diff options
author | Nick Vatamaniuc <vatamane@gmail.com> | 2021-11-20 01:00:08 -0500 |
---|---|---|
committer | Nick Vatamaniuc <nickva@users.noreply.github.com> | 2021-11-22 17:31:31 -0500 |
commit | b78ccf18cb4ed6e183ed0bf4e5cbe40d7a7dc493 (patch) | |
tree | 82158f0b6c7e97e6955bf0c558aac6eb0329b410 | |
parent | 6e87e43fae23647b281ede250ad9f1a68a8f1cde (diff) | |
download | couchdb-b78ccf18cb4ed6e183ed0bf4e5cbe40d7a7dc493.tar.gz |
Apply erlfmt formatting to source tree
These exceptions from main were ported over to 3.x
```
--- a/src/chttpd/src/chttpd.erl
+++ b/src/chttpd/src/chttpd.erl
@@ -491,6 +491,7 @@ extract_cookie(#httpd{mochi_req = MochiReq}) ->
end.
%%% end hack
+%% erlfmt-ignore
set_auth_handlers() ->
AuthenticationDefault = "{chttpd_auth, cookie_authentication_handler},
```
```
--- a/src/couch/src/couch_debug.erl
+++ b/src/couch/src/couch_debug.erl
@@ -49,6 +49,7 @@ help() ->
].
-spec help(Function :: atom()) -> ok.
+%% erlfmt-ignore
help(opened_files) ->
```
553 files changed, 45794 insertions, 38647 deletions
diff --git a/dev/monitor_parent.erl b/dev/monitor_parent.erl index 382f37e9c..0e9e6c5b7 100644 --- a/dev/monitor_parent.erl +++ b/dev/monitor_parent.erl @@ -14,12 +14,10 @@ -export([start/0]). - start() -> {ok, [[PPid]]} = init:get_argument(parent_pid), spawn(fun() -> monitor_parent(PPid) end). - monitor_parent(PPid) -> timer:sleep(1000), case os:type() of diff --git a/rel/plugins/eunit_plugin.erl b/rel/plugins/eunit_plugin.erl index 1de20b394..69003aba6 100644 --- a/rel/plugins/eunit_plugin.erl +++ b/rel/plugins/eunit_plugin.erl @@ -12,21 +12,18 @@ -module(eunit_plugin). - -export([setup_eunit/2]). - setup_eunit(Config, AppFile) -> case is_base_dir(Config) of false -> ok; true -> build_eunit_config(Config, AppFile) end. - %% from https://github.com/ChicagoBoss/ChicagoBoss/blob/master/skel/priv/rebar/boss_plugin.erl is_base_dir(RebarConf) -> - filename:absname(rebar_utils:get_cwd()) =:= rebar_config:get_xconf(RebarConf, base_dir, undefined). - + filename:absname(rebar_utils:get_cwd()) =:= + rebar_config:get_xconf(RebarConf, base_dir, undefined). build_eunit_config(Config0, AppFile) -> Cwd = filename:absname(rebar_utils:get_cwd()), @@ -40,15 +37,16 @@ build_eunit_config(Config0, AppFile) -> Config = rebar_config:set_global(Config3, view_index_dir, ViewIndexDir), rebar_templater:create(Config, AppFile). - cleanup_dirs(Dirs) -> - lists:foreach(fun(Dir) -> - case filelib:is_dir(Dir) of - true -> del_dir(Dir); - false -> ok - end - end, Dirs). - + lists:foreach( + fun(Dir) -> + case filelib:is_dir(Dir) of + true -> del_dir(Dir); + false -> ok + end + end, + Dirs + ). del_dir(Dir) -> All = filelib:wildcard(Dir ++ "/**"), diff --git a/src/chttpd/src/chttpd.erl b/src/chttpd/src/chttpd.erl index aea1ee407..3cfdb5584 100644 --- a/src/chttpd/src/chttpd.erl +++ b/src/chttpd/src/chttpd.erl @@ -17,30 +17,65 @@ -include_lib("couch/include/couch_db.hrl"). -include_lib("chttpd/include/chttpd.hrl"). --export([start_link/0, start_link/1, start_link/2, - stop/0, handle_request/1, handle_request_int/1, - primary_header_value/2, header_value/2, header_value/3, qs_value/2, - qs_value/3, qs/1, qs_json_value/3, path/1, absolute_uri/2, body_length/1, - verify_is_server_admin/1, unquote/1, quote/1, recv/2, recv_chunked/4, - error_info/1, parse_form/1, json_body/1, json_body_obj/1, body/1, - doc_etag/1, make_etag/1, etag_respond/3, etag_match/2, - partition/1, serve_file/3, serve_file/4, - server_header/0, start_chunked_response/3,send_chunk/2, - start_response_length/4, send/2, start_json_response/2, - start_json_response/3, end_json_response/1, send_response/4, +-export([ + start_link/0, start_link/1, start_link/2, + stop/0, + handle_request/1, + handle_request_int/1, + primary_header_value/2, + header_value/2, header_value/3, + qs_value/2, + qs_value/3, + qs/1, + qs_json_value/3, + path/1, + absolute_uri/2, + body_length/1, + verify_is_server_admin/1, + unquote/1, + quote/1, + recv/2, + recv_chunked/4, + error_info/1, + parse_form/1, + json_body/1, + json_body_obj/1, + body/1, + doc_etag/1, + make_etag/1, + etag_respond/3, + etag_match/2, + partition/1, + serve_file/3, serve_file/4, + server_header/0, + start_chunked_response/3, + send_chunk/2, + start_response_length/4, + send/2, + start_json_response/2, + start_json_response/3, + end_json_response/1, + send_response/4, send_response_no_cors/4, - send_method_not_allowed/2, send_error/2, send_error/4, send_redirect/2, - send_chunked_error/2, send_json/2,send_json/3,send_json/4, - validate_ctype/2]). + send_method_not_allowed/2, + send_error/2, send_error/4, + send_redirect/2, + send_chunked_error/2, + send_json/2, send_json/3, send_json/4, + validate_ctype/2 +]). -export([authenticate_request/3]). --export([start_delayed_json_response/2, start_delayed_json_response/3, - start_delayed_json_response/4, +-export([ + start_delayed_json_response/2, start_delayed_json_response/3, start_delayed_json_response/4, start_delayed_chunked_response/3, start_delayed_chunked_response/4, - send_delayed_chunk/2, send_delayed_last_chunk/1, - send_delayed_error/2, end_delayed_json_response/1, - get_delayed_req/1]). + send_delayed_chunk/2, + send_delayed_last_chunk/1, + send_delayed_error/2, + end_delayed_json_response/1, + get_delayed_req/1 +]). -export([ chunked_response_buffer_size/0, @@ -53,8 +88,8 @@ code, headers, chunks, - resp=nil, - buffer_response=false + resp = nil, + buffer_response = false }). -define(DEFAULT_SERVER_OPTIONS, "[{recbuf, undefined}]"). @@ -65,23 +100,28 @@ start_link() -> start_link(http) -> Port = config:get("chttpd", "port", "5984"), start_link(?MODULE, [{port, Port}]); - start_link(https) -> Port = config:get("ssl", "port", "6984"), {ok, Ciphers} = couch_util:parse_term(config:get("ssl", "ciphers", "undefined")), {ok, Versions} = couch_util:parse_term(config:get("ssl", "tls_versions", "undefined")), - {ok, SecureRenegotiate} = couch_util:parse_term(config:get("ssl", "secure_renegotiate", "undefined")), + {ok, SecureRenegotiate} = couch_util:parse_term( + config:get("ssl", "secure_renegotiate", "undefined") + ), ServerOpts0 = - [{cacertfile, config:get("ssl", "cacert_file", undefined)}, - {keyfile, config:get("ssl", "key_file", undefined)}, - {certfile, config:get("ssl", "cert_file", undefined)}, - {password, config:get("ssl", "password", undefined)}, - {secure_renegotiate, SecureRenegotiate}, - {versions, Versions}, - {ciphers, Ciphers}], - - case (couch_util:get_value(keyfile, ServerOpts0) == undefined orelse - couch_util:get_value(certfile, ServerOpts0) == undefined) of + [ + {cacertfile, config:get("ssl", "cacert_file", undefined)}, + {keyfile, config:get("ssl", "key_file", undefined)}, + {certfile, config:get("ssl", "cert_file", undefined)}, + {password, config:get("ssl", "password", undefined)}, + {secure_renegotiate, SecureRenegotiate}, + {versions, Versions}, + {ciphers, Ciphers} + ], + + case + (couch_util:get_value(keyfile, ServerOpts0) == undefined orelse + couch_util:get_value(certfile, ServerOpts0) == undefined) + of true -> io:format("SSL enabled but PEM certificates are missing.", []), throw({error, missing_certs}); @@ -89,66 +129,85 @@ start_link(https) -> ok end, - ServerOpts = [Opt || {_, V}=Opt <- ServerOpts0, V /= undefined], - - ClientOpts = case config:get("ssl", "verify_ssl_certificates", "false") of - "false" -> - []; - "true" -> - FailIfNoPeerCert = case config:get("ssl", "fail_if_no_peer_cert", "false") of - "false" -> false; - "true" -> true - end, - [{depth, list_to_integer(config:get("ssl", - "ssl_certificate_max_depth", "1"))}, - {fail_if_no_peer_cert, FailIfNoPeerCert}, - {verify, verify_peer}] ++ - case config:get("ssl", "verify_fun", undefined) of - undefined -> []; - SpecStr -> - [{verify_fun, couch_httpd:make_arity_3_fun(SpecStr)}] - end - end, + ServerOpts = [Opt || {_, V} = Opt <- ServerOpts0, V /= undefined], + + ClientOpts = + case config:get("ssl", "verify_ssl_certificates", "false") of + "false" -> + []; + "true" -> + FailIfNoPeerCert = + case config:get("ssl", "fail_if_no_peer_cert", "false") of + "false" -> false; + "true" -> true + end, + [ + {depth, + list_to_integer( + config:get( + "ssl", + "ssl_certificate_max_depth", + "1" + ) + )}, + {fail_if_no_peer_cert, FailIfNoPeerCert}, + {verify, verify_peer} + ] ++ + case config:get("ssl", "verify_fun", undefined) of + undefined -> []; + SpecStr -> [{verify_fun, couch_httpd:make_arity_3_fun(SpecStr)}] + end + end, SslOpts = ServerOpts ++ ClientOpts, Options0 = - [{port, Port}, - {ssl, true}, - {ssl_opts, SslOpts}], + [ + {port, Port}, + {ssl, true}, + {ssl_opts, SslOpts} + ], CustomServerOpts = get_server_options("httpsd"), Options = merge_server_options(Options0, CustomServerOpts), start_link(https, Options). start_link(Name, Options) -> - IP = case config:get("chttpd", "bind_address", "any") of - "any" -> any; - Else -> Else - end, + IP = + case config:get("chttpd", "bind_address", "any") of + "any" -> any; + Else -> Else + end, ok = couch_httpd:validate_bind_address(IP), set_auth_handlers(), - Options1 = Options ++ [ - {loop, fun ?MODULE:handle_request/1}, - {name, Name}, - {ip, IP} - ], + Options1 = + Options ++ + [ + {loop, fun ?MODULE:handle_request/1}, + {name, Name}, + {ip, IP} + ], ServerOpts = get_server_options("chttpd"), Options2 = merge_server_options(Options1, ServerOpts), case mochiweb_http:start(Options2) of - {ok, Pid} -> - {ok, Pid}; - {error, Reason} -> - io:format("Failure to start Mochiweb: ~s~n", [Reason]), - {error, Reason} + {ok, Pid} -> + {ok, Pid}; + {error, Reason} -> + io:format("Failure to start Mochiweb: ~s~n", [Reason]), + {error, Reason} end. get_server_options(Module) -> ServerOptsCfg = case Module of - "chttpd" -> config:get(Module, - "server_options", ?DEFAULT_SERVER_OPTIONS); - _ -> config:get(Module, "server_options", "[]") + "chttpd" -> + config:get( + Module, + "server_options", + ?DEFAULT_SERVER_OPTIONS + ); + _ -> + config:get(Module, "server_options", "[]") end, {ok, ServerOpts} = couch_util:parse_term(ServerOptsCfg), ServerOpts. @@ -168,7 +227,8 @@ handle_request(MochiReq0) -> handle_request_int(MochiReq) -> Begin = os:timestamp(), SocketOptsCfg = config:get( - "chttpd", "socket_options", ?DEFAULT_SOCKET_OPTIONS), + "chttpd", "socket_options", ?DEFAULT_SOCKET_OPTIONS + ), {ok, SocketOpts} = couch_util:parse_term(SocketOptsCfg), ok = mochiweb_socket:setopts(MochiReq:get(socket), SocketOpts), @@ -178,49 +238,60 @@ handle_request_int(MochiReq) -> {"/" ++ Path, _, _} = mochiweb_util:urlsplit_path(RawUri), % get requested path - RequestedPath = case MochiReq:get_header_value("x-couchdb-vhost-path") of - undefined -> - case MochiReq:get_header_value("x-couchdb-requested-path") of - undefined -> RawUri; - R -> R - end; - P -> P - end, + RequestedPath = + case MochiReq:get_header_value("x-couchdb-vhost-path") of + undefined -> + case MochiReq:get_header_value("x-couchdb-requested-path") of + undefined -> RawUri; + R -> R + end; + P -> + P + end, Peer = MochiReq:get(peer), Method1 = - case MochiReq:get(method) of - % already an atom - Meth when is_atom(Meth) -> Meth; - - % Non standard HTTP verbs aren't atoms (COPY, MOVE etc) so convert when - % possible (if any module references the atom, then it's existing). - Meth -> couch_util:to_existing_atom(Meth) - end, + case MochiReq:get(method) of + % already an atom + Meth when is_atom(Meth) -> Meth; + % Non standard HTTP verbs aren't atoms (COPY, MOVE etc) so convert when + % possible (if any module references the atom, then it's existing). + Meth -> couch_util:to_existing_atom(Meth) + end, increment_method_stats(Method1), % allow broken HTTP clients to fake a full method vocabulary with an X-HTTP-METHOD-OVERRIDE header MethodOverride = MochiReq:get_primary_header_value("X-HTTP-Method-Override"), - Method2 = case lists:member(MethodOverride, ["GET", "HEAD", "POST", "PUT", "DELETE", "TRACE", "CONNECT", "COPY"]) of - true -> - couch_log:notice("MethodOverride: ~s (real method was ~s)", [MethodOverride, Method1]), - case Method1 of - 'POST' -> couch_util:to_existing_atom(MethodOverride); - _ -> - % Ignore X-HTTP-Method-Override when the original verb isn't POST. - % I'd like to send a 406 error to the client, but that'd require a nasty refactor. - % throw({not_acceptable, <<"X-HTTP-Method-Override may only be used with POST requests.">>}) - Method1 - end; - _ -> Method1 - end, + Method2 = + case + lists:member(MethodOverride, [ + "GET", "HEAD", "POST", "PUT", "DELETE", "TRACE", "CONNECT", "COPY" + ]) + of + true -> + couch_log:notice("MethodOverride: ~s (real method was ~s)", [ + MethodOverride, Method1 + ]), + case Method1 of + 'POST' -> + couch_util:to_existing_atom(MethodOverride); + _ -> + % Ignore X-HTTP-Method-Override when the original verb isn't POST. + % I'd like to send a 406 error to the client, but that'd require a nasty refactor. + % throw({not_acceptable, <<"X-HTTP-Method-Override may only be used with POST requests.">>}) + Method1 + end; + _ -> + Method1 + end, % alias HEAD to GET as mochiweb takes care of stripping the body - Method = case Method2 of - 'HEAD' -> 'GET'; - Other -> Other - end, + Method = + case Method2 of + 'HEAD' -> 'GET'; + Other -> Other + end, Nonce = couch_util:to_hex(crypto:strong_rand_bytes(5)), @@ -231,10 +302,14 @@ handle_request_int(MochiReq) -> original_method = Method1, nonce = Nonce, method = Method, - path_parts = [list_to_binary(unquote(Part)) - || Part <- string:tokens(Path, "/")], - requested_path_parts = [?l2b(unquote(Part)) - || Part <- string:tokens(RequestedPath, "/")] + path_parts = [ + list_to_binary(unquote(Part)) + || Part <- string:tokens(Path, "/") + ], + requested_path_parts = [ + ?l2b(unquote(Part)) + || Part <- string:tokens(RequestedPath, "/") + ] }, % put small token on heap to keep requests synced to backend calls @@ -244,12 +319,13 @@ handle_request_int(MochiReq) -> erlang:put(dont_log_request, true), erlang:put(dont_log_response, true), - {HttpReq2, Response} = case before_request(HttpReq0) of - {ok, HttpReq1} -> - process_request(HttpReq1); - {error, Response0} -> - {HttpReq0, Response0} - end, + {HttpReq2, Response} = + case before_request(HttpReq0) of + {ok, HttpReq1} -> + process_request(HttpReq1); + {error, Response0} -> + {HttpReq0, Response0} + end, {Status, Code, Reason, Resp} = split_response(Response), @@ -345,7 +421,8 @@ catch_error(HttpReq, exit, {mochiweb_recv_error, E}, _Stack) -> Peer, Method, MochiReq:get(raw_path), - E]), + E + ]), exit(normal); catch_error(HttpReq, exit, {uri_too_long, _}, _Stack) -> send_error(HttpReq, request_uri_too_long); @@ -359,12 +436,13 @@ catch_error(HttpReq, Tag, Error, Stack) -> % TODO improve logging and metrics collection for client disconnects case {Tag, Error, Stack} of {exit, normal, [{mochiweb_request, send, _, _} | _]} -> - exit(normal); % Client disconnect (R15+) + % Client disconnect (R15+) + exit(normal); _Else -> send_error(HttpReq, {Error, nil, Stack}) end. -split_response({ok, #delayed_resp{resp=Resp}}) -> +split_response({ok, #delayed_resp{resp = Resp}}) -> {ok, Resp:get(code), undefined, Resp}; split_response({ok, Resp}) -> {ok, Resp:get(code), undefined, Resp}; @@ -401,28 +479,37 @@ maybe_log(#httpd{} = HttpReq, #httpd_resp{should_log = true} = HttpResp) -> Host = MochiReq:get_header_value("Host"), RawUri = MochiReq:get(raw_path), RequestTime = timer:now_diff(EndTime, BeginTime) / 1000, - couch_log:notice("~s ~s ~s ~s ~s ~B ~p ~B", [Host, Peer, User, - Method, RawUri, Code, Status, round(RequestTime)]); + couch_log:notice("~s ~s ~s ~s ~s ~B ~p ~B", [ + Host, + Peer, + User, + Method, + RawUri, + Code, + Status, + round(RequestTime) + ]); maybe_log(_HttpReq, #httpd_resp{should_log = false}) -> ok. - %% HACK: replication currently handles two forms of input, #db{} style %% and #http_db style. We need a third that makes use of fabric. #db{} %% works fine for replicating the dbs and nodes database because they %% aren't sharded. So for now when a local db is specified as the source or %% the target, it's hacked to make it a full url and treated as a remote. -possibly_hack(#httpd{path_parts=[<<"_replicate">>]}=Req) -> +possibly_hack(#httpd{path_parts = [<<"_replicate">>]} = Req) -> {Props0} = chttpd:json_body_obj(Req), Props1 = fix_uri(Req, Props0, <<"source">>), Props2 = fix_uri(Req, Props1, <<"target">>), - Req#httpd{req_body={Props2}}; + Req#httpd{req_body = {Props2}}; possibly_hack(Req) -> Req. check_request_uri_length(Uri) -> - check_request_uri_length(Uri, - chttpd_util:get_chttpd_config("max_uri_length")). + check_request_uri_length( + Uri, + chttpd_util:get_chttpd_config("max_uri_length") + ). check_request_uri_length(_Uri, undefined) -> ok; @@ -445,24 +532,24 @@ check_url_encoding([_ | Rest]) -> fix_uri(Req, Props, Type) -> case replication_uri(Type, Props) of - undefined -> - Props; - Uri0 -> - case is_http(Uri0) of - true -> + undefined -> Props; - false -> - Uri = make_uri(Req, quote(Uri0)), - [{Type,Uri}|proplists:delete(Type,Props)] - end + Uri0 -> + case is_http(Uri0) of + true -> + Props; + false -> + Uri = make_uri(Req, quote(Uri0)), + [{Type, Uri} | proplists:delete(Type, Props)] + end end. replication_uri(Type, PostProps) -> case couch_util:get_value(Type, PostProps) of - {Props} -> - couch_util:get_value(<<"url">>, Props); - Else -> - Else + {Props} -> + couch_util:get_value(<<"url">>, Props); + Else -> + Else end. is_http(<<"http://", _/binary>>) -> @@ -474,13 +561,19 @@ is_http(_) -> make_uri(Req, Raw) -> Port = integer_to_list(mochiweb_socket_server:get(chttpd, port)), - Url = list_to_binary(["http://", config:get("httpd", "bind_address"), - ":", Port, "/", Raw]), + Url = list_to_binary([ + "http://", + config:get("httpd", "bind_address"), + ":", + Port, + "/", + Raw + ]), Headers = [ - {<<"authorization">>, ?l2b(header_value(Req,"authorization",""))}, + {<<"authorization">>, ?l2b(header_value(Req, "authorization", ""))}, {<<"cookie">>, ?l2b(extract_cookie(Req))} ], - {[{<<"url">>,Url}, {<<"headers">>,{Headers}}]}. + {[{<<"url">>, Url}, {<<"headers">>, {Headers}}]}. extract_cookie(#httpd{mochi_req = MochiReq}) -> case MochiReq:get_cookie_value("AuthSession") of @@ -491,6 +584,7 @@ extract_cookie(#httpd{mochi_req = MochiReq}) -> end. %%% end hack +%% erlfmt-ignore set_auth_handlers() -> AuthenticationDefault = "{chttpd_auth, cookie_authentication_handler}, {chttpd_auth, default_authentication_handler}", @@ -525,20 +619,21 @@ authenticate_request(Req) -> authenticate_request(#httpd{} = Req0, AuthModule, AuthFuns) -> Req = Req0#httpd{ auth_module = AuthModule, - authentication_handlers = AuthFuns}, + authentication_handlers = AuthFuns + }, authenticate_request(Req, AuthFuns). % Try authentication handlers in order until one returns a result -authenticate_request(#httpd{user_ctx=#user_ctx{}} = Req, _AuthFuns) -> +authenticate_request(#httpd{user_ctx = #user_ctx{}} = Req, _AuthFuns) -> Req; -authenticate_request(#httpd{} = Req, [{Name, AuthFun}|Rest]) -> +authenticate_request(#httpd{} = Req, [{Name, AuthFun} | Rest]) -> authenticate_request(maybe_set_handler(AuthFun(Req), Name), Rest); -authenticate_request(#httpd{} = Req, [AuthFun|Rest]) -> +authenticate_request(#httpd{} = Req, [AuthFun | Rest]) -> authenticate_request(AuthFun(Req), Rest); authenticate_request(Response, _AuthFuns) -> Response. -maybe_set_handler(#httpd{user_ctx=#user_ctx{} = UserCtx} = Req, Name) -> +maybe_set_handler(#httpd{user_ctx = #user_ctx{} = UserCtx} = Req, Name) -> Req#httpd{user_ctx = UserCtx#user_ctx{handler = Name}}; maybe_set_handler(Else, _) -> Else. @@ -551,16 +646,16 @@ increment_method_stats(Method) -> partition(Path) -> mochiweb_util:partition(Path, "/"). -header_value(#httpd{mochi_req=MochiReq}, Key) -> +header_value(#httpd{mochi_req = MochiReq}, Key) -> MochiReq:get_header_value(Key). -header_value(#httpd{mochi_req=MochiReq}, Key, Default) -> +header_value(#httpd{mochi_req = MochiReq}, Key, Default) -> case MochiReq:get_header_value(Key) of - undefined -> Default; - Value -> Value + undefined -> Default; + Value -> Value end. -primary_header_value(#httpd{mochi_req=MochiReq}, Key) -> +primary_header_value(#httpd{mochi_req = MochiReq}, Key) -> MochiReq:get_primary_header_value(Key). serve_file(Req, RelativePath, DocumentRoot) -> @@ -588,44 +683,52 @@ qs(#httpd{mochi_req = MochiReq, qs = undefined}) -> qs(#httpd{qs = QS}) -> QS. -path(#httpd{mochi_req=MochiReq}) -> +path(#httpd{mochi_req = MochiReq}) -> MochiReq:get(path). -absolute_uri(#httpd{mochi_req=MochiReq, absolute_uri = undefined}, Path) -> +absolute_uri(#httpd{mochi_req = MochiReq, absolute_uri = undefined}, Path) -> XHost = chttpd_util:get_chttpd_config( - "x_forwarded_host", "X-Forwarded-Host"), - Host = case MochiReq:get_header_value(XHost) of - undefined -> - case MochiReq:get_header_value("Host") of - undefined -> - {ok, {Address, Port}} = case MochiReq:get(socket) of - {ssl, SslSocket} -> ssl:sockname(SslSocket); - Socket -> inet:sockname(Socket) - end, - inet_parse:ntoa(Address) ++ ":" ++ integer_to_list(Port); - Value1 -> - Value1 - end; - Value -> Value - end, + "x_forwarded_host", "X-Forwarded-Host" + ), + Host = + case MochiReq:get_header_value(XHost) of + undefined -> + case MochiReq:get_header_value("Host") of + undefined -> + {ok, {Address, Port}} = + case MochiReq:get(socket) of + {ssl, SslSocket} -> ssl:sockname(SslSocket); + Socket -> inet:sockname(Socket) + end, + inet_parse:ntoa(Address) ++ ":" ++ integer_to_list(Port); + Value1 -> + Value1 + end; + Value -> + Value + end, XSsl = chttpd_util:get_chttpd_config("x_forwarded_ssl", "X-Forwarded-Ssl"), - Scheme = case MochiReq:get_header_value(XSsl) of - "on" -> "https"; - _ -> - XProto = chttpd_util:get_chttpd_config( - "x_forwarded_proto", "X-Forwarded-Proto"), - case MochiReq:get_header_value(XProto) of - % Restrict to "https" and "http" schemes only - "https" -> "https"; - _ -> - case MochiReq:get(scheme) of - https -> - "https"; - http -> - "http" - end - end - end, + Scheme = + case MochiReq:get_header_value(XSsl) of + "on" -> + "https"; + _ -> + XProto = chttpd_util:get_chttpd_config( + "x_forwarded_proto", "X-Forwarded-Proto" + ), + case MochiReq:get_header_value(XProto) of + % Restrict to "https" and "http" schemes only + "https" -> + "https"; + _ -> + case MochiReq:get(scheme) of + https -> + "https"; + http -> + "http" + end + end + end, Scheme ++ "://" ++ Host ++ Path; absolute_uri(#httpd{absolute_uri = URI}, Path) -> URI ++ Path. @@ -639,27 +742,28 @@ unquote(UrlEncodedString) -> quote(UrlDecodedString) -> mochiweb_util:quote_plus(UrlDecodedString). -parse_form(#httpd{mochi_req=MochiReq}) -> +parse_form(#httpd{mochi_req = MochiReq}) -> mochiweb_multipart:parse_form(MochiReq). -recv(#httpd{mochi_req=MochiReq}, Len) -> +recv(#httpd{mochi_req = MochiReq}, Len) -> MochiReq:recv(Len). -recv_chunked(#httpd{mochi_req=MochiReq}, MaxChunkSize, ChunkFun, InitState) -> +recv_chunked(#httpd{mochi_req = MochiReq}, MaxChunkSize, ChunkFun, InitState) -> % Fun is called once with each chunk % Fun({Length, Binary}, State) % called with Length == 0 on the last time. MochiReq:stream_body(MaxChunkSize, ChunkFun, InitState). -body_length(#httpd{mochi_req=MochiReq}) -> +body_length(#httpd{mochi_req = MochiReq}) -> MochiReq:get(body_length). -body(#httpd{mochi_req=MochiReq, req_body=ReqBody}) -> +body(#httpd{mochi_req = MochiReq, req_body = ReqBody}) -> case ReqBody of undefined -> % Maximum size of document PUT request body (4GB) MaxSize = chttpd_util:get_chttpd_config_integer( - "max_http_request_size", 4294967296), + "max_http_request_size", 4294967296 + ), Begin = os:timestamp(), try MochiReq:recv_body(MaxSize) @@ -674,38 +778,35 @@ body(#httpd{mochi_req=MochiReq, req_body=ReqBody}) -> validate_ctype(Req, Ctype) -> couch_httpd:validate_ctype(Req, Ctype). -json_body(#httpd{req_body=undefined} = Httpd) -> +json_body(#httpd{req_body = undefined} = Httpd) -> case body(Httpd) of undefined -> throw({bad_request, "Missing request body"}); Body -> ?JSON_DECODE(maybe_decompress(Httpd, Body)) end; - -json_body(#httpd{req_body=ReqBody}) -> +json_body(#httpd{req_body = ReqBody}) -> ReqBody. json_body_obj(Httpd) -> case json_body(Httpd) of {Props} -> {Props}; - _Else -> - throw({bad_request, "Request body must be a JSON object"}) + _Else -> throw({bad_request, "Request body must be a JSON object"}) end. - -doc_etag(#doc{id=Id, body=Body, revs={Start, [DiskRev|_]}}) -> +doc_etag(#doc{id = Id, body = Body, revs = {Start, [DiskRev | _]}}) -> couch_httpd:doc_etag(Id, Body, {Start, DiskRev}). make_etag(Term) -> <<SigInt:128/integer>> = couch_hash:md5_hash(term_to_binary(Term)), - list_to_binary(io_lib:format("\"~.36B\"",[SigInt])). + list_to_binary(io_lib:format("\"~.36B\"", [SigInt])). etag_match(Req, CurrentEtag) when is_binary(CurrentEtag) -> etag_match(Req, binary_to_list(CurrentEtag)); - etag_match(Req, CurrentEtag) -> EtagsToMatch0 = string:tokens( - chttpd:header_value(Req, "If-None-Match", ""), ", "), + chttpd:header_value(Req, "If-None-Match", ""), ", " + ), EtagsToMatch = lists:map(fun strip_weak_prefix/1, EtagsToMatch0), lists:member(CurrentEtag, EtagsToMatch). @@ -716,27 +817,27 @@ strip_weak_prefix(Etag) -> etag_respond(Req, CurrentEtag, RespFun) -> case etag_match(Req, CurrentEtag) of - true -> - % the client has this in their cache. - Headers = [{"ETag", CurrentEtag}], - chttpd:send_response(Req, 304, Headers, <<>>); - false -> - % Run the function. - RespFun() + true -> + % the client has this in their cache. + Headers = [{"ETag", CurrentEtag}], + chttpd:send_response(Req, 304, Headers, <<>>); + false -> + % Run the function. + RespFun() end. -verify_is_server_admin(#httpd{user_ctx=#user_ctx{roles=Roles}}) -> +verify_is_server_admin(#httpd{user_ctx = #user_ctx{roles = Roles}}) -> case lists:member(<<"_admin">>, Roles) of - true -> ok; - false -> throw({unauthorized, <<"You are not a server admin.">>}) + true -> ok; + false -> throw({unauthorized, <<"You are not a server admin.">>}) end. -start_response_length(#httpd{mochi_req=MochiReq}=Req, Code, Headers0, Length) -> +start_response_length(#httpd{mochi_req = MochiReq} = Req, Code, Headers0, Length) -> Headers1 = basic_headers(Req, Headers0), Resp = handle_response(Req, Code, Headers1, Length, start_response_length), case MochiReq:get(method) of - 'HEAD' -> throw({http_head_abort, Resp}); - _ -> ok + 'HEAD' -> throw({http_head_abort, Resp}); + _ -> ok end, {ok, Resp}. @@ -744,12 +845,12 @@ send(Resp, Data) -> Resp:send(Data), {ok, Resp}. -start_chunked_response(#httpd{mochi_req=MochiReq}=Req, Code, Headers0) -> +start_chunked_response(#httpd{mochi_req = MochiReq} = Req, Code, Headers0) -> Headers1 = basic_headers(Req, Headers0), Resp = handle_response(Req, Code, Headers1, chunked, respond), case MochiReq:get(method) of - 'HEAD' -> throw({http_head_abort, Resp}); - _ -> ok + 'HEAD' -> throw({http_head_abort, Resp}); + _ -> ok end, {ok, Resp}. @@ -768,8 +869,14 @@ send_response_no_cors(Req, Code, Headers0, Body) -> couch_httpd:send_response_no_cors(Req, Code, Headers1, Body). send_method_not_allowed(Req, Methods) -> - send_error(Req, 405, [{"Allow", Methods}], <<"method_not_allowed">>, - ?l2b("Only " ++ Methods ++ " allowed"), []). + send_error( + Req, + 405, + [{"Allow", Methods}], + <<"method_not_allowed">>, + ?l2b("Only " ++ Methods ++ " allowed"), + [] + ). send_json(Req, Value) -> send_json(Req, 200, Value). @@ -791,15 +898,12 @@ start_json_response(Req, Code, Headers0) -> end_json_response(Resp) -> couch_httpd:end_json_response(Resp). - start_delayed_json_response(Req, Code) -> start_delayed_json_response(Req, Code, []). - start_delayed_json_response(Req, Code, Headers) -> start_delayed_json_response(Req, Code, Headers, ""). - start_delayed_json_response(Req, Code, Headers, FirstChunk) -> {ok, #delayed_resp{ start_fun = fun start_json_response/3, @@ -807,13 +911,12 @@ start_delayed_json_response(Req, Code, Headers, FirstChunk) -> code = Code, headers = Headers, chunks = [FirstChunk], - buffer_response = buffer_response(Req)}}. - + buffer_response = buffer_response(Req) + }}. start_delayed_chunked_response(Req, Code, Headers) -> start_delayed_chunked_response(Req, Code, Headers, ""). - start_delayed_chunked_response(Req, Code, Headers, FirstChunk) -> {ok, #delayed_resp{ start_fun = fun start_chunked_response/3, @@ -821,34 +924,30 @@ start_delayed_chunked_response(Req, Code, Headers, FirstChunk) -> code = Code, headers = Headers, chunks = [FirstChunk], - buffer_response = buffer_response(Req)}}. - + buffer_response = buffer_response(Req) + }}. -send_delayed_chunk(#delayed_resp{buffer_response=false}=DelayedResp, Chunk) -> - {ok, #delayed_resp{resp=Resp}=DelayedResp1} = +send_delayed_chunk(#delayed_resp{buffer_response = false} = DelayedResp, Chunk) -> + {ok, #delayed_resp{resp = Resp} = DelayedResp1} = start_delayed_response(DelayedResp), {ok, Resp} = send_chunk(Resp, Chunk), {ok, DelayedResp1}; - -send_delayed_chunk(#delayed_resp{buffer_response=true}=DelayedResp, Chunk) -> +send_delayed_chunk(#delayed_resp{buffer_response = true} = DelayedResp, Chunk) -> #delayed_resp{chunks = Chunks} = DelayedResp, {ok, DelayedResp#delayed_resp{chunks = [Chunk | Chunks]}}. - send_delayed_last_chunk(Req) -> send_delayed_chunk(Req, []). - -send_delayed_error(#delayed_resp{req=Req,resp=nil}=DelayedResp, Reason) -> +send_delayed_error(#delayed_resp{req = Req, resp = nil} = DelayedResp, Reason) -> {Code, ErrorStr, ReasonStr} = error_info(Reason), {ok, Resp} = send_error(Req, Code, ErrorStr, ReasonStr), - {ok, DelayedResp#delayed_resp{resp=Resp}}; -send_delayed_error(#delayed_resp{resp=Resp, req=Req}, Reason) -> + {ok, DelayedResp#delayed_resp{resp = Resp}}; +send_delayed_error(#delayed_resp{resp = Resp, req = Req}, Reason) -> update_timeout_stats(Reason, Req), log_error_with_stack_trace(Reason), throw({http_abort, Resp, Reason}). - close_delayed_json_object(Resp, Buffer, Terminator, 0) -> % Use a separate chunk to close the streamed array to maintain strict % compatibility with earlier versions. See COUCHDB-2724 @@ -857,13 +956,11 @@ close_delayed_json_object(Resp, Buffer, Terminator, 0) -> close_delayed_json_object(Resp, Buffer, Terminator, _Threshold) -> send_delayed_chunk(Resp, [Buffer | Terminator]). - -end_delayed_json_response(#delayed_resp{buffer_response=false}=DelayedResp) -> - {ok, #delayed_resp{resp=Resp}} = +end_delayed_json_response(#delayed_resp{buffer_response = false} = DelayedResp) -> + {ok, #delayed_resp{resp = Resp}} = start_delayed_response(DelayedResp), end_json_response(Resp); - -end_delayed_json_response(#delayed_resp{buffer_response=true}=DelayedResp) -> +end_delayed_json_response(#delayed_resp{buffer_response = true} = DelayedResp) -> #delayed_resp{ start_fun = StartFun, req = Req, @@ -872,36 +969,37 @@ end_delayed_json_response(#delayed_resp{buffer_response=true}=DelayedResp) -> chunks = Chunks } = DelayedResp, {ok, Resp} = StartFun(Req, Code, Headers), - lists:foreach(fun - ([]) -> ok; - (Chunk) -> send_chunk(Resp, Chunk) - end, lists:reverse(Chunks)), + lists:foreach( + fun + ([]) -> ok; + (Chunk) -> send_chunk(Resp, Chunk) + end, + lists:reverse(Chunks) + ), end_json_response(Resp). - -get_delayed_req(#delayed_resp{req=#httpd{mochi_req=MochiReq}}) -> +get_delayed_req(#delayed_resp{req = #httpd{mochi_req = MochiReq}}) -> MochiReq; get_delayed_req(Resp) -> Resp:get(request). -start_delayed_response(#delayed_resp{resp=nil}=DelayedResp) -> +start_delayed_response(#delayed_resp{resp = nil} = DelayedResp) -> #delayed_resp{ - start_fun=StartFun, - req=Req, - code=Code, - headers=Headers, - chunks=[FirstChunk] - }=DelayedResp, + start_fun = StartFun, + req = Req, + code = Code, + headers = Headers, + chunks = [FirstChunk] + } = DelayedResp, {ok, Resp} = StartFun(Req, Code, Headers), case FirstChunk of "" -> ok; _ -> {ok, Resp} = send_chunk(Resp, FirstChunk) end, - {ok, DelayedResp#delayed_resp{resp=Resp}}; -start_delayed_response(#delayed_resp{}=DelayedResp) -> + {ok, DelayedResp#delayed_resp{resp = Resp}}; +start_delayed_response(#delayed_resp{} = DelayedResp) -> {ok, DelayedResp}. - buffer_response(Req) -> case chttpd:qs_value(Req, "buffer_response") of "false" -> @@ -912,7 +1010,6 @@ buffer_response(Req) -> config:get_boolean("chttpd", "buffer_response", false) end. - error_info({Error, Reason}) when is_list(Reason) -> error_info({Error, couch_util:to_binary(Reason)}); error_info(bad_request) -> @@ -941,8 +1038,8 @@ error_info({conflict, _}) -> {409, <<"conflict">>, <<"Document update conflict.">>}; error_info({partition_overflow, DocId}) -> Descr = << - "Partition limit exceeded due to update on '", DocId/binary, "'" - >>, + "Partition limit exceeded due to update on '", DocId/binary, "'" + >>, {403, <<"partition_overflow">>, Descr}; error_info({{not_found, missing}, {_, _}}) -> {409, <<"not_found">>, <<"missing_rev">>}; @@ -953,8 +1050,10 @@ error_info({forbidden, Msg}) -> error_info({unauthorized, Msg}) -> {401, <<"unauthorized">>, Msg}; error_info(file_exists) -> - {412, <<"file_exists">>, <<"The database could not be " - "created, the file already exists.">>}; + {412, <<"file_exists">>, << + "The database could not be " + "created, the file already exists." + >>}; error_info({error, {nodedown, Reason}}) -> {412, <<"nodedown">>, Reason}; error_info({maintenance_mode, Node}) -> @@ -970,16 +1069,17 @@ error_info({bad_ctype, Reason}) -> error_info(requested_range_not_satisfiable) -> {416, <<"requested_range_not_satisfiable">>, <<"Requested range not satisfiable">>}; error_info({error, {illegal_database_name, Name}}) -> - Message = <<"Name: '", Name/binary, "'. Only lowercase characters (a-z), ", - "digits (0-9), and any of the characters _, $, (, ), +, -, and / ", - "are allowed. Must begin with a letter.">>, + Message = + <<"Name: '", Name/binary, "'. Only lowercase characters (a-z), ", + "digits (0-9), and any of the characters _, $, (, ), +, -, and / ", + "are allowed. Must begin with a letter.">>, {400, <<"illegal_database_name">>, Message}; error_info({illegal_docid, Reason}) -> {400, <<"illegal_docid">>, Reason}; error_info({illegal_partition, Reason}) -> {400, <<"illegal_partition">>, Reason}; -error_info({_DocID,{illegal_docid,DocID}}) -> - {400, <<"illegal_docid">>,DocID}; +error_info({_DocID, {illegal_docid, DocID}}) -> + {400, <<"illegal_docid">>, DocID}; error_info({error, {database_name_too_long, DbName}}) -> {400, <<"database_name_too_long">>, <<"At least one path segment of `", DbName/binary, "` is too long.">>}; @@ -994,16 +1094,22 @@ error_info({request_entity_too_large, {attachment, AttName}}) -> error_info({request_entity_too_large, DocID}) -> {413, <<"document_too_large">>, DocID}; error_info({error, security_migration_updates_disabled}) -> - {503, <<"security_migration">>, <<"Updates to security docs are disabled during " - "security migration.">>}; + {503, <<"security_migration">>, << + "Updates to security docs are disabled during " + "security migration." + >>}; error_info(all_workers_died) -> - {503, <<"service unvailable">>, <<"Nodes are unable to service this " - "request due to overloading or maintenance mode.">>}; + {503, <<"service unvailable">>, << + "Nodes are unable to service this " + "request due to overloading or maintenance mode." + >>}; error_info(not_implemented) -> {501, <<"not_implemented">>, <<"this feature is not yet implemented">>}; error_info(timeout) -> - {500, <<"timeout">>, <<"The request could not be processed in a reasonable" - " amount of time.">>}; + {500, <<"timeout">>, << + "The request could not be processed in a reasonable" + " amount of time." + >>}; error_info({service_unavailable, Reason}) -> {503, <<"service unavailable">>, Reason}; error_info({timeout, _Reason}) -> @@ -1033,61 +1139,83 @@ maybe_handle_error(Error) -> {500, <<"unknown_error">>, couch_util:to_binary(Error)} end. - -error_headers(#httpd{mochi_req=MochiReq}=Req, 401=Code, ErrorStr, ReasonStr) -> +error_headers(#httpd{mochi_req = MochiReq} = Req, 401 = Code, ErrorStr, ReasonStr) -> % this is where the basic auth popup is triggered case MochiReq:get_header_value("X-CouchDB-WWW-Authenticate") of - undefined -> - case chttpd_util:get_chttpd_config("WWW-Authenticate") of undefined -> - % If the client is a browser and the basic auth popup isn't turned on - % redirect to the session page. - case ErrorStr of - <<"unauthorized">> -> - case chttpd_util:get_chttpd_auth_config( - "authentication_redirect", "/_utils/session.html") of - undefined -> {Code, []}; - AuthRedirect -> - case chttpd_util:get_chttpd_auth_config_boolean( - "require_valid_user", false) of - true -> - % send the browser popup header no matter what if we are require_valid_user - {Code, [{"WWW-Authenticate", "Basic realm=\"server\""}]}; - false -> - case MochiReq:accepts_content_type("application/json") of - true -> - {Code, []}; - false -> - case MochiReq:accepts_content_type("text/html") of - true -> - % Redirect to the path the user requested, not - % the one that is used internally. - UrlReturnRaw = case MochiReq:get_header_value("x-couchdb-vhost-path") of + case chttpd_util:get_chttpd_config("WWW-Authenticate") of + undefined -> + % If the client is a browser and the basic auth popup isn't turned on + % redirect to the session page. + case ErrorStr of + <<"unauthorized">> -> + case + chttpd_util:get_chttpd_auth_config( + "authentication_redirect", "/_utils/session.html" + ) + of undefined -> - MochiReq:get(path); - VHostPath -> - VHostPath - end, - RedirectLocation = lists:flatten([ - AuthRedirect, - "?return=", couch_util:url_encode(UrlReturnRaw), - "&reason=", couch_util:url_encode(ReasonStr) - ]), - {302, [{"Location", absolute_uri(Req, RedirectLocation)}]}; - false -> - {Code, []} - end - end - end - end; - _Else -> - {Code, []} + {Code, []}; + AuthRedirect -> + case + chttpd_util:get_chttpd_auth_config_boolean( + "require_valid_user", false + ) + of + true -> + % send the browser popup header no matter what if we are require_valid_user + {Code, [{"WWW-Authenticate", "Basic realm=\"server\""}]}; + false -> + case + MochiReq:accepts_content_type("application/json") + of + true -> + {Code, []}; + false -> + case + MochiReq:accepts_content_type("text/html") + of + true -> + % Redirect to the path the user requested, not + % the one that is used internally. + UrlReturnRaw = + case + MochiReq:get_header_value( + "x-couchdb-vhost-path" + ) + of + undefined -> + MochiReq:get(path); + VHostPath -> + VHostPath + end, + RedirectLocation = lists:flatten([ + AuthRedirect, + "?return=", + couch_util:url_encode(UrlReturnRaw), + "&reason=", + couch_util:url_encode(ReasonStr) + ]), + {302, [ + {"Location", + absolute_uri( + Req, RedirectLocation + )} + ]}; + false -> + {Code, []} + end + end + end + end; + _Else -> + {Code, []} + end; + Type -> + {Code, [{"WWW-Authenticate", Type}]} end; Type -> {Code, [{"WWW-Authenticate", Type}]} - end; - Type -> - {Code, [{"WWW-Authenticate", Type}]} end; error_headers(_, Code, _, _) -> {Code, []}. @@ -1104,16 +1232,30 @@ send_error(#httpd{} = Req, Code, ErrorStr, ReasonStr) -> send_error(Req, Code, [], ErrorStr, ReasonStr, []). send_error(Req, Code, Headers, ErrorStr, ReasonStr, []) -> - send_json(Req, Code, Headers, - {[{<<"error">>, ErrorStr}, - {<<"reason">>, ReasonStr}]}); + send_json( + Req, + Code, + Headers, + {[ + {<<"error">>, ErrorStr}, + {<<"reason">>, ReasonStr} + ]} + ); send_error(Req, Code, Headers, ErrorStr, ReasonStr, Stack) -> log_error_with_stack_trace({ErrorStr, ReasonStr, Stack}), - send_json(Req, Code, [stack_trace_id(Stack) | Headers], - {[{<<"error">>, ErrorStr}, - {<<"reason">>, ReasonStr} | - case Stack of [] -> []; _ -> [{<<"ref">>, stack_hash(Stack)}] end - ]}). + send_json( + Req, + Code, + [stack_trace_id(Stack) | Headers], + {[ + {<<"error">>, ErrorStr}, + {<<"reason">>, ReasonStr} + | case Stack of + [] -> []; + _ -> [{<<"ref">>, stack_hash(Stack)}] + end + ]} + ). update_timeout_stats(<<"timeout">>, #httpd{requested_path_parts = PathParts}) -> update_timeout_stats(PathParts); @@ -1122,20 +1264,27 @@ update_timeout_stats(timeout, #httpd{requested_path_parts = PathParts}) -> update_timeout_stats(_, _) -> ok. -update_timeout_stats([_, <<"_partition">>, _, <<"_design">>, _, - <<"_view">> | _]) -> +update_timeout_stats([ + _, + <<"_partition">>, + _, + <<"_design">>, + _, + <<"_view">> + | _ +]) -> couch_stats:increment_counter([couchdb, httpd, partition_view_timeouts]); -update_timeout_stats([_, <<"_partition">>, _, <<"_find">>| _]) -> +update_timeout_stats([_, <<"_partition">>, _, <<"_find">> | _]) -> couch_stats:increment_counter([couchdb, httpd, partition_find_timeouts]); -update_timeout_stats([_, <<"_partition">>, _, <<"_explain">>| _]) -> +update_timeout_stats([_, <<"_partition">>, _, <<"_explain">> | _]) -> couch_stats:increment_counter([couchdb, httpd, partition_explain_timeouts]); update_timeout_stats([_, <<"_partition">>, _, <<"_all_docs">> | _]) -> couch_stats:increment_counter([couchdb, httpd, partition_all_docs_timeouts]); update_timeout_stats([_, <<"_design">>, _, <<"_view">> | _]) -> couch_stats:increment_counter([couchdb, httpd, view_timeouts]); -update_timeout_stats([_, <<"_find">>| _]) -> +update_timeout_stats([_, <<"_find">> | _]) -> couch_stats:increment_counter([couchdb, httpd, find_timeouts]); -update_timeout_stats([_, <<"_explain">>| _]) -> +update_timeout_stats([_, <<"_explain">> | _]) -> couch_stats:increment_counter([couchdb, httpd, explain_timeouts]); update_timeout_stats([_, <<"_all_docs">> | _]) -> couch_stats:increment_counter([couchdb, httpd, all_docs_timeouts]); @@ -1146,17 +1295,21 @@ update_timeout_stats(_) -> send_chunked_error(Resp, {_Error, {[{<<"body">>, Reason}]}}) -> send_chunk(Resp, Reason), send_chunk(Resp, []); - send_chunked_error(Resp, Error) -> Stack = json_stack(Error), log_error_with_stack_trace(Error), {Code, ErrorStr, ReasonStr} = error_info(Error), - JsonError = {[{<<"code">>, Code}, - {<<"error">>, ErrorStr}, - {<<"reason">>, ReasonStr} | - case Stack of [] -> []; _ -> [{<<"ref">>, stack_hash(Stack)}] end - ]}, - send_chunk(Resp, ?l2b([$\n,?JSON_ENCODE(JsonError),$\n])), + JsonError = + {[ + {<<"code">>, Code}, + {<<"error">>, ErrorStr}, + {<<"reason">>, ReasonStr} + | case Stack of + [] -> []; + _ -> [{<<"ref">>, stack_hash(Stack)}] + end + ]}, + send_chunk(Resp, ?l2b([$\n, ?JSON_ENCODE(JsonError), $\n])), send_chunk(Resp, []). send_redirect(Req, Path) -> @@ -1184,39 +1337,58 @@ json_stack({_Error, _Reason, Stack}) when is_list(Stack) -> json_stack(_) -> []. -json_stack_item({M,F,A}) -> +json_stack_item({M, F, A}) -> list_to_binary(io_lib:format("~s:~s/~B", [M, F, json_stack_arity(A)])); -json_stack_item({M,F,A,L}) -> +json_stack_item({M, F, A, L}) -> case proplists:get_value(line, L) of - undefined -> json_stack_item({M,F,A}); - Line -> list_to_binary(io_lib:format("~s:~s/~B L~B", - [M, F, json_stack_arity(A), Line])) + undefined -> + json_stack_item({M, F, A}); + Line -> + list_to_binary( + io_lib:format( + "~s:~s/~B L~B", + [M, F, json_stack_arity(A), Line] + ) + ) end; json_stack_item(_) -> <<"bad entry in stacktrace">>. json_stack_arity(A) -> - if is_integer(A) -> A; is_list(A) -> length(A); true -> 0 end. + if + is_integer(A) -> A; + is_list(A) -> length(A); + true -> 0 + end. maybe_decompress(Httpd, Body) -> case header_value(Httpd, "Content-Encoding", "identity") of - "gzip" -> - try - zlib:gunzip(Body) - catch error:data_error -> - throw({bad_request, "Request body is not properly gzipped."}) - end; - "identity" -> - Body; - Else -> - throw({bad_ctype, [Else, " is not a supported content encoding."]}) + "gzip" -> + try + zlib:gunzip(Body) + catch + error:data_error -> + throw({bad_request, "Request body is not properly gzipped."}) + end; + "identity" -> + Body; + Else -> + throw({bad_ctype, [Else, " is not a supported content encoding."]}) end. log_error_with_stack_trace({bad_request, _, _}) -> ok; log_error_with_stack_trace({Error, Reason, Stack}) -> - EFmt = if is_binary(Error) -> "~s"; true -> "~w" end, - RFmt = if is_binary(Reason) -> "~s"; true -> "~w" end, + EFmt = + if + is_binary(Error) -> "~s"; + true -> "~w" + end, + RFmt = + if + is_binary(Reason) -> "~s"; + true -> "~w" + end, Fmt = "req_err(~w) " ++ EFmt ++ " : " ++ RFmt ++ "~n ~p", couch_log:error(Fmt, [stack_hash(Stack), Error, Reason, Stack]); log_error_with_stack_trace(_) -> @@ -1241,9 +1413,10 @@ chunked_response_buffer_size() -> chttpd_util:get_chttpd_config_integer("chunked_response_buffer", 1490). basic_headers(Req, Headers0) -> - Headers = Headers0 - ++ server_header() - ++ couch_httpd_auth:cookie_auth_header(Req, Headers0), + Headers = + Headers0 ++ + server_header() ++ + couch_httpd_auth:cookie_auth_header(Req, Headers0), Headers1 = chttpd_cors:headers(Req, Headers), Headers2 = chttpd_xframe_options:header(Req, Headers1), Headers3 = [reqid(), timing() | Headers2], @@ -1292,52 +1465,83 @@ check_url_encoding_pass_test_() -> check_url_encoding_fail_test_() -> [ - ?_assertThrow({bad_request, invalid_url_encoding}, - check_url_encoding("/dbname%")), - ?_assertThrow({bad_request, invalid_url_encoding}, - check_url_encoding("/dbname/doc_id%")), - ?_assertThrow({bad_request, invalid_url_encoding}, - check_url_encoding("/dbname/doc_id%?rev=1-abcdefgh")), - ?_assertThrow({bad_request, invalid_url_encoding}, - check_url_encoding("/dbname%2")), - ?_assertThrow({bad_request, invalid_url_encoding}, - check_url_encoding("/dbname/doc_id%2")), - ?_assertThrow({bad_request, invalid_url_encoding}, - check_url_encoding("/user%2Fdbname%")), - ?_assertThrow({bad_request, invalid_url_encoding}, - check_url_encoding("/user%2Fdbname/doc_id%")), - ?_assertThrow({bad_request, invalid_url_encoding}, - check_url_encoding("%")), - ?_assertThrow({bad_request, invalid_url_encoding}, - check_url_encoding("/%")), - ?_assertThrow({bad_request, invalid_url_encoding}, - check_url_encoding("/%2")), - ?_assertThrow({bad_request, invalid_url_encoding}, - check_url_encoding("/dbname%2%3A")), - ?_assertThrow({bad_request, invalid_url_encoding}, - check_url_encoding("/dbname%%3Ae")), - ?_assertThrow({bad_request, invalid_url_encoding}, - check_url_encoding("/dbname%2g")), - ?_assertThrow({bad_request, invalid_url_encoding}, - check_url_encoding("/dbname%g2")) + ?_assertThrow( + {bad_request, invalid_url_encoding}, + check_url_encoding("/dbname%") + ), + ?_assertThrow( + {bad_request, invalid_url_encoding}, + check_url_encoding("/dbname/doc_id%") + ), + ?_assertThrow( + {bad_request, invalid_url_encoding}, + check_url_encoding("/dbname/doc_id%?rev=1-abcdefgh") + ), + ?_assertThrow( + {bad_request, invalid_url_encoding}, + check_url_encoding("/dbname%2") + ), + ?_assertThrow( + {bad_request, invalid_url_encoding}, + check_url_encoding("/dbname/doc_id%2") + ), + ?_assertThrow( + {bad_request, invalid_url_encoding}, + check_url_encoding("/user%2Fdbname%") + ), + ?_assertThrow( + {bad_request, invalid_url_encoding}, + check_url_encoding("/user%2Fdbname/doc_id%") + ), + ?_assertThrow( + {bad_request, invalid_url_encoding}, + check_url_encoding("%") + ), + ?_assertThrow( + {bad_request, invalid_url_encoding}, + check_url_encoding("/%") + ), + ?_assertThrow( + {bad_request, invalid_url_encoding}, + check_url_encoding("/%2") + ), + ?_assertThrow( + {bad_request, invalid_url_encoding}, + check_url_encoding("/dbname%2%3A") + ), + ?_assertThrow( + {bad_request, invalid_url_encoding}, + check_url_encoding("/dbname%%3Ae") + ), + ?_assertThrow( + {bad_request, invalid_url_encoding}, + check_url_encoding("/dbname%2g") + ), + ?_assertThrow( + {bad_request, invalid_url_encoding}, + check_url_encoding("/dbname%g2") + ) ]. log_format_test() -> ?assertEqual( "127.0.0.1:15984 127.0.0.1 undefined " "GET /_cluster_setup 201 ok 10000", - test_log_request("/_cluster_setup", undefined)), + test_log_request("/_cluster_setup", undefined) + ), ?assertEqual( "127.0.0.1:15984 127.0.0.1 user_foo " "GET /_all_dbs 201 ok 10000", - test_log_request("/_all_dbs", #user_ctx{name = <<"user_foo">>})), + test_log_request("/_all_dbs", #user_ctx{name = <<"user_foo">>}) + ), %% Utf8Name = unicode:characters_to_binary(Something), - Utf8User = <<227,130,136,227,129,134,227,129,147,227,129,157>>, + Utf8User = <<227, 130, 136, 227, 129, 134, 227, 129, 147, 227, 129, 157>>, ?assertEqual( "127.0.0.1:15984 127.0.0.1 %E3%82%88%E3%81%86%E3%81%93%E3%81%9D " "GET /_all_dbs 201 ok 10000", - test_log_request("/_all_dbs", #user_ctx{name = Utf8User})), + test_log_request("/_all_dbs", #user_ctx{name = Utf8User}) + ), ok. test_log_request(RawPath, UserCtx) -> @@ -1345,14 +1549,14 @@ test_log_request(RawPath, UserCtx) -> MochiReq = mochiweb_request:new(socket, [], 'POST', RawPath, version, Headers), Req = #httpd{ mochi_req = MochiReq, - begin_ts = {1458,588713,124003}, + begin_ts = {1458, 588713, 124003}, original_method = 'GET', peer = "127.0.0.1", nonce = "nonce", user_ctx = UserCtx }, Resp = #httpd_resp{ - end_ts = {1458,588723,124303}, + end_ts = {1458, 588723, 124303}, code = 201, status = ok }, @@ -1366,14 +1570,20 @@ test_log_request(RawPath, UserCtx) -> handle_req_after_auth_test() -> Headers = mochiweb_headers:make([{"HOST", "127.0.0.1:15984"}]), - MochiReq = mochiweb_request:new(socket, [], 'PUT', "/newdb", version, - Headers), + MochiReq = mochiweb_request:new( + socket, + [], + 'PUT', + "/newdb", + version, + Headers + ), UserCtx = #user_ctx{name = <<"retain_user">>}, Roles = [<<"_reader">>], AuthorizedCtx = #user_ctx{name = <<"retain_user">>, roles = Roles}, Req = #httpd{ mochi_req = MochiReq, - begin_ts = {1458,588713,124003}, + begin_ts = {1458, 588713, 124003}, original_method = 'PUT', peer = "127.0.0.1", nonce = "nonce", @@ -1383,18 +1593,22 @@ handle_req_after_auth_test() -> ok = meck:new(chttpd_handlers, [passthrough]), ok = meck:new(chttpd_auth, [passthrough]), ok = meck:expect(chttpd_handlers, url_handler, fun(_Key, _Fun) -> - fun(_Req) -> handled_authorized_req end + fun(_Req) -> handled_authorized_req end end), ok = meck:expect(chttpd_auth, authorize, fun(_Req, _Fun) -> AuthorizedReq end), - ?assertEqual({AuthorizedReq, handled_authorized_req}, - handle_req_after_auth(foo_key, Req)), + ?assertEqual( + {AuthorizedReq, handled_authorized_req}, + handle_req_after_auth(foo_key, Req) + ), ok = meck:expect(chttpd_auth, authorize, fun(_Req, _Fun) -> meck:exception(throw, {http_abort, resp, some_reason}) end), - ?assertEqual({Req, {aborted, resp, some_reason}}, - handle_req_after_auth(foo_key, Req)), + ?assertEqual( + {Req, {aborted, resp, some_reason}}, + handle_req_after_auth(foo_key, Req) + ), ok = meck:unload(chttpd_handlers), ok = meck:unload(chttpd_auth). diff --git a/src/chttpd/src/chttpd_auth.erl b/src/chttpd/src/chttpd_auth.erl index ffae78171..20b5a05f1 100644 --- a/src/chttpd/src/chttpd_auth.erl +++ b/src/chttpd/src/chttpd_auth.erl @@ -27,7 +27,6 @@ -define(SERVICE_ID, chttpd_auth). - %% ------------------------------------------------------------------ %% API Function Definitions %% ------------------------------------------------------------------ @@ -38,7 +37,6 @@ authenticate(HttpReq, Default) -> authorize(HttpReq, Default) -> maybe_handle(authorize, [HttpReq], Default). - %% ------------------------------------------------------------------ %% Default callbacks %% ------------------------------------------------------------------ @@ -55,17 +53,20 @@ proxy_authentication_handler(Req) -> jwt_authentication_handler(Req) -> couch_httpd_auth:jwt_authentication_handler(Req). -party_mode_handler(#httpd{method='POST', path_parts=[<<"_session">>]} = Req) -> +party_mode_handler(#httpd{method = 'POST', path_parts = [<<"_session">>]} = Req) -> % See #1947 - users should always be able to attempt a login - Req#httpd{user_ctx=#user_ctx{}}; -party_mode_handler(#httpd{path_parts=[<<"_up">>]} = Req) -> + Req#httpd{user_ctx = #user_ctx{}}; +party_mode_handler(#httpd{path_parts = [<<"_up">>]} = Req) -> RequireValidUser = config:get_boolean("chttpd", "require_valid_user", false), - RequireValidUserExceptUp = config:get_boolean("chttpd", "require_valid_user_except_for_up", false), + RequireValidUserExceptUp = config:get_boolean( + "chttpd", "require_valid_user_except_for_up", false + ), require_valid_user(Req, RequireValidUser andalso not RequireValidUserExceptUp); - party_mode_handler(Req) -> RequireValidUser = config:get_boolean("chttpd", "require_valid_user", false), - RequireValidUserExceptUp = config:get_boolean("chttpd", "require_valid_user_except_for_up", false), + RequireValidUserExceptUp = config:get_boolean( + "chttpd", "require_valid_user_except_for_up", false + ), require_valid_user(Req, RequireValidUser orelse RequireValidUserExceptUp). require_valid_user(_Req, true) -> @@ -75,13 +76,12 @@ require_valid_user(Req, false) -> [] -> Req#httpd{user_ctx = ?ADMIN_USER}; _ -> - Req#httpd{user_ctx=#user_ctx{}} + Req#httpd{user_ctx = #user_ctx{}} end. handle_session_req(Req) -> couch_httpd_auth:handle_session_req(Req, chttpd_auth_cache). - %% ------------------------------------------------------------------ %% Internal Function Definitions %% ------------------------------------------------------------------ diff --git a/src/chttpd/src/chttpd_auth_cache.erl b/src/chttpd/src/chttpd_auth_cache.erl index 17a31bf55..2173eca95 100644 --- a/src/chttpd/src/chttpd_auth_cache.erl +++ b/src/chttpd/src/chttpd_auth_cache.erl @@ -14,8 +14,14 @@ -behaviour(gen_server). -export([start_link/0, get_user_creds/2, update_user_creds/3, dbname/0]). --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3]). +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3 +]). -export([listen_for_changes/1, changes_callback/2]). -include_lib("couch/include/couch_db.hrl"). @@ -25,7 +31,7 @@ -record(state, { changes_pid, - last_seq="0" + last_seq = "0" }). %% public functions @@ -36,18 +42,21 @@ start_link() -> get_user_creds(Req, UserName) when is_list(UserName) -> get_user_creds(Req, ?l2b(UserName)); get_user_creds(_Req, UserName) when is_binary(UserName) -> - Resp = case couch_auth_cache:get_admin(UserName) of - nil -> - get_from_cache(UserName); - Props -> - case get_from_cache(UserName) of - nil -> - Props; - UserProps when is_list(UserProps) -> - couch_auth_cache:add_roles(Props, - couch_util:get_value(<<"roles">>, UserProps)) - end - end, + Resp = + case couch_auth_cache:get_admin(UserName) of + nil -> + get_from_cache(UserName); + Props -> + case get_from_cache(UserName) of + nil -> + Props; + UserProps when is_list(UserProps) -> + couch_auth_cache:add_roles( + Props, + couch_util:get_value(<<"roles">>, UserProps) + ) + end + end, maybe_validate_user_creds(Resp). update_user_creds(_Req, UserDoc, _Ctx) -> @@ -109,19 +118,23 @@ handle_call(_Call, _From, State) -> handle_cast(_Msg, State) -> {noreply, State}. -handle_info({'DOWN', _, _, Pid, Reason}, #state{changes_pid=Pid} = State) -> - Seq = case Reason of - {seq, EndSeq} -> - EndSeq; - {database_does_not_exist, _} -> - couch_log:notice("~p changes listener died because the _users database does not exist. Create the database to silence this notice.", [?MODULE]), - 0; - _ -> - couch_log:notice("~p changes listener died ~r", [?MODULE, Reason]), - 0 - end, +handle_info({'DOWN', _, _, Pid, Reason}, #state{changes_pid = Pid} = State) -> + Seq = + case Reason of + {seq, EndSeq} -> + EndSeq; + {database_does_not_exist, _} -> + couch_log:notice( + "~p changes listener died because the _users database does not exist. Create the database to silence this notice.", + [?MODULE] + ), + 0; + _ -> + couch_log:notice("~p changes listener died ~r", [?MODULE, Reason]), + 0 + end, erlang:send_after(5000, self(), {start_listener, Seq}), - {noreply, State#state{last_seq=Seq}}; + {noreply, State#state{last_seq = Seq}}; handle_info({start_listener, Seq}, State) -> {noreply, State#state{changes_pid = spawn_changes(Seq)}}; handle_info(_Msg, State) -> @@ -132,7 +145,7 @@ terminate(_Reason, #state{changes_pid = Pid}) when is_pid(Pid) -> terminate(_Reason, _State) -> ok. -code_change(_OldVsn, #state{}=State, _Extra) -> +code_change(_OldVsn, #state{} = State, _Extra) -> {ok, State}. %% private functions @@ -175,14 +188,15 @@ changes_callback({error, _}, EndSeq) -> load_user_from_db(UserName) -> try fabric:open_doc(dbname(), docid(UserName), [?ADMIN_CTX, ejson_body, conflicts]) of - {ok, Doc} -> - {Props} = couch_doc:to_json_obj(Doc, []), - Props; - _Else -> - couch_log:debug("no record of user ~s", [UserName]), - nil - catch error:database_does_not_exist -> - nil + {ok, Doc} -> + {Props} = couch_doc:to_json_obj(Doc, []), + Props; + _Else -> + couch_log:debug("no record of user ~s", [UserName]), + nil + catch + error:database_does_not_exist -> + nil end. dbname() -> @@ -196,23 +210,30 @@ username(<<"org.couchdb.user:", UserName/binary>>) -> ensure_auth_ddoc_exists(DbName, DDocId) -> case fabric:open_doc(DbName, DDocId, [?ADMIN_CTX, ejson_body]) of - {not_found, _Reason} -> - {ok, AuthDesign} = couch_auth_cache:auth_design_doc(DDocId), - update_doc_ignoring_conflict(DbName, AuthDesign, [?ADMIN_CTX]); - {ok, Doc} -> - {Props} = couch_doc:to_json_obj(Doc, []), - case couch_util:get_value(<<"validate_doc_update">>, Props, []) of - ?AUTH_DB_DOC_VALIDATE_FUNCTION -> - ok; - _ -> - Props1 = lists:keyreplace(<<"validate_doc_update">>, 1, Props, - {<<"validate_doc_update">>, - ?AUTH_DB_DOC_VALIDATE_FUNCTION}), - update_doc_ignoring_conflict(DbName, couch_doc:from_json_obj({Props1}), [?ADMIN_CTX]) - end; - {error, Reason} -> - couch_log:notice("Failed to ensure auth ddoc ~s/~s exists for reason: ~p", [DbName, DDocId, Reason]), - ok + {not_found, _Reason} -> + {ok, AuthDesign} = couch_auth_cache:auth_design_doc(DDocId), + update_doc_ignoring_conflict(DbName, AuthDesign, [?ADMIN_CTX]); + {ok, Doc} -> + {Props} = couch_doc:to_json_obj(Doc, []), + case couch_util:get_value(<<"validate_doc_update">>, Props, []) of + ?AUTH_DB_DOC_VALIDATE_FUNCTION -> + ok; + _ -> + Props1 = lists:keyreplace( + <<"validate_doc_update">>, + 1, + Props, + {<<"validate_doc_update">>, ?AUTH_DB_DOC_VALIDATE_FUNCTION} + ), + update_doc_ignoring_conflict(DbName, couch_doc:from_json_obj({Props1}), [ + ?ADMIN_CTX + ]) + end; + {error, Reason} -> + couch_log:notice("Failed to ensure auth ddoc ~s/~s exists for reason: ~p", [ + DbName, DDocId, Reason + ]), + ok end, ok. @@ -229,15 +250,18 @@ maybe_validate_user_creds(nil) -> % throws if UserCreds includes a _conflicts member % returns UserCreds otherwise maybe_validate_user_creds(UserCreds) -> - AllowConflictedUserDocs = config:get_boolean("chttpd_auth", "allow_conflicted_user_docs", false), + AllowConflictedUserDocs = config:get_boolean( + "chttpd_auth", "allow_conflicted_user_docs", false + ), case {couch_util:get_value(<<"_conflicts">>, UserCreds), AllowConflictedUserDocs} of {undefined, _} -> {ok, UserCreds, nil}; {_, true} -> {ok, UserCreds, nil}; {_ConflictList, false} -> - throw({unauthorized, - <<"User document conflicts must be resolved before the document", - " is used for authentication purposes.">> - }) + throw( + {unauthorized, + <<"User document conflicts must be resolved before the document", + " is used for authentication purposes.">>} + ) end. diff --git a/src/chttpd/src/chttpd_auth_request.erl b/src/chttpd/src/chttpd_auth_request.erl index f36203efe..301cf8e7d 100644 --- a/src/chttpd/src/chttpd_auth_request.erl +++ b/src/chttpd/src/chttpd_auth_request.erl @@ -14,106 +14,105 @@ -export([authorize_request/1]). -include_lib("couch/include/couch_db.hrl"). -authorize_request(#httpd{auth=Auth, user_ctx=Ctx} = Req) -> +authorize_request(#httpd{auth = Auth, user_ctx = Ctx} = Req) -> try - authorize_request_int(Req) + authorize_request_int(Req) catch - throw:{forbidden, Msg} -> - case {Auth, Ctx} of - {{cookie_auth_failed, {Error, Reason}}, _} -> - throw({forbidden, {Error, Reason}}); - {_, #user_ctx{name=null}} -> - throw({unauthorized, Msg}); - {_, _} -> - throw({forbidden, Msg}) - end + throw:{forbidden, Msg} -> + case {Auth, Ctx} of + {{cookie_auth_failed, {Error, Reason}}, _} -> + throw({forbidden, {Error, Reason}}); + {_, #user_ctx{name = null}} -> + throw({unauthorized, Msg}); + {_, _} -> + throw({forbidden, Msg}) + end end. -authorize_request_int(#httpd{path_parts=[]}=Req) -> +authorize_request_int(#httpd{path_parts = []} = Req) -> Req; -authorize_request_int(#httpd{path_parts=[<<"favicon.ico">>|_]}=Req) -> +authorize_request_int(#httpd{path_parts = [<<"favicon.ico">> | _]} = Req) -> Req; -authorize_request_int(#httpd{path_parts=[<<"_all_dbs">>|_]}=Req) -> - case config:get_boolean("chttpd", "admin_only_all_dbs", true) of - true -> require_admin(Req); - false -> Req - end; -authorize_request_int(#httpd{path_parts=[<<"_dbs_info">>|_]}=Req) -> +authorize_request_int(#httpd{path_parts = [<<"_all_dbs">> | _]} = Req) -> + case config:get_boolean("chttpd", "admin_only_all_dbs", true) of + true -> require_admin(Req); + false -> Req + end; +authorize_request_int(#httpd{path_parts = [<<"_dbs_info">> | _]} = Req) -> Req; -authorize_request_int(#httpd{path_parts=[<<"_replicator">>], method='PUT'}=Req) -> +authorize_request_int(#httpd{path_parts = [<<"_replicator">>], method = 'PUT'} = Req) -> require_admin(Req); -authorize_request_int(#httpd{path_parts=[<<"_replicator">>], method='DELETE'}=Req) -> +authorize_request_int(#httpd{path_parts = [<<"_replicator">>], method = 'DELETE'} = Req) -> require_admin(Req); -authorize_request_int(#httpd{path_parts=[<<"_replicator">>,<<"_all_docs">>|_]}=Req) -> +authorize_request_int(#httpd{path_parts = [<<"_replicator">>, <<"_all_docs">> | _]} = Req) -> require_admin(Req); -authorize_request_int(#httpd{path_parts=[<<"_replicator">>,<<"_changes">>|_]}=Req) -> +authorize_request_int(#httpd{path_parts = [<<"_replicator">>, <<"_changes">> | _]} = Req) -> require_admin(Req); -authorize_request_int(#httpd{path_parts=[<<"_replicator">>|_]}=Req) -> +authorize_request_int(#httpd{path_parts = [<<"_replicator">> | _]} = Req) -> db_authorization_check(Req); -authorize_request_int(#httpd{path_parts=[<<"_reshard">>|_]}=Req) -> +authorize_request_int(#httpd{path_parts = [<<"_reshard">> | _]} = Req) -> require_admin(Req); -authorize_request_int(#httpd{path_parts=[<<"_users">>], method='PUT'}=Req) -> +authorize_request_int(#httpd{path_parts = [<<"_users">>], method = 'PUT'} = Req) -> require_admin(Req); -authorize_request_int(#httpd{path_parts=[<<"_users">>], method='DELETE'}=Req) -> +authorize_request_int(#httpd{path_parts = [<<"_users">>], method = 'DELETE'} = Req) -> require_admin(Req); -authorize_request_int(#httpd{path_parts=[<<"_users">>,<<"_all_docs">>|_]}=Req) -> +authorize_request_int(#httpd{path_parts = [<<"_users">>, <<"_all_docs">> | _]} = Req) -> require_admin(Req); -authorize_request_int(#httpd{path_parts=[<<"_users">>,<<"_changes">>|_]}=Req) -> +authorize_request_int(#httpd{path_parts = [<<"_users">>, <<"_changes">> | _]} = Req) -> require_admin(Req); -authorize_request_int(#httpd{path_parts=[<<"_users">>|_]}=Req) -> +authorize_request_int(#httpd{path_parts = [<<"_users">> | _]} = Req) -> db_authorization_check(Req); -authorize_request_int(#httpd{path_parts=[<<"_", _/binary>>|_]}=Req) -> +authorize_request_int(#httpd{path_parts = [<<"_", _/binary>> | _]} = Req) -> server_authorization_check(Req); -authorize_request_int(#httpd{path_parts=[_DbName], method='PUT'}=Req) -> +authorize_request_int(#httpd{path_parts = [_DbName], method = 'PUT'} = Req) -> require_admin(Req); -authorize_request_int(#httpd{path_parts=[_DbName], method='DELETE'}=Req) -> +authorize_request_int(#httpd{path_parts = [_DbName], method = 'DELETE'} = Req) -> require_admin(Req); -authorize_request_int(#httpd{path_parts=[_DbName, <<"_compact">>|_]}=Req) -> +authorize_request_int(#httpd{path_parts = [_DbName, <<"_compact">> | _]} = Req) -> require_db_admin(Req); -authorize_request_int(#httpd{path_parts=[_DbName, <<"_view_cleanup">>]}=Req) -> +authorize_request_int(#httpd{path_parts = [_DbName, <<"_view_cleanup">>]} = Req) -> require_db_admin(Req); -authorize_request_int(#httpd{path_parts=[_DbName, <<"_sync_shards">>]}=Req) -> +authorize_request_int(#httpd{path_parts = [_DbName, <<"_sync_shards">>]} = Req) -> require_admin(Req); -authorize_request_int(#httpd{path_parts=[_DbName, <<"_purge">>]}=Req) -> +authorize_request_int(#httpd{path_parts = [_DbName, <<"_purge">>]} = Req) -> require_admin(Req); -authorize_request_int(#httpd{path_parts=[_DbName, <<"_purged_infos_limit">>]}=Req) -> +authorize_request_int(#httpd{path_parts = [_DbName, <<"_purged_infos_limit">>]} = Req) -> require_admin(Req); -authorize_request_int(#httpd{path_parts=[_DbName|_]}=Req) -> +authorize_request_int(#httpd{path_parts = [_DbName | _]} = Req) -> db_authorization_check(Req). - -server_authorization_check(#httpd{path_parts=[<<"_up">>]}=Req) -> +server_authorization_check(#httpd{path_parts = [<<"_up">>]} = Req) -> Req; -server_authorization_check(#httpd{path_parts=[<<"_uuids">>]}=Req) -> +server_authorization_check(#httpd{path_parts = [<<"_uuids">>]} = Req) -> Req; -server_authorization_check(#httpd{path_parts=[<<"_session">>]}=Req) -> +server_authorization_check(#httpd{path_parts = [<<"_session">>]} = Req) -> Req; -server_authorization_check(#httpd{path_parts=[<<"_replicate">>]}=Req) -> +server_authorization_check(#httpd{path_parts = [<<"_replicate">>]} = Req) -> Req; -server_authorization_check(#httpd{path_parts=[<<"_stats">>]}=Req) -> +server_authorization_check(#httpd{path_parts = [<<"_stats">>]} = Req) -> Req; -server_authorization_check(#httpd{path_parts=[<<"_active_tasks">>]}=Req) -> +server_authorization_check(#httpd{path_parts = [<<"_active_tasks">>]} = Req) -> Req; -server_authorization_check(#httpd{path_parts=[<<"_dbs_info">>]}=Req) -> +server_authorization_check(#httpd{path_parts = [<<"_dbs_info">>]} = Req) -> Req; -server_authorization_check(#httpd{method=Method, path_parts=[<<"_utils">>|_]}=Req) - when Method =:= 'HEAD' orelse Method =:= 'GET' -> +server_authorization_check(#httpd{method = Method, path_parts = [<<"_utils">> | _]} = Req) when + Method =:= 'HEAD' orelse Method =:= 'GET' +-> Req; -server_authorization_check(#httpd{path_parts=[<<"_node">>,_ , <<"_stats">>|_]}=Req) -> +server_authorization_check(#httpd{path_parts = [<<"_node">>, _, <<"_stats">> | _]} = Req) -> require_metrics(Req); -server_authorization_check(#httpd{path_parts=[<<"_node">>,_ , <<"_system">>|_]}=Req) -> +server_authorization_check(#httpd{path_parts = [<<"_node">>, _, <<"_system">> | _]} = Req) -> require_metrics(Req); -server_authorization_check(#httpd{path_parts=[<<"_node">>,_ , <<"_prometheus">>|_]}=Req) -> +server_authorization_check(#httpd{path_parts = [<<"_node">>, _, <<"_prometheus">> | _]} = Req) -> require_metrics(Req); -server_authorization_check(#httpd{path_parts=[<<"_", _/binary>>|_]}=Req) -> +server_authorization_check(#httpd{path_parts = [<<"_", _/binary>> | _]} = Req) -> require_admin(Req). -db_authorization_check(#httpd{path_parts=[DbName|_],user_ctx=Ctx}=Req) -> +db_authorization_check(#httpd{path_parts = [DbName | _], user_ctx = Ctx} = Req) -> {_} = fabric:get_security(DbName, [{user_ctx, Ctx}]), Req. - -require_metrics(#httpd{user_ctx=#user_ctx{roles=UserRoles}}=Req) -> +require_metrics(#httpd{user_ctx = #user_ctx{roles = UserRoles}} = Req) -> IsAdmin = lists:member(<<"_admin">>, UserRoles), IsMetrics = lists:member(<<"_metrics">>, UserRoles), case {IsAdmin, IsMetrics} of @@ -126,15 +125,15 @@ require_admin(Req) -> ok = couch_httpd:verify_is_server_admin(Req), Req. -require_db_admin(#httpd{path_parts=[DbName|_],user_ctx=Ctx}=Req) -> +require_db_admin(#httpd{path_parts = [DbName | _], user_ctx = Ctx} = Req) -> Sec = fabric:get_security(DbName, [{user_ctx, Ctx}]), - case is_db_admin(Ctx,Sec) of + case is_db_admin(Ctx, Sec) of true -> Req; - false -> throw({unauthorized, <<"You are not a server or db admin.">>}) + false -> throw({unauthorized, <<"You are not a server or db admin.">>}) end. -is_db_admin(#user_ctx{name=UserName,roles=UserRoles}, {Security}) -> +is_db_admin(#user_ctx{name = UserName, roles = UserRoles}, {Security}) -> {Admins} = couch_util:get_value(<<"admins">>, Security, {[]}), Names = couch_util:get_value(<<"names">>, Admins, []), Roles = couch_util:get_value(<<"roles">>, Admins, []), diff --git a/src/chttpd/src/chttpd_cors.erl b/src/chttpd/src/chttpd_cors.erl index 62b19a0d2..70d3163ec 100644 --- a/src/chttpd/src/chttpd_cors.erl +++ b/src/chttpd/src/chttpd_cors.erl @@ -12,7 +12,6 @@ -module(chttpd_cors). - -export([ maybe_handle_preflight_request/1, maybe_handle_preflight_request/2, @@ -24,14 +23,12 @@ get_cors_config/1 ]). - -include_lib("couch/include/couch_db.hrl"). -include_lib("chttpd/include/chttpd_cors.hrl"). - %% http://www.w3.org/TR/cors/#resource-preflight-requests -maybe_handle_preflight_request(#httpd{method=Method}) when Method /= 'OPTIONS' -> +maybe_handle_preflight_request(#httpd{method = Method}) when Method /= 'OPTIONS' -> not_preflight; maybe_handle_preflight_request(Req) -> case maybe_handle_preflight_request(Req, get_cors_config(Req)) of @@ -41,8 +38,7 @@ maybe_handle_preflight_request(Req) -> chttpd:send_response_no_cors(Req, 204, PreflightHeaders, <<>>) end. - -maybe_handle_preflight_request(#httpd{}=Req, Config) -> +maybe_handle_preflight_request(#httpd{} = Req, Config) -> case is_cors_enabled(Config) of true -> case preflight_request(Req, Config) of @@ -61,7 +57,6 @@ maybe_handle_preflight_request(#httpd{}=Req, Config) -> not_preflight end. - preflight_request(Req, Config) -> case get_origin(Req) of undefined -> @@ -96,70 +91,85 @@ preflight_request(Req, Config) -> end end. - handle_preflight_request(Req, Config, Origin) -> case chttpd:header_value(Req, "Access-Control-Request-Method") of - undefined -> - %% If there is no Access-Control-Request-Method header - %% or if parsing failed, do not set any additional headers - %% and terminate this set of steps. The request is outside - %% the scope of this specification. - %% http://www.w3.org/TR/cors/#resource-preflight-requests - not_preflight; - Method -> - SupportedMethods = get_origin_config(Config, Origin, - <<"allow_methods">>, ?SUPPORTED_METHODS), - - SupportedHeaders = get_origin_config(Config, Origin, - <<"allow_headers">>, ?SUPPORTED_HEADERS), - - - %% get max age - MaxAge = couch_util:get_value(<<"max_age">>, Config, - ?CORS_DEFAULT_MAX_AGE), - - PreflightHeaders0 = maybe_add_credentials(Config, Origin, [ - {"Access-Control-Allow-Origin", binary_to_list(Origin)}, - {"Access-Control-Max-Age", MaxAge}, - {"Access-Control-Allow-Methods", - string:join(SupportedMethods, ", ")}]), - - case lists:member(Method, SupportedMethods) of - true -> - %% method ok , check headers - AccessHeaders = chttpd:header_value(Req, - "Access-Control-Request-Headers"), - {FinalReqHeaders, ReqHeaders} = case AccessHeaders of - undefined -> {"", []}; - "" -> {"", []}; - Headers -> - %% transform header list in something we - %% could check. make sure everything is a - %% list - RH = [to_lower(H) - || H <- split_headers(Headers)], - {Headers, RH} - end, - %% check if headers are supported - case ReqHeaders -- SupportedHeaders of - [] -> - PreflightHeaders = PreflightHeaders0 ++ - [{"Access-Control-Allow-Headers", - FinalReqHeaders}], - {ok, PreflightHeaders}; - _ -> - not_preflight - end; - false -> - %% If method is not a case-sensitive match for any of - %% the values in list of methods do not set any additional - %% headers and terminate this set of steps. + undefined -> + %% If there is no Access-Control-Request-Method header + %% or if parsing failed, do not set any additional headers + %% and terminate this set of steps. The request is outside + %% the scope of this specification. %% http://www.w3.org/TR/cors/#resource-preflight-requests - not_preflight - end + not_preflight; + Method -> + SupportedMethods = get_origin_config( + Config, + Origin, + <<"allow_methods">>, + ?SUPPORTED_METHODS + ), + + SupportedHeaders = get_origin_config( + Config, + Origin, + <<"allow_headers">>, + ?SUPPORTED_HEADERS + ), + + %% get max age + MaxAge = couch_util:get_value( + <<"max_age">>, + Config, + ?CORS_DEFAULT_MAX_AGE + ), + + PreflightHeaders0 = maybe_add_credentials(Config, Origin, [ + {"Access-Control-Allow-Origin", binary_to_list(Origin)}, + {"Access-Control-Max-Age", MaxAge}, + {"Access-Control-Allow-Methods", string:join(SupportedMethods, ", ")} + ]), + + case lists:member(Method, SupportedMethods) of + true -> + %% method ok , check headers + AccessHeaders = chttpd:header_value( + Req, + "Access-Control-Request-Headers" + ), + {FinalReqHeaders, ReqHeaders} = + case AccessHeaders of + undefined -> + {"", []}; + "" -> + {"", []}; + Headers -> + %% transform header list in something we + %% could check. make sure everything is a + %% list + RH = [ + to_lower(H) + || H <- split_headers(Headers) + ], + {Headers, RH} + end, + %% check if headers are supported + case ReqHeaders -- SupportedHeaders of + [] -> + PreflightHeaders = + PreflightHeaders0 ++ + [{"Access-Control-Allow-Headers", FinalReqHeaders}], + {ok, PreflightHeaders}; + _ -> + not_preflight + end; + false -> + %% If method is not a case-sensitive match for any of + %% the values in list of methods do not set any additional + %% headers and terminate this set of steps. + %% http://www.w3.org/TR/cors/#resource-preflight-requests + not_preflight + end end. - headers(Req, RequestHeaders) -> case get_origin(Req) of undefined -> @@ -172,7 +182,6 @@ headers(Req, RequestHeaders) -> headers(Req, RequestHeaders, Origin, get_cors_config(Req)) end. - headers(_Req, RequestHeaders, undefined, _Config) -> RequestHeaders; headers(Req, RequestHeaders, Origin, Config) when is_list(Origin) -> @@ -183,13 +192,13 @@ headers(Req, RequestHeaders, Origin, Config) -> AcceptedOrigins = get_accepted_origins(Req, Config), CorsHeaders = handle_headers(Config, Origin, AcceptedOrigins), ExposedCouchHeaders = couch_util:get_value( - <<"exposed_headers">>, Config, ?COUCH_HEADERS), + <<"exposed_headers">>, Config, ?COUCH_HEADERS + ), maybe_apply_headers(CorsHeaders, RequestHeaders, ExposedCouchHeaders); false -> RequestHeaders end. - maybe_apply_headers([], RequestHeaders, _ExposedCouchHeaders) -> RequestHeaders; maybe_apply_headers(CorsHeaders, RequestHeaders, ExposedCouchHeaders) -> @@ -200,67 +209,64 @@ maybe_apply_headers(CorsHeaders, RequestHeaders, ExposedCouchHeaders) -> %% need to be exposed. %% return: RequestHeaders ++ CorsHeaders ++ ACEH - ExposedHeaders0 = simple_headers([K || {K,_V} <- RequestHeaders]), + ExposedHeaders0 = simple_headers([K || {K, _V} <- RequestHeaders]), %% If Content-Type is not in ExposedHeaders, and the Content-Type %% is not a member of ?SIMPLE_CONTENT_TYPE_VALUES, then add it %% into the list of ExposedHeaders ContentType = proplists:get_value("content-type", ExposedHeaders0), - IncludeContentType = case ContentType of - undefined -> - false; - _ -> - lists:member(string:to_lower(ContentType), ?SIMPLE_CONTENT_TYPE_VALUES) + IncludeContentType = + case ContentType of + undefined -> + false; + _ -> + lists:member(string:to_lower(ContentType), ?SIMPLE_CONTENT_TYPE_VALUES) end, - ExposedHeaders = case IncludeContentType of - false -> - ["content-type" | lists:delete("content-type", ExposedHeaders0)]; - true -> - ExposedHeaders0 + ExposedHeaders = + case IncludeContentType of + false -> + ["content-type" | lists:delete("content-type", ExposedHeaders0)]; + true -> + ExposedHeaders0 end, %% ExposedCouchHeaders may get added later, so expose them by default - ACEH = [{"Access-Control-Expose-Headers", - string:join(ExposedHeaders ++ ExposedCouchHeaders, ", ")}], + ACEH = [ + {"Access-Control-Expose-Headers", string:join(ExposedHeaders ++ ExposedCouchHeaders, ", ")} + ], CorsHeaders ++ RequestHeaders ++ ACEH. - simple_headers(Headers) -> LCHeaders = [to_lower(H) || H <- Headers], lists:filter(fun(H) -> lists:member(H, ?SIMPLE_HEADERS) end, LCHeaders). - to_lower(String) when is_binary(String) -> to_lower(?b2l(String)); to_lower(String) -> string:to_lower(String). - handle_headers(_Config, _Origin, []) -> []; handle_headers(Config, Origin, AcceptedOrigins) -> AcceptAll = lists:member(<<"*">>, AcceptedOrigins), case AcceptAll orelse lists:member(Origin, AcceptedOrigins) of - true -> - make_cors_header(Config, Origin); - false -> - %% If the value of the Origin header is not a - %% case-sensitive match for any of the values - %% in list of origins, do not set any additional - %% headers and terminate this set of steps. - %% http://www.w3.org/TR/cors/#resource-requests - [] + true -> + make_cors_header(Config, Origin); + false -> + %% If the value of the Origin header is not a + %% case-sensitive match for any of the values + %% in list of origins, do not set any additional + %% headers and terminate this set of steps. + %% http://www.w3.org/TR/cors/#resource-requests + [] end. - make_cors_header(Config, Origin) -> Headers = [{"Access-Control-Allow-Origin", binary_to_list(Origin)}], maybe_add_credentials(Config, Origin, Headers). - %% util - maybe_add_credentials(Config, Origin, Headers) -> case allow_credentials(Config, Origin) of false -> @@ -269,13 +275,15 @@ maybe_add_credentials(Config, Origin, Headers) -> Headers ++ [{"Access-Control-Allow-Credentials", "true"}] end. - allow_credentials(_Config, <<"*">>) -> false; allow_credentials(Config, Origin) -> - get_origin_config(Config, Origin, <<"allow_credentials">>, - ?CORS_DEFAULT_ALLOW_CREDENTIALS). - + get_origin_config( + Config, + Origin, + <<"allow_credentials">>, + ?CORS_DEFAULT_ALLOW_CREDENTIALS + ). get_cors_config(#httpd{cors_config = undefined, mochi_req = MochiReq}) -> Host = couch_httpd_vhost:host(MochiReq), @@ -283,24 +291,27 @@ get_cors_config(#httpd{cors_config = undefined, mochi_req = MochiReq}) -> EnableCors = chttpd_util:get_chttpd_config_boolean("enable_cors", false), AllowCredentials = cors_config(Host, "credentials", "false") =:= "true", - AllowHeaders = case cors_config(Host, "headers", undefined) of - undefined -> - ?SUPPORTED_HEADERS; - AllowHeaders0 -> - [to_lower(H) || H <- split_list(AllowHeaders0)] - end, - AllowMethods = case cors_config(Host, "methods", undefined) of - undefined -> - ?SUPPORTED_METHODS; - AllowMethods0 -> - split_list(AllowMethods0) - end, - ExposedHeaders = case cors_config(Host, "exposed_headers", undefined) of - undefined -> - ?COUCH_HEADERS; - ExposedHeaders0 -> - [to_lower(H) || H <- split_list(ExposedHeaders0)] - end, + AllowHeaders = + case cors_config(Host, "headers", undefined) of + undefined -> + ?SUPPORTED_HEADERS; + AllowHeaders0 -> + [to_lower(H) || H <- split_list(AllowHeaders0)] + end, + AllowMethods = + case cors_config(Host, "methods", undefined) of + undefined -> + ?SUPPORTED_METHODS; + AllowMethods0 -> + split_list(AllowMethods0) + end, + ExposedHeaders = + case cors_config(Host, "exposed_headers", undefined) of + undefined -> + ?COUCH_HEADERS; + ExposedHeaders0 -> + [to_lower(H) || H <- split_list(ExposedHeaders0)] + end, MaxAge = cors_config(Host, "max_age", ?CORS_DEFAULT_MAX_AGE), Origins0 = binary_split_list(cors_config(Host, "origins", [])), Origins = [{O, {[]}} || O <- Origins0], @@ -316,25 +327,24 @@ get_cors_config(#httpd{cors_config = undefined, mochi_req = MochiReq}) -> get_cors_config(#httpd{cors_config = Config}) -> Config. - cors_config(Host, Key, Default) -> - config:get(cors_section(Host), Key, - config:get("cors", Key, Default)). - + config:get( + cors_section(Host), + Key, + config:get("cors", Key, Default) + ). cors_section(HostValue) -> HostPort = maybe_strip_scheme(HostValue), Host = hd(string:tokens(HostPort, ":")), "cors:" ++ Host. - maybe_strip_scheme(Host) -> case string:str(Host, "://") of 0 -> Host; N -> string:substr(Host, N + 3) end. - is_cors_enabled(Config) -> case get(disable_couch_httpd_cors) of undefined -> @@ -344,7 +354,6 @@ is_cors_enabled(Config) -> end, couch_util:get_value(<<"enable_cors">>, Config, false). - %% Get a list of {Origin, OriginConfig} tuples %% ie: get_origin_configs(Config) -> %% [ @@ -362,7 +371,6 @@ get_origin_configs(Config) -> {Origins} = couch_util:get_value(<<"origins">>, Config, {[]}), Origins. - %% Get config for an individual Origin %% ie: get_origin_config(Config, <<"http://foo.com">>) -> %% [ @@ -374,15 +382,16 @@ get_origin_config(Config, Origin) -> {OriginConfig} = couch_util:get_value(Origin, OriginConfigs, {[]}), OriginConfig. - %% Get config of a single key for an individual Origin %% ie: get_origin_config(Config, <<"http://foo.com">>, <<"allow_methods">>, []) %% [<<"POST">>] get_origin_config(Config, Origin, Key, Default) -> OriginConfig = get_origin_config(Config, Origin), - couch_util:get_value(Key, OriginConfig, - couch_util:get_value(Key, Config, Default)). - + couch_util:get_value( + Key, + OriginConfig, + couch_util:get_value(Key, Config, Default) + ). get_origin(Req) -> case chttpd:header_value(Req, "Origin") of @@ -392,18 +401,14 @@ get_origin(Req) -> ?l2b(Origin) end. - get_accepted_origins(_Req, Config) -> - lists:map(fun({K,_V}) -> K end, get_origin_configs(Config)). - + lists:map(fun({K, _V}) -> K end, get_origin_configs(Config)). split_list(S) -> re:split(S, "\\s*,\\s*", [trim, {return, list}]). - binary_split_list(S) -> [list_to_binary(E) || E <- split_list(S)]. - split_headers(H) -> - re:split(H, ",\\s*", [{return,list}, trim]). + re:split(H, ",\\s*", [{return, list}, trim]). diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl index 8d45eb779..875df6e00 100644 --- a/src/chttpd/src/chttpd_db.erl +++ b/src/chttpd/src/chttpd_db.erl @@ -18,18 +18,35 @@ -include_lib("couch_mrview/include/couch_mrview.hrl"). -include_lib("mem3/include/mem3.hrl"). --export([handle_request/1, handle_compact_req/2, handle_design_req/2, - db_req/2, couch_doc_open/4,handle_changes_req/2, +-export([ + handle_request/1, + handle_compact_req/2, + handle_design_req/2, + db_req/2, + couch_doc_open/4, + handle_changes_req/2, update_doc_result_to_json/1, update_doc_result_to_json/2, - handle_design_info_req/3, handle_view_cleanup_req/2, - update_doc/4, http_code_from_status/1, - handle_partition_req/2]). - --import(chttpd, - [send_json/2,send_json/3,send_json/4,send_method_not_allowed/2, - start_json_response/2,send_chunk/2,end_json_response/1, - start_chunked_response/3, absolute_uri/2, send/2, - start_response_length/4]). + handle_design_info_req/3, + handle_view_cleanup_req/2, + update_doc/4, + http_code_from_status/1, + handle_partition_req/2 +]). + +-import( + chttpd, + [ + send_json/2, send_json/3, send_json/4, + send_method_not_allowed/2, + start_json_response/2, + send_chunk/2, + end_json_response/1, + start_chunked_response/3, + absolute_uri/2, + send/2, + start_response_length/4 + ] +). -record(doc_query_args, { options = [], @@ -52,38 +69,44 @@ threshold }). --define(IS_ALL_DOCS(T), ( - T == <<"_all_docs">> - orelse T == <<"_local_docs">> - orelse T == <<"_design_docs">>)). +-define(IS_ALL_DOCS(T), + (T == <<"_all_docs">> orelse + T == <<"_local_docs">> orelse + T == <<"_design_docs">>) +). --define(IS_MANGO(T), ( - T == <<"_index">> - orelse T == <<"_find">> - orelse T == <<"_explain">>)). +-define(IS_MANGO(T), + (T == <<"_index">> orelse + T == <<"_find">> orelse + T == <<"_explain">>) +). % Database request handlers -handle_request(#httpd{path_parts=[DbName|RestParts],method=Method}=Req)-> +handle_request(#httpd{path_parts = [DbName | RestParts], method = Method} = Req) -> case {Method, RestParts} of - {'PUT', []} -> - create_db_req(Req, DbName); - {'DELETE', []} -> - % if we get ?rev=... the user is using a faulty script where the - % document id is empty by accident. Let them recover safely. - case chttpd:qs_value(Req, "rev", false) of - false -> delete_db_req(Req, DbName); - _Rev -> throw({bad_request, - "You tried to DELETE a database with a ?=rev parameter. " - ++ "Did you mean to DELETE a document instead?"}) - end; - {_, []} -> - do_db_req(Req, fun db_req/2); - {_, [SecondPart|_]} -> - Handler = chttpd_handlers:db_handler(SecondPart, fun db_req/2), - do_db_req(Req, Handler) + {'PUT', []} -> + create_db_req(Req, DbName); + {'DELETE', []} -> + % if we get ?rev=... the user is using a faulty script where the + % document id is empty by accident. Let them recover safely. + case chttpd:qs_value(Req, "rev", false) of + false -> + delete_db_req(Req, DbName); + _Rev -> + throw( + {bad_request, + "You tried to DELETE a database with a ?=rev parameter. " ++ + "Did you mean to DELETE a document instead?"} + ) + end; + {_, []} -> + do_db_req(Req, fun db_req/2); + {_, [SecondPart | _]} -> + Handler = chttpd_handlers:db_handler(SecondPart, fun db_req/2), + do_db_req(Req, Handler) end. -handle_changes_req(#httpd{method='POST'}=Req, Db) -> +handle_changes_req(#httpd{method = 'POST'} = Req, Db) -> chttpd:validate_ctype(Req, "application/json"), case chttpd:body_length(Req) of 0 -> @@ -92,50 +115,50 @@ handle_changes_req(#httpd{method='POST'}=Req, Db) -> {JsonProps} = chttpd:json_body_obj(Req), handle_changes_req1(Req#httpd{req_body = {JsonProps}}, Db) end; -handle_changes_req(#httpd{method='GET'}=Req, Db) -> +handle_changes_req(#httpd{method = 'GET'} = Req, Db) -> handle_changes_req1(Req, Db); -handle_changes_req(#httpd{path_parts=[_,<<"_changes">>]}=Req, _Db) -> +handle_changes_req(#httpd{path_parts = [_, <<"_changes">>]} = Req, _Db) -> send_method_not_allowed(Req, "GET,POST,HEAD"). -handle_changes_req1(#httpd{}=Req, Db) -> - #changes_args{filter=Raw, style=Style} = Args0 = parse_changes_query(Req), +handle_changes_req1(#httpd{} = Req, Db) -> + #changes_args{filter = Raw, style = Style} = Args0 = parse_changes_query(Req), ChangesArgs = Args0#changes_args{ filter_fun = couch_changes:configure_filter(Raw, Style, Req, Db), db_open_options = [{user_ctx, couch_db:get_user_ctx(Db)}] }, Max = chttpd:chunked_response_buffer_size(), case ChangesArgs#changes_args.feed of - "normal" -> - T0 = os:timestamp(), - {ok, Info} = fabric:get_db_info(Db), - Suffix = mem3:shard_suffix(Db), - Etag = chttpd:make_etag({Info, Suffix}), - DeltaT = timer:now_diff(os:timestamp(), T0) / 1000, - couch_stats:update_histogram([couchdb, dbinfo], DeltaT), - chttpd:etag_respond(Req, Etag, fun() -> + "normal" -> + T0 = os:timestamp(), + {ok, Info} = fabric:get_db_info(Db), + Suffix = mem3:shard_suffix(Db), + Etag = chttpd:make_etag({Info, Suffix}), + DeltaT = timer:now_diff(os:timestamp(), T0) / 1000, + couch_stats:update_histogram([couchdb, dbinfo], DeltaT), + chttpd:etag_respond(Req, Etag, fun() -> + Acc0 = #cacc{ + feed = normal, + etag = Etag, + mochi = Req, + threshold = Max + }, + fabric:changes(Db, fun changes_callback/2, Acc0, ChangesArgs) + end); + Feed when Feed =:= "continuous"; Feed =:= "longpoll"; Feed =:= "eventsource" -> + couch_stats:increment_counter([couchdb, httpd, clients_requesting_changes]), Acc0 = #cacc{ - feed = normal, - etag = Etag, + feed = list_to_atom(Feed), mochi = Req, threshold = Max }, - fabric:changes(Db, fun changes_callback/2, Acc0, ChangesArgs) - end); - Feed when Feed =:= "continuous"; Feed =:= "longpoll"; Feed =:= "eventsource" -> - couch_stats:increment_counter([couchdb, httpd, clients_requesting_changes]), - Acc0 = #cacc{ - feed = list_to_atom(Feed), - mochi = Req, - threshold = Max - }, - try - fabric:changes(Db, fun changes_callback/2, Acc0, ChangesArgs) - after - couch_stats:decrement_counter([couchdb, httpd, clients_requesting_changes]) - end; - _ -> - Msg = <<"Supported `feed` types: normal, continuous, live, longpoll, eventsource">>, - throw({bad_request, Msg}) + try + fabric:changes(Db, fun changes_callback/2, Acc0, ChangesArgs) + after + couch_stats:decrement_counter([couchdb, httpd, clients_requesting_changes]) + end; + _ -> + Msg = <<"Supported `feed` types: normal, continuous, live, longpoll, eventsource">>, + throw({bad_request, Msg}) end. % callbacks for continuous feed (newline-delimited JSON Objects) @@ -149,14 +172,14 @@ changes_callback({change, Change}, #cacc{feed = continuous} = Acc) -> maybe_flush_changes_feed(Acc, Data, Len); changes_callback({stop, EndSeq, Pending}, #cacc{feed = continuous} = Acc) -> #cacc{mochi = Resp, buffer = Buf} = Acc, - Row = {[ - {<<"last_seq">>, EndSeq}, - {<<"pending">>, Pending} - ]}, + Row = + {[ + {<<"last_seq">>, EndSeq}, + {<<"pending">>, Pending} + ]}, Data = [Buf, ?JSON_ENCODE(Row) | "\n"], {ok, Resp1} = chttpd:send_delayed_chunk(Resp, Data), chttpd:end_delayed_json_response(Resp1); - % callbacks for eventsource feed (newline-delimited eventsource Objects) changes_callback(start, #cacc{feed = eventsource} = Acc) -> #cacc{mochi = Req} = Acc, @@ -166,12 +189,15 @@ changes_callback(start, #cacc{feed = eventsource} = Acc) -> ], {ok, Resp} = chttpd:start_delayed_json_response(Req, 200, Headers), {ok, Acc#cacc{mochi = Resp, responding = true}}; -changes_callback({change, {ChangeProp}=Change}, #cacc{feed = eventsource} = Acc) -> +changes_callback({change, {ChangeProp} = Change}, #cacc{feed = eventsource} = Acc) -> chttpd_stats:incr_rows(), Seq = proplists:get_value(seq, ChangeProp), Chunk = [ - "data: ", ?JSON_ENCODE(Change), - "\n", "id: ", ?JSON_ENCODE(Seq), + "data: ", + ?JSON_ENCODE(Change), + "\n", + "id: ", + ?JSON_ENCODE(Seq), "\n\n" ], Len = iolist_size(Chunk), @@ -185,13 +211,16 @@ changes_callback({stop, _EndSeq}, #cacc{feed = eventsource} = Acc) -> #cacc{mochi = Resp, buffer = Buf} = Acc, {ok, Resp1} = chttpd:send_delayed_chunk(Resp, Buf), chttpd:end_delayed_json_response(Resp1); - % callbacks for longpoll and normal (single JSON Object) changes_callback(start, #cacc{feed = normal} = Acc) -> #cacc{etag = Etag, mochi = Req} = Acc, FirstChunk = "{\"results\":[\n", - {ok, Resp} = chttpd:start_delayed_json_response(Req, 200, - [{"ETag",Etag}], FirstChunk), + {ok, Resp} = chttpd:start_delayed_json_response( + Req, + 200, + [{"ETag", Etag}], + FirstChunk + ), {ok, Acc#cacc{mochi = Resp, responding = true}}; changes_callback(start, Acc) -> #cacc{mochi = Req} = Acc, @@ -214,7 +243,6 @@ changes_callback({stop, EndSeq, Pending}, Acc) -> ], {ok, Resp1} = chttpd:close_delayed_json_object(Resp, Buf, Terminator, Max), chttpd:end_delayed_json_response(Resp1); - changes_callback(waiting_for_updates, #cacc{buffer = []} = Acc) -> #cacc{mochi = Resp, chunks_sent = ChunksSent} = Acc, case ChunksSent > 0 of @@ -246,11 +274,12 @@ changes_callback({error, Reason}, #cacc{feed = normal, responding = false} = Acc changes_callback({error, Reason}, Acc) -> chttpd:send_delayed_error(Acc#cacc.mochi, Reason). -maybe_flush_changes_feed(#cacc{bufsize=Size, threshold=Max} = Acc, Data, Len) - when Size > 0 andalso (Size + Len) > Max -> +maybe_flush_changes_feed(#cacc{bufsize = Size, threshold = Max} = Acc, Data, Len) when + Size > 0 andalso (Size + Len) > Max +-> #cacc{buffer = Buffer, mochi = Resp} = Acc, {ok, R1} = chttpd:send_delayed_chunk(Resp, Buffer), - {ok, Acc#cacc{prepend = ",\r\n", buffer = Data, bufsize=Len, mochi = R1}}; + {ok, Acc#cacc{prepend = ",\r\n", buffer = Data, bufsize = Len, mochi = R1}}; maybe_flush_changes_feed(Acc0, Data, Len) -> #cacc{buffer = Buf, bufsize = Size, chunks_sent = ChunksSent} = Acc0, Acc = Acc0#cacc{ @@ -261,7 +290,7 @@ maybe_flush_changes_feed(Acc0, Data, Len) -> }, {ok, Acc}. -handle_compact_req(#httpd{method='POST'}=Req, Db) -> +handle_compact_req(#httpd{method = 'POST'} = Req, Db) -> chttpd:validate_ctype(Req, "application/json"), case Req#httpd.path_parts of [_DbName, <<"_compact">>] -> @@ -276,7 +305,6 @@ handle_compact_req(#httpd{method='POST'}=Req, Db) -> throw(Error) end end; - handle_compact_req(Req, _Db) -> send_method_not_allowed(Req, "POST"). @@ -284,11 +312,9 @@ handle_view_cleanup_req(Req, Db) -> ok = fabric:cleanup_index_files_all_nodes(Db), send_json(Req, 202, {[{ok, true}]}). - -handle_partition_req(#httpd{path_parts=[_,_]}=_Req, _Db) -> +handle_partition_req(#httpd{path_parts = [_, _]} = _Req, _Db) -> throw({bad_request, invalid_partition_req}); - -handle_partition_req(#httpd{method='GET', path_parts=[_,_,PartId]}=Req, Db) -> +handle_partition_req(#httpd{method = 'GET', path_parts = [_, _, PartId]} = Req, Db) -> couch_partition:validate_partition(PartId), case couch_db:is_partitioned(Db) of true -> @@ -297,25 +323,30 @@ handle_partition_req(#httpd{method='GET', path_parts=[_,_,PartId]}=Req, Db) -> false -> throw({bad_request, <<"database is not partitioned">>}) end; - -handle_partition_req(#httpd{method='POST', - path_parts=[_, <<"_partition">>, <<"_", _/binary>>]}, _Db) -> +handle_partition_req( + #httpd{ + method = 'POST', + path_parts = [_, <<"_partition">>, <<"_", _/binary>>] + }, + _Db +) -> Msg = <<"Partition must not start with an underscore">>, throw({illegal_partition, Msg}); - -handle_partition_req(#httpd{path_parts = [_, _, _]}=Req, _Db) -> +handle_partition_req(#httpd{path_parts = [_, _, _]} = Req, _Db) -> send_method_not_allowed(Req, "GET"); - -handle_partition_req(#httpd{path_parts=[DbName, _, PartId | Rest]}=Req, Db) -> +handle_partition_req(#httpd{path_parts = [DbName, _, PartId | Rest]} = Req, Db) -> case couch_db:is_partitioned(Db) of true -> couch_partition:validate_partition(PartId), QS = chttpd:qs(Req), PartIdStr = ?b2l(PartId), QSPartIdStr = couch_util:get_value("partition", QS, PartIdStr), - if QSPartIdStr == PartIdStr -> ok; true -> - Msg = <<"Conflicting value for `partition` in query string">>, - throw({bad_request, Msg}) + if + QSPartIdStr == PartIdStr -> + ok; + true -> + Msg = <<"Conflicting value for `partition` in query string">>, + throw({bad_request, Msg}) end, NewQS = lists:ukeysort(1, [{"partition", PartIdStr} | QS]), NewReq = Req#httpd{ @@ -339,55 +370,59 @@ handle_partition_req(#httpd{path_parts=[DbName, _, PartId | Rest]}=Req, Db) -> false -> throw({bad_request, <<"database is not partitioned">>}) end; - handle_partition_req(Req, _Db) -> chttpd:send_error(Req, not_found). update_partition_stats(PathParts) -> case PathParts of - [<<"_design">> | _] -> - couch_stats:increment_counter([couchdb, httpd, partition_view_requests]); - [<<"_all_docs">> | _] -> - couch_stats:increment_counter([couchdb, httpd, partition_all_docs_requests]); - [<<"_find">> | _] -> - couch_stats:increment_counter([couchdb, httpd, partition_find_requests]); - [<<"_explain">> | _] -> - couch_stats:increment_counter([couchdb, httpd, partition_explain_requests]); - _ -> - ok % ignore path that do not match - end. - + [<<"_design">> | _] -> + couch_stats:increment_counter([couchdb, httpd, partition_view_requests]); + [<<"_all_docs">> | _] -> + couch_stats:increment_counter([couchdb, httpd, partition_all_docs_requests]); + [<<"_find">> | _] -> + couch_stats:increment_counter([couchdb, httpd, partition_find_requests]); + [<<"_explain">> | _] -> + couch_stats:increment_counter([couchdb, httpd, partition_explain_requests]); + _ -> + % ignore path that do not match + ok + end. -handle_design_req(#httpd{ - path_parts=[_DbName, _Design, Name, <<"_",_/binary>> = Action | _Rest] - }=Req, Db) -> +handle_design_req( + #httpd{ + path_parts = [_DbName, _Design, Name, <<"_", _/binary>> = Action | _Rest] + } = Req, + Db +) -> DbName = mem3:dbname(couch_db:name(Db)), case ddoc_cache:open(DbName, <<"_design/", Name/binary>>) of - {ok, DDoc} -> - Handler = chttpd_handlers:design_handler(Action, fun bad_action_req/3), - Handler(Req, Db, DDoc); - Error -> - throw(Error) + {ok, DDoc} -> + Handler = chttpd_handlers:design_handler(Action, fun bad_action_req/3), + Handler(Req, Db, DDoc); + Error -> + throw(Error) end; - handle_design_req(Req, Db) -> db_req(Req, Db). -bad_action_req(#httpd{path_parts=[_, _, Name|FileNameParts]}=Req, Db, _DDoc) -> - db_attachment_req(Req, Db, <<"_design/",Name/binary>>, FileNameParts). +bad_action_req(#httpd{path_parts = [_, _, Name | FileNameParts]} = Req, Db, _DDoc) -> + db_attachment_req(Req, Db, <<"_design/", Name/binary>>, FileNameParts). -handle_design_info_req(#httpd{method='GET'}=Req, Db, #doc{} = DDoc) -> +handle_design_info_req(#httpd{method = 'GET'} = Req, Db, #doc{} = DDoc) -> [_, _, Name, _] = Req#httpd.path_parts, {ok, GroupInfoList} = fabric:get_view_group_info(Db, DDoc), - send_json(Req, 200, {[ - {name, Name}, - {view_index, {GroupInfoList}} - ]}); - + send_json( + Req, + 200, + {[ + {name, Name}, + {view_index, {GroupInfoList}} + ]} + ); handle_design_info_req(Req, _Db, _DDoc) -> send_method_not_allowed(Req, "GET"). -create_db_req(#httpd{}=Req, DbName) -> +create_db_req(#httpd{} = Req, DbName) -> couch_httpd:verify_is_server_admin(Req), ShardsOpt = parse_shards_opt(Req), EngineOpt = parse_engine_opt(Req), @@ -395,202 +430,237 @@ create_db_req(#httpd{}=Req, DbName) -> Options = lists:append([ShardsOpt, [{props, DbProps}], EngineOpt]), DocUrl = absolute_uri(Req, "/" ++ couch_util:url_encode(DbName)), case fabric:create_db(DbName, Options) of - ok -> - send_json(Req, 201, [{"Location", DocUrl}], {[{ok, true}]}); - accepted -> - send_json(Req, 202, [{"Location", DocUrl}], {[{ok, true}]}); - {error, file_exists} -> - chttpd:send_error(Req, file_exists); - Error -> - throw(Error) + ok -> + send_json(Req, 201, [{"Location", DocUrl}], {[{ok, true}]}); + accepted -> + send_json(Req, 202, [{"Location", DocUrl}], {[{ok, true}]}); + {error, file_exists} -> + chttpd:send_error(Req, file_exists); + Error -> + throw(Error) end. -delete_db_req(#httpd{}=Req, DbName) -> +delete_db_req(#httpd{} = Req, DbName) -> couch_httpd:verify_is_server_admin(Req), case fabric:delete_db(DbName, []) of - ok -> - send_json(Req, 200, {[{ok, true}]}); - accepted -> - send_json(Req, 202, {[{ok, true}]}); - Error -> - throw(Error) + ok -> + send_json(Req, 200, {[{ok, true}]}); + accepted -> + send_json(Req, 202, {[{ok, true}]}); + Error -> + throw(Error) end. -do_db_req(#httpd{path_parts=[DbName|_], user_ctx=Ctx}=Req, Fun) -> +do_db_req(#httpd{path_parts = [DbName | _], user_ctx = Ctx} = Req, Fun) -> Shard = hd(mem3:shards(DbName)), Props = couch_util:get_value(props, Shard#shard.opts, []), - Opts = case Ctx of - undefined -> - [{props, Props}]; - #user_ctx{} -> - [{user_ctx, Ctx}, {props, Props}] - end, + Opts = + case Ctx of + undefined -> + [{props, Props}]; + #user_ctx{} -> + [{user_ctx, Ctx}, {props, Props}] + end, {ok, Db} = couch_db:clustered_db(DbName, Opts), Fun(Req, Db). -db_req(#httpd{method='GET',path_parts=[DbName]}=Req, _Db) -> +db_req(#httpd{method = 'GET', path_parts = [DbName]} = Req, _Db) -> % measure the time required to generate the etag, see if it's worth it T0 = os:timestamp(), {ok, DbInfo} = fabric:get_db_info(DbName), DeltaT = timer:now_diff(os:timestamp(), T0) / 1000, couch_stats:update_histogram([couchdb, dbinfo], DeltaT), send_json(Req, {DbInfo}); - -db_req(#httpd{method='POST', path_parts=[DbName], user_ctx=Ctx}=Req, Db) -> +db_req(#httpd{method = 'POST', path_parts = [DbName], user_ctx = Ctx} = Req, Db) -> chttpd:validate_ctype(Req, "application/json"), W = chttpd:qs_value(Req, "w", integer_to_list(mem3:quorum(Db))), - Options = [{user_ctx,Ctx}, {w,W}], + Options = [{user_ctx, Ctx}, {w, W}], Doc = couch_db:doc_from_json_obj_validate(Db, chttpd:json_body(Req)), validate_attachment_names(Doc), - Doc2 = case Doc#doc.id of - <<"">> -> - Doc#doc{id=couch_uuids:new(), revs={0, []}}; - _ -> - Doc - end, + Doc2 = + case Doc#doc.id of + <<"">> -> + Doc#doc{id = couch_uuids:new(), revs = {0, []}}; + _ -> + Doc + end, DocId = Doc2#doc.id, case chttpd:qs_value(Req, "batch") of - "ok" -> - % async_batching - spawn(fun() -> - case catch(fabric:update_doc(Db, Doc2, Options)) of - {ok, _} -> - chttpd_stats:incr_writes(), - ok; - {accepted, _} -> - chttpd_stats:incr_writes(), - ok; - Error -> - couch_log:debug("Batch doc error (~s): ~p",[DocId, Error]) + "ok" -> + % async_batching + spawn(fun() -> + case catch (fabric:update_doc(Db, Doc2, Options)) of + {ok, _} -> + chttpd_stats:incr_writes(), + ok; + {accepted, _} -> + chttpd_stats:incr_writes(), + ok; + Error -> + couch_log:debug("Batch doc error (~s): ~p", [DocId, Error]) end end), - send_json(Req, 202, [], {[ - {ok, true}, - {id, DocId} - ]}); - _Normal -> - % normal - DocUrl = absolute_uri(Req, [$/, couch_util:url_encode(DbName), - $/, couch_util:url_encode(DocId)]), - case fabric:update_doc(Db, Doc2, Options) of - {ok, NewRev} -> - chttpd_stats:incr_writes(), - HttpCode = 201; - {accepted, NewRev} -> - chttpd_stats:incr_writes(), - HttpCode = 202 - end, - send_json(Req, HttpCode, [{"Location", DocUrl}], {[ - {ok, true}, - {id, DocId}, - {rev, couch_doc:rev_to_str(NewRev)} - ]}) + send_json( + Req, + 202, + [], + {[ + {ok, true}, + {id, DocId} + ]} + ); + _Normal -> + % normal + DocUrl = absolute_uri(Req, [ + $/, + couch_util:url_encode(DbName), + $/, + couch_util:url_encode(DocId) + ]), + case fabric:update_doc(Db, Doc2, Options) of + {ok, NewRev} -> + chttpd_stats:incr_writes(), + HttpCode = 201; + {accepted, NewRev} -> + chttpd_stats:incr_writes(), + HttpCode = 202 + end, + send_json( + Req, + HttpCode, + [{"Location", DocUrl}], + {[ + {ok, true}, + {id, DocId}, + {rev, couch_doc:rev_to_str(NewRev)} + ]} + ) end; - -db_req(#httpd{path_parts=[_DbName]}=Req, _Db) -> +db_req(#httpd{path_parts = [_DbName]} = Req, _Db) -> send_method_not_allowed(Req, "DELETE,GET,HEAD,POST"); - -db_req(#httpd{method='POST', path_parts=[DbName, <<"_ensure_full_commit">>], - user_ctx=Ctx}=Req, _Db) -> +db_req( + #httpd{ + method = 'POST', + path_parts = [DbName, <<"_ensure_full_commit">>], + user_ctx = Ctx + } = Req, + _Db +) -> chttpd:validate_ctype(Req, "application/json"), %% use fabric call to trigger a database_does_not_exist exception %% for missing databases that'd return error 404 from chttpd %% get_security used to prefer shards on the same node over other nodes fabric:get_security(DbName, [{user_ctx, Ctx}]), - send_json(Req, 201, {[ - {ok, true}, - {instance_start_time, <<"0">>} - ]}); - -db_req(#httpd{path_parts=[_,<<"_ensure_full_commit">>]}=Req, _Db) -> + send_json( + Req, + 201, + {[ + {ok, true}, + {instance_start_time, <<"0">>} + ]} + ); +db_req(#httpd{path_parts = [_, <<"_ensure_full_commit">>]} = Req, _Db) -> send_method_not_allowed(Req, "POST"); - -db_req(#httpd{method='POST',path_parts=[_,<<"_bulk_docs">>], user_ctx=Ctx}=Req, Db) -> +db_req(#httpd{method = 'POST', path_parts = [_, <<"_bulk_docs">>], user_ctx = Ctx} = Req, Db) -> couch_stats:increment_counter([couchdb, httpd, bulk_requests]), chttpd:validate_ctype(Req, "application/json"), {JsonProps} = chttpd:json_body_obj(Req), - DocsArray = case couch_util:get_value(<<"docs">>, JsonProps) of - undefined -> - throw({bad_request, <<"POST body must include `docs` parameter.">>}); - DocsArray0 when not is_list(DocsArray0) -> - throw({bad_request, <<"`docs` parameter must be an array.">>}); - DocsArray0 -> - DocsArray0 - end, + DocsArray = + case couch_util:get_value(<<"docs">>, JsonProps) of + undefined -> + throw({bad_request, <<"POST body must include `docs` parameter.">>}); + DocsArray0 when not is_list(DocsArray0) -> + throw({bad_request, <<"`docs` parameter must be an array.">>}); + DocsArray0 -> + DocsArray0 + end, couch_stats:update_histogram([couchdb, httpd, bulk_docs], length(DocsArray)), - W = case couch_util:get_value(<<"w">>, JsonProps) of - Value when is_integer(Value) -> - integer_to_list(Value); - _ -> - chttpd:qs_value(Req, "w", integer_to_list(mem3:quorum(Db))) - end, + W = + case couch_util:get_value(<<"w">>, JsonProps) of + Value when is_integer(Value) -> + integer_to_list(Value); + _ -> + chttpd:qs_value(Req, "w", integer_to_list(mem3:quorum(Db))) + end, case chttpd:header_value(Req, "X-Couch-Full-Commit") of - "true" -> - Options = [full_commit, {user_ctx,Ctx}, {w,W}]; - "false" -> - Options = [delay_commit, {user_ctx,Ctx}, {w,W}]; - _ -> - Options = [{user_ctx,Ctx}, {w,W}] + "true" -> + Options = [full_commit, {user_ctx, Ctx}, {w, W}]; + "false" -> + Options = [delay_commit, {user_ctx, Ctx}, {w, W}]; + _ -> + Options = [{user_ctx, Ctx}, {w, W}] end, NewEdits = couch_util:get_value(<<"new_edits">>, JsonProps, true), - Docs = lists:map(fun(JsonObj) -> - Doc = couch_db:doc_from_json_obj_validate(Db, JsonObj), - validate_revs(Doc, NewEdits), - validate_attachment_names(Doc), - case Doc#doc.id of - <<>> -> Doc#doc{id = couch_uuids:new()}; - _ -> Doc - end - end, DocsArray), - case NewEdits of - true -> - Options2 = - case couch_util:get_value(<<"all_or_nothing">>, JsonProps) of - true -> [all_or_nothing|Options]; - _ -> Options + Docs = lists:map( + fun(JsonObj) -> + Doc = couch_db:doc_from_json_obj_validate(Db, JsonObj), + validate_revs(Doc, NewEdits), + validate_attachment_names(Doc), + case Doc#doc.id of + <<>> -> Doc#doc{id = couch_uuids:new()}; + _ -> Doc + end end, - case fabric:update_docs(Db, Docs, Options2) of - {ok, Results} -> - % output the results - chttpd_stats:incr_writes(length(Results)), - DocResults = lists:zipwith(fun update_doc_result_to_json/2, - Docs, Results), - send_json(Req, 201, DocResults); - {accepted, Results} -> - % output the results - chttpd_stats:incr_writes(length(Results)), - DocResults = lists:zipwith(fun update_doc_result_to_json/2, - Docs, Results), - send_json(Req, 202, DocResults); - {aborted, Errors} -> - ErrorsJson = - lists:map(fun update_doc_result_to_json/1, Errors), - send_json(Req, 417, ErrorsJson) - end; - false -> - case fabric:update_docs(Db, Docs, [replicated_changes|Options]) of - {ok, Errors} -> - chttpd_stats:incr_writes(length(Docs)), - ErrorsJson = lists:map(fun update_doc_result_to_json/1, Errors), - send_json(Req, 201, ErrorsJson); - {accepted, Errors} -> - chttpd_stats:incr_writes(length(Docs)), - ErrorsJson = lists:map(fun update_doc_result_to_json/1, Errors), - send_json(Req, 202, ErrorsJson) - end; - _ -> - throw({bad_request, <<"`new_edits` parameter must be a boolean.">>}) + DocsArray + ), + case NewEdits of + true -> + Options2 = + case couch_util:get_value(<<"all_or_nothing">>, JsonProps) of + true -> [all_or_nothing | Options]; + _ -> Options + end, + case fabric:update_docs(Db, Docs, Options2) of + {ok, Results} -> + % output the results + chttpd_stats:incr_writes(length(Results)), + DocResults = lists:zipwith( + fun update_doc_result_to_json/2, + Docs, + Results + ), + send_json(Req, 201, DocResults); + {accepted, Results} -> + % output the results + chttpd_stats:incr_writes(length(Results)), + DocResults = lists:zipwith( + fun update_doc_result_to_json/2, + Docs, + Results + ), + send_json(Req, 202, DocResults); + {aborted, Errors} -> + ErrorsJson = + lists:map(fun update_doc_result_to_json/1, Errors), + send_json(Req, 417, ErrorsJson) + end; + false -> + case fabric:update_docs(Db, Docs, [replicated_changes | Options]) of + {ok, Errors} -> + chttpd_stats:incr_writes(length(Docs)), + ErrorsJson = lists:map(fun update_doc_result_to_json/1, Errors), + send_json(Req, 201, ErrorsJson); + {accepted, Errors} -> + chttpd_stats:incr_writes(length(Docs)), + ErrorsJson = lists:map(fun update_doc_result_to_json/1, Errors), + send_json(Req, 202, ErrorsJson) + end; + _ -> + throw({bad_request, <<"`new_edits` parameter must be a boolean.">>}) end; - -db_req(#httpd{path_parts=[_,<<"_bulk_docs">>]}=Req, _Db) -> +db_req(#httpd{path_parts = [_, <<"_bulk_docs">>]} = Req, _Db) -> send_method_not_allowed(Req, "POST"); - - -db_req(#httpd{method='POST', path_parts=[_, <<"_bulk_get">>], - mochi_req=MochiReq}=Req, Db) -> +db_req( + #httpd{ + method = 'POST', + path_parts = [_, <<"_bulk_get">>], + mochi_req = MochiReq + } = Req, + Db +) -> couch_stats:increment_counter([couchdb, httpd, bulk_requests]), couch_httpd:validate_ctype(Req, "application/json"), {JsonProps} = chttpd:json_body_obj(Req), @@ -603,7 +673,7 @@ db_req(#httpd{method='POST', path_parts=[_, <<"_bulk_get">>], } = bulk_get_parse_doc_query(Req), Options = [{user_ctx, Req#httpd.user_ctx} | Options0], - AcceptJson = MochiReq:accepts_content_type("application/json"), + AcceptJson = MochiReq:accepts_content_type("application/json"), AcceptMixedMp = MochiReq:accepts_content_type("multipart/mixed"), AcceptRelatedMp = MochiReq:accepts_content_type("multipart/related"), AcceptMp = not AcceptJson andalso (AcceptMixedMp orelse AcceptRelatedMp), @@ -611,60 +681,80 @@ db_req(#httpd{method='POST', path_parts=[_, <<"_bulk_get">>], false -> {ok, Resp} = start_json_response(Req, 200), send_chunk(Resp, <<"{\"results\": [">>), - lists:foldl(fun(Doc, Sep) -> - {DocId, Results, Options1} = bulk_get_open_doc_revs(Db, Doc, - Options), - bulk_get_send_docs_json(Resp, DocId, Results, Options1, Sep), - <<",">> - end, <<"">>, Docs), + lists:foldl( + fun(Doc, Sep) -> + {DocId, Results, Options1} = bulk_get_open_doc_revs( + Db, + Doc, + Options + ), + bulk_get_send_docs_json(Resp, DocId, Results, Options1, Sep), + <<",">> + end, + <<"">>, + Docs + ), send_chunk(Resp, <<"]}">>), end_json_response(Resp); true -> OuterBoundary = bulk_get_multipart_boundary(), - MpType = case AcceptMixedMp of - true -> - "multipart/mixed"; - _ -> - "multipart/related" - end, - CType = {"Content-Type", MpType ++ "; boundary=\"" ++ - ?b2l(OuterBoundary) ++ "\""}, + MpType = + case AcceptMixedMp of + true -> + "multipart/mixed"; + _ -> + "multipart/related" + end, + CType = + {"Content-Type", + MpType ++ "; boundary=\"" ++ + ?b2l(OuterBoundary) ++ "\""}, {ok, Resp} = start_chunked_response(Req, 200, [CType]), - lists:foldl(fun(Doc, _Pre) -> - case bulk_get_open_doc_revs(Db, Doc, Options) of - {_, {ok, []}, _Options1} -> - ok; - {_, {ok, Results}, Options1} -> - send_docs_multipart_bulk_get(Results, Options1, - OuterBoundary, Resp); - {DocId, {error, {RevId, Error, Reason}}, _Options1} -> - Json = ?JSON_ENCODE({[ - {<<"id">>, DocId}, - {<<"rev">>, RevId}, - {<<"error">>, Error}, - {<<"reason">>, Reason} - ]}), - couch_httpd:send_chunk(Resp,[ - <<"\r\n--", OuterBoundary/binary>>, - <<"\r\nContent-Type: application/json; error=\"true\"\r\n\r\n">>, - Json - ]) - end - end, <<"">>, Docs), + lists:foldl( + fun(Doc, _Pre) -> + case bulk_get_open_doc_revs(Db, Doc, Options) of + {_, {ok, []}, _Options1} -> + ok; + {_, {ok, Results}, Options1} -> + send_docs_multipart_bulk_get( + Results, + Options1, + OuterBoundary, + Resp + ); + {DocId, {error, {RevId, Error, Reason}}, _Options1} -> + Json = ?JSON_ENCODE( + {[ + {<<"id">>, DocId}, + {<<"rev">>, RevId}, + {<<"error">>, Error}, + {<<"reason">>, Reason} + ]} + ), + couch_httpd:send_chunk(Resp, [ + <<"\r\n--", OuterBoundary/binary>>, + <<"\r\nContent-Type: application/json; error=\"true\"\r\n\r\n">>, + Json + ]) + end + end, + <<"">>, + Docs + ), case Docs of [] -> ok; _ -> - couch_httpd:send_chunk(Resp, <<"\r\n", "--", OuterBoundary/binary, "--\r\n">>) + couch_httpd:send_chunk( + Resp, <<"\r\n", "--", OuterBoundary/binary, "--\r\n">> + ) end, couch_httpd:last_chunk(Resp) end end; -db_req(#httpd{path_parts=[_, <<"_bulk_get">>]}=Req, _Db) -> +db_req(#httpd{path_parts = [_, <<"_bulk_get">>]} = Req, _Db) -> send_method_not_allowed(Req, "POST"); - - -db_req(#httpd{method='POST',path_parts=[_,<<"_purge">>]}=Req, Db) -> +db_req(#httpd{method = 'POST', path_parts = [_, <<"_purge">>]} = Req, Db) -> couch_stats:increment_counter([couchdb, httpd, purge_requests]), chttpd:validate_ctype(Req, "application/json"), W = chttpd:qs_value(Req, "w", integer_to_list(mem3:quorum(Db))), @@ -676,88 +766,94 @@ db_req(#httpd{method='POST',path_parts=[_,<<"_purge">>]}=Req, Db) -> false -> throw({bad_request, "Exceeded maximum number of documents."}); true -> ok end, - RevsLen = lists:foldl(fun({_Id, Revs}, Acc) -> - length(Revs) + Acc - end, 0, IdsRevs2), + RevsLen = lists:foldl( + fun({_Id, Revs}, Acc) -> + length(Revs) + Acc + end, + 0, + IdsRevs2 + ), MaxRevs = config:get_integer("purge", "max_revisions_number", 1000), case RevsLen =< MaxRevs of false -> throw({bad_request, "Exceeded maximum number of revisions."}); true -> ok end, couch_stats:increment_counter([couchdb, document_purges, total], length(IdsRevs2)), - Results2 = case fabric:purge_docs(Db, IdsRevs2, Options) of - {ok, Results} -> - chttpd_stats:incr_writes(length(Results)), - Results; - {accepted, Results} -> - chttpd_stats:incr_writes(length(Results)), - Results - end, + Results2 = + case fabric:purge_docs(Db, IdsRevs2, Options) of + {ok, Results} -> + chttpd_stats:incr_writes(length(Results)), + Results; + {accepted, Results} -> + chttpd_stats:incr_writes(length(Results)), + Results + end, {Code, Json} = purge_results_to_json(IdsRevs2, Results2), send_json(Req, Code, {[{<<"purge_seq">>, null}, {<<"purged">>, {Json}}]}); - -db_req(#httpd{path_parts=[_,<<"_purge">>]}=Req, _Db) -> +db_req(#httpd{path_parts = [_, <<"_purge">>]} = Req, _Db) -> send_method_not_allowed(Req, "POST"); - - -db_req(#httpd{method='GET',path_parts=[_,OP]}=Req, Db) when ?IS_ALL_DOCS(OP) -> +db_req(#httpd{method = 'GET', path_parts = [_, OP]} = Req, Db) when ?IS_ALL_DOCS(OP) -> case chttpd:qs_json_value(Req, "keys", nil) of - Keys when is_list(Keys) -> - all_docs_view(Req, Db, Keys, OP); - nil -> - all_docs_view(Req, Db, undefined, OP); - _ -> - throw({bad_request, "`keys` parameter must be an array."}) + Keys when is_list(Keys) -> + all_docs_view(Req, Db, Keys, OP); + nil -> + all_docs_view(Req, Db, undefined, OP); + _ -> + throw({bad_request, "`keys` parameter must be an array."}) end; - -db_req(#httpd{method='POST', - path_parts=[_, OP, <<"queries">>]}=Req, Db) when ?IS_ALL_DOCS(OP) -> +db_req( + #httpd{ + method = 'POST', + path_parts = [_, OP, <<"queries">>] + } = Req, + Db +) when ?IS_ALL_DOCS(OP) -> Props = chttpd:json_body_obj(Req), case couch_mrview_util:get_view_queries(Props) of undefined -> - throw({bad_request, - <<"POST body must include `queries` parameter.">>}); + throw({bad_request, <<"POST body must include `queries` parameter.">>}); Queries -> multi_all_docs_view(Req, Db, OP, Queries) end; - -db_req(#httpd{path_parts=[_, OP, <<"queries">>]}=Req, - _Db) when ?IS_ALL_DOCS(OP) -> +db_req( + #httpd{path_parts = [_, OP, <<"queries">>]} = Req, + _Db +) when ?IS_ALL_DOCS(OP) -> send_method_not_allowed(Req, "POST"); - -db_req(#httpd{method='POST',path_parts=[_,OP]}=Req, Db) when ?IS_ALL_DOCS(OP) -> +db_req(#httpd{method = 'POST', path_parts = [_, OP]} = Req, Db) when ?IS_ALL_DOCS(OP) -> chttpd:validate_ctype(Req, "application/json"), {Fields} = chttpd:json_body_obj(Req), case couch_util:get_value(<<"keys">>, Fields, nil) of - Keys when is_list(Keys) -> - all_docs_view(Req, Db, Keys, OP); - nil -> - all_docs_view(Req, Db, undefined, OP); - _ -> - throw({bad_request, "`keys` body member must be an array."}) + Keys when is_list(Keys) -> + all_docs_view(Req, Db, Keys, OP); + nil -> + all_docs_view(Req, Db, undefined, OP); + _ -> + throw({bad_request, "`keys` body member must be an array."}) end; - -db_req(#httpd{path_parts=[_,OP]}=Req, _Db) when ?IS_ALL_DOCS(OP) -> +db_req(#httpd{path_parts = [_, OP]} = Req, _Db) when ?IS_ALL_DOCS(OP) -> send_method_not_allowed(Req, "GET,HEAD,POST"); - -db_req(#httpd{method='POST',path_parts=[_,<<"_missing_revs">>]}=Req, Db) -> +db_req(#httpd{method = 'POST', path_parts = [_, <<"_missing_revs">>]} = Req, Db) -> chttpd:validate_ctype(Req, "application/json"), {JsonDocIdRevs} = chttpd:json_body_obj(Req), case fabric:get_missing_revs(Db, JsonDocIdRevs) of {error, Reason} -> chttpd:send_error(Req, Reason); {ok, Results} -> - Results2 = [{Id, couch_doc:revs_to_strs(Revs)} || - {Id, Revs, _} <- Results], - send_json(Req, {[ - {missing_revs, {Results2}} - ]}) + Results2 = [ + {Id, couch_doc:revs_to_strs(Revs)} + || {Id, Revs, _} <- Results + ], + send_json( + Req, + {[ + {missing_revs, {Results2}} + ]} + ) end; - -db_req(#httpd{path_parts=[_,<<"_missing_revs">>]}=Req, _Db) -> +db_req(#httpd{path_parts = [_, <<"_missing_revs">>]} = Req, _Db) -> send_method_not_allowed(Req, "POST"); - -db_req(#httpd{method='POST',path_parts=[_,<<"_revs_diff">>]}=Req, Db) -> +db_req(#httpd{method = 'POST', path_parts = [_, <<"_revs_diff">>]} = Req, Db) -> chttpd:validate_ctype(Req, "application/json"), {JsonDocIdRevs} = chttpd:json_body_obj(Req), case fabric:get_missing_revs(Db, JsonDocIdRevs) of @@ -765,24 +861,31 @@ db_req(#httpd{method='POST',path_parts=[_,<<"_revs_diff">>]}=Req, Db) -> chttpd:send_error(Req, Reason); {ok, Results} -> Results2 = - lists:map(fun({Id, MissingRevs, PossibleAncestors}) -> - {Id, - {[{missing, couch_doc:revs_to_strs(MissingRevs)}] ++ - if PossibleAncestors == [] -> - []; - true -> - [{possible_ancestors, - couch_doc:revs_to_strs(PossibleAncestors)}] - end}} - end, Results), + lists:map( + fun({Id, MissingRevs, PossibleAncestors}) -> + {Id, { + [{missing, couch_doc:revs_to_strs(MissingRevs)}] ++ + if + PossibleAncestors == [] -> + []; + true -> + [ + {possible_ancestors, + couch_doc:revs_to_strs(PossibleAncestors)} + ] + end + }} + end, + Results + ), send_json(Req, {Results2}) end; - -db_req(#httpd{path_parts=[_,<<"_revs_diff">>]}=Req, _Db) -> +db_req(#httpd{path_parts = [_, <<"_revs_diff">>]} = Req, _Db) -> send_method_not_allowed(Req, "POST"); - -db_req(#httpd{method='PUT',path_parts=[_,<<"_security">>],user_ctx=Ctx}=Req, - Db) -> +db_req( + #httpd{method = 'PUT', path_parts = [_, <<"_security">>], user_ctx = Ctx} = Req, + Db +) -> DbName = ?b2l(couch_db:name(Db)), validate_security_can_be_edited(DbName), SecObj = chttpd:json_body(Req), @@ -792,26 +895,22 @@ db_req(#httpd{method='PUT',path_parts=[_,<<"_security">>],user_ctx=Ctx}=Req, Else -> throw(Else) end; - -db_req(#httpd{method='GET',path_parts=[_,<<"_security">>]}=Req, Db) -> +db_req(#httpd{method = 'GET', path_parts = [_, <<"_security">>]} = Req, Db) -> send_json(Req, fabric:get_security(Db)); - -db_req(#httpd{path_parts=[_,<<"_security">>]}=Req, _Db) -> +db_req(#httpd{path_parts = [_, <<"_security">>]} = Req, _Db) -> send_method_not_allowed(Req, "PUT,GET"); - -db_req(#httpd{method='PUT',path_parts=[_,<<"_revs_limit">>],user_ctx=Ctx}=Req, - Db) -> +db_req( + #httpd{method = 'PUT', path_parts = [_, <<"_revs_limit">>], user_ctx = Ctx} = Req, + Db +) -> Limit = chttpd:json_body(Req), - ok = fabric:set_revs_limit(Db, Limit, [{user_ctx,Ctx}]), + ok = fabric:set_revs_limit(Db, Limit, [{user_ctx, Ctx}]), send_json(Req, {[{<<"ok">>, true}]}); - -db_req(#httpd{method='GET',path_parts=[_,<<"_revs_limit">>]}=Req, Db) -> +db_req(#httpd{method = 'GET', path_parts = [_, <<"_revs_limit">>]} = Req, Db) -> send_json(Req, fabric:get_revs_limit(Db)); - -db_req(#httpd{path_parts=[_,<<"_revs_limit">>]}=Req, _Db) -> +db_req(#httpd{path_parts = [_, <<"_revs_limit">>]} = Req, _Db) -> send_method_not_allowed(Req, "PUT,GET"); - -db_req(#httpd{method='PUT',path_parts=[_,<<"_purged_infos_limit">>]}=Req, Db) -> +db_req(#httpd{method = 'PUT', path_parts = [_, <<"_purged_infos_limit">>]} = Req, Db) -> Options = [{user_ctx, Req#httpd.user_ctx}], case chttpd:json_body(Req) of Limit when is_integer(Limit), Limit > 0 -> @@ -821,81 +920,95 @@ db_req(#httpd{method='PUT',path_parts=[_,<<"_purged_infos_limit">>]}=Req, Db) -> Error -> throw(Error) end; - _-> + _ -> throw({bad_request, "`purge_infos_limit` must be positive integer"}) end; - -db_req(#httpd{method='GET',path_parts=[_,<<"_purged_infos_limit">>]}=Req, Db) -> +db_req(#httpd{method = 'GET', path_parts = [_, <<"_purged_infos_limit">>]} = Req, Db) -> send_json(Req, fabric:get_purge_infos_limit(Db)); - % Special case to enable using an unencoded slash in the URL of design docs, % as slashes in document IDs must otherwise be URL encoded. -db_req(#httpd{method='GET', mochi_req=MochiReq, path_parts=[_DbName, <<"_design/", _/binary>> | _]}=Req, _Db) -> +db_req( + #httpd{ + method = 'GET', mochi_req = MochiReq, path_parts = [_DbName, <<"_design/", _/binary>> | _] + } = Req, + _Db +) -> [Head | Tail] = re:split(MochiReq:get(raw_path), "_design%2F", [{return, list}, caseless]), chttpd:send_redirect(Req, Head ++ "_design/" ++ Tail); - -db_req(#httpd{path_parts=[_DbName,<<"_design">>,Name]}=Req, Db) -> - db_doc_req(Req, Db, <<"_design/",Name/binary>>); - -db_req(#httpd{path_parts=[_DbName,<<"_design">>,Name|FileNameParts]}=Req, Db) -> - db_attachment_req(Req, Db, <<"_design/",Name/binary>>, FileNameParts); - - +db_req(#httpd{path_parts = [_DbName, <<"_design">>, Name]} = Req, Db) -> + db_doc_req(Req, Db, <<"_design/", Name/binary>>); +db_req(#httpd{path_parts = [_DbName, <<"_design">>, Name | FileNameParts]} = Req, Db) -> + db_attachment_req(Req, Db, <<"_design/", Name/binary>>, FileNameParts); % Special case to allow for accessing local documents without %2F % encoding the docid. Throws out requests that don't have the second % path part or that specify an attachment name. -db_req(#httpd{path_parts=[_DbName, <<"_local">>]}, _Db) -> +db_req(#httpd{path_parts = [_DbName, <<"_local">>]}, _Db) -> throw({bad_request, <<"Invalid _local document id.">>}); - -db_req(#httpd{path_parts=[_DbName, <<"_local/">>]}, _Db) -> +db_req(#httpd{path_parts = [_DbName, <<"_local/">>]}, _Db) -> throw({bad_request, <<"Invalid _local document id.">>}); - -db_req(#httpd{path_parts=[_DbName, <<"_local">>, Name]}=Req, Db) -> +db_req(#httpd{path_parts = [_DbName, <<"_local">>, Name]} = Req, Db) -> db_doc_req(Req, Db, <<"_local/", Name/binary>>); - -db_req(#httpd{path_parts=[_DbName, <<"_local">> | _Rest]}, _Db) -> +db_req(#httpd{path_parts = [_DbName, <<"_local">> | _Rest]}, _Db) -> throw({bad_request, <<"_local documents do not accept attachments.">>}); - -db_req(#httpd{path_parts=[_, DocId]}=Req, Db) -> +db_req(#httpd{path_parts = [_, DocId]} = Req, Db) -> db_doc_req(Req, Db, DocId); - -db_req(#httpd{method='DELETE', path_parts=[_, DocId | FileNameParts]}=Req, Db) -> +db_req(#httpd{method = 'DELETE', path_parts = [_, DocId | FileNameParts]} = Req, Db) -> chttpd:body(Req), db_attachment_req(Req, Db, DocId, FileNameParts); -db_req(#httpd{path_parts=[_, DocId | FileNameParts]}=Req, Db) -> +db_req(#httpd{path_parts = [_, DocId | FileNameParts]} = Req, Db) -> db_attachment_req(Req, Db, DocId, FileNameParts). multi_all_docs_view(Req, Db, OP, Queries) -> Args0 = couch_mrview_http:parse_params(Req, undefined), - Args1 = Args0#mrargs{view_type=map}, - ArgQueries = lists:map(fun({Query}) -> - QueryArg1 = couch_mrview_http:parse_params(Query, undefined, - Args1, [decoded]), - QueryArgs2 = fabric_util:validate_all_docs_args(Db, QueryArg1), - set_namespace(OP, QueryArgs2) - end, Queries), + Args1 = Args0#mrargs{view_type = map}, + ArgQueries = lists:map( + fun({Query}) -> + QueryArg1 = couch_mrview_http:parse_params( + Query, + undefined, + Args1, + [decoded] + ), + QueryArgs2 = fabric_util:validate_all_docs_args(Db, QueryArg1), + set_namespace(OP, QueryArgs2) + end, + Queries + ), Options = [{user_ctx, Req#httpd.user_ctx}], - VAcc0 = #vacc{db=Db, req=Req, prepend="\r\n"}, + VAcc0 = #vacc{db = Db, req = Req, prepend = "\r\n"}, FirstChunk = "{\"results\":[", - {ok, Resp0} = chttpd:start_delayed_json_response(VAcc0#vacc.req, - 200, [], FirstChunk), - VAcc1 = VAcc0#vacc{resp=Resp0}, - VAcc2 = lists:foldl(fun(Args, Acc0) -> - {ok, Acc1} = fabric:all_docs(Db, Options, - fun view_cb/2, Acc0, Args), - Acc1 - end, VAcc1, ArgQueries), + {ok, Resp0} = chttpd:start_delayed_json_response( + VAcc0#vacc.req, + 200, + [], + FirstChunk + ), + VAcc1 = VAcc0#vacc{resp = Resp0}, + VAcc2 = lists:foldl( + fun(Args, Acc0) -> + {ok, Acc1} = fabric:all_docs( + Db, + Options, + fun view_cb/2, + Acc0, + Args + ), + Acc1 + end, + VAcc1, + ArgQueries + ), {ok, Resp1} = chttpd:send_delayed_chunk(VAcc2#vacc.resp, "\r\n]}"), chttpd:end_delayed_json_response(Resp1). all_docs_view(Req, Db, Keys, OP) -> Args0 = couch_mrview_http:parse_body_and_query(Req, Keys), - Args1 = Args0#mrargs{view_type=map}, + Args1 = Args0#mrargs{view_type = map}, Args2 = fabric_util:validate_all_docs_args(Db, Args1), Args3 = set_namespace(OP, Args2), Options = [{user_ctx, Req#httpd.user_ctx}], Max = chttpd:chunked_response_buffer_size(), - VAcc = #vacc{db=Db, req=Req, threshold=Max}, + VAcc = #vacc{db = Db, req = Req, threshold = Max}, {ok, Resp} = fabric:all_docs(Db, Options, fun view_cb/2, VAcc, Args3), {ok, Resp#vacc.resp}. @@ -906,23 +1019,21 @@ view_cb({row, Row} = Msg, Acc) -> end, chttpd_stats:incr_rows(), couch_mrview_http:view_cb(Msg, Acc); - view_cb(Msg, Acc) -> couch_mrview_http:view_cb(Msg, Acc). -db_doc_req(#httpd{method='DELETE'}=Req, Db, DocId) -> +db_doc_req(#httpd{method = 'DELETE'} = Req, Db, DocId) -> % check for the existence of the doc to handle the 404 case. couch_doc_open(Db, DocId, nil, []), case chttpd:qs_value(Req, "rev") of - undefined -> - Body = {[{<<"_deleted">>,true}]}; - Rev -> - Body = {[{<<"_rev">>, ?l2b(Rev)},{<<"_deleted">>,true}]} + undefined -> + Body = {[{<<"_deleted">>, true}]}; + Rev -> + Body = {[{<<"_rev">>, ?l2b(Rev)}, {<<"_deleted">>, true}]} end, Doc = couch_doc_from_req(Req, Db, DocId, Body), send_updated_doc(Req, Db, DocId, Doc); - -db_doc_req(#httpd{method='GET', mochi_req=MochiReq}=Req, Db, DocId) -> +db_doc_req(#httpd{method = 'GET', mochi_req = MochiReq} = Req, Db, DocId) -> #doc_query_args{ rev = Rev0, open_revs = Revs, @@ -931,116 +1042,130 @@ db_doc_req(#httpd{method='GET', mochi_req=MochiReq}=Req, Db, DocId) -> } = parse_doc_query(Req), Options = [{user_ctx, Req#httpd.user_ctx} | Options0], case Revs of - [] -> - Options2 = - if AttsSince /= nil -> - [{atts_since, AttsSince}, attachments | Options]; - true -> Options - end, - Rev = case lists:member(latest, Options) of - % couch_doc_open will open the winning rev despite of a rev passed - % https://docs.couchdb.org/en/stable/api/document/common.html?highlight=latest#get--db-docid - true -> nil; - false -> Rev0 - end, - Doc = couch_doc_open(Db, DocId, Rev, Options2), - send_doc(Req, Doc, Options2); - _ -> - case fabric:open_revs(Db, DocId, Revs, Options) of - {ok, []} when Revs == all -> - chttpd:send_error(Req, {not_found, missing}); - {ok, Results} -> - chttpd_stats:incr_reads(length(Results)), - case MochiReq:accepts_content_type("multipart/mixed") of - false -> - {ok, Resp} = start_json_response(Req, 200), - send_chunk(Resp, "["), - % We loop through the docs. The first time through the separator - % is whitespace, then a comma on subsequent iterations. - lists:foldl( - fun(Result, AccSeparator) -> - case Result of - {ok, Doc} -> - JsonDoc = couch_doc:to_json_obj(Doc, Options), - Json = ?JSON_ENCODE({[{ok, JsonDoc}]}), - send_chunk(Resp, AccSeparator ++ Json); - {{not_found, missing}, RevId} -> - RevStr = couch_doc:rev_to_str(RevId), - Json = ?JSON_ENCODE({[{<<"missing">>, RevStr}]}), - send_chunk(Resp, AccSeparator ++ Json) - end, - "," % AccSeparator now has a comma - end, - "", Results), - send_chunk(Resp, "]"), - end_json_response(Resp); - true -> - send_docs_multipart(Req, Results, Options) - end; - {error, Error} -> - chttpd:send_error(Req, Error) - end + [] -> + Options2 = + if + AttsSince /= nil -> + [{atts_since, AttsSince}, attachments | Options]; + true -> + Options + end, + Rev = + case lists:member(latest, Options) of + % couch_doc_open will open the winning rev despite of a rev passed + % https://docs.couchdb.org/en/stable/api/document/common.html?highlight=latest#get--db-docid + true -> nil; + false -> Rev0 + end, + Doc = couch_doc_open(Db, DocId, Rev, Options2), + send_doc(Req, Doc, Options2); + _ -> + case fabric:open_revs(Db, DocId, Revs, Options) of + {ok, []} when Revs == all -> + chttpd:send_error(Req, {not_found, missing}); + {ok, Results} -> + chttpd_stats:incr_reads(length(Results)), + case MochiReq:accepts_content_type("multipart/mixed") of + false -> + {ok, Resp} = start_json_response(Req, 200), + send_chunk(Resp, "["), + % We loop through the docs. The first time through the separator + % is whitespace, then a comma on subsequent iterations. + lists:foldl( + fun(Result, AccSeparator) -> + case Result of + {ok, Doc} -> + JsonDoc = couch_doc:to_json_obj(Doc, Options), + Json = ?JSON_ENCODE({[{ok, JsonDoc}]}), + send_chunk(Resp, AccSeparator ++ Json); + {{not_found, missing}, RevId} -> + RevStr = couch_doc:rev_to_str(RevId), + Json = ?JSON_ENCODE({[{<<"missing">>, RevStr}]}), + send_chunk(Resp, AccSeparator ++ Json) + end, + % AccSeparator now has a comma + "," + end, + "", + Results + ), + send_chunk(Resp, "]"), + end_json_response(Resp); + true -> + send_docs_multipart(Req, Results, Options) + end; + {error, Error} -> + chttpd:send_error(Req, Error) + end end; - -db_doc_req(#httpd{method='POST', user_ctx=Ctx}=Req, Db, DocId) -> +db_doc_req(#httpd{method = 'POST', user_ctx = Ctx} = Req, Db, DocId) -> couch_httpd:validate_referer(Req), couch_db:validate_docid(Db, DocId), chttpd:validate_ctype(Req, "multipart/form-data"), W = chttpd:qs_value(Req, "w", integer_to_list(mem3:quorum(Db))), - Options = [{user_ctx,Ctx}, {w,W}], + Options = [{user_ctx, Ctx}, {w, W}], Form = couch_httpd:parse_form(Req), case proplists:is_defined("_doc", Form) of - true -> - Json = ?JSON_DECODE(couch_util:get_value("_doc", Form)), - Doc = couch_doc_from_req(Req, Db, DocId, Json); - false -> - Rev = couch_doc:parse_rev(list_to_binary(couch_util:get_value("_rev", Form))), - Doc = case fabric:open_revs(Db, DocId, [Rev], []) of - {ok, [{ok, Doc0}]} -> - chttpd_stats:incr_reads(), - Doc0; - {error, Error} -> - throw(Error) - end + true -> + Json = ?JSON_DECODE(couch_util:get_value("_doc", Form)), + Doc = couch_doc_from_req(Req, Db, DocId, Json); + false -> + Rev = couch_doc:parse_rev(list_to_binary(couch_util:get_value("_rev", Form))), + Doc = + case fabric:open_revs(Db, DocId, [Rev], []) of + {ok, [{ok, Doc0}]} -> + chttpd_stats:incr_reads(), + Doc0; + {error, Error} -> + throw(Error) + end end, UpdatedAtts = [ couch_att:new([ {name, validate_attachment_name(Name)}, {type, list_to_binary(ContentType)}, {data, Content} - ]) || - {Name, {ContentType, _}, Content} <- - proplists:get_all_values("_attachments", Form) + ]) + || {Name, {ContentType, _}, Content} <- + proplists:get_all_values("_attachments", Form) ], - #doc{atts=OldAtts} = Doc, + #doc{atts = OldAtts} = Doc, OldAtts2 = lists:flatmap( fun(Att) -> OldName = couch_att:fetch(name, Att), case [1 || A <- UpdatedAtts, couch_att:fetch(name, A) == OldName] of - [] -> [Att]; % the attachment wasn't in the UpdatedAtts, return it - _ -> [] % the attachment was in the UpdatedAtts, drop it + % the attachment wasn't in the UpdatedAtts, return it + [] -> [Att]; + % the attachment was in the UpdatedAtts, drop it + _ -> [] end - end, OldAtts), + end, + OldAtts + ), NewDoc = Doc#doc{ atts = UpdatedAtts ++ OldAtts2 }, case fabric:update_doc(Db, NewDoc, Options) of - {ok, NewRev} -> - chttpd_stats:incr_writes(), - HttpCode = 201; - {accepted, NewRev} -> - chttpd_stats:incr_writes(), - HttpCode = 202 + {ok, NewRev} -> + chttpd_stats:incr_writes(), + HttpCode = 201; + {accepted, NewRev} -> + chttpd_stats:incr_writes(), + HttpCode = 202 end, - send_json(Req, HttpCode, [{"ETag", "\"" ++ ?b2l(couch_doc:rev_to_str(NewRev)) ++ "\""}], {[ - {ok, true}, - {id, DocId}, - {rev, couch_doc:rev_to_str(NewRev)} - ]}); - -db_doc_req(#httpd{method='PUT', user_ctx=Ctx}=Req, Db, DocId) -> + send_json( + Req, + HttpCode, + [{"ETag", "\"" ++ ?b2l(couch_doc:rev_to_str(NewRev)) ++ "\""}], + {[ + {ok, true}, + {id, DocId}, + {rev, couch_doc:rev_to_str(NewRev)} + ]} + ); +db_doc_req(#httpd{method = 'PUT', user_ctx = Ctx} = Req, Db, DocId) -> #doc_query_args{ update_type = UpdateType } = parse_doc_query(Req), @@ -1048,194 +1173,257 @@ db_doc_req(#httpd{method='PUT', user_ctx=Ctx}=Req, Db, DocId) -> couch_db:validate_docid(Db, DocId), W = chttpd:qs_value(Req, "w", integer_to_list(mem3:quorum(Db))), - Options = [{user_ctx,Ctx}, {w,W}], - - Loc = absolute_uri(Req, [$/, couch_util:url_encode(DbName), - $/, couch_util:url_encode(DocId)]), + Options = [{user_ctx, Ctx}, {w, W}], + + Loc = absolute_uri(Req, [ + $/, + couch_util:url_encode(DbName), + $/, + couch_util:url_encode(DocId) + ]), RespHeaders = [{"Location", Loc}], case couch_util:to_list(couch_httpd:header_value(Req, "Content-Type")) of - ("multipart/related;" ++ _) = ContentType -> - couch_httpd:check_max_request_length(Req), - couch_httpd_multipart:num_mp_writers(mem3:n(mem3:dbname(DbName), DocId)), - {ok, Doc0, WaitFun, Parser} = couch_doc:doc_from_multi_part_stream(ContentType, - fun() -> receive_request_data(Req) end), - Doc = couch_doc_from_req(Req, Db, DocId, Doc0), - try - Result = send_updated_doc(Req, Db, DocId, Doc, RespHeaders, UpdateType), - WaitFun(), - Result - catch throw:Err -> - % Document rejected by a validate_doc_update function. - couch_httpd_multipart:abort_multipart_stream(Parser), - throw(Err) - end; - _Else -> - case chttpd:qs_value(Req, "batch") of - "ok" -> - % batch - Doc = couch_doc_from_req(Req, Db, DocId, chttpd:json_body(Req)), - - spawn(fun() -> - case catch(fabric:update_doc(Db, Doc, Options)) of - {ok, _} -> - chttpd_stats:incr_writes(), - ok; - {accepted, _} -> - chttpd_stats:incr_writes(), - ok; - Error -> - couch_log:notice("Batch doc error (~s): ~p",[DocId, Error]) - end - end), - send_json(Req, 202, [], {[ - {ok, true}, - {id, DocId} - ]}); - _Normal -> - % normal - Body = chttpd:json_body(Req), - Doc = couch_doc_from_req(Req, Db, DocId, Body), - send_updated_doc(Req, Db, DocId, Doc, RespHeaders, UpdateType) - end + ("multipart/related;" ++ _) = ContentType -> + couch_httpd:check_max_request_length(Req), + couch_httpd_multipart:num_mp_writers(mem3:n(mem3:dbname(DbName), DocId)), + {ok, Doc0, WaitFun, Parser} = couch_doc:doc_from_multi_part_stream( + ContentType, + fun() -> receive_request_data(Req) end + ), + Doc = couch_doc_from_req(Req, Db, DocId, Doc0), + try + Result = send_updated_doc(Req, Db, DocId, Doc, RespHeaders, UpdateType), + WaitFun(), + Result + catch + throw:Err -> + % Document rejected by a validate_doc_update function. + couch_httpd_multipart:abort_multipart_stream(Parser), + throw(Err) + end; + _Else -> + case chttpd:qs_value(Req, "batch") of + "ok" -> + % batch + Doc = couch_doc_from_req(Req, Db, DocId, chttpd:json_body(Req)), + + spawn(fun() -> + case catch (fabric:update_doc(Db, Doc, Options)) of + {ok, _} -> + chttpd_stats:incr_writes(), + ok; + {accepted, _} -> + chttpd_stats:incr_writes(), + ok; + Error -> + couch_log:notice("Batch doc error (~s): ~p", [DocId, Error]) + end + end), + send_json( + Req, + 202, + [], + {[ + {ok, true}, + {id, DocId} + ]} + ); + _Normal -> + % normal + Body = chttpd:json_body(Req), + Doc = couch_doc_from_req(Req, Db, DocId, Body), + send_updated_doc(Req, Db, DocId, Doc, RespHeaders, UpdateType) + end end; - -db_doc_req(#httpd{method='COPY', user_ctx=Ctx}=Req, Db, SourceDocId) -> +db_doc_req(#httpd{method = 'COPY', user_ctx = Ctx} = Req, Db, SourceDocId) -> SourceRev = - case extract_header_rev(Req, chttpd:qs_value(Req, "rev")) of - missing_rev -> nil; - Rev -> Rev - end, + case extract_header_rev(Req, chttpd:qs_value(Req, "rev")) of + missing_rev -> nil; + Rev -> Rev + end, {TargetDocId0, TargetRevs} = couch_httpd_db:parse_copy_destination_header(Req), TargetDocId = list_to_binary(chttpd:unquote(TargetDocId0)), % open old doc Doc = couch_doc_open(Db, SourceDocId, SourceRev, []), % save new doc - case fabric:update_doc(Db, - Doc#doc{id=TargetDocId, revs=TargetRevs}, [{user_ctx,Ctx}]) of - {ok, NewTargetRev} -> - chttpd_stats:incr_writes(), - HttpCode = 201; - {accepted, NewTargetRev} -> - chttpd_stats:incr_writes(), - HttpCode = 202 + case + fabric:update_doc( + Db, + Doc#doc{id = TargetDocId, revs = TargetRevs}, + [{user_ctx, Ctx}] + ) + of + {ok, NewTargetRev} -> + chttpd_stats:incr_writes(), + HttpCode = 201; + {accepted, NewTargetRev} -> + chttpd_stats:incr_writes(), + HttpCode = 202 end, % respond DbName = couch_db:name(Db), {PartRes} = update_doc_result_to_json(TargetDocId, {ok, NewTargetRev}), - Loc = absolute_uri(Req, "/" ++ couch_util:url_encode(DbName) ++ "/" ++ couch_util:url_encode(TargetDocId)), - send_json(Req, HttpCode, - [{"Location", Loc}, - {"ETag", "\"" ++ ?b2l(couch_doc:rev_to_str(NewTargetRev)) ++ "\""}], - {PartRes}); - + Loc = absolute_uri( + Req, "/" ++ couch_util:url_encode(DbName) ++ "/" ++ couch_util:url_encode(TargetDocId) + ), + send_json( + Req, + HttpCode, + [ + {"Location", Loc}, + {"ETag", "\"" ++ ?b2l(couch_doc:rev_to_str(NewTargetRev)) ++ "\""} + ], + {PartRes} + ); db_doc_req(Req, _Db, _DocId) -> send_method_not_allowed(Req, "DELETE,GET,HEAD,POST,PUT,COPY"). send_doc(Req, Doc, Options) -> case Doc#doc.meta of - [] -> - DiskEtag = couch_httpd:doc_etag(Doc), - % output etag only when we have no meta - chttpd:etag_respond(Req, DiskEtag, fun() -> - send_doc_efficiently(Req, Doc, [{"ETag", DiskEtag}], Options) - end); - _ -> - send_doc_efficiently(Req, Doc, [], Options) + [] -> + DiskEtag = couch_httpd:doc_etag(Doc), + % output etag only when we have no meta + chttpd:etag_respond(Req, DiskEtag, fun() -> + send_doc_efficiently(Req, Doc, [{"ETag", DiskEtag}], Options) + end); + _ -> + send_doc_efficiently(Req, Doc, [], Options) end. -send_doc_efficiently(Req, #doc{atts=[]}=Doc, Headers, Options) -> - send_json(Req, 200, Headers, couch_doc:to_json_obj(Doc, Options)); -send_doc_efficiently(#httpd{mochi_req=MochiReq}=Req, #doc{atts=Atts}=Doc, Headers, Options) -> +send_doc_efficiently(Req, #doc{atts = []} = Doc, Headers, Options) -> + send_json(Req, 200, Headers, couch_doc:to_json_obj(Doc, Options)); +send_doc_efficiently(#httpd{mochi_req = MochiReq} = Req, #doc{atts = Atts} = Doc, Headers, Options) -> case lists:member(attachments, Options) of - true -> - Refs = monitor_attachments(Atts), - try - case MochiReq:accepts_content_type("multipart/related") of - false -> - send_json(Req, 200, Headers, couch_doc:to_json_obj(Doc, Options)); true -> - Boundary = couch_uuids:random(), - JsonBytes = ?JSON_ENCODE(couch_doc:to_json_obj(Doc, - [attachments, follows, att_encoding_info | Options])), - {ContentType, Len} = couch_doc:len_doc_to_multi_part_stream( - Boundary,JsonBytes, Atts, true), - CType = {"Content-Type", ContentType}, - {ok, Resp} = start_response_length(Req, 200, [CType|Headers], Len), - couch_doc:doc_to_multi_part_stream(Boundary,JsonBytes,Atts, - fun(Data) -> couch_httpd:send(Resp, Data) end, true) - end - after - demonitor_refs(Refs) - end; - false -> - send_json(Req, 200, Headers, couch_doc:to_json_obj(Doc, Options)) + Refs = monitor_attachments(Atts), + try + case MochiReq:accepts_content_type("multipart/related") of + false -> + send_json(Req, 200, Headers, couch_doc:to_json_obj(Doc, Options)); + true -> + Boundary = couch_uuids:random(), + JsonBytes = ?JSON_ENCODE( + couch_doc:to_json_obj( + Doc, + [attachments, follows, att_encoding_info | Options] + ) + ), + {ContentType, Len} = couch_doc:len_doc_to_multi_part_stream( + Boundary, JsonBytes, Atts, true + ), + CType = {"Content-Type", ContentType}, + {ok, Resp} = start_response_length(Req, 200, [CType | Headers], Len), + couch_doc:doc_to_multi_part_stream( + Boundary, + JsonBytes, + Atts, + fun(Data) -> couch_httpd:send(Resp, Data) end, + true + ) + end + after + demonitor_refs(Refs) + end; + false -> + send_json(Req, 200, Headers, couch_doc:to_json_obj(Doc, Options)) end. send_docs_multipart_bulk_get(Results, Options0, OuterBoundary, Resp) -> InnerBoundary = bulk_get_multipart_boundary(), Options = [attachments, follows, att_encoding_info | Options0], lists:foreach( - fun({ok, #doc{id=Id, revs=Revs, atts=Atts}=Doc}) -> - Refs = monitor_attachments(Doc#doc.atts), - try - JsonBytes = ?JSON_ENCODE(couch_doc:to_json_obj(Doc, Options)), - couch_httpd:send_chunk(Resp, <<"\r\n--", OuterBoundary/binary>>), - case Atts of - [] -> - couch_httpd:send_chunk(Resp, <<"\r\nContent-Type: application/json\r\n\r\n">>); - _ -> - lists:foreach(fun(Header) -> couch_httpd:send_chunk(Resp, Header) end, - bulk_get_multipart_headers(Revs, Id, InnerBoundary)) - end, - couch_doc:doc_to_multi_part_stream(InnerBoundary, JsonBytes, Atts, - fun(Data) -> couch_httpd:send_chunk(Resp, Data) - end, true) - after - demonitor_refs(Refs) - end; - ({{not_found, missing}, RevId}) -> - RevStr = couch_doc:rev_to_str(RevId), - Json = ?JSON_ENCODE({[{<<"rev">>, RevStr}, - {<<"error">>, <<"not_found">>}, - {<<"reason">>, <<"missing">>}]}), - couch_httpd:send_chunk(Resp, - [<<"\r\n--", OuterBoundary/binary>>, - <<"\r\nContent-Type: application/json; error=\"true\"\r\n\r\n">>, - Json]) - end, Results). + fun + ({ok, #doc{id = Id, revs = Revs, atts = Atts} = Doc}) -> + Refs = monitor_attachments(Doc#doc.atts), + try + JsonBytes = ?JSON_ENCODE(couch_doc:to_json_obj(Doc, Options)), + couch_httpd:send_chunk(Resp, <<"\r\n--", OuterBoundary/binary>>), + case Atts of + [] -> + couch_httpd:send_chunk( + Resp, <<"\r\nContent-Type: application/json\r\n\r\n">> + ); + _ -> + lists:foreach( + fun(Header) -> couch_httpd:send_chunk(Resp, Header) end, + bulk_get_multipart_headers(Revs, Id, InnerBoundary) + ) + end, + couch_doc:doc_to_multi_part_stream( + InnerBoundary, + JsonBytes, + Atts, + fun(Data) -> couch_httpd:send_chunk(Resp, Data) end, + true + ) + after + demonitor_refs(Refs) + end; + ({{not_found, missing}, RevId}) -> + RevStr = couch_doc:rev_to_str(RevId), + Json = ?JSON_ENCODE( + {[ + {<<"rev">>, RevStr}, + {<<"error">>, <<"not_found">>}, + {<<"reason">>, <<"missing">>} + ]} + ), + couch_httpd:send_chunk( + Resp, + [ + <<"\r\n--", OuterBoundary/binary>>, + <<"\r\nContent-Type: application/json; error=\"true\"\r\n\r\n">>, + Json + ] + ) + end, + Results + ). send_docs_multipart(Req, Results, Options1) -> OuterBoundary = couch_uuids:random(), InnerBoundary = couch_uuids:random(), Options = [attachments, follows, att_encoding_info | Options1], - CType = {"Content-Type", - "multipart/mixed; boundary=\"" ++ ?b2l(OuterBoundary) ++ "\""}, + CType = {"Content-Type", "multipart/mixed; boundary=\"" ++ ?b2l(OuterBoundary) ++ "\""}, {ok, Resp} = start_chunked_response(Req, 200, [CType]), couch_httpd:send_chunk(Resp, <<"--", OuterBoundary/binary>>), lists:foreach( - fun({ok, #doc{atts=Atts}=Doc}) -> - Refs = monitor_attachments(Doc#doc.atts), - try - JsonBytes = ?JSON_ENCODE(couch_doc:to_json_obj(Doc, Options)), - {ContentType, _Len} = couch_doc:len_doc_to_multi_part_stream( - InnerBoundary, JsonBytes, Atts, true), - couch_httpd:send_chunk(Resp, <<"\r\nContent-Type: ", - ContentType/binary, "\r\n\r\n">>), - couch_doc:doc_to_multi_part_stream(InnerBoundary, JsonBytes, Atts, - fun(Data) -> couch_httpd:send_chunk(Resp, Data) - end, true), - couch_httpd:send_chunk(Resp, <<"\r\n--", OuterBoundary/binary>>) - after - demonitor_refs(Refs) - end; - ({{not_found, missing}, RevId}) -> - RevStr = couch_doc:rev_to_str(RevId), - Json = ?JSON_ENCODE({[{<<"missing">>, RevStr}]}), - couch_httpd:send_chunk(Resp, - [<<"\r\nContent-Type: application/json; error=\"true\"\r\n\r\n">>, - Json, - <<"\r\n--", OuterBoundary/binary>>]) - end, Results), + fun + ({ok, #doc{atts = Atts} = Doc}) -> + Refs = monitor_attachments(Doc#doc.atts), + try + JsonBytes = ?JSON_ENCODE(couch_doc:to_json_obj(Doc, Options)), + {ContentType, _Len} = couch_doc:len_doc_to_multi_part_stream( + InnerBoundary, JsonBytes, Atts, true + ), + couch_httpd:send_chunk( + Resp, <<"\r\nContent-Type: ", ContentType/binary, "\r\n\r\n">> + ), + couch_doc:doc_to_multi_part_stream( + InnerBoundary, + JsonBytes, + Atts, + fun(Data) -> couch_httpd:send_chunk(Resp, Data) end, + true + ), + couch_httpd:send_chunk(Resp, <<"\r\n--", OuterBoundary/binary>>) + after + demonitor_refs(Refs) + end; + ({{not_found, missing}, RevId}) -> + RevStr = couch_doc:rev_to_str(RevId), + Json = ?JSON_ENCODE({[{<<"missing">>, RevStr}]}), + couch_httpd:send_chunk( + Resp, + [ + <<"\r\nContent-Type: application/json; error=\"true\"\r\n\r\n">>, + Json, + <<"\r\n--", OuterBoundary/binary>> + ] + ) + end, + Results + ), couch_httpd:send_chunk(Resp, <<"--">>), couch_httpd:last_chunk(Resp). @@ -1244,7 +1432,7 @@ bulk_get_multipart_headers({0, []}, Id, Boundary) -> <<"\r\nX-Doc-Id: ", Id/binary>>, <<"\r\nContent-Type: multipart/related; boundary=", Boundary/binary, "\r\n\r\n">> ]; -bulk_get_multipart_headers({Start, [FirstRevId|_]}, Id, Boundary) -> +bulk_get_multipart_headers({Start, [FirstRevId | _]}, Id, Boundary) -> RevStr = couch_doc:rev_to_str({Start, FirstRevId}), [ <<"\r\nX-Doc-Id: ", Id/binary>>, @@ -1265,9 +1453,17 @@ receive_request_data(Req, Len) when Len == chunked -> self() ! {chunk, Ref, Binary} end, couch_httpd:recv_chunked(Req, 4096, ChunkFun, ok), - GetChunk = fun GC() -> receive {chunk, Ref, Binary} -> {Binary, GC} end end, - {receive {chunk, Ref, Binary} -> Binary end, GetChunk}; - + GetChunk = fun GC() -> + receive + {chunk, Ref, Binary} -> {Binary, GC} + end + end, + { + receive + {chunk, Ref, Binary} -> Binary + end, + GetChunk + }; receive_request_data(Req, LenLeft) when LenLeft > 0 -> Len = erlang:min(4096, LenLeft), Data = chttpd:recv(Req, Len), @@ -1276,11 +1472,15 @@ receive_request_data(_Req, _) -> throw(<<"expected more data">>). update_doc_result_to_json({{Id, Rev}, Error}) -> - {_Code, Err, Msg} = chttpd:error_info(Error), - {[{id, Id}, {rev, couch_doc:rev_to_str(Rev)}, - {error, Err}, {reason, Msg}]}. - -update_doc_result_to_json(#doc{id=DocId}, Result) -> + {_Code, Err, Msg} = chttpd:error_info(Error), + {[ + {id, Id}, + {rev, couch_doc:rev_to_str(Rev)}, + {error, Err}, + {reason, Msg} + ]}. + +update_doc_result_to_json(#doc{id = DocId}, Result) -> update_doc_result_to_json(DocId, Result); update_doc_result_to_json(DocId, {ok, NewRev}) -> {[{ok, true}, {id, DocId}, {rev, couch_doc:rev_to_str(NewRev)}]}; @@ -1314,20 +1514,30 @@ send_updated_doc(Req, Db, DocId, Json) -> send_updated_doc(Req, Db, DocId, Doc, Headers) -> send_updated_doc(Req, Db, DocId, Doc, Headers, interactive_edit). -send_updated_doc(#httpd{user_ctx=Ctx} = Req, Db, DocId, #doc{deleted=Deleted}=Doc, - Headers, UpdateType) -> +send_updated_doc( + #httpd{user_ctx = Ctx} = Req, + Db, + DocId, + #doc{deleted = Deleted} = Doc, + Headers, + UpdateType +) -> W = chttpd:qs_value(Req, "w", integer_to_list(mem3:quorum(Db))), Options = case couch_httpd:header_value(Req, "X-Couch-Full-Commit") of - "true" -> - [full_commit, UpdateType, {user_ctx,Ctx}, {w,W}]; - "false" -> - [delay_commit, UpdateType, {user_ctx,Ctx}, {w,W}]; - _ -> - [UpdateType, {user_ctx,Ctx}, {w,W}] + "true" -> + [full_commit, UpdateType, {user_ctx, Ctx}, {w, W}]; + "false" -> + [delay_commit, UpdateType, {user_ctx, Ctx}, {w, W}]; + _ -> + [UpdateType, {user_ctx, Ctx}, {w, W}] end, - {Status, {etag, Etag}, Body} = update_doc(Db, DocId, - #doc{deleted=Deleted}=Doc, Options), + {Status, {etag, Etag}, Body} = update_doc( + Db, + DocId, + #doc{deleted = Deleted} = Doc, + Options + ), HttpCode = http_code_from_status(Status), ResponseHeaders = [{"ETag", Etag} | Headers], send_json(Req, HttpCode, ResponseHeaders, Body). @@ -1342,7 +1552,7 @@ http_code_from_status(Status) -> 200 end. -update_doc(Db, DocId, #doc{deleted=Deleted, body=DocBody}=Doc, Options) -> +update_doc(Db, DocId, #doc{deleted = Deleted, body = DocBody} = Doc, Options) -> {_, Ref} = spawn_monitor(fun() -> try fabric:update_doc(Db, Doc, Options) of Resp -> @@ -1356,68 +1566,74 @@ update_doc(Db, DocId, #doc{deleted=Deleted, body=DocBody}=Doc, Options) -> exit({exit_exit, Reason}) end end), - Result = receive - {'DOWN', Ref, _, _, {exit_ok, Ret}} -> - Ret; - {'DOWN', Ref, _, _, {exit_throw, Reason}} -> - throw(Reason); - {'DOWN', Ref, _, _, {exit_error, Reason}} -> - erlang:error(Reason); - {'DOWN', Ref, _, _, {exit_exit, Reason}} -> - erlang:exit(Reason) - end, + Result = + receive + {'DOWN', Ref, _, _, {exit_ok, Ret}} -> + Ret; + {'DOWN', Ref, _, _, {exit_throw, Reason}} -> + throw(Reason); + {'DOWN', Ref, _, _, {exit_error, Reason}} -> + erlang:error(Reason); + {'DOWN', Ref, _, _, {exit_exit, Reason}} -> + erlang:exit(Reason) + end, case Result of - {ok, NewRev} -> - Accepted = false; - {accepted, NewRev} -> - Accepted = true + {ok, NewRev} -> + Accepted = false; + {accepted, NewRev} -> + Accepted = true end, Etag = couch_httpd:doc_etag(DocId, DocBody, NewRev), - Status = case {Accepted, Deleted} of - {true, _} -> - accepted; - {false, true} -> - ok; - {false, false} -> - created - end, + Status = + case {Accepted, Deleted} of + {true, _} -> + accepted; + {false, true} -> + ok; + {false, false} -> + created + end, NewRevStr = couch_doc:rev_to_str(NewRev), Body = {[{ok, true}, {id, DocId}, {rev, NewRevStr}]}, {Status, {etag, Etag}, Body}. -couch_doc_from_req(Req, _Db, DocId, #doc{revs=Revs} = Doc) -> +couch_doc_from_req(Req, _Db, DocId, #doc{revs = Revs} = Doc) -> validate_attachment_names(Doc), - Rev = case chttpd:qs_value(Req, "rev") of - undefined -> - undefined; - QSRev -> - couch_doc:parse_rev(QSRev) - end, + Rev = + case chttpd:qs_value(Req, "rev") of + undefined -> + undefined; + QSRev -> + couch_doc:parse_rev(QSRev) + end, Revs2 = - case Revs of - {Start, [RevId|_]} -> - if Rev /= undefined andalso Rev /= {Start, RevId} -> - throw({bad_request, "Document rev from request body and query " - "string have different values"}); - true -> - case extract_header_rev(Req, {Start, RevId}) of - missing_rev -> {0, []}; - _ -> Revs - end - end; - _ -> - case extract_header_rev(Req, Rev) of - missing_rev -> {0, []}; - {Pos, RevId2} -> {Pos, [RevId2]} - end - end, - Doc#doc{id=DocId, revs=Revs2}; + case Revs of + {Start, [RevId | _]} -> + if + Rev /= undefined andalso Rev /= {Start, RevId} -> + throw( + {bad_request, + "Document rev from request body and query " + "string have different values"} + ); + true -> + case extract_header_rev(Req, {Start, RevId}) of + missing_rev -> {0, []}; + _ -> Revs + end + end; + _ -> + case extract_header_rev(Req, Rev) of + missing_rev -> {0, []}; + {Pos, RevId2} -> {Pos, [RevId2]} + end + end, + Doc#doc{id = DocId, revs = Revs2}; couch_doc_from_req(Req, Db, DocId, Json) -> Doc = couch_db:doc_from_json_obj_validate(Db, Json), couch_doc_from_req(Req, Db, DocId, Doc). - % Useful for debugging % couch_doc_open(Db, DocId) -> % couch_doc_open(Db, DocId, nil, []). @@ -1425,28 +1641,29 @@ couch_doc_from_req(Req, Db, DocId, Json) -> couch_doc_open(Db, DocId, Rev, Options0) -> Options = [{user_ctx, couch_db:get_user_ctx(Db)} | Options0], case Rev of - nil -> % open most recent rev - case fabric:open_doc(Db, DocId, Options) of - {ok, Doc} -> - chttpd_stats:incr_reads(), - Doc; - Error -> - throw(Error) - end; - _ -> % open a specific rev (deletions come back as stubs) - case fabric:open_revs(Db, DocId, [Rev], Options) of - {ok, [{ok, Doc}]} -> - chttpd_stats:incr_reads(), - Doc; - {ok, [{{not_found, missing}, Rev}]} -> - throw(not_found); - {ok, [Else]} -> - throw(Else); - {error, Error} -> - throw(Error) - end - end. - + % open most recent rev + nil -> + case fabric:open_doc(Db, DocId, Options) of + {ok, Doc} -> + chttpd_stats:incr_reads(), + Doc; + Error -> + throw(Error) + end; + % open a specific rev (deletions come back as stubs) + _ -> + case fabric:open_revs(Db, DocId, [Rev], Options) of + {ok, [{ok, Doc}]} -> + chttpd_stats:incr_reads(), + Doc; + {ok, [{{not_found, missing}, Rev}]} -> + throw(not_found); + {ok, [Else]} -> + throw(Else); + {error, Error} -> + throw(Error) + end + end. get_existing_attachment(Atts, FileName) -> % Check if attachment exists, if not throw not_found @@ -1457,228 +1674,298 @@ get_existing_attachment(Atts, FileName) -> % Attachment request handlers -db_attachment_req(#httpd{method='GET',mochi_req=MochiReq}=Req, Db, DocId, FileNameParts) -> - FileName = list_to_binary(mochiweb_util:join(lists:map(fun binary_to_list/1, - FileNameParts),"/")), +db_attachment_req(#httpd{method = 'GET', mochi_req = MochiReq} = Req, Db, DocId, FileNameParts) -> + FileName = list_to_binary( + mochiweb_util:join( + lists:map( + fun binary_to_list/1, + FileNameParts + ), + "/" + ) + ), #doc_query_args{ - rev=Rev, - options=Options + rev = Rev, + options = Options } = parse_doc_query(Req), #doc{ - atts=Atts + atts = Atts } = Doc = couch_doc_open(Db, DocId, Rev, Options), Att = get_existing_attachment(Atts, FileName), - [Type, Enc, DiskLen, AttLen, Md5] = couch_att:fetch([type, encoding, disk_len, att_len, md5], Att), + [Type, Enc, DiskLen, AttLen, Md5] = couch_att:fetch( + [type, encoding, disk_len, att_len, md5], Att + ), Refs = monitor_attachments(Att), try - Etag = case Md5 of - <<>> -> chttpd:doc_etag(Doc); - _ -> "\"" ++ ?b2l(base64:encode(Md5)) ++ "\"" - end, - ReqAcceptsAttEnc = lists:member( - atom_to_list(Enc), - couch_httpd:accepted_encodings(Req) - ), - Headers0 = [ - {"ETag", Etag}, - {"Cache-Control", "must-revalidate"}, - {"Content-Type", binary_to_list(Type)} - ] ++ case ReqAcceptsAttEnc of - true when Enc =/= identity -> - % RFC 2616 says that the 'identify' encoding should not be used in - % the Content-Encoding header - [{"Content-Encoding", atom_to_list(Enc)}]; - _ -> - [] - end ++ case Enc of - identity -> - [{"Accept-Ranges", "bytes"}]; - _ -> - [{"Accept-Ranges", "none"}] - end, - Headers = chttpd_util:maybe_add_csp_header("attachments", Headers0, "sandbox"), - Len = case {Enc, ReqAcceptsAttEnc} of - {identity, _} -> - % stored and served in identity form - DiskLen; - {_, false} when DiskLen =/= AttLen -> - % Stored encoded, but client doesn't accept the encoding we used, - % so we need to decode on the fly. DiskLen is the identity length - % of the attachment. - DiskLen; - {_, true} -> - % Stored and served encoded. AttLen is the encoded length. - AttLen; - _ -> - % We received an encoded attachment and stored it as such, so we - % don't know the identity length. The client doesn't accept the - % encoding, and since we cannot serve a correct Content-Length - % header we'll fall back to a chunked response. - undefined - end, - AttFun = case ReqAcceptsAttEnc of - false -> - fun couch_att:foldl_decode/3; - true -> - fun couch_att:foldl/3 - end, - chttpd:etag_respond( - Req, - Etag, - fun() -> - case Len of - undefined -> - {ok, Resp} = start_chunked_response(Req, 200, Headers), - AttFun(Att, fun(Seg, _) -> send_chunk(Resp, Seg) end, {ok, Resp}), - couch_httpd:last_chunk(Resp); - _ -> - Ranges = parse_ranges(MochiReq:get(range), Len), - case {Enc, Ranges} of - {identity, [{From, To}]} -> - Headers1 = [{"Content-Range", make_content_range(From, To, Len)}] - ++ Headers, - {ok, Resp} = start_response_length(Req, 206, Headers1, To - From + 1), - couch_att:range_foldl(Att, From, To + 1, - fun(Seg, _) -> send(Resp, Seg) end, {ok, Resp}); - {identity, Ranges} when is_list(Ranges) andalso length(Ranges) < 10 -> - send_ranges_multipart(Req, Type, Len, Att, Ranges); + Etag = + case Md5 of + <<>> -> chttpd:doc_etag(Doc); + _ -> "\"" ++ ?b2l(base64:encode(Md5)) ++ "\"" + end, + ReqAcceptsAttEnc = lists:member( + atom_to_list(Enc), + couch_httpd:accepted_encodings(Req) + ), + Headers0 = + [ + {"ETag", Etag}, + {"Cache-Control", "must-revalidate"}, + {"Content-Type", binary_to_list(Type)} + ] ++ + case ReqAcceptsAttEnc of + true when Enc =/= identity -> + % RFC 2616 says that the 'identify' encoding should not be used in + % the Content-Encoding header + [{"Content-Encoding", atom_to_list(Enc)}]; _ -> - Headers1 = Headers ++ - if Enc =:= identity orelse ReqAcceptsAttEnc =:= true -> - [{"Content-MD5", base64:encode(couch_att:fetch(md5, Att))}]; - true -> - [] - end, - {ok, Resp} = start_response_length(Req, 200, Headers1, Len), - AttFun(Att, fun(Seg, _) -> send(Resp, Seg) end, {ok, Resp}) + [] + end ++ + case Enc of + identity -> + [{"Accept-Ranges", "bytes"}]; + _ -> + [{"Accept-Ranges", "none"}] + end, + Headers = chttpd_util:maybe_add_csp_header("attachments", Headers0, "sandbox"), + Len = + case {Enc, ReqAcceptsAttEnc} of + {identity, _} -> + % stored and served in identity form + DiskLen; + {_, false} when DiskLen =/= AttLen -> + % Stored encoded, but client doesn't accept the encoding we used, + % so we need to decode on the fly. DiskLen is the identity length + % of the attachment. + DiskLen; + {_, true} -> + % Stored and served encoded. AttLen is the encoded length. + AttLen; + _ -> + % We received an encoded attachment and stored it as such, so we + % don't know the identity length. The client doesn't accept the + % encoding, and since we cannot serve a correct Content-Length + % header we'll fall back to a chunked response. + undefined + end, + AttFun = + case ReqAcceptsAttEnc of + false -> + fun couch_att:foldl_decode/3; + true -> + fun couch_att:foldl/3 + end, + chttpd:etag_respond( + Req, + Etag, + fun() -> + case Len of + undefined -> + {ok, Resp} = start_chunked_response(Req, 200, Headers), + AttFun(Att, fun(Seg, _) -> send_chunk(Resp, Seg) end, {ok, Resp}), + couch_httpd:last_chunk(Resp); + _ -> + Ranges = parse_ranges(MochiReq:get(range), Len), + case {Enc, Ranges} of + {identity, [{From, To}]} -> + Headers1 = + [{"Content-Range", make_content_range(From, To, Len)}] ++ + Headers, + {ok, Resp} = start_response_length( + Req, 206, Headers1, To - From + 1 + ), + couch_att:range_foldl( + Att, + From, + To + 1, + fun(Seg, _) -> send(Resp, Seg) end, + {ok, Resp} + ); + {identity, Ranges} when is_list(Ranges) andalso length(Ranges) < 10 -> + send_ranges_multipart(Req, Type, Len, Att, Ranges); + _ -> + Headers1 = + Headers ++ + if + Enc =:= identity orelse ReqAcceptsAttEnc =:= true -> + [ + {"Content-MD5", + base64:encode(couch_att:fetch(md5, Att))} + ]; + true -> + [] + end, + {ok, Resp} = start_response_length(Req, 200, Headers1, Len), + AttFun(Att, fun(Seg, _) -> send(Resp, Seg) end, {ok, Resp}) + end end end - end - ) + ) after demonitor_refs(Refs) end; - - -db_attachment_req(#httpd{method=Method, user_ctx=Ctx}=Req, Db, DocId, FileNameParts) - when (Method == 'PUT') or (Method == 'DELETE') -> +db_attachment_req(#httpd{method = Method, user_ctx = Ctx} = Req, Db, DocId, FileNameParts) when + (Method == 'PUT') or (Method == 'DELETE') +-> FileName = validate_attachment_name( - mochiweb_util:join( - lists:map(fun binary_to_list/1, - FileNameParts),"/")), + mochiweb_util:join( + lists:map( + fun binary_to_list/1, + FileNameParts + ), + "/" + ) + ), - NewAtt = case Method of - 'DELETE' -> - []; - _ -> - MimeType = case couch_httpd:header_value(Req,"Content-Type") of - % We could throw an error here or guess by the FileName. - % Currently, just giving it a default. - undefined -> <<"application/octet-stream">>; - CType -> list_to_binary(CType) - end, - Data = fabric:att_receiver(Req, couch_db:name(Db), chttpd:body_length(Req)), - ContentLen = case couch_httpd:header_value(Req,"Content-Length") of - undefined -> undefined; - Length -> list_to_integer(Length) - end, - ContentEnc = string:to_lower(string:strip( - couch_httpd:header_value(Req, "Content-Encoding", "identity") - )), - Encoding = case ContentEnc of - "identity" -> - identity; - "gzip" -> - gzip; - _ -> - throw({ - bad_ctype, - "Only gzip and identity content-encodings are supported" - }) - end, - [couch_att:new([ - {name, FileName}, - {type, MimeType}, - {data, Data}, - {att_len, ContentLen}, - {md5, get_md5_header(Req)}, - {encoding, Encoding} - ])] - end, + NewAtt = + case Method of + 'DELETE' -> + []; + _ -> + MimeType = + case couch_httpd:header_value(Req, "Content-Type") of + % We could throw an error here or guess by the FileName. + % Currently, just giving it a default. + undefined -> <<"application/octet-stream">>; + CType -> list_to_binary(CType) + end, + Data = fabric:att_receiver(Req, couch_db:name(Db), chttpd:body_length(Req)), + ContentLen = + case couch_httpd:header_value(Req, "Content-Length") of + undefined -> undefined; + Length -> list_to_integer(Length) + end, + ContentEnc = string:to_lower( + string:strip( + couch_httpd:header_value(Req, "Content-Encoding", "identity") + ) + ), + Encoding = + case ContentEnc of + "identity" -> + identity; + "gzip" -> + gzip; + _ -> + throw({ + bad_ctype, + "Only gzip and identity content-encodings are supported" + }) + end, + [ + couch_att:new([ + {name, FileName}, + {type, MimeType}, + {data, Data}, + {att_len, ContentLen}, + {md5, get_md5_header(Req)}, + {encoding, Encoding} + ]) + ] + end, - Doc = case extract_header_rev(Req, chttpd:qs_value(Req, "rev")) of - missing_rev -> % make the new doc - if Method =/= 'DELETE' -> ok; true -> - % check for the existence of the doc and attachment - CurrDoc = #doc{} = couch_doc_open(Db, DocId, nil, []), - get_existing_attachment(CurrDoc#doc.atts, FileName) - end, - couch_db:validate_docid(Db, DocId), - #doc{id=DocId}; - Rev -> - case fabric:open_revs(Db, DocId, [Rev], [{user_ctx,Ctx}]) of - {ok, [{ok, Doc0}]} -> - chttpd_stats:incr_reads(), - if Method =/= 'DELETE' -> ok; true -> - % check if attachment exists - get_existing_attachment(Doc0#doc.atts, FileName) + Doc = + case extract_header_rev(Req, chttpd:qs_value(Req, "rev")) of + % make the new doc + missing_rev -> + if + Method =/= 'DELETE' -> + ok; + true -> + % check for the existence of the doc and attachment + CurrDoc = #doc{} = couch_doc_open(Db, DocId, nil, []), + get_existing_attachment(CurrDoc#doc.atts, FileName) end, - Doc0; - {ok, [Error]} -> - throw(Error); - {error, Error} -> - throw(Error) - end - end, + couch_db:validate_docid(Db, DocId), + #doc{id = DocId}; + Rev -> + case fabric:open_revs(Db, DocId, [Rev], [{user_ctx, Ctx}]) of + {ok, [{ok, Doc0}]} -> + chttpd_stats:incr_reads(), + if + Method =/= 'DELETE' -> + ok; + true -> + % check if attachment exists + get_existing_attachment(Doc0#doc.atts, FileName) + end, + Doc0; + {ok, [Error]} -> + throw(Error); + {error, Error} -> + throw(Error) + end + end, - #doc{atts=Atts} = Doc, + #doc{atts = Atts} = Doc, DocEdited = Doc#doc{ atts = NewAtt ++ [A || A <- Atts, couch_att:fetch(name, A) /= FileName] }, W = chttpd:qs_value(Req, "w", integer_to_list(mem3:quorum(Db))), - case fabric:update_doc(Db, DocEdited, [{user_ctx,Ctx}, {w,W}]) of - {ok, UpdatedRev} -> - chttpd_stats:incr_writes(), - HttpCode = 201; - {accepted, UpdatedRev} -> - chttpd_stats:incr_writes(), - HttpCode = 202 + case fabric:update_doc(Db, DocEdited, [{user_ctx, Ctx}, {w, W}]) of + {ok, UpdatedRev} -> + chttpd_stats:incr_writes(), + HttpCode = 201; + {accepted, UpdatedRev} -> + chttpd_stats:incr_writes(), + HttpCode = 202 end, erlang:put(mochiweb_request_recv, true), DbName = couch_db:name(Db), - {Status, Headers} = case Method of - 'DELETE' -> - {200, []}; - _ -> - {HttpCode, [{"Location", absolute_uri(Req, [$/, DbName, $/, couch_util:url_encode(DocId), $/, - couch_util:url_encode(FileName)])}]} + {Status, Headers} = + case Method of + 'DELETE' -> + {200, []}; + _ -> + {HttpCode, [ + {"Location", + absolute_uri(Req, [ + $/, + DbName, + $/, + couch_util:url_encode(DocId), + $/, + couch_util:url_encode(FileName) + ])} + ]} end, - send_json(Req,Status, Headers, {[ - {ok, true}, - {id, DocId}, - {rev, couch_doc:rev_to_str(UpdatedRev)} - ]}); - + send_json( + Req, + Status, + Headers, + {[ + {ok, true}, + {id, DocId}, + {rev, couch_doc:rev_to_str(UpdatedRev)} + ]} + ); db_attachment_req(Req, _Db, _DocId, _FileNameParts) -> send_method_not_allowed(Req, "DELETE,GET,HEAD,PUT"). send_ranges_multipart(Req, ContentType, Len, Att, Ranges) -> Boundary = couch_uuids:random(), - CType = {"Content-Type", - "multipart/byteranges; boundary=\"" ++ ?b2l(Boundary) ++ "\""}, + CType = {"Content-Type", "multipart/byteranges; boundary=\"" ++ ?b2l(Boundary) ++ "\""}, {ok, Resp} = start_chunked_response(Req, 206, [CType]), couch_httpd:send_chunk(Resp, <<"--", Boundary/binary>>), - lists:foreach(fun({From, To}) -> - ContentRange = make_content_range(From, To, Len), - couch_httpd:send_chunk(Resp, - <<"\r\nContent-Type: ", ContentType/binary, "\r\n", - "Content-Range: ", ContentRange/binary, "\r\n", - "\r\n">>), - couch_att:range_foldl(Att, From, To + 1, - fun(Seg, _) -> send_chunk(Resp, Seg) end, {ok, Resp}), - couch_httpd:send_chunk(Resp, <<"\r\n--", Boundary/binary>>) - end, Ranges), + lists:foreach( + fun({From, To}) -> + ContentRange = make_content_range(From, To, Len), + couch_httpd:send_chunk( + Resp, + <<"\r\nContent-Type: ", ContentType/binary, "\r\n", "Content-Range: ", + ContentRange/binary, "\r\n", "\r\n">> + ), + couch_att:range_foldl( + Att, + From, + To + 1, + fun(Seg, _) -> send_chunk(Resp, Seg) end, + {ok, Resp} + ), + couch_httpd:send_chunk(Resp, <<"\r\n--", Boundary/binary>>) + end, + Ranges + ), couch_httpd:send_chunk(Resp, <<"--">>), couch_httpd:last_chunk(Resp), {ok, Resp}. @@ -1692,18 +1979,21 @@ parse_ranges(Ranges, Len) -> parse_ranges([], _Len, Acc) -> lists:reverse(Acc); -parse_ranges([{0, none}|_], _Len, _Acc) -> +parse_ranges([{0, none} | _], _Len, _Acc) -> undefined; -parse_ranges([{From, To}|_], _Len, _Acc) when is_integer(From) andalso is_integer(To) andalso To < From -> +parse_ranges([{From, To} | _], _Len, _Acc) when + is_integer(From) andalso is_integer(To) andalso To < From +-> throw(requested_range_not_satisfiable); -parse_ranges([{From, To}|Rest], Len, Acc) - when is_integer(To) andalso To >= Len -> - parse_ranges([{From, Len-1}] ++ Rest, Len, Acc); -parse_ranges([{none, To}|Rest], Len, Acc) -> +parse_ranges([{From, To} | Rest], Len, Acc) when + is_integer(To) andalso To >= Len +-> + parse_ranges([{From, Len - 1}] ++ Rest, Len, Acc); +parse_ranges([{none, To} | Rest], Len, Acc) -> parse_ranges([{Len - To, Len - 1}] ++ Rest, Len, Acc); -parse_ranges([{From, none}|Rest], Len, Acc) -> +parse_ranges([{From, none} | Rest], Len, Acc) -> parse_ranges([{From, Len - 1}] ++ Rest, Len, Acc); -parse_ranges([{From,To}|Rest], Len, Acc) -> +parse_ranges([{From, To} | Rest], Len, Acc) -> parse_ranges(Rest, Len, [{From, To}] ++ Acc). make_content_range(From, To, Len) -> @@ -1736,27 +2026,34 @@ parse_shards_opt(Req) -> [ {n, parse_shards_opt("n", Req, config:get_integer("cluster", "n", 3))}, {q, parse_shards_opt("q", Req, config:get_integer("cluster", "q", 2))}, - {placement, parse_shards_opt( - "placement", Req, config:get("cluster", "placement"))} + {placement, + parse_shards_opt( + "placement", Req, config:get("cluster", "placement") + )} ]. parse_shards_opt("placement", Req, Default) -> Err = <<"The `placement` value should be in a format `zone:n`.">>, case chttpd:qs_value(Req, "placement", Default) of - Default -> Default; - [] -> throw({bad_request, Err}); + Default -> + Default; + [] -> + throw({bad_request, Err}); Val -> try - true = lists:all(fun(Rule) -> - [_, N] = string:tokens(Rule, ":"), - couch_util:validate_positive_int(N) - end, string:tokens(Val, ",")), + true = lists:all( + fun(Rule) -> + [_, N] = string:tokens(Rule, ":"), + couch_util:validate_positive_int(N) + end, + string:tokens(Val, ",") + ), Val - catch _:_ -> - throw({bad_request, Err}) + catch + _:_ -> + throw({bad_request, Err}) end end; - parse_shards_opt(Param, Req, Default) -> Val = chttpd:qs_value(Req, Param, Default), Err = ?l2b(["The `", Param, "` value should be a positive integer."]), @@ -1765,7 +2062,6 @@ parse_shards_opt(Param, Req, Default) -> false -> throw({bad_request, Err}) end. - parse_engine_opt(Req) -> case chttpd:qs_value(Req, "engine") of undefined -> @@ -1780,7 +2076,6 @@ parse_engine_opt(Req) -> end end. - parse_partitioned_opt(Req) -> case chttpd:qs_value(Req, "partitioned") of undefined -> @@ -1797,130 +2092,142 @@ parse_partitioned_opt(Req) -> throw({bad_request, <<"Invalid `partitioned` parameter">>}) end. - validate_partitioned_db_enabled(Req) -> case couch_flags:is_enabled(partitioned, Req) of - true -> + true -> ok; false -> throw({bad_request, <<"Partitioned feature is not enabled.">>}) end. - parse_doc_query({Key, Value}, Args) -> case {Key, Value} of {"attachments", "true"} -> Options = [attachments | Args#doc_query_args.options], - Args#doc_query_args{options=Options}; + Args#doc_query_args{options = Options}; {"meta", "true"} -> Options = [revs_info, conflicts, deleted_conflicts | Args#doc_query_args.options], - Args#doc_query_args{options=Options}; + Args#doc_query_args{options = Options}; {"revs", "true"} -> Options = [revs | Args#doc_query_args.options], - Args#doc_query_args{options=Options}; + Args#doc_query_args{options = Options}; {"local_seq", "true"} -> Options = [local_seq | Args#doc_query_args.options], - Args#doc_query_args{options=Options}; + Args#doc_query_args{options = Options}; {"revs_info", "true"} -> Options = [revs_info | Args#doc_query_args.options], - Args#doc_query_args{options=Options}; + Args#doc_query_args{options = Options}; {"conflicts", "true"} -> Options = [conflicts | Args#doc_query_args.options], - Args#doc_query_args{options=Options}; + Args#doc_query_args{options = Options}; {"deleted", "true"} -> Options = [deleted | Args#doc_query_args.options], - Args#doc_query_args{options=Options}; + Args#doc_query_args{options = Options}; {"deleted_conflicts", "true"} -> Options = [deleted_conflicts | Args#doc_query_args.options], - Args#doc_query_args{options=Options}; + Args#doc_query_args{options = Options}; {"rev", Rev} -> - Args#doc_query_args{rev=couch_doc:parse_rev(Rev)}; + Args#doc_query_args{rev = couch_doc:parse_rev(Rev)}; {"open_revs", "all"} -> - Args#doc_query_args{open_revs=all}; + Args#doc_query_args{open_revs = all}; {"open_revs", RevsJsonStr} -> JsonArray = ?JSON_DECODE(RevsJsonStr), - Args#doc_query_args{open_revs=couch_doc:parse_revs(JsonArray)}; + Args#doc_query_args{open_revs = couch_doc:parse_revs(JsonArray)}; {"latest", "true"} -> Options = [latest | Args#doc_query_args.options], - Args#doc_query_args{options=Options}; + Args#doc_query_args{options = Options}; {"atts_since", RevsJsonStr} -> JsonArray = ?JSON_DECODE(RevsJsonStr), Args#doc_query_args{atts_since = couch_doc:parse_revs(JsonArray)}; {"new_edits", "false"} -> - Args#doc_query_args{update_type=replicated_changes}; + Args#doc_query_args{update_type = replicated_changes}; {"new_edits", "true"} -> - Args#doc_query_args{update_type=interactive_edit}; + Args#doc_query_args{update_type = interactive_edit}; {"att_encoding_info", "true"} -> Options = [att_encoding_info | Args#doc_query_args.options], - Args#doc_query_args{options=Options}; + Args#doc_query_args{options = Options}; {"r", R} -> - Options = [{r,R} | Args#doc_query_args.options], - Args#doc_query_args{options=Options}; + Options = [{r, R} | Args#doc_query_args.options], + Args#doc_query_args{options = Options}; {"w", W} -> - Options = [{w,W} | Args#doc_query_args.options], - Args#doc_query_args{options=Options}; - _Else -> % unknown key value pair, ignore. + Options = [{w, W} | Args#doc_query_args.options], + Args#doc_query_args{options = Options}; + % unknown key value pair, ignore. + _Else -> Args end. parse_changes_query(Req) -> erlang:erase(changes_seq_interval), - ChangesArgs = lists:foldl(fun({Key, Value}, Args) -> - case {string:to_lower(Key), Value} of - {"feed", "live"} -> - %% sugar for continuous - Args#changes_args{feed="continuous"}; - {"feed", _} -> - Args#changes_args{feed=Value}; - {"descending", "true"} -> - Args#changes_args{dir=rev}; - {"since", _} -> - Args#changes_args{since=Value}; - {"last-event-id", _} -> - Args#changes_args{since=Value}; - {"limit", _} -> - Args#changes_args{limit=list_to_integer(Value)}; - {"style", _} -> - Args#changes_args{style=list_to_existing_atom(Value)}; - {"heartbeat", "true"} -> - Args#changes_args{heartbeat=true}; - {"heartbeat", _} -> - try list_to_integer(Value) of - HeartbeatInteger when HeartbeatInteger > 0 -> - Args#changes_args{heartbeat=HeartbeatInteger}; - _ -> - throw({bad_request, <<"The heartbeat value should be a positive integer (in milliseconds).">>}) - catch error:badarg -> - throw({bad_request, <<"Invalid heartbeat value. Expecting a positive integer value (in milliseconds).">>}) - end; - {"timeout", _} -> - Args#changes_args{timeout=list_to_integer(Value)}; - {"include_docs", "true"} -> - Args#changes_args{include_docs=true}; - {"conflicts", "true"} -> - Args#changes_args{conflicts=true}; - {"attachments", "true"} -> - Options = [attachments | Args#changes_args.doc_options], - Args#changes_args{doc_options=Options}; - {"att_encoding_info", "true"} -> - Options = [att_encoding_info | Args#changes_args.doc_options], - Args#changes_args{doc_options=Options}; - {"filter", _} -> - Args#changes_args{filter=Value}; - {"seq_interval", _} -> - try list_to_integer(Value) of - V when V > 0 -> - erlang:put(changes_seq_interval, V), - Args; - _ -> - throw({bad_request, invalid_seq_interval}) - catch error:badarg -> - throw({bad_request, invalid_seq_interval}) - end; - _Else -> % unknown key value pair, ignore. - Args - end - end, #changes_args{}, chttpd:qs(Req)), + ChangesArgs = lists:foldl( + fun({Key, Value}, Args) -> + case {string:to_lower(Key), Value} of + {"feed", "live"} -> + %% sugar for continuous + Args#changes_args{feed = "continuous"}; + {"feed", _} -> + Args#changes_args{feed = Value}; + {"descending", "true"} -> + Args#changes_args{dir = rev}; + {"since", _} -> + Args#changes_args{since = Value}; + {"last-event-id", _} -> + Args#changes_args{since = Value}; + {"limit", _} -> + Args#changes_args{limit = list_to_integer(Value)}; + {"style", _} -> + Args#changes_args{style = list_to_existing_atom(Value)}; + {"heartbeat", "true"} -> + Args#changes_args{heartbeat = true}; + {"heartbeat", _} -> + try list_to_integer(Value) of + HeartbeatInteger when HeartbeatInteger > 0 -> + Args#changes_args{heartbeat = HeartbeatInteger}; + _ -> + throw( + {bad_request, + <<"The heartbeat value should be a positive integer (in milliseconds).">>} + ) + catch + error:badarg -> + throw( + {bad_request, + <<"Invalid heartbeat value. Expecting a positive integer value (in milliseconds).">>} + ) + end; + {"timeout", _} -> + Args#changes_args{timeout = list_to_integer(Value)}; + {"include_docs", "true"} -> + Args#changes_args{include_docs = true}; + {"conflicts", "true"} -> + Args#changes_args{conflicts = true}; + {"attachments", "true"} -> + Options = [attachments | Args#changes_args.doc_options], + Args#changes_args{doc_options = Options}; + {"att_encoding_info", "true"} -> + Options = [att_encoding_info | Args#changes_args.doc_options], + Args#changes_args{doc_options = Options}; + {"filter", _} -> + Args#changes_args{filter = Value}; + {"seq_interval", _} -> + try list_to_integer(Value) of + V when V > 0 -> + erlang:put(changes_seq_interval, V), + Args; + _ -> + throw({bad_request, invalid_seq_interval}) + catch + error:badarg -> + throw({bad_request, invalid_seq_interval}) + end; + % unknown key value pair, ignore. + _Else -> + Args + end + end, + #changes_args{}, + chttpd:qs(Req) + ), %% if it's an EventSource request with a Last-event-ID header %% that should override the `since` query string, since it's %% probably the browser reconnecting. @@ -1930,57 +2237,68 @@ parse_changes_query(Req) -> undefined -> ChangesArgs; Value -> - ChangesArgs#changes_args{since=Value} + ChangesArgs#changes_args{since = Value} end; _ -> ChangesArgs end. -extract_header_rev(Req, ExplicitRev) when is_binary(ExplicitRev) or is_list(ExplicitRev)-> +extract_header_rev(Req, ExplicitRev) when is_binary(ExplicitRev) or is_list(ExplicitRev) -> extract_header_rev(Req, couch_doc:parse_rev(ExplicitRev)); extract_header_rev(Req, ExplicitRev) -> - Etag = case chttpd:header_value(Req, "If-Match") of - undefined -> undefined; - Value -> couch_doc:parse_rev(string:strip(Value, both, $")) - end, + Etag = + case chttpd:header_value(Req, "If-Match") of + undefined -> undefined; + Value -> couch_doc:parse_rev(string:strip(Value, both, $")) + end, case {ExplicitRev, Etag} of - {undefined, undefined} -> missing_rev; - {_, undefined} -> ExplicitRev; - {undefined, _} -> Etag; - _ when ExplicitRev == Etag -> Etag; - _ -> - throw({bad_request, "Document rev and etag have different values"}) + {undefined, undefined} -> missing_rev; + {_, undefined} -> ExplicitRev; + {undefined, _} -> Etag; + _ when ExplicitRev == Etag -> Etag; + _ -> throw({bad_request, "Document rev and etag have different values"}) end. validate_security_can_be_edited(DbName) -> UserDbName = config:get("chttpd_auth", "authentication_db", "_users"), - CanEditUserSecurityObject = config:get("couchdb","users_db_security_editable","false"), - case {DbName,CanEditUserSecurityObject} of - {UserDbName,"false"} -> + CanEditUserSecurityObject = config:get("couchdb", "users_db_security_editable", "false"), + case {DbName, CanEditUserSecurityObject} of + {UserDbName, "false"} -> Msg = "You can't edit the security object of the user database.", throw({forbidden, Msg}); - {_,_} -> ok + {_, _} -> + ok end. validate_revs(_Doc, true) -> ok; validate_revs(#doc{revs = {0, []}}, false) -> - throw({bad_request, ?l2b("When `new_edits: false`, " ++ - "the document needs `_rev` or `_revisions` specified")}); + throw( + {bad_request, + ?l2b( + "When `new_edits: false`, " ++ + "the document needs `_rev` or `_revisions` specified" + )} + ); validate_revs(_Doc, false) -> ok. validate_attachment_names(Doc) -> - lists:foreach(fun(Att) -> - Name = couch_att:fetch(name, Att), - validate_attachment_name(Name) - end, Doc#doc.atts). + lists:foreach( + fun(Att) -> + Name = couch_att:fetch(name, Att), + validate_attachment_name(Name) + end, + Doc#doc.atts + ). validate_attachment_name(Name) when is_list(Name) -> validate_attachment_name(list_to_binary(Name)); -validate_attachment_name(<<"_",Rest/binary>>) -> - throw({bad_request, <<"Attachment name '_", Rest/binary, - "' starts with prohibited character '_'">>}); +validate_attachment_name(<<"_", Rest/binary>>) -> + throw( + {bad_request, + <<"Attachment name '_", Rest/binary, "' starts with prohibited character '_'">>} + ); validate_attachment_name(Name) -> case couch_util:validate_utf8(Name) of true -> Name; @@ -1989,17 +2307,21 @@ validate_attachment_name(Name) -> -spec monitor_attachments(couch_att:att() | [couch_att:att()]) -> [reference()]. monitor_attachments(Atts) when is_list(Atts) -> - lists:foldl(fun(Att, Monitors) -> - case couch_att:fetch(data, Att) of - {Fd, _} -> - [monitor(process, Fd) | Monitors]; - stub -> - Monitors; - Else -> - couch_log:error("~p from couch_att:fetch(data, ~p)", [Else, Att]), - Monitors - end - end, [], Atts); + lists:foldl( + fun(Att, Monitors) -> + case couch_att:fetch(data, Att) of + {Fd, _} -> + [monitor(process, Fd) | Monitors]; + stub -> + Monitors; + Else -> + couch_log:error("~p from couch_att:fetch(data, ~p)", [Else, Att]), + Monitors + end + end, + [], + Atts + ); monitor_attachments(Att) -> monitor_attachments([Att]). @@ -2015,25 +2337,27 @@ set_namespace(<<"_design_docs">>, Args) -> set_namespace(NS, #mrargs{} = Args) -> couch_mrview_util:set_extra(Args, namespace, NS). - %% /db/_bulk_get stuff bulk_get_parse_doc_query(Req) -> - lists:foldl(fun({Key, Value}, Args) -> - ok = validate_query_param(Key), - parse_doc_query({Key, Value}, Args) - end, #doc_query_args{}, chttpd:qs(Req)). - + lists:foldl( + fun({Key, Value}, Args) -> + ok = validate_query_param(Key), + parse_doc_query({Key, Value}, Args) + end, + #doc_query_args{}, + chttpd:qs(Req) + ). -validate_query_param("open_revs"=Key) -> +validate_query_param("open_revs" = Key) -> throw_bad_query_param(Key); -validate_query_param("new_edits"=Key) -> +validate_query_param("new_edits" = Key) -> throw_bad_query_param(Key); -validate_query_param("w"=Key) -> +validate_query_param("w" = Key) -> throw_bad_query_param(Key); -validate_query_param("rev"=Key) -> +validate_query_param("rev" = Key) -> throw_bad_query_param(Key); -validate_query_param("atts_since"=Key) -> +validate_query_param("atts_since" = Key) -> throw_bad_query_param(Key); validate_query_param(_) -> ok. @@ -2044,11 +2368,9 @@ throw_bad_query_param(Key) when is_binary(Key) -> Msg = <<"\"", Key/binary, "\" query parameter is not acceptable">>, throw({bad_request, Msg}). - bulk_get_open_doc_revs(Db, {Props}, Options) -> bulk_get_open_doc_revs1(Db, Props, Options, {}). - bulk_get_open_doc_revs1(Db, Props, Options, {}) -> case couch_util:get_value(<<"id">>, Props) of undefined -> @@ -2058,8 +2380,9 @@ bulk_get_open_doc_revs1(Db, Props, Options, {}) -> try couch_db:validate_docid(Db, DocId), bulk_get_open_doc_revs1(Db, Props, Options, {DocId}) - catch throw:{Error, Reason} -> - {DocId, {error, {null, Error, Reason}}, Options} + catch + throw:{Error, Reason} -> + {DocId, {error, {null, Error, Reason}}, Options} end end; bulk_get_open_doc_revs1(Db, Props, Options, {DocId}) -> @@ -2068,10 +2391,8 @@ bulk_get_open_doc_revs1(Db, Props, Options, {DocId}) -> case parse_field(<<"rev">>, RevStr) of {error, {RevStr, Error, Reason}} -> {DocId, {error, {RevStr, Error, Reason}}, Options}; - {ok, undefined} -> bulk_get_open_doc_revs1(Db, Props, Options, {DocId, all}); - {ok, Rev} -> bulk_get_open_doc_revs1(Db, Props, Options, {DocId, [Rev]}) end; @@ -2081,10 +2402,8 @@ bulk_get_open_doc_revs1(Db, Props, Options, {DocId, Revs}) -> case parse_field(<<"atts_since">>, AttsSinceStr) of {error, {BadAttsSinceRev, Error, Reason}} -> {DocId, {error, {BadAttsSinceRev, Error, Reason}}, Options}; - {ok, []} -> bulk_get_open_doc_revs1(Db, Props, Options, {DocId, Revs, Options}); - {ok, RevList} -> Options1 = [{atts_since, RevList}, attachments | Options], bulk_get_open_doc_revs1(Db, Props, Options, {DocId, Revs, Options1}) @@ -2102,7 +2421,6 @@ bulk_get_open_doc_revs1(Db, Props, _, {DocId, Revs, Options}) -> {DocId, Else, Options} end. - parse_field(<<"rev">>, undefined) -> {ok, undefined}; parse_field(<<"rev">>, Value) -> @@ -2110,7 +2428,7 @@ parse_field(<<"rev">>, Value) -> Rev = couch_doc:parse_rev(Value), {ok, Rev} catch - throw:{bad_request=Error, Reason} -> + throw:{bad_request = Error, Reason} -> {error, {Value, Error, Reason}} end; parse_field(<<"atts_since">>, undefined) -> @@ -2122,18 +2440,16 @@ parse_field(<<"atts_since">>, Value) when is_list(Value) -> parse_field(<<"atts_since">>, Value) -> {error, {Value, bad_request, <<"att_since value must be array of revs.">>}}. - parse_atts_since([], Acc) -> {ok, lists:reverse(Acc)}; parse_atts_since([RevStr | Rest], Acc) -> case parse_field(<<"rev">>, RevStr) of {ok, Rev} -> parse_atts_since(Rest, [Rev | Acc]); - {error, _}=Error -> + {error, _} = Error -> Error end. - bulk_get_send_docs_json(Resp, DocId, Results, Options, Sep) -> Id = ?JSON_ENCODE(DocId), send_chunk(Resp, [Sep, <<"{\"id\": ">>, Id, <<", \"docs\": [">>]), @@ -2145,37 +2461,45 @@ bulk_get_send_docs_json1(Resp, DocId, {error, {Rev, Error, Reason}}, _) -> bulk_get_send_docs_json1(_Resp, _DocId, {ok, []}, _) -> ok; bulk_get_send_docs_json1(Resp, DocId, {ok, Docs}, Options) -> - lists:foldl(fun(Result, AccSeparator) -> - case Result of - {ok, Doc} -> - JsonDoc = couch_doc:to_json_obj(Doc, Options), - Json = ?JSON_ENCODE({[{ok, JsonDoc}]}), - send_chunk(Resp, [AccSeparator, Json]); - {{Error, Reason}, RevId} -> - RevStr = couch_doc:rev_to_str(RevId), - Json = bulk_get_json_error(DocId, RevStr, Error, Reason), - send_chunk(Resp, [AccSeparator, Json]) + lists:foldl( + fun(Result, AccSeparator) -> + case Result of + {ok, Doc} -> + JsonDoc = couch_doc:to_json_obj(Doc, Options), + Json = ?JSON_ENCODE({[{ok, JsonDoc}]}), + send_chunk(Resp, [AccSeparator, Json]); + {{Error, Reason}, RevId} -> + RevStr = couch_doc:rev_to_str(RevId), + Json = bulk_get_json_error(DocId, RevStr, Error, Reason), + send_chunk(Resp, [AccSeparator, Json]) + end, + <<",">> end, - <<",">> - end, <<"">>, Docs). + <<"">>, + Docs + ). bulk_get_json_error(DocId, Rev, Error, Reason) -> - ?JSON_ENCODE({[{error, {[{<<"id">>, DocId}, - {<<"rev">>, Rev}, - {<<"error">>, Error}, - {<<"reason">>, Reason}]}}]}). - + ?JSON_ENCODE( + {[ + {error, + {[ + {<<"id">>, DocId}, + {<<"rev">>, Rev}, + {<<"error">>, Error}, + {<<"reason">>, Reason} + ]}} + ]} + ). -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). monitor_attachments_test_() -> - {"ignore stubs", - fun () -> - Atts = [couch_att:new([{data, stub}])], - ?_assertEqual([], monitor_attachments(Atts)) - end - }. + {"ignore stubs", fun() -> + Atts = [couch_att:new([{data, stub}])], + ?_assertEqual([], monitor_attachments(Atts)) + end}. parse_partitioned_opt_test_() -> { @@ -2296,18 +2620,20 @@ t_should_allow_valid_placement() -> fun() -> ok end, [ {"single zone", - ?_test(begin - Req = mock_request("/all-test21?placement=az:1"), - Opts = parse_shards_opt(Req), - ?assertEqual("az:1", couch_util:get_value(placement, Opts)) - end)}, + ?_test(begin + Req = mock_request("/all-test21?placement=az:1"), + Opts = parse_shards_opt(Req), + ?assertEqual("az:1", couch_util:get_value(placement, Opts)) + end)}, {"multi zone", - ?_test(begin - Req = mock_request("/all-test21?placement=az:1,co:3"), - Opts = parse_shards_opt(Req), - ?assertEqual("az:1,co:3", - couch_util:get_value(placement, Opts)) - end)} + ?_test(begin + Req = mock_request("/all-test21?placement=az:1,co:3"), + Opts = parse_shards_opt(Req), + ?assertEqual( + "az:1,co:3", + couch_util:get_value(placement, Opts) + ) + end)} ] }. @@ -2325,25 +2651,25 @@ t_should_throw_on_invalid_placement() -> fun() -> ok end, [ {"empty placement", - ?_test(begin - Req = mock_request("/all-test21?placement="), - ?assertThrow({bad_request, Err}, parse_shards_opt(Req)) - end)}, + ?_test(begin + Req = mock_request("/all-test21?placement="), + ?assertThrow({bad_request, Err}, parse_shards_opt(Req)) + end)}, {"invalid format", - ?_test(begin - Req = mock_request("/all-test21?placement=moon"), - ?assertThrow({bad_request, Err}, parse_shards_opt(Req)) - end)}, + ?_test(begin + Req = mock_request("/all-test21?placement=moon"), + ?assertThrow({bad_request, Err}, parse_shards_opt(Req)) + end)}, {"invalid n", - ?_test(begin - Req = mock_request("/all-test21?placement=moon:eagle"), - ?assertThrow({bad_request, Err}, parse_shards_opt(Req)) - end)}, + ?_test(begin + Req = mock_request("/all-test21?placement=moon:eagle"), + ?assertThrow({bad_request, Err}, parse_shards_opt(Req)) + end)}, {"one invalid zone", - ?_test(begin - Req = mock_request("/all-test21?placement=az:1,co:moon"), - ?assertThrow({bad_request, Err}, parse_shards_opt(Req)) - end)} + ?_test(begin + Req = mock_request("/all-test21?placement=az:1,co:moon"), + ?assertThrow({bad_request, Err}, parse_shards_opt(Req)) + end)} ] }. diff --git a/src/chttpd/src/chttpd_epi.erl b/src/chttpd/src/chttpd_epi.erl index ffbd87a07..5536c9e4d 100644 --- a/src/chttpd/src/chttpd_epi.erl +++ b/src/chttpd/src/chttpd_epi.erl @@ -10,7 +10,6 @@ % License for the specific language governing permissions and limitations under % the License. - -module(chttpd_epi). -behaviour(couch_epi_plugin). @@ -33,7 +32,6 @@ providers() -> {chttpd_handlers, chttpd_httpd_handlers} ]. - services() -> [ {chttpd_auth, chttpd_auth}, diff --git a/src/chttpd/src/chttpd_external.erl b/src/chttpd/src/chttpd_external.erl index 12d1fe555..84cfc9620 100644 --- a/src/chttpd/src/chttpd_external.erl +++ b/src/chttpd/src/chttpd_external.erl @@ -18,7 +18,7 @@ -export([json_req_obj_fields/0, json_req_obj/2, json_req_obj/3, json_req_obj/4]). -export([default_or_content_type/2, parse_external_response/1]). --import(chttpd,[send_error/4]). +-import(chttpd, [send_error/4]). -include_lib("couch/include/couch_db.hrl"). @@ -33,9 +33,23 @@ json_req_obj(Req, Db, DocId, Fields) when is_list(Fields) -> {[{Field, json_req_obj_field(Field, Req, Db, DocId)} || Field <- Fields]}. json_req_obj_fields() -> - [<<"info">>, <<"uuid">>, <<"id">>, <<"method">>, <<"requested_path">>, - <<"path">>, <<"raw_path">>, <<"query">>, <<"headers">>, <<"body">>, - <<"peer">>, <<"form">>, <<"cookie">>, <<"userCtx">>, <<"secObj">>]. + [ + <<"info">>, + <<"uuid">>, + <<"id">>, + <<"method">>, + <<"requested_path">>, + <<"path">>, + <<"raw_path">>, + <<"query">>, + <<"headers">>, + <<"body">>, + <<"peer">>, + <<"form">>, + <<"cookie">>, + <<"userCtx">>, + <<"secObj">> + ]. json_req_obj_field(<<"info">>, #httpd{}, Db, _DocId) -> {ok, Info} = get_db_info(Db), @@ -44,51 +58,55 @@ json_req_obj_field(<<"uuid">>, #httpd{}, _Db, _DocId) -> couch_uuids:new(); json_req_obj_field(<<"id">>, #httpd{}, _Db, DocId) -> DocId; -json_req_obj_field(<<"method">>, #httpd{method=Method}, _Db, _DocId) -> +json_req_obj_field(<<"method">>, #httpd{method = Method}, _Db, _DocId) -> Method; -json_req_obj_field(<<"requested_path">>, #httpd{requested_path_parts=Path}, _Db, _DocId) -> +json_req_obj_field(<<"requested_path">>, #httpd{requested_path_parts = Path}, _Db, _DocId) -> Path; -json_req_obj_field(<<"path">>, #httpd{path_parts=Path}, _Db, _DocId) -> +json_req_obj_field(<<"path">>, #httpd{path_parts = Path}, _Db, _DocId) -> Path; -json_req_obj_field(<<"raw_path">>, #httpd{mochi_req=Req}, _Db, _DocId) -> +json_req_obj_field(<<"raw_path">>, #httpd{mochi_req = Req}, _Db, _DocId) -> ?l2b(Req:get(raw_path)); -json_req_obj_field(<<"query">>, #httpd{mochi_req=Req}, _Db, _DocId) -> +json_req_obj_field(<<"query">>, #httpd{mochi_req = Req}, _Db, _DocId) -> json_query_keys(to_json_terms(Req:parse_qs())); -json_req_obj_field(<<"headers">>, #httpd{mochi_req=Req}, _Db, _DocId) -> +json_req_obj_field(<<"headers">>, #httpd{mochi_req = Req}, _Db, _DocId) -> Headers = Req:get(headers), Hlist = mochiweb_headers:to_list(Headers), to_json_terms(Hlist); -json_req_obj_field(<<"body">>, #httpd{req_body=undefined, mochi_req=Req}, _Db, _DocId) -> +json_req_obj_field(<<"body">>, #httpd{req_body = undefined, mochi_req = Req}, _Db, _DocId) -> MaxSize = chttpd_util:get_chttpd_config_integer( - "max_http_request_size", 4294967296), + "max_http_request_size", 4294967296 + ), try Req:recv_body(MaxSize) - catch exit:normal -> - exit({bad_request, <<"Invalid request body">>}) + catch + exit:normal -> + exit({bad_request, <<"Invalid request body">>}) end; -json_req_obj_field(<<"body">>, #httpd{req_body=Body}, _Db, _DocId) -> +json_req_obj_field(<<"body">>, #httpd{req_body = Body}, _Db, _DocId) -> Body; -json_req_obj_field(<<"peer">>, #httpd{peer=undefined, mochi_req=Req}, _, _) -> +json_req_obj_field(<<"peer">>, #httpd{peer = undefined, mochi_req = Req}, _, _) -> ?l2b(Req:get(peer)); -json_req_obj_field(<<"peer">>, #httpd{peer=Peer}, _Db, _DocId) -> +json_req_obj_field(<<"peer">>, #httpd{peer = Peer}, _Db, _DocId) -> ?l2b(Peer); -json_req_obj_field(<<"form">>, #httpd{mochi_req=Req, method=Method}=HttpReq, Db, DocId) -> +json_req_obj_field(<<"form">>, #httpd{mochi_req = Req, method = Method} = HttpReq, Db, DocId) -> Body = json_req_obj_field(<<"body">>, HttpReq, Db, DocId), - ParsedForm = case Req:get_primary_header_value("content-type") of - "application/x-www-form-urlencoded" ++ _ when Method =:= 'POST' orelse Method =:= 'PUT' -> - mochiweb_util:parse_qs(Body); - _ -> - [] - end, + ParsedForm = + case Req:get_primary_header_value("content-type") of + "application/x-www-form-urlencoded" ++ _ when + Method =:= 'POST' orelse Method =:= 'PUT' + -> + mochiweb_util:parse_qs(Body); + _ -> + [] + end, to_json_terms(ParsedForm); -json_req_obj_field(<<"cookie">>, #httpd{mochi_req=Req}, _Db, _DocId) -> +json_req_obj_field(<<"cookie">>, #httpd{mochi_req = Req}, _Db, _DocId) -> to_json_terms(Req:parse_cookie()); json_req_obj_field(<<"userCtx">>, #httpd{}, Db, _DocId) -> couch_util:json_user_ctx(Db); -json_req_obj_field(<<"secObj">>, #httpd{user_ctx=UserCtx}, Db, _DocId) -> +json_req_obj_field(<<"secObj">>, #httpd{user_ctx = UserCtx}, Db, _DocId) -> get_db_security(Db, UserCtx). - get_db_info(Db) -> case couch_db:is_clustered(Db) of true -> @@ -97,7 +115,6 @@ get_db_info(Db) -> couch_db:get_db_info(Db) end. - get_db_security(Db, #user_ctx{}) -> case couch_db:is_clustered(Db) of true -> @@ -106,7 +123,6 @@ get_db_security(Db, #user_ctx{}) -> couch_db:get_security(Db) end. - to_json_terms(Data) -> to_json_terms(Data, []). to_json_terms([], Acc) -> @@ -121,15 +137,15 @@ json_query_keys({Json}) -> json_query_keys([], Acc) -> {lists:reverse(Acc)}; json_query_keys([{<<"startkey">>, Value} | Rest], Acc) -> - json_query_keys(Rest, [{<<"startkey">>, ?JSON_DECODE(Value)}|Acc]); + json_query_keys(Rest, [{<<"startkey">>, ?JSON_DECODE(Value)} | Acc]); json_query_keys([{<<"endkey">>, Value} | Rest], Acc) -> - json_query_keys(Rest, [{<<"endkey">>, ?JSON_DECODE(Value)}|Acc]); + json_query_keys(Rest, [{<<"endkey">>, ?JSON_DECODE(Value)} | Acc]); json_query_keys([{<<"key">>, Value} | Rest], Acc) -> - json_query_keys(Rest, [{<<"key">>, ?JSON_DECODE(Value)}|Acc]); + json_query_keys(Rest, [{<<"key">>, ?JSON_DECODE(Value)} | Acc]); json_query_keys([{<<"descending">>, Value} | Rest], Acc) -> - json_query_keys(Rest, [{<<"descending">>, ?JSON_DECODE(Value)}|Acc]); + json_query_keys(Rest, [{<<"descending">>, ?JSON_DECODE(Value)} | Acc]); json_query_keys([Term | Rest], Acc) -> - json_query_keys(Rest, [Term|Acc]). + json_query_keys(Rest, [Term | Acc]). send_external_response(Req, Response) -> #extern_resp_args{ @@ -141,49 +157,60 @@ send_external_response(Req, Response) -> } = parse_external_response(Response), Headers1 = default_or_content_type(CType, Headers0), case Json of - nil -> - Headers2 = chttpd_util:maybe_add_csp_header("showlist", Headers1, "sandbox"), - chttpd:send_response(Req, Code, Headers2, Data); - Json -> - chttpd:send_json(Req, Code, Headers1, Json) + nil -> + Headers2 = chttpd_util:maybe_add_csp_header("showlist", Headers1, "sandbox"), + chttpd:send_response(Req, Code, Headers2, Data); + Json -> + chttpd:send_json(Req, Code, Headers1, Json) end. parse_external_response({Response}) -> - lists:foldl(fun({Key,Value}, Args) -> - case {Key, Value} of - {"", _} -> - Args; - {<<"code">>, Value} -> - Args#extern_resp_args{code=Value}; - {<<"stop">>, true} -> - Args#extern_resp_args{stop=true}; - {<<"json">>, Value} -> - Args#extern_resp_args{ - json=Value, - ctype="application/json"}; - {<<"body">>, Value} -> - Args#extern_resp_args{data=Value, ctype="text/html; charset=utf-8"}; - {<<"base64">>, Value} -> - Args#extern_resp_args{ - data=base64:decode(Value), - ctype="application/binary" - }; - {<<"headers">>, {Headers}} -> - NewHeaders = lists:map(fun({Header, HVal}) -> - {couch_util:to_list(Header), couch_util:to_list(HVal)} - end, Headers), - Args#extern_resp_args{headers=NewHeaders}; - _ -> % unknown key - Msg = lists:flatten(io_lib:format("Invalid data from external server: ~p", [{Key, Value}])), - throw({external_response_error, Msg}) + lists:foldl( + fun({Key, Value}, Args) -> + case {Key, Value} of + {"", _} -> + Args; + {<<"code">>, Value} -> + Args#extern_resp_args{code = Value}; + {<<"stop">>, true} -> + Args#extern_resp_args{stop = true}; + {<<"json">>, Value} -> + Args#extern_resp_args{ + json = Value, + ctype = "application/json" + }; + {<<"body">>, Value} -> + Args#extern_resp_args{data = Value, ctype = "text/html; charset=utf-8"}; + {<<"base64">>, Value} -> + Args#extern_resp_args{ + data = base64:decode(Value), + ctype = "application/binary" + }; + {<<"headers">>, {Headers}} -> + NewHeaders = lists:map( + fun({Header, HVal}) -> + {couch_util:to_list(Header), couch_util:to_list(HVal)} + end, + Headers + ), + Args#extern_resp_args{headers = NewHeaders}; + % unknown key + _ -> + Msg = lists:flatten( + io_lib:format("Invalid data from external server: ~p", [{Key, Value}]) + ), + throw({external_response_error, Msg}) end - end, #extern_resp_args{}, Response). + end, + #extern_resp_args{}, + Response + ). default_or_content_type(DefaultContentType, Headers) -> IsContentType = fun({X, _}) -> string:to_lower(X) == "content-type" end, case lists:any(IsContentType, Headers) of - false -> - [{"Content-Type", DefaultContentType} | Headers]; - true -> - Headers + false -> + [{"Content-Type", DefaultContentType} | Headers]; + true -> + Headers end. diff --git a/src/chttpd/src/chttpd_handlers.erl b/src/chttpd/src/chttpd_handlers.erl index 930563230..82eee7365 100644 --- a/src/chttpd/src/chttpd_handlers.erl +++ b/src/chttpd/src/chttpd_handlers.erl @@ -57,9 +57,9 @@ select(Handlers, _Default) -> do_select([], Acc) -> Acc; -do_select([{override, Handler}|_], _Acc) -> +do_select([{override, Handler} | _], _Acc) -> [Handler]; -do_select([{default, _}|Rest], Acc) -> +do_select([{default, _} | Rest], Acc) -> do_select(Rest, Acc); do_select([Handler], Acc) -> [Handler | Acc]; @@ -73,14 +73,16 @@ select_override_test() -> ?assertEqual(selected, select([{override, selected}, foo], default)), ?assertEqual(selected, select([foo, {override, selected}], default)), ?assertEqual(selected, select([{override, selected}, {override, bar}], default)), - ?assertError({badmatch,[bar, foo]}, select([foo, bar], default)). + ?assertError({badmatch, [bar, foo]}, select([foo, bar], default)). select_default_override_test() -> ?assertEqual(selected, select([{default, new_default}, selected], old_default)), ?assertEqual(selected, select([selected, {default, new_default}], old_default)), ?assertEqual(selected, select([{default, selected}], old_default)), ?assertEqual(selected, select([], selected)), - ?assertEqual(selected, - select([{default, new_default}, {override, selected}, bar], old_default)). + ?assertEqual( + selected, + select([{default, new_default}, {override, selected}, bar], old_default) + ). -endif. diff --git a/src/chttpd/src/chttpd_httpd_handlers.erl b/src/chttpd/src/chttpd_httpd_handlers.erl index 5e86ea87d..932b52e5f 100644 --- a/src/chttpd/src/chttpd_httpd_handlers.erl +++ b/src/chttpd/src/chttpd_httpd_handlers.erl @@ -14,33 +14,33 @@ -export([url_handler/1, db_handler/1, design_handler/1]). -url_handler(<<>>) -> fun chttpd_misc:handle_welcome_req/1; -url_handler(<<"favicon.ico">>) -> fun chttpd_misc:handle_favicon_req/1; -url_handler(<<"_utils">>) -> fun chttpd_misc:handle_utils_dir_req/1; -url_handler(<<"_all_dbs">>) -> fun chttpd_misc:handle_all_dbs_req/1; -url_handler(<<"_dbs_info">>) -> fun chttpd_misc:handle_dbs_info_req/1; -url_handler(<<"_active_tasks">>) -> fun chttpd_misc:handle_task_status_req/1; -url_handler(<<"_scheduler">>) -> fun couch_replicator_httpd:handle_scheduler_req/1; -url_handler(<<"_node">>) -> fun chttpd_node:handle_node_req/1; +url_handler(<<>>) -> fun chttpd_misc:handle_welcome_req/1; +url_handler(<<"favicon.ico">>) -> fun chttpd_misc:handle_favicon_req/1; +url_handler(<<"_utils">>) -> fun chttpd_misc:handle_utils_dir_req/1; +url_handler(<<"_all_dbs">>) -> fun chttpd_misc:handle_all_dbs_req/1; +url_handler(<<"_dbs_info">>) -> fun chttpd_misc:handle_dbs_info_req/1; +url_handler(<<"_active_tasks">>) -> fun chttpd_misc:handle_task_status_req/1; +url_handler(<<"_scheduler">>) -> fun couch_replicator_httpd:handle_scheduler_req/1; +url_handler(<<"_node">>) -> fun chttpd_node:handle_node_req/1; url_handler(<<"_reload_query_servers">>) -> fun chttpd_misc:handle_reload_query_servers_req/1; -url_handler(<<"_replicate">>) -> fun chttpd_misc:handle_replicate_req/1; -url_handler(<<"_uuids">>) -> fun chttpd_misc:handle_uuids_req/1; -url_handler(<<"_session">>) -> fun chttpd_auth:handle_session_req/1; -url_handler(<<"_up">>) -> fun chttpd_misc:handle_up_req/1; +url_handler(<<"_replicate">>) -> fun chttpd_misc:handle_replicate_req/1; +url_handler(<<"_uuids">>) -> fun chttpd_misc:handle_uuids_req/1; +url_handler(<<"_session">>) -> fun chttpd_auth:handle_session_req/1; +url_handler(<<"_up">>) -> fun chttpd_misc:handle_up_req/1; url_handler(_) -> no_match. db_handler(<<"_view_cleanup">>) -> fun chttpd_db:handle_view_cleanup_req/2; -db_handler(<<"_compact">>) -> fun chttpd_db:handle_compact_req/2; -db_handler(<<"_design">>) -> fun chttpd_db:handle_design_req/2; -db_handler(<<"_partition">>) -> fun chttpd_db:handle_partition_req/2; -db_handler(<<"_temp_view">>) -> fun chttpd_view:handle_temp_view_req/2; -db_handler(<<"_changes">>) -> fun chttpd_db:handle_changes_req/2; +db_handler(<<"_compact">>) -> fun chttpd_db:handle_compact_req/2; +db_handler(<<"_design">>) -> fun chttpd_db:handle_design_req/2; +db_handler(<<"_partition">>) -> fun chttpd_db:handle_partition_req/2; +db_handler(<<"_temp_view">>) -> fun chttpd_view:handle_temp_view_req/2; +db_handler(<<"_changes">>) -> fun chttpd_db:handle_changes_req/2; db_handler(_) -> no_match. -design_handler(<<"_view">>) -> fun chttpd_view:handle_view_req/3; -design_handler(<<"_show">>) -> fun chttpd_show:handle_doc_show_req/3; -design_handler(<<"_list">>) -> fun chttpd_show:handle_view_list_req/3; -design_handler(<<"_update">>) -> fun chttpd_show:handle_doc_update_req/3; -design_handler(<<"_info">>) -> fun chttpd_db:handle_design_info_req/3; +design_handler(<<"_view">>) -> fun chttpd_view:handle_view_req/3; +design_handler(<<"_show">>) -> fun chttpd_show:handle_doc_show_req/3; +design_handler(<<"_list">>) -> fun chttpd_show:handle_view_list_req/3; +design_handler(<<"_update">>) -> fun chttpd_show:handle_doc_update_req/3; +design_handler(<<"_info">>) -> fun chttpd_db:handle_design_info_req/3; design_handler(<<"_rewrite">>) -> fun chttpd_rewrite:handle_rewrite_req/3; design_handler(_) -> no_match. diff --git a/src/chttpd/src/chttpd_misc.erl b/src/chttpd/src/chttpd_misc.erl index 25a0fa77f..6d119572d 100644 --- a/src/chttpd/src/chttpd_misc.erl +++ b/src/chttpd/src/chttpd_misc.erl @@ -31,9 +31,15 @@ -include_lib("couch/include/couch_db.hrl"). -include_lib("couch_mrview/include/couch_mrview.hrl"). --import(chttpd, - [send_json/2,send_json/3,send_method_not_allowed/2, - send_chunk/2,start_chunked_response/3]). +-import( + chttpd, + [ + send_json/2, send_json/3, + send_method_not_allowed/2, + send_chunk/2, + start_chunked_response/3 + ] +). -define(MAX_DB_NUM_FOR_DBS_INFO, 100). @@ -42,19 +48,21 @@ handle_welcome_req(Req) -> handle_welcome_req(Req, <<"Welcome">>). -handle_welcome_req(#httpd{method='GET'}=Req, WelcomeMessage) -> - send_json(Req, {[ - {couchdb, WelcomeMessage}, - {version, list_to_binary(couch_server:get_version())}, - {git_sha, list_to_binary(couch_server:get_git_sha())}, - {uuid, couch_server:get_uuid()}, - {features, get_features()} - ] ++ case config:get("vendor") of - [] -> - []; - Properties -> - [{vendor, {[{?l2b(K), ?l2b(V)} || {K, V} <- Properties]}}] - end +handle_welcome_req(#httpd{method = 'GET'} = Req, WelcomeMessage) -> + send_json(Req, { + [ + {couchdb, WelcomeMessage}, + {version, list_to_binary(couch_server:get_version())}, + {git_sha, list_to_binary(couch_server:get_git_sha())}, + {uuid, couch_server:get_uuid()}, + {features, get_features()} + ] ++ + case config:get("vendor") of + [] -> + []; + Properties -> + [{vendor, {[{?l2b(K), ?l2b(V)} || {K, V} <- Properties]}}] + end }); handle_welcome_req(Req, _) -> send_method_not_allowed(Req, "GET,HEAD"). @@ -70,7 +78,7 @@ get_features() -> handle_favicon_req(Req) -> handle_favicon_req(Req, get_docroot()). -handle_favicon_req(#httpd{method='GET'}=Req, DocumentRoot) -> +handle_favicon_req(#httpd{method = 'GET'} = Req, DocumentRoot) -> {DateNow, TimeNow} = calendar:universal_time(), DaysNow = calendar:date_to_gregorian_days(DateNow), DaysWhenExpires = DaysNow + 365, @@ -87,25 +95,26 @@ handle_favicon_req(Req, _) -> handle_utils_dir_req(Req) -> handle_utils_dir_req(Req, get_docroot()). -handle_utils_dir_req(#httpd{method='GET'}=Req, DocumentRoot) -> +handle_utils_dir_req(#httpd{method = 'GET'} = Req, DocumentRoot) -> "/" ++ UrlPath = chttpd:path(Req), case chttpd:partition(UrlPath) of - {_ActionKey, "/", RelativePath} -> - % GET /_utils/path or GET /_utils/ - CachingHeaders = [{"Cache-Control", "private, must-revalidate"}], - DefaultValues = "child-src 'self' data: blob:; default-src 'self'; img-src 'self' data:; font-src 'self'; " - "script-src 'self' 'unsafe-eval'; style-src 'self' 'unsafe-inline';", - Headers = chttpd_util:maybe_add_csp_header("utils", CachingHeaders, DefaultValues), - chttpd:serve_file(Req, RelativePath, DocumentRoot, Headers); - {_ActionKey, "", _RelativePath} -> - % GET /_utils - RedirectPath = chttpd:path(Req) ++ "/", - chttpd:send_redirect(Req, RedirectPath) + {_ActionKey, "/", RelativePath} -> + % GET /_utils/path or GET /_utils/ + CachingHeaders = [{"Cache-Control", "private, must-revalidate"}], + DefaultValues = + "child-src 'self' data: blob:; default-src 'self'; img-src 'self' data:; font-src 'self'; " + "script-src 'self' 'unsafe-eval'; style-src 'self' 'unsafe-inline';", + Headers = chttpd_util:maybe_add_csp_header("utils", CachingHeaders, DefaultValues), + chttpd:serve_file(Req, RelativePath, DocumentRoot, Headers); + {_ActionKey, "", _RelativePath} -> + % GET /_utils + RedirectPath = chttpd:path(Req) ++ "/", + chttpd:send_redirect(Req, RedirectPath) end; handle_utils_dir_req(Req, _) -> send_method_not_allowed(Req, "GET,HEAD"). -handle_all_dbs_req(#httpd{method='GET'}=Req) -> +handle_all_dbs_req(#httpd{method = 'GET'} = Req) -> Args = couch_mrview_http:parse_params(Req, undefined), ShardDbName = config:get("mem3", "shards_db", "_dbs"), %% shard_db is not sharded but mem3:shards treats it as an edge case @@ -114,8 +123,8 @@ handle_all_dbs_req(#httpd{method='GET'}=Req) -> Etag = couch_httpd:make_etag({Info}), Options = [{user_ctx, Req#httpd.user_ctx}], {ok, Resp} = chttpd:etag_respond(Req, Etag, fun() -> - {ok, Resp} = chttpd:start_delayed_json_response(Req, 200, [{"ETag",Etag}]), - VAcc = #vacc{req=Req,resp=Resp}, + {ok, Resp} = chttpd:start_delayed_json_response(Req, 200, [{"ETag", Etag}]), + VAcc = #vacc{req = Req, resp = Resp}, fabric:all_docs(ShardDbName, Options, fun all_dbs_callback/2, VAcc, Args) end), case is_record(Resp, vacc) of @@ -125,26 +134,27 @@ handle_all_dbs_req(#httpd{method='GET'}=Req) -> handle_all_dbs_req(Req) -> send_method_not_allowed(Req, "GET,HEAD"). -all_dbs_callback({meta, _Meta}, #vacc{resp=Resp0}=Acc) -> +all_dbs_callback({meta, _Meta}, #vacc{resp = Resp0} = Acc) -> {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, "["), - {ok, Acc#vacc{resp=Resp1}}; -all_dbs_callback({row, Row}, #vacc{resp=Resp0}=Acc) -> + {ok, Acc#vacc{resp = Resp1}}; +all_dbs_callback({row, Row}, #vacc{resp = Resp0} = Acc) -> Prepend = couch_mrview_http:prepend_val(Acc), - case couch_util:get_value(id, Row) of <<"_design", _/binary>> -> - {ok, Acc}; - DbName -> - {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, [Prepend, ?JSON_ENCODE(DbName)]), - {ok, Acc#vacc{prepend=",", resp=Resp1}} + case couch_util:get_value(id, Row) of + <<"_design", _/binary>> -> + {ok, Acc}; + DbName -> + {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, [Prepend, ?JSON_ENCODE(DbName)]), + {ok, Acc#vacc{prepend = ",", resp = Resp1}} end; -all_dbs_callback(complete, #vacc{resp=Resp0}=Acc) -> +all_dbs_callback(complete, #vacc{resp = Resp0} = Acc) -> {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, "]"), {ok, Resp2} = chttpd:end_delayed_json_response(Resp1), - {ok, Acc#vacc{resp=Resp2}}; -all_dbs_callback({error, Reason}, #vacc{resp=Resp0}=Acc) -> + {ok, Acc#vacc{resp = Resp2}}; +all_dbs_callback({error, Reason}, #vacc{resp = Resp0} = Acc) -> {ok, Resp1} = chttpd:send_delayed_error(Resp0, Reason), - {ok, Acc#vacc{resp=Resp1}}. + {ok, Acc#vacc{resp = Resp1}}. -handle_dbs_info_req(#httpd{method='POST'}=Req) -> +handle_dbs_info_req(#httpd{method = 'POST'} = Req) -> chttpd:validate_ctype(Req, "application/json"), Props = chttpd:json_body_obj(Req), Keys = couch_mrview_util:get_view_keys(Props), @@ -152,41 +162,52 @@ handle_dbs_info_req(#httpd{method='POST'}=Req) -> undefined -> throw({bad_request, "`keys` member must exist."}); _ -> ok end, - MaxNumber = config:get_integer("chttpd", - "max_db_number_for_dbs_info_req", ?MAX_DB_NUM_FOR_DBS_INFO), + MaxNumber = config:get_integer( + "chttpd", + "max_db_number_for_dbs_info_req", + ?MAX_DB_NUM_FOR_DBS_INFO + ), case length(Keys) =< MaxNumber of true -> ok; false -> throw({bad_request, too_many_keys}) end, {ok, Resp} = chttpd:start_json_response(Req, 200), send_chunk(Resp, "["), - lists:foldl(fun(DbName, AccSeparator) -> - case catch fabric:get_db_info(DbName) of - {ok, Result} -> - Json = ?JSON_ENCODE({[{key, DbName}, {info, {Result}}]}), - send_chunk(Resp, AccSeparator ++ Json); - _ -> - Json = ?JSON_ENCODE({[{key, DbName}, {error, not_found}]}), - send_chunk(Resp, AccSeparator ++ Json) + lists:foldl( + fun(DbName, AccSeparator) -> + case catch fabric:get_db_info(DbName) of + {ok, Result} -> + Json = ?JSON_ENCODE({[{key, DbName}, {info, {Result}}]}), + send_chunk(Resp, AccSeparator ++ Json); + _ -> + Json = ?JSON_ENCODE({[{key, DbName}, {error, not_found}]}), + send_chunk(Resp, AccSeparator ++ Json) + end, + % AccSeparator now has a comma + "," end, - "," % AccSeparator now has a comma - end, "", Keys), + "", + Keys + ), send_chunk(Resp, "]"), chttpd:end_json_response(Resp); handle_dbs_info_req(Req) -> send_method_not_allowed(Req, "POST"). -handle_task_status_req(#httpd{method='GET'}=Req) -> +handle_task_status_req(#httpd{method = 'GET'} = Req) -> ok = chttpd:verify_is_server_admin(Req), {Replies, _BadNodes} = gen_server:multi_call(couch_task_status, all), - Response = lists:flatmap(fun({Node, Tasks}) -> - [{[{node,Node} | Task]} || Task <- Tasks] - end, Replies), + Response = lists:flatmap( + fun({Node, Tasks}) -> + [{[{node, Node} | Task]} || Task <- Tasks] + end, + Replies + ), send_json(Req, lists:sort(Response)); handle_task_status_req(Req) -> send_method_not_allowed(Req, "GET,HEAD"). -handle_replicate_req(#httpd{method='POST', user_ctx=Ctx, req_body=PostBody} = Req) -> +handle_replicate_req(#httpd{method = 'POST', user_ctx = Ctx, req_body = PostBody} = Req) -> chttpd:validate_ctype(Req, "application/json"), %% see HACK in chttpd.erl about replication case replicate(PostBody, Ctx) of @@ -198,11 +219,11 @@ handle_replicate_req(#httpd{method='POST', user_ctx=Ctx, req_body=PostBody} = Re send_json(Req, {[{ok, true} | JsonResults]}); {ok, stopped} -> send_json(Req, 200, {[{ok, stopped}]}); - {error, not_found=Error} -> + {error, not_found = Error} -> chttpd:send_error(Req, Error); - {error, {_, _}=Error} -> + {error, {_, _} = Error} -> chttpd:send_error(Req, Error); - {_, _}=Error -> + {_, _} = Error -> chttpd:send_error(Req, Error) end; handle_replicate_req(Req) -> @@ -210,50 +231,50 @@ handle_replicate_req(Req) -> replicate({Props} = PostBody, Ctx) -> case couch_util:get_value(<<"cancel">>, Props) of - true -> - cancel_replication(PostBody, Ctx); - _ -> - Node = choose_node([ - couch_util:get_value(<<"source">>, Props), - couch_util:get_value(<<"target">>, Props) - ]), - case rpc:call(Node, couch_replicator, replicate, [PostBody, Ctx]) of - {badrpc, Reason} -> - erlang:error(Reason); - Res -> - Res - end + true -> + cancel_replication(PostBody, Ctx); + _ -> + Node = choose_node([ + couch_util:get_value(<<"source">>, Props), + couch_util:get_value(<<"target">>, Props) + ]), + case rpc:call(Node, couch_replicator, replicate, [PostBody, Ctx]) of + {badrpc, Reason} -> + erlang:error(Reason); + Res -> + Res + end end. cancel_replication(PostBody, Ctx) -> {Res, _Bad} = rpc:multicall(couch_replicator, replicate, [PostBody, Ctx]), case [X || {ok, {cancelled, _}} = X <- Res] of - [Success|_] -> - % Report success if at least one node canceled the replication - Success; - [] -> - case lists:usort(Res) of - [UniqueReply] -> - % Report a universally agreed-upon reply - UniqueReply; + [Success | _] -> + % Report success if at least one node canceled the replication + Success; [] -> - {error, badrpc}; - Else -> - % Unclear what to do here -- pick the first error? - % Except try ignoring any {error, not_found} responses - % because we'll always get two of those - hd(Else -- [{error, not_found}]) - end + case lists:usort(Res) of + [UniqueReply] -> + % Report a universally agreed-upon reply + UniqueReply; + [] -> + {error, badrpc}; + Else -> + % Unclear what to do here -- pick the first error? + % Except try ignoring any {error, not_found} responses + % because we'll always get two of those + hd(Else -- [{error, not_found}]) + end end. choose_node(Key) when is_binary(Key) -> Checksum = erlang:crc32(Key), - Nodes = lists:sort([node()|erlang:nodes()]), + Nodes = lists:sort([node() | erlang:nodes()]), lists:nth(1 + Checksum rem length(Nodes), Nodes); choose_node(Key) -> choose_node(term_to_binary(Key)). -handle_reload_query_servers_req(#httpd{method='POST'}=Req) -> +handle_reload_query_servers_req(#httpd{method = 'POST'} = Req) -> chttpd:validate_ctype(Req, "application/json"), ok = couch_proc_manager:reload(), send_json(Req, 200, {[{ok, true}]}); @@ -263,23 +284,21 @@ handle_reload_query_servers_req(Req) -> handle_uuids_req(Req) -> couch_httpd_misc_handlers:handle_uuids_req(Req). - -handle_up_req(#httpd{method='GET'} = Req) -> +handle_up_req(#httpd{method = 'GET'} = Req) -> case config:get("couchdb", "maintenance_mode") of - "true" -> - send_json(Req, 404, {[{status, maintenance_mode}]}); - "nolb" -> - send_json(Req, 404, {[{status, nolb}]}); - _ -> - {ok, {Status}} = mem3_seeds:get_status(), - case couch_util:get_value(status, Status) of - ok -> - send_json(Req, 200, {Status}); - seeding -> - send_json(Req, 404, {Status}) - end + "true" -> + send_json(Req, 404, {[{status, maintenance_mode}]}); + "nolb" -> + send_json(Req, 404, {[{status, nolb}]}); + _ -> + {ok, {Status}} = mem3_seeds:get_status(), + case couch_util:get_value(status, Status) of + ok -> + send_json(Req, 200, {Status}); + seeding -> + send_json(Req, 404, {Status}) + end end; - handle_up_req(Req) -> send_method_not_allowed(Req, "GET,HEAD"). diff --git a/src/chttpd/src/chttpd_node.erl b/src/chttpd/src/chttpd_node.erl index e92a1e506..7379dba02 100644 --- a/src/chttpd/src/chttpd_node.erl +++ b/src/chttpd/src/chttpd_node.erl @@ -21,18 +21,24 @@ -include_lib("couch/include/couch_db.hrl"). --import(chttpd, - [send_json/2,send_json/3,send_method_not_allowed/2, - send_chunk/2,start_chunked_response/3]). +-import( + chttpd, + [ + send_json/2, send_json/3, + send_method_not_allowed/2, + send_chunk/2, + start_chunked_response/3 + ] +). % Node-specific request handler (_config and _stats) % Support _local meaning this node -handle_node_req(#httpd{path_parts=[_, <<"_local">>]}=Req) -> +handle_node_req(#httpd{path_parts = [_, <<"_local">>]} = Req) -> send_json(Req, 200, {[{name, node()}]}); -handle_node_req(#httpd{path_parts=[A, <<"_local">>|Rest]}=Req) -> - handle_node_req(Req#httpd{path_parts=[A, node()] ++ Rest}); +handle_node_req(#httpd{path_parts = [A, <<"_local">> | Rest]} = Req) -> + handle_node_req(Req#httpd{path_parts = [A, node()] ++ Rest}); % GET /_node/$node/_versions -handle_node_req(#httpd{method='GET', path_parts=[_, _Node, <<"_versions">>]}=Req) -> +handle_node_req(#httpd{method = 'GET', path_parts = [_, _Node, <<"_versions">>]} = Req) -> IcuVer = couch_ejson_compare:get_icu_version(), UcaVer = couch_ejson_compare:get_uca_version(), send_json(Req, 200, #{ @@ -47,45 +53,56 @@ handle_node_req(#httpd{method='GET', path_parts=[_, _Node, <<"_versions">>]}=Req version => couch_server:get_spidermonkey_version() } }); -handle_node_req(#httpd{path_parts=[_, _Node, <<"_versions">>]}=Req) -> +handle_node_req(#httpd{path_parts = [_, _Node, <<"_versions">>]} = Req) -> send_method_not_allowed(Req, "GET"); - % GET /_node/$node/_config -handle_node_req(#httpd{method='GET', path_parts=[_, Node, <<"_config">>]}=Req) -> - Grouped = lists:foldl(fun({{Section, Key}, Value}, Acc) -> - case dict:is_key(Section, Acc) of - true -> - dict:append(Section, {list_to_binary(Key), list_to_binary(Value)}, Acc); - false -> - dict:store(Section, [{list_to_binary(Key), list_to_binary(Value)}], Acc) - end - end, dict:new(), call_node(Node, config, all, [])), - KVs = dict:fold(fun(Section, Values, Acc) -> - [{list_to_binary(Section), {Values}} | Acc] - end, [], Grouped), +handle_node_req(#httpd{method = 'GET', path_parts = [_, Node, <<"_config">>]} = Req) -> + Grouped = lists:foldl( + fun({{Section, Key}, Value}, Acc) -> + case dict:is_key(Section, Acc) of + true -> + dict:append(Section, {list_to_binary(Key), list_to_binary(Value)}, Acc); + false -> + dict:store(Section, [{list_to_binary(Key), list_to_binary(Value)}], Acc) + end + end, + dict:new(), + call_node(Node, config, all, []) + ), + KVs = dict:fold( + fun(Section, Values, Acc) -> + [{list_to_binary(Section), {Values}} | Acc] + end, + [], + Grouped + ), send_json(Req, 200, {KVs}); -handle_node_req(#httpd{path_parts=[_, _Node, <<"_config">>]}=Req) -> +handle_node_req(#httpd{path_parts = [_, _Node, <<"_config">>]} = Req) -> send_method_not_allowed(Req, "GET"); % POST /_node/$node/_config/_reload - Flushes unpersisted config values from RAM -handle_node_req(#httpd{method='POST', path_parts=[_, Node, <<"_config">>, <<"_reload">>]}=Req) -> +handle_node_req( + #httpd{method = 'POST', path_parts = [_, Node, <<"_config">>, <<"_reload">>]} = Req +) -> case call_node(Node, config, reload, []) of ok -> send_json(Req, 200, {[{ok, true}]}); {error, Reason} -> chttpd:send_error(Req, {bad_request, Reason}) end; -handle_node_req(#httpd{path_parts=[_, _Node, <<"_config">>, <<"_reload">>]}=Req) -> +handle_node_req(#httpd{path_parts = [_, _Node, <<"_config">>, <<"_reload">>]} = Req) -> send_method_not_allowed(Req, "POST"); % GET /_node/$node/_config/Section -handle_node_req(#httpd{method='GET', path_parts=[_, Node, <<"_config">>, Section]}=Req) -> - KVs = [{list_to_binary(Key), list_to_binary(Value)} - || {Key, Value} <- call_node(Node, config, get, [Section])], +handle_node_req(#httpd{method = 'GET', path_parts = [_, Node, <<"_config">>, Section]} = Req) -> + KVs = [ + {list_to_binary(Key), list_to_binary(Value)} + || {Key, Value} <- call_node(Node, config, get, [Section]) + ], send_json(Req, 200, {KVs}); -handle_node_req(#httpd{path_parts=[_, _Node, <<"_config">>, _Section]}=Req) -> +handle_node_req(#httpd{path_parts = [_, _Node, <<"_config">>, _Section]} = Req) -> send_method_not_allowed(Req, "GET"); % PUT /_node/$node/_config/Section/Key % "value" -handle_node_req(#httpd{method='PUT', path_parts=[_, Node, <<"_config">>, Section, Key]}=Req) -> +handle_node_req(#httpd{method = 'PUT', path_parts = [_, Node, <<"_config">>, Section, Key]} = Req) -> couch_util:check_config_blacklist(Section), Value = couch_util:trim(chttpd:json_body(Req)), Persist = chttpd:header_value(Req, "X-Couch-Persist") /= "false", @@ -99,34 +116,36 @@ handle_node_req(#httpd{method='PUT', path_parts=[_, Node, <<"_config">>, Section chttpd:send_error(Req, {bad_request, Reason}) end; % GET /_node/$node/_config/Section/Key -handle_node_req(#httpd{method='GET', path_parts=[_, Node, <<"_config">>, Section, Key]}=Req) -> +handle_node_req(#httpd{method = 'GET', path_parts = [_, Node, <<"_config">>, Section, Key]} = Req) -> case call_node(Node, config, get, [Section, Key, undefined]) of - undefined -> - throw({not_found, unknown_config_value}); - Value -> - send_json(Req, 200, list_to_binary(Value)) + undefined -> + throw({not_found, unknown_config_value}); + Value -> + send_json(Req, 200, list_to_binary(Value)) end; % DELETE /_node/$node/_config/Section/Key -handle_node_req(#httpd{method='DELETE',path_parts=[_, Node, <<"_config">>, Section, Key]}=Req) -> +handle_node_req( + #httpd{method = 'DELETE', path_parts = [_, Node, <<"_config">>, Section, Key]} = Req +) -> couch_util:check_config_blacklist(Section), Persist = chttpd:header_value(Req, "X-Couch-Persist") /= "false", case call_node(Node, config, get, [Section, Key, undefined]) of - undefined -> - throw({not_found, unknown_config_value}); - OldValue -> - case call_node(Node, config, delete, [Section, Key, Persist]) of - ok -> - send_json(Req, 200, list_to_binary(OldValue)); - {error, Reason} -> - chttpd:send_error(Req, {bad_request, Reason}) - end + undefined -> + throw({not_found, unknown_config_value}); + OldValue -> + case call_node(Node, config, delete, [Section, Key, Persist]) of + ok -> + send_json(Req, 200, list_to_binary(OldValue)); + {error, Reason} -> + chttpd:send_error(Req, {bad_request, Reason}) + end end; -handle_node_req(#httpd{path_parts=[_, _Node, <<"_config">>, _Section, _Key]}=Req) -> +handle_node_req(#httpd{path_parts = [_, _Node, <<"_config">>, _Section, _Key]} = Req) -> send_method_not_allowed(Req, "GET,PUT,DELETE"); -handle_node_req(#httpd{path_parts=[_, _Node, <<"_config">>, _Section, _Key | _]}=Req) -> +handle_node_req(#httpd{path_parts = [_, _Node, <<"_config">>, _Section, _Key | _]} = Req) -> chttpd:send_error(Req, not_found); % GET /_node/$node/_stats -handle_node_req(#httpd{method='GET', path_parts=[_, Node, <<"_stats">> | Path]}=Req) -> +handle_node_req(#httpd{method = 'GET', path_parts = [_, Node, <<"_stats">> | Path]} = Req) -> flush(Node, Req), Stats0 = call_node(Node, couch_stats, fetch, []), Stats = couch_stats_httpd:transform_stats(Stats0), @@ -134,49 +153,54 @@ handle_node_req(#httpd{method='GET', path_parts=[_, Node, <<"_stats">> | Path]}= EJSON0 = couch_stats_httpd:to_ejson(Nested), EJSON1 = couch_stats_httpd:extract_path(Path, EJSON0), chttpd:send_json(Req, EJSON1); -handle_node_req(#httpd{path_parts=[_, _Node, <<"_stats">>]}=Req) -> +handle_node_req(#httpd{path_parts = [_, _Node, <<"_stats">>]} = Req) -> send_method_not_allowed(Req, "GET"); -handle_node_req(#httpd{method='GET', path_parts=[_, Node, <<"_prometheus">>]}=Req) -> +handle_node_req(#httpd{method = 'GET', path_parts = [_, Node, <<"_prometheus">>]} = Req) -> Metrics = call_node(Node, couch_prometheus_server, scrape, []), Version = call_node(Node, couch_prometheus_server, version, []), - Type = "text/plain; version=" ++ Version, + Type = "text/plain; version=" ++ Version, Header = [{<<"Content-Type">>, ?l2b(Type)}], chttpd:send_response(Req, 200, Header, Metrics); -handle_node_req(#httpd{path_parts=[_, _Node, <<"_prometheus">>]}=Req) -> +handle_node_req(#httpd{path_parts = [_, _Node, <<"_prometheus">>]} = Req) -> send_method_not_allowed(Req, "GET"); % GET /_node/$node/_system -handle_node_req(#httpd{method='GET', path_parts=[_, Node, <<"_system">>]}=Req) -> +handle_node_req(#httpd{method = 'GET', path_parts = [_, Node, <<"_system">>]} = Req) -> Stats = call_node(Node, chttpd_node, get_stats, []), EJSON = couch_stats_httpd:to_ejson(Stats), send_json(Req, EJSON); -handle_node_req(#httpd{path_parts=[_, _Node, <<"_system">>]}=Req) -> +handle_node_req(#httpd{path_parts = [_, _Node, <<"_system">>]} = Req) -> send_method_not_allowed(Req, "GET"); % POST /_node/$node/_restart -handle_node_req(#httpd{method='POST', path_parts=[_, Node, <<"_restart">>]}=Req) -> +handle_node_req(#httpd{method = 'POST', path_parts = [_, Node, <<"_restart">>]} = Req) -> call_node(Node, init, restart, []), send_json(Req, 200, {[{ok, true}]}); -handle_node_req(#httpd{path_parts=[_, _Node, <<"_restart">>]}=Req) -> +handle_node_req(#httpd{path_parts = [_, _Node, <<"_restart">>]} = Req) -> send_method_not_allowed(Req, "POST"); -handle_node_req(#httpd{path_parts=[_, Node | PathParts], - mochi_req=MochiReq0}) -> +handle_node_req(#httpd{ + path_parts = [_, Node | PathParts], + mochi_req = MochiReq0 +}) -> % strip /_node/{node} from Req0 before descending further RawUri = MochiReq0:get(raw_path), {_, Query, Fragment} = mochiweb_util:urlsplit_path(RawUri), NewPath0 = "/" ++ lists:join("/", [couch_util:url_encode(P) || P <- PathParts]), NewRawPath = mochiweb_util:urlunsplit_path({NewPath0, Query, Fragment}), MaxSize = chttpd_util:get_chttpd_config_integer( - "max_http_request_size", 4294967296), + "max_http_request_size", 4294967296 + ), NewOpts = [{body, MochiReq0:recv_body(MaxSize)} | MochiReq0:get(opts)], Ref = erlang:make_ref(), - MochiReq = mochiweb_request:new({remote, self(), Ref}, - NewOpts, - MochiReq0:get(method), - NewRawPath, - MochiReq0:get(version), - MochiReq0:get(headers)), + MochiReq = mochiweb_request:new( + {remote, self(), Ref}, + NewOpts, + MochiReq0:get(method), + NewRawPath, + MochiReq0:get(version), + MochiReq0:get(headers) + ), call_node(Node, couch_httpd, handle_request, [MochiReq]), recv_loop(Ref, MochiReq0); -handle_node_req(#httpd{path_parts=[_]}=Req) -> +handle_node_req(#httpd{path_parts = [_]} = Req) -> chttpd:send_error(Req, {bad_request, <<"Incomplete path to _node request">>}); handle_node_req(Req) -> chttpd:send_error(Req, not_found). @@ -207,12 +231,13 @@ recv_loop(Ref, ReqResp) -> end. call_node(Node0, Mod, Fun, Args) when is_binary(Node0) -> - Node1 = try - list_to_existing_atom(?b2l(Node0)) - catch - error:badarg -> - throw({not_found, <<"no such node: ", Node0/binary>>}) - end, + Node1 = + try + list_to_existing_atom(?b2l(Node0)) + catch + error:badarg -> + throw({not_found, <<"no such node: ", Node0/binary>>}) + end, call_node(Node1, Mod, Fun, Args); call_node(Node, Mod, Fun, Args) when is_atom(Node) -> case rpc:call(Node, Mod, Fun, Args) of @@ -232,15 +257,33 @@ flush(Node, Req) -> end. get_stats() -> - Other = erlang:memory(system) - lists:sum([X || {_,X} <- - erlang:memory([atom, code, binary, ets])]), - Memory = [{other, Other} | erlang:memory([atom, atom_used, processes, - processes_used, binary, code, ets])], + Other = + erlang:memory(system) - + lists:sum([ + X + || {_, X} <- + erlang:memory([atom, code, binary, ets]) + ]), + Memory = [ + {other, Other} + | erlang:memory([ + atom, + atom_used, + processes, + processes_used, + binary, + code, + ets + ]) + ], {NumberOfGCs, WordsReclaimed, _} = statistics(garbage_collection), {{input, Input}, {output, Output}} = statistics(io), {CF, CDU} = db_pid_stats(), - MessageQueues0 = [{couch_file, {CF}}, {couch_db_updater, {CDU}}, - {couch_server, couch_server:aggregate_queue_len()}], + MessageQueues0 = [ + {couch_file, {CF}}, + {couch_db_updater, {CDU}}, + {couch_server, couch_server:aggregate_queue_len()} + ], MessageQueues = MessageQueues0 ++ message_queues(registered()), {SQ, DCQ} = run_queues(), [ @@ -283,13 +326,15 @@ db_pid_stats(Mod, Candidates) -> {Mod, init, 1} -> case proplists:get_value(message_queue_len, PI) of undefined -> Acc; - Len -> [Len|Acc] + Len -> [Len | Acc] end; - _ -> + _ -> Acc end end - end, [], Candidates + end, + [], + Candidates ), format_pid_stats(Mailboxes). @@ -308,17 +353,23 @@ format_pid_stats(Mailboxes) -> ]. get_distribution_stats() -> - lists:map(fun({Node, Socket}) -> - {ok, Stats} = inet:getstat(Socket), - {Node, {Stats}} - end, erlang:system_info(dist_ctrl)). + lists:map( + fun({Node, Socket}) -> + {ok, Stats} = inet:getstat(Socket), + {Node, {Stats}} + end, + erlang:system_info(dist_ctrl) + ). message_queues(Registered) -> - lists:map(fun(Name) -> - Type = message_queue_len, - {Type, Length} = process_info(whereis(Name), Type), - {Name, Length} - end, Registered). + lists:map( + fun(Name) -> + Type = message_queue_len, + {Type, Length} = process_info(whereis(Name), Type), + {Name, Length} + end, + Registered + ). %% Workaround for https://bugs.erlang.org/browse/ERL-1355 run_queues() -> diff --git a/src/chttpd/src/chttpd_plugin.erl b/src/chttpd/src/chttpd_plugin.erl index 7ab458170..03d8ad6ac 100644 --- a/src/chttpd/src/chttpd_plugin.erl +++ b/src/chttpd/src/chttpd_plugin.erl @@ -48,7 +48,8 @@ before_response(HttpReq0, Code0, Headers0, Value0) -> before_serve_file(Req0, Code0, Headers0, RelativePath0, DocumentRoot0) -> [HttpReq, Code, Headers, RelativePath, DocumentRoot] = with_pipe(before_serve_file, [ - Req0, Code0, Headers0, RelativePath0, DocumentRoot0]), + Req0, Code0, Headers0, RelativePath0, DocumentRoot0 + ]), {ok, {HttpReq, Code, Headers, RelativePath, DocumentRoot}}. %% ------------------------------------------------------------------ diff --git a/src/chttpd/src/chttpd_prefer_header.erl b/src/chttpd/src/chttpd_prefer_header.erl index de2660399..dbce54e65 100644 --- a/src/chttpd/src/chttpd_prefer_header.erl +++ b/src/chttpd/src/chttpd_prefer_header.erl @@ -18,44 +18,44 @@ maybe_return_minimal/2 ]). - -include_lib("couch/include/couch_db.hrl"). - -define(DEFAULT_PREFER_MINIMAL, "Cache-Control, Content-Length, Content-Range, " - "Content-Type, ETag, Server, Transfer-Encoding, Vary"). - + "Content-Type, ETag, Server, Transfer-Encoding, Vary" +). maybe_return_minimal(#httpd{mochi_req = MochiReq}, Headers) -> case get_prefer_header(MochiReq) of - "return=minimal" -> + "return=minimal" -> filter_headers(Headers, get_header_list()); - _ -> + _ -> Headers end. - get_prefer_header(Req) -> case Req:get_header_value("Prefer") of Value when is_list(Value) -> string:to_lower(Value); - undefined -> + undefined -> undefined end. - filter_headers(Headers, IncludeList) -> - lists:filter(fun({HeaderName, _}) -> - lists:member(HeaderName, IncludeList) - end, Headers). - + lists:filter( + fun({HeaderName, _}) -> + lists:member(HeaderName, IncludeList) + end, + Headers + ). get_header_list() -> - SectionStr = config:get("chttpd", - "prefer_minimal", ?DEFAULT_PREFER_MINIMAL), + SectionStr = config:get( + "chttpd", + "prefer_minimal", + ?DEFAULT_PREFER_MINIMAL + ), split_list(SectionStr). - split_list(S) -> - re:split(S, "\\s*,\\s*", [trim, {return, list}]). + re:split(S, "\\s*,\\s*", [trim, {return, list}]). diff --git a/src/chttpd/src/chttpd_rewrite.erl b/src/chttpd/src/chttpd_rewrite.erl index 22edb3fac..4e77597d4 100644 --- a/src/chttpd/src/chttpd_rewrite.erl +++ b/src/chttpd/src/chttpd_rewrite.erl @@ -12,7 +12,6 @@ % % bind_path is based on bind method from Webmachine - %% @doc Module for URL rewriting by pattern matching. -module(chttpd_rewrite). @@ -25,8 +24,7 @@ -define(SEPARATOR, $\/). -define(MATCH_ALL, {bind, <<"*">>}). - -handle_rewrite_req(#httpd{}=Req, Db, DDoc) -> +handle_rewrite_req(#httpd{} = Req, Db, DDoc) -> RewritesSoFar = erlang:get(?REWRITE_COUNT), MaxRewrites = chttpd_util:get_chttpd_config_integer("rewrite_limit", 100), case RewritesSoFar >= MaxRewrites of @@ -41,36 +39,45 @@ handle_rewrite_req(#httpd{}=Req, Db, DDoc) -> Rules when is_binary(Rules) -> case couch_query_servers:rewrite(Req, Db, DDoc) of undefined -> - chttpd:send_error(Req, 404, <<"rewrite_error">>, - <<"Invalid path.">>); + chttpd:send_error( + Req, + 404, + <<"rewrite_error">>, + <<"Invalid path.">> + ); Rewrite -> do_rewrite(Req, Rewrite) end; undefined -> - chttpd:send_error(Req, 404, <<"rewrite_error">>, - <<"Invalid path.">>) + chttpd:send_error( + Req, + 404, + <<"rewrite_error">>, + <<"Invalid path.">> + ) end. - -get_rules(#doc{body={Props}}) -> +get_rules(#doc{body = {Props}}) -> couch_util:get_value(<<"rewrites">>, Props). - -do_rewrite(#httpd{mochi_req=MochiReq}=Req, {Props}=Rewrite) when is_list(Props) -> +do_rewrite(#httpd{mochi_req = MochiReq} = Req, {Props} = Rewrite) when is_list(Props) -> case couch_util:get_value(<<"code">>, Props) of undefined -> Method = rewrite_method(Req, Rewrite), Headers = rewrite_headers(Req, Rewrite), Path = ?b2l(rewrite_path(Req, Rewrite)), - NewMochiReq = mochiweb_request:new(MochiReq:get(socket), - Method, - Path, - MochiReq:get(version), - Headers), - Body = case couch_util:get_value(<<"body">>, Props) of - undefined -> erlang:get(mochiweb_request_body); - B -> B - end, + NewMochiReq = mochiweb_request:new( + MochiReq:get(socket), + Method, + Path, + MochiReq:get(version), + Headers + ), + Body = + case couch_util:get_value(<<"body">>, Props) of + undefined -> erlang:get(mochiweb_request_body); + B -> B + end, NewMochiReq:cleanup(), case Body of undefined -> []; @@ -86,37 +93,49 @@ do_rewrite(#httpd{mochi_req=MochiReq}=Req, {Props}=Rewrite) when is_list(Props) undefined -> []; {H1} -> H1 end, - rewrite_body(Rewrite)) + rewrite_body(Rewrite) + ) end; -do_rewrite(#httpd{method=Method, - path_parts=[_DbName, <<"_design">>, _DesignName, _Rewrite|PathParts], - mochi_req=MochiReq}=Req, - Rules) when is_list(Rules) -> +do_rewrite( + #httpd{ + method = Method, + path_parts = [_DbName, <<"_design">>, _DesignName, _Rewrite | PathParts], + mochi_req = MochiReq + } = Req, + Rules +) when is_list(Rules) -> % create dispatch list from rules Prefix = path_prefix(Req), QueryList = lists:map(fun decode_query_value/1, chttpd:qs(Req)), - DispatchList = [make_rule(Rule) || {Rule} <- Rules], + DispatchList = [make_rule(Rule) || {Rule} <- Rules], Method1 = couch_util:to_binary(Method), %% get raw path by matching url to a rule. - RawPath = case try_bind_path(DispatchList, Method1, - PathParts, QueryList) of - no_dispatch_path -> - throw(not_found); - {NewPathParts, Bindings} -> - Parts = [quote_plus(X) || X <- NewPathParts], - - % build new path, reencode query args, eventually convert - % them to json - Bindings1 = maybe_encode_bindings(Bindings), - Path = iolist_to_binary([ - string:join(Parts, [?SEPARATOR]), - [["?", mochiweb_util:urlencode(Bindings1)] || Bindings1 =/= []] - ]), - - % if path is relative detect it and rewrite path - safe_relative_path(Prefix, Path) + RawPath = + case + try_bind_path( + DispatchList, + Method1, + PathParts, + QueryList + ) + of + no_dispatch_path -> + throw(not_found); + {NewPathParts, Bindings} -> + Parts = [quote_plus(X) || X <- NewPathParts], + + % build new path, reencode query args, eventually convert + % them to json + Bindings1 = maybe_encode_bindings(Bindings), + Path = iolist_to_binary([ + string:join(Parts, [?SEPARATOR]), + [["?", mochiweb_util:urlencode(Bindings1)] || Bindings1 =/= []] + ]), + + % if path is relative detect it and rewrite path + safe_relative_path(Prefix, Path) end, % normalize final path (fix levels "." and "..") @@ -125,30 +144,32 @@ do_rewrite(#httpd{method=Method, couch_log:debug("rewrite to ~p ~n", [RawPath1]), % build a new mochiweb request - MochiReq1 = mochiweb_request:new(MochiReq:get(socket), - MochiReq:get(method), - RawPath1, - MochiReq:get(version), - MochiReq:get(headers)), + MochiReq1 = mochiweb_request:new( + MochiReq:get(socket), + MochiReq:get(method), + RawPath1, + MochiReq:get(version), + MochiReq:get(headers) + ), % cleanup, It force mochiweb to reparse raw uri. MochiReq1:cleanup(), chttpd:handle_request_int(MochiReq1). - -rewrite_method(#httpd{method=Method}, {Props}) -> +rewrite_method(#httpd{method = Method}, {Props}) -> DefaultMethod = couch_util:to_binary(Method), couch_util:get_value(<<"method">>, Props, DefaultMethod). -rewrite_path(#httpd{}=Req, {Props}=Rewrite) -> +rewrite_path(#httpd{} = Req, {Props} = Rewrite) -> Prefix = path_prefix(Req), - RewritePath = case couch_util:get_value(<<"path">>, Props) of - undefined -> - throw({<<"rewrite_error">>, - <<"Rewrite result must produce a new path.">>}); - P -> P - end, + RewritePath = + case couch_util:get_value(<<"path">>, Props) of + undefined -> + throw({<<"rewrite_error">>, <<"Rewrite result must produce a new path.">>}); + P -> + P + end, SafeRelativePath = safe_relative_path(Prefix, RewritePath), NormalizedPath = normalize_path(SafeRelativePath), QueryParams = rewrite_query_params(Req, Rewrite), @@ -159,30 +180,33 @@ rewrite_path(#httpd{}=Req, {Props}=Rewrite) -> <<NormalizedPath/binary, "?", QueryParams/binary>> end. -rewrite_query_params(#httpd{}=Req, {Props}) -> +rewrite_query_params(#httpd{} = Req, {Props}) -> RequestQS = chttpd:qs(Req), - RewriteQS = case couch_util:get_value(<<"query">>, Props) of - undefined -> RequestQS; - {V} -> V - end, + RewriteQS = + case couch_util:get_value(<<"query">>, Props) of + undefined -> RequestQS; + {V} -> V + end, RewriteQSEsc = [{chttpd:quote(K), chttpd:quote(V)} || {K, V} <- RewriteQS], iolist_to_binary(string:join([[K, "=", V] || {K, V} <- RewriteQSEsc], "&")). -rewrite_headers(#httpd{mochi_req=MochiReq}, {Props}) -> +rewrite_headers(#httpd{mochi_req = MochiReq}, {Props}) -> case couch_util:get_value(<<"headers">>, Props) of undefined -> MochiReq:get(headers); {H} -> mochiweb_headers:enter_from_list( lists:map(fun({Key, Val}) -> {?b2l(Key), ?b2l(Val)} end, H), - MochiReq:get(headers)) + MochiReq:get(headers) + ) end. rewrite_body({Props}) -> - Body = case couch_util:get_value(<<"body">>, Props) of - undefined -> erlang:get(mochiweb_request_body); - B -> B - end, + Body = + case couch_util:get_value(<<"body">>, Props) of + undefined -> erlang:get(mochiweb_request_body); + B -> B + end, case Body of undefined -> []; @@ -191,8 +215,7 @@ rewrite_body({Props}) -> Body end. - -path_prefix(#httpd{path_parts=[DbName, <<"_design">>, DesignName | _]}) -> +path_prefix(#httpd{path_parts = [DbName, <<"_design">>, DesignName | _]}) -> EscapedDesignName = ?l2b(couch_util:url_encode(DesignName)), EscapedDbName = ?l2b(couch_util:url_encode(DbName)), DesignId = <<"_design/", EscapedDesignName/binary>>, @@ -207,7 +230,6 @@ safe_relative_path(Prefix, Path) -> <<Prefix/binary, "/", V1/binary>> end. - quote_plus({bind, X}) -> mochiweb_util:quote_plus(X); quote_plus(X) -> @@ -217,7 +239,7 @@ quote_plus(X) -> %% 404 error not_found is raised try_bind_path([], _Method, _PathParts, _QueryList) -> no_dispatch_path; -try_bind_path([Dispatch|Rest], Method, PathParts, QueryList) -> +try_bind_path([Dispatch | Rest], Method, PathParts, QueryList) -> [{PathParts1, Method1}, RedirectPath, QueryArgs, Formats] = Dispatch, case bind_method(Method1, Method) of true -> @@ -226,22 +248,35 @@ try_bind_path([Dispatch|Rest], Method, PathParts, QueryList) -> Bindings1 = Bindings ++ QueryList, % we parse query args from the rule and fill % it eventually with bindings vars - QueryArgs1 = make_query_list(QueryArgs, Bindings1, - Formats, []), + QueryArgs1 = make_query_list( + QueryArgs, + Bindings1, + Formats, + [] + ), % remove params in QueryLists1 that are already in % QueryArgs1 - Bindings2 = lists:foldl(fun({K, V}, Acc) -> - K1 = to_binding(K), - KV = case couch_util:get_value(K1, QueryArgs1) of - undefined -> [{K1, V}]; - _V1 -> [] + Bindings2 = lists:foldl( + fun({K, V}, Acc) -> + K1 = to_binding(K), + KV = + case couch_util:get_value(K1, QueryArgs1) of + undefined -> [{K1, V}]; + _V1 -> [] + end, + Acc ++ KV end, - Acc ++ KV - end, [], Bindings1), + [], + Bindings1 + ), FinalBindings = Bindings2 ++ QueryArgs1, - NewPathParts = make_new_path(RedirectPath, FinalBindings, - Remaining, []), + NewPathParts = make_new_path( + RedirectPath, + FinalBindings, + Remaining, + [] + ), {NewPathParts, FinalBindings}; fail -> try_bind_path(Rest, Method, PathParts, QueryList) @@ -255,37 +290,51 @@ try_bind_path([Dispatch|Rest], Method, PathParts, QueryList) -> %% passed in url. make_query_list([], _Bindings, _Formats, Acc) -> Acc; -make_query_list([{Key, {Value}}|Rest], Bindings, Formats, Acc) -> +make_query_list([{Key, {Value}} | Rest], Bindings, Formats, Acc) -> Value1 = {Value}, - make_query_list(Rest, Bindings, Formats, [{to_binding(Key), Value1}|Acc]); -make_query_list([{Key, Value}|Rest], Bindings, Formats, Acc) when is_binary(Value) -> + make_query_list(Rest, Bindings, Formats, [{to_binding(Key), Value1} | Acc]); +make_query_list([{Key, Value} | Rest], Bindings, Formats, Acc) when is_binary(Value) -> Value1 = replace_var(Value, Bindings, Formats), - make_query_list(Rest, Bindings, Formats, [{to_binding(Key), Value1}|Acc]); -make_query_list([{Key, Value}|Rest], Bindings, Formats, Acc) when is_list(Value) -> + make_query_list(Rest, Bindings, Formats, [{to_binding(Key), Value1} | Acc]); +make_query_list([{Key, Value} | Rest], Bindings, Formats, Acc) when is_list(Value) -> Value1 = replace_var(Value, Bindings, Formats), - make_query_list(Rest, Bindings, Formats, [{to_binding(Key), Value1}|Acc]); -make_query_list([{Key, Value}|Rest], Bindings, Formats, Acc) -> - make_query_list(Rest, Bindings, Formats, [{to_binding(Key), Value}|Acc]). + make_query_list(Rest, Bindings, Formats, [{to_binding(Key), Value1} | Acc]); +make_query_list([{Key, Value} | Rest], Bindings, Formats, Acc) -> + make_query_list(Rest, Bindings, Formats, [{to_binding(Key), Value} | Acc]). -replace_var(<<"*">>=Value, Bindings, Formats) -> +replace_var(<<"*">> = Value, Bindings, Formats) -> get_var(Value, Bindings, Value, Formats); replace_var(<<":", Var/binary>> = Value, Bindings, Formats) -> get_var(Var, Bindings, Value, Formats); replace_var(Value, _Bindings, _Formats) when is_binary(Value) -> Value; replace_var(Value, Bindings, Formats) when is_list(Value) -> - lists:reverse(lists:foldl(fun - (<<":", Var/binary>>=Value1, Acc) -> - [get_var(Var, Bindings, Value1, Formats)|Acc]; + lists:reverse( + lists:foldl( + fun + (<<":", Var/binary>> = Value1, Acc) -> + [get_var(Var, Bindings, Value1, Formats) | Acc]; (Value1, Acc) -> - [Value1|Acc] - end, [], Value)); + [Value1 | Acc] + end, + [], + Value + ) + ); replace_var(Value, _Bindings, _Formats) -> Value. maybe_json(Key, Value) -> - case lists:member(Key, [<<"key">>, <<"startkey">>, <<"start_key">>, - <<"endkey">>, <<"end_key">>, <<"keys">>]) of + case + lists:member(Key, [ + <<"key">>, + <<"startkey">>, + <<"start_key">>, + <<"endkey">>, + <<"end_key">>, + <<"keys">> + ]) + of true -> ?JSON_ENCODE(Value); false -> @@ -300,7 +349,7 @@ get_var(VarName, Props, Default, Formats) -> maybe_format(VarName, Value, Formats) -> case couch_util:get_value(VarName, Formats) of undefined -> - Value; + Value; Format -> format(Format, Value) end. @@ -325,7 +374,7 @@ format(<<"bool">>, Value) when is_list(Value) -> _ -> Value end; format(_Format, Value) -> - Value. + Value. %% doc: build new patch from bindings. bindings are query args %% (+ dynamic query rewritten if needed) and bindings found in @@ -335,19 +384,18 @@ make_new_path([], _Bindings, _Remaining, Acc) -> make_new_path([?MATCH_ALL], _Bindings, Remaining, Acc) -> Acc1 = lists:reverse(Acc) ++ Remaining, Acc1; -make_new_path([?MATCH_ALL|_Rest], _Bindings, Remaining, Acc) -> +make_new_path([?MATCH_ALL | _Rest], _Bindings, Remaining, Acc) -> Acc1 = lists:reverse(Acc) ++ Remaining, Acc1; -make_new_path([{bind, P}|Rest], Bindings, Remaining, Acc) -> - P2 = case couch_util:get_value({bind, P}, Bindings) of - undefined -> << "undefined">>; - P1 -> - iolist_to_binary(P1) - end, - make_new_path(Rest, Bindings, Remaining, [P2|Acc]); -make_new_path([P|Rest], Bindings, Remaining, Acc) -> - make_new_path(Rest, Bindings, Remaining, [P|Acc]). - +make_new_path([{bind, P} | Rest], Bindings, Remaining, Acc) -> + P2 = + case couch_util:get_value({bind, P}, Bindings) of + undefined -> <<"undefined">>; + P1 -> iolist_to_binary(P1) + end, + make_new_path(Rest, Bindings, Remaining, [P2 | Acc]); +make_new_path([P | Rest], Bindings, Remaining, Acc) -> + make_new_path(Rest, Bindings, Remaining, [P | Acc]). %% @doc If method of the query fith the rule method. If the %% method rule is '*', which is the default, all @@ -360,7 +408,6 @@ bind_method({bind, Method}, Method) -> bind_method(_, _) -> false. - %% @doc bind path. Using the rule from we try to bind variables given %% to the current url by pattern matching bind_path([], [], Bindings) -> @@ -369,63 +416,65 @@ bind_path([?MATCH_ALL], Rest, Bindings) when is_list(Rest) -> {ok, Rest, Bindings}; bind_path(_, [], _) -> fail; -bind_path([{bind, Token}|RestToken],[Match|RestMatch],Bindings) -> - bind_path(RestToken, RestMatch, [{{bind, Token}, Match}|Bindings]); -bind_path([Token|RestToken], [Token|RestMatch], Bindings) -> +bind_path([{bind, Token} | RestToken], [Match | RestMatch], Bindings) -> + bind_path(RestToken, RestMatch, [{{bind, Token}, Match} | Bindings]); +bind_path([Token | RestToken], [Token | RestMatch], Bindings) -> bind_path(RestToken, RestMatch, Bindings); bind_path(_, _, _) -> fail. - %% normalize path. -normalize_path(Path) when is_binary(Path)-> +normalize_path(Path) when is_binary(Path) -> normalize_path(?b2l(Path)); -normalize_path(Path) when is_list(Path)-> +normalize_path(Path) when is_list(Path) -> Segments = normalize_path1(string:tokens(Path, "/"), []), NormalizedPath = string:join(Segments, [?SEPARATOR]), iolist_to_binary(["/", NormalizedPath]). - normalize_path1([], Acc) -> lists:reverse(Acc); -normalize_path1([".."|Rest], Acc) -> - Acc1 = case Acc of - [] -> [".."|Acc]; - [T|_] when T =:= ".." -> [".."|Acc]; - [_|R] -> R - end, +normalize_path1([".." | Rest], Acc) -> + Acc1 = + case Acc of + [] -> [".." | Acc]; + [T | _] when T =:= ".." -> [".." | Acc]; + [_ | R] -> R + end, normalize_path1(Rest, Acc1); -normalize_path1(["."|Rest], Acc) -> +normalize_path1(["." | Rest], Acc) -> normalize_path1(Rest, Acc); -normalize_path1([Path|Rest], Acc) -> - normalize_path1(Rest, [Path|Acc]). - +normalize_path1([Path | Rest], Acc) -> + normalize_path1(Rest, [Path | Acc]). %% @doc transform json rule in erlang for pattern matching make_rule(Rule) -> - Method = case couch_util:get_value(<<"method">>, Rule) of - undefined -> ?MATCH_ALL; - M -> to_binding(M) - end, - QueryArgs = case couch_util:get_value(<<"query">>, Rule) of - undefined -> []; - {Args} -> Args + Method = + case couch_util:get_value(<<"method">>, Rule) of + undefined -> ?MATCH_ALL; + M -> to_binding(M) end, - FromParts = case couch_util:get_value(<<"from">>, Rule) of - undefined -> [?MATCH_ALL]; - From -> - parse_path(From) + QueryArgs = + case couch_util:get_value(<<"query">>, Rule) of + undefined -> []; + {Args} -> Args end, - ToParts = case couch_util:get_value(<<"to">>, Rule) of - undefined -> - throw({error, invalid_rewrite_target}); - To -> - parse_path(To) + FromParts = + case couch_util:get_value(<<"from">>, Rule) of + undefined -> [?MATCH_ALL]; + From -> parse_path(From) + end, + ToParts = + case couch_util:get_value(<<"to">>, Rule) of + undefined -> + throw({error, invalid_rewrite_target}); + To -> + parse_path(To) + end, + Formats = + case couch_util:get_value(<<"formats">>, Rule) of + undefined -> []; + {Fmts} -> Fmts end, - Formats = case couch_util:get_value(<<"formats">>, Rule) of - undefined -> []; - {Fmts} -> Fmts - end, [{FromParts, Method}, ToParts, QueryArgs, Formats]. parse_path(Path) -> @@ -437,43 +486,59 @@ parse_path(Path) -> %% in erlang atom. path_to_list([], Acc, _DotDotCount) -> lists:reverse(Acc); -path_to_list([<<>>|R], Acc, DotDotCount) -> +path_to_list([<<>> | R], Acc, DotDotCount) -> path_to_list(R, Acc, DotDotCount); -path_to_list([<<"*">>|R], Acc, DotDotCount) -> - path_to_list(R, [?MATCH_ALL|Acc], DotDotCount); -path_to_list([<<"..">>|R], Acc, DotDotCount) when DotDotCount == 2 -> +path_to_list([<<"*">> | R], Acc, DotDotCount) -> + path_to_list(R, [?MATCH_ALL | Acc], DotDotCount); +path_to_list([<<"..">> | R], Acc, DotDotCount) when DotDotCount == 2 -> case chttpd_util:get_chttpd_config_boolean("secure_rewrites", true) of false -> - path_to_list(R, [<<"..">>|Acc], DotDotCount+1); + path_to_list(R, [<<"..">> | Acc], DotDotCount + 1); true -> - couch_log:notice("insecure_rewrite_rule ~p blocked", - [lists:reverse(Acc) ++ [<<"..">>] ++ R]), + couch_log:notice( + "insecure_rewrite_rule ~p blocked", + [lists:reverse(Acc) ++ [<<"..">>] ++ R] + ), throw({insecure_rewrite_rule, "too many ../.. segments"}) end; -path_to_list([<<"..">>|R], Acc, DotDotCount) -> - path_to_list(R, [<<"..">>|Acc], DotDotCount+1); -path_to_list([P|R], Acc, DotDotCount) -> - P1 = case P of - <<":", Var/binary>> -> - to_binding(Var); - _ -> P - end, - path_to_list(R, [P1|Acc], DotDotCount). +path_to_list([<<"..">> | R], Acc, DotDotCount) -> + path_to_list(R, [<<"..">> | Acc], DotDotCount + 1); +path_to_list([P | R], Acc, DotDotCount) -> + P1 = + case P of + <<":", Var/binary>> -> + to_binding(Var); + _ -> + P + end, + path_to_list(R, [P1 | Acc], DotDotCount). maybe_encode_bindings([]) -> []; maybe_encode_bindings(Props) -> - lists:foldl(fun + lists:foldl( + fun ({{bind, <<"*">>}, _V}, Acc) -> Acc; ({{bind, K}, V}, Acc) -> V1 = iolist_to_binary(maybe_json(K, V)), - [{K, V1}|Acc] - end, [], Props). - -decode_query_value({K,V}) -> - case lists:member(K, ["key", "startkey", "start_key", - "endkey", "end_key", "keys"]) of + [{K, V1} | Acc] + end, + [], + Props + ). + +decode_query_value({K, V}) -> + case + lists:member(K, [ + "key", + "startkey", + "start_key", + "endkey", + "end_key", + "keys" + ]) + of true -> {to_binding(K), ?JSON_DECODE(V)}; false -> diff --git a/src/chttpd/src/chttpd_show.erl b/src/chttpd/src/chttpd_show.erl index c2c37c66d..e798a98d6 100644 --- a/src/chttpd/src/chttpd_show.erl +++ b/src/chttpd/src/chttpd_show.erl @@ -24,17 +24,20 @@ maybe_open_doc(Db, DocId, Options) -> case fabric:open_doc(Db, DocId, Options) of - {ok, Doc} -> - chttpd_stats:incr_reads(), - Doc; - {not_found, _} -> - nil + {ok, Doc} -> + chttpd_stats:incr_reads(), + Doc; + {not_found, _} -> + nil end. -handle_doc_show_req(#httpd{ - path_parts=[_, _, _, _, ShowName, DocId] - }=Req, Db, DDoc) -> - +handle_doc_show_req( + #httpd{ + path_parts = [_, _, _, _, ShowName, DocId] + } = Req, + Db, + DDoc +) -> % open the doc Options = [conflicts, {user_ctx, Req#httpd.user_ctx}], Doc = maybe_open_doc(Db, DocId, Options), @@ -42,13 +45,15 @@ handle_doc_show_req(#httpd{ % we don't handle revs here b/c they are an internal api % returns 404 if there is no doc with DocId handle_doc_show(Req, Db, DDoc, ShowName, Doc, DocId); - -handle_doc_show_req(#httpd{ - path_parts=[_, _, _, _, ShowName, DocId|Rest] - }=Req, Db, DDoc) -> - - DocParts = [DocId|Rest], - DocId1 = ?l2b(string:join([?b2l(P)|| P <- DocParts], "/")), +handle_doc_show_req( + #httpd{ + path_parts = [_, _, _, _, ShowName, DocId | Rest] + } = Req, + Db, + DDoc +) -> + DocParts = [DocId | Rest], + DocId1 = ?l2b(string:join([?b2l(P) || P <- DocParts], "/")), % open the doc Options = [conflicts, {user_ctx, Req#httpd.user_ctx}], @@ -57,13 +62,15 @@ handle_doc_show_req(#httpd{ % we don't handle revs here b/c they are an internal api % pass 404 docs to the show function handle_doc_show(Req, Db, DDoc, ShowName, Doc, DocId1); - -handle_doc_show_req(#httpd{ - path_parts=[_, _, _, _, ShowName] - }=Req, Db, DDoc) -> +handle_doc_show_req( + #httpd{ + path_parts = [_, _, _, _, ShowName] + } = Req, + Db, + DDoc +) -> % with no docid the doc is nil handle_doc_show(Req, Db, DDoc, ShowName, nil); - handle_doc_show_req(Req, _Db, _DDoc) -> chttpd:send_error(Req, 404, <<"show_error">>, <<"Invalid path.">>). @@ -79,21 +86,25 @@ handle_doc_show(Req, Db, DDoc, ShowName, Doc, DocId) -> JsonReq = chttpd_external:json_req_obj(Req, Db, DocId), JsonDoc = couch_query_servers:json_doc(Doc), [<<"resp">>, ExternalResp] = - couch_query_servers:ddoc_prompt(DDoc, [<<"shows">>, ShowName], - [JsonDoc, JsonReq]), + couch_query_servers:ddoc_prompt( + DDoc, + [<<"shows">>, ShowName], + [JsonDoc, JsonReq] + ), JsonResp = apply_etag(ExternalResp, CurrentEtag), chttpd_external:send_external_response(Req, JsonResp) end). - -show_etag(#httpd{user_ctx=UserCtx}=Req, Doc, DDoc, More) -> +show_etag(#httpd{user_ctx = UserCtx} = Req, Doc, DDoc, More) -> Accept = chttpd:header_value(Req, "Accept"), - DocPart = case Doc of - nil -> nil; - Doc -> chttpd:doc_etag(Doc) - end, - couch_httpd:make_etag({couch_httpd:doc_etag(DDoc), DocPart, Accept, - UserCtx#user_ctx.roles, More}). + DocPart = + case Doc of + nil -> nil; + Doc -> chttpd:doc_etag(Doc) + end, + couch_httpd:make_etag({ + couch_httpd:doc_etag(DDoc), DocPart, Accept, UserCtx#user_ctx.roles, More + }). % /db/_design/foo/update/bar/docid % updates a doc based on a request @@ -101,19 +112,25 @@ show_etag(#httpd{user_ctx=UserCtx}=Req, Doc, DDoc, More) -> % % anything but GET % send_method_not_allowed(Req, "POST,PUT,DELETE,ETC"); -handle_doc_update_req(#httpd{ - path_parts=[_, _, _, _, UpdateName] - }=Req, Db, DDoc) -> +handle_doc_update_req( + #httpd{ + path_parts = [_, _, _, _, UpdateName] + } = Req, + Db, + DDoc +) -> send_doc_update_response(Req, Db, DDoc, UpdateName, nil, null); - -handle_doc_update_req(#httpd{ - path_parts=[_, _, _, _, UpdateName | DocIdParts] - }=Req, Db, DDoc) -> +handle_doc_update_req( + #httpd{ + path_parts = [_, _, _, _, UpdateName | DocIdParts] + } = Req, + Db, + DDoc +) -> DocId = ?l2b(string:join([?b2l(P) || P <- DocIdParts], "/")), Options = [conflicts, {user_ctx, Req#httpd.user_ctx}], Doc = maybe_open_doc(Db, DocId, Options), send_doc_update_response(Req, Db, DDoc, UpdateName, Doc, DocId); - handle_doc_update_req(Req, _Db, _DDoc) -> chttpd:send_error(Req, 404, <<"update_error">>, <<"Invalid path.">>). @@ -125,74 +142,107 @@ send_doc_update_response(Req, Db, DDoc, UpdateName, Doc, DocId) -> Cmd = [<<"updates">>, UpdateName], W = chttpd:qs_value(Req, "w", integer_to_list(mem3:quorum(Db))), UpdateResp = couch_query_servers:ddoc_prompt(DDoc, Cmd, [JsonDoc, JsonReq]), - JsonResp = case UpdateResp of - [<<"up">>, {NewJsonDoc}, {JsonResp0}] -> - case chttpd:header_value(Req, "X-Couch-Full-Commit", "false") of - "true" -> - Options = [full_commit, {user_ctx, Req#httpd.user_ctx}, {w, W}]; - _ -> - Options = [{user_ctx, Req#httpd.user_ctx}, {w, W}] - end, - NewDoc = couch_db:doc_from_json_obj_validate(Db, {NewJsonDoc}), - couch_doc:validate_docid(NewDoc#doc.id), - {UpdateResult, NewRev} = fabric:update_doc(Db, NewDoc, Options), - chttpd_stats:incr_writes(), - NewRevStr = couch_doc:rev_to_str(NewRev), - case {UpdateResult, NewRev} of - {ok, _} -> - Code = 201; - {accepted, _} -> - Code = 202 - end, - {JsonResp1} = apply_headers(JsonResp0, [ - {<<"X-Couch-Update-NewRev">>, NewRevStr}, - {<<"X-Couch-Id">>, couch_util:url_encode(NewDoc#doc.id)} - ]), - {[{<<"code">>, Code} | JsonResp1]}; - [<<"up">>, _Other, {JsonResp0}] -> - {[{<<"code">>, 200} | JsonResp0]} - end, + JsonResp = + case UpdateResp of + [<<"up">>, {NewJsonDoc}, {JsonResp0}] -> + case chttpd:header_value(Req, "X-Couch-Full-Commit", "false") of + "true" -> + Options = [full_commit, {user_ctx, Req#httpd.user_ctx}, {w, W}]; + _ -> + Options = [{user_ctx, Req#httpd.user_ctx}, {w, W}] + end, + NewDoc = couch_db:doc_from_json_obj_validate(Db, {NewJsonDoc}), + couch_doc:validate_docid(NewDoc#doc.id), + {UpdateResult, NewRev} = fabric:update_doc(Db, NewDoc, Options), + chttpd_stats:incr_writes(), + NewRevStr = couch_doc:rev_to_str(NewRev), + case {UpdateResult, NewRev} of + {ok, _} -> + Code = 201; + {accepted, _} -> + Code = 202 + end, + {JsonResp1} = apply_headers(JsonResp0, [ + {<<"X-Couch-Update-NewRev">>, NewRevStr}, + {<<"X-Couch-Id">>, couch_util:url_encode(NewDoc#doc.id)} + ]), + {[{<<"code">>, Code} | JsonResp1]}; + [<<"up">>, _Other, {JsonResp0}] -> + {[{<<"code">>, 200} | JsonResp0]} + end, % todo set location field chttpd_external:send_external_response(Req, JsonResp). - % view-list request with view and list from same design doc. -handle_view_list_req(#httpd{method=Method, - path_parts=[_, _, DesignName, _, ListName, ViewName]}=Req, Db, DDoc) - when Method =:= 'GET' orelse Method =:= 'OPTIONS' -> +handle_view_list_req( + #httpd{ + method = Method, + path_parts = [_, _, DesignName, _, ListName, ViewName] + } = Req, + Db, + DDoc +) when + Method =:= 'GET' orelse Method =:= 'OPTIONS' +-> Keys = chttpd:qs_json_value(Req, "keys", undefined), handle_view_list(Req, Db, DDoc, ListName, {DesignName, ViewName}, Keys); - % view-list request with view and list from different design docs. -handle_view_list_req(#httpd{method=Method, - path_parts=[_, _, _, _, ListName, DesignName, ViewName]}=Req, Db, DDoc) - when Method =:= 'GET' orelse Method =:= 'OPTIONS' -> +handle_view_list_req( + #httpd{ + method = Method, + path_parts = [_, _, _, _, ListName, DesignName, ViewName] + } = Req, + Db, + DDoc +) when + Method =:= 'GET' orelse Method =:= 'OPTIONS' +-> Keys = chttpd:qs_json_value(Req, "keys", undefined), handle_view_list(Req, Db, DDoc, ListName, {DesignName, ViewName}, Keys); - -handle_view_list_req(#httpd{method=Method}=Req, _Db, _DDoc) - when Method =:= 'GET' orelse Method =:= 'OPTIONS' -> +handle_view_list_req(#httpd{method = Method} = Req, _Db, _DDoc) when + Method =:= 'GET' orelse Method =:= 'OPTIONS' +-> chttpd:send_error(Req, 404, <<"list_error">>, <<"Invalid path.">>); - -handle_view_list_req(#httpd{method='POST', - path_parts=[_, _, DesignName, _, ListName, ViewName]}=Req, Db, DDoc) -> +handle_view_list_req( + #httpd{ + method = 'POST', + path_parts = [_, _, DesignName, _, ListName, ViewName] + } = Req, + Db, + DDoc +) -> chttpd:validate_ctype(Req, "application/json"), {Props} = chttpd:json_body(Req), Keys = proplists:get_value(<<"keys">>, Props, undefined), - handle_view_list(Req#httpd{req_body={Props}}, Db, DDoc, ListName, - {DesignName, ViewName}, Keys); - -handle_view_list_req(#httpd{method='POST', - path_parts=[_, _, _, _, ListName, DesignName, ViewName]}=Req, Db, DDoc) -> + handle_view_list( + Req#httpd{req_body = {Props}}, + Db, + DDoc, + ListName, + {DesignName, ViewName}, + Keys + ); +handle_view_list_req( + #httpd{ + method = 'POST', + path_parts = [_, _, _, _, ListName, DesignName, ViewName] + } = Req, + Db, + DDoc +) -> chttpd:validate_ctype(Req, "application/json"), {Props} = chttpd:json_body(Req), Keys = proplists:get_value(<<"keys">>, Props, undefined), - handle_view_list(Req#httpd{req_body={Props}}, Db, DDoc, ListName, - {DesignName, ViewName}, Keys); - -handle_view_list_req(#httpd{method='POST'}=Req, _Db, _DDoc) -> + handle_view_list( + Req#httpd{req_body = {Props}}, + Db, + DDoc, + ListName, + {DesignName, ViewName}, + Keys + ); +handle_view_list_req(#httpd{method = 'POST'} = Req, _Db, _DDoc) -> chttpd:send_error(Req, 404, <<"list_error">>, <<"Invalid path.">>); - handle_view_list_req(Req, _Db, _DDoc) -> chttpd:send_method_not_allowed(Req, "GET,POST,HEAD"). @@ -215,12 +265,18 @@ handle_view_list(Req, Db, DDoc, LName, {ViewDesignName, ViewName}, Keys) -> <<"_all_docs">> -> fabric:all_docs(Db, Options, CB, Acc, QueryArgs); _ -> - fabric:query_view(Db, Options, VDoc, ViewName, - CB, Acc, QueryArgs) + fabric:query_view( + Db, + Options, + VDoc, + ViewName, + CB, + Acc, + QueryArgs + ) end end). - list_cb({row, Row} = Msg, Acc) -> case lists:keymember(doc, 1, Row) of true -> chttpd_stats:incr_reads(); @@ -228,11 +284,9 @@ list_cb({row, Row} = Msg, Acc) -> end, chttpd_stats:incr_rows(), couch_mrview_show:list_cb(Msg, Acc); - list_cb(Msg, Acc) -> couch_mrview_show:list_cb(Msg, Acc). - % Maybe this is in the proplists API % todo move to couch_util json_apply_field(H, {L}) -> @@ -245,7 +299,7 @@ json_apply_field({Key, NewValue}, [{OtherKey, OtherVal} | Headers], Acc) -> json_apply_field({Key, NewValue}, Headers, [{OtherKey, OtherVal} | Acc]); json_apply_field({Key, NewValue}, [], Acc) -> % end of list, add ours - {[{Key, NewValue}|Acc]}. + {[{Key, NewValue} | Acc]}. apply_etag(JsonResp, undefined) -> JsonResp; @@ -264,7 +318,7 @@ apply_headers(JsonResp, []) -> apply_headers(JsonResp, NewHeaders) -> case couch_util:get_value(<<"headers">>, JsonResp) of undefined -> - {[{<<"headers">>, {NewHeaders}}| JsonResp]}; + {[{<<"headers">>, {NewHeaders}} | JsonResp]}; JsonHeaders -> Headers = apply_headers1(JsonHeaders, NewHeaders), NewKV = {<<"headers">>, Headers}, diff --git a/src/chttpd/src/chttpd_stats.erl b/src/chttpd/src/chttpd_stats.erl index b76c5618b..f6eb01659 100644 --- a/src/chttpd/src/chttpd_stats.erl +++ b/src/chttpd/src/chttpd_stats.erl @@ -1,4 +1,3 @@ - % Licensed under the Apache License, Version 2.0 (the "License"); you may not % use this file except in compliance with the License. You may obtain a copy of % the License at @@ -30,21 +29,17 @@ incr_rows/1 ]). - -record(st, { reads = 0, writes = 0, rows = 0 }). - -define(KEY, chttpd_stats). - init() -> put(?KEY, #st{}). - report(HttpReq, HttpResp) -> try case get(?KEY) of @@ -58,7 +53,6 @@ report(HttpReq, HttpResp) -> couch_log:error(Fmt, [T, R, S]) end. - report(HttpReq, HttpResp, St) -> case config:get("chttpd", "stats_reporter") of undefined -> @@ -73,31 +67,24 @@ report(HttpReq, HttpResp, St) -> Mod:report(HttpReq, HttpResp, Reads, Writes, Rows) end. - incr_reads() -> incr(#st.reads, 1). - incr_reads(N) when is_integer(N), N >= 0 -> incr(#st.reads, N). - incr_writes() -> incr(#st.writes, 1). - incr_writes(N) when is_integer(N), N >= 0 -> incr(#st.writes, N). - incr_rows() -> incr(#st.rows, 1). - incr_rows(N) when is_integer(N), N >= 0 -> incr(#st.rows, N). - incr(Idx, Count) -> case get(?KEY) of #st{} = St -> diff --git a/src/chttpd/src/chttpd_sup.erl b/src/chttpd/src/chttpd_sup.erl index 78fc66979..ea4e62f80 100644 --- a/src/chttpd/src/chttpd_sup.erl +++ b/src/chttpd/src/chttpd_sup.erl @@ -51,13 +51,11 @@ init([]) -> }, ?CHILD(chttpd, worker), ?CHILD(chttpd_auth_cache, worker), - {chttpd_auth_cache_lru, - {ets_lru, start_link, [chttpd_auth_cache_lru, lru_opts()]}, - permanent, 5000, worker, [ets_lru]} + {chttpd_auth_cache_lru, {ets_lru, start_link, [chttpd_auth_cache_lru, lru_opts()]}, + permanent, 5000, worker, [ets_lru]} ], - {ok, {{one_for_one, 3, 10}, - couch_epi:register_service(chttpd_epi, Children)}}. + {ok, {{one_for_one, 3, 10}, couch_epi:register_service(chttpd_epi, Children)}}. handle_config_change("chttpd", "bind_address", Value, _, Settings) -> maybe_replace(bind_address, Value, Settings); @@ -78,17 +76,21 @@ settings() -> {bind_address, config:get("chttpd", "bind_address")}, {port, config:get("chttpd", "port")}, {backlog, config:get_integer("chttpd", "backlog", ?DEFAULT_BACKLOG)}, - {server_options, config:get("chttpd", - "server_options", ?DEFAULT_SERVER_OPTIONS)} + {server_options, + config:get( + "chttpd", + "server_options", + ?DEFAULT_SERVER_OPTIONS + )} ]. maybe_replace(Key, Value, Settings) -> case couch_util:get_value(Key, Settings) of - Value -> - {ok, Settings}; - _ -> - chttpd:stop(), - {ok, lists:keyreplace(Key, 1, Settings, {Key, Value})} + Value -> + {ok, Settings}; + _ -> + chttpd:stop(), + {ok, lists:keyreplace(Key, 1, Settings, {Key, Value})} end. lru_opts() -> @@ -105,7 +107,9 @@ append_if_set({_Key, 0}, Opts) -> append_if_set({Key, Value}, Opts) -> couch_log:error( "The value for `~s` should be string convertable " - "to integer which is >= 0 (got `~p`)", [Key, Value]), + "to integer which is >= 0 (got `~p`)", + [Key, Value] + ), Opts. notify_started() -> @@ -115,9 +119,12 @@ notify_error(Error) -> couch_log:error("Error starting Apache CouchDB:~n~n ~p~n~n", [Error]). notify_uris() -> - lists:foreach(fun(Uri) -> - couch_log:info("Apache CouchDB has started on ~s", [Uri]) - end, get_uris()). + lists:foreach( + fun(Uri) -> + couch_log:info("Apache CouchDB has started on ~s", [Uri]) + end, + get_uris() + ). write_uris() -> case config:get("couchdb", "uri_file", undefined) of @@ -130,12 +137,15 @@ write_uris() -> get_uris() -> Ip = config:get("chttpd", "bind_address"), - lists:flatmap(fun(Uri) -> - case get_uri(Uri, Ip) of - undefined -> []; - Else -> [Else] - end - end, [chttpd, couch_httpd, https]). + lists:flatmap( + fun(Uri) -> + case get_uri(Uri, Ip) of + undefined -> []; + Else -> [Else] + end + end, + [chttpd, couch_httpd, https] + ). get_uri(Name, Ip) -> case get_port(Name) of diff --git a/src/chttpd/src/chttpd_test_util.erl b/src/chttpd/src/chttpd_test_util.erl index a1a08eff4..8a849acda 100644 --- a/src/chttpd/src/chttpd_test_util.erl +++ b/src/chttpd/src/chttpd_test_util.erl @@ -16,7 +16,6 @@ -include_lib("couch/include/couch_eunit.hrl"). - start_couch() -> start_couch(?CONFIG_CHAIN). diff --git a/src/chttpd/src/chttpd_util.erl b/src/chttpd/src/chttpd_util.erl index 6c68568fe..ca4ffc8a7 100644 --- a/src/chttpd/src/chttpd_util.erl +++ b/src/chttpd/src/chttpd_util.erl @@ -12,7 +12,6 @@ -module(chttpd_util). - -export([ get_chttpd_config/1, get_chttpd_config/2, @@ -25,43 +24,49 @@ maybe_add_csp_header/3 ]). - get_chttpd_config(Key) -> config:get("chttpd", Key, config:get("httpd", Key)). - get_chttpd_config(Key, Default) -> config:get("chttpd", Key, config:get("httpd", Key, Default)). - get_chttpd_config_integer(Key, Default) -> - config:get_integer("chttpd", Key, - config:get_integer("httpd", Key, Default)). - + config:get_integer( + "chttpd", + Key, + config:get_integer("httpd", Key, Default) + ). get_chttpd_config_boolean(Key, Default) -> - config:get_boolean("chttpd", Key, - config:get_boolean("httpd", Key, Default)). - + config:get_boolean( + "chttpd", + Key, + config:get_boolean("httpd", Key, Default) + ). get_chttpd_auth_config(Key) -> config:get("chttpd_auth", Key, config:get("couch_httpd_auth", Key)). - get_chttpd_auth_config(Key, Default) -> - config:get("chttpd_auth", Key, - config:get("couch_httpd_auth", Key, Default)). - + config:get( + "chttpd_auth", + Key, + config:get("couch_httpd_auth", Key, Default) + ). get_chttpd_auth_config_integer(Key, Default) -> - config:get_integer("chttpd_auth", Key, - config:get_integer("couch_httpd_auth", Key, Default)). - + config:get_integer( + "chttpd_auth", + Key, + config:get_integer("couch_httpd_auth", Key, Default) + ). get_chttpd_auth_config_boolean(Key, Default) -> - config:get_boolean("chttpd_auth", Key, - config:get_boolean("couch_httpd_auth", Key, Default)). - + config:get_boolean( + "chttpd_auth", + Key, + config:get_boolean("couch_httpd_auth", Key, Default) + ). maybe_add_csp_header(Component, OriginalHeaders, DefaultHeaderValue) -> Enabled = config:get_boolean("csp", Component ++ "_enable", true), diff --git a/src/chttpd/src/chttpd_view.erl b/src/chttpd/src/chttpd_view.erl index f73a8b7b1..1d721d189 100644 --- a/src/chttpd/src/chttpd_view.erl +++ b/src/chttpd/src/chttpd_view.erl @@ -18,24 +18,42 @@ multi_query_view(Req, Db, DDoc, ViewName, Queries) -> Args0 = couch_mrview_http:parse_params(Req, undefined), - {ok, #mrst{views=Views}} = couch_mrview_util:ddoc_to_mrst(Db, DDoc), + {ok, #mrst{views = Views}} = couch_mrview_util:ddoc_to_mrst(Db, DDoc), Args1 = couch_mrview_util:set_view_type(Args0, ViewName, Views), - ArgQueries = lists:map(fun({Query}) -> - QueryArg = couch_mrview_http:parse_params(Query, undefined, - Args1, [decoded]), - QueryArg1 = couch_mrview_util:set_view_type(QueryArg, ViewName, Views), - fabric_util:validate_args(Db, DDoc, QueryArg1) - end, Queries), + ArgQueries = lists:map( + fun({Query}) -> + QueryArg = couch_mrview_http:parse_params( + Query, + undefined, + Args1, + [decoded] + ), + QueryArg1 = couch_mrview_util:set_view_type(QueryArg, ViewName, Views), + fabric_util:validate_args(Db, DDoc, QueryArg1) + end, + Queries + ), Options = [{user_ctx, Req#httpd.user_ctx}], - VAcc0 = #vacc{db=Db, req=Req, prepend="\r\n"}, + VAcc0 = #vacc{db = Db, req = Req, prepend = "\r\n"}, FirstChunk = "{\"results\":[", {ok, Resp0} = chttpd:start_delayed_json_response(VAcc0#vacc.req, 200, [], FirstChunk), - VAcc1 = VAcc0#vacc{resp=Resp0}, - VAcc2 = lists:foldl(fun(Args, Acc0) -> - {ok, Acc1} = fabric:query_view(Db, Options, DDoc, ViewName, - fun view_cb/2, Acc0, Args), - Acc1 - end, VAcc1, ArgQueries), + VAcc1 = VAcc0#vacc{resp = Resp0}, + VAcc2 = lists:foldl( + fun(Args, Acc0) -> + {ok, Acc1} = fabric:query_view( + Db, + Options, + DDoc, + ViewName, + fun view_cb/2, + Acc0, + Args + ), + Acc1 + end, + VAcc1, + ArgQueries + ), {ok, Resp1} = chttpd:send_delayed_chunk(VAcc2#vacc.resp, "\r\n]}"), chttpd:end_delayed_json_response(Resp1). @@ -49,13 +67,19 @@ design_doc_view(Req, Db, DDoc, ViewName, Keys) -> fabric_query_view(Db, Req, DDoc, ViewName, Args) -> Max = chttpd:chunked_response_buffer_size(), - VAcc = #vacc{db=Db, req=Req, threshold=Max}, + VAcc = #vacc{db = Db, req = Req, threshold = Max}, Options = [{user_ctx, Req#httpd.user_ctx}], - {ok, Resp} = fabric:query_view(Db, Options, DDoc, ViewName, - fun view_cb/2, VAcc, Args), + {ok, Resp} = fabric:query_view( + Db, + Options, + DDoc, + ViewName, + fun view_cb/2, + VAcc, + Args + ), {ok, Resp#vacc.resp}. - view_cb({row, Row} = Msg, Acc) -> case lists:keymember(doc, 1, Row) of true -> chttpd_stats:incr_reads(); @@ -63,42 +87,56 @@ view_cb({row, Row} = Msg, Acc) -> end, chttpd_stats:incr_rows(), couch_mrview_http:view_cb(Msg, Acc); - view_cb(Msg, Acc) -> couch_mrview_http:view_cb(Msg, Acc). - -handle_view_req(#httpd{method='POST', - path_parts=[_, _, _, _, ViewName, <<"queries">>]}=Req, Db, DDoc) -> +handle_view_req( + #httpd{ + method = 'POST', + path_parts = [_, _, _, _, ViewName, <<"queries">>] + } = Req, + Db, + DDoc +) -> chttpd:validate_ctype(Req, "application/json"), Props = couch_httpd:json_body_obj(Req), case couch_mrview_util:get_view_queries(Props) of undefined -> - throw({bad_request, - <<"POST body must include `queries` parameter.">>}); + throw({bad_request, <<"POST body must include `queries` parameter.">>}); Queries -> multi_query_view(Req, Db, DDoc, ViewName, Queries) end; - -handle_view_req(#httpd{path_parts=[_, _, _, _, _, <<"queries">>]}=Req, - _Db, _DDoc) -> +handle_view_req( + #httpd{path_parts = [_, _, _, _, _, <<"queries">>]} = Req, + _Db, + _DDoc +) -> chttpd:send_method_not_allowed(Req, "POST"); - -handle_view_req(#httpd{method='GET', - path_parts=[_, _, _, _, ViewName]}=Req, Db, DDoc) -> +handle_view_req( + #httpd{ + method = 'GET', + path_parts = [_, _, _, _, ViewName] + } = Req, + Db, + DDoc +) -> couch_stats:increment_counter([couchdb, httpd, view_reads]), Keys = chttpd:qs_json_value(Req, "keys", undefined), design_doc_view(Req, Db, DDoc, ViewName, Keys); - -handle_view_req(#httpd{method='POST', - path_parts=[_, _, _, _, ViewName]}=Req, Db, DDoc) -> +handle_view_req( + #httpd{ + method = 'POST', + path_parts = [_, _, _, _, ViewName] + } = Req, + Db, + DDoc +) -> chttpd:validate_ctype(Req, "application/json"), Props = couch_httpd:json_body_obj(Req), assert_no_queries_param(couch_mrview_util:get_view_queries(Props)), Keys = couch_mrview_util:get_view_keys(Props), couch_stats:increment_counter([couchdb, httpd, view_reads]), design_doc_post_view(Req, Props, Db, DDoc, ViewName, Keys); - handle_view_req(Req, _Db, _DDoc) -> chttpd:send_method_not_allowed(Req, "GET,POST,HEAD"). @@ -115,12 +153,10 @@ assert_no_queries_param(_) -> "The `queries` parameter is no longer supported at this endpoint" }). - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). - check_multi_query_reduce_view_overrides_test_() -> { setup, @@ -137,7 +173,6 @@ check_multi_query_reduce_view_overrides_test_() -> } }. - t_check_include_docs_throw_validation_error() -> ?_test(begin Req = #httpd{qs = []}, @@ -147,7 +182,6 @@ t_check_include_docs_throw_validation_error() -> ?assertThrow(Throw, multi_query_view(Req, Db, ddoc, <<"v">>, [Query])) end). - t_check_user_can_override_individual_query_type() -> ?_test(begin Req = #httpd{qs = []}, @@ -157,7 +191,6 @@ t_check_user_can_override_individual_query_type() -> ?assertEqual(1, meck:num_calls(chttpd, start_delayed_json_response, '_')) end). - setup_all() -> Views = [#mrview{reduce_funs = [{<<"v">>, <<"_count">>}]}], meck:expect(couch_mrview_util, ddoc_to_mrst, 2, {ok, #mrst{views = Views}}), @@ -166,11 +199,9 @@ setup_all() -> meck:expect(chttpd, send_delayed_chunk, 2, {ok, resp}), meck:expect(chttpd, end_delayed_json_response, 1, ok). - teardown_all(_) -> meck:unload(). - setup() -> meck:reset([ chttpd, @@ -178,9 +209,7 @@ setup() -> fabric ]). - teardown(_) -> ok. - -endif. diff --git a/src/chttpd/src/chttpd_xframe_options.erl b/src/chttpd/src/chttpd_xframe_options.erl index 2a43617fa..15865057b 100644 --- a/src/chttpd/src/chttpd_xframe_options.erl +++ b/src/chttpd/src/chttpd_xframe_options.erl @@ -21,7 +21,6 @@ -define(SAMEORIGIN, "SAMEORIGIN"). -define(ALLOWFROM, "ALLOW-FROM "). - -include_lib("couch/include/couch_db.hrl"). % X-Frame-Options protects against clickjacking by limiting whether a response can be used in a @@ -30,33 +29,28 @@ header(Req, Headers) -> header(Req, Headers, get_xframe_config(Req)). - - header(Req, Headers, Config) -> case lists:keyfind(enabled, 1, Config) of - {enabled, true} -> + {enabled, true} -> generate_xframe_header(Req, Headers, Config); - _ -> + _ -> Headers end. - - generate_xframe_header(Req, Headers, Config) -> - XframeOption = case lists:keyfind(same_origin, 1, Config) of - {same_origin, true} -> - ?SAMEORIGIN; - _ -> - check_host(Req, Config) - end, - [{"X-Frame-Options", XframeOption } | Headers]. - - + XframeOption = + case lists:keyfind(same_origin, 1, Config) of + {same_origin, true} -> + ?SAMEORIGIN; + _ -> + check_host(Req, Config) + end, + [{"X-Frame-Options", XframeOption} | Headers]. check_host(#httpd{mochi_req = MochiReq} = Req, Config) -> Host = couch_httpd_vhost:host(MochiReq), case Host of - [] -> + [] -> ?DENY; Host -> FullHost = chttpd:absolute_uri(Req, ""), @@ -66,18 +60,18 @@ check_host(#httpd{mochi_req = MochiReq} = Req, Config) -> true -> ?ALLOWFROM ++ FullHost; false -> ?DENY end - end. - - + end. get_xframe_config(#httpd{xframe_config = undefined}) -> EnableXFrame = chttpd_util:get_chttpd_config_boolean( - "enable_xframe_options", false), + "enable_xframe_options", false + ), SameOrigin = config:get("x_frame_options", "same_origin", "false") =:= "true", - AcceptedHosts = case config:get("x_frame_options", "hosts") of - undefined -> []; - Hosts -> split_list(Hosts) - end, + AcceptedHosts = + case config:get("x_frame_options", "hosts") of + undefined -> []; + Hosts -> split_list(Hosts) + end, [ {enabled, EnableXFrame}, {same_origin, SameOrigin}, @@ -86,15 +80,11 @@ get_xframe_config(#httpd{xframe_config = undefined}) -> get_xframe_config(#httpd{xframe_config = Config}) -> Config. - - get_accepted_hosts(Config) -> case lists:keyfind(hosts, 1, Config) of false -> []; {hosts, AcceptedHosts} -> AcceptedHosts end. - - split_list(S) -> re:split(S, "\\s*,\\s*", [trim, {return, list}]). diff --git a/src/chttpd/test/eunit/chttpd_auth_tests.erl b/src/chttpd/test/eunit/chttpd_auth_tests.erl index b4a8eabfb..7beda9bc7 100644 --- a/src/chttpd/test/eunit/chttpd_auth_tests.erl +++ b/src/chttpd/test/eunit/chttpd_auth_tests.erl @@ -15,7 +15,6 @@ -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). - setup() -> Addr = config:get("chttpd", "bind_address", "127.0.0.1"), Port = mochiweb_socket_server:get(chttpd, port), @@ -25,7 +24,6 @@ setup() -> teardown(_Url) -> ok. - require_valid_user_exception_test_() -> { "_up", @@ -35,7 +33,8 @@ require_valid_user_exception_test_() -> fun chttpd_test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_handle_require_valid_user_except_up_on_up_route/1, fun should_handle_require_valid_user_except_up_on_non_up_routes/1 @@ -45,85 +44,84 @@ require_valid_user_exception_test_() -> }. set_require_user_false() -> - ok = config:set("chttpd", "require_valid_user", "false", _Persist=false). + ok = config:set("chttpd", "require_valid_user", "false", _Persist = false). set_require_user_true() -> - ok = config:set("chttpd", "require_valid_user", "true", _Persist=false). + ok = config:set("chttpd", "require_valid_user", "true", _Persist = false). set_require_user_except_for_up_false() -> - ok = config:set("chttpd", "require_valid_user_except_for_up", "false", _Persist=false). + ok = config:set("chttpd", "require_valid_user_except_for_up", "false", _Persist = false). set_require_user_except_for_up_true() -> - ok = config:set("chttpd", "require_valid_user_except_for_up", "true", _Persist=false). + ok = config:set("chttpd", "require_valid_user_except_for_up", "true", _Persist = false). should_handle_require_valid_user_except_up_on_up_route(_Url) -> - ?_test(begin - % require_valid_user | require_valid_user_except_up | up needs auth - % 1 F | F | F - % 2 F | T | F - % 3 T | F | T - % 4 T | T | F - - UpRequest = #httpd{path_parts=[<<"_up">>]}, - % we use ?ADMIN_USER here because these tests run under admin party - % so this is equivalent to an unauthenticated request - ExpectAuth = {unauthorized, <<"Authentication required.">>}, - ExpectNoAuth = #httpd{user_ctx=?ADMIN_USER,path_parts=[<<"_up">>]}, - - % 1 - set_require_user_false(), - set_require_user_except_for_up_false(), - Result1 = chttpd_auth:party_mode_handler(UpRequest), - ?assertEqual(ExpectNoAuth, Result1), - - % 2 - set_require_user_false(), - set_require_user_except_for_up_true(), - Result2 = chttpd_auth:party_mode_handler(UpRequest), - ?assertEqual(ExpectNoAuth, Result2), - - % 3 - set_require_user_true(), - set_require_user_except_for_up_false(), - ?assertThrow(ExpectAuth, chttpd_auth:party_mode_handler(UpRequest)), - - % 4 - set_require_user_true(), - set_require_user_except_for_up_true(), - Result4 = chttpd_auth:party_mode_handler(UpRequest), - ?assertEqual(ExpectNoAuth, Result4) - - end). + ?_test(begin + % require_valid_user | require_valid_user_except_up | up needs auth + % 1 F | F | F + % 2 F | T | F + % 3 T | F | T + % 4 T | T | F + + UpRequest = #httpd{path_parts = [<<"_up">>]}, + % we use ?ADMIN_USER here because these tests run under admin party + % so this is equivalent to an unauthenticated request + ExpectAuth = {unauthorized, <<"Authentication required.">>}, + ExpectNoAuth = #httpd{user_ctx = ?ADMIN_USER, path_parts = [<<"_up">>]}, + + % 1 + set_require_user_false(), + set_require_user_except_for_up_false(), + Result1 = chttpd_auth:party_mode_handler(UpRequest), + ?assertEqual(ExpectNoAuth, Result1), + + % 2 + set_require_user_false(), + set_require_user_except_for_up_true(), + Result2 = chttpd_auth:party_mode_handler(UpRequest), + ?assertEqual(ExpectNoAuth, Result2), + + % 3 + set_require_user_true(), + set_require_user_except_for_up_false(), + ?assertThrow(ExpectAuth, chttpd_auth:party_mode_handler(UpRequest)), + + % 4 + set_require_user_true(), + set_require_user_except_for_up_true(), + Result4 = chttpd_auth:party_mode_handler(UpRequest), + ?assertEqual(ExpectNoAuth, Result4) + end). should_handle_require_valid_user_except_up_on_non_up_routes(_Url) -> - ?_test(begin - % require_valid_user | require_valid_user_except_up | everything not _up requires auth - % 5 F | F | F - % 6 F | T | T - % 7 T | F | T - % 8 T | T | T - - NonUpRequest = #httpd{path_parts=[<<"/">>]}, - ExpectAuth = {unauthorized, <<"Authentication required.">>}, - ExpectNoAuth = #httpd{user_ctx=?ADMIN_USER,path_parts=[<<"/">>]}, - % 5 - set_require_user_false(), - set_require_user_except_for_up_false(), - Result5 = chttpd_auth:party_mode_handler(NonUpRequest), - ?assertEqual(ExpectNoAuth, Result5), - - % 6 - set_require_user_false(), - set_require_user_except_for_up_true(), - ?assertThrow(ExpectAuth, chttpd_auth:party_mode_handler(NonUpRequest)), - - % 7 - set_require_user_true(), - set_require_user_except_for_up_false(), - ?assertThrow(ExpectAuth, chttpd_auth:party_mode_handler(NonUpRequest)), - - % 8 - set_require_user_true(), - set_require_user_except_for_up_true(), - ?assertThrow(ExpectAuth, chttpd_auth:party_mode_handler(NonUpRequest)) - end). + ?_test(begin + % require_valid_user | require_valid_user_except_up | everything not _up requires auth + % 5 F | F | F + % 6 F | T | T + % 7 T | F | T + % 8 T | T | T + + NonUpRequest = #httpd{path_parts = [<<"/">>]}, + ExpectAuth = {unauthorized, <<"Authentication required.">>}, + ExpectNoAuth = #httpd{user_ctx = ?ADMIN_USER, path_parts = [<<"/">>]}, + % 5 + set_require_user_false(), + set_require_user_except_for_up_false(), + Result5 = chttpd_auth:party_mode_handler(NonUpRequest), + ?assertEqual(ExpectNoAuth, Result5), + + % 6 + set_require_user_false(), + set_require_user_except_for_up_true(), + ?assertThrow(ExpectAuth, chttpd_auth:party_mode_handler(NonUpRequest)), + + % 7 + set_require_user_true(), + set_require_user_except_for_up_false(), + ?assertThrow(ExpectAuth, chttpd_auth:party_mode_handler(NonUpRequest)), + + % 8 + set_require_user_true(), + set_require_user_except_for_up_true(), + ?assertThrow(ExpectAuth, chttpd_auth:party_mode_handler(NonUpRequest)) + end). diff --git a/src/chttpd/test/eunit/chttpd_cors_test.erl b/src/chttpd/test/eunit/chttpd_cors_test.erl index 44644b571..93b080fc6 100644 --- a/src/chttpd/test/eunit/chttpd_cors_test.erl +++ b/src/chttpd/test/eunit/chttpd_cors_test.erl @@ -12,17 +12,16 @@ -module(chttpd_cors_test). - -include_lib("couch/include/couch_db.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("chttpd/include/chttpd_cors.hrl"). - -define(DEFAULT_ORIGIN, "http://example.com"). -define(DEFAULT_ORIGIN_HTTPS, "https://example.com"). -define(EXPOSED_HEADERS, "content-type, accept-ranges, etag, server, x-couch-request-id, " ++ - "x-couch-update-newrev, x-couchdb-body-time"). + "x-couch-update-newrev, x-couchdb-body-time" +). -define(CUSTOM_SUPPORTED_METHODS, ?SUPPORTED_METHODS -- ["CONNECT"]). -define(CUSTOM_SUPPORTED_HEADERS, ["extra" | ?SUPPORTED_HEADERS -- ["pragma"]]). @@ -32,33 +31,31 @@ %% Test helpers - empty_cors_config() -> []. - minimal_cors_config() -> [ {<<"enable_cors">>, true}, {<<"origins">>, {[]}} ]. - simple_cors_config() -> [ {<<"enable_cors">>, true}, - {<<"origins">>, {[ - {list_to_binary(?DEFAULT_ORIGIN), {[]}} - ]}} + {<<"origins">>, + {[ + {list_to_binary(?DEFAULT_ORIGIN), {[]}} + ]}} ]. - wildcard_cors_config() -> [ {<<"enable_cors">>, true}, - {<<"origins">>, {[ - {<<"*">>, {[]}} - ]}} + {<<"origins">>, + {[ + {<<"*">>, {[]}} + ]}} ]. custom_cors_config() -> @@ -68,55 +65,59 @@ custom_cors_config() -> {<<"allow_headers">>, ?CUSTOM_SUPPORTED_HEADERS}, {<<"exposed_headers">>, ?CUSTOM_EXPOSED_HEADERS}, {<<"max_age">>, ?CUSTOM_MAX_AGE}, - {<<"origins">>, {[ - {<<"*">>, {[]}} - ]}} + {<<"origins">>, + {[ + {<<"*">>, {[]}} + ]}} ]. access_control_cors_config(AllowCredentials) -> [ {<<"enable_cors">>, true}, {<<"allow_credentials">>, AllowCredentials}, - {<<"origins">>, {[ - {list_to_binary(?DEFAULT_ORIGIN), {[]}} - ]}}]. - + {<<"origins">>, + {[ + {list_to_binary(?DEFAULT_ORIGIN), {[]}} + ]}} + ]. multiple_cors_config() -> [ {<<"enable_cors">>, true}, - {<<"origins">>, {[ - {list_to_binary(?DEFAULT_ORIGIN), {[]}}, - {<<"https://example.com">>, {[]}}, - {<<"http://example.com:5984">>, {[]}}, - {<<"https://example.com:5984">>, {[]}} - ]}} + {<<"origins">>, + {[ + {list_to_binary(?DEFAULT_ORIGIN), {[]}}, + {<<"https://example.com">>, {[]}}, + {<<"http://example.com:5984">>, {[]}}, + {<<"https://example.com:5984">>, {[]}} + ]}} ]. - mock_request(Method, Path, Headers0) -> HeaderKey = "Access-Control-Request-Method", - Headers = case proplists:get_value(HeaderKey, Headers0, undefined) of - nil -> - proplists:delete(HeaderKey, Headers0); - undefined -> - case Method of - 'OPTIONS' -> - [{HeaderKey, atom_to_list(Method)} | Headers0]; - _ -> - Headers0 - end; - _ -> - Headers0 - end, + Headers = + case proplists:get_value(HeaderKey, Headers0, undefined) of + nil -> + proplists:delete(HeaderKey, Headers0); + undefined -> + case Method of + 'OPTIONS' -> + [{HeaderKey, atom_to_list(Method)} | Headers0]; + _ -> + Headers0 + end; + _ -> + Headers0 + end, Headers1 = mochiweb_headers:make(Headers), MochiReq = mochiweb_request:new(nil, Method, Path, {1, 1}, Headers1), - PathParts = [list_to_binary(chttpd:unquote(Part)) - || Part <- string:tokens(Path, "/")], - #httpd{method=Method, mochi_req=MochiReq, path_parts=PathParts}. - + PathParts = [ + list_to_binary(chttpd:unquote(Part)) + || Part <- string:tokens(Path, "/") + ], + #httpd{method = Method, mochi_req = MochiReq, path_parts = PathParts}. -header(#httpd{}=Req, Key) -> +header(#httpd{} = Req, Key) -> chttpd:header_value(Req, Key); header({mochiweb_response, [_, _, Headers]}, Key) -> %% header(Headers, Key); @@ -124,57 +125,40 @@ header({mochiweb_response, [_, _, Headers]}, Key) -> header(Headers, Key) -> couch_util:get_value(Key, Headers, undefined). - string_headers(H) -> string:join(H, ", "). - assert_not_preflight_(Val) -> ?_assertEqual(not_preflight, Val). - %% CORS disabled tests - cors_disabled_test_() -> {"CORS disabled tests", [ {"Empty user", - {setup, - fun chttpd_test_util:start_couch/0, - fun chttpd_test_util:stop_couch/1, + {setup, fun chttpd_test_util:start_couch/0, fun chttpd_test_util:stop_couch/1, {foreach, fun empty_cors_config/0, [ fun test_no_access_control_method_preflight_request_/1, fun test_no_headers_/1, fun test_no_headers_server_/1, fun test_no_headers_db_/1 - ]} - } - } + ]}}} ]}. - %% CORS enabled tests - cors_enabled_minimal_config_test_() -> {"Minimal CORS enabled, no Origins", - {setup, - fun chttpd_test_util:start_couch/0, - fun chttpd_test_util:stop_couch/1, + {setup, fun chttpd_test_util:start_couch/0, fun chttpd_test_util:stop_couch/1, {foreach, fun minimal_cors_config/0, [ fun test_no_access_control_method_preflight_request_/1, fun test_incorrect_origin_simple_request_/1, fun test_incorrect_origin_preflight_request_/1 - ]} - } - }. - + ]}}}. cors_enabled_simple_config_test_() -> {"Simple CORS config", - {setup, - fun chttpd_test_util:start_couch/0, - fun chttpd_test_util:stop_couch/1, + {setup, fun chttpd_test_util:start_couch/0, fun chttpd_test_util:stop_couch/1, {foreach, fun simple_cors_config/0, [ fun test_no_access_control_method_preflight_request_/1, fun test_preflight_request_/1, @@ -187,28 +171,19 @@ cors_enabled_simple_config_test_() -> fun test_preflight_with_scheme_no_origin_/1, fun test_preflight_with_scheme_port_no_origin_/1, fun test_case_sensitive_mismatch_of_allowed_origins_/1 - ]} - } - }. + ]}}}. cors_enabled_custom_config_test_() -> {"Simple CORS config with custom allow_methods/allow_headers/exposed_headers", - {setup, - fun chttpd_test_util:start_couch/0, - fun chttpd_test_util:stop_couch/1, + {setup, fun chttpd_test_util:start_couch/0, fun chttpd_test_util:stop_couch/1, {foreach, fun custom_cors_config/0, [ fun test_good_headers_preflight_request_with_custom_config_/1, fun test_db_request_with_custom_config_/1 - ]} - } - }. - + ]}}}. cors_enabled_multiple_config_test_() -> {"Multiple options CORS config", - {setup, - fun chttpd_test_util:start_couch/0, - fun chttpd_test_util:stop_couch/1, + {setup, fun chttpd_test_util:start_couch/0, fun chttpd_test_util:stop_couch/1, {foreach, fun multiple_cors_config/0, [ fun test_no_access_control_method_preflight_request_/1, fun test_preflight_request_/1, @@ -218,14 +193,10 @@ cors_enabled_multiple_config_test_() -> fun test_preflight_with_port_with_origin_/1, fun test_preflight_with_scheme_with_origin_/1, fun test_preflight_with_scheme_port_with_origin_/1 - ]} - } - }. - + ]}}}. %% Access-Control-Allow-Credentials tests - %% http://www.w3.org/TR/cors/#supports-credentials %% 6.1.3 %% If the resource supports credentials add a single @@ -245,10 +216,7 @@ db_request_credentials_header_off_test_() -> fun() -> access_control_cors_config(false) end, - fun test_db_request_credentials_header_off_/1 - } - }. - + fun test_db_request_credentials_header_off_/1}}. db_request_credentials_header_on_test_() -> {"Allow credentials enabled", @@ -256,19 +224,13 @@ db_request_credentials_header_on_test_() -> fun() -> access_control_cors_config(true) end, - fun test_db_request_credentials_header_on_/1 - } - }. - + fun test_db_request_credentials_header_on_/1}}. %% CORS wildcard tests - cors_enabled_wildcard_test_() -> {"Wildcard CORS config", - {setup, - fun chttpd_test_util:start_couch/0, - fun chttpd_test_util:stop_couch/1, + {setup, fun chttpd_test_util:start_couch/0, fun chttpd_test_util:stop_couch/1, {foreach, fun wildcard_cors_config/0, [ fun test_no_access_control_method_preflight_request_/1, fun test_preflight_request_/1, @@ -281,30 +243,23 @@ cors_enabled_wildcard_test_() -> fun test_preflight_with_scheme_with_origin_/1, fun test_preflight_with_scheme_port_with_origin_/1, fun test_case_sensitive_mismatch_of_allowed_origins_/1 - ]} - } - }. - + ]}}}. %% Test generators - test_no_headers_(OwnerConfig) -> Req = mock_request('GET', "/", []), assert_not_preflight_(chttpd_cors:maybe_handle_preflight_request(Req, OwnerConfig)). - test_no_headers_server_(OwnerConfig) -> Req = mock_request('GET', "/", [{"Origin", "http://127.0.0.1"}]), assert_not_preflight_(chttpd_cors:maybe_handle_preflight_request(Req, OwnerConfig)). - test_no_headers_db_(OwnerConfig) -> Headers = [{"Origin", "http://127.0.0.1"}], Req = mock_request('GET', "/my_db", Headers), assert_not_preflight_(chttpd_cors:maybe_handle_preflight_request(Req, OwnerConfig)). - test_incorrect_origin_simple_request_(OwnerConfig) -> Req = mock_request('GET', "/", [{"Origin", "http://127.0.0.1"}]), [ @@ -312,7 +267,6 @@ test_incorrect_origin_simple_request_(OwnerConfig) -> assert_not_preflight_(chttpd_cors:maybe_handle_preflight_request(Req, OwnerConfig)) ]. - test_incorrect_origin_preflight_request_(OwnerConfig) -> Headers = [ {"Origin", "http://127.0.0.1"}, @@ -324,7 +278,6 @@ test_incorrect_origin_preflight_request_(OwnerConfig) -> assert_not_preflight_(chttpd_cors:maybe_handle_preflight_request(Req, OwnerConfig)) ]. - test_bad_headers_preflight_request_(OwnerConfig) -> Headers = [ {"Origin", ?DEFAULT_ORIGIN}, @@ -337,7 +290,6 @@ test_bad_headers_preflight_request_(OwnerConfig) -> assert_not_preflight_(chttpd_cors:maybe_handle_preflight_request(Req, OwnerConfig)) ]. - test_good_headers_preflight_request_(OwnerConfig) -> Headers = [ {"Origin", ?DEFAULT_ORIGIN}, @@ -348,12 +300,18 @@ test_good_headers_preflight_request_(OwnerConfig) -> ?assert(chttpd_cors:is_cors_enabled(OwnerConfig)), {ok, Headers1} = chttpd_cors:maybe_handle_preflight_request(Req, OwnerConfig), [ - ?_assertEqual(?DEFAULT_ORIGIN, - header(Headers1, "Access-Control-Allow-Origin")), - ?_assertEqual(string_headers(?SUPPORTED_METHODS), - header(Headers1, "Access-Control-Allow-Methods")), - ?_assertEqual(string_headers(["accept-language"]), - header(Headers1, "Access-Control-Allow-Headers")) + ?_assertEqual( + ?DEFAULT_ORIGIN, + header(Headers1, "Access-Control-Allow-Origin") + ), + ?_assertEqual( + string_headers(?SUPPORTED_METHODS), + header(Headers1, "Access-Control-Allow-Methods") + ), + ?_assertEqual( + string_headers(["accept-language"]), + header(Headers1, "Access-Control-Allow-Headers") + ) ]. test_good_headers_preflight_request_with_custom_config_(OwnerConfig) -> @@ -366,22 +324,31 @@ test_good_headers_preflight_request_with_custom_config_(OwnerConfig) -> Req = mock_request('OPTIONS', "/", Headers), ?assert(chttpd_cors:is_cors_enabled(OwnerConfig)), AllowMethods = couch_util:get_value( - <<"allow_methods">>, OwnerConfig, ?SUPPORTED_METHODS), + <<"allow_methods">>, OwnerConfig, ?SUPPORTED_METHODS + ), MaxAge = couch_util:get_value( - <<"max_age">>, OwnerConfig, ?CORS_DEFAULT_MAX_AGE), + <<"max_age">>, OwnerConfig, ?CORS_DEFAULT_MAX_AGE + ), {ok, Headers1} = chttpd_cors:maybe_handle_preflight_request(Req, OwnerConfig), [ - ?_assertEqual(?DEFAULT_ORIGIN, - header(Headers1, "Access-Control-Allow-Origin")), - ?_assertEqual(string_headers(AllowMethods), - header(Headers1, "Access-Control-Allow-Methods")), - ?_assertEqual(string_headers(["accept-language", "extra"]), - header(Headers1, "Access-Control-Allow-Headers")), - ?_assertEqual(MaxAge, - header(Headers1, "Access-Control-Max-Age")) + ?_assertEqual( + ?DEFAULT_ORIGIN, + header(Headers1, "Access-Control-Allow-Origin") + ), + ?_assertEqual( + string_headers(AllowMethods), + header(Headers1, "Access-Control-Allow-Methods") + ), + ?_assertEqual( + string_headers(["accept-language", "extra"]), + header(Headers1, "Access-Control-Allow-Headers") + ), + ?_assertEqual( + MaxAge, + header(Headers1, "Access-Control-Max-Age") + ) ]. - test_preflight_request_(OwnerConfig) -> Headers = [ {"Origin", ?DEFAULT_ORIGIN}, @@ -390,13 +357,16 @@ test_preflight_request_(OwnerConfig) -> Req = mock_request('OPTIONS', "/", Headers), {ok, Headers1} = chttpd_cors:maybe_handle_preflight_request(Req, OwnerConfig), [ - ?_assertEqual(?DEFAULT_ORIGIN, - header(Headers1, "Access-Control-Allow-Origin")), - ?_assertEqual(string_headers(?SUPPORTED_METHODS), - header(Headers1, "Access-Control-Allow-Methods")) + ?_assertEqual( + ?DEFAULT_ORIGIN, + header(Headers1, "Access-Control-Allow-Origin") + ), + ?_assertEqual( + string_headers(?SUPPORTED_METHODS), + header(Headers1, "Access-Control-Allow-Methods") + ) ]. - test_no_access_control_method_preflight_request_(OwnerConfig) -> Headers = [ {"Origin", ?DEFAULT_ORIGIN}, @@ -405,7 +375,6 @@ test_no_access_control_method_preflight_request_(OwnerConfig) -> Req = mock_request('OPTIONS', "/", Headers), assert_not_preflight_(chttpd_cors:maybe_handle_preflight_request(Req, OwnerConfig)). - test_preflight_request_no_allow_credentials_(OwnerConfig) -> Headers = [ {"Origin", ?DEFAULT_ORIGIN}, @@ -414,15 +383,20 @@ test_preflight_request_no_allow_credentials_(OwnerConfig) -> Req = mock_request('OPTIONS', "/", Headers), {ok, Headers1} = chttpd_cors:maybe_handle_preflight_request(Req, OwnerConfig), [ - ?_assertEqual(?DEFAULT_ORIGIN, - header(Headers1, "Access-Control-Allow-Origin")), - ?_assertEqual(string_headers(?SUPPORTED_METHODS), - header(Headers1, "Access-Control-Allow-Methods")), - ?_assertEqual(undefined, - header(Headers1, "Access-Control-Allow-Credentials")) + ?_assertEqual( + ?DEFAULT_ORIGIN, + header(Headers1, "Access-Control-Allow-Origin") + ), + ?_assertEqual( + string_headers(?SUPPORTED_METHODS), + header(Headers1, "Access-Control-Allow-Methods") + ), + ?_assertEqual( + undefined, + header(Headers1, "Access-Control-Allow-Credentials") + ) ]. - test_preflight_request_empty_request_headers_(OwnerConfig) -> Headers = [ {"Origin", ?DEFAULT_ORIGIN}, @@ -432,25 +406,34 @@ test_preflight_request_empty_request_headers_(OwnerConfig) -> Req = mock_request('OPTIONS', "/", Headers), {ok, Headers1} = chttpd_cors:maybe_handle_preflight_request(Req, OwnerConfig), [ - ?_assertEqual(?DEFAULT_ORIGIN, - header(Headers1, "Access-Control-Allow-Origin")), - ?_assertEqual(string_headers(?SUPPORTED_METHODS), - header(Headers1, "Access-Control-Allow-Methods")), - ?_assertEqual("", - header(Headers1, "Access-Control-Allow-Headers")) + ?_assertEqual( + ?DEFAULT_ORIGIN, + header(Headers1, "Access-Control-Allow-Origin") + ), + ?_assertEqual( + string_headers(?SUPPORTED_METHODS), + header(Headers1, "Access-Control-Allow-Methods") + ), + ?_assertEqual( + "", + header(Headers1, "Access-Control-Allow-Headers") + ) ]. - test_db_request_(OwnerConfig) -> Origin = ?DEFAULT_ORIGIN, Headers = [{"Origin", Origin}], Req = mock_request('GET', "/my_db", Headers), Headers1 = chttpd_cors:headers(Req, Headers, Origin, OwnerConfig), [ - ?_assertEqual(?DEFAULT_ORIGIN, - header(Headers1, "Access-Control-Allow-Origin")), - ?_assertEqual(?EXPOSED_HEADERS, - header(Headers1, "Access-Control-Expose-Headers")) + ?_assertEqual( + ?DEFAULT_ORIGIN, + header(Headers1, "Access-Control-Allow-Origin") + ), + ?_assertEqual( + ?EXPOSED_HEADERS, + header(Headers1, "Access-Control-Expose-Headers") + ) ]. test_db_request_with_custom_config_(OwnerConfig) -> @@ -459,16 +442,21 @@ test_db_request_with_custom_config_(OwnerConfig) -> Req = mock_request('GET', "/my_db", Headers), Headers1 = chttpd_cors:headers(Req, Headers, Origin, OwnerConfig), ExposedHeaders = couch_util:get_value( - <<"exposed_headers">>, OwnerConfig, ?COUCH_HEADERS), + <<"exposed_headers">>, OwnerConfig, ?COUCH_HEADERS + ), [ - ?_assertEqual(?DEFAULT_ORIGIN, - header(Headers1, "Access-Control-Allow-Origin")), - ?_assertEqual(lists:sort(["content-type" | ExposedHeaders]), + ?_assertEqual( + ?DEFAULT_ORIGIN, + header(Headers1, "Access-Control-Allow-Origin") + ), + ?_assertEqual( + lists:sort(["content-type" | ExposedHeaders]), lists:sort( - split_list(header(Headers1, "Access-Control-Expose-Headers")))) + split_list(header(Headers1, "Access-Control-Expose-Headers")) + ) + ) ]. - test_db_preflight_request_(OwnerConfig) -> Headers = [ {"Origin", ?DEFAULT_ORIGIN} @@ -476,13 +464,16 @@ test_db_preflight_request_(OwnerConfig) -> Req = mock_request('OPTIONS', "/my_db", Headers), {ok, Headers1} = chttpd_cors:maybe_handle_preflight_request(Req, OwnerConfig), [ - ?_assertEqual(?DEFAULT_ORIGIN, - header(Headers1, "Access-Control-Allow-Origin")), - ?_assertEqual(string_headers(?SUPPORTED_METHODS), - header(Headers1, "Access-Control-Allow-Methods")) + ?_assertEqual( + ?DEFAULT_ORIGIN, + header(Headers1, "Access-Control-Allow-Origin") + ), + ?_assertEqual( + string_headers(?SUPPORTED_METHODS), + header(Headers1, "Access-Control-Allow-Methods") + ) ]. - test_db_host_origin_request_(OwnerConfig) -> Origin = ?DEFAULT_ORIGIN, Headers = [ @@ -492,13 +483,16 @@ test_db_host_origin_request_(OwnerConfig) -> Req = mock_request('GET', "/my_db", Headers), Headers1 = chttpd_cors:headers(Req, Headers, Origin, OwnerConfig), [ - ?_assertEqual(?DEFAULT_ORIGIN, - header(Headers1, "Access-Control-Allow-Origin")), - ?_assertEqual(?EXPOSED_HEADERS, - header(Headers1, "Access-Control-Expose-Headers")) + ?_assertEqual( + ?DEFAULT_ORIGIN, + header(Headers1, "Access-Control-Allow-Origin") + ), + ?_assertEqual( + ?EXPOSED_HEADERS, + header(Headers1, "Access-Control-Expose-Headers") + ) ]. - test_preflight_origin_helper_(OwnerConfig, Origin, ExpectedOrigin) -> Headers = [ {"Origin", Origin}, @@ -506,76 +500,82 @@ test_preflight_origin_helper_(OwnerConfig, Origin, ExpectedOrigin) -> ], Req = mock_request('OPTIONS', "/", Headers), Headers1 = chttpd_cors:headers(Req, Headers, Origin, OwnerConfig), - [?_assertEqual(ExpectedOrigin, - header(Headers1, "Access-Control-Allow-Origin")) + [ + ?_assertEqual( + ExpectedOrigin, + header(Headers1, "Access-Control-Allow-Origin") + ) ]. - test_preflight_with_port_no_origin_(OwnerConfig) -> Origin = ?DEFAULT_ORIGIN ++ ":5984", test_preflight_origin_helper_(OwnerConfig, Origin, undefined). - test_preflight_with_port_with_origin_(OwnerConfig) -> Origin = ?DEFAULT_ORIGIN ++ ":5984", test_preflight_origin_helper_(OwnerConfig, Origin, Origin). - test_preflight_with_scheme_no_origin_(OwnerConfig) -> test_preflight_origin_helper_(OwnerConfig, ?DEFAULT_ORIGIN_HTTPS, undefined). - test_preflight_with_scheme_with_origin_(OwnerConfig) -> Origin = ?DEFAULT_ORIGIN_HTTPS, test_preflight_origin_helper_(OwnerConfig, Origin, Origin). - test_preflight_with_scheme_port_no_origin_(OwnerConfig) -> Origin = ?DEFAULT_ORIGIN_HTTPS ++ ":5984", test_preflight_origin_helper_(OwnerConfig, Origin, undefined). - test_preflight_with_scheme_port_with_origin_(OwnerConfig) -> Origin = ?DEFAULT_ORIGIN_HTTPS ++ ":5984", test_preflight_origin_helper_(OwnerConfig, Origin, Origin). - test_case_sensitive_mismatch_of_allowed_origins_(OwnerConfig) -> Origin = "http://EXAMPLE.COM", Headers = [{"Origin", Origin}], Req = mock_request('GET', "/", Headers), Headers1 = chttpd_cors:headers(Req, Headers, Origin, OwnerConfig), [ - ?_assertEqual(?DEFAULT_ORIGIN, - header(Headers1, "Access-Control-Allow-Origin")), - ?_assertEqual(?EXPOSED_HEADERS, - header(Headers1, "Access-Control-Expose-Headers")) + ?_assertEqual( + ?DEFAULT_ORIGIN, + header(Headers1, "Access-Control-Allow-Origin") + ), + ?_assertEqual( + ?EXPOSED_HEADERS, + header(Headers1, "Access-Control-Expose-Headers") + ) ]. - test_db_request_credentials_header_off_(OwnerConfig) -> Origin = ?DEFAULT_ORIGIN, Headers = [{"Origin", Origin}], Req = mock_request('GET', "/", Headers), Headers1 = chttpd_cors:headers(Req, Headers, Origin, OwnerConfig), [ - ?_assertEqual(?DEFAULT_ORIGIN, - header(Headers1, "Access-Control-Allow-Origin")), - ?_assertEqual(undefined, - header(Headers1, "Access-Control-Allow-Credentials")) + ?_assertEqual( + ?DEFAULT_ORIGIN, + header(Headers1, "Access-Control-Allow-Origin") + ), + ?_assertEqual( + undefined, + header(Headers1, "Access-Control-Allow-Credentials") + ) ]. - test_db_request_credentials_header_on_(OwnerConfig) -> Origin = ?DEFAULT_ORIGIN, Headers = [{"Origin", Origin}], Req = mock_request('GET', "/", Headers), Headers1 = chttpd_cors:headers(Req, Headers, Origin, OwnerConfig), [ - ?_assertEqual(?DEFAULT_ORIGIN, - header(Headers1, "Access-Control-Allow-Origin")), - ?_assertEqual("true", - header(Headers1, "Access-Control-Allow-Credentials")) + ?_assertEqual( + ?DEFAULT_ORIGIN, + header(Headers1, "Access-Control-Allow-Origin") + ), + ?_assertEqual( + "true", + header(Headers1, "Access-Control-Allow-Credentials") + ) ]. split_list(S) -> diff --git a/src/chttpd/test/eunit/chttpd_csp_tests.erl b/src/chttpd/test/eunit/chttpd_csp_tests.erl index 88194fdf8..4c77c5ab0 100644 --- a/src/chttpd/test/eunit/chttpd_csp_tests.erl +++ b/src/chttpd/test/eunit/chttpd_csp_tests.erl @@ -34,7 +34,9 @@ -define(TDEF(Name), {atom_to_list(Name), fun Name/1}). -define(TDEF(Name, Timeout), {atom_to_list(Name), Timeout, fun Name/1}). -define(TDEF_FE(Name), fun(Arg) -> {atom_to_list(Name), ?_test(Name(Arg))} end). --define(TDEF_FE(Name, Timeout), fun(Arg) -> {atom_to_list(Name), {timeout, Timeout, ?_test(Name(Arg))}} end). +-define(TDEF_FE(Name, Timeout), fun(Arg) -> + {atom_to_list(Name), {timeout, Timeout, ?_test(Name(Arg))}} +end). csp_test_() -> { @@ -101,7 +103,7 @@ sandbox_ddoc_attachments(DbName) -> sandbox_shows(DbName) -> DbUrl = base_url() ++ "/" ++ DbName, - DDocUrl = DbUrl ++ "/" ++ ?DDOC1, + DDocUrl = DbUrl ++ "/" ++ ?DDOC1, Url = DDocUrl ++ "/_show/" ++ ?SHOW1 ++ "/" ++ ?DOC1, ?assertEqual({200, true}, req(get, ?ACC, Url)), config:set("csp", "showlist_enable", "false", false), @@ -109,21 +111,22 @@ sandbox_shows(DbName) -> sandbox_lists(DbName) -> DbUrl = base_url() ++ "/" ++ DbName, - DDocUrl = DbUrl ++ "/" ++ ?DDOC1, + DDocUrl = DbUrl ++ "/" ++ ?DDOC1, Url = DDocUrl ++ "/_list/" ++ ?LIST1 ++ "/" ++ ?VIEW1, ?assertEqual({200, true}, req(get, ?ACC, Url)), config:set("csp", "showlist_enable", "false", false), ?assertEqual({200, false}, req(get, ?ACC, Url)). - should_not_return_any_csp_headers_when_disabled(_DbName) -> - ?_assertEqual(undefined, + ?_assertEqual( + undefined, begin ok = config:set("csp", "utils_enable", "false", false), ok = config:set("csp", "enable", "false", false), {ok, _, Headers, _} = test_request:get(base_url() ++ "/_utils/"), proplists:get_value("Content-Security-Policy", Headers) - end). + end + ). should_apply_default_policy(_DbName) -> ?_assertEqual( @@ -132,7 +135,8 @@ should_apply_default_policy(_DbName) -> begin {ok, _, Headers, _} = test_request:get(base_url() ++ "/_utils/"), proplists:get_value("Content-Security-Policy", Headers) - end). + end + ). should_apply_default_policy_with_legacy_config(_DbName) -> ?_assertEqual( @@ -143,17 +147,23 @@ should_apply_default_policy_with_legacy_config(_DbName) -> ok = config:set("csp", "enable", "true", false), {ok, _, Headers, _} = test_request:get(base_url() ++ "/_utils/"), proplists:get_value("Content-Security-Policy", Headers) - end). + end + ). should_return_custom_policy(_DbName) -> - ?_assertEqual("default-src 'http://example.com';", + ?_assertEqual( + "default-src 'http://example.com';", begin - ok = config:set("csp", "utils_header_value", - "default-src 'http://example.com';", false), + ok = config:set( + "csp", + "utils_header_value", + "default-src 'http://example.com';", + false + ), {ok, _, Headers, _} = test_request:get(base_url() ++ "/_utils/"), proplists:get_value("Content-Security-Policy", Headers) - end). - + end + ). % Utility functions @@ -204,19 +214,18 @@ setup() -> <<?SHOW1>> => <<"function(doc, req) {return '<h1>show1!</h1>';}">> }, <<"lists">> => #{ - <<?LIST1>> => <<"function(head, req) {", - "var row;", - "while(row = getRow()){ send(row.key); };", - "}">> + <<?LIST1>> => + <<"function(head, req) {", "var row;", "while(row = getRow()){ send(row.key); };", + "}">> } }), ok = create_doc(?ACC, DbName, #{<<"_id">> => <<?LDOC1>>}), DbName. cleanup(DbName) -> - config:delete("csp", "utils_enable", _Persist=false), - config:delete("csp", "attachments_enable", _Persist=false), - config:delete("csp", "showlist_enable", _Persist=false), + config:delete("csp", "utils_enable", _Persist = false), + config:delete("csp", "attachments_enable", _Persist = false), + config:delete("csp", "showlist_enable", _Persist = false), DbUrl = base_url() ++ "/" ++ DbName, {200, _} = req(delete, ?ADM, DbUrl), UsersDb = config:get("chttpd_auth", "authentication_db"), @@ -229,8 +238,12 @@ base_url() -> Port = integer_to_list(mochiweb_socket_server:get(chttpd, port)), "http://" ++ Addr ++ ":" ++ Port. -create_user(UsersDb, Name, Pass, Roles) when is_list(UsersDb), - is_binary(Name), is_binary(Pass), is_list(Roles) -> +create_user(UsersDb, Name, Pass, Roles) when + is_list(UsersDb), + is_binary(Name), + is_binary(Pass), + is_list(Roles) +-> Body = #{ <<"name">> => Name, <<"type">> => <<"user">>, diff --git a/src/chttpd/test/eunit/chttpd_db_attachment_size_tests.erl b/src/chttpd/test/eunit/chttpd_db_attachment_size_tests.erl index 227b29c5b..e3975bb6e 100644 --- a/src/chttpd/test/eunit/chttpd_db_attachment_size_tests.erl +++ b/src/chttpd/test/eunit/chttpd_db_attachment_size_tests.erl @@ -19,14 +19,12 @@ -define(PASS, "pass"). -define(AUTH, {basic_auth, {?USER, ?PASS}}). -define(CONTENT_JSON, {"Content-Type", "application/json"}). --define(CONTENT_MULTI_RELATED, {"Content-Type", - "multipart/related;boundary=\"bound\""}). - +-define(CONTENT_MULTI_RELATED, {"Content-Type", "multipart/related;boundary=\"bound\""}). setup() -> Hashed = couch_passwords:hash_admin_password(?PASS), - ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist=false), - ok = config:set("couchdb", "max_attachment_size", "50", _Persist=false), + ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist = false), + ok = config:set("couchdb", "max_attachment_size", "50", _Persist = false), TmpDb = ?tempdb(), Addr = config:get("chttpd", "bind_address", "127.0.0.1"), Port = integer_to_list(mochiweb_socket_server:get(chttpd, port)), @@ -35,13 +33,11 @@ setup() -> add_doc(Url, "doc1"), Url. - teardown(Url) -> delete_db(Url), - ok = config:delete("admins", ?USER, _Persist=false), + ok = config:delete("admins", ?USER, _Persist = false), ok = config:delete("couchdb", "max_attachment_size"). - attachment_size_test_() -> { "chttpd max_attachment_size tests", @@ -51,7 +47,8 @@ attachment_size_test_() -> fun chttpd_test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun put_inline/1, fun put_simple/1, @@ -63,14 +60,12 @@ attachment_size_test_() -> } }. - put_inline(Url) -> - ?_test(begin - Status = put_inline(Url, "doc2", 50), - ?assert(Status =:= 201 orelse Status =:= 202), - ?assertEqual(413, put_inline(Url, "doc3", 51)) - end). - + ?_test(begin + Status = put_inline(Url, "doc2", 50), + ?assert(Status =:= 201 orelse Status =:= 202), + ?assertEqual(413, put_inline(Url, "doc3", 51)) + end). put_simple(Url) -> ?_test(begin @@ -85,9 +80,8 @@ put_simple(Url) -> ?assertEqual(413, Status2) end). - put_simple_chunked(Url) -> - ?_test(begin + ?_test(begin Headers = [{"Content-Type", "app/binary"}], Rev1 = doc_rev(Url, "doc1"), DataFun1 = data_stream_fun(50), @@ -99,7 +93,6 @@ put_simple_chunked(Url) -> ?assertEqual(413, Status2) end). - put_mp_related(Url) -> ?_test(begin Headers = [?CONTENT_MULTI_RELATED], @@ -111,89 +104,84 @@ put_mp_related(Url) -> ?assertEqual(413, Status2) end). - put_chunked_mp_related(Url) -> ?_test(begin - Headers = [?CONTENT_MULTI_RELATED], - Body = mp_body(50), - Status = put_req_chunked(Url ++ "/doc4", Headers, Body), - ?assert(Status =:= 201 orelse Status =:= 202) + Headers = [?CONTENT_MULTI_RELATED], + Body = mp_body(50), + Status = put_req_chunked(Url ++ "/doc4", Headers, Body), + ?assert(Status =:= 201 orelse Status =:= 202) end). - % Helper functions create_db(Url) -> Status = put_req(Url, "{}"), ?assert(Status =:= 201 orelse Status =:= 202). - add_doc(Url, DocId) -> Status = put_req(Url ++ "/" ++ DocId, "{}"), ?assert(Status =:= 201 orelse Status =:= 202). - delete_db(Url) -> {ok, 200, _, _} = test_request:delete(Url, [?AUTH]). - put_inline(Url, DocId, Size) -> - Doc = "{\"_attachments\": {\"att1\":{" + Doc = + "{\"_attachments\": {\"att1\":{" "\"content_type\": \"app/binary\", " - "\"data\": \"" ++ data_b64(Size) ++ "\"" - "}}}", + "\"data\": \"" ++ data_b64(Size) ++ + "\"" + "}}}", put_req(Url ++ "/" ++ DocId, Doc). - mp_body(AttSize) -> AttData = data(AttSize), SizeStr = integer_to_list(AttSize), - string:join([ - "--bound", - - "Content-Type: application/json", + string:join( + [ + "--bound", - "", + "Content-Type: application/json", - "{\"_id\":\"doc2\", \"_attachments\":{\"att\":" - "{\"content_type\":\"app/binary\", \"length\":" ++ SizeStr ++ "," - "\"follows\":true}}}", + "", - "--bound", + "{\"_id\":\"doc2\", \"_attachments\":{\"att\":" + "{\"content_type\":\"app/binary\", \"length\":" ++ SizeStr ++ + "," + "\"follows\":true}}}", - "Content-Disposition: attachment; filename=\"att\"", + "--bound", - "Content-Type: app/binary", + "Content-Disposition: attachment; filename=\"att\"", - "", + "Content-Type: app/binary", - AttData, + "", - "--bound--" - ], "\r\n"). + AttData, + "--bound--" + ], + "\r\n" + ). doc_rev(Url, DocId) -> {200, ResultProps} = get_req(Url ++ "/" ++ DocId), {<<"_rev">>, BinRev} = lists:keyfind(<<"_rev">>, 1, ResultProps), binary_to_list(BinRev). - put_req(Url, Body) -> put_req(Url, [], Body). - put_req(Url, Headers, Body) -> {ok, Status, _, _} = test_request:put(Url, Headers ++ [?AUTH], Body), Status. - put_req_chunked(Url, Headers, Body) -> Opts = [{transfer_encoding, {chunked, 1}}], {ok, Status, _, _} = test_request:put(Url, Headers ++ [?AUTH], Body, Opts), Status. - get_req(Url) -> {ok, Status, _, ResultBody} = test_request:get(Url, [?CONTENT_JSON, ?AUTH]), {[_ | _] = ResultProps} = ?JSON_DECODE(ResultBody), @@ -202,15 +190,14 @@ get_req(Url) -> % Data streaming generator for ibrowse client. ibrowse will repeatedly call the % function with State and it should return {ok, Data, NewState} or eof at end. data_stream_fun(Size) -> - Fun = fun(0) -> eof; (BytesLeft) -> - {ok, <<"x">>, BytesLeft - 1} + Fun = fun + (0) -> eof; + (BytesLeft) -> {ok, <<"x">>, BytesLeft - 1} end, {Fun, Size}. - data(Size) -> string:copies("x", Size). - data_b64(Size) -> base64:encode_to_string(data(Size)). diff --git a/src/chttpd/test/eunit/chttpd_db_bulk_get_multipart_test.erl b/src/chttpd/test/eunit/chttpd_db_bulk_get_multipart_test.erl index 86a8eab1a..91a3eaf19 100644 --- a/src/chttpd/test/eunit/chttpd_db_bulk_get_multipart_test.erl +++ b/src/chttpd/test/eunit/chttpd_db_bulk_get_multipart_test.erl @@ -17,7 +17,6 @@ -define(TIMEOUT, 3000). - setup_all() -> mock(config), mock(chttpd), @@ -27,11 +26,9 @@ setup_all() -> mock(fabric), mock(mochireq). - teardown_all(_) -> meck:unload(). - setup() -> meck:reset([ config, @@ -44,11 +41,9 @@ setup() -> ]), spawn_accumulator(). - teardown(Pid) -> ok = stop_accumulator(Pid). - bulk_get_test_() -> { "/db/_bulk_get tests", @@ -75,36 +70,36 @@ bulk_get_test_() -> } }. - should_require_docs_field(_) -> Req = fake_request({[{}]}), - Db = test_util:fake_db([{name, <<"foo">>}]), + Db = test_util:fake_db([{name, <<"foo">>}]), ?_assertThrow({bad_request, _}, chttpd_db:db_req(Req, Db)). - should_not_accept_specific_query_params(_) -> Req = fake_request({[{<<"docs">>, []}]}), - Db = test_util:fake_db([{name, <<"foo">>}]), - lists:map(fun (Param) -> - {Param, ?_assertThrow({bad_request, _}, begin - BadReq = Req#httpd{qs = [{Param, ""}]}, - chttpd_db:db_req(BadReq, Db) - end)} - end, ["rev", "open_revs", "atts_since", "w", "new_edits"]). - + Db = test_util:fake_db([{name, <<"foo">>}]), + lists:map( + fun(Param) -> + {Param, + ?_assertThrow({bad_request, _}, begin + BadReq = Req#httpd{qs = [{Param, ""}]}, + chttpd_db:db_req(BadReq, Db) + end)} + end, + ["rev", "open_revs", "atts_since", "w", "new_edits"] + ). should_return_empty_results_on_no_docs(Pid) -> Req = fake_request({[{<<"docs">>, []}]}), - Db = test_util:fake_db([{name, <<"foo">>}]), + Db = test_util:fake_db([{name, <<"foo">>}]), chttpd_db:db_req(Req, Db), Results = get_results_from_response(Pid), ?_assertEqual([], Results). - should_get_doc_with_all_revs(Pid) -> DocId = <<"docudoc">>, Req = fake_request(DocId), - Db = test_util:fake_db([{name, <<"foo">>}]), + Db = test_util:fake_db([{name, <<"foo">>}]), DocRevA = #doc{id = DocId, body = {[{<<"_rev">>, <<"1-ABC">>}]}}, DocRevB = #doc{id = DocId, body = {[{<<"_rev">>, <<"1-CDE">>}]}}, @@ -115,95 +110,126 @@ should_get_doc_with_all_revs(Pid) -> Result = get_results_from_response(Pid), ?_assertEqual(DocId, couch_util:get_value(<<"_id">>, Result)). - should_validate_doc_with_bad_id(Pid) -> DocId = <<"_docudoc">>, Req = fake_request(DocId), - Db = test_util:fake_db([{name, <<"foo">>}]), + Db = test_util:fake_db([{name, <<"foo">>}]), chttpd_db:db_req(Req, Db), Result = get_results_from_response(Pid), ?assertEqual(DocId, couch_util:get_value(<<"id">>, Result)), - ?_assertMatch([{<<"id">>, DocId}, - {<<"rev">>, null}, - {<<"error">>, <<"illegal_docid">>}, - {<<"reason">>, _}], Result). - + ?_assertMatch( + [ + {<<"id">>, DocId}, + {<<"rev">>, null}, + {<<"error">>, <<"illegal_docid">>}, + {<<"reason">>, _} + ], + Result + ). should_validate_doc_with_bad_rev(Pid) -> DocId = <<"docudoc">>, Rev = <<"revorev">>, Req = fake_request(DocId, Rev), - Db = test_util:fake_db([{name, <<"foo">>}]), + Db = test_util:fake_db([{name, <<"foo">>}]), chttpd_db:db_req(Req, Db), Result = get_results_from_response(Pid), ?assertEqual(DocId, couch_util:get_value(<<"id">>, Result)), - ?_assertMatch([{<<"id">>, DocId}, - {<<"rev">>, Rev}, - {<<"error">>, <<"bad_request">>}, - {<<"reason">>, _}], Result). - + ?_assertMatch( + [ + {<<"id">>, DocId}, + {<<"rev">>, Rev}, + {<<"error">>, <<"bad_request">>}, + {<<"reason">>, _} + ], + Result + ). should_validate_missing_doc(Pid) -> DocId = <<"docudoc">>, Rev = <<"1-revorev">>, Req = fake_request(DocId, Rev), - Db = test_util:fake_db([{name, <<"foo">>}]), - mock_open_revs([{1,<<"revorev">>}], {ok, []}), + Db = test_util:fake_db([{name, <<"foo">>}]), + mock_open_revs([{1, <<"revorev">>}], {ok, []}), chttpd_db:db_req(Req, Db), Result = get_results_from_response(Pid), ?assertEqual(DocId, couch_util:get_value(<<"id">>, Result)), - ?_assertMatch([{<<"id">>, DocId}, - {<<"rev">>, Rev}, - {<<"error">>, <<"not_found">>}, - {<<"reason">>, _}], Result). - + ?_assertMatch( + [ + {<<"id">>, DocId}, + {<<"rev">>, Rev}, + {<<"error">>, <<"not_found">>}, + {<<"reason">>, _} + ], + Result + ). should_validate_bad_atts_since(Pid) -> DocId = <<"docudoc">>, Rev = <<"1-revorev">>, Req = fake_request(DocId, Rev, <<"badattsince">>), - Db = test_util:fake_db([{name, <<"foo">>}]), - mock_open_revs([{1,<<"revorev">>}], {ok, []}), + Db = test_util:fake_db([{name, <<"foo">>}]), + mock_open_revs([{1, <<"revorev">>}], {ok, []}), chttpd_db:db_req(Req, Db), Result = get_results_from_response(Pid), ?assertEqual(DocId, couch_util:get_value(<<"id">>, Result)), - ?_assertMatch([{<<"id">>, DocId}, - {<<"rev">>, <<"badattsince">>}, - {<<"error">>, <<"bad_request">>}, - {<<"reason">>, _}], Result). - + ?_assertMatch( + [ + {<<"id">>, DocId}, + {<<"rev">>, <<"badattsince">>}, + {<<"error">>, <<"bad_request">>}, + {<<"reason">>, _} + ], + Result + ). should_include_attachments_when_atts_since_specified(_) -> DocId = <<"docudoc">>, Rev = <<"1-revorev">>, Req = fake_request(DocId, Rev, [<<"1-abc">>]), - Db = test_util:fake_db([{name, <<"foo">>}]), - mock_open_revs([{1,<<"revorev">>}], {ok, []}), + Db = test_util:fake_db([{name, <<"foo">>}]), + mock_open_revs([{1, <<"revorev">>}], {ok, []}), chttpd_db:db_req(Req, Db), - ?_assert(meck:called(fabric, open_revs, - ['_', DocId, [{1, <<"revorev">>}], - [{atts_since, [{1, <<"abc">>}]}, attachments, - {user_ctx, undefined}]])). + ?_assert( + meck:called( + fabric, + open_revs, + [ + '_', + DocId, + [{1, <<"revorev">>}], + [ + {atts_since, [{1, <<"abc">>}]}, + attachments, + {user_ctx, undefined} + ] + ] + ) + ). %% helpers fake_request(Payload) when is_tuple(Payload) -> - #httpd{method='POST', path_parts=[<<"db">>, <<"_bulk_get">>], - mochi_req=mochireq, req_body=Payload}; + #httpd{ + method = 'POST', + path_parts = [<<"db">>, <<"_bulk_get">>], + mochi_req = mochireq, + req_body = Payload + }; fake_request(DocId) when is_binary(DocId) -> fake_request({[{<<"docs">>, [{[{<<"id">>, DocId}]}]}]}). @@ -211,25 +237,36 @@ fake_request(DocId, Rev) -> fake_request({[{<<"docs">>, [{[{<<"id">>, DocId}, {<<"rev">>, Rev}]}]}]}). fake_request(DocId, Rev, AttsSince) -> - fake_request({[{<<"docs">>, [{[{<<"id">>, DocId}, - {<<"rev">>, Rev}, - {<<"atts_since">>, AttsSince}]}]}]}). - + fake_request( + {[ + {<<"docs">>, [ + {[ + {<<"id">>, DocId}, + {<<"rev">>, Rev}, + {<<"atts_since">>, AttsSince} + ]} + ]} + ]} + ). mock_open_revs(RevsReq0, RevsResp) -> - ok = meck:expect(fabric, open_revs, - fun(_, _, RevsReq1, _) -> - ?assertEqual(RevsReq0, RevsReq1), - RevsResp - end). - + ok = meck:expect( + fabric, + open_revs, + fun(_, _, RevsReq1, _) -> + ?assertEqual(RevsReq0, RevsReq1), + RevsResp + end + ). mock(mochireq) -> ok = meck:new(mochireq, [non_strict]), ok = meck:expect(mochireq, parse_qs, fun() -> [] end), - ok = meck:expect(mochireq, accepts_content_type, fun("multipart/mixed") -> true; - ("multipart/related") -> true; - (_) -> false end), + ok = meck:expect(mochireq, accepts_content_type, fun + ("multipart/mixed") -> true; + ("multipart/related") -> true; + (_) -> false + end), ok; mock(couch_httpd) -> ok = meck:new(couch_httpd, [passthrough]), @@ -243,7 +280,7 @@ mock(chttpd) -> ok = meck:expect(chttpd, start_chunked_response, fun(_, _, _) -> {ok, nil} end), ok = meck:expect(chttpd, end_json_response, fun(_) -> ok end), ok = meck:expect(chttpd, send_chunk, fun send_chunk/2), - ok = meck:expect(chttpd, json_body_obj, fun (#httpd{req_body=Body}) -> Body end), + ok = meck:expect(chttpd, json_body_obj, fun(#httpd{req_body = Body}) -> Body end), ok; mock(couch_epi) -> ok = meck:new(couch_epi, [passthrough]), @@ -266,7 +303,6 @@ mock(config) -> ok = meck:expect(config, get, fun(_, _, Default) -> Default end), ok. - spawn_accumulator() -> Parent = self(), Pid = spawn(fun() -> accumulator_loop(Parent, []) end), @@ -282,7 +318,7 @@ accumulator_loop(Parent, Acc) -> accumulator_loop(Parent, Acc); {put, Ref, Chunk} -> Parent ! {ok, Ref}, - accumulator_loop(Parent, [Chunk|Acc]) + accumulator_loop(Parent, [Chunk | Acc]) end. stop_accumulator(Pid) -> @@ -295,10 +331,9 @@ stop_accumulator(Pid) -> throw({timeout, <<"process stop timeout">>}) end. - send_chunk(_, []) -> {ok, nil}; -send_chunk(_Req, [H|T]=Chunk) when is_list(Chunk) -> +send_chunk(_Req, [H | T] = Chunk) when is_list(Chunk) -> send_chunk(_Req, H), send_chunk(_Req, T); send_chunk(_, Chunk) -> @@ -311,7 +346,6 @@ send_chunk(_, Chunk) -> throw({timeout, <<"send chunk timeout">>}) end. - get_response(Pid) -> Ref = make_ref(), Pid ! {get, Ref}, @@ -325,8 +359,8 @@ get_response(Pid) -> get_results_from_response(Pid) -> case get_response(Pid) of [] -> - []; + []; Result -> - {Result1} = ?JSON_DECODE(lists:nth(2, Result)), - Result1 + {Result1} = ?JSON_DECODE(lists:nth(2, Result)), + Result1 end. diff --git a/src/chttpd/test/eunit/chttpd_db_bulk_get_test.erl b/src/chttpd/test/eunit/chttpd_db_bulk_get_test.erl index 1a3411254..81dfe098b 100644 --- a/src/chttpd/test/eunit/chttpd_db_bulk_get_test.erl +++ b/src/chttpd/test/eunit/chttpd_db_bulk_get_test.erl @@ -17,7 +17,6 @@ -define(TIMEOUT, 3000). - setup_all() -> mock(config), mock(chttpd), @@ -27,19 +26,15 @@ setup_all() -> mock(fabric), mock(mochireq). - teardown_all(_) -> meck:unload(). - setup() -> spawn_accumulator(). - teardown(Pid) -> ok = stop_accumulator(Pid). - bulk_get_test_() -> { "/db/_bulk_get tests", @@ -66,21 +61,22 @@ bulk_get_test_() -> } }. - should_require_docs_field(_) -> Req = fake_request({[{}]}), ?_assertThrow({bad_request, _}, chttpd_db:db_req(Req, nil)). - should_not_accept_specific_query_params(_) -> Req = fake_request({[{<<"docs">>, []}]}), - lists:map(fun (Param) -> - {Param, ?_assertThrow({bad_request, _}, begin - BadReq = Req#httpd{qs = [{Param, ""}]}, - chttpd_db:db_req(BadReq, nil) - end)} - end, ["rev", "open_revs", "atts_since", "w", "new_edits"]). - + lists:map( + fun(Param) -> + {Param, + ?_assertThrow({bad_request, _}, begin + BadReq = Req#httpd{qs = [{Param, ""}]}, + chttpd_db:db_req(BadReq, nil) + end)} + end, + ["rev", "open_revs", "atts_since", "w", "new_edits"] + ). should_return_empty_results_on_no_docs(Pid) -> Req = fake_request({[{<<"docs">>, []}]}), @@ -88,7 +84,6 @@ should_return_empty_results_on_no_docs(Pid) -> Results = get_results_from_response(Pid), ?_assertEqual([], Results). - should_get_doc_with_all_revs(Pid) -> DocId = <<"docudoc">>, Req = fake_request(DocId), @@ -114,7 +109,6 @@ should_get_doc_with_all_revs(Pid) -> ?_assertEqual([RevA, RevB], [DocA, DocB]). - should_validate_doc_with_bad_id(Pid) -> DocId = <<"_docudoc">>, @@ -130,12 +124,15 @@ should_validate_doc_with_bad_id(Pid) -> Doc = couch_util:get_value(<<"error">>, DocResult), - ?_assertMatch({[{<<"id">>, DocId}, - {<<"rev">>, null}, - {<<"error">>, <<"illegal_docid">>}, - {<<"reason">>, _}]}, - Doc). - + ?_assertMatch( + {[ + {<<"id">>, DocId}, + {<<"rev">>, null}, + {<<"error">>, <<"illegal_docid">>}, + {<<"reason">>, _} + ]}, + Doc + ). should_validate_doc_with_bad_rev(Pid) -> DocId = <<"docudoc">>, @@ -153,19 +150,22 @@ should_validate_doc_with_bad_rev(Pid) -> Doc = couch_util:get_value(<<"error">>, DocResult), - ?_assertMatch({[{<<"id">>, DocId}, - {<<"rev">>, Rev}, - {<<"error">>, <<"bad_request">>}, - {<<"reason">>, _}]}, - Doc). - + ?_assertMatch( + {[ + {<<"id">>, DocId}, + {<<"rev">>, Rev}, + {<<"error">>, <<"bad_request">>}, + {<<"reason">>, _} + ]}, + Doc + ). should_validate_missing_doc(Pid) -> DocId = <<"docudoc">>, Rev = <<"1-revorev">>, Req = fake_request(DocId, Rev), - mock_open_revs([{1,<<"revorev">>}], {ok, []}), + mock_open_revs([{1, <<"revorev">>}], {ok, []}), chttpd_db:db_req(Req, test_util:fake_db([{name, <<"foo">>}])), [{Result}] = get_results_from_response(Pid), @@ -177,19 +177,22 @@ should_validate_missing_doc(Pid) -> Doc = couch_util:get_value(<<"error">>, DocResult), - ?_assertMatch({[{<<"id">>, DocId}, - {<<"rev">>, Rev}, - {<<"error">>, <<"not_found">>}, - {<<"reason">>, _}]}, - Doc). - + ?_assertMatch( + {[ + {<<"id">>, DocId}, + {<<"rev">>, Rev}, + {<<"error">>, <<"not_found">>}, + {<<"reason">>, _} + ]}, + Doc + ). should_validate_bad_atts_since(Pid) -> DocId = <<"docudoc">>, Rev = <<"1-revorev">>, Req = fake_request(DocId, Rev, <<"badattsince">>), - mock_open_revs([{1,<<"revorev">>}], {ok, []}), + mock_open_revs([{1, <<"revorev">>}], {ok, []}), chttpd_db:db_req(Req, test_util:fake_db([{name, <<"foo">>}])), [{Result}] = get_results_from_response(Pid), @@ -201,31 +204,50 @@ should_validate_bad_atts_since(Pid) -> Doc = couch_util:get_value(<<"error">>, DocResult), - ?_assertMatch({[{<<"id">>, DocId}, - {<<"rev">>, <<"badattsince">>}, - {<<"error">>, <<"bad_request">>}, - {<<"reason">>, _}]}, - Doc). - + ?_assertMatch( + {[ + {<<"id">>, DocId}, + {<<"rev">>, <<"badattsince">>}, + {<<"error">>, <<"bad_request">>}, + {<<"reason">>, _} + ]}, + Doc + ). should_include_attachments_when_atts_since_specified(_) -> DocId = <<"docudoc">>, Rev = <<"1-revorev">>, Req = fake_request(DocId, Rev, [<<"1-abc">>]), - mock_open_revs([{1,<<"revorev">>}], {ok, []}), + mock_open_revs([{1, <<"revorev">>}], {ok, []}), chttpd_db:db_req(Req, test_util:fake_db([{name, <<"foo">>}])), - ?_assert(meck:called(fabric, open_revs, - ['_', DocId, [{1, <<"revorev">>}], - [{atts_since, [{1, <<"abc">>}]}, attachments, - {user_ctx, undefined}]])). + ?_assert( + meck:called( + fabric, + open_revs, + [ + '_', + DocId, + [{1, <<"revorev">>}], + [ + {atts_since, [{1, <<"abc">>}]}, + attachments, + {user_ctx, undefined} + ] + ] + ) + ). %% helpers fake_request(Payload) when is_tuple(Payload) -> - #httpd{method='POST', path_parts=[<<"db">>, <<"_bulk_get">>], - mochi_req=mochireq, req_body=Payload}; + #httpd{ + method = 'POST', + path_parts = [<<"db">>, <<"_bulk_get">>], + mochi_req = mochireq, + req_body = Payload + }; fake_request(DocId) when is_binary(DocId) -> fake_request({[{<<"docs">>, [{[{<<"id">>, DocId}]}]}]}). @@ -233,18 +255,27 @@ fake_request(DocId, Rev) -> fake_request({[{<<"docs">>, [{[{<<"id">>, DocId}, {<<"rev">>, Rev}]}]}]}). fake_request(DocId, Rev, AttsSince) -> - fake_request({[{<<"docs">>, [{[{<<"id">>, DocId}, - {<<"rev">>, Rev}, - {<<"atts_since">>, AttsSince}]}]}]}). - + fake_request( + {[ + {<<"docs">>, [ + {[ + {<<"id">>, DocId}, + {<<"rev">>, Rev}, + {<<"atts_since">>, AttsSince} + ]} + ]} + ]} + ). mock_open_revs(RevsReq0, RevsResp) -> - ok = meck:expect(fabric, open_revs, - fun(_, _, RevsReq1, _) -> - ?assertEqual(RevsReq0, RevsReq1), - RevsResp - end). - + ok = meck:expect( + fabric, + open_revs, + fun(_, _, RevsReq1, _) -> + ?assertEqual(RevsReq0, RevsReq1), + RevsResp + end + ). mock(mochireq) -> ok = meck:new(mochireq, [non_strict]), @@ -260,7 +291,7 @@ mock(chttpd) -> ok = meck:expect(chttpd, start_json_response, fun(_, _) -> {ok, nil} end), ok = meck:expect(chttpd, end_json_response, fun(_) -> ok end), ok = meck:expect(chttpd, send_chunk, fun send_chunk/2), - ok = meck:expect(chttpd, json_body_obj, fun (#httpd{req_body=Body}) -> Body end), + ok = meck:expect(chttpd, json_body_obj, fun(#httpd{req_body = Body}) -> Body end), ok; mock(couch_epi) -> ok = meck:new(couch_epi, [passthrough]), @@ -283,7 +314,6 @@ mock(config) -> ok = meck:expect(config, get, fun(_, _, Default) -> Default end), ok. - spawn_accumulator() -> Parent = self(), Pid = spawn(fun() -> accumulator_loop(Parent, []) end), @@ -299,7 +329,7 @@ accumulator_loop(Parent, Acc) -> accumulator_loop(Parent, Acc); {put, Ref, Chunk} -> Parent ! {ok, Ref}, - accumulator_loop(Parent, [Chunk|Acc]) + accumulator_loop(Parent, [Chunk | Acc]) end. stop_accumulator(Pid) -> @@ -312,10 +342,9 @@ stop_accumulator(Pid) -> throw({timeout, <<"process stop timeout">>}) end. - send_chunk(_, []) -> {ok, nil}; -send_chunk(_Req, [H|T]=Chunk) when is_list(Chunk) -> +send_chunk(_Req, [H | T] = Chunk) when is_list(Chunk) -> send_chunk(_Req, H), send_chunk(_Req, T); send_chunk(_, Chunk) -> @@ -328,7 +357,6 @@ send_chunk(_, Chunk) -> throw({timeout, <<"send chunk timeout">>}) end. - get_response(Pid) -> Ref = make_ref(), Pid ! {get, Ref}, @@ -339,7 +367,6 @@ get_response(Pid) -> throw({timeout, <<"get response timeout">>}) end. - get_results_from_response(Pid) -> {Resp} = get_response(Pid), couch_util:get_value(<<"results">>, Resp). diff --git a/src/chttpd/test/eunit/chttpd_db_doc_size_tests.erl b/src/chttpd/test/eunit/chttpd_db_doc_size_tests.erl index 88e2797a3..01ef16f23 100644 --- a/src/chttpd/test/eunit/chttpd_db_doc_size_tests.erl +++ b/src/chttpd/test/eunit/chttpd_db_doc_size_tests.erl @@ -19,15 +19,12 @@ -define(PASS, "pass"). -define(AUTH, {basic_auth, {?USER, ?PASS}}). -define(CONTENT_JSON, {"Content-Type", "application/json"}). --define(CONTENT_MULTI_RELATED, {"Content-Type", - "multipart/related;boundary=\"bound\""}). --define(CONTENT_MULTI_FORM, {"Content-Type", - "multipart/form-data;boundary=\"bound\""}). - +-define(CONTENT_MULTI_RELATED, {"Content-Type", "multipart/related;boundary=\"bound\""}). +-define(CONTENT_MULTI_FORM, {"Content-Type", "multipart/form-data;boundary=\"bound\""}). setup() -> Hashed = couch_passwords:hash_admin_password(?PASS), - ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist=false), + ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist = false), ok = config:set("couchdb", "max_document_size", "50"), TmpDb = ?tempdb(), Addr = config:get("chttpd", "bind_address", "127.0.0.1"), @@ -38,7 +35,7 @@ setup() -> teardown(Url) -> delete_db(Url), - ok = config:delete("admins", ?USER, _Persist=false), + ok = config:delete("admins", ?USER, _Persist = false), ok = config:delete("couchdb", "max_document_size"). create_db(Url) -> @@ -62,7 +59,8 @@ all_test_() -> fun chttpd_test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun post_single_doc/1, fun put_single_doc/1, @@ -76,52 +74,80 @@ all_test_() -> }. post_single_doc(Url) -> - NewDoc = "{\"post_single_doc\": \"some_doc\", - \"_id\": \"testdoc\", \"should_be\" : \"too_large\"}", - {ok, _, _, ResultBody} = test_request:post(Url, - [?CONTENT_JSON, ?AUTH], NewDoc), + NewDoc = + "{\"post_single_doc\": \"some_doc\",\n" + " \"_id\": \"testdoc\", \"should_be\" : \"too_large\"}", + {ok, _, _, ResultBody} = test_request:post( + Url, + [?CONTENT_JSON, ?AUTH], + NewDoc + ), {[ErrorMsg | _]} = ?JSON_DECODE(ResultBody), ?_assertEqual({<<"error">>, <<"document_too_large">>}, ErrorMsg). put_single_doc(Url) -> - NewDoc = "{\"post_single_doc\": \"some_doc\", - \"_id\": \"testdoc\", \"should_be\" : \"too_large\"}", - {ok, _, _, ResultBody} = test_request:put(Url ++ "/" ++ "testid", - [?CONTENT_JSON, ?AUTH], NewDoc), + NewDoc = + "{\"post_single_doc\": \"some_doc\",\n" + " \"_id\": \"testdoc\", \"should_be\" : \"too_large\"}", + {ok, _, _, ResultBody} = test_request:put( + Url ++ "/" ++ "testid", + [?CONTENT_JSON, ?AUTH], + NewDoc + ), {[ErrorMsg | _]} = ?JSON_DECODE(ResultBody), ?_assertEqual({<<"error">>, <<"document_too_large">>}, ErrorMsg). bulk_doc(Url) -> - NewDoc = "{\"docs\": [{\"doc1\": 1}, {\"errordoc\": - \"this_should_be_the_too_large_error_document\"}]}", - {ok, _, _, ResultBody} = test_request:post(Url ++ "/_bulk_docs/", - [?CONTENT_JSON, ?AUTH], NewDoc), + NewDoc = + "{\"docs\": [{\"doc1\": 1}, {\"errordoc\":\n" + " \"this_should_be_the_too_large_error_document\"}]}", + {ok, _, _, ResultBody} = test_request:post( + Url ++ "/_bulk_docs/", + [?CONTENT_JSON, ?AUTH], + NewDoc + ), ResultJson = ?JSON_DECODE(ResultBody), - Expect = {[{<<"error">>,<<"document_too_large">>},{<<"reason">>,<<>>}]}, + Expect = {[{<<"error">>, <<"document_too_large">>}, {<<"reason">>, <<>>}]}, ?_assertEqual(Expect, ResultJson). put_post_doc_attach_inline(Url) -> Body1 = "{\"body\":\"This is a body.\",", - Body2 = lists:concat(["{\"body\":\"This is a body it should fail", - "because there are too many characters.\","]), - DocRest = lists:concat(["\"_attachments\":{\"foo.txt\":{", + Body2 = lists:concat([ + "{\"body\":\"This is a body it should fail", + "because there are too many characters.\"," + ]), + DocRest = lists:concat([ + "\"_attachments\":{\"foo.txt\":{", "\"content_type\":\"text/plain\",", - "\"data\": \"VGhpcyBpcyBhIGJhc2U2NCBlbmNvZGVkIHRleHQ=\"}}}"]), + "\"data\": \"VGhpcyBpcyBhIGJhc2U2NCBlbmNvZGVkIHRleHQ=\"}}}" + ]), Doc1 = lists:concat([Body1, DocRest]), Doc2 = lists:concat([Body2, DocRest]), - {ok, _, _, ResultBody} = test_request:post(Url, - [?CONTENT_JSON, ?AUTH], Doc1), + {ok, _, _, ResultBody} = test_request:post( + Url, + [?CONTENT_JSON, ?AUTH], + Doc1 + ), {[Msg | _]} = ?JSON_DECODE(ResultBody), - {ok, _, _, ResultBody1} = test_request:post(Url, - [?CONTENT_JSON, ?AUTH], Doc2), + {ok, _, _, ResultBody1} = test_request:post( + Url, + [?CONTENT_JSON, ?AUTH], + Doc2 + ), {[Msg1 | _]} = ?JSON_DECODE(ResultBody1), - {ok, _, _, ResultBody2} = test_request:put(Url ++ "/" ++ "accept", - [?CONTENT_JSON, ?AUTH], Doc1), + {ok, _, _, ResultBody2} = test_request:put( + Url ++ "/" ++ "accept", + [?CONTENT_JSON, ?AUTH], + Doc1 + ), {[Msg2 | _]} = ?JSON_DECODE(ResultBody2), - {ok, _, _, ResultBody3} = test_request:put(Url ++ "/" ++ "fail", - [?CONTENT_JSON, ?AUTH], Doc2), + {ok, _, _, ResultBody3} = test_request:put( + Url ++ "/" ++ "fail", + [?CONTENT_JSON, ?AUTH], + Doc2 + ), {[Msg3 | _]} = ?JSON_DECODE(ResultBody3), [ ?_assertEqual({<<"ok">>, true}, Msg), @@ -132,21 +158,31 @@ put_post_doc_attach_inline(Url) -> put_multi_part_related(Url) -> Body1 = "{\"body\":\"This is a body.\",", - Body2 = lists:concat(["{\"body\":\"This is a body it should fail", - "because there are too many characters.\","]), + Body2 = lists:concat([ + "{\"body\":\"This is a body it should fail", + "because there are too many characters.\"," + ]), DocBeg = "--bound\r\nContent-Type: application/json\r\n\r\n", - DocRest = lists:concat(["\"_attachments\":{\"foo.txt\":{\"follows\":true,", + DocRest = lists:concat([ + "\"_attachments\":{\"foo.txt\":{\"follows\":true,", "\"content_type\":\"text/plain\",\"length\":21},\"bar.txt\":", "{\"follows\":true,\"content_type\":\"text/plain\",", "\"length\":20}}}\r\n--bound\r\n\r\nthis is 21 chars long", - "\r\n--bound\r\n\r\nthis is 20 chars lon\r\n--bound--epilogue"]), + "\r\n--bound\r\n\r\nthis is 20 chars lon\r\n--bound--epilogue" + ]), Doc1 = lists:concat([DocBeg, Body1, DocRest]), Doc2 = lists:concat([DocBeg, Body2, DocRest]), - {ok, _, _, ResultBody} = test_request:put(Url ++ "/" ++ "accept", - [?CONTENT_MULTI_RELATED, ?AUTH], Doc1), + {ok, _, _, ResultBody} = test_request:put( + Url ++ "/" ++ "accept", + [?CONTENT_MULTI_RELATED, ?AUTH], + Doc1 + ), {[Msg | _]} = ?JSON_DECODE(ResultBody), - {ok, _, _, ResultBody1} = test_request:put(Url ++ "/" ++ "faildoc", - [?CONTENT_MULTI_RELATED, ?AUTH], Doc2), + {ok, _, _, ResultBody1} = test_request:put( + Url ++ "/" ++ "faildoc", + [?CONTENT_MULTI_RELATED, ?AUTH], + Doc2 + ), {[Msg1 | _]} = ?JSON_DECODE(ResultBody1), [ ?_assertEqual({<<"ok">>, true}, Msg), @@ -155,23 +191,33 @@ put_multi_part_related(Url) -> post_multi_part_form(Url) -> Port = mochiweb_socket_server:get(chttpd, port), - Host = lists:concat([ "http://127.0.0.1:", Port]), + Host = lists:concat(["http://127.0.0.1:", Port]), Referer = {"Referer", Host}, Body1 = "{\"body\":\"This is a body.\"}", - Body2 = lists:concat(["{\"body\":\"This is a body it should fail", - "because there are too many characters.\"}"]), + Body2 = lists:concat([ + "{\"body\":\"This is a body it should fail", + "because there are too many characters.\"}" + ]), DocBeg = "--bound\r\nContent-Disposition: form-data; name=\"_doc\"\r\n\r\n", - DocRest = lists:concat(["\r\n--bound\r\nContent-Disposition:", + DocRest = lists:concat([ + "\r\n--bound\r\nContent-Disposition:", "form-data; name=\"_attachments\"; filename=\"file.txt\"\r\n", "Content-Type: text/plain\r\n\r\ncontents of file.txt\r\n\r\n", - "--bound--"]), + "--bound--" + ]), Doc1 = lists:concat([DocBeg, Body1, DocRest]), Doc2 = lists:concat([DocBeg, Body2, DocRest]), - {ok, _, _, ResultBody} = test_request:post(Url ++ "/" ++ "accept", - [?CONTENT_MULTI_FORM, ?AUTH, Referer], Doc1), + {ok, _, _, ResultBody} = test_request:post( + Url ++ "/" ++ "accept", + [?CONTENT_MULTI_FORM, ?AUTH, Referer], + Doc1 + ), {[Msg | _]} = ?JSON_DECODE(ResultBody), - {ok, _, _, ResultBody1} = test_request:post(Url ++ "/" ++ "fail", - [?CONTENT_MULTI_FORM, ?AUTH, Referer], Doc2), + {ok, _, _, ResultBody1} = test_request:post( + Url ++ "/" ++ "fail", + [?CONTENT_MULTI_FORM, ?AUTH, Referer], + Doc2 + ), {[Msg1 | _]} = ?JSON_DECODE(ResultBody1), [ ?_assertEqual({<<"ok">>, true}, Msg), diff --git a/src/chttpd/test/eunit/chttpd_db_test.erl b/src/chttpd/test/eunit/chttpd_db_test.erl index 15e75393c..07661733f 100644 --- a/src/chttpd/test/eunit/chttpd_db_test.erl +++ b/src/chttpd/test/eunit/chttpd_db_test.erl @@ -23,11 +23,12 @@ -define(DESTHEADER2, {"Destination", "foo%2Fbar%23baz%3Fpow%3Afiz"}). -define(FIXTURE_TXT, ?ABS_PATH(?FILE)). -define(i2l(I), integer_to_list(I)). --define(TIMEOUT, 60). % seconds +% seconds +-define(TIMEOUT, 60). setup() -> Hashed = couch_passwords:hash_admin_password(?PASS), - ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist=false), + ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist = false), TmpDb = ?tempdb(), Addr = config:get("chttpd", "bind_address", "127.0.0.1"), Port = mochiweb_socket_server:get(chttpd, port), @@ -37,16 +38,18 @@ setup() -> teardown(Url) -> delete_db(Url), - ok = config:delete("admins", ?USER, _Persist=false). + ok = config:delete("admins", ?USER, _Persist = false). create_db(Url) -> {ok, Status, _, _} = test_request:put(Url, [?CONTENT_JSON, ?AUTH], "{}"), ?assert(Status =:= 201 orelse Status =:= 202). - create_doc(Url, Id) -> - test_request:put(Url ++ "/" ++ Id, - [?CONTENT_JSON, ?AUTH], "{\"mr\": \"rockoartischocko\"}"). + test_request:put( + Url ++ "/" ++ Id, + [?CONTENT_JSON, ?AUTH], + "{\"mr\": \"rockoartischocko\"}" + ). delete_db(Url) -> {ok, 200, _, _} = test_request:delete(Url, [?AUTH]). @@ -56,10 +59,12 @@ all_test_() -> "chttpd db tests", { setup, - fun chttpd_test_util:start_couch/0, fun chttpd_test_util:stop_couch/1, + fun chttpd_test_util:start_couch/0, + fun chttpd_test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_return_ok_true_on_bulk_update/1, fun should_return_201_new_edits_false_with_revs_on_bulk_update/1, @@ -92,424 +97,511 @@ all_test_() -> } }. - should_return_ok_true_on_bulk_update(Url) -> - {timeout, ?TIMEOUT, ?_assertEqual(true, - begin - {ok, _, _, Body} = create_doc(Url, "testdoc"), - {Json} = ?JSON_DECODE(Body), - Ref = couch_util:get_value(<<"rev">>, Json, undefined), - NewDoc = "{\"docs\": [{\"_rev\": \"" ++ ?b2l(Ref) ++ "\", \"_id\": \"testdoc\"}]}", - {ok, _, _, ResultBody} = test_request:post(Url ++ "/_bulk_docs/", - [?CONTENT_JSON, ?AUTH], NewDoc), - ResultJson = ?JSON_DECODE(ResultBody), - {InnerJson} = lists:nth(1, ResultJson), - couch_util:get_value(<<"ok">>, InnerJson, undefined) - end)}. - + {timeout, ?TIMEOUT, + ?_assertEqual( + true, + begin + {ok, _, _, Body} = create_doc(Url, "testdoc"), + {Json} = ?JSON_DECODE(Body), + Ref = couch_util:get_value(<<"rev">>, Json, undefined), + NewDoc = "{\"docs\": [{\"_rev\": \"" ++ ?b2l(Ref) ++ "\", \"_id\": \"testdoc\"}]}", + {ok, _, _, ResultBody} = test_request:post( + Url ++ "/_bulk_docs/", + [?CONTENT_JSON, ?AUTH], + NewDoc + ), + ResultJson = ?JSON_DECODE(ResultBody), + {InnerJson} = lists:nth(1, ResultJson), + couch_util:get_value(<<"ok">>, InnerJson, undefined) + end + )}. should_return_201_new_edits_false_with_revs_on_bulk_update(Url) -> - {timeout, ?TIMEOUT, ?_test( - begin - {ok, _, _, Body} = create_doc(Url, "dochasrev"), - {Json} = ?JSON_DECODE(Body), - Ref = couch_util:get_value(<<"rev">>, Json, undefined), - NewDoc = "{\"docs\": [{\"_rev\": \"" ++ ?b2l(Ref) ++ - "\", \"_id\": \"dochasrev\"}], \"new_edits\": false}", - {ok, Status, _, ResultBody} = test_request:post(Url ++ - "/_bulk_docs/", [?CONTENT_JSON, ?AUTH], NewDoc), - ?assertEqual(201, Status), - ?assertEqual([], ?JSON_DECODE(ResultBody)) - end)}. - + {timeout, ?TIMEOUT, + ?_test( + begin + {ok, _, _, Body} = create_doc(Url, "dochasrev"), + {Json} = ?JSON_DECODE(Body), + Ref = couch_util:get_value(<<"rev">>, Json, undefined), + NewDoc = + "{\"docs\": [{\"_rev\": \"" ++ ?b2l(Ref) ++ + "\", \"_id\": \"dochasrev\"}], \"new_edits\": false}", + {ok, Status, _, ResultBody} = test_request:post( + Url ++ + "/_bulk_docs/", + [?CONTENT_JSON, ?AUTH], + NewDoc + ), + ?assertEqual(201, Status), + ?assertEqual([], ?JSON_DECODE(ResultBody)) + end + )}. should_return_400_new_edits_false_no_revs_on_bulk_update(Url) -> - {timeout, ?TIMEOUT, ?_test( - begin - {ok, _, _, _} = create_doc(Url, "docnorev"), - NewDoc = "{\"docs\": [{\"_id\": \"docnorev\"}], " - ++ "\"new_edits\": false}", - {ok, Status, _, ResultBody} = test_request:post(Url ++ - "/_bulk_docs/", [?CONTENT_JSON, ?AUTH], NewDoc), - {ResultJson} = ?JSON_DECODE(ResultBody), - ?assertEqual(400, Status), - ?assertEqual(<<"bad_request">>, - couch_util:get_value(<<"error">>, ResultJson)) - end)}. - + {timeout, ?TIMEOUT, + ?_test( + begin + {ok, _, _, _} = create_doc(Url, "docnorev"), + NewDoc = + "{\"docs\": [{\"_id\": \"docnorev\"}], " ++ + "\"new_edits\": false}", + {ok, Status, _, ResultBody} = test_request:post( + Url ++ + "/_bulk_docs/", + [?CONTENT_JSON, ?AUTH], + NewDoc + ), + {ResultJson} = ?JSON_DECODE(ResultBody), + ?assertEqual(400, Status), + ?assertEqual( + <<"bad_request">>, + couch_util:get_value(<<"error">>, ResultJson) + ) + end + )}. should_return_ok_true_on_ensure_full_commit(Url0) -> - {timeout, ?TIMEOUT, ?_test(begin - Url = Url0 ++ "/_ensure_full_commit", - {ok, RC, _, Body} = test_request:post(Url, [?CONTENT_JSON, ?AUTH], []), - {Json} = ?JSON_DECODE(Body), - ?assertEqual(201, RC), - ?assert(couch_util:get_value(<<"ok">>, Json)) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + Url = Url0 ++ "/_ensure_full_commit", + {ok, RC, _, Body} = test_request:post(Url, [?CONTENT_JSON, ?AUTH], []), + {Json} = ?JSON_DECODE(Body), + ?assertEqual(201, RC), + ?assert(couch_util:get_value(<<"ok">>, Json)) + end)}. should_return_404_for_ensure_full_commit_on_no_db(Url0) -> - {timeout, ?TIMEOUT, ?_test(begin - Url = Url0 ++ "-missing-db" ++ "/_ensure_full_commit", - {ok, RC, _, Body} = test_request:post(Url, [?CONTENT_JSON, ?AUTH], []), - {Json} = ?JSON_DECODE(Body), - ?assertEqual(404, RC), - ?assertEqual(<<"not_found">>, couch_util:get_value(<<"error">>, Json)) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + Url = Url0 ++ "-missing-db" ++ "/_ensure_full_commit", + {ok, RC, _, Body} = test_request:post(Url, [?CONTENT_JSON, ?AUTH], []), + {Json} = ?JSON_DECODE(Body), + ?assertEqual(404, RC), + ?assertEqual(<<"not_found">>, couch_util:get_value(<<"error">>, Json)) + end)}. should_accept_live_as_an_alias_for_continuous(Url) -> GetLastSeq = fun(Chunks) -> LastSeqBin = lists:last(Chunks), - {Result} = try ?JSON_DECODE(LastSeqBin) of - Data -> Data - catch - _:_ -> - ?assert(false) % should not happen, abort - end, + {Result} = + try ?JSON_DECODE(LastSeqBin) of + Data -> Data + catch + _:_ -> + % should not happen, abort + ?assert(false) + end, couch_util:get_value(<<"last_seq">>, Result, undefined) end, - {timeout, ?TIMEOUT, ?_test(begin - LastSeq1 = GetLastSeq(wait_non_empty_chunk(Url)), + {timeout, ?TIMEOUT, + ?_test(begin + LastSeq1 = GetLastSeq(wait_non_empty_chunk(Url)), - {ok, _, _, _} = create_doc(Url, "testdoc2"), + {ok, _, _, _} = create_doc(Url, "testdoc2"), - LastSeq2 = GetLastSeq(wait_non_empty_chunk(Url)), - - ?assertNotEqual(LastSeq1, LastSeq2) - end)}. + LastSeq2 = GetLastSeq(wait_non_empty_chunk(Url)), + ?assertNotEqual(LastSeq1, LastSeq2) + end)}. should_return_404_for_delete_att_on_notadoc(Url) -> - {timeout, ?TIMEOUT, ?_test(begin - {ok, RC, _, RespBody} = test_request:delete( - Url ++ "/notadoc/att.pdf", - [?CONTENT_JSON, ?AUTH], - [] - ), - ?assertEqual(404, RC), - ?assertEqual( - {[{<<"error">>,<<"not_found">>}, - {<<"reason">>,<<"missing">>}]}, - jiffy:decode(RespBody) - ), - {ok, RC1, _, _} = test_request:get( - Url ++ "/notadoc", - [?CONTENT_JSON, ?AUTH], - [] - ), - ?assertEqual(404, RC1) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + {ok, RC, _, RespBody} = test_request:delete( + Url ++ "/notadoc/att.pdf", + [?CONTENT_JSON, ?AUTH], + [] + ), + ?assertEqual(404, RC), + ?assertEqual( + {[ + {<<"error">>, <<"not_found">>}, + {<<"reason">>, <<"missing">>} + ]}, + jiffy:decode(RespBody) + ), + {ok, RC1, _, _} = test_request:get( + Url ++ "/notadoc", + [?CONTENT_JSON, ?AUTH], + [] + ), + ?assertEqual(404, RC1) + end)}. should_return_409_for_del_att_without_rev(Url) -> - {timeout, ?TIMEOUT, ?_test(begin - {ok, RC, _, _} = test_request:put( - Url ++ "/testdoc3", - [?CONTENT_JSON, ?AUTH], - jiffy:encode(attachment_doc()) - ), - ?assertEqual(201, RC), - - {ok, RC1, _, _} = test_request:delete( - Url ++ "/testdoc3/file.erl", - [?CONTENT_JSON, ?AUTH], - [] - ), - ?assertEqual(409, RC1) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + {ok, RC, _, _} = test_request:put( + Url ++ "/testdoc3", + [?CONTENT_JSON, ?AUTH], + jiffy:encode(attachment_doc()) + ), + ?assertEqual(201, RC), + + {ok, RC1, _, _} = test_request:delete( + Url ++ "/testdoc3/file.erl", + [?CONTENT_JSON, ?AUTH], + [] + ), + ?assertEqual(409, RC1) + end)}. should_return_200_for_del_att_with_rev(Url) -> - {timeout, ?TIMEOUT, ?_test(begin - {ok, RC, _Headers, RespBody} = test_request:put( - Url ++ "/testdoc4", - [?CONTENT_JSON, ?AUTH], - jiffy:encode(attachment_doc()) - ), - ?assertEqual(201, RC), - - {ResultJson} = ?JSON_DECODE(RespBody), - Rev = couch_util:get_value(<<"rev">>, ResultJson, undefined), - - {ok, RC1, _, _} = test_request:delete( - Url ++ "/testdoc4/file.erl?rev=" ++ Rev, - [?CONTENT_JSON, ?AUTH], - [] - ), - ?assertEqual(200, RC1) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + {ok, RC, _Headers, RespBody} = test_request:put( + Url ++ "/testdoc4", + [?CONTENT_JSON, ?AUTH], + jiffy:encode(attachment_doc()) + ), + ?assertEqual(201, RC), + + {ResultJson} = ?JSON_DECODE(RespBody), + Rev = couch_util:get_value(<<"rev">>, ResultJson, undefined), + + {ok, RC1, _, _} = test_request:delete( + Url ++ "/testdoc4/file.erl?rev=" ++ Rev, + [?CONTENT_JSON, ?AUTH], + [] + ), + ?assertEqual(200, RC1) + end)}. should_return_409_for_put_att_nonexistent_rev(Url) -> - {timeout, ?TIMEOUT, ?_test(begin - {ok, RC, _Headers, RespBody} = test_request:put( - Url ++ "/should_return_404/file.erl?rev=1-000", - [?CONTENT_JSON, ?AUTH], - jiffy:encode(attachment_doc()) - ), - ?assertEqual(409, RC), - ?assertMatch({[ - {<<"error">>,<<"not_found">>}, - {<<"reason">>,<<"missing_rev">>}]}, - ?JSON_DECODE(RespBody)) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + {ok, RC, _Headers, RespBody} = test_request:put( + Url ++ "/should_return_404/file.erl?rev=1-000", + [?CONTENT_JSON, ?AUTH], + jiffy:encode(attachment_doc()) + ), + ?assertEqual(409, RC), + ?assertMatch( + {[ + {<<"error">>, <<"not_found">>}, + {<<"reason">>, <<"missing_rev">>} + ]}, + ?JSON_DECODE(RespBody) + ) + end)}. should_return_update_seq_when_set_on_all_docs(Url) -> - {timeout, ?TIMEOUT, ?_test(begin - [create_doc(Url, "testdoc" ++ ?i2l(I)) || I <- lists:seq(1, 3)], - {ok, RC, _, RespBody} = test_request:get(Url ++ "/_all_docs/" - ++ "?update_seq=true&keys=[\"testdoc1\"]",[?CONTENT_JSON, ?AUTH]), - ?assertEqual(200, RC), - {ResultJson} = ?JSON_DECODE(RespBody), - ?assertNotEqual(undefined, - couch_util:get_value(<<"update_seq">>, ResultJson)), - ?assertNotEqual(undefined, - couch_util:get_value(<<"offset">>, ResultJson)) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + [create_doc(Url, "testdoc" ++ ?i2l(I)) || I <- lists:seq(1, 3)], + {ok, RC, _, RespBody} = test_request:get( + Url ++ "/_all_docs/" ++ + "?update_seq=true&keys=[\"testdoc1\"]", + [?CONTENT_JSON, ?AUTH] + ), + ?assertEqual(200, RC), + {ResultJson} = ?JSON_DECODE(RespBody), + ?assertNotEqual( + undefined, + couch_util:get_value(<<"update_seq">>, ResultJson) + ), + ?assertNotEqual( + undefined, + couch_util:get_value(<<"offset">>, ResultJson) + ) + end)}. should_not_return_update_seq_when_unset_on_all_docs(Url) -> - {timeout, ?TIMEOUT, ?_test(begin - [create_doc(Url, "testdoc" ++ ?i2l(I)) || I <- lists:seq(1, 3)], - {ok, RC, _, RespBody} = test_request:get(Url ++ "/_all_docs/" - ++ "?update_seq=false&keys=[\"testdoc1\"]",[?CONTENT_JSON, ?AUTH]), - ?assertEqual(200, RC), - {ResultJson} = ?JSON_DECODE(RespBody), - ?assertEqual(undefined, - couch_util:get_value(<<"update_seq">>, ResultJson)), - ?assertNotEqual(undefined, - couch_util:get_value(<<"offset">>, ResultJson)) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + [create_doc(Url, "testdoc" ++ ?i2l(I)) || I <- lists:seq(1, 3)], + {ok, RC, _, RespBody} = test_request:get( + Url ++ "/_all_docs/" ++ + "?update_seq=false&keys=[\"testdoc1\"]", + [?CONTENT_JSON, ?AUTH] + ), + ?assertEqual(200, RC), + {ResultJson} = ?JSON_DECODE(RespBody), + ?assertEqual( + undefined, + couch_util:get_value(<<"update_seq">>, ResultJson) + ), + ?assertNotEqual( + undefined, + couch_util:get_value(<<"offset">>, ResultJson) + ) + end)}. should_return_correct_id_on_doc_copy(Url) -> - {timeout, ?TIMEOUT, ?_test(begin - {ok, _, _, _} = create_doc(Url, "testdoc"), - {_, _, _, ResultBody1} = test_request:copy(Url ++ "/testdoc/", - [?CONTENT_JSON, ?AUTH, ?DESTHEADER1]), - {ResultJson1} = ?JSON_DECODE(ResultBody1), - Id1 = couch_util:get_value(<<"id">>, ResultJson1), - - {_, _, _, ResultBody2} = test_request:copy(Url ++ "/testdoc/", - [?CONTENT_JSON, ?AUTH, ?DESTHEADER2]), - {ResultJson2} = ?JSON_DECODE(ResultBody2), - Id2 = couch_util:get_value(<<"id">>, ResultJson2), - [ - ?assertEqual(<<102,111,111,229,149,138,98,97,114>>, Id1), - ?assertEqual(<<"foo/bar#baz?pow:fiz">>, Id2) - ] - end)}. + {timeout, ?TIMEOUT, + ?_test(begin + {ok, _, _, _} = create_doc(Url, "testdoc"), + {_, _, _, ResultBody1} = test_request:copy( + Url ++ "/testdoc/", + [?CONTENT_JSON, ?AUTH, ?DESTHEADER1] + ), + {ResultJson1} = ?JSON_DECODE(ResultBody1), + Id1 = couch_util:get_value(<<"id">>, ResultJson1), + + {_, _, _, ResultBody2} = test_request:copy( + Url ++ "/testdoc/", + [?CONTENT_JSON, ?AUTH, ?DESTHEADER2] + ), + {ResultJson2} = ?JSON_DECODE(ResultBody2), + Id2 = couch_util:get_value(<<"id">>, ResultJson2), + [ + ?assertEqual(<<102, 111, 111, 229, 149, 138, 98, 97, 114>>, Id1), + ?assertEqual(<<"foo/bar#baz?pow:fiz">>, Id2) + ] + end)}. should_return_only_one_ok_on_doc_copy(Url) -> - {timeout, ?TIMEOUT, ?_test(begin - {ok, _, _, _} = create_doc(Url, "testdoc"), - {_, _, _, ResultBody} = test_request:copy(Url ++ "/testdoc", - [?CONTENT_JSON, ?AUTH, ?DESTHEADER1]), - {ResultJson} = jiffy:decode(ResultBody), - NumOks = length(lists:filter(fun({Key, _Value}) -> Key == <<"ok">> end, ResultJson)), - [ - ?assertEqual(1, NumOks) - ] - end)}. + {timeout, ?TIMEOUT, + ?_test(begin + {ok, _, _, _} = create_doc(Url, "testdoc"), + {_, _, _, ResultBody} = test_request:copy( + Url ++ "/testdoc", + [?CONTENT_JSON, ?AUTH, ?DESTHEADER1] + ), + {ResultJson} = jiffy:decode(ResultBody), + NumOks = length(lists:filter(fun({Key, _Value}) -> Key == <<"ok">> end, ResultJson)), + [ + ?assertEqual(1, NumOks) + ] + end)}. attachment_doc() -> {ok, Data} = file:read_file(?FIXTURE_TXT), {[ - {<<"_attachments">>, {[ - {<<"file.erl">>, {[ - {<<"content_type">>, <<"text/plain">>}, - {<<"data">>, base64:encode(Data)} - ]} - }]}} + {<<"_attachments">>, + {[ + {<<"file.erl">>, + {[ + {<<"content_type">>, <<"text/plain">>}, + {<<"data">>, base64:encode(Data)} + ]}} + ]}} ]}. - should_return_400_for_bad_engine(_) -> - {timeout, ?TIMEOUT, ?_test(begin - TmpDb = ?tempdb(), - Addr = config:get("chttpd", "bind_address", "127.0.0.1"), - Port = mochiweb_socket_server:get(chttpd, port), - BaseUrl = lists:concat(["http://", Addr, ":", Port, "/", ?b2l(TmpDb)]), - Url = BaseUrl ++ "?engine=cowabunga", - {ok, Status, _, _} = test_request:put(Url, [?CONTENT_JSON, ?AUTH], "{}"), - ?assertEqual(400, Status) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + TmpDb = ?tempdb(), + Addr = config:get("chttpd", "bind_address", "127.0.0.1"), + Port = mochiweb_socket_server:get(chttpd, port), + BaseUrl = lists:concat(["http://", Addr, ":", Port, "/", ?b2l(TmpDb)]), + Url = BaseUrl ++ "?engine=cowabunga", + {ok, Status, _, _} = test_request:put(Url, [?CONTENT_JSON, ?AUTH], "{}"), + ?assertEqual(400, Status) + end)}. should_not_change_db_proper_after_rewriting_shardmap(_) -> - {timeout, ?TIMEOUT, ?_test(begin - TmpDb = ?tempdb(), - Addr = config:get("chttpd", "bind_address", "127.0.0.1"), - Port = mochiweb_socket_server:get(chttpd, port), - - BaseUrl = lists:concat(["http://", Addr, ":", Port, "/", ?b2l(TmpDb)]), - Url = BaseUrl ++ "?partitioned=true&q=1", - {ok, 201, _, _} = test_request:put(Url, [?CONTENT_JSON, ?AUTH], "{}"), - - ShardDbName = ?l2b(config:get("mem3", "shards_db", "_dbs")), - {ok, ShardDb} = mem3_util:ensure_exists(ShardDbName), - {ok, #doc{body = {Props}}} = couch_db:open_doc( - ShardDb, TmpDb, [ejson_body]), - Shards = mem3_util:build_shards(TmpDb, Props), - - {Prop2} = ?JSON_DECODE(?JSON_ENCODE({Props})), - Shards2 = mem3_util:build_shards(TmpDb, Prop2), - ?assertEqual(Shards2, Shards) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + TmpDb = ?tempdb(), + Addr = config:get("chttpd", "bind_address", "127.0.0.1"), + Port = mochiweb_socket_server:get(chttpd, port), + + BaseUrl = lists:concat(["http://", Addr, ":", Port, "/", ?b2l(TmpDb)]), + Url = BaseUrl ++ "?partitioned=true&q=1", + {ok, 201, _, _} = test_request:put(Url, [?CONTENT_JSON, ?AUTH], "{}"), + + ShardDbName = ?l2b(config:get("mem3", "shards_db", "_dbs")), + {ok, ShardDb} = mem3_util:ensure_exists(ShardDbName), + {ok, #doc{body = {Props}}} = couch_db:open_doc( + ShardDb, TmpDb, [ejson_body] + ), + Shards = mem3_util:build_shards(TmpDb, Props), + + {Prop2} = ?JSON_DECODE(?JSON_ENCODE({Props})), + Shards2 = mem3_util:build_shards(TmpDb, Prop2), + ?assertEqual(Shards2, Shards) + end)}. should_succeed_on_all_docs_with_queries_keys(Url) -> - {timeout, ?TIMEOUT, ?_test(begin - [create_doc(Url, "testdoc" ++ ?i2l(I)) || I <- lists:seq(1, 10)], - QueryDoc = "{\"queries\": [{\"keys\": [ \"testdoc3\", \"testdoc8\"]}]}", - {ok, RC, _, RespBody} = test_request:post(Url ++ "/_all_docs/queries/", - [?CONTENT_JSON, ?AUTH], QueryDoc), - ?assertEqual(200, RC), - {ResultJson} = ?JSON_DECODE(RespBody), - ResultJsonBody = couch_util:get_value(<<"results">>, ResultJson), - {InnerJson} = lists:nth(1, ResultJsonBody), - ?assertEqual(2, length(couch_util:get_value(<<"rows">>, InnerJson))) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + [create_doc(Url, "testdoc" ++ ?i2l(I)) || I <- lists:seq(1, 10)], + QueryDoc = "{\"queries\": [{\"keys\": [ \"testdoc3\", \"testdoc8\"]}]}", + {ok, RC, _, RespBody} = test_request:post( + Url ++ "/_all_docs/queries/", + [?CONTENT_JSON, ?AUTH], + QueryDoc + ), + ?assertEqual(200, RC), + {ResultJson} = ?JSON_DECODE(RespBody), + ResultJsonBody = couch_util:get_value(<<"results">>, ResultJson), + {InnerJson} = lists:nth(1, ResultJsonBody), + ?assertEqual(2, length(couch_util:get_value(<<"rows">>, InnerJson))) + end)}. should_succeed_on_all_docs_with_queries_limit_skip(Url) -> - {timeout, ?TIMEOUT, ?_test(begin - [create_doc(Url, "testdoc" ++ ?i2l(I)) || I <- lists:seq(1, 10)], - QueryDoc = "{\"queries\": [{\"limit\": 5, \"skip\": 2}]}", - {ok, RC, _, RespBody} = test_request:post(Url ++ "/_all_docs/queries/", - [?CONTENT_JSON, ?AUTH], QueryDoc), - ?assertEqual(200, RC), - {ResultJson} = ?JSON_DECODE(RespBody), - ResultJsonBody = couch_util:get_value(<<"results">>, ResultJson), - {InnerJson} = lists:nth(1, ResultJsonBody), - ?assertEqual(2, couch_util:get_value(<<"offset">>, InnerJson)), - ?assertEqual(5, length(couch_util:get_value(<<"rows">>, InnerJson))) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + [create_doc(Url, "testdoc" ++ ?i2l(I)) || I <- lists:seq(1, 10)], + QueryDoc = "{\"queries\": [{\"limit\": 5, \"skip\": 2}]}", + {ok, RC, _, RespBody} = test_request:post( + Url ++ "/_all_docs/queries/", + [?CONTENT_JSON, ?AUTH], + QueryDoc + ), + ?assertEqual(200, RC), + {ResultJson} = ?JSON_DECODE(RespBody), + ResultJsonBody = couch_util:get_value(<<"results">>, ResultJson), + {InnerJson} = lists:nth(1, ResultJsonBody), + ?assertEqual(2, couch_util:get_value(<<"offset">>, InnerJson)), + ?assertEqual(5, length(couch_util:get_value(<<"rows">>, InnerJson))) + end)}. should_succeed_on_all_docs_with_multiple_queries(Url) -> - {timeout, ?TIMEOUT, ?_test(begin - [create_doc(Url, "testdoc" ++ ?i2l(I)) || I <- lists:seq(1, 10)], - QueryDoc = "{\"queries\": [{\"keys\": [ \"testdoc3\", \"testdoc8\"]}, - {\"limit\": 5, \"skip\": 2}]}", - {ok, RC, _, RespBody} = test_request:post(Url ++ "/_all_docs/queries/", - [?CONTENT_JSON, ?AUTH], QueryDoc), - ?assertEqual(200, RC), - {ResultJson} = ?JSON_DECODE(RespBody), - ResultJsonBody = couch_util:get_value(<<"results">>, ResultJson), - {InnerJson1} = lists:nth(1, ResultJsonBody), - ?assertEqual(2, length(couch_util:get_value(<<"rows">>, InnerJson1))), - {InnerJson2} = lists:nth(2, ResultJsonBody), - ?assertEqual(2, couch_util:get_value(<<"offset">>, InnerJson2)), - ?assertEqual(5, length(couch_util:get_value(<<"rows">>, InnerJson2))) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + [create_doc(Url, "testdoc" ++ ?i2l(I)) || I <- lists:seq(1, 10)], + QueryDoc = + "{\"queries\": [{\"keys\": [ \"testdoc3\", \"testdoc8\"]},\n" + " {\"limit\": 5, \"skip\": 2}]}", + {ok, RC, _, RespBody} = test_request:post( + Url ++ "/_all_docs/queries/", + [?CONTENT_JSON, ?AUTH], + QueryDoc + ), + ?assertEqual(200, RC), + {ResultJson} = ?JSON_DECODE(RespBody), + ResultJsonBody = couch_util:get_value(<<"results">>, ResultJson), + {InnerJson1} = lists:nth(1, ResultJsonBody), + ?assertEqual(2, length(couch_util:get_value(<<"rows">>, InnerJson1))), + {InnerJson2} = lists:nth(2, ResultJsonBody), + ?assertEqual(2, couch_util:get_value(<<"offset">>, InnerJson2)), + ?assertEqual(5, length(couch_util:get_value(<<"rows">>, InnerJson2))) + end)}. should_succeed_on_design_docs_with_queries_keys(Url) -> - {timeout, ?TIMEOUT, ?_test(begin - [create_doc(Url, "_design/ddoc" ++ ?i2l(I)) || I <- lists:seq(1, 10)], - QueryDoc = "{\"queries\": [{\"keys\": [ \"_design/ddoc3\", - \"_design/ddoc8\"]}]}", - {ok, RC, _, RespBody} = test_request:post(Url ++ - "/_design_docs/queries/", [?CONTENT_JSON, ?AUTH], QueryDoc), - ?assertEqual(200, RC), - {ResultJson} = ?JSON_DECODE(RespBody), - ResultJsonBody = couch_util:get_value(<<"results">>, ResultJson), - {InnerJson} = lists:nth(1, ResultJsonBody), - ?assertEqual(2, length(couch_util:get_value(<<"rows">>, InnerJson))) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + [create_doc(Url, "_design/ddoc" ++ ?i2l(I)) || I <- lists:seq(1, 10)], + QueryDoc = + "{\"queries\": [{\"keys\": [ \"_design/ddoc3\",\n" + " \"_design/ddoc8\"]}]}", + {ok, RC, _, RespBody} = test_request:post( + Url ++ + "/_design_docs/queries/", + [?CONTENT_JSON, ?AUTH], + QueryDoc + ), + ?assertEqual(200, RC), + {ResultJson} = ?JSON_DECODE(RespBody), + ResultJsonBody = couch_util:get_value(<<"results">>, ResultJson), + {InnerJson} = lists:nth(1, ResultJsonBody), + ?assertEqual(2, length(couch_util:get_value(<<"rows">>, InnerJson))) + end)}. should_succeed_on_design_docs_with_queries_limit_skip(Url) -> - {timeout, ?TIMEOUT, ?_test(begin - [create_doc(Url, "_design/ddoc" ++ ?i2l(I)) || I <- lists:seq(1, 10)], - QueryDoc = "{\"queries\": [{\"limit\": 5, \"skip\": 2}]}", - {ok, RC, _, RespBody} = test_request:post(Url ++ - "/_design_docs/queries/", [?CONTENT_JSON, ?AUTH], QueryDoc), - ?assertEqual(200, RC), - {ResultJson} = ?JSON_DECODE(RespBody), - ResultJsonBody = couch_util:get_value(<<"results">>, ResultJson), - {InnerJson} = lists:nth(1, ResultJsonBody), - ?assertEqual(2, couch_util:get_value(<<"offset">>, InnerJson)), - ?assertEqual(5, length(couch_util:get_value(<<"rows">>, InnerJson))) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + [create_doc(Url, "_design/ddoc" ++ ?i2l(I)) || I <- lists:seq(1, 10)], + QueryDoc = "{\"queries\": [{\"limit\": 5, \"skip\": 2}]}", + {ok, RC, _, RespBody} = test_request:post( + Url ++ + "/_design_docs/queries/", + [?CONTENT_JSON, ?AUTH], + QueryDoc + ), + ?assertEqual(200, RC), + {ResultJson} = ?JSON_DECODE(RespBody), + ResultJsonBody = couch_util:get_value(<<"results">>, ResultJson), + {InnerJson} = lists:nth(1, ResultJsonBody), + ?assertEqual(2, couch_util:get_value(<<"offset">>, InnerJson)), + ?assertEqual(5, length(couch_util:get_value(<<"rows">>, InnerJson))) + end)}. should_succeed_on_design_docs_with_multiple_queries(Url) -> - {timeout, ?TIMEOUT, ?_test(begin - [create_doc(Url, "_design/ddoc" ++ ?i2l(I)) || I <- lists:seq(1, 10)], - QueryDoc = "{\"queries\": [{\"keys\": [ \"_design/ddoc3\", - \"_design/ddoc8\"]}, {\"limit\": 5, \"skip\": 2}]}", - {ok, RC, _, RespBody} = test_request:post(Url ++ - "/_design_docs/queries/", [?CONTENT_JSON, ?AUTH], QueryDoc), - ?assertEqual(200, RC), - {ResultJson} = ?JSON_DECODE(RespBody), - ResultJsonBody = couch_util:get_value(<<"results">>, ResultJson), - {InnerJson1} = lists:nth(1, ResultJsonBody), - ?assertEqual(2, length(couch_util:get_value(<<"rows">>, InnerJson1))), - {InnerJson2} = lists:nth(2, ResultJsonBody), - ?assertEqual(2, couch_util:get_value(<<"offset">>, InnerJson2)), - ?assertEqual(5, length(couch_util:get_value(<<"rows">>, InnerJson2))) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + [create_doc(Url, "_design/ddoc" ++ ?i2l(I)) || I <- lists:seq(1, 10)], + QueryDoc = + "{\"queries\": [{\"keys\": [ \"_design/ddoc3\",\n" + " \"_design/ddoc8\"]}, {\"limit\": 5, \"skip\": 2}]}", + {ok, RC, _, RespBody} = test_request:post( + Url ++ + "/_design_docs/queries/", + [?CONTENT_JSON, ?AUTH], + QueryDoc + ), + ?assertEqual(200, RC), + {ResultJson} = ?JSON_DECODE(RespBody), + ResultJsonBody = couch_util:get_value(<<"results">>, ResultJson), + {InnerJson1} = lists:nth(1, ResultJsonBody), + ?assertEqual(2, length(couch_util:get_value(<<"rows">>, InnerJson1))), + {InnerJson2} = lists:nth(2, ResultJsonBody), + ?assertEqual(2, couch_util:get_value(<<"offset">>, InnerJson2)), + ?assertEqual(5, length(couch_util:get_value(<<"rows">>, InnerJson2))) + end)}. should_succeed_on_local_docs_with_queries_keys(Url) -> - {timeout, ?TIMEOUT, ?_test(begin - [create_doc(Url, "_local/doc" ++ ?i2l(I)) || I <- lists:seq(1, 10)], - QueryDoc = "{\"queries\": [{\"keys\": - [ \"_local/doc3\", \"_local/doc8\"]}]}", - {ok, RC, _, RespBody} = test_request:post(Url ++ "/_local_docs/queries/", - [?CONTENT_JSON, ?AUTH], QueryDoc), - ?assertEqual(200, RC), - {ResultJson} = ?JSON_DECODE(RespBody), - ResultJsonBody = couch_util:get_value(<<"results">>, ResultJson), - {InnerJson} = lists:nth(1, ResultJsonBody), - ?assertEqual(2, length(couch_util:get_value(<<"rows">>, InnerJson))) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + [create_doc(Url, "_local/doc" ++ ?i2l(I)) || I <- lists:seq(1, 10)], + QueryDoc = + "{\"queries\": [{\"keys\":\n" + " [ \"_local/doc3\", \"_local/doc8\"]}]}", + {ok, RC, _, RespBody} = test_request:post( + Url ++ "/_local_docs/queries/", + [?CONTENT_JSON, ?AUTH], + QueryDoc + ), + ?assertEqual(200, RC), + {ResultJson} = ?JSON_DECODE(RespBody), + ResultJsonBody = couch_util:get_value(<<"results">>, ResultJson), + {InnerJson} = lists:nth(1, ResultJsonBody), + ?assertEqual(2, length(couch_util:get_value(<<"rows">>, InnerJson))) + end)}. should_succeed_on_local_docs_with_queries_limit_skip(Url) -> - {timeout, ?TIMEOUT, ?_test(begin - [create_doc(Url, "_local/doc" ++ ?i2l(I)) || I <- lists:seq(1, 10)], - QueryDoc = "{\"queries\": [{\"limit\": 5, \"skip\": 2}]}", - {ok, RC, _, RespBody} = test_request:post(Url ++ - "/_local_docs/queries/", [?CONTENT_JSON, ?AUTH], QueryDoc), - ?assertEqual(200, RC), - {ResultJson} = ?JSON_DECODE(RespBody), - ResultJsonBody = couch_util:get_value(<<"results">>, ResultJson), - {InnerJson} = lists:nth(1, ResultJsonBody), - ?assertEqual(5, length(couch_util:get_value(<<"rows">>, InnerJson))) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + [create_doc(Url, "_local/doc" ++ ?i2l(I)) || I <- lists:seq(1, 10)], + QueryDoc = "{\"queries\": [{\"limit\": 5, \"skip\": 2}]}", + {ok, RC, _, RespBody} = test_request:post( + Url ++ + "/_local_docs/queries/", + [?CONTENT_JSON, ?AUTH], + QueryDoc + ), + ?assertEqual(200, RC), + {ResultJson} = ?JSON_DECODE(RespBody), + ResultJsonBody = couch_util:get_value(<<"results">>, ResultJson), + {InnerJson} = lists:nth(1, ResultJsonBody), + ?assertEqual(5, length(couch_util:get_value(<<"rows">>, InnerJson))) + end)}. should_succeed_on_local_docs_with_multiple_queries(Url) -> - {timeout, ?TIMEOUT, ?_test(begin - [create_doc(Url, "_local/doc" ++ ?i2l(I)) || I <- lists:seq(1, 10)], - QueryDoc = "{\"queries\": [{\"keys\": [ \"_local/doc3\", - \"_local/doc8\"]}, {\"limit\": 5, \"skip\": 2}]}", - {ok, RC, _, RespBody} = test_request:post(Url ++ - "/_local_docs/queries/", [?CONTENT_JSON, ?AUTH], QueryDoc), - ?assertEqual(200, RC), - {ResultJson} = ?JSON_DECODE(RespBody), - ResultJsonBody = couch_util:get_value(<<"results">>, ResultJson), - {InnerJson1} = lists:nth(1, ResultJsonBody), - ?assertEqual(2, length(couch_util:get_value(<<"rows">>, InnerJson1))), - {InnerJson2} = lists:nth(2, ResultJsonBody), - ?assertEqual(5, length(couch_util:get_value(<<"rows">>, InnerJson2))) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + [create_doc(Url, "_local/doc" ++ ?i2l(I)) || I <- lists:seq(1, 10)], + QueryDoc = + "{\"queries\": [{\"keys\": [ \"_local/doc3\",\n" + " \"_local/doc8\"]}, {\"limit\": 5, \"skip\": 2}]}", + {ok, RC, _, RespBody} = test_request:post( + Url ++ + "/_local_docs/queries/", + [?CONTENT_JSON, ?AUTH], + QueryDoc + ), + ?assertEqual(200, RC), + {ResultJson} = ?JSON_DECODE(RespBody), + ResultJsonBody = couch_util:get_value(<<"results">>, ResultJson), + {InnerJson1} = lists:nth(1, ResultJsonBody), + ?assertEqual(2, length(couch_util:get_value(<<"rows">>, InnerJson1))), + {InnerJson2} = lists:nth(2, ResultJsonBody), + ?assertEqual(5, length(couch_util:get_value(<<"rows">>, InnerJson2))) + end)}. should_return_headers_after_starting_continious(Url) -> - {timeout, ?TIMEOUT, ?_test(begin - {ok, _, _, Bin} = - test_request:get(Url ++ "/_changes?feed=live&timeout=1", [?AUTH]), - - Parts = binary:split(Bin, <<"\n">>, [global]), - %% we should receive at least one part even when timeout=1 - ?assertNotEqual([], Parts) - end)}. + {timeout, ?TIMEOUT, + ?_test(begin + {ok, _, _, Bin} = + test_request:get(Url ++ "/_changes?feed=live&timeout=1", [?AUTH]), + + Parts = binary:split(Bin, <<"\n">>, [global]), + %% we should receive at least one part even when timeout=1 + ?assertNotEqual([], Parts) + end)}. wait_non_empty_chunk(Url) -> test_util:wait(fun() -> diff --git a/src/chttpd/test/eunit/chttpd_dbs_info_test.erl b/src/chttpd/test/eunit/chttpd_dbs_info_test.erl index 5b61d8831..b4027eead 100644 --- a/src/chttpd/test/eunit/chttpd_dbs_info_test.erl +++ b/src/chttpd/test/eunit/chttpd_dbs_info_test.erl @@ -20,10 +20,9 @@ -define(AUTH, {basic_auth, {?USER, ?PASS}}). -define(CONTENT_JSON, {"Content-Type", "application/json"}). - setup() -> Hashed = couch_passwords:hash_admin_password(?PASS), - ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist=false), + ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist = false), Addr = config:get("chttpd", "bind_address", "127.0.0.1"), Port = mochiweb_socket_server:get(chttpd, port), Url = lists:concat(["http://", Addr, ":", Port, "/"]), @@ -38,7 +37,7 @@ teardown(Url) -> Db2Url = lists:concat([Url, "db2"]), delete_db(Db1Url), delete_db(Db2Url), - ok = config:delete("admins", ?USER, _Persist=false). + ok = config:delete("admins", ?USER, _Persist = false). create_db(Url) -> {ok, Status, _, _} = test_request:put(Url, [?CONTENT_JSON, ?AUTH], "{}"), @@ -52,10 +51,12 @@ dbs_info_test_() -> "chttpd dbs info tests", { setup, - fun chttpd_test_util:start_couch/0, fun chttpd_test_util:stop_couch/1, + fun chttpd_test_util:start_couch/0, + fun chttpd_test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_return_error_for_get_db_info/1, fun should_return_dbs_info_for_single_db/1, @@ -68,102 +69,136 @@ dbs_info_test_() -> } }. - should_return_error_for_get_db_info(Url) -> ?_test(begin - {ok, Code, _, ResultBody} = test_request:get(Url ++ "/_dbs_info?" - ++ "keys=[\"db1\"]", [?CONTENT_JSON, ?AUTH]), + {ok, Code, _, ResultBody} = test_request:get( + Url ++ "/_dbs_info?" ++ + "keys=[\"db1\"]", + [?CONTENT_JSON, ?AUTH] + ), {Body} = jiffy:decode(ResultBody), [ - ?assertEqual(<<"method_not_allowed">>, - couch_util:get_value(<<"error">>, Body)), + ?assertEqual( + <<"method_not_allowed">>, + couch_util:get_value(<<"error">>, Body) + ), ?assertEqual(405, Code) ] end). - should_return_dbs_info_for_single_db(Url) -> ?_test(begin NewDoc = "{\"keys\": [\"db1\"]}", - {ok, _, _, ResultBody} = test_request:post(Url ++ "/_dbs_info/", - [?CONTENT_JSON, ?AUTH], NewDoc), + {ok, _, _, ResultBody} = test_request:post( + Url ++ "/_dbs_info/", + [?CONTENT_JSON, ?AUTH], + NewDoc + ), BodyJson = jiffy:decode(ResultBody), {Db1Data} = lists:nth(1, BodyJson), [ - ?assertEqual(<<"db1">>, - couch_util:get_value(<<"key">>, Db1Data)), - ?assertNotEqual(undefined, - couch_util:get_value(<<"info">>, Db1Data)) + ?assertEqual( + <<"db1">>, + couch_util:get_value(<<"key">>, Db1Data) + ), + ?assertNotEqual( + undefined, + couch_util:get_value(<<"info">>, Db1Data) + ) ] end). - should_return_dbs_info_for_multiple_dbs(Url) -> ?_test(begin NewDoc = "{\"keys\": [\"db1\", \"db2\"]}", - {ok, _, _, ResultBody} = test_request:post(Url ++ "/_dbs_info/", - [?CONTENT_JSON, ?AUTH], NewDoc), + {ok, _, _, ResultBody} = test_request:post( + Url ++ "/_dbs_info/", + [?CONTENT_JSON, ?AUTH], + NewDoc + ), BodyJson = jiffy:decode(ResultBody), {Db1Data} = lists:nth(1, BodyJson), {Db2Data} = lists:nth(2, BodyJson), [ - ?assertEqual(<<"db1">>, - couch_util:get_value(<<"key">>, Db1Data)), - ?assertNotEqual(undefined, - couch_util:get_value(<<"info">>, Db1Data)), - ?assertEqual(<<"db2">>, - couch_util:get_value(<<"key">>, Db2Data)), - ?assertNotEqual(undefined, - couch_util:get_value(<<"info">>, Db2Data)) + ?assertEqual( + <<"db1">>, + couch_util:get_value(<<"key">>, Db1Data) + ), + ?assertNotEqual( + undefined, + couch_util:get_value(<<"info">>, Db1Data) + ), + ?assertEqual( + <<"db2">>, + couch_util:get_value(<<"key">>, Db2Data) + ), + ?assertNotEqual( + undefined, + couch_util:get_value(<<"info">>, Db2Data) + ) ] end). - should_return_error_for_exceeded_keys(Url) -> ?_test(begin NewDoc = "{\"keys\": [\"db1\", \"db2\"]}", ok = config:set("chttpd", "max_db_number_for_dbs_info_req", "1"), - {ok, Code, _, ResultBody} = test_request:post(Url ++ "/_dbs_info/", - [?CONTENT_JSON, ?AUTH], NewDoc), + {ok, Code, _, ResultBody} = test_request:post( + Url ++ "/_dbs_info/", + [?CONTENT_JSON, ?AUTH], + NewDoc + ), {Body} = jiffy:decode(ResultBody), ok = config:delete("chttpd", "max_db_number_for_dbs_info_req"), [ - ?assertEqual(<<"bad_request">>, - couch_util:get_value(<<"error">>, Body)), + ?assertEqual( + <<"bad_request">>, + couch_util:get_value(<<"error">>, Body) + ), ?assertEqual(400, Code) ] end). - should_return_error_for_missing_keys(Url) -> ?_test(begin NewDoc = "{\"missingkeys\": [\"db1\", \"db2\"]}", - {ok, Code, _, ResultBody} = test_request:post(Url ++ "/_dbs_info/", - [?CONTENT_JSON, ?AUTH], NewDoc), + {ok, Code, _, ResultBody} = test_request:post( + Url ++ "/_dbs_info/", + [?CONTENT_JSON, ?AUTH], + NewDoc + ), {Body} = jiffy:decode(ResultBody), [ - ?assertEqual(<<"bad_request">>, - couch_util:get_value(<<"error">>, Body)), + ?assertEqual( + <<"bad_request">>, + couch_util:get_value(<<"error">>, Body) + ), ?assertEqual(400, Code) ] end). - should_return_dbs_info_for_dbs_with_mixed_state(Url) -> ?_test(begin NewDoc = "{\"keys\": [\"db1\", \"noexisteddb\"]}", - {ok, _, _, ResultBody} = test_request:post(Url ++ "/_dbs_info/", - [?CONTENT_JSON, ?AUTH], NewDoc), + {ok, _, _, ResultBody} = test_request:post( + Url ++ "/_dbs_info/", + [?CONTENT_JSON, ?AUTH], + NewDoc + ), Json = jiffy:decode(ResultBody), {Db1Data} = lists:nth(1, Json), {Db2Data} = lists:nth(2, Json), [ ?assertEqual( - <<"db1">>, couch_util:get_value(<<"key">>, Db1Data)), - ?assertNotEqual(undefined, - couch_util:get_value(<<"info">>, Db1Data)), + <<"db1">>, couch_util:get_value(<<"key">>, Db1Data) + ), + ?assertNotEqual( + undefined, + couch_util:get_value(<<"info">>, Db1Data) + ), ?assertEqual( - <<"noexisteddb">>, couch_util:get_value(<<"key">>, Db2Data)), + <<"noexisteddb">>, couch_util:get_value(<<"key">>, Db2Data) + ), ?assertEqual(undefined, couch_util:get_value(<<"info">>, Db2Data)) ] end). diff --git a/src/chttpd/test/eunit/chttpd_delayed_test.erl b/src/chttpd/test/eunit/chttpd_delayed_test.erl index 63e6cb0e5..4b0fbd55b 100644 --- a/src/chttpd/test/eunit/chttpd_delayed_test.erl +++ b/src/chttpd/test/eunit/chttpd_delayed_test.erl @@ -7,17 +7,20 @@ -define(PASS, "pass"). -define(AUTH, {basic_auth, {?USER, ?PASS}}). -define(CONTENT_JSON, {"Content-Type", "application/json"}). --define(DDOC, "{\"_id\": \"_design/bar\", \"views\": {\"baz\": - {\"map\": \"function(doc) {emit(doc._id, doc._id);}\"}}}"). +-define(DDOC, + "{\"_id\": \"_design/bar\", \"views\": {\"baz\":\n" + " {\"map\": \"function(doc) {emit(doc._id, doc._id);}\"}}}" +). -define(FIXTURE_TXT, ?ABS_PATH(?FILE)). -define(i2l(I), integer_to_list(I)). --define(TIMEOUT, 60). % seconds +% seconds +-define(TIMEOUT, 60). setup() -> Hashed = couch_passwords:hash_admin_password(?PASS), - ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist=false), - ok = config:set("chttpd", "buffer_response", "true", _Persist=false), + ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist = false), + ok = config:set("chttpd", "buffer_response", "true", _Persist = false), TmpDb = ?tempdb(), Addr = config:get("chttpd", "bind_address", "127.0.0.1"), Port = mochiweb_socket_server:get(chttpd, port), @@ -27,26 +30,26 @@ setup() -> teardown(Url) -> delete_db(Url), - ok = config:delete("admins", ?USER, _Persist=false). + ok = config:delete("admins", ?USER, _Persist = false). create_db(Url) -> {ok, Status, _, _} = test_request:put(Url, [?CONTENT_JSON, ?AUTH], "{}"), ?assert(Status =:= 201 orelse Status =:= 202). - delete_db(Url) -> {ok, 200, _, _} = test_request:delete(Url, [?AUTH]). - all_test_() -> { "chttpd delay tests", { setup, - fun chttpd_test_util:start_couch/0, fun chttpd_test_util:stop_couch/1, + fun chttpd_test_util:start_couch/0, + fun chttpd_test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun test_buffer_response_all_docs/1, fun test_buffer_response_changes/1 @@ -55,18 +58,15 @@ all_test_() -> } }. - test_buffer_response_all_docs(Url) -> assert_successful_response(Url ++ "/_all_docs"). - test_buffer_response_changes(Url) -> assert_successful_response(Url ++ "/_changes"). - assert_successful_response(Url) -> - {timeout, ?TIMEOUT, ?_test(begin - {ok, Code, _Headers, _Body} = test_request:get(Url, [?AUTH]), - ?assertEqual(200, Code) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + {ok, Code, _Headers, _Body} = test_request:get(Url, [?AUTH]), + ?assertEqual(200, Code) + end)}. diff --git a/src/chttpd/test/eunit/chttpd_error_info_tests.erl b/src/chttpd/test/eunit/chttpd_error_info_tests.erl index fdb015c08..aefb3bdc5 100644 --- a/src/chttpd/test/eunit/chttpd_error_info_tests.erl +++ b/src/chttpd/test/eunit/chttpd_error_info_tests.erl @@ -14,7 +14,6 @@ -include_lib("eunit/include/eunit.hrl"). - error_info_test() -> Error = <<"error">>, Reason = <<"reason">>, @@ -75,7 +74,7 @@ error_info_test() -> { file_exists, {412, <<"file_exists">>, - <<"The database could not be created, the file already exists.">>} + <<"The database could not be created, the file already exists.">>} }, { {error, {nodedown, Reason}}, {412, <<"nodedown">>, Reason} @@ -102,18 +101,18 @@ error_info_test() -> }, { requested_range_not_satisfiable, - {416, <<"requested_range_not_satisfiable">>, - <<"Requested range not satisfiable">>} + {416, <<"requested_range_not_satisfiable">>, <<"Requested range not satisfiable">>} }, { {error, {illegal_database_name, <<"foo">>}}, - {400, <<"illegal_database_name">>, - <<"Name: 'foo'. Only lowercase characters (a-z), digits (0-9), and any of" - " the characters _, $, (, ), +, -, and / are allowed." - " Must begin with a letter.">>} + {400, <<"illegal_database_name">>, << + "Name: 'foo'. Only lowercase characters (a-z), digits (0-9), and any of" + " the characters _, $, (, ), +, -, and / are allowed." + " Must begin with a letter." + >>} }, { - {Error, {illegal_docid,1}}, + {Error, {illegal_docid, 1}}, {400, <<"illegal_docid">>, 1} }, { @@ -126,20 +125,21 @@ error_info_test() -> }, { not_implemented, - {501, <<"not_implemented">>, - <<"this feature is not yet implemented">>} + {501, <<"not_implemented">>, <<"this feature is not yet implemented">>} }, { timeout, - {500, <<"timeout">>, - <<"The request could not be processed in a reasonable" - " amount of time.">>} + {500, <<"timeout">>, << + "The request could not be processed in a reasonable" + " amount of time." + >>} }, { {timeout, Error}, - {500, <<"timeout">>, - <<"The request could not be processed in a reasonable" - " amount of time.">>} + {500, <<"timeout">>, << + "The request could not be processed in a reasonable" + " amount of time." + >>} }, { {Error, null}, @@ -163,6 +163,9 @@ error_info_test() -> } ], - lists:foreach(fun({Arg, Result}) -> - ?assertEqual(Result, chttpd:error_info(Arg)) - end, ArgResult). + lists:foreach( + fun({Arg, Result}) -> + ?assertEqual(Result, chttpd:error_info(Arg)) + end, + ArgResult + ). diff --git a/src/chttpd/test/eunit/chttpd_external_test.erl b/src/chttpd/test/eunit/chttpd_external_test.erl index 4ed31891d..cd691fbaa 100644 --- a/src/chttpd/test/eunit/chttpd_external_test.erl +++ b/src/chttpd/test/eunit/chttpd_external_test.erl @@ -15,7 +15,6 @@ -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). - setup_mock() -> ok = meck:new([config, couch], [passthrough]), ok = meck:expect(couch_db, is_clustered, 1, false), @@ -91,9 +90,12 @@ json_req_obj_remote_httpd_req_test_() -> should_convert_req_to_json_obj(HttpdReq) -> Expect = expect(), {Result} = chttpd_external:json_req_obj(HttpdReq, <<"fake">>), - lists:map(fun({K, V}) -> - {K, ?_assertEqual(couch_util:get_value(K, Expect), V)} - end, Result). + lists:map( + fun({K, V}) -> + {K, ?_assertEqual(couch_util:get_value(K, Expect), V)} + end, + Result + ). expect() -> [ @@ -110,10 +112,11 @@ expect() -> {<<"peer">>, <<"127.0.0.1">>}, {<<"form">>, {[]}}, {<<"cookie">>, {[]}}, - {<<"userCtx">>, {[ - {<<"db">>,<<"fake">>}, - {<<"name">>,null}, - {<<"roles">>,[]} - ]}}, + {<<"userCtx">>, + {[ + {<<"db">>, <<"fake">>}, + {<<"name">>, null}, + {<<"roles">>, []} + ]}}, {<<"secObj">>, []} ]. diff --git a/src/chttpd/test/eunit/chttpd_handlers_tests.erl b/src/chttpd/test/eunit/chttpd_handlers_tests.erl index 649d82e86..7cca6659d 100644 --- a/src/chttpd/test/eunit/chttpd_handlers_tests.erl +++ b/src/chttpd/test/eunit/chttpd_handlers_tests.erl @@ -15,7 +15,6 @@ -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). - setup() -> Addr = config:get("chttpd", "bind_address", "127.0.0.1"), Port = mochiweb_socket_server:get(chttpd, port), @@ -25,7 +24,6 @@ setup() -> teardown(_Url) -> ok. - replicate_test_() -> { "_replicate", @@ -35,7 +33,8 @@ replicate_test_() -> fun chttpd_test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_escape_dbname_on_replicate/1 ] @@ -43,16 +42,17 @@ replicate_test_() -> } }. - should_escape_dbname_on_replicate(Url) -> ?_test( begin UrlBin = ?l2b(Url), - Request = couch_util:json_encode({[ - {<<"source">>, <<UrlBin/binary, "/foo%2Fbar">>}, - {<<"target">>, <<"bar/baz">>}, - {<<"create_target">>, true} - ]}), + Request = couch_util:json_encode( + {[ + {<<"source">>, <<UrlBin/binary, "/foo%2Fbar">>}, + {<<"target">>, <<"bar/baz">>}, + {<<"create_target">>, true} + ]} + ), {ok, 200, _, Body} = request_replicate(Url ++ "/_replicate", Request), JSON = couch_util:json_decode(Body), @@ -60,8 +60,8 @@ should_escape_dbname_on_replicate(Url) -> Target = json_value(JSON, [<<"target">>, <<"url">>]), ?assertEqual(<<UrlBin/binary, "/foo%2Fbar">>, Source), ?assertEqual(<<UrlBin/binary, "/bar%2Fbaz">>, Target) - end). - + end + ). json_value(JSON, Keys) -> couch_util:get_nested_json_value(JSON, Keys). @@ -80,8 +80,9 @@ request(Method, Url, Headers, Body, {M, F}, MockFun) -> Result = test_request:Method(Url, Headers, Body), ?assert(meck:validate(M)), Result - catch Kind:Reason -> - {Kind, Reason} + catch + Kind:Reason -> + {Kind, Reason} after meck:unload(M) end. diff --git a/src/chttpd/test/eunit/chttpd_open_revs_error_test.erl b/src/chttpd/test/eunit/chttpd_open_revs_error_test.erl index d53d370f8..3eda08ae0 100644 --- a/src/chttpd/test/eunit/chttpd_open_revs_error_test.erl +++ b/src/chttpd/test/eunit/chttpd_open_revs_error_test.erl @@ -19,12 +19,11 @@ -define(PASS, "pass"). -define(AUTH, {basic_auth, {?USER, ?PASS}}). -define(CONTENT_JSON, {"Content-Type", "application/json"}). --define(CONTENT_MULTI_FORM, {"Content-Type", - "multipart/form-data;boundary=\"bound\""}). +-define(CONTENT_MULTI_FORM, {"Content-Type", "multipart/form-data;boundary=\"bound\""}). setup() -> Hashed = couch_passwords:hash_admin_password(?PASS), - ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist=false), + ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist = false), TmpDb = ?tempdb(), Addr = config:get("chttpd", "bind_address", "127.0.0.1"), Port = mochiweb_socket_server:get(chttpd, port), @@ -36,16 +35,18 @@ setup() -> teardown(Url) -> delete_db(Url), (catch meck:unload(fabric)), - ok = config:delete("admins", ?USER, _Persist=false). + ok = config:delete("admins", ?USER, _Persist = false). create_db(Url) -> {ok, Status, _, _} = test_request:put(Url, [?CONTENT_JSON, ?AUTH], "{}"), ?assert(Status =:= 201 orelse Status =:= 202). - create_doc(Url, Id) -> - test_request:put(Url ++ "/" ++ Id, - [?CONTENT_JSON, ?AUTH], "{\"mr\": \"rockoartischocko\"}"). + test_request:put( + Url ++ "/" ++ Id, + [?CONTENT_JSON, ?AUTH], + "{\"mr\": \"rockoartischocko\"}" + ). delete_db(Url) -> {ok, 200, _, _} = test_request:delete(Url, [?AUTH]). @@ -59,7 +60,8 @@ open_revs_error_test_() -> fun chttpd_test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_return_503_error_for_open_revs_get/1, fun should_return_503_error_for_open_revs_post_form/1 @@ -73,31 +75,41 @@ should_return_503_error_for_open_revs_get(Url) -> {Json} = ?JSON_DECODE(Body), Ref = couch_util:get_value(<<"rev">>, Json, undefined), mock_open_revs({error, all_workers_died}), - {ok, Code, _, _} = test_request:get(Url ++ - "/testdoc?rev=" ++ ?b2l(Ref), [?AUTH]), + {ok, Code, _, _} = test_request:get( + Url ++ + "/testdoc?rev=" ++ ?b2l(Ref), + [?AUTH] + ), ?_assertEqual(503, Code). should_return_503_error_for_open_revs_post_form(Url) -> Port = mochiweb_socket_server:get(chttpd, port), - Host = lists:concat([ "http://127.0.0.1:", Port]), + Host = lists:concat(["http://127.0.0.1:", Port]), Referer = {"Referer", Host}, Body1 = "{\"body\":\"This is a body.\"}", DocBeg = "--bound\r\nContent-Disposition: form-data; name=\"_doc\"\r\n\r\n", DocRev = "--bound\r\nContent-Disposition: form-data; name=\"_rev\"\r\n\r\n", - DocRest = "\r\n--bound\r\nContent-Disposition:" + DocRest = + "\r\n--bound\r\nContent-Disposition:" "form-data; name=\"_attachments\"; filename=\"file.txt\"\r\n" "Content-Type: text/plain\r\n\r\ncontents of file.txt\r\n\r\n" "--bound--", Doc1 = lists:concat([DocBeg, Body1, DocRest]), - {ok, _, _, ResultBody} = test_request:post(Url ++ "/" ++ "RevDoc", - [?CONTENT_MULTI_FORM, ?AUTH, Referer], Doc1), + {ok, _, _, ResultBody} = test_request:post( + Url ++ "/" ++ "RevDoc", + [?CONTENT_MULTI_FORM, ?AUTH, Referer], + Doc1 + ), {Json} = ?JSON_DECODE(ResultBody), Ref = couch_util:get_value(<<"rev">>, Json, undefined), - Doc2 = lists:concat([DocRev, ?b2l(Ref) , DocRest]), + Doc2 = lists:concat([DocRev, ?b2l(Ref), DocRest]), mock_open_revs({error, all_workers_died}), - {ok, Code, _, ResultBody1} = test_request:post(Url ++ "/" ++ "RevDoc", - [?CONTENT_MULTI_FORM, ?AUTH, Referer], Doc2), + {ok, Code, _, ResultBody1} = test_request:post( + Url ++ "/" ++ "RevDoc", + [?CONTENT_MULTI_FORM, ?AUTH, Referer], + Doc2 + ), {Json1} = ?JSON_DECODE(ResultBody1), ErrorMessage = couch_util:get_value(<<"error">>, Json1), [ diff --git a/src/chttpd/test/eunit/chttpd_plugin_tests.erl b/src/chttpd/test/eunit/chttpd_plugin_tests.erl index 36572a419..effef589a 100644 --- a/src/chttpd/test/eunit/chttpd_plugin_tests.erl +++ b/src/chttpd/test/eunit/chttpd_plugin_tests.erl @@ -20,7 +20,8 @@ before_serve_file/5 ]). --export([ %% couch_epi_plugin behaviour +%% couch_epi_plugin behaviour +-export([ app/0, providers/0, services/0, @@ -43,7 +44,6 @@ data_subscriptions() -> []. processes() -> []. notify(_, _, _) -> ok. - setup() -> couch_tests:setup([ couch_epi_dispatch:dispatch(chttpd, ?MODULE) @@ -57,7 +57,7 @@ before_request({false, Id}) -> [{false, Id}]; before_request({fail, Id}) -> throw({before_request, Id}). after_request({true, Id}, A) -> [{true, [{after_request, Id}]}, A]; -after_request({false, Id}, A) -> [{false, Id}, A]; +after_request({false, Id}, A) -> [{false, Id}, A]; after_request({fail, Id}, _A) -> throw({after_request, Id}). handle_error({true, Id}) -> [{true, [{handle_error, Id}]}]; @@ -82,7 +82,9 @@ callback_test_() -> { "callback tests", { - setup, fun setup/0, fun teardown/1, + setup, + fun setup/0, + fun teardown/1, [ fun before_request_match/0, fun before_request_no_match/0, @@ -107,81 +109,92 @@ callback_test_() -> } }. - before_request_match() -> ?assertEqual( {ok, {true, [{before_request, foo}]}}, - chttpd_plugin:before_request({true, foo})). + chttpd_plugin:before_request({true, foo}) + ). before_request_no_match() -> ?assertEqual( {ok, {false, foo}}, - chttpd_plugin:before_request({false, foo})). + chttpd_plugin:before_request({false, foo}) + ). before_request_throw() -> ?assertThrow( {before_request, foo}, - chttpd_plugin:before_request({fail, foo})). - + chttpd_plugin:before_request({fail, foo}) + ). after_request_match() -> ?assertEqual( {ok, bar}, - chttpd_plugin:after_request({true, foo}, bar)). + chttpd_plugin:after_request({true, foo}, bar) + ). after_request_no_match() -> ?assertEqual( {ok, bar}, - chttpd_plugin:after_request({false, foo}, bar)). + chttpd_plugin:after_request({false, foo}, bar) + ). after_request_throw() -> ?assertThrow( {after_request, foo}, - chttpd_plugin:after_request({fail, foo}, bar)). - + chttpd_plugin:after_request({fail, foo}, bar) + ). handle_error_match() -> ?assertEqual( {true, [{handle_error, foo}]}, - chttpd_plugin:handle_error({true, foo})). + chttpd_plugin:handle_error({true, foo}) + ). handle_error_no_match() -> ?assertEqual( {false, foo}, - chttpd_plugin:handle_error({false, foo})). + chttpd_plugin:handle_error({false, foo}) + ). handle_error_throw() -> ?assertThrow( {handle_error, foo}, - chttpd_plugin:handle_error({fail, foo})). + chttpd_plugin:handle_error({fail, foo}) + ). before_response_match() -> ?assertEqual( {ok, {{true, [{before_response, foo}]}, 1, 2, 3}}, - chttpd_plugin:before_response({true, foo}, 1, 2, 3)). + chttpd_plugin:before_response({true, foo}, 1, 2, 3) + ). before_response_no_match() -> ?assertEqual( {ok, {{false, foo}, 1, 2, 3}}, - chttpd_plugin:before_response({false, foo}, 1, 2, 3)). + chttpd_plugin:before_response({false, foo}, 1, 2, 3) + ). before_response_throw() -> ?assertThrow( {before_response, foo}, - chttpd_plugin:before_response({fail, foo}, 1, 2, 3)). - + chttpd_plugin:before_response({fail, foo}, 1, 2, 3) + ). before_serve_file_match() -> ?assertEqual( {ok, {{true, [{before_serve_file, foo}]}, 1, 2, 3, 4}}, - chttpd_plugin:before_serve_file({true, foo}, 1, 2, 3, 4)). + chttpd_plugin:before_serve_file({true, foo}, 1, 2, 3, 4) + ). before_serve_file_no_match() -> ?assertEqual( {ok, {{false, foo}, 1, 2, 3, 4}}, - chttpd_plugin:before_serve_file({false, foo}, 1, 2, 3, 4)). + chttpd_plugin:before_serve_file({false, foo}, 1, 2, 3, 4) + ). before_serve_file_throw() -> ?assertThrow( before_serve_file, - chttpd_plugin:before_serve_file({fail, foo}, 1, 2, 3, 4)). + chttpd_plugin:before_serve_file({fail, foo}, 1, 2, 3, 4) + ). diff --git a/src/chttpd/test/eunit/chttpd_prefer_header_test.erl b/src/chttpd/test/eunit/chttpd_prefer_header_test.erl index 1b1195418..55c9f350e 100644 --- a/src/chttpd/test/eunit/chttpd_prefer_header_test.erl +++ b/src/chttpd/test/eunit/chttpd_prefer_header_test.erl @@ -17,52 +17,46 @@ -include_lib("couch/include/couch_db.hrl"). -include_lib("eunit/include/eunit.hrl"). - mock_request(ExcludeHeader) -> Headers = mochiweb_headers:make(ExcludeHeader), MochiReq = mochiweb_request:new(nil, 'GET', "/", {1, 1}, Headers), MochiReq:cleanup(), #httpd{mochi_req = MochiReq}. - default_headers() -> [ - {"Cache-Control","must-revalidate"}, - {"Content-Type","application/json"}, + {"Cache-Control", "must-revalidate"}, + {"Content-Type", "application/json"}, {"Content-Length", "100"}, - {"ETag","\"12343\""}, - {"X-Couch-Request-ID","7bd1adab86"}, - {"X-CouchDB-Body-Time","0"}, + {"ETag", "\"12343\""}, + {"X-Couch-Request-ID", "7bd1adab86"}, + {"X-CouchDB-Body-Time", "0"}, {"Vary", "Accept-Encoding"}, - {"Server","CouchDB/2.1.0-f1a1d7f1c (Erlang OTP/19)"} + {"Server", "CouchDB/2.1.0-f1a1d7f1c (Erlang OTP/19)"} ]. - minimal_options_headers() -> [ - {"Cache-Control","must-revalidate"}, - {"Content-Type","application/json"}, + {"Cache-Control", "must-revalidate"}, + {"Content-Type", "application/json"}, {"Content-Length", "100"}, - {"ETag","\"12343\""}, + {"ETag", "\"12343\""}, {"Vary", "Accept-Encoding"}, - {"Server","CouchDB/2.1.0-f1a1d7f1c (Erlang OTP/19)"} + {"Server", "CouchDB/2.1.0-f1a1d7f1c (Erlang OTP/19)"} ]. - default_no_exclude_header_test() -> Headers = chttpd_prefer_header:maybe_return_minimal( mock_request([]), default_headers() - ), + ), ?assertEqual(default_headers(), Headers). - unsupported_exclude_header_test() -> Req = mock_request([{"prefer", "Wrong"}]), Headers = chttpd_prefer_header:maybe_return_minimal(Req, default_headers()), ?assertEqual(default_headers(), Headers). - empty_header_test() -> Req = mock_request([{"prefer", ""}]), Headers = chttpd_prefer_header:maybe_return_minimal(Req, default_headers()), @@ -70,24 +64,20 @@ empty_header_test() -> setup_all() -> ok = meck:new(config), - ok = meck:expect(config, get, fun("chttpd", "prefer_minimal", _) -> + ok = meck:expect(config, get, fun("chttpd", "prefer_minimal", _) -> "Cache-Control, Content-Length, Content-Type, ETag, Server, Vary" end), ok. - teardown_all(_) -> meck:unload(). - setup() -> meck:reset([config]). - teardown(_) -> ok. - exclude_headers_test_() -> { "Test Prefer headers", @@ -108,19 +98,16 @@ exclude_headers_test_() -> } }. - minimal_options(_) -> Req = mock_request([{"Prefer", "return=minimal"}]), Headers = chttpd_prefer_header:maybe_return_minimal(Req, default_headers()), ?_assertEqual(minimal_options_headers(), Headers). - minimal_options_check_header_case(_) -> Req = mock_request([{"prefer", "return=minimal"}]), Headers = chttpd_prefer_header:maybe_return_minimal(Req, default_headers()), ?_assertEqual(minimal_options_headers(), Headers). - minimal_options_check_header_value_case(_) -> Req = mock_request([{"prefer", "RETURN=MINIMAL"}]), Headers = chttpd_prefer_header:maybe_return_minimal(Req, default_headers()), diff --git a/src/chttpd/test/eunit/chttpd_purge_tests.erl b/src/chttpd/test/eunit/chttpd_purge_tests.erl index ab435682a..a8e1a955d 100644 --- a/src/chttpd/test/eunit/chttpd_purge_tests.erl +++ b/src/chttpd/test/eunit/chttpd_purge_tests.erl @@ -12,20 +12,17 @@ -module(chttpd_purge_tests). - -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). - -define(USER, "chttpd_db_test_admin"). -define(PASS, "pass"). -define(AUTH, {basic_auth, {?USER, ?PASS}}). -define(CONTENT_JSON, {"Content-Type", "application/json"}). - setup() -> Hashed = couch_passwords:hash_admin_password(?PASS), - ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist=false), + ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist = false), TmpDb = ?tempdb(), Addr = config:get("chttpd", "bind_address", "127.0.0.1"), Port = mochiweb_socket_server:get(chttpd, port), @@ -33,35 +30,38 @@ setup() -> create_db(Url), Url. - teardown(Url) -> delete_db(Url), - ok = config:delete("admins", ?USER, _Persist=false). - + ok = config:delete("admins", ?USER, _Persist = false). create_db(Url) -> {ok, Status, _, _} = test_request:put(Url, [?CONTENT_JSON, ?AUTH], "{}"), ?assert(Status =:= 201 orelse Status =:= 202). - create_doc(Url, Id) -> - test_request:put(Url ++ "/" ++ Id, - [?CONTENT_JSON, ?AUTH], "{\"mr\": \"rockoartischocko\"}"). + test_request:put( + Url ++ "/" ++ Id, + [?CONTENT_JSON, ?AUTH], + "{\"mr\": \"rockoartischocko\"}" + ). create_doc(Url, Id, Content) -> - test_request:put(Url ++ "/" ++ Id, - [?CONTENT_JSON, ?AUTH], "{\"mr\": \"" ++ Content ++ "\"}"). - + test_request:put( + Url ++ "/" ++ Id, + [?CONTENT_JSON, ?AUTH], + "{\"mr\": \"" ++ Content ++ "\"}" + ). create_docs(Url, Docs) -> - test_request:post(Url ++ "/_bulk_docs", - [?CONTENT_JSON, ?AUTH], ?JSON_ENCODE({[{docs, Docs}]})). - + test_request:post( + Url ++ "/_bulk_docs", + [?CONTENT_JSON, ?AUTH], + ?JSON_ENCODE({[{docs, Docs}]}) + ). delete_db(Url) -> {ok, 200, _, _} = test_request:delete(Url, [?AUTH]). - purge_test_() -> { "chttpd db tests", @@ -89,24 +89,25 @@ purge_test_() -> } }. - test_empty_purge_request(Url) -> ?_test(begin IdsRevs = "{}", - {ok, Status, _, ResultBody} = test_request:post(Url ++ "/_purge/", - [?CONTENT_JSON, ?AUTH], IdsRevs), + {ok, Status, _, ResultBody} = test_request:post( + Url ++ "/_purge/", + [?CONTENT_JSON, ?AUTH], + IdsRevs + ), ResultJson = ?JSON_DECODE(ResultBody), ?assert(Status =:= 201 orelse Status =:= 202), ?assertEqual( - {[ - {<<"purge_seq">>, null}, - {<<"purged">>,{[]}} - ]}, - ResultJson - ) + {[ + {<<"purge_seq">>, null}, + {<<"purged">>, {[]}} + ]}, + ResultJson + ) end). - test_ok_purge_request(Url) -> ?_test(begin {ok, _, _, Body} = create_doc(Url, "doc1"), @@ -119,230 +120,300 @@ test_ok_purge_request(Url) -> {Json3} = ?JSON_DECODE(Body3), Rev3 = couch_util:get_value(<<"rev">>, Json3, undefined), - IdsRevsEJson = {[ - {<<"doc1">>, [Rev1]}, - {<<"doc2">>, [Rev2]}, - {<<"doc3">>, [Rev3]} - ]}, + IdsRevsEJson = + {[ + {<<"doc1">>, [Rev1]}, + {<<"doc2">>, [Rev2]}, + {<<"doc3">>, [Rev3]} + ]}, IdsRevs = binary_to_list(?JSON_ENCODE(IdsRevsEJson)), - {ok, Status, _, ResultBody} = test_request:post(Url ++ "/_purge/", - [?CONTENT_JSON, ?AUTH], IdsRevs), + {ok, Status, _, ResultBody} = test_request:post( + Url ++ "/_purge/", + [?CONTENT_JSON, ?AUTH], + IdsRevs + ), ResultJson = ?JSON_DECODE(ResultBody), ?assert(Status =:= 201 orelse Status =:= 202), ?assertEqual( - {[ - {<<"purge_seq">>, null}, - {<<"purged">>, {[ + {[ + {<<"purge_seq">>, null}, + {<<"purged">>, + {[ {<<"doc1">>, [Rev1]}, {<<"doc2">>, [Rev2]}, {<<"doc3">>, [Rev3]} ]}} - ]}, - ResultJson - ) + ]}, + ResultJson + ) end). - test_ok_purge_request_with_101_docid(Url) -> ?_test(begin PurgedDocsNum = 101, - Docs = lists:foldl(fun(I, Acc) -> - Id = list_to_binary(integer_to_list(I)), - Doc = {[{<<"_id">>, Id}, {value, I}]}, - [Doc | Acc] - end, [], lists:seq(1, PurgedDocsNum)), + Docs = lists:foldl( + fun(I, Acc) -> + Id = list_to_binary(integer_to_list(I)), + Doc = {[{<<"_id">>, Id}, {value, I}]}, + [Doc | Acc] + end, + [], + lists:seq(1, PurgedDocsNum) + ), {ok, _, _, Body} = create_docs(Url, Docs), BodyJson = ?JSON_DECODE(Body), - PurgeBody = lists:map(fun({DocResp}) -> - Id = couch_util:get_value(<<"id">>, DocResp, undefined), - Rev = couch_util:get_value(<<"rev">>, DocResp, undefined), - {Id, [Rev]} - end, BodyJson), + PurgeBody = lists:map( + fun({DocResp}) -> + Id = couch_util:get_value(<<"id">>, DocResp, undefined), + Rev = couch_util:get_value(<<"rev">>, DocResp, undefined), + {Id, [Rev]} + end, + BodyJson + ), ok = config:set("purge", "max_document_id_number", "101"), try - {ok, Status, _, _} = test_request:post(Url ++ "/_purge/", - [?CONTENT_JSON, ?AUTH], ?JSON_ENCODE({PurgeBody})), + {ok, Status, _, _} = test_request:post( + Url ++ "/_purge/", + [?CONTENT_JSON, ?AUTH], + ?JSON_ENCODE({PurgeBody}) + ), ?assert(Status =:= 201 orelse Status =:= 202) after ok = config:delete("purge", "max_document_id_number") end end). - test_accepted_purge_request(Url) -> ?_test(begin {ok, _, _, Body} = create_doc(Url, "doc1"), {Json} = ?JSON_DECODE(Body), Rev1 = couch_util:get_value(<<"rev">>, Json, undefined), - IdsRevsEJson = {[ - {<<"doc1">>, [Rev1]} - ]}, + IdsRevsEJson = + {[ + {<<"doc1">>, [Rev1]} + ]}, IdsRevs = binary_to_list(?JSON_ENCODE(IdsRevsEJson)), meck:new(fabric, [passthrough]), - meck:expect(fabric, purge_docs, - fun(_, _, _) -> {accepted,[{accepted,[{1, - <<57,27,64,134,152,18,73,243,40,1,141,214,135,104,79,188>>}]}]} + meck:expect( + fabric, + purge_docs, + fun(_, _, _) -> + {accepted, [ + {accepted, [ + {1, + <<57, 27, 64, 134, 152, 18, 73, 243, 40, 1, 141, 214, 135, 104, 79, + 188>>} + ]} + ]} end ), - {ok, Status, _, ResultBody} = test_request:post(Url ++ "/_purge/", - [?CONTENT_JSON, ?AUTH], IdsRevs), + {ok, Status, _, ResultBody} = test_request:post( + Url ++ "/_purge/", + [?CONTENT_JSON, ?AUTH], + IdsRevs + ), ResultJson = ?JSON_DECODE(ResultBody), meck:unload(fabric), ?assert(Status =:= 202), ?assertEqual( {[ {<<"purge_seq">>, null}, - {<<"purged">>, {[ - {<<"doc1">>, [Rev1]} - ]}} + {<<"purged">>, + {[ + {<<"doc1">>, [Rev1]} + ]}} ]}, ResultJson ) end). - test_partial_purge_request(Url) -> ?_test(begin {ok, _, _, Body} = create_doc(Url, "doc1"), {Json} = ?JSON_DECODE(Body), Rev1 = couch_util:get_value(<<"rev">>, Json, undefined), - NewDoc = "{\"new_edits\": false, \"docs\": [{\"_id\": \"doc1\", - \"_revisions\": {\"start\": 1, \"ids\": [\"12345\", \"67890\"]}, - \"content\": \"updated\", \"_rev\": \"" ++ ?b2l(Rev1) ++ "\"}]}", - {ok, _, _, _} = test_request:post(Url ++ "/_bulk_docs/", - [?CONTENT_JSON, ?AUTH], NewDoc), + NewDoc = + "{\"new_edits\": false, \"docs\": [{\"_id\": \"doc1\",\n" + " \"_revisions\": {\"start\": 1, \"ids\": [\"12345\", \"67890\"]},\n" + " \"content\": \"updated\", \"_rev\": \"" ++ ?b2l(Rev1) ++ "\"}]}", + {ok, _, _, _} = test_request:post( + Url ++ "/_bulk_docs/", + [?CONTENT_JSON, ?AUTH], + NewDoc + ), IdsRevsEJson = {[{<<"doc1">>, [Rev1]}]}, IdsRevs = binary_to_list(?JSON_ENCODE(IdsRevsEJson)), - {ok, Status, _, ResultBody} = test_request:post(Url ++ "/_purge/", - [?CONTENT_JSON, ?AUTH], IdsRevs), + {ok, Status, _, ResultBody} = test_request:post( + Url ++ "/_purge/", + [?CONTENT_JSON, ?AUTH], + IdsRevs + ), ResultJson = ?JSON_DECODE(ResultBody), ?assert(Status =:= 201 orelse Status =:= 202), ?assertEqual( {[ {<<"purge_seq">>, null}, - {<<"purged">>, {[ - {<<"doc1">>, [Rev1]} - ]}} + {<<"purged">>, + {[ + {<<"doc1">>, [Rev1]} + ]}} ]}, ResultJson ), - {ok, Status2, _, ResultBody2} = test_request:get(Url - ++ "/doc1/", [?AUTH]), + {ok, Status2, _, ResultBody2} = test_request:get( + Url ++ + "/doc1/", + [?AUTH] + ), {Json2} = ?JSON_DECODE(ResultBody2), Content = couch_util:get_value(<<"content">>, Json2, undefined), ?assertEqual(<<"updated">>, Content), ?assert(Status2 =:= 200) end). - test_mixed_purge_request(Url) -> ?_test(begin {ok, _, _, Body} = create_doc(Url, "doc1"), {Json} = ?JSON_DECODE(Body), Rev1 = couch_util:get_value(<<"rev">>, Json, undefined), - NewDoc = "{\"new_edits\": false, \"docs\": [{\"_id\": \"doc1\", - \"_revisions\": {\"start\": 1, \"ids\": [\"12345\", \"67890\"]}, - \"content\": \"updated\", \"_rev\": \"" ++ ?b2l(Rev1) ++ "\"}]}", - {ok, _, _, _} = test_request:post(Url ++ "/_bulk_docs/", - [?CONTENT_JSON, ?AUTH], NewDoc), + NewDoc = + "{\"new_edits\": false, \"docs\": [{\"_id\": \"doc1\",\n" + " \"_revisions\": {\"start\": 1, \"ids\": [\"12345\", \"67890\"]},\n" + " \"content\": \"updated\", \"_rev\": \"" ++ ?b2l(Rev1) ++ "\"}]}", + {ok, _, _, _} = test_request:post( + Url ++ "/_bulk_docs/", + [?CONTENT_JSON, ?AUTH], + NewDoc + ), {ok, _, _, _Body2} = create_doc(Url, "doc2", "content2"), {ok, _, _, Body3} = create_doc(Url, "doc3", "content3"), {Json3} = ?JSON_DECODE(Body3), Rev3 = couch_util:get_value(<<"rev">>, Json3, undefined), - - IdsRevsEJson = {[ - {<<"doc1">>, [Rev1]}, % partial purge - {<<"doc2">>, [Rev3, Rev1]}, % correct format, but invalid rev - {<<"doc3">>, [Rev3]} % correct format and rev - ]}, + IdsRevsEJson = + {[ + % partial purge + {<<"doc1">>, [Rev1]}, + % correct format, but invalid rev + {<<"doc2">>, [Rev3, Rev1]}, + % correct format and rev + {<<"doc3">>, [Rev3]} + ]}, IdsRevs = binary_to_list(?JSON_ENCODE(IdsRevsEJson)), - {ok, Status, _, Body4} = test_request:post(Url ++ "/_purge/", - [?CONTENT_JSON, ?AUTH], IdsRevs), + {ok, Status, _, Body4} = test_request:post( + Url ++ "/_purge/", + [?CONTENT_JSON, ?AUTH], + IdsRevs + ), ResultJson = ?JSON_DECODE(Body4), ?assert(Status =:= 201 orelse Status =:= 202), ?assertEqual( {[ {<<"purge_seq">>, null}, - {<<"purged">>, {[ - {<<"doc1">>, [Rev1]}, - {<<"doc2">>, []}, - {<<"doc3">>, [Rev3]} - ]}} + {<<"purged">>, + {[ + {<<"doc1">>, [Rev1]}, + {<<"doc2">>, []}, + {<<"doc3">>, [Rev3]} + ]}} ]}, ResultJson ), - {ok, Status2, _, Body5} = test_request:get(Url - ++ "/doc1/", [?AUTH]), + {ok, Status2, _, Body5} = test_request:get( + Url ++ + "/doc1/", + [?AUTH] + ), {Json5} = ?JSON_DECODE(Body5), Content = couch_util:get_value(<<"content">>, Json5, undefined), ?assertEqual(<<"updated">>, Content), ?assert(Status2 =:= 200) end). - test_overmany_ids_or_revs_purge_request(Url) -> ?_test(begin {ok, _, _, Body} = create_doc(Url, "doc1"), {Json} = ?JSON_DECODE(Body), Rev1 = couch_util:get_value(<<"rev">>, Json, undefined), - NewDoc = "{\"new_edits\": false, \"docs\": [{\"_id\": \"doc1\", - \"_revisions\": {\"start\": 1, \"ids\": [\"12345\", \"67890\"]}, - \"content\": \"updated\", \"_rev\": \"" ++ ?b2l(Rev1) ++ "\"}]}", - {ok, _, _, _} = test_request:post(Url ++ "/_bulk_docs/", - [?CONTENT_JSON, ?AUTH], NewDoc), + NewDoc = + "{\"new_edits\": false, \"docs\": [{\"_id\": \"doc1\",\n" + " \"_revisions\": {\"start\": 1, \"ids\": [\"12345\", \"67890\"]},\n" + " \"content\": \"updated\", \"_rev\": \"" ++ ?b2l(Rev1) ++ "\"}]}", + {ok, _, _, _} = test_request:post( + Url ++ "/_bulk_docs/", + [?CONTENT_JSON, ?AUTH], + NewDoc + ), {ok, _, _, _Body2} = create_doc(Url, "doc2", "content2"), {ok, _, _, Body3} = create_doc(Url, "doc3", "content3"), {Json3} = ?JSON_DECODE(Body3), Rev3 = couch_util:get_value(<<"rev">>, Json3, undefined), - IdsRevsEJson = {[ - {<<"doc1">>, [Rev1]}, % partial purge - {<<"doc2">>, [Rev3, Rev1]}, % correct format, but invalid rev - {<<"doc3">>, [Rev3]} % correct format and rev - ]}, + IdsRevsEJson = + {[ + % partial purge + {<<"doc1">>, [Rev1]}, + % correct format, but invalid rev + {<<"doc2">>, [Rev3, Rev1]}, + % correct format and rev + {<<"doc3">>, [Rev3]} + ]}, IdsRevs = binary_to_list(?JSON_ENCODE(IdsRevsEJson)), % Ids larger than expected config:set("purge", "max_document_id_number", "1"), - {ok, Status, _, Body4} = test_request:post(Url ++ "/_purge/", - [?CONTENT_JSON, ?AUTH], IdsRevs), + {ok, Status, _, Body4} = test_request:post( + Url ++ "/_purge/", + [?CONTENT_JSON, ?AUTH], + IdsRevs + ), config:delete("purge", "max_document_id_number"), ResultJson = ?JSON_DECODE(Body4), ?assertEqual(400, Status), - ?assertMatch({[ - {<<"error">>,<<"bad_request">>}, - {<<"reason">>,<<"Exceeded maximum number of documents.">>}]}, - ResultJson), + ?assertMatch( + {[ + {<<"error">>, <<"bad_request">>}, + {<<"reason">>, <<"Exceeded maximum number of documents.">>} + ]}, + ResultJson + ), % Revs larger than expected config:set("purge", "max_revisions_number", "1"), - {ok, Status2, _, Body5} = test_request:post(Url ++ "/_purge/", - [?CONTENT_JSON, ?AUTH], IdsRevs), + {ok, Status2, _, Body5} = test_request:post( + Url ++ "/_purge/", + [?CONTENT_JSON, ?AUTH], + IdsRevs + ), config:delete("purge", "max_revisions_number"), ResultJson2 = ?JSON_DECODE(Body5), ?assertEqual(400, Status2), - ?assertMatch({[ - {<<"error">>,<<"bad_request">>}, - {<<"reason">>,<<"Exceeded maximum number of revisions.">>}]}, - ResultJson2) + ?assertMatch( + {[ + {<<"error">>, <<"bad_request">>}, + {<<"reason">>, <<"Exceeded maximum number of revisions.">>} + ]}, + ResultJson2 + ) end). - test_exceed_limits_on_purge_infos(Url) -> ?_test(begin - {ok, Status1, _, _} = test_request:put(Url ++ "/_purged_infos_limit/", - [?CONTENT_JSON, ?AUTH], "2"), + {ok, Status1, _, _} = test_request:put( + Url ++ "/_purged_infos_limit/", + [?CONTENT_JSON, ?AUTH], + "2" + ), ?assert(Status1 =:= 200), {ok, _, _, Body} = create_doc(Url, "doc1"), @@ -355,53 +426,66 @@ test_exceed_limits_on_purge_infos(Url) -> {Json3} = ?JSON_DECODE(Body3), Rev3 = couch_util:get_value(<<"rev">>, Json3, undefined), - IdsRevsEJson = {[ - {<<"doc1">>, [Rev1]}, - {<<"doc2">>, [Rev2]}, - {<<"doc3">>, [Rev3]} - ]}, + IdsRevsEJson = + {[ + {<<"doc1">>, [Rev1]}, + {<<"doc2">>, [Rev2]}, + {<<"doc3">>, [Rev3]} + ]}, IdsRevs = binary_to_list(?JSON_ENCODE(IdsRevsEJson)), - {ok, Status2, _, ResultBody} = test_request:post(Url ++ "/_purge/", - [?CONTENT_JSON, ?AUTH], IdsRevs), + {ok, Status2, _, ResultBody} = test_request:post( + Url ++ "/_purge/", + [?CONTENT_JSON, ?AUTH], + IdsRevs + ), ResultJson = ?JSON_DECODE(ResultBody), ?assert(Status2 =:= 201 orelse Status2 =:= 202), ?assertEqual( {[ {<<"purge_seq">>, null}, - {<<"purged">>, {[ - {<<"doc1">>, [Rev1]}, - {<<"doc2">>, [Rev2]}, - {<<"doc3">>, [Rev3]} - ]}} + {<<"purged">>, + {[ + {<<"doc1">>, [Rev1]}, + {<<"doc2">>, [Rev2]}, + {<<"doc3">>, [Rev3]} + ]}} ]}, ResultJson ) - end). - should_error_set_purged_docs_limit_to0(Url) -> ?_test(begin - {ok, Status, _, _} = test_request:put(Url ++ "/_purged_infos_limit/", - [?CONTENT_JSON, ?AUTH], "0"), + {ok, Status, _, _} = test_request:put( + Url ++ "/_purged_infos_limit/", + [?CONTENT_JSON, ?AUTH], + "0" + ), ?assert(Status =:= 400) end). - test_timeout_set_purged_infos_limit(Url) -> ?_test(begin meck:new(fabric, [passthrough]), meck:expect(fabric, set_purge_infos_limit, fun(_, _, _) -> - {error, timeout} end), - {ok, Status, _, ResultBody} = test_request:put(Url - ++ "/_purged_infos_limit/", [?CONTENT_JSON, ?AUTH], "2"), + {error, timeout} + end), + {ok, Status, _, ResultBody} = test_request:put( + Url ++ + "/_purged_infos_limit/", + [?CONTENT_JSON, ?AUTH], + "2" + ), meck:unload(fabric), ResultJson = ?JSON_DECODE(ResultBody), ?assert(Status =:= 500), - ?assertMatch({[ - {<<"error">>,<<"error">>}, - {<<"reason">>,<<"timeout">>}]}, - ResultJson) + ?assertMatch( + {[ + {<<"error">>, <<"error">>}, + {<<"reason">>, <<"timeout">>} + ]}, + ResultJson + ) end). diff --git a/src/chttpd/test/eunit/chttpd_security_tests.erl b/src/chttpd/test/eunit/chttpd_security_tests.erl index 0bea9dbcd..d8a39ffc8 100644 --- a/src/chttpd/test/eunit/chttpd_security_tests.erl +++ b/src/chttpd/test/eunit/chttpd_security_tests.erl @@ -27,8 +27,6 @@ -define(TEST_ADMIN_PASS, "test_admin_pass"). -define(TEST_ADMIN_AUTH, {basic_auth, {?TEST_ADMIN, ?TEST_ADMIN_PASS}}). - - -define(CONTENT_JSON, {"Content-Type", "application/json"}). -define(FIXTURE_TXT, ?ABS_PATH(?FILE)). @@ -48,30 +46,33 @@ setup() -> create_db(UsersUrl), create_db(Url), create_design_doc(Url), - create_user(UsersUrl,?TEST_MEMBER,?TEST_MEMBER_PASS,[<<?TEST_MEMBER>>]), - create_user(UsersUrl,?TEST_ADMIN,?TEST_ADMIN_PASS,[<<?TEST_ADMIN>>]), + create_user(UsersUrl, ?TEST_MEMBER, ?TEST_MEMBER_PASS, [<<?TEST_MEMBER>>]), + create_user(UsersUrl, ?TEST_ADMIN, ?TEST_ADMIN_PASS, [<<?TEST_ADMIN>>]), set_security(Url), [Url, UsersUrl]. -teardown([Url,UsersUrl]) -> +teardown([Url, UsersUrl]) -> delete_db(Url), delete_db(UsersUrl), - ok = config:delete("admins", ?USER, _Persist=false). + ok = config:delete("admins", ?USER, _Persist = false). create_db(Url) -> {ok, Status, _, _} = test_request:put(Url, [?CONTENT_JSON, ?AUTH], "{}"), ?assert(Status =:= 201 orelse Status =:= 202). create_design_doc(Url) -> - {ok, Status, _, _} = test_request:put(lists:concat([Url, '/_design/test']), [?CONTENT_JSON, ?AUTH], - "{\"id\":\"_design/test\"}"), + {ok, Status, _, _} = test_request:put( + lists:concat([Url, '/_design/test']), + [?CONTENT_JSON, ?AUTH], + "{\"id\":\"_design/test\"}" + ), ?assert(Status =:= 201 orelse Status =:= 202). set_security(Url) -> SecurityUrl = lists:concat([Url, "/_security"]), SecurityProperties = [ - {<<"admins">>,{[{<<"roles">>,[<<?TEST_ADMIN>>]}]}}, - {<<"members">>,{[{<<"roles">>,[<<?TEST_MEMBER>>]}]}} + {<<"admins">>, {[{<<"roles">>, [<<?TEST_ADMIN>>]}]}}, + {<<"members">>, {[{<<"roles">>, [<<?TEST_MEMBER>>]}]}} ], Body = jiffy:encode({SecurityProperties}), @@ -82,24 +83,27 @@ delete_db(Url) -> {ok, 200, _, _} = test_request:delete(Url, [?AUTH]). create_user(UsersUrl, Name, Password, Roles) -> - - Body = "{\"name\":\"" ++ Name ++ - "\",\"type\":\"user\",\"roles\":" ++ erlang:binary_to_list(jiffy:encode(Roles)) ++ ",\"password\":\"" ++ Password ++"\"}", + Body = + "{\"name\":\"" ++ Name ++ + "\",\"type\":\"user\",\"roles\":" ++ erlang:binary_to_list(jiffy:encode(Roles)) ++ + ",\"password\":\"" ++ Password ++ "\"}", Url = lists:concat([ - UsersUrl, "/org.couchdb.user:", Name]), + UsersUrl, "/org.couchdb.user:", Name + ]), {ok, 201, _, _} = test_request:put(Url, [?CONTENT_JSON, ?AUTH], Body). - all_test_() -> { "chttpd security tests", { setup, - fun chttpd_test_util:start_couch/0, fun chttpd_test_util:stop_couch/1, + fun chttpd_test_util:start_couch/0, + fun chttpd_test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_allow_admin_db_compaction/1, fun should_allow_valid_password_to_create_user/1, @@ -127,10 +131,12 @@ security_object_validate_test_() -> "chttpd security object validate tests", { setup, - fun chttpd_test_util:start_couch/0, fun chttpd_test_util:stop_couch/1, + fun chttpd_test_util:start_couch/0, + fun chttpd_test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_return_ok_for_sec_obj_with_roles/1, fun should_return_ok_for_sec_obj_with_names/1, @@ -144,263 +150,372 @@ security_object_validate_test_() -> } }. -should_allow_admin_db_compaction([Url,_UsersUrl]) -> - ?_assertEqual(true, +should_allow_admin_db_compaction([Url, _UsersUrl]) -> + ?_assertEqual( + true, begin - {ok, _, _, ResultBody} = test_request:post(Url ++ "/_compact", - [?CONTENT_JSON, ?AUTH], ""), + {ok, _, _, ResultBody} = test_request:post( + Url ++ "/_compact", + [?CONTENT_JSON, ?AUTH], + "" + ), ResultJson = ?JSON_DECODE(ResultBody), {InnerJson} = ResultJson, couch_util:get_value(<<"ok">>, InnerJson, undefined) - end). + end + ). should_allow_valid_password_to_create_user([_Url, UsersUrl]) -> - UserDoc = "{\"_id\": \"org.couchdb.user:foo\", \"name\": \"foo\", - \"type\": \"user\", \"roles\": [], \"password\": \"bar\"}", - {ok, _, _, ResultBody} = test_request:post(UsersUrl, - [?CONTENT_JSON, ?AUTH], UserDoc), + UserDoc = + "{\"_id\": \"org.couchdb.user:foo\", \"name\": \"foo\",\n" + " \"type\": \"user\", \"roles\": [], \"password\": \"bar\"}", + {ok, _, _, ResultBody} = test_request:post( + UsersUrl, + [?CONTENT_JSON, ?AUTH], + UserDoc + ), ResultJson = ?JSON_DECODE(ResultBody), {InnerJson} = ResultJson, ?_assertEqual(true, couch_util:get_value(<<"ok">>, InnerJson)). should_disallow_invalid_password_to_create_user([_Url, UsersUrl]) -> - UserDoc = "{\"_id\": \"org.couchdb.user:foo\", \"name\": \"foo\", - \"type\": \"user\", \"roles\": [], \"password\": 123}", - {ok, _, _, ResultBody} = test_request:post(UsersUrl, - [?CONTENT_JSON, ?AUTH], UserDoc), + UserDoc = + "{\"_id\": \"org.couchdb.user:foo\", \"name\": \"foo\",\n" + " \"type\": \"user\", \"roles\": [], \"password\": 123}", + {ok, _, _, ResultBody} = test_request:post( + UsersUrl, + [?CONTENT_JSON, ?AUTH], + UserDoc + ), ResultJson = ?JSON_DECODE(ResultBody), {InnerJson} = ResultJson, ErrType = couch_util:get_value(<<"error">>, InnerJson), ?_assertEqual(<<"forbidden">>, ErrType). -should_disallow_anonymous_db_compaction([Url,_UsersUrl]) -> - {ok, _, _, ResultBody} = test_request:post(Url ++ "/_compact", - [?CONTENT_JSON], ""), +should_disallow_anonymous_db_compaction([Url, _UsersUrl]) -> + {ok, _, _, ResultBody} = test_request:post( + Url ++ "/_compact", + [?CONTENT_JSON], + "" + ), ResultJson = ?JSON_DECODE(ResultBody), {InnerJson} = ResultJson, ErrType = couch_util:get_value(<<"error">>, InnerJson), - ?_assertEqual(<<"unauthorized">>,ErrType). + ?_assertEqual(<<"unauthorized">>, ErrType). -should_disallow_db_member_db_compaction([Url,_UsersUrl]) -> - {ok, _, _, ResultBody} = test_request:post(Url ++ "/_compact", - [?CONTENT_JSON, ?TEST_MEMBER_AUTH], ""), +should_disallow_db_member_db_compaction([Url, _UsersUrl]) -> + {ok, _, _, ResultBody} = test_request:post( + Url ++ "/_compact", + [?CONTENT_JSON, ?TEST_MEMBER_AUTH], + "" + ), ResultJson = ?JSON_DECODE(ResultBody), {InnerJson} = ResultJson, ErrType = couch_util:get_value(<<"error">>, InnerJson), - ?_assertEqual(<<"unauthorized">>,ErrType). + ?_assertEqual(<<"unauthorized">>, ErrType). -should_allow_db_admin_db_compaction([Url,_UsersUrl]) -> - ?_assertEqual(true, +should_allow_db_admin_db_compaction([Url, _UsersUrl]) -> + ?_assertEqual( + true, begin - {ok, _, _, ResultBody} = test_request:post(Url ++ "/_compact", - [?CONTENT_JSON, ?TEST_ADMIN_AUTH], ""), + {ok, _, _, ResultBody} = test_request:post( + Url ++ "/_compact", + [?CONTENT_JSON, ?TEST_ADMIN_AUTH], + "" + ), ResultJson = ?JSON_DECODE(ResultBody), {InnerJson} = ResultJson, couch_util:get_value(<<"ok">>, InnerJson, undefined) - end). + end + ). -should_allow_admin_view_compaction([Url,_UsersUrl]) -> - ?_assertEqual(true, +should_allow_admin_view_compaction([Url, _UsersUrl]) -> + ?_assertEqual( + true, begin - {ok, _, _, ResultBody} = test_request:post(Url ++ "/_compact/test", - [?CONTENT_JSON, ?AUTH], ""), + {ok, _, _, ResultBody} = test_request:post( + Url ++ "/_compact/test", + [?CONTENT_JSON, ?AUTH], + "" + ), ResultJson = ?JSON_DECODE(ResultBody), {InnerJson} = ResultJson, couch_util:get_value(<<"ok">>, InnerJson, undefined) - end). - -should_disallow_anonymous_view_compaction([Url,_UsersUrl]) -> - {ok, _, _, ResultBody} = test_request:post(Url ++ "/_compact/test", - [?CONTENT_JSON], ""), + end + ). + +should_disallow_anonymous_view_compaction([Url, _UsersUrl]) -> + {ok, _, _, ResultBody} = test_request:post( + Url ++ "/_compact/test", + [?CONTENT_JSON], + "" + ), ResultJson = ?JSON_DECODE(ResultBody), {InnerJson} = ResultJson, ErrType = couch_util:get_value(<<"error">>, InnerJson), - ?_assertEqual(<<"unauthorized">>,ErrType). + ?_assertEqual(<<"unauthorized">>, ErrType). -should_allow_admin_db_view_cleanup([Url,_UsersUrl]) -> - ?_assertEqual(true, +should_allow_admin_db_view_cleanup([Url, _UsersUrl]) -> + ?_assertEqual( + true, begin - {ok, _, _, ResultBody} = test_request:post(Url ++ "/_view_cleanup", - [?CONTENT_JSON, ?AUTH], ""), + {ok, _, _, ResultBody} = test_request:post( + Url ++ "/_view_cleanup", + [?CONTENT_JSON, ?AUTH], + "" + ), ResultJson = ?JSON_DECODE(ResultBody), {InnerJson} = ResultJson, couch_util:get_value(<<"ok">>, InnerJson, undefined) - end). - -should_disallow_anonymous_db_view_cleanup([Url,_UsersUrl]) -> - {ok, _, _, ResultBody} = test_request:post(Url ++ "/_view_cleanup", - [?CONTENT_JSON], ""), + end + ). + +should_disallow_anonymous_db_view_cleanup([Url, _UsersUrl]) -> + {ok, _, _, ResultBody} = test_request:post( + Url ++ "/_view_cleanup", + [?CONTENT_JSON], + "" + ), ResultJson = ?JSON_DECODE(ResultBody), {InnerJson} = ResultJson, ErrType = couch_util:get_value(<<"error">>, InnerJson), ?_assertEqual(<<"unauthorized">>, ErrType). -should_allow_admin_purge([Url,_UsersUrl]) -> - ?_assertEqual(null, +should_allow_admin_purge([Url, _UsersUrl]) -> + ?_assertEqual( + null, begin IdsRevs = "{}", - {ok, _, _, ResultBody} = test_request:post(Url ++ "/_purge", - [?CONTENT_JSON, ?AUTH], IdsRevs), + {ok, _, _, ResultBody} = test_request:post( + Url ++ "/_purge", + [?CONTENT_JSON, ?AUTH], + IdsRevs + ), ResultJson = ?JSON_DECODE(ResultBody), {InnerJson} = ResultJson, couch_util:get_value(<<"purge_seq">>, InnerJson, undefined) - end). - -should_disallow_anonymous_purge([Url,_UsersUrl]) -> - {ok, _, _, ResultBody} = test_request:post(Url ++ "/_purge", - [?CONTENT_JSON], ""), + end + ). + +should_disallow_anonymous_purge([Url, _UsersUrl]) -> + {ok, _, _, ResultBody} = test_request:post( + Url ++ "/_purge", + [?CONTENT_JSON], + "" + ), ResultJson = ?JSON_DECODE(ResultBody), {InnerJson} = ResultJson, ErrType = couch_util:get_value(<<"error">>, InnerJson), ?_assertEqual(<<"unauthorized">>, ErrType). -should_disallow_db_member_purge([Url,_UsersUrl]) -> - {ok, _, _, ResultBody} = test_request:post(Url ++ "/_purge", - [?CONTENT_JSON, ?TEST_MEMBER_AUTH], ""), +should_disallow_db_member_purge([Url, _UsersUrl]) -> + {ok, _, _, ResultBody} = test_request:post( + Url ++ "/_purge", + [?CONTENT_JSON, ?TEST_MEMBER_AUTH], + "" + ), ResultJson = ?JSON_DECODE(ResultBody), {InnerJson} = ResultJson, ErrType = couch_util:get_value(<<"error">>, InnerJson), - ?_assertEqual(<<"unauthorized">>,ErrType). + ?_assertEqual(<<"unauthorized">>, ErrType). -should_allow_admin_purged_infos_limit([Url,_UsersUrl]) -> - ?_assertEqual(true, +should_allow_admin_purged_infos_limit([Url, _UsersUrl]) -> + ?_assertEqual( + true, begin - {ok, _, _, ResultBody} = test_request:put(Url - ++ "/_purged_infos_limit/", [?CONTENT_JSON, ?AUTH], "2"), + {ok, _, _, ResultBody} = test_request:put( + Url ++ + "/_purged_infos_limit/", + [?CONTENT_JSON, ?AUTH], + "2" + ), ResultJson = ?JSON_DECODE(ResultBody), {InnerJson} = ResultJson, couch_util:get_value(<<"ok">>, InnerJson, undefined) - end). - -should_disallow_anonymous_purged_infos_limit([Url,_UsersUrl]) -> - {ok, _, _, ResultBody} = test_request:put(Url ++ "/_purged_infos_limit/", - [?CONTENT_JSON, ?TEST_MEMBER_AUTH], "2"), + end + ). + +should_disallow_anonymous_purged_infos_limit([Url, _UsersUrl]) -> + {ok, _, _, ResultBody} = test_request:put( + Url ++ "/_purged_infos_limit/", + [?CONTENT_JSON, ?TEST_MEMBER_AUTH], + "2" + ), ResultJson = ?JSON_DECODE(ResultBody), {InnerJson} = ResultJson, ErrType = couch_util:get_value(<<"error">>, InnerJson), ?_assertEqual(<<"unauthorized">>, ErrType). -should_disallow_db_member_purged_infos_limit([Url,_UsersUrl]) -> - {ok, _, _, ResultBody} = test_request:put(Url ++ "/_purged_infos_limit/", - [?CONTENT_JSON, ?TEST_MEMBER_AUTH], "2"), +should_disallow_db_member_purged_infos_limit([Url, _UsersUrl]) -> + {ok, _, _, ResultBody} = test_request:put( + Url ++ "/_purged_infos_limit/", + [?CONTENT_JSON, ?TEST_MEMBER_AUTH], + "2" + ), ResultJson = ?JSON_DECODE(ResultBody), {InnerJson} = ResultJson, ErrType = couch_util:get_value(<<"error">>, InnerJson), - ?_assertEqual(<<"unauthorized">>,ErrType). + ?_assertEqual(<<"unauthorized">>, ErrType). -should_return_ok_for_sec_obj_with_roles([Url,_UsersUrl]) -> +should_return_ok_for_sec_obj_with_roles([Url, _UsersUrl]) -> SecurityUrl = lists:concat([Url, "/_security"]), SecurityProperties = [ - {<<"admins">>,{[{<<"roles">>,[<<?TEST_ADMIN>>]}]}}, - {<<"members">>,{[{<<"roles">>,[<<?TEST_MEMBER>>]}]}} + {<<"admins">>, {[{<<"roles">>, [<<?TEST_ADMIN>>]}]}}, + {<<"members">>, {[{<<"roles">>, [<<?TEST_MEMBER>>]}]}} ], Body = jiffy:encode({SecurityProperties}), - {ok, Status, _, _} = test_request:put(SecurityUrl, - [?CONTENT_JSON, ?AUTH], Body), + {ok, Status, _, _} = test_request:put( + SecurityUrl, + [?CONTENT_JSON, ?AUTH], + Body + ), ?_assertEqual(200, Status). -should_return_ok_for_sec_obj_with_names([Url,_UsersUrl]) -> +should_return_ok_for_sec_obj_with_names([Url, _UsersUrl]) -> SecurityUrl = lists:concat([Url, "/_security"]), SecurityProperties = [ - {<<"admins">>,{[{<<"names">>,[<<?TEST_ADMIN>>]}]}}, - {<<"members">>,{[{<<"names">>,[<<?TEST_MEMBER>>]}]}} + {<<"admins">>, {[{<<"names">>, [<<?TEST_ADMIN>>]}]}}, + {<<"members">>, {[{<<"names">>, [<<?TEST_MEMBER>>]}]}} ], Body = jiffy:encode({SecurityProperties}), - {ok, Status, _, _} = test_request:put(SecurityUrl, - [?CONTENT_JSON, ?AUTH], Body), + {ok, Status, _, _} = test_request:put( + SecurityUrl, + [?CONTENT_JSON, ?AUTH], + Body + ), ?_assertEqual(200, Status). -should_return_ok_for_sec_obj_with_roles_and_names([Url,_UsersUrl]) -> +should_return_ok_for_sec_obj_with_roles_and_names([Url, _UsersUrl]) -> SecurityUrl = lists:concat([Url, "/_security"]), SecurityProperties = [ - {<<"admins">>, {[{<<"names">>,[<<?TEST_ADMIN>>]}, - {<<"roles">>,[<<?TEST_ADMIN>>]}]}}, - {<<"members">>,{[{<<"names">>,[<<?TEST_MEMBER>>]}, - {<<"roles">>,[<<?TEST_MEMBER>>]}]}} + {<<"admins">>, + {[ + {<<"names">>, [<<?TEST_ADMIN>>]}, + {<<"roles">>, [<<?TEST_ADMIN>>]} + ]}}, + {<<"members">>, + {[ + {<<"names">>, [<<?TEST_MEMBER>>]}, + {<<"roles">>, [<<?TEST_MEMBER>>]} + ]}} ], Body = jiffy:encode({SecurityProperties}), - {ok, Status, _, _} = test_request:put(SecurityUrl, - [?CONTENT_JSON, ?AUTH], Body), + {ok, Status, _, _} = test_request:put( + SecurityUrl, + [?CONTENT_JSON, ?AUTH], + Body + ), ?_assertEqual(200, Status). should_return_error_for_sec_obj_with_incorrect_roles_and_names( - [Url,_UsersUrl]) -> + [Url, _UsersUrl] +) -> SecurityUrl = lists:concat([Url, "/_security"]), SecurityProperties = [ - {<<"admins">>,{[{<<"names">>,[123]}]}}, - {<<"members">>,{[{<<"roles">>,["foo"]}]}} + {<<"admins">>, {[{<<"names">>, [123]}]}}, + {<<"members">>, {[{<<"roles">>, ["foo"]}]}} ], Body = jiffy:encode({SecurityProperties}), - {ok, Status, _, RespBody} = test_request:put(SecurityUrl, - [?CONTENT_JSON, ?AUTH], Body), + {ok, Status, _, RespBody} = test_request:put( + SecurityUrl, + [?CONTENT_JSON, ?AUTH], + Body + ), ResultJson = ?JSON_DECODE(RespBody), [ ?_assertEqual(500, Status), - ?_assertEqual({[ - {<<"error">>,<<"error">>}, - {<<"reason">>,<<"no_majority">>} - ]}, ResultJson) + ?_assertEqual( + {[ + {<<"error">>, <<"error">>}, + {<<"reason">>, <<"no_majority">>} + ]}, + ResultJson + ) ]. -should_return_error_for_sec_obj_with_incorrect_roles([Url,_UsersUrl]) -> +should_return_error_for_sec_obj_with_incorrect_roles([Url, _UsersUrl]) -> SecurityUrl = lists:concat([Url, "/_security"]), SecurityProperties = [ - {<<"admins">>,{[{<<"roles">>,[?TEST_ADMIN]}]}}, - {<<"members">>,{[{<<"roles">>,[<<?TEST_MEMBER>>]}]}} + {<<"admins">>, {[{<<"roles">>, [?TEST_ADMIN]}]}}, + {<<"members">>, {[{<<"roles">>, [<<?TEST_MEMBER>>]}]}} ], Body = jiffy:encode({SecurityProperties}), - {ok, Status, _, RespBody} = test_request:put(SecurityUrl, - [?CONTENT_JSON, ?AUTH], Body), + {ok, Status, _, RespBody} = test_request:put( + SecurityUrl, + [?CONTENT_JSON, ?AUTH], + Body + ), ResultJson = ?JSON_DECODE(RespBody), [ ?_assertEqual(500, Status), - ?_assertEqual({[ - {<<"error">>,<<"error">>}, - {<<"reason">>,<<"no_majority">>} - ]}, ResultJson) + ?_assertEqual( + {[ + {<<"error">>, <<"error">>}, + {<<"reason">>, <<"no_majority">>} + ]}, + ResultJson + ) ]. -should_return_error_for_sec_obj_with_incorrect_names([Url,_UsersUrl]) -> +should_return_error_for_sec_obj_with_incorrect_names([Url, _UsersUrl]) -> SecurityUrl = lists:concat([Url, "/_security"]), SecurityProperties = [ - {<<"admins">>,{[{<<"names">>,[<<?TEST_ADMIN>>]}]}}, - {<<"members">>,{[{<<"names">>,[?TEST_MEMBER]}]}} + {<<"admins">>, {[{<<"names">>, [<<?TEST_ADMIN>>]}]}}, + {<<"members">>, {[{<<"names">>, [?TEST_MEMBER]}]}} ], Body = jiffy:encode({SecurityProperties}), - {ok, Status, _, RespBody} = test_request:put(SecurityUrl, - [?CONTENT_JSON, ?AUTH], Body), + {ok, Status, _, RespBody} = test_request:put( + SecurityUrl, + [?CONTENT_JSON, ?AUTH], + Body + ), ResultJson = ?JSON_DECODE(RespBody), [ ?_assertEqual(500, Status), - ?_assertEqual({[ - {<<"error">>,<<"error">>}, - {<<"reason">>,<<"no_majority">>} - ]}, ResultJson) + ?_assertEqual( + {[ + {<<"error">>, <<"error">>}, + {<<"reason">>, <<"no_majority">>} + ]}, + ResultJson + ) ]. -should_return_error_for_sec_obj_in_user_db([_,_UsersUrl]) -> +should_return_error_for_sec_obj_in_user_db([_, _UsersUrl]) -> SecurityUrl = lists:concat([_UsersUrl, "/_security"]), SecurityProperties = [ - {<<"admins">>, {[{<<"names">>,[<<?TEST_ADMIN>>]}, - {<<"roles">>,[<<?TEST_ADMIN>>]}]}}, - {<<"members">>,{[{<<"names">>,[<<?TEST_MEMBER>>]}, - {<<"roles">>,[<<?TEST_MEMBER>>]}]}} + {<<"admins">>, + {[ + {<<"names">>, [<<?TEST_ADMIN>>]}, + {<<"roles">>, [<<?TEST_ADMIN>>]} + ]}}, + {<<"members">>, + {[ + {<<"names">>, [<<?TEST_MEMBER>>]}, + {<<"roles">>, [<<?TEST_MEMBER>>]} + ]}} ], Body = jiffy:encode({SecurityProperties}), - {ok, Status, _, RespBody} = test_request:put(SecurityUrl, - [?CONTENT_JSON, ?AUTH], Body), + {ok, Status, _, RespBody} = test_request:put( + SecurityUrl, + [?CONTENT_JSON, ?AUTH], + Body + ), ResultJson = ?JSON_DECODE(RespBody), [ ?_assertEqual(403, Status), - ?_assertEqual({[ - {<<"error">>,<<"forbidden">>}, - {<<"reason">>,<<"You can't edit the security object of the user database.">>} - ]}, ResultJson) + ?_assertEqual( + {[ + {<<"error">>, <<"forbidden">>}, + {<<"reason">>, <<"You can't edit the security object of the user database.">>} + ]}, + ResultJson + ) ]. diff --git a/src/chttpd/test/eunit/chttpd_session_tests.erl b/src/chttpd/test/eunit/chttpd_session_tests.erl index 1e1fbf5e4..3d99e3b10 100644 --- a/src/chttpd/test/eunit/chttpd_session_tests.erl +++ b/src/chttpd/test/eunit/chttpd_session_tests.erl @@ -18,18 +18,15 @@ -define(USER, "chttpd_test_admin"). -define(PASS, "pass"). - setup() -> - ok = config:delete("chttpd_auth", "authentication_db", _Persist=false), + ok = config:delete("chttpd_auth", "authentication_db", _Persist = false), Hashed = couch_passwords:hash_admin_password(?PASS), - ok = config:set("admins", ?USER, binary_to_list(Hashed), _Persist=false), + ok = config:set("admins", ?USER, binary_to_list(Hashed), _Persist = false), root_url() ++ "/_session". - cleanup(_) -> - ok = config:delete("chttpd_auth", "authentication_db", _Persist=false), - ok = config:delete("admins", ?USER, _Persist=false). - + ok = config:delete("chttpd_auth", "authentication_db", _Persist = false), + ok = config:delete("admins", ?USER, _Persist = false). session_test_() -> { @@ -51,32 +48,32 @@ session_test_() -> } }. - session_authentication_db_absent(Url) -> - ok = config:delete("chttpd_auth", "authentication_db", _Persist=false), + ok = config:delete("chttpd_auth", "authentication_db", _Persist = false), ?assertThrow({not_found, _}, session_authentication_db(Url)). - session_authentication_db_present(Url) -> Name = "_users", ok = config:set("chttpd_auth", "authentication_db", Name, false), ?assertEqual(list_to_binary(Name), session_authentication_db(Url)). - session_authentication_gzip_request(Url) -> {ok, 200, _, Body} = test_request:request( post, Url, [{"Content-Type", "application/json"}, {"Content-Encoding", "gzip"}], - zlib:gzip(jiffy:encode({[{username, list_to_binary(?USER)}, {password, list_to_binary(?PASS)}]}))), + zlib:gzip( + jiffy:encode({[{username, list_to_binary(?USER)}, {password, list_to_binary(?PASS)}]}) + ) + ), {BodyJson} = jiffy:decode(Body), ?assert(lists:member({<<"name">>, list_to_binary(?USER)}, BodyJson)). session_authentication_db(Url) -> {ok, 200, _, Body} = test_request:get(Url, [{basic_auth, {?USER, ?PASS}}]), couch_util:get_nested_json_value( - jiffy:decode(Body), [<<"info">>, <<"authentication_db">>]). - + jiffy:decode(Body), [<<"info">>, <<"authentication_db">>] + ). root_url() -> Addr = config:get("chttpd", "bind_address", "127.0.0.1"), diff --git a/src/chttpd/test/eunit/chttpd_socket_buffer_size_test.erl b/src/chttpd/test/eunit/chttpd_socket_buffer_size_test.erl index 937880621..bde2c8512 100644 --- a/src/chttpd/test/eunit/chttpd_socket_buffer_size_test.erl +++ b/src/chttpd/test/eunit/chttpd_socket_buffer_size_test.erl @@ -20,26 +20,24 @@ -define(AUTH, {basic_auth, {?USER, ?PASS}}). -define(CONTENT_JSON, {"Content-Type", "application/json"}). - setup(SocketOpts) -> StartCtx = start_couch_with_cfg(SocketOpts), Db = ?tempdb(), create_db(url(Db)), {StartCtx, Db}. - teardown(_, {StartCtx, Db}) -> delete_db(url(Db)), - ok = config:delete("admins", ?USER, _Persist=false), + ok = config:delete("admins", ?USER, _Persist = false), test_util:stop_couch(StartCtx). - socket_buffer_size_test_() -> { "chttpd socket_buffer_size_test", { foreachx, - fun setup/1, fun teardown/2, + fun setup/1, + fun teardown/2, [ {"[{recbuf, undefined}]", fun default_buffer/2}, {"[{recbuf, 1024}]", fun small_recbuf/2}, @@ -48,31 +46,30 @@ socket_buffer_size_test_() -> } }. - small_recbuf(_, {_, Db}) -> - {timeout, 30, ?_test(begin - Id = data(2048), - Response = put_req(url(Db) ++ "/" ++ Id, "{}"), - ?assert(Response =:= 400 orelse Response =:= request_failed) - end)}. - + {timeout, 30, + ?_test(begin + Id = data(2048), + Response = put_req(url(Db) ++ "/" ++ Id, "{}"), + ?assert(Response =:= 400 orelse Response =:= request_failed) + end)}. small_buffer(_, {_, Db}) -> - {timeout, 30, ?_test(begin - Id = data(2048), - Response = put_req(url(Db) ++ "/" ++ Id, "{}"), - ?assert(Response =:= 400 orelse Response =:= request_failed) - end)}. - + {timeout, 30, + ?_test(begin + Id = data(2048), + Response = put_req(url(Db) ++ "/" ++ Id, "{}"), + ?assert(Response =:= 400 orelse Response =:= request_failed) + end)}. default_buffer(_, {_, Db}) -> - {timeout, 30, ?_test(begin - Id = data(7000), - Headers = [{"Blah", data(7000)}], - Status = put_req(url(Db) ++ "/" ++ Id, Headers, "{}"), - ?assert(Status =:= 201 orelse Status =:= 202) - end)}. - + {timeout, 30, + ?_test(begin + Id = data(7000), + Headers = [{"Blah", data(7000)}], + Status = put_req(url(Db) ++ "/" ++ Id, Headers, "{}"), + ?assert(Status =:= 201 orelse Status =:= 202) + end)}. % Helper functions @@ -81,24 +78,19 @@ url() -> Port = integer_to_list(mochiweb_socket_server:get(chttpd, port)), "http://" ++ Addr ++ ":" ++ Port. - url(Db) -> url() ++ "/" ++ ?b2l(Db). - create_db(Url) -> Status = put_req(Url ++ "?q=1&n=1", "{}"), ?assert(Status =:= 201 orelse Status =:= 202). - delete_db(Url) -> {ok, 200, _, _} = test_request:delete(Url, [?AUTH]). - put_req(Url, Body) -> put_req(Url, [], Body). - put_req(Url, Headers, Body) -> AllHeaders = Headers ++ [?CONTENT_JSON, ?AUTH], case test_request:put(Url, AllHeaders, Body) of @@ -106,11 +98,9 @@ put_req(Url, Headers, Body) -> {error, Error} -> Error end. - data(Size) -> string:copies("x", Size). - append_to_cfg_chain(Cfg) -> CfgDir = filename:dirname(lists:last(?CONFIG_CHAIN)), CfgFile = filename:join([CfgDir, "chttpd_socket_buffer_extra_cfg.ini"]), @@ -118,10 +108,9 @@ append_to_cfg_chain(Cfg) -> ok = file:write_file(CfgFile, CfgSect), ?CONFIG_CHAIN ++ [CfgFile]. - start_couch_with_cfg(Cfg) -> CfgChain = append_to_cfg_chain(Cfg), StartCtx = test_util:start_couch(CfgChain, [chttpd]), Hashed = couch_passwords:hash_admin_password(?PASS), - ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist=false), + ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist = false), StartCtx. diff --git a/src/chttpd/test/eunit/chttpd_util_test.erl b/src/chttpd/test/eunit/chttpd_util_test.erl index 41fe6cb23..4ad2b8b83 100644 --- a/src/chttpd/test/eunit/chttpd_util_test.erl +++ b/src/chttpd/test/eunit/chttpd_util_test.erl @@ -12,19 +12,24 @@ -module(chttpd_util_test). - -include_lib("couch/include/couch_eunit.hrl"). -include("chttpd_test.hrl"). - setup() -> - ok = lists:foreach(fun(Section) -> - ok = config_delete_all_keys(Section) - end, ["httpd", "chttpd", "couch_httpd_auth", "chttpd_auth"]), - - ok = config:set("httpd", "authentication_handlers", + ok = lists:foreach( + fun(Section) -> + ok = config_delete_all_keys(Section) + end, + ["httpd", "chttpd", "couch_httpd_auth", "chttpd_auth"] + ), + + ok = config:set( + "httpd", + "authentication_handlers", "{couch_httpd_auth, cookie_authentication_handler}, " - "{couch_httpd_auth, default_authentication_handler}", _Persist = false), + "{couch_httpd_auth, default_authentication_handler}", + _Persist = false + ), ok = config:set("httpd", "backlog", "512", _Persist = false), ok = config:set("chttpd", "require_valid_user", "false", _Persist = false), ok = config:set("httpd", "both_exist", "get_in_httpd", _Persist = false), @@ -36,7 +41,6 @@ setup() -> ok = config:set("couch_httpd_auth", "cha_only", "true", _Persist = false), ok = config:set("chttpd_auth", "ca_only", "1", _Persist = false). - teardown(_) -> ok = config:delete("httpd", "authentication_handlers", _Persist = false), ok = config:delete("httpd", "backlog", _Persist = false), @@ -50,12 +54,13 @@ teardown(_) -> ok = config:delete("couch_httpd_auth", "cha_only", _Persist = false), ok = config:delete("chttpd_auth", "ca_only", _Persist = false). - config_delete_all_keys(Section) -> - lists:foreach(fun({Key, _Val}) -> - ok = config:delete(Section, Key, _Persist = false) - end, config:get(Section)). - + lists:foreach( + fun({Key, _Val}) -> + ok = config:delete(Section, Key, _Persist = false) + end, + config:get(Section) + ). chttpd_util_config_test_() -> { @@ -78,13 +83,11 @@ chttpd_util_config_test_() -> } }. - test_chttpd_behavior(_) -> ?assertEqual("get_in_chttpd", chttpd_util:get_chttpd_config("both_exist")), ?assertEqual(1, chttpd_util:get_chttpd_config_integer("chttpd_only", 0)), ?assert(chttpd_util:get_chttpd_config_boolean("httpd_only", false)). - test_with_undefined_option(_) -> ?assertEqual(undefined, chttpd_util:get_chttpd_config("undefined_option")), ?assertEqual(abc, chttpd_util:get_chttpd_config("undefined_option", abc)), @@ -95,13 +98,11 @@ test_with_undefined_option(_) -> ?assert(chttpd_util:get_chttpd_config("undefined_option", true)), ?assertNot(chttpd_util:get_chttpd_config("undefined_option", false)). - test_auth_behavior(_) -> ?assertEqual("ca", chttpd_util:get_chttpd_auth_config("both_exist")), ?assertEqual(1, chttpd_util:get_chttpd_auth_config_integer("ca_only", 0)), ?assert(chttpd_util:get_chttpd_auth_config_boolean("cha_only", false)). - test_auth_with_undefined_option(_) -> ?assertEqual(undefined, chttpd_util:get_chttpd_auth_config("undefine")), ?assertEqual(abc, chttpd_util:get_chttpd_auth_config("undefine", abc)), diff --git a/src/chttpd/test/eunit/chttpd_view_test.erl b/src/chttpd/test/eunit/chttpd_view_test.erl index 4c224bb4e..ceff2a902 100644 --- a/src/chttpd/test/eunit/chttpd_view_test.erl +++ b/src/chttpd/test/eunit/chttpd_view_test.erl @@ -19,16 +19,19 @@ -define(PASS, "pass"). -define(AUTH, {basic_auth, {?USER, ?PASS}}). -define(CONTENT_JSON, {"Content-Type", "application/json"}). --define(DDOC, "{\"_id\": \"_design/bar\", \"views\": {\"baz\": - {\"map\": \"function(doc) {emit(doc._id, doc._id);}\"}}}"). +-define(DDOC, + "{\"_id\": \"_design/bar\", \"views\": {\"baz\":\n" + " {\"map\": \"function(doc) {emit(doc._id, doc._id);}\"}}}" +). -define(FIXTURE_TXT, ?ABS_PATH(?FILE)). -define(i2l(I), integer_to_list(I)). --define(TIMEOUT, 60). % seconds +% seconds +-define(TIMEOUT, 60). setup() -> Hashed = couch_passwords:hash_admin_password(?PASS), - ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist=false), + ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist = false), TmpDb = ?tempdb(), Addr = config:get("chttpd", "bind_address", "127.0.0.1"), Port = mochiweb_socket_server:get(chttpd, port), @@ -38,16 +41,18 @@ setup() -> teardown(Url) -> delete_db(Url), - ok = config:delete("admins", ?USER, _Persist=false). + ok = config:delete("admins", ?USER, _Persist = false). create_db(Url) -> {ok, Status, _, _} = test_request:put(Url, [?CONTENT_JSON, ?AUTH], "{}"), ?assert(Status =:= 201 orelse Status =:= 202). - create_doc(Url, Id) -> - test_request:put(Url ++ "/" ++ Id, - [?CONTENT_JSON, ?AUTH], "{\"mr\": \"rockoartischocko\"}"). + test_request:put( + Url ++ "/" ++ Id, + [?CONTENT_JSON, ?AUTH], + "{\"mr\": \"rockoartischocko\"}" + ). delete_db(Url) -> {ok, 200, _, _} = test_request:delete(Url, [?AUTH]). @@ -57,10 +62,12 @@ all_view_test_() -> "chttpd view tests", { setup, - fun chttpd_test_util:start_couch/0, fun chttpd_test_util:stop_couch/1, + fun chttpd_test_util:start_couch/0, + fun chttpd_test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_succeed_on_view_with_queries_keys/1, fun should_succeed_on_view_with_queries_limit_skip/1, @@ -70,55 +77,78 @@ all_view_test_() -> } }. - should_succeed_on_view_with_queries_keys(Url) -> - {timeout, ?TIMEOUT, ?_test(begin - [create_doc(Url, "testdoc" ++ ?i2l(I)) || I <- lists:seq(1, 10)], - {ok, _, _, _} = test_request:put(Url ++ "/_design/bar", - [?CONTENT_JSON, ?AUTH], ?DDOC), - QueryDoc = "{\"queries\": [{\"keys\": [ \"testdoc3\", - \"testdoc8\"]}]}", - {ok, _, _, RespBody} = test_request:post(Url ++ "/_design/bar/" - ++ "_view/baz/queries/", [?CONTENT_JSON, ?AUTH], QueryDoc), - {ResultJson} = ?JSON_DECODE(RespBody), - ResultJsonBody = couch_util:get_value(<<"results">>, ResultJson), - {InnerJson} = lists:nth(1, ResultJsonBody), - ?assertEqual(2, length(couch_util:get_value(<<"rows">>, InnerJson))) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + [create_doc(Url, "testdoc" ++ ?i2l(I)) || I <- lists:seq(1, 10)], + {ok, _, _, _} = test_request:put( + Url ++ "/_design/bar", + [?CONTENT_JSON, ?AUTH], + ?DDOC + ), + QueryDoc = + "{\"queries\": [{\"keys\": [ \"testdoc3\",\n" + " \"testdoc8\"]}]}", + {ok, _, _, RespBody} = test_request:post( + Url ++ "/_design/bar/" ++ + "_view/baz/queries/", + [?CONTENT_JSON, ?AUTH], + QueryDoc + ), + {ResultJson} = ?JSON_DECODE(RespBody), + ResultJsonBody = couch_util:get_value(<<"results">>, ResultJson), + {InnerJson} = lists:nth(1, ResultJsonBody), + ?assertEqual(2, length(couch_util:get_value(<<"rows">>, InnerJson))) + end)}. should_succeed_on_view_with_queries_limit_skip(Url) -> - {timeout, ?TIMEOUT, ?_test(begin - [create_doc(Url, "testdoc" ++ ?i2l(I)) || I <- lists:seq(1, 10)], - {ok, _, _, _} = test_request:put(Url ++ "/_design/bar", - [?CONTENT_JSON, ?AUTH], ?DDOC), - QueryDoc = "{\"queries\": [{\"limit\": 5, \"skip\": 2}]}", - {ok, RC, _, RespBody} = test_request:post(Url ++ "/_design/bar/" - ++ "_view/baz/queries/", [?CONTENT_JSON, ?AUTH], QueryDoc), - ?assertEqual(200, RC), - {ResultJson} = ?JSON_DECODE(RespBody), - ResultJsonBody = couch_util:get_value(<<"results">>, ResultJson), - {InnerJson} = lists:nth(1, ResultJsonBody), - ?assertEqual(2, couch_util:get_value(<<"offset">>, InnerJson)), - ?assertEqual(5, length(couch_util:get_value(<<"rows">>, InnerJson))) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + [create_doc(Url, "testdoc" ++ ?i2l(I)) || I <- lists:seq(1, 10)], + {ok, _, _, _} = test_request:put( + Url ++ "/_design/bar", + [?CONTENT_JSON, ?AUTH], + ?DDOC + ), + QueryDoc = "{\"queries\": [{\"limit\": 5, \"skip\": 2}]}", + {ok, RC, _, RespBody} = test_request:post( + Url ++ "/_design/bar/" ++ + "_view/baz/queries/", + [?CONTENT_JSON, ?AUTH], + QueryDoc + ), + ?assertEqual(200, RC), + {ResultJson} = ?JSON_DECODE(RespBody), + ResultJsonBody = couch_util:get_value(<<"results">>, ResultJson), + {InnerJson} = lists:nth(1, ResultJsonBody), + ?assertEqual(2, couch_util:get_value(<<"offset">>, InnerJson)), + ?assertEqual(5, length(couch_util:get_value(<<"rows">>, InnerJson))) + end)}. should_succeed_on_view_with_multiple_queries(Url) -> - {timeout, ?TIMEOUT, ?_test(begin - [create_doc(Url, "testdoc" ++ ?i2l(I)) || I <- lists:seq(1, 10)], - {ok, _, _, _} = test_request:put(Url ++ "/_design/bar", - [?CONTENT_JSON, ?AUTH], ?DDOC), - QueryDoc = "{\"queries\": [{\"keys\": [ \"testdoc3\", - \"testdoc8\"]}, {\"limit\": 5, \"skip\": 2}]}", - {ok, RC, _, RespBody} = test_request:post(Url ++ "/_design/bar/" - ++ "_view/baz/queries/", [?CONTENT_JSON, ?AUTH], QueryDoc), - ?assertEqual(200, RC), - {ResultJson} = ?JSON_DECODE(RespBody), - ResultJsonBody = couch_util:get_value(<<"results">>, ResultJson), - {InnerJson1} = lists:nth(1, ResultJsonBody), - ?assertEqual(2, length(couch_util:get_value(<<"rows">>, InnerJson1))), - {InnerJson2} = lists:nth(2, ResultJsonBody), - ?assertEqual(2, couch_util:get_value(<<"offset">>, InnerJson2)), - ?assertEqual(5, length(couch_util:get_value(<<"rows">>, InnerJson2))) - end)}. + {timeout, ?TIMEOUT, + ?_test(begin + [create_doc(Url, "testdoc" ++ ?i2l(I)) || I <- lists:seq(1, 10)], + {ok, _, _, _} = test_request:put( + Url ++ "/_design/bar", + [?CONTENT_JSON, ?AUTH], + ?DDOC + ), + QueryDoc = + "{\"queries\": [{\"keys\": [ \"testdoc3\",\n" + " \"testdoc8\"]}, {\"limit\": 5, \"skip\": 2}]}", + {ok, RC, _, RespBody} = test_request:post( + Url ++ "/_design/bar/" ++ + "_view/baz/queries/", + [?CONTENT_JSON, ?AUTH], + QueryDoc + ), + ?assertEqual(200, RC), + {ResultJson} = ?JSON_DECODE(RespBody), + ResultJsonBody = couch_util:get_value(<<"results">>, ResultJson), + {InnerJson1} = lists:nth(1, ResultJsonBody), + ?assertEqual(2, length(couch_util:get_value(<<"rows">>, InnerJson1))), + {InnerJson2} = lists:nth(2, ResultJsonBody), + ?assertEqual(2, couch_util:get_value(<<"offset">>, InnerJson2)), + ?assertEqual(5, length(couch_util:get_value(<<"rows">>, InnerJson2))) + end)}. diff --git a/src/chttpd/test/eunit/chttpd_welcome_test.erl b/src/chttpd/test/eunit/chttpd_welcome_test.erl index e427f4dff..7a24efb71 100644 --- a/src/chttpd/test/eunit/chttpd_welcome_test.erl +++ b/src/chttpd/test/eunit/chttpd_welcome_test.erl @@ -20,29 +20,28 @@ -define(AUTH, {basic_auth, {?USER, ?PASS}}). -define(CONTENT_JSON, {"Content-Type", "application/json"}). - setup() -> Hashed = couch_passwords:hash_admin_password(?PASS), - ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist=false), + ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist = false), Addr = config:get("chttpd", "bind_address", "127.0.0.1"), Port = mochiweb_socket_server:get(chttpd, port), Url = lists:concat(["http://", Addr, ":", Port, "/"]), Url. - teardown(_Url) -> - ok = config:delete("admins", ?USER, _Persist=false). - + ok = config:delete("admins", ?USER, _Persist = false). welcome_test_() -> { "chttpd welcome endpoint tests", { setup, - fun chttpd_test_util:start_couch/0, fun chttpd_test_util:stop_couch/1, + fun chttpd_test_util:start_couch/0, + fun chttpd_test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_have_version/1, fun should_have_features/1, @@ -69,7 +68,6 @@ should_have_uuid(Url) -> ?assert(is_list(Features)) end). - should_have_version(Url) -> ?_test(begin {ok, Status, _, Body} = test_request:get(Url, [?CONTENT_JSON, ?AUTH]), @@ -86,7 +84,6 @@ should_have_version(Url) -> ?assert(is_list(Features)) end). - should_have_features(Url) -> ?_test(begin config:enable_feature(snek), diff --git a/src/chttpd/test/eunit/chttpd_xframe_test.erl b/src/chttpd/test/eunit/chttpd_xframe_test.erl index f3e6165bb..ee2a0996b 100644 --- a/src/chttpd/test/eunit/chttpd_xframe_test.erl +++ b/src/chttpd/test/eunit/chttpd_xframe_test.erl @@ -1,6 +1,5 @@ -module(chttpd_xframe_test). - -include_lib("couch/include/couch_db.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -63,7 +62,6 @@ enabled_with_same_origin_test() -> Headers = chttpd_xframe_options:header(mock_request(), [], config_sameorigin()), ?assertEqual(Headers, [{"X-Frame-Options", "SAMEORIGIN"}]). - xframe_host_test_() -> { "xframe host tests", diff --git a/src/couch/src/couch.erl b/src/couch/src/couch.erl index 1c912ac2a..6952c16c8 100644 --- a/src/couch/src/couch.erl +++ b/src/couch/src/couch.erl @@ -18,7 +18,6 @@ restart/0 ]). - deps() -> [ sasl, @@ -32,7 +31,6 @@ deps() -> couch_log ]. - start() -> catch erlang:system_flag(scheduler_bind_type, default_bind), case start_apps(deps()) of @@ -42,26 +40,23 @@ start() -> throw(Else) end. - stop() -> application:stop(couch). - restart() -> init:restart(). - start_apps([]) -> ok; -start_apps([App|Rest]) -> +start_apps([App | Rest]) -> case application:start(App) of - ok -> - start_apps(Rest); - {error, {already_started, App}} -> - start_apps(Rest); - {error, _Reason} when App =:= public_key -> - % ignore on R12B5 - start_apps(Rest); - {error, _Reason} -> - {error, {app_would_not_start, App}} + ok -> + start_apps(Rest); + {error, {already_started, App}} -> + start_apps(Rest); + {error, _Reason} when App =:= public_key -> + % ignore on R12B5 + start_apps(Rest); + {error, _Reason} -> + {error, {app_would_not_start, App}} end. diff --git a/src/couch/src/couch_att.erl b/src/couch/src/couch_att.erl index 12ac4874c..b3b2f23eb 100644 --- a/src/couch/src/couch_att.erl +++ b/src/couch/src/couch_att.erl @@ -60,7 +60,6 @@ -include_lib("couch/include/couch_db.hrl"). - %% Legacy attachment record. This is going to be phased out by the new proplist %% based structure. It's needed for now to allow code to perform lazy upgrades %% while the patch is rolled out to the cluster. Attachments passed as records @@ -79,8 +78,13 @@ md5 = <<>> :: binary(), revpos = 0 :: non_neg_integer(), - data :: stub | follows | binary() | {any(), any()} | - {follows, pid(), reference()} | fun(() -> binary()), + data :: + stub + | follows + | binary() + | {any(), any()} + | {follows, pid(), reference()} + | fun(() -> binary()), %% Encoding of the attachment %% currently supported values are: @@ -90,7 +94,6 @@ encoding = identity :: identity | gzip }). - %% Extensible Attachment Type %% %% The following types describe the known properties for attachment fields @@ -99,57 +102,57 @@ %% be used by upgraded code. If you plan on operating on new data, please add %% an entry here as documentation. - %% The name of the attachment is also used as the mime-part name for file %% downloads. These must be unique per document. -type name_prop() :: {name, binary()}. - %% The mime type of the attachment. This does affect compression of certain %% attachments if the type is found to be configured as a compressable type. %% This is commonly reserved for text/* types but could include other custom %% cases as well. See definition and use of couch_util:compressable_att_type/1. -type type_prop() :: {type, binary()}. - %% The attachment length is similar to disk-length but ignores additional %% encoding that may have occurred. -type att_len_prop() :: {att_len, non_neg_integer()}. - %% The size of the attachment as stored in a disk stream. -type disk_len_prop() :: {disk_len, non_neg_integer()}. - %% This is a digest of the original attachment data as uploaded by the client. %% it's useful for checking validity of contents against other attachment data %% as well as quick digest computation of the enclosing document. -type md5_prop() :: {md5, binary()}. - -type revpos_prop() :: {revpos, 0}. - %% This field is currently overloaded with just about everything. The %% {any(), any()} type is just there until I have time to check the actual %% values expected. Over time this should be split into more than one property %% to allow simpler handling. -type data_prop() :: { - data, stub | follows | binary() | {any(), any()} | - {follows, pid(), reference()} | fun(() -> binary()) + data, + stub + | follows + | binary() + | {any(), any()} + | {follows, pid(), reference()} + | fun(() -> binary()) }. - %% We will occasionally compress our data. See type_prop() for more information %% on when this happens. -type encoding_prop() :: {encoding, identity | gzip}. - -type attachment() :: [ - name_prop() | type_prop() | - att_len_prop() | disk_len_prop() | - md5_prop() | revpos_prop() | - data_prop() | encoding_prop() + name_prop() + | type_prop() + | att_len_prop() + | disk_len_prop() + | md5_prop() + | revpos_prop() + | data_prop() + | encoding_prop() ]. -type disk_att_v1() :: { @@ -178,7 +181,7 @@ -type att() :: #att{} | attachment() | disk_att(). --define(GB, (1024*1024*1024)). +-define(GB, (1024 * 1024 * 1024)). new() -> %% We construct a record by default for compatability. This will be @@ -189,14 +192,13 @@ new() -> %% undefined. #att{}. - -spec new([{atom(), any()}]) -> att(). new(Props) -> store(Props, new()). - --spec fetch([atom()], att()) -> [any()]; - (atom(), att()) -> any(). +-spec fetch + ([atom()], att()) -> [any()]; + (atom(), att()) -> any(). fetch(Fields, Att) when is_list(Fields) -> [fetch(Field, Att) || Field <- Fields]; fetch(Field, Att) when is_list(Att) -> @@ -223,13 +225,15 @@ fetch(encoding, #att{encoding = Encoding}) -> fetch(_, _) -> undefined. - -spec store([{atom(), any()}], att()) -> att(). store(Props, Att0) -> - lists:foldl(fun({Field, Value}, Att) -> - store(Field, Value, Att) - end, Att0, Props). - + lists:foldl( + fun({Field, Value}, Att) -> + store(Field, Value, Att) + end, + Att0, + Props + ). -spec store(atom(), any(), att()) -> att(). store(Field, undefined, Att) when is_list(Att) -> @@ -255,17 +259,14 @@ store(encoding, Encoding, Att) -> store(Field, Value, Att) -> store(Field, Value, upgrade(Att)). - -spec transform(atom(), fun(), att()) -> att(). transform(Field, Fun, Att) -> NewValue = Fun(fetch(Field, Att)), store(Field, NewValue, Att). - is_stub(Att) -> stub == fetch(data, Att). - %% merge_stubs takes all stub attachments and replaces them with on disk %% attachments. It will return {missing, Name} if a stub isn't matched with %% an existing attachment on disk. If the revpos is supplied with the stub @@ -276,7 +277,6 @@ merge_stubs(MemAtts, DiskAtts) -> ), merge_stubs(MemAtts, OnDisk, []). - %% restore spec when R14 support is dropped %% -spec merge_stubs([att()], dict:dict(), [att()]) -> [att()]. merge_stubs([Att | Rest], OnDisk, Merged) -> @@ -305,23 +305,24 @@ merge_stubs([Att | Rest], OnDisk, Merged) -> merge_stubs([], _, Merged) -> {ok, lists:reverse(Merged)}. - size_info([]) -> {ok, []}; size_info(Atts) -> - Info = lists:map(fun(Att) -> - AttLen = fetch(att_len, Att), - case fetch(data, Att) of - {stream, StreamEngine} -> - {ok, SPos} = couch_stream:to_disk_term(StreamEngine), - {SPos, AttLen}; - {_, SPos} -> - {SPos, AttLen} - end - end, Atts), + Info = lists:map( + fun(Att) -> + AttLen = fetch(att_len, Att), + case fetch(data, Att) of + {stream, StreamEngine} -> + {ok, SPos} = couch_stream:to_disk_term(StreamEngine), + {SPos, AttLen}; + {_, SPos} -> + {SPos, AttLen} + end + end, + Atts + ), {ok, lists:usort(Info)}. - %% When converting an attachment to disk term format, attempt to stay with the %% old format when possible. This should help make the attachment lazy upgrade %% as safe as possible, avoiding the need for complicated disk versioning @@ -364,7 +365,6 @@ to_disk_term(Att) -> ), {list_to_tuple(lists:reverse(Base)), Extended}. - %% The new disk term format is a simple wrapper around the legacy format. Base %% properties will remain in a tuple while the new fields and possibly data from %% future extensions will be stored in a list of atom/value pairs. While this is @@ -372,45 +372,45 @@ to_disk_term(Att) -> %% compression to remove these sorts of common bits (block level compression %% with something like a shared dictionary that is checkpointed every now and %% then). -from_disk_term(StreamSrc, {Base, Extended}) - when is_tuple(Base), is_list(Extended) -> +from_disk_term(StreamSrc, {Base, Extended}) when + is_tuple(Base), is_list(Extended) +-> store(Extended, from_disk_term(StreamSrc, Base)); -from_disk_term(StreamSrc, {Name,Type,Sp,AttLen,DiskLen,RevPos,Md5,Enc}) -> +from_disk_term(StreamSrc, {Name, Type, Sp, AttLen, DiskLen, RevPos, Md5, Enc}) -> {ok, Stream} = open_stream(StreamSrc, Sp), #att{ - name=Name, - type=Type, - att_len=AttLen, - disk_len=DiskLen, - md5=Md5, - revpos=RevPos, - data={stream, Stream}, - encoding=upgrade_encoding(Enc) + name = Name, + type = Type, + att_len = AttLen, + disk_len = DiskLen, + md5 = Md5, + revpos = RevPos, + data = {stream, Stream}, + encoding = upgrade_encoding(Enc) }; -from_disk_term(StreamSrc, {Name,Type,Sp,AttLen,RevPos,Md5}) -> +from_disk_term(StreamSrc, {Name, Type, Sp, AttLen, RevPos, Md5}) -> {ok, Stream} = open_stream(StreamSrc, Sp), #att{ - name=Name, - type=Type, - att_len=AttLen, - disk_len=AttLen, - md5=Md5, - revpos=RevPos, - data={stream, Stream} + name = Name, + type = Type, + att_len = AttLen, + disk_len = AttLen, + md5 = Md5, + revpos = RevPos, + data = {stream, Stream} }; -from_disk_term(StreamSrc, {Name,{Type,Sp,AttLen}}) -> +from_disk_term(StreamSrc, {Name, {Type, Sp, AttLen}}) -> {ok, Stream} = open_stream(StreamSrc, Sp), #att{ - name=Name, - type=Type, - att_len=AttLen, - disk_len=AttLen, - md5= <<>>, - revpos=0, - data={stream, Stream} + name = Name, + type = Type, + att_len = AttLen, + disk_len = AttLen, + md5 = <<>>, + revpos = 0, + data = {stream, Stream} }. - %% from_json reads in embedded JSON attachments and creates usable attachment %% values. The attachment may be a stub, from_json(Name, Props) -> @@ -426,7 +426,6 @@ from_json(Name, Props) -> true -> inline_from_json(Att, Props) end. - stub_from_json(Att, Props) -> {DiskLen, EncodedLen, Encoding} = encoded_lengths_from_json(Props), Digest = digest_from_json(Props), @@ -434,21 +433,33 @@ stub_from_json(Att, Props) -> %% the revpos consistency check on stubs when it's not provided in the %% json object. See merge_stubs/3 for the stub check. RevPos = couch_util:get_value(<<"revpos">>, Props), - store([ - {md5, Digest}, {revpos, RevPos}, {data, stub}, {disk_len, DiskLen}, - {att_len, EncodedLen}, {encoding, Encoding} - ], Att). - + store( + [ + {md5, Digest}, + {revpos, RevPos}, + {data, stub}, + {disk_len, DiskLen}, + {att_len, EncodedLen}, + {encoding, Encoding} + ], + Att + ). follow_from_json(Att, Props) -> {DiskLen, EncodedLen, Encoding} = encoded_lengths_from_json(Props), Digest = digest_from_json(Props), RevPos = couch_util:get_value(<<"revpos">>, Props, 0), - store([ - {md5, Digest}, {revpos, RevPos}, {data, follows}, {disk_len, DiskLen}, - {att_len, EncodedLen}, {encoding, Encoding} - ], Att). - + store( + [ + {md5, Digest}, + {revpos, RevPos}, + {data, follows}, + {disk_len, DiskLen}, + {att_len, EncodedLen}, + {encoding, Encoding} + ], + Att + ). inline_from_json(Att, Props) -> B64Data = couch_util:get_value(<<"data">>, Props), @@ -456,19 +467,22 @@ inline_from_json(Att, Props) -> Data -> Length = size(Data), RevPos = couch_util:get_value(<<"revpos">>, Props, 0), - store([ - {data, Data}, {revpos, RevPos}, {disk_len, Length}, - {att_len, Length} - ], Att) + store( + [ + {data, Data}, + {revpos, RevPos}, + {disk_len, Length}, + {att_len, Length} + ], + Att + ) catch _:_ -> Name = fetch(name, Att), - ErrMsg = <<"Invalid attachment data for ", Name/binary>>, + ErrMsg = <<"Invalid attachment data for ", Name/binary>>, throw({bad_request, ErrMsg}) end. - - encoded_lengths_from_json(Props) -> Len = couch_util:get_value(<<"length">>, Props), case couch_util:get_value(<<"encoding">>, Props) of @@ -481,14 +495,12 @@ encoded_lengths_from_json(Props) -> end, {Len, EncodedLen, Encoding}. - digest_from_json(Props) -> case couch_util:get_value(<<"digest">>, Props) of <<"md5-", EncodedMd5/binary>> -> base64:decode(EncodedMd5); _ -> <<>> end. - to_json(Att, OutputData, DataToFollow, ShowEncoding) -> [Name, Data, DiskLen, AttLen, Enc, Type, RevPos, Md5] = fetch( [name, data, disk_len, att_len, encoding, type, revpos, md5], Att @@ -497,42 +509,45 @@ to_json(Att, OutputData, DataToFollow, ShowEncoding) -> {<<"content_type">>, Type}, {<<"revpos">>, RevPos} ], - DigestProp = case base64:encode(Md5) of - <<>> -> []; - Digest -> [{<<"digest">>, <<"md5-", Digest/binary>>}] - end, - DataProps = if - not OutputData orelse Data == stub -> - [{<<"length">>, DiskLen}, {<<"stub">>, true}]; - DataToFollow -> - [{<<"length">>, DiskLen}, {<<"follows">>, true}]; - true -> - AttData = case Enc of - gzip -> zlib:gunzip(to_binary(Att)); - identity -> to_binary(Att) - end, - [{<<"data">>, base64:encode(AttData)}] - end, - EncodingProps = if - ShowEncoding andalso Enc /= identity -> - [ - {<<"encoding">>, couch_util:to_binary(Enc)}, - {<<"encoded_length">>, AttLen} - ]; - true -> - [] - end, - HeadersProp = case fetch(headers, Att) of - undefined -> []; - Headers -> [{<<"headers">>, Headers}] - end, + DigestProp = + case base64:encode(Md5) of + <<>> -> []; + Digest -> [{<<"digest">>, <<"md5-", Digest/binary>>}] + end, + DataProps = + if + not OutputData orelse Data == stub -> + [{<<"length">>, DiskLen}, {<<"stub">>, true}]; + DataToFollow -> + [{<<"length">>, DiskLen}, {<<"follows">>, true}]; + true -> + AttData = + case Enc of + gzip -> zlib:gunzip(to_binary(Att)); + identity -> to_binary(Att) + end, + [{<<"data">>, base64:encode(AttData)}] + end, + EncodingProps = + if + ShowEncoding andalso Enc /= identity -> + [ + {<<"encoding">>, couch_util:to_binary(Enc)}, + {<<"encoded_length">>, AttLen} + ]; + true -> + [] + end, + HeadersProp = + case fetch(headers, Att) of + undefined -> []; + Headers -> [{<<"headers">>, Headers}] + end, {Name, {Props ++ DigestProp ++ DataProps ++ EncodingProps ++ HeadersProp}}. - flush(Db, Att) -> flush_data(Db, fetch(data, Att), Att). - flush_data(Db, Data, Att) when is_binary(Data) -> couch_db:with_stream(Db, Att, fun(OutputStream) -> couch_stream:write(OutputStream, Data) @@ -545,25 +560,29 @@ flush_data(Db, Fun, Att) when is_function(Fun) -> couch_db:with_stream(Db, Att, fun(OutputStream) -> % Fun(MaxChunkSize, WriterFun) must call WriterFun % once for each chunk of the attachment, - Fun(4096, + Fun( + 4096, % WriterFun({Length, Binary}, State) % WriterFun({0, _Footers}, State) % Called with Length == 0 on the last time. % WriterFun returns NewState. - fun({0, Footers}, _Total) -> - F = mochiweb_headers:from_binary(Footers), - case mochiweb_headers:get_value("Content-MD5", F) of - undefined -> - ok; - Md5 -> - {md5, base64:decode(Md5)} - end; - ({Length, Chunk}, Total0) -> - Total = Total0 + Length, - validate_attachment_size(AttName, Total, MaxAttSize), - couch_stream:write(OutputStream, Chunk), - Total - end, 0) + fun + ({0, Footers}, _Total) -> + F = mochiweb_headers:from_binary(Footers), + case mochiweb_headers:get_value("Content-MD5", F) of + undefined -> + ok; + Md5 -> + {md5, base64:decode(Md5)} + end; + ({Length, Chunk}, Total0) -> + Total = Total0 + Length, + validate_attachment_size(AttName, Total, MaxAttSize), + couch_stream:write(OutputStream, Chunk), + Total + end, + 0 + ) end); AttLen -> validate_attachment_size(AttName, AttLen, MaxAttSize), @@ -600,17 +619,18 @@ flush_data(Db, {stream, StreamEngine}, Att) -> end) end. - write_streamed_attachment(_Stream, _F, 0) -> ok; write_streamed_attachment(_Stream, _F, LenLeft) when LenLeft < 0 -> throw({bad_request, <<"attachment longer than expected">>}); write_streamed_attachment(Stream, F, LenLeft) when LenLeft > 0 -> - Bin = try read_next_chunk(F, LenLeft) - catch - {mp_parser_died, normal} -> - throw({bad_request, <<"attachment shorter than expected">>}) - end, + Bin = + try + read_next_chunk(F, LenLeft) + catch + {mp_parser_died, normal} -> + throw({bad_request, <<"attachment shorter than expected">>}) + end, ok = couch_stream:write(Stream, Bin), write_streamed_attachment(Stream, F, LenLeft - iolist_size(Bin)). @@ -619,11 +639,9 @@ read_next_chunk(F, _) when is_function(F, 0) -> read_next_chunk(F, LenLeft) when is_function(F, 1) -> F(lists:min([LenLeft, 16#2000])). - foldl(Att, Fun, Acc) -> foldl(fetch(data, Att), Att, Fun, Acc). - foldl(Bin, _Att, Fun, Acc) when is_binary(Bin) -> Fun(Bin, Acc); foldl({stream, StreamEngine}, Att, Fun, Acc) -> @@ -651,54 +669,51 @@ foldl({follows, Parser, Ref}, Att, Fun, Acc) -> erlang:demonitor(ParserRef, [flush]) end. - range_foldl(Att, From, To, Fun, Acc) -> {stream, StreamEngine} = fetch(data, Att), couch_stream:range_foldl(StreamEngine, From, To, Fun, Acc). - foldl_decode(Att, Fun, Acc) -> case fetch([data, encoding], Att) of [{stream, StreamEngine}, Enc] -> couch_stream:foldl_decode( - StreamEngine, fetch(md5, Att), Enc, Fun, Acc); + StreamEngine, fetch(md5, Att), Enc, Fun, Acc + ); [Fun2, identity] -> fold_streamed_data(Fun2, fetch(att_len, Att), Fun, Acc) end. - to_binary(Att) -> to_binary(fetch(data, Att), Att). - to_binary(Bin, _Att) when is_binary(Bin) -> Bin; to_binary(Iolist, _Att) when is_list(Iolist) -> iolist_to_binary(Iolist); to_binary({stream, _StreamEngine}, Att) -> iolist_to_binary( - lists:reverse(foldl(Att, fun(Bin,Acc) -> [Bin|Acc] end, [])) + lists:reverse(foldl(Att, fun(Bin, Acc) -> [Bin | Acc] end, [])) ); -to_binary(DataFun, Att) when is_function(DataFun)-> +to_binary(DataFun, Att) when is_function(DataFun) -> Len = fetch(att_len, Att), iolist_to_binary( - lists:reverse(fold_streamed_data( - DataFun, - Len, - fun(Data, Acc) -> [Data | Acc] end, - [] - )) + lists:reverse( + fold_streamed_data( + DataFun, + Len, + fun(Data, Acc) -> [Data | Acc] end, + [] + ) + ) ). - fold_streamed_data(_RcvFun, 0, _Fun, Acc) -> Acc; -fold_streamed_data(RcvFun, LenLeft, Fun, Acc) when LenLeft > 0-> +fold_streamed_data(RcvFun, LenLeft, Fun, Acc) when LenLeft > 0 -> Bin = RcvFun(), ResultAcc = Fun(Bin, Acc), fold_streamed_data(RcvFun, LenLeft - size(Bin), Fun, ResultAcc). - %% Upgrade an attachment record to a property list on demand. This is a one-way %% operation as downgrading potentially truncates fields with important data. -spec upgrade(#att{}) -> attachment(). @@ -712,7 +727,6 @@ upgrade(#att{} = Att) -> upgrade(Att) -> Att. - %% Downgrade is exposed for interactive convenience. In practice, unless done %% manually, upgrades are always one-way. downgrade(#att{} = Att) -> @@ -729,7 +743,6 @@ downgrade(Att) -> encoding = fetch(encoding, Att) }. - upgrade_encoding(true) -> gzip; upgrade_encoding(false) -> identity; upgrade_encoding(Encoding) -> Encoding. @@ -744,9 +757,12 @@ max_attachment_size(MaxAttSizeConfig) -> MaxAttSize when is_list(MaxAttSize) -> try list_to_integer(MaxAttSize) of Result -> Result - catch _:_ -> - couch_log:error("invalid config value for max attachment size: ~p ", [MaxAttSize]), - throw(internal_server_error) + catch + _:_ -> + couch_log:error("invalid config value for max attachment size: ~p ", [ + MaxAttSize + ]), + throw(internal_server_error) end; MaxAttSize when is_integer(MaxAttSize) -> MaxAttSize; @@ -755,14 +771,13 @@ max_attachment_size(MaxAttSizeConfig) -> throw(internal_server_error) end. - -validate_attachment_size(AttName, AttSize, MaxAttSize) - when is_integer(AttSize), AttSize > MaxAttSize -> +validate_attachment_size(AttName, AttSize, MaxAttSize) when + is_integer(AttSize), AttSize > MaxAttSize +-> throw({request_entity_too_large, {attachment, AttName}}); validate_attachment_size(_AttName, _AttSize, _MAxAttSize) -> ok. - open_stream(StreamSrc, Data) -> case couch_db:is_db(StreamSrc) of true -> @@ -776,7 +791,6 @@ open_stream(StreamSrc, Data) -> end end. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -785,28 +799,20 @@ open_stream(StreamSrc, Data) -> %% Test utilities - empty_att() -> new(). - upgraded_empty_att() -> new([{headers, undefined}]). - %% Test groups - attachment_upgrade_test_() -> {"Lazy record upgrade tests", [ {"Existing record fields don't upgrade", - {with, empty_att(), [fun test_non_upgrading_fields/1]} - }, - {"New fields upgrade", - {with, empty_att(), [fun test_upgrading_fields/1]} - } + {with, empty_att(), [fun test_non_upgrading_fields/1]}}, + {"New fields upgrade", {with, empty_att(), [fun test_upgrading_fields/1]}} ]}. - attachment_defaults_test_() -> {"Attachment defaults tests", [ {"Records retain old default values", [ @@ -827,14 +833,13 @@ attachment_field_api_test_() -> fun test_transform/0 ]}. - attachment_disk_term_test_() -> BaseAttachment = new([ {name, <<"empty">>}, {type, <<"application/octet-stream">>}, {att_len, 0}, {disk_len, 0}, - {md5, <<212,29,140,217,143,0,178,4,233,128,9,152,236,248,66,126>>}, + {md5, <<212, 29, 140, 217, 143, 0, 178, 4, 233, 128, 9, 152, 236, 248, 66, 126>>}, {revpos, 4}, {data, {stream, {couch_bt_engine_stream, {fake_fd, fake_sp}}}}, {encoding, identity} @@ -843,14 +848,16 @@ attachment_disk_term_test_() -> <<"empty">>, <<"application/octet-stream">>, fake_sp, - 0, 0, 4, - <<212,29,140,217,143,0,178,4,233,128,9,152,236,248,66,126>>, + 0, + 0, + 4, + <<212, 29, 140, 217, 143, 0, 178, 4, 233, 128, 9, 152, 236, 248, 66, 126>>, identity }, Headers = [{<<"X-Foo">>, <<"bar">>}], ExtendedAttachment = store(headers, Headers, BaseAttachment), ExtendedDiskTerm = {BaseDiskTerm, [{headers, Headers}]}, - FakeDb = test_util:fake_db([{engine, {couch_bt_engine, #st{fd=fake_fd}}}]), + FakeDb = test_util:fake_db([{engine, {couch_bt_engine, #st{fd = fake_fd}}}]), {"Disk term tests", [ ?_assertEqual(BaseDiskTerm, to_disk_term(BaseAttachment)), ?_assertEqual(BaseAttachment, from_disk_term(FakeDb, BaseDiskTerm)), @@ -858,7 +865,6 @@ attachment_disk_term_test_() -> ?_assertEqual(ExtendedAttachment, from_disk_term(FakeDb, ExtendedDiskTerm)) ]}. - attachment_json_term_test_() -> Props = [ {<<"content_type">>, <<"application/json">>}, @@ -892,16 +898,13 @@ attachment_json_term_test_() -> ?_assertThrow({bad_request, _}, inline_from_json(Att, InvalidProps)) ]}. - attachment_stub_merge_test_() -> %% Stub merging needs to demonstrate revpos matching, skipping, and missing %% attachment errors. {"Attachment stub merging tests", []}. - %% Test generators - test_non_upgrading_fields(Attachment) -> Pairs = [ {name, "cat.gif"}, @@ -919,8 +922,8 @@ test_non_upgrading_fields(Attachment) -> Updated = store(Field, Value, Attachment), ?assertMatch(#att{}, Updated) end, - Pairs). - + Pairs + ). test_upgrading_fields(Attachment) -> ?assertMatch(#att{}, Attachment), @@ -929,13 +932,11 @@ test_upgrading_fields(Attachment) -> UpdatedHeadersUndefined = store(headers, undefined, Attachment), ?assertMatch(X when is_list(X), UpdatedHeadersUndefined). - test_legacy_defaults(Attachment) -> ?assertEqual(<<>>, fetch(md5, Attachment)), ?assertEqual(0, fetch(revpos, Attachment)), ?assertEqual(identity, fetch(encoding, Attachment)). - test_elided_entries(Attachment) -> ?assertNot(lists:keymember(name, 1, Attachment)), ?assertNot(lists:keymember(type, 1, Attachment)), @@ -943,26 +944,22 @@ test_elided_entries(Attachment) -> ?assertNot(lists:keymember(disk_len, 1, Attachment)), ?assertNot(lists:keymember(data, 1, Attachment)). - test_construction() -> ?assert(new() == new()), Initialized = new([{name, <<"foo.bar">>}, {type, <<"application/qux">>}]), ?assertEqual(<<"foo.bar">>, fetch(name, Initialized)), ?assertEqual(<<"application/qux">>, fetch(type, Initialized)). - test_store_and_fetch() -> Attachment = empty_att(), ?assertEqual(<<"abc">>, fetch(name, store(name, <<"abc">>, Attachment))), ?assertEqual(42, fetch(ans, store(ans, 42, Attachment))). - test_transform() -> Attachment = new([{counter, 0}]), Transformed = transform(counter, fun(Count) -> Count + 1 end, Attachment), ?assertEqual(1, fetch(counter, Transformed)). - max_attachment_size_test_() -> {"Max attachment size tests", [ ?_assertEqual(infinity, max_attachment_size("infinity")), diff --git a/src/couch/src/couch_auth_cache.erl b/src/couch/src/couch_auth_cache.erl index c564cee00..f361ab231 100644 --- a/src/couch/src/couch_auth_cache.erl +++ b/src/couch/src/couch_auth_cache.erl @@ -12,7 +12,6 @@ -module(couch_auth_cache). - -export([ get_user_creds/1, get_user_creds/2, @@ -23,35 +22,33 @@ ensure_users_db_exists/0 ]). - -include_lib("couch/include/couch_db.hrl"). -include_lib("couch/include/couch_js_functions.hrl"). - --spec get_user_creds(UserName::string() | binary()) -> - {ok, Credentials::list(), term()} | nil. +-spec get_user_creds(UserName :: string() | binary()) -> + {ok, Credentials :: list(), term()} | nil. get_user_creds(UserName) -> get_user_creds(nil, UserName). --spec get_user_creds(Req::#httpd{} | nil, UserName::string() | binary()) -> - {ok, Credentials::list(), term()} | nil. +-spec get_user_creds(Req :: #httpd{} | nil, UserName :: string() | binary()) -> + {ok, Credentials :: list(), term()} | nil. get_user_creds(Req, UserName) when is_list(UserName) -> get_user_creds(Req, ?l2b(UserName)); - get_user_creds(_Req, UserName) -> - UserCreds = case get_admin(UserName) of - nil -> - get_from_db(UserName); - Props -> - case get_from_db(UserName) of - nil -> - Props; - UserProps when is_list(UserProps) -> - add_roles(Props, couch_util:get_value(<<"roles">>, UserProps)) - end - end, + UserCreds = + case get_admin(UserName) of + nil -> + get_from_db(UserName); + Props -> + case get_from_db(UserName) of + nil -> + Props; + UserProps when is_list(UserProps) -> + add_roles(Props, couch_util:get_value(<<"roles">>, UserProps)) + end + end, validate_user_creds(UserCreds). update_user_creds(_Req, UserDoc, _AuthCtx) -> @@ -69,31 +66,34 @@ get_admin(UserName) when is_binary(UserName) -> get_admin(?b2l(UserName)); get_admin(UserName) when is_list(UserName) -> case config:get("admins", UserName) of - "-hashed-" ++ HashedPwdAndSalt -> - % the name is an admin, now check to see if there is a user doc - % which has a matching name, salt, and password_sha - [HashedPwd, Salt] = string:tokens(HashedPwdAndSalt, ","), - make_admin_doc(HashedPwd, Salt); - "-pbkdf2-" ++ HashedPwdSaltAndIterations -> - [HashedPwd, Salt, Iterations] = string:tokens(HashedPwdSaltAndIterations, ","), - make_admin_doc(HashedPwd, Salt, Iterations); - _Else -> - nil + "-hashed-" ++ HashedPwdAndSalt -> + % the name is an admin, now check to see if there is a user doc + % which has a matching name, salt, and password_sha + [HashedPwd, Salt] = string:tokens(HashedPwdAndSalt, ","), + make_admin_doc(HashedPwd, Salt); + "-pbkdf2-" ++ HashedPwdSaltAndIterations -> + [HashedPwd, Salt, Iterations] = string:tokens(HashedPwdSaltAndIterations, ","), + make_admin_doc(HashedPwd, Salt, Iterations); + _Else -> + nil end. make_admin_doc(HashedPwd, Salt) -> - [{<<"roles">>, [<<"_admin">>]}, - {<<"salt">>, ?l2b(Salt)}, - {<<"password_scheme">>, <<"simple">>}, - {<<"password_sha">>, ?l2b(HashedPwd)}]. + [ + {<<"roles">>, [<<"_admin">>]}, + {<<"salt">>, ?l2b(Salt)}, + {<<"password_scheme">>, <<"simple">>}, + {<<"password_sha">>, ?l2b(HashedPwd)} + ]. make_admin_doc(DerivedKey, Salt, Iterations) -> - [{<<"roles">>, [<<"_admin">>]}, - {<<"salt">>, ?l2b(Salt)}, - {<<"iterations">>, list_to_integer(Iterations)}, - {<<"password_scheme">>, <<"pbkdf2">>}, - {<<"derived_key">>, ?l2b(DerivedKey)}]. - + [ + {<<"roles">>, [<<"_admin">>]}, + {<<"salt">>, ?l2b(Salt)}, + {<<"iterations">>, list_to_integer(Iterations)}, + {<<"password_scheme">>, <<"pbkdf2">>}, + {<<"derived_key">>, ?l2b(DerivedKey)} + ]. get_from_db(UserName) -> ok = ensure_users_db_exists(), @@ -104,69 +104,69 @@ get_from_db(UserName) -> {DocProps} = couch_doc:to_json_obj(Doc, []), DocProps catch - _:_Error -> - nil + _:_Error -> + nil end end). - validate_user_creds(nil) -> nil; validate_user_creds(UserCreds) -> case couch_util:get_value(<<"_conflicts">>, UserCreds) of - undefined -> - ok; - _ConflictList -> - throw({unauthorized, - <<"User document conflicts must be resolved before the document", - " is used for authentication purposes.">> - }) + undefined -> + ok; + _ConflictList -> + throw( + {unauthorized, + <<"User document conflicts must be resolved before the document", + " is used for authentication purposes.">>} + ) end, {ok, UserCreds, nil}. - users_db() -> DbNameList = config:get("couch_httpd_auth", "authentication_db", "_users"), ?l2b(DbNameList). - ensure_users_db_exists() -> Options = [?ADMIN_CTX, nologifmissing], case couch_db:open(users_db(), Options) of - {ok, Db} -> - ensure_auth_ddoc_exists(Db, <<"_design/_auth">>), - couch_db:close(Db); - _Error -> - {ok, Db} = couch_db:create(users_db(), Options), - ok = ensure_auth_ddoc_exists(Db, <<"_design/_auth">>), - couch_db:close(Db) + {ok, Db} -> + ensure_auth_ddoc_exists(Db, <<"_design/_auth">>), + couch_db:close(Db); + _Error -> + {ok, Db} = couch_db:create(users_db(), Options), + ok = ensure_auth_ddoc_exists(Db, <<"_design/_auth">>), + couch_db:close(Db) end, ok. - ensure_auth_ddoc_exists(Db, DDocId) -> case couch_db:open_doc(Db, DDocId) of - {not_found, _Reason} -> - {ok, AuthDesign} = auth_design_doc(DDocId), - {ok, _Rev} = couch_db:update_doc(Db, AuthDesign, []); - {ok, Doc} -> - {Props} = couch_doc:to_json_obj(Doc, []), - case couch_util:get_value(<<"validate_doc_update">>, Props, []) of - ?AUTH_DB_DOC_VALIDATE_FUNCTION -> - ok; - _ -> - Props1 = lists:keyreplace(<<"validate_doc_update">>, 1, Props, - {<<"validate_doc_update">>, - ?AUTH_DB_DOC_VALIDATE_FUNCTION}), - couch_db:update_doc(Db, couch_doc:from_json_obj({Props1}), []) - end + {not_found, _Reason} -> + {ok, AuthDesign} = auth_design_doc(DDocId), + {ok, _Rev} = couch_db:update_doc(Db, AuthDesign, []); + {ok, Doc} -> + {Props} = couch_doc:to_json_obj(Doc, []), + case couch_util:get_value(<<"validate_doc_update">>, Props, []) of + ?AUTH_DB_DOC_VALIDATE_FUNCTION -> + ok; + _ -> + Props1 = lists:keyreplace( + <<"validate_doc_update">>, + 1, + Props, + {<<"validate_doc_update">>, ?AUTH_DB_DOC_VALIDATE_FUNCTION} + ), + couch_db:update_doc(Db, couch_doc:from_json_obj({Props1}), []) + end end, ok. auth_design_doc(DocId) -> DocProps = [ {<<"_id">>, DocId}, - {<<"language">>,<<"javascript">>}, + {<<"language">>, <<"javascript">>}, {<<"validate_doc_update">>, ?AUTH_DB_DOC_VALIDATE_FUNCTION} ], {ok, couch_doc:from_json_obj({DocProps})}. diff --git a/src/couch/src/couch_base32.erl b/src/couch/src/couch_base32.erl index d8d754f5e..776fe773d 100644 --- a/src/couch/src/couch_base32.erl +++ b/src/couch/src/couch_base32.erl @@ -16,7 +16,6 @@ -define(SET, <<"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567">>). - -spec encode(binary()) -> binary(). encode(Plain) when is_binary(Plain) -> IoList = encode(Plain, 0, byte_size(Plain) * 8, []), @@ -24,54 +23,63 @@ encode(Plain) when is_binary(Plain) -> encode(_Plain, _ByteOffset, 0, Acc) -> Acc; - encode(Plain, ByteOffset, BitsRemaining, Acc) when BitsRemaining == 8 -> <<A:5, B:3>> = binary:part(Plain, ByteOffset, 1), - [<<(binary:at(?SET, A)), - (binary:at(?SET, B bsl 2)), - "======">> | Acc]; - + [<<(binary:at(?SET, A)), (binary:at(?SET, B bsl 2)), "======">> | Acc]; encode(Plain, ByteOffset, BitsRemaining, Acc) when BitsRemaining == 16 -> <<A:5, B:5, C:5, D:1>> = binary:part(Plain, ByteOffset, 2), - [<<(binary:at(?SET, A)), - (binary:at(?SET, B)), - (binary:at(?SET, C)), - (binary:at(?SET, D bsl 4)), - "====">> | Acc]; - + [ + << + (binary:at(?SET, A)), + (binary:at(?SET, B)), + (binary:at(?SET, C)), + (binary:at(?SET, D bsl 4)), + "====" + >> + | Acc + ]; encode(Plain, ByteOffset, BitsRemaining, Acc) when BitsRemaining == 24 -> <<A:5, B:5, C:5, D:5, E:4>> = binary:part(Plain, ByteOffset, 3), - [<<(binary:at(?SET, A)), - (binary:at(?SET, B)), - (binary:at(?SET, C)), - (binary:at(?SET, D)), - (binary:at(?SET, E bsl 1)), - "===">> | Acc]; - + [ + << + (binary:at(?SET, A)), + (binary:at(?SET, B)), + (binary:at(?SET, C)), + (binary:at(?SET, D)), + (binary:at(?SET, E bsl 1)), + "===" + >> + | Acc + ]; encode(Plain, ByteOffset, BitsRemaining, Acc) when BitsRemaining == 32 -> <<A:5, B:5, C:5, D:5, E:5, F:5, G:2>> = binary:part(Plain, ByteOffset, 4), - [<<(binary:at(?SET, A)), - (binary:at(?SET, B)), - (binary:at(?SET, C)), - (binary:at(?SET, D)), - (binary:at(?SET, E)), - (binary:at(?SET, F)), - (binary:at(?SET, G bsl 3)), - "=">> | Acc]; - + [ + << + (binary:at(?SET, A)), + (binary:at(?SET, B)), + (binary:at(?SET, C)), + (binary:at(?SET, D)), + (binary:at(?SET, E)), + (binary:at(?SET, F)), + (binary:at(?SET, G bsl 3)), + "=" + >> + | Acc + ]; encode(Plain, ByteOffset, BitsRemaining, Acc) when BitsRemaining >= 40 -> <<A:5, B:5, C:5, D:5, E:5, F:5, G:5, H:5>> = binary:part(Plain, ByteOffset, 5), - Output = <<(binary:at(?SET, A)), - (binary:at(?SET, B)), - (binary:at(?SET, C)), - (binary:at(?SET, D)), - (binary:at(?SET, E)), - (binary:at(?SET, F)), - (binary:at(?SET, G)), - (binary:at(?SET, H))>>, - encode(Plain, ByteOffset + 5, BitsRemaining - 40, [Output | Acc]). - + Output = << + (binary:at(?SET, A)), + (binary:at(?SET, B)), + (binary:at(?SET, C)), + (binary:at(?SET, D)), + (binary:at(?SET, E)), + (binary:at(?SET, F)), + (binary:at(?SET, G)), + (binary:at(?SET, H)) + >>, + encode(Plain, ByteOffset + 5, BitsRemaining - 40, [Output | Acc]). -spec decode(binary()) -> binary(). decode(Encoded) when is_binary(Encoded) -> @@ -83,39 +91,60 @@ decode(Encoded, ByteOffset, Acc) when ByteOffset == byte_size(Encoded) -> decode(Encoded, ByteOffset, Acc) -> case binary:part(Encoded, ByteOffset, 8) of <<A:1/binary, B:1/binary, "======">> -> - [<<(find_in_set(A)):5, - (find_in_set(B) bsr 2):3>> | Acc]; + [<<(find_in_set(A)):5, (find_in_set(B) bsr 2):3>> | Acc]; <<A:1/binary, B:1/binary, C:1/binary, D:1/binary, "====">> -> - [<<(find_in_set(A)):5, - (find_in_set(B)):5, - (find_in_set(C)):5, - (find_in_set(D) bsr 4):1>> | Acc]; + [ + << + (find_in_set(A)):5, + (find_in_set(B)):5, + (find_in_set(C)):5, + (find_in_set(D) bsr 4):1 + >> + | Acc + ]; <<A:1/binary, B:1/binary, C:1/binary, D:1/binary, E:1/binary, "===">> -> - [<<(find_in_set(A)):5, - (find_in_set(B)):5, - (find_in_set(C)):5, - (find_in_set(D)):5, - (find_in_set(E) bsr 1):4>> | Acc]; - <<A:1/binary, B:1/binary, C:1/binary, D:1/binary, - E:1/binary, F:1/binary, G:1/binary, "=">> -> - [<<(find_in_set(A)):5, - (find_in_set(B)):5, - (find_in_set(C)):5, - (find_in_set(D)):5, - (find_in_set(E)):5, - (find_in_set(F)):5, - (find_in_set(G) bsr 3):2>> | Acc]; - <<A:1/binary, B:1/binary, C:1/binary, D:1/binary, - E:1/binary, F:1/binary, G:1/binary, H:1/binary>> -> - decode(Encoded, ByteOffset + 8, - [<<(find_in_set(A)):5, - (find_in_set(B)):5, - (find_in_set(C)):5, - (find_in_set(D)):5, - (find_in_set(E)):5, - (find_in_set(F)):5, - (find_in_set(G)):5, - (find_in_set(H)):5>> | Acc]) + [ + << + (find_in_set(A)):5, + (find_in_set(B)):5, + (find_in_set(C)):5, + (find_in_set(D)):5, + (find_in_set(E) bsr 1):4 + >> + | Acc + ]; + <<A:1/binary, B:1/binary, C:1/binary, D:1/binary, E:1/binary, F:1/binary, G:1/binary, "=">> -> + [ + << + (find_in_set(A)):5, + (find_in_set(B)):5, + (find_in_set(C)):5, + (find_in_set(D)):5, + (find_in_set(E)):5, + (find_in_set(F)):5, + (find_in_set(G) bsr 3):2 + >> + | Acc + ]; + <<A:1/binary, B:1/binary, C:1/binary, D:1/binary, E:1/binary, F:1/binary, G:1/binary, + H:1/binary>> -> + decode( + Encoded, + ByteOffset + 8, + [ + << + (find_in_set(A)):5, + (find_in_set(B)):5, + (find_in_set(C)):5, + (find_in_set(D)):5, + (find_in_set(E)):5, + (find_in_set(F)):5, + (find_in_set(G)):5, + (find_in_set(H)):5 + >> + | Acc + ] + ) end. find_in_set(Char) -> diff --git a/src/couch/src/couch_bt_engine.erl b/src/couch/src/couch_bt_engine.erl index 48e751a82..7d2390556 100644 --- a/src/couch/src/couch_bt_engine.erl +++ b/src/couch/src/couch_bt_engine.erl @@ -80,12 +80,10 @@ finish_compaction/4 ]). - -export([ init_state/4 ]). - -export([ id_tree_split/1, id_tree_join/2, @@ -105,7 +103,6 @@ purge_seq_tree_join/2 ]). - % Used by the compactor -export([ update_header/2, @@ -113,12 +110,10 @@ copy_props/2 ]). - -include_lib("kernel/include/file.hrl"). -include_lib("couch/include/couch_db.hrl"). -include("couch_bt_engine.hrl"). - exists(FilePath) -> case is_file(FilePath) of true -> @@ -127,7 +122,6 @@ exists(FilePath) -> is_file(FilePath ++ ".compact") end. - delete(RootDir, FilePath, Async) -> %% Delete any leftover compaction files. If we don't do this a %% subsequent request for this DB will try to open them to use @@ -137,70 +131,69 @@ delete(RootDir, FilePath, Async) -> % Delete the actual database file couch_file:delete(RootDir, FilePath, Async). - delete_compaction_files(RootDir, FilePath, DelOpts) -> - lists:foreach(fun(Ext) -> - couch_file:delete(RootDir, FilePath ++ Ext, DelOpts) - end, [".compact", ".compact.data", ".compact.meta"]). - + lists:foreach( + fun(Ext) -> + couch_file:delete(RootDir, FilePath ++ Ext, DelOpts) + end, + [".compact", ".compact.data", ".compact.meta"] + ). init(FilePath, Options) -> {ok, Fd} = open_db_file(FilePath, Options), - Header = case lists:member(create, Options) of - true -> - delete_compaction_files(FilePath), - Header0 = couch_bt_engine_header:new(), - Header1 = init_set_props(Fd, Header0, Options), - ok = couch_file:write_header(Fd, Header1), - Header1; - false -> - case couch_file:read_header(Fd) of - {ok, Header0} -> - Header0; - no_valid_header -> - delete_compaction_files(FilePath), - Header0 = couch_bt_engine_header:new(), - ok = couch_file:write_header(Fd, Header0), - Header0 - end - end, + Header = + case lists:member(create, Options) of + true -> + delete_compaction_files(FilePath), + Header0 = couch_bt_engine_header:new(), + Header1 = init_set_props(Fd, Header0, Options), + ok = couch_file:write_header(Fd, Header1), + Header1; + false -> + case couch_file:read_header(Fd) of + {ok, Header0} -> + Header0; + no_valid_header -> + delete_compaction_files(FilePath), + Header0 = couch_bt_engine_header:new(), + ok = couch_file:write_header(Fd, Header0), + Header0 + end + end, {ok, init_state(FilePath, Fd, Header, Options)}. - terminate(_Reason, St) -> % If the reason we died is because our fd disappeared % then we don't need to try closing it again. Ref = St#st.fd_monitor, - if Ref == closed -> ok; true -> - ok = couch_file:close(St#st.fd), - receive - {'DOWN', Ref, _, _, _} -> - ok + if + Ref == closed -> + ok; + true -> + ok = couch_file:close(St#st.fd), + receive + {'DOWN', Ref, _, _, _} -> + ok after 500 -> ok - end + end end, couch_util:shutdown_sync(St#st.fd), ok. - handle_db_updater_call(Msg, St) -> {stop, {invalid_call, Msg}, {invalid_call, Msg}, St}. - -handle_db_updater_info({'DOWN', Ref, _, _, _}, #st{fd_monitor=Ref} = St) -> - {stop, normal, St#st{fd=undefined, fd_monitor=closed}}. - +handle_db_updater_info({'DOWN', Ref, _, _, _}, #st{fd_monitor = Ref} = St) -> + {stop, normal, St#st{fd = undefined, fd_monitor = closed}}. incref(St) -> {ok, St#st{fd_monitor = erlang:monitor(process, St#st.fd)}}. - decref(St) -> true = erlang:demonitor(St#st.fd_monitor, [flush]), ok. - monitored_by(St) -> case erlang:process_info(St#st.fd, monitored_by) of {monitored_by, Pids} -> @@ -209,33 +202,26 @@ monitored_by(St) -> [] end. - last_activity(#st{fd = Fd}) -> couch_file:last_read(Fd). - get_compacted_seq(#st{header = Header}) -> couch_bt_engine_header:get(Header, compacted_seq). - get_del_doc_count(#st{} = St) -> {ok, Reds} = couch_btree:full_reduce(St#st.id_tree), element(2, Reds). - get_disk_version(#st{header = Header}) -> couch_bt_engine_header:get(Header, disk_version). - get_doc_count(#st{} = St) -> {ok, Reds} = couch_btree:full_reduce(St#st.id_tree), element(1, Reds). - get_epochs(#st{header = Header}) -> couch_bt_engine_header:get(Header, epochs). - get_purge_seq(#st{purge_seq_tree = PurgeSeqTree}) -> Fun = fun({PurgeSeq, _, _, _}, _Reds, _Acc) -> {stop, PurgeSeq} @@ -243,7 +229,6 @@ get_purge_seq(#st{purge_seq_tree = PurgeSeqTree}) -> {ok, _, PurgeSeq} = couch_btree:fold(PurgeSeqTree, Fun, 0, [{dir, rev}]), PurgeSeq. - get_oldest_purge_seq(#st{purge_seq_tree = PurgeSeqTree}) -> Fun = fun({PurgeSeq, _, _, _}, _Reds, _Acc) -> {stop, PurgeSeq} @@ -251,27 +236,25 @@ get_oldest_purge_seq(#st{purge_seq_tree = PurgeSeqTree}) -> {ok, _, PurgeSeq} = couch_btree:fold(PurgeSeqTree, Fun, 0, []), PurgeSeq. - get_purge_infos_limit(#st{header = Header}) -> couch_bt_engine_header:get(Header, purge_infos_limit). - get_revs_limit(#st{header = Header}) -> couch_bt_engine_header:get(Header, revs_limit). - get_size_info(#st{} = St) -> {ok, FileSize} = couch_file:bytes(St#st.fd), {ok, DbReduction} = couch_btree:full_reduce(St#st.id_tree), SizeInfo0 = element(3, DbReduction), - SizeInfo = case SizeInfo0 of - SI when is_record(SI, size_info) -> - SI; - {AS, ES} -> - #size_info{active=AS, external=ES}; - AS -> - #size_info{active=AS} - end, + SizeInfo = + case SizeInfo0 of + SI when is_record(SI, size_info) -> + SI; + {AS, ES} -> + #size_info{active = AS, external = ES}; + AS -> + #size_info{active = AS} + end, ActiveSize = active_size(St, SizeInfo), ExternalSize = SizeInfo#size_info.external, [ @@ -280,7 +263,6 @@ get_size_info(#st{} = St) -> {file, FileSize} ]. - partition_size_cb(traverse, Key, {DC, DDC, Sizes}, {Partition, DCAcc, DDCAcc, SizesAcc}) -> case couch_partition:is_member(Key, Partition) of true -> @@ -288,22 +270,18 @@ partition_size_cb(traverse, Key, {DC, DDC, Sizes}, {Partition, DCAcc, DDCAcc, Si false -> {ok, {Partition, DCAcc, DDCAcc, SizesAcc}} end; - partition_size_cb(visit, FDI, _PrevReds, {Partition, DCAcc, DDCAcc, Acc}) -> InPartition = couch_partition:is_member(FDI#full_doc_info.id, Partition), Deleted = FDI#full_doc_info.deleted, case {InPartition, Deleted} of {true, true} -> - {ok, {Partition, DCAcc, DDCAcc + 1, - reduce_sizes(FDI#full_doc_info.sizes, Acc)}}; + {ok, {Partition, DCAcc, DDCAcc + 1, reduce_sizes(FDI#full_doc_info.sizes, Acc)}}; {true, false} -> - {ok, {Partition, DCAcc + 1, DDCAcc, - reduce_sizes(FDI#full_doc_info.sizes, Acc)}}; + {ok, {Partition, DCAcc + 1, DDCAcc, reduce_sizes(FDI#full_doc_info.sizes, Acc)}}; {false, _} -> {ok, {Partition, DCAcc, DDCAcc, Acc}} end. - get_partition_info(#st{} = St, Partition) -> StartKey = couch_partition:start_key(Partition), EndKey = couch_partition:end_key(Partition), @@ -322,7 +300,6 @@ get_partition_info(#st{} = St, Partition) -> ]} ]. - get_security(#st{header = Header} = St) -> case couch_bt_engine_header:get(Header, security_ptr) of undefined -> @@ -332,7 +309,6 @@ get_security(#st{header = Header} = St) -> SecProps end. - get_props(#st{header = Header} = St) -> case couch_bt_engine_header:get(Header, props_ptr) of undefined -> @@ -342,15 +318,12 @@ get_props(#st{header = Header} = St) -> Props end. - get_update_seq(#st{header = Header}) -> couch_bt_engine_header:get(Header, update_seq). - get_uuid(#st{header = Header}) -> couch_bt_engine_header:get(Header, uuid). - set_revs_limit(#st{header = Header} = St, RevsLimit) -> NewSt = St#st{ header = couch_bt_engine_header:set(Header, [ @@ -360,7 +333,6 @@ set_revs_limit(#st{header = Header} = St, RevsLimit) -> }, {ok, increment_update_seq(NewSt)}. - set_purge_infos_limit(#st{header = Header} = St, PurgeInfosLimit) -> NewSt = St#st{ header = couch_bt_engine_header:set(Header, [ @@ -370,7 +342,6 @@ set_purge_infos_limit(#st{header = Header} = St, PurgeInfosLimit) -> }, {ok, increment_update_seq(NewSt)}. - set_security(#st{header = Header} = St, NewSecurity) -> Options = [{compression, St#st.compression}], {ok, Ptr, _} = couch_file:append_term(St#st.fd, NewSecurity, Options), @@ -382,7 +353,6 @@ set_security(#st{header = Header} = St, NewSecurity) -> }, {ok, increment_update_seq(NewSt)}. - set_props(#st{header = Header} = St, Props) -> Options = [{compression, St#st.compression}], {ok, Ptr, _} = couch_file:append_term(St#st.fd, Props, Options), @@ -394,22 +364,25 @@ set_props(#st{header = Header} = St, Props) -> }, {ok, increment_update_seq(NewSt)}. - open_docs(#st{} = St, DocIds) -> Results = couch_btree:lookup(St#st.id_tree, DocIds), - lists:map(fun - ({ok, FDI}) -> FDI; - (not_found) -> not_found - end, Results). - + lists:map( + fun + ({ok, FDI}) -> FDI; + (not_found) -> not_found + end, + Results + ). open_local_docs(#st{} = St, DocIds) -> Results = couch_btree:lookup(St#st.local_tree, DocIds), - lists:map(fun - ({ok, Doc}) -> Doc; - (not_found) -> not_found - end, Results). - + lists:map( + fun + ({ok, Doc}) -> Doc; + (not_found) -> not_found + end, + Results + ). read_doc_body(#st{} = St, #doc{} = Doc) -> {ok, {Body, Atts}} = couch_file:pread_term(St#st.fd, Doc#doc.body), @@ -418,14 +391,15 @@ read_doc_body(#st{} = St, #doc{} = Doc) -> atts = Atts }. - load_purge_infos(St, UUIDs) -> Results = couch_btree:lookup(St#st.purge_tree, UUIDs), - lists:map(fun - ({ok, Info}) -> Info; - (not_found) -> not_found - end, Results). - + lists:map( + fun + ({ok, Info}) -> Info; + (not_found) -> not_found + end, + Results + ). serialize_doc(#st{} = St, #doc{} = Doc) -> Compress = fun(Term) -> @@ -449,7 +423,6 @@ serialize_doc(#st{} = St, #doc{} = Doc) -> meta = [{comp_body, Body} | Doc#doc.meta] }. - write_doc_body(St, #doc{} = Doc) -> #st{ fd = Fd @@ -457,46 +430,57 @@ write_doc_body(St, #doc{} = Doc) -> {ok, Ptr, Written} = couch_file:append_raw_chunk(Fd, Doc#doc.body), {ok, Doc#doc{body = Ptr}, Written}. - write_doc_infos(#st{} = St, Pairs, LocalDocs) -> #st{ id_tree = IdTree, seq_tree = SeqTree, local_tree = LocalTree } = St, - FinalAcc = lists:foldl(fun({OldFDI, NewFDI}, Acc) -> - {AddAcc, RemIdsAcc, RemSeqsAcc} = Acc, - case {OldFDI, NewFDI} of - {not_found, #full_doc_info{}} -> - {[NewFDI | AddAcc], RemIdsAcc, RemSeqsAcc}; - {#full_doc_info{id = Id}, #full_doc_info{id = Id}} -> - NewAddAcc = [NewFDI | AddAcc], - NewRemSeqsAcc = [OldFDI#full_doc_info.update_seq | RemSeqsAcc], - {NewAddAcc, RemIdsAcc, NewRemSeqsAcc}; - {#full_doc_info{id = Id}, not_found} -> - NewRemIdsAcc = [Id | RemIdsAcc], - NewRemSeqsAcc = [OldFDI#full_doc_info.update_seq | RemSeqsAcc], - {AddAcc, NewRemIdsAcc, NewRemSeqsAcc} - end - end, {[], [], []}, Pairs), + FinalAcc = lists:foldl( + fun({OldFDI, NewFDI}, Acc) -> + {AddAcc, RemIdsAcc, RemSeqsAcc} = Acc, + case {OldFDI, NewFDI} of + {not_found, #full_doc_info{}} -> + {[NewFDI | AddAcc], RemIdsAcc, RemSeqsAcc}; + {#full_doc_info{id = Id}, #full_doc_info{id = Id}} -> + NewAddAcc = [NewFDI | AddAcc], + NewRemSeqsAcc = [OldFDI#full_doc_info.update_seq | RemSeqsAcc], + {NewAddAcc, RemIdsAcc, NewRemSeqsAcc}; + {#full_doc_info{id = Id}, not_found} -> + NewRemIdsAcc = [Id | RemIdsAcc], + NewRemSeqsAcc = [OldFDI#full_doc_info.update_seq | RemSeqsAcc], + {AddAcc, NewRemIdsAcc, NewRemSeqsAcc} + end + end, + {[], [], []}, + Pairs + ), {Add, RemIds, RemSeqs} = FinalAcc, {ok, IdTree2} = couch_btree:add_remove(IdTree, Add, RemIds), {ok, SeqTree2} = couch_btree:add_remove(SeqTree, Add, RemSeqs), - {AddLDocs, RemLDocIds} = lists:foldl(fun(Doc, {AddAcc, RemAcc}) -> - case Doc#doc.deleted of - true -> - {AddAcc, [Doc#doc.id | RemAcc]}; - false -> - {[Doc | AddAcc], RemAcc} - end - end, {[], []}, LocalDocs), + {AddLDocs, RemLDocIds} = lists:foldl( + fun(Doc, {AddAcc, RemAcc}) -> + case Doc#doc.deleted of + true -> + {AddAcc, [Doc#doc.id | RemAcc]}; + false -> + {[Doc | AddAcc], RemAcc} + end + end, + {[], []}, + LocalDocs + ), {ok, LocalTree2} = couch_btree:add_remove(LocalTree, AddLDocs, RemLDocIds), - NewUpdateSeq = lists:foldl(fun(#full_doc_info{update_seq=Seq}, Acc) -> - erlang:max(Seq, Acc) - end, get_update_seq(St), Add), + NewUpdateSeq = lists:foldl( + fun(#full_doc_info{update_seq = Seq}, Acc) -> + erlang:max(Seq, Acc) + end, + get_update_seq(St), + Add + ), NewHeader = couch_bt_engine_header:set(St#st.header, [ {update_seq, NewUpdateSeq} @@ -510,7 +494,6 @@ write_doc_infos(#st{} = St, Pairs, LocalDocs) -> needs_commit = true }}. - purge_docs(#st{} = St, Pairs, PurgeInfos) -> #st{ id_tree = IdTree, @@ -529,10 +512,11 @@ purge_docs(#st{} = St, Pairs, PurgeInfos) -> % We bump NewUpdateSeq because we have to ensure that % indexers see that they need to process the new purge % information. - UpdateSeq = case NewSeq == CurrSeq of - true -> CurrSeq + 1; - false -> NewSeq - end, + UpdateSeq = + case NewSeq == CurrSeq of + true -> CurrSeq + 1; + false -> NewSeq + end, Header = couch_bt_engine_header:set(St#st.header, [ {update_seq, UpdateSeq} ]), @@ -550,7 +534,6 @@ purge_docs(#st{} = St, Pairs, PurgeInfos) -> needs_commit = true }}. - copy_purge_infos(#st{} = St, PurgeInfos) -> #st{ purge_tree = PurgeTree, @@ -559,12 +542,11 @@ copy_purge_infos(#st{} = St, PurgeInfos) -> {ok, PurgeTree2} = couch_btree:add(PurgeTree, PurgeInfos), {ok, PurgeSeqTree2} = couch_btree:add(PurgeSeqTree, PurgeInfos), {ok, St#st{ - purge_tree = PurgeTree2, - purge_seq_tree = PurgeSeqTree2, - needs_commit = true + purge_tree = PurgeTree2, + purge_seq_tree = PurgeSeqTree2, + needs_commit = true }}. - commit_data(St) -> #st{ fd = Fd, @@ -587,32 +569,26 @@ commit_data(St) -> {ok, St} end. - open_write_stream(#st{} = St, Options) -> couch_stream:open({couch_bt_engine_stream, {St#st.fd, []}}, Options). - open_read_stream(#st{} = St, StreamSt) -> {ok, {couch_bt_engine_stream, {St#st.fd, StreamSt}}}. - is_active_stream(#st{} = St, {couch_bt_engine_stream, {Fd, _}}) -> St#st.fd == Fd; is_active_stream(_, _) -> false. - fold_docs(St, UserFun, UserAcc, Options) -> fold_docs_int(St, St#st.id_tree, UserFun, UserAcc, Options). - fold_local_docs(St, UserFun, UserAcc, Options) -> case fold_docs_int(St, St#st.local_tree, UserFun, UserAcc, Options) of {ok, _Reds, FinalAcc} -> {ok, null, FinalAcc}; {ok, FinalAcc} -> {ok, FinalAcc} end. - fold_changes(St, SinceSeq, UserFun, UserAcc, Options) -> Fun = fun drop_reductions/4, InAcc = {UserFun, UserAcc}, @@ -621,13 +597,13 @@ fold_changes(St, SinceSeq, UserFun, UserAcc, Options) -> {_, FinalUserAcc} = OutAcc, {ok, FinalUserAcc}. - fold_purge_infos(St, StartSeq0, UserFun, UserAcc, Options) -> PurgeSeqTree = St#st.purge_seq_tree, StartSeq = StartSeq0 + 1, MinSeq = get_oldest_purge_seq(St), - if MinSeq =< StartSeq -> ok; true -> - erlang:error({invalid_start_purge_seq, StartSeq0}) + if + MinSeq =< StartSeq -> ok; + true -> erlang:error({invalid_start_purge_seq, StartSeq0}) end, Wrapper = fun(Info, _Reds, UAcc) -> UserFun(Info, UAcc) @@ -636,7 +612,6 @@ fold_purge_infos(St, StartSeq0, UserFun, UserAcc, Options) -> {ok, _, OutAcc} = couch_btree:fold(PurgeSeqTree, Wrapper, UserAcc, Opts), {ok, OutAcc}. - count_changes_since(St, SinceSeq) -> BTree = St#st.seq_tree, FoldFun = fun(_SeqStart, PartialReds, 0) -> @@ -646,13 +621,11 @@ count_changes_since(St, SinceSeq) -> {ok, Changes} = couch_btree:fold_reduce(BTree, FoldFun, 0, Opts), Changes. - start_compaction(St, DbName, Options, Parent) -> Args = [St, DbName, Options, Parent], Pid = spawn_link(couch_bt_engine_compactor, start, Args), {ok, St, Pid}. - finish_compaction(OldState, DbName, Options, CompactFilePath) -> {ok, NewState1} = ?MODULE:init(CompactFilePath, Options), OldSeq = get_update_seq(OldState), @@ -661,15 +634,16 @@ finish_compaction(OldState, DbName, Options, CompactFilePath) -> true -> finish_compaction_int(OldState, NewState1); false -> - couch_log:info("Compaction file still behind main file " - "(update seq=~p. compact update seq=~p). Retrying.", - [OldSeq, NewSeq]), + couch_log:info( + "Compaction file still behind main file " + "(update seq=~p. compact update seq=~p). Retrying.", + [OldSeq, NewSeq] + ), ok = decref(NewState1), start_compaction(OldState, DbName, Options, self()) end. - -id_tree_split(#full_doc_info{}=Info) -> +id_tree_split(#full_doc_info{} = Info) -> #full_doc_info{ id = Id, update_seq = Seq, @@ -679,11 +653,9 @@ id_tree_split(#full_doc_info{}=Info) -> } = Info, {Id, {Seq, ?b2i(Deleted), split_sizes(SizeInfo), disk_tree(Tree)}}. - id_tree_join(Id, {HighSeq, Deleted, DiskTree}) -> % Handle old formats before data_size was added id_tree_join(Id, {HighSeq, Deleted, #size_info{}, DiskTree}); - id_tree_join(Id, {HighSeq, Deleted, Sizes, DiskTree}) -> #full_doc_info{ id = Id, @@ -693,29 +665,35 @@ id_tree_join(Id, {HighSeq, Deleted, Sizes, DiskTree}) -> rev_tree = rev_tree(DiskTree) }. - id_tree_reduce(reduce, FullDocInfos) -> - lists:foldl(fun(Info, {NotDeleted, Deleted, Sizes}) -> - Sizes2 = reduce_sizes(Sizes, Info#full_doc_info.sizes), - case Info#full_doc_info.deleted of - true -> - {NotDeleted, Deleted + 1, Sizes2}; - false -> - {NotDeleted + 1, Deleted, Sizes2} - end - end, {0, 0, #size_info{}}, FullDocInfos); + lists:foldl( + fun(Info, {NotDeleted, Deleted, Sizes}) -> + Sizes2 = reduce_sizes(Sizes, Info#full_doc_info.sizes), + case Info#full_doc_info.deleted of + true -> + {NotDeleted, Deleted + 1, Sizes2}; + false -> + {NotDeleted + 1, Deleted, Sizes2} + end + end, + {0, 0, #size_info{}}, + FullDocInfos + ); id_tree_reduce(rereduce, Reds) -> - lists:foldl(fun - ({NotDeleted, Deleted}, {AccNotDeleted, AccDeleted, _AccSizes}) -> - % pre 1.2 format, will be upgraded on compaction - {AccNotDeleted + NotDeleted, AccDeleted + Deleted, nil}; - ({NotDeleted, Deleted, Sizes}, {AccNotDeleted, AccDeleted, AccSizes}) -> - AccSizes2 = reduce_sizes(AccSizes, Sizes), - {AccNotDeleted + NotDeleted, AccDeleted + Deleted, AccSizes2} - end, {0, 0, #size_info{}}, Reds). - - -seq_tree_split(#full_doc_info{}=Info) -> + lists:foldl( + fun + ({NotDeleted, Deleted}, {AccNotDeleted, AccDeleted, _AccSizes}) -> + % pre 1.2 format, will be upgraded on compaction + {AccNotDeleted + NotDeleted, AccDeleted + Deleted, nil}; + ({NotDeleted, Deleted, Sizes}, {AccNotDeleted, AccDeleted, AccSizes}) -> + AccSizes2 = reduce_sizes(AccSizes, Sizes), + {AccNotDeleted + NotDeleted, AccDeleted + Deleted, AccSizes2} + end, + {0, 0, #size_info{}}, + Reds + ). + +seq_tree_split(#full_doc_info{} = Info) -> #full_doc_info{ id = Id, update_seq = Seq, @@ -725,10 +703,8 @@ seq_tree_split(#full_doc_info{}=Info) -> } = Info, {Seq, {Id, ?b2i(Del), split_sizes(SizeInfo), disk_tree(Tree)}}. - seq_tree_join(Seq, {Id, Del, DiskTree}) when is_integer(Del) -> seq_tree_join(Seq, {Id, Del, {0, 0}, DiskTree}); - seq_tree_join(Seq, {Id, Del, Sizes, DiskTree}) when is_integer(Del) -> #full_doc_info{ id = Id, @@ -737,37 +713,39 @@ seq_tree_join(Seq, {Id, Del, Sizes, DiskTree}) when is_integer(Del) -> sizes = join_sizes(Sizes), rev_tree = rev_tree(DiskTree) }; - seq_tree_join(KeySeq, {Id, RevInfos, DeletedRevInfos}) -> % Older versions stored #doc_info records in the seq_tree. % Compact to upgrade. - Revs = lists:map(fun({Rev, Seq, Bp}) -> - #rev_info{rev = Rev, seq = Seq, deleted = false, body_sp = Bp} - end, RevInfos), - DeletedRevs = lists:map(fun({Rev, Seq, Bp}) -> - #rev_info{rev = Rev, seq = Seq, deleted = true, body_sp = Bp} - end, DeletedRevInfos), + Revs = lists:map( + fun({Rev, Seq, Bp}) -> + #rev_info{rev = Rev, seq = Seq, deleted = false, body_sp = Bp} + end, + RevInfos + ), + DeletedRevs = lists:map( + fun({Rev, Seq, Bp}) -> + #rev_info{rev = Rev, seq = Seq, deleted = true, body_sp = Bp} + end, + DeletedRevInfos + ), #doc_info{ id = Id, high_seq = KeySeq, revs = Revs ++ DeletedRevs }. - seq_tree_reduce(reduce, DocInfos) -> % count the number of documents length(DocInfos); seq_tree_reduce(rereduce, Reds) -> lists:sum(Reds). - local_tree_split(#doc{revs = {0, [Rev]}} = Doc) when is_binary(Rev) -> #doc{ id = Id, body = BodyData } = Doc, {Id, {binary_to_integer(Rev), BodyData}}; - local_tree_split(#doc{revs = {0, [Rev]}} = Doc) when is_integer(Rev) -> #doc{ id = Id, @@ -775,14 +753,12 @@ local_tree_split(#doc{revs = {0, [Rev]}} = Doc) when is_integer(Rev) -> } = Doc, {Id, {Rev, BodyData}}. - local_tree_join(Id, {Rev, BodyData}) when is_binary(Rev) -> #doc{ id = Id, revs = {0, [Rev]}, body = BodyData }; - local_tree_join(Id, {Rev, BodyData}) when is_integer(Rev) -> #doc{ id = Id, @@ -790,30 +766,24 @@ local_tree_join(Id, {Rev, BodyData}) when is_integer(Rev) -> body = BodyData }. - purge_tree_split({PurgeSeq, UUID, DocId, Revs}) -> {UUID, {PurgeSeq, DocId, Revs}}. - purge_tree_join(UUID, {PurgeSeq, DocId, Revs}) -> {PurgeSeq, UUID, DocId, Revs}. - purge_seq_tree_split({PurgeSeq, UUID, DocId, Revs}) -> {PurgeSeq, {UUID, DocId, Revs}}. - purge_seq_tree_join(PurgeSeq, {UUID, DocId, Revs}) -> {PurgeSeq, UUID, DocId, Revs}. - purge_tree_reduce(reduce, IdRevs) -> % count the number of purge requests length(IdRevs); purge_tree_reduce(rereduce, Reds) -> lists:sum(Reds). - set_update_seq(#st{header = Header} = St, UpdateSeq) -> {ok, St#st{ header = couch_bt_engine_header:set(Header, [ @@ -822,7 +792,6 @@ set_update_seq(#st{header = Header} = St, UpdateSeq) -> needs_commit = true }}. - copy_security(#st{header = Header} = St, SecProps) -> Options = [{compression, St#st.compression}], {ok, Ptr, _} = couch_file:append_term(St#st.fd, SecProps, Options), @@ -833,7 +802,6 @@ copy_security(#st{header = Header} = St, SecProps) -> needs_commit = true }}. - copy_props(#st{header = Header} = St, Props) -> Options = [{compression, St#st.compression}], {ok, Ptr, _} = couch_file:append_term(St#st.fd, Props, Options), @@ -844,7 +812,6 @@ copy_props(#st{header = Header} = St, Props) -> needs_commit = true }}. - open_db_file(FilePath, Options) -> case couch_file:open(FilePath, Options) of {ok, Fd} -> @@ -866,7 +833,6 @@ open_db_file(FilePath, Options) -> throw(Error) end. - init_state(FilePath, Fd, Header0, Options) -> ok = couch_file:sync(Fd), @@ -878,26 +844,26 @@ init_state(FilePath, Fd, Header0, Options) -> IdTreeState = couch_bt_engine_header:id_tree_state(Header), {ok, IdTree} = couch_btree:open(IdTreeState, Fd, [ - {split, fun ?MODULE:id_tree_split/1}, - {join, fun ?MODULE:id_tree_join/2}, - {reduce, fun ?MODULE:id_tree_reduce/2}, - {compression, Compression} - ]), + {split, fun ?MODULE:id_tree_split/1}, + {join, fun ?MODULE:id_tree_join/2}, + {reduce, fun ?MODULE:id_tree_reduce/2}, + {compression, Compression} + ]), SeqTreeState = couch_bt_engine_header:seq_tree_state(Header), {ok, SeqTree} = couch_btree:open(SeqTreeState, Fd, [ - {split, fun ?MODULE:seq_tree_split/1}, - {join, fun ?MODULE:seq_tree_join/2}, - {reduce, fun ?MODULE:seq_tree_reduce/2}, - {compression, Compression} - ]), + {split, fun ?MODULE:seq_tree_split/1}, + {join, fun ?MODULE:seq_tree_join/2}, + {reduce, fun ?MODULE:seq_tree_reduce/2}, + {compression, Compression} + ]), LocalTreeState = couch_bt_engine_header:local_tree_state(Header), {ok, LocalTree} = couch_btree:open(LocalTreeState, Fd, [ - {split, fun ?MODULE:local_tree_split/1}, - {join, fun ?MODULE:local_tree_join/2}, - {compression, Compression} - ]), + {split, fun ?MODULE:local_tree_split/1}, + {join, fun ?MODULE:local_tree_join/2}, + {compression, Compression} + ]), PurgeTreeState = couch_bt_engine_header:purge_tree_state(Header), {ok, PurgeTree} = couch_btree:open(PurgeTreeState, Fd, [ @@ -940,7 +906,6 @@ init_state(FilePath, Fd, Header0, Options) -> St end. - update_header(St, Header) -> couch_bt_engine_header:set(Header, [ {seq_tree_state, couch_btree:get_state(St#st.seq_tree)}, @@ -950,7 +915,6 @@ update_header(St, Header) -> {purge_seq_tree_state, couch_btree:get_state(St#st.purge_seq_tree)} ]). - increment_update_seq(#st{header = Header} = St) -> UpdateSeq = couch_bt_engine_header:get(Header, update_seq), St#st{ @@ -959,7 +923,6 @@ increment_update_seq(#st{header = Header} = St) -> ]) }. - set_default_security_object(Fd, Header, Compression, Options) -> case couch_bt_engine_header:get(Header, security_ptr) of Pointer when is_integer(Pointer) -> @@ -971,7 +934,6 @@ set_default_security_object(Fd, Header, Compression, Options) -> couch_bt_engine_header:set(Header, security_ptr, Ptr) end. - % This function is here, and not in couch_bt_engine_header % because it requires modifying file contents upgrade_purge_info(Fd, Header) -> @@ -980,24 +942,32 @@ upgrade_purge_info(Fd, Header) -> Header; Ptr when is_tuple(Ptr) -> Header; - PurgeSeq when is_integer(PurgeSeq)-> + PurgeSeq when is_integer(PurgeSeq) -> % Pointer to old purged ids/revs is in purge_seq_tree_state Ptr = couch_bt_engine_header:get(Header, purge_seq_tree_state), case Ptr of nil -> PTS = couch_bt_engine_header:purge_tree_state(Header), - PurgeTreeSt = case PTS of 0 -> nil; Else -> Else end, + PurgeTreeSt = + case PTS of + 0 -> nil; + Else -> Else + end, couch_bt_engine_header:set(Header, [ {purge_tree_state, PurgeTreeSt} ]); _ -> {ok, PurgedIdsRevs} = couch_file:pread_term(Fd, Ptr), - {Infos, _} = lists:foldl(fun({Id, Revs}, {InfoAcc, PSeq}) -> - Info = {PSeq, couch_uuids:random(), Id, Revs}, - {[Info | InfoAcc], PSeq + 1} - end, {[], PurgeSeq}, PurgedIdsRevs), + {Infos, _} = lists:foldl( + fun({Id, Revs}, {InfoAcc, PSeq}) -> + Info = {PSeq, couch_uuids:random(), Id, Revs}, + {[Info | InfoAcc], PSeq + 1} + end, + {[], PurgeSeq}, + PurgedIdsRevs + ), {ok, PurgeTree} = couch_btree:open(nil, Fd, [ {split, fun ?MODULE:purge_tree_split/1}, @@ -1022,7 +992,6 @@ upgrade_purge_info(Fd, Header) -> end end. - init_set_props(Fd, Header, Options) -> case couch_util:get_value(props, Options) of undefined -> @@ -1034,70 +1003,70 @@ init_set_props(Fd, Header, Options) -> couch_bt_engine_header:set(Header, props_ptr, Ptr) end. - delete_compaction_files(FilePath) -> RootDir = config:get("couchdb", "database_dir", "."), DelOpts = [{context, compaction}], delete_compaction_files(RootDir, FilePath, DelOpts). - rev_tree(DiskTree) -> - couch_key_tree:map(fun - (_RevId, {Del, Ptr, Seq}) -> - #leaf{ - deleted = ?i2b(Del), - ptr = Ptr, - seq = Seq - }; - (_RevId, {Del, Ptr, Seq, Size}) -> - #leaf{ - deleted = ?i2b(Del), - ptr = Ptr, - seq = Seq, - sizes = couch_db_updater:upgrade_sizes(Size) - }; - (_RevId, {Del, Ptr, Seq, Sizes, Atts}) -> - #leaf{ - deleted = ?i2b(Del), - ptr = Ptr, - seq = Seq, - sizes = couch_db_updater:upgrade_sizes(Sizes), - atts = Atts - }; - (_RevId, ?REV_MISSING) -> - ?REV_MISSING - end, DiskTree). - + couch_key_tree:map( + fun + (_RevId, {Del, Ptr, Seq}) -> + #leaf{ + deleted = ?i2b(Del), + ptr = Ptr, + seq = Seq + }; + (_RevId, {Del, Ptr, Seq, Size}) -> + #leaf{ + deleted = ?i2b(Del), + ptr = Ptr, + seq = Seq, + sizes = couch_db_updater:upgrade_sizes(Size) + }; + (_RevId, {Del, Ptr, Seq, Sizes, Atts}) -> + #leaf{ + deleted = ?i2b(Del), + ptr = Ptr, + seq = Seq, + sizes = couch_db_updater:upgrade_sizes(Sizes), + atts = Atts + }; + (_RevId, ?REV_MISSING) -> + ?REV_MISSING + end, + DiskTree + ). disk_tree(RevTree) -> - couch_key_tree:map(fun - (_RevId, ?REV_MISSING) -> - ?REV_MISSING; - (_RevId, #leaf{} = Leaf) -> - #leaf{ - deleted = Del, - ptr = Ptr, - seq = Seq, - sizes = Sizes, - atts = Atts - } = Leaf, - {?b2i(Del), Ptr, Seq, split_sizes(Sizes), Atts} - end, RevTree). - - -split_sizes(#size_info{}=SI) -> + couch_key_tree:map( + fun + (_RevId, ?REV_MISSING) -> + ?REV_MISSING; + (_RevId, #leaf{} = Leaf) -> + #leaf{ + deleted = Del, + ptr = Ptr, + seq = Seq, + sizes = Sizes, + atts = Atts + } = Leaf, + {?b2i(Del), Ptr, Seq, split_sizes(Sizes), Atts} + end, + RevTree + ). + +split_sizes(#size_info{} = SI) -> {SI#size_info.active, SI#size_info.external}. - join_sizes({Active, External}) when is_integer(Active), is_integer(External) -> - #size_info{active=Active, external=External}. - + #size_info{active = Active, external = External}. reduce_sizes(nil, _) -> nil; reduce_sizes(_, nil) -> nil; -reduce_sizes(#size_info{}=S1, #size_info{}=S2) -> +reduce_sizes(#size_info{} = S1, #size_info{} = S2) -> #size_info{ active = S1#size_info.active + S2#size_info.active, external = S1#size_info.external + S2#size_info.external @@ -1107,7 +1076,6 @@ reduce_sizes(S1, S2) -> US2 = couch_db_updater:upgrade_sizes(S2), reduce_sizes(US1, US2). - active_size(#st{} = St, #size_info{} = SI) -> Trees = [ St#st.id_tree, @@ -1116,27 +1084,32 @@ active_size(#st{} = St, #size_info{} = SI) -> St#st.purge_tree, St#st.purge_seq_tree ], - lists:foldl(fun(T, Acc) -> - case couch_btree:size(T) of - _ when Acc == null -> - null; - nil -> - null; - Size -> - Acc + Size - end - end, SI#size_info.active, Trees). - + lists:foldl( + fun(T, Acc) -> + case couch_btree:size(T) of + _ when Acc == null -> + null; + nil -> + null; + Size -> + Acc + Size + end + end, + SI#size_info.active, + Trees + ). fold_docs_int(St, Tree, UserFun, UserAcc, Options) -> - Fun = case lists:member(include_deleted, Options) of - true -> fun include_deleted/4; - false -> fun skip_deleted/4 - end, - RedFun = case lists:member(include_reductions, Options) of - true -> fun include_reductions/4; - false -> fun drop_reductions/4 - end, + Fun = + case lists:member(include_deleted, Options) of + true -> fun include_deleted/4; + false -> fun skip_deleted/4 + end, + RedFun = + case lists:member(include_reductions, Options) of + true -> fun include_reductions/4; + false -> fun drop_reductions/4 + end, InAcc = {RedFun, {UserFun, UserAcc}}, {ok, Reds, OutAcc} = couch_btree:fold(Tree, Fun, InAcc, Options), {_, {_, FinalUserAcc}} = OutAcc, @@ -1149,12 +1122,10 @@ fold_docs_int(St, Tree, UserFun, UserAcc, Options) -> {ok, FinalUserAcc} end. - include_deleted(Case, Entry, Reds, {UserFun, UserAcc}) -> {Go, NewUserAcc} = UserFun(Case, Entry, Reds, UserAcc), {Go, {UserFun, NewUserAcc}}. - % First element of the reductions is the total % number of undeleted documents. skip_deleted(traverse, _Entry, {0, _, _} = _Reds, Acc) -> @@ -1165,27 +1136,23 @@ skip_deleted(Case, Entry, Reds, {UserFun, UserAcc}) -> {Go, NewUserAcc} = UserFun(Case, Entry, Reds, UserAcc), {Go, {UserFun, NewUserAcc}}. - include_reductions(visit, FDI, Reds, {UserFun, UserAcc}) -> {Go, NewUserAcc} = UserFun(FDI, Reds, UserAcc), {Go, {UserFun, NewUserAcc}}; include_reductions(_, _, _, Acc) -> {ok, Acc}. - drop_reductions(visit, FDI, _Reds, {UserFun, UserAcc}) -> {Go, NewUserAcc} = UserFun(FDI, UserAcc), {Go, {UserFun, NewUserAcc}}; drop_reductions(_, _, _, Acc) -> {ok, Acc}. - fold_docs_reduce_to_count(Reds) -> RedFun = fun id_tree_reduce/2, FinalRed = couch_btree:final_reduce(RedFun, Reds), element(1, FinalRed). - finish_compaction_int(#st{} = OldSt, #st{} = NewSt1) -> #st{ filepath = FilePath, @@ -1233,10 +1200,11 @@ finish_compaction_int(#st{} = OldSt, #st{} = NewSt1) -> decref(OldSt), % And return our finished new state - {ok, NewSt2#st{ - filepath = FilePath - }, undefined}. - + {ok, + NewSt2#st{ + filepath = FilePath + }, + undefined}. is_file(Path) -> case file:read_file_info(Path, [raw]) of diff --git a/src/couch/src/couch_bt_engine_compactor.erl b/src/couch/src/couch_bt_engine_compactor.erl index 3e356e2e3..8ed55b5c3 100644 --- a/src/couch/src/couch_bt_engine_compactor.erl +++ b/src/couch/src/couch_bt_engine_compactor.erl @@ -12,16 +12,13 @@ -module(couch_bt_engine_compactor). - -export([ start/4 ]). - -include_lib("couch/include/couch_db.hrl"). -include("couch_bt_engine.hrl"). - -record(comp_st, { db_name, old_st, @@ -44,14 +41,12 @@ locs }). - -ifdef(TEST). -define(COMP_EVENT(Name), couch_bt_engine_compactor_ev:event(Name)). -else. -define(COMP_EVENT(Name), ignore). -endif. - start(#st{} = St, DbName, Options, Parent) -> erlang:put(io_priority, {db_compact, DbName}), couch_log:debug("Compaction process spawned for db \"~s\"", [DbName]), @@ -72,9 +67,13 @@ start(#st{} = St, DbName, Options, Parent) -> fun compact_final_sync/1 ], - FinalCompSt = lists:foldl(fun(Stage, CompSt) -> - Stage(CompSt) - end, InitCompSt, Stages), + FinalCompSt = lists:foldl( + fun(Stage, CompSt) -> + Stage(CompSt) + end, + InitCompSt, + Stages + ), #comp_st{ new_st = FinalNewSt, @@ -88,7 +87,6 @@ start(#st{} = St, DbName, Options, Parent) -> Msg = {compact_done, couch_bt_engine, FinalNewSt#st.filepath}, gen_server:cast(Parent, Msg). - open_compaction_files(DbName, OldSt, Options) -> #st{ filepath = DbFilePath, @@ -99,57 +97,59 @@ open_compaction_files(DbName, OldSt, Options) -> {ok, DataFd, DataHdr} = open_compaction_file(DataFile), {ok, MetaFd, MetaHdr} = open_compaction_file(MetaFile), DataHdrIsDbHdr = couch_bt_engine_header:is_header(DataHdr), - CompSt = case {DataHdr, MetaHdr} of - {#comp_header{}=A, #comp_header{}=A} -> - % We're restarting a compaction that did not finish - % before trying to swap out with the original db - DbHeader = A#comp_header.db_header, - St0 = couch_bt_engine:init_state( - DataFile, DataFd, DbHeader, Options), - St1 = bind_emsort(St0, MetaFd, A#comp_header.meta_st), - #comp_st{ - db_name = DbName, - old_st = OldSt, - new_st = St1, - meta_fd = MetaFd, - retry = St0#st.id_tree - }; - _ when DataHdrIsDbHdr -> - % We tried to swap out the compaction but there were - % writes to the database during compaction. Start - % a compaction retry. - Header = couch_bt_engine_header:from(SrcHdr), - ok = reset_compaction_file(MetaFd, Header), - St0 = couch_bt_engine:init_state( - DataFile, DataFd, DataHdr, Options), - St1 = bind_emsort(St0, MetaFd, nil), - #comp_st{ - db_name = DbName, - old_st = OldSt, - new_st = St1, - meta_fd = MetaFd, - retry = St0#st.id_tree - }; - _ -> - % We're starting a compaction from scratch - Header = couch_bt_engine_header:from(SrcHdr), - ok = reset_compaction_file(DataFd, Header), - ok = reset_compaction_file(MetaFd, Header), - St0 = couch_bt_engine:init_state(DataFile, DataFd, Header, Options), - St1 = bind_emsort(St0, MetaFd, nil), - #comp_st{ - db_name = DbName, - old_st = OldSt, - new_st = St1, - meta_fd = MetaFd, - retry = nil - } - end, + CompSt = + case {DataHdr, MetaHdr} of + {#comp_header{} = A, #comp_header{} = A} -> + % We're restarting a compaction that did not finish + % before trying to swap out with the original db + DbHeader = A#comp_header.db_header, + St0 = couch_bt_engine:init_state( + DataFile, DataFd, DbHeader, Options + ), + St1 = bind_emsort(St0, MetaFd, A#comp_header.meta_st), + #comp_st{ + db_name = DbName, + old_st = OldSt, + new_st = St1, + meta_fd = MetaFd, + retry = St0#st.id_tree + }; + _ when DataHdrIsDbHdr -> + % We tried to swap out the compaction but there were + % writes to the database during compaction. Start + % a compaction retry. + Header = couch_bt_engine_header:from(SrcHdr), + ok = reset_compaction_file(MetaFd, Header), + St0 = couch_bt_engine:init_state( + DataFile, DataFd, DataHdr, Options + ), + St1 = bind_emsort(St0, MetaFd, nil), + #comp_st{ + db_name = DbName, + old_st = OldSt, + new_st = St1, + meta_fd = MetaFd, + retry = St0#st.id_tree + }; + _ -> + % We're starting a compaction from scratch + Header = couch_bt_engine_header:from(SrcHdr), + ok = reset_compaction_file(DataFd, Header), + ok = reset_compaction_file(MetaFd, Header), + St0 = couch_bt_engine:init_state(DataFile, DataFd, Header, Options), + St1 = bind_emsort(St0, MetaFd, nil), + #comp_st{ + db_name = DbName, + old_st = OldSt, + new_st = St1, + meta_fd = MetaFd, + retry = nil + } + end, unlink(DataFd), erlang:monitor(process, MetaFd), {ok, CompSt}. - copy_purge_info(#comp_st{} = CompSt) -> #comp_st{ db_name = DbName, @@ -164,25 +164,30 @@ copy_purge_info(#comp_st{} = CompSt) -> OldPSTree = OldSt#st.purge_seq_tree, StartSeq = couch_bt_engine:get_purge_seq(NewSt) + 1, BufferSize = config:get_integer( - "database_compaction", "doc_buffer_size", 524288), + "database_compaction", "doc_buffer_size", 524288 + ), CheckpointAfter = config:get( - "database_compaction", "checkpoint_after", BufferSize * 10), + "database_compaction", "checkpoint_after", BufferSize * 10 + ), EnumFun = fun(Info, _Reds, {StAcc0, InfosAcc, InfosSize, CopiedSize}) -> NewInfosSize = InfosSize + ?term_size(Info), - if NewInfosSize >= BufferSize -> - StAcc1 = copy_purge_infos( - OldSt, StAcc0, [Info | InfosAcc], MinPurgeSeq, Retry), - NewCopiedSize = CopiedSize + NewInfosSize, - if NewCopiedSize >= CheckpointAfter -> - StAcc2 = commit_compaction_data(StAcc1), - {ok, {StAcc2, [], 0, 0}}; + if + NewInfosSize >= BufferSize -> + StAcc1 = copy_purge_infos( + OldSt, StAcc0, [Info | InfosAcc], MinPurgeSeq, Retry + ), + NewCopiedSize = CopiedSize + NewInfosSize, + if + NewCopiedSize >= CheckpointAfter -> + StAcc2 = commit_compaction_data(StAcc1), + {ok, {StAcc2, [], 0, 0}}; + true -> + {ok, {StAcc1, [], 0, NewCopiedSize}} + end; true -> - {ok, {StAcc1, [], 0, NewCopiedSize}} - end; - true -> - NewInfosAcc = [Info | InfosAcc], - {ok, {StAcc0, NewInfosAcc, NewInfosSize, CopiedSize}} + NewInfosAcc = [Info | InfosAcc], + {ok, {StAcc0, NewInfosAcc, NewInfosSize, CopiedSize}} end end, @@ -197,7 +202,6 @@ copy_purge_info(#comp_st{} = CompSt) -> new_st = FinalNewSt }. - copy_purge_infos(OldSt, NewSt0, Infos, MinPurgeSeq, Retry) -> #st{ id_tree = OldIdTree @@ -217,9 +221,12 @@ copy_purge_infos(OldSt, NewSt0, Infos, MinPurgeSeq, Retry) -> } = NewSt1, % Copy over the purge infos - InfosToAdd = lists:filter(fun({PSeq, _, _, _}) -> - PSeq > MinPurgeSeq - end, Infos), + InfosToAdd = lists:filter( + fun({PSeq, _, _, _}) -> + PSeq > MinPurgeSeq + end, + Infos + ), {ok, NewPurgeTree1} = couch_btree:add(NewPurgeTree0, InfosToAdd), {ok, NewPurgeSeqTree1} = couch_btree:add(NewPurgeSeqTree0, InfosToAdd), @@ -232,35 +239,44 @@ copy_purge_infos(OldSt, NewSt0, Infos, MinPurgeSeq, Retry) -> % any of the referenced docs have been completely purged % from the database. Any doc that has been completely purged % must then be removed from our partially compacted database. - NewSt3 = if Retry == nil -> NewSt2; true -> - AllDocIds = [DocId || {_PurgeSeq, _UUID, DocId, _Revs} <- Infos], - UniqDocIds = lists:usort(AllDocIds), - OldIdResults = couch_btree:lookup(OldIdTree, UniqDocIds), - OldZipped = lists:zip(UniqDocIds, OldIdResults), - - % The list of non-existant docs in the database being compacted - MaybeRemDocIds = [DocId || {DocId, not_found} <- OldZipped], - - % Removing anything that exists in the partially compacted database - NewIdResults = couch_btree:lookup(NewIdTree0, MaybeRemDocIds), - ToRemove = [Doc || {ok, Doc} <- NewIdResults, Doc /= {ok, not_found}], - - {RemIds, RemSeqs} = lists:unzip(lists:map(fun(FDI) -> - #full_doc_info{ - id = Id, - update_seq = Seq - } = FDI, - {Id, Seq} - end, ToRemove)), - - {ok, NewIdTree1} = couch_btree:add_remove(NewIdTree0, [], RemIds), - {ok, NewSeqTree1} = couch_btree:add_remove(NewSeqTree0, [], RemSeqs), - - NewSt2#st{ - id_tree = NewIdTree1, - seq_tree = NewSeqTree1 - } - end, + NewSt3 = + if + Retry == nil -> + NewSt2; + true -> + AllDocIds = [DocId || {_PurgeSeq, _UUID, DocId, _Revs} <- Infos], + UniqDocIds = lists:usort(AllDocIds), + OldIdResults = couch_btree:lookup(OldIdTree, UniqDocIds), + OldZipped = lists:zip(UniqDocIds, OldIdResults), + + % The list of non-existant docs in the database being compacted + MaybeRemDocIds = [DocId || {DocId, not_found} <- OldZipped], + + % Removing anything that exists in the partially compacted database + NewIdResults = couch_btree:lookup(NewIdTree0, MaybeRemDocIds), + ToRemove = [Doc || {ok, Doc} <- NewIdResults, Doc /= {ok, not_found}], + + {RemIds, RemSeqs} = lists:unzip( + lists:map( + fun(FDI) -> + #full_doc_info{ + id = Id, + update_seq = Seq + } = FDI, + {Id, Seq} + end, + ToRemove + ) + ), + + {ok, NewIdTree1} = couch_btree:add_remove(NewIdTree0, [], RemIds), + {ok, NewSeqTree1} = couch_btree:add_remove(NewSeqTree0, [], RemSeqs), + + NewSt2#st{ + id_tree = NewIdTree1, + seq_tree = NewSeqTree1 + } + end, Header = couch_bt_engine:update_header(NewSt3, NewSt3#st.header), NewSt4 = NewSt3#st{ @@ -268,7 +284,6 @@ copy_purge_infos(OldSt, NewSt0, Infos, MinPurgeSeq, Retry) -> }, bind_emsort(NewSt4, MetaFd, MetaState). - copy_compact(#comp_st{} = CompSt) -> #comp_st{ db_name = DbName, @@ -282,39 +297,49 @@ copy_compact(#comp_st{} = CompSt) -> NewUpdateSeq = couch_bt_engine:get_update_seq(NewSt0), TotalChanges = couch_bt_engine:count_changes_since(St, NewUpdateSeq), BufferSize = list_to_integer( - config:get("database_compaction", "doc_buffer_size", "524288")), + config:get("database_compaction", "doc_buffer_size", "524288") + ), CheckpointAfter = couch_util:to_integer( - config:get("database_compaction", "checkpoint_after", - BufferSize * 10)), + config:get( + "database_compaction", + "checkpoint_after", + BufferSize * 10 + ) + ), EnumBySeqFun = - fun(DocInfo, _Offset, - {AccNewSt, AccUncopied, AccUncopiedSize, AccCopiedSize}) -> + fun( + DocInfo, + _Offset, + {AccNewSt, AccUncopied, AccUncopiedSize, AccCopiedSize} + ) -> + Seq = + case DocInfo of + #full_doc_info{} -> DocInfo#full_doc_info.update_seq; + #doc_info{} -> DocInfo#doc_info.high_seq + end, - Seq = case DocInfo of - #full_doc_info{} -> DocInfo#full_doc_info.update_seq; - #doc_info{} -> DocInfo#doc_info.high_seq + AccUncopiedSize2 = AccUncopiedSize + ?term_size(DocInfo), + if + AccUncopiedSize2 >= BufferSize -> + NewSt2 = copy_docs( + St, AccNewSt, lists:reverse([DocInfo | AccUncopied]), Retry + ), + AccCopiedSize2 = AccCopiedSize + AccUncopiedSize2, + if + AccCopiedSize2 >= CheckpointAfter -> + {ok, NewSt3} = couch_bt_engine:set_update_seq(NewSt2, Seq), + CommNewSt3 = commit_compaction_data(NewSt3), + {ok, {CommNewSt3, [], 0, 0}}; + true -> + {ok, NewSt3} = couch_bt_engine:set_update_seq(NewSt2, Seq), + {ok, {NewSt3, [], 0, AccCopiedSize2}} + end; + true -> + {ok, {AccNewSt, [DocInfo | AccUncopied], AccUncopiedSize2, AccCopiedSize}} + end end, - AccUncopiedSize2 = AccUncopiedSize + ?term_size(DocInfo), - if AccUncopiedSize2 >= BufferSize -> - NewSt2 = copy_docs( - St, AccNewSt, lists:reverse([DocInfo | AccUncopied]), Retry), - AccCopiedSize2 = AccCopiedSize + AccUncopiedSize2, - if AccCopiedSize2 >= CheckpointAfter -> - {ok, NewSt3} = couch_bt_engine:set_update_seq(NewSt2, Seq), - CommNewSt3 = commit_compaction_data(NewSt3), - {ok, {CommNewSt3, [], 0, 0}}; - true -> - {ok, NewSt3} = couch_bt_engine:set_update_seq(NewSt2, Seq), - {ok, {NewSt3, [], 0, AccCopiedSize2}} - end; - true -> - {ok, {AccNewSt, [DocInfo | AccUncopied], AccUncopiedSize2, - AccCopiedSize}} - end - end, - TaskProps0 = [ {type, database_compaction}, {database, DbName}, @@ -324,24 +349,27 @@ copy_compact(#comp_st{} = CompSt) -> {total_changes, TotalChanges} ], case (Retry =/= nil) and couch_task_status:is_task_added() of - true -> - couch_task_status:update([ - {retry, true}, - {phase, document_copy}, - {progress, 0}, - {changes_done, 0}, - {total_changes, TotalChanges} - ]); - false -> - couch_task_status:add_task(TaskProps0), - couch_task_status:set_update_frequency(500) + true -> + couch_task_status:update([ + {retry, true}, + {phase, document_copy}, + {progress, 0}, + {changes_done, 0}, + {total_changes, TotalChanges} + ]); + false -> + couch_task_status:add_task(TaskProps0), + couch_task_status:set_update_frequency(500) end, ?COMP_EVENT(seq_init), {ok, _, {NewSt2, Uncopied, _, _}} = - couch_btree:foldl(St#st.seq_tree, EnumBySeqFun, + couch_btree:foldl( + St#st.seq_tree, + EnumBySeqFun, {NewSt, [], 0, 0}, - [{start_key, NewUpdateSeq + 1}]), + [{start_key, NewUpdateSeq + 1}] + ), NewSt3 = copy_docs(St, NewSt2, lists:reverse(Uncopied), Retry), @@ -362,146 +390,168 @@ copy_compact(#comp_st{} = CompSt) -> new_st = NewSt6 }. - copy_docs(St, #st{} = NewSt, MixedInfos, Retry) -> - DocInfoIds = [Id || #doc_info{id=Id} <- MixedInfos], + DocInfoIds = [Id || #doc_info{id = Id} <- MixedInfos], LookupResults = couch_btree:lookup(St#st.id_tree, DocInfoIds), % COUCHDB-968, make sure we prune duplicates during compaction - NewInfos0 = lists:usort(fun(#full_doc_info{id=A}, #full_doc_info{id=B}) -> - A =< B - end, merge_lookups(MixedInfos, LookupResults)), - - NewInfos1 = lists:map(fun(Info) -> - {NewRevTree, FinalAcc} = couch_key_tree:mapfold(fun - ({RevPos, RevId}, #leaf{ptr=Sp}=Leaf, leaf, SizesAcc) -> - {Body, AttInfos} = copy_doc_attachments(St, Sp, NewSt), - #size_info{external = OldExternalSize} = Leaf#leaf.sizes, - ExternalSize = case OldExternalSize of - 0 when is_binary(Body) -> - couch_compress:uncompressed_size(Body); - 0 -> - couch_ejson_size:encoded_size(Body); - N -> N + NewInfos0 = lists:usort( + fun(#full_doc_info{id = A}, #full_doc_info{id = B}) -> + A =< B + end, + merge_lookups(MixedInfos, LookupResults) + ), + + NewInfos1 = lists:map( + fun(Info) -> + {NewRevTree, FinalAcc} = couch_key_tree:mapfold( + fun + ({RevPos, RevId}, #leaf{ptr = Sp} = Leaf, leaf, SizesAcc) -> + {Body, AttInfos} = copy_doc_attachments(St, Sp, NewSt), + #size_info{external = OldExternalSize} = Leaf#leaf.sizes, + ExternalSize = + case OldExternalSize of + 0 when is_binary(Body) -> + couch_compress:uncompressed_size(Body); + 0 -> + couch_ejson_size:encoded_size(Body); + N -> + N + end, + Doc0 = #doc{ + id = Info#full_doc_info.id, + revs = {RevPos, [RevId]}, + deleted = Leaf#leaf.deleted, + body = Body, + atts = AttInfos + }, + Doc1 = couch_bt_engine:serialize_doc(NewSt, Doc0), + {ok, Doc2, ActiveSize} = + couch_bt_engine:write_doc_body(NewSt, Doc1), + AttSizes = [{element(3, A), element(4, A)} || A <- AttInfos], + NewLeaf = Leaf#leaf{ + ptr = Doc2#doc.body, + sizes = #size_info{ + active = ActiveSize, + external = ExternalSize + }, + atts = AttSizes + }, + {NewLeaf, couch_db_updater:add_sizes(leaf, NewLeaf, SizesAcc)}; + (_Rev, _Leaf, branch, SizesAcc) -> + {?REV_MISSING, SizesAcc} end, - Doc0 = #doc{ - id = Info#full_doc_info.id, - revs = {RevPos, [RevId]}, - deleted = Leaf#leaf.deleted, - body = Body, - atts = AttInfos - }, - Doc1 = couch_bt_engine:serialize_doc(NewSt, Doc0), - {ok, Doc2, ActiveSize} = - couch_bt_engine:write_doc_body(NewSt, Doc1), - AttSizes = [{element(3,A), element(4,A)} || A <- AttInfos], - NewLeaf = Leaf#leaf{ - ptr = Doc2#doc.body, - sizes = #size_info{ - active = ActiveSize, - external = ExternalSize - }, - atts = AttSizes - }, - {NewLeaf, couch_db_updater:add_sizes(leaf, NewLeaf, SizesAcc)}; - (_Rev, _Leaf, branch, SizesAcc) -> - {?REV_MISSING, SizesAcc} - end, {0, 0, []}, Info#full_doc_info.rev_tree), - {FinalAS, FinalES, FinalAtts} = FinalAcc, - TotalAttSize = lists:foldl(fun({_, S}, A) -> S + A end, 0, FinalAtts), - NewActiveSize = FinalAS + TotalAttSize, - NewExternalSize = FinalES + TotalAttSize, - ?COMP_EVENT(seq_copy), - Info#full_doc_info{ - rev_tree = NewRevTree, - sizes = #size_info{ - active = NewActiveSize, - external = NewExternalSize + {0, 0, []}, + Info#full_doc_info.rev_tree + ), + {FinalAS, FinalES, FinalAtts} = FinalAcc, + TotalAttSize = lists:foldl(fun({_, S}, A) -> S + A end, 0, FinalAtts), + NewActiveSize = FinalAS + TotalAttSize, + NewExternalSize = FinalES + TotalAttSize, + ?COMP_EVENT(seq_copy), + Info#full_doc_info{ + rev_tree = NewRevTree, + sizes = #size_info{ + active = NewActiveSize, + external = NewExternalSize + } } - } - end, NewInfos0), + end, + NewInfos0 + ), Limit = couch_bt_engine:get_revs_limit(St), - NewInfos = lists:map(fun(FDI) -> - FDI#full_doc_info{ - rev_tree = couch_key_tree:stem(FDI#full_doc_info.rev_tree, Limit) - } - end, NewInfos1), + NewInfos = lists:map( + fun(FDI) -> + FDI#full_doc_info{ + rev_tree = couch_key_tree:stem(FDI#full_doc_info.rev_tree, Limit) + } + end, + NewInfos1 + ), RemoveSeqs = - case Retry of - nil -> - []; - OldDocIdTree -> - % Compaction is being rerun to catch up to writes during the - % first pass. This means we may have docs that already exist - % in the seq_tree in the .data file. Here we lookup any old - % update_seqs so that they can be removed. - Ids = [Id || #full_doc_info{id=Id} <- NewInfos], - Existing = couch_btree:lookup(OldDocIdTree, Ids), - [Seq || {ok, #full_doc_info{update_seq=Seq}} <- Existing] - end, + case Retry of + nil -> + []; + OldDocIdTree -> + % Compaction is being rerun to catch up to writes during the + % first pass. This means we may have docs that already exist + % in the seq_tree in the .data file. Here we lookup any old + % update_seqs so that they can be removed. + Ids = [Id || #full_doc_info{id = Id} <- NewInfos], + Existing = couch_btree:lookup(OldDocIdTree, Ids), + [Seq || {ok, #full_doc_info{update_seq = Seq}} <- Existing] + end, {ok, SeqTree} = couch_btree:add_remove( - NewSt#st.seq_tree, NewInfos, RemoveSeqs), + NewSt#st.seq_tree, NewInfos, RemoveSeqs + ), EMSortFd = couch_emsort:get_fd(NewSt#st.id_tree), {ok, LocSizes} = couch_file:append_terms(EMSortFd, NewInfos), - EMSortEntries = lists:zipwith(fun(FDI, {Loc, _}) -> - #full_doc_info{ - id = Id, - update_seq = Seq - } = FDI, - {{Id, Seq}, Loc} - end, NewInfos, LocSizes), + EMSortEntries = lists:zipwith( + fun(FDI, {Loc, _}) -> + #full_doc_info{ + id = Id, + update_seq = Seq + } = FDI, + {{Id, Seq}, Loc} + end, + NewInfos, + LocSizes + ), {ok, IdEms} = couch_emsort:add(NewSt#st.id_tree, EMSortEntries), update_compact_task(length(NewInfos)), - NewSt#st{id_tree=IdEms, seq_tree=SeqTree}. - + NewSt#st{id_tree = IdEms, seq_tree = SeqTree}. copy_doc_attachments(#st{} = SrcSt, SrcSp, DstSt) -> {ok, {BodyData, BinInfos0}} = couch_file:pread_term(SrcSt#st.fd, SrcSp), - BinInfos = case BinInfos0 of - _ when is_binary(BinInfos0) -> - couch_compress:decompress(BinInfos0); - _ when is_list(BinInfos0) -> - % pre 1.2 file format - BinInfos0 - end, + BinInfos = + case BinInfos0 of + _ when is_binary(BinInfos0) -> + couch_compress:decompress(BinInfos0); + _ when is_list(BinInfos0) -> + % pre 1.2 file format + BinInfos0 + end, % copy the bin values NewBinInfos = lists:map( - fun({Name, Type, BinSp, AttLen, RevPos, ExpectedMd5}) -> - % 010 UPGRADE CODE - {ok, SrcStream} = couch_bt_engine:open_read_stream(SrcSt, BinSp), - {ok, DstStream} = couch_bt_engine:open_write_stream(DstSt, []), - ok = couch_stream:copy(SrcStream, DstStream), - {NewStream, AttLen, AttLen, ActualMd5, _IdentityMd5} = - couch_stream:close(DstStream), - {ok, NewBinSp} = couch_stream:to_disk_term(NewStream), - couch_util:check_md5(ExpectedMd5, ActualMd5), - {Name, Type, NewBinSp, AttLen, AttLen, RevPos, ExpectedMd5, identity}; - ({Name, Type, BinSp, AttLen, DiskLen, RevPos, ExpectedMd5, Enc1}) -> - {ok, SrcStream} = couch_bt_engine:open_read_stream(SrcSt, BinSp), - {ok, DstStream} = couch_bt_engine:open_write_stream(DstSt, []), - ok = couch_stream:copy(SrcStream, DstStream), - {NewStream, AttLen, _, ActualMd5, _IdentityMd5} = - couch_stream:close(DstStream), - {ok, NewBinSp} = couch_stream:to_disk_term(NewStream), - couch_util:check_md5(ExpectedMd5, ActualMd5), - Enc = case Enc1 of - true -> - % 0110 UPGRADE CODE - gzip; - false -> - % 0110 UPGRADE CODE - identity; - _ -> - Enc1 - end, - {Name, Type, NewBinSp, AttLen, DiskLen, RevPos, ExpectedMd5, Enc} - end, BinInfos), + fun + ({Name, Type, BinSp, AttLen, RevPos, ExpectedMd5}) -> + % 010 UPGRADE CODE + {ok, SrcStream} = couch_bt_engine:open_read_stream(SrcSt, BinSp), + {ok, DstStream} = couch_bt_engine:open_write_stream(DstSt, []), + ok = couch_stream:copy(SrcStream, DstStream), + {NewStream, AttLen, AttLen, ActualMd5, _IdentityMd5} = + couch_stream:close(DstStream), + {ok, NewBinSp} = couch_stream:to_disk_term(NewStream), + couch_util:check_md5(ExpectedMd5, ActualMd5), + {Name, Type, NewBinSp, AttLen, AttLen, RevPos, ExpectedMd5, identity}; + ({Name, Type, BinSp, AttLen, DiskLen, RevPos, ExpectedMd5, Enc1}) -> + {ok, SrcStream} = couch_bt_engine:open_read_stream(SrcSt, BinSp), + {ok, DstStream} = couch_bt_engine:open_write_stream(DstSt, []), + ok = couch_stream:copy(SrcStream, DstStream), + {NewStream, AttLen, _, ActualMd5, _IdentityMd5} = + couch_stream:close(DstStream), + {ok, NewBinSp} = couch_stream:to_disk_term(NewStream), + couch_util:check_md5(ExpectedMd5, ActualMd5), + Enc = + case Enc1 of + true -> + % 0110 UPGRADE CODE + gzip; + false -> + % 0110 UPGRADE CODE + identity; + _ -> + Enc1 + end, + {Name, Type, NewBinSp, AttLen, DiskLen, RevPos, ExpectedMd5, Enc} + end, + BinInfos + ), {BodyData, NewBinInfos}. - sort_meta_data(#comp_st{new_st = St0} = CompSt) -> ?COMP_EVENT(md_sort_init), NumKVs = couch_emsort:num_kvs(St0#st.id_tree), @@ -521,7 +571,6 @@ sort_meta_data(#comp_st{new_st = St0} = CompSt) -> } }. - copy_meta_data(#comp_st{new_st = St} = CompSt) -> #st{ fd = Fd, @@ -537,11 +586,11 @@ copy_meta_data(#comp_st{new_st = St} = CompSt) -> ]), {ok, Iter} = couch_emsort:iter(Src), Acc0 = #merge_st{ - src_fd=SrcFd, - id_tree=IdTree0, - seq_tree=St#st.seq_tree, - rem_seqs=[], - locs=[] + src_fd = SrcFd, + id_tree = IdTree0, + seq_tree = St#st.seq_tree, + rem_seqs = [], + locs = [] }, ?COMP_EVENT(md_copy_init), NumKVs = couch_emsort:num_kvs(Src), @@ -566,7 +615,6 @@ copy_meta_data(#comp_st{new_st = St} = CompSt) -> } }. - compact_final_sync(#comp_st{new_st = St0} = CompSt) -> ?COMP_EVENT(before_final_sync), {ok, St1} = couch_bt_engine:commit_data(St0), @@ -575,7 +623,6 @@ compact_final_sync(#comp_st{new_st = St0} = CompSt) -> new_st = St1 }. - open_compaction_file(FilePath) -> case couch_file:open(FilePath, [nologifmissing]) of {ok, Fd} -> @@ -588,12 +635,10 @@ open_compaction_file(FilePath) -> {ok, Fd, nil} end. - reset_compaction_file(Fd, Header) -> ok = couch_file:truncate(Fd, 0), ok = couch_file:write_header(Fd, Header). - commit_compaction_data(#comp_st{new_st = St} = CompSt) -> % Compaction needs to write headers to both the data file % and the meta file so if we need to restart we can pick @@ -601,12 +646,10 @@ commit_compaction_data(#comp_st{new_st = St} = CompSt) -> CompSt#comp_st{ new_st = commit_compaction_data(St) }; - commit_compaction_data(#st{} = St) -> commit_compaction_data(St, couch_emsort:get_fd(St#st.id_tree)), commit_compaction_data(St, St#st.fd). - commit_compaction_data(#st{header = OldHeader} = St0, Fd) -> DataState = couch_bt_engine_header:id_tree_state(OldHeader), MetaFd = couch_emsort:get_fd(St0#st.id_tree), @@ -624,17 +667,15 @@ commit_compaction_data(#st{header = OldHeader} = St0, Fd) -> }, bind_emsort(St2, MetaFd, MetaState). - bind_emsort(St, Fd, nil) -> {ok, Ems} = couch_emsort:open(Fd), - St#st{id_tree=Ems}; + St#st{id_tree = Ems}; bind_emsort(St, Fd, {BB, _} = Root) when is_list(BB) -> % Upgrade clause when we find old compaction files bind_emsort(St, Fd, [{root, Root}]); bind_emsort(St, Fd, State) -> {ok, Ems} = couch_emsort:open(Fd, State), - St#st{id_tree=Ems}. - + St#st{id_tree = Ems}. bind_id_tree(St, Fd, State) -> {ok, IdBtree} = couch_btree:open(State, Fd, [ @@ -642,42 +683,41 @@ bind_id_tree(St, Fd, State) -> {join, fun couch_bt_engine:id_tree_join/2}, {reduce, fun couch_bt_engine:id_tree_reduce/2} ]), - St#st{id_tree=IdBtree}. - + St#st{id_tree = IdBtree}. merge_lookups(Infos, []) -> Infos; merge_lookups([], _) -> []; -merge_lookups([#doc_info{}=DI | RestInfos], [{ok, FDI} | RestLookups]) -> +merge_lookups([#doc_info{} = DI | RestInfos], [{ok, FDI} | RestLookups]) -> % Assert we've matched our lookups - if DI#doc_info.id == FDI#full_doc_info.id -> ok; true -> - erlang:error({mismatched_doc_infos, DI#doc_info.id}) + if + DI#doc_info.id == FDI#full_doc_info.id -> ok; + true -> erlang:error({mismatched_doc_infos, DI#doc_info.id}) end, [FDI | merge_lookups(RestInfos, RestLookups)]; merge_lookups([FDI | RestInfos], Lookups) -> [FDI | merge_lookups(RestInfos, Lookups)]. - -merge_docids(Iter, #merge_st{locs=Locs}=Acc) when length(Locs) > 1000 -> +merge_docids(Iter, #merge_st{locs = Locs} = Acc) when length(Locs) > 1000 -> #merge_st{ - src_fd=SrcFd, - id_tree=IdTree0, - seq_tree=SeqTree0, - rem_seqs=RemSeqs + src_fd = SrcFd, + id_tree = IdTree0, + seq_tree = SeqTree0, + rem_seqs = RemSeqs } = Acc, {ok, Infos} = couch_file:pread_terms(SrcFd, Locs), {ok, IdTree1} = couch_btree:add(IdTree0, Infos), {ok, SeqTree1} = couch_btree:add_remove(SeqTree0, [], RemSeqs), Acc1 = Acc#merge_st{ - id_tree=IdTree1, - seq_tree=SeqTree1, - rem_seqs=[], - locs=[] + id_tree = IdTree1, + seq_tree = SeqTree1, + rem_seqs = [], + locs = [] }, update_compact_task(length(Locs)), merge_docids(Iter, Acc1); -merge_docids(Iter, #merge_st{curr=Curr}=Acc) -> +merge_docids(Iter, #merge_st{curr = Curr} = Acc) -> case next_info(Iter, Curr, []) of {NextIter, NewCurr, Loc, Seqs} -> Acc1 = Acc#merge_st{ @@ -697,7 +737,6 @@ merge_docids(Iter, #merge_st{curr=Curr}=Acc) -> Acc end. - next_info(Iter, undefined, []) -> case couch_emsort:next(Iter) of {ok, {{Id, Seq}, Loc}, NextIter} -> @@ -715,15 +754,14 @@ next_info(Iter, {Id, Seq, Loc}, Seqs) -> {finished, Loc, Seqs} end. - update_compact_task(NumChanges) -> [Changes, Total] = couch_task_status:get([changes_done, total_changes]), Changes2 = Changes + NumChanges, - Progress = case Total of - 0 -> - 0; - _ -> - (Changes2 * 100) div Total - end, + Progress = + case Total of + 0 -> + 0; + _ -> + (Changes2 * 100) div Total + end, couch_task_status:update([{changes_done, Changes2}, {progress, Progress}]). - diff --git a/src/couch/src/couch_bt_engine_header.erl b/src/couch/src/couch_bt_engine_header.erl index 3f9f51821..e28f07723 100644 --- a/src/couch/src/couch_bt_engine_header.erl +++ b/src/couch/src/couch_bt_engine_header.erl @@ -12,7 +12,6 @@ -module(couch_bt_engine_header). - -export([ new/0, from/1, @@ -42,7 +41,6 @@ compacted_seq/1 ]). - % This should be updated anytime a header change happens that requires more % than filling in new defaults. % @@ -63,7 +61,8 @@ seq_tree_state = nil, local_tree_state = nil, purge_tree_state = nil, - purge_seq_tree_state = nil, %purge tree: purge_seq -> uuid + %purge tree: purge_seq -> uuid + purge_seq_tree_state = nil, security_ptr = nil, revs_limit = 1000, uuid, @@ -73,17 +72,14 @@ props_ptr }). - -define(PARTITION_DISK_VERSION, 8). - new() -> #db_header{ uuid = couch_uuids:random(), epochs = [{node(), 0}] }. - from(Header0) -> Header = upgrade(Header0), #db_header{ @@ -92,16 +88,15 @@ from(Header0) -> compacted_seq = Header#db_header.compacted_seq }. - is_header(Header) -> try upgrade(Header), true - catch _:_ -> - false + catch + _:_ -> + false end. - upgrade(Header) -> Funs = [ fun upgrade_tuple/1, @@ -110,93 +105,81 @@ upgrade(Header) -> fun upgrade_epochs/1, fun upgrade_compacted_seq/1 ], - lists:foldl(fun(F, HdrAcc) -> - F(HdrAcc) - end, Header, Funs). - + lists:foldl( + fun(F, HdrAcc) -> + F(HdrAcc) + end, + Header, + Funs + ). get(Header, Key) -> ?MODULE:get(Header, Key, undefined). - get(Header, Key, Default) -> get_field(Header, Key, Default). - set(Header, Key, Value) -> ?MODULE:set(Header, [{Key, Value}]). - set(Header0, Fields) -> % A subtlety here is that if a database was open during % the release upgrade that updates to uuids and epochs then % this dynamic upgrade also assigns a uuid and epoch. Header = upgrade(Header0), - lists:foldl(fun({Field, Value}, HdrAcc) -> - set_field(HdrAcc, Field, Value) - end, Header, Fields). - + lists:foldl( + fun({Field, Value}, HdrAcc) -> + set_field(HdrAcc, Field, Value) + end, + Header, + Fields + ). disk_version(Header) -> get_field(Header, disk_version). - latest_disk_version() -> - ?LATEST_DISK_VERSION. - + ?LATEST_DISK_VERSION. update_seq(Header) -> get_field(Header, update_seq). - id_tree_state(Header) -> get_field(Header, id_tree_state). - seq_tree_state(Header) -> get_field(Header, seq_tree_state). - local_tree_state(Header) -> get_field(Header, local_tree_state). - purge_tree_state(Header) -> get_field(Header, purge_tree_state). - purge_seq_tree_state(Header) -> get_field(Header, purge_seq_tree_state). - security_ptr(Header) -> get_field(Header, security_ptr). - revs_limit(Header) -> get_field(Header, revs_limit). - uuid(Header) -> get_field(Header, uuid). - epochs(Header) -> get_field(Header, epochs). - compacted_seq(Header) -> get_field(Header, compacted_seq). - purge_infos_limit(Header) -> get_field(Header, purge_infos_limit). - get_field(Header, Field) -> get_field(Header, Field, undefined). - get_field(Header, Field, Default) -> Idx = index(Field), case Idx > tuple_size(Header) of @@ -204,90 +187,103 @@ get_field(Header, Field, Default) -> false -> element(index(Field), Header) end. - set_field(Header, Field, Value) -> setelement(index(Field), Header, Value). - index(Field) -> couch_util:get_value(Field, indexes()). - indexes() -> Fields = record_info(fields, db_header), Indexes = lists:seq(2, record_info(size, db_header)), lists:zip(Fields, Indexes). - upgrade_tuple(Old) when is_record(Old, db_header) -> Old; upgrade_tuple(Old) when is_tuple(Old) -> NewSize = record_info(size, db_header), - if tuple_size(Old) < NewSize -> ok; true -> - erlang:error({invalid_header_size, Old}) + if + tuple_size(Old) < NewSize -> ok; + true -> erlang:error({invalid_header_size, Old}) end, - {_, New} = lists:foldl(fun(Val, {Idx, Hdr}) -> - {Idx+1, setelement(Idx, Hdr, Val)} - end, {1, #db_header{}}, tuple_to_list(Old)), - if is_record(New, db_header) -> ok; true -> - erlang:error({invalid_header_extension, {Old, New}}) + {_, New} = lists:foldl( + fun(Val, {Idx, Hdr}) -> + {Idx + 1, setelement(Idx, Hdr, Val)} + end, + {1, #db_header{}}, + tuple_to_list(Old) + ), + if + is_record(New, db_header) -> ok; + true -> erlang:error({invalid_header_extension, {Old, New}}) end, New. -define(OLD_DISK_VERSION_ERROR, - "Database files from versions smaller than 0.10.0 are no longer supported"). + "Database files from versions smaller than 0.10.0 are no longer supported" +). -upgrade_disk_version(#db_header{}=Header) -> +upgrade_disk_version(#db_header{} = Header) -> case element(2, Header) of - 1 -> throw({database_disk_version_error, ?OLD_DISK_VERSION_ERROR}); - 2 -> throw({database_disk_version_error, ?OLD_DISK_VERSION_ERROR}); - 3 -> throw({database_disk_version_error, ?OLD_DISK_VERSION_ERROR}); - 4 -> Header#db_header{security_ptr = nil}; % [0.10 - 0.11) - 5 -> Header#db_header{disk_version = ?LATEST_DISK_VERSION}; % pre 1.2 - 6 -> Header#db_header{disk_version = ?LATEST_DISK_VERSION}; % pre clustered purge - 7 -> Header#db_header{disk_version = ?LATEST_DISK_VERSION}; % pre partitioned dbs - ?LATEST_DISK_VERSION -> Header; + 1 -> + throw({database_disk_version_error, ?OLD_DISK_VERSION_ERROR}); + 2 -> + throw({database_disk_version_error, ?OLD_DISK_VERSION_ERROR}); + 3 -> + throw({database_disk_version_error, ?OLD_DISK_VERSION_ERROR}); + % [0.10 - 0.11) + 4 -> + Header#db_header{security_ptr = nil}; + % pre 1.2 + 5 -> + Header#db_header{disk_version = ?LATEST_DISK_VERSION}; + % pre clustered purge + 6 -> + Header#db_header{disk_version = ?LATEST_DISK_VERSION}; + % pre partitioned dbs + 7 -> + Header#db_header{disk_version = ?LATEST_DISK_VERSION}; + ?LATEST_DISK_VERSION -> + Header; _ -> Reason = "Incorrect disk header version", throw({database_disk_version_error, Reason}) end. - -upgrade_uuid(#db_header{}=Header) -> +upgrade_uuid(#db_header{} = Header) -> case Header#db_header.uuid of undefined -> % Upgrading this old db file to a newer % on disk format that includes a UUID. - Header#db_header{uuid=couch_uuids:random()}; + Header#db_header{uuid = couch_uuids:random()}; _ -> Header end. - -upgrade_epochs(#db_header{}=Header) -> - NewEpochs = case Header#db_header.epochs of - undefined -> - % This node is taking over ownership of shard with - % and old version of couch file. Before epochs there - % was always an implicit assumption that a file was - % owned since eternity by the node it was on. This - % just codifies that assumption. - [{node(), 0}]; - [{Node, _} | _] = Epochs0 when Node == node() -> - % Current node is the current owner of this db - Epochs0; - Epochs1 -> - % This node is taking over ownership of this db - % and marking the update sequence where it happened. - [{node(), Header#db_header.update_seq} | Epochs1] - end, +upgrade_epochs(#db_header{} = Header) -> + NewEpochs = + case Header#db_header.epochs of + undefined -> + % This node is taking over ownership of shard with + % and old version of couch file. Before epochs there + % was always an implicit assumption that a file was + % owned since eternity by the node it was on. This + % just codifies that assumption. + [{node(), 0}]; + [{Node, _} | _] = Epochs0 when Node == node() -> + % Current node is the current owner of this db + Epochs0; + Epochs1 -> + % This node is taking over ownership of this db + % and marking the update sequence where it happened. + [{node(), Header#db_header.update_seq} | Epochs1] + end, % Its possible for a node to open a db and claim % ownership but never make a write to the db. This % removes nodes that claimed ownership but never % changed the database. DedupedEpochs = remove_dup_epochs(NewEpochs), - Header#db_header{epochs=DedupedEpochs}. - + Header#db_header{epochs = DedupedEpochs}. % This is slightly relying on the udpate_seq's being sorted % in epochs due to how we only ever push things onto the @@ -296,12 +292,12 @@ upgrade_epochs(#db_header{}=Header) -> % want to remove dupes (by calling a sort on the input to this % function). So for now we don't sort but are relying on the % idea that epochs is always sorted. -remove_dup_epochs([_]=Epochs) -> +remove_dup_epochs([_] = Epochs) -> Epochs; remove_dup_epochs([{N1, S}, {_N2, S}]) -> % Seqs match, keep the most recent owner [{N1, S}]; -remove_dup_epochs([_, _]=Epochs) -> +remove_dup_epochs([_, _] = Epochs) -> % Seqs don't match. Epochs; remove_dup_epochs([{N1, S}, {_N2, S} | Rest]) -> @@ -311,11 +307,10 @@ remove_dup_epochs([{N1, S1}, {N2, S2} | Rest]) -> % Seqs don't match, recurse to check others [{N1, S1} | remove_dup_epochs([{N2, S2} | Rest])]. - -upgrade_compacted_seq(#db_header{}=Header) -> +upgrade_compacted_seq(#db_header{} = Header) -> case Header#db_header.compacted_seq of undefined -> - Header#db_header{compacted_seq=0}; + Header#db_header{compacted_seq = 0}; _ -> Header end. @@ -332,20 +327,30 @@ latest(_Else) -> mk_header(Vsn) -> { - db_header, % record name - Vsn, % disk version - 100, % update_seq - 0, % unused - foo, % id_tree_state - bar, % seq_tree_state - bam, % local_tree_state - flam, % was purge_seq - now purge_tree_state - baz, % was purged_docs - now purge_seq_tree_state - bang, % security_ptr - 999 % revs_limit + % record name + db_header, + % disk version + Vsn, + % update_seq + 100, + % unused + 0, + % id_tree_state + foo, + % seq_tree_state + bar, + % local_tree_state + bam, + % was purge_seq - now purge_tree_state + flam, + % was purged_docs - now purge_seq_tree_state + baz, + % security_ptr + bang, + % revs_limit + 999 }. - -ifdef(run_broken_tests). upgrade_v3_test() -> @@ -388,7 +393,6 @@ upgrade_v5_to_v8_test() -> % Security ptr isn't changed for v5 headers ?assertEqual(bang, security_ptr(NewHeader)). - upgrade_uuid_test() -> Vsn5Header = mk_header(5), @@ -404,7 +408,6 @@ upgrade_uuid_test() -> ResetHeader = from(NewNewHeader), ?assertEqual(uuid(NewHeader), uuid(ResetHeader)). - upgrade_epochs_test() -> Vsn5Header = mk_header(5), @@ -437,15 +440,12 @@ upgrade_epochs_test() -> ResetHeader = from(NewNewHeader), ?assertEqual(OwnedEpochs, epochs(ResetHeader)). - get_uuid_from_old_header_test() -> Vsn5Header = mk_header(5), ?assertEqual(undefined, uuid(Vsn5Header)). - get_epochs_from_old_header_test() -> Vsn5Header = mk_header(5), ?assertEqual(undefined, epochs(Vsn5Header)). - -endif. diff --git a/src/couch/src/couch_bt_engine_stream.erl b/src/couch/src/couch_bt_engine_stream.erl index 431894a50..253877e77 100644 --- a/src/couch/src/couch_bt_engine_stream.erl +++ b/src/couch/src/couch_bt_engine_stream.erl @@ -20,23 +20,18 @@ to_disk_term/1 ]). - foldl({_Fd, []}, _Fun, Acc) -> Acc; - foldl({Fd, [{Pos, _} | Rest]}, Fun, Acc) -> foldl({Fd, [Pos | Rest]}, Fun, Acc); - foldl({Fd, [Bin | Rest]}, Fun, Acc) when is_binary(Bin) -> % We're processing the first bit of data % after we did a seek for a range fold. foldl({Fd, Rest}, Fun, Fun(Bin, Acc)); - foldl({Fd, [Pos | Rest]}, Fun, Acc) when is_integer(Pos) -> {ok, Bin} = couch_file:pread_binary(Fd, Pos), foldl({Fd, Rest}, Fun, Fun(Bin, Acc)). - seek({Fd, [{Pos, Length} | Rest]}, Offset) -> case Length =< Offset of true -> @@ -44,7 +39,6 @@ seek({Fd, [{Pos, Length} | Rest]}, Offset) -> false -> seek({Fd, [Pos | Rest]}, Offset) end; - seek({Fd, [Pos | Rest]}, Offset) when is_integer(Pos) -> {ok, Bin} = couch_file:pread_binary(Fd, Pos), case iolist_size(Bin) =< Offset of @@ -55,16 +49,12 @@ seek({Fd, [Pos | Rest]}, Offset) when is_integer(Pos) -> {ok, {Fd, [Tail | Rest]}} end. - write({Fd, Written}, Data) when is_pid(Fd) -> {ok, Pos, _} = couch_file:append_binary(Fd, Data), {ok, {Fd, [{Pos, iolist_size(Data)} | Written]}}. - finalize({Fd, Written}) -> {ok, {Fd, lists:reverse(Written)}}. - to_disk_term({_Fd, Written}) -> {ok, Written}. - diff --git a/src/couch/src/couch_btree.erl b/src/couch/src/couch_btree.erl index 858ae2b90..b974a22ee 100644 --- a/src/couch/src/couch_btree.erl +++ b/src/couch/src/couch_btree.erl @@ -21,45 +21,45 @@ -define(FILL_RATIO, 0.5). -extract(#btree{extract_kv=undefined}, Value) -> +extract(#btree{extract_kv = undefined}, Value) -> Value; -extract(#btree{extract_kv=Extract}, Value) -> +extract(#btree{extract_kv = Extract}, Value) -> Extract(Value). -assemble(#btree{assemble_kv=undefined}, Key, Value) -> +assemble(#btree{assemble_kv = undefined}, Key, Value) -> {Key, Value}; -assemble(#btree{assemble_kv=Assemble}, Key, Value) -> +assemble(#btree{assemble_kv = Assemble}, Key, Value) -> Assemble(Key, Value). -less(#btree{less=undefined}, A, B) -> +less(#btree{less = undefined}, A, B) -> A < B; -less(#btree{less=Less}, A, B) -> +less(#btree{less = Less}, A, B) -> Less(A, B). % pass in 'nil' for State if a new Btree. open(State, Fd) -> - {ok, #btree{root=State, fd=Fd}}. + {ok, #btree{root = State, fd = Fd}}. set_options(Bt, []) -> Bt; -set_options(Bt, [{split, Extract}|Rest]) -> - set_options(Bt#btree{extract_kv=Extract}, Rest); -set_options(Bt, [{join, Assemble}|Rest]) -> - set_options(Bt#btree{assemble_kv=Assemble}, Rest); -set_options(Bt, [{less, Less}|Rest]) -> - set_options(Bt#btree{less=Less}, Rest); -set_options(Bt, [{reduce, Reduce}|Rest]) -> - set_options(Bt#btree{reduce=Reduce}, Rest); -set_options(Bt, [{compression, Comp}|Rest]) -> - set_options(Bt#btree{compression=Comp}, Rest). +set_options(Bt, [{split, Extract} | Rest]) -> + set_options(Bt#btree{extract_kv = Extract}, Rest); +set_options(Bt, [{join, Assemble} | Rest]) -> + set_options(Bt#btree{assemble_kv = Assemble}, Rest); +set_options(Bt, [{less, Less} | Rest]) -> + set_options(Bt#btree{less = Less}, Rest); +set_options(Bt, [{reduce, Reduce} | Rest]) -> + set_options(Bt#btree{reduce = Reduce}, Rest); +set_options(Bt, [{compression, Comp} | Rest]) -> + set_options(Bt#btree{compression = Comp}, Rest). open(State, Fd, Options) -> - {ok, set_options(#btree{root=State, fd=Fd}, Options)}. + {ok, set_options(#btree{root = State, fd = Fd}, Options)}. -get_state(#btree{root=Root}) -> +get_state(#btree{root = Root}) -> Root. -final_reduce(#btree{reduce=Reduce}, Val) -> +final_reduce(#btree{reduce = Reduce}, Val) -> final_reduce(Reduce, Val); final_reduce(Reduce, {[], []}) -> Reduce(reduce, []); @@ -71,30 +71,42 @@ final_reduce(Reduce, {KVs, Reductions}) -> Red = Reduce(reduce, KVs), final_reduce(Reduce, {[], [Red | Reductions]}). -fold_reduce(#btree{root=Root}=Bt, Fun, Acc, Options) -> +fold_reduce(#btree{root = Root} = Bt, Fun, Acc, Options) -> Dir = couch_util:get_value(dir, Options, fwd), StartKey = couch_util:get_value(start_key, Options), InEndRangeFun = make_key_in_end_range_function(Bt, Dir, Options), KeyGroupFun = get_group_fun(Bt, Options), try {ok, Acc2, GroupedRedsAcc2, GroupedKVsAcc2, GroupedKey2} = - reduce_stream_node(Bt, Dir, Root, StartKey, InEndRangeFun, undefined, [], [], - KeyGroupFun, Fun, Acc), - if GroupedKey2 == undefined -> - {ok, Acc2}; - true -> - case Fun(GroupedKey2, {GroupedKVsAcc2, GroupedRedsAcc2}, Acc2) of - {ok, Acc3} -> {ok, Acc3}; - {stop, Acc3} -> {ok, Acc3} - end + reduce_stream_node( + Bt, + Dir, + Root, + StartKey, + InEndRangeFun, + undefined, + [], + [], + KeyGroupFun, + Fun, + Acc + ), + if + GroupedKey2 == undefined -> + {ok, Acc2}; + true -> + case Fun(GroupedKey2, {GroupedKVsAcc2, GroupedRedsAcc2}, Acc2) of + {ok, Acc3} -> {ok, Acc3}; + {stop, Acc3} -> {ok, Acc3} + end end catch throw:{stop, AccDone} -> {ok, AccDone} end. -full_reduce(#btree{root=nil,reduce=Reduce}) -> +full_reduce(#btree{root = nil, reduce = Reduce}) -> {ok, Reduce(reduce, [])}; -full_reduce(#btree{root=Root}) -> +full_reduce(#btree{root = Root}) -> {ok, element(2, Root)}. size(#btree{root = nil}) -> @@ -114,7 +126,7 @@ get_group_fun(Bt, Options) -> N when is_integer(N), N > 0 -> make_group_fun(Bt, N); undefined -> - couch_util:get_value(key_group_fun, Options, fun(_,_) -> true end) + couch_util:get_value(key_group_fun, Options, fun(_, _) -> true end) end. make_group_fun(Bt, exact) -> @@ -135,7 +147,7 @@ make_group_fun(Bt, GroupLevel) when is_integer(GroupLevel), GroupLevel > 0 -> fun GF({{p, Partition, Key1}, Val1}, {{p, Partition, Key2}, Val2}) -> GF({Key1, Val1}, {Key2, Val2}); - GF({[_|_] = Key1, _}, {[_|_] = Key2, _}) -> + GF({[_ | _] = Key1, _}, {[_ | _] = Key2, _}) -> SL1 = lists:sublist(Key1, GroupLevel), SL2 = lists:sublist(Key2, GroupLevel), case less(Bt, {SL1, nil}, {SL2, nil}) of @@ -175,61 +187,75 @@ convert_fun_arity(Fun) when is_function(Fun, 3) -> (traverse, _K, _Red, AccIn) -> {ok, AccIn} end; convert_fun_arity(Fun) when is_function(Fun, 4) -> - Fun. % Already arity 4 + % Already arity 4 + Fun. make_key_in_end_range_function(Bt, fwd, Options) -> case couch_util:get_value(end_key_gt, Options) of - undefined -> - case couch_util:get_value(end_key, Options) of undefined -> - fun(_Key) -> true end; - LastKey -> - fun(Key) -> not less(Bt, LastKey, Key) end - end; - EndKey -> - fun(Key) -> less(Bt, Key, EndKey) end + case couch_util:get_value(end_key, Options) of + undefined -> + fun(_Key) -> true end; + LastKey -> + fun(Key) -> not less(Bt, LastKey, Key) end + end; + EndKey -> + fun(Key) -> less(Bt, Key, EndKey) end end; make_key_in_end_range_function(Bt, rev, Options) -> case couch_util:get_value(end_key_gt, Options) of - undefined -> - case couch_util:get_value(end_key, Options) of undefined -> - fun(_Key) -> true end; - LastKey -> - fun(Key) -> not less(Bt, Key, LastKey) end - end; - EndKey -> - fun(Key) -> less(Bt, EndKey, Key) end + case couch_util:get_value(end_key, Options) of + undefined -> + fun(_Key) -> true end; + LastKey -> + fun(Key) -> not less(Bt, Key, LastKey) end + end; + EndKey -> + fun(Key) -> less(Bt, EndKey, Key) end end. - foldl(Bt, Fun, Acc) -> fold(Bt, Fun, Acc, []). foldl(Bt, Fun, Acc, Options) -> fold(Bt, Fun, Acc, Options). - -fold(#btree{root=nil}, _Fun, Acc, _Options) -> +fold(#btree{root = nil}, _Fun, Acc, _Options) -> {ok, {[], []}, Acc}; -fold(#btree{root=Root}=Bt, Fun, Acc, Options) -> +fold(#btree{root = Root} = Bt, Fun, Acc, Options) -> Dir = couch_util:get_value(dir, Options, fwd), InRange = make_key_in_end_range_function(Bt, Dir, Options), Result = - case couch_util:get_value(start_key, Options) of - undefined -> - stream_node(Bt, [], Bt#btree.root, InRange, Dir, - convert_fun_arity(Fun), Acc); - StartKey -> - stream_node(Bt, [], Bt#btree.root, StartKey, InRange, Dir, - convert_fun_arity(Fun), Acc) - end, + case couch_util:get_value(start_key, Options) of + undefined -> + stream_node( + Bt, + [], + Bt#btree.root, + InRange, + Dir, + convert_fun_arity(Fun), + Acc + ); + StartKey -> + stream_node( + Bt, + [], + Bt#btree.root, + StartKey, + InRange, + Dir, + convert_fun_arity(Fun), + Acc + ) + end, case Result of - {ok, Acc2}-> - FullReduction = element(2, Root), - {ok, {[], [FullReduction]}, Acc2}; - {stop, LastReduction, Acc2} -> - {ok, LastReduction, Acc2} + {ok, Acc2} -> + FullReduction = element(2, Root), + {ok, {[], [FullReduction]}, Acc2}; + {stop, LastReduction, Acc2} -> + {ok, LastReduction, Acc2} end. add(Bt, InsertKeyValues) -> @@ -240,27 +266,28 @@ add_remove(Bt, InsertKeyValues, RemoveKeys) -> {ok, Bt2}. query_modify(Bt, LookupKeys, InsertValues, RemoveKeys) -> - #btree{root=Root} = Bt, + #btree{root = Root} = Bt, InsertActions = lists:map( fun(KeyValue) -> {Key, Value} = extract(Bt, KeyValue), {insert, Key, Value} - end, InsertValues), + end, + InsertValues + ), RemoveActions = [{remove, Key, nil} || Key <- RemoveKeys], FetchActions = [{fetch, Key, nil} || Key <- LookupKeys], SortFun = fun({OpA, A, _}, {OpB, B, _}) -> case A == B of - % A and B are equal, sort by op. - true -> op_order(OpA) < op_order(OpB); - false -> - less(Bt, A, B) + % A and B are equal, sort by op. + true -> op_order(OpA) < op_order(OpB); + false -> less(Bt, A, B) end end, Actions = lists:sort(SortFun, lists:append([InsertActions, RemoveActions, FetchActions])), {ok, KeyPointers, QueryResults} = modify_node(Bt, Root, Actions, []), {ok, NewRoot} = complete_root(Bt, KeyPointers), - {ok, QueryResults, Bt#btree{root=NewRoot}}. + {ok, QueryResults, Bt#btree{root = NewRoot}}. % for ordering different operations with the same key. % fetch < remove < insert @@ -268,11 +295,12 @@ op_order(fetch) -> 1; op_order(remove) -> 2; op_order(insert) -> 3. -lookup(#btree{root=Root, less=Less}=Bt, Keys) -> - SortedKeys = case Less of - undefined -> lists:sort(Keys); - _ -> lists:sort(Less, Keys) - end, +lookup(#btree{root = Root, less = Less} = Bt, Keys) -> + SortedKeys = + case Less of + undefined -> lists:sort(Keys); + _ -> lists:sort(Less, Keys) + end, {ok, SortedResults} = lookup(Bt, Root, SortedKeys), % We want to return the results in the same order as the keys were input % but we may have changed the order when we sorted. So we need to put the @@ -285,10 +313,10 @@ lookup(Bt, Node, Keys) -> Pointer = element(1, Node), {NodeType, NodeList} = get_node(Bt, Pointer), case NodeType of - kp_node -> - lookup_kpnode(Bt, list_to_tuple(NodeList), 1, Keys, []); - kv_node -> - lookup_kvnode(Bt, list_to_tuple(NodeList), 1, Keys, []) + kp_node -> + lookup_kpnode(Bt, list_to_tuple(NodeList), 1, Keys, []); + kv_node -> + lookup_kvnode(Bt, list_to_tuple(NodeList), 1, Keys, []) end. lookup_kpnode(_Bt, _NodeTuple, _LowerBound, [], Output) -> @@ -300,14 +328,13 @@ lookup_kpnode(Bt, NodeTuple, LowerBound, [FirstLookupKey | _] = LookupKeys, Outp {Key, PointerInfo} = element(N, NodeTuple), SplitFun = fun(LookupKey) -> not less(Bt, Key, LookupKey) end, case lists:splitwith(SplitFun, LookupKeys) of - {[], GreaterQueries} -> - lookup_kpnode(Bt, NodeTuple, N + 1, GreaterQueries, Output); - {LessEqQueries, GreaterQueries} -> - {ok, Results} = lookup(Bt, PointerInfo, LessEqQueries), - lookup_kpnode(Bt, NodeTuple, N + 1, GreaterQueries, lists:reverse(Results, Output)) + {[], GreaterQueries} -> + lookup_kpnode(Bt, NodeTuple, N + 1, GreaterQueries, Output); + {LessEqQueries, GreaterQueries} -> + {ok, Results} = lookup(Bt, PointerInfo, LessEqQueries), + lookup_kpnode(Bt, NodeTuple, N + 1, GreaterQueries, lists:reverse(Results, Output)) end. - lookup_kvnode(_Bt, _NodeTuple, _LowerBound, [], Output) -> {ok, lists:reverse(Output)}; lookup_kvnode(_Bt, NodeTuple, LowerBound, Keys, Output) when tuple_size(NodeTuple) < LowerBound -> @@ -317,24 +344,27 @@ lookup_kvnode(Bt, NodeTuple, LowerBound, [LookupKey | RestLookupKeys], Output) - N = find_first_gteq(Bt, NodeTuple, LowerBound, tuple_size(NodeTuple), LookupKey), {Key, Value} = element(N, NodeTuple), case less(Bt, LookupKey, Key) of - true -> - % LookupKey is less than Key - lookup_kvnode(Bt, NodeTuple, N, RestLookupKeys, [{LookupKey, not_found} | Output]); - false -> - case less(Bt, Key, LookupKey) of true -> - % LookupKey is greater than Key - lookup_kvnode(Bt, NodeTuple, N+1, RestLookupKeys, [{LookupKey, not_found} | Output]); + % LookupKey is less than Key + lookup_kvnode(Bt, NodeTuple, N, RestLookupKeys, [{LookupKey, not_found} | Output]); false -> - % LookupKey is equal to Key - lookup_kvnode(Bt, NodeTuple, N, RestLookupKeys, [{LookupKey, {ok, assemble(Bt, LookupKey, Value)}} | Output]) - end + case less(Bt, Key, LookupKey) of + true -> + % LookupKey is greater than Key + lookup_kvnode(Bt, NodeTuple, N + 1, RestLookupKeys, [ + {LookupKey, not_found} | Output + ]); + false -> + % LookupKey is equal to Key + lookup_kvnode(Bt, NodeTuple, N, RestLookupKeys, [ + {LookupKey, {ok, assemble(Bt, LookupKey, Value)}} | Output + ]) + end end. - complete_root(_Bt, []) -> {ok, nil}; -complete_root(_Bt, [{_Key, PointerInfo}])-> +complete_root(_Bt, [{_Key, PointerInfo}]) -> {ok, PointerInfo}; complete_root(Bt, KPs) -> {ok, ResultKeyPointers} = write_node(Bt, kp_node, KPs), @@ -348,12 +378,12 @@ complete_root(Bt, KPs) -> chunkify(InList) -> BaseChunkSize = get_chunk_size(), case ?term_size(InList) of - Size when Size > BaseChunkSize -> - NumberOfChunksLikely = ((Size div BaseChunkSize) + 1), - ChunkThreshold = Size div NumberOfChunksLikely, - chunkify(InList, ChunkThreshold, [], 0, []); - _Else -> - [InList] + Size when Size > BaseChunkSize -> + NumberOfChunksLikely = ((Size div BaseChunkSize) + 1), + ChunkThreshold = Size div NumberOfChunksLikely, + chunkify(InList, ChunkThreshold, [], 0, []); + _Else -> + [InList] end. chunkify([], _ChunkThreshold, [], 0, OutputChunks) -> @@ -365,58 +395,67 @@ chunkify([], _ChunkThreshold, OutList, _OutListSize, OutputChunks) -> lists:reverse([lists:reverse(OutList) | OutputChunks]); chunkify([InElement | RestInList], ChunkThreshold, OutList, OutListSize, OutputChunks) -> case ?term_size(InElement) of - Size when (Size + OutListSize) > ChunkThreshold andalso OutList /= [] -> - chunkify(RestInList, ChunkThreshold, [], 0, [lists:reverse([InElement | OutList]) | OutputChunks]); - Size -> - chunkify(RestInList, ChunkThreshold, [InElement | OutList], OutListSize + Size, OutputChunks) + Size when (Size + OutListSize) > ChunkThreshold andalso OutList /= [] -> + chunkify(RestInList, ChunkThreshold, [], 0, [ + lists:reverse([InElement | OutList]) | OutputChunks + ]); + Size -> + chunkify( + RestInList, ChunkThreshold, [InElement | OutList], OutListSize + Size, OutputChunks + ) end. --compile({inline,[get_chunk_size/0]}). +-compile({inline, [get_chunk_size/0]}). get_chunk_size() -> try list_to_integer(config:get("couchdb", "btree_chunk_size", "1279")) - catch error:badarg -> - 1279 + catch + error:badarg -> + 1279 end. modify_node(Bt, RootPointerInfo, Actions, QueryOutput) -> - {NodeType, NodeList} = case RootPointerInfo of - nil -> - {kv_node, []}; - _Tuple -> - Pointer = element(1, RootPointerInfo), - get_node(Bt, Pointer) - end, + {NodeType, NodeList} = + case RootPointerInfo of + nil -> + {kv_node, []}; + _Tuple -> + Pointer = element(1, RootPointerInfo), + get_node(Bt, Pointer) + end, NodeTuple = list_to_tuple(NodeList), {ok, NewNodeList, QueryOutput2} = - case NodeType of - kp_node -> modify_kpnode(Bt, NodeTuple, 1, Actions, [], QueryOutput); - kv_node -> modify_kvnode(Bt, NodeTuple, 1, Actions, [], QueryOutput) - end, + case NodeType of + kp_node -> modify_kpnode(Bt, NodeTuple, 1, Actions, [], QueryOutput); + kv_node -> modify_kvnode(Bt, NodeTuple, 1, Actions, [], QueryOutput) + end, case NewNodeList of - [] -> % no nodes remain - {ok, [], QueryOutput2}; - NodeList -> % nothing changed - {LastKey, _LastValue} = element(tuple_size(NodeTuple), NodeTuple), - {ok, [{LastKey, RootPointerInfo}], QueryOutput2}; - _Else2 -> - {ok, ResultList} = case RootPointerInfo of - nil -> - write_node(Bt, NodeType, NewNodeList); - _ -> + % no nodes remain + [] -> + {ok, [], QueryOutput2}; + % nothing changed + NodeList -> {LastKey, _LastValue} = element(tuple_size(NodeTuple), NodeTuple), - OldNode = {LastKey, RootPointerInfo}, - write_node(Bt, OldNode, NodeType, NodeList, NewNodeList) - end, - {ok, ResultList, QueryOutput2} + {ok, [{LastKey, RootPointerInfo}], QueryOutput2}; + _Else2 -> + {ok, ResultList} = + case RootPointerInfo of + nil -> + write_node(Bt, NodeType, NewNodeList); + _ -> + {LastKey, _LastValue} = element(tuple_size(NodeTuple), NodeTuple), + OldNode = {LastKey, RootPointerInfo}, + write_node(Bt, OldNode, NodeType, NodeList, NewNodeList) + end, + {ok, ResultList, QueryOutput2} end. -reduce_node(#btree{reduce=nil}, _NodeType, _NodeList) -> +reduce_node(#btree{reduce = nil}, _NodeType, _NodeList) -> []; -reduce_node(#btree{reduce=R}, kp_node, NodeList) -> +reduce_node(#btree{reduce = R}, kp_node, NodeList) -> R(rereduce, [element(2, Node) || {_K, Node} <- NodeList]); -reduce_node(#btree{reduce=R}=Bt, kv_node, NodeList) -> +reduce_node(#btree{reduce = R} = Bt, kv_node, NodeList) -> R(reduce, [assemble(Bt, K, V) || {K, V} <- NodeList]). reduce_tree_size(kv_node, NodeSize, _KvList) -> @@ -444,17 +483,14 @@ write_node(#btree{fd = Fd, compression = Comp} = Bt, NodeType, NodeList) -> {ok, PtrSizes} = couch_file:append_terms(Fd, ToWrite, WriteOpts), {ok, group_kps(Bt, NodeType, Chunks, PtrSizes)}. - group_kps(_Bt, _NodeType, [], []) -> []; - group_kps(Bt, NodeType, [Chunk | RestChunks], [{Ptr, Size} | RestPtrSizes]) -> {LastKey, _} = lists:last(Chunk), SubTreeSize = reduce_tree_size(NodeType, Size, Chunk), KP = {LastKey, {Ptr, reduce_node(Bt, NodeType, Chunk), SubTreeSize}}, [KP | group_kps(Bt, NodeType, RestChunks, RestPtrSizes)]. - write_node(Bt, _OldNode, NodeType, [], NewList) -> write_node(Bt, NodeType, NewList); write_node(Bt, _OldNode, NodeType, [_], NewList) -> @@ -462,14 +498,16 @@ write_node(Bt, _OldNode, NodeType, [_], NewList) -> write_node(Bt, OldNode, NodeType, OldList, NewList) -> case can_reuse_old_node(OldList, NewList) of {true, Prefix, Suffix} -> - {ok, PrefixKVs} = case Prefix of - [] -> {ok, []}; - _ -> write_node(Bt, NodeType, Prefix) - end, - {ok, SuffixKVs} = case Suffix of - [] -> {ok, []}; - _ -> write_node(Bt, NodeType, Suffix) - end, + {ok, PrefixKVs} = + case Prefix of + [] -> {ok, []}; + _ -> write_node(Bt, NodeType, Prefix) + end, + {ok, SuffixKVs} = + case Suffix of + [] -> {ok, []}; + _ -> write_node(Bt, NodeType, Suffix) + end, Result = PrefixKVs ++ [OldNode] ++ SuffixKVs, {ok, Result}; false -> @@ -481,8 +519,9 @@ can_reuse_old_node(OldList, NewList) -> case old_list_is_prefix(OldList, RestNewList, 0) of {true, Size, Suffix} -> ReuseThreshold = get_chunk_size() * ?FILL_RATIO, - if Size < ReuseThreshold -> false; true -> - {true, Prefix, Suffix} + if + Size < ReuseThreshold -> false; + true -> {true, Prefix, Suffix} end; false -> false @@ -510,38 +549,67 @@ old_list_is_prefix(_OldList, _NewList, _Acc) -> modify_kpnode(Bt, {}, _LowerBound, Actions, [], QueryOutput) -> modify_node(Bt, nil, Actions, QueryOutput); modify_kpnode(_Bt, NodeTuple, LowerBound, [], ResultNode, QueryOutput) -> - {ok, lists:reverse(ResultNode, bounded_tuple_to_list(NodeTuple, LowerBound, - tuple_size(NodeTuple), [])), QueryOutput}; -modify_kpnode(Bt, NodeTuple, LowerBound, - [{_, FirstActionKey, _}|_]=Actions, ResultNode, QueryOutput) -> + {ok, + lists:reverse( + ResultNode, + bounded_tuple_to_list( + NodeTuple, + LowerBound, + tuple_size(NodeTuple), + [] + ) + ), + QueryOutput}; +modify_kpnode( + Bt, + NodeTuple, + LowerBound, + [{_, FirstActionKey, _} | _] = Actions, + ResultNode, + QueryOutput +) -> Sz = tuple_size(NodeTuple), N = find_first_gteq(Bt, NodeTuple, LowerBound, Sz, FirstActionKey), case N =:= Sz of - true -> - % perform remaining actions on last node - {_, PointerInfo} = element(Sz, NodeTuple), - {ok, ChildKPs, QueryOutput2} = - modify_node(Bt, PointerInfo, Actions, QueryOutput), - NodeList = lists:reverse(ResultNode, bounded_tuple_to_list(NodeTuple, LowerBound, - Sz - 1, ChildKPs)), - {ok, NodeList, QueryOutput2}; - false -> - {NodeKey, PointerInfo} = element(N, NodeTuple), - SplitFun = fun({_ActionType, ActionKey, _ActionValue}) -> + true -> + % perform remaining actions on last node + {_, PointerInfo} = element(Sz, NodeTuple), + {ok, ChildKPs, QueryOutput2} = + modify_node(Bt, PointerInfo, Actions, QueryOutput), + NodeList = lists:reverse( + ResultNode, + bounded_tuple_to_list( + NodeTuple, + LowerBound, + Sz - 1, + ChildKPs + ) + ), + {ok, NodeList, QueryOutput2}; + false -> + {NodeKey, PointerInfo} = element(N, NodeTuple), + SplitFun = fun({_ActionType, ActionKey, _ActionValue}) -> not less(Bt, NodeKey, ActionKey) end, - {LessEqQueries, GreaterQueries} = lists:splitwith(SplitFun, Actions), - {ok, ChildKPs, QueryOutput2} = + {LessEqQueries, GreaterQueries} = lists:splitwith(SplitFun, Actions), + {ok, ChildKPs, QueryOutput2} = modify_node(Bt, PointerInfo, LessEqQueries, QueryOutput), - ResultNode2 = lists:reverse(ChildKPs, bounded_tuple_to_revlist(NodeTuple, - LowerBound, N - 1, ResultNode)), - modify_kpnode(Bt, NodeTuple, N+1, GreaterQueries, ResultNode2, QueryOutput2) + ResultNode2 = lists:reverse( + ChildKPs, + bounded_tuple_to_revlist( + NodeTuple, + LowerBound, + N - 1, + ResultNode + ) + ), + modify_kpnode(Bt, NodeTuple, N + 1, GreaterQueries, ResultNode2, QueryOutput2) end. bounded_tuple_to_revlist(_Tuple, Start, End, Tail) when Start > End -> Tail; bounded_tuple_to_revlist(Tuple, Start, End, Tail) -> - bounded_tuple_to_revlist(Tuple, Start+1, End, [element(Start, Tuple)|Tail]). + bounded_tuple_to_revlist(Tuple, Start + 1, End, [element(Start, Tuple) | Tail]). bounded_tuple_to_list(Tuple, Start, End, Tail) -> bounded_tuple_to_list2(Tuple, Start, End, [], Tail). @@ -557,191 +625,438 @@ find_first_gteq(Bt, Tuple, Start, End, Key) -> Mid = Start + ((End - Start) div 2), {TupleKey, _} = element(Mid, Tuple), case less(Bt, TupleKey, Key) of - true -> - find_first_gteq(Bt, Tuple, Mid+1, End, Key); - false -> - find_first_gteq(Bt, Tuple, Start, Mid, Key) + true -> + find_first_gteq(Bt, Tuple, Mid + 1, End, Key); + false -> + find_first_gteq(Bt, Tuple, Start, Mid, Key) end. modify_kvnode(_Bt, NodeTuple, LowerBound, [], ResultNode, QueryOutput) -> - {ok, lists:reverse(ResultNode, bounded_tuple_to_list(NodeTuple, LowerBound, tuple_size(NodeTuple), [])), QueryOutput}; -modify_kvnode(Bt, NodeTuple, LowerBound, [{ActionType, ActionKey, ActionValue} | RestActions], ResultNode, QueryOutput) when LowerBound > tuple_size(NodeTuple) -> + {ok, + lists:reverse( + ResultNode, bounded_tuple_to_list(NodeTuple, LowerBound, tuple_size(NodeTuple), []) + ), + QueryOutput}; +modify_kvnode( + Bt, + NodeTuple, + LowerBound, + [{ActionType, ActionKey, ActionValue} | RestActions], + ResultNode, + QueryOutput +) when LowerBound > tuple_size(NodeTuple) -> case ActionType of - insert -> - modify_kvnode(Bt, NodeTuple, LowerBound, RestActions, [{ActionKey, ActionValue} | ResultNode], QueryOutput); - remove -> - % just drop the action - modify_kvnode(Bt, NodeTuple, LowerBound, RestActions, ResultNode, QueryOutput); - fetch -> - % the key/value must not exist in the tree - modify_kvnode(Bt, NodeTuple, LowerBound, RestActions, ResultNode, [{not_found, {ActionKey, nil}} | QueryOutput]) + insert -> + modify_kvnode( + Bt, + NodeTuple, + LowerBound, + RestActions, + [{ActionKey, ActionValue} | ResultNode], + QueryOutput + ); + remove -> + % just drop the action + modify_kvnode(Bt, NodeTuple, LowerBound, RestActions, ResultNode, QueryOutput); + fetch -> + % the key/value must not exist in the tree + modify_kvnode(Bt, NodeTuple, LowerBound, RestActions, ResultNode, [ + {not_found, {ActionKey, nil}} | QueryOutput + ]) end; -modify_kvnode(Bt, NodeTuple, LowerBound, [{ActionType, ActionKey, ActionValue} | RestActions], AccNode, QueryOutput) -> +modify_kvnode( + Bt, + NodeTuple, + LowerBound, + [{ActionType, ActionKey, ActionValue} | RestActions], + AccNode, + QueryOutput +) -> N = find_first_gteq(Bt, NodeTuple, LowerBound, tuple_size(NodeTuple), ActionKey), {Key, Value} = element(N, NodeTuple), - ResultNode = bounded_tuple_to_revlist(NodeTuple, LowerBound, N - 1, AccNode), + ResultNode = bounded_tuple_to_revlist(NodeTuple, LowerBound, N - 1, AccNode), case less(Bt, ActionKey, Key) of - true -> - case ActionType of - insert -> - % ActionKey is less than the Key, so insert - modify_kvnode(Bt, NodeTuple, N, RestActions, [{ActionKey, ActionValue} | ResultNode], QueryOutput); - remove -> - % ActionKey is less than the Key, just drop the action - modify_kvnode(Bt, NodeTuple, N, RestActions, ResultNode, QueryOutput); - fetch -> - % ActionKey is less than the Key, the key/value must not exist in the tree - modify_kvnode(Bt, NodeTuple, N, RestActions, ResultNode, [{not_found, {ActionKey, nil}} | QueryOutput]) - end; - false -> - % ActionKey and Key are maybe equal. - case less(Bt, Key, ActionKey) of - false -> + true -> case ActionType of - insert -> - modify_kvnode(Bt, NodeTuple, N+1, RestActions, [{ActionKey, ActionValue} | ResultNode], QueryOutput); - remove -> - modify_kvnode(Bt, NodeTuple, N+1, RestActions, ResultNode, QueryOutput); - fetch -> - % ActionKey is equal to the Key, insert into the QueryOuput, but re-process the node - % since an identical action key can follow it. - modify_kvnode(Bt, NodeTuple, N, RestActions, ResultNode, [{ok, assemble(Bt, Key, Value)} | QueryOutput]) + insert -> + % ActionKey is less than the Key, so insert + modify_kvnode( + Bt, + NodeTuple, + N, + RestActions, + [{ActionKey, ActionValue} | ResultNode], + QueryOutput + ); + remove -> + % ActionKey is less than the Key, just drop the action + modify_kvnode(Bt, NodeTuple, N, RestActions, ResultNode, QueryOutput); + fetch -> + % ActionKey is less than the Key, the key/value must not exist in the tree + modify_kvnode(Bt, NodeTuple, N, RestActions, ResultNode, [ + {not_found, {ActionKey, nil}} | QueryOutput + ]) end; - true -> - modify_kvnode(Bt, NodeTuple, N + 1, [{ActionType, ActionKey, ActionValue} | RestActions], [{Key, Value} | ResultNode], QueryOutput) - end + false -> + % ActionKey and Key are maybe equal. + case less(Bt, Key, ActionKey) of + false -> + case ActionType of + insert -> + modify_kvnode( + Bt, + NodeTuple, + N + 1, + RestActions, + [{ActionKey, ActionValue} | ResultNode], + QueryOutput + ); + remove -> + modify_kvnode( + Bt, NodeTuple, N + 1, RestActions, ResultNode, QueryOutput + ); + fetch -> + % ActionKey is equal to the Key, insert into the QueryOuput, but re-process the node + % since an identical action key can follow it. + modify_kvnode(Bt, NodeTuple, N, RestActions, ResultNode, [ + {ok, assemble(Bt, Key, Value)} | QueryOutput + ]) + end; + true -> + modify_kvnode( + Bt, + NodeTuple, + N + 1, + [{ActionType, ActionKey, ActionValue} | RestActions], + [{Key, Value} | ResultNode], + QueryOutput + ) + end end. - -reduce_stream_node(_Bt, _Dir, nil, _KeyStart, _InEndRangeFun, GroupedKey, GroupedKVsAcc, - GroupedRedsAcc, _KeyGroupFun, _Fun, Acc) -> +reduce_stream_node( + _Bt, + _Dir, + nil, + _KeyStart, + _InEndRangeFun, + GroupedKey, + GroupedKVsAcc, + GroupedRedsAcc, + _KeyGroupFun, + _Fun, + Acc +) -> {ok, Acc, GroupedRedsAcc, GroupedKVsAcc, GroupedKey}; -reduce_stream_node(Bt, Dir, Node, KeyStart, InEndRangeFun, GroupedKey, GroupedKVsAcc, - GroupedRedsAcc, KeyGroupFun, Fun, Acc) -> +reduce_stream_node( + Bt, + Dir, + Node, + KeyStart, + InEndRangeFun, + GroupedKey, + GroupedKVsAcc, + GroupedRedsAcc, + KeyGroupFun, + Fun, + Acc +) -> P = element(1, Node), case get_node(Bt, P) of - {kp_node, NodeList} -> - NodeList2 = adjust_dir(Dir, NodeList), - reduce_stream_kp_node(Bt, Dir, NodeList2, KeyStart, InEndRangeFun, GroupedKey, - GroupedKVsAcc, GroupedRedsAcc, KeyGroupFun, Fun, Acc); - {kv_node, KVs} -> - KVs2 = adjust_dir(Dir, KVs), - reduce_stream_kv_node(Bt, Dir, KVs2, KeyStart, InEndRangeFun, GroupedKey, - GroupedKVsAcc, GroupedRedsAcc, KeyGroupFun, Fun, Acc) + {kp_node, NodeList} -> + NodeList2 = adjust_dir(Dir, NodeList), + reduce_stream_kp_node( + Bt, + Dir, + NodeList2, + KeyStart, + InEndRangeFun, + GroupedKey, + GroupedKVsAcc, + GroupedRedsAcc, + KeyGroupFun, + Fun, + Acc + ); + {kv_node, KVs} -> + KVs2 = adjust_dir(Dir, KVs), + reduce_stream_kv_node( + Bt, + Dir, + KVs2, + KeyStart, + InEndRangeFun, + GroupedKey, + GroupedKVsAcc, + GroupedRedsAcc, + KeyGroupFun, + Fun, + Acc + ) end. -reduce_stream_kv_node(Bt, Dir, KVs, KeyStart, InEndRangeFun, - GroupedKey, GroupedKVsAcc, GroupedRedsAcc, - KeyGroupFun, Fun, Acc) -> - +reduce_stream_kv_node( + Bt, + Dir, + KVs, + KeyStart, + InEndRangeFun, + GroupedKey, + GroupedKVsAcc, + GroupedRedsAcc, + KeyGroupFun, + Fun, + Acc +) -> GTEKeyStartKVs = - case KeyStart of - undefined -> - KVs; - _ -> - DropFun = case Dir of - fwd -> - fun({Key, _}) -> less(Bt, Key, KeyStart) end; - rev -> - fun({Key, _}) -> less(Bt, KeyStart, Key) end + case KeyStart of + undefined -> + KVs; + _ -> + DropFun = + case Dir of + fwd -> + fun({Key, _}) -> less(Bt, Key, KeyStart) end; + rev -> + fun({Key, _}) -> less(Bt, KeyStart, Key) end + end, + lists:dropwhile(DropFun, KVs) end, - lists:dropwhile(DropFun, KVs) - end, KVs2 = lists:takewhile( - fun({Key, _}) -> InEndRangeFun(Key) end, GTEKeyStartKVs), - reduce_stream_kv_node2(Bt, KVs2, GroupedKey, GroupedKVsAcc, GroupedRedsAcc, - KeyGroupFun, Fun, Acc). - - -reduce_stream_kv_node2(_Bt, [], GroupedKey, GroupedKVsAcc, GroupedRedsAcc, - _KeyGroupFun, _Fun, Acc) -> + fun({Key, _}) -> InEndRangeFun(Key) end, GTEKeyStartKVs + ), + reduce_stream_kv_node2( + Bt, + KVs2, + GroupedKey, + GroupedKVsAcc, + GroupedRedsAcc, + KeyGroupFun, + Fun, + Acc + ). + +reduce_stream_kv_node2( + _Bt, + [], + GroupedKey, + GroupedKVsAcc, + GroupedRedsAcc, + _KeyGroupFun, + _Fun, + Acc +) -> {ok, Acc, GroupedRedsAcc, GroupedKVsAcc, GroupedKey}; -reduce_stream_kv_node2(Bt, [{Key, Value}| RestKVs], GroupedKey, GroupedKVsAcc, - GroupedRedsAcc, KeyGroupFun, Fun, Acc) -> +reduce_stream_kv_node2( + Bt, + [{Key, Value} | RestKVs], + GroupedKey, + GroupedKVsAcc, + GroupedRedsAcc, + KeyGroupFun, + Fun, + Acc +) -> case GroupedKey of - undefined -> - reduce_stream_kv_node2(Bt, RestKVs, Key, - [assemble(Bt,Key,Value)], [], KeyGroupFun, Fun, Acc); - _ -> - - case KeyGroupFun(GroupedKey, Key) of - true -> - reduce_stream_kv_node2(Bt, RestKVs, GroupedKey, - [assemble(Bt,Key,Value)|GroupedKVsAcc], GroupedRedsAcc, KeyGroupFun, - Fun, Acc); - false -> - case Fun(GroupedKey, {GroupedKVsAcc, GroupedRedsAcc}, Acc) of - {ok, Acc2} -> - reduce_stream_kv_node2(Bt, RestKVs, Key, [assemble(Bt,Key,Value)], - [], KeyGroupFun, Fun, Acc2); - {stop, Acc2} -> - throw({stop, Acc2}) + undefined -> + reduce_stream_kv_node2( + Bt, + RestKVs, + Key, + [assemble(Bt, Key, Value)], + [], + KeyGroupFun, + Fun, + Acc + ); + _ -> + case KeyGroupFun(GroupedKey, Key) of + true -> + reduce_stream_kv_node2( + Bt, + RestKVs, + GroupedKey, + [assemble(Bt, Key, Value) | GroupedKVsAcc], + GroupedRedsAcc, + KeyGroupFun, + Fun, + Acc + ); + false -> + case Fun(GroupedKey, {GroupedKVsAcc, GroupedRedsAcc}, Acc) of + {ok, Acc2} -> + reduce_stream_kv_node2( + Bt, + RestKVs, + Key, + [assemble(Bt, Key, Value)], + [], + KeyGroupFun, + Fun, + Acc2 + ); + {stop, Acc2} -> + throw({stop, Acc2}) + end end - end end. -reduce_stream_kp_node(Bt, Dir, NodeList, KeyStart, InEndRangeFun, - GroupedKey, GroupedKVsAcc, GroupedRedsAcc, - KeyGroupFun, Fun, Acc) -> +reduce_stream_kp_node( + Bt, + Dir, + NodeList, + KeyStart, + InEndRangeFun, + GroupedKey, + GroupedKVsAcc, + GroupedRedsAcc, + KeyGroupFun, + Fun, + Acc +) -> Nodes = - case KeyStart of - undefined -> - NodeList; - _ -> - case Dir of - fwd -> - lists:dropwhile(fun({Key, _}) -> less(Bt, Key, KeyStart) end, NodeList); - rev -> - RevKPs = lists:reverse(NodeList), - case lists:splitwith(fun({Key, _}) -> less(Bt, Key, KeyStart) end, RevKPs) of - {_Before, []} -> + case KeyStart of + undefined -> NodeList; - {Before, [FirstAfter | _]} -> - [FirstAfter | lists:reverse(Before)] - end - end - end, + _ -> + case Dir of + fwd -> + lists:dropwhile(fun({Key, _}) -> less(Bt, Key, KeyStart) end, NodeList); + rev -> + RevKPs = lists:reverse(NodeList), + case + lists:splitwith(fun({Key, _}) -> less(Bt, Key, KeyStart) end, RevKPs) + of + {_Before, []} -> + NodeList; + {Before, [FirstAfter | _]} -> + [FirstAfter | lists:reverse(Before)] + end + end + end, {InRange, MaybeInRange} = lists:splitwith( - fun({Key, _}) -> InEndRangeFun(Key) end, Nodes), - NodesInRange = case MaybeInRange of - [FirstMaybeInRange | _] when Dir =:= fwd -> - InRange ++ [FirstMaybeInRange]; - _ -> - InRange - end, - reduce_stream_kp_node2(Bt, Dir, NodesInRange, KeyStart, InEndRangeFun, - GroupedKey, GroupedKVsAcc, GroupedRedsAcc, KeyGroupFun, Fun, Acc). - - -reduce_stream_kp_node2(Bt, Dir, [{_Key, NodeInfo} | RestNodeList], KeyStart, InEndRangeFun, - undefined, [], [], KeyGroupFun, Fun, Acc) -> + fun({Key, _}) -> InEndRangeFun(Key) end, Nodes + ), + NodesInRange = + case MaybeInRange of + [FirstMaybeInRange | _] when Dir =:= fwd -> + InRange ++ [FirstMaybeInRange]; + _ -> + InRange + end, + reduce_stream_kp_node2( + Bt, + Dir, + NodesInRange, + KeyStart, + InEndRangeFun, + GroupedKey, + GroupedKVsAcc, + GroupedRedsAcc, + KeyGroupFun, + Fun, + Acc + ). + +reduce_stream_kp_node2( + Bt, + Dir, + [{_Key, NodeInfo} | RestNodeList], + KeyStart, + InEndRangeFun, + undefined, + [], + [], + KeyGroupFun, + Fun, + Acc +) -> {ok, Acc2, GroupedRedsAcc2, GroupedKVsAcc2, GroupedKey2} = - reduce_stream_node(Bt, Dir, NodeInfo, KeyStart, InEndRangeFun, undefined, - [], [], KeyGroupFun, Fun, Acc), - reduce_stream_kp_node2(Bt, Dir, RestNodeList, KeyStart, InEndRangeFun, GroupedKey2, - GroupedKVsAcc2, GroupedRedsAcc2, KeyGroupFun, Fun, Acc2); -reduce_stream_kp_node2(Bt, Dir, NodeList, KeyStart, InEndRangeFun, - GroupedKey, GroupedKVsAcc, GroupedRedsAcc, KeyGroupFun, Fun, Acc) -> - {Grouped0, Ungrouped0} = lists:splitwith(fun({Key,_}) -> - KeyGroupFun(GroupedKey, Key) end, NodeList), + reduce_stream_node( + Bt, + Dir, + NodeInfo, + KeyStart, + InEndRangeFun, + undefined, + [], + [], + KeyGroupFun, + Fun, + Acc + ), + reduce_stream_kp_node2( + Bt, + Dir, + RestNodeList, + KeyStart, + InEndRangeFun, + GroupedKey2, + GroupedKVsAcc2, + GroupedRedsAcc2, + KeyGroupFun, + Fun, + Acc2 + ); +reduce_stream_kp_node2( + Bt, + Dir, + NodeList, + KeyStart, + InEndRangeFun, + GroupedKey, + GroupedKVsAcc, + GroupedRedsAcc, + KeyGroupFun, + Fun, + Acc +) -> + {Grouped0, Ungrouped0} = lists:splitwith( + fun({Key, _}) -> + KeyGroupFun(GroupedKey, Key) + end, + NodeList + ), {GroupedNodes, UngroupedNodes} = - case Grouped0 of - [] -> - {Grouped0, Ungrouped0}; - _ -> - [FirstGrouped | RestGrouped] = lists:reverse(Grouped0), - {RestGrouped, [FirstGrouped | Ungrouped0]} - end, + case Grouped0 of + [] -> + {Grouped0, Ungrouped0}; + _ -> + [FirstGrouped | RestGrouped] = lists:reverse(Grouped0), + {RestGrouped, [FirstGrouped | Ungrouped0]} + end, GroupedReds = [element(2, Node) || {_, Node} <- GroupedNodes], case UngroupedNodes of - [{_Key, NodeInfo}|RestNodes] -> - {ok, Acc2, GroupedRedsAcc2, GroupedKVsAcc2, GroupedKey2} = - reduce_stream_node(Bt, Dir, NodeInfo, KeyStart, InEndRangeFun, GroupedKey, - GroupedKVsAcc, GroupedReds ++ GroupedRedsAcc, KeyGroupFun, Fun, Acc), - reduce_stream_kp_node2(Bt, Dir, RestNodes, KeyStart, InEndRangeFun, GroupedKey2, - GroupedKVsAcc2, GroupedRedsAcc2, KeyGroupFun, Fun, Acc2); - [] -> - {ok, Acc, GroupedReds ++ GroupedRedsAcc, GroupedKVsAcc, GroupedKey} + [{_Key, NodeInfo} | RestNodes] -> + {ok, Acc2, GroupedRedsAcc2, GroupedKVsAcc2, GroupedKey2} = + reduce_stream_node( + Bt, + Dir, + NodeInfo, + KeyStart, + InEndRangeFun, + GroupedKey, + GroupedKVsAcc, + GroupedReds ++ GroupedRedsAcc, + KeyGroupFun, + Fun, + Acc + ), + reduce_stream_kp_node2( + Bt, + Dir, + RestNodes, + KeyStart, + InEndRangeFun, + GroupedKey2, + GroupedKVsAcc2, + GroupedRedsAcc2, + KeyGroupFun, + Fun, + Acc2 + ); + [] -> + {ok, Acc, GroupedReds ++ GroupedRedsAcc, GroupedKVsAcc, GroupedKey} end. adjust_dir(fwd, List) -> @@ -753,20 +1068,20 @@ stream_node(Bt, Reds, Node, StartKey, InRange, Dir, Fun, Acc) -> Pointer = element(1, Node), {NodeType, NodeList} = get_node(Bt, Pointer), case NodeType of - kp_node -> - stream_kp_node(Bt, Reds, adjust_dir(Dir, NodeList), StartKey, InRange, Dir, Fun, Acc); - kv_node -> - stream_kv_node(Bt, Reds, adjust_dir(Dir, NodeList), StartKey, InRange, Dir, Fun, Acc) + kp_node -> + stream_kp_node(Bt, Reds, adjust_dir(Dir, NodeList), StartKey, InRange, Dir, Fun, Acc); + kv_node -> + stream_kv_node(Bt, Reds, adjust_dir(Dir, NodeList), StartKey, InRange, Dir, Fun, Acc) end. stream_node(Bt, Reds, Node, InRange, Dir, Fun, Acc) -> Pointer = element(1, Node), {NodeType, NodeList} = get_node(Bt, Pointer), case NodeType of - kp_node -> - stream_kp_node(Bt, Reds, adjust_dir(Dir, NodeList), InRange, Dir, Fun, Acc); - kv_node -> - stream_kv_node2(Bt, Reds, [], adjust_dir(Dir, NodeList), InRange, Dir, Fun, Acc) + kp_node -> + stream_kp_node(Bt, Reds, adjust_dir(Dir, NodeList), InRange, Dir, Fun, Acc); + kv_node -> + stream_kv_node2(Bt, Reds, [], adjust_dir(Dir, NodeList), InRange, Dir, Fun, Acc) end. stream_kp_node(_Bt, _Reds, [], _InRange, _Dir, _Fun, Acc) -> @@ -774,84 +1089,87 @@ stream_kp_node(_Bt, _Reds, [], _InRange, _Dir, _Fun, Acc) -> stream_kp_node(Bt, Reds, [{Key, Node} | Rest], InRange, Dir, Fun, Acc) -> Red = element(2, Node), case Fun(traverse, Key, Red, Acc) of - {ok, Acc2} -> - case stream_node(Bt, Reds, Node, InRange, Dir, Fun, Acc2) of - {ok, Acc3} -> - stream_kp_node(Bt, [Red | Reds], Rest, InRange, Dir, Fun, Acc3); - {stop, LastReds, Acc3} -> - {stop, LastReds, Acc3} - end; - {skip, Acc2} -> - stream_kp_node(Bt, [Red | Reds], Rest, InRange, Dir, Fun, Acc2); - {stop, Acc2} -> - {stop, Reds, Acc2} + {ok, Acc2} -> + case stream_node(Bt, Reds, Node, InRange, Dir, Fun, Acc2) of + {ok, Acc3} -> + stream_kp_node(Bt, [Red | Reds], Rest, InRange, Dir, Fun, Acc3); + {stop, LastReds, Acc3} -> + {stop, LastReds, Acc3} + end; + {skip, Acc2} -> + stream_kp_node(Bt, [Red | Reds], Rest, InRange, Dir, Fun, Acc2); + {stop, Acc2} -> + {stop, Reds, Acc2} end. drop_nodes(_Bt, Reds, _StartKey, []) -> {Reds, []}; drop_nodes(Bt, Reds, StartKey, [{NodeKey, Node} | RestKPs]) -> case less(Bt, NodeKey, StartKey) of - true -> - drop_nodes(Bt, [element(2, Node) | Reds], StartKey, RestKPs); - false -> - {Reds, [{NodeKey, Node} | RestKPs]} + true -> + drop_nodes(Bt, [element(2, Node) | Reds], StartKey, RestKPs); + false -> + {Reds, [{NodeKey, Node} | RestKPs]} end. stream_kp_node(Bt, Reds, KPs, StartKey, InRange, Dir, Fun, Acc) -> {NewReds, NodesToStream} = - case Dir of - fwd -> - % drop all nodes sorting before the key - drop_nodes(Bt, Reds, StartKey, KPs); - rev -> - % keep all nodes sorting before the key, AND the first node to sort after - RevKPs = lists:reverse(KPs), - case lists:splitwith(fun({Key, _Pointer}) -> less(Bt, Key, StartKey) end, RevKPs) of - {_RevsBefore, []} -> - % everything sorts before it - {Reds, KPs}; - {RevBefore, [FirstAfter | Drop]} -> - {[element(2, Node) || {_K, Node} <- Drop] ++ Reds, - [FirstAfter | lists:reverse(RevBefore)]} - end - end, + case Dir of + fwd -> + % drop all nodes sorting before the key + drop_nodes(Bt, Reds, StartKey, KPs); + rev -> + % keep all nodes sorting before the key, AND the first node to sort after + RevKPs = lists:reverse(KPs), + case lists:splitwith(fun({Key, _Pointer}) -> less(Bt, Key, StartKey) end, RevKPs) of + {_RevsBefore, []} -> + % everything sorts before it + {Reds, KPs}; + {RevBefore, [FirstAfter | Drop]} -> + {[element(2, Node) || {_K, Node} <- Drop] ++ Reds, [ + FirstAfter | lists:reverse(RevBefore) + ]} + end + end, case NodesToStream of - [] -> - {ok, Acc}; - [{_Key, Node} | Rest] -> - case stream_node(Bt, NewReds, Node, StartKey, InRange, Dir, Fun, Acc) of - {ok, Acc2} -> - Red = element(2, Node), - stream_kp_node(Bt, [Red | NewReds], Rest, InRange, Dir, Fun, Acc2); - {stop, LastReds, Acc2} -> - {stop, LastReds, Acc2} - end + [] -> + {ok, Acc}; + [{_Key, Node} | Rest] -> + case stream_node(Bt, NewReds, Node, StartKey, InRange, Dir, Fun, Acc) of + {ok, Acc2} -> + Red = element(2, Node), + stream_kp_node(Bt, [Red | NewReds], Rest, InRange, Dir, Fun, Acc2); + {stop, LastReds, Acc2} -> + {stop, LastReds, Acc2} + end end. stream_kv_node(Bt, Reds, KVs, StartKey, InRange, Dir, Fun, Acc) -> DropFun = - case Dir of - fwd -> - fun({Key, _}) -> less(Bt, Key, StartKey) end; - rev -> - fun({Key, _}) -> less(Bt, StartKey, Key) end - end, + case Dir of + fwd -> + fun({Key, _}) -> less(Bt, Key, StartKey) end; + rev -> + fun({Key, _}) -> less(Bt, StartKey, Key) end + end, {LTKVs, GTEKVs} = lists:splitwith(DropFun, KVs), - AssembleLTKVs = [assemble(Bt,K,V) || {K,V} <- LTKVs], + AssembleLTKVs = [assemble(Bt, K, V) || {K, V} <- LTKVs], stream_kv_node2(Bt, Reds, AssembleLTKVs, GTEKVs, InRange, Dir, Fun, Acc). stream_kv_node2(_Bt, _Reds, _PrevKVs, [], _InRange, _Dir, _Fun, Acc) -> {ok, Acc}; -stream_kv_node2(Bt, Reds, PrevKVs, [{K,V} | RestKVs], InRange, Dir, Fun, Acc) -> +stream_kv_node2(Bt, Reds, PrevKVs, [{K, V} | RestKVs], InRange, Dir, Fun, Acc) -> case InRange(K) of - false -> - {stop, {PrevKVs, Reds}, Acc}; - true -> - AssembledKV = assemble(Bt, K, V), - case Fun(visit, AssembledKV, {PrevKVs, Reds}, Acc) of - {ok, Acc2} -> - stream_kv_node2(Bt, Reds, [AssembledKV | PrevKVs], RestKVs, InRange, Dir, Fun, Acc2); - {stop, Acc2} -> - {stop, {PrevKVs, Reds}, Acc2} - end + false -> + {stop, {PrevKVs, Reds}, Acc}; + true -> + AssembledKV = assemble(Bt, K, V), + case Fun(visit, AssembledKV, {PrevKVs, Reds}, Acc) of + {ok, Acc2} -> + stream_kv_node2( + Bt, Reds, [AssembledKV | PrevKVs], RestKVs, InRange, Dir, Fun, Acc2 + ); + {stop, Acc2} -> + {stop, {PrevKVs, Reds}, Acc2} + end end. diff --git a/src/couch/src/couch_changes.erl b/src/couch/src/couch_changes.erl index 2078fed3a..089cda975 100644 --- a/src/couch/src/couch_changes.erl +++ b/src/couch/src/couch_changes.erl @@ -70,62 +70,80 @@ handle_db_changes(Args0, Req, Db0) -> end, Start = fun() -> {ok, Db} = couch_db:reopen(Db0), - StartSeq = case Dir of - rev -> - couch_db:get_update_seq(Db); - fwd -> - Since - end, + StartSeq = + case Dir of + rev -> + couch_db:get_update_seq(Db); + fwd -> + Since + end, {Db, StartSeq} end, % begin timer to deal with heartbeat when filter function fails case Args#changes_args.heartbeat of - undefined -> - erlang:erase(last_changes_heartbeat); - Val when is_integer(Val); Val =:= true -> - put(last_changes_heartbeat, os:timestamp()) + undefined -> + erlang:erase(last_changes_heartbeat); + Val when is_integer(Val); Val =:= true -> + put(last_changes_heartbeat, os:timestamp()) end, case lists:member(Feed, ["continuous", "longpoll", "eventsource"]) of - true -> - fun(CallbackAcc) -> - {Callback, UserAcc} = get_callback_acc(CallbackAcc), - {ok, Listener} = StartListenerFun(), - - {Db, StartSeq} = Start(), - UserAcc2 = start_sending_changes(Callback, UserAcc, Feed), - {Timeout, TimeoutFun} = get_changes_timeout(Args, Callback), - Acc0 = build_acc(Args, Callback, UserAcc2, Db, StartSeq, - <<"">>, Timeout, TimeoutFun), - try - keep_sending_changes( - Args#changes_args{dir=fwd}, - Acc0, - true) - after - couch_event:stop_listener(Listener), - get_rest_updated(ok) % clean out any remaining update messages + true -> + fun(CallbackAcc) -> + {Callback, UserAcc} = get_callback_acc(CallbackAcc), + {ok, Listener} = StartListenerFun(), + + {Db, StartSeq} = Start(), + UserAcc2 = start_sending_changes(Callback, UserAcc, Feed), + {Timeout, TimeoutFun} = get_changes_timeout(Args, Callback), + Acc0 = build_acc( + Args, + Callback, + UserAcc2, + Db, + StartSeq, + <<"">>, + Timeout, + TimeoutFun + ), + try + keep_sending_changes( + Args#changes_args{dir = fwd}, + Acc0, + true + ) + after + couch_event:stop_listener(Listener), + % clean out any remaining update messages + get_rest_updated(ok) + end + end; + false -> + fun(CallbackAcc) -> + {Callback, UserAcc} = get_callback_acc(CallbackAcc), + UserAcc2 = start_sending_changes(Callback, UserAcc, Feed), + {Timeout, TimeoutFun} = get_changes_timeout(Args, Callback), + {Db, StartSeq} = Start(), + Acc0 = build_acc( + Args#changes_args{feed = "normal"}, + Callback, + UserAcc2, + Db, + StartSeq, + <<>>, + Timeout, + TimeoutFun + ), + {ok, #changes_acc{seq = LastSeq, user_acc = UserAcc3}} = + send_changes( + Acc0, + Dir, + true + ), + end_sending_changes(Callback, UserAcc3, LastSeq, Feed) end - end; - false -> - fun(CallbackAcc) -> - {Callback, UserAcc} = get_callback_acc(CallbackAcc), - UserAcc2 = start_sending_changes(Callback, UserAcc, Feed), - {Timeout, TimeoutFun} = get_changes_timeout(Args, Callback), - {Db, StartSeq} = Start(), - Acc0 = build_acc(Args#changes_args{feed="normal"}, Callback, - UserAcc2, Db, StartSeq, <<>>, - Timeout, TimeoutFun), - {ok, #changes_acc{seq = LastSeq, user_acc = UserAcc3}} = - send_changes( - Acc0, - Dir, - true), - end_sending_changes(Callback, UserAcc3, LastSeq, Feed) - end end. - handle_db_event(_DbName, updated, Parent) -> Parent ! updated, {ok, Parent}; @@ -135,7 +153,6 @@ handle_db_event(_DbName, deleted, Parent) -> handle_db_event(_DbName, _Event, Parent) -> {ok, Parent}. - handle_view_event(_DbName, Msg, {Parent, DDocId}) -> case Msg of {index_commit, DDocId} -> @@ -152,17 +169,17 @@ get_callback_acc({Callback, _UserAcc} = Pair) when is_function(Callback, 3) -> get_callback_acc(Callback) when is_function(Callback, 2) -> {fun(Ev, Data, _) -> Callback(Ev, Data) end, ok}. - configure_filter("_doc_ids", Style, Req, _Db) -> {doc_ids, Style, get_doc_ids(Req)}; configure_filter("_selector", Style, Req, _Db) -> - {selector, Style, get_selector_and_fields(Req)}; + {selector, Style, get_selector_and_fields(Req)}; configure_filter("_design", Style, _Req, _Db) -> {design_docs, Style}; configure_filter("_view", Style, Req, Db) -> ViewName = get_view_qs(Req), - if ViewName /= "" -> ok; true -> - throw({bad_request, "`view` filter parameter is not provided."}) + if + ViewName /= "" -> ok; + true -> throw({bad_request, "`view` filter parameter is not provided."}) end, ViewNameParts = string:tokens(ViewName, "/"), case [?l2b(couch_httpd:unquote(Part)) || Part <- ViewNameParts] of @@ -196,10 +213,9 @@ configure_filter(FilterName, Style, Req, Db) -> true -> DIR = fabric_util:doc_id_and_rev(DDoc), {fetch, custom, Style, Req, DIR, FName}; - false-> + false -> {custom, Style, Req, DDoc, FName} end; - [] -> {default, Style}; _Else -> @@ -207,8 +223,7 @@ configure_filter(FilterName, Style, Req, Db) -> throw({bad_request, Msg}) end. - -filter(Db, #full_doc_info{}=FDI, Filter) -> +filter(Db, #full_doc_info{} = FDI, Filter) -> filter(Db, couch_doc:to_doc_info(FDI), Filter); filter(_Db, DocInfo, {default, Style}) -> apply_style(DocInfo, Style); @@ -221,8 +236,10 @@ filter(_Db, DocInfo, {doc_ids, Style, DocIds}) -> end; filter(Db, DocInfo, {selector, Style, {Selector, _Fields}}) -> Docs = open_revs(Db, DocInfo, Style), - Passes = [mango_selector:match(Selector, couch_doc:to_json_obj(Doc, [])) - || Doc <- Docs], + Passes = [ + mango_selector:match(Selector, couch_doc:to_json_obj(Doc, [])) + || Doc <- Docs + ], filter_revs(Passes, Docs); filter(_Db, DocInfo, {design_docs, Style}) -> case DocInfo#doc_info.id of @@ -236,15 +253,15 @@ filter(Db, DocInfo, {view, Style, DDoc, VName}) -> {ok, Passes} = couch_query_servers:filter_view(DDoc, VName, Docs), filter_revs(Passes, Docs); filter(Db, DocInfo, {custom, Style, Req0, DDoc, FName}) -> - Req = case Req0 of - {json_req, _} -> Req0; - #httpd{} -> {json_req, chttpd_external:json_req_obj(Req0, Db)} - end, + Req = + case Req0 of + {json_req, _} -> Req0; + #httpd{} -> {json_req, chttpd_external:json_req_obj(Req0, Db)} + end, Docs = open_revs(Db, DocInfo, Style), {ok, Passes} = couch_query_servers:filter_docs(Req, Db, DDoc, FName, Docs), filter_revs(Passes, Docs). - get_view_qs({json_req, {Props}}) -> {Query} = couch_util:get_value(<<"query">>, Props, {[]}), binary_to_list(couch_util:get_value(<<"view">>, Query, "")); @@ -253,42 +270,43 @@ get_view_qs(Req) -> get_doc_ids({json_req, {Props}}) -> check_docids(couch_util:get_value(<<"doc_ids">>, Props)); -get_doc_ids(#httpd{method='POST'}=Req) -> +get_doc_ids(#httpd{method = 'POST'} = Req) -> couch_httpd:validate_ctype(Req, "application/json"), {Props} = couch_httpd:json_body_obj(Req), check_docids(couch_util:get_value(<<"doc_ids">>, Props)); -get_doc_ids(#httpd{method='GET'}=Req) -> +get_doc_ids(#httpd{method = 'GET'} = Req) -> DocIds = ?JSON_DECODE(couch_httpd:qs_value(Req, "doc_ids", "null")), check_docids(DocIds); get_doc_ids(_) -> throw({bad_request, no_doc_ids_provided}). - get_selector_and_fields({json_req, {Props}}) -> Selector = check_selector(couch_util:get_value(<<"selector">>, Props)), Fields = check_fields(couch_util:get_value(<<"fields">>, Props, nil)), {Selector, Fields}; -get_selector_and_fields(#httpd{method='POST'}=Req) -> +get_selector_and_fields(#httpd{method = 'POST'} = Req) -> couch_httpd:validate_ctype(Req, "application/json"), - get_selector_and_fields({json_req, couch_httpd:json_body_obj(Req)}); + get_selector_and_fields({json_req, couch_httpd:json_body_obj(Req)}); get_selector_and_fields(_) -> throw({bad_request, "Selector must be specified in POST payload"}). - check_docids(DocIds) when is_list(DocIds) -> - lists:foreach(fun - (DocId) when not is_binary(DocId) -> - Msg = "`doc_ids` filter parameter is not a list of doc ids.", - throw({bad_request, Msg}); - (_) -> ok - end, DocIds), + lists:foreach( + fun + (DocId) when not is_binary(DocId) -> + Msg = "`doc_ids` filter parameter is not a list of doc ids.", + throw({bad_request, Msg}); + (_) -> + ok + end, + DocIds + ), DocIds; check_docids(_) -> Msg = "`doc_ids` filter parameter is not a list of doc ids.", throw({bad_request, Msg}). - -check_selector(Selector={_}) -> +check_selector(Selector = {_}) -> try mango_selector:normalize(Selector) catch @@ -299,7 +317,6 @@ check_selector(Selector={_}) -> check_selector(_Selector) -> throw({bad_request, "Selector error: expected a JSON object"}). - check_fields(nil) -> nil; check_fields(Fields) when is_list(Fields) -> @@ -314,7 +331,6 @@ check_fields(Fields) when is_list(Fields) -> check_fields(_Fields) -> throw({bad_request, "Selector error: fields must be JSON array"}). - open_ddoc(Db, DDocId) -> DbName = couch_db:name(Db), case couch_db:is_clustered(Db) of @@ -330,39 +346,38 @@ open_ddoc(Db, DDocId) -> end end. - -check_member_exists(#doc{body={Props}}, Path) -> +check_member_exists(#doc{body = {Props}}, Path) -> couch_util:get_nested_json_value({Props}, Path). - -apply_style(#doc_info{revs=Revs}, main_only) -> - [#rev_info{rev=Rev} | _] = Revs, +apply_style(#doc_info{revs = Revs}, main_only) -> + [#rev_info{rev = Rev} | _] = Revs, [{[{<<"rev">>, couch_doc:rev_to_str(Rev)}]}]; -apply_style(#doc_info{revs=Revs}, all_docs) -> - [{[{<<"rev">>, couch_doc:rev_to_str(R)}]} || #rev_info{rev=R} <- Revs]. - +apply_style(#doc_info{revs = Revs}, all_docs) -> + [{[{<<"rev">>, couch_doc:rev_to_str(R)}]} || #rev_info{rev = R} <- Revs]. open_revs(Db, DocInfo, Style) -> - DocInfos = case Style of - main_only -> [DocInfo]; - all_docs -> [DocInfo#doc_info{revs=[R]}|| R <- DocInfo#doc_info.revs] - end, + DocInfos = + case Style of + main_only -> [DocInfo]; + all_docs -> [DocInfo#doc_info{revs = [R]} || R <- DocInfo#doc_info.revs] + end, OpenOpts = [deleted, conflicts], % Relying on list comprehensions to silence errors OpenResults = [couch_db:open_doc(Db, DI, OpenOpts) || DI <- DocInfos], [Doc || {ok, Doc} <- OpenResults]. - filter_revs(Passes, Docs) -> - lists:flatmap(fun - ({true, #doc{revs={RevPos, [RevId | _]}}}) -> - RevStr = couch_doc:rev_to_str({RevPos, RevId}), - Change = {[{<<"rev">>, RevStr}]}, - [Change]; - (_) -> - [] - end, lists:zip(Passes, Docs)). - + lists:flatmap( + fun + ({true, #doc{revs = {RevPos, [RevId | _]}}}) -> + RevStr = couch_doc:rev_to_str({RevPos, RevId}), + Change = {[{<<"rev">>, RevStr}]}, + [Change]; + (_) -> + [] + end, + lists:zip(Passes, Docs) + ). get_changes_timeout(Args, Callback) -> #changes_args{ @@ -371,29 +386,30 @@ get_changes_timeout(Args, Callback) -> feed = ResponseType } = Args, DefaultTimeout = chttpd_util:get_chttpd_config_integer( - "changes_timeout", 60000), + "changes_timeout", 60000 + ), case Heartbeat of - undefined -> - case Timeout of undefined -> - {DefaultTimeout, fun(UserAcc) -> {stop, UserAcc} end}; - infinity -> - {infinity, fun(UserAcc) -> {stop, UserAcc} end}; + case Timeout of + undefined -> + {DefaultTimeout, fun(UserAcc) -> {stop, UserAcc} end}; + infinity -> + {infinity, fun(UserAcc) -> {stop, UserAcc} end}; + _ -> + {lists:min([DefaultTimeout, Timeout]), fun(UserAcc) -> {stop, UserAcc} end} + end; + true -> + {DefaultTimeout, fun(UserAcc) -> {ok, Callback(timeout, ResponseType, UserAcc)} end}; _ -> - {lists:min([DefaultTimeout, Timeout]), - fun(UserAcc) -> {stop, UserAcc} end} - end; - true -> - {DefaultTimeout, - fun(UserAcc) -> {ok, Callback(timeout, ResponseType, UserAcc)} end}; - _ -> - {lists:min([DefaultTimeout, Heartbeat]), - fun(UserAcc) -> {ok, Callback(timeout, ResponseType, UserAcc)} end} + {lists:min([DefaultTimeout, Heartbeat]), fun(UserAcc) -> + {ok, Callback(timeout, ResponseType, UserAcc)} + end} end. -start_sending_changes(_Callback, UserAcc, ResponseType) - when ResponseType =:= "continuous" - orelse ResponseType =:= "eventsource" -> +start_sending_changes(_Callback, UserAcc, ResponseType) when + ResponseType =:= "continuous" orelse + ResponseType =:= "eventsource" +-> UserAcc; start_sending_changes(Callback, UserAcc, ResponseType) -> Callback(start, ResponseType, UserAcc). @@ -421,8 +437,8 @@ build_acc(Args, Callback, UserAcc, Db, StartSeq, Prepend, Timeout, TimeoutFun) - conflicts = Conflicts, timeout = Timeout, timeout_fun = TimeoutFun, - aggregation_results=[], - aggregation_kvs=[] + aggregation_results = [], + aggregation_kvs = [] }. send_changes(Acc, Dir, FirstRound) -> @@ -440,30 +456,35 @@ send_changes(Acc, Dir, FirstRound) -> couch_db:fold_changes(Db, StartSeq, DbEnumFun, Acc, Opts) end. - can_optimize(true, {doc_ids, _Style, DocIds}) -> - MaxDocIds = config:get_integer("couchdb", - "changes_doc_ids_optimization_threshold", 100), - if length(DocIds) =< MaxDocIds -> - {true, fun send_changes_doc_ids/6}; - true -> - false + MaxDocIds = config:get_integer( + "couchdb", + "changes_doc_ids_optimization_threshold", + 100 + ), + if + length(DocIds) =< MaxDocIds -> + {true, fun send_changes_doc_ids/6}; + true -> + false end; can_optimize(true, {design_docs, _Style}) -> {true, fun send_changes_design_docs/6}; can_optimize(_, _) -> false. - send_changes_doc_ids(Db, StartSeq, Dir, Fun, Acc0, {doc_ids, _Style, DocIds}) -> Results = couch_db:get_full_doc_infos(Db, DocIds), - FullInfos = lists:foldl(fun - (#full_doc_info{}=FDI, Acc) -> [FDI | Acc]; - (not_found, Acc) -> Acc - end, [], Results), + FullInfos = lists:foldl( + fun + (#full_doc_info{} = FDI, Acc) -> [FDI | Acc]; + (not_found, Acc) -> Acc + end, + [], + Results + ), send_lookup_changes(FullInfos, StartSeq, Dir, Db, Fun, Acc0). - send_changes_design_docs(Db, StartSeq, Dir, Fun, Acc0, {design_docs, _Style}) -> FoldFun = fun(FDI, Acc) -> {ok, [FDI | Acc]} end, Opts = [ @@ -474,49 +495,62 @@ send_changes_design_docs(Db, StartSeq, Dir, Fun, Acc0, {design_docs, _Style}) -> {ok, FullInfos} = couch_db:fold_docs(Db, FoldFun, [], Opts), send_lookup_changes(FullInfos, StartSeq, Dir, Db, Fun, Acc0). - send_lookup_changes(FullDocInfos, StartSeq, Dir, Db, Fun, Acc0) -> - FoldFun = case Dir of - fwd -> fun lists:foldl/3; - rev -> fun lists:foldr/3 - end, - GreaterFun = case Dir of - fwd -> fun(A, B) -> A > B end; - rev -> fun(A, B) -> A =< B end - end, - DocInfos = lists:foldl(fun(FDI, Acc) -> - DI = couch_doc:to_doc_info(FDI), - case GreaterFun(DI#doc_info.high_seq, StartSeq) of - true -> [DI | Acc]; - false -> Acc - end - end, [], FullDocInfos), - SortedDocInfos = lists:keysort(#doc_info.high_seq, DocInfos), - FinalAcc = try - FoldFun(fun(DocInfo, Acc) -> - case Fun(DocInfo, Acc) of - {ok, NewAcc} -> - NewAcc; - {stop, NewAcc} -> - throw({stop, NewAcc}) + FoldFun = + case Dir of + fwd -> fun lists:foldl/3; + rev -> fun lists:foldr/3 + end, + GreaterFun = + case Dir of + fwd -> fun(A, B) -> A > B end; + rev -> fun(A, B) -> A =< B end + end, + DocInfos = lists:foldl( + fun(FDI, Acc) -> + DI = couch_doc:to_doc_info(FDI), + case GreaterFun(DI#doc_info.high_seq, StartSeq) of + true -> [DI | Acc]; + false -> Acc end - end, Acc0, SortedDocInfos) - catch - {stop, Acc} -> Acc - end, + end, + [], + FullDocInfos + ), + SortedDocInfos = lists:keysort(#doc_info.high_seq, DocInfos), + FinalAcc = + try + FoldFun( + fun(DocInfo, Acc) -> + case Fun(DocInfo, Acc) of + {ok, NewAcc} -> + NewAcc; + {stop, NewAcc} -> + throw({stop, NewAcc}) + end + end, + Acc0, + SortedDocInfos + ) + catch + {stop, Acc} -> Acc + end, case Dir of fwd -> - FinalAcc0 = case element(1, FinalAcc) of - changes_acc -> % we came here via couch_http or internal call - FinalAcc#changes_acc{seq = couch_db:get_update_seq(Db)}; - fabric_changes_acc -> % we came here via chttpd / fabric / rexi - FinalAcc#fabric_changes_acc{seq = couch_db:get_update_seq(Db)} - end, + FinalAcc0 = + case element(1, FinalAcc) of + % we came here via couch_http or internal call + changes_acc -> + FinalAcc#changes_acc{seq = couch_db:get_update_seq(Db)}; + % we came here via chttpd / fabric / rexi + fabric_changes_acc -> + FinalAcc#fabric_changes_acc{seq = couch_db:get_update_seq(Db)} + end, {ok, FinalAcc0}; - rev -> {ok, FinalAcc} + rev -> + {ok, FinalAcc} end. - keep_sending_changes(Args, Acc0, FirstRound) -> #changes_args{ feed = ResponseType, @@ -527,36 +561,44 @@ keep_sending_changes(Args, Acc0, FirstRound) -> {ok, ChangesAcc} = send_changes(Acc0, fwd, FirstRound), #changes_acc{ - db = Db, callback = Callback, - timeout = Timeout, timeout_fun = TimeoutFun, seq = EndSeq, - prepend = Prepend2, user_acc = UserAcc2, limit = NewLimit + db = Db, + callback = Callback, + timeout = Timeout, + timeout_fun = TimeoutFun, + seq = EndSeq, + prepend = Prepend2, + user_acc = UserAcc2, + limit = NewLimit } = maybe_upgrade_changes_acc(ChangesAcc), couch_db:close(Db), - if Limit > NewLimit, ResponseType == "longpoll" -> - end_sending_changes(Callback, UserAcc2, EndSeq, ResponseType); - true -> - case wait_updated(Timeout, TimeoutFun, UserAcc2) of - {updated, UserAcc4} -> - DbOptions1 = [{user_ctx, couch_db:get_user_ctx(Db)} | DbOptions], - case couch_db:open(couch_db:name(Db), DbOptions1) of - {ok, Db2} -> - ?MODULE:keep_sending_changes( - Args#changes_args{limit=NewLimit}, - ChangesAcc#changes_acc{ - db = Db2, - user_acc = UserAcc4, - seq = EndSeq, - prepend = Prepend2, - timeout = Timeout, - timeout_fun = TimeoutFun}, - false); - _Else -> - end_sending_changes(Callback, UserAcc2, EndSeq, ResponseType) - end; - {stop, UserAcc4} -> - end_sending_changes(Callback, UserAcc4, EndSeq, ResponseType) - end + if + Limit > NewLimit, ResponseType == "longpoll" -> + end_sending_changes(Callback, UserAcc2, EndSeq, ResponseType); + true -> + case wait_updated(Timeout, TimeoutFun, UserAcc2) of + {updated, UserAcc4} -> + DbOptions1 = [{user_ctx, couch_db:get_user_ctx(Db)} | DbOptions], + case couch_db:open(couch_db:name(Db), DbOptions1) of + {ok, Db2} -> + ?MODULE:keep_sending_changes( + Args#changes_args{limit = NewLimit}, + ChangesAcc#changes_acc{ + db = Db2, + user_acc = UserAcc4, + seq = EndSeq, + prepend = Prepend2, + timeout = Timeout, + timeout_fun = TimeoutFun + }, + false + ); + _Else -> + end_sending_changes(Callback, UserAcc2, EndSeq, ResponseType) + end; + {stop, UserAcc4} -> + end_sending_changes(Callback, UserAcc4, EndSeq, ResponseType) + end end. end_sending_changes(Callback, UserAcc, EndSeq, ResponseType) -> @@ -564,46 +606,59 @@ end_sending_changes(Callback, UserAcc, EndSeq, ResponseType) -> changes_enumerator(Value, Acc) -> #changes_acc{ - filter = Filter, callback = Callback, prepend = Prepend, - user_acc = UserAcc, limit = Limit, resp_type = ResponseType, db = Db, - timeout = Timeout, timeout_fun = TimeoutFun + filter = Filter, + callback = Callback, + prepend = Prepend, + user_acc = UserAcc, + limit = Limit, + resp_type = ResponseType, + db = Db, + timeout = Timeout, + timeout_fun = TimeoutFun } = maybe_upgrade_changes_acc(Acc), Results0 = filter(Db, Value, Filter), Results = [Result || Result <- Results0, Result /= null], - Seq = case Value of - #full_doc_info{} -> - Value#full_doc_info.update_seq; - #doc_info{} -> - Value#doc_info.high_seq - end, - Go = if (Limit =< 1) andalso Results =/= [] -> stop; true -> ok end, + Seq = + case Value of + #full_doc_info{} -> + Value#full_doc_info.update_seq; + #doc_info{} -> + Value#doc_info.high_seq + end, + Go = + if + (Limit =< 1) andalso Results =/= [] -> stop; + true -> ok + end, case Results of - [] -> - {Done, UserAcc2} = maybe_heartbeat(Timeout, TimeoutFun, UserAcc), - case Done of - stop -> - {stop, Acc#changes_acc{seq = Seq, user_acc = UserAcc2}}; - ok -> - {Go, Acc#changes_acc{seq = Seq, user_acc = UserAcc2}} - end; - _ -> - if ResponseType =:= "continuous" orelse ResponseType =:= "eventsource" -> - ChangesRow = changes_row(Results, Value, Acc), - UserAcc2 = Callback({change, ChangesRow, <<>>}, ResponseType, UserAcc), - reset_heartbeat(), - {Go, Acc#changes_acc{seq = Seq, user_acc = UserAcc2, limit = Limit - 1}}; - true -> - ChangesRow = changes_row(Results, Value, Acc), - UserAcc2 = Callback({change, ChangesRow, Prepend}, ResponseType, UserAcc), - reset_heartbeat(), - {Go, Acc#changes_acc{ - seq = Seq, prepend = <<",\n">>, - user_acc = UserAcc2, limit = Limit - 1}} - end + [] -> + {Done, UserAcc2} = maybe_heartbeat(Timeout, TimeoutFun, UserAcc), + case Done of + stop -> + {stop, Acc#changes_acc{seq = Seq, user_acc = UserAcc2}}; + ok -> + {Go, Acc#changes_acc{seq = Seq, user_acc = UserAcc2}} + end; + _ -> + if + ResponseType =:= "continuous" orelse ResponseType =:= "eventsource" -> + ChangesRow = changes_row(Results, Value, Acc), + UserAcc2 = Callback({change, ChangesRow, <<>>}, ResponseType, UserAcc), + reset_heartbeat(), + {Go, Acc#changes_acc{seq = Seq, user_acc = UserAcc2, limit = Limit - 1}}; + true -> + ChangesRow = changes_row(Results, Value, Acc), + UserAcc2 = Callback({change, ChangesRow, Prepend}, ResponseType, UserAcc), + reset_heartbeat(), + {Go, Acc#changes_acc{ + seq = Seq, + prepend = <<",\n">>, + user_acc = UserAcc2, + limit = Limit - 1 + }} + end end. - - changes_row(Results, #full_doc_info{} = FDI, Acc) -> changes_row(Results, couch_doc:to_doc_info(FDI), Acc); changes_row(Results, DocInfo, Acc0) -> @@ -611,26 +666,27 @@ changes_row(Results, DocInfo, Acc0) -> #doc_info{ id = Id, high_seq = Seq, revs = [#rev_info{deleted = Del} | _] } = DocInfo, - {[{<<"seq">>, Seq}, {<<"id">>, Id}, {<<"changes">>, Results}] ++ - deleted_item(Del) ++ maybe_get_changes_doc(DocInfo, Acc)}. + { + [{<<"seq">>, Seq}, {<<"id">>, Id}, {<<"changes">>, Results}] ++ + deleted_item(Del) ++ maybe_get_changes_doc(DocInfo, Acc) + }. -maybe_get_changes_doc(Value, #changes_acc{include_docs=true}=Acc) -> +maybe_get_changes_doc(Value, #changes_acc{include_docs = true} = Acc) -> #changes_acc{ db = Db, doc_options = DocOpts, conflicts = Conflicts, filter = Filter } = Acc, - Opts = case Conflicts of - true -> [deleted, conflicts]; - false -> [deleted] - end, + Opts = + case Conflicts of + true -> [deleted, conflicts]; + false -> [deleted] + end, load_doc(Db, Value, Opts, DocOpts, Filter); - maybe_get_changes_doc(_Value, _Acc) -> []. - load_doc(Db, Value, Opts, DocOpts, Filter) -> case couch_index_util:load_doc(Db, Value, Opts) of null -> @@ -639,68 +695,66 @@ load_doc(Db, Value, Opts, DocOpts, Filter) -> [{doc, doc_to_json(Doc, DocOpts, Filter)}] end. - -doc_to_json(Doc, DocOpts, {selector, _Style, {_Selector, Fields}}) - when Fields =/= nil -> +doc_to_json(Doc, DocOpts, {selector, _Style, {_Selector, Fields}}) when + Fields =/= nil +-> mango_fields:extract(couch_doc:to_json_obj(Doc, DocOpts), Fields); doc_to_json(Doc, DocOpts, _Filter) -> couch_doc:to_json_obj(Doc, DocOpts). - deleted_item(true) -> [{<<"deleted">>, true}]; deleted_item(_) -> []. % waits for a updated msg, if there are multiple msgs, collects them. wait_updated(Timeout, TimeoutFun, UserAcc) -> receive - updated -> - get_rest_updated(UserAcc); - deleted -> - {stop, UserAcc} + updated -> + get_rest_updated(UserAcc); + deleted -> + {stop, UserAcc} after Timeout -> {Go, UserAcc2} = TimeoutFun(UserAcc), case Go of - ok -> - ?MODULE:wait_updated(Timeout, TimeoutFun, UserAcc2); - stop -> - {stop, UserAcc2} + ok -> + ?MODULE:wait_updated(Timeout, TimeoutFun, UserAcc2); + stop -> + {stop, UserAcc2} end end. get_rest_updated(UserAcc) -> receive - updated -> - get_rest_updated(UserAcc) + updated -> + get_rest_updated(UserAcc) after 0 -> {updated, UserAcc} end. reset_heartbeat() -> case get(last_changes_heartbeat) of - undefined -> - ok; - _ -> - put(last_changes_heartbeat, os:timestamp()) + undefined -> + ok; + _ -> + put(last_changes_heartbeat, os:timestamp()) end. maybe_heartbeat(Timeout, TimeoutFun, Acc) -> Before = get(last_changes_heartbeat), case Before of - undefined -> - {ok, Acc}; - _ -> - Now = os:timestamp(), - case timer:now_diff(Now, Before) div 1000 >= Timeout of - true -> - Acc2 = TimeoutFun(Acc), - put(last_changes_heartbeat, Now), - Acc2; - false -> - {ok, Acc} - end + undefined -> + {ok, Acc}; + _ -> + Now = os:timestamp(), + case timer:now_diff(Now, Before) div 1000 >= Timeout of + true -> + Acc2 = TimeoutFun(Acc), + put(last_changes_heartbeat, Now), + Acc2; + false -> + {ok, Acc} + end end. - maybe_upgrade_changes_acc(#changes_acc{} = Acc) -> Acc; maybe_upgrade_changes_acc(Acc) when tuple_size(Acc) == 19 -> diff --git a/src/couch/src/couch_compress.erl b/src/couch/src/couch_compress.erl index cfcc2a481..59d692058 100644 --- a/src/couch/src/couch_compress.erl +++ b/src/couch/src/couch_compress.erl @@ -25,21 +25,19 @@ -define(TERM_PREFIX, 131). -define(COMPRESSED_TERM_PREFIX, 131, 80). - get_compression_method() -> case config:get("couchdb", "file_compression") of - undefined -> - ?DEFAULT_COMPRESSION; - Method1 -> - case string:tokens(Method1, "_") of - [Method] -> - list_to_existing_atom(Method); - [Method, Level] -> - {list_to_existing_atom(Method), list_to_integer(Level)} - end + undefined -> + ?DEFAULT_COMPRESSION; + Method1 -> + case string:tokens(Method1, "_") of + [Method] -> + list_to_existing_atom(Method); + [Method, Level] -> + {list_to_existing_atom(Method), list_to_integer(Level)} + end end. - compress(<<?SNAPPY_PREFIX, _/binary>> = Bin, snappy) -> Bin; compress(<<?SNAPPY_PREFIX, _/binary>> = Bin, Method) -> @@ -57,11 +55,11 @@ compress(Term, snappy) -> try {ok, CompressedBin} = snappy:compress(Bin), <<?SNAPPY_PREFIX, CompressedBin/binary>> - catch exit:snappy_nif_not_loaded -> - Bin + catch + exit:snappy_nif_not_loaded -> + Bin end. - decompress(<<?SNAPPY_PREFIX, Rest/binary>>) -> {ok, TermBin} = snappy:decompress(Rest), binary_to_term(TermBin); @@ -70,7 +68,6 @@ decompress(<<?TERM_PREFIX, _/binary>> = Bin) -> decompress(_) -> error(invalid_compression). - is_compressed(<<?SNAPPY_PREFIX, _/binary>>, Method) -> Method =:= snappy; is_compressed(<<?COMPRESSED_TERM_PREFIX, _/binary>>, {deflate, _Level}) -> @@ -84,7 +81,6 @@ is_compressed(Term, _Method) when not is_binary(Term) -> is_compressed(_, _) -> error(invalid_compression). - uncompressed_size(<<?SNAPPY_PREFIX, Rest/binary>>) -> {ok, Size} = snappy:uncompressed_length(Rest), Size; diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl index fdcf23e1b..18ef9c998 100644 --- a/src/couch/src/couch_db.erl +++ b/src/couch/src/couch_db.erl @@ -129,21 +129,22 @@ new_revid/1 ]). - -export([ start_link/4 ]). - -include_lib("couch/include/couch_db.hrl"). -include("couch_db_int.hrl"). -define(DBNAME_REGEX, - "^[a-z][a-z0-9\\_\\$()\\+\\-\\/]*" % use the stock CouchDB regex - "(\\.[0-9]{10,})?$" % but allow an optional shard timestamp at the end + % use the stock CouchDB regex + "^[a-z][a-z0-9\\_\\$()\\+\\-\\/]*" + % but allow an optional shard timestamp at the end + "(\\.[0-9]{10,})?$" ). -define(DEFAULT_COMPRESSIBLE_TYPES, - "text/*, application/javascript, application/json, application/xml"). + "text/*, application/javascript, application/json, application/xml" +). start_link(Engine, DbName, Filepath, Options) -> Arg = {Engine, DbName, Filepath, Options}, @@ -170,10 +171,10 @@ open(DbName, Options) -> close(Db), throw(Error) end; - Else -> Else + Else -> + Else end. - reopen(#db{} = Db) -> % We could have just swapped out the storage engine % for this database during a compaction so we just @@ -184,7 +185,6 @@ reopen(#db{} = Db) -> close(Db) end. - % You shouldn't call this. Its part of the ref counting between % couch_server and couch_db instances. incref(#db{} = Db) -> @@ -200,7 +200,6 @@ clustered_db(DbName, Options) when is_list(Options) -> security = SecProps, options = [{props, Props}] }}; - clustered_db(DbName, #user_ctx{} = UserCtx) -> clustered_db(DbName, [{user_ctx, UserCtx}]). @@ -231,7 +230,7 @@ close(#db{} = Db) -> close(?OLD_DB_REC) -> ok. -is_idle(#db{compactor_pid=nil} = Db) -> +is_idle(#db{compactor_pid = nil} = Db) -> monitored_by(Db) == []; is_idle(_Db) -> false. @@ -245,20 +244,19 @@ monitored_by(Db) -> [] end. - -monitor(#db{main_pid=MainPid}) -> +monitor(#db{main_pid = MainPid}) -> erlang:monitor(process, MainPid). start_compact(#db{} = Db) -> gen_server:call(Db#db.main_pid, start_compact). -cancel_compact(#db{main_pid=Pid}) -> +cancel_compact(#db{main_pid = Pid}) -> gen_server:call(Pid, cancel_compact). wait_for_compaction(Db) -> wait_for_compaction(Db, infinity). -wait_for_compaction(#db{main_pid=Pid}=Db, Timeout) -> +wait_for_compaction(#db{main_pid = Pid} = Db, Timeout) -> Start = os:timestamp(), case gen_server:call(Pid, compactor_pid) of CPid when is_pid(CPid) -> @@ -280,7 +278,7 @@ wait_for_compaction(#db{main_pid=Pid}=Db, Timeout) -> end. delete_doc(Db, Id, Revisions) -> - DeletedDocs = [#doc{id=Id, revs=[Rev], deleted=true} || Rev <- Revisions], + DeletedDocs = [#doc{id = Id, revs = [Rev], deleted = true} || Rev <- Revisions], {ok, [Result]} = update_docs(Db, DeletedDocs, []), {ok, Result}. @@ -290,50 +288,55 @@ open_doc(Db, IdOrDocInfo) -> open_doc(Db, Id, Options) -> increment_stat(Db, [couchdb, database_reads]), case open_doc_int(Db, Id, Options) of - {ok, #doc{deleted=true}=Doc} -> - case lists:member(deleted, Options) of - true -> - apply_open_options({ok, Doc},Options); - false -> - {not_found, deleted} - end; - Else -> - apply_open_options(Else,Options) + {ok, #doc{deleted = true} = Doc} -> + case lists:member(deleted, Options) of + true -> + apply_open_options({ok, Doc}, Options); + false -> + {not_found, deleted} + end; + Else -> + apply_open_options(Else, Options) end. -apply_open_options({ok, Doc},Options) -> - apply_open_options2(Doc,Options); -apply_open_options(Else,_Options) -> +apply_open_options({ok, Doc}, Options) -> + apply_open_options2(Doc, Options); +apply_open_options(Else, _Options) -> Else. -apply_open_options2(Doc,[]) -> +apply_open_options2(Doc, []) -> {ok, Doc}; -apply_open_options2(#doc{atts=Atts0,revs=Revs}=Doc, - [{atts_since, PossibleAncestors}|Rest]) -> +apply_open_options2( + #doc{atts = Atts0, revs = Revs} = Doc, + [{atts_since, PossibleAncestors} | Rest] +) -> RevPos = find_ancestor_rev_pos(Revs, PossibleAncestors), - Atts = lists:map(fun(Att) -> - [AttPos, Data] = couch_att:fetch([revpos, data], Att), - if AttPos > RevPos -> couch_att:store(data, Data, Att); - true -> couch_att:store(data, stub, Att) - end - end, Atts0), - apply_open_options2(Doc#doc{atts=Atts}, Rest); + Atts = lists:map( + fun(Att) -> + [AttPos, Data] = couch_att:fetch([revpos, data], Att), + if + AttPos > RevPos -> couch_att:store(data, Data, Att); + true -> couch_att:store(data, stub, Att) + end + end, + Atts0 + ), + apply_open_options2(Doc#doc{atts = Atts}, Rest); apply_open_options2(Doc, [ejson_body | Rest]) -> apply_open_options2(couch_doc:with_ejson_body(Doc), Rest); -apply_open_options2(Doc,[_|Rest]) -> - apply_open_options2(Doc,Rest). - +apply_open_options2(Doc, [_ | Rest]) -> + apply_open_options2(Doc, Rest). find_ancestor_rev_pos({_, []}, _AttsSinceRevs) -> 0; find_ancestor_rev_pos(_DocRevs, []) -> 0; -find_ancestor_rev_pos({RevPos, [RevId|Rest]}, AttsSinceRevs) -> +find_ancestor_rev_pos({RevPos, [RevId | Rest]}, AttsSinceRevs) -> case lists:member({RevPos, RevId}, AttsSinceRevs) of - true -> - RevPos; - false -> - find_ancestor_rev_pos({RevPos - 1, Rest}, AttsSinceRevs) + true -> + RevPos; + false -> + find_ancestor_rev_pos({RevPos - 1, Rest}, AttsSinceRevs) end. open_doc_revs(Db, Id, Revs, Options) -> @@ -350,39 +353,52 @@ get_missing_revs(Db, IdRevsList) -> find_missing([], []) -> []; -find_missing([{Id, Revs}|RestIdRevs], [FullInfo | RestLookupInfo]) - when is_record(FullInfo, full_doc_info) -> +find_missing([{Id, Revs} | RestIdRevs], [FullInfo | RestLookupInfo]) when + is_record(FullInfo, full_doc_info) +-> case couch_key_tree:find_missing(FullInfo#full_doc_info.rev_tree, Revs) of - [] -> - find_missing(RestIdRevs, RestLookupInfo); - MissingRevs -> - #doc_info{revs=RevsInfo} = couch_doc:to_doc_info(FullInfo), - LeafRevs = [Rev || #rev_info{rev=Rev} <- RevsInfo], - % Find the revs that are possible parents of this rev - PossibleAncestors = - lists:foldl(fun({LeafPos, LeafRevId}, Acc) -> - % this leaf is a "possible ancenstor" of the missing - % revs if this LeafPos lessthan any of the missing revs - case lists:any(fun({MissingPos, _}) -> - LeafPos < MissingPos end, MissingRevs) of - true -> - [{LeafPos, LeafRevId} | Acc]; - false -> - Acc - end - end, [], LeafRevs), - [{Id, MissingRevs, PossibleAncestors} | - find_missing(RestIdRevs, RestLookupInfo)] + [] -> + find_missing(RestIdRevs, RestLookupInfo); + MissingRevs -> + #doc_info{revs = RevsInfo} = couch_doc:to_doc_info(FullInfo), + LeafRevs = [Rev || #rev_info{rev = Rev} <- RevsInfo], + % Find the revs that are possible parents of this rev + PossibleAncestors = + lists:foldl( + fun({LeafPos, LeafRevId}, Acc) -> + % this leaf is a "possible ancenstor" of the missing + % revs if this LeafPos lessthan any of the missing revs + case + lists:any( + fun({MissingPos, _}) -> + LeafPos < MissingPos + end, + MissingRevs + ) + of + true -> + [{LeafPos, LeafRevId} | Acc]; + false -> + Acc + end + end, + [], + LeafRevs + ), + [ + {Id, MissingRevs, PossibleAncestors} + | find_missing(RestIdRevs, RestLookupInfo) + ] end; -find_missing([{Id, Revs}|RestIdRevs], [not_found | RestLookupInfo]) -> +find_missing([{Id, Revs} | RestIdRevs], [not_found | RestLookupInfo]) -> [{Id, Revs, []} | find_missing(RestIdRevs, RestLookupInfo)]. get_doc_info(Db, Id) -> case get_full_doc_info(Db, Id) of - #full_doc_info{} = FDI -> - {ok, couch_doc:to_doc_info(FDI)}; - Else -> - Else + #full_doc_info{} = FDI -> + {ok, couch_doc:to_doc_info(FDI)}; + Else -> + Else end. get_full_doc_info(Db, Id) -> @@ -396,27 +412,39 @@ purge_docs(Db, IdRevs) -> purge_docs(Db, IdRevs, []). -spec purge_docs(#db{}, [{UUId, Id, [Rev]}], [PurgeOption]) -> - {ok, [Reply]} when + {ok, [Reply]} +when UUId :: binary(), Id :: binary() | list(), Rev :: {non_neg_integer(), binary()}, PurgeOption :: interactive_edit | replicated_changes, Reply :: {ok, []} | {ok, [Rev]}. purge_docs(#db{main_pid = Pid} = Db, UUIDsIdsRevs, Options) -> - UUIDsIdsRevs2 = [{UUID, couch_util:to_binary(Id), Revs} - || {UUID, Id, Revs} <- UUIDsIdsRevs], + UUIDsIdsRevs2 = [ + {UUID, couch_util:to_binary(Id), Revs} + || {UUID, Id, Revs} <- UUIDsIdsRevs + ], % Check here if any UUIDs already exist when % we're not replicating purge infos IsRepl = lists:member(replicated_changes, Options), - if IsRepl -> ok; true -> - UUIDs = [UUID || {UUID, _, _} <- UUIDsIdsRevs2], - lists:foreach(fun(Resp) -> - if Resp == not_found -> ok; true -> - Fmt = "Duplicate purge info UIUD: ~s", - Reason = io_lib:format(Fmt, [element(2, Resp)]), - throw({badreq, Reason}) - end - end, get_purge_infos(Db, UUIDs)) + if + IsRepl -> + ok; + true -> + UUIDs = [UUID || {UUID, _, _} <- UUIDsIdsRevs2], + lists:foreach( + fun(Resp) -> + if + Resp == not_found -> + ok; + true -> + Fmt = "Duplicate purge info UIUD: ~s", + Reason = io_lib:format(Fmt, [element(2, Resp)]), + throw({badreq, Reason}) + end + end, + get_purge_infos(Db, UUIDs) + ) end, increment_stat(Db, [couchdb, database_purges]), gen_server:call(Pid, {purge_docs, UUIDsIdsRevs2, Options}). @@ -430,7 +458,6 @@ purge_docs(#db{main_pid = Pid} = Db, UUIDsIdsRevs, Options) -> get_purge_infos(Db, UUIDs) -> couch_db_engine:load_purge_infos(Db, UUIDs). - get_minimum_purge_seq(#db{} = Db) -> PurgeSeq = couch_db_engine:get_purge_seq(Db), OldestPurgeSeq = couch_db_engine:get_oldest_purge_seq(Db), @@ -468,24 +495,31 @@ get_minimum_purge_seq(#db{} = Db) -> {start_key, list_to_binary(?LOCAL_DOC_PREFIX ++ "purge-")} ], {ok, MinIdxSeq} = couch_db:fold_local_docs(Db, FoldFun, InitMinSeq, Opts), - FinalSeq = case MinIdxSeq < PurgeSeq - PurgeInfosLimit of - true -> MinIdxSeq; - false -> erlang:max(0, PurgeSeq - PurgeInfosLimit) - end, + FinalSeq = + case MinIdxSeq < PurgeSeq - PurgeInfosLimit of + true -> MinIdxSeq; + false -> erlang:max(0, PurgeSeq - PurgeInfosLimit) + end, % Log a warning if we've got a purge sequence exceeding the % configured threshold. - if FinalSeq >= (PurgeSeq - PurgeInfosLimit) -> ok; true -> - Fmt = "The purge sequence for '~s' exceeds configured threshold", - couch_log:warning(Fmt, [couch_db:name(Db)]) + if + FinalSeq >= (PurgeSeq - PurgeInfosLimit) -> + ok; + true -> + Fmt = "The purge sequence for '~s' exceeds configured threshold", + couch_log:warning(Fmt, [couch_db:name(Db)]) end, FinalSeq. - purge_client_exists(DbName, DocId, Props) -> % Warn about clients that have not updated their purge % checkpoints in the last "index_lag_warn_seconds" LagWindow = config:get_integer( - "purge", "index_lag_warn_seconds", 86400), % Default 24 hours + % Default 24 hours + "purge", + "index_lag_warn_seconds", + 86400 + ), {Mega, Secs, _} = os:timestamp(), NowSecs = Mega * 1000000 + Secs, @@ -493,43 +527,50 @@ purge_client_exists(DbName, DocId, Props) -> try Exists = couch_db_plugin:is_valid_purge_client(DbName, Props), - if not Exists -> ok; true -> - Updated = couch_util:get_value(<<"updated_on">>, Props), - if is_integer(Updated) and Updated > LagThreshold -> ok; true -> - Diff = NowSecs - Updated, - Fmt1 = "Purge checkpoint '~s' not updated in ~p seconds - in database ~p", - couch_log:error(Fmt1, [DocId, Diff, DbName]) - end + if + not Exists -> + ok; + true -> + Updated = couch_util:get_value(<<"updated_on">>, Props), + if + is_integer(Updated) and Updated > LagThreshold -> + ok; + true -> + Diff = NowSecs - Updated, + Fmt1 = + "Purge checkpoint '~s' not updated in ~p seconds\n" + " in database ~p", + couch_log:error(Fmt1, [DocId, Diff, DbName]) + end end, Exists - catch _:_ -> - % If we fail to check for a client we have to assume that - % it exists. - Fmt2 = "Failed to check purge checkpoint using - document '~p' in database ~p", - couch_log:error(Fmt2, [DocId, DbName]), - true + catch + _:_ -> + % If we fail to check for a client we have to assume that + % it exists. + Fmt2 = + "Failed to check purge checkpoint using\n" + " document '~p' in database ~p", + couch_log:error(Fmt2, [DocId, DbName]), + true end. - -set_purge_infos_limit(#db{main_pid=Pid}=Db, Limit) when Limit > 0 -> +set_purge_infos_limit(#db{main_pid = Pid} = Db, Limit) when Limit > 0 -> check_is_admin(Db), gen_server:call(Pid, {set_purge_infos_limit, Limit}, infinity); set_purge_infos_limit(_Db, _Limit) -> throw(invalid_purge_infos_limit). - get_after_doc_read_fun(#db{after_doc_read = Fun}) -> Fun. get_before_doc_update_fun(#db{before_doc_update = Fun}) -> Fun. -get_committed_update_seq(#db{committed_update_seq=Seq}) -> +get_committed_update_seq(#db{committed_update_seq = Seq}) -> Seq. -get_update_seq(#db{} = Db)-> +get_update_seq(#db{} = Db) -> couch_db_engine:get_update_seq(Db). get_user_ctx(#db{user_ctx = UserCtx}) -> @@ -537,13 +578,13 @@ get_user_ctx(#db{user_ctx = UserCtx}) -> get_user_ctx(?OLD_DB_REC = Db) -> ?OLD_DB_USER_CTX(Db). -get_purge_seq(#db{}=Db) -> +get_purge_seq(#db{} = Db) -> couch_db_engine:get_purge_seq(Db). -get_oldest_purge_seq(#db{}=Db) -> +get_oldest_purge_seq(#db{} = Db) -> couch_db_engine:get_oldest_purge_seq(Db). -get_purge_infos_limit(#db{}=Db) -> +get_purge_infos_limit(#db{} = Db) -> couch_db_engine:get_purge_infos_limit(Db). get_pid(#db{main_pid = Pid}) -> @@ -555,10 +596,10 @@ get_del_doc_count(Db) -> get_doc_count(Db) -> {ok, couch_db_engine:get_doc_count(Db)}. -get_uuid(#db{}=Db) -> +get_uuid(#db{} = Db) -> couch_db_engine:get_uuid(Db). -get_epochs(#db{}=Db) -> +get_epochs(#db{} = Db) -> Epochs = couch_db_engine:get_epochs(Db), validate_epochs(Epochs), Epochs. @@ -569,13 +610,13 @@ get_filepath(#db{filepath = FilePath}) -> get_instance_start_time(#db{instance_start_time = IST}) -> IST. -get_compacted_seq(#db{}=Db) -> +get_compacted_seq(#db{} = Db) -> couch_db_engine:get_compacted_seq(Db). get_compactor_pid(#db{compactor_pid = Pid}) -> Pid. -get_compactor_pid_sync(#db{main_pid=Pid}) -> +get_compactor_pid_sync(#db{main_pid = Pid}) -> case gen_server:call(Pid, compactor_pid, infinity) of CPid when is_pid(CPid) -> CPid; @@ -594,18 +635,21 @@ get_db_info(Db) -> {ok, DelDocCount} = get_del_doc_count(Db), SizeInfo = couch_db_engine:get_size_info(Db), DiskVersion = couch_db_engine:get_disk_version(Db), - Uuid = case get_uuid(Db) of - undefined -> null; - Uuid0 -> Uuid0 - end, - CompactedSeq = case get_compacted_seq(Db) of - undefined -> null; - Else1 -> Else1 - end, - Props = case couch_db_engine:get_props(Db) of - undefined -> null; - Else2 -> {Else2} - end, + Uuid = + case get_uuid(Db) of + undefined -> null; + Uuid0 -> Uuid0 + end, + CompactedSeq = + case get_compacted_seq(Db) of + undefined -> null; + Else1 -> Else1 + end, + Props = + case couch_db_engine:get_props(Db) of + undefined -> null; + Else2 -> {Else2} + end, InfoList = [ {db_name, Name}, {engine, couch_db_engine:get_engine(Db)}, @@ -630,15 +674,15 @@ get_partition_info(#db{} = Db, Partition) when is_binary(Partition) -> get_partition_info(_Db, _Partition) -> throw({bad_request, <<"`partition` is not valid">>}). - get_design_doc(#db{name = <<"shards/", _/binary>> = ShardDbName}, DDocId0) -> DDocId = couch_util:normalize_ddoc_id(DDocId0), DbName = mem3:dbname(ShardDbName), {_, Ref} = spawn_monitor(fun() -> exit(fabric:open_doc(DbName, DDocId, [])) end), - receive {'DOWN', Ref, _, _, Response} -> - Response + receive + {'DOWN', Ref, _, _, Response} -> + Response end; get_design_doc(#db{} = Db, DDocId0) -> DDocId = couch_util:normalize_ddoc_id(DDocId0), @@ -647,8 +691,9 @@ get_design_doc(#db{} = Db, DDocId0) -> get_design_docs(#db{name = <<"shards/", _/binary>> = ShardDbName}) -> DbName = mem3:dbname(ShardDbName), {_, Ref} = spawn_monitor(fun() -> exit(fabric:design_docs(DbName)) end), - receive {'DOWN', Ref, _, _, Response} -> - Response + receive + {'DOWN', Ref, _, _, Response} -> + Response end; get_design_docs(#db{} = Db) -> FoldFun = fun(FDI, Acc) -> {ok, [FDI | Acc]} end, @@ -659,47 +704,51 @@ get_design_doc_count(#db{} = Db) -> FoldFun = fun(_, Acc) -> {ok, Acc + 1} end, fold_design_docs(Db, FoldFun, 0, []). -check_is_admin(#db{user_ctx=UserCtx}=Db) -> +check_is_admin(#db{user_ctx = UserCtx} = Db) -> case is_admin(Db) of - true -> ok; + true -> + ok; false -> Reason = <<"You are not a db or server admin.">>, throw_security_error(UserCtx, Reason) end. -check_is_member(#db{user_ctx=UserCtx}=Db) -> +check_is_member(#db{user_ctx = UserCtx} = Db) -> case is_member(Db) of true -> ok; false -> throw_security_error(UserCtx) end. -is_admin(#db{user_ctx=UserCtx}=Db) -> +is_admin(#db{user_ctx = UserCtx} = Db) -> case couch_db_plugin:check_is_admin(Db) of - true -> true; + true -> + true; false -> {Admins} = get_admins(Db), is_authorized(UserCtx, Admins) end. -is_member(#db{user_ctx=UserCtx}=Db) -> +is_member(#db{user_ctx = UserCtx} = Db) -> case is_admin(Db) of - true -> true; + true -> + true; false -> case is_public_db(Db) of - true -> true; + true -> + true; false -> {Members} = get_members(Db), is_authorized(UserCtx, Members) end end. -is_public_db(#db{}=Db) -> +is_public_db(#db{} = Db) -> {Members} = get_members(Db), Names = couch_util:get_value(<<"names">>, Members, []), Roles = couch_util:get_value(<<"roles">>, Members, []), Names =:= [] andalso Roles =:= []. -is_authorized(#user_ctx{name=UserName,roles=UserRoles}, Security) -> +is_authorized(#user_ctx{name = UserName, roles = UserRoles}, Security) -> Names = couch_util:get_value(<<"names">>, Security, []), Roles = couch_util:get_value(<<"roles">>, Security, []), case check_security(roles, UserRoles, [<<"_admin">> | Roles]) of @@ -720,36 +769,38 @@ check_security(names, null, _) -> check_security(names, UserName, Names) -> lists:member(UserName, Names). -throw_security_error(#user_ctx{name=null}=UserCtx) -> +throw_security_error(#user_ctx{name = null} = UserCtx) -> Reason = <<"You are not authorized to access this db.">>, throw_security_error(UserCtx, Reason); -throw_security_error(#user_ctx{name=_}=UserCtx) -> +throw_security_error(#user_ctx{name = _} = UserCtx) -> Reason = <<"You are not allowed to access this db.">>, throw_security_error(UserCtx, Reason). -throw_security_error(#user_ctx{}=UserCtx, Reason) -> +throw_security_error(#user_ctx{} = UserCtx, Reason) -> Error = security_error_type(UserCtx), throw({Error, Reason}). -security_error_type(#user_ctx{name=null}) -> +security_error_type(#user_ctx{name = null}) -> unauthorized; -security_error_type(#user_ctx{name=_}) -> +security_error_type(#user_ctx{name = _}) -> forbidden. - -get_admins(#db{security=SecProps}) -> +get_admins(#db{security = SecProps}) -> couch_util:get_value(<<"admins">>, SecProps, {[]}). -get_members(#db{security=SecProps}) -> +get_members(#db{security = SecProps}) -> % we fallback to readers here for backwards compatibility - couch_util:get_value(<<"members">>, SecProps, - couch_util:get_value(<<"readers">>, SecProps, {[]})). + couch_util:get_value( + <<"members">>, + SecProps, + couch_util:get_value(<<"readers">>, SecProps, {[]}) + ). -get_security(#db{security=SecProps}) -> +get_security(#db{security = SecProps}) -> {SecProps}; get_security(?OLD_DB_REC = Db) -> {?OLD_DB_SECURITY(Db)}. -set_security(#db{main_pid=Pid}=Db, {NewSecProps}) when is_list(NewSecProps) -> +set_security(#db{main_pid = Pid} = Db, {NewSecProps}) when is_list(NewSecProps) -> check_is_admin(Db), ok = validate_security_object(NewSecProps), gen_server:call(Pid, {set_security, NewSecProps}, infinity); @@ -762,8 +813,11 @@ set_user_ctx(#db{} = Db, UserCtx) -> validate_security_object(SecProps) -> Admins = couch_util:get_value(<<"admins">>, SecProps, {[]}), % we fallback to readers here for backwards compatibility - Members = couch_util:get_value(<<"members">>, SecProps, - couch_util:get_value(<<"readers">>, SecProps, {[]})), + Members = couch_util:get_value( + <<"members">>, + SecProps, + couch_util:get_value(<<"readers">>, SecProps, {[]}) + ), ok = validate_names_and_roles(Admins), ok = validate_names_and_roles(Members), ok. @@ -771,18 +825,18 @@ validate_security_object(SecProps) -> % validate user input validate_names_and_roles({Props}) when is_list(Props) -> case couch_util:get_value(<<"names">>, Props, []) of - Ns when is_list(Ns) -> - [throw("names must be a JSON list of strings") ||N <- Ns, not is_binary(N)], + Ns when is_list(Ns) -> + [throw("names must be a JSON list of strings") || N <- Ns, not is_binary(N)], Ns; - _ -> - throw("names must be a JSON list of strings") + _ -> + throw("names must be a JSON list of strings") end, case couch_util:get_value(<<"roles">>, Props, []) of - Rs when is_list(Rs) -> - [throw("roles must be a JSON list of strings") ||R <- Rs, not is_binary(R)], - Rs; - _ -> - throw("roles must be a JSON list of strings") + Rs when is_list(Rs) -> + [throw("roles must be a JSON list of strings") || R <- Rs, not is_binary(R)], + Rs; + _ -> + throw("roles must be a JSON list of strings") end, ok; validate_names_and_roles(_) -> @@ -791,18 +845,17 @@ validate_names_and_roles(_) -> get_revs_limit(#db{} = Db) -> couch_db_engine:get_revs_limit(Db). -set_revs_limit(#db{main_pid=Pid}=Db, Limit) when Limit > 0 -> +set_revs_limit(#db{main_pid = Pid} = Db, Limit) when Limit > 0 -> check_is_admin(Db), gen_server:call(Pid, {set_revs_limit, Limit}, infinity); set_revs_limit(_Db, _Limit) -> throw(invalid_revs_limit). -name(#db{name=Name}) -> +name(#db{name = Name}) -> Name; name(?OLD_DB_REC = Db) -> ?OLD_DB_NAME(Db). - validate_docid(#db{} = Db, DocId) when is_binary(DocId) -> couch_doc:validate_docid(DocId, name(Db)), case is_partitioned(Db) of @@ -812,7 +865,6 @@ validate_docid(#db{} = Db, DocId) when is_binary(DocId) -> ok end. - doc_from_json_obj_validate(#db{} = Db, DocJson) -> Doc = couch_doc:from_json_obj_validate(DocJson, name(Db)), {Props} = DocJson, @@ -825,22 +877,21 @@ doc_from_json_obj_validate(#db{} = Db, DocJson) -> end, Doc. - update_doc(Db, Doc, Options) -> update_doc(Db, Doc, Options, interactive_edit). update_doc(Db, Doc, Options, UpdateType) -> case update_docs(Db, [Doc], Options, UpdateType) of - {ok, [{ok, NewRev}]} -> - {ok, NewRev}; - {ok, [{{_Id, _Rev}, Error}]} -> - throw(Error); - {ok, [Error]} -> - throw(Error); - {ok, []} -> - % replication success - {Pos, [RevId | _]} = Doc#doc.revs, - {ok, {Pos, RevId}} + {ok, [{ok, NewRev}]} -> + {ok, NewRev}; + {ok, [{{_Id, _Rev}, Error}]} -> + throw(Error); + {ok, [Error]} -> + throw(Error); + {ok, []} -> + % replication success + {Pos, [RevId | _]} = Doc#doc.revs, + {ok, {Pos, RevId}} end. update_docs(Db, Docs) -> @@ -860,30 +911,30 @@ group_alike_docs(Docs) -> group_alike_docs([], Buckets) -> lists:reverse(lists:map(fun lists:reverse/1, Buckets)); -group_alike_docs([Doc|Rest], []) -> +group_alike_docs([Doc | Rest], []) -> group_alike_docs(Rest, [[Doc]]); -group_alike_docs([Doc|Rest], [Bucket|RestBuckets]) -> - [#doc{id=BucketId}|_] = Bucket, +group_alike_docs([Doc | Rest], [Bucket | RestBuckets]) -> + [#doc{id = BucketId} | _] = Bucket, case Doc#doc.id == BucketId of - true -> - % add to existing bucket - group_alike_docs(Rest, [[Doc|Bucket]|RestBuckets]); - false -> - % add to new bucket - group_alike_docs(Rest, [[Doc]|[Bucket|RestBuckets]]) + true -> + % add to existing bucket + group_alike_docs(Rest, [[Doc | Bucket] | RestBuckets]); + false -> + % add to new bucket + group_alike_docs(Rest, [[Doc] | [Bucket | RestBuckets]]) end. -validate_doc_update(#db{}=Db, #doc{id= <<"_design/",_/binary>>}=Doc, _GetDiskDocFun) -> +validate_doc_update(#db{} = Db, #doc{id = <<"_design/", _/binary>>} = Doc, _GetDiskDocFun) -> case catch check_is_admin(Db) of ok -> validate_ddoc(Db, Doc); Error -> Error end; validate_doc_update(#db{validate_doc_funs = undefined} = Db, Doc, Fun) -> ValidationFuns = load_validation_funs(Db), - validate_doc_update(Db#db{validate_doc_funs=ValidationFuns}, Doc, Fun); -validate_doc_update(#db{validate_doc_funs=[]}, _Doc, _GetDiskDocFun) -> + validate_doc_update(Db#db{validate_doc_funs = ValidationFuns}, Doc, Fun); +validate_doc_update(#db{validate_doc_funs = []}, _Doc, _GetDiskDocFun) -> ok; -validate_doc_update(_Db, #doc{id= <<"_local/",_/binary>>}, _GetDiskDocFun) -> +validate_doc_update(_Db, #doc{id = <<"_local/", _/binary>>}, _GetDiskDocFun) -> ok; validate_doc_update(Db, Doc, GetDiskDocFun) -> case get(io_priority) of @@ -911,22 +962,26 @@ validate_doc_update_int(Db, Doc, GetDiskDocFun) -> JsonCtx = couch_util:json_user_ctx(Db), SecObj = get_security(Db), try - [case Fun(Doc, DiskDoc, JsonCtx, SecObj) of - ok -> ok; - Error -> throw(Error) - end || Fun <- Db#db.validate_doc_funs], + [ + case Fun(Doc, DiskDoc, JsonCtx, SecObj) of + ok -> ok; + Error -> throw(Error) + end + || Fun <- Db#db.validate_doc_funs + ], ok catch throw:Error -> Error end end, - couch_stats:update_histogram([couchdb, query_server, vdu_process_time], - Fun). - + couch_stats:update_histogram( + [couchdb, query_server, vdu_process_time], + Fun + ). % to be safe, spawn a middleman here -load_validation_funs(#db{main_pid=Pid, name = <<"shards/", _/binary>>}=Db) -> +load_validation_funs(#db{main_pid = Pid, name = <<"shards/", _/binary>>} = Db) -> {_, Ref} = spawn_monitor(fun() -> exit(ddoc_cache:open(mem3:dbname(Db#db.name), validation_funs)) end), @@ -941,242 +996,326 @@ load_validation_funs(#db{main_pid=Pid, name = <<"shards/", _/binary>>}=Db) -> couch_log:error("could not load validation funs ~p", [Reason]), throw(internal_server_error) end; -load_validation_funs(#db{main_pid=Pid}=Db) -> +load_validation_funs(#db{main_pid = Pid} = Db) -> {ok, DDocInfos} = get_design_docs(Db), - OpenDocs = fun - (#full_doc_info{}=D) -> - {ok, Doc} = open_doc_int(Db, D, [ejson_body]), - Doc + OpenDocs = fun(#full_doc_info{} = D) -> + {ok, Doc} = open_doc_int(Db, D, [ejson_body]), + Doc end, DDocs = lists:map(OpenDocs, DDocInfos), - Funs = lists:flatmap(fun(DDoc) -> - case couch_doc:get_validate_doc_fun(DDoc) of - nil -> []; - Fun -> [Fun] - end - end, DDocs), + Funs = lists:flatmap( + fun(DDoc) -> + case couch_doc:get_validate_doc_fun(DDoc) of + nil -> []; + Fun -> [Fun] + end + end, + DDocs + ), gen_server:cast(Pid, {load_validation_funs, Funs}), Funs. reload_validation_funs(#db{} = Db) -> gen_server:cast(Db#db.main_pid, {load_validation_funs, undefined}). -prep_and_validate_update(Db, #doc{id=Id,revs={RevStart, Revs}}=Doc, - OldFullDocInfo, LeafRevsDict, AllowConflict) -> +prep_and_validate_update( + Db, + #doc{id = Id, revs = {RevStart, Revs}} = Doc, + OldFullDocInfo, + LeafRevsDict, + AllowConflict +) -> case Revs of - [PrevRev|_] -> - case dict:find({RevStart, PrevRev}, LeafRevsDict) of - {ok, {#leaf{deleted=Deleted, ptr=DiskSp}, DiskRevs}} -> - case couch_doc:has_stubs(Doc) of - true -> - DiskDoc = make_doc(Db, Id, Deleted, DiskSp, DiskRevs), - Doc2 = couch_doc:merge_stubs(Doc, DiskDoc), - {validate_doc_update(Db, Doc2, fun() -> DiskDoc end), Doc2}; - false -> - LoadDiskDoc = fun() -> make_doc(Db,Id,Deleted,DiskSp,DiskRevs) end, - {validate_doc_update(Db, Doc, LoadDiskDoc), Doc} + [PrevRev | _] -> + case dict:find({RevStart, PrevRev}, LeafRevsDict) of + {ok, {#leaf{deleted = Deleted, ptr = DiskSp}, DiskRevs}} -> + case couch_doc:has_stubs(Doc) of + true -> + DiskDoc = make_doc(Db, Id, Deleted, DiskSp, DiskRevs), + Doc2 = couch_doc:merge_stubs(Doc, DiskDoc), + {validate_doc_update(Db, Doc2, fun() -> DiskDoc end), Doc2}; + false -> + LoadDiskDoc = fun() -> make_doc(Db, Id, Deleted, DiskSp, DiskRevs) end, + {validate_doc_update(Db, Doc, LoadDiskDoc), Doc} + end; + error when AllowConflict -> + % will generate error if + couch_doc:merge_stubs(Doc, #doc{}), + % there are stubs + {validate_doc_update(Db, Doc, fun() -> nil end), Doc}; + error -> + {conflict, Doc} end; - error when AllowConflict -> - couch_doc:merge_stubs(Doc, #doc{}), % will generate error if - % there are stubs - {validate_doc_update(Db, Doc, fun() -> nil end), Doc}; - error -> - {conflict, Doc} - end; - [] -> - % new doc, and we have existing revs. - % reuse existing deleted doc - if OldFullDocInfo#full_doc_info.deleted orelse AllowConflict -> - {validate_doc_update(Db, Doc, fun() -> nil end), Doc}; - true -> - {conflict, Doc} - end + [] -> + % new doc, and we have existing revs. + % reuse existing deleted doc + if + OldFullDocInfo#full_doc_info.deleted orelse AllowConflict -> + {validate_doc_update(Db, Doc, fun() -> nil end), Doc}; + true -> + {conflict, Doc} + end end. - - -prep_and_validate_updates(_Db, [], [], _AllowConflict, AccPrepped, - AccFatalErrors) -> +prep_and_validate_updates( + _Db, + [], + [], + _AllowConflict, + AccPrepped, + AccFatalErrors +) -> AccPrepped2 = lists:reverse(lists:map(fun lists:reverse/1, AccPrepped)), {AccPrepped2, AccFatalErrors}; -prep_and_validate_updates(Db, [DocBucket|RestBuckets], [not_found|RestLookups], - AllowConflict, AccPrepped, AccErrors) -> +prep_and_validate_updates( + Db, + [DocBucket | RestBuckets], + [not_found | RestLookups], + AllowConflict, + AccPrepped, + AccErrors +) -> % no existing revs are known, {PreppedBucket, AccErrors3} = lists:foldl( - fun(#doc{revs=Revs}=Doc, {AccBucket, AccErrors2}) -> + fun(#doc{revs = Revs} = Doc, {AccBucket, AccErrors2}) -> case couch_doc:has_stubs(Doc) of - true -> - couch_doc:merge_stubs(Doc, #doc{}); % will throw exception - false -> ok + true -> + % will throw exception + couch_doc:merge_stubs(Doc, #doc{}); + false -> + ok end, case Revs of - {0, []} -> - case validate_doc_update(Db, Doc, fun() -> nil end) of - ok -> - {[Doc | AccBucket], AccErrors2}; - Error -> - {AccBucket, [{doc_tag(Doc), Error} | AccErrors2]} - end; - _ -> - % old revs specified but none exist, a conflict - {AccBucket, [{doc_tag(Doc), conflict} | AccErrors2]} + {0, []} -> + case validate_doc_update(Db, Doc, fun() -> nil end) of + ok -> + {[Doc | AccBucket], AccErrors2}; + Error -> + {AccBucket, [{doc_tag(Doc), Error} | AccErrors2]} + end; + _ -> + % old revs specified but none exist, a conflict + {AccBucket, [{doc_tag(Doc), conflict} | AccErrors2]} end end, - {[], AccErrors}, DocBucket), + {[], AccErrors}, + DocBucket + ), - prep_and_validate_updates(Db, RestBuckets, RestLookups, AllowConflict, - [PreppedBucket | AccPrepped], AccErrors3); -prep_and_validate_updates(Db, [DocBucket|RestBuckets], - [#full_doc_info{rev_tree=OldRevTree}=OldFullDocInfo|RestLookups], - AllowConflict, AccPrepped, AccErrors) -> + prep_and_validate_updates( + Db, + RestBuckets, + RestLookups, + AllowConflict, + [PreppedBucket | AccPrepped], + AccErrors3 + ); +prep_and_validate_updates( + Db, + [DocBucket | RestBuckets], + [#full_doc_info{rev_tree = OldRevTree} = OldFullDocInfo | RestLookups], + AllowConflict, + AccPrepped, + AccErrors +) -> Leafs = couch_key_tree:get_all_leafs(OldRevTree), LeafRevsDict = dict:from_list([ - {{Start, RevId}, {Leaf, Revs}} || - {Leaf, {Start, [RevId | _]} = Revs} <- Leafs + {{Start, RevId}, {Leaf, Revs}} + || {Leaf, {Start, [RevId | _]} = Revs} <- Leafs ]), {PreppedBucket, AccErrors3} = lists:foldl( fun(Doc, {Docs2Acc, AccErrors2}) -> - case prep_and_validate_update(Db, Doc, OldFullDocInfo, - LeafRevsDict, AllowConflict) of - {ok, Doc2} -> - {[Doc2 | Docs2Acc], AccErrors2}; - {Error, _} -> - % Record the error - {Docs2Acc, [{doc_tag(Doc), Error} |AccErrors2]} + case + prep_and_validate_update( + Db, + Doc, + OldFullDocInfo, + LeafRevsDict, + AllowConflict + ) + of + {ok, Doc2} -> + {[Doc2 | Docs2Acc], AccErrors2}; + {Error, _} -> + % Record the error + {Docs2Acc, [{doc_tag(Doc), Error} | AccErrors2]} end end, - {[], AccErrors}, DocBucket), - prep_and_validate_updates(Db, RestBuckets, RestLookups, AllowConflict, - [PreppedBucket | AccPrepped], AccErrors3). - + {[], AccErrors}, + DocBucket + ), + prep_and_validate_updates( + Db, + RestBuckets, + RestLookups, + AllowConflict, + [PreppedBucket | AccPrepped], + AccErrors3 + ). update_docs(Db, Docs, Options) -> update_docs(Db, Docs, Options, interactive_edit). - prep_and_validate_replicated_updates(_Db, [], [], AccPrepped, AccErrors) -> - Errors2 = [{{Id, {Pos, Rev}}, Error} || - {#doc{id=Id,revs={Pos,[Rev|_]}}, Error} <- AccErrors], + Errors2 = [ + {{Id, {Pos, Rev}}, Error} + || {#doc{id = Id, revs = {Pos, [Rev | _]}}, Error} <- AccErrors + ], AccPrepped2 = lists:reverse(lists:map(fun lists:reverse/1, AccPrepped)), {AccPrepped2, lists:reverse(Errors2)}; -prep_and_validate_replicated_updates(Db, [Bucket|RestBuckets], [OldInfo|RestOldInfo], AccPrepped, AccErrors) -> +prep_and_validate_replicated_updates( + Db, [Bucket | RestBuckets], [OldInfo | RestOldInfo], AccPrepped, AccErrors +) -> case OldInfo of - not_found -> - {ValidatedBucket, AccErrors3} = lists:foldl( - fun(Doc, {AccPrepped2, AccErrors2}) -> - case couch_doc:has_stubs(Doc) of - true -> - couch_doc:merge_stubs(Doc, #doc{}); % will throw exception - false -> ok - end, - case validate_doc_update(Db, Doc, fun() -> nil end) of - ok -> - {[Doc | AccPrepped2], AccErrors2}; - Error -> - {AccPrepped2, [{Doc, Error} | AccErrors2]} - end - end, - {[], AccErrors}, Bucket), - prep_and_validate_replicated_updates(Db, RestBuckets, RestOldInfo, [ValidatedBucket | AccPrepped], AccErrors3); - #full_doc_info{rev_tree=OldTree} -> - OldLeafs = couch_key_tree:get_all_leafs_full(OldTree), - OldLeafsLU = [{Start, RevId} || {Start, [{RevId, _}|_]} <- OldLeafs], - NewPaths = lists:map(fun couch_doc:to_path/1, Bucket), - NewRevTree = couch_key_tree:multi_merge(OldTree, NewPaths), - Leafs = couch_key_tree:get_all_leafs_full(NewRevTree), - LeafRevsFullDict = dict:from_list( [{{Start, RevId}, FullPath} || {Start, [{RevId, _}|_]}=FullPath <- Leafs]), - {ValidatedBucket, AccErrors3} = - lists:foldl( - fun(#doc{id=Id,revs={Pos, [RevId|_]}}=Doc, {AccValidated, AccErrors2}) -> - IsOldLeaf = lists:member({Pos, RevId}, OldLeafsLU), - case dict:find({Pos, RevId}, LeafRevsFullDict) of - {ok, {Start, Path}} when not IsOldLeaf -> - % our unflushed doc is a leaf node. Go back on the path - % to find the previous rev that's on disk. - - LoadPrevRevFun = fun() -> - make_first_doc_on_disk(Db,Id,Start-1, tl(Path)) - end, - + not_found -> + {ValidatedBucket, AccErrors3} = lists:foldl( + fun(Doc, {AccPrepped2, AccErrors2}) -> case couch_doc:has_stubs(Doc) of - true -> - DiskDoc = case LoadPrevRevFun() of - #doc{} = DiskDoc0 -> - DiskDoc0; + true -> + % will throw exception + couch_doc:merge_stubs(Doc, #doc{}); + false -> + ok + end, + case validate_doc_update(Db, Doc, fun() -> nil end) of + ok -> + {[Doc | AccPrepped2], AccErrors2}; + Error -> + {AccPrepped2, [{Doc, Error} | AccErrors2]} + end + end, + {[], AccErrors}, + Bucket + ), + prep_and_validate_replicated_updates( + Db, RestBuckets, RestOldInfo, [ValidatedBucket | AccPrepped], AccErrors3 + ); + #full_doc_info{rev_tree = OldTree} -> + OldLeafs = couch_key_tree:get_all_leafs_full(OldTree), + OldLeafsLU = [{Start, RevId} || {Start, [{RevId, _} | _]} <- OldLeafs], + NewPaths = lists:map(fun couch_doc:to_path/1, Bucket), + NewRevTree = couch_key_tree:multi_merge(OldTree, NewPaths), + Leafs = couch_key_tree:get_all_leafs_full(NewRevTree), + LeafRevsFullDict = dict:from_list([ + {{Start, RevId}, FullPath} + || {Start, [{RevId, _} | _]} = FullPath <- Leafs + ]), + {ValidatedBucket, AccErrors3} = + lists:foldl( + fun(#doc{id = Id, revs = {Pos, [RevId | _]}} = Doc, {AccValidated, AccErrors2}) -> + IsOldLeaf = lists:member({Pos, RevId}, OldLeafsLU), + case dict:find({Pos, RevId}, LeafRevsFullDict) of + {ok, {Start, Path}} when not IsOldLeaf -> + % our unflushed doc is a leaf node. Go back on the path + % to find the previous rev that's on disk. + + LoadPrevRevFun = fun() -> + make_first_doc_on_disk(Db, Id, Start - 1, tl(Path)) + end, + + case couch_doc:has_stubs(Doc) of + true -> + DiskDoc = + case LoadPrevRevFun() of + #doc{} = DiskDoc0 -> + DiskDoc0; + _ -> + % Force a missing_stub exception + couch_doc:merge_stubs(Doc, #doc{}) + end, + Doc2 = couch_doc:merge_stubs(Doc, DiskDoc), + GetDiskDocFun = fun() -> DiskDoc end; + false -> + Doc2 = Doc, + GetDiskDocFun = LoadPrevRevFun + end, + + case validate_doc_update(Db, Doc2, GetDiskDocFun) of + ok -> + {[Doc2 | AccValidated], AccErrors2}; + Error -> + {AccValidated, [{Doc, Error} | AccErrors2]} + end; _ -> - % Force a missing_stub exception - couch_doc:merge_stubs(Doc, #doc{}) - end, - Doc2 = couch_doc:merge_stubs(Doc, DiskDoc), - GetDiskDocFun = fun() -> DiskDoc end; - false -> - Doc2 = Doc, - GetDiskDocFun = LoadPrevRevFun + % this doc isn't a leaf or already exists in the tree. + % ignore but consider it a success. + {AccValidated, AccErrors2} + end end, - - case validate_doc_update(Db, Doc2, GetDiskDocFun) of - ok -> - {[Doc2 | AccValidated], AccErrors2}; - Error -> - {AccValidated, [{Doc, Error} | AccErrors2]} - end; - _ -> - % this doc isn't a leaf or already exists in the tree. - % ignore but consider it a success. - {AccValidated, AccErrors2} - end - end, - {[], AccErrors}, Bucket), - prep_and_validate_replicated_updates(Db, RestBuckets, RestOldInfo, - [ValidatedBucket | AccPrepped], AccErrors3) + {[], AccErrors}, + Bucket + ), + prep_and_validate_replicated_updates( + Db, + RestBuckets, + RestOldInfo, + [ValidatedBucket | AccPrepped], + AccErrors3 + ) end. - - -new_revid(#doc{body=Body, revs={OldStart,OldRevs}, atts=Atts, deleted=Deleted}) -> - DigestedAtts = lists:foldl(fun(Att, Acc) -> - [N, T, M] = couch_att:fetch([name, type, md5], Att), - case M == <<>> of - true -> Acc; - false -> [{N, T, M} | Acc] - end - end, [], Atts), +new_revid(#doc{body = Body, revs = {OldStart, OldRevs}, atts = Atts, deleted = Deleted}) -> + DigestedAtts = lists:foldl( + fun(Att, Acc) -> + [N, T, M] = couch_att:fetch([name, type, md5], Att), + case M == <<>> of + true -> Acc; + false -> [{N, T, M} | Acc] + end + end, + [], + Atts + ), case DigestedAtts of Atts2 when length(Atts) =/= length(Atts2) -> % We must have old style non-md5 attachments ?l2b(integer_to_list(couch_util:rand32())); Atts2 -> - OldRev = case OldRevs of [] -> 0; [OldRev0|_] -> OldRev0 end, - couch_hash:md5_hash(term_to_binary([Deleted, OldStart, OldRev, Body, Atts2], [{minor_version, 1}])) + OldRev = + case OldRevs of + [] -> 0; + [OldRev0 | _] -> OldRev0 + end, + couch_hash:md5_hash( + term_to_binary([Deleted, OldStart, OldRev, Body, Atts2], [{minor_version, 1}]) + ) end. new_revs([], OutBuckets, IdRevsAcc) -> {lists:reverse(OutBuckets), IdRevsAcc}; -new_revs([Bucket|RestBuckets], OutBuckets, IdRevsAcc) -> +new_revs([Bucket | RestBuckets], OutBuckets, IdRevsAcc) -> {NewBucket, IdRevsAcc3} = lists:mapfoldl( - fun(#doc{revs={Start, RevIds}}=Doc, IdRevsAcc2)-> - NewRevId = new_revid(Doc), - {Doc#doc{revs={Start+1, [NewRevId | RevIds]}}, - [{doc_tag(Doc), {ok, {Start+1, NewRevId}}} | IdRevsAcc2]} - end, IdRevsAcc, Bucket), - new_revs(RestBuckets, [NewBucket|OutBuckets], IdRevsAcc3). - -check_dup_atts(#doc{atts=Atts}=Doc) -> - lists:foldl(fun(Att, Names) -> - Name = couch_att:fetch(name, Att), - case ordsets:is_element(Name, Names) of - true -> throw({bad_request, <<"Duplicate attachments">>}); - false -> ordsets:add_element(Name, Names) - end - end, ordsets:new(), Atts), + fun(#doc{revs = {Start, RevIds}} = Doc, IdRevsAcc2) -> + NewRevId = new_revid(Doc), + {Doc#doc{revs = {Start + 1, [NewRevId | RevIds]}}, [ + {doc_tag(Doc), {ok, {Start + 1, NewRevId}}} | IdRevsAcc2 + ]} + end, + IdRevsAcc, + Bucket + ), + new_revs(RestBuckets, [NewBucket | OutBuckets], IdRevsAcc3). + +check_dup_atts(#doc{atts = Atts} = Doc) -> + lists:foldl( + fun(Att, Names) -> + Name = couch_att:fetch(name, Att), + case ordsets:is_element(Name, Names) of + true -> throw({bad_request, <<"Duplicate attachments">>}); + false -> ordsets:add_element(Name, Names) + end + end, + ordsets:new(), + Atts + ), Doc. tag_docs([]) -> []; -tag_docs([#doc{meta=Meta}=Doc | Rest]) -> - [Doc#doc{meta=[{ref, make_ref()} | Meta]} | tag_docs(Rest)]. +tag_docs([#doc{meta = Meta} = Doc | Rest]) -> + [Doc#doc{meta = [{ref, make_ref()} | Meta]} | tag_docs(Rest)]. -doc_tag(#doc{meta=Meta}) -> +doc_tag(#doc{meta = Meta}) -> case lists:keyfind(ref, 1, Meta) of {ref, Ref} when is_reference(Ref) -> Ref; false -> throw(doc_not_tagged); @@ -1187,58 +1326,105 @@ update_docs(Db, Docs0, Options, replicated_changes) -> Docs = tag_docs(Docs0), PrepValidateFun = fun(Db0, DocBuckets0, ExistingDocInfos) -> - prep_and_validate_replicated_updates(Db0, DocBuckets0, - ExistingDocInfos, [], []) + prep_and_validate_replicated_updates( + Db0, + DocBuckets0, + ExistingDocInfos, + [], + [] + ) end, - {ok, DocBuckets, NonRepDocs, DocErrors} - = before_docs_update(Db, Docs, PrepValidateFun, replicated_changes), + {ok, DocBuckets, NonRepDocs, DocErrors} = + before_docs_update(Db, Docs, PrepValidateFun, replicated_changes), - DocBuckets2 = [[doc_flush_atts(Db, check_dup_atts(Doc)) - || Doc <- Bucket] || Bucket <- DocBuckets], - {ok, _} = write_and_commit(Db, DocBuckets2, - NonRepDocs, [merge_conflicts | Options]), + DocBuckets2 = [ + [ + doc_flush_atts(Db, check_dup_atts(Doc)) + || Doc <- Bucket + ] + || Bucket <- DocBuckets + ], + {ok, _} = write_and_commit( + Db, + DocBuckets2, + NonRepDocs, + [merge_conflicts | Options] + ), {ok, DocErrors}; - update_docs(Db, Docs0, Options, interactive_edit) -> Docs = tag_docs(Docs0), AllOrNothing = lists:member(all_or_nothing, Options), PrepValidateFun = fun(Db0, DocBuckets0, ExistingDocInfos) -> - prep_and_validate_updates(Db0, DocBuckets0, ExistingDocInfos, - AllOrNothing, [], []) + prep_and_validate_updates( + Db0, + DocBuckets0, + ExistingDocInfos, + AllOrNothing, + [], + [] + ) end, - {ok, DocBuckets, NonRepDocs, DocErrors} - = before_docs_update(Db, Docs, PrepValidateFun, interactive_edit), - - if (AllOrNothing) and (DocErrors /= []) -> - RefErrorDict = dict:from_list([{doc_tag(Doc), Doc} || Doc <- Docs]), - {aborted, lists:map(fun({Ref, Error}) -> - #doc{id=Id,revs={Start,RevIds}} = dict:fetch(Ref, RefErrorDict), - case {Start, RevIds} of - {Pos, [RevId | _]} -> {{Id, {Pos, RevId}}, Error}; - {0, []} -> {{Id, {0, <<>>}}, Error} - end - end, DocErrors)}; - true -> - Options2 = if AllOrNothing -> [merge_conflicts]; - true -> [] end ++ Options, - DocBuckets2 = [[ - doc_flush_atts(Db, set_new_att_revpos( - check_dup_atts(Doc))) - || Doc <- B] || B <- DocBuckets], - {DocBuckets3, IdRevs} = new_revs(DocBuckets2, [], []), - - {ok, CommitResults} = write_and_commit(Db, DocBuckets3, - NonRepDocs, Options2), - - ResultsDict = lists:foldl(fun({Key, Resp}, ResultsAcc) -> - dict:store(Key, Resp, ResultsAcc) - end, dict:from_list(IdRevs), CommitResults ++ DocErrors), - {ok, lists:map(fun(Doc) -> - dict:fetch(doc_tag(Doc), ResultsDict) - end, Docs)} + {ok, DocBuckets, NonRepDocs, DocErrors} = + before_docs_update(Db, Docs, PrepValidateFun, interactive_edit), + + if + (AllOrNothing) and (DocErrors /= []) -> + RefErrorDict = dict:from_list([{doc_tag(Doc), Doc} || Doc <- Docs]), + {aborted, + lists:map( + fun({Ref, Error}) -> + #doc{id = Id, revs = {Start, RevIds}} = dict:fetch(Ref, RefErrorDict), + case {Start, RevIds} of + {Pos, [RevId | _]} -> {{Id, {Pos, RevId}}, Error}; + {0, []} -> {{Id, {0, <<>>}}, Error} + end + end, + DocErrors + )}; + true -> + Options2 = + if + AllOrNothing -> [merge_conflicts]; + true -> [] + end ++ Options, + DocBuckets2 = [ + [ + doc_flush_atts( + Db, + set_new_att_revpos( + check_dup_atts(Doc) + ) + ) + || Doc <- B + ] + || B <- DocBuckets + ], + {DocBuckets3, IdRevs} = new_revs(DocBuckets2, [], []), + + {ok, CommitResults} = write_and_commit( + Db, + DocBuckets3, + NonRepDocs, + Options2 + ), + + ResultsDict = lists:foldl( + fun({Key, Resp}, ResultsAcc) -> + dict:store(Key, Resp, ResultsAcc) + end, + dict:from_list(IdRevs), + CommitResults ++ DocErrors + ), + {ok, + lists:map( + fun(Doc) -> + dict:fetch(doc_tag(Doc), ResultsDict) + end, + Docs + )} end. % Returns the first available document on disk. Input list is a full rev path @@ -1246,10 +1432,10 @@ update_docs(Db, Docs0, Options, interactive_edit) -> make_first_doc_on_disk(_Db, _Id, _Pos, []) -> nil; make_first_doc_on_disk(Db, Id, Pos, [{_Rev, #doc{}} | RestPath]) -> - make_first_doc_on_disk(Db, Id, Pos-1, RestPath); -make_first_doc_on_disk(Db, Id, Pos, [{_Rev, ?REV_MISSING}|RestPath]) -> make_first_doc_on_disk(Db, Id, Pos - 1, RestPath); -make_first_doc_on_disk(Db, Id, Pos, [{_Rev, #leaf{deleted=IsDel, ptr=Sp}} |_]=DocPath) -> +make_first_doc_on_disk(Db, Id, Pos, [{_Rev, ?REV_MISSING} | RestPath]) -> + make_first_doc_on_disk(Db, Id, Pos - 1, RestPath); +make_first_doc_on_disk(Db, Id, Pos, [{_Rev, #leaf{deleted = IsDel, ptr = Sp}} | _] = DocPath) -> Revs = [Rev || {Rev, _} <- DocPath], make_doc(Db, Id, IsDel, Sp, {Pos, Revs}). @@ -1267,90 +1453,105 @@ collect_results_with_metrics(Pid, MRef, []) -> collect_results(Pid, MRef, ResultsAcc) -> receive - {result, Pid, Result} -> - collect_results(Pid, MRef, [Result | ResultsAcc]); - {done, Pid} -> - {ok, ResultsAcc}; - {retry, Pid} -> - retry; - {'DOWN', MRef, _, _, Reason} -> - exit(Reason) + {result, Pid, Result} -> + collect_results(Pid, MRef, [Result | ResultsAcc]); + {done, Pid} -> + {ok, ResultsAcc}; + {retry, Pid} -> + retry; + {'DOWN', MRef, _, _, Reason} -> + exit(Reason) end. -write_and_commit(#db{main_pid=Pid, user_ctx=Ctx}=Db, DocBuckets1, - NonRepDocs, Options) -> +write_and_commit( + #db{main_pid = Pid, user_ctx = Ctx} = Db, + DocBuckets1, + NonRepDocs, + Options +) -> DocBuckets = prepare_doc_summaries(Db, DocBuckets1), MergeConflicts = lists:member(merge_conflicts, Options), MRef = erlang:monitor(process, Pid), try Pid ! {update_docs, self(), DocBuckets, NonRepDocs, MergeConflicts}, case collect_results_with_metrics(Pid, MRef, []) of - {ok, Results} -> {ok, Results}; - retry -> - % This can happen if the db file we wrote to was swapped out by - % compaction. Retry by reopening the db and writing to the current file - {ok, Db2} = open(Db#db.name, [{user_ctx, Ctx}]), - DocBuckets2 = [ - [doc_flush_atts(Db2, Doc) || Doc <- Bucket] || - Bucket <- DocBuckets1 - ], - % We only retry once - DocBuckets3 = prepare_doc_summaries(Db2, DocBuckets2), - close(Db2), - Pid ! {update_docs, self(), DocBuckets3, NonRepDocs, MergeConflicts}, - case collect_results_with_metrics(Pid, MRef, []) of - {ok, Results} -> {ok, Results}; - retry -> throw({update_error, compaction_retry}) - end + {ok, Results} -> + {ok, Results}; + retry -> + % This can happen if the db file we wrote to was swapped out by + % compaction. Retry by reopening the db and writing to the current file + {ok, Db2} = open(Db#db.name, [{user_ctx, Ctx}]), + DocBuckets2 = [ + [doc_flush_atts(Db2, Doc) || Doc <- Bucket] + || Bucket <- DocBuckets1 + ], + % We only retry once + DocBuckets3 = prepare_doc_summaries(Db2, DocBuckets2), + close(Db2), + Pid ! {update_docs, self(), DocBuckets3, NonRepDocs, MergeConflicts}, + case collect_results_with_metrics(Pid, MRef, []) of + {ok, Results} -> {ok, Results}; + retry -> throw({update_error, compaction_retry}) + end end after erlang:demonitor(MRef, [flush]) end. - prepare_doc_summaries(Db, BucketList) -> - [lists:map( - fun(#doc{body = Body, atts = Atts} = Doc0) -> - DiskAtts = [couch_att:to_disk_term(Att) || Att <- Atts], - {ok, SizeInfo} = couch_att:size_info(Atts), - AttsStream = case Atts of - [Att | _] -> - {stream, StreamEngine} = couch_att:fetch(data, Att), - StreamEngine; - [] -> - nil + [ + lists:map( + fun(#doc{body = Body, atts = Atts} = Doc0) -> + DiskAtts = [couch_att:to_disk_term(Att) || Att <- Atts], + {ok, SizeInfo} = couch_att:size_info(Atts), + AttsStream = + case Atts of + [Att | _] -> + {stream, StreamEngine} = couch_att:fetch(data, Att), + StreamEngine; + [] -> + nil + end, + Doc1 = Doc0#doc{ + atts = DiskAtts, + meta = + [ + {size_info, SizeInfo}, + {atts_stream, AttsStream}, + {ejson_size, couch_ejson_size:encoded_size(Body)} + ] ++ Doc0#doc.meta + }, + couch_db_engine:serialize_doc(Db, Doc1) end, - Doc1 = Doc0#doc{ - atts = DiskAtts, - meta = [ - {size_info, SizeInfo}, - {atts_stream, AttsStream}, - {ejson_size, couch_ejson_size:encoded_size(Body)} - ] ++ Doc0#doc.meta - }, - couch_db_engine:serialize_doc(Db, Doc1) - end, - Bucket) || Bucket <- BucketList]. - + Bucket + ) + || Bucket <- BucketList + ]. before_docs_update(#db{validate_doc_funs = VDFuns} = Db, Docs, PVFun, UpdateType) -> increment_stat(Db, [couchdb, database_writes]), % Separate _local docs from normal docs IsLocal = fun - (#doc{id= <<?LOCAL_DOC_PREFIX, _/binary>>}) -> true; + (#doc{id = <<?LOCAL_DOC_PREFIX, _/binary>>}) -> true; (_) -> false end, {NonRepDocs, Docs2} = lists:partition(IsLocal, Docs), BucketList = group_alike_docs(Docs2), - DocBuckets = lists:map(fun(Bucket) -> - lists:map(fun(Doc) -> - DocWithBody = couch_doc:with_ejson_body(Doc), - couch_db_plugin:before_doc_update(Db, DocWithBody, UpdateType) - end, Bucket) - end, BucketList), + DocBuckets = lists:map( + fun(Bucket) -> + lists:map( + fun(Doc) -> + DocWithBody = couch_doc:with_ejson_body(Doc), + couch_db_plugin:before_doc_update(Db, DocWithBody, UpdateType) + end, + Bucket + ) + end, + BucketList + ), ValidatePred = fun (#doc{id = <<?DESIGN_DOC_PREFIX, _/binary>>}) -> true; @@ -1363,15 +1564,14 @@ before_docs_update(#db{validate_doc_funs = VDFuns} = Db, Docs, PVFun, UpdateType Ids = [Id || [#doc{id = Id} | _] <- DocBuckets], ExistingDocs = get_full_doc_infos(Db, Ids), {DocBuckets2, DocErrors} = PVFun(Db, DocBuckets, ExistingDocs), - % remove empty buckets + % remove empty buckets DocBuckets3 = [Bucket || Bucket <- DocBuckets2, Bucket /= []], {ok, DocBuckets3, NonRepDocs, DocErrors}; false -> {ok, DocBuckets, NonRepDocs, []} end. - -set_new_att_revpos(#doc{revs={RevPos,_Revs},atts=Atts0}=Doc) -> +set_new_att_revpos(#doc{revs = {RevPos, _Revs}, atts = Atts0} = Doc) -> Atts = lists:map( fun(Att) -> case couch_att:fetch(data, Att) of @@ -1379,29 +1579,36 @@ set_new_att_revpos(#doc{revs={RevPos,_Revs},atts=Atts0}=Doc) -> {stream, _} -> Att; {Fd, _} when is_pid(Fd) -> Att; % write required so update RevPos - _ -> couch_att:store(revpos, RevPos+1, Att) + _ -> couch_att:store(revpos, RevPos + 1, Att) end - end, Atts0), + end, + Atts0 + ), Doc#doc{atts = Atts}. - doc_flush_atts(Db, Doc) -> - Doc#doc{atts=[couch_att:flush(Db, Att) || Att <- Doc#doc.atts]}. - + Doc#doc{atts = [couch_att:flush(Db, Att) || Att <- Doc#doc.atts]}. compressible_att_type(MimeType) when is_binary(MimeType) -> compressible_att_type(?b2l(MimeType)); compressible_att_type(MimeType) -> TypeExpList = re:split( - config:get("attachments", "compressible_types", - ?DEFAULT_COMPRESSIBLE_TYPES), + config:get( + "attachments", + "compressible_types", + ?DEFAULT_COMPRESSIBLE_TYPES + ), "\\s*,\\s*", [{return, list}] ), lists:any( fun(TypeExp) -> - Regexp = ["^\\s*", re:replace(TypeExp, "\\*", ".*"), - "(?:\\s*;.*?)?\\s*", $$], + Regexp = [ + "^\\s*", + re:replace(TypeExp, "\\*", ".*"), + "(?:\\s*;.*?)?\\s*", + $$ + ], re:run(MimeType, Regexp, [caseless]) =/= nomatch end, [T || T <- TypeExpList, T /= []] @@ -1419,74 +1626,80 @@ compressible_att_type(MimeType) -> % pretend that no Content-MD5 exists. with_stream(Db, Att, Fun) -> [InMd5, Type, Enc] = couch_att:fetch([md5, type, encoding], Att), - BufferSize = config:get_integer("couchdb", - "attachment_stream_buffer_size", 4096), - Options = case (Enc =:= identity) andalso compressible_att_type(Type) of - true -> - CompLevel = config:get_integer( - "attachments", "compression_level", 8), - [ - {buffer_size, BufferSize}, - {encoding, gzip}, - {compression_level, CompLevel} - ]; - _ -> - [{buffer_size, BufferSize}] - end, + BufferSize = config:get_integer( + "couchdb", + "attachment_stream_buffer_size", + 4096 + ), + Options = + case (Enc =:= identity) andalso compressible_att_type(Type) of + true -> + CompLevel = config:get_integer( + "attachments", "compression_level", 8 + ), + [ + {buffer_size, BufferSize}, + {encoding, gzip}, + {compression_level, CompLevel} + ]; + _ -> + [{buffer_size, BufferSize}] + end, {ok, OutputStream} = open_write_stream(Db, Options), - ReqMd5 = case Fun(OutputStream) of - {md5, FooterMd5} -> - case InMd5 of - md5_in_footer -> FooterMd5; - _ -> InMd5 - end; - _ -> - InMd5 - end, + ReqMd5 = + case Fun(OutputStream) of + {md5, FooterMd5} -> + case InMd5 of + md5_in_footer -> FooterMd5; + _ -> InMd5 + end; + _ -> + InMd5 + end, {StreamEngine, Len, IdentityLen, Md5, IdentityMd5} = couch_stream:close(OutputStream), couch_util:check_md5(IdentityMd5, ReqMd5), - {AttLen, DiskLen, NewEnc} = case Enc of - identity -> - case {Md5, IdentityMd5} of - {Same, Same} -> - {Len, IdentityLen, identity}; - _ -> - {Len, IdentityLen, gzip} - end; - gzip -> - case couch_att:fetch([att_len, disk_len], Att) of - [AL, DL] when AL =:= undefined orelse DL =:= undefined -> - % Compressed attachment uploaded through the standalone API. - {Len, Len, gzip}; - [AL, DL] -> - % This case is used for efficient push-replication, where a - % compressed attachment is located in the body of multipart - % content-type request. - {AL, DL, gzip} - end - end, - couch_att:store([ - {data, {stream, StreamEngine}}, - {att_len, AttLen}, - {disk_len, DiskLen}, - {md5, Md5}, - {encoding, NewEnc} - ], Att). - + {AttLen, DiskLen, NewEnc} = + case Enc of + identity -> + case {Md5, IdentityMd5} of + {Same, Same} -> + {Len, IdentityLen, identity}; + _ -> + {Len, IdentityLen, gzip} + end; + gzip -> + case couch_att:fetch([att_len, disk_len], Att) of + [AL, DL] when AL =:= undefined orelse DL =:= undefined -> + % Compressed attachment uploaded through the standalone API. + {Len, Len, gzip}; + [AL, DL] -> + % This case is used for efficient push-replication, where a + % compressed attachment is located in the body of multipart + % content-type request. + {AL, DL, gzip} + end + end, + couch_att:store( + [ + {data, {stream, StreamEngine}}, + {att_len, AttLen}, + {disk_len, DiskLen}, + {md5, Md5}, + {encoding, NewEnc} + ], + Att + ). open_write_stream(Db, Options) -> couch_db_engine:open_write_stream(Db, Options). - open_read_stream(Db, AttState) -> couch_db_engine:open_read_stream(Db, AttState). - is_active_stream(Db, StreamEngine) -> couch_db_engine:is_active_stream(Db, StreamEngine). - calculate_start_seq(_Db, _Node, Seq) when is_integer(Seq) -> Seq; calculate_start_seq(Db, Node, {Seq, Uuid}) -> @@ -1498,30 +1711,44 @@ calculate_start_seq(Db, _Node, {Seq, {split, Uuid}, EpochNode}) -> % Find last replicated sequence from split source to target mem3_rep:find_split_target_seq(Db, EpochNode, Uuid, Seq); false -> - couch_log:warning("~p calculate_start_seq not owner " + couch_log:warning( + "~p calculate_start_seq not owner " "db: ~p, seq: ~p, uuid: ~p, epoch_node: ~p, epochs: ~p", - [?MODULE, Db#db.name, Seq, Uuid, EpochNode, get_epochs(Db)]), + [?MODULE, Db#db.name, Seq, Uuid, EpochNode, get_epochs(Db)] + ), 0 end; calculate_start_seq(Db, Node, {Seq, Uuid, EpochNode}) -> case is_prefix(Uuid, get_uuid(Db)) of true -> case is_owner(EpochNode, Seq, get_epochs(Db)) of - true -> Seq; + true -> + Seq; false -> %% Shard might have been moved from another node. We %% matched the uuid already, try to find last viable %% sequence we can use - couch_log:warning( "~p calculate_start_seq not owner, " + couch_log:warning( + "~p calculate_start_seq not owner, " " trying replacement db: ~p, seq: ~p, uuid: ~p, " - "epoch_node: ~p, epochs: ~p", [?MODULE, Db#db.name, - Seq, Uuid, EpochNode, get_epochs(Db)]), + "epoch_node: ~p, epochs: ~p", + [ + ?MODULE, + Db#db.name, + Seq, + Uuid, + EpochNode, + get_epochs(Db) + ] + ), calculate_start_seq(Db, Node, {replace, EpochNode, Uuid, Seq}) end; false -> - couch_log:warning("~p calculate_start_seq uuid prefix mismatch " + couch_log:warning( + "~p calculate_start_seq uuid prefix mismatch " "db: ~p, seq: ~p, uuid: ~p, epoch_node: ~p", - [?MODULE, Db#db.name, Seq, Uuid, EpochNode]), + [?MODULE, Db#db.name, Seq, Uuid, EpochNode] + ), %% The file was rebuilt, most likely in a different %% order, so rewind. 0 @@ -1531,38 +1758,37 @@ calculate_start_seq(Db, _Node, {replace, OriginalNode, Uuid, Seq}) -> true -> try start_seq(get_epochs(Db), OriginalNode, Seq) - catch throw:epoch_mismatch -> - couch_log:warning("~p start_seq duplicate uuid on node: ~p " - "db: ~p, seq: ~p, uuid: ~p, epoch_node: ~p", - [?MODULE, node(), Db#db.name, Seq, Uuid, OriginalNode]), - 0 + catch + throw:epoch_mismatch -> + couch_log:warning( + "~p start_seq duplicate uuid on node: ~p " + "db: ~p, seq: ~p, uuid: ~p, epoch_node: ~p", + [?MODULE, node(), Db#db.name, Seq, Uuid, OriginalNode] + ), + 0 end; false -> {replace, OriginalNode, Uuid, Seq} end. - validate_epochs(Epochs) -> %% Assert uniqueness. case length(Epochs) == length(lists:ukeysort(2, Epochs)) of - true -> ok; + true -> ok; false -> erlang:error(duplicate_epoch) end, %% Assert order. case Epochs == lists:sort(fun({_, A}, {_, B}) -> B =< A end, Epochs) of - true -> ok; + true -> ok; false -> erlang:error(epoch_order) end. - is_prefix(Pattern, Subject) -> - binary:longest_common_prefix([Pattern, Subject]) == size(Pattern). - + binary:longest_common_prefix([Pattern, Subject]) == size(Pattern). is_owner(Node, Seq, Epochs) -> Node =:= owner_of(Epochs, Seq). - owner_of(Db, Seq) when not is_list(Db) -> owner_of(get_epochs(Db), Seq); owner_of([], _Seq) -> @@ -1572,7 +1798,6 @@ owner_of([{EpochNode, EpochSeq} | _Rest], Seq) when Seq > EpochSeq -> owner_of([_ | Rest], Seq) -> owner_of(Rest, Seq). - start_seq([{OrigNode, EpochSeq} | _], OrigNode, Seq) when Seq > EpochSeq -> %% OrigNode is the owner of the Seq so we can safely stream from there Seq; @@ -1586,43 +1811,34 @@ start_seq([_ | Rest], OrigNode, Seq) -> start_seq([], _OrigNode, _Seq) -> throw(epoch_mismatch). - fold_docs(Db, UserFun, UserAcc) -> fold_docs(Db, UserFun, UserAcc, []). fold_docs(Db, UserFun, UserAcc, Options) -> couch_db_engine:fold_docs(Db, UserFun, UserAcc, Options). - fold_local_docs(Db, UserFun, UserAcc, Options) -> couch_db_engine:fold_local_docs(Db, UserFun, UserAcc, Options). - fold_design_docs(Db, UserFun, UserAcc, Options1) -> Options2 = set_design_doc_keys(Options1), couch_db_engine:fold_docs(Db, UserFun, UserAcc, Options2). - fold_changes(Db, StartSeq, UserFun, UserAcc) -> fold_changes(Db, StartSeq, UserFun, UserAcc, []). - fold_changes(Db, StartSeq, UserFun, UserAcc, Opts) -> couch_db_engine:fold_changes(Db, StartSeq, UserFun, UserAcc, Opts). - fold_purge_infos(Db, StartPurgeSeq, Fun, Acc) -> fold_purge_infos(Db, StartPurgeSeq, Fun, Acc, []). - fold_purge_infos(Db, StartPurgeSeq, UFun, UAcc, Opts) -> couch_db_engine:fold_purge_infos(Db, StartPurgeSeq, UFun, UAcc, Opts). - count_changes_since(Db, SinceSeq) -> couch_db_engine:count_changes_since(Db, SinceSeq). - %%% Internal function %%% open_doc_revs_int(Db, IdRevs, Options) -> Ids = [Id || {Id, _Revs} <- IdRevs], @@ -1630,106 +1846,125 @@ open_doc_revs_int(Db, IdRevs, Options) -> lists:zipwith( fun({Id, Revs}, Lookup) -> case Lookup of - #full_doc_info{rev_tree=RevTree} -> - {FoundRevs, MissingRevs} = - case Revs of - all -> - {couch_key_tree:get_all_leafs(RevTree), []}; - _ -> - case lists:member(latest, Options) of - true -> - couch_key_tree:get_key_leafs(RevTree, Revs); - false -> - couch_key_tree:get(RevTree, Revs) - end - end, - FoundResults = - lists:map(fun({Value, {Pos, [Rev|_]}=FoundRevPath}) -> - case Value of - ?REV_MISSING -> - % we have the rev in our list but know nothing about it - {{not_found, missing}, {Pos, Rev}}; - #leaf{deleted=IsDeleted, ptr=SummaryPtr} -> - {ok, make_doc(Db, Id, IsDeleted, SummaryPtr, FoundRevPath)} - end - end, FoundRevs), - Results = FoundResults ++ [{{not_found, missing}, MissingRev} || MissingRev <- MissingRevs], - {ok, Results}; - not_found when Revs == all -> - {ok, []}; - not_found -> - {ok, [{{not_found, missing}, Rev} || Rev <- Revs]} + #full_doc_info{rev_tree = RevTree} -> + {FoundRevs, MissingRevs} = + case Revs of + all -> + {couch_key_tree:get_all_leafs(RevTree), []}; + _ -> + case lists:member(latest, Options) of + true -> + couch_key_tree:get_key_leafs(RevTree, Revs); + false -> + couch_key_tree:get(RevTree, Revs) + end + end, + FoundResults = + lists:map( + fun({Value, {Pos, [Rev | _]} = FoundRevPath}) -> + case Value of + ?REV_MISSING -> + % we have the rev in our list but know nothing about it + {{not_found, missing}, {Pos, Rev}}; + #leaf{deleted = IsDeleted, ptr = SummaryPtr} -> + {ok, make_doc(Db, Id, IsDeleted, SummaryPtr, FoundRevPath)} + end + end, + FoundRevs + ), + Results = + FoundResults ++ + [{{not_found, missing}, MissingRev} || MissingRev <- MissingRevs], + {ok, Results}; + not_found when Revs == all -> + {ok, []}; + not_found -> + {ok, [{{not_found, missing}, Rev} || Rev <- Revs]} end end, - IdRevs, LookupResults). + IdRevs, + LookupResults + ). open_doc_int(Db, <<?LOCAL_DOC_PREFIX, _/binary>> = Id, Options) -> case couch_db_engine:open_local_docs(Db, [Id]) of - [#doc{} = Doc] -> - apply_open_options({ok, Doc}, Options); - [not_found] -> - {not_found, missing} + [#doc{} = Doc] -> + apply_open_options({ok, Doc}, Options); + [not_found] -> + {not_found, missing} end; -open_doc_int(Db, #doc_info{id=Id,revs=[RevInfo|_]}=DocInfo, Options) -> - #rev_info{deleted=IsDeleted,rev={Pos,RevId},body_sp=Bp} = RevInfo, - Doc = make_doc(Db, Id, IsDeleted, Bp, {Pos,[RevId]}), +open_doc_int(Db, #doc_info{id = Id, revs = [RevInfo | _]} = DocInfo, Options) -> + #rev_info{deleted = IsDeleted, rev = {Pos, RevId}, body_sp = Bp} = RevInfo, + Doc = make_doc(Db, Id, IsDeleted, Bp, {Pos, [RevId]}), apply_open_options( - {ok, Doc#doc{meta=doc_meta_info(DocInfo, [], Options)}}, Options); -open_doc_int(Db, #full_doc_info{id=Id,rev_tree=RevTree}=FullDocInfo, Options) -> - #doc_info{revs=[#rev_info{deleted=IsDeleted,rev=Rev,body_sp=Bp}|_]} = + {ok, Doc#doc{meta = doc_meta_info(DocInfo, [], Options)}}, Options + ); +open_doc_int(Db, #full_doc_info{id = Id, rev_tree = RevTree} = FullDocInfo, Options) -> + #doc_info{revs = [#rev_info{deleted = IsDeleted, rev = Rev, body_sp = Bp} | _]} = DocInfo = couch_doc:to_doc_info(FullDocInfo), {[{_, RevPath}], []} = couch_key_tree:get(RevTree, [Rev]), Doc = make_doc(Db, Id, IsDeleted, Bp, RevPath), apply_open_options( - {ok, Doc#doc{meta=doc_meta_info(DocInfo, RevTree, Options)}}, Options); + {ok, Doc#doc{meta = doc_meta_info(DocInfo, RevTree, Options)}}, Options + ); open_doc_int(Db, Id, Options) -> case get_full_doc_info(Db, Id) of - #full_doc_info{} = FullDocInfo -> - open_doc_int(Db, FullDocInfo, Options); - not_found -> - {not_found, missing} + #full_doc_info{} = FullDocInfo -> + open_doc_int(Db, FullDocInfo, Options); + not_found -> + {not_found, missing} end. -doc_meta_info(#doc_info{high_seq=Seq,revs=[#rev_info{rev=Rev}|RestInfo]}, RevTree, Options) -> +doc_meta_info( + #doc_info{high_seq = Seq, revs = [#rev_info{rev = Rev} | RestInfo]}, RevTree, Options +) -> case lists:member(revs_info, Options) of - false -> []; - true -> - {[{Pos, RevPath}],[]} = - couch_key_tree:get_full_key_paths(RevTree, [Rev]), - - [{revs_info, Pos, lists:map( - fun({Rev1, ?REV_MISSING}) -> - {Rev1, missing}; - ({Rev1, Leaf}) -> - case Leaf#leaf.deleted of - true -> - {Rev1, deleted}; - false -> - {Rev1, available} - end - end, RevPath)}] - end ++ - case lists:member(conflicts, Options) of - false -> []; - true -> - case [Rev1 || #rev_info{rev=Rev1,deleted=false} <- RestInfo] of - [] -> []; - ConflictRevs -> [{conflicts, ConflictRevs}] - end - end ++ - case lists:member(deleted_conflicts, Options) of - false -> []; - true -> - case [Rev1 || #rev_info{rev=Rev1,deleted=true} <- RestInfo] of - [] -> []; - DelConflictRevs -> [{deleted_conflicts, DelConflictRevs}] - end - end ++ - case lists:member(local_seq, Options) of - false -> []; - true -> [{local_seq, Seq}] - end. + false -> + []; + true -> + {[{Pos, RevPath}], []} = + couch_key_tree:get_full_key_paths(RevTree, [Rev]), + [ + {revs_info, Pos, + lists:map( + fun + ({Rev1, ?REV_MISSING}) -> + {Rev1, missing}; + ({Rev1, Leaf}) -> + case Leaf#leaf.deleted of + true -> + {Rev1, deleted}; + false -> + {Rev1, available} + end + end, + RevPath + )} + ] + end ++ + case lists:member(conflicts, Options) of + false -> + []; + true -> + case [Rev1 || #rev_info{rev = Rev1, deleted = false} <- RestInfo] of + [] -> []; + ConflictRevs -> [{conflicts, ConflictRevs}] + end + end ++ + case lists:member(deleted_conflicts, Options) of + false -> + []; + true -> + case [Rev1 || #rev_info{rev = Rev1, deleted = true} <- RestInfo] of + [] -> []; + DelConflictRevs -> [{deleted_conflicts, DelConflictRevs}] + end + end ++ + case lists:member(local_seq, Options) of + false -> []; + true -> [{local_seq, Seq}] + end. make_doc(_Db, Id, Deleted, nil = _Bp, RevisionPath) -> #doc{ @@ -1747,29 +1982,29 @@ make_doc(#db{} = Db, Id, Deleted, Bp, {Pos, Revs}) -> body = Bp, deleted = Deleted }), - Doc1 = case Doc0#doc.atts of - BinAtts when is_binary(BinAtts) -> - Doc0#doc{ - atts = couch_compress:decompress(BinAtts) - }; - ListAtts when is_list(ListAtts) -> - Doc0 - end, + Doc1 = + case Doc0#doc.atts of + BinAtts when is_binary(BinAtts) -> + Doc0#doc{ + atts = couch_compress:decompress(BinAtts) + }; + ListAtts when is_list(ListAtts) -> + Doc0 + end, after_doc_read(Db, Doc1#doc{ atts = [couch_att:from_disk_term(Db, T) || T <- Doc1#doc.atts] }). - after_doc_read(#db{} = Db, Doc) -> DocWithBody = couch_doc:with_ejson_body(Doc), couch_db_plugin:after_doc_read(Db, DocWithBody). increment_stat(#db{options = Options}, Stat) -> case lists:member(sys_db, Options) of - true -> - ok; - false -> - couch_stats:increment_counter(Stat) + true -> + ok; + false -> + couch_stats:increment_counter(Stat) end. -spec normalize_dbname(list() | binary()) -> binary(). @@ -1779,23 +2014,22 @@ normalize_dbname(DbName) when is_list(DbName) -> normalize_dbname(DbName) when is_binary(DbName) -> mem3:dbname(couch_util:drop_dot_couch_ext(DbName)). - -spec dbname_suffix(list() | binary()) -> binary(). dbname_suffix(DbName) -> filename:basename(normalize_dbname(DbName)). - validate_dbname(DbName) when is_list(DbName) -> validate_dbname(?l2b(DbName)); validate_dbname(DbName) when is_binary(DbName) -> Normalized = normalize_dbname(DbName), couch_db_plugin:validate_dbname( - DbName, Normalized, fun validate_dbname_int/2). + DbName, Normalized, fun validate_dbname_int/2 + ). validate_dbname_int(DbName, Normalized) when is_binary(DbName) -> DbNoExt = couch_util:drop_dot_couch_ext(DbName), - case re:run(DbNoExt, ?DBNAME_REGEX, [{capture,none}, dollar_endonly]) of + case re:run(DbNoExt, ?DBNAME_REGEX, [{capture, none}, dollar_endonly]) of match -> ok; nomatch -> @@ -1811,76 +2045,81 @@ is_system_db_name(DbName) when is_binary(DbName) -> Normalized = normalize_dbname(DbName), Suffix = filename:basename(Normalized), case {filename:dirname(Normalized), lists:member(Suffix, ?SYSTEM_DATABASES)} of - {<<".">>, Result} -> Result; - {_Prefix, false} -> false; + {<<".">>, Result} -> + Result; + {_Prefix, false} -> + false; {Prefix, true} -> - ReOpts = [{capture,none}, dollar_endonly], + ReOpts = [{capture, none}, dollar_endonly], re:run(Prefix, ?DBNAME_REGEX, ReOpts) == match end. set_design_doc_keys(Options1) -> - Dir = case lists:keyfind(dir, 1, Options1) of - {dir, D0} -> D0; - _ -> fwd - end, + Dir = + case lists:keyfind(dir, 1, Options1) of + {dir, D0} -> D0; + _ -> fwd + end, Options2 = set_design_doc_start_key(Options1, Dir), set_design_doc_end_key(Options2, Dir). - -define(FIRST_DDOC_KEY, <<"_design/">>). -define(LAST_DDOC_KEY, <<"_design0">>). - set_design_doc_start_key(Options, fwd) -> Key1 = couch_util:get_value(start_key, Options, ?FIRST_DDOC_KEY), - Key2 = case Key1 < ?FIRST_DDOC_KEY of - true -> ?FIRST_DDOC_KEY; - false -> Key1 - end, + Key2 = + case Key1 < ?FIRST_DDOC_KEY of + true -> ?FIRST_DDOC_KEY; + false -> Key1 + end, lists:keystore(start_key, 1, Options, {start_key, Key2}); set_design_doc_start_key(Options, rev) -> Key1 = couch_util:get_value(start_key, Options, ?LAST_DDOC_KEY), - Key2 = case Key1 > ?LAST_DDOC_KEY of - true -> ?LAST_DDOC_KEY; - false -> Key1 - end, + Key2 = + case Key1 > ?LAST_DDOC_KEY of + true -> ?LAST_DDOC_KEY; + false -> Key1 + end, lists:keystore(start_key, 1, Options, {start_key, Key2}). - set_design_doc_end_key(Options, fwd) -> case couch_util:get_value(end_key_gt, Options) of undefined -> Key1 = couch_util:get_value(end_key, Options, ?LAST_DDOC_KEY), - Key2 = case Key1 > ?LAST_DDOC_KEY of - true -> ?LAST_DDOC_KEY; - false -> Key1 - end, + Key2 = + case Key1 > ?LAST_DDOC_KEY of + true -> ?LAST_DDOC_KEY; + false -> Key1 + end, lists:keystore(end_key, 1, Options, {end_key, Key2}); EKeyGT -> - Key2 = case EKeyGT > ?LAST_DDOC_KEY of - true -> ?LAST_DDOC_KEY; - false -> EKeyGT - end, + Key2 = + case EKeyGT > ?LAST_DDOC_KEY of + true -> ?LAST_DDOC_KEY; + false -> EKeyGT + end, lists:keystore(end_key_gt, 1, Options, {end_key_gt, Key2}) end; set_design_doc_end_key(Options, rev) -> case couch_util:get_value(end_key_gt, Options) of undefined -> Key1 = couch_util:get_value(end_key, Options, ?LAST_DDOC_KEY), - Key2 = case Key1 < ?FIRST_DDOC_KEY of - true -> ?FIRST_DDOC_KEY; - false -> Key1 - end, + Key2 = + case Key1 < ?FIRST_DDOC_KEY of + true -> ?FIRST_DDOC_KEY; + false -> Key1 + end, lists:keystore(end_key, 1, Options, {end_key, Key2}); EKeyGT -> - Key2 = case EKeyGT < ?FIRST_DDOC_KEY of - true -> ?FIRST_DDOC_KEY; - false -> EKeyGT - end, + Key2 = + case EKeyGT < ?FIRST_DDOC_KEY of + true -> ?FIRST_DDOC_KEY; + false -> EKeyGT + end, lists:keystore(end_key_gt, 1, Options, {end_key_gt, Key2}) end. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -1900,11 +2139,14 @@ teardown(_) -> validate_dbname_success_test_() -> Cases = - generate_cases_with_shards("long/co$mplex-/path+/something") - ++ generate_cases_with_shards("something") - ++ lists:append( - [generate_cases_with_shards(?b2l(SystemDb)) - || SystemDb <- ?SYSTEM_DATABASES]), + generate_cases_with_shards("long/co$mplex-/path+/something") ++ + generate_cases_with_shards("something") ++ + lists:append( + [ + generate_cases_with_shards(?b2l(SystemDb)) + || SystemDb <- ?SYSTEM_DATABASES + ] + ), { setup, fun setup_all/0, @@ -1918,12 +2160,13 @@ validate_dbname_success_test_() -> }. validate_dbname_fail_test_() -> - Cases = generate_cases("_long/co$mplex-/path+/_something") - ++ generate_cases("_something") - ++ generate_cases_with_shards("long/co$mplex-/path+/_something#") - ++ generate_cases_with_shards("long/co$mplex-/path+/some.thing") - ++ generate_cases("!abcdefg/werwej/_users") - ++ generate_cases_with_shards("!abcdefg/werwej/_users"), + Cases = + generate_cases("_long/co$mplex-/path+/_something") ++ + generate_cases("_something") ++ + generate_cases_with_shards("long/co$mplex-/path+/_something#") ++ + generate_cases_with_shards("long/co$mplex-/path+/some.thing") ++ + generate_cases("!abcdefg/werwej/_users") ++ + generate_cases_with_shards("!abcdefg/werwej/_users"), { setup, fun setup_all/0, @@ -1937,41 +2180,56 @@ validate_dbname_fail_test_() -> }. normalize_dbname_test_() -> - Cases = generate_cases_with_shards("long/co$mplex-/path+/_something") - ++ generate_cases_with_shards("_something"), + Cases = + generate_cases_with_shards("long/co$mplex-/path+/_something") ++ + generate_cases_with_shards("_something"), WithExpected = [{?l2b(filename:rootname(A)), B} || {A, B} <- Cases], - [{test_name({Expected, Db}), ?_assertEqual(Expected, normalize_dbname(Db))} - || {Expected, Db} <- WithExpected]. + [ + {test_name({Expected, Db}), ?_assertEqual(Expected, normalize_dbname(Db))} + || {Expected, Db} <- WithExpected + ]. dbname_suffix_test_() -> - Cases = generate_cases_with_shards("long/co$mplex-/path+/_something") - ++ generate_cases_with_shards("_something"), + Cases = + generate_cases_with_shards("long/co$mplex-/path+/_something") ++ + generate_cases_with_shards("_something"), WithExpected = [{?l2b(filename:basename(Arg)), Db} || {Arg, Db} <- Cases], - [{test_name({Expected, Db}), ?_assertEqual(Expected, dbname_suffix(Db))} - || {Expected, Db} <- WithExpected]. + [ + {test_name({Expected, Db}), ?_assertEqual(Expected, dbname_suffix(Db))} + || {Expected, Db} <- WithExpected + ]. is_system_db_name_test_() -> - Cases = lists:append([ - generate_cases_with_shards("long/co$mplex-/path+/" ++ ?b2l(Db)) - || Db <- ?SYSTEM_DATABASES] - ++ [generate_cases_with_shards(?b2l(Db)) || Db <- ?SYSTEM_DATABASES - ]), - WithExpected = [{?l2b(filename:basename(filename:rootname(Arg))), Db} - || {Arg, Db} <- Cases], - [{test_name({Expected, Db}) ++ " in ?SYSTEM_DATABASES", - ?_assert(is_system_db_name(Db))} || {Expected, Db} <- WithExpected]. + Cases = lists:append( + [ + generate_cases_with_shards("long/co$mplex-/path+/" ++ ?b2l(Db)) + || Db <- ?SYSTEM_DATABASES + ] ++ + [generate_cases_with_shards(?b2l(Db)) || Db <- ?SYSTEM_DATABASES] + ), + WithExpected = [ + {?l2b(filename:basename(filename:rootname(Arg))), Db} + || {Arg, Db} <- Cases + ], + [ + {test_name({Expected, Db}) ++ " in ?SYSTEM_DATABASES", ?_assert(is_system_db_name(Db))} + || {Expected, Db} <- WithExpected + ]. should_pass_validate_dbname(DbName) -> {test_name(DbName), ?_assertEqual(ok, validate_dbname(DbName))}. should_fail_validate_dbname(DbName) -> - {test_name(DbName), ?_test(begin - Result = validate_dbname(DbName), - ?assertMatch({error, {illegal_database_name, _}}, Result), - {error, {illegal_database_name, FailedDbName}} = Result, - ?assertEqual(to_binary(DbName), FailedDbName), - ok - end)}. + { + test_name(DbName), + ?_test(begin + Result = validate_dbname(DbName), + ?assertMatch({error, {illegal_database_name, _}}, Result), + {error, {illegal_database_name, FailedDbName}} = Result, + ?assertEqual(to_binary(DbName), FailedDbName), + ok + end) + }. calculate_start_seq_test_() -> { @@ -2087,9 +2345,12 @@ generate_cases_with_shards(DbName) -> DbNameWithShard = add_shard(DbName), DbNameWithShardAndExtension = add_shard(DbName) ++ ".couch", Cases = [ - DbName, ?l2b(DbName), - DbNameWithShard, ?l2b(DbNameWithShard), - DbNameWithShardAndExtension, ?l2b(DbNameWithShardAndExtension) + DbName, + ?l2b(DbName), + DbNameWithShard, + ?l2b(DbNameWithShard), + DbNameWithShardAndExtension, + ?l2b(DbNameWithShardAndExtension) ], [{DbName, Case} || Case <- Cases]. diff --git a/src/couch/src/couch_db_engine.erl b/src/couch/src/couch_db_engine.erl index 918dabcca..de4a42495 100644 --- a/src/couch/src/couch_db_engine.erl +++ b/src/couch/src/couch_db_engine.erl @@ -12,11 +12,9 @@ -module(couch_db_engine). - -include("couch_db.hrl"). -include("couch_db_int.hrl"). - -type filepath() :: iolist(). -type docid() :: binary(). -type rev() :: {non_neg_integer(), binary()}. @@ -26,75 +24,82 @@ -type purge_seq() :: non_neg_integer(). -type doc_pair() :: { - #full_doc_info{} | not_found, - #full_doc_info{} | not_found - }. + #full_doc_info{} | not_found, + #full_doc_info{} | not_found +}. -type doc_pairs() :: [doc_pair()]. -type db_open_options() :: [ - create - ]. + create +]. -type delete_options() :: [ - {context, delete | compaction} | - sync - ]. + {context, delete | compaction} + | sync +]. -type purge_info() :: {purge_seq(), uuid(), docid(), revs()}. --type epochs() :: [{Node::atom(), UpdateSeq::non_neg_integer()}]. --type size_info() :: [{Name::atom(), Size::non_neg_integer()}]. +-type epochs() :: [{Node :: atom(), UpdateSeq :: non_neg_integer()}]. +-type size_info() :: [{Name :: atom(), Size :: non_neg_integer()}]. -type partition_info() :: [ - {partition, Partition::binary()} | - {doc_count, DocCount::non_neg_integer()} | - {doc_del_count, DocDelCount::non_neg_integer()} | - {sizes, size_info()} + {partition, Partition :: binary()} + | {doc_count, DocCount :: non_neg_integer()} + | {doc_del_count, DocDelCount :: non_neg_integer()} + | {sizes, size_info()} ]. -type write_stream_options() :: [ - {buffer_size, Size::pos_integer()} | - {encoding, atom()} | - {compression_level, non_neg_integer()} - ]. + {buffer_size, Size :: pos_integer()} + | {encoding, atom()} + | {compression_level, non_neg_integer()} +]. -type doc_fold_options() :: [ - {start_key, Key::any()} | - {end_key, Key::any()} | - {end_key_gt, Key::any()} | - {dir, fwd | rev} | - include_reductions | - include_deleted - ]. + {start_key, Key :: any()} + | {end_key, Key :: any()} + | {end_key_gt, Key :: any()} + | {dir, fwd | rev} + | include_reductions + | include_deleted +]. -type changes_fold_options() :: [ - {dir, fwd | rev} - ]. + {dir, fwd | rev} +]. -type purge_fold_options() :: [ - {start_key, Key::any()} | - {end_key, Key::any()} | - {end_key_gt, Key::any()} | - {dir, fwd | rev} - ]. + {start_key, Key :: any()} + | {end_key, Key :: any()} + | {end_key_gt, Key :: any()} + | {dir, fwd | rev} +]. -type db_handle() :: any(). --type doc_fold_fun() :: fun((#full_doc_info{}, UserAcc::any()) -> - {ok, NewUserAcc::any()} | - {stop, NewUserAcc::any()}). - --type local_doc_fold_fun() :: fun((#doc{}, UserAcc::any()) -> - {ok, NewUserAcc::any()} | - {stop, NewUserAcc::any()}). - --type changes_fold_fun() :: fun((#doc_info{}, UserAcc::any()) -> - {ok, NewUserAcc::any()} | - {stop, NewUserAcc::any()}). - --type purge_fold_fun() :: fun((purge_info(), UserAcc::any()) -> - {ok, NewUserAcc::any()} | - {stop, NewUserAcc::any()}). - +-type doc_fold_fun() :: fun( + (#full_doc_info{}, UserAcc :: any()) -> + {ok, NewUserAcc :: any()} + | {stop, NewUserAcc :: any()} +). + +-type local_doc_fold_fun() :: fun( + (#doc{}, UserAcc :: any()) -> + {ok, NewUserAcc :: any()} + | {stop, NewUserAcc :: any()} +). + +-type changes_fold_fun() :: fun( + (#doc_info{}, UserAcc :: any()) -> + {ok, NewUserAcc :: any()} + | {stop, NewUserAcc :: any()} +). + +-type purge_fold_fun() :: fun( + (purge_info(), UserAcc :: any()) -> + {ok, NewUserAcc :: any()} + | {stop, NewUserAcc :: any()} +). % This is called by couch_server to determine which % engine should be used for the given database. DbPath @@ -102,8 +107,7 @@ % extension for a given engine. The first engine to % return true is the engine that will be used for the % database. --callback exists(DbPath::filepath()) -> boolean(). - +-callback exists(DbPath :: filepath()) -> boolean(). % This is called by couch_server to delete a database. It % is called from inside the couch_server process which @@ -112,11 +116,11 @@ % context. Although since this is executed in the context % of couch_server it should return relatively quickly. -callback delete( - RootDir::filepath(), - DbPath::filepath(), - DelOpts::delete_options()) -> - ok | {error, Reason::atom()}. - + RootDir :: filepath(), + DbPath :: filepath(), + DelOpts :: delete_options() +) -> + ok | {error, Reason :: atom()}. % This function can be called from multiple contexts. It % will either be called just before a call to delete/3 above @@ -125,11 +129,11 @@ % remove any temporary files used during compaction that % may be used to recover from a failed compaction swap. -callback delete_compaction_files( - RootDir::filepath(), - DbPath::filepath(), - DelOpts::delete_options()) -> - ok. - + RootDir :: filepath(), + DbPath :: filepath(), + DelOpts :: delete_options() +) -> + ok. % This is called from the couch_db_updater:init/1 context. As % such this means that it is guaranteed to only have one process @@ -145,41 +149,36 @@ % its guaranteed that the handle will only ever be mutated % in a single threaded context (ie, within the couch_db_updater % process). --callback init(DbPath::filepath(), db_open_options()) -> - {ok, DbHandle::db_handle()}. - +-callback init(DbPath :: filepath(), db_open_options()) -> + {ok, DbHandle :: db_handle()}. % This is called in the context of couch_db_updater:terminate/2 % and as such has the same properties for init/2. It's guaranteed % to be consistent for a given database but may be called by many % databases concurrently. --callback terminate(Reason::any(), DbHandle::db_handle()) -> Ignored::any(). - +-callback terminate(Reason :: any(), DbHandle :: db_handle()) -> Ignored :: any(). % This is called in the context of couch_db_updater:handle_call/3 % for any message that is unknown. It can be used to handle messages % from asynchronous processes like the engine's compactor if it has one. --callback handle_db_updater_call(Msg::any(), DbHandle::db_handle()) -> - {reply, Resp::any(), NewDbHandle::db_handle()} | - {stop, Reason::any(), Resp::any(), NewDbHandle::db_handle()}. - +-callback handle_db_updater_call(Msg :: any(), DbHandle :: db_handle()) -> + {reply, Resp :: any(), NewDbHandle :: db_handle()} + | {stop, Reason :: any(), Resp :: any(), NewDbHandle :: db_handle()}. % This is called in the context of couch_db_updater:handle_info/2 % and has the same properties as handle_call/3. --callback handle_db_updater_info(Msg::any(), DbHandle::db_handle()) -> - {noreply, NewDbHandle::db_handle()} | - {noreply, NewDbHandle::db_handle(), Timeout::timeout()} | - {stop, Reason::any(), NewDbHandle::db_handle()}. - +-callback handle_db_updater_info(Msg :: any(), DbHandle :: db_handle()) -> + {noreply, NewDbHandle :: db_handle()} + | {noreply, NewDbHandle :: db_handle(), Timeout :: timeout()} + | {stop, Reason :: any(), NewDbHandle :: db_handle()}. % These functions are called by any process opening or closing % a database. As such they need to be able to handle being % called concurrently. For example, the legacy engine uses these % to add monitors to the main engine process. --callback incref(DbHandle::db_handle()) -> {ok, NewDbHandle::db_handle()}. --callback decref(DbHandle::db_handle()) -> ok. --callback monitored_by(DbHande::db_handle()) -> [pid()]. - +-callback incref(DbHandle :: db_handle()) -> {ok, NewDbHandle :: db_handle()}. +-callback decref(DbHandle :: db_handle()) -> ok. +-callback monitored_by(DbHande :: db_handle()) -> [pid()]. % This is called in the context of couch_db_updater:handle_info/2 % and should return the timestamp of the last activity of @@ -187,8 +186,7 @@ % value would be hard to report its ok to just return the % result of os:timestamp/0 as this will just disable idle % databases from automatically closing. --callback last_activity(DbHandle::db_handle()) -> erlang:timestamp(). - +-callback last_activity(DbHandle :: db_handle()) -> erlang:timestamp(). % All of the get_* functions may be called from many % processes concurrently. @@ -196,25 +194,21 @@ % The database should make a note of the update sequence when it % was last compacted. If the database doesn't need compacting it % can just hard code a return value of 0. --callback get_compacted_seq(DbHandle::db_handle()) -> - CompactedSeq::non_neg_integer(). - +-callback get_compacted_seq(DbHandle :: db_handle()) -> + CompactedSeq :: non_neg_integer(). % The number of documents in the database which have all leaf % revisions marked as deleted. --callback get_del_doc_count(DbHandle::db_handle()) -> - DelDocCount::non_neg_integer(). - +-callback get_del_doc_count(DbHandle :: db_handle()) -> + DelDocCount :: non_neg_integer(). % This number is reported in the database info properties and % as such can be any JSON value. --callback get_disk_version(DbHandle::db_handle()) -> Version::json(). - +-callback get_disk_version(DbHandle :: db_handle()) -> Version :: json(). % The number of documents in the database that have one or more % leaf revisions not marked as deleted. --callback get_doc_count(DbHandle::db_handle()) -> DocCount::non_neg_integer(). - +-callback get_doc_count(DbHandle :: db_handle()) -> DocCount :: non_neg_integer(). % The epochs track which node owned the database starting at % a given update sequence. Each time a database is opened it @@ -222,36 +216,29 @@ % for the current node it should add an entry that will be % written the next time a write is performed. An entry is % simply a {node(), CurrentUpdateSeq} tuple. --callback get_epochs(DbHandle::db_handle()) -> Epochs::epochs(). - +-callback get_epochs(DbHandle :: db_handle()) -> Epochs :: epochs(). % Get the current purge sequence known to the engine. This % value should be updated during calls to purge_docs. --callback get_purge_seq(DbHandle::db_handle()) -> purge_seq(). - +-callback get_purge_seq(DbHandle :: db_handle()) -> purge_seq(). % Get the oldest purge sequence known to the engine --callback get_oldest_purge_seq(DbHandle::db_handle()) -> purge_seq(). - +-callback get_oldest_purge_seq(DbHandle :: db_handle()) -> purge_seq(). % Get the purged infos limit. This should just return the last % value that was passed to set_purged_docs_limit/2. --callback get_purge_infos_limit(DbHandle::db_handle()) -> pos_integer(). - +-callback get_purge_infos_limit(DbHandle :: db_handle()) -> pos_integer(). % Get the revision limit. This should just return the last % value that was passed to set_revs_limit/2. --callback get_revs_limit(DbHandle::db_handle()) -> RevsLimit::pos_integer(). - +-callback get_revs_limit(DbHandle :: db_handle()) -> RevsLimit :: pos_integer(). % Get the current security properties. This should just return % the last value that was passed to set_security/2. --callback get_security(DbHandle::db_handle()) -> SecProps::any(). - +-callback get_security(DbHandle :: db_handle()) -> SecProps :: any(). % Get the current properties. --callback get_props(DbHandle::db_handle()) -> Props::[any()]. - +-callback get_props(DbHandle :: db_handle()) -> Props :: [any()]. % This information is displayed in the database info poperties. It % should just be a list of {Name::atom(), Size::non_neg_integer()} @@ -266,8 +253,7 @@ % external - Number of bytes that would be required to represent the % contents outside of the database (for capacity and backup % planning) --callback get_size_info(DbHandle::db_handle()) -> SizeInfo::size_info(). - +-callback get_size_info(DbHandle :: db_handle()) -> SizeInfo :: size_info(). % This returns the information for the given partition. % It should just be a list of {Name::atom(), Size::non_neg_integer()} @@ -277,56 +263,49 @@ % % external - Number of bytes that would be required to represent the % contents of this partition outside of the database --callback get_partition_info(DbHandle::db_handle(), Partition::binary()) -> +-callback get_partition_info(DbHandle :: db_handle(), Partition :: binary()) -> partition_info(). - % The current update sequence of the database. The update % sequence should be incrememnted for every revision added to % the database. --callback get_update_seq(DbHandle::db_handle()) -> UpdateSeq::non_neg_integer(). - +-callback get_update_seq(DbHandle :: db_handle()) -> UpdateSeq :: non_neg_integer(). % Whenever a database is created it should generate a % persistent UUID for identification in case the shard should % ever need to be moved between nodes in a cluster. --callback get_uuid(DbHandle::db_handle()) -> UUID::binary(). - +-callback get_uuid(DbHandle :: db_handle()) -> UUID :: binary(). % These functions are only called by couch_db_updater and % as such are guaranteed to be single threaded calls. The % database should simply store these values somewhere so % they can be returned by the corresponding get_* calls. --callback set_revs_limit(DbHandle::db_handle(), RevsLimit::pos_integer()) -> - {ok, NewDbHandle::db_handle()}. - +-callback set_revs_limit(DbHandle :: db_handle(), RevsLimit :: pos_integer()) -> + {ok, NewDbHandle :: db_handle()}. --callback set_purge_infos_limit(DbHandle::db_handle(), Limit::pos_integer()) -> - {ok, NewDbHandle::db_handle()}. - - --callback set_security(DbHandle::db_handle(), SecProps::any()) -> - {ok, NewDbHandle::db_handle()}. +-callback set_purge_infos_limit(DbHandle :: db_handle(), Limit :: pos_integer()) -> + {ok, NewDbHandle :: db_handle()}. +-callback set_security(DbHandle :: db_handle(), SecProps :: any()) -> + {ok, NewDbHandle :: db_handle()}. % This function is only called by couch_db_updater and % as such is guaranteed to be single threaded calls. The % database should simply store provided property list % unaltered. --callback set_props(DbHandle::db_handle(), Props::any()) -> - {ok, NewDbHandle::db_handle()}. - +-callback set_props(DbHandle :: db_handle(), Props :: any()) -> + {ok, NewDbHandle :: db_handle()}. % Set the current update sequence of the database. The intention is to use this % when copying a database such that the destination update sequence should % match exactly the source update sequence. -callback set_update_seq( - DbHandle::db_handle(), - UpdateSeq::non_neg_integer()) -> - {ok, NewDbHandle::db_handle()}. - + DbHandle :: db_handle(), + UpdateSeq :: non_neg_integer() +) -> + {ok, NewDbHandle :: db_handle()}. % This function will be called by many processes concurrently. % It should return a #full_doc_info{} record or not_found for @@ -337,9 +316,8 @@ % were present in the database when the DbHandle was retrieved % from couch_server. It is currently unknown what would break % if a storage engine deviated from that property. --callback open_docs(DbHandle::db_handle(), DocIds::[docid()]) -> - [#full_doc_info{} | not_found]. - +-callback open_docs(DbHandle :: db_handle(), DocIds :: [docid()]) -> + [#full_doc_info{} | not_found]. % This function will be called by many processes concurrently. % It should return a #doc{} record or not_found for every @@ -349,9 +327,8 @@ % apply to this function (although this function is called % rather less frequently so it may not be as big of an % issue). --callback open_local_docs(DbHandle::db_handle(), DocIds::[docid()]) -> - [#doc{} | not_found]. - +-callback open_local_docs(DbHandle :: db_handle(), DocIds :: [docid()]) -> + [#doc{} | not_found]. % This function will be called from many contexts concurrently. % The provided RawDoc is a #doc{} record that has its body @@ -360,18 +337,16 @@ % This API exists so that storage engines can store document % bodies externally from the #full_doc_info{} record (which % is the traditional approach and is recommended). --callback read_doc_body(DbHandle::db_handle(), RawDoc::doc()) -> - doc(). - +-callback read_doc_body(DbHandle :: db_handle(), RawDoc :: doc()) -> + doc(). % This function will be called from many contexts concurrently. % If the storage engine has a purge_info() record for any of the % provided UUIDs, those purge_info() records should be returned. The % resulting list should have the same length as the input list of % UUIDs. --callback load_purge_infos(DbHandle::db_handle(), [uuid()]) -> - [purge_info() | not_found]. - +-callback load_purge_infos(DbHandle :: db_handle(), [uuid()]) -> + [purge_info() | not_found]. % This function is called concurrently by any client process % that is writing a document. It should accept a #doc{} @@ -382,9 +357,8 @@ % document bodies in parallel by client processes rather % than forcing all compression to occur single threaded % in the context of the couch_db_updater process. --callback serialize_doc(DbHandle::db_handle(), Doc::doc()) -> - doc(). - +-callback serialize_doc(DbHandle :: db_handle(), Doc :: doc()) -> + doc(). % This function is called in the context of a couch_db_updater % which means its single threaded for the given DbHandle. @@ -397,9 +371,8 @@ % The BytesWritten return value is used to determine the number % of active bytes in the database which can is used to make % a determination of when to compact this database. --callback write_doc_body(DbHandle::db_handle(), Doc::doc()) -> - {ok, FlushedDoc::doc(), BytesWritten::non_neg_integer()}. - +-callback write_doc_body(DbHandle :: db_handle(), Doc :: doc()) -> + {ok, FlushedDoc :: doc(), BytesWritten :: non_neg_integer()}. % This function is called from the context of couch_db_updater % and as such is guaranteed single threaded for the given @@ -435,11 +408,11 @@ % batches are non-deterministic (from the point of view of the % client). -callback write_doc_infos( - DbHandle::db_handle(), - Pairs::doc_pairs(), - LocalDocs::[#doc{}]) -> - {ok, NewDbHandle::db_handle()}. - + DbHandle :: db_handle(), + Pairs :: doc_pairs(), + LocalDocs :: [#doc{}] +) -> + {ok, NewDbHandle :: db_handle()}. % This function is called from the context of couch_db_updater % and as such is guaranteed single threaded for the given @@ -470,25 +443,22 @@ % revisions that were requested to be purged. This should be persisted % in such a way that we can efficiently load purge_info() by its UUID % as well as iterate over purge_info() entries in order of their PurgeSeq. --callback purge_docs(DbHandle::db_handle(), [doc_pair()], [purge_info()]) -> - {ok, NewDbHandle::db_handle()}. - +-callback purge_docs(DbHandle :: db_handle(), [doc_pair()], [purge_info()]) -> + {ok, NewDbHandle :: db_handle()}. % This function should be called from a single threaded context and % should be used to copy purge infos from on database to another % when copying a database --callback copy_purge_infos(DbHandle::db_handle(), [purge_info()]) -> - {ok, NewDbHandle::db_handle()}. - +-callback copy_purge_infos(DbHandle :: db_handle(), [purge_info()]) -> + {ok, NewDbHandle :: db_handle()}. % This function is called in the context of couch_db_udpater and % as such is single threaded for any given DbHandle. % % This call is made periodically to ensure that the database has % stored all updates on stable storage. (ie, here is where you fsync). --callback commit_data(DbHandle::db_handle()) -> - {ok, NewDbHande::db_handle()}. - +-callback commit_data(DbHandle :: db_handle()) -> + {ok, NewDbHande :: db_handle()}. % This function is called by multiple processes concurrently. % @@ -502,20 +472,18 @@ % Currently an engine can elect to not implement these API's % by throwing the atom not_supported. -callback open_write_stream( - DbHandle::db_handle(), - Options::write_stream_options()) -> - {ok, pid()}. - + DbHandle :: db_handle(), + Options :: write_stream_options() +) -> + {ok, pid()}. % See the documentation for open_write_stream --callback open_read_stream(DbHandle::db_handle(), StreamDiskInfo::any()) -> - {ok, {Module::atom(), ReadStreamState::any()}}. - +-callback open_read_stream(DbHandle :: db_handle(), StreamDiskInfo :: any()) -> + {ok, {Module :: atom(), ReadStreamState :: any()}}. % See the documentation for open_write_stream --callback is_active_stream(DbHandle::db_handle(), ReadStreamState::any()) -> - boolean(). - +-callback is_active_stream(DbHandle :: db_handle(), ReadStreamState :: any()) -> + boolean(). % This funciton is called by many processes concurrently. % @@ -567,12 +535,12 @@ % that actually happening so a storage engine that includes new results % between invocations shouldn't have any issues. -callback fold_docs( - DbHandle::db_handle(), - UserFold::doc_fold_fun(), - UserAcc::any(), - doc_fold_options()) -> - {ok, LastUserAcc::any()}. - + DbHandle :: db_handle(), + UserFold :: doc_fold_fun(), + UserAcc :: any(), + doc_fold_options() +) -> + {ok, LastUserAcc :: any()}. % This function may be called by many processes concurrently. % @@ -580,12 +548,12 @@ % should only return local documents and the first argument to the % user function is a #doc{} record, not a #full_doc_info{}. -callback fold_local_docs( - DbHandle::db_handle(), - UserFold::local_doc_fold_fun(), - UserAcc::any(), - doc_fold_options()) -> - {ok, LastUserAcc::any()}. - + DbHandle :: db_handle(), + UserFold :: local_doc_fold_fun(), + UserAcc :: any(), + doc_fold_options() +) -> + {ok, LastUserAcc :: any()}. % This function may be called by many processes concurrently. % @@ -608,13 +576,13 @@ % The only option currently supported by the API is the `dir` % option that should behave the same as for fold_docs. -callback fold_changes( - DbHandle::db_handle(), - StartSeq::non_neg_integer(), - UserFold::changes_fold_fun(), - UserAcc::any(), - changes_fold_options()) -> - {ok, LastUserAcc::any()}. - + DbHandle :: db_handle(), + StartSeq :: non_neg_integer(), + UserFold :: changes_fold_fun(), + UserAcc :: any(), + changes_fold_options() +) -> + {ok, LastUserAcc :: any()}. % This function may be called by many processes concurrently. % @@ -623,13 +591,13 @@ % % The StartPurgeSeq parameter indicates where the fold should start *after*. -callback fold_purge_infos( - DbHandle::db_handle(), - StartPurgeSeq::purge_seq(), - UserFold::purge_fold_fun(), - UserAcc::any(), - purge_fold_options()) -> - {ok, LastUserAcc::any()}. - + DbHandle :: db_handle(), + StartPurgeSeq :: purge_seq(), + UserFold :: purge_fold_fun(), + UserAcc :: any(), + purge_fold_options() +) -> + {ok, LastUserAcc :: any()}. % This function may be called by many processes concurrently. % @@ -647,10 +615,10 @@ % _active_tasks entry if the storage engine isn't accounted for by the % client. -callback count_changes_since( - DbHandle::db_handle(), - UpdateSeq::non_neg_integer()) -> - TotalChanges::non_neg_integer(). - + DbHandle :: db_handle(), + UpdateSeq :: non_neg_integer() +) -> + TotalChanges :: non_neg_integer(). % This function is called in the context of couch_db_updater and as % such is guaranteed to be single threaded for the given DbHandle. @@ -666,12 +634,12 @@ % must be the same engine that started the compaction and CompactInfo % is an arbitrary term that's passed to finish_compaction/4. -callback start_compaction( - DbHandle::db_handle(), - DbName::binary(), - Options::db_open_options(), - Parent::pid()) -> - {ok, NewDbHandle::db_handle(), CompactorPid::pid()}. - + DbHandle :: db_handle(), + DbName :: binary(), + Options :: db_open_options(), + Parent :: pid() +) -> + {ok, NewDbHandle :: db_handle(), CompactorPid :: pid()}. % This function is called in the context of couch_db_udpater and as % such is guarnateed to be single threaded for the given DbHandle. @@ -683,12 +651,12 @@ % to update the DbHandle state of the couch_db_updater it can as % finish_compaction/4 is called in the context of the couch_db_updater. -callback finish_compaction( - OldDbHandle::db_handle(), - DbName::binary(), - Options::db_open_options(), - CompactInfo::any()) -> - {ok, CompactedDbHandle::db_handle(), CompactorPid::pid() | undefined}. - + OldDbHandle :: db_handle(), + DbName :: binary(), + Options :: db_open_options(), + CompactInfo :: any() +) -> + {ok, CompactedDbHandle :: db_handle(), CompactorPid :: pid() | undefined}. -export([ exists/2, @@ -757,34 +725,29 @@ trigger_on_compact/1 ]). - exists(Engine, DbPath) -> Engine:exists(DbPath). - delete(Engine, RootDir, DbPath, DelOpts) when is_list(DelOpts) -> Engine:delete(RootDir, DbPath, DelOpts). - -delete_compaction_files(Engine, RootDir, DbPath, DelOpts) - when is_list(DelOpts) -> +delete_compaction_files(Engine, RootDir, DbPath, DelOpts) when + is_list(DelOpts) +-> Engine:delete_compaction_files(RootDir, DbPath, DelOpts). - init(Engine, DbPath, Options) -> case Engine:init(DbPath, Options) of - {ok, EngineState} -> - {ok, {Engine, EngineState}}; - Error -> - throw(Error) + {ok, EngineState} -> + {ok, {Engine, EngineState}}; + Error -> + throw(Error) end. - terminate(Reason, #db{} = Db) -> #db{engine = {Engine, EngineState}} = Db, Engine:terminate(Reason, EngineState). - handle_db_updater_call(Msg, _From, #db{} = Db) -> #db{ engine = {Engine, EngineState} @@ -796,7 +759,6 @@ handle_db_updater_call(Msg, _From, #db{} = Db) -> {stop, Reason, Resp, Db#db{engine = {Engine, NewState}}} end. - handle_db_updater_info(Msg, #db{} = Db) -> #db{ name = Name, @@ -812,98 +774,79 @@ handle_db_updater_info(Msg, #db{} = Db) -> {stop, Reason, Db#db{engine = {Engine, NewState}}} end. - incref(#db{} = Db) -> #db{engine = {Engine, EngineState}} = Db, {ok, NewState} = Engine:incref(EngineState), {ok, Db#db{engine = {Engine, NewState}}}. - decref(#db{} = Db) -> #db{engine = {Engine, EngineState}} = Db, Engine:decref(EngineState). - monitored_by(#db{} = Db) -> #db{engine = {Engine, EngineState}} = Db, Engine:monitored_by(EngineState). - last_activity(#db{} = Db) -> #db{engine = {Engine, EngineState}} = Db, Engine:last_activity(EngineState). - get_engine(#db{} = Db) -> #db{engine = {Engine, _}} = Db, Engine. - get_compacted_seq(#db{} = Db) -> #db{engine = {Engine, EngineState}} = Db, Engine:get_compacted_seq(EngineState). - get_del_doc_count(#db{} = Db) -> #db{engine = {Engine, EngineState}} = Db, Engine:get_del_doc_count(EngineState). - get_disk_version(#db{} = Db) -> #db{engine = {Engine, EngineState}} = Db, Engine:get_disk_version(EngineState). - get_doc_count(#db{} = Db) -> #db{engine = {Engine, EngineState}} = Db, Engine:get_doc_count(EngineState). - get_epochs(#db{} = Db) -> #db{engine = {Engine, EngineState}} = Db, Engine:get_epochs(EngineState). - get_purge_seq(#db{} = Db) -> #db{engine = {Engine, EngineState}} = Db, Engine:get_purge_seq(EngineState). - get_oldest_purge_seq(#db{} = Db) -> #db{engine = {Engine, EngineState}} = Db, Engine:get_oldest_purge_seq(EngineState). - get_purge_infos_limit(#db{} = Db) -> #db{engine = {Engine, EngineState}} = Db, Engine:get_purge_infos_limit(EngineState). - get_revs_limit(#db{} = Db) -> #db{engine = {Engine, EngineState}} = Db, Engine:get_revs_limit(EngineState). - get_security(#db{} = Db) -> #db{engine = {Engine, EngineState}} = Db, Engine:get_security(EngineState). - get_props(#db{} = Db) -> #db{engine = {Engine, EngineState}} = Db, Engine:get_props(EngineState). - get_size_info(#db{} = Db) -> #db{engine = {Engine, EngineState}} = Db, Engine:get_size_info(EngineState). - get_partition_info(#db{} = Db, Partition) -> #db{engine = {Engine, EngineState}} = Db, Engine:get_partition_info(EngineState, Partition). - get_update_seq(#db{} = Db) -> #db{engine = {Engine, EngineState}} = Db, Engine:get_update_seq(EngineState). @@ -912,134 +855,113 @@ get_uuid(#db{} = Db) -> #db{engine = {Engine, EngineState}} = Db, Engine:get_uuid(EngineState). - set_revs_limit(#db{} = Db, RevsLimit) -> #db{engine = {Engine, EngineState}} = Db, {ok, NewSt} = Engine:set_revs_limit(EngineState, RevsLimit), {ok, Db#db{engine = {Engine, NewSt}}}. - set_purge_infos_limit(#db{} = Db, PurgedDocsLimit) -> #db{engine = {Engine, EngineState}} = Db, {ok, NewSt} = Engine:set_purge_infos_limit(EngineState, PurgedDocsLimit), {ok, Db#db{engine = {Engine, NewSt}}}. - set_security(#db{} = Db, SecProps) -> #db{engine = {Engine, EngineState}} = Db, {ok, NewSt} = Engine:set_security(EngineState, SecProps), {ok, Db#db{engine = {Engine, NewSt}}}. - set_props(#db{} = Db, Props) -> #db{engine = {Engine, EngineState}} = Db, {ok, NewSt} = Engine:set_props(EngineState, Props), {ok, Db#db{engine = {Engine, NewSt}}}. - set_update_seq(#db{} = Db, UpdateSeq) -> #db{engine = {Engine, EngineState}} = Db, {ok, NewSt} = Engine:set_update_seq(EngineState, UpdateSeq), {ok, Db#db{engine = {Engine, NewSt}}}. - open_docs(#db{} = Db, DocIds) -> #db{engine = {Engine, EngineState}} = Db, Engine:open_docs(EngineState, DocIds). - open_local_docs(#db{} = Db, DocIds) -> #db{engine = {Engine, EngineState}} = Db, Engine:open_local_docs(EngineState, DocIds). - read_doc_body(#db{} = Db, RawDoc) -> #db{engine = {Engine, EngineState}} = Db, Engine:read_doc_body(EngineState, RawDoc). - load_purge_infos(#db{} = Db, UUIDs) -> #db{engine = {Engine, EngineState}} = Db, Engine:load_purge_infos(EngineState, UUIDs). - serialize_doc(#db{} = Db, #doc{} = Doc) -> #db{engine = {Engine, EngineState}} = Db, Engine:serialize_doc(EngineState, Doc). - write_doc_body(#db{} = Db, #doc{} = Doc) -> #db{engine = {Engine, EngineState}} = Db, Engine:write_doc_body(EngineState, Doc). - write_doc_infos(#db{} = Db, DocUpdates, LocalDocs) -> #db{engine = {Engine, EngineState}} = Db, {ok, NewSt} = Engine:write_doc_infos(EngineState, DocUpdates, LocalDocs), {ok, Db#db{engine = {Engine, NewSt}}}. - purge_docs(#db{} = Db, DocUpdates, Purges) -> #db{engine = {Engine, EngineState}} = Db, {ok, NewSt} = Engine:purge_docs( - EngineState, DocUpdates, Purges), + EngineState, DocUpdates, Purges + ), {ok, Db#db{engine = {Engine, NewSt}}}. - copy_purge_infos(#db{} = Db, Purges) -> #db{engine = {Engine, EngineState}} = Db, {ok, NewSt} = Engine:copy_purge_infos( - EngineState, Purges), + EngineState, Purges + ), {ok, Db#db{engine = {Engine, NewSt}}}. - commit_data(#db{} = Db) -> #db{engine = {Engine, EngineState}} = Db, {ok, NewSt} = Engine:commit_data(EngineState), {ok, Db#db{engine = {Engine, NewSt}}}. - open_write_stream(#db{} = Db, Options) -> #db{engine = {Engine, EngineState}} = Db, Engine:open_write_stream(EngineState, Options). - open_read_stream(#db{} = Db, StreamDiskInfo) -> #db{engine = {Engine, EngineState}} = Db, Engine:open_read_stream(EngineState, StreamDiskInfo). - is_active_stream(#db{} = Db, ReadStreamState) -> #db{engine = {Engine, EngineState}} = Db, Engine:is_active_stream(EngineState, ReadStreamState). - fold_docs(#db{} = Db, UserFun, UserAcc, Options) -> #db{engine = {Engine, EngineState}} = Db, Engine:fold_docs(EngineState, UserFun, UserAcc, Options). - fold_local_docs(#db{} = Db, UserFun, UserAcc, Options) -> #db{engine = {Engine, EngineState}} = Db, Engine:fold_local_docs(EngineState, UserFun, UserAcc, Options). - fold_changes(#db{} = Db, StartSeq, UserFun, UserAcc, Options) -> #db{engine = {Engine, EngineState}} = Db, Engine:fold_changes(EngineState, StartSeq, UserFun, UserAcc, Options). - fold_purge_infos(#db{} = Db, StartPurgeSeq, UserFun, UserAcc, Options) -> #db{engine = {Engine, EngineState}} = Db, Engine:fold_purge_infos( - EngineState, StartPurgeSeq, UserFun, UserAcc, Options). - + EngineState, StartPurgeSeq, UserFun, UserAcc, Options + ). count_changes_since(#db{} = Db, StartSeq) -> #db{engine = {Engine, EngineState}} = Db, Engine:count_changes_since(EngineState, StartSeq). - start_compaction(#db{} = Db) -> #db{ engine = {Engine, EngineState}, @@ -1047,50 +969,53 @@ start_compaction(#db{} = Db) -> options = Options } = Db, {ok, NewEngineState, Pid} = Engine:start_compaction( - EngineState, DbName, Options, self()), + EngineState, DbName, Options, self() + ), {ok, Db#db{ engine = {Engine, NewEngineState}, compactor_pid = Pid }}. - finish_compaction(Db, CompactInfo) -> #db{ engine = {Engine, St}, name = DbName, options = Options } = Db, - NewDb = case Engine:finish_compaction(St, DbName, Options, CompactInfo) of - {ok, NewState, undefined} -> - couch_event:notify(DbName, compacted), - Db#db{ - engine = {Engine, NewState}, - compactor_pid = nil - }; - {ok, NewState, CompactorPid} when is_pid(CompactorPid) -> - Db#db{ - engine = {Engine, NewState}, - compactor_pid = CompactorPid - } - end, + NewDb = + case Engine:finish_compaction(St, DbName, Options, CompactInfo) of + {ok, NewState, undefined} -> + couch_event:notify(DbName, compacted), + Db#db{ + engine = {Engine, NewState}, + compactor_pid = nil + }; + {ok, NewState, CompactorPid} when is_pid(CompactorPid) -> + Db#db{ + engine = {Engine, NewState}, + compactor_pid = CompactorPid + } + end, ok = couch_server:db_updated(NewDb), {ok, NewDb}. - trigger_on_compact(DbName) -> {ok, DDocs} = get_ddocs(DbName), couch_db_plugin:on_compact(DbName, DDocs). - get_ddocs(<<"shards/", _/binary>> = DbName) -> {_, Ref} = spawn_monitor(fun() -> exit(fabric:design_docs(mem3:dbname(DbName))) end), receive {'DOWN', Ref, _, _, {ok, JsonDDocs}} -> - {ok, lists:map(fun(JsonDDoc) -> - couch_doc:from_json_obj(JsonDDoc) - end, JsonDDocs)}; + {ok, + lists:map( + fun(JsonDDoc) -> + couch_doc:from_json_obj(JsonDDoc) + end, + JsonDDocs + )}; {'DOWN', Ref, _, _, Else} -> Else end; diff --git a/src/couch/src/couch_db_epi.erl b/src/couch/src/couch_db_epi.erl index 21879f683..870202bad 100644 --- a/src/couch/src/couch_db_epi.erl +++ b/src/couch/src/couch_db_epi.erl @@ -32,7 +32,6 @@ providers() -> {chttpd_handlers, couch_httpd_handlers} ]. - services() -> [ {couch_db, couch_db_plugin}, diff --git a/src/couch/src/couch_db_header.erl b/src/couch/src/couch_db_header.erl index 355364f9b..9c81ba6d0 100644 --- a/src/couch/src/couch_db_header.erl +++ b/src/couch/src/couch_db_header.erl @@ -12,7 +12,6 @@ -module(couch_db_header). - -export([ new/0, from/1, @@ -37,7 +36,6 @@ compacted_seq/1 ]). - % This should be updated anytime a header change happens that requires more % than filling in new defaults. % @@ -66,14 +64,12 @@ compacted_seq }). - new() -> #db_header{ uuid = couch_uuids:random(), epochs = [{node(), 0}] }. - from(Header0) -> Header = upgrade(Header0), #db_header{ @@ -82,16 +78,15 @@ from(Header0) -> compacted_seq = Header#db_header.compacted_seq }. - is_header(Header) -> try upgrade(Header), true - catch _:_ -> - false + catch + _:_ -> + false end. - upgrade(Header) -> Funs = [ fun upgrade_tuple/1, @@ -100,69 +95,63 @@ upgrade(Header) -> fun upgrade_epochs/1, fun upgrade_compacted_seq/1 ], - lists:foldl(fun(F, HdrAcc) -> - F(HdrAcc) - end, Header, Funs). - + lists:foldl( + fun(F, HdrAcc) -> + F(HdrAcc) + end, + Header, + Funs + ). set(Header0, Fields) -> % A subtlety here is that if a database was open during % the release upgrade that updates to uuids and epochs then % this dynamic upgrade also assigns a uuid and epoch. Header = upgrade(Header0), - lists:foldl(fun({Field, Value}, HdrAcc) -> - set_field(HdrAcc, Field, Value) - end, Header, Fields). - + lists:foldl( + fun({Field, Value}, HdrAcc) -> + set_field(HdrAcc, Field, Value) + end, + Header, + Fields + ). disk_version(Header) -> get_field(Header, disk_version). - update_seq(Header) -> get_field(Header, update_seq). - id_tree_state(Header) -> get_field(Header, id_tree_state). - seq_tree_state(Header) -> get_field(Header, seq_tree_state). - local_tree_state(Header) -> get_field(Header, local_tree_state). - purge_seq(Header) -> get_field(Header, purge_seq). - purged_docs(Header) -> get_field(Header, purged_docs). - security_ptr(Header) -> get_field(Header, security_ptr). - revs_limit(Header) -> get_field(Header, revs_limit). - uuid(Header) -> get_field(Header, uuid). - epochs(Header) -> get_field(Header, epochs). - compacted_seq(Header) -> get_field(Header, compacted_seq). - get_field(Header, Field) -> Idx = index(Field), case Idx > tuple_size(Header) of @@ -170,88 +159,97 @@ get_field(Header, Field) -> false -> element(index(Field), Header) end. - set_field(Header, Field, Value) -> setelement(index(Field), Header, Value). - index(Field) -> couch_util:get_value(Field, indexes()). - indexes() -> Fields = record_info(fields, db_header), Indexes = lists:seq(2, record_info(size, db_header)), lists:zip(Fields, Indexes). - upgrade_tuple(Old) when is_record(Old, db_header) -> Old; upgrade_tuple(Old) when is_tuple(Old) -> NewSize = record_info(size, db_header), - if tuple_size(Old) < NewSize -> ok; true -> - erlang:error({invalid_header_size, Old}) + if + tuple_size(Old) < NewSize -> ok; + true -> erlang:error({invalid_header_size, Old}) end, - {_, New} = lists:foldl(fun(Val, {Idx, Hdr}) -> - {Idx+1, setelement(Idx, Hdr, Val)} - end, {1, #db_header{}}, tuple_to_list(Old)), - if is_record(New, db_header) -> ok; true -> - erlang:error({invalid_header_extension, {Old, New}}) + {_, New} = lists:foldl( + fun(Val, {Idx, Hdr}) -> + {Idx + 1, setelement(Idx, Hdr, Val)} + end, + {1, #db_header{}}, + tuple_to_list(Old) + ), + if + is_record(New, db_header) -> ok; + true -> erlang:error({invalid_header_extension, {Old, New}}) end, New. -define(OLD_DISK_VERSION_ERROR, - "Database files from versions smaller than 0.10.0 are no longer supported"). + "Database files from versions smaller than 0.10.0 are no longer supported" +). -upgrade_disk_version(#db_header{}=Header) -> +upgrade_disk_version(#db_header{} = Header) -> case element(2, Header) of - 1 -> throw({database_disk_version_error, ?OLD_DISK_VERSION_ERROR}); - 2 -> throw({database_disk_version_error, ?OLD_DISK_VERSION_ERROR}); - 3 -> throw({database_disk_version_error, ?OLD_DISK_VERSION_ERROR}); - 4 -> Header#db_header{security_ptr = nil}; % [0.10 - 0.11) - 5 -> Header; % pre 1.2 - ?LATEST_DISK_VERSION -> Header; + 1 -> + throw({database_disk_version_error, ?OLD_DISK_VERSION_ERROR}); + 2 -> + throw({database_disk_version_error, ?OLD_DISK_VERSION_ERROR}); + 3 -> + throw({database_disk_version_error, ?OLD_DISK_VERSION_ERROR}); + % [0.10 - 0.11) + 4 -> + Header#db_header{security_ptr = nil}; + % pre 1.2 + 5 -> + Header; + ?LATEST_DISK_VERSION -> + Header; _ -> Reason = "Incorrect disk header version", throw({database_disk_version_error, Reason}) end. - -upgrade_uuid(#db_header{}=Header) -> +upgrade_uuid(#db_header{} = Header) -> case Header#db_header.uuid of undefined -> % Upgrading this old db file to a newer % on disk format that includes a UUID. - Header#db_header{uuid=couch_uuids:random()}; + Header#db_header{uuid = couch_uuids:random()}; _ -> Header end. - -upgrade_epochs(#db_header{}=Header) -> - NewEpochs = case Header#db_header.epochs of - undefined -> - % This node is taking over ownership of shard with - % and old version of couch file. Before epochs there - % was always an implicit assumption that a file was - % owned since eternity by the node it was on. This - % just codifies that assumption. - [{node(), 0}]; - [{Node, _} | _] = Epochs0 when Node == node() -> - % Current node is the current owner of this db - Epochs0; - Epochs1 -> - % This node is taking over ownership of this db - % and marking the update sequence where it happened. - [{node(), Header#db_header.update_seq} | Epochs1] - end, +upgrade_epochs(#db_header{} = Header) -> + NewEpochs = + case Header#db_header.epochs of + undefined -> + % This node is taking over ownership of shard with + % and old version of couch file. Before epochs there + % was always an implicit assumption that a file was + % owned since eternity by the node it was on. This + % just codifies that assumption. + [{node(), 0}]; + [{Node, _} | _] = Epochs0 when Node == node() -> + % Current node is the current owner of this db + Epochs0; + Epochs1 -> + % This node is taking over ownership of this db + % and marking the update sequence where it happened. + [{node(), Header#db_header.update_seq} | Epochs1] + end, % Its possible for a node to open a db and claim % ownership but never make a write to the db. This % removes nodes that claimed ownership but never % changed the database. DedupedEpochs = remove_dup_epochs(NewEpochs), - Header#db_header{epochs=DedupedEpochs}. - + Header#db_header{epochs = DedupedEpochs}. % This is slightly relying on the udpate_seq's being sorted % in epochs due to how we only ever push things onto the @@ -260,12 +258,12 @@ upgrade_epochs(#db_header{}=Header) -> % want to remove dupes (by calling a sort on the input to this % function). So for now we don't sort but are relying on the % idea that epochs is always sorted. -remove_dup_epochs([_]=Epochs) -> +remove_dup_epochs([_] = Epochs) -> Epochs; remove_dup_epochs([{N1, S}, {_N2, S}]) -> % Seqs match, keep the most recent owner [{N1, S}]; -remove_dup_epochs([_, _]=Epochs) -> +remove_dup_epochs([_, _] = Epochs) -> % Seqs don't match. Epochs; remove_dup_epochs([{N1, S}, {_N2, S} | Rest]) -> @@ -275,11 +273,10 @@ remove_dup_epochs([{N1, S1}, {N2, S2} | Rest]) -> % Seqs don't match, recurse to check others [{N1, S1} | remove_dup_epochs([{N2, S2} | Rest])]. - -upgrade_compacted_seq(#db_header{}=Header) -> +upgrade_compacted_seq(#db_header{} = Header) -> case Header#db_header.compacted_seq of undefined -> - Header#db_header{compacted_seq=0}; + Header#db_header{compacted_seq = 0}; _ -> Header end. @@ -296,20 +293,30 @@ latest(_Else) -> mk_header(Vsn) -> { - db_header, % record name - Vsn, % disk version - 100, % update_seq - 0, % unused - foo, % id_tree_state - bar, % seq_tree_state - bam, % local_tree_state - 1, % purge_seq - baz, % purged_docs - bang, % security_ptr - 999 % revs_limit + % record name + db_header, + % disk version + Vsn, + % update_seq + 100, + % unused + 0, + % id_tree_state + foo, + % seq_tree_state + bar, + % local_tree_state + bam, + % purge_seq + 1, + % purged_docs + baz, + % security_ptr + bang, + % revs_limit + 999 }. - upgrade_v3_test() -> Vsn3Header = mk_header(3), NewHeader = upgrade_tuple(Vsn3Header), @@ -328,9 +335,10 @@ upgrade_v3_test() -> ?assertEqual(undefined, uuid(NewHeader)), ?assertEqual(undefined, epochs(NewHeader)), - ?assertThrow({database_disk_version_error, _}, - upgrade_disk_version(NewHeader)). - + ?assertThrow( + {database_disk_version_error, _}, + upgrade_disk_version(NewHeader) + ). upgrade_v5_test() -> Vsn5Header = mk_header(5), @@ -342,7 +350,6 @@ upgrade_v5_test() -> % Security ptr isn't changed for v5 headers ?assertEqual(bang, security_ptr(NewHeader)). - upgrade_uuid_test() -> Vsn5Header = mk_header(5), @@ -358,7 +365,6 @@ upgrade_uuid_test() -> ResetHeader = from(NewNewHeader), ?assertEqual(uuid(NewHeader), uuid(ResetHeader)). - upgrade_epochs_test() -> Vsn5Header = mk_header(5), @@ -391,15 +397,12 @@ upgrade_epochs_test() -> ResetHeader = from(NewNewHeader), ?assertEqual(OwnedEpochs, epochs(ResetHeader)). - get_uuid_from_old_header_test() -> Vsn5Header = mk_header(5), ?assertEqual(undefined, uuid(Vsn5Header)). - get_epochs_from_old_header_test() -> Vsn5Header = mk_header(5), ?assertEqual(undefined, epochs(Vsn5Header)). - -endif. diff --git a/src/couch/src/couch_db_plugin.erl b/src/couch/src/couch_db_plugin.erl index c3684c6e3..c84edc1b7 100644 --- a/src/couch/src/couch_db_plugin.erl +++ b/src/couch/src/couch_db_plugin.erl @@ -87,10 +87,10 @@ do_apply(Func, Args, Opts) -> maybe_handle(Func, Args, Default) -> Handle = couch_epi:get_handle(?SERVICE_ID), case couch_epi:decide(Handle, ?SERVICE_ID, Func, Args, []) of - no_decision when is_function(Default) -> - apply(Default, Args); - no_decision -> - Default; - {decided, Result} -> - Result + no_decision when is_function(Default) -> + apply(Default, Args); + no_decision -> + Default; + {decided, Result} -> + Result end. diff --git a/src/couch/src/couch_db_split.erl b/src/couch/src/couch_db_split.erl index 1aa86fb37..d219e3731 100644 --- a/src/couch/src/couch_db_split.erl +++ b/src/couch/src/couch_db_split.erl @@ -12,20 +12,16 @@ -module(couch_db_split). - -export([ split/3, copy_local_docs/3, cleanup_target/2 ]). - -include_lib("couch/include/couch_db.hrl"). - -define(DEFAULT_BUFFER_SIZE, 16777216). - -record(state, { source_db, source_uuid, @@ -51,11 +47,11 @@ atts = [] }). - % Public API split(Source, #{} = Targets, PickFun) when - map_size(Targets) >= 2, is_function(PickFun, 3) -> + map_size(Targets) >= 2, is_function(PickFun, 3) +-> case couch_db:open_int(Source, [?ADMIN_CTX]) of {ok, SourceDb} -> Engine = get_engine(SourceDb), @@ -74,16 +70,19 @@ split(Source, #{} = Targets, PickFun) when {error, missing_source} end. - copy_local_docs(Source, #{} = Targets0, PickFun) when - is_binary(Source), is_function(PickFun, 3) -> + is_binary(Source), is_function(PickFun, 3) +-> case couch_db:open_int(Source, [?ADMIN_CTX]) of {ok, SourceDb} -> try - Targets = maps:map(fun(_, DbName) -> - {ok, Db} = couch_db:open_int(DbName, [?ADMIN_CTX]), - #target{db = Db, uuid = couch_db:get_uuid(Db)} - end, Targets0), + Targets = maps:map( + fun(_, DbName) -> + {ok, Db} = couch_db:open_int(DbName, [?ADMIN_CTX]), + #target{db = Db, uuid = couch_db:get_uuid(Db)} + end, + Targets0 + ), SourceName = couch_db:name(SourceDb), try State = #state{ @@ -96,10 +95,13 @@ copy_local_docs(Source, #{} = Targets0, PickFun) when copy_local_docs(State), ok after - maps:map(fun(_, #target{db = Db} = T) -> - couch_db:close(Db), - T#target{db = undefined} - end, Targets) + maps:map( + fun(_, #target{db = Db} = T) -> + couch_db:close(Db), + T#target{db = undefined} + end, + Targets + ) end after couch_db:close(SourceDb) @@ -108,7 +110,6 @@ copy_local_docs(Source, #{} = Targets0, PickFun) when {error, missing_source} end. - cleanup_target(Source, Target) when is_binary(Source), is_binary(Target) -> case couch_db:open_int(Source, [?ADMIN_CTX]) of {ok, SourceDb} -> @@ -121,35 +122,40 @@ cleanup_target(Source, Target) when is_binary(Source), is_binary(Target) -> {error, missing_source} end. - % Private Functions split(SourceDb, Partitioned, Engine, Targets0, PickFun, {M, F, A} = HashFun) -> - Targets = maps:fold(fun(Key, DbName, Map) -> - case couch_db:validate_dbname(DbName) of - ok -> - ok; - {error, E} -> - throw({target_create_error, DbName, E, Map}) + Targets = maps:fold( + fun(Key, DbName, Map) -> + case couch_db:validate_dbname(DbName) of + ok -> + ok; + {error, E} -> + throw({target_create_error, DbName, E, Map}) + end, + case couch_server:lock(DbName, <<"shard splitting">>) of + ok -> + ok; + {error, Err} -> + throw({target_create_error, DbName, Err, Map}) + end, + {ok, Filepath} = couch_server:get_engine_path(DbName, Engine), + Opts = + [create, ?ADMIN_CTX] ++ + case Partitioned of + true -> [{props, [{partitioned, true}, {hash, [M, F, A]}]}]; + false -> [] + end, + case couch_db:start_link(Engine, DbName, Filepath, Opts) of + {ok, Db} -> + Map#{Key => #target{db = Db}}; + {error, Error} -> + throw({target_create_error, DbName, Error, Map}) + end end, - case couch_server:lock(DbName, <<"shard splitting">>) of - ok -> - ok; - {error, Err} -> - throw({target_create_error, DbName, Err, Map}) - end, - {ok, Filepath} = couch_server:get_engine_path(DbName, Engine), - Opts = [create, ?ADMIN_CTX] ++ case Partitioned of - true -> [{props, [{partitioned, true}, {hash, [M, F, A]}]}]; - false -> [] - end, - case couch_db:start_link(Engine, DbName, Filepath, Opts) of - {ok, Db} -> - Map#{Key => #target{db = Db}}; - {error, Error} -> - throw({target_create_error, DbName, Error, Map}) - end - end, #{}, Targets0), + #{}, + Targets0 + ), Seq = couch_db:get_update_seq(SourceDb), State1 = #state{ source_db = SourceDb, @@ -166,24 +172,27 @@ split(SourceDb, Partitioned, Engine, Targets0, PickFun, {M, F, A} = HashFun) -> stop_targets(State6#state.targets), {ok, Seq}. - cleanup_targets(#{} = Targets, Engine) -> - maps:map(fun(_, #target{db = Db} = T) -> - ok = stop_target_db(Db), - DbName = couch_db:name(Db), - delete_target(DbName, Engine), - couch_server:unlock(DbName), - T - end, Targets). - + maps:map( + fun(_, #target{db = Db} = T) -> + ok = stop_target_db(Db), + DbName = couch_db:name(Db), + delete_target(DbName, Engine), + couch_server:unlock(DbName), + T + end, + Targets + ). stop_targets(#{} = Targets) -> - maps:map(fun(_, #target{db = Db} = T) -> - {ok, Db1} = couch_db_engine:commit_data(Db), - ok = stop_target_db(Db1), - T - end, Targets). - + maps:map( + fun(_, #target{db = Db} = T) -> + {ok, Db1} = couch_db_engine:commit_data(Db), + ok = stop_target_db(Db1), + T + end, + Targets + ). stop_target_db(Db) -> couch_db:close(Db), @@ -193,84 +202,91 @@ stop_target_db(Db) -> couch_server:unlock(couch_db:name(Db)), ok. - delete_target(DbName, Engine) -> RootDir = config:get("couchdb", "database_dir", "."), {ok, Filepath} = couch_server:get_engine_path(DbName, Engine), DelOpt = [{context, compaction}, sync], couch_db_engine:delete(Engine, RootDir, Filepath, DelOpt). - pick_target(DocId, #state{} = State, #{} = Targets) -> #state{pickfun = PickFun, hashfun = HashFun} = State, Key = PickFun(DocId, maps:keys(Targets), HashFun), {Key, maps:get(Key, Targets)}. - set_targets_update_seq(#state{targets = Targets} = State) -> Seq = couch_db:get_update_seq(State#state.source_db), - Targets1 = maps:map(fun(_, #target{db = Db} = Target) -> - {ok, Db1} = couch_db_engine:set_update_seq(Db, Seq), - Target#target{db = Db1} - end, Targets), + Targets1 = maps:map( + fun(_, #target{db = Db} = Target) -> + {ok, Db1} = couch_db_engine:set_update_seq(Db, Seq), + Target#target{db = Db1} + end, + Targets + ), State#state{targets = Targets1}. - copy_checkpoints(#state{} = State) -> #state{source_db = Db, source_uuid = SrcUUID, targets = Targets} = State, FoldFun = fun(#doc{id = Id} = Doc, Acc) -> - UpdatedAcc = case Id of - <<?LOCAL_DOC_PREFIX, "shard-sync-", _/binary>> -> - % Transform mem3 internal replicator checkpoints to avoid - % rewinding the changes feed when it sees the new shards - maps:map(fun(_, #target{uuid = TgtUUID, buffer = Docs} = T) -> - Doc1 = update_checkpoint_doc(SrcUUID, TgtUUID, Doc), - T#target{buffer = [Doc1 | Docs]} - end, Acc); - <<?LOCAL_DOC_PREFIX, "purge-", _/binary>> -> - % Copy purge checkpoints to all shards - maps:map(fun(_, #target{buffer = Docs} = T) -> - T#target{buffer = [Doc | Docs]} - end, Acc); - <<?LOCAL_DOC_PREFIX, _/binary>> -> - % Skip copying these that will be done during - % local docs top off right before the shards are switched - Acc - end, + UpdatedAcc = + case Id of + <<?LOCAL_DOC_PREFIX, "shard-sync-", _/binary>> -> + % Transform mem3 internal replicator checkpoints to avoid + % rewinding the changes feed when it sees the new shards + maps:map( + fun(_, #target{uuid = TgtUUID, buffer = Docs} = T) -> + Doc1 = update_checkpoint_doc(SrcUUID, TgtUUID, Doc), + T#target{buffer = [Doc1 | Docs]} + end, + Acc + ); + <<?LOCAL_DOC_PREFIX, "purge-", _/binary>> -> + % Copy purge checkpoints to all shards + maps:map( + fun(_, #target{buffer = Docs} = T) -> + T#target{buffer = [Doc | Docs]} + end, + Acc + ); + <<?LOCAL_DOC_PREFIX, _/binary>> -> + % Skip copying these that will be done during + % local docs top off right before the shards are switched + Acc + end, {ok, UpdatedAcc} end, {ok, Targets1} = couch_db_engine:fold_local_docs(Db, FoldFun, Targets, []), - Targets2 = maps:map(fun(_, #target{db = TDb, buffer = Docs} = T) -> - case Docs of - [] -> - T; - [_ | _] -> - Docs1 = lists:reverse(Docs), - {ok, TDb1} = couch_db_engine:write_doc_infos(TDb, [], Docs1), - {ok, TDb2} = couch_db_engine:commit_data(TDb1), - T#target{db = TDb2, buffer = []} - end - end, Targets1), + Targets2 = maps:map( + fun(_, #target{db = TDb, buffer = Docs} = T) -> + case Docs of + [] -> + T; + [_ | _] -> + Docs1 = lists:reverse(Docs), + {ok, TDb1} = couch_db_engine:write_doc_infos(TDb, [], Docs1), + {ok, TDb2} = couch_db_engine:commit_data(TDb1), + T#target{db = TDb2, buffer = []} + end + end, + Targets1 + ), State#state{targets = Targets2}. - update_checkpoint_doc(Old, New, #doc{body = {Props}} = Doc) -> - NewProps = case couch_util:get_value(<<"target_uuid">>, Props) of - Old -> - replace_kv(Props, {<<"target_uuid">>, Old, New}); - Other when is_binary(Other) -> - replace_kv(Props, {<<"source_uuid">>, Old, New}) - end, + NewProps = + case couch_util:get_value(<<"target_uuid">>, Props) of + Old -> + replace_kv(Props, {<<"target_uuid">>, Old, New}); + Other when is_binary(Other) -> + replace_kv(Props, {<<"source_uuid">>, Old, New}) + end, NewId = update_checkpoint_id(Doc#doc.id, Old, New), Doc#doc{id = NewId, body = {NewProps}}. - update_checkpoint_id(Id, Old, New) -> OldHash = mem3_rep:local_id_hash(Old), NewHash = mem3_rep:local_id_hash(New), binary:replace(Id, OldHash, NewHash). - replace_kv({[]}, _) -> {[]}; replace_kv({KVs}, Replacement) -> @@ -286,30 +302,33 @@ replace_kv({K, V}, Replacement) -> replace_kv(V, _) -> V. - copy_meta(#state{source_db = SourceDb, targets = Targets} = State) -> RevsLimit = couch_db:get_revs_limit(SourceDb), {SecProps} = couch_db:get_security(SourceDb), PurgeLimit = couch_db:get_purge_infos_limit(SourceDb), - Targets1 = maps:map(fun(_, #target{db = Db} = T) -> - {ok, Db1} = couch_db_engine:set_revs_limit(Db, RevsLimit), - {ok, Db2} = couch_db_engine:set_security(Db1, SecProps), - {ok, Db3} = couch_db_engine:set_purge_infos_limit(Db2, PurgeLimit), - T#target{db = Db3} - end, Targets), + Targets1 = maps:map( + fun(_, #target{db = Db} = T) -> + {ok, Db1} = couch_db_engine:set_revs_limit(Db, RevsLimit), + {ok, Db2} = couch_db_engine:set_security(Db1, SecProps), + {ok, Db3} = couch_db_engine:set_purge_infos_limit(Db2, PurgeLimit), + T#target{db = Db3} + end, + Targets + ), State#state{targets = Targets1}. - copy_purge_info(#state{source_db = Db} = State) -> Seq = max(0, couch_db:get_oldest_purge_seq(Db) - 1), {ok, NewState} = couch_db:fold_purge_infos(Db, Seq, fun purge_cb/2, State), - Targets = maps:map(fun(_, #target{} = T) -> - commit_purge_infos(T) - end, NewState#state.targets), + Targets = maps:map( + fun(_, #target{} = T) -> + commit_purge_infos(T) + end, + NewState#state.targets + ), NewState#state{targets = Targets}. - -acc_and_flush(Item, #target{}= Target, MaxBuffer, FlushCb) -> +acc_and_flush(Item, #target{} = Target, MaxBuffer, FlushCb) -> #target{buffer = Buffer, buffer_size = BSize} = Target, BSize1 = BSize + ?term_size(Item), Target1 = Target#target{buffer = [Item | Buffer], buffer_size = BSize1}, @@ -318,37 +337,34 @@ acc_and_flush(Item, #target{}= Target, MaxBuffer, FlushCb) -> false -> Target1 end. - purge_cb({_PSeq, _UUID, Id, _Revs} = PI, #state{targets = Targets} = State) -> {Key, Target} = pick_target(Id, State, Targets), MaxBuffer = State#state.max_buffer_size, Target1 = acc_and_flush(PI, Target, MaxBuffer, fun commit_purge_infos/1), {ok, State#state{targets = Targets#{Key => Target1}}}. - commit_purge_infos(#target{buffer = [], db = Db} = Target) -> Target#target{db = Db}; - commit_purge_infos(#target{buffer = PIs0, db = Db} = Target) -> PIs = lists:reverse(PIs0), {ok, Db1} = couch_db_engine:copy_purge_infos(Db, PIs), {ok, Db2} = couch_db_engine:commit_data(Db1), Target#target{buffer = [], buffer_size = 0, db = Db2}. - copy_docs(#state{source_db = Db} = State) -> {ok, NewState} = couch_db:fold_changes(Db, 0, fun changes_cb/2, State), - CommitTargets = maps:map(fun(_, #target{} = T) -> - commit_docs(T) - end, NewState#state.targets), + CommitTargets = maps:map( + fun(_, #target{} = T) -> + commit_docs(T) + end, + NewState#state.targets + ), NewState#state{targets = CommitTargets}. - % Backwards compatibility clause. Seq trees used to hold #doc_infos at one time changes_cb(#doc_info{id = Id}, #state{source_db = Db} = State) -> [FDI = #full_doc_info{}] = couch_db_engine:open_docs(Db, [Id]), changes_cb(FDI, State); - changes_cb(#full_doc_info{id = Id} = FDI, #state{} = State) -> #state{source_db = SourceDb, targets = Targets} = State, {Key, Target} = pick_target(Id, State, Targets), @@ -357,17 +373,14 @@ changes_cb(#full_doc_info{id = Id} = FDI, #state{} = State) -> Target1 = acc_and_flush(FDI1, Target, MaxBuffer, fun commit_docs/1), {ok, State#state{targets = Targets#{Key => Target1}}}. - commit_docs(#target{buffer = [], db = Db} = Target) -> Target#target{db = Db}; - commit_docs(#target{buffer = FDIs, db = Db} = Target) -> Pairs = [{not_found, FDI} || FDI <- lists:reverse(FDIs)], {ok, Db1} = couch_db_engine:write_doc_infos(Db, Pairs, []), {ok, Db2} = couch_db_engine:commit_data(Db1), Target#target{buffer = [], buffer_size = 0, db = Db2}. - process_fdi(FDI, SourceDb, TargetDb) -> #full_doc_info{id = Id, rev_tree = RTree} = FDI, Acc = #racc{id = Id, source_db = SourceDb, target_db = TargetDb}, @@ -378,10 +391,8 @@ process_fdi(FDI, SourceDb, TargetDb) -> sizes = #size_info{active = Active, external = External} }. - revtree_cb(_Rev, _Leaf, branch, Acc) -> {[], Acc}; - revtree_cb({Pos, RevId}, Leaf, leaf, Acc) -> #racc{id = Id, source_db = SourceDb, target_db = TargetDb} = Acc, #leaf{deleted = Deleted, ptr = Ptr, sizes = LeafSizes} = Leaf, @@ -393,16 +404,20 @@ revtree_cb({Pos, RevId}, Leaf, leaf, Acc) -> }, Doc1 = couch_db_engine:read_doc_body(SourceDb, Doc0), #doc{body = Body, atts = AttInfos0} = Doc1, - External = case LeafSizes#size_info.external of - 0 when is_binary(Body) -> - couch_compress:uncompressed_size(Body); - 0 -> - couch_ejson_size:encoded_size(Body); - N -> N - end, - AttInfos = if not is_binary(AttInfos0) -> AttInfos0; true -> - couch_compress:decompress(AttInfos0) - end, + External = + case LeafSizes#size_info.external of + 0 when is_binary(Body) -> + couch_compress:uncompressed_size(Body); + 0 -> + couch_ejson_size:encoded_size(Body); + N -> + N + end, + AttInfos = + if + not is_binary(AttInfos0) -> AttInfos0; + true -> couch_compress:decompress(AttInfos0) + end, Atts = [process_attachment(Att, SourceDb, TargetDb) || Att <- AttInfos], Doc2 = Doc1#doc{atts = Atts}, Doc3 = couch_db_engine:serialize_doc(TargetDb, Doc2), @@ -417,42 +432,45 @@ revtree_cb({Pos, RevId}, Leaf, leaf, Acc) -> }, {NewLeaf, add_sizes(Active, External, AttSizes, Acc)}. - % This is copied almost verbatim from the compactor -process_attachment({Name, Type, BinSp, AttLen, RevPos, ExpectedMd5}, SourceDb, - TargetDb) -> +process_attachment( + {Name, Type, BinSp, AttLen, RevPos, ExpectedMd5}, + SourceDb, + TargetDb +) -> % 010 upgrade code {ok, SrcStream} = couch_db_engine:open_read_stream(SourceDb, BinSp), {ok, DstStream} = couch_db_engine:open_write_stream(TargetDb, []), ok = couch_stream:copy(SrcStream, DstStream), {NewStream, AttLen, AttLen, ActualMd5, _IdentityMd5} = - couch_stream:close(DstStream), + couch_stream:close(DstStream), {ok, NewBinSp} = couch_stream:to_disk_term(NewStream), couch_util:check_md5(ExpectedMd5, ActualMd5), {Name, Type, NewBinSp, AttLen, AttLen, RevPos, ExpectedMd5, identity}; - -process_attachment({Name, Type, BinSp, AttLen, DiskLen, RevPos, ExpectedMd5, - Enc1}, SourceDb, TargetDb) -> +process_attachment( + {Name, Type, BinSp, AttLen, DiskLen, RevPos, ExpectedMd5, Enc1}, SourceDb, TargetDb +) -> {ok, SrcStream} = couch_db_engine:open_read_stream(SourceDb, BinSp), {ok, DstStream} = couch_db_engine:open_write_stream(TargetDb, []), ok = couch_stream:copy(SrcStream, DstStream), {NewStream, AttLen, _, ActualMd5, _IdentityMd5} = - couch_stream:close(DstStream), + couch_stream:close(DstStream), {ok, NewBinSp} = couch_stream:to_disk_term(NewStream), couch_util:check_md5(ExpectedMd5, ActualMd5), - Enc = case Enc1 of - true -> gzip; % 0110 upgrade code - false -> identity; % 0110 upgrade code - _ -> Enc1 - end, + Enc = + case Enc1 of + % 0110 upgrade code + true -> gzip; + % 0110 upgrade code + false -> identity; + _ -> Enc1 + end, {Name, Type, NewBinSp, AttLen, DiskLen, RevPos, ExpectedMd5, Enc}. - get_engine(Db) -> {ok, DbInfoProps} = couch_db:get_db_info(Db), proplists:get_value(engine, DbInfoProps). - add_sizes(Active, External, Atts, #racc{} = Acc) -> #racc{active = ActiveAcc, external = ExternalAcc, atts = AttsAcc} = Acc, NewActiveAcc = ActiveAcc + Active, @@ -464,41 +482,42 @@ add_sizes(Active, External, Atts, #racc{} = Acc) -> atts = NewAttsAcc }. - total_sizes(#racc{active = Active, external = External, atts = Atts}) -> TotalAtts = lists:foldl(fun({_, S}, A) -> S + A end, 0, Atts), {Active + TotalAtts, External + TotalAtts}. - get_max_buffer_size() -> config:get_integer("reshard", "split_buffer_size", ?DEFAULT_BUFFER_SIZE). - copy_local_docs(#state{source_db = Db, targets = Targets} = State) -> FoldFun = fun(#doc{id = Id} = Doc, Acc) -> - UpdatedAcc = case Id of - <<?LOCAL_DOC_PREFIX, "shard-sync-", _/binary>> -> - Acc; - <<?LOCAL_DOC_PREFIX, "purge-", _/binary>> -> - Acc; - <<?LOCAL_DOC_PREFIX, _/binary>> -> - % Users' and replicator app's checkpoints go to their - % respective shards based on the general hashing algorithm - {Key, Target} = pick_target(Id, State, Acc), - #target{buffer = Docs} = Target, - Acc#{Key => Target#target{buffer = [Doc | Docs]}} - end, + UpdatedAcc = + case Id of + <<?LOCAL_DOC_PREFIX, "shard-sync-", _/binary>> -> + Acc; + <<?LOCAL_DOC_PREFIX, "purge-", _/binary>> -> + Acc; + <<?LOCAL_DOC_PREFIX, _/binary>> -> + % Users' and replicator app's checkpoints go to their + % respective shards based on the general hashing algorithm + {Key, Target} = pick_target(Id, State, Acc), + #target{buffer = Docs} = Target, + Acc#{Key => Target#target{buffer = [Doc | Docs]}} + end, {ok, UpdatedAcc} end, {ok, Targets1} = couch_db:fold_local_docs(Db, FoldFun, Targets, []), - Targets2 = maps:map(fun(_, #target{db = TDb, buffer = Docs} = T) -> - case Docs of - [] -> - T; - [_ | _] -> - Docs1 = lists:reverse(Docs), - {ok, _} = couch_db:update_docs(TDb, Docs1), - T#target{buffer = []} - end - end, Targets1), + Targets2 = maps:map( + fun(_, #target{db = TDb, buffer = Docs} = T) -> + case Docs of + [] -> + T; + [_ | _] -> + Docs1 = lists:reverse(Docs), + {ok, _} = couch_db:update_docs(TDb, Docs1), + T#target{buffer = []} + end + end, + Targets1 + ), State#state{targets = Targets2}. diff --git a/src/couch/src/couch_db_updater.erl b/src/couch/src/couch_db_updater.erl index 535acfad6..710b70510 100644 --- a/src/couch/src/couch_db_updater.erl +++ b/src/couch/src/couch_db_updater.erl @@ -15,14 +15,14 @@ -vsn(1). -export([add_sizes/3, upgrade_sizes/1]). --export([init/1,terminate/2,handle_call/3,handle_cast/2,code_change/3,handle_info/2]). +-export([init/1, terminate/2, handle_call/3, handle_cast/2, code_change/3, handle_info/2]). -include_lib("couch/include/couch_db.hrl"). -include("couch_db_int.hrl"). -define(IDLE_LIMIT_DEFAULT, 61000). --define(DEFAULT_MAX_PARTITION_SIZE, 16#280000000). % 10 GiB - +% 10 GiB +-define(DEFAULT_MAX_PARTITION_SIZE, 16#280000000). -record(merge_acc, { revs_limit, @@ -33,7 +33,6 @@ full_partitions = [] }). - init({Engine, DbName, FilePath, Options0}) -> erlang:put(io_priority, {db_update, DbName}), update_idle_limit_from_config(), @@ -59,7 +58,6 @@ init({Engine, DbName, FilePath, Options0}) -> proc_lib:init_ack(InitError) end. - terminate(Reason, Db) -> couch_util:shutdown_sync(Db#db.compactor_pid), couch_db_engine:terminate(Reason, Db), @@ -81,7 +79,6 @@ handle_call(cancel_compact, _From, #db{compactor_pid = Pid} = Db) -> Db2 = Db#db{compactor_pid = nil}, ok = couch_server:db_updated(Db2), {reply, ok, Db2, idle_limit()}; - handle_call({set_security, NewSec}, _From, #db{} = Db) -> {ok, NewDb} = couch_db_engine:set_security(Db, NewSec), NewSecDb = commit_data(NewDb#db{ @@ -89,36 +86,38 @@ handle_call({set_security, NewSec}, _From, #db{} = Db) -> }), ok = couch_server:db_updated(NewSecDb), {reply, ok, NewSecDb, idle_limit()}; - handle_call({set_revs_limit, Limit}, _From, Db) -> {ok, Db2} = couch_db_engine:set_revs_limit(Db, Limit), Db3 = commit_data(Db2), ok = couch_server:db_updated(Db3), {reply, ok, Db3, idle_limit()}; - handle_call({set_purge_infos_limit, Limit}, _From, Db) -> {ok, Db2} = couch_db_engine:set_purge_infos_limit(Db, Limit), ok = couch_server:db_updated(Db2), {reply, ok, Db2, idle_limit()}; - handle_call({purge_docs, [], _}, _From, Db) -> {reply, {ok, []}, Db, idle_limit()}; - handle_call({purge_docs, PurgeReqs0, Options}, _From, Db) -> % Filter out any previously applied updates during % internal replication IsRepl = lists:member(replicated_changes, Options), - PurgeReqs = if not IsRepl -> PurgeReqs0; true -> - UUIDs = [UUID || {UUID, _Id, _Revs} <- PurgeReqs0], - PurgeInfos = couch_db_engine:load_purge_infos(Db, UUIDs), - lists:flatmap(fun - ({not_found, PReq}) -> [PReq]; - ({{_, _, _, _}, _}) -> [] - end, lists:zip(PurgeInfos, PurgeReqs0)) - end, + PurgeReqs = + if + not IsRepl -> + PurgeReqs0; + true -> + UUIDs = [UUID || {UUID, _Id, _Revs} <- PurgeReqs0], + PurgeInfos = couch_db_engine:load_purge_infos(Db, UUIDs), + lists:flatmap( + fun + ({not_found, PReq}) -> [PReq]; + ({{_, _, _, _}, _}) -> [] + end, + lists:zip(PurgeInfos, PurgeReqs0) + ) + end, {ok, NewDb, Replies} = purge_docs(Db, PurgeReqs), {reply, {ok, Replies}, NewDb, idle_limit()}; - handle_call(Msg, From, Db) -> case couch_db_engine:handle_db_updater_call(Msg, From, Db) of {reply, Resp, NewDb} -> @@ -127,7 +126,6 @@ handle_call(Msg, From, Db) -> Else end. - handle_cast({load_validation_funs, ValidationFuns}, Db) -> Db2 = Db#db{validate_doc_funs = ValidationFuns}, ok = couch_server:db_updated(Db2), @@ -152,65 +150,76 @@ handle_cast(start_compact, Db) -> handle_cast({compact_done, _Engine, CompactInfo}, #db{} = OldDb) -> {ok, NewDb} = couch_db_engine:finish_compaction(OldDb, CompactInfo), {noreply, NewDb}; - handle_cast(wakeup, Db) -> {noreply, Db, idle_limit()}; - handle_cast(Msg, #db{name = Name} = Db) -> - couch_log:error("Database `~s` updater received unexpected cast: ~p", - [Name, Msg]), + couch_log:error( + "Database `~s` updater received unexpected cast: ~p", + [Name, Msg] + ), {stop, Msg, Db}. - -handle_info({update_docs, Client, GroupedDocs, NonRepDocs, MergeConflicts}, - Db) -> +handle_info( + {update_docs, Client, GroupedDocs, NonRepDocs, MergeConflicts}, + Db +) -> GroupedDocs2 = sort_and_tag_grouped_docs(Client, GroupedDocs), - if NonRepDocs == [] -> - {GroupedDocs3, Clients} = collect_updates(GroupedDocs2, - [Client], MergeConflicts); - true -> - GroupedDocs3 = GroupedDocs2, - Clients = [Client] + if + NonRepDocs == [] -> + {GroupedDocs3, Clients} = collect_updates( + GroupedDocs2, + [Client], + MergeConflicts + ); + true -> + GroupedDocs3 = GroupedDocs2, + Clients = [Client] end, NonRepDocs2 = [{Client, NRDoc} || NRDoc <- NonRepDocs], try update_docs_int(Db, GroupedDocs3, NonRepDocs2, MergeConflicts) of - {ok, Db2, UpdatedDDocIds} -> - ok = couch_server:db_updated(Db2), - case {couch_db:get_update_seq(Db), couch_db:get_update_seq(Db2)} of - {Seq, Seq} -> ok; - _ -> couch_event:notify(Db2#db.name, updated) - end, - if NonRepDocs2 /= [] -> - couch_event:notify(Db2#db.name, local_updated); - true -> ok - end, - [catch(ClientPid ! {done, self()}) || ClientPid <- Clients], - Db3 = case length(UpdatedDDocIds) > 0 of - true -> - % Ken and ddoc_cache are the only things that - % use the unspecified ddoc_updated message. We - % should update them to use the new message per - % ddoc. - lists:foreach(fun(DDocId) -> - couch_event:notify(Db2#db.name, {ddoc_updated, DDocId}) - end, UpdatedDDocIds), - couch_event:notify(Db2#db.name, ddoc_updated), - ddoc_cache:refresh(Db2#db.name, UpdatedDDocIds), - refresh_validate_doc_funs(Db2); - false -> - Db2 - end, - {noreply, Db3, hibernate_if_no_idle_limit()} + {ok, Db2, UpdatedDDocIds} -> + ok = couch_server:db_updated(Db2), + case {couch_db:get_update_seq(Db), couch_db:get_update_seq(Db2)} of + {Seq, Seq} -> ok; + _ -> couch_event:notify(Db2#db.name, updated) + end, + if + NonRepDocs2 /= [] -> + couch_event:notify(Db2#db.name, local_updated); + true -> + ok + end, + [catch (ClientPid ! {done, self()}) || ClientPid <- Clients], + Db3 = + case length(UpdatedDDocIds) > 0 of + true -> + % Ken and ddoc_cache are the only things that + % use the unspecified ddoc_updated message. We + % should update them to use the new message per + % ddoc. + lists:foreach( + fun(DDocId) -> + couch_event:notify(Db2#db.name, {ddoc_updated, DDocId}) + end, + UpdatedDDocIds + ), + couch_event:notify(Db2#db.name, ddoc_updated), + ddoc_cache:refresh(Db2#db.name, UpdatedDDocIds), + refresh_validate_doc_funs(Db2); + false -> + Db2 + end, + {noreply, Db3, hibernate_if_no_idle_limit()} catch - throw: retry -> - [catch(ClientPid ! {retry, self()}) || ClientPid <- Clients], + throw:retry -> + [catch (ClientPid ! {retry, self()}) || ClientPid <- Clients], {noreply, Db, hibernate_if_no_idle_limit()} end; handle_info({'EXIT', _Pid, normal}, Db) -> {noreply, Db, idle_limit()}; handle_info({'EXIT', _Pid, Reason}, Db) -> {stop, Reason, Db}; -handle_info(timeout, #db{name=DbName} = Db) -> +handle_info(timeout, #db{name = DbName} = Db) -> IdleLimitMSec = update_idle_limit_from_config(), case couch_db:is_idle(Db) of true -> @@ -230,7 +239,6 @@ handle_info(timeout, #db{name=DbName} = Db) -> % force a thorough garbage collection. gen_server:cast(self(), wakeup), {noreply, Db, hibernate}; - handle_info(Msg, Db) -> case couch_db_engine:handle_db_updater_info(Msg, Db) of {noreply, NewDb} -> @@ -239,7 +247,6 @@ handle_info(Msg, Db) -> Else end. - code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -248,25 +255,28 @@ sort_and_tag_grouped_docs(Client, GroupedDocs) -> % The merge_updates function will fail and the database can end up with % duplicate documents if the incoming groups are not sorted, so as a sanity % check we sort them again here. See COUCHDB-2735. - Cmp = fun([#doc{id=A}|_], [#doc{id=B}|_]) -> A < B end, - lists:map(fun(DocGroup) -> - [{Client, maybe_tag_doc(D)} || D <- DocGroup] - end, lists:sort(Cmp, GroupedDocs)). + Cmp = fun([#doc{id = A} | _], [#doc{id = B} | _]) -> A < B end, + lists:map( + fun(DocGroup) -> + [{Client, maybe_tag_doc(D)} || D <- DocGroup] + end, + lists:sort(Cmp, GroupedDocs) + ). -maybe_tag_doc(#doc{id=Id, revs={Pos,[_Rev|PrevRevs]}, meta=Meta0}=Doc) -> +maybe_tag_doc(#doc{id = Id, revs = {Pos, [_Rev | PrevRevs]}, meta = Meta0} = Doc) -> case lists:keymember(ref, 1, Meta0) of true -> Doc; false -> - Key = {Id, {Pos-1, PrevRevs}}, - Doc#doc{meta=[{ref, Key} | Meta0]} + Key = {Id, {Pos - 1, PrevRevs}}, + Doc#doc{meta = [{ref, Key} | Meta0]} end. -merge_updates([[{_,#doc{id=X}}|_]=A|RestA], [[{_,#doc{id=X}}|_]=B|RestB]) -> - [A++B | merge_updates(RestA, RestB)]; -merge_updates([[{_,#doc{id=X}}|_]|_]=A, [[{_,#doc{id=Y}}|_]|_]=B) when X < Y -> +merge_updates([[{_, #doc{id = X}} | _] = A | RestA], [[{_, #doc{id = X}} | _] = B | RestB]) -> + [A ++ B | merge_updates(RestA, RestB)]; +merge_updates([[{_, #doc{id = X}} | _] | _] = A, [[{_, #doc{id = Y}} | _] | _] = B) when X < Y -> [hd(A) | merge_updates(tl(A), B)]; -merge_updates([[{_,#doc{id=X}}|_]|_]=A, [[{_,#doc{id=Y}}|_]|_]=B) when X > Y -> +merge_updates([[{_, #doc{id = X}} | _] | _] = A, [[{_, #doc{id = Y}} | _] | _] = B) when X > Y -> [hd(B) | merge_updates(A, tl(B))]; merge_updates([], RestB) -> RestB; @@ -283,18 +293,24 @@ collect_updates(GroupedDocsAcc, ClientsAcc, MergeConflicts) -> GroupedDocs2 = sort_and_tag_grouped_docs(Client, GroupedDocs), GroupedDocsAcc2 = merge_updates(GroupedDocsAcc, GroupedDocs2), - collect_updates(GroupedDocsAcc2, [Client | ClientsAcc], - MergeConflicts) + collect_updates( + GroupedDocsAcc2, + [Client | ClientsAcc], + MergeConflicts + ) after 0 -> {GroupedDocsAcc, ClientsAcc} end. - init_db(DbName, FilePath, EngineState, Options) -> % convert start time tuple to microsecs and store as a binary string {MegaSecs, Secs, MicroSecs} = os:timestamp(), - StartTime = ?l2b(io_lib:format("~p", - [(MegaSecs*1000000*1000000) + (Secs*1000000) + MicroSecs])), + StartTime = ?l2b( + io_lib:format( + "~p", + [(MegaSecs * 1000000 * 1000000) + (Secs * 1000000) + MicroSecs] + ) + ), BDU = couch_util:get_value(before_doc_update, Options, nil), ADR = couch_util:get_value(after_doc_read, Options, nil), @@ -319,31 +335,36 @@ init_db(DbName, FilePath, EngineState, Options) -> options = lists:keystore(props, 1, NonCreateOpts, {props, DbProps}) }. - refresh_validate_doc_funs(#db{name = <<"shards/", _/binary>> = Name} = Db) -> spawn(fabric, reset_validation_funs, [mem3:dbname(Name)]), Db#db{validate_doc_funs = undefined}; refresh_validate_doc_funs(Db0) -> - Db = Db0#db{user_ctx=?ADMIN_USER}, + Db = Db0#db{user_ctx = ?ADMIN_USER}, {ok, DesignDocs} = couch_db:get_design_docs(Db), ProcessDocFuns = lists:flatmap( fun(DesignDocInfo) -> {ok, DesignDoc} = couch_db:open_doc_int( - Db, DesignDocInfo, [ejson_body]), + Db, DesignDocInfo, [ejson_body] + ), case couch_doc:get_validate_doc_fun(DesignDoc) of - nil -> []; - Fun -> [Fun] + nil -> []; + Fun -> [Fun] end - end, DesignDocs), - Db#db{validate_doc_funs=ProcessDocFuns}. + end, + DesignDocs + ), + Db#db{validate_doc_funs = ProcessDocFuns}. % rev tree functions flush_trees(_Db, [], AccFlushedTrees) -> {ok, lists:reverse(AccFlushedTrees)}; -flush_trees(#db{} = Db, - [InfoUnflushed | RestUnflushed], AccFlushed) -> - #full_doc_info{update_seq=UpdateSeq, rev_tree=Unflushed} = InfoUnflushed, +flush_trees( + #db{} = Db, + [InfoUnflushed | RestUnflushed], + AccFlushed +) -> + #full_doc_info{update_seq = UpdateSeq, rev_tree = Unflushed} = InfoUnflushed, {Flushed, FinalAcc} = couch_key_tree:mapfold( fun(_Rev, Value, Type, SizesAcc) -> case Value of @@ -353,9 +374,9 @@ flush_trees(#db{} = Db, check_doc_atts(Db, Doc), ExternalSize = get_meta_body_size(Value#doc.meta), {size_info, AttSizeInfo} = - lists:keyfind(size_info, 1, Doc#doc.meta), + lists:keyfind(size_info, 1, Doc#doc.meta), {ok, NewDoc, WrittenSize} = - couch_db_engine:write_doc_body(Db, Doc), + couch_db_engine:write_doc_body(Db, Doc), Leaf = #leaf{ deleted = Doc#doc.deleted, ptr = NewDoc#doc.body, @@ -372,7 +393,10 @@ flush_trees(#db{} = Db, _ -> {Value, SizesAcc} end - end, {0, 0, []}, Unflushed), + end, + {0, 0, []}, + Unflushed + ), {FinalAS, FinalES, FinalAtts} = FinalAcc, TotalAttSize = lists:foldl(fun({_, S}, A) -> S + A end, 0, FinalAtts), NewInfo = InfoUnflushed#full_doc_info{ @@ -384,30 +408,34 @@ flush_trees(#db{} = Db, }, flush_trees(Db, RestUnflushed, [NewInfo | AccFlushed]). - check_doc_atts(Db, Doc) -> {atts_stream, Stream} = lists:keyfind(atts_stream, 1, Doc#doc.meta), % Make sure that the attachments were written to the currently % active attachment stream. If compaction swaps during a write % request we may have to rewrite our attachment bodies. - if Stream == nil -> ok; true -> - case couch_db:is_active_stream(Db, Stream) of - true -> - ok; - false -> - % Stream where the attachments were written to is - % no longer the current attachment stream. This - % can happen when a database is switched at - % compaction time. - couch_log:debug("Stream where the attachments were" - " written has changed." - " Possibly retrying.", []), - throw(retry) - end + if + Stream == nil -> + ok; + true -> + case couch_db:is_active_stream(Db, Stream) of + true -> + ok; + false -> + % Stream where the attachments were written to is + % no longer the current attachment stream. This + % can happen when a database is switched at + % compaction time. + couch_log:debug( + "Stream where the attachments were" + " written has changed." + " Possibly retrying.", + [] + ), + throw(retry) + end end. - -add_sizes(Type, #leaf{sizes=Sizes, atts=AttSizes}, Acc) -> +add_sizes(Type, #leaf{sizes = Sizes, atts = AttSizes}, Acc) -> % Maybe upgrade from disk_size only #size_info{ active = ActiveSize, @@ -415,24 +443,27 @@ add_sizes(Type, #leaf{sizes=Sizes, atts=AttSizes}, Acc) -> } = upgrade_sizes(Sizes), {ASAcc, ESAcc, AttsAcc} = Acc, NewASAcc = ActiveSize + ASAcc, - NewESAcc = ESAcc + if Type == leaf -> ExternalSize; true -> 0 end, + NewESAcc = + ESAcc + + if + Type == leaf -> ExternalSize; + true -> 0 + end, NewAttsAcc = lists:umerge(AttSizes, AttsAcc), {NewASAcc, NewESAcc, NewAttsAcc}. - -upgrade_sizes(#size_info{}=SI) -> +upgrade_sizes(#size_info{} = SI) -> SI; upgrade_sizes({D, E}) -> - #size_info{active=D, external=E}; + #size_info{active = D, external = E}; upgrade_sizes(S) when is_integer(S) -> - #size_info{active=S, external=0}. - + #size_info{active = S, external = 0}. send_result(Client, Doc, NewResult) -> % used to send a result to the client - catch(Client ! {result, self(), {doc_tag(Doc), NewResult}}). + catch (Client ! {result, self(), {doc_tag(Doc), NewResult}}). -doc_tag(#doc{meta=Meta}) -> +doc_tag(#doc{meta = Meta}) -> case lists:keyfind(ref, 1, Meta) of {ref, Ref} -> Ref; false -> throw(no_doc_tag); @@ -452,17 +483,21 @@ merge_rev_trees([NewDocs | RestDocsList], [OldDocInfo | RestOldInfo], Acc) -> % Track doc ids so we can debug large revision trees erlang:put(last_id_merged, OldDocInfo#full_doc_info.id), - NewDocInfo0 = lists:foldl(fun({Client, NewDoc}, OldInfoAcc) -> - NewInfo = merge_rev_tree(OldInfoAcc, NewDoc, Client, MergeConflicts), - case is_overflowed(NewInfo, OldInfoAcc, FullPartitions) of - true when not MergeConflicts -> - DocId = NewInfo#full_doc_info.id, - send_result(Client, NewDoc, {partition_overflow, DocId}), - OldInfoAcc; - _ -> - NewInfo - end - end, OldDocInfo, NewDocs), + NewDocInfo0 = lists:foldl( + fun({Client, NewDoc}, OldInfoAcc) -> + NewInfo = merge_rev_tree(OldInfoAcc, NewDoc, Client, MergeConflicts), + case is_overflowed(NewInfo, OldInfoAcc, FullPartitions) of + true when not MergeConflicts -> + DocId = NewInfo#full_doc_info.id, + send_result(Client, NewDoc, {partition_overflow, DocId}), + OldInfoAcc; + _ -> + NewInfo + end + end, + OldDocInfo, + NewDocs + ), NewDocInfo1 = maybe_stem_full_doc_info(NewDocInfo0, Limit), % When MergeConflicts is false, we updated #full_doc_info.deleted on every % iteration of merge_rev_tree. However, merge_rev_tree does not update @@ -470,39 +505,43 @@ merge_rev_trees([NewDocs | RestDocsList], [OldDocInfo | RestOldInfo], Acc) -> % to know whether the doc is deleted between iterations. Since we still % need to know if the doc is deleted after the merge happens, we have to % set it here. - NewDocInfo2 = case MergeConflicts of - true -> - NewDocInfo1#full_doc_info{ - deleted = couch_doc:is_deleted(NewDocInfo1) - }; - false -> - NewDocInfo1 - end, - if NewDocInfo2 == OldDocInfo -> - % nothing changed - merge_rev_trees(RestDocsList, RestOldInfo, Acc); - true -> - % We have updated the document, give it a new update_seq. Its - % important to note that the update_seq on OldDocInfo should - % be identical to the value on NewDocInfo1. - OldSeq = OldDocInfo#full_doc_info.update_seq, - NewDocInfo3 = NewDocInfo2#full_doc_info{ - update_seq = Acc#merge_acc.cur_seq + 1 - }, - RemoveSeqs = case OldSeq of - 0 -> Acc#merge_acc.rem_seqs; - _ -> [OldSeq | Acc#merge_acc.rem_seqs] + NewDocInfo2 = + case MergeConflicts of + true -> + NewDocInfo1#full_doc_info{ + deleted = couch_doc:is_deleted(NewDocInfo1) + }; + false -> + NewDocInfo1 end, - NewAcc = Acc#merge_acc{ - add_infos = [NewDocInfo3 | Acc#merge_acc.add_infos], - rem_seqs = RemoveSeqs, - cur_seq = Acc#merge_acc.cur_seq + 1 - }, - merge_rev_trees(RestDocsList, RestOldInfo, NewAcc) + if + NewDocInfo2 == OldDocInfo -> + % nothing changed + merge_rev_trees(RestDocsList, RestOldInfo, Acc); + true -> + % We have updated the document, give it a new update_seq. Its + % important to note that the update_seq on OldDocInfo should + % be identical to the value on NewDocInfo1. + OldSeq = OldDocInfo#full_doc_info.update_seq, + NewDocInfo3 = NewDocInfo2#full_doc_info{ + update_seq = Acc#merge_acc.cur_seq + 1 + }, + RemoveSeqs = + case OldSeq of + 0 -> Acc#merge_acc.rem_seqs; + _ -> [OldSeq | Acc#merge_acc.rem_seqs] + end, + NewAcc = Acc#merge_acc{ + add_infos = [NewDocInfo3 | Acc#merge_acc.add_infos], + rem_seqs = RemoveSeqs, + cur_seq = Acc#merge_acc.cur_seq + 1 + }, + merge_rev_trees(RestDocsList, RestOldInfo, NewAcc) end. -merge_rev_tree(OldInfo, NewDoc, Client, false) - when OldInfo#full_doc_info.deleted -> +merge_rev_tree(OldInfo, NewDoc, Client, false) when + OldInfo#full_doc_info.deleted +-> % We're recreating a document that was previously % deleted. To check that this is a recreation from % the root we assert that the new document has a @@ -517,28 +556,29 @@ merge_rev_tree(OldInfo, NewDoc, Client, false) case RevDepth == 1 andalso not NewDeleted of true -> % Update the new doc based on revisions in OldInfo - #doc_info{revs=[WinningRev | _]} = couch_doc:to_doc_info(OldInfo), - #rev_info{rev={OldPos, OldRev}} = WinningRev, - Body = case couch_util:get_value(comp_body, NewDoc#doc.meta) of - CompBody when is_binary(CompBody) -> - couch_compress:decompress(CompBody); - _ -> - NewDoc#doc.body - end, + #doc_info{revs = [WinningRev | _]} = couch_doc:to_doc_info(OldInfo), + #rev_info{rev = {OldPos, OldRev}} = WinningRev, + Body = + case couch_util:get_value(comp_body, NewDoc#doc.meta) of + CompBody when is_binary(CompBody) -> + couch_compress:decompress(CompBody); + _ -> + NewDoc#doc.body + end, RevIdDoc = NewDoc#doc{ revs = {OldPos, [OldRev]}, body = Body }, NewRevId = couch_db:new_revid(RevIdDoc), - NewDoc2 = NewDoc#doc{revs={OldPos + 1, [NewRevId, OldRev]}}, + NewDoc2 = NewDoc#doc{revs = {OldPos + 1, [NewRevId, OldRev]}}, % Merge our modified new doc into the tree - #full_doc_info{rev_tree=OldTree} = OldInfo, + #full_doc_info{rev_tree = OldTree} = OldInfo, NewTree0 = couch_doc:to_path(NewDoc2), case couch_key_tree:merge(OldTree, NewTree0) of {NewTree1, new_leaf} -> % We changed the revision id so inform the caller - send_result(Client, NewDoc, {ok, {OldPos+1, NewRevId}}), + send_result(Client, NewDoc, {ok, {OldPos + 1, NewRevId}}), OldInfo#full_doc_info{ rev_tree = NewTree1, deleted = false @@ -615,34 +655,44 @@ update_docs_int(Db, DocsList, LocalDocs, MergeConflicts) -> UpdateSeq = couch_db_engine:get_update_seq(Db), RevsLimit = couch_db_engine:get_revs_limit(Db), - Ids = [Id || [{_Client, #doc{id=Id}}|_] <- DocsList], + Ids = [Id || [{_Client, #doc{id = Id}} | _] <- DocsList], % lookup up the old documents, if they exist. OldDocLookups = couch_db_engine:open_docs(Db, Ids), - OldDocInfos = lists:zipwith(fun - (_Id, #full_doc_info{} = FDI) -> - FDI; - (Id, not_found) -> - #full_doc_info{id=Id} - end, Ids, OldDocLookups), + OldDocInfos = lists:zipwith( + fun + (_Id, #full_doc_info{} = FDI) -> + FDI; + (Id, not_found) -> + #full_doc_info{id = Id} + end, + Ids, + OldDocLookups + ), %% Get the list of full partitions - FullPartitions = case couch_db:is_partitioned(Db) of - true -> - case max_partition_size() of - N when N =< 0 -> - []; - Max -> - Partitions = lists:usort(lists:flatmap(fun(Id) -> - case couch_partition:extract(Id) of - undefined -> []; - {Partition, _} -> [Partition] - end - end, Ids)), - [P || P <- Partitions, partition_size(Db, P) >= Max] - end; - false -> - [] - end, + FullPartitions = + case couch_db:is_partitioned(Db) of + true -> + case max_partition_size() of + N when N =< 0 -> + []; + Max -> + Partitions = lists:usort( + lists:flatmap( + fun(Id) -> + case couch_partition:extract(Id) of + undefined -> []; + {Partition, _} -> [Partition] + end + end, + Ids + ) + ), + [P || P <- Partitions, partition_size(Db, P) >= Max] + end; + false -> + [] + end, % Merge the new docs into the revision trees. AccIn = #merge_acc{ @@ -668,8 +718,10 @@ update_docs_int(Db, DocsList, LocalDocs, MergeConflicts) -> {ok, Db1} = couch_db_engine:write_doc_infos(Db, Pairs, LocalDocs2), WriteCount = length(IndexFDIs), - couch_stats:increment_counter([couchdb, document_inserts], - WriteCount - length(RemSeqs)), + couch_stats:increment_counter( + [couchdb, document_inserts], + WriteCount - length(RemSeqs) + ), couch_stats:increment_counter([couchdb, document_writes], WriteCount), couch_stats:increment_counter( [couchdb, local_document_writes], @@ -678,26 +730,31 @@ update_docs_int(Db, DocsList, LocalDocs, MergeConflicts) -> % Check if we just updated any design documents, and update the validation % funs if we did. - UpdatedDDocIds = lists:flatmap(fun - (<<"_design/", _/binary>> = Id) -> [Id]; - (_) -> [] - end, Ids), + UpdatedDDocIds = lists:flatmap( + fun + (<<"_design/", _/binary>> = Id) -> [Id]; + (_) -> [] + end, + Ids + ), {ok, commit_data(Db1), UpdatedDDocIds}. - update_local_doc_revs(Docs) -> - lists:foldl(fun({Client, Doc}, Acc) -> - case increment_local_doc_revs(Doc) of - {ok, #doc{revs = {0, [NewRev]}} = NewDoc} -> - send_result(Client, Doc, {ok, {0, integer_to_binary(NewRev)}}), - [NewDoc | Acc]; - {error, Error} -> - send_result(Client, Doc, {error, Error}), - Acc - end - end, [], Docs). - + lists:foldl( + fun({Client, Doc}, Acc) -> + case increment_local_doc_revs(Doc) of + {ok, #doc{revs = {0, [NewRev]}} = NewDoc} -> + send_result(Client, Doc, {ok, {0, integer_to_binary(NewRev)}}), + [NewDoc | Acc]; + {error, Error} -> + send_result(Client, Doc, {error, Error}), + Acc + end + end, + [], + Docs + ). increment_local_doc_revs(#doc{deleted = true} = Doc) -> {ok, Doc#doc{revs = {0, [0]}}}; @@ -707,15 +764,19 @@ increment_local_doc_revs(#doc{revs = {0, [RevStr | _]}} = Doc) -> try PrevRev = binary_to_integer(RevStr), {ok, Doc#doc{revs = {0, [PrevRev + 1]}}} - catch error:badarg -> - {error, <<"Invalid rev format">>} + catch + error:badarg -> + {error, <<"Invalid rev format">>} end; increment_local_doc_revs(#doc{}) -> {error, <<"Invalid rev format">>}. max_partition_size() -> - config:get_integer("couchdb", "max_partition_size", - ?DEFAULT_MAX_PARTITION_SIZE). + config:get_integer( + "couchdb", + "max_partition_size", + ?DEFAULT_MAX_PARTITION_SIZE + ). partition_size(Db, Partition) -> {ok, Info} = couch_db:get_partition_info(Db, Partition), @@ -750,7 +811,6 @@ estimate_size(#full_doc_info{} = FDI) -> purge_docs(Db, []) -> {ok, Db, []}; - purge_docs(Db, PurgeReqs) -> Ids = lists:usort(lists:map(fun({_UUID, Id, _Revs}) -> Id end, PurgeReqs)), FDIs = couch_db_engine:open_docs(Db, Ids), @@ -759,23 +819,30 @@ purge_docs(Db, PurgeReqs) -> IdFDIs = lists:zip(Ids, FDIs), {NewIdFDIs, Replies} = apply_purge_reqs(PurgeReqs, IdFDIs, USeq, []), - Pairs = lists:flatmap(fun({DocId, OldFDI}) -> - {DocId, NewFDI} = lists:keyfind(DocId, 1, NewIdFDIs), - case {OldFDI, NewFDI} of - {not_found, not_found} -> - []; - {#full_doc_info{} = A, #full_doc_info{} = A} -> - []; - {#full_doc_info{}, _} -> - [{OldFDI, NewFDI}] - end - end, IdFDIs), + Pairs = lists:flatmap( + fun({DocId, OldFDI}) -> + {DocId, NewFDI} = lists:keyfind(DocId, 1, NewIdFDIs), + case {OldFDI, NewFDI} of + {not_found, not_found} -> + []; + {#full_doc_info{} = A, #full_doc_info{} = A} -> + []; + {#full_doc_info{}, _} -> + [{OldFDI, NewFDI}] + end + end, + IdFDIs + ), PSeq = couch_db_engine:get_purge_seq(Db), - {RevPInfos, _} = lists:foldl(fun({UUID, DocId, Revs}, {PIAcc, PSeqAcc}) -> - Info = {PSeqAcc + 1, UUID, DocId, Revs}, - {[Info | PIAcc], PSeqAcc + 1} - end, {[], PSeq}, PurgeReqs), + {RevPInfos, _} = lists:foldl( + fun({UUID, DocId, Revs}, {PIAcc, PSeqAcc}) -> + Info = {PSeqAcc + 1, UUID, DocId, Revs}, + {[Info | PIAcc], PSeqAcc + 1} + end, + {[], PSeq}, + PurgeReqs + ), PInfos = lists:reverse(RevPInfos), {ok, Db1} = couch_db_engine:purge_docs(Db, Pairs, PInfos), @@ -784,85 +851,90 @@ purge_docs(Db, PurgeReqs) -> couch_event:notify(Db2#db.name, updated), {ok, Db2, Replies}. - apply_purge_reqs([], IdFDIs, _USeq, Replies) -> {IdFDIs, lists:reverse(Replies)}; - apply_purge_reqs([Req | RestReqs], IdFDIs, USeq, Replies) -> {_UUID, DocId, Revs} = Req, {value, {_, FDI0}, RestIdFDIs} = lists:keytake(DocId, 1, IdFDIs), - {NewFDI, RemovedRevs, NewUSeq} = case FDI0 of - #full_doc_info{rev_tree = Tree} -> - case couch_key_tree:remove_leafs(Tree, Revs) of - {_, []} -> - % No change - {FDI0, [], USeq}; - {[], Removed} -> - % Completely purged - {not_found, Removed, USeq}; - {NewTree, Removed} -> - % Its possible to purge the #leaf{} that contains - % the update_seq where this doc sits in the - % update_seq sequence. Rather than do a bunch of - % complicated checks we just re-label every #leaf{} - % and reinsert it into the update_seq sequence. - {NewTree2, NewUpdateSeq} = couch_key_tree:mapfold(fun - (_RevId, Leaf, leaf, SeqAcc) -> - {Leaf#leaf{seq = SeqAcc + 1}, - SeqAcc + 1}; - (_RevId, Value, _Type, SeqAcc) -> - {Value, SeqAcc} - end, USeq, NewTree), - - FDI1 = FDI0#full_doc_info{ - update_seq = NewUpdateSeq, - rev_tree = NewTree2 - }, - {FDI1, Removed, NewUpdateSeq} - end; - not_found -> - % Not found means nothing to change - {not_found, [], USeq} - end, + {NewFDI, RemovedRevs, NewUSeq} = + case FDI0 of + #full_doc_info{rev_tree = Tree} -> + case couch_key_tree:remove_leafs(Tree, Revs) of + {_, []} -> + % No change + {FDI0, [], USeq}; + {[], Removed} -> + % Completely purged + {not_found, Removed, USeq}; + {NewTree, Removed} -> + % Its possible to purge the #leaf{} that contains + % the update_seq where this doc sits in the + % update_seq sequence. Rather than do a bunch of + % complicated checks we just re-label every #leaf{} + % and reinsert it into the update_seq sequence. + {NewTree2, NewUpdateSeq} = couch_key_tree:mapfold( + fun + (_RevId, Leaf, leaf, SeqAcc) -> + {Leaf#leaf{seq = SeqAcc + 1}, SeqAcc + 1}; + (_RevId, Value, _Type, SeqAcc) -> + {Value, SeqAcc} + end, + USeq, + NewTree + ), + + FDI1 = FDI0#full_doc_info{ + update_seq = NewUpdateSeq, + rev_tree = NewTree2 + }, + {FDI1, Removed, NewUpdateSeq} + end; + not_found -> + % Not found means nothing to change + {not_found, [], USeq} + end, NewIdFDIs = [{DocId, NewFDI} | RestIdFDIs], NewReplies = [{ok, RemovedRevs} | Replies], apply_purge_reqs(RestReqs, NewIdFDIs, NewUSeq, NewReplies). - commit_data(Db) -> {ok, Db1} = couch_db_engine:commit_data(Db), Db1#db{ committed_update_seq = couch_db_engine:get_update_seq(Db) }. - pair_write_info(Old, New) -> - lists:map(fun(FDI) -> - case lists:keyfind(FDI#full_doc_info.id, #full_doc_info.id, Old) of - #full_doc_info{} = OldFDI -> {OldFDI, FDI}; - false -> {not_found, FDI} - end - end, New). - + lists:map( + fun(FDI) -> + case lists:keyfind(FDI#full_doc_info.id, #full_doc_info.id, Old) of + #full_doc_info{} = OldFDI -> {OldFDI, FDI}; + false -> {not_found, FDI} + end + end, + New + ). get_meta_body_size(Meta) -> {ejson_size, ExternalSize} = lists:keyfind(ejson_size, 1, Meta), ExternalSize. - default_security_object(<<"shards/", _/binary>>) -> case config:get("couchdb", "default_security", "admin_only") of "admin_only" -> - [{<<"members">>,{[{<<"roles">>,[<<"_admin">>]}]}}, - {<<"admins">>,{[{<<"roles">>,[<<"_admin">>]}]}}]; + [ + {<<"members">>, {[{<<"roles">>, [<<"_admin">>]}]}}, + {<<"admins">>, {[{<<"roles">>, [<<"_admin">>]}]}} + ]; Everyone when Everyone == "everyone"; Everyone == "admin_local" -> [] end; default_security_object(_DbName) -> case config:get("couchdb", "default_security", "admin_only") of Admin when Admin == "admin_only"; Admin == "admin_local" -> - [{<<"members">>,{[{<<"roles">>,[<<"_admin">>]}]}}, - {<<"admins">>,{[{<<"roles">>,[<<"_admin">>]}]}}]; + [ + {<<"members">>, {[{<<"roles">>, [<<"_admin">>]}]}}, + {<<"admins">>, {[{<<"roles">>, [<<"_admin">>]}]}} + ]; "everyone" -> [] end. @@ -873,12 +945,13 @@ default_security_object(_DbName) -> % Storage Engine) code lands this should be moved to the #db{} record. update_idle_limit_from_config() -> Default = integer_to_list(?IDLE_LIMIT_DEFAULT), - IdleLimit = case config:get("couchdb", "idle_check_timeout", Default) of - "infinity" -> - infinity; - Milliseconds -> - list_to_integer(Milliseconds) - end, + IdleLimit = + case config:get("couchdb", "idle_check_timeout", Default) of + "infinity" -> + infinity; + Milliseconds -> + list_to_integer(Milliseconds) + end, put(idle_limit, IdleLimit), IdleLimit. @@ -893,11 +966,9 @@ hibernate_if_no_idle_limit() -> Timeout end. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). - update_local_doc_revs_test_() -> {inparallel, [ {"Test local doc with valid rev", fun t_good_local_doc/0}, @@ -905,7 +976,6 @@ update_local_doc_revs_test_() -> {"Test deleted local doc", fun t_dead_local_doc/0} ]}. - t_good_local_doc() -> Doc = #doc{ id = <<"_local/alice">>, @@ -915,23 +985,23 @@ t_good_local_doc() -> [NewDoc] = update_local_doc_revs([{self(), Doc}]), ?assertEqual({0, [2]}, NewDoc#doc.revs), {ok, Result} = receive_result(Doc), - ?assertEqual({ok,{0,<<"2">>}}, Result). - + ?assertEqual({ok, {0, <<"2">>}}, Result). t_bad_local_doc() -> - lists:foreach(fun(BadRevs) -> - Doc = #doc{ - id = <<"_local/alice">>, - revs = BadRevs, - meta = [{ref, make_ref()}] - }, - NewDocs = update_local_doc_revs([{self(), Doc}]), - ?assertEqual([], NewDocs), - {ok, Result} = receive_result(Doc), - ?assertEqual({error,<<"Invalid rev format">>}, Result) - end, [{0, [<<"a">>]}, {1, [<<"1">>]}]). - - + lists:foreach( + fun(BadRevs) -> + Doc = #doc{ + id = <<"_local/alice">>, + revs = BadRevs, + meta = [{ref, make_ref()}] + }, + NewDocs = update_local_doc_revs([{self(), Doc}]), + ?assertEqual([], NewDocs), + {ok, Result} = receive_result(Doc), + ?assertEqual({error, <<"Invalid rev format">>}, Result) + end, + [{0, [<<"a">>]}, {1, [<<"1">>]}] + ). t_dead_local_doc() -> Doc = #doc{ @@ -943,8 +1013,7 @@ t_dead_local_doc() -> [NewDoc] = update_local_doc_revs([{self(), Doc}]), ?assertEqual({0, [0]}, NewDoc#doc.revs), {ok, Result} = receive_result(Doc), - ?assertEqual({ok,{0,<<"0">>}}, Result). - + ?assertEqual({ok, {0, <<"0">>}}, Result). receive_result(#doc{meta = Meta}) -> Ref = couch_util:get_value(ref, Meta), diff --git a/src/couch/src/couch_debug.erl b/src/couch/src/couch_debug.erl index 290d095bf..a2f4cdc87 100644 --- a/src/couch/src/couch_debug.erl +++ b/src/couch/src/couch_debug.erl @@ -49,6 +49,7 @@ help() -> ]. -spec help(Function :: atom()) -> ok. +%% erlfmt-ignore help(opened_files) -> io:format(" opened_files() @@ -205,9 +206,11 @@ help(Unknown) -> [{port(), CouchFilePid :: pid(), Fd :: pid() | tuple(), FilePath :: string()}]. opened_files() -> - Info = [couch_file_port_info(Port) - || Port <- erlang:ports(), - {name, "efile"} =:= erlang:port_info(Port, name)], + Info = [ + couch_file_port_info(Port) + || Port <- erlang:ports(), + {name, "efile"} =:= erlang:port_info(Port, name) + ], [I || I <- Info, is_tuple(I)]. couch_file_port_info(Port) -> @@ -223,17 +226,22 @@ couch_file_port_info(Port) -> [{port(), CouchFilePid :: pid(), Fd :: pid() | tuple(), FilePath :: string()}]. opened_files_by_regexp(FileRegExp) -> {ok, RegExp} = re:compile(FileRegExp), - lists:filter(fun({_Port, _Pid, _Fd, Path}) -> - re:run(Path, RegExp) =/= nomatch - end, couch_debug:opened_files()). + lists:filter( + fun({_Port, _Pid, _Fd, Path}) -> + re:run(Path, RegExp) =/= nomatch + end, + couch_debug:opened_files() + ). -spec opened_files_contains(FileNameFragment :: iodata()) -> [{port(), CouchFilePid :: pid(), Fd :: pid() | tuple(), FilePath :: string()}]. opened_files_contains(FileNameFragment) -> - lists:filter(fun({_Port, _Pid, _Fd, Path}) -> - string:str(Path, FileNameFragment) > 0 - end, couch_debug:opened_files()). - + lists:filter( + fun({_Port, _Pid, _Fd, Path}) -> + string:str(Path, FileNameFragment) > 0 + end, + couch_debug:opened_files() + ). process_name(Pid) when is_pid(Pid) -> Info = process_info(Pid, [registered_name, dictionary, initial_call]), @@ -260,7 +268,8 @@ link_tree(RootPid, Info) -> link_tree(RootPid, Info, Fun) -> {_, Result} = link_tree( - RootPid, [links | Info], gb_trees:empty(), 0, [RootPid], Fun), + RootPid, [links | Info], gb_trees:empty(), 0, [RootPid], Fun + ), Result. link_tree(RootPid, Info, Visited0, Pos, [Pid | Rest], Fun) -> @@ -272,21 +281,23 @@ link_tree(RootPid, Info, Visited0, Pos, [Pid | Rest], Fun) -> Visited1 = gb_trees:insert(Pid, Props, Visited0), {links, Children} = lists:keyfind(links, 1, Props), {Visited2, NewTree} = link_tree( - RootPid, Info, Visited1, Pos + 1, Children, Fun), + RootPid, Info, Visited1, Pos + 1, Children, Fun + ), {Visited3, Result} = link_tree( - RootPid, Info, Visited2, Pos, Rest, Fun), - {Visited3, [{Pos, {Pid, Fun(Pid, Props), NewTree}}] ++ Result}; + RootPid, Info, Visited2, Pos, Rest, Fun + ), + {Visited3, [{Pos, {Pid, Fun(Pid, Props), NewTree}}] ++ Result}; none -> Props = info(Pid, Info), Visited1 = gb_trees:insert(Pid, Props, Visited0), {Visited2, Result} = link_tree( - RootPid, Info, Visited1, Pos, Rest, Fun), + RootPid, Info, Visited1, Pos, Rest, Fun + ), {Visited2, [{Pos, {Pid, Fun(Pid, Props), []}}] ++ Result} end; link_tree(_RootPid, _Info, Visited, _Pos, [], _Fun) -> {Visited, []}. - info(Pid, Info) when is_pid(Pid) -> ValidProps = [ backtrace, @@ -340,12 +351,16 @@ info(Port, Info) when is_port(Port) -> port_info(Port, lists:usort(Validated)). port_info(Port, Items) -> - lists:foldl(fun(Item, Acc) -> - case (catch erlang:port_info(Port, Item)) of - {Item, _Value} = Info -> [Info | Acc]; - _Else -> Acc - end - end, [], Items). + lists:foldl( + fun(Item, Acc) -> + case (catch erlang:port_info(Port, Item)) of + {Item, _Value} = Info -> [Info | Acc]; + _Else -> Acc + end + end, + [], + Items + ). mapfold_tree([], Acc, _Fun) -> {[], Acc}; @@ -380,8 +395,10 @@ print_linked_processes(Name) when is_atom(Name) -> print_linked_processes(Pid) when is_pid(Pid) -> Info = [reductions, message_queue_len, memory], TableSpec = [ - {50, left, name}, {12, centre, reductions}, - {19, centre, message_queue_len}, {10, centre, memory} + {50, left, name}, + {12, centre, reductions}, + {19, centre, message_queue_len}, + {10, centre, memory} ], Tree = linked_processes_info(Pid, Info), print_tree(Tree, TableSpec). @@ -390,9 +407,12 @@ id("couch_file:init" ++ _, Pid, _Props) -> case couch_file:process_info(Pid) of {{file_descriptor, prim_file, {Port, Fd}}, FilePath} -> term2str([ - term2str(Fd), ":", - term2str(Port), ":", - shorten_path(FilePath)]); + term2str(Fd), + ":", + term2str(Port), + ":", + shorten_path(FilePath) + ]); undefined -> "" end; @@ -402,8 +422,11 @@ id(_IdStr, _Pid, _Props) -> print_couch_index_server_processes() -> Info = [reductions, message_queue_len, memory], TableSpec = [ - {50, left, name}, {12, centre, reductions}, - {19, centre, message_queue_len}, {14, centre, memory}, {id} + {50, left, name}, + {12, centre, reductions}, + {19, centre, message_queue_len}, + {14, centre, memory}, + {id} ], Tree = link_tree(whereis(couch_index_server), Info, fun(P, Props) -> @@ -476,31 +499,40 @@ random_processes(Pids, 0) -> random_processes(Acc, Depth) -> Caller = self(), Ref = make_ref(), - Pid = case oneof([spawn_link, open_port]) of - spawn_monitor -> - {P, _} = spawn_monitor(fun() -> - Caller ! {Ref, random_processes(Depth - 1)}, - receive looper -> ok end - end), - P; - spawn -> - spawn(fun() -> - Caller ! {Ref, random_processes(Depth - 1)}, - receive looper -> ok end - end); - spawn_link -> - spawn_link(fun() -> - Caller ! {Ref, random_processes(Depth - 1)}, - receive looper -> ok end - end); - open_port -> - spawn_link(fun() -> - Port = erlang:open_port({spawn, "sleep 10"}, []), - true = erlang:link(Port), - Caller ! {Ref, random_processes(Depth - 1)}, - receive looper -> ok end - end) - end, + Pid = + case oneof([spawn_link, open_port]) of + spawn_monitor -> + {P, _} = spawn_monitor(fun() -> + Caller ! {Ref, random_processes(Depth - 1)}, + receive + looper -> ok + end + end), + P; + spawn -> + spawn(fun() -> + Caller ! {Ref, random_processes(Depth - 1)}, + receive + looper -> ok + end + end); + spawn_link -> + spawn_link(fun() -> + Caller ! {Ref, random_processes(Depth - 1)}, + receive + looper -> ok + end + end); + open_port -> + spawn_link(fun() -> + Port = erlang:open_port({spawn, "sleep 10"}, []), + true = erlang:link(Port), + Caller ! {Ref, random_processes(Depth - 1)}, + receive + looper -> ok + end + end) + end, receive {Ref, Pids} -> random_processes([Pid | Pids] ++ Acc, Depth - 1) end. @@ -508,7 +540,6 @@ random_processes(Acc, Depth) -> oneof(Options) -> lists:nth(couch_rand:uniform(length(Options)), Options). - tree() -> [InitialPid | _] = Processes = random_processes(5), {InitialPid, Processes, link_tree(InitialPid)}. @@ -524,7 +555,8 @@ link_tree_test_() -> "link_tree tests", { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_have_same_shape/1, fun should_include_extra_info/1 @@ -534,16 +566,16 @@ link_tree_test_() -> should_have_same_shape({InitialPid, _Processes, Tree}) -> ?_test(begin - InfoTree = linked_processes_info(InitialPid, []), - ?assert(is_equal(InfoTree, Tree)), - ok + InfoTree = linked_processes_info(InitialPid, []), + ?assert(is_equal(InfoTree, Tree)), + ok end). should_include_extra_info({InitialPid, _Processes, _Tree}) -> Info = [reductions, message_queue_len, memory], ?_test(begin - InfoTree = linked_processes_info(InitialPid, Info), - map_tree(InfoTree, fun(Key, {_Id, Props}, _Pos) -> + InfoTree = linked_processes_info(InitialPid, Info), + map_tree(InfoTree, fun(Key, {_Id, Props}, _Pos) -> case Key of Pid when is_pid(Pid) -> ?assert(lists:keymember(reductions, 1, Props)), @@ -553,11 +585,12 @@ should_include_extra_info({InitialPid, _Processes, _Tree}) -> ok end, Props - end), - ok + end), + ok end). -is_equal([], []) -> true; +is_equal([], []) -> + true; is_equal([{Pos, {Pid, _, A}} | RestA], [{Pos, {Pid, _, B}} | RestB]) -> case is_equal(RestA, RestB) of false -> false; diff --git a/src/couch/src/couch_doc.erl b/src/couch/src/couch_doc.erl index ec16d21db..5d44e456c 100644 --- a/src/couch/src/couch_doc.erl +++ b/src/couch/src/couch_doc.erl @@ -12,7 +12,7 @@ -module(couch_doc). --export([to_doc_info/1,to_doc_info_path/1,parse_rev/1,parse_revs/1,rev_to_str/1,revs_to_strs/1]). +-export([to_doc_info/1, to_doc_info_path/1, parse_rev/1, parse_revs/1, rev_to_str/1, revs_to_strs/1]). -export([from_json_obj/1, from_json_obj_validate/1]). -export([from_json_obj/2, from_json_obj_validate/2]). -export([to_json_obj/2, has_stubs/1, merge_stubs/2]). @@ -26,15 +26,14 @@ -export([with_ejson_body/1]). -export([is_deleted/1]). - -include_lib("couch/include/couch_db.hrl"). -spec to_path(#doc{}) -> path(). -to_path(#doc{revs={Start, RevIds}}=Doc) -> +to_path(#doc{revs = {Start, RevIds}} = Doc) -> [Branch] = to_branch(Doc, lists:reverse(RevIds)), {Start - length(RevIds) + 1, Branch}. --spec to_branch(#doc{}, [RevId::binary()]) -> [branch()]. +-spec to_branch(#doc{}, [RevId :: binary()]) -> [branch()]. to_branch(Doc, [RevId]) -> [{RevId, Doc, []}]; to_branch(Doc, [RevId | Rest]) -> @@ -43,8 +42,8 @@ to_branch(Doc, [RevId | Rest]) -> % helpers used by to_json_obj to_json_rev(0, []) -> []; -to_json_rev(Start, [FirstRevId|_]) -> - [{<<"_rev">>, ?l2b([integer_to_list(Start),"-",revid_to_str(FirstRevId)])}]. +to_json_rev(Start, [FirstRevId | _]) -> + [{<<"_rev">>, ?l2b([integer_to_list(Start), "-", revid_to_str(FirstRevId)])}]. to_json_body(true, {Body}) -> Body ++ [{<<"_deleted">>, true}]; @@ -52,53 +51,69 @@ to_json_body(false, {Body}) -> Body. to_json_revisions(Options, Start, RevIds0) -> - RevIds = case proplists:get_value(revs, Options) of + RevIds = + case proplists:get_value(revs, Options) of + true -> + RevIds0; + Num when is_integer(Num), Num > 0 -> + lists:sublist(RevIds0, Num); + _ -> + [] + end, + if + RevIds == [] -> + []; true -> - RevIds0; - Num when is_integer(Num), Num > 0 -> - lists:sublist(RevIds0, Num); - _ -> - [] - end, - if RevIds == [] -> []; true -> - [{<<"_revisions">>, {[{<<"start">>, Start}, - {<<"ids">>, [revid_to_str(R) ||R <- RevIds]}]}}] + [ + {<<"_revisions">>, + {[ + {<<"start">>, Start}, + {<<"ids">>, [revid_to_str(R) || R <- RevIds]} + ]}} + ] end. - revid_to_str(RevId) when size(RevId) =:= 16 -> ?l2b(couch_util:to_hex(RevId)); revid_to_str(RevId) -> RevId. rev_to_str({Pos, RevId}) -> - ?l2b([integer_to_list(Pos),"-",revid_to_str(RevId)]). - + ?l2b([integer_to_list(Pos), "-", revid_to_str(RevId)]). revs_to_strs([]) -> []; -revs_to_strs([{Pos, RevId}| Rest]) -> +revs_to_strs([{Pos, RevId} | Rest]) -> [rev_to_str({Pos, RevId}) | revs_to_strs(Rest)]. to_json_meta(Meta) -> lists:flatmap( - fun({revs_info, Start, RevsInfo}) -> - {JsonRevsInfo, _Pos} = lists:mapfoldl( - fun({RevId, Status}, PosAcc) -> - JsonObj = {[{<<"rev">>, rev_to_str({PosAcc, RevId})}, - {<<"status">>, ?l2b(atom_to_list(Status))}]}, - {JsonObj, PosAcc - 1} - end, Start, RevsInfo), - [{<<"_revs_info">>, JsonRevsInfo}]; - ({local_seq, Seq}) -> - [{<<"_local_seq">>, Seq}]; - ({conflicts, Conflicts}) -> - [{<<"_conflicts">>, revs_to_strs(Conflicts)}]; - ({deleted_conflicts, DConflicts}) -> - [{<<"_deleted_conflicts">>, revs_to_strs(DConflicts)}]; - (_) -> - [] - end, Meta). + fun + ({revs_info, Start, RevsInfo}) -> + {JsonRevsInfo, _Pos} = lists:mapfoldl( + fun({RevId, Status}, PosAcc) -> + JsonObj = + {[ + {<<"rev">>, rev_to_str({PosAcc, RevId})}, + {<<"status">>, ?l2b(atom_to_list(Status))} + ]}, + {JsonObj, PosAcc - 1} + end, + Start, + RevsInfo + ), + [{<<"_revs_info">>, JsonRevsInfo}]; + ({local_seq, Seq}) -> + [{<<"_local_seq">>, Seq}]; + ({conflicts, Conflicts}) -> + [{<<"_conflicts">>, revs_to_strs(Conflicts)}]; + ({deleted_conflicts, DConflicts}) -> + [{<<"_deleted_conflicts">>, revs_to_strs(DConflicts)}]; + (_) -> + [] + end, + Meta + ). to_json_attachments(Attachments, Options) -> to_json_attachments( @@ -117,14 +132,23 @@ to_json_attachments(Atts, OutputData, Follows, ShowEnc) -> to_json_obj(Doc, Options) -> doc_to_json_obj(with_ejson_body(Doc), Options). -doc_to_json_obj(#doc{id=Id,deleted=Del,body=Body,revs={Start, RevIds}, - meta=Meta}=Doc,Options)-> - {[{<<"_id">>, Id}] - ++ to_json_rev(Start, RevIds) - ++ to_json_body(Del, Body) - ++ to_json_revisions(Options, Start, RevIds) - ++ to_json_meta(Meta) - ++ to_json_attachments(Doc#doc.atts, Options) +doc_to_json_obj( + #doc{ + id = Id, + deleted = Del, + body = Body, + revs = {Start, RevIds}, + meta = Meta + } = Doc, + Options +) -> + { + [{<<"_id">>, Id}] ++ + to_json_rev(Start, RevIds) ++ + to_json_body(Del, Body) ++ + to_json_revisions(Options, Start, RevIds) ++ + to_json_meta(Meta) ++ + to_json_attachments(Doc#doc.atts, Options) }. from_json_obj_validate(EJson) -> @@ -135,48 +159,54 @@ from_json_obj_validate(EJson, DbName) -> Doc = from_json_obj(EJson, DbName), case couch_ejson_size:encoded_size(Doc#doc.body) =< MaxSize of true -> - validate_attachment_sizes(Doc#doc.atts), - Doc; + validate_attachment_sizes(Doc#doc.atts), + Doc; false -> throw({request_entity_too_large, Doc#doc.id}) end. - validate_attachment_sizes([]) -> ok; validate_attachment_sizes(Atts) -> MaxAttSize = couch_att:max_attachment_size(), - lists:foreach(fun(Att) -> - AttName = couch_att:fetch(name, Att), - AttSize = couch_att:fetch(att_len, Att), - couch_att:validate_attachment_size(AttName, AttSize, MaxAttSize) - end, Atts). - + lists:foreach( + fun(Att) -> + AttName = couch_att:fetch(name, Att), + AttSize = couch_att:fetch(att_len, Att), + couch_att:validate_attachment_size(AttName, AttSize, MaxAttSize) + end, + Atts + ). from_json_obj({Props}) -> from_json_obj({Props}, undefined). from_json_obj({Props}, DbName) -> - transfer_fields(Props, #doc{body=[]}, DbName); + transfer_fields(Props, #doc{body = []}, DbName); from_json_obj(_Other, _) -> throw({bad_request, "Document must be a JSON object"}). parse_revid(RevId) when size(RevId) =:= 32 -> RevInt = erlang:list_to_integer(?b2l(RevId), 16), - <<RevInt:128>>; + <<RevInt:128>>; parse_revid(RevId) when length(RevId) =:= 32 -> RevInt = erlang:list_to_integer(RevId, 16), - <<RevInt:128>>; + <<RevInt:128>>; parse_revid(RevId) when is_binary(RevId) -> RevId; parse_revid(RevId) when is_list(RevId) -> ?l2b(RevId). - parse_rev(Rev) when is_binary(Rev) -> parse_rev(?b2l(Rev)); parse_rev(Rev) when is_list(Rev) -> - SplitRev = lists:splitwith(fun($-) -> false; (_) -> true end, Rev), + SplitRev = lists:splitwith( + fun + ($-) -> false; + (_) -> true + end, + Rev + ), case SplitRev of {Pos, [$- | RevId]} -> try @@ -185,7 +215,8 @@ parse_rev(Rev) when is_list(Rev) -> catch error:badarg -> throw({bad_request, <<"Invalid rev format">>}) end; - _Else -> throw({bad_request, <<"Invalid rev format">>}) + _Else -> + throw({bad_request, <<"Invalid rev format">>}) end; parse_rev(_BadRev) -> throw({bad_request, <<"Invalid rev format">>}). @@ -197,10 +228,11 @@ parse_revs([Rev | Rest]) -> parse_revs(_) -> throw({bad_request, "Invalid list of revisions"}). - validate_docid(DocId, DbName) -> - case DbName =:= ?l2b(config:get("mem3", "shards_db", "_dbs")) andalso - couch_db:is_system_db_name(DocId) of + case + DbName =:= ?l2b(config:get("mem3", "shards_db", "_dbs")) andalso + couch_db:is_system_db_name(DocId) + of true -> ok; false -> @@ -214,10 +246,11 @@ validate_docid(<<"_design/">>) -> validate_docid(<<"_local/">>) -> throw({illegal_docid, <<"Illegal document id `_local/`">>}); validate_docid(Id) when is_binary(Id) -> - MaxLen = case config:get("couchdb", "max_document_id_length", "infinity") of - "infinity" -> infinity; - IntegerVal -> list_to_integer(IntegerVal) - end, + MaxLen = + case config:get("couchdb", "max_document_id_length", "infinity") of + "infinity" -> infinity; + IntegerVal -> list_to_integer(IntegerVal) + end, case MaxLen > 0 andalso byte_size(Id) > MaxLen of true -> throw({illegal_docid, <<"Document id is too long">>}); false -> ok @@ -227,69 +260,72 @@ validate_docid(Id) when is_binary(Id) -> true -> ok end, case Id of - <<"_design/", _/binary>> -> ok; - <<"_local/", _/binary>> -> ok; - <<"_", _/binary>> -> - case couch_db_plugin:validate_docid(Id) of - true -> - ok; - false -> - throw( - {illegal_docid, - <<"Only reserved document ids may start with underscore.">>}) - end; - _Else -> ok + <<"_design/", _/binary>> -> + ok; + <<"_local/", _/binary>> -> + ok; + <<"_", _/binary>> -> + case couch_db_plugin:validate_docid(Id) of + true -> + ok; + false -> + throw( + {illegal_docid, <<"Only reserved document ids may start with underscore.">>} + ) + end; + _Else -> + ok end; validate_docid(Id) -> couch_log:debug("Document id is not a string: ~p", [Id]), throw({illegal_docid, <<"Document id must be a string">>}). -transfer_fields([], #doc{body=Fields}=Doc, _) -> +transfer_fields([], #doc{body = Fields} = Doc, _) -> % convert fields back to json object - Doc#doc{body={lists:reverse(Fields)}}; - + Doc#doc{body = {lists:reverse(Fields)}}; transfer_fields([{<<"_id">>, Id} | Rest], Doc, DbName) -> validate_docid(Id, DbName), - transfer_fields(Rest, Doc#doc{id=Id}, DbName); - -transfer_fields([{<<"_rev">>, Rev} | Rest], #doc{revs={0, []}}=Doc, DbName) -> + transfer_fields(Rest, Doc#doc{id = Id}, DbName); +transfer_fields([{<<"_rev">>, Rev} | Rest], #doc{revs = {0, []}} = Doc, DbName) -> {Pos, RevId} = parse_rev(Rev), - transfer_fields(Rest, - Doc#doc{revs={Pos, [RevId]}}, DbName); - + transfer_fields( + Rest, + Doc#doc{revs = {Pos, [RevId]}}, + DbName + ); transfer_fields([{<<"_rev">>, _Rev} | Rest], Doc, DbName) -> % we already got the rev from the _revisions transfer_fields(Rest, Doc, DbName); - transfer_fields([{<<"_attachments">>, {JsonBins}} | Rest], Doc, DbName) -> Atts = [couch_att:from_json(Name, Props) || {Name, {Props}} <- JsonBins], - transfer_fields(Rest, Doc#doc{atts=Atts}, DbName); - + transfer_fields(Rest, Doc#doc{atts = Atts}, DbName); transfer_fields([{<<"_revisions">>, {Props}} | Rest], Doc, DbName) -> RevIds = couch_util:get_value(<<"ids">>, Props), Start = couch_util:get_value(<<"start">>, Props), - if not is_integer(Start) -> - throw({doc_validation, "_revisions.start isn't an integer."}); - not is_list(RevIds) -> - throw({doc_validation, "_revisions.ids isn't a array."}); - true -> - ok + if + not is_integer(Start) -> + throw({doc_validation, "_revisions.start isn't an integer."}); + not is_list(RevIds) -> + throw({doc_validation, "_revisions.ids isn't a array."}); + true -> + ok end, - RevIds2 = lists:map(fun(RevId) -> - try - parse_revid(RevId) - catch - error:function_clause -> - throw({doc_validation, "RevId isn't a string"}); - error:badarg -> - throw({doc_validation, "RevId isn't a valid hexadecimal"}) - end - end, RevIds), - transfer_fields(Rest, Doc#doc{revs={Start, RevIds2}}, DbName); - + RevIds2 = lists:map( + fun(RevId) -> + try + parse_revid(RevId) + catch + error:function_clause -> + throw({doc_validation, "RevId isn't a string"}); + error:badarg -> + throw({doc_validation, "RevId isn't a valid hexadecimal"}) + end + end, + RevIds + ), + transfer_fields(Rest, Doc#doc{revs = {Start, RevIds2}}, DbName); transfer_fields([{<<"_deleted">>, B} | Rest], Doc, DbName) when is_boolean(B) -> - transfer_fields(Rest, Doc#doc{deleted=B}, DbName); - + transfer_fields(Rest, Doc#doc{deleted = B}, DbName); % ignored fields transfer_fields([{<<"_revs_info">>, _} | Rest], Doc, DbName) -> transfer_fields(Rest, Doc, DbName); @@ -299,36 +335,49 @@ transfer_fields([{<<"_conflicts">>, _} | Rest], Doc, DbName) -> transfer_fields(Rest, Doc, DbName); transfer_fields([{<<"_deleted_conflicts">>, _} | Rest], Doc, DbName) -> transfer_fields(Rest, Doc, DbName); - % special field for per doc access control, for future compatibility -transfer_fields([{<<"_access">>, _} = Field | Rest], - #doc{body=Fields} = Doc, DbName) -> - transfer_fields(Rest, Doc#doc{body=[Field|Fields]}, DbName); - +transfer_fields( + [{<<"_access">>, _} = Field | Rest], + #doc{body = Fields} = Doc, + DbName +) -> + transfer_fields(Rest, Doc#doc{body = [Field | Fields]}, DbName); % special fields for replication documents -transfer_fields([{<<"_replication_state">>, _} = Field | Rest], - #doc{body=Fields} = Doc, DbName) -> - transfer_fields(Rest, Doc#doc{body=[Field|Fields]}, DbName); -transfer_fields([{<<"_replication_state_time">>, _} = Field | Rest], - #doc{body=Fields} = Doc, DbName) -> - transfer_fields(Rest, Doc#doc{body=[Field|Fields]}, DbName); -transfer_fields([{<<"_replication_state_reason">>, _} = Field | Rest], - #doc{body=Fields} = Doc, DbName) -> - transfer_fields(Rest, Doc#doc{body=[Field|Fields]}, DbName); -transfer_fields([{<<"_replication_id">>, _} = Field | Rest], - #doc{body=Fields} = Doc, DbName) -> - transfer_fields(Rest, Doc#doc{body=[Field|Fields]}, DbName); -transfer_fields([{<<"_replication_stats">>, _} = Field | Rest], - #doc{body=Fields} = Doc, DbName) -> - transfer_fields(Rest, Doc#doc{body=[Field|Fields]}, DbName); - +transfer_fields( + [{<<"_replication_state">>, _} = Field | Rest], + #doc{body = Fields} = Doc, + DbName +) -> + transfer_fields(Rest, Doc#doc{body = [Field | Fields]}, DbName); +transfer_fields( + [{<<"_replication_state_time">>, _} = Field | Rest], + #doc{body = Fields} = Doc, + DbName +) -> + transfer_fields(Rest, Doc#doc{body = [Field | Fields]}, DbName); +transfer_fields( + [{<<"_replication_state_reason">>, _} = Field | Rest], + #doc{body = Fields} = Doc, + DbName +) -> + transfer_fields(Rest, Doc#doc{body = [Field | Fields]}, DbName); +transfer_fields( + [{<<"_replication_id">>, _} = Field | Rest], + #doc{body = Fields} = Doc, + DbName +) -> + transfer_fields(Rest, Doc#doc{body = [Field | Fields]}, DbName); +transfer_fields( + [{<<"_replication_stats">>, _} = Field | Rest], + #doc{body = Fields} = Doc, + DbName +) -> + transfer_fields(Rest, Doc#doc{body = [Field | Fields]}, DbName); % unknown special field -transfer_fields([{<<"_",Name/binary>>, _} | _], _, _) -> - throw({doc_validation, - ?l2b(io_lib:format("Bad special document member: _~s", [Name]))}); - -transfer_fields([Field | Rest], #doc{body=Fields}=Doc, DbName) -> - transfer_fields(Rest, Doc#doc{body=[Field|Fields]}, DbName). +transfer_fields([{<<"_", Name/binary>>, _} | _], _, _) -> + throw({doc_validation, ?l2b(io_lib:format("Bad special document member: _~s", [Name]))}); +transfer_fields([Field | Rest], #doc{body = Fields} = Doc, DbName) -> + transfer_fields(Rest, Doc#doc{body = [Field | Fields]}, DbName). to_doc_info(FullDocInfo) -> {DocInfo, _Path} = to_doc_info_path(FullDocInfo), @@ -340,10 +389,11 @@ max_seq(Tree, UpdateSeq) -> {_Deleted, _DiskPos, OldTreeSeq} -> % Older versions didn't track data sizes. erlang:max(MaxOldSeq, OldTreeSeq); - {_Deleted, _DiskPos, OldTreeSeq, _Size} -> % necessary clause? + % necessary clause? + {_Deleted, _DiskPos, OldTreeSeq, _Size} -> % Older versions didn't store #leaf records. erlang:max(MaxOldSeq, OldTreeSeq); - #leaf{seq=OldTreeSeq} -> + #leaf{seq = OldTreeSeq} -> erlang:max(MaxOldSeq, OldTreeSeq); _ -> MaxOldSeq @@ -351,20 +401,25 @@ max_seq(Tree, UpdateSeq) -> end, couch_key_tree:fold(FoldFun, UpdateSeq, Tree). -to_doc_info_path(#full_doc_info{id=Id,rev_tree=Tree,update_seq=FDISeq}) -> +to_doc_info_path(#full_doc_info{id = Id, rev_tree = Tree, update_seq = FDISeq}) -> RevInfosAndPath = [ - {rev_info(Node), Path} || {_Leaf, Path} = Node <- + {rev_info(Node), Path} + || {_Leaf, Path} = Node <- couch_key_tree:get_all_leafs(Tree) ], SortedRevInfosAndPath = lists:sort( - fun({#rev_info{deleted=DeletedA,rev=RevA}, _PathA}, - {#rev_info{deleted=DeletedB,rev=RevB}, _PathB}) -> + fun( + {#rev_info{deleted = DeletedA, rev = RevA}, _PathA}, + {#rev_info{deleted = DeletedB, rev = RevB}, _PathB} + ) -> % sort descending by {not deleted, rev} {not DeletedA, RevA} > {not DeletedB, RevB} - end, RevInfosAndPath), - [{_RevInfo, WinPath}|_] = SortedRevInfosAndPath, + end, + RevInfosAndPath + ), + [{_RevInfo, WinPath} | _] = SortedRevInfosAndPath, RevInfos = [RevInfo || {RevInfo, _Path} <- SortedRevInfosAndPath], - {#doc_info{id=Id, high_seq=max_seq(Tree, FDISeq), revs=RevInfos}, WinPath}. + {#doc_info{id = Id, high_seq = max_seq(Tree, FDISeq), revs = RevInfos}, WinPath}. rev_info({#leaf{} = Leaf, {Pos, [RevId | _]}}) -> #rev_info{ @@ -381,53 +436,56 @@ rev_info({#doc{} = Doc, {Pos, [RevId | _]}}) -> rev = {Pos, RevId} }. -is_deleted(#full_doc_info{rev_tree=Tree}) -> +is_deleted(#full_doc_info{rev_tree = Tree}) -> is_deleted(Tree); is_deleted(Tree) -> Leafs = couch_key_tree:get_all_leafs(Tree), try - lists:foldl(fun - ({#leaf{deleted=false},_}, _) -> - throw(not_deleted); - ({#doc{deleted=false},_}, _) -> - throw(not_deleted); - (_, Acc) -> - Acc - end, nil, Leafs), + lists:foldl( + fun + ({#leaf{deleted = false}, _}, _) -> + throw(not_deleted); + ({#doc{deleted = false}, _}, _) -> + throw(not_deleted); + (_, Acc) -> + Acc + end, + nil, + Leafs + ), true - catch throw:not_deleted -> - false + catch + throw:not_deleted -> + false end. - get_validate_doc_fun({Props}) -> get_validate_doc_fun(couch_doc:from_json_obj({Props})); -get_validate_doc_fun(#doc{body={Props}}=DDoc) -> +get_validate_doc_fun(#doc{body = {Props}} = DDoc) -> case couch_util:get_value(<<"validate_doc_update">>, Props) of - undefined -> - nil; - _Else -> - fun(EditDoc, DiskDoc, Ctx, SecObj) -> - couch_query_servers:validate_doc_update(DDoc, EditDoc, DiskDoc, Ctx, SecObj) - end + undefined -> + nil; + _Else -> + fun(EditDoc, DiskDoc, Ctx, SecObj) -> + couch_query_servers:validate_doc_update(DDoc, EditDoc, DiskDoc, Ctx, SecObj) + end end. - -has_stubs(#doc{atts=Atts}) -> +has_stubs(#doc{atts = Atts}) -> lists:any(fun couch_att:is_stub/1, Atts); has_stubs(Atts) -> lists:any(fun couch_att:is_stub/1, Atts). merge_stubs(#doc{id = Id}, nil) -> throw({missing_stub, <<"Previous revision missing for document ", Id/binary>>}); -merge_stubs(#doc{id=Id,atts=MemBins}=StubsDoc, #doc{atts=DiskBins}) -> +merge_stubs(#doc{id = Id, atts = MemBins} = StubsDoc, #doc{atts = DiskBins}) -> case couch_att:merge_stubs(MemBins, DiskBins) of {ok, MergedBins} -> StubsDoc#doc{atts = MergedBins}; {missing, Name} -> - throw({missing_stub, - <<"Invalid attachment stub in ", Id/binary, " for ", Name/binary>> - }) + throw( + {missing_stub, <<"Invalid attachment stub in ", Id/binary, " for ", Name/binary>>} + ) end. len_doc_to_multi_part_stream(Boundary, JsonBytes, Atts, SendEncodedAtts) -> @@ -435,28 +493,38 @@ len_doc_to_multi_part_stream(Boundary, JsonBytes, Atts, SendEncodedAtts) -> AttsDecoded = decode_attributes(AttsToInclude, SendEncodedAtts), couch_httpd_multipart:length_multipart_stream(Boundary, JsonBytes, AttsDecoded). - -doc_to_multi_part_stream(Boundary, JsonBytes, Atts, WriteFun, - SendEncodedAtts) -> - AttsToInclude = lists:filter(fun(Att)-> couch_att:fetch(data, Att) /= stub end, Atts), +doc_to_multi_part_stream( + Boundary, + JsonBytes, + Atts, + WriteFun, + SendEncodedAtts +) -> + AttsToInclude = lists:filter(fun(Att) -> couch_att:fetch(data, Att) /= stub end, Atts), AttsDecoded = decode_attributes(AttsToInclude, SendEncodedAtts), - AttFun = case SendEncodedAtts of - false -> fun couch_att:foldl_decode/3; - true -> fun couch_att:foldl/3 - end, + AttFun = + case SendEncodedAtts of + false -> fun couch_att:foldl_decode/3; + true -> fun couch_att:foldl/3 + end, couch_httpd_multipart:encode_multipart_stream( - Boundary, JsonBytes, AttsDecoded, WriteFun, AttFun). + Boundary, JsonBytes, AttsDecoded, WriteFun, AttFun + ). decode_attributes(Atts, SendEncodedAtts) -> - lists:map(fun(Att) -> - [Name, AttLen, DiskLen, Type, Encoding] = - couch_att:fetch([name, att_len, disk_len, type, encoding], Att), - Len = case SendEncodedAtts of - true -> AttLen; - false -> DiskLen - end, - {Att, Name, Len, Type, Encoding} - end, Atts). + lists:map( + fun(Att) -> + [Name, AttLen, DiskLen, Type, Encoding] = + couch_att:fetch([name, att_len, disk_len, type, encoding], Att), + Len = + case SendEncodedAtts of + true -> AttLen; + false -> DiskLen + end, + {Att, Name, Len, Type, Encoding} + end, + Atts + ). doc_from_multi_part_stream(ContentType, DataFun) -> doc_from_multi_part_stream(ContentType, DataFun, make_ref()). @@ -466,25 +534,32 @@ doc_from_multi_part_stream(ContentType, DataFun, Ref) -> doc_from_multi_part_stream(ContentType, DataFun, Ref, ValidateDocLimits) -> case couch_httpd_multipart:decode_multipart_stream(ContentType, DataFun, Ref) of - {{started_open_doc_revs, NewRef}, Parser, _ParserRef} -> - restart_open_doc_revs(Parser, Ref, NewRef); - {{doc_bytes, Ref, DocBytes}, Parser, ParserRef} -> - Doc = case ValidateDocLimits of - true -> - from_json_obj_validate(?JSON_DECODE(DocBytes)); - false -> - from_json_obj(?JSON_DECODE(DocBytes)) - end, - erlang:put(mochiweb_request_recv, true), - % we'll send the Parser process ID to the remote nodes so they can - % retrieve their own copies of the attachment data - WithParser = fun(follows) -> {follows, Parser, Ref}; (D) -> D end, - Atts = [couch_att:transform(data, WithParser, A) || A <- Doc#doc.atts], - WaitFun = fun() -> - receive {'DOWN', ParserRef, _, _, _} -> ok end - end, - {ok, Doc#doc{atts=Atts}, WaitFun, Parser}; - ok -> ok + {{started_open_doc_revs, NewRef}, Parser, _ParserRef} -> + restart_open_doc_revs(Parser, Ref, NewRef); + {{doc_bytes, Ref, DocBytes}, Parser, ParserRef} -> + Doc = + case ValidateDocLimits of + true -> + from_json_obj_validate(?JSON_DECODE(DocBytes)); + false -> + from_json_obj(?JSON_DECODE(DocBytes)) + end, + erlang:put(mochiweb_request_recv, true), + % we'll send the Parser process ID to the remote nodes so they can + % retrieve their own copies of the attachment data + WithParser = fun + (follows) -> {follows, Parser, Ref}; + (D) -> D + end, + Atts = [couch_att:transform(data, WithParser, A) || A <- Doc#doc.atts], + WaitFun = fun() -> + receive + {'DOWN', ParserRef, _, _, _} -> ok + end + end, + {ok, Doc#doc{atts = Atts}, WaitFun, Parser}; + ok -> + ok end. restart_open_doc_revs(Parser, Ref, NewRef) -> @@ -493,7 +568,6 @@ restart_open_doc_revs(Parser, Ref, NewRef) -> flush_parser_messages(Ref), erlang:error({restart_open_doc_revs, NewRef}). - flush_parser_messages(Ref) -> receive {headers, Ref, _} -> @@ -508,7 +582,6 @@ flush_parser_messages(Ref) -> ok end. - with_ejson_body(#doc{body = Body} = Doc) when is_binary(Body) -> Doc#doc{body = couch_compress:decompress(Body)}; with_ejson_body(#doc{body = {_}} = Doc) -> diff --git a/src/couch/src/couch_ejson_compare.erl b/src/couch/src/couch_ejson_compare.erl index b02b9ba7c..628bc2fab 100644 --- a/src/couch/src/couch_ejson_compare.erl +++ b/src/couch/src/couch_ejson_compare.erl @@ -27,10 +27,8 @@ compare_strings_nif/2 ]). - -on_load(init/0). - init() -> NumScheds = erlang:system_info(schedulers), Dir = code:priv_dir(couch), @@ -39,65 +37,54 @@ init() -> % partitioned row comparison less({p, PA, A}, {p, PB, B}) -> less([PA, A], [PB, B]); - less(A, B) -> try less_nif(A, B) catch - error:max_depth_error -> - % The EJSON structure is too deep, fallback to Erlang land. - less_erl(A, B) + error:max_depth_error -> + % The EJSON structure is too deep, fallback to Erlang land. + less_erl(A, B) end. less_json_ids({JsonA, IdA}, {JsonB, IdB}) -> case less(JsonA, JsonB) of - 0 -> - IdA < IdB; - Result -> - Result < 0 + 0 -> + IdA < IdB; + Result -> + Result < 0 end. -less_json(A,B) -> +less_json(A, B) -> less(A, B) < 0. - get_icu_version() -> erlang:nif_error(get_icu_version). - get_uca_version() -> erlang:nif_error(get_uca_version). - less_nif(A, B) -> erlang:nif_error(less_nif_load_error, [A, B]). - compare_strings_nif(A, B) -> erlang:nif_error(compare_string_nif, [A, B]). - -less_erl(A,A) -> 0; - -less_erl(A,B) when is_atom(A), is_atom(B) -> atom_sort(A) - atom_sort(B); -less_erl(A,_) when is_atom(A) -> -1; -less_erl(_,B) when is_atom(B) -> 1; - -less_erl(A,B) when is_number(A), is_number(B) -> A - B; -less_erl(A,_) when is_number(A) -> -1; -less_erl(_,B) when is_number(B) -> 1; - -less_erl(A,B) when is_binary(A), is_binary(B) -> compare_strings_nif(A,B); -less_erl(A,_) when is_binary(A) -> -1; -less_erl(_,B) when is_binary(B) -> 1; - -less_erl(A,B) when is_list(A), is_list(B) -> less_list(A,B); -less_erl(A,_) when is_list(A) -> -1; -less_erl(_,B) when is_list(B) -> 1; - -less_erl({A},{B}) when is_list(A), is_list(B) -> less_props(A,B); -less_erl({A},_) when is_list(A) -> -1; -less_erl(_,{B}) when is_list(B) -> 1. +less_erl(A, A) -> 0; +less_erl(A, B) when is_atom(A), is_atom(B) -> atom_sort(A) - atom_sort(B); +less_erl(A, _) when is_atom(A) -> -1; +less_erl(_, B) when is_atom(B) -> 1; +less_erl(A, B) when is_number(A), is_number(B) -> A - B; +less_erl(A, _) when is_number(A) -> -1; +less_erl(_, B) when is_number(B) -> 1; +less_erl(A, B) when is_binary(A), is_binary(B) -> compare_strings_nif(A, B); +less_erl(A, _) when is_binary(A) -> -1; +less_erl(_, B) when is_binary(B) -> 1; +less_erl(A, B) when is_list(A), is_list(B) -> less_list(A, B); +less_erl(A, _) when is_list(A) -> -1; +less_erl(_, B) when is_list(B) -> 1; +less_erl({A}, {B}) when is_list(A), is_list(B) -> less_props(A, B); +less_erl({A}, _) when is_list(A) -> -1; +less_erl(_, {B}) when is_list(B) -> 1. atom_sort(null) -> 1; atom_sort(false) -> 2; @@ -105,33 +92,33 @@ atom_sort(true) -> 3. less_props([], []) -> 0; -less_props([], [_|_]) -> +less_props([], [_ | _]) -> -1; less_props(_, []) -> 1; -less_props([{AKey, AValue}|RestA], [{BKey, BValue}|RestB]) -> +less_props([{AKey, AValue} | RestA], [{BKey, BValue} | RestB]) -> case compare_strings_nif(AKey, BKey) of - 0 -> - case less_erl(AValue, BValue) of 0 -> - less_props(RestA, RestB); + case less_erl(AValue, BValue) of + 0 -> + less_props(RestA, RestB); + Result -> + Result + end; Result -> Result - end; - Result -> - Result end. less_list([], []) -> 0; -less_list([], [_|_]) -> +less_list([], [_ | _]) -> -1; less_list(_, []) -> 1; -less_list([A|RestA], [B|RestB]) -> - case less_erl(A,B) of - 0 -> - less_list(RestA, RestB); - Result -> - Result +less_list([A | RestA], [B | RestB]) -> + case less_erl(A, B) of + 0 -> + less_list(RestA, RestB); + Result -> + Result end. diff --git a/src/couch/src/couch_ejson_size.erl b/src/couch/src/couch_ejson_size.erl index f5505680f..54a7094ff 100644 --- a/src/couch/src/couch_ejson_size.erl +++ b/src/couch/src/couch_ejson_size.erl @@ -14,85 +14,78 @@ -export([encoded_size/1]). - %% Compound objects encoded_size({[]}) -> - 2; % opening { and closing } - + % opening { and closing } + 2; encoded_size({KVs}) -> % Would add 2 because opening { and closing }, but then inside the LC % would accumulate an extra , at the end so subtract 2 - 1 - 1 + lists:sum([encoded_size(K) + encoded_size(V) + 2 || {K,V} <- KVs]); - + 1 + lists:sum([encoded_size(K) + encoded_size(V) + 2 || {K, V} <- KVs]); encoded_size([]) -> - 2; % opening [ and closing ] - + % opening [ and closing ] + 2; encoded_size(List) when is_list(List) -> % 2 is for [ and ] but inside LC would accumulate an extra , so subtract % 2 - 1 1 + lists:sum([encoded_size(V) + 1 || V <- List]); - %% Floats. encoded_size(0.0) -> 3; - encoded_size(1.0) -> 3; - encoded_size(Float) when is_float(Float), Float < 0.0 -> encoded_size(-Float) + 1; - encoded_size(Float) when is_float(Float), Float < 1.0 -> if - Float =< 1.0e-300 -> 3; % close enough to 0.0 - Float =< 1.0e-100 -> 6; % Xe-YYY - Float =< 1.0e-10 -> 5; % Xe-YY - Float =< 0.01 -> 4; % Xe-Y, 0.0X - true -> 3 % 0.X + % close enough to 0.0 + Float =< 1.0e-300 -> 3; + % Xe-YYY + Float =< 1.0e-100 -> 6; + % Xe-YY + Float =< 1.0e-10 -> 5; + % Xe-Y, 0.0X + Float =< 0.01 -> 4; + % 0.X + true -> 3 end; - encoded_size(Float) when is_float(Float) -> if - Float >= 1.0e100 -> 5; % XeYYY - Float >= 1.0e10 -> 4; % XeYY - true -> 3 % XeY, X.Y + % XeYYY + Float >= 1.0e100 -> 5; + % XeYY + Float >= 1.0e10 -> 4; + % XeY, X.Y + true -> 3 end; - %% Integers encoded_size(0) -> 1; - encoded_size(Integer) when is_integer(Integer), Integer < 0 -> encoded_size(-Integer) + 1; - encoded_size(Integer) when is_integer(Integer) -> if - Integer < 10 -> 1; - Integer < 100 -> 2; - Integer < 1000 -> 3; + Integer < 10 -> 1; + Integer < 100 -> 2; + Integer < 1000 -> 3; Integer < 10000 -> 4; - true -> trunc(math:log10(Integer)) + 1 + true -> trunc(math:log10(Integer)) + 1 end; - %% Strings encoded_size(Binary) when is_binary(Binary) -> 2 + byte_size(Binary); - %% Special terminal symbols as atoms encoded_size(null) -> 4; - encoded_size(true) -> 4; - encoded_size(false) -> 5; - %% Other atoms encoded_size(Atom) when is_atom(Atom) -> diff --git a/src/couch/src/couch_emsort.erl b/src/couch/src/couch_emsort.erl index 430d94e01..9dcc08d67 100644 --- a/src/couch/src/couch_emsort.erl +++ b/src/couch/src/couch_emsort.erl @@ -142,36 +142,30 @@ num_bb = 0 }). - -define(REPORT_INTERVAL, 1000). - open(Fd) -> - {ok, #ems{fd=Fd}}. - + {ok, #ems{fd = Fd}}. open(Fd, Options) -> - {ok, set_options(#ems{fd=Fd}, Options)}. - + {ok, set_options(#ems{fd = Fd}, Options)}. set_options(Ems, []) -> Ems; set_options(Ems, [{root, Root} | Rest]) -> - set_options(Ems#ems{root=Root}, Rest); + set_options(Ems#ems{root = Root}, Rest); set_options(Ems, [{chain_chunk, Count} | Rest]) when is_integer(Count) -> - set_options(Ems#ems{chain_chunk=Count}, Rest); + set_options(Ems#ems{chain_chunk = Count}, Rest); set_options(Ems, [{back_bone_chunk, Count} | Rest]) when is_integer(Count) -> - set_options(Ems#ems{bb_chunk=Count}, Rest); + set_options(Ems#ems{bb_chunk = Count}, Rest); set_options(Ems, [{num_kvs, NumKVs} | Rest]) when is_integer(NumKVs) -> - set_options(Ems#ems{num_kvs=NumKVs}, Rest); + set_options(Ems#ems{num_kvs = NumKVs}, Rest); set_options(Ems, [{num_bb, NumBB} | Rest]) when is_integer(NumBB) -> - set_options(Ems#ems{num_bb=NumBB}, Rest). + set_options(Ems#ems{num_bb = NumBB}, Rest). - -get_fd(#ems{fd=Fd}) -> +get_fd(#ems{fd = Fd}) -> Fd. - get_state(#ems{} = Ems) -> #ems{ root = Root, @@ -184,7 +178,6 @@ get_state(#ems{} = Ems) -> {num_bb, NumBB} ]. - add(Ems, []) -> {ok, Ems}; add(Ems, KVs) -> @@ -195,69 +188,64 @@ add(Ems, KVs) -> num_bb = Ems#ems.num_bb + 1 }}. - -sort(#ems{}=Ems) -> +sort(#ems{} = Ems) -> {ok, Ems1} = merge(Ems), iter(Ems1). - merge(Ems) -> merge(Ems, fun(_) -> ok end). - -merge(#ems{root=undefined}=Ems, _Reporter) -> +merge(#ems{root = undefined} = Ems, _Reporter) -> {ok, Ems}; -merge(#ems{}=Ems, Reporter) -> +merge(#ems{} = Ems, Reporter) -> {ok, decimate(Ems, Reporter)}. - -iter(#ems{root=undefined}=Ems) -> +iter(#ems{root = undefined} = Ems) -> {ok, {Ems, []}}; -iter(#ems{root={BB, nil}}=Ems) -> +iter(#ems{root = {BB, nil}} = Ems) -> Chains = init_chains(Ems, small, BB), {ok, {Ems, Chains}}; -iter(#ems{root={_, _}}) -> +iter(#ems{root = {_, _}}) -> {error, not_merged}. - next({_Ems, []}) -> finished; next({Ems, Chains}) -> {KV, RestChains} = choose_kv(small, Ems, Chains), {ok, KV, {Ems, RestChains}}. - -num_kvs(#ems{num_kvs=NumKVs}) -> +num_kvs(#ems{num_kvs = NumKVs}) -> NumKVs. -num_merges(#ems{bb_chunk=BBChunk, num_bb=NumBB}) -> +num_merges(#ems{bb_chunk = BBChunk, num_bb = NumBB}) -> num_merges(BBChunk, NumBB). - -add_bb_pos(#ems{root=undefined}=Ems, Pos) -> - Ems#ems{root={[Pos], nil}}; -add_bb_pos(#ems{root={BB, Prev}}=Ems, Pos) -> +add_bb_pos(#ems{root = undefined} = Ems, Pos) -> + Ems#ems{root = {[Pos], nil}}; +add_bb_pos(#ems{root = {BB, Prev}} = Ems, Pos) -> {NewBB, NewPrev} = append_item(Ems, {BB, Prev}, Pos, Ems#ems.bb_chunk), - Ems#ems{root={NewBB, NewPrev}}. - + Ems#ems{root = {NewBB, NewPrev}}. write_kvs(Ems, KVs) -> % Write the list of KV's to disk in sorted order in chunks % of 100. Also make sure that the order is so that they % can be streamed in asscending order. {LastKVs, LastPos} = - lists:foldr(fun(KV, Acc) -> - append_item(Ems, Acc, KV, Ems#ems.chain_chunk) - end, {[], nil}, lists:sort(KVs)), + lists:foldr( + fun(KV, Acc) -> + append_item(Ems, Acc, KV, Ems#ems.chain_chunk) + end, + {[], nil}, + lists:sort(KVs) + ), {ok, Final, _} = couch_file:append_term(Ems#ems.fd, {LastKVs, LastPos}), Final. - -decimate(#ems{root={_BB, nil}}=Ems, _Reporter) -> +decimate(#ems{root = {_BB, nil}} = Ems, _Reporter) -> % We have less than bb_chunk backbone pointers so we're % good to start streaming KV's back to the client. Ems; -decimate(#ems{root={BB, NextBB}}=Ems, Reporter) -> +decimate(#ems{root = {BB, NextBB}} = Ems, Reporter) -> % To make sure we have a bounded amount of data in RAM % at any given point we first need to decimate the data % by performing the first couple iterations of a merge @@ -273,15 +261,13 @@ decimate(#ems{root={BB, NextBB}}=Ems, Reporter) -> % Continue deicmating until we have an acceptable bound on % the number of keys to use. - decimate(Ems#ems{root={FwdBB, FwdNextBB}}, Reporter). - + decimate(Ems#ems{root = {FwdBB, FwdNextBB}}, Reporter). merge_back_bone(Ems, Choose, BB, NextBB, Reporter) -> BBPos = merge_chains(Ems, Choose, BB, Reporter), Reporter(length(BB)), merge_rest_back_bone(Ems, Choose, NextBB, {[BBPos], nil}, Reporter). - merge_rest_back_bone(_Ems, _Choose, nil, Acc, _Reporter) -> Acc; merge_rest_back_bone(Ems, Choose, BBPos, Acc, Reporter) -> @@ -290,40 +276,39 @@ merge_rest_back_bone(Ems, Choose, BBPos, Acc, Reporter) -> {NewBB, NewPrev} = append_item(Ems, Acc, NewPos, Ems#ems.bb_chunk), merge_rest_back_bone(Ems, Choose, NextBB, {NewBB, NewPrev}, Reporter). - merge_chains(Ems, Choose, BB, Reporter) -> Chains = init_chains(Ems, Choose, BB), merge_chains(Ems, Choose, Chains, {[], nil}, Reporter, 0). - merge_chains(Ems, _Choose, [], ChainAcc, _Reporter, _Count) -> {ok, CPos, _} = couch_file:append_term(Ems#ems.fd, ChainAcc), CPos; -merge_chains(#ems{chain_chunk=CC}=Ems, Choose, Chains, Acc, Reporter, Count0) -> +merge_chains(#ems{chain_chunk = CC} = Ems, Choose, Chains, Acc, Reporter, Count0) -> {KV, RestChains} = choose_kv(Choose, Ems, Chains), {NewKVs, NewPrev} = append_item(Ems, Acc, KV, CC), - Count1 = case (Count0 + 1) rem ?REPORT_INTERVAL of - 0 -> - Reporter(Count0), - 0; - _ -> - Count0 + 1 - end, + Count1 = + case (Count0 + 1) rem ?REPORT_INTERVAL of + 0 -> + Reporter(Count0), + 0; + _ -> + Count0 + 1 + end, merge_chains(Ems, Choose, RestChains, {NewKVs, NewPrev}, Reporter, Count1). - init_chains(Ems, Choose, BB) -> - Chains = lists:map(fun(CPos) -> - {ok, {KVs, NextKVs}} = couch_file:pread_term(Ems#ems.fd, CPos), - {KVs, NextKVs} - end, BB), + Chains = lists:map( + fun(CPos) -> + {ok, {KVs, NextKVs}} = couch_file:pread_term(Ems#ems.fd, CPos), + {KVs, NextKVs} + end, + BB + ), order_chains(Choose, Chains). - order_chains(small, Chains) -> lists:sort(Chains); order_chains(big, Chains) -> lists:reverse(lists:sort(Chains)). - choose_kv(_Choose, _Ems, [{[KV], nil} | Rest]) -> {KV, Rest}; choose_kv(Choose, Ems, [{[KV], Pos} | RestChains]) -> @@ -338,26 +323,22 @@ choose_kv(Choose, _Ems, [{[KV | RestKVs], Prev} | RestChains]) -> big -> {KV, ins_big_chain(RestChains, {RestKVs, Prev}, [])} end. - -ins_small_chain([{[{K1,_}|_],_}=C1|Rest], {[{K2,_}|_],_}=C2, Acc) when K1<K2 -> +ins_small_chain([{[{K1, _} | _], _} = C1 | Rest], {[{K2, _} | _], _} = C2, Acc) when K1 < K2 -> ins_small_chain(Rest, C2, [C1 | Acc]); ins_small_chain(Rest, Chain, Acc) -> lists:reverse(Acc, [Chain | Rest]). - -ins_big_chain([{[{K1,_}|_],_}=C1|Rest], {[{K2,_}|_],_}=C2, Acc) when K1>K2 -> +ins_big_chain([{[{K1, _} | _], _} = C1 | Rest], {[{K2, _} | _], _} = C2, Acc) when K1 > K2 -> ins_big_chain(Rest, C2, [C1 | Acc]); ins_big_chain(Rest, Chain, Acc) -> lists:reverse(Acc, [Chain | Rest]). - append_item(Ems, {List, Prev}, Pos, Size) when length(List) >= Size -> {ok, PrevList, _} = couch_file:append_term(Ems#ems.fd, {List, Prev}), {[Pos], PrevList}; append_item(_Ems, {List, Prev}, Pos, _Size) -> {[Pos | List], Prev}. - num_merges(BBChunk, NumBB) when NumBB =< BBChunk -> 0; num_merges(BBChunk, NumBB) when NumBB > BBChunk -> diff --git a/src/couch/src/couch_event_sup.erl b/src/couch/src/couch_event_sup.erl index 32f1b9b68..e9fc2e5db 100644 --- a/src/couch/src/couch_event_sup.erl +++ b/src/couch/src/couch_event_sup.erl @@ -20,8 +20,8 @@ -include_lib("couch/include/couch_db.hrl"). --export([start_link/3,start_link/4, stop/1]). --export([init/1, terminate/2, handle_call/3, handle_cast/2, handle_info/2,code_change/3]). +-export([start_link/3, start_link/4, stop/1]). +-export([init/1, terminate/2, handle_call/3, handle_cast/2, handle_info/2, code_change/3]). % % Instead calling the @@ -52,10 +52,10 @@ stop(Pid) -> init({EventMgr, EventHandler, Args}) -> case gen_event:add_sup_handler(EventMgr, EventHandler, Args) of - ok -> - {ok, {EventMgr, EventHandler}}; - {stop, Error} -> - {stop, Error} + ok -> + {ok, {EventMgr, EventHandler}}; + {stop, Error} -> + {stop, Error} end. terminate(_Reason, _State) -> diff --git a/src/couch/src/couch_file.erl b/src/couch/src/couch_file.erl index 2948d685b..0e786525f 100644 --- a/src/couch/src/couch_file.erl +++ b/src/couch/src/couch_file.erl @@ -16,10 +16,10 @@ -include_lib("couch/include/couch_db.hrl"). - -define(INITIAL_WAIT, 60000). -define(MONITOR_CHECK, 10000). --define(SIZE_BLOCK, 16#1000). % 4 KiB +% 4 KiB +-define(SIZE_BLOCK, 16#1000). -define(IS_OLD_STATE(S), is_pid(S#file.db_monitor)). -define(PREFIX_SIZE, 5). -define(DEFAULT_READ_COUNT, 1024). @@ -66,38 +66,48 @@ open(Filepath) -> open(Filepath, []). open(Filepath, Options) -> - case gen_server:start_link(couch_file, - {Filepath, Options, self(), Ref = make_ref()}, []) of - {ok, Fd} -> - {ok, Fd}; - ignore -> - % get the error - receive - {Ref, Pid, {error, Reason} = Error} -> - case process_info(self(), trap_exit) of - {trap_exit, true} -> receive {'EXIT', Pid, _} -> ok end; - {trap_exit, false} -> ok - end, - case {lists:member(nologifmissing, Options), Reason} of - {true, enoent} -> ok; - _ -> - couch_log:error("Could not open file ~s: ~s", - [Filepath, file:format_error(Reason)]) - end, + case + gen_server:start_link( + couch_file, + {Filepath, Options, self(), Ref = make_ref()}, + [] + ) + of + {ok, Fd} -> + {ok, Fd}; + ignore -> + % get the error + receive + {Ref, Pid, {error, Reason} = Error} -> + case process_info(self(), trap_exit) of + {trap_exit, true} -> + receive + {'EXIT', Pid, _} -> ok + end; + {trap_exit, false} -> + ok + end, + case {lists:member(nologifmissing, Options), Reason} of + {true, enoent} -> + ok; + _ -> + couch_log:error( + "Could not open file ~s: ~s", + [Filepath, file:format_error(Reason)] + ) + end, + Error + end; + Error -> + % We can't say much here, because it could be any kind of error. + % Just let it bubble and an encapsulating subcomponent can perhaps + % be more informative. It will likely appear in the SASL log, anyway. Error - end; - Error -> - % We can't say much here, because it could be any kind of error. - % Just let it bubble and an encapsulating subcomponent can perhaps - % be more informative. It will likely appear in the SASL log, anyway. - Error end. - set_db_pid(Fd, Pid) -> gen_server:call(Fd, {set_db_pid, Pid}). - %%---------------------------------------------------------------------- %% Purpose: To append an Erlang term to the end of the file. %% Args: Erlang term to serialize and append to the file. @@ -121,7 +131,6 @@ append_term_md5(Fd, Term, Options) -> Comp = couch_util:get_value(compression, Options, ?DEFAULT_COMPRESSION), append_binary_md5(Fd, couch_compress:compress(Term, Comp)). - %%---------------------------------------------------------------------- %% Purpose: To append an Erlang binary to the end of the file. %% Args: Erlang term to serialize and append to the file. @@ -134,14 +143,15 @@ append_binary(Fd, Bin) -> ioq:call(Fd, {append_bin, assemble_file_chunk(Bin)}, erlang:get(io_priority)). append_binary_md5(Fd, Bin) -> - ioq:call(Fd, + ioq:call( + Fd, {append_bin, assemble_file_chunk(Bin, couch_hash:md5_hash(Bin))}, - erlang:get(io_priority)). + erlang:get(io_priority) + ). append_raw_chunk(Fd, Chunk) -> ioq:call(Fd, {append_bin, Chunk}, erlang:get(io_priority)). - assemble_file_chunk(Bin) -> [<<0:1/integer, (iolist_size(Bin)):31/integer>>, Bin]. @@ -155,12 +165,10 @@ assemble_file_chunk(Bin, Md5) -> %% or {error, Reason}. %%---------------------------------------------------------------------- - pread_term(Fd, Pos) -> {ok, Bin} = pread_binary(Fd, Pos), {ok, couch_compress:decompress(Bin)}. - %%---------------------------------------------------------------------- %% Purpose: Reads a binrary from a file that was written with append_binary %% Args: Pos, the offset into the file where the term is serialized. @@ -172,7 +180,6 @@ pread_binary(Fd, Pos) -> {ok, L} = pread_iolist(Fd, Pos), {ok, iolist_to_binary(L)}. - pread_iolist(Fd, Pos) -> case ioq:call(Fd, {pread_iolist, Pos}, erlang:get(io_priority)) of {ok, IoList, Md5} -> @@ -181,49 +188,52 @@ pread_iolist(Fd, Pos) -> Error end. - pread_terms(Fd, PosList) -> {ok, Bins} = pread_binaries(Fd, PosList), - Terms = lists:map(fun(Bin) -> - couch_compress:decompress(Bin) - end, Bins), + Terms = lists:map( + fun(Bin) -> + couch_compress:decompress(Bin) + end, + Bins + ), {ok, Terms}. - pread_binaries(Fd, PosList) -> {ok, Data} = pread_iolists(Fd, PosList), {ok, lists:map(fun erlang:iolist_to_binary/1, Data)}. - pread_iolists(Fd, PosList) -> case ioq:call(Fd, {pread_iolists, PosList}, erlang:get(io_priority)) of {ok, DataMd5s} -> - Data = lists:zipwith(fun(Pos, {IoList, Md5}) -> - verify_md5(Fd, Pos, IoList, Md5) - end, PosList, DataMd5s), + Data = lists:zipwith( + fun(Pos, {IoList, Md5}) -> + verify_md5(Fd, Pos, IoList, Md5) + end, + PosList, + DataMd5s + ), {ok, Data}; Error -> Error end. - append_terms(Fd, Terms) -> append_terms(Fd, Terms, []). - append_terms(Fd, Terms, Options) -> Comp = couch_util:get_value(compression, Options, ?DEFAULT_COMPRESSION), - Bins = lists:map(fun(Term) -> - couch_compress:compress(Term, Comp) - end, Terms), + Bins = lists:map( + fun(Term) -> + couch_compress:compress(Term, Comp) + end, + Terms + ), append_binaries(Fd, Bins). - append_binaries(Fd, Bins) -> WriteBins = lists:map(fun assemble_file_chunk/1, Bins), ioq:call(Fd, {append_bins, WriteBins}, erlang:get(io_priority)). - %%---------------------------------------------------------------------- %% Purpose: The length of a file, in bytes. %% Returns: {ok, Bytes} @@ -280,36 +290,42 @@ sync(Fd) -> close(Fd) -> gen_server:call(Fd, close, infinity). - delete(RootDir, Filepath) -> delete(RootDir, Filepath, []). delete(RootDir, FullFilePath, Options) -> - EnableRecovery = config:get_boolean("couchdb", - "enable_database_recovery", false), + EnableRecovery = config:get_boolean( + "couchdb", + "enable_database_recovery", + false + ), Async = not lists:member(sync, Options), Context = couch_util:get_value(context, Options, compaction), case Context =:= delete andalso EnableRecovery of true -> rename_file(FullFilePath); false -> - DeleteAfterRename = config:get_boolean("couchdb", - "delete_after_rename", true), + DeleteAfterRename = config:get_boolean( + "couchdb", + "delete_after_rename", + true + ), delete_file(RootDir, FullFilePath, Async, DeleteAfterRename) end. delete_file(RootDir, Filepath, Async, DeleteAfterRename) -> - DelFile = filename:join([RootDir,".delete", ?b2l(couch_uuids:random())]), + DelFile = filename:join([RootDir, ".delete", ?b2l(couch_uuids:random())]), case file:rename(Filepath, DelFile) of - ok when DeleteAfterRename -> - if (Async) -> - spawn(file, delete, [DelFile]), - ok; - true -> - file:delete(DelFile) - end; - Else -> - Else + ok when DeleteAfterRename -> + if + (Async) -> + spawn(file, delete, [DelFile]), + ok; + true -> + file:delete(DelFile) + end; + Else -> + Else end. rename_file(Original) -> @@ -323,14 +339,21 @@ rename_file(Original) -> deleted_filename(Original) -> {{Y, Mon, D}, {H, Min, S}} = calendar:universal_time(), Suffix = lists:flatten( - io_lib:format(".~w~2.10.0B~2.10.0B." - ++ "~2.10.0B~2.10.0B~2.10.0B.deleted" - ++ filename:extension(Original), [Y, Mon, D, H, Min, S])), + io_lib:format( + ".~w~2.10.0B~2.10.0B." ++ + "~2.10.0B~2.10.0B~2.10.0B.deleted" ++ + filename:extension(Original), + [Y, Mon, D, H, Min, S] + ) + ), filename:rootname(Original) ++ Suffix. nuke_dir(RootDelDir, Dir) -> - EnableRecovery = config:get_boolean("couchdb", - "enable_database_recovery", false), + EnableRecovery = config:get_boolean( + "couchdb", + "enable_database_recovery", + false + ), case EnableRecovery of true -> rename_file(Dir); @@ -339,8 +362,11 @@ nuke_dir(RootDelDir, Dir) -> end. delete_dir(RootDelDir, Dir) -> - DeleteAfterRename = config:get_boolean("couchdb", - "delete_after_rename", true), + DeleteAfterRename = config:get_boolean( + "couchdb", + "delete_after_rename", + true + ), FoldFun = fun(File) -> Path = Dir ++ "/" ++ File, case filelib:is_dir(Path) of @@ -359,27 +385,30 @@ delete_dir(RootDelDir, Dir) -> ok end. - init_delete_dir(RootDir) -> - Dir = filename:join(RootDir,".delete"), + Dir = filename:join(RootDir, ".delete"), % note: ensure_dir requires an actual filename companent, which is the % reason for "foo". - filelib:ensure_dir(filename:join(Dir,"foo")), + filelib:ensure_dir(filename:join(Dir, "foo")), spawn(fun() -> - filelib:fold_files(Dir, ".*", true, + filelib:fold_files( + Dir, + ".*", + true, fun(Filename, _) -> ok = file:delete(Filename) - end, ok) + end, + ok + ) end), ok. - read_header(Fd) -> case ioq:call(Fd, find_header, erlang:get(io_priority)) of - {ok, Bin} -> - {ok, binary_to_term(Bin)}; - Else -> - Else + {ok, Bin} -> + {ok, binary_to_term(Bin)}; + Else -> + Else end. write_header(Fd, Data) -> @@ -389,17 +418,14 @@ write_header(Fd, Data) -> FinalBin = <<Md5/binary, Bin/binary>>, ioq:call(Fd, {write_header, FinalBin}, erlang:get(io_priority)). - init_status_error(ReturnPid, Ref, Error) -> ReturnPid ! {Ref, self(), Error}, ignore. - last_read(Fd) when is_pid(Fd) -> Now = os:timestamp(), couch_util:process_dict_get(Fd, read_timestamp, Now). - % server functions init({Filepath, Options, ReturnPid, Ref}) -> @@ -408,66 +434,67 @@ init({Filepath, Options, ReturnPid, Ref}) -> IsSys = lists:member(sys_db, Options), update_read_timestamp(), case lists:member(create, Options) of - true -> - filelib:ensure_dir(Filepath), - case file:open(Filepath, OpenOptions) of - {ok, Fd} -> - %% Save Fd in process dictionary for debugging purposes - put(couch_file_fd, {Fd, Filepath}), - {ok, Length} = file:position(Fd, eof), - case Length > 0 of - true -> - % this means the file already exists and has data. - % FYI: We don't differentiate between empty files and non-existant - % files here. - case lists:member(overwrite, Options) of - true -> - {ok, 0} = file:position(Fd, 0), - ok = file:truncate(Fd), - ok = file:sync(Fd), - maybe_track_open_os_files(Options), - erlang:send_after(?INITIAL_WAIT, self(), maybe_close), - {ok, #file{fd=Fd, is_sys=IsSys, pread_limit=Limit}}; - false -> - ok = file:close(Fd), - init_status_error(ReturnPid, Ref, {error, eexist}) - end; - false -> - maybe_track_open_os_files(Options), - erlang:send_after(?INITIAL_WAIT, self(), maybe_close), - {ok, #file{fd=Fd, is_sys=IsSys, pread_limit=Limit}} - end; - Error -> - init_status_error(ReturnPid, Ref, Error) - end; - false -> - % open in read mode first, so we don't create the file if it doesn't exist. - case file:open(Filepath, [read, raw]) of - {ok, Fd_Read} -> + true -> + filelib:ensure_dir(Filepath), case file:open(Filepath, OpenOptions) of {ok, Fd} -> - %% Save Fd in process dictionary for debugging purposes - put(couch_file_fd, {Fd, Filepath}), - ok = file:close(Fd_Read), - maybe_track_open_os_files(Options), - {ok, Eof} = file:position(Fd, eof), - erlang:send_after(?INITIAL_WAIT, self(), maybe_close), - {ok, #file{fd=Fd, eof=Eof, is_sys=IsSys, pread_limit=Limit}}; - Error -> - init_status_error(ReturnPid, Ref, Error) + %% Save Fd in process dictionary for debugging purposes + put(couch_file_fd, {Fd, Filepath}), + {ok, Length} = file:position(Fd, eof), + case Length > 0 of + true -> + % this means the file already exists and has data. + % FYI: We don't differentiate between empty files and non-existant + % files here. + case lists:member(overwrite, Options) of + true -> + {ok, 0} = file:position(Fd, 0), + ok = file:truncate(Fd), + ok = file:sync(Fd), + maybe_track_open_os_files(Options), + erlang:send_after(?INITIAL_WAIT, self(), maybe_close), + {ok, #file{fd = Fd, is_sys = IsSys, pread_limit = Limit}}; + false -> + ok = file:close(Fd), + init_status_error(ReturnPid, Ref, {error, eexist}) + end; + false -> + maybe_track_open_os_files(Options), + erlang:send_after(?INITIAL_WAIT, self(), maybe_close), + {ok, #file{fd = Fd, is_sys = IsSys, pread_limit = Limit}} + end; + Error -> + init_status_error(ReturnPid, Ref, Error) end; - Error -> - init_status_error(ReturnPid, Ref, Error) - end + false -> + % open in read mode first, so we don't create the file if it doesn't exist. + case file:open(Filepath, [read, raw]) of + {ok, Fd_Read} -> + case file:open(Filepath, OpenOptions) of + {ok, Fd} -> + %% Save Fd in process dictionary for debugging purposes + put(couch_file_fd, {Fd, Filepath}), + ok = file:close(Fd_Read), + maybe_track_open_os_files(Options), + {ok, Eof} = file:position(Fd, eof), + erlang:send_after(?INITIAL_WAIT, self(), maybe_close), + {ok, #file{fd = Fd, eof = Eof, is_sys = IsSys, pread_limit = Limit}}; + Error -> + init_status_error(ReturnPid, Ref, Error) + end; + Error -> + init_status_error(ReturnPid, Ref, Error) + end end. file_open_options(Options) -> - [read, raw, binary] ++ case lists:member(read_only, Options) of - true -> - []; - false -> - [append] - end. + [read, raw, binary] ++ + case lists:member(read_only, Options) of + true -> + []; + false -> + [append] + end. maybe_track_open_os_files(Options) -> case not lists:member(sys_db, Options) of @@ -484,59 +511,62 @@ terminate(_Reason, #file{fd = Fd}) -> handle_call(Msg, From, File) when ?IS_OLD_STATE(File) -> handle_call(Msg, From, upgrade_state(File)); - -handle_call(close, _From, #file{fd=Fd}=File) -> +handle_call(close, _From, #file{fd = Fd} = File) -> {stop, normal, file:close(Fd), File#file{fd = nil}}; - handle_call({pread_iolist, Pos}, _From, File) -> update_read_timestamp(), {LenIolist, NextPos} = read_raw_iolist_int(File, Pos, 4), case iolist_to_binary(LenIolist) of - <<1:1/integer,Len:31/integer>> -> % an MD5-prefixed term - {Md5AndIoList, _} = read_raw_iolist_int(File, NextPos, Len+16), - {Md5, IoList} = extract_md5(Md5AndIoList), - {reply, {ok, IoList, Md5}, File}; - <<0:1/integer,Len:31/integer>> -> - {Iolist, _} = read_raw_iolist_int(File, NextPos, Len), - {reply, {ok, Iolist, <<>>}, File} + % an MD5-prefixed term + <<1:1/integer, Len:31/integer>> -> + {Md5AndIoList, _} = read_raw_iolist_int(File, NextPos, Len + 16), + {Md5, IoList} = extract_md5(Md5AndIoList), + {reply, {ok, IoList, Md5}, File}; + <<0:1/integer, Len:31/integer>> -> + {Iolist, _} = read_raw_iolist_int(File, NextPos, Len), + {reply, {ok, Iolist, <<>>}, File} end; - handle_call({pread_iolists, PosL}, _From, File) -> update_read_timestamp(), LocNums1 = [{Pos, 4} || Pos <- PosL], DataSizes = read_multi_raw_iolists_int(File, LocNums1), - LocNums2 = lists:map(fun({LenIoList, NextPos}) -> - case iolist_to_binary(LenIoList) of - <<1:1/integer, Len:31/integer>> -> % an MD5-prefixed term - {NextPos, Len + 16}; - <<0:1/integer, Len:31/integer>> -> - {NextPos, Len} - end - end, DataSizes), + LocNums2 = lists:map( + fun({LenIoList, NextPos}) -> + case iolist_to_binary(LenIoList) of + % an MD5-prefixed term + <<1:1/integer, Len:31/integer>> -> + {NextPos, Len + 16}; + <<0:1/integer, Len:31/integer>> -> + {NextPos, Len} + end + end, + DataSizes + ), Resps = read_multi_raw_iolists_int(File, LocNums2), - Extracted = lists:zipwith(fun({LenIoList, _}, {IoList, _}) -> - case iolist_to_binary(LenIoList) of - <<1:1/integer, _:31/integer>> -> - {Md5, IoList} = extract_md5(IoList), - {IoList, Md5}; - <<0:1/integer, _:31/integer>> -> - {IoList, <<>>} - end - end, DataSizes, Resps), + Extracted = lists:zipwith( + fun({LenIoList, _}, {IoList, _}) -> + case iolist_to_binary(LenIoList) of + <<1:1/integer, _:31/integer>> -> + {Md5, IoList} = extract_md5(IoList), + {IoList, Md5}; + <<0:1/integer, _:31/integer>> -> + {IoList, <<>>} + end + end, + DataSizes, + Resps + ), {reply, {ok, Extracted}, File}; - handle_call(bytes, _From, #file{fd = Fd} = File) -> {reply, file:position(Fd, eof), File}; - -handle_call({set_db_pid, Pid}, _From, #file{db_monitor=OldRef}=File) -> +handle_call({set_db_pid, Pid}, _From, #file{db_monitor = OldRef} = File) -> case is_reference(OldRef) of true -> demonitor(OldRef, [flush]); false -> ok end, Ref = monitor(process, Pid), - {reply, ok, File#file{db_monitor=Ref}}; - -handle_call(sync, _From, #file{fd=Fd}=File) -> + {reply, ok, File#file{db_monitor = Ref}}; +handle_call(sync, _From, #file{fd = Fd} = File) -> case file:sync(Fd) of ok -> {reply, ok, File}; @@ -547,68 +577,66 @@ handle_call(sync, _From, #file{fd=Fd}=File) -> % can't fathom. {stop, Error, Error, #file{fd = nil}} end; - -handle_call({truncate, Pos}, _From, #file{fd=Fd}=File) -> +handle_call({truncate, Pos}, _From, #file{fd = Fd} = File) -> {ok, Pos} = file:position(Fd, Pos), case file:truncate(Fd) of - ok -> - {reply, ok, File#file{eof = Pos}}; - Error -> - {reply, Error, File} + ok -> + {reply, ok, File#file{eof = Pos}}; + Error -> + {reply, Error, File} end; - handle_call({append_bin, Bin}, _From, #file{fd = Fd, eof = Pos} = File) -> Blocks = make_blocks(Pos rem ?SIZE_BLOCK, Bin), Size = iolist_size(Blocks), case file:write(Fd, Blocks) of - ok -> - {reply, {ok, Pos, Size}, File#file{eof = Pos + Size}}; - Error -> - {reply, Error, reset_eof(File)} + ok -> + {reply, {ok, Pos, Size}, File#file{eof = Pos + Size}}; + Error -> + {reply, Error, reset_eof(File)} end; - handle_call({append_bins, Bins}, _From, #file{fd = Fd, eof = Pos} = File) -> - {BlockResps, FinalPos} = lists:mapfoldl(fun(Bin, PosAcc) -> - Blocks = make_blocks(PosAcc rem ?SIZE_BLOCK, Bin), - Size = iolist_size(Blocks), - {{Blocks, {PosAcc, Size}}, PosAcc + Size} - end, Pos, Bins), + {BlockResps, FinalPos} = lists:mapfoldl( + fun(Bin, PosAcc) -> + Blocks = make_blocks(PosAcc rem ?SIZE_BLOCK, Bin), + Size = iolist_size(Blocks), + {{Blocks, {PosAcc, Size}}, PosAcc + Size} + end, + Pos, + Bins + ), {AllBlocks, Resps} = lists:unzip(BlockResps), case file:write(Fd, AllBlocks) of - ok -> - {reply, {ok, Resps}, File#file{eof = FinalPos}}; - Error -> - {reply, Error, reset_eof(File)} + ok -> + {reply, {ok, Resps}, File#file{eof = FinalPos}}; + Error -> + {reply, Error, reset_eof(File)} end; - handle_call({write_header, Bin}, _From, #file{fd = Fd, eof = Pos} = File) -> BinSize = byte_size(Bin), case Pos rem ?SIZE_BLOCK of - 0 -> - Padding = <<>>; - BlockOffset -> - Padding = <<0:(8*(?SIZE_BLOCK-BlockOffset))>> + 0 -> + Padding = <<>>; + BlockOffset -> + Padding = <<0:(8 * (?SIZE_BLOCK - BlockOffset))>> end, FinalBin = [Padding, <<1, BinSize:32/integer>> | make_blocks(5, [Bin])], case file:write(Fd, FinalBin) of - ok -> - {reply, ok, File#file{eof = Pos + iolist_size(FinalBin)}}; - Error -> - {reply, Error, reset_eof(File)} + ok -> + {reply, ok, File#file{eof = Pos + iolist_size(FinalBin)}}; + Error -> + {reply, Error, reset_eof(File)} end; - handle_call(find_header, _From, #file{fd = Fd, eof = Pos} = File) -> {reply, find_header(Fd, Pos div ?SIZE_BLOCK), File}. handle_cast(close, Fd) -> - {stop,normal,Fd}. + {stop, normal, Fd}. code_change(_OldVsn, State, _Extra) -> {ok, State}. handle_info(Msg, File) when ?IS_OLD_STATE(File) -> handle_info(Msg, upgrade_state(File)); - handle_info(maybe_close, File) -> case is_idle(File) of true -> @@ -617,8 +645,7 @@ handle_info(maybe_close, File) -> erlang:send_after(?MONITOR_CHECK, self(), maybe_close), {noreply, File} end; - -handle_info({'DOWN', Ref, process, _Pid, _Info}, #file{db_monitor=Ref}=File) -> +handle_info({'DOWN', Ref, process, _Pid, _Info}, #file{db_monitor = Ref} = File) -> case is_idle(File) of true -> {stop, normal, File}; false -> {noreply, File} @@ -630,12 +657,13 @@ format_status(_Opt, [PDict, #file{} = File]) -> find_header(Fd, Block) -> case (catch load_header(Fd, Block)) of - {ok, Bin} -> - {ok, Bin}; - _Error -> - ReadCount = config:get_integer( - "couchdb", "find_header_read_count", ?DEFAULT_READ_COUNT), - find_header(Fd, Block -1, ReadCount) + {ok, Bin} -> + {ok, Bin}; + _Error -> + ReadCount = config:get_integer( + "couchdb", "find_header_read_count", ?DEFAULT_READ_COUNT + ), + find_header(Fd, Block - 1, ReadCount) end. load_header(Fd, Block) -> @@ -648,22 +676,22 @@ load_header(Fd, Pos, HeaderLen) -> load_header(Fd, Pos, HeaderLen, RestBlock) -> TotalBytes = calculate_total_read_len(?PREFIX_SIZE, HeaderLen), - RawBin = case TotalBytes =< byte_size(RestBlock) of - true -> - <<RawBin0:TotalBytes/binary, _/binary>> = RestBlock, - RawBin0; - false -> - ReadStart = Pos + ?PREFIX_SIZE + byte_size(RestBlock), - ReadLen = TotalBytes - byte_size(RestBlock), - {ok, Missing} = file:pread(Fd, ReadStart, ReadLen), - <<RestBlock/binary, Missing/binary>> - end, + RawBin = + case TotalBytes =< byte_size(RestBlock) of + true -> + <<RawBin0:TotalBytes/binary, _/binary>> = RestBlock, + RawBin0; + false -> + ReadStart = Pos + ?PREFIX_SIZE + byte_size(RestBlock), + ReadLen = TotalBytes - byte_size(RestBlock), + {ok, Missing} = file:pread(Fd, ReadStart, ReadLen), + <<RestBlock/binary, Missing/binary>> + end, <<Md5Sig:16/binary, HeaderBin/binary>> = iolist_to_binary(remove_block_prefixes(?PREFIX_SIZE, RawBin)), Md5Sig = couch_hash:md5_hash(HeaderBin), {ok, HeaderBin}. - %% Read multiple block locations using a single file:pread/2. -spec find_header(file:fd(), block_id(), non_neg_integer()) -> {ok, binary()} | no_valid_header. @@ -671,23 +699,28 @@ find_header(_Fd, Block, _ReadCount) when Block < 0 -> no_valid_header; find_header(Fd, Block, ReadCount) -> FirstBlock = max(0, Block - ReadCount + 1), - BlockLocations = [?SIZE_BLOCK*B || B <- lists:seq(FirstBlock, Block)], + BlockLocations = [?SIZE_BLOCK * B || B <- lists:seq(FirstBlock, Block)], {ok, DataL} = file:pread(Fd, [{L, ?PREFIX_SIZE} || L <- BlockLocations]), %% Since BlockLocations are ordered from oldest to newest, we rely %% on lists:foldl/3 to reverse the order, making HeaderLocations %% correctly ordered from newest to oldest. - HeaderLocations = lists:foldl(fun - ({Loc, <<1, HeaderSize:32/integer>>}, Acc) -> - [{Loc, HeaderSize} | Acc]; - (_, Acc) -> - Acc - end, [], lists:zip(BlockLocations, DataL)), + HeaderLocations = lists:foldl( + fun + ({Loc, <<1, HeaderSize:32/integer>>}, Acc) -> + [{Loc, HeaderSize} | Acc]; + (_, Acc) -> + Acc + end, + [], + lists:zip(BlockLocations, DataL) + ), case find_newest_header(Fd, HeaderLocations) of {ok, _Location, HeaderBin} -> {ok, HeaderBin}; _ -> ok = file:advise( - Fd, hd(BlockLocations), ReadCount * ?SIZE_BLOCK, dont_need), + Fd, hd(BlockLocations), ReadCount * ?SIZE_BLOCK, dont_need + ), NextBlock = hd(BlockLocations) div ?SIZE_BLOCK - 1, find_header(Fd, NextBlock, ReadCount) end. @@ -704,10 +737,10 @@ find_newest_header(Fd, [{Location, Size} | LocationSizes]) -> find_newest_header(Fd, LocationSizes) end. - --spec read_raw_iolist_int(#file{}, Pos::non_neg_integer(), Len::non_neg_integer()) -> - {Data::iolist(), CurPos::non_neg_integer()}. -read_raw_iolist_int(Fd, {Pos, _Size}, Len) -> % 0110 UPGRADE CODE +-spec read_raw_iolist_int(#file{}, Pos :: non_neg_integer(), Len :: non_neg_integer()) -> + {Data :: iolist(), CurPos :: non_neg_integer()}. +% 0110 UPGRADE CODE +read_raw_iolist_int(Fd, {Pos, _Size}, Len) -> read_raw_iolist_int(Fd, Pos, Len); read_raw_iolist_int(#file{fd = Fd} = File, Pos, Len) -> {Pos, TotalBytes} = get_pread_locnum(File, Pos, Len), @@ -727,33 +760,38 @@ read_raw_iolist_int(#file{fd = Fd} = File, Pos, Len) -> % TODO: check if this is really unused read_multi_raw_iolists_int(#file{fd = Fd} = File, PosLens) -> - LocNums = lists:map(fun({Pos, Len}) -> - get_pread_locnum(File, Pos, Len) - end, PosLens), + LocNums = lists:map( + fun({Pos, Len}) -> + get_pread_locnum(File, Pos, Len) + end, + PosLens + ), {ok, Bins} = file:pread(Fd, LocNums), - lists:zipwith(fun({Pos, TotalBytes}, Bin) -> - <<RawBin:TotalBytes/binary>> = Bin, - {remove_block_prefixes(Pos rem ?SIZE_BLOCK, RawBin), Pos + TotalBytes} - end, LocNums, Bins). - + lists:zipwith( + fun({Pos, TotalBytes}, Bin) -> + <<RawBin:TotalBytes/binary>> = Bin, + {remove_block_prefixes(Pos rem ?SIZE_BLOCK, RawBin), Pos + TotalBytes} + end, + LocNums, + Bins + ). get_pread_locnum(File, Pos, Len) -> BlockOffset = Pos rem ?SIZE_BLOCK, TotalBytes = calculate_total_read_len(BlockOffset, Len), case Pos + TotalBytes of - Size when Size > File#file.eof -> - couch_stats:increment_counter([pread, exceed_eof]), - {_Fd, Filepath} = get(couch_file_fd), - throw({read_beyond_eof, Filepath}); - Size when Size > File#file.pread_limit -> - couch_stats:increment_counter([pread, exceed_limit]), - {_Fd, Filepath} = get(couch_file_fd), - throw({exceed_pread_limit, Filepath, File#file.pread_limit}); - _ -> - {Pos, TotalBytes} + Size when Size > File#file.eof -> + couch_stats:increment_counter([pread, exceed_eof]), + {_Fd, Filepath} = get(couch_file_fd), + throw({read_beyond_eof, Filepath}); + Size when Size > File#file.pread_limit -> + couch_stats:increment_counter([pread, exceed_limit]), + {_Fd, Filepath} = get(couch_file_fd), + throw({exceed_pread_limit, Filepath, File#file.pread_limit}); + _ -> + {Pos, TotalBytes} end. - -spec extract_md5(iolist()) -> {binary(), iolist()}. extract_md5(FullIoList) -> {Md5List, IoList} = split_iolist(FullIoList, 16, []), @@ -763,26 +801,28 @@ calculate_total_read_len(0, FinalLen) -> calculate_total_read_len(1, FinalLen) + 1; calculate_total_read_len(BlockOffset, FinalLen) -> case ?SIZE_BLOCK - BlockOffset of - BlockLeft when BlockLeft >= FinalLen -> - FinalLen; - BlockLeft -> - FinalLen + ((FinalLen - BlockLeft) div (?SIZE_BLOCK -1)) + - if ((FinalLen - BlockLeft) rem (?SIZE_BLOCK -1)) =:= 0 -> 0; - true -> 1 end + BlockLeft when BlockLeft >= FinalLen -> + FinalLen; + BlockLeft -> + FinalLen + ((FinalLen - BlockLeft) div (?SIZE_BLOCK - 1)) + + if + ((FinalLen - BlockLeft) rem (?SIZE_BLOCK - 1)) =:= 0 -> 0; + true -> 1 + end end. remove_block_prefixes(_BlockOffset, <<>>) -> []; -remove_block_prefixes(0, <<_BlockPrefix,Rest/binary>>) -> +remove_block_prefixes(0, <<_BlockPrefix, Rest/binary>>) -> remove_block_prefixes(1, Rest); remove_block_prefixes(BlockOffset, Bin) -> BlockBytesAvailable = ?SIZE_BLOCK - BlockOffset, case size(Bin) of - Size when Size > BlockBytesAvailable -> - <<DataBlock:BlockBytesAvailable/binary,Rest/binary>> = Bin, - [DataBlock | remove_block_prefixes(0, Rest)]; - _Size -> - [Bin] + Size when Size > BlockBytesAvailable -> + <<DataBlock:BlockBytesAvailable/binary, Rest/binary>> = Bin, + [DataBlock | remove_block_prefixes(0, Rest)]; + _Size -> + [Bin] end. make_blocks(_BlockOffset, []) -> @@ -791,16 +831,16 @@ make_blocks(0, IoList) -> [<<0>> | make_blocks(1, IoList)]; make_blocks(BlockOffset, IoList) -> case split_iolist(IoList, (?SIZE_BLOCK - BlockOffset), []) of - {Begin, End} -> - [Begin | make_blocks(0, End)]; - _SplitRemaining -> - IoList + {Begin, End} -> + [Begin | make_blocks(0, End)]; + _SplitRemaining -> + IoList end. %% @doc Returns a tuple where the first element contains the leading SplitAt %% bytes of the original iolist, and the 2nd element is the tail. If SplitAt %% is larger than byte_size(IoList), return the difference. --spec split_iolist(IoList::iolist(), SplitAt::non_neg_integer(), Acc::list()) -> +-spec split_iolist(IoList :: iolist(), SplitAt :: non_neg_integer(), Acc :: list()) -> {iolist(), iolist()} | non_neg_integer(). split_iolist(List, 0, BeginAcc) -> {lists:reverse(BeginAcc), List}; @@ -809,14 +849,14 @@ split_iolist([], SplitAt, _BeginAcc) -> split_iolist([<<Bin/binary>> | Rest], SplitAt, BeginAcc) when SplitAt > byte_size(Bin) -> split_iolist(Rest, SplitAt - byte_size(Bin), [Bin | BeginAcc]); split_iolist([<<Bin/binary>> | Rest], SplitAt, BeginAcc) -> - <<Begin:SplitAt/binary,End/binary>> = Bin, + <<Begin:SplitAt/binary, End/binary>> = Bin, split_iolist([End | Rest], 0, [Begin | BeginAcc]); -split_iolist([Sublist| Rest], SplitAt, BeginAcc) when is_list(Sublist) -> +split_iolist([Sublist | Rest], SplitAt, BeginAcc) when is_list(Sublist) -> case split_iolist(Sublist, SplitAt, BeginAcc) of - {Begin, End} -> - {Begin, [End | Rest]}; - SplitRemaining -> - split_iolist(Rest, SplitAt - (SplitAt - SplitRemaining), [Sublist | BeginAcc]) + {Begin, End} -> + {Begin, [End | Rest]}; + SplitRemaining -> + split_iolist(Rest, SplitAt - (SplitAt - SplitRemaining), [Sublist | BeginAcc]) end; split_iolist([Byte | Rest], SplitAt, BeginAcc) when is_integer(Byte) -> split_iolist(Rest, SplitAt - 1, [Byte | BeginAcc]). @@ -825,29 +865,25 @@ monitored_by_pids() -> {monitored_by, PidsAndRefs} = process_info(self(), monitored_by), lists:filter(fun is_pid/1, PidsAndRefs). - verify_md5(_Fd, _Pos, IoList, <<>>) -> IoList; - verify_md5(Fd, Pos, IoList, Md5) -> case couch_hash:md5_hash(IoList) of Md5 -> IoList; _ -> report_md5_error(Fd, Pos) end. - report_md5_error(Fd, Pos) -> couch_log:emergency("File corruption in ~p at position ~B", [Fd, Pos]), exit({file_corruption, <<"file corruption">>}). - % System dbs aren't monitored by couch_stats_process_tracker -is_idle(#file{is_sys=true}) -> +is_idle(#file{is_sys = true}) -> case monitored_by_pids() of [] -> true; _ -> false end; -is_idle(#file{is_sys=false}) -> +is_idle(#file{is_sys = false}) -> Tracker = whereis(couch_stats_process_tracker), case monitored_by_pids() of [] -> true; @@ -865,10 +901,10 @@ process_info(Pid) -> update_read_timestamp() -> put(read_timestamp, os:timestamp()). -upgrade_state(#file{db_monitor=DbPid}=File) when is_pid(DbPid) -> +upgrade_state(#file{db_monitor = DbPid} = File) when is_pid(DbPid) -> unlink(DbPid), Ref = monitor(process, DbPid), - File#file{db_monitor=Ref}; + File#file{db_monitor = Ref}; upgrade_state(State) -> State. @@ -889,21 +925,26 @@ reset_eof(#file{} = File) -> deleted_filename_test_() -> DbNames = ["dbname", "db.name", "user/dbname"], Fixtures = make_filename_fixtures(DbNames), - lists:map(fun(Fixture) -> - should_create_proper_deleted_filename(Fixture) - end, Fixtures). + lists:map( + fun(Fixture) -> + should_create_proper_deleted_filename(Fixture) + end, + Fixtures + ). should_create_proper_deleted_filename(Before) -> {Before, - ?_test(begin - BeforeExtension = filename:extension(Before), - BeforeBasename = filename:basename(Before, BeforeExtension), - Re = "^" ++ BeforeBasename ++ "\.[0-9]{8}\.[0-9]{6}\.deleted\..*$", - After = deleted_filename(Before), - ?assertEqual(match, - re:run(filename:basename(After), Re, [{capture, none}])), - ?assertEqual(BeforeExtension, filename:extension(After)) - end)}. + ?_test(begin + BeforeExtension = filename:extension(Before), + BeforeBasename = filename:basename(Before, BeforeExtension), + Re = "^" ++ BeforeBasename ++ "\.[0-9]{8}\.[0-9]{6}\.deleted\..*$", + After = deleted_filename(Before), + ?assertEqual( + match, + re:run(filename:basename(After), Re, [{capture, none}]) + ), + ?assertEqual(BeforeExtension, filename:extension(After)) + end)}. make_filename_fixtures(DbNames) -> Formats = [ @@ -912,12 +953,18 @@ make_filename_fixtures(DbNames) -> "shards/00000000-1fffffff/~s.1458336317.couch", ".shards/00000000-1fffffff/~s.1458336317_design", ".shards/00000000-1fffffff/~s.1458336317_design" - "/mrview/3133e28517e89a3e11435dd5ac4ad85a.view" + "/mrview/3133e28517e89a3e11435dd5ac4ad85a.view" ], - lists:flatmap(fun(DbName) -> - lists:map(fun(Format) -> - filename:join("/srv/data", io_lib:format(Format, [DbName])) - end, Formats) - end, DbNames). + lists:flatmap( + fun(DbName) -> + lists:map( + fun(Format) -> + filename:join("/srv/data", io_lib:format(Format, [DbName])) + end, + Formats + ) + end, + DbNames + ). -endif. diff --git a/src/couch/src/couch_flags.erl b/src/couch/src/couch_flags.erl index 5cfe7f6d1..42d585f2e 100644 --- a/src/couch/src/couch_flags.erl +++ b/src/couch/src/couch_flags.erl @@ -64,13 +64,13 @@ -include_lib("mem3/include/mem3.hrl"). -include("couch_db_int.hrl"). --type subject() - :: #db{} - | #httpd{} - | #shard{} - | #ordered_shard{} - | string() - | binary(). +-type subject() :: + #db{} + | #httpd{} + | #shard{} + | #ordered_shard{} + | string() + | binary(). -define(SERVICE_ID, feature_flags). @@ -79,8 +79,10 @@ enabled(Subject) -> Key = maybe_handle(subject_key, [Subject], fun subject_key/1), Handle = couch_epi:get_handle({flags, config}), - lists:usort(enabled(Handle, {<<"/", Key/binary>>}) - ++ enabled(Handle, {couch_db:normalize_dbname(Key)})). + lists:usort( + enabled(Handle, {<<"/", Key/binary>>}) ++ + enabled(Handle, {couch_db:normalize_dbname(Key)}) + ). -spec is_enabled(FlagId :: atom(), subject()) -> boolean(). @@ -106,9 +108,9 @@ enabled(Handle, Key) -> subject_key(#db{name = Name}) -> subject_key(Name); -subject_key(#httpd{path_parts=[Name | _Rest]}) -> +subject_key(#httpd{path_parts = [Name | _Rest]}) -> subject_key(Name); -subject_key(#httpd{path_parts=[]}) -> +subject_key(#httpd{path_parts = []}) -> <<>>; subject_key(#shard{name = Name}) -> subject_key(Name); @@ -120,9 +122,10 @@ subject_key(Name) when is_binary(Name) -> Name. -spec maybe_handle( - Function :: atom(), - Args :: [term()], - Default :: fun((Args :: [term()]) -> term())) -> + Function :: atom(), + Args :: [term()], + Default :: fun((Args :: [term()]) -> term()) +) -> term(). maybe_handle(Func, Args, Default) -> diff --git a/src/couch/src/couch_flags_config.erl b/src/couch/src/couch_flags_config.erl index 104a48257..a50f4411f 100644 --- a/src/couch/src/couch_flags_config.erl +++ b/src/couch/src/couch_flags_config.erl @@ -28,23 +28,26 @@ -define(DATA_INTERVAL, 1000). -define(MAX_FLAG_NAME_LENGTH, 256). --type pattern() - :: binary(). %% non empty binary which optionally can end with * +-type pattern() :: + %% non empty binary which optionally can end with * + binary(). -type flag_id() :: atom(). -type flags() :: list(flag_id()). --type parse_pattern() - :: { - binary(), %% pattern without trainig * if it is present - pattern(), - IsWildCard :: boolean(), %% true if the pattern has training * - PatternSize :: pos_integer() - }. +-type parse_pattern() :: + { + %% pattern without trainig * if it is present + binary(), + pattern(), + %% true if the pattern has training * + IsWildCard :: boolean(), + PatternSize :: pos_integer() + }. --type rule() - :: { +-type rule() :: + { parse_pattern(), EnabledFlags :: flags(), DisabledFlags :: flags() @@ -75,26 +78,29 @@ data() -> data(Config) -> ByPattern = collect_rules(Config), - lists:reverse([{{P}, {P, size(P), E -- D}} || {P, {_, E, D}} <- ByPattern]). + lists:reverse([{{P}, {P, size(P), E -- D}} || {P, {_, E, D}} <- ByPattern]). -spec parse_rules([{Key :: string(), Value :: string()}]) -> [rule()]. parse_rules(Config) -> - lists:filtermap(fun({K, V}) -> - case parse_rule(K, V) of - {error, {Format, Args}} -> - couch_log:error(Format, Args), - false; - Rule -> - {true, Rule} - end - end, Config). + lists:filtermap( + fun({K, V}) -> + case parse_rule(K, V) of + {error, {Format, Args}} -> + couch_log:error(Format, Args), + false; + Rule -> + {true, Rule} + end + end, + Config + ). -spec parse_rule(Key :: string(), Value :: string()) -> rule() | {error, Reason :: term()}. -parse_rule(Key, "true") -> +parse_rule(Key, "true") -> parse_flags(binary:split(list_to_binary(Key), <<"||">>), true); parse_rule(Key, "false") -> parse_flags(binary:split(list_to_binary(Key), <<"||">>), false); @@ -119,29 +125,32 @@ parse_flags([FlagsBin, PatternBin], Value) -> end; parse_flags(_Tokens, _) -> couch_log:error( - "Key should be in the form of `[flags]||pattern` (got ~s)", []), + "Key should be in the form of `[flags]||pattern` (got ~s)", [] + ), false. -spec parse_flags_term(Flags :: binary()) -> [flag_id()] | {error, Reason :: term()}. parse_flags_term(FlagsBin) -> - {Flags, Errors} = lists:splitwith(fun erlang:is_atom/1, - [parse_flag(F) || F <- split_by_comma(FlagsBin)]), + {Flags, Errors} = lists:splitwith( + fun erlang:is_atom/1, + [parse_flag(F) || F <- split_by_comma(FlagsBin)] + ), case Errors of - [] -> - lists:usort(Flags); - _ -> - {error, { - "Cannot parse list of tags: ~n~p", - Errors - }} + [] -> + lists:usort(Flags); + _ -> + {error, { + "Cannot parse list of tags: ~n~p", + Errors + }} end. split_by_comma(Binary) -> case binary:split(Binary, <<",">>, [global]) of - [<<>>] -> []; - Tokens -> Tokens + [<<>>] -> []; + Tokens -> Tokens end. parse_flag(FlagName) when size(FlagName) > ?MAX_FLAG_NAME_LENGTH -> @@ -149,7 +158,7 @@ parse_flag(FlagName) when size(FlagName) > ?MAX_FLAG_NAME_LENGTH -> parse_flag(FlagName) -> FlagNameS = string:strip(binary_to_list(FlagName)), try - list_to_existing_atom(FlagNameS) + list_to_existing_atom(FlagNameS) catch _:_ -> {invalid_flag, FlagName} end. @@ -172,8 +181,10 @@ parse_pattern(PatternBin) -> collect_rules(ConfigData) -> ByKey = by_key(parse_rules(ConfigData)), Keys = lists:sort(fun sort_by_length/2, gb_trees:keys(ByKey)), - FuzzyKeys = lists:sort(fun sort_by_length/2, - [K || {K, {{_, _, true, _}, _, _}} <- gb_trees:to_list(ByKey)]), + FuzzyKeys = lists:sort( + fun sort_by_length/2, + [K || {K, {{_, _, true, _}, _, _}} <- gb_trees:to_list(ByKey)] + ), Rules = collect_rules(lists:reverse(Keys), FuzzyKeys, ByKey), gb_trees:to_list(Rules). @@ -185,17 +196,22 @@ sort_by_length(A, B) -> -spec by_key(Items :: [rule()]) -> Dictionary :: gb_trees:tree(). by_key(Items) -> - lists:foldl(fun({{_, K, _, _}, _, _} = Item, Acc) -> - update_element(Acc, K, Item, fun(Value) -> - update_flags(Value, Item) - end) - end, gb_trees:empty(), Items). + lists:foldl( + fun({{_, K, _, _}, _, _} = Item, Acc) -> + update_element(Acc, K, Item, fun(Value) -> + update_flags(Value, Item) + end) + end, + gb_trees:empty(), + Items + ). -spec update_element( - Tree :: gb_trees:tree(), - Key :: pattern(), - Default :: rule(), - Fun :: fun((Item :: rule()) -> rule())) -> + Tree :: gb_trees:tree(), + Key :: pattern(), + Default :: rule(), + Fun :: fun((Item :: rule()) -> rule()) +) -> gb_trees:tree(). update_element(Tree, Key, Default, Fun) -> @@ -207,9 +223,10 @@ update_element(Tree, Key, Default, Fun) -> end. -spec collect_rules( - Keys :: [pattern()], - FuzzyKeys :: [pattern()], - ByKey :: gb_trees:tree()) -> + Keys :: [pattern()], + FuzzyKeys :: [pattern()], + ByKey :: gb_trees:tree() +) -> gb_trees:tree(). collect_rules([], _, Acc) -> @@ -218,9 +235,10 @@ collect_rules([Current | Rest], Items, Acc) -> collect_rules(Rest, Items -- [Current], inherit_flags(Current, Items, Acc)). -spec inherit_flags( - Current :: pattern(), - FuzzyKeys :: [pattern()], - ByKey :: gb_trees:tree()) -> + Current :: pattern(), + FuzzyKeys :: [pattern()], + ByKey :: gb_trees:tree() +) -> gb_trees:tree(). inherit_flags(_Current, [], Acc) -> @@ -234,9 +252,10 @@ inherit_flags(Current, [Item | Items], Acc) -> end. -spec match_prefix( - AKey :: pattern(), - BKey :: pattern(), - ByKey :: gb_trees:tree()) -> + AKey :: pattern(), + BKey :: pattern(), + ByKey :: gb_trees:tree() +) -> boolean(). match_prefix(AKey, BKey, Acc) -> @@ -257,9 +276,10 @@ match_prefix({{Key0, _, _, _}, _, _}, {{Key1, _, true, S1}, _, _}) -> end. -spec update_flags( - AKey :: pattern(), - BKey :: pattern(), - ByKey :: gb_trees:tree()) -> + AKey :: pattern(), + BKey :: pattern(), + ByKey :: gb_trees:tree() +) -> gb_trees:tree(). update_flags(AKey, BKey, Acc) -> @@ -283,6 +303,7 @@ update_flags({Pattern, E0, D0}, {_, E1, D1}) -> get_config_section(Section) -> try config:get(Section) - catch error:badarg -> + catch + error:badarg -> [] end. diff --git a/src/couch/src/couch_hotp.erl b/src/couch/src/couch_hotp.erl index 4ba81c9bf..cdb8291f3 100644 --- a/src/couch/src/couch_hotp.erl +++ b/src/couch/src/couch_hotp.erl @@ -14,15 +14,16 @@ -export([generate/4]). -generate(Alg, Key, Counter, OutputLen) - when is_atom(Alg), is_binary(Key), is_integer(Counter), is_integer(OutputLen) -> +generate(Alg, Key, Counter, OutputLen) when + is_atom(Alg), is_binary(Key), is_integer(Counter), is_integer(OutputLen) +-> Hmac = couch_util:hmac(Alg, Key, <<Counter:64>>), Offset = binary:last(Hmac) band 16#f, Code = ((binary:at(Hmac, Offset) band 16#7f) bsl 24) + - ((binary:at(Hmac, Offset + 1) band 16#ff) bsl 16) + - ((binary:at(Hmac, Offset + 2) band 16#ff) bsl 8) + - ((binary:at(Hmac, Offset + 3) band 16#ff)), + ((binary:at(Hmac, Offset + 1) band 16#ff) bsl 16) + + ((binary:at(Hmac, Offset + 2) band 16#ff) bsl 8) + + (binary:at(Hmac, Offset + 3) band 16#ff), case OutputLen of 6 -> Code rem 1000000; 7 -> Code rem 10000000; diff --git a/src/couch/src/couch_httpd.erl b/src/couch/src/couch_httpd.erl index 535fc9245..64b68ce3f 100644 --- a/src/couch/src/couch_httpd.erl +++ b/src/couch/src/couch_httpd.erl @@ -18,21 +18,27 @@ -export([start_link/0, start_link/1, stop/0, handle_request/5]). --export([header_value/2,header_value/3,qs_value/2,qs_value/3,qs/1,qs_json_value/3]). --export([path/1,absolute_uri/2,body_length/1]). --export([verify_is_server_admin/1,unquote/1,quote/1,recv/2,recv_chunked/4,error_info/1]). +-export([header_value/2, header_value/3, qs_value/2, qs_value/3, qs/1, qs_json_value/3]). +-export([path/1, absolute_uri/2, body_length/1]). +-export([verify_is_server_admin/1, unquote/1, quote/1, recv/2, recv_chunked/4, error_info/1]). -export([make_fun_spec_strs/1]). -export([make_arity_1_fun/1, make_arity_2_fun/1, make_arity_3_fun/1]). --export([parse_form/1,json_body/1,json_body_obj/1,body/1]). +-export([parse_form/1, json_body/1, json_body_obj/1, body/1]). -export([doc_etag/1, doc_etag/3, make_etag/1, etag_match/2, etag_respond/3, etag_maybe/2]). --export([primary_header_value/2,partition/1,serve_file/3,serve_file/4, server_header/0]). --export([start_chunked_response/3,send_chunk/2,log_request/2]). +-export([primary_header_value/2, partition/1, serve_file/3, serve_file/4, server_header/0]). +-export([start_chunked_response/3, send_chunk/2, log_request/2]). -export([start_response_length/4, start_response/3, send/2]). -export([start_json_response/2, start_json_response/3, end_json_response/1]). --export([send_response/4,send_response_no_cors/4,send_method_not_allowed/2, - send_error/2,send_error/4, send_redirect/2,send_chunked_error/2]). --export([send_json/2,send_json/3,send_json/4,last_chunk/1,parse_multipart_request/3]). --export([accepted_encodings/1,handle_request_int/5,validate_referer/1,validate_ctype/2]). +-export([ + send_response/4, + send_response_no_cors/4, + send_method_not_allowed/2, + send_error/2, send_error/4, + send_redirect/2, + send_chunked_error/2 +]). +-export([send_json/2, send_json/3, send_json/4, last_chunk/1, parse_multipart_request/3]). +-export([accepted_encodings/1, handle_request_int/5, validate_referer/1, validate_ctype/2]). -export([http_1_0_keep_alive/2]). -export([validate_host/1]). -export([validate_bind_address/1]). @@ -47,7 +53,8 @@ -define(DEFAULT_SOCKET_OPTIONS, "[{sndbuf, 262144}]"). -define(DEFAULT_AUTHENTICATION_HANDLERS, "{couch_httpd_auth, cookie_authentication_handler}, " - "{couch_httpd_auth, default_authentication_handler}"). + "{couch_httpd_auth, default_authentication_handler}" +). start_link() -> start_link(http). @@ -58,18 +65,24 @@ start_link(https) -> Port = config:get("ssl", "port", "6984"), {ok, Ciphers} = couch_util:parse_term(config:get("ssl", "ciphers", undefined)), {ok, Versions} = couch_util:parse_term(config:get("ssl", "tls_versions", undefined)), - {ok, SecureRenegotiate} = couch_util:parse_term(config:get("ssl", "secure_renegotiate", undefined)), + {ok, SecureRenegotiate} = couch_util:parse_term( + config:get("ssl", "secure_renegotiate", undefined) + ), ServerOpts0 = - [{cacertfile, config:get("ssl", "cacert_file", undefined)}, - {keyfile, config:get("ssl", "key_file", undefined)}, - {certfile, config:get("ssl", "cert_file", undefined)}, - {password, config:get("ssl", "password", undefined)}, - {secure_renegotiate, SecureRenegotiate}, - {versions, Versions}, - {ciphers, Ciphers}], - - case (couch_util:get_value(keyfile, ServerOpts0) == undefined orelse - couch_util:get_value(certfile, ServerOpts0) == undefined) of + [ + {cacertfile, config:get("ssl", "cacert_file", undefined)}, + {keyfile, config:get("ssl", "key_file", undefined)}, + {certfile, config:get("ssl", "cert_file", undefined)}, + {password, config:get("ssl", "password", undefined)}, + {secure_renegotiate, SecureRenegotiate}, + {versions, Versions}, + {ciphers, Ciphers} + ], + + case + (couch_util:get_value(keyfile, ServerOpts0) == undefined orelse + couch_util:get_value(certfile, ServerOpts0) == undefined) + of true -> couch_log:error("SSL enabled but PEM certificates are missing", []), throw({error, missing_certs}); @@ -77,44 +90,58 @@ start_link(https) -> ok end, - ServerOpts = [Opt || {_, V}=Opt <- ServerOpts0, V /= undefined], - - ClientOpts = case config:get("ssl", "verify_ssl_certificates", "false") of - "false" -> - []; - "true" -> - FailIfNoPeerCert = case config:get("ssl", "fail_if_no_peer_cert", "false") of - "false" -> false; - "true" -> true - end, - [{depth, list_to_integer(config:get("ssl", - "ssl_certificate_max_depth", "1"))}, - {fail_if_no_peer_cert, FailIfNoPeerCert}, - {verify, verify_peer}] ++ - case config:get("ssl", "verify_fun", undefined) of - undefined -> []; - SpecStr -> - [{verify_fun, make_arity_3_fun(SpecStr)}] - end - end, + ServerOpts = [Opt || {_, V} = Opt <- ServerOpts0, V /= undefined], + + ClientOpts = + case config:get("ssl", "verify_ssl_certificates", "false") of + "false" -> + []; + "true" -> + FailIfNoPeerCert = + case config:get("ssl", "fail_if_no_peer_cert", "false") of + "false" -> false; + "true" -> true + end, + [ + {depth, + list_to_integer( + config:get( + "ssl", + "ssl_certificate_max_depth", + "1" + ) + )}, + {fail_if_no_peer_cert, FailIfNoPeerCert}, + {verify, verify_peer} + ] ++ + case config:get("ssl", "verify_fun", undefined) of + undefined -> []; + SpecStr -> [{verify_fun, make_arity_3_fun(SpecStr)}] + end + end, SslOpts = ServerOpts ++ ClientOpts, Options = - [{port, Port}, - {ssl, true}, - {ssl_opts, SslOpts}], + [ + {port, Port}, + {ssl, true}, + {ssl_opts, SslOpts} + ], start_link(https, Options). start_link(Name, Options) -> - BindAddress = case config:get("httpd", "bind_address", "any") of - "any" -> any; - Else -> Else - end, + BindAddress = + case config:get("httpd", "bind_address", "any") of + "any" -> any; + Else -> Else + end, ok = validate_bind_address(BindAddress), {ok, ServerOptions} = couch_util:parse_term( - config:get("httpd", "server_options", "[]")), + config:get("httpd", "server_options", "[]") + ), {ok, SocketOptions} = couch_util:parse_term( - config:get("httpd", "socket_options", ?DEFAULT_SOCKET_OPTIONS)), + config:get("httpd", "socket_options", ?DEFAULT_SOCKET_OPTIONS) + ), set_auth_handlers(), Handlers = get_httpd_handlers(), @@ -123,21 +150,26 @@ start_link(Name, Options) -> % get the same value. couch_server:get_uuid(), - Loop = fun(Req)-> + Loop = fun(Req) -> case SocketOptions of - [] -> - ok; - _ -> - ok = mochiweb_socket:setopts(Req:get(socket), SocketOptions) + [] -> + ok; + _ -> + ok = mochiweb_socket:setopts(Req:get(socket), SocketOptions) end, apply(?MODULE, handle_request, [Req | Handlers]) end, % set mochiweb options - FinalOptions = lists:append([Options, ServerOptions, [ + FinalOptions = lists:append([ + Options, + ServerOptions, + [ {loop, Loop}, {name, Name}, - {ip, BindAddress}]]), + {ip, BindAddress} + ] + ]), % launch mochiweb case mochiweb_http:start(FinalOptions) of @@ -148,21 +180,27 @@ start_link(Name, Options) -> throw({error, Reason}) end. - stop() -> mochiweb_http:stop(couch_httpd), catch mochiweb_http:stop(https). - set_auth_handlers() -> AuthenticationSrcs = make_fun_spec_strs( - config:get("httpd", "authentication_handlers", - ?DEFAULT_AUTHENTICATION_HANDLERS)), + config:get( + "httpd", + "authentication_handlers", + ?DEFAULT_AUTHENTICATION_HANDLERS + ) + ), AuthHandlers = lists:map( - fun(A) -> {auth_handler_name(A), make_arity_1_fun(A)} end, AuthenticationSrcs), - AuthenticationFuns = AuthHandlers ++ [ - fun couch_httpd_auth:party_mode_handler/1 %% must be last - ], + fun(A) -> {auth_handler_name(A), make_arity_1_fun(A)} end, AuthenticationSrcs + ), + AuthenticationFuns = + AuthHandlers ++ + [ + %% must be last + fun couch_httpd_auth:party_mode_handler/1 + ], ok = application:set_env(couch, auth_handlers, AuthenticationFuns). auth_handler_name(SpecStr) -> @@ -174,21 +212,27 @@ get_httpd_handlers() -> UrlHandlersList = lists:map( fun({UrlKey, SpecStr}) -> {?l2b(UrlKey), make_arity_1_fun(SpecStr)} - end, HttpdGlobalHandlers), + end, + HttpdGlobalHandlers + ), {ok, HttpdDbHandlers} = application:get_env(couch, httpd_db_handlers), DbUrlHandlersList = lists:map( fun({UrlKey, SpecStr}) -> {?l2b(UrlKey), make_arity_2_fun(SpecStr)} - end, HttpdDbHandlers), + end, + HttpdDbHandlers + ), {ok, HttpdDesignHandlers} = application:get_env(couch, httpd_design_handlers), DesignUrlHandlersList = lists:map( fun({UrlKey, SpecStr}) -> {?l2b(UrlKey), make_arity_3_fun(SpecStr)} - end, HttpdDesignHandlers), + end, + HttpdDesignHandlers + ), UrlHandlers = dict:from_list(UrlHandlersList), DbUrlHandlers = dict:from_list(DbUrlHandlersList), @@ -200,26 +244,26 @@ get_httpd_handlers() -> % or "{my_module, my_fun, <<"my_arg">>}" make_arity_1_fun(SpecStr) -> case couch_util:parse_term(SpecStr) of - {ok, {Mod, Fun, SpecArg}} -> - fun(Arg) -> Mod:Fun(Arg, SpecArg) end; - {ok, {Mod, Fun}} -> - fun(Arg) -> Mod:Fun(Arg) end + {ok, {Mod, Fun, SpecArg}} -> + fun(Arg) -> Mod:Fun(Arg, SpecArg) end; + {ok, {Mod, Fun}} -> + fun(Arg) -> Mod:Fun(Arg) end end. make_arity_2_fun(SpecStr) -> case couch_util:parse_term(SpecStr) of - {ok, {Mod, Fun, SpecArg}} -> - fun(Arg1, Arg2) -> Mod:Fun(Arg1, Arg2, SpecArg) end; - {ok, {Mod, Fun}} -> - fun(Arg1, Arg2) -> Mod:Fun(Arg1, Arg2) end + {ok, {Mod, Fun, SpecArg}} -> + fun(Arg1, Arg2) -> Mod:Fun(Arg1, Arg2, SpecArg) end; + {ok, {Mod, Fun}} -> + fun(Arg1, Arg2) -> Mod:Fun(Arg1, Arg2) end end. make_arity_3_fun(SpecStr) -> case couch_util:parse_term(SpecStr) of - {ok, {Mod, Fun, SpecArg}} -> - fun(Arg1, Arg2, Arg3) -> Mod:Fun(Arg1, Arg2, Arg3, SpecArg) end; - {ok, {Mod, Fun}} -> - fun(Arg1, Arg2, Arg3) -> Mod:Fun(Arg1, Arg2, Arg3) end + {ok, {Mod, Fun, SpecArg}} -> + fun(Arg1, Arg2, Arg3) -> Mod:Fun(Arg1, Arg2, Arg3, SpecArg) end; + {ok, {Mod, Fun}} -> + fun(Arg1, Arg2, Arg3) -> Mod:Fun(Arg1, Arg2, Arg3) end end. % SpecStr is "{my_module, my_fun}, {my_module2, my_fun2}" @@ -231,15 +275,25 @@ handle_request(MochiReq) -> erlang:put(mochiweb_request_body, Body), apply(?MODULE, handle_request, [MochiReq | get_httpd_handlers()]). -handle_request(MochiReq, DefaultFun, UrlHandlers, DbUrlHandlers, - DesignUrlHandlers) -> +handle_request( + MochiReq, + DefaultFun, + UrlHandlers, + DbUrlHandlers, + DesignUrlHandlers +) -> %% reset rewrite count for new request erlang:put(?REWRITE_COUNT, 0), MochiReq1 = couch_httpd_vhost:dispatch_host(MochiReq), - handle_request_int(MochiReq1, DefaultFun, - UrlHandlers, DbUrlHandlers, DesignUrlHandlers). + handle_request_int( + MochiReq1, + DefaultFun, + UrlHandlers, + DbUrlHandlers, + DesignUrlHandlers + ). handle_request_int(MochiReq, DefaultFun, UrlHandlers, DbUrlHandlers, DesignUrlHandlers) -> @@ -420,8 +474,10 @@ valid_hosts() -> re:split(List, ",", [{return, list}]). check_request_uri_length(Uri) -> - check_request_uri_length(Uri, - chttpd_util:get_chttpd_config("max_uri_length")). + check_request_uri_length( + Uri, + chttpd_util:get_chttpd_config("max_uri_length") + ). check_request_uri_length(_Uri, undefined) -> ok; @@ -444,34 +500,33 @@ validate_referer(Req) -> Host = host_for_request(Req), Referer = header_value(Req, "Referer", fail), case Referer of - fail -> - throw({bad_request, <<"Referer header required.">>}); - Referer -> - {_,RefererHost,_,_,_} = mochiweb_util:urlsplit(Referer), - if - RefererHost =:= Host -> ok; - true -> throw({bad_request, <<"Referer header must match host.">>}) - end + fail -> + throw({bad_request, <<"Referer header required.">>}); + Referer -> + {_, RefererHost, _, _, _} = mochiweb_util:urlsplit(Referer), + if + RefererHost =:= Host -> ok; + true -> throw({bad_request, <<"Referer header must match host.">>}) + end end. validate_ctype(Req, Ctype) -> case header_value(Req, "Content-Type") of - undefined -> - throw({bad_ctype, "Content-Type must be "++Ctype}); - ReqCtype -> - case string:tokens(ReqCtype, ";") of - [Ctype] -> ok; - [Ctype | _Rest] -> ok; - _Else -> - throw({bad_ctype, "Content-Type must be "++Ctype}) - end + undefined -> + throw({bad_ctype, "Content-Type must be " ++ Ctype}); + ReqCtype -> + case string:tokens(ReqCtype, ";") of + [Ctype] -> ok; + [Ctype | _Rest] -> ok; + _Else -> throw({bad_ctype, "Content-Type must be " ++ Ctype}) + end end. - check_max_request_length(Req) -> Len = list_to_integer(header_value(Req, "Content-Length", "0")), MaxLen = chttpd_util:get_chttpd_config_integer( - "max_http_request_size", 4294967296), + "max_http_request_size", 4294967296 + ), case Len > MaxLen of true -> exit({body_too_large, Len}); @@ -479,32 +534,31 @@ check_max_request_length(Req) -> ok end. - % Utilities partition(Path) -> mochiweb_util:partition(Path, "/"). -header_value(#httpd{mochi_req=MochiReq}, Key) -> +header_value(#httpd{mochi_req = MochiReq}, Key) -> MochiReq:get_header_value(Key). -header_value(#httpd{mochi_req=MochiReq}, Key, Default) -> +header_value(#httpd{mochi_req = MochiReq}, Key, Default) -> case MochiReq:get_header_value(Key) of - undefined -> Default; - Value -> Value + undefined -> Default; + Value -> Value end. -primary_header_value(#httpd{mochi_req=MochiReq}, Key) -> +primary_header_value(#httpd{mochi_req = MochiReq}, Key) -> MochiReq:get_primary_header_value(Key). -accepted_encodings(#httpd{mochi_req=MochiReq}) -> +accepted_encodings(#httpd{mochi_req = MochiReq}) -> case MochiReq:accepted_encodings(["gzip", "identity"]) of - bad_accept_encoding_value -> - throw(bad_accept_encoding_value); - [] -> - throw(unacceptable_encoding); - EncList -> - EncList + bad_accept_encoding_value -> + throw(bad_accept_encoding_value); + [] -> + throw(unacceptable_encoding); + EncList -> + EncList end. serve_file(Req, RelativePath, DocumentRoot) -> @@ -514,7 +568,8 @@ serve_file(Req0, RelativePath0, DocumentRoot0, ExtraHeaders) -> Headers0 = basic_headers(Req0, ExtraHeaders), {ok, {Req1, Code1, Headers1, RelativePath1, DocumentRoot1}} = chttpd_plugin:before_serve_file( - Req0, 200, Headers0, RelativePath0, DocumentRoot0), + Req0, 200, Headers0, RelativePath0, DocumentRoot0 + ), log_request(Req1, Code1), #httpd{mochi_req = MochiReq} = Req1, {ok, MochiReq:serve_file(RelativePath1, DocumentRoot1, Headers1)}. @@ -527,53 +582,61 @@ qs_value(Req, Key, Default) -> qs_json_value(Req, Key, Default) -> case qs_value(Req, Key, Default) of - Default -> - Default; - Result -> - ?JSON_DECODE(Result) + Default -> + Default; + Result -> + ?JSON_DECODE(Result) end. -qs(#httpd{mochi_req=MochiReq}) -> +qs(#httpd{mochi_req = MochiReq}) -> MochiReq:parse_qs(). -path(#httpd{mochi_req=MochiReq}) -> +path(#httpd{mochi_req = MochiReq}) -> MochiReq:get(path). -host_for_request(#httpd{mochi_req=MochiReq}) -> +host_for_request(#httpd{mochi_req = MochiReq}) -> XHost = chttpd_util:get_chttpd_config( - "x_forwarded_host", "X-Forwarded-Host"), + "x_forwarded_host", "X-Forwarded-Host" + ), case MochiReq:get_header_value(XHost) of undefined -> case MochiReq:get_header_value("Host") of undefined -> - {ok, {Address, Port}} = case MochiReq:get(socket) of - {ssl, SslSocket} -> ssl:sockname(SslSocket); - Socket -> inet:sockname(Socket) - end, + {ok, {Address, Port}} = + case MochiReq:get(socket) of + {ssl, SslSocket} -> ssl:sockname(SslSocket); + Socket -> inet:sockname(Socket) + end, inet_parse:ntoa(Address) ++ ":" ++ integer_to_list(Port); Value1 -> Value1 end; - Value -> Value + Value -> + Value end. -absolute_uri(#httpd{mochi_req=MochiReq}=Req, [$/ | _] = Path) -> +absolute_uri(#httpd{mochi_req = MochiReq} = Req, [$/ | _] = Path) -> Host = host_for_request(Req), XSsl = chttpd_util:get_chttpd_config("x_forwarded_ssl", "X-Forwarded-Ssl"), - Scheme = case MochiReq:get_header_value(XSsl) of - "on" -> "https"; - _ -> - XProto = chttpd_util:get_chttpd_config( - "x_forwarded_proto", "X-Forwarded-Proto"), - case MochiReq:get_header_value(XProto) of - %% Restrict to "https" and "http" schemes only - "https" -> "https"; - _ -> case MochiReq:get(scheme) of - https -> "https"; - http -> "http" - end - end - end, + Scheme = + case MochiReq:get_header_value(XSsl) of + "on" -> + "https"; + _ -> + XProto = chttpd_util:get_chttpd_config( + "x_forwarded_proto", "X-Forwarded-Proto" + ), + case MochiReq:get_header_value(XProto) of + %% Restrict to "https" and "http" schemes only + "https" -> + "https"; + _ -> + case MochiReq:get(scheme) of + https -> "https"; + http -> "http" + end + end + end, Scheme ++ "://" ++ Host ++ Path; absolute_uri(_Req, _Path) -> throw({bad_request, "path must begin with a /."}). @@ -584,60 +647,63 @@ unquote(UrlEncodedString) -> quote(UrlDecodedString) -> mochiweb_util:quote_plus(UrlDecodedString). -parse_form(#httpd{mochi_req=MochiReq}) -> +parse_form(#httpd{mochi_req = MochiReq}) -> mochiweb_multipart:parse_form(MochiReq). -recv(#httpd{mochi_req=MochiReq}, Len) -> +recv(#httpd{mochi_req = MochiReq}, Len) -> MochiReq:recv(Len). -recv_chunked(#httpd{mochi_req=MochiReq}, MaxChunkSize, ChunkFun, InitState) -> +recv_chunked(#httpd{mochi_req = MochiReq}, MaxChunkSize, ChunkFun, InitState) -> % Fun is called once with each chunk % Fun({Length, Binary}, State) % called with Length == 0 on the last time. - MochiReq:stream_body(MaxChunkSize, ChunkFun, InitState, + MochiReq:stream_body( + MaxChunkSize, + ChunkFun, + InitState, chttpd_util:get_chttpd_config_integer( - "max_http_request_size", 4294967296)). + "max_http_request_size", 4294967296 + ) + ). -body_length(#httpd{mochi_req=MochiReq}) -> +body_length(#httpd{mochi_req = MochiReq}) -> MochiReq:get(body_length). -body(#httpd{mochi_req=MochiReq, req_body=undefined}) -> +body(#httpd{mochi_req = MochiReq, req_body = undefined}) -> MaxSize = chttpd_util:get_chttpd_config_integer( - "max_http_request_size", 4294967296), + "max_http_request_size", 4294967296 + ), MochiReq:recv_body(MaxSize); -body(#httpd{req_body=ReqBody}) -> +body(#httpd{req_body = ReqBody}) -> ReqBody. -json_body(#httpd{req_body=undefined} = Httpd) -> +json_body(#httpd{req_body = undefined} = Httpd) -> case body(Httpd) of undefined -> throw({bad_request, "Missing request body"}); Body -> ?JSON_DECODE(maybe_decompress(Httpd, Body)) end; - -json_body(#httpd{req_body=ReqBody}) -> +json_body(#httpd{req_body = ReqBody}) -> ReqBody. json_body_obj(Httpd) -> case json_body(Httpd) of {Props} -> {Props}; - _Else -> - throw({bad_request, "Request body must be a JSON object"}) + _Else -> throw({bad_request, "Request body must be a JSON object"}) end. - maybe_decompress(Httpd, Body) -> case header_value(Httpd, "Content-Encoding", "identity") of - "gzip" -> - zlib:gunzip(Body); - "identity" -> - Body; - Else -> - throw({bad_ctype, [Else, " is not a supported content encoding."]}) + "gzip" -> + zlib:gunzip(Body); + "identity" -> + Body; + Else -> + throw({bad_ctype, [Else, " is not a supported content encoding."]}) end. -doc_etag(#doc{id=Id, body=Body, revs={Start, [DiskRev|_]}}) -> +doc_etag(#doc{id = Id, body = Body, revs = {Start, [DiskRev | _]}}) -> doc_etag(Id, Body, {Start, DiskRev}). doc_etag(<<"_local/", _/binary>>, Body, {Start, DiskRev}) -> @@ -647,7 +713,7 @@ doc_etag(_Id, _Body, {Start, DiskRev}) -> rev_etag({Start, DiskRev}) -> Rev = couch_doc:rev_to_str({Start, DiskRev}), - <<$", Rev/binary, $">>. + <<$", Rev/binary, $">>. make_etag(Term) -> <<SigInt:128/integer>> = couch_hash:md5_hash(term_to_binary(Term)), @@ -655,20 +721,20 @@ make_etag(Term) -> etag_match(Req, CurrentEtag) when is_binary(CurrentEtag) -> etag_match(Req, binary_to_list(CurrentEtag)); - etag_match(Req, CurrentEtag) -> EtagsToMatch = string:tokens( - header_value(Req, "If-None-Match", ""), ", "), + header_value(Req, "If-None-Match", ""), ", " + ), lists:member(CurrentEtag, EtagsToMatch). etag_respond(Req, CurrentEtag, RespFun) -> case etag_match(Req, CurrentEtag) of - true -> - % the client has this in their cache. - send_response(Req, 304, [{"ETag", CurrentEtag}], <<>>); - false -> - % Run the function. - RespFun() + true -> + % the client has this in their cache. + send_response(Req, 304, [{"ETag", CurrentEtag}], <<>>); + false -> + % Run the function. + RespFun() end. etag_maybe(Req, RespFun) -> @@ -679,15 +745,15 @@ etag_maybe(Req, RespFun) -> send_response(Req, 304, [{"ETag", ETag}], <<>>) end. -verify_is_server_admin(#httpd{user_ctx=UserCtx}) -> +verify_is_server_admin(#httpd{user_ctx = UserCtx}) -> verify_is_server_admin(UserCtx); -verify_is_server_admin(#user_ctx{roles=Roles}) -> +verify_is_server_admin(#user_ctx{roles = Roles}) -> case lists:member(<<"_admin">>, Roles) of - true -> ok; - false -> throw({unauthorized, <<"You are not a server admin.">>}) + true -> ok; + false -> throw({unauthorized, <<"You are not a server admin.">>}) end. -log_request(#httpd{mochi_req=MochiReq,peer=Peer}=Req, Code) -> +log_request(#httpd{mochi_req = MochiReq, peer = Peer} = Req, Code) -> case erlang:get(dont_log_request) of true -> ok; @@ -714,16 +780,16 @@ log_response(Code, Body) -> couch_log:error("httpd ~p error response:~n ~s", [Code, Body]) end. -start_response_length(#httpd{mochi_req=MochiReq}=Req, Code, Headers0, Length) -> +start_response_length(#httpd{mochi_req = MochiReq} = Req, Code, Headers0, Length) -> Headers1 = basic_headers(Req, Headers0), Resp = handle_response(Req, Code, Headers1, Length, start_response_length), case MochiReq:get(method) of - 'HEAD' -> throw({http_head_abort, Resp}); - _ -> ok + 'HEAD' -> throw({http_head_abort, Resp}); + _ -> ok end, {ok, Resp}. -start_response(#httpd{mochi_req=MochiReq}=Req, Code, Headers0) -> +start_response(#httpd{mochi_req = MochiReq} = Req, Code, Headers0) -> Headers1 = basic_headers(Req, Headers0), Resp = handle_response(Req, Code, Headers1, undefined, start_response), case MochiReq:get(method) of @@ -741,9 +807,9 @@ send(Resp, Data) -> no_resp_conn_header([]) -> true; -no_resp_conn_header([{Hdr, V}|Rest]) when is_binary(Hdr)-> - no_resp_conn_header([{?b2l(Hdr), V}|Rest]); -no_resp_conn_header([{Hdr, _}|Rest]) when is_list(Hdr)-> +no_resp_conn_header([{Hdr, V} | Rest]) when is_binary(Hdr) -> + no_resp_conn_header([{?b2l(Hdr), V} | Rest]); +no_resp_conn_header([{Hdr, _} | Rest]) when is_list(Hdr) -> case string:to_lower(Hdr) of "connection" -> false; _ -> no_resp_conn_header(Rest) @@ -760,12 +826,12 @@ http_1_0_keep_alive(Req, Headers) -> false -> Headers end. -start_chunked_response(#httpd{mochi_req=MochiReq}=Req, Code, Headers0) -> +start_chunked_response(#httpd{mochi_req = MochiReq} = Req, Code, Headers0) -> Headers1 = add_headers(Req, Headers0), Resp = handle_response(Req, Code, Headers1, chunked, respond), case MochiReq:get(method) of - 'HEAD' -> throw({http_head_abort, Resp}); - _ -> ok + 'HEAD' -> throw({http_head_abort, Resp}); + _ -> ok end, {ok, Resp}. @@ -774,8 +840,9 @@ send_chunk({remote, Pid, Ref} = Resp, Data) -> {ok, Resp}; send_chunk(Resp, Data) -> case iolist_size(Data) of - 0 -> ok; % do nothing - _ -> Resp:write_chunk(Data) + % do nothing + 0 -> ok; + _ -> Resp:write_chunk(Data) end, {ok, Resp}. @@ -790,17 +857,23 @@ send_response(Req, Code, Headers0, Body) -> Headers1 = chttpd_cors:headers(Req, Headers0), send_response_no_cors(Req, Code, Headers1, Body). -send_response_no_cors(#httpd{mochi_req=MochiReq}=Req, Code, Headers, Body) -> +send_response_no_cors(#httpd{mochi_req = MochiReq} = Req, Code, Headers, Body) -> Headers1 = http_1_0_keep_alive(MochiReq, Headers), Headers2 = basic_headers_no_cors(Req, Headers1), Headers3 = chttpd_xframe_options:header(Req, Headers2), - Headers4 = chttpd_prefer_header:maybe_return_minimal(Req, Headers3), + Headers4 = chttpd_prefer_header:maybe_return_minimal(Req, Headers3), Resp = handle_response(Req, Code, Headers4, Body, respond), log_response(Code, Body), {ok, Resp}. send_method_not_allowed(Req, Methods) -> - send_error(Req, 405, [{"Allow", Methods}], <<"method_not_allowed">>, ?l2b("Only " ++ Methods ++ " allowed")). + send_error( + Req, + 405, + [{"Allow", Methods}], + <<"method_not_allowed">>, + ?l2b("Only " ++ Methods ++ " allowed") + ). send_json(Req, Value) -> send_json(Req, 200, Value). @@ -843,13 +916,18 @@ initialize_jsonp(Req) -> _ -> ok end, case get(jsonp) of - no_jsonp -> []; - [] -> []; + no_jsonp -> + []; + [] -> + []; CallBack -> try % make sure jsonp is configured on (default off) - case chttpd_util:get_chttpd_config_boolean( - "allow_jsonp", false) of + case + chttpd_util:get_chttpd_config_boolean( + "allow_jsonp", false + ) + of true -> validate_callback(CallBack); false -> @@ -889,12 +967,10 @@ validate_callback([Char | Rest]) -> _ when Char == $_ -> ok; _ when Char == $[ -> ok; _ when Char == $] -> ok; - _ -> - throw({bad_request, invalid_callback}) + _ -> throw({bad_request, invalid_callback}) end, validate_callback(Rest). - error_info({Error, Reason}) when is_list(Reason) -> error_info({Error, ?l2b(Reason)}); error_info(bad_request) -> @@ -923,8 +999,10 @@ error_info({forbidden, Msg}) -> error_info({unauthorized, Msg}) -> {401, <<"unauthorized">>, Msg}; error_info(file_exists) -> - {412, <<"file_exists">>, <<"The database could not be " - "created, the file already exists.">>}; + {412, <<"file_exists">>, << + "The database could not be " + "created, the file already exists." + >>}; error_info(request_entity_too_large) -> {413, <<"too_large">>, <<"the request entity is too large">>}; error_info({request_entity_too_large, {attachment, AttName}}) -> @@ -938,9 +1016,10 @@ error_info({bad_ctype, Reason}) -> error_info(requested_range_not_satisfiable) -> {416, <<"requested_range_not_satisfiable">>, <<"Requested range not satisfiable">>}; error_info({error, {illegal_database_name, Name}}) -> - Message = <<"Name: '", Name/binary, "'. Only lowercase characters (a-z), ", - "digits (0-9), and any of the characters _, $, (, ), +, -, and / ", - "are allowed. Must begin with a letter.">>, + Message = + <<"Name: '", Name/binary, "'. Only lowercase characters (a-z), ", + "digits (0-9), and any of the characters _, $, (, ), +, -, and / ", + "are allowed. Must begin with a letter.">>, {400, <<"illegal_database_name">>, Message}; error_info({missing_stub, Reason}) -> {412, <<"missing_stub">>, Reason}; @@ -951,64 +1030,102 @@ error_info({Error, Reason}) -> error_info(Error) -> {500, <<"unknown_error">>, couch_util:to_binary(Error)}. -error_headers(#httpd{mochi_req=MochiReq}=Req, Code, ErrorStr, ReasonStr) -> - if Code == 401 -> - % this is where the basic auth popup is triggered - case MochiReq:get_header_value("X-CouchDB-WWW-Authenticate") of - undefined -> - case chttpd_util:get_chttpd_config("WWW-Authenticate") of - undefined -> - % If the client is a browser and the basic auth popup isn't turned on - % redirect to the session page. - case ErrorStr of - <<"unauthorized">> -> - case chttpd_util:get_chttpd_auth_config( - "authentication_redirect", "/_utils/session.html") of - undefined -> {Code, []}; - AuthRedirect -> - case chttpd_util:get_chttpd_auth_config_boolean( - "require_valid_user", false) of - true -> - % send the browser popup header no matter what if we are require_valid_user - {Code, [{"WWW-Authenticate", "Basic realm=\"server\""}]}; - false -> - case MochiReq:accepts_content_type("application/json") of - true -> - {Code, []}; - false -> - case MochiReq:accepts_content_type("text/html") of - true -> - % Redirect to the path the user requested, not - % the one that is used internally. - UrlReturnRaw = case MochiReq:get_header_value("x-couchdb-vhost-path") of - undefined -> - MochiReq:get(path); - VHostPath -> - VHostPath - end, - RedirectLocation = lists:flatten([ - AuthRedirect, - "?return=", couch_util:url_encode(UrlReturnRaw), - "&reason=", couch_util:url_encode(ReasonStr) - ]), - {302, [{"Location", absolute_uri(Req, RedirectLocation)}]}; - false -> +error_headers(#httpd{mochi_req = MochiReq} = Req, Code, ErrorStr, ReasonStr) -> + if + Code == 401 -> + % this is where the basic auth popup is triggered + case MochiReq:get_header_value("X-CouchDB-WWW-Authenticate") of + undefined -> + case chttpd_util:get_chttpd_config("WWW-Authenticate") of + undefined -> + % If the client is a browser and the basic auth popup isn't turned on + % redirect to the session page. + case ErrorStr of + <<"unauthorized">> -> + case + chttpd_util:get_chttpd_auth_config( + "authentication_redirect", "/_utils/session.html" + ) + of + undefined -> + {Code, []}; + AuthRedirect -> + case + chttpd_util:get_chttpd_auth_config_boolean( + "require_valid_user", false + ) + of + true -> + % send the browser popup header no matter what if we are require_valid_user + {Code, [ + {"WWW-Authenticate", + "Basic realm=\"server\""} + ]}; + false -> + case + MochiReq:accepts_content_type( + "application/json" + ) + of + true -> + {Code, []}; + false -> + case + MochiReq:accepts_content_type( + "text/html" + ) + of + true -> + % Redirect to the path the user requested, not + % the one that is used internally. + UrlReturnRaw = + case + MochiReq:get_header_value( + "x-couchdb-vhost-path" + ) + of + undefined -> + MochiReq:get(path); + VHostPath -> + VHostPath + end, + RedirectLocation = lists:flatten( + [ + AuthRedirect, + "?return=", + couch_util:url_encode( + UrlReturnRaw + ), + "&reason=", + couch_util:url_encode( + ReasonStr + ) + ] + ), + {302, [ + {"Location", + absolute_uri( + Req, + RedirectLocation + )} + ]}; + false -> + {Code, []} + end + end + end + end; + _Else -> {Code, []} - end - end - end + end; + Type -> + {Code, [{"WWW-Authenticate", Type}]} end; - _Else -> - {Code, []} - end; - Type -> - {Code, [{"WWW-Authenticate", Type}]} + Type -> + {Code, [{"WWW-Authenticate", Type}]} end; - Type -> - {Code, [{"WWW-Authenticate", Type}]} - end; - true -> - {Code, []} + true -> + {Code, []} end. send_error(Req, Error) -> @@ -1020,25 +1137,33 @@ send_error(Req, Code, ErrorStr, ReasonStr) -> send_error(Req, Code, [], ErrorStr, ReasonStr). send_error(Req, Code, Headers, ErrorStr, ReasonStr) -> - send_json(Req, Code, Headers, - {[{<<"error">>, ErrorStr}, - {<<"reason">>, ReasonStr}]}). + send_json( + Req, + Code, + Headers, + {[ + {<<"error">>, ErrorStr}, + {<<"reason">>, ReasonStr} + ]} + ). % give the option for list functions to output html or other raw errors send_chunked_error(Resp, {_Error, {[{<<"body">>, Reason}]}}) -> send_chunk(Resp, Reason), last_chunk(Resp); - send_chunked_error(Resp, Error) -> {Code, ErrorStr, ReasonStr} = error_info(Error), - JsonError = {[{<<"code">>, Code}, - {<<"error">>, ErrorStr}, - {<<"reason">>, ReasonStr}]}, - send_chunk(Resp, ?l2b([$\n,?JSON_ENCODE(JsonError),$\n])), + JsonError = + {[ + {<<"code">>, Code}, + {<<"error">>, ErrorStr}, + {<<"reason">>, ReasonStr} + ]}, + send_chunk(Resp, ?l2b([$\n, ?JSON_ENCODE(JsonError), $\n])), last_chunk(Resp). send_redirect(Req, Path) -> - send_response(Req, 301, [{"Location", absolute_uri(Req, Path)}], <<>>). + send_response(Req, 301, [{"Location", absolute_uri(Req, Path)}], <<>>). negotiate_content_type(_Req) -> case get(jsonp) of @@ -1048,27 +1173,33 @@ negotiate_content_type(_Req) -> end. server_header() -> - [{"Server", "CouchDB/" ++ couch_server:get_version() ++ - " (Erlang OTP/" ++ erlang:system_info(otp_release) ++ ")"}]. - + [ + {"Server", + "CouchDB/" ++ couch_server:get_version() ++ + " (Erlang OTP/" ++ erlang:system_info(otp_release) ++ ")"} + ]. -record(mp, {boundary, buffer, data_fun, callback}). - parse_multipart_request(ContentType, DataFun, Callback) -> Boundary0 = iolist_to_binary(get_boundary(ContentType)), Boundary = <<"\r\n--", Boundary0/binary>>, - Mp = #mp{boundary= Boundary, - buffer= <<>>, - data_fun=DataFun, - callback=Callback}, - {Mp2, _NilCallback} = read_until(Mp, <<"--", Boundary0/binary>>, - fun nil_callback/1), - #mp{buffer=Buffer, data_fun=DataFun2, callback=Callback2} = - parse_part_header(Mp2), + Mp = #mp{ + boundary = Boundary, + buffer = <<>>, + data_fun = DataFun, + callback = Callback + }, + {Mp2, _NilCallback} = read_until( + Mp, + <<"--", Boundary0/binary>>, + fun nil_callback/1 + ), + #mp{buffer = Buffer, data_fun = DataFun2, callback = Callback2} = + parse_part_header(Mp2), {Buffer, DataFun2, Callback2}. -nil_callback(_Data)-> +nil_callback(_Data) -> fun nil_callback/1. get_boundary({"multipart/" ++ _, Opts}) -> @@ -1077,83 +1208,102 @@ get_boundary({"multipart/" ++ _, Opts}) -> S end; get_boundary(ContentType) -> - {"multipart/" ++ _ , Opts} = mochiweb_util:parse_header(ContentType), + {"multipart/" ++ _, Opts} = mochiweb_util:parse_header(ContentType), get_boundary({"multipart/", Opts}). - - split_header(<<>>) -> []; split_header(Line) -> - {Name, Rest} = lists:splitwith(fun (C) -> C =/= $: end, - binary_to_list(Line)), - [$: | Value] = case Rest of - [] -> - throw({bad_request, <<"bad part header">>}); - Res -> - Res - end, - [{string:to_lower(string:strip(Name)), - mochiweb_util:parse_header(Value)}]. + {Name, Rest} = lists:splitwith( + fun(C) -> C =/= $: end, + binary_to_list(Line) + ), + [$: | Value] = + case Rest of + [] -> + throw({bad_request, <<"bad part header">>}); + Res -> + Res + end, + [{string:to_lower(string:strip(Name)), mochiweb_util:parse_header(Value)}]. -read_until(#mp{data_fun=DataFun, buffer=Buffer}=Mp, Pattern, Callback) -> +read_until(#mp{data_fun = DataFun, buffer = Buffer} = Mp, Pattern, Callback) -> case couch_util:find_in_binary(Pattern, Buffer) of - not_found -> - Callback2 = Callback(Buffer), - {Buffer2, DataFun2} = DataFun(), - Buffer3 = iolist_to_binary(Buffer2), - read_until(Mp#mp{data_fun=DataFun2,buffer=Buffer3}, Pattern, Callback2); - {partial, 0} -> - {NewData, DataFun2} = DataFun(), - read_until(Mp#mp{data_fun=DataFun2, - buffer= iolist_to_binary([Buffer,NewData])}, - Pattern, Callback); - {partial, Skip} -> - <<DataChunk:Skip/binary, Rest/binary>> = Buffer, - Callback2 = Callback(DataChunk), - {NewData, DataFun2} = DataFun(), - read_until(Mp#mp{data_fun=DataFun2, - buffer= iolist_to_binary([Rest | NewData])}, - Pattern, Callback2); - {exact, 0} -> - PatternLen = size(Pattern), - <<_:PatternLen/binary, Rest/binary>> = Buffer, - {Mp#mp{buffer= Rest}, Callback}; - {exact, Skip} -> - PatternLen = size(Pattern), - <<DataChunk:Skip/binary, _:PatternLen/binary, Rest/binary>> = Buffer, - Callback2 = Callback(DataChunk), - {Mp#mp{buffer= Rest}, Callback2} + not_found -> + Callback2 = Callback(Buffer), + {Buffer2, DataFun2} = DataFun(), + Buffer3 = iolist_to_binary(Buffer2), + read_until(Mp#mp{data_fun = DataFun2, buffer = Buffer3}, Pattern, Callback2); + {partial, 0} -> + {NewData, DataFun2} = DataFun(), + read_until( + Mp#mp{ + data_fun = DataFun2, + buffer = iolist_to_binary([Buffer, NewData]) + }, + Pattern, + Callback + ); + {partial, Skip} -> + <<DataChunk:Skip/binary, Rest/binary>> = Buffer, + Callback2 = Callback(DataChunk), + {NewData, DataFun2} = DataFun(), + read_until( + Mp#mp{ + data_fun = DataFun2, + buffer = iolist_to_binary([Rest | NewData]) + }, + Pattern, + Callback2 + ); + {exact, 0} -> + PatternLen = size(Pattern), + <<_:PatternLen/binary, Rest/binary>> = Buffer, + {Mp#mp{buffer = Rest}, Callback}; + {exact, Skip} -> + PatternLen = size(Pattern), + <<DataChunk:Skip/binary, _:PatternLen/binary, Rest/binary>> = Buffer, + Callback2 = Callback(DataChunk), + {Mp#mp{buffer = Rest}, Callback2} end. - -parse_part_header(#mp{callback=UserCallBack}=Mp) -> - {Mp2, AccCallback} = read_until(Mp, <<"\r\n\r\n">>, - fun(Next) -> acc_callback(Next, []) end), +parse_part_header(#mp{callback = UserCallBack} = Mp) -> + {Mp2, AccCallback} = read_until( + Mp, + <<"\r\n\r\n">>, + fun(Next) -> acc_callback(Next, []) end + ), HeaderData = AccCallback(get_data), Headers = - lists:foldl(fun(Line, Acc) -> - split_header(Line) ++ Acc - end, [], re:split(HeaderData,<<"\r\n">>, [])), + lists:foldl( + fun(Line, Acc) -> + split_header(Line) ++ Acc + end, + [], + re:split(HeaderData, <<"\r\n">>, []) + ), NextCallback = UserCallBack({headers, Headers}), - parse_part_body(Mp2#mp{callback=NextCallback}). - -parse_part_body(#mp{boundary=Prefix, callback=Callback}=Mp) -> - {Mp2, WrappedCallback} = read_until(Mp, Prefix, - fun(Data) -> body_callback_wrapper(Data, Callback) end), + parse_part_body(Mp2#mp{callback = NextCallback}). + +parse_part_body(#mp{boundary = Prefix, callback = Callback} = Mp) -> + {Mp2, WrappedCallback} = read_until( + Mp, + Prefix, + fun(Data) -> body_callback_wrapper(Data, Callback) end + ), Callback2 = WrappedCallback(get_callback), Callback3 = Callback2(body_end), - case check_for_last(Mp2#mp{callback=Callback3}) of - {last, #mp{callback=Callback3}=Mp3} -> - Mp3#mp{callback=Callback3(eof)}; - {more, Mp3} -> - parse_part_header(Mp3) + case check_for_last(Mp2#mp{callback = Callback3}) of + {last, #mp{callback = Callback3} = Mp3} -> + Mp3#mp{callback = Callback3(eof)}; + {more, Mp3} -> + parse_part_header(Mp3) end. -acc_callback(get_data, Acc)-> +acc_callback(get_data, Acc) -> iolist_to_binary(lists:reverse(Acc)); -acc_callback(Data, Acc)-> +acc_callback(Data, Acc) -> fun(Next) -> acc_callback(Next, [Data | Acc]) end. body_callback_wrapper(get_callback, Callback) -> @@ -1162,18 +1312,23 @@ body_callback_wrapper(Data, Callback) -> Callback2 = Callback({body, Data}), fun(Next) -> body_callback_wrapper(Next, Callback2) end. - -check_for_last(#mp{buffer=Buffer, data_fun=DataFun}=Mp) -> +check_for_last(#mp{buffer = Buffer, data_fun = DataFun} = Mp) -> case Buffer of - <<"--",_/binary>> -> {last, Mp}; - <<_, _, _/binary>> -> {more, Mp}; - _ -> % not long enough - {Data, DataFun2} = DataFun(), - check_for_last(Mp#mp{buffer= <<Buffer/binary, Data/binary>>, - data_fun = DataFun2}) + <<"--", _/binary>> -> + {last, Mp}; + <<_, _, _/binary>> -> + {more, Mp}; + % not long enough + _ -> + {Data, DataFun2} = DataFun(), + check_for_last(Mp#mp{ + buffer = <<Buffer/binary, Data/binary>>, + data_fun = DataFun2 + }) end. -validate_bind_address(any) -> ok; +validate_bind_address(any) -> + ok; validate_bind_address(Address) -> case inet_parse:address(Address) of {ok, _} -> ok; @@ -1191,9 +1346,9 @@ basic_headers(Req, Headers0) -> chttpd_cors:headers(Req, Headers2). basic_headers_no_cors(Req, Headers) -> - Headers - ++ server_header() - ++ couch_httpd_auth:cookie_auth_header(Req, Headers). + Headers ++ + server_header() ++ + couch_httpd_auth:cookie_auth_header(Req, Headers). handle_response(Req0, Code0, Headers0, Args0, Type) -> {ok, {Req1, Code1, Headers1, Args1}} = before_response(Req0, Code0, Headers0, Args0), @@ -1259,27 +1414,40 @@ maybe_add_default_headers_test_() -> MustRevalidate = {"Cache-Control", "must-revalidate"}, ApplicationJavascript = {"Content-Type", "application/javascript"}, Cases = [ - {[], - [MustRevalidate, ApplicationJavascript], - "Should add Content-Type and Cache-Control to empty heaeders"}, - - {[NoCache], - [NoCache, ApplicationJavascript], - "Should add Content-Type only if Cache-Control is present"}, - - {[ApplicationJson], - [MustRevalidate, ApplicationJson], - "Should add Cache-Control if Content-Type is present"}, - - {[NoCache, ApplicationJson], - [NoCache, ApplicationJson], - "Should not add headers if Cache-Control and Content-Type are there"} + { + [], + [MustRevalidate, ApplicationJavascript], + "Should add Content-Type and Cache-Control to empty heaeders" + }, + + { + [NoCache], + [NoCache, ApplicationJavascript], + "Should add Content-Type only if Cache-Control is present" + }, + + { + [ApplicationJson], + [MustRevalidate, ApplicationJson], + "Should add Cache-Control if Content-Type is present" + }, + + { + [NoCache, ApplicationJson], + [NoCache, ApplicationJson], + "Should not add headers if Cache-Control and Content-Type are there" + } ], - Tests = lists:map(fun({InitialHeaders, ProperResult, Desc}) -> - {Desc, - ?_assertEqual(ProperResult, - maybe_add_default_headers(DummyRequest, InitialHeaders))} - end, Cases), + Tests = lists:map( + fun({InitialHeaders, ProperResult, Desc}) -> + {Desc, + ?_assertEqual( + ProperResult, + maybe_add_default_headers(DummyRequest, InitialHeaders) + )} + end, + Cases + ), {"Tests adding default headers", Tests}. log_request_test_() -> @@ -1299,27 +1467,24 @@ log_request_test_() -> [ fun() -> should_accept_code_and_message(true) end, fun() -> should_accept_code_and_message(false) end - ] - }. + ]}. should_accept_code_and_message(DontLogFlag) -> erlang:put(dont_log_response, DontLogFlag), - {"with dont_log_response = " ++ atom_to_list(DontLogFlag), - [ - {"Should accept code 200 and string message", - ?_assertEqual(ok, log_response(200, "OK"))}, - {"Should accept code 200 and JSON message", + {"with dont_log_response = " ++ atom_to_list(DontLogFlag), [ + {"Should accept code 200 and string message", ?_assertEqual(ok, log_response(200, "OK"))}, + {"Should accept code 200 and JSON message", ?_assertEqual(ok, log_response(200, {json, {[{ok, true}]}}))}, - {"Should accept code >= 400 and string error", + {"Should accept code >= 400 and string error", ?_assertEqual(ok, log_response(405, method_not_allowed))}, - {"Should accept code >= 400 and JSON error", - ?_assertEqual(ok, - log_response(405, {json, {[{error, method_not_allowed}]}}))}, - {"Should accept code >= 500 and string error", - ?_assertEqual(ok, log_response(500, undef))}, - {"Should accept code >= 500 and JSON error", + {"Should accept code >= 400 and JSON error", + ?_assertEqual( + ok, + log_response(405, {json, {[{error, method_not_allowed}]}}) + )}, + {"Should accept code >= 500 and string error", ?_assertEqual(ok, log_response(500, undef))}, + {"Should accept code >= 500 and JSON error", ?_assertEqual(ok, log_response(500, {json, {[{error, undef}]}}))} - ] - }. + ]}. -endif. diff --git a/src/couch/src/couch_httpd_auth.erl b/src/couch/src/couch_httpd_auth.erl index 01a210d05..7bcb85fba 100644 --- a/src/couch/src/couch_httpd_auth.erl +++ b/src/couch/src/couch_httpd_auth.erl @@ -18,8 +18,10 @@ -export([party_mode_handler/1]). --export([default_authentication_handler/1, default_authentication_handler/2, - special_test_authentication_handler/1]). +-export([ + default_authentication_handler/1, default_authentication_handler/2, + special_test_authentication_handler/1 +]). -export([cookie_authentication_handler/1, cookie_authentication_handler/2]). -export([null_authentication_handler/1]). -export([proxy_authentication_handler/1, proxy_authentification_handler/1]). @@ -33,59 +35,68 @@ -export([jwt_authentication_handler/1]). --import(couch_httpd, [header_value/2, send_json/2, send_json/4, send_method_not_allowed/2, maybe_decompress/2]). +-import(couch_httpd, [ + header_value/2, send_json/2, send_json/4, send_method_not_allowed/2, maybe_decompress/2 +]). --compile({no_auto_import,[integer_to_binary/1, integer_to_binary/2]}). +-compile({no_auto_import, [integer_to_binary/1, integer_to_binary/2]}). party_mode_handler(Req) -> - case chttpd_util:get_chttpd_auth_config_boolean( - "require_valid_user", false) of - true -> - throw({unauthorized, <<"Authentication required.">>}); - false -> - Req#httpd{user_ctx=#user_ctx{}} + case + chttpd_util:get_chttpd_auth_config_boolean( + "require_valid_user", false + ) + of + true -> + throw({unauthorized, <<"Authentication required.">>}); + false -> + Req#httpd{user_ctx = #user_ctx{}} end. special_test_authentication_handler(Req) -> case header_value(Req, "WWW-Authenticate") of - "X-Couch-Test-Auth " ++ NamePass -> - % NamePass is a colon separated string: "joe schmoe:a password". - [Name, Pass] = re:split(NamePass, ":", [{return, list}, {parts, 2}]), - case {Name, Pass} of - {"Jan Lehnardt", "apple"} -> ok; - {"Christopher Lenz", "dog food"} -> ok; - {"Noah Slater", "biggiesmalls endian"} -> ok; - {"Chris Anderson", "mp3"} -> ok; - {"Damien Katz", "pecan pie"} -> ok; - {_, _} -> - throw({unauthorized, <<"Name or password is incorrect.">>}) - end, - Req#httpd{user_ctx=#user_ctx{name=?l2b(Name)}}; - _ -> - % No X-Couch-Test-Auth credentials sent, give admin access so the - % previous authentication can be restored after the test - Req#httpd{user_ctx=?ADMIN_USER} + "X-Couch-Test-Auth " ++ NamePass -> + % NamePass is a colon separated string: "joe schmoe:a password". + [Name, Pass] = re:split(NamePass, ":", [{return, list}, {parts, 2}]), + case {Name, Pass} of + {"Jan Lehnardt", "apple"} -> ok; + {"Christopher Lenz", "dog food"} -> ok; + {"Noah Slater", "biggiesmalls endian"} -> ok; + {"Chris Anderson", "mp3"} -> ok; + {"Damien Katz", "pecan pie"} -> ok; + {_, _} -> throw({unauthorized, <<"Name or password is incorrect.">>}) + end, + Req#httpd{user_ctx = #user_ctx{name = ?l2b(Name)}}; + _ -> + % No X-Couch-Test-Auth credentials sent, give admin access so the + % previous authentication can be restored after the test + Req#httpd{user_ctx = ?ADMIN_USER} end. basic_name_pw(Req) -> AuthorizationHeader = header_value(Req, "Authorization"), case AuthorizationHeader of - "Basic " ++ Base64Value -> - try re:split(base64:decode(Base64Value), ":", - [{return, list}, {parts, 2}]) of - ["_", "_"] -> - % special name and pass to be logged out - nil; - [User, Pass] -> - {User, Pass}; + "Basic " ++ Base64Value -> + try + re:split( + base64:decode(Base64Value), + ":", + [{return, list}, {parts, 2}] + ) + of + ["_", "_"] -> + % special name and pass to be logged out + nil; + [User, Pass] -> + {User, Pass}; + _ -> + nil + catch + error:function_clause -> + throw({bad_request, "Authorization header has invalid base64 value"}) + end; _ -> nil - catch - error:function_clause -> - throw({bad_request, "Authorization header has invalid base64 value"}) - end; - _ -> - nil end. default_authentication_handler(Req) -> @@ -93,42 +104,47 @@ default_authentication_handler(Req) -> default_authentication_handler(Req, AuthModule) -> case basic_name_pw(Req) of - {User, Pass} -> - case AuthModule:get_user_creds(Req, User) of - nil -> - throw({unauthorized, <<"Name or password is incorrect.">>}); - {ok, UserProps, _AuthCtx} -> - reject_if_totp(UserProps), - UserName = ?l2b(User), - Password = ?l2b(Pass), - case authenticate(Password, UserProps) of - true -> - Req#httpd{user_ctx=#user_ctx{ - name=UserName, - roles=couch_util:get_value(<<"roles">>, UserProps, []) - }}; - false -> - authentication_warning(Req, UserName), - throw({unauthorized, <<"Name or password is incorrect.">>}) - end - end; - nil -> - case couch_server:has_admins() of - true -> - Req; - false -> - case chttpd_util:get_chttpd_auth_config_boolean( - "require_valid_user", false) of - true -> Req; - % If no admins, and no user required, then everyone is admin! - % Yay, admin party! - false -> Req#httpd{user_ctx=?ADMIN_USER} + {User, Pass} -> + case AuthModule:get_user_creds(Req, User) of + nil -> + throw({unauthorized, <<"Name or password is incorrect.">>}); + {ok, UserProps, _AuthCtx} -> + reject_if_totp(UserProps), + UserName = ?l2b(User), + Password = ?l2b(Pass), + case authenticate(Password, UserProps) of + true -> + Req#httpd{ + user_ctx = #user_ctx{ + name = UserName, + roles = couch_util:get_value(<<"roles">>, UserProps, []) + } + }; + false -> + authentication_warning(Req, UserName), + throw({unauthorized, <<"Name or password is incorrect.">>}) + end + end; + nil -> + case couch_server:has_admins() of + true -> + Req; + false -> + case + chttpd_util:get_chttpd_auth_config_boolean( + "require_valid_user", false + ) + of + true -> Req; + % If no admins, and no user required, then everyone is admin! + % Yay, admin party! + false -> Req#httpd{user_ctx = ?ADMIN_USER} + end end - end end. null_authentication_handler(Req) -> - Req#httpd{user_ctx=?ADMIN_USER}. + Req#httpd{user_ctx = ?ADMIN_USER}. %% @doc proxy auth handler. % @@ -155,39 +171,53 @@ proxy_authentication_handler(Req) -> %% @deprecated proxy_authentification_handler(Req) -> proxy_authentication_handler(Req). - + proxy_auth_user(Req) -> XHeaderUserName = chttpd_util:get_chttpd_auth_config( - "x_auth_username", "X-Auth-CouchDB-UserName"), + "x_auth_username", "X-Auth-CouchDB-UserName" + ), XHeaderRoles = chttpd_util:get_chttpd_auth_config( - "x_auth_roles", "X-Auth-CouchDB-Roles"), + "x_auth_roles", "X-Auth-CouchDB-Roles" + ), XHeaderToken = chttpd_util:get_chttpd_auth_config( - "x_auth_token", "X-Auth-CouchDB-Token"), + "x_auth_token", "X-Auth-CouchDB-Token" + ), case header_value(Req, XHeaderUserName) of - undefined -> nil; + undefined -> + nil; UserName -> - Roles = case header_value(Req, XHeaderRoles) of - undefined -> []; - Else -> - [?l2b(R) || R <- string:tokens(Else, ",")] - end, - case chttpd_util:get_chttpd_auth_config_boolean( - "proxy_use_secret", false) of + Roles = + case header_value(Req, XHeaderRoles) of + undefined -> []; + Else -> [?l2b(R) || R <- string:tokens(Else, ",")] + end, + case + chttpd_util:get_chttpd_auth_config_boolean( + "proxy_use_secret", false + ) + of true -> case chttpd_util:get_chttpd_auth_config("secret") of undefined -> - Req#httpd{user_ctx=#user_ctx{name=?l2b(UserName), roles=Roles}}; + Req#httpd{user_ctx = #user_ctx{name = ?l2b(UserName), roles = Roles}}; Secret -> - ExpectedToken = couch_util:to_hex(couch_util:hmac(sha, Secret, UserName)), + ExpectedToken = couch_util:to_hex( + couch_util:hmac(sha, Secret, UserName) + ), case header_value(Req, XHeaderToken) of Token when Token == ExpectedToken -> - Req#httpd{user_ctx=#user_ctx{name=?l2b(UserName), - roles=Roles}}; - _ -> nil + Req#httpd{ + user_ctx = #user_ctx{ + name = ?l2b(UserName), + roles = Roles + } + }; + _ -> + nil end end; false -> - Req#httpd{user_ctx=#user_ctx{name=?l2b(UserName), roles=Roles}} + Req#httpd{user_ctx = #user_ctx{name = ?l2b(UserName), roles = Roles}} end end. @@ -198,22 +228,35 @@ jwt_authentication_handler(Req) -> case jwtf:decode(?l2b(Jwt), [alg | RequiredClaims], fun jwtf_keystore:get/2) of {ok, {Claims}} -> case lists:keyfind(<<"sub">>, 1, Claims) of - false -> throw({unauthorized, <<"Token missing sub claim.">>}); - {_, User} -> Req#httpd{user_ctx=#user_ctx{ - name = User, - roles = couch_util:get_value(?l2b(config:get("jwt_auth", "roles_claim_name", "_couchdb.roles")), Claims, []) - }} + false -> + throw({unauthorized, <<"Token missing sub claim.">>}); + {_, User} -> + Req#httpd{ + user_ctx = #user_ctx{ + name = User, + roles = couch_util:get_value( + ?l2b( + config:get( + "jwt_auth", "roles_claim_name", "_couchdb.roles" + ) + ), + Claims, + [] + ) + } + } end; {error, Reason} -> throw(Reason) end; - _ -> Req + _ -> + Req end. get_configured_claims() -> Claims = config:get("jwt_auth", "required_claims", ""), Re = "((?<key1>[a-z]+)|{(?<key2>[a-z]+)\s*,\s*\"(?<val>[^\"]+)\"})", - case re:run(Claims, Re, [global, {capture, [key1, key2, val], binary}]) of + case re:run(Claims, Re, [global, {capture, [key1, key2, val], binary}]) of nomatch when Claims /= "" -> couch_log:error("[jwt_auth] required_claims is set to an invalid value.", []), throw({misconfigured_server, <<"JWT is not configured correctly">>}); @@ -231,61 +274,77 @@ to_claim([<<>>, Key, Value]) -> cookie_authentication_handler(Req) -> cookie_authentication_handler(Req, couch_auth_cache). -cookie_authentication_handler(#httpd{mochi_req=MochiReq}=Req, AuthModule) -> +cookie_authentication_handler(#httpd{mochi_req = MochiReq} = Req, AuthModule) -> case MochiReq:get_cookie_value("AuthSession") of - undefined -> Req; - [] -> Req; - Cookie -> - [User, TimeStr, HashStr] = try - AuthSession = couch_util:decodeBase64Url(Cookie), - [_A, _B, _Cs] = re:split(?b2l(AuthSession), ":", - [{return, list}, {parts, 3}]) - catch - _:_Error -> - Reason = <<"Malformed AuthSession cookie. Please clear your cookies.">>, - throw({bad_request, Reason}) - end, - % Verify expiry and hash - CurrentTime = make_cookie_time(), - case chttpd_util:get_chttpd_auth_config("secret") of undefined -> - couch_log:debug("cookie auth secret is not set",[]), Req; - SecretStr -> - Secret = ?l2b(SecretStr), - case AuthModule:get_user_creds(Req, User) of - nil -> Req; - {ok, UserProps, _AuthCtx} -> - UserSalt = couch_util:get_value(<<"salt">>, UserProps, <<"">>), - FullSecret = <<Secret/binary, UserSalt/binary>>, - ExpectedHash = couch_util:hmac(sha, FullSecret, User ++ ":" ++ TimeStr), - Hash = ?l2b(HashStr), - Timeout = chttpd_util:get_chttpd_auth_config_integer( - "timeout", 600), - couch_log:debug("timeout ~p", [Timeout]), - case (catch erlang:list_to_integer(TimeStr, 16)) of - TimeStamp when CurrentTime < TimeStamp + Timeout -> - case couch_passwords:verify(ExpectedHash, Hash) of - true -> - TimeLeft = TimeStamp + Timeout - CurrentTime, - couch_log:debug("Successful cookie auth as: ~p", - [User]), - Req#httpd{user_ctx=#user_ctx{ - name=?l2b(User), - roles=couch_util:get_value(<<"roles">>, UserProps, []) - }, auth={FullSecret, TimeLeft < Timeout*0.9}}; - _Else -> - Req - end; - _Else -> - Req - end + [] -> + Req; + Cookie -> + [User, TimeStr, HashStr] = + try + AuthSession = couch_util:decodeBase64Url(Cookie), + [_A, _B, _Cs] = re:split( + ?b2l(AuthSession), + ":", + [{return, list}, {parts, 3}] + ) + catch + _:_Error -> + Reason = <<"Malformed AuthSession cookie. Please clear your cookies.">>, + throw({bad_request, Reason}) + end, + % Verify expiry and hash + CurrentTime = make_cookie_time(), + case chttpd_util:get_chttpd_auth_config("secret") of + undefined -> + couch_log:debug("cookie auth secret is not set", []), + Req; + SecretStr -> + Secret = ?l2b(SecretStr), + case AuthModule:get_user_creds(Req, User) of + nil -> + Req; + {ok, UserProps, _AuthCtx} -> + UserSalt = couch_util:get_value(<<"salt">>, UserProps, <<"">>), + FullSecret = <<Secret/binary, UserSalt/binary>>, + ExpectedHash = couch_util:hmac(sha, FullSecret, User ++ ":" ++ TimeStr), + Hash = ?l2b(HashStr), + Timeout = chttpd_util:get_chttpd_auth_config_integer( + "timeout", 600 + ), + couch_log:debug("timeout ~p", [Timeout]), + case (catch erlang:list_to_integer(TimeStr, 16)) of + TimeStamp when CurrentTime < TimeStamp + Timeout -> + case couch_passwords:verify(ExpectedHash, Hash) of + true -> + TimeLeft = TimeStamp + Timeout - CurrentTime, + couch_log:debug( + "Successful cookie auth as: ~p", + [User] + ), + Req#httpd{ + user_ctx = #user_ctx{ + name = ?l2b(User), + roles = couch_util:get_value( + <<"roles">>, UserProps, [] + ) + }, + auth = {FullSecret, TimeLeft < Timeout * 0.9} + }; + _Else -> + Req + end; + _Else -> + Req + end + end end - end end. -cookie_auth_header(#httpd{user_ctx=#user_ctx{name=null}}, _Headers) -> []; -cookie_auth_header(#httpd{user_ctx=#user_ctx{name=User}, auth={Secret, true}}=Req, Headers) -> +cookie_auth_header(#httpd{user_ctx = #user_ctx{name = null}}, _Headers) -> + []; +cookie_auth_header(#httpd{user_ctx = #user_ctx{name = User}, auth = {Secret, true}} = Req, Headers) -> % Note: we only set the AuthSession cookie if: % * a valid AuthSession cookie has been received % * we are outside a 10% timeout window @@ -296,20 +355,24 @@ cookie_auth_header(#httpd{user_ctx=#user_ctx{name=User}, auth={Secret, true}}=Re CookieHeader = couch_util:get_value("Set-Cookie", Headers, ""), Cookies = mochiweb_cookies:parse_cookie(CookieHeader), AuthSession = couch_util:get_value("AuthSession", Cookies), - if AuthSession == undefined -> - TimeStamp = make_cookie_time(), - [cookie_auth_cookie(Req, ?b2l(User), Secret, TimeStamp)]; - true -> - [] + if + AuthSession == undefined -> + TimeStamp = make_cookie_time(), + [cookie_auth_cookie(Req, ?b2l(User), Secret, TimeStamp)]; + true -> + [] end; -cookie_auth_header(_Req, _Headers) -> []. +cookie_auth_header(_Req, _Headers) -> + []. cookie_auth_cookie(Req, User, Secret, TimeStamp) -> SessionData = User ++ ":" ++ erlang:integer_to_list(TimeStamp, 16), Hash = couch_util:hmac(sha, Secret, SessionData), - mochiweb_cookies:cookie("AuthSession", + mochiweb_cookies:cookie( + "AuthSession", couch_util:encodeBase64Url(SessionData ++ ":" ++ ?b2l(Hash)), - [{path, "/"}] ++ cookie_scheme(Req) ++ max_age() ++ cookie_domain() ++ same_site()). + [{path, "/"}] ++ cookie_scheme(Req) ++ max_age() ++ cookie_domain() ++ same_site() + ). ensure_cookie_auth_secret() -> case chttpd_util:get_chttpd_auth_config("secret") of @@ -317,7 +380,8 @@ ensure_cookie_auth_secret() -> NewSecret = ?b2l(couch_uuids:random()), config:set("chttpd_auth", "secret", NewSecret), NewSecret; - Secret -> Secret + Secret -> + Secret end. % session handlers @@ -325,27 +389,32 @@ ensure_cookie_auth_secret() -> handle_session_req(Req) -> handle_session_req(Req, couch_auth_cache). -handle_session_req(#httpd{method='POST', mochi_req=MochiReq}=Req, AuthModule) -> +handle_session_req(#httpd{method = 'POST', mochi_req = MochiReq} = Req, AuthModule) -> ReqBody = MochiReq:recv_body(), - Form = case MochiReq:get_primary_header_value("content-type") of - % content type should be json - "application/x-www-form-urlencoded" ++ _ -> - mochiweb_util:parse_qs(ReqBody); - "application/json" ++ _ -> - {Pairs} = ?JSON_DECODE(maybe_decompress(Req, ReqBody)), - lists:map(fun({Key, Value}) -> - {?b2l(Key), ?b2l(Value)} - end, Pairs); - _ -> - [] - end, + Form = + case MochiReq:get_primary_header_value("content-type") of + % content type should be json + "application/x-www-form-urlencoded" ++ _ -> + mochiweb_util:parse_qs(ReqBody); + "application/json" ++ _ -> + {Pairs} = ?JSON_DECODE(maybe_decompress(Req, ReqBody)), + lists:map( + fun({Key, Value}) -> + {?b2l(Key), ?b2l(Value)} + end, + Pairs + ); + _ -> + [] + end, UserName = ?l2b(extract_username(Form)), Password = ?l2b(couch_util:get_value("password", Form, "")), - couch_log:debug("Attempt Login: ~s",[UserName]), - {ok, UserProps, _AuthCtx} = case AuthModule:get_user_creds(Req, UserName) of - nil -> {ok, [], nil}; - Result -> Result - end, + couch_log:debug("Attempt Login: ~s", [UserName]), + {ok, UserProps, _AuthCtx} = + case AuthModule:get_user_creds(Req, UserName) of + nil -> {ok, [], nil}; + Result -> Result + end, case authenticate(Password, UserProps) of true -> verify_totp(UserProps, Form), @@ -353,68 +422,102 @@ handle_session_req(#httpd{method='POST', mochi_req=MochiReq}=Req, AuthModule) -> Secret = ?l2b(ensure_cookie_auth_secret()), UserSalt = couch_util:get_value(<<"salt">>, UserProps), CurrentTime = make_cookie_time(), - Cookie = cookie_auth_cookie(Req, ?b2l(UserName), <<Secret/binary, UserSalt/binary>>, CurrentTime), + Cookie = cookie_auth_cookie( + Req, ?b2l(UserName), <<Secret/binary, UserSalt/binary>>, CurrentTime + ), % TODO document the "next" feature in Futon - {Code, Headers} = case couch_httpd:qs_value(Req, "next", nil) of - nil -> - {200, [Cookie]}; - Redirect -> - {302, [Cookie, {"Location", couch_httpd:absolute_uri(Req, Redirect)}]} - end, - send_json(Req#httpd{req_body=ReqBody}, Code, Headers, + {Code, Headers} = + case couch_httpd:qs_value(Req, "next", nil) of + nil -> + {200, [Cookie]}; + Redirect -> + {302, [Cookie, {"Location", couch_httpd:absolute_uri(Req, Redirect)}]} + end, + send_json( + Req#httpd{req_body = ReqBody}, + Code, + Headers, {[ {ok, true}, {name, UserName}, {roles, couch_util:get_value(<<"roles">>, UserProps, [])} - ]}); + ]} + ); false -> authentication_warning(Req, UserName), % clear the session - Cookie = mochiweb_cookies:cookie("AuthSession", "", [{path, "/"}] ++ cookie_scheme(Req)), - {Code, Headers} = case couch_httpd:qs_value(Req, "fail", nil) of - nil -> - {401, [Cookie]}; - Redirect -> - {302, [Cookie, {"Location", couch_httpd:absolute_uri(Req, Redirect)}]} - end, - send_json(Req, Code, Headers, {[{error, <<"unauthorized">>},{reason, <<"Name or password is incorrect.">>}]}) + Cookie = mochiweb_cookies:cookie( + "AuthSession", "", [{path, "/"}] ++ cookie_scheme(Req) + ), + {Code, Headers} = + case couch_httpd:qs_value(Req, "fail", nil) of + nil -> + {401, [Cookie]}; + Redirect -> + {302, [Cookie, {"Location", couch_httpd:absolute_uri(Req, Redirect)}]} + end, + send_json( + Req, + Code, + Headers, + {[{error, <<"unauthorized">>}, {reason, <<"Name or password is incorrect.">>}]} + ) end; % get user info % GET /_session -handle_session_req(#httpd{method='GET', user_ctx=UserCtx}=Req, _AuthModule) -> +handle_session_req(#httpd{method = 'GET', user_ctx = UserCtx} = Req, _AuthModule) -> Name = UserCtx#user_ctx.name, ForceLogin = couch_httpd:qs_value(Req, "basic", "false"), case {Name, ForceLogin} of {null, "true"} -> throw({unauthorized, <<"Please login.">>}); {Name, _} -> - send_json(Req, {[ - % remove this ok - {ok, true}, - {<<"userCtx">>, {[ - {name, Name}, - {roles, UserCtx#user_ctx.roles} - ]}}, - {info, {[ - {authentication_handlers, [ - N || {N, _Fun} <- Req#httpd.authentication_handlers]} - ] ++ maybe_value(authenticated, UserCtx#user_ctx.handler, fun(Handler) -> - Handler - end) ++ maybe_value(authentication_db, config:get("chttpd_auth", "authentication_db"), fun(Val) -> - ?l2b(Val) - end)}} - ]}) + send_json( + Req, + {[ + % remove this ok + {ok, true}, + {<<"userCtx">>, + {[ + {name, Name}, + {roles, UserCtx#user_ctx.roles} + ]}}, + {info, { + [ + {authentication_handlers, [ + N + || {N, _Fun} <- Req#httpd.authentication_handlers + ]} + ] ++ + maybe_value(authenticated, UserCtx#user_ctx.handler, fun(Handler) -> + Handler + end) ++ + maybe_value( + authentication_db, + config:get("chttpd_auth", "authentication_db"), + fun(Val) -> + ?l2b(Val) + end + ) + }} + ]} + ) end; % logout by deleting the session -handle_session_req(#httpd{method='DELETE'}=Req, _AuthModule) -> - Cookie = mochiweb_cookies:cookie("AuthSession", "", [{path, "/"}] ++ - cookie_domain() ++ cookie_scheme(Req)), - {Code, Headers} = case couch_httpd:qs_value(Req, "next", nil) of - nil -> - {200, [Cookie]}; - Redirect -> - {302, [Cookie, {"Location", couch_httpd:absolute_uri(Req, Redirect)}]} - end, +handle_session_req(#httpd{method = 'DELETE'} = Req, _AuthModule) -> + Cookie = mochiweb_cookies:cookie( + "AuthSession", + "", + [{path, "/"}] ++ + cookie_domain() ++ cookie_scheme(Req) + ), + {Code, Headers} = + case couch_httpd:qs_value(Req, "next", nil) of + nil -> + {200, [Cookie]}; + Redirect -> + {302, [Cookie, {"Location", couch_httpd:absolute_uri(Req, Redirect)}]} + end, send_json(Req, Code, Headers, {[{ok, true}]}); handle_session_req(Req, _AuthModule) -> send_method_not_allowed(Req, "GET,HEAD,POST,DELETE"). @@ -433,22 +536,25 @@ extract_username(Form) -> end. maybe_value(_Key, undefined, _Fun) -> []; -maybe_value(Key, Else, Fun) -> - [{Key, Fun(Else)}]. +maybe_value(Key, Else, Fun) -> [{Key, Fun(Else)}]. authenticate(Pass, UserProps) -> UserSalt = couch_util:get_value(<<"salt">>, UserProps, <<>>), {PasswordHash, ExpectedHash} = case couch_util:get_value(<<"password_scheme">>, UserProps, <<"simple">>) of - <<"simple">> -> - {couch_passwords:simple(Pass, UserSalt), - couch_util:get_value(<<"password_sha">>, UserProps, nil)}; - <<"pbkdf2">> -> - Iterations = couch_util:get_value(<<"iterations">>, UserProps, 10000), - verify_iterations(Iterations), - {couch_passwords:pbkdf2(Pass, UserSalt, Iterations), - couch_util:get_value(<<"derived_key">>, UserProps, nil)} - end, + <<"simple">> -> + { + couch_passwords:simple(Pass, UserSalt), + couch_util:get_value(<<"password_sha">>, UserProps, nil) + }; + <<"pbkdf2">> -> + Iterations = couch_util:get_value(<<"iterations">>, UserProps, 10000), + verify_iterations(Iterations), + { + couch_passwords:pbkdf2(Pass, UserSalt, Iterations), + couch_util:get_value(<<"derived_key">>, UserProps, nil) + } + end, couch_passwords:verify(PasswordHash, ExpectedHash). verify_iterations(Iterations) when is_integer(Iterations) -> @@ -471,21 +577,25 @@ make_cookie_time() -> {NowMS, NowS, _} = os:timestamp(), NowMS * 1000000 + NowS. -cookie_scheme(#httpd{mochi_req=MochiReq}) -> +cookie_scheme(#httpd{mochi_req = MochiReq}) -> [{http_only, true}] ++ - case MochiReq:get(scheme) of - http -> []; - https -> [{secure, true}] - end. + case MochiReq:get(scheme) of + http -> []; + https -> [{secure, true}] + end. max_age() -> - case chttpd_util:get_chttpd_auth_config_boolean( - "allow_persistent_cookies", true) of + case + chttpd_util:get_chttpd_auth_config_boolean( + "allow_persistent_cookies", true + ) + of false -> []; true -> Timeout = chttpd_util:get_chttpd_auth_config_integer( - "timeout", 600), + "timeout", 600 + ), [{max_age, Timeout}] end. @@ -496,20 +606,22 @@ cookie_domain() -> _ -> [{domain, Domain}] end. - same_site() -> SameSite = chttpd_util:get_chttpd_auth_config("same_site", ""), case string:to_lower(SameSite) of - "" -> []; - "none" -> [{same_site, none}]; - "lax" -> [{same_site, lax}]; - "strict" -> [{same_site, strict}]; + "" -> + []; + "none" -> + [{same_site, none}]; + "lax" -> + [{same_site, lax}]; + "strict" -> + [{same_site, strict}]; _ -> - couch_log:error("invalid config value couch_httpd_auth.same_site: ~p ",[SameSite]), + couch_log:error("invalid config value couch_httpd_auth.same_site: ~p ", [SameSite]), [] end. - reject_if_totp(User) -> case get_totp_config(User) of undefined -> @@ -525,7 +637,8 @@ verify_totp(User, Form) -> {Props} -> Key = couch_base32:decode(couch_util:get_value(<<"key">>, Props)), Alg = couch_util:to_existing_atom( - couch_util:get_value(<<"algorithm">>, Props, <<"sha">>)), + couch_util:get_value(<<"algorithm">>, Props, <<"sha">>) + ), Len = couch_util:get_value(<<"length">>, Props, 6), Token = ?l2b(couch_util:get_value("token", Form, "")), verify_token(Alg, Key, Len, Token) @@ -536,12 +649,17 @@ get_totp_config(User) -> verify_token(Alg, Key, Len, Token) -> Now = make_cookie_time(), - Tokens = [generate_token(Alg, Key, Len, Now - 30), - generate_token(Alg, Key, Len, Now), - generate_token(Alg, Key, Len, Now + 30)], + Tokens = [ + generate_token(Alg, Key, Len, Now - 30), + generate_token(Alg, Key, Len, Now), + generate_token(Alg, Key, Len, Now + 30) + ], %% evaluate all tokens in constant time - Match = lists:foldl(fun(T, Acc) -> couch_util:verify(T, Token) or Acc end, - false, Tokens), + Match = lists:foldl( + fun(T, Acc) -> couch_util:verify(T, Token) or Acc end, + false, + Tokens + ), case Match of true -> ok; @@ -553,17 +671,20 @@ generate_token(Alg, Key, Len, Timestamp) -> integer_to_binary(couch_totp:generate(Alg, Key, Timestamp, 30, Len), Len). integer_to_binary(Int, Len) when is_integer(Int), is_integer(Len) -> - Unpadded = case erlang:function_exported(erlang, integer_to_binary, 1) of - true -> - erlang:integer_to_binary(Int); - false -> - ?l2b(integer_to_list(Int)) - end, + Unpadded = + case erlang:function_exported(erlang, integer_to_binary, 1) of + true -> + erlang:integer_to_binary(Int); + false -> + ?l2b(integer_to_list(Int)) + end, Padding = binary:copy(<<"0">>, Len), Padded = <<Padding/binary, Unpadded/binary>>, binary:part(Padded, byte_size(Padded), -Len). authentication_warning(#httpd{mochi_req = Req}, User) -> Peer = Req:get(peer), - couch_log:warning("~p: Authentication failed for user ~s from ~s", - [?MODULE, User, Peer]). + couch_log:warning( + "~p: Authentication failed for user ~s from ~s", + [?MODULE, User, Peer] + ). diff --git a/src/couch/src/couch_httpd_db.erl b/src/couch/src/couch_httpd_db.erl index 2418c1a4c..e82eea7f7 100644 --- a/src/couch/src/couch_httpd_db.erl +++ b/src/couch/src/couch_httpd_db.erl @@ -16,17 +16,36 @@ -include_lib("couch/include/couch_db.hrl"). --export([handle_request/1, handle_compact_req/2, handle_design_req/2, - db_req/2, couch_doc_open/4, handle_db_changes_req/2, +-export([ + handle_request/1, + handle_compact_req/2, + handle_design_req/2, + db_req/2, + couch_doc_open/4, + handle_db_changes_req/2, update_doc_result_to_json/1, update_doc_result_to_json/2, - handle_design_info_req/3, parse_copy_destination_header/1, - parse_changes_query/2, handle_changes_req/4]). - --import(couch_httpd, - [send_json/2,send_json/3,send_json/4,send_method_not_allowed/2, - start_json_response/2,send_chunk/2,last_chunk/1,end_json_response/1, - start_chunked_response/3, absolute_uri/2, send/2, - start_response_length/4, send_error/4]). + handle_design_info_req/3, + parse_copy_destination_header/1, + parse_changes_query/2, + handle_changes_req/4 +]). + +-import( + couch_httpd, + [ + send_json/2, send_json/3, send_json/4, + send_method_not_allowed/2, + start_json_response/2, + send_chunk/2, + last_chunk/1, + end_json_response/1, + start_chunked_response/3, + absolute_uri/2, + send/2, + start_response_length/4, + send_error/4 + ] +). -record(doc_query_args, { options = [], @@ -37,134 +56,148 @@ }). % Database request handlers -handle_request(#httpd{path_parts=[DbName|RestParts],method=Method, - db_url_handlers=DbUrlHandlers}=Req)-> +handle_request( + #httpd{ + path_parts = [DbName | RestParts], + method = Method, + db_url_handlers = DbUrlHandlers + } = Req +) -> case {Method, RestParts} of - {'PUT', []} -> - create_db_req(Req, DbName); - {'DELETE', []} -> - % if we get ?rev=... the user is using a faulty script where the - % document id is empty by accident. Let them recover safely. - case couch_httpd:qs_value(Req, "rev", false) of - false -> delete_db_req(Req, DbName); - _Rev -> throw({bad_request, - "You tried to DELETE a database with a ?rev= parameter. " - ++ "Did you mean to DELETE a document instead?"}) - end; - {_, []} -> - do_db_req(Req, fun db_req/2); - {_, [SecondPart|_]} -> - Handler = couch_util:dict_find(SecondPart, DbUrlHandlers, fun db_req/2), - do_db_req(Req, Handler) + {'PUT', []} -> + create_db_req(Req, DbName); + {'DELETE', []} -> + % if we get ?rev=... the user is using a faulty script where the + % document id is empty by accident. Let them recover safely. + case couch_httpd:qs_value(Req, "rev", false) of + false -> + delete_db_req(Req, DbName); + _Rev -> + throw( + {bad_request, + "You tried to DELETE a database with a ?rev= parameter. " ++ + "Did you mean to DELETE a document instead?"} + ) + end; + {_, []} -> + do_db_req(Req, fun db_req/2); + {_, [SecondPart | _]} -> + Handler = couch_util:dict_find(SecondPart, DbUrlHandlers, fun db_req/2), + do_db_req(Req, Handler) end. - handle_db_changes_req(Req, Db) -> ChangesArgs = parse_changes_query(Req, Db), ChangesFun = couch_changes:handle_db_changes(ChangesArgs, Req, Db), handle_changes_req(Req, Db, ChangesArgs, ChangesFun). - -handle_changes_req(#httpd{method='POST'}=Req, Db, ChangesArgs, ChangesFun) -> +handle_changes_req(#httpd{method = 'POST'} = Req, Db, ChangesArgs, ChangesFun) -> couch_httpd:validate_ctype(Req, "application/json"), handle_changes_req1(Req, Db, ChangesArgs, ChangesFun); -handle_changes_req(#httpd{method='GET'}=Req, Db, ChangesArgs, ChangesFun) -> +handle_changes_req(#httpd{method = 'GET'} = Req, Db, ChangesArgs, ChangesFun) -> handle_changes_req1(Req, Db, ChangesArgs, ChangesFun); -handle_changes_req(#httpd{}=Req, _Db, _ChangesArgs, _ChangesFun) -> +handle_changes_req(#httpd{} = Req, _Db, _ChangesArgs, _ChangesFun) -> couch_httpd:send_method_not_allowed(Req, "GET,HEAD,POST"). handle_changes_req1(Req, Db, ChangesArgs, ChangesFun) -> DbName = couch_db:name(Db), AuthDbName = ?l2b(config:get("couch_httpd_auth", "authentication_db")), case AuthDbName of - DbName -> - % in the authentication database, _changes is admin-only. - ok = couch_db:check_is_admin(Db); - _Else -> - % on other databases, _changes is free for all. - ok + DbName -> + % in the authentication database, _changes is admin-only. + ok = couch_db:check_is_admin(Db); + _Else -> + % on other databases, _changes is free for all. + ok end, MakeCallback = fun(Resp) -> - fun({change, {ChangeProp}=Change, _}, "eventsource") -> - Seq = proplists:get_value(<<"seq">>, ChangeProp), - couch_httpd:send_chunk(Resp, ["data: ", ?JSON_ENCODE(Change), - "\n", "id: ", ?JSON_ENCODE(Seq), - "\n\n"]); - ({change, Change, _}, "continuous") -> - couch_httpd:send_chunk(Resp, [?JSON_ENCODE(Change) | "\n"]); - ({change, Change, Prepend}, _) -> - couch_httpd:send_chunk(Resp, [Prepend, ?JSON_ENCODE(Change)]); - (start, "eventsource") -> - ok; - (start, "continuous") -> - ok; - (start, _) -> - couch_httpd:send_chunk(Resp, "{\"results\":[\n"); - ({stop, _EndSeq}, "eventsource") -> - couch_httpd:end_json_response(Resp); - ({stop, EndSeq}, "continuous") -> - couch_httpd:send_chunk( - Resp, - [?JSON_ENCODE({[{<<"last_seq">>, EndSeq}]}) | "\n"] - ), - couch_httpd:end_json_response(Resp); - ({stop, EndSeq}, _) -> - couch_httpd:send_chunk( - Resp, - io_lib:format("\n],\n\"last_seq\":~w}\n", [EndSeq]) - ), - couch_httpd:end_json_response(Resp); - (timeout, "eventsource") -> - couch_httpd:send_chunk(Resp, "event: heartbeat\ndata: \n\n"); - (timeout, _) -> - couch_httpd:send_chunk(Resp, "\n") + fun + ({change, {ChangeProp} = Change, _}, "eventsource") -> + Seq = proplists:get_value(<<"seq">>, ChangeProp), + couch_httpd:send_chunk(Resp, [ + "data: ", + ?JSON_ENCODE(Change), + "\n", + "id: ", + ?JSON_ENCODE(Seq), + "\n\n" + ]); + ({change, Change, _}, "continuous") -> + couch_httpd:send_chunk(Resp, [?JSON_ENCODE(Change) | "\n"]); + ({change, Change, Prepend}, _) -> + couch_httpd:send_chunk(Resp, [Prepend, ?JSON_ENCODE(Change)]); + (start, "eventsource") -> + ok; + (start, "continuous") -> + ok; + (start, _) -> + couch_httpd:send_chunk(Resp, "{\"results\":[\n"); + ({stop, _EndSeq}, "eventsource") -> + couch_httpd:end_json_response(Resp); + ({stop, EndSeq}, "continuous") -> + couch_httpd:send_chunk( + Resp, + [?JSON_ENCODE({[{<<"last_seq">>, EndSeq}]}) | "\n"] + ), + couch_httpd:end_json_response(Resp); + ({stop, EndSeq}, _) -> + couch_httpd:send_chunk( + Resp, + io_lib:format("\n],\n\"last_seq\":~w}\n", [EndSeq]) + ), + couch_httpd:end_json_response(Resp); + (timeout, "eventsource") -> + couch_httpd:send_chunk(Resp, "event: heartbeat\ndata: \n\n"); + (timeout, _) -> + couch_httpd:send_chunk(Resp, "\n") end end, - WrapperFun = case ChangesArgs#changes_args.feed of - "normal" -> - {ok, Info} = couch_db:get_db_info(Db), - CurrentEtag = couch_httpd:make_etag(Info), - fun(FeedChangesFun) -> - couch_httpd:etag_respond( - Req, - CurrentEtag, - fun() -> - {ok, Resp} = couch_httpd:start_json_response( - Req, 200, [{"ETag", CurrentEtag}] - ), + WrapperFun = + case ChangesArgs#changes_args.feed of + "normal" -> + {ok, Info} = couch_db:get_db_info(Db), + CurrentEtag = couch_httpd:make_etag(Info), + fun(FeedChangesFun) -> + couch_httpd:etag_respond( + Req, + CurrentEtag, + fun() -> + {ok, Resp} = couch_httpd:start_json_response( + Req, 200, [{"ETag", CurrentEtag}] + ), + FeedChangesFun(MakeCallback(Resp)) + end + ) + end; + "eventsource" -> + Headers = [ + {"Content-Type", "text/event-stream"}, + {"Cache-Control", "no-cache"} + ], + {ok, Resp} = couch_httpd:start_chunked_response(Req, 200, Headers), + fun(FeedChangesFun) -> + FeedChangesFun(MakeCallback(Resp)) + end; + _ -> + % "longpoll" or "continuous" + {ok, Resp} = couch_httpd:start_json_response(Req, 200), + fun(FeedChangesFun) -> FeedChangesFun(MakeCallback(Resp)) end - ) - end; - "eventsource" -> - Headers = [ - {"Content-Type", "text/event-stream"}, - {"Cache-Control", "no-cache"} - ], - {ok, Resp} = couch_httpd:start_chunked_response(Req, 200, Headers), - fun(FeedChangesFun) -> - FeedChangesFun(MakeCallback(Resp)) - end; - _ -> - % "longpoll" or "continuous" - {ok, Resp} = couch_httpd:start_json_response(Req, 200), - fun(FeedChangesFun) -> - FeedChangesFun(MakeCallback(Resp)) - end - end, + end, couch_stats:increment_counter( - [couchdb, httpd, clients_requesting_changes]), + [couchdb, httpd, clients_requesting_changes] + ), try WrapperFun(ChangesFun) after couch_stats:decrement_counter( - [couchdb, httpd, clients_requesting_changes]) + [couchdb, httpd, clients_requesting_changes] + ) end. - - -handle_compact_req(#httpd{method='POST'}=Req, Db) -> +handle_compact_req(#httpd{method = 'POST'} = Req, Db) -> case Req#httpd.path_parts of [_DbName, <<"_compact">>] -> ok = couch_db:check_is_admin(Db), @@ -179,24 +212,30 @@ handle_compact_req(#httpd{method='POST'}=Req, Db) -> ), couch_mrview_http:handle_compact_req(Req, Db, DDoc) end; - handle_compact_req(Req, _Db) -> send_method_not_allowed(Req, "POST"). - -handle_design_req(#httpd{ - path_parts=[_DbName, _Design, DesignName, <<"_",_/binary>> = Action | _Rest], +handle_design_req( + #httpd{ + path_parts = [_DbName, _Design, DesignName, <<"_", _/binary>> = Action | _Rest], design_url_handlers = DesignUrlHandlers - }=Req, Db) -> + } = Req, + Db +) -> case couch_db:is_system_db(Db) of - true -> - case (catch couch_db:check_is_admin(Db)) of - ok -> ok; - _ -> - throw({forbidden, <<"Only admins can access design document", - " actions for system databases.">>}) - end; - false -> ok + true -> + case (catch couch_db:check_is_admin(Db)) of + ok -> + ok; + _ -> + throw( + {forbidden, + <<"Only admins can access design document", + " actions for system databases.">>} + ) + end; + false -> + ok end, % maybe load ddoc through fabric @@ -212,295 +251,315 @@ handle_design_req(#httpd{ throw({not_found, <<"missing handler: ", Action/binary>>}) end), Handler(Req, Db, DDoc); - handle_design_req(Req, Db) -> db_req(Req, Db). -handle_design_info_req(#httpd{ - method='GET', - path_parts=[_DbName, _Design, DesignName, _] - }=Req, Db, _DDoc) -> +handle_design_info_req( + #httpd{ + method = 'GET', + path_parts = [_DbName, _Design, DesignName, _] + } = Req, + Db, + _DDoc +) -> DesignId = <<"_design/", DesignName/binary>>, DDoc = couch_httpd_db:couch_doc_open(Db, DesignId, nil, [ejson_body]), couch_mrview_http:handle_info_req(Req, Db, DDoc). -create_db_req(#httpd{user_ctx=UserCtx}=Req, DbName) -> +create_db_req(#httpd{user_ctx = UserCtx} = Req, DbName) -> ok = couch_httpd:verify_is_server_admin(Req), - Engine = case couch_httpd:qs_value(Req, "engine") of - EngineStr when is_list(EngineStr) -> - [{engine, iolist_to_binary(EngineStr)}]; - _ -> - [] - end, + Engine = + case couch_httpd:qs_value(Req, "engine") of + EngineStr when is_list(EngineStr) -> + [{engine, iolist_to_binary(EngineStr)}]; + _ -> + [] + end, case couch_server:create(DbName, [{user_ctx, UserCtx}] ++ Engine) of - {ok, Db} -> - couch_db:close(Db), - DbUrl = absolute_uri(Req, "/" ++ couch_util:url_encode(DbName)), - send_json(Req, 201, [{"Location", DbUrl}], {[{ok, true}]}); - Error -> - throw(Error) + {ok, Db} -> + couch_db:close(Db), + DbUrl = absolute_uri(Req, "/" ++ couch_util:url_encode(DbName)), + send_json(Req, 201, [{"Location", DbUrl}], {[{ok, true}]}); + Error -> + throw(Error) end. -delete_db_req(#httpd{user_ctx=UserCtx}=Req, DbName) -> +delete_db_req(#httpd{user_ctx = UserCtx} = Req, DbName) -> ok = couch_httpd:verify_is_server_admin(Req), - Options = case couch_httpd:qs_value(Req, "sync") of - "true" -> [sync, {user_ctx, UserCtx}]; - _ -> [{user_ctx, UserCtx}] - end, + Options = + case couch_httpd:qs_value(Req, "sync") of + "true" -> [sync, {user_ctx, UserCtx}]; + _ -> [{user_ctx, UserCtx}] + end, case couch_server:delete(DbName, Options) of - ok -> - send_json(Req, 200, {[{ok, true}]}); - Error -> - throw(Error) + ok -> + send_json(Req, 200, {[{ok, true}]}); + Error -> + throw(Error) end. -do_db_req(#httpd{user_ctx=UserCtx,path_parts=[DbName|_]}=Req, Fun) -> +do_db_req(#httpd{user_ctx = UserCtx, path_parts = [DbName | _]} = Req, Fun) -> case couch_db:open(DbName, [{user_ctx, UserCtx}]) of - {ok, Db} -> - try - Fun(Req, Db) - after - catch couch_db:close(Db) - end; - Error -> - throw(Error) + {ok, Db} -> + try + Fun(Req, Db) + after + catch couch_db:close(Db) + end; + Error -> + throw(Error) end. -db_req(#httpd{method='GET',path_parts=[_DbName]}=Req, Db) -> +db_req(#httpd{method = 'GET', path_parts = [_DbName]} = Req, Db) -> {ok, DbInfo} = couch_db:get_db_info(Db), send_json(Req, {DbInfo}); - -db_req(#httpd{method='POST',path_parts=[_DbName]}=Req, Db) -> +db_req(#httpd{method = 'POST', path_parts = [_DbName]} = Req, Db) -> couch_httpd:validate_ctype(Req, "application/json"), Doc = couch_db:doc_from_json_obj_validate(Db, couch_httpd:json_body(Req)), validate_attachment_names(Doc), - Doc2 = case Doc#doc.id of - <<"">> -> - Doc#doc{id=couch_uuids:new(), revs={0, []}}; - _ -> - Doc - end, + Doc2 = + case Doc#doc.id of + <<"">> -> + Doc#doc{id = couch_uuids:new(), revs = {0, []}}; + _ -> + Doc + end, DocId = Doc2#doc.id, update_doc(Req, Db, DocId, Doc2); - -db_req(#httpd{path_parts=[_DbName]}=Req, _Db) -> +db_req(#httpd{path_parts = [_DbName]} = Req, _Db) -> send_method_not_allowed(Req, "DELETE,GET,HEAD,POST"); - -db_req(#httpd{method='POST',path_parts=[_,<<"_ensure_full_commit">>]}=Req, Db) -> +db_req(#httpd{method = 'POST', path_parts = [_, <<"_ensure_full_commit">>]} = Req, Db) -> couch_httpd:validate_ctype(Req, "application/json"), _ = couch_httpd:body(Req), StartTime = couch_db:get_instance_start_time(Db), - send_json(Req, 201, {[ - {ok, true}, - {instance_start_time, StartTime} - ]}); - -db_req(#httpd{path_parts=[_,<<"_ensure_full_commit">>]}=Req, _Db) -> + send_json( + Req, + 201, + {[ + {ok, true}, + {instance_start_time, StartTime} + ]} + ); +db_req(#httpd{path_parts = [_, <<"_ensure_full_commit">>]} = Req, _Db) -> send_method_not_allowed(Req, "POST"); - -db_req(#httpd{method='POST',path_parts=[_,<<"_bulk_docs">>]}=Req, Db) -> +db_req(#httpd{method = 'POST', path_parts = [_, <<"_bulk_docs">>]} = Req, Db) -> couch_stats:increment_counter([couchdb, httpd, bulk_requests]), couch_httpd:validate_ctype(Req, "application/json"), {JsonProps} = couch_httpd:json_body_obj(Req), case couch_util:get_value(<<"docs">>, JsonProps) of - undefined -> - send_error(Req, 400, <<"bad_request">>, <<"Missing JSON list of 'docs'">>); - DocsArray -> - couch_stats:update_histogram([couchdb, httpd, bulk_docs], length(DocsArray)), - case couch_httpd:header_value(Req, "X-Couch-Full-Commit") of - "true" -> - Options = [full_commit]; - "false" -> - Options = [delay_commit]; - _ -> - Options = [] - end, - case couch_util:get_value(<<"new_edits">>, JsonProps, true) of - true -> - Docs = lists:map( - fun({ObjProps} = JsonObj) -> - Doc = couch_db:doc_from_json_obj_validate(Db, JsonObj), - validate_attachment_names(Doc), - Id = case Doc#doc.id of - <<>> -> couch_uuids:new(); - Id0 -> Id0 - end, - case couch_util:get_value(<<"_rev">>, ObjProps) of - undefined -> - Revs = {0, []}; - Rev -> - {Pos, RevId} = couch_doc:parse_rev(Rev), - Revs = {Pos, [RevId]} - end, - Doc#doc{id=Id,revs=Revs} - end, - DocsArray), - Options2 = - case couch_util:get_value(<<"all_or_nothing">>, JsonProps) of - true -> [all_or_nothing|Options]; - _ -> Options + undefined -> + send_error(Req, 400, <<"bad_request">>, <<"Missing JSON list of 'docs'">>); + DocsArray -> + couch_stats:update_histogram([couchdb, httpd, bulk_docs], length(DocsArray)), + case couch_httpd:header_value(Req, "X-Couch-Full-Commit") of + "true" -> + Options = [full_commit]; + "false" -> + Options = [delay_commit]; + _ -> + Options = [] end, - case couch_db:update_docs(Db, Docs, Options2) of - {ok, Results} -> - % output the results - DocResults = lists:zipwith(fun update_doc_result_to_json/2, - Docs, Results), - send_json(Req, 201, DocResults); - {aborted, Errors} -> - ErrorsJson = - lists:map(fun update_doc_result_to_json/1, Errors), - send_json(Req, 417, ErrorsJson) - end; - false -> - Docs = lists:map(fun(JsonObj) -> - Doc = couch_db:doc_from_json_obj_validate(Db, JsonObj), - validate_attachment_names(Doc), - Doc - end, DocsArray), - {ok, Errors} = couch_db:update_docs(Db, Docs, Options, replicated_changes), - ErrorsJson = - lists:map(fun update_doc_result_to_json/1, Errors), - send_json(Req, 201, ErrorsJson) - end + case couch_util:get_value(<<"new_edits">>, JsonProps, true) of + true -> + Docs = lists:map( + fun({ObjProps} = JsonObj) -> + Doc = couch_db:doc_from_json_obj_validate(Db, JsonObj), + validate_attachment_names(Doc), + Id = + case Doc#doc.id of + <<>> -> couch_uuids:new(); + Id0 -> Id0 + end, + case couch_util:get_value(<<"_rev">>, ObjProps) of + undefined -> + Revs = {0, []}; + Rev -> + {Pos, RevId} = couch_doc:parse_rev(Rev), + Revs = {Pos, [RevId]} + end, + Doc#doc{id = Id, revs = Revs} + end, + DocsArray + ), + Options2 = + case couch_util:get_value(<<"all_or_nothing">>, JsonProps) of + true -> [all_or_nothing | Options]; + _ -> Options + end, + case couch_db:update_docs(Db, Docs, Options2) of + {ok, Results} -> + % output the results + DocResults = lists:zipwith( + fun update_doc_result_to_json/2, + Docs, + Results + ), + send_json(Req, 201, DocResults); + {aborted, Errors} -> + ErrorsJson = + lists:map(fun update_doc_result_to_json/1, Errors), + send_json(Req, 417, ErrorsJson) + end; + false -> + Docs = lists:map( + fun(JsonObj) -> + Doc = couch_db:doc_from_json_obj_validate(Db, JsonObj), + validate_attachment_names(Doc), + Doc + end, + DocsArray + ), + {ok, Errors} = couch_db:update_docs(Db, Docs, Options, replicated_changes), + ErrorsJson = + lists:map(fun update_doc_result_to_json/1, Errors), + send_json(Req, 201, ErrorsJson) + end end; -db_req(#httpd{path_parts=[_,<<"_bulk_docs">>]}=Req, _Db) -> +db_req(#httpd{path_parts = [_, <<"_bulk_docs">>]} = Req, _Db) -> send_method_not_allowed(Req, "POST"); - -db_req(#httpd{method='POST',path_parts=[_,<<"_purge">>]}=Req, Db) -> +db_req(#httpd{method = 'POST', path_parts = [_, <<"_purge">>]} = Req, Db) -> couch_stats:increment_counter([couchdb, httpd, purge_requests]), couch_httpd:validate_ctype(Req, "application/json"), {IdRevs} = couch_httpd:json_body_obj(Req), - PurgeReqs = lists:map(fun({Id, JsonRevs}) -> - {couch_uuids:new(), Id, couch_doc:parse_revs(JsonRevs)} - end, IdRevs), + PurgeReqs = lists:map( + fun({Id, JsonRevs}) -> + {couch_uuids:new(), Id, couch_doc:parse_revs(JsonRevs)} + end, + IdRevs + ), {ok, Replies} = couch_db:purge_docs(Db, PurgeReqs), - Results = lists:zipwith(fun({Id, _}, {ok, Reply}) -> - {Id, couch_doc:revs_to_strs(Reply)} - end, IdRevs, Replies), + Results = lists:zipwith( + fun({Id, _}, {ok, Reply}) -> + {Id, couch_doc:revs_to_strs(Reply)} + end, + IdRevs, + Replies + ), {ok, Db2} = couch_db:reopen(Db), PurgeSeq = couch_db:get_purge_seq(Db2), send_json(Req, 200, {[{purge_seq, PurgeSeq}, {purged, {Results}}]}); - -db_req(#httpd{path_parts=[_,<<"_purge">>]}=Req, _Db) -> +db_req(#httpd{path_parts = [_, <<"_purge">>]} = Req, _Db) -> send_method_not_allowed(Req, "POST"); - -db_req(#httpd{method='POST',path_parts=[_,<<"_missing_revs">>]}=Req, Db) -> +db_req(#httpd{method = 'POST', path_parts = [_, <<"_missing_revs">>]} = Req, Db) -> couch_httpd:validate_ctype(Req, "application/json"), {JsonDocIdRevs} = couch_httpd:json_body_obj(Req), - JsonDocIdRevs2 = [{Id, [couch_doc:parse_rev(RevStr) || RevStr <- RevStrs]} || {Id, RevStrs} <- JsonDocIdRevs], + JsonDocIdRevs2 = [ + {Id, [couch_doc:parse_rev(RevStr) || RevStr <- RevStrs]} + || {Id, RevStrs} <- JsonDocIdRevs + ], {ok, Results} = couch_db:get_missing_revs(Db, JsonDocIdRevs2), Results2 = [{Id, couch_doc:revs_to_strs(Revs)} || {Id, Revs, _} <- Results], - send_json(Req, {[ - {missing_revs, {Results2}} - ]}); - -db_req(#httpd{path_parts=[_,<<"_missing_revs">>]}=Req, _Db) -> + send_json( + Req, + {[ + {missing_revs, {Results2}} + ]} + ); +db_req(#httpd{path_parts = [_, <<"_missing_revs">>]} = Req, _Db) -> send_method_not_allowed(Req, "POST"); - -db_req(#httpd{method='POST',path_parts=[_,<<"_revs_diff">>]}=Req, Db) -> +db_req(#httpd{method = 'POST', path_parts = [_, <<"_revs_diff">>]} = Req, Db) -> couch_httpd:validate_ctype(Req, "application/json"), {JsonDocIdRevs} = couch_httpd:json_body_obj(Req), JsonDocIdRevs2 = [{Id, couch_doc:parse_revs(RevStrs)} || {Id, RevStrs} <- JsonDocIdRevs], {ok, Results} = couch_db:get_missing_revs(Db, JsonDocIdRevs2), Results2 = - lists:map(fun({Id, MissingRevs, PossibleAncestors}) -> - {Id, - {[{missing, couch_doc:revs_to_strs(MissingRevs)}] ++ - if PossibleAncestors == [] -> - []; - true -> - [{possible_ancestors, - couch_doc:revs_to_strs(PossibleAncestors)}] - end}} - end, Results), + lists:map( + fun({Id, MissingRevs, PossibleAncestors}) -> + {Id, { + [{missing, couch_doc:revs_to_strs(MissingRevs)}] ++ + if + PossibleAncestors == [] -> + []; + true -> + [{possible_ancestors, couch_doc:revs_to_strs(PossibleAncestors)}] + end + }} + end, + Results + ), send_json(Req, {Results2}); - -db_req(#httpd{path_parts=[_,<<"_revs_diff">>]}=Req, _Db) -> +db_req(#httpd{path_parts = [_, <<"_revs_diff">>]} = Req, _Db) -> send_method_not_allowed(Req, "POST"); - -db_req(#httpd{method='PUT',path_parts=[_,<<"_security">>]}=Req, Db) -> +db_req(#httpd{method = 'PUT', path_parts = [_, <<"_security">>]} = Req, Db) -> SecObj = couch_httpd:json_body(Req), ok = couch_db:set_security(Db, SecObj), send_json(Req, {[{<<"ok">>, true}]}); - -db_req(#httpd{method='GET',path_parts=[_,<<"_security">>]}=Req, Db) -> +db_req(#httpd{method = 'GET', path_parts = [_, <<"_security">>]} = Req, Db) -> send_json(Req, couch_db:get_security(Db)); - -db_req(#httpd{path_parts=[_,<<"_security">>]}=Req, _Db) -> +db_req(#httpd{path_parts = [_, <<"_security">>]} = Req, _Db) -> send_method_not_allowed(Req, "PUT,GET"); - -db_req(#httpd{method='PUT',path_parts=[_,<<"_revs_limit">>]}=Req, - Db) -> +db_req( + #httpd{method = 'PUT', path_parts = [_, <<"_revs_limit">>]} = Req, + Db +) -> Limit = couch_httpd:json_body(Req), - case is_integer(Limit) of - true -> - ok = couch_db:set_revs_limit(Db, Limit), - send_json(Req, {[{<<"ok">>, true}]}); - false -> - throw({bad_request, <<"Rev limit has to be an integer">>}) - end; - -db_req(#httpd{method='GET',path_parts=[_,<<"_revs_limit">>]}=Req, Db) -> + case is_integer(Limit) of + true -> + ok = couch_db:set_revs_limit(Db, Limit), + send_json(Req, {[{<<"ok">>, true}]}); + false -> + throw({bad_request, <<"Rev limit has to be an integer">>}) + end; +db_req(#httpd{method = 'GET', path_parts = [_, <<"_revs_limit">>]} = Req, Db) -> send_json(Req, couch_db:get_revs_limit(Db)); - -db_req(#httpd{path_parts=[_,<<"_revs_limit">>]}=Req, _Db) -> +db_req(#httpd{path_parts = [_, <<"_revs_limit">>]} = Req, _Db) -> send_method_not_allowed(Req, "PUT,GET"); - % Special case to enable using an unencoded slash in the URL of design docs, % as slashes in document IDs must otherwise be URL encoded. -db_req(#httpd{method='GET',mochi_req=MochiReq, path_parts=[DbName,<<"_design/",_/binary>>|_]}=Req, _Db) -> +db_req( + #httpd{ + method = 'GET', mochi_req = MochiReq, path_parts = [DbName, <<"_design/", _/binary>> | _] + } = Req, + _Db +) -> PathFront = "/" ++ couch_httpd:quote(binary_to_list(DbName)) ++ "/", - [_|PathTail] = re:split(MochiReq:get(raw_path), "_design%2F", - [{return, list}]), - couch_httpd:send_redirect(Req, PathFront ++ "_design/" ++ - mochiweb_util:join(PathTail, "_design%2F")); - -db_req(#httpd{path_parts=[_DbName,<<"_design">>,Name]}=Req, Db) -> - db_doc_req(Req, Db, <<"_design/",Name/binary>>); - -db_req(#httpd{path_parts=[_DbName,<<"_design">>,Name|FileNameParts]}=Req, Db) -> - db_attachment_req(Req, Db, <<"_design/",Name/binary>>, FileNameParts); - - + [_ | PathTail] = re:split( + MochiReq:get(raw_path), + "_design%2F", + [{return, list}] + ), + couch_httpd:send_redirect( + Req, + PathFront ++ "_design/" ++ + mochiweb_util:join(PathTail, "_design%2F") + ); +db_req(#httpd{path_parts = [_DbName, <<"_design">>, Name]} = Req, Db) -> + db_doc_req(Req, Db, <<"_design/", Name/binary>>); +db_req(#httpd{path_parts = [_DbName, <<"_design">>, Name | FileNameParts]} = Req, Db) -> + db_attachment_req(Req, Db, <<"_design/", Name/binary>>, FileNameParts); % Special case to allow for accessing local documents without %2F % encoding the docid. Throws out requests that don't have the second % path part or that specify an attachment name. -db_req(#httpd{path_parts=[_DbName, <<"_local">>]}, _Db) -> +db_req(#httpd{path_parts = [_DbName, <<"_local">>]}, _Db) -> throw({bad_request, <<"Invalid _local document id.">>}); - -db_req(#httpd{path_parts=[_DbName, <<"_local/">>]}, _Db) -> +db_req(#httpd{path_parts = [_DbName, <<"_local/">>]}, _Db) -> throw({bad_request, <<"Invalid _local document id.">>}); - -db_req(#httpd{path_parts=[_DbName, <<"_local">>, Name]}=Req, Db) -> +db_req(#httpd{path_parts = [_DbName, <<"_local">>, Name]} = Req, Db) -> db_doc_req(Req, Db, <<"_local/", Name/binary>>); - -db_req(#httpd{path_parts=[_DbName, <<"_local">> | _Rest]}, _Db) -> +db_req(#httpd{path_parts = [_DbName, <<"_local">> | _Rest]}, _Db) -> throw({bad_request, <<"_local documents do not accept attachments.">>}); - -db_req(#httpd{path_parts=[_, DocId]}=Req, Db) -> +db_req(#httpd{path_parts = [_, DocId]} = Req, Db) -> db_doc_req(Req, Db, DocId); - -db_req(#httpd{path_parts=[_, DocId | FileNameParts]}=Req, Db) -> +db_req(#httpd{path_parts = [_, DocId | FileNameParts]} = Req, Db) -> db_attachment_req(Req, Db, DocId, FileNameParts). -db_doc_req(#httpd{method='DELETE'}=Req, Db, DocId) -> +db_doc_req(#httpd{method = 'DELETE'} = Req, Db, DocId) -> % check for the existence of the doc to handle the 404 case. couch_doc_open(Db, DocId, nil, []), case couch_httpd:qs_value(Req, "rev") of - undefined -> - JsonObj = {[{<<"_deleted">>,true}]}, - Doc = couch_doc_from_req(Req, Db, DocId, JsonObj), - update_doc(Req, Db, DocId, Doc); - Rev -> - JsonObj = {[{<<"_rev">>, ?l2b(Rev)},{<<"_deleted">>,true}]}, - Doc = couch_doc_from_req(Req, Db, DocId, JsonObj), - update_doc(Req, Db, DocId, Doc) + undefined -> + JsonObj = {[{<<"_deleted">>, true}]}, + Doc = couch_doc_from_req(Req, Db, DocId, JsonObj), + update_doc(Req, Db, DocId, Doc); + Rev -> + JsonObj = {[{<<"_rev">>, ?l2b(Rev)}, {<<"_deleted">>, true}]}, + Doc = couch_doc_from_req(Req, Db, DocId, JsonObj), + update_doc(Req, Db, DocId, Doc) end; - db_doc_req(#httpd{method = 'GET', mochi_req = MochiReq} = Req, Db, DocId) -> #doc_query_args{ rev = Rev, @@ -508,206 +567,243 @@ db_doc_req(#httpd{method = 'GET', mochi_req = MochiReq} = Req, Db, DocId) -> options = Options1, atts_since = AttsSince } = parse_doc_query(Req), - Options = case AttsSince of - nil -> - Options1; - RevList when is_list(RevList) -> - [{atts_since, RevList}, attachments | Options1] - end, + Options = + case AttsSince of + nil -> + Options1; + RevList when is_list(RevList) -> + [{atts_since, RevList}, attachments | Options1] + end, case Revs of - [] -> - Doc = couch_doc_open(Db, DocId, Rev, Options), - send_doc(Req, Doc, Options); - _ -> - {ok, Results} = couch_db:open_doc_revs(Db, DocId, Revs, Options), - case MochiReq:accepts_content_type("multipart/mixed") of - false -> - {ok, Resp} = start_json_response(Req, 200), - send_chunk(Resp, "["), - % We loop through the docs. The first time through the separator - % is whitespace, then a comma on subsequent iterations. - lists:foldl( - fun(Result, AccSeparator) -> - case Result of - {ok, Doc} -> - JsonDoc = couch_doc:to_json_obj(Doc, Options), - Json = ?JSON_ENCODE({[{ok, JsonDoc}]}), - send_chunk(Resp, AccSeparator ++ Json); - {{not_found, missing}, RevId} -> - RevStr = couch_doc:rev_to_str(RevId), - Json = ?JSON_ENCODE({[{<<"missing">>, RevStr}]}), - send_chunk(Resp, AccSeparator ++ Json) - end, - "," % AccSeparator now has a comma - end, - "", Results), - send_chunk(Resp, "]"), - end_json_response(Resp); - true -> - send_docs_multipart(Req, Results, Options) - end + [] -> + Doc = couch_doc_open(Db, DocId, Rev, Options), + send_doc(Req, Doc, Options); + _ -> + {ok, Results} = couch_db:open_doc_revs(Db, DocId, Revs, Options), + case MochiReq:accepts_content_type("multipart/mixed") of + false -> + {ok, Resp} = start_json_response(Req, 200), + send_chunk(Resp, "["), + % We loop through the docs. The first time through the separator + % is whitespace, then a comma on subsequent iterations. + lists:foldl( + fun(Result, AccSeparator) -> + case Result of + {ok, Doc} -> + JsonDoc = couch_doc:to_json_obj(Doc, Options), + Json = ?JSON_ENCODE({[{ok, JsonDoc}]}), + send_chunk(Resp, AccSeparator ++ Json); + {{not_found, missing}, RevId} -> + RevStr = couch_doc:rev_to_str(RevId), + Json = ?JSON_ENCODE({[{<<"missing">>, RevStr}]}), + send_chunk(Resp, AccSeparator ++ Json) + end, + % AccSeparator now has a comma + "," + end, + "", + Results + ), + send_chunk(Resp, "]"), + end_json_response(Resp); + true -> + send_docs_multipart(Req, Results, Options) + end end; - - -db_doc_req(#httpd{method='POST'}=Req, Db, DocId) -> +db_doc_req(#httpd{method = 'POST'} = Req, Db, DocId) -> couch_httpd:validate_referer(Req), couch_db:validate_docid(Db, DocId), couch_httpd:validate_ctype(Req, "multipart/form-data"), Form = couch_httpd:parse_form(Req), case couch_util:get_value("_doc", Form) of - undefined -> - Rev = couch_doc:parse_rev(couch_util:get_value("_rev", Form)), - {ok, [{ok, Doc}]} = couch_db:open_doc_revs(Db, DocId, [Rev], []); - Json -> - Doc = couch_doc_from_req(Req, Db, DocId, ?JSON_DECODE(Json)) + undefined -> + Rev = couch_doc:parse_rev(couch_util:get_value("_rev", Form)), + {ok, [{ok, Doc}]} = couch_db:open_doc_revs(Db, DocId, [Rev], []); + Json -> + Doc = couch_doc_from_req(Req, Db, DocId, ?JSON_DECODE(Json)) end, UpdatedAtts = [ couch_att:new([ {name, validate_attachment_name(Name)}, {type, list_to_binary(ContentType)}, {data, Content} - ]) || - {Name, {ContentType, _}, Content} <- - proplists:get_all_values("_attachments", Form) + ]) + || {Name, {ContentType, _}, Content} <- + proplists:get_all_values("_attachments", Form) ], - #doc{atts=OldAtts} = Doc, + #doc{atts = OldAtts} = Doc, OldAtts2 = lists:flatmap( fun(Att) -> OldName = couch_att:fetch(name, Att), case [1 || A <- UpdatedAtts, couch_att:fetch(name, A) == OldName] of - [] -> [Att]; % the attachment wasn't in the UpdatedAtts, return it - _ -> [] % the attachment was in the UpdatedAtts, drop it + % the attachment wasn't in the UpdatedAtts, return it + [] -> [Att]; + % the attachment was in the UpdatedAtts, drop it + _ -> [] end - end, OldAtts), + end, + OldAtts + ), NewDoc = Doc#doc{ atts = UpdatedAtts ++ OldAtts2 }, update_doc(Req, Db, DocId, NewDoc); - -db_doc_req(#httpd{method='PUT'}=Req, Db, DocId) -> +db_doc_req(#httpd{method = 'PUT'} = Req, Db, DocId) -> couch_db:validate_docid(Db, DocId), case couch_util:to_list(couch_httpd:header_value(Req, "Content-Type")) of - ("multipart/related;" ++ _) = ContentType -> - couch_httpd:check_max_request_length(Req), - {ok, Doc0, WaitFun, Parser} = couch_doc:doc_from_multi_part_stream( - ContentType, fun() -> receive_request_data(Req) end), - Doc = couch_doc_from_req(Req, Db, DocId, Doc0), - try - Result = update_doc(Req, Db, DocId, Doc), - WaitFun(), - Result - catch throw:Err -> - % Document rejected by a validate_doc_update function. - couch_httpd_multipart:abort_multipart_stream(Parser), - throw(Err) - end; - _Else -> - Body = couch_httpd:json_body(Req), - Doc = couch_doc_from_req(Req, Db, DocId, Body), - update_doc(Req, Db, DocId, Doc) + ("multipart/related;" ++ _) = ContentType -> + couch_httpd:check_max_request_length(Req), + {ok, Doc0, WaitFun, Parser} = couch_doc:doc_from_multi_part_stream( + ContentType, fun() -> receive_request_data(Req) end + ), + Doc = couch_doc_from_req(Req, Db, DocId, Doc0), + try + Result = update_doc(Req, Db, DocId, Doc), + WaitFun(), + Result + catch + throw:Err -> + % Document rejected by a validate_doc_update function. + couch_httpd_multipart:abort_multipart_stream(Parser), + throw(Err) + end; + _Else -> + Body = couch_httpd:json_body(Req), + Doc = couch_doc_from_req(Req, Db, DocId, Body), + update_doc(Req, Db, DocId, Doc) end; - -db_doc_req(#httpd{method='COPY'}=Req, Db, SourceDocId) -> +db_doc_req(#httpd{method = 'COPY'} = Req, Db, SourceDocId) -> SourceRev = - case extract_header_rev(Req, couch_httpd:qs_value(Req, "rev")) of - missing_rev -> nil; - Rev -> Rev - end, + case extract_header_rev(Req, couch_httpd:qs_value(Req, "rev")) of + missing_rev -> nil; + Rev -> Rev + end, {TargetDocId0, TargetRevs} = parse_copy_destination_header(Req), TargetDocId = list_to_binary(mochiweb_util:unquote(TargetDocId0)), % open old doc Doc = couch_doc_open(Db, SourceDocId, SourceRev, []), % save new doc - update_doc(Req, Db, TargetDocId, Doc#doc{id=TargetDocId, revs=TargetRevs}); - + update_doc(Req, Db, TargetDocId, Doc#doc{id = TargetDocId, revs = TargetRevs}); db_doc_req(Req, _Db, _DocId) -> send_method_not_allowed(Req, "DELETE,GET,HEAD,POST,PUT,COPY"). - send_doc(Req, Doc, Options) -> case Doc#doc.meta of - [] -> - DiskEtag = couch_httpd:doc_etag(Doc), - % output etag only when we have no meta - couch_httpd:etag_respond(Req, DiskEtag, fun() -> - send_doc_efficiently(Req, Doc, [{"ETag", DiskEtag}], Options) - end); - _ -> - send_doc_efficiently(Req, Doc, [], Options) + [] -> + DiskEtag = couch_httpd:doc_etag(Doc), + % output etag only when we have no meta + couch_httpd:etag_respond(Req, DiskEtag, fun() -> + send_doc_efficiently(Req, Doc, [{"ETag", DiskEtag}], Options) + end); + _ -> + send_doc_efficiently(Req, Doc, [], Options) end. - -send_doc_efficiently(Req, #doc{atts=[]}=Doc, Headers, Options) -> - send_json(Req, 200, Headers, couch_doc:to_json_obj(Doc, Options)); -send_doc_efficiently(#httpd{mochi_req = MochiReq} = Req, - #doc{atts = Atts} = Doc, Headers, Options) -> +send_doc_efficiently(Req, #doc{atts = []} = Doc, Headers, Options) -> + send_json(Req, 200, Headers, couch_doc:to_json_obj(Doc, Options)); +send_doc_efficiently( + #httpd{mochi_req = MochiReq} = Req, + #doc{atts = Atts} = Doc, + Headers, + Options +) -> case lists:member(attachments, Options) of - true -> - case MochiReq:accepts_content_type("multipart/related") of - false -> - send_json(Req, 200, Headers, couch_doc:to_json_obj(Doc, Options)); true -> - Boundary = couch_uuids:random(), - JsonBytes = ?JSON_ENCODE(couch_doc:to_json_obj(Doc, - [attachments, follows, att_encoding_info | Options])), - {ContentType, Len} = couch_doc:len_doc_to_multi_part_stream( - Boundary,JsonBytes, Atts, true), - CType = {"Content-Type", ?b2l(ContentType)}, - {ok, Resp} = start_response_length(Req, 200, [CType|Headers], Len), - couch_doc:doc_to_multi_part_stream(Boundary,JsonBytes,Atts, - fun(Data) -> couch_httpd:send(Resp, Data) end, true) - end; - false -> - send_json(Req, 200, Headers, couch_doc:to_json_obj(Doc, Options)) + case MochiReq:accepts_content_type("multipart/related") of + false -> + send_json(Req, 200, Headers, couch_doc:to_json_obj(Doc, Options)); + true -> + Boundary = couch_uuids:random(), + JsonBytes = ?JSON_ENCODE( + couch_doc:to_json_obj( + Doc, + [attachments, follows, att_encoding_info | Options] + ) + ), + {ContentType, Len} = couch_doc:len_doc_to_multi_part_stream( + Boundary, JsonBytes, Atts, true + ), + CType = {"Content-Type", ?b2l(ContentType)}, + {ok, Resp} = start_response_length(Req, 200, [CType | Headers], Len), + couch_doc:doc_to_multi_part_stream( + Boundary, + JsonBytes, + Atts, + fun(Data) -> couch_httpd:send(Resp, Data) end, + true + ) + end; + false -> + send_json(Req, 200, Headers, couch_doc:to_json_obj(Doc, Options)) end. send_docs_multipart(Req, Results, Options1) -> OuterBoundary = couch_uuids:random(), InnerBoundary = couch_uuids:random(), Options = [attachments, follows, att_encoding_info | Options1], - CType = {"Content-Type", - "multipart/mixed; boundary=\"" ++ ?b2l(OuterBoundary) ++ "\""}, + CType = {"Content-Type", "multipart/mixed; boundary=\"" ++ ?b2l(OuterBoundary) ++ "\""}, {ok, Resp} = start_chunked_response(Req, 200, [CType]), couch_httpd:send_chunk(Resp, <<"--", OuterBoundary/binary>>), lists:foreach( - fun({ok, #doc{atts=Atts}=Doc}) -> - JsonBytes = ?JSON_ENCODE(couch_doc:to_json_obj(Doc, Options)), - {ContentType, _Len} = couch_doc:len_doc_to_multi_part_stream( - InnerBoundary, JsonBytes, Atts, true), - couch_httpd:send_chunk(Resp, <<"\r\nContent-Type: ", - ContentType/binary, "\r\n\r\n">>), - couch_doc:doc_to_multi_part_stream(InnerBoundary, JsonBytes, Atts, - fun(Data) -> couch_httpd:send_chunk(Resp, Data) - end, true), - couch_httpd:send_chunk(Resp, <<"\r\n--", OuterBoundary/binary>>); - ({{not_found, missing}, RevId}) -> - RevStr = couch_doc:rev_to_str(RevId), - Json = ?JSON_ENCODE({[{<<"missing">>, RevStr}]}), - couch_httpd:send_chunk(Resp, - [<<"\r\nContent-Type: application/json; error=\"true\"\r\n\r\n">>, - Json, - <<"\r\n--", OuterBoundary/binary>>]) - end, Results), + fun + ({ok, #doc{atts = Atts} = Doc}) -> + JsonBytes = ?JSON_ENCODE(couch_doc:to_json_obj(Doc, Options)), + {ContentType, _Len} = couch_doc:len_doc_to_multi_part_stream( + InnerBoundary, JsonBytes, Atts, true + ), + couch_httpd:send_chunk( + Resp, <<"\r\nContent-Type: ", ContentType/binary, "\r\n\r\n">> + ), + couch_doc:doc_to_multi_part_stream( + InnerBoundary, + JsonBytes, + Atts, + fun(Data) -> couch_httpd:send_chunk(Resp, Data) end, + true + ), + couch_httpd:send_chunk(Resp, <<"\r\n--", OuterBoundary/binary>>); + ({{not_found, missing}, RevId}) -> + RevStr = couch_doc:rev_to_str(RevId), + Json = ?JSON_ENCODE({[{<<"missing">>, RevStr}]}), + couch_httpd:send_chunk( + Resp, + [ + <<"\r\nContent-Type: application/json; error=\"true\"\r\n\r\n">>, + Json, + <<"\r\n--", OuterBoundary/binary>> + ] + ) + end, + Results + ), couch_httpd:send_chunk(Resp, <<"--">>), couch_httpd:last_chunk(Resp). send_ranges_multipart(Req, ContentType, Len, Att, Ranges) -> Boundary = couch_uuids:random(), - CType = {"Content-Type", - "multipart/byteranges; boundary=\"" ++ ?b2l(Boundary) ++ "\""}, + CType = {"Content-Type", "multipart/byteranges; boundary=\"" ++ ?b2l(Boundary) ++ "\""}, {ok, Resp} = start_chunked_response(Req, 206, [CType]), couch_httpd:send_chunk(Resp, <<"--", Boundary/binary>>), - lists:foreach(fun({From, To}) -> - ContentRange = ?l2b(make_content_range(From, To, Len)), - couch_httpd:send_chunk(Resp, - <<"\r\nContent-Type: ", ContentType/binary, "\r\n", - "Content-Range: ", ContentRange/binary, "\r\n", - "\r\n">>), - couch_att:range_foldl(Att, From, To + 1, - fun(Seg, _) -> send_chunk(Resp, Seg) end, {ok, Resp}), - couch_httpd:send_chunk(Resp, <<"\r\n--", Boundary/binary>>) - end, Ranges), + lists:foreach( + fun({From, To}) -> + ContentRange = ?l2b(make_content_range(From, To, Len)), + couch_httpd:send_chunk( + Resp, + <<"\r\nContent-Type: ", ContentType/binary, "\r\n", "Content-Range: ", + ContentRange/binary, "\r\n", "\r\n">> + ), + couch_att:range_foldl( + Att, + From, + To + 1, + fun(Seg, _) -> send_chunk(Resp, Seg) end, + {ok, Resp} + ), + couch_httpd:send_chunk(Resp, <<"\r\n--", Boundary/binary>>) + end, + Ranges + ), couch_httpd:send_chunk(Resp, <<"--">>), couch_httpd:last_chunk(Resp), {ok, Resp}. @@ -726,11 +822,15 @@ make_content_range(From, To, Len) -> io_lib:format("bytes ~B-~B/~B", [From, To, Len]). update_doc_result_to_json({{Id, Rev}, Error}) -> - {_Code, Err, Msg} = couch_httpd:error_info(Error), - {[{id, Id}, {rev, couch_doc:rev_to_str(Rev)}, - {error, Err}, {reason, Msg}]}. - -update_doc_result_to_json(#doc{id=DocId}, Result) -> + {_Code, Err, Msg} = couch_httpd:error_info(Error), + {[ + {id, Id}, + {rev, couch_doc:rev_to_str(Rev)}, + {error, Err}, + {reason, Msg} + ]}. + +update_doc_result_to_json(#doc{id = DocId}, Result) -> update_doc_result_to_json(DocId, Result); update_doc_result_to_json(DocId, {ok, NewRev}) -> {[{ok, true}, {id, DocId}, {rev, couch_doc:rev_to_str(NewRev)}]}; @@ -738,10 +838,11 @@ update_doc_result_to_json(DocId, Error) -> {_Code, ErrorStr, Reason} = couch_httpd:error_info(Error), {[{id, DocId}, {error, ErrorStr}, {reason, Reason}]}. - -update_doc(Req, Db, DocId, #doc{deleted=false}=Doc) -> +update_doc(Req, Db, DocId, #doc{deleted = false} = Doc) -> DbName = couch_db:name(Db), - Loc = absolute_uri(Req, "/" ++ couch_util:url_encode(DbName) ++ "/" ++ couch_util:url_encode(DocId)), + Loc = absolute_uri( + Req, "/" ++ couch_util:url_encode(DbName) ++ "/" ++ couch_util:url_encode(DocId) + ), update_doc(Req, Db, DocId, Doc, [{"Location", Loc}]); update_doc(Req, Db, DocId, Doc) -> update_doc(Req, Db, DocId, Doc, []). @@ -752,70 +853,85 @@ update_doc(Req, Db, DocId, Doc, Headers) -> } = parse_doc_query(Req), update_doc(Req, Db, DocId, Doc, Headers, UpdateType). -update_doc(Req, Db, DocId, #doc{deleted=Deleted}=Doc, Headers, UpdateType) -> +update_doc(Req, Db, DocId, #doc{deleted = Deleted} = Doc, Headers, UpdateType) -> case couch_httpd:header_value(Req, "X-Couch-Full-Commit") of - "true" -> - Options = [full_commit]; - "false" -> - Options = [delay_commit]; - _ -> - Options = [] + "true" -> + Options = [full_commit]; + "false" -> + Options = [delay_commit]; + _ -> + Options = [] end, case couch_httpd:qs_value(Req, "batch") of - "ok" -> - % async batching - spawn(fun() -> - case catch(couch_db:update_doc(Db, Doc, Options, UpdateType)) of - {ok, _} -> ok; - Error -> - couch_log:info("Batch doc error (~s): ~p",[DocId, Error]) + "ok" -> + % async batching + spawn(fun() -> + case catch (couch_db:update_doc(Db, Doc, Options, UpdateType)) of + {ok, _} -> ok; + Error -> couch_log:info("Batch doc error (~s): ~p", [DocId, Error]) end end), - send_json(Req, 202, Headers, {[ - {ok, true}, - {id, DocId} - ]}); - _Normal -> - % normal - {ok, NewRev} = couch_db:update_doc(Db, Doc, Options, UpdateType), - NewRevStr = couch_doc:rev_to_str(NewRev), - ResponseHeaders = [{"ETag", <<"\"", NewRevStr/binary, "\"">>}] ++ Headers, - send_json(Req, - if Deleted orelse Req#httpd.method == 'DELETE' -> 200; - true -> 201 end, - ResponseHeaders, {[ - {ok, true}, - {id, DocId}, - {rev, NewRevStr}]}) + send_json( + Req, + 202, + Headers, + {[ + {ok, true}, + {id, DocId} + ]} + ); + _Normal -> + % normal + {ok, NewRev} = couch_db:update_doc(Db, Doc, Options, UpdateType), + NewRevStr = couch_doc:rev_to_str(NewRev), + ResponseHeaders = [{"ETag", <<"\"", NewRevStr/binary, "\"">>}] ++ Headers, + send_json( + Req, + if + Deleted orelse Req#httpd.method == 'DELETE' -> 200; + true -> 201 + end, + ResponseHeaders, + {[ + {ok, true}, + {id, DocId}, + {rev, NewRevStr} + ]} + ) end. -couch_doc_from_req(Req, _Db, DocId, #doc{revs=Revs}=Doc) -> +couch_doc_from_req(Req, _Db, DocId, #doc{revs = Revs} = Doc) -> validate_attachment_names(Doc), - Rev = case couch_httpd:qs_value(Req, "rev") of - undefined -> - undefined; - QSRev -> - couch_doc:parse_rev(QSRev) - end, + Rev = + case couch_httpd:qs_value(Req, "rev") of + undefined -> + undefined; + QSRev -> + couch_doc:parse_rev(QSRev) + end, Revs2 = - case Revs of - {Start, [RevId|_]} -> - if Rev /= undefined andalso Rev /= {Start, RevId} -> - throw({bad_request, "Document rev from request body and query " - "string have different values"}); - true -> - case extract_header_rev(Req, {Start, RevId}) of - missing_rev -> {0, []}; - _ -> Revs - end - end; - _ -> - case extract_header_rev(Req, Rev) of - missing_rev -> {0, []}; - {Pos, RevId2} -> {Pos, [RevId2]} - end - end, - Doc#doc{id=DocId, revs=Revs2}; + case Revs of + {Start, [RevId | _]} -> + if + Rev /= undefined andalso Rev /= {Start, RevId} -> + throw( + {bad_request, + "Document rev from request body and query " + "string have different values"} + ); + true -> + case extract_header_rev(Req, {Start, RevId}) of + missing_rev -> {0, []}; + _ -> Revs + end + end; + _ -> + case extract_header_rev(Req, Rev) of + missing_rev -> {0, []}; + {Pos, RevId2} -> {Pos, [RevId2]} + end + end, + Doc#doc{id = DocId, revs = Revs2}; couch_doc_from_req(Req, Db, DocId, Json) -> Doc = couch_db:doc_from_json_obj_validate(Db, Json), couch_doc_from_req(Req, Db, DocId, Doc). @@ -826,233 +942,283 @@ couch_doc_from_req(Req, Db, DocId, Json) -> couch_doc_open(Db, DocId, Rev, Options) -> case Rev of - nil -> % open most recent rev - case couch_db:open_doc(Db, DocId, Options) of - {ok, Doc} -> - Doc; - Error -> - throw(Error) - end; - _ -> % open a specific rev (deletions come back as stubs) - case couch_db:open_doc_revs(Db, DocId, [Rev], Options) of - {ok, [{ok, Doc}]} -> - Doc; - {ok, [{{not_found, missing}, Rev}]} -> - throw(not_found); - {ok, [Else]} -> - throw(Else) - end - end. + % open most recent rev + nil -> + case couch_db:open_doc(Db, DocId, Options) of + {ok, Doc} -> + Doc; + Error -> + throw(Error) + end; + % open a specific rev (deletions come back as stubs) + _ -> + case couch_db:open_doc_revs(Db, DocId, [Rev], Options) of + {ok, [{ok, Doc}]} -> + Doc; + {ok, [{{not_found, missing}, Rev}]} -> + throw(not_found); + {ok, [Else]} -> + throw(Else) + end + end. % Attachment request handlers -db_attachment_req(#httpd{method='GET',mochi_req=MochiReq}=Req, Db, DocId, FileNameParts) -> - FileName = list_to_binary(mochiweb_util:join(lists:map(fun binary_to_list/1, FileNameParts),"/")), +db_attachment_req(#httpd{method = 'GET', mochi_req = MochiReq} = Req, Db, DocId, FileNameParts) -> + FileName = list_to_binary( + mochiweb_util:join(lists:map(fun binary_to_list/1, FileNameParts), "/") + ), #doc_query_args{ - rev=Rev, - options=Options + rev = Rev, + options = Options } = parse_doc_query(Req), #doc{ - atts=Atts + atts = Atts } = Doc = couch_doc_open(Db, DocId, Rev, Options), case [A || A <- Atts, couch_att:fetch(name, A) == FileName] of - [] -> - throw({not_found, "Document is missing attachment"}); - [Att] -> - [Type, Enc, DiskLen, AttLen, Md5] = couch_att:fetch([type, encoding, disk_len, att_len, md5], Att), - Etag = case Md5 of - <<>> -> couch_httpd:doc_etag(Doc); - _ -> "\"" ++ ?b2l(base64:encode(Md5)) ++ "\"" - end, - ReqAcceptsAttEnc = lists:member( - atom_to_list(Enc), - couch_httpd:accepted_encodings(Req) - ), - Len = case {Enc, ReqAcceptsAttEnc} of - {identity, _} -> - % stored and served in identity form - DiskLen; - {_, false} when DiskLen =/= AttLen -> - % Stored encoded, but client doesn't accept the encoding we used, - % so we need to decode on the fly. DiskLen is the identity length - % of the attachment. - DiskLen; - {_, true} -> - % Stored and served encoded. AttLen is the encoded length. - AttLen; - _ -> - % We received an encoded attachment and stored it as such, so we - % don't know the identity length. The client doesn't accept the - % encoding, and since we cannot serve a correct Content-Length - % header we'll fall back to a chunked response. - undefined - end, - Headers = [ - {"ETag", Etag}, - {"Cache-Control", "must-revalidate"}, - {"Content-Type", binary_to_list(Type)} - ] ++ case ReqAcceptsAttEnc of - true when Enc =/= identity -> - % RFC 2616 says that the 'identify' encoding should not be used in - % the Content-Encoding header - [{"Content-Encoding", atom_to_list(Enc)}]; - _ -> - [] - end ++ case Enc of - identity -> - [{"Accept-Ranges", "bytes"}]; - _ -> - [{"Accept-Ranges", "none"}] - end, - AttFun = case ReqAcceptsAttEnc of - false -> - fun couch_att:foldl_decode/3; - true -> - fun couch_att:foldl/3 - end, - couch_httpd:etag_respond( - Req, - Etag, - fun() -> - case Len of - undefined -> - {ok, Resp} = start_chunked_response(Req, 200, Headers), - AttFun(Att, fun(Seg, _) -> send_chunk(Resp, Seg) end, {ok, Resp}), - last_chunk(Resp); - _ -> - Ranges = parse_ranges(MochiReq:get(range), Len), - case {Enc, Ranges} of - {identity, [{From, To}]} -> - Headers1 = [{"Content-Range", make_content_range(From, To, Len)}] - ++ Headers, - {ok, Resp} = start_response_length(Req, 206, Headers1, To - From + 1), - couch_att:range_foldl(Att, From, To + 1, - fun(Seg, _) -> send(Resp, Seg) end, {ok, Resp}); - {identity, Ranges} when is_list(Ranges) andalso length(Ranges) < 10 -> - send_ranges_multipart(Req, Type, Len, Att, Ranges); + [] -> + throw({not_found, "Document is missing attachment"}); + [Att] -> + [Type, Enc, DiskLen, AttLen, Md5] = couch_att:fetch( + [type, encoding, disk_len, att_len, md5], Att + ), + Etag = + case Md5 of + <<>> -> couch_httpd:doc_etag(Doc); + _ -> "\"" ++ ?b2l(base64:encode(Md5)) ++ "\"" + end, + ReqAcceptsAttEnc = lists:member( + atom_to_list(Enc), + couch_httpd:accepted_encodings(Req) + ), + Len = + case {Enc, ReqAcceptsAttEnc} of + {identity, _} -> + % stored and served in identity form + DiskLen; + {_, false} when DiskLen =/= AttLen -> + % Stored encoded, but client doesn't accept the encoding we used, + % so we need to decode on the fly. DiskLen is the identity length + % of the attachment. + DiskLen; + {_, true} -> + % Stored and served encoded. AttLen is the encoded length. + AttLen; + _ -> + % We received an encoded attachment and stored it as such, so we + % don't know the identity length. The client doesn't accept the + % encoding, and since we cannot serve a correct Content-Length + % header we'll fall back to a chunked response. + undefined + end, + Headers = + [ + {"ETag", Etag}, + {"Cache-Control", "must-revalidate"}, + {"Content-Type", binary_to_list(Type)} + ] ++ + case ReqAcceptsAttEnc of + true when Enc =/= identity -> + % RFC 2616 says that the 'identify' encoding should not be used in + % the Content-Encoding header + [{"Content-Encoding", atom_to_list(Enc)}]; _ -> - Headers1 = Headers ++ - if Enc =:= identity orelse ReqAcceptsAttEnc =:= true -> - [{"Content-MD5", base64:encode(Md5)}]; - true -> - [] - end, - {ok, Resp} = start_response_length(Req, 200, Headers1, Len), - AttFun(Att, fun(Seg, _) -> send(Resp, Seg) end, {ok, Resp}) + [] + end ++ + case Enc of + identity -> + [{"Accept-Ranges", "bytes"}]; + _ -> + [{"Accept-Ranges", "none"}] + end, + AttFun = + case ReqAcceptsAttEnc of + false -> + fun couch_att:foldl_decode/3; + true -> + fun couch_att:foldl/3 + end, + couch_httpd:etag_respond( + Req, + Etag, + fun() -> + case Len of + undefined -> + {ok, Resp} = start_chunked_response(Req, 200, Headers), + AttFun(Att, fun(Seg, _) -> send_chunk(Resp, Seg) end, {ok, Resp}), + last_chunk(Resp); + _ -> + Ranges = parse_ranges(MochiReq:get(range), Len), + case {Enc, Ranges} of + {identity, [{From, To}]} -> + Headers1 = + [{"Content-Range", make_content_range(From, To, Len)}] ++ + Headers, + {ok, Resp} = start_response_length( + Req, 206, Headers1, To - From + 1 + ), + couch_att:range_foldl( + Att, + From, + To + 1, + fun(Seg, _) -> send(Resp, Seg) end, + {ok, Resp} + ); + {identity, Ranges} when + is_list(Ranges) andalso length(Ranges) < 10 + -> + send_ranges_multipart(Req, Type, Len, Att, Ranges); + _ -> + Headers1 = + Headers ++ + if + Enc =:= identity orelse ReqAcceptsAttEnc =:= true -> + [{"Content-MD5", base64:encode(Md5)}]; + true -> + [] + end, + {ok, Resp} = start_response_length(Req, 200, Headers1, Len), + AttFun(Att, fun(Seg, _) -> send(Resp, Seg) end, {ok, Resp}) + end end end - end - ) + ) end; - - -db_attachment_req(#httpd{method=Method,mochi_req=MochiReq}=Req, Db, DocId, FileNameParts) - when (Method == 'PUT') or (Method == 'DELETE') -> +db_attachment_req( + #httpd{method = Method, mochi_req = MochiReq} = Req, Db, DocId, FileNameParts +) when + (Method == 'PUT') or (Method == 'DELETE') +-> FileName = validate_attachment_name( - mochiweb_util:join( - lists:map(fun binary_to_list/1, - FileNameParts),"/")), - NewAtt = case Method of - 'DELETE' -> - []; - _ -> - MimeType = case couch_httpd:header_value(Req,"Content-Type") of - % We could throw an error here or guess by the FileName. - % Currently, just giving it a default. - undefined -> <<"application/octet-stream">>; - CType -> list_to_binary(CType) - end, - Data = case couch_httpd:body_length(Req) of - undefined -> - <<"">>; - {unknown_transfer_encoding, Unknown} -> - exit({unknown_transfer_encoding, Unknown}); - chunked -> - fun(MaxChunkSize, ChunkFun, InitState) -> - couch_httpd:recv_chunked( - Req, MaxChunkSize, ChunkFun, InitState - ) - end; - 0 -> - <<"">>; - Length when is_integer(Length) -> - Expect = case couch_httpd:header_value(Req, "expect") of + mochiweb_util:join( + lists:map( + fun binary_to_list/1, + FileNameParts + ), + "/" + ) + ), + NewAtt = + case Method of + 'DELETE' -> + []; + _ -> + MimeType = + case couch_httpd:header_value(Req, "Content-Type") of + % We could throw an error here or guess by the FileName. + % Currently, just giving it a default. + undefined -> <<"application/octet-stream">>; + CType -> list_to_binary(CType) + end, + Data = + case couch_httpd:body_length(Req) of undefined -> - undefined; - Value when is_list(Value) -> - string:to_lower(Value) + <<"">>; + {unknown_transfer_encoding, Unknown} -> + exit({unknown_transfer_encoding, Unknown}); + chunked -> + fun(MaxChunkSize, ChunkFun, InitState) -> + couch_httpd:recv_chunked( + Req, MaxChunkSize, ChunkFun, InitState + ) + end; + 0 -> + <<"">>; + Length when is_integer(Length) -> + Expect = + case couch_httpd:header_value(Req, "expect") of + undefined -> + undefined; + Value when is_list(Value) -> + string:to_lower(Value) + end, + case Expect of + "100-continue" -> + MochiReq:start_raw_response({100, gb_trees:empty()}); + _Else -> + ok + end, + fun() -> couch_httpd:recv(Req, 0) end; + Length -> + exit({length_not_integer, Length}) end, - case Expect of - "100-continue" -> - MochiReq:start_raw_response({100, gb_trees:empty()}); - _Else -> - ok + AttLen = + case couch_httpd:header_value(Req, "Content-Length") of + undefined -> undefined; + Len -> list_to_integer(Len) end, - fun() -> couch_httpd:recv(Req, 0) end; - Length -> - exit({length_not_integer, Length}) - end, - AttLen = case couch_httpd:header_value(Req,"Content-Length") of - undefined -> undefined; - Len -> list_to_integer(Len) - end, - ContentEnc = string:to_lower(string:strip( - couch_httpd:header_value(Req,"Content-Encoding","identity") - )), - Encoding = case ContentEnc of - "identity" -> - identity; - "gzip" -> - gzip; - _ -> - throw({ - bad_ctype, - "Only gzip and identity content-encodings are supported" - }) - end, - [couch_att:new([ - {name, FileName}, - {type, MimeType}, - {data, Data}, - {att_len, AttLen}, - {md5, get_md5_header(Req)}, - {encoding, Encoding} - ])] - end, + ContentEnc = string:to_lower( + string:strip( + couch_httpd:header_value(Req, "Content-Encoding", "identity") + ) + ), + Encoding = + case ContentEnc of + "identity" -> + identity; + "gzip" -> + gzip; + _ -> + throw({ + bad_ctype, + "Only gzip and identity content-encodings are supported" + }) + end, + [ + couch_att:new([ + {name, FileName}, + {type, MimeType}, + {data, Data}, + {att_len, AttLen}, + {md5, get_md5_header(Req)}, + {encoding, Encoding} + ]) + ] + end, - Doc = case extract_header_rev(Req, couch_httpd:qs_value(Req, "rev")) of - missing_rev -> % make the new doc - if Method =/= 'DELETE' -> ok; true -> - % check for the existence of the doc to handle the 404 case. - couch_doc_open(Db, DocId, nil, []) - end, - couch_db:validate_docid(Db, DocId), - #doc{id=DocId}; - Rev -> - case couch_db:open_doc_revs(Db, DocId, [Rev], []) of - {ok, [{ok, Doc0}]} -> Doc0; - {ok, [{{not_found, missing}, Rev}]} -> throw(conflict); - {ok, [Error]} -> throw(Error) - end - end, + Doc = + case extract_header_rev(Req, couch_httpd:qs_value(Req, "rev")) of + % make the new doc + missing_rev -> + if + Method =/= 'DELETE' -> + ok; + true -> + % check for the existence of the doc to handle the 404 case. + couch_doc_open(Db, DocId, nil, []) + end, + couch_db:validate_docid(Db, DocId), + #doc{id = DocId}; + Rev -> + case couch_db:open_doc_revs(Db, DocId, [Rev], []) of + {ok, [{ok, Doc0}]} -> Doc0; + {ok, [{{not_found, missing}, Rev}]} -> throw(conflict); + {ok, [Error]} -> throw(Error) + end + end, - #doc{atts=Atts} = Doc, + #doc{atts = Atts} = Doc, DocEdited = Doc#doc{ atts = NewAtt ++ [A || A <- Atts, couch_att:fetch(name, A) /= FileName] }, - Headers = case Method of - 'DELETE' -> - []; - _ -> - [{"Location", absolute_uri(Req, "/" ++ - couch_util:url_encode(couch_db:name(Db)) ++ "/" ++ - couch_util:url_encode(DocId) ++ "/" ++ - couch_util:url_encode(FileName) - )}] - end, + Headers = + case Method of + 'DELETE' -> + []; + _ -> + [ + {"Location", + absolute_uri( + Req, + "/" ++ + couch_util:url_encode(couch_db:name(Db)) ++ "/" ++ + couch_util:url_encode(DocId) ++ "/" ++ + couch_util:url_encode(FileName) + )} + ] + end, update_doc(Req, Db, DocId, DocEdited, Headers); - db_attachment_req(Req, _Db, _DocId, _FileNameParts) -> send_method_not_allowed(Req, "DELETE,GET,HEAD,PUT"). @@ -1065,17 +1231,19 @@ parse_ranges(Ranges, Len) -> parse_ranges([], _Len, Acc) -> lists:reverse(Acc); -parse_ranges([{0, none}|_], _Len, _Acc) -> +parse_ranges([{0, none} | _], _Len, _Acc) -> undefined; -parse_ranges([{From, To}|_], _Len, _Acc) when is_integer(From) andalso is_integer(To) andalso To < From -> +parse_ranges([{From, To} | _], _Len, _Acc) when + is_integer(From) andalso is_integer(To) andalso To < From +-> throw(requested_range_not_satisfiable); -parse_ranges([{From, To}|Rest], Len, Acc) when is_integer(To) andalso To >= Len -> - parse_ranges([{From, Len-1}] ++ Rest, Len, Acc); -parse_ranges([{none, To}|Rest], Len, Acc) -> +parse_ranges([{From, To} | Rest], Len, Acc) when is_integer(To) andalso To >= Len -> + parse_ranges([{From, Len - 1}] ++ Rest, Len, Acc); +parse_ranges([{none, To} | Rest], Len, Acc) -> parse_ranges([{Len - To, Len - 1}] ++ Rest, Len, Acc); -parse_ranges([{From, none}|Rest], Len, Acc) -> +parse_ranges([{From, none} | Rest], Len, Acc) -> parse_ranges([{From, Len - 1}] ++ Rest, Len, Acc); -parse_ranges([{From,To}|Rest], Len, Acc) -> +parse_ranges([{From, To} | Rest], Len, Acc) -> parse_ranges(Rest, Len, [{From, To}] ++ Acc). get_md5_header(Req) -> @@ -1099,99 +1267,111 @@ get_md5_header(Req) -> end. parse_doc_query(Req) -> - lists:foldl(fun({Key,Value}, Args) -> - case {Key, Value} of - {"attachments", "true"} -> - Options = [attachments | Args#doc_query_args.options], - Args#doc_query_args{options=Options}; - {"meta", "true"} -> - Options = [revs_info, conflicts, deleted_conflicts | Args#doc_query_args.options], - Args#doc_query_args{options=Options}; - {"revs", "true"} -> - Options = [revs | Args#doc_query_args.options], - Args#doc_query_args{options=Options}; - {"local_seq", "true"} -> - Options = [local_seq | Args#doc_query_args.options], - Args#doc_query_args{options=Options}; - {"revs_info", "true"} -> - Options = [revs_info | Args#doc_query_args.options], - Args#doc_query_args{options=Options}; - {"conflicts", "true"} -> - Options = [conflicts | Args#doc_query_args.options], - Args#doc_query_args{options=Options}; - {"deleted_conflicts", "true"} -> - Options = [deleted_conflicts | Args#doc_query_args.options], - Args#doc_query_args{options=Options}; - {"rev", Rev} -> - Args#doc_query_args{rev=couch_doc:parse_rev(Rev)}; - {"open_revs", "all"} -> - Args#doc_query_args{open_revs=all}; - {"open_revs", RevsJsonStr} -> - JsonArray = ?JSON_DECODE(RevsJsonStr), - Args#doc_query_args{open_revs=couch_doc:parse_revs(JsonArray)}; - {"latest", "true"} -> - Options = [latest | Args#doc_query_args.options], - Args#doc_query_args{options=Options}; - {"atts_since", RevsJsonStr} -> - JsonArray = ?JSON_DECODE(RevsJsonStr), - Args#doc_query_args{atts_since = couch_doc:parse_revs(JsonArray)}; - {"new_edits", "false"} -> - Args#doc_query_args{update_type=replicated_changes}; - {"new_edits", "true"} -> - Args#doc_query_args{update_type=interactive_edit}; - {"att_encoding_info", "true"} -> - Options = [att_encoding_info | Args#doc_query_args.options], - Args#doc_query_args{options=Options}; - _Else -> % unknown key value pair, ignore. - Args - end - end, #doc_query_args{}, couch_httpd:qs(Req)). + lists:foldl( + fun({Key, Value}, Args) -> + case {Key, Value} of + {"attachments", "true"} -> + Options = [attachments | Args#doc_query_args.options], + Args#doc_query_args{options = Options}; + {"meta", "true"} -> + Options = [ + revs_info, conflicts, deleted_conflicts | Args#doc_query_args.options + ], + Args#doc_query_args{options = Options}; + {"revs", "true"} -> + Options = [revs | Args#doc_query_args.options], + Args#doc_query_args{options = Options}; + {"local_seq", "true"} -> + Options = [local_seq | Args#doc_query_args.options], + Args#doc_query_args{options = Options}; + {"revs_info", "true"} -> + Options = [revs_info | Args#doc_query_args.options], + Args#doc_query_args{options = Options}; + {"conflicts", "true"} -> + Options = [conflicts | Args#doc_query_args.options], + Args#doc_query_args{options = Options}; + {"deleted_conflicts", "true"} -> + Options = [deleted_conflicts | Args#doc_query_args.options], + Args#doc_query_args{options = Options}; + {"rev", Rev} -> + Args#doc_query_args{rev = couch_doc:parse_rev(Rev)}; + {"open_revs", "all"} -> + Args#doc_query_args{open_revs = all}; + {"open_revs", RevsJsonStr} -> + JsonArray = ?JSON_DECODE(RevsJsonStr), + Args#doc_query_args{open_revs = couch_doc:parse_revs(JsonArray)}; + {"latest", "true"} -> + Options = [latest | Args#doc_query_args.options], + Args#doc_query_args{options = Options}; + {"atts_since", RevsJsonStr} -> + JsonArray = ?JSON_DECODE(RevsJsonStr), + Args#doc_query_args{atts_since = couch_doc:parse_revs(JsonArray)}; + {"new_edits", "false"} -> + Args#doc_query_args{update_type = replicated_changes}; + {"new_edits", "true"} -> + Args#doc_query_args{update_type = interactive_edit}; + {"att_encoding_info", "true"} -> + Options = [att_encoding_info | Args#doc_query_args.options], + Args#doc_query_args{options = Options}; + % unknown key value pair, ignore. + _Else -> + Args + end + end, + #doc_query_args{}, + couch_httpd:qs(Req) + ). parse_changes_query(Req, Db) -> - ChangesArgs = lists:foldl(fun({Key, Value}, Args) -> - case {string:to_lower(Key), Value} of - {"feed", "live"} -> - %% sugar for continuous - Args#changes_args{feed="continuous"}; - {"feed", _} -> - Args#changes_args{feed=Value}; - {"descending", "true"} -> - Args#changes_args{dir=rev}; - {"since", "now"} -> - UpdateSeq = couch_util:with_db(couch_db:name(Db), fun(WDb) -> - couch_db:get_update_seq(WDb) - end), - Args#changes_args{since=UpdateSeq}; - {"since", _} -> - Args#changes_args{since=list_to_integer(Value)}; - {"last-event-id", _} -> - Args#changes_args{since=list_to_integer(Value)}; - {"limit", _} -> - Args#changes_args{limit=list_to_integer(Value)}; - {"style", _} -> - Args#changes_args{style=list_to_existing_atom(Value)}; - {"heartbeat", "true"} -> - Args#changes_args{heartbeat=true}; - {"heartbeat", _} -> - Args#changes_args{heartbeat=list_to_integer(Value)}; - {"timeout", _} -> - Args#changes_args{timeout=list_to_integer(Value)}; - {"include_docs", "true"} -> - Args#changes_args{include_docs=true}; - {"attachments", "true"} -> - Opts = Args#changes_args.doc_options, - Args#changes_args{doc_options=[attachments|Opts]}; - {"att_encoding_info", "true"} -> - Opts = Args#changes_args.doc_options, - Args#changes_args{doc_options=[att_encoding_info|Opts]}; - {"conflicts", "true"} -> - Args#changes_args{conflicts=true}; - {"filter", _} -> - Args#changes_args{filter=Value}; - _Else -> % unknown key value pair, ignore. - Args - end - end, #changes_args{}, couch_httpd:qs(Req)), + ChangesArgs = lists:foldl( + fun({Key, Value}, Args) -> + case {string:to_lower(Key), Value} of + {"feed", "live"} -> + %% sugar for continuous + Args#changes_args{feed = "continuous"}; + {"feed", _} -> + Args#changes_args{feed = Value}; + {"descending", "true"} -> + Args#changes_args{dir = rev}; + {"since", "now"} -> + UpdateSeq = couch_util:with_db(couch_db:name(Db), fun(WDb) -> + couch_db:get_update_seq(WDb) + end), + Args#changes_args{since = UpdateSeq}; + {"since", _} -> + Args#changes_args{since = list_to_integer(Value)}; + {"last-event-id", _} -> + Args#changes_args{since = list_to_integer(Value)}; + {"limit", _} -> + Args#changes_args{limit = list_to_integer(Value)}; + {"style", _} -> + Args#changes_args{style = list_to_existing_atom(Value)}; + {"heartbeat", "true"} -> + Args#changes_args{heartbeat = true}; + {"heartbeat", _} -> + Args#changes_args{heartbeat = list_to_integer(Value)}; + {"timeout", _} -> + Args#changes_args{timeout = list_to_integer(Value)}; + {"include_docs", "true"} -> + Args#changes_args{include_docs = true}; + {"attachments", "true"} -> + Opts = Args#changes_args.doc_options, + Args#changes_args{doc_options = [attachments | Opts]}; + {"att_encoding_info", "true"} -> + Opts = Args#changes_args.doc_options, + Args#changes_args{doc_options = [att_encoding_info | Opts]}; + {"conflicts", "true"} -> + Args#changes_args{conflicts = true}; + {"filter", _} -> + Args#changes_args{filter = Value}; + % unknown key value pair, ignore. + _Else -> + Args + end + end, + #changes_args{}, + couch_httpd:qs(Req) + ), %% if it's an EventSource request with a Last-event-ID header %% that should override the `since` query string, since it's %% probably the browser reconnecting. @@ -1201,60 +1381,62 @@ parse_changes_query(Req, Db) -> undefined -> ChangesArgs; Value -> - ChangesArgs#changes_args{since=list_to_integer(Value)} + ChangesArgs#changes_args{since = list_to_integer(Value)} end; _ -> ChangesArgs end. -extract_header_rev(Req, ExplicitRev) when is_binary(ExplicitRev) or is_list(ExplicitRev)-> +extract_header_rev(Req, ExplicitRev) when is_binary(ExplicitRev) or is_list(ExplicitRev) -> extract_header_rev(Req, couch_doc:parse_rev(ExplicitRev)); extract_header_rev(Req, ExplicitRev) -> - Etag = case couch_httpd:header_value(Req, "If-Match") of - undefined -> undefined; - Value -> couch_doc:parse_rev(string:strip(Value, both, $")) - end, + Etag = + case couch_httpd:header_value(Req, "If-Match") of + undefined -> undefined; + Value -> couch_doc:parse_rev(string:strip(Value, both, $")) + end, case {ExplicitRev, Etag} of - {undefined, undefined} -> missing_rev; - {_, undefined} -> ExplicitRev; - {undefined, _} -> Etag; - _ when ExplicitRev == Etag -> Etag; - _ -> - throw({bad_request, "Document rev and etag have different values"}) + {undefined, undefined} -> missing_rev; + {_, undefined} -> ExplicitRev; + {undefined, _} -> Etag; + _ when ExplicitRev == Etag -> Etag; + _ -> throw({bad_request, "Document rev and etag have different values"}) end. - parse_copy_destination_header(Req) -> case couch_httpd:header_value(Req, "Destination") of - undefined -> - throw({bad_request, "Destination header is mandatory for COPY."}); - Destination -> - case re:run(Destination, "^https?://", [{capture, none}]) of - match -> - throw({bad_request, "Destination URL must be relative."}); - nomatch -> - % see if ?rev=revid got appended to the Destination header - case re:run(Destination, "\\?", [{capture, none}]) of - nomatch -> - {list_to_binary(Destination), {0, []}}; - match -> - [DocId, RevQs] = re:split(Destination, "\\?", [{return, list}]), - [_RevQueryKey, Rev] = re:split(RevQs, "=", [{return, list}]), - {Pos, RevId} = couch_doc:parse_rev(Rev), - {list_to_binary(DocId), {Pos, [RevId]}} + undefined -> + throw({bad_request, "Destination header is mandatory for COPY."}); + Destination -> + case re:run(Destination, "^https?://", [{capture, none}]) of + match -> + throw({bad_request, "Destination URL must be relative."}); + nomatch -> + % see if ?rev=revid got appended to the Destination header + case re:run(Destination, "\\?", [{capture, none}]) of + nomatch -> + {list_to_binary(Destination), {0, []}}; + match -> + [DocId, RevQs] = re:split(Destination, "\\?", [{return, list}]), + [_RevQueryKey, Rev] = re:split(RevQs, "=", [{return, list}]), + {Pos, RevId} = couch_doc:parse_rev(Rev), + {list_to_binary(DocId), {Pos, [RevId]}} + end end - end end. validate_attachment_names(Doc) -> - lists:foreach(fun(Att) -> - Name = couch_att:fetch(name, Att), - validate_attachment_name(Name) - end, Doc#doc.atts). + lists:foreach( + fun(Att) -> + Name = couch_att:fetch(name, Att), + validate_attachment_name(Name) + end, + Doc#doc.atts + ). validate_attachment_name(Name) when is_list(Name) -> validate_attachment_name(list_to_binary(Name)); -validate_attachment_name(<<"_",_/binary>>) -> +validate_attachment_name(<<"_", _/binary>>) -> throw({bad_request, <<"Attachment name can't start with '_'">>}); validate_attachment_name(Name) -> case couch_util:validate_utf8(Name) of diff --git a/src/couch/src/couch_httpd_misc_handlers.erl b/src/couch/src/couch_httpd_misc_handlers.erl index ea9c1cb84..d9c591875 100644 --- a/src/couch/src/couch_httpd_misc_handlers.erl +++ b/src/couch/src/couch_httpd_misc_handlers.erl @@ -12,87 +12,104 @@ -module(couch_httpd_misc_handlers). --export([handle_welcome_req/2,handle_favicon_req/2,handle_utils_dir_req/2, +-export([ + handle_welcome_req/2, + handle_favicon_req/2, + handle_utils_dir_req/2, handle_all_dbs_req/1, - handle_uuids_req/1,handle_config_req/1, - handle_task_status_req/1, handle_file_req/2]). - + handle_uuids_req/1, + handle_config_req/1, + handle_task_status_req/1, + handle_file_req/2 +]). -include_lib("couch/include/couch_db.hrl"). --import(couch_httpd, - [send_json/2,send_json/3,send_json/4,send_method_not_allowed/2, - start_json_response/2,send_chunk/2,last_chunk/1,end_json_response/1, - start_chunked_response/3, send_error/4]). +-import( + couch_httpd, + [ + send_json/2, send_json/3, send_json/4, + send_method_not_allowed/2, + start_json_response/2, + send_chunk/2, + last_chunk/1, + end_json_response/1, + start_chunked_response/3, + send_error/4 + ] +). % httpd global handlers -handle_welcome_req(#httpd{method='GET'}=Req, WelcomeMessage) -> - send_json(Req, {[ - {couchdb, WelcomeMessage}, - {uuid, couch_server:get_uuid()}, - {version, list_to_binary(couch_server:get_version())} - ] ++ case config:get("vendor") of - [] -> - []; - Properties -> - [{vendor, {[{?l2b(K), ?l2b(V)} || {K, V} <- Properties]}}] - end +handle_welcome_req(#httpd{method = 'GET'} = Req, WelcomeMessage) -> + send_json(Req, { + [ + {couchdb, WelcomeMessage}, + {uuid, couch_server:get_uuid()}, + {version, list_to_binary(couch_server:get_version())} + ] ++ + case config:get("vendor") of + [] -> + []; + Properties -> + [{vendor, {[{?l2b(K), ?l2b(V)} || {K, V} <- Properties]}}] + end }); handle_welcome_req(Req, _) -> send_method_not_allowed(Req, "GET,HEAD"). -handle_favicon_req(#httpd{method='GET'}=Req, DocumentRoot) -> - {{Year,Month,Day},Time} = erlang:universaltime(), - OneYearFromNow = {{Year+1,Month,Day},Time}, +handle_favicon_req(#httpd{method = 'GET'} = Req, DocumentRoot) -> + {{Year, Month, Day}, Time} = erlang:universaltime(), + OneYearFromNow = {{Year + 1, Month, Day}, Time}, CachingHeaders = [ %favicon should expire a year from now {"Cache-Control", "public, max-age=31536000"}, {"Expires", couch_util:rfc1123_date(OneYearFromNow)} ], couch_httpd:serve_file(Req, "favicon.ico", DocumentRoot, CachingHeaders); - handle_favicon_req(Req, _) -> send_method_not_allowed(Req, "GET,HEAD"). -handle_file_req(#httpd{method='GET'}=Req, Document) -> +handle_file_req(#httpd{method = 'GET'} = Req, Document) -> couch_httpd:serve_file(Req, filename:basename(Document), filename:dirname(Document)); - handle_file_req(Req, _) -> send_method_not_allowed(Req, "GET,HEAD"). handle_utils_dir_req(Req, _) -> - send_error(Req, 410, <<"no_node_local_fauxton">>, - ?l2b("The web interface is no longer available on the node-local port.")). - + send_error( + Req, + 410, + <<"no_node_local_fauxton">>, + ?l2b("The web interface is no longer available on the node-local port.") + ). -handle_all_dbs_req(#httpd{method='GET'}=Req) -> +handle_all_dbs_req(#httpd{method = 'GET'} = Req) -> {ok, DbNames} = couch_server:all_databases(), send_json(Req, DbNames); handle_all_dbs_req(Req) -> send_method_not_allowed(Req, "GET,HEAD"). - -handle_task_status_req(#httpd{method='GET'}=Req) -> +handle_task_status_req(#httpd{method = 'GET'} = Req) -> ok = couch_httpd:verify_is_server_admin(Req), % convert the list of prop lists to a list of json objects send_json(Req, [{Props} || Props <- couch_task_status:all()]); handle_task_status_req(Req) -> send_method_not_allowed(Req, "GET,HEAD"). - -handle_uuids_req(#httpd{method='GET'}=Req) -> - Max = config:get_integer("uuids","max_count", 1000), - Count = try list_to_integer(couch_httpd:qs_value(Req, "count", "1")) of - N when N > Max -> - throw({bad_request, <<"count parameter too large">>}); - N when N < 0 -> - throw({bad_request, <<"count must be a positive integer">>}); - N -> N - catch - error:badarg -> - throw({bad_request, <<"count must be a positive integer">>}) - end, +handle_uuids_req(#httpd{method = 'GET'} = Req) -> + Max = config:get_integer("uuids", "max_count", 1000), + Count = + try list_to_integer(couch_httpd:qs_value(Req, "count", "1")) of + N when N > Max -> + throw({bad_request, <<"count parameter too large">>}); + N when N < 0 -> + throw({bad_request, <<"count must be a positive integer">>}); + N -> + N + catch + error:badarg -> + throw({bad_request, <<"count must be a positive integer">>}) + end, UUIDs = [couch_uuids:new() || _ <- lists:seq(1, Count)], Etag = couch_httpd:make_etag(UUIDs), couch_httpd:etag_respond(Req, Etag, fun() -> @@ -109,51 +126,60 @@ handle_uuids_req(#httpd{method='GET'}=Req) -> handle_uuids_req(Req) -> send_method_not_allowed(Req, "GET"). - % Config request handler - % GET /_config/ % GET /_config -handle_config_req(#httpd{method='GET', path_parts=[_]}=Req) -> +handle_config_req(#httpd{method = 'GET', path_parts = [_]} = Req) -> ok = couch_httpd:verify_is_server_admin(Req), - Grouped = lists:foldl(fun({{Section, Key}, Value}, Acc) -> - case dict:is_key(Section, Acc) of - true -> - dict:append(Section, {list_to_binary(Key), list_to_binary(Value)}, Acc); - false -> - dict:store(Section, [{list_to_binary(Key), list_to_binary(Value)}], Acc) - end - end, dict:new(), config:all()), - KVs = dict:fold(fun(Section, Values, Acc) -> - [{list_to_binary(Section), {Values}} | Acc] - end, [], Grouped), + Grouped = lists:foldl( + fun({{Section, Key}, Value}, Acc) -> + case dict:is_key(Section, Acc) of + true -> + dict:append(Section, {list_to_binary(Key), list_to_binary(Value)}, Acc); + false -> + dict:store(Section, [{list_to_binary(Key), list_to_binary(Value)}], Acc) + end + end, + dict:new(), + config:all() + ), + KVs = dict:fold( + fun(Section, Values, Acc) -> + [{list_to_binary(Section), {Values}} | Acc] + end, + [], + Grouped + ), send_json(Req, 200, {KVs}); % GET /_config/Section -handle_config_req(#httpd{method='GET', path_parts=[_,Section]}=Req) -> +handle_config_req(#httpd{method = 'GET', path_parts = [_, Section]} = Req) -> ok = couch_httpd:verify_is_server_admin(Req), - KVs = [{list_to_binary(Key), list_to_binary(Value)} - || {Key, Value} <- config:get(Section)], + KVs = [ + {list_to_binary(Key), list_to_binary(Value)} + || {Key, Value} <- config:get(Section) + ], send_json(Req, 200, {KVs}); % GET /_config/Section/Key -handle_config_req(#httpd{method='GET', path_parts=[_, Section, Key]}=Req) -> +handle_config_req(#httpd{method = 'GET', path_parts = [_, Section, Key]} = Req) -> ok = couch_httpd:verify_is_server_admin(Req), case config:get(Section, Key, undefined) of - undefined -> - throw({not_found, unknown_config_value}); - Value -> - send_json(Req, 200, list_to_binary(Value)) + undefined -> + throw({not_found, unknown_config_value}); + Value -> + send_json(Req, 200, list_to_binary(Value)) end; % POST /_config/_reload - Flushes unpersisted config values from RAM -handle_config_req(#httpd{method='POST', path_parts=[_, <<"_reload">>]}=Req) -> +handle_config_req(#httpd{method = 'POST', path_parts = [_, <<"_reload">>]} = Req) -> couch_httpd:validate_ctype(Req, "application/json"), _ = couch_httpd:body(Req), ok = couch_httpd:verify_is_server_admin(Req), ok = config:reload(), send_json(Req, 200, {[{ok, true}]}); % PUT or DELETE /_config/Section/Key -handle_config_req(#httpd{method=Method, path_parts=[_, Section, Key]}=Req) - when (Method == 'PUT') or (Method == 'DELETE') -> +handle_config_req(#httpd{method = Method, path_parts = [_, Section, Key]} = Req) when + (Method == 'PUT') or (Method == 'DELETE') +-> ok = couch_httpd:verify_is_server_admin(Req), couch_util:check_config_blacklist(Section), Persist = couch_httpd:header_value(Req, "X-Couch-Persist") /= "false", @@ -169,19 +195,25 @@ handle_config_req(#httpd{method=Method, path_parts=[_, Section, Key]}=Req) % variable itself. FallbackWhitelist = [{<<"chttpd">>, <<"config_whitelist">>}], - Whitelist = case couch_util:parse_term(WhitelistValue) of - {ok, Value} when is_list(Value) -> - Value; - {ok, _NonListValue} -> - FallbackWhitelist; - {error, _} -> - [{WhitelistSection, WhitelistKey}] = FallbackWhitelist, - couch_log:error("Only whitelisting ~s/~s due to error" - " parsing: ~p", - [WhitelistSection, WhitelistKey, - WhitelistValue]), - FallbackWhitelist - end, + Whitelist = + case couch_util:parse_term(WhitelistValue) of + {ok, Value} when is_list(Value) -> + Value; + {ok, _NonListValue} -> + FallbackWhitelist; + {error, _} -> + [{WhitelistSection, WhitelistKey}] = FallbackWhitelist, + couch_log:error( + "Only whitelisting ~s/~s due to error" + " parsing: ~p", + [ + WhitelistSection, + WhitelistKey, + WhitelistValue + ] + ), + FallbackWhitelist + end, IsRequestedKeyVal = fun(Element) -> case Element of @@ -207,8 +239,12 @@ handle_config_req(#httpd{method=Method, path_parts=[_, Section, Key]}=Req) handle_approved_config_req(Req, Persist); _NotWhitelisted -> % Disallow modifying this non-whitelisted variable. - send_error(Req, 400, <<"modification_not_allowed">>, - ?l2b("This config variable is read-only")) + send_error( + Req, + 400, + <<"modification_not_allowed">>, + ?l2b("This config variable is read-only") + ) end end; handle_config_req(Req) -> @@ -218,52 +254,60 @@ handle_config_req(Req) -> % "value" handle_approved_config_req(Req, Persist) -> Query = couch_httpd:qs(Req), - UseRawValue = case lists:keyfind("raw", 1, Query) of - false -> false; % Not specified - {"raw", ""} -> false; % Specified with no value, i.e. "?raw" and "?raw=" - {"raw", "false"} -> false; - {"raw", "true"} -> true; - {"raw", InvalidValue} -> InvalidValue - end, + UseRawValue = + case lists:keyfind("raw", 1, Query) of + % Not specified + false -> false; + % Specified with no value, i.e. "?raw" and "?raw=" + {"raw", ""} -> false; + {"raw", "false"} -> false; + {"raw", "true"} -> true; + {"raw", InvalidValue} -> InvalidValue + end, handle_approved_config_req(Req, Persist, UseRawValue). -handle_approved_config_req(#httpd{method='PUT', path_parts=[_, Section, Key]}=Req, - Persist, UseRawValue) - when UseRawValue =:= false orelse UseRawValue =:= true -> +handle_approved_config_req( + #httpd{method = 'PUT', path_parts = [_, Section, Key]} = Req, + Persist, + UseRawValue +) when + UseRawValue =:= false orelse UseRawValue =:= true +-> RawValue = couch_httpd:json_body(Req), - Value = case UseRawValue of - true -> - % Client requests no change to the provided value. - RawValue; - false -> - % Pre-process the value as necessary. - case Section of - <<"admins">> -> - couch_passwords:hash_admin_password(RawValue); - _ -> - couch_util:trim(RawValue) - end - end, + Value = + case UseRawValue of + true -> + % Client requests no change to the provided value. + RawValue; + false -> + % Pre-process the value as necessary. + case Section of + <<"admins">> -> + couch_passwords:hash_admin_password(RawValue); + _ -> + couch_util:trim(RawValue) + end + end, OldValue = config:get(Section, Key, ""), case config:set(Section, Key, ?b2l(Value), Persist) of - ok -> - send_json(Req, 200, list_to_binary(OldValue)); - Error -> - throw(Error) + ok -> + send_json(Req, 200, list_to_binary(OldValue)); + Error -> + throw(Error) end; - -handle_approved_config_req(#httpd{method='PUT'}=Req, _Persist, UseRawValue) -> +handle_approved_config_req(#httpd{method = 'PUT'} = Req, _Persist, UseRawValue) -> Err = io_lib:format("Bad value for 'raw' option: ~s", [UseRawValue]), send_json(Req, 400, {[{error, ?l2b(Err)}]}); - % DELETE /_config/Section/Key -handle_approved_config_req(#httpd{method='DELETE',path_parts=[_,Section,Key]}=Req, - Persist, _UseRawValue) -> +handle_approved_config_req( + #httpd{method = 'DELETE', path_parts = [_, Section, Key]} = Req, + Persist, + _UseRawValue +) -> case config:get(Section, Key, undefined) of - undefined -> - throw({not_found, unknown_config_value}); - OldValue -> - config:delete(Section, Key, Persist), - send_json(Req, 200, list_to_binary(OldValue)) + undefined -> + throw({not_found, unknown_config_value}); + OldValue -> + config:delete(Section, Key, Persist), + send_json(Req, 200, list_to_binary(OldValue)) end. - diff --git a/src/couch/src/couch_httpd_multipart.erl b/src/couch/src/couch_httpd_multipart.erl index 33795a3a1..ecdf10562 100644 --- a/src/couch/src/couch_httpd_multipart.erl +++ b/src/couch/src/couch_httpd_multipart.erl @@ -30,47 +30,53 @@ decode_multipart_stream(ContentType, DataFun, Ref) -> ParentRef = erlang:monitor(process, Parent), put(mp_parent_ref, ParentRef), num_mp_writers(NumMpWriters), - {<<"--",_/binary>>, _, _} = couch_httpd:parse_multipart_request( - ContentType, DataFun, - fun(Next) -> mp_parse_doc(Next, []) end), + {<<"--", _/binary>>, _, _} = couch_httpd:parse_multipart_request( + ContentType, + DataFun, + fun(Next) -> mp_parse_doc(Next, []) end + ), unlink(Parent) - end), + end), Parser ! {get_doc_bytes, Ref, self()}, receive - {started_open_doc_revs, NewRef} -> - %% FIXME: How to remove the knowledge about this message? - {{started_open_doc_revs, NewRef}, Parser, ParserRef}; - {doc_bytes, Ref, DocBytes} -> - {{doc_bytes, Ref, DocBytes}, Parser, ParserRef}; - {'DOWN', ParserRef, _, _, normal} -> - ok; - {'DOWN', ParserRef, process, Parser, {{nocatch, {Error, Msg}}, _}} -> - couch_log:error("Multipart streamer ~p died with reason ~p", - [ParserRef, Msg]), - throw({Error, Msg}); - {'DOWN', ParserRef, _, _, Reason} -> - couch_log:error("Multipart streamer ~p died with reason ~p", - [ParserRef, Reason]), - throw({error, Reason}) + {started_open_doc_revs, NewRef} -> + %% FIXME: How to remove the knowledge about this message? + {{started_open_doc_revs, NewRef}, Parser, ParserRef}; + {doc_bytes, Ref, DocBytes} -> + {{doc_bytes, Ref, DocBytes}, Parser, ParserRef}; + {'DOWN', ParserRef, _, _, normal} -> + ok; + {'DOWN', ParserRef, process, Parser, {{nocatch, {Error, Msg}}, _}} -> + couch_log:error( + "Multipart streamer ~p died with reason ~p", + [ParserRef, Msg] + ), + throw({Error, Msg}); + {'DOWN', ParserRef, _, _, Reason} -> + couch_log:error( + "Multipart streamer ~p died with reason ~p", + [ParserRef, Reason] + ), + throw({error, Reason}) end. - mp_parse_doc({headers, H}, []) -> case couch_util:get_value("content-type", H) of - {"application/json", _} -> - fun (Next) -> - mp_parse_doc(Next, []) - end; - _ -> - throw({bad_ctype, <<"Content-Type must be application/json">>}) + {"application/json", _} -> + fun(Next) -> + mp_parse_doc(Next, []) + end; + _ -> + throw({bad_ctype, <<"Content-Type must be application/json">>}) end; mp_parse_doc({body, Bytes}, AccBytes) -> - fun (Next) -> + fun(Next) -> mp_parse_doc(Next, [Bytes | AccBytes]) end; mp_parse_doc(body_end, AccBytes) -> - receive {get_doc_bytes, Ref, From} -> - From ! {doc_bytes, Ref, lists:reverse(AccBytes)} + receive + {get_doc_bytes, Ref, From} -> + From ! {doc_bytes, Ref, lists:reverse(AccBytes)} end, fun(Next) -> mp_parse_atts(Next, {Ref, [], 0, orddict:new(), []}) @@ -81,7 +87,7 @@ mp_parse_atts({headers, _}, Acc) -> mp_parse_atts(body_end, Acc) -> fun(Next) -> mp_parse_atts(Next, Acc) end; mp_parse_atts({body, Bytes}, {Ref, Chunks, Offset, Counters, Waiting}) -> - case maybe_send_data({Ref, Chunks++[Bytes], Offset, Counters, Waiting}) of + case maybe_send_data({Ref, Chunks ++ [Bytes], Offset, Counters, Waiting}) of abort_parsing -> fun(Next) -> mp_abort_parse_atts(Next, nil) end; NewAcc -> @@ -91,34 +97,34 @@ mp_parse_atts(eof, {Ref, Chunks, Offset, Counters, Waiting}) -> N = num_mp_writers(), M = length(Counters), case (M == N) andalso Chunks == [] of - true -> - ok; - false -> - ParentRef = get(mp_parent_ref), - receive - abort_parsing -> + true -> ok; - {get_bytes, Ref, From} -> - C2 = update_writer(From, Counters), - case maybe_send_data({Ref, Chunks, Offset, C2, [From|Waiting]}) of - abort_parsing -> - ok; - NewAcc -> - mp_parse_atts(eof, NewAcc) - end; - {'DOWN', ParentRef, _, _, _} -> - exit(mp_reader_coordinator_died); - {'DOWN', WriterRef, _, WriterPid, _} -> - case remove_writer(WriterPid, WriterRef, Counters) of + false -> + ParentRef = get(mp_parent_ref), + receive abort_parsing -> ok; - C2 -> - NewAcc = {Ref, Chunks, Offset, C2, Waiting -- [WriterPid]}, - mp_parse_atts(eof, NewAcc) + {get_bytes, Ref, From} -> + C2 = update_writer(From, Counters), + case maybe_send_data({Ref, Chunks, Offset, C2, [From | Waiting]}) of + abort_parsing -> + ok; + NewAcc -> + mp_parse_atts(eof, NewAcc) + end; + {'DOWN', ParentRef, _, _, _} -> + exit(mp_reader_coordinator_died); + {'DOWN', WriterRef, _, WriterPid, _} -> + case remove_writer(WriterPid, WriterRef, Counters) of + abort_parsing -> + ok; + C2 -> + NewAcc = {Ref, Chunks, Offset, C2, Waiting -- [WriterPid]}, + mp_parse_atts(eof, NewAcc) + end + after 300000 -> + ok end - after 300000 -> - ok - end end. mp_abort_parse_atts(eof, _) -> @@ -127,82 +133,89 @@ mp_abort_parse_atts(_, _) -> fun(Next) -> mp_abort_parse_atts(Next, nil) end. maybe_send_data({Ref, Chunks, Offset, Counters, Waiting}) -> - receive {get_bytes, Ref, From} -> - NewCounters = update_writer(From, Counters), - maybe_send_data({Ref, Chunks, Offset, NewCounters, [From|Waiting]}) + receive + {get_bytes, Ref, From} -> + NewCounters = update_writer(From, Counters), + maybe_send_data({Ref, Chunks, Offset, NewCounters, [From | Waiting]}) after 0 -> % reply to as many writers as possible - NewWaiting = lists:filter(fun(Writer) -> - {_, WhichChunk} = orddict:fetch(Writer, Counters), - ListIndex = WhichChunk - Offset, - if ListIndex =< length(Chunks) -> - Writer ! {bytes, Ref, lists:nth(ListIndex, Chunks)}, - false; - true -> - true - end - end, Waiting), + NewWaiting = lists:filter( + fun(Writer) -> + {_, WhichChunk} = orddict:fetch(Writer, Counters), + ListIndex = WhichChunk - Offset, + if + ListIndex =< length(Chunks) -> + Writer ! {bytes, Ref, lists:nth(ListIndex, Chunks)}, + false; + true -> + true + end + end, + Waiting + ), % check if we can drop a chunk from the head of the list - SmallestIndex = case Counters of - [] -> - 0; - _ -> - lists:min([C || {_WPid, {_WRef, C}} <- Counters]) - end, + SmallestIndex = + case Counters of + [] -> + 0; + _ -> + lists:min([C || {_WPid, {_WRef, C}} <- Counters]) + end, Size = length(Counters), N = num_mp_writers(), - if Size == N andalso SmallestIndex == (Offset+1) -> - NewChunks = tl(Chunks), - NewOffset = Offset+1; - true -> - NewChunks = Chunks, - NewOffset = Offset + if + Size == N andalso SmallestIndex == (Offset + 1) -> + NewChunks = tl(Chunks), + NewOffset = Offset + 1; + true -> + NewChunks = Chunks, + NewOffset = Offset end, % we should wait for a writer if no one has written the last chunk LargestIndex = lists:max([0] ++ [C || {_WPid, {_WRef, C}} <- Counters]), - if LargestIndex >= (Offset + length(Chunks)) -> - % someone has written all possible chunks, keep moving - {Ref, NewChunks, NewOffset, Counters, NewWaiting}; - true -> - ParentRef = get(mp_parent_ref), - receive - abort_parsing -> - abort_parsing; - {'DOWN', ParentRef, _, _, _} -> - exit(mp_reader_coordinator_died); - {'DOWN', WriterRef, _, WriterPid, _} -> - case remove_writer(WriterPid, WriterRef, Counters) of + if + LargestIndex >= (Offset + length(Chunks)) -> + % someone has written all possible chunks, keep moving + {Ref, NewChunks, NewOffset, Counters, NewWaiting}; + true -> + ParentRef = get(mp_parent_ref), + receive abort_parsing -> abort_parsing; - C2 -> - RestWaiting = NewWaiting -- [WriterPid], - NewAcc = {Ref, NewChunks, NewOffset, C2, RestWaiting}, - maybe_send_data(NewAcc) - end; - {get_bytes, Ref, X} -> - C2 = update_writer(X, Counters), - maybe_send_data({Ref, NewChunks, NewOffset, C2, [X|NewWaiting]}) - after 300000 -> - abort_parsing - end + {'DOWN', ParentRef, _, _, _} -> + exit(mp_reader_coordinator_died); + {'DOWN', WriterRef, _, WriterPid, _} -> + case remove_writer(WriterPid, WriterRef, Counters) of + abort_parsing -> + abort_parsing; + C2 -> + RestWaiting = NewWaiting -- [WriterPid], + NewAcc = {Ref, NewChunks, NewOffset, C2, RestWaiting}, + maybe_send_data(NewAcc) + end; + {get_bytes, Ref, X} -> + C2 = update_writer(X, Counters), + maybe_send_data({Ref, NewChunks, NewOffset, C2, [X | NewWaiting]}) + after 300000 -> + abort_parsing + end end end. - update_writer(WriterPid, Counters) -> UpdateFun = fun({WriterRef, Count}) -> {WriterRef, Count + 1} end, - InitialValue = case orddict:find(WriterPid, Counters) of - {ok, IV} -> - IV; - error -> - WriterRef = erlang:monitor(process, WriterPid), - {WriterRef, 1} - end, + InitialValue = + case orddict:find(WriterPid, Counters) of + {ok, IV} -> + IV; + error -> + WriterRef = erlang:monitor(process, WriterPid), + {WriterRef, 1} + end, orddict:update(WriterPid, UpdateFun, InitialValue, Counters). - remove_writer(WriterPid, WriterRef, Counters) -> case orddict:find(WriterPid, Counters) of {ok, {WriterRef, _}} -> @@ -221,11 +234,9 @@ remove_writer(WriterPid, WriterRef, Counters) -> abort_parsing end. - num_mp_writers(N) -> erlang:put(mp_att_writers, N). - num_mp_writers() -> case erlang:get(mp_att_writers) of undefined -> 1; @@ -235,15 +246,21 @@ num_mp_writers() -> encode_multipart_stream(_Boundary, JsonBytes, [], WriteFun, _AttFun) -> WriteFun(JsonBytes); encode_multipart_stream(Boundary, JsonBytes, Atts, WriteFun, AttFun) -> - WriteFun([<<"--", Boundary/binary, - "\r\nContent-Type: application/json\r\n\r\n">>, - JsonBytes, <<"\r\n--", Boundary/binary>>]), + WriteFun([ + <<"--", Boundary/binary, "\r\nContent-Type: application/json\r\n\r\n">>, + JsonBytes, + <<"\r\n--", Boundary/binary>> + ]), atts_to_mp(Atts, Boundary, WriteFun, AttFun). atts_to_mp([], _Boundary, WriteFun, _AttFun) -> WriteFun(<<"--">>); -atts_to_mp([{Att, Name, Len, Type, Encoding} | RestAtts], Boundary, WriteFun, - AttFun) -> +atts_to_mp( + [{Att, Name, Len, Type, Encoding} | RestAtts], + Boundary, + WriteFun, + AttFun +) -> LengthBin = list_to_binary(integer_to_list(Len)), % write headers WriteFun(<<"\r\nContent-Disposition: attachment; filename=\"", Name/binary, "\"">>), @@ -264,40 +281,52 @@ atts_to_mp([{Att, Name, Len, Type, Encoding} | RestAtts], Boundary, WriteFun, atts_to_mp(RestAtts, Boundary, WriteFun, AttFun). length_multipart_stream(Boundary, JsonBytes, Atts) -> - AttsSize = lists:foldl(fun({_Att, Name, Len, Type, Encoding}, AccAttsSize) -> - AccAttsSize + - 4 + % "\r\n\r\n" - length(integer_to_list(Len)) + - Len + - 4 + % "\r\n--" - size(Boundary) + - % attachment headers - % (the length of the Content-Length has already been set) - size(Name) + - size(Type) + - length("\r\nContent-Disposition: attachment; filename=\"\"") + - length("\r\nContent-Type: ") + - length("\r\nContent-Length: ") + - case Encoding of - identity -> - 0; - _ -> - length(atom_to_list(Encoding)) + - length("\r\nContent-Encoding: ") - end - end, 0, Atts), - if AttsSize == 0 -> - {<<"application/json">>, iolist_size(JsonBytes)}; - true -> - {<<"multipart/related; boundary=\"", Boundary/binary, "\"">>, - 2 + % "--" - size(Boundary) + - 36 + % "\r\ncontent-type: application/json\r\n\r\n" - iolist_size(JsonBytes) + - 4 + % "\r\n--" - size(Boundary) + - + AttsSize + - 2 % "--" + AttsSize = lists:foldl( + fun({_Att, Name, Len, Type, Encoding}, AccAttsSize) -> + AccAttsSize + + % "\r\n\r\n" + 4 + + length(integer_to_list(Len)) + + Len + + % "\r\n--" + 4 + + size(Boundary) + + % attachment headers + % (the length of the Content-Length has already been set) + size(Name) + + size(Type) + + length("\r\nContent-Disposition: attachment; filename=\"\"") + + length("\r\nContent-Type: ") + + length("\r\nContent-Length: ") + + case Encoding of + identity -> + 0; + _ -> + length(atom_to_list(Encoding)) + + length("\r\nContent-Encoding: ") + end + end, + 0, + Atts + ), + if + AttsSize == 0 -> + {<<"application/json">>, iolist_size(JsonBytes)}; + true -> + { + <<"multipart/related; boundary=\"", Boundary/binary, "\"">>, + % "--" + 2 + + size(Boundary) + + % "\r\ncontent-type: application/json\r\n\r\n" + 36 + + iolist_size(JsonBytes) + + % "\r\n--" + 4 + + size(Boundary) + + +AttsSize + + % "--" + 2 } end. diff --git a/src/couch/src/couch_httpd_rewrite.erl b/src/couch/src/couch_httpd_rewrite.erl index 40e5c9e3c..97f48a4c0 100644 --- a/src/couch/src/couch_httpd_rewrite.erl +++ b/src/couch/src/couch_httpd_rewrite.erl @@ -12,7 +12,6 @@ % % bind_path is based on bind method from Webmachine - %% @doc Module for URL rewriting by pattern matching. -module(couch_httpd_rewrite). @@ -25,7 +24,6 @@ -define(SEPARATOR, $\/). -define(MATCH_ALL, {bind, <<"*">>}). - %% doc The http rewrite handler. All rewriting is done from %% /dbname/_design/ddocname/_rewrite by default. %% @@ -110,13 +108,15 @@ %% "to": "/some/:foo", %% }} - - -handle_rewrite_req(#httpd{ - path_parts=[DbName, <<"_design">>, DesignName, _Rewrite|PathParts], - method=Method, - mochi_req=MochiReq}=Req, _Db, DDoc) -> - +handle_rewrite_req( + #httpd{ + path_parts = [DbName, <<"_design">>, DesignName, _Rewrite | PathParts], + method = Method, + mochi_req = MochiReq + } = Req, + _Db, + DDoc +) -> % we are in a design handler DesignId = <<"_design/", DesignName/binary>>, Prefix = <<"/", (?l2b(couch_util:url_encode(DbName)))/binary, "/", DesignId/binary>>, @@ -131,19 +131,27 @@ handle_rewrite_req(#httpd{ erlang:put(?REWRITE_COUNT, RewritesSoFar + 1) end, - #doc{body={Props}} = DDoc, + #doc{body = {Props}} = DDoc, % get rules from ddoc case couch_util:get_value(<<"rewrites">>, Props) of undefined -> - couch_httpd:send_error(Req, 404, <<"rewrite_error">>, - <<"Invalid path.">>); + couch_httpd:send_error( + Req, + 404, + <<"rewrite_error">>, + <<"Invalid path.">> + ); Bin when is_binary(Bin) -> - couch_httpd:send_error(Req, 400, <<"rewrite_error">>, - <<"Rewrite rules are a String. They must be a JSON Array.">>); + couch_httpd:send_error( + Req, + 400, + <<"rewrite_error">>, + <<"Rewrite rules are a String. They must be a JSON Array.">> + ); Rules -> % create dispatch list from rules - DispatchList = [make_rule(Rule) || {Rule} <- Rules], + DispatchList = [make_rule(Rule) || {Rule} <- Rules], Method1 = couch_util:to_binary(Method), % get raw path by matching url to a rule. Throws not_found. @@ -155,39 +163,45 @@ handle_rewrite_req(#httpd{ Path0 = string:join(NewPathParts, [?SEPARATOR]), % if path is relative detect it and rewrite path - Path1 = case mochiweb_util:safe_relative_path(Path0) of - undefined -> - ?b2l(Prefix) ++ "/" ++ Path0; - P1 -> - ?b2l(Prefix) ++ "/" ++ P1 - end, + Path1 = + case mochiweb_util:safe_relative_path(Path0) of + undefined -> + ?b2l(Prefix) ++ "/" ++ Path0; + P1 -> + ?b2l(Prefix) ++ "/" ++ P1 + end, Path2 = normalize_path(Path1), - Path3 = case Bindings of - [] -> - Path2; - _ -> - [Path2, "?", mochiweb_util:urlencode(Bindings)] - end, + Path3 = + case Bindings of + [] -> + Path2; + _ -> + [Path2, "?", mochiweb_util:urlencode(Bindings)] + end, RawPath1 = ?b2l(iolist_to_binary(Path3)), % In order to do OAuth correctly, we have to save the % requested path. We use default so chained rewriting % wont replace the original header. - Headers = mochiweb_headers:default("x-couchdb-requested-path", - MochiReq:get(raw_path), - MochiReq:get(headers)), + Headers = mochiweb_headers:default( + "x-couchdb-requested-path", + MochiReq:get(raw_path), + MochiReq:get(headers) + ), couch_log:debug("rewrite to ~p ~n", [RawPath1]), % build a new mochiweb request - MochiReq1 = mochiweb_request:new(MochiReq:get(socket), - MochiReq:get(method), - RawPath1, - MochiReq:get(version), - Headers), + MochiReq1 = mochiweb_request:new( + MochiReq:get(socket), + MochiReq:get(method), + RawPath1, + MochiReq:get(version), + Headers + ), % cleanup, It force mochiweb to reparse raw uri. MochiReq1:cleanup(), @@ -198,14 +212,19 @@ handle_rewrite_req(#httpd{ default_fun = DefaultFun, url_handlers = UrlHandlers, user_ctx = UserCtx, - auth = Auth + auth = Auth } = Req, erlang:put(pre_rewrite_auth, Auth), erlang:put(pre_rewrite_user_ctx, UserCtx), - couch_httpd:handle_request_int(MochiReq1, DefaultFun, - UrlHandlers, DbUrlHandlers, DesignUrlHandlers) - end. + couch_httpd:handle_request_int( + MochiReq1, + DefaultFun, + UrlHandlers, + DbUrlHandlers, + DesignUrlHandlers + ) + end. quote_plus({bind, X}) -> mochiweb_util:quote_plus(X); @@ -216,7 +235,7 @@ quote_plus(X) -> %% 404 error not_found is raised try_bind_path([], _Method, _PathParts, _QueryList) -> throw(not_found); -try_bind_path([Dispatch|Rest], Method, PathParts, QueryList) -> +try_bind_path([Dispatch | Rest], Method, PathParts, QueryList) -> [{PathParts1, Method1}, RedirectPath, QueryArgs, Formats] = Dispatch, case bind_method(Method1, Method) of true -> @@ -225,22 +244,35 @@ try_bind_path([Dispatch|Rest], Method, PathParts, QueryList) -> Bindings1 = Bindings ++ QueryList, % we parse query args from the rule and fill % it eventually with bindings vars - QueryArgs1 = make_query_list(QueryArgs, Bindings1, - Formats, []), + QueryArgs1 = make_query_list( + QueryArgs, + Bindings1, + Formats, + [] + ), % remove params in QueryLists1 that are already in % QueryArgs1 - Bindings2 = lists:foldl(fun({K, V}, Acc) -> - K1 = to_binding(K), - KV = case couch_util:get_value(K1, QueryArgs1) of - undefined -> [{K1, V}]; - _V1 -> [] + Bindings2 = lists:foldl( + fun({K, V}, Acc) -> + K1 = to_binding(K), + KV = + case couch_util:get_value(K1, QueryArgs1) of + undefined -> [{K1, V}]; + _V1 -> [] + end, + Acc ++ KV end, - Acc ++ KV - end, [], Bindings1), + [], + Bindings1 + ), FinalBindings = Bindings2 ++ QueryArgs1, - NewPathParts = make_new_path(RedirectPath, FinalBindings, - Remaining, []), + NewPathParts = make_new_path( + RedirectPath, + FinalBindings, + Remaining, + [] + ), {NewPathParts, FinalBindings}; fail -> try_bind_path(Rest, Method, PathParts, QueryList) @@ -254,37 +286,51 @@ try_bind_path([Dispatch|Rest], Method, PathParts, QueryList) -> %% passed in url. make_query_list([], _Bindings, _Formats, Acc) -> Acc; -make_query_list([{Key, {Value}}|Rest], Bindings, Formats, Acc) -> +make_query_list([{Key, {Value}} | Rest], Bindings, Formats, Acc) -> Value1 = {Value}, - make_query_list(Rest, Bindings, Formats, [{to_binding(Key), Value1}|Acc]); -make_query_list([{Key, Value}|Rest], Bindings, Formats, Acc) when is_binary(Value) -> + make_query_list(Rest, Bindings, Formats, [{to_binding(Key), Value1} | Acc]); +make_query_list([{Key, Value} | Rest], Bindings, Formats, Acc) when is_binary(Value) -> Value1 = replace_var(Value, Bindings, Formats), - make_query_list(Rest, Bindings, Formats, [{to_binding(Key), Value1}|Acc]); -make_query_list([{Key, Value}|Rest], Bindings, Formats, Acc) when is_list(Value) -> + make_query_list(Rest, Bindings, Formats, [{to_binding(Key), Value1} | Acc]); +make_query_list([{Key, Value} | Rest], Bindings, Formats, Acc) when is_list(Value) -> Value1 = replace_var(Value, Bindings, Formats), - make_query_list(Rest, Bindings, Formats, [{to_binding(Key), Value1}|Acc]); -make_query_list([{Key, Value}|Rest], Bindings, Formats, Acc) -> - make_query_list(Rest, Bindings, Formats, [{to_binding(Key), Value}|Acc]). + make_query_list(Rest, Bindings, Formats, [{to_binding(Key), Value1} | Acc]); +make_query_list([{Key, Value} | Rest], Bindings, Formats, Acc) -> + make_query_list(Rest, Bindings, Formats, [{to_binding(Key), Value} | Acc]). -replace_var(<<"*">>=Value, Bindings, Formats) -> +replace_var(<<"*">> = Value, Bindings, Formats) -> get_var(Value, Bindings, Value, Formats); replace_var(<<":", Var/binary>> = Value, Bindings, Formats) -> get_var(Var, Bindings, Value, Formats); replace_var(Value, _Bindings, _Formats) when is_binary(Value) -> Value; replace_var(Value, Bindings, Formats) when is_list(Value) -> - lists:reverse(lists:foldl(fun - (<<":", Var/binary>>=Value1, Acc) -> - [get_var(Var, Bindings, Value1, Formats)|Acc]; + lists:reverse( + lists:foldl( + fun + (<<":", Var/binary>> = Value1, Acc) -> + [get_var(Var, Bindings, Value1, Formats) | Acc]; (Value1, Acc) -> - [Value1|Acc] - end, [], Value)); + [Value1 | Acc] + end, + [], + Value + ) + ); replace_var(Value, _Bindings, _Formats) -> Value. - + maybe_json(Key, Value) -> - case lists:member(Key, [<<"key">>, <<"startkey">>, <<"start_key">>, - <<"endkey">>, <<"end_key">>, <<"keys">>]) of + case + lists:member(Key, [ + <<"key">>, + <<"startkey">>, + <<"start_key">>, + <<"endkey">>, + <<"end_key">>, + <<"keys">> + ]) + of true -> ?JSON_ENCODE(Value); false -> @@ -299,7 +345,7 @@ get_var(VarName, Props, Default, Formats) -> maybe_format(VarName, Value, Formats) -> case couch_util:get_value(VarName, Formats) of undefined -> - Value; + Value; Format -> format(Format, Value) end. @@ -324,7 +370,7 @@ format(<<"bool">>, Value) when is_list(Value) -> _ -> Value end; format(_Format, Value) -> - Value. + Value. %% doc: build new patch from bindings. bindings are query args %% (+ dynamic query rewritten if needed) and bindings found in @@ -334,94 +380,103 @@ make_new_path([], _Bindings, _Remaining, Acc) -> make_new_path([?MATCH_ALL], _Bindings, Remaining, Acc) -> Acc1 = lists:reverse(Acc) ++ Remaining, Acc1; -make_new_path([?MATCH_ALL|_Rest], _Bindings, Remaining, Acc) -> +make_new_path([?MATCH_ALL | _Rest], _Bindings, Remaining, Acc) -> Acc1 = lists:reverse(Acc) ++ Remaining, Acc1; -make_new_path([{bind, P}|Rest], Bindings, Remaining, Acc) -> - P2 = case couch_util:get_value({bind, P}, Bindings) of - undefined -> << "undefined">>; - P1 -> - iolist_to_binary(P1) - end, - make_new_path(Rest, Bindings, Remaining, [P2|Acc]); -make_new_path([P|Rest], Bindings, Remaining, Acc) -> - make_new_path(Rest, Bindings, Remaining, [P|Acc]). - +make_new_path([{bind, P} | Rest], Bindings, Remaining, Acc) -> + P2 = + case couch_util:get_value({bind, P}, Bindings) of + undefined -> <<"undefined">>; + P1 -> iolist_to_binary(P1) + end, + make_new_path(Rest, Bindings, Remaining, [P2 | Acc]); +make_new_path([P | Rest], Bindings, Remaining, Acc) -> + make_new_path(Rest, Bindings, Remaining, [P | Acc]). %% @doc If method of the query fith the rule method. If the %% method rule is '*', which is the default, all %% request method will bind. It allows us to make rules %% depending on HTTP method. -bind_method(?MATCH_ALL, _Method ) -> +bind_method(?MATCH_ALL, _Method) -> true; bind_method({bind, Method}, Method) -> true; bind_method(_, _) -> false. - %% @doc bind path. Using the rule from we try to bind variables given %% to the current url by pattern matching bind_path([], [], Bindings) -> {ok, [], Bindings}; -bind_path([?MATCH_ALL], [Match|_RestMatch]=Rest, Bindings) -> - {ok, Rest, [{?MATCH_ALL, Match}|Bindings]}; +bind_path([?MATCH_ALL], [Match | _RestMatch] = Rest, Bindings) -> + {ok, Rest, [{?MATCH_ALL, Match} | Bindings]}; bind_path(_, [], _) -> fail; -bind_path([{bind, Token}|RestToken],[Match|RestMatch],Bindings) -> - bind_path(RestToken, RestMatch, [{{bind, Token}, Match}|Bindings]); -bind_path([Token|RestToken], [Token|RestMatch], Bindings) -> +bind_path([{bind, Token} | RestToken], [Match | RestMatch], Bindings) -> + bind_path(RestToken, RestMatch, [{{bind, Token}, Match} | Bindings]); +bind_path([Token | RestToken], [Token | RestMatch], Bindings) -> bind_path(RestToken, RestMatch, Bindings); bind_path(_, _, _) -> fail. - %% normalize path. -normalize_path(Path) -> - "/" ++ string:join(normalize_path1(string:tokens(Path, - "/"), []), [?SEPARATOR]). - +normalize_path(Path) -> + "/" ++ + string:join( + normalize_path1( + string:tokens( + Path, + "/" + ), + [] + ), + [?SEPARATOR] + ). normalize_path1([], Acc) -> lists:reverse(Acc); -normalize_path1([".."|Rest], Acc) -> - Acc1 = case Acc of - [] -> [".."|Acc]; - [T|_] when T =:= ".." -> [".."|Acc]; - [_|R] -> R - end, +normalize_path1([".." | Rest], Acc) -> + Acc1 = + case Acc of + [] -> [".." | Acc]; + [T | _] when T =:= ".." -> [".." | Acc]; + [_ | R] -> R + end, normalize_path1(Rest, Acc1); -normalize_path1(["."|Rest], Acc) -> +normalize_path1(["." | Rest], Acc) -> normalize_path1(Rest, Acc); -normalize_path1([Path|Rest], Acc) -> - normalize_path1(Rest, [Path|Acc]). - +normalize_path1([Path | Rest], Acc) -> + normalize_path1(Rest, [Path | Acc]). %% @doc transform json rule in erlang for pattern matching make_rule(Rule) -> - Method = case couch_util:get_value(<<"method">>, Rule) of - undefined -> ?MATCH_ALL; - M -> to_binding(M) - end, - QueryArgs = case couch_util:get_value(<<"query">>, Rule) of - undefined -> []; - {Args} -> Args + Method = + case couch_util:get_value(<<"method">>, Rule) of + undefined -> ?MATCH_ALL; + M -> to_binding(M) end, - FromParts = case couch_util:get_value(<<"from">>, Rule) of - undefined -> [?MATCH_ALL]; - From -> - parse_path(From) + QueryArgs = + case couch_util:get_value(<<"query">>, Rule) of + undefined -> []; + {Args} -> Args end, - ToParts = case couch_util:get_value(<<"to">>, Rule) of - undefined -> - throw({error, invalid_rewrite_target}); - To -> - parse_path(To) + FromParts = + case couch_util:get_value(<<"from">>, Rule) of + undefined -> [?MATCH_ALL]; + From -> parse_path(From) + end, + ToParts = + case couch_util:get_value(<<"to">>, Rule) of + undefined -> + throw({error, invalid_rewrite_target}); + To -> + parse_path(To) + end, + Formats = + case couch_util:get_value(<<"formats">>, Rule) of + undefined -> []; + {Fmts} -> Fmts end, - Formats = case couch_util:get_value(<<"formats">>, Rule) of - undefined -> []; - {Fmts} -> Fmts - end, [{FromParts, Method}, ToParts, QueryArgs, Formats]. parse_path(Path) -> @@ -433,43 +488,59 @@ parse_path(Path) -> %% in erlang atom. path_to_list([], Acc, _DotDotCount) -> lists:reverse(Acc); -path_to_list([<<>>|R], Acc, DotDotCount) -> +path_to_list([<<>> | R], Acc, DotDotCount) -> path_to_list(R, Acc, DotDotCount); -path_to_list([<<"*">>|R], Acc, DotDotCount) -> - path_to_list(R, [?MATCH_ALL|Acc], DotDotCount); -path_to_list([<<"..">>|R], Acc, DotDotCount) when DotDotCount == 2 -> +path_to_list([<<"*">> | R], Acc, DotDotCount) -> + path_to_list(R, [?MATCH_ALL | Acc], DotDotCount); +path_to_list([<<"..">> | R], Acc, DotDotCount) when DotDotCount == 2 -> case chttpd_util:get_chttpd_config_boolean("secure_rewrites", true) of false -> - path_to_list(R, [<<"..">>|Acc], DotDotCount+1); + path_to_list(R, [<<"..">> | Acc], DotDotCount + 1); true -> - couch_log:info("insecure_rewrite_rule ~p blocked", - [lists:reverse(Acc) ++ [<<"..">>] ++ R]), + couch_log:info( + "insecure_rewrite_rule ~p blocked", + [lists:reverse(Acc) ++ [<<"..">>] ++ R] + ), throw({insecure_rewrite_rule, "too many ../.. segments"}) end; -path_to_list([<<"..">>|R], Acc, DotDotCount) -> - path_to_list(R, [<<"..">>|Acc], DotDotCount+1); -path_to_list([P|R], Acc, DotDotCount) -> - P1 = case P of - <<":", Var/binary>> -> - to_binding(Var); - _ -> P - end, - path_to_list(R, [P1|Acc], DotDotCount). +path_to_list([<<"..">> | R], Acc, DotDotCount) -> + path_to_list(R, [<<"..">> | Acc], DotDotCount + 1); +path_to_list([P | R], Acc, DotDotCount) -> + P1 = + case P of + <<":", Var/binary>> -> + to_binding(Var); + _ -> + P + end, + path_to_list(R, [P1 | Acc], DotDotCount). maybe_encode_bindings([]) -> []; -maybe_encode_bindings(Props) -> - lists:foldl(fun +maybe_encode_bindings(Props) -> + lists:foldl( + fun ({{bind, <<"*">>}, _V}, Acc) -> Acc; ({{bind, K}, V}, Acc) -> V1 = iolist_to_binary(maybe_json(K, V)), - [{K, V1}|Acc] - end, [], Props). - -decode_query_value({K,V}) -> - case lists:member(K, ["key", "startkey", "start_key", - "endkey", "end_key", "keys"]) of + [{K, V1} | Acc] + end, + [], + Props + ). + +decode_query_value({K, V}) -> + case + lists:member(K, [ + "key", + "startkey", + "start_key", + "endkey", + "end_key", + "keys" + ]) + of true -> {to_binding(K), ?JSON_DECODE(V)}; false -> diff --git a/src/couch/src/couch_httpd_vhost.erl b/src/couch/src/couch_httpd_vhost.erl index 409631d25..0bff6a36d 100644 --- a/src/couch/src/couch_httpd_vhost.erl +++ b/src/couch/src/couch_httpd_vhost.erl @@ -33,9 +33,10 @@ -define(RELISTEN_DELAY, 5000). -record(vhosts_state, { - vhosts, - vhost_globals, - vhosts_fun}). + vhosts, + vhost_globals, + vhosts_fun +}). %% doc the vhost manager. %% This gen_server keep state of vhosts added to the ini and try to @@ -109,34 +110,44 @@ dispatch_host_int(MochiReq) -> #vhosts_state{ vhost_globals = VHostGlobals, vhosts = VHosts, - vhosts_fun=Fun} = get_state(), + vhosts_fun = Fun + } = get_state(), {"/" ++ VPath, Query, Fragment} = mochiweb_util:urlsplit_path(MochiReq:get(raw_path)), - VPathParts = string:tokens(VPath, "/"), + VPathParts = string:tokens(VPath, "/"), VHost = host(MochiReq), {VHostParts, VhostPort} = split_host_port(VHost), - FinalMochiReq = case try_bind_vhost(VHosts, lists:reverse(VHostParts), - VhostPort, VPathParts) of - no_vhost_matched -> MochiReq; - {VhostTarget, NewPath} -> - case vhost_global(VHostGlobals, MochiReq) of - true -> - MochiReq; - _Else -> - NewPath1 = mochiweb_util:urlunsplit_path({NewPath, Query, - Fragment}), - MochiReq1 = mochiweb_request:new(MochiReq:get(socket), - MochiReq:get(method), - NewPath1, - MochiReq:get(version), - MochiReq:get(headers)), - Fun(MochiReq1, VhostTarget) - end - end, + FinalMochiReq = + case + try_bind_vhost( + VHosts, + lists:reverse(VHostParts), + VhostPort, + VPathParts + ) + of + no_vhost_matched -> + MochiReq; + {VhostTarget, NewPath} -> + case vhost_global(VHostGlobals, MochiReq) of + true -> + MochiReq; + _Else -> + NewPath1 = mochiweb_util:urlunsplit_path({NewPath, Query, Fragment}), + MochiReq1 = mochiweb_request:new( + MochiReq:get(socket), + MochiReq:get(method), + NewPath1, + MochiReq:get(version), + MochiReq:get(headers) + ), + Fun(MochiReq1, VhostTarget) + end + end, FinalMochiReq. -append_path("/"=_Target, "/"=_Path) -> +append_path("/" = _Target, "/" = _Path) -> "/"; append_path(Target, Path) -> Target ++ Path. @@ -148,15 +159,20 @@ redirect_to_vhost(MochiReq, VhostTarget) -> couch_log:debug("Vhost Target: '~p'~n", [Target]), - Headers = mochiweb_headers:enter("x-couchdb-vhost-path", Path, - MochiReq:get(headers)), + Headers = mochiweb_headers:enter( + "x-couchdb-vhost-path", + Path, + MochiReq:get(headers) + ), % build a new mochiweb request - MochiReq1 = mochiweb_request:new(MochiReq:get(socket), - MochiReq:get(method), - Target, - MochiReq:get(version), - Headers), + MochiReq1 = mochiweb_request:new( + MochiReq:get(socket), + MochiReq:get(method), + Target, + MochiReq:get(version), + Headers + ), % cleanup, It force mochiweb to reparse raw uri. MochiReq1:cleanup(), MochiReq1. @@ -164,23 +180,25 @@ redirect_to_vhost(MochiReq, VhostTarget) -> %% if so, then it will not be rewritten, but will run as a normal couchdb request. %* normally you'd use this for _uuids _utils and a few of the others you want to %% keep available on vhosts. You can also use it to make databases 'global'. -vhost_global( VhostGlobals, MochiReq) -> +vhost_global(VhostGlobals, MochiReq) -> RawUri = MochiReq:get(raw_path), {"/" ++ Path, _, _} = mochiweb_util:urlsplit_path(RawUri), - Front = case couch_httpd:partition(Path) of - {"", "", ""} -> - "/"; % Special case the root url handler - {FirstPart, _, _} -> - FirstPart - end, - [true] == [true||V <- VhostGlobals, V == Front]. + Front = + case couch_httpd:partition(Path) of + {"", "", ""} -> + % Special case the root url handler + "/"; + {FirstPart, _, _} -> + FirstPart + end, + [true] == [true || V <- VhostGlobals, V == Front]. %% bind host %% first it try to bind the port then the hostname. try_bind_vhost([], _HostParts, _Port, _PathParts) -> no_vhost_matched; -try_bind_vhost([VhostSpec|Rest], HostParts, Port, PathParts) -> +try_bind_vhost([VhostSpec | Rest], HostParts, Port, PathParts) -> {{VHostParts, VPort, VPath}, Path} = VhostSpec, case bind_port(VPort, Port) of ok -> @@ -191,12 +209,18 @@ try_bind_vhost([VhostSpec|Rest], HostParts, Port, PathParts) -> Path1 = make_target(Path, Bindings, Remainings, []), {make_path(Path1), make_path(PathParts1)}; fail -> - try_bind_vhost(Rest, HostParts, Port, - PathParts) + try_bind_vhost( + Rest, + HostParts, + Port, + PathParts + ) end; - fail -> try_bind_vhost(Rest, HostParts, Port, PathParts) + fail -> + try_bind_vhost(Rest, HostParts, Port, PathParts) end; - fail -> try_bind_vhost(Rest, HostParts, Port, PathParts) + fail -> + try_bind_vhost(Rest, HostParts, Port, PathParts) end. %% doc: build new patch from bindings. bindings are query args @@ -209,72 +233,82 @@ make_target([], _Bindings, _Remaining, Acc) -> make_target([?MATCH_ALL], _Bindings, Remaining, Acc) -> Acc1 = lists:reverse(Acc) ++ Remaining, Acc1; -make_target([?MATCH_ALL|_Rest], _Bindings, Remaining, Acc) -> +make_target([?MATCH_ALL | _Rest], _Bindings, Remaining, Acc) -> Acc1 = lists:reverse(Acc) ++ Remaining, Acc1; -make_target([{bind, P}|Rest], Bindings, Remaining, Acc) -> - P2 = case couch_util:get_value({bind, P}, Bindings) of - undefined -> "undefined"; - P1 -> P1 - end, - make_target(Rest, Bindings, Remaining, [P2|Acc]); -make_target([P|Rest], Bindings, Remaining, Acc) -> - make_target(Rest, Bindings, Remaining, [P|Acc]). +make_target([{bind, P} | Rest], Bindings, Remaining, Acc) -> + P2 = + case couch_util:get_value({bind, P}, Bindings) of + undefined -> "undefined"; + P1 -> P1 + end, + make_target(Rest, Bindings, Remaining, [P2 | Acc]); +make_target([P | Rest], Bindings, Remaining, Acc) -> + make_target(Rest, Bindings, Remaining, [P | Acc]). %% bind port bind_port(Port, Port) -> ok; bind_port('*', _) -> ok; -bind_port(_,_) -> fail. +bind_port(_, _) -> fail. %% bind bhost -bind_vhost([],[], Bindings) -> {ok, Bindings, []}; -bind_vhost([?MATCH_ALL], [], _Bindings) -> fail; -bind_vhost([?MATCH_ALL], Rest, Bindings) -> {ok, Bindings, Rest}; -bind_vhost([], _HostParts, _Bindings) -> fail; -bind_vhost([{bind, Token}|Rest], [Match|RestHost], Bindings) -> - bind_vhost(Rest, RestHost, [{{bind, Token}, Match}|Bindings]); -bind_vhost([Cname|Rest], [Cname|RestHost], Bindings) -> +bind_vhost([], [], Bindings) -> + {ok, Bindings, []}; +bind_vhost([?MATCH_ALL], [], _Bindings) -> + fail; +bind_vhost([?MATCH_ALL], Rest, Bindings) -> + {ok, Bindings, Rest}; +bind_vhost([], _HostParts, _Bindings) -> + fail; +bind_vhost([{bind, Token} | Rest], [Match | RestHost], Bindings) -> + bind_vhost(Rest, RestHost, [{{bind, Token}, Match} | Bindings]); +bind_vhost([Cname | Rest], [Cname | RestHost], Bindings) -> bind_vhost(Rest, RestHost, Bindings); -bind_vhost(_, _, _) -> fail. +bind_vhost(_, _, _) -> + fail. %% bind path bind_path([], PathParts) -> {ok, PathParts}; bind_path(_VPathParts, []) -> fail; -bind_path([Path|VRest],[Path|Rest]) -> - bind_path(VRest, Rest); +bind_path([Path | VRest], [Path | Rest]) -> + bind_path(VRest, Rest); bind_path(_, _) -> fail. % utilities - %% create vhost list from ini host(MochiReq) -> XHost = chttpd_util:get_chttpd_config( - "x_forwarded_host", "X-Forwarded-Host"), + "x_forwarded_host", "X-Forwarded-Host" + ), case MochiReq:get_header_value(XHost) of undefined -> case MochiReq:get_header_value("Host") of undefined -> []; Value1 -> Value1 end; - Value -> Value + Value -> + Value end. make_vhosts() -> - Vhosts = lists:foldl(fun - ({_, ""}, Acc) -> - Acc; - ({Vhost, Path}, Acc) -> - [{parse_vhost(Vhost), split_path(Path)}|Acc] - end, [], config:get("vhosts")), + Vhosts = lists:foldl( + fun + ({_, ""}, Acc) -> + Acc; + ({Vhost, Path}, Acc) -> + [{parse_vhost(Vhost), split_path(Path)} | Acc] + end, + [], + config:get("vhosts") + ), lists:reverse(lists:usort(Vhosts)). - parse_vhost(Vhost) -> case urlsplit_netloc(Vhost, []) of {[], Path} -> @@ -289,15 +323,21 @@ parse_vhost(Vhost) -> {H1, P, string:tokens(Path, "/")} end. - split_host_port(HostAsString) -> case string:rchr(HostAsString, $:) of 0 -> {split_host(HostAsString), '*'}; N -> - HostPart = string:substr(HostAsString, 1, N-1), - case (catch erlang:list_to_integer(string:substr(HostAsString, - N+1, length(HostAsString)))) of + HostPart = string:substr(HostAsString, 1, N - 1), + case + (catch erlang:list_to_integer( + string:substr( + HostAsString, + N + 1, + length(HostAsString) + ) + )) + of {'EXIT', _} -> {split_host(HostAsString), '*'}; Port -> @@ -311,36 +351,34 @@ split_host(HostAsString) -> split_path(Path) -> make_spec(string:tokens(Path, "/"), []). - make_spec([], Acc) -> lists:reverse(Acc); -make_spec([""|R], Acc) -> +make_spec(["" | R], Acc) -> make_spec(R, Acc); -make_spec(["*"|R], Acc) -> - make_spec(R, [?MATCH_ALL|Acc]); -make_spec([P|R], Acc) -> +make_spec(["*" | R], Acc) -> + make_spec(R, [?MATCH_ALL | Acc]); +make_spec([P | R], Acc) -> P1 = parse_var(P), - make_spec(R, [P1|Acc]). - + make_spec(R, [P1 | Acc]). parse_var(P) -> case P of ":" ++ Var -> {bind, Var}; - _ -> P + _ -> + P end. - % mochiweb doesn't export it. urlsplit_netloc("", Acc) -> {lists:reverse(Acc), ""}; -urlsplit_netloc(Rest=[C | _], Acc) when C =:= $/; C =:= $?; C =:= $# -> +urlsplit_netloc(Rest = [C | _], Acc) when C =:= $/; C =:= $?; C =:= $# -> {lists:reverse(Acc), Rest}; urlsplit_netloc([C | Rest], Acc) -> urlsplit_netloc(Rest, [C | Acc]). make_path(Parts) -> - "/" ++ string:join(Parts,[?SEPARATOR]). + "/" ++ string:join(Parts, [?SEPARATOR]). init(_) -> ok = config:listen_for_changes(?MODULE, nil), @@ -348,17 +386,19 @@ init(_) -> %% load configuration {VHostGlobals, VHosts, Fun} = load_conf(), State = #vhosts_state{ - vhost_globals=VHostGlobals, - vhosts=VHosts, - vhosts_fun=Fun}, + vhost_globals = VHostGlobals, + vhosts = VHosts, + vhosts_fun = Fun + }, {ok, State}. handle_call(reload, _From, _State) -> {VHostGlobals, VHosts, Fun} = load_conf(), {reply, ok, #vhosts_state{ - vhost_globals=VHostGlobals, - vhosts=VHosts, - vhosts_fun=Fun}}; + vhost_globals = VHostGlobals, + vhosts = VHosts, + vhosts_fun = Fun + }}; handle_call(get_state, _From, State) -> {reply, State, State}; handle_call(_Msg, _From, State) -> @@ -379,7 +419,6 @@ terminate(_Reason, _State) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. - handle_config_change("vhosts", _, _, _, _) -> {ok, ?MODULE:reload()}; handle_config_change(_, _, _, _, _) -> @@ -392,8 +431,11 @@ handle_config_terminate(_Server, _Reason, _State) -> load_conf() -> %% get vhost globals - VHostGlobals = re:split("_utils, _uuids, _session, _users", "\\s*,\\s*", - [{return, list}]), + VHostGlobals = re:split( + "_utils, _uuids, _session, _users", + "\\s*,\\s*", + [{return, list}] + ), %% build vhosts matching rules VHosts = make_vhosts(), diff --git a/src/couch/src/couch_io_logger.erl b/src/couch/src/couch_io_logger.erl index 188e031cb..f859874b6 100644 --- a/src/couch/src/couch_io_logger.erl +++ b/src/couch/src/couch_io_logger.erl @@ -20,7 +20,6 @@ stop_error/1 ]). - start(undefined) -> ok; start(Dir) -> @@ -42,7 +41,6 @@ start(Dir) -> ok end. - stop_noerror() -> case get(logger_path) of undefined -> @@ -51,7 +49,6 @@ stop_noerror() -> close_logs() end. - stop_error(Err) -> case get(logger_path) of undefined -> @@ -61,21 +58,17 @@ stop_error(Err) -> close_logs() end. - log_output(Data) -> log(get(logger_out_fd), Data). - log_input(Data) -> log(get(logger_in_fd), Data). - unix_time() -> {Mega, Sec, USec} = os:timestamp(), UnixTs = (Mega * 1000000 + Sec) * 1000000 + USec, integer_to_list(UnixTs). - log_name() -> Ts = unix_time(), Pid0 = erlang:pid_to_list(self()), @@ -83,12 +76,10 @@ log_name() -> Pid2 = string:strip(Pid1, right, $>), lists:flatten(io_lib:format("~s_~s", [Ts, Pid2])). - close_logs() -> file:close(get(logger_out_fd)), file:close(get(logger_in_fd)). - save_error_logs(Path, Err) -> Otp = erlang:system_info(otp_release), Msg = io_lib:format("Error: ~p~nNode: ~p~nOTP: ~p~n", [Err, node(), Otp]), @@ -97,10 +88,9 @@ save_error_logs(Path, Err) -> OFd = get(logger_in_fd), file:position(IFd, 0), file:position(OFd, 0), - file:copy(IFd, Path ++ ".out.log"), + file:copy(IFd, Path ++ ".out.log"), file:copy(OFd, Path ++ ".in.log"). - log(undefined, _Data) -> ok; log(Fd, Data) -> diff --git a/src/couch/src/couch_key_tree.erl b/src/couch/src/couch_key_tree.erl index 94150418e..84c786148 100644 --- a/src/couch/src/couch_key_tree.erl +++ b/src/couch/src/couch_key_tree.erl @@ -48,41 +48,43 @@ -module(couch_key_tree). -export([ -count_leafs/1, -find_missing/2, -fold/3, -get/2, -get_all_leafs/1, -get_all_leafs_full/1, -get_full_key_paths/2, -get_key_leafs/2, -map/2, -map_leafs/2, -mapfold/3, -multi_merge/2, -merge/2, -remove_leafs/2, -stem/2 + count_leafs/1, + find_missing/2, + fold/3, + get/2, + get_all_leafs/1, + get_all_leafs_full/1, + get_full_key_paths/2, + get_key_leafs/2, + map/2, + map_leafs/2, + mapfold/3, + multi_merge/2, + merge/2, + remove_leafs/2, + stem/2 ]). -include_lib("couch/include/couch_db.hrl"). --type treenode() :: {Key::term(), Value::term(), [Node::treenode()]}. --type tree() :: {Depth::pos_integer(), [treenode()]}. +-type treenode() :: {Key :: term(), Value :: term(), [Node :: treenode()]}. +-type tree() :: {Depth :: pos_integer(), [treenode()]}. -type revtree() :: [tree()]. - %% @doc Merge multiple paths into the given tree. -spec multi_merge(revtree(), tree()) -> revtree(). multi_merge(RevTree, Trees) -> - lists:foldl(fun(Tree, RevTreeAcc) -> - {NewRevTree, _} = merge(RevTreeAcc, Tree), - NewRevTree - end, RevTree, lists:sort(Trees)). - + lists:foldl( + fun(Tree, RevTreeAcc) -> + {NewRevTree, _} = merge(RevTreeAcc, Tree), + NewRevTree + end, + RevTree, + lists:sort(Trees) + ). %% @doc Merge a path into a tree. -spec merge(revtree(), tree() | path()) -> - {revtree(), new_leaf | new_branch | internal_node}. + {revtree(), new_leaf | new_branch | internal_node}. merge(RevTree, Tree) -> {Merged, Result} = merge_tree(RevTree, Tree, []), {lists:sort(Merged), Result}. @@ -92,12 +94,12 @@ merge(RevTree, Tree) -> %% If it can't find a branch that the new tree merges into, add it as a %% new branch in the RevTree. -spec merge_tree(revtree(), tree() | path(), revtree()) -> - {revtree(), new_leaf | new_branch | internal_node}. + {revtree(), new_leaf | new_branch | internal_node}. merge_tree([], Tree, []) -> {[Tree], new_leaf}; merge_tree([], Tree, MergeAcc) -> - {[Tree|MergeAcc], new_branch}; -merge_tree([{Depth, Nodes} | Rest], {IDepth, INodes}=Tree, MergeAcc) -> + {[Tree | MergeAcc], new_branch}; +merge_tree([{Depth, Nodes} | Rest], {IDepth, INodes} = Tree, MergeAcc) -> % For the intrepid observer following along at home, notice what we're % doing here with (Depth - IDepth). This tells us which of the two % branches (Nodes or INodes) we need to seek into. If Depth > IDepth @@ -125,7 +127,7 @@ merge_tree([{Depth, Nodes} | Rest], {IDepth, INodes}=Tree, MergeAcc) -> %% ends up running out of nodes we know that these two branches can %% not be merged. -spec merge_at([node()], integer(), [node()]) -> - {revtree(), new_leaf | new_branch | internal_node} | fail. + {revtree(), new_leaf | new_branch | internal_node} | fail. merge_at(_Nodes, _Pos, []) -> fail; merge_at([], _Pos, _INodes) -> @@ -172,7 +174,7 @@ merge_at([Tree | Sibs], 0, INodes) -> end. -spec merge_extend(revtree(), revtree()) -> - {revtree(), new_leaf | new_branch | internal_node}. + {revtree(), new_leaf | new_branch | internal_node}. merge_extend([], B) when B =/= [] -> % Most likely the insert branch simply extends this one, so the new % branch is exactly B. Its also possible that B is a branch because @@ -189,7 +191,7 @@ merge_extend([{K, V1, SubA} | NextA], [{K, V2, SubB}]) -> % level in the two branches. {Merged, Result} = merge_extend(SubA, SubB), {[{K, value_pref(V1, V2), Merged} | NextA], Result}; -merge_extend([{K1, _, _}=NodeA | Rest], [{K2, _, _}=NodeB]) when K1 > K2 -> +merge_extend([{K1, _, _} = NodeA | Rest], [{K2, _, _} = NodeB]) when K1 > K2 -> % Keys are ordered so we know this is where the insert branch needs % to be inserted into the tree. We also know that this creates a new % branch so we have a new leaf to report. @@ -200,10 +202,11 @@ merge_extend([Tree | RestA], NextB) -> % key in NextB might be larger than the largest key in RestA which % means we've created a new branch. {Merged, Result0} = merge_extend(RestA, NextB), - Result = case length(Merged) == length(RestA) of - true -> Result0; - false -> new_branch - end, + Result = + case length(Merged) == length(RestA) of + true -> Result0; + false -> new_branch + end, {[Tree | Merged], Result}. find_missing(_Tree, []) -> @@ -228,17 +231,17 @@ find_missing_simple(Pos, [{Key, _, SubTree} | RestTree], SeachKeys) -> SrcKeys3 = find_missing_simple(Pos + 1, SubTree, SrcKeys2), ImpossibleKeys ++ find_missing_simple(Pos, RestTree, SrcKeys3). - filter_leafs([], _Keys, FilteredAcc, RemovedKeysAcc) -> {FilteredAcc, RemovedKeysAcc}; -filter_leafs([{Pos, [{LeafKey, _}|_]} = Path |Rest], Keys, FilteredAcc, RemovedKeysAcc) -> +filter_leafs([{Pos, [{LeafKey, _} | _]} = Path | Rest], Keys, FilteredAcc, RemovedKeysAcc) -> FilteredKeys = lists:delete({Pos, LeafKey}, Keys), - if FilteredKeys == Keys -> - % this leaf is not a key we are looking to remove - filter_leafs(Rest, Keys, [Path | FilteredAcc], RemovedKeysAcc); - true -> - % this did match a key, remove both the node and the input key - filter_leafs(Rest, FilteredKeys, FilteredAcc, [{Pos, LeafKey} | RemovedKeysAcc]) + if + FilteredKeys == Keys -> + % this leaf is not a key we are looking to remove + filter_leafs(Rest, Keys, [Path | FilteredAcc], RemovedKeysAcc); + true -> + % this did match a key, remove both the node and the input key + filter_leafs(Rest, FilteredKeys, FilteredAcc, [{Pos, LeafKey} | RemovedKeysAcc]) end. % Removes any branches from the tree whose leaf node(s) are in the Keys @@ -255,15 +258,18 @@ remove_leafs(Trees, Keys) -> % convert paths back to trees NewTree = lists:foldl( - fun({StartPos, Path},TreeAcc) -> + fun({StartPos, Path}, TreeAcc) -> [SingleTree] = lists:foldl( - fun({K,V},NewTreeAcc) -> [{K,V,NewTreeAcc}] end, [], Path), + fun({K, V}, NewTreeAcc) -> [{K, V, NewTreeAcc}] end, [], Path + ), {NewTrees, _} = merge(TreeAcc, {StartPos, SingleTree}), NewTrees - end, [], SortedPaths), + end, + [], + SortedPaths + ), {NewTree, RemovedKeys}. - % get the leafs in the tree matching the keys. The matching key nodes can be % leafs or an inner nodes. If an inner node, then the leafs for that node % are returned. @@ -274,7 +280,7 @@ get_key_leafs(_, [], Acc) -> {Acc, []}; get_key_leafs([], Keys, Acc) -> {Acc, Keys}; -get_key_leafs([{Pos, Tree}|Rest], Keys, Acc) -> +get_key_leafs([{Pos, Tree} | Rest], Keys, Acc) -> {Gotten, RemainingKeys} = get_key_leafs_simple(Pos, [Tree], Keys, []), get_key_leafs(Rest, RemainingKeys, Gotten ++ Acc). @@ -282,7 +288,7 @@ get_key_leafs_simple(_Pos, _Tree, [], _PathAcc) -> {[], []}; get_key_leafs_simple(_Pos, [], Keys, _PathAcc) -> {[], Keys}; -get_key_leafs_simple(Pos, [{Key, _, SubTree}=Tree | RestTree], Keys, PathAcc) -> +get_key_leafs_simple(Pos, [{Key, _, SubTree} = Tree | RestTree], Keys, PathAcc) -> case lists:delete({Pos, Key}, Keys) of Keys -> % Same list, key not found @@ -300,7 +306,6 @@ get_key_leafs_simple(Pos, [{Key, _, SubTree}=Tree | RestTree], Keys, PathAcc) -> {ChildLeafs ++ SiblingLeafs, Keys4} end. - get_key_leafs_simple2(_Pos, [], Keys, _PathAcc) -> % No more tree to deal with so no more keys to return. {[], Keys}; @@ -320,10 +325,12 @@ get_key_leafs_simple2(Pos, [{Key, _Value, SubTree} | RestTree], Keys, PathAcc) - {SiblingLeafs, Keys4} = get_key_leafs_simple2(Pos, RestTree, Keys3, PathAcc), {ChildLeafs ++ SiblingLeafs, Keys4}. - get(Tree, KeysToGet) -> {KeyPaths, KeysNotFound} = get_full_key_paths(Tree, KeysToGet), - FixedResults = [ {Value, {Pos, [Key0 || {Key0, _} <- Path]}} || {Pos, [{_Key, Value}|_]=Path} <- KeyPaths], + FixedResults = [ + {Value, {Pos, [Key0 || {Key0, _} <- Path]}} + || {Pos, [{_Key, Value} | _] = Path} <- KeyPaths + ], {FixedResults, KeysNotFound}. get_full_key_paths(Tree, Keys) -> @@ -333,11 +340,10 @@ get_full_key_paths(_, [], Acc) -> {Acc, []}; get_full_key_paths([], Keys, Acc) -> {Acc, Keys}; -get_full_key_paths([{Pos, Tree}|Rest], Keys, Acc) -> +get_full_key_paths([{Pos, Tree} | Rest], Keys, Acc) -> {Gotten, RemainingKeys} = get_full_key_paths(Pos, [Tree], Keys, []), get_full_key_paths(Rest, RemainingKeys, Gotten ++ Acc). - get_full_key_paths(_Pos, _Tree, [], _KeyPathAcc) -> {[], []}; get_full_key_paths(_Pos, [], KeysToGet, _KeyPathAcc) -> @@ -345,13 +351,17 @@ get_full_key_paths(_Pos, [], KeysToGet, _KeyPathAcc) -> get_full_key_paths(Pos, [{KeyId, Value, SubTree} | RestTree], KeysToGet, KeyPathAcc) -> KeysToGet2 = KeysToGet -- [{Pos, KeyId}], CurrentNodeResult = - case length(KeysToGet2) =:= length(KeysToGet) of - true -> % not in the key list. - []; - false -> % this node is the key list. return it - [{Pos, [{KeyId, Value} | KeyPathAcc]}] - end, - {KeysGotten, KeysRemaining} = get_full_key_paths(Pos + 1, SubTree, KeysToGet2, [{KeyId, Value} | KeyPathAcc]), + case length(KeysToGet2) =:= length(KeysToGet) of + % not in the key list. + true -> + []; + % this node is the key list. return it + false -> + [{Pos, [{KeyId, Value} | KeyPathAcc]}] + end, + {KeysGotten, KeysRemaining} = get_full_key_paths(Pos + 1, SubTree, KeysToGet2, [ + {KeyId, Value} | KeyPathAcc + ]), {KeysGotten2, KeysRemaining2} = get_full_key_paths(Pos, RestTree, KeysRemaining, KeyPathAcc), {CurrentNodeResult ++ KeysGotten ++ KeysGotten2, KeysRemaining2}. @@ -368,14 +378,15 @@ get_all_leafs_full_simple(_Pos, [], _KeyPathAcc) -> get_all_leafs_full_simple(Pos, [{KeyId, Value, []} | RestTree], KeyPathAcc) -> [{Pos, [{KeyId, Value} | KeyPathAcc]} | get_all_leafs_full_simple(Pos, RestTree, KeyPathAcc)]; get_all_leafs_full_simple(Pos, [{KeyId, Value, SubTree} | RestTree], KeyPathAcc) -> - get_all_leafs_full_simple(Pos + 1, SubTree, [{KeyId, Value} | KeyPathAcc]) ++ get_all_leafs_full_simple(Pos, RestTree, KeyPathAcc). + get_all_leafs_full_simple(Pos + 1, SubTree, [{KeyId, Value} | KeyPathAcc]) ++ + get_all_leafs_full_simple(Pos, RestTree, KeyPathAcc). get_all_leafs(Trees) -> get_all_leafs(Trees, []). get_all_leafs([], Acc) -> Acc; -get_all_leafs([{Pos, Tree}|Rest], Acc) -> +get_all_leafs([{Pos, Tree} | Rest], Acc) -> get_all_leafs(Rest, get_all_leafs_simple(Pos, [Tree], []) ++ Acc). get_all_leafs_simple(_Pos, [], _KeyPathAcc) -> @@ -383,12 +394,12 @@ get_all_leafs_simple(_Pos, [], _KeyPathAcc) -> get_all_leafs_simple(Pos, [{KeyId, Value, []} | RestTree], KeyPathAcc) -> [{Value, {Pos, [KeyId | KeyPathAcc]}} | get_all_leafs_simple(Pos, RestTree, KeyPathAcc)]; get_all_leafs_simple(Pos, [{KeyId, _Value, SubTree} | RestTree], KeyPathAcc) -> - get_all_leafs_simple(Pos + 1, SubTree, [KeyId | KeyPathAcc]) ++ get_all_leafs_simple(Pos, RestTree, KeyPathAcc). - + get_all_leafs_simple(Pos + 1, SubTree, [KeyId | KeyPathAcc]) ++ + get_all_leafs_simple(Pos, RestTree, KeyPathAcc). count_leafs([]) -> 0; -count_leafs([{_Pos,Tree}|Rest]) -> +count_leafs([{_Pos, Tree} | Rest]) -> count_leafs_simple([Tree]) + count_leafs(Rest). count_leafs_simple([]) -> @@ -398,42 +409,49 @@ count_leafs_simple([{_Key, _Value, []} | RestTree]) -> count_leafs_simple([{_Key, _Value, SubTree} | RestTree]) -> count_leafs_simple(SubTree) + count_leafs_simple(RestTree). - fold(_Fun, Acc, []) -> Acc; -fold(Fun, Acc0, [{Pos, Tree}|Rest]) -> +fold(Fun, Acc0, [{Pos, Tree} | Rest]) -> Acc1 = fold_simple(Fun, Acc0, Pos, [Tree]), fold(Fun, Acc1, Rest). fold_simple(_Fun, Acc, _Pos, []) -> Acc; fold_simple(Fun, Acc0, Pos, [{Key, Value, SubTree} | RestTree]) -> - Type = if SubTree == [] -> leaf; true -> branch end, + Type = + if + SubTree == [] -> leaf; + true -> branch + end, Acc1 = Fun({Pos, Key}, Value, Type, Acc0), - Acc2 = fold_simple(Fun, Acc1, Pos+1, SubTree), + Acc2 = fold_simple(Fun, Acc1, Pos + 1, SubTree), fold_simple(Fun, Acc2, Pos, RestTree). - map(_Fun, []) -> []; -map(Fun, [{Pos, Tree}|Rest]) -> +map(Fun, [{Pos, Tree} | Rest]) -> case erlang:fun_info(Fun, arity) of - {arity, 2} -> - [NewTree] = map_simple(fun(A,B,_C) -> Fun(A,B) end, Pos, [Tree]), - [{Pos, NewTree} | map(Fun, Rest)]; - {arity, 3} -> - [NewTree] = map_simple(Fun, Pos, [Tree]), - [{Pos, NewTree} | map(Fun, Rest)] + {arity, 2} -> + [NewTree] = map_simple(fun(A, B, _C) -> Fun(A, B) end, Pos, [Tree]), + [{Pos, NewTree} | map(Fun, Rest)]; + {arity, 3} -> + [NewTree] = map_simple(Fun, Pos, [Tree]), + [{Pos, NewTree} | map(Fun, Rest)] end. map_simple(_Fun, _Pos, []) -> []; map_simple(Fun, Pos, [{Key, Value, SubTree} | RestTree]) -> - Value2 = Fun({Pos, Key}, Value, - if SubTree == [] -> leaf; true -> branch end), + Value2 = Fun( + {Pos, Key}, + Value, + if + SubTree == [] -> leaf; + true -> branch + end + ), [{Key, Value2, map_simple(Fun, Pos + 1, SubTree)} | map_simple(Fun, Pos, RestTree)]. - mapfold(_Fun, Acc, []) -> {[], Acc}; mapfold(Fun, Acc, [{Pos, Tree} | Rest]) -> @@ -444,16 +462,22 @@ mapfold(Fun, Acc, [{Pos, Tree} | Rest]) -> mapfold_simple(_Fun, Acc, _Pos, []) -> {[], Acc}; mapfold_simple(Fun, Acc, Pos, [{Key, Value, SubTree} | RestTree]) -> - {Value2, Acc2} = Fun({Pos, Key}, Value, - if SubTree == [] -> leaf; true -> branch end, Acc), + {Value2, Acc2} = Fun( + {Pos, Key}, + Value, + if + SubTree == [] -> leaf; + true -> branch + end, + Acc + ), {SubTree2, Acc3} = mapfold_simple(Fun, Acc2, Pos + 1, SubTree), {RestTree2, Acc4} = mapfold_simple(Fun, Acc3, Pos, RestTree), {[{Key, Value2, SubTree2} | RestTree2], Acc4}. - map_leafs(_Fun, []) -> []; -map_leafs(Fun, [{Pos, Tree}|Rest]) -> +map_leafs(Fun, [{Pos, Tree} | Rest]) -> [NewTree] = map_leafs_simple(Fun, Pos, [Tree]), [{Pos, NewTree} | map_leafs(Fun, Rest)]. @@ -465,19 +489,22 @@ map_leafs_simple(Fun, Pos, [{Key, Value, []} | RestTree]) -> map_leafs_simple(Fun, Pos, [{Key, Value, SubTree} | RestTree]) -> [{Key, Value, map_leafs_simple(Fun, Pos + 1, SubTree)} | map_leafs_simple(Fun, Pos, RestTree)]. - stem(Trees, Limit) -> try - {_, Branches} = lists:foldl(fun(Tree, {Seen, TreeAcc}) -> - {NewSeen, NewBranches} = stem_tree(Tree, Limit, Seen), - {NewSeen, NewBranches ++ TreeAcc} - end, {sets:new(), []}, Trees), + {_, Branches} = lists:foldl( + fun(Tree, {Seen, TreeAcc}) -> + {NewSeen, NewBranches} = stem_tree(Tree, Limit, Seen), + {NewSeen, NewBranches ++ TreeAcc} + end, + {sets:new(), []}, + Trees + ), lists:sort(Branches) - catch throw:dupe_keys -> - repair_tree(Trees, Limit) + catch + throw:dupe_keys -> + repair_tree(Trees, Limit) end. - stem_tree({Depth, Child}, Limit, Seen) -> case stem_tree(Depth, Child, Limit, Seen) of {NewSeen, _, NewChild, NewBranches} -> @@ -486,41 +513,45 @@ stem_tree({Depth, Child}, Limit, Seen) -> {NewSeen, NewBranches} end. - stem_tree(_Depth, {Key, _Val, []} = Leaf, Limit, Seen) -> {check_key(Key, Seen), Limit - 1, Leaf, []}; - stem_tree(Depth, {Key, Val, Children}, Limit, Seen0) -> Seen1 = check_key(Key, Seen0), - FinalAcc = lists:foldl(fun(Child, Acc) -> - {SeenAcc, LimitPosAcc, ChildAcc, BranchAcc} = Acc, - case stem_tree(Depth + 1, Child, Limit, SeenAcc) of - {NewSeenAcc, LimitPos, NewChild, NewBranches} -> - NewLimitPosAcc = erlang:max(LimitPos, LimitPosAcc), - NewChildAcc = [NewChild | ChildAcc], - NewBranchAcc = NewBranches ++ BranchAcc, - {NewSeenAcc, NewLimitPosAcc, NewChildAcc, NewBranchAcc}; - {NewSeenAcc, LimitPos, NewBranches} -> - NewLimitPosAcc = erlang:max(LimitPos, LimitPosAcc), - NewBranchAcc = NewBranches ++ BranchAcc, - {NewSeenAcc, NewLimitPosAcc, ChildAcc, NewBranchAcc} - end - end, {Seen1, -1, [], []}, Children), + FinalAcc = lists:foldl( + fun(Child, Acc) -> + {SeenAcc, LimitPosAcc, ChildAcc, BranchAcc} = Acc, + case stem_tree(Depth + 1, Child, Limit, SeenAcc) of + {NewSeenAcc, LimitPos, NewChild, NewBranches} -> + NewLimitPosAcc = erlang:max(LimitPos, LimitPosAcc), + NewChildAcc = [NewChild | ChildAcc], + NewBranchAcc = NewBranches ++ BranchAcc, + {NewSeenAcc, NewLimitPosAcc, NewChildAcc, NewBranchAcc}; + {NewSeenAcc, LimitPos, NewBranches} -> + NewLimitPosAcc = erlang:max(LimitPos, LimitPosAcc), + NewBranchAcc = NewBranches ++ BranchAcc, + {NewSeenAcc, NewLimitPosAcc, ChildAcc, NewBranchAcc} + end + end, + {Seen1, -1, [], []}, + Children + ), {FinalSeen, FinalLimitPos, FinalChildren, FinalBranches} = FinalAcc, case FinalLimitPos of N when N > 0, length(FinalChildren) > 0 -> FinalNode = {Key, Val, lists:reverse(FinalChildren)}, {FinalSeen, FinalLimitPos - 1, FinalNode, FinalBranches}; 0 when length(FinalChildren) > 0 -> - NewBranches = lists:map(fun(Child) -> - {Depth + 1, Child} - end, lists:reverse(FinalChildren)), + NewBranches = lists:map( + fun(Child) -> + {Depth + 1, Child} + end, + lists:reverse(FinalChildren) + ), {FinalSeen, -1, NewBranches ++ FinalBranches}; N when N < 0, length(FinalChildren) == 0 -> {FinalSeen, FinalLimitPos - 1, FinalBranches} end. - check_key(Key, Seen) -> case sets:is_element(Key, Seen) of true -> @@ -529,29 +560,40 @@ check_key(Key, Seen) -> sets:add_element(Key, Seen) end. - repair_tree(Trees, Limit) -> % flatten each branch in a tree into a tree path, sort by starting rev # - Paths = lists:sort(lists:map(fun({Pos, Path}) -> - StemmedPath = lists:sublist(Path, Limit), - {Pos + 1 - length(StemmedPath), StemmedPath} - end, get_all_leafs_full(Trees))), + Paths = lists:sort( + lists:map( + fun({Pos, Path}) -> + StemmedPath = lists:sublist(Path, Limit), + {Pos + 1 - length(StemmedPath), StemmedPath} + end, + get_all_leafs_full(Trees) + ) + ), % convert paths back to trees lists:foldl( - fun({StartPos, Path},TreeAcc) -> + fun({StartPos, Path}, TreeAcc) -> [SingleTree] = lists:foldl( - fun({K,V},NewTreeAcc) -> [{K,V,NewTreeAcc}] end, [], Path), + fun({K, V}, NewTreeAcc) -> [{K, V, NewTreeAcc}] end, [], Path + ), {NewTrees, _} = merge(TreeAcc, {StartPos, SingleTree}), NewTrees - end, [], Paths). - - -value_pref(Tuple, _) when is_tuple(Tuple), - (tuple_size(Tuple) == 3 orelse tuple_size(Tuple) == 4) -> + end, + [], + Paths + ). + +value_pref(Tuple, _) when + is_tuple(Tuple), + (tuple_size(Tuple) == 3 orelse tuple_size(Tuple) == 4) +-> Tuple; -value_pref(_, Tuple) when is_tuple(Tuple), - (tuple_size(Tuple) == 3 orelse tuple_size(Tuple) == 4) -> +value_pref(_, Tuple) when + is_tuple(Tuple), + (tuple_size(Tuple) == 3 orelse tuple_size(Tuple) == 4) +-> Tuple; value_pref(?REV_MISSING, Other) -> Other; diff --git a/src/couch/src/couch_lru.erl b/src/couch/src/couch_lru.erl index 618a0144f..1fad20280 100644 --- a/src/couch/src/couch_lru.erl +++ b/src/couch/src/couch_lru.erl @@ -24,14 +24,14 @@ insert(DbName, {Tree0, Dict0}) -> update(DbName, {Tree0, Dict0}) -> case dict:find(DbName, Dict0) of - {ok, Old} -> - New = couch_util:unique_monotonic_integer(), - Tree = gb_trees:insert(New, DbName, gb_trees:delete(Old, Tree0)), - Dict = dict:store(DbName, New, Dict0), - {Tree, Dict}; - error -> - % We closed this database before processing the update. Ignore - {Tree0, Dict0} + {ok, Old} -> + New = couch_util:unique_monotonic_integer(), + Tree = gb_trees:insert(New, DbName, gb_trees:delete(Old, Tree0)), + Dict = dict:store(DbName, New, Dict0), + {Tree, Dict}; + error -> + % We closed this database before processing the update. Ignore + {Tree0, Dict0} end. %% Attempt to close the oldest idle database. @@ -47,21 +47,22 @@ close_int({Lru, DbName, Iter}, {Tree, Dict} = Cache) -> CouchDbsPidToName = couch_server:couch_dbs_pid_to_name(DbName), case ets:update_element(CouchDbs, DbName, {#entry.lock, locked}) of - true -> - [#entry{db = Db, pid = Pid}] = ets:lookup(CouchDbs, DbName), - case couch_db:is_idle(Db) of true -> - true = ets:delete(CouchDbs, DbName), - true = ets:delete(CouchDbsPidToName, Pid), - exit(Pid, kill), - {true, {gb_trees:delete(Lru, Tree), dict:erase(DbName, Dict)}}; + true -> + [#entry{db = Db, pid = Pid}] = ets:lookup(CouchDbs, DbName), + case couch_db:is_idle(Db) of + true -> + true = ets:delete(CouchDbs, DbName), + true = ets:delete(CouchDbsPidToName, Pid), + exit(Pid, kill), + {true, {gb_trees:delete(Lru, Tree), dict:erase(DbName, Dict)}}; + false -> + ElemSpec = {#entry.lock, unlocked}, + true = ets:update_element(CouchDbs, DbName, ElemSpec), + couch_stats:increment_counter([couchdb, couch_server, lru_skip]), + close_int(gb_trees:next(Iter), update(DbName, Cache)) + end; false -> - ElemSpec = {#entry.lock, unlocked}, - true = ets:update_element(CouchDbs, DbName, ElemSpec), - couch_stats:increment_counter([couchdb, couch_server, lru_skip]), - close_int(gb_trees:next(Iter), update(DbName, Cache)) - end; - false -> - NewTree = gb_trees:delete(Lru, Tree), - NewIter = gb_trees:iterator(NewTree), - close_int(gb_trees:next(NewIter), {NewTree, dict:erase(DbName, Dict)}) -end. + NewTree = gb_trees:delete(Lru, Tree), + NewIter = gb_trees:iterator(NewTree), + close_int(gb_trees:next(NewIter), {NewTree, dict:erase(DbName, Dict)}) + end. diff --git a/src/couch/src/couch_multidb_changes.erl b/src/couch/src/couch_multidb_changes.erl index e2bbda3e3..adb1b740f 100644 --- a/src/couch/src/couch_multidb_changes.erl +++ b/src/couch/src/couch_multidb_changes.erl @@ -15,27 +15,27 @@ -behaviour(gen_server). -export([ - start_link/4 + start_link/4 ]). -export([ - init/1, - terminate/2, - handle_call/3, - handle_info/2, - handle_cast/2, - code_change/3 + init/1, + terminate/2, + handle_call/3, + handle_info/2, + handle_cast/2, + code_change/3 ]). -export([ - changes_reader/3, - changes_reader_cb/3 + changes_reader/3, + changes_reader_cb/3 ]). -include_lib("couch/include/couch_db.hrl"). -include_lib("mem3/include/mem3.hrl"). --define(CTX, {user_ctx, #user_ctx{roles=[<<"_admin">>, <<"_replicator">>]}}). +-define(CTX, {user_ctx, #user_ctx{roles = [<<"_admin">>, <<"_replicator">>]}}). -define(AVG_DELAY_MSEC, 10). -define(MAX_DELAY_MSEC, 120000). @@ -68,20 +68,18 @@ -callback db_change(DbName :: binary(), Change :: term(), Context :: term()) -> Context :: term(). - % External API - % Opts list can contain: % - `skip_ddocs` : Skip design docs -spec start_link(binary(), module(), term(), list()) -> {ok, pid()} | ignore | {error, term()}. start_link(DbSuffix, Module, Context, Opts) when - is_binary(DbSuffix), is_atom(Module), is_list(Opts) -> + is_binary(DbSuffix), is_atom(Module), is_list(Opts) +-> gen_server:start_link(?MODULE, [DbSuffix, Module, Context, Opts], []). - % gen_server callbacks init([DbSuffix, Module, Context, Opts]) -> @@ -98,21 +96,21 @@ init([DbSuffix, Module, Context, Opts]) -> skip_ddocs = proplists:is_defined(skip_ddocs, Opts) }}. - terminate(_Reason, _State) -> ok. - -handle_call({change, DbName, Change}, _From, - #state{skip_ddocs=SkipDDocs, mod=Mod, ctx=Ctx} = State) -> +handle_call( + {change, DbName, Change}, + _From, + #state{skip_ddocs = SkipDDocs, mod = Mod, ctx = Ctx} = State +) -> case {SkipDDocs, is_design_doc(Change)} of {true, true} -> {reply, ok, State}; {_, _} -> - {reply, ok, State#state{ctx=Mod:db_change(DbName, Change, Ctx)}} + {reply, ok, State#state{ctx = Mod:db_change(DbName, Change, Ctx)}} end; - -handle_call({checkpoint, DbName, EndSeq}, _From, #state{tid=Ets} = State) -> +handle_call({checkpoint, DbName, EndSeq}, _From, #state{tid = Ets} = State) -> case ets:lookup(Ets, DbName) of [] -> true = ets:insert(Ets, {DbName, EndSeq, false}); @@ -121,11 +119,9 @@ handle_call({checkpoint, DbName, EndSeq}, _From, #state{tid=Ets} = State) -> end, {reply, ok, State}. - handle_cast({resume_scan, DbName}, State) -> {noreply, resume_scan(DbName, State)}. - handle_info({'$couch_event', DbName, Event}, #state{suffix = Suf} = State) -> case Suf =:= couch_db:dbname_suffix(DbName) of true -> @@ -133,23 +129,22 @@ handle_info({'$couch_event', DbName, Event}, #state{suffix = Suf} = State) -> _ -> {noreply, State} end; - handle_info({'DOWN', Ref, _, _, Info}, #state{event_server = Ref} = State) -> {stop, {couch_event_server_died, Info}, State}; - handle_info({'EXIT', From, normal}, #state{scanner = From} = State) -> - {noreply, State#state{scanner=nil}}; - + {noreply, State#state{scanner = nil}}; handle_info({'EXIT', From, Reason}, #state{scanner = From} = State) -> {stop, {scanner_died, Reason}, State}; - handle_info({'EXIT', From, Reason}, #state{pids = Pids} = State) -> couch_log:debug("~p change feed exited ~p", [State#state.suffix, From]), case lists:keytake(From, 2, Pids) of {value, {DbName, From}, NewPids} -> - if Reason == normal -> ok; true -> - Fmt = "~s : Known change feed ~w died :: ~w", - couch_log:error(Fmt, [?MODULE, From, Reason]) + if + Reason == normal -> + ok; + true -> + Fmt = "~s : Known change feed ~w died :: ~w", + couch_log:error(Fmt, [?MODULE, From, Reason]) end, NewState = State#state{pids = NewPids}, case ets:lookup(State#state.tid, DbName) of @@ -165,15 +160,12 @@ handle_info({'EXIT', From, Reason}, #state{pids = Pids} = State) -> couch_log:error(Fmt, [?MODULE, State#state.suffix, From, Reason]), {stop, {unexpected_exit, From, Reason}, State} end; - handle_info(_Msg, State) -> {noreply, State}. - code_change(_OldVsn, State, _Extra) -> {ok, State}. - % Private functions -spec register_with_event_server(pid()) -> reference(). @@ -182,7 +174,6 @@ register_with_event_server(Server) -> couch_event:register_all(Server), Ref. - -spec db_callback(created | deleted | updated, binary(), #state{}) -> #state{}. db_callback(created, DbName, #state{mod = Mod, ctx = Ctx} = State) -> NewState = State#state{ctx = Mod:db_created(DbName, Ctx)}, @@ -194,9 +185,8 @@ db_callback(updated, DbName, State) -> db_callback(_Other, _DbName, State) -> State. - -spec resume_scan(binary(), #state{}) -> #state{}. -resume_scan(DbName, #state{pids=Pids, tid=Ets} = State) -> +resume_scan(DbName, #state{pids = Pids, tid = Ets} = State) -> case {lists:keyfind(DbName, 1, Pids), ets:lookup(Ets, DbName)} of {{DbName, _}, []} -> % Found existing change feed, but not entry in ETS @@ -217,20 +207,18 @@ resume_scan(DbName, #state{pids=Pids, tid=Ets} = State) -> Mod = State#state.mod, Ctx = Mod:db_found(DbName, State#state.ctx), Pid = start_changes_reader(DbName, 0), - State#state{ctx=Ctx, pids=[{DbName, Pid} | Pids]}; + State#state{ctx = Ctx, pids = [{DbName, Pid} | Pids]}; {false, [{DbName, EndSeq, _}]} -> % No existing change feed running. Found existing checkpoint. % Start a new change reader from last checkpoint. true = ets:insert(Ets, {DbName, EndSeq, false}), Pid = start_changes_reader(DbName, EndSeq), - State#state{pids=[{DbName, Pid} | Pids]} - end. - + State#state{pids = [{DbName, Pid} | Pids]} + end. start_changes_reader(DbName, Since) -> spawn_link(?MODULE, changes_reader, [self(), DbName, Since]). - changes_reader(Server, DbName, Since) -> {ok, Db} = couch_db:open_int(DbName, [?CTX, sys_db]), ChangesArgs = #changes_args{ @@ -242,7 +230,6 @@ changes_reader(Server, DbName, Since) -> ChFun = couch_changes:handle_db_changes(ChangesArgs, {json_req, null}, Db), ChFun({fun ?MODULE:changes_reader_cb/3, {Server, DbName}}). - changes_reader_cb({change, Change, _}, _, {Server, DbName}) -> ok = gen_server:call(Server, {change, DbName, Change}, infinity), {Server, DbName}; @@ -252,34 +239,35 @@ changes_reader_cb({stop, EndSeq}, _, {Server, DbName}) -> changes_reader_cb(_, _, Acc) -> Acc. - scan_all_dbs(Server, DbSuffix) when is_pid(Server) -> ok = scan_local_db(Server, DbSuffix), {ok, Db} = mem3_util:ensure_exists( - config:get("mem3", "shards_db", "_dbs")), + config:get("mem3", "shards_db", "_dbs") + ), ChangesFun = couch_changes:handle_db_changes(#changes_args{}, nil, Db), ChangesFun({fun scan_changes_cb/3, {Server, DbSuffix, 1}}), couch_db:close(Db). - scan_changes_cb({change, {Change}, _}, _, {_Server, DbSuffix, _Count} = Acc) -> DbName = couch_util:get_value(<<"id">>, Change), - case DbName of <<"_design/", _/binary>> -> Acc; _Else -> - NameMatch = DbSuffix =:= couch_db:dbname_suffix(DbName), - case {NameMatch, couch_replicator_utils:is_deleted(Change)} of - {false, _} -> - Acc; - {true, true} -> - Acc; - {true, false} -> - Shards = local_shards(DbName), - lists:foldl(fun notify_fold/2, Acc, Shards) - end + case DbName of + <<"_design/", _/binary>> -> + Acc; + _Else -> + NameMatch = DbSuffix =:= couch_db:dbname_suffix(DbName), + case {NameMatch, couch_replicator_utils:is_deleted(Change)} of + {false, _} -> + Acc; + {true, true} -> + Acc; + {true, false} -> + Shards = local_shards(DbName), + lists:foldl(fun notify_fold/2, Acc, Shards) + end end; scan_changes_cb(_, _, Acc) -> Acc. - local_shards(DbName) -> try [ShardName || #shard{name = ShardName} <- mem3:local_shards(DbName)] @@ -288,7 +276,6 @@ local_shards(DbName) -> [] end. - notify_fold(DbName, {Server, DbSuffix, Count}) -> Jitter = jitter(Count), spawn_link(fun() -> @@ -297,7 +284,6 @@ notify_fold(DbName, {Server, DbSuffix, Count}) -> end), {Server, DbSuffix, Count + 1}. - % Jitter is proportional to the number of shards found so far. This is done to % avoid a stampede and notifying the callback function with potentially a large % number of shards back to back during startup. @@ -305,7 +291,6 @@ jitter(N) -> Range = min(2 * N * ?AVG_DELAY_MSEC, ?MAX_DELAY_MSEC), couch_rand:uniform(Range). - scan_local_db(Server, DbSuffix) when is_pid(Server) -> case couch_db:open_int(DbSuffix, [?CTX, sys_db, nologifmissing]) of {ok, Db} -> @@ -315,7 +300,6 @@ scan_local_db(Server, DbSuffix) when is_pid(Server) -> ok end. - is_design_doc({Change}) -> case lists:keyfind(<<"id">>, 1, Change) of false -> @@ -324,13 +308,11 @@ is_design_doc({Change}) -> is_design_doc_id(Id) end. - is_design_doc_id(<<?DESIGN_DOC_PREFIX, _/binary>>) -> true; is_design_doc_id(_) -> false. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -380,7 +362,6 @@ couch_multidb_changes_test_() -> } }. - setup_all() -> mock_logs(), mock_callback_mod(), @@ -389,25 +370,31 @@ setup_all() -> meck:expect(mem3_util, ensure_exists, 1, {ok, dbs}), ChangesFun = meck:val(fun(_) -> ok end), meck:expect(couch_changes, handle_db_changes, 3, ChangesFun), - meck:expect(couch_db, open_int, - fun(?DBNAME, [?CTX, sys_db]) -> {ok, db}; + meck:expect( + couch_db, + open_int, + fun + (?DBNAME, [?CTX, sys_db]) -> {ok, db}; (_, _) -> {not_found, no_db_file} - end), + end + ), meck:expect(couch_db, close, 1, ok), mock_changes_reader(), % create process to stand in for couch_event_server % mocking erlang:monitor doesn't work, so give it real process to monitor - EvtPid = spawn_link(fun() -> receive looper -> ok end end), + EvtPid = spawn_link(fun() -> + receive + looper -> ok + end + end), true = register(couch_event_server, EvtPid), EvtPid. - teardown_all(EvtPid) -> unlink(EvtPid), exit(EvtPid, kill), meck:unload(). - setup() -> meck:reset([ ?MOD, @@ -417,11 +404,9 @@ setup() -> couch_log ]). - teardown(_) -> ok. - t_handle_call_change() -> ?_test(begin State = mock_state(), @@ -431,7 +416,6 @@ t_handle_call_change() -> ?assert(meck:called(?MOD, db_change, [?DBNAME, Change, zig])) end). - t_handle_call_change_filter_design_docs() -> ?_test(begin State0 = mock_state(), @@ -442,7 +426,6 @@ t_handle_call_change_filter_design_docs() -> ?assertNot(meck:called(?MOD, db_change, [?DBNAME, Change, zig])) end). - t_handle_call_checkpoint_new() -> ?_test(begin Tid = mock_ets(), @@ -452,7 +435,6 @@ t_handle_call_checkpoint_new() -> ets:delete(Tid) end). - t_handle_call_checkpoint_existing() -> ?_test(begin Tid = mock_ets(), @@ -463,7 +445,6 @@ t_handle_call_checkpoint_existing() -> ets:delete(Tid) end). - t_handle_info_created() -> ?_test(begin Tid = mock_ets(), @@ -473,18 +454,16 @@ t_handle_info_created() -> ?assert(meck:called(?MOD, db_created, [?DBNAME, zig])) end). - t_handle_info_deleted() -> - ?_test(begin + ?_test(begin State = mock_state(), handle_info_check({'$couch_event', ?DBNAME, deleted}, State), ?assert(meck:validate(?MOD)), ?assert(meck:called(?MOD, db_deleted, [?DBNAME, zig])) end). - t_handle_info_updated() -> - ?_test(begin + ?_test(begin Tid = mock_ets(), State = mock_state(Tid), handle_info_check({'$couch_event', ?DBNAME, updated}, State), @@ -492,9 +471,8 @@ t_handle_info_updated() -> ?assert(meck:called(?MOD, db_found, [?DBNAME, zig])) end). - t_handle_info_other_event() -> - ?_test(begin + ?_test(begin State = mock_state(), handle_info_check({'$couch_event', ?DBNAME, somethingelse}, State), ?assertNot(meck:called(?MOD, db_created, [?DBNAME, somethingelse])), @@ -502,15 +480,13 @@ t_handle_info_other_event() -> ?assertNot(meck:called(?MOD, db_found, [?DBNAME, somethingelse])) end). - t_handle_info_created_other_db() -> - ?_test(begin + ?_test(begin State = mock_state(), handle_info_check({'$couch_event', <<"otherdb">>, created}, State), ?assertNot(meck:called(?MOD, db_created, [?DBNAME, zig])) end). - t_handle_info_scanner_exit_normal() -> ?_test(begin Res = handle_info({'EXIT', spid, normal}, mock_state()), @@ -519,32 +495,28 @@ t_handle_info_scanner_exit_normal() -> ?assertEqual(nil, RState#state.scanner) end). - t_handle_info_scanner_crashed() -> ?_test(begin Res = handle_info({'EXIT', spid, oops}, mock_state()), ?assertMatch({stop, {scanner_died, oops}, _State}, Res) end). - t_handle_info_event_server_exited() -> ?_test(begin Res = handle_info({'DOWN', esref, type, espid, reason}, mock_state()), ?assertMatch({stop, {couch_event_server_died, reason}, _}, Res) end). - t_handle_info_unknown_pid_exited() -> ?_test(begin State0 = mock_state(), - Res0 = handle_info({'EXIT', somepid, normal}, State0), + Res0 = handle_info({'EXIT', somepid, normal}, State0), ?assertMatch({noreply, State0}, Res0), State1 = mock_state(), Res1 = handle_info({'EXIT', somepid, oops}, State1), ?assertMatch({stop, {unexpected_exit, somepid, oops}, State1}, Res1) end). - t_handle_info_change_feed_exited() -> ?_test(begin Tid0 = mock_ets(), @@ -563,7 +535,6 @@ t_handle_info_change_feed_exited() -> ets:delete(Tid1) end). - t_handle_info_change_feed_exited_and_need_rescan() -> ?_test(begin Tid = mock_ets(), @@ -582,7 +553,6 @@ t_handle_info_change_feed_exited_and_need_rescan() -> ets:delete(Tid) end). - t_spawn_changes_reader() -> ?_test(begin Pid = start_changes_reader(?DBNAME, 3), @@ -592,16 +562,20 @@ t_spawn_changes_reader() -> ?assert(meck:validate(couch_db)), ?assert(meck:validate(couch_changes)), ?assert(meck:called(couch_db, open_int, [?DBNAME, [?CTX, sys_db]])), - ?assert(meck:called(couch_changes, handle_db_changes, [ - #changes_args{ - include_docs = true, - since = 3, - feed = "normal", - timeout = infinity - }, {json_req, null}, db])) + ?assert( + meck:called(couch_changes, handle_db_changes, [ + #changes_args{ + include_docs = true, + since = 3, + feed = "normal", + timeout = infinity + }, + {json_req, null}, + db + ]) + ) end). - t_changes_reader_cb_change() -> ?_test(begin {ok, Pid} = start_link(?SUFFIX, ?MOD, zig, []), @@ -613,7 +587,6 @@ t_changes_reader_cb_change() -> exit(Pid, kill) end). - t_changes_reader_cb_stop() -> ?_test(begin {ok, Pid} = start_link(?SUFFIX, ?MOD, zig, []), @@ -626,11 +599,9 @@ t_changes_reader_cb_stop() -> exit(Pid, kill) end). - t_changes_reader_cb_other() -> ?_assertEqual(acc, changes_reader_cb(other, chtype, acc)). - t_handle_call_resume_scan_no_chfeed_no_ets_entry() -> ?_test(begin Tid = mock_ets(), @@ -644,17 +615,21 @@ t_handle_call_resume_scan_no_chfeed_no_ets_entry() -> [{?DBNAME, Pid}] = RState#state.pids, ChArgs = kill_mock_changes_reader_and_get_its_args(Pid), ?assertEqual({self(), ?DBNAME}, ChArgs), - ?assert(meck:called(couch_changes, handle_db_changes, [ - #changes_args{ - include_docs = true, - since = 0, - feed = "normal", - timeout = infinity - }, {json_req, null}, db])), + ?assert( + meck:called(couch_changes, handle_db_changes, [ + #changes_args{ + include_docs = true, + since = 0, + feed = "normal", + timeout = infinity + }, + {json_req, null}, + db + ]) + ), ets:delete(Tid) end). - t_handle_call_resume_scan_chfeed_no_ets_entry() -> ?_test(begin Tid = mock_ets(), @@ -667,7 +642,6 @@ t_handle_call_resume_scan_chfeed_no_ets_entry() -> kill_mock_changes_reader_and_get_its_args(Pid) end). - t_handle_call_resume_scan_chfeed_ets_entry() -> ?_test(begin Tid = mock_ets(), @@ -681,7 +655,6 @@ t_handle_call_resume_scan_chfeed_ets_entry() -> kill_mock_changes_reader_and_get_its_args(Pid) end). - t_handle_call_resume_scan_no_chfeed_ets_entry() -> ?_test(begin Tid = mock_ets(), @@ -694,92 +667,96 @@ t_handle_call_resume_scan_no_chfeed_ets_entry() -> [{?DBNAME, Pid}] = RState#state.pids, ChArgs = kill_mock_changes_reader_and_get_its_args(Pid), ?assertEqual({self(), ?DBNAME}, ChArgs), - ?assert(meck:called(couch_changes, handle_db_changes, [ - #changes_args{ - include_docs = true, - since = 1, - feed = "normal", - timeout = infinity - }, {json_req, null}, db])), + ?assert( + meck:called(couch_changes, handle_db_changes, [ + #changes_args{ + include_docs = true, + since = 1, + feed = "normal", + timeout = infinity + }, + {json_req, null}, + db + ]) + ), ets:delete(Tid) end). - t_start_link() -> ?_test(begin {ok, Pid} = start_link(?SUFFIX, ?MOD, nil, []), ?assert(is_pid(Pid)), - ?assertMatch(#state{ - mod = ?MOD, - suffix = ?SUFFIX, - ctx = nil, - pids = [], - skip_ddocs = false - }, sys:get_state(Pid)), + ?assertMatch( + #state{ + mod = ?MOD, + suffix = ?SUFFIX, + ctx = nil, + pids = [], + skip_ddocs = false + }, + sys:get_state(Pid) + ), unlink(Pid), exit(Pid, kill), ?assert(meck:called(couch_event, register_all, [Pid])) end). - t_start_link_no_ddocs() -> ?_test(begin {ok, Pid} = start_link(?SUFFIX, ?MOD, nil, [skip_ddocs]), ?assert(is_pid(Pid)), - ?assertMatch(#state{ - mod = ?MOD, - suffix = ?SUFFIX, - ctx = nil, - pids = [], - skip_ddocs = true - }, sys:get_state(Pid)), + ?assertMatch( + #state{ + mod = ?MOD, + suffix = ?SUFFIX, + ctx = nil, + pids = [], + skip_ddocs = true + }, + sys:get_state(Pid) + ), unlink(Pid), exit(Pid, kill) end). - t_misc_gen_server_callbacks() -> ?_test(begin ?assertEqual(ok, terminate(reason, state)), ?assertEqual({ok, state}, code_change(old, state, extra)) end). - scan_dbs_test_() -> -{ - setup, - fun() -> - Ctx = test_util:start_couch([mem3, fabric]), - GlobalDb = ?tempdb(), - ok = fabric:create_db(GlobalDb, [?CTX]), - #shard{name = LocalDb} = hd(mem3:local_shards(GlobalDb)), - {Ctx, GlobalDb, LocalDb} - end, - fun({Ctx, GlobalDb, _LocalDb}) -> - fabric:delete_db(GlobalDb, [?CTX]), - test_util:stop_couch(Ctx) - end, - {with, [ - fun t_find_shard/1, - fun t_shard_not_found/1, - fun t_pass_local/1, - fun t_fail_local/1 - ]} -}. - + { + setup, + fun() -> + Ctx = test_util:start_couch([mem3, fabric]), + GlobalDb = ?tempdb(), + ok = fabric:create_db(GlobalDb, [?CTX]), + #shard{name = LocalDb} = hd(mem3:local_shards(GlobalDb)), + {Ctx, GlobalDb, LocalDb} + end, + fun({Ctx, GlobalDb, _LocalDb}) -> + fabric:delete_db(GlobalDb, [?CTX]), + test_util:stop_couch(Ctx) + end, + {with, [ + fun t_find_shard/1, + fun t_shard_not_found/1, + fun t_pass_local/1, + fun t_fail_local/1 + ]} + }. t_find_shard({_, DbName, _}) -> ?_test(begin ?assertEqual(2, length(local_shards(DbName))) end). - t_shard_not_found(_) -> ?_test(begin ?assertEqual([], local_shards(?tempdb())) end). - t_pass_local({_, _, LocalDb}) -> ?_test(begin scan_local_db(self(), LocalDb), @@ -787,11 +764,10 @@ t_pass_local({_, _, LocalDb}) -> {'$gen_cast', Msg} -> ?assertEqual(Msg, {resume_scan, LocalDb}) after 0 -> - ?assert(false) + ?assert(false) end end). - t_fail_local({_, _, LocalDb}) -> ?_test(begin scan_local_db(self(), <<"some_other_db">>), @@ -799,11 +775,10 @@ t_fail_local({_, _, LocalDb}) -> {'$gen_cast', Msg} -> ?assertNotEqual(Msg, {resume_scan, LocalDb}) after 0 -> - ?assert(true) + ?assert(true) end end). - % Test helper functions mock_logs() -> @@ -812,7 +787,6 @@ mock_logs() -> meck:expect(couch_log, info, 2, ok), meck:expect(couch_log, debug, 2, ok). - mock_callback_mod() -> meck:new(?MOD, [non_strict]), meck:expect(?MOD, db_created, fun(_DbName, Ctx) -> Ctx end), @@ -820,7 +794,6 @@ mock_callback_mod() -> meck:expect(?MOD, db_found, fun(_DbName, Ctx) -> Ctx end), meck:expect(?MOD, db_change, fun(_DbName, _Change, Ctx) -> Ctx end). - mock_changes_reader_loop({_CbFun, {Server, DbName}}) -> receive die -> @@ -834,23 +807,23 @@ kill_mock_changes_reader_and_get_its_args(Pid) -> receive {'DOWN', Ref, _, Pid, {Server, DbName}} -> {Server, DbName} - after 1000 -> - erlang:error(spawn_change_reader_timeout) + after 1000 -> + erlang:error(spawn_change_reader_timeout) end. - mock_changes_reader() -> - meck:expect(couch_changes, handle_db_changes, + meck:expect( + couch_changes, + handle_db_changes, fun (_ChArgs, _Req, db) -> fun mock_changes_reader_loop/1; (_ChArgs, _Req, dbs) -> fun(_) -> ok end - end). - + end + ). mock_ets() -> ets:new(multidb_test_ets, [set, public]). - mock_state() -> #state{ mod = ?MOD, @@ -858,19 +831,17 @@ mock_state() -> suffix = ?SUFFIX, event_server = esref, scanner = spid, - pids = []}. - + pids = [] + }. mock_state(Ets) -> State = mock_state(), State#state{tid = Ets}. - mock_state(Ets, Pid) -> State = mock_state(Ets), State#state{pids = [{?DBNAME, Pid}]}. - change_row(Id) when is_binary(Id) -> {[ {<<"seq">>, 1}, @@ -879,13 +850,10 @@ change_row(Id) when is_binary(Id) -> {doc, {[{<<"_id">>, Id}, {<<"_rev">>, <<"1-f00">>}]}} ]}. - handle_call_ok(Msg, State) -> ?assertMatch({reply, ok, _}, handle_call(Msg, from, State)). - handle_info_check(Msg, State) -> ?assertMatch({noreply, _}, handle_info(Msg, State)). - -endif. diff --git a/src/couch/src/couch_native_process.erl b/src/couch/src/couch_native_process.erl index eee8b2860..feea00c3a 100644 --- a/src/couch/src/couch_native_process.erl +++ b/src/couch/src/couch_native_process.erl @@ -41,8 +41,15 @@ -behaviour(gen_server). -vsn(1). --export([start_link/0,init/1,terminate/2,handle_call/3,handle_cast/2,code_change/3, - handle_info/2]). +-export([ + start_link/0, + init/1, + terminate/2, + handle_call/3, + handle_cast/2, + code_change/3, + handle_info/2 +]). -export([set_timeout/2, prompt/2]). -define(STATE, native_proc_state). @@ -74,15 +81,15 @@ prompt(Pid, Data) when is_list(Data) -> init([]) -> V = config:get("query_server_config", "os_process_idle_limit", "300"), Idle = list_to_integer(V) * 1000, - {ok, #evstate{ddocs=dict:new(), idle=Idle}, Idle}. + {ok, #evstate{ddocs = dict:new(), idle = Idle}, Idle}. handle_call({set_timeout, TimeOut}, _From, State) -> - {reply, ok, State#evstate{timeout=TimeOut}, State#evstate.idle}; - + {reply, ok, State#evstate{timeout = TimeOut}, State#evstate.idle}; handle_call({prompt, Data}, _From, State) -> - couch_log:debug("Prompt native qs: ~s",[?JSON_ENCODE(Data)]), - {NewState, Resp} = try run(State, to_binary(Data)) of - {S, R} -> {S, R} + couch_log:debug("Prompt native qs: ~s", [?JSON_ENCODE(Data)]), + {NewState, Resp} = + try run(State, to_binary(Data)) of + {S, R} -> {S, R} catch throw:{error, Why} -> {State, [<<"error">>, Why, Why]} @@ -118,14 +125,14 @@ handle_info(timeout, State) -> gen_server:cast(couch_proc_manager, {os_proc_idle, self()}), erlang:garbage_collect(), {noreply, State, State#evstate.idle}; -handle_info({'EXIT',_,normal}, State) -> +handle_info({'EXIT', _, normal}, State) -> {noreply, State, State#evstate.idle}; -handle_info({'EXIT',_,Reason}, State) -> +handle_info({'EXIT', _, Reason}, State) -> {stop, Reason, State}. terminate(_Reason, _State) -> ok. code_change(_OldVersion, State, _Extra) -> {ok, State}. -run(#evstate{list_pid=Pid}=State, [<<"list_row">>, Row]) when is_pid(Pid) -> +run(#evstate{list_pid = Pid} = State, [<<"list_row">>, Row]) when is_pid(Pid) -> Pid ! {self(), list_row, Row}, receive {Pid, chunks, Data} -> @@ -137,124 +144,137 @@ run(#evstate{list_pid=Pid}=State, [<<"list_row">>, Row]) when is_pid(Pid) -> throw({timeout, list_cleanup}) end, process_flag(trap_exit, erlang:get(do_trap)), - {State#evstate{list_pid=nil}, [<<"end">>, Data]} + {State#evstate{list_pid = nil}, [<<"end">>, Data]} after State#evstate.timeout -> throw({timeout, list_row}) end; -run(#evstate{list_pid=Pid}=State, [<<"list_end">>]) when is_pid(Pid) -> +run(#evstate{list_pid = Pid} = State, [<<"list_end">>]) when is_pid(Pid) -> Pid ! {self(), list_end}, Resp = - receive - {Pid, list_end, Data} -> - receive - {'EXIT', Pid, normal} -> ok - after State#evstate.timeout -> - throw({timeout, list_cleanup}) - end, - [<<"end">>, Data] - after State#evstate.timeout -> - throw({timeout, list_end}) - end, + receive + {Pid, list_end, Data} -> + receive + {'EXIT', Pid, normal} -> ok + after State#evstate.timeout -> + throw({timeout, list_cleanup}) + end, + [<<"end">>, Data] + after State#evstate.timeout -> + throw({timeout, list_end}) + end, process_flag(trap_exit, erlang:get(do_trap)), - {State#evstate{list_pid=nil}, Resp}; -run(#evstate{list_pid=Pid}=State, _Command) when is_pid(Pid) -> + {State#evstate{list_pid = nil}, Resp}; +run(#evstate{list_pid = Pid} = State, _Command) when is_pid(Pid) -> {State, [<<"error">>, list_error, list_error]}; -run(#evstate{ddocs=DDocs}, [<<"reset">>]) -> - {#evstate{ddocs=DDocs}, true}; -run(#evstate{ddocs=DDocs, idle=Idle}, [<<"reset">>, QueryConfig]) -> +run(#evstate{ddocs = DDocs}, [<<"reset">>]) -> + {#evstate{ddocs = DDocs}, true}; +run(#evstate{ddocs = DDocs, idle = Idle}, [<<"reset">>, QueryConfig]) -> NewState = #evstate{ ddocs = DDocs, query_config = QueryConfig, idle = Idle }, {NewState, true}; -run(#evstate{funs=Funs}=State, [<<"add_fun">> , BinFunc]) -> +run(#evstate{funs = Funs} = State, [<<"add_fun">>, BinFunc]) -> FunInfo = makefun(State, BinFunc), - {State#evstate{funs=Funs ++ [FunInfo]}, true}; -run(State, [<<"map_doc">> , Doc]) -> - Resp = lists:map(fun({Sig, Fun}) -> - erlang:put(Sig, []), - Fun(Doc), - lists:reverse(erlang:get(Sig)) - end, State#evstate.funs), + {State#evstate{funs = Funs ++ [FunInfo]}, true}; +run(State, [<<"map_doc">>, Doc]) -> + Resp = lists:map( + fun({Sig, Fun}) -> + erlang:put(Sig, []), + Fun(Doc), + lists:reverse(erlang:get(Sig)) + end, + State#evstate.funs + ), {State, Resp}; run(State, [<<"reduce">>, Funs, KVs]) -> {Keys, Vals} = - lists:foldl(fun([K, V], {KAcc, VAcc}) -> - {[K | KAcc], [V | VAcc]} - end, {[], []}, KVs), + lists:foldl( + fun([K, V], {KAcc, VAcc}) -> + {[K | KAcc], [V | VAcc]} + end, + {[], []}, + KVs + ), Keys2 = lists:reverse(Keys), Vals2 = lists:reverse(Vals), {State, catch reduce(State, Funs, Keys2, Vals2, false)}; run(State, [<<"rereduce">>, Funs, Vals]) -> {State, catch reduce(State, Funs, null, Vals, true)}; -run(#evstate{ddocs=DDocs}=State, [<<"ddoc">>, <<"new">>, DDocId, DDoc]) -> +run(#evstate{ddocs = DDocs} = State, [<<"ddoc">>, <<"new">>, DDocId, DDoc]) -> DDocs2 = store_ddoc(DDocs, DDocId, DDoc), - {State#evstate{ddocs=DDocs2}, true}; -run(#evstate{ddocs=DDocs}=State, [<<"ddoc">>, DDocId | Rest]) -> + {State#evstate{ddocs = DDocs2}, true}; +run(#evstate{ddocs = DDocs} = State, [<<"ddoc">>, DDocId | Rest]) -> DDoc = load_ddoc(DDocs, DDocId), ddoc(State, DDoc, Rest); run(_, Unknown) -> couch_log:error("Native Process: Unknown command: ~p~n", [Unknown]), throw({error, unknown_command}). - + ddoc(State, {DDoc}, [FunPath, Args]) -> % load fun from the FunPath - BFun = lists:foldl(fun - (Key, {Props}) when is_list(Props) -> - couch_util:get_value(Key, Props, nil); - (_Key, Fun) when is_binary(Fun) -> - Fun; - (_Key, nil) -> - throw({error, not_found}); - (_Key, _Fun) -> - throw({error, malformed_ddoc}) - end, {DDoc}, FunPath), + BFun = lists:foldl( + fun + (Key, {Props}) when is_list(Props) -> + couch_util:get_value(Key, Props, nil); + (_Key, Fun) when is_binary(Fun) -> + Fun; + (_Key, nil) -> + throw({error, not_found}); + (_Key, _Fun) -> + throw({error, malformed_ddoc}) + end, + {DDoc}, + FunPath + ), ddoc(State, makefun(State, BFun, {DDoc}), FunPath, Args). ddoc(State, {_, Fun}, [<<"validate_doc_update">>], Args) -> {State, (catch apply(Fun, Args))}; ddoc(State, {_, Fun}, [<<"rewrites">>], Args) -> {State, (catch apply(Fun, Args))}; -ddoc(State, {_, Fun}, [<<"filters">>|_], [Docs, Req]) -> +ddoc(State, {_, Fun}, [<<"filters">> | _], [Docs, Req]) -> FilterFunWrapper = fun(Doc) -> case catch Fun(Doc, Req) of - true -> true; - false -> false; - {'EXIT', Error} -> couch_log:error("~p", [Error]) + true -> true; + false -> false; + {'EXIT', Error} -> couch_log:error("~p", [Error]) end end, Resp = lists:map(FilterFunWrapper, Docs), {State, [true, Resp]}; -ddoc(State, {_, Fun}, [<<"views">>|_], [Docs]) -> +ddoc(State, {_, Fun}, [<<"views">> | _], [Docs]) -> MapFunWrapper = fun(Doc) -> case catch Fun(Doc) of - undefined -> true; - ok -> false; - false -> false; - [_|_] -> true; - {'EXIT', Error} -> couch_log:error("~p", [Error]) + undefined -> true; + ok -> false; + false -> false; + [_ | _] -> true; + {'EXIT', Error} -> couch_log:error("~p", [Error]) end end, Resp = lists:map(MapFunWrapper, Docs), {State, [true, Resp]}; -ddoc(State, {_, Fun}, [<<"shows">>|_], Args) -> - Resp = case (catch apply(Fun, Args)) of - FunResp when is_list(FunResp) -> - FunResp; - {FunResp} -> - [<<"resp">>, {FunResp}]; - FunResp -> - FunResp - end, +ddoc(State, {_, Fun}, [<<"shows">> | _], Args) -> + Resp = + case (catch apply(Fun, Args)) of + FunResp when is_list(FunResp) -> + FunResp; + {FunResp} -> + [<<"resp">>, {FunResp}]; + FunResp -> + FunResp + end, {State, Resp}; -ddoc(State, {_, Fun}, [<<"updates">>|_], Args) -> - Resp = case (catch apply(Fun, Args)) of - [JsonDoc, JsonResp] -> - [<<"up">>, JsonDoc, JsonResp] - end, +ddoc(State, {_, Fun}, [<<"updates">> | _], Args) -> + Resp = + case (catch apply(Fun, Args)) of + [JsonDoc, JsonResp] -> + [<<"up">>, JsonDoc, JsonResp] + end, {State, Resp}; -ddoc(State, {Sig, Fun}, [<<"lists">>|_], Args) -> +ddoc(State, {Sig, Fun}, [<<"lists">> | _], Args) -> Self = self(), SpawnFun = fun() -> LastChunk = (catch apply(Fun, Args)), @@ -270,22 +290,22 @@ ddoc(State, {Sig, Fun}, [<<"lists">>|_], Args) -> ok end, LastChunks = - case erlang:get(Sig) of - undefined -> [LastChunk]; - OtherChunks -> [LastChunk | OtherChunks] - end, + case erlang:get(Sig) of + undefined -> [LastChunk]; + OtherChunks -> [LastChunk | OtherChunks] + end, Self ! {self(), list_end, lists:reverse(LastChunks)} end, erlang:put(do_trap, process_flag(trap_exit, true)), Pid = spawn_link(SpawnFun), Resp = - receive - {Pid, start, Chunks, JsonResp} -> - [<<"start">>, Chunks, JsonResp] - after State#evstate.timeout -> - throw({timeout, list_start}) - end, - {State#evstate{list_pid=Pid}, Resp}. + receive + {Pid, start, Chunks, JsonResp} -> + [<<"start">>, Chunks, JsonResp] + after State#evstate.timeout -> + throw({timeout, list_start}) + end, + {State#evstate{list_pid = Pid}, Resp}. store_ddoc(DDocs, DDocId, DDoc) -> dict:store(DDocId, DDoc, DDocs). @@ -293,7 +313,11 @@ load_ddoc(DDocs, DDocId) -> try dict:fetch(DDocId, DDocs) of {DDoc} -> {DDoc} catch - _:_Else -> throw({error, ?l2b(io_lib:format("Native Query Server missing DDoc with Id: ~s",[DDocId]))}) + _:_Else -> + throw( + {error, + ?l2b(io_lib:format("Native Query Server missing DDoc with Id: ~s", [DDocId]))} + ) end. bindings(State, Sig) -> @@ -316,10 +340,10 @@ bindings(State, Sig, DDoc) -> Send = fun(Chunk) -> Curr = - case erlang:get(Sig) of - undefined -> []; - Else -> Else - end, + case erlang:get(Sig) of + undefined -> []; + Else -> Else + end, erlang:put(Sig, [Chunk | Curr]) end, @@ -329,10 +353,10 @@ bindings(State, Sig, DDoc) -> ok; _ -> Chunks = - case erlang:get(Sig) of - undefined -> []; - CurrChunks -> CurrChunks - end, + case erlang:get(Sig) of + undefined -> []; + CurrChunks -> CurrChunks + end, Self ! {self(), chunks, lists:reverse(Chunks)} end, erlang:put(Sig, []), @@ -343,7 +367,7 @@ bindings(State, Sig, DDoc) -> throw({timeout, list_pid_getrow}) end end, - + FoldRows = fun(Fun, Acc) -> foldrows(GetRow, Fun, Acc) end, Bindings = [ @@ -357,7 +381,8 @@ bindings(State, Sig, DDoc) -> case DDoc of {_Props} -> Bindings ++ [{'DDoc', DDoc}]; - _Else -> Bindings + _Else -> + Bindings end. % thanks to erlview, via: @@ -373,30 +398,41 @@ makefun(State, Source, {DDoc}) -> makefun(_State, Source, BindFuns) when is_list(BindFuns) -> FunStr = binary_to_list(Source), {ok, Tokens, _} = erl_scan:string(FunStr), - Form = case (catch erl_parse:parse_exprs(Tokens)) of - {ok, [ParsedForm]} -> - ParsedForm; - {error, {LineNum, _Mod, [Mesg, Params]}}=Error -> - couch_log:error("Syntax error on line: ~p~n~s~p~n", - [LineNum, Mesg, Params]), - throw(Error) - end, - Bindings = lists:foldl(fun({Name, Fun}, Acc) -> - erl_eval:add_binding(Name, Fun, Acc) - end, erl_eval:new_bindings(), BindFuns), + Form = + case (catch erl_parse:parse_exprs(Tokens)) of + {ok, [ParsedForm]} -> + ParsedForm; + {error, {LineNum, _Mod, [Mesg, Params]}} = Error -> + couch_log:error( + "Syntax error on line: ~p~n~s~p~n", + [LineNum, Mesg, Params] + ), + throw(Error) + end, + Bindings = lists:foldl( + fun({Name, Fun}, Acc) -> + erl_eval:add_binding(Name, Fun, Acc) + end, + erl_eval:new_bindings(), + BindFuns + ), {value, Fun, _} = erl_eval:expr(Form, Bindings), Fun. reduce(State, BinFuns, Keys, Vals, ReReduce) -> - Funs = case is_list(BinFuns) of - true -> - lists:map(fun(BF) -> makefun(State, BF) end, BinFuns); - _ -> - [makefun(State, BinFuns)] - end, - Reds = lists:map(fun({_Sig, Fun}) -> - Fun(Keys, Vals, ReReduce) - end, Funs), + Funs = + case is_list(BinFuns) of + true -> + lists:map(fun(BF) -> makefun(State, BF) end, BinFuns); + _ -> + [makefun(State, BinFuns)] + end, + Reds = lists:map( + fun({_Sig, Fun}) -> + Fun(Keys, Vals, ReReduce) + end, + Funs + ), [true, Reds]. foldrows(GetRow, ProcRow, Acc) -> @@ -416,15 +452,15 @@ start_list_resp(Self, Sig) -> case erlang:get(list_started) of undefined -> Headers = - case erlang:get(list_headers) of - undefined -> {[{<<"headers">>, {[]}}]}; - CurrHdrs -> CurrHdrs - end, + case erlang:get(list_headers) of + undefined -> {[{<<"headers">>, {[]}}]}; + CurrHdrs -> CurrHdrs + end, Chunks = - case erlang:get(Sig) of - undefined -> []; - CurrChunks -> CurrChunks - end, + case erlang:get(Sig) of + undefined -> []; + CurrChunks -> CurrChunks + end, Self ! {self(), start, lists:reverse(Chunks), Headers}, erlang:put(list_started, true), erlang:put(Sig, []), diff --git a/src/couch/src/couch_os_process.erl b/src/couch/src/couch_os_process.erl index 63a241433..da5df5134 100644 --- a/src/couch/src/couch_os_process.erl +++ b/src/couch/src/couch_os_process.erl @@ -23,14 +23,14 @@ -define(PORT_OPTIONS, [stream, {line, 4096}, binary, exit_status, hide]). --record(os_proc, - {command, - port, - writer, - reader, - timeout=5000, - idle - }). +-record(os_proc, { + command, + port, + writer, + reader, + timeout = 5000, + idle +}). start_link(Command) -> start_link(Command, []). @@ -55,7 +55,7 @@ prompt(Pid, Data) -> {ok, Result} -> Result; Error -> - couch_log:error("OS Process Error ~p :: ~p",[Pid,Error]), + couch_log:error("OS Process Error ~p :: ~p", [Pid, Error]), throw(Error) end. @@ -72,21 +72,21 @@ readline(#os_proc{} = OsProc) -> Res. readline(#os_proc{port = Port} = OsProc, Acc) -> receive - {Port, {data, {noeol, Data}}} when is_binary(Acc) -> - readline(OsProc, <<Acc/binary,Data/binary>>); - {Port, {data, {noeol, Data}}} when is_binary(Data) -> - readline(OsProc, Data); - {Port, {data, {noeol, Data}}} -> - readline(OsProc, [Data|Acc]); - {Port, {data, {eol, <<Data/binary>>}}} when is_binary(Acc) -> - [<<Acc/binary,Data/binary>>]; - {Port, {data, {eol, Data}}} when is_binary(Data) -> - [Data]; - {Port, {data, {eol, Data}}} -> - lists:reverse(Acc, Data); - {Port, Err} -> - catch port_close(Port), - throw({os_process_error, Err}) + {Port, {data, {noeol, Data}}} when is_binary(Acc) -> + readline(OsProc, <<Acc/binary, Data/binary>>); + {Port, {data, {noeol, Data}}} when is_binary(Data) -> + readline(OsProc, Data); + {Port, {data, {noeol, Data}}} -> + readline(OsProc, [Data | Acc]); + {Port, {data, {eol, <<Data/binary>>}}} when is_binary(Acc) -> + [<<Acc/binary, Data/binary>>]; + {Port, {data, {eol, Data}}} when is_binary(Data) -> + [Data]; + {Port, {data, {eol, Data}}} -> + lists:reverse(Acc, Data); + {Port, Err} -> + catch port_close(Port), + throw({os_process_error, Err}) after OsProc#os_proc.timeout -> catch port_close(Port), throw({os_process_error, "OS process timed out."}) @@ -95,8 +95,10 @@ readline(#os_proc{port = Port} = OsProc, Acc) -> % Standard JSON functions writejson(OsProc, Data) when is_record(OsProc, os_proc) -> JsonData = ?JSON_ENCODE(Data), - couch_log:debug("OS Process ~p Input :: ~s", - [OsProc#os_proc.port, JsonData]), + couch_log:debug( + "OS Process ~p Input :: ~s", + [OsProc#os_proc.port, JsonData] + ), true = writeline(OsProc, JsonData). readjson(OsProc) when is_record(OsProc, os_proc) -> @@ -109,24 +111,28 @@ readjson(OsProc) when is_record(OsProc, os_proc) -> % command, otherwise return the raw JSON line to the caller. pick_command(Line) catch - throw:abort -> - {json, Line}; - throw:{cmd, _Cmd} -> - case ?JSON_DECODE(Line) of - [<<"log">>, Msg] when is_binary(Msg) -> - % we got a message to log. Log it and continue - couch_log:info("OS Process ~p Log :: ~s", - [OsProc#os_proc.port, Msg]), - readjson(OsProc); - [<<"error">>, Id, Reason] -> - throw({error, {couch_util:to_existing_atom(Id),Reason}}); - [<<"fatal">>, Id, Reason] -> - couch_log:info("OS Process ~p Fatal Error :: ~s ~p", - [OsProc#os_proc.port, Id, Reason]), - throw({couch_util:to_existing_atom(Id),Reason}); - _Result -> - {json, Line} - end + throw:abort -> + {json, Line}; + throw:{cmd, _Cmd} -> + case ?JSON_DECODE(Line) of + [<<"log">>, Msg] when is_binary(Msg) -> + % we got a message to log. Log it and continue + couch_log:info( + "OS Process ~p Log :: ~s", + [OsProc#os_proc.port, Msg] + ), + readjson(OsProc); + [<<"error">>, Id, Reason] -> + throw({error, {couch_util:to_existing_atom(Id), Reason}}); + [<<"fatal">>, Id, Reason] -> + couch_log:info( + "OS Process ~p Fatal Error :: ~s ~p", + [OsProc#os_proc.port, Id, Reason] + ), + throw({couch_util:to_existing_atom(Id), Reason}); + _Result -> + {json, Line} + end end. pick_command(Line) -> @@ -146,7 +152,6 @@ pick_command1(<<"fatal">> = Cmd) -> pick_command1(_) -> throw(abort). - % gen_server API init([Command, Options, PortOptions]) -> couch_io_logger:start(os:getenv("COUCHDB_IO_LOG_DIR")), @@ -155,34 +160,38 @@ init([Command, Options, PortOptions]) -> V = config:get("query_server_config", "os_process_idle_limit", "300"), IdleLimit = list_to_integer(V) * 1000, BaseProc = #os_proc{ - command=Command, - port=open_port({spawn, Spawnkiller ++ " " ++ Command}, PortOptions), - writer=fun ?MODULE:writejson/2, - reader=fun ?MODULE:readjson/1, - idle=IdleLimit + command = Command, + port = open_port({spawn, Spawnkiller ++ " " ++ Command}, PortOptions), + writer = fun ?MODULE:writejson/2, + reader = fun ?MODULE:readjson/1, + idle = IdleLimit }, KillCmd = iolist_to_binary(readline(BaseProc)), Pid = self(), couch_log:debug("OS Process Start :: ~p", [BaseProc#os_proc.port]), spawn(fun() -> - % this ensure the real os process is killed when this process dies. - erlang:monitor(process, Pid), - killer(?b2l(KillCmd)) - end), + % this ensure the real os process is killed when this process dies. + erlang:monitor(process, Pid), + killer(?b2l(KillCmd)) + end), OsProc = - lists:foldl(fun(Opt, Proc) -> - case Opt of - {writer, Writer} when is_function(Writer) -> - Proc#os_proc{writer=Writer}; - {reader, Reader} when is_function(Reader) -> - Proc#os_proc{reader=Reader}; - {timeout, TimeOut} when is_integer(TimeOut) -> - Proc#os_proc{timeout=TimeOut} - end - end, BaseProc, Options), + lists:foldl( + fun(Opt, Proc) -> + case Opt of + {writer, Writer} when is_function(Writer) -> + Proc#os_proc{writer = Writer}; + {reader, Reader} when is_function(Reader) -> + Proc#os_proc{reader = Reader}; + {timeout, TimeOut} when is_integer(TimeOut) -> + Proc#os_proc{timeout = TimeOut} + end + end, + BaseProc, + Options + ), {ok, OsProc, IdleLimit}. -terminate(Reason, #os_proc{port=Port}) -> +terminate(Reason, #os_proc{port = Port}) -> catch port_close(Port), case Reason of normal -> @@ -192,10 +201,10 @@ terminate(Reason, #os_proc{port=Port}) -> end, ok. -handle_call({set_timeout, TimeOut}, _From, #os_proc{idle=Idle}=OsProc) -> - {reply, ok, OsProc#os_proc{timeout=TimeOut}, Idle}; -handle_call({prompt, Data}, _From, #os_proc{idle=Idle}=OsProc) -> - #os_proc{writer=Writer, reader=Reader} = OsProc, +handle_call({set_timeout, TimeOut}, _From, #os_proc{idle = Idle} = OsProc) -> + {reply, ok, OsProc#os_proc{timeout = TimeOut}, Idle}; +handle_call({prompt, Data}, _From, #os_proc{idle = Idle} = OsProc) -> + #os_proc{writer = Writer, reader = Reader} = OsProc, try Writer(OsProc, Data), {reply, {ok, Reader(OsProc)}, OsProc, Idle} @@ -210,7 +219,7 @@ handle_call({prompt, Data}, _From, #os_proc{idle=Idle}=OsProc) -> garbage_collect() end. -handle_cast({send, Data}, #os_proc{writer=Writer, idle=Idle}=OsProc) -> +handle_cast({send, Data}, #os_proc{writer = Writer, idle = Idle} = OsProc) -> try Writer(OsProc, Data), {noreply, OsProc, Idle} @@ -219,31 +228,31 @@ handle_cast({send, Data}, #os_proc{writer=Writer, idle=Idle}=OsProc) -> couch_log:error("Failed sending data: ~p -> ~p", [Data, OsError]), {stop, normal, OsProc} end; -handle_cast(garbage_collect, #os_proc{idle=Idle}=OsProc) -> +handle_cast(garbage_collect, #os_proc{idle = Idle} = OsProc) -> erlang:garbage_collect(), {noreply, OsProc, Idle}; handle_cast(stop, OsProc) -> {stop, normal, OsProc}; -handle_cast(Msg, #os_proc{idle=Idle}=OsProc) -> +handle_cast(Msg, #os_proc{idle = Idle} = OsProc) -> couch_log:debug("OS Proc: Unknown cast: ~p", [Msg]), {noreply, OsProc, Idle}. -handle_info(timeout, #os_proc{idle=Idle}=OsProc) -> +handle_info(timeout, #os_proc{idle = Idle} = OsProc) -> gen_server:cast(couch_proc_manager, {os_proc_idle, self()}), erlang:garbage_collect(), {noreply, OsProc, Idle}; -handle_info({Port, {exit_status, 0}}, #os_proc{port=Port}=OsProc) -> +handle_info({Port, {exit_status, 0}}, #os_proc{port = Port} = OsProc) -> couch_log:info("OS Process terminated normally", []), {stop, normal, OsProc}; -handle_info({Port, {exit_status, Status}}, #os_proc{port=Port}=OsProc) -> +handle_info({Port, {exit_status, Status}}, #os_proc{port = Port} = OsProc) -> couch_log:error("OS Process died with status: ~p", [Status]), {stop, {exit_status, Status}, OsProc}; -handle_info(Msg, #os_proc{idle=Idle}=OsProc) -> +handle_info(Msg, #os_proc{idle = Idle} = OsProc) -> couch_log:debug("OS Proc: Unknown info: ~p", [Msg]), {noreply, OsProc, Idle}. -code_change(_, {os_proc, Cmd, Port, W, R, Timeout} , _) -> - V = config:get("query_server_config","os_process_idle_limit","300"), +code_change(_, {os_proc, Cmd, Port, W, R, Timeout}, _) -> + V = config:get("query_server_config", "os_process_idle_limit", "300"), State = #os_proc{ command = Cmd, port = Port, @@ -257,9 +266,9 @@ code_change(_OldVsn, State, _Extra) -> {ok, State}. killer(KillCmd) -> - receive _ -> - os:cmd(KillCmd) + receive + _ -> + os:cmd(KillCmd) after 1000 -> ?MODULE:killer(KillCmd) end. - diff --git a/src/couch/src/couch_partition.erl b/src/couch/src/couch_partition.erl index f2efcaa5e..101b5b324 100644 --- a/src/couch/src/couch_partition.erl +++ b/src/couch/src/couch_partition.erl @@ -12,7 +12,6 @@ -module(couch_partition). - -export([ extract/1, from_docid/1, @@ -29,10 +28,8 @@ hash/1 ]). - -include_lib("couch/include/couch_db.hrl"). - extract(Value) when is_binary(Value) -> case binary:split(Value, <<":">>) of [Partition, Rest] -> @@ -40,11 +37,9 @@ extract(Value) when is_binary(Value) -> _ -> undefined end; - extract(_) -> undefined. - from_docid(DocId) -> case extract(DocId) of undefined -> @@ -53,7 +48,6 @@ from_docid(DocId) -> Partition end. - is_member(DocId, Partition) -> case extract(DocId) of {Partition, _} -> @@ -62,53 +56,52 @@ is_member(DocId, Partition) -> false end. - start_key(Partition) -> <<Partition/binary, ":">>. - end_key(Partition) -> <<Partition/binary, ";">>. - shard_key(Partition) -> <<Partition/binary, ":foo">>. - validate_dbname(DbName, Options) when is_list(DbName) -> validate_dbname(?l2b(DbName), Options); validate_dbname(DbName, Options) when is_binary(DbName) -> Props = couch_util:get_value(props, Options, []), IsPartitioned = couch_util:get_value(partitioned, Props, false), - if not IsPartitioned -> ok; true -> - - DbsDbName = config:get("mem3", "shards_db", "_dbs"), - NodesDbName = config:get("mem3", "nodes_db", "_nodes"), - UsersDbSuffix = config:get("couchdb", "users_db_suffix", "_users"), - Suffix = couch_db:dbname_suffix(DbName), + if + not IsPartitioned -> + ok; + true -> + DbsDbName = config:get("mem3", "shards_db", "_dbs"), + NodesDbName = config:get("mem3", "nodes_db", "_nodes"), + UsersDbSuffix = config:get("couchdb", "users_db_suffix", "_users"), + Suffix = couch_db:dbname_suffix(DbName), - SysDbNames = [ + SysDbNames = [ iolist_to_binary(DbsDbName), iolist_to_binary(NodesDbName) | ?SYSTEM_DATABASES ], - Suffices = [ + Suffices = [ <<"_replicator">>, <<"_users">>, iolist_to_binary(UsersDbSuffix) ], - IsSysDb = lists:member(DbName, SysDbNames) - orelse lists:member(Suffix, Suffices), + IsSysDb = + lists:member(DbName, SysDbNames) orelse + lists:member(Suffix, Suffices), - if not IsSysDb -> ok; true -> - throw({bad_request, <<"Cannot partition a system database">>}) - end + if + not IsSysDb -> ok; + true -> throw({bad_request, <<"Cannot partition a system database">>}) + end end. - validate_docid(<<"_design/", _/binary>>) -> ok; validate_docid(<<"_local/", _/binary>>) -> @@ -125,7 +118,6 @@ validate_docid(DocId) when is_binary(DocId) -> couch_doc:validate_docid(PartitionedDocId) end. - validate_partition(<<>>) -> throw({illegal_partition, <<"Partition must not be empty">>}); validate_partition(Partition) when is_binary(Partition) -> @@ -153,7 +145,6 @@ validate_partition(Partition) when is_binary(Partition) -> validate_partition(_) -> throw({illegal_partition, <<"Partition must be a string">>}). - % Document ids that start with an underscore % (i.e., _local and _design) do not contain a % partition and thus do not use the partition diff --git a/src/couch/src/couch_passwords.erl b/src/couch/src/couch_passwords.erl index 55ffb359f..828d2f68b 100644 --- a/src/couch/src/couch_passwords.erl +++ b/src/couch/src/couch_passwords.erl @@ -40,98 +40,144 @@ hash_admin_password(ClearPassword) when is_binary(ClearPassword) -> Scheme = chttpd_util:get_chttpd_auth_config("password_scheme", "pbkdf2"), hash_admin_password(Scheme, ClearPassword). -hash_admin_password("simple", ClearPassword) -> % deprecated +% deprecated +hash_admin_password("simple", ClearPassword) -> Salt = couch_uuids:random(), Hash = crypto:hash(sha, <<ClearPassword/binary, Salt/binary>>), ?l2b("-hashed-" ++ couch_util:to_hex(Hash) ++ "," ++ ?b2l(Salt)); hash_admin_password("pbkdf2", ClearPassword) -> Iterations = chttpd_util:get_chttpd_auth_config("iterations", "10"), Salt = couch_uuids:random(), - DerivedKey = couch_passwords:pbkdf2(couch_util:to_binary(ClearPassword), - Salt, list_to_integer(Iterations)), - ?l2b("-pbkdf2-" ++ ?b2l(DerivedKey) ++ "," - ++ ?b2l(Salt) ++ "," - ++ Iterations). + DerivedKey = couch_passwords:pbkdf2( + couch_util:to_binary(ClearPassword), + Salt, + list_to_integer(Iterations) + ), + ?l2b( + "-pbkdf2-" ++ ?b2l(DerivedKey) ++ "," ++ + ?b2l(Salt) ++ "," ++ + Iterations + ). -spec get_unhashed_admins() -> list(). get_unhashed_admins() -> lists:filter( - fun({_User, "-hashed-" ++ _}) -> - false; % already hashed - ({_User, "-pbkdf2-" ++ _}) -> - false; % already hashed - ({_User, _ClearPassword}) -> - true + fun + ({_User, "-hashed-" ++ _}) -> + % already hashed + false; + ({_User, "-pbkdf2-" ++ _}) -> + % already hashed + false; + ({_User, _ClearPassword}) -> + true end, - config:get("admins")). + config:get("admins") + ). %% Current scheme, much stronger. -spec pbkdf2(binary(), binary(), integer()) -> binary(). -pbkdf2(Password, Salt, Iterations) when is_binary(Password), - is_binary(Salt), - is_integer(Iterations), - Iterations > 0 -> +pbkdf2(Password, Salt, Iterations) when + is_binary(Password), + is_binary(Salt), + is_integer(Iterations), + Iterations > 0 +-> {ok, Result} = pbkdf2(Password, Salt, Iterations, ?SHA1_OUTPUT_LENGTH), Result; -pbkdf2(Password, Salt, Iterations) when is_binary(Salt), - is_integer(Iterations), - Iterations > 0 -> +pbkdf2(Password, Salt, Iterations) when + is_binary(Salt), + is_integer(Iterations), + Iterations > 0 +-> Msg = io_lib:format("Password value of '~p' is invalid.", [Password]), throw({forbidden, Msg}); -pbkdf2(Password, Salt, Iterations) when is_binary(Password), - is_integer(Iterations), - Iterations > 0 -> +pbkdf2(Password, Salt, Iterations) when + is_binary(Password), + is_integer(Iterations), + Iterations > 0 +-> Msg = io_lib:format("Salt value of '~p' is invalid.", [Salt]), throw({forbidden, Msg}). --spec pbkdf2(binary(), binary(), integer(), integer()) - -> {ok, binary()} | {error, derived_key_too_long}. -pbkdf2(_Password, _Salt, _Iterations, DerivedLength) - when DerivedLength > ?MAX_DERIVED_KEY_LENGTH -> +-spec pbkdf2(binary(), binary(), integer(), integer()) -> + {ok, binary()} | {error, derived_key_too_long}. +pbkdf2(_Password, _Salt, _Iterations, DerivedLength) when + DerivedLength > ?MAX_DERIVED_KEY_LENGTH +-> {error, derived_key_too_long}; -pbkdf2(Password, Salt, Iterations, DerivedLength) when is_binary(Password), - is_binary(Salt), - is_integer(Iterations), - Iterations > 0, - is_integer(DerivedLength) -> +pbkdf2(Password, Salt, Iterations, DerivedLength) when + is_binary(Password), + is_binary(Salt), + is_integer(Iterations), + Iterations > 0, + is_integer(DerivedLength) +-> L = ceiling(DerivedLength / ?SHA1_OUTPUT_LENGTH), - <<Bin:DerivedLength/binary,_/binary>> = + <<Bin:DerivedLength/binary, _/binary>> = iolist_to_binary(pbkdf2(Password, Salt, Iterations, L, 1, [])), {ok, ?l2b(couch_util:to_hex(Bin))}. --spec pbkdf2(binary(), binary(), integer(), integer(), integer(), iolist()) - -> iolist(). -pbkdf2(_Password, _Salt, _Iterations, BlockCount, BlockIndex, Acc) - when BlockIndex > BlockCount -> +-spec pbkdf2(binary(), binary(), integer(), integer(), integer(), iolist()) -> + iolist(). +pbkdf2(_Password, _Salt, _Iterations, BlockCount, BlockIndex, Acc) when + BlockIndex > BlockCount +-> lists:reverse(Acc); pbkdf2(Password, Salt, Iterations, BlockCount, BlockIndex, Acc) -> Block = pbkdf2(Password, Salt, Iterations, BlockIndex, 1, <<>>, <<>>), - pbkdf2(Password, Salt, Iterations, BlockCount, BlockIndex + 1, [Block|Acc]). + pbkdf2(Password, Salt, Iterations, BlockCount, BlockIndex + 1, [Block | Acc]). --spec pbkdf2(binary(), binary(), integer(), integer(), integer(), - binary(), binary()) -> binary(). -pbkdf2(_Password, _Salt, Iterations, _BlockIndex, Iteration, _Prev, Acc) - when Iteration > Iterations -> +-spec pbkdf2( + binary(), + binary(), + integer(), + integer(), + integer(), + binary(), + binary() +) -> binary(). +pbkdf2(_Password, _Salt, Iterations, _BlockIndex, Iteration, _Prev, Acc) when + Iteration > Iterations +-> Acc; pbkdf2(Password, Salt, Iterations, BlockIndex, 1, _Prev, _Acc) -> - InitialBlock = couch_util:hmac(sha, Password, - <<Salt/binary,BlockIndex:32/integer>>), - pbkdf2(Password, Salt, Iterations, BlockIndex, 2, - InitialBlock, InitialBlock); + InitialBlock = couch_util:hmac( + sha, + Password, + <<Salt/binary, BlockIndex:32/integer>> + ), + pbkdf2( + Password, + Salt, + Iterations, + BlockIndex, + 2, + InitialBlock, + InitialBlock + ); pbkdf2(Password, Salt, Iterations, BlockIndex, Iteration, Prev, Acc) -> Next = couch_util:hmac(sha, Password, Prev), - pbkdf2(Password, Salt, Iterations, BlockIndex, Iteration + 1, - Next, crypto:exor(Next, Acc)). + pbkdf2( + Password, + Salt, + Iterations, + BlockIndex, + Iteration + 1, + Next, + crypto:exor(Next, Acc) + ). %% verify two lists for equality without short-circuits to avoid timing attacks. -spec verify(string(), string(), integer()) -> boolean(). -verify([X|RestX], [Y|RestY], Result) -> +verify([X | RestX], [Y | RestY], Result) -> verify(RestX, RestY, (X bxor Y) bor Result); verify([], [], Result) -> Result == 0. --spec verify(binary(), binary()) -> boolean(); - (list(), list()) -> boolean(). +-spec verify + (binary(), binary()) -> boolean(); + (list(), list()) -> boolean(). verify(<<X/binary>>, <<Y/binary>>) -> verify(?b2l(X), ?b2l(Y)); verify(X, Y) when is_list(X) and is_list(Y) -> @@ -141,7 +187,8 @@ verify(X, Y) when is_list(X) and is_list(Y) -> false -> false end; -verify(_X, _Y) -> false. +verify(_X, _Y) -> + false. -spec ceiling(number()) -> integer(). ceiling(X) -> diff --git a/src/couch/src/couch_primary_sup.erl b/src/couch/src/couch_primary_sup.erl index 73c3de710..4f2917f98 100644 --- a/src/couch/src/couch_primary_sup.erl +++ b/src/couch/src/couch_primary_sup.erl @@ -15,30 +15,20 @@ -export([init/1, start_link/0]). start_link() -> - supervisor:start_link({local,couch_primary_services}, ?MODULE, []). + supervisor:start_link({local, couch_primary_services}, ?MODULE, []). init([]) -> - Children = [ - {couch_task_status, - {couch_task_status, start_link, []}, - permanent, - brutal_kill, - worker, - [couch_task_status]} - ] ++ couch_servers(), + Children = + [ + {couch_task_status, {couch_task_status, start_link, []}, permanent, brutal_kill, worker, + [couch_task_status]} + ] ++ couch_servers(), {ok, {{one_for_one, 10, 3600}, Children}}. - couch_servers() -> N = couch_server:num_servers(), [couch_server(I) || I <- lists:seq(1, N)]. couch_server(N) -> Name = couch_server:couch_server(N), - {Name, - {couch_server, sup_start_link, [N]}, - permanent, - brutal_kill, - worker, - [couch_server] - }. + {Name, {couch_server, sup_start_link, [N]}, permanent, brutal_kill, worker, [couch_server]}. diff --git a/src/couch/src/couch_proc_manager.erl b/src/couch/src/couch_proc_manager.erl index e7a25a6d2..6d86c16a7 100644 --- a/src/couch/src/couch_proc_manager.erl +++ b/src/couch/src/couch_proc_manager.erl @@ -60,7 +60,7 @@ -record(client, { timestamp :: os:timestamp() | '_', - from :: undefined | {pid(), reference()} | '_', + from :: undefined | {pid(), reference()} | '_', lang :: binary() | '_', ddoc :: #doc{} | '_', ddoc_key :: undefined | {DDocId :: docid(), Rev :: revision()} | '_' @@ -77,27 +77,21 @@ t0 = os:timestamp() }). - start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - get_proc_count() -> gen_server:call(?MODULE, get_proc_count). - get_stale_proc_count() -> gen_server:call(?MODULE, get_stale_proc_count). - reload() -> gen_server:call(?MODULE, set_threshold_ts). - terminate_stale_procs() -> gen_server:call(?MODULE, terminate_stale_procs). - init([]) -> process_flag(trap_exit, true), ok = config:listen_for_changes(?MODULE, undefined), @@ -120,50 +114,48 @@ init([]) -> soft_limit = get_soft_limit() }}. - terminate(_Reason, _State) -> - ets:foldl(fun(#proc_int{pid=P}, _) -> - couch_util:shutdown_sync(P) - end, 0, ?PROCS), + ets:foldl( + fun(#proc_int{pid = P}, _) -> + couch_util:shutdown_sync(P) + end, + 0, + ?PROCS + ), ok. - handle_call(get_proc_count, _From, State) -> NumProcs = ets:info(?PROCS, size), NumOpening = ets:info(?OPENING, size), {reply, NumProcs + NumOpening, State}; - handle_call(get_stale_proc_count, _From, State) -> #state{threshold_ts = T0} = State, - MatchSpec = [{#proc_int{t0='$1', _='_'}, [{'<', '$1', {T0}}], [true]}], + MatchSpec = [{#proc_int{t0 = '$1', _ = '_'}, [{'<', '$1', {T0}}], [true]}], {reply, ets:select_count(?PROCS, MatchSpec), State}; - -handle_call({get_proc, #doc{body={Props}}=DDoc, DDocKey}, From, State) -> +handle_call({get_proc, #doc{body = {Props}} = DDoc, DDocKey}, From, State) -> LangStr = couch_util:get_value(<<"language">>, Props, <<"javascript">>), Lang = couch_util:to_binary(LangStr), - Client = #client{from=From, lang=Lang, ddoc=DDoc, ddoc_key=DDocKey}, + Client = #client{from = From, lang = Lang, ddoc = DDoc, ddoc_key = DDocKey}, add_waiting_client(Client), {noreply, flush_waiters(State, Lang)}; - handle_call({get_proc, LangStr}, From, State) -> Lang = couch_util:to_binary(LangStr), - Client = #client{from=From, lang=Lang}, + Client = #client{from = From, lang = Lang}, add_waiting_client(Client), {noreply, flush_waiters(State, Lang)}; - -handle_call({ret_proc, #proc{client=Ref} = Proc}, _From, State) -> +handle_call({ret_proc, #proc{client = Ref} = Proc}, _From, State) -> erlang:demonitor(Ref, [flush]), - NewState = case ets:lookup(?PROCS, Proc#proc.pid) of - [#proc_int{}=ProcInt] -> - return_proc(State, ProcInt); - [] -> - % Proc must've died and we already - % cleared it out of the table in - % the handle_info clause. - State - end, + NewState = + case ets:lookup(?PROCS, Proc#proc.pid) of + [#proc_int{} = ProcInt] -> + return_proc(State, ProcInt); + [] -> + % Proc must've died and we already + % cleared it out of the table in + % the handle_info clause. + State + end, {reply, true, NewState}; - handle_call(set_threshold_ts, _From, State) -> FoldFun = fun (#proc_int{client = undefined} = Proc, StateAcc) -> @@ -173,7 +165,6 @@ handle_call(set_threshold_ts, _From, State) -> end, NewState = ets:foldl(FoldFun, State, ?PROCS), {reply, ok, NewState#state{threshold_ts = os:timestamp()}}; - handle_call(terminate_stale_procs, _From, #state{threshold_ts = Ts1} = State) -> FoldFun = fun (#proc_int{client = undefined, t0 = Ts2} = Proc, StateAcc) -> @@ -188,26 +179,24 @@ handle_call(terminate_stale_procs, _From, #state{threshold_ts = Ts1} = State) -> end, NewState = ets:foldl(FoldFun, State, ?PROCS), {reply, ok, NewState}; - handle_call(_Call, _From, State) -> {reply, ignored, State}. - -handle_cast({os_proc_idle, Pid}, #state{counts=Counts}=State) -> - NewState = case ets:lookup(?PROCS, Pid) of - [#proc_int{client=undefined, lang=Lang}=Proc] -> - case dict:find(Lang, Counts) of - {ok, Count} when Count >= State#state.soft_limit -> - couch_log:info("Closing idle OS Process: ~p", [Pid]), - remove_proc(State, Proc); - {ok, _} -> - State - end; - _ -> - State - end, +handle_cast({os_proc_idle, Pid}, #state{counts = Counts} = State) -> + NewState = + case ets:lookup(?PROCS, Pid) of + [#proc_int{client = undefined, lang = Lang} = Proc] -> + case dict:find(Lang, Counts) of + {ok, Count} when Count >= State#state.soft_limit -> + couch_log:info("Closing idle OS Process: ~p", [Pid]), + remove_proc(State, Proc); + {ok, _} -> + State + end; + _ -> + State + end, {noreply, NewState}; - handle_cast(reload_config, State) -> NewState = State#state{ config = get_proc_config(), @@ -216,29 +205,24 @@ handle_cast(reload_config, State) -> }, maybe_configure_erlang_native_servers(), {noreply, flush_waiters(NewState)}; - handle_cast(_Msg, State) -> {noreply, State}. - handle_info(shutdown, State) -> {stop, shutdown, State}; - -handle_info({'EXIT', Pid, {spawn_ok, Proc0, {ClientPid,_} = From}}, State) -> +handle_info({'EXIT', Pid, {spawn_ok, Proc0, {ClientPid, _} = From}}, State) -> ets:delete(?OPENING, Pid), link(Proc0#proc_int.pid), Proc = assign_proc(ClientPid, Proc0), gen_server:reply(From, {ok, Proc, State#state.config}), {noreply, State}; - handle_info({'EXIT', Pid, spawn_error}, State) -> - [{Pid, #client{lang=Lang}}] = ets:lookup(?OPENING, Pid), + [{Pid, #client{lang = Lang}}] = ets:lookup(?OPENING, Pid), ets:delete(?OPENING, Pid), NewState = State#state{ counts = dict:update_counter(Lang, -1, State#state.counts) }, {noreply, flush_waiters(NewState, Lang)}; - handle_info({'EXIT', Pid, Reason}, State) -> couch_log:info("~p ~p died ~p", [?MODULE, Pid, Reason]), case ets:lookup(?PROCS, Pid) of @@ -248,25 +232,20 @@ handle_info({'EXIT', Pid, Reason}, State) -> [] -> {noreply, State} end; - handle_info({'DOWN', Ref, _, _, _Reason}, State0) -> - case ets:match_object(?PROCS, #proc_int{client=Ref, _='_'}) of + case ets:match_object(?PROCS, #proc_int{client = Ref, _ = '_'}) of [#proc_int{} = Proc] -> {noreply, return_proc(State0, Proc)}; [] -> {noreply, State0} end; - - handle_info(restart_config_listener, State) -> ok = config:listen_for_changes(?MODULE, nil), {noreply, State}; - handle_info(_Msg, State) -> {noreply, State}. - -code_change(_OldVsn, #state{}=State, _Extra) -> +code_change(_OldVsn, #state{} = State, _Extra) -> {ok, State}. handle_config_terminate(_, stop, _) -> @@ -284,7 +263,6 @@ handle_config_change("query_server_config", _, _, _, _) -> handle_config_change(_, _, _, _, _) -> {ok, undefined}. - find_proc(#client{lang = Lang, ddoc_key = undefined}) -> Pred = fun(_) -> true @@ -296,7 +274,7 @@ find_proc(#client{lang = Lang, ddoc = DDoc, ddoc_key = DDocKey} = Client) -> end, case find_proc(Lang, Pred) of not_found -> - case find_proc(Client#client{ddoc_key=undefined}) of + case find_proc(Client#client{ddoc_key = undefined}) of {ok, Proc} -> teach_ddoc(DDoc, DDocKey, Proc); Else -> @@ -313,9 +291,8 @@ find_proc(Lang, Fun) -> {error, Reason} end. - iter_procs(Lang, Fun) when is_binary(Lang) -> - Pattern = #proc_int{lang=Lang, client=undefined, _='_'}, + Pattern = #proc_int{lang = Lang, client = undefined, _ = '_'}, MSpec = [{Pattern, [], ['$_']}], case ets:select_reverse(?PROCS, MSpec, 25) of '$end_of_table' -> @@ -324,7 +301,6 @@ iter_procs(Lang, Fun) when is_binary(Lang) -> iter_procs_int(Continuation, Fun) end. - iter_procs_int({[], Continuation0}, Fun) -> case ets:select_reverse(Continuation0) of '$end_of_table' -> @@ -340,7 +316,6 @@ iter_procs_int({[Proc | Rest], Continuation}, Fun) -> iter_procs_int({Rest, Continuation}, Fun) end. - spawn_proc(State, Client) -> Pid = spawn_link(?MODULE, new_proc, [Client]), ets:insert(?OPENING, {Pid, Client}), @@ -350,36 +325,38 @@ spawn_proc(State, Client) -> counts = dict:update_counter(Lang, 1, Counts) }. - -new_proc(#client{ddoc=undefined, ddoc_key=undefined}=Client) -> - #client{from=From, lang=Lang} = Client, - Resp = try - case new_proc_int(From, Lang) of - {ok, Proc} -> - {spawn_ok, Proc, From}; - Error -> - gen_server:reply(From, {error, Error}), +new_proc(#client{ddoc = undefined, ddoc_key = undefined} = Client) -> + #client{from = From, lang = Lang} = Client, + Resp = + try + case new_proc_int(From, Lang) of + {ok, Proc} -> + {spawn_ok, Proc, From}; + Error -> + gen_server:reply(From, {error, Error}), + spawn_error + end + catch + _:_ -> spawn_error - end - catch _:_ -> - spawn_error - end, + end, exit(Resp); - new_proc(Client) -> - #client{from=From, lang=Lang, ddoc=DDoc, ddoc_key=DDocKey} = Client, - Resp = try - case new_proc_int(From, Lang) of - {ok, NewProc} -> - {ok, Proc} = teach_ddoc(DDoc, DDocKey, NewProc), - {spawn_ok, Proc, From}; - Error -> - gen_server:reply(From, {error, Error}), - spawn_error - end - catch _:_ -> - spawn_error - end, + #client{from = From, lang = Lang, ddoc = DDoc, ddoc_key = DDocKey} = Client, + Resp = + try + case new_proc_int(From, Lang) of + {ok, NewProc} -> + {ok, Proc} = teach_ddoc(DDoc, DDocKey, NewProc), + {spawn_ok, Proc, From}; + Error -> + gen_server:reply(From, {error, Error}), + spawn_error + end + catch + _:_ -> + spawn_error + end, exit(Resp). split_string_if_longer(String, Pos) -> @@ -399,14 +376,17 @@ split_by_char(String, Char) -> get_servers_from_env(Spec) -> SpecLen = length(Spec), % loop over os:getenv(), match SPEC_ - lists:filtermap(fun(EnvStr) -> - case split_string_if_longer(EnvStr, SpecLen) of - {Spec, Rest} -> - {true, split_by_char(Rest, $=)}; - _ -> - false - end - end, os:getenv()). + lists:filtermap( + fun(EnvStr) -> + case split_string_if_longer(EnvStr, SpecLen) of + {Spec, Rest} -> + {true, split_by_char(Rest, $=)}; + _ -> + false + end + end, + os:getenv() + ). get_query_server(LangStr) -> case ets:lookup(?SERVERS, string:to_upper(LangStr)) of @@ -425,39 +405,39 @@ native_query_server_enabled() -> maybe_configure_erlang_native_servers() -> case native_query_server_enabled() of true -> - ets:insert(?SERVERS, [ - {"ERLANG", {couch_native_process, start_link, []}}]); + ets:insert(?SERVERS, [ + {"ERLANG", {couch_native_process, start_link, []}} + ]); _Else -> - ok + ok end. new_proc_int(From, Lang) when is_binary(Lang) -> LangStr = binary_to_list(Lang), case get_query_server(LangStr) of - undefined -> - gen_server:reply(From, {unknown_query_language, Lang}); - {M, F, A} -> - {ok, Pid} = apply(M, F, A), - make_proc(Pid, Lang, M); - Command -> - {ok, Pid} = couch_os_process:start_link(Command), - make_proc(Pid, Lang, couch_os_process) + undefined -> + gen_server:reply(From, {unknown_query_language, Lang}); + {M, F, A} -> + {ok, Pid} = apply(M, F, A), + make_proc(Pid, Lang, M); + Command -> + {ok, Pid} = couch_os_process:start_link(Command), + make_proc(Pid, Lang, couch_os_process) end. - -teach_ddoc(DDoc, {DDocId, _Rev}=DDocKey, #proc_int{ddoc_keys=Keys}=Proc) -> +teach_ddoc(DDoc, {DDocId, _Rev} = DDocKey, #proc_int{ddoc_keys = Keys} = Proc) -> % send ddoc over the wire % we only share the rev with the client we know to update code % but it only keeps the latest copy, per each ddoc, around. true = couch_query_servers:proc_prompt( export_proc(Proc), - [<<"ddoc">>, <<"new">>, DDocId, couch_doc:to_json_obj(DDoc, [])]), + [<<"ddoc">>, <<"new">>, DDocId, couch_doc:to_json_obj(DDoc, [])] + ), % we should remove any other ddocs keys for this docid % because the query server overwrites without the rev - Keys2 = [{D,R} || {D,R} <- Keys, D /= DDocId], + Keys2 = [{D, R} || {D, R} <- Keys, D /= DDocId], % add ddoc to the proc - {ok, Proc#proc_int{ddoc_keys=[DDocKey|Keys2]}}. - + {ok, Proc#proc_int{ddoc_keys = [DDocKey | Keys2]}}. make_proc(Pid, Lang, Mod) when is_binary(Lang) -> Proc = #proc_int{ @@ -470,42 +450,42 @@ make_proc(Pid, Lang, Mod) when is_binary(Lang) -> unlink(Pid), {ok, Proc}. - -assign_proc(Pid, #proc_int{client=undefined}=Proc0) when is_pid(Pid) -> +assign_proc(Pid, #proc_int{client = undefined} = Proc0) when is_pid(Pid) -> Proc = Proc0#proc_int{client = erlang:monitor(process, Pid)}, ets:insert(?PROCS, Proc), export_proc(Proc); -assign_proc(#client{}=Client, #proc_int{client=undefined}=Proc) -> +assign_proc(#client{} = Client, #proc_int{client = undefined} = Proc) -> {Pid, _} = Client#client.from, assign_proc(Pid, Proc). - return_proc(#state{} = State, #proc_int{} = ProcInt) -> #proc_int{pid = Pid, lang = Lang} = ProcInt, - NewState = case is_process_alive(Pid) of true -> - case ProcInt#proc_int.t0 < State#state.threshold_ts of + NewState = + case is_process_alive(Pid) of true -> - remove_proc(State, ProcInt); + case ProcInt#proc_int.t0 < State#state.threshold_ts of + true -> + remove_proc(State, ProcInt); + false -> + gen_server:cast(Pid, garbage_collect), + true = ets:update_element(?PROCS, Pid, [ + {#proc_int.client, undefined} + ]), + State + end; false -> - gen_server:cast(Pid, garbage_collect), - true = ets:update_element(?PROCS, Pid, [ - {#proc_int.client, undefined} - ]), - State - end; - false -> - remove_proc(State, ProcInt) - end, + remove_proc(State, ProcInt) + end, flush_waiters(NewState, Lang). - -remove_proc(State, #proc_int{}=Proc) -> +remove_proc(State, #proc_int{} = Proc) -> ets:delete(?PROCS, Proc#proc_int.pid), - case is_process_alive(Proc#proc_int.pid) of true -> - unlink(Proc#proc_int.pid), - gen_server:cast(Proc#proc_int.pid, stop); - false -> - ok + case is_process_alive(Proc#proc_int.pid) of + true -> + unlink(Proc#proc_int.pid), + gen_server:cast(Proc#proc_int.pid, stop); + false -> + ok end, Counts = State#state.counts, Lang = Proc#proc_int.lang, @@ -513,7 +493,6 @@ remove_proc(State, #proc_int{}=Proc) -> counts = dict:update_counter(Lang, -1, Counts) }. - -spec export_proc(#proc_int{}) -> #proc{}. export_proc(#proc_int{} = ProcInt) -> ProcIntList = tuple_to_list(ProcInt), @@ -521,17 +500,19 @@ export_proc(#proc_int{} = ProcInt) -> [_ | Data] = lists:sublist(ProcIntList, ProcLen), list_to_tuple([proc | Data]). - flush_waiters(State) -> - dict:fold(fun(Lang, Count, StateAcc) -> - case Count < State#state.hard_limit of - true -> - flush_waiters(StateAcc, Lang); - false -> - StateAcc - end - end, State, State#state.counts). - + dict:fold( + fun(Lang, Count, StateAcc) -> + case Count < State#state.hard_limit of + true -> + flush_waiters(StateAcc, Lang); + false -> + StateAcc + end + end, + State, + State#state.counts + ). flush_waiters(State, Lang) -> CanSpawn = can_spawn(State, Lang), @@ -558,31 +539,27 @@ flush_waiters(State, Lang) -> State end. - add_waiting_client(Client) -> - ets:insert(?WAITERS, Client#client{timestamp=os:timestamp()}). + ets:insert(?WAITERS, Client#client{timestamp = os:timestamp()}). -spec get_waiting_client(Lang :: binary()) -> undefined | #client{}. get_waiting_client(Lang) -> - case ets:match_object(?WAITERS, #client{lang=Lang, _='_'}, 1) of + case ets:match_object(?WAITERS, #client{lang = Lang, _ = '_'}, 1) of '$end_of_table' -> undefined; - {[#client{}=Client], _} -> + {[#client{} = Client], _} -> Client end. - remove_waiting_client(#client{timestamp = Timestamp}) -> ets:delete(?WAITERS, Timestamp). - can_spawn(#state{hard_limit = HardLimit, counts = Counts}, Lang) -> case dict:find(Lang, Counts) of {ok, Count} -> Count < HardLimit; error -> true end. - get_proc_config() -> Limit = config:get_boolean("query_server_config", "reduce_limit", true), Timeout = config:get_integer("couchdb", "os_process_timeout", 5000), @@ -591,11 +568,9 @@ get_proc_config() -> {<<"timeout">>, Timeout} ]}. - get_hard_limit() -> LimStr = config:get("query_server_config", "os_process_limit", "100"), list_to_integer(LimStr). - get_soft_limit() -> config:get_integer("query_server_config", "os_process_soft_limit", 100). diff --git a/src/couch/src/couch_query_servers.erl b/src/couch/src/couch_query_servers.erl index 10b8048dd..5dd7c4a4b 100644 --- a/src/couch/src/couch_query_servers.erl +++ b/src/couch/src/couch_query_servers.erl @@ -14,7 +14,7 @@ -export([try_compile/4]). -export([start_doc_map/3, map_doc_raw/2, stop_doc_map/1, raw_to_ejson/1]). --export([reduce/3, rereduce/3,validate_doc_update/5]). +-export([reduce/3, rereduce/3, validate_doc_update/5]). -export([filter_docs/5]). -export([filter_view/3]). -export([finalize/2]). @@ -27,14 +27,17 @@ -include_lib("couch/include/couch_db.hrl"). --define(SUMERROR, <<"The _sum function requires that map values be numbers, " +-define(SUMERROR, << + "The _sum function requires that map values be numbers, " "arrays of numbers, or objects. Objects cannot be mixed with other " "data structures. Objects can be arbitrarily nested, provided that the values " - "for all fields are themselves numbers, arrays of numbers, or objects.">>). - --define(STATERROR, <<"The _stats function requires that map values be numbers " - "or arrays of numbers, not '~p'">>). + "for all fields are themselves numbers, arrays of numbers, or objects." +>>). +-define(STATERROR, << + "The _stats function requires that map values be numbers " + "or arrays of numbers, not '~p'" +>>). try_compile(Proc, FunctionType, FunctionName, FunctionSource) -> try @@ -54,20 +57,21 @@ try_compile(Proc, FunctionType, FunctionName, FunctionSource) -> start_doc_map(Lang, Functions, Lib) -> Proc = get_os_process(Lang), case Lib of - {[]} -> ok; - Lib -> - true = proc_prompt(Proc, [<<"add_lib">>, Lib]) + {[]} -> ok; + Lib -> true = proc_prompt(Proc, [<<"add_lib">>, Lib]) end, - lists:foreach(fun(FunctionSource) -> - true = proc_prompt(Proc, [<<"add_fun">>, FunctionSource]) - end, Functions), + lists:foreach( + fun(FunctionSource) -> + true = proc_prompt(Proc, [<<"add_fun">>, FunctionSource]) + end, + Functions + ), {ok, Proc}. map_doc_raw(Proc, Doc) -> Json = couch_doc:to_json_obj(Doc, []), {ok, proc_prompt_raw(Proc, [<<"map_doc">>, Json])}. - stop_doc_map(nil) -> ok; stop_doc_map(Proc) -> @@ -77,20 +81,24 @@ group_reductions_results([]) -> []; group_reductions_results(List) -> {Heads, Tails} = lists:foldl( - fun([H|T], {HAcc,TAcc}) -> - {[H|HAcc], [T|TAcc]} - end, {[], []}, List), + fun([H | T], {HAcc, TAcc}) -> + {[H | HAcc], [T | TAcc]} + end, + {[], []}, + List + ), case Tails of - [[]|_] -> % no tails left - [Heads]; - _ -> - [Heads | group_reductions_results(Tails)] + % no tails left + [[] | _] -> + [Heads]; + _ -> + [Heads | group_reductions_results(Tails)] end. -finalize(<<"_approx_count_distinct",_/binary>>, Reduction) -> +finalize(<<"_approx_count_distinct", _/binary>>, Reduction) -> true = hyper:is_hyper(Reduction), {ok, round(hyper:card(Reduction))}; -finalize(<<"_stats",_/binary>>, Unpacked) -> +finalize(<<"_stats", _/binary>>, Unpacked) -> {ok, pack_stats(Unpacked)}; finalize(_RedSrc, Reduction) -> {ok, Reduction}. @@ -101,45 +109,51 @@ rereduce(Lang, RedSrcs, ReducedValues) -> Grouped = group_reductions_results(ReducedValues), Results = lists:zipwith( fun - (<<"_", _/binary>> = FunSrc, Values) -> - {ok, [Result]} = builtin_reduce(rereduce, [FunSrc], [[[], V] || V <- Values], []), - Result; - (FunSrc, Values) -> - os_rereduce(Lang, [FunSrc], Values) - end, RedSrcs, Grouped), + (<<"_", _/binary>> = FunSrc, Values) -> + {ok, [Result]} = builtin_reduce(rereduce, [FunSrc], [[[], V] || V <- Values], []), + Result; + (FunSrc, Values) -> + os_rereduce(Lang, [FunSrc], Values) + end, + RedSrcs, + Grouped + ), {ok, Results}. reduce(_Lang, [], _KVs) -> {ok, []}; reduce(Lang, RedSrcs, KVs) -> - {OsRedSrcs, BuiltinReds} = lists:partition(fun - (<<"_", _/binary>>) -> false; - (_OsFun) -> true - end, RedSrcs), + {OsRedSrcs, BuiltinReds} = lists:partition( + fun + (<<"_", _/binary>>) -> false; + (_OsFun) -> true + end, + RedSrcs + ), {ok, OsResults} = os_reduce(Lang, OsRedSrcs, KVs), {ok, BuiltinResults} = builtin_reduce(reduce, BuiltinReds, KVs, []), recombine_reduce_results(RedSrcs, OsResults, BuiltinResults, []). - recombine_reduce_results([], [], [], Acc) -> {ok, lists:reverse(Acc)}; -recombine_reduce_results([<<"_", _/binary>>|RedSrcs], OsResults, [BRes|BuiltinResults], Acc) -> - recombine_reduce_results(RedSrcs, OsResults, BuiltinResults, [BRes|Acc]); -recombine_reduce_results([_OsFun|RedSrcs], [OsR|OsResults], BuiltinResults, Acc) -> - recombine_reduce_results(RedSrcs, OsResults, BuiltinResults, [OsR|Acc]). +recombine_reduce_results([<<"_", _/binary>> | RedSrcs], OsResults, [BRes | BuiltinResults], Acc) -> + recombine_reduce_results(RedSrcs, OsResults, BuiltinResults, [BRes | Acc]); +recombine_reduce_results([_OsFun | RedSrcs], [OsR | OsResults], BuiltinResults, Acc) -> + recombine_reduce_results(RedSrcs, OsResults, BuiltinResults, [OsR | Acc]). os_reduce(_Lang, [], _KVs) -> {ok, []}; os_reduce(Lang, OsRedSrcs, KVs) -> Proc = get_os_process(Lang), - OsResults = try proc_prompt(Proc, [<<"reduce">>, OsRedSrcs, KVs]) of - [true, Reductions] -> Reductions - catch - throw:{reduce_overflow_error, Msg} -> - [{[{reduce_overflow_error, Msg}]} || _ <- OsRedSrcs] - after - ok = ret_os_process(Proc) - end, + OsResults = + try proc_prompt(Proc, [<<"reduce">>, OsRedSrcs, KVs]) of + [true, Reductions] -> Reductions + catch + throw:{reduce_overflow_error, Msg} -> + [{[{reduce_overflow_error, Msg}]} || _ <- OsRedSrcs] + after + ok = ret_os_process(Proc) + end, {ok, OsResults}. os_rereduce(Lang, OsRedSrcs, KVs) -> @@ -158,7 +172,6 @@ os_rereduce(Lang, OsRedSrcs, KVs) -> Error end. - get_overflow_error([]) -> undefined; get_overflow_error([{[{reduce_overflow_error, _}]} = Error | _]) -> @@ -166,26 +179,24 @@ get_overflow_error([{[{reduce_overflow_error, _}]} = Error | _]) -> get_overflow_error([_ | Rest]) -> get_overflow_error(Rest). - builtin_reduce(_Re, [], _KVs, Acc) -> {ok, lists:reverse(Acc)}; -builtin_reduce(Re, [<<"_sum",_/binary>>|BuiltinReds], KVs, Acc) -> +builtin_reduce(Re, [<<"_sum", _/binary>> | BuiltinReds], KVs, Acc) -> Sum = builtin_sum_rows(KVs, 0), Red = check_sum_overflow(?term_size(KVs), ?term_size(Sum), Sum), - builtin_reduce(Re, BuiltinReds, KVs, [Red|Acc]); -builtin_reduce(reduce, [<<"_count",_/binary>>|BuiltinReds], KVs, Acc) -> + builtin_reduce(Re, BuiltinReds, KVs, [Red | Acc]); +builtin_reduce(reduce, [<<"_count", _/binary>> | BuiltinReds], KVs, Acc) -> Count = length(KVs), - builtin_reduce(reduce, BuiltinReds, KVs, [Count|Acc]); -builtin_reduce(rereduce, [<<"_count",_/binary>>|BuiltinReds], KVs, Acc) -> + builtin_reduce(reduce, BuiltinReds, KVs, [Count | Acc]); +builtin_reduce(rereduce, [<<"_count", _/binary>> | BuiltinReds], KVs, Acc) -> Count = builtin_sum_rows(KVs, 0), - builtin_reduce(rereduce, BuiltinReds, KVs, [Count|Acc]); -builtin_reduce(Re, [<<"_stats",_/binary>>|BuiltinReds], KVs, Acc) -> + builtin_reduce(rereduce, BuiltinReds, KVs, [Count | Acc]); +builtin_reduce(Re, [<<"_stats", _/binary>> | BuiltinReds], KVs, Acc) -> Stats = builtin_stats(Re, KVs), - builtin_reduce(Re, BuiltinReds, KVs, [Stats|Acc]); -builtin_reduce(Re, [<<"_approx_count_distinct",_/binary>>|BuiltinReds], KVs, Acc) -> + builtin_reduce(Re, BuiltinReds, KVs, [Stats | Acc]); +builtin_reduce(Re, [<<"_approx_count_distinct", _/binary>> | BuiltinReds], KVs, Acc) -> Distinct = approx_count_distinct(Re, KVs), - builtin_reduce(Re, BuiltinReds, KVs, [Distinct|Acc]). - + builtin_reduce(Re, BuiltinReds, KVs, [Distinct | Acc]). builtin_sum_rows([], Acc) -> Acc; @@ -197,11 +208,13 @@ builtin_sum_rows([[_Key, Value] | RestKVs], Acc) -> throw:{builtin_reduce_error, Obj} -> Obj; throw:{invalid_value, Reason, Cause} -> - {[{<<"error">>, <<"builtin_reduce_error">>}, - {<<"reason">>, Reason}, {<<"caused_by">>, Cause}]} + {[ + {<<"error">>, <<"builtin_reduce_error">>}, + {<<"reason">>, Reason}, + {<<"caused_by">>, Cause} + ]} end. - sum_values(Value, Acc) when is_number(Value), is_number(Acc) -> Acc + Value; sum_values(Value, Acc) when is_list(Value), is_list(Acc) -> @@ -239,12 +252,12 @@ sum_objects(Rest, []) -> sum_arrays([], []) -> []; -sum_arrays([_|_]=Xs, []) -> +sum_arrays([_ | _] = Xs, []) -> Xs; -sum_arrays([], [_|_]=Ys) -> +sum_arrays([], [_ | _] = Ys) -> Ys; -sum_arrays([X|Xs], [Y|Ys]) when is_number(X), is_number(Y) -> - [X+Y | sum_arrays(Xs,Ys)]; +sum_arrays([X | Xs], [Y | Ys]) when is_number(X), is_number(Y) -> + [X + Y | sum_arrays(Xs, Ys)]; sum_arrays(Else, _) -> throw_sum_error(Else). @@ -265,37 +278,42 @@ check_sum_overflow(InSize, OutSize, Sum) -> end. log_sum_overflow(InSize, OutSize) -> - Fmt = "Reduce output must shrink more rapidly: " - "input size: ~b " - "output size: ~b", + Fmt = + "Reduce output must shrink more rapidly: " + "input size: ~b " + "output size: ~b", Msg = iolist_to_binary(io_lib:format(Fmt, [InSize, OutSize])), couch_log:error(Msg, []), Msg. builtin_stats(_, []) -> {0, 0, 0, 0, 0}; -builtin_stats(_, [[_,First]|Rest]) -> - lists:foldl(fun([_Key, Value], Acc) -> - stat_values(Value, Acc) - end, build_initial_accumulator(First), Rest). +builtin_stats(_, [[_, First] | Rest]) -> + lists:foldl( + fun([_Key, Value], Acc) -> + stat_values(Value, Acc) + end, + build_initial_accumulator(First), + Rest + ). stat_values(Value, Acc) when is_list(Value), is_list(Acc) -> lists:zipwith(fun stat_values/2, Value, Acc); stat_values({PreRed}, Acc) when is_list(PreRed) -> stat_values(unpack_stats({PreRed}), Acc); stat_values(Value, Acc) when is_number(Value) -> - stat_values({Value, 1, Value, Value, Value*Value}, Acc); + stat_values({Value, 1, Value, Value, Value * Value}, Acc); stat_values(Value, Acc) when is_number(Acc) -> - stat_values(Value, {Acc, 1, Acc, Acc, Acc*Acc}); + stat_values(Value, {Acc, 1, Acc, Acc, Acc * Acc}); stat_values(Value, Acc) when is_tuple(Value), is_tuple(Acc) -> {Sum0, Cnt0, Min0, Max0, Sqr0} = Value, {Sum1, Cnt1, Min1, Max1, Sqr1} = Acc, { - Sum0 + Sum1, - Cnt0 + Cnt1, - erlang:min(Min0, Min1), - erlang:max(Max0, Max1), - Sqr0 + Sqr1 + Sum0 + Sum1, + Cnt0 + Cnt1, + erlang:min(Min0, Min1), + erlang:max(Max0, Max1), + Sqr0 + Sqr1 }; stat_values(Else, _Acc) -> throw_stat_error(Else). @@ -303,7 +321,7 @@ stat_values(Else, _Acc) -> build_initial_accumulator(L) when is_list(L) -> [build_initial_accumulator(X) || X <- L]; build_initial_accumulator(X) when is_number(X) -> - {X, 1, X, X, X*X}; + {X, 1, X, X, X * X}; build_initial_accumulator({_, _, _, _, _} = AlreadyUnpacked) -> AlreadyUnpacked; build_initial_accumulator({Props}) -> @@ -314,16 +332,21 @@ build_initial_accumulator(Else) -> unpack_stats({PreRed}) when is_list(PreRed) -> { - get_number(<<"sum">>, PreRed), - get_number(<<"count">>, PreRed), - get_number(<<"min">>, PreRed), - get_number(<<"max">>, PreRed), - get_number(<<"sumsqr">>, PreRed) + get_number(<<"sum">>, PreRed), + get_number(<<"count">>, PreRed), + get_number(<<"min">>, PreRed), + get_number(<<"max">>, PreRed), + get_number(<<"sumsqr">>, PreRed) }. - pack_stats({Sum, Cnt, Min, Max, Sqr}) -> - {[{<<"sum">>,Sum}, {<<"count">>,Cnt}, {<<"min">>,Min}, {<<"max">>,Max}, {<<"sumsqr">>,Sqr}]}; + {[ + {<<"sum">>, Sum}, + {<<"count">>, Cnt}, + {<<"min">>, Min}, + {<<"max">>, Max}, + {<<"sumsqr">>, Sqr} + ]}; pack_stats({Packed}) -> % Legacy code path before we had the finalize operation {Packed}; @@ -332,35 +355,43 @@ pack_stats(Stats) when is_list(Stats) -> get_number(Key, Props) -> case couch_util:get_value(Key, Props) of - X when is_number(X) -> - X; - undefined when is_binary(Key) -> - get_number(binary_to_atom(Key, latin1), Props); - undefined -> - Msg = io_lib:format("user _stats input missing required field ~s (~p)", - [Key, Props]), - throw({invalid_value, iolist_to_binary(Msg)}); - Else -> - Msg = io_lib:format("non-numeric _stats input received for ~s: ~w", - [Key, Else]), - throw({invalid_value, iolist_to_binary(Msg)}) + X when is_number(X) -> + X; + undefined when is_binary(Key) -> + get_number(binary_to_atom(Key, latin1), Props); + undefined -> + Msg = io_lib:format( + "user _stats input missing required field ~s (~p)", + [Key, Props] + ), + throw({invalid_value, iolist_to_binary(Msg)}); + Else -> + Msg = io_lib:format( + "non-numeric _stats input received for ~s: ~w", + [Key, Else] + ), + throw({invalid_value, iolist_to_binary(Msg)}) end. % TODO allow customization of precision in the ddoc. approx_count_distinct(reduce, KVs) -> - lists:foldl(fun([[Key, _Id], _Value], Filter) -> - hyper:insert(term_to_binary(Key), Filter) - end, hyper:new(11), KVs); + lists:foldl( + fun([[Key, _Id], _Value], Filter) -> + hyper:insert(term_to_binary(Key), Filter) + end, + hyper:new(11), + KVs + ); approx_count_distinct(rereduce, Reds) -> hyper:union([Filter || [_, Filter] <- Reds]). % use the function stored in ddoc.validate_doc_update to test an update. -spec validate_doc_update(DDoc, EditDoc, DiskDoc, Ctx, SecObj) -> ok when - DDoc :: ddoc(), + DDoc :: ddoc(), EditDoc :: doc(), DiskDoc :: doc() | nil, - Ctx :: user_ctx(), - SecObj :: sec_obj(). + Ctx :: user_ctx(), + SecObj :: sec_obj(). validate_doc_update(DDoc, EditDoc, DiskDoc, Ctx, SecObj) -> JsonEditDoc = couch_doc:to_json_obj(EditDoc, [revs]), @@ -370,8 +401,9 @@ validate_doc_update(DDoc, EditDoc, DiskDoc, Ctx, SecObj) -> [<<"validate_doc_update">>], [JsonEditDoc, JsonDiskDoc, Ctx, SecObj] ), - if Resp == 1 -> ok; true -> - couch_stats:increment_counter([couchdb, query_server, vdu_rejects], 1) + if + Resp == 1 -> ok; + true -> couch_stats:increment_counter([couchdb, query_server, vdu_rejects], 1) end, case Resp of RespCode when RespCode =:= 1; RespCode =:= ok; RespCode =:= true -> @@ -386,11 +418,15 @@ validate_doc_update(DDoc, EditDoc, DiskDoc, Ctx, SecObj) -> throw({unknown_error, Message}) end. - rewrite(Req, Db, DDoc) -> - Fields = [F || F <- chttpd_external:json_req_obj_fields(), - F =/= <<"info">>, F =/= <<"form">>, - F =/= <<"uuid">>, F =/= <<"id">>], + Fields = [ + F + || F <- chttpd_external:json_req_obj_fields(), + F =/= <<"info">>, + F =/= <<"form">>, + F =/= <<"uuid">>, + F =/= <<"id">> + ], JsonReq = chttpd_external:json_req_obj(Req, Db, null, Fields), case couch_query_servers:ddoc_prompt(DDoc, [<<"rewrites">>], [JsonReq]) of {[{<<"forbidden">>, Message}]} -> @@ -399,10 +435,10 @@ rewrite(Req, Db, DDoc) -> throw({unauthorized, Message}); [<<"no_dispatch_rule">>] -> undefined; - [<<"ok">>, {V}=Rewrite] when is_list(V) -> + [<<"ok">>, {V} = Rewrite] when is_list(V) -> ok = validate_rewrite_response(Rewrite), Rewrite; - [<<"ok">>, _] -> + [<<"ok">>, _] -> throw_rewrite_error(<<"bad rewrite">>); V -> couch_log:error("bad rewrite return ~p", [V]), @@ -430,15 +466,17 @@ validate_rewrite_response_field(<<"body">>, Body) when is_binary(Body) -> ok; validate_rewrite_response_field(<<"body">>, _) -> throw_rewrite_error(<<"bad body">>); -validate_rewrite_response_field(<<"headers">>, {Props}=Headers) when is_list(Props) -> +validate_rewrite_response_field(<<"headers">>, {Props} = Headers) when is_list(Props) -> validate_object_fields(Headers); validate_rewrite_response_field(<<"headers">>, _) -> throw_rewrite_error(<<"bad headers">>); -validate_rewrite_response_field(<<"query">>, {Props}=Query) when is_list(Props) -> +validate_rewrite_response_field(<<"query">>, {Props} = Query) when is_list(Props) -> validate_object_fields(Query); validate_rewrite_response_field(<<"query">>, _) -> throw_rewrite_error(<<"bad query">>); -validate_rewrite_response_field(<<"code">>, Code) when is_integer(Code) andalso Code >= 200 andalso Code < 600 -> +validate_rewrite_response_field(<<"code">>, Code) when + is_integer(Code) andalso Code >= 200 andalso Code < 600 +-> ok; validate_rewrite_response_field(<<"code">>, _) -> throw_rewrite_error(<<"bad code">>); @@ -447,24 +485,26 @@ validate_rewrite_response_field(K, V) -> ok. validate_object_fields({Props}) when is_list(Props) -> - lists:foreach(fun - ({Key, Value}) when is_binary(Key) andalso is_binary(Value) -> - ok; - ({Key, Value}) -> - Reason = io_lib:format( - "object key/value must be strings ~p=~p", [Key, Value]), - throw_rewrite_error(Reason); - (Value) -> - throw_rewrite_error(io_lib:format("bad value ~p", [Value])) - end, Props). - + lists:foreach( + fun + ({Key, Value}) when is_binary(Key) andalso is_binary(Value) -> + ok; + ({Key, Value}) -> + Reason = io_lib:format( + "object key/value must be strings ~p=~p", [Key, Value] + ), + throw_rewrite_error(Reason); + (Value) -> + throw_rewrite_error(io_lib:format("bad value ~p", [Value])) + end, + Props + ). -throw_rewrite_error(Reason) when is_list(Reason)-> +throw_rewrite_error(Reason) when is_list(Reason) -> throw_rewrite_error(iolist_to_binary(Reason)); throw_rewrite_error(Reason) when is_binary(Reason) -> throw({rewrite_error, Reason}). - json_doc_options() -> json_doc_options([]). @@ -487,18 +527,19 @@ filter_view(DDoc, VName, Docs) -> {ok, Passes}. filter_docs(Req, Db, DDoc, FName, Docs) -> - JsonReq = case Req of - {json_req, JsonObj} -> - JsonObj; - #httpd{} = HttpReq -> - chttpd_external:json_req_obj(HttpReq, Db) - end, + JsonReq = + case Req of + {json_req, JsonObj} -> + JsonObj; + #httpd{} = HttpReq -> + chttpd_external:json_req_obj(HttpReq, Db) + end, Options = json_doc_options(), JsonDocs = [json_doc(Doc, Options) || Doc <- Docs], try {ok, filter_docs_int(DDoc, FName, JsonReq, JsonDocs)} catch - throw:{os_process_error,{exit_status,1}} -> + throw:{os_process_error, {exit_status, 1}} -> %% batch used too much memory, retry sequentially. Fun = fun(JsonDoc) -> filter_docs_int(DDoc, FName, JsonReq, [JsonDoc]) @@ -507,8 +548,11 @@ filter_docs(Req, Db, DDoc, FName, Docs) -> end. filter_docs_int(DDoc, FName, JsonReq, JsonDocs) -> - [true, Passes] = ddoc_prompt(DDoc, [<<"filters">>, FName], - [JsonDocs, JsonReq]), + [true, Passes] = ddoc_prompt( + DDoc, + [<<"filters">>, FName], + [JsonDocs, JsonReq] + ), Passes. ddoc_proc_prompt({Proc, DDocId}, FunPath, Args) -> @@ -533,12 +577,12 @@ with_ddoc_proc(#doc{id=DDocId,revs={Start, [DiskRev|_]}}=DDoc, Fun) -> end. proc_prompt(Proc, Args) -> - case proc_prompt_raw(Proc, Args) of - {json, Json} -> - raw_to_ejson({json, Json}); - EJson -> - EJson - end. + case proc_prompt_raw(Proc, Args) of + {json, Json} -> + raw_to_ejson({json, Json}); + EJson -> + EJson + end. proc_prompt_raw(#proc{prompt_fun = {Mod, Func}} = Proc, Args) -> apply(Mod, Func, [Proc#proc.pid, Args]). @@ -546,13 +590,16 @@ proc_prompt_raw(#proc{prompt_fun = {Mod, Func}} = Proc, Args) -> raw_to_ejson({json, Json}) -> try ?JSON_DECODE(Json) - catch throw:{invalid_json, {_, invalid_string}} -> - Forced = try - force_utf8(Json) - catch _:_ -> - Json - end, - ?JSON_DECODE(Forced) + catch + throw:{invalid_json, {_, invalid_string}} -> + Forced = + try + force_utf8(Json) + catch + _:_ -> + Json + end, + ?JSON_DECODE(Forced) end; raw_to_ejson(EJson) -> EJson. @@ -561,14 +608,15 @@ force_utf8(Bin) -> case binary:match(Bin, <<"\\u">>) of {Start, 2} -> <<Prefix:Start/binary, Rest1/binary>> = Bin, - {Insert, Rest3} = case check_uescape(Rest1) of - {ok, Skip} -> - <<Skipped:Skip/binary, Rest2/binary>> = Rest1, - {Skipped, Rest2}; - {error, Skip} -> - <<_:Skip/binary, Rest2/binary>> = Rest1, - {<<16#EF, 16#BF, 16#BD>>, Rest2} - end, + {Insert, Rest3} = + case check_uescape(Rest1) of + {ok, Skip} -> + <<Skipped:Skip/binary, Rest2/binary>> = Rest1, + {Skipped, Rest2}; + {error, Skip} -> + <<_:Skip/binary, Rest2/binary>> = Rest1, + {<<16#EF, 16#BF, 16#BD>>, Rest2} + end, RestForced = force_utf8(Rest3), <<Prefix/binary, Insert/binary, RestForced/binary>>; nomatch -> @@ -588,8 +636,9 @@ check_uescape(Data) -> try [_] = xmerl_ucs:from_utf16be(UTF16), {ok, 12} - catch _:_ -> - {error, 6} + catch + _:_ -> + {error, 6} end; {_, _} -> % Found a uescape that's not a low half @@ -628,33 +677,33 @@ get_os_process_timeout() -> get_ddoc_process(#doc{} = DDoc, DDocKey) -> % remove this case statement case gen_server:call(couch_proc_manager, {get_proc, DDoc, DDocKey}, get_os_process_timeout()) of - {ok, Proc, {QueryConfig}} -> - % process knows the ddoc - case (catch proc_prompt(Proc, [<<"reset">>, {QueryConfig}])) of - true -> - proc_set_timeout(Proc, couch_util:get_value(<<"timeout">>, QueryConfig)), - Proc; - _ -> - catch proc_stop(Proc), - get_ddoc_process(DDoc, DDocKey) - end; - Error -> - throw(Error) + {ok, Proc, {QueryConfig}} -> + % process knows the ddoc + case (catch proc_prompt(Proc, [<<"reset">>, {QueryConfig}])) of + true -> + proc_set_timeout(Proc, couch_util:get_value(<<"timeout">>, QueryConfig)), + Proc; + _ -> + catch proc_stop(Proc), + get_ddoc_process(DDoc, DDocKey) + end; + Error -> + throw(Error) end. get_os_process(Lang) -> case gen_server:call(couch_proc_manager, {get_proc, Lang}, get_os_process_timeout()) of - {ok, Proc, {QueryConfig}} -> - case (catch proc_prompt(Proc, [<<"reset">>, {QueryConfig}])) of - true -> - proc_set_timeout(Proc, couch_util:get_value(<<"timeout">>, QueryConfig)), - Proc; - _ -> - catch proc_stop(Proc), - get_os_process(Lang) - end; - Error -> - throw(Error) + {ok, Proc, {QueryConfig}} -> + case (catch proc_prompt(Proc, [<<"reset">>, {QueryConfig}])) of + true -> + proc_set_timeout(Proc, couch_util:get_value(<<"timeout">>, QueryConfig)), + Proc; + _ -> + catch proc_stop(Proc), + get_os_process(Lang) + end; + Error -> + throw(Error) end. ret_os_process(Proc) -> @@ -668,7 +717,6 @@ throw_sum_error(Else) -> throw_stat_error(Else) -> throw({invalid_value, iolist_to_binary(io_lib:format(?STATERROR, [Else]))}). - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -680,19 +728,38 @@ builtin_sum_rows_negative_test() -> % it's only one document. ?assertEqual(A, builtin_sum_rows([["K", A]], [])), {Result} = builtin_sum_rows([["K", A]], [1, 2, 3]), - ?assertEqual({<<"error">>, <<"builtin_reduce_error">>}, - lists:keyfind(<<"error">>, 1, Result)). + ?assertEqual( + {<<"error">>, <<"builtin_reduce_error">>}, + lists:keyfind(<<"error">>, 1, Result) + ). sum_values_test() -> ?assertEqual(3, sum_values(1, 2)), - ?assertEqual([2,4,6], sum_values(1, [1,4,6])), - ?assertEqual([3,5,7], sum_values([3,2,4], [0,3,3])), - X = {[{<<"a">>,1}, {<<"b">>,[1,2]}, {<<"c">>, {[{<<"d">>,3}]}}, - {<<"g">>,1}]}, - Y = {[{<<"a">>,2}, {<<"b">>,3}, {<<"c">>, {[{<<"e">>, 5}]}}, - {<<"f">>,1}, {<<"g">>,1}]}, - Z = {[{<<"a">>,3}, {<<"b">>,[4,2]}, {<<"c">>, {[{<<"d">>,3},{<<"e">>,5}]}}, - {<<"f">>,1}, {<<"g">>,2}]}, + ?assertEqual([2, 4, 6], sum_values(1, [1, 4, 6])), + ?assertEqual([3, 5, 7], sum_values([3, 2, 4], [0, 3, 3])), + X = + {[ + {<<"a">>, 1}, + {<<"b">>, [1, 2]}, + {<<"c">>, {[{<<"d">>, 3}]}}, + {<<"g">>, 1} + ]}, + Y = + {[ + {<<"a">>, 2}, + {<<"b">>, 3}, + {<<"c">>, {[{<<"e">>, 5}]}}, + {<<"f">>, 1}, + {<<"g">>, 1} + ]}, + Z = + {[ + {<<"a">>, 3}, + {<<"b">>, [4, 2]}, + {<<"c">>, {[{<<"d">>, 3}, {<<"e">>, 5}]}}, + {<<"f">>, 1}, + {<<"g">>, 2} + ]}, ?assertEqual(Z, sum_values(X, Y)), ?assertEqual(Z, sum_values(Y, X)). @@ -701,8 +768,12 @@ sum_values_negative_test() -> A = [{[{<<"a">>, 1}]}, {[{<<"a">>, 2}]}, {[{<<"a">>, 3}]}], B = ["error 1", "error 2"], C = [<<"error 3">>, <<"error 4">>], - KV = {[{<<"error">>, <<"builtin_reduce_error">>}, - {<<"reason">>, ?SUMERROR}, {<<"caused_by">>, <<"some cause">>}]}, + KV = + {[ + {<<"error">>, <<"builtin_reduce_error">>}, + {<<"reason">>, ?SUMERROR}, + {<<"caused_by">>, <<"some cause">>} + ]}, ?assertThrow({invalid_value, _, _}, sum_values(A, [1, 2, 3])), ?assertThrow({invalid_value, _, _}, sum_values(A, 0)), ?assertThrow({invalid_value, _, _}, sum_values(B, [1, 2])), @@ -712,48 +783,103 @@ sum_values_negative_test() -> stat_values_test() -> ?assertEqual({1, 2, 0, 1, 1}, stat_values(1, 0)), ?assertEqual({11, 2, 1, 10, 101}, stat_values(1, 10)), - ?assertEqual([{9, 2, 2, 7, 53}, - {14, 2, 3, 11, 130}, - {18, 2, 5, 13, 194} - ], stat_values([2,3,5], [7,11,13])). + ?assertEqual( + [ + {9, 2, 2, 7, 53}, + {14, 2, 3, 11, 130}, + {18, 2, 5, 13, 194} + ], + stat_values([2, 3, 5], [7, 11, 13]) + ). reduce_stats_test() -> - ?assertEqual([ - {[{<<"sum">>,2},{<<"count">>,1},{<<"min">>,2},{<<"max">>,2},{<<"sumsqr">>,4}]} - ], test_reduce(<<"_stats">>, [[[null, key], 2]])), + ?assertEqual( + [ + {[{<<"sum">>, 2}, {<<"count">>, 1}, {<<"min">>, 2}, {<<"max">>, 2}, {<<"sumsqr">>, 4}]} + ], + test_reduce(<<"_stats">>, [[[null, key], 2]]) + ), - ?assertEqual([[ - {[{<<"sum">>,1},{<<"count">>,1},{<<"min">>,1},{<<"max">>,1},{<<"sumsqr">>,1}]}, - {[{<<"sum">>,2},{<<"count">>,1},{<<"min">>,2},{<<"max">>,2},{<<"sumsqr">>,4}]} - ]], test_reduce(<<"_stats">>, [[[null, key],[1,2]]])), + ?assertEqual( + [ + [ + {[ + {<<"sum">>, 1}, + {<<"count">>, 1}, + {<<"min">>, 1}, + {<<"max">>, 1}, + {<<"sumsqr">>, 1} + ]}, + {[ + {<<"sum">>, 2}, + {<<"count">>, 1}, + {<<"min">>, 2}, + {<<"max">>, 2}, + {<<"sumsqr">>, 4} + ]} + ] + ], + test_reduce(<<"_stats">>, [[[null, key], [1, 2]]]) + ), + + ?assertEqual( + {[{<<"sum">>, 2}, {<<"count">>, 1}, {<<"min">>, 2}, {<<"max">>, 2}, {<<"sumsqr">>, 4}]}, + element(2, finalize(<<"_stats">>, {2, 1, 2, 2, 4})) + ), ?assertEqual( - {[{<<"sum">>,2},{<<"count">>,1},{<<"min">>,2},{<<"max">>,2},{<<"sumsqr">>,4}]} - , element(2, finalize(<<"_stats">>, {2, 1, 2, 2, 4}))), - - ?assertEqual([ - {[{<<"sum">>,1},{<<"count">>,1},{<<"min">>,1},{<<"max">>,1},{<<"sumsqr">>,1}]}, - {[{<<"sum">>,2},{<<"count">>,1},{<<"min">>,2},{<<"max">>,2},{<<"sumsqr">>,4}]} - ], element(2, finalize(<<"_stats">>, [ - {1, 1, 1, 1, 1}, - {2, 1, 2, 2, 4} - ]))), - - ?assertEqual([ - {[{<<"sum">>,1},{<<"count">>,1},{<<"min">>,1},{<<"max">>,1},{<<"sumsqr">>,1}]}, - {[{<<"sum">>,2},{<<"count">>,1},{<<"min">>,2},{<<"max">>,2},{<<"sumsqr">>,4}]} - ], element(2, finalize(<<"_stats">>, [ - {1, 1, 1, 1, 1}, - {[{<<"sum">>,2},{<<"count">>,1},{<<"min">>,2},{<<"max">>,2},{<<"sumsqr">>,4}]} - ]))), - - ?assertEqual([ - {[{<<"sum">>,1},{<<"count">>,1},{<<"min">>,1},{<<"max">>,1},{<<"sumsqr">>,1}]}, - {[{<<"sum">>,2},{<<"count">>,1},{<<"min">>,2},{<<"max">>,2},{<<"sumsqr">>,4}]} - ], element(2, finalize(<<"_stats">>, [ - {[{<<"sum">>,1},{<<"count">>,1},{<<"min">>,1},{<<"max">>,1},{<<"sumsqr">>,1}]}, - {2, 1, 2, 2, 4} - ]))), + [ + {[{<<"sum">>, 1}, {<<"count">>, 1}, {<<"min">>, 1}, {<<"max">>, 1}, {<<"sumsqr">>, 1}]}, + {[{<<"sum">>, 2}, {<<"count">>, 1}, {<<"min">>, 2}, {<<"max">>, 2}, {<<"sumsqr">>, 4}]} + ], + element( + 2, + finalize(<<"_stats">>, [ + {1, 1, 1, 1, 1}, + {2, 1, 2, 2, 4} + ]) + ) + ), + + ?assertEqual( + [ + {[{<<"sum">>, 1}, {<<"count">>, 1}, {<<"min">>, 1}, {<<"max">>, 1}, {<<"sumsqr">>, 1}]}, + {[{<<"sum">>, 2}, {<<"count">>, 1}, {<<"min">>, 2}, {<<"max">>, 2}, {<<"sumsqr">>, 4}]} + ], + element( + 2, + finalize(<<"_stats">>, [ + {1, 1, 1, 1, 1}, + {[ + {<<"sum">>, 2}, + {<<"count">>, 1}, + {<<"min">>, 2}, + {<<"max">>, 2}, + {<<"sumsqr">>, 4} + ]} + ]) + ) + ), + + ?assertEqual( + [ + {[{<<"sum">>, 1}, {<<"count">>, 1}, {<<"min">>, 1}, {<<"max">>, 1}, {<<"sumsqr">>, 1}]}, + {[{<<"sum">>, 2}, {<<"count">>, 1}, {<<"min">>, 2}, {<<"max">>, 2}, {<<"sumsqr">>, 4}]} + ], + element( + 2, + finalize(<<"_stats">>, [ + {[ + {<<"sum">>, 1}, + {<<"count">>, 1}, + {<<"min">>, 1}, + {<<"max">>, 1}, + {<<"sumsqr">>, 1} + ]}, + {2, 1, 2, 2, 4} + ]) + ) + ), ok. test_reduce(Reducer, KVs) -> @@ -773,9 +899,12 @@ force_utf8_test() -> % Truncated but we doesn't break replacements <<"\\u0FA">> ], - lists:foreach(fun(Case) -> - ?assertEqual(Case, force_utf8(Case)) - end, Ok), + lists:foreach( + fun(Case) -> + ?assertEqual(Case, force_utf8(Case)) + end, + Ok + ), NotOk = [ <<"\\uDCA5">>, @@ -788,15 +917,18 @@ force_utf8_test() -> <<"\\uD83D\\u00A0">> ], ToJSON = fun(Bin) -> <<34, Bin/binary, 34>> end, - lists:foreach(fun(Case) -> - try - ?assertNotEqual(Case, force_utf8(Case)), - ?assertThrow(_, ?JSON_DECODE(ToJSON(Case))), - ?assertMatch(<<_/binary>>, ?JSON_DECODE(ToJSON(force_utf8(Case)))) - catch - T:R -> - io:format(standard_error, "~p~n~p~n", [T, R]) - end - end, NotOk). + lists:foreach( + fun(Case) -> + try + ?assertNotEqual(Case, force_utf8(Case)), + ?assertThrow(_, ?JSON_DECODE(ToJSON(Case))), + ?assertMatch(<<_/binary>>, ?JSON_DECODE(ToJSON(force_utf8(Case)))) + catch + T:R -> + io:format(standard_error, "~p~n~p~n", [T, R]) + end + end, + NotOk + ). -endif. diff --git a/src/couch/src/couch_rand.erl b/src/couch/src/couch_rand.erl index 67454b8ad..bc30956a4 100644 --- a/src/couch/src/couch_rand.erl +++ b/src/couch/src/couch_rand.erl @@ -12,16 +12,13 @@ -module(couch_rand). - -export([ uniform/0, uniform/1 ]). - uniform() -> rand:uniform(). - uniform(N) -> rand:uniform(N). diff --git a/src/couch/src/couch_secondary_sup.erl b/src/couch/src/couch_secondary_sup.erl index bb7821555..a328c170e 100644 --- a/src/couch/src/couch_secondary_sup.erl +++ b/src/couch/src/couch_secondary_sup.erl @@ -15,16 +15,12 @@ -export([init/1, start_link/0]). start_link() -> - supervisor:start_link({local,couch_secondary_services}, ?MODULE, []). + supervisor:start_link({local, couch_secondary_services}, ?MODULE, []). init([]) -> SecondarySupervisors = [ - {couch_plugin_event, - {gen_event, start_link, [{local, couch_plugin}]}, - permanent, - brutal_kill, - worker, - dynamic} + {couch_plugin_event, {gen_event, start_link, [{local, couch_plugin}]}, permanent, + brutal_kill, worker, dynamic} ], Daemons = [ {index_server, {couch_index_server, start_link, []}}, @@ -33,31 +29,34 @@ init([]) -> {uuids, {couch_uuids, start, []}} ], - MaybeHttp = case http_enabled() of - true -> [{httpd, {couch_httpd, start_link, []}}]; - false -> couch_httpd:set_auth_handlers(), [] - end, + MaybeHttp = + case http_enabled() of + true -> + [{httpd, {couch_httpd, start_link, []}}]; + false -> + couch_httpd:set_auth_handlers(), + [] + end, - MaybeHttps = case https_enabled() of - true -> [{httpsd, {chttpd, start_link, [https]}}]; - false -> [] - end, + MaybeHttps = + case https_enabled() of + true -> [{httpsd, {chttpd, start_link, [https]}}]; + false -> [] + end, - Children = SecondarySupervisors ++ [ - begin - {Module, Fun, Args} = Spec, + Children = + SecondarySupervisors ++ + [ + begin + {Module, Fun, Args} = Spec, - {Name, - {Module, Fun, Args}, - permanent, - brutal_kill, - worker, - [Module]} - end - || {Name, Spec} - <- Daemons ++ MaybeHttp ++ MaybeHttps, Spec /= ""], - {ok, {{one_for_one, 50, 3600}, - couch_epi:register_service(couch_db_epi, Children)}}. + {Name, {Module, Fun, Args}, permanent, brutal_kill, worker, [Module]} + end + || {Name, Spec} <- + Daemons ++ MaybeHttp ++ MaybeHttps, + Spec /= "" + ], + {ok, {{one_for_one, 50, 3600}, couch_epi:register_service(couch_db_epi, Children)}}. http_enabled() -> config:get_boolean("httpd", "enable", false). diff --git a/src/couch/src/couch_server.erl b/src/couch/src/couch_server.erl index 3c72e3357..06be86786 100644 --- a/src/couch/src/couch_server.erl +++ b/src/couch/src/couch_server.erl @@ -15,11 +15,11 @@ -behaviour(config_listener). -vsn(3). --export([open/2,create/2,delete/2,get_version/0,get_version/1,get_git_sha/0,get_uuid/0]). +-export([open/2, create/2, delete/2, get_version/0, get_version/1, get_git_sha/0, get_uuid/0]). -export([all_databases/0, all_databases/2]). --export([init/1, handle_call/3,sup_start_link/1]). --export([handle_cast/2,code_change/3,handle_info/2,terminate/2]). --export([dev_start/0,is_admin/2,has_admins/0,get_stats/0]). +-export([init/1, handle_call/3, sup_start_link/1]). +-export([handle_cast/2, code_change/3, handle_info/2, terminate/2]). +-export([dev_start/0, is_admin/2, has_admins/0, get_stats/0]). -export([close_db_if_idle/1]). -export([delete_compaction_files/1]). -export([exists/1]). @@ -28,7 +28,7 @@ -export([lock/2, unlock/1]). -export([db_updated/1]). -export([num_servers/0, couch_server/1, couch_dbs_pid_to_name/1, couch_dbs/1]). --export([aggregate_queue_len/0,get_spidermonkey_version/0]). +-export([aggregate_queue_len/0, get_spidermonkey_version/0]). % config_listener api -export([handle_config_change/5, handle_config_terminate/3]). @@ -40,18 +40,18 @@ -define(RELISTEN_DELAY, 5000). -define(DEFAULT_ENGINE, "couch"). --record(server,{ +-record(server, { root_dir = [], engines = [], - max_dbs_open=?MAX_DBS_OPEN, - dbs_open=0, - start_time="", - update_lru_on_read=true, + max_dbs_open = ?MAX_DBS_OPEN, + dbs_open = 0, + start_time = "", + update_lru_on_read = true, lru = couch_lru:new(), couch_dbs, couch_dbs_pid_to_name, couch_dbs_locks - }). +}). dev_start() -> couch:stop(), @@ -59,11 +59,12 @@ dev_start() -> couch:start(). get_version() -> - ?COUCHDB_VERSION. %% Defined in rebar.config.script + %% Defined in rebar.config.script + ?COUCHDB_VERSION. get_version(short) -> - %% strip git hash from version string - [Version|_Rest] = string:tokens(get_version(), "+"), - Version. + %% strip git hash from version string + [Version | _Rest] = string:tokens(get_version(), "+"), + Version. get_git_sha() -> ?COUCHDB_GIT_SHA. @@ -73,16 +74,18 @@ get_uuid() -> UUID = couch_uuids:random(), config:set("couchdb", "uuid", ?b2l(UUID)), UUID; - UUID -> ?l2b(UUID) + UUID -> + ?l2b(UUID) end. get_stats() -> Fun = fun(N, {TimeAcc, OpenAcc}) -> - {ok, #server{start_time=Time,dbs_open=Open}} = + {ok, #server{start_time = Time, dbs_open = Open}} = gen_server:call(couch_server(N), get_server), - {max(Time, TimeAcc), Open + OpenAcc} end, + {max(Time, TimeAcc), Open + OpenAcc} + end, {Time, Open} = - lists:foldl(Fun, {0, 0}, lists:seq(1, num_servers())), + lists:foldl(Fun, {0, 0}, lists:seq(1, num_servers())), [{start_time, ?l2b(Time)}, {dbs_open, Open}]. get_spidermonkey_version() -> list_to_binary(?COUCHDB_SPIDERMONKEY_VERSION). @@ -94,31 +97,32 @@ open(DbName, Options) -> try validate_open_or_create(DbName, Options), open_int(DbName, Options) - catch throw:{?MODULE, Error} -> - Error + catch + throw:{?MODULE, Error} -> + Error end. open_int(DbName, Options0) -> Ctx = couch_util:get_value(user_ctx, Options0, #user_ctx{}), case ets:lookup(couch_dbs(DbName), DbName) of - [#entry{db = Db0, lock = Lock} = Entry] when Lock =/= locked -> - update_lru(DbName, Entry#entry.db_options), - {ok, Db1} = couch_db:incref(Db0), - couch_db:set_user_ctx(Db1, Ctx); - _ -> - Options = maybe_add_sys_db_callbacks(DbName, Options0), - Timeout = couch_util:get_value(timeout, Options, infinity), - Create = couch_util:get_value(create_if_missing, Options, false), - case gen_server:call(couch_server(DbName), {open, DbName, Options}, Timeout) of - {ok, Db0} -> + [#entry{db = Db0, lock = Lock} = Entry] when Lock =/= locked -> + update_lru(DbName, Entry#entry.db_options), {ok, Db1} = couch_db:incref(Db0), couch_db:set_user_ctx(Db1, Ctx); - {not_found, no_db_file} when Create -> - couch_log:warning("creating missing database: ~s", [DbName]), - couch_server:create(DbName, Options); - Error -> - Error - end + _ -> + Options = maybe_add_sys_db_callbacks(DbName, Options0), + Timeout = couch_util:get_value(timeout, Options, infinity), + Create = couch_util:get_value(create_if_missing, Options, false), + case gen_server:call(couch_server(DbName), {open, DbName, Options}, Timeout) of + {ok, Db0} -> + {ok, Db1} = couch_db:incref(Db0), + couch_db:set_user_ctx(Db1, Ctx); + {not_found, no_db_file} when Create -> + couch_log:warning("creating missing database: ~s", [DbName]), + couch_server:create(DbName, Options); + Error -> + Error + end end. update_lru(DbName, Options) -> @@ -132,47 +136,48 @@ update_lru(DbName, Options) -> ok end. - create(DbName, Options) -> try validate_open_or_create(DbName, Options), create_int(DbName, Options) - catch throw:{?MODULE, Error} -> - Error + catch + throw:{?MODULE, Error} -> + Error end. create_int(DbName, Options0) -> Options = maybe_add_sys_db_callbacks(DbName, Options0), couch_partition:validate_dbname(DbName, Options), case gen_server:call(couch_server(DbName), {create, DbName, Options}, infinity) of - {ok, Db0} -> - Ctx = couch_util:get_value(user_ctx, Options, #user_ctx{}), - {ok, Db1} = couch_db:incref(Db0), - couch_db:set_user_ctx(Db1, Ctx); - Error -> - Error + {ok, Db0} -> + Ctx = couch_util:get_value(user_ctx, Options, #user_ctx{}), + {ok, Db1} = couch_db:incref(Db0), + couch_db:set_user_ctx(Db1, Ctx); + Error -> + Error end. delete(DbName, Options) -> gen_server:call(couch_server(DbName), {delete, DbName, Options}, infinity). - exists(DbName) -> RootDir = config:get("couchdb", "database_dir", "."), Engines = get_configured_engines(), Possible = get_possible_engines(DbName, RootDir, Engines), Possible /= []. - delete_compaction_files(DbName) -> delete_compaction_files(DbName, []). delete_compaction_files(DbName, DelOpts) when is_list(DbName) -> RootDir = config:get("couchdb", "database_dir", "."), - lists:foreach(fun({Ext, Engine}) -> - FPath = make_filepath(RootDir, DbName, Ext), - couch_db_engine:delete_compaction_files(Engine, RootDir, FPath, DelOpts) - end, get_configured_engines()), + lists:foreach( + fun({Ext, Engine}) -> + FPath = make_filepath(RootDir, DbName, Ext), + couch_db_engine:delete_compaction_files(Engine, RootDir, FPath, DelOpts) + end, + get_configured_engines() + ), ok; delete_compaction_files(DbName, DelOpts) when is_binary(DbName) -> delete_compaction_files(?b2l(DbName), DelOpts). @@ -185,22 +190,32 @@ maybe_add_sys_db_callbacks(DbName, Options) -> IsReplicatorDb = path_ends_with(DbName, "_replicator"), UsersDbSuffix = config:get("couchdb", "users_db_suffix", "_users"), - IsUsersDb = path_ends_with(DbName, "_users") - orelse path_ends_with(DbName, UsersDbSuffix), + IsUsersDb = + path_ends_with(DbName, "_users") orelse + path_ends_with(DbName, UsersDbSuffix), if DbName == DbsDbName -> - [{before_doc_update, fun mem3_bdu:before_doc_update/3}, - sys_db | Options]; + [ + {before_doc_update, fun mem3_bdu:before_doc_update/3}, + sys_db + | Options + ]; DbName == NodesDbName -> [sys_db | Options]; IsReplicatorDb -> - [{before_doc_update, fun couch_replicator_docs:before_doc_update/3}, - {after_doc_read, fun couch_replicator_docs:after_doc_read/2}, - sys_db | Options]; + [ + {before_doc_update, fun couch_replicator_docs:before_doc_update/3}, + {after_doc_read, fun couch_replicator_docs:after_doc_read/2}, + sys_db + | Options + ]; IsUsersDb -> - [{before_doc_update, fun couch_users_db:before_doc_update/3}, - {after_doc_read, fun couch_users_db:after_doc_read/2}, - sys_db | Options]; + [ + {before_doc_update, fun couch_users_db:before_doc_update/3}, + {after_doc_read, fun couch_users_db:after_doc_read/2}, + sys_db + | Options + ]; true -> Options end. @@ -215,11 +230,11 @@ check_dbname(DbName) -> is_admin(User, ClearPwd) -> case config:get("admins", User) of - "-hashed-" ++ HashedPwdAndSalt -> - [HashedPwd, Salt] = string:tokens(HashedPwdAndSalt, ","), - couch_util:to_hex(crypto:hash(sha, ClearPwd ++ Salt)) == HashedPwd; - _Else -> - false + "-hashed-" ++ HashedPwdAndSalt -> + [HashedPwd, Salt] = string:tokens(HashedPwdAndSalt, ","), + couch_util:to_hex(crypto:hash(sha, ClearPwd ++ Salt)) == HashedPwd; + _Else -> + false end. has_admins() -> @@ -233,7 +248,9 @@ hash_admin_passwords(Persist) -> fun({User, ClearPassword}) -> HashedPassword = couch_passwords:hash_admin_password(ClearPassword), config:set("admins", User, ?b2l(HashedPassword), Persist) - end, couch_passwords:get_unhashed_admins()). + end, + couch_passwords:get_unhashed_admins() + ). close_db_if_idle(DbName) -> case ets:lookup(couch_dbs(DbName), DbName) of @@ -243,7 +260,6 @@ close_db_if_idle(DbName) -> ok end. - init([N]) -> couch_util:set_mqd_off_heap(?MODULE), couch_util:set_process_priority(?MODULE, high), @@ -260,7 +276,8 @@ init([N]) -> % Mark if fips is enabled case erlang:function_exported(crypto, info_fips, 0) andalso - crypto:info_fips() == enabled of + crypto:info_fips() == enabled + of true -> config:enable_feature('fips'); false -> @@ -276,7 +293,8 @@ init([N]) -> Engines = get_configured_engines(), MaxDbsOpen = config:get_integer("couchdb", "max_dbs_open", ?MAX_DBS_OPEN), UpdateLruOnRead = config:get_boolean( - "couchdb", "update_lru_on_read", false), + "couchdb", "update_lru_on_read", false + ), ok = config:listen_for_changes(?MODULE, N), ok = couch_file:init_delete_dir(RootDir), hash_admin_passwords(), @@ -295,44 +313,55 @@ init([N]) -> {read_concurrency, true} ]), process_flag(trap_exit, true), - {ok, #server{root_dir=RootDir, - engines = Engines, - max_dbs_open=per_couch_server(MaxDbsOpen), - update_lru_on_read=UpdateLruOnRead, - start_time=couch_util:rfc1123_date(), - couch_dbs=couch_dbs(N), - couch_dbs_pid_to_name=couch_dbs_pid_to_name(N), - couch_dbs_locks=couch_dbs_locks(N)}}. + {ok, #server{ + root_dir = RootDir, + engines = Engines, + max_dbs_open = per_couch_server(MaxDbsOpen), + update_lru_on_read = UpdateLruOnRead, + start_time = couch_util:rfc1123_date(), + couch_dbs = couch_dbs(N), + couch_dbs_pid_to_name = couch_dbs_pid_to_name(N), + couch_dbs_locks = couch_dbs_locks(N) + }}. terminate(Reason, Srv) -> - couch_log:error("couch_server terminating with ~p, state ~2048p", - [Reason, - Srv#server{lru = redacted}]), - ets:foldl(fun(#entry{db = Db}, _) -> - % Filter out any entry records for open_async - % processes that haven't finished. - if Db == undefined -> ok; true -> - couch_util:shutdown_sync(couch_db:get_pid(Db)) - end - end, nil, couch_dbs(Srv)), + couch_log:error( + "couch_server terminating with ~p, state ~2048p", + [ + Reason, + Srv#server{lru = redacted} + ] + ), + ets:foldl( + fun(#entry{db = Db}, _) -> + % Filter out any entry records for open_async + % processes that haven't finished. + if + Db == undefined -> ok; + true -> couch_util:shutdown_sync(couch_db:get_pid(Db)) + end + end, + nil, + couch_dbs(Srv) + ), ok. handle_config_change("couchdb", "database_dir", _, _, _) -> exit(whereis(couch_server), config_change), remove_handler; handle_config_change("couchdb", "update_lru_on_read", "true", _, N) -> - gen_server:call(couch_server(N),{set_update_lru_on_read,true}), + gen_server:call(couch_server(N), {set_update_lru_on_read, true}), {ok, N}; handle_config_change("couchdb", "update_lru_on_read", _, _, N) -> - gen_server:call(couch_server(N),{set_update_lru_on_read,false}), + gen_server:call(couch_server(N), {set_update_lru_on_read, false}), {ok, N}; handle_config_change("couchdb", "max_dbs_open", Max0, _, N) when is_list(Max0) -> Max1 = per_couch_server(list_to_integer(Max0)), - gen_server:call(couch_server(N),{set_max_dbs_open,Max1}), + gen_server:call(couch_server(N), {set_max_dbs_open, Max1}), {ok, N}; handle_config_change("couchdb", "max_dbs_open", _, _, N) -> Max = per_couch_server(?MAX_DBS_OPEN), - gen_server:call(couch_server(N),{set_max_dbs_open,Max}), + gen_server:call(couch_server(N), {set_max_dbs_open, Max}), {ok, N}; handle_config_change("couchdb_engines", _, _, _, N) -> gen_server:call(couch_server(N), reload_engines), @@ -361,57 +390,64 @@ handle_config_terminate(_, stop, _) -> handle_config_terminate(_Server, _Reason, N) -> erlang:send_after(?RELISTEN_DELAY, whereis(?MODULE), {restart_config_listener, N}). - per_couch_server(X) -> erlang:max(1, X div num_servers()). - all_databases() -> {ok, DbList} = all_databases( - fun(DbName, Acc) -> {ok, [DbName | Acc]} end, []), + fun(DbName, Acc) -> {ok, [DbName | Acc]} end, [] + ), {ok, lists:usort(DbList)}. all_databases(Fun, Acc0) -> - {ok, #server{root_dir=Root}} = gen_server:call(couch_server_1, get_server), + {ok, #server{root_dir = Root}} = gen_server:call(couch_server_1, get_server), NormRoot = couch_util:normpath(Root), Extensions = get_engine_extensions(), ExtRegExp = "(" ++ string:join(Extensions, "|") ++ ")", RegExp = - "^[a-z0-9\\_\\$()\\+\\-]*" % stock CouchDB name regex - "(\\.[0-9]{10,})?" % optional shard timestamp - "\\." ++ ExtRegExp ++ "$", % filename extension - FinalAcc = try - couch_util:fold_files(Root, - RegExp, - true, - fun(Filename, AccIn) -> - NormFilename = couch_util:normpath(Filename), - case NormFilename -- NormRoot of - [$/ | RelativeFilename] -> ok; - RelativeFilename -> ok + % stock CouchDB name regex + "^[a-z0-9\\_\\$()\\+\\-]*" + % optional shard timestamp + "(\\.[0-9]{10,})?" + % filename extension + "\\." ++ ExtRegExp ++ "$", + FinalAcc = + try + couch_util:fold_files( + Root, + RegExp, + true, + fun(Filename, AccIn) -> + NormFilename = couch_util:normpath(Filename), + case NormFilename -- NormRoot of + [$/ | RelativeFilename] -> ok; + RelativeFilename -> ok + end, + Ext = filename:extension(RelativeFilename), + case Fun(?l2b(filename:rootname(RelativeFilename, Ext)), AccIn) of + {ok, NewAcc} -> NewAcc; + {stop, NewAcc} -> throw({stop, Fun, NewAcc}) + end end, - Ext = filename:extension(RelativeFilename), - case Fun(?l2b(filename:rootname(RelativeFilename, Ext)), AccIn) of - {ok, NewAcc} -> NewAcc; - {stop, NewAcc} -> throw({stop, Fun, NewAcc}) - end - end, Acc0) - catch throw:{stop, Fun, Acc1} -> - Acc1 - end, + Acc0 + ) + catch + throw:{stop, Fun, Acc1} -> + Acc1 + end, {ok, FinalAcc}. - make_room(Server, Options) -> case lists:member(sys_db, Options) of false -> maybe_close_lru_db(Server); true -> {ok, Server} end. -maybe_close_lru_db(#server{dbs_open=NumOpen, max_dbs_open=MaxOpen}=Server) - when NumOpen < MaxOpen -> +maybe_close_lru_db(#server{dbs_open = NumOpen, max_dbs_open = MaxOpen} = Server) when + NumOpen < MaxOpen +-> {ok, Server}; -maybe_close_lru_db(#server{lru=Lru}=Server) -> +maybe_close_lru_db(#server{lru = Lru} = Server) -> case couch_lru:close(Lru) of {true, NewLru} -> {ok, db_closed(Server#server{lru = NewLru}, [])}; @@ -427,10 +463,11 @@ open_async(Server, From, DbName, Options) -> T0 = os:timestamp(), Opener = spawn_link(fun() -> Res = open_async_int(NoLRUServer, DbName, Options), - IsSuccess = case Res of - {ok, _} -> true; - _ -> false - end, + IsSuccess = + case Res of + {ok, _} -> true; + _ -> false + end, case IsSuccess andalso lists:member(create, Options) of true -> couch_event:notify(DbName, created); @@ -449,10 +486,11 @@ open_async(Server, From, DbName, Options) -> couch_log:info("open_result error ~p for ~s", [Res, DbName]) end end), - ReqType = case lists:member(create, Options) of - true -> create; - false -> open - end, + ReqType = + case lists:member(create, Options) of + true -> create; + false -> open + end, true = ets:insert(couch_dbs(Server), #entry{ name = DbName, pid = Opener, @@ -478,7 +516,7 @@ open_async_int(Server, DbName, Options) -> Error1 end. -handle_call(close_lru, _From, #server{lru=Lru} = Server) -> +handle_call(close_lru, _From, #server{lru = Lru} = Server) -> case couch_lru:close(Lru) of {true, NewLru} -> {reply, ok, db_closed(Server#server{lru = NewLru}, [])}; @@ -488,9 +526,9 @@ handle_call(close_lru, _From, #server{lru=Lru} = Server) -> handle_call(open_dbs_count, _From, Server) -> {reply, Server#server.dbs_open, Server}; handle_call({set_update_lru_on_read, UpdateOnRead}, _From, Server) -> - {reply, ok, Server#server{update_lru_on_read=UpdateOnRead}}; + {reply, ok, Server#server{update_lru_on_read = UpdateOnRead}}; handle_call({set_max_dbs_open, Max}, _From, Server) -> - {reply, ok, Server#server{max_dbs_open=Max}}; + {reply, ok, Server#server{max_dbs_open = Max}}; handle_call(reload_engines, _From, Server) -> {reply, ok, Server#server{engines = get_configured_engines()}}; handle_call(get_server, _From, Server) -> @@ -522,12 +560,13 @@ handle_call({open_result, DbName, {ok, Db}}, {Opener, _}, Server) -> start_time = couch_db:get_instance_start_time(Db) }), true = ets:insert(couch_dbs_pid_to_name(Server), {DbPid, DbName}), - Lru = case couch_db:is_system_db(Db) of - false -> - couch_lru:insert(DbName, Server#server.lru); - true -> - Server#server.lru - end, + Lru = + case couch_db:is_system_db(Db) of + false -> + couch_lru:insert(DbName, Server#server.lru); + true -> + Server#server.lru + end, {reply, ok, Server#server{lru = Lru}}; [#entry{}] -> % A mismatched opener pid means that this open_result message @@ -547,12 +586,13 @@ handle_call({open_result, DbName, Error}, {Opener, _}, Server) -> [gen_server:reply(Waiter, Error) || Waiter <- Waiters], true = ets:delete(couch_dbs(Server), DbName), true = ets:delete(couch_dbs_pid_to_name(Server), Opener), - NewServer = case ReqType of - {create, DbName, Options, CrFrom} -> - open_async(Server, CrFrom, DbName, Options); - _ -> - Server - end, + NewServer = + case ReqType of + {create, DbName, Options, CrFrom} -> + open_async(Server, CrFrom, DbName, Options); + _ -> + Server + end, {reply, ok, db_closed(NewServer, Entry#entry.db_options)}; [#entry{}] -> % A mismatched pid means that this open_result message @@ -561,159 +601,174 @@ handle_call({open_result, DbName, Error}, {Opener, _}, Server) -> end; handle_call({open, DbName, Options}, From, Server) -> case ets:lookup(couch_dbs(Server), DbName) of - [] -> - case make_room(Server, Options) of - {ok, Server2} -> - {noreply, open_async(Server2, From, DbName, Options)}; - CloseError -> - {reply, CloseError, Server} - end; - [#entry{waiters = Waiters} = Entry] when is_list(Waiters) -> - true = ets:insert(couch_dbs(Server), Entry#entry{waiters = [From | Waiters]}), - NumWaiters = length(Waiters), - if NumWaiters =< 10 orelse NumWaiters rem 10 /= 0 -> ok; true -> - Fmt = "~b clients waiting to open db ~s", - couch_log:info(Fmt, [length(Waiters), DbName]) - end, - {noreply, Server}; - [#entry{db = Db}] -> - {reply, {ok, Db}, Server} + [] -> + case make_room(Server, Options) of + {ok, Server2} -> + {noreply, open_async(Server2, From, DbName, Options)}; + CloseError -> + {reply, CloseError, Server} + end; + [#entry{waiters = Waiters} = Entry] when is_list(Waiters) -> + true = ets:insert(couch_dbs(Server), Entry#entry{waiters = [From | Waiters]}), + NumWaiters = length(Waiters), + if + NumWaiters =< 10 orelse NumWaiters rem 10 /= 0 -> + ok; + true -> + Fmt = "~b clients waiting to open db ~s", + couch_log:info(Fmt, [length(Waiters), DbName]) + end, + {noreply, Server}; + [#entry{db = Db}] -> + {reply, {ok, Db}, Server} end; handle_call({create, DbName, Options}, From, Server) -> case ets:lookup(couch_dbs(Server), DbName) of - [] -> - case make_room(Server, Options) of - {ok, Server2} -> + [] -> + case make_room(Server, Options) of + {ok, Server2} -> + CrOptions = [create | Options], + {noreply, open_async(Server2, From, DbName, CrOptions)}; + CloseError -> + {reply, CloseError, Server} + end; + [#entry{req_type = open} = Entry] -> + % We're trying to create a database while someone is in + % the middle of trying to open it. We allow one creator + % to wait while we figure out if it'll succeed. CrOptions = [create | Options], - {noreply, open_async(Server2, From, DbName, CrOptions)}; - CloseError -> - {reply, CloseError, Server} - end; - [#entry{req_type = open} = Entry] -> - % We're trying to create a database while someone is in - % the middle of trying to open it. We allow one creator - % to wait while we figure out if it'll succeed. - CrOptions = [create | Options], - Req = {create, DbName, CrOptions, From}, - true = ets:insert(couch_dbs(Server), Entry#entry{req_type = Req}), - {noreply, Server}; - [_AlreadyRunningDb] -> - {reply, file_exists, Server} + Req = {create, DbName, CrOptions, From}, + true = ets:insert(couch_dbs(Server), Entry#entry{req_type = Req}), + {noreply, Server}; + [_AlreadyRunningDb] -> + {reply, file_exists, Server} end; handle_call({delete, DbName, Options}, _From, Server) -> DbNameList = binary_to_list(DbName), case check_dbname(DbNameList) of - ok -> - Server2 = - case ets:lookup(couch_dbs(Server), DbName) of - [] -> Server; - [#entry{pid = Pid, waiters = Waiters} = Entry] when is_list(Waiters) -> - true = ets:delete(couch_dbs(Server), DbName), - true = ets:delete(couch_dbs_pid_to_name(Server), Pid), - exit(Pid, kill), - [gen_server:reply(Waiter, not_found) || Waiter <- Waiters], - db_closed(Server, Entry#entry.db_options); - [#entry{pid = Pid} = Entry] -> - true = ets:delete(couch_dbs(Server), DbName), - true = ets:delete(couch_dbs_pid_to_name(Server), Pid), - exit(Pid, kill), - db_closed(Server, Entry#entry.db_options) - end, + ok -> + Server2 = + case ets:lookup(couch_dbs(Server), DbName) of + [] -> + Server; + [#entry{pid = Pid, waiters = Waiters} = Entry] when is_list(Waiters) -> + true = ets:delete(couch_dbs(Server), DbName), + true = ets:delete(couch_dbs_pid_to_name(Server), Pid), + exit(Pid, kill), + [gen_server:reply(Waiter, not_found) || Waiter <- Waiters], + db_closed(Server, Entry#entry.db_options); + [#entry{pid = Pid} = Entry] -> + true = ets:delete(couch_dbs(Server), DbName), + true = ets:delete(couch_dbs_pid_to_name(Server), Pid), + exit(Pid, kill), + db_closed(Server, Entry#entry.db_options) + end, - couch_db_plugin:on_delete(DbName, Options), + couch_db_plugin:on_delete(DbName, Options), - DelOpt = [{context, delete} | Options], + DelOpt = [{context, delete} | Options], - % Make sure and remove all compaction data - delete_compaction_files(DbNameList, Options), + % Make sure and remove all compaction data + delete_compaction_files(DbNameList, Options), - {ok, {Engine, FilePath}} = get_engine(Server, DbNameList), - RootDir = Server#server.root_dir, - case couch_db_engine:delete(Engine, RootDir, FilePath, DelOpt) of - ok -> - couch_event:notify(DbName, deleted), - {reply, ok, Server2}; - {error, enoent} -> - {reply, not_found, Server2}; - Else -> - {reply, Else, Server2} - end; - Error -> - {reply, Error, Server} + {ok, {Engine, FilePath}} = get_engine(Server, DbNameList), + RootDir = Server#server.root_dir, + case couch_db_engine:delete(Engine, RootDir, FilePath, DelOpt) of + ok -> + couch_event:notify(DbName, deleted), + {reply, ok, Server2}; + {error, enoent} -> + {reply, not_found, Server2}; + Else -> + {reply, Else, Server2} + end; + Error -> + {reply, Error, Server} end; handle_call({db_updated, Db}, _From, Server0) -> DbName = couch_db:name(Db), StartTime = couch_db:get_instance_start_time(Db), - Server = try ets:lookup_element(couch_dbs(Server0), DbName, #entry.start_time) of - StartTime -> - true = ets:update_element(couch_dbs(Server0), DbName, {#entry.db, Db}), - Lru = case couch_db:is_system_db(Db) of - false -> couch_lru:update(DbName, Server0#server.lru); - true -> Server0#server.lru - end, - Server0#server{lru = Lru}; - _ -> - Server0 - catch _:_ -> - Server0 - end, + Server = + try ets:lookup_element(couch_dbs(Server0), DbName, #entry.start_time) of + StartTime -> + true = ets:update_element(couch_dbs(Server0), DbName, {#entry.db, Db}), + Lru = + case couch_db:is_system_db(Db) of + false -> couch_lru:update(DbName, Server0#server.lru); + true -> Server0#server.lru + end, + Server0#server{lru = Lru}; + _ -> + Server0 + catch + _:_ -> + Server0 + end, {reply, ok, Server}. -handle_cast({update_lru, DbName}, #server{lru = Lru, update_lru_on_read=true} = Server) -> +handle_cast({update_lru, DbName}, #server{lru = Lru, update_lru_on_read = true} = Server) -> {noreply, Server#server{lru = couch_lru:update(DbName, Lru)}}; handle_cast({update_lru, _DbName}, Server) -> {noreply, Server}; handle_cast({close_db_if_idle, DbName}, Server) -> case ets:update_element(couch_dbs(Server), DbName, {#entry.lock, locked}) of - true -> - [#entry{db = Db, db_options = DbOpts}] = ets:lookup(couch_dbs(Server), DbName), - case couch_db:is_idle(Db) of true -> - DbPid = couch_db:get_pid(Db), - true = ets:delete(couch_dbs(Server), DbName), - true = ets:delete(couch_dbs_pid_to_name(Server), DbPid), - exit(DbPid, kill), - {noreply, db_closed(Server, DbOpts)}; + [#entry{db = Db, db_options = DbOpts}] = ets:lookup(couch_dbs(Server), DbName), + case couch_db:is_idle(Db) of + true -> + DbPid = couch_db:get_pid(Db), + true = ets:delete(couch_dbs(Server), DbName), + true = ets:delete(couch_dbs_pid_to_name(Server), DbPid), + exit(DbPid, kill), + {noreply, db_closed(Server, DbOpts)}; + false -> + true = ets:update_element( + couch_dbs(Server), DbName, {#entry.lock, unlocked} + ), + {noreply, Server} + end; false -> - true = ets:update_element( - couch_dbs(Server), DbName, {#entry.lock, unlocked}), {noreply, Server} - end; - false -> - {noreply, Server} end; - handle_cast(Msg, Server) -> {stop, {unknown_cast_message, Msg}, Server}. -code_change(_OldVsn, #server{}=State, _Extra) -> +code_change(_OldVsn, #server{} = State, _Extra) -> {ok, State}. handle_info({'EXIT', _Pid, config_change}, Server) -> {stop, config_change, Server}; handle_info({'EXIT', Pid, Reason}, Server) -> case ets:lookup(couch_dbs_pid_to_name(Server), Pid) of - [{Pid, DbName}] -> - [#entry{waiters = Waiters} = Entry] = ets:lookup(couch_dbs(Server), DbName), - if Reason /= snappy_nif_not_loaded -> ok; true -> - Msg = io_lib:format("To open the database `~s`, Apache CouchDB " - "must be built with Erlang OTP R13B04 or higher.", [DbName]), - couch_log:error(Msg, []) - end, - % We kill databases on purpose so there's no reason - % to log that fact. So we restrict logging to "interesting" - % reasons. - if Reason == normal orelse Reason == killed -> ok; true -> - couch_log:info("db ~s died with reason ~p", [DbName, Reason]) - end, - if not is_list(Waiters) -> ok; true -> - [gen_server:reply(Waiter, Reason) || Waiter <- Waiters] - end, - true = ets:delete(couch_dbs(Server), DbName), - true = ets:delete(couch_dbs_pid_to_name(Server), Pid), - {noreply, db_closed(Server, Entry#entry.db_options)}; - [] -> - {noreply, Server} + [{Pid, DbName}] -> + [#entry{waiters = Waiters} = Entry] = ets:lookup(couch_dbs(Server), DbName), + if + Reason /= snappy_nif_not_loaded -> + ok; + true -> + Msg = io_lib:format( + "To open the database `~s`, Apache CouchDB " + "must be built with Erlang OTP R13B04 or higher.", + [DbName] + ), + couch_log:error(Msg, []) + end, + % We kill databases on purpose so there's no reason + % to log that fact. So we restrict logging to "interesting" + % reasons. + if + Reason == normal orelse Reason == killed -> ok; + true -> couch_log:info("db ~s died with reason ~p", [DbName, Reason]) + end, + if + not is_list(Waiters) -> ok; + true -> [gen_server:reply(Waiter, Reason) || Waiter <- Waiters] + end, + true = ets:delete(couch_dbs(Server), DbName), + true = ets:delete(couch_dbs_pid_to_name(Server), Pid), + {noreply, db_closed(Server, Entry#entry.db_options)}; + [] -> + {noreply, Server} end; handle_info({restart_config_listener, N}, State) -> ok = config:listen_for_changes(?MODULE, N), @@ -723,13 +778,13 @@ handle_info(Info, Server) -> db_opened(Server, Options) -> case lists:member(sys_db, Options) of - false -> Server#server{dbs_open=Server#server.dbs_open + 1}; + false -> Server#server{dbs_open = Server#server.dbs_open + 1}; true -> Server end. db_closed(Server, Options) -> case lists:member(sys_db, Options) of - false -> Server#server{dbs_open=Server#server.dbs_open - 1}; + false -> Server#server{dbs_open = Server#server.dbs_open - 1}; true -> Server end. @@ -757,13 +812,17 @@ validate_open_or_create(DbName, Options) -> get_configured_engines() -> ConfigEntries = config:get("couchdb_engines"), - Engines = lists:flatmap(fun({Extension, ModuleStr}) -> - try - [{Extension, list_to_atom(ModuleStr)}] - catch _T:_R -> - [] - end - end, ConfigEntries), + Engines = lists:flatmap( + fun({Extension, ModuleStr}) -> + try + [{Extension, list_to_atom(ModuleStr)}] + catch + _T:_R -> + [] + end + end, + ConfigEntries + ), case Engines of [] -> [{"couch", couch_bt_engine}]; @@ -771,7 +830,6 @@ get_configured_engines() -> Else end. - get_engine(Server, DbName, Options) -> #server{ root_dir = RootDir, @@ -791,7 +849,6 @@ get_engine(Server, DbName, Options) -> get_engine(Server, DbName) end. - get_engine(Server, DbName) -> #server{ root_dir = RootDir, @@ -807,18 +864,20 @@ get_engine(Server, DbName) -> erlang:error(engine_conflict) end. - get_possible_engines(DbName, RootDir, Engines) -> - lists:foldl(fun({Extension, Engine}, Acc) -> - Path = make_filepath(RootDir, DbName, Extension), - case couch_db_engine:exists(Engine, Path) of - true -> - [{Engine, Path} | Acc]; - false -> - Acc - end - end, [], Engines). - + lists:foldl( + fun({Extension, Engine}, Acc) -> + Path = make_filepath(RootDir, DbName, Extension), + case couch_db_engine:exists(Engine, Path) of + true -> + [{Engine, Path} | Acc]; + false -> + Acc + end + end, + [], + Engines + ). get_default_engine(Server, DbName) -> #server{ @@ -831,15 +890,15 @@ get_default_engine(Server, DbName) -> {Extension, Module} -> {ok, {Module, make_filepath(RootDir, DbName, Extension)}}; false -> - Fmt = "Invalid storage engine extension ~s," - " configured engine extensions are: ~s", + Fmt = + "Invalid storage engine extension ~s," + " configured engine extensions are: ~s", Exts = [E || {E, _} <- Engines], Args = [Extension, string:join(Exts, ", ")], couch_log:error(Fmt, Args), {ok, Default} end. - make_filepath(RootDir, DbName, Extension) when is_binary(RootDir) -> make_filepath(binary_to_list(RootDir), DbName, Extension); make_filepath(RootDir, DbName, Extension) when is_binary(DbName) -> @@ -849,7 +908,6 @@ make_filepath(RootDir, DbName, Extension) when is_binary(Extension) -> make_filepath(RootDir, DbName, Extension) -> filename:join([RootDir, "./" ++ DbName ++ "." ++ Extension]). - get_engine_extensions() -> case config:get("couchdb_engines") of [] -> @@ -858,7 +916,6 @@ get_engine_extensions() -> [Ext || {Ext, _Mod} <- Entries] end. - check_engine(Options) -> case couch_util:get_value(engine, Options) of Ext when is_binary(Ext) -> @@ -874,7 +931,6 @@ check_engine(Options) -> ok end. - get_engine_path(DbName, Engine) when is_binary(DbName), is_atom(Engine) -> RootDir = config:get("couchdb", "database_dir", "."), case lists:keyfind(Engine, 2, get_configured_engines()) of @@ -897,60 +953,48 @@ unlock(DbName) when is_binary(DbName) -> true = ets:delete(couch_dbs_locks(DbName), DbName), ok. - db_updated(Db) -> DbName = couch_db:name(Db), gen_server:call(couch_server(DbName), {db_updated, Db}, infinity). - couch_server(Arg) -> name("couch_server", Arg). - couch_dbs(Arg) -> name("couch_dbs", Arg). - couch_dbs_pid_to_name(Arg) -> name("couch_dbs_pid_to_name", Arg). - couch_dbs_locks(Arg) -> name("couch_dbs_locks", Arg). - name("couch_dbs", #server{} = Server) -> Server#server.couch_dbs; - name("couch_dbs_pid_to_name", #server{} = Server) -> Server#server.couch_dbs_pid_to_name; - name("couch_dbs_locks", #server{} = Server) -> Server#server.couch_dbs_locks; - name(BaseName, DbName) when is_list(DbName) -> name(BaseName, ?l2b(DbName)); - name(BaseName, DbName) when is_binary(DbName) -> N = 1 + erlang:phash2(DbName, num_servers()), name(BaseName, N); - name(BaseName, N) when is_integer(N), N > 0 -> list_to_atom(BaseName ++ "_" ++ integer_to_list(N)). - num_servers() -> erlang:system_info(schedulers). - aggregate_queue_len() -> N = num_servers(), Names = [couch_server(I) || I <- lists:seq(1, N)], - MQs = [process_info(whereis(Name), message_queue_len) || - Name <- Names], + MQs = [ + process_info(whereis(Name), message_queue_len) + || Name <- Names + ], lists:sum([X || {_, X} <- MQs]). - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -1006,10 +1050,13 @@ should_add_sys_db_callbacks() -> "_replicator.couch", "_replicator" ], - lists:foreach(fun(DbName) -> - check_case(DbName, true), - check_case(?l2b(DbName), true) - end, Cases). + lists:foreach( + fun(DbName) -> + check_case(DbName, true), + check_case(?l2b(DbName), true) + end, + Cases + ). should_not_add_sys_db_callbacks() -> Cases = [ @@ -1021,10 +1068,13 @@ should_not_add_sys_db_callbacks() -> "mydb.couch", "mydb" ], - lists:foreach(fun(DbName) -> - check_case(DbName, false), - check_case(?l2b(DbName), false) - end, Cases). + lists:foreach( + fun(DbName) -> + check_case(DbName, false), + check_case(?l2b(DbName), false) + end, + Cases + ). check_case(DbName, IsAdded) -> Options = maybe_add_sys_db_callbacks(DbName, [other_options]), diff --git a/src/couch/src/couch_stream.erl b/src/couch/src/couch_stream.erl index 2ab46d7e7..12b290820 100644 --- a/src/couch/src/couch_stream.erl +++ b/src/couch/src/couch_stream.erl @@ -14,7 +14,6 @@ -behaviour(gen_server). -vsn(1). - -export([ open/1, open/2, @@ -39,16 +38,14 @@ code_change/3 ]). - -include_lib("couch/include/couch_db.hrl"). -define(DEFAULT_BUFFER_SIZE, 4096). - -record(stream, { engine, opener_monitor, - written_pointers=[], + written_pointers = [], buffer_list = [], buffer_len = 0, max_buffer, @@ -62,39 +59,35 @@ end_encoding_fun }). - open({_StreamEngine, _StreamEngineState} = Engine) -> open(Engine, []). - open({_StreamEngine, _StreamEngineState} = Engine, Options) -> gen_server:start_link(?MODULE, {Engine, self(), erlang:get(io_priority), Options}, []). - close(Pid) -> gen_server:call(Pid, close, infinity). - copy(Src, Dst) -> - foldl(Src, fun(Bin, _) -> - ok = write(Dst, Bin) - end, ok). - + foldl( + Src, + fun(Bin, _) -> + ok = write(Dst, Bin) + end, + ok + ). write(_Pid, <<>>) -> ok; write(Pid, Bin) -> gen_server:call(Pid, {write, Bin}, infinity). - to_disk_term({Engine, EngineState}) -> Engine:to_disk_term(EngineState). - foldl({Engine, EngineState}, Fun, Acc) -> Engine:foldl(EngineState, Fun, Acc). - foldl(Engine, <<>>, Fun, Acc) -> foldl(Engine, Fun, Acc); foldl(Engine, Md5, UserFun, UserAcc) -> @@ -103,18 +96,17 @@ foldl(Engine, Md5, UserFun, UserAcc) -> Md5 = couch_hash:md5_hash_final(Md5Acc), OutAcc. - foldl_decode(Engine, Md5, Enc, UserFun, UserAcc1) -> - {DecDataFun, DecEndFun} = case Enc of - gzip -> ungzip_init(); - identity -> identity_enc_dec_funs() - end, + {DecDataFun, DecEndFun} = + case Enc of + gzip -> ungzip_init(); + identity -> identity_enc_dec_funs() + end, InitAcc = {DecDataFun, UserFun, UserAcc1}, {_, _, UserAcc2} = foldl(Engine, Md5, fun foldl_decode/2, InitAcc), DecEndFun(), UserAcc2. - range_foldl(Engine, From, To, UserFun, UserAcc) when To >= From -> NewEngine = do_seek(Engine, From), InitAcc = {To - From, UserFun, UserAcc}, @@ -126,19 +118,16 @@ range_foldl(Engine, From, To, UserFun, UserAcc) when To >= From -> UserAcc3 end. - foldl_md5(Bin, {Md5Acc, UserFun, UserAcc}) -> NewMd5Acc = couch_hash:md5_hash_update(Md5Acc, Bin), {NewMd5Acc, UserFun, UserFun(Bin, UserAcc)}. - foldl_decode(EncBin, {DecFun, UserFun, UserAcc}) -> case DecFun(EncBin) of <<>> -> {DecFun, UserFun, UserAcc}; Dec -> {DecFun, UserFun, UserFun(Dec, UserAcc)} end. - foldl_length(Bin, {Length, UserFun, UserAcc}) -> BinSize = size(Bin), case BinSize =< Length of @@ -151,24 +140,24 @@ foldl_length(Bin, {Length, UserFun, UserAcc}) -> gzip_init(Options) -> case couch_util:get_value(compression_level, Options, 0) of - Lvl when Lvl >= 1 andalso Lvl =< 9 -> - Z = zlib:open(), - % 15 = ?MAX_WBITS (defined in the zlib module) - % the 16 + ?MAX_WBITS formula was obtained by inspecting zlib:gzip/1 - ok = zlib:deflateInit(Z, Lvl, deflated, 16 + 15, 8, default), - { - fun(Data) -> - zlib:deflate(Z, Data) - end, - fun() -> - Last = zlib:deflate(Z, [], finish), - ok = zlib:deflateEnd(Z), - ok = zlib:close(Z), - Last - end - }; - _ -> - identity_enc_dec_funs() + Lvl when Lvl >= 1 andalso Lvl =< 9 -> + Z = zlib:open(), + % 15 = ?MAX_WBITS (defined in the zlib module) + % the 16 + ?MAX_WBITS formula was obtained by inspecting zlib:gzip/1 + ok = zlib:deflateInit(Z, Lvl, deflated, 16 + 15, 8, default), + { + fun(Data) -> + zlib:deflate(Z, Data) + end, + fun() -> + Last = zlib:deflate(Z, [], finish), + ok = zlib:deflateEnd(Z), + ok = zlib:close(Z), + Last + end + }; + _ -> + identity_enc_dec_funs() end. ungzip_init() -> @@ -190,25 +179,24 @@ identity_enc_dec_funs() -> fun() -> [] end }. - init({Engine, OpenerPid, OpenerPriority, Options}) -> erlang:put(io_priority, OpenerPriority), {EncodingFun, EndEncodingFun} = - case couch_util:get_value(encoding, Options, identity) of - identity -> identity_enc_dec_funs(); - gzip -> gzip_init(Options) - end, + case couch_util:get_value(encoding, Options, identity) of + identity -> identity_enc_dec_funs(); + gzip -> gzip_init(Options) + end, {ok, #stream{ - engine=Engine, - opener_monitor=erlang:monitor(process, OpenerPid), - md5=couch_hash:md5_hash_init(), - identity_md5=couch_hash:md5_hash_init(), - encoding_fun=EncodingFun, - end_encoding_fun=EndEncodingFun, - max_buffer=couch_util:get_value( - buffer_size, Options, ?DEFAULT_BUFFER_SIZE) - } - }. + engine = Engine, + opener_monitor = erlang:monitor(process, OpenerPid), + md5 = couch_hash:md5_hash_init(), + identity_md5 = couch_hash:md5_hash_init(), + encoding_fun = EncodingFun, + end_encoding_fun = EndEncodingFun, + max_buffer = couch_util:get_value( + buffer_size, Options, ?DEFAULT_BUFFER_SIZE + ) + }}. terminate(_Reason, _Stream) -> ok. @@ -224,36 +212,42 @@ handle_call({write, Bin}, _From, Stream) -> md5 = Md5, identity_md5 = IdenMd5, identity_len = IdenLen, - encoding_fun = EncodingFun} = Stream, - if BinSize + BufferLen > Max -> - WriteBin = lists:reverse(Buffer, [Bin]), - IdenMd5_2 = couch_hash:md5_hash_update(IdenMd5, WriteBin), - case EncodingFun(WriteBin) of - [] -> - % case where the encoder did some internal buffering - % (zlib does it for example) - NewEngine = Engine, - WrittenLen2 = WrittenLen, - Md5_2 = Md5; - WriteBin2 -> - NewEngine = do_write(Engine, WriteBin2), - WrittenLen2 = WrittenLen + iolist_size(WriteBin2), - Md5_2 = couch_hash:md5_hash_update(Md5, WriteBin2) - end, + encoding_fun = EncodingFun + } = Stream, + if + BinSize + BufferLen > Max -> + WriteBin = lists:reverse(Buffer, [Bin]), + IdenMd5_2 = couch_hash:md5_hash_update(IdenMd5, WriteBin), + case EncodingFun(WriteBin) of + [] -> + % case where the encoder did some internal buffering + % (zlib does it for example) + NewEngine = Engine, + WrittenLen2 = WrittenLen, + Md5_2 = Md5; + WriteBin2 -> + NewEngine = do_write(Engine, WriteBin2), + WrittenLen2 = WrittenLen + iolist_size(WriteBin2), + Md5_2 = couch_hash:md5_hash_update(Md5, WriteBin2) + end, - {reply, ok, Stream#stream{ - engine = NewEngine, - written_len=WrittenLen2, - buffer_list=[], - buffer_len=0, - md5=Md5_2, - identity_md5=IdenMd5_2, - identity_len=IdenLen + BinSize}, hibernate}; - true -> - {reply, ok, Stream#stream{ - buffer_list=[Bin|Buffer], - buffer_len=BufferLen + BinSize, - identity_len=IdenLen + BinSize}} + {reply, ok, + Stream#stream{ + engine = NewEngine, + written_len = WrittenLen2, + buffer_list = [], + buffer_len = 0, + md5 = Md5_2, + identity_md5 = IdenMd5_2, + identity_len = IdenLen + BinSize + }, + hibernate}; + true -> + {reply, ok, Stream#stream{ + buffer_list = [Bin | Buffer], + buffer_len = BufferLen + BinSize, + identity_len = IdenLen + BinSize + }} end; handle_call(close, _From, Stream) -> #stream{ @@ -265,35 +259,36 @@ handle_call(close, _From, Stream) -> identity_md5 = IdenMd5, identity_len = IdenLen, encoding_fun = EncodingFun, - end_encoding_fun = EndEncodingFun} = Stream, + end_encoding_fun = EndEncodingFun + } = Stream, WriteBin = lists:reverse(Buffer), IdenMd5Final = couch_hash:md5_hash_final(couch_hash:md5_hash_update(IdenMd5, WriteBin)), WriteBin2 = EncodingFun(WriteBin) ++ EndEncodingFun(), Md5Final = couch_hash:md5_hash_final(couch_hash:md5_hash_update(Md5, WriteBin2)), - Result = case WriteBin2 of - [] -> - {do_finalize(Engine), WrittenLen, IdenLen, Md5Final, IdenMd5Final}; - _ -> - NewEngine = do_write(Engine, WriteBin2), - StreamLen = WrittenLen + iolist_size(WriteBin2), - {do_finalize(NewEngine), StreamLen, IdenLen, Md5Final, IdenMd5Final} - end, + Result = + case WriteBin2 of + [] -> + {do_finalize(Engine), WrittenLen, IdenLen, Md5Final, IdenMd5Final}; + _ -> + NewEngine = do_write(Engine, WriteBin2), + StreamLen = WrittenLen + iolist_size(WriteBin2), + {do_finalize(NewEngine), StreamLen, IdenLen, Md5Final, IdenMd5Final} + end, erlang:demonitor(MonRef), {stop, normal, Result, Stream}. handle_cast(_Msg, State) -> - {noreply,State}. + {noreply, State}. code_change(_OldVsn, State, _Extra) -> {ok, State}. -handle_info({'DOWN', Ref, _, _, _}, #stream{opener_monitor=Ref} = State) -> +handle_info({'DOWN', Ref, _, _, _}, #stream{opener_monitor = Ref} = State) -> {stop, normal, State}; handle_info(_Info, State) -> {noreply, State}. - do_seek({Engine, EngineState}, Offset) -> {ok, NewState} = Engine:seek(EngineState, Offset), {Engine, NewState}. @@ -305,4 +300,3 @@ do_write({Engine, EngineState}, Data) -> do_finalize({Engine, EngineState}) -> {ok, NewState} = Engine:finalize(EngineState), {Engine, NewState}. - diff --git a/src/couch/src/couch_sup.erl b/src/couch/src/couch_sup.erl index b936c1e5d..033f7115f 100644 --- a/src/couch/src/couch_sup.erl +++ b/src/couch/src/couch_sup.erl @@ -15,7 +15,6 @@ -vsn(1). -behaviour(config_listener). - -export([ start_link/0, init/1, @@ -23,10 +22,8 @@ handle_config_terminate/3 ]). - -include_lib("couch/include/couch_db.hrl"). - start_link() -> assert_admins(), maybe_launch_admin_annoyance_reporter(), @@ -42,43 +39,45 @@ start_link() -> Else end. - init(_Args) -> couch_log:info("Starting ~s", [?MODULE]), - {ok, {{one_for_one,10, 60}, [ - { - config_listener_mon, - {config_listener_mon, start_link, [?MODULE, nil]}, - permanent, - 5000, - worker, - [config_listener_mon] - }, - { - couch_primary_services, - {couch_primary_sup, start_link, []}, - permanent, - infinity, - supervisor, - [couch_primary_sup] - }, - { - couch_secondary_services, - {couch_secondary_sup, start_link, []}, - permanent, - infinity, - supervisor, - [couch_secondary_sup] - } - ]}}. - + {ok, + {{one_for_one, 10, 60}, [ + { + config_listener_mon, + {config_listener_mon, start_link, [?MODULE, nil]}, + permanent, + 5000, + worker, + [config_listener_mon] + }, + { + couch_primary_services, + {couch_primary_sup, start_link, []}, + permanent, + infinity, + supervisor, + [couch_primary_sup] + }, + { + couch_secondary_services, + {couch_secondary_sup, start_link, []}, + permanent, + infinity, + supervisor, + [couch_secondary_sup] + } + ]}}. handle_config_change("daemons", _, _, _, _) -> exit(whereis(?MODULE), shutdown), remove_handler; handle_config_change("couchdb", "util_driver_dir", _, _, _) -> - [Pid] = [P || {collation_driver, P, _, _} - <- supervisor:which_children(couch_primary_services)], + [Pid] = [ + P + || {collation_driver, P, _, _} <- + supervisor:which_children(couch_primary_services) + ], Pid ! reload_driver, {ok, nil}; handle_config_change(_, _, _, _, _) -> @@ -91,44 +90,47 @@ assert_admins() -> couch_log:info("Preflight check: Asserting Admin Account~n", []), case {config:get("admins"), os:getenv("COUCHDB_TEST_ADMIN_PARTY_OVERRIDE")} of {[], false} -> - couch_log:info("~n%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%~n" - ++ " No Admin Account Found, aborting startup. ~n" - ++ " Please configure an admin account in your local.ini file. ~n" - ++ "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%~n", []), + couch_log:info( + "~n%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%~n" ++ + " No Admin Account Found, aborting startup. ~n" ++ + " Please configure an admin account in your local.ini file. ~n" ++ + "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%~n", + [] + ), % Wait a second so the log message can make it to the log timer:sleep(500), erlang:halt(1); - _ -> ok + _ -> + ok end. send_no_admin_account_error_message() -> - couch_log:error("No Admin Account configured." - ++ " Please configure an Admin Account in your local.ini file and restart CouchDB.~n", []), + couch_log:error( + "No Admin Account configured." ++ + " Please configure an Admin Account in your local.ini file and restart CouchDB.~n", + [] + ), FiveMinutes = 5 * 1000 * 60, timer:sleep(FiveMinutes), send_no_admin_account_error_message(). - + maybe_launch_admin_annoyance_reporter() -> case os:getenv("COUCHDB_TEST_ADMIN_PARTY_OVERRIDE") of false -> ok; _ -> spawn_link(fun send_no_admin_account_error_message/0) end. - notify_starting() -> couch_log:info("Apache CouchDB ~s is starting.~n", [ couch_server:get_version() ]). - notify_started() -> couch_log:info("Apache CouchDB has started. Time to relax.~n", []). - notify_error(Error) -> couch_log:error("Error starting Apache CouchDB:~n~n ~p~n~n", [Error]). - write_pidfile() -> case init:get_argument(pidfile) of {ok, [PidFile]} -> diff --git a/src/couch/src/couch_task_status.erl b/src/couch/src/couch_task_status.erl index 74247d63d..42d7c4f62 100644 --- a/src/couch/src/couch_task_status.erl +++ b/src/couch/src/couch_task_status.erl @@ -36,36 +36,30 @@ -define(set(L, K, V), lists:keystore(K, 1, L, {K, V})). - start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - stop() -> gen_server:cast(?MODULE, stop). - all() -> gen_server:call(?MODULE, all). - add_task(Props) -> put(task_status_update, {{0, 0, 0}, 0}), Ts = timestamp(), TaskProps = lists:ukeysort( - 1, [{started_on, Ts}, {updated_on, Ts} | Props]), + 1, [{started_on, Ts}, {updated_on, Ts} | Props] + ), put(task_status_props, TaskProps), gen_server:call(?MODULE, {add_task, TaskProps}). - is_task_added() -> is_list(erlang:get(task_status_props)). - set_update_frequency(Msecs) -> put(task_status_update, {{0, 0, 0}, Msecs * 1000}). - update(Props) -> MergeProps = lists:ukeysort(1, Props), CurrProps = erlang:get(task_status_props), @@ -77,7 +71,6 @@ update(Props) -> persist(TaskProps) end. - get(Props) when is_list(Props) -> TaskProps = erlang:get(task_status_props), [couch_util:get_value(P, TaskProps) || P <- Props]; @@ -85,61 +78,54 @@ get(Prop) -> TaskProps = erlang:get(task_status_props), couch_util:get_value(Prop, TaskProps). - maybe_persist(TaskProps) -> {LastUpdateTime, Frequency} = erlang:get(task_status_update), case timer:now_diff(Now = os:timestamp(), LastUpdateTime) >= Frequency of - true -> - put(task_status_update, {Now, Frequency}), - persist(TaskProps); - false -> - ok + true -> + put(task_status_update, {Now, Frequency}), + persist(TaskProps); + false -> + ok end. - persist(TaskProps0) -> TaskProps = ?set(TaskProps0, updated_on, timestamp(os:timestamp())), put(task_status_props, TaskProps), gen_server:cast(?MODULE, {update_status, self(), TaskProps}). - init([]) -> % read configuration settings and register for configuration changes ets:new(?MODULE, [ordered_set, protected, named_table]), {ok, nil}. - -terminate(_Reason,_State) -> +terminate(_Reason, _State) -> ok. - handle_call({add_task, TaskProps}, {From, _}, Server) -> case ets:lookup(?MODULE, From) of - [] -> - true = ets:insert(?MODULE, {From, TaskProps}), - erlang:monitor(process, From), - {reply, ok, Server}; - [_] -> - {reply, {add_task_error, already_registered}, Server} + [] -> + true = ets:insert(?MODULE, {From, TaskProps}), + erlang:monitor(process, From), + {reply, ok, Server}; + [_] -> + {reply, {add_task_error, already_registered}, Server} end; handle_call(all, _, Server) -> All = [ [{pid, ?l2b(pid_to_list(Pid))}, process_status(Pid) | TaskProps] - || - {Pid, TaskProps} <- ets:tab2list(?MODULE) + || {Pid, TaskProps} <- ets:tab2list(?MODULE) ], {reply, All, Server}. - handle_cast({update_status, Pid, NewProps}, Server) -> case ets:lookup(?MODULE, Pid) of - [{Pid, _CurProps}] -> - couch_log:debug("New task status for ~p: ~p", [Pid, NewProps]), - true = ets:insert(?MODULE, {Pid, NewProps}); - _ -> - % Task finished/died in the meanwhile and we must have received - % a monitor message before this call - ignore. - ok + [{Pid, _CurProps}] -> + couch_log:debug("New task status for ~p: ~p", [Pid, NewProps]), + true = ets:insert(?MODULE, {Pid, NewProps}); + _ -> + % Task finished/died in the meanwhile and we must have received + % a monitor message before this call - ignore. + ok end, {noreply, Server}; handle_cast(stop, State) -> @@ -150,18 +136,15 @@ handle_info({'DOWN', _MonitorRef, _Type, Pid, _Info}, Server) -> ets:delete(?MODULE, Pid), {noreply, Server}. - code_change(_OldVsn, State, _Extra) -> {ok, State}. - timestamp() -> timestamp(os:timestamp()). timestamp({Mega, Secs, _}) -> Mega * 1000000 + Secs. - process_status(Pid) -> case process_info(Pid, status) of undefined -> diff --git a/src/couch/src/couch_totp.erl b/src/couch/src/couch_totp.erl index 56e70d81a..3eff9a583 100644 --- a/src/couch/src/couch_totp.erl +++ b/src/couch/src/couch_totp.erl @@ -14,10 +14,11 @@ -export([generate/5]). -generate(Alg, Key, CounterSecs, StepSecs, OutputLen) - when is_atom(Alg), - is_binary(Key), - is_integer(CounterSecs), - is_integer(StepSecs), - is_integer(OutputLen) -> +generate(Alg, Key, CounterSecs, StepSecs, OutputLen) when + is_atom(Alg), + is_binary(Key), + is_integer(CounterSecs), + is_integer(StepSecs), + is_integer(OutputLen) +-> couch_hotp:generate(Alg, Key, CounterSecs div StepSecs, OutputLen). diff --git a/src/couch/src/couch_users_db.erl b/src/couch/src/couch_users_db.erl index 0c169d3ed..7ef3aee78 100644 --- a/src/couch/src/couch_users_db.erl +++ b/src/couch/src/couch_users_db.erl @@ -42,15 +42,15 @@ % Else % -> save_doc before_doc_update(Doc, Db, _UpdateType) -> - #user_ctx{name=Name} = couch_db:get_user_ctx(Db), + #user_ctx{name = Name} = couch_db:get_user_ctx(Db), DocName = get_doc_name(Doc), case (catch couch_db:check_is_admin(Db)) of - ok -> - save_doc(Doc); - _ when Name =:= DocName orelse Name =:= null -> - save_doc(Doc); - _ -> - throw(not_found) + ok -> + save_doc(Doc); + _ when Name =:= DocName orelse Name =:= null -> + save_doc(Doc); + _ -> + throw(not_found) end. % If newDoc.password == null || newDoc.password == undefined: @@ -60,38 +60,41 @@ before_doc_update(Doc, Db, _UpdateType) -> % newDoc.password_sha = hash_pw(newDoc.password + salt) % newDoc.salt = salt % newDoc.password = null -save_doc(#doc{body={Body}} = Doc) -> +save_doc(#doc{body = {Body}} = Doc) -> %% Support both schemes to smooth migration from legacy scheme Scheme = chttpd_util:get_chttpd_auth_config("password_scheme", "pbkdf2"), case {couch_util:get_value(?PASSWORD, Body), Scheme} of - {null, _} -> % server admins don't have a user-db password entry - Doc; - {undefined, _} -> - Doc; - {ClearPassword, "simple"} -> % deprecated - ok = validate_password(ClearPassword), - Salt = couch_uuids:random(), - PasswordSha = couch_passwords:simple(ClearPassword, Salt), - Body0 = ?replace(Body, ?PASSWORD_SCHEME, ?SIMPLE), - Body1 = ?replace(Body0, ?SALT, Salt), - Body2 = ?replace(Body1, ?PASSWORD_SHA, PasswordSha), - Body3 = proplists:delete(?PASSWORD, Body2), - Doc#doc{body={Body3}}; - {ClearPassword, "pbkdf2"} -> - ok = validate_password(ClearPassword), - Iterations = chttpd_util:get_chttpd_auth_config_integer( - "iterations", 10), - Salt = couch_uuids:random(), - DerivedKey = couch_passwords:pbkdf2(ClearPassword, Salt, Iterations), - Body0 = ?replace(Body, ?PASSWORD_SCHEME, ?PBKDF2), - Body1 = ?replace(Body0, ?ITERATIONS, Iterations), - Body2 = ?replace(Body1, ?DERIVED_KEY, DerivedKey), - Body3 = ?replace(Body2, ?SALT, Salt), - Body4 = proplists:delete(?PASSWORD, Body3), - Doc#doc{body={Body4}}; - {_ClearPassword, Scheme} -> - couch_log:error("[couch_httpd_auth] password_scheme value of '~p' is invalid.", [Scheme]), - throw({forbidden, ?PASSWORD_SERVER_ERROR}) + % server admins don't have a user-db password entry + {null, _} -> + Doc; + {undefined, _} -> + Doc; + % deprecated + {ClearPassword, "simple"} -> + ok = validate_password(ClearPassword), + Salt = couch_uuids:random(), + PasswordSha = couch_passwords:simple(ClearPassword, Salt), + Body0 = ?replace(Body, ?PASSWORD_SCHEME, ?SIMPLE), + Body1 = ?replace(Body0, ?SALT, Salt), + Body2 = ?replace(Body1, ?PASSWORD_SHA, PasswordSha), + Body3 = proplists:delete(?PASSWORD, Body2), + Doc#doc{body = {Body3}}; + {ClearPassword, "pbkdf2"} -> + ok = validate_password(ClearPassword), + Iterations = chttpd_util:get_chttpd_auth_config_integer( + "iterations", 10 + ), + Salt = couch_uuids:random(), + DerivedKey = couch_passwords:pbkdf2(ClearPassword, Salt, Iterations), + Body0 = ?replace(Body, ?PASSWORD_SCHEME, ?PBKDF2), + Body1 = ?replace(Body0, ?ITERATIONS, Iterations), + Body2 = ?replace(Body1, ?DERIVED_KEY, DerivedKey), + Body3 = ?replace(Body2, ?SALT, Salt), + Body4 = proplists:delete(?PASSWORD, Body3), + Doc#doc{body = {Body4}}; + {_ClearPassword, Scheme} -> + couch_log:error("[couch_httpd_auth] password_scheme value of '~p' is invalid.", [Scheme]), + throw({forbidden, ?PASSWORD_SERVER_ERROR}) end. % Validate if a new password matches all RegExp in the password_regexp setting. @@ -104,47 +107,52 @@ validate_password(ClearPassword) -> "[]" -> ok; ValidateConfig -> - RequirementList = case couch_util:parse_term(ValidateConfig) of - {ok, RegExpList} when is_list(RegExpList) -> - RegExpList; - {ok, NonListValue} -> - couch_log:error( - "[couch_httpd_auth] password_regexp value of '~p'" - " is not a list.", - [NonListValue] - ), - throw({forbidden, ?PASSWORD_SERVER_ERROR}); - {error, ErrorInfo} -> - couch_log:error( - "[couch_httpd_auth] password_regexp value of '~p'" - " could not get parsed. ~p", - [ValidateConfig, ErrorInfo] - ), - throw({forbidden, ?PASSWORD_SERVER_ERROR}) - end, - % Check the password on every RegExp. - lists:foreach(fun(RegExpTuple) -> - case get_password_regexp_and_error_msg(RegExpTuple) of - {ok, RegExp, PasswordErrorMsg} -> - check_password(ClearPassword, RegExp, PasswordErrorMsg); - {error} -> + RequirementList = + case couch_util:parse_term(ValidateConfig) of + {ok, RegExpList} when is_list(RegExpList) -> + RegExpList; + {ok, NonListValue} -> + couch_log:error( + "[couch_httpd_auth] password_regexp value of '~p'" + " is not a list.", + [NonListValue] + ), + throw({forbidden, ?PASSWORD_SERVER_ERROR}); + {error, ErrorInfo} -> couch_log:error( - "[couch_httpd_auth] password_regexp part of '~p' " - "is not a RegExp string or " - "a RegExp and Reason tuple.", - [RegExpTuple] + "[couch_httpd_auth] password_regexp value of '~p'" + " could not get parsed. ~p", + [ValidateConfig, ErrorInfo] ), throw({forbidden, ?PASSWORD_SERVER_ERROR}) - end - end, RequirementList), + end, + % Check the password on every RegExp. + lists:foreach( + fun(RegExpTuple) -> + case get_password_regexp_and_error_msg(RegExpTuple) of + {ok, RegExp, PasswordErrorMsg} -> + check_password(ClearPassword, RegExp, PasswordErrorMsg); + {error} -> + couch_log:error( + "[couch_httpd_auth] password_regexp part of '~p' " + "is not a RegExp string or " + "a RegExp and Reason tuple.", + [RegExpTuple] + ), + throw({forbidden, ?PASSWORD_SERVER_ERROR}) + end + end, + RequirementList + ), ok end. % Get the RegExp out of the tuple and combine the the error message. % First is with a Reason string. -get_password_regexp_and_error_msg({RegExp, Reason}) - when is_list(RegExp) andalso is_list(Reason) - andalso length(Reason) > 0 -> +get_password_regexp_and_error_msg({RegExp, Reason}) when + is_list(RegExp) andalso is_list(Reason) andalso + length(Reason) > 0 +-> {ok, RegExp, lists:concat([?REQUIREMENT_ERROR, " ", Reason])}; % With a not correct Reason string. get_password_regexp_and_error_msg({RegExp, _Reason}) when is_list(RegExp) -> @@ -181,36 +189,40 @@ check_password(Password, RegExp, ErrorMsg) -> % -> return doc after_doc_read(#doc{id = <<?DESIGN_DOC_PREFIX, _/binary>>} = Doc, Db) -> case (catch couch_db:check_is_admin(Db)) of - ok -> - Doc; - _ -> - throw({forbidden, - <<"Only administrators can view design docs in the users database.">>}) + ok -> + Doc; + _ -> + throw( + {forbidden, <<"Only administrators can view design docs in the users database.">>} + ) end; after_doc_read(Doc, Db) -> - #user_ctx{name=Name} = couch_db:get_user_ctx(Db), + #user_ctx{name = Name} = couch_db:get_user_ctx(Db), DocName = get_doc_name(Doc), case (catch couch_db:check_is_admin(Db)) of - ok -> - Doc; - _ when Name =:= DocName -> - Doc; - _ -> - Doc1 = strip_non_public_fields(Doc), - case Doc1 of - #doc{body={[]}} -> - throw(not_found); - _ -> - Doc1 - end + ok -> + Doc; + _ when Name =:= DocName -> + Doc; + _ -> + Doc1 = strip_non_public_fields(Doc), + case Doc1 of + #doc{body = {[]}} -> + throw(not_found); + _ -> + Doc1 + end end. -get_doc_name(#doc{id= <<"org.couchdb.user:", Name/binary>>}) -> +get_doc_name(#doc{id = <<"org.couchdb.user:", Name/binary>>}) -> Name; get_doc_name(_) -> undefined. -strip_non_public_fields(#doc{body={Props}}=Doc) -> - Public = re:split(chttpd_util:get_chttpd_auth_config("public_fields", ""), - "\\s*,\\s*", [{return, binary}]), - Doc#doc{body={[{K, V} || {K, V} <- Props, lists:member(K, Public)]}}. +strip_non_public_fields(#doc{body = {Props}} = Doc) -> + Public = re:split( + chttpd_util:get_chttpd_auth_config("public_fields", ""), + "\\s*,\\s*", + [{return, binary}] + ), + Doc#doc{body = {[{K, V} || {K, V} <- Props, lists:member(K, Public)]}}. diff --git a/src/couch/src/couch_util.erl b/src/couch/src/couch_util.erl index c7dc894d7..b7a6ad39a 100644 --- a/src/couch/src/couch_util.erl +++ b/src/couch/src/couch_util.erl @@ -15,14 +15,14 @@ -export([priv_dir/0, normpath/1, fold_files/5]). -export([should_flush/0, should_flush/1, to_existing_atom/1]). -export([rand32/0, implode/2]). --export([abs_pathname/1,abs_pathname/2, trim/1, drop_dot_couch_ext/1]). +-export([abs_pathname/1, abs_pathname/2, trim/1, drop_dot_couch_ext/1]). -export([encodeBase64Url/1, decodeBase64Url/1]). -export([validate_utf8/1, to_hex/1, parse_term/1, dict_find/3]). -export([get_nested_json_value/2, json_user_ctx/1]). -export([proplist_apply_field/2, json_apply_field/2]). -export([to_binary/1, to_integer/1, to_list/1, url_encode/1]). -export([json_encode/1, json_decode/1, json_decode/2]). --export([verify/2,simple_call/2,shutdown_sync/1]). +-export([verify/2, simple_call/2, shutdown_sync/1]). -export([get_value/2, get_value/3]). -export([reorder_results/2]). -export([url_strip_password/1]). @@ -60,7 +60,6 @@ <<"feature_flags">> ]). - priv_dir() -> case code:priv_dir(couch) of {error, bad_name} -> @@ -68,7 +67,8 @@ priv_dir() -> % renaming src/couch to src/couch. Not really worth the hassle. % -Damien code:priv_dir(couchdb); - Dir -> Dir + Dir -> + Dir end. % Normalize a pathname by removing .. and . components. @@ -84,7 +84,6 @@ normparts(["." | RestParts], Acc) -> normparts([Part | RestParts], Acc) -> normparts(RestParts, [Part | Acc]). - % This is implementation is similar the builtin filelib:fold_files/5 % except that this version will run the user supplied function % on directories that match the regular expression as well. @@ -125,13 +124,21 @@ fold_files2([File | Rest], Dir, RegExp, Recursive, Fun, Acc0) -> % works like list_to_existing_atom, except can be list or binary and it % gives you the original value instead of an error if no existing atom. to_existing_atom(V) when is_list(V) -> - try list_to_existing_atom(V) catch _:_ -> V end; + try + list_to_existing_atom(V) + catch + _:_ -> V + end; to_existing_atom(V) when is_binary(V) -> - try list_to_existing_atom(?b2l(V)) catch _:_ -> V end; + try + list_to_existing_atom(?b2l(V)) + catch + _:_ -> V + end; to_existing_atom(V) when is_atom(V) -> V. -shutdown_sync(Pid) when not is_pid(Pid)-> +shutdown_sync(Pid) when not is_pid(Pid) -> ok; shutdown_sync(Pid) -> MRef = erlang:monitor(process, Pid), @@ -139,23 +146,22 @@ shutdown_sync(Pid) -> catch unlink(Pid), catch exit(Pid, shutdown), receive - {'DOWN', MRef, _, _, _} -> - ok + {'DOWN', MRef, _, _, _} -> + ok end after erlang:demonitor(MRef, [flush]) end. - simple_call(Pid, Message) -> MRef = erlang:monitor(process, Pid), try Pid ! {self(), Message}, receive - {Pid, Result} -> - Result; - {'DOWN', MRef, _, _, Reason} -> - exit(Reason) + {Pid, Result} -> + Result; + {'DOWN', MRef, _, _, Reason} -> + exit(Reason) end after erlang:demonitor(MRef, [flush]) @@ -171,28 +177,40 @@ validate_utf8_fast(B, O) -> <<_:O/binary>> -> true; <<_:O/binary, C1, _/binary>> when - C1 < 128 -> + C1 < 128 + -> validate_utf8_fast(B, 1 + O); <<_:O/binary, C1, C2, _/binary>> when - C1 >= 194, C1 =< 223, - C2 >= 128, C2 =< 191 -> + C1 >= 194, + C1 =< 223, + C2 >= 128, + C2 =< 191 + -> validate_utf8_fast(B, 2 + O); <<_:O/binary, C1, C2, C3, _/binary>> when - C1 >= 224, C1 =< 239, - C2 >= 128, C2 =< 191, - C3 >= 128, C3 =< 191 -> + C1 >= 224, + C1 =< 239, + C2 >= 128, + C2 =< 191, + C3 >= 128, + C3 =< 191 + -> validate_utf8_fast(B, 3 + O); <<_:O/binary, C1, C2, C3, C4, _/binary>> when - C1 >= 240, C1 =< 244, - C2 >= 128, C2 =< 191, - C3 >= 128, C3 =< 191, - C4 >= 128, C4 =< 191 -> + C1 >= 240, + C1 =< 244, + C2 >= 128, + C2 =< 191, + C3 >= 128, + C3 =< 191, + C4 >= 128, + C4 =< 191 + -> validate_utf8_fast(B, 4 + O); _ -> false end. - to_hex(<<Hi:4, Lo:4, Rest/binary>>) -> [nibble_to_hex(Hi), nibble_to_hex(Lo) | to_hex(Rest)]; to_hex(<<>>) -> @@ -217,7 +235,6 @@ nibble_to_hex(13) -> $d; nibble_to_hex(14) -> $e; nibble_to_hex(15) -> $f. - parse_term(Bin) when is_binary(Bin) -> parse_term(binary_to_list(Bin)); parse_term(List) -> @@ -229,16 +246,16 @@ get_value(Key, List) -> get_value(Key, List, Default) -> case lists:keysearch(Key, 1, List) of - {value, {Key,Value}} -> - Value; - false -> - Default + {value, {Key, Value}} -> + Value; + false -> + Default end. -get_nested_json_value({Props}, [Key|Keys]) -> +get_nested_json_value({Props}, [Key | Keys]) -> case couch_util:get_value(Key, Props, nil) of - nil -> throw({not_found, <<"missing json key: ", Key/binary>>}); - Value -> get_nested_json_value(Value, Keys) + nil -> throw({not_found, <<"missing json key: ", Key/binary>>}); + Value -> get_nested_json_value(Value, Keys) end; get_nested_json_value(Value, []) -> Value; @@ -256,15 +273,16 @@ json_apply_field({Key, NewValue}, [{Key, _OldVal} | Headers], Acc) -> json_apply_field({Key, NewValue}, [{OtherKey, OtherVal} | Headers], Acc) -> json_apply_field({Key, NewValue}, Headers, [{OtherKey, OtherVal} | Acc]); json_apply_field({Key, NewValue}, [], Acc) -> - {[{Key, NewValue}|Acc]}. + {[{Key, NewValue} | Acc]}. json_user_ctx(Db) -> ShardName = couch_db:name(Db), Ctx = couch_db:get_user_ctx(Db), - {[{<<"db">>, mem3:dbname(ShardName)}, - {<<"name">>,Ctx#user_ctx.name}, - {<<"roles">>,Ctx#user_ctx.roles}]}. - + {[ + {<<"db">>, mem3:dbname(ShardName)}, + {<<"name">>, Ctx#user_ctx.name}, + {<<"roles">>, Ctx#user_ctx.roles} + ]}. % returns a random integer rand32() -> @@ -276,7 +294,7 @@ rand32() -> abs_pathname(" " ++ Filename) -> % strip leading whitspace abs_pathname(Filename); -abs_pathname([$/ |_]=Filename) -> +abs_pathname([$/ | _] = Filename) -> Filename; abs_pathname(Filename) -> {ok, Cwd} = file:get_cwd(), @@ -287,24 +305,25 @@ abs_pathname(Filename, Dir) -> Name = filename:absname(Filename, Dir ++ "/"), OutFilename = filename:join(fix_path_list(filename:split(Name), [])), % If the filename is a dir (last char slash, put back end slash - case string:right(Filename,1) of - "/" -> - OutFilename ++ "/"; - "\\" -> - OutFilename ++ "/"; - _Else-> - OutFilename + case string:right(Filename, 1) of + "/" -> + OutFilename ++ "/"; + "\\" -> + OutFilename ++ "/"; + _Else -> + OutFilename end. % if this as an executable with arguments, seperate out the arguments % ""./foo\ bar.sh -baz=blah" -> {"./foo\ bar.sh", " -baz=blah"} separate_cmd_args("", CmdAcc) -> {lists:reverse(CmdAcc), ""}; -separate_cmd_args("\\ " ++ Rest, CmdAcc) -> % handle skipped value +% handle skipped value +separate_cmd_args("\\ " ++ Rest, CmdAcc) -> separate_cmd_args(Rest, " \\" ++ CmdAcc); separate_cmd_args(" " ++ Rest, CmdAcc) -> {lists:reverse(CmdAcc), " " ++ Rest}; -separate_cmd_args([Char|Rest], CmdAcc) -> +separate_cmd_args([Char | Rest], CmdAcc) -> separate_cmd_args(Rest, [Char | CmdAcc]). % Is a character whitespace (from https://en.wikipedia.org/wiki/Whitespace_character#Unicode)? @@ -341,7 +360,6 @@ is_whitespace(8288) -> true; is_whitespace(65279) -> true; is_whitespace(_Else) -> false. - % removes leading and trailing whitespace from a string trim(String) when is_binary(String) -> % mirror string:trim() behaviour of returning a binary when a binary is passed in @@ -350,7 +368,6 @@ trim(String) -> String2 = lists:dropwhile(fun is_whitespace/1, String), lists:reverse(lists:dropwhile(fun is_whitespace/1, lists:reverse(String2))). - drop_dot_couch_ext(DbName) when is_binary(DbName) -> PrefixLen = size(DbName) - 6, case DbName of @@ -359,48 +376,53 @@ drop_dot_couch_ext(DbName) when is_binary(DbName) -> Else -> Else end; - drop_dot_couch_ext(DbName) when is_list(DbName) -> binary_to_list(drop_dot_couch_ext(iolist_to_binary(DbName))). - % takes a heirarchical list of dirs and removes the dots ".", double dots % ".." and the corresponding parent dirs. fix_path_list([], Acc) -> lists:reverse(Acc); -fix_path_list([".."|Rest], [_PrevAcc|RestAcc]) -> +fix_path_list([".." | Rest], [_PrevAcc | RestAcc]) -> fix_path_list(Rest, RestAcc); -fix_path_list(["."|Rest], Acc) -> +fix_path_list(["." | Rest], Acc) -> fix_path_list(Rest, Acc); fix_path_list([Dir | Rest], Acc) -> fix_path_list(Rest, [Dir | Acc]). - implode(List, Sep) -> implode(List, Sep, []). implode([], _Sep, Acc) -> lists:flatten(lists:reverse(Acc)); implode([H], Sep, Acc) -> - implode([], Sep, [H|Acc]); -implode([H|T], Sep, Acc) -> - implode(T, Sep, [Sep,H|Acc]). - + implode([], Sep, [H | Acc]); +implode([H | T], Sep, Acc) -> + implode(T, Sep, [Sep, H | Acc]). should_flush() -> should_flush(?FLUSH_MAX_MEM). should_flush(MemThreshHold) -> {memory, ProcMem} = process_info(self(), memory), - BinMem = lists:foldl(fun({_Id, Size, _NRefs}, Acc) -> Size+Acc end, - 0, element(2,process_info(self(), binary))), - if ProcMem+BinMem > 2*MemThreshHold -> - garbage_collect(), - {memory, ProcMem2} = process_info(self(), memory), - BinMem2 = lists:foldl(fun({_Id, Size, _NRefs}, Acc) -> Size+Acc end, - 0, element(2,process_info(self(), binary))), - ProcMem2+BinMem2 > MemThreshHold; - true -> false end. + BinMem = lists:foldl( + fun({_Id, Size, _NRefs}, Acc) -> Size + Acc end, + 0, + element(2, process_info(self(), binary)) + ), + if + ProcMem + BinMem > 2 * MemThreshHold -> + garbage_collect(), + {memory, ProcMem2} = process_info(self(), memory), + BinMem2 = lists:foldl( + fun({_Id, Size, _NRefs}, Acc) -> Size + Acc end, + 0, + element(2, process_info(self(), binary)) + ), + ProcMem2 + BinMem2 > MemThreshHold; + true -> + false + end. encodeBase64Url(Url) -> b64url:encode(Url). @@ -410,10 +432,10 @@ decodeBase64Url(Url64) -> dict_find(Key, Dict, DefaultValue) -> case dict:find(Key, Dict) of - {ok, Value} -> - Value; - error -> - DefaultValue + {ok, Value} -> + Value; + error -> + DefaultValue end. to_binary(V) when is_binary(V) -> @@ -448,23 +470,23 @@ to_list(V) -> url_encode(Bin) when is_binary(Bin) -> url_encode(binary_to_list(Bin)); -url_encode([H|T]) -> +url_encode([H | T]) -> if - H >= $a, $z >= H -> - [H|url_encode(T)]; - H >= $A, $Z >= H -> - [H|url_encode(T)]; - H >= $0, $9 >= H -> - [H|url_encode(T)]; - H == $_; H == $.; H == $-; H == $: -> - [H|url_encode(T)]; - true -> - case lists:flatten(io_lib:format("~.16.0B", [H])) of - [X, Y] -> - [$%, X, Y | url_encode(T)]; - [X] -> - [$%, $0, X | url_encode(T)] - end + H >= $a, $z >= H -> + [H | url_encode(T)]; + H >= $A, $Z >= H -> + [H | url_encode(T)]; + H >= $0, $9 >= H -> + [H | url_encode(T)]; + H == $_; H == $.; H == $-; H == $: -> + [H | url_encode(T)]; + true -> + case lists:flatten(io_lib:format("~.16.0B", [H])) of + [X, Y] -> + [$%, X, Y | url_encode(T)]; + [X] -> + [$%, $0, X | url_encode(T)] + end end; url_encode([]) -> []. @@ -483,7 +505,7 @@ json_decode(V, Opts) -> throw({invalid_json, Error}) end. -verify([X|RestX], [Y|RestY], Result) -> +verify([X | RestX], [Y | RestY], Result) -> verify(RestX, RestY, (X bxor Y) bor Result); verify([], [], Result) -> Result == 0. @@ -497,7 +519,8 @@ verify(X, Y) when is_list(X) and is_list(Y) -> false -> false end; -verify(_X, _Y) -> false. +verify(_X, _Y) -> + false. % linear search is faster for small lists, length() is 0.5 ms for 100k list reorder_results(Keys, SortedResults) when length(Keys) < 100 -> @@ -507,10 +530,12 @@ reorder_results(Keys, SortedResults) -> [dict:fetch(Key, KeyDict) || Key <- Keys]. url_strip_password(Url) -> - re:replace(Url, + re:replace( + Url, "(http|https|socks5)://([^:]+):[^@]+@(.*)$", "\\1://\\2:*****@\\3", - [{return, list}]). + [{return, list}] + ). encode_doc_id(#doc{id = Id}) -> encode_doc_id(Id); @@ -528,7 +553,7 @@ normalize_ddoc_id(<<"_design/", _/binary>> = DDocId) -> normalize_ddoc_id(DDocId) when is_binary(DDocId) -> <<"_design/", DDocId/binary>>. -with_db(DbName, Fun) when is_binary(DbName) -> +with_db(DbName, Fun) when is_binary(DbName) -> case couch_db:open_int(DbName, [?ADMIN_CTX]) of {ok, Db} -> try @@ -548,20 +573,26 @@ with_db(Db, Fun) -> end. rfc1123_date() -> - {{YYYY,MM,DD},{Hour,Min,Sec}} = calendar:universal_time(), - DayNumber = calendar:day_of_the_week({YYYY,MM,DD}), + {{YYYY, MM, DD}, {Hour, Min, Sec}} = calendar:universal_time(), + DayNumber = calendar:day_of_the_week({YYYY, MM, DD}), lists:flatten( - io_lib:format("~s, ~2.2.0w ~3.s ~4.4.0w ~2.2.0w:~2.2.0w:~2.2.0w GMT", - [day(DayNumber),DD,month(MM),YYYY,Hour,Min,Sec])). + io_lib:format( + "~s, ~2.2.0w ~3.s ~4.4.0w ~2.2.0w:~2.2.0w:~2.2.0w GMT", + [day(DayNumber), DD, month(MM), YYYY, Hour, Min, Sec] + ) + ). rfc1123_date(undefined) -> undefined; rfc1123_date(UniversalTime) -> - {{YYYY,MM,DD},{Hour,Min,Sec}} = UniversalTime, - DayNumber = calendar:day_of_the_week({YYYY,MM,DD}), + {{YYYY, MM, DD}, {Hour, Min, Sec}} = UniversalTime, + DayNumber = calendar:day_of_the_week({YYYY, MM, DD}), lists:flatten( - io_lib:format("~s, ~2.2.0w ~3.s ~4.4.0w ~2.2.0w:~2.2.0w:~2.2.0w GMT", - [day(DayNumber),DD,month(MM),YYYY,Hour,Min,Sec])). + io_lib:format( + "~s, ~2.2.0w ~3.s ~4.4.0w ~2.2.0w:~2.2.0w:~2.2.0w GMT", + [day(DayNumber), DD, month(MM), YYYY, Hour, Min, Sec] + ) + ). %% day @@ -598,30 +629,32 @@ boolean_to_integer(true) -> boolean_to_integer(false) -> 0. - validate_positive_int(N) when is_list(N) -> try I = list_to_integer(N), validate_positive_int(I) - catch error:badarg -> - false + catch + error:badarg -> + false end; validate_positive_int(N) when is_integer(N), N > 0 -> true; -validate_positive_int(_) -> false. - +validate_positive_int(_) -> + false. find_in_binary(_B, <<>>) -> not_found; - find_in_binary(B, Data) -> case binary:match(Data, [B], []) of - nomatch -> - MatchLength = erlang:min(byte_size(B), byte_size(Data)), - match_prefix_at_end(binary:part(B, {0, MatchLength}), - binary:part(Data, {byte_size(Data), -MatchLength}), - MatchLength, byte_size(Data) - MatchLength); - {Pos, _Len} -> - {exact, Pos} + nomatch -> + MatchLength = erlang:min(byte_size(B), byte_size(Data)), + match_prefix_at_end( + binary:part(B, {0, MatchLength}), + binary:part(Data, {byte_size(Data), -MatchLength}), + MatchLength, + byte_size(Data) - MatchLength + ); + {Pos, _Len} -> + {exact, Pos} end. match_prefix_at_end(Prefix, Data, PrefixLength, N) -> @@ -630,10 +663,14 @@ match_prefix_at_end(Prefix, Data, PrefixLength, N) -> match_rest_of_prefix([], _Prefix, _Data, _PrefixLength, _N) -> not_found; - match_rest_of_prefix([{Pos, _Len} | Rest], Prefix, Data, PrefixLength, N) -> - case binary:match(binary:part(Data, {PrefixLength, Pos - PrefixLength}), - [binary:part(Prefix, {0, PrefixLength - Pos})], []) of + case + binary:match( + binary:part(Data, {PrefixLength, Pos - PrefixLength}), + [binary:part(Prefix, {0, PrefixLength - Pos})], + [] + ) + of nomatch -> match_rest_of_prefix(Rest, Prefix, Data, PrefixLength, N); {_Pos, _Len1} -> @@ -642,44 +679,42 @@ match_rest_of_prefix([{Pos, _Len} | Rest], Prefix, Data, PrefixLength, N) -> callback_exists(Module, Function, Arity) -> case ensure_loaded(Module) of - true -> - InfoList = Module:module_info(exports), - lists:member({Function, Arity}, InfoList); - false -> - false + true -> + InfoList = Module:module_info(exports), + lists:member({Function, Arity}, InfoList); + false -> + false end. validate_callback_exists(Module, Function, Arity) -> case callback_exists(Module, Function, Arity) of - true -> - ok; - false -> - CallbackStr = lists:flatten( - io_lib:format("~w:~w/~w", [Module, Function, Arity])), - throw({error, - {undefined_callback, CallbackStr, {Module, Function, Arity}}}) + true -> + ok; + false -> + CallbackStr = lists:flatten( + io_lib:format("~w:~w/~w", [Module, Function, Arity]) + ), + throw({error, {undefined_callback, CallbackStr, {Module, Function, Arity}}}) end. - check_md5(_NewSig, <<>>) -> ok; check_md5(Sig, Sig) -> ok; check_md5(_, _) -> throw(md5_mismatch). - set_mqd_off_heap(Module) -> case config:get_boolean("off_heap_mqd", atom_to_list(Module), true) of true -> try erlang:process_flag(message_queue_data, off_heap), ok - catch error:badarg -> + catch + error:badarg -> ok end; false -> ok end. - set_process_priority(Module, Level) -> case config:get_boolean("process_priority", atom_to_list(Module), false) of true -> @@ -689,18 +724,17 @@ set_process_priority(Module, Level) -> ok end. - ensure_loaded(Module) when is_atom(Module) -> case code:ensure_loaded(Module) of - {module, Module} -> - true; - {error, embedded} -> - true; - {error, _} -> - false + {module, Module} -> + true; + {error, embedded} -> + true; + {error, _} -> + false end; -ensure_loaded(_Module) -> false. - +ensure_loaded(_Module) -> + false. %% This is especially useful in gen_servers when you need to call %% a function that does a receive as it would hijack incoming messages. @@ -718,11 +752,9 @@ with_proc(M, F, A, Timeout) -> {error, timeout} end. - process_dict_get(Pid, Key) -> process_dict_get(Pid, Key, undefined). - process_dict_get(Pid, Key, DefaultValue) -> case process_info(Pid, dictionary) of {dictionary, Dict} -> @@ -736,21 +768,18 @@ process_dict_get(Pid, Key, DefaultValue) -> DefaultValue end. - unique_monotonic_integer() -> erlang:unique_integer([monotonic, positive]). - check_config_blacklist(Section) -> case lists:member(Section, ?BLACKLIST_CONFIG_SECTIONS) of - true -> - Msg = <<"Config section blacklisted for modification over HTTP API.">>, - throw({forbidden, Msg}); - _ -> - ok + true -> + Msg = <<"Config section blacklisted for modification over HTTP API.">>, + throw({forbidden, Msg}); + _ -> + ok end. - -ifdef(OTP_RELEASE). -if(?OTP_RELEASE >= 22). @@ -765,7 +794,8 @@ hmac(Alg, Key, Message) -> hmac(Alg, Key, Message) -> crypto:hmac(Alg, Key, Message). --endif. % -if(?OTP_RELEASE >= 22) +% -if(?OTP_RELEASE >= 22) +-endif. -else. @@ -773,4 +803,5 @@ hmac(Alg, Key, Message) -> hmac(Alg, Key, Message) -> crypto:hmac(Alg, Key, Message). --endif. % -ifdef(OTP_RELEASE) +% -ifdef(OTP_RELEASE) +-endif. diff --git a/src/couch/src/couch_uuids.erl b/src/couch/src/couch_uuids.erl index 3fffd04b3..be6089dff 100644 --- a/src/couch/src/couch_uuids.erl +++ b/src/couch/src/couch_uuids.erl @@ -127,23 +127,22 @@ utc_random(ClockSeq) -> utc_suffix(Suffix, ClockSeq, Now) -> OsMicros = micros_since_epoch(Now), - NewClockSeq = if - OsMicros =< ClockSeq -> - % Timestamp is lagging, use ClockSeq as Timestamp - ClockSeq + 1; - OsMicros > ClockSeq -> - % Timestamp advanced, use it, and reset ClockSeq with it - OsMicros - end, + NewClockSeq = + if + OsMicros =< ClockSeq -> + % Timestamp is lagging, use ClockSeq as Timestamp + ClockSeq + 1; + OsMicros > ClockSeq -> + % Timestamp advanced, use it, and reset ClockSeq with it + OsMicros + end, Prefix = io_lib:format("~14.16.0b", [NewClockSeq]), {list_to_binary(Prefix ++ Suffix), NewClockSeq}. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). - utc_id_time_does_not_advance_test() -> % Timestamp didn't advance but local clock sequence should and new UUIds % should be generated @@ -156,7 +155,6 @@ utc_id_time_does_not_advance_test() -> ?assertNotEqual(UtcId0, UtcId1), ?assertEqual(ClockSeq1 + 1, ClockSeq2). - utc_id_time_advanced_test() -> % Timestamp advanced, a new UUID generated and also the last clock sequence % is updated to that timestamp. @@ -187,5 +185,4 @@ utc_random_test_time_advance_test() -> ?assertEqual(32, byte_size(UtcRandom)), ?assert(NextClockSeq > micros_since_epoch({1000, 0, 0})). - -endif. diff --git a/src/couch/src/couch_work_queue.erl b/src/couch/src/couch_work_queue.erl index 5d747de82..d767a33be 100644 --- a/src/couch/src/couch_work_queue.erl +++ b/src/couch/src/couch_work_queue.erl @@ -35,21 +35,17 @@ multi_workers = false }). - new(Options) -> gen_server:start_link(couch_work_queue, Options, []). - queue(Wq, Item) when is_binary(Item) -> gen_server:call(Wq, {queue, Item, byte_size(Item)}, infinity); queue(Wq, Item) -> gen_server:call(Wq, {queue, Item, ?term_size(Item)}, infinity). - dequeue(Wq) -> dequeue(Wq, all). - dequeue(Wq, MaxItems) -> try gen_server:call(Wq, {dequeue, MaxItems}, infinity) @@ -57,7 +53,6 @@ dequeue(Wq, MaxItems) -> _:_ -> closed end. - item_count(Wq) -> try gen_server:call(Wq, item_count, infinity) @@ -65,7 +60,6 @@ item_count(Wq) -> _:_ -> closed end. - size(Wq) -> try gen_server:call(Wq, size, infinity) @@ -73,10 +67,8 @@ size(Wq) -> _:_ -> closed end. - close(Wq) -> gen_server:cast(Wq, close). - init(Options) -> Q = #q{ @@ -86,50 +78,47 @@ init(Options) -> }, {ok, Q, hibernate}. - -terminate(_Reason, #q{work_waiters=Workers}) -> +terminate(_Reason, #q{work_waiters = Workers}) -> lists:foreach(fun({W, _}) -> gen_server:reply(W, closed) end, Workers). - handle_call({queue, Item, Size}, From, #q{work_waiters = []} = Q0) -> - Q = Q0#q{size = Q0#q.size + Size, - items = Q0#q.items + 1, - queue = queue:in({Item, Size}, Q0#q.queue)}, - case (Q#q.size >= Q#q.max_size) orelse - (Q#q.items >= Q#q.max_items) of - true -> - {noreply, Q#q{blocked = [From | Q#q.blocked]}, hibernate}; - false -> - {reply, ok, Q, hibernate} + Q = Q0#q{ + size = Q0#q.size + Size, + items = Q0#q.items + 1, + queue = queue:in({Item, Size}, Q0#q.queue) + }, + case + (Q#q.size >= Q#q.max_size) orelse + (Q#q.items >= Q#q.max_items) + of + true -> + {noreply, Q#q{blocked = [From | Q#q.blocked]}, hibernate}; + false -> + {reply, ok, Q, hibernate} end; - handle_call({queue, Item, _}, _From, #q{work_waiters = [{W, _Max} | Rest]} = Q) -> gen_server:reply(W, {ok, [Item]}), {reply, ok, Q#q{work_waiters = Rest}, hibernate}; - handle_call({dequeue, Max}, From, Q) -> #q{work_waiters = Workers, multi_workers = Multi, items = Count} = Q, case {Workers, Multi} of - {[_ | _], false} -> - exit("Only one caller allowed to wait for this work at a time"); - {[_ | _], true} -> - {noreply, Q#q{work_waiters=Workers ++ [{From, Max}]}}; - _ -> - case Count of - 0 -> - {noreply, Q#q{work_waiters=Workers ++ [{From, Max}]}}; - C when C > 0 -> - deliver_queue_items(Max, Q) - end + {[_ | _], false} -> + exit("Only one caller allowed to wait for this work at a time"); + {[_ | _], true} -> + {noreply, Q#q{work_waiters = Workers ++ [{From, Max}]}}; + _ -> + case Count of + 0 -> + {noreply, Q#q{work_waiters = Workers ++ [{From, Max}]}}; + C when C > 0 -> + deliver_queue_items(Max, Q) + end end; - handle_call(item_count, _From, Q) -> {reply, Q#q.items, Q}; - handle_call(size, _From, Q) -> {reply, Q#q.size, Q}. - deliver_queue_items(Max, Q) -> #q{ queue = Queue, @@ -139,48 +128,45 @@ deliver_queue_items(Max, Q) -> blocked = Blocked } = Q, case (Max =:= all) orelse (Max >= Count) of - false -> - {Items, Size2, Queue2, Blocked2} = dequeue_items( - Max, Size, Queue, Blocked, []), - Q2 = Q#q{ - items = Count - Max, size = Size2, blocked = Blocked2, queue = Queue2 - }, - {reply, {ok, Items}, Q2}; - true -> - lists:foreach(fun(F) -> gen_server:reply(F, ok) end, Blocked), - Q2 = Q#q{items = 0, size = 0, blocked = [], queue = queue:new()}, - Items = [Item || {Item, _} <- queue:to_list(Queue)], - case Close of false -> + {Items, Size2, Queue2, Blocked2} = dequeue_items( + Max, Size, Queue, Blocked, [] + ), + Q2 = Q#q{ + items = Count - Max, size = Size2, blocked = Blocked2, queue = Queue2 + }, {reply, {ok, Items}, Q2}; true -> - {stop, normal, {ok, Items}, Q2} - end + lists:foreach(fun(F) -> gen_server:reply(F, ok) end, Blocked), + Q2 = Q#q{items = 0, size = 0, blocked = [], queue = queue:new()}, + Items = [Item || {Item, _} <- queue:to_list(Queue)], + case Close of + false -> + {reply, {ok, Items}, Q2}; + true -> + {stop, normal, {ok, Items}, Q2} + end end. - dequeue_items(0, Size, Queue, Blocked, DequeuedAcc) -> {lists:reverse(DequeuedAcc), Size, Queue, Blocked}; - dequeue_items(NumItems, Size, Queue, Blocked, DequeuedAcc) -> {{value, {Item, ItemSize}}, Queue2} = queue:out(Queue), case Blocked of - [] -> - Blocked2 = Blocked; - [From | Blocked2] -> - gen_server:reply(From, ok) + [] -> + Blocked2 = Blocked; + [From | Blocked2] -> + gen_server:reply(From, ok) end, dequeue_items( - NumItems - 1, Size - ItemSize, Queue2, Blocked2, [Item | DequeuedAcc]). - + NumItems - 1, Size - ItemSize, Queue2, Blocked2, [Item | DequeuedAcc] + ). handle_cast(close, #q{items = 0} = Q) -> {stop, normal, Q}; - handle_cast(close, Q) -> {noreply, Q#q{close_on_dequeue = true}}. - code_change(_OldVsn, State, _Extra) -> {ok, State}. diff --git a/src/couch/src/test_request.erl b/src/couch/src/test_request.erl index 48f49bda6..d7364012f 100644 --- a/src/couch/src/test_request.erl +++ b/src/couch/src/test_request.erl @@ -74,7 +74,6 @@ options(Url, Headers) -> options(Url, Headers, Opts) -> request(options, Url, Headers, [], Opts). - request(Method, Url, Headers) -> request(Method, Url, Headers, []). diff --git a/src/couch/src/test_util.erl b/src/couch/src/test_util.erl index 125e76492..4b802bc49 100644 --- a/src/couch/src/test_util.erl +++ b/src/couch/src/test_util.erl @@ -40,8 +40,7 @@ -record(test_context, {mocked = [], started = [], module}). --define(DEFAULT_APPS, - [inets, ibrowse, ssl, config, couch_epi, couch_event, couch]). +-define(DEFAULT_APPS, [inets, ibrowse, ssl, config, couch_epi, couch_event, couch]). srcdir() -> code:priv_dir(couch) ++ "/../../". @@ -57,9 +56,12 @@ init_code_path() -> "mochiweb", "snappy" ], - lists:foreach(fun(Name) -> - code:add_patha(filename:join([builddir(), "src", Name])) - end, Paths). + lists:foreach( + fun(Name) -> + code:add_patha(filename:join([builddir(), "src", Name])) + end, + Paths + ). source_file(Name) -> filename:join([srcdir(), Name]). @@ -94,21 +96,21 @@ start_applications(Apps) -> start_applications([], Acc) -> lists:reverse(Acc); -start_applications([App|Apps], Acc) when App == kernel; App == stdlib -> +start_applications([App | Apps], Acc) when App == kernel; App == stdlib -> start_applications(Apps, Acc); -start_applications([App|Apps], Acc) -> +start_applications([App | Apps], Acc) -> case application:start(App) of - {error, {already_started, crypto}} -> - start_applications(Apps, [crypto | Acc]); - {error, {already_started, App}} -> - io:format(standard_error, "Application ~s was left running!~n", [App]), - application:stop(App), - start_applications([App|Apps], Acc); - {error, Reason} -> - io:format(standard_error, "Cannot start application '~s', reason ~p~n", [App, Reason]), - throw({error, {cannot_start, App, Reason}}); - ok -> - start_applications(Apps, [App|Acc]) + {error, {already_started, crypto}} -> + start_applications(Apps, [crypto | Acc]); + {error, {already_started, App}} -> + io:format(standard_error, "Application ~s was left running!~n", [App]), + application:stop(App), + start_applications([App | Apps], Acc); + {error, Reason} -> + io:format(standard_error, "Cannot start application '~s', reason ~p~n", [App, Reason]), + throw({error, {cannot_start, App, Reason}}); + ok -> + start_applications(Apps, [App | Acc]) end. stop_applications(Apps) -> @@ -119,12 +121,11 @@ start_config(Chain) -> case config:start_link(Chain) of {ok, Pid} -> {ok, Pid}; - {error, {already_started, OldPid}} -> + {error, {already_started, OldPid}} -> ok = stop_config(OldPid), start_config(Chain) end. - stop_config(Pid) -> Timeout = 1000, case stop_sync(Pid, fun() -> config:stop() end, Timeout) of @@ -150,8 +151,8 @@ stop_sync(Pid, Fun, Timeout) when is_function(Fun) and is_pid(Pid) -> catch unlink(Pid), Res = (catch Fun()), receive - {'DOWN', MRef, _, _, _} -> - Res + {'DOWN', MRef, _, _, _} -> + Res after Timeout -> timeout end @@ -159,7 +160,8 @@ stop_sync(Pid, Fun, Timeout) when is_function(Fun) and is_pid(Pid) -> after erlang:demonitor(MRef, [flush]) end; -stop_sync(_, _, _) -> error(badarg). +stop_sync(_, _, _) -> + error(badarg). stop_sync_throw(Name, Error) -> stop_sync_throw(Name, shutdown, Error). @@ -176,7 +178,8 @@ stop_sync_throw(Pid, Fun, Error, Timeout) -> with_process_restart(Name) -> {Pid, true} = with_process_restart( - Name, fun() -> exit(whereis(Name), shutdown) end), + Name, fun() -> exit(whereis(Name), shutdown) end + ), Pid. with_process_restart(Name, Fun) -> @@ -185,24 +188,26 @@ with_process_restart(Name, Fun) -> with_process_restart(Name, Fun, Timeout) -> Res = stop_sync(Name, Fun), case wait_process(Name, Timeout) of - timeout -> - timeout; - Pid -> - {Pid, Res} + timeout -> + timeout; + Pid -> + {Pid, Res} end. - wait_process(Name) -> wait_process(Name, 5000). wait_process(Name, Timeout) -> - wait(fun() -> - case whereis(Name) of - undefined -> - wait; - Pid -> - Pid - end - end, Timeout). + wait( + fun() -> + case whereis(Name) of + undefined -> + wait; + Pid -> + Pid + end + end, + Timeout + ). wait(Fun) -> wait(Fun, 5000, 50). @@ -218,11 +223,11 @@ wait(_Fun, Timeout, _Delay, Started, Prev) when Prev - Started > Timeout -> timeout; wait(Fun, Timeout, Delay, Started, _Prev) -> case Fun() of - wait -> - ok = timer:sleep(Delay), - wait(Fun, Timeout, Delay, Started, now_us()); - Else -> - Else + wait -> + ok = timer:sleep(Delay), + wait(Fun, Timeout, Delay, Started, now_us()); + Else -> + Else end. wait_value(Fun, Value) -> @@ -260,13 +265,17 @@ stop(#test_context{mocked = Mocked, started = Apps}) -> fake_db(Fields0) -> {ok, Db, Fields} = maybe_set_engine(Fields0), Indexes = lists:zip( - record_info(fields, db), - lists:seq(2, record_info(size, db)) - ), - lists:foldl(fun({FieldName, Value}, Acc) -> - Idx = couch_util:get_value(FieldName, Indexes), - setelement(Idx, Acc, Value) - end, Db, Fields). + record_info(fields, db), + lists:seq(2, record_info(size, db)) + ), + lists:foldl( + fun({FieldName, Value}, Acc) -> + Idx = couch_util:get_value(FieldName, Indexes), + setelement(Idx, Acc, Value) + end, + Db, + Fields + ). maybe_set_engine(Fields0) -> case lists:member(engine, Fields0) of @@ -279,11 +288,24 @@ maybe_set_engine(Fields0) -> end. get_engine_header(Fields) -> - Keys = [disk_version, update_seq, unused, id_tree_state, - seq_tree_state, local_tree_state, purge_seq, purged_docs, - security_ptr, revs_limit, uuid, epochs, compacted_seq], + Keys = [ + disk_version, + update_seq, + unused, + id_tree_state, + seq_tree_state, + local_tree_state, + purge_seq, + purged_docs, + security_ptr, + revs_limit, + uuid, + epochs, + compacted_seq + ], {HeadFields, RestFields} = lists:partition( - fun({K, _}) -> lists:member(K, Keys) end, Fields), + fun({K, _}) -> lists:member(K, Keys) end, Fields + ), Header0 = couch_bt_engine_header:new(), Header = couch_bt_engine_header:set(Header0, HeadFields), {ok, Header, RestFields}. @@ -315,7 +337,7 @@ load_applications_with_stats() -> ok. stats_file_to_app(File) -> - [_Desc, _Priv, App|_] = lists:reverse(filename:split(File)), + [_Desc, _Priv, App | _] = lists:reverse(filename:split(File)), erlang:list_to_atom(App). calculate_start_order(Apps) -> @@ -345,14 +367,19 @@ load_app_deps(App, StartOrder) -> {error, {already_loaded, App}} -> ok end, {ok, Apps} = application:get_key(App, applications), - Deps = case App of - kernel -> Apps; - stdlib -> Apps; - _ -> lists:usort([kernel, stdlib | Apps]) - end, - NewStartOrder = lists:foldl(fun(Dep, Acc) -> - load_app_deps(Dep, Acc) - end, StartOrder, Deps), + Deps = + case App of + kernel -> Apps; + stdlib -> Apps; + _ -> lists:usort([kernel, stdlib | Apps]) + end, + NewStartOrder = lists:foldl( + fun(Dep, Acc) -> + load_app_deps(Dep, Acc) + end, + StartOrder, + Deps + ), [App | NewStartOrder] end. diff --git a/src/couch/test/eunit/chttpd_endpoints_tests.erl b/src/couch/test/eunit/chttpd_endpoints_tests.erl index 3c8586a14..63f67c243 100644 --- a/src/couch/test/eunit/chttpd_endpoints_tests.erl +++ b/src/couch/test/eunit/chttpd_endpoints_tests.erl @@ -15,7 +15,6 @@ -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). - endpoints_test_() -> { "Checking dynamic endpoints", @@ -33,7 +32,6 @@ endpoints_test_() -> } }. - url_handlers() -> Handlers = [ {<<"">>, chttpd_misc, handle_welcome_req}, @@ -53,15 +51,17 @@ url_handlers() -> {<<"_cluster_setup">>, setup_httpd, handle_setup_req} ], - lists:foreach(fun({Path, Mod, Fun}) -> - Handler = chttpd_handlers:url_handler(Path, undefined), - Expect = fun Mod:Fun/1, - ?assertEqual(Expect, Handler) - end, Handlers), + lists:foreach( + fun({Path, Mod, Fun}) -> + Handler = chttpd_handlers:url_handler(Path, undefined), + Expect = fun Mod:Fun/1, + ?assertEqual(Expect, Handler) + end, + Handlers + ), ?assertEqual(undefined, chttpd_handlers:url_handler("foo", undefined)). - db_handlers() -> Handlers = [ {<<"_view_cleanup">>, chttpd_db, handle_view_cleanup_req}, @@ -75,15 +75,17 @@ db_handlers() -> {<<"_find">>, mango_httpd, handle_req} ], - lists:foreach(fun({Path, Mod, Fun}) -> - Handler = chttpd_handlers:db_handler(Path, undefined), - Expect = fun Mod:Fun/2, - ?assertEqual(Expect, Handler) - end, Handlers), + lists:foreach( + fun({Path, Mod, Fun}) -> + Handler = chttpd_handlers:db_handler(Path, undefined), + Expect = fun Mod:Fun/2, + ?assertEqual(Expect, Handler) + end, + Handlers + ), ?assertEqual(undefined, chttpd_handlers:db_handler("bam", undefined)). - design_handlers() -> Handlers = [ {<<"_view">>, chttpd_view, handle_view_req}, @@ -94,10 +96,13 @@ design_handlers() -> {<<"_rewrite">>, chttpd_rewrite, handle_rewrite_req} ], - lists:foreach(fun({Path, Mod, Fun}) -> - Handler = chttpd_handlers:design_handler(Path, undefined), - Expect = fun Mod:Fun/3, - ?assertEqual(Expect, Handler) - end, Handlers), + lists:foreach( + fun({Path, Mod, Fun}) -> + Handler = chttpd_handlers:design_handler(Path, undefined), + Expect = fun Mod:Fun/3, + ?assertEqual(Expect, Handler) + end, + Handlers + ), ?assertEqual(undefined, chttpd_handlers:design_handler("baz", undefined)). diff --git a/src/couch/test/eunit/couch_auth_cache_tests.erl b/src/couch/test/eunit/couch_auth_cache_tests.erl index 71faf77d6..a4c31083a 100644 --- a/src/couch/test/eunit/couch_auth_cache_tests.erl +++ b/src/couch/test/eunit/couch_auth_cache_tests.erl @@ -21,27 +21,31 @@ start() -> test_util:start_couch([ioq]). - setup() -> DbName = ?tempdb(), - config:set("couch_httpd_auth", "authentication_db", - ?b2l(DbName), false), + config:set( + "couch_httpd_auth", + "authentication_db", + ?b2l(DbName), + false + ), DbName. teardown(DbName) -> ok = couch_server:delete(DbName, [?ADMIN_CTX]), ok. - couch_auth_cache_test_() -> { "CouchDB auth cache tests", { setup, - fun start/0, fun test_util:stop_couch/1, + fun start/0, + fun test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_get_nil_on_missed_cache/1, fun should_get_right_password_hash/1, @@ -120,18 +124,18 @@ auth_vdu_test_() -> [missing, user, other] ]), AllPossibleCases = couch_tests_combinatorics:product( - [AllPossibleDocs, AllPossibleDocs]), + [AllPossibleDocs, AllPossibleDocs] + ), ?assertEqual([], AllPossibleCases -- [[A, B] || {A, B, _} <- Cases]), { "Check User doc validation", { setup, - fun test_util:start_couch/0, fun test_util:stop_couch/1, - [ - make_validate_test(Case) || Case <- Cases - ] - } + fun test_util:start_couch/0, + fun test_util:stop_couch/1, + [make_validate_test(Case) || Case <- Cases] + } }. should_get_nil_on_missed_cache(_) -> @@ -142,8 +146,10 @@ should_get_right_password_hash(DbName) -> PasswordHash = hash_password("pass1"), {ok, _} = update_user_doc(DbName, "joe", "pass1"), {ok, Creds, _} = couch_auth_cache:get_user_creds("joe"), - ?assertEqual(PasswordHash, - couch_util:get_value(<<"password_sha">>, Creds)) + ?assertEqual( + PasswordHash, + couch_util:get_value(<<"password_sha">>, Creds) + ) end). should_ensure_doc_hash_equals_cached_one(DbName) -> @@ -162,8 +168,10 @@ should_update_password(DbName) -> {ok, Rev} = update_user_doc(DbName, "joe", "pass1"), {ok, _} = update_user_doc(DbName, "joe", "pass2", Rev), {ok, Creds, _} = couch_auth_cache:get_user_creds("joe"), - ?assertEqual(PasswordHash, - couch_util:get_value(<<"password_sha">>, Creds)) + ?assertEqual( + PasswordHash, + couch_util:get_value(<<"password_sha">>, Creds) + ) end). should_cleanup_cache_after_userdoc_deletion(DbName) -> @@ -183,15 +191,21 @@ should_restore_cache_after_userdoc_recreation(DbName) -> {ok, _} = update_user_doc(DbName, "joe", "pass5"), {ok, Creds, _} = couch_auth_cache:get_user_creds("joe"), - ?assertEqual(PasswordHash, - couch_util:get_value(<<"password_sha">>, Creds)) + ?assertEqual( + PasswordHash, + couch_util:get_value(<<"password_sha">>, Creds) + ) end). should_drop_cache_on_auth_db_change(DbName) -> ?_test(begin {ok, _} = update_user_doc(DbName, "joe", "pass1"), - config:set("couch_httpd_auth", "authentication_db", - ?b2l(?tempdb()), false), + config:set( + "couch_httpd_auth", + "authentication_db", + ?b2l(?tempdb()), + false + ), ?assertEqual(nil, couch_auth_cache:get_user_creds("joe")) end). @@ -202,17 +216,27 @@ should_restore_cache_on_auth_db_change(DbName) -> {ok, Creds, _} = couch_auth_cache:get_user_creds("joe"), DbName1 = ?tempdb(), - config:set("couch_httpd_auth", "authentication_db", - ?b2l(DbName1), false), + config:set( + "couch_httpd_auth", + "authentication_db", + ?b2l(DbName1), + false + ), {ok, _} = update_user_doc(DbName1, "joe", "pass5"), - config:set("couch_httpd_auth", "authentication_db", - ?b2l(DbName), false), + config:set( + "couch_httpd_auth", + "authentication_db", + ?b2l(DbName), + false + ), {ok, Creds, _} = couch_auth_cache:get_user_creds("joe"), - ?assertEqual(PasswordHash, - couch_util:get_value(<<"password_sha">>, Creds)) + ?assertEqual( + PasswordHash, + couch_util:get_value(<<"password_sha">>, Creds) + ) end). should_recover_cache_after_shutdown(DbName) -> @@ -225,7 +249,6 @@ should_recover_cache_after_shutdown(DbName) -> ?assertEqual(PasswordHash, get_user_doc_password_sha(DbName, "joe")) end). - should_get_admin_from_config(_DbName) -> ?_test(begin config:set("admins", "testadmin", "password", false), @@ -245,17 +268,19 @@ update_user_doc(DbName, UserName, Password) -> update_user_doc(DbName, UserName, Password, Rev) -> ok = couch_auth_cache:ensure_users_db_exists(), User = iolist_to_binary(UserName), - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"org.couchdb.user:", User/binary>>}, - {<<"name">>, User}, - {<<"type">>, <<"user">>}, - {<<"salt">>, ?SALT}, - {<<"password_sha">>, hash_password(Password)}, - {<<"roles">>, []} - ] ++ case Rev of - nil -> []; - _ -> [{<<"_rev">>, Rev}] - end + Doc = couch_doc:from_json_obj({ + [ + {<<"_id">>, <<"org.couchdb.user:", User/binary>>}, + {<<"name">>, User}, + {<<"type">>, <<"user">>}, + {<<"salt">>, ?SALT}, + {<<"password_sha">>, hash_password(Password)}, + {<<"roles">>, []} + ] ++ + case Rev of + nil -> []; + _ -> [{<<"_rev">>, Rev}] + end }), {ok, AuthDb} = couch_db:open_int(DbName, [?ADMIN_CTX]), {ok, NewRev} = couch_db:update_doc(AuthDb, Doc, []), @@ -275,13 +300,13 @@ get_doc_rev(DbName, UserName) -> DocId = iolist_to_binary([<<"org.couchdb.user:">>, UserName]), {ok, AuthDb} = couch_db:open_int(DbName, [?ADMIN_CTX]), UpdateRev = - case couch_db:open_doc(AuthDb, DocId, []) of - {ok, Doc} -> - {Props} = couch_doc:to_json_obj(Doc, []), - couch_util:get_value(<<"_rev">>, Props); - {not_found, missing} -> - nil - end, + case couch_db:open_doc(AuthDb, DocId, []) of + {ok, Doc} -> + {Props} = couch_doc:to_json_obj(Doc, []), + couch_util:get_value(<<"_rev">>, Props); + {not_found, missing} -> + nil + end, ok = couch_db:close(AuthDb), {ok, UpdateRev}. @@ -298,15 +323,16 @@ delete_user_doc(DbName, UserName) -> {ok, AuthDb} = couch_db:open_int(DbName, [?ADMIN_CTX]), {ok, Doc} = couch_db:open_doc(AuthDb, DocId, []), {Props} = couch_doc:to_json_obj(Doc, []), - DeletedDoc = couch_doc:from_json_obj({[ - {<<"_id">>, DocId}, - {<<"_rev">>, couch_util:get_value(<<"_rev">>, Props)}, - {<<"_deleted">>, true} - ]}), + DeletedDoc = couch_doc:from_json_obj( + {[ + {<<"_id">>, DocId}, + {<<"_rev">>, couch_util:get_value(<<"_rev">>, Props)}, + {<<"_deleted">>, true} + ]} + ), {ok, _} = couch_db:update_doc(AuthDb, DeletedDoc, []), ok = couch_db:close(AuthDb). - make_validate_test({Old, New, "ok"} = Case) -> {test_id(Case), ?_assertEqual(ok, validate(doc(Old), doc(New)))}; make_validate_test({Old, New, Reason} = Case) -> @@ -314,19 +340,25 @@ make_validate_test({Old, New, Reason} = Case) -> {test_id(Case), ?_assertThrow({forbidden, Failure}, validate(doc(Old), doc(New)))}. test_id({[OldRoles, OldType], [NewRoles, NewType], Result}) -> - lists:flatten(io_lib:format( - "(roles: ~w, type: ~w) -> (roles: ~w, type: ~w) ==> \"~s\"", - [OldRoles, OldType, NewRoles, NewType, Result])). + lists:flatten( + io_lib:format( + "(roles: ~w, type: ~w) -> (roles: ~w, type: ~w) ==> \"~s\"", + [OldRoles, OldType, NewRoles, NewType, Result] + ) + ). doc([Roles, Type]) -> - couch_doc:from_json_obj({[ - {<<"_id">>,<<"org.couchdb.user:foo">>}, - {<<"_rev">>,<<"1-281c81adb1bf10927a6160f246dc0468">>}, - {<<"name">>,<<"foo">>}, - {<<"password_scheme">>,<<"simple">>}, - {<<"salt">>,<<"00000000000000000000000000000000">>}, - {<<"password_sha">>, <<"111111111111111111111111111111111111">>}] - ++ type(Type) ++ roles(Roles)}). + couch_doc:from_json_obj({ + [ + {<<"_id">>, <<"org.couchdb.user:foo">>}, + {<<"_rev">>, <<"1-281c81adb1bf10927a6160f246dc0468">>}, + {<<"name">>, <<"foo">>}, + {<<"password_scheme">>, <<"simple">>}, + {<<"salt">>, <<"00000000000000000000000000000000">>}, + {<<"password_sha">>, <<"111111111111111111111111111111111111">>} + ] ++ + type(Type) ++ roles(Roles) + }). roles(custom) -> [{<<"roles">>, [<<"custom">>]}]; roles(missing) -> []. @@ -336,11 +368,12 @@ type(other) -> [{<<"type">>, <<"other">>}]; type(missing) -> []. validate(DiskDoc, NewDoc) -> - JSONCtx = {[ - {<<"db">>, <<"foo/bar">>}, - {<<"name">>, <<"foo">>}, - {<<"roles">>, [<<"_admin">>]} - ]}, + JSONCtx = + {[ + {<<"db">>, <<"foo/bar">>}, + {<<"name">>, <<"foo">>}, + {<<"roles">>, [<<"_admin">>]} + ]}, validate(DiskDoc, NewDoc, JSONCtx). validate(DiskDoc, NewDoc, JSONCtx) -> diff --git a/src/couch/test/eunit/couch_bt_engine_compactor_ev.erl b/src/couch/test/eunit/couch_bt_engine_compactor_ev.erl index f50be84de..72b780a7f 100644 --- a/src/couch/test/eunit/couch_bt_engine_compactor_ev.erl +++ b/src/couch/test/eunit/couch_bt_engine_compactor_ev.erl @@ -12,7 +12,6 @@ -module(couch_bt_engine_compactor_ev). - -export([ init/0, terminate/0, @@ -24,22 +23,17 @@ event/1 ]). - -define(TAB, couch_db_updater_ev_tab). - init() -> ets:new(?TAB, [set, public, named_table]). - terminate() -> ets:delete(?TAB). - clear() -> ets:delete_all_objects(?TAB). - set_wait(Event) -> Self = self(), WaitFun = fun(_) -> @@ -51,48 +45,48 @@ set_wait(Event) -> end, ContinueFun = fun(Pid) -> Pid ! {Self, go}, - receive {Pid, ok} -> ok end + receive + {Pid, ok} -> ok + end end, ets:insert(?TAB, {Event, WaitFun}), {ok, ContinueFun}. - set_crash(Event) -> Reason = {couch_db_updater_ev_crash, Event}, CrashFun = fun(_) -> exit(Reason) end, ets:insert(?TAB, {Event, CrashFun}), {ok, Reason}. - event(Event) -> - NewEvent = case Event of - seq_init -> - put(?MODULE, 0), - Event; - seq_copy -> - Count = get(?MODULE), - put(?MODULE, Count + 1), - {seq_copy, Count}; - id_init -> - put(?MODULE, 0), - Event; - id_copy -> - Count = get(?MODULE), - put(?MODULE, Count + 1), - {id_copy, Count}; - md_copy_init -> - put(?MODULE, 0), - Event; - md_copy_row -> - Count = get(?MODULE), - put(?MODULE, Count + 1), - {md_copy_row, Count}; - _ -> - Event - end, + NewEvent = + case Event of + seq_init -> + put(?MODULE, 0), + Event; + seq_copy -> + Count = get(?MODULE), + put(?MODULE, Count + 1), + {seq_copy, Count}; + id_init -> + put(?MODULE, 0), + Event; + id_copy -> + Count = get(?MODULE), + put(?MODULE, Count + 1), + {id_copy, Count}; + md_copy_init -> + put(?MODULE, 0), + Event; + md_copy_row -> + Count = get(?MODULE), + put(?MODULE, Count + 1), + {md_copy_row, Count}; + _ -> + Event + end, handle_event(NewEvent). - handle_event(Event) -> try case ets:lookup(?TAB, Event) of @@ -101,6 +95,7 @@ handle_event(Event) -> [] -> ok end - catch error:badarg -> - ok - end.
\ No newline at end of file + catch + error:badarg -> + ok + end. diff --git a/src/couch/test/eunit/couch_bt_engine_compactor_ev_tests.erl b/src/couch/test/eunit/couch_bt_engine_compactor_ev_tests.erl index 090217b4c..007c74d06 100644 --- a/src/couch/test/eunit/couch_bt_engine_compactor_ev_tests.erl +++ b/src/couch/test/eunit/couch_bt_engine_compactor_ev_tests.erl @@ -39,21 +39,28 @@ events() -> [ - init, % The compactor process is spawned - files_opened, % After compaction files have opened + % The compactor process is spawned + init, + % After compaction files have opened + files_opened, - purge_init, % Just before apply purge changes - purge_done, % Just after finish purge updates + % Just before apply purge changes + purge_init, + % Just after finish purge updates + purge_done, % The firs phase is when we copy all document body and attachment % data to the new database file in order of update sequence so % that we can resume on crash. - seq_init, % Before the first change is copied - {seq_copy, 0}, % After change N is copied + % Before the first change is copied + seq_init, + % After change N is copied + {seq_copy, 0}, {seq_copy, ?INIT_DOCS div 2}, {seq_copy, ?INIT_DOCS - 2}, - seq_done, % After last change is copied + % After last change is copied + seq_done, % The id copy phases come in two flavors. Before a compaction % swap is attempted they're copied from the id_tree in the @@ -61,19 +68,27 @@ events() -> % stored in an emsort file on disk. Thus the two sets of % related events here. - md_sort_init, % Just before metadata sort starts - md_sort_done, % Justa after metadata sort finished - md_copy_init, % Just before metadata copy starts - {md_copy_row, 0}, % After docid N is copied + % Just before metadata sort starts + md_sort_init, + % Justa after metadata sort finished + md_sort_done, + % Just before metadata copy starts + md_copy_init, + % After docid N is copied + {md_copy_row, 0}, {md_copy_row, ?INIT_DOCS div 2}, {md_copy_row, ?INIT_DOCS - 2}, - md_copy_done, % Just after the last docid is copied + % Just after the last docid is copied + md_copy_done, % And then the final steps before we finish - before_final_sync, % Just before final sync - after_final_sync, % Just after the final sync - before_notify % Just before the final notification + % Just before final sync + before_final_sync, + % Just after the final sync + after_final_sync, + % Just before the final notification + before_notify ]. % Mark which evens only happen when documents are present @@ -86,7 +101,6 @@ requires_docs({md_copy_row, _}) -> true; requires_docs(md_copy_done) -> true; requires_docs(_) -> false. - % Mark which events only happen when there's write activity during % a compaction. @@ -97,25 +111,21 @@ requires_write({md_copy_row, _}) -> true; requires_write(md_copy_done) -> true; requires_write(_) -> false. - setup() -> purge_module(), ?EV_MOD:init(), test_util:start_couch(). - teardown(Ctx) -> test_util:stop_couch(Ctx), ?EV_MOD:terminate(). - start_empty_db_test(_Event) -> ?EV_MOD:clear(), DbName = ?tempdb(), {ok, _} = couch_db:create(DbName, [?ADMIN_CTX]), DbName. - start_populated_db_test(Event) -> DbName = start_empty_db_test(Event), {ok, Db} = couch_db:open_int(DbName, []), @@ -126,11 +136,9 @@ start_populated_db_test(Event) -> end, DbName. - stop_test(_Event, DbName) -> couch_server:delete(DbName, [?ADMIN_CTX]). - static_empty_db_test_() -> FiltFun = fun(E) -> not (requires_docs(E) or requires_write(E)) @@ -153,7 +161,6 @@ static_empty_db_test_() -> } }. - static_populated_db_test_() -> FiltFun = fun(E) -> not requires_write(E) end, Events = lists:filter(FiltFun, events()) -- [init], @@ -174,7 +181,6 @@ static_populated_db_test_() -> } }. - dynamic_empty_db_test_() -> FiltFun = fun(E) -> not requires_docs(E) end, Events = lists:filter(FiltFun, events()) -- [init], @@ -195,7 +201,6 @@ dynamic_empty_db_test_() -> } }. - dynamic_populated_db_test_() -> Events = events() -- [init], { @@ -215,13 +220,11 @@ dynamic_populated_db_test_() -> } }. - run_static_init(Event, DbName) -> Name = lists:flatten(io_lib:format("~p", [Event])), Test = {timeout, ?TIMEOUT_EUNIT, ?_test(run_static(Event, DbName))}, {Name, Test}. - run_static(Event, DbName) -> {ok, ContinueFun} = ?EV_MOD:set_wait(init), {ok, Reason} = ?EV_MOD:set_crash(Event), @@ -236,13 +239,11 @@ run_static(Event, DbName) -> run_successful_compaction(DbName), couch_db:close(Db). - run_dynamic_init(Event, DbName) -> Name = lists:flatten(io_lib:format("~p", [Event])), Test = {timeout, ?TIMEOUT_EUNIT, ?_test(run_dynamic(Event, DbName))}, {Name, Test}. - run_dynamic(Event, DbName) -> {ok, ContinueFun} = ?EV_MOD:set_wait(init), {ok, Reason} = ?EV_MOD:set_crash(Event), @@ -258,7 +259,6 @@ run_dynamic(Event, DbName) -> run_successful_compaction(DbName), couch_db:close(Db). - run_successful_compaction(DbName) -> ?EV_MOD:clear(), {ok, ContinueFun} = ?EV_MOD:set_wait(init), @@ -274,14 +274,11 @@ run_successful_compaction(DbName) -> validate_compaction(NewDb), couch_db:close(Db). - wait_db_cleared(Db) -> wait_db_cleared(Db, 5). - wait_db_cleared(Db, N) when N < 0 -> erlang:error({db_clear_timeout, couch_db:name(Db)}); - wait_db_cleared(Db, N) -> Tab = couch_server:couch_dbs(couch_db:name(Db)), case ets:lookup(Tab, couch_db:name(Db)) of @@ -290,29 +287,33 @@ wait_db_cleared(Db, N) -> [#entry{db = NewDb}] -> OldPid = couch_db:get_pid(Db), NewPid = couch_db:get_pid(NewDb), - if NewPid /= OldPid -> ok; true -> - timer:sleep(100), - wait_db_cleared(Db, N - 1) + if + NewPid /= OldPid -> + ok; + true -> + timer:sleep(100), + wait_db_cleared(Db, N - 1) end end. - populate_db(_Db, NumDocs) when NumDocs =< 0 -> ok; populate_db(Db, NumDocs) -> String = [$a || _ <- lists:seq(1, erlang:min(NumDocs, 500))], Docs = lists:map( fun(_) -> - couch_doc:from_json_obj({[ - {<<"_id">>, couch_uuids:random()}, - {<<"string">>, list_to_binary(String)} - ]}) + couch_doc:from_json_obj( + {[ + {<<"_id">>, couch_uuids:random()}, + {<<"string">>, list_to_binary(String)} + ]} + ) end, - lists:seq(1, 500)), + lists:seq(1, 500) + ), {ok, _} = couch_db:update_docs(Db, Docs, []), populate_db(Db, NumDocs - 500). - validate_compaction(Db) -> {ok, DocCount} = couch_db:get_doc_count(Db), {ok, DelDocCount} = couch_db:get_del_doc_count(Db), @@ -325,7 +326,6 @@ validate_compaction(Db) -> ?assertEqual(DocCount + DelDocCount, LastCount), ?assertEqual(NumChanges, LastCount). - purge_module() -> case code:which(couch_db_updater) of cover_compiled -> diff --git a/src/couch/test/eunit/couch_bt_engine_compactor_tests.erl b/src/couch/test/eunit/couch_bt_engine_compactor_tests.erl index 4c4c43958..73428b0a9 100644 --- a/src/couch/test/eunit/couch_bt_engine_compactor_tests.erl +++ b/src/couch/test/eunit/couch_bt_engine_compactor_tests.erl @@ -12,15 +12,12 @@ -module(couch_bt_engine_compactor_tests). - -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). - -define(DELAY, 100). -define(WAIT_DELAY_COUNT, 50). - setup() -> DbName = ?tempdb(), {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]), @@ -28,12 +25,10 @@ setup() -> create_docs(DbName), DbName. - teardown(DbName) when is_binary(DbName) -> couch_server:delete(DbName, [?ADMIN_CTX]), ok. - compaction_resume_test_() -> { setup, @@ -49,7 +44,6 @@ compaction_resume_test_() -> } }. - compaction_resume(DbName) -> ?_test(begin check_db_validity(DbName), @@ -66,14 +60,12 @@ compaction_resume(DbName) -> check_db_validity(DbName) end). - check_db_validity(DbName) -> couch_util:with_db(DbName, fun(Db) -> ?assertEqual({ok, 3}, couch_db:get_doc_count(Db)), ?assertEqual(3, couch_db:count_changes_since(Db, 0)) end). - with_mecked_emsort(Fun) -> meck:new(couch_emsort, [passthrough]), meck:expect(couch_emsort, iter, fun(_) -> erlang:error(kaboom) end), @@ -83,35 +75,35 @@ with_mecked_emsort(Fun) -> meck:unload() end. - create_docs(DbName) -> couch_util:with_db(DbName, fun(Db) -> - Doc1 = couch_doc:from_json_obj({[ - {<<"_id">>, <<"doc1">>}, - {<<"value">>, 1} - - ]}), - Doc2 = couch_doc:from_json_obj({[ - {<<"_id">>, <<"doc2">>}, - {<<"value">>, 2} - - ]}), - Doc3 = couch_doc:from_json_obj({[ - {<<"_id">>, <<"doc3">>}, - {<<"value">>, 3} - - ]}), + Doc1 = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"doc1">>}, + {<<"value">>, 1} + ]} + ), + Doc2 = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"doc2">>}, + {<<"value">>, 2} + ]} + ), + Doc3 = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"doc3">>}, + {<<"value">>, 3} + ]} + ), {ok, _} = couch_db:update_docs(Db, [Doc1, Doc2, Doc3]) end). - compact_db(DbName) -> couch_util:with_db(DbName, fun(Db) -> {ok, _} = couch_db:start_compact(Db) end), wait_db_compact_done(DbName, ?WAIT_DELAY_COUNT). - wait_db_compact_done(_DbName, 0) -> Failure = [ {module, ?MODULE}, @@ -123,7 +115,10 @@ wait_db_compact_done(DbName, N) -> IsDone = couch_util:with_db(DbName, fun(Db) -> not is_pid(couch_db:get_compactor_pid(Db)) end), - if IsDone -> ok; true -> - timer:sleep(?DELAY), - wait_db_compact_done(DbName, N - 1) + if + IsDone -> + ok; + true -> + timer:sleep(?DELAY), + wait_db_compact_done(DbName, N - 1) end. diff --git a/src/couch/test/eunit/couch_bt_engine_tests.erl b/src/couch/test/eunit/couch_bt_engine_tests.erl index 3e3ecbf25..56d18d3a4 100644 --- a/src/couch/test/eunit/couch_bt_engine_tests.erl +++ b/src/couch/test/eunit/couch_bt_engine_tests.erl @@ -12,9 +12,7 @@ -module(couch_bt_engine_tests). - -include_lib("eunit/include/eunit.hrl"). - -couch_bt_engine_test_()-> +couch_bt_engine_test_() -> cpse_util:create_tests(couch, couch_bt_engine, "couch"). diff --git a/src/couch/test/eunit/couch_bt_engine_upgrade_tests.erl b/src/couch/test/eunit/couch_bt_engine_upgrade_tests.erl index a2a972caf..62f128a4f 100644 --- a/src/couch/test/eunit/couch_bt_engine_upgrade_tests.erl +++ b/src/couch/test/eunit/couch_bt_engine_upgrade_tests.erl @@ -15,7 +15,8 @@ -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). --define(TIMEOUT, 60). % seconds +% seconds +-define(TIMEOUT, 60). setup(_) -> Ctx = test_util:start_couch(), @@ -30,23 +31,27 @@ setup(_) -> "db_v7_with_2_purge_req.couch", "db_v7_with_1_purge_req_for_2_docs.couch" ], - NewPaths = lists:map(fun(DbFileName) -> - OldDbFilePath = filename:join([?FIXTURESDIR, DbFileName]), - NewDbFilePath = filename:join([DbDir, DbFileName]), - ok = filelib:ensure_dir(NewDbFilePath), - file:delete(NewDbFilePath), - {ok, _} = file:copy(OldDbFilePath, NewDbFilePath), - NewDbFilePath - end, DbFileNames), + NewPaths = lists:map( + fun(DbFileName) -> + OldDbFilePath = filename:join([?FIXTURESDIR, DbFileName]), + NewDbFilePath = filename:join([DbDir, DbFileName]), + ok = filelib:ensure_dir(NewDbFilePath), + file:delete(NewDbFilePath), + {ok, _} = file:copy(OldDbFilePath, NewDbFilePath), + NewDbFilePath + end, + DbFileNames + ), {Ctx, NewPaths}. - teardown(_, {Ctx, Paths}) -> test_util:stop_couch(Ctx), - lists:foreach(fun(Path) -> - file:delete(Path) - end, Paths). - + lists:foreach( + fun(Path) -> + file:delete(Path) + end, + Paths + ). upgrade_test_() -> From = [6, 7], @@ -54,174 +59,182 @@ upgrade_test_() -> "Couch Bt Engine Upgrade tests", { foreachx, - fun setup/1, fun teardown/2, + fun setup/1, + fun teardown/2, [{F, fun t_upgrade_without_purge_req/2} || F <- From] ++ - [{F, fun t_upgrade_with_1_purge_req/2} || F <- From] ++ - [{F, fun t_upgrade_with_N_purge_req/2} || F <- From] ++ - [{F, fun t_upgrade_with_1_purge_req_for_2_docs/2} || F <- From] + [{F, fun t_upgrade_with_1_purge_req/2} || F <- From] ++ + [{F, fun t_upgrade_with_N_purge_req/2} || F <- From] ++ + [{F, fun t_upgrade_with_1_purge_req_for_2_docs/2} || F <- From] } }. - t_upgrade_without_purge_req(VersionFrom, {_Ctx, _NewPaths}) -> - {timeout, ?TIMEOUT, ?_test(begin - % There are three documents in the fixture - % db with zero purge entries - DbName = ?l2b("db_v" ++ integer_to_list(VersionFrom) - ++ "_without_purge_req"), - - ?assertEqual(VersionFrom, get_disk_version_from_header(DbName)), - {ok, UpgradedPurged} = couch_util:with_db(DbName, fun(Db) -> - ?assertEqual(0, couch_db:get_purge_seq(Db)), - couch_db:fold_purge_infos(Db, 0, fun fold_fun/2, []) - end), - ?assertEqual([], UpgradedPurged), - ?assertEqual(8, get_disk_version_from_header(DbName)), - {ok, Rev} = save_doc( - DbName, {[{<<"_id">>, <<"doc4">>}, {<<"v">>, 1}]} - ), - {ok, _} = save_doc(DbName, {[{<<"_id">>, <<"doc5">>}, {<<"v">>, 2}]}), - - couch_util:with_db(DbName, fun(Db) -> - ?assertEqual({ok, 5}, couch_db:get_doc_count(Db)), - ?assertEqual(0, couch_db:get_purge_seq(Db)) - end), - - PurgeReqs = [ - {couch_uuids:random(), <<"doc4">>, [Rev]} - ], - - {ok, [{ok, PRevs}]} = couch_util:with_db(DbName, fun(Db) -> - couch_db:purge_docs(Db, PurgeReqs) - end), - ?assertEqual(PRevs, [Rev]), - - couch_util:with_db(DbName, fun(Db) -> - ?assertEqual({ok, 4}, couch_db:get_doc_count(Db)), - ?assertEqual(1, couch_db:get_purge_seq(Db)) - end) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + % There are three documents in the fixture + % db with zero purge entries + DbName = ?l2b( + "db_v" ++ integer_to_list(VersionFrom) ++ + "_without_purge_req" + ), + + ?assertEqual(VersionFrom, get_disk_version_from_header(DbName)), + {ok, UpgradedPurged} = couch_util:with_db(DbName, fun(Db) -> + ?assertEqual(0, couch_db:get_purge_seq(Db)), + couch_db:fold_purge_infos(Db, 0, fun fold_fun/2, []) + end), + ?assertEqual([], UpgradedPurged), + ?assertEqual(8, get_disk_version_from_header(DbName)), + {ok, Rev} = save_doc( + DbName, {[{<<"_id">>, <<"doc4">>}, {<<"v">>, 1}]} + ), + {ok, _} = save_doc(DbName, {[{<<"_id">>, <<"doc5">>}, {<<"v">>, 2}]}), + + couch_util:with_db(DbName, fun(Db) -> + ?assertEqual({ok, 5}, couch_db:get_doc_count(Db)), + ?assertEqual(0, couch_db:get_purge_seq(Db)) + end), + + PurgeReqs = [ + {couch_uuids:random(), <<"doc4">>, [Rev]} + ], + + {ok, [{ok, PRevs}]} = couch_util:with_db(DbName, fun(Db) -> + couch_db:purge_docs(Db, PurgeReqs) + end), + ?assertEqual(PRevs, [Rev]), + + couch_util:with_db(DbName, fun(Db) -> + ?assertEqual({ok, 4}, couch_db:get_doc_count(Db)), + ?assertEqual(1, couch_db:get_purge_seq(Db)) + end) + end)}. t_upgrade_with_1_purge_req(VersionFrom, {_Ctx, _NewPaths}) -> - {timeout, ?TIMEOUT, ?_test(begin - % There are two documents in the fixture database - % with a single purge entry - DbName = ?l2b("db_v" ++ integer_to_list(VersionFrom) - ++ "_with_1_purge_req"), - - ?assertEqual(VersionFrom, get_disk_version_from_header(DbName)), - {ok, UpgradedPurged} = couch_util:with_db(DbName, fun(Db) -> - ?assertEqual(1, couch_db:get_purge_seq(Db)), - couch_db:fold_purge_infos(Db, 0, fun fold_fun/2, []) - end), - ?assertEqual(8, get_disk_version_from_header(DbName)), - ?assertEqual([{1, <<"doc1">>}], UpgradedPurged), - - {ok, Rev} = save_doc( - DbName, {[{<<"_id">>, <<"doc4">>}, {<<"v">>, 1}]} - ), - {ok, _} = save_doc(DbName, {[{<<"_id">>, <<"doc5">>}, {<<"v">>, 2}]}), - - couch_util:with_db(DbName, fun(Db) -> - ?assertEqual({ok, 4}, couch_db:get_doc_count(Db)), - ?assertEqual(1, couch_db:get_purge_seq(Db)) - end), - - PurgeReqs = [ - {couch_uuids:random(), <<"doc4">>, [Rev]} - ], - - {ok, [{ok, PRevs}]} = couch_util:with_db(DbName, fun(Db) -> - couch_db:purge_docs(Db, PurgeReqs) - end), - ?assertEqual(PRevs, [Rev]), - - couch_util:with_db(DbName, fun(Db) -> - ?assertEqual({ok, 3}, couch_db:get_doc_count(Db)), - ?assertEqual(2, couch_db:get_purge_seq(Db)) - end) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + % There are two documents in the fixture database + % with a single purge entry + DbName = ?l2b( + "db_v" ++ integer_to_list(VersionFrom) ++ + "_with_1_purge_req" + ), + + ?assertEqual(VersionFrom, get_disk_version_from_header(DbName)), + {ok, UpgradedPurged} = couch_util:with_db(DbName, fun(Db) -> + ?assertEqual(1, couch_db:get_purge_seq(Db)), + couch_db:fold_purge_infos(Db, 0, fun fold_fun/2, []) + end), + ?assertEqual(8, get_disk_version_from_header(DbName)), + ?assertEqual([{1, <<"doc1">>}], UpgradedPurged), + + {ok, Rev} = save_doc( + DbName, {[{<<"_id">>, <<"doc4">>}, {<<"v">>, 1}]} + ), + {ok, _} = save_doc(DbName, {[{<<"_id">>, <<"doc5">>}, {<<"v">>, 2}]}), + + couch_util:with_db(DbName, fun(Db) -> + ?assertEqual({ok, 4}, couch_db:get_doc_count(Db)), + ?assertEqual(1, couch_db:get_purge_seq(Db)) + end), + + PurgeReqs = [ + {couch_uuids:random(), <<"doc4">>, [Rev]} + ], + + {ok, [{ok, PRevs}]} = couch_util:with_db(DbName, fun(Db) -> + couch_db:purge_docs(Db, PurgeReqs) + end), + ?assertEqual(PRevs, [Rev]), + + couch_util:with_db(DbName, fun(Db) -> + ?assertEqual({ok, 3}, couch_db:get_doc_count(Db)), + ?assertEqual(2, couch_db:get_purge_seq(Db)) + end) + end)}. t_upgrade_with_N_purge_req(VersionFrom, {_Ctx, _NewPaths}) -> - {timeout, ?TIMEOUT, ?_test(begin - % There is one document in the fixture database - % with two docs that have been purged - DbName = ?l2b("db_v" ++ integer_to_list(VersionFrom) - ++ "_with_2_purge_req"), - - ?assertEqual(VersionFrom, get_disk_version_from_header(DbName)), - {ok, UpgradedPurged} = couch_util:with_db(DbName, fun(Db) -> - ?assertEqual(2, couch_db:get_purge_seq(Db)), - couch_db:fold_purge_infos(Db, 1, fun fold_fun/2, []) - end), - ?assertEqual(8, get_disk_version_from_header(DbName)), - ?assertEqual([{2, <<"doc2">>}], UpgradedPurged), - - {ok, Rev} = save_doc(DbName, {[{<<"_id">>, <<"doc4">>}, {<<"v">>, 1}]}), - {ok, _} = save_doc(DbName, {[{<<"_id">>, <<"doc5">>}, {<<"v">>, 2}]}), - - couch_util:with_db(DbName, fun(Db) -> - ?assertEqual({ok, 3}, couch_db:get_doc_count(Db)), - ?assertEqual(2, couch_db:get_purge_seq(Db)) - end), - - PurgeReqs = [ - {couch_uuids:random(), <<"doc4">>, [Rev]} - ], - - {ok, [{ok, PRevs}]} = couch_util:with_db(DbName, fun(Db) -> - couch_db:purge_docs(Db, PurgeReqs) - end), - ?assertEqual(PRevs, [Rev]), - - couch_util:with_db(DbName, fun(Db) -> - ?assertEqual({ok, 2}, couch_db:get_doc_count(Db)), - ?assertEqual(3, couch_db:get_purge_seq(Db)) - end) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + % There is one document in the fixture database + % with two docs that have been purged + DbName = ?l2b( + "db_v" ++ integer_to_list(VersionFrom) ++ + "_with_2_purge_req" + ), + + ?assertEqual(VersionFrom, get_disk_version_from_header(DbName)), + {ok, UpgradedPurged} = couch_util:with_db(DbName, fun(Db) -> + ?assertEqual(2, couch_db:get_purge_seq(Db)), + couch_db:fold_purge_infos(Db, 1, fun fold_fun/2, []) + end), + ?assertEqual(8, get_disk_version_from_header(DbName)), + ?assertEqual([{2, <<"doc2">>}], UpgradedPurged), + + {ok, Rev} = save_doc(DbName, {[{<<"_id">>, <<"doc4">>}, {<<"v">>, 1}]}), + {ok, _} = save_doc(DbName, {[{<<"_id">>, <<"doc5">>}, {<<"v">>, 2}]}), + + couch_util:with_db(DbName, fun(Db) -> + ?assertEqual({ok, 3}, couch_db:get_doc_count(Db)), + ?assertEqual(2, couch_db:get_purge_seq(Db)) + end), + + PurgeReqs = [ + {couch_uuids:random(), <<"doc4">>, [Rev]} + ], + + {ok, [{ok, PRevs}]} = couch_util:with_db(DbName, fun(Db) -> + couch_db:purge_docs(Db, PurgeReqs) + end), + ?assertEqual(PRevs, [Rev]), + + couch_util:with_db(DbName, fun(Db) -> + ?assertEqual({ok, 2}, couch_db:get_doc_count(Db)), + ?assertEqual(3, couch_db:get_purge_seq(Db)) + end) + end)}. t_upgrade_with_1_purge_req_for_2_docs(VersionFrom, {_Ctx, _NewPaths}) -> - {timeout, ?TIMEOUT, ?_test(begin - % There are two documents (Doc4 and Doc5) in the fixture database - % with three docs (Doc1, Doc2 and Doc3) that have been purged, and - % with one purge req for Doc1 and another purge req for Doc 2 and Doc3 - DbName = ?l2b("db_v" ++ integer_to_list(VersionFrom) - ++ "_with_1_purge_req_for_2_docs"), - - ?assertEqual(VersionFrom, get_disk_version_from_header(DbName)), - {ok, UpgradedPurged} = couch_util:with_db(DbName, fun(Db) -> - ?assertEqual(3, couch_db:get_purge_seq(Db)), - couch_db:fold_purge_infos(Db, 1, fun fold_fun/2, []) - end), - ?assertEqual(8, get_disk_version_from_header(DbName)), - ?assertEqual([{3,<<"doc2">>},{2,<<"doc3">>}], UpgradedPurged), - - {ok, Rev} = save_doc(DbName, {[{<<"_id">>, <<"doc6">>}, {<<"v">>, 1}]}), - {ok, _} = save_doc(DbName, {[{<<"_id">>, <<"doc7">>}, {<<"v">>, 2}]}), - - couch_util:with_db(DbName, fun(Db) -> - ?assertEqual({ok, 4}, couch_db:get_doc_count(Db)), - ?assertEqual(3, couch_db:get_purge_seq(Db)) - end), - - PurgeReqs = [ - {couch_uuids:random(), <<"doc6">>, [Rev]} - ], - - {ok, [{ok, PRevs}]} = couch_util:with_db(DbName, fun(Db) -> - couch_db:purge_docs(Db, PurgeReqs) - end), - ?assertEqual(PRevs, [Rev]), - - couch_util:with_db(DbName, fun(Db) -> - ?assertEqual({ok, 3}, couch_db:get_doc_count(Db)), - ?assertEqual(4, couch_db:get_purge_seq(Db)) - end) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + % There are two documents (Doc4 and Doc5) in the fixture database + % with three docs (Doc1, Doc2 and Doc3) that have been purged, and + % with one purge req for Doc1 and another purge req for Doc 2 and Doc3 + DbName = ?l2b( + "db_v" ++ integer_to_list(VersionFrom) ++ + "_with_1_purge_req_for_2_docs" + ), + + ?assertEqual(VersionFrom, get_disk_version_from_header(DbName)), + {ok, UpgradedPurged} = couch_util:with_db(DbName, fun(Db) -> + ?assertEqual(3, couch_db:get_purge_seq(Db)), + couch_db:fold_purge_infos(Db, 1, fun fold_fun/2, []) + end), + ?assertEqual(8, get_disk_version_from_header(DbName)), + ?assertEqual([{3, <<"doc2">>}, {2, <<"doc3">>}], UpgradedPurged), + + {ok, Rev} = save_doc(DbName, {[{<<"_id">>, <<"doc6">>}, {<<"v">>, 1}]}), + {ok, _} = save_doc(DbName, {[{<<"_id">>, <<"doc7">>}, {<<"v">>, 2}]}), + + couch_util:with_db(DbName, fun(Db) -> + ?assertEqual({ok, 4}, couch_db:get_doc_count(Db)), + ?assertEqual(3, couch_db:get_purge_seq(Db)) + end), + + PurgeReqs = [ + {couch_uuids:random(), <<"doc6">>, [Rev]} + ], + + {ok, [{ok, PRevs}]} = couch_util:with_db(DbName, fun(Db) -> + couch_db:purge_docs(Db, PurgeReqs) + end), + ?assertEqual(PRevs, [Rev]), + + couch_util:with_db(DbName, fun(Db) -> + ?assertEqual({ok, 3}, couch_db:get_doc_count(Db)), + ?assertEqual(4, couch_db:get_purge_seq(Db)) + end) + end)}. save_doc(DbName, Json) -> Doc = couch_doc:from_json_obj(Json), @@ -229,11 +242,9 @@ save_doc(DbName, Json) -> couch_db:update_doc(Db, Doc, []) end). - fold_fun({PSeq, _UUID, Id, _Revs}, Acc) -> {ok, [{PSeq, Id} | Acc]}. - get_disk_version_from_header(DbFileName) -> DbDir = config:get("couchdb", "database_dir"), DbFilePath = filename:join([DbDir, ?l2b(?b2l(DbFileName) ++ ".couch")]), diff --git a/src/couch/test/eunit/couch_btree_tests.erl b/src/couch/test/eunit/couch_btree_tests.erl index c9b791d2c..1c9ba7771 100644 --- a/src/couch/test/eunit/couch_btree_tests.erl +++ b/src/couch/test/eunit/couch_btree_tests.erl @@ -16,13 +16,15 @@ -include_lib("couch/include/couch_db.hrl"). -define(ROWS, 1000). --define(TIMEOUT, 60). % seconds - +% seconds +-define(TIMEOUT, 60). setup() -> {ok, Fd} = couch_file:open(?tempfile(), [create, overwrite]), - {ok, Btree} = couch_btree:open(nil, Fd, [{compression, none}, - {reduce, fun reduce_fun/2}]), + {ok, Btree} = couch_btree:open(nil, Fd, [ + {compression, none}, + {reduce, fun reduce_fun/2} + ]), {Fd, Btree}. setup_kvs(_) -> @@ -35,7 +37,10 @@ setup_red() -> "even" -> {"odd", [{{Key, Idx}, 1} | Acc]}; _ -> {"even", [{{Key, Idx}, 1} | Acc]} end - end, {"odd", []}, lists:seq(1, ?ROWS)), + end, + {"odd", []}, + lists:seq(1, ?ROWS) + ), {Fd, Btree} = setup(), {ok, Btree1} = couch_btree:add_remove(Btree, EvenOddKVs, []), {Fd, Btree1}. @@ -49,7 +54,6 @@ teardown({Fd, _}) -> teardown(_, {Fd, _}) -> teardown(Fd). - kvs_test_funs() -> [ fun should_set_fd_correctly/2, @@ -72,7 +76,6 @@ red_test_funs() -> fun should_reduce_second_half/2 ]. - btree_open_test_() -> {ok, Fd} = couch_file:open(?tempfile(), [create, overwrite]), {ok, Btree} = couch_btree:open(nil, Fd, [{compression, none}]), @@ -88,10 +91,12 @@ sorted_kvs_test_() -> "BTree with sorted keys", { setup, - fun() -> test_util:start(?MODULE, [ioq]) end, fun test_util:stop/1, + fun() -> test_util:start(?MODULE, [ioq]) end, + fun test_util:stop/1, { foreachx, - fun setup_kvs/1, fun teardown/2, + fun setup_kvs/1, + fun teardown/2, [{Sorted, Fun} || Fun <- Funs] } } @@ -105,10 +110,12 @@ rsorted_kvs_test_() -> "BTree with backward sorted keys", { setup, - fun() -> test_util:start(?MODULE, [ioq]) end, fun test_util:stop/1, + fun() -> test_util:start(?MODULE, [ioq]) end, + fun test_util:stop/1, { foreachx, - fun setup_kvs/1, fun teardown/2, + fun setup_kvs/1, + fun teardown/2, [{Reversed, Fun} || Fun <- Funs] } } @@ -122,10 +129,12 @@ shuffled_kvs_test_() -> "BTree with shuffled keys", { setup, - fun() -> test_util:start(?MODULE, [ioq]) end, fun test_util:stop/1, + fun() -> test_util:start(?MODULE, [ioq]) end, + fun test_util:stop/1, { foreachx, - fun setup_kvs/1, fun teardown/2, + fun setup_kvs/1, + fun teardown/2, [{Shuffled, Fun} || Fun <- Funs] } } @@ -136,13 +145,15 @@ reductions_test_() -> "BTree reductions", { setup, - fun() -> test_util:start(?MODULE, [ioq]) end, fun test_util:stop/1, + fun() -> test_util:start(?MODULE, [ioq]) end, + fun test_util:stop/1, [ { "Common tests", { foreach, - fun setup_red/0, fun teardown/1, + fun setup_red/0, + fun teardown/1, [ fun should_reduce_without_specified_direction/1, fun should_reduce_forward/1, @@ -157,7 +168,8 @@ reductions_test_() -> "Forward direction", { foreachx, - fun setup_red/1, fun teardown/2, + fun setup_red/1, + fun teardown/2, [{fwd, F} || F <- red_test_funs()] } }, @@ -165,7 +177,8 @@ reductions_test_() -> "Backward direction", { foreachx, - fun setup_red/1, fun teardown/2, + fun setup_red/1, + fun teardown/2, [{rev, F} || F <- red_test_funs()] } } @@ -175,7 +188,6 @@ reductions_test_() -> } }. - should_set_fd_correctly(_, {Fd, Btree}) -> ?_assertMatch(Fd, Btree#btree.fd). @@ -191,7 +203,7 @@ should_set_reduce_option(_, {_, Btree}) -> ?_assertMatch(ReduceFun, Btree1#btree.reduce). should_fold_over_empty_btree(_, {_, Btree}) -> - {ok, _, EmptyRes} = couch_btree:foldl(Btree, fun(_, X) -> {ok, X+1} end, 0), + {ok, _, EmptyRes} = couch_btree:foldl(Btree, fun(_, X) -> {ok, X + 1} end, 0), ?_assertEqual(EmptyRes, 0). should_add_all_keys(KeyValues, {Fd, Btree}) -> @@ -214,8 +226,10 @@ should_have_lesser_size_than_file(Fd, Btree) -> ?_assert((couch_btree:size(Btree) =< couch_file:bytes(Fd))). should_keep_root_pointer_to_kp_node(Fd, Btree) -> - ?_assertMatch({ok, {kp_node, _}}, - couch_file:pread_term(Fd, element(1, Btree#btree.root))). + ?_assertMatch( + {ok, {kp_node, _}}, + couch_file:pread_term(Fd, element(1, Btree#btree.root)) + ). should_remove_all_keys(KeyValues, Btree) -> Keys = keys(KeyValues), @@ -234,7 +248,10 @@ should_continuously_add_new_kv(KeyValues, {_, Btree}) -> {ok, BtAcc2} = couch_btree:add_remove(BtAcc, [KV], []), ?assert(couch_btree:size(BtAcc2) > PrevSize), {BtAcc2, couch_btree:size(BtAcc2)} - end, {Btree, couch_btree:size(Btree)}, KeyValues), + end, + {Btree, couch_btree:size(Btree)}, + KeyValues + ), { "Should continuously add key-values to btree", [ @@ -250,7 +267,10 @@ should_continuously_remove_keys(KeyValues, {_, Btree}) -> {ok, BtAcc2} = couch_btree:add_remove(BtAcc, [], [K]), ?assert(couch_btree:size(BtAcc2) < PrevSize), {BtAcc2, couch_btree:size(BtAcc2)} - end, {Btree1, couch_btree:size(Btree1)}, KeyValues), + end, + {Btree1, couch_btree:size(Btree1)}, + KeyValues + ), { "Should continuously remove keys from btree", [ @@ -266,48 +286,57 @@ should_insert_keys_in_reversed_order(KeyValues, {_, Btree}) -> {ok, BtAcc2} = couch_btree:add_remove(BtAcc, [KV], []), ?assert(couch_btree:size(BtAcc2) > PrevSize), {BtAcc2, couch_btree:size(BtAcc2)} - end, {Btree, couch_btree:size(Btree)}, KeyValuesRev), + end, + {Btree, couch_btree:size(Btree)}, + KeyValuesRev + ), should_produce_valid_btree(Btree1, KeyValues). should_add_every_odd_key_remove_every_even(KeyValues, {_, Btree}) -> {ok, Btree1} = couch_btree:add_remove(Btree, KeyValues, []), - {_, Rem2Keys0, Rem2Keys1} = lists:foldl(fun(X, {Count, Left, Right}) -> - case Count rem 2 == 0 of - true -> {Count + 1, [X | Left], Right}; - false -> {Count + 1, Left, [X | Right]} - end - end, {0, [], []}, KeyValues), - {timeout, ?TIMEOUT, - ?_assert(test_add_remove(Btree1, Rem2Keys0, Rem2Keys1)) - }. + {_, Rem2Keys0, Rem2Keys1} = lists:foldl( + fun(X, {Count, Left, Right}) -> + case Count rem 2 == 0 of + true -> {Count + 1, [X | Left], Right}; + false -> {Count + 1, Left, [X | Right]} + end + end, + {0, [], []}, + KeyValues + ), + {timeout, ?TIMEOUT, ?_assert(test_add_remove(Btree1, Rem2Keys0, Rem2Keys1))}. should_add_every_even_key_remove_every_old(KeyValues, {_, Btree}) -> {ok, Btree1} = couch_btree:add_remove(Btree, KeyValues, []), - {_, Rem2Keys0, Rem2Keys1} = lists:foldl(fun(X, {Count, Left, Right}) -> - case Count rem 2 == 0 of - true -> {Count + 1, [X | Left], Right}; - false -> {Count + 1, Left, [X | Right]} - end - end, {0, [], []}, KeyValues), - {timeout, ?TIMEOUT, - ?_assert(test_add_remove(Btree1, Rem2Keys1, Rem2Keys0)) - }. - + {_, Rem2Keys0, Rem2Keys1} = lists:foldl( + fun(X, {Count, Left, Right}) -> + case Count rem 2 == 0 of + true -> {Count + 1, [X | Left], Right}; + false -> {Count + 1, Left, [X | Right]} + end + end, + {0, [], []}, + KeyValues + ), + {timeout, ?TIMEOUT, ?_assert(test_add_remove(Btree1, Rem2Keys1, Rem2Keys0))}. should_reduce_without_specified_direction({_, Btree}) -> ?_assertMatch( {ok, [{{"odd", _}, ?ROWS div 2}, {{"even", _}, ?ROWS div 2}]}, - fold_reduce(Btree, [])). + fold_reduce(Btree, []) + ). should_reduce_forward({_, Btree}) -> ?_assertMatch( {ok, [{{"odd", _}, ?ROWS div 2}, {{"even", _}, ?ROWS div 2}]}, - fold_reduce(Btree, [{dir, fwd}])). + fold_reduce(Btree, [{dir, fwd}]) + ). should_reduce_backward({_, Btree}) -> ?_assertMatch( {ok, [{{"even", _}, ?ROWS div 2}, {{"odd", _}, ?ROWS div 2}]}, - fold_reduce(Btree, [{dir, rev}])). + fold_reduce(Btree, [{dir, rev}]) + ). should_reduce_whole_range(fwd, {_, Btree}) -> {SK, EK} = {{"even", 0}, {"odd", ?ROWS - 1}}, @@ -315,20 +344,30 @@ should_reduce_whole_range(fwd, {_, Btree}) -> { "include endkey", ?_assertMatch( - {ok, [{{"odd", 1}, ?ROWS div 2}, - {{"even", 2}, ?ROWS div 2}]}, - fold_reduce(Btree, [{dir, fwd}, - {start_key, SK}, - {end_key, EK}])) + {ok, [ + {{"odd", 1}, ?ROWS div 2}, + {{"even", 2}, ?ROWS div 2} + ]}, + fold_reduce(Btree, [ + {dir, fwd}, + {start_key, SK}, + {end_key, EK} + ]) + ) }, { "exclude endkey", ?_assertMatch( - {ok, [{{"odd", 1}, (?ROWS div 2) - 1}, - {{"even", 2}, ?ROWS div 2}]}, - fold_reduce(Btree, [{dir, fwd}, - {start_key, SK}, - {end_key_gt, EK}])) + {ok, [ + {{"odd", 1}, (?ROWS div 2) - 1}, + {{"even", 2}, ?ROWS div 2} + ]}, + fold_reduce(Btree, [ + {dir, fwd}, + {start_key, SK}, + {end_key_gt, EK} + ]) + ) } ]; should_reduce_whole_range(rev, {_, Btree}) -> @@ -337,20 +376,30 @@ should_reduce_whole_range(rev, {_, Btree}) -> { "include endkey", ?_assertMatch( - {ok, [{{"even", ?ROWS}, ?ROWS div 2}, - {{"odd", ?ROWS - 1}, ?ROWS div 2}]}, - fold_reduce(Btree, [{dir, rev}, - {start_key, SK}, - {end_key, EK}])) + {ok, [ + {{"even", ?ROWS}, ?ROWS div 2}, + {{"odd", ?ROWS - 1}, ?ROWS div 2} + ]}, + fold_reduce(Btree, [ + {dir, rev}, + {start_key, SK}, + {end_key, EK} + ]) + ) }, { "exclude endkey", ?_assertMatch( - {ok, [{{"even", ?ROWS}, (?ROWS div 2) - 1}, - {{"odd", ?ROWS - 1}, ?ROWS div 2}]}, - fold_reduce(Btree, [{dir, rev}, - {start_key, SK}, - {end_key_gt, EK}])) + {ok, [ + {{"even", ?ROWS}, (?ROWS div 2) - 1}, + {{"odd", ?ROWS - 1}, ?ROWS div 2} + ]}, + fold_reduce(Btree, [ + {dir, rev}, + {start_key, SK}, + {end_key_gt, EK} + ]) + ) } ]. @@ -360,19 +409,30 @@ should_reduce_first_half(fwd, {_, Btree}) -> { "include endkey", ?_assertMatch( - {ok, [{{"odd", 1}, ?ROWS div 4}, - {{"even", 2}, ?ROWS div 2}]}, - fold_reduce(Btree, [{dir, fwd}, - {start_key, SK}, {end_key, EK}])) + {ok, [ + {{"odd", 1}, ?ROWS div 4}, + {{"even", 2}, ?ROWS div 2} + ]}, + fold_reduce(Btree, [ + {dir, fwd}, + {start_key, SK}, + {end_key, EK} + ]) + ) }, { "exclude endkey", ?_assertMatch( - {ok, [{{"odd", 1}, (?ROWS div 4) - 1}, - {{"even", 2}, ?ROWS div 2}]}, - fold_reduce(Btree, [{dir, fwd}, - {start_key, SK}, - {end_key_gt, EK}])) + {ok, [ + {{"odd", 1}, (?ROWS div 4) - 1}, + {{"even", 2}, ?ROWS div 2} + ]}, + fold_reduce(Btree, [ + {dir, fwd}, + {start_key, SK}, + {end_key_gt, EK} + ]) + ) } ]; should_reduce_first_half(rev, {_, Btree}) -> @@ -381,20 +441,30 @@ should_reduce_first_half(rev, {_, Btree}) -> { "include endkey", ?_assertMatch( - {ok, [{{"even", ?ROWS}, (?ROWS div 4) + 1}, - {{"odd", ?ROWS - 1}, ?ROWS div 2}]}, - fold_reduce(Btree, [{dir, rev}, - {start_key, SK}, - {end_key, EK}])) + {ok, [ + {{"even", ?ROWS}, (?ROWS div 4) + 1}, + {{"odd", ?ROWS - 1}, ?ROWS div 2} + ]}, + fold_reduce(Btree, [ + {dir, rev}, + {start_key, SK}, + {end_key, EK} + ]) + ) }, { "exclude endkey", ?_assertMatch( - {ok, [{{"even", ?ROWS}, ?ROWS div 4}, - {{"odd", ?ROWS - 1}, ?ROWS div 2}]}, - fold_reduce(Btree, [{dir, rev}, - {start_key, SK}, - {end_key_gt, EK}])) + {ok, [ + {{"even", ?ROWS}, ?ROWS div 4}, + {{"odd", ?ROWS - 1}, ?ROWS div 2} + ]}, + fold_reduce(Btree, [ + {dir, rev}, + {start_key, SK}, + {end_key_gt, EK} + ]) + ) } ]. @@ -404,20 +474,30 @@ should_reduce_second_half(fwd, {_, Btree}) -> { "include endkey", ?_assertMatch( - {ok, [{{"odd", 1}, ?ROWS div 2}, - {{"even", ?ROWS div 2}, (?ROWS div 4) + 1}]}, - fold_reduce(Btree, [{dir, fwd}, - {start_key, SK}, - {end_key, EK}])) + {ok, [ + {{"odd", 1}, ?ROWS div 2}, + {{"even", ?ROWS div 2}, (?ROWS div 4) + 1} + ]}, + fold_reduce(Btree, [ + {dir, fwd}, + {start_key, SK}, + {end_key, EK} + ]) + ) }, { "exclude endkey", ?_assertMatch( - {ok, [{{"odd", 1}, (?ROWS div 2) - 1}, - {{"even", ?ROWS div 2}, (?ROWS div 4) + 1}]}, - fold_reduce(Btree, [{dir, fwd}, - {start_key, SK}, - {end_key_gt, EK}])) + {ok, [ + {{"odd", 1}, (?ROWS div 2) - 1}, + {{"even", ?ROWS div 2}, (?ROWS div 4) + 1} + ]}, + fold_reduce(Btree, [ + {dir, fwd}, + {start_key, SK}, + {end_key_gt, EK} + ]) + ) } ]; should_reduce_second_half(rev, {_, Btree}) -> @@ -426,20 +506,30 @@ should_reduce_second_half(rev, {_, Btree}) -> { "include endkey", ?_assertMatch( - {ok, [{{"even", ?ROWS}, ?ROWS div 2}, - {{"odd", (?ROWS div 2) + 1}, (?ROWS div 4) + 1}]}, - fold_reduce(Btree, [{dir, rev}, - {start_key, SK}, - {end_key, EK}])) + {ok, [ + {{"even", ?ROWS}, ?ROWS div 2}, + {{"odd", (?ROWS div 2) + 1}, (?ROWS div 4) + 1} + ]}, + fold_reduce(Btree, [ + {dir, rev}, + {start_key, SK}, + {end_key, EK} + ]) + ) }, { "exclude endkey", ?_assertMatch( - {ok, [{{"even", ?ROWS}, (?ROWS div 2) - 1}, - {{"odd", (?ROWS div 2) + 1}, (?ROWS div 4) + 1}]}, - fold_reduce(Btree, [{dir, rev}, - {start_key, SK}, - {end_key_gt, EK}])) + {ok, [ + {{"even", ?ROWS}, (?ROWS div 2) - 1}, + {{"odd", (?ROWS div 2) + 1}, (?ROWS div 4) + 1} + ]}, + fold_reduce(Btree, [ + {dir, rev}, + {start_key, SK}, + {end_key_gt, EK} + ]) + ) } ]. @@ -459,9 +549,12 @@ fold_reduce(Btree, Opts) -> FoldFun = fun(GroupedKey, Unreduced, Acc) -> {ok, [{GroupedKey, couch_btree:final_reduce(Btree, Unreduced)} | Acc]} end, - couch_btree:fold_reduce(Btree, FoldFun, [], - [{key_group_fun, GroupFun}] ++ Opts). - + couch_btree:fold_reduce( + Btree, + FoldFun, + [], + [{key_group_fun, GroupFun}] ++ Opts + ). keys(KVs) -> [K || {K, _} <- KVs]. @@ -471,7 +564,6 @@ reduce_fun(reduce, KVs) -> reduce_fun(rereduce, Reds) -> lists:sum(Reds). - shuffle(List) -> randomize(round(math:log(length(List)) + 0.5), List). @@ -481,7 +573,10 @@ randomize(T, List) -> lists:foldl( fun(_E, Acc) -> randomize(Acc) - end, randomize(List), lists:seq(1, (T - 1))). + end, + randomize(List), + lists:seq(1, (T - 1)) + ). randomize(List) -> D = lists:map(fun(A) -> {couch_rand:uniform(), A} end, List), @@ -500,18 +595,24 @@ test_add_remove(Btree, OutKeyValues, RemainingKeyValues) -> fun({K, _}, BtAcc) -> {ok, BtAcc2} = couch_btree:add_remove(BtAcc, [], [K]), BtAcc2 - end, Btree, OutKeyValues), + end, + Btree, + OutKeyValues + ), true = test_btree(Btree2, RemainingKeyValues), Btree3 = lists:foldl( fun(KV, BtAcc) -> {ok, BtAcc2} = couch_btree:add_remove(BtAcc, [KV], []), BtAcc2 - end, Btree2, OutKeyValues), + end, + Btree2, + OutKeyValues + ), true = test_btree(Btree3, OutKeyValues ++ RemainingKeyValues). test_key_access(Btree, List) -> - FoldFun = fun(Element, {[HAcc|TAcc], Count}) -> + FoldFun = fun(Element, {[HAcc | TAcc], Count}) -> case Element == HAcc of true -> {ok, {TAcc, Count + 1}}; _ -> {ok, {TAcc, Count + 1}} @@ -520,8 +621,12 @@ test_key_access(Btree, List) -> Length = length(List), Sorted = lists:sort(List), {ok, _, {[], Length}} = couch_btree:foldl(Btree, FoldFun, {Sorted, 0}), - {ok, _, {[], Length}} = couch_btree:fold(Btree, FoldFun, - {Sorted, 0}, [{dir, rev}]), + {ok, _, {[], Length}} = couch_btree:fold( + Btree, + FoldFun, + {Sorted, 0}, + [{dir, rev}] + ), ok. test_lookup_access(Btree, KeyValues) -> @@ -529,9 +634,15 @@ test_lookup_access(Btree, KeyValues) -> lists:foreach( fun({Key, Value}) -> [{ok, {Key, Value}}] = couch_btree:lookup(Btree, [Key]), - {ok, _, true} = couch_btree:foldl(Btree, FoldFun, - {Key, Value}, [{start_key, Key}]) - end, KeyValues). + {ok, _, true} = couch_btree:foldl( + Btree, + FoldFun, + {Key, Value}, + [{start_key, Key}] + ) + end, + KeyValues + ). test_final_reductions(Btree, KeyValues) -> KVLen = length(KeyValues), @@ -545,18 +656,28 @@ test_final_reductions(Btree, KeyValues) -> CountToEnd = couch_btree:final_reduce(Btree, LeadingReds), {ok, Acc + 1} end, - {LStartKey, _} = case KVLen of - 0 -> {nil, nil}; - _ -> lists:nth(KVLen div 3 + 1, lists:sort(KeyValues)) - end, - {RStartKey, _} = case KVLen of - 0 -> {nil, nil}; - _ -> lists:nth(KVLen div 3, lists:sort(KeyValues)) - end, - {ok, _, FoldLRed} = couch_btree:foldl(Btree, FoldLFun, 0, - [{start_key, LStartKey}]), - {ok, _, FoldRRed} = couch_btree:fold(Btree, FoldRFun, 0, - [{dir, rev}, {start_key, RStartKey}]), + {LStartKey, _} = + case KVLen of + 0 -> {nil, nil}; + _ -> lists:nth(KVLen div 3 + 1, lists:sort(KeyValues)) + end, + {RStartKey, _} = + case KVLen of + 0 -> {nil, nil}; + _ -> lists:nth(KVLen div 3, lists:sort(KeyValues)) + end, + {ok, _, FoldLRed} = couch_btree:foldl( + Btree, + FoldLFun, + 0, + [{start_key, LStartKey}] + ), + {ok, _, FoldRRed} = couch_btree:fold( + Btree, + FoldRFun, + 0, + [{dir, rev}, {start_key, RStartKey}] + ), KVLen = FoldLRed + FoldRRed, ok. diff --git a/src/couch/test/eunit/couch_changes_tests.erl b/src/couch/test/eunit/couch_changes_tests.erl index 848b471f9..02b69f132 100644 --- a/src/couch/test/eunit/couch_changes_tests.erl +++ b/src/couch/test/eunit/couch_changes_tests.erl @@ -28,39 +28,49 @@ setup() -> DbName = ?tempdb(), {ok, Db} = create_db(DbName), - Revs = [R || {ok, R} <- [ - save_doc(Db, {[{<<"_id">>, <<"doc1">>}]}), - save_doc(Db, {[{<<"_id">>, <<"doc2">>}]}), - save_doc(Db, {[{<<"_id">>, <<"doc3">>}]}), - save_doc(Db, {[{<<"_id">>, <<"doc4">>}]}), - save_doc(Db, {[{<<"_id">>, <<"doc5">>}]}) - ]], + Revs = [ + R + || {ok, R} <- [ + save_doc(Db, {[{<<"_id">>, <<"doc1">>}]}), + save_doc(Db, {[{<<"_id">>, <<"doc2">>}]}), + save_doc(Db, {[{<<"_id">>, <<"doc3">>}]}), + save_doc(Db, {[{<<"_id">>, <<"doc4">>}]}), + save_doc(Db, {[{<<"_id">>, <<"doc5">>}]}) + ] + ], Rev = lists:nth(3, Revs), {ok, Db1} = couch_db:reopen(Db), {ok, Rev1} = save_doc(Db1, {[{<<"_id">>, <<"doc3">>}, {<<"_rev">>, Rev}]}), Revs1 = Revs ++ [Rev1], - Revs2 = Revs1 ++ [R || {ok, R} <- [ - save_doc(Db1, {[{<<"_id">>, <<"doc6">>}]}), - save_doc(Db1, {[{<<"_id">>, <<"_design/foo">>}]}), - save_doc(Db1, {[{<<"_id">>, <<"doc7">>}]}), - save_doc(Db1, {[{<<"_id">>, <<"doc8">>}]}) - ]], - config:set("native_query_servers", "erlang", "{couch_native_process, start_link, []}", _Persist=false), + Revs2 = + Revs1 ++ + [ + R + || {ok, R} <- [ + save_doc(Db1, {[{<<"_id">>, <<"doc6">>}]}), + save_doc(Db1, {[{<<"_id">>, <<"_design/foo">>}]}), + save_doc(Db1, {[{<<"_id">>, <<"doc7">>}]}), + save_doc(Db1, {[{<<"_id">>, <<"doc8">>}]}) + ] + ], + config:set( + "native_query_servers", "erlang", "{couch_native_process, start_link, []}", _Persist = false + ), {DbName, list_to_tuple(Revs2)}. teardown({DbName, _}) -> - config:delete("native_query_servers", "erlang", _Persist=false), + config:delete("native_query_servers", "erlang", _Persist = false), delete_db(DbName), ok. - changes_test_() -> { "Changes feed", { setup, - fun test_util:start_couch/0, fun test_util:stop_couch/1, + fun test_util:start_couch/0, + fun test_util:stop_couch/1, [ filter_by_selector(), filter_by_doc_id(), @@ -78,7 +88,8 @@ filter_by_doc_id() -> "Filter _doc_id", { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_filter_by_specific_doc_ids/1, fun should_filter_by_specific_doc_ids_descending/1, @@ -94,7 +105,8 @@ filter_by_selector() -> "Filter _selector", { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_select_basic/1, fun should_select_with_since/1, @@ -108,13 +120,13 @@ filter_by_selector() -> } }. - filter_by_design() -> { "Filter _design", { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_emit_only_design_documents/1 ] @@ -138,7 +150,8 @@ filter_by_filter_function() -> "Filter by filters", { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_filter_by_doc_attribute/1, fun should_filter_by_user_ctx/1 @@ -151,7 +164,8 @@ filter_by_view() -> "Filter _view", { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_filter_by_view/1, fun should_filter_by_erlang_view/1 @@ -164,7 +178,8 @@ continuous_feed() -> "Continuous Feed", { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_filter_continuous_feed_by_specific_doc_ids/1, fun should_end_changes_when_db_deleted/1 @@ -172,7 +187,6 @@ continuous_feed() -> } }. - should_filter_by_specific_doc_ids({DbName, _}) -> ?_test( begin @@ -190,7 +204,8 @@ should_filter_by_specific_doc_ids({DbName, _}) -> ?assertEqual(<<"doc3">>, Id2), ?assertEqual(6, Seq2), ?assertEqual(UpSeq, LastSeq) - end). + end + ). should_filter_by_specific_doc_ids_descending({DbName, _}) -> ?_test( @@ -210,7 +225,8 @@ should_filter_by_specific_doc_ids_descending({DbName, _}) -> ?assertEqual(<<"doc4">>, Id2), ?assertEqual(4, Seq2), ?assertEqual(4, LastSeq) - end). + end + ). should_filter_by_specific_doc_ids_with_since({DbName, _}) -> ?_test( @@ -228,7 +244,8 @@ should_filter_by_specific_doc_ids_with_since({DbName, _}) -> ?assertEqual(<<"doc3">>, Id1), ?assertEqual(6, Seq1), ?assertEqual(UpSeq, LastSeq) - end). + end + ). should_filter_by_specific_doc_ids_no_result({DbName, _}) -> ?_test( @@ -243,7 +260,8 @@ should_filter_by_specific_doc_ids_no_result({DbName, _}) -> ?assertEqual(0, length(Rows)), ?assertEqual(UpSeq, LastSeq) - end). + end + ). should_handle_deleted_docs({DbName, Revs}) -> ?_test( @@ -252,9 +270,12 @@ should_handle_deleted_docs({DbName, Revs}) -> {ok, Db} = couch_db:open_int(DbName, []), {ok, _} = save_doc( Db, - {[{<<"_id">>, <<"doc3">>}, - {<<"_deleted">>, true}, - {<<"_rev">>, Rev3_2}]}), + {[ + {<<"_id">>, <<"doc3">>}, + {<<"_deleted">>, true}, + {<<"_rev">>, Rev3_2} + ]} + ), ChArgs = #changes_args{ filter = "_doc_ids", @@ -270,7 +291,8 @@ should_handle_deleted_docs({DbName, Revs}) -> Rows ), ?assertEqual(11, LastSeq) - end). + end + ). should_filter_continuous_feed_by_specific_doc_ids({DbName, Revs}) -> ?_test( @@ -305,14 +327,29 @@ should_filter_continuous_feed_by_specific_doc_ids({DbName, Revs}) -> Rev4 = element(4, Revs), Rev3_2 = element(6, Revs), - {ok, Rev4_2} = save_doc(Db, {[{<<"_id">>, <<"doc4">>}, - {<<"_rev">>, Rev4}]}), + {ok, Rev4_2} = save_doc( + Db, + {[ + {<<"_id">>, <<"doc4">>}, + {<<"_rev">>, Rev4} + ]} + ), {ok, _} = save_doc(Db, {[{<<"_id">>, <<"doc11">>}]}), - {ok, _} = save_doc(Db, {[{<<"_id">>, <<"doc4">>}, - {<<"_rev">>, Rev4_2}]}), + {ok, _} = save_doc( + Db, + {[ + {<<"_id">>, <<"doc4">>}, + {<<"_rev">>, Rev4_2} + ]} + ), {ok, _} = save_doc(Db, {[{<<"_id">>, <<"doc12">>}]}), - {ok, Rev3_3} = save_doc(Db, {[{<<"_id">>, <<"doc3">>}, - {<<"_rev">>, Rev3_2}]}), + {ok, Rev3_3} = save_doc( + Db, + {[ + {<<"_id">>, <<"doc3">>}, + {<<"_rev">>, Rev3_2} + ]} + ), reset_row_notifications(), ok = unpause(Consumer), ?assertEqual(ok, wait_row_notifications(2)), @@ -327,8 +364,13 @@ should_filter_continuous_feed_by_specific_doc_ids({DbName, Revs}) -> ?assertEqual(17, Row16#row.seq), clear_rows(Consumer), - {ok, _Rev3_4} = save_doc(Db, {[{<<"_id">>, <<"doc3">>}, - {<<"_rev">>, Rev3_3}]}), + {ok, _Rev3_4} = save_doc( + Db, + {[ + {<<"_id">>, <<"doc3">>}, + {<<"_rev">>, Rev3_3} + ]} + ), reset_row_notifications(), ok = unpause(Consumer), ?assertEqual(ok, wait_row_notifications(1)), @@ -340,8 +382,8 @@ should_filter_continuous_feed_by_specific_doc_ids({DbName, Revs}) -> stop_consumer(Consumer), ?assertMatch([#row{seq = 18, id = <<"doc3">>}], FinalRows) - end). - + end + ). should_end_changes_when_db_deleted({DbName, _Revs}) -> ?_test(begin @@ -361,7 +403,6 @@ should_end_changes_when_db_deleted({DbName, _Revs}) -> ok end). - should_select_basic({DbName, _}) -> ?_test( begin @@ -374,7 +415,8 @@ should_select_basic({DbName, _}) -> ?assertEqual(<<"doc3">>, Id), ?assertEqual(6, Seq), ?assertEqual(UpSeq, LastSeq) - end). + end + ). should_select_with_since({DbName, _}) -> ?_test( @@ -389,7 +431,8 @@ should_select_with_since({DbName, _}) -> ?assertEqual(<<"doc8">>, Id), ?assertEqual(10, Seq), ?assertEqual(UpSeq, LastSeq) - end). + end + ). should_select_when_no_result({DbName, _}) -> ?_test( @@ -400,7 +443,8 @@ should_select_when_no_result({DbName, _}) -> {Rows, LastSeq, UpSeq} = run_changes_query(DbName, ChArgs, Req), ?assertEqual(0, length(Rows)), ?assertEqual(UpSeq, LastSeq) - end). + end + ). should_select_with_deleted_docs({DbName, Revs}) -> ?_test( @@ -409,9 +453,12 @@ should_select_with_deleted_docs({DbName, Revs}) -> {ok, Db} = couch_db:open_int(DbName, []), {ok, _} = save_doc( Db, - {[{<<"_id">>, <<"doc3">>}, - {<<"_deleted">>, true}, - {<<"_rev">>, Rev3_2}]}), + {[ + {<<"_id">>, <<"doc3">>}, + {<<"_deleted">>, true}, + {<<"_rev">>, Rev3_2} + ]} + ), ChArgs = #changes_args{filter = "_selector"}, Selector = {[{<<"_id">>, <<"doc3">>}]}, Req = {json_req, {[{<<"selector">>, Selector}]}}, @@ -421,7 +468,8 @@ should_select_with_deleted_docs({DbName, Revs}) -> Rows ), ?assertEqual(11, LastSeq) - end). + end + ). should_select_with_continuous({DbName, Revs}) -> ?_test( @@ -437,8 +485,8 @@ should_select_with_continuous({DbName, Revs}) -> ok = pause(Consumer), Rows = get_rows(Consumer), ?assertMatch( - [#row{seq = 10, id = <<"doc8">>, deleted = false}], - Rows + [#row{seq = 10, id = <<"doc8">>, deleted = false}], + Rows ), clear_rows(Consumer), {ok, _} = save_doc(Db, {[{<<"_id">>, <<"doc01">>}]}), @@ -448,45 +496,60 @@ should_select_with_continuous({DbName, Revs}) -> ?assertEqual([], get_rows(Consumer)), Rev4 = element(4, Revs), Rev8 = element(10, Revs), - {ok, _} = save_doc(Db, {[{<<"_id">>, <<"doc8">>}, - {<<"_rev">>, Rev8}]}), - {ok, _} = save_doc(Db, {[{<<"_id">>, <<"doc4">>}, - {<<"_rev">>, Rev4}]}), + {ok, _} = save_doc( + Db, + {[ + {<<"_id">>, <<"doc8">>}, + {<<"_rev">>, Rev8} + ]} + ), + {ok, _} = save_doc( + Db, + {[ + {<<"_id">>, <<"doc4">>}, + {<<"_rev">>, Rev4} + ]} + ), reset_row_notifications(), ok = unpause(Consumer), ?assertEqual(ok, wait_row_notifications(1)), ok = pause(Consumer), NewRows = get_rows(Consumer), ?assertMatch( - [#row{seq = _, id = <<"doc8">>, deleted = false}], - NewRows + [#row{seq = _, id = <<"doc8">>, deleted = false}], + NewRows ) - end). + end + ). should_stop_selector_when_db_deleted({DbName, _Revs}) -> ?_test( - begin - {ok, _Db} = couch_db:open_int(DbName, []), - ChArgs = #changes_args{filter = "_selector", feed = "continuous"}, - Selector = {[{<<"_id">>, <<"doc3">>}]}, - Req = {json_req, {[{<<"selector">>, Selector}]}}, - Consumer = spawn_consumer(DbName, ChArgs, Req), - ok = pause(Consumer), - ok = couch_server:delete(DbName, [?ADMIN_CTX]), - ok = unpause(Consumer), - {_Rows, _LastSeq} = wait_finished(Consumer), - stop_consumer(Consumer), - ok - end). - + begin + {ok, _Db} = couch_db:open_int(DbName, []), + ChArgs = #changes_args{filter = "_selector", feed = "continuous"}, + Selector = {[{<<"_id">>, <<"doc3">>}]}, + Req = {json_req, {[{<<"selector">>, Selector}]}}, + Consumer = spawn_consumer(DbName, ChArgs, Req), + ok = pause(Consumer), + ok = couch_server:delete(DbName, [?ADMIN_CTX]), + ok = unpause(Consumer), + {_Rows, _LastSeq} = wait_finished(Consumer), + stop_consumer(Consumer), + ok + end + ). should_select_with_empty_fields({DbName, _}) -> ?_test( begin - ChArgs = #changes_args{filter = "_selector", include_docs=true}, + ChArgs = #changes_args{filter = "_selector", include_docs = true}, Selector = {[{<<"_id">>, <<"doc3">>}]}, - Req = {json_req, {[{<<"selector">>, Selector}, - {<<"fields">>, []}]}}, + Req = + {json_req, + {[ + {<<"selector">>, Selector}, + {<<"fields">>, []} + ]}}, {Rows, LastSeq, UpSeq} = run_changes_query(DbName, ChArgs, Req), ?assertEqual(1, length(Rows)), [#row{seq = Seq, id = Id, doc = Doc}] = Rows, @@ -494,15 +557,20 @@ should_select_with_empty_fields({DbName, _}) -> ?assertEqual(6, Seq), ?assertEqual(UpSeq, LastSeq), ?assertMatch({[{_K1, _V1}, {_K2, _V2}]}, Doc) - end). + end + ). should_select_with_fields({DbName, _}) -> ?_test( begin - ChArgs = #changes_args{filter = "_selector", include_docs=true}, + ChArgs = #changes_args{filter = "_selector", include_docs = true}, Selector = {[{<<"_id">>, <<"doc3">>}]}, - Req = {json_req, {[{<<"selector">>, Selector}, - {<<"fields">>, [<<"_id">>, <<"nope">>]}]}}, + Req = + {json_req, + {[ + {<<"selector">>, Selector}, + {<<"fields">>, [<<"_id">>, <<"nope">>]} + ]}}, {Rows, LastSeq, UpSeq} = run_changes_query(DbName, ChArgs, Req), ?assertEqual(1, length(Rows)), [#row{seq = Seq, id = Id, doc = Doc}] = Rows, @@ -510,8 +578,8 @@ should_select_with_fields({DbName, _}) -> ?assertEqual(6, Seq), ?assertEqual(UpSeq, LastSeq), ?assertMatch(Doc, {[{<<"_id">>, <<"doc3">>}]}) - end). - + end + ). should_emit_only_design_documents({DbName, Revs}) -> ?_test( @@ -526,11 +594,15 @@ should_emit_only_design_documents({DbName, Revs}) -> ?assertEqual(UpSeq, LastSeq), ?assertEqual([#row{seq = 8, id = <<"_design/foo">>}], Rows), - {ok, Db} = couch_db:open_int(DbName, [?ADMIN_CTX]), - {ok, _} = save_doc(Db, {[{<<"_id">>, <<"_design/foo">>}, - {<<"_rev">>, element(8, Revs)}, - {<<"_deleted">>, true}]}), + {ok, _} = save_doc( + Db, + {[ + {<<"_id">>, <<"_design/foo">>}, + {<<"_rev">>, element(8, Revs)}, + {<<"_deleted">>, true} + ]} + ), couch_db:close(Db), {Rows2, LastSeq2, _} = run_changes_query(DbName, ChArgs, Req), @@ -539,11 +611,18 @@ should_emit_only_design_documents({DbName, Revs}) -> ?assertEqual(1, length(Rows2)), ?assertEqual(UpSeq2, LastSeq2), - ?assertEqual([#row{seq = 11, - id = <<"_design/foo">>, - deleted = true}], - Rows2) - end). + ?assertEqual( + [ + #row{ + seq = 11, + id = <<"_design/foo">>, + deleted = true + } + ], + Rows2 + ) + end + ). %% should_receive_heartbeats(_) -> %% {timeout, ?TEST_TIMEOUT div 1000, @@ -616,16 +695,21 @@ should_filter_by_doc_attribute({DbName, _}) -> ?_test( begin DDocId = <<"_design/app">>, - DDoc = couch_doc:from_json_obj({[ - {<<"_id">>, DDocId}, - {<<"language">>, <<"javascript">>}, - {<<"filters">>, {[ - {<<"valid">>, <<"function(doc, req) {" - " if (doc._id == 'doc3') {" - " return true; " - "} }">>} - ]}} - ]}), + DDoc = couch_doc:from_json_obj( + {[ + {<<"_id">>, DDocId}, + {<<"language">>, <<"javascript">>}, + {<<"filters">>, + {[ + {<<"valid">>, << + "function(doc, req) {" + " if (doc._id == 'doc3') {" + " return true; " + "} }" + >>} + ]}} + ]} + ), ChArgs = #changes_args{filter = "app/valid"}, Req = {json_req, null}, ok = update_ddoc(DbName, DDoc), @@ -635,28 +719,38 @@ should_filter_by_doc_attribute({DbName, _}) -> ?assertEqual(<<"doc3">>, Id), ?assertEqual(6, Seq), ?assertEqual(UpSeq, LastSeq) - end). + end + ). should_filter_by_user_ctx({DbName, _}) -> ?_test( begin DDocId = <<"_design/app">>, - DDoc = couch_doc:from_json_obj({[ - {<<"_id">>, DDocId}, - {<<"language">>, <<"javascript">>}, - {<<"filters">>, {[ - {<<"valid">>, <<"function(doc, req) {" - " if (req.userCtx.name == doc._id) {" - " return true; " - "} }">>} - ]}} - ]}), + DDoc = couch_doc:from_json_obj( + {[ + {<<"_id">>, DDocId}, + {<<"language">>, <<"javascript">>}, + {<<"filters">>, + {[ + {<<"valid">>, << + "function(doc, req) {" + " if (req.userCtx.name == doc._id) {" + " return true; " + "} }" + >>} + ]}} + ]} + ), ChArgs = #changes_args{filter = "app/valid"}, UserCtx = #user_ctx{name = <<"doc3">>, roles = []}, {ok, DbRec} = couch_db:clustered_db(DbName, UserCtx), - Req = {json_req, {[{ - <<"userCtx">>, couch_util:json_user_ctx(DbRec) - }]}}, + Req = + {json_req, + {[ + { + <<"userCtx">>, couch_util:json_user_ctx(DbRec) + } + ]}}, ok = update_ddoc(DbName, DDoc), {Rows, LastSeq, UpSeq} = run_changes_query(DbName, ChArgs, Req), ?assertEqual(1, length(Rows)), @@ -664,30 +758,42 @@ should_filter_by_user_ctx({DbName, _}) -> ?assertEqual(<<"doc3">>, Id), ?assertEqual(6, Seq), ?assertEqual(UpSeq, LastSeq) - end). + end + ). should_filter_by_view({DbName, _}) -> ?_test( begin DDocId = <<"_design/app">>, - DDoc = couch_doc:from_json_obj({[ - {<<"_id">>, DDocId}, - {<<"language">>, <<"javascript">>}, - {<<"views">>, {[ - {<<"valid">>, {[ - {<<"map">>, <<"function(doc) {" - " if (doc._id == 'doc3') {" - " emit(doc); " - "} }">>} - ]}} - ]}} - ]}), - ChArgs = #changes_args{filter = "_view"}, - Req = {json_req, {[{ - <<"query">>, {[ - {<<"view">>, <<"app/valid">>} + DDoc = couch_doc:from_json_obj( + {[ + {<<"_id">>, DDocId}, + {<<"language">>, <<"javascript">>}, + {<<"views">>, + {[ + {<<"valid">>, + {[ + {<<"map">>, << + "function(doc) {" + " if (doc._id == 'doc3') {" + " emit(doc); " + "} }" + >>} + ]}} + ]}} ]} - }]}}, + ), + ChArgs = #changes_args{filter = "_view"}, + Req = + {json_req, + {[ + { + <<"query">>, + {[ + {<<"view">>, <<"app/valid">>} + ]} + } + ]}}, ok = update_ddoc(DbName, DDoc), {Rows, LastSeq, UpSeq} = run_changes_query(DbName, ChArgs, Req), ?assertEqual(1, length(Rows)), @@ -695,32 +801,44 @@ should_filter_by_view({DbName, _}) -> ?assertEqual(<<"doc3">>, Id), ?assertEqual(6, Seq), ?assertEqual(UpSeq, LastSeq) - end). + end + ). should_filter_by_erlang_view({DbName, _}) -> ?_test( begin DDocId = <<"_design/app">>, - DDoc = couch_doc:from_json_obj({[ - {<<"_id">>, DDocId}, - {<<"language">>, <<"erlang">>}, - {<<"views">>, {[ - {<<"valid">>, {[ - {<<"map">>, <<"fun({Doc}) ->" - " case lists:keyfind(<<\"_id\">>, 1, Doc) of" - " {<<\"_id\">>, <<\"doc3\">>} -> Emit(Doc, null); " - " false -> ok" - " end " - "end.">>} - ]}} - ]}} - ]}), - ChArgs = #changes_args{filter = "_view"}, - Req = {json_req, {[{ - <<"query">>, {[ - {<<"view">>, <<"app/valid">>} + DDoc = couch_doc:from_json_obj( + {[ + {<<"_id">>, DDocId}, + {<<"language">>, <<"erlang">>}, + {<<"views">>, + {[ + {<<"valid">>, + {[ + {<<"map">>, << + "fun({Doc}) ->" + " case lists:keyfind(<<\"_id\">>, 1, Doc) of" + " {<<\"_id\">>, <<\"doc3\">>} -> Emit(Doc, null); " + " false -> ok" + " end " + "end." + >>} + ]}} + ]}} ]} - }]}}, + ), + ChArgs = #changes_args{filter = "_view"}, + Req = + {json_req, + {[ + { + <<"query">>, + {[ + {<<"view">>, <<"app/valid">>} + ]} + } + ]}}, ok = update_ddoc(DbName, DDoc), {Rows, LastSeq, UpSeq} = run_changes_query(DbName, ChArgs, Req), ?assertEqual(1, length(Rows)), @@ -728,7 +846,8 @@ should_filter_by_erlang_view({DbName, _}) -> ?assertEqual(<<"doc3">>, Id), ?assertEqual(6, Seq), ?assertEqual(UpSeq, LastSeq) - end). + end + ). update_ddoc(DbName, DDoc) -> {ok, Db} = couch_db:open_int(DbName, [?ADMIN_CTX]), @@ -752,12 +871,13 @@ save_doc(Db, Json) -> get_rows({Consumer, _}) -> Ref = make_ref(), Consumer ! {get_rows, Ref}, - Resp = receive - {rows, Ref, Rows} -> - Rows - after ?TIMEOUT -> - timeout - end, + Resp = + receive + {rows, Ref, Rows} -> + Rows + after ?TIMEOUT -> + timeout + end, ?assertNotEqual(timeout, Resp), Resp. @@ -776,48 +896,52 @@ get_rows({Consumer, _}) -> clear_rows({Consumer, _}) -> Ref = make_ref(), Consumer ! {reset, Ref}, - Resp = receive - {ok, Ref} -> - ok - after ?TIMEOUT -> - timeout - end, + Resp = + receive + {ok, Ref} -> + ok + after ?TIMEOUT -> + timeout + end, ?assertNotEqual(timeout, Resp), Resp. stop_consumer({Consumer, _}) -> Ref = make_ref(), Consumer ! {stop, Ref}, - Resp = receive - {ok, Ref} -> - ok - after ?TIMEOUT -> - timeout - end, + Resp = + receive + {ok, Ref} -> + ok + after ?TIMEOUT -> + timeout + end, ?assertNotEqual(timeout, Resp), Resp. pause({Consumer, _}) -> Ref = make_ref(), Consumer ! {pause, Ref}, - Resp = receive - {paused, Ref} -> - ok - after ?TIMEOUT -> - timeout - end, + Resp = + receive + {paused, Ref} -> + ok + after ?TIMEOUT -> + timeout + end, ?assertNotEqual(timeout, Resp), Resp. unpause({Consumer, _}) -> Ref = make_ref(), Consumer ! {continue, Ref}, - Resp = receive - {ok, Ref} -> - ok - after ?TIMEOUT -> - timeout - end, + Resp = + receive + {ok, Ref} -> + ok + after ?TIMEOUT -> + timeout + end, ?assertNotEqual(timeout, Resp), Resp. @@ -828,20 +952,23 @@ wait_finished({_, ConsumerRef}) -> {'DOWN', ConsumerRef, _, _, Msg} when Msg == normal; Msg == ok -> ok; {'DOWN', ConsumerRef, _, _, Msg} -> - erlang:error({consumer_died, [ + erlang:error( + {consumer_died, [ + {module, ?MODULE}, + {line, ?LINE}, + {value, Msg} + ]} + ) + after ?TIMEOUT -> + erlang:error( + {consumer_died, [ {module, ?MODULE}, {line, ?LINE}, - {value, Msg} - ]}) - after ?TIMEOUT -> - erlang:error({consumer_died, [ - {module, ?MODULE}, - {line, ?LINE}, - {value, timeout} - ]}) + {value, timeout} + ]} + ) end. - reset_row_notifications() -> receive row -> @@ -850,7 +977,6 @@ reset_row_notifications() -> ok end. - wait_row_notifications(N) -> receive row when N == 1 -> @@ -861,7 +987,6 @@ wait_row_notifications(N) -> timeout end. - spawn_consumer(DbName, ChangesArgs0, Req) -> Parent = self(), spawn_monitor(fun() -> @@ -884,13 +1009,16 @@ spawn_consumer(DbName, ChangesArgs0, Req) -> maybe_pause(Parent, Acc) end, {ok, Db} = couch_db:open_int(DbName, []), - ChangesArgs = case (ChangesArgs0#changes_args.timeout =:= undefined) - andalso (ChangesArgs0#changes_args.heartbeat =:= undefined) of - true -> - ChangesArgs0#changes_args{timeout = 1000, heartbeat = 100}; - false -> - ChangesArgs0 - end, + ChangesArgs = + case + (ChangesArgs0#changes_args.timeout =:= undefined) andalso + (ChangesArgs0#changes_args.heartbeat =:= undefined) + of + true -> + ChangesArgs0#changes_args{timeout = 1000, heartbeat = 100}; + false -> + ChangesArgs0 + end, FeedFun = couch_changes:handle_db_changes(ChangesArgs, Req, Db), try FeedFun({Callback, []}) @@ -920,11 +1048,14 @@ maybe_pause(Parent, Acc) -> Parent ! {ok, Ref}, throw({stop, Acc}); V when V /= updated -> - erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {value, V}, - {reason, "Received unexpected message"}]}) + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {value, V}, + {reason, "Received unexpected message"} + ]} + ) after 0 -> Acc end. diff --git a/src/couch/test/eunit/couch_db_doc_tests.erl b/src/couch/test/eunit/couch_db_doc_tests.erl index 916b63207..dc1ac79e6 100644 --- a/src/couch/test/eunit/couch_db_doc_tests.erl +++ b/src/couch/test/eunit/couch_db_doc_tests.erl @@ -18,7 +18,6 @@ start() -> test_util:start_couch([ioq]). - setup() -> DbName = ?tempdb(), config:set("couchdb", "stem_interactive_updates", "false", false), @@ -26,21 +25,21 @@ setup() -> couch_db:close(Db), DbName. - teardown(DbName) -> ok = couch_server:delete(DbName, [?ADMIN_CTX]), ok. - couch_db_doc_test_() -> { "CouchDB doc tests", { setup, - fun start/0, fun test_util:stop_couch/1, + fun start/0, + fun test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_truncate_number_of_revisions/1, fun should_raise_bad_request_on_invalid_rev/1, @@ -50,7 +49,6 @@ couch_db_doc_test_() -> } }. - should_truncate_number_of_revisions(DbName) -> DocId = <<"foo">>, Db = open_db(DbName), @@ -60,7 +58,6 @@ should_truncate_number_of_revisions(DbName) -> {ok, [{ok, #doc{revs = {11, Revs}}}]} = open_doc_rev(Db, DocId, Rev10), ?_assertEqual(5, length(Revs)). - should_raise_bad_request_on_invalid_rev(DbName) -> DocId = <<"foo">>, InvalidRev1 = <<"foo">>, @@ -70,18 +67,15 @@ should_raise_bad_request_on_invalid_rev(DbName) -> Db = open_db(DbName), create_doc(Db, DocId), [ - {InvalidRev1, - ?_assertThrow(Expect, add_revisions(Db, DocId, InvalidRev1, 1))}, - {InvalidRev2, - ?_assertThrow(Expect, add_revisions(Db, DocId, InvalidRev2, 1))}, - {InvalidRev3, - ?_assertThrow(Expect, add_revisions(Db, DocId, InvalidRev3, 1))} + {InvalidRev1, ?_assertThrow(Expect, add_revisions(Db, DocId, InvalidRev1, 1))}, + {InvalidRev2, ?_assertThrow(Expect, add_revisions(Db, DocId, InvalidRev2, 1))}, + {InvalidRev3, ?_assertThrow(Expect, add_revisions(Db, DocId, InvalidRev3, 1))} ]. should_allow_access_in_doc_keys_test(_DbName) -> Json = <<"{\"_id\":\"foo\",\"_access\":[\"test\"]}">>, EJson = couch_util:json_decode(Json), - Expected = {[{<<"_id">>,<<"foo">>}, {<<"_access">>, [<<"test">>]}]}, + Expected = {[{<<"_id">>, <<"foo">>}, {<<"_access">>, [<<"test">>]}]}, EJson = Expected, Doc = couch_doc:from_json_obj(EJson), NewEJson = couch_doc:to_json_obj(Doc, []), @@ -91,31 +85,33 @@ open_db(DbName) -> {ok, Db} = couch_db:open_int(DbName, [?ADMIN_CTX]), Db. - create_doc(Db, DocId) -> add_revision(Db, DocId, undefined). - open_doc_rev(Db0, DocId, Rev) -> {ok, Db} = couch_db:reopen(Db0), couch_db:open_doc_revs(Db, DocId, [couch_doc:parse_rev(Rev)], []). - add_revision(Db, DocId, undefined) -> add_revision(Db, DocId, []); add_revision(Db, DocId, Rev) when is_binary(Rev) -> add_revision(Db, DocId, [{<<"_rev">>, Rev}]); add_revision(Db0, DocId, Rev) -> {ok, Db} = couch_db:reopen(Db0), - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, DocId}, - {<<"value">>, DocId} - ] ++ Rev}), + Doc = couch_doc:from_json_obj({ + [ + {<<"_id">>, DocId}, + {<<"value">>, DocId} + ] ++ Rev + }), {ok, NewRev} = couch_db:update_doc(Db, Doc, []), couch_doc:rev_to_str(NewRev). - add_revisions(Db, DocId, Rev, N) -> - lists:foldl(fun(_, OldRev) -> - add_revision(Db, DocId, OldRev) - end, Rev, lists:seq(1, N)). + lists:foldl( + fun(_, OldRev) -> + add_revision(Db, DocId, OldRev) + end, + Rev, + lists:seq(1, N) + ). diff --git a/src/couch/test/eunit/couch_db_mpr_tests.erl b/src/couch/test/eunit/couch_db_mpr_tests.erl index bb97c66d7..3a9577a0d 100644 --- a/src/couch/test/eunit/couch_db_mpr_tests.erl +++ b/src/couch/test/eunit/couch_db_mpr_tests.erl @@ -12,7 +12,6 @@ -module(couch_db_mpr_tests). - -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). @@ -24,41 +23,35 @@ -define(CONTENT_JSON, {"Content-Type", "application/json"}). -define(JSON_BODY, "{\"foo\": \"bar\"}"). -define(CONTENT_MULTI_RELATED, - {"Content-Type", "multipart/related;boundary=\"bound\""}). - + {"Content-Type", "multipart/related;boundary=\"bound\""} +). setup() -> Hashed = couch_passwords:hash_admin_password(?PASS), - ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist=false), + ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist = false), TmpDb = ?tempdb(), Addr = config:get("httpd", "bind_address", "127.0.0.1"), Port = mochiweb_socket_server:get(couch_httpd, port), Url = lists:concat(["http://", Addr, ":", Port, "/", ?b2l(TmpDb)]), Url. - teardown(Url) -> catch delete_db(Url), - ok = config:delete("admins", ?USER, _Persist=false). - + ok = config:delete("admins", ?USER, _Persist = false). create_db(Url) -> {ok, Status, _, _} = test_request:put(Url, [?CONTENT_JSON, ?AUTH], "{}"), ?assert(Status =:= 201 orelse Status =:= 202). - delete_db(Url) -> {ok, 200, _, _} = test_request:delete(Url, [?AUTH]). - create_doc(Url, Id, Body, Type) -> test_request:put(Url ++ "/" ++ Id, [Type, ?AUTH], Body). - delete_doc(Url, Id, Rev) -> test_request:delete(Url ++ "/" ++ Id ++ "?rev=" ++ ?b2l(Rev)). - couch_db_mpr_test_() -> { "multi-part attachment tests", @@ -77,30 +70,29 @@ couch_db_mpr_test_() -> } }. - recreate_with_mpr(Url) -> - {timeout, ?TIMEOUT, ?_test(begin - DocId1 = "foo", - DocId2 = "bar", - - create_db(Url), - create_and_delete_doc(Url, DocId1), - Rev1 = create_with_mpr(Url, DocId1), - delete_db(Url), - - create_db(Url), - create_and_delete_doc(Url, DocId1), - % We create a second unrelated doc to change the - % position on disk where the attachment is written - % so that we can assert that the position on disk - % is not included when calculating a revision. - create_and_delete_doc(Url, DocId2), - Rev2 = create_with_mpr(Url, DocId1), - delete_db(Url), - - ?assertEqual(Rev1, Rev2) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + DocId1 = "foo", + DocId2 = "bar", + + create_db(Url), + create_and_delete_doc(Url, DocId1), + Rev1 = create_with_mpr(Url, DocId1), + delete_db(Url), + + create_db(Url), + create_and_delete_doc(Url, DocId1), + % We create a second unrelated doc to change the + % position on disk where the attachment is written + % so that we can assert that the position on disk + % is not included when calculating a revision. + create_and_delete_doc(Url, DocId2), + Rev2 = create_with_mpr(Url, DocId1), + delete_db(Url), + + ?assertEqual(Rev1, Rev2) + end)}. create_and_delete_doc(Url, DocId) -> {ok, _, _, Resp} = create_doc(Url, DocId, ?JSON_BODY, ?CONTENT_JSON), @@ -109,7 +101,6 @@ create_and_delete_doc(Url, DocId) -> ?assert(is_binary(Rev)), {ok, _, _, _} = delete_doc(Url, DocId, Rev). - create_with_mpr(Url, DocId) -> {ok, _, _, Resp} = create_doc(Url, DocId, mpr(), ?CONTENT_MULTI_RELATED), {Props} = ?JSON_DECODE(Resp), @@ -117,19 +108,18 @@ create_with_mpr(Url, DocId) -> ?assert(is_binary(Rev)), Rev. - mpr() -> lists:concat([ "--bound\r\n", "Content-Type: application/json\r\n\r\n", "{", - "\"body\":\"stuff\"," - "\"_attachments\":", - "{\"foo.txt\":{", - "\"follows\":true,", - "\"content_type\":\"text/plain\"," - "\"length\":21", - "}}" + "\"body\":\"stuff\"," + "\"_attachments\":", + "{\"foo.txt\":{", + "\"follows\":true,", + "\"content_type\":\"text/plain\"," + "\"length\":21", + "}}" "}", "\r\n--bound\r\n\r\n", "this is 21 chars long", diff --git a/src/couch/test/eunit/couch_db_plugin_tests.erl b/src/couch/test/eunit/couch_db_plugin_tests.erl index 93551adbc..bcfbffb05 100644 --- a/src/couch/test/eunit/couch_db_plugin_tests.erl +++ b/src/couch/test/eunit/couch_db_plugin_tests.erl @@ -21,7 +21,8 @@ on_delete/2 ]). --export([ %% couch_epi_plugin behaviour +%% couch_epi_plugin behaviour +-export([ app/0, providers/0, services/0, @@ -58,12 +59,15 @@ validate_dbname({false, _Db}, _) -> {decided, false}; validate_dbname({fail, _Db}, _) -> throw(validate_dbname); validate_dbname({pass, _Db}, _) -> no_decision. -before_doc_update({fail, _Doc}, _Db, interactive_edit) -> throw(before_doc_update); -before_doc_update({true, Doc}, Db, interactive_edit) -> [{true, [before_doc_update|Doc]}, Db, interactive_edit]; -before_doc_update({false, Doc}, Db, interactive_edit) -> [{false, Doc}, Db, interactive_edit]. +before_doc_update({fail, _Doc}, _Db, interactive_edit) -> + throw(before_doc_update); +before_doc_update({true, Doc}, Db, interactive_edit) -> + [{true, [before_doc_update | Doc]}, Db, interactive_edit]; +before_doc_update({false, Doc}, Db, interactive_edit) -> + [{false, Doc}, Db, interactive_edit]. after_doc_read({fail, _Doc}, _Db) -> throw(after_doc_read); -after_doc_read({true, Doc}, Db) -> [{true, [after_doc_read|Doc]}, Db]; +after_doc_read({true, Doc}, Db) -> [{true, [after_doc_read | Doc]}, Db]; after_doc_read({false, Doc}, Db) -> [{false, Doc}, Db]. validate_docid({true, _Id}) -> true; @@ -82,7 +86,9 @@ callback_test_() -> { "callback tests", { - setup, fun setup/0, fun teardown/1, + setup, + fun setup/0, + fun teardown/1, [ {"validate_dbname_match", fun validate_dbname_match/0}, {"validate_dbname_no_match", fun validate_dbname_no_match/0}, @@ -112,59 +118,77 @@ callback_test_() -> } }. - validate_dbname_match() -> - ?assert(couch_db_plugin:validate_dbname( - {true, [db]}, db, fun(_, _) -> pass end)). + ?assert( + couch_db_plugin:validate_dbname( + {true, [db]}, db, fun(_, _) -> pass end + ) + ). validate_dbname_no_match() -> - ?assertNot(couch_db_plugin:validate_dbname( - {false, [db]}, db, fun(_, _) -> pass end)). + ?assertNot( + couch_db_plugin:validate_dbname( + {false, [db]}, db, fun(_, _) -> pass end + ) + ). validate_dbname_throw() -> ?assertThrow( validate_dbname, couch_db_plugin:validate_dbname( - {fail, [db]}, db, fun(_, _) -> pass end)). + {fail, [db]}, db, fun(_, _) -> pass end + ) + ). validate_dbname_pass() -> - ?assertEqual(pass, couch_db_plugin:validate_dbname( - {pass, [db]}, db, fun(_, _) -> pass end)). + ?assertEqual( + pass, + couch_db_plugin:validate_dbname( + {pass, [db]}, db, fun(_, _) -> pass end + ) + ). before_doc_update_match() -> ?assertMatch( {true, [before_doc_update, doc]}, couch_db_plugin:before_doc_update( - fake_db(), {true, [doc]}, interactive_edit)). + fake_db(), {true, [doc]}, interactive_edit + ) + ). before_doc_update_no_match() -> ?assertMatch( {false, [doc]}, couch_db_plugin:before_doc_update( - fake_db(), {false, [doc]}, interactive_edit)). + fake_db(), {false, [doc]}, interactive_edit + ) + ). before_doc_update_throw() -> ?assertThrow( before_doc_update, couch_db_plugin:before_doc_update( - fake_db(), {fail, [doc]}, interactive_edit)). - + fake_db(), {fail, [doc]}, interactive_edit + ) + ). after_doc_read_match() -> ?assertMatch( {true, [after_doc_read, doc]}, - couch_db_plugin:after_doc_read(fake_db(), {true, [doc]})). + couch_db_plugin:after_doc_read(fake_db(), {true, [doc]}) + ). after_doc_read_no_match() -> ?assertMatch( {false, [doc]}, - couch_db_plugin:after_doc_read(fake_db(), {false, [doc]})). + couch_db_plugin:after_doc_read(fake_db(), {false, [doc]}) + ). after_doc_read_throw() -> ?assertThrow( after_doc_read, - couch_db_plugin:after_doc_read(fake_db(), {fail, [doc]})). - + couch_db_plugin:after_doc_read(fake_db(), {fail, [doc]}) + ). validate_docid_match() -> ?assert(couch_db_plugin:validate_docid({true, [doc]})). @@ -175,8 +199,8 @@ validate_docid_no_match() -> validate_docid_throw() -> ?assertThrow( validate_docid, - couch_db_plugin:validate_docid({fail, [doc]})). - + couch_db_plugin:validate_docid({fail, [doc]}) + ). check_is_admin_match() -> ?assert(couch_db_plugin:check_is_admin({true, [db]})). @@ -187,19 +211,23 @@ check_is_admin_no_match() -> check_is_admin_throw() -> ?assertThrow( check_is_admin, - couch_db_plugin:check_is_admin({fail, [db]})). + couch_db_plugin:check_is_admin({fail, [db]}) + ). on_delete_match() -> ?assertMatch( - [true], - couch_db_plugin:on_delete(true, [])). + [true], + couch_db_plugin:on_delete(true, []) + ). on_delete_no_match() -> ?assertMatch( - [false], - couch_db_plugin:on_delete(false, [])). + [false], + couch_db_plugin:on_delete(false, []) + ). on_delete_throw() -> ?assertThrow( on_delete, - couch_db_plugin:on_delete(fail, [])). + couch_db_plugin:on_delete(fail, []) + ). diff --git a/src/couch/test/eunit/couch_db_props_upgrade_tests.erl b/src/couch/test/eunit/couch_db_props_upgrade_tests.erl index 40ad283cf..5ca658129 100644 --- a/src/couch/test/eunit/couch_db_props_upgrade_tests.erl +++ b/src/couch/test/eunit/couch_db_props_upgrade_tests.erl @@ -16,7 +16,6 @@ -include_lib("couch/include/couch_db.hrl"). -include_lib("couch_mrview/include/couch_mrview.hrl"). - setup() -> DbName = <<"test">>, DbFileName = "test.couch", @@ -30,12 +29,10 @@ setup() -> DbName. - teardown(DbName) when is_binary(DbName) -> couch_server:delete(DbName, [?ADMIN_CTX]), ok. - old_db_info_test_() -> { "Old database versions work", @@ -56,14 +53,12 @@ old_db_info_test_() -> } }. - can_get_props(DbName) -> ?_test(begin {ok, Db} = couch_db:open_int(DbName, []), Props = couch_db_engine:get_props(Db), ?assert(is_list(Props)) - end). - + end). can_get_db_info(DbName) -> ?_test(begin @@ -71,8 +66,7 @@ can_get_db_info(DbName) -> {ok, Info} = couch_db:get_db_info(Db), Props = couch_util:get_value(props, Info), ?assertEqual({[]}, Props) - end). - + end). can_compact_db(DbName) -> ?_test(begin diff --git a/src/couch/test/eunit/couch_db_split_tests.erl b/src/couch/test/eunit/couch_db_split_tests.erl index b52184a8c..f0baa920b 100644 --- a/src/couch/test/eunit/couch_db_split_tests.erl +++ b/src/couch/test/eunit/couch_db_split_tests.erl @@ -16,8 +16,8 @@ -include_lib("couch/include/couch_db.hrl"). -define(RINGTOP, 2 bsl 31). --define(TIMEOUT, 60). % seconds - +% seconds +-define(TIMEOUT, 60). setup() -> DbName = ?tempdb(), @@ -25,14 +25,12 @@ setup() -> ok = couch_db:close(Db), DbName. - teardown(DbName) -> {ok, Db} = couch_db:open_int(DbName, []), FilePath = couch_db:get_filepath(Db), ok = couch_db:close(Db), ok = file:delete(FilePath). - split_test_() -> Cases = [ {"Should split an empty shard", 0, 2}, @@ -42,16 +40,19 @@ split_test_() -> ], { setup, - fun test_util:start_couch/0, fun test_util:stop/1, + fun test_util:start_couch/0, + fun test_util:stop/1, [ { foreachx, - fun(_) -> setup() end, fun(_, St) -> teardown(St) end, + fun(_) -> setup() end, + fun(_, St) -> teardown(St) end, [{Case, fun should_split_shard/2} || Case <- Cases] }, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_fail_on_missing_source/1, fun should_fail_on_existing_target/1, @@ -63,37 +64,43 @@ split_test_() -> ] }. - should_split_shard({Desc, TotalDocs, Q}, DbName) -> {ok, ExpectSeq} = create_docs(DbName, TotalDocs), Ranges = make_ranges(Q), TMap = make_targets(Ranges), DocsPerRange = TotalDocs div Q, PickFun = make_pickfun(DocsPerRange), - {Desc, timeout, ?TIMEOUT, ?_test(begin - {ok, UpdateSeq} = couch_db_split:split(DbName, TMap, PickFun), - ?assertEqual(ExpectSeq, UpdateSeq), - maps:map(fun(Range, Name) -> - {ok, Db} = couch_db:open_int(Name, []), - FilePath = couch_db:get_filepath(Db), - %% target actually exists - ?assertMatch({ok, _}, file:read_file_info(FilePath)), - %% target's update seq is the same as source's update seq - USeq = couch_db:get_update_seq(Db), - ?assertEqual(ExpectSeq, USeq), - %% target shard has all the expected in its range docs - {ok, DocsInShard} = couch_db:fold_docs(Db, fun(FDI, Acc) -> - DocId = FDI#full_doc_info.id, - ExpectedRange = PickFun(DocId, Ranges, undefined), - ?assertEqual(ExpectedRange, Range), - {ok, Acc + 1} - end, 0), - ?assertEqual(DocsPerRange, DocsInShard), - ok = couch_db:close(Db), - ok = file:delete(FilePath) - end, TMap) - end)}. - + {Desc, timeout, ?TIMEOUT, + ?_test(begin + {ok, UpdateSeq} = couch_db_split:split(DbName, TMap, PickFun), + ?assertEqual(ExpectSeq, UpdateSeq), + maps:map( + fun(Range, Name) -> + {ok, Db} = couch_db:open_int(Name, []), + FilePath = couch_db:get_filepath(Db), + %% target actually exists + ?assertMatch({ok, _}, file:read_file_info(FilePath)), + %% target's update seq is the same as source's update seq + USeq = couch_db:get_update_seq(Db), + ?assertEqual(ExpectSeq, USeq), + %% target shard has all the expected in its range docs + {ok, DocsInShard} = couch_db:fold_docs( + Db, + fun(FDI, Acc) -> + DocId = FDI#full_doc_info.id, + ExpectedRange = PickFun(DocId, Ranges, undefined), + ?assertEqual(ExpectedRange, Range), + {ok, Acc + 1} + end, + 0 + ), + ?assertEqual(DocsPerRange, DocsInShard), + ok = couch_db:close(Db), + ok = file:delete(FilePath) + end, + TMap + ) + end)}. should_fail_on_missing_source(_DbName) -> DbName = ?tempdb(), @@ -102,59 +109,67 @@ should_fail_on_missing_source(_DbName) -> Response = couch_db_split:split(DbName, TMap, fun fake_pickfun/3), ?_assertEqual({error, missing_source}, Response). - should_fail_on_existing_target(DbName) -> Ranges = make_ranges(2), - TMap = maps:map(fun(_, TName) -> - % We create the target but make sure to remove it from the cache so we - % hit the eexist error instaed of already_opened - {ok, Db} = couch_db:create(TName, [?ADMIN_CTX]), - Pid = couch_db:get_pid(Db), - ok = couch_db:close(Db), - exit(Pid, kill), - test_util:wait(fun() -> - case ets:lookup(couch_server:couch_dbs(DbName), TName) of - [] -> ok; - [_ | _] -> wait - end - end), - TName - end, make_targets(Ranges)), + TMap = maps:map( + fun(_, TName) -> + % We create the target but make sure to remove it from the cache so we + % hit the eexist error instaed of already_opened + {ok, Db} = couch_db:create(TName, [?ADMIN_CTX]), + Pid = couch_db:get_pid(Db), + ok = couch_db:close(Db), + exit(Pid, kill), + test_util:wait(fun() -> + case ets:lookup(couch_server:couch_dbs(DbName), TName) of + [] -> ok; + [_ | _] -> wait + end + end), + TName + end, + make_targets(Ranges) + ), Response = couch_db_split:split(DbName, TMap, fun fake_pickfun/3), ?_assertMatch({error, {target_create_error, _, eexist}}, Response). - should_fail_on_invalid_target_name(DbName) -> Ranges = make_ranges(2), - TMap = maps:map(fun([B, _], _) -> - iolist_to_binary(["_$", couch_util:to_hex(<<B:32/integer>>)]) - end, make_targets(Ranges)), - Expect = {error, {target_create_error, <<"_$00000000">>, - {illegal_database_name, <<"_$00000000">>}}}, + TMap = maps:map( + fun([B, _], _) -> + iolist_to_binary(["_$", couch_util:to_hex(<<B:32/integer>>)]) + end, + make_targets(Ranges) + ), + Expect = + {error, {target_create_error, <<"_$00000000">>, {illegal_database_name, <<"_$00000000">>}}}, Response = couch_db_split:split(DbName, TMap, fun fake_pickfun/3), ?_assertMatch(Expect, Response). - should_crash_on_invalid_tmap(DbName) -> Ranges = make_ranges(1), TMap = make_targets(Ranges), - ?_assertError(function_clause, - couch_db_split:split(DbName, TMap, fun fake_pickfun/3)). - + ?_assertError( + function_clause, + couch_db_split:split(DbName, TMap, fun fake_pickfun/3) + ). should_fail_on_opened_target(DbName) -> Ranges = make_ranges(2), - TMap = maps:map(fun(_, TName) -> - % We create and keep the target open but delete - % its file on disk so we don't fail with eexist - {ok, Db} = couch_db:create(TName, [?ADMIN_CTX]), - FilePath = couch_db:get_filepath(Db), - ok = file:delete(FilePath), - TName - end, make_targets(Ranges)), - ?_assertMatch({error, {target_create_error, _, already_opened}}, - couch_db_split:split(DbName, TMap, fun fake_pickfun/3)). - + TMap = maps:map( + fun(_, TName) -> + % We create and keep the target open but delete + % its file on disk so we don't fail with eexist + {ok, Db} = couch_db:create(TName, [?ADMIN_CTX]), + FilePath = couch_db:get_filepath(Db), + ok = file:delete(FilePath), + TName + end, + make_targets(Ranges) + ), + ?_assertMatch( + {error, {target_create_error, _, already_opened}}, + couch_db_split:split(DbName, TMap, fun fake_pickfun/3) + ). copy_local_docs_test_() -> Cases = [ @@ -165,46 +180,55 @@ copy_local_docs_test_() -> ], { setup, - fun test_util:start_couch/0, fun test_util:stop/1, + fun test_util:start_couch/0, + fun test_util:stop/1, [ { foreachx, - fun(_) -> setup() end, fun(_, St) -> teardown(St) end, + fun(_) -> setup() end, + fun(_, St) -> teardown(St) end, [{Case, fun should_copy_local_docs/2} || Case <- Cases] }, {"Should return error on missing source", - fun should_fail_copy_local_on_missing_source/0} + fun should_fail_copy_local_on_missing_source/0} ] }. - should_copy_local_docs({Desc, TotalDocs, Q}, DbName) -> {ok, ExpectSeq} = create_docs(DbName, TotalDocs), Ranges = make_ranges(Q), TMap = make_targets(Ranges), DocsPerRange = TotalDocs div Q, PickFun = make_pickfun(DocsPerRange), - {Desc, timeout, ?TIMEOUT, ?_test(begin - {ok, UpdateSeq} = couch_db_split:split(DbName, TMap, PickFun), - ?assertEqual(ExpectSeq, UpdateSeq), - Response = couch_db_split:copy_local_docs(DbName, TMap, PickFun), - ?assertEqual(ok, Response), - maps:map(fun(Range, Name) -> - {ok, Db} = couch_db:open_int(Name, []), - FilePath = couch_db:get_filepath(Db), - %% target shard has all the expected in its range docs - {ok, DocsInShard} = couch_db:fold_local_docs(Db, fun(Doc, Acc) -> - DocId = Doc#doc.id, - ExpectedRange = PickFun(DocId, Ranges, undefined), - ?assertEqual(ExpectedRange, Range), - {ok, Acc + 1} - end, 0, []), - ?assertEqual(DocsPerRange, DocsInShard), - ok = couch_db:close(Db), - ok = file:delete(FilePath) - end, TMap) - end)}. - + {Desc, timeout, ?TIMEOUT, + ?_test(begin + {ok, UpdateSeq} = couch_db_split:split(DbName, TMap, PickFun), + ?assertEqual(ExpectSeq, UpdateSeq), + Response = couch_db_split:copy_local_docs(DbName, TMap, PickFun), + ?assertEqual(ok, Response), + maps:map( + fun(Range, Name) -> + {ok, Db} = couch_db:open_int(Name, []), + FilePath = couch_db:get_filepath(Db), + %% target shard has all the expected in its range docs + {ok, DocsInShard} = couch_db:fold_local_docs( + Db, + fun(Doc, Acc) -> + DocId = Doc#doc.id, + ExpectedRange = PickFun(DocId, Ranges, undefined), + ?assertEqual(ExpectedRange, Range), + {ok, Acc + 1} + end, + 0, + [] + ), + ?assertEqual(DocsPerRange, DocsInShard), + ok = couch_db:close(Db), + ok = file:delete(FilePath) + end, + TMap + ) + end)}. should_fail_copy_local_on_missing_source() -> DbName = ?tempdb(), @@ -214,23 +238,23 @@ should_fail_copy_local_on_missing_source() -> Response = couch_db_split:copy_local_docs(DbName, TMap, PickFun), ?assertEqual({error, missing_source}, Response). - cleanup_target_test_() -> { setup, - fun test_util:start_couch/0, fun test_util:stop/1, + fun test_util:start_couch/0, + fun test_util:stop/1, [ { setup, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, fun should_delete_existing_targets/1 }, {"Should return error on missing source", - fun should_fail_cleanup_target_on_missing_source/0} + fun should_fail_cleanup_target_on_missing_source/0} ] }. - should_delete_existing_targets(SourceName) -> {ok, ExpectSeq} = create_docs(SourceName, 100), Ranges = make_ranges(2), @@ -239,25 +263,26 @@ should_delete_existing_targets(SourceName) -> ?_test(begin {ok, UpdateSeq} = couch_db_split:split(SourceName, TMap, PickFun), ?assertEqual(ExpectSeq, UpdateSeq), - maps:map(fun(_Range, TargetName) -> - FilePath = couch_util:with_db(TargetName, fun(Db) -> - couch_db:get_filepath(Db) - end), - ?assertMatch({ok, _}, file:read_file_info(FilePath)), - Response = couch_db_split:cleanup_target(SourceName, TargetName), - ?assertEqual(ok, Response), - ?assertEqual({error, enoent}, file:read_file_info(FilePath)) - end, TMap) + maps:map( + fun(_Range, TargetName) -> + FilePath = couch_util:with_db(TargetName, fun(Db) -> + couch_db:get_filepath(Db) + end), + ?assertMatch({ok, _}, file:read_file_info(FilePath)), + Response = couch_db_split:cleanup_target(SourceName, TargetName), + ?assertEqual(ok, Response), + ?assertEqual({error, enoent}, file:read_file_info(FilePath)) + end, + TMap + ) end). - should_fail_cleanup_target_on_missing_source() -> SourceName = ?tempdb(), TargetName = ?tempdb(), Response = couch_db_split:cleanup_target(SourceName, TargetName), ?assertEqual({error, missing_source}, Response). - make_pickfun(DocsPerRange) -> fun(DocId, Ranges, _HashFun) -> Id = docid_to_integer(DocId), @@ -269,41 +294,47 @@ make_pickfun(DocsPerRange) -> end end. - fake_pickfun(_, Ranges, _) -> hd(Ranges). - make_targets([]) -> maps:new(); -make_targets(Ranges) -> - Targets = lists:map(fun(Range) -> - {Range, ?tempdb()} - end, Ranges), +make_targets(Ranges) -> + Targets = lists:map( + fun(Range) -> + {Range, ?tempdb()} + end, + Ranges + ), maps:from_list(Targets). - make_ranges(Q) when Q > 0 -> Incr = (2 bsl 31) div Q, - lists:map(fun - (End) when End >= ?RINGTOP - 1 -> - [End - Incr, ?RINGTOP - 1]; - (End) -> - [End - Incr, End - 1] - end, lists:seq(Incr, ?RINGTOP, Incr)); + lists:map( + fun + (End) when End >= ?RINGTOP - 1 -> + [End - Incr, ?RINGTOP - 1]; + (End) -> + [End - Incr, End - 1] + end, + lists:seq(Incr, ?RINGTOP, Incr) + ); make_ranges(_) -> []. - create_docs(DbName, 0) -> couch_util:with_db(DbName, fun(Db) -> UpdateSeq = couch_db:get_update_seq(Db), {ok, UpdateSeq} end); create_docs(DbName, DocNum) -> - Docs = lists:foldl(fun(I, Acc) -> - [create_doc(I), create_local_doc(I) | Acc] - end, [], lists:seq(DocNum, 1, -1)), + Docs = lists:foldl( + fun(I, Acc) -> + [create_doc(I), create_local_doc(I) | Acc] + end, + [], + lists:seq(DocNum, 1, -1) + ), couch_util:with_db(DbName, fun(Db) -> {ok, _Result} = couch_db:update_docs(Db, Docs), {ok, Db1} = couch_db:reopen(Db), @@ -311,20 +342,16 @@ create_docs(DbName, DocNum) -> {ok, UpdateSeq} end). - create_doc(I) -> create_prefix_id_doc(I, ""). - create_local_doc(I) -> create_prefix_id_doc(I, "_local/"). - create_prefix_id_doc(I, Prefix) -> Id = iolist_to_binary(io_lib:format(Prefix ++ "~3..0B", [I])), couch_doc:from_json_obj({[{<<"_id">>, Id}, {<<"value">>, I}]}). - docid_to_integer(<<"_local/", DocId/binary>>) -> docid_to_integer(DocId); docid_to_integer(DocId) -> diff --git a/src/couch/test/eunit/couch_db_tests.erl b/src/couch/test/eunit/couch_db_tests.erl index d52a15597..82137dc40 100644 --- a/src/couch/test/eunit/couch_db_tests.erl +++ b/src/couch/test/eunit/couch_db_tests.erl @@ -16,14 +16,13 @@ -define(TIMEOUT, 120). - - -create_delete_db_test_()-> +create_delete_db_test_() -> { "Database create/delete tests", { setup, - fun test_util:start_couch/0, fun test_util:stop_couch/1, + fun test_util:start_couch/0, + fun test_util:stop_couch/1, { foreach, fun() -> ?tempdb() end, @@ -35,12 +34,13 @@ create_delete_db_test_()-> } }. -create_delete_multiple_dbs_test_()-> +create_delete_multiple_dbs_test_() -> { "Multiple database create/delete tests", { setup, - fun test_util:start_couch/0, fun test_util:stop_couch/1, + fun test_util:start_couch/0, + fun test_util:stop_couch/1, { foreach, fun() -> [?tempdb() || _ <- lists:seq(1, 6)] end, @@ -57,7 +57,8 @@ create_delete_database_continuously_test_() -> "Continious database create/delete tests", { setup, - fun test_util:start_couch/0, fun test_util:stop_couch/1, + fun test_util:start_couch/0, + fun test_util:stop_couch/1, { foreachx, fun(_) -> ?tempdb() end, @@ -69,12 +70,13 @@ create_delete_database_continuously_test_() -> } }. -open_db_test_()-> +open_db_test_() -> { "Database open tests", { setup, - fun test_util:start_couch/0, fun test_util:stop_couch/1, + fun test_util:start_couch/0, + fun test_util:stop_couch/1, { foreach, fun() -> ?tempdb() end, @@ -87,7 +89,6 @@ open_db_test_()-> } }. - should_create_db(DbName) -> ?_test(begin {ok, Before} = couch_server:all_databases(), @@ -109,10 +110,12 @@ should_delete_db(DbName) -> should_create_multiple_dbs(DbNames) -> ?_test(begin - [gen_server:call(couch_server:couch_server(N), {set_max_dbs_open, 3}) || - N <- lists:seq(1, couch_server:num_servers())], + [ + gen_server:call(couch_server:couch_server(N), {set_max_dbs_open, 3}) + || N <- lists:seq(1, couch_server:num_servers()) + ], {ok, Before} = couch_server:all_databases(), - [?assertNot(lists:member(DbName, Before)) || DbName <- DbNames], + [?assertNot(lists:member(DbName, Before)) || DbName <- DbNames], [?assert(create_db(DbName)) || DbName <- DbNames], {ok, After} = couch_server:all_databases(), [?assert(lists:member(DbName, After)) || DbName <- DbNames] @@ -122,21 +125,27 @@ should_delete_multiple_dbs(DbNames) -> ?_test(begin [?assert(create_db(DbName)) || DbName <- DbNames], {ok, Before} = couch_server:all_databases(), - [?assert(lists:member(DbName, Before)) || DbName <- DbNames], + [?assert(lists:member(DbName, Before)) || DbName <- DbNames], [?assert(delete_db(DbName)) || DbName <- DbNames], {ok, After} = couch_server:all_databases(), [?assertNot(lists:member(DbName, After)) || DbName <- DbNames] end). should_create_delete_database_continuously(Times, DbName) -> - {lists:flatten(io_lib:format("~b times", [Times])), - {timeout, ?TIMEOUT, ?_test(begin - ?assert(create_db(DbName)), - lists:foreach(fun(_) -> - ?assert(delete_db(DbName)), - ?assert(create_db(DbName)) - end, lists:seq(1, Times)) - end)}}. + { + lists:flatten(io_lib:format("~b times", [Times])), + {timeout, ?TIMEOUT, + ?_test(begin + ?assert(create_db(DbName)), + lists:foreach( + fun(_) -> + ?assert(delete_db(DbName)), + ?assert(create_db(DbName)) + end, + lists:seq(1, Times) + ) + end)} + }. should_create_db_if_missing(DbName) -> ?_test(begin @@ -165,8 +174,10 @@ locking_should_work(DbName) -> ?assertEqual({error, {locked, <<"x">>}}, couch_db:create(DbName, [])), ?assertEqual(ok, couch_server:unlock(DbName)), {ok, Db} = couch_db:create(DbName, []), - ?assertEqual({error, already_opened}, - couch_server:lock(DbName, <<>>)), + ?assertEqual( + {error, already_opened}, + couch_server:lock(DbName, <<>>) + ), ok = couch_db:close(Db), catch exit(couch_db:get_pid(Db), kill), @@ -175,11 +186,13 @@ locking_should_work(DbName) -> [] -> ok; [_ | _] -> wait end - end), + end), ?assertEqual(ok, couch_server:lock(DbName, <<"y">>)), - ?assertEqual({error, {locked, <<"y">>}}, - couch_db:open(DbName, [])), + ?assertEqual( + {error, {locked, <<"y">>}}, + couch_db:open(DbName, []) + ), couch_server:unlock(DbName), {ok, Db1} = couch_db:open(DbName, [{create_if_missing, true}]), diff --git a/src/couch/test/eunit/couch_doc_json_tests.erl b/src/couch/test/eunit/couch_doc_json_tests.erl index a68932eca..a004ed8fd 100644 --- a/src/couch/test/eunit/couch_doc_json_tests.erl +++ b/src/couch/test/eunit/couch_doc_json_tests.erl @@ -15,7 +15,6 @@ -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). - setup() -> mock(couch_log), mock(config), @@ -38,27 +37,32 @@ mock(couch_log) -> ok; mock(config) -> meck:new(config, [passthrough]), - meck:expect(config, get_integer, - fun("couchdb", "max_document_size", 8000000) -> 1024 end), + meck:expect( + config, + get_integer, + fun("couchdb", "max_document_size", 8000000) -> 1024 end + ), meck:expect(config, get, fun(_, _) -> undefined end), meck:expect(config, get, fun(_, _, Default) -> Default end), ok. - json_doc_test_() -> { setup, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, fun(_) -> - [{"Document from JSON", [ - from_json_with_dbname_error_cases(), - from_json_with_db_name_success_cases(), - from_json_success_cases(), - from_json_error_cases() - ]}, - {"Document to JSON", [ - to_json_success_cases() - ]}] + [ + {"Document from JSON", [ + from_json_with_dbname_error_cases(), + from_json_with_db_name_success_cases(), + from_json_success_cases(), + from_json_error_cases() + ]}, + {"Document to JSON", [ + to_json_success_cases() + ]} + ] end }. @@ -95,35 +99,42 @@ from_json_success_cases() -> "Non underscore prefixed fields stored in body." }, { - {[{<<"_attachments">>, {[ - {<<"my_attachment.fu">>, {[ - {<<"stub">>, true}, - {<<"content_type">>, <<"application/awesome">>}, - {<<"length">>, 45} - ]}}, - {<<"noahs_private_key.gpg">>, {[ - {<<"data">>, <<"SSBoYXZlIGEgcGV0IGZpc2gh">>}, - {<<"content_type">>, <<"application/pgp-signature">>} - ]}} - ]}}]}, - #doc{atts = [ - couch_att:new([ - {name, <<"my_attachment.fu">>}, - {data, stub}, - {type, <<"application/awesome">>}, - {att_len, 45}, - {disk_len, 45}, - {revpos, undefined} - ]), - couch_att:new([ - {name, <<"noahs_private_key.gpg">>}, - {data, <<"I have a pet fish!">>}, - {type, <<"application/pgp-signature">>}, - {att_len, 18}, - {disk_len, 18}, - {revpos, 0} - ]) + {[ + {<<"_attachments">>, + {[ + {<<"my_attachment.fu">>, + {[ + {<<"stub">>, true}, + {<<"content_type">>, <<"application/awesome">>}, + {<<"length">>, 45} + ]}}, + {<<"noahs_private_key.gpg">>, + {[ + {<<"data">>, <<"SSBoYXZlIGEgcGV0IGZpc2gh">>}, + {<<"content_type">>, <<"application/pgp-signature">>} + ]}} + ]}} ]}, + #doc{ + atts = [ + couch_att:new([ + {name, <<"my_attachment.fu">>}, + {data, stub}, + {type, <<"application/awesome">>}, + {att_len, 45}, + {disk_len, 45}, + {revpos, undefined} + ]), + couch_att:new([ + {name, <<"noahs_private_key.gpg">>}, + {data, <<"I have a pet fish!">>}, + {type, <<"application/pgp-signature">>}, + {att_len, 18}, + {disk_len, 18}, + {revpos, 0} + ]) + ] + }, "Attachments are parsed correctly." }, { @@ -138,11 +149,13 @@ from_json_success_cases() -> }, { {[ - {<<"_revisions">>, - {[{<<"start">>, 4}, - {<<"ids">>, [<<"foo1">>, <<"phi3">>, <<"omega">>]}]}}, - {<<"_rev">>, <<"6-something">>} - ]}, + {<<"_revisions">>, + {[ + {<<"start">>, 4}, + {<<"ids">>, [<<"foo1">>, <<"phi3">>, <<"omega">>]} + ]}}, + {<<"_rev">>, <<"6-something">>} + ]}, #doc{revs = {4, [<<"foo1">>, <<"phi3">>, <<"omega">>]}}, "_revisions attribute are preferred to _rev." }, @@ -171,7 +184,8 @@ from_json_success_cases() -> fun({EJson, Expect, Msg}) -> {Msg, ?_assertMatch(Expect, couch_doc:from_json_obj_validate(EJson))} end, - Cases). + Cases + ). from_json_with_db_name_success_cases() -> Cases = [ @@ -210,7 +224,8 @@ from_json_with_db_name_success_cases() -> fun({EJson, DbName, Expect, Msg}) -> {Msg, ?_assertMatch(Expect, couch_doc:from_json_obj_validate(EJson, DbName))} end, - Cases). + Cases + ). from_json_error_cases() -> Cases = [ @@ -236,8 +251,7 @@ from_json_error_cases() -> }, { {[{<<"_id">>, <<"_random">>}]}, - {illegal_docid, - <<"Only reserved document ids may start with underscore.">>}, + {illegal_docid, <<"Only reserved document ids may start with underscore.">>}, "Disallow arbitrary underscore prefixed docids." }, { @@ -270,8 +284,13 @@ from_json_error_cases() -> "Revision ids must be strings." }, { - {[{<<"_revisions">>, {[{<<"start">>, 0}, - {<<"ids">>, [<<"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx">>]}]}}]}, + {[ + {<<"_revisions">>, + {[ + {<<"start">>, 0}, + {<<"ids">>, [<<"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx">>]} + ]}} + ]}, {doc_validation, "RevId isn't a valid hexadecimal"}, "Revision ids must be a valid hex." }, @@ -284,7 +303,7 @@ from_json_error_cases() -> fun() -> {[ {<<"_id">>, <<"large_doc">>}, - {<<"x">> , << <<"x">> || _ <- lists:seq(1,1025) >>} + {<<"x">>, <<<<"x">> || _ <- lists:seq(1, 1025)>>} ]} end, {request_entity_too_large, <<"large_doc">>}, @@ -292,39 +311,36 @@ from_json_error_cases() -> } ], - lists:map(fun - ({Fun, Expect, Msg}) when is_function(Fun, 0) -> - {Msg, - ?_assertThrow(Expect, couch_doc:from_json_obj_validate(Fun()))}; - ({EJson, Expect, Msg}) -> - {Msg, - ?_assertThrow(Expect, couch_doc:from_json_obj_validate(EJson))}; - ({EJson, Msg}) -> - {Msg, - ?_assertThrow(_, couch_doc:from_json_obj_validate(EJson))} - end, Cases). + lists:map( + fun + ({Fun, Expect, Msg}) when is_function(Fun, 0) -> + {Msg, ?_assertThrow(Expect, couch_doc:from_json_obj_validate(Fun()))}; + ({EJson, Expect, Msg}) -> + {Msg, ?_assertThrow(Expect, couch_doc:from_json_obj_validate(EJson))}; + ({EJson, Msg}) -> + {Msg, ?_assertThrow(_, couch_doc:from_json_obj_validate(EJson))} + end, + Cases + ). from_json_with_dbname_error_cases() -> Cases = [ { {[{<<"_id">>, <<"_random">>}]}, <<"_dbs">>, - {illegal_docid, - <<"Only reserved document ids may start with underscore.">>}, + {illegal_docid, <<"Only reserved document ids may start with underscore.">>}, "Disallow non-system-DB underscore prefixed docids in _dbs database." }, { {[{<<"_id">>, <<"_random">>}]}, <<"foobar">>, - {illegal_docid, - <<"Only reserved document ids may start with underscore.">>}, + {illegal_docid, <<"Only reserved document ids may start with underscore.">>}, "Disallow arbitrary underscore prefixed docids in regular database." }, { {[{<<"_id">>, <<"_users">>}]}, <<"foobar">>, - {illegal_docid, - <<"Only reserved document ids may start with underscore.">>}, + {illegal_docid, <<"Only reserved document ids may start with underscore.">>}, "Disallow system-DB docid _users in regular database." } ], @@ -334,7 +350,8 @@ from_json_with_dbname_error_cases() -> Error = (catch couch_doc:from_json_obj_validate(EJson, DbName)), {Msg, ?_assertMatch(Expect, Error)} end, - Cases). + Cases + ). to_json_success_cases() -> Cases = [ @@ -357,13 +374,14 @@ to_json_success_cases() -> [revs], #doc{revs = {5, [<<"first">>, <<"second">>]}}, {[ - {<<"_id">>, <<>>}, - {<<"_rev">>, <<"5-first">>}, - {<<"_revisions">>, {[ - {<<"start">>, 5}, - {<<"ids">>, [<<"first">>, <<"second">>]} - ]}} - ]}, + {<<"_id">>, <<>>}, + {<<"_rev">>, <<"5-first">>}, + {<<"_revisions">>, + {[ + {<<"start">>, 5}, + {<<"ids">>, [<<"first">>, <<"second">>]} + ]}} + ]}, "_revisions include with revs option" }, { @@ -377,16 +395,18 @@ to_json_success_cases() -> "Deleted docs no longer drop body members." }, { - #doc{meta = [ - {revs_info, 4, [{<<"fin">>, deleted}, {<<"zim">>, missing}]} - ]}, + #doc{ + meta = [ + {revs_info, 4, [{<<"fin">>, deleted}, {<<"zim">>, missing}]} + ] + }, {[ - {<<"_id">>, <<>>}, - {<<"_revs_info">>, [ - {[{<<"rev">>, <<"4-fin">>}, {<<"status">>, <<"deleted">>}]}, - {[{<<"rev">>, <<"3-zim">>}, {<<"status">>, <<"missing">>}]} - ]} - ]}, + {<<"_id">>, <<>>}, + {<<"_revs_info">>, [ + {[{<<"rev">>, <<"4-fin">>}, {<<"status">>, <<"deleted">>}]}, + {[{<<"rev">>, <<"3-zim">>}, {<<"status">>, <<"missing">>}]} + ]} + ]}, "_revs_info field is added correctly." }, { @@ -405,89 +425,102 @@ to_json_success_cases() -> { #doc{meta = [{deleted_conflicts, [{10923, <<"big_cowboy_hat">>}]}]}, {[ - {<<"_id">>, <<>>}, - {<<"_deleted_conflicts">>, [<<"10923-big_cowboy_hat">>]} - ]}, + {<<"_id">>, <<>>}, + {<<"_deleted_conflicts">>, [<<"10923-big_cowboy_hat">>]} + ]}, "_deleted_conflicsts is added as an array of strings." }, { - #doc{atts = [ - couch_att:new([ - {name, <<"big.xml">>}, - {type, <<"xml/sucks">>}, - {data, fun() -> ok end}, - {revpos, 1}, - {att_len, 400}, - {disk_len, 400} - ]), - couch_att:new([ - {name, <<"fast.json">>}, - {type, <<"json/ftw">>}, - {data, <<"{\"so\": \"there!\"}">>}, - {revpos, 1}, - {att_len, 16}, - {disk_len, 16} - ]) - ]}, + #doc{ + atts = [ + couch_att:new([ + {name, <<"big.xml">>}, + {type, <<"xml/sucks">>}, + {data, fun() -> ok end}, + {revpos, 1}, + {att_len, 400}, + {disk_len, 400} + ]), + couch_att:new([ + {name, <<"fast.json">>}, + {type, <<"json/ftw">>}, + {data, <<"{\"so\": \"there!\"}">>}, + {revpos, 1}, + {att_len, 16}, + {disk_len, 16} + ]) + ] + }, {[ - {<<"_id">>, <<>>}, - {<<"_attachments">>, {[ - {<<"big.xml">>, {[ - {<<"content_type">>, <<"xml/sucks">>}, - {<<"revpos">>, 1}, - {<<"length">>, 400}, - {<<"stub">>, true} - ]}}, - {<<"fast.json">>, {[ - {<<"content_type">>, <<"json/ftw">>}, - {<<"revpos">>, 1}, - {<<"length">>, 16}, - {<<"stub">>, true} - ]}} - ]}} + {<<"_id">>, <<>>}, + {<<"_attachments">>, + {[ + {<<"big.xml">>, + {[ + {<<"content_type">>, <<"xml/sucks">>}, + {<<"revpos">>, 1}, + {<<"length">>, 400}, + {<<"stub">>, true} + ]}}, + {<<"fast.json">>, + {[ + {<<"content_type">>, <<"json/ftw">>}, + {<<"revpos">>, 1}, + {<<"length">>, 16}, + {<<"stub">>, true} + ]}} + ]}} ]}, "Attachments attached as stubs only include a length." }, { [attachments], - #doc{atts = [ - couch_att:new([ - {name, <<"stuff.txt">>}, - {type, <<"text/plain">>}, - {data, fun() -> <<"diet pepsi">> end}, - {revpos, 1}, - {att_len, 10}, - {disk_len, 10} - ]), - couch_att:new([ - {name, <<"food.now">>}, - {type, <<"application/food">>}, - {revpos, 1}, - {data, <<"sammich">>} - ]) - ]}, + #doc{ + atts = [ + couch_att:new([ + {name, <<"stuff.txt">>}, + {type, <<"text/plain">>}, + {data, fun() -> <<"diet pepsi">> end}, + {revpos, 1}, + {att_len, 10}, + {disk_len, 10} + ]), + couch_att:new([ + {name, <<"food.now">>}, + {type, <<"application/food">>}, + {revpos, 1}, + {data, <<"sammich">>} + ]) + ] + }, {[ {<<"_id">>, <<>>}, - {<<"_attachments">>, {[ - {<<"stuff.txt">>, {[ - {<<"content_type">>, <<"text/plain">>}, - {<<"revpos">>, 1}, - {<<"data">>, <<"ZGlldCBwZXBzaQ==">>} - ]}}, - {<<"food.now">>, {[ - {<<"content_type">>, <<"application/food">>}, - {<<"revpos">>, 1}, - {<<"data">>, <<"c2FtbWljaA==">>} - ]}} - ]}} + {<<"_attachments">>, + {[ + {<<"stuff.txt">>, + {[ + {<<"content_type">>, <<"text/plain">>}, + {<<"revpos">>, 1}, + {<<"data">>, <<"ZGlldCBwZXBzaQ==">>} + ]}}, + {<<"food.now">>, + {[ + {<<"content_type">>, <<"application/food">>}, + {<<"revpos">>, 1}, + {<<"data">>, <<"c2FtbWljaA==">>} + ]}} + ]}} ]}, "Attachments included inline with attachments option." } ], - lists:map(fun - ({Doc, EJson, Msg}) -> - {Msg, ?_assertMatch(EJson, couch_doc:to_json_obj(Doc, []))}; - ({Options, Doc, EJson, Msg}) -> - {Msg, ?_assertMatch(EJson, couch_doc:to_json_obj(Doc, Options))} - end, Cases). + lists:map( + fun + ({Doc, EJson, Msg}) -> + {Msg, ?_assertMatch(EJson, couch_doc:to_json_obj(Doc, []))}; + ({Options, Doc, EJson, Msg}) -> + {Msg, ?_assertMatch(EJson, couch_doc:to_json_obj(Doc, Options))} + end, + Cases + ). diff --git a/src/couch/test/eunit/couch_doc_tests.erl b/src/couch/test/eunit/couch_doc_tests.erl index fc63d3f30..5a6e11ab2 100644 --- a/src/couch/test/eunit/couch_doc_tests.erl +++ b/src/couch/test/eunit/couch_doc_tests.erl @@ -15,9 +15,9 @@ -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). - -define(REQUEST_FIXTURE, - filename:join([?FIXTURESDIR, "multipart.http"])). + filename:join([?FIXTURESDIR, "multipart.http"]) +). parse_rev_test() -> ?assertEqual({1, <<"123">>}, couch_doc:parse_rev("1-123")), @@ -40,24 +40,30 @@ doc_to_multi_part_stream_test() -> JsonBytes = <<"{\n \"_id\": \"our document goes here\"\n}\n\n">>, AttData = <<"Hello my important document">>, AttLength = size(AttData), - Atts = [couch_att:new([ - {name, <<"test">>}, {data, AttData}, {type, <<"text/plain">>}, - {att_len, AttLength}, {disk_len, AttLength}])], + Atts = [ + couch_att:new([ + {name, <<"test">>}, + {data, AttData}, + {type, <<"text/plain">>}, + {att_len, AttLength}, + {disk_len, AttLength} + ]) + ], couch_doc:doc_to_multi_part_stream(Boundary, JsonBytes, Atts, fun send/1, true), AttLengthStr = integer_to_binary(AttLength), BoundaryLen = size(Boundary), [ - <<"--", Boundary/binary>>, - <<"Content-Type: application/json">>, - <<>>, - JsonBytes, - <<"--", Boundary/binary>>, - <<"Content-Disposition: attachment; filename=\"test\"">>, - <<"Content-Type: text/plain">>, - <<"Content-Length: ", AttLengthStr/binary>>, - <<>>, - AttData, - <<"--", Boundary:BoundaryLen/binary, "--">> + <<"--", Boundary/binary>>, + <<"Content-Type: application/json">>, + <<>>, + JsonBytes, + <<"--", Boundary/binary>>, + <<"Content-Disposition: attachment; filename=\"test\"">>, + <<"Content-Type: text/plain">>, + <<"Content-Length: ", AttLengthStr/binary>>, + <<>>, + AttData, + <<"--", Boundary:BoundaryLen/binary, "--">> ] = collected(), ok. @@ -67,10 +73,17 @@ len_doc_to_multi_part_stream_test() -> ContentType = <<"multipart/related; boundary=\"", Boundary/binary, "\"">>, AttData = <<"Hello my important document">>, AttLength = size(AttData), - Atts = [couch_att:new([ - {name, <<"test">>}, {data, AttData}, {type, <<"text/plain">>}, - {att_len, AttLength}, {disk_len, AttLength}])], - {ContentType, 258} = %% 258 is expected size of the document + Atts = [ + couch_att:new([ + {name, <<"test">>}, + {data, AttData}, + {type, <<"text/plain">>}, + {att_len, AttLength}, + {disk_len, AttLength} + ]) + ], + %% 258 is expected size of the document + {ContentType, 258} = couch_doc:len_doc_to_multi_part_stream(Boundary, JsonBytes, Atts, true), ok. @@ -93,29 +106,46 @@ validate_docid_test_() -> ?_assertEqual(ok, couch_doc:validate_docid(<<"_users">>, <<"_dbs">>)), ?_assertEqual(ok, couch_doc:validate_docid(<<"_replicator">>, <<"_dbs">>)), ?_assertEqual(ok, couch_doc:validate_docid(<<"_global_changes">>, <<"_dbs">>)), - ?_assertThrow({illegal_docid, _}, - couch_doc:validate_docid(<<>>)), - ?_assertThrow({illegal_docid, _}, - couch_doc:validate_docid(<<16#80>>)), - ?_assertThrow({illegal_docid, _}, - couch_doc:validate_docid(<<"_idx">>)), - ?_assertThrow({illegal_docid, _}, - couch_doc:validate_docid(<<"_">>)), - ?_assertThrow({illegal_docid, _}, - couch_doc:validate_docid(<<"_design/">>)), - ?_assertThrow({illegal_docid, _}, - couch_doc:validate_docid(<<"_local/">>)), - ?_assertThrow({illegal_docid, _}, - couch_doc:validate_docid(large_id(1025))), - ?_assertThrow({illegal_docid, _}, - couch_doc:validate_docid(<<"_users">>, <<"foo">>)), - ?_assertThrow({illegal_docid, _}, - couch_doc:validate_docid(<<"_weeee">>, <<"_dbs">>)) - ] - }. + ?_assertThrow( + {illegal_docid, _}, + couch_doc:validate_docid(<<>>) + ), + ?_assertThrow( + {illegal_docid, _}, + couch_doc:validate_docid(<<16#80>>) + ), + ?_assertThrow( + {illegal_docid, _}, + couch_doc:validate_docid(<<"_idx">>) + ), + ?_assertThrow( + {illegal_docid, _}, + couch_doc:validate_docid(<<"_">>) + ), + ?_assertThrow( + {illegal_docid, _}, + couch_doc:validate_docid(<<"_design/">>) + ), + ?_assertThrow( + {illegal_docid, _}, + couch_doc:validate_docid(<<"_local/">>) + ), + ?_assertThrow( + {illegal_docid, _}, + couch_doc:validate_docid(large_id(1025)) + ), + ?_assertThrow( + {illegal_docid, _}, + couch_doc:validate_docid(<<"_users">>, <<"foo">>) + ), + ?_assertThrow( + {illegal_docid, _}, + couch_doc:validate_docid(<<"_weeee">>, <<"_dbs">>) + ) + ]}. large_id(N) -> - << <<"x">> || _ <- lists:seq(1, N) >>. + <<<<"x">> || _ <- lists:seq(1, N)>>. request(start) -> {ok, Doc} = file:read_file(?REQUEST_FIXTURE), @@ -128,7 +158,7 @@ send(Data) -> send(Data, undefined) -> send(Data, []); send(Data, Acc) -> - put(data, [Acc|Data]). + put(data, [Acc | Data]). collected() -> B = binary:replace(iolist_to_binary(get(data)), <<"\r\n">>, <<0>>, [global]), @@ -136,11 +166,14 @@ collected() -> mock_config() -> ok = meck:new(config, [passthrough]), - meck:expect(config, get, - fun("couchdb", "max_document_id_length", "infinity") -> "1024"; - ("couchdb", "max_attachment_size", "infinity") -> "infinity"; - ("couchdb", "max_attachment_size", 1073741824) -> 1073741824; - ("mem3", "shards_db", "_dbs") -> "_dbs"; + meck:expect( + config, + get, + fun + ("couchdb", "max_document_id_length", "infinity") -> "1024"; + ("couchdb", "max_attachment_size", "infinity") -> "infinity"; + ("couchdb", "max_attachment_size", 1073741824) -> 1073741824; + ("mem3", "shards_db", "_dbs") -> "_dbs"; (Key, Val, Default) -> meck:passthrough([Key, Val, Default]) end ). diff --git a/src/couch/test/eunit/couch_ejson_compare_tests.erl b/src/couch/test/eunit/couch_ejson_compare_tests.erl index 1dfbad4ed..f74e40b23 100644 --- a/src/couch/test/eunit/couch_ejson_compare_tests.erl +++ b/src/couch/test/eunit/couch_ejson_compare_tests.erl @@ -12,8 +12,7 @@ -module(couch_ejson_compare_tests). - --define(MAX_UNICODE_STRING, <<255,255,255,255>>). +-define(MAX_UNICODE_STRING, <<255, 255, 255, 255>>). % See mango_idx_view.hrl -define(MAX_JSON_OBJ, {?MAX_UNICODE_STRING}). @@ -52,40 +51,39 @@ {[{<<"b">>, 2}, {<<"c">>, 2}]} ]). - % Propery tests -ifdef(WITH_PROPER). -include_lib("couch/include/couch_eunit_proper.hrl"). - property_test_() -> ?EUNIT_QUICKCHECK(60, 400). - % Properties % The main, nif-based comparison, sorts the test values correctly prop_nif_sorts_correctly() -> Positions = get_positions(?TEST_VALUES), - ?FORALL(A, oneof(?TEST_VALUES), + ?FORALL( + A, + oneof(?TEST_VALUES), ?FORALL(B, oneof(?TEST_VALUES), begin expected_less(A, B, Positions) =:= less_nif(A, B) end) ). - % The erlang fallback comparison sorts the test values correctly prop_erlang_sorts_correctly() -> Positions = get_positions(?TEST_VALUES), - ?FORALL(A, oneof(?TEST_VALUES), + ?FORALL( + A, + oneof(?TEST_VALUES), ?FORALL(B, oneof(?TEST_VALUES), begin expected_less(A, B, Positions) =:= less_erl(A, B) end) ). - % Zero width unicode chars are ignored prop_equivalent_unicode_values() -> ?FORALL({Prefix, Suffix}, {zero_width_list(), zero_width_list()}, begin @@ -93,36 +91,33 @@ prop_equivalent_unicode_values() -> less(<<"a">>, Binary) =:= 0 end). - % Every test value sorts less than the special ?MAX_JSON_OBJ prop_test_values_are_less_than_max_json() -> ?FORALL(V, oneof(?TEST_VALUES), begin less(V, ?MAX_JSON_OBJ) =:= -1 end). - % Any json value sorts less than the special ?MAX_JSON_OBJ prop_any_json_is_less_than_max_json() -> ?FORALL(V, json(), begin less(V, ?MAX_JSON_OBJ) =:= -1 end). - % In general, for any json, the nif collator matches the erlang collator prop_nif_matches_erlang() -> - ?FORALL(A, json(), + ?FORALL( + A, + json(), ?FORALL(B, json(), begin less_nif(A, B) =:= less_erl(A, B) end) ). - % Generators json() -> ?SIZED(Size, json(Size)). - json(0) -> oneof([ null, @@ -133,7 +128,6 @@ json(0) -> [], {[]} ]); - json(Size) -> frequency([ {1, null}, @@ -147,40 +141,30 @@ json(Size) -> {5, ?LAZY(json_object(Size))} ]). - json_number() -> oneof([largeint(), int(), real()]). - json_string() -> utf8(). - json_array(0) -> []; - json_array(Size) -> vector(Size div 2, json(Size div 2)). - json_object(0) -> {[]}; - json_object(Size) -> {vector(Size div 2, {json_string(), json(Size div 2)})}. - zero_width_list() -> ?SIZED(Size, vector(Size, zero_width_chars())). - zero_width_chars() -> oneof([16#200B, 16#200C, 16#200D]). - -endif. - % Regular EUnit tests get_icu_version_test() -> @@ -192,7 +176,6 @@ get_icu_version_test() -> ?assert(is_integer(V3) andalso V3 >= 0), ?assert(is_integer(V4) andalso V4 >= 0). - get_uca_version_test() -> Ver = couch_ejson_compare:get_uca_version(), ?assertMatch({_, _, _, _}, Ver), @@ -202,7 +185,6 @@ get_uca_version_test() -> ?assert(is_integer(V3) andalso V3 >= 0), ?assert(is_integer(V4) andalso V4 >= 0). - max_depth_error_list_test() -> % NIF can handle terms with depth <= 9 Nested9 = nest_list(<<"val">>, 9), @@ -215,7 +197,6 @@ max_depth_error_list_test() -> % Then it should transparently jump to erlang land ?assertEqual(0, less(Nested10, Nested10)). - max_depth_error_obj_test() -> % NIF can handle terms with depth <= 9 Nested9 = nest_obj(<<"k">>, <<"v">>, 9), @@ -228,13 +209,12 @@ max_depth_error_obj_test() -> % Then it should transparently jump to erlang land ?assertEqual(0, less(Nested10, Nested10)). - compare_strings_nif_test() -> ?assertEqual(-1, compare_strings(<<"a">>, <<"b">>)), ?assertEqual(0, compare_strings(<<"a">>, <<"a">>)), ?assertEqual(1, compare_strings(<<"b">>, <<"a">>)), - LargeBin1 = << <<"x">> || _ <- lists:seq(1, 1000000)>>, + LargeBin1 = <<<<"x">> || _ <- lists:seq(1, 1000000)>>, LargeBin2 = <<LargeBin1/binary, "x">>, ?assertEqual(-1, compare_strings(LargeBin1, LargeBin2)), ?assertEqual(1, compare_strings(LargeBin2, LargeBin1)), @@ -244,47 +224,41 @@ compare_strings_nif_test() -> ?assertError(badarg, compare_strings(<<"a">>, 42)), ?assertError(badarg, compare_strings(42, 42)). - % Helper functions less(A, B) -> cmp_norm(couch_ejson_compare:less(A, B)). - less_nif(A, B) -> cmp_norm(couch_ejson_compare:less_nif(A, B)). - less_erl(A, B) -> cmp_norm(couch_ejson_compare:less_erl(A, B)). - compare_strings(A, B) -> couch_ejson_compare:compare_strings_nif(A, B). - nest_list(Val, 0) -> Val; - nest_list(Val, Depth) when is_integer(Depth), Depth > 0 -> [nest_list(Val, Depth - 1)]. - nest_obj(K, V, 1) -> {[{K, V}]}; - nest_obj(K, V, Depth) when is_integer(Depth), Depth > 1 -> {[{K, nest_obj(K, V, Depth - 1)}]}. - % Build a map of #{Val => PositionIndex} for the test values so that when any % two are compared we can verify their position in the test list matches the % compared result get_positions(TestValues) -> - lists:foldl(fun(Val, Acc) -> - Acc#{Val => map_size(Acc)} - end, #{}, TestValues). - + lists:foldl( + fun(Val, Acc) -> + Acc#{Val => map_size(Acc)} + end, + #{}, + TestValues + ). % When two values are compared, check the test values positions index to ensure % the order in the test value list matches the comparison result diff --git a/src/couch/test/eunit/couch_ejson_size_tests.erl b/src/couch/test/eunit/couch_ejson_size_tests.erl index df9168ed1..27803d8b7 100644 --- a/src/couch/test/eunit/couch_ejson_size_tests.erl +++ b/src/couch/test/eunit/couch_ejson_size_tests.erl @@ -14,59 +14,86 @@ -include_lib("eunit/include/eunit.hrl"). --define(HWAIR, $\x{10348}). % 4 byte utf8 encoding --define(EURO, $\x{20ac}). % 3 byte utf8 encoding --define(CENT, $\x{a2}). % 2 byte utf8 encoding - +% 4 byte utf8 encoding +-define(HWAIR, $\x{10348}). +% 3 byte utf8 encoding +-define(EURO, $\x{20ac}). +% 2 byte utf8 encoding +-define(CENT, $\x{a2}). ejson_size_test_() -> - [?_assertEqual(R, couch_ejson_size:encoded_size(Input)) || {R, Input} <- [ - {1, 1}, {1, 1}, {2, -1}, {1, 9}, {2, 10}, {3, -10}, - {2, 11}, {2, 99}, {3, 100}, {3, 999}, {4, 1000}, {4, 9999}, - {5, 10000}, - - {3, 0.0}, {3, 0.1}, {3, 1.0}, {4, -1.0}, {3, 1.0e9}, - {4, 1.0e10}, {5, 1.0e-10}, {5, 1.0e-99}, {6, 1.0e-100}, {3, 1.0e-323}, - - {2, arr_nested(0)}, {22, arr_nested(10)}, {2002, arr_nested(1000)}, - {9, obj_nested(0)}, {69, obj_nested(10)}, {6009, obj_nested(1000)}, - - {4, null}, {4, true}, {5, false}, - - {3, str(1, $x)}, {4, str(1, ?CENT)}, {5, str(1, ?EURO)}, - {6, str(1, ?HWAIR)}, {3, str(1, $\x{1})}, {12, str(10, $x)}, - {22, str(10, ?CENT)}, {32, str(10, ?EURO)}, {42, str(10, ?HWAIR)}, - {12, str(10, $\x{1})} - ]]. - + [ + ?_assertEqual(R, couch_ejson_size:encoded_size(Input)) + || {R, Input} <- [ + {1, 1}, + {1, 1}, + {2, -1}, + {1, 9}, + {2, 10}, + {3, -10}, + {2, 11}, + {2, 99}, + {3, 100}, + {3, 999}, + {4, 1000}, + {4, 9999}, + {5, 10000}, + + {3, 0.0}, + {3, 0.1}, + {3, 1.0}, + {4, -1.0}, + {3, 1.0e9}, + {4, 1.0e10}, + {5, 1.0e-10}, + {5, 1.0e-99}, + {6, 1.0e-100}, + {3, 1.0e-323}, + + {2, arr_nested(0)}, + {22, arr_nested(10)}, + {2002, arr_nested(1000)}, + {9, obj_nested(0)}, + {69, obj_nested(10)}, + {6009, obj_nested(1000)}, + + {4, null}, + {4, true}, + {5, false}, + + {3, str(1, $x)}, + {4, str(1, ?CENT)}, + {5, str(1, ?EURO)}, + {6, str(1, ?HWAIR)}, + {3, str(1, $\x{1})}, + {12, str(10, $x)}, + {22, str(10, ?CENT)}, + {32, str(10, ?EURO)}, + {42, str(10, ?HWAIR)}, + {12, str(10, $\x{1})} + ] + ]. %% Helper functions arr_nested(MaxDepth) -> arr_nested(MaxDepth, 0). - obj_nested(MaxDepth) -> obj_nested(MaxDepth, 0). - obj(N, K, V) -> {[{K, V} || _ <- lists:seq(1, N)]}. - str(N, C) -> unicode:characters_to_binary([C || _ <- lists:seq(1, N)]). - arr_nested(MaxDepth, MaxDepth) -> []; - arr_nested(MaxDepth, Depth) -> [arr_nested(MaxDepth, Depth + 1)]. - obj_nested(MaxDepth, MaxDepth) -> obj(1, <<"k">>, <<"v">>); - obj_nested(MaxDepth, Depth) -> {[{<<"k">>, obj_nested(MaxDepth, Depth + 1)}]}. diff --git a/src/couch/test/eunit/couch_etag_tests.erl b/src/couch/test/eunit/couch_etag_tests.erl index 9d15e483f..72db6008a 100644 --- a/src/couch/test/eunit/couch_etag_tests.erl +++ b/src/couch/test/eunit/couch_etag_tests.erl @@ -18,13 +18,14 @@ local_with_empty_body_test() -> Etag = couch_httpd:doc_etag(<<"_local/local-and-empty">>, {[]}, {0, <<"1">>}), ?assertEqual(Etag, <<"\"5ZVXQYO7VLEOU0TL9VXDNP5PV\"">>). - local_with_body_test() -> - DocBody = {[{<<"hello">>,<<"world">>},{<<"relax">>,true}]}, + DocBody = {[{<<"hello">>, <<"world">>}, {<<"relax">>, true}]}, Etag = couch_httpd:doc_etag(<<"_local/local-with-body">>, DocBody, {0, <<"1">>}), ?assertEqual(Etag, <<"\"CEFXP6WH8OKYIWO1GLGBHKCCA\"">>). normal_doc_uses_rev_test() -> - DocBody = {[{<<"hello">>,<<"world">>},{<<"relax">>,true}]}, - Etag = couch_httpd:doc_etag(<<"nomal-doc">>, DocBody, {1, <<"efda11e34e88ebe31a2f83e84a0435b6">>}), + DocBody = {[{<<"hello">>, <<"world">>}, {<<"relax">>, true}]}, + Etag = couch_httpd:doc_etag( + <<"nomal-doc">>, DocBody, {1, <<"efda11e34e88ebe31a2f83e84a0435b6">>} + ), ?assertEqual(Etag, <<"\"1-efda11e34e88ebe31a2f83e84a0435b6\"">>). diff --git a/src/couch/test/eunit/couch_file_tests.erl b/src/couch/test/eunit/couch_file_tests.erl index 606f4bbf4..1b54cd70e 100644 --- a/src/couch/test/eunit/couch_file_tests.erl +++ b/src/couch/test/eunit/couch_file_tests.erl @@ -18,7 +18,6 @@ -define(setup(F), {setup, fun setup/0, fun teardown/1, F}). -define(foreach(Fs), {foreach, fun setup/0, fun teardown/1, Fs}). - setup() -> {ok, Fd} = couch_file:open(?tempfile(), [create, overwrite]), Fd. @@ -34,7 +33,8 @@ open_close_test_() -> "Test for proper file open and close", { setup, - fun() -> test_util:start(?MODULE, [ioq]) end, fun test_util:stop/1, + fun() -> test_util:start(?MODULE, [ioq]) end, + fun test_util:stop/1, [ should_return_enoent_if_missed(), should_ignore_invalid_flags_with_open(), @@ -49,8 +49,10 @@ should_return_enoent_if_missed() -> ?_assertEqual({error, enoent}, couch_file:open("not a real file")). should_ignore_invalid_flags_with_open() -> - ?_assertMatch({ok, _}, - couch_file:open(?tempfile(), [create, invalid_option])). + ?_assertMatch( + {ok, _}, + couch_file:open(?tempfile(), [create, invalid_option]) + ). should_return_pid_on_file_open(Fd) -> ?_assert(is_pid(Fd)). @@ -63,13 +65,13 @@ should_close_file_properly() -> should_create_empty_new_files(Fd) -> ?_assertMatch({ok, 0}, couch_file:bytes(Fd)). - read_write_test_() -> { "Common file read/write tests", { setup, - fun() -> test_util:start(?MODULE, [ioq]) end, fun test_util:stop/1, + fun() -> test_util:start(?MODULE, [ioq]) end, + fun test_util:stop/1, ?foreach([ fun should_increase_file_size_on_write/1, fun should_return_current_file_size_on_write/1, @@ -86,7 +88,6 @@ read_write_test_() -> } }. - should_increase_file_size_on_write(Fd) -> {ok, 0, _} = couch_file:append_term(Fd, foo), {ok, Size} = couch_file:bytes(Fd), @@ -111,7 +112,7 @@ should_return_term_as_binary_for_reading_binary(Fd) -> ?_assertMatch({ok, Foo}, couch_file:pread_binary(Fd, Pos)). should_read_term_written_as_binary(Fd) -> - {ok, Pos, _} = couch_file:append_binary(Fd, <<131,100,0,3,102,111,111>>), + {ok, Pos, _} = couch_file:append_binary(Fd, <<131, 100, 0, 3, 102, 111, 111>>), ?_assertMatch({ok, foo}, couch_file:pread_term(Fd, Pos)). should_write_and_read_large_binary(Fd) -> @@ -139,8 +140,7 @@ should_not_read_beyond_eof(Fd) -> ok = file:pwrite(Io, Pos, <<0:1/integer, DoubleBin:31/integer>>), file:close(Io), unlink(Fd), - ExpectedError = {badmatch, {'EXIT', {bad_return_value, - {read_beyond_eof, Filepath}}}}, + ExpectedError = {badmatch, {'EXIT', {bad_return_value, {read_beyond_eof, Filepath}}}}, ?_assertError(ExpectedError, couch_file:pread_binary(Fd, Pos)). should_truncate(Fd) -> @@ -180,17 +180,16 @@ should_not_read_more_than_pread_limit(Fd) -> BigBin = list_to_binary(lists:duplicate(100000, 0)), {ok, Pos, _Size} = couch_file:append_binary(Fd, BigBin), unlink(Fd), - ExpectedError = {badmatch, {'EXIT', {bad_return_value, - {exceed_pread_limit, Filepath, 50000}}}}, + ExpectedError = {badmatch, {'EXIT', {bad_return_value, {exceed_pread_limit, Filepath, 50000}}}}, ?_assertError(ExpectedError, couch_file:pread_binary(Fd, Pos)). - header_test_() -> { "File header read/write tests", { setup, - fun() -> test_util:start(?MODULE, [ioq]) end, fun test_util:stop/1, + fun() -> test_util:start(?MODULE, [ioq]) end, + fun test_util:stop/1, [ ?foreach([ fun should_write_and_read_atom_header/1, @@ -208,7 +207,6 @@ header_test_() -> } }. - should_write_and_read_atom_header(Fd) -> ok = couch_file:write_header(Fd, hello), ?_assertMatch({ok, hello}, couch_file:read_header(Fd)). @@ -243,7 +241,6 @@ should_save_headers_larger_than_block_size(Fd) -> couch_file:write_header(Fd, Header), {"COUCHDB-1319", ?_assertMatch({ok, Header}, couch_file:read_header(Fd))}. - should_recover_header_marker_corruption() -> ?_assertMatch( ok, @@ -252,7 +249,8 @@ should_recover_header_marker_corruption() -> ?assertNotMatch(Expect, couch_file:read_header(CouchFd)), file:pwrite(RawFd, HeaderPos, <<0>>), ?assertMatch(Expect, couch_file:read_header(CouchFd)) - end) + end + ) ). should_recover_header_size_corruption() -> @@ -264,7 +262,8 @@ should_recover_header_size_corruption() -> % +1 for 0x1 byte marker file:pwrite(RawFd, HeaderPos + 1, <<10/integer>>), ?assertMatch(Expect, couch_file:read_header(CouchFd)) - end) + end + ) ). should_recover_header_md5sig_corruption() -> @@ -276,7 +275,8 @@ should_recover_header_md5sig_corruption() -> % +5 = +1 for 0x1 byte and +4 for term size. file:pwrite(RawFd, HeaderPos + 5, <<"F01034F88D320B22">>), ?assertMatch(Expect, couch_file:read_header(CouchFd)) - end) + end + ) ). should_recover_header_data_corruption() -> @@ -288,10 +288,10 @@ should_recover_header_data_corruption() -> % +21 = +1 for 0x1 byte, +4 for term size and +16 for MD5 sig file:pwrite(RawFd, HeaderPos + 21, <<"some data goes here!">>), ?assertMatch(Expect, couch_file:read_header(CouchFd)) - end) + end + ) ). - check_header_recovery(CheckFun) -> Path = ?tempfile(), {ok, Fd} = couch_file:open(Path, [create, overwrite]), @@ -322,7 +322,6 @@ write_random_data(Fd, N) -> {ok, _, _} = couch_file:append_term(Fd, Term), write_random_data(Fd, N - 1). - delete_test_() -> { "File delete tests", @@ -350,34 +349,33 @@ delete_test_() -> [ fun(Cfg) -> {"enable_database_recovery = false, context = delete", - make_enable_recovery_test_case(Cfg, false, delete)} + make_enable_recovery_test_case(Cfg, false, delete)} end, fun(Cfg) -> {"enable_database_recovery = true, context = delete", - make_enable_recovery_test_case(Cfg, true, delete)} + make_enable_recovery_test_case(Cfg, true, delete)} end, fun(Cfg) -> {"enable_database_recovery = false, context = compaction", - make_enable_recovery_test_case(Cfg, false, compaction)} + make_enable_recovery_test_case(Cfg, false, compaction)} end, fun(Cfg) -> {"enable_database_recovery = true, context = compaction", - make_enable_recovery_test_case(Cfg, true, compaction)} + make_enable_recovery_test_case(Cfg, true, compaction)} end, fun(Cfg) -> {"delete_after_rename = true", - make_delete_after_rename_test_case(Cfg, true)} + make_delete_after_rename_test_case(Cfg, true)} end, fun(Cfg) -> {"delete_after_rename = false", - make_delete_after_rename_test_case(Cfg, false)} + make_delete_after_rename_test_case(Cfg, false)} end ] } } }. - make_enable_recovery_test_case({RootDir, File}, EnableRecovery, Context) -> meck:expect(config, get_boolean, fun ("couchdb", "enable_database_recovery", _) -> EnableRecovery; @@ -388,10 +386,11 @@ make_enable_recovery_test_case({RootDir, File}, EnableRecovery, Context) -> FileExistsAfter = filelib:is_regular(File), RenamedFiles = filelib:wildcard(filename:rootname(File) ++ "*.deleted.*"), DeletedFiles = filelib:wildcard(RootDir ++ "/.delete/*"), - {ExpectRenamedCount, ExpectDeletedCount} = if - EnableRecovery andalso Context =:= delete -> {1, 0}; - true -> {0, 1} - end, + {ExpectRenamedCount, ExpectDeletedCount} = + if + EnableRecovery andalso Context =:= delete -> {1, 0}; + true -> {0, 1} + end, [ ?_assert(FileExistsBefore), ?_assertNot(FileExistsAfter), @@ -408,14 +407,17 @@ make_delete_after_rename_test_case({RootDir, File}, DeleteAfterRename) -> couch_file:delete(RootDir, File), FileExistsAfter = filelib:is_regular(File), RenamedFiles = filelib:wildcard(filename:join([RootDir, ".delete", "*"])), - ExpectRenamedCount = if DeleteAfterRename -> 0; true -> 1 end, + ExpectRenamedCount = + if + DeleteAfterRename -> 0; + true -> 1 + end, [ ?_assert(FileExistsBefore), ?_assertNot(FileExistsAfter), ?_assertEqual(ExpectRenamedCount, length(RenamedFiles)) ]. - nuke_dir_test_() -> { "Nuke directory tests", @@ -454,27 +456,22 @@ nuke_dir_test_() -> end, [ fun(Cfg) -> - {"enable_database_recovery = false", - make_rename_dir_test_case(Cfg, false)} + {"enable_database_recovery = false", make_rename_dir_test_case(Cfg, false)} end, fun(Cfg) -> - {"enable_database_recovery = true", - make_rename_dir_test_case(Cfg, true)} + {"enable_database_recovery = true", make_rename_dir_test_case(Cfg, true)} end, fun(Cfg) -> - {"delete_after_rename = true", - make_delete_dir_test_case(Cfg, true)} + {"delete_after_rename = true", make_delete_dir_test_case(Cfg, true)} end, fun(Cfg) -> - {"delete_after_rename = false", - make_delete_dir_test_case(Cfg, false)} + {"delete_after_rename = false", make_delete_dir_test_case(Cfg, false)} end ] } } }. - make_rename_dir_test_case({RootDir, ViewDir}, EnableRecovery) -> meck:expect(config, get_boolean, fun ("couchdb", "enable_database_recovery", _) -> EnableRecovery; @@ -486,7 +483,11 @@ make_rename_dir_test_case({RootDir, ViewDir}, EnableRecovery) -> DirExistsAfter = filelib:is_dir(ViewDir), Ext = filename:extension(ViewDir), RenamedDirs = filelib:wildcard(RootDir ++ "/*.deleted" ++ Ext), - ExpectRenamedCount = if EnableRecovery -> 1; true -> 0 end, + ExpectRenamedCount = + if + EnableRecovery -> 1; + true -> 0 + end, [ ?_assert(DirExistsBefore), ?_assertNot(DirExistsAfter), @@ -505,7 +506,11 @@ make_delete_dir_test_case({RootDir, ViewDir}, DeleteAfterRename) -> Ext = filename:extension(ViewDir), RenamedDirs = filelib:wildcard(RootDir ++ "/*.deleted" ++ Ext), RenamedFiles = filelib:wildcard(RootDir ++ "/.delete/*"), - ExpectRenamedCount = if DeleteAfterRename -> 0; true -> 1 end, + ExpectRenamedCount = + if + DeleteAfterRename -> 0; + true -> 1 + end, [ ?_assert(DirExistsBefore), ?_assertNot(DirExistsAfter), @@ -517,7 +522,6 @@ remove_dir(Dir) -> [file:delete(File) || File <- filelib:wildcard(filename:join([Dir, "*"]))], file:del_dir(Dir). - fsync_error_test_() -> { "Test fsync raises errors", @@ -535,12 +539,10 @@ fsync_error_test_() -> } }. - fsync_raises_errors() -> Fd = spawn(fun() -> fake_fsync_fd() end), ?assertError({fsync_error, eio}, couch_file:sync(Fd)). - fake_fsync_fd() -> % Mocking gen_server did not go very % well so faking the couch_file pid diff --git a/src/couch/test/eunit/couch_flags_config_tests.erl b/src/couch/test/eunit/couch_flags_config_tests.erl index 6fe2d5860..05707718b 100644 --- a/src/couch/test/eunit/couch_flags_config_tests.erl +++ b/src/couch/test/eunit/couch_flags_config_tests.erl @@ -19,9 +19,9 @@ couch_flags_config_test_() -> setup, fun setup/0, fun teardown/1, - [fun all_combinations_return_same_result/0] - ++ latest_overide_wins() - ++ [ + [fun all_combinations_return_same_result/0] ++ + latest_overide_wins() ++ + [ {"rules_are_sorted", fun rules_are_sorted/0} ] } @@ -29,50 +29,72 @@ couch_flags_config_test_() -> all_combinations_return_same_result() -> Config = [ - {"foo, bar||*", "true"}, - {"baz, qux||*", "false"}, - {"baz||shards/test*", "true"}, - {"baz||shards/blacklist*", "false"}, - {"bar||shards/test*", "false"}, - {"bar||shards/test/blacklist*", "true"} + {"foo, bar||*", "true"}, + {"baz, qux||*", "false"}, + {"baz||shards/test*", "true"}, + {"baz||shards/blacklist*", "false"}, + {"bar||shards/test*", "false"}, + {"bar||shards/test/blacklist*", "true"} ], Expected = [ - {{<<"shards/test/blacklist*">>},{<<"shards/test/blacklist*">>,22,[bar, baz, foo]}}, - {{<<"shards/test*">>},{<<"shards/test*">>, 12, [baz, foo]}}, - {{<<"shards/blacklist*">>},{<<"shards/blacklist*">>, 17, [bar, foo]}}, - {{<<"*">>},{<<"*">>, 1, [bar, foo]}} + {{<<"shards/test/blacklist*">>}, {<<"shards/test/blacklist*">>, 22, [bar, baz, foo]}}, + {{<<"shards/test*">>}, {<<"shards/test*">>, 12, [baz, foo]}}, + {{<<"shards/blacklist*">>}, {<<"shards/blacklist*">>, 17, [bar, foo]}}, + {{<<"*">>}, {<<"*">>, 1, [bar, foo]}} ], Combinations = couch_tests_combinatorics:permutations(Config), - lists:foreach(fun(Items) -> - ?assertEqual(Expected, couch_flags_config:data(Items)) - end, Combinations). + lists:foreach( + fun(Items) -> + ?assertEqual(Expected, couch_flags_config:data(Items)) + end, + Combinations + ). rules_are_sorted() -> Expected = [ - {{<<"shards/test/exact">>},{<<"shards/test/exact">>, 17, [baz,flag_bar,flag_foo]}}, - {{<<"shards/test/blacklist*">>},{<<"shards/test/blacklist*">>,22,[flag_foo]}}, - {{<<"shards/test*">>},{<<"shards/test*">>, 12, [baz,flag_bar,flag_foo]}}, - {{<<"shards/exact">>},{<<"shards/exact">>, 12, [flag_bar,flag_foo]}}, - {{<<"shards/blacklist*">>},{<<"shards/blacklist*">>, 17, []}}, - {{<<"*">>},{<<"*">>, 1, [flag_foo]}} + {{<<"shards/test/exact">>}, {<<"shards/test/exact">>, 17, [baz, flag_bar, flag_foo]}}, + {{<<"shards/test/blacklist*">>}, {<<"shards/test/blacklist*">>, 22, [flag_foo]}}, + {{<<"shards/test*">>}, {<<"shards/test*">>, 12, [baz, flag_bar, flag_foo]}}, + {{<<"shards/exact">>}, {<<"shards/exact">>, 12, [flag_bar, flag_foo]}}, + {{<<"shards/blacklist*">>}, {<<"shards/blacklist*">>, 17, []}}, + {{<<"*">>}, {<<"*">>, 1, [flag_foo]}} ], ?assertEqual(Expected, couch_flags_config:data(test_config())). latest_overide_wins() -> Cases = [ - {[ - {"flag||*", "false"}, {"flag||a*", "true"}, - {"flag||ab*", "true"}, {"flag||abc*", "true"} - ], true}, - {[ - {"flag||*", "true"}, {"flag||a*", "false"}, - {"flag||ab*", "true"}, {"flag||abc*", "false"} - ], false} + { + [ + {"flag||*", "false"}, + {"flag||a*", "true"}, + {"flag||ab*", "true"}, + {"flag||abc*", "true"} + ], + true + }, + { + [ + {"flag||*", "true"}, + {"flag||a*", "false"}, + {"flag||ab*", "true"}, + {"flag||abc*", "false"} + ], + false + } ], - [{test_id(Rules, Expected), - ?_assertEqual(Expected, lists:member(flag, - flags(hd(couch_flags_config:data(Rules)))))} - || {Rules, Expected} <- Cases]. + [ + { + test_id(Rules, Expected), + ?_assertEqual( + Expected, + lists:member( + flag, + flags(hd(couch_flags_config:data(Rules))) + ) + ) + } + || {Rules, Expected} <- Cases + ]. flags({{_Pattern}, {_Pattern, _Size, Flags}}) -> Flags. @@ -80,7 +102,6 @@ flags({{_Pattern}, {_Pattern, _Size, Flags}}) -> test_id(Items, ExpectedResult) -> lists:flatten(io_lib:format("~p -> ~p", [[P || {P, _} <- Items], ExpectedResult])). - test_config() -> [ {"flag_foo||*", "true"}, @@ -95,22 +116,32 @@ test_config() -> parse_flags_term_test_() -> LongBinary = binary:copy(<<"a">>, ?MAX_FLAG_NAME_LENGTH + 1), - ExpectedError = {error, {"Cannot parse list of tags: ~n~p", - [{too_long, LongBinary}]}}, - ExpectedUnknownError = {error,{"Cannot parse list of tags: ~n~p", - [{invalid_flag,<<"dddddddd">>}]}}, - [ - {"empty binary", ?_assertEqual( - [], couch_flags_config:parse_flags_term(<<>>))}, - {"single flag", ?_assertEqual( - [fff], couch_flags_config:parse_flags_term(<<"fff">>))}, - {"sorted", ?_assertEqual( - [aaa,bbb,fff], couch_flags_config:parse_flags_term(<<"fff,aaa,bbb">>))}, - {"whitespace", ?_assertEqual( - [aaa,bbb,fff], couch_flags_config:parse_flags_term(<<"fff , aaa, bbb ">>))}, - {"error", ?_assertEqual( - ExpectedError, couch_flags_config:parse_flags_term(LongBinary))}, - {"unknown_flag", ?_assertEqual( - ExpectedUnknownError, couch_flags_config:parse_flags_term(<<"dddddddd">>))} - ]. - + ExpectedError = {error, {"Cannot parse list of tags: ~n~p", [{too_long, LongBinary}]}}, + ExpectedUnknownError = + {error, {"Cannot parse list of tags: ~n~p", [{invalid_flag, <<"dddddddd">>}]}}, + [ + {"empty binary", + ?_assertEqual( + [], couch_flags_config:parse_flags_term(<<>>) + )}, + {"single flag", + ?_assertEqual( + [fff], couch_flags_config:parse_flags_term(<<"fff">>) + )}, + {"sorted", + ?_assertEqual( + [aaa, bbb, fff], couch_flags_config:parse_flags_term(<<"fff,aaa,bbb">>) + )}, + {"whitespace", + ?_assertEqual( + [aaa, bbb, fff], couch_flags_config:parse_flags_term(<<"fff , aaa, bbb ">>) + )}, + {"error", + ?_assertEqual( + ExpectedError, couch_flags_config:parse_flags_term(LongBinary) + )}, + {"unknown_flag", + ?_assertEqual( + ExpectedUnknownError, couch_flags_config:parse_flags_term(<<"dddddddd">>) + )} + ]. diff --git a/src/couch/test/eunit/couch_flags_tests.erl b/src/couch/test/eunit/couch_flags_tests.erl index 32ec57b77..e3635e9f2 100644 --- a/src/couch/test/eunit/couch_flags_tests.erl +++ b/src/couch/test/eunit/couch_flags_tests.erl @@ -55,7 +55,9 @@ rules() -> setup() -> %% FIXME after we upgrade couch_epi - application:stop(couch_epi), % in case it's already running from other tests... + + % in case it's already running from other tests... + application:stop(couch_epi), application:unload(couch_epi), application:load(couch_epi), @@ -63,8 +65,7 @@ setup() -> meck:expect(config, get, 1, []), Ctx = test_util:start_couch([couch_epi]), - Ctx. - + Ctx. teardown(Ctx) -> test_util:stop_couch(Ctx), @@ -76,57 +77,65 @@ couch_flags_test_() -> { "test couch_flags", { - setup, fun setup/0, fun teardown/1, - enabled_flags_tests() - ++ is_enabled() -%% ++ match_performance() + setup, + fun setup/0, + fun teardown/1, + enabled_flags_tests() ++ + is_enabled() + %% ++ match_performance() } }. enabled_flags_tests() -> - - [{"enabled_flags_tests", [ - {"flags_default_rule", - ?_assertEqual( - [foo], couch_flags:enabled("something"))}, - {"flags_wildcard_rule", - ?_assertEqual( - [bar, baz, foo], - couch_flags:enabled("shards/test/something"))}, - {"flags_exact_rule", - ?_assertEqual( - [bar, baz, foo], - couch_flags:enabled("shards/test/exact"))}, - {"flags_blacklist_rule", - ?_assertEqual( - [], - couch_flags:enabled("shards/blacklist/4"))} - ]}]. + [ + {"enabled_flags_tests", [ + {"flags_default_rule", + ?_assertEqual( + [foo], couch_flags:enabled("something") + )}, + {"flags_wildcard_rule", + ?_assertEqual( + [bar, baz, foo], + couch_flags:enabled("shards/test/something") + )}, + {"flags_exact_rule", + ?_assertEqual( + [bar, baz, foo], + couch_flags:enabled("shards/test/exact") + )}, + {"flags_blacklist_rule", + ?_assertEqual( + [], + couch_flags:enabled("shards/blacklist/4") + )} + ]} + ]. is_enabled() -> - [{"is_enabled_tests", [ - {"flags_default_rule [enabled]", - ?_assert(couch_flags:is_enabled(foo, "something"))}, - {"flags_default_rule [disabled]", - ?_assertNot(couch_flags:is_enabled(baz, "something"))}, - {"flags_default_rule [not_existent]", - ?_assertNot(couch_flags:is_enabled(non_existent, "something"))}, - - {"flags_wildcard_rule [enabled]", - ?_assert(couch_flags:is_enabled(bar, "shards/test/something"))}, - {"flags_wildcard_rule [not_existent]", - ?_assertNot(couch_flags:is_enabled(non_existent, "shards/test/something"))}, - - {"flags_exact_rule [overide_disbled]", - ?_assert(couch_flags:is_enabled(bar, "shards/test/exact"))}, - {"flags_exact_rule [not_existent]", - ?_assertNot(couch_flags:is_enabled(non_existent, "shards/test/exact"))}, - - {"flags_blacklist_rule [overide_enabled]", - ?_assertNot(couch_flags:is_enabled(foo, "shards/blacklist/4"))}, - {"flags_blacklist_rule [not_existent]", - ?_assertNot(couch_flags:is_enabled(non_existent, "shards/blacklist/4"))} - ]}]. + [ + {"is_enabled_tests", [ + {"flags_default_rule [enabled]", ?_assert(couch_flags:is_enabled(foo, "something"))}, + {"flags_default_rule [disabled]", + ?_assertNot(couch_flags:is_enabled(baz, "something"))}, + {"flags_default_rule [not_existent]", + ?_assertNot(couch_flags:is_enabled(non_existent, "something"))}, + + {"flags_wildcard_rule [enabled]", + ?_assert(couch_flags:is_enabled(bar, "shards/test/something"))}, + {"flags_wildcard_rule [not_existent]", + ?_assertNot(couch_flags:is_enabled(non_existent, "shards/test/something"))}, + + {"flags_exact_rule [overide_disbled]", + ?_assert(couch_flags:is_enabled(bar, "shards/test/exact"))}, + {"flags_exact_rule [not_existent]", + ?_assertNot(couch_flags:is_enabled(non_existent, "shards/test/exact"))}, + + {"flags_blacklist_rule [overide_enabled]", + ?_assertNot(couch_flags:is_enabled(foo, "shards/blacklist/4"))}, + {"flags_blacklist_rule [not_existent]", + ?_assertNot(couch_flags:is_enabled(non_existent, "shards/blacklist/4"))} + ]} + ]. %% match_performance() -> %% [{"match_performance", [ @@ -137,7 +146,6 @@ is_enabled() -> %% end) %% ]}]. - test_config() -> [ {"foo||/*", "true"}, diff --git a/src/couch/test/eunit/couch_index_tests.erl b/src/couch/test/eunit/couch_index_tests.erl index 23c857d6c..368f7a059 100644 --- a/src/couch/test/eunit/couch_index_tests.erl +++ b/src/couch/test/eunit/couch_index_tests.erl @@ -36,10 +36,12 @@ couch_index_ioq_priority_test_() -> "Test ioq_priority for views", { setup, - fun test_util:start_couch/0, fun test_util:stop_couch/1, + fun test_util:start_couch/0, + fun test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun check_io_priority_for_updater/1, fun check_io_priority_for_compactor/1 @@ -48,11 +50,11 @@ couch_index_ioq_priority_test_() -> } }. - check_io_priority_for_updater(DbName) -> ?_test(begin {ok, IndexerPid} = couch_index_server:get_index( - couch_mrview_index, DbName, <<"_design/foo">>), + couch_mrview_index, DbName, <<"_design/foo">> + ), CouchIndexUpdaterPid = updater_pid(IndexerPid), tracer_record(CouchIndexUpdaterPid), @@ -63,15 +65,23 @@ check_io_priority_for_updater(DbName) -> [UpdaterPid] = wait_spawn_event_for_pid(CouchIndexUpdaterPid), [UpdaterMapProcess] = wait_spawn_by_anonymous_fun( - UpdaterPid, '-start_update/4-fun-0-'), + UpdaterPid, '-start_update/4-fun-0-' + ), - ?assert(wait_set_io_priority( - UpdaterMapProcess, {view_update, DbName, <<"_design/foo">>})), + ?assert( + wait_set_io_priority( + UpdaterMapProcess, {view_update, DbName, <<"_design/foo">>} + ) + ), [UpdaterWriterProcess] = wait_spawn_by_anonymous_fun( - UpdaterPid, '-start_update/4-fun-1-'), - ?assert(wait_set_io_priority( - UpdaterWriterProcess, {view_update, DbName, <<"_design/foo">>})), + UpdaterPid, '-start_update/4-fun-1-' + ), + ?assert( + wait_set_io_priority( + UpdaterWriterProcess, {view_update, DbName, <<"_design/foo">>} + ) + ), ok end). @@ -79,7 +89,8 @@ check_io_priority_for_updater(DbName) -> check_io_priority_for_compactor(DbName) -> ?_test(begin {ok, IndexerPid} = couch_index_server:get_index( - couch_mrview_index, DbName, <<"_design/foo">>), + couch_mrview_index, DbName, <<"_design/foo">> + ), {ok, CompactorPid} = couch_index:get_compactor_pid(IndexerPid), tracer_record(CompactorPid), @@ -89,51 +100,65 @@ check_io_priority_for_compactor(DbName) -> wait_spawn_event_for_pid(CompactorPid), [CompactorProcess] = wait_spawn_by_anonymous_fun( - CompactorPid, '-handle_call/3-fun-0-'), - ?assert(wait_set_io_priority( - CompactorProcess, {view_compact, DbName, <<"_design/foo">>})), + CompactorPid, '-handle_call/3-fun-0-' + ), + ?assert( + wait_set_io_priority( + CompactorProcess, {view_compact, DbName, <<"_design/foo">>} + ) + ), ok end). create_docs(DbName) -> {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]), - Doc1 = couch_doc:from_json_obj({[ - {<<"_id">>, <<"doc1">>}, - {<<"value">>, 1} - - ]}), - Doc2 = couch_doc:from_json_obj({[ - {<<"_id">>, <<"doc2">>}, - {<<"value">>, 2} - - ]}), - Doc3 = couch_doc:from_json_obj({[ - {<<"_id">>, <<"doc3">>}, - {<<"value">>, 3} - - ]}), + Doc1 = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"doc1">>}, + {<<"value">>, 1} + ]} + ), + Doc2 = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"doc2">>}, + {<<"value">>, 2} + ]} + ), + Doc3 = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"doc3">>}, + {<<"value">>, 3} + ]} + ), {ok, _} = couch_db:update_docs(Db, [Doc1, Doc2, Doc3]), couch_db:close(Db). create_design_doc(DbName, DDName, ViewName) -> {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]), - DDoc = couch_doc:from_json_obj({[ - {<<"_id">>, DDName}, - {<<"language">>, <<"javascript">>}, - {<<"views">>, {[ - {ViewName, {[ - {<<"map">>, <<"function(doc) { emit(doc.value, null); }">>} - ]}} - ]}} - ]}), + DDoc = couch_doc:from_json_obj( + {[ + {<<"_id">>, DDName}, + {<<"language">>, <<"javascript">>}, + {<<"views">>, + {[ + {ViewName, + {[ + {<<"map">>, <<"function(doc) { emit(doc.value, null); }">>} + ]}} + ]}} + ]} + ), {ok, Rev} = couch_db:update_doc(Db, DDoc, []), couch_db:close(Db), Rev. wait_set_io_priority(Pid, IOPriority) -> - test_util:wait_value(fun() -> - does_process_set_io_priority(Pid, IOPriority) - end, true). + test_util:wait_value( + fun() -> + does_process_set_io_priority(Pid, IOPriority) + end, + true + ). does_process_set_io_priority(Pid, IOPriority) -> PutCallsArgs = find_calls_to_fun(Pid, {erlang, put, 2}), @@ -143,36 +168,47 @@ wait_events(MatchSpec) -> test_util:wait_other_value(fun() -> select(MatchSpec) end, []). find_spawned_by_anonymous_fun(ParentPid, Name) -> - AnonymousFuns = select(ets:fun2ms(fun - ({spawned, Pid, _TS, _Name, _Dict, [PPid, {erlang, apply, [Fun, _]}]}) - when is_function(Fun) andalso PPid =:= ParentPid -> {Pid, Fun} - end)), - lists:filtermap(fun({Pid, Fun}) -> - case erlang:fun_info(Fun, name) of - {name, Name} -> {true, Pid}; - _ -> false - end - end, AnonymousFuns). + AnonymousFuns = select( + ets:fun2ms(fun({spawned, Pid, _TS, _Name, _Dict, [PPid, {erlang, apply, [Fun, _]}]}) when + is_function(Fun) andalso PPid =:= ParentPid + -> + {Pid, Fun} + end) + ), + lists:filtermap( + fun({Pid, Fun}) -> + case erlang:fun_info(Fun, name) of + {name, Name} -> {true, Pid}; + _ -> false + end + end, + AnonymousFuns + ). find_calls_to_fun(Pid, {Module, Function, Arity}) -> - select(ets:fun2ms(fun - ({call, P, _TS, _Name, _Dict, [{M, F, Args}]}) - when length(Args) =:= Arity - andalso M =:= Module - andalso F =:= Function - andalso P =:= Pid - -> Args - end)). + select( + ets:fun2ms(fun({call, P, _TS, _Name, _Dict, [{M, F, Args}]}) when + length(Args) =:= Arity andalso + M =:= Module andalso + F =:= Function andalso + P =:= Pid + -> + Args + end) + ). wait_spawn_event_for_pid(ParentPid) -> - wait_events(ets:fun2ms(fun - ({spawned, Pid, _TS, _Name, _Dict, [P, _]}) when P =:= ParentPid -> Pid - end)). + wait_events( + ets:fun2ms(fun({spawned, Pid, _TS, _Name, _Dict, [P, _]}) when P =:= ParentPid -> Pid end) + ). wait_spawn_by_anonymous_fun(ParentPid, Name) -> - test_util:wait_other_value(fun() -> - find_spawned_by_anonymous_fun(ParentPid, Name) - end, []). + test_util:wait_other_value( + fun() -> + find_spawned_by_anonymous_fun(ParentPid, Name) + end, + [] + ). updater_pid(IndexerPid) -> {links, Links} = process_info(IndexerPid, links), @@ -180,20 +216,25 @@ updater_pid(IndexerPid) -> Pid. select_process_by_name_prefix(Pids, Name) -> - lists:filter(fun(Pid) -> - Key = couch_debug:process_name(Pid), - string:str(Key, Name) =:= 1 - end, Pids). + lists:filter( + fun(Pid) -> + Key = couch_debug:process_name(Pid), + string:str(Key, Name) =:= 1 + end, + Pids + ). select(MatchSpec) -> - lists:filtermap(fun(Event) -> - case ets:test_ms(Event, MatchSpec) of - {ok, false} -> false; - {ok, Result} -> {true, Result}; - _ -> false - end - end, tracer_events()). - + lists:filtermap( + fun(Event) -> + case ets:test_ms(Event, MatchSpec) of + {ok, false} -> false; + {ok, Result} -> {true, Result}; + _ -> false + end + end, + tracer_events() + ). %% ======================== %% Tracer related functions @@ -225,7 +266,7 @@ tracer_collector(Msg, Seq) -> normalize_trace_msg(TraceMsg) -> case tuple_to_list(TraceMsg) of [trace_ts, Pid, Type | Info] -> - {TraceInfo, [Timestamp]} = lists:split(length(Info)-1, Info), + {TraceInfo, [Timestamp]} = lists:split(length(Info) - 1, Info), {Type, Pid, Timestamp, couch_debug:process_name(Pid), process_info(Pid), TraceInfo}; [trace, Pid, Type | TraceInfo] -> {Type, Pid, os:timestamp(), couch_debug:process_name(Pid), process_info(Pid), TraceInfo} diff --git a/src/couch/test/eunit/couch_js_tests.erl b/src/couch/test/eunit/couch_js_tests.erl index c68d60125..1079678da 100644 --- a/src/couch/test/eunit/couch_js_tests.erl +++ b/src/couch/test/eunit/couch_js_tests.erl @@ -13,7 +13,6 @@ -module(couch_js_tests). -include_lib("eunit/include/eunit.hrl"). - couch_js_test_() -> { "Test couchjs", @@ -32,104 +31,131 @@ couch_js_test_() -> } }. - should_create_sandbox() -> % Try and detect whether we can see out of the % sandbox or not. Src = << - "function(doc) {\n" - " try {\n" - " emit(false, typeof(Couch.compile_function));\n" - " } catch (e) {\n" - " emit(true, e.message);\n" - " }\n" - "}\n" + "function(doc) {\n" + " try {\n" + " emit(false, typeof(Couch.compile_function));\n" + " } catch (e) {\n" + " emit(true, e.message);\n" + " }\n" + "}\n" >>, Proc = couch_query_servers:get_os_process(<<"javascript">>), true = couch_query_servers:proc_prompt(Proc, [<<"add_fun">>, Src]), Result = couch_query_servers:proc_prompt(Proc, [<<"map_doc">>, <<"{}">>]), ?assertEqual([[[true, <<"Couch is not defined">>]]], Result). - should_roundtrip_utf8() -> % Try round tripping UTF-8 both directions through % couchjs. These tests use hex encoded values of % Ä (C384) and Ü (C39C) so as to avoid odd editor/Erlang encoding % strangeness. Src = << - "function(doc) {\n" - " emit(doc.value, \"", 16#C3, 16#9C, "\");\n" - "}\n" + "function(doc) {\n" + " emit(doc.value, \"", + 16#C3, + 16#9C, + "\");\n" + "}\n" >>, Proc = couch_query_servers:get_os_process(<<"javascript">>), true = couch_query_servers:proc_prompt(Proc, [<<"add_fun">>, Src]), - Doc = {[ - {<<"value">>, <<16#C3, 16#84>>} - ]}, + Doc = + {[ + {<<"value">>, <<16#C3, 16#84>>} + ]}, Result = couch_query_servers:proc_prompt(Proc, [<<"map_doc">>, Doc]), ?assertEqual([[[<<16#C3, 16#84>>, <<16#C3, 16#9C>>]]], Result). - should_roundtrip_modified_utf8() -> % Mimicing the test case from the mailing list Src = << - "function(doc) {\n" - " emit(doc.value.toLowerCase(), \"", 16#C3, 16#9C, "\");\n" - "}\n" + "function(doc) {\n" + " emit(doc.value.toLowerCase(), \"", + 16#C3, + 16#9C, + "\");\n" + "}\n" >>, Proc = couch_query_servers:get_os_process(<<"javascript">>), true = couch_query_servers:proc_prompt(Proc, [<<"add_fun">>, Src]), - Doc = {[ - {<<"value">>, <<16#C3, 16#84>>} - ]}, + Doc = + {[ + {<<"value">>, <<16#C3, 16#84>>} + ]}, Result = couch_query_servers:proc_prompt(Proc, [<<"map_doc">>, Doc]), ?assertEqual([[[<<16#C3, 16#A4>>, <<16#C3, 16#9C>>]]], Result). - should_replace_broken_utf16() -> % This test reverse the surrogate pair of % the Boom emoji U+1F4A5 Src = << - "function(doc) {\n" - " emit(doc.value.split(\"\").reverse().join(\"\"), 1);\n" - "}\n" + "function(doc) {\n" + " emit(doc.value.split(\"\").reverse().join(\"\"), 1);\n" + "}\n" >>, Proc = couch_query_servers:get_os_process(<<"javascript">>), true = couch_query_servers:proc_prompt(Proc, [<<"add_fun">>, Src]), - Doc = {[ - {<<"value">>, list_to_binary(xmerl_ucs:to_utf8([16#1F4A5]))} - ]}, + Doc = + {[ + {<<"value">>, list_to_binary(xmerl_ucs:to_utf8([16#1F4A5]))} + ]}, Result = couch_query_servers:proc_prompt(Proc, [<<"map_doc">>, Doc]), % Invalid UTF-8 gets replaced with the 16#FFFD replacement % marker Markers = list_to_binary(xmerl_ucs:to_utf8([16#FFFD, 16#FFFD])), ?assertEqual([[[Markers, 1]]], Result). - should_allow_js_string_mutations() -> % This binary corresponds to this string: мама мыла раму % Which I'm told translates to: "mom was washing the frame" MomWashedTheFrame = << - 16#D0, 16#BC, 16#D0, 16#B0, 16#D0, 16#BC, 16#D0, 16#B0, 16#20, - 16#D0, 16#BC, 16#D1, 16#8B, 16#D0, 16#BB, 16#D0, 16#B0, 16#20, - 16#D1, 16#80, 16#D0, 16#B0, 16#D0, 16#BC, 16#D1, 16#83 + 16#D0, + 16#BC, + 16#D0, + 16#B0, + 16#D0, + 16#BC, + 16#D0, + 16#B0, + 16#20, + 16#D0, + 16#BC, + 16#D1, + 16#8B, + 16#D0, + 16#BB, + 16#D0, + 16#B0, + 16#20, + 16#D1, + 16#80, + 16#D0, + 16#B0, + 16#D0, + 16#BC, + 16#D1, + 16#83 >>, Mom = <<16#D0, 16#BC, 16#D0, 16#B0, 16#D0, 16#BC, 16#D0, 16#B0>>, Washed = <<16#D0, 16#BC, 16#D1, 16#8B, 16#D0, 16#BB, 16#D0, 16#B0>>, Src1 = << - "function(doc) {\n" - " emit(\"length\", doc.value.length);\n" - "}\n" + "function(doc) {\n" + " emit(\"length\", doc.value.length);\n" + "}\n" >>, Src2 = << - "function(doc) {\n" - " emit(\"substring\", doc.value.substring(5, 9));\n" - "}\n" + "function(doc) {\n" + " emit(\"substring\", doc.value.substring(5, 9));\n" + "}\n" >>, Src3 = << - "function(doc) {\n" - " emit(\"slice\", doc.value.slice(0, 4));\n" - "}\n" + "function(doc) {\n" + " emit(\"slice\", doc.value.slice(0, 4));\n" + "}\n" >>, Proc = couch_query_servers:get_os_process(<<"javascript">>), true = couch_query_servers:proc_prompt(Proc, [<<"add_fun">>, Src1]), @@ -144,29 +170,30 @@ should_allow_js_string_mutations() -> ], ?assertEqual(Expect, Result). - should_exit_on_oom() -> Src = << - "var state = [];\n" - "function(doc) {\n" - " var val = \"0123456789ABCDEF\";\n" - " for(var i = 0; i < 665535; i++) {\n" - " state.push([val, val]);\n" - " emit(null, null);\n" - " }\n" - "}\n" + "var state = [];\n" + "function(doc) {\n" + " var val = \"0123456789ABCDEF\";\n" + " for(var i = 0; i < 665535; i++) {\n" + " state.push([val, val]);\n" + " emit(null, null);\n" + " }\n" + "}\n" >>, Proc = couch_query_servers:get_os_process(<<"javascript">>), true = couch_query_servers:proc_prompt(Proc, [<<"add_fun">>, Src]), trigger_oom(Proc). trigger_oom(Proc) -> - Status = try - couch_query_servers:proc_prompt(Proc, [<<"map_doc">>, <<"{}">>]), - continue - catch throw:{os_process_error, {exit_status, 1}} -> - done - end, + Status = + try + couch_query_servers:proc_prompt(Proc, [<<"map_doc">>, <<"{}">>]), + continue + catch + throw:{os_process_error, {exit_status, 1}} -> + done + end, case Status of continue -> trigger_oom(Proc); done -> ok diff --git a/src/couch/test/eunit/couch_key_tree_prop_tests.erl b/src/couch/test/eunit/couch_key_tree_prop_tests.erl index 9c09aace5..d6ed26553 100644 --- a/src/couch/test/eunit/couch_key_tree_prop_tests.erl +++ b/src/couch/test/eunit/couch_key_tree_prop_tests.erl @@ -12,49 +12,53 @@ -module(couch_key_tree_prop_tests). - -ifdef(WITH_PROPER). -include_lib("couch/include/couch_eunit_proper.hrl"). - --define(SIZE_REDUCTION, 3). % How much to reduce size with tree depth. --define(MAX_BRANCHES, 4). % Maximum number of branches. +% How much to reduce size with tree depth. +-define(SIZE_REDUCTION, 3). +% Maximum number of branches. +-define(MAX_BRANCHES, 4). -define(RAND_SIZE, 1 bsl 64). - property_test_() -> ?EUNIT_QUICKCHECK(60). - % % Properties % - % Merge random paths from a revtree into itself. Check that no revisions have % been lost in the process and that result is one of the 3 expected values. % prop_revtree_merge_with_subset_of_own_nodes() -> - ?FORALL(Revs, g_revs(), - ?FORALL({RevTree, Branch}, {g_revtree(Revs), g_revtree(Revs, 1)}, - ?IMPLIES(length(Branch) > 0 andalso repeating_revs(levels(RevTree ++ Branch)) == [], + ?FORALL( + Revs, + g_revs(), + ?FORALL( + {RevTree, Branch}, + {g_revtree(Revs), g_revtree(Revs, 1)}, + ?IMPLIES( + length(Branch) > 0 andalso repeating_revs(levels(RevTree ++ Branch)) == [], begin {Merged, Result} = couch_key_tree:merge(RevTree, hd(Branch)), - lists:member(Result, [new_leaf, new_branch, internal_node]) - andalso same_keys(RevTree ++ Branch, Merged) - andalso valid_revtree(Merged) + lists:member(Result, [new_leaf, new_branch, internal_node]) andalso + same_keys(RevTree ++ Branch, Merged) andalso + valid_revtree(Merged) end ) ) ). - % Merge random trees into revtree. % prop_revtree_merge_random_nodes() -> - ?FORALL({RevTree, Branch}, {g_revtree(), g_revtree([], 1)}, - ?IMPLIES(length(Branch) > 0, + ?FORALL( + {RevTree, Branch}, + {g_revtree(), g_revtree([], 1)}, + ?IMPLIES( + length(Branch) > 0, begin {Merged, _} = couch_key_tree:merge(RevTree, hd(Branch)), valid_revtree(Merged) @@ -62,33 +66,35 @@ prop_revtree_merge_random_nodes() -> ) ). - - % Merge mix or random and existing revtree paths into revtree % prop_revtree_merge_some_existing_some_new() -> - ?FORALL(RevTree, g_revtree(), - ?FORALL(Branch, + ?FORALL( + RevTree, + g_revtree(), + ?FORALL( + Branch, begin KeyList = keylist(RevTree), Half = lists:sublist(KeyList, length(KeyList) div 2), g_revtree(Half, 1) end, - ?IMPLIES(length(Branch) > 0 andalso repeating_revs(levels(RevTree ++ Branch)) == [], - begin - {Merged, _} = couch_key_tree:merge(RevTree, hd(Branch)), - valid_revtree(Merged) - end + ?IMPLIES( + length(Branch) > 0 andalso repeating_revs(levels(RevTree ++ Branch)) == [], + begin + {Merged, _} = couch_key_tree:merge(RevTree, hd(Branch)), + valid_revtree(Merged) + end ) ) ). - - % Stem deeper than the current max level. Expect no changes to the revtree % prop_no_change_stemming_deeper_than_current_depth() -> - ?FORALL(RevTree, g_revtree(), + ?FORALL( + RevTree, + g_revtree(), begin StemDepth = depth(RevTree) + 1, Stemmed = couch_key_tree:stem(RevTree, StemDepth), @@ -98,12 +104,13 @@ prop_no_change_stemming_deeper_than_current_depth() -> end ). - % Stem at a random small depth, make sure that resulting tree has % unique revisions and the same number or less revisions than input % prop_stemming_results_in_same_or_less_total_revs() -> - ?FORALL({RevTree, StemDepth}, {g_revtree(), choose(1, 20)}, + ?FORALL( + {RevTree, StemDepth}, + {g_revtree(), choose(1, 20)}, begin Stemmed = couch_key_tree:stem(RevTree, StemDepth), OldRealDepth = real_depth(RevTree), @@ -111,89 +118,95 @@ prop_stemming_results_in_same_or_less_total_revs() -> UniqueStemmedKeys = lists:usort(StemmedKeys), UniqueInputKeys = lists:usort(keylist(RevTree)), NewRealDepth = real_depth(Stemmed), - length(StemmedKeys) == length(UniqueStemmedKeys) - andalso length(UniqueStemmedKeys) =< length(UniqueInputKeys) - andalso OldRealDepth >= NewRealDepth + length(StemmedKeys) == length(UniqueStemmedKeys) andalso + length(UniqueStemmedKeys) =< length(UniqueInputKeys) andalso + OldRealDepth >= NewRealDepth end ). - % Generate a longer path (revtree with no branches) then stem it. % Always expect it to shrink to stemmed depth. prop_stem_path_expect_size_to_get_smaller() -> - ?FORALL({RevTree, StemDepth}, + ?FORALL( + {RevTree, StemDepth}, { ?SIZED(Size, g_revtree(Size * 10, [], 1)), - choose(1,3) + choose(1, 3) }, - ?IMPLIES(real_depth(RevTree) > 3, + ?IMPLIES( + real_depth(RevTree) > 3, begin Stemmed = couch_key_tree:stem(RevTree, StemDepth), StemmedKeys = lists:usort(keylist(Stemmed)), InputKeys = lists:usort(keylist(RevTree)), - length(InputKeys) > length(StemmedKeys) - andalso real_depth(Stemmed) == StemDepth + length(InputKeys) > length(StemmedKeys) andalso + real_depth(Stemmed) == StemDepth end ) ). - % After stemming all leaves are still present prop_after_stemming_all_leaves_are_present() -> - ?FORALL({RevTree, StemDepth}, - {g_revtree(), choose(1,20)}, + ?FORALL( + {RevTree, StemDepth}, + {g_revtree(), choose(1, 20)}, begin OldRealDepth = real_depth(RevTree), OldLeaves = leaves(RevTree), Stemmed = couch_key_tree:stem(RevTree, StemDepth), NewRealDepth = real_depth(Stemmed), NewLeaves = leaves(Stemmed), - valid_revtree(Stemmed) - andalso OldRealDepth >= NewRealDepth - andalso OldLeaves == NewLeaves - + valid_revtree(Stemmed) andalso + OldRealDepth >= NewRealDepth andalso + OldLeaves == NewLeaves end ). - % After stemming paths to root didn't get longer prop_after_stemming_paths_are_shorter() -> - ?FORALL({StemDepth, RevTree}, {choose(2,10), g_revtree()}, + ?FORALL( + {StemDepth, RevTree}, + {choose(2, 10), g_revtree()}, begin OldPaths = paths(RevTree), Stemmed = couch_key_tree:stem(RevTree, StemDepth), NewPaths = paths(Stemmed), - GrowingPaths = orddict:fold(fun(Rev, Path, Acc) -> - OldPath = orddict:fetch(Rev, OldPaths), - case length(Path) > length(OldPath) of - true -> - [{Rev, Path, OldPath}| Acc]; - false -> - Acc - end - end, [], NewPaths), + GrowingPaths = orddict:fold( + fun(Rev, Path, Acc) -> + OldPath = orddict:fetch(Rev, OldPaths), + case length(Path) > length(OldPath) of + true -> + [{Rev, Path, OldPath} | Acc]; + false -> + Acc + end + end, + [], + NewPaths + ), valid_revtree(Stemmed) andalso GrowingPaths == [] end ). - % Check leaf count prop_leaf_count() -> - ?FORALL(RevTree, g_revtree(), + ?FORALL( + RevTree, + g_revtree(), length(leaves(RevTree)) == couch_key_tree:count_leafs(RevTree) ). - % Check get leafs prop_get_leafs() -> - ?FORALL(RevTree, g_revtree(), + ?FORALL( + RevTree, + g_revtree(), begin LeafsFull = couch_key_tree:get_all_leafs(RevTree), lists:usort([Rev || {_V, {_D, [Rev | _]}} <- LeafsFull]) == leaves(RevTree) end ). - % % Generators % @@ -205,39 +218,40 @@ prop_get_leafs() -> g_revtree() -> ?SIZED(Size, g_revtree(Size)). - g_revtree(Size) when is_integer(Size) -> g_revtree(Size, [], ?MAX_BRANCHES); g_revtree(Revs) when is_list(Revs) -> ?SIZED(Size, g_revtree(Size, Revs, ?MAX_BRANCHES)). - g_revtree(Size, Revs) when is_integer(Size), is_list(Revs) -> g_revtree(Size, Revs, ?MAX_BRANCHES); g_revtree(Revs, MaxBranches) when is_list(Revs), is_integer(MaxBranches) -> ?SIZED(Size, g_revtree(Size, Revs, MaxBranches)). - g_revtree(0, _Revs, _MaxBranches) -> []; g_revtree(Size, ERevs, MaxBranches) -> - ?LET({Depth, Revs}, {g_stem_depth(Size), g_revs(Size, ERevs)}, - [{Depth, g_treenode(Size, Revs, MaxBranches)}] + ?LET( + {Depth, Revs}, + {g_stem_depth(Size), g_revs(Size, ERevs)}, + [{Depth, g_treenode(Size, Revs, MaxBranches)}] ). - % Generate a tree node and then recursively generate its children. % g_treenode(0, Revs, _) -> {elements(Revs), x, []}; g_treenode(Size, Revs, MaxBranches) -> - ?LAZY(?LET(N, choose(0, MaxBranches), - begin - [Rev | ChildRevs] = Revs, - {Rev, x, g_nodes(Size div ?SIZE_REDUCTION, N, ChildRevs, MaxBranches)} - end - )). - + ?LAZY( + ?LET( + N, + choose(0, MaxBranches), + begin + [Rev | ChildRevs] = Revs, + {Rev, x, g_nodes(Size div ?SIZE_REDUCTION, N, ChildRevs, MaxBranches)} + end + ) + ). % Generate a list of child nodes. Depending on how many children there are % the pre-generarated revision list is split into that many sublists. @@ -256,82 +270,70 @@ g_nodes(Size, ChildCount, Revs, MaxBranches) -> ordered_nodes(ChildNodes) ). - % Generate each subtree's stem depth % - g_stem_depth(Size) -> - choose(0, expected_height(Size, ?SIZE_REDUCTION) div 2). - + choose(0, expected_height(Size, ?SIZE_REDUCTION) div 2). % Uses the shuffle/1 function to shuffle the input list. Unshuffled list is % used as the shrink value. % -g_shuffle([]) -> []; +g_shuffle([]) -> + []; g_shuffle(L) when is_list(L) -> - ?LET(X, elements(L), [X | g_shuffle(lists:delete(X,L))]). - + ?LET(X, elements(L), [X | g_shuffle(lists:delete(X, L))]). % Wrapper to make a list shuffling generator that doesn't shrink % g_shuffle_noshrink(L) when is_list(L) -> proper_types:noshrink(g_shuffle(L)). - % Generate shuffled sublists up to N items long from a list. % g_shuffled_sublists(L, N) -> ?LET(Shuffled, g_shuffle_noshrink(L), lists:sublist(Shuffled, N)). - % Generate revision lists. % g_revs() -> ?SIZED(Size, g_revs(Size)). - g_revs(Size) when is_integer(Size) -> g_revs(Size, []). - g_revs(Size, Existing) when is_integer(Size), is_list(Existing) -> Expected = keys_needed(Size, ?SIZE_REDUCTION, ?MAX_BRANCHES), Revs = revs(Expected, Existing), case length(Revs) > Expected of - true -> % have extra, try various sublists + % have extra, try various sublists + true -> g_shuffled_sublists(Revs, Expected); false -> proper_types:return(Revs) end. - % % Helper functions % - valid_revtree(RevTree) -> repeating_revs(levels(RevTree)) == [] andalso children_sorted(RevTree). - same_keys(RevTree1, RevTree2) -> Keys1 = lists:usort(keylist(RevTree1)), Keys2 = lists:usort(keylist(RevTree2)), Keys1 == Keys2. - all(L) -> lists:all(fun(E) -> E end, L). - % Generate list of relateively unique large random numbers rand_list(N) when N =< 0 -> []; rand_list(N) -> [rand:uniform(?RAND_SIZE) || _ <- lists:seq(1, N)]. - % Generate a list of revisions to be used as key in revision trees. Expected % must the number of maximum expected nodes in a revision tree. Existing is an % optional list revisions which must be included in the result. The output list @@ -342,7 +344,6 @@ revs(Expected, Existing) when is_integer(Expected), is_list(Existing) -> Need = Expected - length(Existing), lists:usort(lists:append(Existing, rand_list(Need))). - % Get the list of all the keys in a revision tree. The input can also be a % an individual tree (tagged with the depth to virtual root) or a node. % Yes, this is not tail recursive but the idea is to keep it simple. @@ -354,7 +355,6 @@ keylist({K, _V, Nodes}) -> keylist(Nodes) -> lists:append([keylist(Node) || Node <- Nodes]). - % Get the list of leaves from a revision tree. leaves([]) -> []; @@ -367,7 +367,6 @@ leaves({_K, _V, Nodes}) -> leaves(Nodes) -> lists:usort(lists:append([leaves(N) || N <- Nodes])). - % Get paths from leaf to root. Result is an orddict of [{LeafRev, [Rev]}] % paths([]) -> @@ -382,14 +381,20 @@ paths({K, _V, Nodes}) -> CombinedDict = paths_merge_dicts([paths(N) || N <- Nodes]), orddict:map(fun(_LeafKey, Path) -> Path ++ [K] end, CombinedDict). - paths_merge_dicts(Dicts) -> - lists:foldl(fun(D, AccD) -> - orddict:merge(fun(K, V1, V2) -> - throw({found_duplicates, K, V1, V2}) - end, D, AccD) - end, orddict:new(), Dicts). - + lists:foldl( + fun(D, AccD) -> + orddict:merge( + fun(K, V1, V2) -> + throw({found_duplicates, K, V1, V2}) + end, + D, + AccD + ) + end, + orddict:new(), + Dicts + ). % Get lists of all the keys at each depth level. Result is an orddict that % looks like [{depth, [key]}]. The depth used here is the "virtual" depth as @@ -400,32 +405,36 @@ levels([]) -> levels(RevTree) when is_list(RevTree) -> lists:foldl(fun(T, Dict) -> levels(T, Dict) end, orddict:new(), RevTree). - levels({Depth, Node}, Dict) when is_tuple(Node) -> levels(Node, Depth, Dict). - levels({K, _V, Nodes}, Depth, Dict) -> - Dict1 = case orddict:is_key(Depth, Dict) of - true -> orddict:append(Depth, K, Dict); - false -> orddict:store(Depth, [K], Dict) - end, + Dict1 = + case orddict:is_key(Depth, Dict) of + true -> orddict:append(Depth, K, Dict); + false -> orddict:store(Depth, [K], Dict) + end, levels(Nodes, Depth + 1, Dict1); levels(Nodes, Depth, Dict) -> - lists:foldl(fun(Node, AccDict) -> - levels(Node, Depth, AccDict) - end, Dict, Nodes). - + lists:foldl( + fun(Node, AccDict) -> + levels(Node, Depth, AccDict) + end, + Dict, + Nodes + ). % Using the output of leaves/1 as input return any repeating revisions if % there are any at a particular level. Levels which have not revisions are % not returned. % repeating_revs(Dict) -> - orddict:filter(fun(_Depth, Revs) -> - length(lists:usort(Revs)) =/= length(Revs) - end, Dict). - + orddict:filter( + fun(_Depth, Revs) -> + length(lists:usort(Revs)) =/= length(Revs) + end, + Dict + ). % Check that children of all nodes are sorted children_sorted([]) -> @@ -437,7 +446,6 @@ children_sorted({_D, Node}) when is_tuple(Node) -> children_sorted({_K, _V, Nodes}) -> children_sorted(Nodes). - % Get the maximum depth of a revtree. The depth is "virtual" as it takes into % account the distance to the now stemmed root node as indicated by the top % level subtrees. @@ -449,7 +457,6 @@ depth(RevTree) when is_list(RevTree) -> depth({Depth, Node}) when is_tuple(Node) -> depth(Node, Depth - 1). - depth({_K, _V, Nodes}, Depth) -> depth(Nodes, Depth + 1); depth([], Depth) -> @@ -457,7 +464,6 @@ depth([], Depth) -> depth(Nodes, Depth) -> lists:max([depth(Node, Depth) || Node <- Nodes]). - % Get the "real" tree depth, not the virtual one. As revtrees gets stemmed they % will keep their virtual depth but the actual number of nodes in the tree % could be reduced. @@ -467,8 +473,8 @@ real_depth([]) -> real_depth(RevTree) when is_list(RevTree) -> lists:max([real_depth(T) || T <- RevTree]); real_depth({_Depth, Node}) when is_tuple(Node) -> - depth(Node, 0). % Note from here on use the depth/3 function - + % Note from here on use the depth/3 function + depth(Node, 0). % Return an ordered list of revtree nodes. When sorting only immediate keys % (revisions) are looked at and comparison doesn't descent into the treee. @@ -476,7 +482,6 @@ real_depth({_Depth, Node}) when is_tuple(Node) -> ordered_nodes(Nodes) -> lists:sort(fun({K1, _, _}, {K2, _, _}) -> K1 =< K2 end, Nodes). - % Calculate a maximum number of rev tree nodes needed for a tree of a given % height and branchiness. Height is derived from Size and LevelReductionFactor, % that is how big the sample should be and quickly the size parameter would @@ -487,20 +492,17 @@ keys_needed(0, _, _) -> keys_needed(Size, LevelReductionFactor, 1) -> expected_height(Size, LevelReductionFactor); keys_needed(Size, LevelReductionFactor, Branches) -> - Height = expected_height(Size, LevelReductionFactor), + Height = expected_height(Size, LevelReductionFactor), trunc(math:pow(Branches, Height + 1)) + 1. - % Calculate expected tree height for a given sample size and branchiness. % At each step the size is divided by the reduction factor. expected_height(Size, LevelReductionFactor) -> trunc(log(LevelReductionFactor, Size)) + 1. - log(B, X) -> math:log(X) / math:log(B). - % Distribute items in a list into roughly equal chunks of a given size. % distribute(_ChunkSize, []) -> @@ -511,7 +513,6 @@ distribute(ChunkSize, L) -> {L1, L2} = lists:split(ChunkSize, L), [L1 | distribute(ChunkSize, L2)]. - % Split a single (parent) revision list into chunks (sub-lists), one for each % child. Also, for safety, double check that at this point in the process the % list of revisions is sufficiently large. If it isn't something went wrong and diff --git a/src/couch/test/eunit/couch_key_tree_tests.erl b/src/couch/test/eunit/couch_key_tree_tests.erl index 5d9cc8372..f571139c9 100644 --- a/src/couch/test/eunit/couch_key_tree_tests.erl +++ b/src/couch/test/eunit/couch_key_tree_tests.erl @@ -16,8 +16,7 @@ -define(DEPTH, 10). - -key_tree_merge_test_()-> +key_tree_merge_test_() -> { "Key tree merge", [ @@ -40,16 +39,16 @@ key_tree_merge_test_()-> ] }. -key_tree_missing_leaves_test_()-> +key_tree_missing_leaves_test_() -> { - "Missing tree leaves", - [ - should_not_find_missing_leaves(), - should_find_missing_leaves() - ] + "Missing tree leaves", + [ + should_not_find_missing_leaves(), + should_find_missing_leaves() + ] }. -key_tree_remove_leaves_test_()-> +key_tree_remove_leaves_test_() -> { "Remove tree leaves", [ @@ -62,7 +61,7 @@ key_tree_remove_leaves_test_()-> ] }. -key_tree_get_leaves_test_()-> +key_tree_get_leaves_test_() -> { "Leaves retrieving", [ @@ -80,7 +79,7 @@ key_tree_get_leaves_test_()-> ] }. -key_tree_leaf_counting_test_()-> +key_tree_leaf_counting_test_() -> { "Leaf counting", [ @@ -91,7 +90,7 @@ key_tree_leaf_counting_test_()-> ] }. -key_tree_stemming_test_()-> +key_tree_stemming_test_() -> { "Stemming", [ @@ -101,48 +100,71 @@ key_tree_stemming_test_()-> ] }. - -should_merge_with_empty_tree()-> - One = {1, {"1","foo",[]}}, - ?_assertEqual({[One], new_leaf}, - merge_and_stem([], One)). - -should_merge_reflexive()-> - One = {1, {"1","foo",[]}}, - ?_assertEqual({[One], internal_node}, - merge_and_stem([One], One)). - -should_merge_prefix_of_a_tree_with_tree()-> - One = {1, {"1","foo",[]}}, - TwoSibs = [{1, {"1","foo",[]}}, - {1, {"2","foo",[]}}], - ?_assertEqual({TwoSibs, internal_node}, - merge_and_stem(TwoSibs, One)). - -should_produce_conflict_on_merge_with_unrelated_branch()-> - TwoSibs = [{1, {"1","foo",[]}}, - {1, {"2","foo",[]}}], - Three = {1, {"3","foo",[]}}, - ThreeSibs = [{1, {"1","foo",[]}}, - {1, {"2","foo",[]}}, - {1, {"3","foo",[]}}], - ?_assertEqual({ThreeSibs, new_branch}, - merge_and_stem(TwoSibs, Three)). - -should_merge_reflexive_for_child_nodes()-> - TwoChild = {1, {"1","foo", [{"1a", "bar", [{"1aa", "bar", []}]}]}}, - ?_assertEqual({[TwoChild], internal_node}, - merge_and_stem([TwoChild], TwoChild)). - -should_merge_tree_to_itself()-> - TwoChildSibs = {1, {"1","foo", [{"1a", "bar", []}, - {"1b", "bar", []}]}}, +should_merge_with_empty_tree() -> + One = {1, {"1", "foo", []}}, + ?_assertEqual( + {[One], new_leaf}, + merge_and_stem([], One) + ). + +should_merge_reflexive() -> + One = {1, {"1", "foo", []}}, + ?_assertEqual( + {[One], internal_node}, + merge_and_stem([One], One) + ). + +should_merge_prefix_of_a_tree_with_tree() -> + One = {1, {"1", "foo", []}}, + TwoSibs = [ + {1, {"1", "foo", []}}, + {1, {"2", "foo", []}} + ], + ?_assertEqual( + {TwoSibs, internal_node}, + merge_and_stem(TwoSibs, One) + ). + +should_produce_conflict_on_merge_with_unrelated_branch() -> + TwoSibs = [ + {1, {"1", "foo", []}}, + {1, {"2", "foo", []}} + ], + Three = {1, {"3", "foo", []}}, + ThreeSibs = [ + {1, {"1", "foo", []}}, + {1, {"2", "foo", []}}, + {1, {"3", "foo", []}} + ], + ?_assertEqual( + {ThreeSibs, new_branch}, + merge_and_stem(TwoSibs, Three) + ). + +should_merge_reflexive_for_child_nodes() -> + TwoChild = {1, {"1", "foo", [{"1a", "bar", [{"1aa", "bar", []}]}]}}, + ?_assertEqual( + {[TwoChild], internal_node}, + merge_and_stem([TwoChild], TwoChild) + ). + +should_merge_tree_to_itself() -> + TwoChildSibs = + {1, + {"1", "foo", [ + {"1a", "bar", []}, + {"1b", "bar", []} + ]}}, Leafs = couch_key_tree:get_all_leafs([TwoChildSibs]), Paths = lists:map(fun leaf_to_path/1, Leafs), - FinalTree = lists:foldl(fun(Path, TreeAcc) -> - {NewTree, internal_node} = merge_and_stem(TreeAcc, Path), - NewTree - end, [TwoChildSibs], Paths), + FinalTree = lists:foldl( + fun(Path, TreeAcc) -> + {NewTree, internal_node} = merge_and_stem(TreeAcc, Path), + NewTree + end, + [TwoChildSibs], + Paths + ), ?_assertEqual([TwoChildSibs], FinalTree). leaf_to_path({Value, {Start, Keys}}) -> @@ -154,260 +176,355 @@ to_branch(Value, [Key]) -> to_branch(Value, [Key | RestKeys]) -> [{Key, [], to_branch(Value, RestKeys)}]. - -should_merge_tree_of_odd_length()-> - TwoChild = {1, {"1","foo", [{"1a", "bar", [{"1aa", "bar", []}]}]}}, - TwoChildSibs = {1, {"1","foo", [{"1a", "bar", []}, - {"1b", "bar", []}]}}, - TwoChildPlusSibs = {1, {"1","foo", [{"1a", "bar", [{"1aa", "bar", []}]}, - {"1b", "bar", []}]}}, - ?_assertEqual({[TwoChildPlusSibs], new_leaf}, - merge_and_stem([TwoChildSibs], TwoChild)). - -should_merge_tree_with_stem()-> +should_merge_tree_of_odd_length() -> + TwoChild = {1, {"1", "foo", [{"1a", "bar", [{"1aa", "bar", []}]}]}}, + TwoChildSibs = + {1, + {"1", "foo", [ + {"1a", "bar", []}, + {"1b", "bar", []} + ]}}, + TwoChildPlusSibs = + {1, + {"1", "foo", [ + {"1a", "bar", [{"1aa", "bar", []}]}, + {"1b", "bar", []} + ]}}, + ?_assertEqual( + {[TwoChildPlusSibs], new_leaf}, + merge_and_stem([TwoChildSibs], TwoChild) + ). + +should_merge_tree_with_stem() -> Stemmed = {2, {"1a", "bar", []}}, - TwoChildSibs = {1, {"1","foo", [{"1a", "bar", []}, - {"1b", "bar", []}]}}, - - ?_assertEqual({[TwoChildSibs], internal_node}, - merge_and_stem([TwoChildSibs], Stemmed)). - -should_merge_with_stem_at_deeper_level()-> + TwoChildSibs = + {1, + {"1", "foo", [ + {"1a", "bar", []}, + {"1b", "bar", []} + ]}}, + + ?_assertEqual( + {[TwoChildSibs], internal_node}, + merge_and_stem([TwoChildSibs], Stemmed) + ). + +should_merge_with_stem_at_deeper_level() -> Stemmed = {3, {"1bb", "boo", []}}, - TwoChildSibs = {1, {"1","foo", [{"1a", "bar", []}, - {"1b", "bar", [{"1bb", "boo", []}]}]}}, - ?_assertEqual({[TwoChildSibs], internal_node}, - merge_and_stem([TwoChildSibs], Stemmed)). - -should_merge_with_stem_at_deeper_level_with_deeper_paths()-> + TwoChildSibs = + {1, + {"1", "foo", [ + {"1a", "bar", []}, + {"1b", "bar", [{"1bb", "boo", []}]} + ]}}, + ?_assertEqual( + {[TwoChildSibs], internal_node}, + merge_and_stem([TwoChildSibs], Stemmed) + ). + +should_merge_with_stem_at_deeper_level_with_deeper_paths() -> Stemmed = {3, {"1bb", "boo", []}}, - StemmedTwoChildSibs = [{2,{"1a", "bar", []}}, - {2,{"1b", "bar", [{"1bb", "boo", []}]}}], - ?_assertEqual({StemmedTwoChildSibs, internal_node}, - merge_and_stem(StemmedTwoChildSibs, Stemmed)). - -should_merge_single_tree_with_deeper_stem()-> + StemmedTwoChildSibs = [ + {2, {"1a", "bar", []}}, + {2, {"1b", "bar", [{"1bb", "boo", []}]}} + ], + ?_assertEqual( + {StemmedTwoChildSibs, internal_node}, + merge_and_stem(StemmedTwoChildSibs, Stemmed) + ). + +should_merge_single_tree_with_deeper_stem() -> Stemmed = {3, {"1aa", "bar", []}}, - TwoChild = {1, {"1","foo", [{"1a", "bar", [{"1aa", "bar", []}]}]}}, - ?_assertEqual({[TwoChild], internal_node}, - merge_and_stem([TwoChild], Stemmed)). + TwoChild = {1, {"1", "foo", [{"1a", "bar", [{"1aa", "bar", []}]}]}}, + ?_assertEqual( + {[TwoChild], internal_node}, + merge_and_stem([TwoChild], Stemmed) + ). -should_merge_tree_with_large_stem()-> +should_merge_tree_with_large_stem() -> Stemmed = {2, {"1a", "bar", [{"1aa", "bar", []}]}}, - TwoChild = {1, {"1","foo", [{"1a", "bar", [{"1aa", "bar", []}]}]}}, - ?_assertEqual({[TwoChild], internal_node}, - merge_and_stem([TwoChild], Stemmed)). + TwoChild = {1, {"1", "foo", [{"1a", "bar", [{"1aa", "bar", []}]}]}}, + ?_assertEqual( + {[TwoChild], internal_node}, + merge_and_stem([TwoChild], Stemmed) + ). -should_merge_stems()-> +should_merge_stems() -> StemmedA = {2, {"1a", "bar", [{"1aa", "bar", []}]}}, StemmedB = {3, {"1aa", "bar", []}}, - ?_assertEqual({[StemmedA], internal_node}, - merge_and_stem([StemmedA], StemmedB)). + ?_assertEqual( + {[StemmedA], internal_node}, + merge_and_stem([StemmedA], StemmedB) + ). -should_create_conflicts_on_merge()-> - OneChild = {1, {"1","foo",[{"1a", "bar", []}]}}, +should_create_conflicts_on_merge() -> + OneChild = {1, {"1", "foo", [{"1a", "bar", []}]}}, Stemmed = {3, {"1aa", "bar", []}}, - ?_assertEqual({[OneChild, Stemmed], new_branch}, - merge_and_stem([OneChild], Stemmed)). + ?_assertEqual( + {[OneChild, Stemmed], new_branch}, + merge_and_stem([OneChild], Stemmed) + ). -should_create_no_conflicts_on_merge()-> - OneChild = {1, {"1","foo",[{"1a", "bar", []}]}}, +should_create_no_conflicts_on_merge() -> + OneChild = {1, {"1", "foo", [{"1a", "bar", []}]}}, Stemmed = {3, {"1aa", "bar", []}}, - TwoChild = {1, {"1","foo", [{"1a", "bar", [{"1aa", "bar", []}]}]}}, - ?_assertEqual({[TwoChild], new_leaf}, - merge_and_stem([OneChild, Stemmed], TwoChild)). + TwoChild = {1, {"1", "foo", [{"1a", "bar", [{"1aa", "bar", []}]}]}}, + ?_assertEqual( + {[TwoChild], new_leaf}, + merge_and_stem([OneChild, Stemmed], TwoChild) + ). -should_ignore_conflicting_branch()-> +should_ignore_conflicting_branch() -> %% this test is based on couch-902-test-case2.py %% foo has conflicts from replication at depth two %% foo3 is the current value - Foo = {1, {"foo", - "val1", - [{"foo2","val2",[]}, + Foo = + {1, + {"foo", "val1", [ + {"foo2", "val2", []}, {"foo3", "val3", []} - ]}}, + ]}}, %% foo now has an attachment added, which leads to foo4 and val4 %% off foo3 - Bar = {1, {"foo", - [], - [{"foo3", - [], - [{"foo4","val4",[]} - ]}]}}, + Bar = {1, {"foo", [], [{"foo3", [], [{"foo4", "val4", []}]}]}}, %% this is what the merge returns %% note that it ignore the conflicting branch as there's no match - FooBar = {1, {"foo", - "val1", - [{"foo2","val2",[]}, - {"foo3", "val3", [{"foo4","val4",[]}]} - ]}}, + FooBar = + {1, + {"foo", "val1", [ + {"foo2", "val2", []}, + {"foo3", "val3", [{"foo4", "val4", []}]} + ]}}, { "COUCHDB-902", - ?_assertEqual({[FooBar], new_leaf}, - merge_and_stem([Foo], Bar)) + ?_assertEqual( + {[FooBar], new_leaf}, + merge_and_stem([Foo], Bar) + ) }. -should_not_find_missing_leaves()-> - TwoChildSibs = [{0, {"1","foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], - ?_assertEqual([], - couch_key_tree:find_missing(TwoChildSibs, - [{0,"1"}, {1,"1a"}])). - -should_find_missing_leaves()-> +should_not_find_missing_leaves() -> + TwoChildSibs = [{0, {"1", "foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], + ?_assertEqual( + [], + couch_key_tree:find_missing( + TwoChildSibs, + [{0, "1"}, {1, "1a"}] + ) + ). + +should_find_missing_leaves() -> Stemmed1 = [{1, {"1a", "bar", [{"1aa", "bar", []}]}}], Stemmed2 = [{2, {"1aa", "bar", []}}], - TwoChildSibs = [{0, {"1","foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], + TwoChildSibs = [{0, {"1", "foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], [ ?_assertEqual( [{0, "10"}, {100, "x"}], couch_key_tree:find_missing( TwoChildSibs, - [{0,"1"}, {0, "10"}, {1,"1a"}, {100, "x"}])), + [{0, "1"}, {0, "10"}, {1, "1a"}, {100, "x"}] + ) + ), ?_assertEqual( [{0, "1"}, {100, "x"}], couch_key_tree:find_missing( Stemmed1, - [{0,"1"}, {1,"1a"}, {100, "x"}])), + [{0, "1"}, {1, "1a"}, {100, "x"}] + ) + ), ?_assertEqual( - [{0, "1"}, {1,"1a"}, {100, "x"}], + [{0, "1"}, {1, "1a"}, {100, "x"}], couch_key_tree:find_missing( Stemmed2, - [{0,"1"}, {1,"1a"}, {100, "x"}])) + [{0, "1"}, {1, "1a"}, {100, "x"}] + ) + ) ]. -should_have_no_effect_on_removing_no_leaves()-> - TwoChildSibs = [{0, {"1","foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], - ?_assertEqual({TwoChildSibs, []}, - couch_key_tree:remove_leafs(TwoChildSibs, - [])). - -should_have_no_effect_on_removing_non_existant_branch()-> - TwoChildSibs = [{0, {"1","foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], - ?_assertEqual({TwoChildSibs, []}, - couch_key_tree:remove_leafs(TwoChildSibs, - [{0, "1"}])). - -should_remove_leaf()-> - OneChild = [{0, {"1","foo",[{"1a", "bar", []}]}}], - TwoChildSibs = [{0, {"1","foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], - ?_assertEqual({OneChild, [{1, "1b"}]}, - couch_key_tree:remove_leafs(TwoChildSibs, - [{1, "1b"}])). - -should_produce_empty_tree_on_removing_all_leaves()-> - TwoChildSibs = [{0, {"1","foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], - ?_assertEqual({[], [{1, "1b"}, {1, "1a"}]}, - couch_key_tree:remove_leafs(TwoChildSibs, - [{1, "1b"}, {1, "1a"}])). - -should_have_no_effect_on_removing_non_existant_node()-> +should_have_no_effect_on_removing_no_leaves() -> + TwoChildSibs = [{0, {"1", "foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], + ?_assertEqual( + {TwoChildSibs, []}, + couch_key_tree:remove_leafs( + TwoChildSibs, + [] + ) + ). + +should_have_no_effect_on_removing_non_existant_branch() -> + TwoChildSibs = [{0, {"1", "foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], + ?_assertEqual( + {TwoChildSibs, []}, + couch_key_tree:remove_leafs( + TwoChildSibs, + [{0, "1"}] + ) + ). + +should_remove_leaf() -> + OneChild = [{0, {"1", "foo", [{"1a", "bar", []}]}}], + TwoChildSibs = [{0, {"1", "foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], + ?_assertEqual( + {OneChild, [{1, "1b"}]}, + couch_key_tree:remove_leafs( + TwoChildSibs, + [{1, "1b"}] + ) + ). + +should_produce_empty_tree_on_removing_all_leaves() -> + TwoChildSibs = [{0, {"1", "foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], + ?_assertEqual( + {[], [{1, "1b"}, {1, "1a"}]}, + couch_key_tree:remove_leafs( + TwoChildSibs, + [{1, "1b"}, {1, "1a"}] + ) + ). + +should_have_no_effect_on_removing_non_existant_node() -> Stemmed = [{1, {"1a", "bar", [{"1aa", "bar", []}]}}], - ?_assertEqual({Stemmed, []}, - couch_key_tree:remove_leafs(Stemmed, - [{1, "1a"}])). - -should_produce_empty_tree_on_removing_last_leaf()-> + ?_assertEqual( + {Stemmed, []}, + couch_key_tree:remove_leafs( + Stemmed, + [{1, "1a"}] + ) + ). + +should_produce_empty_tree_on_removing_last_leaf() -> Stemmed = [{1, {"1a", "bar", [{"1aa", "bar", []}]}}], - ?_assertEqual({[], [{2, "1aa"}]}, - couch_key_tree:remove_leafs(Stemmed, - [{2, "1aa"}])). - -should_extract_subtree()-> - TwoChildSibs = [{0, {"1","foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], - ?_assertEqual({[{"foo", {0, ["1"]}}],[]}, - couch_key_tree:get(TwoChildSibs, [{0, "1"}])). - -should_extract_subsubtree()-> - TwoChildSibs = [{0, {"1","foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], - ?_assertEqual({[{"bar", {1, ["1a", "1"]}}],[]}, - couch_key_tree:get(TwoChildSibs, [{1, "1a"}])). - -should_gather_non_existant_leaf()-> - TwoChildSibs = [{0, {"1","foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], - ?_assertEqual({[],[{0, "x"}]}, - couch_key_tree:get_key_leafs(TwoChildSibs, [{0, "x"}])). - -should_gather_leaf()-> - TwoChildSibs = [{0, {"1","foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], - ?_assertEqual({[{"bar", {1, ["1a","1"]}}],[]}, - couch_key_tree:get_key_leafs(TwoChildSibs, [{1, "1a"}])). - -shoul_gather_multiple_leaves()-> - TwoChildSibs = [{0, {"1","foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], - ?_assertEqual({[{"bar", {1, ["1a","1"]}},{"bar",{1, ["1b","1"]}}],[]}, - couch_key_tree:get_key_leafs(TwoChildSibs, [{0, "1"}])). + ?_assertEqual( + {[], [{2, "1aa"}]}, + couch_key_tree:remove_leafs( + Stemmed, + [{2, "1aa"}] + ) + ). + +should_extract_subtree() -> + TwoChildSibs = [{0, {"1", "foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], + ?_assertEqual( + {[{"foo", {0, ["1"]}}], []}, + couch_key_tree:get(TwoChildSibs, [{0, "1"}]) + ). + +should_extract_subsubtree() -> + TwoChildSibs = [{0, {"1", "foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], + ?_assertEqual( + {[{"bar", {1, ["1a", "1"]}}], []}, + couch_key_tree:get(TwoChildSibs, [{1, "1a"}]) + ). + +should_gather_non_existant_leaf() -> + TwoChildSibs = [{0, {"1", "foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], + ?_assertEqual( + {[], [{0, "x"}]}, + couch_key_tree:get_key_leafs(TwoChildSibs, [{0, "x"}]) + ). + +should_gather_leaf() -> + TwoChildSibs = [{0, {"1", "foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], + ?_assertEqual( + {[{"bar", {1, ["1a", "1"]}}], []}, + couch_key_tree:get_key_leafs(TwoChildSibs, [{1, "1a"}]) + ). + +shoul_gather_multiple_leaves() -> + TwoChildSibs = [{0, {"1", "foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], + ?_assertEqual( + {[{"bar", {1, ["1a", "1"]}}, {"bar", {1, ["1b", "1"]}}], []}, + couch_key_tree:get_key_leafs(TwoChildSibs, [{0, "1"}]) + ). should_gather_single_leaf_for_multiple_revs() -> - OneChild = [{0, {"1","foo",[{"1a", "bar", []}]}}], + OneChild = [{0, {"1", "foo", [{"1a", "bar", []}]}}], ToFind = [{0, "1"}, {1, "1a"}], - ?_assertEqual({[{"bar", {1, ["1a", "1"]}}],[]}, - couch_key_tree:get_key_leafs(OneChild, ToFind)). + ?_assertEqual( + {[{"bar", {1, ["1a", "1"]}}], []}, + couch_key_tree:get_key_leafs(OneChild, ToFind) + ). should_gather_multiple_for_multiple_revs() -> - TwoChildSibs = [{0, {"1","foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], + TwoChildSibs = [{0, {"1", "foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], ToFind = [{0, "1"}, {1, "1a"}], - ?_assertEqual({[{"bar", {1, ["1a","1"]}},{"bar",{1, ["1b","1"]}}],[]}, - couch_key_tree:get_key_leafs(TwoChildSibs, ToFind)). - -should_retrieve_full_key_path()-> - TwoChildSibs = [{0, {"1","foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], - ?_assertEqual({[{0,[{"1", "foo"}]}],[]}, - couch_key_tree:get_full_key_paths(TwoChildSibs, [{0, "1"}])). - -should_retrieve_full_key_path_for_node()-> - TwoChildSibs = [{0, {"1","foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], - ?_assertEqual({[{1,[{"1a", "bar"},{"1", "foo"}]}],[]}, - couch_key_tree:get_full_key_paths(TwoChildSibs, [{1, "1a"}])). - -should_retrieve_leaves_with_parent_node()-> + ?_assertEqual( + {[{"bar", {1, ["1a", "1"]}}, {"bar", {1, ["1b", "1"]}}], []}, + couch_key_tree:get_key_leafs(TwoChildSibs, ToFind) + ). + +should_retrieve_full_key_path() -> + TwoChildSibs = [{0, {"1", "foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], + ?_assertEqual( + {[{0, [{"1", "foo"}]}], []}, + couch_key_tree:get_full_key_paths(TwoChildSibs, [{0, "1"}]) + ). + +should_retrieve_full_key_path_for_node() -> + TwoChildSibs = [{0, {"1", "foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], + ?_assertEqual( + {[{1, [{"1a", "bar"}, {"1", "foo"}]}], []}, + couch_key_tree:get_full_key_paths(TwoChildSibs, [{1, "1a"}]) + ). + +should_retrieve_leaves_with_parent_node() -> Stemmed = [{1, {"1a", "bar", [{"1aa", "bar", []}]}}], - TwoChildSibs = [{0, {"1","foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], + TwoChildSibs = [{0, {"1", "foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], [ - ?_assertEqual([{2, [{"1aa", "bar"},{"1a", "bar"}]}], - couch_key_tree:get_all_leafs_full(Stemmed)), - ?_assertEqual([{1, [{"1a", "bar"},{"1", "foo"}]}, - {1, [{"1b", "bar"},{"1", "foo"}]}], - couch_key_tree:get_all_leafs_full(TwoChildSibs)) + ?_assertEqual( + [{2, [{"1aa", "bar"}, {"1a", "bar"}]}], + couch_key_tree:get_all_leafs_full(Stemmed) + ), + ?_assertEqual( + [ + {1, [{"1a", "bar"}, {"1", "foo"}]}, + {1, [{"1b", "bar"}, {"1", "foo"}]} + ], + couch_key_tree:get_all_leafs_full(TwoChildSibs) + ) ]. -should_retrieve_all_leaves()-> +should_retrieve_all_leaves() -> Stemmed = [{1, {"1a", "bar", [{"1aa", "bar", []}]}}], - TwoChildSibs = [{0, {"1","foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], + TwoChildSibs = [{0, {"1", "foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], [ - ?_assertEqual([{"bar", {2, ["1aa","1a"]}}], - couch_key_tree:get_all_leafs(Stemmed)), - ?_assertEqual([{"bar", {1, ["1a", "1"]}}, {"bar", {1, ["1b","1"]}}], - couch_key_tree:get_all_leafs(TwoChildSibs)) + ?_assertEqual( + [{"bar", {2, ["1aa", "1a"]}}], + couch_key_tree:get_all_leafs(Stemmed) + ), + ?_assertEqual( + [{"bar", {1, ["1a", "1"]}}, {"bar", {1, ["1b", "1"]}}], + couch_key_tree:get_all_leafs(TwoChildSibs) + ) ]. -should_have_no_leaves_for_empty_tree()-> +should_have_no_leaves_for_empty_tree() -> ?_assertEqual(0, couch_key_tree:count_leafs([])). -should_have_single_leaf_for_tree_with_single_node()-> - ?_assertEqual(1, couch_key_tree:count_leafs([{0, {"1","foo",[]}}])). +should_have_single_leaf_for_tree_with_single_node() -> + ?_assertEqual(1, couch_key_tree:count_leafs([{0, {"1", "foo", []}}])). -should_have_two_leaves_for_tree_with_chindler_siblings()-> - TwoChildSibs = [{0, {"1","foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], +should_have_two_leaves_for_tree_with_chindler_siblings() -> + TwoChildSibs = [{0, {"1", "foo", [{"1a", "bar", []}, {"1b", "bar", []}]}}], ?_assertEqual(2, couch_key_tree:count_leafs(TwoChildSibs)). -should_not_affect_on_leaf_counting_for_stemmed_tree()-> +should_not_affect_on_leaf_counting_for_stemmed_tree() -> ?_assertEqual(1, couch_key_tree:count_leafs([{2, {"1bb", "boo", []}}])). -should_have_no_effect_for_stemming_more_levels_than_exists()-> - TwoChild = [{0, {"1","foo", [{"1a", "bar", [{"1aa", "bar", []}]}]}}], +should_have_no_effect_for_stemming_more_levels_than_exists() -> + TwoChild = [{0, {"1", "foo", [{"1a", "bar", [{"1aa", "bar", []}]}]}}], ?_assertEqual(TwoChild, couch_key_tree:stem(TwoChild, 3)). -should_return_one_deepest_node()-> - TwoChild = [{0, {"1","foo", [{"1a", "bar", [{"1aa", "bar", []}]}]}}], +should_return_one_deepest_node() -> + TwoChild = [{0, {"1", "foo", [{"1a", "bar", [{"1aa", "bar", []}]}]}}], Stemmed = [{2, {"1aa", "bar", []}}], ?_assertEqual(Stemmed, couch_key_tree:stem(TwoChild, 1)). -should_return_two_deepest_nodes()-> - TwoChild = [{0, {"1","foo", [{"1a", "bar", [{"1aa", "bar", []}]}]}}], +should_return_two_deepest_nodes() -> + TwoChild = [{0, {"1", "foo", [{"1a", "bar", [{"1aa", "bar", []}]}]}}], Stemmed = [{1, {"1a", "bar", [{"1aa", "bar", []}]}}], ?_assertEqual(Stemmed, couch_key_tree:stem(TwoChild, 2)). - merge_and_stem(RevTree, Tree) -> {Merged, Result} = couch_key_tree:merge(RevTree, Tree), {couch_key_tree:stem(Merged, ?DEPTH), Result}. diff --git a/src/couch/test/eunit/couch_passwords_tests.erl b/src/couch/test/eunit/couch_passwords_tests.erl index 88de8530f..6b67a99e3 100644 --- a/src/couch/test/eunit/couch_passwords_tests.erl +++ b/src/couch/test/eunit/couch_passwords_tests.erl @@ -14,41 +14,52 @@ -include_lib("couch/include/couch_eunit.hrl"). +pbkdf2_test_() -> + {"PBKDF2", [ + {"Iterations: 1, length: 20", + ?_assertEqual( + {ok, <<"0c60c80f961f0e71f3a9b524af6012062fe037a6">>}, + couch_passwords:pbkdf2(<<"password">>, <<"salt">>, 1, 20) + )}, -pbkdf2_test_()-> - {"PBKDF2", - [ - {"Iterations: 1, length: 20", - ?_assertEqual( - {ok, <<"0c60c80f961f0e71f3a9b524af6012062fe037a6">>}, - couch_passwords:pbkdf2(<<"password">>, <<"salt">>, 1, 20))}, + {"Iterations: 2, length: 20", + ?_assertEqual( + {ok, <<"ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957">>}, + couch_passwords:pbkdf2(<<"password">>, <<"salt">>, 2, 20) + )}, - {"Iterations: 2, length: 20", - ?_assertEqual( - {ok, <<"ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957">>}, - couch_passwords:pbkdf2(<<"password">>, <<"salt">>, 2, 20))}, + {"Iterations: 4096, length: 20", + ?_assertEqual( + {ok, <<"4b007901b765489abead49d926f721d065a429c1">>}, + couch_passwords:pbkdf2(<<"password">>, <<"salt">>, 4096, 20) + )}, - {"Iterations: 4096, length: 20", - ?_assertEqual( - {ok, <<"4b007901b765489abead49d926f721d065a429c1">>}, - couch_passwords:pbkdf2(<<"password">>, <<"salt">>, 4096, 20))}, + {"Iterations: 4096, length: 25", + ?_assertEqual( + {ok, <<"3d2eec4fe41c849b80c8d83662c0e44a8b291a964cf2f07038">>}, + couch_passwords:pbkdf2( + <<"passwordPASSWORDpassword">>, + <<"saltSALTsaltSALTsaltSALTsaltSALTsalt">>, + 4096, + 25 + ) + )}, + {"Null byte", + ?_assertEqual( + {ok, <<"56fa6aa75548099dcc37d7f03425e0c3">>}, + couch_passwords:pbkdf2( + <<"pass\0word">>, + <<"sa\0lt">>, + 4096, + 16 + ) + )}, - {"Iterations: 4096, length: 25", - ?_assertEqual( - {ok, <<"3d2eec4fe41c849b80c8d83662c0e44a8b291a964cf2f07038">>}, - couch_passwords:pbkdf2(<<"passwordPASSWORDpassword">>, - <<"saltSALTsaltSALTsaltSALTsaltSALTsalt">>, - 4096, 25))}, - {"Null byte", - ?_assertEqual( - {ok, <<"56fa6aa75548099dcc37d7f03425e0c3">>}, - couch_passwords:pbkdf2(<<"pass\0word">>, - <<"sa\0lt">>, - 4096, 16))}, - - {timeout, 600, %% this may runs too long on slow hosts - {"Iterations: 16777216 - this may take some time", - ?_assertEqual( - {ok, <<"eefe3d61cd4da4e4e9945b3d6ba2158c2634e984">>}, - couch_passwords:pbkdf2(<<"password">>, <<"salt">>, 16777216, 20) - )}}]}. + %% this may runs too long on slow hosts + {timeout, 600, + {"Iterations: 16777216 - this may take some time", + ?_assertEqual( + {ok, <<"eefe3d61cd4da4e4e9945b3d6ba2158c2634e984">>}, + couch_passwords:pbkdf2(<<"password">>, <<"salt">>, 16777216, 20) + )}} + ]}. diff --git a/src/couch/test/eunit/couch_query_servers_tests.erl b/src/couch/test/eunit/couch_query_servers_tests.erl index 440fc8e1b..01631ba28 100644 --- a/src/couch/test/eunit/couch_query_servers_tests.erl +++ b/src/couch/test/eunit/couch_query_servers_tests.erl @@ -15,24 +15,19 @@ -include_lib("couch/include/couch_db.hrl"). -include_lib("couch/include/couch_eunit.hrl"). - setup() -> meck:new([config, couch_log]). - teardown(_) -> meck:unload(). - setup_oom() -> test_util:start_couch([ioq]). - teardown_oom(Ctx) -> meck:unload(), test_util:stop_couch(Ctx). - sum_overflow_test_() -> { "Test overflow detection in the _sum reduce function", @@ -48,26 +43,27 @@ sum_overflow_test_() -> } }. - filter_oom_test_() -> -{ - "Test recovery from oom in filters", { - setup, - fun setup_oom/0, - fun teardown_oom/1, - [ - fun should_split_large_batches/0 - ] - } -}. + "Test recovery from oom in filters", + { + setup, + fun setup_oom/0, + fun teardown_oom/1, + [ + fun should_split_large_batches/0 + ] + } + }. should_return_error_on_overflow() -> meck:reset([config, couch_log]), meck:expect( - config, get, ["query_server_config", "reduce_limit", "true"], - "true" - ), + config, + get, + ["query_server_config", "reduce_limit", "true"], + "true" + ), meck:expect(couch_log, error, ['_', '_'], ok), KVs = gen_sum_kvs(), {ok, [Result]} = couch_query_servers:reduce(<<"foo">>, [<<"_sum">>], KVs), @@ -75,13 +71,14 @@ should_return_error_on_overflow() -> ?assert(meck:called(config, get, '_')), ?assert(meck:called(couch_log, error, '_')). - should_return_object_on_log() -> meck:reset([config, couch_log]), meck:expect( - config, get, ["query_server_config", "reduce_limit", "true"], - "log" - ), + config, + get, + ["query_server_config", "reduce_limit", "true"], + "log" + ), meck:expect(couch_log, error, ['_', '_'], ok), KVs = gen_sum_kvs(), {ok, [Result]} = couch_query_servers:reduce(<<"foo">>, [<<"_sum">>], KVs), @@ -91,13 +88,14 @@ should_return_object_on_log() -> ?assert(meck:called(config, get, '_')), ?assert(meck:called(couch_log, error, '_')). - should_return_object_on_false() -> meck:reset([config, couch_log]), meck:expect( - config, get, ["query_server_config", "reduce_limit", "true"], - "false" - ), + config, + get, + ["query_server_config", "reduce_limit", "true"], + "false" + ), meck:expect(couch_log, error, ['_', '_'], ok), KVs = gen_sum_kvs(), {ok, [Result]} = couch_query_servers:reduce(<<"foo">>, [<<"_sum">>], KVs), @@ -107,18 +105,19 @@ should_return_object_on_false() -> ?assert(meck:called(config, get, '_')), ?assertNot(meck:called(couch_log, error, '_')). - should_split_large_batches() -> Req = {json_req, {[]}}, Db = undefined, DDoc = #doc{ id = <<"_design/foo">>, revs = {0, [<<"bork bork bork">>]}, - body = {[ - {<<"filters">>, {[ - {<<"bar">>, <<"function(req, doc) {return true;}">>} - ]}} - ]} + body = + {[ + {<<"filters">>, + {[ + {<<"bar">>, <<"function(req, doc) {return true;}">>} + ]}} + ]} }, FName = <<"bar">>, Docs = [ @@ -139,12 +138,17 @@ should_split_large_batches() -> {ok, Ret} = couch_query_servers:filter_docs(Req, Db, DDoc, FName, Docs), ?assertEqual([split_batch, split_batch], Ret). - gen_sum_kvs() -> - lists:map(fun(I) -> - Props = lists:map(fun(_) -> - K = couch_util:encodeBase64Url(crypto:strong_rand_bytes(16)), - {K, 1} - end, lists:seq(1, 20)), - [I, {Props}] - end, lists:seq(1, 10)). + lists:map( + fun(I) -> + Props = lists:map( + fun(_) -> + K = couch_util:encodeBase64Url(crypto:strong_rand_bytes(16)), + {K, 1} + end, + lists:seq(1, 20) + ), + [I, {Props}] + end, + lists:seq(1, 10) + ). diff --git a/src/couch/test/eunit/couch_server_tests.erl b/src/couch/test/eunit/couch_server_tests.erl index 66533d48c..a43106d89 100644 --- a/src/couch/test/eunit/couch_server_tests.erl +++ b/src/couch/test/eunit/couch_server_tests.erl @@ -45,13 +45,13 @@ teardown(rename, Db) -> teardown(_, Db) -> teardown(Db). - delete_db_test_() -> { "Test for proper deletion of db file", { setup, - fun start/0, fun test_util:stop/1, + fun start/0, + fun test_util:stop/1, [ make_test_case(rename, [fun should_rename_on_delete/2]), make_test_case(delete, [fun should_delete/2]) @@ -76,7 +76,8 @@ should_rename_on_delete(_, Db) -> ?assertMatch([_], DeletedFiles), [Renamed] = DeletedFiles, ?assertEqual( - filename:extension(Origin), filename:extension(Renamed)), + filename:extension(Origin), filename:extension(Renamed) + ), ?assert(filelib:is_regular(Renamed)) end). @@ -93,7 +94,6 @@ should_delete(_, Db) -> deleted_files(ViewFile) -> filelib:wildcard(filename:rootname(ViewFile) ++ "*.deleted.*"). - bad_engine_option_test_() -> { setup, @@ -104,19 +104,19 @@ bad_engine_option_test_() -> ] }. - t_bad_engine_option() -> Resp = couch_server:create(?tempdb(), [{engine, <<"cowabunga!">>}]), ?assertEqual(Resp, {error, {invalid_engine_extension, <<"cowabunga!">>}}). - get_engine_path_test_() -> { setup, - fun start/0, fun test_util:stop/1, + fun start/0, + fun test_util:stop/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_return_engine_path/1, fun should_return_invalid_engine_error/1 @@ -124,7 +124,6 @@ get_engine_path_test_() -> } }. - should_return_engine_path(Db) -> DbName = couch_db:name(Db), Engine = couch_db_engine:get_engine(Db), @@ -132,14 +131,12 @@ should_return_engine_path(Db) -> FilePath = couch_db:get_filepath(Db), ?_assertMatch({ok, FilePath}, Resp). - should_return_invalid_engine_error(Db) -> DbName = couch_db:name(Db), Engine = fake_engine, Resp = couch_server:get_engine_path(DbName, Engine), ?_assertMatch({error, {invalid_engine, Engine}}, Resp). - interleaved_requests_test_() -> { setup, @@ -148,7 +145,6 @@ interleaved_requests_test_() -> fun make_interleaved_requests/1 }. - start_interleaved() -> TestDbName = ?tempdb(), meck:new(couch_db, [passthrough]), @@ -180,19 +176,16 @@ start_interleaved() -> end), {test_util:start_couch(), TestDbName}. - stop_interleaved({Ctx, TestDbName}) -> couch_server:delete(TestDbName, [?ADMIN_CTX]), meck:unload(), test_util:stop_couch(Ctx). - make_interleaved_requests({_, TestDbName}) -> [ fun() -> t_interleaved_create_delete_open(TestDbName) end ]. - t_interleaved_create_delete_open(DbName) -> {CrtRef, OpenRef} = {make_ref(), make_ref()}, CrtMsg = {'$gen_call', {self(), CrtRef}, {create, DbName, [?ADMIN_CTX]}}, @@ -247,7 +240,6 @@ t_interleaved_create_delete_open(DbName) -> ?assert(is_process_alive(CouchServer)), check_monitor_not_triggered(CSRef). - get_opener_pid(DbName) -> WaitFun = fun() -> case ets:lookup(couch_server:couch_dbs(DbName), DbName) of @@ -259,23 +251,28 @@ get_opener_pid(DbName) -> end, test_util:wait(WaitFun). - wait_for_open_async_result(CouchServer, Opener) -> WaitFun = fun() -> {_, Messages} = erlang:process_info(CouchServer, messages), - Found = lists:foldl(fun(Msg, Acc) -> - case Msg of - {'$gen_call', {Opener, _}, {open_result, _, {ok, _}}} -> - true; - _ -> - Acc - end - end, false, Messages), - if Found -> ok; true -> wait end + Found = lists:foldl( + fun(Msg, Acc) -> + case Msg of + {'$gen_call', {Opener, _}, {open_result, _, {ok, _}}} -> + true; + _ -> + Acc + end + end, + false, + Messages + ), + if + Found -> ok; + true -> wait + end end, test_util:wait(WaitFun). - check_monitor_not_triggered(Ref) -> receive {'DOWN', Ref, _, _, Reason0} -> @@ -284,7 +281,6 @@ check_monitor_not_triggered(Ref) -> ok end. - get_next_message() -> receive Msg -> diff --git a/src/couch/test/eunit/couch_stream_tests.erl b/src/couch/test/eunit/couch_stream_tests.erl index a7fedf0af..4146a9139 100644 --- a/src/couch/test/eunit/couch_stream_tests.erl +++ b/src/couch/test/eunit/couch_stream_tests.erl @@ -24,16 +24,17 @@ setup() -> teardown({Fd, _}) -> ok = couch_file:close(Fd). - stream_test_() -> { "CouchDB stream tests", { setup, - fun() -> test_util:start(?MODULE, [ioq]) end, fun test_util:stop/1, + fun() -> test_util:start(?MODULE, [ioq]) end, + fun test_util:stop/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_write/1, fun should_write_consecutive/1, @@ -49,7 +50,6 @@ stream_test_() -> } }. - should_write({_, Stream}) -> ?_assertEqual(ok, couch_stream:write(Stream, <<"food">>)). @@ -98,7 +98,10 @@ should_stream_more_with_4K_chunk_size({Fd, _}) -> Data = <<"a1b2c">>, couch_stream:write(Stream, Data), [Data | Acc] - end, [], lists:seq(1, 1024)), + end, + [], + lists:seq(1, 1024) + ), {NewEngine, Length, _, _, _} = couch_stream:close(Stream), {ok, Ptrs} = couch_stream:to_disk_term(NewEngine), ?_assertMatch({[{0, 4100}, {4106, 1020}], 5120}, {Ptrs, Length}). @@ -109,16 +112,17 @@ should_stop_on_normal_exit_of_stream_opener({Fd, _}) -> fun() -> {ok, StreamPid} = couch_stream:open(?ENGINE(Fd)), RunnerPid ! {pid, StreamPid} - end), - StreamPid = receive - {pid, StreamPid0} -> StreamPid0 - end, + end + ), + StreamPid = + receive + {pid, StreamPid0} -> StreamPid0 + end, % Confirm the validity of the test by verifying the stream opener has died ?assertNot(is_process_alive(OpenerPid)), % Verify the stream itself has also died ?_assertNot(is_process_alive(StreamPid)). - read_all(Engine) -> Data = couch_stream:foldl(Engine, fun(Bin, Acc) -> [Bin, Acc] end, []), iolist_to_binary(Data). diff --git a/src/couch/test/eunit/couch_task_status_tests.erl b/src/couch/test/eunit/couch_task_status_tests.erl index 0ec03563b..f888dd596 100644 --- a/src/couch/test/eunit/couch_task_status_tests.erl +++ b/src/couch/test/eunit/couch_task_status_tests.erl @@ -17,27 +17,30 @@ -define(TIMEOUT, 1000). - setup() -> Ctx = test_util:start(?MODULE, [couch_log], [{dont_mock, [config]}]), {ok, TaskStatusPid} = couch_task_status:start_link(), TaskUpdaterPid = spawn(fun() -> loop() end), {TaskStatusPid, TaskUpdaterPid, Ctx}. - -teardown({TaskStatusPid, _, Ctx})-> - test_util:stop_sync_throw(TaskStatusPid, fun() -> - couch_task_status:stop() - end, timeout_error, ?TIMEOUT), +teardown({TaskStatusPid, _, Ctx}) -> + test_util:stop_sync_throw( + TaskStatusPid, + fun() -> + couch_task_status:stop() + end, + timeout_error, + ?TIMEOUT + ), test_util:stop(Ctx). - couch_task_status_test_() -> { "CouchDB task status updates", { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_register_task/1, fun should_set_task_startup_time/1, @@ -51,12 +54,10 @@ couch_task_status_test_() -> fun should_reset_control_update_frequency/1, fun should_track_multiple_tasks/1, fun should_finish_task/1 - ] } }. - should_register_task({_, Pid, _Ctx}) -> ok = call(Pid, add, [{type, replication}, {progress, 0}]), ?_assertEqual(1, length(couch_task_status:all())). @@ -76,8 +77,10 @@ should_set_task_type({_, Pid, _Ctx}) -> should_not_register_multiple_tasks_for_same_pid({_, Pid, _Ctx}) -> ok = call(Pid, add, [{type, replication}, {progress, 0}]), - ?_assertEqual({add_task_error, already_registered}, - call(Pid, add, [{type, compaction}, {progress, 0}])). + ?_assertEqual( + {add_task_error, already_registered}, + call(Pid, add, [{type, compaction}, {progress, 0}]) + ). should_set_task_progress({_, Pid, _Ctx}) -> ok = call(Pid, add, [{type, replication}, {progress, 0}]), @@ -92,10 +95,12 @@ should_update_time_changes_on_task_progress({_, Pid, _Ctx}) -> ?_assert( begin ok = call(Pid, add, [{type, replication}, {progress, 0}]), - ok = timer:sleep(1000), % sleep awhile to customize update time + % sleep awhile to customize update time + ok = timer:sleep(1000), call(Pid, update, [{progress, 25}]), get_task_prop(Pid, updated_on) > get_task_prop(Pid, started_on) - end). + end + ). %%should_control_update_frequency({_, Pid, _Ctx}) -> %% ?_assertEqual(66, @@ -109,7 +114,8 @@ should_update_time_changes_on_task_progress({_, Pid, _Ctx}) -> %% end). should_reset_control_update_frequency({_, Pid, _Ctx}) -> - ?_assertEqual(87, + ?_assertEqual( + 87, begin ok = call(Pid, add, [{type, replication}, {progress, 0}]), call(Pid, update, [{progress, 50}]), @@ -119,7 +125,8 @@ should_reset_control_update_frequency({_, Pid, _Ctx}) -> call(Pid, update_frequency, 0), call(Pid, update, [{progress, 87}]), get_task_prop(Pid, progress) - end). + end + ). should_track_multiple_tasks(_) -> ?_assert(run_multiple_tasks()). @@ -130,7 +137,6 @@ should_finish_task({_, Pid, _Ctx}) -> ok = call(Pid, done), ?_assertEqual(0, length(couch_task_status:all())). - run_multiple_tasks() -> Pid1 = spawn(fun() -> loop() end), Pid2 = spawn(fun() -> loop() end), @@ -161,7 +167,6 @@ run_multiple_tasks() -> true. - loop() -> receive {add, Props, From} -> @@ -188,7 +193,7 @@ call(Pid, done) -> {'DOWN', Ref, _Type, Pid, _Info} -> Res after ?TIMEOUT -> - throw(timeout_error) + throw(timeout_error) end; call(Pid, Command) -> Pid ! {Command, self()}, @@ -217,17 +222,22 @@ get_task_prop(Pid, Prop) -> Acc end end, - [], couch_task_status:all() + [], + couch_task_status:all() ), case couch_util:get_value(Prop, hd(Element), nil) of nil -> - erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {reason, "Could not get property '" - ++ couch_util:to_list(Prop) - ++ "' for task " - ++ pid_to_list(Pid)}]}); + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, + "Could not get property '" ++ + couch_util:to_list(Prop) ++ + "' for task " ++ + pid_to_list(Pid)} + ]} + ); Value -> Value end. diff --git a/src/couch/test/eunit/couch_util_tests.erl b/src/couch/test/eunit/couch_util_tests.erl index 44a5cce0a..c07ddc093 100644 --- a/src/couch/test/eunit/couch_util_tests.erl +++ b/src/couch/test/eunit/couch_util_tests.erl @@ -14,7 +14,6 @@ -include_lib("couch/include/couch_eunit.hrl"). - validate_callback_exists_test_() -> { "validate_callback_exists tests", @@ -33,8 +32,10 @@ implode_test() -> ?assertEqual([1, 38, 2, 38, 3], couch_util:implode([1, 2, 3], "&")). trim_test() -> - lists:map(fun(S) -> ?assertEqual("foo", couch_util:trim(S)) end, - [" foo", "foo ", "\tfoo", " foo ", "foo\t", "foo\n", "\nfoo"]). + lists:map( + fun(S) -> ?assertEqual("foo", couch_util:trim(S)) end, + [" foo", "foo ", "\tfoo", " foo ", "foo\t", "foo\n", "\nfoo"] + ). abs_pathname_test() -> {ok, Cwd} = file:get_cwd(), @@ -44,8 +45,10 @@ flush_test() -> ?assertNot(couch_util:should_flush()), AcquireMem = fun() -> _IntsToAGazillion = lists:seq(1, 200000), - _LotsOfData = lists:map(fun(_) -> <<"foobar">> end, - lists:seq(1, 500000)), + _LotsOfData = lists:map( + fun(_) -> <<"foobar">> end, + lists:seq(1, 500000) + ), _ = list_to_binary(_LotsOfData), %% Allocation 200K tuples puts us above the memory threshold @@ -97,11 +100,20 @@ find_in_binary_test_() -> ], lists:map( fun({Needle, Haystack, Result}) -> - Msg = lists:flatten(io_lib:format("Looking for ~s in ~s", - [Needle, Haystack])), - {Msg, ?_assertMatch(Result, - couch_util:find_in_binary(Needle, Haystack))} - end, Cases). + Msg = lists:flatten( + io_lib:format( + "Looking for ~s in ~s", + [Needle, Haystack] + ) + ), + {Msg, + ?_assertMatch( + Result, + couch_util:find_in_binary(Needle, Haystack) + )} + end, + Cases + ). should_succeed_for_existent_cb() -> ?_assert(couch_util:validate_callback_exists(lists, any, 2)). @@ -115,10 +127,14 @@ should_fail_for_missing_cb() -> lists:map( fun({M, F, A} = MFA) -> Name = lists:flatten(io_lib:format("~w:~w/~w", [M, F, A])), - {Name, ?_assertThrow( - {error, {undefined_callback, Name, MFA}}, - couch_util:validate_callback_exists(M, F, A))} - end, Cases). + {Name, + ?_assertThrow( + {error, {undefined_callback, Name, MFA}}, + couch_util:validate_callback_exists(M, F, A) + )} + end, + Cases + ). to_hex_test_() -> [ diff --git a/src/couch/test/eunit/couch_uuids_tests.erl b/src/couch/test/eunit/couch_uuids_tests.erl index 9ca2c8a84..6546779bb 100644 --- a/src/couch/test/eunit/couch_uuids_tests.erl +++ b/src/couch/test/eunit/couch_uuids_tests.erl @@ -16,17 +16,14 @@ -define(TIMEOUT, 20). - setup_all() -> test_util:start_applications([config]), couch_uuids:start(). - teardown_all(_) -> couch_uuids:stop(), test_util:stop_applications([config]). - uuids_test_() -> { setup, @@ -40,25 +37,21 @@ uuids_test_() -> ] }. - default_algorithm() -> config:delete("uuids", "algorithm", false), check_unique(). - sequential_algorithm() -> config:set("uuids", "algorithm", "sequential", false), check_unique(), check_increment_monotonically(), check_rollover(). - utc_algorithm() -> config:set("uuids", "algorithm", "utc_random", false), check_unique(), check_increment_monotonically(). - utc_id_suffix_algorithm() -> config:set("uuids", "algorithm", "utc_id", false), config:set("uuids", "utc_id_suffix", "bozo", false), @@ -66,36 +59,30 @@ utc_id_suffix_algorithm() -> check_increment_monotonically(), check_preserve_suffix(). - check_unique() -> %% this one may really runs for too long on slow hosts ?assert(test_unique(10000, [couch_uuids:new()])). - check_increment_monotonically() -> ?assert(couch_uuids:new() < couch_uuids:new()). - check_rollover() -> UUID = binary_to_list(couch_uuids:new()), Prefix = element(1, lists:split(26, UUID)), N = gen_until_pref_change(Prefix, 0), ?assert(N >= 5000 andalso N =< 11000). - check_preserve_suffix() -> UUID = binary_to_list(couch_uuids:new()), Suffix = get_suffix(UUID), ?assert(test_same_suffix(10000, Suffix)). - test_unique(0, _) -> true; test_unique(N, UUIDs) -> UUID = couch_uuids:new(), ?assertNot(lists:member(UUID, UUIDs)), - test_unique(N - 1, [UUID| UUIDs]). - + test_unique(N - 1, [UUID | UUIDs]). gen_until_pref_change(_, Count) when Count > 8251 -> Count; @@ -105,7 +92,6 @@ gen_until_pref_change(Prefix, N) -> _ -> N end. - test_same_suffix(0, _) -> true; test_same_suffix(N, Suffix) -> @@ -114,11 +100,9 @@ test_same_suffix(N, Suffix) -> _ -> false end. - get_prefix(UUID) -> element(1, lists:split(26, binary_to_list(UUID))). - get_suffix(UUID) when is_binary(UUID) -> get_suffix(binary_to_list(UUID)); get_suffix(UUID) -> diff --git a/src/couch/test/eunit/couch_work_queue_tests.erl b/src/couch/test/eunit/couch_work_queue_tests.erl index a192230ef..acf0e45dc 100644 --- a/src/couch/test/eunit/couch_work_queue_tests.erl +++ b/src/couch/test/eunit/couch_work_queue_tests.erl @@ -16,7 +16,6 @@ -define(TIMEOUT, 100). - setup(Opts) -> {ok, Q} = couch_work_queue:new(Opts), Producer = spawn_producer(Q), @@ -33,9 +32,11 @@ setup_max_items_and_size() -> setup([{max_size, 160}, {max_items, 3}]). setup_multi_workers() -> - {Q, Producer, Consumer1} = setup([{max_size, 160}, - {max_items, 3}, - {multi_workers, true}]), + {Q, Producer, Consumer1} = setup([ + {max_size, 160}, + {max_items, 3}, + {multi_workers, true} + ]), Consumer2 = spawn_consumer(Q), Consumer3 = spawn_consumer(Q), {Q, Producer, [Consumer1, Consumer2, Consumer3]}. @@ -52,7 +53,6 @@ teardown({Q, Producer, Consumers}) when is_list(Consumers) -> teardown({Q, Producer, Consumer}) -> teardown({Q, Producer, [Consumer]}). - single_consumer_test_() -> { "Single producer and consumer", @@ -61,7 +61,8 @@ single_consumer_test_() -> "Queue with 3 max items", { foreach, - fun setup_max_items/0, fun teardown/1, + fun setup_max_items/0, + fun teardown/1, single_consumer_max_item_count() ++ common_cases() } }, @@ -69,7 +70,8 @@ single_consumer_test_() -> "Queue with max size of 160 bytes", { foreach, - fun setup_max_size/0, fun teardown/1, + fun setup_max_size/0, + fun teardown/1, single_consumer_max_size() ++ common_cases() } }, @@ -77,7 +79,8 @@ single_consumer_test_() -> "Queue with max size of 160 bytes and 3 max items", { foreach, - fun setup_max_items_and_size/0, fun teardown/1, + fun setup_max_items_and_size/0, + fun teardown/1, single_consumer_max_items_and_size() ++ common_cases() } } @@ -92,15 +95,15 @@ multiple_consumers_test_() -> "Queue with max size of 160 bytes and 3 max items", { foreach, - fun setup_multi_workers/0, fun teardown/1, + fun setup_multi_workers/0, + fun teardown/1, common_cases() ++ multiple_consumers() } - } ] }. -common_cases()-> +common_cases() -> [ fun should_block_consumer_on_dequeue_from_empty_queue/1, fun should_consume_right_item/1, @@ -109,7 +112,7 @@ common_cases()-> fun should_be_closed/1 ]. -single_consumer_max_item_count()-> +single_consumer_max_item_count() -> [ fun should_have_no_items_for_new_queue/1, fun should_block_producer_on_full_queue_count/1, @@ -118,7 +121,7 @@ single_consumer_max_item_count()-> fun should_consume_all/1 ]. -single_consumer_max_size()-> +single_consumer_max_size() -> [ fun should_have_zero_size_for_new_queue/1, fun should_block_producer_on_full_queue_size/1, @@ -138,7 +141,6 @@ multiple_consumers() -> fun should_increase_queue_size_on_produce/1 ]. - should_have_no_items_for_new_queue({Q, _, _}) -> ?_assertEqual(0, couch_work_queue:item_count(Q)). @@ -172,8 +174,10 @@ should_consume_right_item({Q, Producer, Consumers}) when is_list(Consumers) -> ?assertEqual(0, couch_work_queue:item_count(Q)), ?assertEqual(0, couch_work_queue:size(Q)), - R = [{ping(C), Item} - || {C, Item} <- lists:zip(Consumers, [Item1, Item2, Item3])], + R = [ + {ping(C), Item} + || {C, Item} <- lists:zip(Consumers, [Item1, Item2, Item3]) + ], ?_assertEqual([{ok, Item1}, {ok, Item2}, {ok, Item3}], R); should_consume_right_item({Q, Producer, Consumer}) -> @@ -284,8 +288,10 @@ should_be_closed({Q, _, Consumers}) when is_list(Consumers) -> ItemsCount = couch_work_queue:item_count(Q), Size = couch_work_queue:size(Q), - ?_assertEqual({[closed, closed, closed], closed, closed}, - {LastConsumerItems, ItemsCount, Size}); + ?_assertEqual( + {[closed, closed, closed], closed, closed}, + {LastConsumerItems, ItemsCount, Size} + ); should_be_closed({Q, _, Consumer}) -> ok = close_queue(Q), @@ -295,14 +301,19 @@ should_be_closed({Q, _, Consumer}) -> ItemsCount = couch_work_queue:item_count(Q), Size = couch_work_queue:size(Q), - ?_assertEqual({closed, closed, closed}, - {LastConsumerItems, ItemsCount, Size}). - + ?_assertEqual( + {closed, closed, closed}, + {LastConsumerItems, ItemsCount, Size} + ). close_queue(Q) -> - test_util:stop_sync(Q, fun() -> - ok = couch_work_queue:close(Q) - end, ?TIMEOUT). + test_util:stop_sync( + Q, + fun() -> + ok = couch_work_queue:close(Q) + end, + ?TIMEOUT + ). spawn_consumer(Q) -> Parent = self(), @@ -365,10 +376,13 @@ produce(Q, Producer, Size, Wait) -> {item, Ref, Item} -> Item after ?TIMEOUT -> - erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {reason, "Timeout asking producer to produce an item"}]}) + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, "Timeout asking producer to produce an item"} + ]} + ) end. ping(Pid) -> @@ -393,10 +407,10 @@ stop(Pid, Name) -> wait_increment(Q, ItemsCount) -> test_util:wait(fun() -> - case couch_work_queue:item_count(Q) > ItemsCount of - true -> - ok; - false -> - wait - end + case couch_work_queue:item_count(Q) > ItemsCount of + true -> + ok; + false -> + wait + end end). diff --git a/src/couch/test/eunit/couchdb_attachments_tests.erl b/src/couch/test/eunit/couchdb_attachments_tests.erl index 04859dbc9..376553985 100644 --- a/src/couch/test/eunit/couchdb_attachments_tests.erl +++ b/src/couch/test/eunit/couchdb_attachments_tests.erl @@ -26,12 +26,15 @@ -define(TIMEWAIT, 1000). -define(i2l(I), integer_to_list(I)). - start() -> Ctx = test_util:start_couch(), % ensure in default compression settings for attachments_compression_tests - config:set("attachments", "compression_level", - ?i2l(?COMPRESSION_LEVEL), false), + config:set( + "attachments", + "compression_level", + ?i2l(?COMPRESSION_LEVEL), + false + ), config:set("attachments", "compressible_types", "text/*", false), Ctx. @@ -46,7 +49,7 @@ setup() -> setup({binary, standalone}) -> {Host, DbName} = setup(), - setup_att(fun create_standalone_png_att/2, Host, DbName, ?FIXTURE_PNG); + setup_att(fun create_standalone_png_att/2, Host, DbName, ?FIXTURE_PNG); setup({text, standalone}) -> {Host, DbName} = setup(), setup_att(fun create_standalone_text_att/2, Host, DbName, ?FIXTURE_TXT); @@ -76,13 +79,13 @@ teardown(DbName) -> ok = couch_server:delete(?l2b(DbName), []), ok. - attachments_test_() -> { "Attachments tests", { setup, - fun start/0, fun test_util:stop_couch/1, + fun start/0, + fun test_util:stop_couch/1, [ attachments_md5_tests(), attachments_compression_tests() @@ -95,7 +98,8 @@ attachments_md5_tests() -> "Attachments MD5 tests", { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_upload_attachment_without_md5/1, fun should_upload_attachment_by_chunks_without_md5/1, @@ -111,12 +115,12 @@ attachments_md5_tests() -> attachments_compression_tests() -> Funs = [ - fun should_get_att_without_accept_gzip_encoding/2, - fun should_get_att_with_accept_gzip_encoding/2, - fun should_get_att_with_accept_deflate_encoding/2, - fun should_return_406_response_on_unsupported_encoding/2, - fun should_get_doc_with_att_data/2, - fun should_get_doc_with_att_data_stub/2 + fun should_get_att_without_accept_gzip_encoding/2, + fun should_get_att_with_accept_gzip_encoding/2, + fun should_get_att_with_accept_deflate_encoding/2, + fun should_return_406_response_on_unsupported_encoding/2, + fun should_get_doc_with_att_data/2, + fun should_get_doc_with_att_data_stub/2 ], { "Attachments compression tests", @@ -133,13 +137,15 @@ attachments_compression_tests() -> "Created already been compressed via Attachments API", { foreachx, - fun setup/1, fun teardown/2, + fun setup/1, + fun teardown/2, [{compressed, Fun} || Fun <- Funs] } }, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_not_create_compressed_att_with_deflate_encoding/1, fun should_not_create_compressed_att_with_compress_encoding/1, @@ -155,7 +161,8 @@ created_attachments_compression_tests(Mod, Funs) -> "Compressiable attachments", { foreachx, - fun setup/1, fun teardown/2, + fun setup/1, + fun teardown/2, [{{text, Mod}, Fun} || Fun <- Funs] } }, @@ -163,14 +170,13 @@ created_attachments_compression_tests(Mod, Funs) -> "Uncompressiable attachments", { foreachx, - fun setup/1, fun teardown/2, + fun setup/1, + fun teardown/2, [{{binary, Mod}, Fun} || Fun <- Funs] } } ]. - - should_upload_attachment_without_md5({Host, DbName}) -> ?_test(begin AttUrl = string:join(["", DbName, ?docid(), "readme.txt"], "/"), @@ -238,9 +244,12 @@ should_upload_attachment_by_chunks_with_valid_md5_trailer({Host, DbName}) -> AttUrl = string:join(["", DbName, ?docid(), "readme.txt"], "/"), AttData = <<"We all live in a yellow submarine!">>, <<Part1:21/binary, Part2:13/binary>> = AttData, - Body = [chunked_body([Part1, Part2]), - "Content-MD5: ", base64:encode(couch_hash:md5_hash(AttData)), - "\r\n\r\n"], + Body = [ + chunked_body([Part1, Part2]), + "Content-MD5: ", + base64:encode(couch_hash:md5_hash(AttData)), + "\r\n\r\n" + ], Headers = [ {"Content-Type", "text/plain"}, {"Host", Host}, @@ -264,11 +273,12 @@ should_reject_attachment_with_invalid_md5({Host, DbName}) -> ], {ok, Code, Json} = request("PUT", AttUrl, Headers, Body), ?assertEqual(400, Code), - ?assertEqual(<<"content_md5_mismatch">>, - get_json(Json, [<<"error">>])) + ?assertEqual( + <<"content_md5_mismatch">>, + get_json(Json, [<<"error">>]) + ) end). - should_reject_chunked_attachment_with_invalid_md5({Host, DbName}) -> ?_test(begin AttUrl = string:join(["", DbName, ?docid(), "readme.txt"], "/"), @@ -283,8 +293,10 @@ should_reject_chunked_attachment_with_invalid_md5({Host, DbName}) -> ], {ok, Code, Json} = request("PUT", AttUrl, Headers, Body), ?assertEqual(400, Code), - ?assertEqual(<<"content_md5_mismatch">>, - get_json(Json, [<<"error">>])) + ?assertEqual( + <<"content_md5_mismatch">>, + get_json(Json, [<<"error">>]) + ) end). should_reject_chunked_attachment_with_invalid_md5_trailer({Host, DbName}) -> @@ -292,9 +304,12 @@ should_reject_chunked_attachment_with_invalid_md5_trailer({Host, DbName}) -> AttUrl = string:join(["", DbName, ?docid(), "readme.txt"], "/"), AttData = <<"We all live in a yellow submarine!">>, <<Part1:21/binary, Part2:13/binary>> = AttData, - Body = [chunked_body([Part1, Part2]), - "Content-MD5: ", base64:encode(<<"foobar!">>), - "\r\n\r\n"], + Body = [ + chunked_body([Part1, Part2]), + "Content-MD5: ", + base64:encode(<<"foobar!">>), + "\r\n\r\n" + ], Headers = [ {"Content-Type", "text/plain"}, {"Host", Host}, @@ -317,7 +332,8 @@ should_get_att_without_accept_gzip_encoding(_, {Data, {_, _, AttUrl}}) -> should_get_att_with_accept_gzip_encoding(compressed, {Data, {_, _, AttUrl}}) -> ?_test(begin {ok, Code, Headers, Body} = test_request:get( - AttUrl, [{"Accept-Encoding", "gzip"}]), + AttUrl, [{"Accept-Encoding", "gzip"}] + ), ?assertEqual(200, Code), ?assert(lists:member({"Content-Encoding", "gzip"}, Headers)), ?assertEqual(Data, zlib:gunzip(iolist_to_binary(Body))) @@ -325,7 +341,8 @@ should_get_att_with_accept_gzip_encoding(compressed, {Data, {_, _, AttUrl}}) -> should_get_att_with_accept_gzip_encoding({text, _}, {Data, {_, _, AttUrl}}) -> ?_test(begin {ok, Code, Headers, Body} = test_request:get( - AttUrl, [{"Accept-Encoding", "gzip"}]), + AttUrl, [{"Accept-Encoding", "gzip"}] + ), ?assertEqual(200, Code), ?assert(lists:member({"Content-Encoding", "gzip"}, Headers)), ?assertEqual(Data, zlib:gunzip(iolist_to_binary(Body))) @@ -333,77 +350,98 @@ should_get_att_with_accept_gzip_encoding({text, _}, {Data, {_, _, AttUrl}}) -> should_get_att_with_accept_gzip_encoding({binary, _}, {Data, {_, _, AttUrl}}) -> ?_test(begin {ok, Code, Headers, Body} = test_request:get( - AttUrl, [{"Accept-Encoding", "gzip"}]), + AttUrl, [{"Accept-Encoding", "gzip"}] + ), ?assertEqual(200, Code), - ?assertEqual(undefined, - couch_util:get_value("Content-Encoding", Headers)), + ?assertEqual( + undefined, + couch_util:get_value("Content-Encoding", Headers) + ), ?assertEqual(Data, iolist_to_binary(Body)) end). should_get_att_with_accept_deflate_encoding(_, {Data, {_, _, AttUrl}}) -> ?_test(begin {ok, Code, Headers, Body} = test_request:get( - AttUrl, [{"Accept-Encoding", "deflate"}]), + AttUrl, [{"Accept-Encoding", "deflate"}] + ), ?assertEqual(200, Code), - ?assertEqual(undefined, - couch_util:get_value("Content-Encoding", Headers)), + ?assertEqual( + undefined, + couch_util:get_value("Content-Encoding", Headers) + ), ?assertEqual(Data, iolist_to_binary(Body)) end). should_return_406_response_on_unsupported_encoding(_, {_, {_, _, AttUrl}}) -> - ?_assertEqual(406, + ?_assertEqual( + 406, begin {ok, Code, _, _} = test_request:get( - AttUrl, [{"Accept-Encoding", "deflate, *;q=0"}]), + AttUrl, [{"Accept-Encoding", "deflate, *;q=0"}] + ), Code - end). + end + ). should_get_doc_with_att_data(compressed, {Data, {_, DocUrl, _}}) -> ?_test(begin Url = DocUrl ++ "?attachments=true", {ok, Code, _, Body} = test_request:get( - Url, [{"Accept", "application/json"}]), + Url, [{"Accept", "application/json"}] + ), ?assertEqual(200, Code), Json = jiffy:decode(Body), AttJson = couch_util:get_nested_json_value( - Json, [<<"_attachments">>, ?ATT_TXT_NAME]), + Json, [<<"_attachments">>, ?ATT_TXT_NAME] + ), AttData = couch_util:get_nested_json_value( - AttJson, [<<"data">>]), + AttJson, [<<"data">>] + ), ?assertEqual( <<"text/plain">>, - couch_util:get_nested_json_value(AttJson,[<<"content_type">>])), + couch_util:get_nested_json_value(AttJson, [<<"content_type">>]) + ), ?assertEqual(Data, base64:decode(AttData)) end); should_get_doc_with_att_data({text, _}, {Data, {_, DocUrl, _}}) -> ?_test(begin Url = DocUrl ++ "?attachments=true", {ok, Code, _, Body} = test_request:get( - Url, [{"Accept", "application/json"}]), + Url, [{"Accept", "application/json"}] + ), ?assertEqual(200, Code), Json = jiffy:decode(Body), AttJson = couch_util:get_nested_json_value( - Json, [<<"_attachments">>, ?ATT_TXT_NAME]), + Json, [<<"_attachments">>, ?ATT_TXT_NAME] + ), AttData = couch_util:get_nested_json_value( - AttJson, [<<"data">>]), + AttJson, [<<"data">>] + ), ?assertEqual( <<"text/plain">>, - couch_util:get_nested_json_value(AttJson,[<<"content_type">>])), + couch_util:get_nested_json_value(AttJson, [<<"content_type">>]) + ), ?assertEqual(Data, base64:decode(AttData)) end); should_get_doc_with_att_data({binary, _}, {Data, {_, DocUrl, _}}) -> ?_test(begin Url = DocUrl ++ "?attachments=true", {ok, Code, _, Body} = test_request:get( - Url, [{"Accept", "application/json"}]), + Url, [{"Accept", "application/json"}] + ), ?assertEqual(200, Code), Json = jiffy:decode(Body), AttJson = couch_util:get_nested_json_value( - Json, [<<"_attachments">>, ?ATT_BIN_NAME]), + Json, [<<"_attachments">>, ?ATT_BIN_NAME] + ), AttData = couch_util:get_nested_json_value( - AttJson, [<<"data">>]), + AttJson, [<<"data">>] + ), ?assertEqual( <<"image/png">>, - couch_util:get_nested_json_value(AttJson,[<<"content_type">>])), + couch_util:get_nested_json_value(AttJson, [<<"content_type">>]) + ), ?assertEqual(Data, base64:decode(AttData)) end). @@ -411,13 +449,17 @@ should_get_doc_with_att_data_stub(compressed, {Data, {_, DocUrl, _}}) -> ?_test(begin Url = DocUrl ++ "?att_encoding_info=true", {ok, Code, _, Body} = test_request:get( - Url, [{"Accept", "application/json"}]), + Url, [{"Accept", "application/json"}] + ), ?assertEqual(200, Code), Json = jiffy:decode(Body), {AttJson} = couch_util:get_nested_json_value( - Json, [<<"_attachments">>, ?ATT_TXT_NAME]), - ?assertEqual(<<"gzip">>, - couch_util:get_value(<<"encoding">>, AttJson)), + Json, [<<"_attachments">>, ?ATT_TXT_NAME] + ), + ?assertEqual( + <<"gzip">>, + couch_util:get_value(<<"encoding">>, AttJson) + ), AttLength = couch_util:get_value(<<"length">>, AttJson), EncLength = couch_util:get_value(<<"encoded_length">>, AttJson), ?assertEqual(AttLength, EncLength), @@ -427,38 +469,55 @@ should_get_doc_with_att_data_stub({text, _}, {Data, {_, DocUrl, _}}) -> ?_test(begin Url = DocUrl ++ "?att_encoding_info=true", {ok, Code, _, Body} = test_request:get( - Url, [{"Accept", "application/json"}]), + Url, [{"Accept", "application/json"}] + ), ?assertEqual(200, Code), Json = jiffy:decode(Body), {AttJson} = couch_util:get_nested_json_value( - Json, [<<"_attachments">>, ?ATT_TXT_NAME]), - ?assertEqual(<<"gzip">>, - couch_util:get_value(<<"encoding">>, AttJson)), + Json, [<<"_attachments">>, ?ATT_TXT_NAME] + ), + ?assertEqual( + <<"gzip">>, + couch_util:get_value(<<"encoding">>, AttJson) + ), AttEncLength = iolist_size(gzip(Data)), - ?assertEqual(AttEncLength, - couch_util:get_value(<<"encoded_length">>, AttJson)), - ?assertEqual(byte_size(Data), - couch_util:get_value(<<"length">>, AttJson)) + ?assertEqual( + AttEncLength, + couch_util:get_value(<<"encoded_length">>, AttJson) + ), + ?assertEqual( + byte_size(Data), + couch_util:get_value(<<"length">>, AttJson) + ) end); should_get_doc_with_att_data_stub({binary, _}, {Data, {_, DocUrl, _}}) -> ?_test(begin Url = DocUrl ++ "?att_encoding_info=true", {ok, Code, _, Body} = test_request:get( - Url, [{"Accept", "application/json"}]), + Url, [{"Accept", "application/json"}] + ), ?assertEqual(200, Code), Json = jiffy:decode(Body), {AttJson} = couch_util:get_nested_json_value( - Json, [<<"_attachments">>, ?ATT_BIN_NAME]), - ?assertEqual(undefined, - couch_util:get_value(<<"encoding">>, AttJson)), - ?assertEqual(undefined, - couch_util:get_value(<<"encoded_length">>, AttJson)), - ?assertEqual(byte_size(Data), - couch_util:get_value(<<"length">>, AttJson)) + Json, [<<"_attachments">>, ?ATT_BIN_NAME] + ), + ?assertEqual( + undefined, + couch_util:get_value(<<"encoding">>, AttJson) + ), + ?assertEqual( + undefined, + couch_util:get_value(<<"encoded_length">>, AttJson) + ), + ?assertEqual( + byte_size(Data), + couch_util:get_value(<<"length">>, AttJson) + ) end). should_not_create_compressed_att_with_deflate_encoding({Host, DbName}) -> - ?_assertEqual(415, + ?_assertEqual( + 415, begin HttpHost = "http://" ++ Host, AttUrl = string:join([HttpHost, DbName, ?docid(), "file.txt"], "/"), @@ -470,14 +529,16 @@ should_not_create_compressed_att_with_deflate_encoding({Host, DbName}) -> ], {ok, Code, _, _} = test_request:put(AttUrl, Headers, Body), Code - end). + end + ). should_not_create_compressed_att_with_compress_encoding({Host, DbName}) -> % Note: As of OTP R13B04, it seems there's no LZW compression % (i.e. UNIX compress utility implementation) lib in OTP. % However there's a simple working Erlang implementation at: % http://scienceblogs.com/goodmath/2008/01/simple_lempelziv_compression_i.php - ?_assertEqual(415, + ?_assertEqual( + 415, begin HttpHost = "http://" ++ Host, AttUrl = string:join([HttpHost, DbName, ?docid(), "file.txt"], "/"), @@ -488,33 +549,42 @@ should_not_create_compressed_att_with_compress_encoding({Host, DbName}) -> ], {ok, Code, _, _} = test_request:put(AttUrl, Headers, Data), Code - end). + end + ). should_create_compressible_att_with_ctype_params({Host, DbName}) -> - {timeout, ?TIMEOUT_EUNIT, ?_test(begin - HttpHost = "http://" ++ Host, - DocUrl = string:join([HttpHost, DbName, ?docid()], "/"), - AttUrl = string:join([DocUrl, ?b2l(?ATT_TXT_NAME)], "/"), - {ok, Data} = file:read_file(?FIXTURE_TXT), - Headers = [{"Content-Type", "text/plain; charset=UTF-8"}], - {ok, Code0, _, _} = test_request:put(AttUrl, Headers, Data), - ?assertEqual(201, Code0), - - {ok, Code1, _, Body} = test_request:get( - DocUrl ++ "?att_encoding_info=true"), - ?assertEqual(200, Code1), - Json = jiffy:decode(Body), - {AttJson} = couch_util:get_nested_json_value( - Json, [<<"_attachments">>, ?ATT_TXT_NAME]), - ?assertEqual(<<"gzip">>, - couch_util:get_value(<<"encoding">>, AttJson)), - AttEncLength = iolist_size(gzip(Data)), - ?assertEqual(AttEncLength, - couch_util:get_value(<<"encoded_length">>, AttJson)), - ?assertEqual(byte_size(Data), - couch_util:get_value(<<"length">>, AttJson)) - end)}. - + {timeout, ?TIMEOUT_EUNIT, + ?_test(begin + HttpHost = "http://" ++ Host, + DocUrl = string:join([HttpHost, DbName, ?docid()], "/"), + AttUrl = string:join([DocUrl, ?b2l(?ATT_TXT_NAME)], "/"), + {ok, Data} = file:read_file(?FIXTURE_TXT), + Headers = [{"Content-Type", "text/plain; charset=UTF-8"}], + {ok, Code0, _, _} = test_request:put(AttUrl, Headers, Data), + ?assertEqual(201, Code0), + + {ok, Code1, _, Body} = test_request:get( + DocUrl ++ "?att_encoding_info=true" + ), + ?assertEqual(200, Code1), + Json = jiffy:decode(Body), + {AttJson} = couch_util:get_nested_json_value( + Json, [<<"_attachments">>, ?ATT_TXT_NAME] + ), + ?assertEqual( + <<"gzip">>, + couch_util:get_value(<<"encoding">>, AttJson) + ), + AttEncLength = iolist_size(gzip(Data)), + ?assertEqual( + AttEncLength, + couch_util:get_value(<<"encoded_length">>, AttJson) + ), + ?assertEqual( + byte_size(Data), + couch_util:get_value(<<"length">>, AttJson) + ) + end)}. compact_after_lowering_attachment_size_limit_test_() -> { @@ -539,33 +609,33 @@ compact_after_lowering_attachment_size_limit_test_() -> } }. - should_compact_after_lowering_attachment_size_limit({_Ctx, DbName}) -> - {timeout, ?TIMEOUT_EUNIT, ?_test(begin - {ok, Db1} = couch_db:open(DbName, [?ADMIN_CTX]), - Doc1 = #doc{id = <<"doc1">>, atts = att(1000)}, - {ok, _} = couch_db:update_doc(Db1, Doc1, []), - couch_db:close(Db1), - config:set("couchdb", "max_attachment_size", "1", _Persist = false), - compact_db(DbName), - {ok, Db2} = couch_db:open_int(DbName, []), - {ok, Doc2} = couch_db:open_doc(Db2, <<"doc1">>), - couch_db:close(Db2), - [Att] = Doc2#doc.atts, - ?assertEqual(1000, couch_att:fetch(att_len, Att)) - end)}. - + {timeout, ?TIMEOUT_EUNIT, + ?_test(begin + {ok, Db1} = couch_db:open(DbName, [?ADMIN_CTX]), + Doc1 = #doc{id = <<"doc1">>, atts = att(1000)}, + {ok, _} = couch_db:update_doc(Db1, Doc1, []), + couch_db:close(Db1), + config:set("couchdb", "max_attachment_size", "1", _Persist = false), + compact_db(DbName), + {ok, Db2} = couch_db:open_int(DbName, []), + {ok, Doc2} = couch_db:open_doc(Db2, <<"doc1">>), + couch_db:close(Db2), + [Att] = Doc2#doc.atts, + ?assertEqual(1000, couch_att:fetch(att_len, Att)) + end)}. att(Size) when is_integer(Size), Size >= 1 -> - [couch_att:new([ - {name, <<"att">>}, - {type, <<"app/binary">>}, - {att_len, Size}, - {data, fun(_Bytes) -> - << <<"x">> || _ <- lists:seq(1, Size) >> - end} - ])]. - + [ + couch_att:new([ + {name, <<"att">>}, + {type, <<"app/binary">>}, + {att_len, Size}, + {data, fun(_Bytes) -> + <<<<"x">> || _ <- lists:seq(1, Size)>> + end} + ]) + ]. compact_db(DbName) -> {ok, Db} = couch_db:open_int(DbName, []), @@ -573,34 +643,35 @@ compact_db(DbName) -> wait_compaction(DbName, "database", ?LINE), ok = couch_db:close(Db). - wait_compaction(DbName, Kind, Line) -> WaitFun = fun() -> - case is_compaction_running(DbName) of - true -> wait; - false -> ok - end + case is_compaction_running(DbName) of + true -> wait; + false -> ok + end end, case test_util:wait(WaitFun, ?TIMEOUT) of timeout -> - erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, Line}, - {reason, "Timeout waiting for " - ++ Kind - ++ " database compaction"}]}); + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, Line}, + {reason, + "Timeout waiting for " ++ + Kind ++ + " database compaction"} + ]} + ); _ -> ok end. - is_compaction_running(DbName) -> {ok, Db} = couch_db:open_int(DbName, []), {ok, DbInfo} = couch_db:get_db_info(Db), couch_db:close(Db), couch_util:get_value(compact_running, DbInfo) =:= true. - internal_replication_after_lowering_attachment_size_limit_test_() -> { "Internal replication after lowering max attachment size", @@ -629,23 +700,23 @@ internal_replication_after_lowering_attachment_size_limit_test_() -> }. should_replicate_after_lowering_attachment_size({_Ctx, SrcName, TgtName}) -> - {timeout, ?TIMEOUT_EUNIT, ?_test(begin - {ok, SrcDb} = couch_db:open(SrcName, [?ADMIN_CTX]), - SrcDoc = #doc{id = <<"doc">>, atts = att(1000)}, - {ok, _} = couch_db:update_doc(SrcDb, SrcDoc, []), - couch_db:close(SrcDb), - config:set("couchdb", "max_attachment_size", "1", _Persist = false), - % Create a pair of "fake" shards - SrcShard = #shard{name = SrcName, node = node()}, - TgtShard = #shard{name = TgtName, node = node()}, - mem3_rep:go(SrcShard, TgtShard, []), - {ok, TgtDb} = couch_db:open_int(TgtName, []), - {ok, TgtDoc} = couch_db:open_doc(TgtDb, <<"doc">>), - couch_db:close(TgtDb), - [Att] = TgtDoc#doc.atts, - ?assertEqual(1000, couch_att:fetch(att_len, Att)) - end)}. - + {timeout, ?TIMEOUT_EUNIT, + ?_test(begin + {ok, SrcDb} = couch_db:open(SrcName, [?ADMIN_CTX]), + SrcDoc = #doc{id = <<"doc">>, atts = att(1000)}, + {ok, _} = couch_db:update_doc(SrcDb, SrcDoc, []), + couch_db:close(SrcDb), + config:set("couchdb", "max_attachment_size", "1", _Persist = false), + % Create a pair of "fake" shards + SrcShard = #shard{name = SrcName, node = node()}, + TgtShard = #shard{name = TgtName, node = node()}, + mem3_rep:go(SrcShard, TgtShard, []), + {ok, TgtDb} = couch_db:open_int(TgtName, []), + {ok, TgtDoc} = couch_db:open_doc(TgtDb, <<"doc">>), + couch_db:close(TgtDb), + [Att] = TgtDoc#doc.atts, + ?assertEqual(1000, couch_att:fetch(att_len, Att)) + end)}. get_json(Json, Path) -> couch_util:get_nested_json_value(Json, Path). @@ -684,12 +755,15 @@ bind_address() -> request(Method, Url, Headers, Body) -> RequestHead = [Method, " ", Url, " HTTP/1.1"], - RequestHeaders = [[string:join([Key, Value], ": "), "\r\n"] - || {Key, Value} <- Headers], + RequestHeaders = [ + [string:join([Key, Value], ": "), "\r\n"] + || {Key, Value} <- Headers + ], Request = [RequestHead, "\r\n", RequestHeaders, "\r\n", Body], Sock = get_socket(), gen_tcp:send(Sock, list_to_binary(lists:flatten(Request))), - timer:sleep(?TIMEWAIT), % must wait to receive complete response + % must wait to receive complete response + timer:sleep(?TIMEWAIT), {ok, R} = gen_tcp:recv(Sock, 0), gen_tcp:close(Sock), [Header, Body1] = re:split(R, "\r\n\r\n", [{return, binary}]), @@ -702,7 +776,8 @@ create_standalone_text_att(Host, DbName) -> {ok, Data} = file:read_file(?FIXTURE_TXT), Url = string:join([Host, DbName, "doc", ?b2l(?ATT_TXT_NAME)], "/"), {ok, Code, _Headers, _Body} = test_request:put( - Url, [{"Content-Type", "text/plain"}], Data), + Url, [{"Content-Type", "text/plain"}], Data + ), ?assertEqual(201, Code), Url. @@ -710,39 +785,48 @@ create_standalone_png_att(Host, DbName) -> {ok, Data} = file:read_file(?FIXTURE_PNG), Url = string:join([Host, DbName, "doc", ?b2l(?ATT_BIN_NAME)], "/"), {ok, Code, _Headers, _Body} = test_request:put( - Url, [{"Content-Type", "image/png"}], Data), + Url, [{"Content-Type", "image/png"}], Data + ), ?assertEqual(201, Code), Url. create_inline_text_att(Host, DbName) -> {ok, Data} = file:read_file(?FIXTURE_TXT), Url = string:join([Host, DbName, "doc"], "/"), - Doc = {[ - {<<"_attachments">>, {[ - {?ATT_TXT_NAME, {[ - {<<"content_type">>, <<"text/plain">>}, - {<<"data">>, base64:encode(Data)} - ]} - }]}} - ]}, + Doc = + {[ + {<<"_attachments">>, + {[ + {?ATT_TXT_NAME, + {[ + {<<"content_type">>, <<"text/plain">>}, + {<<"data">>, base64:encode(Data)} + ]}} + ]}} + ]}, {ok, Code, _Headers, _Body} = test_request:put( - Url, [{"Content-Type", "application/json"}], jiffy:encode(Doc)), + Url, [{"Content-Type", "application/json"}], jiffy:encode(Doc) + ), ?assertEqual(201, Code), string:join([Url, ?b2l(?ATT_TXT_NAME)], "/"). create_inline_png_att(Host, DbName) -> {ok, Data} = file:read_file(?FIXTURE_PNG), Url = string:join([Host, DbName, "doc"], "/"), - Doc = {[ - {<<"_attachments">>, {[ - {?ATT_BIN_NAME, {[ - {<<"content_type">>, <<"image/png">>}, - {<<"data">>, base64:encode(Data)} - ]} - }]}} - ]}, + Doc = + {[ + {<<"_attachments">>, + {[ + {?ATT_BIN_NAME, + {[ + {<<"content_type">>, <<"image/png">>}, + {<<"data">>, base64:encode(Data)} + ]}} + ]}} + ]}, {ok, Code, _Headers, _Body} = test_request:put( - Url, [{"Content-Type", "application/json"}], jiffy:encode(Doc)), + Url, [{"Content-Type", "application/json"}], jiffy:encode(Doc) + ), ?assertEqual(201, Code), string:join([Url, ?b2l(?ATT_BIN_NAME)], "/"). @@ -750,8 +834,10 @@ create_already_compressed_att(Host, DbName) -> {ok, Data} = file:read_file(?FIXTURE_TXT), Url = string:join([Host, DbName, "doc", ?b2l(?ATT_TXT_NAME)], "/"), {ok, Code, _Headers, _Body} = test_request:put( - Url, [{"Content-Type", "text/plain"}, {"Content-Encoding", "gzip"}], - zlib:gzip(Data)), + Url, + [{"Content-Type", "text/plain"}, {"Content-Encoding", "gzip"}], + zlib:gzip(Data) + ), ?assertEqual(201, Code), Url. diff --git a/src/couch/test/eunit/couchdb_auth_tests.erl b/src/couch/test/eunit/couchdb_auth_tests.erl index 19d32d0c5..dfb22dc25 100644 --- a/src/couch/test/eunit/couchdb_auth_tests.erl +++ b/src/couch/test/eunit/couchdb_auth_tests.erl @@ -14,23 +14,21 @@ -include_lib("couch/include/couch_eunit.hrl"). - setup(PortType) -> Hashed = couch_passwords:hash_admin_password("artischocko"), - ok = config:set("admins", "rocko", binary_to_list(Hashed), _Persist=false), + ok = config:set("admins", "rocko", binary_to_list(Hashed), _Persist = false), Addr = config:get("httpd", "bind_address", "127.0.0.1"), lists:concat(["http://", Addr, ":", port(PortType), "/_session"]). setup_require_valid_user(PortType) -> - ok = config:set("chttpd", "require_valid_user", "true", _Persist=false), + ok = config:set("chttpd", "require_valid_user", "true", _Persist = false), setup(PortType). teardown(_, _) -> ok. teardown_require_valid_user(_, _) -> - config:set("chttpd", "require_valid_user", "false", _Persist=false). - + config:set("chttpd", "require_valid_user", "false", _Persist = false). auth_test_() -> Tests = [ @@ -46,7 +44,8 @@ auth_test_() -> "Auth tests", { setup, - fun() -> test_util:start_couch([chttpd]) end, fun test_util:stop_couch/1, + fun() -> test_util:start_couch([chttpd]) end, + fun test_util:stop_couch/1, [ make_test_cases(clustered, Tests), make_test_cases(backdoor, Tests), @@ -64,49 +63,67 @@ make_test_cases(Mod, Funs) -> make_require_valid_user_test_cases(Mod, Funs) -> { lists:flatten(io_lib:format("~s require_valid_user=true", [Mod])), - {foreachx, fun setup_require_valid_user/1, fun teardown_require_valid_user/2, - [{Mod, Fun} || Fun <- Funs]} + {foreachx, fun setup_require_valid_user/1, fun teardown_require_valid_user/2, [ + {Mod, Fun} + || Fun <- Funs + ]} }. should_return_username_on_post_to_session(_PortType, Url) -> - ?_assertEqual(<<"rocko">>, + ?_assertEqual( + <<"rocko">>, begin Hashed = couch_passwords:hash_admin_password(<<"artischocko">>), ok = config:set("admins", "rocko", binary_to_list(Hashed), false), - {ok, _, _, Body} = test_request:post(Url, [{"Content-Type", "application/json"}], - "{\"name\":\"rocko\", \"password\":\"artischocko\"}"), + {ok, _, _, Body} = test_request:post( + Url, + [{"Content-Type", "application/json"}], + "{\"name\":\"rocko\", \"password\":\"artischocko\"}" + ), {Json} = jiffy:decode(Body), proplists:get_value(<<"name">>, Json) - end). + end + ). should_not_return_authenticated_field(_PortType, Url) -> - ?_assertThrow({not_found, _}, + ?_assertThrow( + {not_found, _}, begin couch_util:get_nested_json_value(session(Url), [ - <<"info">>, <<"authenticated">>]) - end). + <<"info">>, <<"authenticated">> + ]) + end + ). should_return_list_of_handlers(backdoor, Url) -> - ?_assertEqual([<<"cookie">>,<<"default">>], + ?_assertEqual( + [<<"cookie">>, <<"default">>], begin couch_util:get_nested_json_value(session(Url), [ - <<"info">>, <<"authentication_handlers">>]) - end); + <<"info">>, <<"authentication_handlers">> + ]) + end + ); should_return_list_of_handlers(clustered, Url) -> - ?_assertEqual([<<"cookie">>,<<"default">>], + ?_assertEqual( + [<<"cookie">>, <<"default">>], begin couch_util:get_nested_json_value(session(Url), [ - <<"info">>, <<"authentication_handlers">>]) - end). - + <<"info">>, <<"authentication_handlers">> + ]) + end + ). %% ------------------------------------------------------------------ %% Internal Function Definitions %% ------------------------------------------------------------------ session(Url) -> - {ok, _, _, Body} = test_request:get(Url, [{"Content-Type", "application/json"}], - "{\"name\":\"rocko\", \"password\":\"artischocko\"}"), + {ok, _, _, Body} = test_request:get( + Url, + [{"Content-Type", "application/json"}], + "{\"name\":\"rocko\", \"password\":\"artischocko\"}" + ), jiffy:decode(Body). port(clustered) -> diff --git a/src/couch/test/eunit/couchdb_cookie_domain_tests.erl b/src/couch/test/eunit/couchdb_cookie_domain_tests.erl index c46352f35..17c41dafe 100755 --- a/src/couch/test/eunit/couchdb_cookie_domain_tests.erl +++ b/src/couch/test/eunit/couchdb_cookie_domain_tests.erl @@ -21,7 +21,7 @@ setup() -> Ctx = test_util:start_couch([chttpd]), Hashed = couch_passwords:hash_admin_password(?PASS), - ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist=false), + ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist = false), Addr = config:get("httpd", "bind_address", "127.0.0.1"), Port = mochiweb_socket_server:get(chttpd, port), Url = ?l2b(io_lib:format("http://~s:~b/_session", [Addr, Port])), @@ -30,7 +30,7 @@ setup() -> {ok, ?b2l(Url), ContentType, ?b2l(Payload), Ctx}. teardown({ok, _, _, _, Ctx}) -> - ok = config:delete("admins", ?USER, _Persist=false), + ok = config:delete("admins", ?USER, _Persist = false), test_util:stop_couch(Ctx). cookie_test_() -> @@ -52,8 +52,12 @@ cookie_test_() -> should_set_cookie_domain(Url, ContentType, Payload) -> ?_test(begin - ok = config:set("couch_httpd_auth", "cookie_domain", - "example.com", false), + ok = config:set( + "couch_httpd_auth", + "cookie_domain", + "example.com", + false + ), {ok, Code, Headers, _} = test_request:post(Url, ContentType, Payload), ?assertEqual(200, Code), Cookie = proplists:get_value("Set-Cookie", Headers), @@ -71,8 +75,12 @@ should_not_set_cookie_domain(Url, ContentType, Payload) -> should_delete_cookie_domain(Url, ContentType, Payload) -> ?_test(begin - ok = config:set("couch_httpd_auth", "cookie_domain", - "example.com", false), + ok = config:set( + "couch_httpd_auth", + "cookie_domain", + "example.com", + false + ), {ok, Code, Headers, _} = test_request:delete(Url, ContentType, Payload), ?assertEqual(200, Code), Cookie = proplists:get_value("Set-Cookie", Headers), diff --git a/src/couch/test/eunit/couchdb_cors_tests.erl b/src/couch/test/eunit/couchdb_cors_tests.erl index 8ec61cc8a..dce07fd28 100644 --- a/src/couch/test/eunit/couchdb_cors_tests.erl +++ b/src/couch/test/eunit/couchdb_cors_tests.erl @@ -20,10 +20,12 @@ -define(TIMEOUT, 1000). -define(_assertEqualLists(A, B), - ?_assertEqual(lists:usort(A), lists:usort(B))). + ?_assertEqual(lists:usort(A), lists:usort(B)) +). -define(assertEqualLists(A, B), - ?assertEqual(lists:usort(A), lists:usort(B))). + ?assertEqual(lists:usort(A), lists:usort(B)) +). start() -> Ctx = test_util:start_couch([ioq]), @@ -46,14 +48,16 @@ setup() -> setup({Mod, VHost}) -> {Host, DbName} = setup(), - Url = case Mod of - server -> - Host; - db -> - Host ++ "/" ++ DbName - end, - DefaultHeaders = [{"Origin", "http://example.com"}] - ++ maybe_append_vhost(VHost), + Url = + case Mod of + server -> + Host; + db -> + Host ++ "/" ++ DbName + end, + DefaultHeaders = + [{"Origin", "http://example.com"}] ++ + maybe_append_vhost(VHost), {Host, DbName, Url, DefaultHeaders}. teardown(DbName) when is_list(DbName) -> @@ -65,7 +69,6 @@ teardown({_, DbName}) -> teardown(_, {_, DbName, _, _}) -> teardown(DbName). - cors_test_() -> Funs = [ fun should_not_allow_origin/2, @@ -85,7 +88,8 @@ cors_test_() -> "CORS (COUCHDB-431)", { setup, - fun start/0, fun test_util:stop_couch/1, + fun start/0, + fun test_util:stop_couch/1, [ cors_tests(Funs), vhost_cors_tests(Funs), @@ -99,7 +103,8 @@ headers_tests() -> "Various headers tests", { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_not_return_cors_headers_for_invalid_origin/1, fun should_not_return_cors_headers_for_invalid_origin_preflight/1, @@ -130,206 +135,288 @@ vhost_cors_tests(Funs) -> make_test_case(Mod, UseVhost, Funs) -> { - case Mod of server -> "Server"; db -> "Database" end, - {foreachx, fun setup/1, fun teardown/2, [{{Mod, UseVhost}, Fun} - || Fun <- Funs]} + case Mod of + server -> "Server"; + db -> "Database" + end, + {foreachx, fun setup/1, fun teardown/2, [ + {{Mod, UseVhost}, Fun} + || Fun <- Funs + ]} }. - should_not_allow_origin(_, {_, _, Url, Headers0}) -> - ?_assertEqual(undefined, + ?_assertEqual( + undefined, begin config:delete("cors", "origins", false), Headers1 = proplists:delete("Origin", Headers0), - Headers = [{"Origin", "http://127.0.0.1"}] - ++ Headers1, + Headers = + [{"Origin", "http://127.0.0.1"}] ++ + Headers1, {ok, _, Resp, _} = test_request:get(Url, Headers), proplists:get_value("Access-Control-Allow-Origin", Resp) - end). + end + ). should_not_allow_origin_with_port_mismatch({_, VHost}, {_, _, Url, _}) -> - ?_assertEqual(undefined, + ?_assertEqual( + undefined, begin - Headers = [{"Origin", "http://example.com:5984"}, - {"Access-Control-Request-Method", "GET"}] - ++ maybe_append_vhost(VHost), + Headers = + [ + {"Origin", "http://example.com:5984"}, + {"Access-Control-Request-Method", "GET"} + ] ++ + maybe_append_vhost(VHost), {ok, _, Resp, _} = test_request:options(Url, Headers), proplists:get_value("Access-Control-Allow-Origin", Resp) - end). + end + ). should_not_allow_origin_with_scheme_mismatch({_, VHost}, {_, _, Url, _}) -> - ?_assertEqual(undefined, + ?_assertEqual( + undefined, begin - Headers = [{"Origin", "http://example.com:5984"}, - {"Access-Control-Request-Method", "GET"}] - ++ maybe_append_vhost(VHost), + Headers = + [ + {"Origin", "http://example.com:5984"}, + {"Access-Control-Request-Method", "GET"} + ] ++ + maybe_append_vhost(VHost), {ok, _, Resp, _} = test_request:options(Url, Headers), proplists:get_value("Access-Control-Allow-Origin", Resp) - end). + end + ). should_not_all_origin_due_case_mismatch({_, VHost}, {_, _, Url, _}) -> - ?_assertEqual(undefined, + ?_assertEqual( + undefined, begin - Headers = [{"Origin", "http://ExAmPlE.CoM"}, - {"Access-Control-Request-Method", "GET"}] - ++ maybe_append_vhost(VHost), + Headers = + [ + {"Origin", "http://ExAmPlE.CoM"}, + {"Access-Control-Request-Method", "GET"} + ] ++ + maybe_append_vhost(VHost), {ok, _, Resp, _} = test_request:options(Url, Headers), proplists:get_value("Access-Control-Allow-Origin", Resp) - end). + end + ). should_make_simple_request(_, {_, _, Url, DefaultHeaders}) -> ?_test(begin {ok, _, Resp, _} = test_request:get(Url, DefaultHeaders), ?assertEqual( undefined, - proplists:get_value("Access-Control-Allow-Credentials", Resp)), + proplists:get_value("Access-Control-Allow-Credentials", Resp) + ), ?assertEqual( "http://example.com", - proplists:get_value("Access-Control-Allow-Origin", Resp)), + proplists:get_value("Access-Control-Allow-Origin", Resp) + ), ?assertEqualLists( ?COUCH_HEADERS ++ list_simple_headers(Resp), - split_list(proplists:get_value("Access-Control-Expose-Headers", Resp))) + split_list(proplists:get_value("Access-Control-Expose-Headers", Resp)) + ) end). should_make_preflight_request(_, {_, _, Url, DefaultHeaders}) -> - ?_assertEqualLists(?SUPPORTED_METHODS, + ?_assertEqualLists( + ?SUPPORTED_METHODS, begin - Headers = DefaultHeaders - ++ [{"Access-Control-Request-Method", "GET"}], + Headers = + DefaultHeaders ++ + [{"Access-Control-Request-Method", "GET"}], {ok, _, Resp, _} = test_request:options(Url, Headers), split_list(proplists:get_value("Access-Control-Allow-Methods", Resp)) - end). + end + ). should_make_prefligh_request_with_port({_, VHost}, {_, _, Url, _}) -> - ?_assertEqual("http://example.com:5984", + ?_assertEqual( + "http://example.com:5984", begin - config:set("cors", "origins", "http://example.com:5984", - false), - Headers = [{"Origin", "http://example.com:5984"}, - {"Access-Control-Request-Method", "GET"}] - ++ maybe_append_vhost(VHost), + config:set( + "cors", + "origins", + "http://example.com:5984", + false + ), + Headers = + [ + {"Origin", "http://example.com:5984"}, + {"Access-Control-Request-Method", "GET"} + ] ++ + maybe_append_vhost(VHost), {ok, _, Resp, _} = test_request:options(Url, Headers), proplists:get_value("Access-Control-Allow-Origin", Resp) - end). + end + ). should_make_prefligh_request_with_scheme({_, VHost}, {_, _, Url, _}) -> - ?_assertEqual("https://example.com:5984", + ?_assertEqual( + "https://example.com:5984", begin - config:set("cors", "origins", "https://example.com:5984", - false), - Headers = [{"Origin", "https://example.com:5984"}, - {"Access-Control-Request-Method", "GET"}] - ++ maybe_append_vhost(VHost), + config:set( + "cors", + "origins", + "https://example.com:5984", + false + ), + Headers = + [ + {"Origin", "https://example.com:5984"}, + {"Access-Control-Request-Method", "GET"} + ] ++ + maybe_append_vhost(VHost), {ok, _, Resp, _} = test_request:options(Url, Headers), proplists:get_value("Access-Control-Allow-Origin", Resp) - end). + end + ). should_make_prefligh_request_with_wildcard_origin({_, VHost}, {_, _, Url, _}) -> - ?_assertEqual("https://example.com:5984", + ?_assertEqual( + "https://example.com:5984", begin config:set("cors", "origins", "*", false), - Headers = [{"Origin", "https://example.com:5984"}, - {"Access-Control-Request-Method", "GET"}] - ++ maybe_append_vhost(VHost), + Headers = + [ + {"Origin", "https://example.com:5984"}, + {"Access-Control-Request-Method", "GET"} + ] ++ + maybe_append_vhost(VHost), {ok, _, Resp, _} = test_request:options(Url, Headers), proplists:get_value("Access-Control-Allow-Origin", Resp) - end). + end + ). should_make_request_with_credentials(_, {_, _, Url, DefaultHeaders}) -> - ?_assertEqual("true", + ?_assertEqual( + "true", begin ok = config:set("cors", "credentials", "true", false), {ok, _, Resp, _} = test_request:options(Url, DefaultHeaders), proplists:get_value("Access-Control-Allow-Credentials", Resp) - end). + end + ). should_make_origin_request_with_auth(_, {_, _, Url, DefaultHeaders}) -> - ?_assertEqual("http://example.com", + ?_assertEqual( + "http://example.com", begin Hashed = couch_passwords:hash_admin_password(<<"test">>), config:set("admins", "test", ?b2l(Hashed), false), {ok, _, Resp, _} = test_request:get( - Url, DefaultHeaders, [{basic_auth, {"test", "test"}}]), + Url, DefaultHeaders, [{basic_auth, {"test", "test"}}] + ), config:delete("admins", "test", false), proplists:get_value("Access-Control-Allow-Origin", Resp) - end). + end + ). should_make_preflight_request_with_auth(_, {_, _, Url, DefaultHeaders}) -> - ?_assertEqualLists(?SUPPORTED_METHODS, + ?_assertEqualLists( + ?SUPPORTED_METHODS, begin Hashed = couch_passwords:hash_admin_password(<<"test">>), config:set("admins", "test", ?b2l(Hashed), false), - Headers = DefaultHeaders - ++ [{"Access-Control-Request-Method", "GET"}], + Headers = + DefaultHeaders ++ + [{"Access-Control-Request-Method", "GET"}], {ok, _, Resp, _} = test_request:options( - Url, Headers, [{basic_auth, {"test", "test"}}]), + Url, Headers, [{basic_auth, {"test", "test"}}] + ), config:delete("admins", "test", false), split_list(proplists:get_value("Access-Control-Allow-Methods", Resp)) - end). + end + ). should_not_return_cors_headers_for_invalid_origin({Host, _}) -> - ?_assertEqual(undefined, + ?_assertEqual( + undefined, begin Headers = [{"Origin", "http://127.0.0.1"}], {ok, _, Resp, _} = test_request:get(Host, Headers), proplists:get_value("Access-Control-Allow-Origin", Resp) - end). + end + ). should_not_return_cors_headers_for_invalid_origin_preflight({Host, _}) -> - ?_assertEqual(undefined, + ?_assertEqual( + undefined, begin - Headers = [{"Origin", "http://127.0.0.1"}, - {"Access-Control-Request-Method", "GET"}], + Headers = [ + {"Origin", "http://127.0.0.1"}, + {"Access-Control-Request-Method", "GET"} + ], {ok, _, Resp, _} = test_request:options(Host, Headers), proplists:get_value("Access-Control-Allow-Origin", Resp) - end). + end + ). should_make_request_against_attachment({Host, DbName}) -> {"COUCHDB-1689", - ?_assertEqual(200, - begin - Url = Host ++ "/" ++ DbName, - {ok, Code0, _, _} = test_request:put( - Url ++ "/doc/file.txt", [{"Content-Type", "text/plain"}], - "hello, couch!"), - ?assert(Code0 =:= 201), - {ok, Code, _, _} = test_request:get( - Url ++ "/doc?attachments=true", - [{"Origin", "http://example.com"}]), - Code - end)}. + ?_assertEqual( + 200, + begin + Url = Host ++ "/" ++ DbName, + {ok, Code0, _, _} = test_request:put( + Url ++ "/doc/file.txt", + [{"Content-Type", "text/plain"}], + "hello, couch!" + ), + ?assert(Code0 =:= 201), + {ok, Code, _, _} = test_request:get( + Url ++ "/doc?attachments=true", + [{"Origin", "http://example.com"}] + ), + Code + end + )}. should_make_range_request_against_attachment({Host, DbName}) -> {"COUCHDB-1689", - ?_assertEqual(206, - begin - Url = Host ++ "/" ++ DbName, - {ok, Code0, _, _} = test_request:put( - Url ++ "/doc/file.txt", - [{"Content-Type", "application/octet-stream"}], - "hello, couch!"), - ?assert(Code0 =:= 201), - {ok, Code, _, _} = test_request:get( - Url ++ "/doc/file.txt", [{"Origin", "http://example.com"}, - {"Range", "bytes=0-6"}]), - Code - end)}. + ?_assertEqual( + 206, + begin + Url = Host ++ "/" ++ DbName, + {ok, Code0, _, _} = test_request:put( + Url ++ "/doc/file.txt", + [{"Content-Type", "application/octet-stream"}], + "hello, couch!" + ), + ?assert(Code0 =:= 201), + {ok, Code, _, _} = test_request:get( + Url ++ "/doc/file.txt", [ + {"Origin", "http://example.com"}, + {"Range", "bytes=0-6"} + ] + ), + Code + end + )}. should_make_request_with_if_none_match_header({Host, DbName}) -> {"COUCHDB-1697", - ?_assertEqual(304, - begin - Url = Host ++ "/" ++ DbName, - {ok, Code0, Headers0, _} = test_request:put( - Url ++ "/doc", [{"Content-Type", "application/json"}], "{}"), - ?assert(Code0 =:= 201), - ETag = proplists:get_value("ETag", Headers0), - {ok, Code, _, _} = test_request:get( - Url ++ "/doc", [{"Origin", "http://example.com"}, - {"If-None-Match", ETag}]), - Code - end)}. - + ?_assertEqual( + 304, + begin + Url = Host ++ "/" ++ DbName, + {ok, Code0, Headers0, _} = test_request:put( + Url ++ "/doc", [{"Content-Type", "application/json"}], "{}" + ), + ?assert(Code0 =:= 201), + ETag = proplists:get_value("ETag", Headers0), + {ok, Code, _, _} = test_request:get( + Url ++ "/doc", [ + {"Origin", "http://example.com"}, + {"If-None-Match", ETag} + ] + ), + Code + end + )}. maybe_append_vhost(true) -> [{"Host", "http://example.com"}]; diff --git a/src/couch/test/eunit/couchdb_db_tests.erl b/src/couch/test/eunit/couchdb_db_tests.erl index 338f2cd3c..2f6993576 100644 --- a/src/couch/test/eunit/couchdb_db_tests.erl +++ b/src/couch/test/eunit/couchdb_db_tests.erl @@ -21,12 +21,10 @@ setup() -> fabric:create_db(DbName), DbName. - teardown(DbName) -> (catch fabric:delete_db(DbName)), ok. - clustered_db_test_() -> { "Checking clustered db API", @@ -39,7 +37,8 @@ clustered_db_test_() -> "DB deletion", { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_close_deleted_db/1, fun should_kill_caller_from_load_validation_funs_for_deleted_db/1 @@ -50,7 +49,6 @@ clustered_db_test_() -> } }. - should_close_deleted_db(DbName) -> ?_test(begin [#shard{name = ShardName} | _] = mem3:shards(DbName), @@ -60,7 +58,7 @@ should_close_deleted_db(DbName) -> fabric:delete_db(DbName), receive {'DOWN', MonitorRef, _Type, _Pid, _Info} -> - ok + ok after 2000 -> throw(timeout_error) end, @@ -71,8 +69,7 @@ should_close_deleted_db(DbName) -> end end), ?assertEqual([], ets:lookup(couch_server:couch_dbs(DbName), DbName)) - end). - + end). should_kill_caller_from_load_validation_funs_for_deleted_db(DbName) -> ?_test(begin @@ -85,7 +82,7 @@ should_kill_caller_from_load_validation_funs_for_deleted_db(DbName) -> {'DOWN', MonitorRef, _Type, _Pid, _Info} -> ok after 2000 -> - throw(timeout_error) + throw(timeout_error) end, ?assertError(database_does_not_exist, couch_db:load_validation_funs(Db)) end). diff --git a/src/couch/test/eunit/couchdb_design_doc_tests.erl b/src/couch/test/eunit/couchdb_design_doc_tests.erl index 653a6cb17..c51d56f0b 100644 --- a/src/couch/test/eunit/couchdb_design_doc_tests.erl +++ b/src/couch/test/eunit/couchdb_design_doc_tests.erl @@ -25,21 +25,21 @@ setup() -> BaseUrl = "http://" ++ Addr ++ ":" ++ Port, {?b2l(DbName), BaseUrl}. - teardown({DbName, _}) -> couch_server:delete(?l2b(DbName), [?ADMIN_CTX]), ok. - design_list_test_() -> { "Check _list functionality", { setup, - fun test_util:start_couch/0, fun test_util:stop_couch/1, + fun test_util:start_couch/0, + fun test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_return_empty_when_plain_return/1, fun should_return_empty_when_no_docs/1 @@ -50,38 +50,50 @@ design_list_test_() -> should_return_empty_when_plain_return({DbName, BaseUrl}) -> ?_test(begin - ?assertEqual(<<>>, - query_text(BaseUrl, DbName, "foo", "_list/plain_return/simple_view")) + ?assertEqual( + <<>>, + query_text(BaseUrl, DbName, "foo", "_list/plain_return/simple_view") + ) end). should_return_empty_when_no_docs({DbName, BaseUrl}) -> ?_test(begin - ?assertEqual(<<>>, - query_text(BaseUrl, DbName, "foo", "_list/simple_render/simple_view")) + ?assertEqual( + <<>>, + query_text(BaseUrl, DbName, "foo", "_list/simple_render/simple_view") + ) end). create_design_doc(DbName, DDName) -> {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]), - DDoc = couch_doc:from_json_obj({[ - {<<"_id">>, DDName}, - {<<"language">>, <<"javascript">>}, - {<<"views">>, {[ - {<<"simple_view">>, {[ - {<<"map">>, <<"function(doc) {emit(doc._id, doc)}">> }, - {<<"reduce">>, <<"function (key, values, rereduce) {return sum(values);}">> } - ]}} - ]}}, - {<<"lists">>, {[ - {<<"plain_return">>, <<"function(head, req) {return;}">>}, - {<<"simple_render">>, <<"function(head, req) {var row; while(row=getRow()) {send(JSON.stringify(row)); }}">>} - ]}} - ]}), + DDoc = couch_doc:from_json_obj( + {[ + {<<"_id">>, DDName}, + {<<"language">>, <<"javascript">>}, + {<<"views">>, + {[ + {<<"simple_view">>, + {[ + {<<"map">>, <<"function(doc) {emit(doc._id, doc)}">>}, + {<<"reduce">>, + <<"function (key, values, rereduce) {return sum(values);}">>} + ]}} + ]}}, + {<<"lists">>, + {[ + {<<"plain_return">>, <<"function(head, req) {return;}">>}, + {<<"simple_render">>, + <<"function(head, req) {var row; while(row=getRow()) {send(JSON.stringify(row)); }}">>} + ]}} + ]} + ), {ok, Rev} = couch_db:update_doc(Db, DDoc, []), couch_db:close(Db), Rev. query_text(BaseUrl, DbName, DDoc, Path) -> {ok, Code, _Headers, Body} = test_request:get( - BaseUrl ++ "/" ++ DbName ++ "/_design/" ++ DDoc ++ "/" ++ Path), + BaseUrl ++ "/" ++ DbName ++ "/_design/" ++ DDoc ++ "/" ++ Path + ), ?assertEqual(200, Code), Body. diff --git a/src/couch/test/eunit/couchdb_file_compression_tests.erl b/src/couch/test/eunit/couchdb_file_compression_tests.erl index 77250337c..75bf18a12 100644 --- a/src/couch/test/eunit/couchdb_file_compression_tests.erl +++ b/src/couch/test/eunit/couchdb_file_compression_tests.erl @@ -25,26 +25,27 @@ setup_all() -> DbName = ?tempdb(), {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]), ok = populate_db(Db, ?DOCS_COUNT), - DDoc = couch_doc:from_json_obj({[ - {<<"_id">>, ?DDOC_ID}, - {<<"language">>, <<"javascript">>}, - {<<"views">>, {[ - {<<"by_id">>, {[ - {<<"map">>, <<"function(doc){emit(doc._id, doc.string);}">>} + DDoc = couch_doc:from_json_obj( + {[ + {<<"_id">>, ?DDOC_ID}, + {<<"language">>, <<"javascript">>}, + {<<"views">>, + {[ + {<<"by_id">>, + {[ + {<<"map">>, <<"function(doc){emit(doc._id, doc.string);}">>} + ]}} ]}} - ]} - } - ]}), + ]} + ), {ok, _} = couch_db:update_doc(Db, DDoc, []), ok = couch_db:close(Db), {Ctx, DbName}. - teardown_all({Ctx, DbName}) -> ok = couch_server:delete(DbName, [?ADMIN_CTX]), test_util:stop_couch(Ctx). - couch_file_compression_test_() -> { "CouchDB file compression tests", @@ -62,13 +63,11 @@ couch_file_compression_test_() -> } }. - should_use_none({_, DbName}) -> run_test(DbName, "none"). should_use_deflate_1({_, DbName}) -> run_test(DbName, "deflate_1"). should_use_deflate_9({_, DbName}) -> run_test(DbName, "deflate_9"). should_use_snappy({_, DbName}) -> run_test(DbName, "snappy"). - should_compare_compression_methods({_, DbName}) -> TestDb = setup_db(DbName), Name = "none > snappy > deflate_1 > deflate_9", @@ -78,7 +77,6 @@ should_compare_compression_methods({_, DbName}) -> couch_server:delete(TestDb, [?ADMIN_CTX]) end. - run_test(DbName, Comp) -> config:set("couchdb", "file_compression", Comp, false), Timeout = 5 + ?TIMEOUT, @@ -93,7 +91,6 @@ run_test(DbName, Comp) -> ok = couch_server:delete(TestDb, [?ADMIN_CTX]) end. - compare_methods(DbName) -> config:set("couchdb", "file_compression", "none", false), ExternalSizePreCompact = db_external_size(DbName), @@ -140,22 +137,23 @@ compare_methods(DbName) -> ?assert(ViewExternalSizeNone =:= ViewExternalSizeSnappy), ?assert(ViewExternalSizeNone =:= ViewExternalSizeDeflate9). - populate_db(_Db, NumDocs) when NumDocs =< 0 -> ok; populate_db(Db, NumDocs) -> Docs = lists:map( fun(_) -> - couch_doc:from_json_obj({[ - {<<"_id">>, couch_uuids:random()}, - {<<"string">>, ?l2b(lists:duplicate(1000, $X))} - ]}) + couch_doc:from_json_obj( + {[ + {<<"_id">>, couch_uuids:random()}, + {<<"string">>, ?l2b(lists:duplicate(1000, $X))} + ]} + ) end, - lists:seq(1, 500)), + lists:seq(1, 500) + ), {ok, _} = couch_db:update_docs(Db, Docs, []), populate_db(Db, NumDocs - 500). - setup_db(SrcDbName) -> TgtDbName = ?tempdb(), TgtDbFileName = binary_to_list(TgtDbName) ++ ".couch", @@ -167,7 +165,6 @@ setup_db(SrcDbName) -> refresh_index(TgtDbName), TgtDbName. - refresh_index(DbName) -> {ok, Db} = couch_db:open_int(DbName, []), {ok, DDoc} = couch_db:open_doc(Db, ?DDOC_ID, [ejson_body]), @@ -224,19 +221,23 @@ external_size(Info) -> wait_compaction(DbName, Kind, Line) -> WaitFun = fun() -> - case is_compaction_running(DbName) of - true -> wait; - false -> ok - end + case is_compaction_running(DbName) of + true -> wait; + false -> ok + end end, case test_util:wait(WaitFun, ?TIMEOUT) of timeout -> - erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, Line}, - {reason, "Timeout waiting for " - ++ Kind - ++ " database compaction"}]}); + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, Line}, + {reason, + "Timeout waiting for " ++ + Kind ++ + " database compaction"} + ]} + ); _ -> ok end. @@ -246,5 +247,5 @@ is_compaction_running(DbName) -> {ok, DbInfo} = couch_db:get_db_info(Db), {ok, ViewInfo} = couch_mrview:get_info(Db, ?DDOC_ID), couch_db:close(Db), - (couch_util:get_value(compact_running, ViewInfo) =:= true) - orelse (couch_util:get_value(compact_running, DbInfo) =:= true). + (couch_util:get_value(compact_running, ViewInfo) =:= true) orelse + (couch_util:get_value(compact_running, DbInfo) =:= true). diff --git a/src/couch/test/eunit/couchdb_location_header_tests.erl b/src/couch/test/eunit/couchdb_location_header_tests.erl index c6c039eb0..08870f8c8 100644 --- a/src/couch/test/eunit/couchdb_location_header_tests.erl +++ b/src/couch/test/eunit/couchdb_location_header_tests.erl @@ -17,7 +17,6 @@ -define(TIMEOUT, 1000). - setup() -> DbName = ?tempdb(), {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]), @@ -32,16 +31,17 @@ teardown({_, DbName}) -> ok = couch_server:delete(?l2b(DbName), [?ADMIN_CTX]), ok. - header_test_() -> { "CouchDB Location Header Tests", { setup, - fun test_util:start_couch/0, fun test_util:stop_couch/1, + fun test_util:start_couch/0, + fun test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_work_with_newlines_in_docs/1, fun should_work_with_newlines_in_attachments/1 @@ -56,10 +56,14 @@ should_work_with_newlines_in_docs({Host, DbName}) -> ?_assertEqual( Url, begin - {ok, _, Headers, _} = test_request:put(Url, - [{"Content-Type", "application/json"}], "{}"), + {ok, _, Headers, _} = test_request:put( + Url, + [{"Content-Type", "application/json"}], + "{}" + ), proplists:get_value("Location", Headers) - end)}. + end + )}. should_work_with_newlines_in_attachments({Host, DbName}) -> Url = Host ++ "/" ++ DbName, @@ -75,4 +79,5 @@ should_work_with_newlines_in_attachments({Host, DbName}) -> ], {ok, _, Headers, _} = test_request:put(AttUrl, Headers0, Body), proplists:get_value("Location", Headers) - end)}. + end + )}. diff --git a/src/couch/test/eunit/couchdb_mrview_cors_tests.erl b/src/couch/test/eunit/couchdb_mrview_cors_tests.erl index 3a560edce..9822542f3 100644 --- a/src/couch/test/eunit/couchdb_mrview_cors_tests.erl +++ b/src/couch/test/eunit/couchdb_mrview_cors_tests.erl @@ -15,24 +15,24 @@ -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). - - --define(DDOC, {[ - {<<"_id">>, <<"_design/foo">>}, - {<<"shows">>, {[ - {<<"bar">>, <<"function(doc, req) {return '<h1>wosh</h1>';}">>} - ]}} -]}). +-define(DDOC, + {[ + {<<"_id">>, <<"_design/foo">>}, + {<<"shows">>, + {[ + {<<"bar">>, <<"function(doc, req) {return '<h1>wosh</h1>';}">>} + ]}} + ]} +). -define(USER, "mrview_cors_test_admin"). -define(PASS, "pass"). -define(AUTH, {basic_auth, {?USER, ?PASS}}). - start() -> Ctx = test_util:start_couch([chttpd]), Hashed = couch_passwords:hash_admin_password(?PASS), - ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist=false), + ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist = false), ok = config:set("chttpd", "enable_cors", "true", false), ok = config:set("vhosts", "example.com", "/", false), Ctx. @@ -49,7 +49,7 @@ setup(PortType) -> {Host, ?b2l(DbName)}. teardown(Ctx) -> - ok = config:delete("admins", ?USER, _Persist=false), + ok = config:delete("admins", ?USER, _Persist = false), test_util:stop_couch(Ctx). teardown(PortType, {_Host, DbName}) -> @@ -61,7 +61,8 @@ cors_test_() -> "CORS for mrview", { setup, - fun start/0, fun teardown/1, + fun start/0, + fun teardown/1, [show_tests()] } }. @@ -83,13 +84,16 @@ make_test_case(Mod, Funs) -> should_make_shows_request(_, {Host, DbName}) -> ?_test(begin - ReqUrl = Host ++ "/" ++ DbName ++ "/_design/foo/_show/bar", - Headers = [{"Origin", "http://example.com"}, - {"Access-Control-Request-Method", "GET"}, ?AUTH], - {ok, _, Resp, Body} = test_request:get(ReqUrl, Headers), - Origin = proplists:get_value("Access-Control-Allow-Origin", Resp), - ?assertEqual("http://example.com", Origin), - ?assertEqual(<<"<h1>wosh</h1>">>, Body) + ReqUrl = Host ++ "/" ++ DbName ++ "/_design/foo/_show/bar", + Headers = [ + {"Origin", "http://example.com"}, + {"Access-Control-Request-Method", "GET"}, + ?AUTH + ], + {ok, _, Resp, Body} = test_request:get(ReqUrl, Headers), + Origin = proplists:get_value("Access-Control-Allow-Origin", Resp), + ?assertEqual("http://example.com", Origin), + ?assertEqual(<<"<h1>wosh</h1>">>, Body) end). create_db(backdoor, DbName) -> @@ -111,7 +115,6 @@ assert_success(create_db, Status) -> true = lists:member(Status, [201, 202]); assert_success(delete_db, Status) -> true = lists:member(Status, [200, 202]). - host_url(PortType) -> "http://" ++ bind_address(PortType) ++ ":" ++ port(PortType). @@ -132,7 +135,6 @@ port(clustered) -> port(backdoor) -> integer_to_list(mochiweb_socket_server:get(couch_httpd, port)). - upload_ddoc(Host, DbName) -> Url = Host ++ "/" ++ DbName ++ "/_design/foo", Body = couch_util:json_encode(?DDOC), diff --git a/src/couch/test/eunit/couchdb_mrview_tests.erl b/src/couch/test/eunit/couchdb_mrview_tests.erl index ec77b190d..606c9c39a 100644 --- a/src/couch/test/eunit/couchdb_mrview_tests.erl +++ b/src/couch/test/eunit/couchdb_mrview_tests.erl @@ -15,41 +15,46 @@ -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). - - --define(DDOC, {[ - {<<"_id">>, <<"_design/foo">>}, - {<<"shows">>, {[ - {<<"bar">>, <<"function(doc, req) {return '<h1>wosh</h1>';}">>} - ]}}, - {<<"updates">>, {[ - {<<"report">>, <<"function(doc, req) {" - "var data = JSON.parse(req.body); " - "return ['test', data];" - "}">>} - ]}}, - {<<"views">>, {[ - {<<"view1">>, {[ - {<<"map">>, <<"function(doc){emit(doc._id, doc._rev)}">>} - ]}} - ]}} -]}). +-define(DDOC, + {[ + {<<"_id">>, <<"_design/foo">>}, + {<<"shows">>, + {[ + {<<"bar">>, <<"function(doc, req) {return '<h1>wosh</h1>';}">>} + ]}}, + {<<"updates">>, + {[ + {<<"report">>, << + "function(doc, req) {" + "var data = JSON.parse(req.body); " + "return ['test', data];" + "}" + >>} + ]}}, + {<<"views">>, + {[ + {<<"view1">>, + {[ + {<<"map">>, <<"function(doc){emit(doc._id, doc._rev)}">>} + ]}} + ]}} + ]} +). -define(USER, "admin"). -define(PASS, "pass"). -define(AUTH, {basic_auth, {?USER, ?PASS}}). - setup_all() -> Ctx = test_util:start_couch([chttpd]), ok = meck:new(mochiweb_socket, [passthrough]), Hashed = couch_passwords:hash_admin_password(?PASS), - ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist=false), + ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist = false), Ctx. teardown_all(Ctx) -> meck:unload(), - ok = config:delete("admins", ?USER, _Persist=false), + ok = config:delete("admins", ?USER, _Persist = false), test_util:stop_couch(Ctx). setup(PortType) -> @@ -108,7 +113,6 @@ mrview_cleanup_index_files_test_() -> } }. - make_test_case(Mod, Funs) -> { lists:flatten(io_lib:format("~s", [Mod])), @@ -122,33 +126,38 @@ make_test_case(Mod, Funs) -> should_return_invalid_request_body(PortType, {Host, DbName}) -> ?_test(begin - ok = create_doc(PortType, ?l2b(DbName), <<"doc_id">>, {[]}), - ReqUrl = Host ++ "/" ++ DbName ++ "/_design/foo/_update/report/doc_id", - {ok, Status, _Headers, Body} = - test_request:post(ReqUrl, [?AUTH], <<"{truncated}">>), - {Props} = jiffy:decode(Body), - ?assertEqual( - <<"bad_request">>, couch_util:get_value(<<"error">>, Props)), - ?assertEqual( - <<"Invalid request body">>, couch_util:get_value(<<"reason">>, Props)), - ?assertEqual(400, Status), - ok + ok = create_doc(PortType, ?l2b(DbName), <<"doc_id">>, {[]}), + ReqUrl = Host ++ "/" ++ DbName ++ "/_design/foo/_update/report/doc_id", + {ok, Status, _Headers, Body} = + test_request:post(ReqUrl, [?AUTH], <<"{truncated}">>), + {Props} = jiffy:decode(Body), + ?assertEqual( + <<"bad_request">>, couch_util:get_value(<<"error">>, Props) + ), + ?assertEqual( + <<"Invalid request body">>, couch_util:get_value(<<"reason">>, Props) + ), + ?assertEqual(400, Status), + ok end). should_return_400_for_wrong_order_of_keys(_PortType, {Host, DbName}) -> Args = [{start_key, "\"bbb\""}, {end_key, "\"aaa\""}], ?_test(begin - ReqUrl = Host ++ "/" ++ DbName - ++ "/_design/foo/_view/view1?" ++ mochiweb_util:urlencode(Args), - {ok, Status, _Headers, Body} = test_request:get(ReqUrl, [?AUTH]), - {Props} = jiffy:decode(Body), - ?assertEqual( - <<"query_parse_error">>, couch_util:get_value(<<"error">>, Props)), - ?assertEqual( + ReqUrl = + Host ++ "/" ++ DbName ++ + "/_design/foo/_view/view1?" ++ mochiweb_util:urlencode(Args), + {ok, Status, _Headers, Body} = test_request:get(ReqUrl, [?AUTH]), + {Props} = jiffy:decode(Body), + ?assertEqual( + <<"query_parse_error">>, couch_util:get_value(<<"error">>, Props) + ), + ?assertEqual( <<"No rows can match your key range, reverse your start_key and end_key or set descending=true">>, - couch_util:get_value(<<"reason">>, Props)), - ?assertEqual(400, Status), - ok + couch_util:get_value(<<"reason">>, Props) + ), + ?assertEqual(400, Status), + ok end). should_cleanup_index_files(_PortType, {Host, DbName}) -> @@ -167,30 +176,34 @@ should_cleanup_index_files(_PortType, {Host, DbName}) -> % It is hard to simulate inactive view. % Since couch_mrview:cleanup is called on view definition change. % That's why we just create extra files in place - ToDelete = lists:map(fun(FilePath) -> - ViewFile = filename:join([ - filename:dirname(FilePath), - "11111111111111111111111111111111.view"]), - file:write_file(ViewFile, <<>>), - ViewFile - end, FileList0), + ToDelete = lists:map( + fun(FilePath) -> + ViewFile = filename:join([ + filename:dirname(FilePath), + "11111111111111111111111111111111.view" + ]), + file:write_file(ViewFile, <<>>), + ViewFile + end, + FileList0 + ), FileList1 = filelib:wildcard(IndexWildCard), ?assertEqual([], lists:usort(FileList1 -- (FileList0 ++ ToDelete))), CleanupUrl = Host ++ "/" ++ DbName ++ "/_view_cleanup", {ok, _Status1, _Headers1, _Body1} = test_request:post( - CleanupUrl, [], <<>>, [?AUTH]), + CleanupUrl, [], <<>>, [?AUTH] + ), test_util:wait(fun() -> - IndexFiles = filelib:wildcard(IndexWildCard), - case lists:usort(FileList0) == lists:usort(IndexFiles) of - false -> wait; - true -> ok - end + IndexFiles = filelib:wildcard(IndexWildCard), + case lists:usort(FileList0) == lists:usort(IndexFiles) of + false -> wait; + true -> ok + end end), ok end). - create_doc(backdoor, DbName, Id, Body) -> JsonDoc = couch_util:json_apply_field({<<"_id">>, Id}, Body), Doc = couch_doc:from_json_obj(JsonDoc), @@ -223,7 +236,6 @@ assert_success(create_db, Status) -> assert_success(delete_db, Status) -> ?assert(lists:member(Status, [200, 202])). - host_url(PortType) -> "http://" ++ bind_address(PortType) ++ ":" ++ port(PortType). @@ -243,7 +255,6 @@ port(clustered) -> port(backdoor) -> integer_to_list(mochiweb_socket_server:get(couch_httpd, port)). - upload_ddoc(Host, DbName) -> Url = Host ++ "/" ++ DbName ++ "/_design/foo", Body = couch_util:json_encode(?DDOC), diff --git a/src/couch/test/eunit/couchdb_os_proc_pool.erl b/src/couch/test/eunit/couchdb_os_proc_pool.erl index b552e114a..620265b32 100644 --- a/src/couch/test/eunit/couchdb_os_proc_pool.erl +++ b/src/couch/test/eunit/couchdb_os_proc_pool.erl @@ -17,7 +17,6 @@ -define(TIMEOUT, 1000). - setup() -> ok = couch_proc_manager:reload(), meck:new(couch_os_process, [passthrough]), @@ -32,15 +31,17 @@ os_proc_pool_test_() -> "OS processes pool tests", { setup, - fun test_util:start_couch/0, fun test_util:stop_couch/1, + fun test_util:start_couch/0, + fun test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ should_block_new_proc_on_full_pool(), should_free_slot_on_proc_unexpected_exit(), should_reuse_known_proc(), -% should_process_waiting_queue_as_fifo(), + % should_process_waiting_queue_as_fifo(), should_reduce_pool_on_idle_os_procs(), should_not_return_broken_process_to_the_pool() ] @@ -48,7 +49,6 @@ os_proc_pool_test_() -> } }. - should_block_new_proc_on_full_pool() -> ?_test(begin Client1 = spawn_client(), @@ -78,12 +78,14 @@ should_block_new_proc_on_full_pool() -> ?assertEqual(Proc1#proc.pid, Proc4#proc.pid), ?assertNotEqual(Proc1#proc.client, Proc4#proc.client), - lists:map(fun(C) -> - ?assertEqual(ok, stop_client(C)) - end, [Client2, Client3, Client4]) + lists:map( + fun(C) -> + ?assertEqual(ok, stop_client(C)) + end, + [Client2, Client3, Client4] + ) end). - should_free_slot_on_proc_unexpected_exit() -> ?_test(begin Client1 = spawn_client(), @@ -119,12 +121,14 @@ should_free_slot_on_proc_unexpected_exit() -> ?assertNotEqual(Proc3#proc.pid, Proc4#proc.pid), ?assertNotEqual(Proc3#proc.client, Proc4#proc.client), - lists:map(fun(C) -> - ?assertEqual(ok, stop_client(C)) - end, [Client2, Client3, Client4]) + lists:map( + fun(C) -> + ?assertEqual(ok, stop_client(C)) + end, + [Client2, Client3, Client4] + ) end). - should_reuse_known_proc() -> ?_test(begin Client1 = spawn_client(<<"ddoc1">>), @@ -150,7 +154,6 @@ should_reuse_known_proc() -> ?assertEqual(ok, stop_client(Client1Again)) end). - %should_process_waiting_queue_as_fifo() -> % ?_test(begin % Client1 = spawn_client(<<"ddoc1">>), @@ -181,12 +184,15 @@ should_reuse_known_proc() -> % ?assertEqual(ok, stop_client(Client5)) % end). - should_reduce_pool_on_idle_os_procs() -> ?_test(begin %% os_process_idle_limit is in sec - config:set("query_server_config", - "os_process_idle_limit", "1", false), + config:set( + "query_server_config", + "os_process_idle_limit", + "1", + false + ), ok = confirm_config("os_process_idle_limit", "1"), Client1 = spawn_client(<<"ddoc1">>), @@ -207,15 +213,22 @@ should_reduce_pool_on_idle_os_procs() -> ?assertEqual(1, couch_proc_manager:get_proc_count()) end). - should_not_return_broken_process_to_the_pool() -> ?_test(begin - config:set("query_server_config", - "os_process_soft_limit", "1", false), + config:set( + "query_server_config", + "os_process_soft_limit", + "1", + false + ), ok = confirm_config("os_process_soft_limit", "1"), - config:set("query_server_config", - "os_process_limit", "1", false), + config:set( + "query_server_config", + "os_process_limit", + "1", + false + ), ok = confirm_config("os_process_limit", "1"), DDoc1 = ddoc(<<"_design/ddoc1">>), @@ -227,9 +240,12 @@ should_not_return_broken_process_to_the_pool() -> ?assertEqual(0, meck:num_calls(couch_os_process, stop, 1)), ?assertEqual(1, couch_proc_manager:get_proc_count()), - ?assertError(bad, couch_query_servers:with_ddoc_proc(DDoc1, fun(_) -> - error(bad) - end)), + ?assertError( + bad, + couch_query_servers:with_ddoc_proc(DDoc1, fun(_) -> + error(bad) + end) + ), ?assertEqual(1, meck:num_calls(couch_os_process, stop, 1)), WaitFun = fun() -> @@ -250,19 +266,21 @@ should_not_return_broken_process_to_the_pool() -> ?assertEqual(1, couch_proc_manager:get_proc_count()) end). - ddoc(DDocId) -> #doc{ id = DDocId, revs = {1, [<<"abc">>]}, - body = {[ - {<<"language">>, <<"javascript">>}, - {<<"views">>, {[ - {<<"v1">>, {[ - {<<"map">>, <<"function(doc) {emit(doc.value,1);}">>} - ]}} - ]}} - ]} + body = + {[ + {<<"language">>, <<"javascript">>}, + {<<"views">>, + {[ + {<<"v1">>, + {[ + {<<"map">>, <<"function(doc) {emit(doc.value,1);}">>} + ]}} + ]}} + ]} }. setup_config() -> @@ -279,11 +297,13 @@ confirm_config(Key, Value, Count) -> Value -> ok; _ when Count > 10 -> - erlang:error({config_setup, [ - {module, ?MODULE}, - {line, ?LINE}, - {value, timeout} - ]}); + erlang:error( + {config_setup, [ + {module, ?MODULE}, + {line, ?LINE}, + {value, timeout} + ]} + ); _ -> %% we need to wait to let gen_server:cast finish timer:sleep(10), @@ -304,7 +324,7 @@ spawn_client(DDocId) -> Ref = make_ref(), Pid = spawn(fun() -> DDocKey = {DDocId, <<"1-abcdefgh">>}, - DDoc = #doc{body={[{<<"language">>, <<"erlang">>}]}}, + DDoc = #doc{body = {[{<<"language">>, <<"erlang">>}]}}, Proc = couch_query_servers:get_ddoc_process(DDoc, DDocKey), loop(Parent, Ref, Proc) end), @@ -324,11 +344,15 @@ get_client_proc({Pid, Ref}, ClientName) -> receive {proc, Ref, Proc} -> Proc after ?TIMEOUT -> - erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {reason, "Timeout getting client " - ++ ClientName ++ " proc"}]}) + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, + "Timeout getting client " ++ + ClientName ++ " proc"} + ]} + ) end. stop_client({Pid, Ref}) -> @@ -354,7 +378,7 @@ loop(Parent, Ref, Proc) -> ping -> Parent ! {pong, Ref}, loop(Parent, Ref, Proc); - get_proc -> + get_proc -> Parent ! {proc, Ref, Proc}, loop(Parent, Ref, Proc); stop -> diff --git a/src/couch/test/eunit/couchdb_update_conflicts_tests.erl b/src/couch/test/eunit/couchdb_update_conflicts_tests.erl index 1329aba27..a7d449a2d 100644 --- a/src/couch/test/eunit/couchdb_update_conflicts_tests.erl +++ b/src/couch/test/eunit/couchdb_update_conflicts_tests.erl @@ -24,12 +24,15 @@ start() -> test_util:start_couch(). - setup() -> DbName = ?tempdb(), {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX, overwrite]), - Doc = couch_doc:from_json_obj({[{<<"_id">>, ?DOC_ID}, - {<<"value">>, 0}]}), + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, ?DOC_ID}, + {<<"value">>, 0} + ]} + ), {ok, Rev} = couch_db:update_doc(Db, Doc, []), ok = couch_db:close(Db), RevStr = couch_doc:rev_to_str(Rev), @@ -43,13 +46,13 @@ teardown({DbName, _}) -> teardown(_, {DbName, _RevStr}) -> teardown({DbName, _RevStr}). - view_indexes_cleanup_test_() -> { "Update conflicts", { setup, - fun start/0, fun test_util:stop_couch/1, + fun start/0, + fun test_util:stop_couch/1, [ concurrent_updates(), bulk_docs_updates() @@ -57,23 +60,27 @@ view_indexes_cleanup_test_() -> } }. -concurrent_updates()-> +concurrent_updates() -> { "Concurrent updates", { foreachx, - fun setup/1, fun teardown/2, - [{NumClients, fun should_concurrently_update_doc/2} - || NumClients <- ?NUM_CLIENTS] + fun setup/1, + fun teardown/2, + [ + {NumClients, fun should_concurrently_update_doc/2} + || NumClients <- ?NUM_CLIENTS + ] } }. -bulk_docs_updates()-> +bulk_docs_updates() -> { "Bulk docs updates", { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_bulk_create_delete_doc/1, fun should_bulk_create_local_doc/1, @@ -82,38 +89,41 @@ bulk_docs_updates()-> } }. +should_concurrently_update_doc(NumClients, {DbName, InitRev}) -> + { + ?i2l(NumClients) ++ " clients", + {inorder, [ + {"update doc", + {timeout, ?TIMEOUT div 1000, + ?_test(concurrent_doc_update(NumClients, DbName, InitRev))}}, + {"ensure in single leaf", ?_test(ensure_in_single_revision_leaf(DbName))} + ]} + }. -should_concurrently_update_doc(NumClients, {DbName, InitRev})-> - {?i2l(NumClients) ++ " clients", - {inorder, - [{"update doc", - {timeout, ?TIMEOUT div 1000, - ?_test(concurrent_doc_update(NumClients, DbName, InitRev))}}, - {"ensure in single leaf", - ?_test(ensure_in_single_revision_leaf(DbName))}]}}. - -should_bulk_create_delete_doc({DbName, InitRev})-> +should_bulk_create_delete_doc({DbName, InitRev}) -> ?_test(bulk_delete_create(DbName, InitRev)). -should_bulk_create_local_doc({DbName, _})-> +should_bulk_create_local_doc({DbName, _}) -> ?_test(bulk_create_local_doc(DbName)). -should_ignore_invalid_local_doc({DbName, _})-> +should_ignore_invalid_local_doc({DbName, _}) -> ?_test(ignore_invalid_local_doc(DbName)). - concurrent_doc_update(NumClients, DbName, InitRev) -> Clients = lists:map( fun(Value) -> - ClientDoc = couch_doc:from_json_obj({[ - {<<"_id">>, ?DOC_ID}, - {<<"_rev">>, InitRev}, - {<<"value">>, Value} - ]}), + ClientDoc = couch_doc:from_json_obj( + {[ + {<<"_id">>, ?DOC_ID}, + {<<"_rev">>, InitRev}, + {<<"value">>, Value} + ]} + ), Pid = spawn_client(DbName, ClientDoc), {Value, Pid, erlang:monitor(process, Pid)} end, - lists:seq(1, NumClients)), + lists:seq(1, NumClients) + ), lists:foreach(fun({_, Pid, _}) -> Pid ! go end, Clients), @@ -125,20 +135,31 @@ concurrent_doc_update(NumClients, DbName, InitRev) -> {'DOWN', MonRef, process, Pid, conflict} -> {AccConflicts + 1, AccValue}; {'DOWN', MonRef, process, Pid, Error} -> - erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {reason, "Client " ++ ?i2l(Value) - ++ " got update error: " - ++ couch_util:to_list(Error)}]}) + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, + "Client " ++ ?i2l(Value) ++ + " got update error: " ++ + couch_util:to_list(Error)} + ]} + ) after ?TIMEOUT div 2 -> - erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {reason, "Timeout waiting for client " - ++ ?i2l(Value) ++ " to die"}]}) + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, + "Timeout waiting for client " ++ + ?i2l(Value) ++ " to die"} + ]} + ) end - end, {0, nil}, Clients), + end, + {0, nil}, + Clients + ), ?assertEqual(NumClients - 1, NumConflicts), {ok, Db} = couch_db:open_int(DbName, []), @@ -171,15 +192,19 @@ ensure_in_single_revision_leaf(DbName) -> bulk_delete_create(DbName, InitRev) -> {ok, Db} = couch_db:open_int(DbName, []), - DeletedDoc = couch_doc:from_json_obj({[ - {<<"_id">>, ?DOC_ID}, - {<<"_rev">>, InitRev}, - {<<"_deleted">>, true} - ]}), - NewDoc = couch_doc:from_json_obj({[ - {<<"_id">>, ?DOC_ID}, - {<<"value">>, 666} - ]}), + DeletedDoc = couch_doc:from_json_obj( + {[ + {<<"_id">>, ?DOC_ID}, + {<<"_rev">>, InitRev}, + {<<"_deleted">>, true} + ]} + ), + NewDoc = couch_doc:from_json_obj( + {[ + {<<"_id">>, ?DOC_ID}, + {<<"value">>, 666} + ]} + ), {ok, Results} = couch_db:update_docs(Db, [DeletedDoc, NewDoc], []), ok = couch_db:close(Db), @@ -189,9 +214,11 @@ bulk_delete_create(DbName, InitRev) -> {ok, Db2} = couch_db:open_int(DbName, []), {ok, [{ok, Doc1}]} = couch_db:open_doc_revs( - Db2, ?DOC_ID, [Rev1], [conflicts, deleted_conflicts]), + Db2, ?DOC_ID, [Rev1], [conflicts, deleted_conflicts] + ), {ok, [{ok, Doc2}]} = couch_db:open_doc_revs( - Db2, ?DOC_ID, [Rev2], [conflicts, deleted_conflicts]), + Db2, ?DOC_ID, [Rev2], [conflicts, deleted_conflicts] + ), ok = couch_db:close(Db2), {Doc1Props} = couch_doc:to_json_obj(Doc1, []), @@ -200,40 +227,75 @@ bulk_delete_create(DbName, InitRev) -> %% Document was deleted ?assert(couch_util:get_value(<<"_deleted">>, Doc1Props)), %% New document not flagged as deleted - ?assertEqual(undefined, couch_util:get_value(<<"_deleted">>, - Doc2Props)), + ?assertEqual( + undefined, + couch_util:get_value( + <<"_deleted">>, + Doc2Props + ) + ), %% New leaf revision has the right value - ?assertEqual(666, couch_util:get_value(<<"value">>, - Doc2Props)), + ?assertEqual( + 666, + couch_util:get_value( + <<"value">>, + Doc2Props + ) + ), %% Deleted document has no conflicts - ?assertEqual(undefined, couch_util:get_value(<<"_conflicts">>, - Doc1Props)), + ?assertEqual( + undefined, + couch_util:get_value( + <<"_conflicts">>, + Doc1Props + ) + ), %% Deleted document has no deleted conflicts - ?assertEqual(undefined, couch_util:get_value(<<"_deleted_conflicts">>, - Doc1Props)), + ?assertEqual( + undefined, + couch_util:get_value( + <<"_deleted_conflicts">>, + Doc1Props + ) + ), %% New leaf revision doesn't have conflicts - ?assertEqual(undefined, couch_util:get_value(<<"_conflicts">>, - Doc1Props)), + ?assertEqual( + undefined, + couch_util:get_value( + <<"_conflicts">>, + Doc1Props + ) + ), %% New leaf revision doesn't have deleted conflicts - ?assertEqual(undefined, couch_util:get_value(<<"_deleted_conflicts">>, - Doc1Props)), + ?assertEqual( + undefined, + couch_util:get_value( + <<"_deleted_conflicts">>, + Doc1Props + ) + ), %% Deleted revision has position 2 ?assertEqual(2, element(1, Rev1)), %% New leaf revision has position 3 ?assertEqual(3, element(1, Rev2)). - bulk_create_local_doc(DbName) -> {ok, Db} = couch_db:open_int(DbName, []), - LocalDoc = couch_doc:from_json_obj({[ - {<<"_id">>, ?LOCAL_DOC_ID}, - {<<"_rev">>, <<"0-1">>} - ]}), - - {ok, Results} = couch_db:update_docs(Db, [LocalDoc], - [], replicated_changes), + LocalDoc = couch_doc:from_json_obj( + {[ + {<<"_id">>, ?LOCAL_DOC_ID}, + {<<"_rev">>, <<"0-1">>} + ]} + ), + + {ok, Results} = couch_db:update_docs( + Db, + [LocalDoc], + [], + replicated_changes + ), ok = couch_db:close(Db), ?assertEqual([], Results), @@ -243,17 +305,22 @@ bulk_create_local_doc(DbName) -> ?assertEqual(?LOCAL_DOC_ID, LocalDoc1#doc.id), ?assertEqual({0, [<<"2">>]}, LocalDoc1#doc.revs). - ignore_invalid_local_doc(DbName) -> {ok, Db} = couch_db:open_int(DbName, []), - LocalDoc = couch_doc:from_json_obj({[ - {<<"_id">>, ?LOCAL_DOC_ID}, - {<<"_rev">>, <<"0-abcdef">>} - ]}), - - {ok, Results} = couch_db:update_docs(Db, [LocalDoc], - [], replicated_changes), + LocalDoc = couch_doc:from_json_obj( + {[ + {<<"_id">>, ?LOCAL_DOC_ID}, + {<<"_rev">>, <<"0-abcdef">>} + ]} + ), + + {ok, Results} = couch_db:update_docs( + Db, + [LocalDoc], + [], + replicated_changes + ), ok = couch_db:close(Db), ?assertEqual([], Results), @@ -262,7 +329,6 @@ ignore_invalid_local_doc(DbName) -> ok = couch_db:close(Db2), ?assertEqual({not_found, missing}, Result2). - spawn_client(DbName, Doc) -> spawn(fun() -> {ok, Db} = couch_db:open_int(DbName, []), @@ -270,11 +336,13 @@ spawn_client(DbName, Doc) -> go -> ok end, erlang:yield(), - Result = try - couch_db:update_doc(Db, Doc, []) - catch _:Error -> - Error - end, + Result = + try + couch_db:update_doc(Db, Doc, []) + catch + _:Error -> + Error + end, ok = couch_db:close(Db), exit(Result) end). diff --git a/src/couch/test/eunit/couchdb_vhosts_tests.erl b/src/couch/test/eunit/couchdb_vhosts_tests.erl index fbe5579cd..635b8f9a6 100644 --- a/src/couch/test/eunit/couchdb_vhosts_tests.erl +++ b/src/couch/test/eunit/couchdb_vhosts_tests.erl @@ -18,30 +18,35 @@ -define(TIMEOUT, 1000). -define(iofmt(S, A), lists:flatten(io_lib:format(S, A))). - setup() -> DbName = ?tempdb(), {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]), - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"doc1">>}, - {<<"value">>, 666} - ]}), + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"doc1">>}, + {<<"value">>, 666} + ]} + ), - Doc1 = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/doc1">>}, - {<<"shows">>, {[ - {<<"test">>, <<"function(doc, req) { - return { json: { - requested_path: '/' + req.requested_path.join('/'), - path: '/' + req.path.join('/')}};}">>} - ]}}, - {<<"rewrites">>, [ - {[ - {<<"from">>, <<"/">>}, - {<<"to">>, <<"_show/test">>} + Doc1 = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/doc1">>}, + {<<"shows">>, + {[ + {<<"test">>, + <<"function(doc, req) {\n" + " return { json: {\n" + " requested_path: '/' + req.requested_path.join('/'),\n" + " path: '/' + req.path.join('/')}};}">>} + ]}}, + {<<"rewrites">>, [ + {[ + {<<"from">>, <<"/">>}, + {<<"to">>, <<"_show/test">>} + ]} ]} ]} - ]}), + ), {ok, _} = couch_db:update_docs(Db, [Doc, Doc1]), couch_db:close(Db), @@ -54,16 +59,17 @@ teardown({_, DbName}) -> ok = couch_server:delete(?l2b(DbName), []), ok. - vhosts_test_() -> { "Virtual Hosts rewrite tests", { setup, - fun test_util:start_couch/0, fun test_util:stop_couch/1, + fun test_util:start_couch/0, + fun test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_return_database_info/1, fun should_return_revs_info/1, @@ -89,103 +95,146 @@ should_return_database_info({Url, DbName}) -> {JsonBody} = jiffy:decode(Body), ?assert(proplists:is_defined(<<"db_name">>, JsonBody)); Else -> - erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {reason, ?iofmt("Request failed: ~p", [Else])}]}) + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, ?iofmt("Request failed: ~p", [Else])} + ]} + ) end end). should_return_revs_info({Url, DbName}) -> ?_test(begin ok = config:set("vhosts", "example.com", "/" ++ DbName, false), - case test_request:get(Url ++ "/doc1?revs_info=true", [], - [{host_header, "example.com"}]) of + case + test_request:get( + Url ++ "/doc1?revs_info=true", + [], + [{host_header, "example.com"}] + ) + of {ok, _, _, Body} -> {JsonBody} = jiffy:decode(Body), ?assert(proplists:is_defined(<<"_revs_info">>, JsonBody)); Else -> - erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {reason, ?iofmt("Request failed: ~p", [Else])}]}) + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, ?iofmt("Request failed: ~p", [Else])} + ]} + ) end end). should_return_virtual_request_path_field_in_request({Url, DbName}) -> ?_test(begin - ok = config:set("vhosts", "example1.com", - "/" ++ DbName ++ "/_design/doc1/_rewrite/", - false), + ok = config:set( + "vhosts", + "example1.com", + "/" ++ DbName ++ "/_design/doc1/_rewrite/", + false + ), case test_request:get(Url, [], [{host_header, "example1.com"}]) of {ok, _, _, Body} -> {Json} = jiffy:decode(Body), - ?assertEqual(<<"/">>, - proplists:get_value(<<"requested_path">>, Json)); + ?assertEqual( + <<"/">>, + proplists:get_value(<<"requested_path">>, Json) + ); Else -> - erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {reason, ?iofmt("Request failed: ~p", [Else])}]}) + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, ?iofmt("Request failed: ~p", [Else])} + ]} + ) end end). should_return_real_request_path_field_in_request({Url, DbName}) -> ?_test(begin - ok = config:set("vhosts", "example1.com", - "/" ++ DbName ++ "/_design/doc1/_rewrite/", - false), + ok = config:set( + "vhosts", + "example1.com", + "/" ++ DbName ++ "/_design/doc1/_rewrite/", + false + ), case test_request:get(Url, [], [{host_header, "example1.com"}]) of {ok, _, _, Body} -> {Json} = jiffy:decode(Body), Path = ?l2b("/" ++ DbName ++ "/_design/doc1/_show/test"), ?assertEqual(Path, proplists:get_value(<<"path">>, Json)); Else -> - erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {reason, ?iofmt("Request failed: ~p", [Else])}]}) + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, ?iofmt("Request failed: ~p", [Else])} + ]} + ) end end). should_match_wildcard_vhost({Url, DbName}) -> ?_test(begin - ok = config:set("vhosts", "*.example.com", - "/" ++ DbName ++ "/_design/doc1/_rewrite", false), + ok = config:set( + "vhosts", + "*.example.com", + "/" ++ DbName ++ "/_design/doc1/_rewrite", + false + ), case test_request:get(Url, [], [{host_header, "test.example.com"}]) of {ok, _, _, Body} -> {Json} = jiffy:decode(Body), Path = ?l2b("/" ++ DbName ++ "/_design/doc1/_show/test"), ?assertEqual(Path, proplists:get_value(<<"path">>, Json)); Else -> - erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {reason, ?iofmt("Request failed: ~p", [Else])}]}) + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, ?iofmt("Request failed: ~p", [Else])} + ]} + ) end end). should_return_db_info_for_wildcard_vhost_for_custom_db({Url, DbName}) -> ?_test(begin - ok = config:set("vhosts", ":dbname.example1.com", - "/:dbname", false), + ok = config:set( + "vhosts", + ":dbname.example1.com", + "/:dbname", + false + ), Host = DbName ++ ".example1.com", case test_request:get(Url, [], [{host_header, Host}]) of {ok, _, _, Body} -> {JsonBody} = jiffy:decode(Body), ?assert(proplists:is_defined(<<"db_name">>, JsonBody)); Else -> - erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {reason, ?iofmt("Request failed: ~p", [Else])}]}) + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, ?iofmt("Request failed: ~p", [Else])} + ]} + ) end end). should_replace_rewrite_variables_for_db_and_doc({Url, DbName}) -> ?_test(begin - ok = config:set("vhosts",":appname.:dbname.example1.com", - "/:dbname/_design/:appname/_rewrite/", false), + ok = config:set( + "vhosts", + ":appname.:dbname.example1.com", + "/:dbname/_design/:appname/_rewrite/", + false + ), Host = "doc1." ++ DbName ++ ".example1.com", case test_request:get(Url, [], [{host_header, Host}]) of {ok, _, _, Body} -> @@ -193,45 +242,61 @@ should_replace_rewrite_variables_for_db_and_doc({Url, DbName}) -> Path = ?l2b("/" ++ DbName ++ "/_design/doc1/_show/test"), ?assertEqual(Path, proplists:get_value(<<"path">>, Json)); Else -> - erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {reason, ?iofmt("Request failed: ~p", [Else])}]}) + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, ?iofmt("Request failed: ~p", [Else])} + ]} + ) end end). should_return_db_info_for_vhost_with_resource({Url, DbName}) -> ?_test(begin - ok = config:set("vhosts", - "example.com/test", "/" ++ DbName, false), + ok = config:set( + "vhosts", + "example.com/test", + "/" ++ DbName, + false + ), ReqUrl = Url ++ "/test", case test_request:get(ReqUrl, [], [{host_header, "example.com"}]) of {ok, _, _, Body} -> {JsonBody} = jiffy:decode(Body), ?assert(proplists:is_defined(<<"db_name">>, JsonBody)); Else -> - erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {reason, ?iofmt("Request failed: ~p", [Else])}]}) + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, ?iofmt("Request failed: ~p", [Else])} + ]} + ) end end). - should_return_revs_info_for_vhost_with_resource({Url, DbName}) -> ?_test(begin - ok = config:set("vhosts", - "example.com/test", "/" ++ DbName, false), + ok = config:set( + "vhosts", + "example.com/test", + "/" ++ DbName, + false + ), ReqUrl = Url ++ "/test/doc1?revs_info=true", case test_request:get(ReqUrl, [], [{host_header, "example.com"}]) of {ok, _, _, Body} -> {JsonBody} = jiffy:decode(Body), ?assert(proplists:is_defined(<<"_revs_info">>, JsonBody)); Else -> - erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {reason, ?iofmt("Request failed: ~p", [Else])}]}) + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, ?iofmt("Request failed: ~p", [Else])} + ]} + ) end end). @@ -245,27 +310,36 @@ should_return_db_info_for_vhost_with_wildcard_resource({Url, DbName}) -> {JsonBody} = jiffy:decode(Body), ?assert(proplists:is_defined(<<"db_name">>, JsonBody)); Else -> - erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {reason, ?iofmt("Request failed: ~p", [Else])}]}) + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, ?iofmt("Request failed: ~p", [Else])} + ]} + ) end end). should_return_path_for_vhost_with_wildcard_host({Url, DbName}) -> ?_test(begin - ok = config:set("vhosts", "*/test1", - "/" ++ DbName ++ "/_design/doc1/_show/test", - false), + ok = config:set( + "vhosts", + "*/test1", + "/" ++ DbName ++ "/_design/doc1/_show/test", + false + ), case test_request:get(Url ++ "/test1") of {ok, _, _, Body} -> {Json} = jiffy:decode(Body), Path = ?l2b("/" ++ DbName ++ "/_design/doc1/_show/test"), ?assertEqual(Path, proplists:get_value(<<"path">>, Json)); Else -> - erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {reason, ?iofmt("Request failed: ~p", [Else])}]}) + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, ?iofmt("Request failed: ~p", [Else])} + ]} + ) end end). diff --git a/src/couch/test/eunit/couchdb_views_tests.erl b/src/couch/test/eunit/couchdb_views_tests.erl index 06e2f03eb..f4d51bdd0 100644 --- a/src/couch/test/eunit/couchdb_views_tests.erl +++ b/src/couch/test/eunit/couchdb_views_tests.erl @@ -48,10 +48,18 @@ setup_legacy() -> DbDir = config:get("couchdb", "database_dir"), ViewDir = config:get("couchdb", "view_index_dir"), - OldViewFilePath = filename:join([ViewDir, ".test_design", "mrview", - OldViewName]), - NewViewFilePath = filename:join([ViewDir, ".test_design", "mrview", - NewViewName]), + OldViewFilePath = filename:join([ + ViewDir, + ".test_design", + "mrview", + OldViewName + ]), + NewViewFilePath = filename:join([ + ViewDir, + ".test_design", + "mrview", + NewViewName + ]), NewDbFilePath = filename:join([DbDir, DbFileName]), @@ -84,10 +92,12 @@ view_indexes_cleanup_test_() -> "View indexes cleanup", { setup, - fun test_util:start_couch/0, fun test_util:stop_couch/1, + fun test_util:start_couch/0, + fun test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_have_two_indexes_alive_before_deletion/1, fun should_cleanup_index_file_after_ddoc_deletion/1, @@ -102,10 +112,12 @@ view_group_db_leaks_test_() -> "View group db leaks", { setup, - fun test_util:start_couch/0, fun test_util:stop_couch/1, + fun test_util:start_couch/0, + fun test_util:stop_couch/1, { foreach, - fun setup_with_docs/0, fun teardown/1, + fun setup_with_docs/0, + fun teardown/1, [ fun couchdb_1138/1, fun couchdb_1309/1 @@ -136,10 +148,12 @@ backup_restore_test_() -> "Upgrade and bugs related tests", { setup, - fun test_util:start_couch/0, fun test_util:stop_couch/1, + fun test_util:start_couch/0, + fun test_util:stop_couch/1, { foreach, - fun setup_with_docs/0, fun teardown/1, + fun setup_with_docs/0, + fun teardown/1, [ fun should_not_remember_docs_in_index_after_backup_restore/1 ] @@ -147,16 +161,17 @@ backup_restore_test_() -> } }. - upgrade_test_() -> { "Upgrade tests", { setup, - fun test_util:start_couch/0, fun test_util:stop_couch/1, + fun test_util:start_couch/0, + fun test_util:stop_couch/1, { foreach, - fun setup_legacy/0, fun teardown_legacy/1, + fun setup_legacy/0, + fun teardown_legacy/1, [ fun should_upgrade_legacy_view_files/1 ] @@ -184,7 +199,7 @@ should_not_remember_docs_in_index_after_backup_restore(DbName) -> ?assert(has_doc("doc2", Rows1)), ?assert(has_doc("doc3", Rows1)), ?assertNot(has_doc("doc666", Rows1)) - end). + end). should_upgrade_legacy_view_files({DbName, Files}) -> ?_test(begin @@ -206,21 +221,23 @@ should_upgrade_legacy_view_files({DbName, Files}) -> % add doc to trigger update DocUrl = db_url(DbName) ++ "/bar", {ok, _, _, _} = test_request:put( - DocUrl, [{"Content-Type", "application/json"}], <<"{\"a\":4}">>), + DocUrl, [{"Content-Type", "application/json"}], <<"{\"a\":4}">> + ), % query view for expected results Rows1 = query_view(DbName, "test", "test"), ?assertEqual(4, length(Rows1)), % ensure new header - timer:sleep(2000), % have to wait for awhile to upgrade the index + + % have to wait for awhile to upgrade the index + timer:sleep(2000), NewHeader = read_header(NewViewFilePath), ?assertMatch(#mrheader{}, NewHeader), NewViewStatus = hd(NewHeader#mrheader.view_states), ?assertEqual(3, tuple_size(NewViewStatus)) end). - should_have_two_indexes_alive_before_deletion({DbName, _}) -> view_cleanup(DbName), ?_assertEqual(2, count_index_files(DbName)). @@ -230,7 +247,7 @@ should_cleanup_index_file_after_ddoc_deletion({DbName, {FooRev, _}}) -> view_cleanup(DbName), ?_assertEqual(1, count_index_files(DbName)). -should_cleanup_all_index_files({DbName, {FooRev, BooRev}})-> +should_cleanup_all_index_files({DbName, {FooRev, BooRev}}) -> delete_design_doc(DbName, <<"_design/foo">>, FooRev), delete_design_doc(DbName, <<"_design/boo">>, BooRev), view_cleanup(DbName), @@ -239,7 +256,8 @@ should_cleanup_all_index_files({DbName, {FooRev, BooRev}})-> couchdb_1138(DbName) -> ?_test(begin {ok, IndexerPid} = couch_index_server:get_index( - couch_mrview_index, DbName, <<"_design/foo">>), + couch_mrview_index, DbName, <<"_design/foo">> + ), ?assert(is_pid(IndexerPid)), ?assert(is_process_alive(IndexerPid)), ?assertEqual(2, count_users(DbName)), @@ -277,7 +295,8 @@ couchdb_1138(DbName) -> couchdb_1309(DbName) -> ?_test(begin {ok, IndexerPid} = couch_index_server:get_index( - couch_mrview_index, DbName, <<"_design/foo">>), + couch_mrview_index, DbName, <<"_design/foo">> + ), ?assert(is_pid(IndexerPid)), ?assert(is_process_alive(IndexerPid)), ?assertEqual(2, count_users(DbName)), @@ -292,18 +311,21 @@ couchdb_1309(DbName) -> ?assert(is_process_alive(IndexerPid)), - update_design_doc(DbName, <<"_design/foo">>, <<"bar">>), + update_design_doc(DbName, <<"_design/foo">>, <<"bar">>), {ok, NewIndexerPid} = couch_index_server:get_index( - couch_mrview_index, DbName, <<"_design/foo">>), + couch_mrview_index, DbName, <<"_design/foo">> + ), ?assert(is_pid(NewIndexerPid)), ?assert(is_process_alive(NewIndexerPid)), ?assertNotEqual(IndexerPid, NewIndexerPid), - UserCnt = case count_users(DbName) of - N when N > 2 -> - timer:sleep(1000), - count_users(DbName); - N -> N - end, + UserCnt = + case count_users(DbName) of + N when N > 2 -> + timer:sleep(1000), + count_users(DbName); + N -> + N + end, ?assertEqual(2, UserCnt), Rows1 = query_view(DbName, "foo", "bar", ok), @@ -312,15 +334,20 @@ couchdb_1309(DbName) -> check_rows_value(Rows2, 1), ?assertEqual(4, length(Rows2)), - ok = stop_indexer( %% FIXME we need to grab monitor earlier - fun() -> ok end, - IndexerPid, ?LINE, - "old view group is not dead after ddoc update"), + %% FIXME we need to grab monitor earlier + ok = stop_indexer( + fun() -> ok end, + IndexerPid, + ?LINE, + "old view group is not dead after ddoc update" + ), ok = stop_indexer( - fun() -> couch_server:delete(DbName, [?ADMIN_USER]) end, - NewIndexerPid, ?LINE, - "new view group did not die after DB deletion") + fun() -> couch_server:delete(DbName, [?ADMIN_USER]) end, + NewIndexerPid, + ?LINE, + "new view group did not die after DB deletion" + ) end). couchdb_1283() -> @@ -328,41 +355,54 @@ couchdb_1283() -> ok = config:set("couchdb", "max_dbs_open", "3", false), {ok, MDb1} = couch_db:create(?tempdb(), [?ADMIN_CTX]), - DDoc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/foo">>}, - {<<"language">>, <<"javascript">>}, - {<<"views">>, {[ - {<<"foo">>, {[ - {<<"map">>, <<"function(doc) { emit(doc._id, null); }">>} - ]}}, - {<<"foo2">>, {[ - {<<"map">>, <<"function(doc) { emit(doc._id, null); }">>} - ]}}, - {<<"foo3">>, {[ - {<<"map">>, <<"function(doc) { emit(doc._id, null); }">>} - ]}}, - {<<"foo4">>, {[ - {<<"map">>, <<"function(doc) { emit(doc._id, null); }">>} - ]}}, - {<<"foo5">>, {[ - {<<"map">>, <<"function(doc) { emit(doc._id, null); }">>} - ]}} - ]}} - ]}), + DDoc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/foo">>}, + {<<"language">>, <<"javascript">>}, + {<<"views">>, + {[ + {<<"foo">>, + {[ + {<<"map">>, <<"function(doc) { emit(doc._id, null); }">>} + ]}}, + {<<"foo2">>, + {[ + {<<"map">>, <<"function(doc) { emit(doc._id, null); }">>} + ]}}, + {<<"foo3">>, + {[ + {<<"map">>, <<"function(doc) { emit(doc._id, null); }">>} + ]}}, + {<<"foo4">>, + {[ + {<<"map">>, <<"function(doc) { emit(doc._id, null); }">>} + ]}}, + {<<"foo5">>, + {[ + {<<"map">>, <<"function(doc) { emit(doc._id, null); }">>} + ]}} + ]}} + ]} + ), {ok, _} = couch_db:update_doc(MDb1, DDoc, []), ok = populate_db(MDb1, 100, 100), query_view(couch_db:name(MDb1), "foo", "foo"), ok = couch_db:close(MDb1), {ok, Pid} = couch_index_server:get_index( - couch_mrview_index, couch_db:name(MDb1), <<"_design/foo">>), + couch_mrview_index, couch_db:name(MDb1), <<"_design/foo">> + ), % Start and pause compacton WaitRef = erlang:make_ref(), meck:expect(couch_mrview_index, compact, fun(Db, State, Opts) -> - receive {WaitRef, From, init} -> ok end, + receive + {WaitRef, From, init} -> ok + end, From ! {WaitRef, inited}, - receive {WaitRef, go} -> ok end, + receive + {WaitRef, go} -> ok + end, meck:passthrough([Db, State, Opts]) end), @@ -373,7 +413,9 @@ couchdb_1283() -> % Make sure that our compactor is waiting for us % before we continue our assertions CPid ! {WaitRef, self(), init}, - receive {WaitRef, inited} -> ok end, + receive + {WaitRef, inited} -> ok + end, % Make sure that a compaction process takes a monitor % on the database's main_pid @@ -382,64 +424,74 @@ couchdb_1283() -> % Finish compaction to and make sure the monitor % disappears CPid ! {WaitRef, go}, - wait_for_process_shutdown(CRef, normal, - {reason, "Failure compacting view group"}), + wait_for_process_shutdown( + CRef, + normal, + {reason, "Failure compacting view group"} + ), % Make sure that the monitor was removed ?assertEqual(false, lists:member(CPid, couch_db:monitored_by(MDb1))) end). wait_for_process_shutdown(Pid, ExpectedReason, Error) -> - receive - {'DOWN', Pid, process, _, Reason} -> - ?assertEqual(ExpectedReason, Reason) - after ?TIMEOUT -> - erlang:error( - {assertion_failed, - [{module, ?MODULE}, {line, ?LINE}, Error]}) - end. - + receive + {'DOWN', Pid, process, _, Reason} -> + ?assertEqual(ExpectedReason, Reason) + after ?TIMEOUT -> + erlang:error( + {assertion_failed, [{module, ?MODULE}, {line, ?LINE}, Error]} + ) + end. create_doc(DbName, DocId) when is_list(DocId) -> create_doc(DbName, ?l2b(DocId)); create_doc(DbName, DocId) when is_binary(DocId) -> {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]), - Doc666 = couch_doc:from_json_obj({[ - {<<"_id">>, DocId}, - {<<"value">>, 999} - ]}), + Doc666 = couch_doc:from_json_obj( + {[ + {<<"_id">>, DocId}, + {<<"value">>, 999} + ]} + ), {ok, _} = couch_db:update_docs(Db, [Doc666]), couch_db:close(Db). create_docs(DbName) -> {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]), - Doc1 = couch_doc:from_json_obj({[ - {<<"_id">>, <<"doc1">>}, - {<<"value">>, 1} - - ]}), - Doc2 = couch_doc:from_json_obj({[ - {<<"_id">>, <<"doc2">>}, - {<<"value">>, 2} - - ]}), - Doc3 = couch_doc:from_json_obj({[ - {<<"_id">>, <<"doc3">>}, - {<<"value">>, 3} - - ]}), + Doc1 = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"doc1">>}, + {<<"value">>, 1} + ]} + ), + Doc2 = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"doc2">>}, + {<<"value">>, 2} + ]} + ), + Doc3 = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"doc3">>}, + {<<"value">>, 3} + ]} + ), {ok, _} = couch_db:update_docs(Db, [Doc1, Doc2, Doc3]), couch_db:close(Db). populate_db(Db, BatchSize, N) when N > 0 -> Docs = lists:map( fun(_) -> - couch_doc:from_json_obj({[ - {<<"_id">>, couch_uuids:new()}, - {<<"value">>, base64:encode(crypto:strong_rand_bytes(1000))} - ]}) + couch_doc:from_json_obj( + {[ + {<<"_id">>, couch_uuids:new()}, + {<<"value">>, base64:encode(crypto:strong_rand_bytes(1000))} + ]} + ) end, - lists:seq(1, BatchSize)), + lists:seq(1, BatchSize) + ), {ok, _} = couch_db:update_docs(Db, Docs, []), populate_db(Db, BatchSize, N - length(Docs)); populate_db(_Db, _, _) -> @@ -447,15 +499,19 @@ populate_db(_Db, _, _) -> create_design_doc(DbName, DDName, ViewName) -> {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]), - DDoc = couch_doc:from_json_obj({[ - {<<"_id">>, DDName}, - {<<"language">>, <<"javascript">>}, - {<<"views">>, {[ - {ViewName, {[ - {<<"map">>, <<"function(doc) { emit(doc.value, null); }">>} - ]}} - ]}} - ]}), + DDoc = couch_doc:from_json_obj( + {[ + {<<"_id">>, DDName}, + {<<"language">>, <<"javascript">>}, + {<<"views">>, + {[ + {ViewName, + {[ + {<<"map">>, <<"function(doc) { emit(doc.value, null); }">>} + ]}} + ]}} + ]} + ), {ok, Rev} = couch_db:update_doc(Db, DDoc, []), couch_db:close(Db), Rev. @@ -465,27 +521,33 @@ update_design_doc(DbName, DDName, ViewName) -> {ok, Doc} = couch_db:open_doc(Db, DDName, [?ADMIN_CTX]), {Props} = couch_doc:to_json_obj(Doc, []), Rev = couch_util:get_value(<<"_rev">>, Props), - DDoc = couch_doc:from_json_obj({[ - {<<"_id">>, DDName}, - {<<"_rev">>, Rev}, - {<<"language">>, <<"javascript">>}, - {<<"views">>, {[ - {ViewName, {[ - {<<"map">>, <<"function(doc) { emit(doc.value, 1); }">>} - ]}} - ]}} - ]}), + DDoc = couch_doc:from_json_obj( + {[ + {<<"_id">>, DDName}, + {<<"_rev">>, Rev}, + {<<"language">>, <<"javascript">>}, + {<<"views">>, + {[ + {ViewName, + {[ + {<<"map">>, <<"function(doc) { emit(doc.value, 1); }">>} + ]}} + ]}} + ]} + ), {ok, NewRev} = couch_db:update_doc(Db, DDoc, [?ADMIN_CTX]), couch_db:close(Db), NewRev. delete_design_doc(DbName, DDName, Rev) -> {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]), - DDoc = couch_doc:from_json_obj({[ - {<<"_id">>, DDName}, - {<<"_rev">>, couch_doc:rev_to_str(Rev)}, - {<<"_deleted">>, true} - ]}), + DDoc = couch_doc:from_json_obj( + {[ + {<<"_id">>, DDName}, + {<<"_rev">>, couch_doc:rev_to_str(Rev)}, + {<<"_deleted">>, true} + ]} + ), {ok, _} = couch_db:update_doc(Db, DDoc, [Rev]), couch_db:close(Db). @@ -499,11 +561,12 @@ query_view(DbName, DDoc, View) -> query_view(DbName, DDoc, View, Stale) -> {ok, Code, _Headers, Body} = test_request:get( - db_url(DbName) ++ "/_design/" ++ DDoc ++ "/_view/" ++ View - ++ case Stale of - false -> []; - _ -> "?stale=" ++ atom_to_list(Stale) - end), + db_url(DbName) ++ "/_design/" ++ DDoc ++ "/_view/" ++ View ++ + case Stale of + false -> []; + _ -> "?stale=" ++ atom_to_list(Stale) + end + ), ?assertEqual(200, Code), {Props} = jiffy:decode(Body), couch_util:get_value(<<"rows">>, Props, []). @@ -512,7 +575,9 @@ check_rows_value(Rows, Value) -> lists:foreach( fun({Row}) -> ?assertEqual(Value, couch_util:get_value(<<"value">>, Row)) - end, Rows). + end, + Rows + ). view_cleanup(DbName) -> {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]), @@ -531,8 +596,12 @@ count_users(DbName) -> count_index_files(DbName) -> % call server to fetch the index files RootDir = config:get("couchdb", "view_index_dir"), - length(filelib:wildcard(RootDir ++ "/." ++ - binary_to_list(DbName) ++ "_design"++"/mrview/*")). + length( + filelib:wildcard( + RootDir ++ "/." ++ + binary_to_list(DbName) ++ "_design" ++ "/mrview/*" + ) + ). has_doc(DocId1, Rows) -> DocId = iolist_to_binary(DocId1), @@ -542,10 +611,11 @@ backup_db_file(DbName) -> {ok, Db} = couch_db:open_int(DbName, []), try SrcPath = couch_db:get_filepath(Db), - Src = if - is_list(SrcPath) -> SrcPath; - true -> binary_to_list(SrcPath) - end, + Src = + if + is_list(SrcPath) -> SrcPath; + true -> binary_to_list(SrcPath) + end, ok = copy_tree(Src, Src ++ ".backup") after couch_db:close(Db) @@ -559,17 +629,21 @@ restore_backup_db_file(DbName) -> exit(DbPid, shutdown), ok = copy_tree(Src ++ ".backup", Src), - test_util:wait(fun() -> - case couch_server:open(DbName, [{timeout, ?TIMEOUT}]) of - {ok, WaitDb} -> - case couch_db:get_pid(WaitDb) == DbPid of - true -> wait; - false -> ok - end; - Else -> - Else - end - end, ?TIMEOUT, ?DELAY). + test_util:wait( + fun() -> + case couch_server:open(DbName, [{timeout, ?TIMEOUT}]) of + {ok, WaitDb} -> + case couch_db:get_pid(WaitDb) == DbPid of + true -> wait; + false -> ok + end; + Else -> + Else + end + end, + ?TIMEOUT, + ?DELAY + ). compact_db(DbName) -> {ok, Db} = couch_db:open_int(DbName, []), @@ -578,20 +652,23 @@ compact_db(DbName) -> wait_db_compact_done(DbName, ?WAIT_DELAY_COUNT). wait_db_compact_done(_DbName, 0) -> - erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {reason, "DB compaction failed to finish"}]}); + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, "DB compaction failed to finish"} + ]} + ); wait_db_compact_done(DbName, N) -> {ok, Db} = couch_db:open_int(DbName, []), ok = couch_db:close(Db), CompactorPid = couch_db:get_compactor_pid(Db), case is_pid(CompactorPid) of - false -> - ok; - true -> - ok = timer:sleep(?DELAY), - wait_db_compact_done(DbName, N - 1) + false -> + ok; + true -> + ok = timer:sleep(?DELAY), + wait_db_compact_done(DbName, N - 1) end. compact_view_group(DbName, DDocId) when is_list(DDocId) -> @@ -601,13 +678,17 @@ compact_view_group(DbName, DDocId) when is_binary(DDocId) -> wait_view_compact_done(DbName, DDocId, 10). wait_view_compact_done(_DbName, _DDocId, 0) -> - erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {reason, "DB compaction failed to finish"}]}); + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, "DB compaction failed to finish"} + ]} + ); wait_view_compact_done(DbName, DDocId, N) -> {ok, Code, _Headers, Body} = test_request:get( - db_url(DbName) ++ "/" ++ ?b2l(DDocId) ++ "/_info"), + db_url(DbName) ++ "/" ++ ?b2l(DDocId) ++ "/_info" + ), ?assertEqual(200, Code), {Info} = jiffy:decode(Body), {IndexInfo} = couch_util:get_value(<<"view_index">>, Info), @@ -628,13 +709,16 @@ read_header(File) -> stop_indexer(StopFun, Pid, Line, Reason) -> case test_util:stop_sync(Pid, StopFun) of - timeout -> - erlang:error( - {assertion_failed, - [{module, ?MODULE}, {line, Line}, - {reason, Reason}]}); - ok -> - ok + timeout -> + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, Line}, + {reason, Reason} + ]} + ); + ok -> + ok end. wait_indexer(IndexerPid) -> diff --git a/src/couch/test/eunit/global_changes_tests.erl b/src/couch/test/eunit/global_changes_tests.erl index 4392aafac..92964bb74 100644 --- a/src/couch/test/eunit/global_changes_tests.erl +++ b/src/couch/test/eunit/global_changes_tests.erl @@ -35,7 +35,7 @@ http_create_db(Name) -> {ok, Status, _, _} = test_request:put(db_url(Name), [?AUTH], ""), true = lists:member(Status, [201, 202]), ok. - + http_delete_db(Name) -> {ok, Status, _, _} = test_request:delete(db_url(Name), [?AUTH]), true = lists:member(Status, [200, 202]), @@ -75,7 +75,8 @@ check_response() -> "Check response", { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_return_correct_response_on_create/1, fun should_return_correct_response_on_update/1 @@ -105,9 +106,11 @@ should_return_correct_response_on_update({Host, DbName}) -> create_doc(Host, DbName, Id) -> Headers = [?AUTH], Url = Host ++ "/" ++ escape(DbName) ++ "/" ++ escape(Id), - Body = jiffy:encode({[ - {key, "value"} - ]}), + Body = jiffy:encode( + {[ + {key, "value"} + ]} + ), {ok, Status, _Headers, _Body} = test_request:put(Url, Headers, Body), ?assert(Status =:= 201 orelse Status =:= 202), timer:sleep(1000), @@ -118,10 +121,12 @@ update_doc(Host, DbName, Id, Value) -> Url = Host ++ "/" ++ escape(DbName) ++ "/" ++ escape(Id), {ok, 200, _Headers0, BinBody} = test_request:get(Url, Headers), [Rev] = decode_response(BinBody, [<<"_rev">>]), - Body = jiffy:encode({[ - {key, Value}, - {'_rev', Rev} - ]}), + Body = jiffy:encode( + {[ + {key, Value}, + {'_rev', Rev} + ]} + ), {ok, Status, _Headers1, _Body} = test_request:put(Url, Headers, Body), ?assert(Status =:= 201 orelse Status =:= 202), timer:sleep(1000), @@ -145,7 +150,7 @@ decode_response(BinBody, ToDecode) -> add_admin(User, Pass) -> Hashed = couch_passwords:hash_admin_password(Pass), - config:set("admins", User, ?b2l(Hashed), _Persist=false). + config:set("admins", User, ?b2l(Hashed), _Persist = false). delete_admin(User) -> config:delete("admins", User, false). diff --git a/src/couch/test/eunit/json_stream_parse_tests.erl b/src/couch/test/eunit/json_stream_parse_tests.erl index e690d7728..ab26be725 100644 --- a/src/couch/test/eunit/json_stream_parse_tests.erl +++ b/src/couch/test/eunit/json_stream_parse_tests.erl @@ -14,83 +14,88 @@ -include_lib("couch/include/couch_eunit.hrl"). --define(CASES, - [ - {1, "1", "integer numeric literial"}, - {3.1416, "3.14160", "float numeric literal"}, % text representation may truncate, trail zeroes - {-1, "-1", "negative integer numeric literal"}, - {-3.1416, "-3.14160", "negative float numeric literal"}, - {12.0e10, "1.20000e+11", "float literal in scientific notation"}, - {1.234E+10, "1.23400e+10", "another float literal in scientific notation"}, - {-1.234E-10, "-1.23400e-10", "negative float literal in scientific notation"}, - {10.0, "1.0e+01", "yet another float literal in scientific notation"}, - {123.456, "1.23456E+2", "yet another float literal in scientific notation"}, - {10.0, "1e1", "yet another float literal in scientific notation"}, - {<<"foo">>, "\"foo\"", "string literal"}, - {<<"foo", 5, "bar">>, "\"foo\\u0005bar\"", "string literal with \\u0005"}, - {<<"">>, "\"\"", "empty string literal"}, - {<<"\n\n\n">>, "\"\\n\\n\\n\"", "only new lines literal"}, - {<<"\" \b\f\r\n\t\"">>, "\"\\\" \\b\\f\\r\\n\\t\\\"\"", - "only white spaces string literal"}, - {null, "null", "null literal"}, - {true, "true", "true literal"}, - {false, "false", "false literal"}, - {<<"null">>, "\"null\"", "null string literal"}, - {<<"true">>, "\"true\"", "true string literal"}, - {<<"false">>, "\"false\"", "false string literal"}, - {{[]}, "{}", "empty object literal"}, - {{[{<<"foo">>, <<"bar">>}]}, "{\"foo\":\"bar\"}", - "simple object literal"}, - {{[{<<"foo">>, <<"bar">>}, {<<"baz">>, 123}]}, - "{\"foo\":\"bar\",\"baz\":123}", "another simple object literal"}, - {[], "[]", "empty array literal"}, - {[[]], "[[]]", "empty array literal inside a single element array literal"}, - {[1, <<"foo">>], "[1,\"foo\"]", "simple non-empty array literal"}, - {[1199344435545.0, 1], "[1199344435545.0,1]", - "another simple non-empty array literal"}, - {[false, true, 321, null], "[false, true, 321, null]", "array of literals"}, - {{[{<<"foo">>, [123]}]}, "{\"foo\":[123]}", - "object literal with an array valued property"}, - {{[{<<"foo">>, {[{<<"bar">>, true}]}}]}, - "{\"foo\":{\"bar\":true}}", "nested object literal"}, - {{[{<<"foo">>, []}, {<<"bar">>, {[{<<"baz">>, true}]}}, - {<<"alice">>, <<"bob">>}]}, - "{\"foo\":[],\"bar\":{\"baz\":true},\"alice\":\"bob\"}", - "complex object literal"}, - {[-123, <<"foo">>, {[{<<"bar">>, []}]}, null], - "[-123,\"foo\",{\"bar\":[]},null]", - "complex array literal"} - ] -). - +-define(CASES, [ + {1, "1", "integer numeric literial"}, + % text representation may truncate, trail zeroes + {3.1416, "3.14160", "float numeric literal"}, + {-1, "-1", "negative integer numeric literal"}, + {-3.1416, "-3.14160", "negative float numeric literal"}, + {12.0e10, "1.20000e+11", "float literal in scientific notation"}, + {1.234E+10, "1.23400e+10", "another float literal in scientific notation"}, + {-1.234E-10, "-1.23400e-10", "negative float literal in scientific notation"}, + {10.0, "1.0e+01", "yet another float literal in scientific notation"}, + {123.456, "1.23456E+2", "yet another float literal in scientific notation"}, + {10.0, "1e1", "yet another float literal in scientific notation"}, + {<<"foo">>, "\"foo\"", "string literal"}, + {<<"foo", 5, "bar">>, "\"foo\\u0005bar\"", "string literal with \\u0005"}, + {<<"">>, "\"\"", "empty string literal"}, + {<<"\n\n\n">>, "\"\\n\\n\\n\"", "only new lines literal"}, + {<<"\" \b\f\r\n\t\"">>, "\"\\\" \\b\\f\\r\\n\\t\\\"\"", "only white spaces string literal"}, + {null, "null", "null literal"}, + {true, "true", "true literal"}, + {false, "false", "false literal"}, + {<<"null">>, "\"null\"", "null string literal"}, + {<<"true">>, "\"true\"", "true string literal"}, + {<<"false">>, "\"false\"", "false string literal"}, + {{[]}, "{}", "empty object literal"}, + {{[{<<"foo">>, <<"bar">>}]}, "{\"foo\":\"bar\"}", "simple object literal"}, + { + {[{<<"foo">>, <<"bar">>}, {<<"baz">>, 123}]}, + "{\"foo\":\"bar\",\"baz\":123}", + "another simple object literal" + }, + {[], "[]", "empty array literal"}, + {[[]], "[[]]", "empty array literal inside a single element array literal"}, + {[1, <<"foo">>], "[1,\"foo\"]", "simple non-empty array literal"}, + {[1199344435545.0, 1], "[1199344435545.0,1]", "another simple non-empty array literal"}, + {[false, true, 321, null], "[false, true, 321, null]", "array of literals"}, + {{[{<<"foo">>, [123]}]}, "{\"foo\":[123]}", "object literal with an array valued property"}, + {{[{<<"foo">>, {[{<<"bar">>, true}]}}]}, "{\"foo\":{\"bar\":true}}", "nested object literal"}, + { + {[ + {<<"foo">>, []}, + {<<"bar">>, {[{<<"baz">>, true}]}}, + {<<"alice">>, <<"bob">>} + ]}, + "{\"foo\":[],\"bar\":{\"baz\":true},\"alice\":\"bob\"}", + "complex object literal" + }, + { + [-123, <<"foo">>, {[{<<"bar">>, []}]}, null], + "[-123,\"foo\",{\"bar\":[]},null]", + "complex array literal" + } +]). raw_json_input_test_() -> Tests = lists:map( fun({EJson, JsonString, Desc}) -> - {Desc, - ?_assert(equiv(EJson, json_stream_parse:to_ejson(JsonString)))} - end, ?CASES), + {Desc, ?_assert(equiv(EJson, json_stream_parse:to_ejson(JsonString)))} + end, + ?CASES + ), {"Tests with raw JSON string as the input", Tests}. one_byte_data_fun_test_() -> Tests = lists:map( fun({EJson, JsonString, Desc}) -> DataFun = fun() -> single_byte_data_fun(JsonString) end, - {Desc, - ?_assert(equiv(EJson, json_stream_parse:to_ejson(DataFun)))} - end, ?CASES), + {Desc, ?_assert(equiv(EJson, json_stream_parse:to_ejson(DataFun)))} + end, + ?CASES + ), {"Tests with a 1 byte output data function as the input", Tests}. test_multiple_bytes_data_fun_test_() -> Tests = lists:map( fun({EJson, JsonString, Desc}) -> DataFun = fun() -> multiple_bytes_data_fun(JsonString) end, - {Desc, - ?_assert(equiv(EJson, json_stream_parse:to_ejson(DataFun)))} - end, ?CASES), + {Desc, ?_assert(equiv(EJson, json_stream_parse:to_ejson(DataFun)))} + end, + ?CASES + ), {"Tests with a multiple bytes output data function as the input", Tests}. - %% Test for equivalence of Erlang terms. %% Due to arbitrary order of construction, equivalent objects might %% compare unequal as erlang terms, so we need to carefully recurse @@ -120,7 +125,8 @@ equiv_object(Props1, Props2) -> fun({{K1, V1}, {K2, V2}}) -> equiv(K1, K2) andalso equiv(V1, V2) end, - Pairs). + Pairs + ). %% Recursively compare tuple elements for equivalence. equiv_list([], []) -> @@ -147,5 +153,5 @@ split(L, N) -> take(0, L, Acc) -> {lists:reverse(Acc), L}; -take(N, [H|L], Acc) -> +take(N, [H | L], Acc) -> take(N - 1, L, [H | Acc]). diff --git a/src/couch/test/eunit/test_web.erl b/src/couch/test/eunit/test_web.erl index b1b3e65c9..8998dad52 100644 --- a/src/couch/test/eunit/test_web.erl +++ b/src/couch/test/eunit/test_web.erl @@ -73,18 +73,18 @@ terminate(_Reason, _State) -> stop() -> mochiweb_http:stop(?SERVER). - handle_call({check_request, Req}, _From, State) when is_function(State, 1) -> - Resp2 = case (catch State(Req)) of - {ok, Resp} -> - {reply, {ok, Resp}, was_ok}; - {raw, Resp} -> - {reply, {raw, Resp}, was_ok}; - {chunked, Resp} -> - {reply, {chunked, Resp}, was_ok}; - Error -> - {reply, {error, Error}, not_ok} - end, + Resp2 = + case (catch State(Req)) of + {ok, Resp} -> + {reply, {ok, Resp}, was_ok}; + {raw, Resp} -> + {reply, {raw, Resp}, was_ok}; + {chunked, Resp} -> + {reply, {chunked, Resp}, was_ok}; + Error -> + {reply, {error, Error}, not_ok} + end, Req:cleanup(), Resp2; handle_call({check_request, _Req}, _From, _State) -> diff --git a/src/couch_dist/src/couch_dist.erl b/src/couch_dist/src/couch_dist.erl index 21afd0a91..9a6b26d07 100644 --- a/src/couch_dist/src/couch_dist.erl +++ b/src/couch_dist/src/couch_dist.erl @@ -12,77 +12,77 @@ -module(couch_dist). --export([childspecs/0, listen/2, accept/1, accept_connection/5, - setup/5, close/1, select/1, is_node_name/1]). +-export([ + childspecs/0, + listen/2, + accept/1, + accept_connection/5, + setup/5, + close/1, + select/1, + is_node_name/1 +]). % Just for tests -export([no_tls/1, get_init_args/0]). - childspecs() -> - {ok, [{ssl_dist_sup, {ssl_dist_sup, start_link, []}, - permanent, infinity, supervisor, [ssl_dist_sup]}]}. - + {ok, [ + {ssl_dist_sup, {ssl_dist_sup, start_link, []}, permanent, infinity, supervisor, [ + ssl_dist_sup + ]} + ]}. listen(Name, Host) -> - NodeName = case is_atom(Name) of - true -> atom_to_list(Name); - false -> Name - end, + NodeName = + case is_atom(Name) of + true -> atom_to_list(Name); + false -> Name + end, Mod = inet_dist(NodeName ++ "@" ++ Host), Mod:listen(NodeName, Host). - accept(Listen) -> Mod = inet_dist(node()), Mod:accept(Listen). - accept_connection(AcceptPid, DistCtrl, MyNode, Allowed, SetupTime) -> Mod = inet_dist(MyNode), Mod:accept_connection(AcceptPid, DistCtrl, MyNode, Allowed, SetupTime). - setup(Node, Type, MyNode, LongOrShortNames, SetupTime) -> Mod = inet_dist(Node), Mod:setup(Node, Type, MyNode, LongOrShortNames, SetupTime). - close(Socket) -> inet_tls_dist:close(Socket). - select(Node) -> inet_tls_dist:select(Node). - is_node_name(Node) -> inet_tls_dist:is_node_name(Node). - get_init_args() -> init:get_argument(couch_dist). - inet_dist(Node) -> case no_tls(Node) of true -> inet_tcp_dist; false -> inet_tls_dist end. - no_tls(NodeName) when is_atom(NodeName) -> no_tls(atom_to_list(NodeName)); - no_tls(NodeName) when is_list(NodeName) -> case ?MODULE:get_init_args() of {ok, Args} -> GlobPatterns = [V || [K, V] <- Args, K == "no_tls"], lists:any(fun(P) -> match(NodeName, P) end, GlobPatterns); - error -> false + error -> + false end. - match(_NodeName, "true") -> true; match(_NodeName, "false") -> @@ -95,11 +95,9 @@ match(NodeName, Pattern) -> end, re:run(NodeName, RE) /= nomatch. - to_re(GlobPattern) -> re:compile([$^, lists:flatmap(fun glob_re/1, GlobPattern), $$]). - glob_re($*) -> ".*"; glob_re($?) -> diff --git a/src/couch_dist/test/eunit/couch_dist_tests.erl b/src/couch_dist/test/eunit/couch_dist_tests.erl index 55e9af376..abe5ff572 100644 --- a/src/couch_dist/test/eunit/couch_dist_tests.erl +++ b/src/couch_dist/test/eunit/couch_dist_tests.erl @@ -14,7 +14,6 @@ -include_lib("eunit/include/eunit.hrl"). - no_tls_test_() -> { "test couch_dist no_tls/1", @@ -33,19 +32,17 @@ no_tls_test_() -> } }. - mock_get_init_args(Reply) -> meck:expect(couch_dist, get_init_args, fun() -> Reply end). - no_tls_test_with_true() -> ?_test( begin mock_get_init_args({ok, [["no_tls", "true"]]}), ?assert(couch_dist:no_tls('abc123')), ?assert(couch_dist:no_tls("123abd")) - end). - + end + ). no_tls_test_with_false() -> ?_test( @@ -53,8 +50,8 @@ no_tls_test_with_false() -> mock_get_init_args({ok, [["no_tls", "false"]]}), ?assertNot(couch_dist:no_tls('abc123')), ?assertNot(couch_dist:no_tls("123abc")) - end). - + end + ). no_tls_test_with_character() -> ?_test( @@ -62,8 +59,8 @@ no_tls_test_with_character() -> mock_get_init_args({ok, [["no_tls", "node@127.0.0.1"]]}), ?assert(couch_dist:no_tls('node@127.0.0.1')), ?assert(couch_dist:no_tls("node@127.0.0.1")) - end). - + end + ). no_tls_test_with_wildcard() -> ?_test( @@ -74,8 +71,8 @@ no_tls_test_with_wildcard() -> ?assert(couch_dist:no_tls("a2")), ?assertNot(couch_dist:no_tls('a')), ?assertNot(couch_dist:no_tls("2")) - end). - + end + ). no_tls_test_with_question_mark() -> ?_test( @@ -85,8 +82,8 @@ no_tls_test_with_question_mark() -> ?assert(couch_dist:no_tls("ab2")), ?assertNot(couch_dist:no_tls('a2')), ?assertNot(couch_dist:no_tls("a")) - end). - + end + ). no_tls_test_with_error() -> ?_test( @@ -94,4 +91,5 @@ no_tls_test_with_error() -> mock_get_init_args(error), ?assertNot(couch_dist:no_tls('abc123')), ?assertNot(couch_dist:no_tls("123abc")) - end). + end + ). diff --git a/src/couch_epi/src/couch_epi.erl b/src/couch_epi/src/couch_epi.erl index 0e5c233ab..c708e5a0b 100644 --- a/src/couch_epi/src/couch_epi.erl +++ b/src/couch_epi/src/couch_epi.erl @@ -17,9 +17,14 @@ %% queries and introspection -export([ - dump/1, get/2, get_value/3, - by_key/1, by_key/2, by_source/1, by_source/2, - keys/1, subscribers/1]). + dump/1, + get/2, + get_value/3, + by_key/1, by_key/2, + by_source/1, by_source/2, + keys/1, + subscribers/1 +]). %% apply -export([apply/5, decide/5]). @@ -51,19 +56,18 @@ -opaque handle() :: module(). --type apply_opt() - :: ignore_errors - | concurrent - | pipe. +-type apply_opt() :: + ignore_errors + | concurrent + | pipe. -type apply_opts() :: [apply_opt()]. --type data_spec() - :: {static_module, module()} - | {callback_module, module()} - | {priv_file, FileName :: string()} - | {file, FileName :: string()}. - +-type data_spec() :: + {static_module, module()} + | {callback_module, module()} + | {priv_file, FileName :: string()} + | {file, FileName :: string()}. %% ------------------------------------------------------------------ %% API Function Definitions @@ -87,93 +91,109 @@ get(Handle, Key) when Handle /= undefined -> get_value(Handle, Subscriber, Key) when Handle /= undefined -> couch_epi_data_gen:get(Handle, Subscriber, Key). - -spec by_key(Handle :: handle()) -> [{Key :: key(), [{Source :: app(), properties()}]}]. by_key(Handle) when Handle /= undefined -> couch_epi_data_gen:by_key(Handle). - -spec by_key(Handle :: handle(), Key :: key()) -> [{Source :: app(), properties()}]. by_key(Handle, Key) when Handle /= undefined -> couch_epi_data_gen:by_key(Handle, Key). - -spec by_source(Handle :: handle()) -> [{Source :: app(), [{Key :: key(), properties()}]}]. by_source(Handle) when Handle /= undefined -> couch_epi_data_gen:by_source(Handle). - -spec by_source(Handle :: handle(), Subscriber :: app()) -> [{Key :: key(), properties()}]. by_source(Handle, Subscriber) when Handle /= undefined -> couch_epi_data_gen:by_source(Handle, Subscriber). - -spec keys(Handle :: handle()) -> [Key :: key()]. keys(Handle) when Handle /= undefined -> couch_epi_data_gen:keys(Handle). - -spec subscribers(Handle :: handle()) -> [Subscriber :: app()]. subscribers(Handle) when Handle /= undefined -> couch_epi_data_gen:subscribers(Handle). --spec apply(Handle :: handle(), ServiceId :: atom(), Function :: atom(), - Args :: [term()], Opts :: apply_opts()) -> [any()]. +-spec apply( + Handle :: handle(), + ServiceId :: atom(), + Function :: atom(), + Args :: [term()], + Opts :: apply_opts() +) -> [any()]. apply(Handle, ServiceId, Function, Args, Opts) when Handle /= undefined -> couch_epi_functions_gen:apply(Handle, ServiceId, Function, Args, Opts). --spec get_handle({ServiceId :: service_id(), Key :: key()}) -> handle(); - (ServiceId :: service_id()) -> handle(). +-spec get_handle + ({ServiceId :: service_id(), Key :: key()}) -> handle(); + (ServiceId :: service_id()) -> handle(). get_handle({_ServiceId, _Key} = EPIKey) -> couch_epi_data_gen:get_handle(EPIKey); get_handle(ServiceId) when is_atom(ServiceId) -> couch_epi_functions_gen:get_handle(ServiceId). --spec any(Handle :: handle(), ServiceId :: atom(), Function :: atom(), - Args :: [term()], Opts :: apply_opts()) -> boolean(). +-spec any( + Handle :: handle(), + ServiceId :: atom(), + Function :: atom(), + Args :: [term()], + Opts :: apply_opts() +) -> boolean(). any(Handle, ServiceId, Function, Args, Opts) when Handle /= undefined -> Replies = apply(Handle, ServiceId, Function, Args, Opts), [] /= [Reply || Reply <- Replies, Reply == true]. --spec all(Handle :: handle(), ServiceId :: atom(), Function :: atom(), - Args :: [term()], Opts :: apply_opts()) -> boolean(). +-spec all( + Handle :: handle(), + ServiceId :: atom(), + Function :: atom(), + Args :: [term()], + Opts :: apply_opts() +) -> boolean(). all(Handle, ServiceId, Function, Args, Opts) when Handle /= undefined -> Replies = apply(Handle, ServiceId, Function, Args, Opts), [] == [Reply || Reply <- Replies, Reply == false]. -spec is_configured( - Handle :: handle(), Function :: atom(), Arity :: pos_integer()) -> boolean(). + Handle :: handle(), Function :: atom(), Arity :: pos_integer() +) -> boolean(). is_configured(Handle, Function, Arity) when Handle /= undefined -> [] /= couch_epi_functions_gen:modules(Handle, Function, Arity). - -spec register_service( - PluginId :: plugin_id(), Children :: [supervisor:child_spec()]) -> - [supervisor:child_spec()]. + PluginId :: plugin_id(), Children :: [supervisor:child_spec()] +) -> + [supervisor:child_spec()]. register_service(Plugin, Children) -> couch_epi_sup:plugin_childspecs(Plugin, Children). --spec decide(Handle :: handle(), ServiceId :: atom(), Function :: atom(), - Args :: [term()], Opts :: apply_opts()) -> - no_decision | {decided, term()}. +-spec decide( + Handle :: handle(), + ServiceId :: atom(), + Function :: atom(), + Args :: [term()], + Opts :: apply_opts() +) -> + no_decision | {decided, term()}. decide(Handle, ServiceId, Function, Args, Opts) when Handle /= undefined -> couch_epi_functions_gen:decide(Handle, ServiceId, Function, Args, Opts). diff --git a/src/couch_epi/src/couch_epi_codechange_monitor.erl b/src/couch_epi/src/couch_epi_codechange_monitor.erl index 738480448..214aea14d 100644 --- a/src/couch_epi/src/couch_epi_codechange_monitor.erl +++ b/src/couch_epi/src/couch_epi_codechange_monitor.erl @@ -24,8 +24,14 @@ %% gen_server Function Exports %% ------------------------------------------------------------------ --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3 +]). %% ------------------------------------------------------------------ %% API Function Definitions diff --git a/src/couch_epi/src/couch_epi_codegen.erl b/src/couch_epi/src/couch_epi_codegen.erl index 89b82a1f9..212a4e31a 100644 --- a/src/couch_epi/src/couch_epi_codegen.erl +++ b/src/couch_epi/src/couch_epi_codegen.erl @@ -25,21 +25,29 @@ generate(ModuleName, Forms0) -> scan(String) -> Exprs = [E || E <- re:split(String, "\\.\n", [{return, list}, trim])], - FormsTokens = lists:foldl(fun(Expr, Acc) -> - case erl_scan:string(Expr) of - {ok, [], _} -> - Acc; - {ok, Tokens, _} -> - [{Expr, fixup_terminator(Tokens)} | Acc] - end - end, [], Exprs), + FormsTokens = lists:foldl( + fun(Expr, Acc) -> + case erl_scan:string(Expr) of + {ok, [], _} -> + Acc; + {ok, Tokens, _} -> + [{Expr, fixup_terminator(Tokens)} | Acc] + end + end, + [], + Exprs + ), lists:reverse(FormsTokens). parse(FormsTokens) -> - ASTForms = lists:foldl(fun(Tokens, Forms) -> - {ok, AST} = parse_form(Tokens), - [AST | Forms] - end, [], FormsTokens), + ASTForms = lists:foldl( + fun(Tokens, Forms) -> + {ok, AST} = parse_form(Tokens), + [AST | Forms] + end, + [], + FormsTokens + ), lists:reverse(ASTForms). format_term(Data) -> @@ -49,11 +57,11 @@ parse_form(Tokens) -> {Expr, Forms} = split_expression(Tokens), case erl_parse:parse_form(Forms) of {ok, AST} -> {ok, AST}; - {error,{_,_, Reason}} -> - {error, Expr, Reason} + {error, {_, _, Reason}} -> {error, Expr, Reason} end. -split_expression({Expr, Forms}) -> {Expr, Forms}; +split_expression({Expr, Forms}) -> + {Expr, Forms}; split_expression(Tokens) -> {Exprs, Forms} = lists:unzip(Tokens), {string:join(Exprs, "\n"), lists:append(Forms)}. @@ -63,14 +71,15 @@ function(Clauses) -> fixup_terminator(Tokens) -> case lists:last(Tokens) of - {dot, _} -> Tokens; - {';', _} -> Tokens; + {dot, _} -> + Tokens; + {';', _} -> + Tokens; Token -> Line = line(Token), Tokens ++ [{dot, Line}] end. - -ifdef(pre18). line(Token) -> diff --git a/src/couch_epi/src/couch_epi_data.erl b/src/couch_epi/src/couch_epi_data.erl index 2bb09f6cf..ec554a40e 100644 --- a/src/couch_epi/src/couch_epi_data.erl +++ b/src/couch_epi/src/couch_epi_data.erl @@ -60,10 +60,13 @@ minimal_interval({_App, #couch_epi_spec{options = Options}}, Min) -> end. locate_sources(Specs) -> - lists:map(fun({ProviderApp, #couch_epi_spec{value = Src}}) -> - {ok, Locator} = locate(ProviderApp, Src), - {ProviderApp, Locator} - end, Specs). + lists:map( + fun({ProviderApp, #couch_epi_spec{value = Src}}) -> + {ok, Locator} = locate(ProviderApp, Src), + {ProviderApp, Locator} + end, + Specs + ). locate(App, {priv_file, FileName}) -> case priv_path(App, FileName) of diff --git a/src/couch_epi/src/couch_epi_data_gen.erl b/src/couch_epi/src/couch_epi_data_gen.erl index 4a283450d..65d689fbf 100644 --- a/src/couch_epi/src/couch_epi_data_gen.erl +++ b/src/couch_epi/src/couch_epi_data_gen.erl @@ -61,57 +61,57 @@ get_handle({Service, Key}) -> %% ------------------------------------------------------------------ preamble() -> - " - -export([by_key/0, by_key/1]). - -export([by_source/0, by_source/1]). - -export([all/0, all/1, get/2]). - -export([version/0, version/1]). - -export([keys/0, subscribers/0]). - -compile({no_auto_import,[get/0, get/1]}). - all() -> - lists:foldl(fun({Key, Defs}, Acc) -> - [D || {_Subscriber, D} <- Defs ] ++ Acc - end, [], by_key()). - - all(Key) -> - lists:foldl(fun({Subscriber, Data}, Acc) -> - [Data | Acc] - end, [], by_key(Key)). - - by_key() -> - [{Key, by_key(Key)} || Key <- keys()]. - - by_key(Key) -> - lists:foldl( - fun(Source, Acc) -> append_if_defined(Source, get(Source, Key), Acc) - end, [], subscribers()). - - - by_source() -> - [{Source, by_source(Source)} || Source <- subscribers()]. - - by_source(Source) -> - lists:foldl( - fun(Key, Acc) -> append_if_defined(Key, get(Source, Key), Acc) - end, [], keys()). - - version() -> - [{Subscriber, version(Subscriber)} || Subscriber <- subscribers()]. - - %% Helper functions - append_if_defined(Type, undefined, Acc) -> Acc; - append_if_defined(Type, Value, Acc) -> [{Type, Value} | Acc]. - " - %% In addition to preamble we also generate following methods - %% get(Source1, Key1) -> Data; - %% get(Source, Key) -> undefined. - - %% version(Source1) -> "HASH"; - %% version(Source) -> {error, {unknown, Source}}. - - %% keys() -> []. - %% subscribers() -> []. - . + "\n" + " -export([by_key/0, by_key/1]).\n" + " -export([by_source/0, by_source/1]).\n" + " -export([all/0, all/1, get/2]).\n" + " -export([version/0, version/1]).\n" + " -export([keys/0, subscribers/0]).\n" + " -compile({no_auto_import,[get/0, get/1]}).\n" + " all() ->\n" + " lists:foldl(fun({Key, Defs}, Acc) ->\n" + " [D || {_Subscriber, D} <- Defs ] ++ Acc\n" + " end, [], by_key()).\n" + "\n" + " all(Key) ->\n" + " lists:foldl(fun({Subscriber, Data}, Acc) ->\n" + " [Data | Acc]\n" + " end, [], by_key(Key)).\n" + "\n" + " by_key() ->\n" + " [{Key, by_key(Key)} || Key <- keys()].\n" + "\n" + " by_key(Key) ->\n" + " lists:foldl(\n" + " fun(Source, Acc) -> append_if_defined(Source, get(Source, Key), Acc)\n" + " end, [], subscribers()).\n" + "\n" + "\n" + " by_source() ->\n" + " [{Source, by_source(Source)} || Source <- subscribers()].\n" + "\n" + " by_source(Source) ->\n" + " lists:foldl(\n" + " fun(Key, Acc) -> append_if_defined(Key, get(Source, Key), Acc)\n" + " end, [], keys()).\n" + "\n" + " version() ->\n" + " [{Subscriber, version(Subscriber)} || Subscriber <- subscribers()].\n" + "\n" + " %% Helper functions\n" + " append_if_defined(Type, undefined, Acc) -> Acc;\n" + " append_if_defined(Type, Value, Acc) -> [{Type, Value} | Acc].\n" + " " +%% In addition to preamble we also generate following methods +%% get(Source1, Key1) -> Data; +%% get(Source, Key) -> undefined. + +%% version(Source1) -> "HASH"; +%% version(Source) -> {error, {unknown, Source}}. + +%% keys() -> []. +%% subscribers() -> []. +. generate(Handle, Defs) -> GetFunForms = couch_epi_codegen:function(getters(Defs)), @@ -119,9 +119,10 @@ generate(Handle, Defs) -> KeysForms = keys_method(Defs), SubscribersForms = subscribers_method(Defs), - Forms = couch_epi_codegen:scan(preamble()) - ++ GetFunForms ++ VersionFunForms - ++ KeysForms ++ SubscribersForms, + Forms = + couch_epi_codegen:scan(preamble()) ++ + GetFunForms ++ VersionFunForms ++ + KeysForms ++ SubscribersForms, couch_epi_codegen:generate(Handle, Forms). @@ -135,22 +136,30 @@ subscribers_method(Defs) -> getters(Defs) -> DefaultClause = "get(_S, _K) -> undefined.", - fold_defs(Defs, [couch_epi_codegen:scan(DefaultClause)], + fold_defs( + Defs, + [couch_epi_codegen:scan(DefaultClause)], fun({Source, Key, Data}, Acc) -> getter(Source, Key, Data) ++ Acc - end). + end + ). version_method(Defs) -> DefaultClause = "version(S) -> {error, {unknown, S}}.", - lists:foldl(fun({Source, Data}, Clauses) -> - version(Source, Data) ++ Clauses - end, [couch_epi_codegen:scan(DefaultClause)], Defs). + lists:foldl( + fun({Source, Data}, Clauses) -> + version(Source, Data) ++ Clauses + end, + [couch_epi_codegen:scan(DefaultClause)], + Defs + ). getter(Source, Key, Data) -> D = couch_epi_codegen:format_term(Data), Src = atom_to_list(Source), couch_epi_codegen:scan( - "get(" ++ Src ++ ", " ++ format_key(Key) ++ ") ->" ++ D ++ ";"). + "get(" ++ Src ++ ", " ++ format_key(Key) ++ ") ->" ++ D ++ ";" + ). version(Source, Data) -> Src = atom_to_list(Source), @@ -183,7 +192,6 @@ format_key(Key) -> module_name({Service, Key}) when is_list(Service) andalso is_list(Key) -> list_to_atom(string:join([atom_to_list(?MODULE), Service, Key], "_")). - get_current_definitions(Handle) -> if_exists(Handle, by_source, 0, [], fun() -> Handle:by_source() @@ -205,11 +213,19 @@ defined_subscribers(Defs) -> [Source || {Source, _} <- Defs]. fold_defs(Defs, Acc, Fun) -> - lists:foldr(fun({Source, SourceData}, Clauses) -> - lists:foldr(fun({Key, Data}, InAcc) -> - Fun({Source, Key, Data}, InAcc) - end, [], SourceData) ++ Clauses - end, Acc, Defs). + lists:foldr( + fun({Source, SourceData}, Clauses) -> + lists:foldr( + fun({Key, Data}, InAcc) -> + Fun({Source, Key, Data}, InAcc) + end, + [], + SourceData + ) ++ Clauses + end, + Acc, + Defs + ). %% ------------------------------------------------------------------ %% Tests @@ -243,15 +259,20 @@ basic_test() -> ?assertEqual("3KZ4EG4WBF4J683W8GSDDPYR3", Module:version(app1)), ?assertEqual("4EFUU47W9XDNMV9RMZSSJQU3Y", Module:version(app2)), - ?assertEqual({error,{unknown,bad}}, Module:version(bad)), + ?assertEqual({error, {unknown, bad}}, Module:version(bad)), ?assertEqual( - [{app1,"3KZ4EG4WBF4J683W8GSDDPYR3"}, - {app2,"4EFUU47W9XDNMV9RMZSSJQU3Y"}], lists:usort(Module:version())), + [ + {app1, "3KZ4EG4WBF4J683W8GSDDPYR3"}, + {app2, "4EFUU47W9XDNMV9RMZSSJQU3Y"} + ], + lists:usort(Module:version()) + ), ?assertEqual( - [{app1,[some_nice_data]},{app2,"other data"}], - lists:usort(Module:by_key(foo))), + [{app1, [some_nice_data]}, {app2, "other data"}], + lists:usort(Module:by_key(foo)) + ), ?assertEqual([], lists:usort(Module:by_key(bad))), @@ -260,8 +281,8 @@ basic_test() -> {bar, [{app2, {"even more data"}}]}, {foo, [{app2, "other data"}, {app1, [some_nice_data]}]} ], - lists:usort(Module:by_key())), - + lists:usort(Module:by_key()) + ), ?assertEqual(Defs1, lists:usort(Module:by_source(app1))), ?assertEqual(Defs2, lists:usort(Module:by_source(app2))), @@ -273,10 +294,12 @@ basic_test() -> {app1, [{foo, [some_nice_data]}]}, {app2, [{foo, "other data"}, {bar, {"even more data"}}]} ], - lists:usort(Module:by_source())), + lists:usort(Module:by_source()) + ), ?assertEqual( - lists:usort([Data1, Data2, Data3]), lists:usort(Module:all())), + lists:usort([Data1, Data2, Data3]), lists:usort(Module:all()) + ), ?assertEqual(lists:usort([Data1, Data2]), lists:usort(Module:all(foo))), ?assertEqual([], lists:usort(Module:all(bad))), ok. diff --git a/src/couch_epi/src/couch_epi_functions.erl b/src/couch_epi/src/couch_epi_functions.erl index ac9373928..1c5fd3403 100644 --- a/src/couch_epi/src/couch_epi_functions.erl +++ b/src/couch_epi/src/couch_epi_functions.erl @@ -43,7 +43,11 @@ definitions(Modules) -> [{M, M:module_info(exports) -- Blacklist} || M <- Modules]. group(KV) -> - Dict = lists:foldr(fun({K,V}, D) -> - dict:append_list(K, V, D) - end, dict:new(), KV), + Dict = lists:foldr( + fun({K, V}, D) -> + dict:append_list(K, V, D) + end, + dict:new(), + KV + ), [{K, lists:reverse(V)} || {K, V} <- dict:to_list(Dict)]. diff --git a/src/couch_epi/src/couch_epi_functions_gen.erl b/src/couch_epi/src/couch_epi_functions_gen.erl index 7408593b8..d7364c044 100644 --- a/src/couch_epi/src/couch_epi_functions_gen.erl +++ b/src/couch_epi/src/couch_epi_functions_gen.erl @@ -45,20 +45,30 @@ get_handle(ServiceId) -> apply(ServiceId, Function, Args, Opts) when is_atom(ServiceId) -> apply(get_handle(ServiceId), ServiceId, Function, Args, Opts). --spec apply(Handle :: atom(), ServiceId :: atom(), Function :: atom(), - Args :: [term()], Opts :: couch_epi:apply_opts()) -> [any()]. +-spec apply( + Handle :: atom(), + ServiceId :: atom(), + Function :: atom(), + Args :: [term()], + Opts :: couch_epi:apply_opts() +) -> [any()]. apply(Handle, _ServiceId, Function, Args, Opts) -> DispatchOpts = parse_opts(Opts), Modules = providers(Handle, Function, length(Args), DispatchOpts), dispatch(Handle, Modules, Function, Args, DispatchOpts). --spec decide(Handle :: atom(), ServiceId :: atom(), Function :: atom(), - Args :: [term()], Opts :: couch_epi:apply_opts()) -> - no_decision | {decided, term()}. +-spec decide( + Handle :: atom(), + ServiceId :: atom(), + Function :: atom(), + Args :: [term()], + Opts :: couch_epi:apply_opts() +) -> + no_decision | {decided, term()}. decide(Handle, _ServiceId, Function, Args, Opts) -> - DispatchOpts = parse_opts([interruptible|Opts]), + DispatchOpts = parse_opts([interruptible | Opts]), Modules = providers(Handle, Function, length(Args), DispatchOpts), dispatch(Handle, Modules, Function, Args, DispatchOpts). @@ -67,33 +77,33 @@ decide(Handle, _ServiceId, Function, Args, Opts) -> %% ------------------------------------------------------------------ preamble() -> - " - -export([version/0, version/1]). - -export([providers/0, providers/2]). - -export([definitions/0, definitions/1]). - -export([dispatch/3]). - -export([callbacks/2]). - - version() -> - [{Provider, version(Provider)} || Provider <- providers()]. - - definitions() -> - [{Provider, definitions(Provider)} || Provider <- providers()]. - - callbacks(Provider, Function) -> - []. - - " - %% In addition to preamble we also generate following methods - %% dispatch(Module, Function, [A1, A2]) -> Module:Function(A1, A2); - - %% version(Source1) -> "HASH"; - %% version(Source) -> {error, {unknown, Source}}. - - %% providers() -> []. - %% providers(Function, Arity) -> []. - %% definitions(Provider) -> [{Module, [{Fun, Arity}]}]. - . + "\n" + " -export([version/0, version/1]).\n" + " -export([providers/0, providers/2]).\n" + " -export([definitions/0, definitions/1]).\n" + " -export([dispatch/3]).\n" + " -export([callbacks/2]).\n" + "\n" + " version() ->\n" + " [{Provider, version(Provider)} || Provider <- providers()].\n" + "\n" + " definitions() ->\n" + " [{Provider, definitions(Provider)} || Provider <- providers()].\n" + "\n" + " callbacks(Provider, Function) ->\n" + " [].\n" + "\n" + " " +%% In addition to preamble we also generate following methods +%% dispatch(Module, Function, [A1, A2]) -> Module:Function(A1, A2); + +%% version(Source1) -> "HASH"; +%% version(Source) -> {error, {unknown, Source}}. + +%% providers() -> []. +%% providers(Function, Arity) -> []. +%% definitions(Provider) -> [{Module, [{Fun, Arity}]}]. +. generate(Handle, Defs) -> DispatchFunForms = couch_epi_codegen:function(dispatchers(Defs)), @@ -103,10 +113,11 @@ generate(Handle, Defs) -> ProvidersForms = couch_epi_codegen:function(providers_method(Defs)), DefinitionsForms = couch_epi_codegen:function(definitions_method(Defs)), - Forms = couch_epi_codegen:scan(preamble()) - ++ DispatchFunForms ++ VersionFunForms - ++ ProvidersForms ++ AllProvidersForms - ++ DefinitionsForms, + Forms = + couch_epi_codegen:scan(preamble()) ++ + DispatchFunForms ++ VersionFunForms ++ + ProvidersForms ++ AllProvidersForms ++ + DefinitionsForms, couch_epi_codegen:generate(Handle, Forms). @@ -117,9 +128,13 @@ all_providers_method(Defs) -> providers_method(Defs) -> Providers = providers_by_function(Defs), DefaultClause = "providers(_, _) -> [].", - lists:foldl(fun({{Fun, Arity}, Modules}, Clauses) -> - providers(Fun, Arity, Modules) ++ Clauses - end, [couch_epi_codegen:scan(DefaultClause)], Providers). + lists:foldl( + fun({{Fun, Arity}, Modules}, Clauses) -> + providers(Fun, Arity, Modules) ++ Clauses + end, + [couch_epi_codegen:scan(DefaultClause)], + Providers + ). providers(Function, Arity, Modules) -> ArityStr = integer_to_list(Arity), @@ -127,26 +142,38 @@ providers(Function, Arity, Modules) -> Fun = atom_to_list(Function), %% providers(Function, Arity) -> [Module]; couch_epi_codegen:scan( - "providers(" ++ Fun ++ "," ++ ArityStr ++ ") ->" ++ Mods ++ ";"). + "providers(" ++ Fun ++ "," ++ ArityStr ++ ") ->" ++ Mods ++ ";" + ). dispatchers(Defs) -> DefaultClause = "dispatch(_Module, _Fun, _Args) -> ok.", - fold_defs(Defs, [couch_epi_codegen:scan(DefaultClause)], + fold_defs( + Defs, + [couch_epi_codegen:scan(DefaultClause)], fun({_Source, Module, Function, Arity}, Acc) -> dispatcher(Module, Function, Arity) ++ Acc - end). + end + ). version_method(Defs) -> DefaultClause = "version(S) -> {error, {unknown, S}}.", - lists:foldl(fun({Source, SrcDefs}, Clauses) -> - version(Source, SrcDefs) ++ Clauses - end, [couch_epi_codegen:scan(DefaultClause)], Defs). + lists:foldl( + fun({Source, SrcDefs}, Clauses) -> + version(Source, SrcDefs) ++ Clauses + end, + [couch_epi_codegen:scan(DefaultClause)], + Defs + ). definitions_method(Defs) -> DefaultClause = "definitions(S) -> {error, {unknown, S}}.", - lists:foldl(fun({Source, SrcDefs}, Clauses) -> - definition(Source, SrcDefs) ++ Clauses - end, [couch_epi_codegen:scan(DefaultClause)], Defs). + lists:foldl( + fun({Source, SrcDefs}, Clauses) -> + definition(Source, SrcDefs) ++ Clauses + end, + [couch_epi_codegen:scan(DefaultClause)], + Defs + ). definition(Source, Defs) -> Src = atom_to_list(Source), @@ -159,27 +186,28 @@ dispatcher(Module, Function, 0) -> %% dispatch(Module, Function, []) -> Module:Function(); couch_epi_codegen:scan( - "dispatch(" ++ M ++ "," ++ Fun ++ ", []) ->" - ++ M ++ ":" ++ Fun ++ "();"); + "dispatch(" ++ M ++ "," ++ Fun ++ ", []) ->" ++ + M ++ ":" ++ Fun ++ "();" + ); dispatcher(Module, Function, Arity) -> Args = args_string(Arity), M = atom_to_list(Module), Fun = atom_to_list(Function), %% dispatch(Module, Function, [A1, A2]) -> Module:Function(A1, A2); couch_epi_codegen:scan( - "dispatch(" ++ M ++ "," ++ Fun ++ ", [" ++ Args ++ "]) ->" - ++ M ++ ":" ++ Fun ++ "(" ++ Args ++ ");"). + "dispatch(" ++ M ++ "," ++ Fun ++ ", [" ++ Args ++ "]) ->" ++ + M ++ ":" ++ Fun ++ "(" ++ Args ++ ");" + ). args_string(Arity) -> - Vars = ["A" ++ integer_to_list(Seq) || Seq <- lists:seq(1, Arity)], + Vars = ["A" ++ integer_to_list(Seq) || Seq <- lists:seq(1, Arity)], string:join(Vars, ", "). version(Source, SrcDefs) -> Modules = [Module || {Module, _Exports} <- SrcDefs], couch_epi_codegen:scan( - "version(" ++ atom_to_list(Source) ++ ") ->" ++ hash(Modules) ++ ";"). - - + "version(" ++ atom_to_list(Source) ++ ") ->" ++ hash(Modules) ++ ";" + ). %% ------------------------------------------------------------------ %% Helper functions @@ -204,26 +232,48 @@ defined_providers(Defs) -> %% Defs = [{Source, [{Module, [{Fun, Arity}]}]}] fold_defs(Defs, Acc, Fun) -> - lists:foldl(fun({Source, SourceData}, Clauses) -> - lists:foldl(fun({Module, Exports}, ExportsAcc) -> - lists:foldl(fun({Function, Arity}, InAcc) -> - Fun({Source, Module, Function, Arity}, InAcc) - end, [], Exports) ++ ExportsAcc - end, [], SourceData) ++ Clauses - end, Acc, Defs). + lists:foldl( + fun({Source, SourceData}, Clauses) -> + lists:foldl( + fun({Module, Exports}, ExportsAcc) -> + lists:foldl( + fun({Function, Arity}, InAcc) -> + Fun({Source, Module, Function, Arity}, InAcc) + end, + [], + Exports + ) ++ ExportsAcc + end, + [], + SourceData + ) ++ Clauses + end, + Acc, + Defs + ). providers_by_function(Defs) -> - Providers = fold_defs(Defs, [], + Providers = fold_defs( + Defs, + [], fun({_Source, Module, Function, Arity}, Acc) -> [{{Function, Arity}, Module} | Acc] end ), - Dict = lists:foldl(fun({K, V}, Acc) -> - dict:update(K, fun(Modules) -> - append_if_missing(Modules, V) - end, [V], Acc) - - end, dict:new(), Providers), + Dict = lists:foldl( + fun({K, V}, Acc) -> + dict:update( + K, + fun(Modules) -> + append_if_missing(Modules, V) + end, + [V], + Acc + ) + end, + dict:new(), + Providers + ), dict:to_list(Dict). append_if_missing(List, Value) -> @@ -238,36 +288,75 @@ hash(Modules) -> dispatch(_Handle, _Modules, _Func, _Args, #opts{concurrent = true, pipe = true}) -> throw({error, {incompatible_options, [concurrent, pipe]}}); -dispatch(Handle, Modules, Function, Args, - #opts{pipe = true, ignore_errors = true}) -> - lists:foldl(fun(Module, Acc) -> - try +dispatch( + Handle, + Modules, + Function, + Args, + #opts{pipe = true, ignore_errors = true} +) -> + lists:foldl( + fun(Module, Acc) -> + try + Handle:dispatch(Module, Function, Acc) + catch + _:_ -> + Acc + end + end, + Args, + Modules + ); +dispatch( + Handle, + Modules, + Function, + Args, + #opts{pipe = true} +) -> + lists:foldl( + fun(Module, Acc) -> Handle:dispatch(Module, Function, Acc) - catch _:_ -> - Acc - end - end, Args, Modules); -dispatch(Handle, Modules, Function, Args, - #opts{pipe = true}) -> - lists:foldl(fun(Module, Acc) -> - Handle:dispatch(Module, Function, Acc) - end, Args, Modules); -dispatch(Handle, Modules, Function, Args, - #opts{interruptible = true}) -> + end, + Args, + Modules + ); +dispatch( + Handle, + Modules, + Function, + Args, + #opts{interruptible = true} +) -> apply_while(Modules, Handle, Function, Args); dispatch(Handle, Modules, Function, Args, #opts{} = Opts) -> [do_dispatch(Handle, Module, Function, Args, Opts) || Module <- Modules]. -do_dispatch(Handle, Module, Function, Args, - #opts{concurrent = true, ignore_errors = true}) -> +do_dispatch( + Handle, + Module, + Function, + Args, + #opts{concurrent = true, ignore_errors = true} +) -> spawn(fun() -> (catch Handle:dispatch(Module, Function, Args)) end); -do_dispatch(Handle, Module, Function, Args, - #opts{ignore_errors = true}) -> +do_dispatch( + Handle, + Module, + Function, + Args, + #opts{ignore_errors = true} +) -> (catch Handle:dispatch(Module, Function, Args)); -do_dispatch(Handle, Module, Function, Args, - #opts{concurrent = true}) -> +do_dispatch( + Handle, + Module, + Function, + Args, + #opts{concurrent = true} +) -> spawn(fun() -> Handle:dispatch(Module, Function, Args) end); do_dispatch(Handle, Module, Function, Args, #opts{}) -> Handle:dispatch(Module, Function, Args). @@ -285,13 +374,13 @@ apply_while([Module | Modules], Handle, Function, Args) -> parse_opts(Opts) -> parse_opts(Opts, #opts{}). -parse_opts([ignore_errors|Rest], #opts{} = Acc) -> +parse_opts([ignore_errors | Rest], #opts{} = Acc) -> parse_opts(Rest, Acc#opts{ignore_errors = true}); -parse_opts([pipe|Rest], #opts{} = Acc) -> +parse_opts([pipe | Rest], #opts{} = Acc) -> parse_opts(Rest, Acc#opts{pipe = true}); -parse_opts([concurrent|Rest], #opts{} = Acc) -> +parse_opts([concurrent | Rest], #opts{} = Acc) -> parse_opts(Rest, Acc#opts{concurrent = true}); -parse_opts([interruptible|Rest], #opts{} = Acc) -> +parse_opts([interruptible | Rest], #opts{} = Acc) -> parse_opts(Rest, Acc#opts{interruptible = true}); parse_opts([], Acc) -> Acc. @@ -324,16 +413,17 @@ basic_test() -> generate(Module, [{app1, Defs}, {app2, Defs}]), Exports = lists:sort([ - {callbacks,2}, - {version,1}, - {providers,2}, - {definitions,1}, - {module_info,0}, - {version,0}, - {dispatch,3}, - {providers,0}, - {module_info,1}, - {definitions,0}]), + {callbacks, 2}, + {version, 1}, + {providers, 2}, + {definitions, 1}, + {module_info, 0}, + {version, 0}, + {dispatch, 3}, + {providers, 0}, + {module_info, 1}, + {definitions, 0} + ]), ?assertEqual(Exports, lists:sort(Module:module_info(exports))), ?assertEqual([app1, app2], lists:sort(Module:providers())), @@ -356,19 +446,19 @@ generate_module(Name, Body) -> couch_epi_codegen:generate(Name, Tokens). decide_module(decide) -> - " - -export([inc/1]). - - inc(A) -> - {decided, A + 1}. - "; + "\n" + " -export([inc/1]).\n" + "\n" + " inc(A) ->\n" + " {decided, A + 1}.\n" + " "; decide_module(no_decision) -> - " - -export([inc/1]). - - inc(_A) -> - no_decision. - ". + "\n" + " -export([inc/1]).\n" + "\n" + " inc(_A) ->\n" + " no_decision.\n" + " ". decide_test() -> ok = generate_module(decide, decide_module(decide)), @@ -380,12 +470,12 @@ decide_test() -> DecideFirstHandle = decide_first_handle, ok = generate(DecideFirstHandle, [DecideDef, NoDecissionDef]), ?assertMatch([decide, no_decision], DecideFirstHandle:providers(inc, 1)), - ?assertMatch({decided,4}, decide(DecideFirstHandle, anything, inc, [3], [])), + ?assertMatch({decided, 4}, decide(DecideFirstHandle, anything, inc, [3], [])), DecideSecondHandle = decide_second_handle, ok = generate(DecideSecondHandle, [NoDecissionDef, DecideDef]), ?assertMatch([no_decision, decide], DecideSecondHandle:providers(inc, 1)), - ?assertMatch({decided,4}, decide(DecideSecondHandle, anything, inc, [3], [])), + ?assertMatch({decided, 4}, decide(DecideSecondHandle, anything, inc, [3], [])), NoDecissionHandle = no_decision_handle, ok = generate(NoDecissionHandle, [NoDecissionDef]), diff --git a/src/couch_epi/src/couch_epi_module_keeper.erl b/src/couch_epi/src/couch_epi_module_keeper.erl index 36376fec0..97420ea7b 100644 --- a/src/couch_epi/src/couch_epi_module_keeper.erl +++ b/src/couch_epi/src/couch_epi_module_keeper.erl @@ -12,7 +12,6 @@ -module(couch_epi_module_keeper). - -behaviour(gen_server). %% ------------------------------------------------------------------ @@ -22,17 +21,29 @@ -export([start_link/3, stop/1]). -export([reload/1]). - %% ------------------------------------------------------------------ %% gen_server Function Exports %% ------------------------------------------------------------------ --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3 +]). -record(state, { - codegen, module, key, type, handle, hash, kind, - timer = {undefined, undefined}}). + codegen, + module, + key, + type, + handle, + hash, + kind, + timer = {undefined, undefined} +}). %% ------------------------------------------------------------------ %% API Function Definitions @@ -41,7 +52,8 @@ start_link(Type, Key, Codegen) -> Handle = Codegen:get_handle(Key), gen_server:start_link( - {local, Handle}, ?MODULE, [Type, Codegen, Key, Handle], []). + {local, Handle}, ?MODULE, [Type, Codegen, Key, Handle], [] + ). stop(Server) -> catch gen_server:call(Server, stop). @@ -151,8 +163,9 @@ safe_set(Hash, Data, #state{} = State) -> OldData = CodeGen:get_current_definitions(Handle), ok = CodeGen:generate(Handle, Data), {ok, OldData, State#state{hash = Hash}} - catch Class:Reason -> - {{Class, Reason}, State} + catch + Class:Reason -> + {{Class, Reason}, State} end. notify(Key, OldData, NewData, Defs) -> diff --git a/src/couch_epi/src/couch_epi_plugin.erl b/src/couch_epi/src/couch_epi_plugin.erl index 2cb1f3ebe..1ec09d8dc 100644 --- a/src/couch_epi/src/couch_epi_plugin.erl +++ b/src/couch_epi/src/couch_epi_plugin.erl @@ -28,25 +28,25 @@ %% Types Definitions %% ------------------------------------------------------------------ --type kind() - :: providers - | data_providers - | services - | data_subscriptions - . +-type kind() :: + providers + | data_providers + | services + | data_subscriptions. --type key() - :: {ServiceId :: couch_epi:service_id(), Key :: couch_epi:key()} - | couch_epi:service_id(). +-type key() :: + {ServiceId :: couch_epi:service_id(), Key :: couch_epi:key()} + | couch_epi:service_id(). -callback app() -> couch_epi:app(). -callback providers() -> [{couch_epi:service_id(), module()}]. -callback services() -> [{couch_epi:service_id(), module()}]. -callback data_subscriptions() -> [{couch_epi:service_id(), couch_epi:key()}]. --callback data_providers() -> [ - {couch_epi:key(), couch_epi:data_spec()} +-callback data_providers() -> + [ + {couch_epi:key(), couch_epi:data_spec()} | {couch_epi:key(), couch_epi:data_spec(), [couch_epi:data_spec_opt()]} -]. + ]. -callback processes() -> [{couch_epi:plugin_id(), [supervisor:child_spec()]}]. -callback notify(Key :: term(), Old :: term(), New :: term()) -> ok. @@ -58,8 +58,7 @@ definitions(Plugins) -> lists:append([extract_definitions(Plugin) || Plugin <- Plugins]). plugin_processes(Plugin, Plugins) -> - lists:append([ - Specs || P0 <- Plugins, {P1, Specs} <- P0:processes(), P1 =:= Plugin]). + lists:append([Specs || P0 <- Plugins, {P1, Specs} <- P0:processes(), P1 =:= Plugin]). grouped_definitions(Plugins) -> Defs = lists:append([extract_definitions(Plugin) || Plugin <- Plugins]), @@ -87,7 +86,6 @@ notify_plugin(Plugin, Key, OldData, NewData) -> App = Plugin:app(), Plugin:notify(Key, app_data(App, OldData), app_data(App, NewData)). - app_data(App, Data) -> case lists:keyfind(App, 1, Data) of {App, AppData} -> AppData; @@ -100,12 +98,11 @@ filter_by_key(Definitions, Kind, Key) -> by_key(#couch_epi_spec{kind = Kind, key = Key}, Kind, Key) -> true; by_key(_, _, _) -> false. - extract_definitions(Plugin) -> - specs(Plugin, providers) - ++ specs(Plugin, data_providers) - ++ specs(Plugin, services) - ++ specs(Plugin, data_subscriptions). + specs(Plugin, providers) ++ + specs(Plugin, data_providers) ++ + specs(Plugin, services) ++ + specs(Plugin, data_subscriptions). -spec group_specs(Specs :: [#couch_epi_spec{}]) -> GroupedSpecs when GroupedSpecs :: @@ -113,15 +110,23 @@ extract_definitions(Plugin) -> group_specs(Specs) -> Grouped = group( - [{{Kind, Key}, group([{App, Spec}])} - || #couch_epi_spec{kind = Kind, key = Key, app = App} = Spec <- Specs]), + [ + {{Kind, Key}, group([{App, Spec}])} + || #couch_epi_spec{kind = Kind, key = Key, app = App} = Spec <- Specs + ] + ), [{K, lists:reverse(V)} || {K, V} <- Grouped]. - group(KV) -> - dict:to_list(lists:foldr(fun({K,V}, D) -> - dict:append_list(K, V, D) - end, dict:new(), KV)). + dict:to_list( + lists:foldr( + fun({K, V}, D) -> + dict:append_list(K, V, D) + end, + dict:new(), + KV + ) + ). specs(Plugin, Kind) -> [spec(parse(Spec, Kind), Plugin, Kind) || Spec <- Plugin:Kind()]. @@ -156,7 +161,6 @@ type(services, _) -> couch_epi_functions; type(data_providers, _) -> couch_epi_data; type(data_subscriptions, _) -> undefined. - %% ------------------------------------------------------------------ %% Tests %% ------------------------------------------------------------------ @@ -165,66 +169,66 @@ type(data_subscriptions, _) -> undefined. -include_lib("eunit/include/eunit.hrl"). plugin_module(foo_epi) -> - " - -compile([export_all]). - - app() -> foo. - providers() -> - [ - {chttpd_handlers, foo_provider}, - {bar_handlers, bar_provider1}, - {bar_handlers, bar_provider2} - ]. - - services() -> - [ - {foo_handlers, foo_service} - ]. - - data_providers() -> - [ - {{foo_service, data1}, {file, \"abs_file\"}, [{interval, 5000}]}, - {{foo_service, data2}, {priv_file, \"priv_file\"}}, - {{foo_service, data3}, {module, foo_data}} - ]. - - data_subscriptions() -> - [ - {stats, foo_definitions} - ]. - - processes() -> []. - - notify(_, _, _) -> ok. - "; + "\n" + " -compile([export_all]).\n" + "\n" + " app() -> foo.\n" + " providers() ->\n" + " [\n" + " {chttpd_handlers, foo_provider},\n" + " {bar_handlers, bar_provider1},\n" + " {bar_handlers, bar_provider2}\n" + " ].\n" + "\n" + " services() ->\n" + " [\n" + " {foo_handlers, foo_service}\n" + " ].\n" + "\n" + " data_providers() ->\n" + " [\n" + " {{foo_service, data1}, {file, \"abs_file\"}, [{interval, 5000}]},\n" + " {{foo_service, data2}, {priv_file, \"priv_file\"}},\n" + " {{foo_service, data3}, {module, foo_data}}\n" + " ].\n" + "\n" + " data_subscriptions() ->\n" + " [\n" + " {stats, foo_definitions}\n" + " ].\n" + "\n" + " processes() -> [].\n" + "\n" + " notify(_, _, _) -> ok.\n" + " "; plugin_module(bar_epi) -> - " - -compile([export_all]). - - app() -> bar. - providers() -> - [ - {chttpd_handlers, bar_provider}, - {bar_handlers, bar_provider} - ]. - - services() -> - [ - {bar_handlers, bar_service} - ]. - - data_providers() -> - []. - - data_subscriptions() -> - [ - {foo_service, data1} - ]. - - processes() -> []. - - notify(_, _, _) -> ok. - ". + "\n" + " -compile([export_all]).\n" + "\n" + " app() -> bar.\n" + " providers() ->\n" + " [\n" + " {chttpd_handlers, bar_provider},\n" + " {bar_handlers, bar_provider}\n" + " ].\n" + "\n" + " services() ->\n" + " [\n" + " {bar_handlers, bar_service}\n" + " ].\n" + "\n" + " data_providers() ->\n" + " [].\n" + "\n" + " data_subscriptions() ->\n" + " [\n" + " {foo_service, data1}\n" + " ].\n" + "\n" + " processes() -> [].\n" + "\n" + " notify(_, _, _) -> ok.\n" + " ". generate_module(Name, Body) -> Tokens = couch_epi_codegen:scan(Body), @@ -234,7 +238,7 @@ generate_modules(Kind, Providers) -> [generate_module(P, Kind(P)) || P <- Providers]. provider_modules_order_test() -> - [ok,ok] = generate_modules(fun plugin_module/1, [foo_epi, bar_epi]), + [ok, ok] = generate_modules(fun plugin_module/1, [foo_epi, bar_epi]), ok = application:set_env(couch_epi, plugins, [foo_epi, bar_epi]), Expected = [ {foo, bar_provider1}, @@ -249,7 +253,7 @@ provider_modules_order_test() -> ok. providers_order_test() -> - [ok,ok] = generate_modules(fun plugin_module/1, [foo_epi, bar_epi]), + [ok, ok] = generate_modules(fun plugin_module/1, [foo_epi, bar_epi]), Expected = [ {foo, bar_provider1}, {foo, bar_provider2}, @@ -331,7 +335,8 @@ definitions_test() -> key = chttpd_handlers, value = foo_provider, codegen = couch_epi_functions_gen, - type = couch_epi_functions}, + type = couch_epi_functions + }, #couch_epi_spec{ behaviour = foo_epi, app = foo, @@ -340,14 +345,15 @@ definitions_test() -> key = foo_handlers, value = foo_service, codegen = couch_epi_functions_gen, - type = couch_epi_functions}, + type = couch_epi_functions + }, #couch_epi_spec{ behaviour = foo_epi, app = foo, kind = data_providers, options = [{interval, 5000}], key = {foo_service, data1}, - value = {file,"abs_file"}, + value = {file, "abs_file"}, codegen = couch_epi_data_gen, type = couch_epi_data }, @@ -382,7 +388,7 @@ definitions_test() -> } ]), - [ok,ok] = generate_modules(fun plugin_module/1, [foo_epi, bar_epi]), + [ok, ok] = generate_modules(fun plugin_module/1, [foo_epi, bar_epi]), Tests = lists:zip(Expected, lists:sort(definitions([foo_epi, bar_epi]))), [?assertEqual(Expect, Result) || {Expect, Result} <- Tests], ok. diff --git a/src/couch_epi/src/couch_epi_sup.erl b/src/couch_epi/src/couch_epi_sup.erl index 477cbe79e..aca423a7d 100644 --- a/src/couch_epi/src/couch_epi_sup.erl +++ b/src/couch_epi/src/couch_epi_sup.erl @@ -61,7 +61,7 @@ plugin_childspecs(Plugin, Children) -> %% =================================================================== init([]) -> - {ok, { {one_for_one, 5, 10}, keepers()} }. + {ok, {{one_for_one, 5, 10}, keepers()}}. %% ------------------------------------------------------------------ %% Internal Function Definitions @@ -79,13 +79,16 @@ plugin_childspecs(Plugin, Plugins, Children) -> merge(ExtraChildren, Children) ++ childspecs(Definitions). childspecs(Definitions) -> - lists:map(fun({{Kind, Key}, Defs}) -> - CodeGen = couch_epi_plugin:codegen(Kind), - Handle = CodeGen:get_handle(Key), - Modules = lists:append([modules(Spec) || {_App, Spec} <- Defs]), - Name = service_name(Key) ++ "|" ++ atom_to_list(Kind), - code_monitor(Name, [Handle], [Handle|Modules]) - end, Definitions). + lists:map( + fun({{Kind, Key}, Defs}) -> + CodeGen = couch_epi_plugin:codegen(Kind), + Handle = CodeGen:get_handle(Key), + Modules = lists:append([modules(Spec) || {_App, Spec} <- Defs]), + Name = service_name(Key) ++ "|" ++ atom_to_list(Kind), + code_monitor(Name, [Handle], [Handle | Modules]) + end, + Definitions + ). %% ------------------------------------------------------------------ %% Helper Function Definitions @@ -95,21 +98,36 @@ remove_duplicates(Definitions) -> lists:ukeysort(1, Definitions). keeper_childspecs(Definitions) -> - lists:map(fun({{Kind, Key}, _Specs}) -> - Name = service_name(Key) ++ "|keeper", - CodeGen = couch_epi_plugin:codegen(Kind), - Handle = CodeGen:get_handle(Key), - keeper(Name, [provider_kind(Kind), Key, CodeGen], [Handle]) - end, Definitions). + lists:map( + fun({{Kind, Key}, _Specs}) -> + Name = service_name(Key) ++ "|keeper", + CodeGen = couch_epi_plugin:codegen(Kind), + Handle = CodeGen:get_handle(Key), + keeper(Name, [provider_kind(Kind), Key, CodeGen], [Handle]) + end, + Definitions + ). keeper(Name, Args, Modules) -> - {"couch_epi|" ++ Name, {couch_epi_module_keeper, start_link, - Args}, permanent, 5000, worker, Modules}. + { + "couch_epi|" ++ Name, + {couch_epi_module_keeper, start_link, Args}, + permanent, + 5000, + worker, + Modules + }. code_monitor(Name, Args, Modules0) -> Modules = [couch_epi_codechange_monitor | Modules0], - {"couch_epi_codechange_monitor|" ++ Name, - {couch_epi_codechange_monitor, start_link, Args}, permanent, 5000, worker, Modules}. + { + "couch_epi_codechange_monitor|" ++ Name, + {couch_epi_codechange_monitor, start_link, Args}, + permanent, + 5000, + worker, + Modules + }. provider_kind(services) -> providers; provider_kind(data_subscriptions) -> data_providers; @@ -138,5 +156,8 @@ merge([], Children) -> merge([{Id, _, _, _, _, _} = Spec | Rest], Children) -> merge(Rest, lists:keystore(Id, 1, Children, Spec)); merge([#{id := Id} = Spec | Rest], Children) -> - Replace = fun(#{id := I}) when I == Id -> Spec; (E) -> E end, + Replace = fun + (#{id := I}) when I == Id -> Spec; + (E) -> E + end, merge(Rest, lists:map(Replace, Children)). diff --git a/src/couch_epi/src/couch_epi_util.erl b/src/couch_epi/src/couch_epi_util.erl index ea4b10ea8..2c86a96e2 100644 --- a/src/couch_epi/src/couch_epi_util.erl +++ b/src/couch_epi/src/couch_epi_util.erl @@ -23,7 +23,7 @@ module_version(Module) -> hash(Term) -> <<SigInt:128/integer>> = couch_hash:md5_hash(term_to_binary(Term)), - lists:flatten(io_lib:format("\"~.36B\"",[SigInt])). + lists:flatten(io_lib:format("\"~.36B\"", [SigInt])). module_exists(Module) -> erlang:function_exported(Module, module_info, 0). diff --git a/src/couch_epi/test/eunit/couch_epi_basic_test.erl b/src/couch_epi/test/eunit/couch_epi_basic_test.erl index 5ba6c9f87..a99e9f900 100644 --- a/src/couch_epi/test/eunit/couch_epi_basic_test.erl +++ b/src/couch_epi/test/eunit/couch_epi_basic_test.erl @@ -28,63 +28,56 @@ -define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}). - start_link() -> ok. - %% BEGIN couch_epi_plugin behaviour callbacks - app() -> test_app. - providers() -> [ {my_service, provider1}, {my_service, provider2} ]. - services() -> [ {my_service, ?MODULE} ]. - data_providers() -> [ {{test_app, descriptions}, {static_module, ?MODULE}, [{interval, 100}]} ]. - data_subscriptions() -> [ {test_app, descriptions} ]. - processes() -> [ {?MODULE, [?CHILD(extra_process, worker)]}, - {?MODULE, [{to_replace, {new, start_link, [bar]}, - permanent, 5000, worker, [bar]}]}, - {?MODULE, [#{id => to_replace_map, - start => {new, start_link, [bar]}, modules => [bar]}]} + {?MODULE, [{to_replace, {new, start_link, [bar]}, permanent, 5000, worker, [bar]}]}, + {?MODULE, [ + #{ + id => to_replace_map, + start => {new, start_link, [bar]}, + modules => [bar] + } + ]} ]. - notify(_Key, _OldData, _NewData) -> ok. - %% END couch_epi_plugin behaviour callbacks - parse_child_id(Id) when is_atom(Id) -> Id; parse_child_id(Id) -> - ["couch_epi_codechange_monitor", ServiceName, KindStr] - = string:tokens(Id, "|"), + ["couch_epi_codechange_monitor", ServiceName, KindStr] = + string:tokens(Id, "|"), Kind = list_to_atom(KindStr), case string:tokens(ServiceName, ":") of [ServiceId, Key] -> @@ -93,7 +86,6 @@ parse_child_id(Id) -> {list_to_atom(Key), Kind} end. - -include_lib("eunit/include/eunit.hrl"). basic_test() -> @@ -101,49 +93,75 @@ basic_test() -> {extra_process, [], [extra_process]}, {to_replace, [bar], [bar]}, {to_replace_map, [bar], [bar]}, - {{my_service, providers}, + {{my_service, providers}, [couch_epi_functions_gen_my_service], [ + couch_epi_codechange_monitor, + couch_epi_functions_gen_my_service, + provider1, + provider2 + ]}, + { + {my_service, services}, [couch_epi_functions_gen_my_service], - [couch_epi_codechange_monitor, couch_epi_functions_gen_my_service, - provider1, provider2]}, - {{my_service, services}, - [couch_epi_functions_gen_my_service], - lists:sort([couch_epi_codechange_monitor, - couch_epi_functions_gen_my_service, ?MODULE])}, - {{{test_app, descriptions}, data_subscriptions}, + lists:sort([ + couch_epi_codechange_monitor, + couch_epi_functions_gen_my_service, + ?MODULE + ]) + }, + { + {{test_app, descriptions}, data_subscriptions}, [couch_epi_data_gen_test_app_descriptions], - lists:sort([couch_epi_codechange_monitor, - couch_epi_data_gen_test_app_descriptions, ?MODULE])}, - {{{test_app, descriptions}, data_providers}, + lists:sort([ + couch_epi_codechange_monitor, + couch_epi_data_gen_test_app_descriptions, + ?MODULE + ]) + }, + { + {{test_app, descriptions}, data_providers}, [couch_epi_data_gen_test_app_descriptions], - lists:sort([couch_epi_codechange_monitor, - couch_epi_data_gen_test_app_descriptions, ?MODULE])} + lists:sort([ + couch_epi_codechange_monitor, + couch_epi_data_gen_test_app_descriptions, + ?MODULE + ]) + } ], ToReplace = [ {to_replace, {old, start_link, [foo]}, permanent, 5000, worker, [foo]}, #{id => to_replace_map, start => {old, start_link, [foo]}} ], - Children = lists:sort(couch_epi_sup:plugin_childspecs( - ?MODULE, [?MODULE], ToReplace)), - - Results = lists:map(fun - ({Id, {_M, _F, Args}, _, _, _, Modules}) -> - {parse_child_id(Id), Args, lists:sort(Modules)}; - (#{id := Id, start := {_M, _F, Args}, modules := Modules}) -> - {parse_child_id(Id), Args, lists:sort(Modules)} - end, Children), + Children = lists:sort( + couch_epi_sup:plugin_childspecs( + ?MODULE, [?MODULE], ToReplace + ) + ), + + Results = lists:map( + fun + ({Id, {_M, _F, Args}, _, _, _, Modules}) -> + {parse_child_id(Id), Args, lists:sort(Modules)}; + (#{id := Id, start := {_M, _F, Args}, modules := Modules}) -> + {parse_child_id(Id), Args, lists:sort(Modules)} + end, + Children + ), Tests = lists:zip(lists:sort(Expected), lists:sort(Results)), [?assertEqual(Expect, Result) || {Expect, Result} <- Tests], - ExpectedChild = {to_replace, {new, start_link, [bar]}, - permanent, 5000, worker, [bar]}, + ExpectedChild = {to_replace, {new, start_link, [bar]}, permanent, 5000, worker, [bar]}, ?assertEqual( ExpectedChild, - lists:keyfind(to_replace, 1, Children)), - - ExpectedMapChildSpec = #{id => to_replace_map, - start => {new, start_link, [bar]}, modules => [bar]}, + lists:keyfind(to_replace, 1, Children) + ), + + ExpectedMapChildSpec = #{ + id => to_replace_map, + start => {new, start_link, [bar]}, + modules => [bar] + }, [MapChildSpec] = [E || #{id := to_replace_map} = E <- Children], ?assertEqual(ExpectedMapChildSpec, MapChildSpec), ok. diff --git a/src/couch_epi/test/eunit/couch_epi_tests.erl b/src/couch_epi/test/eunit/couch_epi_tests.erl index 12d8610c1..08307fe30 100644 --- a/src/couch_epi/test/eunit/couch_epi_tests.erl +++ b/src/couch_epi/test/eunit/couch_epi_tests.erl @@ -24,72 +24,77 @@ -define(TIMEOUT, 5000). -define(RELOAD_WAIT, 1000). --define(temp_atom, - fun() -> - {A, B, C} = os:timestamp(), - list_to_atom(lists:flatten(io_lib:format("~p~p~p", [A, B, C]))) - end). - --define(MODULE1(Name), " - -export([inc/2, fail/2]). - - inc(KV, A) -> - Reply = A + 1, - couch_epi_tests:save(KV, inc1, Reply), - [KV, Reply]. - - fail(KV, A) -> - inc(KV, A). -"). - --define(MODULE2(Name), " - -export([inc/2, fail/2]). - - inc(KV, A) -> - Reply = A + 1, - couch_epi_tests:save(KV, inc2, Reply), - [KV, Reply]. - - fail(KV, _A) -> - couch_epi_tests:save(KV, inc2, check_error), - throw(check_error). -"). - --define(DATA_MODULE1(Name), " - -export([data/0]). - - data() -> - [ - {[complex, key, 1], [ - {type, counter}, - {desc, foo} - ]} - ]. -"). - --define(DATA_MODULE2(Name), " - -export([data/0]). - - data() -> - [ - {[complex, key, 2], [ - {type, counter}, - {desc, bar} - ]}, - {[complex, key, 1], [ - {type, counter}, - {desc, updated_foo} - ]} - ]. -"). - --define(DATA_MODULE3(Name, Kv), " - -export([data/0]). - -data() -> - {ok, Data} = couch_epi_tests:get('" ++ atom_to_list(Kv) ++ "', data), - Data. -"). +-define(temp_atom, fun() -> + {A, B, C} = os:timestamp(), + list_to_atom(lists:flatten(io_lib:format("~p~p~p", [A, B, C]))) +end). + +-define(MODULE1(Name), + "\n" + " -export([inc/2, fail/2]).\n" + "\n" + " inc(KV, A) ->\n" + " Reply = A + 1,\n" + " couch_epi_tests:save(KV, inc1, Reply),\n" + " [KV, Reply].\n" + "\n" + " fail(KV, A) ->\n" + " inc(KV, A).\n" +). + +-define(MODULE2(Name), + "\n" + " -export([inc/2, fail/2]).\n" + "\n" + " inc(KV, A) ->\n" + " Reply = A + 1,\n" + " couch_epi_tests:save(KV, inc2, Reply),\n" + " [KV, Reply].\n" + "\n" + " fail(KV, _A) ->\n" + " couch_epi_tests:save(KV, inc2, check_error),\n" + " throw(check_error).\n" +). + +-define(DATA_MODULE1(Name), + "\n" + " -export([data/0]).\n" + "\n" + " data() ->\n" + " [\n" + " {[complex, key, 1], [\n" + " {type, counter},\n" + " {desc, foo}\n" + " ]}\n" + " ].\n" +). + +-define(DATA_MODULE2(Name), + "\n" + " -export([data/0]).\n" + "\n" + " data() ->\n" + " [\n" + " {[complex, key, 2], [\n" + " {type, counter},\n" + " {desc, bar}\n" + " ]},\n" + " {[complex, key, 1], [\n" + " {type, counter},\n" + " {desc, updated_foo}\n" + " ]}\n" + " ].\n" +). + +-define(DATA_MODULE3(Name, Kv), + "\n" + " -export([data/0]).\n" + "\n" + "data() ->\n" + " {ok, Data} = couch_epi_tests:get('" ++ atom_to_list(Kv) ++ + "', data),\n" + " Data.\n" +). %% ------------------------------------------------------------------ %% couch_epi_plugin behaviour @@ -98,69 +103,76 @@ data() -> plugin_module([KV, Spec]) when is_tuple(Spec) -> SpecStr = io_lib:format("~w", [Spec]), KVStr = "'" ++ atom_to_list(KV) ++ "'", - " - -compile([export_all]). - - app() -> test_app. - providers() -> - []. - - services() -> - []. - - data_providers() -> - [ - {{test_app, descriptions}, " ++ SpecStr ++ ", [{interval, 100}]} - ]. - - data_subscriptions() -> - [ - {test_app, descriptions} - ]. - - processes() -> []. - - notify(Key, OldData, Data) -> - couch_epi_tests:notify_cb(Key, OldData, Data, " ++ KVStr ++ "). - "; + "\n" + " -compile([export_all]).\n" + "\n" + " app() -> test_app.\n" + " providers() ->\n" + " [].\n" + "\n" + " services() ->\n" + " [].\n" + "\n" + " data_providers() ->\n" + " [\n" + " {{test_app, descriptions}, " ++ SpecStr ++ + ", [{interval, 100}]}\n" + " ].\n" + "\n" + " data_subscriptions() ->\n" + " [\n" + " {test_app, descriptions}\n" + " ].\n" + "\n" + " processes() -> [].\n" + "\n" + " notify(Key, OldData, Data) ->\n" + " couch_epi_tests:notify_cb(Key, OldData, Data, " ++ KVStr ++ + ").\n" + " "; plugin_module([KV, Provider]) when is_atom(Provider) -> KVStr = "'" ++ atom_to_list(KV) ++ "'", - " - -compile([export_all]). - - app() -> test_app. - providers() -> - [ - {my_service, " ++ atom_to_list(Provider) ++ "} - ]. - - services() -> - [ - {my_service, " ++ atom_to_list(Provider) ++ "} - ]. - - data_providers() -> - []. - - data_subscriptions() -> - []. - - processes() -> []. - - notify(Key, OldData, Data) -> - couch_epi_tests:notify_cb(Key, OldData, Data, " ++ KVStr ++ "). - ". - + "\n" + " -compile([export_all]).\n" + "\n" + " app() -> test_app.\n" + " providers() ->\n" + " [\n" + " {my_service, " ++ atom_to_list(Provider) ++ + "}\n" + " ].\n" + "\n" + " services() ->\n" + " [\n" + " {my_service, " ++ atom_to_list(Provider) ++ + "}\n" + " ].\n" + "\n" + " data_providers() ->\n" + " [].\n" + "\n" + " data_subscriptions() ->\n" + " [].\n" + "\n" + " processes() -> [].\n" + "\n" + " notify(Key, OldData, Data) ->\n" + " couch_epi_tests:notify_cb(Key, OldData, Data, " ++ KVStr ++ + ").\n" + " ". notify_cb(Key, OldData, Data, KV) -> save(KV, is_called, {Key, OldData, Data}). start_epi(Plugins) -> application:load(couch_epi), - PluginsModules = lists:map(fun({Module, Body}) -> - ok = generate_module(Module, Body), - Module - end, Plugins), + PluginsModules = lists:map( + fun({Module, Body}) -> + ok = generate_module(Module, Body), + Module + end, + Plugins + ), application:set_env(couch_epi, plugins, PluginsModules), application:start(couch_epi). @@ -176,13 +188,13 @@ setup(data_file) -> Pid = whereis(couch_epi:get_handle(Key)), - #ctx{ file = File, key = Key, handle = couch_epi:get_handle(Key), kv = KV, - pid = Pid}; + pid = Pid + }; setup(static_data_module) -> error_logger:tty(false), @@ -201,7 +213,8 @@ setup(static_data_module) -> handle = Handle, modules = [Handle, provider], kv = KV, - pid = Pid}; + pid = Pid + }; setup(callback_data_module) -> error_logger:tty(false), @@ -224,11 +237,12 @@ setup(callback_data_module) -> Handle = couch_epi:get_handle(Key), #ctx{ - key = Key, - handle = Handle, - modules = [Handle, provider], - kv = KV, - pid = Pid}; + key = Key, + handle = Handle, + modules = [Handle, provider], + kv = KV, + pid = Pid + }; setup(functions) -> Key = my_service, error_logger:tty(false), @@ -251,7 +265,8 @@ setup(functions) -> handle = Handle, modules = [Handle, provider1, provider2], kv = KV, - pid = Pid}; + pid = Pid + }; setup({options, _Opts}) -> setup(functions). @@ -308,7 +323,6 @@ epi_data_source_test_() -> [make_case("Check query API for: ", Cases, Funs)] }. - epi_apply_test_() -> { "epi dispatch tests", @@ -338,7 +352,6 @@ epi_providers_order_test_() -> } }. - epi_reload_test_() -> Cases = [ data_file, @@ -364,19 +377,21 @@ apply_options_test_() -> [make_case("Apply with options: ", Setups, Funs)] }. - make_case(Msg, {Tag, P}, Funs) -> Cases = [{Tag, Case} || Case <- P], make_case(Msg, Cases, Funs); make_case(Msg, P, Funs) -> - [{format_case_name(Msg, Case), [ - { - foreachx, fun setup/1, fun teardown/2, - [ - {Case, make_fun(Fun, 2)} || Fun <- Funs - ] - } - ]} || Case <- P]. + [ + {format_case_name(Msg, Case), [ + { + foreachx, + fun setup/1, + fun teardown/2, + [{Case, make_fun(Fun, 2)} || Fun <- Funs] + } + ]} + || Case <- P + ]. make_fun(Fun, Arity) -> {arity, A} = lists:keyfind(arity, 1, erlang:fun_info(Fun)), @@ -405,8 +420,8 @@ ensure_notified_when_changed(functions, #ctx{key = Key} = Ctx) -> update(functions, Ctx), Result = get(Ctx, is_called), ExpectedDefs = [ - {provider1,[{inc,2},{fail,2}]}, - {provider2,[{inc,2},{fail,2}]} + {provider1, [{inc, 2}, {fail, 2}]}, + {provider2, [{inc, 2}, {fail, 2}]} ], ?assertEqual({ok, {Key, ExpectedDefs, ExpectedDefs}}, Result), ok @@ -425,7 +440,8 @@ ensure_notified_when_changed(Case, #ctx{key = Key} = Ctx) -> ?assertMatch(ExpectedData, lists:usort(Data)), ?assertMatch( [{[complex, key, 1], [{type, counter}, {desc, foo}]}], - lists:usort(OldData)) + lists:usort(OldData) + ) end). ensure_not_notified_when_no_change(_Case, #ctx{key = Key} = Ctx) -> @@ -463,15 +479,19 @@ check_broken_pipe(#ctx{handle = Handle, kv = KV, key = Key} = Ctx) -> ensure_fail_pipe(#ctx{handle = Handle, kv = KV, key = Key}) -> ?_test(begin - ?assertThrow(check_error, - couch_epi:apply(Handle, Key, fail, [KV, 2], [pipe])), + ?assertThrow( + check_error, + couch_epi:apply(Handle, Key, fail, [KV, 2], [pipe]) + ), ok end). ensure_fail(#ctx{handle = Handle, kv = KV, key = Key}) -> ?_test(begin - ?assertThrow(check_error, - couch_epi:apply(Handle, Key, fail, [KV, 2], [])), + ?assertThrow( + check_error, + couch_epi:apply(Handle, Key, fail, [KV, 2], []) + ), ok end). @@ -483,52 +503,56 @@ check_dump(_Case, #ctx{handle = Handle}) -> ?_test(begin ?assertMatch( [[{type, counter}, {desc, foo}]], - couch_epi:dump(Handle)) + couch_epi:dump(Handle) + ) end). check_get(_Case, #ctx{handle = Handle}) -> ?_test(begin ?assertMatch( [[{type, counter}, {desc, foo}]], - couch_epi:get(Handle, [complex,key, 1])) + couch_epi:get(Handle, [complex, key, 1]) + ) end). check_get_value(_Case, #ctx{handle = Handle}) -> ?_test(begin ?assertMatch( [{type, counter}, {desc, foo}], - couch_epi:get_value(Handle, test_app, [complex,key, 1])) + couch_epi:get_value(Handle, test_app, [complex, key, 1]) + ) end). check_by_key(_Case, #ctx{handle = Handle}) -> ?_test(begin ?assertMatch( - [{[complex, key, 1], - [{test_app, [{type, counter}, {desc, foo}]}]}], - couch_epi:by_key(Handle)), + [{[complex, key, 1], [{test_app, [{type, counter}, {desc, foo}]}]}], + couch_epi:by_key(Handle) + ), ?assertMatch( [{test_app, [{type, counter}, {desc, foo}]}], - couch_epi:by_key(Handle, [complex, key, 1])) + couch_epi:by_key(Handle, [complex, key, 1]) + ) end). check_by_source(_Case, #ctx{handle = Handle}) -> ?_test(begin ?assertMatch( - [{test_app, - [{[complex,key, 1], [{type, counter}, {desc, foo}]}]}], - couch_epi:by_source(Handle)), + [{test_app, [{[complex, key, 1], [{type, counter}, {desc, foo}]}]}], + couch_epi:by_source(Handle) + ), ?assertMatch( - [{[complex,key, 1], [{type, counter}, {desc, foo}]}], - couch_epi:by_source(Handle, test_app)) + [{[complex, key, 1], [{type, counter}, {desc, foo}]}], + couch_epi:by_source(Handle, test_app) + ) end). check_keys(_Case, #ctx{handle = Handle}) -> - ?_assertMatch([[complex,key,1]], couch_epi:keys(Handle)). + ?_assertMatch([[complex, key, 1]], couch_epi:keys(Handle)). check_subscribers(_Case, #ctx{handle = Handle}) -> ?_assertMatch([test_app], couch_epi:subscribers(Handle)). - ensure_reload_if_manually_triggered(Case, #ctx{pid = Pid, key = Key} = Ctx) -> ?_test(begin subscribe(Ctx, test_app, Key), @@ -538,8 +562,10 @@ ensure_reload_if_manually_triggered(Case, #ctx{pid = Pid, key = Key} = Ctx) -> ?assertNotEqual(error, get(Ctx, is_called)) end). -ensure_reload_if_changed(data_file = Case, - #ctx{key = Key, handle = Handle} = Ctx) -> +ensure_reload_if_changed( + data_file = Case, + #ctx{key = Key, handle = Handle} = Ctx +) -> ?_test(begin Version = Handle:version(), subscribe(Ctx, test_app, Key), @@ -548,19 +574,24 @@ ensure_reload_if_changed(data_file = Case, ?assertNotEqual(Version, Handle:version()), ?assertNotEqual(error, get(Ctx, is_called)) end); -ensure_reload_if_changed(Case, - #ctx{key = Key, handle = Handle} = Ctx) -> +ensure_reload_if_changed( + Case, + #ctx{key = Key, handle = Handle} = Ctx +) -> ?_test(begin Version = Handle:version(), subscribe(Ctx, test_app, Key), update(Case, Ctx), ?assertNotEqual(Version, Handle:version()), - timer:sleep(?RELOAD_WAIT), %% Allow some time for notify to be called + %% Allow some time for notify to be called + timer:sleep(?RELOAD_WAIT), ?assertNotEqual(error, get(Ctx, is_called)) end). -ensure_no_reload_when_no_change(functions, - #ctx{pid = Pid, key = Key, handle = Handle, modules = Modules} = Ctx) -> +ensure_no_reload_when_no_change( + functions, + #ctx{pid = Pid, key = Key, handle = Handle, modules = Modules} = Ctx +) -> ?_test(begin Version = Handle:version(), subscribe(Ctx, test_app, Key), @@ -568,8 +599,10 @@ ensure_no_reload_when_no_change(functions, ?assertEqual(Version, Handle:version()), ?assertEqual(error, get(Ctx, is_called)) end); -ensure_no_reload_when_no_change(_Case, - #ctx{key = Key, handle = Handle} = Ctx) -> +ensure_no_reload_when_no_change( + _Case, + #ctx{key = Key, handle = Handle} = Ctx +) -> ?_test(begin Version = Handle:version(), subscribe(Ctx, test_app, Key), @@ -638,7 +671,8 @@ wait_update(Ctx) -> error -> timer:sleep(?RELOAD_WAIT), wait_update(Ctx); - _ -> ok + _ -> + ok end. %% ------------ diff --git a/src/couch_event/src/couch_event.erl b/src/couch_event/src/couch_event.erl index 9f8e501df..2579349d7 100644 --- a/src/couch_event/src/couch_event.erl +++ b/src/couch_event/src/couch_event.erl @@ -29,37 +29,28 @@ unregister/1 ]). - -define(SERVER, couch_event_server). - notify(DbName, Event) -> gen_server:cast(?SERVER, {notify, DbName, Event}). - listen(Module, Function, State, Options) -> couch_event_listener_mfa:enter_loop(Module, Function, State, Options). - link_listener(Module, Function, State, Options) -> couch_event_listener_mfa:start_link(Module, Function, State, Options). - stop_listener(Pid) -> couch_event_listener_mfa:stop(Pid). - register(Pid, DbName) -> gen_server:call(?SERVER, {register, Pid, [DbName]}). - register_many(Pid, DbNames) when is_list(DbNames) -> gen_server:call(?SERVER, {register, Pid, DbNames}). - register_all(Pid) -> gen_server:call(?SERVER, {register, Pid, [all_dbs]}). - unregister(Pid) -> gen_server:call(?SERVER, {unregister, Pid}). diff --git a/src/couch_event/src/couch_event_app.erl b/src/couch_event/src/couch_event_app.erl index 3a8341b9e..19621f0bf 100644 --- a/src/couch_event/src/couch_event_app.erl +++ b/src/couch_event/src/couch_event_app.erl @@ -18,10 +18,8 @@ stop/1 ]). - start(_StartType, _StartArgs) -> couch_event_sup2:start_link(). - stop(_State) -> ok. diff --git a/src/couch_event/src/couch_event_listener.erl b/src/couch_event/src/couch_event_listener.erl index a9ed33199..40f1a5c65 100644 --- a/src/couch_event/src/couch_event_listener.erl +++ b/src/couch_event/src/couch_event_listener.erl @@ -12,7 +12,6 @@ -module(couch_event_listener). - -export([ start/3, start/4, @@ -27,13 +26,11 @@ loop/2 ]). - -record(st, { module, state }). - -callback init(Arg :: term()) -> term(). @@ -49,12 +46,10 @@ -callback handle_info(Message :: term(), State :: term()) -> term(). - start(Mod, Arg, Options) -> Pid = erlang:spawn(?MODULE, do_init, [Mod, Arg, Options]), {ok, Pid}. - start(Name, Mod, Arg, Options) -> case where(Name) of undefined -> @@ -63,12 +58,10 @@ start(Name, Mod, Arg, Options) -> {error, {already_started, Pid}} end. - start_link(Mod, Arg, Options) -> Pid = erlang:spawn_link(?MODULE, do_init, [Mod, Arg, Options]), {ok, Pid}. - start_link(Name, Mod, Arg, Options) -> case where(Name) of undefined -> @@ -77,30 +70,26 @@ start_link(Name, Mod, Arg, Options) -> {error, {already_started, Pid}} end. - enter_loop(Module, State, Options) -> ok = register_listeners(Options), - ?MODULE:loop(#st{module=Module, state=State}, infinity). - + ?MODULE:loop(#st{module = Module, state = State}, infinity). cast(Pid, Message) -> Pid ! {'$couch_event_cast', Message}, ok. - do_init(Module, Arg, Options) -> ok = maybe_name_process(Options), ok = register_listeners(Options), case (catch Module:init(Arg)) of {ok, State} -> - ?MODULE:loop(#st{module=Module, state=State}, infinity); + ?MODULE:loop(#st{module = Module, state = State}, infinity); {ok, State, Timeout} when is_integer(Timeout), Timeout >= 0 -> - ?MODULE:loop(#st{module=Module, state=State}, Timeout); + ?MODULE:loop(#st{module = Module, state = State}, Timeout); Else -> erlang:exit(Else) end. - loop(St, Timeout) -> receive {'$couch_event', DbName, Event} -> @@ -113,7 +102,6 @@ loop(St, Timeout) -> do_info(St, timeout) end. - maybe_name_process(Options) -> case proplists:lookup(name, Options) of {name, Name} -> @@ -127,7 +115,6 @@ maybe_name_process(Options) -> ok end. - register_listeners(Options) -> case get_all_dbnames(Options) of all_dbs -> @@ -137,85 +124,79 @@ register_listeners(Options) -> end, ok. - -do_event(#st{module=Module, state=State}=St, DbName, Event) -> +do_event(#st{module = Module, state = State} = St, DbName, Event) -> case (catch Module:handle_event(DbName, Event, State)) of {ok, NewState} -> - ?MODULE:loop(St#st{state=NewState}, infinity); + ?MODULE:loop(St#st{state = NewState}, infinity); {ok, NewState, Timeout} when is_integer(Timeout), Timeout >= 0 -> - ?MODULE:loop(St#st{state=NewState}, Timeout); + ?MODULE:loop(St#st{state = NewState}, Timeout); {stop, Reason, NewState} -> - do_terminate(Reason, St#st{state=NewState}); + do_terminate(Reason, St#st{state = NewState}); Else -> erlang:error(Else) end. - -do_cast(#st{module=Module, state=State}=St, Message) -> +do_cast(#st{module = Module, state = State} = St, Message) -> case (catch Module:handle_cast(Message, State)) of {ok, NewState} -> - ?MODULE:loop(St#st{state=NewState}, infinity); + ?MODULE:loop(St#st{state = NewState}, infinity); {ok, NewState, Timeout} when is_integer(Timeout), Timeout >= 0 -> - ?MODULE:loop(St#st{state=NewState}, Timeout); + ?MODULE:loop(St#st{state = NewState}, Timeout); {stop, Reason, NewState} -> - do_terminate(Reason, St#st{state=NewState}); + do_terminate(Reason, St#st{state = NewState}); Else -> erlang:error(Else) end. - -do_info(#st{module=Module, state=State}=St, Message) -> +do_info(#st{module = Module, state = State} = St, Message) -> case (catch Module:handle_info(Message, State)) of {ok, NewState} -> - ?MODULE:loop(St#st{state=NewState}, infinity); + ?MODULE:loop(St#st{state = NewState}, infinity); {ok, NewState, Timeout} when is_integer(Timeout), Timeout >= 0 -> - ?MODULE:loop(St#st{state=NewState}, Timeout); + ?MODULE:loop(St#st{state = NewState}, Timeout); {stop, Reason, NewState} -> - do_terminate(Reason, St#st{state=NewState}); + do_terminate(Reason, St#st{state = NewState}); Else -> erlang:error(Else) end. - -do_terminate(Reason, #st{module=Module, state=State}) -> +do_terminate(Reason, #st{module = Module, state = State}) -> % Order matters. We want to make sure Module:terminate/1 % is called even if couch_event:unregister/1 hangs % indefinitely. catch Module:terminate(Reason, State), catch couch_event:unregister(self()), - Status = case Reason of - normal -> normal; - shutdown -> normal; - ignore -> normal; - Else -> Else - end, + Status = + case Reason of + normal -> normal; + shutdown -> normal; + ignore -> normal; + Else -> Else + end, erlang:exit(Status). - where({global, Name}) -> global:whereis_name(Name); where({local, Name}) -> whereis(Name). - -name_register({global, Name}=GN) -> +name_register({global, Name} = GN) -> case global:register_name(Name, self()) of yes -> true; no -> {false, where(GN)} end; -name_register({local, Name}=LN) -> +name_register({local, Name} = LN) -> try register(Name, self()) of true -> true - catch error:_ -> - {false, where(LN)} + catch + error:_ -> + {false, where(LN)} end. - get_all_dbnames(Options) -> case proplists:get_value(all_dbs, Options) of true -> all_dbs; _ -> get_all_dbnames(Options, []) end. - get_all_dbnames([], []) -> erlang:error(no_dbnames_provided); get_all_dbnames([], Acc) -> @@ -227,7 +208,6 @@ get_all_dbnames([{dbnames, DbNames} | Rest], Acc) when is_list(DbNames) -> get_all_dbnames([_Ignored | Rest], Acc) -> get_all_dbnames(Rest, Acc). - convert_dbname_list([]) -> []; convert_dbname_list([DbName | Rest]) when is_binary(DbName) -> diff --git a/src/couch_event/src/couch_event_listener_mfa.erl b/src/couch_event/src/couch_event_listener_mfa.erl index 9be58880a..b4cd9148a 100644 --- a/src/couch_event/src/couch_event_listener_mfa.erl +++ b/src/couch_event/src/couch_event_listener_mfa.erl @@ -13,7 +13,6 @@ -module(couch_event_listener_mfa). -behavior(couch_event_listener). - -export([ start_link/4, enter_loop/4, @@ -28,7 +27,6 @@ handle_info/2 ]). - -record(st, { mod, func, @@ -36,24 +34,24 @@ parent }). - start_link(Mod, Func, State, Options) -> - Parent = case proplists:get_value(parent, Options) of - P when is_pid(P) -> P; - _ -> self() - end, + Parent = + case proplists:get_value(parent, Options) of + P when is_pid(P) -> P; + _ -> self() + end, Arg = {Parent, Mod, Func, State}, couch_event_listener:start_link(?MODULE, Arg, Options). - enter_loop(Mod, Func, State, Options) -> - Parent = case proplists:get_value(parent, Options) of - P when is_pid(P) -> - erlang:monitor(process, P), - P; - _ -> - undefined - end, + Parent = + case proplists:get_value(parent, Options) of + P when is_pid(P) -> + erlang:monitor(process, P), + P; + _ -> + undefined + end, St = #st{ mod = Mod, func = Func, @@ -62,11 +60,9 @@ enter_loop(Mod, Func, State, Options) -> }, couch_event_listener:enter_loop(?MODULE, St, Options). - stop(Pid) -> couch_event_listener:cast(Pid, shutdown). - init({Parent, Mod, Func, State}) -> erlang:monitor(process, Parent), {ok, #st{ @@ -76,32 +72,25 @@ init({Parent, Mod, Func, State}) -> parent = Parent }}. - terminate(_Reason, _MFA) -> ok. - -handle_event(DbName, Event, #st{mod=Mod, func=Func, state=State}=St) -> +handle_event(DbName, Event, #st{mod = Mod, func = Func, state = State} = St) -> case (catch Mod:Func(DbName, Event, State)) of {ok, NewState} -> - {ok, St#st{state=NewState}}; + {ok, St#st{state = NewState}}; stop -> {stop, normal, St}; Else -> erlang:error(Else) end. - handle_cast(shutdown, St) -> {stop, normal, St}; - handle_cast(_Msg, St) -> {ok, St}. - -handle_info({'DOWN', _Ref, process, Parent, _Reason}, #st{parent=Parent}=St) -> +handle_info({'DOWN', _Ref, process, Parent, _Reason}, #st{parent = Parent} = St) -> {stop, normal, St}; - handle_info(_Msg, St) -> {ok, St}. - diff --git a/src/couch_event/src/couch_event_os_listener.erl b/src/couch_event/src/couch_event_os_listener.erl index 4de0a4416..ef379402a 100644 --- a/src/couch_event/src/couch_event_os_listener.erl +++ b/src/couch_event/src/couch_event_os_listener.erl @@ -14,7 +14,6 @@ -behaviour(gen_server). -vsn(1). - -export([ start_link/1 ]). @@ -28,49 +27,41 @@ code_change/3 ]). - start_link(Exe) when is_list(Exe) -> gen_server:start_link(?MODULE, Exe, []). - init(Exe) -> process_flag(trap_exit, true), ok = couch_event:register_all(self()), couch_os_process:start_link(Exe, []). - terminate(_Reason, Pid) when is_pid(Pid) -> couch_os_process:stop(Pid); terminate(_Reason, _Pid) -> ok. - handle_call(Msg, From, Pid) -> couch_log:notice("~s ignoring call ~w from ~w", [?MODULE, Msg, From]), {reply, ignored, Pid, 0}. - handle_cast(Msg, Pid) -> couch_log:notice("~s ignoring cast ~w", [?MODULE, Msg]), {noreply, Pid, 0}. - handle_info({'$couch_event', DbName, Event}, Pid) -> - Obj = {[ - {db, DbName}, - {type, list_to_binary(atom_to_list(Event))} - ]}, + Obj = + {[ + {db, DbName}, + {type, list_to_binary(atom_to_list(Event))} + ]}, ok = couch_os_process:send(Pid, Obj), {noreply, Pid}; - handle_info({'EXIT', Pid, Reason}, Pid) -> couch_log:error("Update notificatio process ~w died: ~w", [Pid, Reason]), {stop, normal, nil}; - handle_info(Msg, Pid) -> couch_log:notice("~s ignoring info ~w", [?MODULE, Msg]), {noreply, Pid, 0}. - code_change(_OldVsn, St, _Extra) -> {ok, St}. diff --git a/src/couch_event/src/couch_event_server.erl b/src/couch_event/src/couch_event_server.erl index 321e8fafd..f633a8409 100644 --- a/src/couch_event/src/couch_event_server.erl +++ b/src/couch_event/src/couch_event_server.erl @@ -14,7 +14,6 @@ -behaviour(gen_server). -vsn(1). - -export([ start_link/0 ]). @@ -28,20 +27,16 @@ code_change/3 ]). - -include("couch_event_int.hrl"). - -record(st, { by_pid, by_dbname }). - start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, nil, []). - init(_) -> {ok, ByPid} = khash:new(), {ok, ByDbName} = khash:new(), @@ -50,11 +45,9 @@ init(_) -> by_dbname = ByDbName }}. - terminate(_Reason, _St) -> ok. - handle_call({register, Pid, NewDbNames}, _From, St) -> case khash:get(St#st.by_pid, Pid) of undefined -> @@ -65,32 +58,28 @@ handle_call({register, Pid, NewDbNames}, _From, St) -> register(St, ReuseRef, Pid, NewDbNames) end, {reply, ok, St}; - handle_call({unregister, Pid}, _From, St) -> - Reply = case khash:get(St#st.by_pid, Pid) of - undefined -> - not_registered; - {Ref, OldDbNames} -> - unregister(St, Pid, OldDbNames), - erlang:demonitor(Ref, [flush]), - ok - end, + Reply = + case khash:get(St#st.by_pid, Pid) of + undefined -> + not_registered; + {Ref, OldDbNames} -> + unregister(St, Pid, OldDbNames), + erlang:demonitor(Ref, [flush]), + ok + end, {reply, Reply, St}; - handle_call(Msg, From, St) -> couch_log:notice("~s ignoring call ~w from ~w", [?MODULE, Msg, From]), {reply, ignored, St}. - handle_cast({notify, DbName, Event}, St) -> notify_listeners(St#st.by_dbname, DbName, Event), {noreply, St}; - handle_cast(Msg, St) -> couch_log:notice("~s ignoring cast ~w", [?MODULE, Msg]), {noreply, St}. - handle_info({'DOWN', Ref, process, Pid, _Reason}, St) -> case khash:get(St#st.by_pid, Pid) of {Ref, OldDbNames} -> @@ -99,35 +88,38 @@ handle_info({'DOWN', Ref, process, Pid, _Reason}, St) -> ok end, {noreply, St}; - - handle_info(Msg, St) -> couch_log:notice("~s ignoring info ~w", [?MODULE, Msg]), {noreply, St}. - code_change(_OldVsn, St, _Extra) -> {ok, St}. - notify_listeners(ByDbName, DbName, Event) -> Msg = {'$couch_event', DbName, Event}, notify_listeners(khash:get(ByDbName, all_dbs), Msg), notify_listeners(khash:get(ByDbName, DbName), Msg). - notify_listeners(undefined, _) -> ok; notify_listeners(Listeners, Msg) -> - khash:fold(Listeners, fun(Pid, _, _) -> Pid ! Msg, nil end, nil). - + khash:fold( + Listeners, + fun(Pid, _, _) -> + Pid ! Msg, + nil + end, + nil + ). register(St, Ref, Pid, DbNames) -> khash:put(St#st.by_pid, Pid, {Ref, DbNames}), - lists:foreach(fun(DbName) -> - add_listener(St#st.by_dbname, DbName, Pid) - end, DbNames). - + lists:foreach( + fun(DbName) -> + add_listener(St#st.by_dbname, DbName, Pid) + end, + DbNames + ). add_listener(ByDbName, DbName, Pid) -> case khash:lookup(ByDbName, DbName) of @@ -139,18 +131,20 @@ add_listener(ByDbName, DbName, Pid) -> khash:put(ByDbName, DbName, NewListeners) end. - unregister(St, Pid, OldDbNames) -> ok = khash:del(St#st.by_pid, Pid), - lists:foreach(fun(DbName) -> - rem_listener(St#st.by_dbname, DbName, Pid) - end, OldDbNames). - + lists:foreach( + fun(DbName) -> + rem_listener(St#st.by_dbname, DbName, Pid) + end, + OldDbNames + ). rem_listener(ByDbName, DbName, Pid) -> {value, Listeners} = khash:lookup(ByDbName, DbName), khash:del(Listeners, Pid), Size = khash:size(Listeners), - if Size > 0 -> ok; true -> - khash:del(ByDbName, DbName) + if + Size > 0 -> ok; + true -> khash:del(ByDbName, DbName) end. diff --git a/src/couch_event/src/couch_event_sup2.erl b/src/couch_event/src/couch_event_sup2.erl index 2d88b93d4..a815c440b 100644 --- a/src/couch_event/src/couch_event_sup2.erl +++ b/src/couch_event/src/couch_event_sup2.erl @@ -19,26 +19,18 @@ -module(couch_event_sup2). -behavior(supervisor). - -export([ start_link/0, init/1 ]). - start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, nil). - init(_) -> Children = [ - {couch_event_server, - {couch_event_server, start_link, []}, - permanent, - 5000, - worker, - [couch_event_server] - } + {couch_event_server, {couch_event_server, start_link, []}, permanent, 5000, worker, [ + couch_event_server + ]} ], {ok, {{one_for_one, 5, 10}, Children}}. - diff --git a/src/couch_index/src/couch_index.erl b/src/couch_index/src/couch_index.erl index 83eadc706..a6b62be7c 100644 --- a/src/couch_index/src/couch_index.erl +++ b/src/couch_index/src/couch_index.erl @@ -26,48 +26,40 @@ -export([init/1, terminate/2, code_change/3]). -export([handle_call/3, handle_cast/2, handle_info/2]). - -include_lib("couch/include/couch_db.hrl"). - --define(CHECK_INTERVAL, 600000). % 10 minutes +% 10 minutes +-define(CHECK_INTERVAL, 600000). -record(st, { mod, idx_state, updater, compactor, - waiters=[], - committed=true, - shutdown=false + waiters = [], + committed = true, + shutdown = false }). - start_link({Module0, IdxState0}) -> [Module, IdxState] = couch_index_plugin:before_open(Module0, IdxState0), proc_lib:start_link(?MODULE, init, [{Module, IdxState}]). - stop(Pid) -> gen_server:cast(Pid, stop). - get_state(Pid, RequestSeq) -> gen_server:call(Pid, {get_state, RequestSeq}, infinity). - get_info(Pid) -> gen_server:call(Pid, get_info, group_info_timeout_msec()). - trigger_update(Pid, UpdateSeq) -> gen_server:cast(Pid, {trigger_update, UpdateSeq}). - compact(Pid) -> compact(Pid, []). - compact(Pid, Options) -> {ok, CPid} = gen_server:call(Pid, compact), case lists:member(monitor, Options) of @@ -75,7 +67,6 @@ compact(Pid, Options) -> false -> ok end. - get_compactor_pid(Pid) -> gen_server:call(Pid, get_compactor_pid). @@ -96,10 +87,10 @@ init({Mod, IdxState}) -> {ok, UPid} = couch_index_updater:start_link(self(), Mod), {ok, CPid} = couch_index_compactor:start_link(self(), Mod), State = #st{ - mod=Mod, - idx_state=NewIdxState, - updater=UPid, - compactor=CPid + mod = Mod, + idx_state = NewIdxState, + updater = UPid, + compactor = CPid }, Args = [ Mod:get(db_name, IdxState), @@ -113,9 +104,8 @@ init({Mod, IdxState}) -> proc_lib:init_ack(Other) end. - terminate(Reason0, State) -> - #st{mod=Mod, idx_state=IdxState}=State, + #st{mod = Mod, idx_state = IdxState} = State, case Reason0 of {shutdown, ddoc_updated} -> Mod:shutdown(IdxState), @@ -136,24 +126,24 @@ terminate(Reason0, State) -> couch_log:debug("Closing index for db: ~s idx: ~s sig: ~p because ~r", Args), ok. - handle_call({get_state, ReqSeq}, From, State) -> #st{ - mod=Mod, - idx_state=IdxState, - waiters=Waiters + mod = Mod, + idx_state = IdxState, + waiters = Waiters } = State, IdxSeq = Mod:get(update_seq, IdxState), case ReqSeq =< IdxSeq of true -> {reply, {ok, IdxState}, State}; - _ -> % View update required + % View update required + _ -> couch_index_updater:run(State#st.updater, IdxState), Waiters2 = [{From, ReqSeq} | Waiters], - {noreply, State#st{waiters=Waiters2}, infinity} + {noreply, State#st{waiters = Waiters2}, infinity} end; handle_call(get_info, _From, State) -> - #st{mod=Mod} = State, + #st{mod = Mod} = State, IdxState = State#st.idx_state, {ok, Info0} = Mod:get(info, IdxState), IsUpdating = couch_index_updater:is_running(State#st.updater), @@ -162,21 +152,23 @@ handle_call(get_info, _From, State) -> GetCommSeq = fun(Db) -> couch_db:get_committed_update_seq(Db) end, DbName = Mod:get(db_name, IdxState), CommittedSeq = couch_util:with_db(DbName, GetCommSeq), - Info = Info0 ++ [ - {updater_running, IsUpdating}, - {compact_running, IsCompacting}, - {waiting_commit, State#st.committed == false}, - {waiting_clients, length(State#st.waiters)}, - {pending_updates, max(CommittedSeq - IdxSeq, 0)} - ], + Info = + Info0 ++ + [ + {updater_running, IsUpdating}, + {compact_running, IsCompacting}, + {waiting_commit, State#st.committed == false}, + {waiting_clients, length(State#st.waiters)}, + {pending_updates, max(CommittedSeq - IdxSeq, 0)} + ], {reply, {ok, Info}, State}; handle_call(reset, _From, State) -> #st{ - mod=Mod, - idx_state=IdxState + mod = Mod, + idx_state = IdxState } = State, {ok, NewIdxState} = Mod:reset(IdxState), - {reply, {ok, NewIdxState}, State#st{idx_state=NewIdxState}}; + {reply, {ok, NewIdxState}, State#st{idx_state = NewIdxState}}; handle_call(compact, _From, State) -> Resp = couch_index_compactor:run(State#st.compactor, State#st.idx_state), {reply, Resp, State}; @@ -184,8 +176,8 @@ handle_call(get_compactor_pid, _From, State) -> {reply, {ok, State#st.compactor}, State}; handle_call({compacted, NewIdxState}, _From, State) -> #st{ - mod=Mod, - idx_state=OldIdxState + mod = Mod, + idx_state = OldIdxState } = State, assert_signature_match(Mod, OldIdxState, NewIdxState), NewSeq = Mod:get(update_seq, NewIdxState), @@ -215,8 +207,8 @@ handle_call({compaction_failed, Reason}, _From, State) -> handle_cast({trigger_update, UpdateSeq}, State) -> #st{ - mod=Mod, - idx_state=IdxState + mod = Mod, + idx_state = IdxState } = State, case UpdateSeq =< Mod:get(update_seq, IdxState) of true -> @@ -236,8 +228,8 @@ handle_cast({updated, NewIdxState}, State) -> end; handle_cast({new_state, NewIdxState}, State) -> #st{ - mod=Mod, - idx_state=OldIdxState + mod = Mod, + idx_state = OldIdxState } = State, OldFd = Mod:get(fd, OldIdxState), NewFd = Mod:get(fd, NewIdxState), @@ -257,9 +249,9 @@ handle_cast({new_state, NewIdxState}, State) -> false -> ok end, {noreply, State#st{ - idx_state=NewIdxState, - waiters=Rest, - committed=false + idx_state = NewIdxState, + waiters = Rest, + committed = false }}; false -> Fmt = "Ignoring update from old indexer for db: ~s idx: ~s", @@ -272,25 +264,26 @@ handle_cast({new_state, NewIdxState}, State) -> end; handle_cast({update_error, Error}, State) -> send_all(State#st.waiters, Error), - {noreply, State#st{waiters=[]}}; + {noreply, State#st{waiters = []}}; handle_cast(stop, State) -> {stop, normal, State}; handle_cast(delete, State) -> - #st{mod=Mod, idx_state=IdxState} = State, + #st{mod = Mod, idx_state = IdxState} = State, ok = Mod:delete(IdxState), {stop, normal, State}; handle_cast({ddoc_updated, DDocResult}, State) -> #st{mod = Mod, idx_state = IdxState} = State, - Shutdown = case DDocResult of - {not_found, deleted} -> - true; - {ok, DDoc} -> - DbName = Mod:get(db_name, IdxState), - couch_util:with_db(DbName, fun(Db) -> - {ok, NewIdxState} = Mod:init(Db, DDoc), - Mod:get(signature, NewIdxState) =/= Mod:get(signature, IdxState) - end) - end, + Shutdown = + case DDocResult of + {not_found, deleted} -> + true; + {ok, DDoc} -> + DbName = Mod:get(db_name, IdxState), + couch_util:with_db(DbName, fun(Db) -> + {ok, NewIdxState} = Mod:init(Db, DDoc), + Mod:get(signature, NewIdxState) =/= Mod:get(signature, IdxState) + end) + end, case Shutdown of true -> {stop, {shutdown, ddoc_updated}, State#st{shutdown = true}}; @@ -319,10 +312,10 @@ handle_cast(ddoc_updated, State) -> handle_cast(_Mesg, State) -> {stop, unhandled_cast, State}. -handle_info(commit, #st{committed=true}=State) -> +handle_info(commit, #st{committed = true} = State) -> {noreply, State}; handle_info(commit, State) -> - #st{mod=Mod, idx_state=IdxState} = State, + #st{mod = Mod, idx_state = IdxState} = State, DbName = Mod:get(db_name, IdxState), IdxName = Mod:get(idx_name, IdxState), GetCommSeq = fun(Db) -> couch_db:get_committed_update_seq(Db) end, @@ -332,7 +325,7 @@ handle_info(commit, State) -> % Commit the updates ok = Mod:commit(IdxState), couch_event:notify(DbName, {index_commit, IdxName}), - {noreply, State#st{committed=true}}; + {noreply, State#st{committed = true}}; _ -> % We can't commit the header because the database seq that's % fully committed to disk is still behind us. If we committed @@ -366,25 +359,23 @@ handle_info(maybe_close, State) -> erlang:send_after(?CHECK_INTERVAL, self(), maybe_close), {noreply, State} end; -handle_info({'DOWN', _, _, _Pid, _}, #st{mod=Mod, idx_state=IdxState}=State) -> +handle_info({'DOWN', _, _, _Pid, _}, #st{mod = Mod, idx_state = IdxState} = State) -> Args = [Mod:get(db_name, IdxState), Mod:get(idx_name, IdxState)], couch_log:debug("Index shutdown by monitor notice for db: ~s idx: ~s", Args), catch send_all(State#st.waiters, shutdown), - {stop, normal, State#st{waiters=[]}}. + {stop, normal, State#st{waiters = []}}. code_change(_OldVsn, State, _Extra) -> {ok, State}. -maybe_restart_updater(#st{waiters=[]}) -> +maybe_restart_updater(#st{waiters = []}) -> ok; -maybe_restart_updater(#st{idx_state=IdxState}=State) -> +maybe_restart_updater(#st{idx_state = IdxState} = State) -> couch_index_updater:run(State#st.updater, IdxState). - send_all(Waiters, Reply) -> [gen_server:reply(From, Reply) || {From, _} <- Waiters]. - send_replies(Waiters, UpdateSeq, IdxState) -> Pred = fun({_, S}) -> S =< UpdateSeq end, {ToSend, Remaining} = lists:partition(Pred, Waiters), @@ -399,9 +390,9 @@ assert_signature_match(Mod, OldIdxState, NewIdxState) -> commit_compacted(NewIdxState, State) -> #st{ - mod=Mod, - idx_state=OldIdxState, - updater=Updater + mod = Mod, + idx_state = OldIdxState, + updater = Updater } = State, {ok, NewIdxState1} = Mod:swap_compacted(OldIdxState, NewIdxState), % Restart the indexer if it's running. @@ -414,9 +405,9 @@ commit_compacted(NewIdxState, State) -> false -> ok end, State#st{ - idx_state=NewIdxState1, - committed=false - }. + idx_state = NewIdxState1, + committed = false + }. is_recompaction_enabled(IdxState, #st{mod = Mod}) -> DbName = binary_to_list(Mod:get(db_name, IdxState)), @@ -449,7 +440,6 @@ get_value(Section, Key) -> commit_delay() -> config:get_integer("query_server_config", "commit_freq", 5) * 1000. - group_info_timeout_msec() -> Timeout = config:get("query_server_config", "group_info_timeout", "5000"), case Timeout of @@ -459,7 +449,6 @@ group_info_timeout_msec() -> list_to_integer(Milliseconds) end. - -ifdef(TEST). -include_lib("couch/include/couch_eunit.hrl"). @@ -468,7 +457,7 @@ get(db_name, _, _) -> get(idx_name, _, _) -> <<"idx_name">>; get(signature, _, _) -> - <<61,237,157,230,136,93,96,201,204,17,137,186,50,249,44,135>>. + <<61, 237, 157, 230, 136, 93, 96, 201, 204, 17, 137, 186, 50, 249, 44, 135>>. setup_all() -> Ctx = test_util:start_couch(), @@ -526,7 +515,7 @@ recompaction_configuration_tests() -> EnabledCases = [ [undefined, undefined, undefined], - [undefined, undefined,"enabled"], + [undefined, undefined, "enabled"], [undefined, "enabled", undefined], [undefined, "disabled", "enabled"], [undefined, "enabled", "enabled"], @@ -563,21 +552,26 @@ recompaction_configuration_tests() -> ?assertEqual([], AllCases -- (EnabledCases ++ DisabledCases)), - [{Settings, fun should_not_call_recompact/2} || Settings <- DisabledCases] - ++ - [{Settings, fun should_call_recompact/2} || Settings <- EnabledCases]. + [{Settings, fun should_not_call_recompact/2} || Settings <- DisabledCases] ++ + [{Settings, fun should_call_recompact/2} || Settings <- EnabledCases]. should_call_recompact(Settings, {IdxState, State}) -> - {test_id(Settings), ?_test(begin - ?assert(is_recompaction_enabled(IdxState, State)), - ok - end)}. + { + test_id(Settings), + ?_test(begin + ?assert(is_recompaction_enabled(IdxState, State)), + ok + end) + }. should_not_call_recompact(Settings, {IdxState, State}) -> - {test_id(Settings), ?_test(begin - ?assertNot(is_recompaction_enabled(IdxState, State)), - ok - end)}. + { + test_id(Settings), + ?_test(begin + ?assertNot(is_recompaction_enabled(IdxState, State)), + ok + end) + }. to_string(undefined) -> "undefined"; to_string(Value) -> Value. @@ -586,7 +580,6 @@ test_id(Settings0) -> Settings1 = [to_string(Value) || Value <- Settings0], "[ " ++ lists:flatten(string:join(Settings1, " , ")) ++ " ]". - get_group_timeout_info_test_() -> { foreach, @@ -598,25 +591,28 @@ get_group_timeout_info_test_() -> ] }. - t_group_timeout_info_integer() -> - ?_test(begin - meck:expect(config, get, + ?_test(begin + meck:expect( + config, + get, fun("query_server_config", "group_info_timeout", _) -> - "5001" - end), + "5001" + end + ), ?assertEqual(5001, group_info_timeout_msec()) end). - t_group_timeout_info_infinity() -> - ?_test(begin - meck:expect(config, get, + ?_test(begin + meck:expect( + config, + get, fun("query_server_config", "group_info_timeout", _) -> "infinity" - end), + end + ), ?assertEqual(infinity, group_info_timeout_msec()) end). - -endif. diff --git a/src/couch_index/src/couch_index_compactor.erl b/src/couch_index/src/couch_index_compactor.erl index 8849cf67d..8b592d140 100644 --- a/src/couch_index/src/couch_index_compactor.erl +++ b/src/couch_index/src/couch_index_compactor.erl @@ -13,7 +13,6 @@ -module(couch_index_compactor). -behaviour(gen_server). - %% API -export([start_link/2, run/2, cancel/1, is_running/1, get_compacting_pid/1]). @@ -21,29 +20,23 @@ -export([init/1, terminate/2, code_change/3]). -export([handle_call/3, handle_cast/2, handle_info/2]). - -include_lib("couch/include/couch_db.hrl"). - -record(st, { idx, mod, pid }). - start_link(Index, Module) -> gen_server:start_link(?MODULE, {Index, Module}, []). - run(Pid, IdxState) -> gen_server:call(Pid, {compact, IdxState}). - cancel(Pid) -> gen_server:call(Pid, cancel). - is_running(Pid) -> gen_server:call(Pid, is_running). @@ -52,39 +45,35 @@ get_compacting_pid(Pid) -> init({Index, Module}) -> process_flag(trap_exit, true), - {ok, #st{idx=Index, mod=Module}}. - + {ok, #st{idx = Index, mod = Module}}. terminate(_Reason, State) -> couch_util:shutdown_sync(State#st.pid), ok. - -handle_call({compact, _}, _From, #st{pid=Pid}=State) when is_pid(Pid) -> +handle_call({compact, _}, _From, #st{pid = Pid} = State) when is_pid(Pid) -> {reply, {ok, Pid}, State}; -handle_call({compact, IdxState}, _From, #st{idx=Idx}=State) -> +handle_call({compact, IdxState}, _From, #st{idx = Idx} = State) -> Pid = spawn_link(fun() -> compact(Idx, State#st.mod, IdxState) end), - {reply, {ok, Pid}, State#st{pid=Pid}}; -handle_call(cancel, _From, #st{pid=undefined}=State) -> + {reply, {ok, Pid}, State#st{pid = Pid}}; +handle_call(cancel, _From, #st{pid = undefined} = State) -> {reply, ok, State}; -handle_call(cancel, _From, #st{pid=Pid}=State) -> +handle_call(cancel, _From, #st{pid = Pid} = State) -> unlink(Pid), exit(Pid, kill), - {reply, ok, State#st{pid=undefined}}; -handle_call(get_compacting_pid, _From, #st{pid=Pid}=State) -> + {reply, ok, State#st{pid = undefined}}; +handle_call(get_compacting_pid, _From, #st{pid = Pid} = State) -> {reply, {ok, Pid}, State}; -handle_call(is_running, _From, #st{pid=Pid}=State) when is_pid(Pid) -> +handle_call(is_running, _From, #st{pid = Pid} = State) when is_pid(Pid) -> {reply, true, State}; handle_call(is_running, _From, State) -> {reply, false, State}. - handle_cast(_Mesg, State) -> {stop, unknown_cast, State}. - -handle_info({'EXIT', Pid, normal}, #st{pid=Pid}=State) -> - {noreply, State#st{pid=undefined}}; +handle_info({'EXIT', Pid, normal}, #st{pid = Pid} = State) -> + {noreply, State#st{pid = undefined}}; handle_info({'EXIT', Pid, Reason}, #st{pid = Pid} = State) -> #st{idx = Idx, mod = Mod} = State, {ok, IdxState} = gen_server:call(Idx, {compaction_failed, Reason}), @@ -95,16 +84,14 @@ handle_info({'EXIT', Pid, Reason}, #st{pid = Pid} = State) -> {noreply, State#st{pid = undefined}}; handle_info({'EXIT', _Pid, normal}, State) -> {noreply, State}; -handle_info({'EXIT', Pid, _Reason}, #st{idx=Pid}=State) -> +handle_info({'EXIT', Pid, _Reason}, #st{idx = Pid} = State) -> {stop, normal, State}; handle_info(_Mesg, State) -> {stop, unknown_info, State}. - code_change(_OldVsn, State, _Extra) -> {ok, State}. - compact(Parent, Mod, IdxState) -> DbName = Mod:get(db_name, IdxState), %% We use with_db here to make sure we hold db open diff --git a/src/couch_index/src/couch_index_plugin_couch_db.erl b/src/couch_index/src/couch_index_plugin_couch_db.erl index 0af22e396..b90fa6cf2 100644 --- a/src/couch_index/src/couch_index_plugin_couch_db.erl +++ b/src/couch_index/src/couch_index_plugin_couch_db.erl @@ -17,10 +17,8 @@ on_compact/2 ]). - is_valid_purge_client(DbName, Props) -> couch_mrview_index:verify_index_exists(DbName, Props). - on_compact(DbName, DDocs) -> couch_mrview_index:ensure_local_purge_docs(DbName, DDocs). diff --git a/src/couch_index/src/couch_index_server.erl b/src/couch_index/src/couch_index_server.erl index 6bebff2d8..77f91cc5b 100644 --- a/src/couch_index/src/couch_index_server.erl +++ b/src/couch_index/src/couch_index_server.erl @@ -40,37 +40,37 @@ start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - validate(Db, DDoc) -> LoadModFun = fun ({ModNameList, "true"}) -> try [list_to_existing_atom(ModNameList)] - catch error:badarg -> - [] + catch + error:badarg -> + [] end; ({_ModNameList, _Enabled}) -> [] end, - ValidateFun = fun - (ModName) -> - ModName:validate(Db, DDoc) + ValidateFun = fun(ModName) -> + ModName:validate(Db, DDoc) end, EnabledIndexers = lists:flatmap(LoadModFun, config:get("indexers")), lists:foreach(ValidateFun, EnabledIndexers). - -get_index(Module, <<"shards/", _/binary>> = DbName, DDoc) - when is_record(DDoc, doc) -> +get_index(Module, <<"shards/", _/binary>> = DbName, DDoc) when + is_record(DDoc, doc) +-> get_index(Module, DbName, DDoc, nil); get_index(Module, <<"shards/", _/binary>> = DbName, DDoc) -> {Pid, Ref} = spawn_monitor(fun() -> exit(fabric:open_doc(mem3:dbname(DbName), DDoc, [ejson_body, ?ADMIN_CTX])) end), - receive {'DOWN', Ref, process, Pid, {ok, Doc}} -> - get_index(Module, DbName, Doc, nil); - {'DOWN', Ref, process, Pid, Error} -> - Error + receive + {'DOWN', Ref, process, Pid, {ok, Doc}} -> + get_index(Module, DbName, Doc, nil); + {'DOWN', Ref, process, Pid, Error} -> + Error after 61000 -> erlang:demonitor(Ref, [flush]), {error, timeout} @@ -80,7 +80,6 @@ get_index(Module, DbName, DDoc) when is_binary(DbName) -> get_index(Module, Db, DDoc) -> get_index(Module, couch_db:name(Db), DDoc). - get_index(Module, DbName, DDoc, Fun) when is_binary(DbName) -> couch_util:with_db(DbName, fun(Db) -> get_index(Module, Db, DDoc, Fun) @@ -99,7 +98,6 @@ get_index(Module, Db, DDoc, _Fun) -> {ok, InitState} = Module:init(Db, DDoc), get_index(Module, InitState). - get_index(Module, IdxState) -> DbName = Module:get(db_name, IdxState), Sig = Module:get(signature, IdxState), @@ -110,7 +108,8 @@ get_index(Module, IdxState) -> [] -> Args = [Pid, DbName, DDocId, Sig], gen_server:cast(?MODULE, {add_to_ets, Args}); - _ -> ok + _ -> + ok end, {ok, Pid}; _ -> @@ -118,7 +117,6 @@ get_index(Module, IdxState) -> gen_server:call(?MODULE, {get_index, Args}, infinity) end. - init([]) -> process_flag(trap_exit, true), ok = config:listen_for_changes(?MODULE, couch_index_util:root_dir()), @@ -128,16 +126,14 @@ init([]) -> couch_event:link_listener(?MODULE, handle_db_event, nil, [all_dbs]), RootDir = couch_index_util:root_dir(), couch_file:init_delete_dir(RootDir), - {ok, #st{root_dir=RootDir}}. - + {ok, #st{root_dir = RootDir}}. terminate(_Reason, _State) -> Pids = [Pid || {Pid, _} <- ets:tab2list(?BY_PID)], lists:map(fun couch_util:shutdown_sync/1, Pids), ok. - -handle_call({get_index, {_Mod, _IdxState, DbName, Sig}=Args}, From, State) -> +handle_call({get_index, {_Mod, _IdxState, DbName, Sig} = Args}, From, State) -> case ets:lookup(?BY_SIG, {DbName, Sig}) of [] -> spawn_link(fun() -> new_index(Args) end), @@ -164,7 +160,6 @@ handle_call({reset_indexes, DbName}, _From, State) -> reset_indexes(DbName, State#st.root_dir), {reply, ok, State}. - handle_cast({reset_indexes, DbName}, State) -> reset_indexes(DbName, State#st.root_dir), {noreply, State}; @@ -173,7 +168,8 @@ handle_cast({add_to_ets, [Pid, DbName, DDocId, Sig]}, State) -> case ets:lookup(?BY_PID, Pid) of [{Pid, {DbName, Sig}}] when is_pid(Pid) -> ets:insert(?BY_DB, {DbName, {DDocId, Sig}}); - _ -> ok + _ -> + ok end, {noreply, State}; handle_cast({rem_from_ets, [DbName, DDocId, Sig]}, State) -> @@ -183,8 +179,11 @@ handle_cast({rem_from_ets, [DbName, DDocId, Sig]}, State) -> handle_info({'EXIT', Pid, Reason}, Server) -> case ets:lookup(?BY_PID, Pid) of [{Pid, {DbName, Sig}}] -> - DDocIds = [DDocId || {_, {DDocId, _}} - <- ets:match_object(?BY_DB, {DbName, {'$1', Sig}})], + DDocIds = [ + DDocId + || {_, {DDocId, _}} <- + ets:match_object(?BY_DB, {DbName, {'$1', Sig}}) + ], rem_from_ets(DbName, Sig, DDocIds, Pid); [] when Reason /= normal -> exit(Reason); @@ -199,11 +198,9 @@ handle_info(Msg, State) -> couch_log:warning("~p did not expect ~p", [?MODULE, Msg]), {noreply, State}. - code_change(_OldVsn, State, _Extra) -> {ok, State}. - handle_config_change("couchdb", "index_dir", RootDir, _, RootDir) -> {ok, RootDir}; handle_config_change("couchdb", "view_index_dir", RootDir, _, RootDir) -> @@ -228,19 +225,24 @@ new_index({Mod, IdxState, DbName, Sig}) -> case couch_index:start_link({Mod, IdxState}) of {ok, Pid} -> ok = gen_server:call( - ?MODULE, {async_open, {DbName, DDocId, Sig}, {ok, Pid}}), + ?MODULE, {async_open, {DbName, DDocId, Sig}, {ok, Pid}} + ), unlink(Pid); Error -> ok = gen_server:call( - ?MODULE, {async_error, {DbName, DDocId, Sig}, Error}) + ?MODULE, {async_error, {DbName, DDocId, Sig}, Error} + ) end. - reset_indexes(DbName, Root) -> % shutdown all the updaters and clear the files, the db got changed - SigDDocIds = lists:foldl(fun({_, {DDocId, Sig}}, DDict) -> - dict:append(Sig, DDocId, DDict) - end, dict:new(), ets:lookup(?BY_DB, DbName)), + SigDDocIds = lists:foldl( + fun({_, {DDocId, Sig}}, DDict) -> + dict:append(Sig, DDocId, DDict) + end, + dict:new(), + ets:lookup(?BY_DB, DbName) + ), Fun = fun({Sig, DDocIds}) -> [{_, Pid}] = ets:lookup(?BY_SIG, {DbName, Sig}), unlink(Pid), @@ -248,9 +250,8 @@ reset_indexes(DbName, Root) -> receive {'EXIT', Pid, _} -> ok - after - 0 -> - ok + after 0 -> + ok end, rem_from_ets(DbName, Sig, DDocIds, Pid) end, @@ -258,20 +259,20 @@ reset_indexes(DbName, Root) -> Path = couch_index_util:index_dir("", DbName), couch_file:nuke_dir(Root, Path). - add_to_ets(DbName, Sig, DDocId, Pid) -> ets:insert(?BY_SIG, {{DbName, Sig}, Pid}), ets:insert(?BY_PID, {Pid, {DbName, Sig}}), ets:insert(?BY_DB, {DbName, {DDocId, Sig}}). - rem_from_ets(DbName, Sig, DDocIds, Pid) -> ets:delete(?BY_SIG, {DbName, Sig}), ets:delete(?BY_PID, Pid), - lists:foreach(fun(DDocId) -> - ets:delete_object(?BY_DB, {DbName, {DDocId, Sig}}) - end, DDocIds). - + lists:foreach( + fun(DDocId) -> + ets:delete_object(?BY_DB, {DbName, {DDocId, Sig}}) + end, + DDocIds + ). handle_db_event(DbName, created, St) -> gen_server:cast(?MODULE, {reset_indexes, DbName}), @@ -279,44 +280,57 @@ handle_db_event(DbName, created, St) -> handle_db_event(DbName, deleted, St) -> gen_server:cast(?MODULE, {reset_indexes, DbName}), {ok, St}; -handle_db_event(<<"shards/", _/binary>> = DbName, {ddoc_updated, - DDocId}, St) -> +handle_db_event(<<"shards/", _/binary>> = DbName, {ddoc_updated, DDocId}, St) -> DDocResult = couch_util:with_db(DbName, fun(Db) -> couch_db:open_doc(Db, DDocId, [ejson_body, ?ADMIN_CTX]) end), - LocalShards = try mem3:local_shards(mem3:dbname(DbName)) - catch error:database_does_not_exist -> - [] - end, + LocalShards = + try + mem3:local_shards(mem3:dbname(DbName)) + catch + error:database_does_not_exist -> + [] + end, DbShards = [mem3:name(Sh) || Sh <- LocalShards], - lists:foreach(fun(DbShard) -> - lists:foreach(fun({_DbShard, {_DDocId, Sig}}) -> - % check if there are other ddocs with the same Sig for the same db - SigDDocs = ets:match_object(?BY_DB, {DbShard, {'$1', Sig}}), - if length(SigDDocs) > 1 -> - % remove records from ?BY_DB for this DDoc - Args = [DbShard, DDocId, Sig], - gen_server:cast(?MODULE, {rem_from_ets, Args}); - true -> - % single DDoc with this Sig - close couch_index processes - case ets:lookup(?BY_SIG, {DbShard, Sig}) of - [{_, IndexPid}] -> (catch - gen_server:cast(IndexPid, {ddoc_updated, DDocResult})); - [] -> [] - end - end - end, ets:match_object(?BY_DB, {DbShard, {DDocId, '$1'}})) - end, DbShards), + lists:foreach( + fun(DbShard) -> + lists:foreach( + fun({_DbShard, {_DDocId, Sig}}) -> + % check if there are other ddocs with the same Sig for the same db + SigDDocs = ets:match_object(?BY_DB, {DbShard, {'$1', Sig}}), + if + length(SigDDocs) > 1 -> + % remove records from ?BY_DB for this DDoc + Args = [DbShard, DDocId, Sig], + gen_server:cast(?MODULE, {rem_from_ets, Args}); + true -> + % single DDoc with this Sig - close couch_index processes + case ets:lookup(?BY_SIG, {DbShard, Sig}) of + [{_, IndexPid}] -> + (catch gen_server:cast(IndexPid, {ddoc_updated, DDocResult})); + [] -> + [] + end + end + end, + ets:match_object(?BY_DB, {DbShard, {DDocId, '$1'}}) + ) + end, + DbShards + ), {ok, St}; handle_db_event(DbName, {ddoc_updated, DDocId}, St) -> - lists:foreach(fun({_DbName, {_DDocId, Sig}}) -> - case ets:lookup(?BY_SIG, {DbName, Sig}) of - [{_, IndexPid}] -> - (catch gen_server:cast(IndexPid, ddoc_updated)); - [] -> - ok - end - end, ets:match_object(?BY_DB, {DbName, {DDocId, '$1'}})), + lists:foreach( + fun({_DbName, {_DDocId, Sig}}) -> + case ets:lookup(?BY_SIG, {DbName, Sig}) of + [{_, IndexPid}] -> + (catch gen_server:cast(IndexPid, ddoc_updated)); + [] -> + ok + end + end, + ets:match_object(?BY_DB, {DbName, {DDocId, '$1'}}) + ), {ok, St}; handle_db_event(_DbName, _Event, St) -> {ok, St}. diff --git a/src/couch_index/src/couch_index_sup.erl b/src/couch_index/src/couch_index_sup.erl index 2d4f671e2..eea4cc3ab 100644 --- a/src/couch_index/src/couch_index_sup.erl +++ b/src/couch_index/src/couch_index_sup.erl @@ -16,9 +16,8 @@ -export([start_link/1]). - start_link(Args) -> - supervisor:start_link({local,?MODULE}, ?MODULE, Args). + supervisor:start_link({local, ?MODULE}, ?MODULE, Args). init([]) -> {ok, {{one_for_one, 3, 10}, couch_epi:register_service(couch_index_epi, [])}}. diff --git a/src/couch_index/src/couch_index_updater.erl b/src/couch_index/src/couch_index_updater.erl index fb15db052..fe2150505 100644 --- a/src/couch_index/src/couch_index_updater.erl +++ b/src/couch_index/src/couch_index_updater.erl @@ -13,7 +13,6 @@ -module(couch_index_updater). -behaviour(gen_server). - %% API -export([start_link/2, run/2, is_running/1, update/2, restart/2]). @@ -29,48 +28,40 @@ -record(st, { idx, mod, - pid=nil + pid = nil }). - start_link(Index, Module) -> gen_server:start_link(?MODULE, {Index, Module}, []). - run(Pid, IdxState) -> gen_server:call(Pid, {update, IdxState}). - is_running(Pid) -> gen_server:call(Pid, is_running). - update(Mod, State) -> update(nil, Mod, State). - restart(Pid, IdxState) -> gen_server:call(Pid, {restart, IdxState}). - init({Index, Module}) -> process_flag(trap_exit, true), - {ok, #st{idx=Index, mod=Module}}. - + {ok, #st{idx = Index, mod = Module}}. terminate(_Reason, State) -> couch_util:shutdown_sync(State#st.pid), ok. - -handle_call({update, _IdxState}, _From, #st{pid=Pid}=State) when is_pid(Pid) -> +handle_call({update, _IdxState}, _From, #st{pid = Pid} = State) when is_pid(Pid) -> {reply, ok, State}; -handle_call({update, IdxState}, _From, #st{idx=Idx, mod=Mod}=State) -> +handle_call({update, IdxState}, _From, #st{idx = Idx, mod = Mod} = State) -> Args = [Mod:get(db_name, IdxState), Mod:get(idx_name, IdxState)], couch_log:info("Starting index update for db: ~s idx: ~s", Args), Pid = spawn_link(?MODULE, update, [Idx, Mod, IdxState]), - {reply, ok, State#st{pid=Pid}}; -handle_call({restart, IdxState}, _From, #st{idx=Idx, mod=Mod}=State) -> + {reply, ok, State#st{pid = Pid}}; +handle_call({restart, IdxState}, _From, #st{idx = Idx, mod = Mod} = State) -> Args = [Mod:get(db_name, IdxState), Mod:get(idx_name, IdxState)], couch_log:info("Restarting index update for db: ~s idx: ~s", Args), Pid = State#st.pid, @@ -86,46 +77,42 @@ handle_call({restart, IdxState}, _From, #st{idx=Idx, mod=Mod}=State) -> ok end, NewPid = spawn_link(?MODULE, update, [Idx, State#st.mod, IdxState]), - {reply, ok, State#st{pid=NewPid}}; -handle_call(is_running, _From, #st{pid=Pid}=State) when is_pid(Pid) -> + {reply, ok, State#st{pid = NewPid}}; +handle_call(is_running, _From, #st{pid = Pid} = State) when is_pid(Pid) -> {reply, true, State}; handle_call(is_running, _From, State) -> {reply, false, State}. - handle_cast(_Mesg, State) -> {stop, unknown_cast, State}. - -handle_info({'EXIT', _, {updated, Pid, IdxState}}, #st{pid=Pid}=State) -> +handle_info({'EXIT', _, {updated, Pid, IdxState}}, #st{pid = Pid} = State) -> Mod = State#st.mod, Args = [Mod:get(db_name, IdxState), Mod:get(idx_name, IdxState)], couch_log:info("Index update finished for db: ~s idx: ~s", Args), ok = gen_server:cast(State#st.idx, {updated, IdxState}), - {noreply, State#st{pid=undefined}}; -handle_info({'EXIT', _, {reset, Pid}}, #st{idx=Idx, pid=Pid}=State) -> + {noreply, State#st{pid = undefined}}; +handle_info({'EXIT', _, {reset, Pid}}, #st{idx = Idx, pid = Pid} = State) -> {ok, NewIdxState} = gen_server:call(State#st.idx, reset), Pid2 = spawn_link(?MODULE, update, [Idx, State#st.mod, NewIdxState]), - {noreply, State#st{pid=Pid2}}; -handle_info({'EXIT', Pid, normal}, #st{pid=Pid}=State) -> - {noreply, State#st{pid=undefined}}; + {noreply, State#st{pid = Pid2}}; +handle_info({'EXIT', Pid, normal}, #st{pid = Pid} = State) -> + {noreply, State#st{pid = undefined}}; handle_info({'EXIT', Pid, {{nocatch, Error}, _Trace}}, State) -> handle_info({'EXIT', Pid, Error}, State); -handle_info({'EXIT', Pid, Error}, #st{pid=Pid}=State) -> +handle_info({'EXIT', Pid, Error}, #st{pid = Pid} = State) -> ok = gen_server:cast(State#st.idx, {update_error, Error}), - {noreply, State#st{pid=undefined}}; -handle_info({'EXIT', Pid, _Reason}, #st{idx=Pid}=State) -> + {noreply, State#st{pid = undefined}}; +handle_info({'EXIT', Pid, _Reason}, #st{idx = Pid} = State) -> {stop, normal, State}; handle_info({'EXIT', _Pid, normal}, State) -> {noreply, State}; handle_info(_Mesg, State) -> {stop, unknown_info, State}. - code_change(_OldVsn, State, _Extra) -> {ok, State}. - update(Idx, Mod, IdxState) -> DbName = Mod:get(db_name, IdxState), IndexName = Mod:get(idx_name, IdxState), @@ -134,10 +121,11 @@ update(Idx, Mod, IdxState) -> UpdateOpts = Mod:get(update_options, IdxState), CommittedOnly = lists:member(committed_only, UpdateOpts), IncludeDesign = lists:member(include_design, UpdateOpts), - DocOpts = case lists:member(local_seq, UpdateOpts) of - true -> [conflicts, deleted_conflicts, local_seq]; - _ -> [conflicts, deleted_conflicts] - end, + DocOpts = + case lists:member(local_seq, UpdateOpts) of + true -> [conflicts, deleted_conflicts, local_seq]; + _ -> [conflicts, deleted_conflicts] + end, couch_util:with_db(DbName, fun(Db) -> DbUpdateSeq = couch_db:get_update_seq(Db), @@ -149,14 +137,14 @@ update(Idx, Mod, IdxState) -> {ok, PurgedIdxState} = purge_index(Db, Mod, IdxState), GetSeq = fun - (#full_doc_info{update_seq=Seq}) -> Seq; - (#doc_info{high_seq=Seq}) -> Seq + (#full_doc_info{update_seq = Seq}) -> Seq; + (#doc_info{high_seq = Seq}) -> Seq end, GetInfo = fun - (#full_doc_info{id=Id, update_seq=Seq, deleted=Del}=FDI) -> + (#full_doc_info{id = Id, update_seq = Seq, deleted = Del} = FDI) -> {Id, Seq, Del, couch_doc:to_doc_info(FDI)}; - (#doc_info{id=Id, high_seq=Seq, revs=[RI|_]}=DI) -> + (#doc_info{id = Id, high_seq = Seq, revs = [RI | _]} = DI) -> {Id, Seq, RI#rev_info.deleted, DI} end, @@ -167,7 +155,7 @@ update(Idx, Mod, IdxState) -> {false, <<"_design/", _/binary>>} -> {nil, Seq}; _ when Deleted -> - {#doc{id=DocId, deleted=true}, Seq}; + {#doc{id = DocId, deleted = true}, Seq}; _ -> {ok, Doc} = couch_db:open_doc_int(Db, DocInfo, DocOpts), {Doc, Seq} @@ -186,11 +174,11 @@ update(Idx, Mod, IdxState) -> end end, {ok, InitIdxState} = Mod:start_update( - Idx, - PurgedIdxState, - TotalChanges, - NumPurgeChanges - ), + Idx, + PurgedIdxState, + TotalChanges, + NumPurgeChanges + ), Acc0 = {InitIdxState, true}, {ok, Acc} = couch_db:fold_changes(Db, CurrSeq, Proc, Acc0, []), @@ -198,41 +186,45 @@ update(Idx, Mod, IdxState) -> % If we didn't bail due to hitting the last committed seq we need % to send our last update_seq through. - {ok, LastIdxSt} = case SendLast of - true -> - Mod:process_doc(nil, DbUpdateSeq, ProcIdxSt); - _ -> - {ok, ProcIdxSt} - end, + {ok, LastIdxSt} = + case SendLast of + true -> + Mod:process_doc(nil, DbUpdateSeq, ProcIdxSt); + _ -> + {ok, ProcIdxSt} + end, {ok, FinalIdxState} = Mod:finish_update(LastIdxSt), exit({updated, self(), FinalIdxState}) end). - purge_index(Db, Mod, IdxState) -> DbPurgeSeq = couch_db:get_purge_seq(Db), IdxPurgeSeq = Mod:get(purge_seq, IdxState), - if IdxPurgeSeq == DbPurgeSeq -> {ok, IdxState}; true -> - FoldFun = fun({PurgeSeq, _UUId, Id, Revs}, Acc) -> - Mod:purge(Db, PurgeSeq, [{Id, Revs}], Acc) - end, - {ok, NewStateAcc} = try - couch_db:fold_purge_infos( - Db, - IdxPurgeSeq, - FoldFun, - IdxState, - [] - ) - catch error:{invalid_start_purge_seq, _} -> - exit({reset, self()}) - end, - Mod:update_local_purge_doc(Db, NewStateAcc), - {ok, NewStateAcc} + if + IdxPurgeSeq == DbPurgeSeq -> + {ok, IdxState}; + true -> + FoldFun = fun({PurgeSeq, _UUId, Id, Revs}, Acc) -> + Mod:purge(Db, PurgeSeq, [{Id, Revs}], Acc) + end, + {ok, NewStateAcc} = + try + couch_db:fold_purge_infos( + Db, + IdxPurgeSeq, + FoldFun, + IdxState, + [] + ) + catch + error:{invalid_start_purge_seq, _} -> + exit({reset, self()}) + end, + Mod:update_local_purge_doc(Db, NewStateAcc), + {ok, NewStateAcc} end. - count_pending_purged_docs_since(Db, Mod, IdxState) -> DbPurgeSeq = couch_db:get_purge_seq(Db), IdxPurgeSeq = Mod:get(purge_seq, IdxState), diff --git a/src/couch_index/src/couch_index_util.erl b/src/couch_index/src/couch_index_util.erl index dcb33b5b0..3a7d283bf 100644 --- a/src/couch_index/src/couch_index_util.erl +++ b/src/couch_index/src/couch_index_util.erl @@ -17,10 +17,8 @@ -include_lib("couch/include/couch_db.hrl"). - root_dir() -> - config:get("couchdb", "view_index_dir"). - + config:get("couchdb", "view_index_dir"). index_dir(Module, DbName) when is_binary(DbName) -> DbDir = "." ++ binary_to_list(DbName) ++ "_design", @@ -28,33 +26,32 @@ index_dir(Module, DbName) when is_binary(DbName) -> index_dir(Module, Db) -> index_dir(Module, couch_db:name(Db)). - index_file(Module, DbName, FileName) -> filename:join(index_dir(Module, DbName), FileName). - -load_doc(Db, #doc_info{}=DI, Opts) -> +load_doc(Db, #doc_info{} = DI, Opts) -> Deleted = lists:member(deleted, Opts), case (catch couch_db:open_doc(Db, DI, Opts)) of - {ok, #doc{deleted=false}=Doc} -> Doc; - {ok, #doc{deleted=true}=Doc} when Deleted -> Doc; + {ok, #doc{deleted = false} = Doc} -> Doc; + {ok, #doc{deleted = true} = Doc} when Deleted -> Doc; _Else -> null end; load_doc(Db, {DocId, Rev}, Opts) -> case (catch load_doc(Db, DocId, Rev, Opts)) of - #doc{deleted=false} = Doc -> Doc; + #doc{deleted = false} = Doc -> Doc; _ -> null end. - load_doc(Db, DocId, Rev, Options) -> case Rev of - nil -> % open most recent rev + % open most recent rev + nil -> case (catch couch_db:open_doc(Db, DocId, Options)) of {ok, Doc} -> Doc; _Error -> null end; - _ -> % open a specific rev (deletions come back as stubs) + % open a specific rev (deletions come back as stubs) + _ -> case (catch couch_db:open_doc_revs(Db, DocId, [Rev], Options)) of {ok, [{ok, Doc}]} -> Doc; {ok, [{{not_found, missing}, Rev}]} -> null; @@ -62,17 +59,16 @@ load_doc(Db, DocId, Rev, Options) -> end end. - sort_lib({Lib}) -> sort_lib(Lib, []). sort_lib([], LAcc) -> lists:keysort(1, LAcc); -sort_lib([{LName, {LObj}}|Rest], LAcc) -> - LSorted = sort_lib(LObj, []), % descend into nested object - sort_lib(Rest, [{LName, LSorted}|LAcc]); -sort_lib([{LName, LCode}|Rest], LAcc) -> - sort_lib(Rest, [{LName, LCode}|LAcc]). - +sort_lib([{LName, {LObj}} | Rest], LAcc) -> + % descend into nested object + LSorted = sort_lib(LObj, []), + sort_lib(Rest, [{LName, LSorted} | LAcc]); +sort_lib([{LName, LCode} | Rest], LAcc) -> + sort_lib(Rest, [{LName, LCode} | LAcc]). hexsig(Sig) -> couch_util:to_hex(binary_to_list(Sig)). diff --git a/src/couch_index/test/eunit/couch_index_compaction_tests.erl b/src/couch_index/test/eunit/couch_index_compaction_tests.erl index ab493a969..fee04fd9c 100644 --- a/src/couch_index/test/eunit/couch_index_compaction_tests.erl +++ b/src/couch_index/test/eunit/couch_index_compaction_tests.erl @@ -17,7 +17,6 @@ -define(WAIT_TIMEOUT, 1000). - setup_all() -> Ctx = test_util:start_couch(), meck:new([test_index], [non_strict]), @@ -41,8 +40,13 @@ fake_index(DbName) -> ok = meck:expect(test_index, open, fun(_Db, State) -> {ok, State} end), - ok = meck:expect(test_index, compact, ['_', '_', '_'], - meck:seq([{ok, 9}, {ok, 10}])), %% to trigger recompaction + ok = meck:expect( + test_index, + compact, + ['_', '_', '_'], + %% to trigger recompaction + meck:seq([{ok, 9}, {ok, 10}]) + ), ok = meck:expect(test_index, commit, ['_'], ok), ok = meck:expect(test_index, get, fun (db_name, _) -> @@ -50,7 +54,7 @@ fake_index(DbName) -> (idx_name, _) -> <<"idx_name">>; (signature, _) -> - <<61,237,157,230,136,93,96,201,204,17,137,186,50,249,44,135>>; + <<61, 237, 157, 230, 136, 93, 96, 201, 204, 17, 137, 186, 50, 249, 44, 135>>; (update_seq, Seq) -> Seq end), @@ -80,7 +84,6 @@ compaction_test_() -> } }. - hold_db_for_recompaction({Db, Idx}) -> ?_test(begin ?assertNot(is_opened(Db)), @@ -105,12 +108,15 @@ hold_db_for_recompaction({Db, Idx}) -> end). wait_db_close(Db) -> - test_util:wait(fun() -> - case is_opened(Db) of - false -> ok; - true -> wait - end - end, ?WAIT_TIMEOUT). + test_util:wait( + fun() -> + case is_opened(Db) of + false -> ok; + true -> wait + end + end, + ?WAIT_TIMEOUT + ). is_opened(Db) -> Monitors = [M || M <- couch_db:monitored_by(Db), M =/= self()], diff --git a/src/couch_index/test/eunit/couch_index_ddoc_updated_tests.erl b/src/couch_index/test/eunit/couch_index_ddoc_updated_tests.erl index 0e23adf91..7bee8baae 100644 --- a/src/couch_index/test/eunit/couch_index_ddoc_updated_tests.erl +++ b/src/couch_index/test/eunit/couch_index_ddoc_updated_tests.erl @@ -15,7 +15,6 @@ -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). - start() -> fake_index(), Ctx = test_util:start_couch([mem3, fabric]), @@ -23,56 +22,68 @@ start() -> ok = fabric:create_db(DbName, [?ADMIN_CTX]), {Ctx, DbName}. - stop({Ctx, DbName}) -> meck:unload(test_index), ok = fabric:delete_db(DbName, [?ADMIN_CTX]), DbDir = config:get("couchdb", "database_dir", "."), WaitFun = fun() -> - filelib:fold_files(DbDir, <<".*", DbName/binary, "\.[0-9]+.*">>, - true, fun(_F, _A) -> wait end, ok) + filelib:fold_files( + DbDir, + <<".*", DbName/binary, "\.[0-9]+.*">>, + true, + fun(_F, _A) -> wait end, + ok + ) end, ok = test_util:wait(WaitFun), test_util:stop_couch(Ctx), ok. - ddoc_update_test_() -> { "Check ddoc update actions", { setup, - fun start/0, fun stop/1, + fun start/0, + fun stop/1, fun check_all_indexers_exit_on_ddoc_change/1 } }. - check_all_indexers_exit_on_ddoc_change({_Ctx, DbName}) -> ?_test(begin - [DbShard1 | RestDbShards] = lists:map(fun(Sh) -> - {ok, ShardDb} = couch_db:open(mem3:name(Sh), []), - ShardDb - end, mem3:local_shards(mem3:dbname(DbName))), + [DbShard1 | RestDbShards] = lists:map( + fun(Sh) -> + {ok, ShardDb} = couch_db:open(mem3:name(Sh), []), + ShardDb + end, + mem3:local_shards(mem3:dbname(DbName)) + ), % create a DDoc on Db1 DDocID = <<"idx_name">>, - DDocJson = couch_doc:from_json_obj({[ - {<<"_id">>, DDocID}, - {<<"value">>, 1} - ]}), + DDocJson = couch_doc:from_json_obj( + {[ + {<<"_id">>, DDocID}, + {<<"value">>, 1} + ]} + ), {ok, _Rev} = couch_db:update_doc(DbShard1, DDocJson, []), {ok, DbShard} = couch_db:reopen(DbShard1), {ok, DDoc} = couch_db:open_doc( - DbShard, DDocID, [ejson_body, ?ADMIN_CTX]), + DbShard, DDocID, [ejson_body, ?ADMIN_CTX] + ), DbShards = [DbShard | RestDbShards], N = length(DbShards), % run couch_index process for each shard database ok = meck:reset(test_index), - lists:foreach(fun(ShardDb) -> - couch_index_server:get_index(test_index, ShardDb, DDoc) - end, DbShards), + lists:foreach( + fun(ShardDb) -> + couch_index_server:get_index(test_index, ShardDb, DDoc) + end, + DbShards + ), IndexesBefore = get_indexes_by_ddoc(DDocID, N), ?assertEqual(N, length(IndexesBefore)), @@ -81,17 +92,20 @@ check_all_indexers_exit_on_ddoc_change({_Ctx, DbName}) -> ?assertEqual(N, length(AliveBefore)), % update ddoc - DDocJson2 = couch_doc:from_json_obj({[ - {<<"_id">>, DDocID}, - {<<"value">>, 2}, - {<<"_rev">>, couch_doc:rev_to_str(DDoc#doc.revs)} - ]}), + DDocJson2 = couch_doc:from_json_obj( + {[ + {<<"_id">>, DDocID}, + {<<"value">>, 2}, + {<<"_rev">>, couch_doc:rev_to_str(DDoc#doc.revs)} + ]} + ), {ok, _} = couch_db:update_doc(DbShard, DDocJson2, []), % assert that all index processes exit after ddoc updated ok = meck:reset(test_index), couch_index_server:handle_db_event( - couch_db:name(DbShard), {ddoc_updated, DDocID}, {st, ""}), + couch_db:name(DbShard), {ddoc_updated, DDocID}, {st, ""} + ), ok = meck:wait(N, test_index, init, ['_', '_'], 5000), IndexesAfter = get_indexes_by_ddoc(DDocID, 0), @@ -103,7 +117,6 @@ check_all_indexers_exit_on_ddoc_change({_Ctx, DbName}) -> ok end). - fake_index() -> ok = meck:new([test_index], [non_strict]), ok = meck:expect(test_index, init, fun(Db, DDoc) -> @@ -124,11 +137,11 @@ fake_index() -> end), ok = meck:expect(test_index, shutdown, ['_'], ok). - get_indexes_by_ddoc(DDocID, N) -> Indexes = test_util:wait(fun() -> Indxs = ets:match_object( - couchdb_indexes_by_db, {'$1', {DDocID, '$2'}}), + couchdb_indexes_by_db, {'$1', {DDocID, '$2'}} + ), case length(Indxs) == N of true -> Indxs; @@ -136,10 +149,13 @@ get_indexes_by_ddoc(DDocID, N) -> wait end end), - lists:foldl(fun({DbName, {_DDocID, Sig}}, Acc) -> - case ets:lookup(couchdb_indexes_by_sig, {DbName, Sig}) of - [{_, Pid}] -> [Pid|Acc]; - _ -> Acc - end - end, [], Indexes). - + lists:foldl( + fun({DbName, {_DDocID, Sig}}, Acc) -> + case ets:lookup(couchdb_indexes_by_sig, {DbName, Sig}) of + [{_, Pid}] -> [Pid | Acc]; + _ -> Acc + end + end, + [], + Indexes + ). diff --git a/src/couch_log/src/couch_log.erl b/src/couch_log/src/couch_log.erl index a8dc5d48d..b8a1ca4bd 100644 --- a/src/couch_log/src/couch_log.erl +++ b/src/couch_log/src/couch_log.erl @@ -12,7 +12,6 @@ -module(couch_log). - -export([ debug/2, info/2, @@ -26,44 +25,34 @@ set_level/1 ]). - -spec debug(string(), list()) -> ok. debug(Fmt, Args) -> log(debug, Fmt, Args). - -spec info(string(), list()) -> ok. info(Fmt, Args) -> log(info, Fmt, Args). - -spec notice(string(), list()) -> ok. notice(Fmt, Args) -> log(notice, Fmt, Args). - -spec warning(string(), list()) -> ok. warning(Fmt, Args) -> log(warning, Fmt, Args). - -spec error(string(), list()) -> ok. error(Fmt, Args) -> log(error, Fmt, Args). - -spec critical(string(), list()) -> ok. critical(Fmt, Args) -> log(critical, Fmt, Args). - -spec alert(string(), list()) -> ok. alert(Fmt, Args) -> log(alert, Fmt, Args). - -spec emergency(string(), list()) -> ok. emergency(Fmt, Args) -> log(emergency, Fmt, Args). - -spec set_level(atom() | string() | integer()) -> true. set_level(Level) -> config:set("log", "level", couch_log_util:level_to_string(Level)). - -spec log(atom(), string(), list()) -> ok. log(Level, Fmt, Args) -> case couch_log_util:should_log(Level) of diff --git a/src/couch_log/src/couch_log_app.erl b/src/couch_log/src/couch_log_app.erl index 91a8ecc4d..28c8bb193 100644 --- a/src/couch_log/src/couch_log_app.erl +++ b/src/couch_log/src/couch_log_app.erl @@ -16,7 +16,6 @@ -export([start/2, stop/1]). - start(_Type, _StartArgs) -> couch_log_sup:start_link(). diff --git a/src/couch_log/src/couch_log_config.erl b/src/couch_log/src/couch_log_config.erl index 55925c39f..925973178 100644 --- a/src/couch_log/src/couch_log_config.erl +++ b/src/couch_log/src/couch_log_config.erl @@ -14,23 +14,19 @@ -module(couch_log_config). - -export([ init/0, reconfigure/0, get/1 ]). - -define(MOD_NAME, couch_log_config_dyn). -define(ERL_FILE, "couch_log_config_dyn.erl"). - -spec init() -> ok. init() -> reconfigure(). - -spec reconfigure() -> ok. reconfigure() -> {ok, ?MOD_NAME, Bin} = compile:forms(forms(), [verbose, report_errors]), @@ -38,12 +34,10 @@ reconfigure() -> {module, ?MOD_NAME} = code:load_binary(?MOD_NAME, ?ERL_FILE, Bin), ok. - -spec get(atom()) -> term(). get(Key) -> ?MOD_NAME:get(Key). - -spec entries() -> [string()]. entries() -> [ @@ -52,17 +46,19 @@ entries() -> {max_message_size, "max_message_size", "16000"}, {strip_last_msg, "strip_last_msg", "true"}, {filter_fields, "filter_fields", "[pid, registered_name, error_info, messages]"} - ]. - + ]. -spec forms() -> [erl_syntax:syntaxTree()]. forms() -> - GetFunClauses = lists:map(fun({FunKey, CfgKey, Default}) -> - FunVal = transform(FunKey, config:get("log", CfgKey, Default)), - Patterns = [erl_syntax:abstract(FunKey)], - Bodies = [erl_syntax:abstract(FunVal)], - erl_syntax:clause(Patterns, none, Bodies) - end, entries()), + GetFunClauses = lists:map( + fun({FunKey, CfgKey, Default}) -> + FunVal = transform(FunKey, config:get("log", CfgKey, Default)), + Patterns = [erl_syntax:abstract(FunKey)], + Bodies = [erl_syntax:abstract(FunVal)], + erl_syntax:clause(Patterns, none, Bodies) + end, + entries() + ), Statements = [ % -module(?MOD_NAME) @@ -74,11 +70,14 @@ forms() -> % -export([lookup/1]). erl_syntax:attribute( erl_syntax:atom(export), - [erl_syntax:list([ - erl_syntax:arity_qualifier( - erl_syntax:atom(get), - erl_syntax:integer(1)) - ])] + [ + erl_syntax:list([ + erl_syntax:arity_qualifier( + erl_syntax:atom(get), + erl_syntax:integer(1) + ) + ]) + ] ), % list(Key) -> Value. @@ -86,27 +85,22 @@ forms() -> ], [erl_syntax:revert(X) || X <- Statements]. - transform(level, LevelStr) -> couch_log_util:level_to_atom(LevelStr); - transform(level_int, LevelStr) -> Level = couch_log_util:level_to_atom(LevelStr), couch_log_util:level_to_integer(Level); - transform(max_message_size, SizeStr) -> try list_to_integer(SizeStr) of Size -> Size - catch _:_ -> - 16000 + catch + _:_ -> + 16000 end; - transform(strip_last_msg, "false") -> false; - transform(strip_last_msg, _) -> true; - transform(filter_fields, FieldsStr) -> Default = [pid, registered_name, error_info, messages], case parse_term(FieldsStr) of @@ -121,7 +115,6 @@ transform(filter_fields, FieldsStr) -> Default end. - parse_term(List) -> {ok, Tokens, _} = erl_scan:string(List ++ "."), erl_parse:parse_term(Tokens). diff --git a/src/couch_log/src/couch_log_config_dyn.erl b/src/couch_log/src/couch_log_config_dyn.erl index 1e1c927ae..ff781d3a0 100644 --- a/src/couch_log/src/couch_log_config_dyn.erl +++ b/src/couch_log/src/couch_log_config_dyn.erl @@ -17,12 +17,10 @@ -module(couch_log_config_dyn). - -export([ get/1 ]). - get(level) -> info; get(level_int) -> 2; get(max_message_size) -> 16000; diff --git a/src/couch_log/src/couch_log_error_logger_h.erl b/src/couch_log/src/couch_log_error_logger_h.erl index c0765c61a..ff7ae045f 100644 --- a/src/couch_log/src/couch_log_error_logger_h.erl +++ b/src/couch_log/src/couch_log_error_logger_h.erl @@ -14,10 +14,8 @@ % https://github.com/basho/lager which is available under the % above marked ASFL v2 license. - -module(couch_log_error_logger_h). - -behaviour(gen_event). -export([ @@ -29,29 +27,22 @@ code_change/3 ]). - init(_) -> {ok, undefined}. - terminate(_Reason, _St) -> ok. - handle_call(_, St) -> {ok, ignored, St}. - handle_event(Event, St) -> Entry = couch_log_formatter:format(Event), ok = couch_log_server:log(Entry), {ok, St}. - handle_info(_, St) -> {ok, St}. - code_change(_OldVsn, State, _Extra) -> {ok, State}. - diff --git a/src/couch_log/src/couch_log_formatter.erl b/src/couch_log/src/couch_log_formatter.erl index 3553666f6..2ce0fba6d 100644 --- a/src/couch_log/src/couch_log_formatter.erl +++ b/src/couch_log/src/couch_log_formatter.erl @@ -14,10 +14,8 @@ % from lager's error_logger_lager_h.erl which is available % under the ASFv2 license. - -module(couch_log_formatter). - -export([ format/4, format/3, @@ -29,13 +27,10 @@ format_args/3 ]). - -include("couch_log.hrl"). - -define(DEFAULT_TRUNCATION, 1024). - format(Level, Pid, Fmt, Args) -> #log_entry{ level = couch_log_util:level_to_atom(Level), @@ -45,7 +40,6 @@ format(Level, Pid, Fmt, Args) -> time_stamp = couch_log_util:iso8601_timestamp() }. - format(Level, Pid, Msg) -> #log_entry{ level = couch_log_util:level_to_atom(Level), @@ -55,79 +49,82 @@ format(Level, Pid, Msg) -> time_stamp = couch_log_util:iso8601_timestamp() }. - format(Event) -> try do_format(Event) catch Tag:Err -> Msg = "Encountered error ~w when formatting ~w", - format(error, self(), Msg, [{Tag, Err}, Event]) + format(error, self(), Msg, [{Tag, Err}, Event]) end. - do_format({error, _GL, {Pid, "** Generic server " ++ _, Args}}) -> %% gen_server terminate [Name, LastMsg0, State, Reason | Extra] = Args, - LastMsg = case couch_log_config:get(strip_last_msg) of - true -> - redacted; - false -> - LastMsg0 - end, - MsgFmt = "gen_server ~w terminated with reason: ~s~n" ++ - " last msg: ~p~n state: ~p~n extra: ~p", + LastMsg = + case couch_log_config:get(strip_last_msg) of + true -> + redacted; + false -> + LastMsg0 + end, + MsgFmt = + "gen_server ~w terminated with reason: ~s~n" ++ + " last msg: ~p~n state: ~p~n extra: ~p", MsgArgs = [Name, format_reason(Reason), LastMsg, State, Extra], format(error, Pid, MsgFmt, MsgArgs); - do_format({error, _GL, {Pid, "** State machine " ++ _, Args}}) -> %% gen_fsm terminate [Name, LastMsg0, StateName, State, Reason | Extra] = Args, - LastMsg = case couch_log_config:get(strip_last_msg) of - true -> - redacted; - false -> - LastMsg0 - end, - MsgFmt = "gen_fsm ~w in state ~w terminated with reason: ~s~n" ++ - " last msg: ~p~n state: ~p~n extra: ~p", + LastMsg = + case couch_log_config:get(strip_last_msg) of + true -> + redacted; + false -> + LastMsg0 + end, + MsgFmt = + "gen_fsm ~w in state ~w terminated with reason: ~s~n" ++ + " last msg: ~p~n state: ~p~n extra: ~p", MsgArgs = [Name, StateName, format_reason(Reason), LastMsg, State, Extra], format(error, Pid, MsgFmt, MsgArgs); - do_format({error, _GL, {Pid, "** gen_event handler" ++ _, Args}}) -> %% gen_event handler terminate [ID, Name, LastMsg0, State, Reason] = Args, - LastMsg = case couch_log_config:get(strip_last_msg) of - true -> - redacted; - false -> - LastMsg0 - end, - MsgFmt = "gen_event ~w installed in ~w terminated with reason: ~s~n" ++ - " last msg: ~p~n state: ~p", + LastMsg = + case couch_log_config:get(strip_last_msg) of + true -> + redacted; + false -> + LastMsg0 + end, + MsgFmt = + "gen_event ~w installed in ~w terminated with reason: ~s~n" ++ + " last msg: ~p~n state: ~p", MsgArgs = [ID, Name, format_reason(Reason), LastMsg, State], format(error, Pid, MsgFmt, MsgArgs); - do_format({error, _GL, {emulator, "~s~n", [Msg]}}) when is_list(Msg) -> % These messages are for whenever any process exits due % to a throw or error. We intercept here to remove the % extra newlines. NewMsg = lists:sublist(Msg, length(Msg) - 1), format(error, emulator, NewMsg); - do_format({error, _GL, {Pid, Fmt, Args}}) -> format(error, Pid, Fmt, Args); - do_format({error_report, _GL, {Pid, std_error, D}}) -> format(error, Pid, print_silly_list(D)); - do_format({error_report, _GL, {Pid, supervisor_report, D}}) -> case lists:sort(D) of - [{errorContext, Ctx}, {offender, Off}, - {reason, Reason}, {supervisor, Name}] -> + [ + {errorContext, Ctx}, + {offender, Off}, + {reason, Reason}, + {supervisor, Name} + ] -> Offender = format_offender(Off), - MsgFmt = "Supervisor ~w had child ~s exit " ++ - "with reason ~s in context ~w", + MsgFmt = + "Supervisor ~w had child ~s exit " ++ + "with reason ~s in context ~w", Args = [ supervisor_name(Name), Offender, @@ -138,20 +135,15 @@ do_format({error_report, _GL, {Pid, supervisor_report, D}}) -> _ -> format(error, Pid, "SUPERVISOR REPORT " ++ print_silly_list(D)) end; - do_format({error_report, _GL, {Pid, crash_report, [Report, Neighbors]}}) -> Msg = "CRASH REPORT " ++ format_crash_report(Report, Neighbors), format(error, Pid, Msg); - do_format({warning_msg, _GL, {Pid, Fmt, Args}}) -> format(warning, Pid, Fmt, Args); - do_format({warning_report, _GL, {Pid, std_warning, Report}}) -> format(warning, Pid, print_silly_list(Report)); - do_format({info_msg, _GL, {Pid, Fmt, Args}}) -> format(info, Pid, Fmt, Args); - do_format({info_report, _GL, {Pid, std_info, D}}) when is_list(D) -> case lists:sort(D) of [{application, App}, {exited, Reason}, {type, _Type}] -> @@ -160,10 +152,8 @@ do_format({info_report, _GL, {Pid, std_info, D}}) when is_list(D) -> _ -> format(info, Pid, print_silly_list(D)) end; - do_format({info_report, _GL, {Pid, std_info, D}}) -> format(info, Pid, "~w", [D]); - do_format({info_report, _GL, {Pid, progress, D}}) -> case lists:sort(D) of [{application, App}, {started_at, Node}] -> @@ -177,25 +167,25 @@ do_format({info_report, _GL, {Pid, progress, D}}) -> _ -> format(info, Pid, "PROGRESS REPORT " ++ print_silly_list(D)) end; - do_format(Event) -> format(warning, self(), "Unexpected error_logger event ~w", [Event]). - format_crash_report(Report, Neighbours) -> Pid = get_value(pid, Report), - Name = case get_value(registered_name, Report) of - undefined -> - pid_to_list(Pid); - Atom -> - io_lib:format("~s (~w)", [Atom, Pid]) - end, + Name = + case get_value(registered_name, Report) of + undefined -> + pid_to_list(Pid); + Atom -> + io_lib:format("~s (~w)", [Atom, Pid]) + end, {Class, Reason, Trace} = get_value(error_info, Report), ReasonStr = format_reason({Reason, Trace}), - Type = case Class of - exit -> "exited"; - _ -> "crashed" - end, + Type = + case Class of + exit -> "exited"; + _ -> "crashed" + end, MsgFmt = "Process ~s with ~w neighbors ~s with reason: ~s", Args = [Name, length(Neighbours), Type, ReasonStr], Msg = io_lib:format(MsgFmt, Args), @@ -206,7 +196,6 @@ format_crash_report(Report, Neighbours) -> Msg ++ "; " ++ print_silly_list(Rest) end. - format_offender(Off) -> case get_value(mfargs, Off) of undefined -> @@ -219,129 +208,131 @@ format_offender(Off) -> %% In 2014 the error report changed from `name' to %% `id', so try that first. - Name = case get_value(id, Off) of - undefined -> - get_value(name, Off); - Id -> - Id - end, + Name = + case get_value(id, Off) of + undefined -> + get_value(name, Off); + Id -> + Id + end, Args = [Name, MFA, get_value(pid, Off)], io_lib:format("~p started with ~s at ~w", Args) end. - format_reason({'function not exported', [{M, F, A} | Trace]}) -> - ["call to unexported function ", format_mfa({M, F, A}), - " at ", format_trace(Trace)]; - + [ + "call to unexported function ", + format_mfa({M, F, A}), + " at ", + format_trace(Trace) + ]; format_reason({'function not exported' = C, [{M, F, A, _Props} | Rest]}) -> %% Drop line number from undefined function format_reason({C, [{M, F, A} | Rest]}); - format_reason({undef, [MFA | Trace]}) -> - ["call to undefined function ", format_mfa(MFA), - " at ", format_trace(Trace)]; - + [ + "call to undefined function ", + format_mfa(MFA), + " at ", + format_trace(Trace) + ]; format_reason({bad_return, {MFA, Val}}) -> ["bad return value ", print_val(Val), " from ", format_mfa(MFA)]; - format_reason({bad_return_value, Val}) -> ["bad return value ", print_val(Val)]; - format_reason({{bad_return_value, Val}, MFA}) -> ["bad return value ", print_val(Val), " at ", format_mfa(MFA)]; - format_reason({{badrecord, Record}, Trace}) -> ["bad record ", print_val(Record), " at ", format_trace(Trace)]; - format_reason({{case_clause, Val}, Trace}) -> ["no case clause matching ", print_val(Val), " at ", format_trace(Trace)]; - format_reason({function_clause, [MFA | Trace]}) -> - ["no function clause matching ", format_mfa(MFA), - " at ", format_trace(Trace)]; - + [ + "no function clause matching ", + format_mfa(MFA), + " at ", + format_trace(Trace) + ]; format_reason({if_clause, Trace}) -> - ["no true branch found while evaluating if expression at ", - format_trace(Trace)]; - + [ + "no true branch found while evaluating if expression at ", + format_trace(Trace) + ]; format_reason({{try_clause, Val}, Trace}) -> ["no try clause matching ", print_val(Val), " at ", format_trace(Trace)]; - format_reason({badarith, Trace}) -> ["bad arithmetic expression at ", format_trace(Trace)]; - format_reason({{badmatch, Val}, Trace}) -> - ["no match of right hand value ", print_val(Val), - " at ", format_trace(Trace)]; - + [ + "no match of right hand value ", + print_val(Val), + " at ", + format_trace(Trace) + ]; format_reason({emfile, Trace}) -> - ["maximum number of file descriptors exhausted, check ulimit -n; ", - format_trace(Trace)]; - + [ + "maximum number of file descriptors exhausted, check ulimit -n; ", + format_trace(Trace) + ]; format_reason({system_limit, [{M, F, A} | Trace]}) -> - Limit = case {M, F} of - {erlang, open_port} -> - "maximum number of ports exceeded"; - {erlang, spawn} -> - "maximum number of processes exceeded"; - {erlang, spawn_opt} -> - "maximum number of processes exceeded"; - {erlang, list_to_atom} -> - "tried to create an atom larger than 255, or maximum atom count exceeded"; - {ets, new} -> - "maximum number of ETS tables exceeded"; - _ -> - format_mfa({M, F, A}) - end, + Limit = + case {M, F} of + {erlang, open_port} -> + "maximum number of ports exceeded"; + {erlang, spawn} -> + "maximum number of processes exceeded"; + {erlang, spawn_opt} -> + "maximum number of processes exceeded"; + {erlang, list_to_atom} -> + "tried to create an atom larger than 255, or maximum atom count exceeded"; + {ets, new} -> + "maximum number of ETS tables exceeded"; + _ -> + format_mfa({M, F, A}) + end, ["system limit: ", Limit, " at ", format_trace(Trace)]; - format_reason({badarg, [MFA | Trace]}) -> - ["bad argument in call to ", format_mfa(MFA), - " at ", format_trace(Trace)]; - + [ + "bad argument in call to ", + format_mfa(MFA), + " at ", + format_trace(Trace) + ]; format_reason({{badarg, Stack}, _}) -> format_reason({badarg, Stack}); - format_reason({{badarity, {Fun, Args}}, Trace}) -> {arity, Arity} = lists:keyfind(arity, 1, erlang:fun_info(Fun)), MsgFmt = "function called with wrong arity of ~w instead of ~w at ", [io_lib:format(MsgFmt, [length(Args), Arity]), format_trace(Trace)]; - format_reason({noproc, MFA}) -> ["no such process or port in call to ", format_mfa(MFA)]; - format_reason({{badfun, Term}, Trace}) -> ["bad function ", print_val(Term), " called at ", format_trace(Trace)]; - -format_reason({Reason, [{M, F, A} | _] = Trace}) - when is_atom(M), is_atom(F), is_integer(A) -> +format_reason({Reason, [{M, F, A} | _] = Trace}) when + is_atom(M), is_atom(F), is_integer(A) +-> [format_reason(Reason), " at ", format_trace(Trace)]; - -format_reason({Reason, [{M, F, A} | _] = Trace}) - when is_atom(M), is_atom(F), is_list(A) -> +format_reason({Reason, [{M, F, A} | _] = Trace}) when + is_atom(M), is_atom(F), is_list(A) +-> [format_reason(Reason), " at ", format_trace(Trace)]; - -format_reason({Reason, [{M, F, A, Props} | _] = Trace}) - when is_atom(M), is_atom(F), is_integer(A), is_list(Props) -> +format_reason({Reason, [{M, F, A, Props} | _] = Trace}) when + is_atom(M), is_atom(F), is_integer(A), is_list(Props) +-> [format_reason(Reason), " at ", format_trace(Trace)]; - -format_reason({Reason, [{M, F, A, Props} | _] = Trace}) - when is_atom(M), is_atom(F), is_list(A), is_list(Props) -> +format_reason({Reason, [{M, F, A, Props} | _] = Trace}) when + is_atom(M), is_atom(F), is_list(A), is_list(Props) +-> [format_reason(Reason), " at ", format_trace(Trace)]; - format_reason(Reason) -> {Str, _} = couch_log_trunc_io:print(Reason, 500), Str. - format_mfa({M, F, A}) when is_list(A) -> {FmtStr, Args} = format_args(A, [], []), io_lib:format("~w:~w(" ++ FmtStr ++ ")", [M, F | Args]); - format_mfa({M, F, A}) when is_integer(A) -> io_lib:format("~w:~w/~w", [M, F, A]); - format_mfa({M, F, A, Props}) when is_list(Props) -> case get_value(line, Props) of undefined -> @@ -349,47 +340,35 @@ format_mfa({M, F, A, Props}) when is_list(Props) -> Line -> [format_mfa({M, F, A}), io_lib:format("(line:~w)", [Line])] end; - format_mfa(Trace) when is_list(Trace) -> format_trace(Trace); - format_mfa(Other) -> io_lib:format("~w", [Other]). - format_trace([MFA]) -> [trace_mfa(MFA)]; - format_trace([MFA | Rest]) -> [trace_mfa(MFA), " <= ", format_trace(Rest)]; - format_trace(Other) -> io_lib:format("~w", [Other]). - trace_mfa({M, F, A}) when is_list(A) -> format_mfa({M, F, length(A)}); - trace_mfa({M, F, A, Props}) when is_list(A) -> format_mfa({M, F, length(A), Props}); - trace_mfa(Other) -> format_mfa(Other). - format_args([], FmtAcc, ArgsAcc) -> {string:join(lists:reverse(FmtAcc), ", "), lists:reverse(ArgsAcc)}; - -format_args([H|T], FmtAcc, ArgsAcc) -> +format_args([H | T], FmtAcc, ArgsAcc) -> {Str, _} = couch_log_trunc_io:print(H, 100), format_args(T, ["~s" | FmtAcc], [Str | ArgsAcc]). - maybe_truncate(Fmt, Args) -> MaxMsgSize = couch_log_config:get(max_message_size), couch_log_trunc_io:format(Fmt, Args, MaxMsgSize). - maybe_truncate(Msg) -> MaxMsgSize = couch_log_config:get(max_message_size), case iolist_size(Msg) > MaxMsgSize of @@ -402,7 +381,6 @@ maybe_truncate(Msg) -> Msg end. - print_silly_list(L) when is_list(L) -> case couch_log_util:string_p(L) of true -> @@ -410,23 +388,21 @@ print_silly_list(L) when is_list(L) -> _ -> print_silly_list(L, [], []) end; - print_silly_list(L) -> {Str, _} = couch_log_trunc_io:print(L, ?DEFAULT_TRUNCATION), Str. - print_silly_list([], Fmt, Acc) -> - couch_log_trunc_io:format(string:join(lists:reverse(Fmt), ", "), - lists:reverse(Acc), ?DEFAULT_TRUNCATION); - + couch_log_trunc_io:format( + string:join(lists:reverse(Fmt), ", "), + lists:reverse(Acc), + ?DEFAULT_TRUNCATION + ); print_silly_list([{K, V} | T], Fmt, Acc) -> print_silly_list(T, ["~p: ~p" | Fmt], [V, K | Acc]); - print_silly_list([H | T], Fmt, Acc) -> print_silly_list(T, ["~p" | Fmt], [H | Acc]). - print_val(Val) -> {Str, _} = couch_log_trunc_io:print(Val, 500), Str. @@ -439,7 +415,6 @@ filter_silly_list(KV) -> filter_silly_list([], _) -> []; - filter_silly_list([{K, V} | T], Filter) -> case lists:member(K, Filter) of true -> @@ -447,11 +422,9 @@ filter_silly_list([{K, V} | T], Filter) -> false -> [{K, V} | filter_silly_list(T, Filter)] end; - filter_silly_list([H | T], Filter) -> [H | filter_silly_list(T, Filter)]. - get_value(Key, Value) -> get_value(Key, Value, undefined). diff --git a/src/couch_log/src/couch_log_monitor.erl b/src/couch_log/src/couch_log_monitor.erl index 96d7f3698..b5ac0a844 100644 --- a/src/couch_log/src/couch_log_monitor.erl +++ b/src/couch_log/src/couch_log_monitor.erl @@ -15,7 +15,6 @@ -behaviour(gen_server). -vsn(1). - -export([ start_link/0 ]). @@ -29,14 +28,11 @@ code_change/3 ]). - -define(HANDLER_MOD, couch_log_error_logger_h). - start_link() -> gen_server:start_link(?MODULE, [], []). - % OTP_RELEASE defined in OTP >= 21 only -ifdef(OTP_RELEASE). @@ -55,26 +51,19 @@ init(_) -> -endif. - terminate(_, _) -> ok. - handle_call(_Msg, _From, St) -> {reply, ignored, St}. - handle_cast(_Msg, St) -> {noreply, St}. - handle_info({gen_event_EXIT, ?HANDLER_MOD, Reason}, St) -> {stop, Reason, St}; - - handle_info(_Msg, St) -> {noreply, St}. - code_change(_, State, _) -> {ok, State}. diff --git a/src/couch_log/src/couch_log_server.erl b/src/couch_log/src/couch_log_server.erl index 8432b9aa3..05cf92a75 100644 --- a/src/couch_log/src/couch_log_server.erl +++ b/src/couch_log/src/couch_log_server.erl @@ -13,7 +13,6 @@ -module(couch_log_server). -behavior(gen_server). - -export([ start_link/0, reconfigure/0, @@ -21,42 +20,35 @@ ]). -export([ - init/1, - terminate/2, - handle_call/3, - handle_cast/2, - handle_info/2, - code_change/3 + init/1, + terminate/2, + handle_call/3, + handle_cast/2, + handle_info/2, + code_change/3 ]). - -include("couch_log.hrl"). - -record(st, { writer }). - -ifdef(TEST). -define(SEND(Entry), gen_server:call(?MODULE, {log, Entry})). -else. -define(SEND(Entry), gen_server:cast(?MODULE, {log, Entry})). -endif. - start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - reconfigure() -> gen_server:call(?MODULE, reconfigure). - log(Entry) -> ?SEND(Entry). - init(_) -> couch_util:set_mqd_off_heap(?MODULE), process_flag(trap_exit, true), @@ -64,17 +56,14 @@ init(_) -> writer = couch_log_writer:init() }}. - terminate(Reason, St) -> ok = couch_log_writer:terminate(Reason, St#st.writer). - handle_call(reconfigure, _From, St) -> ok = couch_log_writer:terminate(reconfiguring, St#st.writer), {reply, ok, St#st{ writer = couch_log_writer:init() }}; - handle_call({log, Entry}, _From, St) -> % We re-check if we should log here in case an operator % adjusted the log level and then realized it was a bad @@ -86,22 +75,18 @@ handle_call({log, Entry}, _From, St) -> false -> {reply, ok, St} end; - handle_call(Ignore, From, St) -> Args = [?MODULE, Ignore], Entry = couch_log_formatter:format(error, ?MODULE, "~s ignored ~p", Args), handle_call({log, Entry}, From, St). - handle_cast(Msg, St) -> {reply, ok, NewSt} = handle_call(Msg, nil, St), {noreply, NewSt}. - handle_info(Msg, St) -> {reply, ok, NewSt} = handle_call(Msg, nil, St), {noreply, NewSt}. - code_change(_Vsn, St, _Extra) -> {ok, St}. diff --git a/src/couch_log/src/couch_log_sup.erl b/src/couch_log/src/couch_log_sup.erl index 6cb8d7395..0167192d8 100644 --- a/src/couch_log/src/couch_log_sup.erl +++ b/src/couch_log/src/couch_log_sup.erl @@ -23,12 +23,10 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). - init([]) -> ok = couch_log_config:init(), {ok, {{one_for_one, 10, 10}, children()}}. - children() -> [ { @@ -74,7 +72,6 @@ handle_config_change("log", Key, _, _, S) -> end, notify_listeners(), {ok, S}; - handle_config_change(_, _, _, _, S) -> {ok, S}. @@ -84,9 +81,12 @@ handle_config_terminate(_Server, _Reason, _State) -> -ifdef(TEST). notify_listeners() -> Listeners = application:get_env(couch_log, config_listeners, []), - lists:foreach(fun(L) -> - L ! couch_log_config_change_finished - end, Listeners). + lists:foreach( + fun(L) -> + L ! couch_log_config_change_finished + end, + Listeners + ). -else. notify_listeners() -> ok. diff --git a/src/couch_log/src/couch_log_trunc_io.erl b/src/couch_log/src/couch_log_trunc_io.erl index 636dfdc1f..9736e87e1 100644 --- a/src/couch_log/src/couch_log_trunc_io.erl +++ b/src/couch_log/src/couch_log_trunc_io.erl @@ -36,33 +36,37 @@ -module(couch_log_trunc_io). -author('matthias@corelatus.se'). %% And thanks to Chris Newcombe for a bug fix --export([format/3, format/4, print/2, print/3, fprint/2, fprint/3, safe/2]). % interface functions + +% interface functions +-export([format/3, format/4, print/2, print/3, fprint/2, fprint/3, safe/2]). -version("$Id: trunc_io.erl,v 1.11 2009-02-23 12:01:06 matthias Exp $"). -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). -endif. --type option() :: {'depth', integer()} +-type option() :: + {'depth', integer()} | {'lists_as_strings', boolean()} | {'force_strings', boolean()}. -type options() :: [option()]. -record(print_options, { - %% negative depth means no depth limiting - depth = -1 :: integer(), - %% whether to print lists as strings, if possible - lists_as_strings = true :: boolean(), - %% force strings, or binaries to be printed as a string, - %% even if they're not printable - force_strings = false :: boolean() - }). + %% negative depth means no depth limiting + depth = -1 :: integer(), + %% whether to print lists as strings, if possible + lists_as_strings = true :: boolean(), + %% force strings, or binaries to be printed as a string, + %% even if they're not printable + force_strings = false :: boolean() +}). format(Fmt, Args, Max) -> format(Fmt, Args, Max, []). format(Fmt, Args, Max, Options) -> - try couch_log_trunc_io_fmt:format(Fmt, Args, Max, Options) + try + couch_log_trunc_io_fmt:format(Fmt, Args, Max, Options) catch _What:_Why -> erlang:error(badarg, [Fmt, Args]) @@ -74,7 +78,6 @@ format(Fmt, Args, Max, Options) -> fprint(Term, Max) -> fprint(Term, Max, []). - %% @doc Returns an flattened list containing the ASCII representation of the given %% term. -spec fprint(term(), pos_integer(), options()) -> string(). @@ -108,30 +111,29 @@ print(Term, Max) -> print(Term, Max, Options) when is_list(Options) -> %% need to convert the proplist to a record print(Term, Max, prepare_options(Options, #print_options{})); - -print(Term, _Max, #print_options{force_strings=true}) when not is_list(Term), not is_binary(Term), not is_atom(Term) -> +print(Term, _Max, #print_options{force_strings = true}) when + not is_list(Term), not is_binary(Term), not is_atom(Term) +-> erlang:error(badarg); - print(_, Max, _Options) when Max < 0 -> {"...", 3}; -print(_, _, #print_options{depth=0}) -> {"...", 3}; - - +print(_, _, #print_options{depth = 0}) -> + {"...", 3}; %% @doc We assume atoms, floats, funs, integers, PIDs, ports and refs never need %% to be truncated. This isn't strictly true, someone could make an %% arbitrarily long bignum. Let's assume that won't happen unless someone %% is being malicious. %% -print(Atom, _Max, #print_options{force_strings=NoQuote}) when is_atom(Atom) -> +print(Atom, _Max, #print_options{force_strings = NoQuote}) when is_atom(Atom) -> L = atom_to_list(Atom), - R = case atom_needs_quoting_start(L) andalso not NoQuote of - true -> lists:flatten([$', L, $']); - false -> L - end, + R = + case atom_needs_quoting_start(L) andalso not NoQuote of + true -> lists:flatten([$', L, $']); + false -> L + end, {R, length(R)}; - -print(<<>>, _Max, #print_options{depth=1}) -> +print(<<>>, _Max, #print_options{depth = 1}) -> {"<<>>", 4}; -print(Bin, _Max, #print_options{depth=1}) when is_binary(Bin) -> +print(Bin, _Max, #print_options{depth = 1}) when is_binary(Bin) -> {"<<...>>", 7}; print(<<>>, _Max, Options) -> case Options#print_options.force_strings of @@ -140,70 +142,79 @@ print(<<>>, _Max, Options) -> false -> {"<<>>", 4} end; - print(Binary, 0, _Options) when is_bitstring(Binary) -> {"<<..>>", 6}; - print(Bin, Max, _Options) when is_binary(Bin), Max < 2 -> {"<<...>>", 7}; print(Binary, Max, Options) when is_binary(Binary) -> B = binary_to_list(Binary, 1, lists:min([Max, byte_size(Binary)])), - {Res, Length} = case Options#print_options.lists_as_strings orelse - Options#print_options.force_strings of - true -> - Depth = Options#print_options.depth, - MaxSize = (Depth - 1) * 4, - %% check if we need to truncate based on depth - In = case Depth > -1 andalso MaxSize < length(B) andalso - not Options#print_options.force_strings of - true -> - string:substr(B, 1, MaxSize); - false -> B - end, - MaxLen = case Options#print_options.force_strings of - true -> - Max; - false -> - %% make room for the leading doublequote - Max - 1 - end, - try alist(In, MaxLen, Options) of - {L0, Len0} -> - case Options#print_options.force_strings of - false -> - case B /= In of - true -> - {[$", L0, "..."], Len0+4}; - false -> - {[$"|L0], Len0+1} - end; + {Res, Length} = + case + Options#print_options.lists_as_strings orelse + Options#print_options.force_strings + of + true -> + Depth = Options#print_options.depth, + MaxSize = (Depth - 1) * 4, + %% check if we need to truncate based on depth + In = + case + Depth > -1 andalso MaxSize < length(B) andalso + not Options#print_options.force_strings + of true -> - {L0, Len0} - end - catch - throw:{unprintable, C} -> - Index = string:chr(In, C), - case Index > 1 andalso Options#print_options.depth =< Index andalso - Options#print_options.depth > -1 andalso - not Options#print_options.force_strings of + string:substr(B, 1, MaxSize); + false -> + B + end, + MaxLen = + case Options#print_options.force_strings of true -> - %% print first Index-1 characters followed by ... - {L0, Len0} = alist_start(string:substr(In, 1, Index - 1), Max - 1, Options), - {L0++"...", Len0+3}; + Max; false -> - list_body(In, Max-4, dec_depth(Options), true) - end - end; - _ -> - list_body(B, Max-4, dec_depth(Options), true) - end, + %% make room for the leading doublequote + Max - 1 + end, + try alist(In, MaxLen, Options) of + {L0, Len0} -> + case Options#print_options.force_strings of + false -> + case B /= In of + true -> + {[$", L0, "..."], Len0 + 4}; + false -> + {[$" | L0], Len0 + 1} + end; + true -> + {L0, Len0} + end + catch + throw:{unprintable, C} -> + Index = string:chr(In, C), + case + Index > 1 andalso Options#print_options.depth =< Index andalso + Options#print_options.depth > -1 andalso + not Options#print_options.force_strings + of + true -> + %% print first Index-1 characters followed by ... + {L0, Len0} = alist_start( + string:substr(In, 1, Index - 1), Max - 1, Options + ), + {L0 ++ "...", Len0 + 3}; + false -> + list_body(In, Max - 4, dec_depth(Options), true) + end + end; + _ -> + list_body(B, Max - 4, dec_depth(Options), true) + end, case Options#print_options.force_strings of true -> {Res, Length}; _ -> - {["<<", Res, ">>"], Length+4} + {["<<", Res, ">>"], Length + 4} end; - %% bitstrings are binary's evil brother who doesn't end on an 8 bit boundary. %% This makes printing them extremely annoying, so list_body/list_bodyc has %% some magic for dealing with the output of bitstring_to_list, which returns @@ -214,27 +225,26 @@ print({inline_bitstring, B}, _Max, _Options) when is_bitstring(B) -> <<Value:Size>> = B, ValueStr = integer_to_list(Value), SizeStr = integer_to_list(Size), - {[ValueStr, $:, SizeStr], length(ValueStr) + length(SizeStr) +1}; + {[ValueStr, $:, SizeStr], length(ValueStr) + length(SizeStr) + 1}; print(BitString, Max, Options) when is_bitstring(BitString) -> - BL = case byte_size(BitString) > Max of - true -> - binary_to_list(BitString, 1, Max); - _ -> - R = erlang:bitstring_to_list(BitString), - {Bytes, [Bits]} = lists:splitwith(fun erlang:is_integer/1, R), - %% tag the trailing bits with a special tuple we catch when - %% list_body calls print again - Bytes ++ [{inline_bitstring, Bits}] - end, + BL = + case byte_size(BitString) > Max of + true -> + binary_to_list(BitString, 1, Max); + _ -> + R = erlang:bitstring_to_list(BitString), + {Bytes, [Bits]} = lists:splitwith(fun erlang:is_integer/1, R), + %% tag the trailing bits with a special tuple we catch when + %% list_body calls print again + Bytes ++ [{inline_bitstring, Bits}] + end, {X, Len0} = list_body(BL, Max - 4, dec_depth(Options), true), {["<<", X, ">>"], Len0 + 4}; - print(Float, _Max, _Options) when is_float(Float) -> %% use the same function io_lib:format uses to print floats %% float_to_list is way too verbose. L = io_lib_format:fwrite_g(Float), {L, length(L)}; - print(Fun, Max, _Options) when is_function(Fun) -> L = erlang:fun_to_list(Fun), case length(L) > Max of @@ -245,42 +255,36 @@ print(Fun, Max, _Options) when is_function(Fun) -> _ -> {L, length(L)} end; - print(Integer, _Max, _Options) when is_integer(Integer) -> L = integer_to_list(Integer), {L, length(L)}; - print(Pid, _Max, _Options) when is_pid(Pid) -> L = pid_to_list(Pid), {L, length(L)}; - print(Ref, _Max, _Options) when is_reference(Ref) -> L = erlang:ref_to_list(Ref), {L, length(L)}; - print(Port, _Max, _Options) when is_port(Port) -> L = erlang:port_to_list(Port), {L, length(L)}; - print({'$lager_record', Name, Fields}, Max, Options) -> Leader = "#" ++ atom_to_list(Name) ++ "{", {RC, Len} = record_fields(Fields, Max - length(Leader) + 1, dec_depth(Options)), {[Leader, RC, "}"], Len + length(Leader) + 1}; - print(Tuple, Max, Options) when is_tuple(Tuple) -> - {TC, Len} = tuple_contents(Tuple, Max-2, Options), + {TC, Len} = tuple_contents(Tuple, Max - 2, Options), {[${, TC, $}], Len + 2}; - print(List, Max, Options) when is_list(List) -> - case Options#print_options.lists_as_strings orelse - Options#print_options.force_strings of + case + Options#print_options.lists_as_strings orelse + Options#print_options.force_strings + of true -> alist_start(List, Max, dec_depth(Options)); _ -> {R, Len} = list_body(List, Max - 2, dec_depth(Options), false), {[$[, R, $]], Len + 2} end; - print(Map, Max, Options) -> case erlang:is_builtin(erlang, is_map, 1) andalso erlang:is_map(Map) of true -> @@ -297,43 +301,52 @@ tuple_contents(Tuple, Max, Options) -> %% Format the inside of a list, i.e. do not add a leading [ or trailing ]. %% Returns {List, Length} -list_body([], _Max, _Options, _Tuple) -> {[], 0}; +list_body([], _Max, _Options, _Tuple) -> + {[], 0}; list_body(_, Max, _Options, _Tuple) when Max < 4 -> {"...", 3}; -list_body(_, _Max, #print_options{depth=0}, _Tuple) -> {"...", 3}; -list_body([H], Max, Options=#print_options{depth=1}, _Tuple) -> +list_body(_, _Max, #print_options{depth = 0}, _Tuple) -> + {"...", 3}; +list_body([H], Max, Options = #print_options{depth = 1}, _Tuple) -> print(H, Max, Options); -list_body([H|_], Max, Options=#print_options{depth=1}, Tuple) -> - {List, Len} = print(H, Max-4, Options), - Sep = case Tuple of - true -> $,; - false -> $| - end, +list_body([H | _], Max, Options = #print_options{depth = 1}, Tuple) -> + {List, Len} = print(H, Max - 4, Options), + Sep = + case Tuple of + true -> $,; + false -> $| + end, {[List ++ [Sep | "..."]], Len + 4}; -list_body([H|T], Max, Options, Tuple) -> +list_body([H | T], Max, Options, Tuple) -> {List, Len} = print(H, Max, Options), {Final, FLen} = list_bodyc(T, Max - Len, Options, Tuple), - {[List|Final], FLen + Len}; -list_body(X, Max, Options, _Tuple) -> %% improper list + {[List | Final], FLen + Len}; +%% improper list +list_body(X, Max, Options, _Tuple) -> {List, Len} = print(X, Max - 1, Options), - {[$|,List], Len + 1}. + {[$|, List], Len + 1}. -list_bodyc([], _Max, _Options, _Tuple) -> {[], 0}; +list_bodyc([], _Max, _Options, _Tuple) -> + {[], 0}; list_bodyc(_, Max, _Options, _Tuple) when Max < 5 -> {",...", 4}; -list_bodyc(_, _Max, #print_options{depth=1}, true) -> {",...", 4}; -list_bodyc(_, _Max, #print_options{depth=1}, false) -> {"|...", 4}; -list_bodyc([H|T], Max, #print_options{depth=Depth} = Options, Tuple) -> +list_bodyc(_, _Max, #print_options{depth = 1}, true) -> + {",...", 4}; +list_bodyc(_, _Max, #print_options{depth = 1}, false) -> + {"|...", 4}; +list_bodyc([H | T], Max, #print_options{depth = Depth} = Options, Tuple) -> {List, Len} = print(H, Max, dec_depth(Options)), {Final, FLen} = list_bodyc(T, Max - Len - 1, dec_depth(Options), Tuple), - Sep = case Depth == 1 andalso not Tuple of - true -> $|; - _ -> $, - end, - {[Sep, List|Final], FLen + Len + 1}; -list_bodyc(X, Max, Options, _Tuple) -> %% improper list + Sep = + case Depth == 1 andalso not Tuple of + true -> $|; + _ -> $, + end, + {[Sep, List | Final], FLen + Len + 1}; +%% improper list +list_bodyc(X, Max, Options, _Tuple) -> {List, Len} = print(X, Max - 1, Options), - {[$|,List], Len + 1}. + {[$|, List], Len + 1}. -map_body(Map, Max, #print_options{depth=Depth}) when Max < 4; Depth =:= 0 -> +map_body(Map, Max, #print_options{depth = Depth}) when Max < 4; Depth =:= 0 -> case erlang:map_size(Map) of 0 -> {[], 0}; _ -> {"...", 3} @@ -353,7 +366,7 @@ map_body(Map, Max, Options) -> map_bodyc([], _Max, _Options) -> {[], 0}; -map_bodyc(_Rest, Max,#print_options{depth=Depth}) when Max < 5; Depth =:= 0 -> +map_bodyc(_Rest, Max, #print_options{depth = Depth}) when Max < 5; Depth =:= 0 -> {",...", 4}; map_bodyc([{Key, Value} | Rest], Max, Options) -> {KeyStr, KeyLen} = print(Key, Max - 5, Options), @@ -370,70 +383,86 @@ map_bodyc([{Key, Value} | Rest], Max, Options) -> %% [0,65,66] -> [0,65,66] %% [65,b,66] -> "A"[b,66] %% -alist_start([], _Max, #print_options{force_strings=true}) -> {"", 0}; -alist_start([], _Max, _Options) -> {"[]", 2}; +alist_start([], _Max, #print_options{force_strings = true}) -> + {"", 0}; +alist_start([], _Max, _Options) -> + {"[]", 2}; alist_start(_, Max, _Options) when Max < 4 -> {"...", 3}; -alist_start(_, _Max, #print_options{depth=0}) -> {"[...]", 5}; -alist_start(L, Max, #print_options{force_strings=true} = Options) -> +alist_start(_, _Max, #print_options{depth = 0}) -> + {"[...]", 5}; +alist_start(L, Max, #print_options{force_strings = true} = Options) -> alist(L, Max, Options); %alist_start([H|_T], _Max, #print_options{depth=1}) when is_integer(H) -> {[$[, H, $|, $., $., $., $]], 7}; -alist_start([H|T], Max, Options) when is_integer(H), H >= 16#20, H =< 16#7e -> % definitely printable - try alist([H|T], Max -1, Options) of + +% definitely printable +alist_start([H | T], Max, Options) when is_integer(H), H >= 16#20, H =< 16#7e -> + try alist([H | T], Max - 1, Options) of {L, Len} -> - {[$"|L], Len + 1} + {[$" | L], Len + 1} catch throw:{unprintable, _} -> - {R, Len} = list_body([H|T], Max-2, Options, false), + {R, Len} = list_body([H | T], Max - 2, Options, false), {[$[, R, $]], Len + 2} end; -alist_start([H|T], Max, Options) when is_integer(H), H >= 16#a0, H =< 16#ff -> % definitely printable - try alist([H|T], Max -1, Options) of +% definitely printable +alist_start([H | T], Max, Options) when is_integer(H), H >= 16#a0, H =< 16#ff -> + try alist([H | T], Max - 1, Options) of {L, Len} -> - {[$"|L], Len + 1} + {[$" | L], Len + 1} catch throw:{unprintable, _} -> - {R, Len} = list_body([H|T], Max-2, Options, false), + {R, Len} = list_body([H | T], Max - 2, Options, false), {[$[, R, $]], Len + 2} end; -alist_start([H|T], Max, Options) when H =:= $\t; H =:= $\n; H =:= $\r; H =:= $\v; H =:= $\e; H=:= $\f; H=:= $\b -> - try alist([H|T], Max -1, Options) of +alist_start([H | T], Max, Options) when + H =:= $\t; H =:= $\n; H =:= $\r; H =:= $\v; H =:= $\e; H =:= $\f; H =:= $\b +-> + try alist([H | T], Max - 1, Options) of {L, Len} -> - {[$"|L], Len + 1} + {[$" | L], Len + 1} catch throw:{unprintable, _} -> - {R, Len} = list_body([H|T], Max-2, Options, false), + {R, Len} = list_body([H | T], Max - 2, Options, false), {[$[, R, $]], Len + 2} end; alist_start(L, Max, Options) -> - {R, Len} = list_body(L, Max-2, Options, false), + {R, Len} = list_body(L, Max - 2, Options, false), {[$[, R, $]], Len + 2}. -alist([], _Max, #print_options{force_strings=true}) -> {"", 0}; -alist([], _Max, _Options) -> {"\"", 1}; -alist(_, Max, #print_options{force_strings=true}) when Max < 4 -> {"...", 3}; -alist(_, Max, #print_options{force_strings=false}) when Max < 5 -> {"...\"", 4}; -alist([H|T], Max, Options = #print_options{force_strings=false,lists_as_strings=true}) when H =:= $"; H =:= $\\ -> +alist([], _Max, #print_options{force_strings = true}) -> + {"", 0}; +alist([], _Max, _Options) -> + {"\"", 1}; +alist(_, Max, #print_options{force_strings = true}) when Max < 4 -> {"...", 3}; +alist(_, Max, #print_options{force_strings = false}) when Max < 5 -> {"...\"", 4}; +alist([H | T], Max, Options = #print_options{force_strings = false, lists_as_strings = true}) when + H =:= $"; H =:= $\\ +-> %% preserve escaping around quotes - {L, Len} = alist(T, Max-1, Options), - {[$\\,H|L], Len + 2}; -alist([H|T], Max, Options) when is_integer(H), H >= 16#20, H =< 16#7e -> % definitely printable - {L, Len} = alist(T, Max-1, Options), - {[H|L], Len + 1}; -alist([H|T], Max, Options) when is_integer(H), H >= 16#a0, H =< 16#ff -> % definitely printable - {L, Len} = alist(T, Max-1, Options), - {[H|L], Len + 1}; -alist([H|T], Max, Options) when H =:= $\t; H =:= $\n; H =:= $\r; H =:= $\v; H =:= $\e; H=:= $\f; H=:= $\b -> - {L, Len} = alist(T, Max-1, Options), + {L, Len} = alist(T, Max - 1, Options), + {[$\\, H | L], Len + 2}; +% definitely printable +alist([H | T], Max, Options) when is_integer(H), H >= 16#20, H =< 16#7e -> + {L, Len} = alist(T, Max - 1, Options), + {[H | L], Len + 1}; +% definitely printable +alist([H | T], Max, Options) when is_integer(H), H >= 16#a0, H =< 16#ff -> + {L, Len} = alist(T, Max - 1, Options), + {[H | L], Len + 1}; +alist([H | T], Max, Options) when + H =:= $\t; H =:= $\n; H =:= $\r; H =:= $\v; H =:= $\e; H =:= $\f; H =:= $\b +-> + {L, Len} = alist(T, Max - 1, Options), case Options#print_options.force_strings of true -> - {[H|L], Len + 1}; + {[H | L], Len + 1}; _ -> - {[escape(H)|L], Len + 1} + {[escape(H) | L], Len + 1} end; -alist([H|T], Max, #print_options{force_strings=true} = Options) when is_integer(H) -> - {L, Len} = alist(T, Max-1, Options), - {[H|L], Len + 1}; -alist([H|T], Max, Options = #print_options{force_strings=true}) when is_binary(H); is_list(H) -> +alist([H | T], Max, #print_options{force_strings = true} = Options) when is_integer(H) -> + {L, Len} = alist(T, Max - 1, Options), + {[H | L], Len + 1}; +alist([H | T], Max, Options = #print_options{force_strings = true}) when is_binary(H); is_list(H) -> {List, Len} = print(H, Max, Options), case (Max - Len) =< 0 of true -> @@ -442,28 +471,31 @@ alist([H|T], Max, Options = #print_options{force_strings=true}) when is_binary(H false -> %% no need to decrement depth, as we're in printable string mode {Final, FLen} = alist(T, Max - Len, Options), - {[List|Final], FLen+Len} + {[List | Final], FLen + Len} end; -alist(_, _, #print_options{force_strings=true}) -> +alist(_, _, #print_options{force_strings = true}) -> erlang:error(badarg); -alist([H|_L], _Max, _Options) -> +alist([H | _L], _Max, _Options) -> throw({unprintable, H}); alist(H, _Max, _Options) -> %% improper list throw({unprintable, H}). %% is the first character in the atom alphabetic & lowercase? -atom_needs_quoting_start([H|T]) when H >= $a, H =< $z -> +atom_needs_quoting_start([H | T]) when H >= $a, H =< $z -> atom_needs_quoting(T); atom_needs_quoting_start(_) -> true. atom_needs_quoting([]) -> false; -atom_needs_quoting([H|T]) when (H >= $a andalso H =< $z); - (H >= $A andalso H =< $Z); - (H >= $0 andalso H =< $9); - H == $@; H == $_ -> +atom_needs_quoting([H | T]) when + (H >= $a andalso H =< $z); + (H >= $A andalso H =< $Z); + (H >= $0 andalso H =< $9); + H == $@; + H == $_ +-> atom_needs_quoting(T); atom_needs_quoting(_) -> true. @@ -471,15 +503,15 @@ atom_needs_quoting(_) -> -spec prepare_options(options(), #print_options{}) -> #print_options{}. prepare_options([], Options) -> Options; -prepare_options([{depth, Depth}|T], Options) when is_integer(Depth) -> - prepare_options(T, Options#print_options{depth=Depth}); -prepare_options([{lists_as_strings, Bool}|T], Options) when is_boolean(Bool) -> +prepare_options([{depth, Depth} | T], Options) when is_integer(Depth) -> + prepare_options(T, Options#print_options{depth = Depth}); +prepare_options([{lists_as_strings, Bool} | T], Options) when is_boolean(Bool) -> prepare_options(T, Options#print_options{lists_as_strings = Bool}); -prepare_options([{force_strings, Bool}|T], Options) when is_boolean(Bool) -> +prepare_options([{force_strings, Bool} | T], Options) when is_boolean(Bool) -> prepare_options(T, Options#print_options{force_strings = Bool}). -dec_depth(#print_options{depth=Depth} = Options) when Depth > 0 -> - Options#print_options{depth=Depth-1}; +dec_depth(#print_options{depth = Depth} = Options) when Depth > 0 -> + Options#print_options{depth = Depth - 1}; dec_depth(Options) -> Options. @@ -493,20 +525,20 @@ escape($\v) -> "\\v". record_fields([], _, _) -> {"", 0}; -record_fields(_, Max, #print_options{depth=D}) when Max < 4; D == 0 -> +record_fields(_, Max, #print_options{depth = D}) when Max < 4; D == 0 -> {"...", 3}; -record_fields([{Field, Value}|T], Max, Options) -> - {ExtraChars, Terminator} = case T of - [] -> - {1, []}; - _ -> - {2, ","} - end, +record_fields([{Field, Value} | T], Max, Options) -> + {ExtraChars, Terminator} = + case T of + [] -> + {1, []}; + _ -> + {2, ","} + end, {FieldStr, FieldLen} = print(Field, Max - ExtraChars, Options), {ValueStr, ValueLen} = print(Value, Max - (FieldLen + ExtraChars), Options), {Final, FLen} = record_fields(T, Max - (FieldLen + ValueLen + ExtraChars), dec_depth(Options)), - {[FieldStr++"="++ValueStr++Terminator|Final], FLen + FieldLen + ValueLen + ExtraChars}. - + {[FieldStr ++ "=" ++ ValueStr ++ Terminator | Final], FLen + FieldLen + ValueLen + ExtraChars}. -ifdef(TEST). %%-------------------- @@ -516,19 +548,27 @@ format_test() -> ?assertEqual("foobar", lists:flatten(format("~s", [["foo", $b, $a, $r]], 50))), ?assertEqual("[\"foo\",98,97,114]", lists:flatten(format("~p", [["foo", $b, $a, $r]], 50))), ?assertEqual("[\"foo\",98,97,114]", lists:flatten(format("~P", [["foo", $b, $a, $r], 10], 50))), - ?assertEqual("[[102,111,111],98,97,114]", lists:flatten(format("~w", [["foo", $b, $a, $r]], 50))), + ?assertEqual( + "[[102,111,111],98,97,114]", lists:flatten(format("~w", [["foo", $b, $a, $r]], 50)) + ), %% complex ones ?assertEqual(" foobar", lists:flatten(format("~10s", [["foo", $b, $a, $r]], 50))), ?assertEqual("f", lists:flatten(format("~1s", [["foo", $b, $a, $r]], 50))), ?assertEqual("[\"foo\",98,97,114]", lists:flatten(format("~22p", [["foo", $b, $a, $r]], 50))), - ?assertEqual("[\"foo\",98,97,114]", lists:flatten(format("~22P", [["foo", $b, $a, $r], 10], 50))), + ?assertEqual( + "[\"foo\",98,97,114]", lists:flatten(format("~22P", [["foo", $b, $a, $r], 10], 50)) + ), ?assertEqual("**********", lists:flatten(format("~10W", [["foo", $b, $a, $r], 10], 50))), - ?assertEqual("[[102,111,111],98,97,114]", lists:flatten(format("~25W", [["foo", $b, $a, $r], 10], 50))), + ?assertEqual( + "[[102,111,111],98,97,114]", lists:flatten(format("~25W", [["foo", $b, $a, $r], 10], 50)) + ), % Note these next two diverge from io_lib:format; the field width is % ignored, when it should be used as max line length. ?assertEqual("[\"foo\",98,97,114]", lists:flatten(format("~10p", [["foo", $b, $a, $r]], 50))), - ?assertEqual("[\"foo\",98,97,114]", lists:flatten(format("~10P", [["foo", $b, $a, $r], 10], 50))), + ?assertEqual( + "[\"foo\",98,97,114]", lists:flatten(format("~10P", [["foo", $b, $a, $r], 10], 50)) + ), ok. atom_quoting_test() -> @@ -545,12 +585,15 @@ sane_float_printing_test() -> ?assertEqual("1.0", lists:flatten(format("~p", [1.0], 50))), ?assertEqual("1.23456789", lists:flatten(format("~p", [1.23456789], 50))), ?assertEqual("1.23456789", lists:flatten(format("~p", [1.234567890], 50))), - ?assertEqual("0.3333333333333333", lists:flatten(format("~p", [1/3], 50))), + ?assertEqual("0.3333333333333333", lists:flatten(format("~p", [1 / 3], 50))), ?assertEqual("0.1234567", lists:flatten(format("~p", [0.1234567], 50))), ok. float_inside_list_test() -> - ?assertEqual("[97,38.233913133184835,99]", lists:flatten(format("~p", [[$a, 38.233913133184835, $c]], 50))), + ?assertEqual( + "[97,38.233913133184835,99]", + lists:flatten(format("~p", [[$a, 38.233913133184835, $c]], 50)) + ), ?assertError(badarg, lists:flatten(format("~s", [[$a, 38.233913133184835, $c]], 50))), ok. @@ -572,14 +615,18 @@ binary_printing_test() -> ?assertEqual("<<\"hello\">>", lists:flatten(format("~p", [<<"hello">>], 50))), ?assertEqual("<<104,101,108,108,111>>", lists:flatten(format("~w", [<<"hello">>], 50))), ?assertEqual("<<1,2,3,4>>", lists:flatten(format("~p", [<<1, 2, 3, 4>>], 50))), - ?assertEqual([1,2,3,4], lists:flatten(format("~s", [<<1, 2, 3, 4>>], 50))), + ?assertEqual([1, 2, 3, 4], lists:flatten(format("~s", [<<1, 2, 3, 4>>], 50))), ?assertEqual("hello", lists:flatten(format("~s", [<<"hello">>], 50))), ?assertEqual("hello\nworld", lists:flatten(format("~s", [<<"hello\nworld">>], 50))), ?assertEqual("<<\"hello\\nworld\">>", lists:flatten(format("~p", [<<"hello\nworld">>], 50))), - ?assertEqual("<<\"\\\"hello world\\\"\">>", lists:flatten(format("~p", [<<"\"hello world\"">>], 50))), + ?assertEqual( + "<<\"\\\"hello world\\\"\">>", lists:flatten(format("~p", [<<"\"hello world\"">>], 50)) + ), ?assertEqual("<<\"hello\\\\world\">>", lists:flatten(format("~p", [<<"hello\\world">>], 50))), ?assertEqual("<<\"hello\\\\\world\">>", lists:flatten(format("~p", [<<"hello\\\world">>], 50))), - ?assertEqual("<<\"hello\\\\\\\\world\">>", lists:flatten(format("~p", [<<"hello\\\\world">>], 50))), + ?assertEqual( + "<<\"hello\\\\\\\\world\">>", lists:flatten(format("~p", [<<"hello\\\\world">>], 50)) + ), ?assertEqual("<<\"hello\\bworld\">>", lists:flatten(format("~p", [<<"hello\bworld">>], 50))), ?assertEqual("<<\"hello\\tworld\">>", lists:flatten(format("~p", [<<"hello\tworld">>], 50))), ?assertEqual("<<\"hello\\nworld\">>", lists:flatten(format("~p", [<<"hello\nworld">>], 50))), @@ -594,20 +641,68 @@ binary_printing_test() -> ok. bitstring_printing_test() -> - ?assertEqual("<<1,2,3,1:7>>", lists:flatten(format("~p", - [<<1, 2, 3, 1:7>>], 100))), - ?assertEqual("<<1:7>>", lists:flatten(format("~p", - [<<1:7>>], 100))), - ?assertEqual("<<1,2,3,...>>", lists:flatten(format("~p", - [<<1, 2, 3, 1:7>>], 12))), - ?assertEqual("<<1,2,3,...>>", lists:flatten(format("~p", - [<<1, 2, 3, 1:7>>], 13))), - ?assertEqual("<<1,2,3,1:7>>", lists:flatten(format("~p", - [<<1, 2, 3, 1:7>>], 14))), + ?assertEqual( + "<<1,2,3,1:7>>", + lists:flatten( + format( + "~p", + [<<1, 2, 3, 1:7>>], + 100 + ) + ) + ), + ?assertEqual( + "<<1:7>>", + lists:flatten( + format( + "~p", + [<<1:7>>], + 100 + ) + ) + ), + ?assertEqual( + "<<1,2,3,...>>", + lists:flatten( + format( + "~p", + [<<1, 2, 3, 1:7>>], + 12 + ) + ) + ), + ?assertEqual( + "<<1,2,3,...>>", + lists:flatten( + format( + "~p", + [<<1, 2, 3, 1:7>>], + 13 + ) + ) + ), + ?assertEqual( + "<<1,2,3,1:7>>", + lists:flatten( + format( + "~p", + [<<1, 2, 3, 1:7>>], + 14 + ) + ) + ), ?assertEqual("<<..>>", lists:flatten(format("~p", [<<1:7>>], 0))), ?assertEqual("<<...>>", lists:flatten(format("~p", [<<1:7>>], 1))), - ?assertEqual("[<<1>>,<<2>>]", lists:flatten(format("~p", [[<<1>>, <<2>>]], - 100))), + ?assertEqual( + "[<<1>>,<<2>>]", + lists:flatten( + format( + "~p", + [[<<1>>, <<2>>]], + 100 + ) + ) + ), ?assertEqual("{<<1:7>>}", lists:flatten(format("~p", [{<<1:7>>}], 50))), ok. @@ -617,48 +712,91 @@ list_printing_test() -> ?assertEqual("", lists:flatten(format("~s", [[]], 50))), ?assertEqual("...", lists:flatten(format("~s", [[]], -1))), ?assertEqual("[[]]", lists:flatten(format("~p", [[[]]], 50))), - ?assertEqual("[13,11,10,8,5,4]", lists:flatten(format("~p", [[13,11,10,8,5,4]], 50))), - ?assertEqual("\"\\rabc\"", lists:flatten(format("~p", [[13,$a, $b, $c]], 50))), - ?assertEqual("[1,2,3|4]", lists:flatten(format("~p", [[1, 2, 3|4]], 50))), - ?assertEqual("[...]", lists:flatten(format("~p", [[1, 2, 3,4]], 4))), + ?assertEqual("[13,11,10,8,5,4]", lists:flatten(format("~p", [[13, 11, 10, 8, 5, 4]], 50))), + ?assertEqual("\"\\rabc\"", lists:flatten(format("~p", [[13, $a, $b, $c]], 50))), + ?assertEqual("[1,2,3|4]", lists:flatten(format("~p", [[1, 2, 3 | 4]], 50))), + ?assertEqual("[...]", lists:flatten(format("~p", [[1, 2, 3, 4]], 4))), ?assertEqual("[1,...]", lists:flatten(format("~p", [[1, 2, 3, 4]], 6))), ?assertEqual("[1,...]", lists:flatten(format("~p", [[1, 2, 3, 4]], 7))), ?assertEqual("[1,2,...]", lists:flatten(format("~p", [[1, 2, 3, 4]], 8))), - ?assertEqual("[1|4]", lists:flatten(format("~p", [[1|4]], 50))), + ?assertEqual("[1|4]", lists:flatten(format("~p", [[1 | 4]], 50))), ?assertEqual("[1]", lists:flatten(format("~p", [[1]], 50))), - ?assertError(badarg, lists:flatten(format("~s", [[1|4]], 50))), + ?assertError(badarg, lists:flatten(format("~s", [[1 | 4]], 50))), ?assertEqual("\"hello...\"", lists:flatten(format("~p", ["hello world"], 10))), ?assertEqual("hello w...", lists:flatten(format("~s", ["hello world"], 10))), ?assertEqual("hello world\r\n", lists:flatten(format("~s", ["hello world\r\n"], 50))), ?assertEqual("\rhello world\r\n", lists:flatten(format("~s", ["\rhello world\r\n"], 50))), - ?assertEqual("\"\\rhello world\\r\\n\"", lists:flatten(format("~p", ["\rhello world\r\n"], 50))), - ?assertEqual("[13,104,101,108,108,111,32,119,111,114,108,100,13,10]", lists:flatten(format("~w", ["\rhello world\r\n"], 60))), + ?assertEqual( + "\"\\rhello world\\r\\n\"", lists:flatten(format("~p", ["\rhello world\r\n"], 50)) + ), + ?assertEqual( + "[13,104,101,108,108,111,32,119,111,114,108,100,13,10]", + lists:flatten(format("~w", ["\rhello world\r\n"], 60)) + ), ?assertEqual("...", lists:flatten(format("~s", ["\rhello world\r\n"], 3))), - ?assertEqual("[22835963083295358096932575511191922182123945984,...]", - lists:flatten(format("~p", [ - [22835963083295358096932575511191922182123945984, - 22835963083295358096932575511191922182123945984]], 9))), - ?assertEqual("[22835963083295358096932575511191922182123945984,...]", - lists:flatten(format("~p", [ - [22835963083295358096932575511191922182123945984, - 22835963083295358096932575511191922182123945984]], 53))), + ?assertEqual( + "[22835963083295358096932575511191922182123945984,...]", + lists:flatten( + format( + "~p", + [ + [ + 22835963083295358096932575511191922182123945984, + 22835963083295358096932575511191922182123945984 + ] + ], + 9 + ) + ) + ), + ?assertEqual( + "[22835963083295358096932575511191922182123945984,...]", + lists:flatten( + format( + "~p", + [ + [ + 22835963083295358096932575511191922182123945984, + 22835963083295358096932575511191922182123945984 + ] + ], + 53 + ) + ) + ), %%improper list - ?assertEqual("[1,2,3|4]", lists:flatten(format("~P", [[1|[2|[3|4]]], 5], 50))), - ?assertEqual("[1|1]", lists:flatten(format("~P", [[1|1], 5], 50))), - ?assertEqual("[9|9]", lists:flatten(format("~p", [[9|9]], 50))), + ?assertEqual("[1,2,3|4]", lists:flatten(format("~P", [[1 | [2 | [3 | 4]]], 5], 50))), + ?assertEqual("[1|1]", lists:flatten(format("~P", [[1 | 1], 5], 50))), + ?assertEqual("[9|9]", lists:flatten(format("~p", [[9 | 9]], 50))), ok. iolist_printing_test() -> - ?assertEqual("iolist: HelloIamaniolist", - lists:flatten(format("iolist: ~s", [[$H, $e, $l, $l, $o, "I", ["am", [<<"an">>], [$i, $o, $l, $i, $s, $t]]]], 1000))), - ?assertEqual("123...", - lists:flatten(format("~s", [[<<"123456789">>, "HellIamaniolist"]], 6))), - ?assertEqual("123456...", - lists:flatten(format("~s", [[<<"123456789">>, "HellIamaniolist"]], 9))), - ?assertEqual("123456789H...", - lists:flatten(format("~s", [[<<"123456789">>, "HellIamaniolist"]], 13))), - ?assertEqual("123456789HellIamaniolist", - lists:flatten(format("~s", [[<<"123456789">>, "HellIamaniolist"]], 30))), + ?assertEqual( + "iolist: HelloIamaniolist", + lists:flatten( + format( + "iolist: ~s", + [[$H, $e, $l, $l, $o, "I", ["am", [<<"an">>], [$i, $o, $l, $i, $s, $t]]]], + 1000 + ) + ) + ), + ?assertEqual( + "123...", + lists:flatten(format("~s", [[<<"123456789">>, "HellIamaniolist"]], 6)) + ), + ?assertEqual( + "123456...", + lists:flatten(format("~s", [[<<"123456789">>, "HellIamaniolist"]], 9)) + ), + ?assertEqual( + "123456789H...", + lists:flatten(format("~s", [[<<"123456789">>, "HellIamaniolist"]], 13)) + ), + ?assertEqual( + "123456789HellIamaniolist", + lists:flatten(format("~s", [[<<"123456789">>, "HellIamaniolist"]], 30)) + ), ok. @@ -671,22 +809,48 @@ tuple_printing_test() -> ?assertEqual("{...}", lists:flatten(format("~p", [{foo}], 3))), ?assertEqual("{...}", lists:flatten(format("~p", [{foo}], 4))), ?assertEqual("{...}", lists:flatten(format("~p", [{foo}], 5))), - ?assertEqual("{foo,...}", lists:flatten(format("~p", [{foo,bar}], 6))), - ?assertEqual("{foo,...}", lists:flatten(format("~p", [{foo,bar}], 7))), - ?assertEqual("{foo,...}", lists:flatten(format("~p", [{foo,bar}], 9))), - ?assertEqual("{foo,bar}", lists:flatten(format("~p", [{foo,bar}], 10))), - ?assertEqual("{22835963083295358096932575511191922182123945984,...}", - lists:flatten(format("~w", [ + ?assertEqual("{foo,...}", lists:flatten(format("~p", [{foo, bar}], 6))), + ?assertEqual("{foo,...}", lists:flatten(format("~p", [{foo, bar}], 7))), + ?assertEqual("{foo,...}", lists:flatten(format("~p", [{foo, bar}], 9))), + ?assertEqual("{foo,bar}", lists:flatten(format("~p", [{foo, bar}], 10))), + ?assertEqual( + "{22835963083295358096932575511191922182123945984,...}", + lists:flatten( + format( + "~w", + [ {22835963083295358096932575511191922182123945984, - 22835963083295358096932575511191922182123945984}], 10))), - ?assertEqual("{22835963083295358096932575511191922182123945984,...}", - lists:flatten(format("~w", [ + 22835963083295358096932575511191922182123945984} + ], + 10 + ) + ) + ), + ?assertEqual( + "{22835963083295358096932575511191922182123945984,...}", + lists:flatten( + format( + "~w", + [ + {22835963083295358096932575511191922182123945984, bar} + ], + 10 + ) + ) + ), + ?assertEqual( + "{22835963083295358096932575511191922182123945984,...}", + lists:flatten( + format( + "~w", + [ {22835963083295358096932575511191922182123945984, - bar}], 10))), - ?assertEqual("{22835963083295358096932575511191922182123945984,...}", - lists:flatten(format("~w", [ - {22835963083295358096932575511191922182123945984, - 22835963083295358096932575511191922182123945984}], 53))), + 22835963083295358096932575511191922182123945984} + ], + 53 + ) + ) + ), ok. map_printing_test() -> @@ -698,40 +862,102 @@ map_printing_test() -> ?assertError(badarg, lists:flatten(format("~s", [maps:new()], 50))), ?assertEqual("#{...}", lists:flatten(format("~p", [maps:from_list([{bar, foo}])], 1))), ?assertEqual("#{...}", lists:flatten(format("~p", [maps:from_list([{bar, foo}])], 6))), - ?assertEqual("#{bar => ...}", lists:flatten(format("~p", [maps:from_list([{bar, foo}])], 7))), - ?assertEqual("#{bar => ...}", lists:flatten(format("~p", [maps:from_list([{bar, foo}])], 9))), - ?assertEqual("#{bar => foo}", lists:flatten(format("~p", [maps:from_list([{bar, foo}])], 10))), - ?assertEqual("#{bar => ...,...}", lists:flatten(format("~p", [maps:from_list([{bar, foo}, {foo, bar}])], 9))), - ?assertEqual("#{bar => foo,...}", lists:flatten(format("~p", [maps:from_list([{bar, foo}, {foo, bar}])], 10))), - ?assertEqual("#{bar => foo,...}", lists:flatten(format("~p", [maps:from_list([{bar, foo}, {foo, bar}])], 17))), - ?assertEqual("#{bar => foo,foo => ...}", lists:flatten(format("~p", [maps:from_list([{bar, foo}, {foo, bar}])], 18))), - ?assertEqual("#{bar => foo,foo => ...}", lists:flatten(format("~p", [maps:from_list([{bar, foo}, {foo, bar}])], 19))), - ?assertEqual("#{bar => foo,foo => ...}", lists:flatten(format("~p", [maps:from_list([{bar, foo}, {foo, bar}])], 20))), - ?assertEqual("#{bar => foo,foo => bar}", lists:flatten(format("~p", [maps:from_list([{bar, foo}, {foo, bar}])], 21))), - ?assertEqual("#{22835963083295358096932575511191922182123945984 => ...}", - lists:flatten(format("~w", [ - maps:from_list([{22835963083295358096932575511191922182123945984, - 22835963083295358096932575511191922182123945984}])], 10))), - ?assertEqual("#{22835963083295358096932575511191922182123945984 => ...}", - lists:flatten(format("~w", [ - maps:from_list([{22835963083295358096932575511191922182123945984, - bar}])], 10))), - ?assertEqual("#{22835963083295358096932575511191922182123945984 => ...}", - lists:flatten(format("~w", [ - maps:from_list([{22835963083295358096932575511191922182123945984, - bar}])], 53))), - ?assertEqual("#{22835963083295358096932575511191922182123945984 => bar}", - lists:flatten(format("~w", [ - maps:from_list([{22835963083295358096932575511191922182123945984, - bar}])], 54))), + ?assertEqual( + "#{bar => ...}", lists:flatten(format("~p", [maps:from_list([{bar, foo}])], 7)) + ), + ?assertEqual( + "#{bar => ...}", lists:flatten(format("~p", [maps:from_list([{bar, foo}])], 9)) + ), + ?assertEqual( + "#{bar => foo}", lists:flatten(format("~p", [maps:from_list([{bar, foo}])], 10)) + ), + ?assertEqual( + "#{bar => ...,...}", + lists:flatten(format("~p", [maps:from_list([{bar, foo}, {foo, bar}])], 9)) + ), + ?assertEqual( + "#{bar => foo,...}", + lists:flatten(format("~p", [maps:from_list([{bar, foo}, {foo, bar}])], 10)) + ), + ?assertEqual( + "#{bar => foo,...}", + lists:flatten(format("~p", [maps:from_list([{bar, foo}, {foo, bar}])], 17)) + ), + ?assertEqual( + "#{bar => foo,foo => ...}", + lists:flatten(format("~p", [maps:from_list([{bar, foo}, {foo, bar}])], 18)) + ), + ?assertEqual( + "#{bar => foo,foo => ...}", + lists:flatten(format("~p", [maps:from_list([{bar, foo}, {foo, bar}])], 19)) + ), + ?assertEqual( + "#{bar => foo,foo => ...}", + lists:flatten(format("~p", [maps:from_list([{bar, foo}, {foo, bar}])], 20)) + ), + ?assertEqual( + "#{bar => foo,foo => bar}", + lists:flatten(format("~p", [maps:from_list([{bar, foo}, {foo, bar}])], 21)) + ), + ?assertEqual( + "#{22835963083295358096932575511191922182123945984 => ...}", + lists:flatten( + format( + "~w", + [ + maps:from_list([ + {22835963083295358096932575511191922182123945984, + 22835963083295358096932575511191922182123945984} + ]) + ], + 10 + ) + ) + ), + ?assertEqual( + "#{22835963083295358096932575511191922182123945984 => ...}", + lists:flatten( + format( + "~w", + [ + maps:from_list([{22835963083295358096932575511191922182123945984, bar}]) + ], + 10 + ) + ) + ), + ?assertEqual( + "#{22835963083295358096932575511191922182123945984 => ...}", + lists:flatten( + format( + "~w", + [ + maps:from_list([{22835963083295358096932575511191922182123945984, bar}]) + ], + 53 + ) + ) + ), + ?assertEqual( + "#{22835963083295358096932575511191922182123945984 => bar}", + lists:flatten( + format( + "~w", + [ + maps:from_list([{22835963083295358096932575511191922182123945984, bar}]) + ], + 54 + ) + ) + ), ok; false -> ok end. unicode_test() -> - ?assertEqual([231,167,129], lists:flatten(format("~s", [<<231,167,129>>], 50))), - ?assertEqual([31169], lists:flatten(format("~ts", [<<231,167,129>>], 50))), + ?assertEqual([231, 167, 129], lists:flatten(format("~s", [<<231, 167, 129>>], 50))), + ?assertEqual([31169], lists:flatten(format("~ts", [<<231, 167, 129>>], 50))), ok. depth_limit_test() -> @@ -754,21 +980,54 @@ depth_limit_test() -> case erlang:is_builtin(erlang, is_map, 1) of true -> - ?assertEqual("#{a => #{...}}", - lists:flatten(format("~P", - [maps:from_list([{a, maps:from_list([{b, maps:from_list([{c, d}])}])}]), 2], 50))), - ?assertEqual("#{a => #{b => #{...}}}", - lists:flatten(format("~P", - [maps:from_list([{a, maps:from_list([{b, maps:from_list([{c, d}])}])}]), 3], 50))), - ?assertEqual("#{a => #{b => #{c => d}}}", - lists:flatten(format("~P", - [maps:from_list([{a, maps:from_list([{b, maps:from_list([{c, d}])}])}]), 4], 50))), + ?assertEqual( + "#{a => #{...}}", + lists:flatten( + format( + "~P", + [maps:from_list([{a, maps:from_list([{b, maps:from_list([{c, d}])}])}]), 2], + 50 + ) + ) + ), + ?assertEqual( + "#{a => #{b => #{...}}}", + lists:flatten( + format( + "~P", + [maps:from_list([{a, maps:from_list([{b, maps:from_list([{c, d}])}])}]), 3], + 50 + ) + ) + ), + ?assertEqual( + "#{a => #{b => #{c => d}}}", + lists:flatten( + format( + "~P", + [maps:from_list([{a, maps:from_list([{b, maps:from_list([{c, d}])}])}]), 4], + 50 + ) + ) + ), ?assertEqual("#{}", lists:flatten(format("~P", [maps:new(), 1], 50))), - ?assertEqual("#{...}", lists:flatten(format("~P", [maps:from_list([{1,1}, {2,2}, {3,3}]), 1], 50))), - ?assertEqual("#{1 => 1,...}", lists:flatten(format("~P", [maps:from_list([{1,1}, {2,2}, {3,3}]), 2], 50))), - ?assertEqual("#{1 => 1,2 => 2,...}", lists:flatten(format("~P", [maps:from_list([{1,1}, {2,2}, {3,3}]), 3], 50))), - ?assertEqual("#{1 => 1,2 => 2,3 => 3}", lists:flatten(format("~P", [maps:from_list([{1,1}, {2,2}, {3,3}]), 4], 50))), + ?assertEqual( + "#{...}", + lists:flatten(format("~P", [maps:from_list([{1, 1}, {2, 2}, {3, 3}]), 1], 50)) + ), + ?assertEqual( + "#{1 => 1,...}", + lists:flatten(format("~P", [maps:from_list([{1, 1}, {2, 2}, {3, 3}]), 2], 50)) + ), + ?assertEqual( + "#{1 => 1,2 => 2,...}", + lists:flatten(format("~P", [maps:from_list([{1, 1}, {2, 2}, {3, 3}]), 3], 50)) + ), + ?assertEqual( + "#{1 => 1,2 => 2,3 => 3}", + lists:flatten(format("~P", [maps:from_list([{1, 1}, {2, 2}, {3, 3}]), 4], 50)) + ), ok; false -> @@ -776,8 +1035,14 @@ depth_limit_test() -> end, ?assertEqual("{\"a\",[...]}", lists:flatten(format("~P", [{"a", ["b", ["c", ["d"]]]}, 3], 50))), - ?assertEqual("{\"a\",[\"b\",[[...]|...]]}", lists:flatten(format("~P", [{"a", ["b", ["c", ["d"]]]}, 6], 50))), - ?assertEqual("{\"a\",[\"b\",[\"c\",[\"d\"]]]}", lists:flatten(format("~P", [{"a", ["b", ["c", ["d"]]]}, 9], 50))), + ?assertEqual( + "{\"a\",[\"b\",[[...]|...]]}", + lists:flatten(format("~P", [{"a", ["b", ["c", ["d"]]]}, 6], 50)) + ), + ?assertEqual( + "{\"a\",[\"b\",[\"c\",[\"d\"]]]}", + lists:flatten(format("~P", [{"a", ["b", ["c", ["d"]]]}, 9], 50)) + ), ?assertEqual("[...]", lists:flatten(format("~P", [[1, 2, 3], 1], 50))), ?assertEqual("[1|...]", lists:flatten(format("~P", [[1, 2, 3], 2], 50))), @@ -808,21 +1073,23 @@ depth_limit_test() -> %% depth limiting for some reason works in 4 byte chunks on printable binaries? ?assertEqual("<<\"hell\"...>>", lists:flatten(format("~P", [<<"hello world">>, 2], 50))), - ?assertEqual("<<\"abcd\"...>>", lists:flatten(format("~P", [<<$a, $b, $c, $d, $e, 0>>, 2], 50))), + ?assertEqual( + "<<\"abcd\"...>>", lists:flatten(format("~P", [<<$a, $b, $c, $d, $e, 0>>, 2], 50)) + ), %% I don't even know... ?assertEqual("<<>>", lists:flatten(format("~P", [<<>>, 1], 50))), ?assertEqual("<<>>", lists:flatten(format("~W", [<<>>, 1], 50))), - ?assertEqual("{abc,<<\"abc\\\"\">>}", lists:flatten(format("~P", [{abc,<<"abc\"">>}, 4], 50))), + ?assertEqual("{abc,<<\"abc\\\"\">>}", lists:flatten(format("~P", [{abc, <<"abc\"">>}, 4], 50))), ok. print_terms_without_format_string_test() -> ?assertError(badarg, format({hello, world}, [], 50)), ?assertError(badarg, format([{google, bomb}], [], 50)), - ?assertError(badarg, format([$h,$e,$l,$l,$o, 3594], [], 50)), - ?assertEqual("helloworld", lists:flatten(format([$h,$e,$l,$l,$o, "world"], [], 50))), + ?assertError(badarg, format([$h, $e, $l, $l, $o, 3594], [], 50)), + ?assertEqual("helloworld", lists:flatten(format([$h, $e, $l, $l, $o, "world"], [], 50))), ?assertEqual("hello", lists:flatten(format(<<"hello">>, [], 50))), ?assertEqual("hello", lists:flatten(format('hello', [], 50))), ?assertError(badarg, format(<<1, 2, 3, 1:7>>, [], 100)), @@ -830,9 +1097,9 @@ print_terms_without_format_string_test() -> ok. improper_io_list_test() -> - ?assertEqual(">hello", lists:flatten(format('~s', [[$>|<<"hello">>]], 50))), - ?assertEqual(">hello", lists:flatten(format('~ts', [[$>|<<"hello">>]], 50))), - ?assertEqual("helloworld", lists:flatten(format('~ts', [[<<"hello">>|<<"world">>]], 50))), + ?assertEqual(">hello", lists:flatten(format('~s', [[$> | <<"hello">>]], 50))), + ?assertEqual(">hello", lists:flatten(format('~ts', [[$> | <<"hello">>]], 50))), + ?assertEqual("helloworld", lists:flatten(format('~ts', [[<<"hello">> | <<"world">>]], 50))), ok. --endif.
\ No newline at end of file +-endif. diff --git a/src/couch_log/src/couch_log_trunc_io_fmt.erl b/src/couch_log/src/couch_log_trunc_io_fmt.erl index 77f0b2e0d..cf18019ad 100644 --- a/src/couch_log/src/couch_log_trunc_io_fmt.erl +++ b/src/couch_log/src/couch_log_trunc_io_fmt.erl @@ -22,12 +22,11 @@ %% lager_Format. -module(couch_log_trunc_io_fmt). - -export([format/3, format/4]). -record(options, { - chomp = false :: boolean() - }). + chomp = false :: boolean() +}). format(FmtStr, Args, MaxLen) -> format(FmtStr, Args, MaxLen, []). @@ -46,11 +45,15 @@ format(FmtStr, Args, MaxLen, Opts) when is_list(FmtStr) -> {Cs2, MaxLen2} = build(Cs, [], MaxLen, Options), %% count how many terms remain {Count, StrLen} = lists:foldl( - fun({_C, _As, _F, _Adj, _P, _Pad, _Enc}, {Terms, Chars}) -> + fun + ({_C, _As, _F, _Adj, _P, _Pad, _Enc}, {Terms, Chars}) -> {Terms + 1, Chars}; (_, {Terms, Chars}) -> {Terms, Chars + 1} - end, {0, 0}, Cs2), + end, + {0, 0}, + Cs2 + ), build2(Cs2, Count, MaxLen2 - StrLen); false -> erlang:error(badarg) @@ -58,114 +61,116 @@ format(FmtStr, Args, MaxLen, Opts) when is_list(FmtStr) -> format(_FmtStr, _Args, _MaxLen, _Opts) -> erlang:error(badarg). -collect([$~|Fmt0], Args0) -> - {C,Fmt1,Args1} = collect_cseq(Fmt0, Args0), - [C|collect(Fmt1, Args1)]; -collect([C|Fmt], Args) -> - [C|collect(Fmt, Args)]; -collect([], []) -> []. +collect([$~ | Fmt0], Args0) -> + {C, Fmt1, Args1} = collect_cseq(Fmt0, Args0), + [C | collect(Fmt1, Args1)]; +collect([C | Fmt], Args) -> + [C | collect(Fmt, Args)]; +collect([], []) -> + []. collect_cseq(Fmt0, Args0) -> - {F,Ad,Fmt1,Args1} = field_width(Fmt0, Args0), - {P,Fmt2,Args2} = precision(Fmt1, Args1), - {Pad,Fmt3,Args3} = pad_char(Fmt2, Args2), - {Encoding,Fmt4,Args4} = encoding(Fmt3, Args3), - {C,As,Fmt5,Args5} = collect_cc(Fmt4, Args4), - {{C,As,F,Ad,P,Pad,Encoding},Fmt5,Args5}. - -encoding([$t|Fmt],Args) -> - {unicode,Fmt,Args}; -encoding(Fmt,Args) -> - {latin1,Fmt,Args}. - -field_width([$-|Fmt0], Args0) -> - {F,Fmt,Args} = field_value(Fmt0, Args0), + {F, Ad, Fmt1, Args1} = field_width(Fmt0, Args0), + {P, Fmt2, Args2} = precision(Fmt1, Args1), + {Pad, Fmt3, Args3} = pad_char(Fmt2, Args2), + {Encoding, Fmt4, Args4} = encoding(Fmt3, Args3), + {C, As, Fmt5, Args5} = collect_cc(Fmt4, Args4), + {{C, As, F, Ad, P, Pad, Encoding}, Fmt5, Args5}. + +encoding([$t | Fmt], Args) -> + {unicode, Fmt, Args}; +encoding(Fmt, Args) -> + {latin1, Fmt, Args}. + +field_width([$- | Fmt0], Args0) -> + {F, Fmt, Args} = field_value(Fmt0, Args0), field_width(-F, Fmt, Args); field_width(Fmt0, Args0) -> - {F,Fmt,Args} = field_value(Fmt0, Args0), + {F, Fmt, Args} = field_value(Fmt0, Args0), field_width(F, Fmt, Args). field_width(F, Fmt, Args) when F < 0 -> - {-F,left,Fmt,Args}; + {-F, left, Fmt, Args}; field_width(F, Fmt, Args) when F >= 0 -> - {F,right,Fmt,Args}. + {F, right, Fmt, Args}. -precision([$.|Fmt], Args) -> +precision([$. | Fmt], Args) -> field_value(Fmt, Args); precision(Fmt, Args) -> - {none,Fmt,Args}. + {none, Fmt, Args}. -field_value([$*|Fmt], [A|Args]) when is_integer(A) -> - {A,Fmt,Args}; -field_value([C|Fmt], Args) when is_integer(C), C >= $0, C =< $9 -> - field_value([C|Fmt], Args, 0); +field_value([$* | Fmt], [A | Args]) when is_integer(A) -> + {A, Fmt, Args}; +field_value([C | Fmt], Args) when is_integer(C), C >= $0, C =< $9 -> + field_value([C | Fmt], Args, 0); field_value(Fmt, Args) -> - {none,Fmt,Args}. + {none, Fmt, Args}. -field_value([C|Fmt], Args, F) when is_integer(C), C >= $0, C =< $9 -> - field_value(Fmt, Args, 10*F + (C - $0)); -field_value(Fmt, Args, F) -> %Default case - {F,Fmt,Args}. +field_value([C | Fmt], Args, F) when is_integer(C), C >= $0, C =< $9 -> + field_value(Fmt, Args, 10 * F + (C - $0)); +%Default case +field_value(Fmt, Args, F) -> + {F, Fmt, Args}. -pad_char([$.,$*|Fmt], [Pad|Args]) -> {Pad,Fmt,Args}; -pad_char([$.,Pad|Fmt], Args) -> {Pad,Fmt,Args}; -pad_char(Fmt, Args) -> {$\s,Fmt,Args}. +pad_char([$., $* | Fmt], [Pad | Args]) -> {Pad, Fmt, Args}; +pad_char([$., Pad | Fmt], Args) -> {Pad, Fmt, Args}; +pad_char(Fmt, Args) -> {$\s, Fmt, Args}. %% collect_cc([FormatChar], [Argument]) -> %% {Control,[ControlArg],[FormatChar],[Arg]}. %% Here we collect the argments for each control character. %% Be explicit to cause failure early. -collect_cc([$w|Fmt], [A|Args]) -> {$w,[A],Fmt,Args}; -collect_cc([$p|Fmt], [A|Args]) -> {$p,[A],Fmt,Args}; -collect_cc([$W|Fmt], [A,Depth|Args]) -> {$W,[A,Depth],Fmt,Args}; -collect_cc([$P|Fmt], [A,Depth|Args]) -> {$P,[A,Depth],Fmt,Args}; -collect_cc([$s|Fmt], [A|Args]) -> {$s,[A],Fmt,Args}; -collect_cc([$r|Fmt], [A|Args]) -> {$r,[A],Fmt,Args}; -collect_cc([$e|Fmt], [A|Args]) -> {$e,[A],Fmt,Args}; -collect_cc([$f|Fmt], [A|Args]) -> {$f,[A],Fmt,Args}; -collect_cc([$g|Fmt], [A|Args]) -> {$g,[A],Fmt,Args}; -collect_cc([$b|Fmt], [A|Args]) -> {$b,[A],Fmt,Args}; -collect_cc([$B|Fmt], [A|Args]) -> {$B,[A],Fmt,Args}; -collect_cc([$x|Fmt], [A,Prefix|Args]) -> {$x,[A,Prefix],Fmt,Args}; -collect_cc([$X|Fmt], [A,Prefix|Args]) -> {$X,[A,Prefix],Fmt,Args}; -collect_cc([$+|Fmt], [A|Args]) -> {$+,[A],Fmt,Args}; -collect_cc([$#|Fmt], [A|Args]) -> {$#,[A],Fmt,Args}; -collect_cc([$c|Fmt], [A|Args]) -> {$c,[A],Fmt,Args}; -collect_cc([$~|Fmt], Args) when is_list(Args) -> {$~,[],Fmt,Args}; -collect_cc([$n|Fmt], Args) when is_list(Args) -> {$n,[],Fmt,Args}; -collect_cc([$i|Fmt], [A|Args]) -> {$i,[A],Fmt,Args}. - +collect_cc([$w | Fmt], [A | Args]) -> {$w, [A], Fmt, Args}; +collect_cc([$p | Fmt], [A | Args]) -> {$p, [A], Fmt, Args}; +collect_cc([$W | Fmt], [A, Depth | Args]) -> {$W, [A, Depth], Fmt, Args}; +collect_cc([$P | Fmt], [A, Depth | Args]) -> {$P, [A, Depth], Fmt, Args}; +collect_cc([$s | Fmt], [A | Args]) -> {$s, [A], Fmt, Args}; +collect_cc([$r | Fmt], [A | Args]) -> {$r, [A], Fmt, Args}; +collect_cc([$e | Fmt], [A | Args]) -> {$e, [A], Fmt, Args}; +collect_cc([$f | Fmt], [A | Args]) -> {$f, [A], Fmt, Args}; +collect_cc([$g | Fmt], [A | Args]) -> {$g, [A], Fmt, Args}; +collect_cc([$b | Fmt], [A | Args]) -> {$b, [A], Fmt, Args}; +collect_cc([$B | Fmt], [A | Args]) -> {$B, [A], Fmt, Args}; +collect_cc([$x | Fmt], [A, Prefix | Args]) -> {$x, [A, Prefix], Fmt, Args}; +collect_cc([$X | Fmt], [A, Prefix | Args]) -> {$X, [A, Prefix], Fmt, Args}; +collect_cc([$+ | Fmt], [A | Args]) -> {$+, [A], Fmt, Args}; +collect_cc([$# | Fmt], [A | Args]) -> {$#, [A], Fmt, Args}; +collect_cc([$c | Fmt], [A | Args]) -> {$c, [A], Fmt, Args}; +collect_cc([$~ | Fmt], Args) when is_list(Args) -> {$~, [], Fmt, Args}; +collect_cc([$n | Fmt], Args) when is_list(Args) -> {$n, [], Fmt, Args}; +collect_cc([$i | Fmt], [A | Args]) -> {$i, [A], Fmt, Args}. %% build([Control], Pc, Indentation) -> [Char]. %% Interpret the control structures. Count the number of print %% remaining and only calculate indentation when necessary. Must also %% be smart when calculating indentation for characters in format. -build([{$n, _, _, _, _, _, _}], Acc, MaxLen, #options{chomp=true}) -> +build([{$n, _, _, _, _, _, _}], Acc, MaxLen, #options{chomp = true}) -> %% trailing ~n, ignore {lists:reverse(Acc), MaxLen}; -build([{C,As,F,Ad,P,Pad,Enc}|Cs], Acc, MaxLen, O) -> +build([{C, As, F, Ad, P, Pad, Enc} | Cs], Acc, MaxLen, O) -> {S, MaxLen2} = control(C, As, F, Ad, P, Pad, Enc, MaxLen), - build(Cs, [S|Acc], MaxLen2, O); -build([$\n], Acc, MaxLen, #options{chomp=true}) -> + build(Cs, [S | Acc], MaxLen2, O); +build([$\n], Acc, MaxLen, #options{chomp = true}) -> %% trailing \n, ignore {lists:reverse(Acc), MaxLen}; -build([$\n|Cs], Acc, MaxLen, O) -> - build(Cs, [$\n|Acc], MaxLen - 1, O); -build([$\t|Cs], Acc, MaxLen, O) -> - build(Cs, [$\t|Acc], MaxLen - 1, O); -build([C|Cs], Acc, MaxLen, O) -> - build(Cs, [C|Acc], MaxLen - 1, O); +build([$\n | Cs], Acc, MaxLen, O) -> + build(Cs, [$\n | Acc], MaxLen - 1, O); +build([$\t | Cs], Acc, MaxLen, O) -> + build(Cs, [$\t | Acc], MaxLen - 1, O); +build([C | Cs], Acc, MaxLen, O) -> + build(Cs, [C | Acc], MaxLen - 1, O); build([], Acc, MaxLen, _O) -> {lists:reverse(Acc), MaxLen}. -build2([{C,As,F,Ad,P,Pad,Enc}|Cs], Count, MaxLen) -> +build2([{C, As, F, Ad, P, Pad, Enc} | Cs], Count, MaxLen) -> {S, Len} = control2(C, As, F, Ad, P, Pad, Enc, MaxLen div Count), - [S|build2(Cs, Count - 1, MaxLen - Len)]; -build2([C|Cs], Count, MaxLen) -> - [C|build2(Cs, Count, MaxLen)]; -build2([], _, _) -> []. + [S | build2(Cs, Count - 1, MaxLen - Len)]; +build2([C | Cs], Count, MaxLen) -> + [C | build2(Cs, Count, MaxLen)]; +build2([], _, _) -> + []. %% control(FormatChar, [Argument], FieldWidth, Adjust, Precision, PadChar, %% Indentation) -> [Char] @@ -187,20 +192,26 @@ control($b, [A], F, Adj, P, Pad, _Enc, L) when is_integer(A) -> control($B, [A], F, Adj, P, Pad, _Enc, L) when is_integer(A) -> Res = unprefixed_integer(A, F, Adj, base(P), Pad, false), {Res, L - lists:flatlength(Res)}; -control($x, [A,Prefix], F, Adj, P, Pad, _Enc, L) when is_integer(A), - is_atom(Prefix) -> +control($x, [A, Prefix], F, Adj, P, Pad, _Enc, L) when + is_integer(A), + is_atom(Prefix) +-> Res = prefixed_integer(A, F, Adj, base(P), Pad, atom_to_list(Prefix), true), {Res, L - lists:flatlength(Res)}; -control($x, [A,Prefix], F, Adj, P, Pad, _Enc, L) when is_integer(A) -> - true = io_lib:deep_char_list(Prefix), %Check if Prefix a character list +control($x, [A, Prefix], F, Adj, P, Pad, _Enc, L) when is_integer(A) -> + %Check if Prefix a character list + true = io_lib:deep_char_list(Prefix), Res = prefixed_integer(A, F, Adj, base(P), Pad, Prefix, true), {Res, L - lists:flatlength(Res)}; -control($X, [A,Prefix], F, Adj, P, Pad, _Enc, L) when is_integer(A), - is_atom(Prefix) -> +control($X, [A, Prefix], F, Adj, P, Pad, _Enc, L) when + is_integer(A), + is_atom(Prefix) +-> Res = prefixed_integer(A, F, Adj, base(P), Pad, atom_to_list(Prefix), false), {Res, L - lists:flatlength(Res)}; -control($X, [A,Prefix], F, Adj, P, Pad, _Enc, L) when is_integer(A) -> - true = io_lib:deep_char_list(Prefix), %Check if Prefix a character list +control($X, [A, Prefix], F, Adj, P, Pad, _Enc, L) when is_integer(A) -> + %Check if Prefix a character list + true = io_lib:deep_char_list(Prefix), Res = prefixed_integer(A, F, Adj, base(P), Pad, Prefix, false), {Res, L - lists:flatlength(Res)}; control($+, [A], F, Adj, P, Pad, _Enc, L) when is_integer(A) -> @@ -241,11 +252,11 @@ control2($w, [A], F, Adj, P, Pad, _Enc, L) -> control2($p, [A], _F, _Adj, _P, _Pad, _Enc, L) -> Term = couch_log_trunc_io:fprint(A, L, [{lists_as_strings, true}]), {Term, lists:flatlength(Term)}; -control2($W, [A,Depth], F, Adj, P, Pad, _Enc, L) when is_integer(Depth) -> +control2($W, [A, Depth], F, Adj, P, Pad, _Enc, L) when is_integer(Depth) -> Term = couch_log_trunc_io:fprint(A, L, [{depth, Depth}, {lists_as_strings, false}]), Res = term(Term, F, Adj, P, Pad), {Res, lists:flatlength(Res)}; -control2($P, [A,Depth], _F, _Adj, _P, _Pad, _Enc, L) when is_integer(Depth) -> +control2($P, [A, Depth], _F, _Adj, _P, _Pad, _Enc, L) when is_integer(Depth) -> Term = couch_log_trunc_io:fprint(A, L, [{depth, Depth}, {lists_as_strings, true}]), {Term, lists:flatlength(Term)}; control2($s, [L0], F, Adj, P, Pad, latin1, L) -> @@ -261,18 +272,18 @@ control2($r, [R], F, Adj, P, Pad, _Enc, _L) -> Res = string(List, F, Adj, P, Pad), {Res, lists:flatlength(Res)}. -iolist_to_chars([C|Cs]) when is_integer(C), C >= $\000, C =< $\377 -> +iolist_to_chars([C | Cs]) when is_integer(C), C >= $\000, C =< $\377 -> [C | iolist_to_chars(Cs)]; -iolist_to_chars([I|Cs]) -> +iolist_to_chars([I | Cs]) -> [iolist_to_chars(I) | iolist_to_chars(Cs)]; iolist_to_chars([]) -> []; iolist_to_chars(B) when is_binary(B) -> binary_to_list(B). -cdata_to_chars([C|Cs]) when is_integer(C), C >= $\000 -> +cdata_to_chars([C | Cs]) when is_integer(C), C >= $\000 -> [C | cdata_to_chars(Cs)]; -cdata_to_chars([I|Cs]) -> +cdata_to_chars([I | Cs]) -> [cdata_to_chars(I) | cdata_to_chars(Cs)]; cdata_to_chars([]) -> []; @@ -284,12 +295,12 @@ cdata_to_chars(B) when is_binary(B) -> make_options([], Options) -> Options; -make_options([{chomp, Bool}|T], Options) when is_boolean(Bool) -> - make_options(T, Options#options{chomp=Bool}). +make_options([{chomp, Bool} | T], Options) when is_boolean(Bool) -> + make_options(T, Options#options{chomp = Bool}). -ifdef(UNICODE_AS_BINARIES). uniconv(C) -> - unicode:characters_to_binary(C,unicode). + unicode:characters_to_binary(C, unicode). -else. uniconv(C) -> C. @@ -305,21 +316,28 @@ base(B) when is_integer(B) -> %% Adjust the characters within the field if length less than Max padding %% with PadChar. -term(T, none, _Adj, none, _Pad) -> T; -term(T, none, Adj, P, Pad) -> term(T, P, Adj, P, Pad); +term(T, none, _Adj, none, _Pad) -> + T; +term(T, none, Adj, P, Pad) -> + term(T, P, Adj, P, Pad); term(T, F, Adj, P0, Pad) -> L = lists:flatlength(T), - P = case P0 of none -> erlang:min(L, F); _ -> P0 end, + P = + case P0 of + none -> erlang:min(L, F); + _ -> P0 + end, if L > P -> - adjust(chars($*, P), chars(Pad, F-P), Adj); + adjust(chars($*, P), chars(Pad, F - P), Adj); F >= P -> - adjust(T, chars(Pad, F-L), Adj) + adjust(T, chars(Pad, F - L), Adj) end. %% fwrite_e(Float, Field, Adjust, Precision, PadChar) -fwrite_e(Fl, none, Adj, none, Pad) -> %Default values +%Default values +fwrite_e(Fl, none, Adj, none, Pad) -> fwrite_e(Fl, none, Adj, 6, Pad); fwrite_e(Fl, none, _Adj, P, _Pad) when P >= 2 -> float_e(Fl, float_data(Fl), P); @@ -328,12 +346,13 @@ fwrite_e(Fl, F, Adj, none, Pad) -> fwrite_e(Fl, F, Adj, P, Pad) when P >= 2 -> term(float_e(Fl, float_data(Fl), P), F, Adj, F, Pad). -float_e(Fl, Fd, P) when Fl < 0.0 -> %Negative numbers - [$-|float_e(-Fl, Fd, P)]; -float_e(_Fl, {Ds,E}, P) -> - case float_man(Ds, 1, P-1) of - {[$0|Fs],true} -> [[$1|Fs]|float_exp(E)]; - {Fs,false} -> [Fs|float_exp(E-1)] +%Negative numbers +float_e(Fl, Fd, P) when Fl < 0.0 -> + [$- | float_e(-Fl, Fd, P)]; +float_e(_Fl, {Ds, E}, P) -> + case float_man(Ds, 1, P - 1) of + {[$0 | Fs], true} -> [[$1 | Fs] | float_exp(E)]; + {Fs, false} -> [Fs | float_exp(E - 1)] end. %% float_man([Digit], Icount, Dcount) -> {[Chars],CarryFlag}. @@ -342,38 +361,43 @@ float_e(_Fl, {Ds,E}, P) -> %% caller decide what to do at top. float_man(Ds, 0, Dc) -> - {Cs,C} = float_man(Ds, Dc), - {[$.|Cs],C}; -float_man([D|Ds], I, Dc) -> - case float_man(Ds, I-1, Dc) of - {Cs,true} when D =:= $9 -> {[$0|Cs],true}; - {Cs,true} -> {[D+1|Cs],false}; - {Cs,false} -> {[D|Cs],false} + {Cs, C} = float_man(Ds, Dc), + {[$. | Cs], C}; +float_man([D | Ds], I, Dc) -> + case float_man(Ds, I - 1, Dc) of + {Cs, true} when D =:= $9 -> {[$0 | Cs], true}; + {Cs, true} -> {[D + 1 | Cs], false}; + {Cs, false} -> {[D | Cs], false} end; -float_man([], I, Dc) -> %Pad with 0's - {string:chars($0, I, [$.|string:chars($0, Dc)]),false}. - -float_man([D|_], 0) when D >= $5 -> {[],true}; -float_man([_|_], 0) -> {[],false}; -float_man([D|Ds], Dc) -> - case float_man(Ds, Dc-1) of - {Cs,true} when D =:= $9 -> {[$0|Cs],true}; - {Cs,true} -> {[D+1|Cs],false}; - {Cs,false} -> {[D|Cs],false} +%Pad with 0's +float_man([], I, Dc) -> + {string:chars($0, I, [$. | string:chars($0, Dc)]), false}. + +float_man([D | _], 0) when D >= $5 -> {[], true}; +float_man([_ | _], 0) -> + {[], false}; +float_man([D | Ds], Dc) -> + case float_man(Ds, Dc - 1) of + {Cs, true} when D =:= $9 -> {[$0 | Cs], true}; + {Cs, true} -> {[D + 1 | Cs], false}; + {Cs, false} -> {[D | Cs], false} end; -float_man([], Dc) -> {string:chars($0, Dc),false}. %Pad with 0's +%Pad with 0's +float_man([], Dc) -> + {string:chars($0, Dc), false}. %% float_exp(Exponent) -> [Char]. %% Generate the exponent of a floating point number. Always include sign. float_exp(E) when E >= 0 -> - [$e,$+|integer_to_list(E)]; + [$e, $+ | integer_to_list(E)]; float_exp(E) -> - [$e|integer_to_list(E)]. + [$e | integer_to_list(E)]. %% fwrite_f(FloatData, Field, Adjust, Precision, PadChar) -fwrite_f(Fl, none, Adj, none, Pad) -> %Default values +%Default values +fwrite_f(Fl, none, Adj, none, Pad) -> fwrite_f(Fl, none, Adj, 6, Pad); fwrite_f(Fl, none, _Adj, P, _Pad) when P >= 1 -> float_f(Fl, float_data(Fl), P); @@ -383,13 +407,15 @@ fwrite_f(Fl, F, Adj, P, Pad) when P >= 1 -> term(float_f(Fl, float_data(Fl), P), F, Adj, F, Pad). float_f(Fl, Fd, P) when Fl < 0.0 -> - [$-|float_f(-Fl, Fd, P)]; -float_f(Fl, {Ds,E}, P) when E =< 0 -> - float_f(Fl, {string:chars($0, -E+1, Ds),1}, P); %Prepend enough 0's -float_f(_Fl, {Ds,E}, P) -> + [$- | float_f(-Fl, Fd, P)]; +float_f(Fl, {Ds, E}, P) when E =< 0 -> + %Prepend enough 0's + float_f(Fl, {string:chars($0, -E + 1, Ds), 1}, P); +float_f(_Fl, {Ds, E}, P) -> case float_man(Ds, E, P) of - {Fs,true} -> "1" ++ Fs; %Handle carry - {Fs,false} -> Fs + %Handle carry + {Fs, true} -> "1" ++ Fs; + {Fs, false} -> Fs end. %% float_data([FloatChar]) -> {[Digit],Exponent} @@ -397,11 +423,11 @@ float_f(_Fl, {Ds,E}, P) -> float_data(Fl) -> float_data(float_to_list(Fl), []). -float_data([$e|E], Ds) -> - {lists:reverse(Ds),list_to_integer(E)+1}; -float_data([D|Cs], Ds) when D >= $0, D =< $9 -> - float_data(Cs, [D|Ds]); -float_data([_|Cs], Ds) -> +float_data([$e | E], Ds) -> + {lists:reverse(Ds), list_to_integer(E) + 1}; +float_data([D | Cs], Ds) when D >= $0, D =< $9 -> + float_data(Cs, [D | Ds]); +float_data([_ | Cs], Ds) -> float_data(Cs, Ds). %% fwrite_g(Float, Field, Adjust, Precision, PadChar) @@ -413,83 +439,98 @@ fwrite_g(Fl, F, Adj, none, Pad) -> fwrite_g(Fl, F, Adj, 6, Pad); fwrite_g(Fl, F, Adj, P, Pad) when P >= 1 -> A = abs(Fl), - E = if A < 1.0e-1 -> -2; - A < 1.0e0 -> -1; - A < 1.0e1 -> 0; - A < 1.0e2 -> 1; - A < 1.0e3 -> 2; - A < 1.0e4 -> 3; - true -> fwrite_f - end, - if P =< 1, E =:= -1; - P-1 > E, E >= -1 -> - fwrite_f(Fl, F, Adj, P-1-E, Pad); - P =< 1 -> - fwrite_e(Fl, F, Adj, 2, Pad); - true -> - fwrite_e(Fl, F, Adj, P, Pad) + E = + if + A < 1.0e-1 -> -2; + A < 1.0e0 -> -1; + A < 1.0e1 -> 0; + A < 1.0e2 -> 1; + A < 1.0e3 -> 2; + A < 1.0e4 -> 3; + true -> fwrite_f + end, + if + P =< 1, E =:= -1; + P - 1 > E, E >= -1 -> + fwrite_f(Fl, F, Adj, P - 1 - E, Pad); + P =< 1 -> + fwrite_e(Fl, F, Adj, 2, Pad); + true -> + fwrite_e(Fl, F, Adj, P, Pad) end. - %% string(String, Field, Adjust, Precision, PadChar) -string(S, none, _Adj, none, _Pad) -> S; +string(S, none, _Adj, none, _Pad) -> + S; string(S, F, Adj, none, Pad) -> string_field(S, F, Adj, lists:flatlength(S), Pad); string(S, none, _Adj, P, Pad) -> string_field(S, P, left, lists:flatlength(S), Pad); string(S, F, Adj, P, Pad) when F >= P -> N = lists:flatlength(S), - if F > P -> - if N > P -> - adjust(flat_trunc(S, P), chars(Pad, F-P), Adj); + if + F > P -> + if + N > P -> + adjust(flat_trunc(S, P), chars(Pad, F - P), Adj); N < P -> - adjust([S|chars(Pad, P-N)], chars(Pad, F-P), Adj); - true -> % N == P - adjust(S, chars(Pad, F-P), Adj) + adjust([S | chars(Pad, P - N)], chars(Pad, F - P), Adj); + % N == P + true -> + adjust(S, chars(Pad, F - P), Adj) end; - true -> % F == P - string_field(S, F, Adj, N, Pad) + % F == P + true -> + string_field(S, F, Adj, N, Pad) end. string_field(S, F, _Adj, N, _Pad) when N > F -> flat_trunc(S, F); string_field(S, F, Adj, N, Pad) when N < F -> - adjust(S, chars(Pad, F-N), Adj); -string_field(S, _, _, _, _) -> % N == F + adjust(S, chars(Pad, F - N), Adj); +% N == F +string_field(S, _, _, _, _) -> S. %% unprefixed_integer(Int, Field, Adjust, Base, PadChar, Lowercase) %% -> [Char]. -unprefixed_integer(Int, F, Adj, Base, Pad, Lowercase) - when Base >= 2, Base =< 1+$Z-$A+10 -> - if Int < 0 -> +unprefixed_integer(Int, F, Adj, Base, Pad, Lowercase) when + Base >= 2, Base =< 1 + $Z - $A + 10 +-> + if + Int < 0 -> S = cond_lowercase(erlang:integer_to_list(-Int, Base), Lowercase), - term([$-|S], F, Adj, none, Pad); - true -> - S = cond_lowercase(erlang:integer_to_list(Int, Base), Lowercase), - term(S, F, Adj, none, Pad) + term([$- | S], F, Adj, none, Pad); + true -> + S = cond_lowercase(erlang:integer_to_list(Int, Base), Lowercase), + term(S, F, Adj, none, Pad) end. %% prefixed_integer(Int, Field, Adjust, Base, PadChar, Prefix, Lowercase) %% -> [Char]. -prefixed_integer(Int, F, Adj, Base, Pad, Prefix, Lowercase) - when Base >= 2, Base =< 1+$Z-$A+10 -> - if Int < 0 -> +prefixed_integer(Int, F, Adj, Base, Pad, Prefix, Lowercase) when + Base >= 2, Base =< 1 + $Z - $A + 10 +-> + if + Int < 0 -> S = cond_lowercase(erlang:integer_to_list(-Int, Base), Lowercase), - term([$-,Prefix|S], F, Adj, none, Pad); - true -> - S = cond_lowercase(erlang:integer_to_list(Int, Base), Lowercase), - term([Prefix|S], F, Adj, none, Pad) + term([$-, Prefix | S], F, Adj, none, Pad); + true -> + S = cond_lowercase(erlang:integer_to_list(Int, Base), Lowercase), + term([Prefix | S], F, Adj, none, Pad) end. %% char(Char, Field, Adjust, Precision, PadChar) -> [Char]. -char(C, none, _Adj, none, _Pad) -> [C]; -char(C, F, _Adj, none, _Pad) -> chars(C, F); -char(C, none, _Adj, P, _Pad) -> chars(C, P); +char(C, none, _Adj, none, _Pad) -> + [C]; +char(C, F, _Adj, none, _Pad) -> + chars(C, F); +char(C, none, _Adj, P, _Pad) -> + chars(C, P); char(C, F, Adj, P, Pad) when F >= P -> adjust(chars(C, P), chars(Pad, F - P), Adj). @@ -503,8 +544,8 @@ newline(F, right, _P, _Pad) -> chars($\n, F). %% adjust(Data, [], _) -> Data; -adjust(Data, Pad, left) -> [Data|Pad]; -adjust(Data, Pad, right) -> [Pad|Data]. +adjust(Data, Pad, left) -> [Data | Pad]; +adjust(Data, Pad, right) -> [Pad | Data]. %% Flatten and truncate a deep list to at most N elements. flat_trunc(List, N) when is_integer(N), N >= 0 -> @@ -512,8 +553,8 @@ flat_trunc(List, N) when is_integer(N), N >= 0 -> flat_trunc(L, 0, R) when is_list(L) -> lists:reverse(R); -flat_trunc([H|T], N, R) -> - flat_trunc(T, N-1, [H|R]); +flat_trunc([H | T], N, R) -> + flat_trunc(T, N - 1, [H | R]); flat_trunc([], _, R) -> lists:reverse(R). @@ -524,15 +565,15 @@ chars(_C, 0) -> chars(C, 1) -> [C]; chars(C, 2) -> - [C,C]; + [C, C]; chars(C, 3) -> - [C,C,C]; + [C, C, C]; chars(C, N) when is_integer(N), (N band 1) =:= 0 -> S = chars(C, N bsr 1), - [S|S]; + [S | S]; chars(C, N) when is_integer(N) -> S = chars(C, N bsr 1), - [C,S|S]. + [C, S | S]. %chars(C, N, Tail) -> % [chars(C, N)|Tail]. @@ -541,12 +582,12 @@ chars(C, N) when is_integer(N) -> cond_lowercase(String, true) -> lowercase(String); -cond_lowercase(String,false) -> +cond_lowercase(String, false) -> String. -lowercase([H|T]) when is_integer(H), H >= $A, H =< $Z -> - [(H-$A+$a)|lowercase(T)]; -lowercase([H|T]) -> - [H|lowercase(T)]; +lowercase([H | T]) when is_integer(H), H >= $A, H =< $Z -> + [(H - $A + $a) | lowercase(T)]; +lowercase([H | T]) -> + [H | lowercase(T)]; lowercase([]) -> - [].
\ No newline at end of file + []. diff --git a/src/couch_log/src/couch_log_util.erl b/src/couch_log/src/couch_log_util.erl index c8b8e54ea..8be11e12d 100644 --- a/src/couch_log/src/couch_log_util.erl +++ b/src/couch_log/src/couch_log_util.erl @@ -12,7 +12,6 @@ -module(couch_log_util). - -export([ should_log/1, iso8601_timestamp/0, @@ -25,26 +24,21 @@ string_p/1 ]). - -include("couch_log.hrl"). - -spec should_log(#log_entry{} | atom()) -> boolean(). should_log(#log_entry{level = Level}) -> should_log(Level); - should_log(Level) -> level_to_integer(Level) >= couch_log_config:get(level_int). - -spec iso8601_timestamp() -> string(). iso8601_timestamp() -> - {_,_,Micro} = Now = os:timestamp(), - {{Year,Month,Date},{Hour,Minute,Second}} = calendar:now_to_datetime(Now), + {_, _, Micro} = Now = os:timestamp(), + {{Year, Month, Date}, {Hour, Minute, Second}} = calendar:now_to_datetime(Now), Format = "~4.10.0B-~2.10.0B-~2.10.0BT~2.10.0B:~2.10.0B:~2.10.0B.~6.10.0BZ", io_lib:format(Format, [Year, Month, Date, Hour, Minute, Second, Micro]). - -spec get_msg_id() -> string(). get_msg_id() -> case erlang:get(nonce) of @@ -52,78 +46,73 @@ get_msg_id() -> MsgId -> MsgId end. - -spec level_to_integer(atom() | string() | integer()) -> integer(). level_to_integer(L) when L >= 0, L =< 9 -> L; -level_to_integer(debug) -> 1; -level_to_integer(info) -> 2; -level_to_integer(notice) -> 3; -level_to_integer(warning) -> 4; -level_to_integer(warn) -> 4; -level_to_integer(error) -> 5; -level_to_integer(err) -> 5; -level_to_integer(critical) -> 6; -level_to_integer(crit) -> 6; -level_to_integer(alert) -> 7; -level_to_integer(emergency) -> 8; -level_to_integer(emerg) -> 8; -level_to_integer(none) -> 9; -level_to_integer("debug") -> 1; -level_to_integer("info") -> 2; -level_to_integer("notice") -> 3; -level_to_integer("warning") -> 4; -level_to_integer("warn") -> 4; -level_to_integer("error") -> 5; -level_to_integer("err") -> 5; -level_to_integer("critical") -> 6; -level_to_integer("crit") -> 6; -level_to_integer("alert") -> 7; -level_to_integer("emergency") -> 8; -level_to_integer("emerg") -> 8; -level_to_integer("none") -> 9; -level_to_integer("1") -> 1; -level_to_integer("2") -> 2; -level_to_integer("3") -> 3; -level_to_integer("4") -> 4; -level_to_integer("5") -> 5; -level_to_integer("6") -> 6; -level_to_integer("7") -> 7; -level_to_integer("8") -> 8; -level_to_integer("9") -> 9. - +level_to_integer(debug) -> 1; +level_to_integer(info) -> 2; +level_to_integer(notice) -> 3; +level_to_integer(warning) -> 4; +level_to_integer(warn) -> 4; +level_to_integer(error) -> 5; +level_to_integer(err) -> 5; +level_to_integer(critical) -> 6; +level_to_integer(crit) -> 6; +level_to_integer(alert) -> 7; +level_to_integer(emergency) -> 8; +level_to_integer(emerg) -> 8; +level_to_integer(none) -> 9; +level_to_integer("debug") -> 1; +level_to_integer("info") -> 2; +level_to_integer("notice") -> 3; +level_to_integer("warning") -> 4; +level_to_integer("warn") -> 4; +level_to_integer("error") -> 5; +level_to_integer("err") -> 5; +level_to_integer("critical") -> 6; +level_to_integer("crit") -> 6; +level_to_integer("alert") -> 7; +level_to_integer("emergency") -> 8; +level_to_integer("emerg") -> 8; +level_to_integer("none") -> 9; +level_to_integer("1") -> 1; +level_to_integer("2") -> 2; +level_to_integer("3") -> 3; +level_to_integer("4") -> 4; +level_to_integer("5") -> 5; +level_to_integer("6") -> 6; +level_to_integer("7") -> 7; +level_to_integer("8") -> 8; +level_to_integer("9") -> 9. -spec level_to_atom(atom() | string() | integer()) -> atom(). -level_to_atom(L) when is_atom(L) -> L; -level_to_atom("1") -> debug; -level_to_atom("debug") -> debug; -level_to_atom("2") -> info; -level_to_atom("info") -> info; -level_to_atom("3") -> notice; -level_to_atom("notice") -> notice; -level_to_atom("4") -> warning; -level_to_atom("warning") -> warning; -level_to_atom("warn") -> warning; -level_to_atom("5") -> error; -level_to_atom("error") -> error; -level_to_atom("err") -> error; -level_to_atom("6") -> critical; -level_to_atom("critical") -> critical; -level_to_atom("crit") -> critical; -level_to_atom("7") -> alert; -level_to_atom("alert") -> alert; -level_to_atom("8") -> emergency; -level_to_atom("emergency") -> emergency; -level_to_atom("emerg") -> emergency; -level_to_atom("9") -> none; -level_to_atom("none") -> none; +level_to_atom(L) when is_atom(L) -> L; +level_to_atom("1") -> debug; +level_to_atom("debug") -> debug; +level_to_atom("2") -> info; +level_to_atom("info") -> info; +level_to_atom("3") -> notice; +level_to_atom("notice") -> notice; +level_to_atom("4") -> warning; +level_to_atom("warning") -> warning; +level_to_atom("warn") -> warning; +level_to_atom("5") -> error; +level_to_atom("error") -> error; +level_to_atom("err") -> error; +level_to_atom("6") -> critical; +level_to_atom("critical") -> critical; +level_to_atom("crit") -> critical; +level_to_atom("7") -> alert; +level_to_atom("alert") -> alert; +level_to_atom("8") -> emergency; +level_to_atom("emergency") -> emergency; +level_to_atom("emerg") -> emergency; +level_to_atom("9") -> none; +level_to_atom("none") -> none; level_to_atom(V) when is_integer(V) -> level_to_atom(integer_to_list(V)); -level_to_atom(V) when is_list(V) -> info. - - -level_to_string(L) when is_atom(L) -> atom_to_list(L); -level_to_string(L) -> atom_to_list(level_to_atom(L)). - +level_to_atom(V) when is_list(V) -> info. +level_to_string(L) when is_atom(L) -> atom_to_list(L); +level_to_string(L) -> atom_to_list(level_to_atom(L)). % From error_logger_file_h via lager_stdlib.erl string_p([]) -> @@ -131,19 +120,28 @@ string_p([]) -> string_p(Term) -> string_p1(Term). -string_p1([H|T]) when is_integer(H), H >= $\s, H < 256 -> +string_p1([H | T]) when is_integer(H), H >= $\s, H < 256 -> + string_p1(T); +string_p1([$\n | T]) -> + string_p1(T); +string_p1([$\r | T]) -> + string_p1(T); +string_p1([$\t | T]) -> + string_p1(T); +string_p1([$\v | T]) -> + string_p1(T); +string_p1([$\b | T]) -> + string_p1(T); +string_p1([$\f | T]) -> + string_p1(T); +string_p1([$\e | T]) -> string_p1(T); -string_p1([$\n|T]) -> string_p1(T); -string_p1([$\r|T]) -> string_p1(T); -string_p1([$\t|T]) -> string_p1(T); -string_p1([$\v|T]) -> string_p1(T); -string_p1([$\b|T]) -> string_p1(T); -string_p1([$\f|T]) -> string_p1(T); -string_p1([$\e|T]) -> string_p1(T); -string_p1([H|T]) when is_list(H) -> +string_p1([H | T]) when is_list(H) -> case string_p1(H) of true -> string_p1(T); - _ -> false + _ -> false end; -string_p1([]) -> true; -string_p1(_) -> false. +string_p1([]) -> + true; +string_p1(_) -> + false. diff --git a/src/couch_log/src/couch_log_writer.erl b/src/couch_log/src/couch_log_writer.erl index 5e28a0775..18bb557ae 100644 --- a/src/couch_log/src/couch_log_writer.erl +++ b/src/couch_log/src/couch_log_writer.erl @@ -13,28 +13,22 @@ % @doc Modules wishing to handle writing log % messages should implement this behavior. - -module(couch_log_writer). - -export([ init/0, terminate/2, write/2 ]). - -include("couch_log.hrl"). - -define(DEFAULT_WRITER, couch_log_writer_stderr). - --callback init() -> {ok, State::term()}. --callback terminate(Reason::term(), State::term()) -> ok. --callback write(LogEntry::#log_entry{}, State::term()) -> - {ok, NewState::term()}. - +-callback init() -> {ok, State :: term()}. +-callback terminate(Reason :: term(), State :: term()) -> ok. +-callback write(LogEntry :: #log_entry{}, State :: term()) -> + {ok, NewState :: term()}. -spec init() -> {atom(), term()}. init() -> @@ -42,18 +36,15 @@ init() -> {ok, St} = Writer:init(), {Writer, St}. - -spec terminate(term(), {atom(), term()}) -> ok. terminate(Reason, {Writer, St}) -> ok = Writer:terminate(Reason, St). - -spec write(#log_entry{}, {atom(), term()}) -> {atom(), term()}. write(Entry, {Writer, St}) -> {ok, NewSt} = Writer:write(Entry, St), {Writer, NewSt}. - get_writer_mod() -> WriterStr = config:get("log", "writer", "stderr"), ModName1 = to_atom("couch_log_writer_" ++ WriterStr), @@ -70,14 +61,13 @@ get_writer_mod() -> end end. - to_atom(Str) -> try list_to_existing_atom(Str) of Atom -> Atom - catch _:_ -> - undefined + catch + _:_ -> + undefined end. - mod_exists(ModName) -> code:which(ModName) /= non_existing. diff --git a/src/couch_log/src/couch_log_writer_file.erl b/src/couch_log/src/couch_log_writer_file.erl index 1fe35a8ab..9b7255050 100644 --- a/src/couch_log/src/couch_log_writer_file.erl +++ b/src/couch_log/src/couch_log_writer_file.erl @@ -13,18 +13,15 @@ -module(couch_log_writer_file). -behaviour(couch_log_writer). - -export([ init/0, terminate/2, write/2 ]). - -include_lib("kernel/include/file.hrl"). -include("couch_log.hrl"). - -record(st, { file_path, fd, @@ -32,17 +29,14 @@ last_check }). - -define(CHECK_INTERVAL, 30000000). - -ifdef(TEST). -export([ maybe_reopen/1 ]). -endif. - init() -> FilePath = config:get("log", "file", "./couch.log"), Opts = [append, raw] ++ buffer_opt(), @@ -69,14 +63,12 @@ init() -> EnsureDirError end. - terminate(_, St) -> % Apparently delayed_write can require two closes file:close(St#st.fd), file:close(St#st.fd), ok. - write(Entry, St) -> {ok, NewSt} = maybe_reopen(St), #log_entry{ @@ -99,7 +91,6 @@ write(Entry, St) -> ok = file:write(NewSt#st.fd, [Data, Msg, "\n"]), {ok, NewSt}. - buffer_opt() -> WriteBuffer = config:get_integer("log", "write_buffer", 0), WriteDelay = config:get_integer("log", "write_delay", 0), @@ -110,7 +101,6 @@ buffer_opt() -> [] end. - maybe_reopen(St) -> #st{ last_check = LastCheck @@ -121,7 +111,6 @@ maybe_reopen(St) -> false -> {ok, St} end. - reopen(St) -> case file:read_file_info(St#st.file_path) of {ok, FInfo} -> diff --git a/src/couch_log/src/couch_log_writer_journald.erl b/src/couch_log/src/couch_log_writer_journald.erl index 02a9c6900..c2bdd940c 100644 --- a/src/couch_log/src/couch_log_writer_journald.erl +++ b/src/couch_log/src/couch_log_writer_journald.erl @@ -13,25 +13,20 @@ -module(couch_log_writer_journald). -behaviour(couch_log_writer). - -export([ init/0, terminate/2, write/2 ]). - -include("couch_log.hrl"). - init() -> {ok, nil}. - terminate(_, _St) -> ok. - write(Entry, St) -> #log_entry{ level = Level, @@ -51,19 +46,18 @@ write(Entry, St) -> io:format(standard_error, [Data, Msg, "\n"], []), {ok, St}. - % log level mapping from sd-daemon(3) % https://www.freedesktop.org/software/systemd/man/sd-daemon.html -spec level_for_journald(atom()) -> integer(). level_for_journald(Level) when is_atom(Level) -> case Level of - debug -> 7; - info -> 6; - notice -> 5; - warning -> 4; - error -> 3; - critical -> 2; - alert -> 1; - emergency -> 0; - _ -> 3 + debug -> 7; + info -> 6; + notice -> 5; + warning -> 4; + error -> 3; + critical -> 2; + alert -> 1; + emergency -> 0; + _ -> 3 end. diff --git a/src/couch_log/src/couch_log_writer_stderr.erl b/src/couch_log/src/couch_log_writer_stderr.erl index 7c5fc6ca0..01e350971 100644 --- a/src/couch_log/src/couch_log_writer_stderr.erl +++ b/src/couch_log/src/couch_log_writer_stderr.erl @@ -13,25 +13,20 @@ -module(couch_log_writer_stderr). -behaviour(couch_log_writer). - -export([ init/0, terminate/2, write/2 ]). - -include("couch_log.hrl"). - init() -> {ok, nil}. - terminate(_, _St) -> ok. - write(Entry, St) -> #log_entry{ level = Level, diff --git a/src/couch_log/src/couch_log_writer_syslog.erl b/src/couch_log/src/couch_log_writer_syslog.erl index e3a6fc4b6..b95cf018c 100644 --- a/src/couch_log/src/couch_log_writer_syslog.erl +++ b/src/couch_log/src/couch_log_writer_syslog.erl @@ -13,17 +13,14 @@ -module(couch_log_writer_syslog). -behavior(couch_log_writer). - -export([ init/0, terminate/2, write/2 ]). - -include("couch_log.hrl"). - -record(st, { socket, host, @@ -34,10 +31,8 @@ facility }). - -define(SYSLOG_VERSION, 1). - -ifdef(TEST). -export([ get_facility/1, @@ -45,20 +40,20 @@ ]). -endif. - init() -> {ok, Socket} = gen_udp:open(0), - Host = case config:get("log", "syslog_host") of - undefined -> - undefined; - SysLogHost -> - case inet:getaddr(SysLogHost, inet) of - {ok, Address} -> - Address; - _ -> - undefined - end + Host = + case config:get("log", "syslog_host") of + undefined -> + undefined; + SysLogHost -> + case inet:getaddr(SysLogHost, inet) of + {ok, Address} -> + Address; + _ -> + undefined + end end, {ok, #st{ @@ -71,11 +66,9 @@ init() -> facility = get_facility(config:get("log", "syslog_facility", "local2")) }}. - terminate(_Reason, St) -> gen_udp:close(St#st.socket). - write(Entry, St) -> #log_entry{ level = Level, @@ -98,10 +91,8 @@ write(Entry, St) -> ok = send(St, [Pre, Msg, $\n]), {ok, St}. - -send(#st{host=undefined}, Packet) -> +send(#st{host = undefined}, Packet) -> io:format(standard_error, "~s", [Packet]); - send(St, Packet) -> #st{ socket = Socket, @@ -110,53 +101,101 @@ send(St, Packet) -> } = St, gen_udp:send(Socket, Host, Port, Packet). - get_facility(Name) -> - FacId = case Name of - "kern" -> 0; % Kernel messages - "user" -> 1; % Random user-level messages - "mail" -> 2; % Mail system - "daemon" -> 3; % System daemons - "auth" -> 4; % Security/Authorization messages - "syslog" -> 5; % Internal Syslog messages - "lpr" -> 6; % Line printer subsystem - "news" -> 7; % Network news subsystems - "uucp" -> 8; % UUCP subsystem - "clock" -> 9; % Clock daemon - "authpriv" -> 10; % Security/Authorization messages - "ftp" -> 11; % FTP daemon - "ntp" -> 12; % NTP subsystem - "audit" -> 13; % Log audit - "alert" -> 14; % Log alert - "cron" -> 15; % Scheduling daemon - "local0" -> 16; % Local use 0 - "local1" -> 17; % Local use 1 - "local2" -> 18; % Local use 2 - "local3" -> 19; % Local use 3 - "local4" -> 20; % Local use 4 - "local5" -> 21; % Local use 5 - "local6" -> 22; % Local use 6 - "local7" -> 23; % Local use 7 - _ -> - try list_to_integer(Name) of - N when N >= 0, N =< 23 -> N; - _ -> 23 - catch _:_ -> - 23 - end - end, + FacId = + case Name of + % Kernel messages + "kern" -> + 0; + % Random user-level messages + "user" -> + 1; + % Mail system + "mail" -> + 2; + % System daemons + "daemon" -> + 3; + % Security/Authorization messages + "auth" -> + 4; + % Internal Syslog messages + "syslog" -> + 5; + % Line printer subsystem + "lpr" -> + 6; + % Network news subsystems + "news" -> + 7; + % UUCP subsystem + "uucp" -> + 8; + % Clock daemon + "clock" -> + 9; + % Security/Authorization messages + "authpriv" -> + 10; + % FTP daemon + "ftp" -> + 11; + % NTP subsystem + "ntp" -> + 12; + % Log audit + "audit" -> + 13; + % Log alert + "alert" -> + 14; + % Scheduling daemon + "cron" -> + 15; + % Local use 0 + "local0" -> + 16; + % Local use 1 + "local1" -> + 17; + % Local use 2 + "local2" -> + 18; + % Local use 3 + "local3" -> + 19; + % Local use 4 + "local4" -> + 20; + % Local use 5 + "local5" -> + 21; + % Local use 6 + "local6" -> + 22; + % Local use 7 + "local7" -> + 23; + _ -> + try list_to_integer(Name) of + N when N >= 0, N =< 23 -> N; + _ -> 23 + catch + _:_ -> + 23 + end + end, FacId bsl 3. - get_level(Name) when is_atom(Name) -> case Name of - debug -> 7; - info -> 6; - notice -> 5; - warning -> 4; - error -> 3; - critical -> 2; - alert -> 1; - emergency -> 0; - _ -> 3 + debug -> 7; + info -> 6; + notice -> 5; + warning -> 4; + error -> 3; + critical -> 2; + alert -> 1; + emergency -> 0; + _ -> 3 end. diff --git a/src/couch_log/test/eunit/couch_log_config_listener_test.erl b/src/couch_log/test/eunit/couch_log_config_listener_test.erl index 07abae1ff..c955972ff 100644 --- a/src/couch_log/test/eunit/couch_log_config_listener_test.erl +++ b/src/couch_log/test/eunit/couch_log_config_listener_test.erl @@ -12,22 +12,16 @@ -module(couch_log_config_listener_test). - -include_lib("couch_log/include/couch_log.hrl"). -include_lib("eunit/include/eunit.hrl"). -define(TIMEOUT, 1000). couch_log_config_test_() -> - {setup, - fun couch_log_test_util:start/0, - fun couch_log_test_util:stop/1, - [ - fun check_restart_listener/0, - fun check_ignore_non_log/0 - ] - }. - + {setup, fun couch_log_test_util:start/0, fun couch_log_test_util:stop/1, [ + fun check_restart_listener/0, + fun check_ignore_non_log/0 + ]}. check_restart_listener() -> Listener1 = get_listener(), @@ -41,16 +35,20 @@ check_restart_listener() -> receive {'DOWN', Ref, process, _, _} -> ?assertNot(is_process_alive(Listener1)) - after ?TIMEOUT -> - erlang:error({timeout, config_listener_mon_death}) + after ?TIMEOUT -> + erlang:error({timeout, config_listener_mon_death}) end, - NewHandler = test_util:wait(fun() -> - case get_handler() of - not_found -> wait; - Reply -> Reply - end - end, ?TIMEOUT, 20), + NewHandler = test_util:wait( + fun() -> + case get_handler() of + not_found -> wait; + Reply -> Reply + end + end, + ?TIMEOUT, + 20 + ), ?assertEqual(Handler1, NewHandler), Listener2 = get_listener(), @@ -67,7 +65,6 @@ check_ignore_non_log() -> end, ?assertError(config_change_timeout, Run()). - get_handler() -> FoldFun = fun ({config_listener, {couch_log_sup, _}} = H, not_found) -> diff --git a/src/couch_log/test/eunit/couch_log_config_test.erl b/src/couch_log/test/eunit/couch_log_config_test.erl index e47a52bc2..df7cdf977 100644 --- a/src/couch_log/test/eunit/couch_log_config_test.erl +++ b/src/couch_log/test/eunit/couch_log_config_test.erl @@ -12,28 +12,22 @@ -module(couch_log_config_test). - -include_lib("couch_log/include/couch_log.hrl"). -include_lib("eunit/include/eunit.hrl"). -define(T(Name), {atom_to_list(Name), fun Name/0}). couch_log_config_test_() -> - {setup, - fun couch_log_test_util:start/0, - fun couch_log_test_util:stop/1, - [ - ?T(check_level), - ?T(check_max_message_size), - ?T(check_bad_level), - ?T(check_bad_max_message_size), - ?T(check_strip_last_msg), - ?T(check_bad_strip_last_msg), - ?T(check_filter_fields), - ?T(check_bad_filter_fields) - ] - }. - + {setup, fun couch_log_test_util:start/0, fun couch_log_test_util:stop/1, [ + ?T(check_level), + ?T(check_max_message_size), + ?T(check_bad_level), + ?T(check_bad_max_message_size), + ?T(check_strip_last_msg), + ?T(check_bad_strip_last_msg), + ?T(check_filter_fields), + ?T(check_bad_filter_fields) + ]}. check_level() -> % Default level is info @@ -57,7 +51,6 @@ check_level() -> ?assertEqual(2, couch_log_config:get(level_int)) end). - check_max_message_size() -> % Default is 16000 ?assertEqual(16000, couch_log_config:get(max_message_size)), @@ -72,7 +65,6 @@ check_max_message_size() -> ?assertEqual(16000, couch_log_config:get(max_message_size)) end). - check_bad_level() -> % Default level is info ?assertEqual(info, couch_log_config:get(level)), @@ -95,7 +87,6 @@ check_bad_level() -> ?assertEqual(2, couch_log_config:get(level_int)) end). - check_bad_max_message_size() -> % Default level is 16000 ?assertEqual(16000, couch_log_config:get(max_message_size)), @@ -114,7 +105,6 @@ check_bad_max_message_size() -> ?assertEqual(16000, couch_log_config:get(max_message_size)) end). - check_strip_last_msg() -> % Default is true ?assertEqual(true, couch_log_config:get(strip_last_msg)), @@ -147,7 +137,6 @@ check_bad_strip_last_msg() -> ?assertEqual(true, couch_log_config:get(strip_last_msg)) end). - check_filter_fields() -> Default = [pid, registered_name, error_info, messages], ?assertEqual(Default, couch_log_config:get(filter_fields)), diff --git a/src/couch_log/test/eunit/couch_log_error_logger_h_test.erl b/src/couch_log/test/eunit/couch_log_error_logger_h_test.erl index b78598fa4..cb053d611 100644 --- a/src/couch_log/test/eunit/couch_log_error_logger_h_test.erl +++ b/src/couch_log/test/eunit/couch_log_error_logger_h_test.erl @@ -12,23 +12,15 @@ -module(couch_log_error_logger_h_test). - -include_lib("eunit/include/eunit.hrl"). - -define(HANDLER, couch_log_error_logger_h). - couch_log_error_logger_h_test_() -> - {setup, - fun couch_log_test_util:start/0, - fun couch_log_test_util:stop/1, - [ - fun handler_ignores_unknown_messages/0, - fun coverage_test/0 - ] - }. - + {setup, fun couch_log_test_util:start/0, fun couch_log_test_util:stop/1, [ + fun handler_ignores_unknown_messages/0, + fun coverage_test/0 + ]}. handler_ignores_unknown_messages() -> Handlers1 = gen_event:which_handlers(error_logger), @@ -39,7 +31,6 @@ handler_ignores_unknown_messages() -> Handlers2 = gen_event:which_handlers(error_logger), ?assert(lists:member(?HANDLER, Handlers2)). - coverage_test() -> Resp = couch_log_error_logger_h:code_change(foo, bazinga, baz), ?assertEqual({ok, bazinga}, Resp). diff --git a/src/couch_log/test/eunit/couch_log_formatter_test.erl b/src/couch_log/test/eunit/couch_log_formatter_test.erl index 24de346c6..d516c2bc5 100644 --- a/src/couch_log/test/eunit/couch_log_formatter_test.erl +++ b/src/couch_log/test/eunit/couch_log_formatter_test.erl @@ -12,23 +12,19 @@ -module(couch_log_formatter_test). - -include("couch_log.hrl"). -include_lib("eunit/include/eunit.hrl"). - truncate_fmt_test() -> Msg = [0 || _ <- lists:seq(1, 1048576)], Entry = couch_log_formatter:format(info, self(), "~w", [Msg]), ?assert(length(Entry#log_entry.msg) =< 16000). - truncate_test() -> Msg = [0 || _ <- lists:seq(1, 1048576)], Entry = couch_log_formatter:format(info, self(), Msg), ?assert(length(Entry#log_entry.msg) =< 16000). - format_reason_test() -> MsgFmt = "This is a reason: ~r", Reason = {foo, [{x, k, 3}, {c, d, 2}]}, @@ -36,7 +32,6 @@ format_reason_test() -> Formatted = "This is a reason: foo at x:k/3 <= c:d/2", ?assertEqual(Formatted, lists:flatten(Entry#log_entry.msg)). - crashing_formatting_test() -> Pid = self(), Event = { @@ -45,7 +40,8 @@ crashing_formatting_test() -> { Pid, "** Generic server and some stuff", - [a_gen_server, {foo, bar}, server_state] % not enough args! + % not enough args! + [a_gen_server, {foo, bar}, server_state] } }, ?assertMatch( @@ -59,7 +55,6 @@ crashing_formatting_test() -> "Encountered error {error,{badmatch" ]). - gen_server_error_test() -> Pid = self(), Event = { @@ -86,7 +81,6 @@ gen_server_error_test() -> "extra: \\[\\]" ]). - gen_server_error_with_extra_args_test() -> Pid = self(), Event = { @@ -113,7 +107,6 @@ gen_server_error_with_extra_args_test() -> "extra: \\[sad,args\\]" ]). - gen_fsm_error_test() -> Pid = self(), Event = { @@ -122,7 +115,7 @@ gen_fsm_error_test() -> { Pid, "** State machine did a thing", - [a_gen_fsm, {ohai,there}, state_name, curr_state, barf] + [a_gen_fsm, {ohai, there}, state_name, curr_state, barf] } }, ?assertMatch( @@ -140,7 +133,6 @@ gen_fsm_error_test() -> "extra: \\[\\]" ]). - gen_fsm_error_with_extra_args_test() -> Pid = self(), Event = { @@ -149,7 +141,7 @@ gen_fsm_error_with_extra_args_test() -> { Pid, "** State machine did a thing", - [a_gen_fsm, {ohai,there}, state_name, curr_state, barf, sad, args] + [a_gen_fsm, {ohai, there}, state_name, curr_state, barf, sad, args] } }, ?assertMatch( @@ -167,7 +159,6 @@ gen_fsm_error_with_extra_args_test() -> "extra: \\[sad,args\\]" ]). - gen_event_error_test() -> Pid = self(), Event = { @@ -179,7 +170,7 @@ gen_event_error_test() -> [ handler_id, a_gen_event, - {ohai,there}, + {ohai, there}, curr_state, barf ] @@ -199,7 +190,6 @@ gen_event_error_test() -> "state: curr_state" ]). - emulator_error_test() -> Event = { error, @@ -219,7 +209,6 @@ emulator_error_test() -> do_format(Event) ). - normal_error_test() -> Pid = self(), Event = { @@ -243,7 +232,6 @@ normal_error_test() -> do_format(Event) ). - error_report_std_error_test() -> Pid = self(), Event = { @@ -264,7 +252,6 @@ error_report_std_error_test() -> do_format(Event) ). - supervisor_report_test() -> Pid = self(), % A standard supervisor report @@ -382,7 +369,6 @@ supervisor_report_test() -> do_format(Event4) ). - crash_report_test() -> Pid = self(), % A standard crash report @@ -395,11 +381,12 @@ crash_report_test() -> [ [ {pid, list_to_pid("<0.2.0>")}, - {error_info, { - exit, - undef, - [{mod_name, fun_name, [a, b]}] - }} + {error_info, + { + exit, + undef, + [{mod_name, fun_name, [a, b]}] + }} ], [list_to_pid("<0.3.0>"), list_to_pid("<0.4.0>")] ] @@ -429,11 +416,12 @@ crash_report_test() -> [ {pid, list_to_pid("<0.2.0>")}, {registered_name, couch_log_server}, - {error_info, { - exit, - undef, - [{mod_name, fun_name, [a, b]}] - }} + {error_info, + { + exit, + undef, + [{mod_name, fun_name, [a, b]}] + }} ], [list_to_pid("<0.3.0>"), list_to_pid("<0.4.0>")] ] @@ -453,11 +441,12 @@ crash_report_test() -> [ {pid, list_to_pid("<0.2.0>")}, {registered_name, couch_log_server}, - {error_info, { - killed, - undef, - [{mod_name, fun_name, [a, b]}] - }} + {error_info, + { + killed, + undef, + [{mod_name, fun_name, [a, b]}] + }} ], [list_to_pid("<0.3.0>"), list_to_pid("<0.4.0>")] ] @@ -476,11 +465,12 @@ crash_report_test() -> [ [ {pid, list_to_pid("<0.2.0>")}, - {error_info, { - killed, - undef, - [{mod_name, fun_name, [a, b]}] - }}, + {error_info, + { + killed, + undef, + [{mod_name, fun_name, [a, b]}] + }}, {another, entry}, yep ], @@ -492,7 +482,6 @@ crash_report_test() -> "; another: entry, yep" ]). - warning_report_test() -> Pid = self(), % A warning message @@ -532,7 +521,6 @@ warning_report_test() -> do_format(Event2) ). - info_report_test() -> Pid = self(), % An info message @@ -615,7 +603,6 @@ info_report_test() -> do_format(Event4) ). - progress_report_test() -> Pid = self(), % Application started @@ -656,8 +643,9 @@ progress_report_test() -> #log_entry{ level = debug, pid = Pid, - msg = "Supervisor sup_dude started mod_name:fun_name/1" - " at pid <0.5.0>" + msg = + "Supervisor sup_dude started mod_name:fun_name/1" + " at pid <0.5.0>" }, do_format(Event2) ), @@ -680,7 +668,6 @@ progress_report_test() -> do_format(Event3) ). - log_unknown_event_test() -> Pid = self(), ?assertMatch( @@ -692,7 +679,6 @@ log_unknown_event_test() -> do_format(an_unknown_event) ). - format_reason_test_() -> Cases = [ { @@ -805,14 +791,15 @@ format_reason_test_() -> } ], [ - {Msg, fun() -> ?assertEqual( - Msg, - lists:flatten(couch_log_formatter:format_reason(Reason)) - ) end} - || {Reason, Msg} <- Cases + {Msg, fun() -> + ?assertEqual( + Msg, + lists:flatten(couch_log_formatter:format_reason(Reason)) + ) + end} + || {Reason, Msg} <- Cases ]. - coverage_test() -> % MFA's that aren't ?assertEqual(["foo"], couch_log_formatter:format_mfa(foo)), @@ -830,11 +817,13 @@ coverage_test() -> level = error, msg = "foobar" }, - do_format({ - error_report, - erlang:group_leader(), - {self(), std_error, "foobar"} - }) + do_format( + { + error_report, + erlang:group_leader(), + {self(), std_error, "foobar"} + } + ) ), % Excercising print_silly_list @@ -843,11 +832,13 @@ coverage_test() -> level = error, msg = "dang" }, - do_format({ - error_report, - erlang:group_leader(), - {self(), std_error, dang} - }) + do_format( + { + error_report, + erlang:group_leader(), + {self(), std_error, dang} + } + ) ). gen_server_error_with_last_msg_test() -> @@ -889,7 +880,7 @@ gen_event_error_with_last_msg_test() -> [ handler_id, a_gen_event, - {ohai,there}, + {ohai, there}, curr_state, barf ] @@ -911,7 +902,6 @@ gen_event_error_with_last_msg_test() -> ]) end). - gen_fsm_error_with_last_msg_test() -> Pid = self(), Event = { @@ -920,7 +910,7 @@ gen_fsm_error_with_last_msg_test() -> { Pid, "** State machine did a thing", - [a_gen_fsm, {ohai,there}, state_name, curr_state, barf] + [a_gen_fsm, {ohai, there}, state_name, curr_state, barf] } }, ?assertMatch( @@ -940,7 +930,6 @@ gen_fsm_error_with_last_msg_test() -> ]) end). - with_last(Fun) -> meck:new(couch_log_config_dyn, [passthrough]), try @@ -963,10 +952,8 @@ do_format(Event) -> time_stamp = lists:flatten(E#log_entry.time_stamp) }. - do_matches(_, []) -> ok; - do_matches(#log_entry{msg = Msg} = E, [Pattern | RestPatterns]) -> case re:run(Msg, Pattern) of {match, _} -> diff --git a/src/couch_log/test/eunit/couch_log_monitor_test.erl b/src/couch_log/test/eunit/couch_log_monitor_test.erl index eec008522..ceeb98b4e 100644 --- a/src/couch_log/test/eunit/couch_log_monitor_test.erl +++ b/src/couch_log/test/eunit/couch_log_monitor_test.erl @@ -12,24 +12,16 @@ -module(couch_log_monitor_test). - -include_lib("eunit/include/eunit.hrl"). - -define(HANDLER, couch_log_error_logger_h). - couch_log_monitor_test_() -> - {setup, - fun couch_log_test_util:start/0, - fun couch_log_test_util:stop/1, - [ - fun monitor_ignores_unknown_messages/0, - fun monitor_restarts_handler/0, - fun coverage_test/0 - ] - }. - + {setup, fun couch_log_test_util:start/0, fun couch_log_test_util:stop/1, [ + fun monitor_ignores_unknown_messages/0, + fun monitor_restarts_handler/0, + fun coverage_test/0 + ]}. monitor_ignores_unknown_messages() -> Pid1 = get_monitor_pid(), @@ -41,7 +33,6 @@ monitor_ignores_unknown_messages() -> timer:sleep(250), ?assert(is_process_alive(Pid1)). - monitor_restarts_handler() -> Pid1 = get_monitor_pid(), error_logger:delete_report_handler(?HANDLER), @@ -55,12 +46,10 @@ monitor_restarts_handler() -> Handlers = gen_event:which_handlers(error_logger), ?assert(lists:member(?HANDLER, Handlers)). - coverage_test() -> Resp = couch_log_monitor:code_change(foo, bazinga, baz), ?assertEqual({ok, bazinga}, Resp). - get_monitor_pid() -> Children = supervisor:which_children(couch_log_sup), [MonPid] = [Pid || {couch_log_monitor, Pid, _, _} <- Children, is_pid(Pid)], diff --git a/src/couch_log/test/eunit/couch_log_server_test.erl b/src/couch_log/test/eunit/couch_log_server_test.erl index 7af570e90..a2334b048 100644 --- a/src/couch_log/test/eunit/couch_log_server_test.erl +++ b/src/couch_log/test/eunit/couch_log_server_test.erl @@ -12,23 +12,16 @@ -module(couch_log_server_test). - -include("couch_log.hrl"). -include_lib("eunit/include/eunit.hrl"). - couch_log_server_test_() -> - {setup, - fun couch_log_test_util:start/0, - fun couch_log_test_util:stop/1, - [ - fun check_can_reconfigure/0, - fun check_can_restart/0, - fun check_can_cast_log_entry/0, - fun check_logs_ignored_messages/0 - ] - }. - + {setup, fun couch_log_test_util:start/0, fun couch_log_test_util:stop/1, [ + fun check_can_reconfigure/0, + fun check_can_restart/0, + fun check_can_cast_log_entry/0, + fun check_logs_ignored_messages/0 + ]}. check_can_reconfigure() -> couch_log:error("a message", []), @@ -44,7 +37,6 @@ check_can_reconfigure() -> ?assertEqual('$end_of_table', couch_log_test_util:last_log_key()) end). - check_can_restart() -> Pid1 = whereis(couch_log_server), Ref = erlang:monitor(process, Pid1), @@ -65,7 +57,6 @@ check_can_restart() -> ?assertNotEqual(Pid2, Pid1), ?assert(is_process_alive(Pid2)). - check_can_cast_log_entry() -> Entry = #log_entry{ level = critical, @@ -75,10 +66,10 @@ check_can_cast_log_entry() -> time_stamp = "2016-07-20-almost-my-birthday" }, ok = gen_server:cast(couch_log_server, {log, Entry}), - timer:sleep(500), % totes gross + % totes gross + timer:sleep(500), ?assertEqual(Entry, couch_log_test_util:last_log()). - check_logs_ignored_messages() -> gen_server:call(couch_log_server, a_call), ?assertMatch( @@ -91,7 +82,8 @@ check_logs_ignored_messages() -> ), gen_server:cast(couch_log_server, a_cast), - timer:sleep(500), % yes gross + % yes gross + timer:sleep(500), ?assertMatch( #log_entry{ level = error, @@ -102,7 +94,8 @@ check_logs_ignored_messages() -> ), couch_log_server ! an_info, - timer:sleep(500), % still gross + % still gross + timer:sleep(500), ?assertMatch( #log_entry{ level = error, @@ -112,7 +105,6 @@ check_logs_ignored_messages() -> couch_log_test_util:last_log() ). - coverage_test() -> Resp = couch_log_server:code_change(foo, bazinga, baz), ?assertEqual({ok, bazinga}, Resp). diff --git a/src/couch_log/test/eunit/couch_log_test.erl b/src/couch_log/test/eunit/couch_log_test.erl index c7195f65f..1538934b3 100644 --- a/src/couch_log/test/eunit/couch_log_test.erl +++ b/src/couch_log/test/eunit/couch_log_test.erl @@ -12,24 +12,17 @@ -module(couch_log_test). - -include_lib("couch_log/include/couch_log.hrl"). -include_lib("eunit/include/eunit.hrl"). - couch_log_test_() -> - {setup, - fun couch_log_test_util:start/0, - fun couch_log_test_util:stop/1, - gen() ++ [fun check_set_level/0] - }. - + {setup, fun couch_log_test_util:start/0, fun couch_log_test_util:stop/1, + gen() ++ [fun check_set_level/0]}. check_set_level() -> couch_log:set_level(crit), ?assertEqual("crit", config:get("log", "level")). - levels() -> [ debug, @@ -43,17 +36,17 @@ levels() -> none ]. - gen() -> - lists:map(fun(L) -> - Name = "Test log level: " ++ couch_log_util:level_to_string(L), - {Name, fun() -> check_levels(L, levels()) end} - end, levels() -- [none]). - + lists:map( + fun(L) -> + Name = "Test log level: " ++ couch_log_util:level_to_string(L), + {Name, fun() -> check_levels(L, levels()) end} + end, + levels() -- [none] + ). check_levels(_, []) -> ok; - check_levels(TestLevel, [CfgLevel | RestLevels]) -> TestInt = couch_log_util:level_to_integer(TestLevel), CfgInt = couch_log_util:level_to_integer(CfgLevel), @@ -78,7 +71,6 @@ check_levels(TestLevel, [CfgLevel | RestLevels]) -> end), check_levels(TestLevel, RestLevels). - new_msg() -> Bin = list_to_binary([couch_rand:uniform(255) || _ <- lists:seq(1, 16)]), couch_util:to_hex(Bin). diff --git a/src/couch_log/test/eunit/couch_log_test_util.erl b/src/couch_log/test/eunit/couch_log_test_util.erl index 00f3981fc..9a170bdbd 100644 --- a/src/couch_log/test/eunit/couch_log_test_util.erl +++ b/src/couch_log/test/eunit/couch_log_test_util.erl @@ -25,7 +25,6 @@ -include("couch_log.hrl"). - start() -> remove_error_loggers(), application:set_env(config, ini_files, config_files()), @@ -35,13 +34,11 @@ start() -> meck:new(couch_stats), ok = meck:expect(couch_stats, increment_counter, ['_'], ok). - stop(_) -> application:stop(config), application:stop(couch_log), meck:unload(couch_stats). - with_level(Name, Fun) -> with_config_listener(fun() -> try @@ -54,7 +51,6 @@ with_level(Name, Fun) -> end end). - with_config_listener(Fun) -> Listener = self(), try @@ -64,7 +60,6 @@ with_config_listener(Fun) -> rem_listener(Listener) end. - wait_for_config() -> receive couch_log_config_change_finished -> ok @@ -72,47 +67,53 @@ wait_for_config() -> erlang:error(config_change_timeout) end. - with_meck(Mods, Fun) -> - lists:foreach(fun(M) -> - case M of - {Name, Opts} -> meck:new(Name, Opts); - Name -> meck:new(Name) - end - end, Mods), + lists:foreach( + fun(M) -> + case M of + {Name, Opts} -> meck:new(Name, Opts); + Name -> meck:new(Name) + end + end, + Mods + ), try Fun() after - lists:foreach(fun(M) -> - case M of - {Name, _} -> meck:unload(Name); - Name -> meck:unload(Name) - end - end, Mods) + lists:foreach( + fun(M) -> + case M of + {Name, _} -> meck:unload(Name); + Name -> meck:unload(Name) + end + end, + Mods + ) end. - ignore_common_loggers() -> IgnoreSet = [ application_controller, config, config_event ], - lists:foreach(fun(Proc) -> - disable_logs_from(Proc) - end, IgnoreSet). - + lists:foreach( + fun(Proc) -> + disable_logs_from(Proc) + end, + IgnoreSet + ). disable_logs_from(Pid) when is_pid(Pid) -> - Ignored = case application:get_env(couch_log, ignored_pids) of - {ok, L} when is_list(L) -> - lists:usort([Pid | L]); - _E -> - [Pid] - end, + Ignored = + case application:get_env(couch_log, ignored_pids) of + {ok, L} when is_list(L) -> + lists:usort([Pid | L]); + _E -> + [Pid] + end, IgnoredAlive = [P || P <- Ignored, is_process_alive(P)], application:set_env(couch_log, ignored_pids, IgnoredAlive); - disable_logs_from(Name) when is_atom(Name) -> case whereis(Name) of P when is_pid(P) -> @@ -121,48 +122,49 @@ disable_logs_from(Name) when is_atom(Name) -> erlang:error({unknown_pid_name, Name}) end. - last_log_key() -> ets:last(?COUCH_LOG_TEST_TABLE). - last_log() -> [{_, Entry}] = ets:lookup(?COUCH_LOG_TEST_TABLE, last_log_key()), Entry. - remove_error_loggers() -> ErrorLoggerPid = whereis(error_logger), - if ErrorLoggerPid == undefined -> ok; true -> - lists:foreach(fun(Handler) -> - error_logger:delete_report_handler(Handler) - end, gen_event:which_handlers(ErrorLoggerPid)) + if + ErrorLoggerPid == undefined -> + ok; + true -> + lists:foreach( + fun(Handler) -> + error_logger:delete_report_handler(Handler) + end, + gen_event:which_handlers(ErrorLoggerPid) + ) end. - config_files() -> Path = filename:dirname(code:which(?MODULE)), Name = filename:join(Path, "couch_log_test.ini"), ok = file:write_file(Name, "[log]\nwriter = ets\n"), [Name]. - add_listener(Listener) -> - Listeners = case application:get_env(couch_log, config_listeners) of - {ok, L} when is_list(L) -> - lists:usort([Listener | L]); - _ -> - [Listener] - end, + Listeners = + case application:get_env(couch_log, config_listeners) of + {ok, L} when is_list(L) -> + lists:usort([Listener | L]); + _ -> + [Listener] + end, application:set_env(couch_log, config_listeners, Listeners). - rem_listener(Listener) -> - Listeners = case application:get_env(couch_lig, config_listeners) of - {ok, L} when is_list(L) -> - L -- [Listener]; - _ -> - [] - end, + Listeners = + case application:get_env(couch_lig, config_listeners) of + {ok, L} when is_list(L) -> + L -- [Listener]; + _ -> + [] + end, application:set_env(couch_log, config_listeners, Listeners). - diff --git a/src/couch_log/test/eunit/couch_log_trunc_io_fmt_test.erl b/src/couch_log/test/eunit/couch_log_trunc_io_fmt_test.erl index 77d555440..8d1fdeffb 100644 --- a/src/couch_log/test/eunit/couch_log_trunc_io_fmt_test.erl +++ b/src/couch_log/test/eunit/couch_log_trunc_io_fmt_test.erl @@ -12,22 +12,22 @@ -module(couch_log_trunc_io_fmt_test). - -include_lib("eunit/include/eunit.hrl"). - format_test_() -> - lists:map(fun({Fmt, Args, Expect}) -> - Name = io_lib:format("~p", [Expect]), - {lists:flatten(Name), - ?_assertEqual( - Expect, - lists:flatten(couch_log_trunc_io_fmt:format(Fmt, Args, 1024)) - ) - } - end, cases()). - - + lists:map( + fun({Fmt, Args, Expect}) -> + Name = io_lib:format("~p", [Expect]), + { + lists:flatten(Name), + ?_assertEqual( + Expect, + lists:flatten(couch_log_trunc_io_fmt:format(Fmt, Args, 1024)) + ) + } + end, + cases() + ). chomp_test() -> R1 = couch_log_trunc_io_fmt:format("\n", [], 1024, [{chomp, true}]), @@ -35,7 +35,6 @@ chomp_test() -> R2 = couch_log_trunc_io_fmt:format("~n", [], 1024, [{chomp, true}]), ?assertEqual("", lists:flatten(R2)). - cases() -> [ {"", [], ""}, diff --git a/src/couch_log/test/eunit/couch_log_util_test.erl b/src/couch_log/test/eunit/couch_log_util_test.erl index e97911aa9..ade968146 100644 --- a/src/couch_log/test/eunit/couch_log_util_test.erl +++ b/src/couch_log/test/eunit/couch_log_util_test.erl @@ -12,25 +12,24 @@ -module(couch_log_util_test). - -include_lib("couch_log/include/couch_log.hrl"). -include_lib("eunit/include/eunit.hrl"). - get_message_id_test() -> ?assertEqual("--------", couch_log_util:get_msg_id()), erlang:put(nonce, "deadbeef"), ?assertEqual("deadbeef", couch_log_util:get_msg_id()), erlang:put(nonce, undefined). - level_to_atom_test() -> - lists:foreach(fun(L) -> - ?assert(is_atom(couch_log_util:level_to_atom(L))), - ?assert(is_integer(couch_log_util:level_to_integer(L))), - ?assert(is_list(couch_log_util:level_to_string(L))) - end, levels()). - + lists:foreach( + fun(L) -> + ?assert(is_atom(couch_log_util:level_to_atom(L))), + ?assert(is_integer(couch_log_util:level_to_integer(L))), + ?assert(is_list(couch_log_util:level_to_string(L))) + end, + levels() + ). string_p_test() -> ?assertEqual(false, couch_log_util:string_p([])), @@ -43,13 +42,50 @@ string_p_test() -> ?assertEqual(true, couch_log_util:string_p([$\f])), ?assertEqual(true, couch_log_util:string_p([$\e])). - levels() -> [ - 1, 2, 3, 4, 5, 6, 7, 8, 9, - "1", "2", "3", "4", "5", "6", "7", "8", "9", - debug, info, notice, warning, warn, error, err, - critical, crit, alert, emergency, emerg, none, - "debug", "info", "notice", "warning", "warn", "error", "err", - "critical", "crit", "alert", "emergency", "emerg", "none" + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + debug, + info, + notice, + warning, + warn, + error, + err, + critical, + crit, + alert, + emergency, + emerg, + none, + "debug", + "info", + "notice", + "warning", + "warn", + "error", + "err", + "critical", + "crit", + "alert", + "emergency", + "emerg", + "none" ]. diff --git a/src/couch_log/test/eunit/couch_log_writer_ets.erl b/src/couch_log/test/eunit/couch_log_writer_ets.erl index d5fd327ac..7ddb9f39e 100644 --- a/src/couch_log/test/eunit/couch_log_writer_ets.erl +++ b/src/couch_log/test/eunit/couch_log_writer_ets.erl @@ -13,27 +13,22 @@ -module(couch_log_writer_ets). -behaviour(couch_log_writer). - -export([ init/0, terminate/2, write/2 ]). - -include("couch_log.hrl"). - init() -> ets:new(?COUCH_LOG_TEST_TABLE, [named_table, public, ordered_set]), {ok, 0}. - terminate(_, _St) -> ets:delete(?COUCH_LOG_TEST_TABLE), ok. - write(Entry0, St) -> Entry = Entry0#log_entry{ msg = lists:flatten(Entry0#log_entry.msg), diff --git a/src/couch_log/test/eunit/couch_log_writer_file_test.erl b/src/couch_log/test/eunit/couch_log_writer_file_test.erl index ba042610a..2e40088f4 100644 --- a/src/couch_log/test/eunit/couch_log_writer_file_test.erl +++ b/src/couch_log/test/eunit/couch_log_writer_file_test.erl @@ -12,65 +12,54 @@ -module(couch_log_writer_file_test). - -include_lib("kernel/include/file.hrl"). -include_lib("couch_log/include/couch_log.hrl"). -include_lib("eunit/include/eunit.hrl"). - -define(WRITER, couch_log_writer_file). - couch_log_writer_file_test_() -> - {setup, - fun couch_log_test_util:start/0, - fun couch_log_test_util:stop/1, - [ - fun check_init_terminate/0, - fun() -> - couch_log_test_util:with_meck( - [{filelib, [unstick]}], - fun check_ensure_dir_fail/0 - ) - end, - fun() -> - couch_log_test_util:with_meck( - [{file, [unstick, passthrough]}], - fun check_open_fail/0 - ) - end, - fun() -> - couch_log_test_util:with_meck( - [{file, [unstick, passthrough]}], - fun check_read_file_info_fail/0 - ) - end, - fun check_file_write/0, - fun check_buffered_file_write/0, - fun check_reopen/0 - ] - }. - + {setup, fun couch_log_test_util:start/0, fun couch_log_test_util:stop/1, [ + fun check_init_terminate/0, + fun() -> + couch_log_test_util:with_meck( + [{filelib, [unstick]}], + fun check_ensure_dir_fail/0 + ) + end, + fun() -> + couch_log_test_util:with_meck( + [{file, [unstick, passthrough]}], + fun check_open_fail/0 + ) + end, + fun() -> + couch_log_test_util:with_meck( + [{file, [unstick, passthrough]}], + fun check_read_file_info_fail/0 + ) + end, + fun check_file_write/0, + fun check_buffered_file_write/0, + fun check_reopen/0 + ]}. check_init_terminate() -> {ok, St} = ?WRITER:init(), ok = ?WRITER:terminate(stop, St). - check_ensure_dir_fail() -> meck:expect(filelib, ensure_dir, 1, {error, eperm}), ?assertEqual({error, eperm}, ?WRITER:init()), ?assert(meck:called(filelib, ensure_dir, 1)), ?assert(meck:validate(filelib)). - check_open_fail() -> meck:expect(file, open, 2, {error, enotfound}), ?assertEqual({error, enotfound}, ?WRITER:init()), ?assert(meck:called(file, open, 2)), ?assert(meck:validate(file)). - check_read_file_info_fail() -> RFI = fun ("./couch.log") -> {error, enoent}; @@ -81,12 +70,12 @@ check_read_file_info_fail() -> ?assert(meck:called(file, read_file_info, 1)), ?assert(meck:validate(file)). - check_file_write() -> % Make sure we have an empty log for this test IsFile = filelib:is_file("./couch.log"), - if not IsFile -> ok; true -> - file:delete("./couch.log") + if + not IsFile -> ok; + true -> file:delete("./couch.log") end, Entry = #log_entry{ @@ -104,12 +93,12 @@ check_file_write() -> Expect = <<"[info] time_stamp nonode@nohost <0.1.0> msg_id stuff\n">>, ?assertEqual(Expect, Data). - check_buffered_file_write() -> % Make sure we have an empty log for this test IsFile = filelib:is_file("./couch.log"), - if not IsFile -> ok; true -> - file:delete("./couch.log") + if + not IsFile -> ok; + true -> file:delete("./couch.log") end, config:set("log", "write_buffer", "1024"), @@ -135,7 +124,6 @@ check_buffered_file_write() -> Expect = <<"[info] time_stamp nonode@nohost <0.1.0> msg_id stuff\n">>, ?assertEqual(Expect, Data). - check_reopen() -> {ok, St1} = clear_clock(?WRITER:init()), {ok, St2} = clear_clock(couch_log_writer_file:maybe_reopen(St1)), @@ -160,10 +148,8 @@ check_reopen() -> ?assert(element(3, St4) /= element(3, St2)) end. - clear_clock({ok, St}) -> {ok, clear_clock(St)}; - clear_clock(St) -> {st, Path, Fd, INode, _} = St, {st, Path, Fd, INode, {0, 0, 0}}. diff --git a/src/couch_log/test/eunit/couch_log_writer_stderr_test.erl b/src/couch_log/test/eunit/couch_log_writer_stderr_test.erl index 1e99263dd..04f1e9a41 100644 --- a/src/couch_log/test/eunit/couch_log_writer_stderr_test.erl +++ b/src/couch_log/test/eunit/couch_log_writer_stderr_test.erl @@ -12,35 +12,26 @@ -module(couch_log_writer_stderr_test). - -include_lib("couch_log/include/couch_log.hrl"). -include_lib("eunit/include/eunit.hrl"). - -define(WRITER, couch_log_writer_stderr). - couch_log_writer_stderr_test_() -> - {setup, - fun couch_log_test_util:start/0, - fun couch_log_test_util:stop/1, - [ - fun check_init_terminate/0, - fun() -> - couch_log_test_util:with_meck( - [{io, [unstick]}], - fun check_write/0 - ) - end - ] - }. - + {setup, fun couch_log_test_util:start/0, fun couch_log_test_util:stop/1, [ + fun check_init_terminate/0, + fun() -> + couch_log_test_util:with_meck( + [{io, [unstick]}], + fun check_write/0 + ) + end + ]}. check_init_terminate() -> {ok, St} = ?WRITER:init(), ok = ?WRITER:terminate(stop, St). - check_write() -> meck:expect(io, format, 3, ok), diff --git a/src/couch_log/test/eunit/couch_log_writer_syslog_test.erl b/src/couch_log/test/eunit/couch_log_writer_syslog_test.erl index c32b5c6bf..5a3f89520 100644 --- a/src/couch_log/test/eunit/couch_log_writer_syslog_test.erl +++ b/src/couch_log/test/eunit/couch_log_writer_syslog_test.erl @@ -12,41 +12,32 @@ -module(couch_log_writer_syslog_test). - -include_lib("couch_log/include/couch_log.hrl"). -include_lib("eunit/include/eunit.hrl"). - -define(WRITER, couch_log_writer_syslog). - couch_log_writer_syslog_test_() -> - {setup, - fun couch_log_test_util:start/0, - fun couch_log_test_util:stop/1, - [ - fun check_init_terminate/0, - fun() -> - couch_log_test_util:with_meck( - [{io, [unstick]}], - fun check_stderr_write/0 - ) - end, - fun() -> - couch_log_test_util:with_meck( - [{gen_udp, [unstick]}], - fun check_udp_send/0 - ) - end - ] - }. - + {setup, fun couch_log_test_util:start/0, fun couch_log_test_util:stop/1, [ + fun check_init_terminate/0, + fun() -> + couch_log_test_util:with_meck( + [{io, [unstick]}], + fun check_stderr_write/0 + ) + end, + fun() -> + couch_log_test_util:with_meck( + [{gen_udp, [unstick]}], + fun check_udp_send/0 + ) + end + ]}. check_init_terminate() -> {ok, St} = ?WRITER:init(), ok = ?WRITER:terminate(stop, St). - check_stderr_write() -> meck:expect(io, format, 3, ok), @@ -64,7 +55,6 @@ check_stderr_write() -> ?assert(meck:called(io, format, 3)), ?assert(meck:validate(io)). - check_udp_send() -> meck:expect(gen_udp, open, 1, {ok, socket}), meck:expect(gen_udp, send, 4, ok), @@ -91,32 +81,64 @@ check_udp_send() -> ?assert(meck:called(gen_udp, close, 1)), ?assert(meck:validate(gen_udp)). - facility_test() -> Names = [ - "kern", "user", "mail", "daemon", "auth", "syslog", "lpr", - "news", "uucp", "clock", "authpriv", "ftp", "ntp", "audit", - "alert", "cron", "local0", "local1", "local2", "local3", - "local4", "local5", "local6", "local7" + "kern", + "user", + "mail", + "daemon", + "auth", + "syslog", + "lpr", + "news", + "uucp", + "clock", + "authpriv", + "ftp", + "ntp", + "audit", + "alert", + "cron", + "local0", + "local1", + "local2", + "local3", + "local4", + "local5", + "local6", + "local7" ], - lists:foldl(fun(Name, Id) -> - IdStr = lists:flatten(io_lib:format("~w", [Id])), - ?assertEqual(Id bsl 3, couch_log_writer_syslog:get_facility(Name)), - ?assertEqual(Id bsl 3, couch_log_writer_syslog:get_facility(IdStr)), - Id + 1 - end, 0, Names), + lists:foldl( + fun(Name, Id) -> + IdStr = lists:flatten(io_lib:format("~w", [Id])), + ?assertEqual(Id bsl 3, couch_log_writer_syslog:get_facility(Name)), + ?assertEqual(Id bsl 3, couch_log_writer_syslog:get_facility(IdStr)), + Id + 1 + end, + 0, + Names + ), ?assertEqual(23 bsl 3, couch_log_writer_syslog:get_facility("foo")), ?assertEqual(23 bsl 3, couch_log_writer_syslog:get_facility("-1")), ?assertEqual(23 bsl 3, couch_log_writer_syslog:get_facility("24")). - level_test() -> Levels = [ - emergency, alert, critical, error, - warning, notice, info, debug + emergency, + alert, + critical, + error, + warning, + notice, + info, + debug ], - lists:foldl(fun(Name, Id) -> - ?assertEqual(Id, couch_log_writer_syslog:get_level(Name)), - Id + 1 - end, 0, Levels), + lists:foldl( + fun(Name, Id) -> + ?assertEqual(Id, couch_log_writer_syslog:get_level(Name)), + Id + 1 + end, + 0, + Levels + ), ?assertEqual(3, couch_log_writer_syslog:get_level(foo)). diff --git a/src/couch_log/test/eunit/couch_log_writer_test.erl b/src/couch_log/test/eunit/couch_log_writer_test.erl index d0bb347fe..e758c9f60 100644 --- a/src/couch_log/test/eunit/couch_log_writer_test.erl +++ b/src/couch_log/test/eunit/couch_log_writer_test.erl @@ -12,20 +12,13 @@ -module(couch_log_writer_test). - -include_lib("couch_log/include/couch_log.hrl"). -include_lib("eunit/include/eunit.hrl"). - couch_log_writer_test_() -> - {setup, - fun couch_log_test_util:start/0, - fun couch_log_test_util:stop/1, - [ - fun check_writer_change/0 - ] - }. - + {setup, fun couch_log_test_util:start/0, fun couch_log_test_util:stop/1, [ + fun check_writer_change/0 + ]}. check_writer_change() -> % Change to file and back @@ -51,4 +44,3 @@ check_writer_change() -> couch_log_test_util:wait_for_config(), ?assertEqual(0, ets:info(?COUCH_LOG_TEST_TABLE, size)) end). - diff --git a/src/couch_mrview/src/couch_mrview.erl b/src/couch_mrview/src/couch_mrview.erl index 1cdc91809..798b939c7 100644 --- a/src/couch_mrview/src/couch_mrview.erl +++ b/src/couch_mrview/src/couch_mrview.erl @@ -27,7 +27,7 @@ -record(mracc, { db, - meta_sent=false, + meta_sent = false, total_rows, offset, limit, @@ -36,82 +36,100 @@ doc_info, callback, user_acc, - last_go=ok, + last_go = ok, reduce_fun, finalizer, update_seq, args }). - - validate_ddoc_fields(DDoc) -> MapFuncType = map_function_type(DDoc), - lists:foreach(fun(Path) -> - validate_ddoc_fields(DDoc, Path) - end, [ - [{<<"filters">>, object}, {any, [object, string]}], - [{<<"language">>, string}], - [{<<"lists">>, object}, {any, [object, string]}], - [{<<"options">>, object}], - [{<<"options">>, object}, {<<"include_design">>, boolean}], - [{<<"options">>, object}, {<<"local_seq">>, boolean}], - [{<<"options">>, object}, {<<"partitioned">>, boolean}], - [{<<"rewrites">>, [string, array]}], - [{<<"shows">>, object}, {any, [object, string]}], - [{<<"updates">>, object}, {any, [object, string]}], - [{<<"validate_doc_update">>, string}], - [{<<"views">>, object}, {<<"lib">>, object}], - [{<<"views">>, object}, {any, object}, {<<"map">>, MapFuncType}], - [{<<"views">>, object}, {any, object}, {<<"reduce">>, string}] - ]), + lists:foreach( + fun(Path) -> + validate_ddoc_fields(DDoc, Path) + end, + [ + [{<<"filters">>, object}, {any, [object, string]}], + [{<<"language">>, string}], + [{<<"lists">>, object}, {any, [object, string]}], + [{<<"options">>, object}], + [{<<"options">>, object}, {<<"include_design">>, boolean}], + [{<<"options">>, object}, {<<"local_seq">>, boolean}], + [{<<"options">>, object}, {<<"partitioned">>, boolean}], + [{<<"rewrites">>, [string, array]}], + [{<<"shows">>, object}, {any, [object, string]}], + [{<<"updates">>, object}, {any, [object, string]}], + [{<<"validate_doc_update">>, string}], + [{<<"views">>, object}, {<<"lib">>, object}], + [{<<"views">>, object}, {any, object}, {<<"map">>, MapFuncType}], + [{<<"views">>, object}, {any, object}, {<<"reduce">>, string}] + ] + ), require_map_function_for_views(DDoc), ok. require_map_function_for_views({Props}) -> case couch_util:get_value(<<"views">>, Props) of - undefined -> ok; + undefined -> + ok; {Views} -> - lists:foreach(fun - ({<<"lib">>, _}) -> ok; - ({Key, {Value}}) -> - case couch_util:get_value(<<"map">>, Value) of - undefined -> throw({invalid_design_doc, - <<"View `", Key/binary, "` must contain map function">>}); - _ -> ok - end - end, Views), + lists:foreach( + fun + ({<<"lib">>, _}) -> + ok; + ({Key, {Value}}) -> + case couch_util:get_value(<<"map">>, Value) of + undefined -> + throw( + {invalid_design_doc, + <<"View `", Key/binary, "` must contain map function">>} + ); + _ -> + ok + end + end, + Views + ), ok end. validate_ddoc_fields(DDoc, Path) -> case validate_ddoc_fields(DDoc, Path, []) of - ok -> ok; + ok -> + ok; {error, {FailedPath0, Type0}} -> FailedPath = iolist_to_binary(join(FailedPath0, <<".">>)), Type = format_type(Type0), - throw({invalid_design_doc, - <<"`", FailedPath/binary, "` field must have ", - Type/binary, " type">>}) + throw( + {invalid_design_doc, + <<"`", FailedPath/binary, "` field must have ", Type/binary, " type">>} + ) end. validate_ddoc_fields(undefined, _, _) -> ok; validate_ddoc_fields(_, [], _) -> ok; -validate_ddoc_fields({KVS}=Props, [{any, Type} | Rest], Acc) -> - lists:foldl(fun - ({Key, _}, ok) -> - validate_ddoc_fields(Props, [{Key, Type} | Rest], Acc); - ({_, _}, {error, _}=Error) -> - Error - end, ok, KVS); -validate_ddoc_fields({KVS}=Props, [{Key, Type} | Rest], Acc) -> +validate_ddoc_fields({KVS} = Props, [{any, Type} | Rest], Acc) -> + lists:foldl( + fun + ({Key, _}, ok) -> + validate_ddoc_fields(Props, [{Key, Type} | Rest], Acc); + ({_, _}, {error, _} = Error) -> + Error + end, + ok, + KVS + ); +validate_ddoc_fields({KVS} = Props, [{Key, Type} | Rest], Acc) -> case validate_ddoc_field(Props, {Key, Type}) of ok -> - validate_ddoc_fields(couch_util:get_value(Key, KVS), - Rest, - [Key | Acc]); + validate_ddoc_fields( + couch_util:get_value(Key, KVS), + Rest, + [Key | Acc] + ); error -> {error, {[Key | Acc], Type}}; {error, Key1} -> @@ -123,10 +141,14 @@ validate_ddoc_field(undefined, Type) when is_atom(Type) -> validate_ddoc_field(_, any) -> ok; validate_ddoc_field(Value, Types) when is_list(Types) -> - lists:foldl(fun - (_, ok) -> ok; - (Type, _) -> validate_ddoc_field(Value, Type) - end, error, Types); + lists:foldl( + fun + (_, ok) -> ok; + (Type, _) -> validate_ddoc_field(Value, Type) + end, + error, + Types + ); validate_ddoc_field(Value, string) when is_binary(Value) -> ok; validate_ddoc_field(Value, array) when is_list(Value) -> @@ -165,36 +187,38 @@ format_type(Types) when is_list(Types) -> join(L, Sep) -> join(L, Sep, []). -join([H|[]], _, Acc) -> +join([H | []], _, Acc) -> [H | Acc]; -join([H|T], Sep, Acc) -> +join([H | T], Sep, Acc) -> join(T, Sep, [Sep, H | Acc]). - -validate(Db, DDoc) -> +validate(Db, DDoc) -> ok = validate_ddoc_fields(DDoc#doc.body), GetName = fun (#mrview{map_names = [Name | _]}) -> Name; (#mrview{reduce_funs = [{Name, _} | _]}) -> Name; (_) -> null end, - ValidateView = fun(Proc, #mrview{def=MapSrc, reduce_funs=Reds}=View) -> + ValidateView = fun(Proc, #mrview{def = MapSrc, reduce_funs = Reds} = View) -> couch_query_servers:try_compile(Proc, map, GetName(View), MapSrc), - lists:foreach(fun - ({_RedName, <<"_sum", _/binary>>}) -> - ok; - ({_RedName, <<"_count", _/binary>>}) -> - ok; - ({_RedName, <<"_stats", _/binary>>}) -> - ok; - ({_RedName, <<"_approx_count_distinct", _/binary>>}) -> - ok; - ({_RedName, <<"_", _/binary>> = Bad}) -> - Msg = ["`", Bad, "` is not a supported reduce function."], - throw({invalid_design_doc, Msg}); - ({RedName, RedSrc}) -> - couch_query_servers:try_compile(Proc, reduce, RedName, RedSrc) - end, Reds) + lists:foreach( + fun + ({_RedName, <<"_sum", _/binary>>}) -> + ok; + ({_RedName, <<"_count", _/binary>>}) -> + ok; + ({_RedName, <<"_stats", _/binary>>}) -> + ok; + ({_RedName, <<"_approx_count_distinct", _/binary>>}) -> + ok; + ({_RedName, <<"_", _/binary>> = Bad}) -> + Msg = ["`", Bad, "` is not a supported reduce function."], + throw({invalid_design_doc, Msg}); + ({RedName, RedSrc}) -> + couch_query_servers:try_compile(Proc, reduce, RedName, RedSrc) + end, + Reds + ) end, {ok, #mrst{ language = Lang, @@ -204,9 +228,12 @@ validate(Db, DDoc) -> case {couch_db:is_partitioned(Db), Partitioned} of {false, true} -> - throw({invalid_design_doc, - <<"partitioned option cannot be true in a " - "non-partitioned database.">>}); + throw( + {invalid_design_doc, << + "partitioned option cannot be true in a " + "non-partitioned database." + >>} + ); {_, _} -> ok end, @@ -220,16 +247,15 @@ validate(Db, DDoc) -> after couch_query_servers:ret_os_process(Proc) end - catch {unknown_query_language, _Lang} -> - %% Allow users to save ddocs written in unknown languages - ok + catch + {unknown_query_language, _Lang} -> + %% Allow users to save ddocs written in unknown languages + ok end. - query_all_docs(Db, Args) -> query_all_docs(Db, Args, fun default_cb/2, []). - query_all_docs(Db, Args, Callback, Acc) when is_list(Args) -> query_all_docs(Db, to_mrargs(Args), Callback, Acc); query_all_docs(Db, Args0, Callback, Acc) -> @@ -237,44 +263,41 @@ query_all_docs(Db, Args0, Callback, Acc) -> {ok, Info} = couch_db:get_db_info(WDb), couch_index_util:hexsig(couch_hash:md5_hash(term_to_binary(Info))) end), - Args1 = Args0#mrargs{view_type=map}, + Args1 = Args0#mrargs{view_type = map}, Args2 = couch_mrview_util:validate_all_docs_args(Db, Args1), - {ok, Acc1} = case Args2#mrargs.preflight_fun of - PFFun when is_function(PFFun, 2) -> PFFun(Sig, Acc); - _ -> {ok, Acc} - end, + {ok, Acc1} = + case Args2#mrargs.preflight_fun of + PFFun when is_function(PFFun, 2) -> PFFun(Sig, Acc); + _ -> {ok, Acc} + end, all_docs_fold(Db, Args2, Callback, Acc1). - query_view(Db, DDoc, VName) -> query_view(Db, DDoc, VName, #mrargs{}). - query_view(Db, DDoc, VName, Args) when is_list(Args) -> query_view(Db, DDoc, VName, to_mrargs(Args), fun default_cb/2, []); query_view(Db, DDoc, VName, Args) -> query_view(Db, DDoc, VName, Args, fun default_cb/2, []). - query_view(Db, DDoc, VName, Args, Callback, Acc) when is_list(Args) -> query_view(Db, DDoc, VName, to_mrargs(Args), Callback, Acc); query_view(Db, DDoc, VName, Args0, Callback, Acc0) -> case couch_mrview_util:get_view(Db, DDoc, VName, Args0) of {ok, VInfo, Sig, Args} -> - {ok, Acc1} = case Args#mrargs.preflight_fun of - PFFun when is_function(PFFun, 2) -> PFFun(Sig, Acc0); - _ -> {ok, Acc0} - end, + {ok, Acc1} = + case Args#mrargs.preflight_fun of + PFFun when is_function(PFFun, 2) -> PFFun(Sig, Acc0); + _ -> {ok, Acc0} + end, query_view(Db, VInfo, Args, Callback, Acc1); ddoc_updated -> Callback(ok, ddoc_updated) end. - get_view_index_pid(Db, DDoc, ViewName, Args0) -> couch_mrview_util:get_view_index_pid(Db, DDoc, ViewName, Args0). - query_view(Db, {Type, View, Ref}, Args, Callback, Acc) -> try case Type of @@ -285,12 +308,10 @@ query_view(Db, {Type, View, Ref}, Args, Callback, Acc) -> erlang:demonitor(Ref, [flush]) end. - get_info(Db, DDoc) -> {ok, Pid} = couch_index_server:get_index(couch_mrview_index, Db, DDoc), couch_index:get_info(Pid). - trigger_update(Db, DDoc) -> trigger_update(Db, DDoc, couch_db:get_update_seq(Db)). @@ -300,22 +321,27 @@ trigger_update(Db, DDoc, UpdateSeq) -> %% get informations on a view get_view_info(Db, DDoc, VName) -> - {ok, {_, View, _}, _, _Args} = couch_mrview_util:get_view(Db, DDoc, VName, - #mrargs{}), + {ok, {_, View, _}, _, _Args} = couch_mrview_util:get_view( + Db, + DDoc, + VName, + #mrargs{} + ), %% get the total number of rows - {ok, TotalRows} = couch_mrview_util:get_row_count(View), - - {ok, [{update_seq, View#mrview.update_seq}, - {purge_seq, View#mrview.purge_seq}, - {total_rows, TotalRows}]}. + {ok, TotalRows} = couch_mrview_util:get_row_count(View), + {ok, [ + {update_seq, View#mrview.update_seq}, + {purge_seq, View#mrview.purge_seq}, + {total_rows, TotalRows} + ]}. %% @doc refresh a view index -refresh(DbName, DDoc) when is_binary(DbName)-> +refresh(DbName, DDoc) when is_binary(DbName) -> UpdateSeq = couch_util:with_db(DbName, fun(WDb) -> - couch_db:get_update_seq(WDb) - end), + couch_db:get_update_seq(WDb) + end), case couch_index_server:get_index(couch_mrview_index, DbName, DDoc) of {ok, Pid} -> @@ -326,48 +352,43 @@ refresh(DbName, DDoc) when is_binary(DbName)-> Error -> {error, Error} end; - refresh(Db, DDoc) -> refresh(couch_db:name(Db), DDoc). compact(Db, DDoc) -> compact(Db, DDoc, []). - compact(Db, DDoc, Opts) -> {ok, Pid} = couch_index_server:get_index(couch_mrview_index, Db, DDoc), couch_index:compact(Pid, Opts). - cancel_compaction(Db, DDoc) -> {ok, IPid} = couch_index_server:get_index(couch_mrview_index, Db, DDoc), {ok, CPid} = couch_index:get_compactor_pid(IPid), ok = couch_index_compactor:cancel(CPid), % Cleanup the compaction file if it exists - {ok, #mrst{sig=Sig, db_name=DbName}} = couch_index:get_state(IPid, 0), + {ok, #mrst{sig = Sig, db_name = DbName}} = couch_index:get_state(IPid, 0), couch_mrview_util:delete_compaction_file(DbName, Sig), ok. - cleanup(Db) -> couch_mrview_cleanup:run(Db). - -all_docs_fold(Db, #mrargs{keys=undefined}=Args, Callback, UAcc) -> +all_docs_fold(Db, #mrargs{keys = undefined} = Args, Callback, UAcc) -> ReduceFun = get_reduce_fun(Args), Total = get_total_rows(Db, Args), UpdateSeq = get_update_seq(Db, Args), Acc = #mracc{ - db=Db, - total_rows=Total, - limit=Args#mrargs.limit, - skip=Args#mrargs.skip, - callback=Callback, - user_acc=UAcc, - reduce_fun=ReduceFun, - update_seq=UpdateSeq, - args=Args + db = Db, + total_rows = Total, + limit = Args#mrargs.limit, + skip = Args#mrargs.skip, + callback = Callback, + user_acc = UAcc, + reduce_fun = ReduceFun, + update_seq = UpdateSeq, + args = Args }, [Opts1] = couch_mrview_util:all_docs_key_opts(Args), % TODO: This is a terrible hack for now. We'll probably have @@ -375,243 +396,262 @@ all_docs_fold(Db, #mrargs{keys=undefined}=Args, Callback, UAcc) -> % a btree. For now non-btree's will just have to pass 0 or % some fake reductions to get an offset. Opts2 = [include_reductions | Opts1], - FunName = case couch_util:get_value(namespace, Args#mrargs.extra) of - <<"_design">> -> fold_design_docs; - <<"_local">> -> fold_local_docs; - _ -> fold_docs - end, + FunName = + case couch_util:get_value(namespace, Args#mrargs.extra) of + <<"_design">> -> fold_design_docs; + <<"_local">> -> fold_local_docs; + _ -> fold_docs + end, {ok, Offset, FinalAcc} = couch_db:FunName(Db, fun map_fold/3, Acc, Opts2), finish_fold(FinalAcc, [{total, Total}, {offset, Offset}]); -all_docs_fold(Db, #mrargs{direction=Dir, keys=Keys0}=Args, Callback, UAcc) -> +all_docs_fold(Db, #mrargs{direction = Dir, keys = Keys0} = Args, Callback, UAcc) -> ReduceFun = get_reduce_fun(Args), Total = get_total_rows(Db, Args), UpdateSeq = get_update_seq(Db, Args), Acc = #mracc{ - db=Db, - total_rows=Total, - limit=Args#mrargs.limit, - skip=Args#mrargs.skip, - callback=Callback, - user_acc=UAcc, - reduce_fun=ReduceFun, - update_seq=UpdateSeq, - args=Args + db = Db, + total_rows = Total, + limit = Args#mrargs.limit, + skip = Args#mrargs.skip, + callback = Callback, + user_acc = UAcc, + reduce_fun = ReduceFun, + update_seq = UpdateSeq, + args = Args }, % Backwards compatibility hack. The old _all_docs iterates keys % in reverse if descending=true was passed. Here we'll just % reverse the list instead. - Keys = if Dir =:= fwd -> Keys0; true -> lists:reverse(Keys0) end, + Keys = + if + Dir =:= fwd -> Keys0; + true -> lists:reverse(Keys0) + end, FoldFun = fun(Key, Acc0) -> DocInfo = (catch couch_db:get_doc_info(Db, Key)), - {Doc, Acc1} = case DocInfo of - {ok, #doc_info{id=Id, revs=[RevInfo | _RestRevs]}=DI} -> - Rev = couch_doc:rev_to_str(RevInfo#rev_info.rev), - Props = [{rev, Rev}] ++ case RevInfo#rev_info.deleted of - true -> [{deleted, true}]; - false -> [] - end, - {{{Id, Id}, {Props}}, Acc0#mracc{doc_info=DI}}; - not_found -> - {{{Key, error}, not_found}, Acc0} - end, + {Doc, Acc1} = + case DocInfo of + {ok, #doc_info{id = Id, revs = [RevInfo | _RestRevs]} = DI} -> + Rev = couch_doc:rev_to_str(RevInfo#rev_info.rev), + Props = + [{rev, Rev}] ++ + case RevInfo#rev_info.deleted of + true -> [{deleted, true}]; + false -> [] + end, + {{{Id, Id}, {Props}}, Acc0#mracc{doc_info = DI}}; + not_found -> + {{{Key, error}, not_found}, Acc0} + end, {_, Acc2} = map_fold(Doc, {[], [{0, 0, 0}]}, Acc1), Acc2 end, FinalAcc = lists:foldl(FoldFun, Acc, Keys), finish_fold(FinalAcc, [{total, Total}]). - map_fold(Db, View, Args, Callback, UAcc) -> {ok, Total} = couch_mrview_util:get_row_count(View), Acc = #mracc{ - db=Db, - total_rows=Total, - limit=Args#mrargs.limit, - skip=Args#mrargs.skip, - callback=Callback, - user_acc=UAcc, - reduce_fun=fun couch_mrview_util:reduce_to_count/1, - update_seq=View#mrview.update_seq, - args=Args + db = Db, + total_rows = Total, + limit = Args#mrargs.limit, + skip = Args#mrargs.skip, + callback = Callback, + user_acc = UAcc, + reduce_fun = fun couch_mrview_util:reduce_to_count/1, + update_seq = View#mrview.update_seq, + args = Args }, OptList = couch_mrview_util:key_opts(Args), - {Reds, Acc2} = lists:foldl(fun(Opts, {_, Acc0}) -> - {ok, R, A} = couch_mrview_util:fold(View, fun map_fold/3, Acc0, Opts), - {R, A} - end, {nil, Acc}, OptList), + {Reds, Acc2} = lists:foldl( + fun(Opts, {_, Acc0}) -> + {ok, R, A} = couch_mrview_util:fold(View, fun map_fold/3, Acc0, Opts), + {R, A} + end, + {nil, Acc}, + OptList + ), Offset = couch_mrview_util:reduce_to_count(Reds), finish_fold(Acc2, [{total, Total}, {offset, Offset}]). - map_fold(#full_doc_info{} = FullDocInfo, OffsetReds, Acc) -> % matches for _all_docs and translates #full_doc_info{} -> KV pair case couch_doc:to_doc_info(FullDocInfo) of - #doc_info{id=Id, revs=[#rev_info{deleted=false, rev=Rev}|_]} = DI -> + #doc_info{id = Id, revs = [#rev_info{deleted = false, rev = Rev} | _]} = DI -> Value = {[{rev, couch_doc:rev_to_str(Rev)}]}, - map_fold({{Id, Id}, Value}, OffsetReds, Acc#mracc{doc_info=DI}); - #doc_info{revs=[#rev_info{deleted=true}|_]} -> + map_fold({{Id, Id}, Value}, OffsetReds, Acc#mracc{doc_info = DI}); + #doc_info{revs = [#rev_info{deleted = true} | _]} -> {ok, Acc} end; -map_fold(_KV, _Offset, #mracc{skip=N}=Acc) when N > 0 -> - {ok, Acc#mracc{skip=N-1, last_go=ok}}; -map_fold(KV, OffsetReds, #mracc{offset=undefined}=Acc) -> +map_fold(_KV, _Offset, #mracc{skip = N} = Acc) when N > 0 -> + {ok, Acc#mracc{skip = N - 1, last_go = ok}}; +map_fold(KV, OffsetReds, #mracc{offset = undefined} = Acc) -> #mracc{ - total_rows=Total, - callback=Callback, - user_acc=UAcc0, - reduce_fun=Reduce, - update_seq=UpdateSeq, - args=Args + total_rows = Total, + callback = Callback, + user_acc = UAcc0, + reduce_fun = Reduce, + update_seq = UpdateSeq, + args = Args } = Acc, Offset = Reduce(OffsetReds), Meta = make_meta(Args, UpdateSeq, [{total, Total}, {offset, Offset}]), {Go, UAcc1} = Callback(Meta, UAcc0), - Acc1 = Acc#mracc{meta_sent=true, offset=Offset, user_acc=UAcc1, last_go=Go}, + Acc1 = Acc#mracc{meta_sent = true, offset = Offset, user_acc = UAcc1, last_go = Go}, case Go of ok -> map_fold(KV, OffsetReds, Acc1); stop -> {stop, Acc1} end; -map_fold(_KV, _Offset, #mracc{limit=0}=Acc) -> +map_fold(_KV, _Offset, #mracc{limit = 0} = Acc) -> {stop, Acc}; map_fold({{Key, Id}, Val}, _Offset, Acc) -> #mracc{ - db=Db, - limit=Limit, - doc_info=DI, - callback=Callback, - user_acc=UAcc0, - args=Args + db = Db, + limit = Limit, + doc_info = DI, + callback = Callback, + user_acc = UAcc0, + args = Args } = Acc, - Doc = case DI of - #doc_info{} -> couch_mrview_util:maybe_load_doc(Db, DI, Args); - _ -> couch_mrview_util:maybe_load_doc(Db, Id, Val, Args) - end, + Doc = + case DI of + #doc_info{} -> couch_mrview_util:maybe_load_doc(Db, DI, Args); + _ -> couch_mrview_util:maybe_load_doc(Db, Id, Val, Args) + end, Row = [{id, Id}, {key, Key}, {value, Val}] ++ Doc, {Go, UAcc1} = Callback({row, Row}, UAcc0), {Go, Acc#mracc{ - limit=Limit-1, - doc_info=undefined, - user_acc=UAcc1, - last_go=Go + limit = Limit - 1, + doc_info = undefined, + user_acc = UAcc1, + last_go = Go }}; map_fold(#doc{id = <<"_local/", _/binary>>} = Doc, _Offset, #mracc{} = Acc) -> #mracc{ - limit=Limit, - callback=Callback, - user_acc=UAcc0, - args=Args + limit = Limit, + callback = Callback, + user_acc = UAcc0, + args = Args } = Acc, #doc{ id = DocId, revs = {Pos, [RevId | _]} } = Doc, Rev = {Pos, RevId}, - Row = [ - {id, DocId}, - {key, DocId}, - {value, {[{rev, couch_doc:rev_to_str(Rev)}]}} - ] ++ if not Args#mrargs.include_docs -> []; true -> - [{doc, couch_doc:to_json_obj(Doc, Args#mrargs.doc_options)}] - end, + Row = + [ + {id, DocId}, + {key, DocId}, + {value, {[{rev, couch_doc:rev_to_str(Rev)}]}} + ] ++ + if + not Args#mrargs.include_docs -> []; + true -> [{doc, couch_doc:to_json_obj(Doc, Args#mrargs.doc_options)}] + end, {Go, UAcc1} = Callback({row, Row}, UAcc0), {Go, Acc#mracc{ - limit=Limit-1, - reduce_fun=undefined, - doc_info=undefined, - user_acc=UAcc1, - last_go=Go + limit = Limit - 1, + reduce_fun = undefined, + doc_info = undefined, + user_acc = UAcc1, + last_go = Go }}. -red_fold(Db, {NthRed, _Lang, View}=RedView, Args, Callback, UAcc) -> - Finalizer = case couch_util:get_value(finalizer, Args#mrargs.extra) of - undefined -> - {_, FunSrc} = lists:nth(NthRed, View#mrview.reduce_funs), - FunSrc; - CustomFun-> - CustomFun - end, +red_fold(Db, {NthRed, _Lang, View} = RedView, Args, Callback, UAcc) -> + Finalizer = + case couch_util:get_value(finalizer, Args#mrargs.extra) of + undefined -> + {_, FunSrc} = lists:nth(NthRed, View#mrview.reduce_funs), + FunSrc; + CustomFun -> + CustomFun + end, Acc = #mracc{ - db=Db, - total_rows=null, - limit=Args#mrargs.limit, - skip=Args#mrargs.skip, - group_level=Args#mrargs.group_level, - callback=Callback, - user_acc=UAcc, - update_seq=View#mrview.update_seq, - finalizer=Finalizer, - args=Args + db = Db, + total_rows = null, + limit = Args#mrargs.limit, + skip = Args#mrargs.skip, + group_level = Args#mrargs.group_level, + callback = Callback, + user_acc = UAcc, + update_seq = View#mrview.update_seq, + finalizer = Finalizer, + args = Args }, Grouping = {key_group_level, Args#mrargs.group_level}, OptList = couch_mrview_util:key_opts(Args, [Grouping]), - Acc2 = lists:foldl(fun(Opts, Acc0) -> - {ok, Acc1} = - couch_mrview_util:fold_reduce(RedView, fun red_fold/3, Acc0, Opts), - Acc1 - end, Acc, OptList), + Acc2 = lists:foldl( + fun(Opts, Acc0) -> + {ok, Acc1} = + couch_mrview_util:fold_reduce(RedView, fun red_fold/3, Acc0, Opts), + Acc1 + end, + Acc, + OptList + ), finish_fold(Acc2, []). red_fold({p, _Partition, Key}, Red, Acc) -> red_fold(Key, Red, Acc); -red_fold(_Key, _Red, #mracc{skip=N}=Acc) when N > 0 -> - {ok, Acc#mracc{skip=N-1, last_go=ok}}; -red_fold(Key, Red, #mracc{meta_sent=false}=Acc) -> +red_fold(_Key, _Red, #mracc{skip = N} = Acc) when N > 0 -> + {ok, Acc#mracc{skip = N - 1, last_go = ok}}; +red_fold(Key, Red, #mracc{meta_sent = false} = Acc) -> #mracc{ - args=Args, - callback=Callback, - user_acc=UAcc0, - update_seq=UpdateSeq + args = Args, + callback = Callback, + user_acc = UAcc0, + update_seq = UpdateSeq } = Acc, Meta = make_meta(Args, UpdateSeq, []), {Go, UAcc1} = Callback(Meta, UAcc0), - Acc1 = Acc#mracc{user_acc=UAcc1, meta_sent=true, last_go=Go}, + Acc1 = Acc#mracc{user_acc = UAcc1, meta_sent = true, last_go = Go}, case Go of ok -> red_fold(Key, Red, Acc1); _ -> {Go, Acc1} end; -red_fold(_Key, _Red, #mracc{limit=0} = Acc) -> +red_fold(_Key, _Red, #mracc{limit = 0} = Acc) -> {stop, Acc}; -red_fold(_Key, Red, #mracc{group_level=0} = Acc) -> +red_fold(_Key, Red, #mracc{group_level = 0} = Acc) -> #mracc{ - finalizer=Finalizer, - limit=Limit, - callback=Callback, - user_acc=UAcc0 + finalizer = Finalizer, + limit = Limit, + callback = Callback, + user_acc = UAcc0 } = Acc, Row = [{key, null}, {value, maybe_finalize(Red, Finalizer)}], {Go, UAcc1} = Callback({row, Row}, UAcc0), - {Go, Acc#mracc{user_acc=UAcc1, limit=Limit-1, last_go=Go}}; -red_fold(Key, Red, #mracc{group_level=exact} = Acc) -> + {Go, Acc#mracc{user_acc = UAcc1, limit = Limit - 1, last_go = Go}}; +red_fold(Key, Red, #mracc{group_level = exact} = Acc) -> #mracc{ - finalizer=Finalizer, - limit=Limit, - callback=Callback, - user_acc=UAcc0 + finalizer = Finalizer, + limit = Limit, + callback = Callback, + user_acc = UAcc0 } = Acc, Row = [{key, Key}, {value, maybe_finalize(Red, Finalizer)}], {Go, UAcc1} = Callback({row, Row}, UAcc0), - {Go, Acc#mracc{user_acc=UAcc1, limit=Limit-1, last_go=Go}}; -red_fold(K, Red, #mracc{group_level=I} = Acc) when I > 0, is_list(K) -> + {Go, Acc#mracc{user_acc = UAcc1, limit = Limit - 1, last_go = Go}}; +red_fold(K, Red, #mracc{group_level = I} = Acc) when I > 0, is_list(K) -> #mracc{ - finalizer=Finalizer, - limit=Limit, - callback=Callback, - user_acc=UAcc0 + finalizer = Finalizer, + limit = Limit, + callback = Callback, + user_acc = UAcc0 } = Acc, Row = [{key, lists:sublist(K, I)}, {value, maybe_finalize(Red, Finalizer)}], {Go, UAcc1} = Callback({row, Row}, UAcc0), - {Go, Acc#mracc{user_acc=UAcc1, limit=Limit-1, last_go=Go}}; -red_fold(K, Red, #mracc{group_level=I} = Acc) when I > 0 -> + {Go, Acc#mracc{user_acc = UAcc1, limit = Limit - 1, last_go = Go}}; +red_fold(K, Red, #mracc{group_level = I} = Acc) when I > 0 -> #mracc{ - finalizer=Finalizer, - limit=Limit, - callback=Callback, - user_acc=UAcc0 + finalizer = Finalizer, + limit = Limit, + callback = Callback, + user_acc = UAcc0 } = Acc, Row = [{key, K}, {value, maybe_finalize(Red, Finalizer)}], {Go, UAcc1} = Callback({row, Row}, UAcc0), - {Go, Acc#mracc{user_acc=UAcc1, limit=Limit-1, last_go=Go}}. + {Go, Acc#mracc{user_acc = UAcc1, limit = Limit - 1, last_go = Go}}. maybe_finalize(Red, null) -> Red; @@ -619,31 +659,31 @@ maybe_finalize(Red, RedSrc) -> {ok, Finalized} = couch_query_servers:finalize(RedSrc, Red), Finalized. -finish_fold(#mracc{last_go=ok, update_seq=UpdateSeq}=Acc, ExtraMeta) -> - #mracc{callback=Callback, user_acc=UAcc, args=Args}=Acc, +finish_fold(#mracc{last_go = ok, update_seq = UpdateSeq} = Acc, ExtraMeta) -> + #mracc{callback = Callback, user_acc = UAcc, args = Args} = Acc, % Possible send meta info Meta = make_meta(Args, UpdateSeq, ExtraMeta), - {Go, UAcc1} = case Acc#mracc.meta_sent of - false -> Callback(Meta, UAcc); - _ -> {ok, Acc#mracc.user_acc} - end, + {Go, UAcc1} = + case Acc#mracc.meta_sent of + false -> Callback(Meta, UAcc); + _ -> {ok, Acc#mracc.user_acc} + end, % Notify callback that the fold is complete. - {_, UAcc2} = case Go of - ok -> Callback(complete, UAcc1); - _ -> {ok, UAcc1} - end, + {_, UAcc2} = + case Go of + ok -> Callback(complete, UAcc1); + _ -> {ok, UAcc1} + end, {ok, UAcc2}; -finish_fold(#mracc{user_acc=UAcc}, _ExtraMeta) -> +finish_fold(#mracc{user_acc = UAcc}, _ExtraMeta) -> {ok, UAcc}. - make_meta(Args, UpdateSeq, Base) -> case Args#mrargs.update_seq of true -> {meta, Base ++ [{update_seq, UpdateSeq}]}; _ -> {meta, Base} end. - get_reduce_fun(#mrargs{extra = Extra}) -> case couch_util:get_value(namespace, Extra) of <<"_local">> -> @@ -652,7 +692,6 @@ get_reduce_fun(#mrargs{extra = Extra}) -> fun couch_mrview_util:all_docs_reduce_to_count/1 end. - get_total_rows(Db, #mrargs{extra = Extra}) -> case couch_util:get_value(namespace, Extra) of <<"_local">> -> @@ -665,7 +704,6 @@ get_total_rows(Db, #mrargs{extra = Extra}) -> couch_util:get_value(doc_count, Info) end. - get_update_seq(Db, #mrargs{extra = Extra}) -> case couch_util:get_value(namespace, Extra) of <<"_local">> -> @@ -674,7 +712,6 @@ get_update_seq(Db, #mrargs{extra = Extra}) -> couch_db:get_update_seq(Db) end. - default_cb(complete, Acc) -> {ok, lists:reverse(Acc)}; default_cb({final, Info}, []) -> @@ -686,13 +723,15 @@ default_cb(ok, ddoc_updated) -> default_cb(Row, Acc) -> {ok, [Row | Acc]}. - to_mrargs(KeyList) -> - lists:foldl(fun({Key, Value}, Acc) -> - Index = lookup_index(couch_util:to_existing_atom(Key)), - setelement(Index, Acc, Value) - end, #mrargs{}, KeyList). - + lists:foldl( + fun({Key, Value}, Acc) -> + Index = lookup_index(couch_util:to_existing_atom(Key)), + setelement(Index, Acc, Value) + end, + #mrargs{}, + KeyList + ). lookup_index(Key) -> Index = lists:zip( diff --git a/src/couch_mrview/src/couch_mrview_cleanup.erl b/src/couch_mrview/src/couch_mrview_cleanup.erl index e0cb1c64f..417605c55 100644 --- a/src/couch_mrview/src/couch_mrview_cleanup.erl +++ b/src/couch_mrview/src/couch_mrview_cleanup.erl @@ -14,24 +14,26 @@ -export([run/1]). - -include_lib("couch/include/couch_db.hrl"). -include_lib("couch_mrview/include/couch_mrview.hrl"). - run(Db) -> RootDir = couch_index_util:root_dir(), DbName = couch_db:name(Db), {ok, DesignDocs} = couch_db:get_design_docs(Db), - SigFiles = lists:foldl(fun(DDocInfo, SFAcc) -> - {ok, DDoc} = couch_db:open_doc_int(Db, DDocInfo, [ejson_body]), - {ok, InitState} = couch_mrview_util:ddoc_to_mrst(DbName, DDoc), - Sig = InitState#mrst.sig, - IFName = couch_mrview_util:index_file(DbName, Sig), - CFName = couch_mrview_util:compaction_file(DbName, Sig), - [IFName, CFName | SFAcc] - end, [], [DD || DD <- DesignDocs, DD#full_doc_info.deleted == false]), + SigFiles = lists:foldl( + fun(DDocInfo, SFAcc) -> + {ok, DDoc} = couch_db:open_doc_int(Db, DDocInfo, [ejson_body]), + {ok, InitState} = couch_mrview_util:ddoc_to_mrst(DbName, DDoc), + Sig = InitState#mrst.sig, + IFName = couch_mrview_util:index_file(DbName, Sig), + CFName = couch_mrview_util:compaction_file(DbName, Sig), + [IFName, CFName | SFAcc] + end, + [], + [DD || DD <- DesignDocs, DD#full_doc_info.deleted == false] + ), IdxDir = couch_index_util:index_dir(mrview, DbName), DiskFiles = filelib:wildcard(filename:join(IdxDir, "*")), @@ -39,21 +41,28 @@ run(Db) -> % We need to delete files that have no ddoc. ToDelete = DiskFiles -- SigFiles, - lists:foreach(fun(FN) -> - couch_log:debug("Deleting stale view file: ~s", [FN]), - couch_file:delete(RootDir, FN, [sync]), - case couch_mrview_util:verify_view_filename(FN) of - true -> - Sig = couch_mrview_util:get_signature_from_filename(FN), - DocId = couch_mrview_util:get_local_purge_doc_id(Sig), - case couch_db:open_doc(Db, DocId, []) of - {ok, LocalPurgeDoc} -> - couch_db:update_doc(Db, - LocalPurgeDoc#doc{deleted=true}, [?ADMIN_CTX]); - {not_found, _} -> - ok - end; - false -> ok - end - end, ToDelete), + lists:foreach( + fun(FN) -> + couch_log:debug("Deleting stale view file: ~s", [FN]), + couch_file:delete(RootDir, FN, [sync]), + case couch_mrview_util:verify_view_filename(FN) of + true -> + Sig = couch_mrview_util:get_signature_from_filename(FN), + DocId = couch_mrview_util:get_local_purge_doc_id(Sig), + case couch_db:open_doc(Db, DocId, []) of + {ok, LocalPurgeDoc} -> + couch_db:update_doc( + Db, + LocalPurgeDoc#doc{deleted = true}, + [?ADMIN_CTX] + ); + {not_found, _} -> + ok + end; + false -> + ok + end + end, + ToDelete + ), ok. diff --git a/src/couch_mrview/src/couch_mrview_compactor.erl b/src/couch_mrview/src/couch_mrview_compactor.erl index d42edc054..28e5a9b3d 100644 --- a/src/couch_mrview/src/couch_mrview_compactor.erl +++ b/src/couch_mrview/src/couch_mrview_compactor.erl @@ -18,12 +18,12 @@ -export([compact/3, swap_compacted/2, remove_compacted/1]). -record(acc, { - btree = nil, - last_id = nil, - kvs = [], - kvs_size = 0, - changes = 0, - total_changes + btree = nil, + last_id = nil, + kvs = [], + kvs_size = 0, + changes = 0, + total_changes }). -define(DEFAULT_RECOMPACT_RETRY_COUNT, 3). @@ -36,12 +36,12 @@ compact(_Db, State, Opts) -> compact(State) -> #mrst{ - db_name=DbName, - idx_name=IdxName, - sig=Sig, - update_seq=Seq, - id_btree=IdBtree, - views=Views + db_name = DbName, + idx_name = IdxName, + sig = Sig, + update_seq = Seq, + id_btree = IdBtree, + views = Views } = State, erlang:put(io_priority, {view_compact, DbName, IdxName}), @@ -65,7 +65,9 @@ compact(State) -> {ok, Kvs} = couch_mrview_util:get_row_count(View), Acc + Kvs end, - NumDocIds, Views), + NumDocIds, + Views + ), couch_task_status:add_task([ {type, view_compaction}, @@ -83,24 +85,29 @@ compact(State) -> FoldFun = fun({DocId, ViewIdKeys} = KV, Acc) -> #acc{btree = Bt, kvs = Kvs, kvs_size = KvsSize} = Acc, - NewKvs = case Kvs of - [{DocId, OldViewIdKeys} | Rest] -> - couch_log:error("Dupes of ~s in ~s ~s", - [DocId, DbName, IdxName]), - [{DocId, ViewIdKeys ++ OldViewIdKeys} | Rest]; - _ -> - [KV | Kvs] - end, + NewKvs = + case Kvs of + [{DocId, OldViewIdKeys} | Rest] -> + couch_log:error( + "Dupes of ~s in ~s ~s", + [DocId, DbName, IdxName] + ), + [{DocId, ViewIdKeys ++ OldViewIdKeys} | Rest]; + _ -> + [KV | Kvs] + end, KvsSize2 = KvsSize + ?term_size(KV), case KvsSize2 >= BufferSize of true -> {ok, Bt2} = couch_btree:add(Bt, lists:reverse(NewKvs)), Acc2 = update_task(Acc, length(NewKvs)), {ok, Acc2#acc{ - btree = Bt2, kvs = [], kvs_size = 0, last_id = DocId}}; + btree = Bt2, kvs = [], kvs_size = 0, last_id = DocId + }}; _ -> {ok, Acc#acc{ - kvs = NewKvs, kvs_size = KvsSize2, last_id = DocId}} + kvs = NewKvs, kvs_size = KvsSize2, last_id = DocId + }} end end, @@ -110,26 +117,26 @@ compact(State) -> {ok, NewIdBtree} = couch_btree:add(Bt3, lists:reverse(Uncopied)), FinalAcc2 = update_task(FinalAcc, length(Uncopied)), - - {NewViews, _} = lists:mapfoldl(fun({View, EmptyView}, Acc) -> - compact_view(View, EmptyView, BufferSize, Acc) - end, FinalAcc2, lists:zip(Views, EmptyViews)), + {NewViews, _} = lists:mapfoldl( + fun({View, EmptyView}, Acc) -> + compact_view(View, EmptyView, BufferSize, Acc) + end, + FinalAcc2, + lists:zip(Views, EmptyViews) + ), unlink(EmptyState#mrst.fd), {ok, EmptyState#mrst{ - id_btree=NewIdBtree, - views=NewViews, - update_seq=Seq + id_btree = NewIdBtree, + views = NewViews, + update_seq = Seq }}. - recompact(State) -> recompact(State, recompact_retry_count()). -recompact(#mrst{db_name=DbName, idx_name=IdxName}, 0) -> - erlang:error({exceeded_recompact_retry_count, - [{db_name, DbName}, {idx_name, IdxName}]}); - +recompact(#mrst{db_name = DbName, idx_name = IdxName}, 0) -> + erlang:error({exceeded_recompact_retry_count, [{db_name, DbName}, {idx_name, IdxName}]}); recompact(State, RetryCount) -> Self = self(), link(State#mrst.fd), @@ -159,27 +166,35 @@ recompact_retry_count() -> ?DEFAULT_RECOMPACT_RETRY_COUNT ). - %% @spec compact_view(View, EmptyView, Retry, Acc) -> {CompactView, NewAcc} -compact_view(#mrview{id_num=VID}=View, EmptyView, BufferSize, Acc0) -> - - {NewBt, FinalAcc} = compact_view_btree(View#mrview.btree, - EmptyView#mrview.btree, - VID, BufferSize, Acc0), +compact_view(#mrview{id_num = VID} = View, EmptyView, BufferSize, Acc0) -> + {NewBt, FinalAcc} = compact_view_btree( + View#mrview.btree, + EmptyView#mrview.btree, + VID, + BufferSize, + Acc0 + ), - {EmptyView#mrview{btree=NewBt, - update_seq=View#mrview.update_seq, - purge_seq=View#mrview.purge_seq}, FinalAcc}. + { + EmptyView#mrview{ + btree = NewBt, + update_seq = View#mrview.update_seq, + purge_seq = View#mrview.purge_seq + }, + FinalAcc + }. compact_view_btree(Btree, EmptyBtree, VID, BufferSize, Acc0) -> Fun = fun(KV, #acc{btree = Bt, kvs = Kvs, kvs_size = KvsSize} = Acc) -> KvsSize2 = KvsSize + ?term_size(KV), - if KvsSize2 >= BufferSize -> - {ok, Bt2} = couch_btree:add(Bt, lists:reverse([KV | Kvs])), - Acc2 = update_task(VID, Acc, 1 + length(Kvs)), - {ok, Acc2#acc{btree = Bt2, kvs = [], kvs_size = 0}}; - true -> - {ok, Acc#acc{kvs = [KV | Kvs], kvs_size = KvsSize2}} + if + KvsSize2 >= BufferSize -> + {ok, Bt2} = couch_btree:add(Bt, lists:reverse([KV | Kvs])), + Acc2 = update_task(VID, Acc, 1 + length(Kvs)), + {ok, Acc2#acc{btree = Bt2, kvs = [], kvs_size = 0}}; + true -> + {ok, Acc#acc{kvs = [KV | Kvs], kvs_size = KvsSize2}} end end, @@ -193,11 +208,18 @@ compact_view_btree(Btree, EmptyBtree, VID, BufferSize, Acc0) -> update_task(Acc, ChangesInc) -> update_task(null, Acc, ChangesInc). - -update_task(VID, #acc{changes=Changes, total_changes=Total}=Acc, ChangesInc) -> - Phase = if is_integer(VID) -> view; true -> ids end, +update_task(VID, #acc{changes = Changes, total_changes = Total} = Acc, ChangesInc) -> + Phase = + if + is_integer(VID) -> view; + true -> ids + end, Changes2 = Changes + ChangesInc, - Progress = if Total == 0 -> 0; true -> (Changes2 * 100) div Total end, + Progress = + if + Total == 0 -> 0; + true -> (Changes2 * 100) div Total + end, couch_task_status:update([ {phase, Phase}, {view, VID}, @@ -207,15 +229,14 @@ update_task(VID, #acc{changes=Changes, total_changes=Total}=Acc, ChangesInc) -> ]), Acc#acc{changes = Changes2}. - swap_compacted(OldState, NewState) -> #mrst{ fd = Fd } = OldState, #mrst{ - sig=Sig, - db_name=DbName, - fd=NewFd + sig = Sig, + db_name = DbName, + fd = NewFd } = NewState, link(NewState#mrst.fd), @@ -227,16 +248,18 @@ swap_compacted(OldState, NewState) -> {ok, Pre} = couch_file:bytes(Fd), {ok, Post} = couch_file:bytes(NewFd), - couch_log:notice("Compaction swap for view ~s ~p ~p", [IndexFName, - Pre, Post]), + couch_log:notice("Compaction swap for view ~s ~p ~p", [ + IndexFName, + Pre, + Post + ]), ok = couch_file:delete(RootDir, IndexFName), ok = file:rename(CompactFName, IndexFName), unlink(OldState#mrst.fd), erlang:demonitor(OldState#mrst.fd_monitor, [flush]), - {ok, NewState#mrst{fd_monitor=Ref}}. - + {ok, NewState#mrst{fd_monitor = Ref}}. remove_compacted(#mrst{sig = Sig, db_name = DbName} = State) -> RootDir = couch_index_util:root_dir(), @@ -244,7 +267,6 @@ remove_compacted(#mrst{sig = Sig, db_name = DbName} = State) -> ok = couch_file:delete(RootDir, CompactFName), {ok, State}. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -268,27 +290,28 @@ recompact_test_() -> recompact_success_after_progress() -> ?_test(begin - ok = meck:expect(couch_index_updater, update, fun - (Pid, _, #mrst{update_seq=0} = State) -> - Pid ! {'$gen_cast', {new_state, State#mrst{update_seq = 1}}}, - timer:sleep(100), - exit({updated, self(), State#mrst{update_seq = 2}}) + ok = meck:expect(couch_index_updater, update, fun(Pid, _, #mrst{update_seq = 0} = State) -> + Pid ! {'$gen_cast', {new_state, State#mrst{update_seq = 1}}}, + timer:sleep(100), + exit({updated, self(), State#mrst{update_seq = 2}}) end), - State = #mrst{fd=self(), update_seq=0}, + State = #mrst{fd = self(), update_seq = 0}, ?assertEqual({ok, State#mrst{update_seq = 2}}, recompact(State)) end). recompact_exceeded_retry_count() -> ?_test(begin - ok = meck:expect(couch_index_updater, update, + ok = meck:expect( + couch_index_updater, + update, fun(_, _, _) -> exit(error) - end), + end + ), ok = meck:expect(couch_log, warning, fun(_, _) -> ok end), - State = #mrst{fd=self(), db_name=foo, idx_name=bar}, - ExpectedError = {exceeded_recompact_retry_count, - [{db_name, foo}, {idx_name, bar}]}, - ?assertError(ExpectedError, recompact(State)) + State = #mrst{fd = self(), db_name = foo, idx_name = bar}, + ExpectedError = {exceeded_recompact_retry_count, [{db_name, foo}, {idx_name, bar}]}, + ?assertError(ExpectedError, recompact(State)) end). -endif. diff --git a/src/couch_mrview/src/couch_mrview_http.erl b/src/couch_mrview/src/couch_mrview_http.erl index 802739b82..fa3fab386 100644 --- a/src/couch_mrview/src/couch_mrview_http.erl +++ b/src/couch_mrview/src/couch_mrview_http.erl @@ -43,37 +43,41 @@ -include_lib("couch/include/couch_db.hrl"). -include_lib("couch_mrview/include/couch_mrview.hrl"). - -handle_all_docs_req(#httpd{method='GET'}=Req, Db) -> +handle_all_docs_req(#httpd{method = 'GET'} = Req, Db) -> all_docs_req(Req, Db, undefined); -handle_all_docs_req(#httpd{method='POST'}=Req, Db) -> +handle_all_docs_req(#httpd{method = 'POST'} = Req, Db) -> chttpd:validate_ctype(Req, "application/json"), Keys = couch_mrview_util:get_view_keys(chttpd:json_body_obj(Req)), all_docs_req(Req, Db, Keys); handle_all_docs_req(Req, _Db) -> chttpd:send_method_not_allowed(Req, "GET,POST,HEAD"). -handle_local_docs_req(#httpd{method='GET'}=Req, Db) -> +handle_local_docs_req(#httpd{method = 'GET'} = Req, Db) -> all_docs_req(Req, Db, undefined, <<"_local">>); -handle_local_docs_req(#httpd{method='POST'}=Req, Db) -> +handle_local_docs_req(#httpd{method = 'POST'} = Req, Db) -> chttpd:validate_ctype(Req, "application/json"), Keys = couch_mrview_util:get_view_keys(chttpd:json_body_obj(Req)), all_docs_req(Req, Db, Keys, <<"_local">>); handle_local_docs_req(Req, _Db) -> chttpd:send_method_not_allowed(Req, "GET,POST,HEAD"). -handle_design_docs_req(#httpd{method='GET'}=Req, Db) -> +handle_design_docs_req(#httpd{method = 'GET'} = Req, Db) -> all_docs_req(Req, Db, undefined, <<"_design">>); -handle_design_docs_req(#httpd{method='POST'}=Req, Db) -> +handle_design_docs_req(#httpd{method = 'POST'} = Req, Db) -> chttpd:validate_ctype(Req, "application/json"), Keys = couch_mrview_util:get_view_keys(chttpd:json_body_obj(Req)), all_docs_req(Req, Db, Keys, <<"_design">>); handle_design_docs_req(Req, _Db) -> chttpd:send_method_not_allowed(Req, "GET,POST,HEAD"). -handle_reindex_req(#httpd{method='POST', - path_parts=[_, _, DName,<<"_reindex">>]}=Req, - Db, _DDoc) -> +handle_reindex_req( + #httpd{ + method = 'POST', + path_parts = [_, _, DName, <<"_reindex">>] + } = Req, + Db, + _DDoc +) -> chttpd:validate_ctype(Req, "application/json"), ok = couch_db:check_is_admin(Db), couch_mrview:trigger_update(Db, <<"_design/", DName/binary>>), @@ -81,23 +85,30 @@ handle_reindex_req(#httpd{method='POST', handle_reindex_req(Req, _Db, _DDoc) -> chttpd:send_method_not_allowed(Req, "POST"). - -handle_view_req(#httpd{method='GET', - path_parts=[_, _, DDocName, _, VName, <<"_info">>]}=Req, - Db, _DDoc) -> +handle_view_req( + #httpd{ + method = 'GET', + path_parts = [_, _, DDocName, _, VName, <<"_info">>] + } = Req, + Db, + _DDoc +) -> DbName = couch_db:name(Db), - DDocId = <<"_design/", DDocName/binary >>, + DDocId = <<"_design/", DDocName/binary>>, {ok, Info} = couch_mrview:get_view_info(DbName, DDocId, VName), - FinalInfo = [{db_name, DbName}, - {ddoc, DDocId}, - {view, VName}] ++ Info, + FinalInfo = + [ + {db_name, DbName}, + {ddoc, DDocId}, + {view, VName} + ] ++ Info, chttpd:send_json(Req, 200, {FinalInfo}); -handle_view_req(#httpd{method='GET'}=Req, Db, DDoc) -> +handle_view_req(#httpd{method = 'GET'} = Req, Db, DDoc) -> [_, _, _, _, ViewName] = Req#httpd.path_parts, couch_stats:increment_counter([couchdb, httpd, view_reads]), design_doc_view(Req, Db, DDoc, ViewName, undefined); -handle_view_req(#httpd{method='POST'}=Req, Db, DDoc) -> +handle_view_req(#httpd{method = 'POST'} = Req, Db, DDoc) -> chttpd:validate_ctype(Req, "application/json"), [_, _, _, _, ViewName] = Req#httpd.path_parts, Props = chttpd:json_body_obj(Req), @@ -122,8 +133,7 @@ handle_view_req(#httpd{method='POST'}=Req, Db, DDoc) -> handle_view_req(Req, _Db, _DDoc) -> chttpd:send_method_not_allowed(Req, "GET,POST,HEAD"). - -handle_temp_view_req(#httpd{method='POST'}=Req, Db) -> +handle_temp_view_req(#httpd{method = 'POST'} = Req, Db) -> chttpd:validate_ctype(Req, "application/json"), ok = couch_db:check_is_admin(Db), {Body} = chttpd:json_body_obj(Req), @@ -134,19 +144,21 @@ handle_temp_view_req(#httpd{method='POST'}=Req, Db) -> handle_temp_view_req(Req, _Db) -> chttpd:send_method_not_allowed(Req, "POST"). - -handle_info_req(#httpd{method='GET'}=Req, Db, DDoc) -> +handle_info_req(#httpd{method = 'GET'} = Req, Db, DDoc) -> [_, _, Name, _] = Req#httpd.path_parts, {ok, Info} = couch_mrview:get_info(Db, DDoc), - chttpd:send_json(Req, 200, {[ - {name, Name}, - {view_index, {Info}} - ]}); + chttpd:send_json( + Req, + 200, + {[ + {name, Name}, + {view_index, {Info}} + ]} + ); handle_info_req(Req, _Db, _DDoc) -> chttpd:send_method_not_allowed(Req, "GET"). - -handle_compact_req(#httpd{method='POST'}=Req, Db, DDoc) -> +handle_compact_req(#httpd{method = 'POST'} = Req, Db, DDoc) -> chttpd:validate_ctype(Req, "application/json"), ok = couch_db:check_is_admin(Db), ok = couch_mrview:compact(Db, DDoc), @@ -154,8 +166,7 @@ handle_compact_req(#httpd{method='POST'}=Req, Db, DDoc) -> handle_compact_req(Req, _Db, _DDoc) -> chttpd:send_method_not_allowed(Req, "POST"). - -handle_cleanup_req(#httpd{method='POST'}=Req, Db) -> +handle_cleanup_req(#httpd{method = 'POST'} = Req, Db) -> chttpd:validate_ctype(Req, "application/json"), ok = couch_db:check_is_admin(Db), ok = couch_mrview:cleanup(Db), @@ -163,29 +174,30 @@ handle_cleanup_req(#httpd{method='POST'}=Req, Db) -> handle_cleanup_req(Req, _Db) -> chttpd:send_method_not_allowed(Req, "POST"). - all_docs_req(Req, Db, Keys) -> all_docs_req(Req, Db, Keys, undefined). all_docs_req(Req, Db, Keys, NS) -> case is_restricted(Db, NS) of - true -> - case (catch couch_db:check_is_admin(Db)) of - ok -> - do_all_docs_req(Req, Db, Keys, NS); - _ when NS == <<"_local">> -> - throw({forbidden, <<"Only admins can access _local_docs">>}); - _ -> - case is_public_fields_configured(Db) of - true -> + true -> + case (catch couch_db:check_is_admin(Db)) of + ok -> do_all_docs_req(Req, Db, Keys, NS); - false -> - throw({forbidden, <<"Only admins can access _all_docs", - " of system databases.">>}) - end - end; - false -> - do_all_docs_req(Req, Db, Keys, NS) + _ when NS == <<"_local">> -> + throw({forbidden, <<"Only admins can access _local_docs">>}); + _ -> + case is_public_fields_configured(Db) of + true -> + do_all_docs_req(Req, Db, Keys, NS); + false -> + throw( + {forbidden, + <<"Only admins can access _all_docs", " of system databases.">>} + ) + end + end; + false -> + do_all_docs_req(Req, Db, Keys, NS) end. is_restricted(_Db, <<"_local">>) -> @@ -196,18 +208,19 @@ is_restricted(Db, _) -> is_public_fields_configured(Db) -> DbName = ?b2l(couch_db:name(Db)), case config:get("couch_httpd_auth", "authentication_db", "_users") of - DbName -> - UsersDbPublic = chttpd_util:get_chttpd_auth_config( - "users_db_public", "false"), - PublicFields = chttpd_util:get_chttpd_auth_config("public_fields"), - case {UsersDbPublic, PublicFields} of - {"true", PublicFields} when PublicFields =/= undefined -> - true; - {_, _} -> + DbName -> + UsersDbPublic = chttpd_util:get_chttpd_auth_config( + "users_db_public", "false" + ), + PublicFields = chttpd_util:get_chttpd_auth_config("public_fields"), + case {UsersDbPublic, PublicFields} of + {"true", PublicFields} when PublicFields =/= undefined -> + true; + {_, _} -> + false + end; + _ -> false - end; - _ -> - false end. do_all_docs_req(Req, Db, Keys, NS) -> @@ -216,14 +229,16 @@ do_all_docs_req(Req, Db, Keys, NS) -> ETagFun = fun(Sig, Acc0) -> check_view_etag(Sig, Acc0, Req) end, - Args = Args1#mrargs{preflight_fun=ETagFun}, + Args = Args1#mrargs{preflight_fun = ETagFun}, {ok, Resp} = couch_httpd:etag_maybe(Req, fun() -> Max = chttpd:chunked_response_buffer_size(), - VAcc0 = #vacc{db=Db, req=Req, threshold=Max}, + VAcc0 = #vacc{db = Db, req = Req, threshold = Max}, DbName = ?b2l(couch_db:name(Db)), - UsersDbName = config:get("couch_httpd_auth", - "authentication_db", - "_users"), + UsersDbName = config:get( + "couch_httpd_auth", + "authentication_db", + "_users" + ), IsAdmin = is_admin(Db), Callback = get_view_callback(DbName, UsersDbName, IsAdmin), couch_mrview:query_all_docs(Db, Args, Callback, VAcc0) @@ -238,13 +253,12 @@ set_namespace(NS, #mrargs{extra = Extra} = Args) -> is_admin(Db) -> case catch couch_db:check_is_admin(Db) of - {unauthorized, _} -> - false; - ok -> - true + {unauthorized, _} -> + false; + ok -> + true end. - % admin users always get all fields get_view_callback(_, _, true) -> fun view_cb/2; @@ -256,16 +270,15 @@ get_view_callback(_DbName, _DbName, false) -> get_view_callback(_, _, _) -> fun view_cb/2. - design_doc_view(Req, Db, DDoc, ViewName, Keys) -> Args0 = parse_params(Req, Keys), ETagFun = fun(Sig, Acc0) -> check_view_etag(Sig, Acc0, Req) end, - Args = Args0#mrargs{preflight_fun=ETagFun}, + Args = Args0#mrargs{preflight_fun = ETagFun}, {ok, Resp} = couch_httpd:etag_maybe(Req, fun() -> Max = chttpd:chunked_response_buffer_size(), - VAcc0 = #vacc{db=Db, req=Req, threshold=Max}, + VAcc0 = #vacc{db = Db, req = Req, threshold = Max}, couch_mrview:query_view(Db, DDoc, ViewName, Args, fun view_cb/2, VAcc0) end), case is_record(Resp, vacc) of @@ -273,30 +286,36 @@ design_doc_view(Req, Db, DDoc, ViewName, Keys) -> _ -> {ok, Resp} end. - multi_query_view(Req, Db, DDoc, ViewName, Queries) -> Args0 = parse_params(Req, undefined), {ok, _, _, Args1} = couch_mrview_util:get_view(Db, DDoc, ViewName, Args0), - ArgQueries = lists:map(fun({Query}) -> - QueryArg = parse_params(Query, undefined, Args1), - couch_mrview_util:validate_args(Db, DDoc, QueryArg) - end, Queries), + ArgQueries = lists:map( + fun({Query}) -> + QueryArg = parse_params(Query, undefined, Args1), + couch_mrview_util:validate_args(Db, DDoc, QueryArg) + end, + Queries + ), {ok, Resp2} = couch_httpd:etag_maybe(Req, fun() -> Max = chttpd:chunked_response_buffer_size(), - VAcc0 = #vacc{db=Db, req=Req, prepend="\r\n", threshold=Max}, + VAcc0 = #vacc{db = Db, req = Req, prepend = "\r\n", threshold = Max}, %% TODO: proper calculation of etag Etag = [$", couch_uuids:new(), $"], Headers = [{"ETag", Etag}], FirstChunk = "{\"results\":[", {ok, Resp0} = chttpd:start_delayed_json_response(VAcc0#vacc.req, 200, Headers, FirstChunk), - VAcc1 = VAcc0#vacc{resp=Resp0}, - VAcc2 = lists:foldl(fun(Args, Acc0) -> - {ok, Acc1} = couch_mrview:query_view(Db, DDoc, ViewName, Args, fun view_cb/2, Acc0), - Acc1 - end, VAcc1, ArgQueries), + VAcc1 = VAcc0#vacc{resp = Resp0}, + VAcc2 = lists:foldl( + fun(Args, Acc0) -> + {ok, Acc1} = couch_mrview:query_view(Db, DDoc, ViewName, Args, fun view_cb/2, Acc0), + Acc1 + end, + VAcc1, + ArgQueries + ), {ok, Resp1} = chttpd:send_delayed_chunk(VAcc2#vacc.resp, "\r\n]}"), {ok, Resp2} = chttpd:end_delayed_json_response(Resp1), - {ok, VAcc2#vacc{resp=Resp2}} + {ok, VAcc2#vacc{resp = Resp2}} end), case is_record(Resp2, vacc) of true -> {ok, Resp2#vacc.resp}; @@ -304,94 +323,99 @@ multi_query_view(Req, Db, DDoc, ViewName, Queries) -> end. filtered_view_cb({row, Row0}, Acc) -> - Row1 = lists:map(fun({doc, null}) -> - {doc, null}; - ({doc, Body}) -> - Doc = couch_users_db:strip_non_public_fields(#doc{body=Body}), - {doc, Doc#doc.body}; - (KV) -> - KV - end, Row0), + Row1 = lists:map( + fun + ({doc, null}) -> + {doc, null}; + ({doc, Body}) -> + Doc = couch_users_db:strip_non_public_fields(#doc{body = Body}), + {doc, Doc#doc.body}; + (KV) -> + KV + end, + Row0 + ), view_cb({row, Row1}, Acc); filtered_view_cb(Obj, Acc) -> view_cb(Obj, Acc). - %% these clauses start (and possibly end) the response -view_cb({error, Reason}, #vacc{resp=undefined}=Acc) -> +view_cb({error, Reason}, #vacc{resp = undefined} = Acc) -> {ok, Resp} = chttpd:send_error(Acc#vacc.req, Reason), - {ok, Acc#vacc{resp=Resp}}; - -view_cb(complete, #vacc{resp=undefined}=Acc) -> + {ok, Acc#vacc{resp = Resp}}; +view_cb(complete, #vacc{resp = undefined} = Acc) -> % Nothing in view {ok, Resp} = chttpd:send_json(Acc#vacc.req, 200, {[{rows, []}]}), - {ok, Acc#vacc{resp=Resp}}; - -view_cb(Msg, #vacc{resp=undefined}=Acc) -> + {ok, Acc#vacc{resp = Resp}}; +view_cb(Msg, #vacc{resp = undefined} = Acc) -> %% Start response Headers = [], {ok, Resp} = chttpd:start_delayed_json_response(Acc#vacc.req, 200, Headers), - view_cb(Msg, Acc#vacc{resp=Resp, should_close=true}); - + view_cb(Msg, Acc#vacc{resp = Resp, should_close = true}); %% --------------------------------------------------- %% From here on down, the response has been started. -view_cb({error, Reason}, #vacc{resp=Resp}=Acc) -> +view_cb({error, Reason}, #vacc{resp = Resp} = Acc) -> {ok, Resp1} = chttpd:send_delayed_error(Resp, Reason), - {ok, Acc#vacc{resp=Resp1}}; - -view_cb(complete, #vacc{resp=Resp, buffer=Buf, threshold=Max}=Acc) -> + {ok, Acc#vacc{resp = Resp1}}; +view_cb(complete, #vacc{resp = Resp, buffer = Buf, threshold = Max} = Acc) -> % Finish view output and possibly end the response {ok, Resp1} = chttpd:close_delayed_json_object(Resp, Buf, "\r\n]}", Max), case Acc#vacc.should_close of true -> {ok, Resp2} = chttpd:end_delayed_json_response(Resp1), - {ok, Acc#vacc{resp=Resp2}}; + {ok, Acc#vacc{resp = Resp2}}; _ -> - {ok, Acc#vacc{resp=Resp1, meta_sent=false, row_sent=false, - prepend=",\r\n", buffer=[], bufsize=0}} + {ok, Acc#vacc{ + resp = Resp1, + meta_sent = false, + row_sent = false, + prepend = ",\r\n", + buffer = [], + bufsize = 0 + }} end; - -view_cb({meta, Meta}, #vacc{meta_sent=false, row_sent=false}=Acc) -> +view_cb({meta, Meta}, #vacc{meta_sent = false, row_sent = false} = Acc) -> % Sending metadata as we've not sent it or any row yet - Parts = case couch_util:get_value(total, Meta) of - undefined -> []; - Total -> [io_lib:format("\"total_rows\":~p", [Total])] - end ++ case couch_util:get_value(offset, Meta) of - undefined -> []; - Offset -> [io_lib:format("\"offset\":~p", [Offset])] - end ++ case couch_util:get_value(update_seq, Meta) of - undefined -> []; - null -> - ["\"update_seq\":null"]; - UpdateSeq when is_integer(UpdateSeq) -> - [io_lib:format("\"update_seq\":~B", [UpdateSeq])]; - UpdateSeq when is_binary(UpdateSeq) -> - [io_lib:format("\"update_seq\":\"~s\"", [UpdateSeq])] - end ++ ["\"rows\":["], + Parts = + case couch_util:get_value(total, Meta) of + undefined -> []; + Total -> [io_lib:format("\"total_rows\":~p", [Total])] + end ++ + case couch_util:get_value(offset, Meta) of + undefined -> []; + Offset -> [io_lib:format("\"offset\":~p", [Offset])] + end ++ + case couch_util:get_value(update_seq, Meta) of + undefined -> + []; + null -> + ["\"update_seq\":null"]; + UpdateSeq when is_integer(UpdateSeq) -> + [io_lib:format("\"update_seq\":~B", [UpdateSeq])]; + UpdateSeq when is_binary(UpdateSeq) -> + [io_lib:format("\"update_seq\":\"~s\"", [UpdateSeq])] + end ++ ["\"rows\":["], Chunk = [prepend_val(Acc), "{", string:join(Parts, ","), "\r\n"], {ok, AccOut} = maybe_flush_response(Acc, Chunk, iolist_size(Chunk)), - {ok, AccOut#vacc{prepend="", meta_sent=true}}; - -view_cb({meta, _Meta}, #vacc{}=Acc) -> + {ok, AccOut#vacc{prepend = "", meta_sent = true}}; +view_cb({meta, _Meta}, #vacc{} = Acc) -> %% ignore metadata {ok, Acc}; - -view_cb({row, Row}, #vacc{meta_sent=false}=Acc) -> +view_cb({row, Row}, #vacc{meta_sent = false} = Acc) -> %% sorted=false and row arrived before meta % Adding another row Chunk = [prepend_val(Acc), "{\"rows\":[\r\n", row_to_json(Row)], - maybe_flush_response(Acc#vacc{meta_sent=true, row_sent=true}, Chunk, iolist_size(Chunk)); - -view_cb({row, Row}, #vacc{meta_sent=true}=Acc) -> + maybe_flush_response(Acc#vacc{meta_sent = true, row_sent = true}, Chunk, iolist_size(Chunk)); +view_cb({row, Row}, #vacc{meta_sent = true} = Acc) -> % Adding another row Chunk = [prepend_val(Acc), row_to_json(Row)], - maybe_flush_response(Acc#vacc{row_sent=true}, Chunk, iolist_size(Chunk)). + maybe_flush_response(Acc#vacc{row_sent = true}, Chunk, iolist_size(Chunk)). - -maybe_flush_response(#vacc{bufsize=Size, threshold=Max} = Acc, Data, Len) - when Size > 0 andalso (Size + Len) > Max -> +maybe_flush_response(#vacc{bufsize = Size, threshold = Max} = Acc, Data, Len) when + Size > 0 andalso (Size + Len) > Max +-> #vacc{buffer = Buffer, resp = Resp} = Acc, {ok, R1} = chttpd:send_delayed_chunk(Resp, Buffer), {ok, Acc#vacc{prepend = ",\r\n", buffer = Data, bufsize = Len, resp = R1}}; @@ -404,7 +428,7 @@ maybe_flush_response(Acc0, Data, Len) -> }, {ok, Acc}. -prepend_val(#vacc{prepend=Prepend}) -> +prepend_val(#vacc{prepend = Prepend}) -> case Prepend of undefined -> ""; @@ -412,72 +436,83 @@ prepend_val(#vacc{prepend=Prepend}) -> Prepend end. - row_to_json(Row) -> Id = couch_util:get_value(id, Row), row_to_json(Id, Row). - row_to_json(error, Row) -> % Special case for _all_docs request with KEYS to % match prior behavior. Key = couch_util:get_value(key, Row), Val = couch_util:get_value(value, Row), Reason = couch_util:get_value(reason, Row), - ReasonProp = if Reason == undefined -> []; true -> - [{reason, Reason}] - end, + ReasonProp = + if + Reason == undefined -> []; + true -> [{reason, Reason}] + end, Obj = {[{key, Key}, {error, Val}] ++ ReasonProp}, ?JSON_ENCODE(Obj); row_to_json(Id0, Row) -> - Id = case Id0 of - undefined -> []; - Id0 -> [{id, Id0}] - end, + Id = + case Id0 of + undefined -> []; + Id0 -> [{id, Id0}] + end, Key = couch_util:get_value(key, Row, null), Val = couch_util:get_value(value, Row), - Doc = case couch_util:get_value(doc, Row) of - undefined -> []; - Doc0 -> [{doc, Doc0}] - end, + Doc = + case couch_util:get_value(doc, Row) of + undefined -> []; + Doc0 -> [{doc, Doc0}] + end, Obj = {Id ++ [{key, Key}, {value, Val}] ++ Doc}, ?JSON_ENCODE(Obj). - -parse_params(#httpd{}=Req, Keys) -> +parse_params(#httpd{} = Req, Keys) -> parse_params(chttpd:qs(Req), Keys); parse_params(Props, Keys) -> Args = #mrargs{}, parse_params(Props, Keys, Args). - parse_params(Props, Keys, Args) -> parse_params(Props, Keys, Args, []). -parse_params(Props, Keys, #mrargs{}=Args0, Options) -> +parse_params(Props, Keys, #mrargs{} = Args0, Options) -> IsDecoded = lists:member(decoded, Options), - Args1 = case lists:member(keep_group_level, Options) of - true -> - Args0; - _ -> - % group_level set to undefined to detect if explicitly set by user - Args0#mrargs{keys=Keys, group=undefined, group_level=undefined} - end, - lists:foldl(fun({K, V}, Acc) -> - parse_param(K, V, Acc, IsDecoded) - end, Args1, Props). - - -parse_body_and_query(#httpd{method='POST'} = Req, Keys) -> + Args1 = + case lists:member(keep_group_level, Options) of + true -> + Args0; + _ -> + % group_level set to undefined to detect if explicitly set by user + Args0#mrargs{keys = Keys, group = undefined, group_level = undefined} + end, + lists:foldl( + fun({K, V}, Acc) -> + parse_param(K, V, Acc, IsDecoded) + end, + Args1, + Props + ). + +parse_body_and_query(#httpd{method = 'POST'} = Req, Keys) -> Props = chttpd:json_body_obj(Req), parse_body_and_query(Req, Props, Keys); - parse_body_and_query(Req, Keys) -> - parse_params(chttpd:qs(Req), Keys, #mrargs{keys=Keys, group=undefined, - group_level=undefined}, [keep_group_level]). + parse_params( + chttpd:qs(Req), + Keys, + #mrargs{ + keys = Keys, + group = undefined, + group_level = undefined + }, + [keep_group_level] + ). parse_body_and_query(Req, {Props}, Keys) -> - Args = #mrargs{keys=Keys, group=undefined, group_level=undefined}, + Args = #mrargs{keys = Keys, group = undefined, group_level = undefined}, BodyArgs = parse_params(Props, Keys, Args, [decoded]), parse_params(chttpd:qs(Req), Keys, BodyArgs, [keep_group_level]). @@ -488,101 +523,101 @@ parse_param(Key, Val, Args, IsDecoded) -> "" -> Args; "reduce" -> - Args#mrargs{reduce=parse_boolean(Val)}; + Args#mrargs{reduce = parse_boolean(Val)}; "key" when IsDecoded -> - Args#mrargs{start_key=Val, end_key=Val}; + Args#mrargs{start_key = Val, end_key = Val}; "key" -> JsonKey = ?JSON_DECODE(Val), - Args#mrargs{start_key=JsonKey, end_key=JsonKey}; + Args#mrargs{start_key = JsonKey, end_key = JsonKey}; "keys" when IsDecoded -> - Args#mrargs{keys=Val}; + Args#mrargs{keys = Val}; "keys" -> - Args#mrargs{keys=?JSON_DECODE(Val)}; + Args#mrargs{keys = ?JSON_DECODE(Val)}; "startkey" when IsDecoded -> - Args#mrargs{start_key=Val}; + Args#mrargs{start_key = Val}; "start_key" when IsDecoded -> - Args#mrargs{start_key=Val}; + Args#mrargs{start_key = Val}; "startkey" -> - Args#mrargs{start_key=?JSON_DECODE(Val)}; + Args#mrargs{start_key = ?JSON_DECODE(Val)}; "start_key" -> - Args#mrargs{start_key=?JSON_DECODE(Val)}; + Args#mrargs{start_key = ?JSON_DECODE(Val)}; "startkey_docid" -> - Args#mrargs{start_key_docid=couch_util:to_binary(Val)}; + Args#mrargs{start_key_docid = couch_util:to_binary(Val)}; "start_key_doc_id" -> - Args#mrargs{start_key_docid=couch_util:to_binary(Val)}; + Args#mrargs{start_key_docid = couch_util:to_binary(Val)}; "endkey" when IsDecoded -> - Args#mrargs{end_key=Val}; + Args#mrargs{end_key = Val}; "end_key" when IsDecoded -> - Args#mrargs{end_key=Val}; + Args#mrargs{end_key = Val}; "endkey" -> - Args#mrargs{end_key=?JSON_DECODE(Val)}; + Args#mrargs{end_key = ?JSON_DECODE(Val)}; "end_key" -> - Args#mrargs{end_key=?JSON_DECODE(Val)}; + Args#mrargs{end_key = ?JSON_DECODE(Val)}; "endkey_docid" -> - Args#mrargs{end_key_docid=couch_util:to_binary(Val)}; + Args#mrargs{end_key_docid = couch_util:to_binary(Val)}; "end_key_doc_id" -> - Args#mrargs{end_key_docid=couch_util:to_binary(Val)}; + Args#mrargs{end_key_docid = couch_util:to_binary(Val)}; "limit" -> - Args#mrargs{limit=parse_pos_int(Val)}; + Args#mrargs{limit = parse_pos_int(Val)}; "stale" when Val == "ok" orelse Val == <<"ok">> -> - Args#mrargs{stable=true, update=false}; + Args#mrargs{stable = true, update = false}; "stale" when Val == "update_after" orelse Val == <<"update_after">> -> - Args#mrargs{stable=true, update=lazy}; + Args#mrargs{stable = true, update = lazy}; "stale" -> throw({query_parse_error, <<"Invalid value for `stale`.">>}); "stable" when Val == "true" orelse Val == <<"true">> orelse Val == true -> - Args#mrargs{stable=true}; + Args#mrargs{stable = true}; "stable" when Val == "false" orelse Val == <<"false">> orelse Val == false -> - Args#mrargs{stable=false}; + Args#mrargs{stable = false}; "stable" -> throw({query_parse_error, <<"Invalid value for `stable`.">>}); "update" when Val == "true" orelse Val == <<"true">> orelse Val == true -> - Args#mrargs{update=true}; + Args#mrargs{update = true}; "update" when Val == "false" orelse Val == <<"false">> orelse Val == false -> - Args#mrargs{update=false}; + Args#mrargs{update = false}; "update" when Val == "lazy" orelse Val == <<"lazy">> -> - Args#mrargs{update=lazy}; + Args#mrargs{update = lazy}; "update" -> throw({query_parse_error, <<"Invalid value for `update`.">>}); "descending" -> case parse_boolean(Val) of - true -> Args#mrargs{direction=rev}; - _ -> Args#mrargs{direction=fwd} + true -> Args#mrargs{direction = rev}; + _ -> Args#mrargs{direction = fwd} end; "skip" -> - Args#mrargs{skip=parse_pos_int(Val)}; + Args#mrargs{skip = parse_pos_int(Val)}; "group" -> - Args#mrargs{group=parse_boolean(Val)}; + Args#mrargs{group = parse_boolean(Val)}; "group_level" -> - Args#mrargs{group_level=parse_pos_int(Val)}; + Args#mrargs{group_level = parse_pos_int(Val)}; "inclusive_end" -> - Args#mrargs{inclusive_end=parse_boolean(Val)}; + Args#mrargs{inclusive_end = parse_boolean(Val)}; "include_docs" -> - Args#mrargs{include_docs=parse_boolean(Val)}; + Args#mrargs{include_docs = parse_boolean(Val)}; "attachments" -> case parse_boolean(Val) of - true -> - Opts = Args#mrargs.doc_options, - Args#mrargs{doc_options=[attachments|Opts]}; - false -> - Args + true -> + Opts = Args#mrargs.doc_options, + Args#mrargs{doc_options = [attachments | Opts]}; + false -> + Args end; "att_encoding_info" -> case parse_boolean(Val) of - true -> - Opts = Args#mrargs.doc_options, - Args#mrargs{doc_options=[att_encoding_info|Opts]}; - false -> - Args + true -> + Opts = Args#mrargs.doc_options, + Args#mrargs{doc_options = [att_encoding_info | Opts]}; + false -> + Args end; "update_seq" -> - Args#mrargs{update_seq=parse_boolean(Val)}; + Args#mrargs{update_seq = parse_boolean(Val)}; "conflicts" -> - Args#mrargs{conflicts=parse_boolean(Val)}; + Args#mrargs{conflicts = parse_boolean(Val)}; "callback" -> - Args#mrargs{callback=couch_util:to_binary(Val)}; + Args#mrargs{callback = couch_util:to_binary(Val)}; "sorted" -> - Args#mrargs{sorted=parse_boolean(Val)}; + Args#mrargs{sorted = parse_boolean(Val)}; "partition" -> Partition = couch_util:to_binary(Val), couch_partition:validate_partition(Partition), @@ -590,52 +625,50 @@ parse_param(Key, Val, Args, IsDecoded) -> _ -> BKey = couch_util:to_binary(Key), BVal = couch_util:to_binary(Val), - Args#mrargs{extra=[{BKey, BVal} | Args#mrargs.extra]} + Args#mrargs{extra = [{BKey, BVal} | Args#mrargs.extra]} end. - parse_boolean(true) -> true; parse_boolean(false) -> false; - parse_boolean(Val) when is_binary(Val) -> parse_boolean(?b2l(Val)); - parse_boolean(Val) -> case string:to_lower(Val) of - "true" -> true; - "false" -> false; - _ -> - Msg = io_lib:format("Invalid boolean parameter: ~p", [Val]), - throw({query_parse_error, ?l2b(Msg)}) + "true" -> + true; + "false" -> + false; + _ -> + Msg = io_lib:format("Invalid boolean parameter: ~p", [Val]), + throw({query_parse_error, ?l2b(Msg)}) end. parse_int(Val) when is_integer(Val) -> Val; parse_int(Val) -> case (catch list_to_integer(Val)) of - IntVal when is_integer(IntVal) -> - IntVal; - _ -> - Msg = io_lib:format("Invalid value for integer: ~p", [Val]), - throw({query_parse_error, ?l2b(Msg)}) + IntVal when is_integer(IntVal) -> + IntVal; + _ -> + Msg = io_lib:format("Invalid value for integer: ~p", [Val]), + throw({query_parse_error, ?l2b(Msg)}) end. parse_pos_int(Val) -> case parse_int(Val) of - IntVal when IntVal >= 0 -> - IntVal; - _ -> - Fmt = "Invalid value for positive integer: ~p", - Msg = io_lib:format(Fmt, [Val]), - throw({query_parse_error, ?l2b(Msg)}) + IntVal when IntVal >= 0 -> + IntVal; + _ -> + Fmt = "Invalid value for positive integer: ~p", + Msg = io_lib:format(Fmt, [Val]), + throw({query_parse_error, ?l2b(Msg)}) end. - check_view_etag(Sig, Acc0, Req) -> ETag = chttpd:make_etag(Sig), case chttpd:etag_match(Req, ETag) of true -> throw({etag_match, ETag}); - false -> {ok, Acc0#vacc{etag=ETag}} + false -> {ok, Acc0#vacc{etag = ETag}} end. diff --git a/src/couch_mrview/src/couch_mrview_index.erl b/src/couch_mrview/src/couch_mrview_index.erl index 68f1d2322..a024d35c8 100644 --- a/src/couch_mrview/src/couch_mrview_index.erl +++ b/src/couch_mrview/src/couch_mrview_index.erl @@ -12,7 +12,6 @@ -module(couch_mrview_index). - -export([get/2]). -export([init/2, open/2, close/1, reset/1, delete/1, shutdown/1]). -export([start_update/4, purge/4, process_doc/3, finish_update/1, commit/1]). @@ -24,7 +23,6 @@ -include_lib("couch/include/couch_db.hrl"). -include_lib("couch_mrview/include/couch_mrview.hrl"). - get(db_name, #mrst{db_name = DbName}) -> DbName; get(idx_name, #mrst{idx_name = IdxName}) -> @@ -39,9 +37,18 @@ get(update_options, #mrst{design_opts = Opts}) -> IncDesign = couch_util:get_value(<<"include_design">>, Opts, false), LocalSeq = couch_util:get_value(<<"local_seq">>, Opts, false), Partitioned = couch_util:get_value(<<"partitioned">>, Opts, false), - if IncDesign -> [include_design]; true -> [] end - ++ if LocalSeq -> [local_seq]; true -> [] end - ++ if Partitioned -> [partitioned]; true -> [] end; + if + IncDesign -> [include_design]; + true -> [] + end ++ + if + LocalSeq -> [local_seq]; + true -> [] + end ++ + if + Partitioned -> [partitioned]; + true -> [] + end; get(fd, #mrst{fd = Fd}) -> Fd; get(language, #mrst{language = Language}) -> @@ -69,11 +76,12 @@ get(info, State) -> {ok, [ {signature, list_to_binary(couch_index_util:hexsig(Sig))}, {language, Lang}, - {sizes, {[ - {file, FileSize}, - {active, ActiveSize}, - {external, ExternalSize} - ]}}, + {sizes, + {[ + {file, FileSize}, + {active, ActiveSize}, + {external, ExternalSize} + ]}}, {update_seq, UpdateSeq}, {purge_seq, PurgeSeq}, {update_options, UpdateOptions} @@ -81,16 +89,14 @@ get(info, State) -> get(Other, _) -> throw({unknown_index_property, Other}). - init(Db, DDoc) -> {ok, State} = couch_mrview_util:ddoc_to_mrst(couch_db:name(Db), DDoc), {ok, set_partitioned(Db, State)}. - open(Db, State0) -> #mrst{ - db_name=DbName, - sig=Sig + db_name = DbName, + sig = Sig } = State = set_partitioned(Db, State0), IndexFName = couch_mrview_util:index_file(DbName, Sig), @@ -128,14 +134,18 @@ open(Db, State0) -> ensure_local_purge_doc(Db, NewSt), {ok, NewSt}; {ok, {WrongSig, _}} -> - couch_log:error("~s has the wrong signature: expected: ~p but got ~p", - [IndexFName, Sig, WrongSig]), + couch_log:error( + "~s has the wrong signature: expected: ~p but got ~p", + [IndexFName, Sig, WrongSig] + ), NewSt = couch_mrview_util:reset_index(Db, Fd, State), ensure_local_purge_doc(Db, NewSt), {ok, NewSt}; {ok, Else} -> - couch_log:error("~s has a bad header: got ~p", - [IndexFName, Else]), + couch_log:error( + "~s has a bad header: got ~p", + [IndexFName, Else] + ), NewSt = couch_mrview_util:reset_index(Db, Fd, State), ensure_local_purge_doc(Db, NewSt), {ok, NewSt}; @@ -145,17 +155,17 @@ open(Db, State0) -> {ok, NewSt} end; {error, Reason} = Error -> - couch_log:error("Failed to open view file '~s': ~s", - [IndexFName, file:format_error(Reason)]), + couch_log:error( + "Failed to open view file '~s': ~s", + [IndexFName, file:format_error(Reason)] + ), Error end. - close(State) -> erlang:demonitor(State#mrst.fd_monitor, [flush]), couch_file:close(State#mrst.fd). - % This called after ddoc_updated event occurrs, and % before we shutdown couch_index process. % We unlink couch_index from corresponding couch_file and demonitor it. @@ -167,19 +177,16 @@ shutdown(State) -> erlang:demonitor(State#mrst.fd_monitor, [flush]), unlink(State#mrst.fd). - -delete(#mrst{db_name=DbName, sig=Sig}=State) -> +delete(#mrst{db_name = DbName, sig = Sig} = State) -> couch_file:close(State#mrst.fd), catch couch_mrview_util:delete_files(DbName, Sig). - reset(State) -> couch_util:with_db(State#mrst.db_name, fun(Db) -> NewState = couch_mrview_util:reset_index(Db, State#mrst.fd, State), {ok, NewState} end). - start_update(PartialDest, State, NumChanges, NumChangesDone) -> couch_mrview_updater:start_update( PartialDest, @@ -188,94 +195,93 @@ start_update(PartialDest, State, NumChanges, NumChangesDone) -> NumChangesDone ). - purge(Db, PurgeSeq, PurgedIdRevs, State) -> couch_mrview_updater:purge(Db, PurgeSeq, PurgedIdRevs, State). - process_doc(Doc, Seq, State) -> couch_mrview_updater:process_doc(Doc, Seq, State). - finish_update(State) -> couch_mrview_updater:finish_update(State). - commit(State) -> Header = {State#mrst.sig, couch_mrview_util:make_header(State)}, couch_file:write_header(State#mrst.fd, Header). - compact(Db, State, Opts) -> couch_mrview_compactor:compact(Db, State, Opts). - swap_compacted(OldState, NewState) -> couch_mrview_compactor:swap_compacted(OldState, NewState). - remove_compacted(State) -> couch_mrview_compactor:remove_compacted(State). - index_file_exists(State) -> #mrst{ - db_name=DbName, - sig=Sig + db_name = DbName, + sig = Sig } = State, IndexFName = couch_mrview_util:index_file(DbName, Sig), filelib:is_file(IndexFName). - verify_index_exists(DbName, Props) -> try Type = couch_util:get_value(<<"type">>, Props), - if Type =/= <<"mrview">> -> false; true -> - DDocId = couch_util:get_value(<<"ddoc_id">>, Props), - couch_util:with_db(DbName, fun(Db) -> - case couch_db:get_design_doc(Db, DDocId) of - {ok, #doc{} = DDoc} -> - {ok, IdxState} = couch_mrview_util:ddoc_to_mrst( - DbName, DDoc), - IdxSig = IdxState#mrst.sig, - SigInLocal = couch_util:get_value( - <<"signature">>, Props), - couch_index_util:hexsig(IdxSig) == SigInLocal; - {not_found, _} -> - false - end - end) + if + Type =/= <<"mrview">> -> + false; + true -> + DDocId = couch_util:get_value(<<"ddoc_id">>, Props), + couch_util:with_db(DbName, fun(Db) -> + case couch_db:get_design_doc(Db, DDocId) of + {ok, #doc{} = DDoc} -> + {ok, IdxState} = couch_mrview_util:ddoc_to_mrst( + DbName, DDoc + ), + IdxSig = IdxState#mrst.sig, + SigInLocal = couch_util:get_value( + <<"signature">>, Props + ), + couch_index_util:hexsig(IdxSig) == SigInLocal; + {not_found, _} -> + false + end + end) end - catch _:_ -> - false + catch + _:_ -> + false end. - set_partitioned(Db, State) -> #mrst{ design_opts = DesignOpts } = State, DbPartitioned = couch_db:is_partitioned(Db), ViewPartitioned = couch_util:get_value( - <<"partitioned">>, DesignOpts, DbPartitioned), + <<"partitioned">>, DesignOpts, DbPartitioned + ), IsPartitioned = DbPartitioned andalso ViewPartitioned, State#mrst{partitioned = IsPartitioned}. - ensure_local_purge_docs(DbName, DDocs) -> couch_util:with_db(DbName, fun(Db) -> - lists:foreach(fun(DDoc) -> - try couch_mrview_util:ddoc_to_mrst(DbName, DDoc) of - {ok, MRSt} -> - ensure_local_purge_doc(Db, MRSt) - catch _:_ -> - ok - end - end, DDocs) + lists:foreach( + fun(DDoc) -> + try couch_mrview_util:ddoc_to_mrst(DbName, DDoc) of + {ok, MRSt} -> + ensure_local_purge_doc(Db, MRSt) + catch + _:_ -> + ok + end + end, + DDocs + ) end). - -ensure_local_purge_doc(Db, #mrst{}=State) -> +ensure_local_purge_doc(Db, #mrst{} = State) -> Sig = couch_index_util:hexsig(get(signature, State)), DocId = couch_mrview_util:get_local_purge_doc_id(Sig), case couch_db:open_doc(Db, DocId, []) of @@ -285,33 +291,33 @@ ensure_local_purge_doc(Db, #mrst{}=State) -> ok end. - create_local_purge_doc(Db, State) -> PurgeSeq = couch_db:get_purge_seq(Db), update_local_purge_doc(Db, State, PurgeSeq). - update_local_purge_doc(Db, State) -> update_local_purge_doc(Db, State, get(purge_seq, State)). - update_local_purge_doc(Db, State, PSeq) -> Sig = couch_index_util:hexsig(State#mrst.sig), DocId = couch_mrview_util:get_local_purge_doc_id(Sig), {Mega, Secs, _} = os:timestamp(), NowSecs = Mega * 1000000 + Secs, - BaseDoc = couch_doc:from_json_obj({[ - {<<"_id">>, DocId}, - {<<"type">>, <<"mrview">>}, - {<<"purge_seq">>, PSeq}, - {<<"updated_on">>, NowSecs}, - {<<"ddoc_id">>, get(idx_name, State)}, - {<<"signature">>, Sig} - ]}), - Doc = case couch_db:open_doc(Db, DocId, []) of - {ok, #doc{revs = Revs}} -> - BaseDoc#doc{revs = Revs}; - {not_found, _} -> - BaseDoc - end, + BaseDoc = couch_doc:from_json_obj( + {[ + {<<"_id">>, DocId}, + {<<"type">>, <<"mrview">>}, + {<<"purge_seq">>, PSeq}, + {<<"updated_on">>, NowSecs}, + {<<"ddoc_id">>, get(idx_name, State)}, + {<<"signature">>, Sig} + ]} + ), + Doc = + case couch_db:open_doc(Db, DocId, []) of + {ok, #doc{revs = Revs}} -> + BaseDoc#doc{revs = Revs}; + {not_found, _} -> + BaseDoc + end, couch_db:update_doc(Db, Doc, []). diff --git a/src/couch_mrview/src/couch_mrview_show.erl b/src/couch_mrview/src/couch_mrview_show.erl index 0268b706e..3e95be9cc 100644 --- a/src/couch_mrview/src/couch_mrview_show.erl +++ b/src/couch_mrview/src/couch_mrview_show.erl @@ -33,23 +33,28 @@ maybe_open_doc(Db, DocId) -> {not_found, _} -> nil end. -handle_doc_show_req(#httpd{ - path_parts=[_, _, _, _, ShowName, DocId] - }=Req, Db, DDoc) -> - +handle_doc_show_req( + #httpd{ + path_parts = [_, _, _, _, ShowName, DocId] + } = Req, + Db, + DDoc +) -> % open the doc Doc = maybe_open_doc(Db, DocId), % we don't handle revs here b/c they are an internal api % returns 404 if there is no doc with DocId handle_doc_show(Req, Db, DDoc, ShowName, Doc, DocId); - -handle_doc_show_req(#httpd{ - path_parts=[_, _, _, _, ShowName, DocId|Rest] - }=Req, Db, DDoc) -> - - DocParts = [DocId|Rest], - DocId1 = ?l2b(string:join([?b2l(P)|| P <- DocParts], "/")), +handle_doc_show_req( + #httpd{ + path_parts = [_, _, _, _, ShowName, DocId | Rest] + } = Req, + Db, + DDoc +) -> + DocParts = [DocId | Rest], + DocId1 = ?l2b(string:join([?b2l(P) || P <- DocParts], "/")), % open the doc Doc = maybe_open_doc(Db, DocId1), @@ -57,13 +62,15 @@ handle_doc_show_req(#httpd{ % we don't handle revs here b/c they are an internal api % pass 404 docs to the show function handle_doc_show(Req, Db, DDoc, ShowName, Doc, DocId1); - -handle_doc_show_req(#httpd{ - path_parts=[_, _, _, _, ShowName] - }=Req, Db, DDoc) -> +handle_doc_show_req( + #httpd{ + path_parts = [_, _, _, _, ShowName] + } = Req, + Db, + DDoc +) -> % with no docid the doc is nil handle_doc_show(Req, Db, DDoc, ShowName, nil); - handle_doc_show_req(Req, _Db, _DDoc) -> chttpd:send_error(Req, 404, <<"show_error">>, <<"Invalid path.">>). @@ -77,21 +84,29 @@ handle_doc_show(Req, Db, DDoc, ShowName, Doc, DocId) -> JsonReq = chttpd_external:json_req_obj(Req, Db, DocId), JsonDoc = couch_query_servers:json_doc(Doc), [<<"resp">>, ExternalResp] = - couch_query_servers:ddoc_prompt(DDoc, [<<"shows">>, ShowName], - [JsonDoc, JsonReq]), + couch_query_servers:ddoc_prompt( + DDoc, + [<<"shows">>, ShowName], + [JsonDoc, JsonReq] + ), JsonResp = apply_etag(ExternalResp, CurrentEtag), chttpd_external:send_external_response(Req, JsonResp) end). - -show_etag(#httpd{user_ctx=UserCtx}=Req, Doc, DDoc, More) -> +show_etag(#httpd{user_ctx = UserCtx} = Req, Doc, DDoc, More) -> Accept = chttpd:header_value(Req, "Accept"), - DocPart = case Doc of - nil -> nil; - Doc -> chttpd:doc_etag(Doc) - end, - chttpd:make_etag({chttpd:doc_etag(DDoc), DocPart, Accept, - {UserCtx#user_ctx.name, UserCtx#user_ctx.roles}, More}). + DocPart = + case Doc of + nil -> nil; + Doc -> chttpd:doc_etag(Doc) + end, + chttpd:make_etag({ + chttpd:doc_etag(DDoc), + DocPart, + Accept, + {UserCtx#user_ctx.name, UserCtx#user_ctx.roles}, + More + }). % updates a doc based on a request % handle_doc_update_req(#httpd{method = 'GET'}=Req, _Db, _DDoc) -> @@ -101,20 +116,25 @@ show_etag(#httpd{user_ctx=UserCtx}=Req, Doc, DDoc, More) -> % This call is creating a new doc using an _update function to % modify the provided request body. % /db/_design/foo/_update/bar -handle_doc_update_req(#httpd{ - path_parts=[_, _, _, _, UpdateName] - }=Req, Db, DDoc) -> +handle_doc_update_req( + #httpd{ + path_parts = [_, _, _, _, UpdateName] + } = Req, + Db, + DDoc +) -> send_doc_update_response(Req, Db, DDoc, UpdateName, nil, null); - % /db/_design/foo/_update/bar/docid -handle_doc_update_req(#httpd{ - path_parts=[_, _, _, _, UpdateName | DocIdParts] - }=Req, Db, DDoc) -> +handle_doc_update_req( + #httpd{ + path_parts = [_, _, _, _, UpdateName | DocIdParts] + } = Req, + Db, + DDoc +) -> DocId = ?l2b(string:join([?b2l(P) || P <- DocIdParts], "/")), Doc = maybe_open_doc(Db, DocId), send_doc_update_response(Req, Db, DDoc, UpdateName, Doc, DocId); - - handle_doc_update_req(Req, _Db, _DDoc) -> chttpd:send_error(Req, 404, <<"update_error">>, <<"Invalid path.">>). @@ -123,32 +143,36 @@ send_doc_update_response(Req, Db, DDoc, UpdateName, Doc, DocId) -> JsonDoc = couch_query_servers:json_doc(Doc), Cmd = [<<"updates">>, UpdateName], UpdateResp = couch_query_servers:ddoc_prompt(DDoc, Cmd, [JsonDoc, JsonReq]), - JsonResp = case UpdateResp of - [<<"up">>, {NewJsonDoc}, {JsonResp0}] -> - case chttpd:header_value( - Req, "X-Couch-Full-Commit", "false") of - "true" -> - Options = [full_commit, {user_ctx, Req#httpd.user_ctx}]; - _ -> - Options = [{user_ctx, Req#httpd.user_ctx}] - end, - NewDoc = couch_db:doc_from_json_obj_validate(Db, {NewJsonDoc}), - {ok, NewRev} = couch_db:update_doc(Db, NewDoc, Options), - NewRevStr = couch_doc:rev_to_str(NewRev), - {JsonResp1} = apply_headers(JsonResp0, [ - {<<"X-Couch-Update-NewRev">>, NewRevStr}, - {<<"X-Couch-Id">>, couch_util:url_encode(NewDoc#doc.id)} - ]), - {[{<<"code">>, 201} | JsonResp1]}; - [<<"up">>, _Other, {JsonResp0}] -> - {[{<<"code">>, 200} | JsonResp0]} - end, + JsonResp = + case UpdateResp of + [<<"up">>, {NewJsonDoc}, {JsonResp0}] -> + case + chttpd:header_value( + Req, "X-Couch-Full-Commit", "false" + ) + of + "true" -> + Options = [full_commit, {user_ctx, Req#httpd.user_ctx}]; + _ -> + Options = [{user_ctx, Req#httpd.user_ctx}] + end, + NewDoc = couch_db:doc_from_json_obj_validate(Db, {NewJsonDoc}), + {ok, NewRev} = couch_db:update_doc(Db, NewDoc, Options), + NewRevStr = couch_doc:rev_to_str(NewRev), + {JsonResp1} = apply_headers(JsonResp0, [ + {<<"X-Couch-Update-NewRev">>, NewRevStr}, + {<<"X-Couch-Id">>, couch_util:url_encode(NewDoc#doc.id)} + ]), + {[{<<"code">>, 201} | JsonResp1]}; + [<<"up">>, _Other, {JsonResp0}] -> + {[{<<"code">>, 200} | JsonResp0]} + end, % todo set location field chttpd_external:send_external_response(Req, JsonResp). - -handle_view_list_req(#httpd{method=Method}=Req, Db, DDoc) - when Method =:= 'GET' orelse Method =:= 'OPTIONS' -> +handle_view_list_req(#httpd{method = Method} = Req, Db, DDoc) when + Method =:= 'GET' orelse Method =:= 'OPTIONS' +-> case Req#httpd.path_parts of [_, _, _DName, _, LName, VName] -> % Same design doc for view and list @@ -161,7 +185,7 @@ handle_view_list_req(#httpd{method=Method}=Req, Db, DDoc) _ -> chttpd:send_error(Req, 404, <<"list_error">>, <<"Bad path.">>) end; -handle_view_list_req(#httpd{method='POST'}=Req, Db, DDoc) -> +handle_view_list_req(#httpd{method = 'POST'} = Req, Db, DDoc) -> chttpd:validate_ctype(Req, "application/json"), {Props} = chttpd:json_body_obj(Req), Keys = proplists:get_value(<<"keys">>, Props), @@ -179,7 +203,6 @@ handle_view_list_req(#httpd{method='POST'}=Req, Db, DDoc) -> handle_view_list_req(Req, _Db, _DDoc) -> chttpd:send_method_not_allowed(Req, "GET,POST,HEAD"). - handle_view_list(Req, Db, DDoc, LName, VDDoc, VName, Keys) -> Args0 = couch_mrview_http:parse_body_and_query(Req, Keys), ETagFun = fun(BaseSig, Acc0) -> @@ -191,67 +214,73 @@ handle_view_list(Req, Db, DDoc, LName, VDDoc, VName, Keys) -> ETag = chttpd:make_etag({BaseSig, Parts}), case chttpd:etag_match(Req, ETag) of true -> throw({etag_match, ETag}); - false -> {ok, Acc0#lacc{etag=ETag}} + false -> {ok, Acc0#lacc{etag = ETag}} end end, - Args = Args0#mrargs{preflight_fun=ETagFun}, + Args = Args0#mrargs{preflight_fun = ETagFun}, couch_httpd:etag_maybe(Req, fun() -> couch_query_servers:with_ddoc_proc(DDoc, fun(QServer) -> - Acc = #lacc{db=Db, req=Req, qserver=QServer, lname=LName}, + Acc = #lacc{db = Db, req = Req, qserver = QServer, lname = LName}, case VName of - <<"_all_docs">> -> - couch_mrview:query_all_docs(Db, Args, fun list_cb/2, Acc); - _ -> - couch_mrview:query_view(Db, VDDoc, VName, Args, fun list_cb/2, Acc) + <<"_all_docs">> -> + couch_mrview:query_all_docs(Db, Args, fun list_cb/2, Acc); + _ -> + couch_mrview:query_view(Db, VDDoc, VName, Args, fun list_cb/2, Acc) end end) end). - -list_cb({meta, Meta}, #lacc{code=undefined} = Acc) -> - MetaProps = case couch_util:get_value(total, Meta) of - undefined -> []; - Total -> [{total_rows, Total}] - end ++ case couch_util:get_value(offset, Meta) of - undefined -> []; - Offset -> [{offset, Offset}] - end ++ case couch_util:get_value(update_seq, Meta) of - undefined -> []; - UpdateSeq -> [{update_seq, UpdateSeq}] - end, +list_cb({meta, Meta}, #lacc{code = undefined} = Acc) -> + MetaProps = + case couch_util:get_value(total, Meta) of + undefined -> []; + Total -> [{total_rows, Total}] + end ++ + case couch_util:get_value(offset, Meta) of + undefined -> []; + Offset -> [{offset, Offset}] + end ++ + case couch_util:get_value(update_seq, Meta) of + undefined -> []; + UpdateSeq -> [{update_seq, UpdateSeq}] + end, start_list_resp({MetaProps}, Acc); -list_cb({row, Row}, #lacc{code=undefined} = Acc) -> +list_cb({row, Row}, #lacc{code = undefined} = Acc) -> {ok, NewAcc} = start_list_resp({[]}, Acc), send_list_row(Row, NewAcc); list_cb({row, Row}, Acc) -> send_list_row(Row, Acc); list_cb(complete, Acc) -> #lacc{qserver = {Proc, _}, req = Req, resp = Resp0} = Acc, - if Resp0 =:= nil -> - {ok, #lacc{resp = Resp}} = start_list_resp({[]}, Acc); - true -> - Resp = Resp0 + if + Resp0 =:= nil -> + {ok, #lacc{resp = Resp}} = start_list_resp({[]}, Acc); + true -> + Resp = Resp0 end, case couch_query_servers:proc_prompt(Proc, [<<"list_end">>]) of [<<"end">>, Data, Headers] -> - Acc2 = fixup_headers(Headers, Acc#lacc{resp=Resp}), + Acc2 = fixup_headers(Headers, Acc#lacc{resp = Resp}), #lacc{resp = Resp2} = send_non_empty_chunk(Acc2, Data); [<<"end">>, Data] -> - #lacc{resp = Resp2} = send_non_empty_chunk(Acc#lacc{resp=Resp}, Data) + #lacc{resp = Resp2} = send_non_empty_chunk(Acc#lacc{resp = Resp}, Data) end, last_chunk(Req, Resp2), {ok, Resp2}. start_list_resp(Head, Acc) -> - #lacc{db=Db, req=Req, qserver=QServer, lname=LName} = Acc, + #lacc{db = Db, req = Req, qserver = QServer, lname = LName} = Acc, JsonReq = json_req_obj(Req, Db), - [<<"start">>,Chunk,JsonResp] = couch_query_servers:ddoc_proc_prompt(QServer, - [<<"lists">>, LName], [Head, JsonReq]), + [<<"start">>, Chunk, JsonResp] = couch_query_servers:ddoc_proc_prompt( + QServer, + [<<"lists">>, LName], + [Head, JsonReq] + ), Acc2 = send_non_empty_chunk(fixup_headers(JsonResp, Acc), Chunk), {ok, Acc2}. -fixup_headers(Headers, #lacc{etag=ETag} = Acc) -> +fixup_headers(Headers, #lacc{etag = ETag} = Acc) -> Headers2 = apply_etag(Headers, ETag), #extern_resp_args{ code = Code, @@ -260,61 +289,66 @@ fixup_headers(Headers, #lacc{etag=ETag} = Acc) -> } = chttpd_external:parse_external_response(Headers2), Headers3 = chttpd_external:default_or_content_type(CType, ExtHeaders), Headers4 = chttpd_util:maybe_add_csp_header("showlist", Headers3, "sandbox"), - Acc#lacc{code=Code, headers=Headers4}. + Acc#lacc{code = Code, headers = Headers4}. send_list_row(Row, #lacc{qserver = {Proc, _}, req = Req, resp = Resp} = Acc) -> - RowObj = case couch_util:get_value(id, Row) of - undefined -> []; - Id -> [{id, Id}] - end ++ case couch_util:get_value(key, Row) of - undefined -> []; - Key -> [{key, Key}] - end ++ case couch_util:get_value(value, Row) of - undefined -> []; - Val -> [{value, Val}] - end ++ case couch_util:get_value(doc, Row) of - undefined -> []; - Doc -> [{doc, Doc}] - end, + RowObj = + case couch_util:get_value(id, Row) of + undefined -> []; + Id -> [{id, Id}] + end ++ + case couch_util:get_value(key, Row) of + undefined -> []; + Key -> [{key, Key}] + end ++ + case couch_util:get_value(value, Row) of + undefined -> []; + Val -> [{value, Val}] + end ++ + case couch_util:get_value(doc, Row) of + undefined -> []; + Doc -> [{doc, Doc}] + end, try couch_query_servers:proc_prompt(Proc, [<<"list_row">>, {RowObj}]) of - [<<"chunks">>, Chunk, Headers] -> - Acc2 = send_non_empty_chunk(fixup_headers(Headers, Acc), Chunk), - {ok, Acc2}; - [<<"chunks">>, Chunk] -> - Acc2 = send_non_empty_chunk(Acc, Chunk), - {ok, Acc2}; - [<<"end">>, Chunk, Headers] -> - #lacc{resp = Resp2} = send_non_empty_chunk(fixup_headers(Headers, Acc), Chunk), - {ok, Resp3} = last_chunk(Req, Resp2), - {stop, Resp3}; - [<<"end">>, Chunk] -> - #lacc{resp = Resp2} = send_non_empty_chunk(Acc, Chunk), - {ok, Resp3} = last_chunk(Req, Resp2), - {stop, Resp3} - catch Error -> - {ok, Resp2} = case Resp of - undefined -> - {Code, _, _} = chttpd:error_info(Error), - #lacc{req=Req, headers=Headers} = Acc, - chttpd:start_chunked_response(Req, Code, Headers); - _ -> - {ok, Resp} - end, - {ok, Resp3} = chttpd:send_chunked_error(Resp2, Error), - {stop, Resp3} + [<<"chunks">>, Chunk, Headers] -> + Acc2 = send_non_empty_chunk(fixup_headers(Headers, Acc), Chunk), + {ok, Acc2}; + [<<"chunks">>, Chunk] -> + Acc2 = send_non_empty_chunk(Acc, Chunk), + {ok, Acc2}; + [<<"end">>, Chunk, Headers] -> + #lacc{resp = Resp2} = send_non_empty_chunk(fixup_headers(Headers, Acc), Chunk), + {ok, Resp3} = last_chunk(Req, Resp2), + {stop, Resp3}; + [<<"end">>, Chunk] -> + #lacc{resp = Resp2} = send_non_empty_chunk(Acc, Chunk), + {ok, Resp3} = last_chunk(Req, Resp2), + {stop, Resp3} + catch + Error -> + {ok, Resp2} = + case Resp of + undefined -> + {Code, _, _} = chttpd:error_info(Error), + #lacc{req = Req, headers = Headers} = Acc, + chttpd:start_chunked_response(Req, Code, Headers); + _ -> + {ok, Resp} + end, + {ok, Resp3} = chttpd:send_chunked_error(Resp2, Error), + {stop, Resp3} end. send_non_empty_chunk(Acc, []) -> Acc; -send_non_empty_chunk(#lacc{resp=undefined} = Acc, Chunk) -> - #lacc{req=Req, code=Code, headers=Headers} = Acc, +send_non_empty_chunk(#lacc{resp = undefined} = Acc, Chunk) -> + #lacc{req = Req, code = Code, headers = Headers} = Acc, {ok, Resp} = chttpd:start_chunked_response(Req, Code, Headers), send_non_empty_chunk(Acc#lacc{resp = Resp}, Chunk); -send_non_empty_chunk(#lacc{resp=Resp} = Acc, Chunk) -> +send_non_empty_chunk(#lacc{resp = Resp} = Acc, Chunk) -> chttpd:send_chunk(Resp, Chunk), Acc. - apply_etag(JsonResp, undefined) -> JsonResp; apply_etag({ExternalResponse}, CurrentEtag) -> @@ -332,7 +366,7 @@ apply_headers(JsonResp, []) -> apply_headers(JsonResp, NewHeaders) -> case couch_util:get_value(<<"headers">>, JsonResp) of undefined -> - {[{<<"headers">>, {NewHeaders}}| JsonResp]}; + {[{<<"headers">>, {NewHeaders}} | JsonResp]}; JsonHeaders -> Headers = apply_headers1(JsonHeaders, NewHeaders), NewKV = {<<"headers">>, Headers}, @@ -344,13 +378,11 @@ apply_headers1(JsonHeaders, [{Key, Value} | Rest]) -> apply_headers1(JsonHeaders, []) -> JsonHeaders. - % Maybe this is in the proplists API % todo move to couch_util json_apply_field(H, {L}) -> json_apply_field(H, L, []). - json_apply_field({Key, NewValue}, [{Key, _OldVal} | Headers], Acc) -> % drop matching keys json_apply_field({Key, NewValue}, Headers, Acc); @@ -359,8 +391,7 @@ json_apply_field({Key, NewValue}, [{OtherKey, OtherVal} | Headers], Acc) -> json_apply_field({Key, NewValue}, Headers, [{OtherKey, OtherVal} | Acc]); json_apply_field({Key, NewValue}, [], Acc) -> % end of list, add ours - {[{Key, NewValue}|Acc]}. - + {[{Key, NewValue} | Acc]}. % This loads the db info if we have a fully loaded db record, but we might not % have the db locally on this node, so then load the info through fabric. @@ -371,7 +402,9 @@ json_req_obj(Req, Db) -> % and json_req_obj calls fabric:get_db_info() JRO = fun() -> exit(chttpd_external:json_req_obj(Req, Db)) end, {Pid, Ref} = spawn_monitor(JRO), - receive {'DOWN', Ref, process, Pid, JsonReq} -> JsonReq end; + receive + {'DOWN', Ref, process, Pid, JsonReq} -> JsonReq + end; false -> chttpd_external:json_req_obj(Req, Db) end. @@ -381,7 +414,6 @@ last_chunk(Req, undefined) -> last_chunk(_Req, Resp) -> chttpd:send_chunk(Resp, []). - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -395,7 +427,7 @@ apply_headers_test_() -> should_apply_headers() -> ?_test(begin JsonResp = [{<<"code">>, 201}], - Headers = [{<<"foo">>, <<"bar">>}], + Headers = [{<<"foo">>, <<"bar">>}], {Props} = apply_headers(JsonResp, Headers), JsonHeaders = couch_util:get_value(<<"headers">>, Props), ?assertEqual({Headers}, JsonHeaders) @@ -404,7 +436,7 @@ should_apply_headers() -> should_apply_headers_with_merge() -> ?_test(begin BaseHeaders = [{<<"bar">>, <<"baz">>}], - NewHeaders = [{<<"foo">>, <<"bar">>}], + NewHeaders = [{<<"foo">>, <<"bar">>}], JsonResp = [ {<<"code">>, 201}, {<<"headers">>, {BaseHeaders}} @@ -418,7 +450,7 @@ should_apply_headers_with_merge() -> should_apply_headers_with_merge_overwrite() -> ?_test(begin BaseHeaders = [{<<"foo">>, <<"bar">>}], - NewHeaders = [{<<"foo">>, <<"baz">>}], + NewHeaders = [{<<"foo">>, <<"baz">>}], JsonResp = [ {<<"code">>, 201}, {<<"headers">>, {BaseHeaders}} @@ -428,7 +460,6 @@ should_apply_headers_with_merge_overwrite() -> ?assertEqual({NewHeaders}, JsonHeaders) end). - send_list_row_test_() -> Cases = couch_tests_combinatorics:product([ [ @@ -439,26 +470,40 @@ send_list_row_test_() -> [ req, undefined - ]]), + ] + ]), {"Ensure send_list_row returns a valid response on end or error", {setup, fun setup/0, fun(_) -> meck:unload() end, [ { lists:flatten(io_lib:format("~s -- ~p", [N, R])), should_return_valid_response(F, R) - } || [{N, F}, R] <- Cases - ]} - }. + } + || [{N, F}, R] <- Cases + ]}}. setup() -> ok = application:start(config, permanent), - ok = meck:expect(chttpd, send_chunk, - fun(Resp, _) -> {ok, Resp} end), - ok = meck:expect(chttpd, send_chunked_error, - fun(Resp, _) -> {ok, Resp} end), - ok = meck:expect(chttpd, start_chunked_response, - fun(_, _, _) -> {ok, resp} end), - ok = meck:expect(chttpd_external, parse_external_response, 1, - #extern_resp_args{headers = []}). + ok = meck:expect( + chttpd, + send_chunk, + fun(Resp, _) -> {ok, Resp} end + ), + ok = meck:expect( + chttpd, + send_chunked_error, + fun(Resp, _) -> {ok, Resp} end + ), + ok = meck:expect( + chttpd, + start_chunked_response, + fun(_, _, _) -> {ok, resp} end + ), + ok = meck:expect( + chttpd_external, + parse_external_response, + 1, + #extern_resp_args{headers = []} + ). should_return_valid_response(Spec, Req) -> ?_test(begin diff --git a/src/couch_mrview/src/couch_mrview_test_util.erl b/src/couch_mrview/src/couch_mrview_test_util.erl index 2dfa64e61..918988ea3 100644 --- a/src/couch_mrview/src/couch_mrview_test_util.erl +++ b/src/couch_mrview/src/couch_mrview_test_util.erl @@ -18,17 +18,14 @@ -include_lib("couch/include/couch_db.hrl"). -include_lib("couch/include/couch_eunit.hrl"). - init_db(Name, Type) -> init_db(Name, Type, 10). - init_db(Name, Type, Count) -> {ok, Db} = new_db(Name, Type), Docs = make_docs(Type, Count), save_docs(Db, Docs). - new_db(Name, Type) when Type == local; Type == design -> couch_server:delete(Name, [?ADMIN_CTX]), couch_db:create(Name, [?ADMIN_CTX]); @@ -44,80 +41,96 @@ save_docs(Db, Docs) -> {ok, _} = couch_db:update_docs(Db, Docs, []), couch_db:reopen(Db). - make_docs(local, Count) -> [local_doc(I) || I <- lists:seq(1, Count)]; make_docs(design, Count) -> - lists:foldl(fun(I, Acc) -> - [doc(I), ddoc(I) | Acc] - end, [], lists:seq(1, Count)); + lists:foldl( + fun(I, Acc) -> + [doc(I), ddoc(I) | Acc] + end, + [], + lists:seq(1, Count) + ); make_docs(_, Count) -> [doc(I) || I <- lists:seq(1, Count)]. - make_docs(_, Since, Count) -> [doc(I) || I <- lists:seq(Since, Count)]. - ddoc(map) -> - couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/bar">>}, - {<<"views">>, {[ - {<<"baz">>, {[ - {<<"map">>, <<"function(doc) {emit(doc.val, doc.val);}">>} - ]}}, - {<<"bing">>, {[ - {<<"map">>, <<"function(doc) {}">>} - ]}}, - {<<"zing">>, {[ - {<<"map">>, << - "function(doc) {\n" - " if(doc.foo !== undefined)\n" - " emit(doc.foo, 0);\n" - "}" - >>} - ]}} - ]}} - ]}); + couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/bar">>}, + {<<"views">>, + {[ + {<<"baz">>, + {[ + {<<"map">>, <<"function(doc) {emit(doc.val, doc.val);}">>} + ]}}, + {<<"bing">>, + {[ + {<<"map">>, <<"function(doc) {}">>} + ]}}, + {<<"zing">>, + {[ + {<<"map">>, << + "function(doc) {\n" + " if(doc.foo !== undefined)\n" + " emit(doc.foo, 0);\n" + "}" + >>} + ]}} + ]}} + ]} + ); ddoc(red) -> - couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/red">>}, - {<<"views">>, {[ - {<<"baz">>, {[ - {<<"map">>, << - "function(doc) {\n" - " emit([doc.val % 2, doc.val], doc.val);\n" - "}\n" - >>}, - {<<"reduce">>, <<"function(keys, vals) {return sum(vals);}">>} - ]}}, - {<<"zing">>, {[ - {<<"map">>, << - "function(doc) {\n" - " if(doc.foo !== undefined)\n" - " emit(doc.foo, null);\n" - "}" - >>}, - {<<"reduce">>, <<"_count">>} - ]}} - ]}} - ]}); + couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/red">>}, + {<<"views">>, + {[ + {<<"baz">>, + {[ + {<<"map">>, << + "function(doc) {\n" + " emit([doc.val % 2, doc.val], doc.val);\n" + "}\n" + >>}, + {<<"reduce">>, <<"function(keys, vals) {return sum(vals);}">>} + ]}}, + {<<"zing">>, + {[ + {<<"map">>, << + "function(doc) {\n" + " if(doc.foo !== undefined)\n" + " emit(doc.foo, null);\n" + "}" + >>}, + {<<"reduce">>, <<"_count">>} + ]}} + ]}} + ]} + ); ddoc(Id) -> - couch_doc:from_json_obj({[ - {<<"_id">>, list_to_binary(io_lib:format("_design/bar~2..0b", [Id]))}, - {<<"views">>, {[]}} - ]}). - + couch_doc:from_json_obj( + {[ + {<<"_id">>, list_to_binary(io_lib:format("_design/bar~2..0b", [Id]))}, + {<<"views">>, {[]}} + ]} + ). doc(Id) -> - couch_doc:from_json_obj({[ - {<<"_id">>, list_to_binary(integer_to_list(Id))}, - {<<"val">>, Id} - ]}). - + couch_doc:from_json_obj( + {[ + {<<"_id">>, list_to_binary(integer_to_list(Id))}, + {<<"val">>, Id} + ]} + ). local_doc(Id) -> - couch_doc:from_json_obj({[ - {<<"_id">>, list_to_binary(io_lib:format("_local/~b", [Id]))}, - {<<"val">>, Id} - ]}). + couch_doc:from_json_obj( + {[ + {<<"_id">>, list_to_binary(io_lib:format("_local/~b", [Id]))}, + {<<"val">>, Id} + ]} + ). diff --git a/src/couch_mrview/src/couch_mrview_update_notifier.erl b/src/couch_mrview/src/couch_mrview_update_notifier.erl index 803d39747..ac91131a0 100644 --- a/src/couch_mrview/src/couch_mrview_update_notifier.erl +++ b/src/couch_mrview/src/couch_mrview_update_notifier.erl @@ -20,7 +20,9 @@ -include_lib("couch/include/couch_db.hrl"). start_link(Exec) -> - couch_event_sup:start_link(couch_mrview_update, {couch_mrview_update_notifier, make_ref()}, Exec). + couch_event_sup:start_link( + couch_mrview_update, {couch_mrview_update_notifier, make_ref()}, Exec + ). notify(Event) -> gen_event:notify(couch_mrview_update, Event). diff --git a/src/couch_mrview/src/couch_mrview_updater.erl b/src/couch_mrview/src/couch_mrview_updater.erl index 522367c1d..969a82028 100644 --- a/src/couch_mrview/src/couch_mrview_updater.erl +++ b/src/couch_mrview/src/couch_mrview_updater.erl @@ -26,22 +26,25 @@ start_update(Partial, State, NumChanges, NumChangesDone) -> {ok, DocQueue} = couch_work_queue:new(QueueOpts), {ok, WriteQueue} = couch_work_queue:new(QueueOpts), InitState = State#mrst{ - first_build=State#mrst.update_seq==0, - partial_resp_pid=Partial, - doc_acc=[], - doc_queue=DocQueue, - write_queue=WriteQueue + first_build = State#mrst.update_seq == 0, + partial_resp_pid = Partial, + doc_acc = [], + doc_queue = DocQueue, + write_queue = WriteQueue }, Self = self(), MapFun = fun() -> - erlang:put(io_priority, - {view_update, State#mrst.db_name, State#mrst.idx_name}), - Progress = case NumChanges of - 0 -> 0; - _ -> (NumChangesDone * 100) div NumChanges - end, + erlang:put( + io_priority, + {view_update, State#mrst.db_name, State#mrst.idx_name} + ), + Progress = + case NumChanges of + 0 -> 0; + _ -> (NumChangesDone * 100) div NumChanges + end, couch_task_status:add_task([ {indexer_pid, ?l2b(pid_to_list(Partial))}, {type, indexer}, @@ -55,8 +58,10 @@ start_update(Partial, State, NumChanges, NumChangesDone) -> map_docs(Self, InitState) end, WriteFun = fun() -> - erlang:put(io_priority, - {view_update, State#mrst.db_name, State#mrst.idx_name}), + erlang:put( + io_priority, + {view_update, State#mrst.db_name, State#mrst.idx_name} + ), write_results(Self, InitState) end, spawn_link(MapFun), @@ -64,12 +69,11 @@ start_update(Partial, State, NumChanges, NumChangesDone) -> {ok, InitState}. - purge(_Db, PurgeSeq, PurgedIdRevs, State) -> #mrst{ - id_btree=IdBtree, - views=Views, - partitioned=Partitioned + id_btree = IdBtree, + views = Views, + partitioned = Partitioned } = State, Ids = [Id || {Id, _Revs} <- PurgedIdRevs], @@ -81,10 +85,14 @@ purge(_Db, PurgeSeq, PurgedIdRevs, State) -> ({ViewNum, {Key, Seq, _Op}}, DictAcc2) -> dict:append(ViewNum, {Key, Seq, DocId}, DictAcc2); ({ViewNum, RowKey0}, DictAcc2) -> - RowKey = if not Partitioned -> RowKey0; true -> - [{RK, _}] = inject_partition([{RowKey0, DocId}]), - RK - end, + RowKey = + if + not Partitioned -> + RowKey0; + true -> + [{RK, _}] = inject_partition([{RowKey0, DocId}]), + RK + end, dict:append(ViewNum, {RowKey, DocId}, DictAcc2) end, lists:foldl(FoldFun, DictAcc, ViewNumRowKeys); @@ -93,54 +101,54 @@ purge(_Db, PurgeSeq, PurgedIdRevs, State) -> end, KeysToRemove = lists:foldl(MakeDictFun, dict:new(), Lookups), - RemKeysFun = fun(#mrview{id_num=ViewId}=View) -> + RemKeysFun = fun(#mrview{id_num = ViewId} = View) -> ToRem = couch_util:dict_find(ViewId, KeysToRemove, []), {ok, VBtree2} = couch_btree:add_remove(View#mrview.btree, [], ToRem), - NewPurgeSeq = case VBtree2 =/= View#mrview.btree of - true -> PurgeSeq; - _ -> View#mrview.purge_seq - end, - View#mrview{btree=VBtree2, purge_seq=NewPurgeSeq} + NewPurgeSeq = + case VBtree2 =/= View#mrview.btree of + true -> PurgeSeq; + _ -> View#mrview.purge_seq + end, + View#mrview{btree = VBtree2, purge_seq = NewPurgeSeq} end, Views2 = lists:map(RemKeysFun, Views), {ok, State#mrst{ - id_btree=IdBtree2, - views=Views2, - purge_seq=PurgeSeq + id_btree = IdBtree2, + views = Views2, + purge_seq = PurgeSeq }}. - -process_doc(Doc, Seq, #mrst{doc_acc=Acc}=State) when length(Acc) > 100 -> +process_doc(Doc, Seq, #mrst{doc_acc = Acc} = State) when length(Acc) > 100 -> couch_work_queue:queue(State#mrst.doc_queue, lists:reverse(Acc)), - process_doc(Doc, Seq, State#mrst{doc_acc=[]}); -process_doc(nil, Seq, #mrst{doc_acc=Acc}=State) -> - {ok, State#mrst{doc_acc=[{nil, Seq, nil} | Acc]}}; -process_doc(#doc{id=Id, deleted=true}, Seq, #mrst{doc_acc=Acc}=State) -> - {ok, State#mrst{doc_acc=[{Id, Seq, deleted} | Acc]}}; -process_doc(#doc{id=Id}=Doc, Seq, #mrst{doc_acc=Acc}=State) -> - {ok, State#mrst{doc_acc=[{Id, Seq, Doc} | Acc]}}. - - -finish_update(#mrst{doc_acc=Acc}=State) -> - if Acc /= [] -> - couch_work_queue:queue(State#mrst.doc_queue, Acc); - true -> ok + process_doc(Doc, Seq, State#mrst{doc_acc = []}); +process_doc(nil, Seq, #mrst{doc_acc = Acc} = State) -> + {ok, State#mrst{doc_acc = [{nil, Seq, nil} | Acc]}}; +process_doc(#doc{id = Id, deleted = true}, Seq, #mrst{doc_acc = Acc} = State) -> + {ok, State#mrst{doc_acc = [{Id, Seq, deleted} | Acc]}}; +process_doc(#doc{id = Id} = Doc, Seq, #mrst{doc_acc = Acc} = State) -> + {ok, State#mrst{doc_acc = [{Id, Seq, Doc} | Acc]}}. + +finish_update(#mrst{doc_acc = Acc} = State) -> + if + Acc /= [] -> + couch_work_queue:queue(State#mrst.doc_queue, Acc); + true -> + ok end, couch_work_queue:close(State#mrst.doc_queue), receive {new_state, NewState} -> {ok, NewState#mrst{ - first_build=undefined, - partial_resp_pid=undefined, - doc_acc=undefined, - doc_queue=undefined, - write_queue=undefined, - qserver=nil + first_build = undefined, + partial_resp_pid = undefined, + doc_acc = undefined, + doc_queue = undefined, + write_queue = undefined, + qserver = nil }} end. - map_docs(Parent, #mrst{db_name = DbName, idx_name = IdxName} = State0) -> erlang:put(io_priority, {view_update, DbName, IdxName}), case couch_work_queue:dequeue(State0#mrst.doc_queue) of @@ -150,10 +158,11 @@ map_docs(Parent, #mrst{db_name = DbName, idx_name = IdxName} = State0) -> {ok, Dequeued} -> % Run all the non deleted docs through the view engine and % then pass the results on to the writer process. - State1 = case State0#mrst.qserver of - nil -> start_query_server(State0); - _ -> State0 - end, + State1 = + case State0#mrst.qserver of + nil -> start_query_server(State0); + _ -> State0 + end, QServer = State1#mrst.qserver, DocFun = fun ({nil, Seq, _}, {SeqAcc, Results}) -> @@ -174,38 +183,37 @@ map_docs(Parent, #mrst{db_name = DbName, idx_name = IdxName} = State0) -> map_docs(Parent, State1) end. - write_results(Parent, #mrst{} = State) -> case accumulate_writes(State, State#mrst.write_queue, nil) of stop -> Parent ! {new_state, State}; {Go, {Seq, ViewKVs, DocIdKeys}} -> NewState = write_kvs(State, Seq, ViewKVs, DocIdKeys), - if Go == stop -> - Parent ! {new_state, NewState}; - true -> - send_partial(NewState#mrst.partial_resp_pid, NewState), - write_results(Parent, NewState) + if + Go == stop -> + Parent ! {new_state, NewState}; + true -> + send_partial(NewState#mrst.partial_resp_pid, NewState), + write_results(Parent, NewState) end end. - start_query_server(State) -> #mrst{ - language=Language, - lib=Lib, - views=Views + language = Language, + lib = Lib, + views = Views } = State, Defs = [View#mrview.def || View <- Views], {ok, QServer} = couch_query_servers:start_doc_map(Language, Defs, Lib), - State#mrst{qserver=QServer}. - + State#mrst{qserver = QServer}. accumulate_writes(State, W, Acc0) -> - {Seq, ViewKVs, DocIdKVs} = case Acc0 of - nil -> {0, [{V#mrview.id_num, []} || V <- State#mrst.views], []}; - _ -> Acc0 - end, + {Seq, ViewKVs, DocIdKVs} = + case Acc0 of + nil -> {0, [{V#mrview.id_num, []} || V <- State#mrst.views], []}; + _ -> Acc0 + end, case couch_work_queue:dequeue(W) of closed when Seq == 0 -> stop; @@ -219,15 +227,13 @@ accumulate_writes(State, W, Acc0) -> end end. - accumulate_more(NumDocIds, Acc) -> % check if we have enough items now MinItems = config:get("view_updater", "min_writer_items", "100"), MinSize = config:get("view_updater", "min_writer_size", "16777216"), CurrMem = ?term_size(Acc), - NumDocIds < list_to_integer(MinItems) - andalso CurrMem < list_to_integer(MinSize). - + NumDocIds < list_to_integer(MinItems) andalso + CurrMem < list_to_integer(MinSize). merge_results([], SeqAcc, ViewKVs, DocIdKeys) -> {SeqAcc, ViewKVs, DocIdKeys}; @@ -238,7 +244,6 @@ merge_results([{Seq, Results} | Rest], SeqAcc, ViewKVs, DocIdKeys) -> {ViewKVs1, DocIdKeys1} = lists:foldl(Fun, {ViewKVs, DocIdKeys}, Results), merge_results(Rest, erlang:max(Seq, SeqAcc), ViewKVs1, DocIdKeys1). - merge_results({DocId, []}, ViewKVs, DocIdKeys) -> {ViewKVs, [{DocId, []} | DocIdKeys]}; merge_results({DocId, RawResults}, ViewKVs, DocIdKeys) -> @@ -252,7 +257,6 @@ merge_results({DocId, RawResults}, ViewKVs, DocIdKeys) -> {ViewKVs1, [ViewIdKeys | DocIdKeys]} end. - insert_results(DocId, [], [], ViewKVs, ViewIdKeys) -> {lists:reverse(ViewKVs), {DocId, ViewIdKeys}}; insert_results(DocId, [KVs | RKVs], [{Id, VKVs} | RVKVs], VKVAcc, VIdKeys) -> @@ -266,62 +270,67 @@ insert_results(DocId, [KVs | RKVs], [{Id, VKVs} | RVKVs], VKVAcc, VIdKeys) -> end, InitAcc = {[], VIdKeys}, couch_stats:increment_counter([couchdb, mrview, emits], length(KVs)), - {Duped, VIdKeys0} = lists:foldl(CombineDupesFun, InitAcc, - lists:sort(KVs)), + {Duped, VIdKeys0} = lists:foldl( + CombineDupesFun, + InitAcc, + lists:sort(KVs) + ), FinalKVs = [{{Key, DocId}, Val} || {Key, Val} <- Duped] ++ VKVs, insert_results(DocId, RKVs, RVKVs, [{Id, FinalKVs} | VKVAcc], VIdKeys0). - write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys) -> #mrst{ - id_btree=IdBtree, - first_build=FirstBuild, - partitioned=Partitioned + id_btree = IdBtree, + first_build = FirstBuild, + partitioned = Partitioned } = State, {ok, ToRemove, IdBtree2} = update_id_btree(IdBtree, DocIdKeys, FirstBuild), ToRemByView = collapse_rem_keys(ToRemove, dict:new()), - UpdateView = fun(#mrview{id_num=ViewId}=View, {ViewId, KVs0}) -> + UpdateView = fun(#mrview{id_num = ViewId} = View, {ViewId, KVs0}) -> ToRem0 = couch_util:dict_find(ViewId, ToRemByView, []), - {KVs, ToRem} = case Partitioned of - true -> - KVs1 = inject_partition(KVs0), - ToRem1 = inject_partition(ToRem0), - {KVs1, ToRem1}; - false -> - {KVs0, ToRem0} - end, + {KVs, ToRem} = + case Partitioned of + true -> + KVs1 = inject_partition(KVs0), + ToRem1 = inject_partition(ToRem0), + {KVs1, ToRem1}; + false -> + {KVs0, ToRem0} + end, {ok, VBtree2} = couch_btree:add_remove(View#mrview.btree, KVs, ToRem), - NewUpdateSeq = case VBtree2 =/= View#mrview.btree of - true -> UpdateSeq; - _ -> View#mrview.update_seq - end, + NewUpdateSeq = + case VBtree2 =/= View#mrview.btree of + true -> UpdateSeq; + _ -> View#mrview.update_seq + end, - View2 = View#mrview{btree=VBtree2, update_seq=NewUpdateSeq}, + View2 = View#mrview{btree = VBtree2, update_seq = NewUpdateSeq}, maybe_notify(State, View2, KVs, ToRem), View2 end, State#mrst{ - views=lists:zipwith(UpdateView, State#mrst.views, ViewKVs), - update_seq=UpdateSeq, - id_btree=IdBtree2 + views = lists:zipwith(UpdateView, State#mrst.views, ViewKVs), + update_seq = UpdateSeq, + id_btree = IdBtree2 }. - inject_partition(Rows) -> - lists:map(fun - ({{Key, DocId}, Value}) -> - % Adding a row to the view - {Partition, _} = couch_partition:extract(DocId), - {{{p, Partition, Key}, DocId}, Value}; - ({Key, DocId}) -> - % Removing a row based on values in id_tree - {Partition, _} = couch_partition:extract(DocId), - {{p, Partition, Key}, DocId} - end, Rows). - + lists:map( + fun + ({{Key, DocId}, Value}) -> + % Adding a row to the view + {Partition, _} = couch_partition:extract(DocId), + {{{p, Partition, Key}, DocId}, Value}; + ({Key, DocId}) -> + % Removing a row based on values in id_tree + {Partition, _} = couch_partition:extract(DocId), + {{p, Partition, Key}, DocId} + end, + Rows + ). update_id_btree(Btree, DocIdKeys, true) -> ToAdd = [{Id, DIKeys} || {Id, DIKeys} <- DocIdKeys, DIKeys /= []], @@ -332,37 +341,38 @@ update_id_btree(Btree, DocIdKeys, _) -> ToRem = [Id || {Id, DIKeys} <- DocIdKeys, DIKeys == []], couch_btree:query_modify(Btree, ToFind, ToAdd, ToRem). - collapse_rem_keys([], Acc) -> Acc; collapse_rem_keys([{ok, {DocId, ViewIdKeys}} | Rest], Acc) -> - NewAcc = lists:foldl(fun({ViewId, Key}, Acc2) -> - dict:append(ViewId, {Key, DocId}, Acc2) - end, Acc, ViewIdKeys), + NewAcc = lists:foldl( + fun({ViewId, Key}, Acc2) -> + dict:append(ViewId, {Key, DocId}, Acc2) + end, + Acc, + ViewIdKeys + ), collapse_rem_keys(Rest, NewAcc); collapse_rem_keys([{not_found, _} | Rest], Acc) -> collapse_rem_keys(Rest, Acc). - send_partial(Pid, State) when is_pid(Pid) -> gen_server:cast(Pid, {new_state, State}); send_partial(_, _) -> ok. - update_task(NumChanges) -> [Changes, Total] = couch_task_status:get([changes_done, total_changes]), Changes2 = Changes + NumChanges, - Progress = case Total of - 0 -> - % updater restart after compaction finishes - 0; - _ -> - (Changes2 * 100) div Total - end, + Progress = + case Total of + 0 -> + % updater restart after compaction finishes + 0; + _ -> + (Changes2 * 100) div Total + end, couch_task_status:update([{progress, Progress}, {changes_done, Changes2}]). - maybe_notify(State, View, KVs, ToRem) -> Updated = fun() -> [Key || {{Key, _}, _} <- KVs] diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl index d318a3f4a..b7220f71f 100644 --- a/src/couch_mrview/src/couch_mrview_util.erl +++ b/src/couch_mrview/src/couch_mrview_util.erl @@ -38,39 +38,54 @@ -define(GET_VIEW_RETRY_DELAY, 50). -define(LOWEST_KEY, null). -define(HIGHEST_KEY, {<<255, 255, 255, 255>>}). --define(LOWEST(A, B), (if A < B -> A; true -> B end)). --define(HIGHEST(A, B), (if A > B -> A; true -> B end)). +-define(LOWEST(A, B), + (if + A < B -> A; + true -> B + end) +). +-define(HIGHEST(A, B), + (if + A > B -> A; + true -> B + end) +). -include_lib("couch/include/couch_db.hrl"). -include_lib("couch_mrview/include/couch_mrview.hrl"). - get_local_purge_doc_id(Sig) -> ?l2b(?LOCAL_DOC_PREFIX ++ "purge-mrview-" ++ Sig). - get_value_from_options(Key, Options) -> case couch_util:get_value(Key, Options) of undefined -> Reason = <<"'", Key/binary, "' must exists in options.">>, throw({bad_request, Reason}); - Value -> Value + Value -> + Value end. - verify_view_filename(FileName) -> FilePathList = filename:split(FileName), PureFN = lists:last(FilePathList), case filename:extension(PureFN) of ".view" -> Sig = filename:basename(PureFN), - case [Ch || Ch <- Sig, not (((Ch >= $0) and (Ch =< $9)) - orelse ((Ch >= $a) and (Ch =< $f)) - orelse ((Ch >= $A) and (Ch =< $F)))] == [] of + case + [ + Ch + || Ch <- Sig, + not (((Ch >= $0) and (Ch =< $9)) orelse + ((Ch >= $a) and (Ch =< $f)) orelse + ((Ch >= $A) and (Ch =< $F))) + ] == [] + of true -> true; false -> false end; - _ -> false + _ -> + false end. get_signature_from_filename(FileName) -> @@ -82,7 +97,7 @@ get_view(Db, DDoc, ViewName, Args0) -> case get_view_index_state(Db, DDoc, ViewName, Args0) of {ok, State, Args2} -> Ref = erlang:monitor(process, State#mrst.fd), - #mrst{language=Lang, views=Views} = State, + #mrst{language = Lang, views = Views} = State, {Type, View, Args3} = extract_view(Lang, Args2, ViewName, Views), check_range(Args3, view_cmp(View)), Sig = view_sig(Db, State, View, Args3), @@ -91,7 +106,6 @@ get_view(Db, DDoc, ViewName, Args0) -> ddoc_updated end. - get_view_index_pid(Db, DDoc, ViewName, Args0) -> ArgCheck = fun(InitState) -> Args1 = set_view_type(Args0, ViewName, InitState#mrst.views), @@ -99,7 +113,6 @@ get_view_index_pid(Db, DDoc, ViewName, Args0) -> end, couch_index_server:get_index(?MOD, Db, DDoc, ArgCheck). - get_view_index_state(Db, DDoc, ViewName, Args0) -> get_view_index_state(Db, DDoc, ViewName, Args0, ?GET_VIEW_RETRY_COUNT). @@ -112,17 +125,18 @@ get_view_index_state(Db, DDoc, ViewName, Args0, RetryCount) -> UpdateSeq = couch_util:with_db(Db, fun(WDb) -> couch_db:get_update_seq(WDb) end), - State = case Args#mrargs.update of - lazy -> - spawn(fun() -> - catch couch_index:get_state(Pid, UpdateSeq) - end), - couch_index:get_state(Pid, 0); - false -> - couch_index:get_state(Pid, 0); - _ -> - couch_index:get_state(Pid, UpdateSeq) - end, + State = + case Args#mrargs.update of + lazy -> + spawn(fun() -> + catch couch_index:get_state(Pid, UpdateSeq) + end), + couch_index:get_state(Pid, 0); + false -> + couch_index:get_state(Pid, 0); + _ -> + couch_index:get_state(Pid, UpdateSeq) + end, case State of {ok, State0} -> {ok, State0, Args}; ddoc_updated -> ddoc_updated; @@ -138,33 +152,37 @@ get_view_index_state(Db, DDoc, ViewName, Args0, RetryCount) -> throw(Error) end. - -ddoc_to_mrst(DbName, #doc{id=Id, body={Fields}}) -> - MakeDict = fun({Name, {MRFuns}}, DictBySrcAcc) -> - case couch_util:get_value(<<"map">>, MRFuns) of - MapSrc when MapSrc /= undefined -> - RedSrc = couch_util:get_value(<<"reduce">>, MRFuns, null), - {ViewOpts} = couch_util:get_value(<<"options">>, MRFuns, {[]}), - View = case dict:find({MapSrc, ViewOpts}, DictBySrcAcc) of - {ok, View0} -> View0; - error -> #mrview{def=MapSrc, options=ViewOpts} - end, - {MapNames, RedSrcs} = case RedSrc of - null -> - MNames = [Name | View#mrview.map_names], - {MNames, View#mrview.reduce_funs}; - _ -> - RedFuns = [{Name, RedSrc} | View#mrview.reduce_funs], - {View#mrview.map_names, RedFuns} - end, - View2 = View#mrview{map_names=MapNames, reduce_funs=RedSrcs}, - dict:store({MapSrc, ViewOpts}, View2, DictBySrcAcc); - undefined -> - DictBySrcAcc - end; +ddoc_to_mrst(DbName, #doc{id = Id, body = {Fields}}) -> + MakeDict = fun + ({Name, {MRFuns}}, DictBySrcAcc) -> + case couch_util:get_value(<<"map">>, MRFuns) of + MapSrc when MapSrc /= undefined -> + RedSrc = couch_util:get_value(<<"reduce">>, MRFuns, null), + {ViewOpts} = couch_util:get_value(<<"options">>, MRFuns, {[]}), + View = + case dict:find({MapSrc, ViewOpts}, DictBySrcAcc) of + {ok, View0} -> View0; + error -> #mrview{def = MapSrc, options = ViewOpts} + end, + {MapNames, RedSrcs} = + case RedSrc of + null -> + MNames = [Name | View#mrview.map_names], + {MNames, View#mrview.reduce_funs}; + _ -> + RedFuns = [{Name, RedSrc} | View#mrview.reduce_funs], + {View#mrview.map_names, RedFuns} + end, + View2 = View#mrview{map_names = MapNames, reduce_funs = RedSrcs}, + dict:store({MapSrc, ViewOpts}, View2, DictBySrcAcc); + undefined -> + DictBySrcAcc + end; ({Name, Else}, DictBySrcAcc) -> - couch_log:error("design_doc_to_view_group ~s views ~p", - [Name, Else]), + couch_log:error( + "design_doc_to_view_group ~s views ~p", + [Name, Else] + ), DictBySrcAcc end, {DesignOpts} = proplists:get_value(<<"options">>, Fields, {[]}), @@ -174,7 +192,7 @@ ddoc_to_mrst(DbName, #doc{id=Id, body={Fields}}) -> BySrc = lists:foldl(MakeDict, dict:new(), RawViews), NumViews = fun({_, View}, N) -> - {View#mrview{id_num=N}, N+1} + {View#mrview{id_num = N}, N + 1} end, {Views, _} = lists:mapfoldl(NumViews, 0, lists:sort(dict:to_list(BySrc))), @@ -182,17 +200,16 @@ ddoc_to_mrst(DbName, #doc{id=Id, body={Fields}}) -> Lib = couch_util:get_value(<<"lib">>, RawViews, {[]}), IdxState = #mrst{ - db_name=DbName, - idx_name=Id, - lib=Lib, - views=Views, - language=Language, - design_opts=DesignOpts, - partitioned=Partitioned + db_name = DbName, + idx_name = Id, + lib = Lib, + views = Views, + language = Language, + design_opts = DesignOpts, + partitioned = Partitioned }, SigInfo = {Views, Language, DesignOpts, couch_index_util:sort_lib(Lib)}, - {ok, IdxState#mrst{sig=couch_hash:md5_hash(term_to_binary(SigInfo))}}. - + {ok, IdxState#mrst{sig = couch_hash:md5_hash(term_to_binary(SigInfo))}}. set_view_type(_Args, _ViewName, []) -> throw({not_found, missing_named_view}); @@ -201,48 +218,44 @@ set_view_type(Args, ViewName, [View | Rest]) -> case lists:member(ViewName, RedNames) of true -> case Args#mrargs.reduce of - false -> Args#mrargs{view_type=map}; - _ -> Args#mrargs{view_type=red} + false -> Args#mrargs{view_type = map}; + _ -> Args#mrargs{view_type = red} end; false -> case lists:member(ViewName, View#mrview.map_names) of - true -> Args#mrargs{view_type=map}; + true -> Args#mrargs{view_type = map}; false -> set_view_type(Args, ViewName, Rest) end end. - set_extra(#mrargs{} = Args, Key, Value) -> Extra0 = Args#mrargs.extra, Extra1 = lists:ukeysort(1, [{Key, Value} | Extra0]), Args#mrargs{extra = Extra1}. - get_extra(#mrargs{} = Args, Key) -> couch_util:get_value(Key, Args#mrargs.extra). get_extra(#mrargs{} = Args, Key, Default) -> couch_util:get_value(Key, Args#mrargs.extra, Default). - extract_view(_Lang, _Args, _ViewName, []) -> throw({not_found, missing_named_view}); -extract_view(Lang, #mrargs{view_type=map}=Args, Name, [View | Rest]) -> +extract_view(Lang, #mrargs{view_type = map} = Args, Name, [View | Rest]) -> Names = View#mrview.map_names ++ [N || {N, _} <- View#mrview.reduce_funs], case lists:member(Name, Names) of true -> {map, View, Args}; _ -> extract_view(Lang, Args, Name, Rest) end; -extract_view(Lang, #mrargs{view_type=red}=Args, Name, [View | Rest]) -> +extract_view(Lang, #mrargs{view_type = red} = Args, Name, [View | Rest]) -> RedNames = [N || {N, _} <- View#mrview.reduce_funs], case lists:member(Name, RedNames) of true -> {red, {index_of(Name, RedNames), Lang, View}, Args}; false -> extract_view(Lang, Args, Name, Rest) end. - -view_sig(Db, State, View, #mrargs{include_docs=true}=Args) -> - BaseSig = view_sig(Db, State, View, Args#mrargs{include_docs=false}), +view_sig(Db, State, View, #mrargs{include_docs = true} = Args) -> + BaseSig = view_sig(Db, State, View, Args#mrargs{include_docs = false}), UpdateSeq = couch_db:get_update_seq(Db), PurgeSeq = couch_db:get_purge_seq(Db), Term = view_sig_term(BaseSig, UpdateSeq, PurgeSeq), @@ -254,8 +267,8 @@ view_sig(_Db, State, View, Args0) -> UpdateSeq = View#mrview.update_seq, PurgeSeq = View#mrview.purge_seq, Args = Args0#mrargs{ - preflight_fun=undefined, - extra=[] + preflight_fun = undefined, + extra = [] }, Term = view_sig_term(Sig, UpdateSeq, PurgeSeq, Args), couch_index_util:hexsig(couch_hash:md5_hash(term_to_binary(Term))). @@ -266,26 +279,25 @@ view_sig_term(BaseSig, UpdateSeq, PurgeSeq) -> view_sig_term(BaseSig, UpdateSeq, PurgeSeq, Args) -> {BaseSig, UpdateSeq, PurgeSeq, Args}. - -init_state(Db, Fd, #mrst{views=Views}=State, nil) -> +init_state(Db, Fd, #mrst{views = Views} = State, nil) -> PurgeSeq = couch_db:get_purge_seq(Db), Header = #mrheader{ - seq=0, - purge_seq=PurgeSeq, - id_btree_state=nil, - view_states=[make_view_state(#mrview{}) || _ <- Views] + seq = 0, + purge_seq = PurgeSeq, + id_btree_state = nil, + view_states = [make_view_state(#mrview{}) || _ <- Views] }, init_state(Db, Fd, State, Header); init_state(Db, Fd, State, Header) -> #mrst{ - language=Lang, - views=Views + language = Lang, + views = Views } = State, #mrheader{ - seq=Seq, - purge_seq=PurgeSeq, - id_btree_state=IdBtreeState, - view_states=ViewStates + seq = Seq, + purge_seq = PurgeSeq, + id_btree_state = IdBtreeState, + view_states = ViewStates } = maybe_update_header(Header), IdBtOpts = [ @@ -297,12 +309,12 @@ init_state(Db, Fd, State, Header) -> Views2 = lists:zipwith(OpenViewFun, ViewStates, Views), State#mrst{ - fd=Fd, - fd_monitor=erlang:monitor(process, Fd), - update_seq=Seq, - purge_seq=PurgeSeq, - id_btree=IdBtree, - views=Views2 + fd = Fd, + fd_monitor = erlang:monitor(process, Fd), + update_seq = Seq, + purge_seq = PurgeSeq, + id_btree = IdBtree, + views = Views2 }. open_view(_Db, Fd, Lang, ViewState, View) -> @@ -317,38 +329,42 @@ open_view(_Db, Fd, Lang, ViewState, View) -> ], {ok, Btree} = couch_btree:open(BTState, Fd, ViewBtOpts), - View#mrview{btree=Btree, - update_seq=get_update_seq(ViewState), - purge_seq=get_purge_seq(ViewState)}. - + View#mrview{ + btree = Btree, + update_seq = get_update_seq(ViewState), + purge_seq = get_purge_seq(ViewState) + }. temp_view_to_ddoc({Props}) -> Language = couch_util:get_value(<<"language">>, Props, <<"javascript">>), Options = couch_util:get_value(<<"options">>, Props, {[]}), View0 = [{<<"map">>, couch_util:get_value(<<"map">>, Props)}], - View1 = View0 ++ case couch_util:get_value(<<"reduce">>, Props) of - RedSrc when is_binary(RedSrc) -> [{<<"reduce">>, RedSrc}]; - _ -> [] - end, - DDoc = {[ - {<<"_id">>, couch_uuids:random()}, - {<<"language">>, Language}, - {<<"options">>, Options}, - {<<"views">>, {[ - {<<"temp">>, {View1}} - ]}} - ]}, + View1 = + View0 ++ + case couch_util:get_value(<<"reduce">>, Props) of + RedSrc when is_binary(RedSrc) -> [{<<"reduce">>, RedSrc}]; + _ -> [] + end, + DDoc = + {[ + {<<"_id">>, couch_uuids:random()}, + {<<"language">>, Language}, + {<<"options">>, Options}, + {<<"views">>, + {[ + {<<"temp">>, {View1}} + ]}} + ]}, couch_doc:from_json_obj(DDoc). - -get_row_count(#mrview{btree=Bt}) -> - Count = case couch_btree:full_reduce(Bt) of - {ok, {Count0, _Reds, _}} -> Count0; - {ok, {Count0, _Reds}} -> Count0 - end, +get_row_count(#mrview{btree = Bt}) -> + Count = + case couch_btree:full_reduce(Bt) of + {ok, {Count0, _Reds, _}} -> Count0; + {ok, {Count0, _Reds}} -> Count0 + end, {ok, Count}. - all_docs_reduce_to_count(Reductions) -> Reduce = fun couch_bt_engine:id_tree_reduce/2, {Count, _, _} = couch_btree:final_reduce(Reduce, Reductions), @@ -361,8 +377,7 @@ reduce_to_count(Reductions) -> FinalReduction = couch_btree:final_reduce(CountReduceFun, Reductions), get_count(FinalReduction). - -fold(#mrview{btree=Bt}, Fun, Acc, Opts) -> +fold(#mrview{btree = Bt}, Fun, Acc, Opts) -> WrapperFun = fun(KV, Reds, Acc2) -> fold_fun(Fun, expand_dups([KV], []), Reds, Acc2) end, @@ -370,19 +385,18 @@ fold(#mrview{btree=Bt}, Fun, Acc, Opts) -> fold_fun(_Fun, [], _, Acc) -> {ok, Acc}; -fold_fun(Fun, [KV|Rest], {KVReds, Reds}, Acc) -> +fold_fun(Fun, [KV | Rest], {KVReds, Reds}, Acc) -> case Fun(KV, {KVReds, Reds}, Acc) of {ok, Acc2} -> - fold_fun(Fun, Rest, {[KV|KVReds], Reds}, Acc2); + fold_fun(Fun, Rest, {[KV | KVReds], Reds}, Acc2); {stop, Acc2} -> {stop, Acc2} end. - -fold_reduce({NthRed, Lang, View}, Fun, Acc, Options) -> +fold_reduce({NthRed, Lang, View}, Fun, Acc, Options) -> #mrview{ - btree=Bt, - reduce_funs=RedFuns + btree = Bt, + reduce_funs = RedFuns } = View, ReduceFun = make_user_reds_reduce_fun(Lang, RedFuns, NthRed), @@ -395,13 +409,11 @@ fold_reduce({NthRed, Lang, View}, Fun, Acc, Options) -> couch_btree:fold_reduce(Bt, WrapperFun, Acc, Options). - validate_args(Db, DDoc, Args0) -> {ok, State} = couch_mrview_index:init(Db, DDoc), Args1 = apply_limit(State#mrst.partitioned, Args0), validate_args(State, Args1). - validate_args(#mrst{} = State, Args0) -> Args = validate_args(Args0), @@ -410,45 +422,55 @@ validate_args(#mrst{} = State, Args0) -> case {ViewPartitioned, Partition} of {true, undefined} -> - Msg1 = <<"`partition` parameter is mandatory " - "for queries to this view.">>, + Msg1 = << + "`partition` parameter is mandatory " + "for queries to this view." + >>, mrverror(Msg1); {true, _} -> apply_partition(Args, Partition); {false, undefined} -> Args; {false, Value} when is_binary(Value) -> - Msg2 = <<"`partition` parameter is not " - "supported in this design doc">>, + Msg2 = << + "`partition` parameter is not " + "supported in this design doc" + >>, mrverror(Msg2) end. - apply_limit(ViewPartitioned, Args) -> Options = Args#mrargs.extra, IgnorePQLimit = lists:keyfind(ignore_partition_query_limit, 1, Options), - LimitType = case {ViewPartitioned, IgnorePQLimit} of - {true, false} -> "partition_query_limit"; - {true, _} -> "query_limit"; - {false, _} -> "query_limit" - end, + LimitType = + case {ViewPartitioned, IgnorePQLimit} of + {true, false} -> "partition_query_limit"; + {true, _} -> "query_limit"; + {false, _} -> "query_limit" + end, - MaxLimit = config:get_integer("query_server_config", - LimitType, ?MAX_VIEW_LIMIT), + MaxLimit = config:get_integer( + "query_server_config", + LimitType, + ?MAX_VIEW_LIMIT + ), % Set the highest limit possible if a user has not % specified a limit - Args1 = case Args#mrargs.limit == ?MAX_VIEW_LIMIT of - true -> Args#mrargs{limit = MaxLimit}; - false -> Args - end, + Args1 = + case Args#mrargs.limit == ?MAX_VIEW_LIMIT of + true -> Args#mrargs{limit = MaxLimit}; + false -> Args + end, - if Args1#mrargs.limit =< MaxLimit -> Args1; true -> - Fmt = "Limit is too large, must not exceed ~p", - mrverror(io_lib:format(Fmt, [MaxLimit])) + if + Args1#mrargs.limit =< MaxLimit -> + Args1; + true -> + Fmt = "Limit is too large, must not exceed ~p", + mrverror(io_lib:format(Fmt, [MaxLimit])) end. - validate_all_docs_args(Db, Args0) -> Args = validate_args(Args0), @@ -465,7 +487,6 @@ validate_all_docs_args(Db, Args0) -> Args end. - validate_args(Args) -> GroupLevel = determine_group_level(Args), Reduce = Args#mrargs.reduce, @@ -480,11 +501,13 @@ validate_args(Args) -> end, case {Args#mrargs.view_type, GroupLevel, Args#mrargs.keys} of - {red, exact, _} -> ok; + {red, exact, _} -> + ok; {red, _, KeyList} when is_list(KeyList) -> Msg = <<"Multi-key fetchs for reduce views must use `group=true`">>, mrverror(Msg); - _ -> ok + _ -> + ok end, case Args#mrargs.keys of @@ -493,13 +516,18 @@ validate_args(Args) -> _ -> mrverror(<<"`keys` must be an array of strings.">>) end, - case {Args#mrargs.keys, Args#mrargs.start_key, - Args#mrargs.end_key} of - {undefined, _, _} -> ok; - {[], _, _} -> ok; - {[_|_], undefined, undefined} -> ok; - _ -> mrverror(<<"`keys` is incompatible with `key`" - ", `start_key` and `end_key`">>) + case {Args#mrargs.keys, Args#mrargs.start_key, Args#mrargs.end_key} of + {undefined, _, _} -> + ok; + {[], _, _} -> + ok; + {[_ | _], undefined, undefined} -> + ok; + _ -> + mrverror(<< + "`keys` is incompatible with `key`" + ", `start_key` and `end_key`" + >>) end, case Args#mrargs.start_key_docid of @@ -571,17 +599,19 @@ validate_args(Args) -> {red, _} -> mrverror(<<"`conflicts` is invalid for reduce views.">>) end, - SKDocId = case {Args#mrargs.direction, Args#mrargs.start_key_docid} of - {fwd, undefined} -> <<>>; - {rev, undefined} -> <<255>>; - {_, SKDocId1} -> SKDocId1 - end, + SKDocId = + case {Args#mrargs.direction, Args#mrargs.start_key_docid} of + {fwd, undefined} -> <<>>; + {rev, undefined} -> <<255>>; + {_, SKDocId1} -> SKDocId1 + end, - EKDocId = case {Args#mrargs.direction, Args#mrargs.end_key_docid} of - {fwd, undefined} -> <<255>>; - {rev, undefined} -> <<>>; - {_, EKDocId1} -> EKDocId1 - end, + EKDocId = + case {Args#mrargs.direction, Args#mrargs.end_key_docid} of + {fwd, undefined} -> <<255>>; + {rev, undefined} -> <<>>; + {_, EKDocId1} -> EKDocId1 + end, case is_boolean(Args#mrargs.sorted) of true -> ok; @@ -595,32 +625,30 @@ validate_args(Args) -> end, Args#mrargs{ - start_key_docid=SKDocId, - end_key_docid=EKDocId, - group_level=GroupLevel + start_key_docid = SKDocId, + end_key_docid = EKDocId, + group_level = GroupLevel }. - -determine_group_level(#mrargs{group=undefined, group_level=undefined}) -> +determine_group_level(#mrargs{group = undefined, group_level = undefined}) -> 0; -determine_group_level(#mrargs{group=false, group_level=undefined}) -> +determine_group_level(#mrargs{group = false, group_level = undefined}) -> 0; -determine_group_level(#mrargs{group=false, group_level=Level}) when Level > 0 -> +determine_group_level(#mrargs{group = false, group_level = Level}) when Level > 0 -> mrverror(<<"Can't specify group=false and group_level>0 at the same time">>); -determine_group_level(#mrargs{group=true, group_level=undefined}) -> +determine_group_level(#mrargs{group = true, group_level = undefined}) -> exact; -determine_group_level(#mrargs{group_level=GroupLevel}) -> +determine_group_level(#mrargs{group_level = GroupLevel}) -> GroupLevel. -apply_partition(#mrargs{keys=[{p, _, _} | _]} = Args, _Partition) -> - Args; % already applied - -apply_partition(#mrargs{keys=Keys} = Args, Partition) when Keys /= undefined -> - Args#mrargs{keys=[{p, Partition, K} || K <- Keys]}; - -apply_partition(#mrargs{start_key={p, _, _}, end_key={p, _, _}} = Args, _Partition) -> - Args; % already applied. - +apply_partition(#mrargs{keys = [{p, _, _} | _]} = Args, _Partition) -> + % already applied + Args; +apply_partition(#mrargs{keys = Keys} = Args, Partition) when Keys /= undefined -> + Args#mrargs{keys = [{p, Partition, K} || K <- Keys]}; +apply_partition(#mrargs{start_key = {p, _, _}, end_key = {p, _, _}} = Args, _Partition) -> + % already applied. + Args; apply_partition(Args, Partition) -> #mrargs{ direction = Dir, @@ -628,13 +656,22 @@ apply_partition(Args, Partition) -> end_key = EndKey } = Args, - {DefSK, DefEK} = case Dir of - fwd -> {?LOWEST_KEY, ?HIGHEST_KEY}; - rev -> {?HIGHEST_KEY, ?LOWEST_KEY} - end, + {DefSK, DefEK} = + case Dir of + fwd -> {?LOWEST_KEY, ?HIGHEST_KEY}; + rev -> {?HIGHEST_KEY, ?LOWEST_KEY} + end, - SK0 = if StartKey /= undefined -> StartKey; true -> DefSK end, - EK0 = if EndKey /= undefined -> EndKey; true -> DefEK end, + SK0 = + if + StartKey /= undefined -> StartKey; + true -> DefSK + end, + EK0 = + if + EndKey /= undefined -> EndKey; + true -> DefEK + end, Args#mrargs{ start_key = {p, Partition, SK0}, @@ -650,92 +687,101 @@ apply_all_docs_partition(#mrargs{} = Args, Partition) -> end_key = EndKey } = Args, - {DefSK, DefEK} = case Dir of - fwd -> - { - couch_partition:start_key(Partition), - couch_partition:end_key(Partition) - }; - rev -> - { - couch_partition:end_key(Partition), - couch_partition:start_key(Partition) - } - end, + {DefSK, DefEK} = + case Dir of + fwd -> + { + couch_partition:start_key(Partition), + couch_partition:end_key(Partition) + }; + rev -> + { + couch_partition:end_key(Partition), + couch_partition:start_key(Partition) + } + end, - SK0 = if StartKey == undefined -> DefSK; true -> StartKey end, - EK0 = if EndKey == undefined -> DefEK; true -> EndKey end, + SK0 = + if + StartKey == undefined -> DefSK; + true -> StartKey + end, + EK0 = + if + EndKey == undefined -> DefEK; + true -> EndKey + end, - {SK1, EK1} = case Dir of - fwd -> {?HIGHEST(DefSK, SK0), ?LOWEST(DefEK, EK0)}; - rev -> {?LOWEST(DefSK, SK0), ?HIGHEST(DefEK, EK0)} - end, + {SK1, EK1} = + case Dir of + fwd -> {?HIGHEST(DefSK, SK0), ?LOWEST(DefEK, EK0)}; + rev -> {?LOWEST(DefSK, SK0), ?HIGHEST(DefEK, EK0)} + end, Args#mrargs{ start_key = SK1, end_key = EK1 }. - -check_range(#mrargs{start_key=undefined}, _Cmp) -> +check_range(#mrargs{start_key = undefined}, _Cmp) -> ok; -check_range(#mrargs{end_key=undefined}, _Cmp) -> +check_range(#mrargs{end_key = undefined}, _Cmp) -> ok; -check_range(#mrargs{start_key=K, end_key=K}, _Cmp) -> +check_range(#mrargs{start_key = K, end_key = K}, _Cmp) -> ok; check_range(Args, Cmp) -> #mrargs{ - direction=Dir, - start_key=SK, - start_key_docid=SKD, - end_key=EK, - end_key_docid=EKD + direction = Dir, + start_key = SK, + start_key_docid = SKD, + end_key = EK, + end_key_docid = EKD } = Args, case {Dir, Cmp({SK, SKD}, {EK, EKD})} of {fwd, false} -> - throw({query_parse_error, - <<"No rows can match your key range, reverse your ", - "start_key and end_key or set descending=true">>}); + throw( + {query_parse_error, + <<"No rows can match your key range, reverse your ", + "start_key and end_key or set descending=true">>} + ); {rev, true} -> - throw({query_parse_error, - <<"No rows can match your key range, reverse your ", - "start_key and end_key or set descending=false">>}); - _ -> ok + throw( + {query_parse_error, + <<"No rows can match your key range, reverse your ", + "start_key and end_key or set descending=false">>} + ); + _ -> + ok end. - view_cmp({_Nth, _Lang, View}) -> view_cmp(View); view_cmp(View) -> fun(A, B) -> couch_btree:less(View#mrview.btree, A, B) end. - make_header(State) -> #mrst{ - update_seq=Seq, - purge_seq=PurgeSeq, - id_btree=IdBtree, - views=Views + update_seq = Seq, + purge_seq = PurgeSeq, + id_btree = IdBtree, + views = Views } = State, #mrheader{ - seq=Seq, - purge_seq=PurgeSeq, - id_btree_state=get_btree_state(IdBtree), - view_states=[make_view_state(V) || V <- Views] + seq = Seq, + purge_seq = PurgeSeq, + id_btree_state = get_btree_state(IdBtree), + view_states = [make_view_state(V) || V <- Views] }. - index_file(DbName, Sig) -> FileName = couch_index_util:hexsig(Sig) ++ ".view", couch_index_util:index_file(mrview, DbName, FileName). - compaction_file(DbName, Sig) -> FileName = couch_index_util:hexsig(Sig) ++ ".compact.view", couch_index_util:index_file(mrview, DbName, FileName). - open_file(FName) -> case couch_file:open(FName, [nologifmissing]) of {ok, Fd} -> {ok, Fd}; @@ -743,20 +789,16 @@ open_file(FName) -> Error -> Error end. - delete_files(DbName, Sig) -> delete_index_file(DbName, Sig), delete_compaction_file(DbName, Sig). - delete_index_file(DbName, Sig) -> delete_file(index_file(DbName, Sig)). - delete_compaction_file(DbName, Sig) -> delete_file(compaction_file(DbName, Sig)). - delete_file(FName) -> case filelib:is_file(FName) of true -> @@ -766,87 +808,91 @@ delete_file(FName) -> ok end. - -reset_index(Db, Fd, #mrst{sig=Sig}=State) -> +reset_index(Db, Fd, #mrst{sig = Sig} = State) -> ok = couch_file:truncate(Fd, 0), ok = couch_file:write_header(Fd, {Sig, nil}), init_state(Db, Fd, reset_state(State), nil). - reset_state(State) -> State#mrst{ - fd=nil, - qserver=nil, - update_seq=0, - id_btree=nil, - views=[View#mrview{btree=nil} || View <- State#mrst.views] + fd = nil, + qserver = nil, + update_seq = 0, + id_btree = nil, + views = [View#mrview{btree = nil} || View <- State#mrst.views] }. - all_docs_key_opts(#mrargs{extra = Extra} = Args) -> all_docs_key_opts(Args, Extra). -all_docs_key_opts(#mrargs{keys=undefined}=Args, Extra) -> - all_docs_key_opts(Args#mrargs{keys=[]}, Extra); -all_docs_key_opts(#mrargs{keys=[], direction=Dir}=Args, Extra) -> +all_docs_key_opts(#mrargs{keys = undefined} = Args, Extra) -> + all_docs_key_opts(Args#mrargs{keys = []}, Extra); +all_docs_key_opts(#mrargs{keys = [], direction = Dir} = Args, Extra) -> [[{dir, Dir}] ++ ad_skey_opts(Args) ++ ad_ekey_opts(Args) ++ Extra]; -all_docs_key_opts(#mrargs{keys=Keys, direction=Dir}=Args, Extra) -> - lists:map(fun(K) -> - [{dir, Dir}] - ++ ad_skey_opts(Args#mrargs{start_key=K}) - ++ ad_ekey_opts(Args#mrargs{end_key=K}) - ++ Extra - end, Keys). - +all_docs_key_opts(#mrargs{keys = Keys, direction = Dir} = Args, Extra) -> + lists:map( + fun(K) -> + [{dir, Dir}] ++ + ad_skey_opts(Args#mrargs{start_key = K}) ++ + ad_ekey_opts(Args#mrargs{end_key = K}) ++ + Extra + end, + Keys + ). -ad_skey_opts(#mrargs{start_key=SKey}) when is_binary(SKey) -> +ad_skey_opts(#mrargs{start_key = SKey}) when is_binary(SKey) -> [{start_key, SKey}]; -ad_skey_opts(#mrargs{start_key_docid=SKeyDocId}) -> +ad_skey_opts(#mrargs{start_key_docid = SKeyDocId}) -> [{start_key, SKeyDocId}]. - -ad_ekey_opts(#mrargs{end_key=EKey}=Args) when is_binary(EKey) -> - Type = if Args#mrargs.inclusive_end -> end_key; true -> end_key_gt end, +ad_ekey_opts(#mrargs{end_key = EKey} = Args) when is_binary(EKey) -> + Type = + if + Args#mrargs.inclusive_end -> end_key; + true -> end_key_gt + end, [{Type, EKey}]; -ad_ekey_opts(#mrargs{end_key_docid=EKeyDocId}=Args) -> - Type = if Args#mrargs.inclusive_end -> end_key; true -> end_key_gt end, +ad_ekey_opts(#mrargs{end_key_docid = EKeyDocId} = Args) -> + Type = + if + Args#mrargs.inclusive_end -> end_key; + true -> end_key_gt + end, [{Type, EKeyDocId}]. - key_opts(Args) -> key_opts(Args, []). -key_opts(#mrargs{keys=undefined, direction=Dir}=Args, Extra) -> +key_opts(#mrargs{keys = undefined, direction = Dir} = Args, Extra) -> [[{dir, Dir}] ++ skey_opts(Args) ++ ekey_opts(Args) ++ Extra]; -key_opts(#mrargs{keys=Keys, direction=Dir}=Args, Extra) -> - lists:map(fun(K) -> - [{dir, Dir}] - ++ skey_opts(Args#mrargs{start_key=K}) - ++ ekey_opts(Args#mrargs{end_key=K}) - ++ Extra - end, Keys). - +key_opts(#mrargs{keys = Keys, direction = Dir} = Args, Extra) -> + lists:map( + fun(K) -> + [{dir, Dir}] ++ + skey_opts(Args#mrargs{start_key = K}) ++ + ekey_opts(Args#mrargs{end_key = K}) ++ + Extra + end, + Keys + ). -skey_opts(#mrargs{start_key=undefined}) -> +skey_opts(#mrargs{start_key = undefined}) -> []; -skey_opts(#mrargs{start_key=SKey, start_key_docid=SKeyDocId}) -> +skey_opts(#mrargs{start_key = SKey, start_key_docid = SKeyDocId}) -> [{start_key, {SKey, SKeyDocId}}]. - -ekey_opts(#mrargs{end_key=undefined}) -> +ekey_opts(#mrargs{end_key = undefined}) -> []; -ekey_opts(#mrargs{end_key=EKey, end_key_docid=EKeyDocId}=Args) -> +ekey_opts(#mrargs{end_key = EKey, end_key_docid = EKeyDocId} = Args) -> case Args#mrargs.inclusive_end of true -> [{end_key, {EKey, EKeyDocId}}]; false -> [{end_key_gt, {EKey, reverse_key_default(EKeyDocId)}}] end. - reverse_key_default(<<>>) -> <<255>>; reverse_key_default(<<255>>) -> <<>>; reverse_key_default(Key) -> Key. - reduced_external_size(Tree) -> case couch_btree:full_reduce(Tree) of {ok, {_, _, Size}} -> Size; @@ -854,35 +900,31 @@ reduced_external_size(Tree) -> {ok, {_, _}} -> 0 end. - calculate_external_size(Views) -> SumFun = fun - (#mrview{btree=nil}, Acc) -> + (#mrview{btree = nil}, Acc) -> Acc; - (#mrview{btree=Bt}, Acc) -> + (#mrview{btree = Bt}, Acc) -> Acc + reduced_external_size(Bt) end, {ok, lists:foldl(SumFun, 0, Views)}. - calculate_active_size(Views) -> FoldFun = fun - (#mrview{btree=nil}, Acc) -> + (#mrview{btree = nil}, Acc) -> Acc; - (#mrview{btree=Bt}, Acc) -> + (#mrview{btree = Bt}, Acc) -> Acc + couch_btree:size(Bt) end, {ok, lists:foldl(FoldFun, 0, Views)}. - detuple_kvs([], Acc) -> lists:reverse(Acc); detuple_kvs([KV | Rest], Acc) -> - {{Key,Id},Value} = KV, + {{Key, Id}, Value} = KV, NKV = [[Key, Id], Value], detuple_kvs(Rest, [NKV | Acc]). - expand_dups([], Acc) -> lists:reverse(Acc); expand_dups([{Key, {dups, Vals}} | Rest], Acc) -> @@ -891,56 +933,49 @@ expand_dups([{Key, {dups, Vals}} | Rest], Acc) -> expand_dups([KV | Rest], Acc) -> expand_dups(Rest, [KV | Acc]). - -maybe_load_doc(_Db, _DI, #mrargs{include_docs=false}) -> +maybe_load_doc(_Db, _DI, #mrargs{include_docs = false}) -> []; -maybe_load_doc(Db, #doc_info{}=DI, #mrargs{conflicts=true, doc_options=Opts}) -> +maybe_load_doc(Db, #doc_info{} = DI, #mrargs{conflicts = true, doc_options = Opts}) -> doc_row(couch_index_util:load_doc(Db, DI, [conflicts]), Opts); -maybe_load_doc(Db, #doc_info{}=DI, #mrargs{doc_options=Opts}) -> +maybe_load_doc(Db, #doc_info{} = DI, #mrargs{doc_options = Opts}) -> doc_row(couch_index_util:load_doc(Db, DI, []), Opts). - -maybe_load_doc(_Db, _Id, _Val, #mrargs{include_docs=false}) -> +maybe_load_doc(_Db, _Id, _Val, #mrargs{include_docs = false}) -> []; -maybe_load_doc(Db, Id, Val, #mrargs{conflicts=true, doc_options=Opts}) -> +maybe_load_doc(Db, Id, Val, #mrargs{conflicts = true, doc_options = Opts}) -> doc_row(couch_index_util:load_doc(Db, docid_rev(Id, Val), [conflicts]), Opts); -maybe_load_doc(Db, Id, Val, #mrargs{doc_options=Opts}) -> +maybe_load_doc(Db, Id, Val, #mrargs{doc_options = Opts}) -> doc_row(couch_index_util:load_doc(Db, docid_rev(Id, Val), []), Opts). - doc_row(null, _Opts) -> [{doc, null}]; doc_row(Doc, Opts) -> [{doc, couch_doc:to_json_obj(Doc, Opts)}]. - docid_rev(Id, {Props}) -> DocId = couch_util:get_value(<<"_id">>, Props, Id), - Rev = case couch_util:get_value(<<"_rev">>, Props, nil) of - nil -> nil; - Rev0 -> couch_doc:parse_rev(Rev0) - end, + Rev = + case couch_util:get_value(<<"_rev">>, Props, nil) of + nil -> nil; + Rev0 -> couch_doc:parse_rev(Rev0) + end, {DocId, Rev}; docid_rev(Id, _) -> {Id, nil}. - index_of(Key, List) -> index_of(Key, List, 1). - index_of(_, [], _) -> throw({error, missing_named_view}); index_of(Key, [Key | _], Idx) -> Idx; index_of(Key, [_ | Rest], Idx) -> - index_of(Key, Rest, Idx+1). - + index_of(Key, Rest, Idx + 1). mrverror(Mesg) -> throw({query_parse_error, Mesg}). - %% Updates 2.x view files to 3.x or later view files %% transparently, the first time the 2.x view file is opened by %% 3.x or later. @@ -960,11 +995,11 @@ maybe_update_index_file(State) -> % open in read-only mode so we don't create % the file if it doesn't exist. case file:open(NewIndexFile, [read, raw]) of - {ok, Fd_Read} -> - % the new index file exists, there is nothing to do here. - file:close(Fd_Read); - _Error -> - update_index_file(State) + {ok, Fd_Read} -> + % the new index file exists, there is nothing to do here. + file:close(Fd_Read); + _Error -> + update_index_file(State) end. update_index_file(State) -> @@ -975,26 +1010,35 @@ update_index_file(State) -> % If we have an old index, rename it to the new position. case file:read_file_info(IndexFile) of - {ok, _FileInfo} -> - % Crash if the rename fails for any reason. - % If the target exists, e.g. the next request will find the - % new file and we are good. We might need to catch this - % further up to avoid a full server crash. - NewIndexFile = index_file(DbName, State#mrst.sig), - couch_log:notice("Attempting to update legacy view index file" - " from ~p to ~s", [IndexFile, NewIndexFile]), - ok = filelib:ensure_dir(NewIndexFile), - ok = file:rename(IndexFile, NewIndexFile), - couch_log:notice("Successfully updated legacy view index file" - " ~s", [IndexFile]), - Sig; - {error, enoent} -> - % Ignore missing index file - ok; - {error, Reason} -> - couch_log:error("Failed to update legacy view index file" - " ~s : ~s", [IndexFile, file:format_error(Reason)]), - ok + {ok, _FileInfo} -> + % Crash if the rename fails for any reason. + % If the target exists, e.g. the next request will find the + % new file and we are good. We might need to catch this + % further up to avoid a full server crash. + NewIndexFile = index_file(DbName, State#mrst.sig), + couch_log:notice( + "Attempting to update legacy view index file" + " from ~p to ~s", + [IndexFile, NewIndexFile] + ), + ok = filelib:ensure_dir(NewIndexFile), + ok = file:rename(IndexFile, NewIndexFile), + couch_log:notice( + "Successfully updated legacy view index file" + " ~s", + [IndexFile] + ), + Sig; + {error, enoent} -> + % Ignore missing index file + ok; + {error, Reason} -> + couch_log:error( + "Failed to update legacy view index file" + " ~s : ~s", + [IndexFile, file:format_error(Reason)] + ), + ok end. sig_vsn_2x(State) -> @@ -1010,21 +1054,21 @@ sig_vsn_2x(State) -> couch_hash:md5_hash(term_to_binary(SigInfo)). old_view_format(View, SI, KSI) -> -{ - mrview, - View#mrview.id_num, - View#mrview.update_seq, - View#mrview.purge_seq, - View#mrview.map_names, - View#mrview.reduce_funs, - View#mrview.def, - View#mrview.btree, - nil, - nil, - SI, - KSI, - View#mrview.options -}. + { + mrview, + View#mrview.id_num, + View#mrview.update_seq, + View#mrview.purge_seq, + View#mrview.map_names, + View#mrview.reduce_funs, + View#mrview.def, + View#mrview.btree, + nil, + nil, + SI, + KSI, + View#mrview.options + }. maybe_update_header(#mrheader{} = Header) -> Header; @@ -1050,7 +1094,6 @@ make_view_state({BTState, _SeqBTState, _KSeqBTState, UpdateSeq, PurgeSeq}) -> make_view_state(nil) -> {nil, 0, 0}. - get_key_btree_state(ViewState) -> element(1, ViewState). @@ -1066,18 +1109,14 @@ get_count(Reduction) -> get_user_reds(Reduction) -> element(2, Reduction). - % This is for backwards compatibility for seq btree reduces get_external_size_reds(Reduction) when is_integer(Reduction) -> 0; - get_external_size_reds(Reduction) when tuple_size(Reduction) == 2 -> 0; - get_external_size_reds(Reduction) when tuple_size(Reduction) == 3 -> element(3, Reduction). - make_reduce_fun(Lang, ReduceFuns) -> FunSrcs = [FunSrc || {_, FunSrc} <- ReduceFuns], fun @@ -1093,20 +1132,21 @@ make_reduce_fun(Lang, ReduceFuns) -> ExtAcc = ExtAcc0 + get_external_size_reds(Red), {CountsAcc, URedsAcc, ExtAcc} end, - {Counts, UReds, ExternalSize} = lists:foldl(ExtractFun, - {0, [], 0}, Reds), + {Counts, UReds, ExternalSize} = lists:foldl( + ExtractFun, + {0, [], 0}, + Reds + ), {ok, Result} = couch_query_servers:rereduce(Lang, FunSrcs, UReds), {Counts, Result, ExternalSize} end. - maybe_define_less_fun(#mrview{options = Options}) -> case couch_util:get_value(<<"collation">>, Options) of <<"raw">> -> undefined; _ -> fun couch_ejson_compare:less_json_ids/2 end. - count_reduce(reduce, KVs) -> CountFun = fun ({_, {dups, Vals}}, Acc) -> Acc + length(Vals); @@ -1121,7 +1161,6 @@ count_reduce(rereduce, Reds) -> Count = lists:foldl(CountFun, 0, Reds), {Count, []}. - make_user_reds_reduce_fun(Lang, ReduceFuns, NthRed) -> LPad = lists:duplicate(NthRed - 1, []), RPad = lists:duplicate(length(ReduceFuns) - NthRed, []), @@ -1140,18 +1179,15 @@ make_user_reds_reduce_fun(Lang, ReduceFuns, NthRed) -> {0, LPad ++ Result ++ RPad} end. - get_btree_state(nil) -> nil; get_btree_state(#btree{} = Btree) -> couch_btree:get_state(Btree). - -extract_view_reduce({red, {N, _Lang, #mrview{reduce_funs=Reds}}, _Ref}) -> +extract_view_reduce({red, {N, _Lang, #mrview{reduce_funs = Reds}}, _Ref}) -> {_Name, FunSrc} = lists:nth(N, Reds), FunSrc. - get_view_keys({Props}) -> case couch_util:get_value(<<"keys">>, Props) of undefined -> @@ -1162,7 +1198,6 @@ get_view_keys({Props}) -> throw({bad_request, "`keys` member must be an array."}) end. - get_view_queries({Props}) -> case couch_util:get_value(<<"queries">>, Props) of undefined -> @@ -1173,8 +1208,11 @@ get_view_queries({Props}) -> throw({bad_request, "`queries` member must be an array."}) end. - kv_external_size(KVList, Reduction) -> - lists:foldl(fun([[Key, _], Value], Acc) -> - ?term_size(Key) + ?term_size(Value) + Acc - end, ?term_size(Reduction), KVList). + lists:foldl( + fun([[Key, _], Value], Acc) -> + ?term_size(Key) + ?term_size(Value) + Acc + end, + ?term_size(Reduction), + KVList + ). diff --git a/src/couch_mrview/test/eunit/couch_mrview_all_docs_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_all_docs_tests.erl index bf8eb7e5b..1a81d4f0a 100644 --- a/src/couch_mrview/test/eunit/couch_mrview_all_docs_tests.erl +++ b/src/couch_mrview/test/eunit/couch_mrview_all_docs_tests.erl @@ -17,8 +17,6 @@ -define(TIMEOUT, 1000). - - setup() -> {ok, Db} = couch_mrview_test_util:init_db(?tempdb(), map), Db. @@ -28,16 +26,17 @@ teardown(Db) -> couch_server:delete(couch_db:name(Db), [?ADMIN_CTX]), ok. - all_docs_test_() -> { "_all_docs view tests", { setup, - fun test_util:start_couch/0, fun test_util:stop_couch/1, + fun test_util:start_couch/0, + fun test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_query/1, fun should_query_with_range/1, @@ -50,47 +49,50 @@ all_docs_test_() -> } }. - should_query(Db) -> Result = run_query(Db, []), - Expect = {ok, [ - {meta, [{total, 11}, {offset, 0}]}, - mk_row(<<"1">>, <<"1-08d53a5760b95fce6df2e2c5b008be39">>), - mk_row(<<"10">>, <<"1-a05b6ea2bc0243949f103d5b4f15f71e">>), - mk_row(<<"2">>, <<"1-b57c77a9e6f7574ca6469f0d6dcd78bb">>), - mk_row(<<"3">>, <<"1-7fbf84d56f8017880974402d60f5acd6">>), - mk_row(<<"4">>, <<"1-fcaf5852c08ffb239ac8ce16c409f253">>), - mk_row(<<"5">>, <<"1-aaac5d460fd40f9286e57b9bf12e23d2">>), - mk_row(<<"6">>, <<"1-aca21c2e7bc5f8951424fcfc5d1209d8">>), - mk_row(<<"7">>, <<"1-4374aeec17590d82f16e70f318116ad9">>), - mk_row(<<"8">>, <<"1-55b9a29311341e07ec0a7ca13bc1b59f">>), - mk_row(<<"9">>, <<"1-558c8487d9aee25399a91b5d31d90fe2">>), - mk_row(<<"_design/bar">>, <<"1-a44e1dd1994a7717bf89c894ebd1f081">>) - ]}, + Expect = + {ok, [ + {meta, [{total, 11}, {offset, 0}]}, + mk_row(<<"1">>, <<"1-08d53a5760b95fce6df2e2c5b008be39">>), + mk_row(<<"10">>, <<"1-a05b6ea2bc0243949f103d5b4f15f71e">>), + mk_row(<<"2">>, <<"1-b57c77a9e6f7574ca6469f0d6dcd78bb">>), + mk_row(<<"3">>, <<"1-7fbf84d56f8017880974402d60f5acd6">>), + mk_row(<<"4">>, <<"1-fcaf5852c08ffb239ac8ce16c409f253">>), + mk_row(<<"5">>, <<"1-aaac5d460fd40f9286e57b9bf12e23d2">>), + mk_row(<<"6">>, <<"1-aca21c2e7bc5f8951424fcfc5d1209d8">>), + mk_row(<<"7">>, <<"1-4374aeec17590d82f16e70f318116ad9">>), + mk_row(<<"8">>, <<"1-55b9a29311341e07ec0a7ca13bc1b59f">>), + mk_row(<<"9">>, <<"1-558c8487d9aee25399a91b5d31d90fe2">>), + mk_row(<<"_design/bar">>, <<"1-a44e1dd1994a7717bf89c894ebd1f081">>) + ]}, ?_assertEqual(Expect, Result). should_query_with_range(Db) -> Result = run_query(Db, [{start_key, <<"3">>}, {end_key, <<"5">>}]), - Expect = {ok, [ - {meta, [{total, 11}, {offset, 3}]}, - mk_row(<<"3">>, <<"1-7fbf84d56f8017880974402d60f5acd6">>), - mk_row(<<"4">>, <<"1-fcaf5852c08ffb239ac8ce16c409f253">>), - mk_row(<<"5">>, <<"1-aaac5d460fd40f9286e57b9bf12e23d2">>) - ]}, + Expect = + {ok, [ + {meta, [{total, 11}, {offset, 3}]}, + mk_row(<<"3">>, <<"1-7fbf84d56f8017880974402d60f5acd6">>), + mk_row(<<"4">>, <<"1-fcaf5852c08ffb239ac8ce16c409f253">>), + mk_row(<<"5">>, <<"1-aaac5d460fd40f9286e57b9bf12e23d2">>) + ]}, ?_assertEqual(Expect, Result). should_query_with_range_rev(Db) -> Result = run_query(Db, [ {direction, rev}, - {start_key, <<"5">>}, {end_key, <<"3">>}, + {start_key, <<"5">>}, + {end_key, <<"3">>}, {inclusive_end, true} ]), - Expect = {ok, [ - {meta, [{total, 11}, {offset, 5}]}, - mk_row(<<"5">>, <<"1-aaac5d460fd40f9286e57b9bf12e23d2">>), - mk_row(<<"4">>, <<"1-fcaf5852c08ffb239ac8ce16c409f253">>), - mk_row(<<"3">>, <<"1-7fbf84d56f8017880974402d60f5acd6">>) - ]}, + Expect = + {ok, [ + {meta, [{total, 11}, {offset, 5}]}, + mk_row(<<"5">>, <<"1-aaac5d460fd40f9286e57b9bf12e23d2">>), + mk_row(<<"4">>, <<"1-fcaf5852c08ffb239ac8ce16c409f253">>), + mk_row(<<"3">>, <<"1-7fbf84d56f8017880974402d60f5acd6">>) + ]}, ?_assertEqual(Expect, Result). should_query_with_limit_and_skip(Db) -> @@ -99,12 +101,13 @@ should_query_with_limit_and_skip(Db) -> {limit, 3}, {skip, 3} ]), - Expect = {ok, [ - {meta, [{total, 11}, {offset, 5}]}, - mk_row(<<"5">>, <<"1-aaac5d460fd40f9286e57b9bf12e23d2">>), - mk_row(<<"6">>, <<"1-aca21c2e7bc5f8951424fcfc5d1209d8">>), - mk_row(<<"7">>, <<"1-4374aeec17590d82f16e70f318116ad9">>) - ]}, + Expect = + {ok, [ + {meta, [{total, 11}, {offset, 5}]}, + mk_row(<<"5">>, <<"1-aaac5d460fd40f9286e57b9bf12e23d2">>), + mk_row(<<"6">>, <<"1-aca21c2e7bc5f8951424fcfc5d1209d8">>), + mk_row(<<"7">>, <<"1-4374aeec17590d82f16e70f318116ad9">>) + ]}, ?_assertEqual(Expect, Result). should_query_with_include_docs(Db) -> @@ -113,26 +116,28 @@ should_query_with_include_docs(Db) -> {end_key, <<"8">>}, {include_docs, true} ]), - Doc = {[ - {<<"_id">>,<<"8">>}, - {<<"_rev">>, <<"1-55b9a29311341e07ec0a7ca13bc1b59f">>}, - {<<"val">>, 8} - ]}, + Doc = + {[ + {<<"_id">>, <<"8">>}, + {<<"_rev">>, <<"1-55b9a29311341e07ec0a7ca13bc1b59f">>}, + {<<"val">>, 8} + ]}, Val = {[{rev, <<"1-55b9a29311341e07ec0a7ca13bc1b59f">>}]}, - Expect = {ok, [ - {meta, [{total, 11}, {offset, 8}]}, - {row, [{id, <<"8">>}, {key, <<"8">>}, {value, Val}, {doc, Doc}]} - ]}, + Expect = + {ok, [ + {meta, [{total, 11}, {offset, 8}]}, + {row, [{id, <<"8">>}, {key, <<"8">>}, {value, Val}, {doc, Doc}]} + ]}, ?_assertEqual(Expect, Result). should_query_empty_views(Db) -> Result = couch_mrview:query_view(Db, <<"_design/bar">>, <<"bing">>), - Expect = {ok, [ - {meta, [{total, 0}, {offset, 0}]} - ]}, + Expect = + {ok, [ + {meta, [{total, 0}, {offset, 0}]} + ]}, ?_assertEqual(Expect, Result). - mk_row(Id, Rev) -> {row, [{id, Id}, {key, Id}, {value, {[{rev, Rev}]}}]}. diff --git a/src/couch_mrview/test/eunit/couch_mrview_collation_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_collation_tests.erl index 0d2afbe3c..c00b97b33 100644 --- a/src/couch_mrview/test/eunit/couch_mrview_collation_tests.erl +++ b/src/couch_mrview/test/eunit/couch_mrview_collation_tests.erl @@ -56,10 +56,8 @@ % Values with depth > 10 trigger the erlang collation fallback in couch_ejson_compare {[{<<"x">>, [[[[[[[[[[[<<"y">>]]]]]]]]]]]}]} - ]). - setup() -> {ok, Db1} = couch_mrview_test_util:new_db(?tempdb(), map), Docs = [couch_mrview_test_util:ddoc(red) | make_docs()], @@ -71,16 +69,17 @@ teardown(Db) -> couch_server:delete(couch_db:name(Db), [?ADMIN_CTX]), ok. - collation_test_() -> { "Collation tests", { setup, - fun test_util:start_couch/0, fun test_util:stop_couch/1, + fun test_util:start_couch/0, + fun test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_collate_fwd/1, fun should_collate_rev/1, @@ -96,7 +95,6 @@ collation_test_() -> } }. - should_collate_fwd(Db) -> {ok, Results} = run_query(Db, []), Expect = [{meta, [{total, length(?VALUES)}, {offset, 0}]}] ++ rows(), @@ -108,100 +106,127 @@ should_collate_rev(Db) -> ?_assertEquiv(Expect, Results). should_collate_range_(Db) -> - Index = lists:zip(lists:seq(0, length(?VALUES)-1), ?VALUES), - lists:map(fun(V) -> - {ok, Results} = run_query(Db, [{start_key, V}, {end_key, V}]), - Expect = [ - {meta, [{total, length(?VALUES)}, find_offset(Index, V)]} | - find_matching_rows(Index, V) - ], - ?_assertEquiv(Expect, Results) - end, ?VALUES). + Index = lists:zip(lists:seq(0, length(?VALUES) - 1), ?VALUES), + lists:map( + fun(V) -> + {ok, Results} = run_query(Db, [{start_key, V}, {end_key, V}]), + Expect = [ + {meta, [{total, length(?VALUES)}, find_offset(Index, V)]} + | find_matching_rows(Index, V) + ], + ?_assertEquiv(Expect, Results) + end, + ?VALUES + ). find_offset(Index, Value) -> - [{Offset, _} | _] = lists:dropwhile(fun({_, V}) -> - couch_ejson_compare:less(Value, V) =/= 0 - end, Index), + [{Offset, _} | _] = lists:dropwhile( + fun({_, V}) -> + couch_ejson_compare:less(Value, V) =/= 0 + end, + Index + ), {offset, Offset}. find_matching_rows(Index, Value) -> - Matches = lists:filter(fun({_, V}) -> - couch_ejson_compare:less(Value, V) =:= 0 - end, Index), - lists:map(fun({Id, V}) -> - {row, [{id, list_to_binary(integer_to_list(Id))}, {key, V}, {value, 0}]} - end, Matches). + Matches = lists:filter( + fun({_, V}) -> + couch_ejson_compare:less(Value, V) =:= 0 + end, + Index + ), + lists:map( + fun({Id, V}) -> + {row, [{id, list_to_binary(integer_to_list(Id))}, {key, V}, {value, 0}]} + end, + Matches + ). should_collate_with_inclusive_end_fwd(Db) -> Opts = [{end_key, <<"b">>}, {inclusive_end, true}], {ok, Rows0} = run_query(Db, Opts), LastRow = lists:last(Rows0), - Expect = {row, [{id,<<"10">>}, {key,<<"b">>}, {value,0}]}, + Expect = {row, [{id, <<"10">>}, {key, <<"b">>}, {value, 0}]}, ?_assertEqual(Expect, LastRow). should_collate_with_inclusive_end_rev(Db) -> Opts = [{end_key, <<"b">>}, {inclusive_end, true}, {direction, rev}], {ok, Rows} = run_query(Db, Opts), LastRow = lists:last(Rows), - Expect = {row, [{id,<<"10">>}, {key,<<"b">>}, {value,0}]}, + Expect = {row, [{id, <<"10">>}, {key, <<"b">>}, {value, 0}]}, ?_assertEqual(Expect, LastRow). should_collate_without_inclusive_end_fwd(Db) -> Opts = [{end_key, <<"b">>}, {inclusive_end, false}], {ok, Rows0} = run_query(Db, Opts), LastRow = lists:last(Rows0), - Expect = {row, [{id,<<"9">>}, {key,<<"aa">>}, {value,0}]}, + Expect = {row, [{id, <<"9">>}, {key, <<"aa">>}, {value, 0}]}, ?_assertEqual(Expect, LastRow). should_collate_without_inclusive_end_rev(Db) -> Opts = [{end_key, <<"b">>}, {inclusive_end, false}, {direction, rev}], {ok, Rows} = run_query(Db, Opts), LastRow = lists:last(Rows), - Expect = {row, [{id,<<"11">>}, {key,<<"B">>}, {value,0}]}, + Expect = {row, [{id, <<"11">>}, {key, <<"B">>}, {value, 0}]}, ?_assertEqual(Expect, LastRow). should_collate_with_endkey_docid(Db) -> ?_test(begin {ok, Rows0} = run_query(Db, [ - {end_key, <<"b">>}, {end_key_docid, <<"10">>}, + {end_key, <<"b">>}, + {end_key_docid, <<"10">>}, {inclusive_end, false} ]), Result0 = lists:last(Rows0), - Expect0 = {row, [{id,<<"9">>}, {key,<<"aa">>}, {value,0}]}, + Expect0 = {row, [{id, <<"9">>}, {key, <<"aa">>}, {value, 0}]}, ?assertEqual(Expect0, Result0), {ok, Rows1} = run_query(Db, [ - {end_key, <<"b">>}, {end_key_docid, <<"11">>}, + {end_key, <<"b">>}, + {end_key_docid, <<"11">>}, {inclusive_end, false} ]), Result1 = lists:last(Rows1), - Expect1 = {row, [{id,<<"10">>}, {key,<<"b">>}, {value,0}]}, + Expect1 = {row, [{id, <<"10">>}, {key, <<"b">>}, {value, 0}]}, ?assertEqual(Expect1, Result1) end). should_use_collator_for_reduce_grouping(Db) -> - UniqueKeys = lists:usort(fun(A, B) -> - not couch_ejson_compare:less_json(B, A) - end, ?VALUES), - {ok, [{meta,_} | Rows]} = reduce_query(Db, [{group_level, exact}]), + UniqueKeys = lists:usort( + fun(A, B) -> + not couch_ejson_compare:less_json(B, A) + end, + ?VALUES + ), + {ok, [{meta, _} | Rows]} = reduce_query(Db, [{group_level, exact}]), ?_assertEqual(length(UniqueKeys), length(Rows)). make_docs() -> - {Docs, _} = lists:foldl(fun(V, {Docs0, Count}) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, list_to_binary(integer_to_list(Count))}, - {<<"foo">>, V} - ]}), - {[Doc | Docs0], Count+1} - end, {[], 0}, ?VALUES), + {Docs, _} = lists:foldl( + fun(V, {Docs0, Count}) -> + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, list_to_binary(integer_to_list(Count))}, + {<<"foo">>, V} + ]} + ), + {[Doc | Docs0], Count + 1} + end, + {[], 0}, + ?VALUES + ), Docs. rows() -> - {Rows, _} = lists:foldl(fun(V, {Rows0, Count}) -> - Id = list_to_binary(integer_to_list(Count)), - Row = {row, [{id, Id}, {key, V}, {value, 0}]}, - {[Row | Rows0], Count+1} - end, {[], 0}, ?VALUES), + {Rows, _} = lists:foldl( + fun(V, {Rows0, Count}) -> + Id = list_to_binary(integer_to_list(Count)), + Row = {row, [{id, Id}, {key, V}, {value, 0}]}, + {[Row | Rows0], Count + 1} + end, + {[], 0}, + ?VALUES + ), lists:reverse(Rows). run_query(Db, Opts) -> diff --git a/src/couch_mrview/test/eunit/couch_mrview_compact_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_compact_tests.erl index 7664becdc..df035c649 100644 --- a/src/couch_mrview/test/eunit/couch_mrview_compact_tests.erl +++ b/src/couch_mrview/test/eunit/couch_mrview_compact_tests.erl @@ -17,7 +17,6 @@ -define(TIMEOUT, 1000). - setup() -> {ok, Db} = couch_mrview_test_util:init_db(?tempdb(), map, 1000), ok = meck:new(couch_mrview_compactor, [passthrough]), @@ -29,16 +28,17 @@ teardown(Db) -> couch_server:delete(couch_db:name(Db), [?ADMIN_CTX]), ok. - compaction_test_() -> { "Compaction tests", { setup, - fun test_util:start_couch/0, fun test_util:stop_couch/1, + fun test_util:start_couch/0, + fun test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_swap/1, fun should_remove/1 @@ -47,7 +47,6 @@ compaction_test_() -> } }. - should_swap(Db) -> ?_test(begin couch_mrview:query_view(Db, <<"_design/bar">>, <<"baz">>), @@ -57,9 +56,12 @@ should_swap(Db) -> {'DOWN', MonRef, process, _, _} -> ok after ?TIMEOUT -> erlang:error( - {assertion_failed, - [{module, ?MODULE}, {line, ?LINE}, - {reason, "compaction failed"}]}) + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, "compaction failed"} + ]} + ) end, QPid ! {self(), continue}, receive @@ -67,13 +69,15 @@ should_swap(Db) -> ?assertEqual(1000, Count) after ?TIMEOUT -> erlang:error( - {assertion_failed, - [{module, ?MODULE}, {line, ?LINE}, - {reason, "query failed"}]}) + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, "query failed"} + ]} + ) end end). - should_remove(Db) -> ?_test(begin DDoc = <<"_design/bar">>, @@ -87,29 +91,40 @@ should_remove(Db) -> receive {'DOWN', MonRef, process, _, crash} -> meck:wait(couch_mrview_compactor, remove_compacted, '_', 100), - ?assertEqual(1, meck:num_calls( - couch_mrview_compactor, remove_compacted, '_', IndexPid)), + ?assertEqual( + 1, + meck:num_calls( + couch_mrview_compactor, remove_compacted, '_', IndexPid + ) + ), ?assert(is_process_alive(IndexPid)), ?assert(is_process_alive(CompactorPid)) after ?TIMEOUT -> erlang:error( {assertion_failed, [ - {module, ?MODULE}, {line, ?LINE}, - {reason, "compaction didn't exit :/"}]}) + {module, ?MODULE}, + {line, ?LINE}, + {reason, "compaction didn't exit :/"} + ]} + ) end end). - start_query(Db) -> Self = self(), Pid = spawn(fun() -> CB = fun - (_, wait) -> receive {Self, continue} -> {ok, 0} end; - ({row, _}, Count) -> {ok, Count+1}; - (_, Count) -> {ok, Count} + (_, wait) -> + receive + {Self, continue} -> {ok, 0} + end; + ({row, _}, Count) -> + {ok, Count + 1}; + (_, Count) -> + {ok, Count} end, {ok, Result} = - couch_mrview:query_view(Db, <<"_design/bar">>, <<"baz">>, [], CB, wait), + couch_mrview:query_view(Db, <<"_design/bar">>, <<"baz">>, [], CB, wait), Self ! {self(), Result} end), {ok, Pid}. diff --git a/src/couch_mrview/test/eunit/couch_mrview_ddoc_updated_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_ddoc_updated_tests.erl index 4310157eb..2a6299448 100644 --- a/src/couch_mrview/test/eunit/couch_mrview_ddoc_updated_tests.erl +++ b/src/couch_mrview/test/eunit/couch_mrview_ddoc_updated_tests.erl @@ -17,30 +17,33 @@ -define(TIMEOUT, 1000). - setup() -> Name = ?tempdb(), couch_server:delete(Name, [?ADMIN_CTX]), {ok, Db} = couch_db:create(Name, [?ADMIN_CTX]), - DDoc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/bar">>}, - {<<"views">>, {[ - {<<"baz">>, {[ - {<<"map">>, << - "function(doc) {\n" - " emit(doc.val, doc.val);\n" - "}" - >>} - ]}} - ]}} - ]}), + DDoc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/bar">>}, + {<<"views">>, + {[ + {<<"baz">>, + {[ + {<<"map">>, << + "function(doc) {\n" + " emit(doc.val, doc.val);\n" + "}" + >>} + ]}} + ]}} + ]} + ), [Doc1 | Docs999] = couch_mrview_test_util:make_docs(map, 100), {ok, _} = couch_db:update_docs(Db, [DDoc, Doc1], []), {ok, Db2} = couch_db:reopen(Db), % run a query with 1 doc to initialize couch_index process CB = fun - ({row, _}, Count) -> {ok, Count+1}; + ({row, _}, Count) -> {ok, Count + 1}; (_, Count) -> {ok, Count} end, {ok, _} = @@ -63,16 +66,17 @@ teardown(Db) -> couch_server:delete(couch_db:name(Db), [?ADMIN_CTX]), ok. - ddoc_update_test_() -> { "Check ddoc update actions", { setup, - fun test_util:start_couch/0, fun test_util:stop_couch/1, + fun test_util:start_couch/0, + fun test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun check_indexing_stops_on_ddoc_change/1 ] @@ -80,7 +84,6 @@ ddoc_update_test_() -> } }. - check_indexing_stops_on_ddoc_change(Db) -> ?_test(begin DDocID = <<"_design/bar">>, @@ -91,17 +94,20 @@ check_indexing_stops_on_ddoc_change(Db) -> ?assertEqual(1, length(AliveBefore)), {ok, DDoc} = couch_db:open_doc(Db, DDocID, [ejson_body, ?ADMIN_CTX]), - DDocJson2 = couch_doc:from_json_obj({[ - {<<"_id">>, DDocID}, - {<<"_deleted">>, true}, - {<<"_rev">>, couch_doc:rev_to_str(DDoc#doc.revs)} - ]}), + DDocJson2 = couch_doc:from_json_obj( + {[ + {<<"_id">>, DDocID}, + {<<"_deleted">>, true}, + {<<"_rev">>, couch_doc:rev_to_str(DDoc#doc.revs)} + ]} + ), % spawn a process for query Self = self(), QPid = spawn(fun() -> {ok, Result} = couch_mrview:query_view( - Db, <<"_design/bar">>, <<"baz">>, []), + Db, <<"_design/bar">>, <<"baz">>, [] + ), Self ! {self(), Result} end), @@ -112,8 +118,12 @@ check_indexing_stops_on_ddoc_change(Db) -> ?assertEqual(Msg, ddoc_updated) after ?TIMEOUT -> erlang:error( - {assertion_failed, [{module, ?MODULE}, {line, ?LINE}, - {reason, "test failed"}]}) + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, "test failed"} + ]} + ) end, %% assert that previously running indexes are gone @@ -123,11 +133,11 @@ check_indexing_stops_on_ddoc_change(Db) -> ?assertEqual(0, length(AliveAfter)) end). - get_indexes_by_ddoc(DDocID, N) -> Indexes = test_util:wait(fun() -> Indxs = ets:match_object( - couchdb_indexes_by_db, {'$1', {DDocID, '$2'}}), + couchdb_indexes_by_db, {'$1', {DDocID, '$2'}} + ), case length(Indxs) == N of true -> Indxs; @@ -135,11 +145,13 @@ get_indexes_by_ddoc(DDocID, N) -> wait end end), - lists:foldl(fun({DbName, {_DDocID, Sig}}, Acc) -> - case ets:lookup(couchdb_indexes_by_sig, {DbName, Sig}) of - [{_, Pid}] -> [Pid|Acc]; - _ -> Acc - end - end, [], Indexes). - - + lists:foldl( + fun({DbName, {_DDocID, Sig}}, Acc) -> + case ets:lookup(couchdb_indexes_by_sig, {DbName, Sig}) of + [{_, Pid}] -> [Pid | Acc]; + _ -> Acc + end + end, + [], + Indexes + ). diff --git a/src/couch_mrview/test/eunit/couch_mrview_ddoc_validation_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_ddoc_validation_tests.erl index ce2be8904..3e4cbc84f 100644 --- a/src/couch_mrview/test/eunit/couch_mrview_ddoc_validation_tests.erl +++ b/src/couch_mrview/test/eunit/couch_mrview_ddoc_validation_tests.erl @@ -31,10 +31,12 @@ ddoc_validation_test_() -> "ddoc validation tests", { setup, - fun test_util:start_couch/0, fun test_util:stop_couch/1, + fun test_util:start_couch/0, + fun test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_reject_invalid_js_map/1, fun should_reject_invalid_js_reduce/1, @@ -79,344 +81,477 @@ ddoc_validation_test_() -> }. should_reject_invalid_js_map(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_reject_invalid_js_map">>}, - {<<"views">>, {[ - {<<"foo">>, {[ - {<<"map">>, <<"function(doc) }{">>} - ]}} - ]}} - ]}), + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_reject_invalid_js_map">>}, + {<<"views">>, + {[ + {<<"foo">>, + {[ + {<<"map">>, <<"function(doc) }{">>} + ]}} + ]}} + ]} + ), ?_assertThrow( {bad_request, compilation_error, _}, - couch_db:update_doc(Db, Doc, [])). + couch_db:update_doc(Db, Doc, []) + ). should_reject_invalid_js_reduce(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_reject_invalid_js_reduce">>}, - {<<"views">>, {[ - {<<"foo">>, {[ - {<<"map">>, <<"function(doc) { emit(null); }">>}, - {<<"reduce">>, <<"function(k, v, r) }{}">>} - ]}} - ]}} - ]}), + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_reject_invalid_js_reduce">>}, + {<<"views">>, + {[ + {<<"foo">>, + {[ + {<<"map">>, <<"function(doc) { emit(null); }">>}, + {<<"reduce">>, <<"function(k, v, r) }{}">>} + ]}} + ]}} + ]} + ), ?_assertThrow( {bad_request, compilation_error, _}, - couch_db:update_doc(Db, Doc, [])). + couch_db:update_doc(Db, Doc, []) + ). should_reject_invalid_builtin_reduce(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_reject_invalid_builtin_reduce">>}, - {<<"views">>, {[ - {<<"foo">>, {[ - {<<"map">>, <<"function(doc) { emit(null); }">>}, - {<<"reduce">>, <<"_foobar">>} - ]}} - ]}} - ]}), + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_reject_invalid_builtin_reduce">>}, + {<<"views">>, + {[ + {<<"foo">>, + {[ + {<<"map">>, <<"function(doc) { emit(null); }">>}, + {<<"reduce">>, <<"_foobar">>} + ]}} + ]}} + ]} + ), ?_assertThrow( {bad_request, invalid_design_doc, _}, - couch_db:update_doc(Db, Doc, [])). + couch_db:update_doc(Db, Doc, []) + ). should_reject_non_object_options(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_reject_non_object_options">>}, - {<<"options">>, <<"invalid">>} - ]}), - ?_assertThrow({bad_request, invalid_design_doc, _}, - couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_reject_non_object_options">>}, + {<<"options">>, <<"invalid">>} + ]} + ), + ?_assertThrow( + {bad_request, invalid_design_doc, _}, + couch_db:update_doc(Db, Doc, []) + ). should_reject_non_object_filters(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_reject_non_object_filters">>}, - {<<"filters">>, <<"invalid">>} - ]}), - ?_assertThrow({bad_request, invalid_design_doc, _}, - couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_reject_non_object_filters">>}, + {<<"filters">>, <<"invalid">>} + ]} + ), + ?_assertThrow( + {bad_request, invalid_design_doc, _}, + couch_db:update_doc(Db, Doc, []) + ). should_accept_obj_in_filters(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_accept_obj_in_filters">>}, - {<<"filters">>, ?LIB} - ]}), + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_accept_obj_in_filters">>}, + {<<"filters">>, ?LIB} + ]} + ), ?_assertMatch({ok, _}, couch_db:update_doc(Db, Doc, [])). should_reject_non_object_lists(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_reject_non_object_lists">>}, - {<<"lists">>, <<"invalid">>} - ]}), - ?_assertThrow({bad_request, invalid_design_doc, _}, - couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_reject_non_object_lists">>}, + {<<"lists">>, <<"invalid">>} + ]} + ), + ?_assertThrow( + {bad_request, invalid_design_doc, _}, + couch_db:update_doc(Db, Doc, []) + ). should_reject_non_object_shows(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_reject_non_object_shows">>}, - {<<"shows">>, <<"invalid">>} - ]}), - ?_assertThrow({bad_request, invalid_design_doc, _}, - couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_reject_non_object_shows">>}, + {<<"shows">>, <<"invalid">>} + ]} + ), + ?_assertThrow( + {bad_request, invalid_design_doc, _}, + couch_db:update_doc(Db, Doc, []) + ). should_accept_obj_in_shows(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_accept_obj_in_shows">>}, - {<<"shows">>, ?LIB} - ]}), + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_accept_obj_in_shows">>}, + {<<"shows">>, ?LIB} + ]} + ), ?_assertMatch({ok, _}, couch_db:update_doc(Db, Doc, [])). should_reject_non_object_updates(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_reject_non_object_updates">>}, - {<<"updates">>, <<"invalid">>} - ]}), - ?_assertThrow({bad_request, invalid_design_doc, _}, - couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_reject_non_object_updates">>}, + {<<"updates">>, <<"invalid">>} + ]} + ), + ?_assertThrow( + {bad_request, invalid_design_doc, _}, + couch_db:update_doc(Db, Doc, []) + ). should_accept_obj_in_updates(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_accept_obj_in_updates">>}, - {<<"updates">>, ?LIB} - ]}), + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_accept_obj_in_updates">>}, + {<<"updates">>, ?LIB} + ]} + ), ?_assertMatch({ok, _}, couch_db:update_doc(Db, Doc, [])). should_reject_non_object_views(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_reject_non_object_views">>}, - {<<"views">>, <<"invalid">>} - ]}), - ?_assertThrow({bad_request, invalid_design_doc, _}, - couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_reject_non_object_views">>}, + {<<"views">>, <<"invalid">>} + ]} + ), + ?_assertThrow( + {bad_request, invalid_design_doc, _}, + couch_db:update_doc(Db, Doc, []) + ). should_reject_non_string_language(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_reject_non_string_language">>}, - {<<"language">>, 1} - ]}), - ?_assertThrow({bad_request, invalid_design_doc, _}, - couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_reject_non_string_language">>}, + {<<"language">>, 1} + ]} + ), + ?_assertThrow( + {bad_request, invalid_design_doc, _}, + couch_db:update_doc(Db, Doc, []) + ). should_reject_non_string_validate_doc_update(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_reject_non_string_vdu">>}, - {<<"validate_doc_update">>, 1} - ]}), - ?_assertThrow({bad_request, invalid_design_doc, _}, - couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_reject_non_string_vdu">>}, + {<<"validate_doc_update">>, 1} + ]} + ), + ?_assertThrow( + {bad_request, invalid_design_doc, _}, + couch_db:update_doc(Db, Doc, []) + ). should_accept_string_rewrites(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_reject_non_array_rewrites">>}, - {<<"rewrites">>, <<"function(req){}">>} - ]}), - ?_assertMatch({ok,_}, couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_reject_non_array_rewrites">>}, + {<<"rewrites">>, <<"function(req){}">>} + ]} + ), + ?_assertMatch({ok, _}, couch_db:update_doc(Db, Doc, [])). should_reject_bad_rewrites(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_reject_non_array_rewrites">>}, - {<<"rewrites">>, 42} - ]}), - ?_assertThrow({bad_request, invalid_design_doc, _}, - couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_reject_non_array_rewrites">>}, + {<<"rewrites">>, 42} + ]} + ), + ?_assertThrow( + {bad_request, invalid_design_doc, _}, + couch_db:update_doc(Db, Doc, []) + ). should_accept_option(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_accept_options">>}, - {<<"options">>, {[ {<<"option1">>, <<"function(doc,req){}">>} ]}} - ]}), - ?_assertMatch({ok,_}, couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_accept_options">>}, + {<<"options">>, {[{<<"option1">>, <<"function(doc,req){}">>}]}} + ]} + ), + ?_assertMatch({ok, _}, couch_db:update_doc(Db, Doc, [])). should_accept_any_option(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_accept_any_option">>}, - {<<"options">>, {[ {<<"option1">>, true} ]}} - ]}), - ?_assertMatch({ok,_}, couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_accept_any_option">>}, + {<<"options">>, {[{<<"option1">>, true}]}} + ]} + ), + ?_assertMatch({ok, _}, couch_db:update_doc(Db, Doc, [])). should_accept_filter(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_accept_filters">>}, - {<<"filters">>, {[ {<<"filter1">>, <<"function(doc,req){}">>} ]}} - ]}), - ?_assertMatch({ok,_}, couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_accept_filters">>}, + {<<"filters">>, {[{<<"filter1">>, <<"function(doc,req){}">>}]}} + ]} + ), + ?_assertMatch({ok, _}, couch_db:update_doc(Db, Doc, [])). should_reject_non_string_or_obj_filter_function(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_reject_non_string_or_obj_filter_function">>}, - {<<"filters">>, {[ {<<"filter1">>, 1} ]}} - ]}), - ?_assertThrow({bad_request, invalid_design_doc, _}, - couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_reject_non_string_or_obj_filter_function">>}, + {<<"filters">>, {[{<<"filter1">>, 1}]}} + ]} + ), + ?_assertThrow( + {bad_request, invalid_design_doc, _}, + couch_db:update_doc(Db, Doc, []) + ). should_accept_list(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_accept_lists">>}, - {<<"lists">>, {[ {<<"list1">>, <<"function(doc,req){}">>} ]}} - ]}), - ?_assertMatch({ok,_}, couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_accept_lists">>}, + {<<"lists">>, {[{<<"list1">>, <<"function(doc,req){}">>}]}} + ]} + ), + ?_assertMatch({ok, _}, couch_db:update_doc(Db, Doc, [])). should_reject_non_string_or_obj_list_function(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_reject_non_string_or_obj_list_function">>}, - {<<"lists">>, {[ {<<"list1">>, 1} ]}} - ]}), - ?_assertThrow({bad_request, invalid_design_doc, _}, - couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_reject_non_string_or_obj_list_function">>}, + {<<"lists">>, {[{<<"list1">>, 1}]}} + ]} + ), + ?_assertThrow( + {bad_request, invalid_design_doc, _}, + couch_db:update_doc(Db, Doc, []) + ). should_accept_obj_in_lists(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_accept_obj_in_lists">>}, - {<<"lists">>, ?LIB} - ]}), + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_accept_obj_in_lists">>}, + {<<"lists">>, ?LIB} + ]} + ), ?_assertMatch({ok, _}, couch_db:update_doc(Db, Doc, [])). - should_accept_show(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_accept_shows">>}, - {<<"shows">>, {[ {<<"show1">>, <<"function(doc,req){}">>} ]}} - ]}), - ?_assertMatch({ok,_}, couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_accept_shows">>}, + {<<"shows">>, {[{<<"show1">>, <<"function(doc,req){}">>}]}} + ]} + ), + ?_assertMatch({ok, _}, couch_db:update_doc(Db, Doc, [])). should_reject_non_string_or_obj_show_function(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_reject_non_string_or_obj_show_function">>}, - {<<"shows">>, {[ {<<"show1">>, 1} ]}} - ]}), - ?_assertThrow({bad_request, invalid_design_doc, _}, - couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_reject_non_string_or_obj_show_function">>}, + {<<"shows">>, {[{<<"show1">>, 1}]}} + ]} + ), + ?_assertThrow( + {bad_request, invalid_design_doc, _}, + couch_db:update_doc(Db, Doc, []) + ). should_accept_update(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_accept_updates">>}, - {<<"updates">>, {[ {<<"update1">>, <<"function(doc,req){}">>} ]}} - ]}), - ?_assertMatch({ok,_}, couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_accept_updates">>}, + {<<"updates">>, {[{<<"update1">>, <<"function(doc,req){}">>}]}} + ]} + ), + ?_assertMatch({ok, _}, couch_db:update_doc(Db, Doc, [])). should_reject_non_string_or_obj_update_function(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_reject_non_string_or_obj_update_function">>}, - {<<"updates">>, {[ {<<"update1">>, 1} ]}} - ]}), - ?_assertThrow({bad_request, invalid_design_doc, _}, - couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_reject_non_string_or_obj_update_function">>}, + {<<"updates">>, {[{<<"update1">>, 1}]}} + ]} + ), + ?_assertThrow( + {bad_request, invalid_design_doc, _}, + couch_db:update_doc(Db, Doc, []) + ). should_accept_view(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_accept_view">>}, - {<<"views">>, {[ - {<<"view1">>, {[{<<"map">>, <<"function(d){}">>}]}} - ]}} - ]}), - ?_assertMatch({ok,_}, couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_accept_view">>}, + {<<"views">>, + {[ + {<<"view1">>, {[{<<"map">>, <<"function(d){}">>}]}} + ]}} + ]} + ), + ?_assertMatch({ok, _}, couch_db:update_doc(Db, Doc, [])). should_accept_view_with_reduce(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_accept_view_with_reduce">>}, - {<<"views">>, {[ - {<<"view1">>, {[ - {<<"map">>, <<"function(d){}">>}, - {<<"reduce">>,<<"function(d){}">>} - ]}} - ]}} - ]}), - ?_assertMatch({ok,_}, couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_accept_view_with_reduce">>}, + {<<"views">>, + {[ + {<<"view1">>, + {[ + {<<"map">>, <<"function(d){}">>}, + {<<"reduce">>, <<"function(d){}">>} + ]}} + ]}} + ]} + ), + ?_assertMatch({ok, _}, couch_db:update_doc(Db, Doc, [])). should_accept_view_with_lib(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_accept_view_with_lib">>}, - {<<"views">>, {[ - {<<"view1">>, {[ - {<<"map">>, <<"function(d){}">>} - ]}}, - {<<"lib">>, {[ - {<<"lib1">>, <<"x=42">>} - ]}} - ]}} - ]}), - ?_assertMatch({ok,_}, couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_accept_view_with_lib">>}, + {<<"views">>, + {[ + {<<"view1">>, + {[ + {<<"map">>, <<"function(d){}">>} + ]}}, + {<<"lib">>, + {[ + {<<"lib1">>, <<"x=42">>} + ]}} + ]}} + ]} + ), + ?_assertMatch({ok, _}, couch_db:update_doc(Db, Doc, [])). should_reject_view_that_is_not_an_object(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_reject_non_object_view">>}, - {<<"views">>, {[{<<"view1">>, <<"thisisbad">>}]}} - ]}), - ?_assertThrow({bad_request, invalid_design_doc, _}, - couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_reject_non_object_view">>}, + {<<"views">>, {[{<<"view1">>, <<"thisisbad">>}]}} + ]} + ), + ?_assertThrow( + {bad_request, invalid_design_doc, _}, + couch_db:update_doc(Db, Doc, []) + ). should_reject_view_without_map_function(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_accept_view_without_map">>}, - {<<"views">>, {[ - {<<"view1">>, {[]}} - ]}} - ]}), - ?_assertThrow({bad_request, invalid_design_doc, _}, - couch_db:update_doc(Db, Doc, [])). - + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_accept_view_without_map">>}, + {<<"views">>, + {[ + {<<"view1">>, {[]}} + ]}} + ]} + ), + ?_assertThrow( + {bad_request, invalid_design_doc, _}, + couch_db:update_doc(Db, Doc, []) + ). should_reject_view_with_non_string_map_function(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_reject_view_with_nonstr_map">>}, - {<<"views">>, {[ - {<<"view1">>, {[ - {<<"map">>,{[]}} - ]}} - ]}} - ]}), - ?_assertThrow({bad_request, invalid_design_doc, _}, - couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_reject_view_with_nonstr_map">>}, + {<<"views">>, + {[ + {<<"view1">>, + {[ + {<<"map">>, {[]}} + ]}} + ]}} + ]} + ), + ?_assertThrow( + {bad_request, invalid_design_doc, _}, + couch_db:update_doc(Db, Doc, []) + ). should_reject_view_with_non_string_reduce_function(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_reject_view_with_nonstr_reduce">>}, - {<<"views">>, {[ - {<<"view1">>, {[ - {<<"map">>,<<"function(d){}">>}, - {<<"reduce">>,1} - ]}} - ]}} - ]}), - ?_assertThrow({bad_request, invalid_design_doc, _}, - couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_reject_view_with_nonstr_reduce">>}, + {<<"views">>, + {[ + {<<"view1">>, + {[ + {<<"map">>, <<"function(d){}">>}, + {<<"reduce">>, 1} + ]}} + ]}} + ]} + ), + ?_assertThrow( + {bad_request, invalid_design_doc, _}, + couch_db:update_doc(Db, Doc, []) + ). should_accept_any_in_lib(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_accept_any_in_lib">>}, - {<<"views">>, {[ - {<<"view1">>, {[ - {<<"map">>, <<"function(d){}">>} - ]}}, - {<<"lib">>, {[{<<"lib1">>, {[]}}]}} - ]}} - ]}), - ?_assertMatch({ok,_}, couch_db:update_doc(Db, Doc, [])). - + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_accept_any_in_lib">>}, + {<<"views">>, + {[ + {<<"view1">>, + {[ + {<<"map">>, <<"function(d){}">>} + ]}}, + {<<"lib">>, {[{<<"lib1">>, {[]}}]}} + ]}} + ]} + ), + ?_assertMatch({ok, _}, couch_db:update_doc(Db, Doc, [])). should_accept_map_object_for_queries(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_accept_map_objects_for_queries">>}, - {<<"language">>, <<"query">>}, - {<<"views">>, {[ - {<<"view1">>, {[ - {<<"map">>, {[ - {<<"x">>, <<"y">>} + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_accept_map_objects_for_queries">>}, + {<<"language">>, <<"query">>}, + {<<"views">>, + {[ + {<<"view1">>, + {[ + {<<"map">>, + {[ + {<<"x">>, <<"y">>} + ]}} + ]}} ]}} - ]}} - ]}} - ]}), - ?_assertMatch({ok,_}, couch_db:update_doc(Db, Doc, [])). - + ]} + ), + ?_assertMatch({ok, _}, couch_db:update_doc(Db, Doc, [])). should_reject_map_non_objects_for_queries(Db) -> - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/should_reject_map_non_objects__with_nonstr_reduce">>}, - {<<"language">>, <<"query">>}, - {<<"views">>, {[ - {<<"view1">>, {[ - {<<"map">>, <<"function(d){}">>} - ]}} - ]}} - ]}), - ?_assertThrow({bad_request, invalid_design_doc, _}, - couch_db:update_doc(Db, Doc, [])). + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/should_reject_map_non_objects__with_nonstr_reduce">>}, + {<<"language">>, <<"query">>}, + {<<"views">>, + {[ + {<<"view1">>, + {[ + {<<"map">>, <<"function(d){}">>} + ]}} + ]}} + ]} + ), + ?_assertThrow( + {bad_request, invalid_design_doc, _}, + couch_db:update_doc(Db, Doc, []) + ). diff --git a/src/couch_mrview/test/eunit/couch_mrview_design_docs_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_design_docs_tests.erl index aedd42865..b1de2839d 100644 --- a/src/couch_mrview/test/eunit/couch_mrview_design_docs_tests.erl +++ b/src/couch_mrview/test/eunit/couch_mrview_design_docs_tests.erl @@ -17,8 +17,6 @@ -define(TIMEOUT, 1000). - - setup() -> {ok, Db} = couch_mrview_test_util:init_db(?tempdb(), design), Db. @@ -28,16 +26,17 @@ teardown(Db) -> couch_server:delete(couch_db:name(Db), [?ADMIN_CTX]), ok. - design_docs_test_() -> { "_design_docs view tests", { setup, - fun test_util:start_couch/0, fun test_util:stop_couch/1, + fun test_util:start_couch/0, + fun test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_query/1, fun should_query_with_range/1, @@ -49,22 +48,22 @@ design_docs_test_() -> } }. - should_query(Db) -> Result = run_query(Db, []), - Expect = {ok, [ - {meta, [{total, 10}, {offset, 10}]}, - mk_row(<<"_design/bar01">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), - mk_row(<<"_design/bar02">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), - mk_row(<<"_design/bar03">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), - mk_row(<<"_design/bar04">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), - mk_row(<<"_design/bar05">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), - mk_row(<<"_design/bar06">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), - mk_row(<<"_design/bar07">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), - mk_row(<<"_design/bar08">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), - mk_row(<<"_design/bar09">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), - mk_row(<<"_design/bar10">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>) - ]}, + Expect = + {ok, [ + {meta, [{total, 10}, {offset, 10}]}, + mk_row(<<"_design/bar01">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar02">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar03">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar04">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar05">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar06">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar07">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar08">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar09">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar10">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>) + ]}, ?_assertEqual(Expect, Result). should_query_with_range(Db) -> @@ -72,26 +71,29 @@ should_query_with_range(Db) -> {start_key, <<"_design/bar03">>}, {end_key, <<"_design/bar05">>} ]), - Expect = {ok, [ - {meta, [{total, 10}, {offset, 12}]}, - mk_row(<<"_design/bar03">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), - mk_row(<<"_design/bar04">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), - mk_row(<<"_design/bar05">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>) - ]}, + Expect = + {ok, [ + {meta, [{total, 10}, {offset, 12}]}, + mk_row(<<"_design/bar03">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar04">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar05">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>) + ]}, ?_assertEqual(Expect, Result). should_query_with_range_rev(Db) -> Result = run_query(Db, [ {direction, rev}, - {start_key, <<"_design/bar05">>}, {end_key, <<"_design/bar03">>}, + {start_key, <<"_design/bar05">>}, + {end_key, <<"_design/bar03">>}, {inclusive_end, true} ]), - Expect = {ok, [ - {meta, [{total, 10}, {offset, 5}]}, - mk_row(<<"_design/bar05">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), - mk_row(<<"_design/bar04">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), - mk_row(<<"_design/bar03">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>) - ]}, + Expect = + {ok, [ + {meta, [{total, 10}, {offset, 5}]}, + mk_row(<<"_design/bar05">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar04">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar03">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>) + ]}, ?_assertEqual(Expect, Result). should_query_with_limit_and_skip(Db) -> @@ -100,12 +102,13 @@ should_query_with_limit_and_skip(Db) -> {limit, 3}, {skip, 3} ]), - Expect = {ok, [ - {meta, [{total, 10}, {offset, 14}]}, - mk_row(<<"_design/bar05">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), - mk_row(<<"_design/bar06">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), - mk_row(<<"_design/bar07">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>) - ]}, + Expect = + {ok, [ + {meta, [{total, 10}, {offset, 14}]}, + mk_row(<<"_design/bar05">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar06">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar07">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>) + ]}, ?_assertEqual(Expect, Result). should_query_with_include_docs(Db) -> @@ -114,20 +117,25 @@ should_query_with_include_docs(Db) -> {end_key, <<"_design/bar08">>}, {include_docs, true} ]), - Doc = {[ - {<<"_id">>,<<"_design/bar08">>}, - {<<"_rev">>,<<"1-0b24e44a44af45e51e562fd124ce3007">>}, - {<<"views">>,{[]}} - ]}, + Doc = + {[ + {<<"_id">>, <<"_design/bar08">>}, + {<<"_rev">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>}, + {<<"views">>, {[]}} + ]}, Val = {[{rev, <<"1-0b24e44a44af45e51e562fd124ce3007">>}]}, - Expect = {ok, [ - {meta, [{total, 10}, {offset, 17}]}, - {row, [{id, <<"_design/bar08">>}, {key, <<"_design/bar08">>}, - {value, Val}, {doc, Doc}]} - ]}, + Expect = + {ok, [ + {meta, [{total, 10}, {offset, 17}]}, + {row, [ + {id, <<"_design/bar08">>}, + {key, <<"_design/bar08">>}, + {value, Val}, + {doc, Doc} + ]} + ]}, ?_assertEqual(Expect, Result). - mk_row(Id, Rev) -> {row, [{id, Id}, {key, Id}, {value, {[{rev, Rev}]}}]}. diff --git a/src/couch_mrview/test/eunit/couch_mrview_http_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_http_tests.erl index bd11c7ad8..bfa4965a4 100644 --- a/src/couch_mrview/test/eunit/couch_mrview_http_tests.erl +++ b/src/couch_mrview/test/eunit/couch_mrview_http_tests.erl @@ -15,14 +15,23 @@ -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch_mrview/include/couch_mrview.hrl"). - mrview_http_test_() -> [ - ?_assertEqual(#mrargs{group_level=undefined, group=true}, - couch_mrview_http:parse_params([{"group", "true"}], - undefined, #mrargs{})), + ?_assertEqual( + #mrargs{group_level = undefined, group = true}, + couch_mrview_http:parse_params( + [{"group", "true"}], + undefined, + #mrargs{} + ) + ), - ?_assertEqual(#mrargs{group_level=1, group=undefined}, - couch_mrview_http:parse_params([{"group_level", "1"}], - undefined, #mrargs{})) + ?_assertEqual( + #mrargs{group_level = 1, group = undefined}, + couch_mrview_http:parse_params( + [{"group_level", "1"}], + undefined, + #mrargs{} + ) + ) ]. diff --git a/src/couch_mrview/test/eunit/couch_mrview_index_info_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_index_info_tests.erl index c4c765feb..bf64eaea3 100644 --- a/src/couch_mrview/test/eunit/couch_mrview_index_info_tests.erl +++ b/src/couch_mrview/test/eunit/couch_mrview_index_info_tests.erl @@ -17,20 +17,17 @@ -define(TIMEOUT, 1000). - setup() -> {ok, Db} = couch_mrview_test_util:init_db(?tempdb(), map), couch_mrview:query_view(Db, <<"_design/bar">>, <<"baz">>), {ok, Info} = couch_mrview:get_info(Db, <<"_design/bar">>), {Db, Info}. - teardown({Db, _}) -> couch_db:close(Db), couch_server:delete(couch_db:name(Db), [?ADMIN_CTX]), ok. - view_info_test_() -> { "Views index tests", @@ -57,50 +54,41 @@ view_info_test_() -> } }. - sig_is_binary({_, Info}) -> ?_assert(is_binary(prop(signature, Info))). - language_is_js({_, Info}) -> ?_assertEqual(<<"javascript">>, prop(language, Info)). - file_size_is_non_neg_int({_, Info}) -> ?_assert(check_non_neg_int([sizes, file], Info)). - active_size_is_non_neg_int({_, Info}) -> ?_assert(check_non_neg_int([sizes, active], Info)). - external_size_is_non_neg_int({_, Info}) -> ?_assert(check_non_neg_int([sizes, external], Info)). - active_size_less_than_file_size({_, Info}) -> ?_assert(prop([sizes, active], Info) < prop([sizes, file], Info)). - update_seq_is_non_neg_int({_, Info}) -> ?_assert(check_non_neg_int(update_seq, Info)). - purge_seq_is_non_neg_int({_, Info}) -> ?_assert(check_non_neg_int(purge_seq, Info)). - update_opts_is_bin_list({_, Info}) -> Opts = prop(update_options, Info), - ?_assert(is_list(Opts) andalso - (Opts == [] orelse lists:all([is_binary(B) || B <- Opts]))). - + ?_assert( + is_list(Opts) andalso + (Opts == [] orelse lists:all([is_binary(B) || B <- Opts])) + ). check_non_neg_int(Key, Info) -> Size = prop(Key, Info), is_integer(Size) andalso Size >= 0. - prop(Key, {Props}) when is_list(Props) -> prop(Key, Props); prop([Key], Info) -> diff --git a/src/couch_mrview/test/eunit/couch_mrview_local_docs_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_local_docs_tests.erl index b0d25469a..7c812eeb0 100644 --- a/src/couch_mrview/test/eunit/couch_mrview_local_docs_tests.erl +++ b/src/couch_mrview/test/eunit/couch_mrview_local_docs_tests.erl @@ -17,8 +17,6 @@ -define(TIMEOUT, 1000). - - setup() -> {ok, Db} = couch_mrview_test_util:init_db(?tempdb(), local), Db. @@ -28,16 +26,17 @@ teardown(Db) -> couch_server:delete(couch_db:name(Db), [?ADMIN_CTX]), ok. - all_docs_test_() -> { "_local_docs view tests", { setup, - fun test_util:start_couch/0, fun test_util:stop_couch/1, + fun test_util:start_couch/0, + fun test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_query/1, fun should_query_with_range/1, @@ -50,22 +49,22 @@ all_docs_test_() -> } }. - should_query(Db) -> Result = run_query(Db, []), - Expect = {ok, [ - {meta, [{total, null}, {offset, null}]}, - mk_row(1), - mk_row(10), - mk_row(2), - mk_row(3), - mk_row(4), - mk_row(5), - mk_row(6), - mk_row(7), - mk_row(8), - mk_row(9) - ]}, + Expect = + {ok, [ + {meta, [{total, null}, {offset, null}]}, + mk_row(1), + mk_row(10), + mk_row(2), + mk_row(3), + mk_row(4), + mk_row(5), + mk_row(6), + mk_row(7), + mk_row(8), + mk_row(9) + ]}, ?_assertEqual(Expect, Result). should_query_with_range(Db) -> @@ -73,26 +72,29 @@ should_query_with_range(Db) -> {start_key, <<"_local/3">>}, {end_key, <<"_local/5">>} ]), - Expect = {ok, [ - {meta, [{total, null}, {offset, null}]}, - mk_row(3), - mk_row(4), - mk_row(5) - ]}, + Expect = + {ok, [ + {meta, [{total, null}, {offset, null}]}, + mk_row(3), + mk_row(4), + mk_row(5) + ]}, ?_assertEqual(Expect, Result). should_query_with_range_rev(Db) -> Result = run_query(Db, [ {direction, rev}, - {start_key, <<"_local/5">>}, {end_key, <<"_local/3">>}, + {start_key, <<"_local/5">>}, + {end_key, <<"_local/3">>}, {inclusive_end, true} ]), - Expect = {ok, [ - {meta, [{total, null}, {offset, null}]}, - mk_row(5), - mk_row(4), - mk_row(3) - ]}, + Expect = + {ok, [ + {meta, [{total, null}, {offset, null}]}, + mk_row(5), + mk_row(4), + mk_row(3) + ]}, ?_assertEqual(Expect, Result). should_query_with_limit_and_skip(Db) -> @@ -101,12 +103,13 @@ should_query_with_limit_and_skip(Db) -> {limit, 3}, {skip, 3} ]), - Expect = {ok, [ - {meta, [{total, null}, {offset, null}]}, - mk_row(5), - mk_row(6), - mk_row(7) - ]}, + Expect = + {ok, [ + {meta, [{total, null}, {offset, null}]}, + mk_row(5), + mk_row(6), + mk_row(7) + ]}, ?_assertEqual(Expect, Result). should_query_with_include_docs(Db) -> @@ -116,15 +119,21 @@ should_query_with_include_docs(Db) -> {include_docs, true} ]), {row, Doc0} = mk_row(8), - Doc = Doc0 ++ [{doc, {[ - {<<"_id">>, <<"_local/8">>}, - {<<"_rev">>, <<"0-1">>}, - {<<"val">>, 8} - ]}}], - Expect = {ok, [ - {meta, [{total, null}, {offset, null}]}, - {row, Doc} - ]}, + Doc = + Doc0 ++ + [ + {doc, + {[ + {<<"_id">>, <<"_local/8">>}, + {<<"_rev">>, <<"0-1">>}, + {<<"val">>, 8} + ]}} + ], + Expect = + {ok, [ + {meta, [{total, null}, {offset, null}]}, + {row, Doc} + ]}, ?_assertEqual(Expect, Result). should_query_with_update_seq(Db) -> @@ -133,10 +142,11 @@ should_query_with_update_seq(Db) -> {limit, 1}, {update_seq, true} ]), - Expect = {ok, [ - {meta, [{total, null}, {offset, null}, {update_seq, null}]}, - mk_row(2) - ]}, + Expect = + {ok, [ + {meta, [{total, null}, {offset, null}, {update_seq, null}]}, + mk_row(2) + ]}, ?_assertEqual(Expect, Result). mk_row(IntId) -> diff --git a/src/couch_mrview/test/eunit/couch_mrview_map_views_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_map_views_tests.erl index 805dc6c74..0f8357a98 100644 --- a/src/couch_mrview/test/eunit/couch_mrview_map_views_tests.erl +++ b/src/couch_mrview/test/eunit/couch_mrview_map_views_tests.erl @@ -17,7 +17,6 @@ -define(TIMEOUT, 1000). - setup() -> {ok, Db} = couch_mrview_test_util:init_db(?tempdb(), map), Db. @@ -27,16 +26,17 @@ teardown(Db) -> couch_server:delete(couch_db:name(Db), [?ADMIN_CTX]), ok. - map_views_test_() -> { "Map views", { setup, - fun test_util:start_couch/0, fun test_util:stop_couch/1, + fun test_util:start_couch/0, + fun test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_map/1, fun should_map_with_range/1, @@ -49,36 +49,38 @@ map_views_test_() -> } }. - should_map(Db) -> Result = run_query(Db, []), - Expect = {ok, [ - {meta, [{total, 10}, {offset, 0}]}, - {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}, - {row, [{id, <<"6">>}, {key, 6}, {value, 6}]}, - {row, [{id, <<"7">>}, {key, 7}, {value, 7}]}, - {row, [{id, <<"8">>}, {key, 8}, {value, 8}]}, - {row, [{id, <<"9">>}, {key, 9}, {value, 9}]}, - {row, [{id, <<"10">>}, {key, 10}, {value, 10}]} - ]}, + Expect = + {ok, [ + {meta, [{total, 10}, {offset, 0}]}, + {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}, + {row, [{id, <<"6">>}, {key, 6}, {value, 6}]}, + {row, [{id, <<"7">>}, {key, 7}, {value, 7}]}, + {row, [{id, <<"8">>}, {key, 8}, {value, 8}]}, + {row, [{id, <<"9">>}, {key, 9}, {value, 9}]}, + {row, [{id, <<"10">>}, {key, 10}, {value, 10}]} + ]}, ?_assertEqual(Expect, Result). should_map_with_range(Db) -> Result = run_query(Db, [ {direction, rev}, - {start_key, 5}, {end_key, 3}, + {start_key, 5}, + {end_key, 3}, {inclusive_end, true} ]), - Expect = {ok, [ - {meta, [{total, 10}, {offset, 5}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]} - ]}, + Expect = + {ok, [ + {meta, [{total, 10}, {offset, 5}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]} + ]}, ?_assertEqual(Expect, Result). should_map_with_limit_and_skip(Db) -> @@ -87,12 +89,13 @@ should_map_with_limit_and_skip(Db) -> {limit, 3}, {skip, 3} ]), - Expect = {ok, [ - {meta, [{total, 10}, {offset, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}, - {row, [{id, <<"6">>}, {key, 6}, {value, 6}]}, - {row, [{id, <<"7">>}, {key, 7}, {value, 7}]} - ]}, + Expect = + {ok, [ + {meta, [{total, 10}, {offset, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}, + {row, [{id, <<"6">>}, {key, 6}, {value, 6}]}, + {row, [{id, <<"7">>}, {key, 7}, {value, 7}]} + ]}, ?_assertEqual(Expect, Result). should_map_with_include_docs(Db) -> @@ -101,35 +104,41 @@ should_map_with_include_docs(Db) -> {end_key, 8}, {include_docs, true} ]), - Doc = {[ - {<<"_id">>,<<"8">>}, - {<<"_rev">>, <<"1-55b9a29311341e07ec0a7ca13bc1b59f">>}, - {<<"val">>,8} - ]}, - Expect = {ok, [ - {meta, [{total, 10}, {offset, 7}]}, - {row, [{id, <<"8">>}, {key, 8}, {value, 8}, {doc, Doc}]} - ]}, + Doc = + {[ + {<<"_id">>, <<"8">>}, + {<<"_rev">>, <<"1-55b9a29311341e07ec0a7ca13bc1b59f">>}, + {<<"val">>, 8} + ]}, + Expect = + {ok, [ + {meta, [{total, 10}, {offset, 7}]}, + {row, [{id, <<"8">>}, {key, 8}, {value, 8}, {doc, Doc}]} + ]}, ?_assertEqual(Expect, Result). should_map_empty_views(Db) -> Result = couch_mrview:query_view(Db, <<"_design/bar">>, <<"bing">>), - Expect = {ok, [ - {meta, [{total, 0}, {offset, 0}]} - ]}, + Expect = + {ok, [ + {meta, [{total, 0}, {offset, 0}]} + ]}, ?_assertEqual(Expect, Result). should_give_ext_size_seq_indexed_test(Db) -> - DDoc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/seqdoc">>}, - {<<"options">>, {[{<<"seq_indexed">>, true}]}}, - {<<"views">>, {[ - {<<"view1">>, {[ - {<<"map">>, <<"function(doc){emit(doc._id, doc._id);}">>} + DDoc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/seqdoc">>}, + {<<"options">>, {[{<<"seq_indexed">>, true}]}}, + {<<"views">>, + {[ + {<<"view1">>, + {[ + {<<"map">>, <<"function(doc){emit(doc._id, doc._id);}">>} + ]}} ]}} - ]} - } - ]}), + ]} + ), {ok, _} = couch_db:update_doc(Db, DDoc, []), {ok, Db1} = couch_db:open_int(couch_db:name(Db), []), {ok, DDoc1} = couch_db:open_doc(Db1, <<"_design/seqdoc">>, [ejson_body]), @@ -139,6 +148,5 @@ should_give_ext_size_seq_indexed_test(Db) -> ok = couch_db:close(Db1), ?_assert(is_number(Size)). - run_query(Db, Opts) -> couch_mrview:query_view(Db, <<"_design/bar">>, <<"baz">>, Opts). diff --git a/src/couch_mrview/test/eunit/couch_mrview_purge_docs_fabric_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_purge_docs_fabric_tests.erl index b2969bba0..3207a3da3 100644 --- a/src/couch_mrview/test/eunit/couch_mrview_purge_docs_fabric_tests.erl +++ b/src/couch_mrview/test/eunit/couch_mrview_purge_docs_fabric_tests.erl @@ -17,20 +17,18 @@ -include_lib("mem3/include/mem3.hrl"). -include_lib("couch_mrview/include/couch_mrview.hrl"). --define(TIMEOUT, 60). % seconds - +% seconds +-define(TIMEOUT, 60). setup_all() -> Ctx = test_util:start_couch([fabric, mem3]), meck:new(couch_mrview_index, [passthrough]), Ctx. - teardown_all(Ctx) -> meck:unload(), test_util:stop_couch(Ctx). - setup() -> DbName = ?tempdb(), ok = fabric:create_db(DbName, [?ADMIN_CTX, {q, 1}]), @@ -40,11 +38,9 @@ setup() -> end), DbName. - teardown(DbName) -> ok = fabric:delete_db(DbName, [?ADMIN_CTX]). - view_purge_fabric_test_() -> { "Map views", @@ -64,152 +60,185 @@ view_purge_fabric_test_() -> } }. - test_purge_verify_index(DbName) -> - {timeout, ?TIMEOUT, ?_test(begin - Docs1 = couch_mrview_test_util:make_docs(normal, 5), - {ok, _} = fabric:update_docs(DbName, Docs1, [?ADMIN_CTX]), - {ok, _} = fabric:update_doc( - DbName, - couch_mrview_test_util:ddoc(map), - [?ADMIN_CTX] - ), - - Result1 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}), - Expect1 = {ok, [ - {meta, [{total, 5}, {offset, 0}]}, - {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, - ?assertEqual(Expect1, Result1), - - {ok, #doc{body = {Props1}}} = get_local_purge_doc(DbName), - ?assertEqual(0, couch_util:get_value(<<"purge_seq">>, Props1)), - ShardNames = [Sh || #shard{name = Sh} <- mem3:local_shards(DbName)], - [ShardDbName | _Rest ] = ShardNames, - ?assertEqual(true, couch_mrview_index:verify_index_exists( - ShardDbName, Props1)), - - purge_docs(DbName, [<<"1">>]), - - Result2 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}), - Expect2 = {ok, [ - {meta, [{total, 4}, {offset, 0}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, - ?assertEqual(Expect2, Result2), - - {ok, #doc{body = {Props2}}} = get_local_purge_doc(DbName), - ?assertEqual(1, couch_util:get_value(<<"purge_seq">>, Props2)), - ?assertEqual(true, couch_mrview_index:verify_index_exists( - ShardDbName, Props2)) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + Docs1 = couch_mrview_test_util:make_docs(normal, 5), + {ok, _} = fabric:update_docs(DbName, Docs1, [?ADMIN_CTX]), + {ok, _} = fabric:update_doc( + DbName, + couch_mrview_test_util:ddoc(map), + [?ADMIN_CTX] + ), + + Result1 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}), + Expect1 = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect1, Result1), + + {ok, #doc{body = {Props1}}} = get_local_purge_doc(DbName), + ?assertEqual(0, couch_util:get_value(<<"purge_seq">>, Props1)), + ShardNames = [Sh || #shard{name = Sh} <- mem3:local_shards(DbName)], + [ShardDbName | _Rest] = ShardNames, + ?assertEqual( + true, + couch_mrview_index:verify_index_exists( + ShardDbName, Props1 + ) + ), + + purge_docs(DbName, [<<"1">>]), + + Result2 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}), + Expect2 = + {ok, [ + {meta, [{total, 4}, {offset, 0}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect2, Result2), + + {ok, #doc{body = {Props2}}} = get_local_purge_doc(DbName), + ?assertEqual(1, couch_util:get_value(<<"purge_seq">>, Props2)), + ?assertEqual( + true, + couch_mrview_index:verify_index_exists( + ShardDbName, Props2 + ) + ) + end)}. test_purge_hook_before_compaction(DbName) -> - {timeout, ?TIMEOUT, ?_test(begin - Docs1 = couch_mrview_test_util:make_docs(normal, 5), - {ok, _} = fabric:update_docs(DbName, Docs1, [?ADMIN_CTX]), - {ok, _} = fabric:update_doc( - DbName, - couch_mrview_test_util:ddoc(map), - [?ADMIN_CTX] - ), - - Result1 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}), - Expect1 = {ok, [ - {meta, [{total, 5}, {offset, 0}]}, - {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, - ?assertEqual(Expect1, Result1), - - purge_docs(DbName, [<<"1">>]), - - Result2 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}), - Expect2 = {ok, [ - {meta, [{total, 4}, {offset, 0}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, - ?assertEqual(Expect2, Result2), - - {ok, #doc{body = {Props1}}} = get_local_purge_doc(DbName), - ?assertEqual(1, couch_util:get_value(<<"purge_seq">>, Props1)), - - [ShardName | _] = local_shards(DbName), - couch_util:with_db(ShardName, fun(Db) -> - {ok, _} = couch_db:start_compact(Db) - end), - wait_compaction(ShardName, ?LINE), - - ?assertEqual(ok, meck:wait(1, couch_mrview_index, - ensure_local_purge_docs, '_', 5000) - ), - - % Make sure compaction didn't change the update seq - {ok, #doc{body = {Props1}}} = get_local_purge_doc(DbName), - ?assertEqual(1, couch_util:get_value(<<"purge_seq">>, Props1)), - - purge_docs(DbName, [<<"2">>]), - - couch_util:with_db(ShardName, fun(Db) -> - {ok, _} = couch_db:start_compact(Db) - end), - wait_compaction(ShardName, ?LINE), - - ?assertEqual(ok, meck:wait(2, couch_mrview_index, - ensure_local_purge_docs, '_', 5000) - ), - - % Make sure compaction after a purge didn't overwrite - % the local purge doc for the index - {ok, #doc{body = {Props2}}} = get_local_purge_doc(DbName), - ?assertEqual(1, couch_util:get_value(<<"purge_seq">>, Props2)), - - % Force another update to ensure that we update - % the local doc appropriate after compaction - Result3 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}), - Expect3 = {ok, [ - {meta, [{total, 3}, {offset, 0}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, - ?assertEqual(Expect3, Result3), - - {ok, #doc{body = {Props3}}} = get_local_purge_doc(DbName), - ?assertEqual(2, couch_util:get_value(<<"purge_seq">>, Props3)), - - % Check that if the local doc doesn't exist that one - % is created for the index on compaction - delete_local_purge_doc(DbName), - ?assertMatch({not_found, _}, get_local_purge_doc(DbName)), - - couch_util:with_db(ShardName, fun(Db) -> - {ok, _} = couch_db:start_compact(Db) - end), - wait_compaction(ShardName, ?LINE), - - ?assertEqual(ok, meck:wait(3, couch_mrview_index, - ensure_local_purge_docs, '_', 5000) - ), - - {ok, #doc{body = {Props4}}} = get_local_purge_doc(DbName), - ?assertEqual(2, couch_util:get_value(<<"purge_seq">>, Props4)) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + Docs1 = couch_mrview_test_util:make_docs(normal, 5), + {ok, _} = fabric:update_docs(DbName, Docs1, [?ADMIN_CTX]), + {ok, _} = fabric:update_doc( + DbName, + couch_mrview_test_util:ddoc(map), + [?ADMIN_CTX] + ), + + Result1 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}), + Expect1 = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect1, Result1), + + purge_docs(DbName, [<<"1">>]), + + Result2 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}), + Expect2 = + {ok, [ + {meta, [{total, 4}, {offset, 0}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect2, Result2), + + {ok, #doc{body = {Props1}}} = get_local_purge_doc(DbName), + ?assertEqual(1, couch_util:get_value(<<"purge_seq">>, Props1)), + + [ShardName | _] = local_shards(DbName), + couch_util:with_db(ShardName, fun(Db) -> + {ok, _} = couch_db:start_compact(Db) + end), + wait_compaction(ShardName, ?LINE), + + ?assertEqual( + ok, + meck:wait( + 1, + couch_mrview_index, + ensure_local_purge_docs, + '_', + 5000 + ) + ), + + % Make sure compaction didn't change the update seq + {ok, #doc{body = {Props1}}} = get_local_purge_doc(DbName), + ?assertEqual(1, couch_util:get_value(<<"purge_seq">>, Props1)), + + purge_docs(DbName, [<<"2">>]), + + couch_util:with_db(ShardName, fun(Db) -> + {ok, _} = couch_db:start_compact(Db) + end), + wait_compaction(ShardName, ?LINE), + + ?assertEqual( + ok, + meck:wait( + 2, + couch_mrview_index, + ensure_local_purge_docs, + '_', + 5000 + ) + ), + + % Make sure compaction after a purge didn't overwrite + % the local purge doc for the index + {ok, #doc{body = {Props2}}} = get_local_purge_doc(DbName), + ?assertEqual(1, couch_util:get_value(<<"purge_seq">>, Props2)), + + % Force another update to ensure that we update + % the local doc appropriate after compaction + Result3 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}), + Expect3 = + {ok, [ + {meta, [{total, 3}, {offset, 0}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect3, Result3), + + {ok, #doc{body = {Props3}}} = get_local_purge_doc(DbName), + ?assertEqual(2, couch_util:get_value(<<"purge_seq">>, Props3)), + + % Check that if the local doc doesn't exist that one + % is created for the index on compaction + delete_local_purge_doc(DbName), + ?assertMatch({not_found, _}, get_local_purge_doc(DbName)), + + couch_util:with_db(ShardName, fun(Db) -> + {ok, _} = couch_db:start_compact(Db) + end), + wait_compaction(ShardName, ?LINE), + + ?assertEqual( + ok, + meck:wait( + 3, + couch_mrview_index, + ensure_local_purge_docs, + '_', + 5000 + ) + ), + + {ok, #doc{body = {Props4}}} = get_local_purge_doc(DbName), + ?assertEqual(2, couch_util:get_value(<<"purge_seq">>, Props4)) + end)}. get_local_purge_doc(DbName) -> {ok, DDoc} = fabric:open_doc(DbName, <<"_design/bar">>, []), @@ -222,7 +251,6 @@ get_local_purge_doc(DbName) -> couch_db:open_doc(Db, DocId, []) end). - delete_local_purge_doc(DbName) -> {ok, DDoc} = fabric:open_doc(DbName, <<"_design/bar">>, []), {ok, IdxState} = couch_mrview_util:ddoc_to_mrst(DbName, DDoc), @@ -235,21 +263,21 @@ delete_local_purge_doc(DbName) -> {ok, _} = couch_db:update_doc(Db, NewDoc, []) end). - get_rev(#full_doc_info{} = FDI) -> #doc_info{ revs = [#rev_info{} = PrevRev | _] } = couch_doc:to_doc_info(FDI), PrevRev#rev_info.rev. - purge_docs(DbName, DocIds) -> - lists:foreach(fun(DocId) -> - FDI = fabric:get_full_doc_info(DbName, DocId, []), - Rev = get_rev(FDI), - {ok, [{ok, _}]} = fabric:purge_docs(DbName, [{DocId, [Rev]}], []) - end, DocIds). - + lists:foreach( + fun(DocId) -> + FDI = fabric:get_full_doc_info(DbName, DocId, []), + Rev = get_rev(FDI), + {ok, [{ok, _}]} = fabric:purge_docs(DbName, [{DocId, [Rev]}], []) + end, + DocIds + ). wait_compaction(DbName, Line) -> WaitFun = fun() -> @@ -260,23 +288,23 @@ wait_compaction(DbName, Line) -> end, case test_util:wait(WaitFun, 10000) of timeout -> - erlang:error({assertion_failed, [ + erlang:error( + {assertion_failed, [ {module, ?MODULE}, {line, Line}, {reason, "Timeout waiting for database compaction"} - ]}); + ]} + ); _ -> ok end. - is_compaction_running(DbName) -> {ok, DbInfo} = couch_util:with_db(DbName, fun(Db) -> couch_db:get_db_info(Db) end), couch_util:get_value(compact_running, DbInfo). - local_shards(DbName) -> try [ShardName || #shard{name = ShardName} <- mem3:local_shards(DbName)] diff --git a/src/couch_mrview/test/eunit/couch_mrview_purge_docs_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_purge_docs_tests.erl index 62e1410cb..63c5de458 100644 --- a/src/couch_mrview/test/eunit/couch_mrview_purge_docs_tests.erl +++ b/src/couch_mrview/test/eunit/couch_mrview_purge_docs_tests.erl @@ -18,7 +18,6 @@ -define(TIMEOUT, 1000). - setup() -> meck:new(couch_index_updater, [passthrough]), {ok, Db} = couch_mrview_test_util:init_db(?tempdb(), map, 5), @@ -57,51 +56,52 @@ view_purge_test_() -> } }. - test_purge_single(Db) -> ?_test(begin Result = run_query(Db, []), - Expect = {ok, [ - {meta, [{total, 5}, {offset, 0}]}, - {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, + Expect = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, ?assertEqual(Expect, Result), FDI = couch_db:get_full_doc_info(Db, <<"1">>), Rev = get_rev(FDI), {ok, [{ok, _PRevs}]} = couch_db:purge_docs( - Db, - [{<<"UUID1">>, <<"1">>, [Rev]}] - ), + Db, + [{<<"UUID1">>, <<"1">>, [Rev]}] + ), {ok, Db2} = couch_db:reopen(Db), Result2 = run_query(Db2, []), - Expect2 = {ok, [ - {meta, [{total, 4}, {offset, 0}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, + Expect2 = + {ok, [ + {meta, [{total, 4}, {offset, 0}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, ?assertEqual(Expect2, Result2) end). - test_purge_single_for_docid_with_list(Db) -> ?_test(begin Result = run_query(Db, []), - Expect = {ok, [ - {meta, [{total, 5}, {offset, 0}]}, - {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, + Expect = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, ?assertEqual(Expect, Result), FDI = couch_db:get_full_doc_info(Db, <<"1">>), @@ -113,35 +113,39 @@ test_purge_single_for_docid_with_list(Db) -> {ok, Db2} = couch_db:reopen(Db), Result2 = run_query(Db2, []), - Expect2 = {ok, [ - {meta, [{total, 4}, {offset, 0}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, + Expect2 = + {ok, [ + {meta, [{total, 4}, {offset, 0}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, ?assertEqual(Expect2, Result2) end). test_purge_partial(Db) -> ?_test(begin Result = run_query(Db, []), - Expect = {ok, [ - {meta, [{total, 5}, {offset, 0}]}, - {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, + Expect = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, ?assertEqual(Expect, Result), - FDI1 = couch_db:get_full_doc_info(Db, <<"1">>), Rev1 = get_rev(FDI1), - Update = {[ - {'_id', <<"1">>}, - {'_rev', couch_doc:rev_to_str({1, [couch_hash:md5_hash(<<"1.2">>)]})}, - {'val', 1.2} - ]}, + FDI1 = couch_db:get_full_doc_info(Db, <<"1">>), + Rev1 = get_rev(FDI1), + Update = + {[ + {'_id', <<"1">>}, + {'_rev', couch_doc:rev_to_str({1, [couch_hash:md5_hash(<<"1.2">>)]})}, + {'val', 1.2} + ]}, {ok, [_Rev2]} = save_docs(Db, [Update], [replicated_changes]), PurgeInfos = [{<<"UUID1">>, <<"1">>, [Rev1]}], @@ -150,34 +154,38 @@ test_purge_partial(Db) -> {ok, Db2} = couch_db:reopen(Db), Result2 = run_query(Db2, []), - Expect2 = {ok, [ - {meta, [{total, 5}, {offset, 0}]}, - {row, [{id, <<"1">>}, {key, 1.2}, {value, 1.2}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, + Expect2 = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, <<"1">>}, {key, 1.2}, {value, 1.2}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, ?assertEqual(Expect2, Result2) end). - test_purge_complete(Db) -> ?_test(begin Result = run_query(Db, []), - Expect = {ok, [ - {meta, [{total, 5}, {offset, 0}]}, - {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, + Expect = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, ?assertEqual(Expect, Result), - FDI1 = couch_db:get_full_doc_info(Db, <<"1">>), Rev1 = get_rev(FDI1), - FDI2 = couch_db:get_full_doc_info(Db, <<"2">>), Rev2 = get_rev(FDI2), - FDI5 = couch_db:get_full_doc_info(Db, <<"5">>), Rev5 = get_rev(FDI5), + FDI1 = couch_db:get_full_doc_info(Db, <<"1">>), + Rev1 = get_rev(FDI1), + FDI2 = couch_db:get_full_doc_info(Db, <<"2">>), + Rev2 = get_rev(FDI2), + FDI5 = couch_db:get_full_doc_info(Db, <<"5">>), + Rev5 = get_rev(FDI5), PurgeInfos = [ {<<"UUID1">>, <<"1">>, [Rev1]}, @@ -188,31 +196,35 @@ test_purge_complete(Db) -> {ok, Db2} = couch_db:reopen(Db), Result2 = run_query(Db2, []), - Expect2 = {ok, [ - {meta, [{total, 2}, {offset, 0}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]} - ]}, + Expect2 = + {ok, [ + {meta, [{total, 2}, {offset, 0}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]} + ]}, ?assertEqual(Expect2, Result2) end). - test_purge_complete_for_docid_with_list(Db) -> ?_test(begin Result = run_query(Db, []), - Expect = {ok, [ - {meta, [{total, 5}, {offset, 0}]}, - {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, + Expect = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, ?assertEqual(Expect, Result), - FDI1 = couch_db:get_full_doc_info(Db, <<"1">>), Rev1 = get_rev(FDI1), - FDI2 = couch_db:get_full_doc_info(Db, <<"2">>), Rev2 = get_rev(FDI2), - FDI5 = couch_db:get_full_doc_info(Db, <<"5">>), Rev5 = get_rev(FDI5), + FDI1 = couch_db:get_full_doc_info(Db, <<"1">>), + Rev1 = get_rev(FDI1), + FDI2 = couch_db:get_full_doc_info(Db, <<"2">>), + Rev2 = get_rev(FDI2), + FDI5 = couch_db:get_full_doc_info(Db, <<"5">>), + Rev5 = get_rev(FDI5), PurgeInfos = [ {<<"UUID1">>, "1", [Rev1]}, @@ -223,26 +235,27 @@ test_purge_complete_for_docid_with_list(Db) -> {ok, Db2} = couch_db:reopen(Db), Result2 = run_query(Db2, []), - Expect2 = {ok, [ - {meta, [{total, 2}, {offset, 0}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]} - ]}, + Expect2 = + {ok, [ + {meta, [{total, 2}, {offset, 0}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]} + ]}, ?assertEqual(Expect2, Result2) end). - test_purge_nochange(Db) -> ?_test(begin Result = run_query(Db, []), - Expect = {ok, [ - {meta, [{total, 5}, {offset, 0}]}, - {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, + Expect = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, ?assertEqual(Expect, Result), FDI1 = couch_db:get_full_doc_info(Db, <<"1">>), @@ -255,40 +268,44 @@ test_purge_nochange(Db) -> {ok, Db2} = couch_db:reopen(Db), Result2 = run_query(Db2, []), - Expect2 = {ok, [ - {meta, [{total, 5}, {offset, 0}]}, - {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, + Expect2 = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, ?assertEqual(Expect2, Result2) end). - test_purge_index_reset(Db) -> ?_test(begin ok = couch_db:set_purge_infos_limit(Db, 2), {ok, Db1} = couch_db:reopen(Db), Result = run_query(Db1, []), - Expect = {ok, [ - {meta, [{total, 5}, {offset, 0}]}, - {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, + Expect = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, ?assertEqual(Expect, Result), - PurgeInfos = lists:map(fun(I) -> - DocId = list_to_binary(integer_to_list(I)), - FDI = couch_db:get_full_doc_info(Db, DocId), - Rev = get_rev(FDI), - {couch_uuids:random(), DocId, [Rev]} - end, lists:seq(1, 5)), + PurgeInfos = lists:map( + fun(I) -> + DocId = list_to_binary(integer_to_list(I)), + FDI = couch_db:get_full_doc_info(Db, DocId), + Rev = get_rev(FDI), + {couch_uuids:random(), DocId, [Rev]} + end, + lists:seq(1, 5) + ), {ok, _} = couch_db:purge_docs(Db1, PurgeInfos), {ok, Db2} = couch_db:reopen(Db1), @@ -309,22 +326,22 @@ test_purge_index_reset(Db) -> {ok, Db3} = couch_db:reopen(Db2), Result2 = run_query(Db3, []), - Expect2 = {ok, [ - {meta, [{total, 0}, {offset, 0}]} - ]}, + Expect2 = + {ok, [ + {meta, [{total, 0}, {offset, 0}]} + ]}, ?assertEqual(Expect2, Result2), % Assert that we had a reset meck:wait( - 1, - couch_index_updater, - handle_info, - [{'EXIT', '_', {reset, '_'}}, '_'], - 5000 - ) + 1, + couch_index_updater, + handle_info, + [{'EXIT', '_', {reset, '_'}}, '_'], + 5000 + ) end). - test_purge_compact_size_check(Db) -> ?_test(begin DbName = couch_db:name(Db), @@ -334,24 +351,28 @@ test_purge_compact_size_check(Db) -> DiskSizeBefore = db_disk_size(DbName), PurgedDocsNum = 150, - IdsRevs = lists:foldl(fun(Id, CIdRevs) -> - Id1 = docid(Id), - FDI1 = couch_db:get_full_doc_info(Db1, Id1), - Rev1 = get_rev(FDI1), - UUID1 = uuid(Id), - [{UUID1, Id1, [Rev1]} | CIdRevs] - end, [], lists:seq(1, PurgedDocsNum)), + IdsRevs = lists:foldl( + fun(Id, CIdRevs) -> + Id1 = docid(Id), + FDI1 = couch_db:get_full_doc_info(Db1, Id1), + Rev1 = get_rev(FDI1), + UUID1 = uuid(Id), + [{UUID1, Id1, [Rev1]} | CIdRevs] + end, + [], + lists:seq(1, PurgedDocsNum) + ), {ok, _} = couch_db:purge_docs(Db1, IdsRevs), {ok, Db2} = couch_db:reopen(Db1), _Result1 = run_query(Db2, []), {ok, PurgedIdRevs} = couch_db:fold_purge_infos( - Db2, - 0, - fun fold_fun/2, - [], - [] - ), + Db2, + 0, + fun fold_fun/2, + [], + [] + ), ?assertEqual(PurgedDocsNum, length(PurgedIdRevs)), config:set("couchdb", "file_compression", "snappy", false), @@ -363,7 +384,6 @@ test_purge_compact_size_check(Db) -> ?assert(DiskSizeBefore > DiskSizeAfter) end). - test_purge_compact_for_stale_purge_cp_without_client(Db) -> ?_test(begin DbName = couch_db:name(Db), @@ -378,23 +398,27 @@ test_purge_compact_for_stale_purge_cp_without_client(Db) -> % purge 150 documents PurgedDocsNum = 150, - PurgeInfos = lists:foldl(fun(Id, CIdRevs) -> - Id1 = docid(Id), - FDI1 = couch_db:get_full_doc_info(Db1, Id1), - Rev1 = get_rev(FDI1), - UUID1 = uuid(Id), - [{UUID1, Id1, [Rev1]} | CIdRevs] - end, [], lists:seq(1, PurgedDocsNum)), + PurgeInfos = lists:foldl( + fun(Id, CIdRevs) -> + Id1 = docid(Id), + FDI1 = couch_db:get_full_doc_info(Db1, Id1), + Rev1 = get_rev(FDI1), + UUID1 = uuid(Id), + [{UUID1, Id1, [Rev1]} | CIdRevs] + end, + [], + lists:seq(1, PurgedDocsNum) + ), {ok, _} = couch_db:purge_docs(Db1, PurgeInfos), {ok, Db2} = couch_db:reopen(Db1), {ok, PurgedIdRevs} = couch_db:fold_purge_infos( - Db2, - 0, - fun fold_fun/2, - [], - [] - ), + Db2, + 0, + fun fold_fun/2, + [], + [] + ), ?assertEqual(PurgedDocsNum, length(PurgedIdRevs)), % run compaction to trigger pruning of purge tree @@ -407,16 +431,15 @@ test_purge_compact_for_stale_purge_cp_without_client(Db) -> {ok, Db4} = couch_db:reopen(Db3), OldestPSeq = couch_db:get_oldest_purge_seq(Db4), {ok, PurgedIdRevs2} = couch_db:fold_purge_infos( - Db4, - OldestPSeq - 1, - fun fold_fun/2, - [], - [] - ), + Db4, + OldestPSeq - 1, + fun fold_fun/2, + [], + [] + ), ?assertEqual(PurgedDocsLimit, length(PurgedIdRevs2)) end). - test_purge_compact_for_stale_purge_cp_with_client(Db) -> ?_test(begin DbName = couch_db:name(Db), @@ -432,36 +455,44 @@ test_purge_compact_for_stale_purge_cp_with_client(Db) -> % first purge 30 documents PurgedDocsNum1 = 30, - IdsRevs = lists:foldl(fun(Id, CIdRevs) -> - Id1 = docid(Id), - FDI1 = couch_db:get_full_doc_info(Db1, Id1), - Rev1 = get_rev(FDI1), - UUID1 = uuid(Id), - [{UUID1, Id1, [Rev1]} | CIdRevs] - end, [], lists:seq(1, PurgedDocsNum1)), + IdsRevs = lists:foldl( + fun(Id, CIdRevs) -> + Id1 = docid(Id), + FDI1 = couch_db:get_full_doc_info(Db1, Id1), + Rev1 = get_rev(FDI1), + UUID1 = uuid(Id), + [{UUID1, Id1, [Rev1]} | CIdRevs] + end, + [], + lists:seq(1, PurgedDocsNum1) + ), {ok, _} = couch_db:purge_docs(Db1, IdsRevs), {ok, Db2} = couch_db:reopen(Db1), % run query again to reflect purge request to mrview _Result1 = run_query(Db2, []), {ok, PurgedIdRevs} = couch_db:fold_purge_infos( - Db2, - 0, - fun fold_fun/2, - [], - [] - ), + Db2, + 0, + fun fold_fun/2, + [], + [] + ), ?assertEqual(PurgedDocsNum1, length(PurgedIdRevs)), % then purge 120 documents PurgedDocsNum2 = 150, - IdsRevs2 = lists:foldl(fun(Id, CIdRevs) -> - Id1 = docid(Id), - FDI1 = couch_db:get_full_doc_info(Db1, Id1), - Rev1 = get_rev(FDI1), - UUID1 = uuid(Id), - [{UUID1, Id1, [Rev1]} | CIdRevs] - end, [], lists:seq(PurgedDocsNum1 + 1, PurgedDocsNum2)), + IdsRevs2 = lists:foldl( + fun(Id, CIdRevs) -> + Id1 = docid(Id), + FDI1 = couch_db:get_full_doc_info(Db1, Id1), + Rev1 = get_rev(FDI1), + UUID1 = uuid(Id), + [{UUID1, Id1, [Rev1]} | CIdRevs] + end, + [], + lists:seq(PurgedDocsNum1 + 1, PurgedDocsNum2) + ), {ok, _} = couch_db:purge_docs(Db2, IdsRevs2), % run compaction to trigger pruning of purge tree @@ -475,16 +506,15 @@ test_purge_compact_for_stale_purge_cp_with_client(Db) -> {ok, Db4} = couch_db:reopen(Db3), OldestPSeq = couch_db:get_oldest_purge_seq(Db4), {ok, PurgedIdRevs2} = couch_db:fold_purge_infos( - Db4, - OldestPSeq - 1, - fun fold_fun/2, - [], - [] - ), + Db4, + OldestPSeq - 1, + fun fold_fun/2, + [], + [] + ), ?assertEqual(PurgedDocsNum2 - PurgedDocsNum1, length(PurgedIdRevs2)) end). - get_local_purge_doc(Db) -> {ok, DDoc} = couch_db:open_doc(Db, <<"_design/bar">>, []), {ok, IdxState} = couch_mrview_util:ddoc_to_mrst(couch_db:name(Db), DDoc), @@ -493,48 +523,50 @@ get_local_purge_doc(Db) -> DocId = couch_mrview_util:get_local_purge_doc_id(HexSig), couch_db:open_doc(Db, DocId, []). - run_query(Db, Opts) -> couch_mrview:query_view(Db, <<"_design/bar">>, <<"baz">>, Opts). - save_docs(Db, JsonDocs, Options) -> - Docs = lists:map(fun(JDoc) -> - couch_doc:from_json_obj(?JSON_DECODE(?JSON_ENCODE(JDoc))) - end, JsonDocs), + Docs = lists:map( + fun(JDoc) -> + couch_doc:from_json_obj(?JSON_DECODE(?JSON_ENCODE(JDoc))) + end, + JsonDocs + ), Opts = [full_commit | Options], case lists:member(replicated_changes, Options) of true -> {ok, []} = couch_db:update_docs( - Db, Docs, Opts, replicated_changes), - {ok, lists:map(fun(Doc) -> - {Pos, [RevId | _]} = Doc#doc.revs, - {Pos, RevId} - end, Docs)}; + Db, Docs, Opts, replicated_changes + ), + {ok, + lists:map( + fun(Doc) -> + {Pos, [RevId | _]} = Doc#doc.revs, + {Pos, RevId} + end, + Docs + )}; false -> {ok, Resp} = couch_db:update_docs(Db, Docs, Opts), {ok, [Rev || {ok, Rev} <- Resp]} end. - get_rev(#full_doc_info{} = FDI) -> #doc_info{ revs = [#rev_info{} = PrevRev | _] } = couch_doc:to_doc_info(FDI), PrevRev#rev_info.rev. - db_disk_size(DbName) -> {ok, Db} = couch_db:open_int(DbName, []), {ok, Info} = couch_db:get_db_info(Db), ok = couch_db:close(Db), active_size(Info). - active_size(Info) -> couch_util:get_nested_json_value({Info}, [sizes, active]). - wait_compaction(DbName, Kind, Line) -> WaitFun = fun() -> case is_compaction_running(DbName) of @@ -544,32 +576,32 @@ wait_compaction(DbName, Kind, Line) -> end, case test_util:wait(WaitFun, 10000) of timeout -> - erlang:error({assertion_failed, - [{module, ?MODULE}, + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, {line, Line}, - {reason, "Timeout waiting for " - ++ Kind - ++ " database compaction"}]}); + {reason, + "Timeout waiting for " ++ + Kind ++ + " database compaction"} + ]} + ); _ -> ok end. - is_compaction_running(DbName) -> {ok, Db} = couch_db:open_int(DbName, []), {ok, DbInfo} = couch_db:get_db_info(Db), couch_db:close(Db), couch_util:get_value(compact_running, DbInfo). - fold_fun({_PSeq, _UUID, Id, Revs}, Acc) -> {ok, [{Id, Revs} | Acc]}. - docid(I) -> list_to_binary(integer_to_list(I)). - uuid(I) -> Str = io_lib:format("UUID~4..0b", [I]), iolist_to_binary(Str). diff --git a/src/couch_mrview/test/eunit/couch_mrview_red_views_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_red_views_tests.erl index b83686113..b6042b6c7 100644 --- a/src/couch_mrview/test/eunit/couch_mrview_red_views_tests.erl +++ b/src/couch_mrview/test/eunit/couch_mrview_red_views_tests.erl @@ -17,7 +17,6 @@ -define(TIMEOUT, 1000). - setup() -> {ok, Db} = couch_mrview_test_util:init_db(?tempdb(), red), Db. @@ -27,16 +26,17 @@ teardown(Db) -> couch_server:delete(couch_db:name(Db), [?ADMIN_CTX]), ok. - reduce_views_test_() -> { "Reduce views", { setup, - fun test_util:start_couch/0, fun test_util:stop_couch/1, + fun test_util:start_couch/0, + fun test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_reduce_basic/1, fun should_reduce_key_range/1, @@ -47,49 +47,51 @@ reduce_views_test_() -> } }. - should_reduce_basic(Db) -> Result = run_query(Db, []), - Expect = {ok, [ - {meta, []}, - {row, [{key, null}, {value, 55}]} - ]}, + Expect = + {ok, [ + {meta, []}, + {row, [{key, null}, {value, 55}]} + ]}, ?_assertEqual(Expect, Result). should_reduce_key_range(Db) -> Result = run_query(Db, [{start_key, [0, 2]}, {end_key, [0, 4]}]), - Expect = {ok, [ - {meta, []}, - {row, [{key, null}, {value, 6}]} - ]}, + Expect = + {ok, [ + {meta, []}, + {row, [{key, null}, {value, 6}]} + ]}, ?_assertEqual(Expect, Result). should_reduce_with_group_level(Db) -> Result = run_query(Db, [{group_level, 1}]), - Expect = {ok, [ - {meta, []}, - {row, [{key, [0]}, {value, 30}]}, - {row, [{key, [1]}, {value, 25}]} - ]}, + Expect = + {ok, [ + {meta, []}, + {row, [{key, [0]}, {value, 30}]}, + {row, [{key, [1]}, {value, 25}]} + ]}, ?_assertEqual(Expect, Result). should_reduce_with_group_exact(Db) -> Result = run_query(Db, [{group_level, exact}]), - Expect = {ok, [ - {meta, []}, - {row, [{key, [0, 2]}, {value, 2}]}, - {row, [{key, [0, 4]}, {value, 4}]}, - {row, [{key, [0, 6]}, {value, 6}]}, - {row, [{key, [0, 8]}, {value, 8}]}, - {row, [{key, [0, 10]}, {value, 10}]}, - {row, [{key, [1, 1]}, {value, 1}]}, - {row, [{key, [1, 3]}, {value, 3}]}, - {row, [{key, [1, 5]}, {value, 5}]}, - {row, [{key, [1, 7]}, {value, 7}]}, - {row, [{key, [1, 9]}, {value, 9}]} - ]}, + Expect = + {ok, [ + {meta, []}, + {row, [{key, [0, 2]}, {value, 2}]}, + {row, [{key, [0, 4]}, {value, 4}]}, + {row, [{key, [0, 6]}, {value, 6}]}, + {row, [{key, [0, 8]}, {value, 8}]}, + {row, [{key, [0, 10]}, {value, 10}]}, + {row, [{key, [1, 1]}, {value, 1}]}, + {row, [{key, [1, 3]}, {value, 3}]}, + {row, [{key, [1, 5]}, {value, 5}]}, + {row, [{key, [1, 7]}, {value, 7}]}, + {row, [{key, [1, 9]}, {value, 9}]} + ]}, ?_assertEqual(Expect, Result). - run_query(Db, Opts) -> couch_mrview:query_view(Db, <<"_design/red">>, <<"baz">>, Opts). diff --git a/src/couch_mrview/test/eunit/couch_mrview_util_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_util_tests.erl index 7046c9bb2..a495fd82c 100644 --- a/src/couch_mrview/test/eunit/couch_mrview_util_tests.erl +++ b/src/couch_mrview/test/eunit/couch_mrview_util_tests.erl @@ -15,25 +15,23 @@ -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch_mrview/include/couch_mrview.hrl"). - - couch_mrview_util_test_() -> [ - ?_assertEqual(0, validate_group_level(undefined, undefined)), - ?_assertEqual(exact, validate_group_level(true, undefined)), - ?_assertEqual(0, validate_group_level(false, undefined)), - ?_assertEqual(1, validate_group_level(undefined, 1)), - ?_assertEqual(0, validate_group_level(true, 0)), - ?_assertEqual(0, validate_group_level(undefined, 0)), - ?_assertEqual(1, validate_group_level(true, 1)), - ?_assertEqual(0, validate_group_level(false, 0)), - ?_assertThrow({query_parse_error, - <<"Can't specify group=false and group_level>0 at the same time">>}, - validate_group_level(false,1)) + ?_assertEqual(0, validate_group_level(undefined, undefined)), + ?_assertEqual(exact, validate_group_level(true, undefined)), + ?_assertEqual(0, validate_group_level(false, undefined)), + ?_assertEqual(1, validate_group_level(undefined, 1)), + ?_assertEqual(0, validate_group_level(true, 0)), + ?_assertEqual(0, validate_group_level(undefined, 0)), + ?_assertEqual(1, validate_group_level(true, 1)), + ?_assertEqual(0, validate_group_level(false, 0)), + ?_assertThrow( + {query_parse_error, <<"Can't specify group=false and group_level>0 at the same time">>}, + validate_group_level(false, 1) + ) ]. validate_group_level(Group, GroupLevel) -> - Args0 = #mrargs{group=Group, group_level=GroupLevel, view_type=red}, + Args0 = #mrargs{group = Group, group_level = GroupLevel, view_type = red}, Args1 = couch_mrview_util:validate_args(Args0), Args1#mrargs.group_level. - diff --git a/src/couch_peruser/src/couch_peruser.erl b/src/couch_peruser/src/couch_peruser.erl index 886fb4f6e..c87ffb2fb 100644 --- a/src/couch_peruser/src/couch_peruser.erl +++ b/src/couch_peruser/src/couch_peruser.erl @@ -18,8 +18,15 @@ -include_lib("mem3/include/mem3.hrl"). % gen_server callbacks --export([start_link/0, init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). +-export([ + start_link/0, + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3 +]). -export([init_changes_handler/1, changes_handler/3]). @@ -52,8 +59,10 @@ -define(DEFAULT_USERDB_PREFIX, "userdb-"). -define(RELISTEN_DELAY, 5000). --define(DEFAULT_QUIET_PERIOD, 60). % seconds --define(DEFAULT_START_PERIOD, 5). % seconds +% seconds +-define(DEFAULT_QUIET_PERIOD, 60). +% seconds +-define(DEFAULT_START_PERIOD, 5). %% %% Please leave in the commented-out couch_log:debug calls, thanks! — Jan @@ -66,267 +75,346 @@ start_link() -> init_state() -> couch_log:debug("peruser: starting on node ~p in pid ~p", [node(), self()]), case config:get_boolean("couch_peruser", "enable", false) of - false -> - couch_log:debug("peruser: disabled on node ~p", [node()]), - #state{}; - true -> - couch_log:debug("peruser: enabled on node ~p", [node()]), - DbName = ?l2b(config:get( - "couch_httpd_auth", "authentication_db", "_users")), - DeleteDbs = config:get_boolean("couch_peruser", "delete_dbs", false), - Q = config:get_integer("couch_peruser", "q", 1), - Prefix = config:get("couch_peruser", "database_prefix", ?DEFAULT_USERDB_PREFIX), - case couch_db:validate_dbname(Prefix) of - ok -> ok; - Error -> - couch_log:error("couch_peruser can't proceed as illegal database prefix ~p. - Error: ~p", [Prefix, Error]), - throw(Error) - end, - - - % set up cluster-stable listener - Period = abs(config:get_integer("couch_peruser", "cluster_quiet_period", - ?DEFAULT_QUIET_PERIOD)), - StartPeriod = abs(config:get_integer("couch_peruser", - "cluster_start_period", ?DEFAULT_START_PERIOD)), - - {ok, Mem3Cluster} = mem3_cluster:start_link(?MODULE, self(), StartPeriod, - Period), - - #state{ - parent = self(), - db_name = DbName, - delete_dbs = DeleteDbs, - mem3_cluster_pid = Mem3Cluster, - cluster_stable = false, - q_for_peruser_db = Q, - peruser_dbname_prefix = ?l2b(Prefix) - } + false -> + couch_log:debug("peruser: disabled on node ~p", [node()]), + #state{}; + true -> + couch_log:debug("peruser: enabled on node ~p", [node()]), + DbName = ?l2b( + config:get( + "couch_httpd_auth", "authentication_db", "_users" + ) + ), + DeleteDbs = config:get_boolean("couch_peruser", "delete_dbs", false), + Q = config:get_integer("couch_peruser", "q", 1), + Prefix = config:get("couch_peruser", "database_prefix", ?DEFAULT_USERDB_PREFIX), + case couch_db:validate_dbname(Prefix) of + ok -> + ok; + Error -> + couch_log:error( + "couch_peruser can't proceed as illegal database prefix ~p.\n" + " Error: ~p", + [Prefix, Error] + ), + throw(Error) + end, + + % set up cluster-stable listener + Period = abs( + config:get_integer( + "couch_peruser", + "cluster_quiet_period", + ?DEFAULT_QUIET_PERIOD + ) + ), + StartPeriod = abs( + config:get_integer( + "couch_peruser", + "cluster_start_period", + ?DEFAULT_START_PERIOD + ) + ), + + {ok, Mem3Cluster} = mem3_cluster:start_link( + ?MODULE, + self(), + StartPeriod, + Period + ), + + #state{ + parent = self(), + db_name = DbName, + delete_dbs = DeleteDbs, + mem3_cluster_pid = Mem3Cluster, + cluster_stable = false, + q_for_peruser_db = Q, + peruser_dbname_prefix = ?l2b(Prefix) + } end. - -spec start_listening(State :: #state{}) -> #state{} | ok. -start_listening(#state{states=ChangesStates}=State) - when length(ChangesStates) > 0 -> +start_listening(#state{states = ChangesStates} = State) when + length(ChangesStates) > 0 +-> % couch_log:debug("peruser: start_listening() already run on node ~p in pid ~p", [node(), self()]), State; -start_listening(#state{db_name=DbName, delete_dbs=DeleteDbs, - q_for_peruser_db = Q, peruser_dbname_prefix = Prefix} = State) -> +start_listening( + #state{ + db_name = DbName, + delete_dbs = DeleteDbs, + q_for_peruser_db = Q, + peruser_dbname_prefix = Prefix + } = State +) -> % couch_log:debug("peruser: start_listening() on node ~p", [node()]), try - States = lists:map(fun (A) -> - S = #changes_state{ - parent = State#state.parent, - db_name = A#shard.name, - delete_dbs = DeleteDbs, - q_for_peruser_db = Q, - peruser_dbname_prefix = Prefix - }, - {Pid, Ref} = spawn_opt( - ?MODULE, init_changes_handler, [S], [link, monitor]), - S#changes_state{changes_pid=Pid, changes_ref=Ref} - end, mem3:local_shards(DbName)), + States = lists:map( + fun(A) -> + S = #changes_state{ + parent = State#state.parent, + db_name = A#shard.name, + delete_dbs = DeleteDbs, + q_for_peruser_db = Q, + peruser_dbname_prefix = Prefix + }, + {Pid, Ref} = spawn_opt( + ?MODULE, init_changes_handler, [S], [link, monitor] + ), + S#changes_state{changes_pid = Pid, changes_ref = Ref} + end, + mem3:local_shards(DbName) + ), % couch_log:debug("peruser: start_listening() States ~p", [States]), State#state{states = States, cluster_stable = true} - catch error:database_does_not_exist -> - couch_log:warning("couch_peruser can't proceed as underlying database (~s) is missing, disables itself.", [DbName]), - config:set("couch_peruser", "enable", "false", lists:concat([binary_to_list(DbName), " is missing"])) + catch + error:database_does_not_exist -> + couch_log:warning( + "couch_peruser can't proceed as underlying database (~s) is missing, disables itself.", + [DbName] + ), + config:set( + "couch_peruser", + "enable", + "false", + lists:concat([binary_to_list(DbName), " is missing"]) + ) end. -spec init_changes_handler(ChangesState :: #changes_state{}) -> ok. -init_changes_handler(#changes_state{db_name=DbName} = ChangesState) -> +init_changes_handler(#changes_state{db_name = DbName} = ChangesState) -> % couch_log:debug("peruser: init_changes_handler() on DbName ~p", [DbName]), try {ok, Db} = couch_db:open_int(DbName, [?ADMIN_CTX, sys_db]), FunAcc = {fun ?MODULE:changes_handler/3, ChangesState}, (couch_changes:handle_db_changes( - #changes_args{feed="continuous", timeout=infinity}, - {json_req, null}, - Db))(FunAcc) - catch error:database_does_not_exist -> - ok + #changes_args{feed = "continuous", timeout = infinity}, + {json_req, null}, + Db + ))( + FunAcc + ) + catch + error:database_does_not_exist -> + ok end. -type db_change() :: {atom(), tuple(), binary()}. -spec changes_handler( Change :: db_change(), ResultType :: any(), - ChangesState :: #changes_state{}) -> #changes_state{}. + ChangesState :: #changes_state{} +) -> #changes_state{}. changes_handler( {change, {Doc}, _Prepend}, _ResType, - ChangesState=#changes_state{db_name=DbName, q_for_peruser_db = Q, - peruser_dbname_prefix = Prefix}) -> + ChangesState = #changes_state{ + db_name = DbName, + q_for_peruser_db = Q, + peruser_dbname_prefix = Prefix + } +) -> % couch_log:debug("peruser: changes_handler() on DbName/Doc ~p/~p", [DbName, Doc]), case couch_util:get_value(<<"id">>, Doc) of - <<"org.couchdb.user:",User/binary>> = DocId -> - case should_handle_doc(DbName, DocId) of - true -> - case couch_util:get_value(<<"deleted">>, Doc, false) of - false -> - UserDb = ensure_user_db(Prefix, User, Q), - ok = ensure_security(User, UserDb, fun add_user/3), - ChangesState; - true -> - case ChangesState#changes_state.delete_dbs of + <<"org.couchdb.user:", User/binary>> = DocId -> + case should_handle_doc(DbName, DocId) of true -> - _UserDb = delete_user_db(Prefix, User), - ChangesState; + case couch_util:get_value(<<"deleted">>, Doc, false) of + false -> + UserDb = ensure_user_db(Prefix, User, Q), + ok = ensure_security(User, UserDb, fun add_user/3), + ChangesState; + true -> + case ChangesState#changes_state.delete_dbs of + true -> + _UserDb = delete_user_db(Prefix, User), + ChangesState; + false -> + UserDb = user_db_name(Prefix, User), + ok = ensure_security(User, UserDb, fun remove_user/3), + ChangesState + end + end; false -> - UserDb = user_db_name(Prefix, User), - ok = ensure_security(User, UserDb, fun remove_user/3), ChangesState - end end; - false -> + _ -> ChangesState - end; - _ -> - ChangesState end; changes_handler(_Event, _ResType, ChangesState) -> ChangesState. --spec should_handle_doc(ShardName :: binary(), DocId::binary()) -> boolean(). +-spec should_handle_doc(ShardName :: binary(), DocId :: binary()) -> boolean(). should_handle_doc(ShardName, DocId) -> case is_stable() of - false -> - % when the cluster is unstable, we have already stopped all Listeners - % the next stable event will restart all listeners and pick up this - % doc change - couch_log:debug("peruser: skipping, cluster unstable ~s/~s", - [ShardName, DocId]), - false; - true -> - should_handle_doc_int(ShardName, DocId) + false -> + % when the cluster is unstable, we have already stopped all Listeners + % the next stable event will restart all listeners and pick up this + % doc change + couch_log:debug( + "peruser: skipping, cluster unstable ~s/~s", + [ShardName, DocId] + ), + false; + true -> + should_handle_doc_int(ShardName, DocId) end. -spec should_handle_doc_int( ShardName :: binary(), - DocId :: binary()) -> boolean(). + DocId :: binary() +) -> boolean(). should_handle_doc_int(ShardName, DocId) -> DbName = mem3:dbname(ShardName), Live = [erlang:node() | erlang:nodes()], Shards = mem3:shards(DbName, DocId), - Nodes = [N || #shard{node=N} <- Shards, lists:member(N, Live)], + Nodes = [N || #shard{node = N} <- Shards, lists:member(N, Live)], case mem3:owner(DbName, DocId, Nodes) of - ThisNode when ThisNode =:= node() -> - couch_log:debug("peruser: handling ~s/~s", [DbName, DocId]), - true; % do the database action - _OtherNode -> - couch_log:debug("peruser: skipping ~s/~s", [DbName, DocId]), - false - end. - --spec delete_user_db(Prefix:: binary(), User :: binary()) -> binary(). + ThisNode when ThisNode =:= node() -> + couch_log:debug("peruser: handling ~s/~s", [DbName, DocId]), + % do the database action + true; + _OtherNode -> + couch_log:debug("peruser: skipping ~s/~s", [DbName, DocId]), + false + end. + +-spec delete_user_db(Prefix :: binary(), User :: binary()) -> binary(). delete_user_db(Prefix, User) -> UserDb = user_db_name(Prefix, User), try case fabric:delete_db(UserDb, [?ADMIN_CTX]) of - ok -> ok; - accepted -> ok + ok -> ok; + accepted -> ok end - catch error:database_does_not_exist -> - ok + catch + error:database_does_not_exist -> + ok end, UserDb. --spec ensure_user_db(Prefix:: binary(), User :: binary(), Q :: integer()) -> binary(). +-spec ensure_user_db(Prefix :: binary(), User :: binary(), Q :: integer()) -> binary(). ensure_user_db(Prefix, User, Q) -> UserDb = user_db_name(Prefix, User), try {ok, _DbInfo} = fabric:get_db_info(UserDb) - catch error:database_does_not_exist -> - case fabric:create_db(UserDb, [?ADMIN_CTX, {q, integer_to_list(Q)}]) of - {error, file_exists} -> ok; - ok -> ok; - accepted -> ok - end + catch + error:database_does_not_exist -> + case fabric:create_db(UserDb, [?ADMIN_CTX, {q, integer_to_list(Q)}]) of + {error, file_exists} -> ok; + ok -> ok; + accepted -> ok + end end, UserDb. -spec add_user( User :: binary(), Properties :: tuple(), - Acc :: tuple()) -> tuple(). + Acc :: tuple() +) -> tuple(). add_user(User, Prop, {Modified, SecProps}) -> {PropValue} = couch_util:get_value(Prop, SecProps, {[]}), Names = couch_util:get_value(<<"names">>, PropValue, []), case lists:member(User, Names) of - true -> - {Modified, SecProps}; - false -> - {true, - lists:keystore( - Prop, 1, SecProps, - {Prop, - {lists:keystore( - <<"names">>, 1, PropValue, - {<<"names">>, [User | Names]})}})} + true -> + {Modified, SecProps}; + false -> + {true, + lists:keystore( + Prop, + 1, + SecProps, + {Prop, + {lists:keystore( + <<"names">>, + 1, + PropValue, + {<<"names">>, [User | Names]} + )}} + )} end. -spec remove_user( User :: binary(), Properties :: tuple(), - Acc :: tuple()) -> tuple(). + Acc :: tuple() +) -> tuple(). remove_user(User, Prop, {Modified, SecProps}) -> {PropValue} = couch_util:get_value(Prop, SecProps, {[]}), Names = couch_util:get_value(<<"names">>, PropValue, []), case lists:member(User, Names) of - false -> - {Modified, SecProps}; - true -> - {true, - lists:keystore( - Prop, 1, SecProps, - {Prop, - {lists:keystore( - <<"names">>, 1, PropValue, - {<<"names">>, lists:delete(User, Names)})}})} + false -> + {Modified, SecProps}; + true -> + {true, + lists:keystore( + Prop, + 1, + SecProps, + {Prop, + {lists:keystore( + <<"names">>, + 1, + PropValue, + {<<"names">>, lists:delete(User, Names)} + )}} + )} end. -spec ensure_security( User :: binary(), UserDb :: binary(), - TransformFun :: fun()) -> ok. + TransformFun :: fun() +) -> ok. ensure_security(User, UserDb, TransformFun) -> case fabric:get_all_security(UserDb, [?ADMIN_CTX]) of - {error, no_majority} -> - % TODO: make sure this is still true: single node, ignore - ok; - {ok, Shards} -> - {_ShardInfo, {SecProps}} = hd(Shards), - % assert that shards have the same security object - true = lists:all(fun ({_, {SecProps1}}) -> - SecProps =:= SecProps1 - end, Shards), - case lists:foldl( - fun (Prop, SAcc) -> TransformFun(User, Prop, SAcc) end, - {false, SecProps}, - [<<"admins">>, <<"members">>]) of - {false, _} -> + {error, no_majority} -> + % TODO: make sure this is still true: single node, ignore ok; - {true, SecProps1} -> - ok = fabric:set_security(UserDb, {SecProps1}, [?ADMIN_CTX]) - end + {ok, Shards} -> + {_ShardInfo, {SecProps}} = hd(Shards), + % assert that shards have the same security object + true = lists:all( + fun({_, {SecProps1}}) -> + SecProps =:= SecProps1 + end, + Shards + ), + case + lists:foldl( + fun(Prop, SAcc) -> TransformFun(User, Prop, SAcc) end, + {false, SecProps}, + [<<"admins">>, <<"members">>] + ) + of + {false, _} -> + ok; + {true, SecProps1} -> + ok = fabric:set_security(UserDb, {SecProps1}, [?ADMIN_CTX]) + end end. -spec user_db_name(Prefix :: binary(), User :: binary()) -> binary(). user_db_name(Prefix, User) -> HexUser = list_to_binary( - [string:to_lower(integer_to_list(X, 16)) || <<X>> <= User]), - <<Prefix/binary,HexUser/binary>>. + [string:to_lower(integer_to_list(X, 16)) || <<X>> <= User] + ), + <<Prefix/binary, HexUser/binary>>. -spec exit_changes(State :: #state{}) -> ok. exit_changes(State) -> - lists:foreach(fun (ChangesState) -> - demonitor(ChangesState#changes_state.changes_ref, [flush]), - unlink(ChangesState#changes_state.changes_pid), - exit(ChangesState#changes_state.changes_pid, kill) - end, State#state.states). + lists:foreach( + fun(ChangesState) -> + demonitor(ChangesState#changes_state.changes_ref, [flush]), + unlink(ChangesState#changes_state.changes_pid), + exit(ChangesState#changes_state.changes_pid, kill) + end, + State#state.states + ). -spec is_stable() -> true | false. is_stable() -> @@ -364,7 +452,6 @@ handle_call(is_stable, _From, #state{cluster_stable = IsStable} = State) -> handle_call(_Msg, _From, State) -> {reply, error, State}. - handle_cast(update_config, State) when State#state.states =/= undefined -> exit_changes(State), {noreply, init_state()}; @@ -386,10 +473,16 @@ handle_info({'DOWN', _Ref, _, _, _Reason}, State) -> {stop, normal, State}; handle_info({config_change, "couch_peruser", _, _, _}, State) -> handle_cast(update_config, State); -handle_info({ - config_change, - "couch_httpd_auth", - "authentication_db", _, _}, State) -> +handle_info( + { + config_change, + "couch_httpd_auth", + "authentication_db", + _, + _ + }, + State +) -> handle_cast(update_config, State); handle_info({gen_event_EXIT, _Handler, _Reason}, State) -> erlang:send_after(?RELISTEN_DELAY, self(), restart_config_listener), diff --git a/src/couch_peruser/src/couch_peruser_app.erl b/src/couch_peruser/src/couch_peruser_app.erl index 770c08237..ab0e04444 100644 --- a/src/couch_peruser/src/couch_peruser_app.erl +++ b/src/couch_peruser/src/couch_peruser_app.erl @@ -16,11 +16,8 @@ -export([start/2, stop/1]). - start(_Type, _StartArgs) -> couch_peruser_sup:start_link(). - stop(_State) -> ok. - diff --git a/src/couch_peruser/src/couch_peruser_sup.erl b/src/couch_peruser/src/couch_peruser_sup.erl index b89a36324..a9a789ce6 100644 --- a/src/couch_peruser/src/couch_peruser_sup.erl +++ b/src/couch_peruser/src/couch_peruser_sup.erl @@ -19,11 +19,8 @@ %% Helper macro for declaring children of supervisor -define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}). - start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). - init([]) -> - {ok, { {one_for_one, 5, 10}, [?CHILD(couch_peruser, worker)]}}. - + {ok, {{one_for_one, 5, 10}, [?CHILD(couch_peruser, worker)]}}. diff --git a/src/couch_peruser/test/eunit/couch_peruser_test.erl b/src/couch_peruser/test/eunit/couch_peruser_test.erl index 5ddbe7a5a..3ba344e9d 100644 --- a/src/couch_peruser/test/eunit/couch_peruser_test.erl +++ b/src/couch_peruser/test/eunit/couch_peruser_test.erl @@ -24,7 +24,7 @@ setup_all() -> TestCtx = test_util:start_couch([chttpd]), ok = application:start(couch_peruser), Hashed = couch_passwords:hash_admin_password(?ADMIN_PASSWORD), - ok = config:set("admins", ?ADMIN_USERNAME, ?b2l(Hashed), _Persist=false), + ok = config:set("admins", ?ADMIN_USERNAME, ?b2l(Hashed), _Persist = false), TestCtx. teardown_all(TestCtx) -> @@ -52,18 +52,21 @@ teardown(TestAuthDb) -> set_config("cluster", "n", "3"), do_request(delete, get_cluster_base_url() ++ "/" ++ ?b2l(TestAuthDb)), do_request(delete, get_base_url() ++ "/" ++ ?b2l(TestAuthDb)), - lists:foreach(fun(DbName) -> - case binary:part(DbName, 0, 7) of - <<"userdb-">> -> delete_db(DbName); - _ -> ok - end - end, all_dbs()). + lists:foreach( + fun(DbName) -> + case binary:part(DbName, 0, 7) of + <<"userdb-">> -> delete_db(DbName); + _ -> ok + end + end, + all_dbs() + ). set_config(Section, Key, Value) -> - ok = config:set(Section, Key, Value, _Persist=false). + ok = config:set(Section, Key, Value, _Persist = false). delete_config(Section, Key) -> - ok = config:delete(Section, Key, _Persist=false). + ok = config:delete(Section, Key, _Persist = false). do_request(Method, Url) -> Headers = [{basic_auth, {?ADMIN_USERNAME, ?ADMIN_PASSWORD}}], @@ -72,12 +75,14 @@ do_request(Method, Url) -> do_request(Method, Url, Body) -> Headers = [ {basic_auth, {?ADMIN_USERNAME, ?ADMIN_PASSWORD}}, - {"Content-Type", "application/json"}], + {"Content-Type", "application/json"} + ], {ok, _, _, _} = test_request:request(Method, Url, Headers, Body). do_anon_request(Method, Url, Body) -> Headers = [ - {"Content-Type", "application/json"}], + {"Content-Type", "application/json"} + ], {ok, _, _, _} = test_request:request(Method, Url, Headers, Body). create_db(DbName) -> @@ -87,22 +92,31 @@ delete_db(DbName) -> {ok, _, _, _} = do_request(delete, get_cluster_base_url() ++ "/" ++ ?b2l(DbName)). create_user(AuthDb, Name) -> - Body = "{\"name\":\"" ++ Name ++ - "\",\"type\":\"user\",\"roles\":[],\"password\":\"secret\"}", + Body = + "{\"name\":\"" ++ Name ++ + "\",\"type\":\"user\",\"roles\":[],\"password\":\"secret\"}", Url = lists:concat([ - get_cluster_base_url(), "/", ?b2l(AuthDb), "/org.couchdb.user:", Name]), + get_cluster_base_url(), "/", ?b2l(AuthDb), "/org.couchdb.user:", Name + ]), {ok, 201, _, _} = do_request(put, Url, Body). create_anon_user(AuthDb, Name) -> - Body = "{\"name\":\"" ++ Name ++ - "\",\"type\":\"user\",\"roles\":[],\"password\":\"secret\"}", + Body = + "{\"name\":\"" ++ Name ++ + "\",\"type\":\"user\",\"roles\":[],\"password\":\"secret\"}", Url = lists:concat([ - get_cluster_base_url(), "/", ?b2l(AuthDb), "/org.couchdb.user:", Name]), + get_cluster_base_url(), "/", ?b2l(AuthDb), "/org.couchdb.user:", Name + ]), {ok, 201, _, _} = do_anon_request(put, Url, Body). delete_user(AuthDb, Name) -> - Url = lists:concat([get_cluster_base_url(), "/", ?b2l(AuthDb), - "/org.couchdb.user:", Name]), + Url = lists:concat([ + get_cluster_base_url(), + "/", + ?b2l(AuthDb), + "/org.couchdb.user:", + Name + ]), {ok, 200, _, Body} = do_request(get, Url), {DocProps} = jiffy:decode(Body), Rev = proplists:get_value(<<"_rev">>, DocProps), @@ -110,7 +124,8 @@ delete_user(AuthDb, Name) -> get_security(DbName) -> Url = lists:concat([ - get_cluster_base_url(), "/", ?b2l(DbName), "/_security"]), + get_cluster_base_url(), "/", ?b2l(DbName), "/_security" + ]), test_util:wait(fun() -> {ok, 200, _, Body} = do_request(get, Url), case jiffy:decode(Body) of @@ -121,7 +136,8 @@ get_security(DbName) -> set_security(DbName, SecurityProperties) -> Url = lists:concat([ - get_cluster_base_url(), "/", ?b2l(DbName), "/_security"]), + get_cluster_base_url(), "/", ?b2l(DbName), "/_security" + ]), Body = jiffy:encode({SecurityProperties}), {ok, 200, _, _} = do_request(put, Url, Body). @@ -143,7 +159,6 @@ get_cluster_base_url() -> Port = integer_to_list(mochiweb_socket_server:get(chttpd, port)), "http://" ++ Addr ++ ":" ++ Port. - should_create_user_db_with_default(TestAuthDb) -> ?_test(begin create_user(TestAuthDb, "foo"), @@ -288,42 +303,42 @@ should_delete_user_db_with_custom_special_prefix(TestAuthDb) -> end). should_reflect_config_changes(TestAuthDb) -> - {timeout, 10000, ?_test(begin - User = "baz", - UserDbName = <<"userdb-62617a">>, - set_config("couch_peruser", "delete_dbs", "true"), - create_user(TestAuthDb, User), - wait_for_db_create(UserDbName), - AfterCreate1 = lists:member(UserDbName, all_dbs()), - delete_user(TestAuthDb, User), - timer:sleep(?WAIT_FOR_USER_DELETE_TIMEOUT), - wait_for_db_delete(UserDbName), - AfterDelete1 = lists:member(UserDbName, all_dbs()), - create_user(TestAuthDb, User), - wait_for_db_create(UserDbName), - AfterCreate2 = lists:member(UserDbName, all_dbs()), - set_config("couch_peruser", "delete_dbs", "false"), - delete_user(TestAuthDb, User), - timer:sleep(?WAIT_FOR_USER_DELETE_TIMEOUT), - AfterDelete2 = lists:member(UserDbName, all_dbs()), - create_user(TestAuthDb, User), - wait_for_db_create(UserDbName), - set_config("couch_peruser", "delete_dbs", "true"), - delete_user(TestAuthDb, User), - wait_for_db_delete(UserDbName), - AfterDelete3 = lists:member(UserDbName, all_dbs()), - set_config("couch_peruser", "enable", "false"), - create_user(TestAuthDb, User), - timer:sleep(?WAIT_FOR_USER_DELETE_TIMEOUT), - AfterCreate3 = lists:member(UserDbName, all_dbs()), - ?assert(AfterCreate1), - ?assertNot(AfterDelete1), - ?assert(AfterCreate2), - ?assert(AfterDelete2), - ?assertNot(AfterDelete3), - ?assertNot(AfterCreate3) - end)}. - + {timeout, 10000, + ?_test(begin + User = "baz", + UserDbName = <<"userdb-62617a">>, + set_config("couch_peruser", "delete_dbs", "true"), + create_user(TestAuthDb, User), + wait_for_db_create(UserDbName), + AfterCreate1 = lists:member(UserDbName, all_dbs()), + delete_user(TestAuthDb, User), + timer:sleep(?WAIT_FOR_USER_DELETE_TIMEOUT), + wait_for_db_delete(UserDbName), + AfterDelete1 = lists:member(UserDbName, all_dbs()), + create_user(TestAuthDb, User), + wait_for_db_create(UserDbName), + AfterCreate2 = lists:member(UserDbName, all_dbs()), + set_config("couch_peruser", "delete_dbs", "false"), + delete_user(TestAuthDb, User), + timer:sleep(?WAIT_FOR_USER_DELETE_TIMEOUT), + AfterDelete2 = lists:member(UserDbName, all_dbs()), + create_user(TestAuthDb, User), + wait_for_db_create(UserDbName), + set_config("couch_peruser", "delete_dbs", "true"), + delete_user(TestAuthDb, User), + wait_for_db_delete(UserDbName), + AfterDelete3 = lists:member(UserDbName, all_dbs()), + set_config("couch_peruser", "enable", "false"), + create_user(TestAuthDb, User), + timer:sleep(?WAIT_FOR_USER_DELETE_TIMEOUT), + AfterCreate3 = lists:member(UserDbName, all_dbs()), + ?assert(AfterCreate1), + ?assertNot(AfterDelete1), + ?assert(AfterCreate2), + ?assert(AfterDelete2), + ?assertNot(AfterDelete3), + ?assertNot(AfterCreate3) + end)}. should_add_user_to_db_admins(TestAuthDb) -> ?_test(begin @@ -332,8 +347,9 @@ should_add_user_to_db_admins(TestAuthDb) -> create_user(TestAuthDb, User), wait_for_db_create(UserDbName), ?assertEqual( - {[{<<"names">>,[<<"qux">>]}]}, - proplists:get_value(<<"admins">>, get_security(UserDbName))) + {[{<<"names">>, [<<"qux">>]}]}, + proplists:get_value(<<"admins">>, get_security(UserDbName)) + ) end). should_add_user_to_db_members(TestAuthDb) -> @@ -343,8 +359,9 @@ should_add_user_to_db_members(TestAuthDb) -> create_user(TestAuthDb, User), wait_for_db_create(UserDbName), ?assertEqual( - {[{<<"names">>,[<<"qux">>]}]}, - proplists:get_value(<<"members">>, get_security(UserDbName))) + {[{<<"names">>, [<<"qux">>]}]}, + proplists:get_value(<<"members">>, get_security(UserDbName)) + ) end). should_not_remove_existing_db_admins(TestAuthDb) -> @@ -352,15 +369,17 @@ should_not_remove_existing_db_admins(TestAuthDb) -> User = "qux", UserDbName = <<"userdb-717578">>, SecurityProperties = [ - {<<"admins">>,{[{<<"names">>,[<<"foo">>,<<"bar">>]}]}}, - {<<"members">>,{[{<<"names">>,[<<"baz">>,<<"pow">>]}]}} + {<<"admins">>, {[{<<"names">>, [<<"foo">>, <<"bar">>]}]}}, + {<<"members">>, {[{<<"names">>, [<<"baz">>, <<"pow">>]}]}} ], create_db(UserDbName), set_security(UserDbName, SecurityProperties), create_user(TestAuthDb, User), wait_for_security_create(<<"admins">>, User, UserDbName), - {AdminProperties} = proplists:get_value(<<"admins">>, - get_security(UserDbName)), + {AdminProperties} = proplists:get_value( + <<"admins">>, + get_security(UserDbName) + ), AdminNames = proplists:get_value(<<"names">>, AdminProperties), ?assert(lists:member(<<"foo">>, AdminNames)), ?assert(lists:member(<<"bar">>, AdminNames)), @@ -372,15 +391,17 @@ should_not_remove_existing_db_members(TestAuthDb) -> User = "qux", UserDbName = <<"userdb-717578">>, SecurityProperties = [ - {<<"admins">>,{[{<<"names">>,[<<"pow">>,<<"wow">>]}]}}, - {<<"members">>,{[{<<"names">>,[<<"pow">>,<<"wow">>]}]}} + {<<"admins">>, {[{<<"names">>, [<<"pow">>, <<"wow">>]}]}}, + {<<"members">>, {[{<<"names">>, [<<"pow">>, <<"wow">>]}]}} ], create_db(UserDbName), set_security(UserDbName, SecurityProperties), create_user(TestAuthDb, User), wait_for_security_create(<<"members">>, User, UserDbName), - {MemberProperties} = proplists:get_value(<<"members">>, - get_security(UserDbName)), + {MemberProperties} = proplists:get_value( + <<"members">>, + get_security(UserDbName) + ), MemberNames = proplists:get_value(<<"names">>, MemberProperties), ?assert(lists:member(<<"pow">>, MemberNames)), ?assert(lists:member(<<"wow">>, MemberNames)), @@ -392,23 +413,27 @@ should_remove_user_from_db_admins(TestAuthDb) -> User = "qux", UserDbName = <<"userdb-717578">>, SecurityProperties = [ - {<<"admins">>,{[{<<"names">>,[<<"foo">>,<<"bar">>]}]}}, - {<<"members">>,{[{<<"names">>,[<<"baz">>,<<"pow">>]}]}} + {<<"admins">>, {[{<<"names">>, [<<"foo">>, <<"bar">>]}]}}, + {<<"members">>, {[{<<"names">>, [<<"baz">>, <<"pow">>]}]}} ], create_db(UserDbName), set_security(UserDbName, SecurityProperties), create_user(TestAuthDb, User), wait_for_security_create(<<"admins">>, User, UserDbName), - {AdminProperties} = proplists:get_value(<<"admins">>, - get_security(UserDbName)), + {AdminProperties} = proplists:get_value( + <<"admins">>, + get_security(UserDbName) + ), AdminNames = proplists:get_value(<<"names">>, AdminProperties), FooBefore = lists:member(<<"foo">>, AdminNames), BarBefore = lists:member(<<"bar">>, AdminNames), QuxBefore = lists:member(<<"qux">>, AdminNames), delete_user(TestAuthDb, User), wait_for_security_delete(<<"admins">>, User, UserDbName), - {NewAdminProperties} = proplists:get_value(<<"admins">>, - get_security(UserDbName)), + {NewAdminProperties} = proplists:get_value( + <<"admins">>, + get_security(UserDbName) + ), NewAdminNames = proplists:get_value(<<"names">>, NewAdminProperties), FooAfter = lists:member(<<"foo">>, NewAdminNames), BarAfter = lists:member(<<"bar">>, NewAdminNames), @@ -426,23 +451,27 @@ should_remove_user_from_db_members(TestAuthDb) -> User = "qux", UserDbName = <<"userdb-717578">>, SecurityProperties = [ - {<<"admins">>,{[{<<"names">>,[<<"pow">>,<<"wow">>]}]}}, - {<<"members">>,{[{<<"names">>,[<<"pow">>,<<"wow">>]}]}} + {<<"admins">>, {[{<<"names">>, [<<"pow">>, <<"wow">>]}]}}, + {<<"members">>, {[{<<"names">>, [<<"pow">>, <<"wow">>]}]}} ], create_db(UserDbName), set_security(UserDbName, SecurityProperties), create_user(TestAuthDb, User), wait_for_security_create(<<"members">>, User, UserDbName), - {MemberProperties} = proplists:get_value(<<"members">>, - get_security(UserDbName)), + {MemberProperties} = proplists:get_value( + <<"members">>, + get_security(UserDbName) + ), MemberNames = proplists:get_value(<<"names">>, MemberProperties), PowBefore = lists:member(<<"pow">>, MemberNames), WowBefore = lists:member(<<"wow">>, MemberNames), QuxBefore = lists:member(<<"qux">>, MemberNames), delete_user(TestAuthDb, User), wait_for_security_delete(<<"members">>, User, UserDbName), - {NewMemberProperties} = proplists:get_value(<<"members">>, - get_security(UserDbName)), + {NewMemberProperties} = proplists:get_value( + <<"members">>, + get_security(UserDbName) + ), NewMemberNames = proplists:get_value(<<"names">>, NewMemberProperties), PowAfter = lists:member(<<"pow">>, NewMemberNames), WowAfter = lists:member(<<"wow">>, NewMemberNames), @@ -455,12 +484,11 @@ should_remove_user_from_db_members(TestAuthDb) -> ?assertNot(QuxAfter) end). - - wait_for_db_create(UserDbName) -> test_util:wait(fun() -> case all_dbs_with_errors() of - {error, _, _ , _} -> wait; + {error, _, _, _} -> + wait; {ok, _, _, AllDbs} -> case lists:member(UserDbName, AllDbs) of true -> true; @@ -472,7 +500,8 @@ wait_for_db_create(UserDbName) -> wait_for_db_delete(UserDbName) -> test_util:wait(fun() -> case all_dbs_with_errors() of - {ok, 500, _ , _} -> wait; + {ok, 500, _, _} -> + wait; {ok, _, _, AllDbs} -> case not lists:member(UserDbName, AllDbs) of true -> true; @@ -508,10 +537,12 @@ couch_peruser_test_() -> "couch_peruser test", { setup, - fun setup_all/0, fun teardown_all/1, + fun setup_all/0, + fun teardown_all/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_create_anon_user_db_with_default/1, fun should_create_anon_user_db_with_custom_prefix/1, diff --git a/src/couch_plugins/src/couch_plugins.erl b/src/couch_plugins/src/couch_plugins.erl index 139a878bf..97834134b 100644 --- a/src/couch_plugins/src/couch_plugins.erl +++ b/src/couch_plugins/src/couch_plugins.erl @@ -17,63 +17,62 @@ % couch_plugins:install({"geocouch", "http://people.apache.org/~jan/", "couchdb1.2.x_v0.3.0-11-gd83ba22", [{"R15B03", "ZetgdHj2bY2w37buulWVf3USOZs="}]}). plugin_dir() -> - couch_config:get("couchdb", "plugin_dir"). + couch_config:get("couchdb", "plugin_dir"). log(T) -> - couch_log:debug("[couch_plugins] ~p ~n", [T]). + couch_log:debug("[couch_plugins] ~p ~n", [T]). %% "geocouch", "http://localhost:8000/dist", "1.0.0" -type plugin() :: {string(), string(), string(), list()}. -spec install(plugin()) -> ok | {error, string()}. -install({Name, _BaseUrl, Version, Checksums}=Plugin) -> - log("Installing " ++ Name), +install({Name, _BaseUrl, Version, Checksums} = Plugin) -> + log("Installing " ++ Name), - {ok, LocalFilename} = download(Plugin), - log("downloaded to " ++ LocalFilename), + {ok, LocalFilename} = download(Plugin), + log("downloaded to " ++ LocalFilename), - ok = verify_checksum(LocalFilename, Checksums), - log("checksum verified"), + ok = verify_checksum(LocalFilename, Checksums), + log("checksum verified"), - ok = untargz(LocalFilename), - log("extraction done"), + ok = untargz(LocalFilename), + log("extraction done"), - ok = add_code_path(Name, Version), - log("added code path"), + ok = add_code_path(Name, Version), + log("added code path"), - ok = register_plugin(Name, Version), - log("registered plugin"), + ok = register_plugin(Name, Version), + log("registered plugin"), - load_config(Name, Version), - log("loaded config"), + load_config(Name, Version), + log("loaded config"), - ok. + ok. % Idempotent uninstall, if you uninstall a non-existant % plugin, you get an `ok`. -spec uninstall(plugin()) -> ok | {error, string()}. uninstall({Name, _BaseUrl, Version, _Checksums}) -> - % unload config - ok = unload_config(Name, Version), - log("config unloaded"), + % unload config + ok = unload_config(Name, Version), + log("config unloaded"), - % delete files - ok = delete_files(Name, Version), - log("files deleted"), + % delete files + ok = delete_files(Name, Version), + log("files deleted"), - % delete code path - ok = del_code_path(Name, Version), - log("deleted code path"), + % delete code path + ok = del_code_path(Name, Version), + log("deleted code path"), - % unregister plugin - ok = unregister_plugin(Name), - log("unregistered plugin"), + % unregister plugin + ok = unregister_plugin(Name), + log("unregistered plugin"), - % done - ok. + % done + ok. %% * * * - %% Plugin Registration %% On uninstall: %% - add plugins/name = version to config @@ -82,15 +81,14 @@ uninstall({Name, _BaseUrl, Version, _Checksums}) -> -spec register_plugin(string(), string()) -> ok. register_plugin(Name, Version) -> - couch_config:set("plugins", Name, Version). + couch_config:set("plugins", Name, Version). -spec unregister_plugin(string()) -> ok. unregister_plugin(Name) -> - couch_config:delete("plugins", Name). + couch_config:delete("plugins", Name). %% * * * - %% Load Config %% Parses <plugindir>/priv/default.d/<pluginname.ini> and applies %% the contents to the config system, or removes them on uninstall @@ -105,8 +103,10 @@ unload_config(Name, Version) -> -spec loop_config(string(), string(), function()) -> ok. loop_config(Name, Version, Fun) -> - lists:foreach(fun(File) -> load_config_file(File, Fun) end, - filelib:wildcard(file_names(Name, Version))). + lists:foreach( + fun(File) -> load_config_file(File, Fun) end, + filelib:wildcard(file_names(Name, Version)) + ). -spec load_config_file(string(), function()) -> ok. load_config_file(File, Fun) -> @@ -123,13 +123,18 @@ delete_config({{Section, Key}, _Value}) -> -spec file_names(string(), string()) -> string(). file_names(Name, Version) -> - filename:join( - [plugin_dir(), get_file_slug(Name, Version), - "priv", "default.d", "*.ini"]). + filename:join( + [ + plugin_dir(), + get_file_slug(Name, Version), + "priv", + "default.d", + "*.ini" + ] + ). %% * * * - %% Code Path Management %% The Erlang code path is where the Erlang runtime looks for `.beam` %% files to load on, say, `application:load()`. Since plugin directories @@ -138,137 +143,151 @@ file_names(Name, Version) -> -spec add_code_path(string(), string()) -> ok | {error, bad_directory}. add_code_path(Name, Version) -> - PluginPath = plugin_dir() ++ "/" ++ get_file_slug(Name, Version) ++ "/ebin", - case code:add_path(PluginPath) of - true -> ok; - Else -> - couch_log:error("Failed to add PluginPath: '~s'", [PluginPath]), - Else - end. + PluginPath = plugin_dir() ++ "/" ++ get_file_slug(Name, Version) ++ "/ebin", + case code:add_path(PluginPath) of + true -> + ok; + Else -> + couch_log:error("Failed to add PluginPath: '~s'", [PluginPath]), + Else + end. -spec del_code_path(string(), string()) -> ok | {error, atom()}. del_code_path(Name, Version) -> - PluginPath = plugin_dir() ++ "/" ++ get_file_slug(Name, Version) ++ "/ebin", - case code:del_path(PluginPath) of - true -> ok; - _Else -> - couch_log:debug("Failed to delete PluginPath: '~s', ignoring", - [PluginPath]), - ok - end. + PluginPath = plugin_dir() ++ "/" ++ get_file_slug(Name, Version) ++ "/ebin", + case code:del_path(PluginPath) of + true -> + ok; + _Else -> + couch_log:debug( + "Failed to delete PluginPath: '~s', ignoring", + [PluginPath] + ), + ok + end. %% * * * - -spec untargz(string()) -> {ok, string()} | {error, string()}. untargz(Filename) -> - % read .gz file - {ok, GzData} = file:read_file(Filename), - % gunzip - log("unzipped"), - TarData = zlib:gunzip(GzData), - ok = filelib:ensure_dir(plugin_dir()), - % untar - erl_tar:extract({binary, TarData}, [{cwd, plugin_dir()}, keep_old_files]). + % read .gz file + {ok, GzData} = file:read_file(Filename), + % gunzip + log("unzipped"), + TarData = zlib:gunzip(GzData), + ok = filelib:ensure_dir(plugin_dir()), + % untar + erl_tar:extract({binary, TarData}, [{cwd, plugin_dir()}, keep_old_files]). -spec delete_files(string(), string()) -> ok | {error, atom()}. delete_files(Name, Version) -> - PluginPath = plugin_dir() ++ "/" ++ get_file_slug(Name, Version), - mochitemp:rmtempdir(PluginPath). - + PluginPath = plugin_dir() ++ "/" ++ get_file_slug(Name, Version), + mochitemp:rmtempdir(PluginPath). % downloads a pluygin .tar.gz into a local plugins directory -spec download(string()) -> ok | {error, string()}. -download({Name, _BaseUrl, Version, _Checksums}=Plugin) -> - TargetFile = filename:join(mochitemp:gettempdir(), get_filename(Name, Version)), - case file_exists(TargetFile) of - %% wipe and redownload - true -> file:delete(TargetFile); - _Else -> ok - end, - Url = get_url(Plugin), - HTTPOptions = [ - {connect_timeout, 30*1000}, % 30 seconds - {timeout, 30*1000} % 30 seconds - ], - % todo: windows - Options = [ - {stream, TargetFile}, % /tmp/something - {body_format, binary}, - {full_result, false} - ], - % todo: reduce to just httpc:request() - case httpc:request(get, {Url, []}, HTTPOptions, Options) of - {ok, _Result} -> - log("downloading " ++ Url), - {ok, TargetFile}; - Error -> Error - end. +download({Name, _BaseUrl, Version, _Checksums} = Plugin) -> + TargetFile = filename:join(mochitemp:gettempdir(), get_filename(Name, Version)), + case file_exists(TargetFile) of + %% wipe and redownload + true -> file:delete(TargetFile); + _Else -> ok + end, + Url = get_url(Plugin), + HTTPOptions = [ + % 30 seconds + {connect_timeout, 30 * 1000}, + % 30 seconds + {timeout, 30 * 1000} + ], + % todo: windows + Options = [ + % /tmp/something + {stream, TargetFile}, + {body_format, binary}, + {full_result, false} + ], + % todo: reduce to just httpc:request() + case httpc:request(get, {Url, []}, HTTPOptions, Options) of + {ok, _Result} -> + log("downloading " ++ Url), + {ok, TargetFile}; + Error -> + Error + end. -spec verify_checksum(string(), list()) -> ok | {error, string()}. verify_checksum(Filename, Checksums) -> - - CouchDBVersion = couchdb_version(), - case proplists:get_value(CouchDBVersion, Checksums) of - undefined -> - couch_log:error("[couch_plugins] Can't find checksum for CouchDB Version" - " '~s'", [CouchDBVersion]), - {error, no_couchdb_checksum}; - OTPChecksum -> - OTPRelease = erlang:system_info(otp_release), - case proplists:get_value(OTPRelease, OTPChecksum) of - undefined -> - couch_log:error("[couch_plugins] Can't find checksum for Erlang Version" - " '~s'", [OTPRelease]), - {error, no_erlang_checksum}; - Checksum -> - do_verify_checksum(Filename, Checksum) - end - end. + CouchDBVersion = couchdb_version(), + case proplists:get_value(CouchDBVersion, Checksums) of + undefined -> + couch_log:error( + "[couch_plugins] Can't find checksum for CouchDB Version" + " '~s'", + [CouchDBVersion] + ), + {error, no_couchdb_checksum}; + OTPChecksum -> + OTPRelease = erlang:system_info(otp_release), + case proplists:get_value(OTPRelease, OTPChecksum) of + undefined -> + couch_log:error( + "[couch_plugins] Can't find checksum for Erlang Version" + " '~s'", + [OTPRelease] + ), + {error, no_erlang_checksum}; + Checksum -> + do_verify_checksum(Filename, Checksum) + end + end. -spec do_verify_checksum(string(), string()) -> ok | {error, string()}. do_verify_checksum(Filename, Checksum) -> - couch_log:debug("Checking Filename: ~s", [Filename]), - case file:read_file(Filename) of - {ok, Data} -> - ComputedChecksum = binary_to_list(base64:encode(crypto:hash(sha, Data))), - case ComputedChecksum of - Checksum -> ok; - _Else -> - couch_log:error("Checksum mismatch. Wanted: '~p'. Got '~p'", - [Checksum, ComputedChecksum]), - {error, checksum_mismatch} - end; - Error -> Error - end. - + couch_log:debug("Checking Filename: ~s", [Filename]), + case file:read_file(Filename) of + {ok, Data} -> + ComputedChecksum = binary_to_list(base64:encode(crypto:hash(sha, Data))), + case ComputedChecksum of + Checksum -> + ok; + _Else -> + couch_log:error( + "Checksum mismatch. Wanted: '~p'. Got '~p'", + [Checksum, ComputedChecksum] + ), + {error, checksum_mismatch} + end; + Error -> + Error + end. %% utils -spec get_url(plugin()) -> string(). get_url({Name, BaseUrl, Version, _Checksums}) -> - BaseUrl ++ "/" ++ get_filename(Name, Version). + BaseUrl ++ "/" ++ get_filename(Name, Version). -spec get_filename(string(), string()) -> string(). get_filename(Name, Version) -> - get_file_slug(Name, Version) ++ ".tar.gz". + get_file_slug(Name, Version) ++ ".tar.gz". -spec get_file_slug(string(), string()) -> string(). get_file_slug(Name, Version) -> - % OtpRelease does not include patch levels like the -1 in R15B03-1 - OTPRelease = erlang:system_info(otp_release), - CouchDBVersion = couchdb_version(), - string:join([Name, Version, OTPRelease, CouchDBVersion], "-"). + % OtpRelease does not include patch levels like the -1 in R15B03-1 + OTPRelease = erlang:system_info(otp_release), + CouchDBVersion = couchdb_version(), + string:join([Name, Version, OTPRelease, CouchDBVersion], "-"). -spec file_exists(string()) -> boolean(). file_exists(Filename) -> - does_file_exist(file:read_file_info(Filename)). + does_file_exist(file:read_file_info(Filename)). -spec does_file_exist(term()) -> boolean(). does_file_exist({error, enoent}) -> false; does_file_exist(_Else) -> true. couchdb_version() -> - couch_server:get_version(short). + couch_server:get_version(short). % installing a plugin: % - POST /_plugins -d {plugin-def} @@ -301,4 +320,3 @@ couchdb_version() -> % geocouch-{geocouch_version}-{erlang_version}/ebin % [geocouch-{geocouch_version}-{erlang_version}/config/config.erlt] % [geocouch-{geocouch_version}-{erlang_version}/share/] - diff --git a/src/couch_plugins/src/couch_plugins_httpd.erl b/src/couch_plugins/src/couch_plugins_httpd.erl index 90a09a5a5..784f040fc 100644 --- a/src/couch_plugins/src/couch_plugins_httpd.erl +++ b/src/couch_plugins/src/couch_plugins_httpd.erl @@ -15,7 +15,7 @@ -include_lib("couch/include/couch_db.hrl"). -handle_req(#httpd{method='POST'}=Req) -> +handle_req(#httpd{method = 'POST'} = Req) -> ok = couch_httpd:verify_is_server_admin(Req), couch_httpd:validate_ctype(Req, "application/json"), @@ -29,15 +29,15 @@ handle_req(#httpd{method='POST'}=Req) -> Plugin = {Name, Url, Version, Checksums}, case do_install(Delete, Plugin) of - ok -> - couch_httpd:send_json(Req, 202, {[{ok, true}]}); - Error -> - couch_log:debug("Plugin Spec: ~p", [PluginSpec]), - couch_httpd:send_error(Req, {bad_request, Error}) + ok -> + couch_httpd:send_json(Req, 202, {[{ok, true}]}); + Error -> + couch_log:debug("Plugin Spec: ~p", [PluginSpec]), + couch_httpd:send_error(Req, {bad_request, Error}) end; % handles /_plugins/<pluginname>/<file> % serves <plugin_dir>/<pluginname>-<pluginversion>-<otpversion>-<couchdbversion>/<file> -handle_req(#httpd{method='GET',path_parts=[_, Name0 | Path0]}=Req) -> +handle_req(#httpd{method = 'GET', path_parts = [_, Name0 | Path0]} = Req) -> Name = ?b2l(Name0), Path = lists:map(fun binary_to_list/1, Path0), OTPRelease = erlang:system_info(otp_release), @@ -51,15 +51,19 @@ handle_req(Req) -> couch_httpd:send_method_not_allowed(Req, "POST"). plugin_dir() -> - couch_config:get("couchdb", "plugin_dir"). + couch_config:get("couchdb", "plugin_dir"). do_install(false, Plugin) -> couch_plugins:install(Plugin); do_install(true, Plugin) -> couch_plugins:uninstall(Plugin). parse_checksums(Checksums) -> - lists:map(fun({K, {V}}) -> - {binary_to_list(K), parse_checksums(V)}; - ({K, V}) -> - {binary_to_list(K), binary_to_list(V)} - end, Checksums). + lists:map( + fun + ({K, {V}}) -> + {binary_to_list(K), parse_checksums(V)}; + ({K, V}) -> + {binary_to_list(K), binary_to_list(V)} + end, + Checksums + ). diff --git a/src/couch_prometheus/src/couch_prometheus_http.erl b/src/couch_prometheus/src/couch_prometheus_http.erl index bd0c4c6f9..b3df1ea4b 100644 --- a/src/couch_prometheus/src/couch_prometheus_http.erl +++ b/src/couch_prometheus/src/couch_prometheus_http.erl @@ -23,10 +23,11 @@ -include_lib("couch/include/couch_db.hrl"). start_link() -> - IP = case config:get("prometheus", "bind_address", "any") of - "any" -> any; - Else -> Else - end, + IP = + case config:get("prometheus", "bind_address", "any") of + "any" -> any; + Else -> Else + end, Port = config:get("prometheus", "port"), ok = couch_httpd:validate_bind_address(IP), @@ -47,7 +48,7 @@ start_link() -> handle_request(MochiReq) -> RawUri = MochiReq:get(raw_path), {"/" ++ Path, _, _} = mochiweb_util:urlsplit_path(RawUri), - PathParts = string:tokens(Path, "/"), + PathParts = string:tokens(Path, "/"), try case PathParts of ["_node", Node, "_prometheus"] -> @@ -55,16 +56,19 @@ handle_request(MochiReq) -> _ -> send_error(MochiReq, 404, <<"not_found">>, <<>>) end - catch T:R -> - Body = list_to_binary(io_lib:format("~p:~p", [T, R])), - send_error(MochiReq, 500, <<"server_error">>, Body) + catch + T:R -> + Body = list_to_binary(io_lib:format("~p:~p", [T, R])), + send_error(MochiReq, 500, <<"server_error">>, Body) end. send_prometheus(MochiReq, Node) -> Type = "text/plain; version=" ++ ?PROMETHEUS_VERSION, - Headers = couch_httpd:server_header() ++ [ - {<<"Content-Type">>, ?l2b(Type)} - ], + Headers = + couch_httpd:server_header() ++ + [ + {<<"Content-Type">>, ?l2b(Type)} + ], Body = call_node(Node, couch_prometheus_server, scrape, []), send_resp(MochiReq, 200, Headers, Body). @@ -73,23 +77,29 @@ send_resp(MochiReq, Status, ExtraHeaders, Body) -> MochiReq:respond({Status, Headers, Body}). send_error(MochiReq, Code, Error, Reason) -> - Headers = couch_httpd:server_header() ++ [ - {<<"Content-Type">>, <<"application/json">>} - ], - JsonError = {[{<<"error">>, Error}, - {<<"reason">>, Reason}]}, + Headers = + couch_httpd:server_header() ++ + [ + {<<"Content-Type">>, <<"application/json">>} + ], + JsonError = + {[ + {<<"error">>, Error}, + {<<"reason">>, Reason} + ]}, Body = ?JSON_ENCODE(JsonError), MochiReq:respond({Code, Headers, Body}). call_node("_local", Mod, Fun, Args) -> call_node(node(), Mod, Fun, Args); call_node(Node0, Mod, Fun, Args) when is_list(Node0) -> - Node1 = try - list_to_existing_atom(Node0) - catch - error:badarg -> - NoNode = list_to_binary(Node0), - throw({not_found, <<"no such node: ", NoNode/binary>>}) + Node1 = + try + list_to_existing_atom(Node0) + catch + error:badarg -> + NoNode = list_to_binary(Node0), + throw({not_found, <<"no such node: ", NoNode/binary>>}) end, call_node(Node1, Mod, Fun, Args); call_node(Node, Mod, Fun, Args) when is_atom(Node) -> diff --git a/src/couch_prometheus/src/couch_prometheus_server.erl b/src/couch_prometheus/src/couch_prometheus_server.erl index e97df04a4..701483a38 100644 --- a/src/couch_prometheus/src/couch_prometheus_server.erl +++ b/src/couch_prometheus/src/couch_prometheus_server.erl @@ -48,7 +48,7 @@ start_link() -> init([]) -> Metrics = refresh_metrics(), RT = update_refresh_timer(), - {ok, #st{metrics=Metrics, refresh=RT}}. + {ok, #st{metrics = Metrics, refresh = RT}}. scrape() -> {ok, Metrics} = gen_server:call(?MODULE, scrape), @@ -57,13 +57,13 @@ scrape() -> version() -> ?PROMETHEUS_VERSION. -handle_call(scrape, _from, #st{metrics = Metrics}=State) -> +handle_call(scrape, _from, #st{metrics = Metrics} = State) -> {reply, {ok, Metrics}, State}; -handle_call(refresh, _from, #st{refresh=OldRT} = State) -> +handle_call(refresh, _from, #st{refresh = OldRT} = State) -> timer:cancel(OldRT), Metrics = refresh_metrics(), RT = update_refresh_timer(), - {reply, ok, State#st{metrics=Metrics, refresh=RT}}; + {reply, ok, State#st{metrics = Metrics, refresh = RT}}; handle_call(Msg, _From, State) -> {stop, {unknown_call, Msg}, error, State}. @@ -73,7 +73,7 @@ handle_cast(Msg, State) -> handle_info(refresh, State) -> Metrics = refresh_metrics(), RT = update_refresh_timer(), - {noreply, State#st{metrics=Metrics, refresh=RT}}; + {noreply, State#st{metrics = Metrics, refresh = RT}}; handle_info(Msg, State) -> {stop, {unknown_info, Msg}, State}. @@ -86,15 +86,23 @@ code_change(_OldVsn, State, _Extra) -> refresh_metrics() -> CouchDB = get_couchdb_stats(), System = couch_stats_httpd:to_ejson(get_system_stats()), - couch_prometheus_util:to_bin(lists:map(fun(Line) -> - io_lib:format("~s~n", [Line]) - end, CouchDB ++ System)). + couch_prometheus_util:to_bin( + lists:map( + fun(Line) -> + io_lib:format("~s~n", [Line]) + end, + CouchDB ++ System + ) + ). get_couchdb_stats() -> Stats = lists:sort(couch_stats:fetch()), - lists:flatmap(fun({Path, Info}) -> - couch_to_prom(Path, Info, Stats) - end, Stats). + lists:flatmap( + fun({Path, Info}) -> + couch_to_prom(Path, Info, Stats) + end, + Stats + ). get_system_stats() -> lists:flatten([ @@ -111,9 +119,12 @@ get_uptime_stat() -> to_prom(uptime_seconds, counter, couch_app:uptime() div 1000). get_vm_stats() -> - MemLabels = lists:map(fun({Type, Value}) -> - {[{memory_type, Type}], Value} - end, erlang:memory()), + MemLabels = lists:map( + fun({Type, Value}) -> + {[{memory_type, Type}], Value} + end, + erlang:memory() + ), {NumGCs, WordsReclaimed, _} = erlang:statistics(garbage_collection), CtxSwitches = element(1, erlang:statistics(context_switches)), Reds = element(1, erlang:statistics(reductions)), @@ -137,14 +148,17 @@ get_io_stats() -> ]. get_message_queue_stats() -> - Queues = lists:map(fun(Name) -> - case process_info(whereis(Name), message_queue_len) of - {message_queue_len, N} -> - N; - _ -> - 0 - end - end, registered()), + Queues = lists:map( + fun(Name) -> + case process_info(whereis(Name), message_queue_len) of + {message_queue_len, N} -> + N; + _ -> + 0 + end + end, + registered() + ), [ to_prom(erlang_message_queues, gauge, lists:sum(Queues)), to_prom(erlang_message_queue_min, gauge, lists:min(Queues)), @@ -153,13 +167,14 @@ get_message_queue_stats() -> get_run_queue_stats() -> %% Workaround for https://bugs.erlang.org/browse/ERL-1355 - {Normal, Dirty} = case erlang:system_info(dirty_cpu_schedulers) > 0 of - false -> - {statistics(run_queue), 0}; - true -> - [DCQ | SQs] = lists:reverse(statistics(run_queue_lengths)), - {lists:sum(SQs), DCQ} - end, + {Normal, Dirty} = + case erlang:system_info(dirty_cpu_schedulers) > 0 of + false -> + {statistics(run_queue), 0}; + true -> + [DCQ | SQs] = lists:reverse(statistics(run_queue_lengths)), + {lists:sum(SQs), DCQ} + end, [ to_prom(erlang_scheduler_queues, gauge, Normal), to_prom(erlang_dirty_cpu_scheduler_queues, gauge, Dirty) diff --git a/src/couch_prometheus/src/couch_prometheus_sup.erl b/src/couch_prometheus/src/couch_prometheus_sup.erl index 8d8c7e078..45a884fad 100644 --- a/src/couch_prometheus/src/couch_prometheus_sup.erl +++ b/src/couch_prometheus/src/couch_prometheus_sup.erl @@ -26,7 +26,8 @@ start_link() -> init([]) -> {ok, { - {one_for_one, 5, 10}, [ + {one_for_one, 5, 10}, + [ ?CHILD(couch_prometheus_server, worker) ] ++ maybe_start_prometheus_http() }}. diff --git a/src/couch_prometheus/src/couch_prometheus_util.erl b/src/couch_prometheus/src/couch_prometheus_util.erl index c3b58cb3a..ea2cdf737 100644 --- a/src/couch_prometheus/src/couch_prometheus_util.erl +++ b/src/couch_prometheus/src/couch_prometheus_util.erl @@ -10,7 +10,7 @@ % License for the specific language governing permissions and limitations under % the License. --module(couch_prometheus_util ). +-module(couch_prometheus_util). -export([ couch_to_prom/3, @@ -25,7 +25,6 @@ couch_to_prom([couch_log, level, alert], Info, _All) -> to_prom(couch_log_requests_total, counter, {[{level, alert}], val(Info)}); couch_to_prom([couch_log, level, Level], Info, _All) -> to_prom(couch_log_requests_total, {[{level, Level}], val(Info)}); - couch_to_prom([couch_replicator, checkpoints, failure], Info, _All) -> to_prom(couch_replicator_checkpoints_failure_total, counter, val(Info)); couch_to_prom([couch_replicator, checkpoints, success], Info, All) -> @@ -41,7 +40,6 @@ couch_to_prom([couch_replicator, stream_responses, failure], Info, _All) -> couch_to_prom([couch_replicator, stream_responses, success], Info, All) -> Total = val(Info) + val([couch_replicator, stream_responses, failure], All), to_prom(couch_replicator_stream_responses_total, counter, Total); - couch_to_prom([couchdb, auth_cache_hits], Info, All) -> Total = val(Info) + val([couchdb, auth_cache_misses], All), to_prom(auth_cache_requests_total, counter, Total); @@ -53,7 +51,6 @@ couch_to_prom([couchdb, httpd_request_methods, Method], Info, _All) -> to_prom(httpd_request_methods, {[{method, Method}], val(Info)}); couch_to_prom([couchdb, httpd_status_codes, Code], Info, _All) -> to_prom(httpd_status_codes, {[{code, Code}], val(Info)}); - couch_to_prom([ddoc_cache, hit], Info, All) -> Total = val(Info) + val([ddoc_cache, miss], All), to_prom(ddoc_cache_requests_total, counter, Total); @@ -61,21 +58,17 @@ couch_to_prom([ddoc_cache, miss], Info, _All) -> to_prom(ddoc_cache_requests_failures_total, counter, val(Info)); couch_to_prom([ddoc_cache, recovery], Info, _All) -> to_prom(ddoc_cache_requests_recovery_total, counter, val(Info)); - couch_to_prom([fabric, read_repairs, failure], Info, _All) -> to_prom(fabric_read_repairs_failures_total, counter, val(Info)); couch_to_prom([fabric, read_repairs, success], Info, All) -> Total = val(Info) + val([fabric, read_repairs, failure], All), to_prom(fabric_read_repairs_total, counter, Total); - couch_to_prom([rexi, streams, timeout, init_stream], Info, _All) -> to_prom(rexi_streams_timeout_total, counter, {[{stage, init_stream}], val(Info)}); couch_to_prom([rexi_streams, timeout, Stage], Info, _All) -> to_prom(rexi_streams_timeout_total, {[{stage, Stage}], val(Info)}); - couch_to_prom([couchdb | Rest], Info, All) -> couch_to_prom(Rest, Info, All); - couch_to_prom(Path, Info, _All) -> case lists:keyfind(type, 1, Info) of {type, counter} -> @@ -94,16 +87,20 @@ to_prom(Metric, Type, Data) -> to_prom(Metric, Instances) when is_list(Instances) -> lists:flatmap(fun(Inst) -> to_prom(Metric, Inst) end, Instances); to_prom(Metric, {Labels, Value}) -> - LabelParts = lists:map(fun({K, V}) -> - lists:flatten(io_lib:format("~s=\"~s\"", [to_bin(K), to_bin(V)])) - end, Labels), - MetricStr = case length(LabelParts) > 0 of - true -> - LabelStr = string:join(LabelParts, ", "), - lists:flatten(io_lib:format("~s{~s}", [to_prom_name(Metric), LabelStr])); - false -> - lists:flatten(io_lib:format("~s", [to_prom_name(Metric)])) - end, + LabelParts = lists:map( + fun({K, V}) -> + lists:flatten(io_lib:format("~s=\"~s\"", [to_bin(K), to_bin(V)])) + end, + Labels + ), + MetricStr = + case length(LabelParts) > 0 of + true -> + LabelStr = string:join(LabelParts, ", "), + lists:flatten(io_lib:format("~s{~s}", [to_prom_name(Metric), LabelStr])); + false -> + lists:flatten(io_lib:format("~s", [to_prom_name(Metric)])) + end, [to_bin(io_lib:format("~s ~p", [MetricStr, Value]))]; to_prom(Metric, Value) -> [to_bin(io_lib:format("~s ~p", [to_prom_name(Metric), Value]))]. @@ -114,18 +111,21 @@ to_prom_summary(Path, Info) -> {arithmetic_mean, Mean} = lists:keyfind(arithmetic_mean, 1, Value), {percentile, Percentiles} = lists:keyfind(percentile, 1, Value), {n, Count} = lists:keyfind(n, 1, Value), - Quantiles = lists:map(fun({Perc, Val0}) -> - % Prometheus uses seconds, so we need to covert milliseconds to seconds - Val = Val0/1000, - case Perc of - 50 -> {[{quantile, <<"0.5">>}], Val}; - 75 -> {[{quantile, <<"0.75">>}], Val}; - 90 -> {[{quantile, <<"0.9">>}], Val}; - 95 -> {[{quantile, <<"0.95">>}], Val}; - 99 -> {[{quantile, <<"0.99">>}], Val}; - 999 -> {[{quantile, <<"0.999">>}], Val} - end - end, Percentiles), + Quantiles = lists:map( + fun({Perc, Val0}) -> + % Prometheus uses seconds, so we need to covert milliseconds to seconds + Val = Val0 / 1000, + case Perc of + 50 -> {[{quantile, <<"0.5">>}], Val}; + 75 -> {[{quantile, <<"0.75">>}], Val}; + 90 -> {[{quantile, <<"0.9">>}], Val}; + 95 -> {[{quantile, <<"0.95">>}], Val}; + 99 -> {[{quantile, <<"0.99">>}], Val}; + 999 -> {[{quantile, <<"0.999">>}], Val} + end + end, + Percentiles + ), SumMetric = path_to_name(Path ++ ["seconds", "sum"]), SumStat = to_prom(SumMetric, Count * Mean), CountMetric = path_to_name(Path ++ ["seconds", "count"]), @@ -136,9 +136,12 @@ to_prom_name(Metric) -> to_bin(io_lib:format("couchdb_~s", [Metric])). path_to_name(Path) -> - Parts = lists:map(fun(Part) -> - io_lib:format("~s", [Part]) - end, Path), + Parts = lists:map( + fun(Part) -> + io_lib:format("~s", [Part]) + end, + Path + ), string:join(Parts, "_"). counter_metric(Path) -> @@ -163,4 +166,4 @@ val(Data) -> val(Key, Stats) -> {Key, Data} = lists:keyfind(Key, 1, Stats), - val(Data).
\ No newline at end of file + val(Data). diff --git a/src/couch_prometheus/test/eunit/couch_prometheus_e2e_tests.erl b/src/couch_prometheus/test/eunit/couch_prometheus_e2e_tests.erl index 5b8adfd1d..f986fc6b0 100644 --- a/src/couch_prometheus/test/eunit/couch_prometheus_e2e_tests.erl +++ b/src/couch_prometheus/test/eunit/couch_prometheus_e2e_tests.erl @@ -26,7 +26,7 @@ start() -> setup() -> Hashed = couch_passwords:hash_admin_password(?PASS), - ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist=false), + ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist = false), ok = config:set_integer("stats", "interval", 2), ok = config:set_integer("couch_prometheus", "interval", 1), Port = mochiweb_socket_server:get(chttpd, port), @@ -40,10 +40,12 @@ couch_prometheus_e2e_test_() -> "Prometheus E2E Tests", { setup, - fun start/0, fun test_util:stop_couch/1, + fun start/0, + fun test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun node_call_chttpd/1, fun node_call_prometheus_http/1, @@ -56,11 +58,11 @@ couch_prometheus_e2e_test_() -> % normal chttpd path via cluster port node_call_chttpd(Url) -> - {ok, RC1, _, _} = test_request:get( - Url, - [?CONTENT_JSON, ?AUTH], - [] - ), + {ok, RC1, _, _} = test_request:get( + Url, + [?CONTENT_JSON, ?AUTH], + [] + ), ?_assertEqual(200, RC1). % normal chttpd path via cluster port @@ -84,14 +86,14 @@ node_call_prometheus_http(_) -> maybe_start_http_server("true"), Url = construct_url(?PROM_PORT), {ok, RC1, _, _} = test_request:get( - Url, - [?CONTENT_JSON, ?AUTH] - ), + Url, + [?CONTENT_JSON, ?AUTH] + ), % since this port doesn't require auth, this should work {ok, RC2, _, _} = test_request:get( - Url, - [?CONTENT_JSON] - ), + Url, + [?CONTENT_JSON] + ), delete_db(Url), ?_assertEqual({200, 200}, {RC1, RC2}). @@ -100,16 +102,16 @@ deny_prometheus_http(_) -> maybe_start_http_server("false"), Url = construct_url(?PROM_PORT), Response = test_request:get( - Url, - [?CONTENT_JSON, ?AUTH], - [] - ), - ?_assertEqual({error,{conn_failed,{error,econnrefused}}}, Response). + Url, + [?CONTENT_JSON, ?AUTH], + [] + ), + ?_assertEqual({error, {conn_failed, {error, econnrefused}}}, Response). maybe_start_http_server(Additional) -> test_util:stop_applications([couch_prometheus, chttpd]), Hashed = couch_passwords:hash_admin_password(?PASS), - ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist=false), + ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist = false), ok = config:set("prometheus", "additional_port", Additional), ok = config:set("prometheus", "port", ?PROM_PORT), test_util:start_applications([couch_prometheus, chttpd]). @@ -126,18 +128,24 @@ delete_db(Url) -> {ok, 200, _, _} = test_request:delete(Url, [?AUTH]). create_doc(Url, Id) -> - test_request:put(Url ++ "/" ++ Id, - [?CONTENT_JSON, ?AUTH], "{\"mr\": \"rockoartischocko\"}"). + test_request:put( + Url ++ "/" ++ Id, + [?CONTENT_JSON, ?AUTH], + "{\"mr\": \"rockoartischocko\"}" + ). wait_for_metrics(Url, Value, Timeout) -> - test_util:wait(fun() -> + test_util:wait( + fun() -> {ok, _, _, Body} = test_request:get( - Url, - [?CONTENT_JSON, ?AUTH], - [] - ), - case string:find(Body, Value) of - nomatch -> wait; - M -> M - end - end, Timeout). + Url, + [?CONTENT_JSON, ?AUTH], + [] + ), + case string:find(Body, Value) of + nomatch -> wait; + M -> M + end + end, + Timeout + ). diff --git a/src/couch_prometheus/test/eunit/couch_prometheus_util_tests.erl b/src/couch_prometheus/test/eunit/couch_prometheus_util_tests.erl index 8fe17e561..65828db62 100644 --- a/src/couch_prometheus/test/eunit/couch_prometheus_util_tests.erl +++ b/src/couch_prometheus/test/eunit/couch_prometheus_util_tests.erl @@ -21,39 +21,49 @@ couch_prometheus_util_test_() -> [ - ?_assertEqual(<<"couchdb_ddoc_cache 10">>, - test_to_prom_output(ddoc_cache, counter, 10)), - ?_assertEqual(<<"couchdb_httpd_status_codes{code=\"200\"} 3">>, - test_to_prom_output(httpd_status_codes, counter, {[{code, 200}], 3})), - ?_assertEqual(<<"couchdb_temperature_celsius 36">>, - test_to_prom_output(temperature_celsius, gauge, 36)), - ?_assertEqual(<<"couchdb_mango_query_time_seconds{quantile=\"0.75\"} 4.5">>, + ?_assertEqual( + <<"couchdb_ddoc_cache 10">>, + test_to_prom_output(ddoc_cache, counter, 10) + ), + ?_assertEqual( + <<"couchdb_httpd_status_codes{code=\"200\"} 3">>, + test_to_prom_output(httpd_status_codes, counter, {[{code, 200}], 3}) + ), + ?_assertEqual( + <<"couchdb_temperature_celsius 36">>, + test_to_prom_output(temperature_celsius, gauge, 36) + ), + ?_assertEqual( + <<"couchdb_mango_query_time_seconds{quantile=\"0.75\"} 4.5">>, test_to_prom_sum_output([mango_query_time], [ - {value, - [ - {min,0.0}, - {max,0.0}, - {arithmetic_mean,0.0}, - {geometric_mean,0.0}, - {harmonic_mean,0.0}, - {median,0.0},{variance,0.0}, - {standard_deviation,0.0}, - {skewness,0.0},{kurtosis,0.0}, - {percentile,[ - {50,0.0}, - {75, 4500}, - {90,0.0}, - {95,0.0}, - {99,0.0}, - {999,0.0}]}, - {histogram,[ - {0,0}]}, - {n,0} - ] - }, - {type,histogram}, + {value, [ + {min, 0.0}, + {max, 0.0}, + {arithmetic_mean, 0.0}, + {geometric_mean, 0.0}, + {harmonic_mean, 0.0}, + {median, 0.0}, + {variance, 0.0}, + {standard_deviation, 0.0}, + {skewness, 0.0}, + {kurtosis, 0.0}, + {percentile, [ + {50, 0.0}, + {75, 4500}, + {90, 0.0}, + {95, 0.0}, + {99, 0.0}, + {999, 0.0} + ]}, + {histogram, [ + {0, 0} + ]}, + {n, 0} + ]}, + {type, histogram}, {desc, <<"length of time processing a mango query">>} - ])) + ]) + ) ]. test_to_prom_output(Metric, Type, Val) -> @@ -62,4 +72,4 @@ test_to_prom_output(Metric, Type, Val) -> test_to_prom_sum_output(Metric, Info) -> Out = to_prom_summary(Metric, Info), - lists:nth(3, Out).
\ No newline at end of file + lists:nth(3, Out). diff --git a/src/couch_pse_tests/src/cpse_gather.erl b/src/couch_pse_tests/src/cpse_gather.erl index 7804d419e..346eca29b 100644 --- a/src/couch_pse_tests/src/cpse_gather.erl +++ b/src/couch_pse_tests/src/cpse_gather.erl @@ -12,12 +12,10 @@ -module(cpse_gather). - -export([ module/1 ]). - module(ModName) -> Exports = ModName:module_info(exports), @@ -26,15 +24,19 @@ module(ModName) -> SetupTest = get_fun(ModName, setup_each, 0, Exports), TeardownTest = get_fun(ModName, teardown_each, 1, Exports), - RevTests = lists:foldl(fun({Fun, Arity}, Acc) -> - case {atom_to_list(Fun), Arity} of - {[$c, $p, $s, $e, $_ | _], Arity} when Arity == 0; Arity == 1 -> - TestFun = make_test_fun(ModName, Fun, Arity), - [TestFun | Acc]; - _ -> - Acc - end - end, [], Exports), + RevTests = lists:foldl( + fun({Fun, Arity}, Acc) -> + case {atom_to_list(Fun), Arity} of + {[$c, $p, $s, $e, $_ | _], Arity} when Arity == 0; Arity == 1 -> + TestFun = make_test_fun(ModName, Fun, Arity), + [TestFun | Acc]; + _ -> + Acc + end + end, + [], + Exports + ), Tests = lists:reverse(RevTests), { @@ -52,21 +54,18 @@ module(ModName) -> ] }. - get_setup_all(ModName, Exports) -> case lists:member({setup_all, 0}, Exports) of true -> fun ModName:setup_all/0; false -> fun cpse_util:setup_all/0 end. - get_teardown_all(ModName, Exports) -> case lists:member({teardown_all, 1}, Exports) of true -> fun ModName:teardown_all/1; false -> fun cpse_util:teardown_all/1 end. - get_fun(ModName, FunName, Arity, Exports) -> case lists:member({FunName, Arity}, Exports) of true -> fun ModName:FunName/Arity; @@ -74,22 +73,23 @@ get_fun(ModName, FunName, Arity, Exports) -> false when Arity == 1 -> fun(_) -> ok end end. - make_test_fun(Module, Fun, Arity) -> Name = atom_to_list(Fun), case Arity of 0 -> fun(_) -> - {timeout, 60, {Name, fun() -> - process_flag(trap_exit, true), - Module:Fun() - end}} + {timeout, 60, + {Name, fun() -> + process_flag(trap_exit, true), + Module:Fun() + end}} end; 1 -> fun(Arg) -> - {timeout, 60, {Name, fun() -> - process_flag(trap_exit, true), - Module:Fun(Arg) - end}} + {timeout, 60, + {Name, fun() -> + process_flag(trap_exit, true), + Module:Fun(Arg) + end}} end end. diff --git a/src/couch_pse_tests/src/cpse_test_attachments.erl b/src/couch_pse_tests/src/cpse_test_attachments.erl index ddd1077d1..4447b8120 100644 --- a/src/couch_pse_tests/src/cpse_test_attachments.erl +++ b/src/couch_pse_tests/src/cpse_test_attachments.erl @@ -14,27 +14,23 @@ -compile(export_all). -compile(nowarn_export_all). - -include_lib("eunit/include/eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). - setup_each() -> {ok, Db} = cpse_util:create_db(), Db. - teardown_each(Db) -> ok = couch_server:delete(couch_db:name(Db), []). - cpse_write_attachment(Db1) -> AttBin = crypto:strong_rand_bytes(32768), try [Att0] = cpse_util:prep_atts(Db1, [ - {<<"ohai.txt">>, AttBin} - ]), + {<<"ohai.txt">>, AttBin} + ]), {stream, Stream} = couch_att:fetch(data, Att0), ?assertEqual(true, couch_db_engine:is_active_stream(Db1, Stream)), @@ -61,19 +57,21 @@ cpse_write_attachment(Db1) -> }, Doc1 = couch_db_engine:read_doc_body(Db3, Doc0), - Atts1 = if not is_binary(Doc1#doc.atts) -> Doc1#doc.atts; true -> - couch_compress:decompress(Doc1#doc.atts) - end, + Atts1 = + if + not is_binary(Doc1#doc.atts) -> Doc1#doc.atts; + true -> couch_compress:decompress(Doc1#doc.atts) + end, StreamSrc = fun(Sp) -> couch_db_engine:open_read_stream(Db3, Sp) end, [Att1] = [couch_att:from_disk_term(StreamSrc, T) || T <- Atts1], ReadBin = couch_att:to_binary(Att1), ?assertEqual(AttBin, ReadBin) - catch throw:not_supported -> - ok + catch + throw:not_supported -> + ok end. - % N.B. This test may be overly specific for some theoretical % storage engines that don't re-initialize their % attachments streams when restarting (for instance if @@ -84,8 +82,8 @@ cpse_inactive_stream(Db1) -> try [Att0] = cpse_util:prep_atts(Db1, [ - {<<"ohai.txt">>, AttBin} - ]), + {<<"ohai.txt">>, AttBin} + ]), {stream, Stream} = couch_att:fetch(data, Att0), ?assertEqual(true, couch_db_engine:is_active_stream(Db1, Stream)), @@ -94,6 +92,7 @@ cpse_inactive_stream(Db1) -> {ok, Db2} = couch_db:reopen(Db1), ?assertEqual(false, couch_db_engine:is_active_stream(Db2, Stream)) - catch throw:not_supported -> - ok + catch + throw:not_supported -> + ok end. diff --git a/src/couch_pse_tests/src/cpse_test_compaction.erl b/src/couch_pse_tests/src/cpse_test_compaction.erl index 6bc470b2f..3be95db60 100644 --- a/src/couch_pse_tests/src/cpse_test_compaction.erl +++ b/src/couch_pse_tests/src/cpse_test_compaction.erl @@ -14,20 +14,16 @@ -compile(export_all). -compile(nowarn_export_all). - -include_lib("eunit/include/eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). - setup_each() -> {ok, Db} = cpse_util:create_db(), Db. - teardown_each(Db) -> ok = couch_server:delete(couch_db:name(Db), []). - cpse_compact_empty(Db1) -> Term1 = cpse_util:db_as_term(Db1), @@ -39,7 +35,6 @@ cpse_compact_empty(Db1) -> Diff = cpse_util:term_diff(Term1, Term2), ?assertEqual(nodiff, Diff). - cpse_compact_doc(Db1) -> Actions = [{create, {<<"foo">>, {[]}}}], {ok, Db2} = cpse_util:apply_actions(Db1, Actions), @@ -53,7 +48,6 @@ cpse_compact_doc(Db1) -> Diff = cpse_util:term_diff(Term1, Term2), ?assertEqual(nodiff, Diff). - cpse_compact_local_doc(Db1) -> Actions = [{create, {<<"_local/foo">>, {[]}}}], {ok, Db2} = cpse_util:apply_actions(Db1, Actions), @@ -67,16 +61,21 @@ cpse_compact_local_doc(Db1) -> Diff = cpse_util:term_diff(Term1, Term2), ?assertEqual(nodiff, Diff). - cpse_compact_with_everything(Db1) -> % Add a whole bunch of docs - DocActions = lists:map(fun(Seq) -> - {create, {docid(Seq), {[{<<"int">>, Seq}]}}} - end, lists:seq(1, 1000)), - - LocalActions = lists:map(fun(I) -> - {create, {local_docid(I), {[{<<"int">>, I}]}}} - end, lists:seq(1, 25)), + DocActions = lists:map( + fun(Seq) -> + {create, {docid(Seq), {[{<<"int">>, Seq}]}}} + end, + lists:seq(1, 1000) + ), + + LocalActions = lists:map( + fun(I) -> + {create, {local_docid(I), {[{<<"int">>, I}]}}} + end, + lists:seq(1, 25) + ), Actions1 = DocActions ++ LocalActions, @@ -110,11 +109,13 @@ cpse_compact_with_everything(Db1) -> ], {ok, PIdRevs4} = couch_db_engine:fold_purge_infos( - Db4, 0, fun fold_fun/2, [], []), + Db4, 0, fun fold_fun/2, [], [] + ), ?assertEqual(PurgedIdRevs, PIdRevs4), - {ok, Db5} = try - [Att0, Att1, Att2, Att3, Att4] = cpse_util:prep_atts(Db4, [ + {ok, Db5} = + try + [Att0, Att1, Att2, Att3, Att4] = cpse_util:prep_atts(Db4, [ {<<"ohai.txt">>, crypto:strong_rand_bytes(2048)}, {<<"stuff.py">>, crypto:strong_rand_bytes(32768)}, {<<"a.erl">>, crypto:strong_rand_bytes(29)}, @@ -122,15 +123,16 @@ cpse_compact_with_everything(Db1) -> {<<"a.app">>, crypto:strong_rand_bytes(400)} ]), - Actions4 = [ - {create, {<<"small_att">>, {[]}, [Att0]}}, - {create, {<<"large_att">>, {[]}, [Att1]}}, - {create, {<<"multi_att">>, {[]}, [Att2, Att3, Att4]}} - ], - cpse_util:apply_actions(Db4, Actions4) - catch throw:not_supported -> - {ok, Db4} - end, + Actions4 = [ + {create, {<<"small_att">>, {[]}, [Att0]}}, + {create, {<<"large_att">>, {[]}, [Att1]}}, + {create, {<<"multi_att">>, {[]}, [Att2, Att3, Att4]}} + ], + cpse_util:apply_actions(Db4, Actions4) + catch + throw:not_supported -> + {ok, Db4} + end, {ok, Db6} = couch_db:reopen(Db5), Term1 = cpse_util:db_as_term(Db6), @@ -150,11 +152,13 @@ cpse_compact_with_everything(Db1) -> Diff = cpse_util:term_diff(Term1, Term2), ?assertEqual(nodiff, Diff). - cpse_recompact_updates(Db1) -> - Actions1 = lists:map(fun(Seq) -> - {create, {docid(Seq), {[{<<"int">>, Seq}]}}} - end, lists:seq(1, 1000)), + Actions1 = lists:map( + fun(Seq) -> + {create, {docid(Seq), {[{<<"int">>, Seq}]}}} + end, + lists:seq(1, 1000) + ), {ok, Db2} = cpse_util:apply_batch(Db1, Actions1), {ok, Compactor} = couch_db:start_compact(Db2), @@ -177,11 +181,13 @@ cpse_recompact_updates(Db1) -> Diff = cpse_util:term_diff(Term1, Term2), ?assertEqual(nodiff, Diff). - cpse_purge_during_compact(Db1) -> - Actions1 = lists:map(fun(Seq) -> - {create, {docid(Seq), {[{<<"int">>, Seq}]}}} - end, lists:seq(1, 1000)), + Actions1 = lists:map( + fun(Seq) -> + {create, {docid(Seq), {[{<<"int">>, Seq}]}}} + end, + lists:seq(1, 1000) + ), Actions2 = [ {create, {<<"foo">>, {[]}}}, {create, {<<"bar">>, {[]}}}, @@ -216,11 +222,13 @@ cpse_purge_during_compact(Db1) -> Diff = cpse_util:term_diff(Term1, Term2), ?assertEqual(nodiff, Diff). - cpse_multiple_purge_during_compact(Db1) -> - Actions1 = lists:map(fun(Seq) -> - {create, {docid(Seq), {[{<<"int">>, Seq}]}}} - end, lists:seq(1, 1000)), + Actions1 = lists:map( + fun(Seq) -> + {create, {docid(Seq), {[{<<"int">>, Seq}]}}} + end, + lists:seq(1, 1000) + ), Actions2 = [ {create, {<<"foo">>, {[]}}}, {create, {<<"bar">>, {[]}}}, @@ -233,7 +241,6 @@ cpse_multiple_purge_during_compact(Db1) -> ], {ok, Db3} = cpse_util:apply_actions(Db2, Actions3), - {ok, Pid} = couch_db:start_compact(Db3), catch erlang:suspend_process(Pid), @@ -261,31 +268,39 @@ cpse_multiple_purge_during_compact(Db1) -> Diff = cpse_util:term_diff(Term1, Term2), ?assertEqual(nodiff, Diff). - cpse_compact_purged_docs_limit(Db1) -> NumDocs = 1200, - {RActions, RIds} = lists:foldl(fun(Id, {CActions, CIds}) -> - Id1 = docid(Id), - Action = {create, {Id1, {[{<<"int">>, Id}]}}}, - {[Action| CActions], [Id1| CIds]} - end, {[], []}, lists:seq(1, NumDocs)), + {RActions, RIds} = lists:foldl( + fun(Id, {CActions, CIds}) -> + Id1 = docid(Id), + Action = {create, {Id1, {[{<<"int">>, Id}]}}}, + {[Action | CActions], [Id1 | CIds]} + end, + {[], []}, + lists:seq(1, NumDocs) + ), Ids = lists:reverse(RIds), {ok, Db2} = cpse_util:apply_batch(Db1, lists:reverse(RActions)), FDIs = couch_db_engine:open_docs(Db2, Ids), - RActions2 = lists:foldl(fun(FDI, CActions) -> - Id = FDI#full_doc_info.id, - PrevRev = cpse_util:prev_rev(FDI), - Rev = PrevRev#rev_info.rev, - [{purge, {Id, Rev}}| CActions] - end, [], FDIs), + RActions2 = lists:foldl( + fun(FDI, CActions) -> + Id = FDI#full_doc_info.id, + PrevRev = cpse_util:prev_rev(FDI), + Rev = PrevRev#rev_info.rev, + [{purge, {Id, Rev}} | CActions] + end, + [], + FDIs + ), {ok, Db3} = cpse_util:apply_batch(Db2, lists:reverse(RActions2)), % check that before compaction all NumDocs of purge_requests % are in purge_tree, % even if NumDocs=1200 is greater than purged_docs_limit=1000 {ok, PurgedIdRevs} = couch_db_engine:fold_purge_infos( - Db3, 0, fun fold_fun/2, [], []), + Db3, 0, fun fold_fun/2, [], [] + ), ?assertEqual(1, couch_db_engine:get_oldest_purge_seq(Db3)), ?assertEqual(NumDocs, length(PurgedIdRevs)), @@ -298,21 +313,19 @@ cpse_compact_purged_docs_limit(Db1) -> PurgedDocsLimit = couch_db_engine:get_purge_infos_limit(Db4), OldestPSeq = couch_db_engine:get_oldest_purge_seq(Db4), {ok, PurgedIdRevs2} = couch_db_engine:fold_purge_infos( - Db4, OldestPSeq - 1, fun fold_fun/2, [], []), + Db4, OldestPSeq - 1, fun fold_fun/2, [], [] + ), ExpectedOldestPSeq = NumDocs - PurgedDocsLimit + 1, ?assertEqual(ExpectedOldestPSeq, OldestPSeq), ?assertEqual(PurgedDocsLimit, length(PurgedIdRevs2)). - docid(I) -> Str = io_lib:format("~4..0b", [I]), iolist_to_binary(Str). - local_docid(I) -> Str = io_lib:format("_local/~4..0b", [I]), iolist_to_binary(Str). - fold_fun({_PSeq, _UUID, Id, Revs}, Acc) -> {ok, [{Id, Revs} | Acc]}. diff --git a/src/couch_pse_tests/src/cpse_test_copy_purge_infos.erl b/src/couch_pse_tests/src/cpse_test_copy_purge_infos.erl index 4e41430d3..a32f866b4 100644 --- a/src/couch_pse_tests/src/cpse_test_copy_purge_infos.erl +++ b/src/couch_pse_tests/src/cpse_test_copy_purge_infos.erl @@ -14,35 +14,35 @@ -compile(export_all). -compile(nowarn_export_all). - -include_lib("eunit/include/eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). - -define(NUM_DOCS, 100). - setup_each() -> {ok, SrcDb} = cpse_util:create_db(), {ok, SrcDb2} = create_and_purge(SrcDb), {ok, TrgDb} = cpse_util:create_db(), {SrcDb2, TrgDb}. - teardown_each({SrcDb, TrgDb}) -> ok = couch_server:delete(couch_db:name(SrcDb), []), ok = couch_server:delete(couch_db:name(TrgDb), []). - cpse_copy_empty_purged_info({_, Db}) -> {ok, Db1} = couch_db_engine:copy_purge_infos(Db, []), ?assertEqual(ok, cpse_util:assert_each_prop(Db1, [{purge_infos, []}])). - cpse_copy_purged_info({SrcDb, TrgDb}) -> - {ok, RPIs} = couch_db_engine:fold_purge_infos(SrcDb, 0, fun(PI, Acc) -> - {ok, [PI | Acc]} - end, [], []), + {ok, RPIs} = couch_db_engine:fold_purge_infos( + SrcDb, + 0, + fun(PI, Acc) -> + {ok, [PI | Acc]} + end, + [], + [] + ), PIs = lists:reverse(RPIs), AEPFold = fun({PSeq, UUID, Id, Revs}, {CPSeq, CPurges}) -> {max(PSeq, CPSeq), [{UUID, Id, Revs} | CPurges]} @@ -53,30 +53,36 @@ cpse_copy_purged_info({SrcDb, TrgDb}) -> AssertProps = [{purge_seq, PurgeSeq}, {purge_infos, Purges}], ?assertEqual(ok, cpse_util:assert_each_prop(TrgDb2, AssertProps)). - create_and_purge(Db) -> - {RActions, RIds} = lists:foldl(fun(Id, {CActions, CIds}) -> - Id1 = docid(Id), - Action = {create, {Id1, {[{<<"int">>, Id}]}}}, - {[Action| CActions], [Id1| CIds]} - end, {[], []}, lists:seq(1, ?NUM_DOCS)), + {RActions, RIds} = lists:foldl( + fun(Id, {CActions, CIds}) -> + Id1 = docid(Id), + Action = {create, {Id1, {[{<<"int">>, Id}]}}}, + {[Action | CActions], [Id1 | CIds]} + end, + {[], []}, + lists:seq(1, ?NUM_DOCS) + ), Actions = lists:reverse(RActions), Ids = lists:reverse(RIds), {ok, Db1} = cpse_util:apply_batch(Db, Actions), FDIs = couch_db_engine:open_docs(Db1, Ids), - RActions2 = lists:foldl(fun(FDI, CActions) -> - Id = FDI#full_doc_info.id, - PrevRev = cpse_util:prev_rev(FDI), - Rev = PrevRev#rev_info.rev, - Action = {purge, {Id, Rev}}, - [Action| CActions] - end, [], FDIs), + RActions2 = lists:foldl( + fun(FDI, CActions) -> + Id = FDI#full_doc_info.id, + PrevRev = cpse_util:prev_rev(FDI), + Rev = PrevRev#rev_info.rev, + Action = {purge, {Id, Rev}}, + [Action | CActions] + end, + [], + FDIs + ), Actions2 = lists:reverse(RActions2), {ok, Db2} = cpse_util:apply_batch(Db1, Actions2), {ok, Db2}. - docid(I) -> Str = io_lib:format("~4..0b", [I]), iolist_to_binary(Str). diff --git a/src/couch_pse_tests/src/cpse_test_fold_changes.erl b/src/couch_pse_tests/src/cpse_test_fold_changes.erl index 436396276..91f7c63e9 100644 --- a/src/couch_pse_tests/src/cpse_test_fold_changes.erl +++ b/src/couch_pse_tests/src/cpse_test_fold_changes.erl @@ -14,37 +14,34 @@ -compile(export_all). -compile(nowarn_export_all). - -include_lib("eunit/include/eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). - -define(NUM_DOCS, 25). - setup_each() -> {ok, Db} = cpse_util:create_db(), Db. - teardown_each(Db) -> ok = couch_server:delete(couch_db:name(Db), []). - cpse_empty_changes(Db) -> ?assertEqual(0, couch_db_engine:count_changes_since(Db, 0)), - ?assertEqual({ok, []}, - couch_db_engine:fold_changes(Db, 0, fun fold_fun/2, [], [])). - + ?assertEqual( + {ok, []}, + couch_db_engine:fold_changes(Db, 0, fun fold_fun/2, [], []) + ). cpse_single_change(Db1) -> Actions = [{create, {<<"a">>, {[]}}}], {ok, Db2} = cpse_util:apply_actions(Db1, Actions), ?assertEqual(1, couch_db_engine:count_changes_since(Db2, 0)), - ?assertEqual({ok, [{<<"a">>, 1}]}, - couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], [])). - + ?assertEqual( + {ok, [{<<"a">>, 1}]}, + couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], []) + ). cpse_two_changes(Db1) -> Actions = [ @@ -55,10 +52,9 @@ cpse_two_changes(Db1) -> ?assertEqual(2, couch_db_engine:count_changes_since(Db2, 0)), {ok, Changes} = - couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], []), + couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], []), ?assertEqual([{<<"a">>, 1}, {<<"b">>, 2}], lists:reverse(Changes)). - cpse_two_changes_batch(Db1) -> Actions = [ {batch, [ @@ -70,10 +66,9 @@ cpse_two_changes_batch(Db1) -> ?assertEqual(2, couch_db_engine:count_changes_since(Db2, 0)), {ok, Changes} = - couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], []), + couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], []), ?assertEqual([{<<"a">>, 1}, {<<"b">>, 2}], lists:reverse(Changes)). - cpse_two_changes_batch_sorted(Db1) -> Actions = [ {batch, [ @@ -85,10 +80,9 @@ cpse_two_changes_batch_sorted(Db1) -> ?assertEqual(2, couch_db_engine:count_changes_since(Db2, 0)), {ok, Changes} = - couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], []), + couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], []), ?assertEqual([{<<"a">>, 1}, {<<"b">>, 2}], lists:reverse(Changes)). - cpse_update_one(Db1) -> Actions = [ {create, {<<"a">>, {[]}}}, @@ -97,9 +91,10 @@ cpse_update_one(Db1) -> {ok, Db2} = cpse_util:apply_actions(Db1, Actions), ?assertEqual(1, couch_db_engine:count_changes_since(Db2, 0)), - ?assertEqual({ok, [{<<"a">>, 2}]}, - couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], [])). - + ?assertEqual( + {ok, [{<<"a">>, 2}]}, + couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], []) + ). cpse_update_first_of_two(Db1) -> Actions = [ @@ -111,10 +106,9 @@ cpse_update_first_of_two(Db1) -> ?assertEqual(2, couch_db_engine:count_changes_since(Db2, 0)), {ok, Changes} = - couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], []), + couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], []), ?assertEqual([{<<"b">>, 2}, {<<"a">>, 3}], lists:reverse(Changes)). - cpse_update_second_of_two(Db1) -> Actions = [ {create, {<<"a">>, {[]}}}, @@ -125,14 +119,18 @@ cpse_update_second_of_two(Db1) -> ?assertEqual(2, couch_db_engine:count_changes_since(Db2, 0)), {ok, Changes} = - couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], []), + couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], []), ?assertEqual([{<<"a">>, 1}, {<<"b">>, 3}], lists:reverse(Changes)). - cpse_check_mutation_ordering(Db1) -> - Actions = shuffle(lists:map(fun(Seq) -> - {create, {docid(Seq), {[]}}} - end, lists:seq(1, ?NUM_DOCS))), + Actions = shuffle( + lists:map( + fun(Seq) -> + {create, {docid(Seq), {[]}}} + end, + lists:seq(1, ?NUM_DOCS) + ) + ), DocIdOrder = [DocId || {_, {DocId, _}} <- Actions], DocSeqs = lists:zip(DocIdOrder, lists:seq(1, ?NUM_DOCS)), @@ -141,45 +139,44 @@ cpse_check_mutation_ordering(Db1) -> % First lets see that we can get the correct % suffix/prefix starting at every update sequence - lists:foreach(fun(Seq) -> - {ok, Suffix} = + lists:foreach( + fun(Seq) -> + {ok, Suffix} = couch_db_engine:fold_changes(Db2, Seq, fun fold_fun/2, [], []), - ?assertEqual(lists:nthtail(Seq, DocSeqs), lists:reverse(Suffix)), + ?assertEqual(lists:nthtail(Seq, DocSeqs), lists:reverse(Suffix)), - {ok, Prefix} = couch_db_engine:fold_changes( - Db2, Seq, fun fold_fun/2, [], [{dir, rev}]), - ?assertEqual(lists:sublist(DocSeqs, Seq + 1), Prefix) - end, lists:seq(0, ?NUM_DOCS)), + {ok, Prefix} = couch_db_engine:fold_changes( + Db2, Seq, fun fold_fun/2, [], [{dir, rev}] + ), + ?assertEqual(lists:sublist(DocSeqs, Seq + 1), Prefix) + end, + lists:seq(0, ?NUM_DOCS) + ), ok = do_mutation_ordering(Db2, ?NUM_DOCS + 1, DocSeqs, []). - do_mutation_ordering(Db, _Seq, [], FinalDocSeqs) -> {ok, RevOrder} = couch_db_engine:fold_changes(Db, 0, fun fold_fun/2, [], []), ?assertEqual(FinalDocSeqs, lists:reverse(RevOrder)), ok; - do_mutation_ordering(Db, Seq, [{DocId, _OldSeq} | Rest], DocSeqAcc) -> Actions = [{update, {DocId, {[]}}}], {ok, NewDb} = cpse_util:apply_actions(Db, Actions), NewAcc = DocSeqAcc ++ [{DocId, Seq}], Expected = Rest ++ NewAcc, {ok, RevOrder} = - couch_db_engine:fold_changes(NewDb, 0, fun fold_fun/2, [], []), + couch_db_engine:fold_changes(NewDb, 0, fun fold_fun/2, [], []), ?assertEqual(Expected, lists:reverse(RevOrder)), do_mutation_ordering(NewDb, Seq + 1, Rest, NewAcc). - shuffle(List) -> Paired = [{couch_rand:uniform(), I} || I <- List], Sorted = lists:sort(Paired), [I || {_, I} <- Sorted]. - -fold_fun(#full_doc_info{id=Id, update_seq=Seq}, Acc) -> +fold_fun(#full_doc_info{id = Id, update_seq = Seq}, Acc) -> {ok, [{Id, Seq} | Acc]}. - docid(I) -> Str = io_lib:format("~4..0b", [I]), iolist_to_binary(Str). diff --git a/src/couch_pse_tests/src/cpse_test_fold_docs.erl b/src/couch_pse_tests/src/cpse_test_fold_docs.erl index d43930c4a..2d6eb7a9d 100644 --- a/src/couch_pse_tests/src/cpse_test_fold_docs.erl +++ b/src/couch_pse_tests/src/cpse_test_fold_docs.erl @@ -14,70 +14,53 @@ -compile(export_all). -compile(nowarn_export_all). - -include_lib("eunit/include/eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). - -define(NUM_DOCS, 100). - setup_each() -> cpse_util:dbname(). - teardown_each(DbName) -> ok = couch_server:delete(DbName, []). - cpse_fold_all(DbName) -> fold_all(DbName, fold_docs, fun docid/1). - cpse_fold_all_local(DbName) -> fold_all(DbName, fold_local_docs, fun local_docid/1). - cpse_fold_start_key(DbName) -> fold_start_key(DbName, fold_docs, fun docid/1). - cpse_fold_start_key_local(DbName) -> fold_start_key(DbName, fold_local_docs, fun local_docid/1). - cpse_fold_end_key(DbName) -> fold_end_key(DbName, fold_docs, fun docid/1). - cpse_fold_end_key_local(DbName) -> fold_end_key(DbName, fold_local_docs, fun local_docid/1). - cpse_fold_end_key_gt(DbName) -> fold_end_key_gt(DbName, fold_docs, fun docid/1). - cpse_fold_end_key_gt_local(DbName) -> fold_end_key_gt(DbName, fold_local_docs, fun local_docid/1). - cpse_fold_range(DbName) -> fold_range(DbName, fold_docs, fun docid/1). - cpse_fold_range_local(DbName) -> fold_range(DbName, fold_local_docs, fun local_docid/1). - cpse_fold_stop(DbName) -> fold_user_fun_stop(DbName, fold_docs, fun docid/1). - cpse_fold_stop_local(DbName) -> fold_user_fun_stop(DbName, fold_local_docs, fun local_docid/1). - % This is a loose test but we have to have this until % I figure out what to do about the total_rows/offset % meta data included in _all_docs @@ -89,7 +72,6 @@ cpse_fold_include_reductions(DbName) -> ?assert(is_integer(Count)), ?assert(Count >= 0). - fold_all(DbName, FoldFun, DocIdFun) -> DocIds = [DocIdFun(I) || I <- lists:seq(1, ?NUM_DOCS)], {ok, Db} = init_db(DbName, DocIdFun), @@ -103,7 +85,6 @@ fold_all(DbName, FoldFun, DocIdFun) -> ?assertEqual(?NUM_DOCS, length(DocIdAccRev)), ?assertEqual(DocIds, DocIdAccRev). - fold_start_key(DbName, FoldFun, DocIdFun) -> {ok, Db} = init_db(DbName, DocIdFun), @@ -114,138 +95,153 @@ fold_start_key(DbName, FoldFun, DocIdFun) -> DocIdsFwd = [DocIdFun(I) || I <- lists:seq(StartKeyNum, ?NUM_DOCS)], DocIdsRev = [DocIdFun(I) || I <- lists:seq(1, StartKeyNum)], - ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ + ?assertEqual( + {ok, []}, + couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {start_key, <<255>>} - ])), + ]) + ), - ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ + ?assertEqual( + {ok, []}, + couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {start_key, <<"">>} - ])), + ]) + ), {ok, AllDocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ - {start_key, <<"">>} - ]), + {start_key, <<"">>} + ]), ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), ?assertEqual(AllDocIds, lists:reverse(AllDocIdAccFwd)), {ok, AllDocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ - {dir, rev}, - {start_key, <<255>>} - ]), + {dir, rev}, + {start_key, <<255>>} + ]), ?assertEqual(length(AllDocIds), length(AllDocIdAccRev)), ?assertEqual(AllDocIds, AllDocIdAccRev), {ok, DocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ - {start_key, StartKey} - ]), + {start_key, StartKey} + ]), ?assertEqual(length(DocIdsFwd), length(DocIdAccFwd)), ?assertEqual(DocIdsFwd, lists:reverse(DocIdAccFwd)), {ok, DocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ - {dir, rev}, - {start_key, StartKey} - ]), + {dir, rev}, + {start_key, StartKey} + ]), ?assertEqual(length(DocIdsRev), length(DocIdAccRev)), ?assertEqual(DocIdsRev, DocIdAccRev). - fold_end_key(DbName, FoldFun, DocIdFun) -> {ok, Db} = init_db(DbName, DocIdFun), EndKeyNum = ?NUM_DOCS div 4, EndKey = DocIdFun(EndKeyNum), - ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ + ?assertEqual( + {ok, []}, + couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {end_key, <<"">>} - ])), + ]) + ), - ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ + ?assertEqual( + {ok, []}, + couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {end_key, <<255>>} - ])), + ]) + ), AllDocIds = [DocIdFun(I) || I <- lists:seq(1, ?NUM_DOCS)], {ok, AllDocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ - {end_key, <<255>>} - ]), + {end_key, <<255>>} + ]), ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), ?assertEqual(AllDocIds, lists:reverse(AllDocIdAccFwd)), {ok, AllDocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ - {dir, rev}, - {end_key, <<"">>} - ]), + {dir, rev}, + {end_key, <<"">>} + ]), ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), ?assertEqual(AllDocIds, AllDocIdAccRev), DocIdsFwd = [DocIdFun(I) || I <- lists:seq(1, EndKeyNum)], {ok, DocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ - {end_key, EndKey} - ]), + {end_key, EndKey} + ]), ?assertEqual(length(DocIdsFwd), length(DocIdAccFwd)), ?assertEqual(DocIdsFwd, lists:reverse(DocIdAccFwd)), DocIdsRev = [DocIdFun(I) || I <- lists:seq(EndKeyNum, ?NUM_DOCS)], {ok, DocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ - {dir, rev}, - {end_key, EndKey} - ]), + {dir, rev}, + {end_key, EndKey} + ]), ?assertEqual(length(DocIdsRev), length(DocIdAccRev)), ?assertEqual(DocIdsRev, DocIdAccRev). - fold_end_key_gt(DbName, FoldFun, DocIdFun) -> {ok, Db} = init_db(DbName, DocIdFun), EndKeyNum = ?NUM_DOCS div 4, EndKey = DocIdFun(EndKeyNum), - ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ + ?assertEqual( + {ok, []}, + couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {end_key_gt, <<"">>} - ])), + ]) + ), - ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ + ?assertEqual( + {ok, []}, + couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {end_key_gt, <<255>>} - ])), + ]) + ), AllDocIds = [DocIdFun(I) || I <- lists:seq(1, ?NUM_DOCS)], {ok, AllDocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ - {end_key_gt, <<255>>} - ]), + {end_key_gt, <<255>>} + ]), ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), ?assertEqual(AllDocIds, lists:reverse(AllDocIdAccFwd)), {ok, AllDocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ - {dir, rev}, - {end_key_gt, <<"">>} - ]), + {dir, rev}, + {end_key_gt, <<"">>} + ]), ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), ?assertEqual(AllDocIds, AllDocIdAccRev), DocIdsFwd = [DocIdFun(I) || I <- lists:seq(1, EndKeyNum - 1)], {ok, DocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ - {end_key_gt, EndKey} - ]), + {end_key_gt, EndKey} + ]), ?assertEqual(length(DocIdsFwd), length(DocIdAccFwd)), ?assertEqual(DocIdsFwd, lists:reverse(DocIdAccFwd)), DocIdsRev = [DocIdFun(I) || I <- lists:seq(EndKeyNum + 1, ?NUM_DOCS)], {ok, DocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ - {dir, rev}, - {end_key_gt, EndKey} - ]), + {dir, rev}, + {end_key_gt, EndKey} + ]), ?assertEqual(length(DocIdsRev), length(DocIdAccRev)), ?assertEqual(DocIdsRev, DocIdAccRev). - fold_range(DbName, FoldFun, DocIdFun) -> {ok, Db} = init_db(DbName, DocIdFun), @@ -255,133 +251,153 @@ fold_range(DbName, FoldFun, DocIdFun) -> StartKey = DocIdFun(StartKeyNum), EndKey = DocIdFun(EndKeyNum), - ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ + ?assertEqual( + {ok, []}, + couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {start_key, <<"">>}, {end_key, <<"">>} - ])), + ]) + ), - ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ + ?assertEqual( + {ok, []}, + couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {start_key, <<"">>}, {end_key, <<255>>} - ])), + ]) + ), AllDocIds = [DocIdFun(I) || I <- lists:seq(1, ?NUM_DOCS)], {ok, AllDocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ - {start_key, <<"">>}, - {end_key, <<255>>} - ]), + {start_key, <<"">>}, + {end_key, <<255>>} + ]), ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), ?assertEqual(AllDocIds, lists:reverse(AllDocIdAccFwd)), {ok, AllDocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ - {dir, rev}, - {start_key, <<255>>}, - {end_key_gt, <<"">>} - ]), + {dir, rev}, + {start_key, <<255>>}, + {end_key_gt, <<"">>} + ]), ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), ?assertEqual(AllDocIds, AllDocIdAccRev), DocIdsFwd = [DocIdFun(I) || I <- lists:seq(StartKeyNum, EndKeyNum)], {ok, DocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ - {start_key, StartKey}, - {end_key, EndKey} - ]), + {start_key, StartKey}, + {end_key, EndKey} + ]), ?assertEqual(length(DocIdsFwd), length(DocIdAccFwd)), ?assertEqual(DocIdsFwd, lists:reverse(DocIdAccFwd)), DocIdsRev = [DocIdFun(I) || I <- lists:seq(StartKeyNum, EndKeyNum)], - ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ + ?assertEqual( + {ok, []}, + couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {start_key, StartKey}, {end_key, EndKey} - ])), + ]) + ), {ok, DocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ - {dir, rev}, - {start_key, EndKey}, - {end_key, StartKey} - ]), + {dir, rev}, + {start_key, EndKey}, + {end_key, StartKey} + ]), ?assertEqual(length(DocIdsRev), length(DocIdAccRev)), ?assertEqual(DocIdsRev, DocIdAccRev). - fold_user_fun_stop(DbName, FoldFun, DocIdFun) -> {ok, Db} = init_db(DbName, DocIdFun), StartKeyNum = ?NUM_DOCS div 4, StartKey = DocIdFun(StartKeyNum), - ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_stop/2, [], [ + ?assertEqual( + {ok, []}, + couch_db_engine:FoldFun(Db, fun fold_stop/2, [], [ {start_key, <<255>>} - ])), + ]) + ), - ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_stop/2, [], [ + ?assertEqual( + {ok, []}, + couch_db_engine:FoldFun(Db, fun fold_stop/2, [], [ {dir, rev}, {start_key, <<"">>} - ])), + ]) + ), SuffixDocIds = [DocIdFun(I) || I <- lists:seq(?NUM_DOCS - 3, ?NUM_DOCS)], {ok, SuffixDocIdAcc} = couch_db_engine:FoldFun(Db, fun fold_stop/2, [], [ - {start_key, DocIdFun(?NUM_DOCS - 3)} - ]), + {start_key, DocIdFun(?NUM_DOCS - 3)} + ]), ?assertEqual(length(SuffixDocIds), length(SuffixDocIdAcc)), ?assertEqual(SuffixDocIds, lists:reverse(SuffixDocIdAcc)), PrefixDocIds = [DocIdFun(I) || I <- lists:seq(1, 3)], {ok, PrefixDocIdAcc} = couch_db_engine:FoldFun(Db, fun fold_stop/2, [], [ - {dir, rev}, - {start_key, DocIdFun(3)} - ]), + {dir, rev}, + {start_key, DocIdFun(3)} + ]), ?assertEqual(3, length(PrefixDocIdAcc)), ?assertEqual(PrefixDocIds, PrefixDocIdAcc), - FiveDocIdsFwd = [DocIdFun(I) - || I <- lists:seq(StartKeyNum, StartKeyNum + 5)], + FiveDocIdsFwd = [ + DocIdFun(I) + || I <- lists:seq(StartKeyNum, StartKeyNum + 5) + ], {ok, FiveDocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_stop/2, [], [ - {start_key, StartKey} - ]), + {start_key, StartKey} + ]), ?assertEqual(length(FiveDocIdsFwd), length(FiveDocIdAccFwd)), ?assertEqual(FiveDocIdsFwd, lists:reverse(FiveDocIdAccFwd)), - FiveDocIdsRev = [DocIdFun(I) - || I <- lists:seq(StartKeyNum - 5, StartKeyNum)], + FiveDocIdsRev = [ + DocIdFun(I) + || I <- lists:seq(StartKeyNum - 5, StartKeyNum) + ], {ok, FiveDocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_stop/2, [], [ - {dir, rev}, - {start_key, StartKey} - ]), + {dir, rev}, + {start_key, StartKey} + ]), ?assertEqual(length(FiveDocIdsRev), length(FiveDocIdAccRev)), ?assertEqual(FiveDocIdsRev, FiveDocIdAccRev). - init_db(DbName, DocIdFun) -> {ok, Db1} = cpse_util:create_db(DbName), - Actions = lists:map(fun(Id) -> - {create, {DocIdFun(Id), {[{<<"int">>, Id}]}}} - end, lists:seq(1, ?NUM_DOCS)), + Actions = lists:map( + fun(Id) -> + {create, {DocIdFun(Id), {[{<<"int">>, Id}]}}} + end, + lists:seq(1, ?NUM_DOCS) + ), cpse_util:apply_actions(Db1, [{batch, Actions}]). - fold_fun(Doc, Acc) -> - Id = case Doc of - #doc{id = Id0} -> Id0; - #full_doc_info{id = Id0} -> Id0 - end, + Id = + case Doc of + #doc{id = Id0} -> Id0; + #full_doc_info{id = Id0} -> Id0 + end, {ok, [Id | Acc]}. - fold_stop(Doc, Acc) -> - Id = case Doc of - #doc{id = Id0} -> Id0; - #full_doc_info{id = Id0} -> Id0 - end, + Id = + case Doc of + #doc{id = Id0} -> Id0; + #full_doc_info{id = Id0} -> Id0 + end, case length(Acc) of N when N =< 4 -> {ok, [Id | Acc]}; @@ -389,12 +405,10 @@ fold_stop(Doc, Acc) -> {stop, [Id | Acc]} end. - docid(I) -> Str = io_lib:format("~4..0b", [I]), iolist_to_binary(Str). - local_docid(I) -> Str = io_lib:format("_local/~4..0b", [I]), iolist_to_binary(Str). diff --git a/src/couch_pse_tests/src/cpse_test_fold_purge_infos.erl b/src/couch_pse_tests/src/cpse_test_fold_purge_infos.erl index 4826c5d9c..6225cbdb0 100644 --- a/src/couch_pse_tests/src/cpse_test_fold_purge_infos.erl +++ b/src/couch_pse_tests/src/cpse_test_fold_purge_infos.erl @@ -14,54 +14,60 @@ -compile(export_all). -compile(nowarn_export_all). - -include_lib("eunit/include/eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). - -define(NUM_DOCS, 100). - setup_each() -> {ok, Db} = cpse_util:create_db(), Db. - teardown_each(Db) -> ok = couch_server:delete(couch_db:name(Db), []). - cpse_empty_purged_docs(Db) -> - ?assertEqual({ok, []}, couch_db_engine:fold_purge_infos( - Db, 0, fun fold_fun/2, [], [])). - + ?assertEqual( + {ok, []}, + couch_db_engine:fold_purge_infos( + Db, 0, fun fold_fun/2, [], [] + ) + ). cpse_all_purged_docs(Db1) -> - {RActions, RIds} = lists:foldl(fun(Id, {CActions, CIds}) -> - Id1 = docid(Id), - Action = {create, {Id1, {[{<<"int">>, Id}]}}}, - {[Action| CActions], [Id1| CIds]} - end, {[], []}, lists:seq(1, ?NUM_DOCS)), + {RActions, RIds} = lists:foldl( + fun(Id, {CActions, CIds}) -> + Id1 = docid(Id), + Action = {create, {Id1, {[{<<"int">>, Id}]}}}, + {[Action | CActions], [Id1 | CIds]} + end, + {[], []}, + lists:seq(1, ?NUM_DOCS) + ), Actions = lists:reverse(RActions), Ids = lists:reverse(RIds), {ok, Db2} = cpse_util:apply_batch(Db1, Actions), FDIs = couch_db_engine:open_docs(Db2, Ids), - {RevActions2, RevIdRevs} = lists:foldl(fun(FDI, {CActions, CIdRevs}) -> - Id = FDI#full_doc_info.id, - PrevRev = cpse_util:prev_rev(FDI), - Rev = PrevRev#rev_info.rev, - Action = {purge, {Id, Rev}}, - {[Action| CActions], [{Id, [Rev]}| CIdRevs]} - end, {[], []}, FDIs), + {RevActions2, RevIdRevs} = lists:foldl( + fun(FDI, {CActions, CIdRevs}) -> + Id = FDI#full_doc_info.id, + PrevRev = cpse_util:prev_rev(FDI), + Rev = PrevRev#rev_info.rev, + Action = {purge, {Id, Rev}}, + {[Action | CActions], [{Id, [Rev]} | CIdRevs]} + end, + {[], []}, + FDIs + ), {Actions2, IdsRevs} = {lists:reverse(RevActions2), lists:reverse(RevIdRevs)}, {ok, Db3} = cpse_util:apply_batch(Db2, Actions2), {ok, PurgedIdRevs} = couch_db_engine:fold_purge_infos( - Db3, 0, fun fold_fun/2, [], []), + Db3, 0, fun fold_fun/2, [], [] + ), ?assertEqual(IdsRevs, lists:reverse(PurgedIdRevs)). - cpse_start_seq(Db1) -> Actions1 = [ {create, {docid(1), {[{<<"int">>, 1}]}}}, @@ -74,22 +80,26 @@ cpse_start_seq(Db1) -> {ok, Db2} = cpse_util:apply_actions(Db1, Actions1), FDIs = couch_db_engine:open_docs(Db2, Ids), - {RActions2, RIdRevs} = lists:foldl(fun(FDI, {CActions, CIdRevs}) -> - Id = FDI#full_doc_info.id, - PrevRev = cpse_util:prev_rev(FDI), - Rev = PrevRev#rev_info.rev, - Action = {purge, {Id, Rev}}, - {[Action| CActions], [{Id, [Rev]}| CIdRevs]} - end, {[], []}, FDIs), + {RActions2, RIdRevs} = lists:foldl( + fun(FDI, {CActions, CIdRevs}) -> + Id = FDI#full_doc_info.id, + PrevRev = cpse_util:prev_rev(FDI), + Rev = PrevRev#rev_info.rev, + Action = {purge, {Id, Rev}}, + {[Action | CActions], [{Id, [Rev]} | CIdRevs]} + end, + {[], []}, + FDIs + ), {ok, Db3} = cpse_util:apply_actions(Db2, lists:reverse(RActions2)), StartSeq = 3, StartSeqIdRevs = lists:nthtail(StartSeq, lists:reverse(RIdRevs)), {ok, PurgedIdRevs} = couch_db_engine:fold_purge_infos( - Db3, StartSeq, fun fold_fun/2, [], []), + Db3, StartSeq, fun fold_fun/2, [], [] + ), ?assertEqual(StartSeqIdRevs, lists:reverse(PurgedIdRevs)). - cpse_id_rev_repeated(Db1) -> Actions1 = [ {create, {<<"foo">>, {[{<<"vsn">>, 1}]}}}, @@ -106,7 +116,8 @@ cpse_id_rev_repeated(Db1) -> {ok, Db3} = cpse_util:apply_actions(Db2, Actions2), {ok, PurgedIdRevs1} = couch_db_engine:fold_purge_infos( - Db3, 0, fun fold_fun/2, [], []), + Db3, 0, fun fold_fun/2, [], [] + ), ExpectedPurgedIdRevs1 = [ {<<"foo">>, [Rev1]} ], @@ -117,7 +128,8 @@ cpse_id_rev_repeated(Db1) -> % purge the same Id,Rev when the doc still exists {ok, Db4} = cpse_util:apply_actions(Db3, Actions2), {ok, PurgedIdRevs2} = couch_db_engine:fold_purge_infos( - Db4, 0, fun fold_fun/2, [], []), + Db4, 0, fun fold_fun/2, [], [] + ), ExpectedPurgedIdRevs2 = [ {<<"foo">>, [Rev1]}, {<<"foo">>, [Rev1]} @@ -134,7 +146,8 @@ cpse_id_rev_repeated(Db1) -> {ok, Db5} = cpse_util:apply_actions(Db4, Actions3), {ok, PurgedIdRevs3} = couch_db_engine:fold_purge_infos( - Db5, 0, fun fold_fun/2, [], []), + Db5, 0, fun fold_fun/2, [], [] + ), ExpectedPurgedIdRevs3 = [ {<<"foo">>, [Rev1]}, {<<"foo">>, [Rev1]}, @@ -147,7 +160,8 @@ cpse_id_rev_repeated(Db1) -> {ok, Db6} = cpse_util:apply_actions(Db5, Actions3), {ok, PurgedIdRevs4} = couch_db_engine:fold_purge_infos( - Db6, 0, fun fold_fun/2, [], []), + Db6, 0, fun fold_fun/2, [], [] + ), ExpectedPurgedIdRevs4 = [ {<<"foo">>, [Rev1]}, {<<"foo">>, [Rev1]}, @@ -157,11 +171,9 @@ cpse_id_rev_repeated(Db1) -> ?assertEqual(ExpectedPurgedIdRevs4, lists:reverse(PurgedIdRevs4)), ?assertEqual(4, couch_db_engine:get_purge_seq(Db6)). - fold_fun({_PSeq, _UUID, Id, Revs}, Acc) -> {ok, [{Id, Revs} | Acc]}. - docid(I) -> Str = io_lib:format("~4..0b", [I]), iolist_to_binary(Str). diff --git a/src/couch_pse_tests/src/cpse_test_get_set_props.erl b/src/couch_pse_tests/src/cpse_test_get_set_props.erl index d49f67f49..773f1d0dc 100644 --- a/src/couch_pse_tests/src/cpse_test_get_set_props.erl +++ b/src/couch_pse_tests/src/cpse_test_get_set_props.erl @@ -14,18 +14,14 @@ -compile(export_all). -compile(nowarn_export_all). - -include_lib("eunit/include/eunit.hrl"). - setup_each() -> cpse_util:dbname(). - teardown_each(DbName) -> ok = couch_server:delete(DbName, []). - cpse_default_props(DbName) -> {ok, {_App, Engine, _Extension}} = application:get_env(couch, test_engine), {ok, Db} = cpse_util:create_db(DbName), @@ -46,16 +42,18 @@ cpse_default_props(DbName) -> ?assertEqual([{Node, 0}], couch_db_engine:get_epochs(Db)), ?assertEqual(0, couch_db_engine:get_compacted_seq(Db)). - --define(ADMIN_ONLY_SEC_PROPS, {[ - {<<"members">>, {[ - {<<"roles">>, [<<"_admin">>]} - ]}}, - {<<"admins">>, {[ - {<<"roles">>, [<<"_admin">>]} - ]}} -]}). - +-define(ADMIN_ONLY_SEC_PROPS, + {[ + {<<"members">>, + {[ + {<<"roles">>, [<<"_admin">>]} + ]}}, + {<<"admins">>, + {[ + {<<"roles">>, [<<"_admin">>]} + ]}} + ]} +). cpse_admin_only_security(DbName) -> Config = [{"couchdb", "default_security", "admin_only"}], @@ -70,16 +68,13 @@ cpse_admin_only_security(DbName) -> couch_log:error("~n~n~n~n~s -> ~s~n~n", [couch_db:name(Db1), couch_db:name(Db2)]), ?assertEqual(?ADMIN_ONLY_SEC_PROPS, couch_db:get_security(Db2)). - cpse_set_security(DbName) -> SecProps = {[{<<"foo">>, <<"bar">>}]}, check_prop_set(DbName, get_security, set_security, {[]}, SecProps). - cpse_set_revs_limit(DbName) -> check_prop_set(DbName, get_revs_limit, set_revs_limit, 1000, 50). - check_prop_set(DbName, GetFun, SetFun, Default, Value) -> {ok, Db0} = cpse_util:create_db(DbName), diff --git a/src/couch_pse_tests/src/cpse_test_open_close_delete.erl b/src/couch_pse_tests/src/cpse_test_open_close_delete.erl index d9b589fd6..c63a05bea 100644 --- a/src/couch_pse_tests/src/cpse_test_open_close_delete.erl +++ b/src/couch_pse_tests/src/cpse_test_open_close_delete.erl @@ -14,55 +14,49 @@ -compile(export_all). -compile(nowarn_export_all). - -include_lib("eunit/include/eunit.hrl"). - setup_each() -> cpse_util:dbname(). - teardown_each(DbName) -> case couch_server:exists(DbName) of true -> ok = couch_server:delete(DbName, []); false -> ok end. - cpse_open_non_existent(DbName) -> % Try twice to check that a failed open doesn't create % the database for some reason. ?assertEqual({not_found, no_db_file}, cpse_util:open_db(DbName)), ?assertEqual({not_found, no_db_file}, cpse_util:open_db(DbName)). - cpse_open_create(DbName) -> ?assertEqual(false, couch_server:exists(DbName)), ?assertEqual({not_found, no_db_file}, cpse_util:open_db(DbName)), ?assertMatch({ok, _}, cpse_util:create_db(DbName)), ?assertEqual(true, couch_server:exists(DbName)). - cpse_open_when_exists(DbName) -> ?assertEqual(false, couch_server:exists(DbName)), ?assertEqual({not_found, no_db_file}, cpse_util:open_db(DbName)), ?assertMatch({ok, _}, cpse_util:create_db(DbName)), ?assertEqual(file_exists, cpse_util:create_db(DbName)). - cpse_terminate(DbName) -> ?assertEqual(false, couch_server:exists(DbName)), ?assertEqual({not_found, no_db_file}, cpse_util:open_db(DbName)), ?assertEqual(ok, cycle_db(DbName, create_db)), ?assertEqual(true, couch_server:exists(DbName)). - cpse_rapid_recycle(DbName) -> ?assertEqual(ok, cycle_db(DbName, create_db)), - lists:foreach(fun(_) -> - ?assertEqual(ok, cycle_db(DbName, open_db)) - end, lists:seq(1, 100)). - + lists:foreach( + fun(_) -> + ?assertEqual(ok, cycle_db(DbName, open_db)) + end, + lists:seq(1, 100) + ). cpse_delete(DbName) -> ?assertEqual(false, couch_server:exists(DbName)), @@ -71,7 +65,6 @@ cpse_delete(DbName) -> ?assertEqual(ok, couch_server:delete(DbName, [])), ?assertEqual(false, couch_server:exists(DbName)). - cycle_db(DbName, Type) -> {ok, Db} = cpse_util:Type(DbName), cpse_util:shutdown_db(Db). diff --git a/src/couch_pse_tests/src/cpse_test_purge_bad_checkpoints.erl b/src/couch_pse_tests/src/cpse_test_purge_bad_checkpoints.erl index c7a85c7e4..bddbdb699 100644 --- a/src/couch_pse_tests/src/cpse_test_purge_bad_checkpoints.erl +++ b/src/couch_pse_tests/src/cpse_test_purge_bad_checkpoints.erl @@ -14,11 +14,9 @@ -compile(export_all). -compile(nowarn_export_all). - -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). - setup_each() -> {ok, Db1} = cpse_util:create_db(), {ok, Revs} = cpse_util:save_docs(couch_db:name(Db1), [ @@ -33,20 +31,21 @@ setup_each() -> {[{'_id', foo8}, {vsn, 8}]}, {[{'_id', foo9}, {vsn, 9}]} ]), - PInfos = lists:map(fun(Idx) -> - DocId = iolist_to_binary(["foo", $0 + Idx]), - Rev = lists:nth(Idx + 1, Revs), - {cpse_util:uuid(), DocId, [Rev]} - end, lists:seq(0, 9)), + PInfos = lists:map( + fun(Idx) -> + DocId = iolist_to_binary(["foo", $0 + Idx]), + Rev = lists:nth(Idx + 1, Revs), + {cpse_util:uuid(), DocId, [Rev]} + end, + lists:seq(0, 9) + ), {ok, _} = cpse_util:purge(couch_db:name(Db1), PInfos), {ok, Db2} = couch_db:reopen(Db1), Db2. - teardown_each(Db) -> ok = couch_server:delete(couch_db:name(Db), []). - cpse_bad_purge_seq(Db1) -> Db2 = save_local_doc(Db1, <<"foo">>), ?assertEqual(0, couch_db:get_minimum_purge_seq(Db2)), @@ -55,7 +54,6 @@ cpse_bad_purge_seq(Db1) -> {ok, Db3} = couch_db:reopen(Db2), ?assertEqual(1, couch_db:get_minimum_purge_seq(Db3)). - cpse_verify_non_boolean(Db1) -> Db2 = save_local_doc(Db1, 2), ?assertEqual(0, couch_db:get_minimum_purge_seq(Db2)), @@ -64,17 +62,22 @@ cpse_verify_non_boolean(Db1) -> {ok, Db3} = couch_db:reopen(Db2), ?assertEqual(5, couch_db:get_minimum_purge_seq(Db3)). - save_local_doc(Db1, PurgeSeq) -> {Mega, Secs, _} = os:timestamp(), NowSecs = Mega * 1000000 + Secs, - Doc = couch_doc:from_json_obj(?JSON_DECODE(?JSON_ENCODE({[ - {<<"_id">>, <<"_local/purge-test-stuff">>}, - {<<"purge_seq">>, PurgeSeq}, - {<<"timestamp_utc">>, NowSecs}, - {<<"verify_options">>, {[{<<"signature">>, <<"stuff">>}]}}, - {<<"type">>, <<"test">>} - ]}))), + Doc = couch_doc:from_json_obj( + ?JSON_DECODE( + ?JSON_ENCODE( + {[ + {<<"_id">>, <<"_local/purge-test-stuff">>}, + {<<"purge_seq">>, PurgeSeq}, + {<<"timestamp_utc">>, NowSecs}, + {<<"verify_options">>, {[{<<"signature">>, <<"stuff">>}]}}, + {<<"type">>, <<"test">>} + ]} + ) + ) + ), {ok, _} = couch_db:update_doc(Db1, Doc, []), {ok, Db2} = couch_db:reopen(Db1), Db2. diff --git a/src/couch_pse_tests/src/cpse_test_purge_docs.erl b/src/couch_pse_tests/src/cpse_test_purge_docs.erl index 60a072da6..f0ed3d747 100644 --- a/src/couch_pse_tests/src/cpse_test_purge_docs.erl +++ b/src/couch_pse_tests/src/cpse_test_purge_docs.erl @@ -14,23 +14,18 @@ -compile(export_all). -compile(nowarn_export_all). - -include_lib("eunit/include/eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). - -define(REV_DEPTH, 100). - setup_each() -> {ok, Db} = cpse_util:create_db(), couch_db:name(Db). - teardown_each(DbName) -> ok = couch_server:delete(DbName, []). - cpse_purge_simple(DbName) -> {ok, Rev} = cpse_util:save_doc(DbName, {[{'_id', foo1}, {vsn, 1.1}]}), @@ -57,7 +52,6 @@ cpse_purge_simple(DbName) -> {purge_infos, PurgeInfos} ]). - cpse_purge_simple_info_check(DbName) -> {ok, Rev} = cpse_util:save_doc(DbName, {[{'_id', foo1}, {vsn, 1.1}]}), PurgeInfos = [ @@ -72,7 +66,6 @@ cpse_purge_simple_info_check(DbName) -> ?assertMatch([{1, <<_/binary>>, <<"foo1">>, [Rev]}], AllInfos). - cpse_purge_empty_db(DbName) -> PurgeInfos = [ {cpse_util:uuid(), <<"foo">>, [{0, <<0>>}]} @@ -90,7 +83,6 @@ cpse_purge_empty_db(DbName) -> {purge_infos, PurgeInfos} ]). - cpse_purge_single_docid(DbName) -> {ok, [Rev1, _Rev2]} = cpse_util:save_docs(DbName, [ {[{'_id', foo1}, {vsn, 1}]}, @@ -121,7 +113,6 @@ cpse_purge_single_docid(DbName) -> {purge_infos, PurgeInfos} ]). - cpse_purge_multiple_docids(DbName) -> {ok, [Rev1, Rev2]} = cpse_util:save_docs(DbName, [ {[{'_id', foo1}, {vsn, 1.1}]}, @@ -156,7 +147,6 @@ cpse_purge_multiple_docids(DbName) -> {purge_infos, PurgeInfos} ]). - cpse_purge_no_docids(DbName) -> {ok, [_Rev1, _Rev2]} = cpse_util:save_docs(DbName, [ {[{'_id', foo1}, {vsn, 1}]}, @@ -183,15 +173,15 @@ cpse_purge_no_docids(DbName) -> {purge_infos, []} ]). - cpse_purge_rev_path(DbName) -> {ok, Rev1} = cpse_util:save_doc(DbName, {[{'_id', foo}, {vsn, 1}]}), - Update = {[ - {<<"_id">>, <<"foo">>}, - {<<"_rev">>, couch_doc:rev_to_str(Rev1)}, - {<<"_deleted">>, true}, - {<<"vsn">>, 2} - ]}, + Update = + {[ + {<<"_id">>, <<"foo">>}, + {<<"_rev">>, couch_doc:rev_to_str(Rev1)}, + {<<"_deleted">>, true}, + {<<"vsn">>, 2} + ]}, {ok, Rev2} = cpse_util:save_doc(DbName, Update), cpse_util:assert_db_props(?MODULE, ?LINE, DbName, [ @@ -219,18 +209,22 @@ cpse_purge_rev_path(DbName) -> {purge_infos, PurgeInfos} ]). - cpse_purge_deep_revision_path(DbName) -> {ok, InitRev} = cpse_util:save_doc(DbName, {[{'_id', bar}, {vsn, 0}]}), - LastRev = lists:foldl(fun(Count, PrevRev) -> - Update = {[ - {'_id', bar}, - {'_rev', couch_doc:rev_to_str(PrevRev)}, - {vsn, Count} - ]}, - {ok, NewRev} = cpse_util:save_doc(DbName, Update), - NewRev - end, InitRev, lists:seq(1, ?REV_DEPTH)), + LastRev = lists:foldl( + fun(Count, PrevRev) -> + Update = + {[ + {'_id', bar}, + {'_rev', couch_doc:rev_to_str(PrevRev)}, + {vsn, Count} + ]}, + {ok, NewRev} = cpse_util:save_doc(DbName, Update), + NewRev + end, + InitRev, + lists:seq(1, ?REV_DEPTH) + ), PurgeInfos = [ {cpse_util:uuid(), <<"bar">>, [LastRev]} @@ -248,14 +242,14 @@ cpse_purge_deep_revision_path(DbName) -> {purge_infos, PurgeInfos} ]). - cpse_purge_partial_revs(DbName) -> {ok, Rev1} = cpse_util:save_doc(DbName, {[{'_id', foo}, {vsn, <<"1.1">>}]}), - Update = {[ - {'_id', foo}, - {'_rev', couch_doc:rev_to_str({1, [couch_hash:md5_hash(<<"1.2">>)]})}, - {vsn, <<"1.2">>} - ]}, + Update = + {[ + {'_id', foo}, + {'_rev', couch_doc:rev_to_str({1, [couch_hash:md5_hash(<<"1.2">>)]})}, + {vsn, <<"1.2">>} + ]}, {ok, [_Rev2]} = cpse_util:save_docs(DbName, [Update], [replicated_changes]), PurgeInfos = [ @@ -274,7 +268,6 @@ cpse_purge_partial_revs(DbName) -> {purge_infos, PurgeInfos} ]). - cpse_purge_missing_docid(DbName) -> {ok, [Rev1, _Rev2]} = cpse_util:save_docs(DbName, [ {[{'_id', foo1}, {vsn, 1}]}, @@ -305,7 +298,6 @@ cpse_purge_missing_docid(DbName) -> {purge_infos, PurgeInfos} ]). - cpse_purge_duplicate_docids(DbName) -> {ok, [Rev1, _Rev2]} = cpse_util:save_docs(DbName, [ {[{'_id', foo1}, {vsn, 1}]}, @@ -338,14 +330,14 @@ cpse_purge_duplicate_docids(DbName) -> {purge_infos, PurgeInfos} ]). - cpse_purge_internal_revision(DbName) -> {ok, Rev1} = cpse_util:save_doc(DbName, {[{'_id', foo}, {vsn, 1}]}), - Update = {[ - {'_id', foo}, - {'_rev', couch_doc:rev_to_str(Rev1)}, - {vsn, 2} - ]}, + Update = + {[ + {'_id', foo}, + {'_rev', couch_doc:rev_to_str(Rev1)}, + {vsn, 2} + ]}, {ok, _Rev2} = cpse_util:save_doc(DbName, Update), PurgeInfos = [ @@ -364,7 +356,6 @@ cpse_purge_internal_revision(DbName) -> {purge_infos, PurgeInfos} ]). - cpse_purge_missing_revision(DbName) -> {ok, [_Rev1, Rev2]} = cpse_util:save_docs(DbName, [ {[{'_id', foo1}, {vsn, 1}]}, @@ -387,14 +378,14 @@ cpse_purge_missing_revision(DbName) -> {purge_infos, PurgeInfos} ]). - cpse_purge_repeated_revisions(DbName) -> {ok, Rev1} = cpse_util:save_doc(DbName, {[{'_id', foo}, {vsn, <<"1.1">>}]}), - Update = {[ - {'_id', foo}, - {'_rev', couch_doc:rev_to_str({1, [couch_hash:md5_hash(<<"1.2">>)]})}, - {vsn, <<"1.2">>} - ]}, + Update = + {[ + {'_id', foo}, + {'_rev', couch_doc:rev_to_str({1, [couch_hash:md5_hash(<<"1.2">>)]})}, + {vsn, <<"1.2">>} + ]}, {ok, [Rev2]} = cpse_util:save_docs(DbName, [Update], [replicated_changes]), cpse_util:assert_db_props(?MODULE, ?LINE, DbName, [ @@ -424,7 +415,6 @@ cpse_purge_repeated_revisions(DbName) -> {purge_infos, PurgeInfos1} ]). - cpse_purge_repeated_uuid(DbName) -> {ok, Rev} = cpse_util:save_doc(DbName, {[{'_id', foo1}, {vsn, 1.1}]}), @@ -459,6 +449,5 @@ cpse_purge_repeated_uuid(DbName) -> {purge_infos, PurgeInfos} ]). - fold_all_infos(Info, Acc) -> {ok, [Info | Acc]}. diff --git a/src/couch_pse_tests/src/cpse_test_purge_replication.erl b/src/couch_pse_tests/src/cpse_test_purge_replication.erl index 20dcc2f81..5ec04b711 100644 --- a/src/couch_pse_tests/src/cpse_test_purge_replication.erl +++ b/src/couch_pse_tests/src/cpse_test_purge_replication.erl @@ -14,27 +14,22 @@ -compile(export_all). -compile(nowarn_export_all). - -include_lib("eunit/include/eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). -include_lib("mem3/include/mem3.hrl"). - setup_all() -> cpse_util:setup_all([mem3, fabric, couch_replicator]). - setup_each() -> {ok, Src} = cpse_util:create_db(), {ok, Tgt} = cpse_util:create_db(), {couch_db:name(Src), couch_db:name(Tgt)}. - teardown_each({SrcDb, TgtDb}) -> ok = couch_server:delete(SrcDb, []), ok = couch_server:delete(TgtDb, []). - cpse_purge_http_replication({Source, Target}) -> {ok, Rev1} = cpse_util:save_doc(Source, {[{'_id', foo}, {vsn, 1}]}), @@ -47,10 +42,11 @@ cpse_purge_http_replication({Source, Target}) -> {purge_infos, []} ]), - RepObject = {[ - {<<"source">>, db_url(Source)}, - {<<"target">>, db_url(Target)} - ]}, + RepObject = + {[ + {<<"source">>, db_url(Source)}, + {<<"target">>, db_url(Target)} + ]}, {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER), {ok, Doc1} = cpse_util:open_doc(Target, foo), @@ -99,10 +95,11 @@ cpse_purge_http_replication({Source, Target}) -> % Show that replicating from the target % back to the source reintroduces the doc - RepObject2 = {[ - {<<"source">>, db_url(Target)}, - {<<"target">>, db_url(Source)} - ]}, + RepObject2 = + {[ + {<<"source">>, db_url(Target)}, + {<<"target">>, db_url(Source)} + ]}, {ok, _} = couch_replicator:replicate(RepObject2, ?ADMIN_USER), {ok, Doc3} = cpse_util:open_doc(Source, foo), @@ -118,7 +115,6 @@ cpse_purge_http_replication({Source, Target}) -> {purge_infos, PurgeInfos} ]). - cpse_purge_internal_repl_disabled({Source, Target}) -> cpse_util:with_config([{"mem3", "replicate_purges", "false"}], fun() -> repl(Source, Target), @@ -151,7 +147,6 @@ cpse_purge_internal_repl_disabled({Source, Target}) -> ?assertMatch({ok, #doc_info{}}, cpse_util:open_doc(Target, <<"foo1">>)) end). - cpse_purge_repl_simple_pull({Source, Target}) -> repl(Source, Target), @@ -165,7 +160,6 @@ cpse_purge_repl_simple_pull({Source, Target}) -> ?assertEqual([Rev], PRevs), repl(Source, Target). - cpse_purge_repl_simple_push({Source, Target}) -> repl(Source, Target), @@ -179,7 +173,6 @@ cpse_purge_repl_simple_push({Source, Target}) -> ?assertEqual([Rev], PRevs), repl(Source, Target). - repl(Source, Target) -> SrcShard = make_shard(Source), TgtShard = make_shard(Target), @@ -192,7 +185,6 @@ repl(Source, Target) -> Diff = cpse_util:term_diff(SrcTerm, TgtTerm), ?assertEqual(nodiff, Diff). - make_shard(DbName) -> #shard{ name = DbName, @@ -201,7 +193,6 @@ make_shard(DbName) -> range = [0, 16#FFFFFFFF] }. - db_url(DbName) -> Addr = config:get("httpd", "bind_address", "127.0.0.1"), Port = mochiweb_socket_server:get(couch_httpd, port), diff --git a/src/couch_pse_tests/src/cpse_test_purge_seqs.erl b/src/couch_pse_tests/src/cpse_test_purge_seqs.erl index 6a546580c..f9d87945e 100644 --- a/src/couch_pse_tests/src/cpse_test_purge_seqs.erl +++ b/src/couch_pse_tests/src/cpse_test_purge_seqs.erl @@ -14,20 +14,16 @@ -compile(export_all). -compile(nowarn_export_all). - -include_lib("eunit/include/eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). - setup_each() -> {ok, Db} = cpse_util:create_db(), couch_db:name(Db). - teardown_each(DbName) -> ok = couch_server:delete(DbName, []). - cpse_increment_purge_seq_on_complete_purge(DbName) -> {ok, Rev1} = cpse_util:save_doc(DbName, {[{'_id', foo1}, {vsn, 1.1}]}), {ok, Rev2} = cpse_util:save_doc(DbName, {[{'_id', foo2}, {vsn, 1.2}]}), @@ -68,7 +64,6 @@ cpse_increment_purge_seq_on_complete_purge(DbName) -> {purge_infos, PurgeInfos1 ++ PurgeInfos2} ]). - cpse_increment_purge_multiple_times(DbName) -> {ok, Rev1} = cpse_util:save_doc(DbName, {[{'_id', foo1}, {vsn, 1.1}]}), {ok, Rev2} = cpse_util:save_doc(DbName, {[{'_id', foo2}, {vsn, 1.2}]}), @@ -97,14 +92,14 @@ cpse_increment_purge_multiple_times(DbName) -> {purge_infos, PurgeInfos1} ]). - cpse_increment_purge_seq_on_partial_purge(DbName) -> {ok, Rev1} = cpse_util:save_doc(DbName, {[{'_id', foo1}, {vsn, <<"1.1">>}]}), - Update = {[ - {'_id', foo1}, - {'_rev', couch_doc:rev_to_str({1, [couch_hash:md5_hash(<<"1.2">>)]})}, - {vsn, <<"1.2">>} - ]}, + Update = + {[ + {'_id', foo1}, + {'_rev', couch_doc:rev_to_str({1, [couch_hash:md5_hash(<<"1.2">>)]})}, + {vsn, <<"1.2">>} + ]}, {ok, [_Rev2]} = cpse_util:save_docs(DbName, [Update], [replicated_changes]), cpse_util:assert_db_props(?MODULE, ?LINE, DbName, [ diff --git a/src/couch_pse_tests/src/cpse_test_read_write_docs.erl b/src/couch_pse_tests/src/cpse_test_read_write_docs.erl index a2151340a..f51e50aec 100644 --- a/src/couch_pse_tests/src/cpse_test_read_write_docs.erl +++ b/src/couch_pse_tests/src/cpse_test_read_write_docs.erl @@ -14,20 +14,16 @@ -compile(export_all). -compile(nowarn_export_all). - -include_lib("eunit/include/eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). - setup_each() -> {ok, Db} = cpse_util:create_db(), Db. - teardown_each(Db) -> ok = couch_server:delete(couch_db:name(Db), []). - cpse_read_docs_from_empty_db(Db) -> ?assertEqual([not_found], couch_db_engine:open_docs(Db, [<<"foo">>])), ?assertEqual( @@ -35,7 +31,6 @@ cpse_read_docs_from_empty_db(Db) -> couch_db_engine:open_docs(Db, [<<"a">>, <<"b">>]) ). - cpse_read_empty_local_docs(Db) -> {LocalA, LocalB} = {<<"_local/a">>, <<"_local/b">>}, ?assertEqual([not_found], couch_db_engine:open_local_docs(Db, [LocalA])), @@ -44,7 +39,6 @@ cpse_read_empty_local_docs(Db) -> couch_db_engine:open_local_docs(Db, [LocalA, LocalB]) ). - cpse_write_one_doc(Db1) -> ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), @@ -79,12 +73,13 @@ cpse_write_one_doc(Db1) -> }, Doc1 = couch_db_engine:read_doc_body(Db3, Doc0), - Body1 = if not is_binary(Doc1#doc.body) -> Doc1#doc.body; true -> - couch_compress:decompress(Doc1#doc.body) - end, + Body1 = + if + not is_binary(Doc1#doc.body) -> Doc1#doc.body; + true -> couch_compress:decompress(Doc1#doc.body) + end, ?assertEqual({[{<<"vsn">>, 1}]}, Body1). - cpse_write_two_docs(Db1) -> ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), @@ -106,7 +101,6 @@ cpse_write_two_docs(Db1) -> Resps = couch_db_engine:open_docs(Db3, [<<"foo">>, <<"bar">>]), ?assertEqual(false, lists:member(not_found, Resps)). - cpse_write_three_doc_batch(Db1) -> ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), @@ -131,7 +125,6 @@ cpse_write_three_doc_batch(Db1) -> Resps = couch_db_engine:open_docs(Db3, [<<"foo">>, <<"bar">>, <<"baz">>]), ?assertEqual(false, lists:member(not_found, Resps)). - cpse_update_doc(Db1) -> ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), @@ -167,13 +160,14 @@ cpse_update_doc(Db1) -> }, Doc1 = couch_db_engine:read_doc_body(Db3, Doc0), - Body1 = if not is_binary(Doc1#doc.body) -> Doc1#doc.body; true -> - couch_compress:decompress(Doc1#doc.body) - end, + Body1 = + if + not is_binary(Doc1#doc.body) -> Doc1#doc.body; + true -> couch_compress:decompress(Doc1#doc.body) + end, ?assertEqual({[{<<"vsn">>, 2}]}, Body1). - cpse_delete_doc(Db1) -> ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), @@ -207,13 +201,14 @@ cpse_delete_doc(Db1) -> }, Doc1 = couch_db_engine:read_doc_body(Db3, Doc0), - Body1 = if not is_binary(Doc1#doc.body) -> Doc1#doc.body; true -> - couch_compress:decompress(Doc1#doc.body) - end, + Body1 = + if + not is_binary(Doc1#doc.body) -> Doc1#doc.body; + true -> couch_compress:decompress(Doc1#doc.body) + end, ?assertEqual({[]}, Body1). - cpse_write_local_doc(Db1) -> ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), @@ -235,7 +230,6 @@ cpse_write_local_doc(Db1) -> [#doc{} = Doc] = couch_db_engine:open_local_docs(Db3, [<<"_local/foo">>]), ?assertEqual({[{<<"yay">>, false}]}, Doc#doc.body). - cpse_write_mixed_batch(Db1) -> ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), @@ -262,7 +256,6 @@ cpse_write_mixed_batch(Db1) -> [not_found] = couch_db_engine:open_local_docs(Db3, [<<"bar">>]), [#doc{}] = couch_db_engine:open_local_docs(Db3, [<<"_local/foo">>]). - cpse_update_local_doc(Db1) -> ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), @@ -285,7 +278,6 @@ cpse_update_local_doc(Db1) -> [#doc{} = Doc] = couch_db_engine:open_local_docs(Db3, [<<"_local/foo">>]), ?assertEqual({[{<<"stuff">>, null}]}, Doc#doc.body). - cpse_delete_local_doc(Db1) -> ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), @@ -306,6 +298,6 @@ cpse_delete_local_doc(Db1) -> [not_found] = couch_db_engine:open_docs(Db3, [<<"_local/foo">>]), ?assertEqual( - [not_found], - couch_db_engine:open_local_docs(Db3, [<<"_local/foo">>]) - ). + [not_found], + couch_db_engine:open_local_docs(Db3, [<<"_local/foo">>]) + ). diff --git a/src/couch_pse_tests/src/cpse_test_ref_counting.erl b/src/couch_pse_tests/src/cpse_test_ref_counting.erl index cb115a785..a0123d1ca 100644 --- a/src/couch_pse_tests/src/cpse_test_ref_counting.erl +++ b/src/couch_pse_tests/src/cpse_test_ref_counting.erl @@ -14,23 +14,18 @@ -compile(export_all). -compile(nowarn_export_all). - -include_lib("eunit/include/eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). - -define(NUM_CLIENTS, 1000). - setup_each() -> {ok, Db} = cpse_util:create_db(), {Db, self()}. - teardown_each({Db, _}) -> ok = couch_server:delete(couch_db:name(Db), []). - cpse_empty_monitors({Db, Pid}) -> Pids = couch_db_engine:monitored_by(Db), ?assert(is_list(Pids)), @@ -41,7 +36,6 @@ cpse_empty_monitors({Db, Pid}) -> ], ?assertEqual([], Pids -- Expected). - cpse_incref_decref({Db, _}) -> {Pid, _} = Client = start_client(Db), wait_client(Client), @@ -54,11 +48,13 @@ cpse_incref_decref({Db, _}) -> Pids2 = couch_db_engine:monitored_by(Db), ?assert(not lists:member(Pid, Pids2)). - cpse_incref_decref_many({Db, _}) -> - Clients = lists:map(fun(_) -> - start_client(Db) - end, lists:seq(1, ?NUM_CLIENTS)), + Clients = lists:map( + fun(_) -> + start_client(Db) + end, + lists:seq(1, ?NUM_CLIENTS) + ), lists:foreach(fun(C) -> wait_client(C) end, Clients), @@ -71,7 +67,6 @@ cpse_incref_decref_many({Db, _}) -> Pids2 = couch_db_engine:monitored_by(Db), ?assertEqual(3, length(Pids2)). - start_client(Db0) -> spawn_monitor(fun() -> {ok, Db1} = couch_db:open_int(couch_db:name(Db0), []), @@ -92,7 +87,6 @@ start_client(Db0) -> end end). - wait_client({Pid, _Ref}) -> Pid ! {waiting, self()}, receive @@ -101,7 +95,6 @@ wait_client({Pid, _Ref}) -> erlang:error(timeout) end. - close_client({Pid, Ref}) -> Pid ! close, receive @@ -110,4 +103,3 @@ close_client({Pid, Ref}) -> after 1000 -> erlang:error(timeout) end. - diff --git a/src/couch_pse_tests/src/cpse_util.erl b/src/couch_pse_tests/src/cpse_util.erl index 55622e925..bcbea4487 100644 --- a/src/couch_pse_tests/src/cpse_util.erl +++ b/src/couch_pse_tests/src/cpse_util.erl @@ -14,11 +14,9 @@ -compile(export_all). -compile(nowarn_export_all). - -include_lib("eunit/include/eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). - -define(TEST_MODULES, [ cpse_test_open_close_delete, cpse_test_get_set_props, @@ -36,29 +34,27 @@ cpse_test_purge_seqs ]). - -define(SHUTDOWN_TIMEOUT, 5000). -define(COMPACTOR_TIMEOUT, 50000). -define(ATTACHMENT_WRITE_TIMEOUT, 10000). -define(MAKE_DOC_SUMMARY_TIMEOUT, 5000). - create_tests(EngineApp, Extension) -> create_tests(EngineApp, EngineApp, Extension). - create_tests(EngineApp, EngineModule, Extension) -> TestEngine = {EngineApp, EngineModule, Extension}, application:set_env(couch, test_engine, TestEngine), - lists:map(fun(TestMod) -> - {atom_to_list(TestMod), cpse_gather:module(TestMod)} - end, ?TEST_MODULES). - + lists:map( + fun(TestMod) -> + {atom_to_list(TestMod), cpse_gather:module(TestMod)} + end, + ?TEST_MODULES + ). setup_all() -> setup_all([]). - setup_all(ExtraApps) -> Ctx = test_util:start_couch(ExtraApps), {ok, {_, EngineMod, Extension}} = application:get_env(couch, test_engine), @@ -68,20 +64,16 @@ setup_all(ExtraApps) -> config:set("mem3", "replicate_purges", "true", false), Ctx. - teardown_all(Ctx) -> test_util:stop_couch(Ctx). - rootdir() -> config:get("couchdb", "database_dir", "."). - dbname() -> UUID = couch_uuids:random(), <<"db-", UUID/binary>>. - get_engine() -> case application:get_env(couch, test_engine) of {ok, {_App, _Mod, Extension}} -> @@ -90,21 +82,17 @@ get_engine() -> <<"couch">> end. - create_db() -> create_db(dbname()). - create_db(DbName) -> Engine = get_engine(), couch_db:create(DbName, [{engine, Engine}, ?ADMIN_CTX]). - open_db(DbName) -> Engine = get_engine(), couch_db:open_int(DbName, [{engine, Engine}, ?ADMIN_CTX]). - shutdown_db(Db) -> Pid = couch_db:get_pid(Db), Ref = erlang:monitor(process, Pid), @@ -116,38 +104,47 @@ shutdown_db(Db) -> erlang:error(database_shutdown_timeout) end, test_util:wait(fun() -> - case ets:member(couch_server:couch_dbs(couch_db:name(Db)), - couch_db:name(Db)) of + case + ets:member( + couch_server:couch_dbs(couch_db:name(Db)), + couch_db:name(Db) + ) + of true -> wait; false -> ok end end). - save_doc(DbName, Json) -> {ok, [Rev]} = save_docs(DbName, [Json], []), {ok, Rev}. - save_docs(DbName, JsonDocs) -> save_docs(DbName, JsonDocs, []). - save_docs(DbName, JsonDocs, Options) -> - Docs = lists:map(fun(JDoc) -> - couch_doc:from_json_obj(?JSON_DECODE(?JSON_ENCODE(JDoc))) - end, JsonDocs), + Docs = lists:map( + fun(JDoc) -> + couch_doc:from_json_obj(?JSON_DECODE(?JSON_ENCODE(JDoc))) + end, + JsonDocs + ), Opts = [full_commit | Options], {ok, Db} = couch_db:open_int(DbName, []), try case lists:member(replicated_changes, Options) of true -> {ok, []} = couch_db:update_docs( - Db, Docs, Opts, replicated_changes), - {ok, lists:map(fun(Doc) -> - {Pos, [RevId | _]} = Doc#doc.revs, - {Pos, RevId} - end, Docs)}; + Db, Docs, Opts, replicated_changes + ), + {ok, + lists:map( + fun(Doc) -> + {Pos, [RevId | _]} = Doc#doc.revs, + {Pos, RevId} + end, + Docs + )}; false -> {ok, Resp} = couch_db:update_docs(Db, Docs, Opts), {ok, [Rev || {ok, Rev} <- Resp]} @@ -156,7 +153,6 @@ save_docs(DbName, JsonDocs, Options) -> couch_db:close(Db) end. - open_doc(DbName, DocId0) -> DocId = ?JSON_DECODE(?JSON_ENCODE(DocId0)), {ok, Db} = couch_db:open_int(DbName, []), @@ -166,15 +162,16 @@ open_doc(DbName, DocId0) -> couch_db:close(Db) end. - purge(DbName, PurgeInfos) -> purge(DbName, PurgeInfos, []). - purge(DbName, PurgeInfos0, Options) when is_list(PurgeInfos0) -> - PurgeInfos = lists:map(fun({UUID, DocIdJson, Revs}) -> - {UUID, ?JSON_DECODE(?JSON_ENCODE(DocIdJson)), Revs} - end, PurgeInfos0), + PurgeInfos = lists:map( + fun({UUID, DocIdJson, Revs}) -> + {UUID, ?JSON_DECODE(?JSON_ENCODE(DocIdJson)), Revs} + end, + PurgeInfos0 + ), {ok, Db} = couch_db:open_int(DbName, []), try couch_db:purge_docs(Db, PurgeInfos, Options) @@ -182,31 +179,29 @@ purge(DbName, PurgeInfos0, Options) when is_list(PurgeInfos0) -> couch_db:close(Db) end. - uuid() -> couch_uuids:random(). - assert_db_props(Module, Line, DbName, Props) when is_binary(DbName) -> {ok, Db} = couch_db:open_int(DbName, []), try assert_db_props(Module, Line, Db, Props) - catch error:{assertEqual, Props} -> - {_, Rest} = proplists:split(Props, [module, line]), - erlang:error({assertEqual, [{module, Module}, {line, Line} | Rest]}) + catch + error:{assertEqual, Props} -> + {_, Rest} = proplists:split(Props, [module, line]), + erlang:error({assertEqual, [{module, Module}, {line, Line} | Rest]}) after couch_db:close(Db) end; - assert_db_props(Module, Line, Db, Props) -> try assert_each_prop(Db, Props) - catch error:{assertEqual, Props} -> - {_, Rest} = proplists:split(Props, [module, line]), - erlang:error({assertEqual, [{module, Module}, {line, Line} | Rest]}) + catch + error:{assertEqual, Props} -> + {_, Rest} = proplists:split(Props, [module, line]), + erlang:error({assertEqual, [{module, Module}, {line, Line} | Rest]}) end. - assert_each_prop(_Db, []) -> ok; assert_each_prop(Db, [{doc_count, Expect} | Rest]) -> @@ -234,51 +229,47 @@ assert_each_prop(Db, [{purge_infos, Expect} | Rest]) -> ?assertEqual(Expect, lists:reverse(PurgeInfos)), assert_each_prop(Db, Rest). - aep_changes(_A, Acc) -> {ok, Acc + 1}. - aep_fold({_PSeq, UUID, Id, Revs}, Acc) -> {ok, [{UUID, Id, Revs} | Acc]}. - apply_actions(DbName, Actions) when is_binary(DbName) -> {ok, Db0} = couch_db:open_int(DbName, [?ADMIN_CTX]), {ok, Db1} = apply_actions(Db0, Actions), couch_db:close(Db1), ok; - apply_actions(Db, []) -> {ok, Db}; - apply_actions(Db, [Action | Rest]) -> {ok, NewDb} = apply_action(Db, Action), apply_actions(NewDb, Rest). - apply_action(Db, {batch, BatchActions}) -> apply_batch(Db, BatchActions); - apply_action(Db, Action) -> apply_batch(Db, [Action]). - apply_batch(Db, Actions) -> AccIn = {[], [], [], []}, - AccOut = lists:foldl(fun(Action, Acc) -> - {DocAcc, ConfAcc, LDocAcc, PurgeAcc} = Acc, - case gen_write(Db, Action) of - {update, Doc} -> - {[Doc | DocAcc], ConfAcc, LDocAcc, PurgeAcc}; - {conflict, Doc} -> - {DocAcc, [Doc | ConfAcc], LDocAcc, PurgeAcc}; - {local, Doc} -> - {DocAcc, ConfAcc, [Doc | LDocAcc], PurgeAcc}; - {purge, PurgeInfo} -> - {DocAcc, ConfAcc, LDocAcc, [PurgeInfo | PurgeAcc]} - end - end, AccIn, Actions), + AccOut = lists:foldl( + fun(Action, Acc) -> + {DocAcc, ConfAcc, LDocAcc, PurgeAcc} = Acc, + case gen_write(Db, Action) of + {update, Doc} -> + {[Doc | DocAcc], ConfAcc, LDocAcc, PurgeAcc}; + {conflict, Doc} -> + {DocAcc, [Doc | ConfAcc], LDocAcc, PurgeAcc}; + {local, Doc} -> + {DocAcc, ConfAcc, [Doc | LDocAcc], PurgeAcc}; + {purge, PurgeInfo} -> + {DocAcc, ConfAcc, LDocAcc, [PurgeInfo | PurgeAcc]} + end + end, + AccIn, + Actions + ), {Docs0, Conflicts0, LDocs0, PurgeInfos0} = AccOut, Docs = lists:reverse(Docs0), @@ -293,37 +284,37 @@ apply_batch(Db, Actions) -> {ok, []} = couch_db:update_docs(Db, Conflicts, [], replicated_changes), {ok, Db2} = couch_db:reopen(Db1), - if PurgeInfos == [] -> ok; true -> - {ok, _} = couch_db:purge_docs(Db2, PurgeInfos) + if + PurgeInfos == [] -> ok; + true -> {ok, _} = couch_db:purge_docs(Db2, PurgeInfos) end, couch_db:reopen(Db2). - gen_write(Db, {Action, {<<"_local/", _/binary>> = DocId, Body}}) -> - PrevRev = case couch_db:open_doc(Db, DocId) of - {not_found, _} -> - 0; - {ok, #doc{revs = {0, []}}} -> - 0; - {ok, #doc{revs = {0, [RevStr | _]}}} -> - binary_to_integer(RevStr) - end, - {RevId, Deleted} = case Action of - Action when Action == create; Action == update -> - {PrevRev + 1, false}; - delete -> - {0, true} - end, + PrevRev = + case couch_db:open_doc(Db, DocId) of + {not_found, _} -> + 0; + {ok, #doc{revs = {0, []}}} -> + 0; + {ok, #doc{revs = {0, [RevStr | _]}}} -> + binary_to_integer(RevStr) + end, + {RevId, Deleted} = + case Action of + Action when Action == create; Action == update -> + {PrevRev + 1, false}; + delete -> + {0, true} + end, {local, #doc{ id = DocId, revs = {0, [list_to_binary(integer_to_list(RevId))]}, body = Body, deleted = Deleted }}; - gen_write(Db, {Action, {DocId, Body}}) -> gen_write(Db, {Action, {DocId, Body, []}}); - gen_write(Db, {create, {DocId, Body, Atts}}) -> {not_found, _} = couch_db:open_doc(Db, DocId), {update, #doc{ @@ -333,11 +324,13 @@ gen_write(Db, {create, {DocId, Body, Atts}}) -> body = Body, atts = Atts }}; - gen_write(_Db, {purge, {DocId, PrevRevs0, _}}) -> - PrevRevs = if is_list(PrevRevs0) -> PrevRevs0; true -> [PrevRevs0] end, + PrevRevs = + if + is_list(PrevRevs0) -> PrevRevs0; + true -> [PrevRevs0] + end, {purge, {couch_uuids:random(), DocId, PrevRevs}}; - gen_write(Db, {Action, {DocId, Body, Atts}}) -> #full_doc_info{} = PrevFDI = couch_db:get_full_doc_info(Db, DocId), @@ -351,16 +344,18 @@ gen_write(Db, {Action, {DocId, Body, Atts}}) -> NewRev = gen_rev(Action, DocId, PrevRev, Body, Atts), - Deleted = case Action of - update -> false; - conflict -> false; - delete -> true - end, + Deleted = + case Action of + update -> false; + conflict -> false; + delete -> true + end, - Type = case Action of - conflict -> conflict; - _ -> update - end, + Type = + case Action of + conflict -> conflict; + _ -> update + end, {Type, #doc{ id = DocId, @@ -370,7 +365,6 @@ gen_write(Db, {Action, {DocId, Body, Atts}}) -> atts = Atts }}. - gen_rev(A, DocId, {Pos, Rev}, Body, Atts) when A == update; A == delete -> NewRev = couch_hash:md5_hash(term_to_binary({DocId, Rev, Body, Atts})), {Pos + 1, [NewRev, Rev]}; @@ -379,26 +373,24 @@ gen_rev(conflict, DocId, _, Body, Atts) -> NewRev = couch_hash:md5_hash(term_to_binary({DocId, UUID, Body, Atts})), {1, [NewRev]}. - prep_atts(_Db, []) -> []; - prep_atts(Db, [{FileName, Data} | Rest]) -> {_, Ref} = spawn_monitor(fun() -> {ok, Stream} = couch_db:open_write_stream(Db, []), exit(write_att(Stream, FileName, Data, Data)) end), - Att = receive - {'DOWN', Ref, _, _, {{no_catch, not_supported}, _}} -> - throw(not_supported); - {'DOWN', Ref, _, _, Resp} -> - Resp + Att = + receive + {'DOWN', Ref, _, _, {{no_catch, not_supported}, _}} -> + throw(not_supported); + {'DOWN', Ref, _, _, Resp} -> + Resp after ?ATTACHMENT_WRITE_TIMEOUT -> erlang:error(attachment_write_timeout) - end, + end, [Att | prep_atts(Db, Rest)]. - write_att(Stream, FileName, OrigData, <<>>) -> {StreamEngine, Len, Len, Md5, Md5} = couch_stream:close(Stream), couch_util:check_md5(Md5, couch_hash:md5_hash(OrigData)), @@ -412,26 +404,24 @@ write_att(Stream, FileName, OrigData, <<>>) -> {md5, Md5}, {encoding, identity} ]); - write_att(Stream, FileName, OrigData, Data) -> - {Chunk, Rest} = case size(Data) > 4096 of - true -> - <<Head:4096/binary, Tail/binary>> = Data, - {Head, Tail}; - false -> - {Data, <<>>} - end, + {Chunk, Rest} = + case size(Data) > 4096 of + true -> + <<Head:4096/binary, Tail/binary>> = Data, + {Head, Tail}; + false -> + {Data, <<>>} + end, ok = couch_stream:write(Stream, Chunk), write_att(Stream, FileName, OrigData, Rest). - prev_rev(#full_doc_info{} = FDI) -> #doc_info{ revs = [#rev_info{} = PrevRev | _] } = couch_doc:to_doc_info(FDI), PrevRev. - db_as_term(Db) -> db_as_term(Db, compact). @@ -439,7 +429,6 @@ db_as_term(DbName, Type) when is_binary(DbName) -> couch_util:with_db(DbName, fun(Db) -> db_as_term(Db, Type) end); - db_as_term(Db, Type) -> [ {props, db_props_as_term(Db, Type)}, @@ -449,7 +438,6 @@ db_as_term(Db, Type) -> {purged_docs, db_purged_docs_as_term(Db)} ]. - db_props_as_term(Db, Type) -> Props0 = [ get_doc_count, @@ -463,27 +451,36 @@ db_props_as_term(Db, Type) -> get_uuid, get_epochs ], - Props = if Type /= replication -> Props0; true -> - Props0 -- [get_uuid] - end, - lists:map(fun(Fun) -> - {Fun, couch_db_engine:Fun(Db)} - end, Props). - + Props = + if + Type /= replication -> Props0; + true -> Props0 -- [get_uuid] + end, + lists:map( + fun(Fun) -> + {Fun, couch_db_engine:Fun(Db)} + end, + Props + ). db_docs_as_term(Db) -> FoldFun = fun(FDI, Acc) -> {ok, [FDI | Acc]} end, {ok, FDIs} = couch_db:fold_docs(Db, FoldFun, [], []), - lists:reverse(lists:map(fun(FDI) -> - fdi_to_term(Db, FDI) - end, FDIs)). - + lists:reverse( + lists:map( + fun(FDI) -> + fdi_to_term(Db, FDI) + end, + FDIs + ) + ). db_local_docs_as_term(Db, Type) -> FoldFun = fun(Doc, Acc) -> case Doc#doc.id of - <<?LOCAL_DOC_PREFIX, "purge-mem3", _/binary>> - when Type == replication -> + <<?LOCAL_DOC_PREFIX, "purge-mem3", _/binary>> when + Type == replication + -> {ok, Acc}; _ -> {ok, [Doc | Acc]} @@ -492,14 +489,17 @@ db_local_docs_as_term(Db, Type) -> {ok, LDocs} = couch_db:fold_local_docs(Db, FoldFun, [], []), lists:reverse(LDocs). - db_changes_as_term(Db) -> FoldFun = fun(FDI, Acc) -> {ok, [FDI | Acc]} end, {ok, Changes} = couch_db:fold_changes(Db, 0, FoldFun, [], []), - lists:reverse(lists:map(fun(FDI) -> - fdi_to_term(Db, FDI) - end, Changes)). - + lists:reverse( + lists:map( + fun(FDI) -> + fdi_to_term(Db, FDI) + end, + Changes + ) + ). db_purged_docs_as_term(Db) -> InitPSeq = couch_db_engine:get_oldest_purge_seq(Db) - 1, @@ -507,18 +507,22 @@ db_purged_docs_as_term(Db) -> {ok, [{PSeq, UUID, Id, Revs} | Acc]} end, {ok, PDocs} = couch_db_engine:fold_purge_infos( - Db, InitPSeq, FoldFun, [], []), + Db, InitPSeq, FoldFun, [], [] + ), lists:reverse(PDocs). - fdi_to_term(Db, FDI) -> #full_doc_info{ id = DocId, rev_tree = OldTree } = FDI, - {NewRevTree, _} = couch_key_tree:mapfold(fun(Rev, Node, Type, Acc) -> - tree_to_term(Rev, Node, Type, Acc, DocId) - end, Db, OldTree), + {NewRevTree, _} = couch_key_tree:mapfold( + fun(Rev, Node, Type, Acc) -> + tree_to_term(Rev, Node, Type, Acc, DocId) + end, + Db, + OldTree + ), FDI#full_doc_info{ rev_tree = NewRevTree, % Blank out sizes because we allow storage @@ -530,10 +534,8 @@ fdi_to_term(Db, FDI) -> } }. - tree_to_term(_Rev, _Leaf, branch, Acc, _DocId) -> {?REV_MISSING, Acc}; - tree_to_term({Pos, RevId}, #leaf{} = Leaf, leaf, Db, DocId) -> #leaf{ deleted = Deleted, @@ -549,13 +551,17 @@ tree_to_term({Pos, RevId}, #leaf{} = Leaf, leaf, Db, DocId) -> Doc1 = couch_db_engine:read_doc_body(Db, Doc0), - Body = if not is_binary(Doc1#doc.body) -> Doc1#doc.body; true -> - couch_compress:decompress(Doc1#doc.body) - end, + Body = + if + not is_binary(Doc1#doc.body) -> Doc1#doc.body; + true -> couch_compress:decompress(Doc1#doc.body) + end, - Atts1 = if not is_binary(Doc1#doc.atts) -> Doc1#doc.atts; true -> - couch_compress:decompress(Doc1#doc.atts) - end, + Atts1 = + if + not is_binary(Doc1#doc.atts) -> Doc1#doc.atts; + true -> couch_compress:decompress(Doc1#doc.atts) + end, StreamSrc = fun(Sp) -> couch_db:open_read_stream(Db, Sp) end, Atts2 = [couch_att:from_disk_term(StreamSrc, Att) || Att <- Atts1], @@ -568,34 +574,25 @@ tree_to_term({Pos, RevId}, #leaf{} = Leaf, leaf, Db, DocId) -> }, {NewLeaf, Db}. - att_to_term(Att) -> Bin = couch_att:to_binary(Att), couch_att:store(data, Bin, Att). - term_diff(T1, T2) when is_tuple(T1), is_tuple(T2) -> tuple_diff(tuple_to_list(T1), tuple_to_list(T2)); - term_diff(L1, L2) when is_list(L1), is_list(L2) -> list_diff(L1, L2); - term_diff(V1, V2) when V1 == V2 -> nodiff; - term_diff(V1, V2) -> {V1, V2}. - tuple_diff([], []) -> nodiff; - tuple_diff([T1 | _], []) -> {longer, T1}; - tuple_diff([], [T2 | _]) -> {shorter, T2}; - tuple_diff([T1 | R1], [T2 | R2]) -> case term_diff(T1, T2) of nodiff -> @@ -604,16 +601,12 @@ tuple_diff([T1 | R1], [T2 | R2]) -> {T1, Else} end. - list_diff([], []) -> nodiff; - list_diff([T1 | _], []) -> {longer, T1}; - list_diff([], [T2 | _]) -> {shorter, T2}; - list_diff([T1 | R1], [T2 | R2]) -> case term_diff(T1, T2) of nodiff -> @@ -622,7 +615,6 @@ list_diff([T1 | R1], [T2 | R2]) -> {T1, Else} end. - compact(Db) -> {ok, Pid} = couch_db:start_compact(Db), Ref = erlang:monitor(process, Pid), @@ -656,7 +648,6 @@ compact(Db) -> end end). - with_config(Config, Fun) -> OldConfig = apply_config(Config), try @@ -665,10 +656,8 @@ with_config(Config, Fun) -> apply_config(OldConfig) end. - apply_config([]) -> []; - apply_config([{Section, Key, Value} | Rest]) -> Orig = config:get(Section, Key), case Value of diff --git a/src/couch_replicator/src/couch_replicator.erl b/src/couch_replicator/src/couch_replicator.erl index 413628143..39b3903ea 100644 --- a/src/couch_replicator/src/couch_replicator.erl +++ b/src/couch_replicator/src/couch_replicator.erl @@ -30,13 +30,20 @@ -define(DESIGN_DOC_CREATION_DELAY_MSEC, 1000). -define(REPLICATION_STATES, [ - initializing, % Just added to scheduler - error, % Could not be turned into a replication job - running, % Scheduled and running - pending, % Scheduled and waiting to run - crashing, % Scheduled but crashing, backed off by the scheduler - completed, % Non-continuous (normal) completed replication - failed % Terminal failure, will not be retried anymore + % Just added to scheduler + initializing, + % Could not be turned into a replication job + error, + % Scheduled and running + running, + % Scheduled and waiting to run + pending, + % Scheduled but crashing, backed off by the scheduler + crashing, + % Non-continuous (normal) completed replication + completed, + % Terminal failure, will not be retried anymore + failed ]). -import(couch_util, [ @@ -44,101 +51,96 @@ get_value/3 ]). - -spec replicate({[_]}, any()) -> - {ok, {continuous, binary()}} | - {ok, {[_]}} | - {ok, {cancelled, binary()}} | - {error, any()} | - no_return(). + {ok, {continuous, binary()}} + | {ok, {[_]}} + | {ok, {cancelled, binary()}} + | {error, any()} + | no_return(). replicate(PostBody, Ctx) -> {ok, Rep0} = couch_replicator_utils:parse_rep_doc(PostBody, Ctx), Rep = Rep0#rep{start_time = os:timestamp()}, #rep{id = RepId, options = Options, user_ctx = UserCtx} = Rep, case get_value(cancel, Options, false) of - true -> - CancelRepId = case get_value(id, Options, nil) of - nil -> - RepId; - RepId2 -> - RepId2 - end, - case check_authorization(CancelRepId, UserCtx) of - ok -> - cancel_replication(CancelRepId); - not_found -> - {error, not_found} - end; - false -> - check_authorization(RepId, UserCtx), - {ok, Listener} = rep_result_listener(RepId), - Result = do_replication_loop(Rep), - couch_replicator_notifier:stop(Listener), - Result + true -> + CancelRepId = + case get_value(id, Options, nil) of + nil -> + RepId; + RepId2 -> + RepId2 + end, + case check_authorization(CancelRepId, UserCtx) of + ok -> + cancel_replication(CancelRepId); + not_found -> + {error, not_found} + end; + false -> + check_authorization(RepId, UserCtx), + {ok, Listener} = rep_result_listener(RepId), + Result = do_replication_loop(Rep), + couch_replicator_notifier:stop(Listener), + Result end. - -spec do_replication_loop(#rep{}) -> {ok, {continuous, binary()}} | {ok, tuple()} | {error, any()}. do_replication_loop(#rep{id = {BaseId, Ext} = Id, options = Options} = Rep) -> ok = couch_replicator_scheduler:add_job(Rep), case get_value(continuous, Options, false) of - true -> - {ok, {continuous, ?l2b(BaseId ++ Ext)}}; - false -> - wait_for_result(Id) + true -> + {ok, {continuous, ?l2b(BaseId ++ Ext)}}; + false -> + wait_for_result(Id) end. - -spec rep_result_listener(rep_id()) -> {ok, pid()}. rep_result_listener(RepId) -> ReplyTo = self(), {ok, _Listener} = couch_replicator_notifier:start_link( - fun({_, RepId2, _} = Ev) when RepId2 =:= RepId -> + fun + ({_, RepId2, _} = Ev) when RepId2 =:= RepId -> ReplyTo ! Ev; (_) -> ok - end). - + end + ). -spec wait_for_result(rep_id()) -> {ok, {[_]}} | {error, any()}. wait_for_result(RepId) -> receive - {finished, RepId, RepResult} -> - {ok, RepResult}; - {error, RepId, Reason} -> - {error, Reason} + {finished, RepId, RepResult} -> + {ok, RepResult}; + {error, RepId, Reason} -> + {error, Reason} end. - -spec cancel_replication(rep_id()) -> {ok, {cancelled, binary()}} | {error, not_found}. cancel_replication({BasedId, Extension} = RepId) -> FullRepId = BasedId ++ Extension, couch_log:notice("Canceling replication '~s' ...", [FullRepId]), case couch_replicator_scheduler:rep_state(RepId) of - #rep{} -> - ok = couch_replicator_scheduler:remove_job(RepId), - couch_log:notice("Replication '~s' cancelled", [FullRepId]), - {ok, {cancelled, ?l2b(FullRepId)}}; - nil -> - couch_log:notice("Replication '~s' not found", [FullRepId]), - {error, not_found} + #rep{} -> + ok = couch_replicator_scheduler:remove_job(RepId), + couch_log:notice("Replication '~s' cancelled", [FullRepId]), + {ok, {cancelled, ?l2b(FullRepId)}}; + nil -> + couch_log:notice("Replication '~s' not found", [FullRepId]), + {error, not_found} end. - -spec replication_states() -> [atom()]. replication_states() -> ?REPLICATION_STATES. - -spec strip_url_creds(binary() | {[_]}) -> binary(). strip_url_creds(Endpoint) -> - try - couch_replicator_docs:parse_rep_db(Endpoint, [], []) of - #httpdb{url = Url} -> - iolist_to_binary(couch_util:url_strip_password(Url)) + try couch_replicator_docs:parse_rep_db(Endpoint, [], []) of + #httpdb{url = Url} -> + iolist_to_binary(couch_util:url_strip_password(Url)) catch throw:{error, local_endpoints_not_supported} -> Endpoint; @@ -148,39 +150,39 @@ strip_url_creds(Endpoint) -> null end. - -spec job(binary()) -> {ok, {[_]}} | {error, not_found}. job(JobId0) when is_binary(JobId0) -> JobId = couch_replicator_ids:convert(JobId0), {Res, _Bad} = rpc:multicall(couch_replicator_scheduler, job, [JobId]), case [JobInfo || {ok, JobInfo} <- Res] of - [JobInfo| _] -> + [JobInfo | _] -> {ok, JobInfo}; [] -> {error, not_found} end. - -spec restart_job(binary() | list() | rep_id()) -> {ok, {[_]}} | {error, not_found}. restart_job(JobId0) -> JobId = couch_replicator_ids:convert(JobId0), {Res, _} = rpc:multicall(couch_replicator_scheduler, restart_job, [JobId]), case [JobInfo || {ok, JobInfo} <- Res] of - [JobInfo| _] -> + [JobInfo | _] -> {ok, JobInfo}; [] -> {error, not_found} end. - -spec active_doc(binary(), binary()) -> {ok, {[_]}} | {error, not_found}. active_doc(DbName, DocId) -> try Shards = mem3:shards(DbName), Live = [node() | nodes()], - Nodes = lists:usort([N || #shard{node=N} <- Shards, - lists:member(N, Live)]), + Nodes = lists:usort([ + N + || #shard{node = N} <- Shards, + lists:member(N, Live) + ]), Owner = mem3:owner(DbName, DocId, Nodes), case active_doc_rpc(DbName, DocId, [Owner]) of {ok, DocInfo} -> @@ -194,7 +196,6 @@ active_doc(DbName, DocId) -> active_doc_rpc(DbName, DocId, [node()]) end. - -spec active_doc_rpc(binary(), binary(), [node()]) -> {ok, {[_]}} | {error, not_found}. active_doc_rpc(_DbName, _DocId, []) -> @@ -202,8 +203,12 @@ active_doc_rpc(_DbName, _DocId, []) -> active_doc_rpc(DbName, DocId, [Node]) when Node =:= node() -> couch_replicator_doc_processor:doc(DbName, DocId); active_doc_rpc(DbName, DocId, Nodes) -> - {Res, _Bad} = rpc:multicall(Nodes, couch_replicator_doc_processor, doc, - [DbName, DocId]), + {Res, _Bad} = rpc:multicall( + Nodes, + couch_replicator_doc_processor, + doc, + [DbName, DocId] + ), case [DocInfo || {ok, DocInfo} <- Res] of [DocInfo | _] -> {ok, DocInfo}; @@ -211,7 +216,6 @@ active_doc_rpc(DbName, DocId, Nodes) -> {error, not_found} end. - -spec doc(binary(), binary(), any()) -> {ok, {[_]}} | {error, not_found}. doc(RepDb, DocId, UserCtx) -> case active_doc(RepDb, DocId) of @@ -221,17 +225,15 @@ doc(RepDb, DocId, UserCtx) -> doc_from_db(RepDb, DocId, UserCtx) end. - -spec doc_from_db(binary(), binary(), any()) -> {ok, {[_]}} | {error, not_found}. doc_from_db(RepDb, DocId, UserCtx) -> case fabric:open_doc(RepDb, DocId, [UserCtx, ejson_body]) of {ok, Doc} -> {ok, info_from_doc(RepDb, couch_doc:to_json_obj(Doc, []))}; - {not_found, _Reason} -> + {not_found, _Reason} -> {error, not_found} end. - -spec info_from_doc(binary(), {[_]}) -> {[_]}. info_from_doc(RepDb, {Props}) -> DocId = get_value(<<"_id">>, Props), @@ -239,27 +241,28 @@ info_from_doc(RepDb, {Props}) -> Target = get_value(<<"target">>, Props), State0 = state_atom(get_value(<<"_replication_state">>, Props, null)), StateTime = get_value(<<"_replication_state_time">>, Props, null), - {State1, StateInfo, ErrorCount, StartTime} = case State0 of - completed -> - {InfoP} = get_value(<<"_replication_stats">>, Props, {[]}), - case lists:keytake(<<"start_time">>, 1, InfoP) of - {value, {_, Time}, InfoP1} -> - {State0, {InfoP1}, 0, Time}; - false -> - case lists:keytake(start_time, 1, InfoP) of - {value, {_, Time}, InfoP1} -> - {State0, {InfoP1}, 0, Time}; - false -> - {State0, {InfoP}, 0, null} + {State1, StateInfo, ErrorCount, StartTime} = + case State0 of + completed -> + {InfoP} = get_value(<<"_replication_stats">>, Props, {[]}), + case lists:keytake(<<"start_time">>, 1, InfoP) of + {value, {_, Time}, InfoP1} -> + {State0, {InfoP1}, 0, Time}; + false -> + case lists:keytake(start_time, 1, InfoP) of + {value, {_, Time}, InfoP1} -> + {State0, {InfoP1}, 0, Time}; + false -> + {State0, {InfoP}, 0, null} end - end; - failed -> - Info = get_value(<<"_replication_state_reason">>, Props, nil), - EJsonInfo = couch_replicator_utils:ejson_state_info(Info), - {State0, EJsonInfo, 1, StateTime}; - _OtherState -> - {null, null, 0, null} - end, + end; + failed -> + Info = get_value(<<"_replication_state_reason">>, Props, nil), + EJsonInfo = couch_replicator_utils:ejson_state_info(Info), + {State0, EJsonInfo, 1, StateTime}; + _OtherState -> + {null, null, 0, null} + end, {[ {doc_id, DocId}, {database, RepDb}, @@ -271,29 +274,27 @@ info_from_doc(RepDb, {Props}) -> {info, StateInfo}, {start_time, StartTime}, {last_updated, StateTime} - ]}. - + ]}. state_atom(<<"triggered">>) -> - triggered; % This handles a legacy case were document wasn't converted yet + % This handles a legacy case were document wasn't converted yet + triggered; state_atom(State) when is_binary(State) -> erlang:binary_to_existing_atom(State, utf8); state_atom(State) when is_atom(State) -> State. - -spec check_authorization(rep_id(), #user_ctx{}) -> ok | not_found. check_authorization(RepId, #user_ctx{name = Name} = Ctx) -> case couch_replicator_scheduler:rep_state(RepId) of - #rep{user_ctx = #user_ctx{name = Name}} -> - ok; - #rep{} -> - couch_httpd:verify_is_server_admin(Ctx); - nil -> - not_found + #rep{user_ctx = #user_ctx{name = Name}} -> + ok; + #rep{} -> + couch_httpd:verify_is_server_admin(Ctx); + nil -> + not_found end. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -301,8 +302,8 @@ check_authorization(RepId, #user_ctx{name = Name} = Ctx) -> authorization_test_() -> { foreach, - fun () -> ok end, - fun (_) -> meck:unload() end, + fun() -> ok end, + fun(_) -> meck:unload() end, [ t_admin_is_always_authorized(), t_username_must_match(), @@ -310,7 +311,6 @@ authorization_test_() -> ] }. - t_admin_is_always_authorized() -> ?_test(begin expect_rep_user_ctx(<<"someuser">>, <<"_admin">>), @@ -318,20 +318,23 @@ t_admin_is_always_authorized() -> ?assertEqual(ok, check_authorization(<<"RepId">>, UserCtx)) end). - t_username_must_match() -> - ?_test(begin + ?_test(begin expect_rep_user_ctx(<<"user">>, <<"somerole">>), UserCtx1 = #user_ctx{name = <<"user">>, roles = [<<"somerole">>]}, ?assertEqual(ok, check_authorization(<<"RepId">>, UserCtx1)), UserCtx2 = #user_ctx{name = <<"other">>, roles = [<<"somerole">>]}, - ?assertThrow({unauthorized, _}, check_authorization(<<"RepId">>, - UserCtx2)) + ?assertThrow( + {unauthorized, _}, + check_authorization( + <<"RepId">>, + UserCtx2 + ) + ) end). - t_replication_not_found() -> - ?_test(begin + ?_test(begin meck:expect(couch_replicator_scheduler, rep_state, 1, nil), UserCtx1 = #user_ctx{name = <<"user">>, roles = [<<"somerole">>]}, ?assertEqual(not_found, check_authorization(<<"RepId">>, UserCtx1)), @@ -339,17 +342,18 @@ t_replication_not_found() -> ?assertEqual(not_found, check_authorization(<<"RepId">>, UserCtx2)) end). - expect_rep_user_ctx(Name, Role) -> - meck:expect(couch_replicator_scheduler, rep_state, + meck:expect( + couch_replicator_scheduler, + rep_state, fun(_Id) -> UserCtx = #user_ctx{name = Name, roles = [Role]}, #rep{user_ctx = UserCtx} - end). - + end + ). strip_url_creds_test_() -> - { + { setup, fun() -> meck:expect(config, get, fun(_, _, Default) -> Default end) @@ -365,11 +369,9 @@ strip_url_creds_test_() -> ] }. - t_strip_local_db_creds() -> ?_test(?assertEqual(<<"localdb">>, strip_url_creds(<<"localdb">>))). - t_strip_http_basic_creds() -> ?_test(begin Url1 = <<"http://adm:pass@host/db">>, @@ -379,22 +381,24 @@ t_strip_http_basic_creds() -> Url3 = <<"http://adm:pass@host:80/db">>, ?assertEqual(<<"http://host:80/db/">>, strip_url_creds(Url3)), Url4 = <<"http://adm:pass@host/db?a=b&c=d">>, - ?assertEqual(<<"http://host/db?a=b&c=d">>, - strip_url_creds(Url4)) + ?assertEqual( + <<"http://host/db?a=b&c=d">>, + strip_url_creds(Url4) + ) end). - t_strip_http_props_creds() -> ?_test(begin Props1 = {[{<<"url">>, <<"http://adm:pass@host/db">>}]}, ?assertEqual(<<"http://host/db/">>, strip_url_creds(Props1)), - Props2 = {[ {<<"url">>, <<"http://host/db">>}, - {<<"headers">>, {[{<<"Authorization">>, <<"Basic pa55">>}]}} - ]}, + Props2 = + {[ + {<<"url">>, <<"http://host/db">>}, + {<<"headers">>, {[{<<"Authorization">>, <<"Basic pa55">>}]}} + ]}, ?assertEqual(<<"http://host/db/">>, strip_url_creds(Props2)) end). - t_strip_url_creds_errors() -> ?_test(begin Bad1 = {[{<<"url">>, <<"http://adm:pass/bad">>}]}, @@ -412,5 +416,4 @@ t_strip_url_creds_errors() -> ?assertEqual(null, strip_url_creds(Bad5)) end). - -endif. diff --git a/src/couch_replicator/src/couch_replicator_api_wrap.erl b/src/couch_replicator/src/couch_replicator_api_wrap.erl index 8dc7f2f01..193f8dad4 100644 --- a/src/couch_replicator/src/couch_replicator_api_wrap.erl +++ b/src/couch_replicator/src/couch_replicator_api_wrap.erl @@ -40,17 +40,17 @@ changes_since/5, db_uri/1, normalize_db/1 - ]). +]). -import(couch_replicator_httpc, [ send_req/3 - ]). +]). -import(couch_util, [ encode_doc_id/1, get_value/2, get_value/3 - ]). +]). -define(MAX_WAIT, 5 * 60 * 1000). @@ -59,14 +59,11 @@ db_uri(#httpdb{url = Url}) -> couch_util:url_strip_password(Url); - db_uri(DbName) when is_binary(DbName) -> ?b2l(DbName); - db_uri(Db) -> db_uri(couch_db:name(Db)). - db_open(Db) -> db_open(Db, false, []). @@ -74,40 +71,48 @@ db_open(#httpdb{} = Db1, Create, CreateParams) -> {ok, Db} = couch_replicator_httpc:setup(Db1), try case Create of - false -> - ok; - true -> - Db2 = maybe_append_create_query_params(Db, CreateParams), - send_req(Db2, [{method, put}], - fun(401, _, _) -> - throw({unauthorized, ?l2b(db_uri(Db2))}); + false -> + ok; + true -> + Db2 = maybe_append_create_query_params(Db, CreateParams), + send_req( + Db2, + [{method, put}], + fun + (401, _, _) -> + throw({unauthorized, ?l2b(db_uri(Db2))}); + (403, _, _) -> + throw({forbidden, ?l2b(db_uri(Db2))}); + (_, _, _) -> + ok + end + ) + end, + send_req( + Db, + [{method, get}], + fun + (200, _, {Props}) -> + UpdateSeq = get_value(<<"update_seq">>, Props), + InstanceStart = get_value(<<"instance_start_time">>, Props), + case {UpdateSeq, InstanceStart} of + {undefined, _} -> + throw({db_not_found, ?l2b(db_uri(Db))}); + {_, undefined} -> + throw({db_not_found, ?l2b(db_uri(Db))}); + _ -> + {ok, Db} + end; + (200, _, _Body) -> + throw({db_not_found, ?l2b(db_uri(Db))}); + (401, _, _) -> + throw({unauthorized, ?l2b(db_uri(Db))}); (403, _, _) -> - throw({forbidden, ?l2b(db_uri(Db2))}); + throw({forbidden, ?l2b(db_uri(Db))}); (_, _, _) -> - ok - end) - end, - send_req(Db, [{method, get}], - fun(200, _, {Props}) -> - UpdateSeq = get_value(<<"update_seq">>, Props), - InstanceStart = get_value(<<"instance_start_time">>, Props), - case {UpdateSeq, InstanceStart} of - {undefined, _} -> - throw({db_not_found, ?l2b(db_uri(Db))}); - {_, undefined} -> - throw({db_not_found, ?l2b(db_uri(Db))}); - _ -> - {ok, Db} - end; - (200, _, _Body) -> - throw({db_not_found, ?l2b(db_uri(Db))}); - (401, _, _) -> - throw({unauthorized, ?l2b(db_uri(Db))}); - (403, _, _) -> - throw({forbidden, ?l2b(db_uri(Db))}); - (_, _, _) -> - throw({db_not_found, ?l2b(db_uri(Db))}) - end) + throw({db_not_found, ?l2b(db_uri(Db))}) + end + ) catch throw:Error -> db_close(Db), @@ -125,12 +130,14 @@ db_close(#httpdb{httpc_pool = Pool} = HttpDb) -> unlink(Pool), ok = couch_replicator_httpc_pool:stop(Pool). - get_db_info(#httpdb{} = Db) -> - send_req(Db, [], + send_req( + Db, + [], fun(200, _, {Props}) -> {ok, Props} - end). + end + ). get_pending_count(#httpdb{} = Db, Seq) when is_number(Seq) -> % Source looks like Apache CouchDB and not Cloudant so we fall @@ -151,52 +158,63 @@ get_pending_count(#httpdb{} = Db, Seq) -> get_view_info(#httpdb{} = Db, DDocId, ViewName) -> Path = io_lib:format("~s/_view/~s/_info", [DDocId, ViewName]), - send_req(Db, [{path, Path}], + send_req( + Db, + [{path, Path}], fun(200, _, {Props}) -> {VInfo} = couch_util:get_value(<<"view_index">>, Props, {[]}), {ok, VInfo} - end). - + end + ). ensure_full_commit(#httpdb{} = Db) -> send_req( Db, - [{method, post}, {path, "_ensure_full_commit"}, - {headers, [{"Content-Type", "application/json"}]}], - fun(201, _, {Props}) -> - {ok, get_value(<<"instance_start_time">>, Props)}; - (_, _, {Props}) -> - {error, get_value(<<"error">>, Props)} - end). - + [ + {method, post}, + {path, "_ensure_full_commit"}, + {headers, [{"Content-Type", "application/json"}]} + ], + fun + (201, _, {Props}) -> + {ok, get_value(<<"instance_start_time">>, Props)}; + (_, _, {Props}) -> + {error, get_value(<<"error">>, Props)} + end + ). get_missing_revs(#httpdb{} = Db, IdRevs) -> JsonBody = {[{Id, couch_doc:revs_to_strs(Revs)} || {Id, Revs} <- IdRevs]}, send_req( Db, - [{method, post}, {path, "_revs_diff"}, {body, ?JSON_ENCODE(JsonBody)}, - {headers, [{"Content-Type", "application/json"}]}], - fun(200, _, {Props}) -> - ConvertToNativeFun = fun({Id, {Result}}) -> - MissingRevs = couch_doc:parse_revs( - get_value(<<"missing">>, Result) - ), - PossibleAncestors = couch_doc:parse_revs( - get_value(<<"possible_ancestors">>, Result, []) - ), - {Id, MissingRevs, PossibleAncestors} - end, - {ok, lists:map(ConvertToNativeFun, Props)}; - (ErrCode, _, ErrMsg) when is_integer(ErrCode) -> - {error, {revs_diff_failed, ErrCode, ErrMsg}} - end). - + [ + {method, post}, + {path, "_revs_diff"}, + {body, ?JSON_ENCODE(JsonBody)}, + {headers, [{"Content-Type", "application/json"}]} + ], + fun + (200, _, {Props}) -> + ConvertToNativeFun = fun({Id, {Result}}) -> + MissingRevs = couch_doc:parse_revs( + get_value(<<"missing">>, Result) + ), + PossibleAncestors = couch_doc:parse_revs( + get_value(<<"possible_ancestors">>, Result, []) + ), + {Id, MissingRevs, PossibleAncestors} + end, + {ok, lists:map(ConvertToNativeFun, Props)}; + (ErrCode, _, ErrMsg) when is_integer(ErrCode) -> + {error, {revs_diff_failed, ErrCode, ErrMsg}} + end + ). open_doc_revs(#httpdb{retries = 0} = HttpDb, Id, Revs, Options, _Fun, _Acc) -> Path = encode_doc_id(Id), QS = options_to_query_args(HttpDb, Path, [revs, {open_revs, Revs} | Options]), Url = couch_util:url_strip_password( - couch_replicator_httpc:full_url(HttpDb, [{path,Path}, {qs,QS}]) + couch_replicator_httpc:full_url(HttpDb, [{path, Path}, {qs, QS}]) ), couch_log:error("Replication crashing because GET ~s failed", [Url]), exit(kaboom); @@ -206,15 +224,15 @@ open_doc_revs(#httpdb{} = HttpDb, Id, Revs, Options, Fun, Acc) -> {Pid, Ref} = spawn_monitor(fun() -> Self = self(), Callback = fun - (200, Headers, StreamDataFun) -> - remote_open_doc_revs_streamer_start(Self), - {<<"--">>, _, _} = couch_httpd:parse_multipart_request( - header_value("Content-Type", Headers), - StreamDataFun, - fun mp_parse_mixed/1 - ); - (414, _, _) -> - exit(request_uri_too_long) + (200, Headers, StreamDataFun) -> + remote_open_doc_revs_streamer_start(Self), + {<<"--">>, _, _} = couch_httpd:parse_multipart_request( + header_value("Content-Type", Headers), + StreamDataFun, + fun mp_parse_mixed/1 + ); + (414, _, _) -> + exit(request_uri_too_long) end, Streamer = spawn_link(fun() -> Params = [ @@ -241,12 +259,12 @@ open_doc_revs(#httpdb{} = HttpDb, Id, Revs, Options, Fun, Acc) -> exit(Streamer, {streamer_parent_died, Self}); {'DOWN', Ref, process, Self, _} -> ok - end + end end), receive - {started_open_doc_revs, Ref} -> - Ret = receive_docs_loop(Streamer, Fun, Id, Revs, Ref, Acc), - exit({exit_ok, Ret}) + {started_open_doc_revs, Ref} -> + Ret = receive_docs_loop(Streamer, Fun, Id, Revs, Ref, Acc), + exit({exit_ok, Ret}) end end), receive @@ -254,7 +272,7 @@ open_doc_revs(#httpdb{} = HttpDb, Id, Revs, Options, Fun, Acc) -> Ret; {'DOWN', Ref, process, Pid, {{nocatch, missing_doc}, _}} -> throw(missing_doc); - {'DOWN', Ref, process, Pid, {{nocatch, {missing_stub,_} = Stub}, _}} -> + {'DOWN', Ref, process, Pid, {{nocatch, {missing_stub, _} = Stub}, _}} -> throw(Stub); {'DOWN', Ref, process, Pid, {http_request_failed, _, _, max_backoff}} -> exit(max_backoff); @@ -264,19 +282,27 @@ open_doc_revs(#httpdb{} = HttpDb, Id, Revs, Options, Fun, Acc) -> true -> throw(request_uri_too_long); false -> - couch_log:info("Reducing url length to ~B because of" - " 414 response", [NewMaxLen]), - Options1 = lists:keystore(max_url_len, 1, Options, - {max_url_len, NewMaxLen}), + couch_log:info( + "Reducing url length to ~B because of" + " 414 response", + [NewMaxLen] + ), + Options1 = lists:keystore( + max_url_len, + 1, + Options, + {max_url_len, NewMaxLen} + ), open_doc_revs(HttpDb, Id, Revs, Options1, Fun, Acc) end; {'DOWN', Ref, process, Pid, Else} -> Url = couch_util:url_strip_password( - couch_replicator_httpc:full_url(HttpDb, [{path,Path}, {qs,QS}]) + couch_replicator_httpc:full_url(HttpDb, [{path, Path}, {qs, QS}]) ), #httpdb{retries = Retries, wait = Wait0} = HttpDb, Wait = 2 * erlang:min(Wait0 * 2, ?MAX_WAIT), - couch_log:notice("Retrying GET to ~s in ~p seconds due to error ~w", + couch_log:notice( + "Retrying GET to ~s in ~p seconds due to error ~w", [Url, Wait / 1000, error_reason(Else)] ), ok = timer:sleep(Wait), @@ -287,7 +313,6 @@ open_doc_revs(#httpdb{} = HttpDb, Id, Revs, Options, Fun, Acc) -> open_doc_revs(RetryDb, Id, Revs, Options, Fun, Acc) end. - error_reason({http_request_failed, "GET", _Url, {error, timeout}}) -> timeout; error_reason({http_request_failed, "GET", _Url, {error, {_, req_timedout}}}) -> @@ -301,35 +326,44 @@ open_doc(#httpdb{} = Db, Id, Options) -> send_req( Db, [{path, encode_doc_id(Id)}, {qs, options_to_query_args(Options, [])}], - fun(200, _, Body) -> - {ok, couch_doc:from_json_obj(Body)}; - (_, _, {Props}) -> - {error, get_value(<<"error">>, Props)} - end). - + fun + (200, _, Body) -> + {ok, couch_doc:from_json_obj(Body)}; + (_, _, {Props}) -> + {error, get_value(<<"error">>, Props)} + end + ). update_doc(Db, Doc, Options) -> update_doc(Db, Doc, Options, interactive_edit). update_doc(#httpdb{} = HttpDb, #doc{id = DocId} = Doc, Options, Type) -> - QArgs = case Type of - replicated_changes -> - [{"new_edits", "false"}]; - _ -> - [] - end ++ options_to_query_args(Options, []), + QArgs = + case Type of + replicated_changes -> + [{"new_edits", "false"}]; + _ -> + [] + end ++ options_to_query_args(Options, []), Boundary = couch_uuids:random(), JsonBytes = ?JSON_ENCODE( couch_doc:to_json_obj( - Doc, [revs, attachments, follows, att_encoding_info | Options])), - {ContentType, Len} = couch_doc:len_doc_to_multi_part_stream(Boundary, - JsonBytes, Doc#doc.atts, true), - Headers = case lists:member(delay_commit, Options) of - true -> - [{"X-Couch-Full-Commit", "false"}]; - false -> - [] - end ++ [{"Content-Type", ?b2l(ContentType)}, {"Content-Length", Len}], + Doc, [revs, attachments, follows, att_encoding_info | Options] + ) + ), + {ContentType, Len} = couch_doc:len_doc_to_multi_part_stream( + Boundary, + JsonBytes, + Doc#doc.atts, + true + ), + Headers = + case lists:member(delay_commit, Options) of + true -> + [{"X-Couch-Full-Commit", "false"}]; + false -> + [] + end ++ [{"Content-Type", ?b2l(ContentType)}, {"Content-Length", Len}], Body = {fun stream_doc/1, {JsonBytes, Doc#doc.atts, Boundary, Len}}, send_req( % A crash here bubbles all the way back up to run_user_fun inside @@ -337,27 +371,33 @@ update_doc(#httpdb{} = HttpDb, #doc{id = DocId} = Doc, Options, Type) -> % appropriate course of action, since we've already started streaming % the response body from the GET request. HttpDb#httpdb{retries = 0}, - [{method, put}, {path, encode_doc_id(DocId)}, - {qs, QArgs}, {headers, Headers}, {body, Body}], - fun(Code, _, {Props}) when Code =:= 200 orelse Code =:= 201 orelse Code =:= 202 -> + [ + {method, put}, + {path, encode_doc_id(DocId)}, + {qs, QArgs}, + {headers, Headers}, + {body, Body} + ], + fun + (Code, _, {Props}) when Code =:= 200 orelse Code =:= 201 orelse Code =:= 202 -> {ok, couch_doc:parse_rev(get_value(<<"rev">>, Props))}; (409, _, _) -> throw(conflict); (Code, _, {Props}) -> case {Code, get_value(<<"error">>, Props)} of - {401, <<"unauthorized">>} -> - throw({unauthorized, get_value(<<"reason">>, Props)}); - {403, <<"forbidden">>} -> - throw({forbidden, get_value(<<"reason">>, Props)}); - {412, <<"missing_stub">>} -> - throw({missing_stub, get_value(<<"reason">>, Props)}); - {413, _} -> - {error, request_body_too_large}; - {_, Error} -> - {error, Error} + {401, <<"unauthorized">>} -> + throw({unauthorized, get_value(<<"reason">>, Props)}); + {403, <<"forbidden">>} -> + throw({forbidden, get_value(<<"reason">>, Props)}); + {412, <<"missing_stub">>} -> + throw({missing_stub, get_value(<<"reason">>, Props)}); + {413, _} -> + {error, request_body_too_large}; + {_, Error} -> + {error, Error} end - end). - + end + ). update_docs(Db, DocList, Options) -> update_docs(Db, DocList, Options, interactive_edit). @@ -366,26 +406,30 @@ update_docs(_Db, [], _Options, _UpdateType) -> {ok, []}; update_docs(#httpdb{} = HttpDb, DocList, Options, UpdateType) -> FullCommit = atom_to_list(not lists:member(delay_commit, Options)), - Prefix = case UpdateType of - replicated_changes -> - <<"{\"new_edits\":false,\"docs\":[">>; - interactive_edit -> - <<"{\"docs\":[">> - end, + Prefix = + case UpdateType of + replicated_changes -> + <<"{\"new_edits\":false,\"docs\":[">>; + interactive_edit -> + <<"{\"docs\":[">> + end, Suffix = <<"]}">>, % Note: nginx and other servers don't like PUT/POST requests without % a Content-Length header, so we can't do a chunked transfer encoding % and JSON encode each doc only before sending it through the socket. {Docs, Len} = lists:mapfoldl( - fun(#doc{} = Doc, Acc) -> - Json = ?JSON_ENCODE(couch_doc:to_json_obj(Doc, [revs, attachments])), - {Json, Acc + iolist_size(Json)}; - (Doc, Acc) -> - {Doc, Acc + iolist_size(Doc)} + fun + (#doc{} = Doc, Acc) -> + Json = ?JSON_ENCODE(couch_doc:to_json_obj(Doc, [revs, attachments])), + {Json, Acc + iolist_size(Json)}; + (Doc, Acc) -> + {Doc, Acc + iolist_size(Doc)} end, byte_size(Prefix) + byte_size(Suffix) + length(DocList) - 1, - DocList), - BodyFun = fun(eof) -> + DocList + ), + BodyFun = fun + (eof) -> eof; ([]) -> {ok, Suffix, eof}; @@ -403,153 +447,195 @@ update_docs(#httpdb{} = HttpDb, DocList, Options, UpdateType) -> ], send_req( HttpDb, - [{method, post}, {path, "_bulk_docs"}, - {body, {BodyFun, [prefix | Docs]}}, {headers, Headers}], - fun(201, _, Results) when is_list(Results) -> + [ + {method, post}, + {path, "_bulk_docs"}, + {body, {BodyFun, [prefix | Docs]}}, + {headers, Headers} + ], + fun + (201, _, Results) when is_list(Results) -> {ok, bulk_results_to_errors(DocList, Results, remote)}; - (413, _, _) -> + (413, _, _) -> {error, request_body_too_large}; - (417, _, Results) when is_list(Results) -> + (417, _, Results) when is_list(Results) -> {ok, bulk_results_to_errors(DocList, Results, remote)}; - (ErrCode, _, ErrMsg) when is_integer(ErrCode) -> + (ErrCode, _, ErrMsg) when is_integer(ErrCode) -> {error, {bulk_docs_failed, ErrCode, ErrMsg}} - end). - - -changes_since(#httpdb{headers = Headers1, timeout = InactiveTimeout} = HttpDb, - Style, StartSeq, UserFun, Options) -> + end + ). + +changes_since( + #httpdb{headers = Headers1, timeout = InactiveTimeout} = HttpDb, + Style, + StartSeq, + UserFun, + Options +) -> Timeout = erlang:max(1000, InactiveTimeout div 3), - BaseQArgs = case get_value(continuous, Options, false) of - false -> - [{"feed", "normal"}]; - true -> - [{"feed", "continuous"}] - end ++ [ - {"style", atom_to_list(Style)}, {"since", ?JSON_ENCODE(StartSeq)}, - {"timeout", integer_to_list(Timeout)} - ], + BaseQArgs = + case get_value(continuous, Options, false) of + false -> + [{"feed", "normal"}]; + true -> + [{"feed", "continuous"}] + end ++ + [ + {"style", atom_to_list(Style)}, + {"since", ?JSON_ENCODE(StartSeq)}, + {"timeout", integer_to_list(Timeout)} + ], DocIds = get_value(doc_ids, Options), Selector = get_value(selector, Options), - {QArgs, Method, Body, Headers} = case {DocIds, Selector} of - {undefined, undefined} -> - QArgs1 = maybe_add_changes_filter_q_args(BaseQArgs, Options), - {QArgs1, get, [], Headers1}; - {undefined, _} when is_tuple(Selector) -> - Headers2 = [{"Content-Type", "application/json"} | Headers1], - JsonSelector = ?JSON_ENCODE({[{<<"selector">>, Selector}]}), - {[{"filter", "_selector"} | BaseQArgs], post, JsonSelector, Headers2}; - {_, undefined} when is_list(DocIds) -> - Headers2 = [{"Content-Type", "application/json"} | Headers1], - JsonDocIds = ?JSON_ENCODE({[{<<"doc_ids">>, DocIds}]}), - {[{"filter", "_doc_ids"} | BaseQArgs], post, JsonDocIds, Headers2} - end, + {QArgs, Method, Body, Headers} = + case {DocIds, Selector} of + {undefined, undefined} -> + QArgs1 = maybe_add_changes_filter_q_args(BaseQArgs, Options), + {QArgs1, get, [], Headers1}; + {undefined, _} when is_tuple(Selector) -> + Headers2 = [{"Content-Type", "application/json"} | Headers1], + JsonSelector = ?JSON_ENCODE({[{<<"selector">>, Selector}]}), + {[{"filter", "_selector"} | BaseQArgs], post, JsonSelector, Headers2}; + {_, undefined} when is_list(DocIds) -> + Headers2 = [{"Content-Type", "application/json"} | Headers1], + JsonDocIds = ?JSON_ENCODE({[{<<"doc_ids">>, DocIds}]}), + {[{"filter", "_doc_ids"} | BaseQArgs], post, JsonDocIds, Headers2} + end, try send_req( HttpDb, - [{method, Method}, {path, "_changes"}, {qs, QArgs}, - {headers, Headers}, {body, Body}, - {ibrowse_options, [{stream_to, {self(), once}}]}], - fun(200, _, DataStreamFun) -> + [ + {method, Method}, + {path, "_changes"}, + {qs, QArgs}, + {headers, Headers}, + {body, Body}, + {ibrowse_options, [{stream_to, {self(), once}}]} + ], + fun + (200, _, DataStreamFun) -> parse_changes_feed(Options, UserFun, DataStreamFun); (405, _, _) when is_list(DocIds) -> % CouchDB versions < 1.1.0 don't have the builtin % _changes feed filter "_doc_ids" neither support POST - send_req(HttpDb, [{method, get}, {path, "_changes"}, - {qs, BaseQArgs}, {headers, Headers1}, - {ibrowse_options, [{stream_to, {self(), once}}]}], + send_req( + HttpDb, + [ + {method, get}, + {path, "_changes"}, + {qs, BaseQArgs}, + {headers, Headers1}, + {ibrowse_options, [{stream_to, {self(), once}}]} + ], fun(200, _, DataStreamFun2) -> - UserFun2 = fun(#doc_info{id = Id} = DocInfo) -> - case lists:member(Id, DocIds) of - true -> - UserFun(DocInfo); - false -> - ok - end; - (LastSeq) -> - UserFun(LastSeq) + UserFun2 = fun + (#doc_info{id = Id} = DocInfo) -> + case lists:member(Id, DocIds) of + true -> + UserFun(DocInfo); + false -> + ok + end; + (LastSeq) -> + UserFun(LastSeq) end, - parse_changes_feed(Options, UserFun2, - DataStreamFun2) - end); - (ErrCode, _, ErrMsg) when is_integer(ErrCode) -> + parse_changes_feed( + Options, + UserFun2, + DataStreamFun2 + ) + end + ); + (ErrCode, _, ErrMsg) when is_integer(ErrCode) -> throw({retry_limit, {changes_req_failed, ErrCode, ErrMsg}}) - end) + end + ) catch exit:{http_request_failed, _, _, max_backoff} -> exit(max_backoff); - exit:{http_request_failed, _, _, {error, {connection_closed, - mid_stream}}} -> + exit:{http_request_failed, _, _, {error, {connection_closed, mid_stream}}} -> throw(retry_no_limit); exit:{http_request_failed, _, _, _} = Error -> throw({retry_limit, Error}) end. - % internal functions maybe_add_changes_filter_q_args(BaseQS, Options) -> case get_value(filter, Options) of - undefined -> - BaseQS; - FilterName -> - %% get list of view attributes - ViewFields0 = [atom_to_list(F) || F <- record_info(fields, mrargs)], - ViewFields = ["key" | ViewFields0], - - {Params} = get_value(query_params, Options, {[]}), - [{"filter", ?b2l(FilterName)} | lists:foldl( - fun({K, V}, QSAcc) -> - Ks = couch_util:to_list(K), - case lists:keymember(Ks, 1, QSAcc) of - true -> - QSAcc; - false when FilterName =:= <<"_view">> -> - V1 = case lists:member(Ks, ViewFields) of - true -> ?JSON_ENCODE(V); - false -> couch_util:to_list(V) + undefined -> + BaseQS; + FilterName -> + %% get list of view attributes + ViewFields0 = [atom_to_list(F) || F <- record_info(fields, mrargs)], + ViewFields = ["key" | ViewFields0], + + {Params} = get_value(query_params, Options, {[]}), + [ + {"filter", ?b2l(FilterName)} + | lists:foldl( + fun({K, V}, QSAcc) -> + Ks = couch_util:to_list(K), + case lists:keymember(Ks, 1, QSAcc) of + true -> + QSAcc; + false when FilterName =:= <<"_view">> -> + V1 = + case lists:member(Ks, ViewFields) of + true -> ?JSON_ENCODE(V); + false -> couch_util:to_list(V) + end, + [{Ks, V1} | QSAcc]; + false -> + [{Ks, couch_util:to_list(V)} | QSAcc] + end end, - [{Ks, V1} | QSAcc]; - false -> - [{Ks, couch_util:to_list(V)} | QSAcc] - end - end, - BaseQS, Params)] + BaseQS, + Params + ) + ] end. parse_changes_feed(Options, UserFun, DataStreamFun) -> case get_value(continuous, Options, false) of - true -> - continuous_changes(DataStreamFun, UserFun); - false -> - EventFun = fun(Ev) -> - changes_ev1(Ev, fun(DocInfo, _) -> UserFun(DocInfo) end, []) - end, - json_stream_parse:events(DataStreamFun, EventFun) + true -> + continuous_changes(DataStreamFun, UserFun); + false -> + EventFun = fun(Ev) -> + changes_ev1(Ev, fun(DocInfo, _) -> UserFun(DocInfo) end, []) + end, + json_stream_parse:events(DataStreamFun, EventFun) end. - options_to_query_args(HttpDb, Path, Options0) -> case lists:keytake(max_url_len, 1, Options0) of - false -> MaxLen = ?MAX_URL_LEN, Options = Options0; - {value, {max_url_len, MaxLen}, Options} -> ok + false -> + MaxLen = ?MAX_URL_LEN, + Options = Options0; + {value, {max_url_len, MaxLen}, Options} -> + ok end, case lists:keytake(atts_since, 1, Options) of - false -> - options_to_query_args(Options, []); - {value, {atts_since, []}, Options2} -> - options_to_query_args(Options2, []); - {value, {atts_since, PAs}, Options2} -> - QueryArgs1 = options_to_query_args(Options2, []), - FullUrl = couch_replicator_httpc:full_url( - HttpDb, [{path, Path}, {qs, QueryArgs1}]), - RevList = atts_since_arg( - length("GET " ++ FullUrl ++ " HTTP/1.1\r\n") + - length("&atts_since=") + 6, % +6 = % encoded [ and ] - PAs, MaxLen, []), - [{"atts_since", ?b2l(iolist_to_binary(?JSON_ENCODE(RevList)))} | QueryArgs1] + false -> + options_to_query_args(Options, []); + {value, {atts_since, []}, Options2} -> + options_to_query_args(Options2, []); + {value, {atts_since, PAs}, Options2} -> + QueryArgs1 = options_to_query_args(Options2, []), + FullUrl = couch_replicator_httpc:full_url( + HttpDb, [{path, Path}, {qs, QueryArgs1}] + ), + RevList = atts_since_arg( + length("GET " ++ FullUrl ++ " HTTP/1.1\r\n") + + % +6 = % encoded [ and ] + length("&atts_since=") + 6, + PAs, + MaxLen, + [] + ), + [{"atts_since", ?b2l(iolist_to_binary(?JSON_ENCODE(RevList)))} | QueryArgs1] end. - options_to_query_args([], Acc) -> lists:reverse(Acc); options_to_query_args([ejson_body | Rest], Acc) -> @@ -570,22 +656,22 @@ atts_since_arg(_UrlLen, [], _MaxLen, Acc) -> lists:reverse(Acc); atts_since_arg(UrlLen, [PA | Rest], MaxLen, Acc) -> RevStr = couch_doc:rev_to_str(PA), - NewUrlLen = case Rest of - [] -> - % plus 2 double quotes (% encoded) - UrlLen + size(RevStr) + 6; - _ -> - % plus 2 double quotes and a comma (% encoded) - UrlLen + size(RevStr) + 9 - end, + NewUrlLen = + case Rest of + [] -> + % plus 2 double quotes (% encoded) + UrlLen + size(RevStr) + 6; + _ -> + % plus 2 double quotes and a comma (% encoded) + UrlLen + size(RevStr) + 9 + end, case NewUrlLen >= MaxLen of - true -> - lists:reverse(Acc); - false -> - atts_since_arg(NewUrlLen, Rest, MaxLen, [RevStr | Acc]) + true -> + lists:reverse(Acc); + false -> + atts_since_arg(NewUrlLen, Rest, MaxLen, [RevStr | Acc]) end. - % TODO: A less verbose, more elegant and automatic restart strategy for % the exported open_doc_revs/6 function. The restart should be % transparent to the caller like any other Couch API function exported @@ -596,51 +682,55 @@ receive_docs_loop(Streamer, Fun, Id, Revs, Ref, Acc) -> erlang:put(open_doc_revs, {Id, Revs, Ref, Streamer}), receive_docs(Streamer, Fun, Ref, Acc) catch - error:{restart_open_doc_revs, NewRef} -> - receive_docs_loop(Streamer, Fun, Id, Revs, NewRef, Acc) + error:{restart_open_doc_revs, NewRef} -> + receive_docs_loop(Streamer, Fun, Id, Revs, NewRef, Acc) end. receive_docs(Streamer, UserFun, Ref, UserAcc) -> Streamer ! {get_headers, Ref, self()}, receive - {started_open_doc_revs, NewRef} -> - restart_remote_open_doc_revs(Ref, NewRef); - {headers, Ref, Headers} -> - case header_value("content-type", Headers) of - {"multipart/related", _} = ContentType -> - % Skip document body and attachment size limits validation here - % since these should be validated by the replication target - case couch_doc:doc_from_multi_part_stream( - ContentType, - fun() -> receive_doc_data(Streamer, Ref) end, - Ref, _ValidateDocLimits = false) of - {ok, Doc, WaitFun, Parser} -> - case run_user_fun(UserFun, {ok, Doc}, UserAcc, Ref) of - {ok, UserAcc2} -> - ok; - {skip, UserAcc2} -> - couch_httpd_multipart:abort_multipart_stream(Parser) - end, - WaitFun(), - receive_docs(Streamer, UserFun, Ref, UserAcc2) + {started_open_doc_revs, NewRef} -> + restart_remote_open_doc_revs(Ref, NewRef); + {headers, Ref, Headers} -> + case header_value("content-type", Headers) of + {"multipart/related", _} = ContentType -> + % Skip document body and attachment size limits validation here + % since these should be validated by the replication target + case + couch_doc:doc_from_multi_part_stream( + ContentType, + fun() -> receive_doc_data(Streamer, Ref) end, + Ref, + _ValidateDocLimits = false + ) + of + {ok, Doc, WaitFun, Parser} -> + case run_user_fun(UserFun, {ok, Doc}, UserAcc, Ref) of + {ok, UserAcc2} -> + ok; + {skip, UserAcc2} -> + couch_httpd_multipart:abort_multipart_stream(Parser) + end, + WaitFun(), + receive_docs(Streamer, UserFun, Ref, UserAcc2) + end; + {"application/json", []} -> + Doc = couch_doc:from_json_obj( + ?JSON_DECODE(receive_all(Streamer, Ref, [])) + ), + {_, UserAcc2} = run_user_fun(UserFun, {ok, Doc}, UserAcc, Ref), + receive_docs(Streamer, UserFun, Ref, UserAcc2); + {"application/json", [{"error", "true"}]} -> + {ErrorProps} = ?JSON_DECODE(receive_all(Streamer, Ref, [])), + Rev = get_value(<<"missing">>, ErrorProps), + Result = {{not_found, missing}, couch_doc:parse_rev(Rev)}, + {_, UserAcc2} = run_user_fun(UserFun, Result, UserAcc, Ref), + receive_docs(Streamer, UserFun, Ref, UserAcc2) end; - {"application/json", []} -> - Doc = couch_doc:from_json_obj( - ?JSON_DECODE(receive_all(Streamer, Ref, []))), - {_, UserAcc2} = run_user_fun(UserFun, {ok, Doc}, UserAcc, Ref), - receive_docs(Streamer, UserFun, Ref, UserAcc2); - {"application/json", [{"error","true"}]} -> - {ErrorProps} = ?JSON_DECODE(receive_all(Streamer, Ref, [])), - Rev = get_value(<<"missing">>, ErrorProps), - Result = {{not_found, missing}, couch_doc:parse_rev(Rev)}, - {_, UserAcc2} = run_user_fun(UserFun, Result, UserAcc, Ref), - receive_docs(Streamer, UserFun, Ref, UserAcc2) - end; - {done, Ref} -> - {ok, UserAcc} + {done, Ref} -> + {ok, UserAcc} end. - run_user_fun(UserFun, Arg, UserAcc, OldRef) -> {Pid, Ref} = spawn_monitor(fun() -> try UserFun(Arg, UserAcc) of @@ -670,78 +760,76 @@ run_user_fun(UserFun, Arg, UserAcc, OldRef) -> erlang:exit(Reason) end. - restart_remote_open_doc_revs(Ref, NewRef) -> receive - {body_bytes, Ref, _} -> - restart_remote_open_doc_revs(Ref, NewRef); - {body_done, Ref} -> - restart_remote_open_doc_revs(Ref, NewRef); - {done, Ref} -> - restart_remote_open_doc_revs(Ref, NewRef); - {headers, Ref, _} -> - restart_remote_open_doc_revs(Ref, NewRef) + {body_bytes, Ref, _} -> + restart_remote_open_doc_revs(Ref, NewRef); + {body_done, Ref} -> + restart_remote_open_doc_revs(Ref, NewRef); + {done, Ref} -> + restart_remote_open_doc_revs(Ref, NewRef); + {headers, Ref, _} -> + restart_remote_open_doc_revs(Ref, NewRef) after 0 -> erlang:error({restart_open_doc_revs, NewRef}) end. - remote_open_doc_revs_streamer_start(Parent) -> receive - {get_headers, _Ref, Parent} -> - remote_open_doc_revs_streamer_start(Parent); - {next_bytes, _Ref, Parent} -> - remote_open_doc_revs_streamer_start(Parent) + {get_headers, _Ref, Parent} -> + remote_open_doc_revs_streamer_start(Parent); + {next_bytes, _Ref, Parent} -> + remote_open_doc_revs_streamer_start(Parent) after 0 -> Parent ! {started_open_doc_revs, make_ref()} end. - receive_all(Streamer, Ref, Acc) -> Streamer ! {next_bytes, Ref, self()}, receive - {started_open_doc_revs, NewRef} -> - restart_remote_open_doc_revs(Ref, NewRef); - {body_bytes, Ref, Bytes} -> - receive_all(Streamer, Ref, [Bytes | Acc]); - {body_done, Ref} -> - lists:reverse(Acc) + {started_open_doc_revs, NewRef} -> + restart_remote_open_doc_revs(Ref, NewRef); + {body_bytes, Ref, Bytes} -> + receive_all(Streamer, Ref, [Bytes | Acc]); + {body_done, Ref} -> + lists:reverse(Acc) end. - mp_parse_mixed(eof) -> - receive {get_headers, Ref, From} -> - From ! {done, Ref} + receive + {get_headers, Ref, From} -> + From ! {done, Ref} end; mp_parse_mixed({headers, H}) -> - receive {get_headers, Ref, From} -> - From ! {headers, Ref, H} + receive + {get_headers, Ref, From} -> + From ! {headers, Ref, H} end, fun mp_parse_mixed/1; mp_parse_mixed({body, Bytes}) -> - receive {next_bytes, Ref, From} -> - From ! {body_bytes, Ref, Bytes} + receive + {next_bytes, Ref, From} -> + From ! {body_bytes, Ref, Bytes} end, fun mp_parse_mixed/1; mp_parse_mixed(body_end) -> - receive {next_bytes, Ref, From} -> - From ! {body_done, Ref}; - {get_headers, Ref, From} -> - self() ! {get_headers, Ref, From} + receive + {next_bytes, Ref, From} -> + From ! {body_done, Ref}; + {get_headers, Ref, From} -> + self() ! {get_headers, Ref, From} end, fun mp_parse_mixed/1. - receive_doc_data(Streamer, Ref) -> Streamer ! {next_bytes, Ref, self()}, receive - {body_bytes, Ref, Bytes} -> - {Bytes, fun() -> receive_doc_data(Streamer, Ref) end}; - {body_done, Ref} -> - {<<>>, fun() -> receive_doc_data(Streamer, Ref) end} + {body_bytes, Ref, Bytes} -> + {Bytes, fun() -> receive_doc_data(Streamer, Ref) end}; + {body_done, Ref} -> + {<<>>, fun() -> receive_doc_data(Streamer, Ref) end} end. - changes_ev1(object_start, UserFun, UserAcc) -> fun(Ev) -> changes_ev2(Ev, UserFun, UserAcc) end. @@ -755,11 +843,13 @@ changes_ev3(array_start, UserFun, UserAcc) -> changes_ev_loop(object_start, UserFun, UserAcc) -> fun(Ev) -> - json_stream_parse:collect_object(Ev, + json_stream_parse:collect_object( + Ev, fun(Obj) -> UserAcc2 = UserFun(json_to_doc_info(Obj), UserAcc), fun(Ev2) -> changes_ev_loop(Ev2, UserFun, UserAcc2) end - end) + end + ) end; changes_ev_loop(array_end, _UserFun, _UserAcc) -> fun(_Ev) -> changes_ev_done() end. @@ -770,80 +860,106 @@ changes_ev_done() -> continuous_changes(DataFun, UserFun) -> {DataFun2, _, Rest} = json_stream_parse:events( DataFun, - fun(Ev) -> parse_changes_line(Ev, UserFun) end), + fun(Ev) -> parse_changes_line(Ev, UserFun) end + ), continuous_changes(fun() -> {Rest, DataFun2} end, UserFun). parse_changes_line(object_start, UserFun) -> fun(Ev) -> - json_stream_parse:collect_object(Ev, - fun(Obj) -> UserFun(json_to_doc_info(Obj)) end) + json_stream_parse:collect_object( + Ev, + fun(Obj) -> UserFun(json_to_doc_info(Obj)) end + ) end. json_to_doc_info({Props}) -> case get_value(<<"changes">>, Props) of - undefined -> - {last_seq, get_value(<<"last_seq">>, Props)}; - Changes -> - RevsInfo0 = lists:map( - fun({Change}) -> - Rev = couch_doc:parse_rev(get_value(<<"rev">>, Change)), - Del = couch_replicator_utils:is_deleted(Change), - #rev_info{rev=Rev, deleted=Del} - end, Changes), - - RevsInfo = case get_value(<<"removed">>, Props) of - true -> - [_ | RevsInfo1] = RevsInfo0, - RevsInfo1; - _ -> - RevsInfo0 - end, + undefined -> + {last_seq, get_value(<<"last_seq">>, Props)}; + Changes -> + RevsInfo0 = lists:map( + fun({Change}) -> + Rev = couch_doc:parse_rev(get_value(<<"rev">>, Change)), + Del = couch_replicator_utils:is_deleted(Change), + #rev_info{rev = Rev, deleted = Del} + end, + Changes + ), - #doc_info{ - id = get_value(<<"id">>, Props), - high_seq = get_value(<<"seq">>, Props), - revs = RevsInfo - } + RevsInfo = + case get_value(<<"removed">>, Props) of + true -> + [_ | RevsInfo1] = RevsInfo0, + RevsInfo1; + _ -> + RevsInfo0 + end, + + #doc_info{ + id = get_value(<<"id">>, Props), + high_seq = get_value(<<"seq">>, Props), + revs = RevsInfo + } end. bulk_results_to_errors(Docs, {ok, Results}, interactive_edit) -> - lists:reverse(lists:foldl( - fun({_, {ok, _}}, Acc) -> - Acc; - ({#doc{id = Id, revs = {Pos, [RevId | _]}}, Error}, Acc) -> - {_, Error, Reason} = couch_httpd:error_info(Error), - [ {[{id, Id}, {rev, rev_to_str({Pos, RevId})}, - {error, Error}, {reason, Reason}]} | Acc ] - end, - [], lists:zip(Docs, Results))); - + lists:reverse( + lists:foldl( + fun + ({_, {ok, _}}, Acc) -> + Acc; + ({#doc{id = Id, revs = {Pos, [RevId | _]}}, Error}, Acc) -> + {_, Error, Reason} = couch_httpd:error_info(Error), + [ + {[ + {id, Id}, + {rev, rev_to_str({Pos, RevId})}, + {error, Error}, + {reason, Reason} + ]} + | Acc + ] + end, + [], + lists:zip(Docs, Results) + ) + ); bulk_results_to_errors(Docs, {ok, Results}, replicated_changes) -> bulk_results_to_errors(Docs, {aborted, Results}, interactive_edit); - bulk_results_to_errors(_Docs, {aborted, Results}, interactive_edit) -> lists:map( fun({{Id, Rev}, Err}) -> {_, Error, Reason} = couch_httpd:error_info(Err), {[{id, Id}, {rev, rev_to_str(Rev)}, {error, Error}, {reason, Reason}]} end, - Results); - + Results + ); bulk_results_to_errors(_Docs, Results, remote) -> - lists:reverse(lists:foldl( - fun({Props}, Acc) -> - case get_value(<<"error">>, Props, get_value(error, Props)) of - undefined -> - Acc; - Error -> - Id = get_value(<<"id">>, Props, get_value(id, Props)), - Rev = get_value(<<"rev">>, Props, get_value(rev, Props)), - Reason = get_value(<<"reason">>, Props, get_value(reason, Props)), - [ {[{id, Id}, {rev, rev_to_str(Rev)}, - {error, Error}, {reason, Reason}]} | Acc ] - end - end, - [], Results)). - + lists:reverse( + lists:foldl( + fun({Props}, Acc) -> + case get_value(<<"error">>, Props, get_value(error, Props)) of + undefined -> + Acc; + Error -> + Id = get_value(<<"id">>, Props, get_value(id, Props)), + Rev = get_value(<<"rev">>, Props, get_value(rev, Props)), + Reason = get_value(<<"reason">>, Props, get_value(reason, Props)), + [ + {[ + {id, Id}, + {rev, rev_to_str(Rev)}, + {error, Error}, + {reason, Reason} + ]} + | Acc + ] + end + end, + [], + Results + ) + ). rev_to_str({_Pos, _Id} = Rev) -> couch_doc:rev_to_str(Rev); @@ -852,18 +968,19 @@ rev_to_str(Rev) -> write_fun() -> fun(Data) -> - receive {get_data, Ref, From} -> - From ! {data, Ref, Data} + receive + {get_data, Ref, From} -> + From ! {data, Ref, Data} end end. stream_doc({JsonBytes, Atts, Boundary, Len}) -> case erlang:erase({doc_streamer, Boundary}) of - Pid when is_pid(Pid) -> - unlink(Pid), - exit(Pid, kill); - _ -> - ok + Pid when is_pid(Pid) -> + unlink(Pid), + exit(Pid, kill); + _ -> + ok end, DocStreamer = spawn_link( couch_doc, @@ -878,8 +995,9 @@ stream_doc({0, Id}) -> stream_doc({LenLeft, Id}) when LenLeft > 0 -> Ref = make_ref(), erlang:get({doc_streamer, Id}) ! {get_data, Ref, self()}, - receive {data, Ref, Data} -> - {ok, Data, {LenLeft - iolist_size(Data), Id}} + receive + {data, Ref, Data} -> + {ok, Data, {LenLeft - iolist_size(Data), Id}} end. header_value(Key, Headers) -> @@ -894,7 +1012,6 @@ header_value(Key, Headers, Default) -> Default end. - % Normalize an #httpdb{} or #db{} record such that it can be used for % comparisons. This means remove things like pids and also sort options / props. normalize_db(#httpdb{} = HttpDb) -> @@ -907,40 +1024,34 @@ normalize_db(#httpdb{} = HttpDb) -> retries = HttpDb#httpdb.retries, http_connections = HttpDb#httpdb.http_connections }; - normalize_db(<<DbName/binary>>) -> DbName. - maybe_append_create_query_params(Db, []) -> Db; - maybe_append_create_query_params(Db, CreateParams) -> NewUrl = Db#httpdb.url ++ "?" ++ mochiweb_util:urlencode(CreateParams), Db#httpdb{url = NewUrl}. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). - normalize_http_db_test() -> - HttpDb = #httpdb{ + HttpDb = #httpdb{ url = "http://host/db", auth_props = [{"key", "val"}], - headers = [{"k2","v2"}, {"k1","v1"}], + headers = [{"k2", "v2"}, {"k1", "v1"}], timeout = 30000, ibrowse_options = [{k2, v2}, {k1, v1}], retries = 10, http_connections = 20 }, Expected = HttpDb#httpdb{ - headers = [{"k1","v1"}, {"k2","v2"}], + headers = [{"k1", "v1"}, {"k2", "v2"}], ibrowse_options = [{k1, v1}, {k2, v2}] }, ?assertEqual(Expected, normalize_db(HttpDb)), ?assertEqual(<<"local">>, normalize_db(<<"local">>)). - -endif. diff --git a/src/couch_replicator/src/couch_replicator_auth.erl b/src/couch_replicator/src/couch_replicator_auth.erl index 272e10af5..e5c024f7e 100644 --- a/src/couch_replicator/src/couch_replicator_auth.erl +++ b/src/couch_replicator/src/couch_replicator_auth.erl @@ -12,7 +12,6 @@ -module(couch_replicator_auth). - -export([ initialize/1, update_headers/2, @@ -20,17 +19,13 @@ cleanup/1 ]). - -include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl"). - -type headers() :: [{string(), string()}]. -type code() :: non_neg_integer(). - -define(DEFAULT_PLUGINS, "couch_replicator_auth_session,couch_replicator_auth_noop"). - % Behavior API % Note for plugin developers: consider using the "auth" field in the source and @@ -49,7 +44,6 @@ -callback cleanup(term()) -> ok. - % Main API -spec initialize(#httpdb{}) -> {ok, #httpdb{}} | {error, term()}. @@ -61,13 +55,11 @@ initialize(#httpdb{auth_context = nil} = HttpDb) -> {error, Error} end. - -spec update_headers(#httpdb{}, headers()) -> {headers(), #httpdb{}}. update_headers(#httpdb{auth_context = {Mod, Context}} = HttpDb, Headers) -> {Headers1, Context1} = Mod:update_headers(Context, Headers), {Headers1, HttpDb#httpdb{auth_context = {Mod, Context1}}}. - -spec handle_response(#httpdb{}, code(), headers()) -> {continue | retry, term()}. handle_response(#httpdb{} = HttpDb, Code, Headers) -> @@ -75,13 +67,11 @@ handle_response(#httpdb{} = HttpDb, Code, Headers) -> {Res, Context1} = Mod:handle_response(Context, Code, Headers), {Res, HttpDb#httpdb{auth_context = {Mod, Context1}}}. - -spec cleanup(#httpdb{}) -> #httpdb{}. cleanup(#httpdb{auth_context = {Module, Context}} = HttpDb) -> ok = Module:cleanup(Context), HttpDb#httpdb{auth_context = nil}. - % Private helper functions -spec get_plugin_modules() -> [atom()]. @@ -89,7 +79,6 @@ get_plugin_modules() -> Plugins1 = config:get("replicator", "auth_plugins", ?DEFAULT_PLUGINS), [list_to_atom(Plugin) || Plugin <- string:tokens(Plugins1, ",")]. - try_initialize([], _HttpDb) -> {error, no_more_auth_plugins_left_to_try}; try_initialize([Mod | Modules], HttpDb) -> diff --git a/src/couch_replicator/src/couch_replicator_auth_noop.erl b/src/couch_replicator/src/couch_replicator_auth_noop.erl index 5dbf13335..e2a7ee839 100644 --- a/src/couch_replicator/src/couch_replicator_auth_noop.erl +++ b/src/couch_replicator/src/couch_replicator_auth_noop.erl @@ -12,10 +12,8 @@ -module(couch_replicator_auth_noop). - -behavior(couch_replicator_auth). - -export([ initialize/1, update_headers/2, @@ -23,30 +21,24 @@ cleanup/1 ]). - -include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl"). - -type headers() :: [{string(), string()}]. -type code() :: non_neg_integer(). - -spec initialize(#httpdb{}) -> {ok, #httpdb{}, term()} | ignore. initialize(#httpdb{} = HttpDb) -> {ok, HttpDb, nil}. - -spec update_headers(term(), headers()) -> {headers(), term()}. update_headers(Context, Headers) -> {Headers, Context}. - -spec handle_response(term(), code(), headers()) -> {continue | retry, term()}. handle_response(Context, _Code, _Headers) -> {continue, Context}. - -spec cleanup(term()) -> ok. cleanup(_Context) -> ok. diff --git a/src/couch_replicator/src/couch_replicator_auth_session.erl b/src/couch_replicator/src/couch_replicator_auth_session.erl index 4f70cd668..d29600706 100644 --- a/src/couch_replicator/src/couch_replicator_auth_session.erl +++ b/src/couch_replicator/src/couch_replicator_auth_session.erl @@ -10,7 +10,6 @@ % License for the specific language governing permissions and limitations under % the License. - % This is the replicator session auth plugin. It implements session based % authentication for the replicator. The only public API are the functions from % the couch_replicator_auth behaviour. Most of the logic and state is in the @@ -46,14 +45,11 @@ % ensure if something goes wrong and one of the endpoints issues invalid % cookies, replicator won't be stuck in a busy loop refreshing them. - -module(couch_replicator_auth_session). - -behaviour(couch_replicator_auth). -behaviour(gen_server). - -export([ initialize/1, update_headers/2, @@ -71,11 +67,9 @@ format_status/2 ]). - -include_lib("ibrowse/include/ibrowse.hrl"). -include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl"). - -type headers() :: [{string(), string()}]. -type code() :: non_neg_integer(). -type time_sec() :: non_neg_integer(). @@ -84,7 +78,6 @@ -define(MIN_UPDATE_INTERVAL_SEC, 5). -define(DEFAULT_REFRESH_INTERVAL_SEC, 550). - -record(state, { epoch = 0 :: non_neg_integer(), cookie :: string() | undefined, @@ -94,12 +87,11 @@ httpdb_pool :: pid(), httpdb_ibrowse_options = [] :: list(), session_url :: string(), - next_refresh = infinity :: infinity | non_neg_integer(), + next_refresh = infinity :: infinity | non_neg_integer(), refresh_tstamp = 0 :: non_neg_integer(), require_valid_user = false :: boolean() }). - % Behavior API callbacks -spec initialize(#httpdb{}) -> @@ -117,37 +109,31 @@ initialize(#httpdb{} = HttpDb) -> ignore end. - -spec update_headers(term(), headers()) -> {headers(), term()}. update_headers({Pid, Epoch, Timeout}, Headers) -> Args = {update_headers, Headers, Epoch}, {Headers1, Epoch1} = gen_server:call(Pid, Args, Timeout * 10), {Headers1, {Pid, Epoch1, Timeout}}. - -spec handle_response(term(), code(), headers()) -> {continue | retry, term()}. handle_response({Pid, Epoch, Timeout}, Code, Headers) -> - Args = {handle_response, Code, Headers, Epoch}, + Args = {handle_response, Code, Headers, Epoch}, {Retry, Epoch1} = gen_server:call(Pid, Args, Timeout * 10), {Retry, {Pid, Epoch1, Timeout}}. - -spec cleanup(term()) -> ok. cleanup({Pid, _Epoch, Timeout}) -> gen_server:call(Pid, stop, Timeout * 10). - %% gen_server functions init([#state{} = State]) -> {ok, State}. - terminate(_Reason, _State) -> ok. - handle_call({update_headers, Headers, _Epoch}, _From, State) -> case maybe_refresh(State) of {ok, State1} -> @@ -159,29 +145,23 @@ handle_call({update_headers, Headers, _Epoch}, _From, State) -> couch_log:error(LogMsg, [?MODULE, Error]), {stop, Error, State} end; - handle_call({handle_response, Code, Headers, Epoch}, _From, State) -> {Retry, State1} = process_response(Code, Headers, Epoch, State), {reply, {Retry, State1#state.epoch}, State1}; - handle_call(stop, _From, State) -> {stop, normal, ok, State}. - handle_cast(Msg, State) -> couch_log:error("~p: Received un-expected cast ~p", [?MODULE, Msg]), {noreply, State}. - handle_info(Msg, State) -> couch_log:error("~p : Received un-expected message ~p", [?MODULE, Msg]), {noreply, State}. - code_change(_OldVsn, State, _Extra) -> {ok, State}. - format_status(_Opt, [_PDict, State]) -> [ {epoch, State#state.epoch}, @@ -190,10 +170,8 @@ format_status(_Opt, [_PDict, State]) -> {refresh_tstamp, State#state.refresh_tstamp} ]. - %% Private helper functions - -spec init_state(#httpdb{}) -> {ok, #httpdb{}, #state{}} | {error, term()} | ignore. init_state(#httpdb{} = HttpDb) -> @@ -234,7 +212,6 @@ init_state(#httpdb{} = HttpDb) -> {error, Error} end. - -spec extract_creds(#httpdb{}) -> {ok, string(), string(), #httpdb{}} | {error, term()}. extract_creds(#httpdb{} = HttpDb) -> @@ -248,9 +225,12 @@ extract_creds(#httpdb{} = HttpDb) -> {ok, User, Pass, HttpDb1} end. - --spec process_response(non_neg_integer(), headers(), - non_neg_integer(), #state{}) -> {retry | continue, #state{}}. +-spec process_response( + non_neg_integer(), + headers(), + non_neg_integer(), + #state{} +) -> {retry | continue, #state{}}. process_response(403, _Headers, Epoch, State) -> process_auth_failure(Epoch, State); process_response(401, _Headers, Epoch, State) -> @@ -258,25 +238,26 @@ process_response(401, _Headers, Epoch, State) -> process_response(Code, Headers, _Epoch, State) when Code >= 200, Code < 300 -> % If server noticed cookie is about to time out it can send a new cookie in % the response headers. Take advantage of that and refresh the cookie. - State1 = case maybe_update_cookie(Headers, State) of - {ok, UpdatedState} -> - UpdatedState; - {error, cookie_not_found} -> - State; - {error, Other} -> - LogMsg = "~p : Could not parse cookie from response headers ~p", - couch_log:error(LogMsg, [?MODULE, Other]), - State - end, + State1 = + case maybe_update_cookie(Headers, State) of + {ok, UpdatedState} -> + UpdatedState; + {error, cookie_not_found} -> + State; + {error, Other} -> + LogMsg = "~p : Could not parse cookie from response headers ~p", + couch_log:error(LogMsg, [?MODULE, Other]), + State + end, {continue, State1}; process_response(_Code, _Headers, _Epoch, State) -> {continue, State}. - -spec process_auth_failure(non_neg_integer(), #state{}) -> {retry | continue, #state{}}. -process_auth_failure(Epoch, #state{epoch = StateEpoch} = State) - when StateEpoch > Epoch -> +process_auth_failure(Epoch, #state{epoch = StateEpoch} = State) when + StateEpoch > Epoch +-> % This request used an outdated cookie, tell it to immediately retry % and it will pick up the current cookie when its headers are updated {retry, State}; @@ -293,7 +274,6 @@ process_auth_failure(Epoch, #state{epoch = Epoch} = State) -> {retry, schedule_refresh(now_sec(), State)} end. - -spec get_session_url(string()) -> string(). get_session_url(Url) -> #url{ @@ -311,14 +291,12 @@ get_session_url(Url) -> lists:concat([Proto, "://", Host, "/_session"]) end. - -spec schedule_refresh(non_neg_integer(), #state{}) -> #state{}. schedule_refresh(T, #state{next_refresh = Tc} = State) when T < Tc -> State#state{next_refresh = T}; schedule_refresh(_, #state{} = State) -> State. - -spec maybe_refresh(#state{}) -> {ok, #state{}} | {error, term()}. maybe_refresh(#state{next_refresh = T} = State) -> case now_sec() >= T of @@ -328,21 +306,20 @@ maybe_refresh(#state{next_refresh = T} = State) -> {ok, State} end. - -spec refresh(#state{}) -> {ok, #state{}} | {error, term()}. refresh(#state{session_url = Url, user = User, pass = Pass} = State) -> - Body = mochiweb_util:urlencode([{name, User}, {password, Pass}]), + Body = mochiweb_util:urlencode([{name, User}, {password, Pass}]), Headers0 = [{"Content-Type", "application/x-www-form-urlencoded"}], - Headers = case State#state.require_valid_user of - true -> - Headers0 ++ [{"Authorization", "Basic " ++ b64creds(User, Pass)}]; - false -> - Headers0 - end, + Headers = + case State#state.require_valid_user of + true -> + Headers0 ++ [{"Authorization", "Basic " ++ b64creds(User, Pass)}]; + false -> + Headers0 + end, Result = http_request(State, Url, Headers, post, Body), http_response(Result, State). - -spec http_request(#state{}, string(), headers(), atom(), iolist()) -> {ok, string(), headers(), binary()} | {error, term()}. http_request(#state{httpdb_pool = Pool} = State, Url, Headers, Method, Body) -> @@ -354,20 +331,26 @@ http_request(#state{httpdb_pool = Pool} = State, Url, Headers, Method, Body) -> ], {ok, Wrk} = couch_replicator_httpc_pool:get_worker(Pool), try - Result = ibrowse:send_req_direct(Wrk, Url, Headers, Method, Body, Opts, - Timeout), + Result = ibrowse:send_req_direct( + Wrk, + Url, + Headers, + Method, + Body, + Opts, + Timeout + ), case Result of {ok, _, ResultHeaders, _} -> stop_worker_if_server_requested(ResultHeaders, Wrk); _Other -> ok - end, - Result + end, + Result after ok = couch_replicator_httpc_pool:release_worker_sync(Pool, Wrk) end. - -spec stop_worker_if_server_requested(headers(), pid()) -> ok. stop_worker_if_server_requested(ResultHeaders0, Worker) -> ResultHeaders = mochiweb_headers:make(ResultHeaders0), @@ -383,13 +366,16 @@ stop_worker_if_server_requested(ResultHeaders0, Worker) -> ok end. - --spec http_response({ok, string(), headers(), binary()} | {error, term()}, - #state{}) -> {ok, #state{}} | {error, term()}. +-spec http_response( + {ok, string(), headers(), binary()} | {error, term()}, + #state{} +) -> {ok, #state{}} | {error, term()}. http_response({ok, "200", Headers, _}, State) -> maybe_update_cookie(Headers, State); -http_response({ok, "401", Headers0, _}, #state{session_url = Url, - user = User}) -> +http_response({ok, "401", Headers0, _}, #state{ + session_url = Url, + user = User +}) -> Headers = mochiweb_headers:make(Headers0), case mochiweb_headers:get_value("WWW-Authenticate", Headers) of undefined -> @@ -406,7 +392,6 @@ http_response({ok, Code, _, _}, #state{session_url = Url, user = User}) -> http_response({error, Error}, #state{session_url = Url, user = User}) -> {error, {session_request_failed, Url, User, Error}}. - -spec parse_cookie(list()) -> {ok, age(), string()} | {error, term()}. parse_cookie(Headers0) -> Headers = mochiweb_headers:make(Headers0), @@ -425,14 +410,11 @@ parse_cookie(Headers0) -> end end. - -spec parse_max_age(list()) -> age(). parse_max_age(CaseInsKVs) -> case mochiweb_headers:get_value("Max-Age", CaseInsKVs) of String when is_list(String) -> - try - list_to_integer(String) - of + try list_to_integer(String) of MaxAge when MaxAge >= 0 -> MaxAge; _ -> @@ -445,7 +427,6 @@ parse_max_age(CaseInsKVs) -> undefined end. - -spec maybe_update_cookie(headers(), #state{}) -> {ok, string()} | {error, term()}. maybe_update_cookie(ResponseHeaders, State) -> @@ -456,7 +437,6 @@ maybe_update_cookie(ResponseHeaders, State) -> {error, Error} end. - -spec update_cookie(#state{}, string(), time_sec(), age()) -> #state{}. update_cookie(#state{cookie = Cookie} = State, Cookie, _, _) -> State; @@ -469,72 +449,72 @@ update_cookie(#state{epoch = Epoch} = State, Cookie, NowSec, MaxAge) -> }, schedule_refresh(NextRefresh, NewState). - -spec next_refresh(time_sec(), age(), time_sec()) -> time_sec(). next_refresh(NowSec, undefined, RefreshInterval) -> NowSec + RefreshInterval; - next_refresh(NowSec, MaxAge, _) when is_integer(MaxAge) -> % Apply a fudge factor to account for delays in receving the cookie % and / or time adjustments happening over a longer period of time NowSec + trunc(MaxAge * 0.9). - -spec cookie_age_sec(#state{}, time_sec()) -> time_sec(). cookie_age_sec(#state{refresh_tstamp = RefreshTs}, Now) -> max(0, Now - RefreshTs). - -spec now_sec() -> time_sec(). now_sec() -> {Mega, Sec, _Micro} = os:timestamp(), Mega * 1000000 + Sec. - -spec min_update_interval() -> time_sec(). min_update_interval() -> - config:get_integer("replicator", "session_min_update_interval", - ?MIN_UPDATE_INTERVAL_SEC). - + config:get_integer( + "replicator", + "session_min_update_interval", + ?MIN_UPDATE_INTERVAL_SEC + ). -spec refresh_interval() -> integer(). refresh_interval() -> - config:get_integer("replicator", "session_refresh_interval_sec", - ?DEFAULT_REFRESH_INTERVAL_SEC). - - + config:get_integer( + "replicator", + "session_refresh_interval_sec", + ?DEFAULT_REFRESH_INTERVAL_SEC + ). -spec b64creds(string(), string()) -> string(). b64creds(User, Pass) -> base64:encode_to_string(User ++ ":" ++ Pass). - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). - get_session_url_test_() -> - [?_assertEqual(SessionUrl, get_session_url(Url)) || {Url, SessionUrl} <- [ - {"http://host/db", "http://host/_session"}, - {"http://127.0.0.1/db", "http://127.0.0.1/_session"}, - {"http://host/x/y/z", "http://host/_session"}, - {"http://host:5984/db", "http://host:5984/_session"}, - {"https://host/db?q=1", "https://host/_session"} - ]]. - + [ + ?_assertEqual(SessionUrl, get_session_url(Url)) + || {Url, SessionUrl} <- [ + {"http://host/db", "http://host/_session"}, + {"http://127.0.0.1/db", "http://127.0.0.1/_session"}, + {"http://host/x/y/z", "http://host/_session"}, + {"http://host:5984/db", "http://host:5984/_session"}, + {"https://host/db?q=1", "https://host/_session"} + ] + ]. extract_creds_success_test() -> - HttpDb = #httpdb{auth_props = [ - {<<"basic">>, {[ - {<<"username">>, <<"u2">>}, - {<<"password">>, <<"p2">>} - ]}} - ]}, + HttpDb = #httpdb{ + auth_props = [ + {<<"basic">>, + {[ + {<<"username">>, <<"u2">>}, + {<<"password">>, <<"p2">>} + ]}} + ] + }, ?assertEqual({ok, "u2", "p2", #httpdb{}}, extract_creds(HttpDb)), ?assertEqual({error, missing_credentials}, extract_creds(#httpdb{})). - cookie_update_test_() -> { setup, @@ -562,7 +542,6 @@ cookie_update_test_() -> } }. - t_do_refresh_without_max_age() -> ?_test(begin State = #state{next_refresh = 0}, @@ -573,7 +552,6 @@ t_do_refresh_without_max_age() -> ?assert(540 < RefreshInterval andalso RefreshInterval =< 550) end). - t_do_refresh_with_max_age() -> ?_test(begin State = #state{next_refresh = 0}, @@ -585,7 +563,6 @@ t_do_refresh_with_max_age() -> ?assert(80 < RefreshInterval andalso RefreshInterval =< 90) end). - t_dont_refresh() -> ?_test(begin State = #state{ @@ -602,7 +579,6 @@ t_dont_refresh() -> ?assertMatch(State2, State3) end). - t_process_auth_failure() -> ?_test(begin State = #state{epoch = 1, refresh_tstamp = 0}, @@ -611,21 +587,18 @@ t_process_auth_failure() -> ?assert(NextRefresh =< now_sec()) end). - t_process_auth_failure_stale_epoch() -> ?_test(begin State = #state{epoch = 3}, ?assertMatch({retry, State}, process_auth_failure(2, State)) end). - t_process_auth_failure_too_frequent() -> ?_test(begin State = #state{epoch = 4, refresh_tstamp = now_sec()}, ?assertMatch({continue, _}, process_auth_failure(4, State)) end). - t_process_ok_update_cookie() -> ?_test(begin Headers = [{"set-CookiE", "AuthSession=xyz; Path=/;"}, {"X", "y"}], @@ -636,7 +609,6 @@ t_process_ok_update_cookie() -> ?assertMatch({continue, #state{cookie = "xyz", epoch = 2}}, Res2) end). - t_process_ok_no_cookie() -> ?_test(begin Headers = [{"X", "y"}], @@ -645,37 +617,34 @@ t_process_ok_no_cookie() -> ?assertMatch({continue, State}, Res) end). - t_init_state_fails_on_401() -> ?_test(begin mock_http_401_response(), {error, Error} = init_state(httpdb("http://u:p@h")), - SessionUrl = "http://h/_session", + SessionUrl = "http://h/_session", ?assertEqual({session_request_unauthorized, SessionUrl, "u"}, Error) end). - t_init_state_401_with_require_valid_user() -> ?_test(begin mock_http_401_response_with_require_valid_user(), - ?assertMatch({ok, #httpdb{}, #state{cookie = "Cookie"}}, - init_state(httpdb("http://u:p@h"))) + ?assertMatch( + {ok, #httpdb{}, #state{cookie = "Cookie"}}, + init_state(httpdb("http://u:p@h")) + ) end). - t_init_state_404() -> ?_test(begin mock_http_404_response(), ?assertEqual(ignore, init_state(httpdb("http://u:p@h"))) end). - t_init_state_no_creds() -> ?_test(begin ?_assertEqual(ignore, init_state(httpdb("http://h"))) end). - t_init_state_http_error() -> ?_test(begin mock_http_error_response(), @@ -684,11 +653,9 @@ t_init_state_http_error() -> ?assertEqual({session_request_failed, SessionUrl, "u", x}, Error) end). - httpdb(Url) -> couch_replicator_utils:normalize_basic_auth(#httpdb{url = Url}). - setup_all() -> meck:expect(couch_replicator_httpc_pool, get_worker, 1, {ok, worker}), meck:expect(couch_replicator_httpc_pool, release_worker_sync, 2, ok), @@ -696,11 +663,9 @@ setup_all() -> mock_http_cookie_response("Abc"), ok. - teardown_all(_) -> meck:unload(). - setup() -> meck:reset([ config, @@ -708,44 +673,37 @@ setup() -> ibrowse ]). - teardown(_) -> ok. - mock_http_cookie_response(Cookie) -> Resp = {ok, "200", [{"Set-Cookie", "AuthSession=" ++ Cookie}], []}, meck:expect(ibrowse, send_req_direct, 7, Resp). - mock_http_cookie_response_with_age(Cookie, Age) -> AgeKV = "Max-Age=" ++ Age, CookieKV = "AuthSession=" ++ Cookie, Resp = {ok, "200", [{"Set-Cookie", CookieKV ++ ";" ++ AgeKV}], []}, meck:expect(ibrowse, send_req_direct, 7, Resp). - mock_http_401_response() -> meck:expect(ibrowse, send_req_direct, 7, {ok, "401", [], []}). - mock_http_401_response_with_require_valid_user() -> Resp1 = {ok, "401", [{"WWW-Authenticate", "Basic realm=\"server\""}], []}, Resp2 = {ok, "200", [{"Set-Cookie", "AuthSession=Cookie"}], []}, meck:expect(ibrowse, send_req_direct, 7, meck:seq([Resp1, Resp2])). - mock_http_404_response() -> meck:expect(ibrowse, send_req_direct, 7, {ok, "404", [], []}). - mock_http_error_response() -> meck:expect(ibrowse, send_req_direct, 7, {error, x}). - parse_max_age_test_() -> - [?_assertEqual(R, parse_max_age(mochiweb_headers:make([{"Max-Age", A}]))) - || {A, R} <- [ + [ + ?_assertEqual(R, parse_max_age(mochiweb_headers:make([{"Max-Age", A}]))) + || {A, R} <- [ {"-10", undefined}, {"\ufeff", undefined}, {"*", undefined}, @@ -758,5 +716,4 @@ parse_max_age_test_() -> ] ]. - -endif. diff --git a/src/couch_replicator/src/couch_replicator_changes_reader.erl b/src/couch_replicator/src/couch_replicator_changes_reader.erl index 2e4df5365..83080b6fb 100644 --- a/src/couch_replicator/src/couch_replicator_changes_reader.erl +++ b/src/couch_replicator/src/couch_replicator_changes_reader.erl @@ -28,25 +28,37 @@ start_link(StartSeq, #httpdb{} = Db, ChangesQueue, Options) -> Parent = self(), - {ok, spawn_link(fun() -> - put(last_seq, StartSeq), - put(retries_left, Db#httpdb.retries), - ?MODULE:read_changes(Parent, StartSeq, Db#httpdb{retries = 0}, - ChangesQueue, Options) - end)}; + {ok, + spawn_link(fun() -> + put(last_seq, StartSeq), + put(retries_left, Db#httpdb.retries), + ?MODULE:read_changes( + Parent, + StartSeq, + Db#httpdb{retries = 0}, + ChangesQueue, + Options + ) + end)}; start_link(StartSeq, Db, ChangesQueue, Options) -> Parent = self(), - {ok, spawn_link(fun() -> - ?MODULE:read_changes(Parent, StartSeq, Db, ChangesQueue, Options) - end)}. + {ok, + spawn_link(fun() -> + ?MODULE:read_changes(Parent, StartSeq, Db, ChangesQueue, Options) + end)}. read_changes(Parent, StartSeq, Db, ChangesQueue, Options) -> Continuous = couch_util:get_value(continuous, Options), try - couch_replicator_api_wrap:changes_since(Db, all_docs, StartSeq, + couch_replicator_api_wrap:changes_since( + Db, + all_docs, + StartSeq, fun(Item) -> process_change(Item, {Parent, Db, ChangesQueue, Continuous}) - end, Options), + end, + Options + ), couch_work_queue:close(ChangesQueue) catch throw:recurse -> @@ -56,63 +68,79 @@ read_changes(Parent, StartSeq, Db, ChangesQueue, Options) -> LS = get(last_seq), read_changes(Parent, LS, Db, ChangesQueue, Options); throw:{retry_limit, Error} -> - couch_stats:increment_counter( - [couch_replicator, changes_read_failures] - ), - case get(retries_left) of - N when N > 0 -> - put(retries_left, N - 1), - LastSeq = get(last_seq), - Db2 = case LastSeq of - StartSeq -> - couch_log:notice("Retrying _changes request to source database ~s" - " with since=~p in ~p seconds", - [couch_replicator_api_wrap:db_uri(Db), LastSeq, Db#httpdb.wait / 1000]), - ok = timer:sleep(Db#httpdb.wait), - Db#httpdb{wait = 2 * Db#httpdb.wait}; - _ -> - couch_log:notice("Retrying _changes request to source database ~s" - " with since=~p", [couch_replicator_api_wrap:db_uri(Db), LastSeq]), - Db - end, - read_changes(Parent, LastSeq, Db2, ChangesQueue, Options); - _ -> - exit(Error) - end + couch_stats:increment_counter( + [couch_replicator, changes_read_failures] + ), + case get(retries_left) of + N when N > 0 -> + put(retries_left, N - 1), + LastSeq = get(last_seq), + Db2 = + case LastSeq of + StartSeq -> + couch_log:notice( + "Retrying _changes request to source database ~s" + " with since=~p in ~p seconds", + [ + couch_replicator_api_wrap:db_uri(Db), + LastSeq, + Db#httpdb.wait / 1000 + ] + ), + ok = timer:sleep(Db#httpdb.wait), + Db#httpdb{wait = 2 * Db#httpdb.wait}; + _ -> + couch_log:notice( + "Retrying _changes request to source database ~s" + " with since=~p", + [couch_replicator_api_wrap:db_uri(Db), LastSeq] + ), + Db + end, + read_changes(Parent, LastSeq, Db2, ChangesQueue, Options); + _ -> + exit(Error) + end end. - process_change(#doc_info{id = <<>>} = DocInfo, {_, Db, _, _}) -> % Previous CouchDB releases had a bug which allowed a doc with an empty ID % to be inserted into databases. Such doc is impossible to GET. - couch_log:error("Replicator: ignoring document with empty ID in " + couch_log:error( + "Replicator: ignoring document with empty ID in " "source database `~s` (_changes sequence ~p)", - [couch_replicator_api_wrap:db_uri(Db), DocInfo#doc_info.high_seq]); - + [couch_replicator_api_wrap:db_uri(Db), DocInfo#doc_info.high_seq] + ); process_change(#doc_info{id = Id} = DocInfo, {Parent, Db, ChangesQueue, _}) -> case is_doc_id_too_long(byte_size(Id)) of true -> SourceDb = couch_replicator_api_wrap:db_uri(Db), - couch_log:error("Replicator: document id `~s...` from source db " - " `~64s` is too long, ignoring.", [Id, SourceDb]), + couch_log:error( + "Replicator: document id `~s...` from source db " + " `~64s` is too long, ignoring.", + [Id, SourceDb] + ), Stats = couch_replicator_stats:new([{doc_write_failures, 1}]), ok = gen_server:call(Parent, {add_stats, Stats}, infinity); false -> ok = couch_work_queue:queue(ChangesQueue, DocInfo), put(last_seq, DocInfo#doc_info.high_seq) end; - process_change({last_seq, LS}, {_Parent, _, ChangesQueue, true = _Continuous}) -> % LS should never be undefined, but it doesn't hurt to be defensive inside % the replicator. - Seq = case LS of undefined -> get(last_seq); _ -> LS end, + Seq = + case LS of + undefined -> get(last_seq); + _ -> LS + end, OldSeq = get(last_seq), - if Seq == OldSeq -> ok; true -> - ok = couch_work_queue:queue(ChangesQueue, {last_seq, Seq}) + if + Seq == OldSeq -> ok; + true -> ok = couch_work_queue:queue(ChangesQueue, {last_seq, Seq}) end, put(last_seq, Seq), throw(recurse); - process_change({last_seq, _}, _) -> % This clause is unreachable today, but let's plan ahead for the future % where we checkpoint against last_seq instead of the sequence of the last diff --git a/src/couch_replicator/src/couch_replicator_clustering.erl b/src/couch_replicator/src/couch_replicator_clustering.erl index 18de1e825..8db320433 100644 --- a/src/couch_replicator/src/couch_replicator_clustering.erl +++ b/src/couch_replicator/src/couch_replicator_clustering.erl @@ -10,7 +10,6 @@ % License for the specific language governing permissions and limitations under % the License. - % Maintain cluster membership and stability notifications for replications. % On changes to cluster membership, broadcast events to `replication` gen_event. % Listeners will get `{cluster, stable}` or `{cluster, unstable}` events. @@ -23,7 +22,6 @@ % This module is also in charge of calculating ownership of replications based % on where their _replicator db documents shards live. - -module(couch_replicator_clustering). -behaviour(gen_server). @@ -64,8 +62,10 @@ -include_lib("couch/include/couch_db.hrl"). -include_lib("mem3/include/mem3.hrl"). --define(DEFAULT_QUIET_PERIOD, 60). % seconds --define(DEFAULT_START_PERIOD, 5). % seconds +% seconds +-define(DEFAULT_QUIET_PERIOD, 60). +% seconds +-define(DEFAULT_START_PERIOD, 5). -define(RELISTEN_DELAY, 5000). -record(state, { @@ -73,12 +73,10 @@ cluster_stable :: boolean() }). - -spec start_link() -> {ok, pid()} | ignore | {error, term()}. start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - % owner/2 function computes ownership for a {DbName, DocId} tuple % `unstable` if cluster is considered to be unstable i.e. it has changed % recently, or returns node() which of the owner. @@ -94,23 +92,22 @@ owner(<<"shards/", _/binary>> = DbName, DocId) -> owner(_DbName, _DocId) -> node(). - -spec is_stable() -> true | false. is_stable() -> gen_server:call(?MODULE, is_stable). - -spec link_cluster_event_listener(atom(), atom(), list()) -> pid(). -link_cluster_event_listener(Mod, Fun, Args) - when is_atom(Mod), is_atom(Fun), is_list(Args) -> +link_cluster_event_listener(Mod, Fun, Args) when + is_atom(Mod), is_atom(Fun), is_list(Args) +-> CallbackFun = - fun(Event = {cluster, _}) -> erlang:apply(Mod, Fun, Args ++ [Event]); - (_) -> ok + fun + (Event = {cluster, _}) -> erlang:apply(Mod, Fun, Args ++ [Event]); + (_) -> ok end, {ok, Pid} = couch_replicator_notifier:start_link(CallbackFun), Pid. - % Mem3 cluster callbacks cluster_unstable(Server) -> @@ -127,80 +124,80 @@ cluster_stable(Server) -> couch_log:notice("~s : cluster stable", [?MODULE]), Server. - % gen_server callbacks init([]) -> ok = config:listen_for_changes(?MODULE, nil), - Period = abs(config:get_integer("replicator", "cluster_quiet_period", - ?DEFAULT_QUIET_PERIOD)), - StartPeriod = abs(config:get_integer("replicator", "cluster_start_period", - ?DEFAULT_START_PERIOD)), + Period = abs( + config:get_integer( + "replicator", + "cluster_quiet_period", + ?DEFAULT_QUIET_PERIOD + ) + ), + StartPeriod = abs( + config:get_integer( + "replicator", + "cluster_start_period", + ?DEFAULT_START_PERIOD + ) + ), couch_stats:update_gauge([couch_replicator, cluster_is_stable], 0), - {ok, Mem3Cluster} = mem3_cluster:start_link(?MODULE, self(), StartPeriod, - Period), + {ok, Mem3Cluster} = mem3_cluster:start_link( + ?MODULE, + self(), + StartPeriod, + Period + ), {ok, #state{mem3_cluster_pid = Mem3Cluster, cluster_stable = false}}. - terminate(_Reason, _State) -> ok. - handle_call(is_stable, _From, #state{cluster_stable = IsStable} = State) -> {reply, IsStable, State}; - handle_call(set_stable, _From, State) -> {reply, ok, State#state{cluster_stable = true}}; - handle_call(set_unstable, _From, State) -> {reply, ok, State#state{cluster_stable = false}}. - handle_cast({set_period, Period}, #state{mem3_cluster_pid = Pid} = State) -> ok = mem3_cluster:set_period(Pid, Period), {noreply, State}. - handle_info(restart_config_listener, State) -> ok = config:listen_for_changes(?MODULE, nil), {noreply, State}. - code_change(_OldVsn, State, _Extra) -> {ok, State}. - %% Internal functions - handle_config_change("replicator", "cluster_quiet_period", V, _, S) -> ok = gen_server:cast(?MODULE, {set_period, list_to_integer(V)}), {ok, S}; handle_config_change(_, _, _, _, S) -> {ok, S}. - -handle_config_terminate(_, stop, _) -> ok; +handle_config_terminate(_, stop, _) -> + ok; handle_config_terminate(_S, _R, _St) -> Pid = whereis(?MODULE), erlang:send_after(?RELISTEN_DELAY, Pid, restart_config_listener). - -spec owner_int(binary(), binary()) -> node(). owner_int(ShardName, DocId) -> DbName = mem3:dbname(ShardName), Live = [node() | nodes()], Shards = mem3:shards(DbName, DocId), - Nodes = [N || #shard{node=N} <- Shards, lists:member(N, Live)], + Nodes = [N || #shard{node = N} <- Shards, lists:member(N, Live)], mem3:owner(DbName, DocId, Nodes). - - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). - replicator_clustering_test_() -> { setup, @@ -217,7 +214,6 @@ replicator_clustering_test_() -> } }. - t_stable_callback() -> ?_test(begin ?assertEqual(false, is_stable()), @@ -225,7 +221,6 @@ t_stable_callback() -> ?assertEqual(true, is_stable()) end). - t_unstable_callback() -> ?_test(begin cluster_stable(whereis(?MODULE)), @@ -234,7 +229,6 @@ t_unstable_callback() -> ?assertEqual(false, is_stable()) end). - setup_all() -> meck:expect(couch_log, notice, 2, ok), meck:expect(config, get, fun(_, _, Default) -> Default end), @@ -242,11 +236,9 @@ setup_all() -> meck:expect(couch_stats, update_gauge, 2, ok), meck:expect(couch_replicator_notifier, notify, 1, ok). - teardown_all(_) -> meck:unload(). - setup() -> meck:reset([ config, @@ -258,22 +250,20 @@ setup() -> {ok, Pid} = start_link(), Pid. - teardown(Pid) -> stop_clustering_process(Pid). - stop_clustering_process() -> stop_clustering_process(whereis(?MODULE)). - stop_clustering_process(undefined) -> ok; - stop_clustering_process(Pid) when is_pid(Pid) -> Ref = erlang:monitor(process, Pid), unlink(Pid), exit(Pid, kill), - receive {'DOWN', Ref, _, _, _} -> ok end. + receive + {'DOWN', Ref, _, _, _} -> ok + end. -endif. diff --git a/src/couch_replicator/src/couch_replicator_connection.erl b/src/couch_replicator/src/couch_replicator_connection.erl index d1c3d93fc..a158d2609 100644 --- a/src/couch_replicator/src/couch_replicator_connection.erl +++ b/src/couch_replicator/src/couch_replicator_connection.erl @@ -20,18 +20,18 @@ ]). -export([ - init/1, - terminate/2, - handle_call/3, - handle_info/2, - handle_cast/2, - code_change/3 + init/1, + terminate/2, + handle_call/3, + handle_info/2, + handle_cast/2, + code_change/3 ]). -export([ - acquire/1, - acquire/2, - release/1 + acquire/1, + acquire/2, + release/1 ]). -export([ @@ -44,7 +44,6 @@ -define(DEFAULT_CLOSE_INTERVAL, 90000). -define(RELISTEN_DELAY, 5000). - -record(state, { close_interval, timer @@ -59,40 +58,43 @@ mref }). - start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - init([]) -> process_flag(trap_exit, true), - ?MODULE = ets:new(?MODULE, [named_table, public, - {keypos, #connection.worker}]), + ?MODULE = ets:new(?MODULE, [ + named_table, + public, + {keypos, #connection.worker} + ]), ok = config:listen_for_changes(?MODULE, nil), - Interval = config:get_integer("replicator", "connection_close_interval", - ?DEFAULT_CLOSE_INTERVAL), + Interval = config:get_integer( + "replicator", + "connection_close_interval", + ?DEFAULT_CLOSE_INTERVAL + ), Timer = erlang:send_after(Interval, self(), close_idle_connections), ibrowse:add_config([ {inactivity_timeout, Interval}, {worker_trap_exits, false} ]), - {ok, #state{close_interval=Interval, timer=Timer}}. + {ok, #state{close_interval = Interval, timer = Timer}}. acquire(Url) -> acquire(Url, undefined). acquire(Url, ProxyUrl) when is_binary(Url) -> acquire(binary_to_list(Url), ProxyUrl); - acquire(Url, ProxyUrl) when is_binary(ProxyUrl) -> acquire(Url, binary_to_list(ProxyUrl)); - acquire(Url0, ProxyUrl0) -> Url = couch_util:url_strip_password(Url0), - ProxyUrl = case ProxyUrl0 of - undefined -> undefined; - _ -> couch_util:url_strip_password(ProxyUrl0) - end, + ProxyUrl = + case ProxyUrl0 of + undefined -> undefined; + _ -> couch_util:url_strip_password(ProxyUrl0) + end, case gen_server:call(?MODULE, {acquire, Url, ProxyUrl}) of {ok, Worker} -> link(Worker), @@ -105,28 +107,37 @@ acquire(Url0, ProxyUrl0) -> {error, Reason} end. - release(Worker) -> unlink(Worker), gen_server:cast(?MODULE, {release, Worker}). - handle_call({acquire, Url, ProxyUrl}, From, State) -> {Pid, _Ref} = From, case {ibrowse_lib:parse_url(Url), parse_proxy_url(ProxyUrl)} of - {#url{host=Host, port=Port}, #url{host=ProxyHost, port=ProxyPort}} -> + {#url{host = Host, port = Port}, #url{host = ProxyHost, port = ProxyPort}} -> Pat = #connection{ - host=Host, port=Port, - proxy_host=ProxyHost, proxy_port=ProxyPort, - mref=undefined, _='_'}, + host = Host, + port = Port, + proxy_host = ProxyHost, + proxy_port = ProxyPort, + mref = undefined, + _ = '_' + }, case ets:match_object(?MODULE, Pat, 1) of '$end_of_table' -> {reply, {error, all_allocated}, State}; {[Worker], _Cont} -> - couch_stats:increment_counter([couch_replicator, connection, - acquires]), - ets:insert(?MODULE, Worker#connection{mref=monitor(process, - Pid)}), + couch_stats:increment_counter([ + couch_replicator, + connection, + acquires + ]), + ets:insert(?MODULE, Worker#connection{ + mref = monitor( + process, + Pid + ) + }), {reply, {ok, Worker#connection.worker}, State} end; {{error, invalid_uri}, _} -> @@ -134,26 +145,30 @@ handle_call({acquire, Url, ProxyUrl}, From, State) -> {_, {error, invalid_uri}} -> {reply, {error, invalid_uri}, State} end; - handle_call({create, Url, ProxyUrl, Worker}, From, State) -> {Pid, _Ref} = From, case {ibrowse_lib:parse_url(Url), parse_proxy_url(ProxyUrl)} of - {#url{host=Host, port=Port}, #url{host=ProxyHost, port=ProxyPort}} -> + {#url{host = Host, port = Port}, #url{host = ProxyHost, port = ProxyPort}} -> link(Worker), - couch_stats:increment_counter([couch_replicator, connection, - creates]), + couch_stats:increment_counter([ + couch_replicator, + connection, + creates + ]), true = ets:insert_new( ?MODULE, #connection{ - host=Host, port=Port, - proxy_host=ProxyHost, proxy_port=ProxyPort, - worker=Worker, - mref=monitor(process, Pid)} + host = Host, + port = Port, + proxy_host = ProxyHost, + proxy_port = ProxyPort, + worker = Worker, + mref = monitor(process, Pid) + } ), {reply, ok, State} end. - handle_cast({release, WorkerPid}, State) -> couch_stats:increment_counter([couch_replicator, connection, releases]), case ets:lookup(?MODULE, WorkerPid) of @@ -162,39 +177,45 @@ handle_cast({release, WorkerPid}, State) -> MRef when is_reference(MRef) -> demonitor(MRef, [flush]); undefined -> ok end, - ets:insert(?MODULE, Worker#connection{mref=undefined}); + ets:insert(?MODULE, Worker#connection{mref = undefined}); [] -> ok end, {noreply, State}; - handle_cast({connection_close_interval, V}, State) -> erlang:cancel_timer(State#state.timer), NewTimer = erlang:send_after(V, self(), close_idle_connections), ibrowse:add_config([{inactivity_timeout, V}]), - {noreply, State#state{close_interval=V, timer=NewTimer}}. - + {noreply, State#state{close_interval = V, timer = NewTimer}}. % owner crashed handle_info({'DOWN', Ref, process, _Pid, _Reason}, State) -> - couch_stats:increment_counter([couch_replicator, connection, - owner_crashes]), - Conns = ets:match_object(?MODULE, #connection{mref = Ref, _='_'}), - lists:foreach(fun(Conn) -> - couch_stats:increment_counter([couch_replicator, connection, closes]), - delete_worker(Conn) - end, Conns), + couch_stats:increment_counter([ + couch_replicator, + connection, + owner_crashes + ]), + Conns = ets:match_object(?MODULE, #connection{mref = Ref, _ = '_'}), + lists:foreach( + fun(Conn) -> + couch_stats:increment_counter([couch_replicator, connection, closes]), + delete_worker(Conn) + end, + Conns + ), {noreply, State}; - % worker crashed handle_info({'EXIT', Pid, Reason}, State) -> - couch_stats:increment_counter([couch_replicator, connection, - worker_crashes]), + couch_stats:increment_counter([ + couch_replicator, + connection, + worker_crashes + ]), case ets:lookup(?MODULE, Pid) of [] -> ok; [Worker] -> - #connection{host=Host, port=Port} = Worker, + #connection{host = Host, port = Port} = Worker, maybe_log_worker_death(Host, Port, Reason), case Worker#connection.mref of MRef when is_reference(MRef) -> demonitor(MRef, [flush]); @@ -203,42 +224,38 @@ handle_info({'EXIT', Pid, Reason}, State) -> ets:delete(?MODULE, Pid) end, {noreply, State}; - handle_info(close_idle_connections, State) -> #state{ - close_interval=Interval, - timer=Timer + close_interval = Interval, + timer = Timer } = State, - Conns = ets:match_object(?MODULE, #connection{mref=undefined, _='_'}), - lists:foreach(fun(Conn) -> - couch_stats:increment_counter([couch_replicator, connection, closes]), - delete_worker(Conn) - end, Conns), + Conns = ets:match_object(?MODULE, #connection{mref = undefined, _ = '_'}), + lists:foreach( + fun(Conn) -> + couch_stats:increment_counter([couch_replicator, connection, closes]), + delete_worker(Conn) + end, + Conns + ), erlang:cancel_timer(Timer), NewTimer = erlang:send_after(Interval, self(), close_idle_connections), - {noreply, State#state{timer=NewTimer}}; - + {noreply, State#state{timer = NewTimer}}; handle_info(restart_config_listener, State) -> ok = config:listen_for_changes(?MODULE, nil), {noreply, State}. - code_change(_OldVsn, State, _Extra) -> {ok, State}. - terminate(_Reason, _State) -> ok. - maybe_log_worker_death(_Host, _Port, normal) -> ok; - maybe_log_worker_death(Host, Port, Reason) -> ErrMsg = "Replication connection to: ~p:~p died with reason ~p", couch_log:info(ErrMsg, [Host, Port, Reason]). - -spec delete_worker(#connection{}) -> ok. delete_worker(Worker) -> ets:delete(?MODULE, Worker#connection.worker), @@ -246,25 +263,19 @@ delete_worker(Worker) -> spawn(fun() -> ibrowse_http_client:stop(Worker#connection.worker) end), ok. - handle_config_change("replicator", "connection_close_interval", V, _, S) -> - ok = gen_server:cast(?MODULE, {connection_close_interval, - list_to_integer(V)}), + ok = gen_server:cast(?MODULE, {connection_close_interval, list_to_integer(V)}), {ok, S}; - handle_config_change(_, _, _, _, S) -> {ok, S}. - handle_config_terminate(_, stop, _) -> ok; - handle_config_terminate(_, _, _) -> Pid = whereis(?MODULE), erlang:send_after(?RELISTEN_DELAY, Pid, restart_config_listener). - parse_proxy_url(undefined) -> - #url{host=undefined, port=undefined}; + #url{host = undefined, port = undefined}; parse_proxy_url(ProxyUrl) -> ibrowse_lib:parse_url(ProxyUrl). diff --git a/src/couch_replicator/src/couch_replicator_db_changes.erl b/src/couch_replicator/src/couch_replicator_db_changes.erl index 92b0222c4..947af51b4 100644 --- a/src/couch_replicator/src/couch_replicator_db_changes.erl +++ b/src/couch_replicator/src/couch_replicator_db_changes.erl @@ -15,42 +15,42 @@ -behaviour(gen_server). -export([ - start_link/0 + start_link/0 ]). -export([ - init/1, - terminate/2, - handle_call/3, - handle_info/2, - handle_cast/2, - code_change/3 + init/1, + terminate/2, + handle_call/3, + handle_info/2, + handle_cast/2, + code_change/3 ]). -export([ - notify_cluster_event/2 + notify_cluster_event/2 ]). -record(state, { - event_listener :: pid(), - mdb_changes :: pid() | nil + event_listener :: pid(), + mdb_changes :: pid() | nil }). - -spec notify_cluster_event(pid(), {cluster, any()}) -> ok. notify_cluster_event(Server, {cluster, _} = Event) -> gen_server:cast(Server, Event). - -spec start_link() -> {ok, pid()} | ignore | {error, any()}. start_link() -> gen_server:start_link(?MODULE, [], []). - init([]) -> - EvtPid = couch_replicator_clustering:link_cluster_event_listener(?MODULE, - notify_cluster_event, [self()]), + EvtPid = couch_replicator_clustering:link_cluster_event_listener( + ?MODULE, + notify_cluster_event, + [self()] + ), State = #state{event_listener = EvtPid, mdb_changes = nil}, case couch_replicator_clustering:is_stable() of true -> @@ -59,45 +59,40 @@ init([]) -> {ok, State} end. - terminate(_Reason, _State) -> ok. - handle_call(_Msg, _From, State) -> {reply, {error, invalid_call}, State}. - handle_cast({cluster, unstable}, State) -> {noreply, stop_mdb_changes(State)}; - handle_cast({cluster, stable}, State) -> {noreply, restart_mdb_changes(State)}. - handle_info(_Msg, State) -> {noreply, State}. - code_change(_OldVsn, State, _Extra) -> {ok, State}. - -spec restart_mdb_changes(#state{}) -> #state{}. restart_mdb_changes(#state{mdb_changes = nil} = State) -> Suffix = <<"_replicator">>, CallbackMod = couch_replicator_doc_processor, Options = [skip_ddocs], - {ok, Pid} = couch_multidb_changes:start_link(Suffix, CallbackMod, nil, - Options), + {ok, Pid} = couch_multidb_changes:start_link( + Suffix, + CallbackMod, + nil, + Options + ), couch_stats:increment_counter([couch_replicator, db_scans]), couch_log:notice("Started replicator db changes listener ~p", [Pid]), State#state{mdb_changes = Pid}; - restart_mdb_changes(#state{mdb_changes = _Pid} = State) -> restart_mdb_changes(stop_mdb_changes(State)). - -spec stop_mdb_changes(#state{}) -> #state{}. stop_mdb_changes(#state{mdb_changes = nil} = State) -> State; diff --git a/src/couch_replicator/src/couch_replicator_doc_processor.erl b/src/couch_replicator/src/couch_replicator_doc_processor.erl index ed6670615..436d7c44d 100644 --- a/src/couch_replicator/src/couch_replicator_doc_processor.erl +++ b/src/couch_replicator/src/couch_replicator_doc_processor.erl @@ -20,12 +20,12 @@ ]). -export([ - init/1, - terminate/2, - handle_call/3, - handle_info/2, - handle_cast/2, - code_change/3 + init/1, + terminate/2, + handle_call/3, + handle_info/2, + handle_cast/2, + code_change/3 ]). -export([ @@ -54,15 +54,15 @@ ]). -define(DEFAULT_UPDATE_DOCS, false). --define(ERROR_MAX_BACKOFF_EXPONENT, 12). % ~ 1 day on average +% ~ 1 day on average +-define(ERROR_MAX_BACKOFF_EXPONENT, 12). -define(TS_DAY_SEC, 86400). -define(INITIAL_BACKOFF_EXPONENT, 64). -define(MIN_FILTER_DELAY_SEC, 60). --type filter_type() :: nil | view | user | docids | mango. +-type filter_type() :: nil | view | user | docids | mango. -type repstate() :: initializing | error | scheduled. - -record(rdoc, { id :: db_doc_id() | '_' | {any(), '_'}, state :: repstate() | '_', @@ -75,7 +75,6 @@ last_updated :: erlang:timestamp() | '_' }). - % couch_multidb_changes API callbacks db_created(DbName, Server) -> @@ -83,35 +82,31 @@ db_created(DbName, Server) -> couch_replicator_docs:ensure_rep_ddoc_exists(DbName), Server. - db_deleted(DbName, Server) -> couch_stats:increment_counter([couch_replicator, docs, dbs_deleted]), ok = gen_server:call(?MODULE, {clean_up_replications, DbName}, infinity), Server. - db_found(DbName, Server) -> couch_stats:increment_counter([couch_replicator, docs, dbs_found]), couch_replicator_docs:ensure_rep_ddoc_exists(DbName), Server. - db_change(DbName, {ChangeProps} = Change, Server) -> couch_stats:increment_counter([couch_replicator, docs, db_changes]), try ok = process_change(DbName, Change) catch - exit:{Error, {gen_server, call, [?MODULE, _, _]}} -> - ErrMsg = "~p exited ~p while processing change from db ~p", - couch_log:error(ErrMsg, [?MODULE, Error, DbName]); - _Tag:Error -> - {RepProps} = get_json_value(doc, ChangeProps), - DocId = get_json_value(<<"_id">>, RepProps), - couch_replicator_docs:update_failed(DbName, DocId, Error) + exit:{Error, {gen_server, call, [?MODULE, _, _]}} -> + ErrMsg = "~p exited ~p while processing change from db ~p", + couch_log:error(ErrMsg, [?MODULE, Error, DbName]); + _Tag:Error -> + {RepProps} = get_json_value(doc, ChangeProps), + DocId = get_json_value(<<"_id">>, RepProps), + couch_replicator_docs:update_failed(DbName, DocId, Error) end, Server. - -spec get_worker_ref(db_doc_id()) -> reference() | nil. get_worker_ref({DbName, DocId}) when is_binary(DbName), is_binary(DocId) -> case ets:lookup(?MODULE, {DbName, DocId}) of @@ -123,46 +118,43 @@ get_worker_ref({DbName, DocId}) when is_binary(DbName), is_binary(DocId) -> nil end. - % Cluster membership change notification callback -spec notify_cluster_event(pid(), {cluster, any()}) -> ok. notify_cluster_event(Server, {cluster, _} = Event) -> gen_server:cast(Server, Event). - process_change(DbName, {Change}) -> {RepProps} = JsonRepDoc = get_json_value(doc, Change), DocId = get_json_value(<<"_id">>, RepProps), Owner = couch_replicator_clustering:owner(DbName, DocId), Id = {DbName, DocId}, case {Owner, get_json_value(deleted, Change, false)} of - {_, true} -> - ok = gen_server:call(?MODULE, {removed, Id}, infinity); - {unstable, false} -> - couch_log:notice("Not starting '~s' as cluster is unstable", [DocId]); - {ThisNode, false} when ThisNode =:= node() -> - case get_json_value(<<"_replication_state">>, RepProps) of - undefined -> - ok = process_updated(Id, JsonRepDoc); - <<"triggered">> -> - maybe_remove_state_fields(DbName, DocId), - ok = process_updated(Id, JsonRepDoc); - <<"completed">> -> - ok = gen_server:call(?MODULE, {completed, Id}, infinity); - <<"error">> -> - % Handle replications started from older versions of replicator - % which wrote transient errors to replication docs - maybe_remove_state_fields(DbName, DocId), - ok = process_updated(Id, JsonRepDoc); - <<"failed">> -> + {_, true} -> + ok = gen_server:call(?MODULE, {removed, Id}, infinity); + {unstable, false} -> + couch_log:notice("Not starting '~s' as cluster is unstable", [DocId]); + {ThisNode, false} when ThisNode =:= node() -> + case get_json_value(<<"_replication_state">>, RepProps) of + undefined -> + ok = process_updated(Id, JsonRepDoc); + <<"triggered">> -> + maybe_remove_state_fields(DbName, DocId), + ok = process_updated(Id, JsonRepDoc); + <<"completed">> -> + ok = gen_server:call(?MODULE, {completed, Id}, infinity); + <<"error">> -> + % Handle replications started from older versions of replicator + % which wrote transient errors to replication docs + maybe_remove_state_fields(DbName, DocId), + ok = process_updated(Id, JsonRepDoc); + <<"failed">> -> + ok + end; + {Owner, false} -> ok - end; - {Owner, false} -> - ok end, ok. - maybe_remove_state_fields(DbName, DocId) -> case update_docs() of true -> @@ -171,7 +163,6 @@ maybe_remove_state_fields(DbName, DocId) -> couch_replicator_docs:remove_state_fields(DbName, DocId) end. - process_updated({DbName, _DocId} = Id, JsonRepDoc) -> % Parsing replication doc (but not calculating the id) could throw an % exception which would indicate this document is malformed. This exception @@ -180,53 +171,54 @@ process_updated({DbName, _DocId} = Id, JsonRepDoc) -> % problem. Rep0 = couch_replicator_docs:parse_rep_doc_without_id(JsonRepDoc), Rep = Rep0#rep{db_name = DbName, start_time = os:timestamp()}, - Filter = case couch_replicator_filters:parse(Rep#rep.options) of - {ok, nil} -> - nil; - {ok, {user, _FName, _QP}} -> - user; - {ok, {view, _FName, _QP}} -> - view; - {ok, {docids, _DocIds}} -> - docids; - {ok, {mango, _Selector}} -> - mango; - {error, FilterError} -> - throw(FilterError) - end, + Filter = + case couch_replicator_filters:parse(Rep#rep.options) of + {ok, nil} -> + nil; + {ok, {user, _FName, _QP}} -> + user; + {ok, {view, _FName, _QP}} -> + view; + {ok, {docids, _DocIds}} -> + docids; + {ok, {mango, _Selector}} -> + mango; + {error, FilterError} -> + throw(FilterError) + end, gen_server:call(?MODULE, {updated, Id, Rep, Filter}, infinity). - % Doc processor gen_server API and callbacks start_link() -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). init([]) -> - ?MODULE = ets:new(?MODULE, [named_table, {keypos, #rdoc.id}, - {read_concurrency, true}, {write_concurrency, true}]), - couch_replicator_clustering:link_cluster_event_listener(?MODULE, - notify_cluster_event, [self()]), + ?MODULE = ets:new(?MODULE, [ + named_table, + {keypos, #rdoc.id}, + {read_concurrency, true}, + {write_concurrency, true} + ]), + couch_replicator_clustering:link_cluster_event_listener( + ?MODULE, + notify_cluster_event, + [self()] + ), {ok, nil}. - terminate(_Reason, _State) -> ok. - handle_call({updated, Id, Rep, Filter}, _From, State) -> ok = updated_doc(Id, Rep, Filter), {reply, ok, State}; - handle_call({removed, Id}, _From, State) -> ok = removed_doc(Id), {reply, ok, State}; - handle_call({completed, Id}, _From, State) -> true = ets:delete(?MODULE, Id), {reply, ok, State}; - handle_call({clean_up_replications, DbName}, _From, State) -> ok = removed_db(DbName), {reply, ok, State}. @@ -234,29 +226,29 @@ handle_call({clean_up_replications, DbName}, _From, State) -> handle_cast({cluster, unstable}, State) -> % Ignoring unstable state transition {noreply, State}; - handle_cast({cluster, stable}, State) -> % Membership changed recheck all the replication document ownership nil = ets:foldl(fun cluster_membership_foldl/2, nil, ?MODULE), {noreply, State}; - handle_cast(Msg, State) -> {stop, {error, unexpected_message, Msg}, State}. - -handle_info({'DOWN', _, _, _, #doc_worker_result{id = Id, wref = Ref, - result = Res}}, State) -> +handle_info( + {'DOWN', _, _, _, #doc_worker_result{ + id = Id, + wref = Ref, + result = Res + }}, + State +) -> ok = worker_returned(Ref, Id, Res), {noreply, State}; - handle_info(_Msg, State) -> {noreply, State}. - code_change(_OldVsn, State, _Extra) -> {ok, State}. - % Doc processor gen_server private helper functions % Handle doc update -- add to ets, then start a worker to try to turn it into @@ -289,7 +281,6 @@ updated_doc(Id, Rep, Filter) -> ok end. - % Return current #rep{} record if any. If replication hasn't been submitted % to the scheduler yet, #rep{} record will be in the document processor's % ETS table, otherwise query scheduler for the #rep{} record. @@ -308,81 +299,81 @@ current_rep({DbName, DocId}) when is_binary(DbName), is_binary(DocId) -> Rep end. - -spec worker_returned(reference(), db_doc_id(), rep_start_result()) -> ok. worker_returned(Ref, Id, {ok, RepId}) -> case ets:lookup(?MODULE, Id) of - [#rdoc{worker = Ref} = Row] -> - Row0 = Row#rdoc{ - state = scheduled, - errcnt = 0, - worker = nil, - last_updated = os:timestamp() - }, - NewRow = case Row0 of - #rdoc{rid = RepId, filter = user} -> - % Filtered replication id didn't change. - Row0; - #rdoc{rid = nil, filter = user} -> - % Calculated new replication id for a filtered replication. Make - % sure to schedule another check as filter code could change. - % Replication starts could have been failing, so also clear - % error count. - Row0#rdoc{rid = RepId}; - #rdoc{rid = OldRepId, filter = user} -> - % Replication id of existing replication job with filter has - % changed. Remove old replication job from scheduler and - % schedule check to check for future changes. - ok = couch_replicator_scheduler:remove_job(OldRepId), - Msg = io_lib:format("Replication id changed: ~p -> ~p", [ - OldRepId, RepId]), - Row0#rdoc{rid = RepId, info = couch_util:to_binary(Msg)}; - #rdoc{rid = nil} -> - % Calculated new replication id for non-filtered replication. - % Remove replication doc body, after this we won't need it - % anymore. - Row0#rdoc{rep=nil, rid=RepId, info=nil} - end, - true = ets:insert(?MODULE, NewRow), - ok = maybe_update_doc_triggered(Row#rdoc.rep, RepId), - ok = maybe_start_worker(Id); - _ -> - ok % doc could have been deleted, ignore + [#rdoc{worker = Ref} = Row] -> + Row0 = Row#rdoc{ + state = scheduled, + errcnt = 0, + worker = nil, + last_updated = os:timestamp() + }, + NewRow = + case Row0 of + #rdoc{rid = RepId, filter = user} -> + % Filtered replication id didn't change. + Row0; + #rdoc{rid = nil, filter = user} -> + % Calculated new replication id for a filtered replication. Make + % sure to schedule another check as filter code could change. + % Replication starts could have been failing, so also clear + % error count. + Row0#rdoc{rid = RepId}; + #rdoc{rid = OldRepId, filter = user} -> + % Replication id of existing replication job with filter has + % changed. Remove old replication job from scheduler and + % schedule check to check for future changes. + ok = couch_replicator_scheduler:remove_job(OldRepId), + Msg = io_lib:format("Replication id changed: ~p -> ~p", [ + OldRepId, RepId + ]), + Row0#rdoc{rid = RepId, info = couch_util:to_binary(Msg)}; + #rdoc{rid = nil} -> + % Calculated new replication id for non-filtered replication. + % Remove replication doc body, after this we won't need it + % anymore. + Row0#rdoc{rep = nil, rid = RepId, info = nil} + end, + true = ets:insert(?MODULE, NewRow), + ok = maybe_update_doc_triggered(Row#rdoc.rep, RepId), + ok = maybe_start_worker(Id); + _ -> + % doc could have been deleted, ignore + ok end, ok; - worker_returned(_Ref, _Id, ignore) -> ok; - worker_returned(Ref, Id, {temporary_error, Reason}) -> case ets:lookup(?MODULE, Id) of - [#rdoc{worker = Ref, errcnt = ErrCnt} = Row] -> - NewRow = Row#rdoc{ - rid = nil, - state = error, - info = Reason, - errcnt = ErrCnt + 1, - worker = nil, - last_updated = os:timestamp() - }, - true = ets:insert(?MODULE, NewRow), - ok = maybe_update_doc_error(NewRow#rdoc.rep, Reason), - ok = maybe_start_worker(Id); - _ -> - ok % doc could have been deleted, ignore + [#rdoc{worker = Ref, errcnt = ErrCnt} = Row] -> + NewRow = Row#rdoc{ + rid = nil, + state = error, + info = Reason, + errcnt = ErrCnt + 1, + worker = nil, + last_updated = os:timestamp() + }, + true = ets:insert(?MODULE, NewRow), + ok = maybe_update_doc_error(NewRow#rdoc.rep, Reason), + ok = maybe_start_worker(Id); + _ -> + % doc could have been deleted, ignore + ok end, ok; - worker_returned(Ref, Id, {permanent_failure, _Reason}) -> case ets:lookup(?MODULE, Id) of - [#rdoc{worker = Ref}] -> - true = ets:delete(?MODULE, Id); - _ -> - ok % doc could have been deleted, ignore + [#rdoc{worker = Ref}] -> + true = ets:delete(?MODULE, Id); + _ -> + % doc could have been deleted, ignore + ok end, ok. - -spec maybe_update_doc_error(#rep{}, any()) -> ok. maybe_update_doc_error(Rep, Reason) -> case update_docs() of @@ -392,7 +383,6 @@ maybe_update_doc_error(Rep, Reason) -> ok end. - -spec maybe_update_doc_triggered(#rep{}, rep_id()) -> ok. maybe_update_doc_triggered(Rep, RepId) -> case update_docs() of @@ -402,7 +392,6 @@ maybe_update_doc_triggered(Rep, RepId) -> ok end. - -spec error_backoff(non_neg_integer()) -> seconds(). error_backoff(ErrCnt) -> Exp = min(ErrCnt, ?ERROR_MAX_BACKOFF_EXPONENT), @@ -411,7 +400,6 @@ error_backoff(ErrCnt) -> % on average. Then 1 minute and so on. couch_rand:uniform(?INITIAL_BACKOFF_EXPONENT bsl Exp). - -spec filter_backoff() -> seconds(). filter_backoff() -> Total = ets:info(?MODULE, size), @@ -424,7 +412,6 @@ filter_backoff() -> Range = 1 + min(2 * (Total / 10), ?TS_DAY_SEC), ?MIN_FILTER_DELAY_SEC + couch_rand:uniform(round(Range)). - % Document removed from db -- clear ets table and remove all scheduled jobs -spec removed_doc(db_doc_id()) -> ok. removed_doc({DbName, DocId} = Id) -> @@ -432,7 +419,6 @@ removed_doc({DbName, DocId} = Id) -> RepIds = couch_replicator_scheduler:find_jobs_by_doc(DbName, DocId), lists:foreach(fun couch_replicator_scheduler:remove_job/1, RepIds). - % Whole db shard is gone -- remove all its ets rows and stop jobs -spec removed_db(binary()) -> ok. removed_db(DbName) -> @@ -441,32 +427,30 @@ removed_db(DbName) -> RepIds = couch_replicator_scheduler:find_jobs_by_dbname(DbName), lists:foreach(fun couch_replicator_scheduler:remove_job/1, RepIds). - % Spawn a worker process which will attempt to calculate a replication id, then % start a replication. Returns a process monitor reference. The worker is % guaranteed to exit with rep_start_result() type only. -spec maybe_start_worker(db_doc_id()) -> ok. maybe_start_worker(Id) -> case ets:lookup(?MODULE, Id) of - [] -> - ok; - [#rdoc{state = scheduled, filter = Filter}] when Filter =/= user -> - ok; - [#rdoc{rep = Rep} = Doc] -> - % For any replication with a user created filter function, periodically - % (every `filter_backoff/0` seconds) to try to see if the user filter - % has changed by using a worker to check for changes. When the worker - % returns check if replication ID has changed. If it hasn't keep - % checking (spawn another worker and so on). If it has stop the job - % with the old ID and continue checking. - Wait = get_worker_wait(Doc), - Ref = make_ref(), - true = ets:insert(?MODULE, Doc#rdoc{worker = Ref}), - couch_replicator_doc_processor_worker:spawn_worker(Id, Rep, Wait, Ref), - ok + [] -> + ok; + [#rdoc{state = scheduled, filter = Filter}] when Filter =/= user -> + ok; + [#rdoc{rep = Rep} = Doc] -> + % For any replication with a user created filter function, periodically + % (every `filter_backoff/0` seconds) to try to see if the user filter + % has changed by using a worker to check for changes. When the worker + % returns check if replication ID has changed. If it hasn't keep + % checking (spawn another worker and so on). If it has stop the job + % with the old ID and continue checking. + Wait = get_worker_wait(Doc), + Ref = make_ref(), + true = ets:insert(?MODULE, Doc#rdoc{worker = Ref}), + couch_replicator_doc_processor_worker:spawn_worker(Id, Rep, Wait, Ref), + ok end. - -spec get_worker_wait(#rdoc{}) -> seconds(). get_worker_wait(#rdoc{state = scheduled, filter = user}) -> filter_backoff(); @@ -475,45 +459,52 @@ get_worker_wait(#rdoc{state = error, errcnt = ErrCnt}) -> get_worker_wait(#rdoc{state = initializing}) -> 0. - -spec update_docs() -> boolean(). update_docs() -> config:get_boolean("replicator", "update_docs", ?DEFAULT_UPDATE_DOCS). - % _scheduler/docs HTTP endpoint helpers -spec docs([atom()]) -> [{[_]}] | []. docs(States) -> HealthThreshold = couch_replicator_scheduler:health_threshold(), - ets:foldl(fun(RDoc, Acc) -> - case ejson_doc(RDoc, HealthThreshold) of - nil -> - Acc; % Could have been deleted if job just completed - {Props} = EJson -> - {state, DocState} = lists:keyfind(state, 1, Props), - case ejson_doc_state_filter(DocState, States) of - true -> - [EJson | Acc]; - false -> - Acc - end - end - end, [], ?MODULE). - + ets:foldl( + fun(RDoc, Acc) -> + case ejson_doc(RDoc, HealthThreshold) of + nil -> + % Could have been deleted if job just completed + Acc; + {Props} = EJson -> + {state, DocState} = lists:keyfind(state, 1, Props), + case ejson_doc_state_filter(DocState, States) of + true -> + [EJson | Acc]; + false -> + Acc + end + end + end, + [], + ?MODULE + ). -spec doc(binary(), binary()) -> {ok, {[_]}} | {error, not_found}. doc(Db, DocId) -> HealthThreshold = couch_replicator_scheduler:health_threshold(), - Res = (catch ets:foldl(fun(RDoc, nil) -> - {Shard, RDocId} = RDoc#rdoc.id, - case {mem3:dbname(Shard), RDocId} of - {Db, DocId} -> - throw({found, ejson_doc(RDoc, HealthThreshold)}); - {_OtherDb, _OtherDocId} -> - nil - end - end, nil, ?MODULE)), + Res = + (catch ets:foldl( + fun(RDoc, nil) -> + {Shard, RDocId} = RDoc#rdoc.id, + case {mem3:dbname(Shard), RDocId} of + {Db, DocId} -> + throw({found, ejson_doc(RDoc, HealthThreshold)}); + {_OtherDb, _OtherDocId} -> + nil + end + end, + nil, + ?MODULE + )), case Res of {found, DocInfo} -> {ok, DocInfo}; @@ -521,7 +512,6 @@ doc(Db, DocId) -> {error, not_found} end. - -spec doc_lookup(binary(), binary(), integer()) -> {ok, {[_]}} | {error, not_found}. doc_lookup(Db, DocId, HealthThreshold) -> @@ -532,14 +522,12 @@ doc_lookup(Db, DocId, HealthThreshold) -> {error, not_found} end. - -spec ejson_rep_id(rep_id() | nil) -> binary() | null. ejson_rep_id(nil) -> null; ejson_rep_id({BaseId, Ext}) -> iolist_to_binary([BaseId, Ext]). - -spec ejson_doc(#rdoc{}, non_neg_integer()) -> {[_]} | nil. ejson_doc(#rdoc{state = scheduled} = RDoc, HealthThreshold) -> #rdoc{id = {DbName, DocId}, rid = RepId} = RDoc, @@ -552,18 +540,18 @@ ejson_doc(#rdoc{state = scheduled} = RDoc, HealthThreshold) -> {doc_id, DocId}, {database, DbName}, {id, ejson_rep_id(RepId)}, - {node, node()} | JobProps + {node, node()} + | JobProps ]} end; - ejson_doc(#rdoc{state = RepState} = RDoc, _HealthThreshold) -> #rdoc{ - id = {DbName, DocId}, - info = StateInfo, - rid = RepId, - errcnt = ErrorCount, - last_updated = StateTime, - rep = Rep + id = {DbName, DocId}, + info = StateInfo, + rid = RepId, + errcnt = ErrorCount, + last_updated = StateTime, + rep = Rep } = RDoc, {[ {doc_id, DocId}, @@ -577,14 +565,12 @@ ejson_doc(#rdoc{state = RepState} = RDoc, _HealthThreshold) -> {start_time, couch_replicator_utils:iso8601(Rep#rep.start_time)} ]}. - -spec ejson_doc_state_filter(atom(), [atom()]) -> boolean(). ejson_doc_state_filter(_DocState, []) -> true; ejson_doc_state_filter(State, States) when is_list(States), is_atom(State) -> lists:member(State, States). - -spec cluster_membership_foldl(#rdoc{}, nil) -> nil. cluster_membership_foldl(#rdoc{id = {DbName, DocId} = Id, rid = RepId}, nil) -> case couch_replicator_clustering:owner(DbName, DocId) of @@ -599,7 +585,6 @@ cluster_membership_foldl(#rdoc{id = {DbName, DocId} = Id, rid = RepId}, nil) -> nil end. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -611,7 +596,6 @@ cluster_membership_foldl(#rdoc{id = {DbName, DocId} = Id, rid = RepId}, nil) -> -define(R1, {"1", ""}). -define(R2, {"2", ""}). - doc_processor_test_() -> { setup, @@ -640,7 +624,6 @@ doc_processor_test_() -> } }. - % Can't parse replication doc, so should write failure state to document. t_bad_change() -> ?_test(begin @@ -648,7 +631,6 @@ t_bad_change() -> ?assert(updated_doc_with_failed_state()) end). - % Regular change, parse to a #rep{} and then add job. t_regular_change() -> ?_test(begin @@ -658,15 +640,13 @@ t_regular_change() -> ?assert(started_worker({?DB, ?DOC1})) end). - % Handle cases where doc processor exits or crashes while processing a change t_change_with_doc_processor_crash() -> ?_test(begin mock_existing_jobs_lookup([]), ?assertEqual(acc, db_change(?EXIT_DB, change(), acc)), ?assert(failed_state_not_updated()) - end). - + end). % Regular change, parse to a #rep{} and then add job but there is already % a running job with same Id found. @@ -678,7 +658,6 @@ t_change_with_existing_job() -> ?assert(started_worker({?DB, ?DOC1})) end). - % Change is a deletion, and job is running, so remove job. t_deleted_change() -> ?_test(begin @@ -687,7 +666,6 @@ t_deleted_change() -> ?assert(removed_job(?R2)) end). - % Change is in `triggered` state. Remove legacy state and add job. t_triggered_change() -> ?_test(begin @@ -698,7 +676,6 @@ t_triggered_change() -> ?assert(started_worker({?DB, ?DOC1})) end). - % Change is in `completed` state, so skip over it. t_completed_change() -> ?_test(begin @@ -708,7 +685,6 @@ t_completed_change() -> ?assert(did_not_spawn_worker()) end). - % Completed change comes for what used to be an active job. In this case % remove entry from doc_processor's ets (because there is no linkage or % callback mechanism for scheduler to tell doc_processsor a replication just @@ -723,7 +699,6 @@ t_active_replication_completed() -> ?assertNot(ets:member(?MODULE, {?DB, ?DOC1})) end). - % Change is in `error` state. Remove legacy state and retry % running the job. This state was used for transient erorrs which are not % written to the document anymore. @@ -736,7 +711,6 @@ t_error_change() -> ?assert(started_worker({?DB, ?DOC1})) end). - % Change is in `failed` state. This is a terminal state and it will not % be tried again, so skip over it. t_failed_change() -> @@ -747,27 +721,24 @@ t_failed_change() -> ?assert(did_not_spawn_worker()) end). - % Normal change, but according to cluster ownership algorithm, replication % belongs to a different node, so this node should skip it. t_change_for_different_node() -> - ?_test(begin + ?_test(begin meck:expect(couch_replicator_clustering, owner, 2, different_node), ?assertEqual(ok, process_change(?DB, change())), ?assert(did_not_spawn_worker()) - end). - + end). % Change handled when cluster is unstable (nodes are added or removed), so % job is not added. A rescan will be triggered soon and change will be % evaluated again. t_change_when_cluster_unstable() -> - ?_test(begin - meck:expect(couch_replicator_clustering, owner, 2, unstable), - ?assertEqual(ok, process_change(?DB, change())), - ?assert(did_not_spawn_worker()) - end). - + ?_test(begin + meck:expect(couch_replicator_clustering, owner, 2, unstable), + ?assertEqual(ok, process_change(?DB, change())), + ?assert(did_not_spawn_worker()) + end). % Check if docs/0 function produces expected ejson after adding a job t_ejson_docs() -> @@ -776,12 +747,17 @@ t_ejson_docs() -> ?assertEqual(ok, process_change(?DB, change())), ?assert(ets:member(?MODULE, {?DB, ?DOC1})), EJsonDocs = docs([]), - ?assertMatch([{[_|_]}], EJsonDocs), + ?assertMatch([{[_ | _]}], EJsonDocs), [{DocProps}] = EJsonDocs, - {value, StateTime, DocProps1} = lists:keytake(last_updated, 1, - DocProps), - ?assertMatch({last_updated, BinVal1} when is_binary(BinVal1), - StateTime), + {value, StateTime, DocProps1} = lists:keytake( + last_updated, + 1, + DocProps + ), + ?assertMatch( + {last_updated, BinVal1} when is_binary(BinVal1), + StateTime + ), {value, StartTime, DocProps2} = lists:keytake(start_time, 1, DocProps1), ?assertMatch({start_time, BinVal2} when is_binary(BinVal2), StartTime), ExpectedProps = [ @@ -796,11 +772,10 @@ t_ejson_docs() -> ?assertEqual(ExpectedProps, lists:usort(DocProps2)) end). - % Check that when cluster membership changes records from doc processor and job % scheduler get removed t_cluster_membership_foldl() -> - ?_test(begin + ?_test(begin mock_existing_jobs_lookup([test_rep(?R1)]), ?assertEqual(ok, process_change(?DB, change())), meck:expect(couch_replicator_clustering, owner, 2, different_node), @@ -809,8 +784,7 @@ t_cluster_membership_foldl() -> meck:wait(2, couch_replicator_scheduler, find_jobs_by_doc, 2, 5000), ?assertNot(ets:member(?MODULE, {?DB, ?DOC1})), ?assert(removed_job(?R1)) - end). - + end). get_worker_ref_test_() -> { @@ -830,10 +804,8 @@ get_worker_ref_test_() -> end) }. - % Test helper functions - setup_all() -> meck:expect(couch_log, info, 2, ok), meck:expect(couch_log, notice, 2, ok), @@ -842,8 +814,12 @@ setup_all() -> meck:expect(config, get, fun(_, _, Default) -> Default end), meck:expect(config, listen_for_changes, 2, ok), meck:expect(couch_replicator_clustering, owner, 2, node()), - meck:expect(couch_replicator_clustering, link_cluster_event_listener, 3, - ok), + meck:expect( + couch_replicator_clustering, + link_cluster_event_listener, + 3, + ok + ), meck:expect(couch_replicator_doc_processor_worker, spawn_worker, fun ({?EXIT_DB, _}, _, _, _) -> exit(kapow); (_, _, _, _) -> pid @@ -852,11 +828,9 @@ setup_all() -> meck:expect(couch_replicator_docs, remove_state_fields, 2, ok), meck:expect(couch_replicator_docs, update_failed, 3, ok). - teardown_all(_) -> meck:unload(). - setup() -> meck:reset([ config, @@ -873,30 +847,29 @@ setup() -> unlink(Pid), Pid. - teardown(Pid) -> - test_util:stop_sync(Pid, kill, 1000). % 1s wait should suffice - + % 1s wait should suffice + test_util:stop_sync(Pid, kill, 1000). removed_state_fields() -> meck:called(couch_replicator_docs, remove_state_fields, [?DB, ?DOC1]). - started_worker(_Id) -> 1 == meck:num_calls(couch_replicator_doc_processor_worker, spawn_worker, 4). - removed_job(Id) -> meck:called(couch_replicator_scheduler, remove_job, [test_rep(Id)]). - did_not_remove_state_fields() -> 0 == meck:num_calls(couch_replicator_docs, remove_state_fields, '_'). - did_not_spawn_worker() -> - 0 == meck:num_calls(couch_replicator_doc_processor_worker, spawn_worker, - '_'). + 0 == + meck:num_calls( + couch_replicator_doc_processor_worker, + spawn_worker, + '_' + ). updated_doc_with_failed_state() -> 1 == meck:num_calls(couch_replicator_docs, update_failed, '_'). @@ -910,53 +883,52 @@ mock_existing_jobs_lookup(ExistingJobs) -> (?DB, ?DOC1) -> ExistingJobs end). - test_rep(Id) -> - #rep{id = Id, start_time = {0, 0, 0}}. - + #rep{id = Id, start_time = {0, 0, 0}}. change() -> {[ {<<"id">>, ?DOC1}, - {doc, {[ - {<<"_id">>, ?DOC1}, - {<<"source">>, <<"http://srchost.local/src">>}, - {<<"target">>, <<"http://tgthost.local/tgt">>} - ]}} + {doc, + {[ + {<<"_id">>, ?DOC1}, + {<<"source">>, <<"http://srchost.local/src">>}, + {<<"target">>, <<"http://tgthost.local/tgt">>} + ]}} ]}. - change(State) -> {[ {<<"id">>, ?DOC1}, - {doc, {[ - {<<"_id">>, ?DOC1}, - {<<"source">>, <<"http://srchost.local/src">>}, - {<<"target">>, <<"http://tgthost.local/tgt">>}, - {<<"_replication_state">>, State} - ]}} + {doc, + {[ + {<<"_id">>, ?DOC1}, + {<<"source">>, <<"http://srchost.local/src">>}, + {<<"target">>, <<"http://tgthost.local/tgt">>}, + {<<"_replication_state">>, State} + ]}} ]}. - deleted_change() -> {[ {<<"id">>, ?DOC1}, {<<"deleted">>, true}, - {doc, {[ - {<<"_id">>, ?DOC1}, - {<<"source">>, <<"http://srchost.local/src">>}, - {<<"target">>, <<"http://tgthost.local/tgt">>} - ]}} + {doc, + {[ + {<<"_id">>, ?DOC1}, + {<<"source">>, <<"http://srchost.local/src">>}, + {<<"target">>, <<"http://tgthost.local/tgt">>} + ]}} ]}. - bad_change() -> {[ {<<"id">>, ?DOC2}, - {doc, {[ - {<<"_id">>, ?DOC2}, - {<<"source">>, <<"src">>} - ]}} + {doc, + {[ + {<<"_id">>, ?DOC2}, + {<<"source">>, <<"src">>} + ]}} ]}. -endif. diff --git a/src/couch_replicator/src/couch_replicator_doc_processor_worker.erl b/src/couch_replicator/src/couch_replicator_doc_processor_worker.erl index a4c829323..5d971151b 100644 --- a/src/couch_replicator/src/couch_replicator_doc_processor_worker.erl +++ b/src/couch_replicator/src/couch_replicator_doc_processor_worker.erl @@ -32,7 +32,6 @@ % hung forever. -define(WORKER_TIMEOUT_MSEC, 61000). - % Spawn a worker which attempts to calculate replication id then add a % replication job to scheduler. This function create a monitor to the worker % a worker will then exit with the #doc_worker_result{} record within @@ -46,7 +45,6 @@ spawn_worker(Id, Rep, WaitSec, WRef) -> end), Pid. - % Private functions -spec worker_fun(db_doc_id(), #rep{}, seconds(), reference()) -> no_return(). @@ -72,13 +70,15 @@ worker_fun(Id, Rep, WaitSec, WRef) -> exit(Pid, kill), {DbName, DocId} = Id, TimeoutSec = round(?WORKER_TIMEOUT_MSEC / 1000), - Msg = io_lib:format("Replication for db ~p doc ~p failed to start due " - "to timeout after ~B seconds", [DbName, DocId, TimeoutSec]), + Msg = io_lib:format( + "Replication for db ~p doc ~p failed to start due " + "to timeout after ~B seconds", + [DbName, DocId, TimeoutSec] + ), Result = {temporary_error, couch_util:to_binary(Msg)}, exit(#doc_worker_result{id = Id, wref = WRef, result = Result}) end. - % Try to start a replication. Used by a worker. This function should return % rep_start_result(), also throws {filter_fetch_error, Reason} if cannot fetch % filter.It can also block for an indeterminate amount of time while fetching @@ -86,51 +86,54 @@ worker_fun(Id, Rep, WaitSec, WRef) -> maybe_start_replication(Id, RepWithoutId, WRef) -> Rep = couch_replicator_docs:update_rep_id(RepWithoutId), case maybe_add_job_to_scheduler(Id, Rep, WRef) of - ignore -> - ignore; - {ok, RepId} -> - {ok, RepId}; - {temporary_error, Reason} -> - {temporary_error, Reason}; - {permanent_failure, Reason} -> - {DbName, DocId} = Id, - couch_replicator_docs:update_failed(DbName, DocId, Reason), - {permanent_failure, Reason} + ignore -> + ignore; + {ok, RepId} -> + {ok, RepId}; + {temporary_error, Reason} -> + {temporary_error, Reason}; + {permanent_failure, Reason} -> + {DbName, DocId} = Id, + couch_replicator_docs:update_failed(DbName, DocId, Reason), + {permanent_failure, Reason} end. - -spec maybe_add_job_to_scheduler(db_doc_id(), #rep{}, reference()) -> - rep_start_result(). + rep_start_result(). maybe_add_job_to_scheduler({DbName, DocId}, Rep, WRef) -> RepId = Rep#rep.id, case couch_replicator_scheduler:rep_state(RepId) of - nil -> - % Before adding a job check that this worker is still the current - % worker. This is to handle a race condition where a worker which was - % sleeping and then checking a replication filter may inadvertently - % re-add a replication which was already deleted. - case couch_replicator_doc_processor:get_worker_ref({DbName, DocId}) of - WRef -> - ok = couch_replicator_scheduler:add_job(Rep), + nil -> + % Before adding a job check that this worker is still the current + % worker. This is to handle a race condition where a worker which was + % sleeping and then checking a replication filter may inadvertently + % re-add a replication which was already deleted. + case couch_replicator_doc_processor:get_worker_ref({DbName, DocId}) of + WRef -> + ok = couch_replicator_scheduler:add_job(Rep), + {ok, RepId}; + _NilOrOtherWRef -> + ignore + end; + #rep{doc_id = DocId} -> {ok, RepId}; - _NilOrOtherWRef -> - ignore - end; - #rep{doc_id = DocId} -> - {ok, RepId}; - #rep{doc_id = null} -> - Msg = io_lib:format("Replication `~s` specified by document `~s`" - " already running as a transient replication, started via" - " `_replicate` API endpoint", [pp_rep_id(RepId), DocId]), - {temporary_error, couch_util:to_binary(Msg)}; - #rep{db_name = OtherDb, doc_id = OtherDocId} -> - Msg = io_lib:format("Replication `~s` specified by document `~s`" - " already started, triggered by document `~s` from db `~s`", - [pp_rep_id(RepId), DocId, OtherDocId, mem3:dbname(OtherDb)]), - {permanent_failure, couch_util:to_binary(Msg)} + #rep{doc_id = null} -> + Msg = io_lib:format( + "Replication `~s` specified by document `~s`" + " already running as a transient replication, started via" + " `_replicate` API endpoint", + [pp_rep_id(RepId), DocId] + ), + {temporary_error, couch_util:to_binary(Msg)}; + #rep{db_name = OtherDb, doc_id = OtherDocId} -> + Msg = io_lib:format( + "Replication `~s` specified by document `~s`" + " already started, triggered by document `~s` from db `~s`", + [pp_rep_id(RepId), DocId, OtherDocId, mem3:dbname(OtherDb)] + ), + {permanent_failure, couch_util:to_binary(Msg)} end. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -139,7 +142,6 @@ maybe_add_job_to_scheduler({DbName, DocId}, Rep, WRef) -> -define(DOC1, <<"doc1">>). -define(R1, {"ad08e05057046eabe898a2572bbfb573", ""}). - doc_processor_worker_test_() -> { foreach, @@ -156,94 +158,105 @@ doc_processor_worker_test_() -> ] }. - % Replication is already running, with same doc id. Ignore change. t_should_add_job() -> - ?_test(begin - Id = {?DB, ?DOC1}, - Rep = couch_replicator_docs:parse_rep_doc_without_id(change()), - ?assertEqual({ok, ?R1}, maybe_start_replication(Id, Rep, nil)), - ?assert(added_job()) - end). - + ?_test(begin + Id = {?DB, ?DOC1}, + Rep = couch_replicator_docs:parse_rep_doc_without_id(change()), + ?assertEqual({ok, ?R1}, maybe_start_replication(Id, Rep, nil)), + ?assert(added_job()) + end). % Replication is already running, with same doc id. Ignore change. t_already_running_same_docid() -> - ?_test(begin - Id = {?DB, ?DOC1}, - mock_already_running(?DB, ?DOC1), - Rep = couch_replicator_docs:parse_rep_doc_without_id(change()), - ?assertEqual({ok, ?R1}, maybe_start_replication(Id, Rep, nil)), - ?assert(did_not_add_job()) - end). - + ?_test(begin + Id = {?DB, ?DOC1}, + mock_already_running(?DB, ?DOC1), + Rep = couch_replicator_docs:parse_rep_doc_without_id(change()), + ?assertEqual({ok, ?R1}, maybe_start_replication(Id, Rep, nil)), + ?assert(did_not_add_job()) + end). % There is a transient replication with same replication id running. Ignore. t_already_running_transient() -> - ?_test(begin - Id = {?DB, ?DOC1}, - mock_already_running(null, null), - Rep = couch_replicator_docs:parse_rep_doc_without_id(change()), - ?assertMatch({temporary_error, _}, maybe_start_replication(Id, Rep, - nil)), - ?assert(did_not_add_job()) - end). - + ?_test(begin + Id = {?DB, ?DOC1}, + mock_already_running(null, null), + Rep = couch_replicator_docs:parse_rep_doc_without_id(change()), + ?assertMatch( + {temporary_error, _}, + maybe_start_replication( + Id, + Rep, + nil + ) + ), + ?assert(did_not_add_job()) + end). % There is a duplicate replication potentially from a different db and doc. % Write permanent failure to doc. t_already_running_other_db_other_doc() -> - ?_test(begin - Id = {?DB, ?DOC1}, - mock_already_running(<<"otherdb">>, <<"otherdoc">>), - Rep = couch_replicator_docs:parse_rep_doc_without_id(change()), - ?assertMatch({permanent_failure, _}, maybe_start_replication(Id, Rep, - nil)), - ?assert(did_not_add_job()), - 1 == meck:num_calls(couch_replicator_docs, update_failed, '_') - end). - + ?_test(begin + Id = {?DB, ?DOC1}, + mock_already_running(<<"otherdb">>, <<"otherdoc">>), + Rep = couch_replicator_docs:parse_rep_doc_without_id(change()), + ?assertMatch( + {permanent_failure, _}, + maybe_start_replication( + Id, + Rep, + nil + ) + ), + ?assert(did_not_add_job()), + 1 == meck:num_calls(couch_replicator_docs, update_failed, '_') + end). % Should spawn worker t_spawn_worker() -> - ?_test(begin - Id = {?DB, ?DOC1}, - Rep = couch_replicator_docs:parse_rep_doc_without_id(change()), - WRef = make_ref(), - meck:expect(couch_replicator_doc_processor, get_worker_ref, 1, WRef), - Pid = spawn_worker(Id, Rep, 0, WRef), - Res = receive {'DOWN', _Ref, process, Pid, Reason} -> Reason - after 1000 -> timeout end, - Expect = #doc_worker_result{id = Id, wref = WRef, result = {ok, ?R1}}, - ?assertEqual(Expect, Res), - ?assert(added_job()) - end). - + ?_test(begin + Id = {?DB, ?DOC1}, + Rep = couch_replicator_docs:parse_rep_doc_without_id(change()), + WRef = make_ref(), + meck:expect(couch_replicator_doc_processor, get_worker_ref, 1, WRef), + Pid = spawn_worker(Id, Rep, 0, WRef), + Res = + receive + {'DOWN', _Ref, process, Pid, Reason} -> Reason + after 1000 -> timeout + end, + Expect = #doc_worker_result{id = Id, wref = WRef, result = {ok, ?R1}}, + ?assertEqual(Expect, Res), + ?assert(added_job()) + end). % Should not add job if by the time worker got to fetching the filter % and getting a replication id, replication doc was deleted t_ignore_if_doc_deleted() -> - ?_test(begin - Id = {?DB, ?DOC1}, - Rep = couch_replicator_docs:parse_rep_doc_without_id(change()), - meck:expect(couch_replicator_doc_processor, get_worker_ref, 1, nil), - ?assertEqual(ignore, maybe_start_replication(Id, Rep, make_ref())), - ?assertNot(added_job()) - end). - + ?_test(begin + Id = {?DB, ?DOC1}, + Rep = couch_replicator_docs:parse_rep_doc_without_id(change()), + meck:expect(couch_replicator_doc_processor, get_worker_ref, 1, nil), + ?assertEqual(ignore, maybe_start_replication(Id, Rep, make_ref())), + ?assertNot(added_job()) + end). % Should not add job if by the time worker got to fetchign the filter % and building a replication id, another worker was spawned. t_ignore_if_worker_ref_does_not_match() -> ?_test(begin - Id = {?DB, ?DOC1}, - Rep = couch_replicator_docs:parse_rep_doc_without_id(change()), - meck:expect(couch_replicator_doc_processor, get_worker_ref, 1, - make_ref()), - ?assertEqual(ignore, maybe_start_replication(Id, Rep, make_ref())), - ?assertNot(added_job()) - end). - + Id = {?DB, ?DOC1}, + Rep = couch_replicator_docs:parse_rep_doc_without_id(change()), + meck:expect( + couch_replicator_doc_processor, + get_worker_ref, + 1, + make_ref() + ), + ?assertEqual(ignore, maybe_start_replication(Id, Rep, make_ref())), + ?assertNot(added_job()) + end). % Test helper functions @@ -256,29 +269,27 @@ setup() -> meck:expect(couch_replicator_doc_processor, get_worker_ref, 1, nil), ok. - teardown(_) -> meck:unload(). - mock_already_running(DbName, DocId) -> - meck:expect(couch_replicator_scheduler, rep_state, - fun(RepId) -> #rep{id = RepId, doc_id = DocId, db_name = DbName} end). - + meck:expect( + couch_replicator_scheduler, + rep_state, + fun(RepId) -> #rep{id = RepId, doc_id = DocId, db_name = DbName} end + ). added_job() -> 1 == meck:num_calls(couch_replicator_scheduler, add_job, '_'). - did_not_add_job() -> 0 == meck:num_calls(couch_replicator_scheduler, add_job, '_'). - change() -> {[ - {<<"_id">>, ?DOC1}, - {<<"source">>, <<"http://srchost.local/src">>}, - {<<"target">>, <<"http://tgthost.local/tgt">>} - ]}. + {<<"_id">>, ?DOC1}, + {<<"source">>, <<"http://srchost.local/src">>}, + {<<"target">>, <<"http://tgthost.local/tgt">>} + ]}. -endif. diff --git a/src/couch_replicator/src/couch_replicator_docs.erl b/src/couch_replicator/src/couch_replicator_docs.erl index 7c60e8a43..bcab46747 100644 --- a/src/couch_replicator/src/couch_replicator_docs.erl +++ b/src/couch_replicator/src/couch_replicator_docs.erl @@ -30,7 +30,6 @@ update_error/2 ]). - -include_lib("couch/include/couch_db.hrl"). -include_lib("ibrowse/include/ibrowse.hrl"). -include_lib("mem3/include/mem3.hrl"). @@ -49,45 +48,51 @@ get_json_value/3 ]). - -define(REP_DB_NAME, <<"_replicator">>). -define(REP_DESIGN_DOC, <<"_design/_replicator">>). -define(OWNER, <<"owner">>). --define(CTX, {user_ctx, #user_ctx{roles=[<<"_admin">>, <<"_replicator">>]}}). +-define(CTX, {user_ctx, #user_ctx{roles = [<<"_admin">>, <<"_replicator">>]}}). -define(replace(L, K, V), lists:keystore(K, 1, L, {K, V})). - remove_state_fields(DbName, DocId) -> update_rep_doc(DbName, DocId, [ {<<"_replication_state">>, undefined}, {<<"_replication_state_time">>, undefined}, {<<"_replication_state_reason">>, undefined}, {<<"_replication_id">>, undefined}, - {<<"_replication_stats">>, undefined}]). - + {<<"_replication_stats">>, undefined} + ]). -spec update_doc_completed(binary(), binary(), [_]) -> any(). update_doc_completed(DbName, DocId, Stats) -> update_rep_doc(DbName, DocId, [ {<<"_replication_state">>, <<"completed">>}, {<<"_replication_state_reason">>, undefined}, - {<<"_replication_stats">>, {Stats}}]), - couch_stats:increment_counter([couch_replicator, docs, - completed_state_updates]). - + {<<"_replication_stats">>, {Stats}} + ]), + couch_stats:increment_counter([ + couch_replicator, + docs, + completed_state_updates + ]). -spec update_failed(binary(), binary(), any()) -> any(). update_failed(DbName, DocId, Error) -> Reason = error_reason(Error), - couch_log:error("Error processing replication doc `~s` from `~s`: ~s", - [DocId, DbName, Reason]), + couch_log:error( + "Error processing replication doc `~s` from `~s`: ~s", + [DocId, DbName, Reason] + ), update_rep_doc(DbName, DocId, [ {<<"_replication_state">>, <<"failed">>}, {<<"_replication_stats">>, undefined}, - {<<"_replication_state_reason">>, Reason}]), - couch_stats:increment_counter([couch_replicator, docs, - failed_state_updates]). - + {<<"_replication_state_reason">>, Reason} + ]), + couch_stats:increment_counter([ + couch_replicator, + docs, + failed_state_updates + ]). -spec update_triggered(#rep{}, rep_id()) -> ok. update_triggered(Rep, {Base, Ext}) -> @@ -99,27 +104,28 @@ update_triggered(Rep, {Base, Ext}) -> {<<"_replication_state">>, <<"triggered">>}, {<<"_replication_state_reason">>, undefined}, {<<"_replication_id">>, iolist_to_binary([Base, Ext])}, - {<<"_replication_stats">>, undefined}]), + {<<"_replication_stats">>, undefined} + ]), ok. - -spec update_error(#rep{}, any()) -> ok. update_error(#rep{db_name = DbName, doc_id = DocId, id = RepId}, Error) -> Reason = error_reason(Error), - BinRepId = case RepId of - {Base, Ext} -> - iolist_to_binary([Base, Ext]); - _Other -> - null - end, + BinRepId = + case RepId of + {Base, Ext} -> + iolist_to_binary([Base, Ext]); + _Other -> + null + end, update_rep_doc(DbName, DocId, [ {<<"_replication_state">>, <<"error">>}, {<<"_replication_state_reason">>, Reason}, {<<"_replication_stats">>, undefined}, - {<<"_replication_id">>, BinRepId}]), + {<<"_replication_id">>, BinRepId} + ]), ok. - -spec ensure_rep_ddoc_exists(binary()) -> ok. ensure_rep_ddoc_exists(RepDb) -> case mem3:belongs(RepDb, ?REP_DESIGN_DOC) of @@ -129,7 +135,6 @@ ensure_rep_ddoc_exists(RepDb) -> ok end. - -spec ensure_rep_ddoc_exists(binary(), binary()) -> ok. ensure_rep_ddoc_exists(RepDb, DDocId) -> case open_rep_doc(RepDb, DDocId) of @@ -163,21 +168,18 @@ ensure_rep_ddoc_exists(RepDb, DDocId) -> end, ok. - -spec ensure_cluster_rep_ddoc_exists(binary()) -> ok. ensure_cluster_rep_ddoc_exists(RepDb) -> DDocId = ?REP_DESIGN_DOC, [#shard{name = DbShard} | _] = mem3:shards(RepDb, DDocId), ensure_rep_ddoc_exists(DbShard, DDocId). - -spec compare_ejson({[_]}, {[_]}) -> boolean(). compare_ejson(EJson1, EJson2) -> EjsonSorted1 = couch_replicator_filters:ejsort(EJson1), EjsonSorted2 = couch_replicator_filters:ejsort(EJson2), EjsonSorted1 == EjsonSorted2. - -spec replication_design_doc_props(binary()) -> [_]. replication_design_doc_props(DDocId) -> [ @@ -186,7 +188,6 @@ replication_design_doc_props(DDocId) -> {<<"validate_doc_update">>, ?REP_DB_DOC_VALIDATE_FUN} ]. - % Note: parse_rep_doc can handle filtered replications. During parsing of the % replication doc it will make possibly remote http requests to the source % database. If failure or parsing of filter docs fails, parse_doc throws a @@ -195,32 +196,32 @@ replication_design_doc_props(DDocId) -> % on netowrk availability of the source db and other factors. -spec parse_rep_doc({[_]}) -> #rep{}. parse_rep_doc(RepDoc) -> - {ok, Rep} = try - parse_rep_doc(RepDoc, rep_user_ctx(RepDoc)) - catch - throw:{error, Reason} -> - throw({bad_rep_doc, Reason}); - throw:{filter_fetch_error, Reason} -> - throw({filter_fetch_error, Reason}); - Tag:Err -> - throw({bad_rep_doc, to_binary({Tag, Err})}) - end, + {ok, Rep} = + try + parse_rep_doc(RepDoc, rep_user_ctx(RepDoc)) + catch + throw:{error, Reason} -> + throw({bad_rep_doc, Reason}); + throw:{filter_fetch_error, Reason} -> + throw({filter_fetch_error, Reason}); + Tag:Err -> + throw({bad_rep_doc, to_binary({Tag, Err})}) + end, Rep. - -spec parse_rep_doc_without_id({[_]}) -> #rep{}. parse_rep_doc_without_id(RepDoc) -> - {ok, Rep} = try - parse_rep_doc_without_id(RepDoc, rep_user_ctx(RepDoc)) - catch - throw:{error, Reason} -> - throw({bad_rep_doc, Reason}); - Tag:Err -> - throw({bad_rep_doc, to_binary({Tag, Err})}) - end, + {ok, Rep} = + try + parse_rep_doc_without_id(RepDoc, rep_user_ctx(RepDoc)) + catch + throw:{error, Reason} -> + throw({bad_rep_doc, Reason}); + Tag:Err -> + throw({bad_rep_doc, to_binary({Tag, Err})}) + end, Rep. - -spec parse_rep_doc({[_]}, #user_ctx{}) -> {ok, #rep{}}. parse_rep_doc(Doc, UserCtx) -> {ok, Rep} = parse_rep_doc_without_id(Doc, UserCtx), @@ -238,44 +239,45 @@ parse_rep_doc(Doc, UserCtx) -> {ok, update_rep_id(Rep)} end. - -spec parse_rep_doc_without_id({[_]}, #user_ctx{}) -> {ok, #rep{}}. parse_rep_doc_without_id({Props}, UserCtx) -> {SrcProxy, TgtProxy} = parse_proxy_settings(Props), Opts = make_options(Props), - case get_value(cancel, Opts, false) andalso - (get_value(id, Opts, nil) =/= nil) of - true -> - {ok, #rep{options = Opts, user_ctx = UserCtx}}; - false -> - Source = parse_rep_db(get_value(<<"source">>, Props), SrcProxy, Opts), - Target = parse_rep_db(get_value(<<"target">>, Props), TgtProxy, Opts), - {Type, View} = case couch_replicator_filters:view_type(Props, Opts) of - {error, Error} -> - throw({bad_request, Error}); - Result -> - Result - end, - Rep = #rep{ - source = Source, - target = Target, - options = Opts, - user_ctx = UserCtx, - type = Type, - view = View, - doc_id = get_value(<<"_id">>, Props, null) - }, - % Check if can parse filter code, if not throw exception - case couch_replicator_filters:parse(Opts) of - {error, FilterError} -> - throw({error, FilterError}); - {ok, _Filter} -> - ok - end, - {ok, Rep} + case + get_value(cancel, Opts, false) andalso + (get_value(id, Opts, nil) =/= nil) + of + true -> + {ok, #rep{options = Opts, user_ctx = UserCtx}}; + false -> + Source = parse_rep_db(get_value(<<"source">>, Props), SrcProxy, Opts), + Target = parse_rep_db(get_value(<<"target">>, Props), TgtProxy, Opts), + {Type, View} = + case couch_replicator_filters:view_type(Props, Opts) of + {error, Error} -> + throw({bad_request, Error}); + Result -> + Result + end, + Rep = #rep{ + source = Source, + target = Target, + options = Opts, + user_ctx = UserCtx, + type = Type, + view = View, + doc_id = get_value(<<"_id">>, Props, null) + }, + % Check if can parse filter code, if not throw exception + case couch_replicator_filters:parse(Opts) of + {error, FilterError} -> + throw({error, FilterError}); + {ok, _Filter} -> + ok + end, + {ok, Rep} end. - parse_proxy_settings(Props) when is_list(Props) -> Proxy = get_value(<<"proxy">>, Props, <<>>), SrcProxy = get_value(<<"source_proxy">>, Props, <<>>), @@ -285,7 +287,7 @@ parse_proxy_settings(Props) when is_list(Props) -> true when SrcProxy =/= <<>> -> Error = "`proxy` is mutually exclusive with `source_proxy`", throw({bad_request, Error}); - true when TgtProxy =/= <<>> -> + true when TgtProxy =/= <<>> -> Error = "`proxy` is mutually exclusive with `target_proxy`", throw({bad_request, Error}); true -> @@ -294,7 +296,6 @@ parse_proxy_settings(Props) when is_list(Props) -> {SrcProxy, TgtProxy} end. - % Update a #rep{} record with a replication_id. Calculating the id might involve % fetching a filter from the source db, and so it could fail intermetently. % In case of a failure to fetch the filter this function will throw a @@ -303,11 +304,9 @@ update_rep_id(Rep) -> RepId = couch_replicator_ids:replication_id(Rep), Rep#rep{id = RepId}. - update_rep_doc(RepDbName, RepDocId, KVs) -> update_rep_doc(RepDbName, RepDocId, KVs, 1). - update_rep_doc(RepDbName, RepDocId, KVs, Wait) when is_binary(RepDocId) -> try case open_rep_doc(RepDbName, RepDocId) of @@ -323,36 +322,40 @@ update_rep_doc(RepDbName, RepDocId, KVs, Wait) when is_binary(RepDocId) -> ok = timer:sleep(couch_rand:uniform(erlang:min(128, Wait)) * 100), update_rep_doc(RepDbName, RepDocId, KVs, Wait * 2) end; - update_rep_doc(RepDbName, #doc{body = {RepDocBody}} = RepDoc, KVs, _Try) -> NewRepDocBody = lists:foldl( - fun({K, undefined}, Body) -> + fun + ({K, undefined}, Body) -> lists:keydelete(K, 1, Body); - ({<<"_replication_state">> = K, State} = KV, Body) -> + ({<<"_replication_state">> = K, State} = KV, Body) -> case get_json_value(K, Body) of - State -> - Body; - _ -> - Body1 = lists:keystore(K, 1, Body, KV), - Timestamp = couch_replicator_utils:iso8601(os:timestamp()), - lists:keystore( - <<"_replication_state_time">>, 1, Body1, - {<<"_replication_state_time">>, Timestamp}) + State -> + Body; + _ -> + Body1 = lists:keystore(K, 1, Body, KV), + Timestamp = couch_replicator_utils:iso8601(os:timestamp()), + lists:keystore( + <<"_replication_state_time">>, + 1, + Body1, + {<<"_replication_state_time">>, Timestamp} + ) end; ({K, _V} = KV, Body) -> lists:keystore(K, 1, Body, KV) end, - RepDocBody, KVs), + RepDocBody, + KVs + ), case NewRepDocBody of - RepDocBody -> - ok; - _ -> - % Might not succeed - when the replication doc is deleted right - % before this update (not an error, ignore). - save_rep_doc(RepDbName, RepDoc#doc{body = {NewRepDocBody}}) + RepDocBody -> + ok; + _ -> + % Might not succeed - when the replication doc is deleted right + % before this update (not an error, ignore). + save_rep_doc(RepDbName, RepDoc#doc{body = {NewRepDocBody}}) end. - open_rep_doc(DbName, DocId) -> case couch_db:open_int(DbName, [?CTX, sys_db]) of {ok, Db} -> @@ -365,7 +368,6 @@ open_rep_doc(DbName, DocId) -> Else end. - save_rep_doc(DbName, Doc) -> {ok, Db} = couch_db:open_int(DbName, [?CTX, sys_db]), try @@ -382,27 +384,26 @@ save_rep_doc(DbName, Doc) -> couch_db:close(Db) end. - -spec rep_user_ctx({[_]}) -> #user_ctx{}. rep_user_ctx({RepDoc}) -> case get_json_value(<<"user_ctx">>, RepDoc) of - undefined -> - #user_ctx{}; - {UserCtx} -> - #user_ctx{ - name = get_json_value(<<"name">>, UserCtx, null), - roles = get_json_value(<<"roles">>, UserCtx, []) - } + undefined -> + #user_ctx{}; + {UserCtx} -> + #user_ctx{ + name = get_json_value(<<"name">>, UserCtx, null), + roles = get_json_value(<<"roles">>, UserCtx, []) + } end. - -spec parse_rep_db({[_]} | binary(), binary(), [_]) -> #httpd{} | binary(). parse_rep_db({Props}, Proxy, Options) -> ProxyParams = parse_proxy_params(Proxy), - ProxyURL = case ProxyParams of - [] -> undefined; - _ -> binary_to_list(Proxy) - end, + ProxyURL = + case ProxyParams of + [] -> undefined; + _ -> binary_to_list(Proxy) + end, Url = maybe_add_trailing_slash(get_value(<<"url">>, Props)), {AuthProps} = get_value(<<"auth">>, Props, {[]}), {BinHeaders} = get_value(<<"headers">>, Props, {[]}), @@ -412,36 +413,36 @@ parse_rep_db({Props}, Proxy, Options) -> url = Url, auth_props = AuthProps, headers = lists:ukeymerge(1, Headers, DefaultHeaders), - ibrowse_options = lists:keysort(1, - [{socket_options, get_value(socket_options, Options)} | - ProxyParams ++ ssl_params(Url)]), + ibrowse_options = lists:keysort( + 1, + [ + {socket_options, get_value(socket_options, Options)} + | ProxyParams ++ ssl_params(Url) + ] + ), timeout = get_value(connection_timeout, Options), http_connections = get_value(http_connections, Options), retries = get_value(retries, Options), proxy_url = ProxyURL }, couch_replicator_utils:normalize_basic_auth(HttpDb); - parse_rep_db(<<"http://", _/binary>> = Url, Proxy, Options) -> parse_rep_db({[{<<"url">>, Url}]}, Proxy, Options); - parse_rep_db(<<"https://", _/binary>> = Url, Proxy, Options) -> parse_rep_db({[{<<"url">>, Url}]}, Proxy, Options); - parse_rep_db(<<_/binary>>, _Proxy, _Options) -> throw({error, local_endpoints_not_supported}); - parse_rep_db(undefined, _Proxy, _Options) -> throw({error, <<"Missing replicator database">>}). - -spec maybe_add_trailing_slash(binary() | list()) -> list(). maybe_add_trailing_slash(Url) when is_binary(Url) -> maybe_add_trailing_slash(?b2l(Url)); maybe_add_trailing_slash(Url) -> case lists:member($?, Url) of true -> - Url; % skip if there are query params + % skip if there are query params + Url; false -> case lists:last(Url) of $/ -> @@ -451,7 +452,6 @@ maybe_add_trailing_slash(Url) -> end end. - -spec make_options([_]) -> [_]. make_options(Props) -> Options0 = lists:ukeysort(1, convert_options(Props)), @@ -462,43 +462,55 @@ make_options(Props) -> DefTimeout = config:get_integer("replicator", "connection_timeout", 30000), DefRetries = config:get_integer("replicator", "retries_per_request", 5), UseCheckpoints = config:get_boolean("replicator", "use_checkpoints", true), - DefCheckpointInterval = config:get_integer("replicator", - "checkpoint_interval", 30000), + DefCheckpointInterval = config:get_integer( + "replicator", + "checkpoint_interval", + 30000 + ), {ok, DefSocketOptions} = couch_util:parse_term( - config:get("replicator", "socket_options", - "[{keepalive, true}, {nodelay, false}]")), - lists:ukeymerge(1, Options, lists:keysort(1, [ - {connection_timeout, DefTimeout}, - {retries, DefRetries}, - {http_connections, DefConns}, - {socket_options, DefSocketOptions}, - {worker_batch_size, DefBatchSize}, - {worker_processes, DefWorkers}, - {use_checkpoints, UseCheckpoints}, - {checkpoint_interval, DefCheckpointInterval} - ])). - + config:get( + "replicator", + "socket_options", + "[{keepalive, true}, {nodelay, false}]" + ) + ), + lists:ukeymerge( + 1, + Options, + lists:keysort(1, [ + {connection_timeout, DefTimeout}, + {retries, DefRetries}, + {http_connections, DefConns}, + {socket_options, DefSocketOptions}, + {worker_batch_size, DefBatchSize}, + {worker_processes, DefWorkers}, + {use_checkpoints, UseCheckpoints}, + {checkpoint_interval, DefCheckpointInterval} + ]) + ). -spec convert_options([_]) -> [_]. -convert_options([])-> +convert_options([]) -> []; -convert_options([{<<"cancel">>, V} | _R]) when not is_boolean(V)-> +convert_options([{<<"cancel">>, V} | _R]) when not is_boolean(V) -> throw({bad_request, <<"parameter `cancel` must be a boolean">>}); convert_options([{<<"cancel">>, V} | R]) -> [{cancel, V} | convert_options(R)]; -convert_options([{IdOpt, V} | R]) when IdOpt =:= <<"_local_id">>; - IdOpt =:= <<"replication_id">>; IdOpt =:= <<"id">> -> +convert_options([{IdOpt, V} | R]) when + IdOpt =:= <<"_local_id">>; + IdOpt =:= <<"replication_id">>; + IdOpt =:= <<"id">> +-> [{id, couch_replicator_ids:convert(V)} | convert_options(R)]; -convert_options([{<<"create_target">>, V} | _R]) when not is_boolean(V)-> +convert_options([{<<"create_target">>, V} | _R]) when not is_boolean(V) -> throw({bad_request, <<"parameter `create_target` must be a boolean">>}); convert_options([{<<"create_target">>, V} | R]) -> [{create_target, V} | convert_options(R)]; convert_options([{<<"create_target_params">>, V} | _R]) when not is_tuple(V) -> - throw({bad_request, - <<"parameter `create_target_params` must be a JSON object">>}); + throw({bad_request, <<"parameter `create_target_params` must be a JSON object">>}); convert_options([{<<"create_target_params">>, V} | R]) -> [{create_target_params, V} | convert_options(R)]; -convert_options([{<<"continuous">>, V} | _R]) when not is_boolean(V)-> +convert_options([{<<"continuous">>, V} | _R]) when not is_boolean(V) -> throw({bad_request, <<"parameter `continuous` must be a boolean">>}); convert_options([{<<"continuous">>, V} | R]) -> [{continuous, V} | convert_options(R)]; @@ -538,10 +550,10 @@ convert_options([{<<"use_checkpoints">>, V} | R]) -> [{use_checkpoints, V} | convert_options(R)]; convert_options([{<<"checkpoint_interval">>, V} | R]) -> [{checkpoint_interval, couch_util:to_integer(V)} | convert_options(R)]; -convert_options([_ | R]) -> % skip unknown option +% skip unknown option +convert_options([_ | R]) -> convert_options(R). - -spec check_options([_]) -> [_]. check_options(Options) -> DocIds = lists:keyfind(doc_ids, 1, Options), @@ -552,12 +564,9 @@ check_options(Options) -> {false, false, _} -> Options; {false, _, false} -> Options; {_, false, false} -> Options; - _ -> - throw({bad_request, - "`doc_ids`,`filter`,`selector` are mutually exclusive"}) + _ -> throw({bad_request, "`doc_ids`,`filter`,`selector` are mutually exclusive"}) end. - -spec parse_proxy_params(binary() | [_]) -> [_]. parse_proxy_params(ProxyUrl) when is_binary(ProxyUrl) -> parse_proxy_params(?b2l(ProxyUrl)); @@ -571,15 +580,17 @@ parse_proxy_params(ProxyUrl) -> password = Passwd, protocol = Protocol } = ibrowse_lib:parse_url(ProxyUrl), - Params = [ - {proxy_host, Host}, - {proxy_port, Port} - ] ++ case is_list(User) andalso is_list(Passwd) of - false -> - []; - true -> - [{proxy_user, User}, {proxy_password, Passwd}] - end, + Params = + [ + {proxy_host, Host}, + {proxy_port, Port} + ] ++ + case is_list(User) andalso is_list(Passwd) of + false -> + []; + true -> + [{proxy_user, User}, {proxy_password, Passwd}] + end, case Protocol of socks5 -> [proxy_to_socks5(Param) || Param <- Params]; @@ -587,7 +598,6 @@ parse_proxy_params(ProxyUrl) -> Params end. - -spec proxy_to_socks5({atom(), string()}) -> {atom(), string()}. proxy_to_socks5({proxy_host, Val}) -> {socks5_host, Val}; @@ -598,36 +608,45 @@ proxy_to_socks5({proxy_user, Val}) -> proxy_to_socks5({proxy_password, Val}) -> {socks5_password, Val}. - -spec ssl_params([_]) -> [_]. ssl_params(Url) -> case ibrowse_lib:parse_url(Url) of - #url{protocol = https} -> - Depth = config:get_integer("replicator", - "ssl_certificate_max_depth", 3), - VerifyCerts = config:get_boolean("replicator", - "verify_ssl_certificates", false), - CertFile = config:get("replicator", "cert_file", undefined), - KeyFile = config:get("replicator", "key_file", undefined), - Password = config:get("replicator", "password", undefined), - SslOpts = [{depth, Depth} | ssl_verify_options(VerifyCerts)], - SslOpts1 = case CertFile /= undefined andalso KeyFile /= undefined of - true -> - case Password of - undefined -> - [{certfile, CertFile}, {keyfile, KeyFile}] ++ SslOpts; - _ -> - [{certfile, CertFile}, {keyfile, KeyFile}, - {password, Password}] ++ SslOpts - end; - false -> SslOpts - end, - [{is_ssl, true}, {ssl_options, SslOpts1}]; - #url{protocol = http} -> - [] + #url{protocol = https} -> + Depth = config:get_integer( + "replicator", + "ssl_certificate_max_depth", + 3 + ), + VerifyCerts = config:get_boolean( + "replicator", + "verify_ssl_certificates", + false + ), + CertFile = config:get("replicator", "cert_file", undefined), + KeyFile = config:get("replicator", "key_file", undefined), + Password = config:get("replicator", "password", undefined), + SslOpts = [{depth, Depth} | ssl_verify_options(VerifyCerts)], + SslOpts1 = + case CertFile /= undefined andalso KeyFile /= undefined of + true -> + case Password of + undefined -> + [{certfile, CertFile}, {keyfile, KeyFile}] ++ SslOpts; + _ -> + [ + {certfile, CertFile}, + {keyfile, KeyFile}, + {password, Password} + ] ++ SslOpts + end; + false -> + SslOpts + end, + [{is_ssl, true}, {ssl_options, SslOpts1}]; + #url{protocol = http} -> + [] end. - -spec ssl_verify_options(true | false) -> [_]. ssl_verify_options(true) -> CAFile = config:get("replicator", "ssl_trusted_certificates_file"), @@ -635,75 +654,85 @@ ssl_verify_options(true) -> ssl_verify_options(false) -> [{verify, verify_none}]. - --spec before_doc_update(#doc{}, Db::any(), couch_db:update_type()) -> #doc{}. +-spec before_doc_update(#doc{}, Db :: any(), couch_db:update_type()) -> #doc{}. before_doc_update(#doc{id = <<?DESIGN_DOC_PREFIX, _/binary>>} = Doc, _Db, _UpdateType) -> Doc; before_doc_update(#doc{body = {Body}} = Doc, Db, _UpdateType) -> #user_ctx{ - roles = Roles, - name = Name + roles = Roles, + name = Name } = couch_db:get_user_ctx(Db), case lists:member(<<"_replicator">>, Roles) of - true -> - Doc; - false -> - case couch_util:get_value(?OWNER, Body) of - undefined -> - Doc#doc{body = {?replace(Body, ?OWNER, Name)}}; - Name -> + true -> Doc; - Other -> - case (catch couch_db:check_is_admin(Db)) of - ok when Other =:= null -> - Doc#doc{body = {?replace(Body, ?OWNER, Name)}}; - ok -> - Doc; - _ -> - throw({forbidden, <<"Can't update replication documents", - " from other users.">>}) + false -> + case couch_util:get_value(?OWNER, Body) of + undefined -> + Doc#doc{body = {?replace(Body, ?OWNER, Name)}}; + Name -> + Doc; + Other -> + case (catch couch_db:check_is_admin(Db)) of + ok when Other =:= null -> + Doc#doc{body = {?replace(Body, ?OWNER, Name)}}; + ok -> + Doc; + _ -> + throw( + {forbidden, + <<"Can't update replication documents", " from other users.">>} + ) + end end - end end. - --spec after_doc_read(#doc{}, Db::any()) -> #doc{}. +-spec after_doc_read(#doc{}, Db :: any()) -> #doc{}. after_doc_read(#doc{id = <<?DESIGN_DOC_PREFIX, _/binary>>} = Doc, _Db) -> Doc; after_doc_read(#doc{body = {Body}} = Doc, Db) -> #user_ctx{name = Name} = couch_db:get_user_ctx(Db), case (catch couch_db:check_is_admin(Db)) of - ok -> - Doc; - _ -> - case couch_util:get_value(?OWNER, Body) of - Name -> + ok -> Doc; - _Other -> - Source = strip_credentials(couch_util:get_value(<<"source">>, -Body)), - Target = strip_credentials(couch_util:get_value(<<"target">>, -Body)), - NewBody0 = ?replace(Body, <<"source">>, Source), - NewBody = ?replace(NewBody0, <<"target">>, Target), - #doc{revs = {Pos, [_ | Revs]}} = Doc, - NewDoc = Doc#doc{body = {NewBody}, revs = {Pos - 1, Revs}}, - NewRevId = couch_db:new_revid(NewDoc), - NewDoc#doc{revs = {Pos, [NewRevId | Revs]}} - end + _ -> + case couch_util:get_value(?OWNER, Body) of + Name -> + Doc; + _Other -> + Source = strip_credentials( + couch_util:get_value( + <<"source">>, + Body + ) + ), + Target = strip_credentials( + couch_util:get_value( + <<"target">>, + Body + ) + ), + NewBody0 = ?replace(Body, <<"source">>, Source), + NewBody = ?replace(NewBody0, <<"target">>, Target), + #doc{revs = {Pos, [_ | Revs]}} = Doc, + NewDoc = Doc#doc{body = {NewBody}, revs = {Pos - 1, Revs}}, + NewRevId = couch_db:new_revid(NewDoc), + NewDoc#doc{revs = {Pos, [NewRevId | Revs]}} + end end. - --spec strip_credentials(undefined) -> undefined; +-spec strip_credentials + (undefined) -> undefined; (binary()) -> binary(); ({[_]}) -> {[_]}. strip_credentials(undefined) -> undefined; strip_credentials(Url) when is_binary(Url) -> - re:replace(Url, + re:replace( + Url, "http(s)?://(?:[^:]+):[^@]+@(.*)$", "http\\1://\\2", - [{return, binary}]); + [{return, binary}] + ); strip_credentials({Props0}) -> Props1 = lists:keydelete(<<"headers">>, 1, Props0), % Strip "auth" just like headers, for replication plugins it can be a place @@ -711,26 +740,23 @@ strip_credentials({Props0}) -> Props2 = lists:keydelete(<<"auth">>, 1, Props1), {Props2}. - error_reason({shutdown, Error}) -> error_reason(Error); error_reason({bad_rep_doc, Reason}) -> to_binary(Reason); -error_reason({error, {Error, Reason}}) - when is_atom(Error), is_binary(Reason) -> +error_reason({error, {Error, Reason}}) when + is_atom(Error), is_binary(Reason) +-> to_binary(io_lib:format("~s: ~s", [Error, Reason])); error_reason({error, Reason}) -> to_binary(Reason); error_reason(Reason) -> to_binary(Reason). - -ifdef(TEST). - -include_lib("couch/include/couch_eunit.hrl"). - check_options_pass_values_test() -> ?assertEqual(check_options([]), []), ?assertEqual(check_options([baz, {other, fiz}]), [baz, {other, fiz}]), @@ -738,88 +764,120 @@ check_options_pass_values_test() -> ?assertEqual(check_options([{filter, x}]), [{filter, x}]), ?assertEqual(check_options([{selector, x}]), [{selector, x}]). - check_options_fail_values_test() -> - ?assertThrow({bad_request, _}, - check_options([{doc_ids, x}, {filter, y}])), - ?assertThrow({bad_request, _}, - check_options([{doc_ids, x}, {selector, y}])), - ?assertThrow({bad_request, _}, - check_options([{filter, x}, {selector, y}])), - ?assertThrow({bad_request, _}, - check_options([{doc_ids, x}, {selector, y}, {filter, z}])). - + ?assertThrow( + {bad_request, _}, + check_options([{doc_ids, x}, {filter, y}]) + ), + ?assertThrow( + {bad_request, _}, + check_options([{doc_ids, x}, {selector, y}]) + ), + ?assertThrow( + {bad_request, _}, + check_options([{filter, x}, {selector, y}]) + ), + ?assertThrow( + {bad_request, _}, + check_options([{doc_ids, x}, {selector, y}, {filter, z}]) + ). check_convert_options_pass_test() -> ?assertEqual([], convert_options([])), ?assertEqual([], convert_options([{<<"random">>, 42}])), - ?assertEqual([{cancel, true}], - convert_options([{<<"cancel">>, true}])), - ?assertEqual([{create_target, true}], - convert_options([{<<"create_target">>, true}])), - ?assertEqual([{continuous, true}], - convert_options([{<<"continuous">>, true}])), - ?assertEqual([{doc_ids, [<<"id">>]}], - convert_options([{<<"doc_ids">>, [<<"id">>]}])), - ?assertEqual([{selector, {key, value}}], - convert_options([{<<"selector">>, {key, value}}])). - + ?assertEqual( + [{cancel, true}], + convert_options([{<<"cancel">>, true}]) + ), + ?assertEqual( + [{create_target, true}], + convert_options([{<<"create_target">>, true}]) + ), + ?assertEqual( + [{continuous, true}], + convert_options([{<<"continuous">>, true}]) + ), + ?assertEqual( + [{doc_ids, [<<"id">>]}], + convert_options([{<<"doc_ids">>, [<<"id">>]}]) + ), + ?assertEqual( + [{selector, {key, value}}], + convert_options([{<<"selector">>, {key, value}}]) + ). check_convert_options_fail_test() -> - ?assertThrow({bad_request, _}, - convert_options([{<<"cancel">>, <<"true">>}])), - ?assertThrow({bad_request, _}, - convert_options([{<<"create_target">>, <<"true">>}])), - ?assertThrow({bad_request, _}, - convert_options([{<<"continuous">>, <<"true">>}])), - ?assertThrow({bad_request, _}, - convert_options([{<<"doc_ids">>, not_a_list}])), - ?assertThrow({bad_request, _}, - convert_options([{<<"selector">>, [{key, value}]}])). + ?assertThrow( + {bad_request, _}, + convert_options([{<<"cancel">>, <<"true">>}]) + ), + ?assertThrow( + {bad_request, _}, + convert_options([{<<"create_target">>, <<"true">>}]) + ), + ?assertThrow( + {bad_request, _}, + convert_options([{<<"continuous">>, <<"true">>}]) + ), + ?assertThrow( + {bad_request, _}, + convert_options([{<<"doc_ids">>, not_a_list}]) + ), + ?assertThrow( + {bad_request, _}, + convert_options([{<<"selector">>, [{key, value}]}]) + ). check_strip_credentials_test() -> - [?assertEqual(Expected, strip_credentials(Body)) || {Expected, Body} <- [ - { - undefined, - undefined - }, - { - <<"https://remote_server/database">>, - <<"https://foo:bar@remote_server/database">> - }, - { - {[{<<"_id">>, <<"foo">>}]}, - {[{<<"_id">>, <<"foo">>}, {<<"headers">>, <<"bar">>}]} - }, - { - {[{<<"_id">>, <<"foo">>}, {<<"other">>, <<"bar">>}]}, - {[{<<"_id">>, <<"foo">>}, {<<"other">>, <<"bar">>}]} - }, - { - {[{<<"_id">>, <<"foo">>}]}, - {[{<<"_id">>, <<"foo">>}, {<<"headers">>, <<"baz">>}]} - }, - { - {[{<<"_id">>, <<"foo">>}]}, - {[{<<"_id">>, <<"foo">>}, {<<"auth">>, <<"pluginsecret">>}]} - } - ]]. - + [ + ?assertEqual(Expected, strip_credentials(Body)) + || {Expected, Body} <- [ + { + undefined, + undefined + }, + { + <<"https://remote_server/database">>, + <<"https://foo:bar@remote_server/database">> + }, + { + {[{<<"_id">>, <<"foo">>}]}, + {[{<<"_id">>, <<"foo">>}, {<<"headers">>, <<"bar">>}]} + }, + { + {[{<<"_id">>, <<"foo">>}, {<<"other">>, <<"bar">>}]}, + {[{<<"_id">>, <<"foo">>}, {<<"other">>, <<"bar">>}]} + }, + { + {[{<<"_id">>, <<"foo">>}]}, + {[{<<"_id">>, <<"foo">>}, {<<"headers">>, <<"baz">>}]} + }, + { + {[{<<"_id">>, <<"foo">>}]}, + {[{<<"_id">>, <<"foo">>}, {<<"auth">>, <<"pluginsecret">>}]} + } + ] + ]. parse_proxy_params_test() -> - ?assertEqual([ - {proxy_host, "foo.com"}, - {proxy_port, 443}, - {proxy_user, "u"}, - {proxy_password, "p"} - ], parse_proxy_params("https://u:p@foo.com")), - ?assertEqual([ - {socks5_host, "foo.com"}, - {socks5_port, 1080}, - {socks5_user, "u"}, - {socks5_password, "p"} - ], parse_proxy_params("socks5://u:p@foo.com")). - + ?assertEqual( + [ + {proxy_host, "foo.com"}, + {proxy_port, 443}, + {proxy_user, "u"}, + {proxy_password, "p"} + ], + parse_proxy_params("https://u:p@foo.com") + ), + ?assertEqual( + [ + {socks5_host, "foo.com"}, + {socks5_port, 1080}, + {socks5_user, "u"}, + {socks5_password, "p"} + ], + parse_proxy_params("socks5://u:p@foo.com") + ). setup() -> DbName = ?tempdb(), @@ -828,12 +886,10 @@ setup() -> create_vdu(DbName), DbName. - teardown(DbName) when is_binary(DbName) -> couch_server:delete(DbName, [?ADMIN_CTX]), ok. - create_vdu(DbName) -> couch_util:with_db(DbName, fun(Db) -> VduFun = <<"function(newdoc, olddoc, userctx) {throw({'forbidden':'fail'})}">>, @@ -844,48 +900,51 @@ create_vdu(DbName) -> {ok, _} = couch_db:update_docs(Db, [Doc]) end). - update_replicator_doc_with_bad_vdu_test_() -> { setup, fun test_util:start_couch/0, fun test_util:stop_couch/1, { - foreach, fun setup/0, fun teardown/1, + foreach, + fun setup/0, + fun teardown/1, [ fun t_vdu_does_not_crash_on_save/1 ] } }. - t_vdu_does_not_crash_on_save(DbName) -> ?_test(begin Doc = #doc{id = <<"some_id">>, body = {[{<<"foo">>, 42}]}}, ?assertEqual({ok, forbidden}, save_rep_doc(DbName, Doc)) end). - local_replication_endpoint_error_test_() -> - { + { foreach, - fun () -> meck:expect(config, get, - fun(_, _, Default) -> Default end) + fun() -> + meck:expect( + config, + get, + fun(_, _, Default) -> Default end + ) end, - fun (_) -> meck:unload() end, + fun(_) -> meck:unload() end, [ t_error_on_local_endpoint() ] }. - t_error_on_local_endpoint() -> ?_test(begin - RepDoc = {[ - {<<"_id">>, <<"someid">>}, - {<<"source">>, <<"localdb">>}, - {<<"target">>, <<"http://somehost.local/tgt">>} - ]}, + RepDoc = + {[ + {<<"_id">>, <<"someid">>}, + {<<"source">>, <<"localdb">>}, + {<<"target">>, <<"http://somehost.local/tgt">>} + ]}, Expect = local_endpoints_not_supported, ?assertThrow({bad_rep_doc, Expect}, parse_rep_doc_without_id(RepDoc)) end). diff --git a/src/couch_replicator/src/couch_replicator_fabric.erl b/src/couch_replicator/src/couch_replicator_fabric.erl index 1650105b5..6e5ebfc25 100644 --- a/src/couch_replicator/src/couch_replicator_fabric.erl +++ b/src/couch_replicator/src/couch_replicator_fabric.erl @@ -13,7 +13,7 @@ -module(couch_replicator_fabric). -export([ - docs/5 + docs/5 ]). -include_lib("fabric/include/fabric.hrl"). @@ -24,7 +24,8 @@ docs(DbName, Options, QueryArgs, Callback, Acc) -> Shards = mem3:shards(DbName), Workers0 = fabric_util:submit_jobs( - Shards, couch_replicator_fabric_rpc, docs, [Options, QueryArgs]), + Shards, couch_replicator_fabric_rpc, docs, [Options, QueryArgs] + ), RexiMon = fabric_util:create_monitors(Workers0), try case fabric_streams:start(Workers0, #shard.ref) of @@ -50,7 +51,6 @@ docs(DbName, Options, QueryArgs, Callback, Acc) -> rexi_monitor:stop(RexiMon) end. - docs_int(DbName, Workers, QueryArgs, Callback, Acc0) -> #mrargs{limit = Limit, skip = Skip} = QueryArgs, State = #collector{ @@ -63,22 +63,28 @@ docs_int(DbName, Workers, QueryArgs, Callback, Acc0) -> user_acc = Acc0, update_seq = nil }, - case rexi_utils:recv(Workers, #shard.ref, fun handle_message/3, - State, infinity, 5000) of - {ok, NewState} -> - {ok, NewState#collector.user_acc}; - {timeout, NewState} -> - Callback({error, timeout}, NewState#collector.user_acc); - {error, Resp} -> - {ok, Resp} + case + rexi_utils:recv( + Workers, + #shard.ref, + fun handle_message/3, + State, + infinity, + 5000 + ) + of + {ok, NewState} -> + {ok, NewState#collector.user_acc}; + {timeout, NewState} -> + Callback({error, timeout}, NewState#collector.user_acc); + {error, Resp} -> + {ok, Resp} end. handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _, State) -> fabric_view:check_down_shards(State, NodeRef); - handle_message({rexi_EXIT, Reason}, Worker, State) -> fabric_view:handle_worker_exit(State, Worker, Reason); - handle_message({meta, Meta0}, {Worker, From}, State) -> Tot = couch_util:get_value(total, Meta0, 0), Off = couch_util:get_value(offset, Meta0, 0), @@ -97,50 +103,46 @@ handle_message({meta, Meta0}, {Worker, From}, State) -> Total = Total0 + Tot, Offset = Offset0 + Off, case fabric_dict:any(0, Counters1) of - true -> - {ok, State#collector{ - counters = Counters1, - total_rows = Total, - offset = Offset - }}; - false -> - FinalOffset = erlang:min(Total, Offset+State#collector.skip), - Meta = [{total, Total}, {offset, FinalOffset}], - {Go, Acc} = Callback({meta, Meta}, AccIn), - {Go, State#collector{ - counters = fabric_dict:decrement_all(Counters1), - total_rows = Total, - offset = FinalOffset, - user_acc = Acc - }} + true -> + {ok, State#collector{ + counters = Counters1, + total_rows = Total, + offset = Offset + }}; + false -> + FinalOffset = erlang:min(Total, Offset + State#collector.skip), + Meta = [{total, Total}, {offset, FinalOffset}], + {Go, Acc} = Callback({meta, Meta}, AccIn), + {Go, State#collector{ + counters = fabric_dict:decrement_all(Counters1), + total_rows = Total, + offset = FinalOffset, + user_acc = Acc + }} end; - handle_message(#view_row{id = Id, doc = Doc} = Row0, {Worker, From}, State) -> #collector{query_args = Args, counters = Counters0, rows = Rows0} = State, case maybe_fetch_and_filter_doc(Id, Doc, State) of {[_ | _]} = NewDoc -> Row = Row0#view_row{doc = NewDoc}, Dir = Args#mrargs.direction, - Rows = merge_row(Dir, Row#view_row{worker={Worker, From}}, Rows0), + Rows = merge_row(Dir, Row#view_row{worker = {Worker, From}}, Rows0), Counters1 = fabric_dict:update_counter(Worker, 1, Counters0), - State1 = State#collector{rows=Rows, counters=Counters1}, + State1 = State#collector{rows = Rows, counters = Counters1}, fabric_view:maybe_send_row(State1); skip -> rexi:stream_ack(From), {ok, State} end; - handle_message(complete, Worker, State) -> Counters = fabric_dict:update_counter(Worker, 1, State#collector.counters), fabric_view:maybe_send_row(State#collector{counters = Counters}). - merge_row(fwd, Row, Rows) -> lists:keymerge(#view_row.id, [Row], Rows); merge_row(rev, Row, Rows) -> lists:rkeymerge(#view_row.id, [Row], Rows). - maybe_fetch_and_filter_doc(Id, undecided, State) -> #collector{db_name = DbName, query_args = #mrargs{extra = Extra}} = State, FilterStates = proplists:get_value(filter_states, Extra), @@ -149,7 +151,8 @@ maybe_fetch_and_filter_doc(Id, undecided, State) -> DocState = couch_util:get_value(state, Props), couch_replicator_utils:filter_state(DocState, FilterStates, DocInfo); {error, not_found} -> - skip % could have been deleted + % could have been deleted + skip end; maybe_fetch_and_filter_doc(_Id, Doc, _State) -> Doc. diff --git a/src/couch_replicator/src/couch_replicator_fabric_rpc.erl b/src/couch_replicator/src/couch_replicator_fabric_rpc.erl index d67f87548..daeb86e60 100644 --- a/src/couch_replicator/src/couch_replicator_fabric_rpc.erl +++ b/src/couch_replicator/src/couch_replicator_fabric_rpc.erl @@ -13,14 +13,13 @@ -module(couch_replicator_fabric_rpc). -export([ - docs/3 + docs/3 ]). -include_lib("fabric/include/fabric.hrl"). -include_lib("couch/include/couch_db.hrl"). -include_lib("couch_mrview/include/couch_mrview.hrl"). - docs(DbName, Options, Args0) -> set_io_priority(DbName, Options), #mrargs{skip = Skip, limit = Limit, extra = Extra} = Args0, @@ -31,7 +30,6 @@ docs(DbName, Options, Args0) -> Acc = {DbName, FilterStates, HealthThreshold}, couch_mrview:query_all_docs(Db, Args, fun docs_cb/2, Acc). - docs_cb({meta, Meta}, Acc) -> ok = rexi:stream2({meta, Meta}), {ok, Acc}; @@ -54,16 +52,14 @@ docs_cb(complete, Acc) -> ok = rexi:stream_last(complete), {ok, Acc}. - set_io_priority(DbName, Options) -> case lists:keyfind(io_priority, 1, Options) of - {io_priority, Pri} -> - erlang:put(io_priority, Pri); - false -> - erlang:put(io_priority, {interactive, DbName}) + {io_priority, Pri} -> + erlang:put(io_priority, Pri); + false -> + erlang:put(io_priority, {interactive, DbName}) end. - %% Get the state of the replication document. If it is found and has a terminal %% state then it can be filtered and either included in the results or skipped. %% If it is not in a terminal state, look it up in the local doc processor ETS @@ -80,8 +76,13 @@ rep_doc_state(Shard, Id, {[_ | _]} = Doc, States, HealthThreshold) -> null -> % Fetch from local doc processor. If there, filter by state. % If not there, mark as undecided. Let coordinator figure it out. - case couch_replicator_doc_processor:doc_lookup(Shard, Id, - HealthThreshold) of + case + couch_replicator_doc_processor:doc_lookup( + Shard, + Id, + HealthThreshold + ) + of {ok, EtsInfo} -> State = get_doc_state(EtsInfo), couch_replicator_utils:filter_state(State, States, EtsInfo); @@ -92,6 +93,5 @@ rep_doc_state(Shard, Id, {[_ | _]} = Doc, States, HealthThreshold) -> couch_replicator_utils:filter_state(OtherState, States, DocInfo) end. - -get_doc_state({Props})-> +get_doc_state({Props}) -> couch_util:get_value(state, Props). diff --git a/src/couch_replicator/src/couch_replicator_filters.erl b/src/couch_replicator/src/couch_replicator_filters.erl index c8980001a..aab8e80b3 100644 --- a/src/couch_replicator/src/couch_replicator_filters.erl +++ b/src/couch_replicator/src/couch_replicator_filters.erl @@ -21,19 +21,18 @@ -include_lib("couch/include/couch_db.hrl"). - % Parse the filter from replication options proplist. % Return {ok, {FilterType,...}} | {error, ParseError}. % For `user` filter, i.e. filters specified as user code % in source database, this code doesn't fetch the filter % code, but only returns the name of the filter. -spec parse([_]) -> - {ok, nil} | - {ok, {view, binary(), {[_]}}} | - {ok, {user, {binary(), binary()}, {[_]}}} | - {ok, {docids, [_]}} | - {ok, {mango, {[_]}}} | - {error, binary()}. + {ok, nil} + | {ok, {view, binary(), {[_]}}} + | {ok, {user, {binary(), binary()}, {[_]}}} + | {ok, {docids, [_]}} + | {ok, {mango, {[_]}}} + | {error, binary()}. parse(Options) -> Filter = couch_util:get_value(filter, Options), DocIds = couch_util:get_value(doc_ids, Options), @@ -59,7 +58,6 @@ parse(Options) -> {error, list_to_binary(Err)} end. - % Fetches body of filter function from source database. Guaranteed to either % return {ok, Body} or an {error, Reason}. Also assume this function might % block due to network / socket issues for an undeterminted amount of time. @@ -86,18 +84,17 @@ fetch(DDocName, FilterName, Source) -> {error, couch_util:to_binary(Reason)} end. - % Get replication type and view (if any) from replication document props -spec view_type([_], [_]) -> {view, {binary(), binary()}} | {db, nil} | {error, binary()}. view_type(Props, Options) -> case couch_util:get_value(<<"filter">>, Props) of <<"_view">> -> - {QP} = couch_util:get_value(query_params, Options, {[]}), + {QP} = couch_util:get_value(query_params, Options, {[]}), ViewParam = couch_util:get_value(<<"view">>, QP), case re:split(ViewParam, <<"/">>) of [DName, ViewName] -> - {view, {<< "_design/", DName/binary >>, ViewName}}; + {view, {<<"_design/", DName/binary>>, ViewName}}; _ -> {error, <<"Invalid `view` parameter.">>} end; @@ -105,57 +102,70 @@ view_type(Props, Options) -> {db, nil} end. - % Private functions fetch_internal(DDocName, FilterName, Source) -> - Db = case (catch couch_replicator_api_wrap:db_open(Source)) of - {ok, Db0} -> - Db0; - DbError -> - DbErrorMsg = io_lib:format("Could not open source database `~s`: ~s", - [couch_replicator_api_wrap:db_uri(Source), - couch_util:to_binary(DbError)]), - throw({fetch_error, iolist_to_binary(DbErrorMsg)}) - end, - try - Body = case (catch couch_replicator_api_wrap:open_doc( - Db, <<"_design/", DDocName/binary>>, [ejson_body])) of - {ok, #doc{body = Body0}} -> - Body0; - DocError -> - DocErrorMsg = io_lib:format( - "Couldn't open document `_design/~s` from source " - "database `~s`: ~s", [DDocName, - couch_replicator_api_wrap:db_uri(Source), - couch_util:to_binary(DocError)] - ), - throw({fetch_error, iolist_to_binary(DocErrorMsg)}) + Db = + case (catch couch_replicator_api_wrap:db_open(Source)) of + {ok, Db0} -> + Db0; + DbError -> + DbErrorMsg = io_lib:format( + "Could not open source database `~s`: ~s", + [ + couch_replicator_api_wrap:db_uri(Source), + couch_util:to_binary(DbError) + ] + ), + throw({fetch_error, iolist_to_binary(DbErrorMsg)}) end, + try + Body = + case + (catch couch_replicator_api_wrap:open_doc( + Db, <<"_design/", DDocName/binary>>, [ejson_body] + )) + of + {ok, #doc{body = Body0}} -> + Body0; + DocError -> + DocErrorMsg = io_lib:format( + "Couldn't open document `_design/~s` from source " + "database `~s`: ~s", + [ + DDocName, + couch_replicator_api_wrap:db_uri(Source), + couch_util:to_binary(DocError) + ] + ), + throw({fetch_error, iolist_to_binary(DocErrorMsg)}) + end, try Code = couch_util:get_nested_json_value( - Body, [<<"filters">>, FilterName]), + Body, [<<"filters">>, FilterName] + ), re:replace(Code, [$^, "\s*(.*?)\s*", $$], "\\1", [{return, binary}]) - catch - _Tag:CodeError -> - CodeErrorMsg = io_lib:format( - "Couldn't parse filter code from document ~s on `~s` " - " Error: ~s", [DDocName, - couch_replicator_api_wrap:db_uri(Source), - couch_util:to_binary(CodeError)] - ), - throw({fetch_error, CodeErrorMsg}) - end + catch + _Tag:CodeError -> + CodeErrorMsg = io_lib:format( + "Couldn't parse filter code from document ~s on `~s` " + " Error: ~s", + [ + DDocName, + couch_replicator_api_wrap:db_uri(Source), + couch_util:to_binary(CodeError) + ] + ), + throw({fetch_error, CodeErrorMsg}) + end after couch_replicator_api_wrap:db_close(Db) end. - -spec query_params([_]) -> {[_]}. -query_params(Options)-> +query_params(Options) -> couch_util:get_value(query_params, Options, {[]}). - parse_user_filter(Filter) -> case re:run(Filter, "(.*?)/(.*)", [{capture, [1, 2], binary}]) of {match, [DDocName0, FilterName0]} -> @@ -164,31 +174,27 @@ parse_user_filter(Filter) -> {error, <<"Invalid filter. Must match `ddocname/filtername`.">>} end. - % Sort an EJSON object's properties to attempt % to generate a unique representation. This is used % to reduce the chance of getting different % replication checkpoints for the same Mango selector -ejsort({V})-> +ejsort({V}) -> ejsort_props(V, []); ejsort(V) when is_list(V) -> ejsort_array(V, []); ejsort(V) -> V. - -ejsort_props([], Acc)-> +ejsort_props([], Acc) -> {lists:keysort(1, Acc)}; -ejsort_props([{K, V}| R], Acc) -> +ejsort_props([{K, V} | R], Acc) -> ejsort_props(R, [{K, ejsort(V)} | Acc]). - -ejsort_array([], Acc)-> +ejsort_array([], Acc) -> lists:reverse(Acc); ejsort_array([V | R], Acc) -> ejsort_array(R, [ejsort(V) | Acc]). - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -200,14 +206,15 @@ ejsort_basic_values_test() -> ?assertEqual(ejsort([]), []), ?assertEqual(ejsort({[]}), {[]}). - ejsort_compound_values_test() -> ?assertEqual(ejsort([2, 1, 3, <<"a">>]), [2, 1, 3, <<"a">>]), - Ej1 = {[{<<"a">>, 0}, {<<"c">>, 0}, {<<"b">>, 0}]}, - Ej1s = {[{<<"a">>, 0}, {<<"b">>, 0}, {<<"c">>, 0}]}, + Ej1 = {[{<<"a">>, 0}, {<<"c">>, 0}, {<<"b">>, 0}]}, + Ej1s = {[{<<"a">>, 0}, {<<"b">>, 0}, {<<"c">>, 0}]}, ?assertEqual(ejsort(Ej1), Ej1s), Ej2 = {[{<<"x">>, Ej1}, {<<"z">>, Ej1}, {<<"y">>, [Ej1, Ej1]}]}, - ?assertEqual(ejsort(Ej2), - {[{<<"x">>, Ej1s}, {<<"y">>, [Ej1s, Ej1s]}, {<<"z">>, Ej1s}]}). + ?assertEqual( + ejsort(Ej2), + {[{<<"x">>, Ej1s}, {<<"y">>, [Ej1s, Ej1s]}, {<<"z">>, Ej1s}]} + ). -endif. diff --git a/src/couch_replicator/src/couch_replicator_httpc.erl b/src/couch_replicator/src/couch_replicator_httpc.erl index a2af51898..67e3f8474 100644 --- a/src/couch_replicator/src/couch_replicator_httpc.erl +++ b/src/couch_replicator/src/couch_replicator_httpc.erl @@ -39,7 +39,6 @@ % where we may end up processing an unbounded number of messages. -define(MAX_DISCARDED_MESSAGES, 16). - setup(Db) -> #httpdb{ httpc_pool = nil, @@ -47,8 +46,11 @@ setup(Db) -> http_connections = MaxConns, proxy_url = ProxyUrl } = Db, - {ok, Pid} = couch_replicator_httpc_pool:start_link(Url, ProxyUrl, - [{max_connections, MaxConns}]), + {ok, Pid} = couch_replicator_httpc_pool:start_link( + Url, + ProxyUrl, + [{max_connections, MaxConns}] + ), case couch_replicator_auth:initialize(Db#httpdb{httpc_pool = Pid}) of {ok, Db1} -> {ok, Db1}; @@ -59,42 +61,47 @@ setup(Db) -> throw({replication_auth_error, Error}) end. - send_req(HttpDb, Params1, Callback) -> put(?STREAM_STATUS, init), couch_stats:increment_counter([couch_replicator, requests]), - Params2 = ?replace(Params1, qs, - [{K, ?b2l(iolist_to_binary(V))} || {K, V} <- get_value(qs, Params1, [])]), - Params = ?replace(Params2, ibrowse_options, - lists:keysort(1, get_value(ibrowse_options, Params2, []))), + Params2 = ?replace( + Params1, + qs, + [{K, ?b2l(iolist_to_binary(V))} || {K, V} <- get_value(qs, Params1, [])] + ), + Params = ?replace( + Params2, + ibrowse_options, + lists:keysort(1, get_value(ibrowse_options, Params2, [])) + ), {Worker, Response, HttpDb1} = send_ibrowse_req(HttpDb, Params), - Ret = try - process_response(Response, Worker, HttpDb1, Params, Callback) - catch - throw:{retry, NewHttpDb0, NewParams0} -> - {retry, NewHttpDb0, NewParams0} - after - Pool = HttpDb1#httpdb.httpc_pool, - case get(?STOP_HTTP_WORKER) of - stop -> - ok = stop_and_release_worker(Pool, Worker), - erase(?STOP_HTTP_WORKER); - undefined -> - ok = couch_replicator_httpc_pool:release_worker(Pool, Worker) + Ret = + try + process_response(Response, Worker, HttpDb1, Params, Callback) + catch + throw:{retry, NewHttpDb0, NewParams0} -> + {retry, NewHttpDb0, NewParams0} + after + Pool = HttpDb1#httpdb.httpc_pool, + case get(?STOP_HTTP_WORKER) of + stop -> + ok = stop_and_release_worker(Pool, Worker), + erase(?STOP_HTTP_WORKER); + undefined -> + ok = couch_replicator_httpc_pool:release_worker(Pool, Worker) + end, + clean_mailbox(Response) end, - clean_mailbox(Response) - end, % This is necessary to keep this tail-recursive. Calling % send_req in the catch clause would turn it into a body % recursive call accidentally. case Ret of - {retry, #httpdb{}=NewHttpDb, NewParams} -> + {retry, #httpdb{} = NewHttpDb, NewParams} -> send_req(NewHttpDb, NewParams, Callback); _ -> Ret end. - send_ibrowse_req(#httpdb{headers = BaseHeaders} = HttpDb0, Params) -> Method = get_value(method, Params, get), UserHeaders = get_value(headers, Params, []), @@ -103,32 +110,40 @@ send_ibrowse_req(#httpdb{headers = BaseHeaders} = HttpDb0, Params) -> Url = full_url(HttpDb, Params), Body = get_value(body, Params, []), case get_value(path, Params) == "_changes" of - true -> - Timeout = infinity; - false -> - Timeout = case config:get("replicator", "request_timeout", "infinity") of - "infinity" -> infinity; - Milliseconds -> list_to_integer(Milliseconds) - end + true -> + Timeout = infinity; + false -> + Timeout = + case config:get("replicator", "request_timeout", "infinity") of + "infinity" -> infinity; + Milliseconds -> list_to_integer(Milliseconds) + end end, {ok, Worker} = couch_replicator_httpc_pool:get_worker(HttpDb#httpdb.httpc_pool), - BasicAuthOpts = case couch_replicator_utils:get_basic_auth_creds(HttpDb) of - {undefined, undefined} -> - []; - {User, Pass} when is_list(User), is_list(Pass) -> - [{basic_auth, {User, Pass}}] - end, - IbrowseOptions = BasicAuthOpts ++ [ - {response_format, binary}, {inactivity_timeout, HttpDb#httpdb.timeout} | - lists:ukeymerge(1, get_value(ibrowse_options, Params, []), - HttpDb#httpdb.ibrowse_options) - ], + BasicAuthOpts = + case couch_replicator_utils:get_basic_auth_creds(HttpDb) of + {undefined, undefined} -> + []; + {User, Pass} when is_list(User), is_list(Pass) -> + [{basic_auth, {User, Pass}}] + end, + IbrowseOptions = + BasicAuthOpts ++ + [ + {response_format, binary}, + {inactivity_timeout, HttpDb#httpdb.timeout} + | lists:ukeymerge( + 1, + get_value(ibrowse_options, Params, []), + HttpDb#httpdb.ibrowse_options + ) + ], backoff_before_request(Worker, HttpDb, Params), Response = ibrowse:send_req_direct( - Worker, Url, Headers2, Method, Body, IbrowseOptions, Timeout), + Worker, Url, Headers2, Method, Body, IbrowseOptions, Timeout + ), {Worker, Response, HttpDb}. - %% Stop worker, wait for it to die, then release it. Make sure it is dead before %% releasing it to the pool, so there is not race triggered recycling it again. %% The reason is recycling a dying worker, could end up that worker returning @@ -146,8 +161,6 @@ stop_and_release_worker(Pool, Worker) -> process_response({error, sel_conn_closed}, Worker, HttpDb, Params, _Cb) -> put(?STOP_HTTP_WORKER, stop), maybe_retry(sel_conn_closed, Worker, HttpDb, Params); - - %% This clause handles un-expected connection closing during pipelined requests. %% For example, if server responds to a request, sets Connection: close header %% and closes the socket, ibrowse will detect that error when it sends @@ -155,78 +168,85 @@ process_response({error, sel_conn_closed}, Worker, HttpDb, Params, _Cb) -> process_response({error, connection_closing}, Worker, HttpDb, Params, _Cb) -> put(?STOP_HTTP_WORKER, stop), maybe_retry({error, connection_closing}, Worker, HttpDb, Params); - process_response({ibrowse_req_id, ReqId}, Worker, HttpDb, Params, Callback) -> process_stream_response(ReqId, Worker, HttpDb, Params, Callback); - process_response({ok, Code, Headers, Body}, Worker, HttpDb, Params, Callback) -> case list_to_integer(Code) of - R when R =:= 301 ; R =:= 302 ; R =:= 303 -> - backoff_success(HttpDb, Params), - do_redirect(Worker, R, Headers, HttpDb, Params, Callback); - 429 -> - backoff(HttpDb, Params); - Error when Error =:= 408 ; Error >= 500 -> - couch_stats:increment_counter([couch_replicator, responses, failure]), - maybe_retry({code, Error}, Worker, HttpDb, Params); - Ok when Ok >= 200 , Ok < 500 -> - backoff_success(HttpDb, Params), - couch_stats:increment_counter([couch_replicator, responses, success]), - EJson = case Body of - <<>> -> - null; - Json -> - ?JSON_DECODE(Json) - end, - process_auth_response(HttpDb, Ok, Headers, Params), - if Ok =:= 413 -> put(?STOP_HTTP_WORKER, stop); true -> ok end, - Callback(Ok, Headers, EJson) + R when R =:= 301; R =:= 302; R =:= 303 -> + backoff_success(HttpDb, Params), + do_redirect(Worker, R, Headers, HttpDb, Params, Callback); + 429 -> + backoff(HttpDb, Params); + Error when Error =:= 408; Error >= 500 -> + couch_stats:increment_counter([couch_replicator, responses, failure]), + maybe_retry({code, Error}, Worker, HttpDb, Params); + Ok when Ok >= 200, Ok < 500 -> + backoff_success(HttpDb, Params), + couch_stats:increment_counter([couch_replicator, responses, success]), + EJson = + case Body of + <<>> -> + null; + Json -> + ?JSON_DECODE(Json) + end, + process_auth_response(HttpDb, Ok, Headers, Params), + if + Ok =:= 413 -> put(?STOP_HTTP_WORKER, stop); + true -> ok + end, + Callback(Ok, Headers, EJson) end; - process_response(Error, Worker, HttpDb, Params, _Callback) -> maybe_retry(Error, Worker, HttpDb, Params). - process_stream_response(ReqId, Worker, HttpDb, Params, Callback) -> receive - {ibrowse_async_headers, ReqId, Code, Headers} -> - case list_to_integer(Code) of - R when R =:= 301 ; R =:= 302 ; R =:= 303 -> - backoff_success(HttpDb, Params), - do_redirect(Worker, R, Headers, HttpDb, Params, Callback); - 429 -> - Timeout = couch_replicator_rate_limiter:max_interval(), - backoff(HttpDb#httpdb{timeout = Timeout}, Params); - Error when Error =:= 408 ; Error >= 500 -> + {ibrowse_async_headers, ReqId, Code, Headers} -> + case list_to_integer(Code) of + R when R =:= 301; R =:= 302; R =:= 303 -> + backoff_success(HttpDb, Params), + do_redirect(Worker, R, Headers, HttpDb, Params, Callback); + 429 -> + Timeout = couch_replicator_rate_limiter:max_interval(), + backoff(HttpDb#httpdb{timeout = Timeout}, Params); + Error when Error =:= 408; Error >= 500 -> + couch_stats:increment_counter( + [couch_replicator, stream_responses, failure] + ), + report_error(Worker, HttpDb, Params, {code, Error}); + Ok when Ok >= 200, Ok < 500 -> + backoff_success(HttpDb, Params), + HttpDb1 = process_auth_response(HttpDb, Ok, Headers, Params), + StreamDataFun = fun() -> + stream_data_self(HttpDb1, Params, Worker, ReqId, Callback) + end, + put(?STREAM_STATUS, {streaming, Worker}), + if + Ok =:= 413 -> put(?STOP_HTTP_WORKER, stop); + true -> ok + end, + ibrowse:stream_next(ReqId), + try + Ret = Callback(Ok, Headers, StreamDataFun), + Ret + catch + throw:{maybe_retry_req, connection_closed} -> + maybe_retry( + {connection_closed, mid_stream}, + Worker, + HttpDb1, + Params + ); + throw:{maybe_retry_req, Err} -> + maybe_retry(Err, Worker, HttpDb1, Params) + end + end; + {ibrowse_async_response, ReqId, {error, _} = Error} -> couch_stats:increment_counter( [couch_replicator, stream_responses, failure] ), - report_error(Worker, HttpDb, Params, {code, Error}); - Ok when Ok >= 200 , Ok < 500 -> - backoff_success(HttpDb, Params), - HttpDb1 = process_auth_response(HttpDb, Ok, Headers, Params), - StreamDataFun = fun() -> - stream_data_self(HttpDb1, Params, Worker, ReqId, Callback) - end, - put(?STREAM_STATUS, {streaming, Worker}), - if Ok =:= 413 -> put(?STOP_HTTP_WORKER, stop); true -> ok end, - ibrowse:stream_next(ReqId), - try - Ret = Callback(Ok, Headers, StreamDataFun), - Ret - catch - throw:{maybe_retry_req, connection_closed} -> - maybe_retry({connection_closed, mid_stream}, - Worker, HttpDb1, Params); - throw:{maybe_retry_req, Err} -> - maybe_retry(Err, Worker, HttpDb1, Params) - end - end; - {ibrowse_async_response, ReqId, {error, _} = Error} -> - couch_stats:increment_counter( - [couch_replicator, stream_responses, failure] - ), - maybe_retry(Error, Worker, HttpDb, Params) + maybe_retry(Error, Worker, HttpDb, Params) after HttpDb#httpdb.timeout + 500 -> % Note: ibrowse should always reply with timeouts, but this doesn't % seem to be always true when there's a very high rate of requests @@ -234,7 +254,6 @@ process_stream_response(ReqId, Worker, HttpDb, Params, Callback) -> maybe_retry(timeout, Worker, HttpDb, Params) end. - process_auth_response(HttpDb, Code, Headers, Params) -> case couch_replicator_auth:handle_response(HttpDb, Code, Headers) of {continue, HttpDb1} -> @@ -244,7 +263,6 @@ process_auth_response(HttpDb, Code, Headers, Params) -> throw({retry, HttpDb1, Params}) end. - % Only streaming HTTP requests send messages back from % the ibrowse worker process. We can detect that based % on the ibrowse_req_id format. This just drops all @@ -254,7 +272,6 @@ process_auth_response(HttpDb, Code, Headers, Params) -> clean_mailbox(ReqId) -> clean_mailbox(ReqId, ?MAX_DISCARDED_MESSAGES). - clean_mailbox(_ReqId, 0) -> case get(?STREAM_STATUS) of {streaming, Worker} -> @@ -285,14 +302,13 @@ clean_mailbox({ibrowse_req_id, ReqId}, Count) when Count > 0 -> {ibrowse_async_response_end, ReqId} -> put(?STREAM_STATUS, ended), ok - after 0 -> - ok + after 0 -> + ok end end; clean_mailbox(_, Count) when Count > 0 -> ok. - discard_message(ReqId, Worker, Count) -> ibrowse:stream_next(ReqId), receive @@ -306,12 +322,14 @@ discard_message(ReqId, Worker, Count) -> exit({timeout, ibrowse_stream_cleanup}) end. - maybe_retry(Error, Worker, #httpdb{retries = 0} = HttpDb, Params) -> report_error(Worker, HttpDb, Params, {error, Error}); - -maybe_retry(Error, Worker, #httpdb{retries = Retries, wait = Wait} = HttpDb, - Params) -> +maybe_retry( + Error, + Worker, + #httpdb{retries = Retries, wait = Wait} = HttpDb, + Params +) -> case total_error_time_exceeded(HttpDb) of true -> report_error(Worker, HttpDb, Params, {error, Error}); @@ -324,14 +342,12 @@ maybe_retry(Error, Worker, #httpdb{retries = Retries, wait = Wait} = HttpDb, throw({retry, HttpDb2, Params}) end. - % When retrying, check to make total time spent retrying a request is below % the current scheduler health threshold. The goal is to not exceed the % threshold, otherwise the job which keep retrying too long will still be % considered healthy. total_error_time_exceeded(#httpdb{first_error_timestamp = nil}) -> false; - total_error_time_exceeded(#httpdb{first_error_timestamp = ErrorTimestamp}) -> HealthThresholdSec = couch_replicator_scheduler:health_threshold(), % Theshold value is halved because in the calling code the next step @@ -340,23 +356,21 @@ total_error_time_exceeded(#httpdb{first_error_timestamp = ErrorTimestamp}) -> ThresholdUSec = (HealthThresholdSec / 2) * 1000000, timer:now_diff(os:timestamp(), ErrorTimestamp) > ThresholdUSec. - % Remember the first time an error occurs. This value is used later to check % the total time spend retrying a request. Because retrying is cursive, on % successful result #httpdb{} record is reset back to the original value. update_first_error_timestamp(#httpdb{first_error_timestamp = nil} = HttpDb) -> HttpDb#httpdb{first_error_timestamp = os:timestamp()}; - update_first_error_timestamp(HttpDb) -> HttpDb. - log_retry_error(Params, HttpDb, Wait, Error) -> Method = string:to_upper(atom_to_list(get_value(method, Params, get))), Url = couch_util:url_strip_password(full_url(HttpDb, Params)), - couch_log:notice("Retrying ~s request to ~s in ~p seconds due to error ~s", - [Method, Url, Wait / 1000, error_cause(Error)]). - + couch_log:notice( + "Retrying ~s request to ~s in ~p seconds due to error ~s", + [Method, Url, Wait / 1000, error_cause(Error)] + ). report_error(_Worker, HttpDb, Params, Error) -> Method = string:to_upper(atom_to_list(get_value(method, Params, get))), @@ -364,60 +378,60 @@ report_error(_Worker, HttpDb, Params, Error) -> do_report_error(Url, Method, Error), exit({http_request_failed, Method, Url, Error}). - do_report_error(Url, Method, {code, Code}) -> - couch_log:error("Replicator, request ~s to ~p failed. The received " - "HTTP error code is ~p", [Method, Url, Code]); - + couch_log:error( + "Replicator, request ~s to ~p failed. The received " + "HTTP error code is ~p", + [Method, Url, Code] + ); do_report_error(FullUrl, Method, Error) -> - couch_log:error("Replicator, request ~s to ~p failed due to error ~s", - [Method, FullUrl, error_cause(Error)]). - + couch_log:error( + "Replicator, request ~s to ~p failed due to error ~s", + [Method, FullUrl, error_cause(Error)] + ). error_cause({error, Cause}) -> lists:flatten(io_lib:format("~p", [Cause])); error_cause(Cause) -> lists:flatten(io_lib:format("~p", [Cause])). - stream_data_self(#httpdb{timeout = T} = HttpDb, Params, Worker, ReqId, Cb) -> case accumulate_messages(ReqId, [], T + 500) of - {Data, ibrowse_async_response} -> - ibrowse:stream_next(ReqId), - {Data, fun() -> stream_data_self(HttpDb, Params, Worker, ReqId, Cb) end}; - {Data, ibrowse_async_response_end} -> - put(?STREAM_STATUS, ended), - {Data, fun() -> throw({maybe_retry_req, more_data_expected}) end} + {Data, ibrowse_async_response} -> + ibrowse:stream_next(ReqId), + {Data, fun() -> stream_data_self(HttpDb, Params, Worker, ReqId, Cb) end}; + {Data, ibrowse_async_response_end} -> + put(?STREAM_STATUS, ended), + {Data, fun() -> throw({maybe_retry_req, more_data_expected}) end} end. accumulate_messages(ReqId, Acc, Timeout) -> receive - {ibrowse_async_response, ReqId, {error, Error}} -> - throw({maybe_retry_req, Error}); - {ibrowse_async_response, ReqId, <<>>} -> - accumulate_messages(ReqId, Acc, Timeout); - {ibrowse_async_response, ReqId, Data} -> - accumulate_messages(ReqId, [Data | Acc], 0); - {ibrowse_async_response_end, ReqId} -> - {iolist_to_binary(lists:reverse(Acc)), ibrowse_async_response_end} + {ibrowse_async_response, ReqId, {error, Error}} -> + throw({maybe_retry_req, Error}); + {ibrowse_async_response, ReqId, <<>>} -> + accumulate_messages(ReqId, Acc, Timeout); + {ibrowse_async_response, ReqId, Data} -> + accumulate_messages(ReqId, [Data | Acc], 0); + {ibrowse_async_response_end, ReqId} -> + {iolist_to_binary(lists:reverse(Acc)), ibrowse_async_response_end} after Timeout -> % Note: ibrowse should always reply with timeouts, but this doesn't % seem to be always true when there's a very high rate of requests % and many open connections. - if Acc =:= [] -> - throw({maybe_retry_req, timeout}); - true -> - {iolist_to_binary(lists:reverse(Acc)), ibrowse_async_response} + if + Acc =:= [] -> + throw({maybe_retry_req, timeout}); + true -> + {iolist_to_binary(lists:reverse(Acc)), ibrowse_async_response} end end. - full_url(#httpdb{url = BaseUrl}, Params) -> Path = get_value(path, Params, []), QueryArgs = get_value(qs, Params, []), BaseUrl ++ Path ++ query_args_to_string(QueryArgs, []). - query_args_to_string([], []) -> ""; query_args_to_string([], Acc) -> @@ -425,13 +439,11 @@ query_args_to_string([], Acc) -> query_args_to_string([{K, V} | Rest], Acc) -> query_args_to_string(Rest, [K ++ "=" ++ couch_httpd:quote(V) | Acc]). - do_redirect(_Worker, Code, Headers, #httpdb{url = Url} = HttpDb, Params, _Cb) -> RedirectUrl = redirect_url(Headers, Url), {HttpDb2, Params2} = after_redirect(RedirectUrl, Code, HttpDb, Params), throw({retry, HttpDb2, Params2}). - redirect_url(RespHeaders, OrigUrl) -> MochiHeaders = mochiweb_headers:make(RespHeaders), RedUrl = mochiweb_headers:get_value("Location", MochiHeaders), @@ -439,25 +451,28 @@ redirect_url(RespHeaders, OrigUrl) -> host = Host, host_type = HostType, port = Port, - path = Path, % includes query string + % includes query string + path = Path, protocol = Proto } = ibrowse_lib:parse_url(RedUrl), #url{ username = User, password = Passwd } = ibrowse_lib:parse_url(OrigUrl), - Creds = case is_list(User) andalso is_list(Passwd) of - true -> - User ++ ":" ++ Passwd ++ "@"; - false -> - [] - end, - HostPart = case HostType of - ipv6_address -> - "[" ++ Host ++ "]"; - _ -> - Host - end, + Creds = + case is_list(User) andalso is_list(Passwd) of + true -> + User ++ ":" ++ Passwd ++ "@"; + false -> + [] + end, + HostPart = + case HostType of + ipv6_address -> + "[" ++ Host ++ "]"; + _ -> + Host + end, atom_to_list(Proto) ++ "://" ++ Creds ++ HostPart ++ ":" ++ integer_to_list(Port) ++ Path. @@ -470,24 +485,20 @@ after_redirect(RedirectUrl, HttpDb, Params) -> Params2 = lists:keydelete(path, 1, lists:keydelete(qs, 1, Params)), {HttpDb#httpdb{url = RedirectUrl}, Params2}. - backoff_key(HttpDb, Params) -> Method = get_value(method, Params, get), Url = HttpDb#httpdb.url, {Url, Method}. - backoff(HttpDb, Params) -> Key = backoff_key(HttpDb, Params), couch_replicator_rate_limiter:failure(Key), throw({retry, HttpDb, Params}). - backoff_success(HttpDb, Params) -> Key = backoff_key(HttpDb, Params), couch_replicator_rate_limiter:success(Key). - backoff_before_request(Worker, HttpDb, Params) -> Key = backoff_key(HttpDb, Params), Limit = couch_replicator_rate_limiter:max_interval(), @@ -500,25 +511,27 @@ backoff_before_request(Worker, HttpDb, Params) -> ok end. - merge_headers(Headers1, Headers2) when is_list(Headers1), is_list(Headers2) -> Empty = mochiweb_headers:empty(), Merged = mochiweb_headers:enter_from_list(Headers1 ++ Headers2, Empty), mochiweb_headers:to_list(Merged). - -ifdef(TEST). -include_lib("couch/include/couch_eunit.hrl"). - merge_headers_test() -> ?assertEqual([], merge_headers([], [])), ?assertEqual([{"a", "x"}], merge_headers([], [{"a", "x"}])), ?assertEqual([{"a", "x"}], merge_headers([{"a", "x"}], [])), ?assertEqual([{"a", "y"}], merge_headers([{"A", "x"}], [{"a", "y"}])), - ?assertEqual([{"a", "y"}, {"B", "x"}], merge_headers([{"B", "x"}], - [{"a", "y"}])), + ?assertEqual( + [{"a", "y"}, {"B", "x"}], + merge_headers( + [{"B", "x"}], + [{"a", "y"}] + ) + ), ?assertEqual([{"a", "y"}], merge_headers([{"A", "z"}, {"a", "y"}], [])), ?assertEqual([{"a", "y"}], merge_headers([], [{"A", "z"}, {"a", "y"}])). diff --git a/src/couch_replicator/src/couch_replicator_httpc_pool.erl b/src/couch_replicator/src/couch_replicator_httpc_pool.erl index dc9a749ac..ff070f922 100644 --- a/src/couch_replicator/src/couch_replicator_httpc_pool.erl +++ b/src/couch_replicator/src/couch_replicator_httpc_pool.erl @@ -31,13 +31,15 @@ -record(state, { url, proxy_url, - limit, % max # of workers allowed + % max # of workers allowed + limit, workers = [], - waiting = queue:new(), % blocked clients waiting for a worker - callers = [] % clients who've been given a worker + % blocked clients waiting for a worker + waiting = queue:new(), + % clients who've been given a worker + callers = [] }). - start_link(Url, Options) -> start_link(Url, undefined, Options). @@ -47,11 +49,9 @@ start_link(Url, ProxyUrl, Options) -> stop(Pool) -> ok = gen_server:call(Pool, stop, infinity). - get_worker(Pool) -> {ok, _Worker} = gen_server:call(Pool, get_worker, infinity). - release_worker(Pool, Worker) -> ok = gen_server:cast(Pool, {release_worker, Worker}). @@ -67,7 +67,6 @@ init({Url, ProxyUrl, Options}) -> }, {ok, State}. - handle_call(get_worker, From, State) -> #state{ waiting = Waiting, @@ -78,22 +77,20 @@ handle_call(get_worker, From, State) -> workers = Workers } = State, case length(Workers) >= Limit of - true -> - {noreply, State#state{waiting = queue:in(From, Waiting)}}; - false -> - % If the call to acquire fails, the worker pool will crash with a - % badmatch. - {ok, Worker} = couch_replicator_connection:acquire(Url, ProxyUrl), - NewState = State#state{ - workers = [Worker | Workers], - callers = monitor_client(Callers, Worker, From) - }, - {reply, {ok, Worker}, NewState} + true -> + {noreply, State#state{waiting = queue:in(From, Waiting)}}; + false -> + % If the call to acquire fails, the worker pool will crash with a + % badmatch. + {ok, Worker} = couch_replicator_connection:acquire(Url, ProxyUrl), + NewState = State#state{ + workers = [Worker | Workers], + callers = monitor_client(Callers, Worker, From) + }, + {reply, {ok, Worker}, NewState} end; - handle_call(stop, _From, State) -> {stop, normal, ok, State}; - handle_call({release_worker_sync, Worker}, _From, State) -> {reply, ok, release_worker_internal(Worker, State)}. @@ -115,8 +112,10 @@ handle_info({'EXIT', Pid, _Reason}, State) -> Workers2 -> case queue:out(Waiting) of {empty, _} -> - {noreply, State#state{workers = Workers2, - callers = NewCallers0}}; + {noreply, State#state{ + workers = Workers2, + callers = NewCallers0 + }}; {{value, From}, Waiting2} -> {ok, Worker} = couch_replicator_connection:acquire(Url, ProxyUrl), NewCallers1 = monitor_client(NewCallers0, Worker, From), @@ -129,7 +128,6 @@ handle_info({'EXIT', Pid, _Reason}, State) -> {noreply, NewState} end end; - handle_info({'DOWN', Ref, process, _, _}, #state{callers = Callers} = State) -> case lists:keysearch(Ref, 2, Callers) of {value, {Worker, Ref}} -> @@ -138,10 +136,9 @@ handle_info({'DOWN', Ref, process, _, _}, #state{callers = Callers} = State) -> {noreply, State} end. -code_change(_OldVsn, #state{}=State, _Extra) -> +code_change(_OldVsn, #state{} = State, _Extra) -> {ok, State}. - terminate(_Reason, _State) -> ok. @@ -150,10 +147,14 @@ format_status(_Opt, [_PDict, State]) -> url = Url, proxy_url = ProxyUrl } = State, - [{data, [{"State", State#state{ - url = couch_util:url_strip_password(Url), - proxy_url = couch_util:url_strip_password(ProxyUrl) - }}]}]. + [ + {data, [ + {"State", State#state{ + url = couch_util:url_strip_password(Url), + proxy_url = couch_util:url_strip_password(ProxyUrl) + }} + ]} + ]. monitor_client(Callers, Worker, {ClientPid, _}) -> [{Worker, erlang:monitor(process, ClientPid)} | Callers]. @@ -170,29 +171,31 @@ demonitor_client(Callers, Worker) -> release_worker_internal(Worker, State) -> #state{waiting = Waiting, callers = Callers} = State, NewCallers0 = demonitor_client(Callers, Worker), - case is_process_alive(Worker) andalso - lists:member(Worker, State#state.workers) of - true -> - Workers = case queue:out(Waiting) of - {empty, Waiting2} -> - NewCallers1 = NewCallers0, - couch_replicator_connection:release(Worker), - State#state.workers -- [Worker]; - {{value, From}, Waiting2} -> - NewCallers1 = monitor_client(NewCallers0, Worker, From), - gen_server:reply(From, {ok, Worker}), - State#state.workers - end, - NewState = State#state{ - workers = Workers, - waiting = Waiting2, - callers = NewCallers1 - }, - NewState; - false -> - State#state{callers = NewCallers0} - end. - + case + is_process_alive(Worker) andalso + lists:member(Worker, State#state.workers) + of + true -> + Workers = + case queue:out(Waiting) of + {empty, Waiting2} -> + NewCallers1 = NewCallers0, + couch_replicator_connection:release(Worker), + State#state.workers -- [Worker]; + {{value, From}, Waiting2} -> + NewCallers1 = monitor_client(NewCallers0, Worker, From), + gen_server:reply(From, {ok, Worker}), + State#state.workers + end, + NewState = State#state{ + workers = Workers, + waiting = Waiting2, + callers = NewCallers1 + }, + NewState; + false -> + State#state{callers = NewCallers0} + end. -ifdef(TEST). @@ -210,4 +213,4 @@ format_status_test_() -> ok end). --endif.
\ No newline at end of file +-endif. diff --git a/src/couch_replicator/src/couch_replicator_httpd.erl b/src/couch_replicator/src/couch_replicator_httpd.erl index 6efd49335..77c78efe2 100644 --- a/src/couch_replicator/src/couch_replicator_httpd.erl +++ b/src/couch_replicator/src/couch_replicator_httpd.erl @@ -30,50 +30,68 @@ to_binary/1 ]). - -define(DEFAULT_TASK_LIMIT, 100). -define(REPDB, <<"_replicator">>). % This is a macro so it can be used as a guard --define(ISREPDB(X), X =:= ?REPDB orelse binary_part(X, {byte_size(X), -12}) - =:= <<"/_replicator">>). - - -handle_scheduler_req(#httpd{method='GET', path_parts=[_,<<"jobs">>]}=Req) -> - Limit = couch_replicator_httpd_util:parse_int_param(Req, "limit", - ?DEFAULT_TASK_LIMIT, 0, infinity), - Skip = couch_replicator_httpd_util:parse_int_param(Req, "skip", 0, 0, - infinity), +-define(ISREPDB(X), + X =:= ?REPDB orelse + binary_part(X, {byte_size(X), -12}) =:= + <<"/_replicator">> +). + +handle_scheduler_req(#httpd{method = 'GET', path_parts = [_, <<"jobs">>]} = Req) -> + Limit = couch_replicator_httpd_util:parse_int_param( + Req, + "limit", + ?DEFAULT_TASK_LIMIT, + 0, + infinity + ), + Skip = couch_replicator_httpd_util:parse_int_param( + Req, + "skip", + 0, + 0, + infinity + ), {Replies, _BadNodes} = rpc:multicall(couch_replicator_scheduler, jobs, []), Flatlist = lists:concat(Replies), % couch_replicator_scheduler:job_ejson/1 guarantees {id, Id} to be the % the first item in the list - Sorted = lists:sort(fun({[{id,A}|_]},{[{id,B}|_]}) -> A =< B end, Flatlist), + Sorted = lists:sort(fun({[{id, A} | _]}, {[{id, B} | _]}) -> A =< B end, Flatlist), Total = length(Sorted), Offset = min(Skip, Total), - Sublist = lists:sublist(Sorted, Offset+1, Limit), - Sublist1 = [couch_replicator_httpd_util:update_db_name(Task) - || Task <- Sublist], + Sublist = lists:sublist(Sorted, Offset + 1, Limit), + Sublist1 = [ + couch_replicator_httpd_util:update_db_name(Task) + || Task <- Sublist + ], send_json(Req, {[{total_rows, Total}, {offset, Offset}, {jobs, Sublist1}]}); -handle_scheduler_req(#httpd{method='GET', path_parts=[_,<<"jobs">>,JobId]}=Req) -> +handle_scheduler_req(#httpd{method = 'GET', path_parts = [_, <<"jobs">>, JobId]} = Req) -> case couch_replicator:job(JobId) of {ok, JobInfo} -> send_json(Req, couch_replicator_httpd_util:update_db_name(JobInfo)); {error, not_found} -> throw(not_found) end; -handle_scheduler_req(#httpd{method='GET', path_parts=[_,<<"docs">>]}=Req) -> +handle_scheduler_req(#httpd{method = 'GET', path_parts = [_, <<"docs">>]} = Req) -> handle_scheduler_docs(?REPDB, Req); -handle_scheduler_req(#httpd{method='GET', path_parts=[_,<<"docs">>,Db]}=Req) - when ?ISREPDB(Db) -> +handle_scheduler_req(#httpd{method = 'GET', path_parts = [_, <<"docs">>, Db]} = Req) when + ?ISREPDB(Db) +-> handle_scheduler_docs(Db, Req); -handle_scheduler_req(#httpd{method='GET', path_parts=[_,<<"docs">>,Db,DocId]} - = Req) when ?ISREPDB(Db) -> +handle_scheduler_req( + #httpd{method = 'GET', path_parts = [_, <<"docs">>, Db, DocId]} = + Req +) when ?ISREPDB(Db) -> handle_scheduler_doc(Db, DocId, Req); % Allow users to pass in unencoded _replicator database names (/ are not % escaped). This is possible here because _replicator is not a valid document % ID so can disambiguate between an element of a db path and the document ID. -handle_scheduler_req(#httpd{method='GET', path_parts=[_,<<"docs">>|Unquoted]} - = Req) -> +handle_scheduler_req( + #httpd{method = 'GET', path_parts = [_, <<"docs">> | Unquoted]} = + Req +) -> case parse_unquoted_docs_path(Unquoted) of {db_only, Db} -> handle_scheduler_docs(Db, Req); @@ -82,38 +100,37 @@ handle_scheduler_req(#httpd{method='GET', path_parts=[_,<<"docs">>|Unquoted]} {error, invalid} -> throw(bad_request) end; -handle_scheduler_req(#httpd{method='GET'} = Req) -> +handle_scheduler_req(#httpd{method = 'GET'} = Req) -> send_json(Req, 404, {[{error, <<"not found">>}]}); handle_scheduler_req(Req) -> send_method_not_allowed(Req, "GET,HEAD"). - handle_req(#httpd{method = 'POST', user_ctx = UserCtx} = Req) -> couch_httpd:validate_ctype(Req, "application/json"), RepDoc = {Props} = couch_httpd:json_body_obj(Req), couch_replicator_httpd_util:validate_rep_props(Props), case couch_replicator:replicate(RepDoc, UserCtx) of - {error, {Error, Reason}} -> - send_json( - Req, 500, - {[{error, to_binary(Error)}, {reason, to_binary(Reason)}]}); - {error, not_found} -> - % Tried to cancel a replication that didn't exist. - send_json(Req, 404, {[{error, <<"not found">>}]}); - {error, Reason} -> - send_json(Req, 500, {[{error, to_binary(Reason)}]}); - {ok, {cancelled, RepId}} -> - send_json(Req, 200, {[{ok, true}, {<<"_local_id">>, RepId}]}); - {ok, {continuous, RepId}} -> - send_json(Req, 202, {[{ok, true}, {<<"_local_id">>, RepId}]}); - {ok, {HistoryResults}} -> - send_json(Req, {[{ok, true} | HistoryResults]}) + {error, {Error, Reason}} -> + send_json( + Req, + 500, + {[{error, to_binary(Error)}, {reason, to_binary(Reason)}]} + ); + {error, not_found} -> + % Tried to cancel a replication that didn't exist. + send_json(Req, 404, {[{error, <<"not found">>}]}); + {error, Reason} -> + send_json(Req, 500, {[{error, to_binary(Reason)}]}); + {ok, {cancelled, RepId}} -> + send_json(Req, 200, {[{ok, true}, {<<"_local_id">>, RepId}]}); + {ok, {continuous, RepId}} -> + send_json(Req, 202, {[{ok, true}, {<<"_local_id">>, RepId}]}); + {ok, {HistoryResults}} -> + send_json(Req, {[{ok, true} | HistoryResults]}) end; - handle_req(Req) -> send_method_not_allowed(Req, "POST"). - handle_scheduler_docs(Db, Req) when is_binary(Db) -> VArgs0 = couch_mrview_http:parse_params(Req, undefined), StatesQs = chttpd:qs_value(Req, "states"), @@ -130,8 +147,7 @@ handle_scheduler_docs(Db, Req) when is_binary(Db) -> Acc = couch_replicator_httpd_util:docs_acc_new(Req, Db, Max), Cb = fun couch_replicator_httpd_util:docs_cb/2, {ok, RAcc} = couch_replicator_fabric:docs(Db, Opts, VArgs2, Cb, Acc), - {ok, couch_replicator_httpd_util:docs_acc_response(RAcc)}. - + {ok, couch_replicator_httpd_util:docs_acc_response(RAcc)}. handle_scheduler_doc(Db, DocId, Req) when is_binary(Db), is_binary(DocId) -> UserCtx = Req#httpd.user_ctx, @@ -142,7 +158,6 @@ handle_scheduler_doc(Db, DocId, Req) when is_binary(Db), is_binary(DocId) -> throw(not_found) end. - parse_unquoted_docs_path([_, _ | _] = Unquoted) -> DbAndAfter = lists:dropwhile(fun(E) -> E =/= ?REPDB end, Unquoted), BeforeRDb = lists:takewhile(fun(E) -> E =/= ?REPDB end, Unquoted), @@ -155,19 +170,21 @@ parse_unquoted_docs_path([_, _ | _] = Unquoted) -> {db_and_doc, filename:join(BeforeRDb ++ [?REPDB]), DocId} end. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). unquoted_scheduler_docs_path_test_() -> - [?_assertEqual(Res, parse_unquoted_docs_path(Path)) || {Res, Path} <- [ - {{error, invalid}, [<<"a">>,<< "b">>]}, - {{db_only, <<"a/_replicator">>}, [<<"a">>, ?REPDB]}, - {{db_only, <<"a/b/_replicator">>}, [<<"a">>, <<"b">>, ?REPDB]}, - {{db_and_doc, <<"_replicator">>, <<"x">>}, [?REPDB, <<"x">>]}, - {{db_and_doc, <<"a/_replicator">>, <<"x">>}, [<<"a">>, ?REPDB, <<"x">>]}, - {{error, invalid}, [<<"a/_replicator">>,<<"x">>]} - ]]. + [ + ?_assertEqual(Res, parse_unquoted_docs_path(Path)) + || {Res, Path} <- [ + {{error, invalid}, [<<"a">>, <<"b">>]}, + {{db_only, <<"a/_replicator">>}, [<<"a">>, ?REPDB]}, + {{db_only, <<"a/b/_replicator">>}, [<<"a">>, <<"b">>, ?REPDB]}, + {{db_and_doc, <<"_replicator">>, <<"x">>}, [?REPDB, <<"x">>]}, + {{db_and_doc, <<"a/_replicator">>, <<"x">>}, [<<"a">>, ?REPDB, <<"x">>]}, + {{error, invalid}, [<<"a/_replicator">>, <<"x">>]} + ] + ]. -endif. diff --git a/src/couch_replicator/src/couch_replicator_httpd_util.erl b/src/couch_replicator/src/couch_replicator_httpd_util.erl index 624eddd2f..ddcc179d4 100644 --- a/src/couch_replicator/src/couch_replicator_httpd_util.erl +++ b/src/couch_replicator/src/couch_replicator_httpd_util.erl @@ -35,61 +35,64 @@ to_binary/1 ]). - parse_replication_state_filter(undefined) -> - []; % This is the default (wildcard) filter + % This is the default (wildcard) filter + []; parse_replication_state_filter(States) when is_list(States) -> AllStates = couch_replicator:replication_states(), StrStates = [string:to_lower(S) || S <- string:tokens(States, ",")], - AtomStates = try - [list_to_existing_atom(S) || S <- StrStates] - catch error:badarg -> - Msg1 = io_lib:format("States must be one or more of ~w", [AllStates]), - throw({query_parse_error, ?l2b(Msg1)}) - end, + AtomStates = + try + [list_to_existing_atom(S) || S <- StrStates] + catch + error:badarg -> + Msg1 = io_lib:format("States must be one or more of ~w", [AllStates]), + throw({query_parse_error, ?l2b(Msg1)}) + end, AllSet = sets:from_list(AllStates), StatesSet = sets:from_list(AtomStates), Diff = sets:to_list(sets:subtract(StatesSet, AllSet)), case Diff of - [] -> - AtomStates; - _ -> - Args = [Diff, AllStates], - Msg2 = io_lib:format("Unknown states ~w. Choose from: ~w", Args), - throw({query_parse_error, ?l2b(Msg2)}) + [] -> + AtomStates; + _ -> + Args = [Diff, AllStates], + Msg2 = io_lib:format("Unknown states ~w. Choose from: ~w", Args), + throw({query_parse_error, ?l2b(Msg2)}) end. - parse_int_param(Req, Param, Default, Min, Max) -> - IntVal = try - list_to_integer(chttpd:qs_value(Req, Param, integer_to_list(Default))) - catch error:badarg -> - Msg1 = io_lib:format("~s must be an integer", [Param]), - throw({query_parse_error, ?l2b(Msg1)}) - end, + IntVal = + try + list_to_integer(chttpd:qs_value(Req, Param, integer_to_list(Default))) + catch + error:badarg -> + Msg1 = io_lib:format("~s must be an integer", [Param]), + throw({query_parse_error, ?l2b(Msg1)}) + end, case IntVal >= Min andalso IntVal =< Max of - true -> - IntVal; - false -> - Msg2 = io_lib:format("~s not in range of [~w,~w]", [Param, Min, Max]), - throw({query_parse_error, ?l2b(Msg2)}) + true -> + IntVal; + false -> + Msg2 = io_lib:format("~s not in range of [~w,~w]", [Param, Min, Max]), + throw({query_parse_error, ?l2b(Msg2)}) end. - validate_rep_props([]) -> ok; -validate_rep_props([{<<"query_params">>, {Params}}|Rest]) -> - lists:foreach(fun - ({_,V}) when is_binary(V) -> ok; - ({K,_}) -> throw({bad_request, - <<K/binary," value must be a string.">>}) - end, Params), +validate_rep_props([{<<"query_params">>, {Params}} | Rest]) -> + lists:foreach( + fun + ({_, V}) when is_binary(V) -> ok; + ({K, _}) -> throw({bad_request, <<K/binary, " value must be a string.">>}) + end, + Params + ), validate_rep_props(Rest); -validate_rep_props([_|Rest]) -> +validate_rep_props([_ | Rest]) -> validate_rep_props(Rest). - -prepend_val(#vacc{prepend=Prepend}) -> +prepend_val(#vacc{prepend = Prepend}) -> case Prepend of undefined -> ""; @@ -97,9 +100,9 @@ prepend_val(#vacc{prepend=Prepend}) -> Prepend end. - -maybe_flush_response(#vacc{bufsize=Size, threshold=Max} = Acc, Data, Len) - when Size > 0 andalso (Size + Len) > Max -> +maybe_flush_response(#vacc{bufsize = Size, threshold = Max} = Acc, Data, Len) when + Size > 0 andalso (Size + Len) > Max +-> #vacc{buffer = Buffer, resp = Resp} = Acc, {ok, R1} = chttpd:send_delayed_chunk(Resp, Buffer), {ok, Acc#vacc{prepend = ",\r\n", buffer = Data, bufsize = Len, resp = R1}}; @@ -113,71 +116,69 @@ maybe_flush_response(Acc0, Data, Len) -> {ok, Acc}. docs_acc_new(Req, Db, Threshold) -> - #vacc{db=Db, req=Req, threshold=Threshold}. + #vacc{db = Db, req = Req, threshold = Threshold}. docs_acc_response(#vacc{resp = Resp}) -> Resp. -docs_cb({error, Reason}, #vacc{resp=undefined}=Acc) -> +docs_cb({error, Reason}, #vacc{resp = undefined} = Acc) -> {ok, Resp} = chttpd:send_error(Acc#vacc.req, Reason), - {ok, Acc#vacc{resp=Resp}}; - -docs_cb(complete, #vacc{resp=undefined}=Acc) -> + {ok, Acc#vacc{resp = Resp}}; +docs_cb(complete, #vacc{resp = undefined} = Acc) -> % Nothing in view {ok, Resp} = chttpd:send_json(Acc#vacc.req, 200, {[{rows, []}]}), - {ok, Acc#vacc{resp=Resp}}; - -docs_cb(Msg, #vacc{resp=undefined}=Acc) -> + {ok, Acc#vacc{resp = Resp}}; +docs_cb(Msg, #vacc{resp = undefined} = Acc) -> %% Start response Headers = [], {ok, Resp} = chttpd:start_delayed_json_response(Acc#vacc.req, 200, Headers), - docs_cb(Msg, Acc#vacc{resp=Resp, should_close=true}); - -docs_cb({error, Reason}, #vacc{resp=Resp}=Acc) -> + docs_cb(Msg, Acc#vacc{resp = Resp, should_close = true}); +docs_cb({error, Reason}, #vacc{resp = Resp} = Acc) -> {ok, Resp1} = chttpd:send_delayed_error(Resp, Reason), - {ok, Acc#vacc{resp=Resp1}}; - -docs_cb(complete, #vacc{resp=Resp, buffer=Buf, threshold=Max}=Acc) -> + {ok, Acc#vacc{resp = Resp1}}; +docs_cb(complete, #vacc{resp = Resp, buffer = Buf, threshold = Max} = Acc) -> % Finish view output and possibly end the response {ok, Resp1} = chttpd:close_delayed_json_object(Resp, Buf, "\r\n]}", Max), case Acc#vacc.should_close of true -> {ok, Resp2} = chttpd:end_delayed_json_response(Resp1), - {ok, Acc#vacc{resp=Resp2}}; + {ok, Acc#vacc{resp = Resp2}}; _ -> - {ok, Acc#vacc{resp=Resp1, meta_sent=false, row_sent=false, - prepend=",\r\n", buffer=[], bufsize=0}} + {ok, Acc#vacc{ + resp = Resp1, + meta_sent = false, + row_sent = false, + prepend = ",\r\n", + buffer = [], + bufsize = 0 + }} end; - -docs_cb({meta, Meta}, #vacc{meta_sent=false, row_sent=false}=Acc) -> +docs_cb({meta, Meta}, #vacc{meta_sent = false, row_sent = false} = Acc) -> % Sending metadata as we've not sent it or any row yet - Parts = case couch_util:get_value(total, Meta) of - undefined -> []; - Total -> [io_lib:format("\"total_rows\":~p", [adjust_total(Total)])] - end ++ case couch_util:get_value(offset, Meta) of - undefined -> []; - Offset -> [io_lib:format("\"offset\":~p", [Offset])] - end ++ ["\"docs\":["], + Parts = + case couch_util:get_value(total, Meta) of + undefined -> []; + Total -> [io_lib:format("\"total_rows\":~p", [adjust_total(Total)])] + end ++ + case couch_util:get_value(offset, Meta) of + undefined -> []; + Offset -> [io_lib:format("\"offset\":~p", [Offset])] + end ++ ["\"docs\":["], Chunk = [prepend_val(Acc), "{", string:join(Parts, ","), "\r\n"], {ok, AccOut} = maybe_flush_response(Acc, Chunk, iolist_size(Chunk)), - {ok, AccOut#vacc{prepend="", meta_sent=true}}; - - -docs_cb({meta, _Meta}, #vacc{}=Acc) -> + {ok, AccOut#vacc{prepend = "", meta_sent = true}}; +docs_cb({meta, _Meta}, #vacc{} = Acc) -> %% ignore metadata {ok, Acc}; - -docs_cb({row, Row}, #vacc{meta_sent=false}=Acc) -> +docs_cb({row, Row}, #vacc{meta_sent = false} = Acc) -> %% sorted=false and row arrived before meta % Adding another row Chunk = [prepend_val(Acc), "{\"docs\":[\r\n", row_to_json(Row)], - maybe_flush_response(Acc#vacc{meta_sent=true, row_sent=true}, Chunk, iolist_size(Chunk)); - -docs_cb({row, Row}, #vacc{meta_sent=true}=Acc) -> + maybe_flush_response(Acc#vacc{meta_sent = true, row_sent = true}, Chunk, iolist_size(Chunk)); +docs_cb({row, Row}, #vacc{meta_sent = true} = Acc) -> % Adding another row Chunk = [prepend_val(Acc), row_to_json(Row)], - maybe_flush_response(Acc#vacc{row_sent=true}, Chunk, iolist_size(Chunk)). - + maybe_flush_response(Acc#vacc{row_sent = true}, Chunk, iolist_size(Chunk)). update_db_name({Props}) -> {value, {database, DbName}, Props1} = lists:keytake(database, 1, Props), @@ -193,7 +194,6 @@ row_to_json(Row) -> Doc1 = update_db_name(Doc0), ?JSON_ENCODE(Doc1). - %% Adjust Total as there is an automatically created validation design doc adjust_total(Total) when is_integer(Total), Total > 0 -> Total - 1; diff --git a/src/couch_replicator/src/couch_replicator_ids.erl b/src/couch_replicator/src/couch_replicator_ids.erl index 316e6a28a..6543cf069 100644 --- a/src/couch_replicator/src/couch_replicator_ids.erl +++ b/src/couch_replicator/src/couch_replicator_ids.erl @@ -34,7 +34,6 @@ replication_id(#rep{options = Options} = Rep) -> BaseId = replication_id(Rep, ?REP_ID_VERSION), {BaseId, maybe_append_options([continuous, create_target], Options)}. - % Versioned clauses for generating replication IDs. % If a change is made to how replications are identified, % please add a new clause and increase ?REP_ID_VERSION. @@ -44,37 +43,34 @@ replication_id(#rep{} = Rep, 4) -> SrcInfo = get_v4_endpoint(Rep#rep.source), TgtInfo = get_v4_endpoint(Rep#rep.target), maybe_append_filters([UUID, SrcInfo, TgtInfo], Rep); - replication_id(#rep{} = Rep, 3) -> UUID = couch_server:get_uuid(), Src = get_rep_endpoint(Rep#rep.source), Tgt = get_rep_endpoint(Rep#rep.target), maybe_append_filters([UUID, Src, Tgt], Rep); - replication_id(#rep{} = Rep, 2) -> {ok, HostName} = inet:gethostname(), - Port = case (catch mochiweb_socket_server:get(couch_httpd, port)) of - P when is_number(P) -> - P; - _ -> - % On restart we might be called before the couch_httpd process is - % started. - % TODO: we might be under an SSL socket server only, or both under - % SSL and a non-SSL socket. - % ... mochiweb_socket_server:get(https, port) - config:get_integer("httpd", "port", 5984) - end, + Port = + case (catch mochiweb_socket_server:get(couch_httpd, port)) of + P when is_number(P) -> + P; + _ -> + % On restart we might be called before the couch_httpd process is + % started. + % TODO: we might be under an SSL socket server only, or both under + % SSL and a non-SSL socket. + % ... mochiweb_socket_server:get(https, port) + config:get_integer("httpd", "port", 5984) + end, Src = get_rep_endpoint(Rep#rep.source), Tgt = get_rep_endpoint(Rep#rep.target), maybe_append_filters([HostName, Port, Src, Tgt], Rep); - replication_id(#rep{} = Rep, 1) -> {ok, HostName} = inet:gethostname(), Src = get_rep_endpoint(Rep#rep.source), Tgt = get_rep_endpoint(Rep#rep.target), maybe_append_filters([HostName, Src, Tgt], Rep). - -spec convert([_] | binary() | {string(), string()}) -> {string(), string()}. convert(Id) when is_list(Id) -> convert(?l2b(Id)); @@ -87,59 +83,62 @@ convert(Id0) when is_binary(Id0) -> convert({BaseId, Ext} = Id) when is_list(BaseId), is_list(Ext) -> Id. - % Private functions -maybe_append_filters(Base, - #rep{source = Source, options = Options}) -> - Base2 = Base ++ - case couch_replicator_filters:parse(Options) of - {ok, nil} -> - []; - {ok, {view, Filter, QueryParams}} -> - [Filter, QueryParams]; - {ok, {user, {Doc, Filter}, QueryParams}} -> - case couch_replicator_filters:fetch(Doc, Filter, Source) of - {ok, Code} -> - [Code, QueryParams]; - {error, Error} -> - throw({filter_fetch_error, Error}) - end; - {ok, {docids, DocIds}} -> - [DocIds]; - {ok, {mango, Selector}} -> - [Selector]; - {error, FilterParseError} -> - throw({error, FilterParseError}) - end, +maybe_append_filters( + Base, + #rep{source = Source, options = Options} +) -> + Base2 = + Base ++ + case couch_replicator_filters:parse(Options) of + {ok, nil} -> + []; + {ok, {view, Filter, QueryParams}} -> + [Filter, QueryParams]; + {ok, {user, {Doc, Filter}, QueryParams}} -> + case couch_replicator_filters:fetch(Doc, Filter, Source) of + {ok, Code} -> + [Code, QueryParams]; + {error, Error} -> + throw({filter_fetch_error, Error}) + end; + {ok, {docids, DocIds}} -> + [DocIds]; + {ok, {mango, Selector}} -> + [Selector]; + {error, FilterParseError} -> + throw({error, FilterParseError}) + end, couch_util:to_hex(couch_hash:md5_hash(term_to_binary(Base2))). - maybe_append_options(Options, RepOptions) -> - lists:foldl(fun(Option, Acc) -> - Acc ++ - case couch_util:get_value(Option, RepOptions, false) of - true -> - "+" ++ atom_to_list(Option); - false -> - "" - end - end, [], Options). - + lists:foldl( + fun(Option, Acc) -> + Acc ++ + case couch_util:get_value(Option, RepOptions, false) of + true -> + "+" ++ atom_to_list(Option); + false -> + "" + end + end, + [], + Options + ). -get_rep_endpoint(#httpdb{url=Url, headers=Headers}) -> +get_rep_endpoint(#httpdb{url = Url, headers = Headers}) -> DefaultHeaders = (#httpdb{})#httpdb.headers, {remote, Url, Headers -- DefaultHeaders}. - get_v4_endpoint(#httpdb{} = HttpDb) -> {remote, Url, Headers} = get_rep_endpoint(HttpDb), {User, _} = couch_replicator_utils:get_basic_auth_creds(HttpDb), {Host, NonDefaultPort, Path} = get_v4_url_info(Url), - OAuth = undefined, % Keep this to ensure checkpoints don't change + % Keep this to ensure checkpoints don't change + OAuth = undefined, {remote, User, Host, NonDefaultPort, Path, Headers, OAuth}. - get_v4_url_info(Url) when is_binary(Url) -> get_v4_url_info(binary_to_list(Url)); get_v4_url_info(Url) -> @@ -158,7 +157,6 @@ get_v4_url_info(Url) -> {Host, NonDefaultPort, Path} end. - get_non_default_port(https, 443) -> default; get_non_default_port(http, 80) -> @@ -168,112 +166,117 @@ get_non_default_port(http, 5984) -> get_non_default_port(_Schema, Port) -> Port. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). - replication_id_convert_test_() -> - [?_assertEqual(Expected, convert(Id)) || {Expected, Id} <- [ - {{"abc", ""}, "abc"}, - {{"abc", ""}, <<"abc">>}, - {{"abc", "+x+y"}, <<"abc+x+y">>}, - {{"abc", "+x+y"}, {"abc", "+x+y"}}, - {{"abc", "+x+y"}, <<"abc x y">>} - ]]. + [ + ?_assertEqual(Expected, convert(Id)) + || {Expected, Id} <- [ + {{"abc", ""}, "abc"}, + {{"abc", ""}, <<"abc">>}, + {{"abc", "+x+y"}, <<"abc+x+y">>}, + {{"abc", "+x+y"}, {"abc", "+x+y"}}, + {{"abc", "+x+y"}, <<"abc x y">>} + ] + ]. http_v4_endpoint_test_() -> - [?_assertMatch({remote, User, Host, Port, Path, HeadersNoAuth, undefined}, - begin - HttpDb = #httpdb{url = Url, headers = Headers, auth_props = Auth}, - HttpDb1 = couch_replicator_utils:normalize_basic_auth(HttpDb), - get_v4_endpoint(HttpDb1) - end) || - {{User, Host, Port, Path, HeadersNoAuth}, {Url, Headers, Auth}} <- [ - { - {undefined, "host", default, "/", []}, - {"http://host", [], []} - }, - { - {undefined, "host", default, "/", []}, - {"https://host", [], []} - }, - { - {undefined, "host", default, "/", []}, - {"http://host:5984", [], []} - }, - { - {undefined, "host", 1, "/", []}, - {"http://host:1", [], []} - }, - { - {undefined, "host", 2, "/", []}, - {"https://host:2", [], []} - }, - { - {undefined, "host", default, "/", [{"h","v"}]}, - {"http://host", [{"h","v"}], []} - }, - { - {undefined, "host", default, "/a/b", []}, - {"http://host/a/b", [], []} - }, - { - {"user", "host", default, "/", []}, - {"http://user:pass@host", [], []} - }, - { - {"user", "host", 3, "/", []}, - {"http://user:pass@host:3", [], []} - }, - { - {"user", "host", default, "/", []}, - {"http://user:newpass@host", [], []} - }, - { - {"user", "host", default, "/", []}, - {"http://host", [basic_auth("user","pass")], []} - }, - { - {"user", "host", default, "/", []}, - {"http://host", [basic_auth("user","newpass")], []} - }, - { - {"user3", "host", default, "/", []}, - {"http://user1:pass1@host", [basic_auth("user2","pass2")], - auth_props("user3", "pass3")} - }, - { - {"user2", "host", default, "/", [{"h", "v"}]}, - {"http://host", [{"h", "v"}, basic_auth("user","pass")], - auth_props("user2", "pass2")} - }, - { - {"user", "host", default, "/", [{"h", "v"}]}, - {"http://host", [{"h", "v"}], auth_props("user", "pass")} - }, - { - {undefined, "random_junk", undefined, undefined}, - {"random_junk", [], []} - }, - { - {undefined, "host", default, "/", []}, - {"http://host", [{"Authorization", "Basic bad"}], []} - } + [ + ?_assertMatch( + {remote, User, Host, Port, Path, HeadersNoAuth, undefined}, + begin + HttpDb = #httpdb{url = Url, headers = Headers, auth_props = Auth}, + HttpDb1 = couch_replicator_utils:normalize_basic_auth(HttpDb), + get_v4_endpoint(HttpDb1) + end + ) + || {{User, Host, Port, Path, HeadersNoAuth}, {Url, Headers, Auth}} <- [ + { + {undefined, "host", default, "/", []}, + {"http://host", [], []} + }, + { + {undefined, "host", default, "/", []}, + {"https://host", [], []} + }, + { + {undefined, "host", default, "/", []}, + {"http://host:5984", [], []} + }, + { + {undefined, "host", 1, "/", []}, + {"http://host:1", [], []} + }, + { + {undefined, "host", 2, "/", []}, + {"https://host:2", [], []} + }, + { + {undefined, "host", default, "/", [{"h", "v"}]}, + {"http://host", [{"h", "v"}], []} + }, + { + {undefined, "host", default, "/a/b", []}, + {"http://host/a/b", [], []} + }, + { + {"user", "host", default, "/", []}, + {"http://user:pass@host", [], []} + }, + { + {"user", "host", 3, "/", []}, + {"http://user:pass@host:3", [], []} + }, + { + {"user", "host", default, "/", []}, + {"http://user:newpass@host", [], []} + }, + { + {"user", "host", default, "/", []}, + {"http://host", [basic_auth("user", "pass")], []} + }, + { + {"user", "host", default, "/", []}, + {"http://host", [basic_auth("user", "newpass")], []} + }, + { + {"user3", "host", default, "/", []}, + {"http://user1:pass1@host", [basic_auth("user2", "pass2")], + auth_props("user3", "pass3")} + }, + { + {"user2", "host", default, "/", [{"h", "v"}]}, + {"http://host", [{"h", "v"}, basic_auth("user", "pass")], + auth_props("user2", "pass2")} + }, + { + {"user", "host", default, "/", [{"h", "v"}]}, + {"http://host", [{"h", "v"}], auth_props("user", "pass")} + }, + { + {undefined, "random_junk", undefined, undefined}, + {"random_junk", [], []} + }, + { + {undefined, "host", default, "/", []}, + {"http://host", [{"Authorization", "Basic bad"}], []} + } ] ]. - basic_auth(User, Pass) -> B64Auth = base64:encode_to_string(User ++ ":" ++ Pass), {"Authorization", "Basic " ++ B64Auth}. - auth_props(User, Pass) when is_list(User), is_list(Pass) -> - [{<<"basic">>, {[ - {<<"username">>, list_to_binary(User)}, - {<<"password">>, list_to_binary(Pass)} - ]}}]. + [ + {<<"basic">>, + {[ + {<<"username">>, list_to_binary(User)}, + {<<"password">>, list_to_binary(Pass)} + ]}} + ]. -endif. diff --git a/src/couch_replicator/src/couch_replicator_job_sup.erl b/src/couch_replicator/src/couch_replicator_job_sup.erl index 9ea65e85f..e3d15c041 100644 --- a/src/couch_replicator/src/couch_replicator_job_sup.erl +++ b/src/couch_replicator/src/couch_replicator_job_sup.erl @@ -20,7 +20,7 @@ ]). start_link() -> - supervisor:start_link({local,?MODULE}, ?MODULE, []). + supervisor:start_link({local, ?MODULE}, ?MODULE, []). %%============================================================================= %% supervisor callbacks diff --git a/src/couch_replicator/src/couch_replicator_notifier.erl b/src/couch_replicator/src/couch_replicator_notifier.erl index f7640a349..451ec5de7 100644 --- a/src/couch_replicator/src/couch_replicator_notifier.erl +++ b/src/couch_replicator/src/couch_replicator_notifier.erl @@ -25,8 +25,11 @@ -include_lib("couch/include/couch_db.hrl"). start_link(FunAcc) -> - couch_event_sup:start_link(couch_replication, - {couch_replicator_notifier, make_ref()}, FunAcc). + couch_event_sup:start_link( + couch_replication, + {couch_replicator_notifier, make_ref()}, + FunAcc + ). notify(Event) -> gen_event:notify(couch_replication, Event). @@ -34,7 +37,6 @@ notify(Event) -> stop(Pid) -> couch_event_sup:stop(Pid). - init(FunAcc) -> {ok, FunAcc}. diff --git a/src/couch_replicator/src/couch_replicator_rate_limiter.erl b/src/couch_replicator/src/couch_replicator_rate_limiter.erl index b7b70945c..5d2c184b8 100644 --- a/src/couch_replicator/src/couch_replicator_rate_limiter.erl +++ b/src/couch_replicator/src/couch_replicator_rate_limiter.erl @@ -10,7 +10,6 @@ % License for the specific language governing permissions and limitations under % the License. - % This module implements rate limiting based on a variation the additive % increase / multiplicative decrease feedback control algorithm. % @@ -36,29 +35,28 @@ % function is the current period value. Caller then might decide to sleep for % that amount of time before or after each request. - -module(couch_replicator_rate_limiter). -behaviour(gen_server). -export([ - start_link/0 + start_link/0 ]). -export([ - init/1, - terminate/2, - handle_call/3, - handle_info/2, - handle_cast/2, - code_change/3 + init/1, + terminate/2, + handle_call/3, + handle_info/2, + handle_cast/2, + code_change/3 ]). -export([ - interval/1, - max_interval/0, - failure/1, - success/1 + interval/1, + max_interval/0, + failure/1, + success/1 ]). % Types @@ -66,7 +64,6 @@ -type interval() :: non_neg_integer(). -type msec() :: non_neg_integer(). - % Definitions % Main parameters of the algorithm. The factor is the multiplicative part and @@ -98,79 +95,67 @@ % use something similar to solve the ACK compression problem). -define(SENSITIVITY_TIME_WINDOW, 80). - -record(state, {timer}). -record(rec, {id, backoff, ts}). - -spec start_link() -> {ok, pid()} | ignore | {error, term()}. start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - -spec interval(key()) -> interval(). interval(Key) -> {Interval, _Timestamp} = interval_and_timestamp(Key), Interval. - -spec max_interval() -> interval(). max_interval() -> ?MAX_INTERVAL. - -spec failure(key()) -> interval(). failure(Key) -> {Interval, Timestamp} = interval_and_timestamp(Key), update_failure(Key, Interval, Timestamp, now_msec()). - -spec success(key()) -> interval(). success(Key) -> {Interval, Timestamp} = interval_and_timestamp(Key), update_success(Key, Interval, Timestamp, now_msec()). - % gen_server callbacks init([]) -> couch_replicator_rate_limiter_tables:create(#rec.id), {ok, #state{timer = new_timer()}}. - terminate(_Reason, _State) -> ok. - handle_call(_Msg, _From, State) -> {reply, invalid, State}. - handle_cast(_, State) -> {noreply, State}. - handle_info(cleanup, #state{timer = Timer}) -> erlang:cancel_timer(Timer), TIds = couch_replicator_rate_limiter_tables:tids(), [cleanup_table(TId, now_msec() - ?MAX_INTERVAL) || TId <- TIds], {noreply, #state{timer = new_timer()}}. - code_change(_OldVsn, State, _Extra) -> {ok, State}. - % Private functions -spec update_success(any(), interval(), msec(), msec()) -> interval(). update_success(_Key, _Interval, _Timestamp = 0, _Now) -> - 0; % No ets entry. Keep it that way and don't insert a new one. - -update_success(_Key, Interval, Timestamp, Now) - when Now - Timestamp =< ?SENSITIVITY_TIME_WINDOW -> - Interval; % Ignore too frequent updates. - + % No ets entry. Keep it that way and don't insert a new one. + 0; +update_success(_Key, Interval, Timestamp, Now) when + Now - Timestamp =< ?SENSITIVITY_TIME_WINDOW +-> + % Ignore too frequent updates. + Interval; update_success(Key, Interval, Timestamp, Now) -> DecayedInterval = time_decay(Now - Timestamp, Interval), AdditiveFactor = additive_factor(DecayedInterval), @@ -186,19 +171,18 @@ update_success(Key, Interval, Timestamp, Now) -> insert(Key, NewInterval, Now) end. - -spec update_failure(any(), interval(), msec(), msec()) -> interval(). -update_failure(_Key, Interval, Timestamp, Now) - when Now - Timestamp =< ?SENSITIVITY_TIME_WINDOW -> - Interval; % Ignore too frequent updates. - +update_failure(_Key, Interval, Timestamp, Now) when + Now - Timestamp =< ?SENSITIVITY_TIME_WINDOW +-> + % Ignore too frequent updates. + Interval; update_failure(Key, Interval, _Timestamp, Now) -> Interval1 = erlang:max(Interval, ?BASE_INTERVAL), Interval2 = round(Interval1 * ?BACKOFF_FACTOR), Interval3 = erlang:min(Interval2, ?MAX_INTERVAL), insert(Key, Interval3, Now). - -spec insert(any(), interval(), msec()) -> interval(). insert(Key, Interval, Timestamp) -> Entry = #rec{id = Key, backoff = Interval, ts = Timestamp}, @@ -206,7 +190,6 @@ insert(Key, Interval, Timestamp) -> ets:insert(Table, Entry), Interval. - -spec interval_and_timestamp(key()) -> {interval(), msec()}. interval_and_timestamp(Key) -> Table = couch_replicator_rate_limiter_tables:term_to_table(Key), @@ -217,16 +200,13 @@ interval_and_timestamp(Key) -> {Interval, Timestamp} end. - -spec time_decay(msec(), interval()) -> interval(). time_decay(Dt, Interval) when Dt > ?TIME_DECAY_THRESHOLD -> DecayedInterval = Interval - ?TIME_DECAY_FACTOR * Dt, erlang:max(round(DecayedInterval), 0); - time_decay(_Dt, Interval) -> Interval. - % Calculate additive factor. Ideally it would be a constant but in this case % it is a step function to help handle larger values as they are approaching % the backoff limit. Large success values closer to the limit add some @@ -243,18 +223,15 @@ additive_factor(Interval) when Interval > 100 -> additive_factor(_Interval) -> ?BASE_INTERVAL. - -spec new_timer() -> reference(). new_timer() -> erlang:send_after(?MAX_INTERVAL * 2, self(), cleanup). - -spec now_msec() -> msec(). now_msec() -> {Mega, Sec, Micro} = os:timestamp(), ((Mega * 1000000) + Sec) * 1000 + Micro div 1000. - -spec cleanup_table(atom(), msec()) -> non_neg_integer(). cleanup_table(Tid, LimitMSec) -> Head = #rec{ts = '$1', _ = '_'}, diff --git a/src/couch_replicator/src/couch_replicator_rate_limiter_tables.erl b/src/couch_replicator/src/couch_replicator_rate_limiter_tables.erl index 72892b410..2e2556888 100644 --- a/src/couch_replicator/src/couch_replicator_rate_limiter_tables.erl +++ b/src/couch_replicator/src/couch_replicator_rate_limiter_tables.erl @@ -10,7 +10,6 @@ % License for the specific language governing permissions and limitations under % the License. - % Maintain cluster membership and stability notifications for replications. % On changes to cluster membership, broadcast events to `replication` gen_event. % Listeners will get `{cluster, stable}` or `{cluster, unstable}` events. @@ -26,37 +25,32 @@ -module(couch_replicator_rate_limiter_tables). -export([ - create/1, - tids/0, - term_to_table/1 + create/1, + tids/0, + term_to_table/1 ]). -define(SHARDS_N, 16). - -spec create(non_neg_integer()) -> ok. create(KeyPos) -> Opts = [named_table, public, {keypos, KeyPos}, {read_concurrency, true}], [ets:new(list_to_atom(TableName), Opts) || TableName <- table_names()], ok. - -spec tids() -> [atom()]. tids() -> [list_to_existing_atom(TableName) || TableName <- table_names()]. - -spec term_to_table(any()) -> atom(). term_to_table(Term) -> PHash = erlang:phash2(Term), list_to_existing_atom(table_name(PHash rem ?SHARDS_N)). - -spec table_names() -> [string()]. table_names() -> [table_name(N) || N <- lists:seq(0, ?SHARDS_N - 1)]. - -spec table_name(non_neg_integer()) -> string(). table_name(Id) when is_integer(Id), Id >= 0 andalso Id < ?SHARDS_N -> atom_to_list(?MODULE) ++ "_" ++ integer_to_list(Id). diff --git a/src/couch_replicator/src/couch_replicator_scheduler.erl b/src/couch_replicator/src/couch_replicator_scheduler.erl index d3b5b71a4..f544865af 100644 --- a/src/couch_replicator/src/couch_replicator_scheduler.erl +++ b/src/couch_replicator/src/couch_replicator_scheduler.erl @@ -20,28 +20,28 @@ ]). -export([ - init/1, - terminate/2, - handle_call/3, - handle_info/2, - handle_cast/2, - code_change/3, - format_status/2 + init/1, + terminate/2, + handle_call/3, + handle_info/2, + handle_cast/2, + code_change/3, + format_status/2 ]). -export([ - add_job/1, - remove_job/1, - reschedule/0, - rep_state/1, - find_jobs_by_dbname/1, - find_jobs_by_doc/2, - job_summary/2, - health_threshold/0, - jobs/0, - job/1, - restart_job/1, - update_job_stats/2 + add_job/1, + remove_job/1, + reschedule/0, + rep_state/1, + find_jobs_by_dbname/1, + find_jobs_by_doc/2, + job_summary/2, + health_threshold/0, + jobs/0, + job/1, + restart_job/1, + update_job_stats/2 ]). %% config_listener callbacks @@ -59,7 +59,6 @@ -include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl"). -include_lib("couch/include/couch_db.hrl"). - %% definitions -define(MAX_BACKOFF_EXPONENT, 10). -define(BACKOFF_INTERVAL_MICROS, 30 * 1000 * 1000). @@ -72,7 +71,6 @@ -define(DEFAULT_MAX_HISTORY, 20). -define(DEFAULT_SCHEDULER_INTERVAL, 60000). - -record(state, { interval = ?DEFAULT_SCHEDULER_INTERVAL, timer, @@ -88,14 +86,12 @@ crashed_n = 0 :: non_neg_integer() }). - %% public functions -spec start_link() -> {ok, pid()} | ignore | {error, term()}. start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - -spec add_job(#rep{}) -> ok. add_job(#rep{} = Rep) when Rep#rep.id /= undefined -> case existing_replication(Rep) of @@ -110,47 +106,44 @@ add_job(#rep{} = Rep) when Rep#rep.id /= undefined -> ok end. - -spec remove_job(job_id()) -> ok. remove_job(Id) -> gen_server:call(?MODULE, {remove_job, Id}, infinity). - -spec reschedule() -> ok. % Trigger a manual reschedule. Used for testing and/or ops. reschedule() -> gen_server:call(?MODULE, reschedule, infinity). - -spec rep_state(rep_id()) -> #rep{} | nil. rep_state(RepId) -> case (catch ets:lookup_element(?MODULE, RepId, #job.rep)) of - {'EXIT',{badarg, _}} -> + {'EXIT', {badarg, _}} -> nil; Rep -> Rep end. - -spec job_summary(job_id(), non_neg_integer()) -> [_] | nil. job_summary(JobId, HealthThreshold) -> case job_by_id(JobId) of {ok, #job{pid = Pid, history = History, rep = Rep}} -> ErrorCount = consecutive_crashes(History, HealthThreshold), - {State, Info} = case {Pid, ErrorCount} of - {undefined, 0} -> - case History of - [{{crashed, Error}, _When} | _] -> - {crashing, crash_reason_json(Error)}; - [_ | _] -> - {pending, Rep#rep.stats} - end; - {undefined, ErrorCount} when ErrorCount > 0 -> - [{{crashed, Error}, _When} | _] = History, - {crashing, crash_reason_json(Error)}; - {Pid, ErrorCount} when is_pid(Pid) -> - {running, Rep#rep.stats} - end, + {State, Info} = + case {Pid, ErrorCount} of + {undefined, 0} -> + case History of + [{{crashed, Error}, _When} | _] -> + {crashing, crash_reason_json(Error)}; + [_ | _] -> + {pending, Rep#rep.stats} + end; + {undefined, ErrorCount} when ErrorCount > 0 -> + [{{crashed, Error}, _When} | _] = History, + {crashing, crash_reason_json(Error)}; + {Pid, ErrorCount} when is_pid(Pid) -> + {running, Rep#rep.stats} + end, [ {source, iolist_to_binary(ejson_url(Rep#rep.source))}, {target, iolist_to_binary(ejson_url(Rep#rep.target))}, @@ -158,22 +151,20 @@ job_summary(JobId, HealthThreshold) -> {info, couch_replicator_utils:ejson_state_info(Info)}, {error_count, ErrorCount}, {last_updated, last_updated(History)}, - {start_time, - couch_replicator_utils:iso8601(Rep#rep.start_time)}, + {start_time, couch_replicator_utils:iso8601(Rep#rep.start_time)}, {source_proxy, job_proxy_url(Rep#rep.source)}, {target_proxy, job_proxy_url(Rep#rep.target)} ]; {error, not_found} -> - nil % Job might have just completed + % Job might have just completed + nil end. - job_proxy_url(#httpdb{proxy_url = ProxyUrl}) when is_list(ProxyUrl) -> list_to_binary(couch_util:url_strip_password(ProxyUrl)); job_proxy_url(_Endpoint) -> null. - % Health threshold is the minimum amount of time an unhealthy job should run % crashing before it is considered to be healthy again. HealtThreashold should % not be 0 as jobs could start and immediately crash, and it shouldn't be @@ -181,9 +172,11 @@ job_proxy_url(_Endpoint) -> % job is back to normal. -spec health_threshold() -> non_neg_integer(). health_threshold() -> - config:get_integer("replicator", "health_threshold", - ?DEFAULT_HEALTH_THRESHOLD_SEC). - + config:get_integer( + "replicator", + "health_threshold", + ?DEFAULT_HEALTH_THRESHOLD_SEC + ). -spec find_jobs_by_dbname(binary()) -> list(#rep{}). find_jobs_by_dbname(DbName) -> @@ -191,14 +184,12 @@ find_jobs_by_dbname(DbName) -> MatchSpec = #job{id = '$1', rep = Rep, _ = '_'}, [RepId || [RepId] <- ets:match(?MODULE, MatchSpec)]. - -spec find_jobs_by_doc(binary(), binary()) -> list(#rep{}). find_jobs_by_doc(DbName, DocId) -> - Rep = #rep{db_name = DbName, doc_id = DocId, _ = '_'}, + Rep = #rep{db_name = DbName, doc_id = DocId, _ = '_'}, MatchSpec = #job{id = '$1', rep = Rep, _ = '_'}, [RepId || [RepId] <- ets:match(?MODULE, MatchSpec)]. - -spec restart_job(binary() | list() | rep_id()) -> {ok, {[_]}} | {error, not_found}. restart_job(JobId) -> @@ -211,28 +202,39 @@ restart_job(JobId) -> job(JobId) end. - -spec update_job_stats(job_id(), term()) -> ok. update_job_stats(JobId, Stats) -> gen_server:cast(?MODULE, {update_job_stats, JobId, Stats}). - %% gen_server functions init(_) -> config:enable_feature('scheduler'), - EtsOpts = [named_table, {keypos, #job.id}, {read_concurrency, true}, - {write_concurrency, true}], + EtsOpts = [ + named_table, + {keypos, #job.id}, + {read_concurrency, true}, + {write_concurrency, true} + ], ?MODULE = ets:new(?MODULE, EtsOpts), ok = couch_replicator_share:init(), ok = config:listen_for_changes(?MODULE, nil), - Interval = config:get_integer("replicator", "interval", - ?DEFAULT_SCHEDULER_INTERVAL), + Interval = config:get_integer( + "replicator", + "interval", + ?DEFAULT_SCHEDULER_INTERVAL + ), MaxJobs = config:get_integer("replicator", "max_jobs", ?DEFAULT_MAX_JOBS), - MaxChurn = config:get_integer("replicator", "max_churn", - ?DEFAULT_MAX_CHURN), - MaxHistory = config:get_integer("replicator", "max_history", - ?DEFAULT_MAX_HISTORY), + MaxChurn = config:get_integer( + "replicator", + "max_churn", + ?DEFAULT_MAX_CHURN + ), + MaxHistory = config:get_integer( + "replicator", + "max_history", + ?DEFAULT_MAX_HISTORY + ), Timer = erlang:send_after(Interval, self(), reschedule), State = #state{ interval = Interval, @@ -244,7 +246,6 @@ init(_) -> }, {ok, State}. - handle_call({add_job, Job}, _From, State) -> ok = maybe_remove_job_int(Job#job.id, State), true = add_job_int(Job), @@ -253,50 +254,51 @@ handle_call({add_job, Job}, _From, State) -> TotalJobs = ets:info(?MODULE, size), couch_stats:update_gauge([couch_replicator, jobs, total], TotalJobs), {reply, ok, State}; - handle_call({remove_job, Id}, _From, State) -> ok = maybe_remove_job_int(Id, State), {reply, ok, State}; - handle_call(reschedule, _From, State) -> ok = reschedule(State), {reply, ok, State}; - handle_call(_, _From, State) -> {noreply, State}. - -handle_cast({set_max_jobs, MaxJobs}, State) when is_integer(MaxJobs), - MaxJobs >= 0 -> +handle_cast({set_max_jobs, MaxJobs}, State) when + is_integer(MaxJobs), + MaxJobs >= 0 +-> couch_log:notice("~p: max_jobs set to ~B", [?MODULE, MaxJobs]), {noreply, State#state{max_jobs = MaxJobs}}; - -handle_cast({set_max_churn, MaxChurn}, State) when is_integer(MaxChurn), - MaxChurn > 0 -> +handle_cast({set_max_churn, MaxChurn}, State) when + is_integer(MaxChurn), + MaxChurn > 0 +-> couch_log:notice("~p: max_churn set to ~B", [?MODULE, MaxChurn]), {noreply, State#state{max_churn = MaxChurn}}; - -handle_cast({set_max_history, MaxHistory}, State) when is_integer(MaxHistory), - MaxHistory > 0 -> +handle_cast({set_max_history, MaxHistory}, State) when + is_integer(MaxHistory), + MaxHistory > 0 +-> couch_log:notice("~p: max_history set to ~B", [?MODULE, MaxHistory]), {noreply, State#state{max_history = MaxHistory}}; - -handle_cast({set_interval, Interval}, State) when is_integer(Interval), - Interval > 0 -> +handle_cast({set_interval, Interval}, State) when + is_integer(Interval), + Interval > 0 +-> couch_log:notice("~p: interval set to ~B", [?MODULE, Interval]), {noreply, State#state{interval = Interval}}; - -handle_cast({update_shares, Key, Shares}, State) when is_binary(Key), - is_integer(Shares), Shares >= 0 -> +handle_cast({update_shares, Key, Shares}, State) when + is_binary(Key), + is_integer(Shares), + Shares >= 0 +-> couch_log:notice("~p: shares for ~s set to ~B", [?MODULE, Key, Shares]), couch_replicator_share:update_shares(Key, Shares), {noreply, State}; - -handle_cast({reset_shares, Key}, State) when is_binary(Key) -> +handle_cast({reset_shares, Key}, State) when is_binary(Key) -> couch_log:notice("~p: shares for ~s reset to default", [?MODULE, Key]), couch_replicator_share:reset_shares(Key), {noreply, State}; - handle_cast({update_job_stats, JobId, Stats}, State) -> case rep_state(JobId) of nil -> @@ -306,18 +308,15 @@ handle_cast({update_job_stats, JobId, Stats}, State) -> true = ets:update_element(?MODULE, JobId, {#job.rep, NewRep}) end, {noreply, State}; - handle_cast(UnexpectedMsg, State) -> couch_log:error("~p: received un-expected cast ~p", [?MODULE, UnexpectedMsg]), {noreply, State}. - handle_info(reschedule, State) -> ok = reschedule(State), erlang:cancel_timer(State#state.timer), Timer = erlang:send_after(State#state.interval, self(), reschedule), {noreply, State#state{timer = Timer}}; - handle_info({'DOWN', _Ref, process, Pid, normal}, State) -> {ok, Job} = job_by_pid(Pid), couch_log:notice("~p: Job ~p completed normally", [?MODULE, Job#job.id]), @@ -326,82 +325,66 @@ handle_info({'DOWN', _Ref, process, Pid, normal}, State) -> remove_job_int(Job), update_running_jobs_stats(State#state.stats_pid), {noreply, State}; - handle_info({'DOWN', _Ref, process, Pid, Reason0}, State) -> {ok, Job} = job_by_pid(Pid), - Reason = case Reason0 of - {shutdown, ShutdownReason} -> ShutdownReason; - Other -> Other - end, + Reason = + case Reason0 of + {shutdown, ShutdownReason} -> ShutdownReason; + Other -> Other + end, Interval = State#state.interval, couch_replicator_share:charge(Job, Interval, os:timestamp()), ok = handle_crashed_job(Job, Reason, State), {noreply, State}; - handle_info(restart_config_listener, State) -> ok = config:listen_for_changes(?MODULE, nil), {noreply, State}; - handle_info(_, State) -> {noreply, State}. - code_change(_OldVsn, State, _Extra) -> {ok, State}. - terminate(_Reason, _State) -> couch_replicator_share:clear(), ok. - format_status(_Opt, [_PDict, State]) -> [ - {max_jobs, State#state.max_jobs}, - {running_jobs, running_job_count()}, - {pending_jobs, pending_job_count()} + {max_jobs, State#state.max_jobs}, + {running_jobs, running_job_count()}, + {pending_jobs, pending_job_count()} ]. - %% config listener functions handle_config_change("replicator", "max_jobs", V, _, S) -> ok = gen_server:cast(?MODULE, {set_max_jobs, list_to_integer(V)}), {ok, S}; - handle_config_change("replicator", "max_churn", V, _, S) -> ok = gen_server:cast(?MODULE, {set_max_churn, list_to_integer(V)}), {ok, S}; - handle_config_change("replicator", "interval", V, _, S) -> ok = gen_server:cast(?MODULE, {set_interval, list_to_integer(V)}), {ok, S}; - handle_config_change("replicator", "max_history", V, _, S) -> ok = gen_server:cast(?MODULE, {set_max_history, list_to_integer(V)}), {ok, S}; - handle_config_change("replicator.shares", Key, deleted, _, S) -> ok = gen_server:cast(?MODULE, {reset_shares, list_to_binary(Key)}), {ok, S}; - handle_config_change("replicator.shares", Key, V, _, S) -> - ok = gen_server:cast(?MODULE, {update_shares, list_to_binary(Key), - list_to_integer(V)}), + ok = gen_server:cast(?MODULE, {update_shares, list_to_binary(Key), list_to_integer(V)}), {ok, S}; - handle_config_change(_, _, _, _, S) -> {ok, S}. - handle_config_terminate(_, stop, _) -> ok; - handle_config_terminate(_, _, _) -> Pid = whereis(?MODULE), erlang:send_after(?RELISTEN_DELAY, Pid, restart_config_listener). - %% Private functions % Handle crashed jobs. Handling differs between transient and permanent jobs. @@ -420,7 +403,6 @@ handle_crashed_job(#job{rep = #rep{db_name = null}} = Job, Reason, State) -> remove_job_int(Job), update_running_jobs_stats(State#state.stats_pid), ok; - handle_crashed_job(Job, Reason, State) -> ok = update_state_crashed(Job, Reason, State), case couch_replicator_doc_processor:update_docs() of @@ -442,7 +424,6 @@ handle_crashed_job(Job, Reason, State) -> ok end. - % Attempt to start a newly added job. First quickly check if total jobs % already exceed max jobs, then do a more expensive check which runs a % select (an O(n) operation) to check pending jobs specifically. @@ -459,7 +440,6 @@ maybe_start_newly_added_job(Job, State) -> ok end. - % Return up to a given number of oldest, not recently crashed jobs. Try to be % memory efficient and use ets:foldl to accumulate jobs. -spec pending_jobs(non_neg_integer()) -> [#job{}]. @@ -468,36 +448,34 @@ pending_jobs(0) -> % other function clause it will crash as gb_sets:largest assumes set is not % empty. []; - pending_jobs(Count) when is_integer(Count), Count > 0 -> - Set0 = gb_sets:new(), % [{{Priority, LastStart}, Job},...] + % [{{Priority, LastStart}, Job},...] + Set0 = gb_sets:new(), Now = os:timestamp(), Acc0 = {Set0, Now, Count, health_threshold()}, {Set1, _, _, _} = ets:foldl(fun pending_fold/2, Acc0, ?MODULE), [Job || {_PriorityKey, Job} <- gb_sets:to_list(Set1)]. - pending_fold(#job{pid = Pid}, Acc) when is_pid(Pid) -> Acc; - pending_fold(Job, {Set, Now, Count, HealthThreshold}) -> Healthy = not_recently_crashed(Job, Now, HealthThreshold), - Set1 = case {Healthy, gb_sets:size(Set) >= Count} of - {true, true} -> - % Job is healthy but already reached accumulated limit, so might - % have to replace one of the accumulated jobs - pending_maybe_replace(Job, Set); - {true, false} -> - % Job is healthy and we haven't reached the limit, so add job - % to accumulator - gb_sets:add_element({start_priority_key(Job), Job}, Set); - {false, _} -> - % This job is not healthy (has crashed too recently), so skip it. - Set - end, + Set1 = + case {Healthy, gb_sets:size(Set) >= Count} of + {true, true} -> + % Job is healthy but already reached accumulated limit, so might + % have to replace one of the accumulated jobs + pending_maybe_replace(Job, Set); + {true, false} -> + % Job is healthy and we haven't reached the limit, so add job + % to accumulator + gb_sets:add_element({start_priority_key(Job), Job}, Set); + {false, _} -> + % This job is not healthy (has crashed too recently), so skip it. + Set + end, {Set1, Now, Count, HealthThreshold}. - % Replace Job in the accumulator if it has a higher priority (lower priority % value) than the lowest priority there. Job priority is indexed by % {FairSharePiority, LastStarted} tuples. If the FairSharePriority is the same @@ -526,16 +504,13 @@ pending_maybe_replace(Job, Set) -> start_priority_key(#job{} = Job) -> {couch_replicator_share:priority(Job#job.id), last_started(Job)}. - start_jobs(Count, State) -> [start_job_int(Job, State) || Job <- pending_jobs(Count)], ok. - -spec stop_jobs(non_neg_integer(), boolean(), #state{}) -> non_neg_integer(). stop_jobs(Count, _, _) when is_integer(Count), Count =< 0 -> 0; - stop_jobs(Count, IsContinuous, State) when is_integer(Count) -> Running0 = running_jobs(), ContinuousPred = fun(Job) -> is_continuous(Job) =:= IsContinuous end, @@ -544,7 +519,6 @@ stop_jobs(Count, IsContinuous, State) when is_integer(Count) -> Running3 = lists:sublist(lists:sort(Running2), Count), length([stop_job_int(Job, State) || {_SortKey, Job} <- Running3]). - % Lower priority jobs have higher priority values, so we negate them, that way % when sorted, they'll come up first. If priorities are equal, jobs are sorted % by the lowest starting times as jobs with lowest start time have been running @@ -553,7 +527,6 @@ stop_jobs(Count, IsContinuous, State) when is_integer(Count) -> stop_priority_key(#job{} = Job) -> {-couch_replicator_share:priority(Job#job.id), last_started(Job)}. - not_recently_crashed(#job{history = History}, Now, HealthThreshold) -> case History of [{added, _When}] -> @@ -566,7 +539,6 @@ not_recently_crashed(#job{history = History}, Now, HealthThreshold) -> timer:now_diff(Now, LatestCrashT) >= backoff_micros(CrashCount) end. - % Count consecutive crashes. A crash happens when there is a `crashed` event % within a short period of time (configurable) after any other event. It could % be `crashed, started` for jobs crashing quickly after starting, `crashed, @@ -583,40 +555,39 @@ not_recently_crashed(#job{history = History}, Now, HealthThreshold) -> consecutive_crashes(History, HealthThreshold) when is_list(History) -> consecutive_crashes(History, HealthThreshold, 0). - -spec consecutive_crashes(history(), non_neg_integer(), non_neg_integer()) -> - non_neg_integer(). + non_neg_integer(). consecutive_crashes([], _HealthThreashold, Count) -> Count; - -consecutive_crashes([{{crashed, _}, CrashT}, {_, PrevT} = PrevEvent | Rest], - HealthThreshold, Count) -> +consecutive_crashes( + [{{crashed, _}, CrashT}, {_, PrevT} = PrevEvent | Rest], + HealthThreshold, + Count +) -> case timer:now_diff(CrashT, PrevT) > HealthThreshold * 1000000 of true -> Count; false -> consecutive_crashes([PrevEvent | Rest], HealthThreshold, Count + 1) end; - -consecutive_crashes([{stopped, _}, {started, _} | _], _HealthThreshold, - Count) -> +consecutive_crashes( + [{stopped, _}, {started, _} | _], + _HealthThreshold, + Count +) -> Count; - consecutive_crashes([_ | Rest], HealthThreshold, Count) -> consecutive_crashes(Rest, HealthThreshold, Count). - -spec latest_crash_timestamp(history()) -> erlang:timestamp(). latest_crash_timestamp([]) -> - {0, 0, 0}; % Used to avoid special-casing "no crash" when doing now_diff - + % Used to avoid special-casing "no crash" when doing now_diff + {0, 0, 0}; latest_crash_timestamp([{{crashed, _Reason}, When} | _]) -> When; - latest_crash_timestamp([_Event | Rest]) -> latest_crash_timestamp(Rest). - -spec backoff_micros(non_neg_integer()) -> non_neg_integer(). backoff_micros(CrashCount) -> % When calculating the backoff interval treat consecutive crash count as the @@ -626,13 +597,11 @@ backoff_micros(CrashCount) -> BackoffExp = erlang:min(CrashCount - 1, ?MAX_BACKOFF_EXPONENT), (1 bsl BackoffExp) * ?BACKOFF_INTERVAL_MICROS. - -spec add_job_int(#job{}) -> boolean(). add_job_int(#job{} = Job) -> couch_replicator_share:job_added(Job), ets:insert_new(?MODULE, Job). - -spec maybe_remove_job_int(job_id(), #state{}) -> ok. maybe_remove_job_int(JobId, State) -> case job_by_id(JobId) of @@ -644,98 +613,99 @@ maybe_remove_job_int(JobId, State) -> true = remove_job_int(Job), couch_stats:increment_counter([couch_replicator, jobs, removes]), TotalJobs = ets:info(?MODULE, size), - couch_stats:update_gauge([couch_replicator, jobs, total], - TotalJobs), + couch_stats:update_gauge( + [couch_replicator, jobs, total], + TotalJobs + ), update_running_jobs_stats(State#state.stats_pid), ok; {error, not_found} -> ok end. - start_job_int(#job{pid = Pid}, _State) when Pid /= undefined -> ok; - start_job_int(#job{} = Job0, State) -> Job = maybe_optimize_job_for_rate_limiting(Job0), case couch_replicator_scheduler_sup:start_child(Job#job.rep) of {ok, Child} -> Ref = monitor(process, Child), ok = update_state_started(Job, Child, Ref, State), - couch_log:notice("~p: Job ~p started as ~p", - [?MODULE, Job#job.id, Child]); + couch_log:notice( + "~p: Job ~p started as ~p", + [?MODULE, Job#job.id, Child] + ); {error, {already_started, OtherPid}} when node(OtherPid) =:= node() -> Ref = monitor(process, OtherPid), ok = update_state_started(Job, OtherPid, Ref, State), - couch_log:notice("~p: Job ~p already running as ~p. Most likely" + couch_log:notice( + "~p: Job ~p already running as ~p. Most likely" " because replicator scheduler was restarted", - [?MODULE, Job#job.id, OtherPid]); + [?MODULE, Job#job.id, OtherPid] + ); {error, {already_started, OtherPid}} when node(OtherPid) =/= node() -> CrashMsg = "Duplicate replication running on another node", - couch_log:notice("~p: Job ~p already running as ~p. Most likely" + couch_log:notice( + "~p: Job ~p already running as ~p. Most likely" " because a duplicate replication is running on another node", - [?MODULE, Job#job.id, OtherPid]), + [?MODULE, Job#job.id, OtherPid] + ), ok = update_state_crashed(Job, CrashMsg, State); {error, Reason} -> - couch_log:notice("~p: Job ~p failed to start for reason ~p", - [?MODULE, Job, Reason]), + couch_log:notice( + "~p: Job ~p failed to start for reason ~p", + [?MODULE, Job, Reason] + ), ok = update_state_crashed(Job, Reason, State) end. - -spec stop_job_int(#job{}, #state{}) -> ok | {error, term()}. stop_job_int(#job{pid = undefined}, _State) -> ok; - stop_job_int(#job{} = Job, State) -> ok = couch_replicator_scheduler_sup:terminate_child(Job#job.pid), demonitor(Job#job.monitor, [flush]), ok = update_state_stopped(Job, State), - couch_log:notice("~p: Job ~p stopped as ~p", - [?MODULE, Job#job.id, Job#job.pid]). - + couch_log:notice( + "~p: Job ~p stopped as ~p", + [?MODULE, Job#job.id, Job#job.pid] + ). -spec remove_job_int(#job{}) -> true. remove_job_int(#job{} = Job) -> couch_replicator_share:job_removed(Job), ets:delete(?MODULE, Job#job.id). - -spec running_job_count() -> non_neg_integer(). running_job_count() -> ets:info(?MODULE, size) - pending_job_count(). - -spec running_jobs() -> [#job{}]. running_jobs() -> - ets:select(?MODULE, [{#job{pid = '$1', _='_'}, [{is_pid, '$1'}], ['$_']}]). - + ets:select(?MODULE, [{#job{pid = '$1', _ = '_'}, [{is_pid, '$1'}], ['$_']}]). -spec pending_job_count() -> non_neg_integer(). pending_job_count() -> - ets:select_count(?MODULE, [{#job{pid=undefined, _='_'}, [], [true]}]). - + ets:select_count(?MODULE, [{#job{pid = undefined, _ = '_'}, [], [true]}]). -spec job_by_pid(pid()) -> {ok, #job{}} | {error, not_found}. job_by_pid(Pid) when is_pid(Pid) -> - case ets:match_object(?MODULE, #job{pid=Pid, _='_'}) of + case ets:match_object(?MODULE, #job{pid = Pid, _ = '_'}) of [] -> {error, not_found}; - [#job{}=Job] -> + [#job{} = Job] -> {ok, Job} end. - -spec job_by_id(job_id()) -> {ok, #job{}} | {error, not_found}. job_by_id(Id) -> case ets:lookup(?MODULE, Id) of [] -> {error, not_found}; - [#job{}=Job] -> + [#job{} = Job] -> {ok, Job} end. - -spec update_state_stopped(#job{}, #state{}) -> ok. update_state_stopped(Job, State) -> Job1 = reset_job_process(Job), @@ -744,7 +714,6 @@ update_state_stopped(Job, State) -> couch_stats:increment_counter([couch_replicator, jobs, stops]), ok. - -spec update_state_started(#job{}, pid(), reference(), #state{}) -> ok. update_state_started(Job, Pid, Ref, State) -> Job1 = set_job_process(Job, Pid, Ref), @@ -753,7 +722,6 @@ update_state_started(Job, Pid, Ref, State) -> couch_stats:increment_counter([couch_replicator, jobs, starts]), ok. - -spec update_state_crashed(#job{}, any(), #state{}) -> ok. update_state_crashed(Job, Reason, State) -> Job1 = reset_job_process(Job), @@ -762,17 +730,14 @@ update_state_crashed(Job, Reason, State) -> couch_stats:increment_counter([couch_replicator, jobs, crashes]), ok. - -spec set_job_process(#job{}, pid(), reference()) -> #job{}. set_job_process(#job{} = Job, Pid, Ref) when is_pid(Pid), is_reference(Ref) -> Job#job{pid = Pid, monitor = Ref}. - -spec reset_job_process(#job{}) -> #job{}. reset_job_process(#job{} = Job) -> Job#job{pid = undefined, monitor = undefined}. - -spec reschedule(#state{}) -> ok. reschedule(#state{interval = Interval} = State) -> couch_replicator_share:update(running_jobs(), Interval, os:timestamp()), @@ -780,50 +745,48 @@ reschedule(#state{interval = Interval} = State) -> rotate_jobs(State, StopCount), update_running_jobs_stats(State#state.stats_pid). - -spec stop_excess_jobs(#state{}, non_neg_integer()) -> non_neg_integer(). stop_excess_jobs(State, Running) -> - #state{max_jobs=MaxJobs} = State, + #state{max_jobs = MaxJobs} = State, StopCount = max(0, Running - MaxJobs), Stopped = stop_jobs(StopCount, true, State), OneshotLeft = StopCount - Stopped, stop_jobs(OneshotLeft, false, State), StopCount. - start_pending_jobs(State) -> - #state{max_jobs=MaxJobs} = State, + #state{max_jobs = MaxJobs} = State, Running = running_job_count(), Pending = pending_job_count(), - if Running < MaxJobs, Pending > 0 -> - start_jobs(MaxJobs - Running, State); - true -> - ok + if + Running < MaxJobs, Pending > 0 -> + start_jobs(MaxJobs - Running, State); + true -> + ok end. - -spec rotate_jobs(#state{}, non_neg_integer()) -> ok. rotate_jobs(State, ChurnSoFar) -> - #state{max_jobs=MaxJobs, max_churn=MaxChurn} = State, + #state{max_jobs = MaxJobs, max_churn = MaxChurn} = State, Running = running_job_count(), Pending = pending_job_count(), % Reduce MaxChurn by the number of already stopped jobs in the % current rescheduling cycle. Churn = max(0, MaxChurn - ChurnSoFar), SlotsAvailable = MaxJobs - Running, - if SlotsAvailable >= 0 -> - % If there is are enough SlotsAvailable reduce StopCount to avoid - % unnesessarily stopping jobs. `stop_jobs/3` ignores 0 or negative - % values so we don't worry about that here. - StopCount = lists:min([Pending - SlotsAvailable, Running, Churn]), - stop_jobs(StopCount, true, State), - StartCount = max(0, MaxJobs - running_job_count()), - start_jobs(StartCount, State); - true -> - ok + if + SlotsAvailable >= 0 -> + % If there is are enough SlotsAvailable reduce StopCount to avoid + % unnesessarily stopping jobs. `stop_jobs/3` ignores 0 or negative + % values so we don't worry about that here. + StopCount = lists:min([Pending - SlotsAvailable, Running, Churn]), + stop_jobs(StopCount, true, State), + StartCount = max(0, MaxJobs - running_job_count()), + start_jobs(StartCount, State); + true -> + ok end. - -spec last_started(#job{}) -> erlang:timestamp(). last_started(#job{} = Job) -> case lists:keyfind(started, 1, Job#job.history) of @@ -833,7 +796,6 @@ last_started(#job{} = Job) -> When end. - -spec update_history(#job{}, event_type(), erlang:timestamp(), #state{}) -> #job{}. update_history(Job, Type, When, State) -> @@ -841,35 +803,38 @@ update_history(Job, Type, When, State) -> History1 = lists:sublist(History0, State#state.max_history), Job#job{history = History1}. - -spec ejson_url(#httpdb{} | binary()) -> binary(). -ejson_url(#httpdb{}=Httpdb) -> +ejson_url(#httpdb{} = Httpdb) -> couch_util:url_strip_password(Httpdb#httpdb.url); ejson_url(DbName) when is_binary(DbName) -> DbName. - -spec job_ejson(#job{}) -> {[_ | _]}. job_ejson(Job) -> Rep = Job#job.rep, Source = ejson_url(Rep#rep.source), Target = ejson_url(Rep#rep.target), - History = lists:map(fun({Type, When}) -> - EventProps = case Type of - {crashed, Reason} -> - [{type, crashed}, {reason, crash_reason_json(Reason)}]; - Type -> - [{type, Type}] + History = lists:map( + fun({Type, When}) -> + EventProps = + case Type of + {crashed, Reason} -> + [{type, crashed}, {reason, crash_reason_json(Reason)}]; + Type -> + [{type, Type}] + end, + {[{timestamp, couch_replicator_utils:iso8601(When)} | EventProps]} end, - {[{timestamp, couch_replicator_utils:iso8601(When)} | EventProps]} - end, Job#job.history), + Job#job.history + ), {BaseID, Ext} = Job#job.id, - Pid = case Job#job.pid of - undefined -> - null; - P when is_pid(P) -> - ?l2b(pid_to_list(P)) - end, + Pid = + case Job#job.pid of + undefined -> + null; + P when is_pid(P) -> + ?l2b(pid_to_list(P)) + end, {[ {id, iolist_to_binary([BaseID, Ext])}, {pid, Pid}, @@ -884,12 +849,10 @@ job_ejson(Job) -> {start_time, couch_replicator_utils:iso8601(Rep#rep.start_time)} ]}. - -spec jobs() -> [[tuple()]]. jobs() -> ets:foldl(fun(Job, Acc) -> [job_ejson(Job) | Acc] end, [], ?MODULE). - -spec job(job_id()) -> {ok, {[_ | _]}} | {error, not_found}. job(JobId) -> case job_by_id(JobId) of @@ -899,7 +862,6 @@ job(JobId) -> Error end. - crash_reason_json({_CrashType, Info}) when is_binary(Info) -> Info; crash_reason_json(Reason) when is_binary(Reason) -> @@ -907,22 +869,23 @@ crash_reason_json(Reason) when is_binary(Reason) -> crash_reason_json(Error) -> couch_replicator_utils:rep_error_to_binary(Error). - -spec last_updated([_]) -> binary(). last_updated([{_Type, When} | _]) -> couch_replicator_utils:iso8601(When). - -spec is_continuous(#job{}) -> boolean(). is_continuous(#job{rep = Rep}) -> couch_util:get_value(continuous, Rep#rep.options, false). - % If job crashed last time because it was rate limited, try to % optimize some options to help the job make progress. -spec maybe_optimize_job_for_rate_limiting(#job{}) -> #job{}. -maybe_optimize_job_for_rate_limiting(Job = #job{history = - [{{crashed, max_backoff}, _} | _]}) -> +maybe_optimize_job_for_rate_limiting( + Job = #job{ + history = + [{{crashed, max_backoff}, _} | _] + } +) -> Opts = [ {checkpoint_interval, 5000}, {worker_processes, 2}, @@ -934,7 +897,6 @@ maybe_optimize_job_for_rate_limiting(Job = #job{history = maybe_optimize_job_for_rate_limiting(Job) -> Job. - -spec optimize_int_option({atom(), any()}, #rep{}) -> #rep{}. optimize_int_option({Key, Val}, #rep{options = Options} = Rep) -> case couch_util:get_value(Key, Options) of @@ -947,7 +909,6 @@ optimize_int_option({Key, Val}, #rep{options = Options} = Rep) -> Rep end. - % Updater is a separate process. It receives `update_stats` messages and % updates scheduler stats from the scheduler jobs table. Updates are % performed no more frequently than once per ?STATS_UPDATE_WAIT milliseconds. @@ -956,11 +917,9 @@ update_running_jobs_stats(StatsPid) when is_pid(StatsPid) -> StatsPid ! update_stats, ok. - start_stats_updater() -> erlang:spawn_link(?MODULE, stats_updater_loop, [undefined]). - stats_updater_loop(Timer) -> receive update_stats when Timer == undefined -> @@ -975,31 +934,28 @@ stats_updater_loop(Timer) -> erlang:exit({stats_updater_bad_msg, Else}) end. - -spec stats_updater_refresh() -> ok. stats_updater_refresh() -> #stats_acc{ - pending_n = PendingN, - running_n = RunningN, - crashed_n = CrashedN - } = ets:foldl(fun stats_fold/2, #stats_acc{}, ?MODULE), + pending_n = PendingN, + running_n = RunningN, + crashed_n = CrashedN + } = ets:foldl(fun stats_fold/2, #stats_acc{}, ?MODULE), couch_stats:update_gauge([couch_replicator, jobs, pending], PendingN), couch_stats:update_gauge([couch_replicator, jobs, running], RunningN), couch_stats:update_gauge([couch_replicator, jobs, crashed], CrashedN), ok. - -spec stats_fold(#job{}, #stats_acc{}) -> #stats_acc{}. stats_fold(#job{pid = undefined, history = [{added, _}]}, Acc) -> Acc#stats_acc{pending_n = Acc#stats_acc.pending_n + 1}; stats_fold(#job{pid = undefined, history = [{stopped, _} | _]}, Acc) -> Acc#stats_acc{pending_n = Acc#stats_acc.pending_n + 1}; stats_fold(#job{pid = undefined, history = [{{crashed, _}, _} | _]}, Acc) -> - Acc#stats_acc{crashed_n =Acc#stats_acc.crashed_n + 1}; + Acc#stats_acc{crashed_n = Acc#stats_acc.crashed_n + 1}; stats_fold(#job{pid = P, history = [{started, _} | _]}, Acc) when is_pid(P) -> Acc#stats_acc{running_n = Acc#stats_acc.running_n + 1}. - -spec existing_replication(#rep{}) -> boolean(). existing_replication(#rep{} = NewRep) -> case job_by_id(NewRep#rep.id) of @@ -1011,67 +967,85 @@ existing_replication(#rep{} = NewRep) -> false end. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). - backoff_micros_test_() -> BaseInterval = ?BACKOFF_INTERVAL_MICROS, - [?_assertEqual(R * BaseInterval, backoff_micros(N)) || {R, N} <- [ - {1, 1}, {2, 2}, {4, 3}, {8, 4}, {16, 5}, {32, 6}, {64, 7}, {128, 8}, - {256, 9}, {512, 10}, {1024, 11}, {1024, 12} - ]]. - + [ + ?_assertEqual(R * BaseInterval, backoff_micros(N)) + || {R, N} <- [ + {1, 1}, + {2, 2}, + {4, 3}, + {8, 4}, + {16, 5}, + {32, 6}, + {64, 7}, + {128, 8}, + {256, 9}, + {512, 10}, + {1024, 11}, + {1024, 12} + ] + ]. consecutive_crashes_test_() -> Threshold = ?DEFAULT_HEALTH_THRESHOLD_SEC, - [?_assertEqual(R, consecutive_crashes(H, Threshold)) || {R, H} <- [ - {0, []}, - {0, [added()]}, - {0, [stopped()]}, - {0, [crashed()]}, - {1, [crashed(), added()]}, - {1, [crashed(), crashed()]}, - {1, [crashed(), stopped()]}, - {3, [crashed(), crashed(), crashed(), added()]}, - {2, [crashed(), crashed(), stopped()]}, - {1, [crashed(), started(), added()]}, - {2, [crashed(3), started(2), crashed(1), started(0)]}, - {0, [stopped(3), started(2), crashed(1), started(0)]}, - {1, [crashed(3), started(2), stopped(1), started(0)]}, - {0, [crashed(999), started(0)]}, - {1, [crashed(999), started(998), crashed(997), started(0)]} - ]]. - + [ + ?_assertEqual(R, consecutive_crashes(H, Threshold)) + || {R, H} <- [ + {0, []}, + {0, [added()]}, + {0, [stopped()]}, + {0, [crashed()]}, + {1, [crashed(), added()]}, + {1, [crashed(), crashed()]}, + {1, [crashed(), stopped()]}, + {3, [crashed(), crashed(), crashed(), added()]}, + {2, [crashed(), crashed(), stopped()]}, + {1, [crashed(), started(), added()]}, + {2, [crashed(3), started(2), crashed(1), started(0)]}, + {0, [stopped(3), started(2), crashed(1), started(0)]}, + {1, [crashed(3), started(2), stopped(1), started(0)]}, + {0, [crashed(999), started(0)]}, + {1, [crashed(999), started(998), crashed(997), started(0)]} + ] + ]. consecutive_crashes_non_default_threshold_test_() -> - [?_assertEqual(R, consecutive_crashes(H, T)) || {R, H, T} <- [ - {0, [crashed(11), started(0)], 10}, - {1, [crashed(10), started(0)], 10} - ]]. - + [ + ?_assertEqual(R, consecutive_crashes(H, T)) + || {R, H, T} <- [ + {0, [crashed(11), started(0)], 10}, + {1, [crashed(10), started(0)], 10} + ] + ]. latest_crash_timestamp_test_() -> - [?_assertEqual({0, R, 0}, latest_crash_timestamp(H)) || {R, H} <- [ - {0, [added()]}, - {1, [crashed(1)]}, - {3, [crashed(3), started(2), crashed(1), started(0)]}, - {1, [started(3), stopped(2), crashed(1), started(0)]} - ]]. - + [ + ?_assertEqual({0, R, 0}, latest_crash_timestamp(H)) + || {R, H} <- [ + {0, [added()]}, + {1, [crashed(1)]}, + {3, [crashed(3), started(2), crashed(1), started(0)]}, + {1, [started(3), stopped(2), crashed(1), started(0)]} + ] + ]. last_started_test_() -> - [?_assertEqual({0, R, 0}, last_started(testjob(H))) || {R, H} <- [ - {0, [added()]}, - {0, [crashed(1)]}, - {1, [started(1)]}, - {1, [added(), started(1)]}, - {2, [started(2), started(1)]}, - {2, [crashed(3), started(2), started(1)]} - ]]. - + [ + ?_assertEqual({0, R, 0}, last_started(testjob(H))) + || {R, H} <- [ + {0, [added()]}, + {0, [crashed(1)]}, + {1, [started(1)]}, + {1, [added(), started(1)]}, + {2, [started(2), started(1)]}, + {2, [crashed(3), started(2), started(1)]} + ] + ]. longest_running_test() -> J0 = testjob([crashed()]), @@ -1084,7 +1058,6 @@ longest_running_test() -> ?assertEqual([J1, J2], Sort([J2, J1])), ?assertEqual([J0, J1, J2], Sort([J2, J1, J0])). - scheduler_test_() -> { setup, @@ -1131,9 +1104,8 @@ scheduler_test_() -> } }. - t_pending_jobs_simple() -> - ?_test(begin + ?_test(begin Job1 = oneshot(1), Job2 = oneshot(2), setup_jobs([Job2, Job1]), @@ -1143,9 +1115,8 @@ t_pending_jobs_simple() -> ?assertEqual([Job1, Job2], pending_jobs(3)) end). - t_pending_jobs_skip_crashed() -> - ?_test(begin + ?_test(begin Job = oneshot(1), Ts = os:timestamp(), History = [crashed(Ts), started(Ts) | Job#job.history], @@ -1158,9 +1129,8 @@ t_pending_jobs_skip_crashed() -> ?assertEqual([Job2, Job3], pending_jobs(3)) end). - t_pending_jobs_skip_running() -> - ?_test(begin + ?_test(begin Job1 = continuous(1), Job2 = continuous_running(2), Job3 = oneshot(3), @@ -1170,7 +1140,6 @@ t_pending_jobs_skip_running() -> ?assertEqual([Job1, Job3], pending_jobs(4)) end). - t_one_job_starts() -> ?_test(begin setup_jobs([oneshot(1)]), @@ -1179,7 +1148,6 @@ t_one_job_starts() -> ?assertEqual({1, 0}, run_stop_count()) end). - t_no_jobs_start_if_max_is_0() -> ?_test(begin setup_jobs([oneshot(1)]), @@ -1187,7 +1155,6 @@ t_no_jobs_start_if_max_is_0() -> ?assertEqual({0, 1}, run_stop_count()) end). - t_one_job_starts_if_max_is_1() -> ?_test(begin setup_jobs([oneshot(1), oneshot(2)]), @@ -1195,7 +1162,6 @@ t_one_job_starts_if_max_is_1() -> ?assertEqual({1, 1}, run_stop_count()) end). - t_max_churn_does_not_throttle_initial_start() -> ?_test(begin setup_jobs([oneshot(1), oneshot(2)]), @@ -1203,7 +1169,6 @@ t_max_churn_does_not_throttle_initial_start() -> ?assertEqual({2, 0}, run_stop_count()) end). - t_excess_oneshot_only_jobs() -> ?_test(begin setup_jobs([oneshot_running(1), oneshot_running(2)]), @@ -1214,7 +1179,6 @@ t_excess_oneshot_only_jobs() -> ?assertEqual({0, 2}, run_stop_count()) end). - t_excess_continuous_only_jobs() -> ?_test(begin setup_jobs([continuous_running(1), continuous_running(2)]), @@ -1225,7 +1189,6 @@ t_excess_continuous_only_jobs() -> ?assertEqual({0, 2}, run_stop_count()) end). - t_excess_prefer_continuous_first() -> ?_test(begin Jobs = [ @@ -1245,7 +1208,6 @@ t_excess_prefer_continuous_first() -> ?assertEqual({0, 1}, oneshot_run_stop_count()) end). - t_stop_oldest_first() -> ?_test(begin Jobs = [ @@ -1261,7 +1223,6 @@ t_stop_oldest_first() -> ?assertEqual([7], jobs_running()) end). - t_start_oldest_first() -> ?_test(begin setup_jobs([continuous(7), continuous(2), continuous(5)]), @@ -1275,7 +1236,6 @@ t_start_oldest_first() -> ?assertEqual([2], jobs_stopped()) end). - t_jobs_churn_even_if_not_all_max_jobs_are_running() -> ?_test(begin setup_jobs([ @@ -1288,9 +1248,8 @@ t_jobs_churn_even_if_not_all_max_jobs_are_running() -> ?assertEqual([7], jobs_stopped()) end). - t_jobs_dont_churn_if_there_are_available_running_slots() -> - ?_test(begin + ?_test(begin setup_jobs([ continuous_running(1), continuous_running(2) @@ -1301,9 +1260,8 @@ t_jobs_dont_churn_if_there_are_available_running_slots() -> ?assertEqual(0, meck:num_calls(couch_replicator_scheduler_sup, start_child, 1)) end). - t_start_only_pending_jobs_do_not_churn_existing_ones() -> - ?_test(begin + ?_test(begin setup_jobs([ continuous(1), continuous_running(2) @@ -1314,7 +1272,6 @@ t_start_only_pending_jobs_do_not_churn_existing_ones() -> ?assertEqual({2, 0}, run_stop_count()) end). - t_dont_stop_if_nothing_pending() -> ?_test(begin setup_jobs([continuous_running(1), continuous_running(2)]), @@ -1322,7 +1279,6 @@ t_dont_stop_if_nothing_pending() -> ?assertEqual({2, 0}, run_stop_count()) end). - t_max_churn_limits_number_of_rotated_jobs() -> ?_test(begin Jobs = [ @@ -1336,7 +1292,6 @@ t_max_churn_limits_number_of_rotated_jobs() -> ?assertEqual([2, 3], jobs_stopped()) end). - t_if_pending_less_than_running_start_all_pending() -> ?_test(begin Jobs = [ @@ -1351,7 +1306,6 @@ t_if_pending_less_than_running_start_all_pending() -> ?assertEqual([1, 2, 5], jobs_running()) end). - t_running_less_than_pending_swap_all_running() -> ?_test(begin Jobs = [ @@ -1366,7 +1320,6 @@ t_running_less_than_pending_swap_all_running() -> ?assertEqual([3, 4, 5], jobs_stopped()) end). - t_oneshot_dont_get_rotated() -> ?_test(begin setup_jobs([oneshot_running(1), continuous(2)]), @@ -1374,7 +1327,6 @@ t_oneshot_dont_get_rotated() -> ?assertEqual([1], jobs_running()) end). - t_rotate_continuous_only_if_mixed() -> ?_test(begin setup_jobs([continuous(1), oneshot_running(2), continuous_running(3)]), @@ -1382,7 +1334,6 @@ t_rotate_continuous_only_if_mixed() -> ?assertEqual([1, 2], jobs_running()) end). - t_oneshot_dont_get_starting_priority() -> ?_test(begin setup_jobs([continuous(1), oneshot(2), continuous_running(3)]), @@ -1390,7 +1341,6 @@ t_oneshot_dont_get_starting_priority() -> ?assertEqual([1], jobs_running()) end). - % This tested in other test cases, it is here to mainly make explicit a property % of one-shot replications -- they can starve other jobs if they "take control" % of all the available scheduler slots. @@ -1407,7 +1357,6 @@ t_oneshot_will_hog_the_scheduler() -> ?assertEqual([1, 2], jobs_running()) end). - t_if_excess_is_trimmed_rotation_still_happens() -> ?_test(begin Jobs = [ @@ -1420,12 +1369,11 @@ t_if_excess_is_trimmed_rotation_still_happens() -> ?assertEqual([1], jobs_running()) end). - t_if_transient_job_crashes_it_gets_removed() -> ?_test(begin Pid = mock_pid(), Rep = continuous_rep(), - Job = #job{ + Job = #job{ id = job1, pid = Pid, history = [added()], @@ -1434,17 +1382,18 @@ t_if_transient_job_crashes_it_gets_removed() -> setup_jobs([Job]), ?assertEqual(1, ets:info(?MODULE, size)), State = #state{max_history = 3, stats_pid = self()}, - {noreply, State} = handle_info({'DOWN', r1, process, Pid, failed}, - State), + {noreply, State} = handle_info( + {'DOWN', r1, process, Pid, failed}, + State + ), ?assertEqual(0, ets:info(?MODULE, size)) - end). - + end). t_if_permanent_job_crashes_it_stays_in_ets() -> ?_test(begin Pid = mock_pid(), Rep = continuous_rep(), - Job = #job{ + Job = #job{ id = job1, pid = Pid, history = [added()], @@ -1457,14 +1406,15 @@ t_if_permanent_job_crashes_it_stays_in_ets() -> max_history = 3, stats_pid = self() }, - {noreply, State} = handle_info({'DOWN', r1, process, Pid, failed}, - State), + {noreply, State} = handle_info( + {'DOWN', r1, process, Pid, failed}, + State + ), ?assertEqual(1, ets:info(?MODULE, size)), [Job1] = ets:lookup(?MODULE, job1), [Latest | _] = Job1#job.history, ?assertMatch({{crashed, failed}, _}, Latest) - end). - + end). t_existing_jobs() -> ?_test(begin @@ -1479,11 +1429,10 @@ t_existing_jobs() -> ?assertNot(existing_replication(NewRep#rep{options = []})) end). - t_job_summary_running() -> ?_test(begin Rep = rep(<<"s">>, <<"t">>), - Job = #job{ + Job = #job{ id = job1, pid = mock_pid(), history = [added()], @@ -1501,10 +1450,9 @@ t_job_summary_running() -> ?assertEqual({Stats}, proplists:get_value(info, Summary1)) end). - t_job_summary_pending() -> ?_test(begin - Job = #job{ + Job = #job{ id = job1, pid = undefined, history = [stopped(20), started(10), added()], @@ -1522,10 +1470,9 @@ t_job_summary_pending() -> ?assertEqual({Stats}, proplists:get_value(info, Summary1)) end). - t_job_summary_crashing_once() -> ?_test(begin - Job = #job{ + Job = #job{ id = job1, history = [crashed(?DEFAULT_HEALTH_THRESHOLD_SEC + 1), started(0)], rep = rep(<<"s">>, <<"t">>) @@ -1538,10 +1485,9 @@ t_job_summary_crashing_once() -> ?assertEqual(0, proplists:get_value(error_count, Summary)) end). - t_job_summary_crashing_many_times() -> ?_test(begin - Job = #job{ + Job = #job{ id = job1, history = [crashed(4), started(3), crashed(2), started(1)], rep = rep(<<"s">>, <<"t">>) @@ -1554,7 +1500,6 @@ t_job_summary_crashing_many_times() -> ?assertEqual(2, proplists:get_value(error_count, Summary)) end). - t_job_summary_proxy_fields() -> ?_test(begin Src = #httpdb{ @@ -1565,20 +1510,23 @@ t_job_summary_proxy_fields() -> url = "http://t", proxy_url = "socks5://u:p@tproxy:34" }, - Job = #job{ + Job = #job{ id = job1, history = [started(10), added()], rep = rep(Src, Tgt) }, setup_jobs([Job]), Summary = job_summary(job1, ?DEFAULT_HEALTH_THRESHOLD_SEC), - ?assertEqual(<<"http://u:*****@sproxy:12">>, - proplists:get_value(source_proxy, Summary)), - ?assertEqual(<<"socks5://u:*****@tproxy:34">>, - proplists:get_value(target_proxy, Summary)) + ?assertEqual( + <<"http://u:*****@sproxy:12">>, + proplists:get_value(source_proxy, Summary) + ), + ?assertEqual( + <<"socks5://u:*****@tproxy:34">>, + proplists:get_value(target_proxy, Summary) + ) end). - % Test helper functions setup_all() -> @@ -1595,14 +1543,11 @@ setup_all() -> meck:expect(couch_replicator_scheduler_sup, start_child, 1, {ok, Pid}), couch_replicator_share:init(). - - teardown_all(_) -> couch_replicator_share:clear(), catch ets:delete(?MODULE), meck:unload(). - setup() -> meck:reset([ couch_log, @@ -1611,40 +1556,40 @@ setup() -> config ]). - teardown(_) -> ok. - setup_jobs(Jobs) when is_list(Jobs) -> ?MODULE = ets:new(?MODULE, [named_table, {keypos, #job.id}]), ets:insert(?MODULE, Jobs). - all_jobs() -> lists:usort(ets:tab2list(?MODULE)). - jobs_stopped() -> [Job#job.id || Job <- all_jobs(), Job#job.pid =:= undefined]. - jobs_running() -> [Job#job.id || Job <- all_jobs(), Job#job.pid =/= undefined]. - run_stop_count() -> {length(jobs_running()), length(jobs_stopped())}. - oneshot_run_stop_count() -> - Running = [Job#job.id || Job <- all_jobs(), Job#job.pid =/= undefined, - not is_continuous(Job)], - Stopped = [Job#job.id || Job <- all_jobs(), Job#job.pid =:= undefined, - not is_continuous(Job)], + Running = [ + Job#job.id + || Job <- all_jobs(), + Job#job.pid =/= undefined, + not is_continuous(Job) + ], + Stopped = [ + Job#job.id + || Job <- all_jobs(), + Job#job.pid =:= undefined, + not is_continuous(Job) + ], {length(Running), length(Stopped)}. - mock_state(MaxJobs) -> #state{ max_jobs = MaxJobs, @@ -1661,35 +1606,29 @@ mock_state(MaxJobs, MaxChurn) -> stats_pid = self() }. - rep() -> #rep{options = [], user_ctx = #user_ctx{}}. - rep(Src, Tgt) -> Rep = rep(), Rep#rep{source = Src, target = Tgt}. - continuous_rep() -> #rep{options = [{continuous, true}], user_ctx = #user_ctx{}}. - continuous_rep(Src, Tgt) -> Rep = continuous_rep(), Rep#rep{source = Src, target = Tgt}. - continuous(Id) when is_integer(Id) -> Started = Id, - Hist = [stopped(Started+1), started(Started), added()], + Hist = [stopped(Started + 1), started(Started), added()], #job{ id = Id, history = Hist, rep = continuous_rep() }. - continuous_running(Id) when is_integer(Id) -> Started = Id, Pid = mock_pid(), @@ -1701,13 +1640,11 @@ continuous_running(Id) when is_integer(Id) -> monitor = monitor(process, Pid) }. - oneshot(Id) when is_integer(Id) -> Started = Id, Hist = [stopped(Started + 1), started(Started), added()], #job{id = Id, history = Hist, rep = rep()}. - oneshot_running(Id) when is_integer(Id) -> Started = Id, Pid = mock_pid(), @@ -1719,43 +1656,34 @@ oneshot_running(Id) when is_integer(Id) -> monitor = monitor(process, Pid) }. - testjob(Hist) when is_list(Hist) -> #job{history = Hist}. - mock_pid() -> - list_to_pid("<0.999.999>"). + list_to_pid("<0.999.999>"). crashed() -> crashed(0). - -crashed(WhenSec) when is_integer(WhenSec)-> +crashed(WhenSec) when is_integer(WhenSec) -> {{crashed, some_reason}, {0, WhenSec, 0}}; crashed({MSec, Sec, USec}) -> {{crashed, some_reason}, {MSec, Sec, USec}}. - started() -> started(0). - -started(WhenSec) when is_integer(WhenSec)-> +started(WhenSec) when is_integer(WhenSec) -> {started, {0, WhenSec, 0}}; - started({MSec, Sec, USec}) -> {started, {MSec, Sec, USec}}. - stopped() -> stopped(0). - stopped(WhenSec) -> {stopped, {0, WhenSec, 0}}. - added() -> {added, {0, 0, 0}}. diff --git a/src/couch_replicator/src/couch_replicator_scheduler_job.erl b/src/couch_replicator/src/couch_replicator_scheduler_job.erl index db8edfbef..f300dd44e 100644 --- a/src/couch_replicator/src/couch_replicator_scheduler_job.erl +++ b/src/couch_replicator/src/couch_replicator_scheduler_job.erl @@ -15,17 +15,17 @@ -behaviour(gen_server). -export([ - start_link/1 + start_link/1 ]). -export([ - init/1, - terminate/2, - handle_call/3, - handle_info/2, - handle_cast/2, - code_change/3, - format_status/2 + init/1, + terminate/2, + handle_call/3, + handle_info/2, + handle_cast/2, + code_change/3, + format_status/2 ]). -include_lib("couch/include/couch_db.hrl"). @@ -42,7 +42,6 @@ pp_rep_id/1 ]). - -define(LOWEST_SEQ, 0). -define(DEFAULT_CHECKPOINT_INTERVAL, 30000). -define(STARTUP_JITTER_DEFAULT, 5000). @@ -65,7 +64,8 @@ rep_starttime, src_starttime, tgt_starttime, - timer, % checkpoint timer + % checkpoint timer + timer, changes_queue, changes_manager, changes_reader, @@ -79,7 +79,6 @@ view = nil }). - start_link(#rep{id = {BaseId, Ext}, source = Src, target = Tgt} = Rep) -> RepChildId = BaseId ++ Ext, Source = couch_replicator_api_wrap:db_uri(Src), @@ -90,17 +89,17 @@ start_link(#rep{id = {BaseId, Ext}, source = Src, target = Tgt} = Rep) -> {ok, Pid} -> {ok, Pid}; {error, Reason} -> - couch_log:warning("failed to start replication `~s` (`~s` -> `~s`)", - [RepChildId, Source, Target]), + couch_log:warning( + "failed to start replication `~s` (`~s` -> `~s`)", + [RepChildId, Source, Target] + ), {error, Reason} end. - init(InitArgs) -> {ok, InitArgs, 0}. - -do_init(#rep{options = Options, id = {BaseId, Ext}, user_ctx=UserCtx} = Rep) -> +do_init(#rep{options = Options, id = {BaseId, Ext}, user_ctx = UserCtx} = Rep) -> process_flag(trap_exit, true), timer:sleep(startup_jitter()), @@ -137,23 +136,27 @@ do_init(#rep{options = Options, id = {BaseId, Ext}, user_ctx=UserCtx} = Rep) -> fun(_) -> couch_stats:increment_counter([couch_replicator, workers_started]), {ok, Pid} = couch_replicator_worker:start_link( - self(), Source, Target, ChangesManager, MaxConns), + self(), Source, Target, ChangesManager, MaxConns + ), Pid end, - lists:seq(1, NumWorkers)), - - couch_task_status:add_task([ - {type, replication}, - {user, UserCtx#user_ctx.name}, - {replication_id, ?l2b(BaseId ++ Ext)}, - {database, Rep#rep.db_name}, - {doc_id, Rep#rep.doc_id}, - {source, ?l2b(SourceName)}, - {target, ?l2b(TargetName)}, - {continuous, get_value(continuous, Options, false)}, - {source_seq, HighestSeq}, - {checkpoint_interval, CheckpointInterval} - ] ++ rep_stats(State)), + lists:seq(1, NumWorkers) + ), + + couch_task_status:add_task( + [ + {type, replication}, + {user, UserCtx#user_ctx.name}, + {replication_id, ?l2b(BaseId ++ Ext)}, + {database, Rep#rep.db_name}, + {doc_id, Rep#rep.doc_id}, + {source, ?l2b(SourceName)}, + {target, ?l2b(TargetName)}, + {continuous, get_value(continuous, Options, false)}, + {source_seq, HighestSeq}, + {checkpoint_interval, CheckpointInterval} + ] ++ rep_stats(State) + ), couch_task_status:set_update_frequency(1000), % Until OTP R14B03: @@ -172,44 +175,59 @@ do_init(#rep{options = Options, id = {BaseId, Ext}, user_ctx=UserCtx} = Rep) -> doc_update_triggered(Rep), {ok, State#rep_state{ - changes_queue = ChangesQueue, - changes_manager = ChangesManager, - changes_reader = ChangesReader, - workers = Workers - } - }. - + changes_queue = ChangesQueue, + changes_manager = ChangesManager, + changes_reader = ChangesReader, + workers = Workers + }}. handle_call({add_stats, Stats}, From, State) -> gen_server:reply(From, ok), NewStats = couch_replicator_utils:sum_stats(State#rep_state.stats, Stats), {noreply, State#rep_state{stats = NewStats}}; - -handle_call({report_seq_done, Seq, StatsInc}, From, - #rep_state{seqs_in_progress = SeqsInProgress, highest_seq_done = HighestDone, - current_through_seq = ThroughSeq, stats = Stats} = State) -> +handle_call( + {report_seq_done, Seq, StatsInc}, + From, + #rep_state{ + seqs_in_progress = SeqsInProgress, + highest_seq_done = HighestDone, + current_through_seq = ThroughSeq, + stats = Stats + } = State +) -> gen_server:reply(From, ok), - {NewThroughSeq0, NewSeqsInProgress} = case SeqsInProgress of - [] -> - {Seq, []}; - [Seq | Rest] -> - {Seq, Rest}; - [_ | _] -> - {ThroughSeq, ordsets:del_element(Seq, SeqsInProgress)} - end, + {NewThroughSeq0, NewSeqsInProgress} = + case SeqsInProgress of + [] -> + {Seq, []}; + [Seq | Rest] -> + {Seq, Rest}; + [_ | _] -> + {ThroughSeq, ordsets:del_element(Seq, SeqsInProgress)} + end, NewHighestDone = lists:max([HighestDone, Seq]), - NewThroughSeq = case NewSeqsInProgress of - [] -> - lists:max([NewThroughSeq0, NewHighestDone]); - _ -> - NewThroughSeq0 - end, - couch_log:debug("Worker reported seq ~p, through seq was ~p, " + NewThroughSeq = + case NewSeqsInProgress of + [] -> + lists:max([NewThroughSeq0, NewHighestDone]); + _ -> + NewThroughSeq0 + end, + couch_log:debug( + "Worker reported seq ~p, through seq was ~p, " "new through seq is ~p, highest seq done was ~p, " "new highest seq done is ~p~n" "Seqs in progress were: ~p~nSeqs in progress are now: ~p", - [Seq, ThroughSeq, NewThroughSeq, HighestDone, - NewHighestDone, SeqsInProgress, NewSeqsInProgress]), + [ + Seq, + ThroughSeq, + NewThroughSeq, + HighestDone, + NewHighestDone, + SeqsInProgress, + NewSeqsInProgress + ] + ), NewState = State#rep_state{ stats = couch_replicator_utils:sum_stats(Stats, StatsInc), current_through_seq = NewThroughSeq, @@ -219,23 +237,22 @@ handle_call({report_seq_done, Seq, StatsInc}, From, update_task(NewState), {noreply, NewState}. - handle_cast(checkpoint, State) -> case do_checkpoint(State) of - {ok, NewState} -> - couch_stats:increment_counter([couch_replicator, checkpoints, success]), - {noreply, NewState#rep_state{timer = start_timer(State)}}; - Error -> - couch_stats:increment_counter([couch_replicator, checkpoints, failure]), - {stop, Error, State} + {ok, NewState} -> + couch_stats:increment_counter([couch_replicator, checkpoints, success]), + {noreply, NewState#rep_state{timer = start_timer(State)}}; + Error -> + couch_stats:increment_counter([couch_replicator, checkpoints, failure]), + {stop, Error, State} end; - -handle_cast({report_seq, Seq}, - #rep_state{seqs_in_progress = SeqsInProgress} = State) -> +handle_cast( + {report_seq, Seq}, + #rep_state{seqs_in_progress = SeqsInProgress} = State +) -> NewSeqsInProgress = ordsets:add_element(Seq, SeqsInProgress), {noreply, State#rep_state{seqs_in_progress = NewSeqsInProgress}}. - handle_info(shutdown, St) -> {stop, shutdown, St}; @@ -329,34 +346,36 @@ handle_info(timeout, InitArgs) -> {stop, {shutdown, ShutdownReason}, ShutdownState} end. - -terminate(normal, #rep_state{rep_details = #rep{id = RepId} = Rep, - checkpoint_history = CheckpointHistory} = State) -> +terminate( + normal, + #rep_state{ + rep_details = #rep{id = RepId} = Rep, + checkpoint_history = CheckpointHistory + } = State +) -> terminate_cleanup(State), couch_replicator_notifier:notify({finished, RepId, CheckpointHistory}), doc_update_completed(Rep, rep_stats(State)); - terminate(shutdown, #rep_state{rep_details = #rep{id = RepId}} = State) -> % Replication stopped via _scheduler_sup:terminate_child/1, which can be % occur during regular scheduler operation or when job is removed from % the scheduler. - State1 = case do_checkpoint(State) of - {ok, NewState} -> - NewState; - Error -> - LogMsg = "~p : Failed last checkpoint. Job: ~p Error: ~p", - couch_log:error(LogMsg, [?MODULE, RepId, Error]), - State - end, + State1 = + case do_checkpoint(State) of + {ok, NewState} -> + NewState; + Error -> + LogMsg = "~p : Failed last checkpoint. Job: ~p Error: ~p", + couch_log:error(LogMsg, [?MODULE, RepId, Error]), + State + end, couch_replicator_notifier:notify({stopped, RepId, <<"stopped">>}), terminate_cleanup(State1); - terminate({shutdown, max_backoff}, {error, InitArgs}) -> #rep{id = {BaseId, Ext} = RepId} = InitArgs, couch_stats:increment_counter([couch_replicator, failed_starts]), couch_log:warning("Replication `~s` reached max backoff ", [BaseId ++ Ext]), couch_replicator_notifier:notify({error, RepId, max_backoff}); - terminate({shutdown, {error, Error}}, {error, Class, Stack, InitArgs}) -> #rep{ id = {BaseId, Ext} = RepId, @@ -369,35 +388,44 @@ terminate({shutdown, {error, Error}}, {error, Class, Stack, InitArgs}) -> Target = couch_replicator_api_wrap:db_uri(Target0), RepIdStr = BaseId ++ Ext, Msg = "~p:~p: Replication ~s failed to start ~p -> ~p doc ~p:~p stack:~p", - couch_log:error(Msg, [Class, Error, RepIdStr, Source, Target, DbName, - DocId, Stack]), + couch_log:error(Msg, [ + Class, + Error, + RepIdStr, + Source, + Target, + DbName, + DocId, + Stack + ]), couch_stats:increment_counter([couch_replicator, failed_starts]), couch_replicator_notifier:notify({error, RepId, Error}); - terminate({shutdown, max_backoff}, State) -> #rep_state{ source_name = Source, target_name = Target, rep_details = #rep{id = {BaseId, Ext} = RepId} } = State, - couch_log:error("Replication `~s` (`~s` -> `~s`) reached max backoff", - [BaseId ++ Ext, Source, Target]), + couch_log:error( + "Replication `~s` (`~s` -> `~s`) reached max backoff", + [BaseId ++ Ext, Source, Target] + ), terminate_cleanup(State), couch_replicator_notifier:notify({error, RepId, max_backoff}); - terminate({shutdown, Reason}, State) -> % Unwrap so when reporting we don't have an extra {shutdown, ...} tuple % wrapped around the message terminate(Reason, State); - terminate(Reason, State) -> -#rep_state{ + #rep_state{ source_name = Source, target_name = Target, rep_details = #rep{id = {BaseId, Ext} = RepId} } = State, - couch_log:error("Replication `~s` (`~s` -> `~s`) failed: ~s", - [BaseId ++ Ext, Source, Target, to_binary(Reason)]), + couch_log:error( + "Replication `~s` (`~s` -> `~s`) failed: ~s", + [BaseId ++ Ext, Source, Target, to_binary(Reason)] + ), terminate_cleanup(State), couch_replicator_notifier:notify({error, RepId, Reason}). @@ -406,28 +434,26 @@ terminate_cleanup(State) -> couch_replicator_api_wrap:db_close(State#rep_state.source), couch_replicator_api_wrap:db_close(State#rep_state.target). - -code_change(_OldVsn, #rep_state{}=State, _Extra) -> +code_change(_OldVsn, #rep_state{} = State, _Extra) -> {ok, State}. - format_status(_Opt, [_PDict, State]) -> #rep_state{ - source = Source, - target = Target, - rep_details = RepDetails, - start_seq = StartSeq, - source_seq = SourceSeq, - committed_seq = CommitedSeq, - current_through_seq = ThroughSeq, - highest_seq_done = HighestSeqDone, - session_id = SessionId + source = Source, + target = Target, + rep_details = RepDetails, + start_seq = StartSeq, + source_seq = SourceSeq, + committed_seq = CommitedSeq, + current_through_seq = ThroughSeq, + highest_seq_done = HighestSeqDone, + session_id = SessionId } = state_strip_creds(State), #rep{ - id = RepId, - options = Options, - doc_id = DocId, - db_name = DbName + id = RepId, + options = Options, + doc_id = DocId, + db_name = DbName } = RepDetails, [ {rep_id, RepId}, @@ -444,25 +470,26 @@ format_status(_Opt, [_PDict, State]) -> {highest_seq_done, HighestSeqDone} ]. - startup_jitter() -> - Jitter = config:get_integer("replicator", "startup_jitter", - ?STARTUP_JITTER_DEFAULT), + Jitter = config:get_integer( + "replicator", + "startup_jitter", + ?STARTUP_JITTER_DEFAULT + ), couch_rand:uniform(erlang:max(1, Jitter)). - headers_strip_creds([], Acc) -> lists:reverse(Acc); headers_strip_creds([{Key, Value0} | Rest], Acc) -> - Value = case string:to_lower(Key) of - "authorization" -> - "****"; - _ -> - Value0 - end, + Value = + case string:to_lower(Key) of + "authorization" -> + "****"; + _ -> + Value0 + end, headers_strip_creds(Rest, [{Key, Value} | Acc]). - httpdb_strip_creds(#httpdb{url = Url, headers = Headers} = HttpDb) -> HttpDb#httpdb{ url = couch_util:url_strip_password(Url), @@ -471,14 +498,12 @@ httpdb_strip_creds(#httpdb{url = Url, headers = Headers} = HttpDb) -> httpdb_strip_creds(LocalDb) -> LocalDb. - rep_strip_creds(#rep{source = Source, target = Target} = Rep) -> Rep#rep{ source = httpdb_strip_creds(Source), target = httpdb_strip_creds(Target) }. - state_strip_creds(#rep_state{rep_details = Rep, source = Source, target = Target} = State) -> % #rep_state contains the source and target at the top level and also % in the nested #rep_details record @@ -488,7 +513,6 @@ state_strip_creds(#rep_state{rep_details = Rep, source = Source, target = Target target = httpdb_strip_creds(Target) }. - adjust_maxconn(Src = #httpdb{http_connections = 1}, RepId) -> Msg = "Adjusting minimum number of HTTP source connections to 2 for ~p", couch_log:notice(Msg, [RepId]), @@ -496,7 +520,6 @@ adjust_maxconn(Src = #httpdb{http_connections = 1}, RepId) -> adjust_maxconn(Src, _RepId) -> Src. - -spec doc_update_triggered(#rep{}) -> ok. doc_update_triggered(#rep{db_name = null}) -> ok; @@ -507,62 +530,78 @@ doc_update_triggered(#rep{id = RepId, doc_id = DocId} = Rep) -> false -> ok end, - couch_log:notice("Document `~s` triggered replication `~s`", - [DocId, pp_rep_id(RepId)]), + couch_log:notice( + "Document `~s` triggered replication `~s`", + [DocId, pp_rep_id(RepId)] + ), ok. - -spec doc_update_completed(#rep{}, list()) -> ok. doc_update_completed(#rep{db_name = null}, _Stats) -> ok; -doc_update_completed(#rep{id = RepId, doc_id = DocId, db_name = DbName, - start_time = StartTime}, Stats0) -> +doc_update_completed( + #rep{ + id = RepId, + doc_id = DocId, + db_name = DbName, + start_time = StartTime + }, + Stats0 +) -> Stats = Stats0 ++ [{start_time, couch_replicator_utils:iso8601(StartTime)}], couch_replicator_docs:update_doc_completed(DbName, DocId, Stats), - couch_log:notice("Replication `~s` completed (triggered by `~s`)", - [pp_rep_id(RepId), DocId]), + couch_log:notice( + "Replication `~s` completed (triggered by `~s`)", + [pp_rep_id(RepId), DocId] + ), ok. - -do_last_checkpoint(#rep_state{seqs_in_progress = [], - highest_seq_done = {_Ts, ?LOWEST_SEQ}} = State) -> +do_last_checkpoint( + #rep_state{ + seqs_in_progress = [], + highest_seq_done = {_Ts, ?LOWEST_SEQ} + } = State +) -> {stop, normal, cancel_timer(State)}; -do_last_checkpoint(#rep_state{seqs_in_progress = [], - highest_seq_done = Seq} = State) -> +do_last_checkpoint( + #rep_state{ + seqs_in_progress = [], + highest_seq_done = Seq + } = State +) -> case do_checkpoint(State#rep_state{current_through_seq = Seq}) of - {ok, NewState} -> - couch_stats:increment_counter([couch_replicator, checkpoints, success]), - {stop, normal, cancel_timer(NewState)}; - Error -> - couch_stats:increment_counter([couch_replicator, checkpoints, failure]), - {stop, Error, State} + {ok, NewState} -> + couch_stats:increment_counter([couch_replicator, checkpoints, success]), + {stop, normal, cancel_timer(NewState)}; + Error -> + couch_stats:increment_counter([couch_replicator, checkpoints, failure]), + {stop, Error, State} end. - start_timer(State) -> After = State#rep_state.checkpoint_interval, case timer:apply_after(After, gen_server, cast, [self(), checkpoint]) of - {ok, Ref} -> - Ref; - Error -> - couch_log:error("Replicator, error scheduling checkpoint: ~p", [Error]), - nil + {ok, Ref} -> + Ref; + Error -> + couch_log:error("Replicator, error scheduling checkpoint: ~p", [Error]), + nil end. - cancel_timer(#rep_state{timer = nil} = State) -> State; cancel_timer(#rep_state{timer = Timer} = State) -> {ok, cancel} = timer:cancel(Timer), State#rep_state{timer = nil}. - init_state(Rep) -> #rep{ id = {BaseId, _Ext}, - source = Src0, target = Tgt, + source = Src0, + target = Tgt, options = Options, - type = Type, view = View, + type = Type, + view = View, start_time = StartTime, stats = ArgStats0 } = Rep, @@ -570,8 +609,11 @@ init_state(Rep) -> Src = adjust_maxconn(Src0, BaseId), {ok, Source} = couch_replicator_api_wrap:db_open(Src), {CreateTargetParams} = get_value(create_target_params, Options, {[]}), - {ok, Target} = couch_replicator_api_wrap:db_open(Tgt, - get_value(create_target, Options, false), CreateTargetParams), + {ok, Target} = couch_replicator_api_wrap:db_open( + Tgt, + get_value(create_target, Options, false), + CreateTargetParams + ), {ok, SourceInfo} = couch_replicator_api_wrap:get_db_info(Source), {ok, TargetInfo} = couch_replicator_api_wrap:get_db_info(Target), @@ -581,10 +623,11 @@ init_state(Rep) -> {StartSeq0, History} = compare_replication_logs(SourceLog, TargetLog), ArgStats1 = couch_replicator_stats:new(ArgStats0), - HistoryStats = case History of - [{[_ | _] = HProps} | _] -> couch_replicator_stats:new(HProps); - _ -> couch_replicator_stats:new() - end, + HistoryStats = + case History of + [{[_ | _] = HProps} | _] -> couch_replicator_stats:new(HProps); + _ -> couch_replicator_stats:new() + end, Stats = couch_replicator_stats:max_stats(ArgStats1, HistoryStats), StartSeq1 = get_value(since_seq, Options, StartSeq0), @@ -592,7 +635,7 @@ init_state(Rep) -> SourceSeq = get_value(<<"update_seq">>, SourceInfo, ?LOWEST_SEQ), - #doc{body={CheckpointHistory}} = SourceLog, + #doc{body = {CheckpointHistory}} = SourceLog, State = #rep_state{ rep_details = Rep, source_name = couch_replicator_api_wrap:db_uri(Source), @@ -600,7 +643,7 @@ init_state(Rep) -> source = Source, target = Target, history = History, - checkpoint_history = {[{<<"no_changes">>, true}| CheckpointHistory]}, + checkpoint_history = {[{<<"no_changes">>, true} | CheckpointHistory]}, start_seq = StartSeq, current_through_seq = StartSeq, committed_seq = StartSeq, @@ -612,43 +655,51 @@ init_state(Rep) -> session_id = couch_uuids:random(), source_seq = SourceSeq, use_checkpoints = get_value(use_checkpoints, Options, true), - checkpoint_interval = get_value(checkpoint_interval, Options, - ?DEFAULT_CHECKPOINT_INTERVAL), + checkpoint_interval = get_value( + checkpoint_interval, + Options, + ?DEFAULT_CHECKPOINT_INTERVAL + ), type = Type, view = View, stats = Stats }, State#rep_state{timer = start_timer(State)}. - find_and_migrate_logs(DbList, #rep{id = {BaseId, _}} = Rep) -> LogId = ?l2b(?LOCAL_DOC_PREFIX ++ BaseId), fold_replication_logs(DbList, ?REP_ID_VERSION, LogId, LogId, Rep, []). - fold_replication_logs([], _Vsn, _LogId, _NewId, _Rep, Acc) -> lists:reverse(Acc); - fold_replication_logs([Db | Rest] = Dbs, Vsn, LogId, NewId, Rep, Acc) -> case couch_replicator_api_wrap:open_doc(Db, LogId, [ejson_body]) of - {error, <<"not_found">>} when Vsn > 1 -> - OldRepId = couch_replicator_utils:replication_id(Rep, Vsn - 1), - fold_replication_logs(Dbs, Vsn - 1, - ?l2b(?LOCAL_DOC_PREFIX ++ OldRepId), NewId, Rep, Acc); - {error, <<"not_found">>} -> - fold_replication_logs( - Rest, ?REP_ID_VERSION, NewId, NewId, Rep, [#doc{id = NewId} | Acc]); - {ok, Doc} when LogId =:= NewId -> - fold_replication_logs( - Rest, ?REP_ID_VERSION, NewId, NewId, Rep, [Doc | Acc]); - {ok, Doc} -> - MigratedLog = #doc{id = NewId, body = Doc#doc.body}, - maybe_save_migrated_log(Rep, Db, MigratedLog, Doc#doc.id), - fold_replication_logs( - Rest, ?REP_ID_VERSION, NewId, NewId, Rep, [MigratedLog | Acc]) + {error, <<"not_found">>} when Vsn > 1 -> + OldRepId = couch_replicator_utils:replication_id(Rep, Vsn - 1), + fold_replication_logs( + Dbs, + Vsn - 1, + ?l2b(?LOCAL_DOC_PREFIX ++ OldRepId), + NewId, + Rep, + Acc + ); + {error, <<"not_found">>} -> + fold_replication_logs( + Rest, ?REP_ID_VERSION, NewId, NewId, Rep, [#doc{id = NewId} | Acc] + ); + {ok, Doc} when LogId =:= NewId -> + fold_replication_logs( + Rest, ?REP_ID_VERSION, NewId, NewId, Rep, [Doc | Acc] + ); + {ok, Doc} -> + MigratedLog = #doc{id = NewId, body = Doc#doc.body}, + maybe_save_migrated_log(Rep, Db, MigratedLog, Doc#doc.id), + fold_replication_logs( + Rest, ?REP_ID_VERSION, NewId, NewId, Rep, [MigratedLog | Acc] + ) end. - maybe_save_migrated_log(Rep, Db, #doc{} = Doc, OldId) -> case get_value(use_checkpoints, Rep#rep.options, true) of true -> @@ -659,49 +710,50 @@ maybe_save_migrated_log(Rep, Db, #doc{} = Doc, OldId) -> ok end. - spawn_changes_manager(Parent, ChangesQueue, BatchSize) -> spawn_link(fun() -> changes_manager_loop_open(Parent, ChangesQueue, BatchSize, 1) end). - changes_manager_loop_open(Parent, ChangesQueue, BatchSize, Ts) -> receive - {get_changes, From} -> - case couch_work_queue:dequeue(ChangesQueue, BatchSize) of - closed -> - From ! {closed, self()}; - {ok, ChangesOrLastSeqs} -> - ReportSeq = case lists:last(ChangesOrLastSeqs) of - {last_seq, Seq} -> - {Ts, Seq}; - #doc_info{high_seq = Seq} -> - {Ts, Seq} + {get_changes, From} -> + case couch_work_queue:dequeue(ChangesQueue, BatchSize) of + closed -> + From ! {closed, self()}; + {ok, ChangesOrLastSeqs} -> + ReportSeq = + case lists:last(ChangesOrLastSeqs) of + {last_seq, Seq} -> + {Ts, Seq}; + #doc_info{high_seq = Seq} -> + {Ts, Seq} + end, + Changes = lists:filter( + fun + (#doc_info{}) -> + true; + ({last_seq, _Seq}) -> + false + end, + ChangesOrLastSeqs + ), + ok = gen_server:cast(Parent, {report_seq, ReportSeq}), + From ! {changes, self(), Changes, ReportSeq} end, - Changes = lists:filter( - fun(#doc_info{}) -> - true; - ({last_seq, _Seq}) -> - false - end, ChangesOrLastSeqs), - ok = gen_server:cast(Parent, {report_seq, ReportSeq}), - From ! {changes, self(), Changes, ReportSeq} - end, - changes_manager_loop_open(Parent, ChangesQueue, BatchSize, Ts + 1) + changes_manager_loop_open(Parent, ChangesQueue, BatchSize, Ts + 1) end. - -do_checkpoint(#rep_state{use_checkpoints=false} = State) -> - NewState = State#rep_state{checkpoint_history = {[{<<"use_checkpoints">>, false}]} }, +do_checkpoint(#rep_state{use_checkpoints = false} = State) -> + NewState = State#rep_state{checkpoint_history = {[{<<"use_checkpoints">>, false}]}}, {ok, NewState}; -do_checkpoint(#rep_state{current_through_seq=Seq, committed_seq=Seq} = State) -> +do_checkpoint(#rep_state{current_through_seq = Seq, committed_seq = Seq} = State) -> update_task(State), {ok, State}; do_checkpoint(State) -> #rep_state{ - source_name=SourceName, - target_name=TargetName, + source_name = SourceName, + target_name = TargetName, source = Source, target = Target, history = OldHistory, @@ -717,119 +769,135 @@ do_checkpoint(State) -> session_id = SessionId } = State, case commit_to_both(Source, Target) of - {source_error, Reason} -> - {checkpoint_commit_failure, - <<"Failure on source commit: ", (to_binary(Reason))/binary>>}; - {target_error, Reason} -> - {checkpoint_commit_failure, - <<"Failure on target commit: ", (to_binary(Reason))/binary>>}; - {SrcInstanceStartTime, TgtInstanceStartTime} -> - couch_log:notice("recording a checkpoint for `~s` -> `~s` at source update_seq ~p", - [SourceName, TargetName, NewSeq]), - LocalStartTime = calendar:now_to_local_time(ReplicationStartTime), - StartTime = ?l2b(httpd_util:rfc1123_date(LocalStartTime)), - EndTime = ?l2b(httpd_util:rfc1123_date()), - NewHistoryEntry = {[ - {<<"session_id">>, SessionId}, - {<<"start_time">>, StartTime}, - {<<"end_time">>, EndTime}, - {<<"start_last_seq">>, StartSeq}, - {<<"end_last_seq">>, NewSeq}, - {<<"recorded_seq">>, NewSeq}, - {<<"missing_checked">>, couch_replicator_stats:missing_checked(Stats)}, - {<<"missing_found">>, couch_replicator_stats:missing_found(Stats)}, - {<<"docs_read">>, couch_replicator_stats:docs_read(Stats)}, - {<<"docs_written">>, couch_replicator_stats:docs_written(Stats)}, - {<<"doc_write_failures">>, couch_replicator_stats:doc_write_failures(Stats)} - ]}, - BaseHistory = [ - {<<"session_id">>, SessionId}, - {<<"source_last_seq">>, NewSeq}, - {<<"replication_id_version">>, ?REP_ID_VERSION} - ] ++ case get_value(doc_ids, Options) of - undefined -> - []; - _DocIds -> - % backwards compatibility with the result of a replication by - % doc IDs in versions 0.11.x and 1.0.x - % TODO: deprecate (use same history format, simplify code) - [ - {<<"start_time">>, StartTime}, - {<<"end_time">>, EndTime}, - {<<"docs_read">>, couch_replicator_stats:docs_read(Stats)}, - {<<"docs_written">>, couch_replicator_stats:docs_written(Stats)}, - {<<"doc_write_failures">>, couch_replicator_stats:doc_write_failures(Stats)} - ] - end, - % limit history to 50 entries - NewRepHistory = { - BaseHistory ++ - [{<<"history">>, lists:sublist([NewHistoryEntry | OldHistory], 50)}] - }, - - try - {SrcRevPos, SrcRevId} = update_checkpoint( - Source, SourceLog#doc{body = NewRepHistory}, source), - {TgtRevPos, TgtRevId} = update_checkpoint( - Target, TargetLog#doc{body = NewRepHistory}, target), - NewState = State#rep_state{ - checkpoint_history = NewRepHistory, - committed_seq = NewTsSeq, - source_log = SourceLog#doc{revs={SrcRevPos, [SrcRevId]}}, - target_log = TargetLog#doc{revs={TgtRevPos, [TgtRevId]}} + {source_error, Reason} -> + {checkpoint_commit_failure, + <<"Failure on source commit: ", (to_binary(Reason))/binary>>}; + {target_error, Reason} -> + {checkpoint_commit_failure, + <<"Failure on target commit: ", (to_binary(Reason))/binary>>}; + {SrcInstanceStartTime, TgtInstanceStartTime} -> + couch_log:notice( + "recording a checkpoint for `~s` -> `~s` at source update_seq ~p", + [SourceName, TargetName, NewSeq] + ), + LocalStartTime = calendar:now_to_local_time(ReplicationStartTime), + StartTime = ?l2b(httpd_util:rfc1123_date(LocalStartTime)), + EndTime = ?l2b(httpd_util:rfc1123_date()), + NewHistoryEntry = + {[ + {<<"session_id">>, SessionId}, + {<<"start_time">>, StartTime}, + {<<"end_time">>, EndTime}, + {<<"start_last_seq">>, StartSeq}, + {<<"end_last_seq">>, NewSeq}, + {<<"recorded_seq">>, NewSeq}, + {<<"missing_checked">>, couch_replicator_stats:missing_checked(Stats)}, + {<<"missing_found">>, couch_replicator_stats:missing_found(Stats)}, + {<<"docs_read">>, couch_replicator_stats:docs_read(Stats)}, + {<<"docs_written">>, couch_replicator_stats:docs_written(Stats)}, + {<<"doc_write_failures">>, couch_replicator_stats:doc_write_failures(Stats)} + ]}, + BaseHistory = + [ + {<<"session_id">>, SessionId}, + {<<"source_last_seq">>, NewSeq}, + {<<"replication_id_version">>, ?REP_ID_VERSION} + ] ++ + case get_value(doc_ids, Options) of + undefined -> + []; + _DocIds -> + % backwards compatibility with the result of a replication by + % doc IDs in versions 0.11.x and 1.0.x + % TODO: deprecate (use same history format, simplify code) + [ + {<<"start_time">>, StartTime}, + {<<"end_time">>, EndTime}, + {<<"docs_read">>, couch_replicator_stats:docs_read(Stats)}, + {<<"docs_written">>, couch_replicator_stats:docs_written(Stats)}, + {<<"doc_write_failures">>, + couch_replicator_stats:doc_write_failures(Stats)} + ] + end, + % limit history to 50 entries + NewRepHistory = { + BaseHistory ++ + [{<<"history">>, lists:sublist([NewHistoryEntry | OldHistory], 50)}] }, - update_task(NewState), - {ok, NewState} - catch throw:{checkpoint_commit_failure, _} = Failure -> - Failure - end; - {SrcInstanceStartTime, _NewTgtInstanceStartTime} -> - {checkpoint_commit_failure, <<"Target database out of sync. " - "Try to increase max_dbs_open at the target's server.">>}; - {_NewSrcInstanceStartTime, TgtInstanceStartTime} -> - {checkpoint_commit_failure, <<"Source database out of sync. " - "Try to increase max_dbs_open at the source's server.">>}; - {_NewSrcInstanceStartTime, _NewTgtInstanceStartTime} -> - {checkpoint_commit_failure, <<"Source and target databases out of " - "sync. Try to increase max_dbs_open at both servers.">>} - end. + try + {SrcRevPos, SrcRevId} = update_checkpoint( + Source, SourceLog#doc{body = NewRepHistory}, source + ), + {TgtRevPos, TgtRevId} = update_checkpoint( + Target, TargetLog#doc{body = NewRepHistory}, target + ), + NewState = State#rep_state{ + checkpoint_history = NewRepHistory, + committed_seq = NewTsSeq, + source_log = SourceLog#doc{revs = {SrcRevPos, [SrcRevId]}}, + target_log = TargetLog#doc{revs = {TgtRevPos, [TgtRevId]}} + }, + update_task(NewState), + {ok, NewState} + catch + throw:{checkpoint_commit_failure, _} = Failure -> + Failure + end; + {SrcInstanceStartTime, _NewTgtInstanceStartTime} -> + {checkpoint_commit_failure, << + "Target database out of sync. " + "Try to increase max_dbs_open at the target's server." + >>}; + {_NewSrcInstanceStartTime, TgtInstanceStartTime} -> + {checkpoint_commit_failure, << + "Source database out of sync. " + "Try to increase max_dbs_open at the source's server." + >>}; + {_NewSrcInstanceStartTime, _NewTgtInstanceStartTime} -> + {checkpoint_commit_failure, << + "Source and target databases out of " + "sync. Try to increase max_dbs_open at both servers." + >>} + end. update_checkpoint(Db, Doc, DbType) -> try update_checkpoint(Db, Doc) - catch throw:{checkpoint_commit_failure, Reason} -> - throw({checkpoint_commit_failure, - <<"Error updating the ", (to_binary(DbType))/binary, - " checkpoint document: ", (to_binary(Reason))/binary>>}) + catch + throw:{checkpoint_commit_failure, Reason} -> + throw( + {checkpoint_commit_failure, + <<"Error updating the ", (to_binary(DbType))/binary, " checkpoint document: ", + (to_binary(Reason))/binary>>} + ) end. - update_checkpoint(Db, #doc{id = LogId, body = LogBody} = Doc) -> try case couch_replicator_api_wrap:update_doc(Db, Doc, [delay_commit]) of - {ok, PosRevId} -> - PosRevId; - {error, Reason} -> - throw({checkpoint_commit_failure, Reason}) - end - catch throw:conflict -> - case (catch couch_replicator_api_wrap:open_doc(Db, LogId, [ejson_body])) of - {ok, #doc{body = LogBody, revs = {Pos, [RevId | _]}}} -> - % This means that we were able to update successfully the - % checkpoint doc in a previous attempt but we got a connection - % error (timeout for e.g.) before receiving the success response. - % Therefore the request was retried and we got a conflict, as the - % revision we sent is not the current one. - % We confirm this by verifying the doc body we just got is the same - % that we have just sent. - {Pos, RevId}; - _ -> - throw({checkpoint_commit_failure, conflict}) + {ok, PosRevId} -> + PosRevId; + {error, Reason} -> + throw({checkpoint_commit_failure, Reason}) end + catch + throw:conflict -> + case (catch couch_replicator_api_wrap:open_doc(Db, LogId, [ejson_body])) of + {ok, #doc{body = LogBody, revs = {Pos, [RevId | _]}}} -> + % This means that we were able to update successfully the + % checkpoint doc in a previous attempt but we got a connection + % error (timeout for e.g.) before receiving the success response. + % Therefore the request was retried and we got a conflict, as the + % revision we sent is not the current one. + % We confirm this by verifying the doc body we just got is the same + % that we have just sent. + {Pos, RevId}; + _ -> + throw({checkpoint_commit_failure, conflict}) + end end. - commit_to_both(Source, Target) -> % commit the src async ParentPid = self(), @@ -837,90 +905,102 @@ commit_to_both(Source, Target) -> fun() -> Result = (catch couch_replicator_api_wrap:ensure_full_commit(Source)), ParentPid ! {self(), Result} - end), + end + ), % commit tgt sync TargetResult = (catch couch_replicator_api_wrap:ensure_full_commit(Target)), - SourceResult = receive - {SrcCommitPid, Result} -> - unlink(SrcCommitPid), - receive {'EXIT', SrcCommitPid, _} -> ok after 0 -> ok end, - Result; - {'EXIT', SrcCommitPid, Reason} -> - {error, Reason} - end, + SourceResult = + receive + {SrcCommitPid, Result} -> + unlink(SrcCommitPid), + receive + {'EXIT', SrcCommitPid, _} -> ok + after 0 -> ok + end, + Result; + {'EXIT', SrcCommitPid, Reason} -> + {error, Reason} + end, case TargetResult of - {ok, TargetStartTime} -> - case SourceResult of - {ok, SourceStartTime} -> - {SourceStartTime, TargetStartTime}; - SourceError -> - {source_error, SourceError} - end; - TargetError -> - {target_error, TargetError} + {ok, TargetStartTime} -> + case SourceResult of + {ok, SourceStartTime} -> + {SourceStartTime, TargetStartTime}; + SourceError -> + {source_error, SourceError} + end; + TargetError -> + {target_error, TargetError} end. - compare_replication_logs(SrcDoc, TgtDoc) -> - #doc{body={RepRecProps}} = SrcDoc, - #doc{body={RepRecPropsTgt}} = TgtDoc, - case get_value(<<"session_id">>, RepRecProps) == - get_value(<<"session_id">>, RepRecPropsTgt) of - true -> - % if the records have the same session id, - % then we have a valid replication history - OldSeqNum = get_value(<<"source_last_seq">>, RepRecProps, ?LOWEST_SEQ), - OldHistory = get_value(<<"history">>, RepRecProps, []), - {OldSeqNum, OldHistory}; - false -> - SourceHistory = get_value(<<"history">>, RepRecProps, []), - TargetHistory = get_value(<<"history">>, RepRecPropsTgt, []), - couch_log:notice("Replication records differ. " - "Scanning histories to find a common ancestor.", []), - couch_log:debug("Record on source:~p~nRecord on target:~p~n", - [RepRecProps, RepRecPropsTgt]), - compare_rep_history(SourceHistory, TargetHistory) + #doc{body = {RepRecProps}} = SrcDoc, + #doc{body = {RepRecPropsTgt}} = TgtDoc, + case + get_value(<<"session_id">>, RepRecProps) == + get_value(<<"session_id">>, RepRecPropsTgt) + of + true -> + % if the records have the same session id, + % then we have a valid replication history + OldSeqNum = get_value(<<"source_last_seq">>, RepRecProps, ?LOWEST_SEQ), + OldHistory = get_value(<<"history">>, RepRecProps, []), + {OldSeqNum, OldHistory}; + false -> + SourceHistory = get_value(<<"history">>, RepRecProps, []), + TargetHistory = get_value(<<"history">>, RepRecPropsTgt, []), + couch_log:notice( + "Replication records differ. " + "Scanning histories to find a common ancestor.", + [] + ), + couch_log:debug( + "Record on source:~p~nRecord on target:~p~n", + [RepRecProps, RepRecPropsTgt] + ), + compare_rep_history(SourceHistory, TargetHistory) end. - compare_rep_history(S, T) when S =:= [] orelse T =:= [] -> couch_log:notice("no common ancestry -- performing full replication", []), {?LOWEST_SEQ, []}; compare_rep_history([{S} | SourceRest], [{T} | TargetRest] = Target) -> SourceId = get_value(<<"session_id">>, S), case has_session_id(SourceId, Target) of - true -> - RecordSeqNum = get_value(<<"recorded_seq">>, S, ?LOWEST_SEQ), - couch_log:notice("found a common replication record with source_seq ~p", - [RecordSeqNum]), - {RecordSeqNum, SourceRest}; - false -> - TargetId = get_value(<<"session_id">>, T), - case has_session_id(TargetId, SourceRest) of true -> - RecordSeqNum = get_value(<<"recorded_seq">>, T, ?LOWEST_SEQ), - couch_log:notice("found a common replication record with source_seq ~p", - [RecordSeqNum]), - {RecordSeqNum, TargetRest}; + RecordSeqNum = get_value(<<"recorded_seq">>, S, ?LOWEST_SEQ), + couch_log:notice( + "found a common replication record with source_seq ~p", + [RecordSeqNum] + ), + {RecordSeqNum, SourceRest}; false -> - compare_rep_history(SourceRest, TargetRest) - end + TargetId = get_value(<<"session_id">>, T), + case has_session_id(TargetId, SourceRest) of + true -> + RecordSeqNum = get_value(<<"recorded_seq">>, T, ?LOWEST_SEQ), + couch_log:notice( + "found a common replication record with source_seq ~p", + [RecordSeqNum] + ), + {RecordSeqNum, TargetRest}; + false -> + compare_rep_history(SourceRest, TargetRest) + end end. - has_session_id(_SessionId, []) -> false; has_session_id(SessionId, [{Props} | Rest]) -> case get_value(<<"session_id">>, Props, nil) of - SessionId -> - true; - _Else -> - has_session_id(SessionId, Rest) + SessionId -> + true; + _Else -> + has_session_id(SessionId, Rest) end. - get_pending_count(St) -> Rep = St#rep_state.rep_details, Timeout = get_value(connection_timeout, Rep#rep.options), @@ -941,36 +1021,35 @@ get_pending_count(St) -> NewPendingCount end. - -get_pending_count_int(#rep_state{source = #httpdb{} = Db0}=St) -> +get_pending_count_int(#rep_state{source = #httpdb{} = Db0} = St) -> {_, Seq} = St#rep_state.highest_seq_done, Db = Db0#httpdb{retries = 3}, case (catch couch_replicator_api_wrap:get_pending_count(Db, Seq)) of - {ok, Pending} -> - Pending; - _ -> - null + {ok, Pending} -> + Pending; + _ -> + null end; -get_pending_count_int(#rep_state{source = Db}=St) -> +get_pending_count_int(#rep_state{source = Db} = St) -> {_, Seq} = St#rep_state.highest_seq_done, {ok, Pending} = couch_replicator_api_wrap:get_pending_count(Db, Seq), Pending. - update_task(State) -> #rep_state{ rep_details = #rep{id = JobId}, current_through_seq = {_, ThroughSeq}, highest_seq_done = {_, HighestSeq} } = State, - Status = rep_stats(State) ++ [ - {source_seq, HighestSeq}, - {through_seq, ThroughSeq} - ], + Status = + rep_stats(State) ++ + [ + {source_seq, HighestSeq}, + {through_seq, ThroughSeq} + ], couch_replicator_scheduler:update_job_stats(JobId, Status), couch_task_status:update(Status). - rep_stats(State) -> #rep_state{ committed_seq = {_, CommittedSeq}, @@ -986,68 +1065,80 @@ rep_stats(State) -> {checkpointed_source_seq, CommittedSeq} ]. - replication_start_error({unauthorized, DbUri}) -> {unauthorized, <<"unauthorized to access or create database ", DbUri/binary>>}; replication_start_error({db_not_found, DbUri}) -> {db_not_found, <<"could not open ", DbUri/binary>>}; -replication_start_error({http_request_failed, _Method, Url0, - {error, {error, {conn_failed, {error, nxdomain}}}}}) -> +replication_start_error( + {http_request_failed, _Method, Url0, {error, {error, {conn_failed, {error, nxdomain}}}}} +) -> Url = ?l2b(couch_util:url_strip_password(Url0)), {nxdomain, <<"could not resolve ", Url/binary>>}; -replication_start_error({http_request_failed, Method0, Url0, - {error, {code, Code}}}) when is_integer(Code) -> +replication_start_error({http_request_failed, Method0, Url0, {error, {code, Code}}}) when + is_integer(Code) +-> Url = ?l2b(couch_util:url_strip_password(Url0)), Method = ?l2b(Method0), {http_error_code, Code, <<Method/binary, " ", Url/binary>>}; replication_start_error(Error) -> Error. - log_replication_start(#rep_state{rep_details = Rep} = RepState) -> #rep{ - id = {BaseId, Ext}, - doc_id = DocId, - db_name = DbName, - options = Options + id = {BaseId, Ext}, + doc_id = DocId, + db_name = DbName, + options = Options } = Rep, Id = BaseId ++ Ext, Workers = get_value(worker_processes, Options), BatchSize = get_value(worker_batch_size, Options), #rep_state{ - source_name = Source, % credentials already stripped - target_name = Target, % credentials already stripped - session_id = Sid + % credentials already stripped + source_name = Source, + % credentials already stripped + target_name = Target, + session_id = Sid } = RepState, - From = case DbName of - ShardName when is_binary(ShardName) -> - io_lib:format("from doc ~s:~s", [mem3:dbname(ShardName), DocId]); - _ -> - "from _replicate endpoint" - end, - Msg = "Starting replication ~s (~s -> ~s) ~s worker_procesess:~p" + From = + case DbName of + ShardName when is_binary(ShardName) -> + io_lib:format("from doc ~s:~s", [mem3:dbname(ShardName), DocId]); + _ -> + "from _replicate endpoint" + end, + Msg = + "Starting replication ~s (~s -> ~s) ~s worker_procesess:~p" " worker_batch_size:~p session_id:~s", couch_log:notice(Msg, [Id, Source, Target, From, Workers, BatchSize, Sid]). - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). - replication_start_error_test() -> - ?assertEqual({unauthorized, <<"unauthorized to access or create database" - " http://x/y">>}, replication_start_error({unauthorized, - <<"http://x/y">>})), - ?assertEqual({db_not_found, <<"could not open http://x/y">>}, - replication_start_error({db_not_found, <<"http://x/y">>})), - ?assertEqual({nxdomain,<<"could not resolve http://x/y">>}, - replication_start_error({http_request_failed, "GET", "http://x/y", - {error, {error, {conn_failed, {error, nxdomain}}}}})), - ?assertEqual({http_error_code,503,<<"GET http://x/y">>}, - replication_start_error({http_request_failed, "GET", "http://x/y", - {error, {code, 503}}})). - + ?assertEqual( + {unauthorized, << + "unauthorized to access or create database" + " http://x/y" + >>}, + replication_start_error({unauthorized, <<"http://x/y">>}) + ), + ?assertEqual( + {db_not_found, <<"could not open http://x/y">>}, + replication_start_error({db_not_found, <<"http://x/y">>}) + ), + ?assertEqual( + {nxdomain, <<"could not resolve http://x/y">>}, + replication_start_error( + {http_request_failed, "GET", "http://x/y", + {error, {error, {conn_failed, {error, nxdomain}}}}} + ) + ), + ?assertEqual( + {http_error_code, 503, <<"GET http://x/y">>}, + replication_start_error({http_request_failed, "GET", "http://x/y", {error, {code, 503}}}) + ). scheduler_job_format_status_test() -> Source = <<"http://u:p@h1/d1">>, @@ -1085,5 +1176,4 @@ scheduler_job_format_status_test() -> ?assertEqual(<<"4">>, proplists:get_value(current_through_seq, Format)), ?assertEqual(<<"5">>, proplists:get_value(highest_seq_done, Format)). - -endif. diff --git a/src/couch_replicator/src/couch_replicator_scheduler_sup.erl b/src/couch_replicator/src/couch_replicator_scheduler_sup.erl index 8ab55f838..1d5104312 100644 --- a/src/couch_replicator/src/couch_replicator_scheduler_sup.erl +++ b/src/couch_replicator/src/couch_replicator_scheduler_sup.erl @@ -23,24 +23,20 @@ %% supervisor api -export([ - init/1 + init/1 ]). - %% includes -include("couch_replicator.hrl"). - %% public functions start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). - start_child(#rep{} = Rep) -> supervisor:start_child(?MODULE, [Rep]). - terminate_child(Pid) -> supervisor:terminate_child(?MODULE, Pid). @@ -48,7 +44,8 @@ terminate_child(Pid) -> init(_Args) -> Start = {couch_replicator_scheduler_job, start_link, []}, - Restart = temporary, % A crashed job is not entitled to immediate restart. + % A crashed job is not entitled to immediate restart. + Restart = temporary, Shutdown = 5000, Type = worker, Modules = [couch_replicator_scheduler_job], diff --git a/src/couch_replicator/src/couch_replicator_share.erl b/src/couch_replicator/src/couch_replicator_share.erl index c1e52b2b8..8c9fa029a 100644 --- a/src/couch_replicator/src/couch_replicator_share.erl +++ b/src/couch_replicator/src/couch_replicator_share.erl @@ -53,7 +53,6 @@ % % [1] : https://proteusmaster.urcf.drexel.edu/urcfwiki/images/KayLauderFairShare.pdf - -module(couch_replicator_share). -export([ @@ -68,11 +67,9 @@ charge/3 ]). - -include_lib("couch/include/couch_db.hrl"). -include("couch_replicator.hrl"). - % Usage coefficient decays historic usage every scheduling cycle. For example, % the usage value for a job running 1 minute is 60000000 (i.e microseconds / % minute), then if the job stops running it will take about 26 cycles (minutes) @@ -101,7 +98,6 @@ % -define(DEFAULT_PRIORITY_COEFF, 0.98). - -define(MIN_SHARES, 1). -define(MAX_SHARES, 1000). -define(DEFAULT_SHARES, 100). @@ -112,34 +108,34 @@ -define(CHARGES, couch_replicator_stopped_usage). -define(NUM_JOBS, couch_replicator_num_jobs). - init() -> EtsOpts = [named_table, public], - ?SHARES = ets:new(?SHARES, EtsOpts), % {Key, Shares} - ?PRIORITIES = ets:new(?PRIORITIES, EtsOpts), % {JobId, Priority} - ?USAGE = ets:new(?USAGE, EtsOpts), % {Key, Usage} - ?CHARGES = ets:new(?CHARGES, EtsOpts), % {Key, Charges} - ?NUM_JOBS = ets:new(?NUM_JOBS, EtsOpts), % {Key, NumJobs} + % {Key, Shares} + ?SHARES = ets:new(?SHARES, EtsOpts), + % {JobId, Priority} + ?PRIORITIES = ets:new(?PRIORITIES, EtsOpts), + % {Key, Usage} + ?USAGE = ets:new(?USAGE, EtsOpts), + % {Key, Charges} + ?CHARGES = ets:new(?CHARGES, EtsOpts), + % {Key, NumJobs} + ?NUM_JOBS = ets:new(?NUM_JOBS, EtsOpts), lists:foreach(fun({K, V}) -> update_shares(K, V) end, get_config_shares()). - clear() -> Tables = [?SHARES, ?PRIORITIES, ?USAGE, ?CHARGES, ?NUM_JOBS], lists:foreach(fun(T) -> catch ets:delete(T) end, Tables). - % This should be called when user updates the replicator.shares config section % update_shares(Key, Shares) when is_integer(Shares) -> ets:insert(?SHARES, {Key, bounded(Shares, ?MIN_SHARES, ?MAX_SHARES)}). - % Called when the config value is deleted and shares are reset to the default % value. reset_shares(Key) -> ets:delete(?SHARES, Key). - job_added(#job{} = Job) -> Key = key(Job), % If the entry is not present {Key, 0} is used as the default @@ -148,7 +144,6 @@ job_added(#job{} = Job) -> % new jobs don't get to be at priority 0 (highest). update_priority(Job). - job_removed(#job{} = Job) -> Key = key(Job), ets:delete(?PRIORITIES, Job#job.id), @@ -160,7 +155,6 @@ job_removed(#job{} = Job) -> end, ok. - % This is the main algorithm update function. It should be called during each % rescheduling cycle with a list of running jobs, the interval from the % scheduler (in milliseconds), and the current timestamp. @@ -180,7 +174,6 @@ update(RunningJobs, Interval, {_, _, _} = Now) -> decay_priorities(), lists:foreach(fun(Job) -> update_priority(Job) end, RunningJobs). - priority(JobId) -> % Not found means it was removed because it's value was 0 case ets:lookup(?PRIORITIES, JobId) of @@ -188,38 +181,32 @@ priority(JobId) -> [] -> 0 end. - charge(#job{pid = undefined}, _, _) -> 0; - charge(#job{} = Job, Interval, {_, _, _} = Now) when is_integer(Interval) -> Key = key(Job), Charges = job_charges(Job, Interval, Now), % If the entry is not present {Key, 0} is used as the default ets:update_counter(?CHARGES, Key, Charges, {Key, 0}). - usage(Key) -> case ets:lookup(?USAGE, Key) of [{_, Usage}] -> Usage; [] -> 0 end. - num_jobs(Key) -> case ets:lookup(?NUM_JOBS, Key) of [{_, NumJobs}] -> NumJobs; [] -> 0 end. - shares(Key) -> case ets:lookup(?SHARES, Key) of [{_, Shares}] -> Shares; [] -> ?DEFAULT_SHARES end. - % In [1] this described in the "Decay of Process Priorities" section % decay_priorities() -> @@ -228,7 +215,6 @@ decay_priorities() -> % is missing we assume it is 0 clear_zero(?PRIORITIES). - % This is the main part of the alrgorithm. In [1] it is described in the % "Priority Adjustment" section. % @@ -240,20 +226,22 @@ update_priority(#job{} = Job) -> % If the entry is not present {Id, 0} is used as the default ets:update_counter(?PRIORITIES, Id, trunc(Priority), {Id, 0}). - % This is the "User-Level Scheduling" part from [1] % update_usage() -> decay(?USAGE, usage_coeff()), clear_zero(?USAGE), - ets:foldl(fun({Key, Charges}, _) -> - % If the entry is not present {Key, 0} is used as the default - ets:update_counter(?USAGE, Key, Charges, {Key, 0}) - end, 0, ?CHARGES), + ets:foldl( + fun({Key, Charges}, _) -> + % If the entry is not present {Key, 0} is used as the default + ets:update_counter(?USAGE, Key, Charges, {Key, 0}) + end, + 0, + ?CHARGES + ), % Start each interval with a fresh charges table ets:delete_all_objects(?CHARGES). - % Private helper functions decay(Ets, Coeff) when is_atom(Ets) -> @@ -263,11 +251,9 @@ decay(Ets, Coeff) when is_atom(Ets) -> Result = {{'$1', {trunc, {'*', '$2', {const, Coeff}}}}}, ets:select_replace(Ets, [{Head, [], [Result]}]). - clear_zero(Ets) when is_atom(Ets) -> ets:select_delete(Ets, [{{'_', '$1'}, [{'=<', '$1', 0}], [true]}]). - key(#job{} = Job) -> Rep = Job#job.rep, case is_binary(Rep#rep.db_name) of @@ -275,7 +261,6 @@ key(#job{} = Job) -> false -> (Rep#rep.user_ctx)#user_ctx.name end. - % Jobs are charged based on the amount of time the job was running during the % last scheduling interval. The time units used are microseconds in order to % have a large enough usage values so that when priority is calculated the @@ -294,25 +279,25 @@ job_charges(#job{} = Job, IntervalMSec, {_, _, _} = Now) -> IntervalUSec = IntervalMSec * 1000, bounded(TimeRunning, 0, IntervalUSec). - last_started(#job{} = Job) -> case lists:keyfind(started, 1, Job#job.history) of - false -> {0, 0, 0}; % In case user set too low of a max history + % In case user set too low of a max history + false -> {0, 0, 0}; {started, When} -> When end. - bounded(Val, Min, Max) -> max(Min, min(Max, Val)). - % Config helper functions get_config_shares() -> - lists:map(fun({K, V}) -> - {list_to_binary(K), int_val(V, ?DEFAULT_SHARES)} - end, config:get("replicator.shares")). - + lists:map( + fun({K, V}) -> + {list_to_binary(K), int_val(V, ?DEFAULT_SHARES)} + end, + config:get("replicator.shares") + ). priority_coeff() -> % This is the K2 coefficient from [1] @@ -320,14 +305,12 @@ priority_coeff() -> Val = float_val(config:get("replicator", "priority_coeff"), Default), bounded(Val, 0.0, 1.0). - usage_coeff() -> % This is the K1 coefficient from [1] Default = ?DEFAULT_USAGE_COEFF, Val = float_val(config:get("replicator", "usage_coeff"), Default), bounded(Val, 0.0, 1.0). - int_val(Str, Default) when is_list(Str) -> try list_to_integer(Str) of Val -> Val @@ -336,10 +319,8 @@ int_val(Str, Default) when is_list(Str) -> Default end. - float_val(undefined, Default) -> Default; - float_val(Str, Default) when is_list(Str) -> try list_to_float(Str) of Val -> Val @@ -348,14 +329,12 @@ float_val(Str, Default) when is_list(Str) -> Default end. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch_replicator/test/eunit/couch_replicator_test.hrl"). - -define(DB1, <<"db1">>). -define(DB2, <<"db2">>). -define(DB3, <<"db3">>). @@ -363,7 +342,6 @@ float_val(Str, Default) when is_list(Str) -> -define(J2, <<"j2">>). -define(J3, <<"j3">>). - fair_share_test_() -> { setup, @@ -395,31 +373,26 @@ fair_share_test_() -> } }. - setup_all() -> test_util:start_couch(). - teardown_all(Ctx) -> config_delete("priority_coeff"), config_delete("usage_coeff"), config_shares_delete(), test_util:stop_couch(Ctx). - setup() -> init(), ok. - teardown(_) -> clear(), config_delete("priority_coeff"), config_delete("usage_coeff"), config_shares_delete(). - -init_works(_)-> +init_works(_) -> Tables = [?SHARES, ?PRIORITIES, ?USAGE, ?CHARGES, ?NUM_JOBS], [?assert(is_list(ets:info(T))) || T <- Tables], ?assertEqual(#{}, tab2map(?SHARES)), @@ -432,7 +405,6 @@ init_works(_)-> ?assertEqual(200, shares(?DB1)), ?assertEqual(#{?DB1 => 200}, tab2map(?SHARES)). - shares_are_updated_and_reset(_) -> ?assertEqual(#{}, tab2map(?SHARES)), @@ -451,7 +423,6 @@ shares_are_updated_and_reset(_) -> update_shares(?DB1, 1001), ?assertEqual(1000, shares(?DB1)). - jobs_are_added_and_removed(_) -> job_added(job(?J1, ?DB1)), ?assertEqual(1, num_jobs(?DB1)), @@ -479,7 +450,6 @@ jobs_are_added_and_removed(_) -> ?assertEqual(0, priority(?J2)), ?assertEqual(#{}, tab2map(?PRIORITIES)). - can_fetch_job_priority(_) -> job_added(job(?J1, ?DB1)), ?assertEqual(0, priority(?J1)), @@ -490,7 +460,6 @@ can_fetch_job_priority(_) -> ets:delete(?PRIORITIES, ?J1), ?assertEqual(0, priority(?J1)). - jobs_are_charged(_) -> Job1 = running_job(?J1, ?DB1), job_added(Job1), @@ -518,7 +487,6 @@ jobs_are_charged(_) -> job_removed(Job2), ?assertEqual(#{?DB1 => 2000001}, tab2map(?CHARGES)). - usage_is_updated(_) -> Job = running_job(?J1, ?DB1), job_added(Job), @@ -549,7 +517,6 @@ usage_is_updated(_) -> ?assertEqual(0, usage(?DB1)), ?assertEqual(#{}, tab2map(?USAGE)). - priority_coefficient_works(_) -> job_added(job(?J1, ?DB1)), ets:insert(?PRIORITIES, {?J1, 1000}), @@ -578,7 +545,6 @@ priority_coefficient_works(_) -> ?assertEqual(0, priority(?J1)), ?assertEqual(#{}, tab2map(?PRIORITIES)). - priority_decays_when_jobs_stop_running(_) -> Job = running_job(?J1, ?DB1), job_added(Job), @@ -593,7 +559,6 @@ priority_decays_when_jobs_stop_running(_) -> [reschedule(0, {[], Pending}) || _ <- lists:seq(1, 500)], ?assertEqual(0, priority(?J1)). - priority_increases_when_jobs_run(_) -> Job = running_job(?J1, ?DB1), job_added(Job), @@ -617,7 +582,6 @@ priority_increases_when_jobs_run(_) -> Pm = priority(?J1), ?assertEqual(Pn, Pm). - two_dbs_equal_shares_equal_number_of_jobs(_) -> update_shares(?DB1, 100), update_shares(?DB2, 100), @@ -626,7 +590,6 @@ two_dbs_equal_shares_equal_number_of_jobs(_) -> ?assert(49 =< Db1 andalso Db1 =< 51), ?assert(49 =< Db2 andalso Db2 =< 51). - two_dbs_unequal_shares_equal_number_of_jobs(_) -> update_shares(?DB1, 100), update_shares(?DB1, 900), @@ -635,7 +598,6 @@ two_dbs_unequal_shares_equal_number_of_jobs(_) -> ?assert(89 =< Db1 andalso Db1 =< 91), ?assert(9 =< Db2 andalso Db2 =< 11). - two_dbs_equal_shares_unequal_number_of_jobs(_) -> update_shares(?DB1, 100), update_shares(?DB2, 100), @@ -644,7 +606,6 @@ two_dbs_equal_shares_unequal_number_of_jobs(_) -> ?assert(49 =< Db1 andalso Db1 =< 51), ?assert(49 =< Db2 andalso Db2 =< 51). - two_dbs_unequal_shares_unequal_number_of_jobs(_) -> update_shares(?DB1, 1), update_shares(?DB2, 100), @@ -653,7 +614,6 @@ two_dbs_unequal_shares_unequal_number_of_jobs(_) -> ?assert(0 =< Db1 andalso Db1 =< 2), ?assert(98 =< Db2 andalso Db2 =< 100). - three_dbs_equal_shares_equal_number_of_jobs(_) -> update_shares(?DB1, 100), update_shares(?DB2, 100), @@ -664,18 +624,16 @@ three_dbs_equal_shares_equal_number_of_jobs(_) -> ?assert(32 =< Db2 andalso Db2 =< 34), ?assert(32 =< Db3 andalso Db3 =< 34). - three_dbs_unequal_shares_equal_number_of_jobs(_) -> update_shares(?DB1, 100), update_shares(?DB2, 700), update_shares(?DB3, 200), Jobs = jobs(#{?DB1 => {25, 75}, ?DB2 => {25, 75}, ?DB3 => {25, 75}}), #{?DB1 := Db1, ?DB2 := Db2, ?DB3 := Db3} = run_scheduler(1000, 10, Jobs), - ?assert(9 =< Db1 andalso Db1 =< 11), + ?assert(9 =< Db1 andalso Db1 =< 11), ?assert(69 =< Db2 andalso Db2 =< 71), ?assert(19 =< Db3 andalso Db3 =< 21). - three_dbs_equal_shares_unequal_number_of_jobs(_) -> update_shares(?DB1, 100), update_shares(?DB2, 100), @@ -686,7 +644,6 @@ three_dbs_equal_shares_unequal_number_of_jobs(_) -> ?assert(32 =< Db2 andalso Db2 =< 34), ?assert(32 =< Db3 andalso Db3 =< 34). - three_dbs_unequal_shares_unequal_number_of_jobs(_) -> update_shares(?DB1, 1000), update_shares(?DB2, 100), @@ -697,31 +654,26 @@ three_dbs_unequal_shares_unequal_number_of_jobs(_) -> ?assert(9 =< Db2 andalso Db2 =< 11), ?assert(2 =< Db3 andalso Db3 =< 4). - config_set(K, V) -> config:set("replicator", K, V, _Persist = false). - config_delete(K) -> config:delete("replicator", K, _Persist = false). - config_share_set(K, V) -> config:set("replicator.shares", K, V, _Persist = false). - config_shares_delete() -> - [config:delete("replicator.shares", K, _Persist = false) || - {K, _} <- config:get("replicator.shares")]. - + [ + config:delete("replicator.shares", K, _Persist = false) + || {K, _} <- config:get("replicator.shares") + ]. tab2map(T) when is_atom(T) -> maps:from_list(ets:tab2list(T)). - job(rand, Db) -> job(rand:uniform(1 bsl 59), Db); - job(Id, Db) -> Job = #job{ id = Id, @@ -732,25 +684,21 @@ job(Id, Db) -> }, stop(Job). - running_job(Id, Db) -> run(job(Id, Db)). - run(#job{} = Job) -> Job#job{ pid = list_to_pid("<0.9999.999>"), history = [{started, {0, 0, 0}}, {added, {0, 0, 0}}] }. - stop(#job{} = Job) -> Job#job{ pid = undefined, history = [{added, {0, 0, 0}}] }. - % Simple scheduler simulator. Start and stop N jobs and do the % accounting steps. Return a new list of running and pending jobs. If % N is 0 then jobs which were running stay running and jobs were @@ -773,33 +721,42 @@ reschedule(N, {Running, Pending}) -> {Running2, Pending2}. - % Run a few scheduling cycles and calculate usage percentage for each db % run_scheduler(Cycles, Churn, Jobs0) -> Acc0 = {#{}, Jobs0}, - {Sum, _} = lists:foldl(fun(_CycleCnt, {UsageAcc, {Running, _} = Jobs}) -> - UsageAcc1 = lists:foldl(fun(#job{} = Job, Acc) -> - Db = Job#job.rep#rep.db_name, - maps:update_with(Db, fun(V) -> V + 1 end, 0, Acc) - end, UsageAcc, Running), - {UsageAcc1, reschedule(Churn, Jobs)} - end, Acc0, lists:seq(1, Cycles)), + {Sum, _} = lists:foldl( + fun(_CycleCnt, {UsageAcc, {Running, _} = Jobs}) -> + UsageAcc1 = lists:foldl( + fun(#job{} = Job, Acc) -> + Db = Job#job.rep#rep.db_name, + maps:update_with(Db, fun(V) -> V + 1 end, 0, Acc) + end, + UsageAcc, + Running + ), + {UsageAcc1, reschedule(Churn, Jobs)} + end, + Acc0, + lists:seq(1, Cycles) + ), Total = maps:fold(fun(_, V, Acc) -> Acc + V end, 0, Sum), maps:map(fun(_Db, V) -> round(V / Total * 100) end, Sum). - % Dbs = #{Db => {RunningCount, PendingCount} % jobs(#{} = Dbs) -> - maps:fold(fun(Db, {RCnt, PCnt}, {Running, Pending}) -> - RJobs = [running_job(rand, Db) || _ <- lists:seq(1, RCnt)], - PJobs = [job(rand, Db) || _ <- lists:seq(1, PCnt)], - [job_added(Job) || Job <- RJobs ++ PJobs], - {Running ++ RJobs, Pending ++ PJobs} - end, {[], []}, Dbs). - + maps:fold( + fun(Db, {RCnt, PCnt}, {Running, Pending}) -> + RJobs = [running_job(rand, Db) || _ <- lists:seq(1, RCnt)], + PJobs = [job(rand, Db) || _ <- lists:seq(1, PCnt)], + [job_added(Job) || Job <- RJobs ++ PJobs], + {Running ++ RJobs, Pending ++ PJobs} + end, + {[], []}, + Dbs + ). -endif. diff --git a/src/couch_replicator/src/couch_replicator_stats.erl b/src/couch_replicator/src/couch_replicator_stats.erl index 37848b3ee..e1f23a1bc 100644 --- a/src/couch_replicator/src/couch_replicator_stats.erl +++ b/src/couch_replicator/src/couch_replicator_stats.erl @@ -63,31 +63,25 @@ increment(Field, Stats) -> orddict:update_counter(Field, 1, Stats). sum_stats(S1, S2) -> - orddict:merge(fun(_, V1, V2) -> V1+V2 end, S1, S2). + orddict:merge(fun(_, V1, V2) -> V1 + V2 end, S1, S2). max_stats(S1, S2) -> orddict:merge(fun(_, V1, V2) -> max(V1, V2) end, S1, S2). - % Handle initializing from a status object, which uses same values but % different field names, as well as from ejson props from the checkpoint % history % -fmap({missing_found, _}) -> true; -fmap({missing_revisions_found, V}) -> {true, {missing_found, V}}; -fmap({<<"missing_found">>, V}) -> {true, {missing_found, V}}; - -fmap({missing_checked, _}) -> true; -fmap({revisions_checked, V}) -> {true, {missing_checked, V}}; -fmap({<<"missing_checked">>, V}) -> {true, {missing_checked, V}}; - -fmap({docs_read, _}) -> true; -fmap({<<"docs_read">>, V}) -> {true, {docs_read, V}}; - -fmap({docs_written, _}) -> true; -fmap({<<"docs_written">>, V}) -> {true, {docs_written, V}}; - -fmap({doc_write_failures, _}) -> true; -fmap({<<"doc_write_failures">>, V}) -> {true, {doc_write_failures, V}}; - -fmap({_, _}) -> false. +fmap({missing_found, _}) -> true; +fmap({missing_revisions_found, V}) -> {true, {missing_found, V}}; +fmap({<<"missing_found">>, V}) -> {true, {missing_found, V}}; +fmap({missing_checked, _}) -> true; +fmap({revisions_checked, V}) -> {true, {missing_checked, V}}; +fmap({<<"missing_checked">>, V}) -> {true, {missing_checked, V}}; +fmap({docs_read, _}) -> true; +fmap({<<"docs_read">>, V}) -> {true, {docs_read, V}}; +fmap({docs_written, _}) -> true; +fmap({<<"docs_written">>, V}) -> {true, {docs_written, V}}; +fmap({doc_write_failures, _}) -> true; +fmap({<<"doc_write_failures">>, V}) -> {true, {doc_write_failures, V}}; +fmap({_, _}) -> false. diff --git a/src/couch_replicator/src/couch_replicator_sup.erl b/src/couch_replicator/src/couch_replicator_sup.erl index cd4512c54..33eee8659 100644 --- a/src/couch_replicator/src/couch_replicator_sup.erl +++ b/src/couch_replicator/src/couch_replicator_sup.erl @@ -20,53 +20,21 @@ start_link() -> init(_Args) -> Children = [ - {couch_replication_event, - {gen_event, start_link, [{local, couch_replication}]}, - permanent, - brutal_kill, - worker, - dynamic}, - {couch_replicator_clustering, - {couch_replicator_clustering, start_link, []}, - permanent, - brutal_kill, - worker, - [couch_replicator_clustering]}, - {couch_replicator_connection, - {couch_replicator_connection, start_link, []}, - permanent, - brutal_kill, - worker, - [couch_replicator_connection]}, - {couch_replicator_rate_limiter, - {couch_replicator_rate_limiter, start_link, []}, - permanent, - brutal_kill, - worker, - [couch_replicator_rate_limiter]}, - {couch_replicator_scheduler_sup, - {couch_replicator_scheduler_sup, start_link, []}, - permanent, - infinity, - supervisor, - [couch_replicator_scheduler_sup]}, - {couch_replicator_scheduler, - {couch_replicator_scheduler, start_link, []}, - permanent, - brutal_kill, - worker, - [couch_replicator_scheduler]}, - {couch_replicator_doc_processor, - {couch_replicator_doc_processor, start_link, []}, - permanent, - brutal_kill, - worker, - [couch_replicator_doc_processor]}, - {couch_replicator_db_changes, - {couch_replicator_db_changes, start_link, []}, - permanent, - brutal_kill, - worker, - [couch_multidb_changes]} + {couch_replication_event, {gen_event, start_link, [{local, couch_replication}]}, permanent, + brutal_kill, worker, dynamic}, + {couch_replicator_clustering, {couch_replicator_clustering, start_link, []}, permanent, + brutal_kill, worker, [couch_replicator_clustering]}, + {couch_replicator_connection, {couch_replicator_connection, start_link, []}, permanent, + brutal_kill, worker, [couch_replicator_connection]}, + {couch_replicator_rate_limiter, {couch_replicator_rate_limiter, start_link, []}, permanent, + brutal_kill, worker, [couch_replicator_rate_limiter]}, + {couch_replicator_scheduler_sup, {couch_replicator_scheduler_sup, start_link, []}, + permanent, infinity, supervisor, [couch_replicator_scheduler_sup]}, + {couch_replicator_scheduler, {couch_replicator_scheduler, start_link, []}, permanent, + brutal_kill, worker, [couch_replicator_scheduler]}, + {couch_replicator_doc_processor, {couch_replicator_doc_processor, start_link, []}, + permanent, brutal_kill, worker, [couch_replicator_doc_processor]}, + {couch_replicator_db_changes, {couch_replicator_db_changes, start_link, []}, permanent, + brutal_kill, worker, [couch_multidb_changes]} ], - {ok, {{rest_for_one,10,1}, Children}}. + {ok, {{rest_for_one, 10, 1}, Children}}. diff --git a/src/couch_replicator/src/couch_replicator_utils.erl b/src/couch_replicator/src/couch_replicator_utils.erl index dbadb3787..b2bc34078 100644 --- a/src/couch_replicator/src/couch_replicator_utils.erl +++ b/src/couch_replicator/src/couch_replicator_utils.erl @@ -13,24 +13,23 @@ -module(couch_replicator_utils). -export([ - parse_rep_doc/2, - replication_id/2, - sum_stats/2, - is_deleted/1, - rep_error_to_binary/1, - get_json_value/2, - get_json_value/3, - pp_rep_id/1, - iso8601/1, - filter_state/3, - normalize_rep/1, - ejson_state_info/1, - get_basic_auth_creds/1, - remove_basic_auth_creds/1, - normalize_basic_auth/1 + parse_rep_doc/2, + replication_id/2, + sum_stats/2, + is_deleted/1, + rep_error_to_binary/1, + get_json_value/2, + get_json_value/3, + pp_rep_id/1, + iso8601/1, + filter_state/3, + normalize_rep/1, + ejson_state_info/1, + get_basic_auth_creds/1, + remove_basic_auth_creds/1, + normalize_basic_auth/1 ]). - -include_lib("ibrowse/include/ibrowse.hrl"). -include_lib("couch/include/couch_db.hrl"). -include("couch_replicator.hrl"). @@ -41,22 +40,20 @@ get_value/3 ]). - rep_error_to_binary(Error) -> couch_util:to_binary(error_reason(Error)). - error_reason({shutdown, Error}) -> error_reason(Error); -error_reason({error, {Error, Reason}}) - when is_atom(Error), is_binary(Reason) -> +error_reason({error, {Error, Reason}}) when + is_atom(Error), is_binary(Reason) +-> io_lib:format("~s: ~s", [Error, Reason]); error_reason({error, Reason}) -> Reason; error_reason(Reason) -> Reason. - get_json_value(Key, Props) -> get_json_value(Key, Props, undefined). @@ -77,7 +74,6 @@ get_json_value(Key, Props, Default) when is_binary(Key) -> Else end. - % pretty-print replication id -spec pp_rep_id(#rep{} | rep_id()) -> string(). pp_rep_id(#rep{id = RepId}) -> @@ -85,34 +81,28 @@ pp_rep_id(#rep{id = RepId}) -> pp_rep_id({Base, Extension}) -> Base ++ Extension. - % NV: TODO: this function is not used outside api wrap module % consider moving it there during final cleanup is_deleted(Change) -> get_json_value(<<"deleted">>, Change, false). - % NV: TODO: proxy some functions which used to be here, later remove % these and replace calls to their respective modules replication_id(Rep, Version) -> couch_replicator_ids:replication_id(Rep, Version). - sum_stats(S1, S2) -> couch_replicator_stats:sum_stats(S1, S2). - parse_rep_doc(Props, UserCtx) -> couch_replicator_docs:parse_rep_doc(Props, UserCtx). - -spec iso8601(erlang:timestamp()) -> binary(). iso8601({_Mega, _Sec, _Micro} = Timestamp) -> {{Y, Mon, D}, {H, Min, S}} = calendar:now_to_universal_time(Timestamp), Format = "~B-~2..0B-~2..0BT~2..0B:~2..0B:~2..0BZ", iolist_to_binary(io_lib:format(Format, [Y, Mon, D, H, Min, S])). - %% Filter replication info ejson by state provided. If it matches return %% the input value, if it doesn't return 'skip'. This is used from replicator %% fabric coordinator and worker. @@ -129,7 +119,6 @@ filter_state(State, States, Info) -> skip end. - remove_basic_auth_from_headers(Headers) -> Headers1 = mochiweb_headers:make(Headers), case mochiweb_headers:get_value("Authorization", Headers1) of @@ -140,14 +129,12 @@ remove_basic_auth_from_headers(Headers) -> maybe_remove_basic_auth(string:to_lower(Basic), Base64, Headers1) end. - maybe_remove_basic_auth("basic", " " ++ Base64, Headers) -> Headers1 = mochiweb_headers:delete_any("Authorization", Headers), {decode_basic_creds(Base64), mochiweb_headers:to_list(Headers1)}; maybe_remove_basic_auth(_, _, Headers) -> {{undefined, undefined}, mochiweb_headers:to_list(Headers)}. - decode_basic_creds(Base64) -> try re:split(base64:decode(Base64), ":", [{return, list}, {parts, 2}]) of [User, Pass] -> @@ -160,47 +147,45 @@ decode_basic_creds(Base64) -> {undefined, undefined} end. - % Normalize a #rep{} record such that it doesn't contain time dependent fields % pids (like httpc pools), and options / props are sorted. This function would % used during comparisons. -spec normalize_rep(#rep{} | nil) -> #rep{} | nil. normalize_rep(nil) -> nil; - -normalize_rep(#rep{} = Rep)-> +normalize_rep(#rep{} = Rep) -> #rep{ source = couch_replicator_api_wrap:normalize_db(Rep#rep.source), target = couch_replicator_api_wrap:normalize_db(Rep#rep.target), - options = Rep#rep.options, % already sorted in make_options/1 + % already sorted in make_options/1 + options = Rep#rep.options, type = Rep#rep.type, view = Rep#rep.view, doc_id = Rep#rep.doc_id, db_name = Rep#rep.db_name }. - -spec ejson_state_info(binary() | nil) -> binary() | null. ejson_state_info(nil) -> null; ejson_state_info(Info) when is_binary(Info) -> {[{<<"error">>, Info}]}; ejson_state_info([]) -> - null; % Status not set yet => null for compatibility reasons + % Status not set yet => null for compatibility reasons + null; ejson_state_info([{_, _} | _] = Info) -> {Info}; ejson_state_info(Info) -> ErrMsg = couch_replicator_utils:rep_error_to_binary(Info), {[{<<"error">>, ErrMsg}]}. - -spec get_basic_auth_creds(#httpdb{}) -> {string(), string()} | {undefined, undefined}. get_basic_auth_creds(#httpdb{auth_props = AuthProps}) -> case couch_util:get_value(<<"basic">>, AuthProps) of undefined -> {undefined, undefined}; - {UserPass} when is_list(UserPass) -> + {UserPass} when is_list(UserPass) -> User = couch_util:get_value(<<"username">>, UserPass), Pass = couch_util:get_value(<<"password">>, UserPass), case {User, Pass} of @@ -213,31 +198,30 @@ get_basic_auth_creds(#httpdb{auth_props = AuthProps}) -> {undefined, undefined} end. - -spec remove_basic_auth_creds(#httpd{}) -> #httpdb{}. remove_basic_auth_creds(#httpdb{auth_props = Props} = HttpDb) -> Props1 = lists:keydelete(<<"basic">>, 1, Props), HttpDb#httpdb{auth_props = Props1}. - -spec set_basic_auth_creds(string(), string(), #httpd{}) -> #httpdb{}. set_basic_auth_creds(undefined, undefined, #httpdb{} = HttpDb) -> HttpDb; -set_basic_auth_creds(User, Pass, #httpdb{} = HttpDb) - when is_list(User), is_list(Pass) -> +set_basic_auth_creds(User, Pass, #httpdb{} = HttpDb) when + is_list(User), is_list(Pass) +-> HttpDb1 = remove_basic_auth_creds(HttpDb), Props = HttpDb1#httpdb.auth_props, - UserPass = {[ - {<<"username">>, list_to_binary(User)}, - {<<"password">>, list_to_binary(Pass)} - ]}, + UserPass = + {[ + {<<"username">>, list_to_binary(User)}, + {<<"password">>, list_to_binary(Pass)} + ]}, Props1 = lists:keystore(<<"basic">>, 1, Props, {<<"basic">>, UserPass}), HttpDb1#httpdb{auth_props = Props1}. - -spec extract_creds_from_url(string()) -> - {ok, {string() | undefined, string() | undefined}, string()} | - {error, term()}. + {ok, {string() | undefined, string() | undefined}, string()} + | {error, term()}. extract_creds_from_url(Url) -> case ibrowse_lib:parse_url(Url) of {error, Error} -> @@ -253,7 +237,6 @@ extract_creds_from_url(Url) -> {ok, {User, Pass}, NoCreds} end. - % Normalize basic auth credentials so they are set only in the auth props % object. If multiple basic auth credentials are provided, the resulting % credentials are picked in the following order. @@ -265,35 +248,39 @@ extract_creds_from_url(Url) -> normalize_basic_auth(#httpdb{} = HttpDb) -> #httpdb{url = Url, headers = Headers} = HttpDb, {HeaderCreds, HeadersNoCreds} = remove_basic_auth_from_headers(Headers), - {UrlCreds, UrlWithoutCreds} = case extract_creds_from_url(Url) of - {ok, Creds = {_, _}, UrlNoCreds} -> - {Creds, UrlNoCreds}; - {error, _Error} -> - % Don't crash replicator if user provided an invalid - % userinfo part - {undefined, undefined} - end, + {UrlCreds, UrlWithoutCreds} = + case extract_creds_from_url(Url) of + {ok, Creds = {_, _}, UrlNoCreds} -> + {Creds, UrlNoCreds}; + {error, _Error} -> + % Don't crash replicator if user provided an invalid + % userinfo part + {undefined, undefined} + end, AuthCreds = {_, _} = get_basic_auth_creds(HttpDb), HttpDb1 = remove_basic_auth_creds(HttpDb#httpdb{ url = UrlWithoutCreds, headers = HeadersNoCreds }), - {User, Pass} = case {AuthCreds, UrlCreds, HeaderCreds} of - {{U, P}, {_, _}, {_, _}} when is_list(U), is_list(P) -> {U, P}; - {{_, _}, {U, P}, {_, _}} when is_list(U), is_list(P) -> {U, P}; - {{_, _}, {_, _}, {U, P}} -> {U, P} - end, + {User, Pass} = + case {AuthCreds, UrlCreds, HeaderCreds} of + {{U, P}, {_, _}, {_, _}} when is_list(U), is_list(P) -> {U, P}; + {{_, _}, {U, P}, {_, _}} when is_list(U), is_list(P) -> {U, P}; + {{_, _}, {_, _}, {U, P}} -> {U, P} + end, set_basic_auth_creds(User, Pass, HttpDb1). - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). remove_basic_auth_from_headers_test_() -> - [?_assertMatch({{User, Pass}, NoAuthHeaders}, - remove_basic_auth_from_headers(Headers)) || - {{User, Pass, NoAuthHeaders}, Headers} <- [ + [ + ?_assertMatch( + {{User, Pass}, NoAuthHeaders}, + remove_basic_auth_from_headers(Headers) + ) + || {{User, Pass, NoAuthHeaders}, Headers} <- [ { {undefined, undefined, []}, [] @@ -332,39 +319,42 @@ remove_basic_auth_from_headers_test_() -> ] ]. - b64creds(User, Pass) -> base64:encode_to_string(User ++ ":" ++ Pass). - normalize_rep_test_() -> { setup, - fun() -> meck:expect(config, get, - fun(_, _, Default) -> Default end) + fun() -> + meck:expect( + config, + get, + fun(_, _, Default) -> Default end + ) end, fun(_) -> meck:unload() end, ?_test(begin - EJson1 = {[ - {<<"source">>, <<"http://host.com/source_db">>}, - {<<"target">>, <<"http://target.local/db">>}, - {<<"doc_ids">>, [<<"a">>, <<"c">>, <<"b">>]}, - {<<"other_field">>, <<"some_value">>} - ]}, + EJson1 = + {[ + {<<"source">>, <<"http://host.com/source_db">>}, + {<<"target">>, <<"http://target.local/db">>}, + {<<"doc_ids">>, [<<"a">>, <<"c">>, <<"b">>]}, + {<<"other_field">>, <<"some_value">>} + ]}, Rep1 = couch_replicator_docs:parse_rep_doc_without_id(EJson1), - EJson2 = {[ - {<<"other_field">>, <<"unrelated">>}, - {<<"target">>, <<"http://target.local/db">>}, - {<<"source">>, <<"http://host.com/source_db">>}, - {<<"doc_ids">>, [<<"c">>, <<"a">>, <<"b">>]}, - {<<"other_field2">>, <<"unrelated2">>} - ]}, + EJson2 = + {[ + {<<"other_field">>, <<"unrelated">>}, + {<<"target">>, <<"http://target.local/db">>}, + {<<"source">>, <<"http://host.com/source_db">>}, + {<<"doc_ids">>, [<<"c">>, <<"a">>, <<"b">>]}, + {<<"other_field2">>, <<"unrelated2">>} + ]}, Rep2 = couch_replicator_docs:parse_rep_doc_without_id(EJson2), ?assertEqual(normalize_rep(Rep1), normalize_rep(Rep2)) end) }. - get_basic_auth_creds_test() -> Check = fun(Props) -> get_basic_auth_creds(#httpdb{auth_props = Props}) @@ -384,7 +374,6 @@ get_basic_auth_creds_test() -> UserPass3 = {[{<<"username">>, <<"u">>}, {<<"password">>, null}]}, ?assertEqual({undefined, undefined}, Check([{<<"basic">>, UserPass3}])). - remove_basic_auth_creds_test() -> Check = fun(Props) -> HttpDb = remove_basic_auth_creds(#httpdb{auth_props = Props}), @@ -395,21 +384,28 @@ remove_basic_auth_creds_test() -> ?assertEqual([{<<"other">>, {[]}}], Check([{<<"other">>, {[]}}])), - ?assertEqual([], Check([ - {<<"basic">>, {[ - {<<"username">>, <<"u">>}, - {<<"password">>, <<"p">>} - ]}} - ])), - - ?assertEqual([{<<"other">>, {[]}}], Check([ - {<<"basic">>, {[ - {<<"username">>, <<"u">>}, - {<<"password">>, <<"p">>} - ]}}, - {<<"other">>, {[]}} - ])). - + ?assertEqual( + [], + Check([ + {<<"basic">>, + {[ + {<<"username">>, <<"u">>}, + {<<"password">>, <<"p">>} + ]}} + ]) + ), + + ?assertEqual( + [{<<"other">>, {[]}}], + Check([ + {<<"basic">>, + {[ + {<<"username">>, <<"u">>}, + {<<"password">>, <<"p">>} + ]}}, + {<<"other">>, {[]}} + ]) + ). set_basic_auth_creds_test() -> Check = fun(User, Pass, Props) -> @@ -419,121 +415,158 @@ set_basic_auth_creds_test() -> ?assertEqual([], Check(undefined, undefined, [])), - ?assertEqual([{<<"other">>, {[]}}], Check(undefined, undefined, - [{<<"other">>, {[]}}])), - - ?assertEqual([ - {<<"basic">>, {[ - {<<"username">>, <<"u">>}, - {<<"password">>, <<"p">>} - ]}} - ], Check("u", "p", [])), - - ?assertEqual([ - {<<"other">>, {[]}}, - {<<"basic">>, {[ - {<<"username">>, <<"u">>}, - {<<"password">>, <<"p">>} - ]}} - ], Check("u", "p", [{<<"other">>, {[]}}])). - + ?assertEqual( + [{<<"other">>, {[]}}], + Check( + undefined, + undefined, + [{<<"other">>, {[]}}] + ) + ), + + ?assertEqual( + [ + {<<"basic">>, + {[ + {<<"username">>, <<"u">>}, + {<<"password">>, <<"p">>} + ]}} + ], + Check("u", "p", []) + ), + + ?assertEqual( + [ + {<<"other">>, {[]}}, + {<<"basic">>, + {[ + {<<"username">>, <<"u">>}, + {<<"password">>, <<"p">>} + ]}} + ], + Check("u", "p", [{<<"other">>, {[]}}]) + ). normalize_basic_creds_test_() -> DefaultHeaders = (#httpdb{})#httpdb.headers, - [?_assertEqual(Expect, normalize_basic_auth(Input)) || {Input, Expect} <- [ - { - #httpdb{url = "http://u:p@x.y/db"}, - #httpdb{url = "http://x.y/db", auth_props = auth_props("u", "p")} - }, - { - #httpdb{url = "http://u:p@h:80/db"}, - #httpdb{url = "http://h:80/db", auth_props = auth_props("u", "p")} - }, - { - #httpdb{url = "https://u:p@h/db"}, - #httpdb{url = "https://h/db", auth_props = auth_props("u", "p")} - }, - { - #httpdb{url = "http://u:p@[2001:db8:a1b:12f9::1]/db"}, - #httpdb{url = "http://[2001:db8:a1b:12f9::1]/db", - auth_props = auth_props("u", "p")} - }, - { - #httpdb{ - url = "http://h/db", - headers = DefaultHeaders ++ [ - {"Authorization", "Basic " ++ b64creds("u", "p")} - ] + [ + ?_assertEqual(Expect, normalize_basic_auth(Input)) + || {Input, Expect} <- [ + { + #httpdb{url = "http://u:p@x.y/db"}, + #httpdb{url = "http://x.y/db", auth_props = auth_props("u", "p")} }, - #httpdb{url = "http://h/db", auth_props = auth_props("u", "p")} - }, - { - #httpdb{ - url = "http://h/db", - headers = DefaultHeaders ++ [ - {"Authorization", "Basic " ++ b64creds("u", "p@")} - ] + { + #httpdb{url = "http://u:p@h:80/db"}, + #httpdb{url = "http://h:80/db", auth_props = auth_props("u", "p")} }, - #httpdb{url = "http://h/db", auth_props = auth_props("u", "p@")} - }, - { - #httpdb{ - url = "http://h/db", - headers = DefaultHeaders ++ [ - {"Authorization", "Basic " ++ b64creds("u", "p@%40")} - ] + { + #httpdb{url = "https://u:p@h/db"}, + #httpdb{url = "https://h/db", auth_props = auth_props("u", "p")} }, - #httpdb{url = "http://h/db", auth_props = auth_props("u", "p@%40")} - }, - { - #httpdb{ - url = "http://h/db", - headers = DefaultHeaders ++ [ - {"aUthoriZation", "bASIC " ++ b64creds("U", "p")} - ] + { + #httpdb{url = "http://u:p@[2001:db8:a1b:12f9::1]/db"}, + #httpdb{ + url = "http://[2001:db8:a1b:12f9::1]/db", + auth_props = auth_props("u", "p") + } }, - #httpdb{url = "http://h/db", auth_props = auth_props("U", "p")} - }, - { - #httpdb{ - url = "http://u1:p1@h/db", - headers = DefaultHeaders ++ [ - {"Authorization", "Basic " ++ b64creds("u2", "p2")} - ] + { + #httpdb{ + url = "http://h/db", + headers = + DefaultHeaders ++ + [ + {"Authorization", "Basic " ++ b64creds("u", "p")} + ] + }, + #httpdb{url = "http://h/db", auth_props = auth_props("u", "p")} }, - #httpdb{url ="http://h/db", auth_props = auth_props("u1", "p1")} - }, - { - #httpdb{ - url = "http://u1:p1@h/db", - auth_props = [{<<"basic">>, {[ - {<<"username">>, <<"u2">>}, - {<<"password">>, <<"p2">>} - ]}}] + { + #httpdb{ + url = "http://h/db", + headers = + DefaultHeaders ++ + [ + {"Authorization", "Basic " ++ b64creds("u", "p@")} + ] + }, + #httpdb{url = "http://h/db", auth_props = auth_props("u", "p@")} }, - #httpdb{url = "http://h/db", auth_props = auth_props("u2", "p2")} - }, - { - #httpdb{ - url = "http://u1:p1@h/db", - auth_props = [{<<"basic">>, {[ - {<<"username">>, <<"u2">>}, - {<<"password">>, <<"p2">>} - ]}}], - headers = DefaultHeaders ++ [ - {"Authorization", "Basic " ++ b64creds("u3", "p3")} - ] + { + #httpdb{ + url = "http://h/db", + headers = + DefaultHeaders ++ + [ + {"Authorization", "Basic " ++ b64creds("u", "p@%40")} + ] + }, + #httpdb{url = "http://h/db", auth_props = auth_props("u", "p@%40")} }, - #httpdb{url = "http://h/db", auth_props = auth_props("u2", "p2")} - } - ]]. - + { + #httpdb{ + url = "http://h/db", + headers = + DefaultHeaders ++ + [ + {"aUthoriZation", "bASIC " ++ b64creds("U", "p")} + ] + }, + #httpdb{url = "http://h/db", auth_props = auth_props("U", "p")} + }, + { + #httpdb{ + url = "http://u1:p1@h/db", + headers = + DefaultHeaders ++ + [ + {"Authorization", "Basic " ++ b64creds("u2", "p2")} + ] + }, + #httpdb{url = "http://h/db", auth_props = auth_props("u1", "p1")} + }, + { + #httpdb{ + url = "http://u1:p1@h/db", + auth_props = [ + {<<"basic">>, + {[ + {<<"username">>, <<"u2">>}, + {<<"password">>, <<"p2">>} + ]}} + ] + }, + #httpdb{url = "http://h/db", auth_props = auth_props("u2", "p2")} + }, + { + #httpdb{ + url = "http://u1:p1@h/db", + auth_props = [ + {<<"basic">>, + {[ + {<<"username">>, <<"u2">>}, + {<<"password">>, <<"p2">>} + ]}} + ], + headers = + DefaultHeaders ++ + [ + {"Authorization", "Basic " ++ b64creds("u3", "p3")} + ] + }, + #httpdb{url = "http://h/db", auth_props = auth_props("u2", "p2")} + } + ] + ]. auth_props(User, Pass) when is_list(User), is_list(Pass) -> - [{<<"basic">>, {[ - {<<"username">>, list_to_binary(User)}, - {<<"password">>, list_to_binary(Pass)} - ]}}]. - + [ + {<<"basic">>, + {[ + {<<"username">>, list_to_binary(User)}, + {<<"password">>, list_to_binary(Pass)} + ]}} + ]. -endif. diff --git a/src/couch_replicator/src/couch_replicator_worker.erl b/src/couch_replicator/src/couch_replicator_worker.erl index eb8beaaa9..f66a019e2 100644 --- a/src/couch_replicator/src/couch_replicator_worker.erl +++ b/src/couch_replicator/src/couch_replicator_worker.erl @@ -27,8 +27,11 @@ -include("couch_replicator.hrl"). % TODO: maybe make both buffer max sizes configurable --define(DOC_BUFFER_BYTE_SIZE, 512 * 1024). % for remote targets --define(STATS_DELAY, 10000000). % 10 seconds (in microseconds) + +% for remote targets +-define(DOC_BUFFER_BYTE_SIZE, 512 * 1024). +% 10 seconds (in microseconds) +-define(STATS_DELAY, 10000000). -define(MISSING_DOC_RETRY_MSEC, 2000). -import(couch_util, [ @@ -36,7 +39,6 @@ get_value/3 ]). - -record(batch, { docs = [], size = 0 @@ -56,12 +58,10 @@ batch = #batch{} }). - - start_link(Cp, #httpdb{} = Source, Target, ChangesManager, MaxConns) -> gen_server:start_link( - ?MODULE, {Cp, Source, Target, ChangesManager, MaxConns}, []). - + ?MODULE, {Cp, Source, Target, ChangesManager, MaxConns}, [] + ). init({Cp, Source, Target, ChangesManager, MaxConns}) -> process_flag(trap_exit, true), @@ -79,108 +79,122 @@ init({Cp, Source, Target, ChangesManager, MaxConns}) -> }, {ok, State}. - -handle_call({fetch_doc, {_Id, _Revs, _PAs} = Params}, {Pid, _} = From, - #state{loop = Pid, readers = Readers, pending_fetch = nil, - source = Src, target = Tgt, max_parallel_conns = MaxConns} = State) -> +handle_call( + {fetch_doc, {_Id, _Revs, _PAs} = Params}, + {Pid, _} = From, + #state{ + loop = Pid, + readers = Readers, + pending_fetch = nil, + source = Src, + target = Tgt, + max_parallel_conns = MaxConns + } = State +) -> case length(Readers) of - Size when Size < MaxConns -> - Reader = spawn_doc_reader(Src, Tgt, Params), - NewState = State#state{ - readers = [Reader | Readers] - }, - {reply, ok, NewState}; - _ -> - NewState = State#state{ - pending_fetch = {From, Params} - }, - {noreply, NewState} + Size when Size < MaxConns -> + Reader = spawn_doc_reader(Src, Tgt, Params), + NewState = State#state{ + readers = [Reader | Readers] + }, + {reply, ok, NewState}; + _ -> + NewState = State#state{ + pending_fetch = {From, Params} + }, + {noreply, NewState} end; - handle_call({batch_doc, Doc}, From, State) -> gen_server:reply(From, ok), {noreply, maybe_flush_docs(Doc, State)}; - handle_call({add_stats, IncStats}, From, #state{stats = Stats} = State) -> gen_server:reply(From, ok), NewStats = couch_replicator_utils:sum_stats(Stats, IncStats), NewStats2 = maybe_report_stats(State#state.cp, NewStats), {noreply, State#state{stats = NewStats2}}; - -handle_call(flush, {Pid, _} = From, - #state{loop = Pid, writer = nil, flush_waiter = nil, - target = Target, batch = Batch} = State) -> - State2 = case State#state.readers of - [] -> - State#state{writer = spawn_writer(Target, Batch)}; - _ -> - State - end, +handle_call( + flush, + {Pid, _} = From, + #state{ + loop = Pid, + writer = nil, + flush_waiter = nil, + target = Target, + batch = Batch + } = State +) -> + State2 = + case State#state.readers of + [] -> + State#state{writer = spawn_writer(Target, Batch)}; + _ -> + State + end, {noreply, State2#state{flush_waiter = From}}. - handle_cast(Msg, State) -> {stop, {unexpected_async_call, Msg}, State}. - handle_info({'EXIT', Pid, normal}, #state{loop = Pid} = State) -> #state{ - batch = #batch{docs = []}, readers = [], writer = nil, - pending_fetch = nil, flush_waiter = nil + batch = #batch{docs = []}, + readers = [], + writer = nil, + pending_fetch = nil, + flush_waiter = nil } = State, {stop, normal, State}; - handle_info({'EXIT', Pid, normal}, #state{writer = Pid} = State) -> {noreply, after_full_flush(State)}; - handle_info({'EXIT', Pid, normal}, #state{writer = nil} = State) -> #state{ - readers = Readers, writer = Writer, batch = Batch, - source = Source, target = Target, - pending_fetch = Fetch, flush_waiter = FlushWaiter + readers = Readers, + writer = Writer, + batch = Batch, + source = Source, + target = Target, + pending_fetch = Fetch, + flush_waiter = FlushWaiter } = State, case Readers -- [Pid] of - Readers -> - {noreply, State}; - Readers2 -> - State2 = case Fetch of - nil -> - case (FlushWaiter =/= nil) andalso (Writer =:= nil) andalso - (Readers2 =:= []) of - true -> - State#state{ - readers = Readers2, - writer = spawn_writer(Target, Batch) - }; - false -> - State#state{readers = Readers2} - end; - {From, FetchParams} -> - Reader = spawn_doc_reader(Source, Target, FetchParams), - gen_server:reply(From, ok), - State#state{ - readers = [Reader | Readers2], - pending_fetch = nil - } - end, - {noreply, State2} + Readers -> + {noreply, State}; + Readers2 -> + State2 = + case Fetch of + nil -> + case + (FlushWaiter =/= nil) andalso (Writer =:= nil) andalso + (Readers2 =:= []) + of + true -> + State#state{ + readers = Readers2, + writer = spawn_writer(Target, Batch) + }; + false -> + State#state{readers = Readers2} + end; + {From, FetchParams} -> + Reader = spawn_doc_reader(Source, Target, FetchParams), + gen_server:reply(From, ok), + State#state{ + readers = [Reader | Readers2], + pending_fetch = nil + } + end, + {noreply, State2} end; - handle_info({'EXIT', _Pid, max_backoff}, State) -> {stop, {shutdown, max_backoff}, State}; - handle_info({'EXIT', _Pid, {bulk_docs_failed, _, _} = Err}, State) -> {stop, {shutdown, Err}, State}; - handle_info({'EXIT', _Pid, {revs_diff_failed, _, _} = Err}, State) -> {stop, {shutdown, Err}, State}; - handle_info({'EXIT', _Pid, {http_request_failed, _, _, _} = Err}, State) -> {stop, {shutdown, Err}, State}; - handle_info({'EXIT', Pid, Reason}, State) -> - {stop, {process_died, Pid, Reason}, State}. - + {stop, {process_died, Pid, Reason}, State}. terminate(_Reason, _State) -> ok. @@ -208,31 +222,28 @@ format_status(_Opt, [_PDict, State]) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. - queue_fetch_loop(Source, Target, Parent, Cp, ChangesManager) -> ChangesManager ! {get_changes, self()}, receive - {closed, ChangesManager} -> - ok; - {changes, ChangesManager, [], ReportSeq} -> - Stats = couch_replicator_stats:new(), - ok = gen_server:call(Cp, {report_seq_done, ReportSeq, Stats}, infinity), - queue_fetch_loop(Source, Target, Parent, Cp, ChangesManager); - {changes, ChangesManager, Changes, ReportSeq} -> - {IdRevs, Stats0} = find_missing(Changes, Target), - ok = gen_server:call(Parent, {add_stats, Stats0}, infinity), - remote_process_batch(IdRevs, Parent), - {ok, Stats} = gen_server:call(Parent, flush, infinity), - ok = gen_server:call(Cp, {report_seq_done, ReportSeq, Stats}, infinity), - erlang:put(last_stats_report, os:timestamp()), - couch_log:debug("Worker reported completion of seq ~p", [ReportSeq]), - queue_fetch_loop(Source, Target, Parent, Cp, ChangesManager) + {closed, ChangesManager} -> + ok; + {changes, ChangesManager, [], ReportSeq} -> + Stats = couch_replicator_stats:new(), + ok = gen_server:call(Cp, {report_seq_done, ReportSeq, Stats}, infinity), + queue_fetch_loop(Source, Target, Parent, Cp, ChangesManager); + {changes, ChangesManager, Changes, ReportSeq} -> + {IdRevs, Stats0} = find_missing(Changes, Target), + ok = gen_server:call(Parent, {add_stats, Stats0}, infinity), + remote_process_batch(IdRevs, Parent), + {ok, Stats} = gen_server:call(Parent, flush, infinity), + ok = gen_server:call(Cp, {report_seq_done, ReportSeq, Stats}, infinity), + erlang:put(last_stats_report, os:timestamp()), + couch_log:debug("Worker reported completion of seq ~p", [ReportSeq]), + queue_fetch_loop(Source, Target, Parent, Cp, ChangesManager) end. - remote_process_batch([], _Parent) -> ok; - remote_process_batch([{Id, Revs, PAs} | Rest], Parent) -> % When the source is a remote database, we fetch a single document revision % per HTTP request. This is mostly to facilitate retrying of HTTP requests @@ -242,44 +253,56 @@ remote_process_batch([{Id, Revs, PAs} | Rest], Parent) -> fun(Rev) -> ok = gen_server:call(Parent, {fetch_doc, {Id, [Rev], PAs}}, infinity) end, - Revs), + Revs + ), remote_process_batch(Rest, Parent). - spawn_doc_reader(Source, Target, FetchParams) -> Parent = self(), spawn_link(fun() -> fetch_doc( - Source, FetchParams, fun remote_doc_handler/2, {Parent, Target}) + Source, FetchParams, fun remote_doc_handler/2, {Parent, Target} + ) end). - fetch_doc(Source, {Id, Revs, PAs}, DocHandler, Acc) -> try couch_replicator_api_wrap:open_doc_revs( - Source, Id, Revs, [{atts_since, PAs}, latest], DocHandler, Acc) + Source, Id, Revs, [{atts_since, PAs}, latest], DocHandler, Acc + ) catch - throw:missing_doc -> - couch_log:error("Retrying fetch and update of document `~s` as it is " - "unexpectedly missing. Missing revisions are: ~s", - [Id, couch_doc:revs_to_strs(Revs)]), - WaitMSec = config:get_integer("replicator", "missing_doc_retry_msec", - ?MISSING_DOC_RETRY_MSEC), - timer:sleep(WaitMSec), - couch_replicator_api_wrap:open_doc_revs(Source, Id, Revs, [latest], DocHandler, Acc); - throw:{missing_stub, _} -> - couch_log:error("Retrying fetch and update of document `~s` due to out of " - "sync attachment stubs. Missing revisions are: ~s", - [Id, couch_doc:revs_to_strs(Revs)]), - WaitMSec = config:get_integer("replicator", "missing_doc_retry_msec", - ?MISSING_DOC_RETRY_MSEC), - timer:sleep(WaitMSec), - couch_replicator_api_wrap:open_doc_revs(Source, Id, Revs, [latest], DocHandler, Acc) + throw:missing_doc -> + couch_log:error( + "Retrying fetch and update of document `~s` as it is " + "unexpectedly missing. Missing revisions are: ~s", + [Id, couch_doc:revs_to_strs(Revs)] + ), + WaitMSec = config:get_integer( + "replicator", + "missing_doc_retry_msec", + ?MISSING_DOC_RETRY_MSEC + ), + timer:sleep(WaitMSec), + couch_replicator_api_wrap:open_doc_revs(Source, Id, Revs, [latest], DocHandler, Acc); + throw:{missing_stub, _} -> + couch_log:error( + "Retrying fetch and update of document `~s` due to out of " + "sync attachment stubs. Missing revisions are: ~s", + [Id, couch_doc:revs_to_strs(Revs)] + ), + WaitMSec = config:get_integer( + "replicator", + "missing_doc_retry_msec", + ?MISSING_DOC_RETRY_MSEC + ), + timer:sleep(WaitMSec), + couch_replicator_api_wrap:open_doc_revs(Source, Id, Revs, [latest], DocHandler, Acc) end. - -remote_doc_handler({ok, #doc{id = <<?DESIGN_DOC_PREFIX, _/binary>>} = Doc}, - Acc) -> +remote_doc_handler( + {ok, #doc{id = <<?DESIGN_DOC_PREFIX, _/binary>>} = Doc}, + Acc +) -> % Flush design docs in their own PUT requests to correctly process % authorization failures for design doc updates. couch_log:debug("Worker flushing design doc", []), @@ -297,34 +320,33 @@ remote_doc_handler({ok, #doc{atts = []} = Doc}, {Parent, _} = Acc) -> remote_doc_handler({{not_found, missing}, _}, _Acc) -> throw(missing_doc). - doc_handler_flush_doc(#doc{} = Doc, {Parent, Target} = Acc) -> Stats = couch_replicator_stats:new([{docs_read, 1}]), Success = (flush_doc(Target, Doc) =:= ok), - {Result, Stats2} = case Success of - true -> - {{ok, Acc}, couch_replicator_stats:increment(docs_written, Stats)}; - false -> - {{skip, Acc}, couch_replicator_stats:increment(doc_write_failures, Stats)} - end, + {Result, Stats2} = + case Success of + true -> + {{ok, Acc}, couch_replicator_stats:increment(docs_written, Stats)}; + false -> + {{skip, Acc}, couch_replicator_stats:increment(doc_write_failures, Stats)} + end, ok = gen_server:call(Parent, {add_stats, Stats2}, infinity), Result. - spawn_writer(Target, #batch{docs = DocList, size = Size}) -> case {Target, Size > 0} of - {#httpdb{}, true} -> - couch_log:debug("Worker flushing doc batch of size ~p bytes", [Size]); - _ -> - ok + {#httpdb{}, true} -> + couch_log:debug("Worker flushing doc batch of size ~p bytes", [Size]); + _ -> + ok end, Parent = self(), spawn_link( fun() -> Stats = flush_docs(Target, DocList), ok = gen_server:call(Parent, {add_stats, Stats}, infinity) - end). - + end + ). after_full_flush(#state{stats = Stats, flush_waiter = Waiter} = State) -> gen_server:reply(Waiter, {ok, Stats}), @@ -336,11 +358,12 @@ after_full_flush(#state{stats = Stats, flush_waiter = Waiter} = State) -> batch = #batch{} }. - -maybe_flush_docs(Doc,State) -> +maybe_flush_docs(Doc, State) -> #state{ - target = Target, batch = Batch, - stats = Stats, cp = Cp + target = Target, + batch = Batch, + stats = Stats, + cp = Cp } = State, {Batch2, WStats} = maybe_flush_docs(Target, Batch, Doc), Stats2 = couch_replicator_stats:sum_stats(Stats, WStats), @@ -348,39 +371,47 @@ maybe_flush_docs(Doc,State) -> Stats4 = maybe_report_stats(Cp, Stats3), State#state{stats = Stats4, batch = Batch2}. - maybe_flush_docs(#httpdb{} = Target, Batch, Doc) -> #batch{docs = DocAcc, size = SizeAcc} = Batch, JsonDoc = ?JSON_ENCODE(couch_doc:to_json_obj(Doc, [revs, attachments])), case SizeAcc + iolist_size(JsonDoc) of - SizeAcc2 when SizeAcc2 > ?DOC_BUFFER_BYTE_SIZE -> - couch_log:debug("Worker flushing doc batch of size ~p bytes", [SizeAcc2]), - Stats = flush_docs(Target, [JsonDoc | DocAcc]), - {#batch{}, Stats}; - SizeAcc2 -> - Stats = couch_replicator_stats:new(), - {#batch{docs = [JsonDoc | DocAcc], size = SizeAcc2}, Stats} + SizeAcc2 when SizeAcc2 > ?DOC_BUFFER_BYTE_SIZE -> + couch_log:debug("Worker flushing doc batch of size ~p bytes", [SizeAcc2]), + Stats = flush_docs(Target, [JsonDoc | DocAcc]), + {#batch{}, Stats}; + SizeAcc2 -> + Stats = couch_replicator_stats:new(), + {#batch{docs = [JsonDoc | DocAcc], size = SizeAcc2}, Stats} end. - flush_docs(_Target, []) -> couch_replicator_stats:new(); flush_docs(Target, DocList) -> - FlushResult = couch_replicator_api_wrap:update_docs(Target, DocList, - [delay_commit], replicated_changes), + FlushResult = couch_replicator_api_wrap:update_docs( + Target, + DocList, + [delay_commit], + replicated_changes + ), handle_flush_docs_result(FlushResult, Target, DocList). - handle_flush_docs_result({error, request_body_too_large}, _Target, [Doc]) -> couch_log:error("Replicator: failed to write doc ~p. Too large", [Doc]), couch_replicator_stats:new([{doc_write_failures, 1}]); handle_flush_docs_result({error, request_body_too_large}, Target, DocList) -> Len = length(DocList), {DocList1, DocList2} = lists:split(Len div 2, DocList), - couch_log:notice("Replicator: couldn't write batch of size ~p to ~p because" + couch_log:notice( + "Replicator: couldn't write batch of size ~p to ~p because" " request body is too large. Splitting batch into 2 separate batches of" - " sizes ~p and ~p", [Len, couch_replicator_api_wrap:db_uri(Target), - length(DocList1), length(DocList2)]), + " sizes ~p and ~p", + [ + Len, + couch_replicator_api_wrap:db_uri(Target), + length(DocList1), + length(DocList2) + ] + ), Stats1 = flush_docs(Target, DocList1), Stats2 = flush_docs(Target, DocList2), couch_replicator_stats:sum_stats(Stats1, Stats2); @@ -388,11 +419,20 @@ handle_flush_docs_result({ok, Errors}, Target, DocList) -> DbUri = couch_replicator_api_wrap:db_uri(Target), lists:foreach( fun({Props}) -> - couch_log:error("Replicator: couldn't write document `~s`, revision" - " `~s`, to target database `~s`. Error: `~s`, reason: `~s`.", [ - get_value(id, Props, ""), get_value(rev, Props, ""), DbUri, - get_value(error, Props, ""), get_value(reason, Props, "")]) - end, Errors), + couch_log:error( + "Replicator: couldn't write document `~s`, revision" + " `~s`, to target database `~s`. Error: `~s`, reason: `~s`.", + [ + get_value(id, Props, ""), + get_value(rev, Props, ""), + DbUri, + get_value(error, Props, ""), + get_value(reason, Props, "") + ] + ) + end, + Errors + ), couch_replicator_stats:new([ {docs_written, length(DocList) - length(Errors)}, {doc_write_failures, length(Errors)} @@ -400,74 +440,90 @@ handle_flush_docs_result({ok, Errors}, Target, DocList) -> handle_flush_docs_result({error, {bulk_docs_failed, _, _} = Err}, _, _) -> exit(Err). - flush_doc(Target, #doc{id = Id, revs = {Pos, [RevId | _]}} = Doc) -> try couch_replicator_api_wrap:update_doc(Target, Doc, [], replicated_changes) of - {ok, _} -> - ok; - Error -> - couch_log:error("Replicator: error writing document `~s` to `~s`: ~s", - [Id, couch_replicator_api_wrap:db_uri(Target), couch_util:to_binary(Error)]), - Error + {ok, _} -> + ok; + Error -> + couch_log:error( + "Replicator: error writing document `~s` to `~s`: ~s", + [Id, couch_replicator_api_wrap:db_uri(Target), couch_util:to_binary(Error)] + ), + Error catch - throw:{missing_stub, _} = MissingStub -> - throw(MissingStub); - throw:{Error, Reason} -> - couch_log:error("Replicator: couldn't write document `~s`, revision `~s`," - " to target database `~s`. Error: `~s`, reason: `~s`.", - [Id, couch_doc:rev_to_str({Pos, RevId}), - couch_replicator_api_wrap:db_uri(Target), to_binary(Error), to_binary(Reason)]), - {error, Error}; - throw:Err -> - couch_log:error("Replicator: couldn't write document `~s`, revision `~s`," - " to target database `~s`. Error: `~s`.", - [Id, couch_doc:rev_to_str({Pos, RevId}), - couch_replicator_api_wrap:db_uri(Target), to_binary(Err)]), - {error, Err} + throw:{missing_stub, _} = MissingStub -> + throw(MissingStub); + throw:{Error, Reason} -> + couch_log:error( + "Replicator: couldn't write document `~s`, revision `~s`," + " to target database `~s`. Error: `~s`, reason: `~s`.", + [ + Id, + couch_doc:rev_to_str({Pos, RevId}), + couch_replicator_api_wrap:db_uri(Target), + to_binary(Error), + to_binary(Reason) + ] + ), + {error, Error}; + throw:Err -> + couch_log:error( + "Replicator: couldn't write document `~s`, revision `~s`," + " to target database `~s`. Error: `~s`.", + [ + Id, + couch_doc:rev_to_str({Pos, RevId}), + couch_replicator_api_wrap:db_uri(Target), + to_binary(Err) + ] + ), + {error, Err} end. - find_missing(DocInfos, Target) -> - {IdRevs, AllRevsCount} = lists:foldr(fun - (#doc_info{revs = []}, {IdRevAcc, CountAcc}) -> - {IdRevAcc, CountAcc}; - (#doc_info{id = Id, revs = RevsInfo}, {IdRevAcc, CountAcc}) -> - Revs = [Rev || #rev_info{rev = Rev} <- RevsInfo], - {[{Id, Revs} | IdRevAcc], CountAcc + length(Revs)} - end, {[], 0}, DocInfos), - - - Missing = case couch_replicator_api_wrap:get_missing_revs(Target, IdRevs) of - {ok, Result} -> Result; - {error, Error} -> exit(Error) - end, + {IdRevs, AllRevsCount} = lists:foldr( + fun + (#doc_info{revs = []}, {IdRevAcc, CountAcc}) -> + {IdRevAcc, CountAcc}; + (#doc_info{id = Id, revs = RevsInfo}, {IdRevAcc, CountAcc}) -> + Revs = [Rev || #rev_info{rev = Rev} <- RevsInfo], + {[{Id, Revs} | IdRevAcc], CountAcc + length(Revs)} + end, + {[], 0}, + DocInfos + ), + + Missing = + case couch_replicator_api_wrap:get_missing_revs(Target, IdRevs) of + {ok, Result} -> Result; + {error, Error} -> exit(Error) + end, MissingRevsCount = lists:foldl( fun({_Id, MissingRevs, _PAs}, Acc) -> Acc + length(MissingRevs) end, - 0, Missing), + 0, + Missing + ), Stats = couch_replicator_stats:new([ {missing_checked, AllRevsCount}, {missing_found, MissingRevsCount} ]), {Missing, Stats}. - maybe_report_stats(Cp, Stats) -> Now = os:timestamp(), case timer:now_diff(erlang:get(last_stats_report), Now) >= ?STATS_DELAY of - true -> - ok = gen_server:call(Cp, {add_stats, Stats}, infinity), - erlang:put(last_stats_report, Now), - couch_replicator_stats:new(); - false -> - Stats + true -> + ok = gen_server:call(Cp, {add_stats, Stats}, infinity), + erlang:put(last_stats_report, Now), + couch_replicator_stats:new(); + false -> + Stats end. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). - replication_worker_format_status_test() -> State = #state{ cp = self(), diff --git a/src/couch_replicator/src/json_stream_parse.erl b/src/couch_replicator/src/json_stream_parse.erl index b63e01152..3478b9830 100644 --- a/src/couch_replicator/src/json_stream_parse.erl +++ b/src/couch_replicator/src/json_stream_parse.erl @@ -12,15 +12,12 @@ -module(json_stream_parse). - -export([events/2, to_ejson/1, collect_object/2]). -define(IS_WS(X), (X == $\ orelse X == $\t orelse X == $\n orelse X == $\r)). -define(IS_DELIM(X), (X == $} orelse X == $] orelse X == $,)). -define(IS_DIGIT(X), (X >= $0 andalso X =< $9)). - - % Parses the json into events. % % The DataFun param is a function that produces the data for parsing. When @@ -49,11 +46,11 @@ % {key, KeyString}, and the following event is the value, or start of the % value (array_start, object_start). % -events(Data,EventFun) when is_list(Data)-> - events(list_to_binary(Data),EventFun); -events(Data,EventFun) when is_binary(Data)-> - events(fun() -> {Data, fun() -> done end} end,EventFun); -events(DataFun,EventFun) -> +events(Data, EventFun) when is_list(Data) -> + events(list_to_binary(Data), EventFun); +events(Data, EventFun) when is_binary(Data) -> + events(fun() -> {Data, fun() -> done end} end, EventFun); +events(DataFun, EventFun) -> parse_one(DataFun, EventFun, <<>>). % converts the JSON directly to the erlang represention of Json @@ -62,7 +59,6 @@ to_ejson(DF) -> [[EJson]] = make_ejson(EF(get_results), [[]]), EJson. - % This function is used to return complete objects while parsing streams. % % Return this function from inside an event function right after getting an @@ -98,228 +94,224 @@ to_ejson(DF) -> collect_object(Ev, ReturnControl) -> collect_object(Ev, 0, ReturnControl, [object_start]). - - % internal methods -parse_one(DF,EF,Acc) -> +parse_one(DF, EF, Acc) -> case toke(DF, Acc) of - none -> - none; - {Token, DF2, Rest} -> - case Token of - "{" -> - EF2 = EF(object_start), - {DF3, EF3, Rest2} = parse_object(DF2, EF2, Rest), - {DF3, EF3(object_end), Rest2}; - "[" -> - EF2 = EF(array_start), - {DF3, EF3, Rest2} = parse_array(DF2, EF2, Rest), - {DF3, EF3(array_end), Rest2}; - Int when is_integer(Int)-> - {DF2, EF(Int), Rest}; - Float when is_float(Float)-> - {DF2, EF(Float), Rest}; - Atom when is_atom(Atom)-> - {DF2, EF(Atom), Rest}; - String when is_binary(String)-> - {DF2, EF(String), Rest}; - _OtherToken -> - err(unexpected_token) - end + none -> + none; + {Token, DF2, Rest} -> + case Token of + "{" -> + EF2 = EF(object_start), + {DF3, EF3, Rest2} = parse_object(DF2, EF2, Rest), + {DF3, EF3(object_end), Rest2}; + "[" -> + EF2 = EF(array_start), + {DF3, EF3, Rest2} = parse_array(DF2, EF2, Rest), + {DF3, EF3(array_end), Rest2}; + Int when is_integer(Int) -> + {DF2, EF(Int), Rest}; + Float when is_float(Float) -> + {DF2, EF(Float), Rest}; + Atom when is_atom(Atom) -> + {DF2, EF(Atom), Rest}; + String when is_binary(String) -> + {DF2, EF(String), Rest}; + _OtherToken -> + err(unexpected_token) + end end. -must_parse_one(DF,EF,Acc,Error)-> +must_parse_one(DF, EF, Acc, Error) -> case parse_one(DF, EF, Acc) of - none -> - err(Error); - Else -> - Else + none -> + err(Error); + Else -> + Else end. must_toke(DF, Data, Error) -> case toke(DF, Data) of - none -> - err(Error); - Result -> - Result + none -> + err(Error); + Result -> + Result end. toke(DF, <<>>) -> case DF() of - done -> - none; - {Data, DF2} -> - toke(DF2, Data) + done -> + none; + {Data, DF2} -> + toke(DF2, Data) end; -toke(DF, <<C,Rest/binary>>) when ?IS_WS(C)-> +toke(DF, <<C, Rest/binary>>) when ?IS_WS(C) -> toke(DF, Rest); -toke(DF, <<${,Rest/binary>>) -> +toke(DF, <<${, Rest/binary>>) -> {"{", DF, Rest}; -toke(DF, <<$},Rest/binary>>) -> +toke(DF, <<$}, Rest/binary>>) -> {"}", DF, Rest}; -toke(DF, <<$[,Rest/binary>>) -> +toke(DF, <<$[, Rest/binary>>) -> {"[", DF, Rest}; -toke(DF, <<$],Rest/binary>>) -> +toke(DF, <<$], Rest/binary>>) -> {"]", DF, Rest}; -toke(DF, <<$",Rest/binary>>) -> - toke_string(DF,Rest,[]); -toke(DF, <<$,,Rest/binary>>) -> +toke(DF, <<$", Rest/binary>>) -> + toke_string(DF, Rest, []); +toke(DF, <<$,, Rest/binary>>) -> {",", DF, Rest}; -toke(DF, <<$:,Rest/binary>>) -> +toke(DF, <<$:, Rest/binary>>) -> {":", DF, Rest}; -toke(DF, <<$-,Rest/binary>>) -> - {<<C,_/binary>> = Data, DF2} = must_df(DF,1,Rest,expected_number), +toke(DF, <<$-, Rest/binary>>) -> + {<<C, _/binary>> = Data, DF2} = must_df(DF, 1, Rest, expected_number), case ?IS_DIGIT(C) of - true -> - toke_number_leading(DF2, Data, "-"); - false -> - err(expected_number) + true -> + toke_number_leading(DF2, Data, "-"); + false -> + err(expected_number) end; -toke(DF, <<C,_/binary>> = Data) when ?IS_DIGIT(C) -> +toke(DF, <<C, _/binary>> = Data) when ?IS_DIGIT(C) -> toke_number_leading(DF, Data, []); -toke(DF, <<$t,Rest/binary>>) -> +toke(DF, <<$t, Rest/binary>>) -> {Data, DF2} = must_match(<<"rue">>, DF, Rest), {true, DF2, Data}; -toke(DF, <<$f,Rest/binary>>) -> +toke(DF, <<$f, Rest/binary>>) -> {Data, DF2} = must_match(<<"alse">>, DF, Rest), {false, DF2, Data}; -toke(DF, <<$n,Rest/binary>>) -> +toke(DF, <<$n, Rest/binary>>) -> {Data, DF2} = must_match(<<"ull">>, DF, Rest), {null, DF2, Data}; toke(_, _) -> err(bad_token). - must_match(Pattern, DF, Data) -> Size = size(Pattern), case must_df(DF, Size, Data, bad_token) of - {<<Pattern:Size/binary,Data2/binary>>, DF2} -> - {Data2, DF2}; - {_, _} -> - err(bad_token) + {<<Pattern:Size/binary, Data2/binary>>, DF2} -> + {Data2, DF2}; + {_, _} -> + err(bad_token) end. -must_df(DF,Error)-> +must_df(DF, Error) -> case DF() of - done -> - err(Error); - {Data, DF2} -> - {Data, DF2} - end. - - -must_df(DF,NeedLen,Acc,Error)-> - if size(Acc) >= NeedLen -> - {Acc, DF}; - true -> - case DF() of done -> err(Error); {Data, DF2} -> - must_df(DF2, NeedLen, <<Acc/binary, Data/binary>>, Error) - end + {Data, DF2} end. +must_df(DF, NeedLen, Acc, Error) -> + if + size(Acc) >= NeedLen -> + {Acc, DF}; + true -> + case DF() of + done -> + err(Error); + {Data, DF2} -> + must_df(DF2, NeedLen, <<Acc/binary, Data/binary>>, Error) + end + end. -parse_object(DF,EF,Acc) -> +parse_object(DF, EF, Acc) -> case must_toke(DF, Acc, unterminated_object) of - {String, DF2, Rest} when is_binary(String)-> - EF2 = EF({key,String}), - case must_toke(DF2,Rest,unterminated_object) of - {":", DF3, Rest2} -> - {DF4, EF3, Rest3} = must_parse_one(DF3, EF2, Rest2, expected_value), - case must_toke(DF4,Rest3, unterminated_object) of - {",", DF5, Rest4} -> - parse_object(DF5, EF3, Rest4); - {"}", DF5, Rest4} -> - {DF5, EF3, Rest4}; - {_, _, _} -> - err(unexpected_token) + {String, DF2, Rest} when is_binary(String) -> + EF2 = EF({key, String}), + case must_toke(DF2, Rest, unterminated_object) of + {":", DF3, Rest2} -> + {DF4, EF3, Rest3} = must_parse_one(DF3, EF2, Rest2, expected_value), + case must_toke(DF4, Rest3, unterminated_object) of + {",", DF5, Rest4} -> + parse_object(DF5, EF3, Rest4); + {"}", DF5, Rest4} -> + {DF5, EF3, Rest4}; + {_, _, _} -> + err(unexpected_token) + end; + _Else -> + err(expected_colon) end; - _Else -> - err(expected_colon) - end; - {"}", DF2, Rest} -> - {DF2, EF, Rest}; - {_, _, _} -> - err(unexpected_token) - end. - -parse_array0(DF,EF,Acc) -> - case toke(DF, Acc) of - none -> - err(unterminated_array); - {",", DF2, Rest} -> - parse_array(DF2,EF,Rest); - {"]", DF2, Rest} -> - {DF2,EF,Rest}; - _ -> - err(unexpected_token) + {"}", DF2, Rest} -> + {DF2, EF, Rest}; + {_, _, _} -> + err(unexpected_token) end. -parse_array(DF,EF,Acc) -> +parse_array0(DF, EF, Acc) -> case toke(DF, Acc) of - none -> - err(unterminated_array); - {Token, DF2, Rest} -> - case Token of - "{" -> - EF2 = EF(object_start), - {DF3, EF3, Rest2} = parse_object(DF2, EF2, Rest), - parse_array0(DF3, EF3(object_end), Rest2); - "[" -> - EF2 = EF(array_start), - {DF3, EF3, Rest2} = parse_array(DF2, EF2, Rest), - parse_array0(DF3, EF3(array_end), Rest2); - Int when is_integer(Int)-> - parse_array0(DF2, EF(Int), Rest); - Float when is_float(Float)-> - parse_array0(DF2, EF(Float), Rest); - Atom when is_atom(Atom)-> - parse_array0(DF2, EF(Atom), Rest); - String when is_binary(String)-> - parse_array0(DF2, EF(String), Rest); - "]" -> + none -> + err(unterminated_array); + {",", DF2, Rest} -> + parse_array(DF2, EF, Rest); + {"]", DF2, Rest} -> {DF2, EF, Rest}; _ -> err(unexpected_token) - end end. +parse_array(DF, EF, Acc) -> + case toke(DF, Acc) of + none -> + err(unterminated_array); + {Token, DF2, Rest} -> + case Token of + "{" -> + EF2 = EF(object_start), + {DF3, EF3, Rest2} = parse_object(DF2, EF2, Rest), + parse_array0(DF3, EF3(object_end), Rest2); + "[" -> + EF2 = EF(array_start), + {DF3, EF3, Rest2} = parse_array(DF2, EF2, Rest), + parse_array0(DF3, EF3(array_end), Rest2); + Int when is_integer(Int) -> + parse_array0(DF2, EF(Int), Rest); + Float when is_float(Float) -> + parse_array0(DF2, EF(Float), Rest); + Atom when is_atom(Atom) -> + parse_array0(DF2, EF(Atom), Rest); + String when is_binary(String) -> + parse_array0(DF2, EF(String), Rest); + "]" -> + {DF2, EF, Rest}; + _ -> + err(unexpected_token) + end + end. toke_string(DF, <<>>, Acc) -> {Data, DF2} = must_df(DF, unterminated_string), toke_string(DF2, Data, Acc); -toke_string(DF, <<$\\,$",Rest/binary>>, Acc) -> +toke_string(DF, <<$\\, $", Rest/binary>>, Acc) -> toke_string(DF, Rest, [$" | Acc]); -toke_string(DF, <<$\\,$\\,Rest/binary>>, Acc) -> +toke_string(DF, <<$\\, $\\, Rest/binary>>, Acc) -> toke_string(DF, Rest, [$\\ | Acc]); -toke_string(DF, <<$\\,$/,Rest/binary>>, Acc) -> +toke_string(DF, <<$\\, $/, Rest/binary>>, Acc) -> toke_string(DF, Rest, [$/ | Acc]); -toke_string(DF, <<$\\,$b,Rest/binary>>, Acc) -> +toke_string(DF, <<$\\, $b, Rest/binary>>, Acc) -> toke_string(DF, Rest, [$\b | Acc]); -toke_string(DF, <<$\\,$f,Rest/binary>>, Acc) -> +toke_string(DF, <<$\\, $f, Rest/binary>>, Acc) -> toke_string(DF, Rest, [$\f | Acc]); -toke_string(DF, <<$\\,$n,Rest/binary>>, Acc) -> +toke_string(DF, <<$\\, $n, Rest/binary>>, Acc) -> toke_string(DF, Rest, [$\n | Acc]); -toke_string(DF, <<$\\,$r,Rest/binary>>, Acc) -> +toke_string(DF, <<$\\, $r, Rest/binary>>, Acc) -> toke_string(DF, Rest, [$\r | Acc]); -toke_string(DF, <<$\\,$t,Rest/binary>>, Acc) -> +toke_string(DF, <<$\\, $t, Rest/binary>>, Acc) -> toke_string(DF, Rest, [$\t | Acc]); -toke_string(DF, <<$\\,$u,Rest/binary>>, Acc) -> - {<<A,B,C,D,Data/binary>>, DF2} = must_df(DF,4,Rest,missing_hex), +toke_string(DF, <<$\\, $u, Rest/binary>>, Acc) -> + {<<A, B, C, D, Data/binary>>, DF2} = must_df(DF, 4, Rest, missing_hex), UTFChar = erlang:list_to_integer([A, B, C, D], 16), - if UTFChar == 16#FFFF orelse UTFChar == 16#FFFE -> - err(invalid_utf_char); - true -> - ok + if + UTFChar == 16#FFFF orelse UTFChar == 16#FFFE -> + err(invalid_utf_char); + true -> + ok end, Chars = xmerl_ucs:to_utf8(UTFChar), toke_string(DF2, Data, lists:reverse(Chars) ++ Acc); toke_string(DF, <<$\\>>, Acc) -> {Data, DF2} = must_df(DF, unterminated_string), - toke_string(DF2, <<$\\,Data/binary>>, Acc); + toke_string(DF2, <<$\\, Data/binary>>, Acc); toke_string(_DF, <<$\\, _/binary>>, _Acc) -> err(bad_escape); toke_string(DF, <<$", Rest/binary>>, Acc) -> @@ -327,72 +319,74 @@ toke_string(DF, <<$", Rest/binary>>, Acc) -> toke_string(DF, <<C, Rest/binary>>, Acc) -> toke_string(DF, Rest, [C | Acc]). - -toke_number_leading(DF, <<Digit,Rest/binary>>, Acc) - when ?IS_DIGIT(Digit) -> +toke_number_leading(DF, <<Digit, Rest/binary>>, Acc) when + ?IS_DIGIT(Digit) +-> toke_number_leading(DF, Rest, [Digit | Acc]); -toke_number_leading(DF, <<C,_/binary>>=Rest, Acc) - when ?IS_WS(C) orelse ?IS_DELIM(C) -> +toke_number_leading(DF, <<C, _/binary>> = Rest, Acc) when + ?IS_WS(C) orelse ?IS_DELIM(C) +-> {list_to_integer(lists:reverse(Acc)), DF, Rest}; toke_number_leading(DF, <<>>, Acc) -> case DF() of - done -> - {list_to_integer(lists:reverse(Acc)), fun() -> done end, <<>>}; - {Data, DF2} -> - toke_number_leading(DF2, Data, Acc) + done -> + {list_to_integer(lists:reverse(Acc)), fun() -> done end, <<>>}; + {Data, DF2} -> + toke_number_leading(DF2, Data, Acc) end; toke_number_leading(DF, <<$., Rest/binary>>, Acc) -> - toke_number_trailing(DF, Rest, [$.|Acc]); + toke_number_trailing(DF, Rest, [$. | Acc]); toke_number_leading(DF, <<$e, Rest/binary>>, Acc) -> - toke_number_exponent(DF, Rest, [$e, $0, $.|Acc]); + toke_number_exponent(DF, Rest, [$e, $0, $. | Acc]); toke_number_leading(DF, <<$E, Rest/binary>>, Acc) -> - toke_number_exponent(DF, Rest, [$e, $0, $.|Acc]); + toke_number_exponent(DF, Rest, [$e, $0, $. | Acc]); toke_number_leading(_, _, _) -> err(unexpected_character_in_number). -toke_number_trailing(DF, <<Digit,Rest/binary>>, Acc) - when ?IS_DIGIT(Digit) -> +toke_number_trailing(DF, <<Digit, Rest/binary>>, Acc) when + ?IS_DIGIT(Digit) +-> toke_number_trailing(DF, Rest, [Digit | Acc]); -toke_number_trailing(DF, <<C,_/binary>>=Rest, Acc) - when ?IS_WS(C) orelse ?IS_DELIM(C) -> +toke_number_trailing(DF, <<C, _/binary>> = Rest, Acc) when + ?IS_WS(C) orelse ?IS_DELIM(C) +-> {list_to_float(lists:reverse(Acc)), DF, Rest}; toke_number_trailing(DF, <<>>, Acc) -> case DF() of - done -> - {list_to_float(lists:reverse(Acc)), fun() -> done end, <<>>}; - {Data, DF2} -> - toke_number_trailing(DF2, Data, Acc) + done -> + {list_to_float(lists:reverse(Acc)), fun() -> done end, <<>>}; + {Data, DF2} -> + toke_number_trailing(DF2, Data, Acc) end; -toke_number_trailing(DF, <<"e", Rest/binary>>, [C|_]=Acc) when C /= $. -> - toke_number_exponent(DF, Rest, [$e|Acc]); -toke_number_trailing(DF, <<"E", Rest/binary>>, [C|_]=Acc) when C /= $. -> - toke_number_exponent(DF, Rest, [$e|Acc]); +toke_number_trailing(DF, <<"e", Rest/binary>>, [C | _] = Acc) when C /= $. -> + toke_number_exponent(DF, Rest, [$e | Acc]); +toke_number_trailing(DF, <<"E", Rest/binary>>, [C | _] = Acc) when C /= $. -> + toke_number_exponent(DF, Rest, [$e | Acc]); toke_number_trailing(_, _, _) -> err(unexpected_character_in_number). - -toke_number_exponent(DF, <<Digit,Rest/binary>>, Acc) when ?IS_DIGIT(Digit) -> +toke_number_exponent(DF, <<Digit, Rest/binary>>, Acc) when ?IS_DIGIT(Digit) -> toke_number_exponent(DF, Rest, [Digit | Acc]); -toke_number_exponent(DF, <<Sign,Rest/binary>>, [$e|_]=Acc) - when Sign == $+ orelse Sign == $- -> +toke_number_exponent(DF, <<Sign, Rest/binary>>, [$e | _] = Acc) when + Sign == $+ orelse Sign == $- +-> toke_number_exponent(DF, Rest, [Sign | Acc]); -toke_number_exponent(DF, <<C,_/binary>>=Rest, Acc) - when ?IS_WS(C) orelse ?IS_DELIM(C) -> +toke_number_exponent(DF, <<C, _/binary>> = Rest, Acc) when + ?IS_WS(C) orelse ?IS_DELIM(C) +-> {list_to_float(lists:reverse(Acc)), DF, Rest}; toke_number_exponent(DF, <<>>, Acc) -> case DF() of - done -> - {list_to_float(lists:reverse(Acc)), fun() -> done end, <<>>}; - {Data, DF2} -> - toke_number_exponent(DF2, Data, Acc) + done -> + {list_to_float(lists:reverse(Acc)), fun() -> done end, <<>>}; + {Data, DF2} -> + toke_number_exponent(DF2, Data, Acc) end; toke_number_exponent(_, _, _) -> - err(unexpected_character_in_number). - - -err(Error)-> - throw({parse_error,Error}). + err(unexpected_character_in_number). +err(Error) -> + throw({parse_error, Error}). make_ejson([], Stack) -> Stack; @@ -404,8 +398,8 @@ make_ejson([object_start | RevEvs], [ObjValues, PrevValues | RestStack]) -> make_ejson(RevEvs, [[{ObjValues} | PrevValues] | RestStack]); make_ejson([object_end | RevEvs], Stack) -> make_ejson(RevEvs, [[] | Stack]); -make_ejson([{key, String} | RevEvs], [[PrevValue|RestObject] | RestStack] = _Stack) -> - make_ejson(RevEvs, [[{String, PrevValue}|RestObject] | RestStack]); +make_ejson([{key, String} | RevEvs], [[PrevValue | RestObject] | RestStack] = _Stack) -> + make_ejson(RevEvs, [[{String, PrevValue} | RestObject] | RestStack]); make_ejson([Value | RevEvs], [Vals | RestStack] = _Stack) -> make_ejson(RevEvs, [[Value | Vals] | RestStack]). @@ -414,7 +408,6 @@ collect_events(get_results, Acc) -> collect_events(Ev, Acc) -> fun(NextEv) -> collect_events(NextEv, [Ev | Acc]) end. - collect_object(object_end, 0, ReturnControl, Acc) -> [[Obj]] = make_ejson([object_end | Acc], [[]]), ReturnControl(Obj); diff --git a/src/couch_replicator/test/eunit/couch_replicator_attachments_too_large.erl b/src/couch_replicator/test/eunit/couch_replicator_attachments_too_large.erl index ac4bb84f3..2d58f847e 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_attachments_too_large.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_attachments_too_large.erl @@ -16,7 +16,6 @@ -include_lib("couch/include/couch_db.hrl"). -include_lib("couch_replicator/src/couch_replicator.hrl"). - setup(_) -> Ctx = test_util:start_couch([couch_replicator]), Source = create_db(), @@ -24,47 +23,47 @@ setup(_) -> Target = create_db(), {Ctx, {Source, Target}}. - teardown(_, {Ctx, {Source, Target}}) -> delete_db(Source), delete_db(Target), config:delete("couchdb", "max_attachment_size"), ok = test_util:stop_couch(Ctx). - attachment_too_large_replication_test_() -> Pairs = [{remote, remote}], { "Attachment size too large replication tests", { foreachx, - fun setup/1, fun teardown/2, + fun setup/1, + fun teardown/2, [{Pair, fun should_succeed/2} || Pair <- Pairs] ++ - [{Pair, fun should_fail/2} || Pair <- Pairs] + [{Pair, fun should_fail/2} || Pair <- Pairs] } }. - should_succeed({From, To}, {_Ctx, {Source, Target}}) -> - RepObject = {[ - {<<"source">>, db_url(From, Source)}, - {<<"target">>, db_url(To, Target)} - ]}, + RepObject = + {[ + {<<"source">>, db_url(From, Source)}, + {<<"target">>, db_url(To, Target)} + ]}, config:set("couchdb", "max_attachment_size", "1000", _Persist = false), {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER), ?_assertEqual(ok, couch_replicator_test_helper:compare_dbs(Source, Target)). - should_fail({From, To}, {_Ctx, {Source, Target}}) -> - RepObject = {[ - {<<"source">>, db_url(From, Source)}, - {<<"target">>, db_url(To, Target)} - ]}, + RepObject = + {[ + {<<"source">>, db_url(From, Source)}, + {<<"target">>, db_url(To, Target)} + ]}, config:set("couchdb", "max_attachment_size", "999", _Persist = false), {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER), - ?_assertError({badmatch, {not_found, missing}}, - couch_replicator_test_helper:compare_dbs(Source, Target)). - + ?_assertError( + {badmatch, {not_found, missing}}, + couch_replicator_test_helper:compare_dbs(Source, Target) + ). create_db() -> DbName = ?tempdb(), @@ -72,7 +71,6 @@ create_db() -> ok = couch_db:close(Db), DbName. - create_doc_with_attachment(DbName, DocId, AttSize) -> {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]), Doc = #doc{id = DocId, atts = att(AttSize)}, @@ -80,22 +78,21 @@ create_doc_with_attachment(DbName, DocId, AttSize) -> couch_db:close(Db), ok. - att(Size) when is_integer(Size), Size >= 1 -> - [couch_att:new([ - {name, <<"att">>}, - {type, <<"app/binary">>}, - {att_len, Size}, - {data, fun(_Bytes) -> - << <<"x">> || _ <- lists:seq(1, Size) >> - end} - ])]. - + [ + couch_att:new([ + {name, <<"att">>}, + {type, <<"app/binary">>}, + {att_len, Size}, + {data, fun(_Bytes) -> + <<<<"x">> || _ <- lists:seq(1, Size)>> + end} + ]) + ]. delete_db(DbName) -> ok = couch_server:delete(DbName, [?ADMIN_CTX]). - db_url(remote, DbName) -> Addr = config:get("httpd", "bind_address", "127.0.0.1"), Port = mochiweb_socket_server:get(couch_httpd, port), diff --git a/src/couch_replicator/test/eunit/couch_replicator_compact_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_compact_tests.erl index 997c84863..1c093d58c 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_compact_tests.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_compact_tests.erl @@ -60,25 +60,29 @@ compact_test_() -> "Compaction during replication tests", { foreachx, - fun setup/1, fun teardown/2, - [{Pair, fun should_populate_replicate_compact/2} - || Pair <- Pairs] + fun setup/1, + fun teardown/2, + [ + {Pair, fun should_populate_replicate_compact/2} + || Pair <- Pairs + ] } }. - should_populate_replicate_compact({From, To}, {_Ctx, {Source, Target}}) -> {ok, RepPid, RepId} = replicate(Source, Target), - {lists:flatten(io_lib:format("~p -> ~p", [From, To])), - {inorder, [ - should_run_replication(RepPid, RepId, Source, Target), - should_all_processes_be_alive(RepPid, Source, Target), - should_populate_and_compact(RepPid, Source, Target, 50, 3), - should_wait_target_in_sync(Source, Target), - should_ensure_replication_still_running(RepPid, RepId, Source, Target), - should_cancel_replication(RepId, RepPid), - should_compare_databases(Source, Target) - ]}}. + { + lists:flatten(io_lib:format("~p -> ~p", [From, To])), + {inorder, [ + should_run_replication(RepPid, RepId, Source, Target), + should_all_processes_be_alive(RepPid, Source, Target), + should_populate_and_compact(RepPid, Source, Target, 50, 3), + should_wait_target_in_sync(Source, Target), + should_ensure_replication_still_running(RepPid, RepId, Source, Target), + should_cancel_replication(RepId, RepPid), + should_compare_databases(Source, Target) + ]} + }. should_all_processes_be_alive(RepPid, Source, Target) -> ?_test(begin @@ -96,18 +100,20 @@ should_ensure_replication_still_running(RepPid, RepId, Source, Target) -> ?_test(check_active_tasks(RepPid, RepId, Source, Target)). check_active_tasks(RepPid, {BaseId, Ext} = _RepId, Src, Tgt) -> - Source = case Src of - {remote, NameSrc} -> - <<(db_url(NameSrc))/binary, $/>>; - _ -> - Src - end, - Target = case Tgt of - {remote, NameTgt} -> - <<(db_url(NameTgt))/binary, $/>>; - _ -> - Tgt - end, + Source = + case Src of + {remote, NameSrc} -> + <<(db_url(NameSrc))/binary, $/>>; + _ -> + Src + end, + Target = + case Tgt of + {remote, NameTgt} -> + <<(db_url(NameTgt))/binary, $/>>; + _ -> + Tgt + end, FullRepId = ?l2b(BaseId ++ Ext), Pid = ?l2b(pid_to_list(RepPid)), RepTasks = wait_for_task_status(), @@ -129,10 +135,12 @@ check_active_tasks(RepPid, {BaseId, Ext} = _RepId, Src, Tgt) -> ?assert(is_integer(Pending)). replication_tasks() -> - lists:filter(fun(P) -> - couch_util:get_value(type, P) =:= replication - end, couch_task_status:all()). - + lists:filter( + fun(P) -> + couch_util:get_value(type, P) =:= replication + end, + couch_task_status:all() + ). wait_for_task_status() -> test_util:wait(fun() -> @@ -151,66 +159,73 @@ should_cancel_replication(RepId, RepPid) -> end). should_populate_and_compact(RepPid, Source, Target, BatchSize, Rounds) -> - {timeout, ?TIMEOUT_EUNIT, ?_test(begin - {ok, SourceDb0} = reopen_db(Source), - Writer = spawn_writer(SourceDb0), - lists:foreach( - fun(N) -> - {ok, SourceDb} = reopen_db(Source), - {ok, TargetDb} = reopen_db(Target), - pause_writer(Writer), - - compact_db("source", SourceDb), - ?assert(is_process_alive(RepPid)), - ?assert(is_process_alive(couch_db:get_pid(SourceDb))), - wait_for_compaction("source", SourceDb), - - compact_db("target", TargetDb), - ?assert(is_process_alive(RepPid)), - ?assert(is_process_alive(couch_db:get_pid(TargetDb))), - wait_for_compaction("target", TargetDb), - - {ok, SourceDb2} = reopen_db(SourceDb), - {ok, TargetDb2} = reopen_db(TargetDb), - - resume_writer(Writer), - wait_writer(Writer, BatchSize * N), - - compact_db("source", SourceDb2), - ?assert(is_process_alive(RepPid)), - ?assert(is_process_alive(couch_db:get_pid(SourceDb2))), - pause_writer(Writer), - wait_for_compaction("source", SourceDb2), - resume_writer(Writer), - - compact_db("target", TargetDb2), - ?assert(is_process_alive(RepPid)), - ?assert(is_process_alive(couch_db:get_pid(TargetDb2))), - pause_writer(Writer), - wait_for_compaction("target", TargetDb2), - resume_writer(Writer) - end, lists:seq(1, Rounds)), - stop_writer(Writer) - end)}. + {timeout, ?TIMEOUT_EUNIT, + ?_test(begin + {ok, SourceDb0} = reopen_db(Source), + Writer = spawn_writer(SourceDb0), + lists:foreach( + fun(N) -> + {ok, SourceDb} = reopen_db(Source), + {ok, TargetDb} = reopen_db(Target), + pause_writer(Writer), + + compact_db("source", SourceDb), + ?assert(is_process_alive(RepPid)), + ?assert(is_process_alive(couch_db:get_pid(SourceDb))), + wait_for_compaction("source", SourceDb), + + compact_db("target", TargetDb), + ?assert(is_process_alive(RepPid)), + ?assert(is_process_alive(couch_db:get_pid(TargetDb))), + wait_for_compaction("target", TargetDb), + + {ok, SourceDb2} = reopen_db(SourceDb), + {ok, TargetDb2} = reopen_db(TargetDb), + + resume_writer(Writer), + wait_writer(Writer, BatchSize * N), + + compact_db("source", SourceDb2), + ?assert(is_process_alive(RepPid)), + ?assert(is_process_alive(couch_db:get_pid(SourceDb2))), + pause_writer(Writer), + wait_for_compaction("source", SourceDb2), + resume_writer(Writer), + + compact_db("target", TargetDb2), + ?assert(is_process_alive(RepPid)), + ?assert(is_process_alive(couch_db:get_pid(TargetDb2))), + pause_writer(Writer), + wait_for_compaction("target", TargetDb2), + resume_writer(Writer) + end, + lists:seq(1, Rounds) + ), + stop_writer(Writer) + end)}. should_wait_target_in_sync({remote, Source}, Target) -> should_wait_target_in_sync(Source, Target); should_wait_target_in_sync(Source, {remote, Target}) -> should_wait_target_in_sync(Source, Target); should_wait_target_in_sync(Source, Target) -> - {timeout, ?TIMEOUT_EUNIT, ?_assert(begin - {ok, SourceDb} = couch_db:open_int(Source, []), - {ok, SourceInfo} = couch_db:get_db_info(SourceDb), - ok = couch_db:close(SourceDb), - SourceDocCount = couch_util:get_value(doc_count, SourceInfo), - wait_target_in_sync_loop(SourceDocCount, Target, 300) - end)}. + {timeout, ?TIMEOUT_EUNIT, + ?_assert(begin + {ok, SourceDb} = couch_db:open_int(Source, []), + {ok, SourceInfo} = couch_db:get_db_info(SourceDb), + ok = couch_db:close(SourceDb), + SourceDocCount = couch_util:get_value(doc_count, SourceInfo), + wait_target_in_sync_loop(SourceDocCount, Target, 300) + end)}. wait_target_in_sync_loop(_DocCount, _TargetName, 0) -> erlang:error( - {assertion_failed, - [{module, ?MODULE}, {line, ?LINE}, - {reason, "Could not get source and target databases in sync"}]}); + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, "Could not get source and target databases in sync"} + ]} + ); wait_target_in_sync_loop(DocCount, {remote, TargetName}, RetriesLeft) -> wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft); wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) -> @@ -231,33 +246,41 @@ should_compare_databases({remote, Source}, Target) -> should_compare_databases(Source, {remote, Target}) -> should_compare_databases(Source, Target); should_compare_databases(Source, Target) -> - {timeout, 35, ?_test(begin - {ok, SourceDb} = couch_db:open_int(Source, []), - {ok, TargetDb} = couch_db:open_int(Target, []), - Fun = fun(FullDocInfo, Acc) -> - {ok, Doc} = couch_db:open_doc(SourceDb, FullDocInfo), - {Props} = DocJson = couch_doc:to_json_obj(Doc, [attachments]), - DocId = couch_util:get_value(<<"_id">>, Props), - DocTarget = case couch_db:open_doc(TargetDb, DocId) of - {ok, DocT} -> - DocT; - Error -> - erlang:error( - {assertion_failed, - [{module, ?MODULE}, {line, ?LINE}, - {reason, lists:concat(["Error opening document '", - ?b2l(DocId), "' from target: ", - couch_util:to_list(Error)])}]}) + {timeout, 35, + ?_test(begin + {ok, SourceDb} = couch_db:open_int(Source, []), + {ok, TargetDb} = couch_db:open_int(Target, []), + Fun = fun(FullDocInfo, Acc) -> + {ok, Doc} = couch_db:open_doc(SourceDb, FullDocInfo), + {Props} = DocJson = couch_doc:to_json_obj(Doc, [attachments]), + DocId = couch_util:get_value(<<"_id">>, Props), + DocTarget = + case couch_db:open_doc(TargetDb, DocId) of + {ok, DocT} -> + DocT; + Error -> + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, + lists:concat([ + "Error opening document '", + ?b2l(DocId), + "' from target: ", + couch_util:to_list(Error) + ])} + ]} + ) + end, + DocTargetJson = couch_doc:to_json_obj(DocTarget, [attachments]), + ?assertEqual(DocJson, DocTargetJson), + {ok, Acc} end, - DocTargetJson = couch_doc:to_json_obj(DocTarget, [attachments]), - ?assertEqual(DocJson, DocTargetJson), - {ok, Acc} - end, - {ok, _} = couch_db:fold_docs(SourceDb, Fun, [], []), - ok = couch_db:close(SourceDb), - ok = couch_db:close(TargetDb) - end)}. - + {ok, _} = couch_db:fold_docs(SourceDb, Fun, [], []), + ok = couch_db:close(SourceDb), + ok = couch_db:close(TargetDb) + end)}. reopen_db({remote, Db}) -> reopen_db(Db); @@ -268,7 +291,6 @@ reopen_db(DbName) when is_binary(DbName) -> reopen_db(Db) -> reopen_db(couch_db:name(Db)). - compact_db(Type, Db0) -> Name = couch_db:name(Db0), {ok, Db} = couch_db:open_int(Name, []), @@ -281,18 +303,35 @@ compact_db(Type, Db0) -> ok; {'DOWN', MonRef, process, CompactPid, Reason} -> erlang:error( - {assertion_failed, - [{module, ?MODULE}, {line, ?LINE}, - {reason, - lists:concat(["Error compacting ", Type, " database ", - ?b2l(Name), ": ", - couch_util:to_list(Reason)])}]}) + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, + lists:concat([ + "Error compacting ", + Type, + " database ", + ?b2l(Name), + ": ", + couch_util:to_list(Reason) + ])} + ]} + ) after ?TIMEOUT -> erlang:error( - {assertion_failed, - [{module, ?MODULE}, {line, ?LINE}, - {reason, lists:concat(["Compaction for ", Type, " database ", - ?b2l(Name), " didn't finish"])}]}) + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, + lists:concat([ + "Compaction for ", + Type, + " database ", + ?b2l(Name), + " didn't finish" + ])} + ]} + ) end, ok = couch_db:close(Db). @@ -304,31 +343,37 @@ wait_for_compaction(Type, Db) -> ok; {error, Reason} -> erlang:error( - {assertion_failed, - [{module, ?MODULE}, {line, ?LINE}, - {reason, lists:concat(["Compaction of ", Type, - " database failed with: ", Reason])}]}) + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, + lists:concat([ + "Compaction of ", + Type, + " database failed with: ", + Reason + ])} + ]} + ) end. replicate({remote, Db}, Target) -> replicate(db_url(Db), Target); - replicate(Source, {remote, Db}) -> replicate(Source, db_url(Db)); - replicate(Source, Target) -> - RepObject = {[ - {<<"source">>, Source}, - {<<"target">>, Target}, - {<<"continuous">>, true} - ]}, + RepObject = + {[ + {<<"source">>, Source}, + {<<"target">>, Target}, + {<<"continuous">>, true} + ]}, {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_USER), ok = couch_replicator_scheduler:add_job(Rep), couch_replicator_scheduler:reschedule(), Pid = get_pid(Rep#rep.id), {ok, Pid, Rep#rep.id}. - wait_writer(Pid, NumDocs) -> case get_writer_num_docs_written(Pid) of N when N >= NumDocs -> @@ -342,7 +387,6 @@ spawn_writer(Db) -> Pid = spawn(fun() -> writer_loop(Db, Parent, 0) end), Pid. - pause_writer(Pid) -> Ref = make_ref(), Pid ! {pause, Ref}, @@ -350,10 +394,13 @@ pause_writer(Pid) -> {paused, Ref} -> ok after ?TIMEOUT_WRITER -> - erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {reason, "Failed to pause source database writer"}]}) + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, "Failed to pause source database writer"} + ]} + ) end. resume_writer(Pid) -> @@ -363,10 +410,13 @@ resume_writer(Pid) -> {ok, Ref} -> ok after ?TIMEOUT_WRITER -> - erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {reason, "Failed to pause source database writer"}]}) + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, "Failed to pause source database writer"} + ]} + ) end. get_writer_num_docs_written(Pid) -> @@ -376,11 +426,15 @@ get_writer_num_docs_written(Pid) -> {count, Ref, Count} -> Count after ?TIMEOUT_WRITER -> - erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {reason, "Timeout getting number of documents written" - " from source database writer"}]}) + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, + "Timeout getting number of documents written" + " from source database writer"} + ]} + ) end. stop_writer(Pid) -> @@ -393,38 +447,52 @@ stop_writer(Pid) -> {'DOWN', MonRef, process, Pid, _Reason} -> DocsWritten after ?TIMEOUT -> - erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {reason, "Timeout stopping source database writer"}]}) + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, "Timeout stopping source database writer"} + ]} + ) end after ?TIMEOUT_WRITER -> - erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {reason, "Timeout stopping source database writer"}]}) + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, "Timeout stopping source database writer"} + ]} + ) end. writer_loop(Db0, Parent, Counter) -> DbName = couch_db:name(Db0), {ok, Data} = file:read_file(?ATTFILE), maybe_pause(Parent, Counter), - Docs = lists:map(fun(I) -> - couch_doc:from_json_obj({[ - {<<"_id">>, ?l2b(integer_to_list(Counter + I))}, - {<<"value">>, Counter + I}, - {<<"_attachments">>, {[ - {<<"icon1.png">>, {[ - {<<"data">>, base64:encode(Data)}, - {<<"content_type">>, <<"image/png">>} - ]}}, - {<<"icon2.png">>, {[ - {<<"data">>, base64:encode(iolist_to_binary([Data, Data]))}, - {<<"content_type">>, <<"image/png">>} - ]}} - ]}} - ]}) - end, lists:seq(1, ?WRITE_BATCH_SIZE)), + Docs = lists:map( + fun(I) -> + couch_doc:from_json_obj( + {[ + {<<"_id">>, ?l2b(integer_to_list(Counter + I))}, + {<<"value">>, Counter + I}, + {<<"_attachments">>, + {[ + {<<"icon1.png">>, + {[ + {<<"data">>, base64:encode(Data)}, + {<<"content_type">>, <<"image/png">>} + ]}}, + {<<"icon2.png">>, + {[ + {<<"data">>, base64:encode(iolist_to_binary([Data, Data]))}, + {<<"content_type">>, <<"image/png">>} + ]}} + ]}} + ]} + ) + end, + lists:seq(1, ?WRITE_BATCH_SIZE) + ), maybe_pause(Parent, Counter), {ok, Db} = couch_db:open_int(DbName, []), {ok, _} = couch_db:update_docs(Db, Docs, []), diff --git a/src/couch_replicator/test/eunit/couch_replicator_connection_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_connection_tests.erl index e75cc5a63..7adbb6852 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_connection_tests.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_connection_tests.erl @@ -17,7 +17,6 @@ -define(TIMEOUT, 1000). - setup() -> Host = config:get("httpd", "bind_address", "127.0.0.1"), Port = config:get("httpd", "port", "5984"), @@ -26,16 +25,17 @@ setup() -> teardown(_) -> ok. - httpc_pool_test_() -> { "replicator connection sharing tests", { setup, - fun() -> test_util:start_couch([couch_replicator]) end, fun test_util:stop_couch/1, + fun() -> test_util:start_couch([couch_replicator]) end, + fun test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun connections_shared_after_release/1, fun connections_not_shared_after_owner_death/1, @@ -50,7 +50,6 @@ httpc_pool_test_() -> } }. - connections_shared_after_release({Host, Port}) -> ?_test(begin URL = "http://" ++ Host ++ ":" ++ Port, @@ -66,7 +65,6 @@ connections_shared_after_release({Host, Port}) -> end end). - connections_not_shared_after_owner_death({Host, Port}) -> ?_test(begin URL = "http://" ++ Host ++ ":" ++ Port, @@ -80,14 +78,15 @@ connections_not_shared_after_owner_death({Host, Port}) -> {ok, Pid2} = couch_replicator_connection:acquire(URL), ?assertNotEqual(Pid, Pid2), MRef = monitor(process, Pid), - receive {'DOWN', MRef, process, Pid, _Reason} -> - ?assert(not is_process_alive(Pid)); - Other -> throw(Other) + receive + {'DOWN', MRef, process, Pid, _Reason} -> + ?assert(not is_process_alive(Pid)); + Other -> + throw(Other) end end end). - idle_connections_closed({Host, Port}) -> ?_test(begin URL = "http://" ++ Host ++ ":" ++ Port, @@ -103,7 +102,6 @@ idle_connections_closed({Host, Port}) -> ?assert(not ets:member(couch_replicator_connection, Pid)) end). - test_owner_monitors({Host, Port}) -> ?_test(begin URL = "http://" ++ Host ++ ":" ++ Port, @@ -111,21 +109,28 @@ test_owner_monitors({Host, Port}) -> assert_monitors_equal([{process, self()}]), couch_replicator_connection:release(Worker0), assert_monitors_equal([]), - {Workers, Monitors} = lists:foldl(fun(_, {WAcc, MAcc}) -> - {ok, Worker1} = couch_replicator_connection:acquire(URL), - MAcc1 = [{process, self()} | MAcc], - assert_monitors_equal(MAcc1), - {[Worker1 | WAcc], MAcc1} - end, {[], []}, lists:seq(1,5)), - lists:foldl(fun(Worker2, Acc) -> - [_ | NewAcc] = Acc, - couch_replicator_connection:release(Worker2), - assert_monitors_equal(NewAcc), - NewAcc - end, Monitors, Workers) + {Workers, Monitors} = lists:foldl( + fun(_, {WAcc, MAcc}) -> + {ok, Worker1} = couch_replicator_connection:acquire(URL), + MAcc1 = [{process, self()} | MAcc], + assert_monitors_equal(MAcc1), + {[Worker1 | WAcc], MAcc1} + end, + {[], []}, + lists:seq(1, 5) + ), + lists:foldl( + fun(Worker2, Acc) -> + [_ | NewAcc] = Acc, + couch_replicator_connection:release(Worker2), + assert_monitors_equal(NewAcc), + NewAcc + end, + Monitors, + Workers + ) end). - worker_discards_creds_on_create({Host, Port}) -> ?_test(begin {User, Pass, B64Auth} = user_pass(), @@ -136,89 +141,81 @@ worker_discards_creds_on_create({Host, Port}) -> ?assert(string:str(Internals, Pass) =:= 0) end). - worker_discards_url_creds_after_request({Host, _}) -> ?_test(begin - {User, Pass, B64Auth} = user_pass(), - {Port, ServerPid} = server(), - PortStr = integer_to_list(Port), - URL = "http://" ++ User ++ ":" ++ Pass ++ "@" ++ Host ++ ":" ++ PortStr, - {ok, WPid} = couch_replicator_connection:acquire(URL), - ?assertMatch({ok, "200", _, _}, send_req(WPid, URL, [], [])), - Internals = worker_internals(WPid), - ?assert(string:str(Internals, B64Auth) =:= 0), - ?assert(string:str(Internals, Pass) =:= 0), - couch_replicator_connection:release(WPid), - unlink(ServerPid), - exit(ServerPid, kill) + {User, Pass, B64Auth} = user_pass(), + {Port, ServerPid} = server(), + PortStr = integer_to_list(Port), + URL = "http://" ++ User ++ ":" ++ Pass ++ "@" ++ Host ++ ":" ++ PortStr, + {ok, WPid} = couch_replicator_connection:acquire(URL), + ?assertMatch({ok, "200", _, _}, send_req(WPid, URL, [], [])), + Internals = worker_internals(WPid), + ?assert(string:str(Internals, B64Auth) =:= 0), + ?assert(string:str(Internals, Pass) =:= 0), + couch_replicator_connection:release(WPid), + unlink(ServerPid), + exit(ServerPid, kill) end). - worker_discards_creds_in_headers_after_request({Host, _}) -> ?_test(begin - {_User, Pass, B64Auth} = user_pass(), - {Port, ServerPid} = server(), - PortStr = integer_to_list(Port), - URL = "http://" ++ Host ++ ":" ++ PortStr, - {ok, WPid} = couch_replicator_connection:acquire(URL), - Headers = [{"Authorization", "Basic " ++ B64Auth}], - ?assertMatch({ok, "200", _, _}, send_req(WPid, URL, Headers, [])), - Internals = worker_internals(WPid), - ?assert(string:str(Internals, B64Auth) =:= 0), - ?assert(string:str(Internals, Pass) =:= 0), - couch_replicator_connection:release(WPid), - unlink(ServerPid), - exit(ServerPid, kill) + {_User, Pass, B64Auth} = user_pass(), + {Port, ServerPid} = server(), + PortStr = integer_to_list(Port), + URL = "http://" ++ Host ++ ":" ++ PortStr, + {ok, WPid} = couch_replicator_connection:acquire(URL), + Headers = [{"Authorization", "Basic " ++ B64Auth}], + ?assertMatch({ok, "200", _, _}, send_req(WPid, URL, Headers, [])), + Internals = worker_internals(WPid), + ?assert(string:str(Internals, B64Auth) =:= 0), + ?assert(string:str(Internals, Pass) =:= 0), + couch_replicator_connection:release(WPid), + unlink(ServerPid), + exit(ServerPid, kill) end). - worker_discards_proxy_creds_after_request({Host, _}) -> ?_test(begin - {User, Pass, B64Auth} = user_pass(), - {Port, ServerPid} = server(), - PortStr = integer_to_list(Port), - URL = "http://" ++ Host ++ ":" ++ PortStr, - {ok, WPid} = couch_replicator_connection:acquire(URL), - Opts = [ - {proxy_host, Host}, - {proxy_port, Port}, - {proxy_user, User}, - {proxy_pass, Pass} - ], - ?assertMatch({ok, "200", _, _}, send_req(WPid, URL, [], Opts)), - Internals = worker_internals(WPid), - ?assert(string:str(Internals, B64Auth) =:= 0), - ?assert(string:str(Internals, Pass) =:= 0), - couch_replicator_connection:release(WPid), - unlink(ServerPid), - exit(ServerPid, kill) + {User, Pass, B64Auth} = user_pass(), + {Port, ServerPid} = server(), + PortStr = integer_to_list(Port), + URL = "http://" ++ Host ++ ":" ++ PortStr, + {ok, WPid} = couch_replicator_connection:acquire(URL), + Opts = [ + {proxy_host, Host}, + {proxy_port, Port}, + {proxy_user, User}, + {proxy_pass, Pass} + ], + ?assertMatch({ok, "200", _, _}, send_req(WPid, URL, [], Opts)), + Internals = worker_internals(WPid), + ?assert(string:str(Internals, B64Auth) =:= 0), + ?assert(string:str(Internals, Pass) =:= 0), + couch_replicator_connection:release(WPid), + unlink(ServerPid), + exit(ServerPid, kill) end). - send_req(WPid, URL, Headers, Opts) -> ibrowse:send_req_direct(WPid, URL, Headers, get, [], Opts). - user_pass() -> User = "specialuser", Pass = "averysecretpassword", B64Auth = ibrowse_lib:encode_base64(User ++ ":" ++ Pass), {User, Pass, B64Auth}. - worker_internals(Pid) -> Dict = io_lib:format("~p", [erlang:process_info(Pid, dictionary)]), State = io_lib:format("~p", [sys:get_state(Pid)]), lists:flatten([Dict, State]). - server() -> {ok, LSock} = gen_tcp:listen(0, [{recbuf, 256}, {active, false}]), {ok, LPort} = inet:port(LSock), SPid = spawn_link(fun() -> server_responder(LSock) end), {LPort, SPid}. - server_responder(LSock) -> {ok, Sock} = gen_tcp:accept(LSock), case gen_tcp:recv(Sock, 0) of @@ -234,7 +231,6 @@ server_responder(LSock) -> end, server_responder(LSock). - assert_monitors_equal(ShouldBe) -> sys:get_status(couch_replicator_connection), {monitors, Monitors} = process_info(whereis(couch_replicator_connection), monitors), diff --git a/src/couch_replicator/test/eunit/couch_replicator_create_target_with_options_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_create_target_with_options_tests.erl index dae4cd122..8adcd25bd 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_create_target_with_options_tests.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_create_target_with_options_tests.erl @@ -22,23 +22,22 @@ setup() -> Ctx = test_util:start_couch([fabric, mem3, couch_replicator, chttpd]), Hashed = couch_passwords:hash_admin_password(?PASSWORD), - ok = config:set("admins", ?USERNAME, ?b2l(Hashed), _Persist=false), + ok = config:set("admins", ?USERNAME, ?b2l(Hashed), _Persist = false), Source = ?tempdb(), Target = ?tempdb(), {Ctx, {Source, Target}}. - teardown({Ctx, {_Source, _Target}}) -> config:delete("admins", ?USERNAME), ok = test_util:stop_couch(Ctx). - create_target_with_options_replication_test_() -> { "Create target with range partitions tests", { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_create_target_with_q_4/1, fun should_create_target_with_q_2_n_1/1, @@ -48,14 +47,14 @@ create_target_with_options_replication_test_() -> } }. - should_create_target_with_q_4({_Ctx, {Source, Target}}) -> - RepObject = {[ - {<<"source">>, db_url(Source)}, - {<<"target">>, db_url(Target)}, - {<<"create_target">>, true}, - {<<"create_target_params">>, {[{<<"q">>, <<"4">>}]}} - ]}, + RepObject = + {[ + {<<"source">>, db_url(Source)}, + {<<"target">>, db_url(Target)}, + {<<"create_target">>, true}, + {<<"create_target_params">>, {[{<<"q">>, <<"4">>}]}} + ]}, create_db(Source), create_doc(Source), {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER), @@ -66,15 +65,14 @@ should_create_target_with_q_4({_Ctx, {Source, Target}}) -> delete_db(Target), ?_assertEqual(4, couch_util:get_value(q, ClusterInfo)). - should_create_target_with_q_2_n_1({_Ctx, {Source, Target}}) -> - RepObject = {[ - {<<"source">>, db_url(Source)}, - {<<"target">>, db_url(Target)}, - {<<"create_target">>, true}, - {<<"create_target_params">>, - {[{<<"q">>, <<"2">>}, {<<"n">>, <<"1">>}]}} - ]}, + RepObject = + {[ + {<<"source">>, db_url(Source)}, + {<<"target">>, db_url(Target)}, + {<<"create_target">>, true}, + {<<"create_target_params">>, {[{<<"q">>, <<"2">>}, {<<"n">>, <<"1">>}]}} + ]}, create_db(Source), create_doc(Source), {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER), @@ -88,13 +86,13 @@ should_create_target_with_q_2_n_1({_Ctx, {Source, Target}}) -> ?_assertEqual(1, couch_util:get_value(n, ClusterInfo)) ]. - should_create_target_with_default({_Ctx, {Source, Target}}) -> - RepObject = {[ - {<<"source">>, db_url(Source)}, - {<<"target">>, db_url(Target)}, - {<<"create_target">>, true} - ]}, + RepObject = + {[ + {<<"source">>, db_url(Source)}, + {<<"target">>, db_url(Target)}, + {<<"create_target">>, true} + ]}, create_db(Source), create_doc(Source), {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER), @@ -106,14 +104,14 @@ should_create_target_with_default({_Ctx, {Source, Target}}) -> delete_db(Target), ?_assertEqual(Q, couch_util:get_value(q, ClusterInfo)). - should_not_create_target_with_q_any({_Ctx, {Source, Target}}) -> - RepObject = {[ - {<<"source">>, db_url(Source)}, - {<<"target">>, db_url(Target)}, - {<<"create_target">>, false}, - {<<"create_target_params">>, {[{<<"q">>, <<"1">>}]}} - ]}, + RepObject = + {[ + {<<"source">>, db_url(Source)}, + {<<"target">>, db_url(Target)}, + {<<"create_target">>, false}, + {<<"create_target_params">>, {[{<<"q">>, <<"1">>}]}} + ]}, create_db(Source), create_doc(Source), {error, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER), @@ -121,23 +119,26 @@ should_not_create_target_with_q_any({_Ctx, {Source, Target}}) -> delete_db(Source), ?_assertEqual(false, DbExist). - create_doc(DbName) -> Body = {[{<<"foo">>, <<"bar">>}]}, NewDoc = #doc{body = Body}, {ok, _} = fabric:update_doc(DbName, NewDoc, [?ADMIN_CTX]). - create_db(DbName) -> ok = fabric:create_db(DbName, [?ADMIN_CTX]). - delete_db(DbName) -> ok = fabric:delete_db(DbName, [?ADMIN_CTX]). - db_url(DbName) -> Addr = config:get("chttpd", "bind_address", "127.0.0.1"), Port = mochiweb_socket_server:get(chttpd, port), - ?l2b(io_lib:format("http://~s:~s@~s:~b/~s", [?USERNAME, ?PASSWORD, Addr, - Port, DbName])). + ?l2b( + io_lib:format("http://~s:~s@~s:~b/~s", [ + ?USERNAME, + ?PASSWORD, + Addr, + Port, + DbName + ]) + ). diff --git a/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl index 6b4f95c25..7778bd77d 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl @@ -16,29 +16,24 @@ -include_lib("couch/include/couch_db.hrl"). -include_lib("couch_replicator/src/couch_replicator.hrl"). - setup_all() -> test_util:start_couch([couch_replicator, chttpd, mem3, fabric]). - teardown_all(Ctx) -> ok = test_util:stop_couch(Ctx). - setup() -> meck:unload(), Source = setup_db(), Target = setup_db(), {Source, Target}. - teardown({Source, Target}) -> meck:unload(), teardown_db(Source), teardown_db(Target), ok. - error_reporting_test_() -> { setup, @@ -59,7 +54,6 @@ error_reporting_test_() -> } }. - t_fail_bulk_docs({Source, Target}) -> ?_test(begin populate_db(Source, 1, 5), @@ -76,7 +70,6 @@ t_fail_bulk_docs({Source, Target}) -> couch_replicator_notifier:stop(Listener) end). - t_fail_changes_reader({Source, Target}) -> ?_test(begin populate_db(Source, 1, 5), @@ -93,7 +86,6 @@ t_fail_changes_reader({Source, Target}) -> couch_replicator_notifier:stop(Listener) end). - t_fail_revs_diff({Source, Target}) -> ?_test(begin populate_db(Source, 1, 5), @@ -110,7 +102,6 @@ t_fail_revs_diff({Source, Target}) -> couch_replicator_notifier:stop(Listener) end). - t_fail_changes_queue({Source, Target}) -> ?_test(begin populate_db(Source, 1, 5), @@ -130,7 +121,6 @@ t_fail_changes_queue({Source, Target}) -> couch_replicator_notifier:stop(Listener) end). - t_fail_changes_manager({Source, Target}) -> ?_test(begin populate_db(Source, 1, 5), @@ -150,7 +140,6 @@ t_fail_changes_manager({Source, Target}) -> couch_replicator_notifier:stop(Listener) end). - t_fail_changes_reader_proc({Source, Target}) -> ?_test(begin populate_db(Source, 1, 5), @@ -170,9 +159,10 @@ t_fail_changes_reader_proc({Source, Target}) -> couch_replicator_notifier:stop(Listener) end). - mock_fail_req(Path, Return) -> - meck:expect(ibrowse, send_req_direct, + meck:expect( + ibrowse, + send_req_direct, fun(W, Url, Headers, Meth, Body, Opts, TOut) -> Args = [W, Url, Headers, Meth, Body, Opts, TOut], {ok, {_, _, _, _, UPath, _}} = http_uri:parse(Url), @@ -180,18 +170,19 @@ mock_fail_req(Path, Return) -> true -> Return; false -> meck:passthrough(Args) end - end). - + end + ). rep_result_listener(RepId) -> ReplyTo = self(), {ok, _Listener} = couch_replicator_notifier:start_link( - fun({_, RepId2, _} = Ev) when RepId2 =:= RepId -> + fun + ({_, RepId2, _} = Ev) when RepId2 =:= RepId -> ReplyTo ! Ev; (_) -> ok - end). - + end + ). wait_rep_result(RepId) -> receive @@ -199,19 +190,15 @@ wait_rep_result(RepId) -> {error, RepId, Reason} -> {error, Reason} end. - - setup_db() -> DbName = ?tempdb(), {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]), ok = couch_db:close(Db), DbName. - teardown_db(DbName) -> ok = couch_server:delete(DbName, [?ADMIN_CTX]). - populate_db(DbName, Start, End) -> {ok, Db} = couch_db:open_int(DbName, []), Docs = lists:foldl( @@ -220,11 +207,12 @@ populate_db(DbName, Start, End) -> Doc = #doc{id = Id, body = {[]}}, [Doc | Acc] end, - [], lists:seq(Start, End)), + [], + lists:seq(Start, End) + ), {ok, _} = couch_db:update_docs(Db, Docs, []), ok = couch_db:close(Db). - wait_target_in_sync(Source, Target) -> {ok, SourceDb} = couch_db:open_int(Source, []), {ok, SourceInfo} = couch_db:get_db_info(SourceDb), @@ -232,13 +220,14 @@ wait_target_in_sync(Source, Target) -> SourceDocCount = couch_util:get_value(doc_count, SourceInfo), wait_target_in_sync_loop(SourceDocCount, Target, 300). - wait_target_in_sync_loop(_DocCount, _TargetName, 0) -> - erlang:error({assertion_failed, [ - {module, ?MODULE}, {line, ?LINE}, - {reason, "Could not get source and target databases in sync"} - ]}); - + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, "Could not get source and target databases in sync"} + ]} + ); wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) -> {ok, Target} = couch_db:open_int(TargetName, []), {ok, TargetInfo} = couch_db:get_db_info(Target), @@ -252,19 +241,19 @@ wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) -> wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft - 1) end. - replicate(Source, Target) -> SrcUrl = couch_replicator_test_helper:db_url(Source), TgtUrl = couch_replicator_test_helper:db_url(Target), - RepObject = {[ - {<<"source">>, SrcUrl}, - {<<"target">>, TgtUrl}, - {<<"continuous">>, true}, - {<<"worker_processes">>, 1}, - {<<"retries_per_request">>, 1}, - % Low connection timeout so _changes feed gets restarted quicker - {<<"connection_timeout">>, 3000} - ]}, + RepObject = + {[ + {<<"source">>, SrcUrl}, + {<<"target">>, TgtUrl}, + {<<"continuous">>, true}, + {<<"worker_processes">>, 1}, + {<<"retries_per_request">>, 1}, + % Low connection timeout so _changes feed gets restarted quicker + {<<"connection_timeout">>, 3000} + ]}, {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_USER), ok = couch_replicator_scheduler:add_job(Rep), couch_replicator_scheduler:reschedule(), diff --git a/src/couch_replicator/test/eunit/couch_replicator_filtered_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_filtered_tests.erl index 7ac9a4d71..41f1772e6 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_filtered_tests.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_filtered_tests.erl @@ -16,35 +16,43 @@ -include_lib("couch/include/couch_db.hrl"). -include_lib("couch_replicator/src/couch_replicator.hrl"). --define(DDOC, {[ - {<<"_id">>, <<"_design/filter_ddoc">>}, - {<<"filters">>, {[ - {<<"testfilter">>, <<" - function(doc, req){if (doc.class == 'mammal') return true;} - ">>}, - {<<"queryfilter">>, <<" - function(doc, req) { - if (doc.class && req.query.starts) { - return doc.class.indexOf(req.query.starts) === 0; - } - else { - return false; - } - } - ">>} - ]}}, - {<<"views">>, {[ - {<<"mammals">>, {[ - {<<"map">>, <<" - function(doc) { - if (doc.class == 'mammal') { - emit(doc._id, null); - } - } - ">>} - ]}} - ]}} -]}). +-define(DDOC, + {[ + {<<"_id">>, <<"_design/filter_ddoc">>}, + {<<"filters">>, + {[ + {<<"testfilter">>, + <<"\n" + " function(doc, req){if (doc.class == 'mammal') return true;}\n" + " ">>}, + {<<"queryfilter">>, + <<"\n" + " function(doc, req) {\n" + " if (doc.class && req.query.starts) {\n" + " return doc.class.indexOf(req.query.starts) === 0;\n" + " }\n" + " else {\n" + " return false;\n" + " }\n" + " }\n" + " ">>} + ]}}, + {<<"views">>, + {[ + {<<"mammals">>, + {[ + {<<"map">>, + <<"\n" + " function(doc) {\n" + " if (doc.class == 'mammal') {\n" + " emit(doc._id, null);\n" + " }\n" + " }\n" + " ">>} + ]}} + ]}} + ]} +). setup(_) -> Ctx = test_util:start_couch([couch_replicator]), @@ -65,7 +73,8 @@ filtered_replication_test_() -> "Filtered replication tests", { foreachx, - fun setup/1, fun teardown/2, + fun setup/1, + fun teardown/2, [{Pair, fun should_succeed/2} || Pair <- Pairs] } }. @@ -76,7 +85,8 @@ query_filtered_replication_test_() -> "Filtered with query replication tests", { foreachx, - fun setup/1, fun teardown/2, + fun setup/1, + fun teardown/2, [{Pair, fun should_succeed_with_query/2} || Pair <- Pairs] } }. @@ -87,17 +97,19 @@ view_filtered_replication_test_() -> "Filtered with a view replication tests", { foreachx, - fun setup/1, fun teardown/2, + fun setup/1, + fun teardown/2, [{Pair, fun should_succeed_with_view/2} || Pair <- Pairs] } }. should_succeed({From, To}, {_Ctx, {Source, Target}}) -> - RepObject = {[ - {<<"source">>, db_url(From, Source)}, - {<<"target">>, db_url(To, Target)}, - {<<"filter">>, <<"filter_ddoc/testfilter">>} - ]}, + RepObject = + {[ + {<<"source">>, db_url(From, Source)}, + {<<"target">>, db_url(To, Target)}, + {<<"filter">>, <<"filter_ddoc/testfilter">>} + ]}, {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER), %% FilteredFun is an Erlang version of following JS function %% function(doc, req){if (doc.class == 'mammal') return true;} @@ -107,22 +119,24 @@ should_succeed({From, To}, {_Ctx, {Source, Target}}) -> {ok, TargetDbInfo, AllReplies} = compare_dbs(Source, Target, FilterFun), {lists:flatten(io_lib:format("~p -> ~p", [From, To])), [ {"Target DB has proper number of docs", - ?_assertEqual(1, proplists:get_value(doc_count, TargetDbInfo))}, + ?_assertEqual(1, proplists:get_value(doc_count, TargetDbInfo))}, {"Target DB doesn't have deleted docs", - ?_assertEqual(0, proplists:get_value(doc_del_count, TargetDbInfo))}, + ?_assertEqual(0, proplists:get_value(doc_del_count, TargetDbInfo))}, {"All the docs filtered as expected", - ?_assert(lists:all(fun(Valid) -> Valid end, AllReplies))} + ?_assert(lists:all(fun(Valid) -> Valid end, AllReplies))} ]}. should_succeed_with_query({From, To}, {_Ctx, {Source, Target}}) -> - RepObject = {[ - {<<"source">>, db_url(From, Source)}, - {<<"target">>, db_url(To, Target)}, - {<<"filter">>, <<"filter_ddoc/queryfilter">>}, - {<<"query_params">>, {[ - {<<"starts">>, <<"a">>} - ]}} - ]}, + RepObject = + {[ + {<<"source">>, db_url(From, Source)}, + {<<"target">>, db_url(To, Target)}, + {<<"filter">>, <<"filter_ddoc/queryfilter">>}, + {<<"query_params">>, + {[ + {<<"starts">>, <<"a">>} + ]}} + ]}, {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER), FilterFun = fun(_DocId, {Props}) -> case couch_util:get_value(<<"class">>, Props) of @@ -133,22 +147,24 @@ should_succeed_with_query({From, To}, {_Ctx, {Source, Target}}) -> {ok, TargetDbInfo, AllReplies} = compare_dbs(Source, Target, FilterFun), {lists:flatten(io_lib:format("~p -> ~p", [From, To])), [ {"Target DB has proper number of docs", - ?_assertEqual(2, proplists:get_value(doc_count, TargetDbInfo))}, + ?_assertEqual(2, proplists:get_value(doc_count, TargetDbInfo))}, {"Target DB doesn't have deleted docs", - ?_assertEqual(0, proplists:get_value(doc_del_count, TargetDbInfo))}, + ?_assertEqual(0, proplists:get_value(doc_del_count, TargetDbInfo))}, {"All the docs filtered as expected", - ?_assert(lists:all(fun(Valid) -> Valid end, AllReplies))} + ?_assert(lists:all(fun(Valid) -> Valid end, AllReplies))} ]}. should_succeed_with_view({From, To}, {_Ctx, {Source, Target}}) -> - RepObject = {[ - {<<"source">>, db_url(From, Source)}, - {<<"target">>, db_url(To, Target)}, - {<<"filter">>, <<"_view">>}, - {<<"query_params">>, {[ - {<<"view">>, <<"filter_ddoc/mammals">>} - ]}} - ]}, + RepObject = + {[ + {<<"source">>, db_url(From, Source)}, + {<<"target">>, db_url(To, Target)}, + {<<"filter">>, <<"_view">>}, + {<<"query_params">>, + {[ + {<<"view">>, <<"filter_ddoc/mammals">>} + ]}} + ]}, {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER), FilterFun = fun(_DocId, {Props}) -> couch_util:get_value(<<"class">>, Props) == <<"mammal">> @@ -156,11 +172,11 @@ should_succeed_with_view({From, To}, {_Ctx, {Source, Target}}) -> {ok, TargetDbInfo, AllReplies} = compare_dbs(Source, Target, FilterFun), {lists:flatten(io_lib:format("~p -> ~p", [From, To])), [ {"Target DB has proper number of docs", - ?_assertEqual(1, proplists:get_value(doc_count, TargetDbInfo))}, + ?_assertEqual(1, proplists:get_value(doc_count, TargetDbInfo))}, {"Target DB doesn't have deleted docs", - ?_assertEqual(0, proplists:get_value(doc_del_count, TargetDbInfo))}, + ?_assertEqual(0, proplists:get_value(doc_del_count, TargetDbInfo))}, {"All the docs filtered as expected", - ?_assert(lists:all(fun(Valid) -> Valid end, AllReplies))} + ?_assert(lists:all(fun(Valid) -> Valid end, AllReplies))} ]}. compare_dbs(Source, Target, FilterFun) -> @@ -173,10 +189,10 @@ compare_dbs(Source, Target, FilterFun) -> case FilterFun(DocId, SourceDoc) of true -> ValidReply = {ok, DocId, SourceDoc} == TargetReply, - {ok, [ValidReply|Acc]}; + {ok, [ValidReply | Acc]}; false -> ValidReply = {not_found, missing} == TargetReply, - {ok, [ValidReply|Acc]} + {ok, [ValidReply | Acc]} end end, {ok, AllReplies} = couch_db:fold_docs(SourceDb, Fun, [], []), @@ -203,30 +219,34 @@ create_db() -> create_docs(DbName) -> {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]), DDoc = couch_doc:from_json_obj(?DDOC), - Doc1 = couch_doc:from_json_obj({[ - {<<"_id">>, <<"doc1">>}, - {<<"class">>, <<"mammal">>}, - {<<"value">>, 1} - - ]}), - Doc2 = couch_doc:from_json_obj({[ - {<<"_id">>, <<"doc2">>}, - {<<"class">>, <<"amphibians">>}, - {<<"value">>, 2} - - ]}), - Doc3 = couch_doc:from_json_obj({[ - {<<"_id">>, <<"doc3">>}, - {<<"class">>, <<"reptiles">>}, - {<<"value">>, 3} - - ]}), - Doc4 = couch_doc:from_json_obj({[ - {<<"_id">>, <<"doc4">>}, - {<<"class">>, <<"arthropods">>}, - {<<"value">>, 2} - - ]}), + Doc1 = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"doc1">>}, + {<<"class">>, <<"mammal">>}, + {<<"value">>, 1} + ]} + ), + Doc2 = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"doc2">>}, + {<<"class">>, <<"amphibians">>}, + {<<"value">>, 2} + ]} + ), + Doc3 = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"doc3">>}, + {<<"class">>, <<"reptiles">>}, + {<<"value">>, 3} + ]} + ), + Doc4 = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"doc4">>}, + {<<"class">>, <<"arthropods">>}, + {<<"value">>, 2} + ]} + ), {ok, _} = couch_db:update_docs(Db, [DDoc, Doc1, Doc2, Doc3, Doc4]), couch_db:close(Db). diff --git a/src/couch_replicator/test/eunit/couch_replicator_httpc_pool_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_httpc_pool_tests.erl index c4ad4e9b6..31f1da48e 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_httpc_pool_tests.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_httpc_pool_tests.erl @@ -17,23 +17,23 @@ -define(TIMEOUT, 1000). - setup() -> spawn_pool(). teardown(Pool) -> stop_pool(Pool). - httpc_pool_test_() -> { "httpc pool tests", { setup, - fun() -> test_util:start_couch([couch_replicator]) end, fun test_util:stop_couch/1, + fun() -> test_util:start_couch([couch_replicator]) end, + fun test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun should_block_new_clients_when_full/1, fun should_replace_worker_on_death/1 @@ -42,7 +42,6 @@ httpc_pool_test_() -> } }. - should_block_new_clients_when_full(Pool) -> ?_test(begin Client1 = spawn_client(Pool), @@ -77,7 +76,9 @@ should_block_new_clients_when_full(Pool) -> lists:foreach( fun(C) -> ?assertEqual(ok, stop_client(C)) - end, [Client2, Client3, Client4]) + end, + [Client2, Client3, Client4] + ) end). should_replace_worker_on_death(Pool) -> @@ -100,7 +101,6 @@ should_replace_worker_on_death(Pool) -> ?assertEqual(ok, stop_client(Client2)) end). - spawn_client(Pool) -> Parent = self(), Ref = make_ref(), @@ -126,9 +126,12 @@ get_client_worker({Pid, Ref}, ClientName) -> Worker after ?TIMEOUT -> erlang:error( - {assertion_failed, - [{module, ?MODULE}, {line, ?LINE}, - {reason, "Timeout getting client " ++ ClientName ++ " worker"}]}) + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, "Timeout getting client " ++ ClientName ++ " worker"} + ]} + ) end. stop_client({Pid, Ref}) -> @@ -155,7 +158,7 @@ loop(Parent, Ref, Worker, Pool) -> ping -> Parent ! {pong, Ref}, loop(Parent, Ref, Worker, Pool); - get_worker -> + get_worker -> Parent ! {worker, Ref, Worker}, loop(Parent, Ref, Worker, Pool); stop -> @@ -167,7 +170,8 @@ spawn_pool() -> Host = config:get("httpd", "bind_address", "127.0.0.1"), Port = config:get("httpd", "port", "5984"), {ok, Pool} = couch_replicator_httpc_pool:start_link( - "http://" ++ Host ++ ":" ++ Port, [{max_connections, 3}]), + "http://" ++ Host ++ ":" ++ Port, [{max_connections, 3}] + ), Pool. stop_pool(Pool) -> diff --git a/src/couch_replicator/test/eunit/couch_replicator_id_too_long_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_id_too_long_tests.erl index a4696c4b8..9ed415a29 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_id_too_long_tests.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_id_too_long_tests.erl @@ -16,7 +16,6 @@ -include_lib("couch/include/couch_db.hrl"). -include_lib("couch_replicator/src/couch_replicator.hrl"). - setup(_) -> Ctx = test_util:start_couch([couch_replicator]), Source = create_db(), @@ -24,47 +23,47 @@ setup(_) -> Target = create_db(), {Ctx, {Source, Target}}. - teardown(_, {Ctx, {Source, Target}}) -> delete_db(Source), delete_db(Target), config:set("replicator", "max_document_id_length", "infinity"), ok = test_util:stop_couch(Ctx). - id_too_long_replication_test_() -> Pairs = [{remote, remote}], { "Doc id too long tests", { foreachx, - fun setup/1, fun teardown/2, + fun setup/1, + fun teardown/2, [{Pair, fun should_succeed/2} || Pair <- Pairs] ++ - [{Pair, fun should_fail/2} || Pair <- Pairs] + [{Pair, fun should_fail/2} || Pair <- Pairs] } }. - should_succeed({From, To}, {_Ctx, {Source, Target}}) -> - RepObject = {[ - {<<"source">>, db_url(From, Source)}, - {<<"target">>, db_url(To, Target)} - ]}, + RepObject = + {[ + {<<"source">>, db_url(From, Source)}, + {<<"target">>, db_url(To, Target)} + ]}, config:set("replicator", "max_document_id_length", "5"), {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER), ?_assertEqual(ok, couch_replicator_test_helper:compare_dbs(Source, Target)). - should_fail({From, To}, {_Ctx, {Source, Target}}) -> - RepObject = {[ - {<<"source">>, db_url(From, Source)}, - {<<"target">>, db_url(To, Target)} - ]}, + RepObject = + {[ + {<<"source">>, db_url(From, Source)}, + {<<"target">>, db_url(To, Target)} + ]}, config:set("replicator", "max_document_id_length", "4"), {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER), - ?_assertError({badmatch, {not_found, missing}}, - couch_replicator_test_helper:compare_dbs(Source, Target)). - + ?_assertError( + {badmatch, {not_found, missing}}, + couch_replicator_test_helper:compare_dbs(Source, Target) + ). create_db() -> DbName = ?tempdb(), @@ -72,18 +71,15 @@ create_db() -> ok = couch_db:close(Db), DbName. - create_doc(DbName) -> {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]), Doc = couch_doc:from_json_obj({[{<<"_id">>, <<"12345">>}]}), {ok, _} = couch_db:update_doc(Db, Doc, []), couch_db:close(Db). - delete_db(DbName) -> ok = couch_server:delete(DbName, [?ADMIN_CTX]). - db_url(remote, DbName) -> Addr = config:get("httpd", "bind_address", "127.0.0.1"), Port = mochiweb_socket_server:get(couch_httpd, port), diff --git a/src/couch_replicator/test/eunit/couch_replicator_large_atts_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_large_atts_tests.erl index 27c89a0cd..2f0e2a1f0 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_large_atts_tests.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_large_atts_tests.erl @@ -26,7 +26,6 @@ -define(DOCS_COUNT, 11). -define(TIMEOUT_EUNIT, 120). - setup() -> DbName = ?tempdb(), {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]), @@ -61,18 +60,24 @@ large_atts_test_() -> "Replicate docs with large attachments", { foreachx, - fun setup/1, fun teardown/2, - [{Pair, fun should_populate_replicate_compact/2} - || Pair <- Pairs] + fun setup/1, + fun teardown/2, + [ + {Pair, fun should_populate_replicate_compact/2} + || Pair <- Pairs + ] } }. - should_populate_replicate_compact({From, To}, {_Ctx, {Source, Target}}) -> - {lists:flatten(io_lib:format("~p -> ~p", [From, To])), - {inorder, [should_populate_source(Source), - should_replicate(Source, Target), - should_compare_databases(Source, Target)]}}. + { + lists:flatten(io_lib:format("~p -> ~p", [From, To])), + {inorder, [ + should_populate_source(Source), + should_replicate(Source, Target), + should_compare_databases(Source, Target) + ]} + }. should_populate_source({remote, Source}) -> should_populate_source(Source); @@ -93,7 +98,6 @@ should_compare_databases(Source, {remote, Target}) -> should_compare_databases(Source, Target) -> {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target))}. - populate_db(DbName, DocCount) -> {ok, Db} = couch_db:open_int(DbName, []), Docs = lists:foldl( @@ -108,7 +112,9 @@ populate_db(DbName, DocCount) -> }, [Doc | Acc] end, - [], lists:seq(1, DocCount)), + [], + lists:seq(1, DocCount) + ), {ok, _} = couch_db:update_docs(Db, Docs, []), couch_db:close(Db). diff --git a/src/couch_replicator/test/eunit/couch_replicator_many_leaves_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_many_leaves_tests.erl index c7933b472..86daa808f 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_many_leaves_tests.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_many_leaves_tests.erl @@ -38,7 +38,6 @@ setup() -> ok = couch_db:close(Db), DbName. - setup(remote) -> {remote, setup()}; setup({A, B}) -> @@ -65,23 +64,27 @@ docs_with_many_leaves_test_() -> "Replicate documents with many leaves", { foreachx, - fun setup/1, fun teardown/2, - [{Pair, fun should_populate_replicate_compact/2} - || Pair <- Pairs] + fun setup/1, + fun teardown/2, + [ + {Pair, fun should_populate_replicate_compact/2} + || Pair <- Pairs + ] } }. - should_populate_replicate_compact({From, To}, {_Ctx, {Source, Target}}) -> - {lists:flatten(io_lib:format("~p -> ~p", [From, To])), - {inorder, [ - should_populate_source(Source), - should_replicate(Source, Target), - should_verify_target(Source, Target), - should_add_attachments_to_source(Source), - should_replicate(Source, Target), - should_verify_target(Source, Target) - ]}}. + { + lists:flatten(io_lib:format("~p -> ~p", [From, To])), + {inorder, [ + should_populate_source(Source), + should_replicate(Source, Target), + should_verify_target(Source, Target), + should_add_attachments_to_source(Source), + should_replicate(Source, Target), + should_verify_target(Source, Target) + ]} + }. should_populate_source({remote, Source}) -> should_populate_source(Source); @@ -100,35 +103,39 @@ should_verify_target({remote, Source}, Target) -> should_verify_target(Source, {remote, Target}) -> should_verify_target(Source, Target); should_verify_target(Source, Target) -> - {timeout, ?TIMEOUT_EUNIT, ?_test(begin - {ok, SourceDb} = couch_db:open_int(Source, []), - {ok, TargetDb} = couch_db:open_int(Target, []), - verify_target(SourceDb, TargetDb, ?DOCS_CONFLICTS), - ok = couch_db:close(SourceDb), - ok = couch_db:close(TargetDb) - end)}. + {timeout, ?TIMEOUT_EUNIT, + ?_test(begin + {ok, SourceDb} = couch_db:open_int(Source, []), + {ok, TargetDb} = couch_db:open_int(Target, []), + verify_target(SourceDb, TargetDb, ?DOCS_CONFLICTS), + ok = couch_db:close(SourceDb), + ok = couch_db:close(TargetDb) + end)}. should_add_attachments_to_source({remote, Source}) -> should_add_attachments_to_source(Source); should_add_attachments_to_source(Source) -> - {timeout, ?TIMEOUT_EUNIT, ?_test(begin - {ok, SourceDb} = couch_db:open_int(Source, [?ADMIN_CTX]), - add_attachments(SourceDb, ?NUM_ATTS, ?DOCS_CONFLICTS), - ok = couch_db:close(SourceDb) - end)}. + {timeout, ?TIMEOUT_EUNIT, + ?_test(begin + {ok, SourceDb} = couch_db:open_int(Source, [?ADMIN_CTX]), + add_attachments(SourceDb, ?NUM_ATTS, ?DOCS_CONFLICTS), + ok = couch_db:close(SourceDb) + end)}. populate_db(DbName) -> {ok, Db} = couch_db:open_int(DbName, [?ADMIN_CTX]), lists:foreach( - fun({DocId, NumConflicts}) -> + fun({DocId, NumConflicts}) -> Value = <<"0">>, Doc = #doc{ id = DocId, - body = {[ {<<"value">>, Value} ]} + body = {[{<<"value">>, Value}]} }, {ok, _} = couch_db:update_doc(Db, Doc, [?ADMIN_CTX]), {ok, _} = add_doc_siblings(Db, DocId, NumConflicts) - end, ?DOCS_CONFLICTS), + end, + ?DOCS_CONFLICTS + ), couch_db:close(Db). add_doc_siblings(Db, DocId, NumLeaves) when NumLeaves > 0 -> @@ -137,17 +144,21 @@ add_doc_siblings(Db, DocId, NumLeaves) when NumLeaves > 0 -> add_doc_siblings(Db, _DocId, 0, AccDocs, AccRevs) -> {ok, []} = couch_db:update_docs(Db, AccDocs, [], replicated_changes), {ok, AccRevs}; - add_doc_siblings(Db, DocId, NumLeaves, AccDocs, AccRevs) -> Value = ?l2b(?i2l(NumLeaves)), Rev = couch_hash:md5_hash(Value), Doc = #doc{ id = DocId, revs = {1, [Rev]}, - body = {[ {<<"value">>, Value} ]} + body = {[{<<"value">>, Value}]} }, - add_doc_siblings(Db, DocId, NumLeaves - 1, - [Doc | AccDocs], [{1, Rev} | AccRevs]). + add_doc_siblings( + Db, + DocId, + NumLeaves - 1, + [Doc | AccDocs], + [{1, Rev} | AccRevs] + ). verify_target(_SourceDb, _TargetDb, []) -> ok; @@ -156,12 +167,14 @@ verify_target(SourceDb, TargetDb, [{DocId, NumConflicts} | Rest]) -> SourceDb, DocId, all, - [conflicts, deleted_conflicts]), + [conflicts, deleted_conflicts] + ), {ok, TargetLookups} = couch_db:open_doc_revs( TargetDb, DocId, all, - [conflicts, deleted_conflicts]), + [conflicts, deleted_conflicts] + ), SourceDocs = [Doc || {ok, Doc} <- SourceLookups], TargetDocs = [Doc || {ok, Doc} <- TargetLookups], Total = NumConflicts + 1, @@ -172,34 +185,45 @@ verify_target(SourceDb, TargetDb, [{DocId, NumConflicts} | Rest]) -> TargetJson = couch_doc:to_json_obj(TargetDoc, [attachments]), ?assertEqual(SourceJson, TargetJson) end, - lists:zip(SourceDocs, TargetDocs)), + lists:zip(SourceDocs, TargetDocs) + ), verify_target(SourceDb, TargetDb, Rest). -add_attachments(_SourceDb, _NumAtts, []) -> +add_attachments(_SourceDb, _NumAtts, []) -> ok; -add_attachments(SourceDb, NumAtts, [{DocId, NumConflicts} | Rest]) -> +add_attachments(SourceDb, NumAtts, [{DocId, NumConflicts} | Rest]) -> {ok, SourceLookups} = couch_db:open_doc_revs(SourceDb, DocId, all, []), SourceDocs = [Doc || {ok, Doc} <- SourceLookups], Total = NumConflicts + 1, ?assertEqual(Total, length(SourceDocs)), NewDocs = lists:foldl( fun(#doc{atts = Atts, revs = {Pos, [Rev | _]}} = Doc, Acc) -> - NewAtts = lists:foldl(fun(I, AttAcc) -> - AttData = crypto:strong_rand_bytes(100), - NewAtt = couch_att:new([ - {name, ?io2b(["att_", ?i2l(I), "_", - couch_doc:rev_to_str({Pos, Rev})])}, - {type, <<"application/foobar">>}, - {att_len, byte_size(AttData)}, - {data, AttData} - ]), - [NewAtt | AttAcc] - end, [], lists:seq(1, NumAtts)), + NewAtts = lists:foldl( + fun(I, AttAcc) -> + AttData = crypto:strong_rand_bytes(100), + NewAtt = couch_att:new([ + {name, + ?io2b([ + "att_", + ?i2l(I), + "_", + couch_doc:rev_to_str({Pos, Rev}) + ])}, + {type, <<"application/foobar">>}, + {att_len, byte_size(AttData)}, + {data, AttData} + ]), + [NewAtt | AttAcc] + end, + [], + lists:seq(1, NumAtts) + ), [Doc#doc{atts = Atts ++ NewAtts} | Acc] end, - [], SourceDocs), + [], + SourceDocs + ), {ok, UpdateResults} = couch_db:update_docs(SourceDb, NewDocs, []), NewRevs = [R || {ok, R} <- UpdateResults], ?assertEqual(length(NewDocs), length(NewRevs)), add_attachments(SourceDb, NumAtts, Rest). - diff --git a/src/couch_replicator/test/eunit/couch_replicator_missing_stubs_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_missing_stubs_tests.erl index ff08b5ee5..ff3b5ee98 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_missing_stubs_tests.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_missing_stubs_tests.erl @@ -56,24 +56,28 @@ missing_stubs_test_() -> "Replicate docs with missing stubs (COUCHDB-1365)", { foreachx, - fun setup/1, fun teardown/2, - [{Pair, fun should_replicate_docs_with_missed_att_stubs/2} - || Pair <- Pairs] + fun setup/1, + fun teardown/2, + [ + {Pair, fun should_replicate_docs_with_missed_att_stubs/2} + || Pair <- Pairs + ] } }. - should_replicate_docs_with_missed_att_stubs({From, To}, {_Ctx, {Source, Target}}) -> - {lists:flatten(io_lib:format("~p -> ~p", [From, To])), - {inorder, [ - should_populate_source(Source), - should_set_target_revs_limit(Target, ?REVS_LIMIT), - should_replicate(Source, Target), - should_compare_databases(Source, Target), - should_update_source_docs(Source, ?REVS_LIMIT * 2), - should_replicate(Source, Target), - should_compare_databases(Source, Target) - ]}}. + { + lists:flatten(io_lib:format("~p -> ~p", [From, To])), + {inorder, [ + should_populate_source(Source), + should_set_target_revs_limit(Target, ?REVS_LIMIT), + should_replicate(Source, Target), + should_compare_databases(Source, Target), + should_update_source_docs(Source, ?REVS_LIMIT * 2), + should_replicate(Source, Target), + should_compare_databases(Source, Target) + ]} + }. should_populate_source({remote, Source}) -> should_populate_source(Source); @@ -108,7 +112,6 @@ should_update_source_docs({remote, Source}, Times) -> should_update_source_docs(Source, Times) -> {timeout, ?TIMEOUT_EUNIT, ?_test(update_db_docs(Source, Times))}. - populate_db(DbName) -> {ok, Db} = couch_db:open_int(DbName, []), AttData = crypto:strong_rand_bytes(6000), @@ -120,7 +123,7 @@ populate_db(DbName) -> {type, <<"application/foobar">>}, {att_len, byte_size(AttData)}, {data, AttData} - ]) + ]) ] }, {ok, _} = couch_db:update_doc(Db, Doc, []), @@ -132,7 +135,8 @@ update_db_docs(DbName, Times) -> Db, fun(FDI, Acc) -> db_fold_fun(FDI, Acc) end, {DbName, Times}, - []), + [] + ), ok = couch_db:close(Db). db_fold_fun(FullDocInfo, {DbName, Times}) -> @@ -149,6 +153,7 @@ db_fold_fun(FullDocInfo, {DbName, Times}) -> NewRev end, {element(1, Doc#doc.revs), hd(element(2, Doc#doc.revs))}, - lists:seq(1, Times)), + lists:seq(1, Times) + ), ok = couch_db:close(Db), {ok, {DbName, Times}}. diff --git a/src/couch_replicator/test/eunit/couch_replicator_proxy_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_proxy_tests.erl index da46b8a26..ca1816b33 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_proxy_tests.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_proxy_tests.erl @@ -16,24 +16,23 @@ -include_lib("couch_replicator/src/couch_replicator.hrl"). -include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl"). - setup() -> ok. - teardown(_) -> ok. - replicator_proxy_test_() -> { "replicator proxy tests", { setup, - fun() -> test_util:start_couch([couch_replicator]) end, fun test_util:stop_couch/1, + fun() -> test_util:start_couch([couch_replicator]) end, + fun test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun parse_rep_doc_without_proxy/1, fun parse_rep_doc_with_proxy/1, @@ -45,72 +44,80 @@ replicator_proxy_test_() -> } }. - parse_rep_doc_without_proxy(_) -> ?_test(begin - NoProxyDoc = {[ - {<<"source">>, <<"http://unproxied.com">>}, - {<<"target">>, <<"http://otherunproxied.com">>} - ]}, + NoProxyDoc = + {[ + {<<"source">>, <<"http://unproxied.com">>}, + {<<"target">>, <<"http://otherunproxied.com">>} + ]}, Rep = couch_replicator_docs:parse_rep_doc(NoProxyDoc), ?assertEqual((Rep#rep.source)#httpdb.proxy_url, undefined), ?assertEqual((Rep#rep.target)#httpdb.proxy_url, undefined) end). - parse_rep_doc_with_proxy(_) -> ?_test(begin ProxyURL = <<"http://myproxy.com">>, - ProxyDoc = {[ - {<<"source">>, <<"http://unproxied.com">>}, - {<<"target">>, <<"http://otherunproxied.com">>}, - {<<"proxy">>, ProxyURL} - ]}, + ProxyDoc = + {[ + {<<"source">>, <<"http://unproxied.com">>}, + {<<"target">>, <<"http://otherunproxied.com">>}, + {<<"proxy">>, ProxyURL} + ]}, Rep = couch_replicator_docs:parse_rep_doc(ProxyDoc), ?assertEqual((Rep#rep.source)#httpdb.proxy_url, binary_to_list(ProxyURL)), ?assertEqual((Rep#rep.target)#httpdb.proxy_url, binary_to_list(ProxyURL)) end). - parse_rep_source_target_proxy(_) -> ?_test(begin SrcProxyURL = <<"http://mysrcproxy.com">>, TgtProxyURL = <<"http://mytgtproxy.com:9999">>, - ProxyDoc = {[ - {<<"source">>, <<"http://unproxied.com">>}, - {<<"target">>, <<"http://otherunproxied.com">>}, - {<<"source_proxy">>, SrcProxyURL}, - {<<"target_proxy">>, TgtProxyURL} - ]}, + ProxyDoc = + {[ + {<<"source">>, <<"http://unproxied.com">>}, + {<<"target">>, <<"http://otherunproxied.com">>}, + {<<"source_proxy">>, SrcProxyURL}, + {<<"target_proxy">>, TgtProxyURL} + ]}, Rep = couch_replicator_docs:parse_rep_doc(ProxyDoc), - ?assertEqual((Rep#rep.source)#httpdb.proxy_url, - binary_to_list(SrcProxyURL)), - ?assertEqual((Rep#rep.target)#httpdb.proxy_url, - binary_to_list(TgtProxyURL)) + ?assertEqual( + (Rep#rep.source)#httpdb.proxy_url, + binary_to_list(SrcProxyURL) + ), + ?assertEqual( + (Rep#rep.target)#httpdb.proxy_url, + binary_to_list(TgtProxyURL) + ) end). - mutually_exclusive_proxy_and_source_proxy(_) -> ?_test(begin - ProxyDoc = {[ - {<<"source">>, <<"http://unproxied.com">>}, - {<<"target">>, <<"http://otherunproxied.com">>}, - {<<"proxy">>, <<"oldstyleproxy.local">>}, - {<<"source_proxy">>, <<"sourceproxy.local">>} - ]}, - ?assertThrow({bad_rep_doc, _}, - couch_replicator_docs:parse_rep_doc(ProxyDoc)) + ProxyDoc = + {[ + {<<"source">>, <<"http://unproxied.com">>}, + {<<"target">>, <<"http://otherunproxied.com">>}, + {<<"proxy">>, <<"oldstyleproxy.local">>}, + {<<"source_proxy">>, <<"sourceproxy.local">>} + ]}, + ?assertThrow( + {bad_rep_doc, _}, + couch_replicator_docs:parse_rep_doc(ProxyDoc) + ) end). - mutually_exclusive_proxy_and_target_proxy(_) -> ?_test(begin - ProxyDoc = {[ - {<<"source">>, <<"http://unproxied.com">>}, - {<<"target">>, <<"http://otherunproxied.com">>}, - {<<"proxy">>, <<"oldstyleproxy.local">>}, - {<<"target_proxy">>, <<"targetproxy.local">>} - ]}, - ?assertThrow({bad_rep_doc, _}, - couch_replicator_docs:parse_rep_doc(ProxyDoc)) + ProxyDoc = + {[ + {<<"source">>, <<"http://unproxied.com">>}, + {<<"target">>, <<"http://otherunproxied.com">>}, + {<<"proxy">>, <<"oldstyleproxy.local">>}, + {<<"target_proxy">>, <<"targetproxy.local">>} + ]}, + ?assertThrow( + {bad_rep_doc, _}, + couch_replicator_docs:parse_rep_doc(ProxyDoc) + ) end). diff --git a/src/couch_replicator/test/eunit/couch_replicator_rate_limiter_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_rate_limiter_tests.erl index 034550aec..a214d4607 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_rate_limiter_tests.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_rate_limiter_tests.erl @@ -2,7 +2,6 @@ -include_lib("couch/include/couch_eunit.hrl"). - rate_limiter_test_() -> { foreach, @@ -18,19 +17,16 @@ rate_limiter_test_() -> ] }. - t_new_key() -> ?_test(begin ?assertEqual(0, couch_replicator_rate_limiter:interval({"foo", get})) end). - t_1_failure() -> ?_test(begin ?assertEqual(24, couch_replicator_rate_limiter:failure({"foo", get})) end). - t_2_failures() -> ?_test(begin couch_replicator_rate_limiter:failure({"foo", get}), @@ -39,7 +35,6 @@ t_2_failures() -> ?assertEqual(29, Interval) end). - t_2_failures_back_to_back() -> ?_test(begin couch_replicator_rate_limiter:failure({"foo", get}), @@ -47,7 +42,6 @@ t_2_failures_back_to_back() -> ?assertEqual(24, Interval) end). - t_success_threshold() -> ?_test(begin Interval = couch_replicator_rate_limiter:success({"foo", get}), @@ -56,7 +50,6 @@ t_success_threshold() -> ?assertEqual(0, Interval) end). - t_1_failure_2_successes() -> ?_test(begin couch_replicator_rate_limiter:failure({"foo", get}), @@ -68,16 +61,13 @@ t_1_failure_2_successes() -> ?assertEqual(0, Succ2) end). - low_pass_filter_delay() -> timer:sleep(100). - setup() -> {ok, Pid} = couch_replicator_rate_limiter:start_link(), Pid. - teardown(Pid) -> Ref = erlang:monitor(process, Pid), unlink(Pid), diff --git a/src/couch_replicator/test/eunit/couch_replicator_retain_stats_between_job_runs.erl b/src/couch_replicator/test/eunit/couch_replicator_retain_stats_between_job_runs.erl index a9a0fc943..ba29a3c68 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_retain_stats_between_job_runs.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_retain_stats_between_job_runs.erl @@ -19,27 +19,22 @@ -define(DELAY, 500). -define(TIMEOUT, 60000). - setup_all() -> test_util:start_couch([couch_replicator, chttpd, mem3, fabric]). - teardown_all(Ctx) -> ok = test_util:stop_couch(Ctx). - setup() -> Source = setup_db(), Target = setup_db(), {Source, Target}. - teardown({Source, Target}) -> teardown_db(Source), teardown_db(Target), ok. - stats_retained_test_() -> { setup, @@ -56,7 +51,6 @@ stats_retained_test_() -> } }. - t_stats_retained_by_scheduler({Source, Target}) -> ?_test(begin {ok, _} = add_vdu(Target), @@ -76,13 +70,13 @@ t_stats_retained_by_scheduler({Source, Target}) -> couch_replicator_scheduler:remove_job(RepId) end). - t_stats_retained_on_job_removal({Source, Target}) -> ?_test(begin {ok, _} = add_vdu(Target), populate_db_reject_even_docs(Source, 1, 10), {ok, _, RepId} = replicate(Source, Target), - wait_target_in_sync(6, Target), % 5 + 1 vdu + % 5 + 1 vdu + wait_target_in_sync(6, Target), check_active_tasks(10, 5, 5), check_scheduler_jobs(10, 5, 5), @@ -91,7 +85,8 @@ t_stats_retained_on_job_removal({Source, Target}) -> populate_db_reject_even_docs(Source, 11, 20), {ok, _, RepId} = replicate(Source, Target), - wait_target_in_sync(11, Target), % 6 + 5 + % 6 + 5 + wait_target_in_sync(11, Target), check_scheduler_jobs(20, 10, 10), check_active_tasks(20, 10, 10), @@ -100,7 +95,8 @@ t_stats_retained_on_job_removal({Source, Target}) -> populate_db_reject_even_docs(Source, 21, 30), {ok, _, RepId} = replicate(Source, Target), - wait_target_in_sync(16, Target), % 11 + 5 + % 11 + 5 + wait_target_in_sync(16, Target), check_scheduler_jobs(30, 15, 15), check_active_tasks(30, 15, 15), @@ -108,19 +104,16 @@ t_stats_retained_on_job_removal({Source, Target}) -> couch_replicator_scheduler:remove_job(RepId) end). - setup_db() -> DbName = ?tempdb(), {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]), ok = couch_db:close(Db), DbName. - teardown_db(DbName) -> ok = couch_server:delete(DbName, [?ADMIN_CTX]), ok. - stop_job(RepPid) -> Ref = erlang:monitor(process, RepPid), gen_server:cast(couch_replicator_scheduler, {set_max_jobs, 0}), @@ -131,20 +124,22 @@ stop_job(RepPid) -> erlang:error(timeout) end. - start_job() -> gen_server:cast(couch_replicator_scheduler, {set_max_jobs, 500}), couch_replicator_scheduler:reschedule(). - check_active_tasks(DocsRead, DocsWritten, DocsFailed) -> RepTask = wait_for_task_status(DocsWritten), ?assertNotEqual(timeout, RepTask), ?assertEqual(DocsRead, couch_util:get_value(docs_read, RepTask)), ?assertEqual(DocsWritten, couch_util:get_value(docs_written, RepTask)), - ?assertEqual(DocsFailed, couch_util:get_value(doc_write_failures, - RepTask)). - + ?assertEqual( + DocsFailed, + couch_util:get_value( + doc_write_failures, + RepTask + ) + ). check_scheduler_jobs(DocsRead, DocsWritten, DocFailed) -> Info = wait_scheduler_info(DocsWritten), @@ -160,17 +155,19 @@ check_scheduler_jobs(DocsRead, DocsWritten, DocFailed) -> ?assertMatch(#{<<"docs_written">> := DocsWritten}, Info), ?assertMatch(#{<<"doc_write_failures">> := DocFailed}, Info). - replication_tasks() -> - lists:filter(fun(P) -> - couch_util:get_value(type, P) =:= replication - end, couch_task_status:all()). - + lists:filter( + fun(P) -> + couch_util:get_value(type, P) =:= replication + end, + couch_task_status:all() + ). wait_for_task_status(DocsWritten) -> test_util:wait(fun() -> case replication_tasks() of - [] -> wait; + [] -> + wait; [RepTask] -> case couch_util:get_value(docs_written, RepTask) of DocsWritten -> RepTask; @@ -179,12 +176,13 @@ wait_for_task_status(DocsWritten) -> end end). - wait_scheduler_info(DocsWritten) -> test_util:wait(fun() -> case scheduler_jobs() of - [] -> wait; - [#{<<"info">> := null}] -> wait; + [] -> + wait; + [#{<<"info">> := null}] -> + wait; [#{<<"info">> := Info}] -> case maps:get(<<"docs_written">>, Info, undefined) of DocsWritten -> Info; @@ -193,7 +191,6 @@ wait_scheduler_info(DocsWritten) -> end end). - populate_db_reject_even_docs(DbName, Start, End) -> BodyFun = fun(Id) -> case Id rem 2 == 0 of @@ -203,7 +200,6 @@ populate_db_reject_even_docs(DbName, Start, End) -> end, populate_db(DbName, Start, End, BodyFun). - populate_db(DbName, Start, End, BodyFun) when is_function(BodyFun, 1) -> {ok, Db} = couch_db:open_int(DbName, []), Docs = lists:foldl( @@ -212,27 +208,29 @@ populate_db(DbName, Start, End, BodyFun) when is_function(BodyFun, 1) -> Doc = #doc{id = Id, body = BodyFun(DocIdCounter)}, [Doc | Acc] end, - [], lists:seq(Start, End)), + [], + lists:seq(Start, End) + ), {ok, _} = couch_db:update_docs(Db, Docs, []), ok = couch_db:close(Db). - wait_target_in_sync(DocCount, Target) when is_integer(DocCount) -> wait_target_in_sync_loop(DocCount, Target, 300). - wait_target_in_sync_loop(_DocCount, _TargetName, 0) -> - erlang:error({assertion_failed, [ - {module, ?MODULE}, {line, ?LINE}, - {reason, "Could not get source and target databases in sync"} - ]}); - + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, "Could not get source and target databases in sync"} + ]} + ); wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) -> {ok, Target} = couch_db:open_int(TargetName, []), {ok, TargetInfo} = couch_db:get_db_info(Target), ok = couch_db:close(Target), TargetDocCount = couch_util:get_value(doc_count, TargetInfo), - case TargetDocCount == DocCount of + case TargetDocCount == DocCount of true -> true; false -> @@ -240,22 +238,21 @@ wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) -> wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft - 1) end. - replicate(Source, Target) -> SrcUrl = couch_replicator_test_helper:db_url(Source), TgtUrl = couch_replicator_test_helper:db_url(Target), - RepObject = {[ - {<<"source">>, SrcUrl}, - {<<"target">>, TgtUrl}, - {<<"continuous">>, true} - ]}, + RepObject = + {[ + {<<"source">>, SrcUrl}, + {<<"target">>, TgtUrl}, + {<<"continuous">>, true} + ]}, {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_USER), ok = couch_replicator_scheduler:add_job(Rep), couch_replicator_scheduler:reschedule(), Pid = couch_replicator_test_helper:get_pid(Rep#rep.id), {ok, Pid, Rep#rep.id}. - scheduler_jobs() -> Addr = config:get("chttpd", "bind_address", "127.0.0.1"), Port = mochiweb_socket_server:get(chttpd, port), @@ -264,16 +261,14 @@ scheduler_jobs() -> Json = jiffy:decode(Body, [return_maps]), maps:get(<<"jobs">>, Json). - vdu() -> - <<"function(newDoc, oldDoc, userCtx) { - if(newDoc.nope === true) { - throw({forbidden: 'nope'}); - } else { - return; - } - }">>. - + <<"function(newDoc, oldDoc, userCtx) {\n" + " if(newDoc.nope === true) {\n" + " throw({forbidden: 'nope'});\n" + " } else {\n" + " return;\n" + " }\n" + " }">>. add_vdu(DbName) -> DocProps = [ diff --git a/src/couch_replicator/test/eunit/couch_replicator_selector_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_selector_tests.erl index 5026c1435..8f61a638c 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_selector_tests.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_selector_tests.erl @@ -16,7 +16,6 @@ -include_lib("couch/include/couch_db.hrl"). -include_lib("couch_replicator/src/couch_replicator.hrl"). - setup(_) -> Ctx = test_util:start_couch([couch_replicator]), Source = create_db(), @@ -36,17 +35,19 @@ selector_replication_test_() -> "Selector filtered replication tests", { foreachx, - fun setup/1, fun teardown/2, + fun setup/1, + fun teardown/2, [{Pair, fun should_succeed/2} || Pair <- Pairs] } }. should_succeed({From, To}, {_Ctx, {Source, Target}}) -> - RepObject = {[ - {<<"source">>, db_url(From, Source)}, - {<<"target">>, db_url(To, Target)}, - {<<"selector">>, {[{<<"_id">>, <<"doc2">>}]}} - ]}, + RepObject = + {[ + {<<"source">>, db_url(From, Source)}, + {<<"target">>, db_url(To, Target)}, + {<<"selector">>, {[{<<"_id">>, <<"doc2">>}]}} + ]}, {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER), %% FilteredFun is an Erlang version of following mango selector FilterFun = fun(_DocId, {Props}) -> @@ -55,9 +56,9 @@ should_succeed({From, To}, {_Ctx, {Source, Target}}) -> {ok, TargetDbInfo, AllReplies} = compare_dbs(Source, Target, FilterFun), {lists:flatten(io_lib:format("~p -> ~p", [From, To])), [ {"Target DB has proper number of docs", - ?_assertEqual(1, proplists:get_value(doc_count, TargetDbInfo))}, + ?_assertEqual(1, proplists:get_value(doc_count, TargetDbInfo))}, {"All the docs selected as expected", - ?_assert(lists:all(fun(Valid) -> Valid end, AllReplies))} + ?_assert(lists:all(fun(Valid) -> Valid end, AllReplies))} ]}. compare_dbs(Source, Target, FilterFun) -> @@ -70,10 +71,10 @@ compare_dbs(Source, Target, FilterFun) -> case FilterFun(DocId, SourceDoc) of true -> ValidReply = {ok, DocId, SourceDoc} == TargetReply, - {ok, [ValidReply|Acc]}; + {ok, [ValidReply | Acc]}; false -> ValidReply = {not_found, missing} == TargetReply, - {ok, [ValidReply|Acc]} + {ok, [ValidReply | Acc]} end end, {ok, AllReplies} = couch_db:fold_docs(SourceDb, Fun, [], []), @@ -99,12 +100,16 @@ create_db() -> create_docs(DbName) -> {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]), - Doc1 = couch_doc:from_json_obj({[ - {<<"_id">>, <<"doc1">>} - ]}), - Doc2 = couch_doc:from_json_obj({[ - {<<"_id">>, <<"doc2">>} - ]}), + Doc1 = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"doc1">>} + ]} + ), + Doc2 = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"doc2">>} + ]} + ), {ok, _} = couch_db:update_docs(Db, [Doc1, Doc2]), couch_db:close(Db). diff --git a/src/couch_replicator/test/eunit/couch_replicator_small_max_request_size_target.erl b/src/couch_replicator/test/eunit/couch_replicator_small_max_request_size_target.erl index 91bda33fa..3b020927d 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_small_max_request_size_target.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_small_max_request_size_target.erl @@ -11,17 +11,14 @@ -define(TIMEOUT_EUNIT, 360). - setup() -> DbName = ?tempdb(), {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]), ok = couch_db:close(Db), DbName. - setup(remote) -> {remote, setup()}; - setup({A, B}) -> Ctx = test_util:start_couch([couch_replicator]), config:set("chttpd", "max_http_request_size", "10000", false), @@ -29,7 +26,6 @@ setup({A, B}) -> Target = setup(B), {Ctx, {Source, Target}}. - teardown({remote, DbName}) -> teardown(DbName); teardown(DbName) -> @@ -42,144 +38,147 @@ teardown(_, {Ctx, {Source, Target}}) -> ok = application:stop(couch_replicator), ok = test_util:stop_couch(Ctx). - reduce_max_request_size_test_() -> Pairs = [{remote, remote}], { "Replicate docs when target has a small max_http_request_size", { foreachx, - fun setup/1, fun teardown/2, - [{Pair, fun should_replicate_all_docs/2} - || Pair <- Pairs] - ++ [{Pair, fun should_replicate_one/2} - || Pair <- Pairs] - % Disabled. See issue 574. Sometimes PUTs with a doc and - % attachment which exceed maximum request size are simply - % closed instead of returning a 413 request. That makes these - % tests flaky. - ++ [{Pair, fun should_replicate_one_with_attachment/2} - || Pair <- Pairs] + fun setup/1, + fun teardown/2, + [ + {Pair, fun should_replicate_all_docs/2} + || Pair <- Pairs + ] ++ + [ + {Pair, fun should_replicate_one/2} + || Pair <- Pairs + ] ++ + % Disabled. See issue 574. Sometimes PUTs with a doc and + % attachment which exceed maximum request size are simply + % closed instead of returning a 413 request. That makes these + % tests flaky. + [ + {Pair, fun should_replicate_one_with_attachment/2} + || Pair <- Pairs + ] } }. - % Test documents which are below max_http_request_size but when batched, batch size % will be greater than max_http_request_size. Replicator could automatically split % the batch into smaller batches and POST those separately. should_replicate_all_docs({From, To}, {_Ctx, {Source, Target}}) -> - {lists:flatten(io_lib:format("~p -> ~p", [From, To])), - {inorder, [should_populate_source(Source), - should_replicate(Source, Target), - should_compare_databases(Source, Target, [])]}}. - + { + lists:flatten(io_lib:format("~p -> ~p", [From, To])), + {inorder, [ + should_populate_source(Source), + should_replicate(Source, Target), + should_compare_databases(Source, Target, []) + ]} + }. % If a document is too large to post as a single request, that document is % skipped but replication overall will make progress and not crash. should_replicate_one({From, To}, {_Ctx, {Source, Target}}) -> - {lists:flatten(io_lib:format("~p -> ~p", [From, To])), - {inorder, [should_populate_source_one_large_one_small(Source), - should_replicate(Source, Target), - should_compare_databases(Source, Target, [<<"doc0">>])]}}. - + { + lists:flatten(io_lib:format("~p -> ~p", [From, To])), + {inorder, [ + should_populate_source_one_large_one_small(Source), + should_replicate(Source, Target), + should_compare_databases(Source, Target, [<<"doc0">>]) + ]} + }. % If a document has an attachment > 64 * 1024 bytes, replicator will switch to % POST-ing individual documents directly and skip bulk_docs. Test that case % separately % See note in main test function why this was disabled. should_replicate_one_with_attachment({From, To}, {_Ctx, {Source, Target}}) -> - {lists:flatten(io_lib:format("~p -> ~p", [From, To])), - {inorder, [should_populate_source_one_large_attachment(Source), - should_populate_source(Source), - should_replicate(Source, Target), - should_compare_databases(Source, Target, [<<"doc0">>])]}}. - + { + lists:flatten(io_lib:format("~p -> ~p", [From, To])), + {inorder, [ + should_populate_source_one_large_attachment(Source), + should_populate_source(Source), + should_replicate(Source, Target), + should_compare_databases(Source, Target, [<<"doc0">>]) + ]} + }. should_populate_source({remote, Source}) -> should_populate_source(Source); - should_populate_source(Source) -> {timeout, ?TIMEOUT_EUNIT, ?_test(add_docs(Source, 5, 3000, 0))}. - should_populate_source_one_large_one_small({remote, Source}) -> should_populate_source_one_large_one_small(Source); - should_populate_source_one_large_one_small(Source) -> {timeout, ?TIMEOUT_EUNIT, ?_test(one_large_one_small(Source, 12000, 3000))}. - should_populate_source_one_large_attachment({remote, Source}) -> - should_populate_source_one_large_attachment(Source); - + should_populate_source_one_large_attachment(Source); should_populate_source_one_large_attachment(Source) -> - {timeout, ?TIMEOUT_EUNIT, ?_test(one_large_attachment(Source, 70000, 70000))}. - + {timeout, ?TIMEOUT_EUNIT, ?_test(one_large_attachment(Source, 70000, 70000))}. should_replicate({remote, Source}, Target) -> should_replicate(db_url(Source), Target); - should_replicate(Source, {remote, Target}) -> should_replicate(Source, db_url(Target)); - should_replicate(Source, Target) -> {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target))}. - should_compare_databases({remote, Source}, Target, ExceptIds) -> should_compare_databases(Source, Target, ExceptIds); - should_compare_databases(Source, {remote, Target}, ExceptIds) -> should_compare_databases(Source, Target, ExceptIds); - should_compare_databases(Source, Target, ExceptIds) -> {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target, ExceptIds))}. - binary_chunk(Size) when is_integer(Size), Size > 0 -> - << <<"x">> || _ <- lists:seq(1, Size) >>. - + <<<<"x">> || _ <- lists:seq(1, Size)>>. add_docs(DbName, DocCount, DocSize, AttSize) -> - [begin - DocId = iolist_to_binary(["doc", integer_to_list(Id)]), - add_doc(DbName, DocId, DocSize, AttSize) - end || Id <- lists:seq(1, DocCount)], + [ + begin + DocId = iolist_to_binary(["doc", integer_to_list(Id)]), + add_doc(DbName, DocId, DocSize, AttSize) + end + || Id <- lists:seq(1, DocCount) + ], ok. - one_large_one_small(DbName, Large, Small) -> add_doc(DbName, <<"doc0">>, Large, 0), add_doc(DbName, <<"doc1">>, Small, 0). - one_large_attachment(DbName, Size, AttSize) -> - add_doc(DbName, <<"doc0">>, Size, AttSize). - + add_doc(DbName, <<"doc0">>, Size, AttSize). add_doc(DbName, DocId, Size, AttSize) when is_binary(DocId) -> - {ok, Db} = couch_db:open_int(DbName, []), - Doc0 = #doc{id = DocId, body = {[{<<"x">>, binary_chunk(Size)}]}}, - Doc = Doc0#doc{atts = atts(AttSize)}, - {ok, _} = couch_db:update_doc(Db, Doc, []), - couch_db:close(Db). - + {ok, Db} = couch_db:open_int(DbName, []), + Doc0 = #doc{id = DocId, body = {[{<<"x">>, binary_chunk(Size)}]}}, + Doc = Doc0#doc{atts = atts(AttSize)}, + {ok, _} = couch_db:update_doc(Db, Doc, []), + couch_db:close(Db). atts(0) -> []; - atts(Size) -> - [couch_att:new([ - {name, <<"att1">>}, - {type, <<"app/binary">>}, - {att_len, Size}, - {data, fun(Bytes) -> binary_chunk(Bytes) end} - ])]. - + [ + couch_att:new([ + {name, <<"att1">>}, + {type, <<"app/binary">>}, + {att_len, Size}, + {data, fun(Bytes) -> binary_chunk(Bytes) end} + ]) + ]. replicate(Source, Target) -> - replicate({[ - {<<"source">>, Source}, - {<<"target">>, Target}, - {<<"worker_processes">>, "1"} % This make batch_size predictable - ]}). + replicate( + {[ + {<<"source">>, Source}, + {<<"target">>, Target}, + % This make batch_size predictable + {<<"worker_processes">>, "1"} + ]} + ). diff --git a/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl b/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl index fd0409164..4044e7c72 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl @@ -13,11 +13,9 @@ replicate/2 ]). - compare_dbs(Source, Target) -> compare_dbs(Source, Target, []). - compare_dbs(Source, Target, ExceptIds) -> {ok, SourceDb} = couch_db:open_int(Source, []), {ok, TargetDb} = couch_db:open_int(Target, []), @@ -39,13 +37,14 @@ compare_dbs(Source, Target, ExceptIds) -> ok = couch_db:close(SourceDb), ok = couch_db:close(TargetDb). - compare_docs(Doc1, Doc2) -> ?assertEqual(Doc1#doc.body, Doc2#doc.body), #doc{atts = Atts1} = Doc1, #doc{atts = Atts2} = Doc2, - ?assertEqual(lists:sort([couch_att:fetch(name, Att) || Att <- Atts1]), - lists:sort([couch_att:fetch(name, Att) || Att <- Atts2])), + ?assertEqual( + lists:sort([couch_att:fetch(name, Att) || Att <- Atts1]), + lists:sort([couch_att:fetch(name, Att) || Att <- Atts2]) + ), FunCompareAtts = fun(Att) -> AttName = couch_att:fetch(name, Att), {ok, AttTarget} = find_att(Atts2, AttName), @@ -67,18 +66,25 @@ compare_docs(Doc1, Doc2) -> ?assert(is_integer(couch_att:fetch(att_len, Att))), ?assert(is_integer(couch_att:fetch(disk_len, AttTarget))), ?assert(is_integer(couch_att:fetch(att_len, AttTarget))), - ?assertEqual(couch_att:fetch(disk_len, Att), - couch_att:fetch(disk_len, AttTarget)), - ?assertEqual(couch_att:fetch(att_len, Att), - couch_att:fetch(att_len, AttTarget)), - ?assertEqual(couch_att:fetch(type, Att), - couch_att:fetch(type, AttTarget)), - ?assertEqual(couch_att:fetch(md5, Att), - couch_att:fetch(md5, AttTarget)) + ?assertEqual( + couch_att:fetch(disk_len, Att), + couch_att:fetch(disk_len, AttTarget) + ), + ?assertEqual( + couch_att:fetch(att_len, Att), + couch_att:fetch(att_len, AttTarget) + ), + ?assertEqual( + couch_att:fetch(type, Att), + couch_att:fetch(type, AttTarget) + ), + ?assertEqual( + couch_att:fetch(md5, Att), + couch_att:fetch(md5, AttTarget) + ) end, lists:foreach(FunCompareAtts, Atts1). - find_att([], _Name) -> nil; find_att([Att | Rest], Name) -> @@ -89,38 +95,44 @@ find_att([Att | Rest], Name) -> find_att(Rest, Name) end. - att_md5(Att) -> Md50 = couch_att:foldl( Att, fun(Chunk, Acc) -> couch_hash:md5_hash_update(Acc, Chunk) end, - couch_hash:md5_hash_init()), + couch_hash:md5_hash_init() + ), couch_hash:md5_hash_final(Md50). att_decoded_md5(Att) -> Md50 = couch_att:foldl_decode( Att, fun(Chunk, Acc) -> couch_hash:md5_hash_update(Acc, Chunk) end, - couch_hash:md5_hash_init()), + couch_hash:md5_hash_init() + ), couch_hash:md5_hash_final(Md50). db_url(DbName) -> iolist_to_binary([ - "http://", config:get("httpd", "bind_address", "127.0.0.1"), - ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)), - "/", DbName + "http://", + config:get("httpd", "bind_address", "127.0.0.1"), + ":", + integer_to_list(mochiweb_socket_server:get(couch_httpd, port)), + "/", + DbName ]). get_pid(RepId) -> - Pid = global:whereis_name({couch_replicator_scheduler_job,RepId}), + Pid = global:whereis_name({couch_replicator_scheduler_job, RepId}), ?assert(is_pid(Pid)), Pid. replicate(Source, Target) -> - replicate({[ - {<<"source">>, Source}, - {<<"target">>, Target} - ]}). + replicate( + {[ + {<<"source">>, Source}, + {<<"target">>, Target} + ]} + ). replicate({[_ | _]} = RepObject) -> {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_USER), diff --git a/src/couch_replicator/test/eunit/couch_replicator_use_checkpoints_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_use_checkpoints_tests.erl index 8e4a21dbb..a23f415c0 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_use_checkpoints_tests.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_use_checkpoints_tests.erl @@ -25,19 +25,24 @@ -define(i2l(I), integer_to_list(I)). -define(io2b(Io), iolist_to_binary(Io)). - start(false) -> fun ({finished, _, {CheckpointHistory}}) -> - ?assertEqual([{<<"use_checkpoints">>,false}], CheckpointHistory); + ?assertEqual([{<<"use_checkpoints">>, false}], CheckpointHistory); (_) -> ok end; start(true) -> fun ({finished, _, {CheckpointHistory}}) -> - ?assertNotEqual(false, lists:keyfind(<<"session_id">>, - 1, CheckpointHistory)); + ?assertNotEqual( + false, + lists:keyfind( + <<"session_id">>, + 1, + CheckpointHistory + ) + ); (_) -> ok end. @@ -79,9 +84,12 @@ use_checkpoints_test_() -> "Replication use_checkpoints feature tests", { foreachx, - fun start/1, fun stop/2, - [{UseCheckpoints, fun use_checkpoints_tests/2} - || UseCheckpoints <- [false, true]] + fun start/1, + fun stop/2, + [ + {UseCheckpoints, fun use_checkpoints_tests/2} + || UseCheckpoints <- [false, true] + ] } }. @@ -91,21 +99,26 @@ use_checkpoints_tests(UseCheckpoints, Fun) -> "use_checkpoints: " ++ atom_to_list(UseCheckpoints), { foreachx, - fun setup/1, fun teardown/2, - [{{UseCheckpoints, Fun, Pair}, fun should_test_checkpoints/2} - || Pair <- Pairs] + fun setup/1, + fun teardown/2, + [ + {{UseCheckpoints, Fun, Pair}, fun should_test_checkpoints/2} + || Pair <- Pairs + ] } }. should_test_checkpoints({UseCheckpoints, _, {From, To}}, {_Ctx, {Source, Target, _}}) -> should_test_checkpoints(UseCheckpoints, {From, To}, {Source, Target}). should_test_checkpoints(UseCheckpoints, {From, To}, {Source, Target}) -> - {lists:flatten(io_lib:format("~p -> ~p", [From, To])), - {inorder, [ - should_populate_source(Source, ?DOCS_COUNT), - should_replicate(Source, Target, UseCheckpoints), - should_compare_databases(Source, Target) - ]}}. + { + lists:flatten(io_lib:format("~p -> ~p", [From, To])), + {inorder, [ + should_populate_source(Source, ?DOCS_COUNT), + should_replicate(Source, Target, UseCheckpoints), + should_compare_databases(Source, Target) + ]} + }. should_populate_source({remote, Source}, DocCount) -> should_populate_source(Source, DocCount); @@ -126,7 +139,6 @@ should_compare_databases(Source, {remote, Target}) -> should_compare_databases(Source, Target) -> {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target))}. - populate_db(DbName, DocCount) -> {ok, Db} = couch_db:open_int(DbName, []), Docs = lists:foldl( @@ -135,11 +147,13 @@ populate_db(DbName, DocCount) -> Value = ?io2b(["val", ?i2l(DocIdCounter)]), Doc = #doc{ id = Id, - body = {[ {<<"value">>, Value} ]} + body = {[{<<"value">>, Value}]} }, [Doc | Acc] end, - [], lists:seq(1, DocCount)), + [], + lists:seq(1, DocCount) + ), {ok, _} = couch_db:update_docs(Db, Docs, []), ok = couch_db:close(Db). @@ -150,16 +164,24 @@ compare_dbs(Source, Target) -> {ok, Doc} = couch_db:open_doc(SourceDb, FullDocInfo), {Props} = DocJson = couch_doc:to_json_obj(Doc, [attachments]), DocId = couch_util:get_value(<<"_id">>, Props), - DocTarget = case couch_db:open_doc(TargetDb, DocId) of - {ok, DocT} -> - DocT; - Error -> - erlang:error( - {assertion_failed, - [{module, ?MODULE}, {line, ?LINE}, - {reason, lists:concat(["Error opening document '", - ?b2l(DocId), "' from target: ", - couch_util:to_list(Error)])}]}) + DocTarget = + case couch_db:open_doc(TargetDb, DocId) of + {ok, DocT} -> + DocT; + Error -> + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, + lists:concat([ + "Error opening document '", + ?b2l(DocId), + "' from target: ", + couch_util:to_list(Error) + ])} + ]} + ) end, DocTargetJson = couch_doc:to_json_obj(DocTarget, [attachments]), ?assertEqual(DocJson, DocTargetJson), @@ -170,9 +192,10 @@ compare_dbs(Source, Target) -> ok = couch_db:close(TargetDb). replicate(Source, Target, UseCheckpoints) -> - replicate({[ - {<<"source">>, Source}, - {<<"target">>, Target}, - {<<"use_checkpoints">>, UseCheckpoints} - ]}). - + replicate( + {[ + {<<"source">>, Source}, + {<<"target">>, Target}, + {<<"use_checkpoints">>, UseCheckpoints} + ]} + ). diff --git a/src/couch_stats/src/couch_stats.erl b/src/couch_stats/src/couch_stats.erl index 4fde14acb..e0303fc0f 100644 --- a/src/couch_stats/src/couch_stats.erl +++ b/src/couch_stats/src/couch_stats.erl @@ -29,10 +29,8 @@ update_gauge/2 ]). - -include("couch_stats.hrl"). - -type response() :: ok | {error, unknown_metric}. -type stat() :: {any(), [{atom(), any()}]}. @@ -95,8 +93,9 @@ decrement_counter(Name) -> decrement_counter(Name, Value) -> notify_existing_metric(Name, {dec, Value}, counter). --spec update_histogram(any(), number()) -> response(); - (any(), function()) -> any(). +-spec update_histogram + (any(), number()) -> response(); + (any(), function()) -> any(). update_histogram(Name, Fun) when is_function(Fun, 0) -> Begin = os:timestamp(), Result = Fun(), @@ -118,9 +117,10 @@ update_gauge(Name, Value) -> notify_existing_metric(Name, Op, Type) -> try ok = folsom_metrics:notify_existing_metric(Name, Op, Type) - catch _:_ -> - error_logger:error_msg("unknown metric: ~p", [Name]), - {error, unknown_metric} + catch + _:_ -> + error_logger:error_msg("unknown metric: ~p", [Name]), + {error, unknown_metric} end. -spec sample_type(any(), atom()) -> stat(). diff --git a/src/couch_stats/src/couch_stats_aggregator.erl b/src/couch_stats/src/couch_stats_aggregator.erl index 0416636c9..34b28bfd6 100644 --- a/src/couch_stats/src/couch_stats_aggregator.erl +++ b/src/couch_stats/src/couch_stats_aggregator.erl @@ -30,7 +30,6 @@ terminate/2 ]). - -include("couch_stats.hrl"). -record(st, { @@ -57,17 +56,17 @@ init([]) -> {ok, Descs} = reload_metrics(), CT = erlang:send_after(get_interval(collect), self(), collect), RT = erlang:send_after(get_interval(reload), self(), reload), - {ok, #st{descriptions=Descs, stats=[], collect_timer=CT, reload_timer=RT}}. + {ok, #st{descriptions = Descs, stats = [], collect_timer = CT, reload_timer = RT}}. -handle_call(fetch, _from, #st{stats = Stats}=State) -> +handle_call(fetch, _from, #st{stats = Stats} = State) -> {reply, {ok, Stats}, State}; handle_call(flush, _From, State) -> {reply, ok, collect(State)}; -handle_call(reload, _from, #st{reload_timer=OldRT} = State) -> +handle_call(reload, _from, #st{reload_timer = OldRT} = State) -> timer:cancel(OldRT), {ok, Descriptions} = reload_metrics(), RT = update_timer(reload), - {reply, ok, State#st{descriptions=Descriptions, reload_timer=RT}}; + {reply, ok, State#st{descriptions = Descriptions, reload_timer = RT}}; handle_call(Msg, _From, State) -> {stop, {unknown_call, Msg}, error, State}. @@ -78,7 +77,7 @@ handle_info(collect, State) -> {noreply, collect(State)}; handle_info(reload, State) -> {ok, Descriptions} = reload_metrics(), - {noreply, State#st{descriptions=Descriptions}}; + {noreply, State#st{descriptions = Descriptions}}; handle_info(Msg, State) -> {stop, {unknown_info, Msg}, State}. @@ -101,7 +100,10 @@ reload_metrics() -> ToDelete = sets:subtract(ExistingSet, CurrentSet), ToCreate = sets:subtract(CurrentSet, ExistingSet), sets:fold( - fun({Name, _}, _) -> couch_stats:delete(Name), nil end, + fun({Name, _}, _) -> + couch_stats:delete(Name), + nil + end, nil, ToDelete ), @@ -141,16 +143,16 @@ load_metrics_for_application(AppName) -> end end. -collect(#st{collect_timer=OldCT} = State) -> +collect(#st{collect_timer = OldCT} = State) -> timer:cancel(OldCT), Stats = lists:map( fun({Name, Props}) -> - {Name, [{value, couch_stats:sample(Name)}|Props]} + {Name, [{value, couch_stats:sample(Name)} | Props]} end, State#st.descriptions ), CT = update_timer(collect), - State#st{stats=Stats, collect_timer=CT}. + State#st{stats = Stats, collect_timer = CT}. update_timer(Type) -> Interval = get_interval(Type), diff --git a/src/couch_stats/src/couch_stats_httpd.erl b/src/couch_stats/src/couch_stats_httpd.erl index 0c24d8856..b40ba6094 100644 --- a/src/couch_stats/src/couch_stats_httpd.erl +++ b/src/couch_stats/src/couch_stats_httpd.erl @@ -18,7 +18,7 @@ %% exported for use by chttpd_misc -export([transform_stats/1, nest/1, to_ejson/1, extract_path/2]). -handle_stats_req(#httpd{method='GET', path_parts=[_ | Path]}=Req) -> +handle_stats_req(#httpd{method = 'GET', path_parts = [_ | Path]} = Req) -> flush(Req), Stats0 = couch_stats:fetch(), Stats = transform_stats(Stats0), @@ -27,7 +27,6 @@ handle_stats_req(#httpd{method='GET', path_parts=[_ | Path]}=Req) -> EJSON1 = extract_path(Path, EJSON0), couch_httpd:send_json(Req, EJSON1). - transform_stats(Stats) -> transform_stats(Stats, []). @@ -37,51 +36,55 @@ transform_stats([{Key, Props} | Rest], Acc) -> {_, Type} = proplists:lookup(type, Props), transform_stats(Rest, [{Key, transform_stat(Type, Props)} | Acc]). - transform_stat(counter, Props) -> Props; transform_stat(gauge, Props) -> Props; transform_stat(histogram, Props) -> - lists:map(fun - ({value, Value}) -> - {value, lists:map(fun - ({Key, List}) when Key == percentile; Key == histogram -> - {Key, [tuple_to_list(Item) || Item <- List]}; - (Else) -> - Else - end, Value)}; - (Else) -> - Else - end, Props). - + lists:map( + fun + ({value, Value}) -> + {value, + lists:map( + fun + ({Key, List}) when Key == percentile; Key == histogram -> + {Key, [tuple_to_list(Item) || Item <- List]}; + (Else) -> + Else + end, + Value + )}; + (Else) -> + Else + end, + Props + ). nest(Proplist) -> nest(Proplist, []). nest([], Acc) -> Acc; -nest([{[Key|Keys], Value}|Rest], Acc) -> - Acc1 = case proplists:lookup(Key, Acc) of - {Key, Old} -> - [{Key, nest([{Keys, Value}], Old)}|proplists:delete(Key, Acc)]; - none -> - Term = lists:foldr(fun(K, A) -> [{K, A}] end, Value, Keys), - [{Key, Term}|Acc] - end, +nest([{[Key | Keys], Value} | Rest], Acc) -> + Acc1 = + case proplists:lookup(Key, Acc) of + {Key, Old} -> + [{Key, nest([{Keys, Value}], Old)} | proplists:delete(Key, Acc)]; + none -> + Term = lists:foldr(fun(K, A) -> [{K, A}] end, Value, Keys), + [{Key, Term} | Acc] + end, nest(Rest, Acc1). - -to_ejson([{_, _}|_]=Proplist) -> +to_ejson([{_, _} | _] = Proplist) -> EJSONProps = lists:map( - fun({Key, Value}) -> {maybe_format_key(Key), to_ejson(Value)} end, - Proplist + fun({Key, Value}) -> {maybe_format_key(Key), to_ejson(Value)} end, + Proplist ), {EJSONProps}; to_ejson(NotAProplist) -> NotAProplist. - extract_path([], EJSON) -> EJSON; extract_path([Key | Rest], {Props}) -> @@ -94,7 +97,6 @@ extract_path([Key | Rest], {Props}) -> extract_path([_ | _], _NotAnObject) -> null. - maybe_format_key(Key) when is_list(Key) -> list_to_binary(Key); maybe_format_key(Key) when is_atom(Key) -> diff --git a/src/couch_stats/src/couch_stats_process_tracker.erl b/src/couch_stats/src/couch_stats_process_tracker.erl index fef955efd..c53f0f887 100644 --- a/src/couch_stats/src/couch_stats_process_tracker.erl +++ b/src/couch_stats/src/couch_stats_process_tracker.erl @@ -28,9 +28,7 @@ terminate/2 ]). --record(st, { - -}). +-record(st, {}). -spec track(any()) -> ok. track(Name) -> @@ -60,7 +58,7 @@ handle_cast(Msg, State) -> error_logger:error_msg("~p received unknown cast ~p", [?MODULE, Msg]), {noreply, State}. -handle_info({'DOWN', Ref, _, _, _}=Msg, State) -> +handle_info({'DOWN', Ref, _, _, _} = Msg, State) -> case ets:lookup(?MODULE, Ref) of [] -> error_logger:error_msg( diff --git a/src/couch_stats/src/couch_stats_sup.erl b/src/couch_stats/src/couch_stats_sup.erl index 55755bb83..2a92ac69c 100644 --- a/src/couch_stats/src/couch_stats_sup.erl +++ b/src/couch_stats/src/couch_stats_sup.erl @@ -24,12 +24,11 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). - init([]) -> - {ok, { - {one_for_one, 5, 10}, [ - ?CHILD(couch_stats_aggregator, worker), - ?CHILD(couch_stats_process_tracker, worker) - ] - }}. - + {ok, + { + {one_for_one, 5, 10}, [ + ?CHILD(couch_stats_aggregator, worker), + ?CHILD(couch_stats_process_tracker, worker) + ] + }}. diff --git a/src/couch_tests/setups/couch_epi_dispatch.erl b/src/couch_tests/setups/couch_epi_dispatch.erl index 9c0b6b0b0..0094780d4 100644 --- a/src/couch_tests/setups/couch_epi_dispatch.erl +++ b/src/couch_tests/setups/couch_epi_dispatch.erl @@ -27,14 +27,17 @@ notify/3 ]). - %% ------------------------------------------------------------------ %% API functions definitions %% ------------------------------------------------------------------ dispatch(ServiceId, CallbackModule) -> - couch_tests:new(?MODULE, dispatch, - setup_dispatch(ServiceId, CallbackModule), teardown_dispatch()). + couch_tests:new( + ?MODULE, + dispatch, + setup_dispatch(ServiceId, CallbackModule), + teardown_dispatch() + ). %% ------------------------------------------------------------------ %% setups and teardowns diff --git a/src/couch_tests/src/couch_tests.erl b/src/couch_tests/src/couch_tests.erl index 5dff3c5e1..de80addf5 100644 --- a/src/couch_tests/src/couch_tests.erl +++ b/src/couch_tests/src/couch_tests.erl @@ -116,19 +116,22 @@ validate_fixture(#couch_tests_fixture{} = Fixture0, Args, Opts) -> StartedAppsAfterTeardown = Ctx1#couch_tests_ctx.started_apps, validate_and_report([ - {equal, "Expected applications before calling fixture (~p) " - "to be equal to applications after its calling", - AppsBefore, AppsAfter}, - {equal, "Expected list of started applications (~p) " - "to be equal to #couch_tests_fixture.apps (~p)", - AppsStarted, FixtureApps}, - {equal, "Expected list of started applications (~p) " - "to be equal to #couch_tests_ctx.started_apps (~p)", - AppsStarted, StartedAppsBeforeTeardown}, - {equal, "Expected list of stopped applications (~p) " - "to be equal to #couch_tests_ctx.stopped_apps (~p)", - AppsStarted, StoppedAppsAfterTeardown}, - {equal, "Expected empty list ~i of #couch_tests_ctx.started_apps (~p) " + {equal, + "Expected applications before calling fixture (~p) " + "to be equal to applications after its calling", AppsBefore, AppsAfter}, + {equal, + "Expected list of started applications (~p) " + "to be equal to #couch_tests_fixture.apps (~p)", AppsStarted, FixtureApps}, + {equal, + "Expected list of started applications (~p) " + "to be equal to #couch_tests_ctx.started_apps (~p)", AppsStarted, + StartedAppsBeforeTeardown}, + {equal, + "Expected list of stopped applications (~p) " + "to be equal to #couch_tests_ctx.stopped_apps (~p)", AppsStarted, + StoppedAppsAfterTeardown}, + {equal, + "Expected empty list ~i of #couch_tests_ctx.started_apps (~p) " "after teardown", [], StartedAppsAfterTeardown} ]). @@ -151,16 +154,19 @@ validate_and_report(Sheet) -> %% Helper functions definitions %% ------------------------------------------------------------------ - do_setup([#couch_tests_fixture{setup = Setup} = Fixture | Rest], Ctx0, Acc) -> Ctx1 = Ctx0#couch_tests_ctx{started_apps = []}, #couch_tests_ctx{started_apps = Apps} = Ctx2 = Setup(Fixture, Ctx1), Ctx3 = Ctx2#couch_tests_ctx{started_apps = []}, do_setup(Rest, Ctx3, [Fixture#couch_tests_fixture{apps = Apps} | Acc]); do_setup([], Ctx, Acc) -> - Apps = lists:foldl(fun(#couch_tests_fixture{apps = A}, AppsAcc) -> - A ++ AppsAcc - end, [], Acc), + Apps = lists:foldl( + fun(#couch_tests_fixture{apps = A}, AppsAcc) -> + A ++ AppsAcc + end, + [], + Acc + ), Ctx#couch_tests_ctx{chain = lists:reverse(Acc), started_apps = Apps}. do_teardown(Fixture, Ctx0) -> @@ -175,14 +181,14 @@ do_start_applications([], Acc) -> lists:reverse(Acc); do_start_applications([App | Apps], Acc) -> case application:start(App) of - {error, {already_started, _}} -> - do_start_applications(Apps, Acc); - {error, {not_started, Dep}} -> - do_start_applications([Dep, App | Apps], Acc); - {error, {not_running, Dep}} -> - do_start_applications([Dep, App | Apps], Acc); - ok -> - do_start_applications(Apps, [App | Acc]) + {error, {already_started, _}} -> + do_start_applications(Apps, Acc); + {error, {not_started, Dep}} -> + do_start_applications([Dep, App | Apps], Acc); + {error, {not_running, Dep}} -> + do_start_applications([Dep, App | Apps], Acc); + ok -> + do_start_applications(Apps, [App | Acc]) end. stop_applications(Apps) -> @@ -192,26 +198,25 @@ do_stop_applications([], Acc) -> lists:reverse(Acc); do_stop_applications([App | Apps], Acc) -> case application:stop(App) of - {error, _} -> - do_stop_applications(Apps, Acc); - ok -> - do_stop_applications(Apps, [App | Acc]) + {error, _} -> + do_stop_applications(Apps, Acc); + ok -> + do_stop_applications(Apps, [App | Acc]) end. -remove_duplicates([]) -> +remove_duplicates([]) -> []; remove_duplicates([H | T]) -> [H | [X || X <- remove_duplicates(T), X /= H]]. applications() -> - lists:usort([App || {App, _, _} <-application:which_applications()]). + lists:usort([App || {App, _, _} <- application:which_applications()]). do_validate({equal, _Message, Arg, Arg}, Acc) -> Acc; do_validate({equal, Message, Arg1, Arg2}, Acc) -> [io_lib:format(Message, [Arg1, Arg2]) | Acc]. - %% ------------------------------------------------------------------ %% Tests %% ------------------------------------------------------------------ diff --git a/src/couch_tests/src/couch_tests_combinatorics.erl b/src/couch_tests/src/couch_tests_combinatorics.erl index 343336277..f1ee6dd2e 100644 --- a/src/couch_tests/src/couch_tests_combinatorics.erl +++ b/src/couch_tests/src/couch_tests_combinatorics.erl @@ -65,7 +65,7 @@ powerset(X, [H | T], Acc) -> permutations([]) -> [[]]; -permutations(L) -> +permutations(L) -> [[H | T] || H <- L, T <- permutations(L -- [H])]. %% @doc product({Items1, Items2, ..., ItemsN}) @@ -83,7 +83,7 @@ permutations(L) -> %% ] -spec product(Elements :: list()) -> [list()]. -product([H]) -> +product([H]) -> [[A] || A <- H]; product([H | T]) -> [[A | B] || A <- H, B <- product(T)]. @@ -109,7 +109,6 @@ product([H | T]) -> binary_combinations(NBits) -> product(lists:duplicate(NBits, [true, false])). - %% @doc combinations(N, Items). %% Generate all combinations by choosing N values from a given list of Items %% in sorted order. Each combination is sorted and the entire table is sorted. diff --git a/src/couch_tests/test/couch_tests_app_tests.erl b/src/couch_tests/test/couch_tests_app_tests.erl index 6f9c7e419..97f5c1750 100644 --- a/src/couch_tests/test/couch_tests_app_tests.erl +++ b/src/couch_tests/test/couch_tests_app_tests.erl @@ -25,52 +25,64 @@ teardown(Mocks) -> %% ------------------------------------------------------------------ dummy_setup() -> - couch_tests:new(?MODULE, dummy_setup, + couch_tests:new( + ?MODULE, + dummy_setup, fun(_Fixture, Ctx) -> Ctx end, - fun(_Fixture, Ctx) -> Ctx end). - + fun(_Fixture, Ctx) -> Ctx end + ). setup1(Arg1) -> - couch_tests:new(?MODULE, setup1, + couch_tests:new( + ?MODULE, + setup1, fun(Fixture, Ctx0) -> - Ctx1 = couch_tests:start_applications([asn1], Ctx0), - couch_tests:set_state(Fixture, Ctx1, {Arg1}) + Ctx1 = couch_tests:start_applications([asn1], Ctx0), + couch_tests:set_state(Fixture, Ctx1, {Arg1}) end, fun(_Fixture, Ctx) -> - couch_tests:stop_applications([asn1], Ctx) - end). + couch_tests:stop_applications([asn1], Ctx) + end + ). setup2(Arg1, Arg2) -> - couch_tests:new(?MODULE, setup2, + couch_tests:new( + ?MODULE, + setup2, fun(Fixture, Ctx0) -> - Ctx1 = couch_tests:start_applications([public_key], Ctx0), - couch_tests:set_state(Fixture, Ctx1, {Arg1, Arg2}) + Ctx1 = couch_tests:start_applications([public_key], Ctx0), + couch_tests:set_state(Fixture, Ctx1, {Arg1, Arg2}) end, fun(_Fixture, Ctx) -> - Ctx - end). - + Ctx + end + ). couch_tests_test_() -> { "couch_tests tests", { - foreach, fun setup/0, fun teardown/1, + foreach, + fun setup/0, + fun teardown/1, [ {"chained setup", fun chained_setup/0} ] } }. - chained_setup() -> ?assert(meck:validate(application)), ?assertEqual([], history(application, start)), - Ctx0 = couch_tests:setup([ - setup1(foo), - dummy_setup(), - setup2(bar, baz) - ], [], []), + Ctx0 = couch_tests:setup( + [ + setup1(foo), + dummy_setup(), + setup2(bar, baz) + ], + [], + [] + ), ?assertEqual([asn1, public_key], history(application, start)), ?assertEqual([asn1, public_key], couch_tests:get(started_apps, Ctx0)), @@ -96,7 +108,10 @@ unmock(application) -> history(Module, Function) -> Self = self(), - [A || {Pid, {M, F, [A]}, _Result} <- meck:history(Module) - , Pid =:= Self - , M =:= Module - , F =:= Function]. + [ + A + || {Pid, {M, F, [A]}, _Result} <- meck:history(Module), + Pid =:= Self, + M =:= Module, + F =:= Function + ]. diff --git a/src/custodian/src/custodian.erl b/src/custodian/src/custodian.erl index a16c925b5..5cb7d930c 100644 --- a/src/custodian/src/custodian.erl +++ b/src/custodian/src/custodian.erl @@ -18,4 +18,4 @@ report() -> custodian_util:report(). summary() -> - custodian_util:summary(). + custodian_util:summary(). diff --git a/src/custodian/src/custodian_db_checker.erl b/src/custodian/src/custodian_db_checker.erl index f9ab8c85e..96cf24a30 100644 --- a/src/custodian/src/custodian_db_checker.erl +++ b/src/custodian/src/custodian_db_checker.erl @@ -14,10 +14,8 @@ -behaviour(gen_server). -vsn(1). - -export([start_link/0]). - -export([ init/1, terminate/2, @@ -31,75 +29,64 @@ check_dbs/0 ]). - -record(st, { checker }). - start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - init(_) -> process_flag(trap_exit, true), net_kernel:monitor_nodes(true), {ok, restart_checker(#st{})}. - terminate(_Reason, St) -> couch_util:shutdown_sync(St#st.checker), ok. - handle_call(Msg, _From, St) -> {stop, {invalid_call, Msg}, {invalid_call, Msg}, St}. - handle_cast(refresh, St) -> {noreply, restart_checker(St)}; - handle_cast(Msg, St) -> {stop, {invalid_cast, Msg}, St}. - handle_info({nodeup, _}, St) -> {noreply, restart_checker(St)}; - handle_info({nodedown, _}, St) -> {noreply, restart_checker(St)}; - -handle_info({'EXIT', Pid, normal}, #st{checker=Pid}=St) -> - {noreply, St#st{checker=undefined}}; - -handle_info({'EXIT', Pid, Reason}, #st{checker=Pid}=St) -> +handle_info({'EXIT', Pid, normal}, #st{checker = Pid} = St) -> + {noreply, St#st{checker = undefined}}; +handle_info({'EXIT', Pid, Reason}, #st{checker = Pid} = St) -> couch_log:notice("custodian db checker died ~p", [Reason]), - {noreply, restart_checker(St#st{checker=undefined})}; - + {noreply, restart_checker(St#st{checker = undefined})}; handle_info(Msg, St) -> {stop, {invalid_info, Msg}, St}. - code_change(_OldVsn, St, _Extra) -> {ok, St}. - -restart_checker(#st{checker=undefined}=St) -> +restart_checker(#st{checker = undefined} = St) -> Pid = spawn_link(fun ?MODULE:check_dbs/0), - St#st{checker=Pid}; -restart_checker(#st{checker=Pid}=St) when is_pid(Pid) -> + St#st{checker = Pid}; +restart_checker(#st{checker = Pid} = St) when is_pid(Pid) -> St. - check_dbs() -> {ok, DbsDb} = custodian_util:ensure_dbs_exists(), try - Missing = lists:foldl(fun(DbName, Count) -> - case check_db(DbsDb, DbName) of - ok -> Count; - missing -> Count + 1 - end - end, 0, get_dbs()), + Missing = lists:foldl( + fun(DbName, Count) -> + case check_db(DbsDb, DbName) of + ok -> Count; + missing -> Count + 1 + end + end, + 0, + get_dbs() + ), case Missing == 0 of true -> clear_missing_dbs_alert(); false -> ok @@ -108,7 +95,6 @@ check_dbs() -> couch_db:close(DbsDb) end. - check_db(DbsDb, DbName) when is_binary(DbName) -> try case couch_db:open_doc(DbsDb, DbName, []) of @@ -118,24 +104,22 @@ check_db(DbsDb, DbName) when is_binary(DbName) -> send_missing_db_alert(DbName), missing end - catch _:_ -> - send_missing_db_alert(DbName), - missing + catch + _:_ -> + send_missing_db_alert(DbName), + missing end. - get_dbs() -> lists:flatten([ get_users_db(), get_stats_db() ]). - get_users_db() -> UsersDb = chttpd_auth_cache:dbname(), [list_to_binary(UsersDb)]. - get_stats_db() -> case application:get_env(ioq, stats_db) of {ok, DbName} when is_binary(DbName) -> @@ -146,12 +130,10 @@ get_stats_db() -> [] end. - send_missing_db_alert(DbName) -> couch_log:notice("Missing system database ~s", [DbName]), ?CUSTODIAN_MONITOR:send_missing_db_alert(DbName). - clear_missing_dbs_alert() -> couch_log:notice("All system databases exist.", []), ?CUSTODIAN_MONITOR:clear_missing_dbs_alert(). diff --git a/src/custodian/src/custodian_monitor.erl b/src/custodian/src/custodian_monitor.erl index 3cca046ed..29a347374 100644 --- a/src/custodian/src/custodian_monitor.erl +++ b/src/custodian/src/custodian_monitor.erl @@ -12,17 +12,15 @@ -module(custodian_monitor). - % N.B. that callback return values are ignored -callback send_missing_db_alert(DbName :: binary()) -> Ignored :: any(). - -callback clear_missing_dbs_alert() -> Ignored :: any(). - -callback send_event( - Name :: string(), Count :: non_neg_integer(), Description :: string()) -> + Name :: string(), Count :: non_neg_integer(), Description :: string() +) -> Ignored :: any(). diff --git a/src/custodian/src/custodian_noop_monitor.erl b/src/custodian/src/custodian_noop_monitor.erl index 5c793aeca..4cdd6d1d3 100644 --- a/src/custodian/src/custodian_noop_monitor.erl +++ b/src/custodian/src/custodian_noop_monitor.erl @@ -12,24 +12,19 @@ -module(custodian_noop_monitor). - -behaviour(custodian_monitor). - -export([ send_missing_db_alert/1, clear_missing_dbs_alert/0, send_event/3 ]). - send_missing_db_alert(_DbName) -> false. - clear_missing_dbs_alert() -> false. - send_event(_Name, _Count, _Description) -> false. diff --git a/src/custodian/src/custodian_server.erl b/src/custodian/src/custodian_server.erl index 0c8b87e87..e8bdc13c9 100644 --- a/src/custodian/src/custodian_server.erl +++ b/src/custodian/src/custodian_server.erl @@ -19,8 +19,14 @@ -export([start_link/0]). % gen_server api. --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - code_change/3, terminate/2]). +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + code_change/3, + terminate/2 +]). % exported for callback. -export([ @@ -35,7 +41,7 @@ -record(state, { event_listener, shard_checker, - rescan=false + rescan = false }). -define(VSN_0_2_7, 184129240591641721395874905059581858099). @@ -46,7 +52,6 @@ -define(RELISTEN_DELAY, 5000). -endif. - % public functions. start_link() -> @@ -69,9 +74,10 @@ init(_) -> net_kernel:monitor_nodes(true), ok = config:listen_for_changes(?MODULE, nil), {ok, LisPid} = start_event_listener(), - {ok, start_shard_checker(#state{ - event_listener=LisPid - })}. + {ok, + start_shard_checker(#state{ + event_listener = LisPid + })}. handle_call(_Msg, _From, State) -> {noreply, State}. @@ -81,29 +87,24 @@ handle_cast(refresh, State) -> handle_info({nodeup, _}, State) -> {noreply, start_shard_checker(State)}; - handle_info({nodedown, _}, State) -> {noreply, start_shard_checker(State)}; - -handle_info({'EXIT', Pid, normal}, #state{shard_checker=Pid}=State) -> - NewState = State#state{shard_checker=undefined}, +handle_info({'EXIT', Pid, normal}, #state{shard_checker = Pid} = State) -> + NewState = State#state{shard_checker = undefined}, case State#state.rescan of true -> {noreply, start_shard_checker(NewState)}; false -> {noreply, NewState} end; - -handle_info({'EXIT', Pid, Reason}, #state{shard_checker=Pid}=State) -> +handle_info({'EXIT', Pid, Reason}, #state{shard_checker = Pid} = State) -> couch_log:notice("custodian shard checker died ~p", [Reason]), - NewState = State#state{shard_checker=undefined}, + NewState = State#state{shard_checker = undefined}, {noreply, start_shard_checker(NewState)}; - -handle_info({'EXIT', Pid, Reason}, #state{event_listener=Pid}=State) -> +handle_info({'EXIT', Pid, Reason}, #state{event_listener = Pid} = State) -> couch_log:notice("custodian update notifier died ~p", [Reason]), {ok, Pid1} = start_event_listener(), - {noreply, State#state{event_listener=Pid1}}; - + {noreply, State#state{event_listener = Pid1}}; handle_info(restart_config_listener, State) -> ok = config:listen_for_changes(?MODULE, nil), {noreply, State}. @@ -116,26 +117,24 @@ terminate(_Reason, State) -> code_change(?VSN_0_2_7, State, _Extra) -> ok = config:listen_for_changes(?MODULE, nil), {ok, State}; -code_change(_OldVsn, #state{}=State, _Extra) -> +code_change(_OldVsn, #state{} = State, _Extra) -> {ok, State}. % private functions - -start_shard_checker(#state{shard_checker=undefined}=State) -> +start_shard_checker(#state{shard_checker = undefined} = State) -> State#state{ - shard_checker=spawn_link(fun ?MODULE:check_shards/0), - rescan=false + shard_checker = spawn_link(fun ?MODULE:check_shards/0), + rescan = false }; -start_shard_checker(#state{shard_checker=Pid}=State) when is_pid(Pid) -> - State#state{rescan=true}. - +start_shard_checker(#state{shard_checker = Pid} = State) when is_pid(Pid) -> + State#state{rescan = true}. start_event_listener() -> DbName = mem3_sync:shards_db(), couch_event:link_listener( - ?MODULE, handle_db_event, nil, [{dbname, DbName}] - ). + ?MODULE, handle_db_event, nil, [{dbname, DbName}] + ). handle_db_event(_DbName, updated, _St) -> gen_server:cast(?MODULE, refresh), @@ -146,7 +145,6 @@ handle_db_event(_DbName, _Event, _St) -> check_shards() -> [send_event(Item) || Item <- custodian:summary()]. - send_event({_, Count} = Item) -> Description = describe(Item), Name = check_name(Item), @@ -160,13 +158,28 @@ send_event({_, Count} = Item) -> end, ?CUSTODIAN_MONITOR:send_event(Name, Count, Description). - describe({{safe, N}, Count}) -> - lists:concat([Count, " ", shards(Count), " in cluster with only ", N, - " ", copies(N), " on nodes that are currently up"]); + lists:concat([ + Count, + " ", + shards(Count), + " in cluster with only ", + N, + " ", + copies(N), + " on nodes that are currently up" + ]); describe({{live, N}, Count}) -> - lists:concat([Count, " ", shards(Count), " in cluster with only ", - N, " ", copies(N), " on nodes not in maintenance mode"]); + lists:concat([ + Count, + " ", + shards(Count), + " in cluster with only ", + N, + " ", + copies(N), + " on nodes not in maintenance mode" + ]); describe({conflicted, Count}) -> lists:concat([Count, " conflicted ", shards(Count), " in cluster"]). @@ -185,7 +198,6 @@ copies(1) -> copies(_) -> "copies". - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -200,7 +212,7 @@ config_update_test_() -> fun t_restart_config_listener/1 ] } -}. + }. t_restart_config_listener(_) -> ?_test(begin diff --git a/src/custodian/src/custodian_sup.erl b/src/custodian/src/custodian_sup.erl index 97dbd2321..c2be7c861 100644 --- a/src/custodian/src/custodian_sup.erl +++ b/src/custodian/src/custodian_sup.erl @@ -35,11 +35,11 @@ start_link() -> %% =================================================================== init([]) -> - {ok, { - {one_for_one, 5, 10}, - [ - ?CHILD(custodian_server, worker), - ?CHILD(custodian_db_checker, worker) - ] - }}. - + {ok, + { + {one_for_one, 5, 10}, + [ + ?CHILD(custodian_server, worker), + ?CHILD(custodian_db_checker, worker) + ] + }}. diff --git a/src/custodian/src/custodian_util.erl b/src/custodian/src/custodian_util.erl index 6d5a56093..866bcacb1 100644 --- a/src/custodian/src/custodian_util.erl +++ b/src/custodian/src/custodian_util.erl @@ -26,22 +26,26 @@ %% public functions. summary() -> - Dict0 = dict:from_list([{conflicted, 0}] ++ - [{{live, N}, 0} || N <- lists:seq(0, cluster_n() - 1)] ++ - [{{safe, N}, 0} || N <- lists:seq(0, cluster_n() - 1)]), - Fun = fun(_Id, _Range, {conflicted, _N}, Dict) -> - dict:update_counter(conflicted, 1, Dict); - (_Id, _Range, Item, Dict) -> - dict:update_counter(Item, 1, Dict) + Dict0 = dict:from_list( + [{conflicted, 0}] ++ + [{{live, N}, 0} || N <- lists:seq(0, cluster_n() - 1)] ++ + [{{safe, N}, 0} || N <- lists:seq(0, cluster_n() - 1)] + ), + Fun = fun + (_Id, _Range, {conflicted, _N}, Dict) -> + dict:update_counter(conflicted, 1, Dict); + (_Id, _Range, Item, Dict) -> + dict:update_counter(Item, 1, Dict) end, dict:to_list(fold_dbs(Dict0, Fun)). report() -> - Fun = fun(Id, _Range, {conflicted, N}, Acc) -> - [{Id, {conflicted, N}} | Acc]; - (Id, Range, Item, Acc) -> - [{Id, Range, Item} | Acc] - end, + Fun = fun + (Id, _Range, {conflicted, N}, Acc) -> + [{Id, {conflicted, N}} | Acc]; + (Id, Range, Item, Acc) -> + [{Id, Range, Item} | Acc] + end, fold_dbs([], Fun). ensure_dbs_exists() -> @@ -57,7 +61,7 @@ fold_dbs(Acc, Fun) -> Live = Safe -- maintenance_nodes(Safe), {ok, Db} = ensure_dbs_exists(), try - State0 = #state{live=Live, safe=Safe, callback=Fun, db=Db, acc=Acc}, + State0 = #state{live = Live, safe = Safe, callback = Fun, db = Db, acc = Acc}, {ok, State1} = couch_db:fold_docs(Db, fun fold_dbs1/2, State0, []), State1#state.acc after @@ -66,17 +70,17 @@ fold_dbs(Acc, Fun) -> fold_dbs1(#full_doc_info{id = <<"_design/", _/binary>>}, Acc) -> {ok, Acc}; -fold_dbs1(#full_doc_info{deleted=true}, Acc) -> +fold_dbs1(#full_doc_info{deleted = true}, Acc) -> {ok, Acc}; fold_dbs1(#full_doc_info{id = Id} = FDI, State) -> - InternalAcc = case count_conflicts(FDI) of - 0 -> - State#state.acc; - ConflictCount -> - (State#state.callback)(Id, null, {conflicted, ConflictCount}, State#state.acc) - end, - fold_dbs(Id, load_shards(State#state.db, FDI), State#state{acc=InternalAcc}). - + InternalAcc = + case count_conflicts(FDI) of + 0 -> + State#state.acc; + ConflictCount -> + (State#state.callback)(Id, null, {conflicted, ConflictCount}, State#state.acc) + end, + fold_dbs(Id, load_shards(State#state.db, FDI), State#state{acc = InternalAcc}). fold_dbs(Id, Shards, State) -> IsSafe = fun(#shard{node = N}) -> lists:member(N, State#state.safe) end, @@ -85,27 +89,36 @@ fold_dbs(Id, Shards, State) -> SafeShards = lists:filter(IsSafe, Shards), TargetN = mem3_util:calculate_max_n(Shards), Acc0 = State#state.acc, - Acc1 = case mem3_util:calculate_max_n(LiveShards) of - LiveN when LiveN < TargetN -> - LiveRanges = get_range_counts(LiveN, LiveShards, Shards), - lists:foldl(fun({Range, N}, FAcc) -> - (State#state.callback)(Id, Range, {live, N}, FAcc) - end, Acc0, LiveRanges); - _ -> - Acc0 - end, - Acc2 = case mem3_util:calculate_max_n(SafeShards) of - SafeN when SafeN < TargetN -> - SafeRanges = get_range_counts(SafeN, SafeShards, Shards), - lists:foldl(fun({Range, N}, FAcc) -> - (State#state.callback)(Id, Range, {safe, N}, FAcc) - end, Acc1, SafeRanges); - _ -> - Acc1 - end, + Acc1 = + case mem3_util:calculate_max_n(LiveShards) of + LiveN when LiveN < TargetN -> + LiveRanges = get_range_counts(LiveN, LiveShards, Shards), + lists:foldl( + fun({Range, N}, FAcc) -> + (State#state.callback)(Id, Range, {live, N}, FAcc) + end, + Acc0, + LiveRanges + ); + _ -> + Acc0 + end, + Acc2 = + case mem3_util:calculate_max_n(SafeShards) of + SafeN when SafeN < TargetN -> + SafeRanges = get_range_counts(SafeN, SafeShards, Shards), + lists:foldl( + fun({Range, N}, FAcc) -> + (State#state.callback)(Id, Range, {safe, N}, FAcc) + end, + Acc1, + SafeRanges + ); + _ -> + Acc1 + end, {ok, State#state{acc = Acc2}}. - get_range_counts(MaxN, Shards, AllShards) -> Ranges = ranges(Shards), AllRanges = ranges(AllShards), @@ -131,16 +144,23 @@ get_range_counts(MaxN, Shards, AllShards) -> RangeCounts1 = maps:filter(fun(_, N) -> N =< MaxN end, RangeCounts), lists:sort(maps:to_list(RangeCounts1)). - update_counts(Ranges, Acc0, Init, UpdateFun) -> - lists:foldl(fun({B, E}, Acc) -> - maps:update_with({B, E}, UpdateFun, Init, Acc) - end, Acc0, Ranges). - + lists:foldl( + fun({B, E}, Acc) -> + maps:update_with({B, E}, UpdateFun, Init, Acc) + end, + Acc0, + Ranges + ). ranges(Shards) -> - lists:map(fun(S) -> [B, E] = mem3:range(S), {B, E} end, Shards). - + lists:map( + fun(S) -> + [B, E] = mem3:range(S), + {B, E} + end, + Shards + ). get_n_rings(N, Ranges, Rings) when N =< 0 -> {Ranges, Rings}; @@ -148,7 +168,6 @@ get_n_rings(N, Ranges, Rings) -> Ring = mem3_util:get_ring(Ranges), get_n_rings(N - 1, Ranges -- Ring, Rings ++ Ring). - cluster_n() -> config:get_integer("cluster", "n", 3). @@ -169,19 +188,18 @@ maybe_redirect(Nodes) -> maybe_redirect([], Acc) -> Acc; -maybe_redirect([Node|Rest], Acc) -> +maybe_redirect([Node | Rest], Acc) -> case config:get("mem3.redirects", atom_to_list(Node)) of undefined -> - maybe_redirect(Rest, [Node|Acc]); + maybe_redirect(Rest, [Node | Acc]); Redirect -> - maybe_redirect(Rest, [list_to_atom(Redirect)|Acc]) + maybe_redirect(Rest, [list_to_atom(Redirect) | Acc]) end. count_conflicts(#full_doc_info{rev_tree = T}) -> - Leafs = [1 || {#leaf{deleted=false}, _} <- couch_key_tree:get_all_leafs(T)], + Leafs = [1 || {#leaf{deleted = false}, _} <- couch_key_tree:get_all_leafs(T)], length(Leafs) - 1. - % Ensure the design doc which was added 3.2.0 is deleted as we switched to using a BDU % function instead. After a few releases this function could be removed as well % @@ -203,58 +221,63 @@ ensure_custodian_ddoc_is_deleted(Db) -> end end. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). - get_range_counts_test_() -> - [?_assertEqual(Res, get_range_counts(N, Shards, AllShards)) || {N, Shards, - AllShards, Res} <- [ - % No shards are present. There is a full range shard that would - % fit. Report that range as missing. - {0, [], [full()], [{{0, ?RING_END}, 0}]}, - - % Can't complete the ring. But would complete it if had the - % {2, ?RING_END} interval available. - {0, [sh(0, 1)], [sh(0, 1), sh(2, ?RING_END)], [{{2, ?RING_END}, 0}]}, - - % Can complete the ring only 1 time. Report that range as the - % one available with a count of 1 - {1, [full()], [full(), full()], [{{0, ?RING_END}, 1}]}, - - % Can complete the ring only 1 time with a full range shard, but - % there is also {2, ?RING_END} that would complete another the - % the ring as well if {0, 1} was present. - {1, [sh(2, ?RING_END), full()], [sh(0, 1), sh(2, ?RING_END), full()], - [ + [ + ?_assertEqual(Res, get_range_counts(N, Shards, AllShards)) + || {N, Shards, AllShards, Res} <- [ + % No shards are present. There is a full range shard that would + % fit. Report that range as missing. + {0, [], [full()], [{{0, ?RING_END}, 0}]}, + + % Can't complete the ring. But would complete it if had the + % {2, ?RING_END} interval available. + {0, [sh(0, 1)], [sh(0, 1), sh(2, ?RING_END)], [{{2, ?RING_END}, 0}]}, + + % Can complete the ring only 1 time. Report that range as the + % one available with a count of 1 + {1, [full()], [full(), full()], [{{0, ?RING_END}, 1}]}, + + % Can complete the ring only 1 time with a full range shard, but + % there is also {2, ?RING_END} that would complete another the + % the ring as well if {0, 1} was present. + {1, [sh(2, ?RING_END), full()], [sh(0, 1), sh(2, ?RING_END), full()], [ {{0, 1}, 0}, {{0, ?RING_END}, 1}, {{2, ?RING_END}, 1} - ] - }, - - % Can complete the ring 2 times [{0, 2},{3, ?RING_END)] and full(), - % and there is remnant of a 5, 9 range that would comlete the ring - % as well if {0, 4} and {10, ?RING_END} were present. So report - {2, [sh(0, 2), sh(3, ?RING_END), sh(5, 9), full()], [sh(0, 2), sh(3, - ?RING_END), full(), sh(0, 4), sh(5, 9), sh(10, ?RING_END)], - [ - {{0, 2}, 1}, - {{0, 4}, 0}, - {{0, ?RING_END}, 1}, - {{3, ?RING_END}, 1}, - {{5, 9}, 1}, - {{10, ?RING_END}, 0} - ] - } - ]]. - + ]}, + + % Can complete the ring 2 times [{0, 2},{3, ?RING_END)] and full(), + % and there is remnant of a 5, 9 range that would comlete the ring + % as well if {0, 4} and {10, ?RING_END} were present. So report + {2, [sh(0, 2), sh(3, ?RING_END), sh(5, 9), full()], + [ + sh(0, 2), + sh( + 3, + ?RING_END + ), + full(), + sh(0, 4), + sh(5, 9), + sh(10, ?RING_END) + ], + [ + {{0, 2}, 1}, + {{0, 4}, 0}, + {{0, ?RING_END}, 1}, + {{3, ?RING_END}, 1}, + {{5, 9}, 1}, + {{10, ?RING_END}, 0} + ]} + ] + ]. full() -> #shard{range = [0, ?RING_END]}. - sh(B, E) -> #shard{range = [B, E]}. diff --git a/src/ddoc_cache/src/ddoc_cache.erl b/src/ddoc_cache/src/ddoc_cache.erl index 50cac3039..747abc753 100644 --- a/src/ddoc_cache/src/ddoc_cache.erl +++ b/src/ddoc_cache/src/ddoc_cache.erl @@ -12,7 +12,6 @@ -module(ddoc_cache). - -export([ open_doc/2, open_doc/3, @@ -24,37 +23,31 @@ open/2 ]). - open_doc(DbName, DocId) -> Key = {ddoc_cache_entry_ddocid, {DbName, DocId}}, ddoc_cache_lru:open(Key). - open_doc(DbName, DocId, RevId) -> Key = {ddoc_cache_entry_ddocid_rev, {DbName, DocId, RevId}}, ddoc_cache_lru:open(Key). - open_validation_funs(DbName) -> Key = {ddoc_cache_entry_validation_funs, DbName}, ddoc_cache_lru:open(Key). - open_custom(DbName, Mod) -> Key = {ddoc_cache_entry_custom, {DbName, Mod}}, ddoc_cache_lru:open(Key). - refresh(ShardDbName, DDocIds) when is_list(DDocIds) -> DbName = mem3:dbname(ShardDbName), ddoc_cache_lru:refresh(DbName, DDocIds). - open(DbName, validation_funs) -> open_validation_funs(DbName); open(DbName, Module) when is_atom(Module) -> open_custom(DbName, Module); -open(DbName, <<"_design/", _/binary>>=DDocId) when is_binary(DbName) -> +open(DbName, <<"_design/", _/binary>> = DDocId) when is_binary(DbName) -> open_doc(DbName, DDocId); open(DbName, DDocId) when is_binary(DDocId) -> open_doc(DbName, <<"_design/", DDocId/binary>>). diff --git a/src/ddoc_cache/src/ddoc_cache_app.erl b/src/ddoc_cache/src/ddoc_cache_app.erl index 5afa7ac95..3f2f02d5d 100644 --- a/src/ddoc_cache/src/ddoc_cache_app.erl +++ b/src/ddoc_cache/src/ddoc_cache_app.erl @@ -13,13 +13,10 @@ -module(ddoc_cache_app). -behaviour(application). - -export([start/2, stop/1]). - start(_StartType, _StartArgs) -> ddoc_cache_sup:start_link(). - stop(_State) -> ok. diff --git a/src/ddoc_cache/src/ddoc_cache_entry.erl b/src/ddoc_cache/src/ddoc_cache_entry.erl index 32d3ec1a7..5a1711dd8 100644 --- a/src/ddoc_cache/src/ddoc_cache_entry.erl +++ b/src/ddoc_cache/src/ddoc_cache_entry.erl @@ -43,17 +43,14 @@ do_open/1 ]). - -include("ddoc_cache.hrl"). - -ifndef(TEST). -define(ENTRY_SHUTDOWN_TIMEOUT, 5000). -else. -define(ENTRY_SHUTDOWN_TIMEOUT, 500). -endif. - -record(st, { key, val, @@ -63,28 +60,22 @@ accessed }). - dbname({Mod, Arg}) -> Mod:dbname(Arg). - ddocid({Mod, Arg}) -> Mod:ddocid(Arg). - recover({Mod, Arg}) -> Mod:recover(Arg). - insert({Mod, Arg}, Value) -> Mod:insert(Arg, Value). - start_link(Key, Default) -> Pid = proc_lib:spawn_link(?MODULE, init, [{Key, Default}]), {ok, Pid}. - shutdown(Pid) -> Ref = erlang:monitor(process, Pid), ok = gen_server:cast(Pid, shutdown), @@ -98,7 +89,6 @@ shutdown(Pid) -> erlang:exit({timeout, {entry_shutdown, Pid}}) end. - open(Pid, Key) -> try Resp = gen_server:call(Pid, open), @@ -118,15 +108,12 @@ open(Pid, Key) -> recover(Key) end. - accessed(Pid) -> gen_server:cast(Pid, accessed). - refresh(Pid) -> gen_server:cast(Pid, force_refresh). - init({Key, undefined}) -> true = ets:update_element(?CACHE, Key, {#entry.pid, self()}), St = #st{ @@ -137,7 +124,6 @@ init({Key, undefined}) -> }, ?EVENT(started, Key), gen_server:enter_loop(?MODULE, [], St); - init({Key, Wrapped}) -> Default = ddoc_cache_value:unwrap(Wrapped), Updates = [ @@ -158,7 +144,6 @@ init({Key, Wrapped}) -> ?EVENT(default_started, Key), gen_server:enter_loop(?MODULE, [], St, hibernate). - terminate(_Reason, St) -> #st{ key = Key, @@ -172,30 +157,30 @@ terminate(_Reason, St) -> true = ets:select_delete(?CACHE, CacheMSpec) < 2, % We may have already deleted our LRU entry % during shutdown - if Ts == undefined -> ok; true -> - LruMSpec = [{{{Ts, Key, self()}}, [], [true]}], - true = ets:select_delete(?LRU, LruMSpec) < 2 + if + Ts == undefined -> + ok; + true -> + LruMSpec = [{{{Ts, Key, self()}}, [], [true]}], + true = ets:select_delete(?LRU, LruMSpec) < 2 end, % Blow away any current opener if it exists - if not is_pid(Pid) -> ok; true -> - catch exit(Pid, kill) + if + not is_pid(Pid) -> ok; + true -> catch exit(Pid, kill) end, ok. - handle_call(open, From, #st{opener = Pid} = St) when is_pid(Pid) -> NewSt = St#st{ waiters = [From | St#st.waiters] }, {noreply, NewSt}; - handle_call(open, _From, St) -> {reply, St#st.val, St}; - handle_call(Msg, _From, St) -> {stop, {bad_call, Msg}, {bad_call, Msg}, St}. - handle_cast(accessed, St) -> ?EVENT(accessed, St#st.key), drain_accessed(), @@ -203,7 +188,6 @@ handle_cast(accessed, St) -> accessed = St#st.accessed + 1 }, {noreply, update_lru(NewSt)}; - handle_cast(force_refresh, St) -> % If we had frequent design document updates % they could end up racing accessed events and @@ -211,18 +195,18 @@ handle_cast(force_refresh, St) -> % cache. To prevent this we just make sure that % accessed is set to at least 1 before we % execute a refresh. - NewSt = if St#st.accessed > 0 -> St; true -> - St#st{accessed = 1} - end, + NewSt = + if + St#st.accessed > 0 -> St; + true -> St#st{accessed = 1} + end, % We remove the cache entry value so that any % new client comes to us for the refreshed % value. true = ets:update_element(?CACHE, St#st.key, {#entry.val, undefined}), handle_cast(refresh, NewSt); - handle_cast(refresh, #st{accessed = 0} = St) -> {stop, normal, St}; - handle_cast(refresh, #st{opener = Ref} = St) when is_reference(Ref) -> #st{ key = Key @@ -233,7 +217,6 @@ handle_cast(refresh, #st{opener = Ref} = St) when is_reference(Ref) -> accessed = 0 }, {noreply, NewSt}; - handle_cast(refresh, #st{opener = Pid} = St) when is_pid(Pid) -> catch exit(Pid, kill), receive @@ -244,15 +227,12 @@ handle_cast(refresh, #st{opener = Pid} = St) when is_pid(Pid) -> accessed = 0 }, {noreply, NewSt}; - handle_cast(shutdown, St) -> remove_from_cache(St), {stop, normal, St}; - handle_cast(Msg, St) -> {stop, {bad_cast, Msg}, St}. - handle_info({'DOWN', _, _, Pid, Resp}, #st{key = Key, opener = Pid} = St) -> case Resp of {open_ok, Key, {ok, Val}} -> @@ -275,26 +255,22 @@ handle_info({'DOWN', _, _, Pid, Resp}, #st{key = Key, opener = Pid} = St) -> respond(St#st.waiters, {Status, Other}), {stop, normal, NewSt} end; - handle_info(Msg, St) -> {stop, {bad_info, Msg}, St}. - code_change(_, St, _) -> {ok, St}. - spawn_opener(Key) -> {Pid, _} = erlang:spawn_monitor(?MODULE, do_open, [Key]), Pid. - start_timer() -> TimeOut = config:get_integer( - "ddoc_cache", "refresh_timeout", ?REFRESH_TIMEOUT), + "ddoc_cache", "refresh_timeout", ?REFRESH_TIMEOUT + ), erlang:send_after(TimeOut, self(), {'$gen_cast', refresh}). - do_open(Key) -> try recover(Key) of Resp -> @@ -303,26 +279,21 @@ do_open(Key) -> erlang:exit({open_error, Key, {T, R, S}}) end. - update_lru(#st{key = Key, ts = Ts} = St) -> remove_from_lru(Ts, Key), NewTs = os:timestamp(), true = ets:insert(?LRU, {{NewTs, Key, self()}}), St#st{ts = NewTs}. - update_cache(#st{val = undefined} = St, Val) -> true = ets:update_element(?CACHE, St#st.key, {#entry.val, Val}), ?EVENT(inserted, St#st.key); - update_cache(#st{val = V1} = _St, V2) when {open_ok, {ok, V2}} == V1 -> ?EVENT(update_noop, _St#st.key); - update_cache(St, Val) -> true = ets:update_element(?CACHE, St#st.key, {#entry.val, Val}), ?EVENT(updated, {St#st.key, Val}). - remove_from_cache(St) -> #st{ key = Key, @@ -335,14 +306,15 @@ remove_from_cache(St) -> ?EVENT(removed, St#st.key), ok. - remove_from_lru(Ts, Key) -> - if Ts == undefined -> ok; true -> - LruMSpec = [{{{Ts, Key, self()}}, [], [true]}], - 1 = ets:select_delete(?LRU, LruMSpec) + if + Ts == undefined -> + ok; + true -> + LruMSpec = [{{{Ts, Key, self()}}, [], [true]}], + 1 = ets:select_delete(?LRU, LruMSpec) end. - drain_accessed() -> receive {'$gen_cast', accessed} -> @@ -351,6 +323,5 @@ drain_accessed() -> ok end. - respond(Waiters, Resp) -> [gen_server:reply(W, Resp) || W <- Waiters]. diff --git a/src/ddoc_cache/src/ddoc_cache_entry_custom.erl b/src/ddoc_cache/src/ddoc_cache_entry_custom.erl index 9eaf16f34..8747b46bc 100644 --- a/src/ddoc_cache/src/ddoc_cache_entry_custom.erl +++ b/src/ddoc_cache/src/ddoc_cache_entry_custom.erl @@ -12,7 +12,6 @@ -module(ddoc_cache_entry_custom). - -export([ dbname/1, ddocid/1, @@ -20,18 +19,14 @@ insert/2 ]). - dbname({DbName, _}) -> DbName. - ddocid(_) -> no_ddocid. - recover({DbName, Mod}) -> Mod:recover(DbName). - insert(_, _) -> ok. diff --git a/src/ddoc_cache/src/ddoc_cache_entry_ddocid.erl b/src/ddoc_cache/src/ddoc_cache_entry_ddocid.erl index 5248469fb..cf40725e4 100644 --- a/src/ddoc_cache/src/ddoc_cache_entry_ddocid.erl +++ b/src/ddoc_cache/src/ddoc_cache_entry_ddocid.erl @@ -12,7 +12,6 @@ -module(ddoc_cache_entry_ddocid). - -export([ dbname/1, ddocid/1, @@ -20,27 +19,21 @@ insert/2 ]). - -include_lib("couch/include/couch_db.hrl"). - dbname({DbName, _}) -> DbName. - ddocid({_, DDocId}) -> DDocId. - recover({DbName, DDocId}) -> fabric:open_doc(DbName, DDocId, [ejson_body, ?ADMIN_CTX]). - insert({DbName, DDocId}, {ok, #doc{revs = Revs} = DDoc}) -> {Depth, [RevId | _]} = Revs, Rev = {Depth, RevId}, Key = {ddoc_cache_entry_ddocid_rev, {DbName, DDocId, Rev}}, spawn(fun() -> ddoc_cache_lru:insert(Key, DDoc) end); - insert(_, _) -> ok. diff --git a/src/ddoc_cache/src/ddoc_cache_entry_ddocid_rev.erl b/src/ddoc_cache/src/ddoc_cache_entry_ddocid_rev.erl index 868fa7789..5126f5210 100644 --- a/src/ddoc_cache/src/ddoc_cache_entry_ddocid_rev.erl +++ b/src/ddoc_cache/src/ddoc_cache_entry_ddocid_rev.erl @@ -12,7 +12,6 @@ -module(ddoc_cache_entry_ddocid_rev). - -export([ dbname/1, ddocid/1, @@ -20,28 +19,21 @@ insert/2 ]). - -include_lib("couch/include/couch_db.hrl"). - dbname({DbName, _, _}) -> DbName. - ddocid({_, DDocId, _}) -> DDocId. - recover({DbName, DDocId, Rev}) -> Opts = [ejson_body, ?ADMIN_CTX], {ok, [Resp]} = fabric:open_revs(DbName, DDocId, [Rev], Opts), Resp. - insert({DbName, DDocId, _Rev}, {ok, #doc{} = DDoc}) -> Key = {ddoc_cache_entry_ddocid, {DbName, DDocId}}, spawn(fun() -> ddoc_cache_lru:insert(Key, DDoc) end); - insert(_, _) -> ok. - diff --git a/src/ddoc_cache/src/ddoc_cache_entry_validation_funs.erl b/src/ddoc_cache/src/ddoc_cache_entry_validation_funs.erl index 2182dead6..bcd122252 100644 --- a/src/ddoc_cache/src/ddoc_cache_entry_validation_funs.erl +++ b/src/ddoc_cache/src/ddoc_cache_entry_validation_funs.erl @@ -12,7 +12,6 @@ -module(ddoc_cache_entry_validation_funs). - -export([ dbname/1, ddocid/1, @@ -20,25 +19,24 @@ insert/2 ]). - dbname(DbName) -> DbName. - ddocid(_) -> no_ddocid. - recover(DbName) -> {ok, DDocs} = fabric:design_docs(mem3:dbname(DbName)), - Funs = lists:flatmap(fun(DDoc) -> - case couch_doc:get_validate_doc_fun(DDoc) of - nil -> []; - Fun -> [Fun] - end - end, DDocs), + Funs = lists:flatmap( + fun(DDoc) -> + case couch_doc:get_validate_doc_fun(DDoc) of + nil -> []; + Fun -> [Fun] + end + end, + DDocs + ), {ok, Funs}. - insert(_, _) -> ok. diff --git a/src/ddoc_cache/src/ddoc_cache_lru.erl b/src/ddoc_cache/src/ddoc_cache_lru.erl index 28a8a64c4..10b4aa0f0 100644 --- a/src/ddoc_cache/src/ddoc_cache_lru.erl +++ b/src/ddoc_cache/src/ddoc_cache_lru.erl @@ -14,7 +14,6 @@ -behaviour(gen_server). -vsn(1). - -export([ start_link/0, open/1, @@ -35,24 +34,21 @@ handle_db_event/3 ]). - -include("ddoc_cache.hrl"). - -define(OPENER, ddoc_cache_opener). - -record(st, { - pids, % pid -> key - dbs, % dbname -> docid -> key -> pid + % pid -> key + pids, + % dbname -> docid -> key -> pid + dbs, evictor }). - start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - open(Key) -> try ets:lookup(?CACHE, Key) of [] -> @@ -66,12 +62,12 @@ open(Key) -> couch_stats:increment_counter([ddoc_cache, hit]), ddoc_cache_entry:accessed(Pid), {ok, Val} - catch _:_ -> - couch_stats:increment_counter([ddoc_cache, recovery]), - ddoc_cache_entry:recover(Key) + catch + _:_ -> + couch_stats:increment_counter([ddoc_cache, recovery]), + ddoc_cache_entry:recover(Key) end. - insert(Key, Value) -> case ets:lookup(?CACHE, Key) of [] -> @@ -81,27 +77,26 @@ insert(Key, Value) -> ok end. - refresh(DbName, DDocIds) -> gen_server:cast(?MODULE, {refresh, DbName, DDocIds}). - init(_) -> couch_util:set_mqd_off_heap(?MODULE), process_flag(trap_exit, true), BaseOpts = [public, named_table], - CacheOpts = [ - set, - {read_concurrency, true}, - {keypos, #entry.key} - ] ++ BaseOpts, + CacheOpts = + [ + set, + {read_concurrency, true}, + {keypos, #entry.key} + ] ++ BaseOpts, ets:new(?CACHE, CacheOpts), ets:new(?LRU, [ordered_set, {write_concurrency, true}] ++ BaseOpts), {ok, Pids} = khash:new(), {ok, Dbs} = khash:new(), {ok, Evictor} = couch_event:link_listener( - ?MODULE, handle_db_event, nil, [all_dbs] - ), + ?MODULE, handle_db_event, nil, [all_dbs] + ), ?EVENT(lru_init, nil), {ok, #st{ pids = Pids, @@ -109,7 +104,6 @@ init(_) -> evictor = Evictor }}. - terminate(_Reason, St) -> case is_pid(St#st.evictor) of true -> exit(St#st.evictor, kill); @@ -117,7 +111,6 @@ terminate(_Reason, St) -> end, ok. - handle_call({start, Key, Default}, _From, St) -> #st{ pids = Pids, @@ -141,40 +134,43 @@ handle_call({start, Key, Default}, _From, St) -> [#entry{pid = Pid}] -> {reply, {ok, Pid}, St} end; - handle_call(Msg, _From, St) -> {stop, {invalid_call, Msg}, {invalid_call, Msg}, St}. - handle_cast({evict, DbName}, St) -> gen_server:abcast(mem3:nodes(), ?OPENER, {do_evict, DbName}), {noreply, St}; - handle_cast({refresh, DbName, DDocIds}, St) -> gen_server:abcast(mem3:nodes(), ?OPENER, {do_evict, DbName, DDocIds}), {noreply, St}; - handle_cast({do_evict, DbName}, St) -> #st{ dbs = Dbs } = St, - ToRem = case khash:lookup(Dbs, DbName) of - {value, DDocIds} -> - AccOut = khash:fold(DDocIds, fun(_, Keys, Acc1) -> - khash:to_list(Keys) ++ Acc1 - end, []), - ?EVENT(evicted, DbName), - AccOut; - not_found -> - ?EVENT(evict_noop, DbName), - [] - end, - lists:foreach(fun({Key, Pid}) -> - remove_entry(St, Key, Pid) - end, ToRem), + ToRem = + case khash:lookup(Dbs, DbName) of + {value, DDocIds} -> + AccOut = khash:fold( + DDocIds, + fun(_, Keys, Acc1) -> + khash:to_list(Keys) ++ Acc1 + end, + [] + ), + ?EVENT(evicted, DbName), + AccOut; + not_found -> + ?EVENT(evict_noop, DbName), + [] + end, + lists:foreach( + fun({Key, Pid}) -> + remove_entry(St, Key, Pid) + end, + ToRem + ), khash:del(Dbs, DbName), {noreply, St}; - handle_cast({do_refresh, DbName, DDocIdList}, St) -> #st{ dbs = Dbs @@ -185,28 +181,32 @@ handle_cast({do_refresh, DbName, DDocIdList}, St) -> % design documents. case khash:lookup(Dbs, DbName) of {value, DDocIds} -> - lists:foreach(fun(DDocId) -> - case khash:lookup(DDocIds, DDocId) of - {value, Keys} -> - khash:fold(Keys, fun(_, Pid, _) -> - ddoc_cache_entry:refresh(Pid) - end, nil); - not_found -> - ok - end - end, [no_ddocid | DDocIdList]); + lists:foreach( + fun(DDocId) -> + case khash:lookup(DDocIds, DDocId) of + {value, Keys} -> + khash:fold( + Keys, + fun(_, Pid, _) -> + ddoc_cache_entry:refresh(Pid) + end, + nil + ); + not_found -> + ok + end + end, + [no_ddocid | DDocIdList] + ); not_found -> ok end, {noreply, St}; - handle_cast(Msg, St) -> {stop, {invalid_cast, Msg}, St}. - handle_info({'EXIT', Pid, Reason}, #st{evictor = Pid} = St) -> {stop, Reason, St}; - handle_info({'EXIT', Pid, normal}, St) -> % This clause handles when an entry starts % up but encounters an error or uncacheable @@ -218,34 +218,29 @@ handle_info({'EXIT', Pid, normal}, St) -> khash:del(Pids, Pid), remove_key(St, Key), {noreply, St}; - handle_info(Msg, St) -> {stop, {invalid_info, Msg}, St}. - code_change(_OldVsn, St, _Extra) -> {ok, St}. - handle_db_event(ShardDbName, created, St) -> gen_server:cast(?MODULE, {evict, mem3:dbname(ShardDbName)}), {ok, St}; - handle_db_event(ShardDbName, deleted, St) -> gen_server:cast(?MODULE, {evict, mem3:dbname(ShardDbName)}), {ok, St}; - handle_db_event(_DbName, _Event, St) -> {ok, St}. - lru_start(Key, DoInsert) -> case gen_server:call(?MODULE, {start, Key, undefined}, infinity) of {ok, Pid} -> couch_stats:increment_counter([ddoc_cache, miss]), Resp = ddoc_cache_entry:open(Pid, Key), - if not DoInsert -> ok; true -> - ddoc_cache_entry:insert(Key, Resp) + if + not DoInsert -> ok; + true -> ddoc_cache_entry:insert(Key, Resp) end, Resp; full -> @@ -253,23 +248,23 @@ lru_start(Key, DoInsert) -> ddoc_cache_entry:recover(Key) end. - trim(_, 0) -> full; - trim(St, MaxSize) -> CurSize = ets:info(?CACHE, memory) * erlang:system_info(wordsize), - if CurSize =< MaxSize -> ok; true -> - case ets:first(?LRU) of - {_Ts, Key, Pid} -> - remove_entry(St, Key, Pid), - trim(St, MaxSize); - '$end_of_table' -> - full - end + if + CurSize =< MaxSize -> + ok; + true -> + case ets:first(?LRU) of + {_Ts, Key, Pid} -> + remove_entry(St, Key, Pid), + trim(St, MaxSize); + '$end_of_table' -> + full + end end. - remove_entry(St, Key, Pid) -> #st{ pids = Pids @@ -279,7 +274,6 @@ remove_entry(St, Key, Pid) -> khash:del(Pids, Pid), remove_key(St, Key). - store_key(Dbs, Key, Pid) -> DbName = ddoc_cache_entry:dbname(Key), DDocId = ddoc_cache_entry:ddocid(Key), @@ -298,7 +292,6 @@ store_key(Dbs, Key, Pid) -> khash:put(Dbs, DbName, DDocIds) end. - remove_key(St, Key) -> #st{ dbs = Dbs @@ -317,7 +310,6 @@ remove_key(St, Key) -> _ -> ok end. - unlink_and_flush(Pid) -> erlang:unlink(Pid), % Its possible that the entry process has already exited before diff --git a/src/ddoc_cache/src/ddoc_cache_opener.erl b/src/ddoc_cache/src/ddoc_cache_opener.erl index 52de54217..b6b3dc9d5 100644 --- a/src/ddoc_cache/src/ddoc_cache_opener.erl +++ b/src/ddoc_cache/src/ddoc_cache_opener.erl @@ -14,7 +14,6 @@ -behaviour(gen_server). -vsn(1). - -export([ start_link/0 ]). @@ -28,39 +27,31 @@ code_change/3 ]). - start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - init(_) -> {ok, nil}. terminate(_Reason, _St) -> ok. - handle_call(Msg, _From, St) -> {stop, {invalid_call, Msg}, {invalid_call, Msg}, St}. - % The do_evict clauses are upgrades while we're % in a rolling reboot. handle_cast({do_evict, _} = Msg, St) -> gen_server:cast(ddoc_cache_lru, Msg), {noreply, St}; - handle_cast({do_evict, DbName, DDocIds}, St) -> gen_server:cast(ddoc_cache_lru, {do_refresh, DbName, DDocIds}), {noreply, St}; - handle_cast(Msg, St) -> {stop, {invalid_cast, Msg}, St}. - handle_info(Msg, St) -> {stop, {invalid_info, Msg}, St}. - code_change(_OldVsn, State, _Extra) -> {ok, State}. diff --git a/src/ddoc_cache/src/ddoc_cache_sup.erl b/src/ddoc_cache/src/ddoc_cache_sup.erl index 6fff9ef4f..cf1a1aafb 100644 --- a/src/ddoc_cache/src/ddoc_cache_sup.erl +++ b/src/ddoc_cache/src/ddoc_cache_sup.erl @@ -13,17 +13,14 @@ -module(ddoc_cache_sup). -behaviour(supervisor). - -export([ start_link/0, init/1 ]). - start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). - init([]) -> Children = [ { diff --git a/src/ddoc_cache/src/ddoc_cache_value.erl b/src/ddoc_cache/src/ddoc_cache_value.erl index 21a5bb549..59585ee58 100644 --- a/src/ddoc_cache/src/ddoc_cache_value.erl +++ b/src/ddoc_cache/src/ddoc_cache_value.erl @@ -12,16 +12,13 @@ -module(ddoc_cache_value). - -export([ wrap/1, unwrap/1 ]). - wrap(Value) -> {?MODULE, term_to_binary(Value)}. - unwrap({?MODULE, Bin}) when is_binary(Bin) -> binary_to_term(Bin). diff --git a/src/ddoc_cache/test/eunit/ddoc_cache_basic_test.erl b/src/ddoc_cache/test/eunit/ddoc_cache_basic_test.erl index b576d88bb..54c8c585b 100644 --- a/src/ddoc_cache/test/eunit/ddoc_cache_basic_test.erl +++ b/src/ddoc_cache/test/eunit/ddoc_cache_basic_test.erl @@ -12,32 +12,26 @@ -module(ddoc_cache_basic_test). - -export([ recover/1 ]). - -include_lib("couch/include/couch_db.hrl"). -include_lib("eunit/include/eunit.hrl"). -include("ddoc_cache_test.hrl"). - recover(DbName) -> {ok, {DbName, totes_custom}}. - start_couch() -> Ctx = ddoc_cache_tutil:start_couch(), meck:new(ddoc_cache_ev, [passthrough]), Ctx. - stop_couch(Ctx) -> meck:unload(), ddoc_cache_tutil:stop_couch(Ctx). - check_basic_test_() -> { setup, @@ -54,7 +48,6 @@ check_basic_test_() -> ]) }. - check_no_vdu_test_() -> { setup, @@ -66,7 +59,6 @@ check_no_vdu_test_() -> ]) }. - cache_ddoc({DbName, _}) -> ddoc_cache_tutil:clear(), meck:reset(ddoc_cache_ev), @@ -80,7 +72,6 @@ cache_ddoc({DbName, _}) -> ?assertEqual(Resp1, Resp2), ?assertEqual(2, ets:info(?CACHE, size)). - cache_ddoc_rev({DbName, _}) -> ddoc_cache_tutil:clear(), meck:reset(ddoc_cache_ev), @@ -100,7 +91,6 @@ cache_ddoc_rev({DbName, _}) -> ?assertMatch({ok, #doc{id = ?FOOBAR}}, Resp3), ?assertEqual(2, ets:info(?CACHE, size)). - cache_vdu({DbName, _}) -> ddoc_cache_tutil:clear(), ?assertEqual(0, ets:info(?CACHE, size)), @@ -111,7 +101,6 @@ cache_vdu({DbName, _}) -> ?assertEqual(Resp1, Resp2), ?assertEqual(1, ets:info(?CACHE, size)). - cache_custom({DbName, _}) -> ddoc_cache_tutil:clear(), ?assertEqual(0, ets:info(?CACHE, size)), @@ -122,7 +111,6 @@ cache_custom({DbName, _}) -> ?assertEqual(Resp1, Resp2), ?assertEqual(1, ets:info(?CACHE, size)). - cache_ddoc_refresher_unchanged({DbName, _}) -> ddoc_cache_tutil:clear(), meck:reset(ddoc_cache_ev), @@ -136,7 +124,6 @@ cache_ddoc_refresher_unchanged({DbName, _}) -> Tab2 = lists:sort(ets:tab2list(?CACHE)), ?assertEqual(Tab2, Tab1). - dont_cache_not_found({DbName, _}) -> DDocId = <<"_design/not_found">>, ddoc_cache_tutil:clear(), @@ -145,7 +132,6 @@ dont_cache_not_found({DbName, _}) -> ?assertEqual(0, ets:info(?CACHE, size)), ?assertEqual(0, ets:info(?LRU, size)). - deprecated_api_works({DbName, _}) -> ddoc_cache_tutil:clear(), {ok, _} = ddoc_cache:open(DbName, ?FOOBAR), @@ -153,7 +139,6 @@ deprecated_api_works({DbName, _}) -> {ok, _} = ddoc_cache:open(DbName, ?MODULE), {ok, _} = ddoc_cache:open(DbName, validation_funs). - cache_no_vdu_no_ddoc({DbName, _}) -> ddoc_cache_tutil:clear(), Resp = ddoc_cache:open_validation_funs(DbName), @@ -161,7 +146,6 @@ cache_no_vdu_no_ddoc({DbName, _}) -> ?assertEqual(1, ets:info(?CACHE, size)), ?assertEqual(1, ets:info(?LRU, size)). - cache_no_vdu_empty_ddoc({DbName, _}) -> ddoc_cache_tutil:clear(), DDoc = #doc{ diff --git a/src/ddoc_cache/test/eunit/ddoc_cache_coverage_test.erl b/src/ddoc_cache/test/eunit/ddoc_cache_coverage_test.erl index b1a185bdc..d2d0559c6 100644 --- a/src/ddoc_cache/test/eunit/ddoc_cache_coverage_test.erl +++ b/src/ddoc_cache/test/eunit/ddoc_cache_coverage_test.erl @@ -12,12 +12,10 @@ -module(ddoc_cache_coverage_test). - -include_lib("couch/include/couch_db.hrl"). -include_lib("eunit/include/eunit.hrl"). -include("ddoc_cache_test.hrl"). - coverage_test_() -> { setup, @@ -29,13 +27,11 @@ coverage_test_() -> ] }. - restart_lru() -> send_bad_messages(ddoc_cache_lru), ?assertEqual(ok, ddoc_cache_lru:terminate(bang, {st, a, b, c})), ?assertEqual({ok, foo}, ddoc_cache_lru:code_change(1, foo, [])). - stop_on_evictor_death() -> meck:new(ddoc_cache_ev, [passthrough]), try @@ -54,7 +50,6 @@ stop_on_evictor_death() -> meck:unload() end. - send_bad_messages(Name) -> wait_for_restart(Name, fun() -> ?assertEqual({invalid_call, foo}, gen_server:call(Name, foo)) @@ -66,7 +61,6 @@ send_bad_messages(Name) -> whereis(Name) ! foo end). - wait_for_restart(Server, Fun) -> Ref = erlang:monitor(process, whereis(Server)), Fun(), diff --git a/src/ddoc_cache/test/eunit/ddoc_cache_disabled_test.erl b/src/ddoc_cache/test/eunit/ddoc_cache_disabled_test.erl index d46bdde32..d6538e4a3 100644 --- a/src/ddoc_cache/test/eunit/ddoc_cache_disabled_test.erl +++ b/src/ddoc_cache/test/eunit/ddoc_cache_disabled_test.erl @@ -12,18 +12,15 @@ -module(ddoc_cache_disabled_test). - -include_lib("couch/include/couch_db.hrl"). -include_lib("eunit/include/eunit.hrl"). -include("ddoc_cache_test.hrl"). - start_couch() -> Ctx = ddoc_cache_tutil:start_couch(), config:set("ddoc_cache", "max_size", "0", false), Ctx. - check_disabled_test_() -> { setup, @@ -36,7 +33,6 @@ check_disabled_test_() -> ]) }. - resp_ok({DbName, _}) -> ddoc_cache_tutil:clear(), Resp = ddoc_cache:open_doc(DbName, ?FOOBAR), @@ -44,7 +40,6 @@ resp_ok({DbName, _}) -> ?assertEqual(0, ets:info(?CACHE, size)), ?assertEqual(0, ets:info(?LRU, size)). - resp_not_found({DbName, _}) -> ddoc_cache_tutil:clear(), Resp = ddoc_cache:open_doc(DbName, <<"_design/not_found">>), @@ -52,7 +47,6 @@ resp_not_found({DbName, _}) -> ?assertEqual(0, ets:info(?CACHE, size)), ?assertEqual(0, ets:info(?LRU, size)). - check_effectively_disabled({DbName, _}) -> config:set("ddoc_cache", "max_size", "1", false), ddoc_cache_tutil:clear(), diff --git a/src/ddoc_cache/test/eunit/ddoc_cache_entry_test.erl b/src/ddoc_cache/test/eunit/ddoc_cache_entry_test.erl index c992bea8d..fdba0f030 100644 --- a/src/ddoc_cache/test/eunit/ddoc_cache_entry_test.erl +++ b/src/ddoc_cache/test/eunit/ddoc_cache_entry_test.erl @@ -12,35 +12,28 @@ -module(ddoc_cache_entry_test). - -export([ recover/1 ]). - -include_lib("couch/include/couch_db.hrl"). -include_lib("eunit/include/eunit.hrl"). -include("ddoc_cache_test.hrl"). - recover(<<"foo">>) -> timer:sleep(30000); - recover(DbName) -> {ok, {DbName, such_custom}}. - start_couch() -> Ctx = ddoc_cache_tutil:start_couch(), meck:new(ddoc_cache_ev, [passthrough]), Ctx. - stop_couch(Ctx) -> meck:unload(), ddoc_cache_tutil:stop_couch(Ctx). - check_entry_test_() -> { setup, @@ -57,7 +50,6 @@ check_entry_test_() -> ]) }. - cancel_and_replace_opener(_) -> Key = {ddoc_cache_entry_custom, {<<"foo">>, ?MODULE}}, true = ets:insert_new(?CACHE, #entry{key = Key}), @@ -65,7 +57,9 @@ cancel_and_replace_opener(_) -> Opener1 = element(4, sys:get_state(Entry)), Ref1 = erlang:monitor(process, Opener1), gen_server:cast(Entry, force_refresh), - receive {'DOWN', Ref1, _, _, _} -> ok end, + receive + {'DOWN', Ref1, _, _, _} -> ok + end, Opener2 = element(4, sys:get_state(Entry)), ?assert(Opener2 /= Opener1), ?assert(is_process_alive(Opener2)), @@ -73,34 +67,38 @@ cancel_and_replace_opener(_) -> unlink(Entry), ddoc_cache_entry:shutdown(Entry). - condenses_access_messages({DbName, _}) -> meck:reset(ddoc_cache_ev), Key = {ddoc_cache_entry_custom, {DbName, ?MODULE}}, true = ets:insert(?CACHE, #entry{key = Key}), {ok, Entry} = ddoc_cache_entry:start_link(Key, undefined), erlang:suspend_process(Entry), - lists:foreach(fun(_) -> - gen_server:cast(Entry, accessed) - end, lists:seq(1, 100)), + lists:foreach( + fun(_) -> + gen_server:cast(Entry, accessed) + end, + lists:seq(1, 100) + ), erlang:resume_process(Entry), meck:wait(1, ddoc_cache_ev, event, [accessed, Key], 1000), ?assertError( - timeout, - meck:wait(2, ddoc_cache_ev, event, [accessed, Key], 100) - ), + timeout, + meck:wait(2, ddoc_cache_ev, event, [accessed, Key], 100) + ), unlink(Entry), ddoc_cache_entry:shutdown(Entry). - kill_opener_on_terminate(_) -> - Pid = spawn(fun() -> receive _ -> ok end end), + Pid = spawn(fun() -> + receive + _ -> ok + end + end), ?assert(is_process_alive(Pid)), St = {st, key, val, Pid, waiters, ts, accessed}, ?assertEqual(ok, ddoc_cache_entry:terminate(normal, St)), ?assert(not is_process_alive(Pid)). - evict_when_not_accessed(_) -> meck:reset(ddoc_cache_ev), Key = {ddoc_cache_entry_custom, {<<"bar">>, ?MODULE}}, @@ -116,17 +114,17 @@ evict_when_not_accessed(_) -> AccessCount2 = element(7, sys:get_state(Entry)), ?assertEqual(0, AccessCount2), ok = gen_server:cast(Entry, refresh), - receive {'DOWN', Ref, _, _, Reason} -> Reason end, + receive + {'DOWN', Ref, _, _, Reason} -> Reason + end, ?assertEqual(normal, Reason), ?assertEqual(0, ets:info(?CACHE, size)). - open_dead_entry({DbName, _}) -> Pid = spawn(fun() -> ok end), Key = {ddoc_cache_entry_custom, {DbName, ?MODULE}}, ?assertEqual(recover(DbName), ddoc_cache_entry:open(Pid, Key)). - handles_bad_messages(_) -> CallExpect = {stop, {bad_call, foo}, {bad_call, foo}, baz}, CastExpect = {stop, {bad_cast, foo}, bar}, @@ -135,25 +133,24 @@ handles_bad_messages(_) -> ?assertEqual(CastExpect, ddoc_cache_entry:handle_cast(foo, bar)), ?assertEqual(InfoExpect, ddoc_cache_entry:handle_info(foo, bar)). - handles_code_change(_) -> CCExpect = {ok, bar}, ?assertEqual(CCExpect, ddoc_cache_entry:code_change(foo, bar, baz)). - handles_bad_shutdown_test_() -> - {timeout, 10, ?_test(begin - ErrorPid = spawn(fun() -> - receive - _ -> exit(bad_shutdown) - end - end), - ?assertExit(bad_shutdown, ddoc_cache_entry:shutdown(ErrorPid)), - NotDeadYetPid = spawn(fun() -> - timer:sleep(infinity) - end), - ?assertExit( + {timeout, 10, + ?_test(begin + ErrorPid = spawn(fun() -> + receive + _ -> exit(bad_shutdown) + end + end), + ?assertExit(bad_shutdown, ddoc_cache_entry:shutdown(ErrorPid)), + NotDeadYetPid = spawn(fun() -> + timer:sleep(infinity) + end), + ?assertExit( {timeout, {entry_shutdown, NotDeadYetPid}}, ddoc_cache_entry:shutdown(NotDeadYetPid) ) - end)}. + end)}. diff --git a/src/ddoc_cache/test/eunit/ddoc_cache_ev.erl b/src/ddoc_cache/test/eunit/ddoc_cache_ev.erl index a451342cf..ded2469a4 100644 --- a/src/ddoc_cache/test/eunit/ddoc_cache_ev.erl +++ b/src/ddoc_cache/test/eunit/ddoc_cache_ev.erl @@ -16,6 +16,5 @@ event/2 ]). - event(Name, Arg) -> couch_log:error("~s :: ~s :: ~p", [?MODULE, Name, Arg]). diff --git a/src/ddoc_cache/test/eunit/ddoc_cache_eviction_test.erl b/src/ddoc_cache/test/eunit/ddoc_cache_eviction_test.erl index bd61afc37..68f9f2efe 100644 --- a/src/ddoc_cache/test/eunit/ddoc_cache_eviction_test.erl +++ b/src/ddoc_cache/test/eunit/ddoc_cache_eviction_test.erl @@ -12,33 +12,27 @@ -module(ddoc_cache_eviction_test). - -export([ recover/1 ]). - -include_lib("couch/include/couch_db.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("mem3/include/mem3.hrl"). -include("ddoc_cache_test.hrl"). - recover(DbName) -> {ok, {DbName, totes_custom}}. - start_couch() -> Ctx = ddoc_cache_tutil:start_couch(), meck:new(ddoc_cache_ev, [passthrough]), Ctx. - stop_couch(Ctx) -> meck:unload(), ddoc_cache_tutil:stop_couch(Ctx). - check_eviction_test_() -> { setup, @@ -51,7 +45,6 @@ check_eviction_test_() -> ]) }. - evict_all({DbName, _}) -> ddoc_cache_tutil:clear(), meck:reset(ddoc_cache_ev), @@ -67,7 +60,6 @@ evict_all({DbName, _}) -> meck:wait(4, ddoc_cache_ev, event, [removed, '_'], 1000), ?assertEqual(0, ets:info(?CACHE, size)). - dont_evict_all_unrelated({DbName, _}) -> ddoc_cache_tutil:clear(), meck:reset(ddoc_cache_ev), @@ -82,7 +74,6 @@ dont_evict_all_unrelated({DbName, _}) -> meck:wait(ddoc_cache_ev, event, [evict_noop, <<"test">>], 1000), ?assertEqual(4, ets:info(?CACHE, size)). - check_upgrade_clause({DbName, _}) -> ddoc_cache_tutil:clear(), meck:reset(ddoc_cache_ev), diff --git a/src/ddoc_cache/test/eunit/ddoc_cache_lru_test.erl b/src/ddoc_cache/test/eunit/ddoc_cache_lru_test.erl index 9a5391587..d1dac869a 100644 --- a/src/ddoc_cache/test/eunit/ddoc_cache_lru_test.erl +++ b/src/ddoc_cache/test/eunit/ddoc_cache_lru_test.erl @@ -12,39 +12,33 @@ -module(ddoc_cache_lru_test). - -export([ recover/1 ]). - -include_lib("couch/include/couch_db.hrl"). -include_lib("eunit/include/eunit.hrl"). -include("ddoc_cache_test.hrl"). - recover(<<"pause", _/binary>>) -> - receive go -> ok end, + receive + go -> ok + end, {ok, paused}; - recover(<<"big", _/binary>>) -> {ok, [couch_rand:uniform() || _ <- lists:seq(1, 8192)]}; - recover(DbName) -> {ok, DbName}. - start_couch() -> Ctx = ddoc_cache_tutil:start_couch(), meck:new(ddoc_cache_ev, [passthrough]), Ctx. - stop_couch(Ctx) -> meck:unload(), ddoc_cache_tutil:stop_couch(Ctx). - check_not_started_test() -> % Starting couch, but not ddoc_cache { @@ -73,33 +67,42 @@ check_lru_test_() -> ]) }. - check_multi_start(_) -> ddoc_cache_tutil:clear(), meck:reset(ddoc_cache_ev), Key = {ddoc_cache_entry_custom, {<<"pause">>, ?MODULE}}, % These will all get sent through ddoc_cache_lru - Clients = lists:map(fun(_) -> - spawn_monitor(fun() -> - ddoc_cache_lru:open(Key) - end) - end, lists:seq(1, 10)), + Clients = lists:map( + fun(_) -> + spawn_monitor(fun() -> + ddoc_cache_lru:open(Key) + end) + end, + lists:seq(1, 10) + ), meck:wait(ddoc_cache_ev, event, [started, Key], 1000), - lists:foreach(fun({Pid, _Ref}) -> - ?assert(is_process_alive(Pid)) - end, Clients), + lists:foreach( + fun({Pid, _Ref}) -> + ?assert(is_process_alive(Pid)) + end, + Clients + ), [#entry{pid = Pid}] = ets:tab2list(?CACHE), Opener = element(4, sys:get_state(Pid)), OpenerRef = erlang:monitor(process, Opener), ?assert(is_process_alive(Opener)), Opener ! go, - receive {'DOWN', OpenerRef, _, _, _} -> ok end, - lists:foreach(fun({_, Ref}) -> - receive - {'DOWN', Ref, _, _, normal} -> ok - end - end, Clients). - + receive + {'DOWN', OpenerRef, _, _, _} -> ok + end, + lists:foreach( + fun({_, Ref}) -> + receive + {'DOWN', Ref, _, _, normal} -> ok + end + end, + Clients + ). check_multi_open(_) -> ddoc_cache_tutil:clear(), @@ -112,24 +115,38 @@ check_multi_open(_) -> ddoc_cache_lru:open(Key) end), meck:wait(ddoc_cache_ev, event, [started, Key], 1000), - Clients = [Client1] ++ lists:map(fun(_) -> - spawn_monitor(fun() -> - ddoc_cache_lru:open(Key) - end) - end, lists:seq(1, 9)), - lists:foreach(fun({Pid, _Ref}) -> - ?assert(is_process_alive(Pid)) - end, Clients), + Clients = + [Client1] ++ + lists:map( + fun(_) -> + spawn_monitor(fun() -> + ddoc_cache_lru:open(Key) + end) + end, + lists:seq(1, 9) + ), + lists:foreach( + fun({Pid, _Ref}) -> + ?assert(is_process_alive(Pid)) + end, + Clients + ), [#entry{pid = Pid}] = ets:tab2list(?CACHE), Opener = element(4, sys:get_state(Pid)), OpenerRef = erlang:monitor(process, Opener), ?assert(is_process_alive(Opener)), Opener ! go, - receive {'DOWN', OpenerRef, _, _, _} -> ok end, - lists:foreach(fun({_, Ref}) -> - receive {'DOWN', Ref, _, _, normal} -> ok end - end, Clients). - + receive + {'DOWN', OpenerRef, _, _, _} -> ok + end, + lists:foreach( + fun({_, Ref}) -> + receive + {'DOWN', Ref, _, _, normal} -> ok + end + end, + Clients + ). check_capped_size(_) -> % The extra factor of two in the size checks is @@ -141,19 +158,24 @@ check_capped_size(_) -> MaxSize = 1048576, ddoc_cache_tutil:clear(), meck:reset(ddoc_cache_ev), - lists:foreach(fun(I) -> - DbName = list_to_binary("big_" ++ integer_to_list(I)), - ddoc_cache:open_custom(DbName, ?MODULE), - meck:wait(I, ddoc_cache_ev, event, [started, '_'], 1000), - ?assert(cache_size() < MaxSize * 2) - end, lists:seq(1, 25)), - lists:foreach(fun(I) -> - DbName = list_to_binary("big_" ++ integer_to_list(I)), - ddoc_cache:open_custom(DbName, ?MODULE), - meck:wait(I, ddoc_cache_ev, event, [started, '_'], 1000), - ?assert(cache_size() < MaxSize * 2) - end, lists:seq(26, 100)). - + lists:foreach( + fun(I) -> + DbName = list_to_binary("big_" ++ integer_to_list(I)), + ddoc_cache:open_custom(DbName, ?MODULE), + meck:wait(I, ddoc_cache_ev, event, [started, '_'], 1000), + ?assert(cache_size() < MaxSize * 2) + end, + lists:seq(1, 25) + ), + lists:foreach( + fun(I) -> + DbName = list_to_binary("big_" ++ integer_to_list(I)), + ddoc_cache:open_custom(DbName, ?MODULE), + meck:wait(I, ddoc_cache_ev, event, [started, '_'], 1000), + ?assert(cache_size() < MaxSize * 2) + end, + lists:seq(26, 100) + ). check_cache_refill({DbName, _}) -> ddoc_cache_tutil:clear(), @@ -168,10 +190,13 @@ check_cache_refill({DbName, _}) -> {ddoc_cache_entry_ddocid, {DbName, DDocId}} end, - lists:foreach(fun(I) -> - Key = InitDDoc(I), - meck:wait(ddoc_cache_ev, event, [started, Key], 1000) - end, lists:seq(1, 5)), + lists:foreach( + fun(I) -> + Key = InitDDoc(I), + meck:wait(ddoc_cache_ev, event, [started, Key], 1000) + end, + lists:seq(1, 5) + ), ShardName = mem3:name(hd(mem3:shards(DbName))), {ok, _} = ddoc_cache_lru:handle_db_event(ShardName, deleted, foo), @@ -179,11 +204,13 @@ check_cache_refill({DbName, _}) -> meck:wait(10, ddoc_cache_ev, event, [removed, '_'], 1000), ?assertEqual(0, ets:info(?CACHE, size)), - lists:foreach(fun(I) -> - Key = InitDDoc(I), - meck:wait(ddoc_cache_ev, event, [started, Key], 1000) - end, lists:seq(6, 10)). - + lists:foreach( + fun(I) -> + Key = InitDDoc(I), + meck:wait(ddoc_cache_ev, event, [started, Key], 1000) + end, + lists:seq(6, 10) + ). check_evict_and_exit(_) -> ddoc_cache_tutil:clear(), @@ -214,6 +241,5 @@ check_evict_and_exit(_) -> timer:sleep(500), ?assertEqual({messages, []}, process_info(self(), messages)). - cache_size() -> ets:info(?CACHE, memory) * erlang:system_info(wordsize). diff --git a/src/ddoc_cache/test/eunit/ddoc_cache_no_cache_test.erl b/src/ddoc_cache/test/eunit/ddoc_cache_no_cache_test.erl index 96682910c..8da535294 100644 --- a/src/ddoc_cache/test/eunit/ddoc_cache_no_cache_test.erl +++ b/src/ddoc_cache/test/eunit/ddoc_cache_no_cache_test.erl @@ -12,29 +12,25 @@ -module(ddoc_cache_no_cache_test). - -include_lib("couch/include/couch_db.hrl"). -include_lib("eunit/include/eunit.hrl"). - ddoc(DDocId) -> {ok, #doc{ id = DDocId, revs = {1, [<<"deadbeefdeadbeef">>]}, - body = {[ - {<<"ohai">>, null} - ]} + body = + {[ + {<<"ohai">>, null} + ]} }}. - not_found(_DDocId) -> {not_found, missing}. - return_error(_DDocId) -> {error, timeout}. - no_cache_test_() -> { "ddoc_cache no cache test", @@ -76,12 +72,10 @@ no_cache_open_ok_test(_, _) -> Resp = ddoc_cache:open_doc(<<"foo">>, <<"bar">>), ?_assertEqual(ddoc(<<"bar">>), Resp). - no_cache_open_not_found_test(_, _) -> Resp = ddoc_cache:open_doc(<<"foo">>, <<"baz">>), ?_assertEqual(not_found(<<"baz">>), Resp). - no_cache_open_error_test(_, _) -> Resp = ddoc_cache:open_doc(<<"foo">>, <<"bif">>), ?_assertEqual(return_error(<<"bif">>), Resp). diff --git a/src/ddoc_cache/test/eunit/ddoc_cache_open_error_test.erl b/src/ddoc_cache/test/eunit/ddoc_cache_open_error_test.erl index c7379d26a..8e71b1270 100644 --- a/src/ddoc_cache/test/eunit/ddoc_cache_open_error_test.erl +++ b/src/ddoc_cache/test/eunit/ddoc_cache_open_error_test.erl @@ -12,12 +12,10 @@ -module(ddoc_cache_open_error_test). - -include_lib("couch/include/couch_db.hrl"). -include_lib("eunit/include/eunit.hrl"). -include("ddoc_cache_test.hrl"). - start_couch() -> Ctx = ddoc_cache_tutil:start_couch(), meck:expect(fabric, open_doc, fun(_, ?FOOBAR, _) -> @@ -25,12 +23,10 @@ start_couch() -> end), Ctx. - stop_couch(Ctx) -> meck:unload(), ddoc_cache_tutil:stop_couch(Ctx). - check_open_error_test_() -> { setup, @@ -41,6 +37,5 @@ check_open_error_test_() -> ]) }. - handle_open_error({DbName, _}) -> ?assertError(test_kaboom, ddoc_cache:open_doc(DbName, ?FOOBAR)). diff --git a/src/ddoc_cache/test/eunit/ddoc_cache_open_test.erl b/src/ddoc_cache/test/eunit/ddoc_cache_open_test.erl index 73d644f71..f9a9460e7 100644 --- a/src/ddoc_cache/test/eunit/ddoc_cache_open_test.erl +++ b/src/ddoc_cache/test/eunit/ddoc_cache_open_test.erl @@ -23,85 +23,80 @@ -include_lib("eunit/include/eunit.hrl"). -include("ddoc_cache_test.hrl"). - %% behaviour callbacks dbname(DbName) -> DbName. - ddocid(_) -> no_ddocid. - recover({deleted, _DbName}) -> erlang:error(database_does_not_exist); recover(DbName) -> ddoc_cache_entry_validation_funs:recover(DbName). - insert(_, _) -> ok. - start_couch() -> Ctx = ddoc_cache_tutil:start_couch(), meck:new(ddoc_cache_entry_validation_funs, [passthrough]), - meck:expect(ddoc_cache_entry_validation_funs, recover, - ['_'], meck:passthrough()), + meck:expect( + ddoc_cache_entry_validation_funs, + recover, + ['_'], + meck:passthrough() + ), Ctx. - stop_couch(Ctx) -> meck:unload(), ddoc_cache_tutil:stop_couch(Ctx). - check_open_error_test_() -> - { - setup, - fun start_couch/0, - fun stop_couch/1, - ddoc_cache_tutil:with([ - {"should_return_database_does_not_exist", - fun should_return_database_does_not_exist/1}, - {"should_not_call_recover_when_database_does_not_exist", - fun should_not_call_recover_when_database_does_not_exist/1}, - {"should_call_recover_when_needed", - fun should_call_recover_when_needed/1}, - {"should_call_recover_when_needed", - fun should_not_crash_lru_process/1} - ]) + { + setup, + fun start_couch/0, + fun stop_couch/1, + ddoc_cache_tutil:with([ + {"should_return_database_does_not_exist", fun should_return_database_does_not_exist/1}, + {"should_not_call_recover_when_database_does_not_exist", + fun should_not_call_recover_when_database_does_not_exist/1}, + {"should_call_recover_when_needed", fun should_call_recover_when_needed/1}, + {"should_call_recover_when_needed", fun should_not_crash_lru_process/1} + ]) }. - should_return_database_does_not_exist({DbName, _}) -> ?assertError( database_does_not_exist, - ddoc_cache_lru:open({?MODULE, {deleted, DbName}})). - + ddoc_cache_lru:open({?MODULE, {deleted, DbName}}) + ). should_not_call_recover_when_database_does_not_exist({DbName, _}) -> meck:reset(ddoc_cache_entry_validation_funs), ?assertError( - database_does_not_exist, - ddoc_cache_lru:open({?MODULE, {deleted, DbName}})), + database_does_not_exist, + ddoc_cache_lru:open({?MODULE, {deleted, DbName}}) + ), ?assertError( timeout, - meck:wait(1, ddoc_cache_entry_validation_funs, recover, '_', 100)). - + meck:wait(1, ddoc_cache_entry_validation_funs, recover, '_', 100) + ). should_call_recover_when_needed({DbName, _}) -> meck:reset(ddoc_cache_entry_validation_funs), ddoc_cache_lru:open({?MODULE, DbName}), ?assertEqual( ok, - meck:wait(1, ddoc_cache_entry_validation_funs, recover, '_', 500)). - + meck:wait(1, ddoc_cache_entry_validation_funs, recover, '_', 500) + ). should_not_crash_lru_process({DbName, _}) -> LRUPid = whereis(ddoc_cache_lru), ?assert(is_process_alive(LRUPid)), ?assertError( database_does_not_exist, - ddoc_cache_lru:open({?MODULE, {deleted, DbName}})), + ddoc_cache_lru:open({?MODULE, {deleted, DbName}}) + ), ?assert(is_process_alive(LRUPid)). diff --git a/src/ddoc_cache/test/eunit/ddoc_cache_opener_test.erl b/src/ddoc_cache/test/eunit/ddoc_cache_opener_test.erl index c3846360c..ba5bff057 100644 --- a/src/ddoc_cache/test/eunit/ddoc_cache_opener_test.erl +++ b/src/ddoc_cache/test/eunit/ddoc_cache_opener_test.erl @@ -12,12 +12,10 @@ -module(ddoc_cache_opener_test). - -include_lib("couch/include/couch_db.hrl"). -include_lib("eunit/include/eunit.hrl"). -include("ddoc_cache_test.hrl"). - empty_hull_test() -> InitExpect = {ok, nil}, TermExpect = ok, diff --git a/src/ddoc_cache/test/eunit/ddoc_cache_refresh_test.erl b/src/ddoc_cache/test/eunit/ddoc_cache_refresh_test.erl index 24ae346d4..2db5b6cd2 100644 --- a/src/ddoc_cache/test/eunit/ddoc_cache_refresh_test.erl +++ b/src/ddoc_cache/test/eunit/ddoc_cache_refresh_test.erl @@ -12,32 +12,26 @@ -module(ddoc_cache_refresh_test). - -export([ recover/1 ]). - -include_lib("couch/include/couch_db.hrl"). -include_lib("eunit/include/eunit.hrl"). -include("ddoc_cache_test.hrl"). - recover(DbName) -> {ok, {DbName, rand_string()}}. - start_couch() -> Ctx = ddoc_cache_tutil:start_couch(), meck:new(ddoc_cache_ev, [passthrough]), Ctx. - stop_couch(Ctx) -> meck:unload(), ddoc_cache_tutil:stop_couch(Ctx). - check_refresh_test_() -> { setup, @@ -53,7 +47,6 @@ check_refresh_test_() -> ]) }. - refresh_ddoc({DbName, _}) -> ddoc_cache_tutil:clear(), meck:reset(ddoc_cache_ev), @@ -74,7 +67,6 @@ refresh_ddoc({DbName, _}) -> ?assertMatch({ok, Expect}, ddoc_cache:open_doc(DbName, ?FOOBAR)), ?assertEqual(2, ets:info(?CACHE, size)). - refresh_ddoc_rev({DbName, _}) -> ddoc_cache_tutil:clear(), meck:reset(ddoc_cache_ev), @@ -95,7 +87,6 @@ refresh_ddoc_rev({DbName, _}) -> ?assertMatch({ok, RevDDoc}, ddoc_cache:open_doc(DbName, ?FOOBAR, Rev)), ?assertEqual(2, ets:info(?CACHE, size)). - refresh_vdu({DbName, _}) -> ddoc_cache_tutil:clear(), meck:reset(ddoc_cache_ev), @@ -107,7 +98,6 @@ refresh_vdu({DbName, _}) -> ?assertMatch({ok, []}, ddoc_cache:open_validation_funs(DbName)), ?assertEqual(1, ets:info(?CACHE, size)). - refresh_custom({DbName, _}) -> ddoc_cache_tutil:clear(), meck:reset(ddoc_cache_ev), @@ -118,7 +108,6 @@ refresh_custom({DbName, _}) -> ?assertNotEqual({ok, Resp1}, ddoc_cache:open_custom(DbName, ?MODULE)), ?assertEqual(1, ets:info(?CACHE, size)). - refresh_multiple({DbName, _}) -> ddoc_cache_tutil:clear(), meck:reset(ddoc_cache_ev), @@ -147,7 +136,6 @@ refresh_multiple({DbName, _}) -> ?assertEqual({ok, DDoc}, ddoc_cache:open_doc(DbName, ?FOOBAR, Rev)), ?assertEqual(2, ets:info(?CACHE, size)). - check_upgrade_clause({DbName, _}) -> ddoc_cache_tutil:clear(), meck:reset(ddoc_cache_ev), @@ -156,18 +144,15 @@ check_upgrade_clause({DbName, _}) -> gen_server:cast(ddoc_cache_opener, {do_evict, DbName, [?FOOBAR]}), meck:wait(ddoc_cache_ev, event, [update_noop, Key], 1000). - rand_string() -> Bin = crypto:strong_rand_bytes(8), to_hex(Bin, []). - to_hex(<<>>, Acc) -> list_to_binary(lists:reverse(Acc)); to_hex(<<C1:4, C2:4, Rest/binary>>, Acc) -> to_hex(Rest, [hexdig(C1), hexdig(C2) | Acc]). - hexdig(C) when C >= 0, C =< 9 -> C + $0; hexdig(C) when C >= 10, C =< 15 -> diff --git a/src/ddoc_cache/test/eunit/ddoc_cache_remove_test.erl b/src/ddoc_cache/test/eunit/ddoc_cache_remove_test.erl index e40518529..f974dd804 100644 --- a/src/ddoc_cache/test/eunit/ddoc_cache_remove_test.erl +++ b/src/ddoc_cache/test/eunit/ddoc_cache_remove_test.erl @@ -12,18 +12,15 @@ -module(ddoc_cache_remove_test). - -export([ recover/1 ]). - -include_lib("couch/include/couch_db.hrl"). -include_lib("mem3/include/mem3.hrl"). -include_lib("eunit/include/eunit.hrl"). -include("ddoc_cache_test.hrl"). - recover(DbName) -> {ok, #doc{body = {Body}}} = fabric:open_doc(DbName, ?CUSTOM, [?ADMIN_CTX]), case couch_util:get_value(<<"status">>, Body) of @@ -35,18 +32,15 @@ recover(DbName) -> erlang:error(thpppt) end. - start_couch() -> Ctx = ddoc_cache_tutil:start_couch(), meck:new(ddoc_cache_ev, [passthrough]), Ctx. - stop_couch(Ctx) -> meck:unload(), ddoc_cache_tutil:stop_couch(Ctx). - check_refresh_test_() -> { setup, @@ -61,7 +55,6 @@ check_refresh_test_() -> ]) }. - remove_ddoc({DbName, _}) -> ddoc_cache_tutil:clear(), meck:reset(ddoc_cache_ev), @@ -88,7 +81,6 @@ remove_ddoc({DbName, _}) -> ?assertMatch({not_found, deleted}, ddoc_cache:open_doc(DbName, ?FOOBAR)), ?assertEqual(1, ets:info(?CACHE, size)). - remove_ddoc_rev({DbName, _}) -> ddoc_cache_tutil:clear(), meck:reset(ddoc_cache_ev), @@ -100,8 +92,8 @@ remove_ddoc_rev({DbName, _}) -> % Notice the sort so that we know we're getting the % revid version second. - [_, #entry{key = Key, val = DDoc, pid = Pid}] - = lists:sort(ets:tab2list(?CACHE)), + [_, #entry{key = Key, val = DDoc, pid = Pid}] = + lists:sort(ets:tab2list(?CACHE)), NewDDoc = DDoc#doc{ body = {[{<<"an">>, <<"update">>}]} @@ -109,19 +101,21 @@ remove_ddoc_rev({DbName, _}) -> {ok, _} = fabric:update_doc(DbName, NewDDoc, [?ADMIN_CTX]), meck:wait(ddoc_cache_ev, event, [update_noop, Key], 1000), % Compact the database so that the old rev is removed - lists:foreach(fun(Shard) -> - do_compact(Shard#shard.name) - end, mem3:local_shards(DbName)), + lists:foreach( + fun(Shard) -> + do_compact(Shard#shard.name) + end, + mem3:local_shards(DbName) + ), % Trigger a refresh rather than wait for the timeout ddoc_cache_entry:refresh(Pid), meck:wait(ddoc_cache_ev, event, [removed, Key], 1000), ?assertMatch( - {{not_found, missing}, _}, - ddoc_cache:open_doc(DbName, ?VDU, Rev) - ), + {{not_found, missing}, _}, + ddoc_cache:open_doc(DbName, ?VDU, Rev) + ), ?assertEqual(1, ets:info(?CACHE, size)). - remove_ddoc_rev_only({DbName, _}) -> ddoc_cache_tutil:clear(), meck:reset(ddoc_cache_ev), @@ -141,9 +135,12 @@ remove_ddoc_rev_only({DbName, _}) -> meck:wait(ddoc_cache_ev, event, [updated, '_'], 1000), meck:wait(ddoc_cache_ev, event, [update_noop, RevKey], 1000), % Compact the database so that the old rev is removed - lists:foreach(fun(Shard) -> - do_compact(Shard#shard.name) - end, mem3:local_shards(DbName)), + lists:foreach( + fun(Shard) -> + do_compact(Shard#shard.name) + end, + mem3:local_shards(DbName) + ), % Trigger a refresh rather than wait for the timeout ddoc_cache_entry:refresh(NoRevPid), ddoc_cache_entry:refresh(RevPid), @@ -151,9 +148,9 @@ remove_ddoc_rev_only({DbName, _}) -> meck:wait(ddoc_cache_ev, event, [removed, RevKey], 1000), ?assertMatch({ok, _}, ddoc_cache:open_doc(DbName, ?VDU)), ?assertMatch( - {{not_found, missing}, _}, - ddoc_cache:open_doc(DbName, ?VDU, Rev) - ), + {{not_found, missing}, _}, + ddoc_cache:open_doc(DbName, ?VDU, Rev) + ), ?assertEqual(1, ets:info(?CACHE, size)). remove_custom_not_ok({DbName, _}) -> @@ -171,7 +168,6 @@ remove_custom_not_ok({DbName, _}) -> ?assertEqual({ruh, roh}, ddoc_cache:open_custom(DbName, ?MODULE)), ?assertEqual(0, ets:info(?CACHE, size)). - remove_custom_error({DbName, _}) -> ddoc_cache_tutil:clear(), meck:reset(ddoc_cache_ev), @@ -187,14 +183,12 @@ remove_custom_error({DbName, _}) -> ?assertError(thpppt, ddoc_cache:open_custom(DbName, ?MODULE)), ?assertEqual(0, ets:info(?CACHE, size)). - init_custom_ddoc(DbName) -> Body = {[{<<"status">>, <<"ok">>}]}, {ok, Doc} = fabric:open_doc(DbName, ?CUSTOM, [?ADMIN_CTX]), NewDoc = Doc#doc{body = Body}, {ok, _} = fabric:update_doc(DbName, NewDoc, [?ADMIN_CTX]). - do_compact(ShardName) -> {ok, Db} = couch_db:open_int(ShardName, []), try @@ -209,16 +203,19 @@ do_compact(ShardName) -> end, wait_for_compaction(ShardName). - wait_for_compaction(ShardName) -> {ok, Db} = couch_db:open_int(ShardName, []), - CompactRunning = try - {ok, Info} = couch_db:get_db_info(Db), - couch_util:get_value(compact_running, Info) - after - couch_db:close(Db) - end, - if not CompactRunning -> ok; true -> - timer:sleep(100), - wait_for_compaction(ShardName) - end.
\ No newline at end of file + CompactRunning = + try + {ok, Info} = couch_db:get_db_info(Db), + couch_util:get_value(compact_running, Info) + after + couch_db:close(Db) + end, + if + not CompactRunning -> + ok; + true -> + timer:sleep(100), + wait_for_compaction(ShardName) + end. diff --git a/src/ddoc_cache/test/eunit/ddoc_cache_tutil.erl b/src/ddoc_cache/test/eunit/ddoc_cache_tutil.erl index b34d4b163..cc7d694f0 100644 --- a/src/ddoc_cache/test/eunit/ddoc_cache_tutil.erl +++ b/src/ddoc_cache/test/eunit/ddoc_cache_tutil.erl @@ -12,7 +12,6 @@ -module(ddoc_cache_tutil). - -export([ start_couch/0, start_couch/1, @@ -24,36 +23,31 @@ with/1 ]). - -include_lib("couch/include/couch_db.hrl"). -include_lib("couch/include/couch_eunit.hrl"). - start_couch() -> start_couch([{write_ddocs, true}]). - start_couch(Options) -> WriteDDocs = couch_util:get_value(write_ddocs, Options, true), purge_modules(), Ctx = test_util:start_couch(?CONFIG_CHAIN, [chttpd, ddoc_cache]), TmpDb = ?tempdb(), ok = fabric:create_db(TmpDb, [{q, "1"}, {n, "1"}]), - if not WriteDDocs -> ok; true -> - {ok, _} = fabric:update_docs(TmpDb, ddocs(), [?ADMIN_CTX]) + if + not WriteDDocs -> ok; + true -> {ok, _} = fabric:update_docs(TmpDb, ddocs(), [?ADMIN_CTX]) end, {TmpDb, Ctx}. - stop_couch({_TmpDb, Ctx}) -> test_util:stop_couch(Ctx). - clear() -> application:stop(ddoc_cache), application:start(ddoc_cache). - get_rev(DbName, DDocId) -> {_, Ref} = erlang:spawn_monitor(fun() -> {ok, #doc{revs = Revs}} = fabric:open_doc(DbName, DDocId, [?ADMIN_CTX]), @@ -64,48 +58,52 @@ get_rev(DbName, DDocId) -> {'DOWN', Ref, _, _, Rev} -> Rev end. - ddocs() -> FooBar = #doc{ id = <<"_design/foobar">>, - body = {[ - {<<"foo">>, <<"bar">>} - ]} + body = + {[ + {<<"foo">>, <<"bar">>} + ]} }, VDU = #doc{ id = <<"_design/vdu">>, - body = {[ - {<<"validate_doc_update">>, <<"function(doc) {return;}">>} - ]} + body = + {[ + {<<"validate_doc_update">>, <<"function(doc) {return;}">>} + ]} }, Custom = #doc{ id = <<"_design/custom">>, - body = {[ - {<<"status">>, <<"ok">>}, - {<<"custom">>, <<"hotrod">>} - ]} + body = + {[ + {<<"status">>, <<"ok">>}, + {<<"custom">>, <<"hotrod">>} + ]} }, [FooBar, VDU, Custom]. - purge_modules() -> case application:get_key(ddoc_cache, modules) of {ok, Mods} -> - lists:foreach(fun(Mod) -> - case code:which(Mod) of - cover_compiled -> - ok; - _ -> - code:delete(Mod), - code:purge(Mod) - end - end, Mods); + lists:foreach( + fun(Mod) -> + case code:which(Mod) of + cover_compiled -> + ok; + _ -> + code:delete(Mod), + code:purge(Mod) + end + end, + Mods + ); undefined -> ok end. %% eunit implementation of {with, Tests} doesn't detect test name correctly with(Tests) -> - fun(ArgsTuple) -> - [{Name, ?_test(Fun(ArgsTuple))} || {Name, Fun} <- Tests] - end. + fun(ArgsTuple) -> + [{Name, ?_test(Fun(ArgsTuple))} || {Name, Fun} <- Tests] + end. diff --git a/src/dreyfus/src/clouseau_rpc.erl b/src/dreyfus/src/clouseau_rpc.erl index b8190b32c..908182793 100644 --- a/src/dreyfus/src/clouseau_rpc.erl +++ b/src/dreyfus/src/clouseau_rpc.erl @@ -10,7 +10,6 @@ % License for the specific language governing permissions and limitations under % the License. - %% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- -module(clouseau_rpc). @@ -55,12 +54,12 @@ search(Ref, Args) -> case rpc(Ref, {search, Args}) of {ok, Response} when is_list(Response) -> {ok, #top_docs{ - update_seq = couch_util:get_value(update_seq, Response), - total_hits = couch_util:get_value(total_hits, Response), - hits = couch_util:get_value(hits, Response), - counts = couch_util:get_value(counts, Response), - ranges = couch_util:get_value(ranges, Response) - }}; + update_seq = couch_util:get_value(update_seq, Response), + total_hits = couch_util:get_value(total_hits, Response), + hits = couch_util:get_value(hits, Response), + counts = couch_util:get_value(counts, Response), + ranges = couch_util:get_value(ranges, Response) + }}; Else -> Else end. @@ -81,7 +80,7 @@ cleanup(DbName) -> gen_server:cast({cleanup, clouseau()}, {cleanup, DbName}). rename(DbName) -> - gen_server:cast({cleanup, clouseau()}, {rename, DbName}). + gen_server:cast({cleanup, clouseau()}, {rename, DbName}). cleanup(DbName, ActiveSigs) -> gen_server:cast({cleanup, clouseau()}, {cleanup, DbName, ActiveSigs}). diff --git a/src/dreyfus/src/dreyfus_app.erl b/src/dreyfus/src/dreyfus_app.erl index 7cd7f4a31..3f2fc5086 100644 --- a/src/dreyfus/src/dreyfus_app.erl +++ b/src/dreyfus/src/dreyfus_app.erl @@ -10,7 +10,6 @@ % License for the specific language governing permissions and limitations under % the License. - %% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- -module(dreyfus_app). diff --git a/src/dreyfus/src/dreyfus_bookmark.erl b/src/dreyfus/src/dreyfus_bookmark.erl index 9a2979b25..e33087a27 100644 --- a/src/dreyfus/src/dreyfus_bookmark.erl +++ b/src/dreyfus/src/dreyfus_bookmark.erl @@ -10,7 +10,6 @@ % License for the specific language governing permissions and limitations under % the License. - %% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- -module(dreyfus_bookmark). @@ -25,7 +24,6 @@ add_missing_shards/2 ]). - update(_Sort, Bookmark, []) -> Bookmark; update(relevance, Bookmark, [#sortable{} = Sortable | Rest]) -> @@ -45,46 +43,46 @@ update(Sort, Bookmark, [#sortable{} = Sortable | Rest]) -> B2 = fabric_view:remove_overlapping_shards(Shard, B1), update(Sort, B2, Rest). - -unpack(DbName, #index_query_args{bookmark=nil} = Args) -> +unpack(DbName, #index_query_args{bookmark = nil} = Args) -> fabric_dict:init(dreyfus_util:get_shards(DbName, Args), nil); unpack(DbName, #index_query_args{} = Args) -> unpack(DbName, Args#index_query_args.bookmark); unpack(DbName, Packed) when is_binary(Packed) -> - lists:map(fun({Node, Range, After}) -> - case mem3:get_shard(DbName, Node, Range) of - {ok, Shard} -> - {Shard, After}; - {error, not_found} -> - PlaceHolder = #shard{ - node = Node, - range = Range, - dbname = DbName, - _='_' - }, - {PlaceHolder, After} - end - end, binary_to_term(couch_util:decodeBase64Url(Packed))). - + lists:map( + fun({Node, Range, After}) -> + case mem3:get_shard(DbName, Node, Range) of + {ok, Shard} -> + {Shard, After}; + {error, not_found} -> + PlaceHolder = #shard{ + node = Node, + range = Range, + dbname = DbName, + _ = '_' + }, + {PlaceHolder, After} + end + end, + binary_to_term(couch_util:decodeBase64Url(Packed)) + ). pack(nil) -> null; pack(Workers) -> - Workers1 = [{N,R,A} || {#shard{node=N, range=R}, A} <- Workers, A =/= nil], - Bin = term_to_binary(Workers1, [compressed, {minor_version,1}]), + Workers1 = [{N, R, A} || {#shard{node = N, range = R}, A} <- Workers, A =/= nil], + Bin = term_to_binary(Workers1, [compressed, {minor_version, 1}]), couch_util:encodeBase64Url(Bin). - add_missing_shards(Bookmark, LiveShards) -> {BookmarkShards, _} = lists:unzip(Bookmark), add_missing_shards(Bookmark, BookmarkShards, LiveShards). - add_missing_shards(Bookmark, _, []) -> Bookmark; add_missing_shards(Bookmark, BMShards, [H | T]) -> - Bookmark1 = case lists:keymember(H#shard.range, #shard.range, BMShards) of - true -> Bookmark; - false -> fabric_dict:store(H, nil, Bookmark) - end, + Bookmark1 = + case lists:keymember(H#shard.range, #shard.range, BMShards) of + true -> Bookmark; + false -> fabric_dict:store(H, nil, Bookmark) + end, add_missing_shards(Bookmark1, BMShards, T). diff --git a/src/dreyfus/src/dreyfus_config.erl b/src/dreyfus/src/dreyfus_config.erl index b7555c1d0..df138f35d 100644 --- a/src/dreyfus/src/dreyfus_config.erl +++ b/src/dreyfus/src/dreyfus_config.erl @@ -1,13 +1,14 @@ - -module(dreyfus_config). +-module(dreyfus_config). - -export([data/0, get/1]). +-export([data/0, get/1]). data() -> try config:get("dreyfus_blacklist") - catch error:badarg -> - % lazy workaround to address issue with epi invocation on startup - [] + catch + error:badarg -> + % lazy workaround to address issue with epi invocation on startup + [] end. get(Key) -> diff --git a/src/dreyfus/src/dreyfus_epi.erl b/src/dreyfus/src/dreyfus_epi.erl index cb07f8a34..22c2c90a8 100644 --- a/src/dreyfus/src/dreyfus_epi.erl +++ b/src/dreyfus/src/dreyfus_epi.erl @@ -23,7 +23,6 @@ providers() -> {chttpd_handlers, dreyfus_httpd_handlers} ]. - services() -> []. @@ -32,8 +31,7 @@ data_subscriptions() -> data_providers() -> [ - {{dreyfus, black_list}, {callback_module, dreyfus_config}, - [{interval, ?DATA_INTERVAL}]} + {{dreyfus, black_list}, {callback_module, dreyfus_config}, [{interval, ?DATA_INTERVAL}]} ]. processes() -> @@ -41,6 +39,9 @@ processes() -> notify(_Key, _Old, _New) -> Listeners = application:get_env(dreyfus, config_listeners, []), - lists:foreach(fun(L) -> - L ! dreyfus_config_change_finished - end, Listeners). + lists:foreach( + fun(L) -> + L ! dreyfus_config_change_finished + end, + Listeners + ). diff --git a/src/dreyfus/src/dreyfus_fabric.erl b/src/dreyfus/src/dreyfus_fabric.erl index 0b25a6cc6..5689c1d4e 100644 --- a/src/dreyfus/src/dreyfus_fabric.erl +++ b/src/dreyfus/src/dreyfus_fabric.erl @@ -10,7 +10,6 @@ % License for the specific language governing permissions and limitations under % the License. - %% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- -module(dreyfus_fabric). @@ -23,98 +22,167 @@ get_json_docs(DbName, DocIds) -> fabric:all_docs(DbName, fun callback/2, [], [{keys, DocIds}, {include_docs, true}]). -callback({meta,_}, Acc) -> +callback({meta, _}, Acc) -> {ok, Acc}; callback({error, Reason}, _Acc) -> {error, Reason}; callback({row, Row}, Acc) -> {id, Id} = lists:keyfind(id, 1, Row), - {ok, [{Id, lists:keyfind(doc, 1, Row)}|Acc]}; + {ok, [{Id, lists:keyfind(doc, 1, Row)} | Acc]}; callback(complete, Acc) -> {ok, lists:reverse(Acc)}; callback(timeout, _Acc) -> {error, timeout}. -handle_error_message({rexi_DOWN, _, {_, NodeRef}, _}, _Worker, - Counters, _Replacements, _StartFun, _StartArgs, RingOpts) -> +handle_error_message( + {rexi_DOWN, _, {_, NodeRef}, _}, + _Worker, + Counters, + _Replacements, + _StartFun, + _StartArgs, + RingOpts +) -> case fabric_util:remove_down_workers(Counters, NodeRef, RingOpts) of - {ok, NewCounters} -> - {ok, NewCounters}; - error -> - {error, {nodedown, <<"progress not possible">>}} + {ok, NewCounters} -> + {ok, NewCounters}; + error -> + {error, {nodedown, <<"progress not possible">>}} end; -handle_error_message({rexi_EXIT, {maintenance_mode, _}}, Worker, - Counters, Replacements, StartFun, StartArgs, RingOpts) -> - handle_replacement(Worker, Counters, Replacements, StartFun, StartArgs, - RingOpts); -handle_error_message({rexi_EXIT, Reason}, Worker, - Counters, _Replacements, _StartFun, _StartArgs, RingOpts) -> +handle_error_message( + {rexi_EXIT, {maintenance_mode, _}}, + Worker, + Counters, + Replacements, + StartFun, + StartArgs, + RingOpts +) -> + handle_replacement( + Worker, + Counters, + Replacements, + StartFun, + StartArgs, + RingOpts + ); +handle_error_message( + {rexi_EXIT, Reason}, + Worker, + Counters, + _Replacements, + _StartFun, + _StartArgs, + RingOpts +) -> handle_error(Reason, Worker, Counters, RingOpts); -handle_error_message({error, Reason}, Worker, - Counters, _Replacements, _StartFun, _StartArgs, RingOpts) -> +handle_error_message( + {error, Reason}, + Worker, + Counters, + _Replacements, + _StartFun, + _StartArgs, + RingOpts +) -> handle_error(Reason, Worker, Counters, RingOpts); -handle_error_message({'EXIT', Reason}, Worker, - Counters, _Replacements, _StartFun, _StartArgs, RingOpts) -> +handle_error_message( + {'EXIT', Reason}, + Worker, + Counters, + _Replacements, + _StartFun, + _StartArgs, + RingOpts +) -> handle_error({exit, Reason}, Worker, Counters, RingOpts); -handle_error_message(Reason, Worker, Counters, - _Replacements, _StartFun, _StartArgs, RingOpts) -> +handle_error_message( + Reason, + Worker, + Counters, + _Replacements, + _StartFun, + _StartArgs, + RingOpts +) -> couch_log:error("Unexpected error during request: ~p", [Reason]), handle_error(Reason, Worker, Counters, RingOpts). handle_error(Reason, Worker, Counters0, RingOpts) -> Counters = fabric_dict:erase(Worker, Counters0), case fabric_ring:is_progress_possible(Counters, RingOpts) of - true -> - {ok, Counters}; - false -> - {error, Reason} + true -> + {ok, Counters}; + false -> + {error, Reason} end. -handle_replacement(Worker, OldCntrs0, OldReplacements, StartFun, StartArgs, - RingOpts) -> - OldCounters = lists:filter(fun({#shard{ref=R}, _}) -> - R /= Worker#shard.ref - end, OldCntrs0), +handle_replacement( + Worker, + OldCntrs0, + OldReplacements, + StartFun, + StartArgs, + RingOpts +) -> + OldCounters = lists:filter( + fun({#shard{ref = R}, _}) -> + R /= Worker#shard.ref + end, + OldCntrs0 + ), case lists:keytake(Worker#shard.range, 1, OldReplacements) of {value, {_Range, Replacements}, NewReplacements} -> - NewCounters = lists:foldl(fun(Repl, CounterAcc) -> - NewCounter = start_replacement(StartFun, StartArgs, Repl), - fabric_dict:store(NewCounter, nil, CounterAcc) - end, OldCounters, Replacements), + NewCounters = lists:foldl( + fun(Repl, CounterAcc) -> + NewCounter = start_replacement(StartFun, StartArgs, Repl), + fabric_dict:store(NewCounter, nil, CounterAcc) + end, + OldCounters, + Replacements + ), true = fabric_ring:is_progress_possible(NewCounters, RingOpts), NewRefs = fabric_dict:fetch_keys(NewCounters), {new_refs, NewRefs, NewCounters, NewReplacements}; false -> - handle_error({nodedown, <<"progress not possible">>}, - Worker, OldCounters, RingOpts) + handle_error( + {nodedown, <<"progress not possible">>}, + Worker, + OldCounters, + RingOpts + ) end. start_replacement(StartFun, StartArgs, Shard) -> [DDoc, IndexName, QueryArgs] = StartArgs, - After = case QueryArgs#index_query_args.bookmark of - Bookmark when is_list(Bookmark) -> - lists:foldl(fun({#shard{range=R0}, After0}, Acc) -> - case R0 == Shard#shard.range of - true -> After0; - false -> Acc - end - end, nil, Bookmark); - _ -> - nil - end, - QueryArgs1 = QueryArgs#index_query_args{bookmark=After}, + After = + case QueryArgs#index_query_args.bookmark of + Bookmark when is_list(Bookmark) -> + lists:foldl( + fun({#shard{range = R0}, After0}, Acc) -> + case R0 == Shard#shard.range of + true -> After0; + false -> Acc + end + end, + nil, + Bookmark + ); + _ -> + nil + end, + QueryArgs1 = QueryArgs#index_query_args{bookmark = After}, StartArgs1 = [DDoc, IndexName, QueryArgs1], - Ref = rexi:cast(Shard#shard.node, - {dreyfus_rpc, StartFun, - [Shard#shard.name|StartArgs1]}), + Ref = rexi:cast( + Shard#shard.node, + {dreyfus_rpc, StartFun, [Shard#shard.name | StartArgs1]} + ), Shard#shard{ref = Ref}. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). - node_down_test() -> [S1, S2, S3] = [ mk_shard("n1", [0, 4]), @@ -122,24 +190,23 @@ node_down_test() -> mk_shard("n2", [0, ?RING_END]) ], [W1, W2, W3] = [ - S1#shard{ref = make_ref()}, - S2#shard{ref = make_ref()}, - S3#shard{ref = make_ref()} + S1#shard{ref = make_ref()}, + S2#shard{ref = make_ref()}, + S3#shard{ref = make_ref()} ], Counters1 = fabric_dict:init([W1, W2, W3], nil), N1 = S1#shard.node, Msg1 = {rexi_DOWN, nil, {nil, N1}, nil}, - Res1 = handle_error_message(Msg1, nil, Counters1, nil, nil, nil, []), + Res1 = handle_error_message(Msg1, nil, Counters1, nil, nil, nil, []), ?assertEqual({ok, [{W3, nil}]}, Res1), {ok, Counters2} = Res1, N2 = S3#shard.node, Msg2 = {rexi_DOWN, nil, {nil, N2}, nil}, - Res2 = handle_error_message(Msg2, nil, Counters2, nil, nil, nil, []), + Res2 = handle_error_message(Msg2, nil, Counters2, nil, nil, nil, []), ?assertEqual({error, {nodedown, <<"progress not possible">>}}, Res2). - worker_error_test() -> [S1, S2] = [ mk_shard("n1", [0, ?RING_END]), @@ -154,7 +221,6 @@ worker_error_test() -> {ok, Counters2} = Res1, ?assertEqual({error, boom}, handle_error(boom, W2, Counters2, [])). - node_down_with_partitions_test() -> [S1, S2] = [ mk_shard("n1", [0, 4]), @@ -169,20 +235,20 @@ node_down_with_partitions_test() -> N1 = S1#shard.node, Msg1 = {rexi_DOWN, nil, {nil, N1}, nil}, - Res1 = handle_error_message(Msg1, nil, Counters1, nil, nil, nil, RingOpts), + Res1 = handle_error_message(Msg1, nil, Counters1, nil, nil, nil, RingOpts), ?assertEqual({ok, [{W2, nil}]}, Res1), {ok, Counters2} = Res1, N2 = S2#shard.node, Msg2 = {rexi_DOWN, nil, {nil, N2}, nil}, - Res2 = handle_error_message(Msg2, nil, Counters2, nil, nil, nil, RingOpts), + Res2 = handle_error_message(Msg2, nil, Counters2, nil, nil, nil, RingOpts), ?assertEqual({error, {nodedown, <<"progress not possible">>}}, Res2). - worker_error_with_partitions_test() -> [S1, S2] = [ mk_shard("n1", [0, 4]), - mk_shard("n2", [0, 8])], + mk_shard("n2", [0, 8]) + ], [W1, W2] = [ S1#shard{ref = make_ref()}, S2#shard{ref = make_ref()} @@ -196,7 +262,6 @@ worker_error_with_partitions_test() -> {ok, Counters2} = Res1, ?assertEqual({error, boom}, handle_error(boom, W2, Counters2, RingOpts)). - mk_shard(Name, Range) -> Node = list_to_atom(Name), BName = list_to_binary(Name), diff --git a/src/dreyfus/src/dreyfus_fabric_cleanup.erl b/src/dreyfus/src/dreyfus_fabric_cleanup.erl index 681712748..e2710744d 100644 --- a/src/dreyfus/src/dreyfus_fabric_cleanup.erl +++ b/src/dreyfus/src/dreyfus_fabric_cleanup.erl @@ -10,7 +10,6 @@ % License for the specific language governing permissions and limitations under % the License. - %% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- -module(dreyfus_fabric_cleanup). @@ -23,22 +22,30 @@ go(DbName) -> {ok, DesignDocs} = fabric:design_docs(DbName), - ActiveSigs = lists:usort(lists:flatmap(fun active_sigs/1, - [couch_doc:from_json_obj(DD) || DD <- DesignDocs])), + ActiveSigs = lists:usort( + lists:flatmap( + fun active_sigs/1, + [couch_doc:from_json_obj(DD) || DD <- DesignDocs] + ) + ), cleanup_local_purge_doc(DbName, ActiveSigs), clouseau_rpc:cleanup(DbName, ActiveSigs), ok. -active_sigs(#doc{body={Fields}}=Doc) -> +active_sigs(#doc{body = {Fields}} = Doc) -> try {RawIndexes} = couch_util:get_value(<<"indexes">>, Fields, {[]}), {IndexNames, _} = lists:unzip(RawIndexes), - [begin - {ok, Index} = dreyfus_index:design_doc_to_index(Doc, IndexName), - Index#index.sig - end || IndexName <- IndexNames] - catch error:{badmatch, _Error} -> - [] + [ + begin + {ok, Index} = dreyfus_index:design_doc_to_index(Doc, IndexName), + Index#index.sig + end + || IndexName <- IndexNames + ] + catch + error:{badmatch, _Error} -> + [] end. cleanup_local_purge_doc(DbName, ActiveSigs) -> @@ -49,30 +56,50 @@ cleanup_local_purge_doc(DbName, ActiveSigs) -> DirListStrs = filelib:wildcard(Pattern), DirList = [iolist_to_binary(DL) || DL <- DirListStrs], LocalShards = mem3:local_shards(DbName), - ActiveDirs = lists:foldl(fun(LS, AccOuter) -> - lists:foldl(fun(Sig, AccInner) -> - DirName = filename:join([BaseDir, LS#shard.name, Sig]), - [DirName | AccInner] - end, AccOuter, ActiveSigs) - end, [], LocalShards), + ActiveDirs = lists:foldl( + fun(LS, AccOuter) -> + lists:foldl( + fun(Sig, AccInner) -> + DirName = filename:join([BaseDir, LS#shard.name, Sig]), + [DirName | AccInner] + end, + AccOuter, + ActiveSigs + ) + end, + [], + LocalShards + ), DeadDirs = DirList -- ActiveDirs, - lists:foreach(fun(IdxDir) -> - Sig = dreyfus_util:get_signature_from_idxdir(IdxDir), - case Sig of undefined -> ok; _ -> - DocId = dreyfus_util:get_local_purge_doc_id(Sig), - LocalShards = mem3:local_shards(DbName), - lists:foreach(fun(LS) -> - ShardDbName = LS#shard.name, - {ok, ShardDb} = couch_db:open_int(ShardDbName, []), - case couch_db:open_doc(ShardDb, DocId, []) of - {ok, LocalPurgeDoc} -> - couch_db:update_doc(ShardDb, - LocalPurgeDoc#doc{deleted=true}, [?ADMIN_CTX]); - {not_found, _} -> - ok - end, - couch_db:close(ShardDb) - end, LocalShards) - end - end, DeadDirs). + lists:foreach( + fun(IdxDir) -> + Sig = dreyfus_util:get_signature_from_idxdir(IdxDir), + case Sig of + undefined -> + ok; + _ -> + DocId = dreyfus_util:get_local_purge_doc_id(Sig), + LocalShards = mem3:local_shards(DbName), + lists:foreach( + fun(LS) -> + ShardDbName = LS#shard.name, + {ok, ShardDb} = couch_db:open_int(ShardDbName, []), + case couch_db:open_doc(ShardDb, DocId, []) of + {ok, LocalPurgeDoc} -> + couch_db:update_doc( + ShardDb, + LocalPurgeDoc#doc{deleted = true}, + [?ADMIN_CTX] + ); + {not_found, _} -> + ok + end, + couch_db:close(ShardDb) + end, + LocalShards + ) + end + end, + DeadDirs + ). diff --git a/src/dreyfus/src/dreyfus_fabric_group1.erl b/src/dreyfus/src/dreyfus_fabric_group1.erl index bdae6f040..1edfd653f 100644 --- a/src/dreyfus/src/dreyfus_fabric_group1.erl +++ b/src/dreyfus/src/dreyfus_fabric_group1.erl @@ -10,7 +10,6 @@ % License for the specific language governing permissions and limitations under % the License. - %% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- -module(dreyfus_fabric_group1). @@ -35,14 +34,16 @@ go(DbName, GroupId, IndexName, QueryArgs) when is_binary(GroupId) -> {ok, DDoc} = fabric:open_doc(DbName, <<"_design/", GroupId/binary>>, []), dreyfus_util:maybe_deny_index(DbName, GroupId, IndexName), go(DbName, DDoc, IndexName, QueryArgs); - -go(DbName, DDoc, IndexName, #index_query_args{}=QueryArgs) -> +go(DbName, DDoc, IndexName, #index_query_args{} = QueryArgs) -> DesignName = dreyfus_util:get_design_docid(DDoc), dreyfus_util:maybe_deny_index(DbName, DesignName, IndexName), Shards = dreyfus_util:get_shards(DbName, QueryArgs), RingOpts = dreyfus_util:get_ring_opts(QueryArgs, Shards), - Workers = fabric_util:submit_jobs(Shards, dreyfus_rpc, group1, [DDoc, - IndexName, dreyfus_util:export(QueryArgs)]), + Workers = fabric_util:submit_jobs(Shards, dreyfus_rpc, group1, [ + DDoc, + IndexName, + dreyfus_util:export(QueryArgs) + ]), Replacements = fabric_view:get_shard_replacements(DbName, Workers), Counters = fabric_dict:init(Workers, nil), RexiMon = fabric_util:create_monitors(Workers), @@ -56,8 +57,14 @@ go(DbName, DDoc, IndexName, #index_query_args{}=QueryArgs) -> ring_opts = RingOpts }, try - rexi_utils:recv(Workers, #shard.ref, fun handle_message/3, - State, infinity, 1000 * 60 * 60) + rexi_utils:recv( + Workers, + #shard.ref, + fun handle_message/3, + State, + infinity, + 1000 * 60 * 60 + ) after rexi_monitor:stop(RexiMon), fabric_util:cleanup(Workers) @@ -67,34 +74,43 @@ go(DbName, DDoc, IndexName, OldArgs) -> handle_message({ok, NewTopGroups}, Shard, State0) -> State = upgrade_state(State0), - #state{top_groups=TopGroups, limit=Limit, sort=Sort} = State, + #state{top_groups = TopGroups, limit = Limit, sort = Sort} = State, case fabric_dict:lookup_element(Shard, State#state.counters) of - undefined -> - %% already heard from someone else in this range - {ok, State}; - nil -> - C1 = fabric_dict:store(Shard, ok, State#state.counters), - C2 = fabric_view:remove_overlapping_shards(Shard, C1), - MergedTopGroups = merge_top_groups(TopGroups, make_sortable(Shard, NewTopGroups), Limit, Sort), - State1 = State#state{ - counters=C2, - top_groups=MergedTopGroups - }, - case fabric_dict:any(nil, C2) of - true -> - {ok, State1}; - false -> - {stop, remove_sortable(MergedTopGroups)} - end + undefined -> + %% already heard from someone else in this range + {ok, State}; + nil -> + C1 = fabric_dict:store(Shard, ok, State#state.counters), + C2 = fabric_view:remove_overlapping_shards(Shard, C1), + MergedTopGroups = merge_top_groups( + TopGroups, make_sortable(Shard, NewTopGroups), Limit, Sort + ), + State1 = State#state{ + counters = C2, + top_groups = MergedTopGroups + }, + case fabric_dict:any(nil, C2) of + true -> + {ok, State1}; + false -> + {stop, remove_sortable(MergedTopGroups)} + end end; - handle_message(Error, Worker, State0) -> State = upgrade_state(State0), - case dreyfus_fabric:handle_error_message(Error, Worker, - State#state.counters, State#state.replacements, - group1, State#state.start_args, State#state.ring_opts) of + case + dreyfus_fabric:handle_error_message( + Error, + Worker, + State#state.counters, + State#state.replacements, + group1, + State#state.start_args, + State#state.ring_opts + ) + of {ok, Counters} -> - {ok, State#state{counters=Counters}}; + {ok, State#state{counters = Counters}}; {new_refs, NewRefs, NewCounters, NewReplacements} -> NewState = State#state{ counters = NewCounters, @@ -107,8 +123,11 @@ handle_message(Error, Worker, State0) -> merge_top_groups(TopGroupsA, TopGroupsB, Limit, Sort) -> MergedGroups0 = TopGroupsA ++ TopGroupsB, - GNs = lists:usort([N || #sortable{item={N,_}} <- MergedGroups0]), - MergedGroups = [merge_top_group(Sort, [S || #sortable{item={N,_}}=S <- MergedGroups0, N =:= GN]) || GN <- GNs], + GNs = lists:usort([N || #sortable{item = {N, _}} <- MergedGroups0]), + MergedGroups = [ + merge_top_group(Sort, [S || #sortable{item = {N, _}} = S <- MergedGroups0, N =:= GN]) + || GN <- GNs + ], lists:sublist(dreyfus_util:sort(Sort, MergedGroups), Limit). merge_top_group(_Sort, [Group]) -> @@ -117,13 +136,18 @@ merge_top_group(Sort, [_, _] = Groups) -> hd(dreyfus_util:sort(Sort, Groups)). make_sortable(Shard, TopGroups) -> - [#sortable{item=G, order=Order, shard=Shard} || {_Name, Order}=G <- TopGroups]. + [#sortable{item = G, order = Order, shard = Shard} || {_Name, Order} = G <- TopGroups]. remove_sortable(Sortables) -> - [Item || #sortable{item=Item} <- Sortables]. + [Item || #sortable{item = Item} <- Sortables]. upgrade_state({state, Limit, Sort, TopGroups, Counters}) -> - #state{limit=Limit, sort=Sort, top_groups=TopGroups, counters=Counters, - replacements=[]}; -upgrade_state(#state{}=State) -> + #state{ + limit = Limit, + sort = Sort, + top_groups = TopGroups, + counters = Counters, + replacements = [] + }; +upgrade_state(#state{} = State) -> State. diff --git a/src/dreyfus/src/dreyfus_fabric_group2.erl b/src/dreyfus/src/dreyfus_fabric_group2.erl index 8d864dd0c..e962c7e4a 100644 --- a/src/dreyfus/src/dreyfus_fabric_group2.erl +++ b/src/dreyfus/src/dreyfus_fabric_group2.erl @@ -10,7 +10,6 @@ % License for the specific language governing permissions and limitations under % the License. - %% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- -module(dreyfus_fabric_group2). @@ -37,14 +36,17 @@ go(DbName, GroupId, IndexName, QueryArgs) when is_binary(GroupId) -> {ok, DDoc} = fabric:open_doc(DbName, <<"_design/", GroupId/binary>>, []), dreyfus_util:maybe_deny_index(DbName, GroupId, IndexName), go(DbName, DDoc, IndexName, QueryArgs); - -go(DbName, DDoc, IndexName, #index_query_args{}=QueryArgs) -> +go(DbName, DDoc, IndexName, #index_query_args{} = QueryArgs) -> DesignName = dreyfus_util:get_design_docid(DDoc), dreyfus_util:maybe_deny_index(DbName, DesignName, IndexName), Shards = dreyfus_util:get_shards(DbName, QueryArgs), RingOpts = dreyfus_util:get_ring_opts(QueryArgs, Shards), - Workers = fabric_util:submit_jobs(Shards, dreyfus_rpc, group2, - [DDoc, IndexName, dreyfus_util:export(QueryArgs)]), + Workers = fabric_util:submit_jobs( + Shards, + dreyfus_rpc, + group2, + [DDoc, IndexName, dreyfus_util:export(QueryArgs)] + ), Replacements = fabric_view:get_shard_replacements(DbName, Workers), Counters = fabric_dict:init(Workers, nil), RexiMon = fabric_util:create_monitors(Workers), @@ -60,8 +62,14 @@ go(DbName, DDoc, IndexName, #index_query_args{}=QueryArgs) -> ring_opts = RingOpts }, try - rexi_utils:recv(Workers, #shard.ref, fun handle_message/3, - State, infinity, 1000 * 60 * 60) + rexi_utils:recv( + Workers, + #shard.ref, + fun handle_message/3, + State, + infinity, + 1000 * 60 * 60 + ) after rexi_monitor:stop(RexiMon), fabric_util:cleanup(Workers) @@ -69,45 +77,59 @@ go(DbName, DDoc, IndexName, #index_query_args{}=QueryArgs) -> go(DbName, DDoc, IndexName, OldArgs) -> go(DbName, DDoc, IndexName, dreyfus_util:upgrade(OldArgs)). - -handle_message({ok, NewTotalHits, NewTotalGroupedHits, NewTopGroups}, - Shard, State0) -> +handle_message( + {ok, NewTotalHits, NewTotalGroupedHits, NewTopGroups}, + Shard, + State0 +) -> State = upgrade_state(State0), - #state{total_hits=TotalHits, total_grouped_hits=TotalGroupedHits, - top_groups=TopGroups, limit=Limit, sort=Sort} = State, + #state{ + total_hits = TotalHits, + total_grouped_hits = TotalGroupedHits, + top_groups = TopGroups, + limit = Limit, + sort = Sort + } = State, case fabric_dict:lookup_element(Shard, State#state.counters) of - undefined -> - %% already heard from someone else in this range - {ok, State}; - nil -> - C1 = fabric_dict:store(Shard, ok, State#state.counters), - C2 = fabric_view:remove_overlapping_shards(Shard, C1), - MergedTotalHits = NewTotalHits + TotalHits, - MergedTotalGroupedHits = NewTotalGroupedHits + TotalGroupedHits, - Sortable = make_sortable(Shard, NewTopGroups), - MergedTopGroups = merge_top_groups(TopGroups, Sortable, Limit, Sort), - State1 = State#state{ - counters=C2, - total_hits=MergedTotalHits, - total_grouped_hits=MergedTotalGroupedHits, - top_groups=MergedTopGroups - }, - case fabric_dict:any(nil, C2) of - true -> - {ok, State1}; - false -> - {stop, {MergedTotalHits, MergedTotalGroupedHits, - remove_sortable(MergedTopGroups)}} - end + undefined -> + %% already heard from someone else in this range + {ok, State}; + nil -> + C1 = fabric_dict:store(Shard, ok, State#state.counters), + C2 = fabric_view:remove_overlapping_shards(Shard, C1), + MergedTotalHits = NewTotalHits + TotalHits, + MergedTotalGroupedHits = NewTotalGroupedHits + TotalGroupedHits, + Sortable = make_sortable(Shard, NewTopGroups), + MergedTopGroups = merge_top_groups(TopGroups, Sortable, Limit, Sort), + State1 = State#state{ + counters = C2, + total_hits = MergedTotalHits, + total_grouped_hits = MergedTotalGroupedHits, + top_groups = MergedTopGroups + }, + case fabric_dict:any(nil, C2) of + true -> + {ok, State1}; + false -> + {stop, + {MergedTotalHits, MergedTotalGroupedHits, remove_sortable(MergedTopGroups)}} + end end; - handle_message(Error, Worker, State0) -> State = upgrade_state(State0), - case dreyfus_fabric:handle_error_message(Error, Worker, - State#state.counters, State#state.replacements, - group2, State#state.start_args, State#state.ring_opts) of + case + dreyfus_fabric:handle_error_message( + Error, + Worker, + State#state.counters, + State#state.replacements, + group2, + State#state.start_args, + State#state.ring_opts + ) + of {ok, Counters} -> - {ok, State#state{counters=Counters}}; + {ok, State#state{counters = Counters}}; {new_refs, NewRefs, NewCounters, NewReplacements} -> NewState = State#state{ counters = NewCounters, @@ -121,15 +143,16 @@ handle_message(Error, Worker, State0) -> merge_top_groups([], TopGroups, _Limit, _Sort) -> TopGroups; merge_top_groups(TopGroupsA, TopGroupsB, Limit, Sort) -> - lists:zipwith(fun(A,B) -> merge_top_group(A, B, Limit, Sort) end, - TopGroupsA, - TopGroupsB). + lists:zipwith( + fun(A, B) -> merge_top_group(A, B, Limit, Sort) end, + TopGroupsA, + TopGroupsB + ). merge_top_group({Name, TotalA, HitsA}, {Name, TotalB, HitsB}, Limit, Sort) -> MergedHits = lists:sublist(dreyfus_util:sort(Sort, HitsA ++ HitsB), Limit), {Name, TotalA + TotalB, MergedHits}. - make_sortable(Shard, TopGroups) -> [make_sortable_group(Shard, TopGroup) || TopGroup <- TopGroups]. @@ -137,7 +160,7 @@ make_sortable_group(Shard, {Name, TotalHits, Hits}) -> {Name, TotalHits, [make_sortable_hit(Shard, Hit) || Hit <- Hits]}. make_sortable_hit(Shard, Hit) -> - #sortable{item=Hit, order=Hit#hit.order, shard=Shard}. + #sortable{item = Hit, order = Hit#hit.order, shard = Shard}. remove_sortable(SortableGroups) -> [remove_sortable_group(G) || G <- SortableGroups]. @@ -148,11 +171,15 @@ remove_sortable_group({Name, TotalHits, SortableHits}) -> remove_sortable_hit(SortableHit) -> SortableHit#sortable.item. -upgrade_state({state, Limit, Sort, TotalHits, TotalGroupedHits, - TopGroups, Counters}) -> - #state{limit = Limit, sort = Sort, total_hits = TotalHits, - total_grouped_hits = TotalGroupedHits, - top_groups = TopGroups, counters = Counters, - replacements = []}; +upgrade_state({state, Limit, Sort, TotalHits, TotalGroupedHits, TopGroups, Counters}) -> + #state{ + limit = Limit, + sort = Sort, + total_hits = TotalHits, + total_grouped_hits = TotalGroupedHits, + top_groups = TopGroups, + counters = Counters, + replacements = [] + }; upgrade_state(#state{} = State) -> State. diff --git a/src/dreyfus/src/dreyfus_fabric_info.erl b/src/dreyfus/src/dreyfus_fabric_info.erl index e217bc0ef..5ca589c1d 100644 --- a/src/dreyfus/src/dreyfus_fabric_info.erl +++ b/src/dreyfus/src/dreyfus_fabric_info.erl @@ -10,7 +10,6 @@ % License for the specific language governing permissions and limitations under % the License. - %% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- -module(dreyfus_fabric_info). @@ -25,7 +24,6 @@ go(DbName, DDocId, IndexName, InfoLevel) when is_binary(DDocId) -> {ok, DDoc} = fabric:open_doc(DbName, <<"_design/", DDocId/binary>>, []), dreyfus_util:maybe_deny_index(DbName, DDocId, IndexName), go(DbName, DDoc, IndexName, InfoLevel); - go(DbName, DDoc, IndexName, InfoLevel) -> DesignName = dreyfus_util:get_design_docid(DDoc), dreyfus_util:maybe_deny_index(DbName, DesignName, IndexName), @@ -39,70 +37,74 @@ go(DbName, DDoc, IndexName, InfoLevel) -> rexi_monitor:stop(RexiMon) end. -handle_message({rexi_DOWN, _, {_,NodeRef},_}, _Worker, {Counters, Acc}) -> +handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _Worker, {Counters, Acc}) -> case fabric_util:remove_down_workers(Counters, NodeRef) of - {ok, NewCounters} -> - {ok, {NewCounters, Acc}}; - error -> - {error, {nodedown, <<"progress not possible">>}} + {ok, NewCounters} -> + {ok, {NewCounters, Acc}}; + error -> + {error, {nodedown, <<"progress not possible">>}} end; - handle_message({rexi_EXIT, Reason}, Worker, {Counters, Acc}) -> NewCounters = fabric_dict:erase(Worker, Counters), case fabric_ring:is_progress_possible(NewCounters) of - true -> - {ok, {NewCounters, Acc}}; - false -> - {error, Reason} + true -> + {ok, {NewCounters, Acc}}; + false -> + {error, Reason} end; - handle_message({ok, Info}, Worker, {Counters, Acc}) -> case fabric_dict:lookup_element(Worker, Counters) of - undefined -> - % already heard from someone else in this range - {ok, {Counters, Acc}}; - nil -> - C1 = fabric_dict:store(Worker, ok, Counters), - C2 = fabric_view:remove_overlapping_shards(Worker, C1), - case fabric_dict:any(nil, C2) of - true -> - {ok, {C2, [Info|Acc]}}; - false -> - {stop, merge_results(lists:flatten([Info|Acc]))} - end + undefined -> + % already heard from someone else in this range + {ok, {Counters, Acc}}; + nil -> + C1 = fabric_dict:store(Worker, ok, Counters), + C2 = fabric_view:remove_overlapping_shards(Worker, C1), + case fabric_dict:any(nil, C2) of + true -> + {ok, {C2, [Info | Acc]}}; + false -> + {stop, merge_results(lists:flatten([Info | Acc]))} + end end; - handle_message({error, Reason}, Worker, {Counters, Acc}) -> NewCounters = fabric_dict:erase(Worker, Counters), case fabric_ring:is_progress_possible(NewCounters) of - true -> - {ok, {NewCounters, Acc}}; - false -> - {error, Reason} + true -> + {ok, {NewCounters, Acc}}; + false -> + {error, Reason} end; handle_message({'EXIT', _}, Worker, {Counters, Acc}) -> NewCounters = fabric_dict:erase(Worker, Counters), case fabric_ring:is_progress_possible(NewCounters) of - true -> - {ok, {NewCounters, Acc}}; - false -> - {error, {nodedown, <<"progress not possible">>}} + true -> + {ok, {NewCounters, Acc}}; + false -> + {error, {nodedown, <<"progress not possible">>}} end. merge_results(Info) -> - Dict = lists:foldl(fun({K,V},D0) -> orddict:append(K,V,D0) end, - orddict:new(), Info), - orddict:fold(fun - (disk_size, X, Acc) -> - [{disk_size, lists:sum(X)} | Acc]; - (doc_count, X, Acc) -> - [{doc_count, lists:sum(X)} | Acc]; - (doc_del_count, X, Acc) -> - [{doc_del_count, lists:sum(X)} | Acc]; - (committed_seq, X, Acc) -> - [{committed_seq, lists:sum(X)} | Acc]; - (pending_seq, X, Acc) -> - [{pending_seq, lists:sum(X)} | Acc]; - (_, _, Acc) -> - Acc - end, [], Dict). + Dict = lists:foldl( + fun({K, V}, D0) -> orddict:append(K, V, D0) end, + orddict:new(), + Info + ), + orddict:fold( + fun + (disk_size, X, Acc) -> + [{disk_size, lists:sum(X)} | Acc]; + (doc_count, X, Acc) -> + [{doc_count, lists:sum(X)} | Acc]; + (doc_del_count, X, Acc) -> + [{doc_del_count, lists:sum(X)} | Acc]; + (committed_seq, X, Acc) -> + [{committed_seq, lists:sum(X)} | Acc]; + (pending_seq, X, Acc) -> + [{pending_seq, lists:sum(X)} | Acc]; + (_, _, Acc) -> + Acc + end, + [], + Dict + ). diff --git a/src/dreyfus/src/dreyfus_fabric_search.erl b/src/dreyfus/src/dreyfus_fabric_search.erl index 8edaa385a..7e78e5fc3 100644 --- a/src/dreyfus/src/dreyfus_fabric_search.erl +++ b/src/dreyfus/src/dreyfus_fabric_search.erl @@ -10,7 +10,6 @@ % License for the specific language governing permissions and limitations under % the License. - %% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- -module(dreyfus_fabric_search). @@ -32,48 +31,61 @@ }). go(DbName, GroupId, IndexName, QueryArgs) when is_binary(GroupId) -> - {ok, DDoc} = fabric:open_doc(DbName, <<"_design/", GroupId/binary>>, - [ejson_body]), + {ok, DDoc} = fabric:open_doc( + DbName, + <<"_design/", GroupId/binary>>, + [ejson_body] + ), dreyfus_util:maybe_deny_index(DbName, GroupId, IndexName), go(DbName, DDoc, IndexName, QueryArgs); - -go(DbName, DDoc, IndexName, #index_query_args{bookmark=nil}=QueryArgs) -> +go(DbName, DDoc, IndexName, #index_query_args{bookmark = nil} = QueryArgs) -> DesignName = dreyfus_util:get_design_docid(DDoc), dreyfus_util:maybe_deny_index(DbName, DesignName, IndexName), Shards = dreyfus_util:get_shards(DbName, QueryArgs), RingOpts = dreyfus_util:get_ring_opts(QueryArgs, Shards), - Workers = fabric_util:submit_jobs(Shards, dreyfus_rpc, search, - [DDoc, IndexName, dreyfus_util:export(QueryArgs)]), + Workers = fabric_util:submit_jobs( + Shards, + dreyfus_rpc, + search, + [DDoc, IndexName, dreyfus_util:export(QueryArgs)] + ), Counters = fabric_dict:init(Workers, nil), go(DbName, DDoc, IndexName, QueryArgs, Counters, Counters, RingOpts); - -go(DbName, DDoc, IndexName, #index_query_args{}=QueryArgs) -> - Bookmark0 = try dreyfus_bookmark:unpack(DbName, QueryArgs) - catch - _:_ -> - throw({bad_request, "Invalid bookmark parameter supplied"}) - end, +go(DbName, DDoc, IndexName, #index_query_args{} = QueryArgs) -> + Bookmark0 = + try + dreyfus_bookmark:unpack(DbName, QueryArgs) + catch + _:_ -> + throw({bad_request, "Invalid bookmark parameter supplied"}) + end, Shards = dreyfus_util:get_shards(DbName, QueryArgs), LiveNodes = [node() | nodes()], - LiveShards = [S || #shard{node=Node} = S <- Shards, lists:member(Node, LiveNodes)], + LiveShards = [S || #shard{node = Node} = S <- Shards, lists:member(Node, LiveNodes)], Bookmark1 = dreyfus_bookmark:add_missing_shards(Bookmark0, LiveShards), - Counters0 = lists:flatmap(fun({#shard{name=Name, node=N} = Shard, After}) -> - QueryArgs1 = dreyfus_util:export(QueryArgs#index_query_args{ - bookmark = After - }), - case lists:member(Shard, LiveShards) of - true -> - Ref = rexi:cast(N, {dreyfus_rpc, search, - [Name, DDoc, IndexName, QueryArgs1]}), - [Shard#shard{ref = Ref}]; - false -> - lists:map(fun(#shard{name=Name2, node=N2} = NewShard) -> - Ref = rexi:cast(N2, {dreyfus_rpc, search, - [Name2, DDoc, IndexName, QueryArgs1]}), - NewShard#shard{ref = Ref} - end, find_replacement_shards(Shard, LiveShards)) - end - end, Bookmark1), + Counters0 = lists:flatmap( + fun({#shard{name = Name, node = N} = Shard, After}) -> + QueryArgs1 = dreyfus_util:export(QueryArgs#index_query_args{ + bookmark = After + }), + case lists:member(Shard, LiveShards) of + true -> + Ref = rexi:cast(N, {dreyfus_rpc, search, [Name, DDoc, IndexName, QueryArgs1]}), + [Shard#shard{ref = Ref}]; + false -> + lists:map( + fun(#shard{name = Name2, node = N2} = NewShard) -> + Ref = rexi:cast( + N2, {dreyfus_rpc, search, [Name2, DDoc, IndexName, QueryArgs1]} + ), + NewShard#shard{ref = Ref} + end, + find_replacement_shards(Shard, LiveShards) + ) + end + end, + Bookmark1 + ), Counters = fabric_dict:init(Counters0, nil), WorkerShards = fabric_dict:fetch_keys(Counters), RingOpts = dreyfus_util:get_ring_opts(QueryArgs, WorkerShards), @@ -95,73 +107,92 @@ go(DbName, DDoc, IndexName, QueryArgs, Counters, Bookmark, RingOpts) -> State = #state{ limit = Limit, sort = Sort, - top_docs = #top_docs{total_hits=0,hits=[]}, + top_docs = #top_docs{total_hits = 0, hits = []}, counters = Counters, start_args = [DDoc, IndexName, QueryArgs], replacements = Replacements, ring_opts = RingOpts - }, + }, RexiMon = fabric_util:create_monitors(Workers), - try rexi_utils:recv(Workers, #shard.ref, fun handle_message/3, - State, infinity, 1000 * 60 * 60) of - {ok, Result} -> - #state{top_docs=TopDocs} = Result, - #top_docs{total_hits=TotalHits, hits=Hits, - counts=Counts, ranges=Ranges} = TopDocs, - case RawBookmark of - true -> - {ok, Bookmark, TotalHits, Hits, Counts, Ranges}; - false -> - Bookmark1 = dreyfus_bookmark:update(Sort, Bookmark, Hits), - Hits1 = remove_sortable(Hits), - {ok, Bookmark1, TotalHits, Hits1, Counts, Ranges} - end; - {error, Reason} -> - {error, Reason} + try + rexi_utils:recv( + Workers, + #shard.ref, + fun handle_message/3, + State, + infinity, + 1000 * 60 * 60 + ) + of + {ok, Result} -> + #state{top_docs = TopDocs} = Result, + #top_docs{ + total_hits = TotalHits, + hits = Hits, + counts = Counts, + ranges = Ranges + } = TopDocs, + case RawBookmark of + true -> + {ok, Bookmark, TotalHits, Hits, Counts, Ranges}; + false -> + Bookmark1 = dreyfus_bookmark:update(Sort, Bookmark, Hits), + Hits1 = remove_sortable(Hits), + {ok, Bookmark1, TotalHits, Hits1, Counts, Ranges} + end; + {error, Reason} -> + {error, Reason} after rexi_monitor:stop(RexiMon), fabric_util:cleanup(Workers) end. -handle_message({ok, #top_docs{}=NewTopDocs}, Shard, State0) -> +handle_message({ok, #top_docs{} = NewTopDocs}, Shard, State0) -> State = upgrade_state(State0), - #state{top_docs=TopDocs, limit=Limit, sort=Sort} = State, + #state{top_docs = TopDocs, limit = Limit, sort = Sort} = State, case fabric_dict:lookup_element(Shard, State#state.counters) of - undefined -> - %% already heard from someone else in this range - {ok, State}; - nil -> - C1 = fabric_dict:store(Shard, ok, State#state.counters), - C2 = fabric_view:remove_overlapping_shards(Shard, C1), - Sortable = make_sortable(Shard, NewTopDocs), - MergedTopDocs = merge_top_docs(TopDocs, Sortable, Limit, Sort), - State1 = State#state{ - counters=C2, - top_docs=MergedTopDocs - }, - case fabric_dict:any(nil, C2) of - true -> - {ok, State1}; - false -> - {stop, State1} - end + undefined -> + %% already heard from someone else in this range + {ok, State}; + nil -> + C1 = fabric_dict:store(Shard, ok, State#state.counters), + C2 = fabric_view:remove_overlapping_shards(Shard, C1), + Sortable = make_sortable(Shard, NewTopDocs), + MergedTopDocs = merge_top_docs(TopDocs, Sortable, Limit, Sort), + State1 = State#state{ + counters = C2, + top_docs = MergedTopDocs + }, + case fabric_dict:any(nil, C2) of + true -> + {ok, State1}; + false -> + {stop, State1} + end end; - % upgrade clause handle_message({ok, {top_docs, UpdateSeq, TotalHits, Hits}}, Shard, State) -> TopDocs = #top_docs{ - update_seq = UpdateSeq, - total_hits = TotalHits, - hits = Hits}, + update_seq = UpdateSeq, + total_hits = TotalHits, + hits = Hits + }, handle_message({ok, TopDocs}, Shard, State); - handle_message(Error, Worker, State0) -> State = upgrade_state(State0), - case dreyfus_fabric:handle_error_message(Error, Worker, - State#state.counters, State#state.replacements, - search, State#state.start_args, State#state.ring_opts) of + case + dreyfus_fabric:handle_error_message( + Error, + Worker, + State#state.counters, + State#state.replacements, + search, + State#state.start_args, + State#state.ring_opts + ) + of {ok, Counters} -> - {ok, State#state{counters=Counters}}; + {ok, State#state{counters = Counters}}; {new_refs, NewRefs, NewCounters, NewReplacements} -> NewState = State#state{ counters = NewCounters, @@ -172,36 +203,45 @@ handle_message(Error, Worker, State0) -> Else end. -find_replacement_shards(#shard{range=Range}, AllShards) -> +find_replacement_shards(#shard{range = Range}, AllShards) -> [Shard || Shard <- AllShards, Shard#shard.range =:= Range]. -make_sortable(Shard, #top_docs{}=TopDocs) -> +make_sortable(Shard, #top_docs{} = TopDocs) -> Hits = make_sortable(Shard, TopDocs#top_docs.hits), - TopDocs#top_docs{hits=Hits}; + TopDocs#top_docs{hits = Hits}; make_sortable(Shard, List) when is_list(List) -> make_sortable(Shard, List, []). make_sortable(_, [], Acc) -> lists:reverse(Acc); -make_sortable(Shard, [#hit{}=Hit|Rest], Acc) -> - make_sortable(Shard, Rest, [#sortable{item=Hit, order=Hit#hit.order, shard=Shard} | Acc]). +make_sortable(Shard, [#hit{} = Hit | Rest], Acc) -> + make_sortable(Shard, Rest, [#sortable{item = Hit, order = Hit#hit.order, shard = Shard} | Acc]). remove_sortable(List) -> remove_sortable(List, []). remove_sortable([], Acc) -> lists:reverse(Acc); -remove_sortable([#sortable{item=Item} | Rest], Acc) -> +remove_sortable([#sortable{item = Item} | Rest], Acc) -> remove_sortable(Rest, [Item | Acc]). -merge_top_docs(#top_docs{}=TopDocsA, #top_docs{}=TopDocsB, Limit, Sort) -> +merge_top_docs(#top_docs{} = TopDocsA, #top_docs{} = TopDocsB, Limit, Sort) -> MergedTotal = sum_element(#top_docs.total_hits, TopDocsA, TopDocsB), - MergedHits = lists:sublist(dreyfus_util:sort(Sort, - TopDocsA#top_docs.hits ++ TopDocsB#top_docs.hits), Limit), + MergedHits = lists:sublist( + dreyfus_util:sort( + Sort, + TopDocsA#top_docs.hits ++ TopDocsB#top_docs.hits + ), + Limit + ), MergedCounts = merge_facets(TopDocsA#top_docs.counts, TopDocsB#top_docs.counts), MergedRanges = merge_facets(TopDocsA#top_docs.ranges, TopDocsB#top_docs.ranges), - #top_docs{total_hits=MergedTotal, hits=MergedHits, - counts=MergedCounts, ranges=MergedRanges}. + #top_docs{ + total_hits = MergedTotal, + hits = MergedHits, + counts = MergedCounts, + ranges = MergedRanges + }. merge_facets(undefined, undefined) -> undefined; @@ -218,26 +258,35 @@ merge_facets_int(FacetsA, []) -> FacetsA; merge_facets_int([], FacetsB) -> FacetsB; -merge_facets_int([{KA, _, _}=A | RA], [{KB, _, _} | _]=FB) when KA < KB -> +merge_facets_int([{KA, _, _} = A | RA], [{KB, _, _} | _] = FB) when KA < KB -> [A | merge_facets_int(RA, FB)]; merge_facets_int([{KA, VA, CA} | RA], [{KB, VB, CB} | RB]) when KA =:= KB -> - [{KA, VA+VB, merge_facets_int(CA, CB)} | merge_facets_int(RA, RB)]; -merge_facets_int([{KA, _, _} | _]=FA, [{KB, _, _}=B | RB]) when KA > KB -> + [{KA, VA + VB, merge_facets_int(CA, CB)} | merge_facets_int(RA, RB)]; +merge_facets_int([{KA, _, _} | _] = FA, [{KB, _, _} = B | RB]) when KA > KB -> [B | merge_facets_int(FA, RB)]. sort_facets([]) -> []; sort_facets(Facets) -> - lists:sort(lists:map(fun({K, V, C}) -> {K, V, sort_facets(C)} end, - Facets)). + lists:sort( + lists:map( + fun({K, V, C}) -> {K, V, sort_facets(C)} end, + Facets + ) + ). sum_element(N, T1, T2) -> element(N, T1) + element(N, T2). upgrade_state({state, Limit, Sort, TopDocs, Counters}) -> - #state{limit=Limit, sort=Sort, top_docs=TopDocs, counters=Counters, - replacements=[]}; -upgrade_state(#state{}=State) -> + #state{ + limit = Limit, + sort = Sort, + top_docs = TopDocs, + counters = Counters, + replacements = [] + }; +upgrade_state(#state{} = State) -> State. -ifdef(TEST). @@ -248,23 +297,38 @@ merge_facets_test() -> ?assertEqual([{foo, 1.0, []}], merge_facets([{foo, 1.0, []}], [])), % one level, one key - ?assertEqual([{foo, 3.0, []}], - merge_facets([{foo, 1.0, []}], - [{foo, 2.0, []}])), + ?assertEqual( + [{foo, 3.0, []}], + merge_facets( + [{foo, 1.0, []}], + [{foo, 2.0, []}] + ) + ), % one level, two keys - ?assertEqual([{bar, 6.0, []}, {foo, 9.0, []}], - merge_facets([{foo, 1.0, []}, {bar, 2.0, []}], - [{bar, 4.0, []}, {foo, 8.0, []}])), + ?assertEqual( + [{bar, 6.0, []}, {foo, 9.0, []}], + merge_facets( + [{foo, 1.0, []}, {bar, 2.0, []}], + [{bar, 4.0, []}, {foo, 8.0, []}] + ) + ), % multi level, multi keys - ?assertEqual([{foo, 2.0, [{bar, 2.0, []}]}], - merge_facets([{foo, 1.0, [{bar, 1.0, []}]}], - [{foo, 1.0, [{bar, 1.0, []}]}])), - - ?assertEqual([{foo, 5.0, [{bar, 7.0, [{bar, 1.0, []}, {baz, 3.0, []}, {foo, 6.5, []}]}]}], - merge_facets([{foo, 1.0, [{bar, 2.0, [{baz, 3.0, []}, {foo, 0.5, []}]}]}], - [{foo, 4.0, [{bar, 5.0, [{foo, 6.0, []}, {bar, 1.0, []}]}]}])). - + ?assertEqual( + [{foo, 2.0, [{bar, 2.0, []}]}], + merge_facets( + [{foo, 1.0, [{bar, 1.0, []}]}], + [{foo, 1.0, [{bar, 1.0, []}]}] + ) + ), + + ?assertEqual( + [{foo, 5.0, [{bar, 7.0, [{bar, 1.0, []}, {baz, 3.0, []}, {foo, 6.5, []}]}]}], + merge_facets( + [{foo, 1.0, [{bar, 2.0, [{baz, 3.0, []}, {foo, 0.5, []}]}]}], + [{foo, 4.0, [{bar, 5.0, [{foo, 6.0, []}, {bar, 1.0, []}]}]}] + ) + ). -endif. diff --git a/src/dreyfus/src/dreyfus_httpd.erl b/src/dreyfus/src/dreyfus_httpd.erl index 007dace8f..39d205b95 100644 --- a/src/dreyfus/src/dreyfus_httpd.erl +++ b/src/dreyfus/src/dreyfus_httpd.erl @@ -10,139 +10,188 @@ % License for the specific language governing permissions and limitations under % the License. - %% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- -module(dreyfus_httpd). --export([handle_search_req/3, handle_info_req/3, handle_disk_size_req/3, - handle_cleanup_req/2, handle_analyze_req/1]). +-export([ + handle_search_req/3, + handle_info_req/3, + handle_disk_size_req/3, + handle_cleanup_req/2, + handle_analyze_req/1 +]). -include("dreyfus.hrl"). -include_lib("couch/include/couch_db.hrl"). --import(chttpd, [send_method_not_allowed/2, send_json/2, send_json/3, - send_error/2]). +-import(chttpd, [ + send_method_not_allowed/2, + send_json/2, send_json/3, + send_error/2 +]). handle_search_req(Req, Db, DDoc) -> handle_search_req(Req, Db, DDoc, 0, 500). -handle_search_req(#httpd{method=Method, path_parts=[_, _, _, _, IndexName]}=Req - ,Db, DDoc, RetryCount, RetryPause) - when Method == 'GET'; Method == 'POST' -> +handle_search_req( + #httpd{method = Method, path_parts = [_, _, _, _, IndexName]} = Req, + Db, + DDoc, + RetryCount, + RetryPause +) when + Method == 'GET'; Method == 'POST' +-> DbName = couch_db:name(Db), Start = os:timestamp(), - QueryArgs = #index_query_args{ - include_docs = IncludeDocs, - grouping = Grouping - } = parse_index_params(Req, Db), + QueryArgs = + #index_query_args{ + include_docs = IncludeDocs, + grouping = Grouping + } = parse_index_params(Req, Db), validate_search_restrictions(Db, DDoc, QueryArgs), - Response = case Grouping#grouping.by of - nil -> - case dreyfus_fabric_search:go(DbName, DDoc, IndexName, QueryArgs) of - {ok, Bookmark0, TotalHits, Hits0} -> % legacy clause - Hits = hits_to_json(DbName, IncludeDocs, Hits0), - Bookmark = dreyfus_bookmark:pack(Bookmark0), - send_json(Req, 200, {[ - {total_rows, TotalHits}, - {bookmark, Bookmark}, - {rows, Hits} - ]}); - {ok, Bookmark0, TotalHits, Hits0, Counts0, Ranges0} -> - Hits = hits_to_json(DbName, IncludeDocs, Hits0), - Bookmark = dreyfus_bookmark:pack(Bookmark0), - Counts = case Counts0 of - undefined -> - []; - _ -> - [{counts, facets_to_json(Counts0)}] - end, - Ranges = case Ranges0 of - undefined -> - []; - _ -> - [{ranges, facets_to_json(Ranges0)}] - end, - send_json(Req, 200, {[ - {total_rows, TotalHits}, - {bookmark, Bookmark}, - {rows, Hits} - ] ++ Counts ++ Ranges - }); - {error, Reason} -> - handle_error(Req, Db, DDoc, RetryCount, RetryPause, Reason) - end; - _ -> - % ensure limit in group query >0 - UseNewApi = Grouping#grouping.new_api, - case dreyfus_fabric_group1:go(DbName, DDoc, IndexName, QueryArgs) of - {ok, []} -> - send_grouped_response(Req, {0, 0, []}, UseNewApi); - {ok, TopGroups} -> - QueryArgs1 = QueryArgs#index_query_args{grouping=Grouping#grouping{groups=TopGroups}}, - case dreyfus_fabric_group2:go(DbName, DDoc, - IndexName, QueryArgs1) of - {ok, {TotalHits, TotalGroupedHits, Groups0}} -> - Groups = [group_to_json(DbName, IncludeDocs, Group, UseNewApi) || Group <- Groups0], - send_grouped_response(Req, {TotalHits, TotalGroupedHits, Groups}, UseNewApi); - {error, Reason} -> - handle_error(Req, Db, DDoc, RetryCount, RetryPause, Reason) - end; - {error, Reason} -> - handle_error(Req, Db, DDoc, RetryCount, RetryPause, Reason) - end - end, + Response = + case Grouping#grouping.by of + nil -> + case dreyfus_fabric_search:go(DbName, DDoc, IndexName, QueryArgs) of + % legacy clause + {ok, Bookmark0, TotalHits, Hits0} -> + Hits = hits_to_json(DbName, IncludeDocs, Hits0), + Bookmark = dreyfus_bookmark:pack(Bookmark0), + send_json( + Req, + 200, + {[ + {total_rows, TotalHits}, + {bookmark, Bookmark}, + {rows, Hits} + ]} + ); + {ok, Bookmark0, TotalHits, Hits0, Counts0, Ranges0} -> + Hits = hits_to_json(DbName, IncludeDocs, Hits0), + Bookmark = dreyfus_bookmark:pack(Bookmark0), + Counts = + case Counts0 of + undefined -> + []; + _ -> + [{counts, facets_to_json(Counts0)}] + end, + Ranges = + case Ranges0 of + undefined -> + []; + _ -> + [{ranges, facets_to_json(Ranges0)}] + end, + send_json(Req, 200, { + [ + {total_rows, TotalHits}, + {bookmark, Bookmark}, + {rows, Hits} + ] ++ Counts ++ Ranges + }); + {error, Reason} -> + handle_error(Req, Db, DDoc, RetryCount, RetryPause, Reason) + end; + _ -> + % ensure limit in group query >0 + UseNewApi = Grouping#grouping.new_api, + case dreyfus_fabric_group1:go(DbName, DDoc, IndexName, QueryArgs) of + {ok, []} -> + send_grouped_response(Req, {0, 0, []}, UseNewApi); + {ok, TopGroups} -> + QueryArgs1 = QueryArgs#index_query_args{ + grouping = Grouping#grouping{groups = TopGroups} + }, + case + dreyfus_fabric_group2:go( + DbName, + DDoc, + IndexName, + QueryArgs1 + ) + of + {ok, {TotalHits, TotalGroupedHits, Groups0}} -> + Groups = [ + group_to_json(DbName, IncludeDocs, Group, UseNewApi) + || Group <- Groups0 + ], + send_grouped_response( + Req, {TotalHits, TotalGroupedHits, Groups}, UseNewApi + ); + {error, Reason} -> + handle_error(Req, Db, DDoc, RetryCount, RetryPause, Reason) + end; + {error, Reason} -> + handle_error(Req, Db, DDoc, RetryCount, RetryPause, Reason) + end + end, RequestTime = timer:now_diff(os:timestamp(), Start) div 1000, couch_stats:update_histogram([dreyfus, httpd, search], RequestTime), Response; -handle_search_req(#httpd{path_parts=[_, _, _, _, _]}=Req, _Db, _DDoc, _RetryCount, _RetryPause) -> +handle_search_req(#httpd{path_parts = [_, _, _, _, _]} = Req, _Db, _DDoc, _RetryCount, _RetryPause) -> send_method_not_allowed(Req, "GET,POST"); handle_search_req(Req, _Db, _DDoc, _RetryCount, _RetryPause) -> send_error(Req, {bad_request, "path not recognized"}). -handle_info_req(#httpd{method='GET', path_parts=[_, _, _, _, IndexName]}=Req - ,Db, #doc{id=Id}=DDoc) -> +handle_info_req( + #httpd{method = 'GET', path_parts = [_, _, _, _, IndexName]} = Req, + Db, + #doc{id = Id} = DDoc +) -> DbName = couch_db:name(Db), case dreyfus_fabric_info:go(DbName, DDoc, IndexName, info) of {ok, IndexInfoList} -> - send_json(Req, 200, {[ - {name, <<Id/binary,"/",IndexName/binary>>}, - {search_index, {IndexInfoList}} - ]}); + send_json( + Req, + 200, + {[ + {name, <<Id/binary, "/", IndexName/binary>>}, + {search_index, {IndexInfoList}} + ]} + ); {error, Reason} -> send_error(Req, Reason) end; -handle_info_req(#httpd{path_parts=[_, _, _, _, _]}=Req, _Db, _DDoc) -> +handle_info_req(#httpd{path_parts = [_, _, _, _, _]} = Req, _Db, _DDoc) -> send_method_not_allowed(Req, "GET"); handle_info_req(Req, _Db, _DDoc) -> send_error(Req, {bad_request, "path not recognized"}). -handle_disk_size_req(#httpd{method='GET', path_parts=[_, _, _, _, IndexName]}=Req, Db, #doc{id=Id}=DDoc) -> +handle_disk_size_req( + #httpd{method = 'GET', path_parts = [_, _, _, _, IndexName]} = Req, Db, #doc{id = Id} = DDoc +) -> DbName = couch_db:name(Db), case dreyfus_fabric_info:go(DbName, DDoc, IndexName, disk_size) of {ok, IndexInfoList} -> - send_json(Req, 200, {[ - {name, <<Id/binary,"/",IndexName/binary>>}, - {search_index, {IndexInfoList}} - ]}); + send_json( + Req, + 200, + {[ + {name, <<Id/binary, "/", IndexName/binary>>}, + {search_index, {IndexInfoList}} + ]} + ); {error, Reason} -> send_error(Req, Reason) end; -handle_disk_size_req(#httpd{path_parts=[_, _, _, _, _]}=Req, _Db, _DDoc) -> +handle_disk_size_req(#httpd{path_parts = [_, _, _, _, _]} = Req, _Db, _DDoc) -> send_method_not_allowed(Req, "GET"); handle_disk_size_req(Req, _Db, _DDoc) -> send_error(Req, {bad_request, "path not recognized"}). -handle_cleanup_req(#httpd{method='POST'}=Req, Db) -> +handle_cleanup_req(#httpd{method = 'POST'} = Req, Db) -> ok = dreyfus_fabric_cleanup:go(couch_db:name(Db)), send_json(Req, 202, {[{ok, true}]}); handle_cleanup_req(Req, _Db) -> send_method_not_allowed(Req, "POST"). -handle_analyze_req(#httpd{method='GET'}=Req) -> +handle_analyze_req(#httpd{method = 'GET'} = Req) -> Analyzer = couch_httpd:qs_value(Req, "analyzer"), Text = couch_httpd:qs_value(Req, "text"), analyze(Req, Analyzer, Text); -handle_analyze_req(#httpd{method='POST'}=Req) -> +handle_analyze_req(#httpd{method = 'POST'} = Req) -> couch_httpd:validate_ctype(Req, "application/json"), {Fields} = chttpd:json_body_obj(Req), Analyzer = couch_util:get_value(<<"analyzer">>, Fields), @@ -159,7 +208,7 @@ analyze(Req, Analyzer, Text) -> ok; _ when is_binary(Analyzer) -> ok; - {[_|_]} -> + {[_ | _]} -> ok; _ -> throw({bad_request, "analyzer parameter must be a string or an object"}) @@ -174,86 +223,101 @@ analyze(Req, Analyzer, Text) -> _ -> throw({bad_request, "text parameter must be a string"}) end, - case clouseau_rpc:analyze(couch_util:to_binary(Analyzer), - couch_util:to_binary(Text)) of + case + clouseau_rpc:analyze( + couch_util:to_binary(Analyzer), + couch_util:to_binary(Text) + ) + of {ok, Tokens} -> send_json(Req, 200, {[{tokens, Tokens}]}); {error, Reason} -> send_error(Req, Reason) end. -parse_index_params(#httpd{method='GET'}=Req, Db) -> - IndexParams = lists:flatmap(fun({K, V}) -> parse_index_param(K, V) end, - chttpd:qs(Req)), +parse_index_params(#httpd{method = 'GET'} = Req, Db) -> + IndexParams = lists:flatmap( + fun({K, V}) -> parse_index_param(K, V) end, + chttpd:qs(Req) + ), parse_index_params(IndexParams, Db); -parse_index_params(#httpd{method='POST'}=Req, Db) -> +parse_index_params(#httpd{method = 'POST'} = Req, Db) -> {JsonBody} = chttpd:json_body_obj(Req), - QSEntry = case chttpd:qs_value(Req, "partition") of - undefined -> []; - StrVal -> [{<<"partition">>, ?l2b(StrVal)}] - end, - IndexParams = lists:flatmap(fun({K, V}) -> - parse_json_index_param(K, V) - end, QSEntry ++ JsonBody), + QSEntry = + case chttpd:qs_value(Req, "partition") of + undefined -> []; + StrVal -> [{<<"partition">>, ?l2b(StrVal)}] + end, + IndexParams = lists:flatmap( + fun({K, V}) -> + parse_json_index_param(K, V) + end, + QSEntry ++ JsonBody + ), ensure_unique_partition(IndexParams), parse_index_params(IndexParams, Db); parse_index_params(IndexParams, Db) -> - DefaultLimit = case fabric_util:is_partitioned(Db) of - true -> - list_to_integer(config:get("dreyfus", "limit_partitions", "2000")); - false -> - list_to_integer(config:get("dreyfus", "limit", "25")) - end, - Args = #index_query_args{limit=DefaultLimit}, - lists:foldl(fun({K, V}, Args2) -> - validate_index_query(K, V, Args2) - end, Args, IndexParams). + DefaultLimit = + case fabric_util:is_partitioned(Db) of + true -> + list_to_integer(config:get("dreyfus", "limit_partitions", "2000")); + false -> + list_to_integer(config:get("dreyfus", "limit", "25")) + end, + Args = #index_query_args{limit = DefaultLimit}, + lists:foldl( + fun({K, V}, Args2) -> + validate_index_query(K, V, Args2) + end, + Args, + IndexParams + ). validate_index_query(q, Value, Args) -> - Args#index_query_args{q=Value}; + Args#index_query_args{q = Value}; validate_index_query(partition, Value, Args) -> - Args#index_query_args{partition=Value}; + Args#index_query_args{partition = Value}; validate_index_query(stale, Value, Args) -> - Args#index_query_args{stale=Value}; + Args#index_query_args{stale = Value}; validate_index_query(limit, Value, Args) -> - Args#index_query_args{limit=Value}; + Args#index_query_args{limit = Value}; validate_index_query(include_docs, Value, Args) -> - Args#index_query_args{include_docs=Value}; + Args#index_query_args{include_docs = Value}; validate_index_query(include_fields, Value, Args) -> - Args#index_query_args{include_fields=Value}; + Args#index_query_args{include_fields = Value}; validate_index_query(bookmark, Value, Args) -> - Args#index_query_args{bookmark=Value}; + Args#index_query_args{bookmark = Value}; validate_index_query(sort, Value, Args) -> - Args#index_query_args{sort=Value}; -validate_index_query(group_by, Value, #index_query_args{grouping=Grouping}=Args) -> - Args#index_query_args{grouping=Grouping#grouping{by=Value, new_api=false}}; -validate_index_query(group_field, Value, #index_query_args{grouping=Grouping}=Args) -> - Args#index_query_args{grouping=Grouping#grouping{by=Value, new_api=true}}; -validate_index_query(group_sort, Value, #index_query_args{grouping=Grouping}=Args) -> - Args#index_query_args{grouping=Grouping#grouping{sort=Value}}; -validate_index_query(group_limit, Value, #index_query_args{grouping=Grouping}=Args) -> - Args#index_query_args{grouping=Grouping#grouping{limit=Value}}; + Args#index_query_args{sort = Value}; +validate_index_query(group_by, Value, #index_query_args{grouping = Grouping} = Args) -> + Args#index_query_args{grouping = Grouping#grouping{by = Value, new_api = false}}; +validate_index_query(group_field, Value, #index_query_args{grouping = Grouping} = Args) -> + Args#index_query_args{grouping = Grouping#grouping{by = Value, new_api = true}}; +validate_index_query(group_sort, Value, #index_query_args{grouping = Grouping} = Args) -> + Args#index_query_args{grouping = Grouping#grouping{sort = Value}}; +validate_index_query(group_limit, Value, #index_query_args{grouping = Grouping} = Args) -> + Args#index_query_args{grouping = Grouping#grouping{limit = Value}}; validate_index_query(stable, Value, Args) -> - Args#index_query_args{stable=Value}; + Args#index_query_args{stable = Value}; validate_index_query(counts, Value, Args) -> - Args#index_query_args{counts=Value}; + Args#index_query_args{counts = Value}; validate_index_query(ranges, Value, Args) -> - Args#index_query_args{ranges=Value}; -validate_index_query(drilldown, [[_|_]|_] = Value, Args) -> - Args#index_query_args{drilldown=Value}; + Args#index_query_args{ranges = Value}; +validate_index_query(drilldown, [[_ | _] | _] = Value, Args) -> + Args#index_query_args{drilldown = Value}; validate_index_query(drilldown, Value, Args) -> DrillDown = Args#index_query_args.drilldown, - Args#index_query_args{drilldown=[Value|DrillDown]}; + Args#index_query_args{drilldown = [Value | DrillDown]}; validate_index_query(highlight_fields, Value, Args) -> - Args#index_query_args{highlight_fields=Value}; + Args#index_query_args{highlight_fields = Value}; validate_index_query(highlight_pre_tag, Value, Args) -> - Args#index_query_args{highlight_pre_tag=Value}; + Args#index_query_args{highlight_pre_tag = Value}; validate_index_query(highlight_post_tag, Value, Args) -> - Args#index_query_args{highlight_post_tag=Value}; + Args#index_query_args{highlight_post_tag = Value}; validate_index_query(highlight_number, Value, Args) -> - Args#index_query_args{highlight_number=Value}; + Args#index_query_args{highlight_number = Value}; validate_index_query(highlight_size, Value, Args) -> - Args#index_query_args{highlight_size=Value}; + Args#index_query_args{highlight_size = Value}; validate_index_query(extra, _Value, Args) -> Args. @@ -359,8 +423,10 @@ parse_json_index_param(Key, Value) -> parse_bool_param(_, Val) when is_boolean(Val) -> Val; -parse_bool_param(_, "true") -> true; -parse_bool_param(_, "false") -> false; +parse_bool_param(_, "true") -> + true; +parse_bool_param(_, "false") -> + false; parse_bool_param(Name, Val) -> Msg = io_lib:format("Invalid value for ~s: ~p", [Name, Val]), throw({query_parse_error, ?l2b(Msg)}). @@ -369,72 +435,76 @@ parse_int_param(_, Val) when is_integer(Val) -> Val; parse_int_param(Name, Val) -> case (catch list_to_integer(Val)) of - IntVal when is_integer(IntVal) -> - IntVal; - _ -> - Msg = io_lib:format("Invalid value for ~s: ~p", [Name, Val]), - throw({query_parse_error, ?l2b(Msg)}) + IntVal when is_integer(IntVal) -> + IntVal; + _ -> + Msg = io_lib:format("Invalid value for ~s: ~p", [Name, Val]), + throw({query_parse_error, ?l2b(Msg)}) end. parse_positive_int_param(Name, Val, Prop, Default) -> MaximumVal = list_to_integer( - config:get("dreyfus", Prop, Default)), + config:get("dreyfus", Prop, Default) + ), case parse_int_param(Name, Val) of - IntVal when IntVal > MaximumVal -> - Fmt = "Value for ~s is too large, must not exceed ~p", - Msg = io_lib:format(Fmt, [Name, MaximumVal]), - throw({query_parse_error, ?l2b(Msg)}); - IntVal when IntVal > 0 -> - IntVal; - IntVal when IntVal =< 0 -> - Fmt = "~s must be greater than zero", - Msg = io_lib:format(Fmt, [Name]), - throw({query_parse_error, ?l2b(Msg)}); - _ -> - Fmt = "Invalid value for ~s: ~p", - Msg = io_lib:format(Fmt, [Name, Val]), - throw({query_parse_error, ?l2b(Msg)}) + IntVal when IntVal > MaximumVal -> + Fmt = "Value for ~s is too large, must not exceed ~p", + Msg = io_lib:format(Fmt, [Name, MaximumVal]), + throw({query_parse_error, ?l2b(Msg)}); + IntVal when IntVal > 0 -> + IntVal; + IntVal when IntVal =< 0 -> + Fmt = "~s must be greater than zero", + Msg = io_lib:format(Fmt, [Name]), + throw({query_parse_error, ?l2b(Msg)}); + _ -> + Fmt = "Invalid value for ~s: ~p", + Msg = io_lib:format(Fmt, [Name, Val]), + throw({query_parse_error, ?l2b(Msg)}) end. parse_positive_int_param2(Name, Val) -> case parse_int_param(Name, Val) of - IntVal when IntVal > 0 -> - IntVal; - IntVal when IntVal =< 0 -> - Fmt = "~s must be greater than zero", - Msg = io_lib:format(Fmt, [Name]), - throw({query_parse_error, ?l2b(Msg)}); - _ -> - Fmt = "Invalid value for ~s: ~p", - Msg = io_lib:format(Fmt, [Name, Val]), - throw({query_parse_error, ?l2b(Msg)}) + IntVal when IntVal > 0 -> + IntVal; + IntVal when IntVal =< 0 -> + Fmt = "~s must be greater than zero", + Msg = io_lib:format(Fmt, [Name]), + throw({query_parse_error, ?l2b(Msg)}); + _ -> + Fmt = "Invalid value for ~s: ~p", + Msg = io_lib:format(Fmt, [Name, Val]), + throw({query_parse_error, ?l2b(Msg)}) end. parse_non_negative_int_param(Name, Val, Prop, Default) -> MaximumVal = list_to_integer( - config:get("dreyfus", Prop, Default)), + config:get("dreyfus", Prop, Default) + ), case parse_int_param(Name, Val) of - IntVal when IntVal > MaximumVal -> - Fmt = "Value for ~s is too large, must not exceed ~p", - Msg = io_lib:format(Fmt, [Name, MaximumVal]), - throw({query_parse_error, ?l2b(Msg)}); - IntVal when IntVal >= 0 -> - IntVal; - IntVal when IntVal < 0 -> - Fmt = "~s must be greater than or equal to zero", - Msg = io_lib:format(Fmt, [Name]), - throw({query_parse_error, ?l2b(Msg)}); - _ -> - Fmt = "Invalid value for ~s: ~p", - Msg = io_lib:format(Fmt, [Name, Val]), - throw({query_parse_error, ?l2b(Msg)}) + IntVal when IntVal > MaximumVal -> + Fmt = "Value for ~s is too large, must not exceed ~p", + Msg = io_lib:format(Fmt, [Name, MaximumVal]), + throw({query_parse_error, ?l2b(Msg)}); + IntVal when IntVal >= 0 -> + IntVal; + IntVal when IntVal < 0 -> + Fmt = "~s must be greater than or equal to zero", + Msg = io_lib:format(Fmt, [Name]), + throw({query_parse_error, ?l2b(Msg)}); + _ -> + Fmt = "Invalid value for ~s: ~p", + Msg = io_lib:format(Fmt, [Name, Val]), + throw({query_parse_error, ?l2b(Msg)}) end. - ensure_unique_partition(IndexParams) -> - Partitions = lists:filter(fun({Key, _Val}) -> - Key == partition - end, IndexParams), + Partitions = lists:filter( + fun({Key, _Val}) -> + Key == partition + end, + IndexParams + ), case length(lists:usort(Partitions)) > 1 of true -> Msg = <<"Multiple conflicting values for `partition` provided">>, @@ -443,7 +513,6 @@ ensure_unique_partition(IndexParams) -> ok end. - validate_search_restrictions(Db, DDoc, Args) -> #index_query_args{ q = Query, @@ -485,8 +554,10 @@ validate_search_restrictions(Db, DDoc, Args) -> {true, true} -> ok; {true, false} -> - Msg3 = <<"`partition` parameter is mandatory " - "for queries to this index.">>, + Msg3 = << + "`partition` parameter is mandatory " + "for queries to this index." + >>, throw({bad_request, Msg3}); {false, true} -> Msg4 = <<"`partition` not supported on this index">>, @@ -497,7 +568,8 @@ validate_search_restrictions(Db, DDoc, Args) -> true -> MaxLimit = config:get("dreyfus", "max_limit", "2000"), parse_non_negative_int_param( - "limit", Limit, "max_limit_partitions", MaxLimit); + "limit", Limit, "max_limit_partitions", MaxLimit + ); false -> MaxLimit = config:get("dreyfus", "max_limit", "200"), parse_non_negative_int_param("limit", Limit, "max_limit", MaxLimit) @@ -505,52 +577,69 @@ validate_search_restrictions(Db, DDoc, Args) -> DefaultArgs = #index_query_args{}, - case is_binary(Partition) andalso ( - Counts /= DefaultArgs#index_query_args.counts - orelse Drilldown /= DefaultArgs#index_query_args.drilldown - orelse Ranges /= DefaultArgs#index_query_args.ranges - orelse GroupSort /= DefaultArgs#index_query_args.grouping#grouping.sort - orelse GroupBy /= DefaultArgs#index_query_args.grouping#grouping.by - orelse GroupLimit /= DefaultArgs#index_query_args.grouping#grouping.limit - ) of + case + is_binary(Partition) andalso + (Counts /= DefaultArgs#index_query_args.counts orelse + Drilldown /= DefaultArgs#index_query_args.drilldown orelse + Ranges /= DefaultArgs#index_query_args.ranges orelse + GroupSort /= DefaultArgs#index_query_args.grouping#grouping.sort orelse + GroupBy /= DefaultArgs#index_query_args.grouping#grouping.by orelse + GroupLimit /= DefaultArgs#index_query_args.grouping#grouping.limit) + of true -> - Msg5 = <<"`partition` and any of `drilldown`, `ranges`, `group_field`, `group_sort`, `group_limit` or `group_by` are incompatible">>, + Msg5 = + <<"`partition` and any of `drilldown`, `ranges`, `group_field`, `group_sort`, `group_limit` or `group_by` are incompatible">>, throw({bad_request, Msg5}); false -> ok end. - get_view_partition_option(#doc{body = {Props}}, Default) -> {Options} = couch_util:get_value(<<"options">>, Props, {[]}), couch_util:get_value(<<"partitioned">>, Options, Default). - hits_to_json(DbName, IncludeDocs, Hits) -> {Ids, HitData} = lists:unzip(lists:map(fun get_hit_data/1, Hits)), chttpd_stats:incr_rows(length(Hits)), - if IncludeDocs -> - chttpd_stats:incr_reads(length(Hits)), - {ok, JsonDocs} = dreyfus_fabric:get_json_docs(DbName, Ids), - lists:zipwith(fun(Hit, {Id, Doc}) -> - case Hit of - {Id, Order, Fields} -> - {[{id, Id}, {order, Order}, {fields, {Fields}}, Doc]}; - {Id, Order, Fields, Highlights} -> - {[{id, Id}, {order, Order}, {fields, {Fields}}, - {highlights, {Highlights}}, Doc]} - end - end, HitData, JsonDocs); - - true -> - lists:map(fun(Hit) -> - case Hit of - {Id, Order, Fields} -> - {[{id, Id}, {order, Order}, {fields, {Fields}}]}; - {Id, Order, Fields, Highlights} -> - {[{id, Id}, {order, Order}, {fields, {Fields}}, {highlights, {Highlights}}]} - end - end, HitData) + if + IncludeDocs -> + chttpd_stats:incr_reads(length(Hits)), + {ok, JsonDocs} = dreyfus_fabric:get_json_docs(DbName, Ids), + lists:zipwith( + fun(Hit, {Id, Doc}) -> + case Hit of + {Id, Order, Fields} -> + {[{id, Id}, {order, Order}, {fields, {Fields}}, Doc]}; + {Id, Order, Fields, Highlights} -> + {[ + {id, Id}, + {order, Order}, + {fields, {Fields}}, + {highlights, {Highlights}}, + Doc + ]} + end + end, + HitData, + JsonDocs + ); + true -> + lists:map( + fun(Hit) -> + case Hit of + {Id, Order, Fields} -> + {[{id, Id}, {order, Order}, {fields, {Fields}}]}; + {Id, Order, Fields, Highlights} -> + {[ + {id, Id}, + {order, Order}, + {fields, {Fields}}, + {highlights, {Highlights}} + ]} + end + end, + HitData + ) end. get_hit_data(Hit) -> @@ -565,13 +654,16 @@ get_hit_data(Hit) -> end. group_to_json(DbName, IncludeDocs, {Name, TotalHits, Hits}, UseNewApi) -> - {TotalHitsKey, HitsKey} = case UseNewApi of - true -> {total_rows, rows}; - false -> {total_hits, hits} - end, - {[{by, Name}, - {TotalHitsKey, TotalHits}, - {HitsKey, hits_to_json(DbName, IncludeDocs, Hits)}]}. + {TotalHitsKey, HitsKey} = + case UseNewApi of + true -> {total_rows, rows}; + false -> {total_hits, hits} + end, + {[ + {by, Name}, + {TotalHitsKey, TotalHits}, + {HitsKey, hits_to_json(DbName, IncludeDocs, Hits)} + ]}. facets_to_json(Facets) -> {[facet_to_json(F) || F <- Facets]}. @@ -583,10 +675,13 @@ facet_to_json({K0, _V0, C0}) -> {hd(K0), facets_to_json(C2)}. send_grouped_response(Req, {TotalHits, TotalGroupedHits, Groups}, UseNewApi) -> - GroupResponsePairs = case UseNewApi of - true -> [{total_rows, TotalHits}, {groups, Groups}]; - false -> [{total_hits, TotalHits}, {total_grouped_hits, TotalGroupedHits}, {groups, Groups}] - end, + GroupResponsePairs = + case UseNewApi of + true -> + [{total_rows, TotalHits}, {groups, Groups}]; + false -> + [{total_hits, TotalHits}, {total_grouped_hits, TotalGroupedHits}, {groups, Groups}] + end, send_json(Req, 200, {GroupResponsePairs}). handle_error(Req, Db, DDoc, RetryCount, RetryPause, {exit, _} = Err) -> diff --git a/src/dreyfus/src/dreyfus_httpd_handlers.erl b/src/dreyfus/src/dreyfus_httpd_handlers.erl index bf2be23b1..05188c252 100644 --- a/src/dreyfus/src/dreyfus_httpd_handlers.erl +++ b/src/dreyfus/src/dreyfus_httpd_handlers.erl @@ -10,7 +10,6 @@ % License for the specific language governing permissions and limitations under % the License. - %% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- -module(dreyfus_httpd_handlers). @@ -20,10 +19,10 @@ url_handler(<<"_search_analyze">>) -> fun dreyfus_httpd:handle_analyze_req/1; url_handler(_) -> no_match. -db_handler(<<"_search_cleanup">>) -> fun dreyfus_httpd:handle_cleanup_req/2; +db_handler(<<"_search_cleanup">>) -> fun dreyfus_httpd:handle_cleanup_req/2; db_handler(_) -> no_match. -design_handler(<<"_search">>) -> fun dreyfus_httpd:handle_search_req/3; +design_handler(<<"_search">>) -> fun dreyfus_httpd:handle_search_req/3; design_handler(<<"_search_info">>) -> fun dreyfus_httpd:handle_info_req/3; design_handler(<<"_search_disk_size">>) -> fun dreyfus_httpd:handle_disk_size_req/3; design_handler(_) -> no_match. diff --git a/src/dreyfus/src/dreyfus_index.erl b/src/dreyfus/src/dreyfus_index.erl index 2bf560f37..df3e68f84 100644 --- a/src/dreyfus/src/dreyfus_index.erl +++ b/src/dreyfus/src/dreyfus_index.erl @@ -10,7 +10,6 @@ % License for the specific language governing permissions and limitations under % the License. - %% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- %% A dreyfus_index gen_server is linked to its clouseau twin. @@ -21,23 +20,35 @@ -include_lib("couch/include/couch_db.hrl"). -include("dreyfus.hrl"). - % public api. --export([start_link/2, design_doc_to_index/2, await/2, search/2, info/1, - group1/2, group2/2, - design_doc_to_indexes/1]). +-export([ + start_link/2, + design_doc_to_index/2, + await/2, + search/2, + info/1, + group1/2, + group2/2, + design_doc_to_indexes/1 +]). % gen_server api. --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3]). +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3 +]). % private definitions. -record(state, { dbname, index, - updater_pid=nil, - index_pid=nil, - waiting_list=[] + updater_pid = nil, + index_pid = nil, + waiting_list = [] }). % exported for callback. @@ -75,11 +86,11 @@ info(Pid0) -> %% clouseau pid. to_index_pid(Pid) -> case node(Pid) == node() of - true -> gen_server:call(Pid, get_index_pid, infinity); + true -> gen_server:call(Pid, get_index_pid, infinity); false -> Pid end. -design_doc_to_indexes(#doc{body={Fields}}=Doc) -> +design_doc_to_indexes(#doc{body = {Fields}} = Doc) -> RawIndexes = couch_util:get_value(<<"indexes">>, Fields, {[]}), case RawIndexes of {IndexList} when is_list(IndexList) -> @@ -87,12 +98,14 @@ design_doc_to_indexes(#doc{body={Fields}}=Doc) -> lists:flatmap( fun(IndexName) -> case (catch design_doc_to_index(Doc, IndexName)) of - {ok, #index{}=Index} -> [Index]; + {ok, #index{} = Index} -> [Index]; _ -> [] end end, - IndexNames); - _ -> [] + IndexNames + ); + _ -> + [] end. % gen_server functions. @@ -101,14 +114,18 @@ init({DbName, Index}) -> process_flag(trap_exit, true), case open_index(DbName, Index) of {ok, Pid, Seq} -> - State=#state{ - dbname=DbName, - index=Index#index{current_seq=Seq, dbname=DbName}, - index_pid=Pid - }, + State = #state{ + dbname = DbName, + index = Index#index{current_seq = Seq, dbname = DbName}, + index_pid = Pid + }, case couch_db:open_int(DbName, []) of {ok, Db} -> - try couch_db:monitor(Db) after couch_db:close(Db) end, + try + couch_db:monitor(Db) + after + couch_db:close(Db) + end, dreyfus_util:maybe_create_local_purge_doc(Db, Pid, Index), proc_lib:init_ack({ok, self()}), gen_server:enter_loop(?MODULE, [], State); @@ -119,122 +136,155 @@ init({DbName, Index}) -> proc_lib:init_ack(Error) end. -handle_call({await, RequestSeq}, From, - #state{ - index=#index{dbname=DbName,name=IdxName,ddoc_id=DDocId,current_seq=Seq}=Index, - index_pid=IndexPid, - updater_pid=nil, - waiting_list=WaitList - }=State) when RequestSeq > Seq -> +handle_call( + {await, RequestSeq}, + From, + #state{ + index = + #index{dbname = DbName, name = IdxName, ddoc_id = DDocId, current_seq = Seq} = Index, + index_pid = IndexPid, + updater_pid = nil, + waiting_list = WaitList + } = State +) when RequestSeq > Seq -> DbName2 = mem3:dbname(DbName), <<"_design/", GroupId/binary>> = DDocId, - NewState = case dreyfus_util:in_black_list(DbName2, GroupId, IdxName) of - false -> - UpPid = spawn_link(fun() -> - dreyfus_index_updater:update(IndexPid,Index) - end), - State#state{ - updater_pid=UpPid, - waiting_list=[{From,RequestSeq}|WaitList] - }; - _ -> - couch_log:notice("Index Blocked from Updating - db: ~p," - " ddocid: ~p name: ~p", [DbName, DDocId, IdxName]), - State - end, + NewState = + case dreyfus_util:in_black_list(DbName2, GroupId, IdxName) of + false -> + UpPid = spawn_link(fun() -> + dreyfus_index_updater:update(IndexPid, Index) + end), + State#state{ + updater_pid = UpPid, + waiting_list = [{From, RequestSeq} | WaitList] + }; + _ -> + couch_log:notice( + "Index Blocked from Updating - db: ~p," + " ddocid: ~p name: ~p", + [DbName, DDocId, IdxName] + ), + State + end, {noreply, NewState}; -handle_call({await, RequestSeq}, _From, - #state{index=#index{current_seq=Seq}}=State) when RequestSeq =< Seq -> +handle_call( + {await, RequestSeq}, + _From, + #state{index = #index{current_seq = Seq}} = State +) when RequestSeq =< Seq -> {reply, {ok, State#state.index_pid, Seq}, State}; -handle_call({await, RequestSeq}, From, #state{waiting_list=WaitList}=State) -> +handle_call({await, RequestSeq}, From, #state{waiting_list = WaitList} = State) -> {noreply, State#state{ - waiting_list=[{From,RequestSeq}|WaitList] + waiting_list = [{From, RequestSeq} | WaitList] }}; - -handle_call(get_index_pid, _From, State) -> % upgrade +% upgrade +handle_call(get_index_pid, _From, State) -> {reply, State#state.index_pid, State}; - -handle_call({search, QueryArgs0}, _From, State) -> % obsolete +% obsolete +handle_call({search, QueryArgs0}, _From, State) -> Reply = search_int(State#state.index_pid, QueryArgs0), {reply, Reply, State}; - -handle_call({group1, QueryArgs0}, _From, State) -> % obsolete +% obsolete +handle_call({group1, QueryArgs0}, _From, State) -> Reply = group1_int(State#state.index_pid, QueryArgs0), {reply, Reply, State}; - -handle_call({group2, QueryArgs0}, _From, State) -> % obsolete +% obsolete +handle_call({group2, QueryArgs0}, _From, State) -> Reply = group2_int(State#state.index_pid, QueryArgs0), {reply, Reply, State}; - -handle_call(info, _From, State) -> % obsolete +% obsolete +handle_call(info, _From, State) -> Reply = info_int(State#state.index_pid), {reply, Reply, State}. handle_cast(_Msg, State) -> {noreply, State}. -handle_info({'EXIT', FromPid, {updated, NewSeq}}, - #state{ - index=#index{dbname=DbName,name=IdxName,ddoc_id=DDocId}=Index0, - index_pid=IndexPid, - updater_pid=UpPid, - waiting_list=WaitList - }=State) when UpPid == FromPid -> - Index = Index0#index{current_seq=NewSeq}, +handle_info( + {'EXIT', FromPid, {updated, NewSeq}}, + #state{ + index = #index{dbname = DbName, name = IdxName, ddoc_id = DDocId} = Index0, + index_pid = IndexPid, + updater_pid = UpPid, + waiting_list = WaitList + } = State +) when UpPid == FromPid -> + Index = Index0#index{current_seq = NewSeq}, case reply_with_index(IndexPid, Index, WaitList) of - [] -> - {noreply, State#state{index=Index, - updater_pid=nil, - waiting_list=[] - }}; - StillWaiting -> - DbName2 = mem3:dbname(DbName), - <<"_design/", GroupId/binary>> = DDocId, - Pid = case dreyfus_util:in_black_list(DbName2, GroupId, IdxName) of - true -> - couch_log:notice("Index Blocked from Updating - db: ~p, ddocid: ~p" - " name: ~p", [DbName, GroupId, IdxName]), - nil; - false -> - spawn_link(fun() -> - dreyfus_index_updater:update(IndexPid, Index) - end) - end, - {noreply, State#state{index=Index, - updater_pid=Pid, - waiting_list=StillWaiting - }} + [] -> + {noreply, State#state{ + index = Index, + updater_pid = nil, + waiting_list = [] + }}; + StillWaiting -> + DbName2 = mem3:dbname(DbName), + <<"_design/", GroupId/binary>> = DDocId, + Pid = + case dreyfus_util:in_black_list(DbName2, GroupId, IdxName) of + true -> + couch_log:notice( + "Index Blocked from Updating - db: ~p, ddocid: ~p" + " name: ~p", + [DbName, GroupId, IdxName] + ), + nil; + false -> + spawn_link(fun() -> + dreyfus_index_updater:update(IndexPid, Index) + end) + end, + {noreply, State#state{ + index = Index, + updater_pid = Pid, + waiting_list = StillWaiting + }} end; handle_info({'EXIT', _, {updated, _}}, State) -> {noreply, State}; -handle_info({'EXIT', FromPid, Reason}, #state{ - index=Index, - index_pid=IndexPid, - waiting_list=WaitList - }=State) when FromPid == IndexPid -> +handle_info( + {'EXIT', FromPid, Reason}, + #state{ + index = Index, + index_pid = IndexPid, + waiting_list = WaitList + } = State +) when FromPid == IndexPid -> couch_log:notice( - "index for ~p closed with reason ~p", [index_name(Index), Reason]), + "index for ~p closed with reason ~p", [index_name(Index), Reason] + ), [gen_server:reply(Pid, {error, Reason}) || {Pid, _} <- WaitList], {stop, normal, State}; -handle_info({'EXIT', FromPid, Reason}, #state{ - index=Index, - updater_pid=UpPid, - waiting_list=WaitList - }=State) when FromPid == UpPid -> - couch_log:info("Shutting down index server ~p, updater ~p closing w/ reason ~w", - [index_name(Index), UpPid, Reason]), +handle_info( + {'EXIT', FromPid, Reason}, + #state{ + index = Index, + updater_pid = UpPid, + waiting_list = WaitList + } = State +) when FromPid == UpPid -> + couch_log:info( + "Shutting down index server ~p, updater ~p closing w/ reason ~w", + [index_name(Index), UpPid, Reason] + ), [gen_server:reply(Pid, {error, Reason}) || {Pid, _} <- WaitList], {stop, normal, State}; handle_info({'EXIT', Pid, Reason}, State) -> % probably dreyfus_index_manager. couch_log:notice("Unknown pid ~p closed with reason ~p", [Pid, Reason]), {stop, normal, State}; -handle_info({'DOWN',_,_,Pid,Reason}, #state{ - index=Index, - waiting_list=WaitList - }=State) -> - couch_log:info("Shutting down index server ~p, db ~p closing w/ reason ~w", - [index_name(Index), Pid, Reason]), +handle_info( + {'DOWN', _, _, Pid, Reason}, + #state{ + index = Index, + waiting_list = WaitList + } = State +) -> + couch_log:info( + "Shutting down index server ~p, db ~p closing w/ reason ~w", + [index_name(Index), Pid, Reason] + ), [gen_server:reply(P, {error, Reason}) || {P, _} <- WaitList], {stop, normal, State}. @@ -246,8 +296,8 @@ code_change(_OldVsn, State, _Extra) -> % private functions. -open_index(DbName, #index{analyzer=Analyzer, sig=Sig}) -> - Path = <<DbName/binary,"/",Sig/binary>>, +open_index(DbName, #index{analyzer = Analyzer, sig = Sig}) -> + Path = <<DbName/binary, "/", Sig/binary>>, case clouseau_rpc:open_index(self(), Path, Analyzer) of {ok, Pid} -> case clouseau_rpc:get_update_seq(Pid) of @@ -260,11 +310,11 @@ open_index(DbName, #index{analyzer=Analyzer, sig=Sig}) -> Error end. -design_doc_to_index(#doc{id=Id,body={Fields}}, IndexName) -> +design_doc_to_index(#doc{id = Id, body = {Fields}}, IndexName) -> Language = couch_util:get_value(<<"language">>, Fields, <<"javascript">>), {RawIndexes} = couch_util:get_value(<<"indexes">>, Fields, {[]}), - InvalidDDocError = {invalid_design_doc, - <<"index `", IndexName/binary, "` must have parameter `index`">>}, + InvalidDDocError = + {invalid_design_doc, <<"index `", IndexName/binary, "` must have parameter `index`">>}, case lists:keyfind(IndexName, 1, RawIndexes) of false -> {error, {not_found, <<IndexName/binary, " not found.">>}}; @@ -274,15 +324,21 @@ design_doc_to_index(#doc{id=Id,body={Fields}}, IndexName) -> undefined -> {error, InvalidDDocError}; Def -> - Sig = ?l2b(couch_util:to_hex(couch_hash:md5_hash( - term_to_binary({Analyzer, Def})))), + Sig = ?l2b( + couch_util:to_hex( + couch_hash:md5_hash( + term_to_binary({Analyzer, Def}) + ) + ) + ), {ok, #index{ - analyzer=Analyzer, - ddoc_id=Id, - def=Def, - def_lang=Language, - name=IndexName, - sig=Sig}} + analyzer = Analyzer, + ddoc_id = Id, + def = Def, + def_lang = Language, + name = IndexName, + sig = Sig + }} end; _ -> {error, InvalidDDocError} @@ -293,49 +349,51 @@ reply_with_index(IndexPid, Index, WaitList) -> reply_with_index(_IndexPid, _Index, [], Acc) -> Acc; -reply_with_index(IndexPid, #index{current_seq=IndexSeq}=Index, [{Pid, Seq}|Rest], Acc) when Seq =< IndexSeq -> +reply_with_index(IndexPid, #index{current_seq = IndexSeq} = Index, [{Pid, Seq} | Rest], Acc) when + Seq =< IndexSeq +-> gen_server:reply(Pid, {ok, IndexPid, IndexSeq}), reply_with_index(IndexPid, Index, Rest, Acc); -reply_with_index(IndexPid, Index, [{Pid, Seq}|Rest], Acc) -> - reply_with_index(IndexPid, Index, Rest, [{Pid, Seq}|Acc]). +reply_with_index(IndexPid, Index, [{Pid, Seq} | Rest], Acc) -> + reply_with_index(IndexPid, Index, Rest, [{Pid, Seq} | Acc]). -index_name(#index{dbname=DbName,ddoc_id=DDocId,name=IndexName}) -> +index_name(#index{dbname = DbName, ddoc_id = DDocId, name = IndexName}) -> <<DbName/binary, " ", DDocId/binary, " ", IndexName/binary>>. args_to_proplist(#index_query_args{} = Args) -> [ - {'query', Args#index_query_args.q}, - {partition, Args#index_query_args.partition}, - {limit, Args#index_query_args.limit}, - {refresh, Args#index_query_args.stale =:= false}, - {'after', Args#index_query_args.bookmark}, - {sort, Args#index_query_args.sort}, - {include_fields, Args#index_query_args.include_fields}, - {counts, Args#index_query_args.counts}, - {ranges, Args#index_query_args.ranges}, - {drilldown, Args#index_query_args.drilldown}, - {highlight_fields, Args#index_query_args.highlight_fields}, - {highlight_pre_tag, Args#index_query_args.highlight_pre_tag}, - {highlight_post_tag, Args#index_query_args.highlight_post_tag}, - {highlight_number, Args#index_query_args.highlight_number}, - {highlight_size, Args#index_query_args.highlight_size} + {'query', Args#index_query_args.q}, + {partition, Args#index_query_args.partition}, + {limit, Args#index_query_args.limit}, + {refresh, Args#index_query_args.stale =:= false}, + {'after', Args#index_query_args.bookmark}, + {sort, Args#index_query_args.sort}, + {include_fields, Args#index_query_args.include_fields}, + {counts, Args#index_query_args.counts}, + {ranges, Args#index_query_args.ranges}, + {drilldown, Args#index_query_args.drilldown}, + {highlight_fields, Args#index_query_args.highlight_fields}, + {highlight_pre_tag, Args#index_query_args.highlight_pre_tag}, + {highlight_post_tag, Args#index_query_args.highlight_post_tag}, + {highlight_number, Args#index_query_args.highlight_number}, + {highlight_size, Args#index_query_args.highlight_size} ]. args_to_proplist2(#index_query_args{} = Args) -> [ - {'query', Args#index_query_args.q}, - {field, Args#index_query_args.grouping#grouping.by}, - {refresh, Args#index_query_args.stale =:= false}, - {groups, Args#index_query_args.grouping#grouping.groups}, - {group_sort, Args#index_query_args.grouping#grouping.sort}, - {sort, Args#index_query_args.sort}, - {limit, Args#index_query_args.limit}, - {include_fields, Args#index_query_args.include_fields}, - {highlight_fields, Args#index_query_args.highlight_fields}, - {highlight_pre_tag, Args#index_query_args.highlight_pre_tag}, - {highlight_post_tag, Args#index_query_args.highlight_post_tag}, - {highlight_number, Args#index_query_args.highlight_number}, - {highlight_size, Args#index_query_args.highlight_size} + {'query', Args#index_query_args.q}, + {field, Args#index_query_args.grouping#grouping.by}, + {refresh, Args#index_query_args.stale =:= false}, + {groups, Args#index_query_args.grouping#grouping.groups}, + {group_sort, Args#index_query_args.grouping#grouping.sort}, + {sort, Args#index_query_args.sort}, + {limit, Args#index_query_args.limit}, + {include_fields, Args#index_query_args.include_fields}, + {highlight_fields, Args#index_query_args.highlight_fields}, + {highlight_pre_tag, Args#index_query_args.highlight_pre_tag}, + {highlight_post_tag, Args#index_query_args.highlight_post_tag}, + {highlight_number, Args#index_query_args.highlight_number}, + {highlight_size, Args#index_query_args.highlight_size} ]. search_int(Pid, QueryArgs0) -> @@ -355,8 +413,15 @@ group1_int(Pid, QueryArgs0) -> sort = Sort } } = QueryArgs, - clouseau_rpc:group1(Pid, Query, GroupBy, Stale =:= false, Sort, - Offset, Limit). + clouseau_rpc:group1( + Pid, + Query, + GroupBy, + Stale =:= false, + Sort, + Offset, + Limit + ). group2_int(Pid, QueryArgs0) -> QueryArgs = dreyfus_util:upgrade(QueryArgs0), diff --git a/src/dreyfus/src/dreyfus_index_manager.erl b/src/dreyfus/src/dreyfus_index_manager.erl index 9c6cad2ae..f0dbbec64 100644 --- a/src/dreyfus/src/dreyfus_index_manager.erl +++ b/src/dreyfus/src/dreyfus_index_manager.erl @@ -10,7 +10,6 @@ % License for the specific language governing permissions and limitations under % the License. - %% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- -module(dreyfus_index_manager). @@ -26,8 +25,14 @@ -export([start_link/0, get_index/2, get_disk_size/2]). % gen_server api. --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3]). +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3 +]). -export([handle_db_event/3]). @@ -38,7 +43,7 @@ start_link() -> get_index(DbName, Index) -> gen_server:call(?MODULE, {get_index, DbName, Index}, infinity). -get_disk_size(DbName, #index{sig=Sig}) -> +get_disk_size(DbName, #index{sig = Sig}) -> Path = <<DbName/binary, "/", Sig/binary>>, clouseau_rpc:disk_size(Path). @@ -52,20 +57,19 @@ init([]) -> process_flag(trap_exit, true), {ok, nil}. -handle_call({get_index, DbName, #index{sig=Sig}=Index}, From, State) -> +handle_call({get_index, DbName, #index{sig = Sig} = Index}, From, State) -> case ets:lookup(?BY_SIG, {DbName, Sig}) of - [] -> - Pid = spawn_link(fun() -> new_index(DbName, Index) end), - ets:insert(?BY_PID, {Pid, opening, {DbName, Sig}}), - ets:insert(?BY_SIG, {{DbName,Sig}, [From]}), - {noreply, State}; - [{_, WaitList}] when is_list(WaitList) -> - ets:insert(?BY_SIG, {{DbName, Sig}, [From | WaitList]}), - {noreply, State}; - [{_, ExistingPid}] -> - {reply, {ok, ExistingPid}, State} + [] -> + Pid = spawn_link(fun() -> new_index(DbName, Index) end), + ets:insert(?BY_PID, {Pid, opening, {DbName, Sig}}), + ets:insert(?BY_SIG, {{DbName, Sig}, [From]}), + {noreply, State}; + [{_, WaitList}] when is_list(WaitList) -> + ets:insert(?BY_SIG, {{DbName, Sig}, [From | WaitList]}), + {noreply, State}; + [{_, ExistingPid}] -> + {reply, {ok, ExistingPid}, State} end; - handle_call({open_ok, DbName, Sig, NewPid}, {OpenerPid, _}, State) -> link(NewPid), [{_, WaitList}] = ets:lookup(?BY_SIG, {DbName, Sig}), @@ -73,7 +77,6 @@ handle_call({open_ok, DbName, Sig, NewPid}, {OpenerPid, _}, State) -> ets:delete(?BY_PID, OpenerPid), add_to_ets(NewPid, DbName, Sig), {reply, ok, State}; - handle_call({open_error, DbName, Sig, Error}, {OpenerPid, _}, State) -> [{_, WaitList}] = ets:lookup(?BY_SIG, {DbName, Sig}), [gen_server:reply(From, Error) || From <- WaitList], @@ -84,27 +87,28 @@ handle_call({open_error, DbName, Sig, Error}, {OpenerPid, _}, State) -> handle_cast({cleanup, DbName}, State) -> clouseau_rpc:cleanup(DbName), {noreply, State}; - handle_cast({rename, DbName}, State) -> clouseau_rpc:rename(DbName), {noreply, State}. handle_info({'EXIT', FromPid, Reason}, State) -> case ets:lookup(?BY_PID, FromPid) of - [] -> - if Reason =/= normal -> - couch_log:error("Exit on non-updater process: ~p", [Reason]), - exit(Reason); - true -> ok - end; - % Using Reason /= normal to force a match error - % if we didn't delete the Pid in a handle_call - % message for some reason. - [{_, opening, {DbName, Sig}}] when Reason /= normal -> - Msg = {open_error, DbName, Sig, Reason}, - {reply, ok, _} = handle_call(Msg, {FromPid, nil}, State); - [{_, {DbName, Sig}}] -> - delete_from_ets(FromPid, DbName, Sig) + [] -> + if + Reason =/= normal -> + couch_log:error("Exit on non-updater process: ~p", [Reason]), + exit(Reason); + true -> + ok + end; + % Using Reason /= normal to force a match error + % if we didn't delete the Pid in a handle_call + % message for some reason. + [{_, opening, {DbName, Sig}}] when Reason /= normal -> + Msg = {open_error, DbName, Sig, Reason}, + {reply, ok, _} = handle_call(Msg, {FromPid, nil}, State); + [{_, {DbName, Sig}}] -> + delete_from_ets(FromPid, DbName, Sig) end, {noreply, State}. @@ -120,8 +124,11 @@ handle_db_event(DbName, created, _St) -> gen_server:cast(?MODULE, {cleanup, DbName}), {ok, nil}; handle_db_event(DbName, deleted, _St) -> - RecoveryEnabled = config:get_boolean("couchdb", - "enable_database_recovery", false), + RecoveryEnabled = config:get_boolean( + "couchdb", + "enable_database_recovery", + false + ), case RecoveryEnabled of true -> gen_server:cast(?MODULE, {rename, DbName}); @@ -133,15 +140,15 @@ handle_db_event(DbName, deleted, _St) -> handle_db_event(_DbName, _Event, _St) -> {ok, nil}. -new_index(DbName, #index{sig=Sig}=Index) -> +new_index(DbName, #index{sig = Sig} = Index) -> case (catch dreyfus_index:start_link(DbName, Index)) of - {ok, NewPid} -> - Msg = {open_ok, DbName, Sig, NewPid}, - ok = gen_server:call(?MODULE, Msg, infinity), - unlink(NewPid); - Error -> - Msg = {open_error, DbName, Sig, Error}, - ok = gen_server:call(?MODULE, Msg, infinity) + {ok, NewPid} -> + Msg = {open_ok, DbName, Sig, NewPid}, + ok = gen_server:call(?MODULE, Msg, infinity), + unlink(NewPid); + Error -> + Msg = {open_error, DbName, Sig, Error}, + ok = gen_server:call(?MODULE, Msg, infinity) end. add_to_ets(Pid, DbName, Sig) -> @@ -151,4 +158,3 @@ add_to_ets(Pid, DbName, Sig) -> delete_from_ets(Pid, DbName, Sig) -> true = ets:delete(?BY_PID, Pid), true = ets:delete(?BY_SIG, {DbName, Sig}). - diff --git a/src/dreyfus/src/dreyfus_index_updater.erl b/src/dreyfus/src/dreyfus_index_updater.erl index 87edef0ad..6edc5a257 100644 --- a/src/dreyfus/src/dreyfus_index_updater.erl +++ b/src/dreyfus/src/dreyfus_index_updater.erl @@ -10,7 +10,6 @@ % License for the specific language governing permissions and limitations under % the License. - %% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- -module(dreyfus_index_updater). @@ -70,10 +69,10 @@ update(IndexPid, Index) -> couch_db:close(Db) end. -load_docs(FDI, {I, IndexPid, Db, Proc, Total, LastCommitTime, ExcludeIdRevs}=Acc) -> +load_docs(FDI, {I, IndexPid, Db, Proc, Total, LastCommitTime, ExcludeIdRevs} = Acc) -> couch_task_status:update([{changes_done, I}, {progress, (I * 100) div Total}]), DI = couch_doc:to_doc_info(FDI), - #doc_info{id=Id, high_seq=Seq, revs=[#rev_info{rev=Rev}|_]} = DI, + #doc_info{id = Id, high_seq = Seq, revs = [#rev_info{rev = Rev} | _]} = DI, %check if it is processed in purge_index to avoid update the index again. case lists:member({Id, Rev}, ExcludeIdRevs) of true -> ok; @@ -83,9 +82,9 @@ load_docs(FDI, {I, IndexPid, Db, Proc, Total, LastCommitTime, ExcludeIdRevs}=Acc case timer:now_diff(Now = erlang:timestamp(), LastCommitTime) >= 60000000 of true -> ok = clouseau_rpc:commit(IndexPid, Seq), - {ok, {I+1, IndexPid, Db, Proc, Total, Now, ExcludeIdRevs}}; + {ok, {I + 1, IndexPid, Db, Proc, Total, Now, ExcludeIdRevs}}; false -> - {ok, setelement(1, Acc, I+1)} + {ok, setelement(1, Acc, I + 1)} end. purge_index(Db, IndexPid, Index) -> @@ -94,26 +93,29 @@ purge_index(Db, IndexPid, Index) -> try true = proc_prompt(Proc, [<<"add_fun">>, Index#index.def]), FoldFun = fun({PurgeSeq, _UUID, Id, _Revs}, {Acc, _}) -> - Acc0 = case couch_db:get_full_doc_info(Db, Id) of - not_found -> - ok = clouseau_rpc:delete(IndexPid, Id), - Acc; - FDI -> - DI = couch_doc:to_doc_info(FDI), - #doc_info{id=Id, revs=[#rev_info{rev=Rev}|_]} = DI, - case lists:member({Id, Rev}, Acc) of - true -> Acc; - false -> - update_or_delete_index(IndexPid, Db, DI, Proc), - [{Id, Rev} | Acc] - end - end, + Acc0 = + case couch_db:get_full_doc_info(Db, Id) of + not_found -> + ok = clouseau_rpc:delete(IndexPid, Id), + Acc; + FDI -> + DI = couch_doc:to_doc_info(FDI), + #doc_info{id = Id, revs = [#rev_info{rev = Rev} | _]} = DI, + case lists:member({Id, Rev}, Acc) of + true -> + Acc; + false -> + update_or_delete_index(IndexPid, Db, DI, Proc), + [{Id, Rev} | Acc] + end + end, update_task(1), {ok, {Acc0, PurgeSeq}} end, {ok, {ExcludeList, NewPurgeSeq}} = couch_db:fold_purge_infos( - Db, IdxPurgeSeq, FoldFun, {[], 0}, []), + Db, IdxPurgeSeq, FoldFun, {[], 0}, [] + ), clouseau_rpc:set_purge_seq(IndexPid, NewPurgeSeq), update_local_doc(Db, Index, NewPurgeSeq), {ok, ExcludeList} @@ -127,7 +129,7 @@ count_pending_purged_docs_since(Db, IndexPid) -> DbPurgeSeq - IdxPurgeSeq. update_or_delete_index(IndexPid, Db, DI, Proc) -> - #doc_info{id=Id, revs=[#rev_info{deleted=Del}|_]} = DI, + #doc_info{id = Id, revs = [#rev_info{deleted = Del} | _]} = DI, case Del of true -> ok = clouseau_rpc:delete(IndexPid, Id); @@ -138,12 +140,12 @@ update_or_delete_index(IndexPid, Db, DI, Proc) -> false -> {ok, Doc} = couch_db:open_doc(Db, DI, []), Json = couch_doc:to_json_obj(Doc, []), - [Fields|_] = proc_prompt(Proc, [<<"index_doc">>, Json]), + [Fields | _] = proc_prompt(Proc, [<<"index_doc">>, Json]), Fields1 = [list_to_tuple(Field) || Field <- Fields], Fields2 = maybe_add_partition(Db, Id, Fields1), case Fields2 of [] -> ok = clouseau_rpc:delete(IndexPid, Id); - _ -> ok = clouseau_rpc:update(IndexPid, Id, Fields2) + _ -> ok = clouseau_rpc:update(IndexPid, Id, Fields2) end end end. @@ -156,12 +158,13 @@ update_local_doc(Db, Index, PurgeSeq) -> update_task(NumChanges) -> [Changes, Total] = couch_task_status:get([changes_done, total_changes]), Changes2 = Changes + NumChanges, - Progress = case Total of - 0 -> - 0; - _ -> - (Changes2 * 100) div Total - end, + Progress = + case Total of + 0 -> + 0; + _ -> + (Changes2 * 100) div Total + end, couch_task_status:update([{progress, Progress}, {changes_done, Changes2}]). maybe_skip_doc(Db, <<"_design/", _/binary>>) -> diff --git a/src/dreyfus/src/dreyfus_plugin_couch_db.erl b/src/dreyfus/src/dreyfus_plugin_couch_db.erl index b9f48ba74..a55c26373 100644 --- a/src/dreyfus/src/dreyfus_plugin_couch_db.erl +++ b/src/dreyfus/src/dreyfus_plugin_couch_db.erl @@ -17,10 +17,8 @@ on_compact/2 ]). - is_valid_purge_client(DbName, Props) -> dreyfus_util:verify_index_exists(DbName, Props). - on_compact(DbName, DDocs) -> dreyfus_util:ensure_local_purge_docs(DbName, DDocs). diff --git a/src/dreyfus/src/dreyfus_rpc.erl b/src/dreyfus/src/dreyfus_rpc.erl index cc50d0999..08b719435 100644 --- a/src/dreyfus/src/dreyfus_rpc.erl +++ b/src/dreyfus/src/dreyfus_rpc.erl @@ -10,7 +10,6 @@ % License for the specific language governing permissions and limitations under % the License. - %% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- -module(dreyfus_rpc). @@ -110,7 +109,6 @@ get_or_create_db(DbName, Options) -> Else end. - calculate_seqs(Db, Stale) -> LastSeq = couch_db:get_update_seq(Db), if diff --git a/src/dreyfus/src/dreyfus_sup.erl b/src/dreyfus/src/dreyfus_sup.erl index d855a822e..e19203af2 100644 --- a/src/dreyfus/src/dreyfus_sup.erl +++ b/src/dreyfus/src/dreyfus_sup.erl @@ -10,7 +10,6 @@ % License for the specific language governing permissions and limitations under % the License. - %% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- -module(dreyfus_sup). @@ -25,8 +24,7 @@ init(_Args) -> Children = [ child(dreyfus_index_manager) ], - {ok, {{one_for_one,10,1}, - couch_epi:register_service(dreyfus_epi, Children)}}. + {ok, {{one_for_one, 10, 1}, couch_epi:register_service(dreyfus_epi, Children)}}. child(Child) -> {Child, {Child, start_link, []}, permanent, 1000, worker, [Child]}. diff --git a/src/dreyfus/src/dreyfus_util.erl b/src/dreyfus/src/dreyfus_util.erl index 05ecdb621..301d3887a 100644 --- a/src/dreyfus/src/dreyfus_util.erl +++ b/src/dreyfus/src/dreyfus_util.erl @@ -10,7 +10,6 @@ % License for the specific language governing permissions and limitations under % the License. - %% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- -module(dreyfus_util). @@ -33,7 +32,6 @@ verify_index_exists/2 ]). - get_shards(DbName, #index_query_args{partition = nil} = Args) -> case use_ushards(Args) of true -> @@ -52,20 +50,22 @@ get_shards(DbName, #index_query_args{partition = Partition} = Args) -> get_shards(DbName, Args) -> get_shards(DbName, upgrade(Args)). -use_ushards(#index_query_args{stale=ok}) -> +use_ushards(#index_query_args{stale = ok}) -> true; -use_ushards(#index_query_args{stable=true}) -> +use_ushards(#index_query_args{stable = true}) -> true; use_ushards(#index_query_args{}) -> false. - get_ring_opts(#index_query_args{partition = nil}, _Shards) -> []; get_ring_opts(#index_query_args{}, Shards) -> - Shards1 = lists:map(fun(#shard{} = S) -> - S#shard{ref = undefined} - end, Shards), + Shards1 = lists:map( + fun(#shard{} = S) -> + S#shard{ref = undefined} + end, + Shards + ), [{any, Shards1}]. -spec sort(Order :: relevance | [any()], [#sortable{}]) -> [#sortable{}]. @@ -79,70 +79,80 @@ stash_items(List) -> stash_item(Item) -> Ref = make_ref(), - {Item#sortable{item=Ref}, {Ref, Item#sortable.item}}. + {Item#sortable{item = Ref}, {Ref, Item#sortable.item}}. unstash_items(List, Stash) -> [unstash_item(Item, Stash) || Item <- List]. unstash_item(Stashed, Stash) -> {_, Item} = lists:keyfind(Stashed#sortable.item, 1, Stash), - Stashed#sortable{item=Item}. + Stashed#sortable{item = Item}. -spec sort(Order :: relevance | [any()], #sortable{}, #sortable{}) -> boolean(). -sort(relevance, #sortable{}=A, #sortable{}=B) -> +sort(relevance, #sortable{} = A, #sortable{} = B) -> sort2(pad([<<"-">>], <<"">>, length(A#sortable.order)), A, B); -sort(Sort, #sortable{}=A, #sortable{}=B) when is_binary(Sort) -> +sort(Sort, #sortable{} = A, #sortable{} = B) when is_binary(Sort) -> sort2(pad([Sort], <<"">>, length(A#sortable.order)), A, B); -sort(Sort, #sortable{}=A, #sortable{}=B) when is_list(Sort) -> +sort(Sort, #sortable{} = A, #sortable{} = B) when is_list(Sort) -> sort2(pad(Sort, <<"">>, length(A#sortable.order)), A, B). -spec sort2([any()], #sortable{}, #sortable{}) -> boolean(). -sort2([<<"-",_/binary>>|_], #sortable{order=[A|_]}, #sortable{order=[B|_]}) when A =/= B -> +sort2([<<"-", _/binary>> | _], #sortable{order = [A | _]}, #sortable{order = [B | _]}) when + A =/= B +-> A > B; -sort2([_|_], #sortable{order=[A|_]}, #sortable{order=[B|_]}) when A =/= B -> +sort2([_ | _], #sortable{order = [A | _]}, #sortable{order = [B | _]}) when A =/= B -> A < B; -sort2([], #sortable{shard=#shard{range=A}}, #sortable{shard=#shard{range=B}}) -> +sort2([], #sortable{shard = #shard{range = A}}, #sortable{shard = #shard{range = B}}) -> % arbitrary tie-breaker A =< B; -sort2([_|Rest], #sortable{order=[_|RestA]}=SortableA, #sortable{order=[_|RestB]}=SortableB) -> - sort2(Rest, SortableA#sortable{order=RestA}, SortableB#sortable{order=RestB}). +sort2( + [_ | Rest], + #sortable{order = [_ | RestA]} = SortableA, + #sortable{order = [_ | RestB]} = SortableB +) -> + sort2(Rest, SortableA#sortable{order = RestA}, SortableB#sortable{order = RestB}). pad(List, _Padding, Length) when length(List) >= Length -> List; pad(List, Padding, Length) -> pad(List ++ [Padding], Padding, Length). -upgrade(#index_query_args{}=Args) -> +upgrade(#index_query_args{} = Args) -> Args; -upgrade({index_query_args, Query, Limit, Stale, IncludeDocs, Bookmark, - Sort, Grouping, Stable}) -> +upgrade({index_query_args, Query, Limit, Stale, IncludeDocs, Bookmark, Sort, Grouping, Stable}) -> #index_query_args{ - q = Query, - limit = Limit, - stale = Stale, - include_docs = IncludeDocs, - bookmark = Bookmark, - sort = Sort, - grouping = Grouping, - stable = Stable}; -upgrade({index_query_args, Query, Limit, Stale, IncludeDocs, Bookmark, - Sort, Grouping, Stable, Counts, Ranges, Drilldown}) -> + q = Query, + limit = Limit, + stale = Stale, + include_docs = IncludeDocs, + bookmark = Bookmark, + sort = Sort, + grouping = Grouping, + stable = Stable + }; +upgrade( + {index_query_args, Query, Limit, Stale, IncludeDocs, Bookmark, Sort, Grouping, Stable, Counts, + Ranges, Drilldown} +) -> #index_query_args{ - q = Query, - limit = Limit, - stale = Stale, - include_docs = IncludeDocs, - bookmark = Bookmark, - sort = Sort, - grouping = Grouping, - stable = Stable, - counts=Counts, - ranges = Ranges, - drilldown = Drilldown}; -upgrade({index_query_args, Query, Limit, Stale, IncludeDocs, Bookmark, - Sort, Grouping, Stable, Counts, Ranges, Drilldown, - IncludeFields, HighlightFields, HighlightPreTag, HighlightPostTag, - HighlightNumber, HighlightSize}) -> + q = Query, + limit = Limit, + stale = Stale, + include_docs = IncludeDocs, + bookmark = Bookmark, + sort = Sort, + grouping = Grouping, + stable = Stable, + counts = Counts, + ranges = Ranges, + drilldown = Drilldown + }; +upgrade( + {index_query_args, Query, Limit, Stale, IncludeDocs, Bookmark, Sort, Grouping, Stable, Counts, + Ranges, Drilldown, IncludeFields, HighlightFields, HighlightPreTag, HighlightPostTag, + HighlightNumber, HighlightSize} +) -> #index_query_args{ q = Query, limit = Limit, @@ -150,7 +160,7 @@ upgrade({index_query_args, Query, Limit, Stale, IncludeDocs, Bookmark, include_docs = IncludeDocs, bookmark = Bookmark, sort = Sort, - grouping = Grouping, + grouping = Grouping, stable = Stable, counts = Counts, ranges = Ranges, @@ -162,10 +172,11 @@ upgrade({index_query_args, Query, Limit, Stale, IncludeDocs, Bookmark, highlight_number = HighlightNumber, highlight_size = HighlightSize }; -upgrade({index_query_args, Query, Limit, Stale, IncludeDocs, Bookmark, - Sort, Grouping, Stable, Counts, Ranges, Drilldown, - IncludeFields, HighlightFields, HighlightPreTag, HighlightPostTag, - HighlightNumber, HighlightSize, RawBookmark}) -> +upgrade( + {index_query_args, Query, Limit, Stale, IncludeDocs, Bookmark, Sort, Grouping, Stable, Counts, + Ranges, Drilldown, IncludeFields, HighlightFields, HighlightPreTag, HighlightPostTag, + HighlightNumber, HighlightSize, RawBookmark} +) -> #index_query_args{ q = Query, limit = Limit, @@ -173,7 +184,7 @@ upgrade({index_query_args, Query, Limit, Stale, IncludeDocs, Bookmark, include_docs = IncludeDocs, bookmark = Bookmark, sort = Sort, - grouping = Grouping, + grouping = Grouping, stable = Stable, counts = Counts, ranges = Ranges, @@ -187,56 +198,43 @@ upgrade({index_query_args, Query, Limit, Stale, IncludeDocs, Bookmark, raw_bookmark = RawBookmark }. -export(#index_query_args{partition = nil, counts = nil, ranges = nil, - drilldown = [], include_fields = nil, highlight_fields = nil} = Args) -> +export( + #index_query_args{ + partition = nil, + counts = nil, + ranges = nil, + drilldown = [], + include_fields = nil, + highlight_fields = nil + } = Args +) -> % Ensure existing searches work during the upgrade by creating an % #index_query_args record in the old format - {index_query_args, - Args#index_query_args.q, - Args#index_query_args.limit, - Args#index_query_args.stale, - Args#index_query_args.include_docs, - Args#index_query_args.bookmark, - Args#index_query_args.sort, - Args#index_query_args.grouping, - Args#index_query_args.stable - }; -export(#index_query_args{partition = nil, include_fields = nil, - highlight_fields = nil} = Args) -> - {index_query_args, - Args#index_query_args.q, - Args#index_query_args.limit, - Args#index_query_args.stale, - Args#index_query_args.include_docs, - Args#index_query_args.bookmark, - Args#index_query_args.sort, - Args#index_query_args.grouping, - Args#index_query_args.stable, - Args#index_query_args.counts, - Args#index_query_args.ranges, - Args#index_query_args.drilldown - }; + {index_query_args, Args#index_query_args.q, Args#index_query_args.limit, + Args#index_query_args.stale, Args#index_query_args.include_docs, + Args#index_query_args.bookmark, Args#index_query_args.sort, Args#index_query_args.grouping, + Args#index_query_args.stable}; +export( + #index_query_args{ + partition = nil, + include_fields = nil, + highlight_fields = nil + } = Args +) -> + {index_query_args, Args#index_query_args.q, Args#index_query_args.limit, + Args#index_query_args.stale, Args#index_query_args.include_docs, + Args#index_query_args.bookmark, Args#index_query_args.sort, Args#index_query_args.grouping, + Args#index_query_args.stable, Args#index_query_args.counts, Args#index_query_args.ranges, + Args#index_query_args.drilldown}; export(#index_query_args{partition = nil} = Args) -> - {index_query_args, - Args#index_query_args.q, - Args#index_query_args.limit, - Args#index_query_args.stale, - Args#index_query_args.include_docs, - Args#index_query_args.bookmark, - Args#index_query_args.sort, - Args#index_query_args.grouping, - Args#index_query_args.stable, - Args#index_query_args.counts, - Args#index_query_args.ranges, - Args#index_query_args.drilldown, - Args#index_query_args.include_fields, - Args#index_query_args.highlight_fields, - Args#index_query_args.highlight_pre_tag, - Args#index_query_args.highlight_post_tag, - Args#index_query_args.highlight_number, - Args#index_query_args.highlight_size, - Args#index_query_args.raw_bookmark - }; + {index_query_args, Args#index_query_args.q, Args#index_query_args.limit, + Args#index_query_args.stale, Args#index_query_args.include_docs, + Args#index_query_args.bookmark, Args#index_query_args.sort, Args#index_query_args.grouping, + Args#index_query_args.stable, Args#index_query_args.counts, Args#index_query_args.ranges, + Args#index_query_args.drilldown, Args#index_query_args.include_fields, + Args#index_query_args.highlight_fields, Args#index_query_args.highlight_pre_tag, + Args#index_query_args.highlight_post_tag, Args#index_query_args.highlight_number, + Args#index_query_args.highlight_size, Args#index_query_args.raw_bookmark}; export(QueryArgs) -> QueryArgs. @@ -246,14 +244,20 @@ time(Metric, {M, F, A}) when is_list(Metric) -> erlang:apply(M, F, A) after Length = timer:now_diff(os:timestamp(), Start) / 1000, - couch_stats:update_histogram([dreyfus | Metric], Length) + couch_stats:update_histogram([dreyfus | Metric], Length) end. -in_black_list(DbName, GroupId, IndexName) when is_binary(DbName), - is_binary(GroupId), is_binary(IndexName) -> +in_black_list(DbName, GroupId, IndexName) when + is_binary(DbName), + is_binary(GroupId), + is_binary(IndexName) +-> in_black_list(?b2l(DbName), ?b2l(GroupId), ?b2l(IndexName)); -in_black_list(DbName, GroupId, IndexName) when is_list(DbName), - is_list(GroupId), is_list(IndexName) -> +in_black_list(DbName, GroupId, IndexName) when + is_list(DbName), + is_list(GroupId), + is_list(IndexName) +-> in_black_list(lists:flatten([DbName, ".", GroupId, ".", IndexName])); in_black_list(_DbName, _GroupId, _IndexName) -> false. @@ -269,9 +273,13 @@ in_black_list(_IndexEntry) -> maybe_deny_index(DbName, GroupId, IndexName) -> case in_black_list(DbName, GroupId, IndexName) of true -> - Reason = ?l2b(io_lib:format("Index <~s, ~s, ~s>, is BlackListed", - [?b2l(DbName), ?b2l(GroupId), ?b2l(IndexName)])), - throw ({bad_request, Reason}); + Reason = ?l2b( + io_lib:format( + "Index <~s, ~s, ~s>, is BlackListed", + [?b2l(DbName), ?b2l(GroupId), ?b2l(IndexName)] + ) + ), + throw({bad_request, Reason}); _ -> ok end. @@ -284,31 +292,43 @@ get_value_from_options(Key, Options) -> undefined -> Reason = binary_to_list(Key) ++ " must exist in Options.", throw({bad_request, Reason}); - Value -> Value + Value -> + Value end. ensure_local_purge_docs(DbName, DDocs) -> couch_util:with_db(DbName, fun(Db) -> - lists:foreach(fun(DDoc) -> - #doc{body = {Props}} = DDoc, - case couch_util:get_value(<<"indexes">>, Props) of - undefined -> false; - _ -> - try dreyfus_index:design_doc_to_indexes(DDoc) of - SIndexes -> ensure_local_purge_doc(Db, SIndexes) - catch _:_ -> - ok - end - end - end, DDocs) + lists:foreach( + fun(DDoc) -> + #doc{body = {Props}} = DDoc, + case couch_util:get_value(<<"indexes">>, Props) of + undefined -> + false; + _ -> + try dreyfus_index:design_doc_to_indexes(DDoc) of + SIndexes -> ensure_local_purge_doc(Db, SIndexes) + catch + _:_ -> + ok + end + end + end, + DDocs + ) end). ensure_local_purge_doc(Db, SIndexes) -> - if SIndexes =/= [] -> - lists:map(fun(SIndex) -> - maybe_create_local_purge_doc(Db, SIndex) - end, SIndexes); - true -> ok end. + if + SIndexes =/= [] -> + lists:map( + fun(SIndex) -> + maybe_create_local_purge_doc(Db, SIndex) + end, + SIndexes + ); + true -> + ok + end. maybe_create_local_purge_doc(Db, Index) -> DocId = dreyfus_util:get_local_purge_doc_id(Index#index.sig), @@ -316,7 +336,8 @@ maybe_create_local_purge_doc(Db, Index) -> {not_found, _} -> DbPurgeSeq = couch_db:get_purge_seq(Db), DocContent = dreyfus_util:get_local_purge_doc_body( - Db, DocId, DbPurgeSeq, Index), + Db, DocId, DbPurgeSeq, Index + ), couch_db:update_doc(Db, DocContent, []); _ -> ok @@ -329,7 +350,8 @@ maybe_create_local_purge_doc(Db, IndexPid, Index) -> DbPurgeSeq = couch_db:get_purge_seq(Db), clouseau_rpc:set_purge_seq(IndexPid, DbPurgeSeq), DocContent = dreyfus_util:get_local_purge_doc_body( - Db, DocId, DbPurgeSeq, Index), + Db, DocId, DbPurgeSeq, Index + ), couch_db:update_doc(Db, DocContent, []); _ -> ok @@ -341,12 +363,20 @@ get_local_purge_doc_id(Sig) -> get_signature_from_idxdir(IdxDir) -> IdxDirList = filename:split(IdxDir), Sig = lists:last(IdxDirList), - Sig2 = if not is_binary(Sig) -> Sig; true -> - binary_to_list(Sig) - end, - case [Ch || Ch <- Sig2, not (((Ch >= $0) and (Ch =< $9)) - orelse ((Ch >= $a) and (Ch =< $f)) - orelse ((Ch >= $A) and (Ch =< $F)))] == [] of + Sig2 = + if + not is_binary(Sig) -> Sig; + true -> binary_to_list(Sig) + end, + case + [ + Ch + || Ch <- Sig2, + not (((Ch >= $0) and (Ch =< $9)) orelse + ((Ch >= $a) and (Ch =< $f)) orelse + ((Ch >= $A) and (Ch =< $F))) + ] == [] + of true -> Sig; false -> undefined end. @@ -359,37 +389,43 @@ get_local_purge_doc_body(_, LocalDocId, PurgeSeq, Index) -> } = Index, {Mega, Secs, _} = os:timestamp(), NowSecs = Mega * 1000000 + Secs, - JsonList = {[ - {<<"_id">>, LocalDocId}, - {<<"purge_seq">>, PurgeSeq}, - {<<"updated_on">>, NowSecs}, - {<<"indexname">>, IdxName}, - {<<"ddoc_id">>, DDocId}, - {<<"signature">>, Sig}, - {<<"type">>, <<"dreyfus">>} - ]}, + JsonList = + {[ + {<<"_id">>, LocalDocId}, + {<<"purge_seq">>, PurgeSeq}, + {<<"updated_on">>, NowSecs}, + {<<"indexname">>, IdxName}, + {<<"ddoc_id">>, DDocId}, + {<<"signature">>, Sig}, + {<<"type">>, <<"dreyfus">>} + ]}, couch_doc:from_json_obj(JsonList). verify_index_exists(DbName, Props) -> try Type = couch_util:get_value(<<"type">>, Props), - if Type =/= <<"dreyfus">> -> false; true -> - DDocId = couch_util:get_value(<<"ddoc_id">>, Props), - IndexName = couch_util:get_value(<<"indexname">>, Props), - Sig = couch_util:get_value(<<"signature">>, Props), - couch_util:with_db(DbName, fun(Db) -> - case couch_db:get_design_doc(Db, DDocId) of - {ok, #doc{} = DDoc} -> - {ok, IdxState} = dreyfus_index:design_doc_to_index( - DDoc, IndexName), - IdxState#index.sig == Sig; - {not_found, _} -> - false - end - end) + if + Type =/= <<"dreyfus">> -> + false; + true -> + DDocId = couch_util:get_value(<<"ddoc_id">>, Props), + IndexName = couch_util:get_value(<<"indexname">>, Props), + Sig = couch_util:get_value(<<"signature">>, Props), + couch_util:with_db(DbName, fun(Db) -> + case couch_db:get_design_doc(Db, DDocId) of + {ok, #doc{} = DDoc} -> + {ok, IdxState} = dreyfus_index:design_doc_to_index( + DDoc, IndexName + ), + IdxState#index.sig == Sig; + {not_found, _} -> + false + end + end) end - catch _:_ -> - false + catch + _:_ -> + false end. -ifdef(TEST). @@ -405,29 +441,36 @@ empty_test() -> ?assertEqual([], ?SORT([], [])). primary_asc_test() -> - ?assertMatch([#sortable{order=[1]}, #sortable{order=[2]}], - ?SORT([?ASC], [#sortable{order=[2]}, #sortable{order=[1]}])). + ?assertMatch( + [#sortable{order = [1]}, #sortable{order = [2]}], + ?SORT([?ASC], [#sortable{order = [2]}, #sortable{order = [1]}]) + ). primary_desc_test() -> - ?assertMatch([#sortable{order=[2]}, #sortable{order=[1]}], - ?SORT([?DESC], [#sortable{order=[1]}, #sortable{order=[2]}])). + ?assertMatch( + [#sortable{order = [2]}, #sortable{order = [1]}], + ?SORT([?DESC], [#sortable{order = [1]}, #sortable{order = [2]}]) + ). secondary_asc_test() -> - ?assertMatch([#sortable{order=[1, 1]}, #sortable{order=[1, 2]}], - ?SORT([?ASC, ?ASC], [#sortable{order=[1, 2]}, #sortable{order=[1, 1]}])). + ?assertMatch( + [#sortable{order = [1, 1]}, #sortable{order = [1, 2]}], + ?SORT([?ASC, ?ASC], [#sortable{order = [1, 2]}, #sortable{order = [1, 1]}]) + ). secondary_desc_test() -> - ?assertMatch([#sortable{order=[1, 2]}, #sortable{order=[1, 1]}], - ?SORT([?DESC, ?DESC], [#sortable{order=[1, 1]}, #sortable{order=[1, 2]}])). + ?assertMatch( + [#sortable{order = [1, 2]}, #sortable{order = [1, 1]}], + ?SORT([?DESC, ?DESC], [#sortable{order = [1, 1]}, #sortable{order = [1, 2]}]) + ). stash_test() -> - {Stashed, Stash} = stash_items([#sortable{order=foo, item=bar}]), + {Stashed, Stash} = stash_items([#sortable{order = foo, item = bar}]), First = hd(Stashed), ?assert(is_reference(First#sortable.item)), Unstashed = hd(unstash_items(Stashed, Stash)), ?assertEqual(Unstashed#sortable.item, bar). - ring_opts_test() -> Shards = [#shard{name = foo, ref = make_ref()}], @@ -435,7 +478,9 @@ ring_opts_test() -> ?assertEqual([], get_ring_opts(QArgs1, Shards)), QArgs2 = #index_query_args{partition = <<"x">>}, - ?assertMatch([{any, [#shard{name = foo, ref = undefined}]}], - get_ring_opts(QArgs2, Shards)). + ?assertMatch( + [{any, [#shard{name = foo, ref = undefined}]}], + get_ring_opts(QArgs2, Shards) + ). -endif. diff --git a/src/dreyfus/test/dreyfus_blacklist_await_test.erl b/src/dreyfus/test/dreyfus_blacklist_await_test.erl index 82665eb02..de3d629e5 100644 --- a/src/dreyfus/test/dreyfus_blacklist_await_test.erl +++ b/src/dreyfus/test/dreyfus_blacklist_await_test.erl @@ -41,10 +41,12 @@ dreyfus_blacklist_await_test_() -> "dreyfus black_list_doc await tests", { setup, - fun start/0, fun stop/1, + fun start/0, + fun stop/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun do_not_await_1/0 ] @@ -54,16 +56,24 @@ dreyfus_blacklist_await_test_() -> do_not_await_1() -> ok = meck:new(dreyfus_index, [passthrough]), - Denied = lists:flatten([?b2l(?DBNAME), ".", "black_list_doc", ".", - "my_index"]), + Denied = lists:flatten([ + ?b2l(?DBNAME), + ".", + "black_list_doc", + ".", + "my_index" + ]), config:set("dreyfus_blacklist", Denied, "true"), dreyfus_test_util:wait_config_change(Denied, "true"), - Index = #index{dbname=?DBNAME, name=?INDEX_NAME, ddoc_id=?DDOC_ID}, + Index = #index{dbname = ?DBNAME, name = ?INDEX_NAME, ddoc_id = ?DDOC_ID}, State = create_state(?DBNAME, Index, nil, nil, []), Msg = "Index Blocked from Updating - db: ~p, ddocid: ~p name: ~p", Return = wait_log_message(Msg, fun() -> - {noreply, _NewState} = dreyfus_index:handle_call({await, 1}, - self(), State) + {noreply, _NewState} = dreyfus_index:handle_call( + {await, 1}, + self(), + State + ) end), ?assertEqual(Return, ok). diff --git a/src/dreyfus/test/dreyfus_blacklist_request_test.erl b/src/dreyfus/test/dreyfus_blacklist_request_test.erl index 8e5598ae1..3e466327d 100644 --- a/src/dreyfus/test/dreyfus_blacklist_request_test.erl +++ b/src/dreyfus/test/dreyfus_blacklist_request_test.erl @@ -41,10 +41,12 @@ dreyfus_blacklist_request_test_() -> "dreyfus blacklist request tests", { setup, - fun start/0, fun stop/1, + fun start/0, + fun stop/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun deny_fabric_requests/0, fun allow_fabric_request/0 @@ -61,29 +63,88 @@ deny_fabric_requests() -> Denied = "mydb.myddocid.myindexname", config:set("dreyfus_blacklist", Denied, "true"), dreyfus_test_util:wait_config_change(Denied, "true"), - ?assertThrow({bad_request, Reason}, dreyfus_fabric_search:go(<<"mydb">>, - <<"myddocid">>, <<"myindexname">>, QueryArgs)), - ?assertThrow({bad_request, Reason}, dreyfus_fabric_group1:go(<<"mydb">>, - <<"myddocid">>, <<"myindexname">>, QueryArgs)), - ?assertThrow({bad_request, Reason}, dreyfus_fabric_group2:go(<<"mydb">>, - <<"myddocid">>, <<"myindexname">>, QueryArgs)), - ?assertThrow({bad_request, Reason}, dreyfus_fabric_info:go(<<"mydb">>, - <<"myddocid">>, <<"myindexname">>, QueryArgs)), - ?assertThrow({bad_request, Reason}, dreyfus_fabric_search:go(<<"mydb">>, - DDoc, <<"myindexname">>, IndexQueryArgs)), - ?assertThrow({bad_request, Reason}, dreyfus_fabric_group1:go(<<"mydb">>, - DDoc, <<"myindexname">>, IndexQueryArgs)), - ?assertThrow({bad_request, Reason}, dreyfus_fabric_group2:go(<<"mydb">>, - DDoc, <<"myindexname">>, IndexQueryArgs)), - ?assertThrow({bad_request, Reason}, dreyfus_fabric_info:go(<<"mydb">>, - DDoc, <<"myindexname">>, IndexQueryArgs)). + ?assertThrow( + {bad_request, Reason}, + dreyfus_fabric_search:go( + <<"mydb">>, + <<"myddocid">>, + <<"myindexname">>, + QueryArgs + ) + ), + ?assertThrow( + {bad_request, Reason}, + dreyfus_fabric_group1:go( + <<"mydb">>, + <<"myddocid">>, + <<"myindexname">>, + QueryArgs + ) + ), + ?assertThrow( + {bad_request, Reason}, + dreyfus_fabric_group2:go( + <<"mydb">>, + <<"myddocid">>, + <<"myindexname">>, + QueryArgs + ) + ), + ?assertThrow( + {bad_request, Reason}, + dreyfus_fabric_info:go( + <<"mydb">>, + <<"myddocid">>, + <<"myindexname">>, + QueryArgs + ) + ), + ?assertThrow( + {bad_request, Reason}, + dreyfus_fabric_search:go( + <<"mydb">>, + DDoc, + <<"myindexname">>, + IndexQueryArgs + ) + ), + ?assertThrow( + {bad_request, Reason}, + dreyfus_fabric_group1:go( + <<"mydb">>, + DDoc, + <<"myindexname">>, + IndexQueryArgs + ) + ), + ?assertThrow( + {bad_request, Reason}, + dreyfus_fabric_group2:go( + <<"mydb">>, + DDoc, + <<"myindexname">>, + IndexQueryArgs + ) + ), + ?assertThrow( + {bad_request, Reason}, + dreyfus_fabric_info:go( + <<"mydb">>, + DDoc, + <<"myindexname">>, + IndexQueryArgs + ) + ). allow_fabric_request() -> ok = meck:new(dreyfus_fabric_search, [passthrough]), - ok = meck:expect(dreyfus_fabric_search, go, - fun(A, GroupId, B, C) when is_binary(GroupId) -> + ok = meck:expect( + dreyfus_fabric_search, + go, + fun(A, GroupId, B, C) when is_binary(GroupId) -> meck:passthrough([A, GroupId, B, C]) - end), + end + ), ok = meck:expect(dreyfus_fabric_search, go, fun(_, _, _, _) -> ok end), @@ -91,6 +152,13 @@ allow_fabric_request() -> QueryArgs = #index_query_args{}, config:set("dreyfus_blacklist", Denied, "true"), dreyfus_test_util:wait_config_change(Denied, "true"), - ?assertEqual(ok, dreyfus_fabric_search:go(<<"mydb">>, - <<"myddocid">>, <<"indexnotthere">>, QueryArgs)), + ?assertEqual( + ok, + dreyfus_fabric_search:go( + <<"mydb">>, + <<"myddocid">>, + <<"indexnotthere">>, + QueryArgs + ) + ), ok = meck:unload(dreyfus_fabric_search). diff --git a/src/dreyfus/test/dreyfus_config_test.erl b/src/dreyfus/test/dreyfus_config_test.erl index 775e49d7f..9ae0e56e0 100644 --- a/src/dreyfus/test/dreyfus_config_test.erl +++ b/src/dreyfus/test/dreyfus_config_test.erl @@ -12,13 +12,11 @@ -module(dreyfus_config_test). - -include_lib("couch_log/include/couch_log.hrl"). -include_lib("eunit/include/eunit.hrl"). -define(TIMEOUT, 1000). - start() -> test_util:start_couch([dreyfus]). @@ -33,10 +31,12 @@ dreyfus_config_test_() -> "dreyfus config tests", { setup, - fun start/0, fun test_util:stop_couch/1, + fun start/0, + fun test_util:stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun check_black_list/0, fun check_delete_from_blacklist/0 @@ -54,9 +54,12 @@ check_black_list() -> ok = config:set("dreyfus_blacklist", Index3, "true"), dreyfus_test_util:wait_config_change(Index3, "true"), FinalBl = [Index3, Index2, Index], - lists:foreach(fun (I) -> - ?assertEqual("true", dreyfus_config:get(I)) - end, FinalBl). + lists:foreach( + fun(I) -> + ?assertEqual("true", dreyfus_config:get(I)) + end, + FinalBl + ). check_delete_from_blacklist() -> Index = "mydb.myddocid.myindexname", diff --git a/src/dreyfus/test/dreyfus_purge_test.erl b/src/dreyfus/test/dreyfus_purge_test.erl index 9b24d6f64..e64a046a7 100644 --- a/src/dreyfus/test/dreyfus_purge_test.erl +++ b/src/dreyfus/test/dreyfus_purge_test.erl @@ -17,19 +17,33 @@ -include_lib("couch/include/couch_eunit.hrl"). -include_lib("mem3/include/mem3.hrl"). - --export([test_purge_single/0, test_purge_multiple/0, test_purge_multiple2/0, - test_purge_conflict/0, test_purge_conflict2/0, test_purge_conflict3/0, test_purge_conflict4/0, - test_purge_update/0, test_purge_update2/0, - test_delete/0, test_delete_purge_conflict/0, test_delete_conflict/0, - test_all/0]). --export([test_verify_index_exists1/0, test_verify_index_exists2/0, test_verify_index_exists_failed/0, - test_local_doc/0, test_delete_local_doc/0, test_purge_search/0]). +-export([ + test_purge_single/0, + test_purge_multiple/0, + test_purge_multiple2/0, + test_purge_conflict/0, + test_purge_conflict2/0, + test_purge_conflict3/0, + test_purge_conflict4/0, + test_purge_update/0, + test_purge_update2/0, + test_delete/0, + test_delete_purge_conflict/0, + test_delete_conflict/0, + test_all/0 +]). +-export([ + test_verify_index_exists1/0, + test_verify_index_exists2/0, + test_verify_index_exists_failed/0, + test_local_doc/0, + test_delete_local_doc/0, + test_purge_search/0 +]). -compile(export_all). -compile(nowarn_export_all). - test_all() -> test_purge_single(), test_purge_multiple(), @@ -75,8 +89,13 @@ test_purge_multiple() -> ?assertEqual(HitCount1, 5), %purge 5 docs - purge_docs(DbName, [<<"apple">>, <<"tomato">>, <<"cherry">>, <<"haw">>, - <<"strawberry">>]), + purge_docs(DbName, [ + <<"apple">>, + <<"tomato">>, + <<"cherry">>, + <<"haw">>, + <<"strawberry">> + ]), %second search request {ok, _, HitCount2, _, _, _} = dreyfus_search(DbName, Query), @@ -130,36 +149,64 @@ test_purge_conflict() -> %first search {ok, _, RedHitCount1, _RedHits1, _, _} = dreyfus_search( - TargetDbName, <<"color:red">>), + TargetDbName, <<"color:red">> + ), {ok, _, GreenHitCount1, _GreenHits1, _, _} = dreyfus_search( - TargetDbName, <<"color:green">>), + TargetDbName, <<"color:green">> + ), ?assertEqual(5, RedHitCount1 + GreenHitCount1), %do replicate and make conflicted docs - {ok, _} = fabric:update_doc(<<"_replicator">>, make_replicate_doc( - SourceDbName, TargetDbName), [?ADMIN_CTX]), + {ok, _} = fabric:update_doc( + <<"_replicator">>, + make_replicate_doc( + SourceDbName, TargetDbName + ), + [?ADMIN_CTX] + ), %%check doc version - wait_for_replicate(TargetDbName, [<<"apple">>, <<"tomato">>, <<"cherry">>, - <<"haw">>, <<"strawberry">>], 2, 5), + wait_for_replicate( + TargetDbName, + [ + <<"apple">>, + <<"tomato">>, + <<"cherry">>, + <<"haw">>, + <<"strawberry">> + ], + 2, + 5 + ), %second search {ok, _, RedHitCount2, _RedHits2, _, _} = dreyfus_search( - TargetDbName, <<"color:red">>), + TargetDbName, <<"color:red">> + ), {ok, _, GreenHitCount2, _GreenHits2, _, _} = dreyfus_search( - TargetDbName, <<"color:green">>), + TargetDbName, <<"color:green">> + ), ?assertEqual(5, RedHitCount2 + GreenHitCount2), - purge_docs(TargetDbName, [<<"apple">>, <<"tomato">>, <<"cherry">>, <<"haw">>, - <<"strawberry">>]), + purge_docs(TargetDbName, [ + <<"apple">>, + <<"tomato">>, + <<"cherry">>, + <<"haw">>, + <<"strawberry">> + ]), %third search - {ok, _, RedHitCount3, _RedHits3, _, _} = dreyfus_search(TargetDbName, - <<"color:red">>), - {ok, _, GreenHitCount3, _GreenHits3, _, _} = dreyfus_search(TargetDbName, - <<"color:green">>), + {ok, _, RedHitCount3, _RedHits3, _, _} = dreyfus_search( + TargetDbName, + <<"color:red">> + ), + {ok, _, GreenHitCount3, _GreenHits3, _, _} = dreyfus_search( + TargetDbName, + <<"color:green">> + ), ?assertEqual(5, RedHitCount3 + GreenHitCount3), ?assertEqual(RedHitCount2, GreenHitCount3), @@ -179,38 +226,71 @@ test_purge_conflict2() -> create_db_docs(TargetDbName, <<"green">>), %first search - {ok, _, RedHitCount1, _RedHits1, _, _} = dreyfus_search(TargetDbName, - <<"color:red">>), - {ok, _, GreenHitCount1, _GreenHits1, _, _} = dreyfus_search(TargetDbName, - <<"color:green">>), + {ok, _, RedHitCount1, _RedHits1, _, _} = dreyfus_search( + TargetDbName, + <<"color:red">> + ), + {ok, _, GreenHitCount1, _GreenHits1, _, _} = dreyfus_search( + TargetDbName, + <<"color:green">> + ), ?assertEqual(5, RedHitCount1 + GreenHitCount1), %do replicate and make conflicted docs - {ok, _} = fabric:update_doc(<<"_replicator">>, make_replicate_doc( - SourceDbName, TargetDbName), [?ADMIN_CTX]), + {ok, _} = fabric:update_doc( + <<"_replicator">>, + make_replicate_doc( + SourceDbName, TargetDbName + ), + [?ADMIN_CTX] + ), - wait_for_replicate(TargetDbName, [<<"apple">>, <<"tomato">>, <<"cherry">>, - <<"haw">>, <<"strawberry">>], 2, 5), + wait_for_replicate( + TargetDbName, + [ + <<"apple">>, + <<"tomato">>, + <<"cherry">>, + <<"haw">>, + <<"strawberry">> + ], + 2, + 5 + ), %second search {ok, _, RedHitCount2, _RedHits2, _, _} = dreyfus_search( - TargetDbName, <<"color:red">>), + TargetDbName, <<"color:red">> + ), {ok, _, GreenHitCount2, _GreenHits2, _, _} = dreyfus_search( - TargetDbName, <<"color:green">>), + TargetDbName, <<"color:green">> + ), ?assertEqual(5, RedHitCount2 + GreenHitCount2), - purge_docs(TargetDbName, [<<"apple">>, <<"tomato">>, <<"cherry">>, - <<"haw">>, <<"strawberry">>]), - purge_docs(TargetDbName, [<<"apple">>, <<"tomato">>, <<"cherry">>, - <<"haw">>, <<"strawberry">>]), + purge_docs(TargetDbName, [ + <<"apple">>, + <<"tomato">>, + <<"cherry">>, + <<"haw">>, + <<"strawberry">> + ]), + purge_docs(TargetDbName, [ + <<"apple">>, + <<"tomato">>, + <<"cherry">>, + <<"haw">>, + <<"strawberry">> + ]), %third search {ok, _, RedHitCount3, _RedHits3, _, _} = dreyfus_search( - TargetDbName, <<"color:red">>), + TargetDbName, <<"color:red">> + ), {ok, _, GreenHitCount3, _GreenHits3, _, _} = dreyfus_search( - TargetDbName, <<"color:green">>), + TargetDbName, <<"color:green">> + ), ?assertEqual(0, RedHitCount3 + GreenHitCount3), @@ -218,7 +298,6 @@ test_purge_conflict2() -> delete_db(TargetDbName), ok. - test_purge_conflict3() -> %create dbs and docs SourceDbName = db_name(), @@ -230,47 +309,80 @@ test_purge_conflict3() -> %first search {ok, _, RedHitCount1, _RedHits1, _, _} = dreyfus_search( - TargetDbName, <<"color:red">>), + TargetDbName, <<"color:red">> + ), {ok, _, GreenHitCount1, _GreenHits1, _, _} = dreyfus_search( - TargetDbName, <<"color:green">>), + TargetDbName, <<"color:green">> + ), ?assertEqual(5, RedHitCount1 + GreenHitCount1), %do replicate and make conflicted docs - {ok, _} = fabric:update_doc(<<"_replicator">>, make_replicate_doc( - SourceDbName, TargetDbName), [?ADMIN_CTX]), + {ok, _} = fabric:update_doc( + <<"_replicator">>, + make_replicate_doc( + SourceDbName, TargetDbName + ), + [?ADMIN_CTX] + ), %%check doc version - wait_for_replicate(TargetDbName, [<<"apple">>, <<"tomato">>, <<"cherry">>, - <<"haw">>, <<"strawberry">>], 2, 5), + wait_for_replicate( + TargetDbName, + [ + <<"apple">>, + <<"tomato">>, + <<"cherry">>, + <<"haw">>, + <<"strawberry">> + ], + 2, + 5 + ), %second search {ok, _, RedHitCount2, _RedHits2, _, _} = dreyfus_search( - TargetDbName, <<"color:red">>), + TargetDbName, <<"color:red">> + ), {ok, _, GreenHitCount2, _GreenHits2, _, _} = dreyfus_search( - TargetDbName, <<"color:green">>), + TargetDbName, <<"color:green">> + ), ?assertEqual(5, RedHitCount2 + GreenHitCount2), - purge_docs(TargetDbName, [<<"apple">>, <<"tomato">>, <<"cherry">>, - <<"haw">>, <<"strawberry">>]), + purge_docs(TargetDbName, [ + <<"apple">>, + <<"tomato">>, + <<"cherry">>, + <<"haw">>, + <<"strawberry">> + ]), %third search {ok, _, RedHitCount3, _RedHits3, _, _} = dreyfus_search( - TargetDbName, <<"color:red">>), + TargetDbName, <<"color:red">> + ), {ok, _, GreenHitCount3, _GreenHits3, _, _} = dreyfus_search( - TargetDbName, <<"color:green">>), + TargetDbName, <<"color:green">> + ), ?assertEqual(5, RedHitCount3 + GreenHitCount3), ?assertEqual(RedHitCount2, GreenHitCount3), ?assertEqual(GreenHitCount2, RedHitCount3), - purge_docs(TargetDbName, [<<"apple">>, <<"tomato">>, <<"cherry">>, - <<"haw">>, <<"strawberry">>]), + purge_docs(TargetDbName, [ + <<"apple">>, + <<"tomato">>, + <<"cherry">>, + <<"haw">>, + <<"strawberry">> + ]), {ok, _, RedHitCount4, _, _, _} = dreyfus_search( - TargetDbName, <<"color:red">>), + TargetDbName, <<"color:red">> + ), {ok, _, GreenHitCount4, _, _, _} = dreyfus_search( - TargetDbName, <<"color:green">>), + TargetDbName, <<"color:green">> + ), ?assertEqual(0, RedHitCount4 + GreenHitCount4), @@ -289,36 +401,62 @@ test_purge_conflict4() -> %first search {ok, _, RedHitCount1, _RedHits1, _, _} = dreyfus_search( - TargetDbName, <<"color:red">>), + TargetDbName, <<"color:red">> + ), {ok, _, GreenHitCount1, _GreenHits1, _, _} = dreyfus_search( - TargetDbName, <<"color:green">>), + TargetDbName, <<"color:green">> + ), ?assertEqual(5, RedHitCount1 + GreenHitCount1), %do replicate and make conflicted docs - {ok, _} = fabric:update_doc(<<"_replicator">>, make_replicate_doc( - SourceDbName, TargetDbName), [?ADMIN_CTX]), + {ok, _} = fabric:update_doc( + <<"_replicator">>, + make_replicate_doc( + SourceDbName, TargetDbName + ), + [?ADMIN_CTX] + ), %%check doc version - wait_for_replicate(TargetDbName, [<<"apple">>, <<"tomato">>, <<"cherry">>, - <<"haw">>, <<"strawberry">>], 2, 5), + wait_for_replicate( + TargetDbName, + [ + <<"apple">>, + <<"tomato">>, + <<"cherry">>, + <<"haw">>, + <<"strawberry">> + ], + 2, + 5 + ), %second search {ok, _, RedHitCount2, _RedHits2, _, _} = dreyfus_search( - TargetDbName, <<"color:red">>), + TargetDbName, <<"color:red">> + ), {ok, _, GreenHitCount2, _GreenHits2, _, _} = dreyfus_search( - TargetDbName, <<"color:green">>), + TargetDbName, <<"color:green">> + ), ?assertEqual(5, RedHitCount2 + GreenHitCount2), - purge_docs_with_all_revs(TargetDbName, [<<"apple">>, <<"tomato">>, - <<"cherry">>, <<"haw">>, <<"strawberry">>]), + purge_docs_with_all_revs(TargetDbName, [ + <<"apple">>, + <<"tomato">>, + <<"cherry">>, + <<"haw">>, + <<"strawberry">> + ]), %third search {ok, _, RedHitCount3, _RedHits3, _, _} = dreyfus_search( - TargetDbName, <<"color:red">>), + TargetDbName, <<"color:red">> + ), {ok, _, GreenHitCount3, _GreenHits3, _, _} = dreyfus_search( - TargetDbName, <<"color:green">>), + TargetDbName, <<"color:green">> + ), ?assertEqual(0, RedHitCount3 + GreenHitCount3), @@ -341,12 +479,14 @@ test_purge_update() -> %update doc Rev = get_rev(DbName, <<"apple">>), - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"apple">>}, - {<<"_rev">>, couch_doc:rev_to_str(Rev)}, - {<<"color">>, <<"green">>}, - {<<"size">>, 8} - ]}), + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"apple">>}, + {<<"_rev">>, couch_doc:rev_to_str(Rev)}, + {<<"color">>, <<"green">>}, + {<<"size">>, 8} + ]} + ), {ok, _} = fabric:update_docs(DbName, [Doc], [?ADMIN_CTX]), %second search request @@ -433,36 +573,62 @@ test_delete_conflict() -> %first search {ok, _, RedHitCount1, _RedHits1, _, _} = dreyfus_search( - TargetDbName, <<"color:red">>), + TargetDbName, <<"color:red">> + ), {ok, _, GreenHitCount1, _GreenHits1, _, _} = dreyfus_search( - TargetDbName, <<"color:green">>), + TargetDbName, <<"color:green">> + ), ?assertEqual(5, RedHitCount1 + GreenHitCount1), %do replicate and make conflicted docs - {ok, _} = fabric:update_doc(<<"_replicator">>, make_replicate_doc( - SourceDbName, TargetDbName), [?ADMIN_CTX]), + {ok, _} = fabric:update_doc( + <<"_replicator">>, + make_replicate_doc( + SourceDbName, TargetDbName + ), + [?ADMIN_CTX] + ), - wait_for_replicate(TargetDbName, [<<"apple">>, <<"tomato">>, <<"cherry">>, - <<"haw">>, <<"strawberry">>], 2, 5), + wait_for_replicate( + TargetDbName, + [ + <<"apple">>, + <<"tomato">>, + <<"cherry">>, + <<"haw">>, + <<"strawberry">> + ], + 2, + 5 + ), %second search {ok, _, RedHitCount2, _RedHits2, _, _} = dreyfus_search( - TargetDbName, <<"color:red">>), + TargetDbName, <<"color:red">> + ), {ok, _, GreenHitCount2, _GreenHits2, _, _} = dreyfus_search( - TargetDbName, <<"color:green">>), + TargetDbName, <<"color:green">> + ), ?assertEqual(5, RedHitCount2 + GreenHitCount2), %delete docs - delete_docs(TargetDbName, [<<"apple">>, <<"tomato">>, <<"cherry">>, - <<"haw">>, <<"strawberry">>]), + delete_docs(TargetDbName, [ + <<"apple">>, + <<"tomato">>, + <<"cherry">>, + <<"haw">>, + <<"strawberry">> + ]), %third search {ok, _, RedHitCount3, _RedHits3, _, _} = dreyfus_search( - TargetDbName, <<"color:red">>), + TargetDbName, <<"color:red">> + ), {ok, _, GreenHitCount3, _GreenHits3, _, _} = dreyfus_search( - TargetDbName, <<"color:green">>), + TargetDbName, <<"color:green">> + ), ?assertEqual(5, RedHitCount3 + GreenHitCount3), ?assertEqual(RedHitCount2, GreenHitCount3), @@ -483,40 +649,71 @@ test_delete_purge_conflict() -> %first search {ok, _, RedHitCount1, _RedHits1, _, _} = dreyfus_search( - TargetDbName, <<"color:red">>), + TargetDbName, <<"color:red">> + ), {ok, _, GreenHitCount1, _GreenHits1, _, _} = dreyfus_search( - TargetDbName, <<"color:green">>), + TargetDbName, <<"color:green">> + ), ?assertEqual(5, RedHitCount1 + GreenHitCount1), %do replicate and make conflicted docs - {ok, _} = fabric:update_doc(<<"_replicator">>, make_replicate_doc( - SourceDbName, TargetDbName), [?ADMIN_CTX]), + {ok, _} = fabric:update_doc( + <<"_replicator">>, + make_replicate_doc( + SourceDbName, TargetDbName + ), + [?ADMIN_CTX] + ), - wait_for_replicate(TargetDbName, [<<"apple">>, <<"tomato">>, <<"cherry">>, - <<"haw">>, <<"strawberry">>], 2, 5), + wait_for_replicate( + TargetDbName, + [ + <<"apple">>, + <<"tomato">>, + <<"cherry">>, + <<"haw">>, + <<"strawberry">> + ], + 2, + 5 + ), %second search {ok, _, RedHitCount2, _RedHits2, _, _} = dreyfus_search( - TargetDbName, <<"color:red">>), + TargetDbName, <<"color:red">> + ), {ok, _, GreenHitCount2, _GreenHits2, _, _} = dreyfus_search( - TargetDbName, <<"color:green">>), + TargetDbName, <<"color:green">> + ), ?assertEqual(5, RedHitCount2 + GreenHitCount2), %purge docs - purge_docs(TargetDbName, [<<"apple">>, <<"tomato">>, <<"cherry">>, - <<"haw">>, <<"strawberry">>]), + purge_docs(TargetDbName, [ + <<"apple">>, + <<"tomato">>, + <<"cherry">>, + <<"haw">>, + <<"strawberry">> + ]), %delete docs - delete_docs(TargetDbName, [<<"apple">>, <<"tomato">>, <<"cherry">>, - <<"haw">>, <<"strawberry">>]), + delete_docs(TargetDbName, [ + <<"apple">>, + <<"tomato">>, + <<"cherry">>, + <<"haw">>, + <<"strawberry">> + ]), %third search {ok, _, RedHitCount3, _RedHits3, _, _} = dreyfus_search( - TargetDbName, <<"color:red">>), + TargetDbName, <<"color:red">> + ), {ok, _, GreenHitCount3, _GreenHits3, _, _} = dreyfus_search( - TargetDbName, <<"color:green">>), + TargetDbName, <<"color:green">> + ), ?assertEqual(RedHitCount3, 0), ?assertEqual(GreenHitCount3, 0), @@ -533,26 +730,33 @@ test_local_doc() -> {ok, _, HitCount1, _, _, _} = dreyfus_search(DbName, <<"apple">>), ?assertEqual(HitCount1, 1), - purge_docs(DbName, [<<"apple">>, <<"tomato">>, <<"cherry">>, - <<"strawberry">>]), + purge_docs(DbName, [ + <<"apple">>, + <<"tomato">>, + <<"cherry">>, + <<"strawberry">> + ]), {ok, _, HitCount2, _, _, _} = dreyfus_search(DbName, <<"apple">>), ?assertEqual(HitCount2, 0), %get local doc - [Sig|_] = get_sigs(DbName), + [Sig | _] = get_sigs(DbName), LocalId = dreyfus_util:get_local_purge_doc_id(Sig), LocalShards = mem3:local_shards(DbName), - PurgeSeqs = lists:map(fun(Shard) -> - {ok, Db} = couch_db:open_int(Shard#shard.name, [?ADMIN_CTX]), - {ok, LDoc} = couch_db:open_doc(Db, LocalId, []), - {Props} = couch_doc:to_json_obj(LDoc, []), - dreyfus_util:get_value_from_options(<<"updated_on">>, Props), - PurgeSeq = dreyfus_util:get_value_from_options(<<"purge_seq">>, Props), - Type = dreyfus_util:get_value_from_options(<<"type">>, Props), - ?assertEqual(<<"dreyfus">>, Type), - couch_db:close(Db), - PurgeSeq - end, LocalShards), + PurgeSeqs = lists:map( + fun(Shard) -> + {ok, Db} = couch_db:open_int(Shard#shard.name, [?ADMIN_CTX]), + {ok, LDoc} = couch_db:open_doc(Db, LocalId, []), + {Props} = couch_doc:to_json_obj(LDoc, []), + dreyfus_util:get_value_from_options(<<"updated_on">>, Props), + PurgeSeq = dreyfus_util:get_value_from_options(<<"purge_seq">>, Props), + Type = dreyfus_util:get_value_from_options(<<"type">>, Props), + ?assertEqual(<<"dreyfus">>, Type), + couch_db:close(Db), + PurgeSeq + end, + LocalShards + ), ?assertEqual(lists:sum(PurgeSeqs), 4), delete_db(DbName), @@ -571,11 +775,14 @@ test_verify_index_exists1() -> ?assertEqual(HitCount2, 0), ShardNames = [Sh || #shard{name = Sh} <- mem3:local_shards(DbName)], - [ShardDbName | _Rest ] = ShardNames, + [ShardDbName | _Rest] = ShardNames, {ok, Db} = couch_db:open(ShardDbName, [?ADMIN_CTX]), - {ok, LDoc} = couch_db:open_doc(Db, + {ok, LDoc} = couch_db:open_doc( + Db, dreyfus_util:get_local_purge_doc_id( - <<"49e82c2a910b1046b55cc45ad058a7ee">>), [] + <<"49e82c2a910b1046b55cc45ad058a7ee">> + ), + [] ), #doc{body = {Props}} = LDoc, ?assertEqual(true, dreyfus_util:verify_index_exists(ShardDbName, Props)), @@ -590,11 +797,14 @@ test_verify_index_exists2() -> ?assertEqual(HitCount1, 1), ShardNames = [Sh || #shard{name = Sh} <- mem3:local_shards(DbName)], - [ShardDbName | _Rest ] = ShardNames, + [ShardDbName | _Rest] = ShardNames, {ok, Db} = couch_db:open(ShardDbName, [?ADMIN_CTX]), - {ok, LDoc} = couch_db:open_doc(Db, + {ok, LDoc} = couch_db:open_doc( + Db, dreyfus_util:get_local_purge_doc_id( - <<"49e82c2a910b1046b55cc45ad058a7ee">>), [] + <<"49e82c2a910b1046b55cc45ad058a7ee">> + ), + [] ), #doc{body = {Props}} = LDoc, ?assertEqual(true, dreyfus_util:verify_index_exists(ShardDbName, Props)), @@ -610,55 +820,63 @@ test_verify_index_exists_failed() -> ?assertEqual(HitCount1, 1), ShardNames = [Sh || #shard{name = Sh} <- mem3:local_shards(DbName)], - [ShardDbName | _Rest ] = ShardNames, + [ShardDbName | _Rest] = ShardNames, {ok, Db} = couch_db:open(ShardDbName, [?ADMIN_CTX]), - {ok, LDoc} = couch_db:open_doc(Db, + {ok, LDoc} = couch_db:open_doc( + Db, dreyfus_util:get_local_purge_doc_id( - <<"49e82c2a910b1046b55cc45ad058a7ee">>), [] + <<"49e82c2a910b1046b55cc45ad058a7ee">> + ), + [] ), #doc{body = {Options}} = LDoc, OptionsDbErr = [ - {<<"indexname">>, - dreyfus_util:get_value_from_options(<<"indexname">>, Options)}, - {<<"ddoc_id">>, - dreyfus_util:get_value_from_options(<<"ddoc_id">>, Options)}, - {<<"signature">>, - dreyfus_util:get_value_from_options(<<"signature">>, Options)} + {<<"indexname">>, dreyfus_util:get_value_from_options(<<"indexname">>, Options)}, + {<<"ddoc_id">>, dreyfus_util:get_value_from_options(<<"ddoc_id">>, Options)}, + {<<"signature">>, dreyfus_util:get_value_from_options(<<"signature">>, Options)} ], - ?assertEqual(false, dreyfus_util:verify_index_exists( - ShardDbName, OptionsDbErr)), + ?assertEqual( + false, + dreyfus_util:verify_index_exists( + ShardDbName, OptionsDbErr + ) + ), OptionsIdxErr = [ {<<"indexname">>, <<"someindex">>}, - {<<"ddoc_id">>, - dreyfus_util:get_value_from_options(<<"ddoc_id">>, Options)}, - {<<"signature">>, - dreyfus_util:get_value_from_options(<<"signature">>, Options)} + {<<"ddoc_id">>, dreyfus_util:get_value_from_options(<<"ddoc_id">>, Options)}, + {<<"signature">>, dreyfus_util:get_value_from_options(<<"signature">>, Options)} ], - ?assertEqual(false, dreyfus_util:verify_index_exists( - ShardDbName, OptionsIdxErr)), + ?assertEqual( + false, + dreyfus_util:verify_index_exists( + ShardDbName, OptionsIdxErr + ) + ), OptionsDDocErr = [ - {<<"indexname">>, - dreyfus_util:get_value_from_options(<<"indexname">>, Options)}, - {<<"ddoc_id">>, - <<"somedesigndoc">>}, - {<<"signature">>, - dreyfus_util:get_value_from_options(<<"signature">>, Options)} + {<<"indexname">>, dreyfus_util:get_value_from_options(<<"indexname">>, Options)}, + {<<"ddoc_id">>, <<"somedesigndoc">>}, + {<<"signature">>, dreyfus_util:get_value_from_options(<<"signature">>, Options)} ], - ?assertEqual(false, dreyfus_util:verify_index_exists( - ShardDbName, OptionsDDocErr)), + ?assertEqual( + false, + dreyfus_util:verify_index_exists( + ShardDbName, OptionsDDocErr + ) + ), OptionsSigErr = [ - {<<"indexname">>, - dreyfus_util:get_value_from_options(<<"indexname">>, Options)}, - {<<"ddoc_id">>, - dreyfus_util:get_value_from_options(<<"ddoc_id">>, Options)}, - {<<"signature">>, - <<"12345678901234567890123456789012">>} + {<<"indexname">>, dreyfus_util:get_value_from_options(<<"indexname">>, Options)}, + {<<"ddoc_id">>, dreyfus_util:get_value_from_options(<<"ddoc_id">>, Options)}, + {<<"signature">>, <<"12345678901234567890123456789012">>} ], - ?assertEqual(false, dreyfus_util:verify_index_exists( - ShardDbName, OptionsSigErr)), + ?assertEqual( + false, + dreyfus_util:verify_index_exists( + ShardDbName, OptionsSigErr + ) + ), delete_db(DbName), ok. @@ -676,9 +894,10 @@ test_delete_local_doc() -> ?assertEqual(HitCount2, 0), LDocId = dreyfus_util:get_local_purge_doc_id( - <<"49e82c2a910b1046b55cc45ad058a7ee">>), + <<"49e82c2a910b1046b55cc45ad058a7ee">> + ), ShardNames = [Sh || #shard{name = Sh} <- mem3:local_shards(DbName)], - [ShardDbName | _Rest ] = ShardNames, + [ShardDbName | _Rest] = ShardNames, {ok, Db} = couch_db:open(ShardDbName, [?ADMIN_CTX]), {ok, _} = couch_db:open_doc(Db, LDocId, []), @@ -686,7 +905,6 @@ test_delete_local_doc() -> io:format("DbName ~p~n", [DbName]), ?debugFmt("Converting ... ~n~p~n", [DbName]), - dreyfus_fabric_cleanup:go(DbName), {ok, Db2} = couch_db:open(ShardDbName, [?ADMIN_CTX]), {not_found, _} = couch_db:open_doc(Db2, LDocId, []), @@ -706,8 +924,10 @@ test_purge_search() -> %private API db_name() -> iolist_to_binary([ - "dreyfus-test-db-", [ - integer_to_list(I) || I <- [ + "dreyfus-test-db-", + [ + integer_to_list(I) + || I <- [ erlang:unique_integer([positive]), rand:uniform(10000) ] @@ -751,11 +971,13 @@ make_docs(Count, Color) -> [make_doc(I, Color) || I <- lists:seq(1, Count)]. make_doc(Id, Color) -> - couch_doc:from_json_obj({[ - {<<"_id">>, get_value(Id)}, - {<<"color">>, Color}, - {<<"size">>, 1} - ]}). + couch_doc:from_json_obj( + {[ + {<<"_id">>, get_value(Id)}, + {<<"color">>, Color}, + {<<"size">>, 1} + ]} + ). get_value(Key) -> case Key of @@ -772,34 +994,43 @@ get_value(Key) -> end. make_design_doc(dreyfus) -> - couch_doc:from_json_obj({[ - {<<"_id">>, <<"_design/search">>}, - {<<"language">>, <<"javascript">>}, - {<<"indexes">>, {[ - {<<"index">>, {[ - {<<"analyzer">>, <<"standard">>}, - {<<"index">>, << - "function (doc) { \n" - " index(\"default\", doc._id);\n" - " if(doc.color) {\n" - " index(\"color\", doc.color);\n" - " }\n" - " if(doc.size) {\n" - " index(\"size\", doc.size);\n" - " }\n" - "}" - >>} - ]}} - ]}} - ]}). + couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"_design/search">>}, + {<<"language">>, <<"javascript">>}, + {<<"indexes">>, + {[ + {<<"index">>, + {[ + {<<"analyzer">>, <<"standard">>}, + {<<"index">>, << + "function (doc) { \n" + " index(\"default\", doc._id);\n" + " if(doc.color) {\n" + " index(\"color\", doc.color);\n" + " }\n" + " if(doc.size) {\n" + " index(\"size\", doc.size);\n" + " }\n" + "}" + >>} + ]}} + ]}} + ]} + ). make_replicate_doc(SourceDbName, TargetDbName) -> - couch_doc:from_json_obj({[ - {<<"_id">>, list_to_binary("replicate_fm_" ++ - binary_to_list(SourceDbName) ++ "_to_" ++ binary_to_list(TargetDbName))}, - {<<"source">>, list_to_binary("http://localhost:15984/" ++ SourceDbName)}, - {<<"target">>, list_to_binary("http://localhost:15984/" ++ TargetDbName)} - ]}). + couch_doc:from_json_obj( + {[ + {<<"_id">>, + list_to_binary( + "replicate_fm_" ++ + binary_to_list(SourceDbName) ++ "_to_" ++ binary_to_list(TargetDbName) + )}, + {<<"source">>, list_to_binary("http://localhost:15984/" ++ SourceDbName)}, + {<<"target">>, list_to_binary("http://localhost:15984/" ++ TargetDbName)} + ]} + ). get_rev(DbName, DocId) -> FDI = fabric:get_full_doc_info(DbName, DocId, []), @@ -808,20 +1039,22 @@ get_rev(DbName, DocId) -> get_revs(DbName, DocId) -> FDI = fabric:get_full_doc_info(DbName, DocId, []), - #doc_info{ revs = Revs } = couch_doc:to_doc_info(FDI), + #doc_info{revs = Revs} = couch_doc:to_doc_info(FDI), [Rev#rev_info.rev || Rev <- Revs]. update_doc(_, _, 0) -> ok; update_doc(DbName, DocId, Times) -> Rev = get_rev(DbName, DocId), - Doc = couch_doc:from_json_obj({[ - {<<"_id">>, <<"apple">>}, - {<<"_rev">>, couch_doc:rev_to_str(Rev)}, - {<<"size">>, 1001 - Times} - ]}), + Doc = couch_doc:from_json_obj( + {[ + {<<"_id">>, <<"apple">>}, + {<<"_rev">>, couch_doc:rev_to_str(Rev)}, + {<<"size">>, 1001 - Times} + ]} + ), {ok, _} = fabric:update_docs(DbName, [Doc], [?ADMIN_CTX]), - update_doc(DbName, DocId, Times-1). + update_doc(DbName, DocId, Times - 1). delete_docs(DbName, DocIds) -> lists:foreach( @@ -831,43 +1064,55 @@ delete_docs(DbName, DocIds) -> delete_doc(DbName, DocId) -> Rev = get_rev(DbName, DocId), - DDoc = couch_doc:from_json_obj({[ - {<<"_id">>, DocId}, - {<<"_rev">>, couch_doc:rev_to_str(Rev)}, - {<<"_deleted">>, true} - ]}), + DDoc = couch_doc:from_json_obj( + {[ + {<<"_id">>, DocId}, + {<<"_rev">>, couch_doc:rev_to_str(Rev)}, + {<<"_deleted">>, true} + ]} + ), {ok, _} = fabric:update_doc(DbName, DDoc, [?ADMIN_CTX]), ok. wait_for_replicate(_, _, _, 0) -> couch_log:notice("[~p] wait time out", [?MODULE]), ok; -wait_for_replicate(DbName, DocIds, ExpectRevCount ,TimeOut) - when is_list(DocIds) -> - [wait_for_replicate(DbName, DocId, ExpectRevCount ,TimeOut) || DocId <- DocIds]; -wait_for_replicate(DbName, DocId, ExpectRevCount ,TimeOut) -> +wait_for_replicate(DbName, DocIds, ExpectRevCount, TimeOut) when + is_list(DocIds) +-> + [wait_for_replicate(DbName, DocId, ExpectRevCount, TimeOut) || DocId <- DocIds]; +wait_for_replicate(DbName, DocId, ExpectRevCount, TimeOut) -> FDI = fabric:get_full_doc_info(DbName, DocId, []), - #doc_info{ revs = Revs } = couch_doc:to_doc_info(FDI), + #doc_info{revs = Revs} = couch_doc:to_doc_info(FDI), case erlang:length(Revs) of ExpectRevCount -> - couch_log:notice("[~p] wait end by expect, time used:~p, DocId:~p", - [?MODULE, 5-TimeOut, DocId]), + couch_log:notice( + "[~p] wait end by expect, time used:~p, DocId:~p", + [?MODULE, 5 - TimeOut, DocId] + ), ok; true -> timer:sleep(1000), - wait_for_replicate(DbName, DocId, ExpectRevCount ,TimeOut-1) + wait_for_replicate(DbName, DocId, ExpectRevCount, TimeOut - 1) end, ok. get_sigs(DbName) -> {ok, DesignDocs} = fabric:design_docs(DbName), - lists:usort(lists:flatmap(fun active_sigs/1, - [couch_doc:from_json_obj(DD) || DD <- DesignDocs])). + lists:usort( + lists:flatmap( + fun active_sigs/1, + [couch_doc:from_json_obj(DD) || DD <- DesignDocs] + ) + ). -active_sigs(#doc{body={Fields}}=Doc) -> +active_sigs(#doc{body = {Fields}} = Doc) -> {RawIndexes} = couch_util:get_value(<<"indexes">>, Fields, {[]}), {IndexNames, _} = lists:unzip(RawIndexes), - [begin - {ok, Index} = dreyfus_index:design_doc_to_index(Doc, IndexName), - Index#index.sig - end || IndexName <- IndexNames]. + [ + begin + {ok, Index} = dreyfus_index:design_doc_to_index(Doc, IndexName), + Index#index.sig + end + || IndexName <- IndexNames + ]. diff --git a/src/fabric/src/fabric.erl b/src/fabric/src/fabric.erl index aaaa7e011..5840cd63c 100644 --- a/src/fabric/src/fabric.erl +++ b/src/fabric/src/fabric.erl @@ -17,27 +17,57 @@ -include_lib("couch_mrview/include/couch_mrview.hrl"). % DBs --export([all_dbs/0, all_dbs/1, create_db/1, create_db/2, delete_db/1, - delete_db/2, get_db_info/1, get_doc_count/1, get_doc_count/2, - set_revs_limit/3, set_security/2, set_security/3, - get_revs_limit/1, get_security/1, get_security/2, +-export([ + all_dbs/0, all_dbs/1, + create_db/1, create_db/2, + delete_db/1, + delete_db/2, + get_db_info/1, + get_doc_count/1, get_doc_count/2, + set_revs_limit/3, + set_security/2, set_security/3, + get_revs_limit/1, + get_security/1, get_security/2, get_all_security/1, get_all_security/2, - get_purge_infos_limit/1, set_purge_infos_limit/3, - compact/1, compact/2, get_partition_info/2]). + get_purge_infos_limit/1, + set_purge_infos_limit/3, + compact/1, compact/2, + get_partition_info/2 +]). % Documents --export([open_doc/3, open_revs/4, get_doc_info/3, get_full_doc_info/3, - get_missing_revs/2, get_missing_revs/3, update_doc/3, update_docs/3, - purge_docs/3, att_receiver/3]). +-export([ + open_doc/3, + open_revs/4, + get_doc_info/3, + get_full_doc_info/3, + get_missing_revs/2, get_missing_revs/3, + update_doc/3, + update_docs/3, + purge_docs/3, + att_receiver/3 +]). % Views --export([all_docs/4, all_docs/5, changes/4, query_view/3, query_view/4, - query_view/6, query_view/7, get_view_group_info/2, end_changes/0]). +-export([ + all_docs/4, all_docs/5, + changes/4, + query_view/3, query_view/4, query_view/6, query_view/7, + get_view_group_info/2, + end_changes/0 +]). % miscellany --export([design_docs/1, reset_validation_funs/1, cleanup_index_files/0, - cleanup_index_files/1, cleanup_index_files_all_nodes/1, dbname/1, - inactive_index_files/1, db_uuids/1]). +-export([ + design_docs/1, + reset_validation_funs/1, + cleanup_index_files/0, + cleanup_index_files/1, + cleanup_index_files_all_nodes/1, + dbname/1, + inactive_index_files/1, + db_uuids/1 +]). -include_lib("fabric/include/fabric.hrl"). @@ -54,19 +84,21 @@ all_dbs() -> all_dbs(<<>>). %% @doc returns a list of all database names --spec all_dbs(Prefix::iodata()) -> {ok, [binary()]}. +-spec all_dbs(Prefix :: iodata()) -> {ok, [binary()]}. all_dbs(Prefix) when is_binary(Prefix) -> Length = byte_size(Prefix), - MatchingDbs = mem3:fold_shards(fun(#shard{dbname=DbName}, Acc) -> - case DbName of - <<Prefix:Length/binary, _/binary>> -> - [DbName | Acc]; - _ -> - Acc - end - end, []), + MatchingDbs = mem3:fold_shards( + fun(#shard{dbname = DbName}, Acc) -> + case DbName of + <<Prefix:Length/binary, _/binary>> -> + [DbName | Acc]; + _ -> + Acc + end + end, + [] + ), {ok, lists:usort(MatchingDbs)}; - %% @equiv all_dbs(list_to_binary(Prefix)) all_dbs(Prefix) when is_list(Prefix) -> all_dbs(list_to_binary(Prefix)). @@ -76,40 +108,39 @@ all_dbs(Prefix) when is_list(Prefix) -> %% etc. -spec get_db_info(dbname()) -> {ok, [ - {instance_start_time, binary()} | - {doc_count, non_neg_integer()} | - {doc_del_count, non_neg_integer()} | - {purge_seq, non_neg_integer()} | - {compact_running, boolean()} | - {disk_size, non_neg_integer()} | - {disk_format_version, pos_integer()} + {instance_start_time, binary()} + | {doc_count, non_neg_integer()} + | {doc_del_count, non_neg_integer()} + | {purge_seq, non_neg_integer()} + | {compact_running, boolean()} + | {disk_size, non_neg_integer()} + | {disk_format_version, pos_integer()} ]}. get_db_info(DbName) -> fabric_db_info:go(dbname(DbName)). %% @doc returns the size of a given partition --spec get_partition_info(dbname(), Partition::binary()) -> +-spec get_partition_info(dbname(), Partition :: binary()) -> {ok, [ - {db_name, binary()} | - {partition, binary()} | - {doc_count, non_neg_integer()} | - {doc_del_count, non_neg_integer()} | - {sizes, json_obj()} + {db_name, binary()} + | {partition, binary()} + | {doc_count, non_neg_integer()} + | {doc_del_count, non_neg_integer()} + | {sizes, json_obj()} ]}. get_partition_info(DbName, Partition) -> fabric_db_partition_info:go(dbname(DbName), Partition). - %% @doc the number of docs in a database %% @equiv get_doc_count(DbName, <<"_all_docs">>) get_doc_count(DbName) -> get_doc_count(DbName, <<"_all_docs">>). %% @doc the number of design docs in a database --spec get_doc_count(dbname(), Namespace::binary()) -> - {ok, non_neg_integer() | null} | - {error, atom()} | - {error, atom(), any()}. +-spec get_doc_count(dbname(), Namespace :: binary()) -> + {ok, non_neg_integer() | null} + | {error, atom()} + | {error, atom(), any()}. get_doc_count(DbName, <<"_all_docs">>) -> fabric_db_doc_count:go(dbname(DbName)); get_doc_count(DbName, <<"_design">>) -> @@ -150,29 +181,38 @@ set_revs_limit(DbName, Limit, Options) when is_integer(Limit), Limit > 0 -> -spec get_revs_limit(dbname()) -> pos_integer() | no_return(). get_revs_limit(DbName) -> {ok, Db} = fabric_util:get_db(dbname(DbName), [?ADMIN_CTX]), - try couch_db:get_revs_limit(Db) after catch couch_db:close(Db) end. + try + couch_db:get_revs_limit(Db) + after + catch couch_db:close(Db) + end. %% @doc sets the readers/writers/admin permissions for a database --spec set_security(dbname(), SecObj::json_obj()) -> ok. +-spec set_security(dbname(), SecObj :: json_obj()) -> ok. set_security(DbName, SecObj) -> fabric_db_meta:set_security(dbname(DbName), SecObj, [?ADMIN_CTX]). %% @doc sets the readers/writers/admin permissions for a database --spec set_security(dbname(), SecObj::json_obj(), [option()]) -> ok. +-spec set_security(dbname(), SecObj :: json_obj(), [option()]) -> ok. set_security(DbName, SecObj, Options) -> fabric_db_meta:set_security(dbname(DbName), SecObj, opts(Options)). %% @doc sets the upper bound for the number of stored purge requests -spec set_purge_infos_limit(dbname(), pos_integer(), [option()]) -> ok. -set_purge_infos_limit(DbName, Limit, Options) - when is_integer(Limit), Limit > 0 -> +set_purge_infos_limit(DbName, Limit, Options) when + is_integer(Limit), Limit > 0 +-> fabric_db_meta:set_purge_infos_limit(dbname(DbName), Limit, opts(Options)). %% @doc retrieves the upper bound for the number of stored purge requests -spec get_purge_infos_limit(dbname()) -> pos_integer() | no_return(). get_purge_infos_limit(DbName) -> {ok, Db} = fabric_util:get_db(dbname(DbName), [?ADMIN_CTX]), - try couch_db:get_purge_infos_limit(Db) after catch couch_db:close(Db) end. + try + couch_db:get_purge_infos_limit(Db) + after + catch couch_db:close(Db) + end. get_security(DbName) -> get_security(DbName, [?ADMIN_CTX]). @@ -181,80 +221,88 @@ get_security(DbName) -> -spec get_security(dbname()) -> json_obj() | no_return(). get_security(DbName, Options) -> {ok, Db} = fabric_util:get_db(dbname(DbName), opts(Options)), - try couch_db:get_security(Db) after catch couch_db:close(Db) end. + try + couch_db:get_security(Db) + after + catch couch_db:close(Db) + end. %% @doc retrieve the security object for all shards of a database -spec get_all_security(dbname()) -> - {ok, [{#shard{}, json_obj()}]} | - {error, no_majority | timeout} | - {error, atom(), any()}. + {ok, [{#shard{}, json_obj()}]} + | {error, no_majority | timeout} + | {error, atom(), any()}. get_all_security(DbName) -> get_all_security(DbName, []). %% @doc retrieve the security object for all shards of a database -spec get_all_security(dbname(), [option()]) -> - {ok, [{#shard{}, json_obj()}]} | - {error, no_majority | timeout} | - {error, atom(), any()}. + {ok, [{#shard{}, json_obj()}]} + | {error, no_majority | timeout} + | {error, atom(), any()}. get_all_security(DbName, Options) -> fabric_db_meta:get_all_security(dbname(DbName), opts(Options)). compact(DbName) -> - [rexi:cast(Node, {fabric_rpc, compact, [Name]}) || - #shard{node=Node, name=Name} <- mem3:shards(dbname(DbName))], + [ + rexi:cast(Node, {fabric_rpc, compact, [Name]}) + || #shard{node = Node, name = Name} <- mem3:shards(dbname(DbName)) + ], ok. compact(DbName, DesignName) -> - [rexi:cast(Node, {fabric_rpc, compact, [Name, DesignName]}) || - #shard{node=Node, name=Name} <- mem3:shards(dbname(DbName))], + [ + rexi:cast(Node, {fabric_rpc, compact, [Name, DesignName]}) + || #shard{node = Node, name = Name} <- mem3:shards(dbname(DbName)) + ], ok. % doc operations %% @doc retrieve the doc with a given id -spec open_doc(dbname(), docid(), [option()]) -> - {ok, #doc{}} | - {not_found, missing | deleted} | - {timeout, any()} | - {error, any()} | - {error, any() | any()}. + {ok, #doc{}} + | {not_found, missing | deleted} + | {timeout, any()} + | {error, any()} + | {error, any() | any()}. open_doc(DbName, Id, Options) -> case proplists:get_value(doc_info, Options) of - undefined -> - fabric_doc_open:go(dbname(DbName), docid(Id), opts(Options)); - Else -> - {error, {invalid_option, {doc_info, Else}}} + undefined -> + fabric_doc_open:go(dbname(DbName), docid(Id), opts(Options)); + Else -> + {error, {invalid_option, {doc_info, Else}}} end. %% @doc retrieve a collection of revisions, possible all -spec open_revs(dbname(), docid(), [revision()] | all, [option()]) -> - {ok, [{ok, #doc{}} | {{not_found,missing}, revision()}]} | - {timeout, any()} | - {error, any()} | - {error, any(), any()}. + {ok, [{ok, #doc{}} | {{not_found, missing}, revision()}]} + | {timeout, any()} + | {error, any()} + | {error, any(), any()}. open_revs(DbName, Id, Revs, Options) -> fabric_doc_open_revs:go(dbname(DbName), docid(Id), Revs, opts(Options)). %% @doc Retrieves an information on a document with a given id -spec get_doc_info(dbname(), docid(), [options()]) -> - {ok, #doc_info{}} | - {not_found, missing} | - {timeout, any()} | - {error, any()} | - {error, any() | any()}. + {ok, #doc_info{}} + | {not_found, missing} + | {timeout, any()} + | {error, any()} + | {error, any() | any()}. get_doc_info(DbName, Id, Options) -> - Options1 = [doc_info|Options], + Options1 = [doc_info | Options], fabric_doc_open:go(dbname(DbName), docid(Id), opts(Options1)). %% @doc Retrieves a full information on a document with a given id -spec get_full_doc_info(dbname(), docid(), [options()]) -> - {ok, #full_doc_info{}} | - {not_found, missing | deleted} | - {timeout, any()} | - {error, any()} | - {error, any() | any()}. + {ok, #full_doc_info{}} + | {not_found, missing | deleted} + | {timeout, any()} + | {error, any()} + | {error, any() | any()}. get_full_doc_info(DbName, Id, Options) -> - Options1 = [{doc_info, full}|Options], + Options1 = [{doc_info, full} | Options], fabric_doc_open:go(dbname(DbName), docid(Id), opts(Options1)). %% @equiv get_missing_revs(DbName, IdsRevs, []) @@ -262,7 +310,7 @@ get_missing_revs(DbName, IdsRevs) -> get_missing_revs(DbName, IdsRevs, []). %% @doc retrieve missing revisions for a list of `{Id, Revs}' --spec get_missing_revs(dbname(),[{docid(), [revision()]}], [option()]) -> +-spec get_missing_revs(dbname(), [{docid(), [revision()]}], [option()]) -> {ok, [{docid(), any(), [any()]}]}. get_missing_revs(DbName, IdsRevs, Options) when is_list(IdsRevs) -> Sanitized = [idrevs(IdR) || IdR <- IdsRevs], @@ -274,20 +322,20 @@ get_missing_revs(DbName, IdsRevs, Options) when is_list(IdsRevs) -> {ok, any()} | any(). update_doc(DbName, Doc, Options) -> case update_docs(DbName, [Doc], opts(Options)) of - {ok, [{ok, NewRev}]} -> - {ok, NewRev}; - {accepted, [{accepted, NewRev}]} -> - {accepted, NewRev}; - {ok, [{{_Id, _Rev}, Error}]} -> - throw(Error); - {ok, [Error]} -> - throw(Error); - {ok, []} -> - % replication success - #doc{revs = {Pos, [RevId | _]}} = doc(DbName, Doc), - {ok, {Pos, RevId}}; - {error, [Error]} -> - throw(Error) + {ok, [{ok, NewRev}]} -> + {ok, NewRev}; + {accepted, [{accepted, NewRev}]} -> + {accepted, NewRev}; + {ok, [{{_Id, _Rev}, Error}]} -> + throw(Error); + {ok, [Error]} -> + throw(Error); + {ok, []} -> + % replication success + #doc{revs = {Pos, [RevId | _]}} = doc(DbName, Doc), + {ok, {Pos, RevId}}; + {error, [Error]} -> + throw(Error) end. %% @doc update a list of docs @@ -296,7 +344,8 @@ update_doc(DbName, Doc, Options) -> update_docs(DbName, Docs0, Options) -> try Docs1 = docs(DbName, Docs0), - fabric_doc_update:go(dbname(DbName), Docs1, opts(Options)) of + fabric_doc_update:go(dbname(DbName), Docs1, opts(Options)) + of {ok, Results} -> {ok, Results}; {accepted, Results} -> @@ -305,27 +354,34 @@ update_docs(DbName, Docs0, Options) -> {error, Error}; Error -> throw(Error) - catch {aborted, PreCommitFailures} -> - {aborted, PreCommitFailures} + catch + {aborted, PreCommitFailures} -> + {aborted, PreCommitFailures} end. - %% @doc purge revisions for a list '{Id, Revs}' %% returns {ok, {PurgeSeq, Results}} -spec purge_docs(dbname(), [{docid(), [revision()]}], [option()]) -> - {ok, [{Health, [revision()]}] | {error, any()}} when + {ok, [{Health, [revision()]}] | {error, any()}} +when Health :: ok | accepted. purge_docs(DbName, IdsRevs, Options) when is_list(IdsRevs) -> IdsRevs2 = [idrevs(IdRs) || IdRs <- IdsRevs], fabric_doc_purge:go(dbname(DbName), IdsRevs2, opts(Options)). - %% @doc spawns a process to upload attachment data and %% returns a fabric attachment receiver context tuple %% with the spawned middleman process, an empty binary, %% or exits with an error tuple {Error, Arg} --spec att_receiver(#httpd{}, dbname(), Length :: undefined | chunked | pos_integer() | - {unknown_transfer_encoding, any()}) -> +-spec att_receiver( + #httpd{}, + dbname(), + Length :: + undefined + | chunked + | pos_integer() + | {unknown_transfer_encoding, any()} +) -> {fabric_attachment_receiver, pid(), chunked | pos_integer()} | binary(). att_receiver(Req, DbName, Length) -> fabric_doc_atts:receiver(Req, DbName, Length). @@ -340,26 +396,28 @@ all_docs(DbName, Callback, Acc, QueryArgs) -> %% "http://wiki.apache.org/couchdb/HTTP_Document_API#All_Documents"> %% all_docs</a> for details -spec all_docs( - dbname(), [{atom(), any()}], callback(), [] | tuple(), - #mrargs{} | [option()]) -> + dbname(), + [{atom(), any()}], + callback(), + [] | tuple(), + #mrargs{} | [option()] +) -> {ok, any()} | {error, Reason :: term()}. all_docs(DbName, Options, Callback, Acc0, #mrargs{} = QueryArgs) when - is_function(Callback, 2) -> + is_function(Callback, 2) +-> fabric_view_all_docs:go(dbname(DbName), opts(Options), QueryArgs, Callback, Acc0); - %% @doc convenience function that takes a keylist rather than a record %% @equiv all_docs(DbName, Callback, Acc0, kl_to_query_args(QueryArgs)) all_docs(DbName, Options, Callback, Acc0, QueryArgs) -> all_docs(DbName, Options, Callback, Acc0, kl_to_query_args(QueryArgs)). - --spec changes(dbname(), callback(), any(), #changes_args{} | [{atom(),any()}]) -> +-spec changes(dbname(), callback(), any(), #changes_args{} | [{atom(), any()}]) -> {ok, any()}. -changes(DbName, Callback, Acc0, #changes_args{}=Options) -> +changes(DbName, Callback, Acc0, #changes_args{} = Options) -> Feed = Options#changes_args.feed, fabric_view_changes:go(dbname(DbName), Feed, Options, Callback, Acc0); - %% @doc convenience function, takes keylist instead of record %% @equiv changes(DbName, Callback, Acc0, kl_to_changes_args(Options)) changes(DbName, Callback, Acc0, Options) -> @@ -375,22 +433,28 @@ query_view(DbName, DesignName, ViewName, QueryArgs) -> Callback = fun default_callback/2, query_view(DbName, DesignName, ViewName, Callback, [], QueryArgs). - %% @equiv query_view(DbName, DesignName, [], %% ViewName, fun default_callback/2, [], QueryArgs) query_view(DbName, DDoc, ViewName, Callback, Acc, QueryArgs) -> query_view(DbName, [], DDoc, ViewName, Callback, Acc, QueryArgs). - %% @doc execute a given view. %% There are many additional query args that can be passed to a view, %% see <a href="http://wiki.apache.org/couchdb/HTTP_view_API#Querying_Options"> %% query args</a> for details. --spec query_view(dbname(), [{atom(), any()}] | [], - #doc{} | binary(), iodata(), callback(), any(), #mrargs{}) -> +-spec query_view( + dbname(), + [{atom(), any()}] | [], + #doc{} | binary(), + iodata(), + callback(), + any(), + #mrargs{} +) -> any(). -query_view(Db, Options, GroupId, ViewName, Callback, Acc0, QueryArgs) - when is_binary(GroupId) -> +query_view(Db, Options, GroupId, ViewName, Callback, Acc0, QueryArgs) when + is_binary(GroupId) +-> DbName = dbname(Db), {ok, DDoc} = ddoc_cache:open(DbName, <<"_design/", GroupId/binary>>), query_view(Db, Options, DDoc, ViewName, Callback, Acc0, QueryArgs); @@ -398,13 +462,13 @@ query_view(Db, Options, DDoc, ViewName, Callback, Acc0, QueryArgs0) -> DbName = dbname(Db), View = name(ViewName), case fabric_util:is_users_db(DbName) of - true -> - FakeDb = fabric_util:open_cluster_db(DbName, Options), - couch_users_db:after_doc_read(DDoc, FakeDb); - false -> - ok + true -> + FakeDb = fabric_util:open_cluster_db(DbName, Options), + couch_users_db:after_doc_read(DDoc, FakeDb); + false -> + ok end, - {ok, #mrst{views=Views, language=Lang}} = + {ok, #mrst{views = Views, language = Lang}} = couch_mrview_util:ddoc_to_mrst(DbName, DDoc), QueryArgs1 = couch_mrview_util:set_view_type(QueryArgs0, View, Views), QueryArgs2 = fabric_util:validate_args(Db, DDoc, QueryArgs1), @@ -437,24 +501,24 @@ query_view(Db, Options, DDoc, ViewName, Callback, Acc0, QueryArgs0) -> %% is running and so forth -spec get_view_group_info(dbname(), #doc{} | docid()) -> {ok, [ - {signature, binary()} | - {language, binary()} | - {disk_size, non_neg_integer()} | - {compact_running, boolean()} | - {updater_running, boolean()} | - {waiting_commit, boolean()} | - {waiting_clients, non_neg_integer()} | - {update_seq, pos_integer()} | - {purge_seq, non_neg_integer()} | - {sizes, [ - {active, non_neg_integer()} | - {external, non_neg_integer()} | - {file, non_neg_integer()} - ]} | - {updates_pending, [ - {minimum, non_neg_integer()} | - {preferred, non_neg_integer()} | - {total, non_neg_integer()} + {signature, binary()} + | {language, binary()} + | {disk_size, non_neg_integer()} + | {compact_running, boolean()} + | {updater_running, boolean()} + | {waiting_commit, boolean()} + | {waiting_clients, non_neg_integer()} + | {update_seq, pos_integer()} + | {purge_seq, non_neg_integer()} + | {sizes, [ + {active, non_neg_integer()} + | {external, non_neg_integer()} + | {file, non_neg_integer()} + ]} + | {updates_pending, [ + {minimum, non_neg_integer()} + | {preferred, non_neg_integer()} + | {total, non_neg_integer()} ]} ]}. get_view_group_info(DbName, DesignId) -> @@ -467,23 +531,25 @@ end_changes() -> %% @doc retrieve all the design docs from a database -spec design_docs(dbname()) -> {ok, [json_obj()]} | {error, Reason :: term()}. design_docs(DbName) -> - Extra = case get(io_priority) of - undefined -> []; - Else -> [{io_priority, Else}] - end, + Extra = + case get(io_priority) of + undefined -> []; + Else -> [{io_priority, Else}] + end, QueryArgs0 = #mrargs{ - include_docs=true, - extra=Extra + include_docs = true, + extra = Extra }, QueryArgs = set_namespace(<<"_design">>, QueryArgs0), - Callback = fun({meta, _}, []) -> - {ok, []}; - ({row, Props}, Acc) -> - {ok, [couch_util:get_value(doc, Props) | Acc]}; - (complete, Acc) -> - {ok, lists:reverse(Acc)}; - ({error, Reason}, _Acc) -> - {error, Reason} + Callback = fun + ({meta, _}, []) -> + {ok, []}; + ({row, Props}, Acc) -> + {ok, [couch_util:get_value(doc, Props) | Acc]}; + (complete, Acc) -> + {ok, lists:reverse(Acc)}; + ({error, Reason}, _Acc) -> + {error, Reason} end, fabric:all_docs(dbname(DbName), [?ADMIN_CTX], Callback, [], QueryArgs). @@ -492,8 +558,10 @@ design_docs(DbName) -> %% NOTE: This function probably doesn't belong here as part fo the API -spec reset_validation_funs(dbname()) -> [reference()]. reset_validation_funs(DbName) -> - [rexi:cast(Node, {fabric_rpc, reset_validation_funs, [Name]}) || - #shard{node=Node, name=Name} <- mem3:shards(DbName)]. + [ + rexi:cast(Node, {fabric_rpc, reset_validation_funs, [Name]}) + || #shard{node = Node, name = Name} <- mem3:shards(DbName) + ]. %% @doc clean up index files for all Dbs -spec cleanup_index_files() -> [ok]. @@ -504,15 +572,19 @@ cleanup_index_files() -> %% @doc clean up index files for a specific db -spec cleanup_index_files(dbname()) -> ok. cleanup_index_files(DbName) -> - try lists:foreach( - fun(File) -> - file:delete(File) - end, inactive_index_files(DbName)) + try + lists:foreach( + fun(File) -> + file:delete(File) + end, + inactive_index_files(DbName) + ) catch error:Error -> couch_log:error( "~p:cleanup_index_files. Error: ~p", - [?MODULE, Error]), + [?MODULE, Error] + ), ok end. @@ -521,31 +593,48 @@ cleanup_index_files(DbName) -> inactive_index_files(DbName) -> {ok, DesignDocs} = fabric:design_docs(DbName), - ActiveSigs = maps:from_list(lists:map(fun(#doc{id = GroupId}) -> - {ok, Info} = fabric:get_view_group_info(DbName, GroupId), - {binary_to_list(couch_util:get_value(signature, Info)), nil} - end, [couch_doc:from_json_obj(DD) || DD <- DesignDocs])), - - FileList = lists:flatmap(fun(#shard{name = ShardName}) -> - IndexDir = couch_index_util:index_dir(mrview, ShardName), - filelib:wildcard([IndexDir, "/*"]) - end, mem3:local_shards(dbname(DbName))), - - if ActiveSigs =:= [] -> FileList; true -> - %% <sig>.view and <sig>.compact.view where <sig> is in ActiveSigs - %% will be excluded from FileList because they are active view - %% files and should not be deleted. - lists:filter(fun(FilePath) -> - not maps:is_key(get_view_sig_from_filename(FilePath), ActiveSigs) - end, FileList) + ActiveSigs = maps:from_list( + lists:map( + fun(#doc{id = GroupId}) -> + {ok, Info} = fabric:get_view_group_info(DbName, GroupId), + {binary_to_list(couch_util:get_value(signature, Info)), nil} + end, + [couch_doc:from_json_obj(DD) || DD <- DesignDocs] + ) + ), + + FileList = lists:flatmap( + fun(#shard{name = ShardName}) -> + IndexDir = couch_index_util:index_dir(mrview, ShardName), + filelib:wildcard([IndexDir, "/*"]) + end, + mem3:local_shards(dbname(DbName)) + ), + + if + ActiveSigs =:= [] -> + FileList; + true -> + %% <sig>.view and <sig>.compact.view where <sig> is in ActiveSigs + %% will be excluded from FileList because they are active view + %% files and should not be deleted. + lists:filter( + fun(FilePath) -> + not maps:is_key(get_view_sig_from_filename(FilePath), ActiveSigs) + end, + FileList + ) end. %% @doc clean up index files for a specific db on all nodes -spec cleanup_index_files_all_nodes(dbname()) -> [reference()]. cleanup_index_files_all_nodes(DbName) -> - lists:foreach(fun(Node) -> - rexi:cast(Node, {?MODULE, cleanup_index_files, [DbName]}) - end, mem3:nodes()). + lists:foreach( + fun(Node) -> + rexi:cast(Node, {?MODULE, cleanup_index_files, [DbName]}) + end, + mem3:nodes() + ). %% some simple type validation and transcoding dbname(DbName) when is_list(DbName) -> @@ -555,12 +644,13 @@ dbname(DbName) when is_binary(DbName) -> dbname(Db) -> try couch_db:name(Db) - catch error:badarg -> - erlang:error({illegal_database_name, Db}) + catch + error:badarg -> + erlang:error({illegal_database_name, Db}) end. %% @doc get db shard uuids --spec db_uuids(dbname()) -> map(). +-spec db_uuids(dbname()) -> map(). db_uuids(DbName) -> fabric_db_uuids:go(dbname(DbName)). @@ -580,15 +670,16 @@ docs(_Db, Docs) -> doc(_Db, #doc{} = Doc) -> Doc; doc(Db0, {_} = Doc) -> - Db = case couch_db:is_db(Db0) of - true -> - Db0; - false -> - Shard = hd(mem3:shards(Db0)), - Props = couch_util:get_value(props, Shard#shard.opts, []), - {ok, Db1} = couch_db:clustered_db(Db0, [{props, Props}]), - Db1 - end, + Db = + case couch_db:is_db(Db0) of + true -> + Db0; + false -> + Shard = hd(mem3:shards(Db0)), + Props = couch_util:get_value(props, Shard#shard.opts, []), + {ok, Db1} = couch_db:clustered_db(Db0, [{props, Props}]), + Db1 + end, couch_db:doc_from_json_obj_validate(Db, Doc); doc(_Db, Doc) -> erlang:error({illegal_doc_format, Doc}). @@ -616,15 +707,15 @@ opts(Options) -> add_option(Key, Options) -> case couch_util:get_value(Key, Options) of - undefined -> - case erlang:get(Key) of undefined -> - Options; - Value -> - [{Key, Value} | Options] - end; - _ -> - Options + case erlang:get(Key) of + undefined -> + Options; + Value -> + [{Key, Value} | Options] + end; + _ -> + Options end. default_callback(complete, Acc) -> @@ -632,7 +723,7 @@ default_callback(complete, Acc) -> default_callback(Row, Acc) -> {ok, [Row | Acc]}. -is_reduce_view(#mrargs{view_type=ViewType}) -> +is_reduce_view(#mrargs{view_type = ViewType}) -> ViewType =:= red; is_reduce_view({Reduce, _, _}) -> Reduce =:= red. @@ -651,29 +742,38 @@ kl_to_query_args(KeyList) -> %% note that record_info is only known at compile time %% so the code must be written in this way. For each new %% record type add a case clause -lookup_index(Key,RecName) -> +lookup_index(Key, RecName) -> Indexes = case RecName of - changes_args -> - lists:zip(record_info(fields, changes_args), - lists:seq(2, record_info(size, changes_args))); - mrargs -> - lists:zip(record_info(fields, mrargs), - lists:seq(2, record_info(size, mrargs))) + changes_args -> + lists:zip( + record_info(fields, changes_args), + lists:seq(2, record_info(size, changes_args)) + ); + mrargs -> + lists:zip( + record_info(fields, mrargs), + lists:seq(2, record_info(size, mrargs)) + ) end, couch_util:get_value(Key, Indexes). %% @doc convert a keylist to record with given `RecName' %% @see lookup_index -kl_to_record(KeyList,RecName) -> - Acc0 = case RecName of - changes_args -> #changes_args{}; - mrargs -> #mrargs{} - end, - lists:foldl(fun({Key, Value}, Acc) -> - Index = lookup_index(couch_util:to_existing_atom(Key),RecName), - setelement(Index, Acc, Value) - end, Acc0, KeyList). +kl_to_record(KeyList, RecName) -> + Acc0 = + case RecName of + changes_args -> #changes_args{}; + mrargs -> #mrargs{} + end, + lists:foldl( + fun({Key, Value}, Acc) -> + Index = lookup_index(couch_util:to_existing_atom(Key), RecName), + setelement(Index, Acc, Value) + end, + Acc0, + KeyList + ). set_namespace(NS, #mrargs{extra = Extra} = Args) -> Args#mrargs{extra = [{namespace, NS} | Extra]}. @@ -686,11 +786,16 @@ get_view_sig_from_filename(FilePath) -> update_doc_test_() -> { - "Update doc tests", { - setup, fun setup/0, fun teardown/1, - fun(Ctx) -> [ - should_throw_conflict(Ctx) - ] end + "Update doc tests", + { + setup, + fun setup/0, + fun teardown/1, + fun(Ctx) -> + [ + should_throw_conflict(Ctx) + ] + end } }. @@ -699,35 +804,39 @@ should_throw_conflict(Doc) -> ?assertThrow(conflict, update_doc(<<"test-db">>, Doc, [])) end). - setup() -> Doc = #doc{ id = <<"test_doc">>, - revs = {3, [<<5,68,252,180,43,161,216,223,26,119,71,219,212,229, - 159,113>>]}, - body = {[{<<"foo">>,<<"asdf">>},{<<"author">>,<<"tom">>}]}, - atts = [], deleted = false, meta = [] + revs = {3, [<<5, 68, 252, 180, 43, 161, 216, 223, 26, 119, 71, 219, 212, 229, 159, 113>>]}, + body = {[{<<"foo">>, <<"asdf">>}, {<<"author">>, <<"tom">>}]}, + atts = [], + deleted = false, + meta = [] }, ok = application:ensure_started(config), ok = meck:expect(mem3, shards, fun(_, _) -> [] end), ok = meck:expect(mem3, quorum, fun(_) -> 1 end), ok = meck:expect(rexi, cast, fun(_, _) -> ok end), - ok = meck:expect(rexi_utils, recv, + ok = meck:expect( + rexi_utils, + recv, fun(_, _, _, _, _, _) -> {ok, {error, [{Doc, conflict}]}} - end), - ok = meck:expect(couch_util, reorder_results, + end + ), + ok = meck:expect( + couch_util, + reorder_results, fun(_, [{_, Res}]) -> [Res] - end), + end + ), ok = meck:expect(fabric_util, create_monitors, fun(_) -> ok end), ok = meck:expect(rexi_monitor, stop, fun(_) -> ok end), Doc. - teardown(_) -> meck:unload(), ok = application:stop(config). - -endif. diff --git a/src/fabric/src/fabric_db_create.erl b/src/fabric/src/fabric_db_create.erl index 1048cb0bb..38770aea4 100644 --- a/src/fabric/src/fabric_db_create.erl +++ b/src/fabric/src/fabric_db_create.erl @@ -17,61 +17,60 @@ -include_lib("mem3/include/mem3.hrl"). -include_lib("couch/include/couch_db.hrl"). - %% @doc Create a new database, and all its partition files across the cluster %% Options is proplist with user_ctx, n, q, validate_name go(DbName, Options) -> case validate_dbname(DbName, Options) of - ok -> - couch_partition:validate_dbname(DbName, Options), - case db_exists(DbName) of - true -> - {error, file_exists}; - false -> - {Shards, Doc} = generate_shard_map(DbName, Options), - CreateShardResult = create_shard_files(Shards, Options), - case CreateShardResult of - enametoolong -> - {error, {database_name_too_long, DbName}}; - _ -> - case {CreateShardResult, create_shard_db_doc(Doc)} of - {ok, {ok, Status}} -> - Status; - {ok, {error, conflict} = ShardDocError} -> - % Check if it is just a race to create the shard doc - case db_exists(DbName) of - true -> {error, file_exists}; - false -> ShardDocError - end; - {file_exists, {ok, _}} -> + ok -> + couch_partition:validate_dbname(DbName, Options), + case db_exists(DbName) of + true -> {error, file_exists}; - {_, Error} -> - Error - end - end - end; - Error -> - Error + false -> + {Shards, Doc} = generate_shard_map(DbName, Options), + CreateShardResult = create_shard_files(Shards, Options), + case CreateShardResult of + enametoolong -> + {error, {database_name_too_long, DbName}}; + _ -> + case {CreateShardResult, create_shard_db_doc(Doc)} of + {ok, {ok, Status}} -> + Status; + {ok, {error, conflict} = ShardDocError} -> + % Check if it is just a race to create the shard doc + case db_exists(DbName) of + true -> {error, file_exists}; + false -> ShardDocError + end; + {file_exists, {ok, _}} -> + {error, file_exists}; + {_, Error} -> + Error + end + end + end; + Error -> + Error end. validate_dbname(DbName, Options) -> case couch_util:get_value(validate_name, Options, true) of - false -> - ok; - true -> - couch_db:validate_dbname(DbName) + false -> + ok; + true -> + couch_db:validate_dbname(DbName) end. generate_shard_map(DbName, Options) -> {MegaSecs, Secs, _} = os:timestamp(), - Suffix = "." ++ integer_to_list(MegaSecs*1000000 + Secs), - Shards = mem3:choose_shards(DbName, [{shard_suffix,Suffix} | Options]), + Suffix = "." ++ integer_to_list(MegaSecs * 1000000 + Secs), + Shards = mem3:choose_shards(DbName, [{shard_suffix, Suffix} | Options]), case mem3_util:open_db_doc(DbName) of - {ok, Doc} -> - % the DB already exists, and may have a different Suffix - ok; - {not_found, _} -> - Doc = make_document(Shards, Suffix, Options) + {ok, Doc} -> + % the DB already exists, and may have a different Suffix + ok; + {not_found, _} -> + Doc = make_document(Shards, Suffix, Options) end, {Shards, Doc}. @@ -79,56 +78,53 @@ create_shard_files(Shards, Options) -> Workers = fabric_util:submit_jobs(Shards, create_db, [Options]), RexiMon = fabric_util:create_monitors(Shards), try fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Workers) of - {error, file_exists} -> - file_exists; - {error, enametoolong} -> - enametoolong; - {timeout, DefunctWorkers} -> - fabric_util:log_timeout(DefunctWorkers, "create_db"), - {error, timeout}; - _ -> - ok + {error, file_exists} -> + file_exists; + {error, enametoolong} -> + enametoolong; + {timeout, DefunctWorkers} -> + fabric_util:log_timeout(DefunctWorkers, "create_db"), + {error, timeout}; + _ -> + ok after rexi_monitor:stop(RexiMon) end. handle_message({error, enametoolong}, _, _) -> {error, enametoolong}; - handle_message(file_exists, _, _) -> {error, file_exists}; - handle_message({rexi_DOWN, _, {_, Node}, _}, _, Workers) -> case lists:filter(fun(S) -> S#shard.node =/= Node end, Workers) of - [] -> - {stop, ok}; - RemainingWorkers -> - {ok, RemainingWorkers} + [] -> + {stop, ok}; + RemainingWorkers -> + {ok, RemainingWorkers} end; - handle_message(_, Worker, Workers) -> case lists:delete(Worker, Workers) of - [] -> - {stop, ok}; - RemainingWorkers -> - {ok, RemainingWorkers} + [] -> + {stop, ok}; + RemainingWorkers -> + {ok, RemainingWorkers} end. create_shard_db_doc(Doc) -> - Shards = [#shard{node=N} || N <- mem3:nodes()], + Shards = [#shard{node = N} || N <- mem3:nodes()], RexiMon = fabric_util:create_monitors(Shards), Workers = fabric_util:submit_jobs(Shards, create_shard_db_doc, [Doc]), Acc0 = {length(Shards), fabric_dict:init(Workers, nil)}, try fabric_util:recv(Workers, #shard.ref, fun handle_db_update/3, Acc0) of - {timeout, {_, WorkersDict}} -> - DefunctWorkers = fabric_util:remove_done_workers(WorkersDict, nil), - fabric_util:log_timeout( - DefunctWorkers, - "create_shard_db_doc" - ), - {error, timeout}; - Else -> - Else + {timeout, {_, WorkersDict}} -> + DefunctWorkers = fabric_util:remove_done_workers(WorkersDict, nil), + fabric_util:log_timeout( + DefunctWorkers, + "create_shard_db_doc" + ), + {error, timeout}; + Else -> + Else after rexi_monitor:stop(RexiMon) end. @@ -136,57 +132,68 @@ create_shard_db_doc(Doc) -> handle_db_update({rexi_DOWN, _, {_, Node}, _}, _Worker, {W, Counters}) -> New = fabric_dict:filter(fun(S, _) -> S#shard.node =/= Node end, Counters), maybe_stop(W, New); - handle_db_update({rexi_EXIT, _Reason}, Worker, {W, Counters}) -> maybe_stop(W, fabric_dict:erase(Worker, Counters)); - handle_db_update(conflict, _, _) -> % just fail when we get any conflicts {error, conflict}; - handle_db_update(Msg, Worker, {W, Counters}) -> maybe_stop(W, fabric_dict:store(Worker, Msg, Counters)). maybe_stop(W, Counters) -> case fabric_dict:any(nil, Counters) of - true -> - {ok, {W, Counters}}; - false -> - case lists:sum([1 || {_, ok} <- Counters]) of - NumOk when NumOk >= (W div 2 +1) -> - {stop, ok}; - NumOk when NumOk > 0 -> - {stop, accepted}; - _ -> - {error, internal_server_error} - end + true -> + {ok, {W, Counters}}; + false -> + case lists:sum([1 || {_, ok} <- Counters]) of + NumOk when NumOk >= (W div 2 + 1) -> + {stop, ok}; + NumOk when NumOk > 0 -> + {stop, accepted}; + _ -> + {error, internal_server_error} + end end. -make_document([#shard{dbname=DbName}|_] = Shards, Suffix, Options) -> +make_document([#shard{dbname = DbName} | _] = Shards, Suffix, Options) -> {RawOut, ByNodeOut, ByRangeOut} = - lists:foldl(fun(#shard{node=N, range=[B,E]}, {Raw, ByNode, ByRange}) -> - Range = ?l2b([couch_util:to_hex(<<B:32/integer>>), "-", - couch_util:to_hex(<<E:32/integer>>)]), - Node = couch_util:to_binary(N), - {[[<<"add">>, Range, Node] | Raw], orddict:append(Node, Range, ByNode), - orddict:append(Range, Node, ByRange)} - end, {[], [], []}, Shards), - EngineProp = case couch_util:get_value(engine, Options) of - E when is_binary(E) -> [{<<"engine">>, E}]; - _ -> [] - end, - DbProps = case couch_util:get_value(props, Options) of - Props when is_list(Props) -> [{<<"props">>, {Props}}]; - _ -> [] - end, + lists:foldl( + fun(#shard{node = N, range = [B, E]}, {Raw, ByNode, ByRange}) -> + Range = ?l2b([ + couch_util:to_hex(<<B:32/integer>>), + "-", + couch_util:to_hex(<<E:32/integer>>) + ]), + Node = couch_util:to_binary(N), + { + [[<<"add">>, Range, Node] | Raw], + orddict:append(Node, Range, ByNode), + orddict:append(Range, Node, ByRange) + } + end, + {[], [], []}, + Shards + ), + EngineProp = + case couch_util:get_value(engine, Options) of + E when is_binary(E) -> [{<<"engine">>, E}]; + _ -> [] + end, + DbProps = + case couch_util:get_value(props, Options) of + Props when is_list(Props) -> [{<<"props">>, {Props}}]; + _ -> [] + end, #doc{ id = DbName, - body = {[ - {<<"shard_suffix">>, Suffix}, - {<<"changelog">>, lists:sort(RawOut)}, - {<<"by_node">>, {[{K,lists:sort(V)} || {K,V} <- ByNodeOut]}}, - {<<"by_range">>, {[{K,lists:sort(V)} || {K,V} <- ByRangeOut]}} - ] ++ EngineProp ++ DbProps} + body = { + [ + {<<"shard_suffix">>, Suffix}, + {<<"changelog">>, lists:sort(RawOut)}, + {<<"by_node">>, {[{K, lists:sort(V)} || {K, V} <- ByNodeOut]}}, + {<<"by_range">>, {[{K, lists:sort(V)} || {K, V} <- ByRangeOut]}} + ] ++ EngineProp ++ DbProps + } }. db_exists(DbName) -> is_list(catch mem3:shards(DbName)). @@ -205,24 +212,20 @@ db_exists_test_() -> ] }. - setup_all() -> meck:new(mem3). - teardown_all(_) -> meck:unload(). - db_exists_for_existing_db() -> Mock = fun(DbName) when is_binary(DbName) -> - [#shard{dbname = DbName, range = [0,100]}] + [#shard{dbname = DbName, range = [0, 100]}] end, ok = meck:expect(mem3, shards, Mock), ?assertEqual(true, db_exists(<<"foobar">>)), ?assertEqual(true, meck:validate(mem3)). - db_exists_for_missing_db() -> Mock = fun(DbName) -> erlang:error(database_does_not_exist, DbName) diff --git a/src/fabric/src/fabric_db_delete.erl b/src/fabric/src/fabric_db_delete.erl index c146cb6cd..a257b0d6e 100644 --- a/src/fabric/src/fabric_db_delete.erl +++ b/src/fabric/src/fabric_db_delete.erl @@ -24,34 +24,34 @@ go(DbName, _Options) -> Shards = mem3:shards(DbName), % delete doc from shard_db try delete_shard_db_doc(DbName) of - {ok, ok} -> - ok; - {ok, accepted} -> - accepted; - {ok, not_found} -> - erlang:error(database_does_not_exist, DbName); - Error -> - Error + {ok, ok} -> + ok; + {ok, accepted} -> + accepted; + {ok, not_found} -> + erlang:error(database_does_not_exist, DbName); + Error -> + Error after % delete the shard files fabric_util:submit_jobs(Shards, delete_db, []) end. delete_shard_db_doc(Doc) -> - Shards = [#shard{node=N} || N <- mem3:nodes()], + Shards = [#shard{node = N} || N <- mem3:nodes()], RexiMon = fabric_util:create_monitors(Shards), Workers = fabric_util:submit_jobs(Shards, delete_shard_db_doc, [Doc]), Acc0 = {length(Shards), fabric_dict:init(Workers, nil)}, try fabric_util:recv(Workers, #shard.ref, fun handle_db_update/3, Acc0) of - {timeout, {_, WorkersDict}} -> - DefunctWorkers = fabric_util:remove_done_workers(WorkersDict, nil), - fabric_util:log_timeout( - DefunctWorkers, - "delete_shard_db_doc" - ), - {error, timeout}; - Else -> - Else + {timeout, {_, WorkersDict}} -> + DefunctWorkers = fabric_util:remove_done_workers(WorkersDict, nil), + fabric_util:log_timeout( + DefunctWorkers, + "delete_shard_db_doc" + ), + {error, timeout}; + Else -> + Else after rexi_monitor:stop(RexiMon) end. @@ -59,40 +59,37 @@ delete_shard_db_doc(Doc) -> handle_db_update({rexi_DOWN, _, {_, Node}, _}, _Worker, {W, Counters}) -> New = fabric_dict:filter(fun(S, _) -> S#shard.node =/= Node end, Counters), maybe_stop(W, New); - handle_db_update({rexi_EXIT, _Reason}, Worker, {W, Counters}) -> maybe_stop(W, fabric_dict:erase(Worker, Counters)); - handle_db_update(conflict, _, _) -> % just fail when we get any conflicts {error, conflict}; - handle_db_update(Msg, Worker, {W, Counters}) -> maybe_stop(W, fabric_dict:store(Worker, Msg, Counters)). maybe_stop(W, Counters) -> case fabric_dict:any(nil, Counters) of - true -> - {ok, {W, Counters}}; - false -> - {Ok,NotFound} = fabric_dict:fold(fun count_replies/3, {0,0}, Counters), - case {Ok + NotFound, Ok, NotFound} of - {W, 0, W} -> - {#shard{dbname=Name}, _} = hd(Counters), - couch_log:warning("~p not_found ~d", [?MODULE, Name]), - {stop, not_found}; - {W, _, _} -> - {stop, ok}; - {_, M, _} when M > 0 -> - {stop,accepted}; - _ -> - {error, internal_server_error} - end + true -> + {ok, {W, Counters}}; + false -> + {Ok, NotFound} = fabric_dict:fold(fun count_replies/3, {0, 0}, Counters), + case {Ok + NotFound, Ok, NotFound} of + {W, 0, W} -> + {#shard{dbname = Name}, _} = hd(Counters), + couch_log:warning("~p not_found ~d", [?MODULE, Name]), + {stop, not_found}; + {W, _, _} -> + {stop, ok}; + {_, M, _} when M > 0 -> + {stop, accepted}; + _ -> + {error, internal_server_error} + end end. count_replies(_, ok, {Ok, NotFound}) -> - {Ok+1, NotFound}; + {Ok + 1, NotFound}; count_replies(_, not_found, {Ok, NotFound}) -> - {Ok, NotFound+1}; + {Ok, NotFound + 1}; count_replies(_, _, Acc) -> Acc. diff --git a/src/fabric/src/fabric_db_doc_count.erl b/src/fabric/src/fabric_db_doc_count.erl index a91014b7c..b2ab35b81 100644 --- a/src/fabric/src/fabric_db_doc_count.erl +++ b/src/fabric/src/fabric_db_doc_count.erl @@ -24,28 +24,26 @@ go(DbName) -> RexiMon = fabric_util:create_monitors(Shards), Acc0 = {fabric_dict:init(Workers, nil), []}, try fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of - {timeout, {WorkersDict, _}} -> - DefunctWorkers = fabric_util:remove_done_workers(WorkersDict, nil), - fabric_util:log_timeout(DefunctWorkers, "get_doc_count"), - {error, timeout}; - Else -> - Else + {timeout, {WorkersDict, _}} -> + DefunctWorkers = fabric_util:remove_done_workers(WorkersDict, nil), + fabric_util:log_timeout(DefunctWorkers, "get_doc_count"), + {error, timeout}; + Else -> + Else after rexi_monitor:stop(RexiMon) end. -handle_message({rexi_DOWN, _, {_,NodeRef},_}, _Shard, {Counters, Resps}) -> +handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _Shard, {Counters, Resps}) -> case fabric_ring:node_down(NodeRef, Counters, Resps) of {ok, Counters1} -> {ok, {Counters1, Resps}}; error -> {error, {nodedown, <<"progress not possible">>}} end; - handle_message({rexi_EXIT, Reason}, Shard, {Counters, Resps}) -> case fabric_ring:handle_error(Shard, Counters, Resps) of {ok, Counters1} -> {ok, {Counters1, Resps}}; error -> {error, Reason} end; - handle_message({ok, Count}, Shard, {Counters, Resps}) -> case fabric_ring:handle_response(Shard, Count, Counters, Resps) of {ok, {Counters1, Resps1}} -> @@ -54,7 +52,6 @@ handle_message({ok, Count}, Shard, {Counters, Resps}) -> Total = fabric_dict:fold(fun(_, C, A) -> A + C end, 0, Resps1), {stop, Total} end; - handle_message(Reason, Shard, {Counters, Resps}) -> case fabric_ring:handle_error(Shard, Counters, Resps) of {ok, Counters1} -> {ok, {Counters1, Resps}}; diff --git a/src/fabric/src/fabric_db_info.erl b/src/fabric/src/fabric_db_info.erl index 586f282c2..2366420c8 100644 --- a/src/fabric/src/fabric_db_info.erl +++ b/src/fabric/src/fabric_db_info.erl @@ -23,11 +23,10 @@ go(DbName) -> RexiMon = fabric_util:create_monitors(Shards), Fun = fun handle_message/3, {ok, ClusterInfo} = get_cluster_info(Shards), - CInfo = [{cluster, ClusterInfo}], + CInfo = [{cluster, ClusterInfo}], Acc0 = {fabric_dict:init(Workers, nil), [], CInfo}, try case fabric_util:recv(Workers, #shard.ref, Fun, Acc0) of - {ok, Acc} -> {ok, Acc}; {timeout, {WorkersDict, _, _}} -> @@ -47,19 +46,16 @@ go(DbName) -> rexi_monitor:stop(RexiMon) end. - -handle_message({rexi_DOWN, _, {_,NodeRef},_}, _, {Counters, Resps, CInfo}) -> +handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _, {Counters, Resps, CInfo}) -> case fabric_ring:node_down(NodeRef, Counters, Resps) of {ok, Counters1} -> {ok, {Counters1, Resps, CInfo}}; error -> {error, {nodedown, <<"progress not possible">>}} end; - handle_message({rexi_EXIT, Reason}, Shard, {Counters, Resps, CInfo}) -> case fabric_ring:handle_error(Shard, Counters, Resps) of {ok, Counters1} -> {ok, {Counters1, Resps, CInfo}}; error -> {error, Reason} end; - handle_message({ok, Info}, Shard, {Counters, Resps, CInfo}) -> case fabric_ring:handle_response(Shard, Info, Counters, Resps) of {ok, {Counters1, Resps1}} -> @@ -67,84 +63,102 @@ handle_message({ok, Info}, Shard, {Counters, Resps, CInfo}) -> {stop, Resps1} -> {stop, build_final_response(CInfo, Shard#shard.dbname, Resps1)} end; - handle_message(Reason, Shard, {Counters, Resps, CInfo}) -> case fabric_ring:handle_error(Shard, Counters, Resps) of {ok, Counters1} -> {ok, {Counters1, Resps, CInfo}}; error -> {error, Reason} end. - build_final_response(CInfo, DbName, Responses) -> - AccF = fabric_dict:fold(fun(Shard, Info, {Seqs, PSeqs, Infos}) -> - Seq = build_seq(Shard, Info), - PSeq = couch_util:get_value(purge_seq, Info), - {[{Shard, Seq} | Seqs], [{Shard, PSeq} | PSeqs], [Info | Infos]} - end, {[], [], []}, Responses), + AccF = fabric_dict:fold( + fun(Shard, Info, {Seqs, PSeqs, Infos}) -> + Seq = build_seq(Shard, Info), + PSeq = couch_util:get_value(purge_seq, Info), + {[{Shard, Seq} | Seqs], [{Shard, PSeq} | PSeqs], [Info | Infos]} + end, + {[], [], []}, + Responses + ), {Seqs, PSeqs, Infos} = AccF, - PackedSeq = fabric_view_changes:pack_seqs(Seqs), + PackedSeq = fabric_view_changes:pack_seqs(Seqs), PackedPSeq = fabric_view_changes:pack_seqs(PSeqs), MergedInfos = merge_results(lists:flatten([CInfo | Infos])), Sequences = [{purge_seq, PackedPSeq}, {update_seq, PackedSeq}], [{db_name, DbName}] ++ Sequences ++ MergedInfos. - build_seq(#shard{node = Node}, Info) when is_list(Info) -> Seq = couch_util:get_value(update_seq, Info), Uuid = couch_util:get_value(uuid, Info), PrefixLen = fabric_util:get_uuid_prefix_len(), {Seq, binary:part(Uuid, {0, PrefixLen}), Node}. - merge_results(Info) -> - Dict = lists:foldl(fun({K,V},D0) -> orddict:append(K,V,D0) end, - orddict:new(), Info), - orddict:fold(fun - (doc_count, X, Acc) -> - [{doc_count, lists:sum(X)} | Acc]; - (doc_del_count, X, Acc) -> - [{doc_del_count, lists:sum(X)} | Acc]; - (compact_running, X, Acc) -> - [{compact_running, lists:member(true, X)} | Acc]; - (sizes, X, Acc) -> - [{sizes, {merge_object(X)}} | Acc]; - (disk_format_version, X, Acc) -> - [{disk_format_version, lists:max(X)} | Acc]; - (cluster, [X], Acc) -> - [{cluster, {X}} | Acc]; - (props, Xs, Acc) -> - [{props, {merge_object(Xs)}} | Acc]; - (_K, _V, Acc) -> - Acc - end, [{instance_start_time, <<"0">>}], Dict). + Dict = lists:foldl( + fun({K, V}, D0) -> orddict:append(K, V, D0) end, + orddict:new(), + Info + ), + orddict:fold( + fun + (doc_count, X, Acc) -> + [{doc_count, lists:sum(X)} | Acc]; + (doc_del_count, X, Acc) -> + [{doc_del_count, lists:sum(X)} | Acc]; + (compact_running, X, Acc) -> + [{compact_running, lists:member(true, X)} | Acc]; + (sizes, X, Acc) -> + [{sizes, {merge_object(X)}} | Acc]; + (disk_format_version, X, Acc) -> + [{disk_format_version, lists:max(X)} | Acc]; + (cluster, [X], Acc) -> + [{cluster, {X}} | Acc]; + (props, Xs, Acc) -> + [{props, {merge_object(Xs)}} | Acc]; + (_K, _V, Acc) -> + Acc + end, + [{instance_start_time, <<"0">>}], + Dict + ). merge_object(Objects) -> - Dict = lists:foldl(fun({Props}, D) -> - lists:foldl(fun({K,V},D0) -> orddict:append(K,V,D0) end, D, Props) - end, orddict:new(), Objects), - orddict:fold(fun - (Key, [X | _] = Xs, Acc) when is_integer(X) -> - [{Key, lists:sum(Xs)} | Acc]; - (Key, [X | _] = Xs, Acc) when is_boolean(X) -> - [{Key, lists:all(fun all_true/1, Xs)} | Acc]; - (_Key, _Xs, Acc) -> - Acc - end, [], Dict). + Dict = lists:foldl( + fun({Props}, D) -> + lists:foldl(fun({K, V}, D0) -> orddict:append(K, V, D0) end, D, Props) + end, + orddict:new(), + Objects + ), + orddict:fold( + fun + (Key, [X | _] = Xs, Acc) when is_integer(X) -> + [{Key, lists:sum(Xs)} | Acc]; + (Key, [X | _] = Xs, Acc) when is_boolean(X) -> + [{Key, lists:all(fun all_true/1, Xs)} | Acc]; + (_Key, _Xs, Acc) -> + Acc + end, + [], + Dict + ). all_true(true) -> true; all_true(_) -> false. get_cluster_info(Shards) -> - Dict = lists:foldl(fun(#shard{range = R}, Acc) -> - dict:update_counter(R, 1, Acc) - end, dict:new(), Shards), + Dict = lists:foldl( + fun(#shard{range = R}, Acc) -> + dict:update_counter(R, 1, Acc) + end, + dict:new(), + Shards + ), Q = dict:size(Dict), N = dict:fold(fun(_, X, Acc) -> max(X, Acc) end, 0, Dict), %% defaults as per mem3:quorum/1 WR = N div 2 + 1, {ok, [{q, Q}, {n, N}, {w, WR}, {r, WR}]}. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -155,7 +169,6 @@ get_cluster_info_test_() -> fun get_cluster_info_test_generator/1 }. - setup() -> Quorums = [1, 2, 3], Shards = [1, 3, 5, 8, 12, 24], @@ -164,8 +177,7 @@ setup() -> get_cluster_info_test_generator([]) -> []; get_cluster_info_test_generator([{N, Q} | Rest]) -> - {generator, - fun() -> + {generator, fun() -> Nodes = lists:seq(1, 8), Shards = mem3_util:create_partition_map(<<"foo">>, N, Q, Nodes), {ok, Info} = get_cluster_info(Shards), diff --git a/src/fabric/src/fabric_db_meta.erl b/src/fabric/src/fabric_db_meta.erl index 348b06d51..1013b958d 100644 --- a/src/fabric/src/fabric_db_meta.erl +++ b/src/fabric/src/fabric_db_meta.erl @@ -12,8 +12,12 @@ -module(fabric_db_meta). --export([set_revs_limit/3, set_security/3, get_all_security/2, - set_purge_infos_limit/3]). +-export([ + set_revs_limit/3, + set_security/3, + get_all_security/2, + set_purge_infos_limit/3 +]). -include_lib("fabric/include/fabric.hrl"). -include_lib("mem3/include/mem3.hrl"). @@ -25,20 +29,19 @@ num_workers }). - set_revs_limit(DbName, Limit, Options) -> Shards = mem3:shards(DbName), Workers = fabric_util:submit_jobs(Shards, set_revs_limit, [Limit, Options]), Handler = fun handle_revs_message/3, Acc0 = {Workers, length(Workers) - 1}, case fabric_util:recv(Workers, #shard.ref, Handler, Acc0) of - {ok, ok} -> - ok; - {timeout, {DefunctWorkers, _}} -> - fabric_util:log_timeout(DefunctWorkers, "set_revs_limit"), - {error, timeout}; - Error -> - Error + {ok, ok} -> + ok; + {timeout, {DefunctWorkers, _}} -> + fabric_util:log_timeout(DefunctWorkers, "set_revs_limit"), + {error, timeout}; + Error -> + Error end. handle_revs_message(ok, _, {_Workers, 0}) -> @@ -48,7 +51,6 @@ handle_revs_message(ok, Worker, {Workers, Waiting}) -> handle_revs_message(Error, _, _Acc) -> {error, Error}. - set_purge_infos_limit(DbName, Limit, Options) -> Shards = mem3:shards(DbName), Workers = fabric_util:submit_jobs(Shards, set_purge_infos_limit, [Limit, Options]), @@ -71,35 +73,34 @@ handle_purge_message(ok, Worker, {Workers, Waiting}) -> handle_purge_message(Error, _, _Acc) -> {error, Error}. - set_security(DbName, SecObj, Options) -> Shards = mem3:shards(DbName), RexiMon = fabric_util:create_monitors(Shards), Workers = fabric_util:submit_jobs(Shards, set_security, [SecObj, Options]), Handler = fun handle_set_message/3, Acc = #acc{ - workers=Workers, - finished=[], - num_workers=length(Workers) + workers = Workers, + finished = [], + num_workers = length(Workers) }, try fabric_util:recv(Workers, #shard.ref, Handler, Acc) of - {ok, #acc{finished=Finished}} -> - case check_sec_set(length(Workers), Finished) of - ok -> ok; - Error -> Error - end; - {timeout, #acc{workers=DefunctWorkers}} -> - fabric_util:log_timeout(DefunctWorkers, "set_security"), - {error, timeout}; - Error -> - Error + {ok, #acc{finished = Finished}} -> + case check_sec_set(length(Workers), Finished) of + ok -> ok; + Error -> Error + end; + {timeout, #acc{workers = DefunctWorkers}} -> + fabric_util:log_timeout(DefunctWorkers, "set_security"), + {error, timeout}; + Error -> + Error after rexi_monitor:stop(RexiMon) end. -handle_set_message({rexi_DOWN, _, {_, Node}, _}, _, #acc{workers=Wrkrs}=Acc) -> +handle_set_message({rexi_DOWN, _, {_, Node}, _}, _, #acc{workers = Wrkrs} = Acc) -> RemWorkers = lists:filter(fun(S) -> S#shard.node =/= Node end, Wrkrs), - maybe_finish_set(Acc#acc{workers=RemWorkers}); + maybe_finish_set(Acc#acc{workers = RemWorkers}); handle_set_message(ok, W, Acc) -> NewAcc = Acc#acc{ workers = (Acc#acc.workers -- [W]), @@ -115,9 +116,9 @@ handle_set_message(Error, W, Acc) -> NewAcc = Acc#acc{workers = (Acc#acc.workers -- [W])}, maybe_finish_set(NewAcc). -maybe_finish_set(#acc{workers=[]}=Acc) -> +maybe_finish_set(#acc{workers = []} = Acc) -> {stop, Acc}; -maybe_finish_set(#acc{finished=Finished, num_workers=NumWorkers}=Acc) -> +maybe_finish_set(#acc{finished = Finished, num_workers = NumWorkers} = Acc) -> case check_sec_set(NumWorkers, Finished) of ok -> {stop, Acc}; _ -> {ok, Acc} @@ -126,8 +127,9 @@ maybe_finish_set(#acc{finished=Finished, num_workers=NumWorkers}=Acc) -> check_sec_set(NumWorkers, SetWorkers) -> try check_sec_set_int(NumWorkers, SetWorkers) - catch throw:Reason -> - {error, Reason} + catch + throw:Reason -> + {error, Reason} end. check_sec_set_int(NumWorkers, SetWorkers) -> @@ -143,41 +145,41 @@ check_sec_set_int(NumWorkers, SetWorkers) -> end, ok. - get_all_security(DbName, Options) -> - Shards = case proplists:get_value(shards, Options) of - Shards0 when is_list(Shards0) -> Shards0; - _ -> mem3:shards(DbName) - end, + Shards = + case proplists:get_value(shards, Options) of + Shards0 when is_list(Shards0) -> Shards0; + _ -> mem3:shards(DbName) + end, RexiMon = fabric_util:create_monitors(Shards), Workers = fabric_util:submit_jobs(Shards, get_all_security, [[?ADMIN_CTX]]), Handler = fun handle_get_message/3, Acc = #acc{ - workers=Workers, - finished=[], - num_workers=length(Workers) + workers = Workers, + finished = [], + num_workers = length(Workers) }, try fabric_util:recv(Workers, #shard.ref, Handler, Acc) of - {ok, #acc{finished=SecObjs}} when length(SecObjs) > length(Workers) / 2 -> - {ok, SecObjs}; - {ok, _} -> - {error, no_majority}; - {timeout, #acc{workers=DefunctWorkers}} -> - fabric_util:log_timeout( - DefunctWorkers, - "get_all_security" - ), - {error, timeout}; - Error -> - Error + {ok, #acc{finished = SecObjs}} when length(SecObjs) > length(Workers) / 2 -> + {ok, SecObjs}; + {ok, _} -> + {error, no_majority}; + {timeout, #acc{workers = DefunctWorkers}} -> + fabric_util:log_timeout( + DefunctWorkers, + "get_all_security" + ), + {error, timeout}; + Error -> + Error after rexi_monitor:stop(RexiMon) end. -handle_get_message({rexi_DOWN, _, {_, Node}, _}, _, #acc{workers=Wrkrs}=Acc) -> +handle_get_message({rexi_DOWN, _, {_, Node}, _}, _, #acc{workers = Wrkrs} = Acc) -> RemWorkers = lists:filter(fun(S) -> S#shard.node =/= Node end, Wrkrs), - maybe_finish_get(Acc#acc{workers=RemWorkers}); -handle_get_message({Props}=SecObj, W, Acc) when is_list(Props) -> + maybe_finish_get(Acc#acc{workers = RemWorkers}); +handle_get_message({Props} = SecObj, W, Acc) when is_list(Props) -> NewAcc = Acc#acc{ workers = (Acc#acc.workers -- [W]), finished = [{W, SecObj} | Acc#acc.finished] @@ -192,7 +194,7 @@ handle_get_message(Error, W, Acc) -> NewAcc = Acc#acc{workers = (Acc#acc.workers -- [W])}, maybe_finish_get(NewAcc). -maybe_finish_get(#acc{workers=[]}=Acc) -> +maybe_finish_get(#acc{workers = []} = Acc) -> {stop, Acc}; maybe_finish_get(Acc) -> {ok, Acc}. diff --git a/src/fabric/src/fabric_db_partition_info.erl b/src/fabric/src/fabric_db_partition_info.erl index 954c52db2..efc895534 100644 --- a/src/fabric/src/fabric_db_partition_info.erl +++ b/src/fabric/src/fabric_db_partition_info.erl @@ -17,14 +17,12 @@ -include_lib("fabric/include/fabric.hrl"). -include_lib("mem3/include/mem3.hrl"). - -record(acc, { counters, replies, ring_opts }). - go(DbName, Partition) -> Shards = mem3:shards(DbName, couch_partition:shard_key(Partition)), Workers = fabric_util:submit_jobs(Shards, get_partition_info, [Partition]), @@ -33,11 +31,12 @@ go(DbName, Partition) -> Acc0 = #acc{ counters = fabric_dict:init(Workers, nil), replies = [], - ring_opts = [{any, Shards}] + ring_opts = [{any, Shards}] }, try case fabric_util:recv(Workers, #shard.ref, Fun, Acc0) of - {ok, Res} -> {ok, Res}; + {ok, Res} -> + {ok, Res}; {timeout, {WorkersDict, _}} -> DefunctWorkers = fabric_util:remove_done_workers( WorkersDict, @@ -48,32 +47,31 @@ go(DbName, Partition) -> "get_partition_info" ), {error, timeout}; - {error, Error} -> throw(Error) + {error, Error} -> + throw(Error) end after rexi_monitor:stop(RexiMon) end. -handle_message({rexi_DOWN, _, {_,NodeRef},_}, _Shard, #acc{} = Acc) -> +handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _Shard, #acc{} = Acc) -> #acc{counters = Counters, ring_opts = RingOpts} = Acc, case fabric_util:remove_down_workers(Counters, NodeRef, RingOpts) of - {ok, NewCounters} -> - {ok, Acc#acc{counters = NewCounters}}; - error -> - {error, {nodedown, <<"progress not possible">>}} + {ok, NewCounters} -> + {ok, Acc#acc{counters = NewCounters}}; + error -> + {error, {nodedown, <<"progress not possible">>}} end; - handle_message({rexi_EXIT, Reason}, Shard, #acc{} = Acc) -> #acc{counters = Counters, ring_opts = RingOpts} = Acc, NewCounters = fabric_dict:erase(Shard, Counters), case fabric_ring:is_progress_possible(NewCounters, RingOpts) of - true -> - {ok, Acc#acc{counters = NewCounters}}; - false -> - {error, Reason} + true -> + {ok, Acc#acc{counters = NewCounters}}; + false -> + {error, Reason} end; - -handle_message({ok, Info}, #shard{dbname=Name} = Shard, #acc{} = Acc) -> +handle_message({ok, Info}, #shard{dbname = Name} = Shard, #acc{} = Acc) -> #acc{counters = Counters, replies = Replies} = Acc, Replies1 = [Info | Replies], Counters1 = fabric_dict:erase(Shard, Counters), @@ -85,11 +83,9 @@ handle_message({ok, Info}, #shard{dbname=Name} = Shard, #acc{} = Acc) -> false -> {ok, Acc#acc{counters = Counters1, replies = Replies1}} end; - handle_message(_, _, #acc{} = Acc) -> {ok, Acc}. - get_max_partition_size(Max, []) -> Max; get_max_partition_size(MaxInfo, [NextInfo | Rest]) -> @@ -105,21 +101,18 @@ get_max_partition_size(MaxInfo, [NextInfo | Rest]) -> get_max_partition_size(MaxInfo, Rest) end. - % for JS to work nicely we need to convert the size list % to a jiffy object format_partition(PartitionInfo) -> {value, {sizes, Size}, PartitionInfo1} = lists:keytake(sizes, 1, PartitionInfo), [{sizes, {Size}} | PartitionInfo1]. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). - node_down_test() -> - [S1, S2] = [mk_shard("n1", [0, 4]), mk_shard("n2", [0, 8])], + [S1, S2] = [mk_shard("n1", [0, 4]), mk_shard("n2", [0, 8])], Acc1 = #acc{ counters = fabric_dict:init([S1, S2], nil), ring_opts = [{any, [S1, S2]}] @@ -130,12 +123,13 @@ node_down_test() -> ?assertEqual([{S2, nil}], Acc2#acc.counters), N2 = S2#shard.node, - ?assertEqual({error, {nodedown, <<"progress not possible">>}}, - handle_message({rexi_DOWN, nil, {nil, N2}, nil}, nil, Acc2)). - + ?assertEqual( + {error, {nodedown, <<"progress not possible">>}}, + handle_message({rexi_DOWN, nil, {nil, N2}, nil}, nil, Acc2) + ). worker_exit_test() -> - [S1, S2] = [mk_shard("n1", [0, 4]), mk_shard("n2", [0, 8])], + [S1, S2] = [mk_shard("n1", [0, 4]), mk_shard("n2", [0, 8])], Acc1 = #acc{ counters = fabric_dict:init([S1, S2], nil), ring_opts = [{any, [S1, S2]}] @@ -146,7 +140,6 @@ worker_exit_test() -> ?assertEqual({error, bam}, handle_message({rexi_EXIT, bam}, S2, Acc2)). - mk_shard(Name, Range) -> Node = list_to_atom(Name), BName = list_to_binary(Name), diff --git a/src/fabric/src/fabric_db_update_listener.erl b/src/fabric/src/fabric_db_update_listener.erl index fb2937be1..78ccf5a4d 100644 --- a/src/fabric/src/fabric_db_update_listener.erl +++ b/src/fabric/src/fabric_db_update_listener.erl @@ -45,18 +45,19 @@ go(Parent, ParentRef, DbName, Timeout) -> %% This is not a common pattern for rexi but to enable the calling %% process to communicate via handle_message/3 we "fake" it as a %% a spawned worker. - Workers = [#worker{ref=ParentRef, pid=Parent} | Notifiers], + Workers = [#worker{ref = ParentRef, pid = Parent} | Notifiers], Acc = #acc{ parent = Parent, state = unset, shards = Shards }, - Resp = try - receive_results(Workers, Acc, Timeout) - after - rexi_monitor:stop(RexiMon), - stop_cleanup_monitor(MonPid) - end, + Resp = + try + receive_results(Workers, Acc, Timeout) + after + rexi_monitor:stop(RexiMon), + stop_cleanup_monitor(MonPid) + end, case Resp of {ok, _} -> ok; {error, Error} -> erlang:error(Error); @@ -64,13 +65,20 @@ go(Parent, ParentRef, DbName, Timeout) -> end. start_update_notifiers(Shards) -> - EndPointDict = lists:foldl(fun(#shard{node=Node, name=Name}, Acc) -> - dict:append(Node, Name, Acc) - end, dict:new(), Shards), - lists:map(fun({Node, DbNames}) -> - Ref = rexi:cast(Node, {?MODULE, start_update_notifier, [DbNames]}), - #worker{ref=Ref, node=Node} - end, dict:to_list(EndPointDict)). + EndPointDict = lists:foldl( + fun(#shard{node = Node, name = Name}, Acc) -> + dict:append(Node, Name, Acc) + end, + dict:new(), + Shards + ), + lists:map( + fun({Node, DbNames}) -> + Ref = rexi:cast(Node, {?MODULE, start_update_notifier, [DbNames]}), + #worker{ref = Ref, node = Node} + end, + dict:to_list(EndPointDict) + ). % rexi endpoint start_update_notifier(DbNames) -> @@ -132,41 +140,39 @@ wait_db_updated({Pid, Ref}) -> receive_results(Workers, Acc0, Timeout) -> Fun = fun handle_message/3, case rexi_utils:recv(Workers, #worker.ref, Fun, Acc0, infinity, Timeout) of - {timeout, #acc{state=updated}=Acc} -> - receive_results(Workers, Acc, Timeout); - {timeout, #acc{state=waiting}=Acc} -> - erlang:send(Acc#acc.parent, {state, self(), timeout}), - receive_results(Workers, Acc#acc{state=unset}, Timeout); - {timeout, Acc} -> - receive_results(Workers, Acc#acc{state=timeout}, Timeout); - {_, Acc} -> - {ok, Acc} + {timeout, #acc{state = updated} = Acc} -> + receive_results(Workers, Acc, Timeout); + {timeout, #acc{state = waiting} = Acc} -> + erlang:send(Acc#acc.parent, {state, self(), timeout}), + receive_results(Workers, Acc#acc{state = unset}, Timeout); + {timeout, Acc} -> + receive_results(Workers, Acc#acc{state = timeout}, Timeout); + {_, Acc} -> + {ok, Acc} end. - handle_message({rexi_DOWN, _, {_, Node}, _}, _Worker, Acc) -> handle_error(Node, {nodedown, Node}, Acc); handle_message({rexi_EXIT, _Reason}, Worker, Acc) -> handle_error(Worker#worker.node, {worker_exit, Worker}, Acc); handle_message({gen_event_EXIT, Node, Reason}, _Worker, Acc) -> handle_error(Node, {gen_event_EXIT, Node, Reason}, Acc); -handle_message(db_updated, _Worker, #acc{state=waiting}=Acc) -> +handle_message(db_updated, _Worker, #acc{state = waiting} = Acc) -> % propagate message to calling controller erlang:send(Acc#acc.parent, {state, self(), updated}), - {ok, Acc#acc{state=unset}}; + {ok, Acc#acc{state = unset}}; handle_message(db_updated, _Worker, Acc) -> - {ok, Acc#acc{state=updated}}; + {ok, Acc#acc{state = updated}}; handle_message(db_deleted, _Worker, _Acc) -> {stop, ok}; -handle_message(get_state, _Worker, #acc{state=unset}=Acc) -> - {ok, Acc#acc{state=waiting}}; +handle_message(get_state, _Worker, #acc{state = unset} = Acc) -> + {ok, Acc#acc{state = waiting}}; handle_message(get_state, _Worker, Acc) -> erlang:send(Acc#acc.parent, {state, self(), Acc#acc.state}), - {ok, Acc#acc{state=unset}}; + {ok, Acc#acc{state = unset}}; handle_message(done, _, _) -> {stop, ok}. - handle_error(Node, Reason, #acc{shards = Shards} = Acc) -> Rest = lists:filter(fun(#shard{node = N}) -> N /= Node end, Shards), case fabric_ring:is_progress_possible([{R, nil} || R <- Rest]) of diff --git a/src/fabric/src/fabric_db_uuids.erl b/src/fabric/src/fabric_db_uuids.erl index a440d74c2..12931a3d1 100644 --- a/src/fabric/src/fabric_db_uuids.erl +++ b/src/fabric/src/fabric_db_uuids.erl @@ -12,14 +12,11 @@ -module(fabric_db_uuids). - -export([go/1]). - -include_lib("fabric/include/fabric.hrl"). -include_lib("mem3/include/mem3.hrl"). - go(DbName) when is_binary(DbName) -> Shards = mem3:live_shards(DbName, [node() | nodes()]), Workers = fabric_util:submit_jobs(Shards, get_uuid, []), @@ -36,30 +33,30 @@ go(DbName) when is_binary(DbName) -> rexi_monitor:stop(RexiMon) end. - -handle_message({rexi_DOWN, _, {_, NodeRef},_}, _Shard, {Cntrs, Res}) -> +handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _Shard, {Cntrs, Res}) -> case fabric_ring:node_down(NodeRef, Cntrs, Res, [all]) of {ok, Cntrs1} -> {ok, {Cntrs1, Res}}; error -> {error, {nodedown, <<"progress not possible">>}} end; - handle_message({rexi_EXIT, Reason}, Shard, {Cntrs, Res}) -> case fabric_ring:handle_error(Shard, Cntrs, Res, [all]) of {ok, Cntrs1} -> {ok, {Cntrs1, Res}}; error -> {error, Reason} end; - handle_message(Uuid, Shard, {Cntrs, Res}) when is_binary(Uuid) -> case fabric_ring:handle_response(Shard, Uuid, Cntrs, Res, [all]) of {ok, {Cntrs1, Res1}} -> {ok, {Cntrs1, Res1}}; {stop, Res1} -> - Uuids = fabric_dict:fold(fun(#shard{} = S, Id, #{} = Acc) -> - Acc#{Id => S#shard{ref = undefined}} - end, #{}, Res1), + Uuids = fabric_dict:fold( + fun(#shard{} = S, Id, #{} = Acc) -> + Acc#{Id => S#shard{ref = undefined}} + end, + #{}, + Res1 + ), {stop, Uuids} end; - handle_message(Reason, Shard, {Cntrs, Res}) -> case fabric_ring:handle_error(Shard, Cntrs, Res, [all]) of {ok, Cntrs1} -> {ok, {Cntrs1, Res}}; diff --git a/src/fabric/src/fabric_design_doc_count.erl b/src/fabric/src/fabric_design_doc_count.erl index b0efc3007..f6f866a24 100644 --- a/src/fabric/src/fabric_design_doc_count.erl +++ b/src/fabric/src/fabric_design_doc_count.erl @@ -34,18 +34,16 @@ go(DbName) -> rexi_monitor:stop(RexiMon) end. -handle_message({rexi_DOWN, _, {_,NodeRef},_}, _Shard, {Counters, Resps}) -> +handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _Shard, {Counters, Resps}) -> case fabric_ring:node_down(NodeRef, Counters, Resps) of {ok, Counters1} -> {ok, {Counters1, Resps}}; error -> {error, {nodedown, <<"progress not possible">>}} end; - handle_message({rexi_EXIT, Reason}, Shard, {Counters, Resps}) -> case fabric_ring:handle_error(Shard, Counters, Resps) of {ok, Counters1} -> {ok, {Counters1, Resps}}; error -> {error, Reason} end; - handle_message({ok, Count}, Shard, {Counters, Resps}) -> case fabric_ring:handle_response(Shard, Count, Counters, Resps) of {ok, {Counters1, Resps1}} -> @@ -54,7 +52,6 @@ handle_message({ok, Count}, Shard, {Counters, Resps}) -> Total = fabric_dict:fold(fun(_, C, A) -> A + C end, 0, Resps1), {stop, Total} end; - handle_message(Reason, Shard, {Counters, Resps}) -> case fabric_ring:handle_error(Shard, Counters, Resps) of {ok, Counters1} -> {ok, {Counters1, Resps}}; diff --git a/src/fabric/src/fabric_dict.erl b/src/fabric/src/fabric_dict.erl index b63ed2095..8395221b4 100644 --- a/src/fabric/src/fabric_dict.erl +++ b/src/fabric/src/fabric_dict.erl @@ -27,7 +27,7 @@ fetch_keys(Dict) -> orddict:fetch_keys(Dict). decrement_all(Dict) -> - [{K,V-1} || {K,V} <- Dict]. + [{K, V - 1} || {K, V} <- Dict]. store(Key, Value, Dict) -> orddict:store(Key, Value, Dict). @@ -38,7 +38,6 @@ erase(Key, Dict) -> update_counter(Key, Incr, Dict0) -> orddict:update_counter(Key, Incr, Dict0). - lookup_element(Key, Dict) -> couch_util:get_value(Key, Dict). diff --git a/src/fabric/src/fabric_doc_atts.erl b/src/fabric/src/fabric_doc_atts.erl index bd0687b30..80a36ee51 100644 --- a/src/fabric/src/fabric_doc_atts.erl +++ b/src/fabric/src/fabric_doc_atts.erl @@ -22,7 +22,6 @@ receiver_callback/2 ]). - receiver(_Req, _DbName, undefined) -> <<"">>; receiver(_Req, _DbName, {unknown_transfer_encoding, Unknown}) -> @@ -39,7 +38,6 @@ receiver(Req, DbName, Length) when is_integer(Length) -> receiver(_Req, _DbName, Length) -> exit({length_not_integer, Length}). - receiver_callback(Middleman, chunked) -> fun(4096, ChunkFun, State) -> write_chunks(Middleman, ChunkFun, State) @@ -57,36 +55,35 @@ receiver_callback(Middleman, Length) when is_integer(Length) -> end end. - %% %% internal %% maybe_send_continue(#httpd{mochi_req = MochiReq} = Req) -> case couch_httpd:header_value(Req, "expect") of - undefined -> - ok; - Expect -> - case string:to_lower(Expect) of - "100-continue" -> - MochiReq:start_raw_response({100, gb_trees:empty()}); - _ -> - ok - end + undefined -> + ok; + Expect -> + case string:to_lower(Expect) of + "100-continue" -> + MochiReq:start_raw_response({100, gb_trees:empty()}); + _ -> + ok + end end. write_chunks(MiddleMan, ChunkFun, State) -> MiddleMan ! {self(), gimme_data}, Timeout = fabric_util:attachments_timeout(), receive - {MiddleMan, ChunkRecordList} -> - rexi:reply(attachment_chunk_received), - case flush_chunks(ChunkRecordList, ChunkFun, State) of - {continue, NewState} -> - write_chunks(MiddleMan, ChunkFun, NewState); - {done, NewState} -> - NewState - end + {MiddleMan, ChunkRecordList} -> + rexi:reply(attachment_chunk_received), + case flush_chunks(ChunkRecordList, ChunkFun, State) of + {continue, NewState} -> + write_chunks(MiddleMan, ChunkFun, NewState); + {done, NewState} -> + NewState + end after Timeout -> exit(timeout) end. @@ -102,24 +99,27 @@ flush_chunks([Chunk | Rest], ChunkFun, State) -> receive_unchunked_attachment(_Req, 0) -> ok; receive_unchunked_attachment(Req, Length) -> - receive {MiddleMan, go} -> - Data = couch_httpd:recv(Req, 0), - MiddleMan ! {self(), Data} + receive + {MiddleMan, go} -> + Data = couch_httpd:recv(Req, 0), + MiddleMan ! {self(), Data} end, receive_unchunked_attachment(Req, Length - size(Data)). middleman(Req, DbName, chunked) -> % spawn a process to actually receive the uploaded data RcvFun = fun(ChunkRecord, ok) -> - receive {From, go} -> From ! {self(), ChunkRecord} end, ok + receive + {From, go} -> From ! {self(), ChunkRecord} + end, + ok end, - Receiver = spawn(fun() -> couch_httpd:recv_chunked(Req,4096,RcvFun,ok) end), + Receiver = spawn(fun() -> couch_httpd:recv_chunked(Req, 4096, RcvFun, ok) end), % take requests from the DB writers and get data from the receiver N = mem3:n(DbName), Timeout = fabric_util:attachments_timeout(), middleman_loop(Receiver, N, [], [], Timeout); - middleman(Req, DbName, Length) -> Receiver = spawn(fun() -> receive_unchunked_attachment(Req, Length) end), N = mem3:n(DbName), @@ -127,43 +127,50 @@ middleman(Req, DbName, Length) -> middleman_loop(Receiver, N, [], [], Timeout). middleman_loop(Receiver, N, Counters0, ChunkList0, Timeout) -> - receive {From, gimme_data} -> - % Figure out how far along this writer (From) is in the list - ListIndex = case fabric_dict:lookup_element(From, Counters0) of - undefined -> 0; - I -> I - end, - - % Talk to the receiver to get another chunk if necessary - ChunkList1 = if ListIndex == length(ChunkList0) -> - Receiver ! {self(), go}, - receive - {Receiver, ChunkRecord} -> - ChunkList0 ++ [ChunkRecord] - end; - true -> ChunkList0 end, - - % reply to the writer - Reply = lists:nthtail(ListIndex, ChunkList1), - From ! {self(), Reply}, - - % Update the counter for this writer - Counters1 = fabric_dict:update_counter(From, length(Reply), Counters0), - - % Drop any chunks that have been sent to all writers - Size = fabric_dict:size(Counters1), - NumToDrop = lists:min([I || {_, I} <- Counters1]), - - {ChunkList3, Counters3} = - if Size == N andalso NumToDrop > 0 -> - ChunkList2 = lists:nthtail(NumToDrop, ChunkList1), - Counters2 = [{F, I-NumToDrop} || {F, I} <- Counters1], - {ChunkList2, Counters2}; - true -> - {ChunkList1, Counters1} - end, - - middleman_loop(Receiver, N, Counters3, ChunkList3, Timeout) + receive + {From, gimme_data} -> + % Figure out how far along this writer (From) is in the list + ListIndex = + case fabric_dict:lookup_element(From, Counters0) of + undefined -> 0; + I -> I + end, + + % Talk to the receiver to get another chunk if necessary + ChunkList1 = + if + ListIndex == length(ChunkList0) -> + Receiver ! {self(), go}, + receive + {Receiver, ChunkRecord} -> + ChunkList0 ++ [ChunkRecord] + end; + true -> + ChunkList0 + end, + + % reply to the writer + Reply = lists:nthtail(ListIndex, ChunkList1), + From ! {self(), Reply}, + + % Update the counter for this writer + Counters1 = fabric_dict:update_counter(From, length(Reply), Counters0), + + % Drop any chunks that have been sent to all writers + Size = fabric_dict:size(Counters1), + NumToDrop = lists:min([I || {_, I} <- Counters1]), + + {ChunkList3, Counters3} = + if + Size == N andalso NumToDrop > 0 -> + ChunkList2 = lists:nthtail(NumToDrop, ChunkList1), + Counters2 = [{F, I - NumToDrop} || {F, I} <- Counters1], + {ChunkList2, Counters2}; + true -> + {ChunkList1, Counters1} + end, + + middleman_loop(Receiver, N, Counters3, ChunkList3, Timeout) after Timeout -> exit(Receiver, kill), ok diff --git a/src/fabric/src/fabric_doc_missing_revs.erl b/src/fabric/src/fabric_doc_missing_revs.erl index 993c21dc2..ffd408f4e 100644 --- a/src/fabric/src/fabric_doc_missing_revs.erl +++ b/src/fabric/src/fabric_doc_missing_revs.erl @@ -23,48 +23,57 @@ go(DbName, AllIdsRevs) -> go(_, [], _) -> {ok, []}; go(DbName, AllIdsRevs, Options) -> - Workers = lists:map(fun({#shard{name=Name, node=Node} = Shard, IdsRevs}) -> - Ref = rexi:cast(Node, {fabric_rpc, get_missing_revs, [Name, IdsRevs, - Options]}), - Shard#shard{ref=Ref} - end, group_idrevs_by_shard(DbName, AllIdsRevs)), - ResultDict = dict:from_list([{Id, {{nil,Revs},[]}} || {Id, Revs} <- AllIdsRevs]), + Workers = lists:map( + fun({#shard{name = Name, node = Node} = Shard, IdsRevs}) -> + Ref = rexi:cast( + Node, + {fabric_rpc, get_missing_revs, [ + Name, + IdsRevs, + Options + ]} + ), + Shard#shard{ref = Ref} + end, + group_idrevs_by_shard(DbName, AllIdsRevs) + ), + ResultDict = dict:from_list([{Id, {{nil, Revs}, []}} || {Id, Revs} <- AllIdsRevs]), RexiMon = fabric_util:create_monitors(Workers), Acc0 = {length(Workers), ResultDict, Workers}, try fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of - {timeout, {_, _, DefunctWorkers}} -> - fabric_util:log_timeout( - DefunctWorkers, - "get_missing_revs" - ), - {error, timeout}; - Else -> - Else + {timeout, {_, _, DefunctWorkers}} -> + fabric_util:log_timeout( + DefunctWorkers, + "get_missing_revs" + ), + {error, timeout}; + Else -> + Else after rexi_monitor:stop(RexiMon) end. -handle_message({rexi_DOWN, _, {_,NodeRef},_}, _Shard, {_WorkerLen, ResultDict, Workers}) -> - NewWorkers = [W || #shard{node=Node} = W <- Workers, Node =/= NodeRef], +handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _Shard, {_WorkerLen, ResultDict, Workers}) -> + NewWorkers = [W || #shard{node = Node} = W <- Workers, Node =/= NodeRef], skip_message({fabric_dict:size(NewWorkers), ResultDict, NewWorkers}); handle_message({rexi_EXIT, _}, Worker, {W, D, Workers}) -> - skip_message({W-1,D,lists:delete(Worker, Workers)}); + skip_message({W - 1, D, lists:delete(Worker, Workers)}); handle_message({ok, Results}, _Worker, {1, D0, _}) -> D = update_dict(D0, Results), {stop, dict:fold(fun force_reply/3, [], D)}; handle_message({ok, Results}, Worker, {WaitingCount, D0, Workers}) -> D = update_dict(D0, Results), case dict:fold(fun maybe_reply/3, {stop, []}, D) of - continue -> - % still haven't heard about some Ids - {ok, {WaitingCount - 1, D, lists:delete(Worker,Workers)}}; - {stop, FinalReply} -> - % finished, stop the rest of the jobs - fabric_util:cleanup(lists:delete(Worker,Workers)), - {stop, FinalReply} + continue -> + % still haven't heard about some Ids + {ok, {WaitingCount - 1, D, lists:delete(Worker, Workers)}}; + {stop, FinalReply} -> + % finished, stop the rest of the jobs + fabric_util:cleanup(lists:delete(Worker, Workers)), + {stop, FinalReply} end. -force_reply(Id, {{nil,Revs}, Anc}, Acc) -> +force_reply(Id, {{nil, Revs}, Anc}, Acc) -> % never heard about this ID, assume it's missing [{Id, Revs, Anc} | Acc]; force_reply(_, {[], _}, Acc) -> @@ -82,14 +91,24 @@ maybe_reply(Id, {Revs, Anc}, {stop, Acc}) -> {stop, [{Id, Revs, Anc} | Acc]}. group_idrevs_by_shard(DbName, IdsRevs) -> - dict:to_list(lists:foldl(fun({Id, Revs}, D0) -> - lists:foldl(fun(Shard, D1) -> - dict:append(Shard, {Id, Revs}, D1) - end, D0, mem3:shards(DbName,Id)) - end, dict:new(), IdsRevs)). + dict:to_list( + lists:foldl( + fun({Id, Revs}, D0) -> + lists:foldl( + fun(Shard, D1) -> + dict:append(Shard, {Id, Revs}, D1) + end, + D0, + mem3:shards(DbName, Id) + ) + end, + dict:new(), + IdsRevs + ) + ). update_dict(D0, KVs) -> - lists:foldl(fun({K,V,A}, D1) -> dict:store(K, {V,A}, D1) end, D0, KVs). + lists:foldl(fun({K, V, A}, D1) -> dict:store(K, {V, A}, D1) end, D0, KVs). skip_message({0, Dict, _Workers}) -> {stop, dict:fold(fun force_reply/3, [], Dict)}; diff --git a/src/fabric/src/fabric_doc_open.erl b/src/fabric/src/fabric_doc_open.erl index 8ef604b60..ba348112c 100644 --- a/src/fabric/src/fabric_doc_open.erl +++ b/src/fabric/src/fabric_doc_open.erl @@ -18,7 +18,6 @@ -include_lib("mem3/include/mem3.hrl"). -include_lib("couch/include/couch_db.hrl"). - -record(acc, { dbname, workers, @@ -29,15 +28,18 @@ q_reply }). - go(DbName, Id, Options) -> - Handler = case proplists:get_value(doc_info, Options) of - true -> get_doc_info; - full -> get_full_doc_info; - undefined -> open_doc - end, - Workers = fabric_util:submit_jobs(mem3:shards(DbName,Id), Handler, - [Id, [deleted|Options]]), + Handler = + case proplists:get_value(doc_info, Options) of + true -> get_doc_info; + full -> get_full_doc_info; + undefined -> open_doc + end, + Workers = fabric_util:submit_jobs( + mem3:shards(DbName, Id), + Handler, + [Id, [deleted | Options]] + ), SuppressDeletedDoc = not lists:member(deleted, Options), N = mem3:n(DbName), R = couch_util:get_value(r, Options, integer_to_list(mem3:quorum(DbName))), @@ -50,64 +52,65 @@ go(DbName, Id, Options) -> }, RexiMon = fabric_util:create_monitors(Workers), try fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of - {ok, #acc{}=Acc} when Handler =:= open_doc -> - Reply = handle_response(Acc), - format_reply(Reply, SuppressDeletedDoc); - {ok, #acc{state = r_not_met}} -> - {error, quorum_not_met}; - {ok, #acc{q_reply = QuorumReply}} -> - format_reply(QuorumReply, SuppressDeletedDoc); - {timeout, #acc{workers=DefunctWorkers}} -> - fabric_util:log_timeout(DefunctWorkers, atom_to_list(Handler)), - {error, timeout}; - Error -> - Error + {ok, #acc{} = Acc} when Handler =:= open_doc -> + Reply = handle_response(Acc), + format_reply(Reply, SuppressDeletedDoc); + {ok, #acc{state = r_not_met}} -> + {error, quorum_not_met}; + {ok, #acc{q_reply = QuorumReply}} -> + format_reply(QuorumReply, SuppressDeletedDoc); + {timeout, #acc{workers = DefunctWorkers}} -> + fabric_util:log_timeout(DefunctWorkers, atom_to_list(Handler)), + {error, timeout}; + Error -> + Error after rexi_monitor:stop(RexiMon) end. handle_message({rexi_DOWN, _, {_, Node}, _}, _Worker, Acc) -> - NewWorkers = [W || #shard{node=N}=W <- Acc#acc.workers, N /= Node], + NewWorkers = [W || #shard{node = N} = W <- Acc#acc.workers, N /= Node], case NewWorkers of - [] -> - {stop, Acc#acc{workers=[]}}; - _ -> - {ok, Acc#acc{workers=NewWorkers}} + [] -> + {stop, Acc#acc{workers = []}}; + _ -> + {ok, Acc#acc{workers = NewWorkers}} end; handle_message({rexi_EXIT, _Reason}, Worker, Acc) -> NewWorkers = lists:delete(Worker, Acc#acc.workers), case NewWorkers of - [] -> - {stop, Acc#acc{workers=[]}}; - _ -> - {ok, Acc#acc{workers=NewWorkers}} + [] -> + {stop, Acc#acc{workers = []}}; + _ -> + {ok, Acc#acc{workers = NewWorkers}} end; handle_message(Reply, Worker, Acc) -> NewReplies = fabric_util:update_counter(Reply, 1, Acc#acc.replies), - NewNodeRevs = case Reply of - {ok, #doc{revs = {Pos, [Rev | _]}}} -> - [{Worker#shard.node, [{Pos, Rev}]} | Acc#acc.node_revs]; - _ -> - Acc#acc.node_revs - end, + NewNodeRevs = + case Reply of + {ok, #doc{revs = {Pos, [Rev | _]}}} -> + [{Worker#shard.node, [{Pos, Rev}]} | Acc#acc.node_revs]; + _ -> + Acc#acc.node_revs + end, NewAcc = Acc#acc{replies = NewReplies, node_revs = NewNodeRevs}, case is_r_met(Acc#acc.workers, NewReplies, Acc#acc.r) of - {true, QuorumReply} -> - fabric_util:cleanup(lists:delete(Worker, Acc#acc.workers)), - {stop, NewAcc#acc{workers=[], state=r_met, q_reply=QuorumReply}}; - wait_for_more -> - NewWorkers = lists:delete(Worker, Acc#acc.workers), - {ok, NewAcc#acc{workers=NewWorkers}}; - no_more_workers -> - {stop, NewAcc#acc{workers=[]}} + {true, QuorumReply} -> + fabric_util:cleanup(lists:delete(Worker, Acc#acc.workers)), + {stop, NewAcc#acc{workers = [], state = r_met, q_reply = QuorumReply}}; + wait_for_more -> + NewWorkers = lists:delete(Worker, Acc#acc.workers), + {ok, NewAcc#acc{workers = NewWorkers}}; + no_more_workers -> + {stop, NewAcc#acc{workers = []}} end. -handle_response(#acc{state=r_met, replies=Replies, q_reply=QuorumReply}=Acc) -> +handle_response(#acc{state = r_met, replies = Replies, q_reply = QuorumReply} = Acc) -> case {Replies, fabric_util:remove_ancestors(Replies, [])} of {[_], [_]} -> % Complete agreement amongst all copies QuorumReply; - {[_|_], [{_, {QuorumReply, _}}]} -> + {[_ | _], [{_, {QuorumReply, _}}]} -> % Any divergent replies are ancestors of the QuorumReply, % repair the document asynchronously spawn(fun() -> read_repair(Acc) end), @@ -120,68 +123,70 @@ handle_response(Acc) -> read_repair(Acc). is_r_met(Workers, Replies, R) -> - case lists:dropwhile(fun({_,{_, Count}}) -> Count < R end, Replies) of - [{_,{QuorumReply, _}} | _] -> - {true, QuorumReply}; - [] when length(Workers) > 1 -> - wait_for_more; - [] -> - no_more_workers + case lists:dropwhile(fun({_, {_, Count}}) -> Count < R end, Replies) of + [{_, {QuorumReply, _}} | _] -> + {true, QuorumReply}; + [] when length(Workers) > 1 -> + wait_for_more; + [] -> + no_more_workers end. -read_repair(#acc{dbname=DbName, replies=Replies, node_revs=NodeRevs}) -> - Docs = [Doc || {_, {{ok, #doc{}=Doc}, _}} <- Replies], +read_repair(#acc{dbname = DbName, replies = Replies, node_revs = NodeRevs}) -> + Docs = [Doc || {_, {{ok, #doc{} = Doc}, _}} <- Replies], case Docs of - % omit local docs from read repair - [#doc{id = <<?LOCAL_DOC_PREFIX, _/binary>>} | _] -> - choose_reply(Docs); - [#doc{id=Id} | _] -> - Opts = [?ADMIN_CTX, replicated_changes, {read_repair, NodeRevs}], - Res = fabric:update_docs(DbName, Docs, Opts), - case Res of - {ok, []} -> - couch_stats:increment_counter([fabric, read_repairs, success]); - _ -> - couch_stats:increment_counter([fabric, read_repairs, failure]), - couch_log:notice("read_repair ~s ~s ~p", [DbName, Id, Res]) - end, - choose_reply(Docs); - [] -> - % Try hard to return some sort of information - % to the client. - Values = [V || {_, {V, _}} <- Replies], - case lists:member({not_found, missing}, Values) of - true -> - {not_found, missing}; - false when length(Values) > 0 -> - % Sort for stability in responses in - % case we have some weird condition - hd(lists:sort(Values)); - false -> - {error, read_failure} - end + % omit local docs from read repair + [#doc{id = <<?LOCAL_DOC_PREFIX, _/binary>>} | _] -> + choose_reply(Docs); + [#doc{id = Id} | _] -> + Opts = [?ADMIN_CTX, replicated_changes, {read_repair, NodeRevs}], + Res = fabric:update_docs(DbName, Docs, Opts), + case Res of + {ok, []} -> + couch_stats:increment_counter([fabric, read_repairs, success]); + _ -> + couch_stats:increment_counter([fabric, read_repairs, failure]), + couch_log:notice("read_repair ~s ~s ~p", [DbName, Id, Res]) + end, + choose_reply(Docs); + [] -> + % Try hard to return some sort of information + % to the client. + Values = [V || {_, {V, _}} <- Replies], + case lists:member({not_found, missing}, Values) of + true -> + {not_found, missing}; + false when length(Values) > 0 -> + % Sort for stability in responses in + % case we have some weird condition + hd(lists:sort(Values)); + false -> + {error, read_failure} + end end. choose_reply(Docs) -> % Sort descending by {not deleted, rev}. This should match % the logic of couch_doc:to_doc_info/1. - [Winner | _] = lists:sort(fun(DocA, DocB) -> - InfoA = {not DocA#doc.deleted, DocA#doc.revs}, - InfoB = {not DocB#doc.deleted, DocB#doc.revs}, - InfoA > InfoB - end, Docs), + [Winner | _] = lists:sort( + fun(DocA, DocB) -> + InfoA = {not DocA#doc.deleted, DocA#doc.revs}, + InfoB = {not DocB#doc.deleted, DocB#doc.revs}, + InfoA > InfoB + end, + Docs + ), {ok, Winner}. -format_reply({ok, #full_doc_info{deleted=true}}, true) -> +format_reply({ok, #full_doc_info{deleted = true}}, true) -> {not_found, deleted}; -format_reply({ok, #doc{deleted=true}}, true) -> +format_reply({ok, #doc{deleted = true}}, true) -> {not_found, deleted}; format_reply(not_found, _) -> {not_found, missing}; format_reply(Else, _) -> Else. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -195,23 +200,18 @@ format_reply(Else, _) -> rexi_monitor ]). - setup_all() -> meck:new(?MECK_MODS, [passthrough]). - teardown_all(_) -> meck:unload(). - setup() -> meck:reset(?MECK_MODS). - teardown(_) -> ok. - open_doc_test_() -> { setup, @@ -234,7 +234,6 @@ open_doc_test_() -> } }. - t_is_r_met() -> ?_test(begin Workers0 = [], @@ -249,18 +248,24 @@ t_is_r_met() -> {{true, bar}, [fabric_util:kv(bar, 1), fabric_util:kv(bar, 2)], 2}, {{true, bar}, [fabric_util:kv(bar, 2), fabric_util:kv(foo, 1)], 2} ], - lists:foreach(fun({Expect, Replies, Q}) -> - ?assertEqual(Expect, is_r_met(Workers0, Replies, Q)) - end, SuccessCases), + lists:foreach( + fun({Expect, Replies, Q}) -> + ?assertEqual(Expect, is_r_met(Workers0, Replies, Q)) + end, + SuccessCases + ), WaitForMoreCases = [ {[fabric_util:kv(foo, 1)], 2}, {[fabric_util:kv(foo, 2)], 3}, {[fabric_util:kv(foo, 1), fabric_util:kv(bar, 1)], 2} ], - lists:foreach(fun({Replies, Q}) -> - ?assertEqual(wait_for_more, is_r_met(Workers2, Replies, Q)) - end, WaitForMoreCases), + lists:foreach( + fun({Replies, Q}) -> + ?assertEqual(wait_for_more, is_r_met(Workers2, Replies, Q)) + end, + WaitForMoreCases + ), FailureCases = [ {Workers0, [fabric_util:kv(foo, 1)], 2}, @@ -268,73 +273,73 @@ t_is_r_met() -> {Workers1, [fabric_util:kv(foo, 1), fabric_util:kv(bar, 1)], 2}, {Workers1, [fabric_util:kv(foo, 2)], 3} ], - lists:foreach(fun({Workers, Replies, Q}) -> - ?assertEqual(no_more_workers, is_r_met(Workers, Replies, Q)) - end, FailureCases) + lists:foreach( + fun({Workers, Replies, Q}) -> + ?assertEqual(no_more_workers, is_r_met(Workers, Replies, Q)) + end, + FailureCases + ) end). - t_handle_message_down() -> Node0 = 'foo@localhost', Node1 = 'bar@localhost', Down0 = {rexi_DOWN, nil, {nil, Node0}, nil}, Down1 = {rexi_DOWN, nil, {nil, Node1}, nil}, - Workers0 = [#shard{node=Node0} || _ <- [a, b]], - Worker1 = #shard{node=Node1}, + Workers0 = [#shard{node = Node0} || _ <- [a, b]], + Worker1 = #shard{node = Node1}, Workers1 = Workers0 ++ [Worker1], ?_test(begin % Stop when no more workers are left ?assertEqual( - {stop, #acc{workers=[]}}, - handle_message(Down0, nil, #acc{workers=Workers0}) + {stop, #acc{workers = []}}, + handle_message(Down0, nil, #acc{workers = Workers0}) ), % Continue when we have more workers ?assertEqual( - {ok, #acc{workers=[Worker1]}}, - handle_message(Down0, nil, #acc{workers=Workers1}) + {ok, #acc{workers = [Worker1]}}, + handle_message(Down0, nil, #acc{workers = Workers1}) ), % A second DOWN removes the remaining workers ?assertEqual( - {stop, #acc{workers=[]}}, - handle_message(Down1, nil, #acc{workers=[Worker1]}) + {stop, #acc{workers = []}}, + handle_message(Down1, nil, #acc{workers = [Worker1]}) ) end). - t_handle_message_exit() -> Exit = {rexi_EXIT, nil}, - Worker0 = #shard{ref=erlang:make_ref()}, - Worker1 = #shard{ref=erlang:make_ref()}, + Worker0 = #shard{ref = erlang:make_ref()}, + Worker1 = #shard{ref = erlang:make_ref()}, ?_test(begin % Only removes the specified worker ?assertEqual( - {ok, #acc{workers=[Worker1]}}, - handle_message(Exit, Worker0, #acc{workers=[Worker0, Worker1]}) + {ok, #acc{workers = [Worker1]}}, + handle_message(Exit, Worker0, #acc{workers = [Worker0, Worker1]}) ), ?assertEqual( - {ok, #acc{workers=[Worker0]}}, - handle_message(Exit, Worker1, #acc{workers=[Worker0, Worker1]}) + {ok, #acc{workers = [Worker0]}}, + handle_message(Exit, Worker1, #acc{workers = [Worker0, Worker1]}) ), % We bail if it was the last worker ?assertEqual( - {stop, #acc{workers=[]}}, - handle_message(Exit, Worker0, #acc{workers=[Worker0]}) + {stop, #acc{workers = []}}, + handle_message(Exit, Worker0, #acc{workers = [Worker0]}) ) end). - t_handle_message_reply() -> - Worker0 = #shard{ref=erlang:make_ref()}, - Worker1 = #shard{ref=erlang:make_ref()}, - Worker2 = #shard{ref=erlang:make_ref()}, + Worker0 = #shard{ref = erlang:make_ref()}, + Worker1 = #shard{ref = erlang:make_ref()}, + Worker2 = #shard{ref = erlang:make_ref()}, Workers = [Worker0, Worker1, Worker2], - Acc0 = #acc{workers=Workers, r=2, replies=[]}, + Acc0 = #acc{workers = Workers, r = 2, replies = []}, ?_test(begin meck:expect(rexi, kill_all, fun(_) -> ok end), @@ -342,19 +347,19 @@ t_handle_message_reply() -> % Test that we continue when we haven't met R yet ?assertMatch( {ok, #acc{ - workers=[Worker0, Worker1], - replies=[{foo, {foo, 1}}] + workers = [Worker0, Worker1], + replies = [{foo, {foo, 1}}] }}, handle_message(foo, Worker2, Acc0) ), ?assertMatch( {ok, #acc{ - workers=[Worker0, Worker1], - replies=[{bar, {bar, 1}}, {foo, {foo, 1}}] + workers = [Worker0, Worker1], + replies = [{bar, {bar, 1}}, {foo, {foo, 1}}] }}, handle_message(bar, Worker2, Acc0#acc{ - replies=[{foo, {foo, 1}}] + replies = [{foo, {foo, 1}}] }) ), @@ -363,18 +368,18 @@ t_handle_message_reply() -> % is returned. Bit subtle on the assertions here. ?assertMatch( - {stop, #acc{workers=[], replies=[{foo, {foo, 1}}]}}, - handle_message(foo, Worker0, Acc0#acc{workers=[Worker0]}) + {stop, #acc{workers = [], replies = [{foo, {foo, 1}}]}}, + handle_message(foo, Worker0, Acc0#acc{workers = [Worker0]}) ), ?assertMatch( {stop, #acc{ - workers=[], - replies=[{bar, {bar, 1}}, {foo, {foo, 1}}] + workers = [], + replies = [{bar, {bar, 1}}, {foo, {foo, 1}}] }}, handle_message(bar, Worker0, Acc0#acc{ - workers=[Worker0], - replies=[{foo, {foo, 1}}] + workers = [Worker0], + replies = [{foo, {foo, 1}}] }) ), @@ -383,43 +388,42 @@ t_handle_message_reply() -> ?assertMatch( {stop, #acc{ - workers=[], - replies=[{foo, {foo, 2}}], - state=r_met, - q_reply=foo + workers = [], + replies = [{foo, {foo, 2}}], + state = r_met, + q_reply = foo }}, handle_message(foo, Worker1, Acc0#acc{ - workers=[Worker0, Worker1], - replies=[{foo, {foo, 1}}] + workers = [Worker0, Worker1], + replies = [{foo, {foo, 1}}] }) ), ?assertEqual( {stop, #acc{ - workers=[], - r=1, - replies=[{foo, {foo, 1}}], - state=r_met, - q_reply=foo + workers = [], + r = 1, + replies = [{foo, {foo, 1}}], + state = r_met, + q_reply = foo }}, - handle_message(foo, Worker0, Acc0#acc{r=1}) + handle_message(foo, Worker0, Acc0#acc{r = 1}) ), ?assertMatch( {stop, #acc{ - workers=[], - replies=[{bar, {bar, 1}}, {foo, {foo, 2}}], - state=r_met, - q_reply=foo + workers = [], + replies = [{bar, {bar, 1}}, {foo, {foo, 2}}], + state = r_met, + q_reply = foo }}, handle_message(foo, Worker0, Acc0#acc{ - workers=[Worker0], - replies=[{bar, {bar, 1}}, {foo, {foo, 1}}] + workers = [Worker0], + replies = [{bar, {bar, 1}}, {foo, {foo, 1}}] }) ) end). - t_store_node_revs() -> W1 = #shard{node = w1, ref = erlang:make_ref()}, W2 = #shard{node = w2, ref = erlang:make_ref()}, @@ -449,9 +453,9 @@ t_store_node_revs() -> Acc1 = InitAcc#acc{node_revs = [{w1, [{1, <<"foo">>}]}]}, {ok, #acc{node_revs = NodeRevs4}} = handle_message(Foo2, W2, Acc1), ?assertEqual( - [{w2, [{2, <<"foo2">>}]}, {w1, [{1, <<"foo">>}]}], - NodeRevs4 - ), + [{w2, [{2, <<"foo2">>}]}, {w1, [{1, <<"foo">>}]}], + NodeRevs4 + ), % Make sure rexi_DOWN doesn't modify node_revs Down = {rexi_DOWN, nil, {nil, w1}, nil}, @@ -473,15 +477,14 @@ t_store_node_revs() -> {ok, Acc3} = handle_message(Foo2, W2, Acc2), {stop, Acc4} = handle_message(NFM, W3, Acc3), ?assertEqual( - [{w2, [{2, <<"foo2">>}]}, {w1, [{1, <<"foo">>}]}], - Acc4#acc.node_revs - ) + [{w2, [{2, <<"foo2">>}]}, {w1, [{1, <<"foo">>}]}], + Acc4#acc.node_revs + ) end). - t_read_repair() -> - Foo1 = {ok, #doc{revs = {1,[<<"foo">>]}}}, - Foo2 = {ok, #doc{revs = {2,[<<"foo2">>,<<"foo">>]}}}, + Foo1 = {ok, #doc{revs = {1, [<<"foo">>]}}}, + Foo2 = {ok, #doc{revs = {2, [<<"foo2">>, <<"foo">>]}}}, NFM = {not_found, missing}, ?_test(begin @@ -492,33 +495,32 @@ t_read_repair() -> meck:expect(fabric, update_docs, fun(_, [_], _) -> {ok, []} end), Acc0 = #acc{ dbname = <<"name">>, - replies = [fabric_util:kv(Foo1,1)] + replies = [fabric_util:kv(Foo1, 1)] }, ?assertEqual(Foo1, read_repair(Acc0)), meck:expect(fabric, update_docs, fun(_, [_, _], _) -> {ok, []} end), Acc1 = #acc{ dbname = <<"name">>, - replies = [fabric_util:kv(Foo1,1), fabric_util:kv(Foo2,1)] + replies = [fabric_util:kv(Foo1, 1), fabric_util:kv(Foo2, 1)] }, ?assertEqual(Foo2, read_repair(Acc1)), % Test when we have nothing but errors - Acc2 = #acc{replies=[fabric_util:kv(NFM, 1)]}, + Acc2 = #acc{replies = [fabric_util:kv(NFM, 1)]}, ?assertEqual(NFM, read_repair(Acc2)), - Acc3 = #acc{replies=[fabric_util:kv(NFM,1), fabric_util:kv(foo,2)]}, + Acc3 = #acc{replies = [fabric_util:kv(NFM, 1), fabric_util:kv(foo, 2)]}, ?assertEqual(NFM, read_repair(Acc3)), - Acc4 = #acc{replies=[fabric_util:kv(foo,1), fabric_util:kv(bar,1)]}, + Acc4 = #acc{replies = [fabric_util:kv(foo, 1), fabric_util:kv(bar, 1)]}, ?assertEqual(bar, read_repair(Acc4)) end). - t_handle_response_quorum_met() -> - Foo1 = {ok, #doc{revs = {1,[<<"foo">>]}}}, - Foo2 = {ok, #doc{revs = {2,[<<"foo2">>,<<"foo">>]}}}, - Bar1 = {ok, #doc{revs = {1,[<<"bar">>]}}}, + Foo1 = {ok, #doc{revs = {1, [<<"foo">>]}}}, + Foo2 = {ok, #doc{revs = {2, [<<"foo2">>, <<"foo">>]}}}, + Bar1 = {ok, #doc{revs = {1, [<<"bar">>]}}}, ?_test(begin meck:expect(couch_log, notice, fun(_, _) -> ok end), @@ -526,34 +528,34 @@ t_handle_response_quorum_met() -> meck:expect(couch_stats, increment_counter, fun(_) -> ok end), BasicOkAcc = #acc{ - state=r_met, - replies=[fabric_util:kv(Foo1,2)], - q_reply=Foo1 + state = r_met, + replies = [fabric_util:kv(Foo1, 2)], + q_reply = Foo1 }, ?assertEqual(Foo1, handle_response(BasicOkAcc)), WithAncestorsAcc = #acc{ - state=r_met, - replies=[fabric_util:kv(Foo1,1), fabric_util:kv(Foo2,2)], - q_reply=Foo2 + state = r_met, + replies = [fabric_util:kv(Foo1, 1), fabric_util:kv(Foo2, 2)], + q_reply = Foo2 }, ?assertEqual(Foo2, handle_response(WithAncestorsAcc)), % This also checks when the quorum isn't the most recent % revision. DeeperWinsAcc = #acc{ - state=r_met, - replies=[fabric_util:kv(Foo1,2), fabric_util:kv(Foo2,1)], - q_reply=Foo1 + state = r_met, + replies = [fabric_util:kv(Foo1, 2), fabric_util:kv(Foo2, 1)], + q_reply = Foo1 }, ?assertEqual(Foo2, handle_response(DeeperWinsAcc)), % Check that we return the proper doc based on rev % (ie, pos is equal) BiggerRevWinsAcc = #acc{ - state=r_met, - replies=[fabric_util:kv(Foo1,1), fabric_util:kv(Bar1,2)], - q_reply=Bar1 + state = r_met, + replies = [fabric_util:kv(Foo1, 1), fabric_util:kv(Bar1, 2)], + q_reply = Bar1 }, ?assertEqual(Foo1, handle_response(BiggerRevWinsAcc)) @@ -561,7 +563,6 @@ t_handle_response_quorum_met() -> % read_repair_test for those conditions. end). - t_get_doc_info() -> ?_test(begin meck:expect(fabric, update_docs, fun(_, _, _) -> {ok, []} end), @@ -603,7 +604,7 @@ t_get_doc_info() -> end), Rsp3 = fabric_doc_open:go("test", "one", [{doc_info, full}]), ?assertEqual({not_found, deleted}, Rsp3), - {ok, Rec2} = fabric_doc_open:go("test", "one", [{doc_info, full},deleted]), + {ok, Rec2} = fabric_doc_open:go("test", "one", [{doc_info, full}, deleted]), ?assert(is_record(Rec2, full_doc_info)) end). diff --git a/src/fabric/src/fabric_doc_open_revs.erl b/src/fabric/src/fabric_doc_open_revs.erl index 3d7b9dc3c..284187bff 100644 --- a/src/fabric/src/fabric_doc_open_revs.erl +++ b/src/fabric/src/fabric_doc_open_revs.erl @@ -34,8 +34,11 @@ }). go(DbName, Id, Revs, Options) -> - Workers = fabric_util:submit_jobs(mem3:shards(DbName,Id), open_revs, - [Id, Revs, Options]), + Workers = fabric_util:submit_jobs( + mem3:shards(DbName, Id), + open_revs, + [Id, Revs, Options] + ), R = couch_util:get_value(r, Options, integer_to_list(mem3:quorum(DbName))), State = #state{ dbname = DbName, @@ -48,34 +51,31 @@ go(DbName, Id, Revs, Options) -> }, RexiMon = fabric_util:create_monitors(Workers), try fabric_util:recv(Workers, #shard.ref, fun handle_message/3, State) of - {ok, all_workers_died} -> - {error, all_workers_died}; - {ok, Replies} -> - {ok, Replies}; - {timeout, #state{workers=DefunctWorkers}} -> - fabric_util:log_timeout(DefunctWorkers, "open_revs"), - {error, timeout}; - Else -> - Else + {ok, all_workers_died} -> + {error, all_workers_died}; + {ok, Replies} -> + {ok, Replies}; + {timeout, #state{workers = DefunctWorkers}} -> + fabric_util:log_timeout(DefunctWorkers, "open_revs"), + {error, timeout}; + Else -> + Else after rexi_monitor:stop(RexiMon) end. - -handle_message({rexi_DOWN, _, {_,NodeRef},_}, _Worker, #state{workers=Workers}=State) -> +handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _Worker, #state{workers = Workers} = State) -> NewState = State#state{ workers = lists:keydelete(NodeRef, #shard.node, Workers), reply_error_count = State#state.reply_error_count + 1 }, handle_message({ok, []}, nil, NewState); - -handle_message({rexi_EXIT, _}, Worker, #state{workers=Workers}=State) -> +handle_message({rexi_EXIT, _}, Worker, #state{workers = Workers} = State) -> NewState = State#state{ workers = lists:delete(Worker, Workers), reply_error_count = State#state.reply_error_count + 1 }, handle_message({ok, []}, nil, NewState); - handle_message({ok, RawReplies}, Worker, State) -> #state{ dbname = DbName, @@ -96,30 +96,40 @@ handle_message({ok, RawReplies}, Worker, State) -> % Do not count error replies when checking quorum RealReplyCount = ReplyCount + 1 - ReplyErrorCount, QuorumReplies = RealReplyCount >= R, - {NewReplies, QuorumMet, Repair} = case IsTree of - true -> - {NewReplies0, AllInternal, Repair0} = + {NewReplies, QuorumMet, Repair} = + case IsTree of + true -> + {NewReplies0, AllInternal, Repair0} = tree_replies(PrevReplies, tree_sort(RawReplies)), - NumLeafs = couch_key_tree:count_leafs(PrevReplies), - SameNumRevs = length(RawReplies) == NumLeafs, - QMet = AllInternal andalso SameNumRevs andalso QuorumReplies, - % Don't set repair=true on the first reply - {NewReplies0, QMet, (ReplyCount > 0) and Repair0}; - false -> - {NewReplies0, MinCount} = dict_replies(PrevReplies, RawReplies), - {NewReplies0, MinCount >= R, false} - end, - NewNodeRevs = if Worker == nil -> PrevNodeRevs; true -> - IdRevs = lists:foldl(fun - ({ok, #doc{revs = {Pos, [Rev | _]}}}, Acc) -> - [{Pos, Rev} | Acc]; - (_, Acc) -> - Acc - end, [], RawReplies), - if IdRevs == [] -> PrevNodeRevs; true -> - [{Worker#shard.node, IdRevs} | PrevNodeRevs] - end - end, + NumLeafs = couch_key_tree:count_leafs(PrevReplies), + SameNumRevs = length(RawReplies) == NumLeafs, + QMet = AllInternal andalso SameNumRevs andalso QuorumReplies, + % Don't set repair=true on the first reply + {NewReplies0, QMet, (ReplyCount > 0) and Repair0}; + false -> + {NewReplies0, MinCount} = dict_replies(PrevReplies, RawReplies), + {NewReplies0, MinCount >= R, false} + end, + NewNodeRevs = + if + Worker == nil -> + PrevNodeRevs; + true -> + IdRevs = lists:foldl( + fun + ({ok, #doc{revs = {Pos, [Rev | _]}}}, Acc) -> + [{Pos, Rev} | Acc]; + (_, Acc) -> + Acc + end, + [], + RawReplies + ), + if + IdRevs == [] -> PrevNodeRevs; + true -> [{Worker#shard.node, IdRevs} | PrevNodeRevs] + end + end, Complete = (ReplyCount =:= (WorkerCount - 1)), @@ -127,13 +137,13 @@ handle_message({ok, RawReplies}, Worker, State) -> true -> fabric_util:cleanup(lists:delete(Worker, Workers)), maybe_read_repair( - DbName, - IsTree, - NewReplies, - NewNodeRevs, - ReplyCount + 1, - InRepair orelse Repair - ), + DbName, + IsTree, + NewReplies, + NewNodeRevs, + ReplyCount + 1, + InRepair orelse Repair + ), {stop, format_reply(IsTree, NewReplies, RealReplyCount)}; false -> {ok, State#state{ @@ -145,10 +155,8 @@ handle_message({ok, RawReplies}, Worker, State) -> }} end. - tree_replies(RevTree, []) -> {RevTree, true, false}; - tree_replies(RevTree0, [{ok, Doc} | Rest]) -> {RevTree1, Done, Repair} = tree_replies(RevTree0, Rest), Path = couch_doc:to_path(Doc), @@ -160,7 +168,6 @@ tree_replies(RevTree0, [{ok, Doc} | Rest]) -> {RevTree2, _} -> {RevTree2, false, true} end; - tree_replies(RevTree0, [{{not_found, missing}, {Pos, Rev}} | Rest]) -> {RevTree1, Done, Repair} = tree_replies(RevTree0, Rest), Node = {Rev, ?REV_MISSING, []}, @@ -172,34 +179,30 @@ tree_replies(RevTree0, [{{not_found, missing}, {Pos, Rev}} | Rest]) -> {RevTree2, false, Repair} end. - tree_sort(Replies) -> SortFun = fun(A, B) -> sort_key(A) =< sort_key(B) end, lists:sort(SortFun, Replies). - sort_key({ok, #doc{revs = {Pos, [Rev | _]}}}) -> {Pos, Rev}; sort_key({{not_found, _}, {Pos, Rev}}) -> {Pos, Rev}. - dict_replies(Dict, []) -> case [Count || {_Key, {_Reply, Count}} <- Dict] of [] -> {Dict, 0}; Counts -> {Dict, lists:min(Counts)} end; - dict_replies(Dict, [Reply | Rest]) -> NewDict = fabric_util:update_counter(Reply, 1, Dict), dict_replies(NewDict, Rest). - maybe_read_repair(Db, IsTree, Replies, NodeRevs, ReplyCount, DoRepair) -> - Docs = case IsTree of - true -> tree_repair_docs(Replies, DoRepair); - false -> dict_repair_docs(Replies, ReplyCount) - end, + Docs = + case IsTree of + true -> tree_repair_docs(Replies, DoRepair); + false -> dict_repair_docs(Replies, ReplyCount) + end, case Docs of [] -> ok; @@ -207,22 +210,19 @@ maybe_read_repair(Db, IsTree, Replies, NodeRevs, ReplyCount, DoRepair) -> erlang:spawn(fun() -> read_repair(Db, Docs, NodeRevs) end) end. - tree_repair_docs(_Replies, false) -> []; - tree_repair_docs(Replies, true) -> Leafs = couch_key_tree:get_all_leafs(Replies), [Doc || {Doc, {_Pos, _}} <- Leafs, is_record(Doc, doc)]. - dict_repair_docs(Replies, ReplyCount) -> NeedsRepair = lists:any(fun({_, {_, C}}) -> C < ReplyCount end, Replies), - if not NeedsRepair -> []; true -> - [Doc || {_, {{ok, Doc}, _}} <- Replies] + if + not NeedsRepair -> []; + true -> [Doc || {_, {{ok, Doc}, _}} <- Replies] end. - read_repair(Db, Docs, NodeRevs) -> Opts = [?ADMIN_CTX, replicated_changes, {read_repair, NodeRevs}], Res = fabric:update_docs(Db, Docs, Opts), @@ -235,57 +235,63 @@ read_repair(Db, Docs, NodeRevs) -> couch_log:notice("read_repair ~s ~s ~p", [Db, Id, Res]) end. - format_reply(_, _, RealReplyCount) when RealReplyCount =< 0 -> all_workers_died; - format_reply(true, Replies, _) -> tree_format_replies(Replies); - format_reply(false, Replies, _) -> dict_format_replies(Replies). - tree_format_replies(RevTree) -> Leafs = couch_key_tree:get_all_leafs(RevTree), - lists:sort(lists:map(fun(Reply) -> - case Reply of - {?REV_MISSING, {Pos, [Rev]}} -> - {{not_found, missing}, {Pos, Rev}}; - {Doc, _} when is_record(Doc, doc) -> - {ok, Doc} - end - end, Leafs)). - + lists:sort( + lists:map( + fun(Reply) -> + case Reply of + {?REV_MISSING, {Pos, [Rev]}} -> + {{not_found, missing}, {Pos, Rev}}; + {Doc, _} when is_record(Doc, doc) -> + {ok, Doc} + end + end, + Leafs + ) + ). dict_format_replies(Dict) -> Replies0 = [Reply || {_, {Reply, _}} <- Dict], - AllFoundRevs = lists:foldl(fun(Reply, Acc) -> - case Reply of - {ok, #doc{revs = {Pos, [RevId | _]}}} -> - [{Pos, RevId} | Acc]; - _ -> - Acc - end - end, [], Replies0), + AllFoundRevs = lists:foldl( + fun(Reply, Acc) -> + case Reply of + {ok, #doc{revs = {Pos, [RevId | _]}}} -> + [{Pos, RevId} | Acc]; + _ -> + Acc + end + end, + [], + Replies0 + ), %% Drop any not_found replies for which we %% found the revision on a different node. - Replies1 = lists:filter(fun(Reply) -> - case Reply of - {{not_found, missing}, Rev} -> - not lists:member(Rev, AllFoundRevs); - _ -> - true - end - end, Replies0), + Replies1 = lists:filter( + fun(Reply) -> + case Reply of + {{not_found, missing}, Rev} -> + not lists:member(Rev, AllFoundRevs); + _ -> + true + end + end, + Replies0 + ), % Remove replies with shorter revision % paths for a given revision. collapse_duplicate_revs(Replies1). - collapse_duplicate_revs(Replies) -> % The collapse logic requires that replies are % sorted so that shorter rev paths are in @@ -296,27 +302,23 @@ collapse_duplicate_revs(Replies) -> % of C. collapse_duplicate_revs_int(lists:sort(Replies)). - collapse_duplicate_revs_int([]) -> []; - collapse_duplicate_revs_int([{ok, Doc1}, {ok, Doc2} | Rest]) -> {D1, R1} = Doc1#doc.revs, {D2, R2} = Doc2#doc.revs, - Head = case D1 == D2 andalso lists:prefix(R1, R2) of - true -> []; - false -> [{ok, Doc1}] - end, + Head = + case D1 == D2 andalso lists:prefix(R1, R2) of + true -> []; + false -> [{ok, Doc1}] + end, Head ++ collapse_duplicate_revs([{ok, Doc2} | Rest]); - collapse_duplicate_revs_int([Reply | Rest]) -> [Reply | collapse_duplicate_revs(Rest)]. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). - setup_all() -> config:start_link([]), meck:new([fabric, couch_stats, couch_log]), @@ -326,13 +328,10 @@ setup_all() -> meck:expect(couch_log, notice, fun(_, _) -> ok end), meck:expect(fabric_util, cleanup, fun(_) -> ok end). - - teardown_all(_) -> meck:unload(), config:stop(). - setup() -> meck:reset([ couch_log, @@ -341,37 +340,31 @@ setup() -> fabric_util ]). - teardown(_) -> ok. - state0(Revs, Latest) -> #state{ worker_count = 3, workers = - [#shard{node='node1'}, #shard{node='node2'}, #shard{node='node3'}], + [#shard{node = 'node1'}, #shard{node = 'node2'}, #shard{node = 'node3'}], r = 2, revs = Revs, latest = Latest }. - -revs() -> [{1,<<"foo">>}, {1,<<"bar">>}, {1,<<"baz">>}]. - +revs() -> [{1, <<"foo">>}, {1, <<"bar">>}, {1, <<"baz">>}]. foo1() -> {ok, #doc{revs = {1, [<<"foo">>]}}}. foo2() -> {ok, #doc{revs = {2, [<<"foo2">>, <<"foo">>]}}}. foo2stemmed() -> {ok, #doc{revs = {2, [<<"foo2">>]}}}. -fooNF() -> {{not_found, missing}, {1,<<"foo">>}}. +fooNF() -> {{not_found, missing}, {1, <<"foo">>}}. foo2NF() -> {{not_found, missing}, {2, <<"foo2">>}}. bar1() -> {ok, #doc{revs = {1, [<<"bar">>]}}}. -barNF() -> {{not_found, missing}, {1,<<"bar">>}}. -bazNF() -> {{not_found, missing}, {1,<<"baz">>}}. +barNF() -> {{not_found, missing}, {1, <<"bar">>}}. +bazNF() -> {{not_found, missing}, {1, <<"baz">>}}. baz1() -> {ok, #doc{revs = {1, [<<"baz">>]}}}. - - open_doc_revs_test_() -> { setup, @@ -414,52 +407,47 @@ open_doc_revs_test_() -> } }. - % Tests for revs=all - check_empty_response_not_quorum() -> % Simple smoke test that we don't think we're % done with a first empty response - W1 = #shard{node='node1'}, - W2 = #shard{node='node2'}, - W3 = #shard{node='node3'}, + W1 = #shard{node = 'node1'}, + W2 = #shard{node = 'node2'}, + W3 = #shard{node = 'node3'}, ?_assertMatch( {ok, #state{workers = [W2, W3]}}, handle_message({ok, []}, W1, state0(all, false)) ). - check_basic_response() -> % Check that we've handle a response - W1 = #shard{node='node1'}, - W2 = #shard{node='node2'}, - W3 = #shard{node='node3'}, + W1 = #shard{node = 'node1'}, + W2 = #shard{node = 'node2'}, + W3 = #shard{node = 'node3'}, ?_assertMatch( {ok, #state{reply_count = 1, workers = [W2, W3]}}, handle_message({ok, [foo1(), bar1()]}, W1, state0(all, false)) ). - check_finish_quorum() -> % Two messages with the same revisions means we're done ?_test(begin - W1 = #shard{node='node1'}, - W2 = #shard{node='node2'}, + W1 = #shard{node = 'node1'}, + W2 = #shard{node = 'node2'}, S0 = state0(all, false), {ok, S1} = handle_message({ok, [foo1(), bar1()]}, W1, S0), Expect = {stop, [bar1(), foo1()]}, ?assertEqual(Expect, handle_message({ok, [foo1(), bar1()]}, W2, S1)) end). - check_finish_quorum_newer() -> % We count a descendant of a revision for quorum so % foo1 should count for foo2 which means we're finished. % We also validate that read_repair was triggered. ?_test(begin - W1 = #shard{node='node1'}, - W2 = #shard{node='node2'}, + W1 = #shard{node = 'node1'}, + W2 = #shard{node = 'node2'}, S0 = state0(all, false), {ok, S1} = handle_message({ok, [foo1(), bar1()]}, W1, S0), Expect = {stop, [bar1(), foo2()]}, @@ -472,14 +460,13 @@ check_finish_quorum_newer() -> ) end). - check_no_quorum_on_second() -> % Quorum not yet met for the foo revision so we % would wait for w3 ?_test(begin - W1 = #shard{node='node1'}, - W2 = #shard{node='node2'}, - W3 = #shard{node='node3'}, + W1 = #shard{node = 'node1'}, + W2 = #shard{node = 'node2'}, + W3 = #shard{node = 'node3'}, S0 = state0(all, false), {ok, S1} = handle_message({ok, [foo1(), bar1()]}, W1, S0), ?assertMatch( @@ -488,15 +475,14 @@ check_no_quorum_on_second() -> ) end). - check_done_on_third() -> % The third message of three means we're done no matter % what. Every revision seen in this pattern should be % included. ?_test(begin - W1 = #shard{node='node1'}, - W2 = #shard{node='node2'}, - W3 = #shard{node='node3'}, + W1 = #shard{node = 'node1'}, + W2 = #shard{node = 'node2'}, + W3 = #shard{node = 'node3'}, S0 = state0(all, false), {ok, S1} = handle_message({ok, [foo1(), bar1()]}, W1, S0), {ok, S2} = handle_message({ok, [bar1()]}, W2, S1), @@ -504,15 +490,13 @@ check_done_on_third() -> ?assertEqual(Expect, handle_message({ok, [bar1()]}, W3, S2)) end). - % Tests for a specific list of revs - check_specific_revs_first_msg() -> ?_test(begin - W1 = #shard{node='node1'}, - W2 = #shard{node='node2'}, - W3 = #shard{node='node3'}, + W1 = #shard{node = 'node1'}, + W2 = #shard{node = 'node2'}, + W3 = #shard{node = 'node3'}, S0 = state0(revs(), false), ?assertMatch( {ok, #state{reply_count = 1, workers = [W2, W3]}}, @@ -520,11 +504,10 @@ check_specific_revs_first_msg() -> ) end). - check_revs_done_on_agreement() -> ?_test(begin - W1 = #shard{node='node1'}, - W2 = #shard{node='node2'}, + W1 = #shard{node = 'node1'}, + W2 = #shard{node = 'node2'}, S0 = state0(revs(), false), Msg = {ok, [foo1(), bar1(), bazNF()]}, {ok, S1} = handle_message(Msg, W1, S0), @@ -532,11 +515,10 @@ check_revs_done_on_agreement() -> ?assertEqual(Expect, handle_message(Msg, W2, S1)) end). - check_latest_true() -> ?_test(begin - W1 = #shard{node='node1'}, - W2 = #shard{node='node2'}, + W1 = #shard{node = 'node1'}, + W2 = #shard{node = 'node2'}, S0 = state0(revs(), true), Msg1 = {ok, [foo2(), bar1(), bazNF()]}, Msg2 = {ok, [foo2(), bar1(), bazNF()]}, @@ -545,11 +527,10 @@ check_latest_true() -> ?assertEqual(Expect, handle_message(Msg2, W2, S1)) end). - check_ancestor_counted_in_quorum() -> ?_test(begin - W1 = #shard{node='node1'}, - W2 = #shard{node='node2'}, + W1 = #shard{node = 'node1'}, + W2 = #shard{node = 'node2'}, S0 = state0(revs(), true), Msg1 = {ok, [foo1(), bar1(), bazNF()]}, Msg2 = {ok, [foo2(), bar1(), bazNF()]}, @@ -564,11 +545,10 @@ check_ancestor_counted_in_quorum() -> ?assertEqual(Expect, handle_message(Msg1, W1, S2)) end). - check_not_found_counts_for_descendant() -> ?_test(begin - W1 = #shard{node='node1'}, - W2 = #shard{node='node2'}, + W1 = #shard{node = 'node1'}, + W2 = #shard{node = 'node2'}, S0 = state0(revs(), true), Msg1 = {ok, [foo1(), bar1(), bazNF()]}, Msg2 = {ok, [foo1(), bar1(), baz1()]}, @@ -583,12 +563,11 @@ check_not_found_counts_for_descendant() -> ?assertEqual(Expect, handle_message(Msg1, W1, S2)) end). - check_worker_error_skipped() -> ?_test(begin - W1 = #shard{node='node1'}, - W2 = #shard{node='node2'}, - W3 = #shard{node='node3'}, + W1 = #shard{node = 'node1'}, + W2 = #shard{node = 'node2'}, + W3 = #shard{node = 'node3'}, S0 = state0(revs(), true), Msg1 = {ok, [foo1(), bar1(), baz1()]}, Msg2 = {rexi_EXIT, reason}, @@ -600,12 +579,11 @@ check_worker_error_skipped() -> ?assertEqual(Expect, handle_message(Msg3, W3, S2)) end). - check_quorum_only_counts_valid_responses() -> ?_test(begin - W1 = #shard{node='node1'}, - W2 = #shard{node='node2'}, - W3 = #shard{node='node3'}, + W1 = #shard{node = 'node1'}, + W2 = #shard{node = 'node2'}, + W3 = #shard{node = 'node3'}, S0 = state0(revs(), true), Msg1 = {rexi_EXIT, reason}, Msg2 = {rexi_EXIT, reason}, @@ -617,12 +595,11 @@ check_quorum_only_counts_valid_responses() -> ?assertEqual(Expect, handle_message(Msg3, W3, S2)) end). - check_empty_list_when_no_workers_reply() -> ?_test(begin - W1 = #shard{node='node1'}, - W2 = #shard{node='node2'}, - W3 = #shard{node='node3'}, + W1 = #shard{node = 'node1'}, + W2 = #shard{node = 'node2'}, + W3 = #shard{node = 'node3'}, S0 = state0(revs(), true), Msg1 = {rexi_EXIT, reason}, Msg2 = {rexi_EXIT, reason}, @@ -634,7 +611,6 @@ check_empty_list_when_no_workers_reply() -> ?assertEqual(Expect, handle_message(Msg3, W3, S2)) end). - check_node_rev_stored() -> ?_test(begin W1 = #shard{node = node1}, @@ -644,7 +620,6 @@ check_node_rev_stored() -> ?assertEqual([{node1, [{1, <<"foo">>}]}], S1#state.node_revs) end). - check_node_rev_store_head_only() -> ?_test(begin W1 = #shard{node = node1}, @@ -654,7 +629,6 @@ check_node_rev_store_head_only() -> ?assertEqual([{node1, [{2, <<"foo2">>}]}], S1#state.node_revs) end). - check_node_rev_store_multiple() -> ?_test(begin W1 = #shard{node = node1}, @@ -662,12 +636,11 @@ check_node_rev_store_multiple() -> {ok, S1} = handle_message({ok, [foo1(), foo2()]}, W1, S0), ?assertEqual( - [{node1, [{2, <<"foo2">>}, {1, <<"foo">>}]}], - S1#state.node_revs - ) + [{node1, [{2, <<"foo2">>}, {1, <<"foo">>}]}], + S1#state.node_revs + ) end). - check_node_rev_dont_store_errors() -> ?_test(begin W1 = #shard{node = node1}, @@ -677,7 +650,6 @@ check_node_rev_dont_store_errors() -> ?assertEqual([], S1#state.node_revs) end). - check_node_rev_store_non_errors() -> ?_test(begin W1 = #shard{node = node1}, @@ -687,7 +659,6 @@ check_node_rev_store_non_errors() -> ?assertEqual([{node1, [{1, <<"foo">>}]}], S1#state.node_revs) end). - check_node_rev_store_concatenate() -> ?_test(begin W2 = #shard{node = node2}, @@ -696,12 +667,11 @@ check_node_rev_store_concatenate() -> {ok, S2} = handle_message({ok, [foo2()]}, W2, S1), ?assertEqual( - [{node2, [{2, <<"foo2">>}]}, {node1, [{1, <<"foo">>}]}], - S2#state.node_revs - ) + [{node2, [{2, <<"foo2">>}]}, {node1, [{1, <<"foo">>}]}], + S2#state.node_revs + ) end). - check_node_rev_store_concantenate_multiple() -> ?_test(begin W2 = #shard{node = node2}, @@ -710,15 +680,14 @@ check_node_rev_store_concantenate_multiple() -> {ok, S2} = handle_message({ok, [foo2(), bar1()]}, W2, S1), ?assertEqual( - [ - {node2, [{1, <<"bar">>}, {2, <<"foo2">>}]}, - {node1, [{1, <<"foo">>}]} - ], - S2#state.node_revs - ) + [ + {node2, [{1, <<"bar">>}, {2, <<"foo2">>}]}, + {node1, [{1, <<"foo">>}]} + ], + S2#state.node_revs + ) end). - check_node_rev_unmodified_on_down_or_exit() -> ?_test(begin W2 = #shard{node = node2}, @@ -728,19 +697,18 @@ check_node_rev_unmodified_on_down_or_exit() -> Down = {rexi_DOWN, nodedown, {nil, node()}, nil}, {ok, S2} = handle_message(Down, W2, S1), ?assertEqual( - [{node1, [{1, <<"foo">>}]}], - S2#state.node_revs - ), + [{node1, [{1, <<"foo">>}]}], + S2#state.node_revs + ), Exit = {rexi_EXIT, reason}, {ok, S3} = handle_message(Exit, W2, S1), ?assertEqual( - [{node1, [{1, <<"foo">>}]}], - S3#state.node_revs - ) + [{node1, [{1, <<"foo">>}]}], + S3#state.node_revs + ) end). - check_not_found_replies_are_removed_when_doc_found() -> ?_test(begin Replies = replies_to_dict([foo1(), bar1(), fooNF()]), @@ -786,7 +754,6 @@ check_not_found_removed_and_longer_rev_list() -> ?assertEqual(Expect, dict_format_replies(Replies)) end). - replies_to_dict(Replies) -> [reply_to_element(R) || R <- Replies]. diff --git a/src/fabric/src/fabric_doc_purge.erl b/src/fabric/src/fabric_doc_purge.erl index 3492f88c5..64221ab0e 100644 --- a/src/fabric/src/fabric_doc_purge.erl +++ b/src/fabric/src/fabric_doc_purge.erl @@ -12,16 +12,13 @@ -module(fabric_doc_purge). - -export([ go/3 ]). - -include_lib("fabric/include/fabric.hrl"). -include_lib("mem3/include/mem3.hrl"). - -record(acc, { worker_uuids, resps, @@ -29,7 +26,6 @@ w }). - go(_, [], _) -> {ok, []}; go(DbName, IdsRevs, Options) -> @@ -37,20 +33,32 @@ go(DbName, IdsRevs, Options) -> {UUIDs, Reqs} = create_reqs(IdsRevs, [], []), % Fire off rexi workers for each shard. - {Workers, WorkerUUIDs} = dict:fold(fun(Shard, ShardReqs, {Ws, WUUIDs}) -> - #shard{name = ShardDbName, node = Node} = Shard, - Args = [ShardDbName, ShardReqs, Options], - Ref = rexi:cast(Node, {fabric_rpc, purge_docs, Args}), - Worker = Shard#shard{ref=Ref}, - ShardUUIDs = [UUID || {UUID, _Id, _Revs} <- ShardReqs], - {[Worker | Ws], [{Worker, ShardUUIDs} | WUUIDs]} - end, {[], []}, group_reqs_by_shard(DbName, Reqs)), - - UUIDCounts = lists:foldl(fun({_Worker, WUUIDs}, CountAcc) -> - lists:foldl(fun(UUID, InnerCountAcc) -> - dict:update_counter(UUID, 1, InnerCountAcc) - end, CountAcc, WUUIDs) - end, dict:new(), WorkerUUIDs), + {Workers, WorkerUUIDs} = dict:fold( + fun(Shard, ShardReqs, {Ws, WUUIDs}) -> + #shard{name = ShardDbName, node = Node} = Shard, + Args = [ShardDbName, ShardReqs, Options], + Ref = rexi:cast(Node, {fabric_rpc, purge_docs, Args}), + Worker = Shard#shard{ref = Ref}, + ShardUUIDs = [UUID || {UUID, _Id, _Revs} <- ShardReqs], + {[Worker | Ws], [{Worker, ShardUUIDs} | WUUIDs]} + end, + {[], []}, + group_reqs_by_shard(DbName, Reqs) + ), + + UUIDCounts = lists:foldl( + fun({_Worker, WUUIDs}, CountAcc) -> + lists:foldl( + fun(UUID, InnerCountAcc) -> + dict:update_counter(UUID, 1, InnerCountAcc) + end, + CountAcc, + WUUIDs + ) + end, + dict:new(), + WorkerUUIDs + ), RexiMon = fabric_util:create_monitors(Workers), Timeout = fabric_util:request_timeout(), @@ -60,29 +68,37 @@ go(DbName, IdsRevs, Options) -> uuid_counts = UUIDCounts, w = w(DbName, Options) }, - Acc2 = try rexi_utils:recv(Workers, #shard.ref, - fun handle_message/3, Acc0, infinity, Timeout) of - {ok, Acc1} -> - Acc1; - {timeout, Acc1} -> - #acc{ - worker_uuids = WorkerUUIDs, - resps = Resps - } = Acc1, - DefunctWorkers = [Worker || {Worker, _} <- WorkerUUIDs], - fabric_util:log_timeout(DefunctWorkers, "purge_docs"), - NewResps = append_errors(timeout, WorkerUUIDs, Resps), - Acc1#acc{worker_uuids = [], resps = NewResps}; - Else -> - Else - after - rexi_monitor:stop(RexiMon) - end, + Acc2 = + try + rexi_utils:recv( + Workers, + #shard.ref, + fun handle_message/3, + Acc0, + infinity, + Timeout + ) + of + {ok, Acc1} -> + Acc1; + {timeout, Acc1} -> + #acc{ + worker_uuids = WorkerUUIDs, + resps = Resps + } = Acc1, + DefunctWorkers = [Worker || {Worker, _} <- WorkerUUIDs], + fabric_util:log_timeout(DefunctWorkers, "purge_docs"), + NewResps = append_errors(timeout, WorkerUUIDs, Resps), + Acc1#acc{worker_uuids = [], resps = NewResps}; + Else -> + Else + after + rexi_monitor:stop(RexiMon) + end, FinalResps = format_resps(UUIDs, Acc2), {resp_health(FinalResps), FinalResps}. - handle_message({rexi_DOWN, _, {_, Node}, _}, _Worker, Acc) -> #acc{ worker_uuids = WorkerUUIDs, @@ -92,7 +108,6 @@ handle_message({rexi_DOWN, _, {_, Node}, _}, _Worker, Acc) -> {Failed, Rest} = lists:partition(Pred, WorkerUUIDs), NewResps = append_errors(internal_server_error, Failed, Resps), maybe_stop(Acc#acc{worker_uuids = Rest, resps = NewResps}); - handle_message({rexi_EXIT, _}, Worker, Acc) -> #acc{ worker_uuids = WorkerUUIDs, @@ -101,7 +116,6 @@ handle_message({rexi_EXIT, _}, Worker, Acc) -> {value, WorkerPair, Rest} = lists:keytake(Worker, 1, WorkerUUIDs), NewResps = append_errors(internal_server_error, [WorkerPair], Resps), maybe_stop(Acc#acc{worker_uuids = Rest, resps = NewResps}); - handle_message({ok, Replies}, Worker, Acc) -> #acc{ worker_uuids = WorkerUUIDs, @@ -110,43 +124,49 @@ handle_message({ok, Replies}, Worker, Acc) -> {value, {_W, UUIDs}, Rest} = lists:keytake(Worker, 1, WorkerUUIDs), NewResps = append_resps(UUIDs, Replies, Resps), maybe_stop(Acc#acc{worker_uuids = Rest, resps = NewResps}); - handle_message({bad_request, Msg}, _, _) -> throw({bad_request, Msg}). - create_reqs([], UUIDs, Reqs) -> {lists:reverse(UUIDs), lists:reverse(Reqs)}; - create_reqs([{Id, Revs} | RestIdsRevs], UUIDs, Reqs) -> UUID = couch_uuids:new(), NewUUIDs = [UUID | UUIDs], NewReqs = [{UUID, Id, Revs} | Reqs], create_reqs(RestIdsRevs, NewUUIDs, NewReqs). - group_reqs_by_shard(DbName, Reqs) -> - lists:foldl(fun({_UUID, Id, _Revs} = Req, D0) -> - lists:foldl(fun(Shard, D1) -> - dict:append(Shard, Req, D1) - end, D0, mem3:shards(DbName, Id)) - end, dict:new(), Reqs). - + lists:foldl( + fun({_UUID, Id, _Revs} = Req, D0) -> + lists:foldl( + fun(Shard, D1) -> + dict:append(Shard, Req, D1) + end, + D0, + mem3:shards(DbName, Id) + ) + end, + dict:new(), + Reqs + ). w(DbName, Options) -> try list_to_integer(couch_util:get_value(w, Options)) - catch _:_ -> - mem3:quorum(DbName) + catch + _:_ -> + mem3:quorum(DbName) end. - append_errors(Type, WorkerUUIDs, Resps) -> - lists:foldl(fun({_Worker, UUIDs}, RespAcc) -> - Errors = [{error, Type} || _UUID <- UUIDs], - append_resps(UUIDs, Errors, RespAcc) - end, Resps, WorkerUUIDs). - + lists:foldl( + fun({_Worker, UUIDs}, RespAcc) -> + Errors = [{error, Type} || _UUID <- UUIDs], + append_resps(UUIDs, Errors, RespAcc) + end, + Resps, + WorkerUUIDs + ). append_resps([], [], Resps) -> Resps; @@ -154,24 +174,27 @@ append_resps([UUID | RestUUIDs], [Reply | RestReplies], Resps) -> NewResps = dict:append(UUID, Reply, Resps), append_resps(RestUUIDs, RestReplies, NewResps). - maybe_stop(#acc{worker_uuids = []} = Acc) -> {stop, Acc}; maybe_stop(#acc{resps = Resps, uuid_counts = Counts, w = W} = Acc) -> try - dict:fold(fun(UUID, UUIDResps, _) -> - UUIDCount = dict:fetch(UUID, Counts), - case has_quorum(UUIDResps, UUIDCount, W) of - true -> ok; - false -> throw(keep_going) - end - end, nil, Resps), + dict:fold( + fun(UUID, UUIDResps, _) -> + UUIDCount = dict:fetch(UUID, Counts), + case has_quorum(UUIDResps, UUIDCount, W) of + true -> ok; + false -> throw(keep_going) + end + end, + nil, + Resps + ), {stop, Acc} - catch throw:keep_going -> - {ok, Acc} + catch + throw:keep_going -> + {ok, Acc} end. - format_resps(UUIDs, #acc{} = Acc) -> #acc{ resps = Resps, @@ -185,19 +208,22 @@ format_resps(UUIDs, #acc{} = Acc) -> [{UUID, Error} | ReplyAcc]; _ -> AllRevs = lists:usort(lists:flatten(OkReplies)), - IsOk = length(OkReplies) >= W - andalso length(lists:usort(OkReplies)) == 1, - Health = if IsOk -> ok; true -> accepted end, + IsOk = + length(OkReplies) >= W andalso + length(lists:usort(OkReplies)) == 1, + Health = + if + IsOk -> ok; + true -> accepted + end, [{UUID, {Health, AllRevs}} | ReplyAcc] end end, FinalReplies = dict:fold(FoldFun, [], Resps), couch_util:reorder_results(UUIDs, FinalReplies); - format_resps(_UUIDs, Else) -> Else. - resp_health(Resps) -> Healths = lists:usort([H || {H, _} <- Resps]), HasError = lists:member(error, Healths), @@ -210,12 +236,15 @@ resp_health(Resps) -> true -> error end. - has_quorum(Resps, Count, W) -> OkResps = [R || {ok, _} = R <- Resps], - OkCounts = lists:foldl(fun(R, Acc) -> - orddict:update_counter(R, 1, Acc) - end, orddict:new(), OkResps), + OkCounts = lists:foldl( + fun(R, Acc) -> + orddict:update_counter(R, 1, Acc) + end, + orddict:new(), + OkResps + ), MaxOk = lists:max([0 | element(2, lists:unzip(OkCounts))]), if MaxOk >= W -> true; @@ -223,7 +252,6 @@ has_quorum(Resps, Count, W) -> true -> false end. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -250,17 +278,14 @@ purge_test_() -> ] }. - setup() -> meck:new(couch_log), meck:expect(couch_log, warning, fun(_, _) -> ok end), meck:expect(couch_log, notice, fun(_, _) -> ok end). - teardown(_) -> meck:unload(). - t_w2_ok() -> ?_test(begin Acc0 = create_init_acc(2), @@ -280,7 +305,6 @@ t_w2_ok() -> ?assertEqual(ok, resp_health(Resps)) end). - t_w3_ok() -> ?_test(begin Acc0 = create_init_acc(3), @@ -303,7 +327,6 @@ t_w3_ok() -> ?assertEqual(ok, resp_health(Resps)) end). - t_w2_mixed_accepted() -> ?_test(begin Acc0 = create_init_acc(2), @@ -331,7 +354,6 @@ t_w2_mixed_accepted() -> ?assertEqual(accepted, resp_health(Resps)) end). - t_w3_mixed_accepted() -> ?_test(begin Acc0 = create_init_acc(3), @@ -359,7 +381,6 @@ t_w3_mixed_accepted() -> ?assertEqual(accepted, resp_health(Resps)) end). - t_w2_exit1_ok() -> ?_test(begin Acc0 = create_init_acc(2), @@ -384,7 +405,6 @@ t_w2_exit1_ok() -> ?assertEqual(ok, resp_health(Resps)) end). - t_w2_exit2_accepted() -> ?_test(begin Acc0 = create_init_acc(2), @@ -409,7 +429,6 @@ t_w2_exit2_accepted() -> ?assertEqual(accepted, resp_health(Resps)) end). - t_w2_exit3_error() -> ?_test(begin Acc0 = create_init_acc(2), @@ -436,7 +455,6 @@ t_w2_exit3_error() -> ?assertEqual(error, resp_health(Resps)) end). - t_w4_accepted() -> % Make sure we return when all workers have responded % rather than wait around for a timeout if a user asks @@ -464,7 +482,6 @@ t_w4_accepted() -> ?assertEqual(accepted, resp_health(Resps)) end). - t_mixed_ok_accepted() -> ?_test(begin WorkerUUIDs = [ @@ -500,7 +517,6 @@ t_mixed_ok_accepted() -> ?assertEqual(accepted, resp_health(Resps)) end). - t_mixed_errors() -> ?_test(begin WorkerUUIDs = [ @@ -535,7 +551,6 @@ t_mixed_errors() -> ?assertEqual(error, resp_health(Resps)) end). - create_init_acc(W) -> UUID1 = <<"uuid1">>, UUID2 = <<"uuid2">>, @@ -546,9 +561,12 @@ create_init_acc(W) -> % Create our worker_uuids. We're relying on the fact that % we're using a fake Q=1 db so we don't have to worry % about any hashing here. - WorkerUUIDs = lists:map(fun(Shard) -> - {Shard#shard{ref = erlang:make_ref()}, [UUID1, UUID2]} - end, Shards), + WorkerUUIDs = lists:map( + fun(Shard) -> + {Shard#shard{ref = erlang:make_ref()}, [UUID1, UUID2]} + end, + Shards + ), #acc{ worker_uuids = WorkerUUIDs, @@ -557,15 +575,17 @@ create_init_acc(W) -> w = W }. - worker(N, #acc{worker_uuids = WorkerUUIDs}) -> {Worker, _} = lists:nth(N, WorkerUUIDs), Worker. - check_quorum(Acc, Expect) -> - dict:fold(fun(_Shard, Resps, _) -> - ?assertEqual(Expect, has_quorum(Resps, 3, Acc#acc.w)) - end, nil, Acc#acc.resps). + dict:fold( + fun(_Shard, Resps, _) -> + ?assertEqual(Expect, has_quorum(Resps, 3, Acc#acc.w)) + end, + nil, + Acc#acc.resps + ). -endif. diff --git a/src/fabric/src/fabric_doc_update.erl b/src/fabric/src/fabric_doc_update.erl index 69babc14b..62e180ae2 100644 --- a/src/fabric/src/fabric_doc_update.erl +++ b/src/fabric/src/fabric_doc_update.erl @@ -25,48 +25,53 @@ go(DbName, AllDocs0, Opts) -> AllDocs = tag_docs(AllDocs1), validate_atomic_update(DbName, AllDocs, lists:member(all_or_nothing, Opts)), Options = lists:delete(all_or_nothing, Opts), - GroupedDocs = lists:map(fun({#shard{name=Name, node=Node} = Shard, Docs}) -> - Docs1 = untag_docs(Docs), - Ref = rexi:cast(Node, {fabric_rpc, update_docs, [Name,Docs1,Options]}), - {Shard#shard{ref=Ref}, Docs} - end, group_docs_by_shard(DbName, AllDocs)), + GroupedDocs = lists:map( + fun({#shard{name = Name, node = Node} = Shard, Docs}) -> + Docs1 = untag_docs(Docs), + Ref = rexi:cast(Node, {fabric_rpc, update_docs, [Name, Docs1, Options]}), + {Shard#shard{ref = Ref}, Docs} + end, + group_docs_by_shard(DbName, AllDocs) + ), {Workers, _} = lists:unzip(GroupedDocs), RexiMon = fabric_util:create_monitors(Workers), W = couch_util:get_value(w, Options, integer_to_list(mem3:quorum(DbName))), - Acc0 = {length(Workers), length(AllDocs), list_to_integer(W), GroupedDocs, - dict:new()}, + Acc0 = {length(Workers), length(AllDocs), list_to_integer(W), GroupedDocs, dict:new()}, Timeout = fabric_util:request_timeout(), try rexi_utils:recv(Workers, #shard.ref, fun handle_message/3, Acc0, infinity, Timeout) of - {ok, {Health, Results}} - when Health =:= ok; Health =:= accepted; Health =:= error -> - {Health, [R || R <- couch_util:reorder_results(AllDocs, Results), R =/= noreply]}; - {timeout, Acc} -> - {_, _, W1, GroupedDocs1, DocReplDict} = Acc, - {DefunctWorkers, _} = lists:unzip(GroupedDocs1), - fabric_util:log_timeout(DefunctWorkers, "update_docs"), - {Health, _, Resp} = dict:fold(fun force_reply/3, {ok, W1, []}, - DocReplDict), - {Health, [R || R <- couch_util:reorder_results(AllDocs, Resp), R =/= noreply]}; - Else -> - Else + {ok, {Health, Results}} when + Health =:= ok; Health =:= accepted; Health =:= error + -> + {Health, [R || R <- couch_util:reorder_results(AllDocs, Results), R =/= noreply]}; + {timeout, Acc} -> + {_, _, W1, GroupedDocs1, DocReplDict} = Acc, + {DefunctWorkers, _} = lists:unzip(GroupedDocs1), + fabric_util:log_timeout(DefunctWorkers, "update_docs"), + {Health, _, Resp} = dict:fold( + fun force_reply/3, + {ok, W1, []}, + DocReplDict + ), + {Health, [R || R <- couch_util:reorder_results(AllDocs, Resp), R =/= noreply]}; + Else -> + Else after rexi_monitor:stop(RexiMon) end. -handle_message({rexi_DOWN, _, {_,NodeRef},_}, _Worker, Acc0) -> +handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _Worker, Acc0) -> {_, LenDocs, W, GroupedDocs, DocReplyDict} = Acc0, - NewGrpDocs = [X || {#shard{node=N}, _} = X <- GroupedDocs, N =/= NodeRef], + NewGrpDocs = [X || {#shard{node = N}, _} = X <- GroupedDocs, N =/= NodeRef], skip_message({length(NewGrpDocs), LenDocs, W, NewGrpDocs, DocReplyDict}); - handle_message({rexi_EXIT, _}, Worker, Acc0) -> - {WC,LenDocs,W,GrpDocs,DocReplyDict} = Acc0, - NewGrpDocs = lists:keydelete(Worker,1,GrpDocs), - skip_message({WC-1,LenDocs,W,NewGrpDocs,DocReplyDict}); + {WC, LenDocs, W, GrpDocs, DocReplyDict} = Acc0, + NewGrpDocs = lists:keydelete(Worker, 1, GrpDocs), + skip_message({WC - 1, LenDocs, W, NewGrpDocs, DocReplyDict}); handle_message(internal_server_error, Worker, Acc0) -> % happens when we fail to load validation functions in an RPC worker - {WC,LenDocs,W,GrpDocs,DocReplyDict} = Acc0, - NewGrpDocs = lists:keydelete(Worker,1,GrpDocs), - skip_message({WC-1,LenDocs,W,NewGrpDocs,DocReplyDict}); + {WC, LenDocs, W, GrpDocs, DocReplyDict} = Acc0, + NewGrpDocs = lists:keydelete(Worker, 1, GrpDocs), + skip_message({WC - 1, LenDocs, W, NewGrpDocs, DocReplyDict}); handle_message(attachment_chunk_received, _Worker, Acc0) -> {ok, Acc0}; handle_message({ok, Replies}, Worker, Acc0) -> @@ -74,21 +79,24 @@ handle_message({ok, Replies}, Worker, Acc0) -> {value, {_, Docs}, NewGrpDocs} = lists:keytake(Worker, 1, GroupedDocs), DocReplyDict = append_update_replies(Docs, Replies, DocReplyDict0), case {WaitingCount, dict:size(DocReplyDict)} of - {1, _} -> - % last message has arrived, we need to conclude things - {Health, W, Reply} = dict:fold(fun force_reply/3, {ok, W, []}, - DocReplyDict), - {stop, {Health, Reply}}; - {_, DocCount} -> - % we've got at least one reply for each document, let's take a look - case dict:fold(fun maybe_reply/3, {stop,W,[]}, DocReplyDict) of - continue -> - {ok, {WaitingCount - 1, DocCount, W, NewGrpDocs, DocReplyDict}}; - {stop, W, FinalReplies} -> - {stop, {ok, FinalReplies}} - end; - _ -> - {ok, {WaitingCount - 1, DocCount, W, NewGrpDocs, DocReplyDict}} + {1, _} -> + % last message has arrived, we need to conclude things + {Health, W, Reply} = dict:fold( + fun force_reply/3, + {ok, W, []}, + DocReplyDict + ), + {stop, {Health, Reply}}; + {_, DocCount} -> + % we've got at least one reply for each document, let's take a look + case dict:fold(fun maybe_reply/3, {stop, W, []}, DocReplyDict) of + continue -> + {ok, {WaitingCount - 1, DocCount, W, NewGrpDocs, DocReplyDict}}; + {stop, W, FinalReplies} -> + {stop, {ok, FinalReplies}} + end; + _ -> + {ok, {WaitingCount - 1, DocCount, W, NewGrpDocs, DocReplyDict}} end; handle_message({missing_stub, Stub}, _, _) -> throw({missing_stub, Stub}); @@ -106,53 +114,61 @@ before_doc_update(DbName, Docs, Opts) -> {true, _} -> %% cluster db is expensive to create so we only do it if we have to Db = fabric_util:open_cluster_db(DbName, Opts), - [couch_replicator_docs:before_doc_update(Doc, Db, replicated_changes) - || Doc <- Docs]; + [ + couch_replicator_docs:before_doc_update(Doc, Db, replicated_changes) + || Doc <- Docs + ]; {_, true} -> %% cluster db is expensive to create so we only do it if we have to Db = fabric_util:open_cluster_db(DbName, Opts), - [couch_users_db:before_doc_update(Doc, Db, interactive_edit) - || Doc <- Docs]; + [ + couch_users_db:before_doc_update(Doc, Db, interactive_edit) + || Doc <- Docs + ]; _ -> Docs end. tag_docs([]) -> []; -tag_docs([#doc{meta=Meta}=Doc | Rest]) -> - [Doc#doc{meta=[{ref, make_ref()} | Meta]} | tag_docs(Rest)]. +tag_docs([#doc{meta = Meta} = Doc | Rest]) -> + [Doc#doc{meta = [{ref, make_ref()} | Meta]} | tag_docs(Rest)]. untag_docs([]) -> []; -untag_docs([#doc{meta=Meta}=Doc | Rest]) -> - [Doc#doc{meta=lists:keydelete(ref, 1, Meta)} | untag_docs(Rest)]. +untag_docs([#doc{meta = Meta} = Doc | Rest]) -> + [Doc#doc{meta = lists:keydelete(ref, 1, Meta)} | untag_docs(Rest)]. force_reply(Doc, [], {_, W, Acc}) -> {error, W, [{Doc, {error, internal_server_error}} | Acc]}; -force_reply(Doc, [FirstReply|_] = Replies, {Health, W, Acc}) -> +force_reply(Doc, [FirstReply | _] = Replies, {Health, W, Acc}) -> case update_quorum_met(W, Replies) of - {true, Reply} -> - {Health, W, [{Doc,Reply} | Acc]}; - false -> - case [Reply || {ok, Reply} <- Replies] of - [] -> - % check if all errors are identical, if so inherit health - case lists:all(fun(E) -> E =:= FirstReply end, Replies) of - true -> - CounterKey = [fabric, doc_update, errors], - couch_stats:increment_counter(CounterKey), - {Health, W, [{Doc, FirstReply} | Acc]}; - false -> - CounterKey = [fabric, doc_update, mismatched_errors], - couch_stats:increment_counter(CounterKey), - {error, W, [{Doc, FirstReply} | Acc]} - end; - [AcceptedRev | _] -> - CounterKey = [fabric, doc_update, write_quorum_errors], - couch_stats:increment_counter(CounterKey), - NewHealth = case Health of ok -> accepted; _ -> Health end, - {NewHealth, W, [{Doc, {accepted,AcceptedRev}} | Acc]} - end + {true, Reply} -> + {Health, W, [{Doc, Reply} | Acc]}; + false -> + case [Reply || {ok, Reply} <- Replies] of + [] -> + % check if all errors are identical, if so inherit health + case lists:all(fun(E) -> E =:= FirstReply end, Replies) of + true -> + CounterKey = [fabric, doc_update, errors], + couch_stats:increment_counter(CounterKey), + {Health, W, [{Doc, FirstReply} | Acc]}; + false -> + CounterKey = [fabric, doc_update, mismatched_errors], + couch_stats:increment_counter(CounterKey), + {error, W, [{Doc, FirstReply} | Acc]} + end; + [AcceptedRev | _] -> + CounterKey = [fabric, doc_update, write_quorum_errors], + couch_stats:increment_counter(CounterKey), + NewHealth = + case Health of + ok -> accepted; + _ -> Health + end, + {NewHealth, W, [{Doc, {accepted, AcceptedRev}} | Acc]} + end end. maybe_reply(_, _, continue) -> @@ -160,21 +176,24 @@ maybe_reply(_, _, continue) -> continue; maybe_reply(Doc, Replies, {stop, W, Acc}) -> case update_quorum_met(W, Replies) of - {true, Reply} -> - {stop, W, [{Doc, Reply} | Acc]}; - false -> - continue + {true, Reply} -> + {stop, W, [{Doc, Reply} | Acc]}; + false -> + continue end. update_quorum_met(W, Replies) -> - Counters = lists:foldl(fun(R,D) -> orddict:update_counter(R,1,D) end, - orddict:new(), Replies), + Counters = lists:foldl( + fun(R, D) -> orddict:update_counter(R, 1, D) end, + orddict:new(), + Replies + ), GoodReplies = lists:filter(fun good_reply/1, Counters), case lists:dropwhile(fun({_, Count}) -> Count < W end, GoodReplies) of - [] -> - false; - [{FinalReply, _} | _] -> - {true, FinalReply} + [] -> + false; + [{FinalReply, _} | _] -> + {true, FinalReply} end. good_reply({{ok, _}, _}) -> @@ -186,18 +205,28 @@ good_reply(_) -> -spec group_docs_by_shard(binary(), [#doc{}]) -> [{#shard{}, [#doc{}]}]. group_docs_by_shard(DbName, Docs) -> - dict:to_list(lists:foldl(fun(#doc{id=Id} = Doc, D0) -> - lists:foldl(fun(Shard, D1) -> - dict:append(Shard, Doc, D1) - end, D0, mem3:shards(DbName,Id)) - end, dict:new(), Docs)). + dict:to_list( + lists:foldl( + fun(#doc{id = Id} = Doc, D0) -> + lists:foldl( + fun(Shard, D1) -> + dict:append(Shard, Doc, D1) + end, + D0, + mem3:shards(DbName, Id) + ) + end, + dict:new(), + Docs + ) + ). append_update_replies([], [], DocReplyDict) -> DocReplyDict; -append_update_replies([Doc|Rest], [], Dict0) -> +append_update_replies([Doc | Rest], [], Dict0) -> % icky, if replicated_changes only errors show up in result append_update_replies(Rest, [], dict:append(Doc, noreply, Dict0)); -append_update_replies([Doc|Rest1], [Reply|Rest2], Dict0) -> +append_update_replies([Doc | Rest1], [Reply | Rest2], Dict0) -> append_update_replies(Rest1, Rest2, dict:append(Doc, Reply, Dict0)). skip_message({0, _, W, _, DocReplyDict}) -> @@ -213,27 +242,29 @@ validate_atomic_update(_DbName, AllDocs, true) -> % to basically extract the prep_and_validate_updates function from couch_db % and only run that, without actually writing in case of a success. Error = {not_implemented, <<"all_or_nothing is not supported">>}, - PreCommitFailures = lists:map(fun(#doc{id=Id, revs = {Pos,Revs}}) -> - case Revs of [] -> RevId = <<>>; [RevId|_] -> ok end, - {{Id, {Pos, RevId}}, Error} - end, AllDocs), + PreCommitFailures = lists:map( + fun(#doc{id = Id, revs = {Pos, Revs}}) -> + case Revs of + [] -> RevId = <<>>; + [RevId | _] -> ok + end, + {{Id, {Pos, RevId}}, Error} + end, + AllDocs + ), throw({aborted, PreCommitFailures}). - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). - setup_all() -> meck:new([couch_log, couch_stats]), - meck:expect(couch_log, warning, fun(_,_) -> ok end), + meck:expect(couch_log, warning, fun(_, _) -> ok end), meck:expect(couch_stats, increment_counter, fun(_) -> ok end). - teardown_all(_) -> meck:unload(). - doc_update_test_() -> { setup, @@ -246,132 +277,151 @@ doc_update_test_() -> ] }. - % eunits doc_update1() -> - Doc1 = #doc{revs = {1,[<<"foo">>]}}, - Doc2 = #doc{revs = {1,[<<"bar">>]}}, + Doc1 = #doc{revs = {1, [<<"foo">>]}}, + Doc2 = #doc{revs = {1, [<<"bar">>]}}, Docs = [Doc1], Docs2 = [Doc2, Doc1], - Dict = dict:from_list([{Doc,[]} || Doc <- Docs]), - Dict2 = dict:from_list([{Doc,[]} || Doc <- Docs2]), + Dict = dict:from_list([{Doc, []} || Doc <- Docs]), + Dict2 = dict:from_list([{Doc, []} || Doc <- Docs2]), Shards = - mem3_util:create_partition_map("foo",3,1,["node1","node2","node3"]), - GroupedDocs = group_docs_by_shard_hack(<<"foo">>,Shards,Docs), - + mem3_util:create_partition_map("foo", 3, 1, ["node1", "node2", "node3"]), + GroupedDocs = group_docs_by_shard_hack(<<"foo">>, Shards, Docs), % test for W = 2 - AccW2 = {length(Shards), length(Docs), list_to_integer("2"), GroupedDocs, - Dict}, + AccW2 = {length(Shards), length(Docs), list_to_integer("2"), GroupedDocs, Dict}, - {ok,{WaitingCountW2_1,_,_,_,_}=AccW2_1} = - handle_message({ok, [{ok, Doc1}]},hd(Shards),AccW2), - ?assertEqual(WaitingCountW2_1,2), - {stop, FinalReplyW2 } = - handle_message({ok, [{ok, Doc1}]},lists:nth(2,Shards),AccW2_1), - ?assertEqual({ok, [{Doc1, {ok,Doc1}}]},FinalReplyW2), + {ok, {WaitingCountW2_1, _, _, _, _} = AccW2_1} = + handle_message({ok, [{ok, Doc1}]}, hd(Shards), AccW2), + ?assertEqual(WaitingCountW2_1, 2), + {stop, FinalReplyW2} = + handle_message({ok, [{ok, Doc1}]}, lists:nth(2, Shards), AccW2_1), + ?assertEqual({ok, [{Doc1, {ok, Doc1}}]}, FinalReplyW2), % test for W = 3 - AccW3 = {length(Shards), length(Docs), list_to_integer("3"), GroupedDocs, - Dict}, + AccW3 = {length(Shards), length(Docs), list_to_integer("3"), GroupedDocs, Dict}, - {ok,{WaitingCountW3_1,_,_,_,_}=AccW3_1} = - handle_message({ok, [{ok, Doc1}]},hd(Shards),AccW3), - ?assertEqual(WaitingCountW3_1,2), + {ok, {WaitingCountW3_1, _, _, _, _} = AccW3_1} = + handle_message({ok, [{ok, Doc1}]}, hd(Shards), AccW3), + ?assertEqual(WaitingCountW3_1, 2), - {ok,{WaitingCountW3_2,_,_,_,_}=AccW3_2} = - handle_message({ok, [{ok, Doc1}]},lists:nth(2,Shards),AccW3_1), - ?assertEqual(WaitingCountW3_2,1), + {ok, {WaitingCountW3_2, _, _, _, _} = AccW3_2} = + handle_message({ok, [{ok, Doc1}]}, lists:nth(2, Shards), AccW3_1), + ?assertEqual(WaitingCountW3_2, 1), - {stop, FinalReplyW3 } = - handle_message({ok, [{ok, Doc1}]},lists:nth(3,Shards),AccW3_2), - ?assertEqual({ok, [{Doc1, {ok,Doc1}}]},FinalReplyW3), + {stop, FinalReplyW3} = + handle_message({ok, [{ok, Doc1}]}, lists:nth(3, Shards), AccW3_2), + ?assertEqual({ok, [{Doc1, {ok, Doc1}}]}, FinalReplyW3), % test w quorum > # shards, which should fail immediately - Shards2 = mem3_util:create_partition_map("foo",1,1,["node1"]), - GroupedDocs2 = group_docs_by_shard_hack(<<"foo">>,Shards2,Docs), + Shards2 = mem3_util:create_partition_map("foo", 1, 1, ["node1"]), + GroupedDocs2 = group_docs_by_shard_hack(<<"foo">>, Shards2, Docs), AccW4 = {length(Shards2), length(Docs), list_to_integer("2"), GroupedDocs2, Dict}, Bool = - case handle_message({ok, [{ok, Doc1}]},hd(Shards2),AccW4) of - {stop, _Reply} -> - true; - _ -> false - end, - ?assertEqual(Bool,true), + case handle_message({ok, [{ok, Doc1}]}, hd(Shards2), AccW4) of + {stop, _Reply} -> + true; + _ -> + false + end, + ?assertEqual(Bool, true), % Docs with no replies should end up as {error, internal_server_error} - SA1 = #shard{node=a, range=1}, - SB1 = #shard{node=b, range=1}, - SA2 = #shard{node=a, range=2}, - SB2 = #shard{node=b, range=2}, - GroupedDocs3 = [{SA1,[Doc1]}, {SB1,[Doc1]}, {SA2,[Doc2]}, {SB2,[Doc2]}], + SA1 = #shard{node = a, range = 1}, + SB1 = #shard{node = b, range = 1}, + SA2 = #shard{node = a, range = 2}, + SB2 = #shard{node = b, range = 2}, + GroupedDocs3 = [{SA1, [Doc1]}, {SB1, [Doc1]}, {SA2, [Doc2]}, {SB2, [Doc2]}], StW5_0 = {length(GroupedDocs3), length(Docs2), 2, GroupedDocs3, Dict2}, {ok, StW5_1} = handle_message({ok, [{ok, "A"}]}, SA1, StW5_0), {ok, StW5_2} = handle_message({rexi_EXIT, nil}, SB1, StW5_1), {ok, StW5_3} = handle_message({rexi_EXIT, nil}, SA2, StW5_2), {stop, ReplyW5} = handle_message({rexi_EXIT, nil}, SB2, StW5_3), ?assertEqual( - {error, [{Doc1,{accepted,"A"}},{Doc2,{error,internal_server_error}}]}, + {error, [{Doc1, {accepted, "A"}}, {Doc2, {error, internal_server_error}}]}, ReplyW5 ). doc_update2() -> - Doc1 = #doc{revs = {1,[<<"foo">>]}}, - Doc2 = #doc{revs = {1,[<<"bar">>]}}, + Doc1 = #doc{revs = {1, [<<"foo">>]}}, + Doc2 = #doc{revs = {1, [<<"bar">>]}}, Docs = [Doc2, Doc1], Shards = - mem3_util:create_partition_map("foo",3,1,["node1","node2","node3"]), - GroupedDocs = group_docs_by_shard_hack(<<"foo">>,Shards,Docs), - Acc0 = {length(Shards), length(Docs), list_to_integer("2"), GroupedDocs, - dict:from_list([{Doc,[]} || Doc <- Docs])}, - - {ok,{WaitingCount1,_,_,_,_}=Acc1} = - handle_message({ok, [{ok, Doc1},{ok, Doc2}]},hd(Shards),Acc0), - ?assertEqual(WaitingCount1,2), - - {ok,{WaitingCount2,_,_,_,_}=Acc2} = - handle_message({rexi_EXIT, 1},lists:nth(2,Shards),Acc1), - ?assertEqual(WaitingCount2,1), + mem3_util:create_partition_map("foo", 3, 1, ["node1", "node2", "node3"]), + GroupedDocs = group_docs_by_shard_hack(<<"foo">>, Shards, Docs), + Acc0 = { + length(Shards), + length(Docs), + list_to_integer("2"), + GroupedDocs, + dict:from_list([{Doc, []} || Doc <- Docs]) + }, + + {ok, {WaitingCount1, _, _, _, _} = Acc1} = + handle_message({ok, [{ok, Doc1}, {ok, Doc2}]}, hd(Shards), Acc0), + ?assertEqual(WaitingCount1, 2), + + {ok, {WaitingCount2, _, _, _, _} = Acc2} = + handle_message({rexi_EXIT, 1}, lists:nth(2, Shards), Acc1), + ?assertEqual(WaitingCount2, 1), {stop, Reply} = - handle_message({rexi_EXIT, 1},lists:nth(3,Shards),Acc2), + handle_message({rexi_EXIT, 1}, lists:nth(3, Shards), Acc2), - ?assertEqual({accepted, [{Doc1,{accepted,Doc2}}, {Doc2,{accepted,Doc1}}]}, - Reply). + ?assertEqual( + {accepted, [{Doc1, {accepted, Doc2}}, {Doc2, {accepted, Doc1}}]}, + Reply + ). doc_update3() -> - Doc1 = #doc{revs = {1,[<<"foo">>]}}, - Doc2 = #doc{revs = {1,[<<"bar">>]}}, + Doc1 = #doc{revs = {1, [<<"foo">>]}}, + Doc2 = #doc{revs = {1, [<<"bar">>]}}, Docs = [Doc2, Doc1], Shards = - mem3_util:create_partition_map("foo",3,1,["node1","node2","node3"]), - GroupedDocs = group_docs_by_shard_hack(<<"foo">>,Shards,Docs), - Acc0 = {length(Shards), length(Docs), list_to_integer("2"), GroupedDocs, - dict:from_list([{Doc,[]} || Doc <- Docs])}, - - {ok,{WaitingCount1,_,_,_,_}=Acc1} = - handle_message({ok, [{ok, Doc1},{ok, Doc2}]},hd(Shards),Acc0), - ?assertEqual(WaitingCount1,2), - - {ok,{WaitingCount2,_,_,_,_}=Acc2} = - handle_message({rexi_EXIT, 1},lists:nth(2,Shards),Acc1), - ?assertEqual(WaitingCount2,1), + mem3_util:create_partition_map("foo", 3, 1, ["node1", "node2", "node3"]), + GroupedDocs = group_docs_by_shard_hack(<<"foo">>, Shards, Docs), + Acc0 = { + length(Shards), + length(Docs), + list_to_integer("2"), + GroupedDocs, + dict:from_list([{Doc, []} || Doc <- Docs]) + }, + + {ok, {WaitingCount1, _, _, _, _} = Acc1} = + handle_message({ok, [{ok, Doc1}, {ok, Doc2}]}, hd(Shards), Acc0), + ?assertEqual(WaitingCount1, 2), + + {ok, {WaitingCount2, _, _, _, _} = Acc2} = + handle_message({rexi_EXIT, 1}, lists:nth(2, Shards), Acc1), + ?assertEqual(WaitingCount2, 1), {stop, Reply} = - handle_message({ok, [{ok, Doc1},{ok, Doc2}]},lists:nth(3,Shards),Acc2), + handle_message({ok, [{ok, Doc1}, {ok, Doc2}]}, lists:nth(3, Shards), Acc2), - ?assertEqual({ok, [{Doc1, {ok, Doc2}},{Doc2, {ok,Doc1}}]},Reply). + ?assertEqual({ok, [{Doc1, {ok, Doc2}}, {Doc2, {ok, Doc1}}]}, Reply). % needed for testing to avoid having to start the mem3 application group_docs_by_shard_hack(_DbName, Shards, Docs) -> - dict:to_list(lists:foldl(fun(#doc{id=_Id} = Doc, D0) -> - lists:foldl(fun(Shard, D1) -> - dict:append(Shard, Doc, D1) - end, D0, Shards) - end, dict:new(), Docs)). + dict:to_list( + lists:foldl( + fun(#doc{id = _Id} = Doc, D0) -> + lists:foldl( + fun(Shard, D1) -> + dict:append(Shard, Doc, D1) + end, + D0, + Shards + ) + end, + dict:new(), + Docs + ) + ). -endif. diff --git a/src/fabric/src/fabric_group_info.erl b/src/fabric/src/fabric_group_info.erl index 91bdf01b0..c7d7293fd 100644 --- a/src/fabric/src/fabric_group_info.erl +++ b/src/fabric/src/fabric_group_info.erl @@ -21,8 +21,7 @@ go(DbName, GroupId) when is_binary(GroupId) -> {ok, DDoc} = fabric:open_doc(DbName, GroupId, [?ADMIN_CTX]), go(DbName, DDoc); - -go(DbName, #doc{id=DDocId}) -> +go(DbName, #doc{id = DDocId}) -> Shards = mem3:shards(DbName), Ushards = mem3:ushards(DbName), Workers = fabric_util:submit_jobs(Shards, group_info, [DDocId]), @@ -30,63 +29,66 @@ go(DbName, #doc{id=DDocId}) -> USet = sets:from_list([{Id, N} || #shard{name = Id, node = N} <- Ushards]), Acc = {fabric_dict:init(Workers, nil), [], USet}, try fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc) of - {timeout, {WorkersDict, _, _}} -> - DefunctWorkers = fabric_util:remove_done_workers(WorkersDict, nil), - fabric_util:log_timeout(DefunctWorkers, "group_info"), - {error, timeout}; - Else -> - Else + {timeout, {WorkersDict, _, _}} -> + DefunctWorkers = fabric_util:remove_done_workers(WorkersDict, nil), + fabric_util:log_timeout(DefunctWorkers, "group_info"), + {error, timeout}; + Else -> + Else after rexi_monitor:stop(RexiMon) end. -handle_message({rexi_DOWN, _, {_,NodeRef},_}, _, {Counters, Resps, USet}) -> +handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _, {Counters, Resps, USet}) -> case fabric_ring:node_down(NodeRef, Counters, Resps) of {ok, Counters1} -> {ok, {Counters1, Resps, USet}}; error -> {error, {nodedown, <<"progress not possible">>}} end; - handle_message({rexi_EXIT, Reason}, Shard, {Counters, Resps, USet}) -> case fabric_ring:handle_error(Shard, Counters, Resps) of {ok, Counters1} -> {ok, {Counters1, Resps, USet}}; error -> {error, Reason} end; - handle_message({ok, Info}, Shard, {Counters, Resps, USet}) -> - case fabric_ring:handle_response(Shard, Info, Counters, Resps) of + case fabric_ring:handle_response(Shard, Info, Counters, Resps) of {ok, {Counters1, Resps1}} -> {ok, {Counters1, Resps1, USet}}; {stop, Resps1} -> {stop, build_final_response(USet, Resps1)} end; - handle_message(Reason, Shard, {Counters, Resps, USet}) -> case fabric_ring:handle_error(Shard, Counters, Resps) of {ok, Counters1} -> {ok, {Counters1, Resps, USet}}; error -> {error, Reason} end. - build_final_response(USet, Responses) -> - AccF = fabric_dict:fold(fun(#shard{name = Id, node = Node}, Info, Acc) -> - IsPreferred = sets:is_element({Id, Node}, USet), - dict:append(Id, {Node, IsPreferred, Info}, Acc) - end, dict:new(), Responses), + AccF = fabric_dict:fold( + fun(#shard{name = Id, node = Node}, Info, Acc) -> + IsPreferred = sets:is_element({Id, Node}, USet), + dict:append(Id, {Node, IsPreferred, Info}, Acc) + end, + dict:new(), + Responses + ), Pending = aggregate_pending(AccF), Infos = get_infos(AccF), [{updates_pending, {Pending}} | merge_results(Infos)]. - get_infos(Acc) -> Values = [V || {_, V} <- dict:to_list(Acc)], lists:flatten([Info || {_Node, _Pref, Info} <- lists:flatten(Values)]). aggregate_pending(Dict) -> {Preferred, Total, Minimum} = - dict:fold(fun(_Name, Results, {P, T, M}) -> - {Preferred, Total, Minimum} = calculate_pending(Results), - {P + Preferred, T + Total, M + Minimum} - end, {0, 0, 0}, Dict), + dict:fold( + fun(_Name, Results, {P, T, M}) -> + {Preferred, Total, Minimum} = calculate_pending(Results), + {P + Preferred, T + Total, M + Minimum} + end, + {0, 0, 0}, + Dict + ), [ {minimum, Minimum}, {preferred, Preferred}, @@ -94,46 +96,64 @@ aggregate_pending(Dict) -> ]. calculate_pending(Results) -> - lists:foldl(fun - ({_Node, true, Info}, {P, T, V}) -> - Pending = couch_util:get_value(pending_updates, Info), - {P + Pending, T + Pending, min(Pending, V)}; - ({_Node, false, Info}, {P, T, V}) -> - Pending = couch_util:get_value(pending_updates, Info), - {P, T + Pending, min(Pending, V)} - end, {0, 0, infinity}, Results). + lists:foldl( + fun + ({_Node, true, Info}, {P, T, V}) -> + Pending = couch_util:get_value(pending_updates, Info), + {P + Pending, T + Pending, min(Pending, V)}; + ({_Node, false, Info}, {P, T, V}) -> + Pending = couch_util:get_value(pending_updates, Info), + {P, T + Pending, min(Pending, V)} + end, + {0, 0, infinity}, + Results + ). merge_results(Info) -> - Dict = lists:foldl(fun({K,V},D0) -> orddict:append(K,V,D0) end, - orddict:new(), Info), - orddict:fold(fun - (signature, [X | _], Acc) -> - [{signature, X} | Acc]; - (language, [X | _], Acc) -> - [{language, X} | Acc]; - (sizes, X, Acc) -> - [{sizes, {merge_object(X)}} | Acc]; - (compact_running, X, Acc) -> - [{compact_running, lists:member(true, X)} | Acc]; - (updater_running, X, Acc) -> - [{updater_running, lists:member(true, X)} | Acc]; - (waiting_commit, X, Acc) -> - [{waiting_commit, lists:member(true, X)} | Acc]; - (waiting_clients, X, Acc) -> - [{waiting_clients, lists:sum(X)} | Acc]; - (update_seq, X, Acc) -> - [{update_seq, lists:sum(X)} | Acc]; - (purge_seq, X, Acc) -> - [{purge_seq, lists:sum(X)} | Acc]; - (_, _, Acc) -> - Acc - end, [], Dict). + Dict = lists:foldl( + fun({K, V}, D0) -> orddict:append(K, V, D0) end, + orddict:new(), + Info + ), + orddict:fold( + fun + (signature, [X | _], Acc) -> + [{signature, X} | Acc]; + (language, [X | _], Acc) -> + [{language, X} | Acc]; + (sizes, X, Acc) -> + [{sizes, {merge_object(X)}} | Acc]; + (compact_running, X, Acc) -> + [{compact_running, lists:member(true, X)} | Acc]; + (updater_running, X, Acc) -> + [{updater_running, lists:member(true, X)} | Acc]; + (waiting_commit, X, Acc) -> + [{waiting_commit, lists:member(true, X)} | Acc]; + (waiting_clients, X, Acc) -> + [{waiting_clients, lists:sum(X)} | Acc]; + (update_seq, X, Acc) -> + [{update_seq, lists:sum(X)} | Acc]; + (purge_seq, X, Acc) -> + [{purge_seq, lists:sum(X)} | Acc]; + (_, _, Acc) -> + Acc + end, + [], + Dict + ). merge_object(Objects) -> - Dict = lists:foldl(fun({Props}, D) -> - lists:foldl(fun({K,V},D0) -> orddict:append(K,V,D0) end, D, Props) - end, orddict:new(), Objects), - orddict:fold(fun - (Key, X, Acc) -> + Dict = lists:foldl( + fun({Props}, D) -> + lists:foldl(fun({K, V}, D0) -> orddict:append(K, V, D0) end, D, Props) + end, + orddict:new(), + Objects + ), + orddict:fold( + fun(Key, X, Acc) -> [{Key, lists:sum(X)} | Acc] - end, [], Dict). + end, + [], + Dict + ). diff --git a/src/fabric/src/fabric_ring.erl b/src/fabric/src/fabric_ring.erl index bad0f42d1..2bb7d717f 100644 --- a/src/fabric/src/fabric_ring.erl +++ b/src/fabric/src/fabric_ring.erl @@ -12,7 +12,6 @@ -module(fabric_ring). - -export([ is_progress_possible/1, is_progress_possible/2, @@ -25,21 +24,17 @@ handle_response/5 ]). - -include_lib("fabric/include/fabric.hrl"). -include_lib("mem3/include/mem3.hrl"). - -type fabric_dict() :: [{#shard{}, any()}]. -type ring_opts() :: [atom() | tuple()]. - %% @doc looks for a fully covered keyrange in the list of counters -spec is_progress_possible(fabric_dict()) -> boolean(). is_progress_possible(Counters) -> is_progress_possible(Counters, []). - %% @doc looks for a fully covered keyrange in the list of counters %% This version take ring option to configure how progress will %% be checked. By default, [], checks that the full ring is covered. @@ -47,41 +42,39 @@ is_progress_possible(Counters) -> is_progress_possible(Counters, RingOpts) -> is_progress_possible(Counters, [], 0, ?RING_END, RingOpts). - -spec get_shard_replacements(binary(), [#shard{}]) -> [#shard{}]. get_shard_replacements(DbName, UsedShards0) -> % We only want to generate a replacements list from shards % that aren't already used. AllLiveShards = mem3:live_shards(DbName, [node() | nodes()]), - UsedShards = [S#shard{ref=undefined} || S <- UsedShards0], + UsedShards = [S#shard{ref = undefined} || S <- UsedShards0], get_shard_replacements_int(AllLiveShards -- UsedShards, UsedShards). - -spec node_down(node(), fabric_dict(), fabric_dict()) -> {ok, fabric_dict()} | error. node_down(Node, Workers, Responses) -> node_down(Node, Workers, Responses, []). - -spec node_down(node(), fabric_dict(), fabric_dict(), ring_opts()) -> {ok, fabric_dict()} | error. node_down(Node, Workers, Responses, RingOpts) -> {B, E} = range_bounds(Workers, Responses), - Workers1 = fabric_dict:filter(fun(#shard{node = N}, _) -> - N =/= Node - end, Workers), + Workers1 = fabric_dict:filter( + fun(#shard{node = N}, _) -> + N =/= Node + end, + Workers + ), case is_progress_possible(Workers1, Responses, B, E, RingOpts) of true -> {ok, Workers1}; false -> error end. - -spec handle_error(#shard{}, fabric_dict(), fabric_dict()) -> {ok, fabric_dict()} | error. handle_error(Shard, Workers, Responses) -> handle_error(Shard, Workers, Responses, []). - -spec handle_error(#shard{}, fabric_dict(), fabric_dict(), ring_opts()) -> {ok, fabric_dict()} | error. handle_error(Shard, Workers, Responses, RingOpts) -> @@ -92,20 +85,28 @@ handle_error(Shard, Workers, Responses, RingOpts) -> false -> error end. - -spec handle_response(#shard{}, any(), fabric_dict(), fabric_dict()) -> {ok, {fabric_dict(), fabric_dict()}} | {stop, fabric_dict()}. handle_response(Shard, Response, Workers, Responses) -> handle_response(Shard, Response, Workers, Responses, []). - --spec handle_response(#shard{}, any(), fabric_dict(), fabric_dict(), - ring_opts()) -> +-spec handle_response( + #shard{}, + any(), + fabric_dict(), + fabric_dict(), + ring_opts() +) -> {ok, {fabric_dict(), fabric_dict()}} | {stop, fabric_dict()}. handle_response(Shard, Response, Workers, Responses, RingOpts) -> - handle_response(Shard, Response, Workers, Responses, RingOpts, - fun stop_workers/1). - + handle_response( + Shard, + Response, + Workers, + Responses, + RingOpts, + fun stop_workers/1 + ). % Worker response handler. Gets reponses from shard and puts them in the list % until they complete a full ring. Then kill unused responses and remaining @@ -139,7 +140,6 @@ handle_response(Shard, Response, Workers, Responses, RingOpts, CleanupCb) -> handle_response_all(Workers1, Responses1) end. - handle_response_ring(Workers, Responses, CleanupCb) -> {MinB, MaxE} = range_bounds(Workers, Responses), Ranges = lists:map(fun({R, _, _}) -> R end, Responses), @@ -159,7 +159,6 @@ handle_response_ring(Workers, Responses, CleanupCb) -> {stop, fabric_dict:from_list(UsedResponses)} end. - handle_response_any(Shard, Response, Workers, Any, CleanupCb) -> case lists:member(Shard#shard{ref = undefined}, Any) of true -> @@ -169,7 +168,6 @@ handle_response_any(Shard, Response, Workers, Any, CleanupCb) -> {ok, {Workers, []}} end. - handle_response_all(Workers, Responses) -> case fabric_dict:size(Workers) =:= 0 of true -> @@ -178,7 +176,6 @@ handle_response_all(Workers, Responses) -> {ok, {Workers, Responses}} end. - % Check if workers still waiting and the already received responses could % still form a continous range. The range won't always be the full ring, and % the bounds are computed based on the minimum and maximum interval beginning @@ -191,19 +188,21 @@ handle_response_all(Workers, Responses) -> % might look like: 00-ff, 00-ff, 07-ff. Even if both 00-ff workers exit, % progress can still be made with the remaining 07-ff copy. % --spec is_progress_possible(fabric_dict(), [{any(), #shard{}, any()}], - non_neg_integer(), non_neg_integer(), ring_opts()) -> boolean(). +-spec is_progress_possible( + fabric_dict(), + [{any(), #shard{}, any()}], + non_neg_integer(), + non_neg_integer(), + ring_opts() +) -> boolean(). is_progress_possible([], [], _, _, _) -> false; - is_progress_possible(Counters, Responses, MinB, MaxE, []) -> ResponseRanges = lists:map(fun({{B, E}, _, _}) -> {B, E} end, Responses), Ranges = fabric_util:worker_ranges(Counters) ++ ResponseRanges, mem3_util:get_ring(Ranges, MinB, MaxE) =/= []; - is_progress_possible(Counters, _Responses, _, _, [all]) -> fabric_dict:size(Counters) > 0; - is_progress_possible(Counters, Responses, _, _, [{any, AnyShards}]) -> InAny = fun(S) -> lists:member(S#shard{ref = undefined}, AnyShards) end, case fabric_dict:filter(fun(S, _) -> InAny(S) end, Counters) of @@ -216,30 +215,37 @@ is_progress_possible(Counters, Responses, _, _, [{any, AnyShards}]) -> true end. - get_shard_replacements_int(UnusedShards, UsedShards) -> % If we have more than one copy of a range then we don't % want to try and add a replacement to any copy. - RangeCounts = lists:foldl(fun(#shard{range=R}, Acc) -> - dict:update_counter(R, 1, Acc) - end, dict:new(), UsedShards), + RangeCounts = lists:foldl( + fun(#shard{range = R}, Acc) -> + dict:update_counter(R, 1, Acc) + end, + dict:new(), + UsedShards + ), % For each seq shard range with a count of 1, find any % possible replacements from the unused shards. The % replacement list is keyed by range. - lists:foldl(fun(#shard{range = [B, E] = Range}, Acc) -> - case dict:find(Range, RangeCounts) of - {ok, 1} -> - Repls = mem3_util:non_overlapping_shards(UnusedShards, B, E), - % Only keep non-empty lists of replacements - if Repls == [] -> Acc; true -> - [{Range, Repls} | Acc] - end; - _ -> - Acc - end - end, [], UsedShards). - + lists:foldl( + fun(#shard{range = [B, E] = Range}, Acc) -> + case dict:find(Range, RangeCounts) of + {ok, 1} -> + Repls = mem3_util:non_overlapping_shards(UnusedShards, B, E), + % Only keep non-empty lists of replacements + if + Repls == [] -> Acc; + true -> [{Range, Repls} | Acc] + end; + _ -> + Acc + end + end, + [], + UsedShards + ). range_bounds(Workers, Responses) -> RespRanges = lists:map(fun({R, _, _}) -> R end, Responses), @@ -247,31 +253,24 @@ range_bounds(Workers, Responses) -> {Bs, Es} = lists:unzip(Ranges), {lists:min(Bs), lists:max(Es)}. - get_responses([], _) -> []; - get_responses([Range | Ranges], [{Range, Shard, Value} | Resps]) -> [{Shard, Value} | get_responses(Ranges, Resps)]; - get_responses(Ranges, [_DupeRangeResp | Resps]) -> get_responses(Ranges, Resps). - stop_unused_workers(_, _, _, undefined) -> ok; - stop_unused_workers(Workers, AllResponses, UsedResponses, CleanupCb) -> WorkerShards = [S || {S, _} <- Workers], - Used = [S || {S, _} <- UsedResponses], + Used = [S || {S, _} <- UsedResponses], Unused = [S || {_, S, _} <- AllResponses, not lists:member(S, Used)], CleanupCb(WorkerShards ++ Unused). - stop_workers(Shards) when is_list(Shards) -> rexi:kill_all([{Node, Ref} || #shard{node = Node, ref = Ref} <- Shards]). - % Unit tests is_progress_possible_full_range_test() -> @@ -297,7 +296,6 @@ is_progress_possible_full_range_test() -> T7 = [[0, 10], [13, 20], [21, ?RING_END], [9, 12]], ?assertEqual(false, is_progress_possible(mk_cnts(T7))). - is_progress_possible_with_responses_test() -> C1 = mk_cnts([[0, ?RING_END]]), ?assertEqual(true, is_progress_possible(C1, [], 0, ?RING_END, [])), @@ -322,7 +320,6 @@ is_progress_possible_with_responses_test() -> ?assertEqual(false, is_progress_possible([], RS1, 5, 8, [])), ?assertEqual(true, is_progress_possible([], RS1, 7, 8, [])). - is_progress_possible_with_ring_opts_any_test() -> Opts = [{any, [mk_shard("n1", [0, 5]), mk_shard("n2", [3, 10])]}], C1 = [{mk_shard("n1", [0, ?RING_END]), nil}], @@ -340,23 +337,31 @@ is_progress_possible_with_ring_opts_any_test() -> C2 = [{mk_shard("n1", [0, 5]), nil}], ?assertEqual(true, is_progress_possible(C2, [], 0, ?RING_END, Opts)). - is_progress_possible_with_ring_opts_all_test() -> C1 = [{mk_shard("n1", [0, ?RING_END]), nil}], ?assertEqual(true, is_progress_possible(C1, [], 0, ?RING_END, [all])), ?assertEqual(false, is_progress_possible([], [], 0, ?RING_END, [all])). - get_shard_replacements_test() -> - Unused = [mk_shard(N, [B, E]) || {N, B, E} <- [ - {"n1", 11, 20}, {"n1", 21, ?RING_END}, - {"n2", 0, 4}, {"n2", 5, 10}, {"n2", 11, 20}, - {"n3", 0, 21, ?RING_END} - ]], - Used = [mk_shard(N, [B, E]) || {N, B, E} <- [ - {"n2", 21, ?RING_END}, - {"n3", 0, 10}, {"n3", 11, 20} - ]], + Unused = [ + mk_shard(N, [B, E]) + || {N, B, E} <- [ + {"n1", 11, 20}, + {"n1", 21, ?RING_END}, + {"n2", 0, 4}, + {"n2", 5, 10}, + {"n2", 11, 20}, + {"n3", 0, 21, ?RING_END} + ] + ], + Used = [ + mk_shard(N, [B, E]) + || {N, B, E} <- [ + {"n2", 21, ?RING_END}, + {"n3", 0, 10}, + {"n3", 11, 20} + ] + ], Res = lists:sort(get_shard_replacements_int(Unused, Used)), % Notice that [0, 10] range can be replaces by spawning the % [0, 4] and [5, 10] workers on n1 @@ -367,7 +372,6 @@ get_shard_replacements_test() -> ], ?assertEqual(Expect, Res). - handle_response_basic_test() -> Shard1 = mk_shard("n1", [0, 1]), Shard2 = mk_shard("n1", [2, ?RING_END]), @@ -383,7 +387,6 @@ handle_response_basic_test() -> Result2 = handle_response(Shard2, 43, Workers2, Responses1, [], undefined), ?assertEqual({stop, [{Shard1, 42}, {Shard2, 43}]}, Result2). - handle_response_incomplete_ring_test() -> Shard1 = mk_shard("n1", [0, 1]), Shard2 = mk_shard("n1", [2, 10]), @@ -399,7 +402,6 @@ handle_response_incomplete_ring_test() -> Result2 = handle_response(Shard2, 43, Workers2, Responses1, [], undefined), ?assertEqual({stop, [{Shard1, 42}, {Shard2, 43}]}, Result2). - handle_response_multiple_copies_test() -> Shard1 = mk_shard("n1", [0, 1]), Shard2 = mk_shard("n2", [0, 1]), @@ -421,7 +423,6 @@ handle_response_multiple_copies_test() -> % that responded first is included in the ring. ?assertEqual({stop, [{Shard1, 42}, {Shard3, 44}]}, Result3). - handle_response_backtracking_test() -> Shard1 = mk_shard("n1", [0, 5]), Shard2 = mk_shard("n1", [10, ?RING_END]), @@ -445,7 +446,6 @@ handle_response_backtracking_test() -> Result4 = handle_response(Shard4, 45, Workers4, Responses3, [], undefined), ?assertEqual({stop, [{Shard3, 44}, {Shard4, 45}]}, Result4). - handle_response_ring_opts_any_test() -> Shard1 = mk_shard("n1", [0, 5]), Shard2 = mk_shard("n2", [0, 1]), @@ -469,7 +469,6 @@ handle_response_ring_opts_any_test() -> Result3 = handle_response(Shard3, 44, Workers3, [], Opts, undefined), ?assertEqual({stop, [{Shard3, 44}]}, Result3). - handle_response_ring_opts_all_test() -> Shard1 = mk_shard("n1", [0, 5]), Shard2 = mk_shard("n2", [0, 1]), @@ -493,7 +492,6 @@ handle_response_ring_opts_all_test() -> Result3 = handle_response(W3, 44, Workers3, [], [all], undefined), ?assertMatch({stop, [_ | _]}, Result3). - handle_error_test() -> Shard1 = mk_shard("n1", [0, 5]), Shard2 = mk_shard("n1", [10, ?RING_END]), @@ -516,7 +514,6 @@ handle_error_test() -> {ok, {Workers4, Responses3}} = Result3, ?assertEqual(error, handle_error(Shard4, Workers4, Responses3)). - node_down_test() -> Shard1 = mk_shard("n1", [0, 5]), Shard2 = mk_shard("n1", [10, ?RING_END]), @@ -547,20 +544,16 @@ node_down_test() -> ?assertEqual(error, node_down(n3, Workers5, Responses3)). - mk_cnts(Ranges) -> Shards = lists:map(fun mk_shard/1, Ranges), fabric_dict:init([S#shard{ref = make_ref()} || S <- Shards], nil). - mk_resps(RangeNameVals) -> [{{B, E}, mk_shard(Name, [B, E]), V} || {Name, B, E, V} <- RangeNameVals]. - mk_shard([B, E]) when is_integer(B), is_integer(E) -> #shard{range = [B, E]}. - mk_shard(Name, Range) -> Node = list_to_atom(Name), BName = list_to_binary(Name), diff --git a/src/fabric/src/fabric_rpc.erl b/src/fabric/src/fabric_rpc.erl index 4330f92be..a90c94ade 100644 --- a/src/fabric/src/fabric_rpc.erl +++ b/src/fabric/src/fabric_rpc.erl @@ -12,21 +12,47 @@ -module(fabric_rpc). --export([get_db_info/1, get_doc_count/1, get_design_doc_count/1, - get_update_seq/1]). --export([open_doc/3, open_revs/4, get_doc_info/3, get_full_doc_info/3, - get_missing_revs/2, get_missing_revs/3, update_docs/3]). +-export([ + get_db_info/1, + get_doc_count/1, + get_design_doc_count/1, + get_update_seq/1 +]). +-export([ + open_doc/3, + open_revs/4, + get_doc_info/3, + get_full_doc_info/3, + get_missing_revs/2, get_missing_revs/3, + update_docs/3 +]). -export([all_docs/3, changes/3, map_view/4, reduce_view/4, group_info/2]). --export([create_db/1, create_db/2, delete_db/1, reset_validation_funs/1, - set_security/3, set_revs_limit/3, create_shard_db_doc/2, - delete_shard_db_doc/2, get_partition_info/2]). +-export([ + create_db/1, create_db/2, + delete_db/1, + reset_validation_funs/1, + set_security/3, + set_revs_limit/3, + create_shard_db_doc/2, + delete_shard_db_doc/2, + get_partition_info/2 +]). -export([get_all_security/2, open_shard/2]). -export([compact/1, compact/2]). -export([get_purge_seq/2, purge_docs/3, set_purge_infos_limit/3]). --export([get_db_info/2, get_doc_count/2, get_design_doc_count/2, - get_update_seq/2, changes/4, map_view/5, reduce_view/5, - group_info/3, update_mrview/4, get_uuid/1]). +-export([ + get_db_info/2, + get_doc_count/2, + get_design_doc_count/2, + get_update_seq/2, + changes/4, + map_view/5, + reduce_view/5, + group_info/3, + update_mrview/4, + get_uuid/1 +]). -include_lib("fabric/include/fabric.hrl"). -include_lib("couch/include/couch_db.hrl"). @@ -45,79 +71,86 @@ changes(DbName, #changes_args{} = Args, StartSeq, DbOptions) -> changes(DbName, Options, StartVector, DbOptions) -> set_io_priority(DbName, DbOptions), Args0 = lists:keyfind(changes_args, 1, Options), - #changes_args{dir=Dir, filter_fun=Filter} = Args0, - Args = case Filter of - {fetch, custom, Style, Req, {DDocId, Rev}, FName} -> - {ok, DDoc} = ddoc_cache:open_doc(mem3:dbname(DbName), DDocId, Rev), - Args0#changes_args{ - filter_fun={custom, Style, Req, DDoc, FName} - }; - {fetch, view, Style, {DDocId, Rev}, VName} -> - {ok, DDoc} = ddoc_cache:open_doc(mem3:dbname(DbName), DDocId, Rev), - Args0#changes_args{filter_fun={view, Style, DDoc, VName}}; - _ -> - Args0 - end, + #changes_args{dir = Dir, filter_fun = Filter} = Args0, + Args = + case Filter of + {fetch, custom, Style, Req, {DDocId, Rev}, FName} -> + {ok, DDoc} = ddoc_cache:open_doc(mem3:dbname(DbName), DDocId, Rev), + Args0#changes_args{ + filter_fun = {custom, Style, Req, DDoc, FName} + }; + {fetch, view, Style, {DDocId, Rev}, VName} -> + {ok, DDoc} = ddoc_cache:open_doc(mem3:dbname(DbName), DDocId, Rev), + Args0#changes_args{filter_fun = {view, Style, DDoc, VName}}; + _ -> + Args0 + end, DbOpenOptions = Args#changes_args.db_open_options ++ DbOptions, case get_or_create_db(DbName, DbOpenOptions) of - {ok, Db} -> - StartSeq = calculate_start_seq(Db, node(), StartVector), - Enum = fun changes_enumerator/2, - Opts = [{dir,Dir}], - Acc0 = #fabric_changes_acc{ - db = Db, - seq = StartSeq, - args = Args, - options = Options, - pending = couch_db:count_changes_since(Db, StartSeq), - epochs = couch_db:get_epochs(Db) - }, - try - {ok, #fabric_changes_acc{seq=LastSeq, pending=Pending, epochs=Epochs}} = - do_changes(Db, StartSeq, Enum, Acc0, Opts), - rexi:stream_last({complete, [ - {seq, {LastSeq, uuid(Db), couch_db:owner_of(Epochs, LastSeq)}}, - {pending, Pending} - ]}) - after - couch_db:close(Db) - end; - Error -> - rexi:stream_last(Error) + {ok, Db} -> + StartSeq = calculate_start_seq(Db, node(), StartVector), + Enum = fun changes_enumerator/2, + Opts = [{dir, Dir}], + Acc0 = #fabric_changes_acc{ + db = Db, + seq = StartSeq, + args = Args, + options = Options, + pending = couch_db:count_changes_since(Db, StartSeq), + epochs = couch_db:get_epochs(Db) + }, + try + {ok, #fabric_changes_acc{seq = LastSeq, pending = Pending, epochs = Epochs}} = + do_changes(Db, StartSeq, Enum, Acc0, Opts), + rexi:stream_last( + {complete, [ + {seq, {LastSeq, uuid(Db), couch_db:owner_of(Epochs, LastSeq)}}, + {pending, Pending} + ]} + ) + after + couch_db:close(Db) + end; + Error -> + rexi:stream_last(Error) end. do_changes(Db, StartSeq, Enum, Acc0, Opts) -> - #fabric_changes_acc { + #fabric_changes_acc{ args = Args } = Acc0, - #changes_args { + #changes_args{ filter = Filter } = Args, case Filter of "_doc_ids" -> % optimised code path, we’re looking up all doc_ids in the by-id instead of filtering % the entire by-seq tree to find the doc_ids one by one - #changes_args { + #changes_args{ filter_fun = {doc_ids, Style, DocIds}, dir = Dir } = Args, - couch_changes:send_changes_doc_ids(Db, StartSeq, Dir, Enum, Acc0, {doc_ids, Style, DocIds}); + couch_changes:send_changes_doc_ids( + Db, StartSeq, Dir, Enum, Acc0, {doc_ids, Style, DocIds} + ); "_design_docs" -> % optimised code path, we’re looking up all design_docs in the by-id instead of % filtering the entire by-seq tree to find the design_docs one by one - #changes_args { + #changes_args{ filter_fun = {design_docs, Style}, dir = Dir } = Args, - couch_changes:send_changes_design_docs(Db, StartSeq, Dir, Enum, Acc0, {design_docs, Style}); + couch_changes:send_changes_design_docs( + Db, StartSeq, Dir, Enum, Acc0, {design_docs, Style} + ); _ -> couch_db:fold_changes(Db, StartSeq, Enum, Acc0, Opts) end. all_docs(DbName, Options, Args0) -> case fabric_util:upgrade_mrargs(Args0) of - #mrargs{keys=undefined} = Args -> + #mrargs{keys = undefined} = Args -> set_io_priority(DbName, Options), {ok, Db} = get_or_create_db(DbName, Options), CB = get_view_cb(Args), @@ -129,7 +162,8 @@ update_mrview(DbName, {DDocId, Rev}, ViewName, Args0) -> couch_util:with_db(DbName, fun(Db) -> UpdateSeq = couch_db:get_update_seq(Db), {ok, Pid, _} = couch_mrview:get_view_index_pid( - Db, DDoc, ViewName, fabric_util:upgrade_mrargs(Args0)), + Db, DDoc, ViewName, fabric_util:upgrade_mrargs(Args0) + ), couch_index:get_state(Pid, UpdateSeq) end). @@ -158,19 +192,21 @@ reduce_view(DbName, DDoc, ViewName, Args0, DbOptions) -> set_io_priority(DbName, DbOptions), Args = fabric_util:upgrade_mrargs(Args0), {ok, Db} = get_or_create_db(DbName, DbOptions), - VAcc0 = #vacc{db=Db}, + VAcc0 = #vacc{db = Db}, couch_mrview:query_view(Db, DDoc, ViewName, Args, fun reduce_cb/2, VAcc0). create_db(DbName) -> create_db(DbName, []). create_db(DbName, Options) -> - rexi:reply(case couch_server:create(DbName, Options) of - {ok, _} -> - ok; - Error -> - Error - end). + rexi:reply( + case couch_server:create(DbName, Options) of + {ok, _} -> + ok; + Error -> + Error + end + ). create_shard_db_doc(_, Doc) -> rexi:reply(mem3_util:write_db_doc(Doc)). @@ -213,12 +249,13 @@ get_update_seq(DbName, DbOptions) -> with_db(DbName, DbOptions, {couch_db, get_update_seq, []}). set_security(DbName, SecObj, Options0) -> - Options = case lists:keyfind(io_priority, 1, Options0) of - false -> - [{io_priority, {db_meta, security}}|Options0]; - _ -> - Options0 - end, + Options = + case lists:keyfind(io_priority, 1, Options0) of + false -> + [{io_priority, {db_meta, security}} | Options0]; + _ -> + Options0 + end, with_db(DbName, Options, {couch_db, set_security, [SecObj]}). get_all_security(DbName, Options) -> @@ -248,38 +285,46 @@ get_missing_revs(DbName, IdRevsList) -> get_missing_revs(DbName, IdRevsList, Options) -> % reimplement here so we get [] for Ids with no missing revs in response set_io_priority(DbName, Options), - rexi:reply(case get_or_create_db(DbName, Options) of - {ok, Db} -> - Ids = [Id1 || {Id1, _Revs} <- IdRevsList], - {ok, lists:zipwith(fun({Id, Revs}, FullDocInfoResult) -> - case FullDocInfoResult of - #full_doc_info{rev_tree=RevisionTree} = FullInfo -> - MissingRevs = couch_key_tree:find_missing(RevisionTree, Revs), - {Id, MissingRevs, possible_ancestors(FullInfo, MissingRevs)}; - not_found -> - {Id, Revs, []} - end - end, IdRevsList, couch_db:get_full_doc_infos(Db, Ids))}; - Error -> - Error - end). + rexi:reply( + case get_or_create_db(DbName, Options) of + {ok, Db} -> + Ids = [Id1 || {Id1, _Revs} <- IdRevsList], + {ok, + lists:zipwith( + fun({Id, Revs}, FullDocInfoResult) -> + case FullDocInfoResult of + #full_doc_info{rev_tree = RevisionTree} = FullInfo -> + MissingRevs = couch_key_tree:find_missing(RevisionTree, Revs), + {Id, MissingRevs, possible_ancestors(FullInfo, MissingRevs)}; + not_found -> + {Id, Revs, []} + end + end, + IdRevsList, + couch_db:get_full_doc_infos(Db, Ids) + )}; + Error -> + Error + end + ). update_docs(DbName, Docs0, Options) -> - {Docs1, Type} = case couch_util:get_value(read_repair, Options) of - NodeRevs when is_list(NodeRevs) -> - Filtered = read_repair_filter(DbName, Docs0, NodeRevs, Options), - {Filtered, replicated_changes}; - undefined -> - X = case proplists:get_value(replicated_changes, Options) of - true -> replicated_changes; - _ -> interactive_edit - end, - {Docs0, X} - end, + {Docs1, Type} = + case couch_util:get_value(read_repair, Options) of + NodeRevs when is_list(NodeRevs) -> + Filtered = read_repair_filter(DbName, Docs0, NodeRevs, Options), + {Filtered, replicated_changes}; + undefined -> + X = + case proplists:get_value(replicated_changes, Options) of + true -> replicated_changes; + _ -> interactive_edit + end, + {Docs0, X} + end, Docs2 = make_att_readers(Docs1), with_db(DbName, Options, {couch_db, update_docs, [Docs2, Options, Type]}). - get_purge_seq(DbName, Options) -> with_db(DbName, Options, {couch_db, get_purge_seq, []}). @@ -295,18 +340,19 @@ group_info(DbName, DDocId, DbOptions) -> reset_validation_funs(DbName) -> case get_or_create_db(DbName, []) of - {ok, Db} -> - couch_db:reload_validation_funs(Db); - _ -> - ok + {ok, Db} -> + couch_db:reload_validation_funs(Db); + _ -> + ok end. open_shard(Name, Opts) -> set_io_priority(Name, Opts), try rexi:reply(mem3_util:get_or_create_db(Name, Opts)) - catch exit:{timeout, _} -> - couch_stats:increment_counter([fabric, open_shard, timeouts]) + catch + exit:{timeout, _} -> + couch_stats:increment_counter([fabric, open_shard, timeouts]) end. compact(DbName) -> @@ -314,7 +360,8 @@ compact(DbName) -> compact(ShardName, DesignName) -> {ok, Pid} = couch_index_server:get_index( - couch_mrview_index, ShardName, <<"_design/", DesignName/binary>>), + couch_mrview_index, ShardName, <<"_design/", DesignName/binary>> + ), Ref = erlang:make_ref(), Pid ! {'$gen_call', {self(), Ref}, compact}. @@ -342,7 +389,6 @@ with_db(DbName, Options, {M,F,A}) -> rexi:reply(Error) end. - read_repair_filter(DbName, Docs, NodeRevs, Options) -> set_io_priority(DbName, Options), case get_or_create_db(DbName, Options) of @@ -356,7 +402,6 @@ read_repair_filter(DbName, Docs, NodeRevs, Options) -> rexi:reply(Error) end. - % A read repair operation may have been triggered by a node % that was out of sync with the local node. Thus, any time % we receive a read repair request we need to check if we @@ -393,16 +438,21 @@ read_repair_filter(Db, Docs, NodeRevs) -> % For each node we scan the purge infos to filter out any % revisions that have been locally purged since we last % replicated to the remote node's shard copy. - AllowableRevs = lists:foldl(fun({Node, Revs}, RevAcc) -> - {Node, StartSeq} = lists:keyfind(Node, 1, NodeSeqs), - FoldFun = fun({_PSeq, _UUID, PDocId, PRevs}, InnerAcc) -> - if PDocId /= DocId -> {ok, InnerAcc}; true -> - {ok, InnerAcc -- PRevs} - end + AllowableRevs = lists:foldl( + fun({Node, Revs}, RevAcc) -> + {Node, StartSeq} = lists:keyfind(Node, 1, NodeSeqs), + FoldFun = fun({_PSeq, _UUID, PDocId, PRevs}, InnerAcc) -> + if + PDocId /= DocId -> {ok, InnerAcc}; + true -> {ok, InnerAcc -- PRevs} + end + end, + {ok, FiltRevs} = couch_db:fold_purge_infos(Db, StartSeq, FoldFun, Revs), + lists:usort(FiltRevs ++ RevAcc) end, - {ok, FiltRevs} = couch_db:fold_purge_infos(Db, StartSeq, FoldFun, Revs), - lists:usort(FiltRevs ++ RevAcc) - end, [], RecentNodeRevs), + [], + RecentNodeRevs + ), % Finally, filter the doc updates to only include revisions % that have not been purged locally. @@ -411,7 +461,6 @@ read_repair_filter(Db, Docs, NodeRevs) -> end, lists:filter(DocFiltFun, Docs). - get_node_seqs(Db, Nodes) -> % Gather the list of {Node, PurgeSeq} pairs for all nodes % that are present in our read repair group @@ -439,12 +488,9 @@ get_node_seqs(Db, Nodes) -> {ok, NodeBinSeqs} = couch_db:fold_local_docs(Db, FoldFun, InitAcc, Opts), [{list_to_existing_atom(binary_to_list(N)), S} || {N, S} <- NodeBinSeqs]. - - get_or_create_db(DbName, Options) -> mem3_util:get_or_create_db_int(DbName, Options). - get_view_cb(#mrargs{extra = Options}) -> case couch_util:get_value(callback, Options) of {Mod, Fun} when is_atom(Mod), is_atom(Fun) -> @@ -455,7 +501,6 @@ get_view_cb(#mrargs{extra = Options}) -> get_view_cb(_) -> fun view_cb/2. - view_cb({meta, Meta}, Acc) -> % Map function starting ok = rexi:stream2({meta, Meta}), @@ -477,7 +522,6 @@ view_cb(complete, Acc) -> view_cb(ok, ddoc_updated) -> rexi:reply({ok, ddoc_updated}). - reduce_cb({meta, Meta}, Acc) -> % Map function starting ok = rexi:stream2({meta, Meta}), @@ -496,11 +540,10 @@ reduce_cb(complete, Acc) -> reduce_cb(ok, ddoc_updated) -> rexi:reply({ok, ddoc_updated}). - changes_enumerator(#full_doc_info{} = FDI, Acc) -> changes_enumerator(couch_doc:to_doc_info(FDI), Acc); -changes_enumerator(#doc_info{id= <<"_local/", _/binary>>, high_seq=Seq}, Acc) -> - {ok, Acc#fabric_changes_acc{seq = Seq, pending = Acc#fabric_changes_acc.pending-1}}; +changes_enumerator(#doc_info{id = <<"_local/", _/binary>>, high_seq = Seq}, Acc) -> + {ok, Acc#fabric_changes_acc{seq = Seq, pending = Acc#fabric_changes_acc.pending - 1}}; changes_enumerator(DocInfo, Acc) -> #fabric_changes_acc{ db = Db, @@ -513,63 +556,82 @@ changes_enumerator(DocInfo, Acc) -> pending = Pending, epochs = Epochs } = Acc, - #doc_info{id=Id, high_seq=Seq, revs=[#rev_info{deleted=Del}|_]} = DocInfo, + #doc_info{id = Id, high_seq = Seq, revs = [#rev_info{deleted = Del} | _]} = DocInfo, case [X || X <- couch_changes:filter(Db, DocInfo, Filter), X /= null] of - [] -> - ChangesRow = {no_pass, [ - {pending, Pending-1}, - {seq, {Seq, uuid(Db), couch_db:owner_of(Epochs, Seq)}} - ]}; - Results -> - Opts = if Conflicts -> [conflicts | DocOptions]; true -> DocOptions end, - ChangesRow = {change, [ - {pending, Pending-1}, - {seq, {Seq, uuid(Db), couch_db:owner_of(Epochs, Seq)}}, - {id, Id}, - {changes, Results}, - {deleted, Del} | - if IncludeDocs -> [doc_member(Db, DocInfo, Opts, Filter)]; true -> [] end - ]} + [] -> + ChangesRow = + {no_pass, [ + {pending, Pending - 1}, + {seq, {Seq, uuid(Db), couch_db:owner_of(Epochs, Seq)}} + ]}; + Results -> + Opts = + if + Conflicts -> [conflicts | DocOptions]; + true -> DocOptions + end, + ChangesRow = + {change, [ + {pending, Pending - 1}, + {seq, {Seq, uuid(Db), couch_db:owner_of(Epochs, Seq)}}, + {id, Id}, + {changes, Results}, + {deleted, Del} + | if + IncludeDocs -> [doc_member(Db, DocInfo, Opts, Filter)]; + true -> [] + end + ]} end, ok = rexi:stream2(ChangesRow), - {ok, Acc#fabric_changes_acc{seq = Seq, pending = Pending-1}}. + {ok, Acc#fabric_changes_acc{seq = Seq, pending = Pending - 1}}. doc_member(Shard, DocInfo, Opts, Filter) -> case couch_db:open_doc(Shard, DocInfo, [deleted | Opts]) of - {ok, Doc} -> - {doc, maybe_filtered_json_doc(Doc, Opts, Filter)}; - Error -> - Error + {ok, Doc} -> + {doc, maybe_filtered_json_doc(Doc, Opts, Filter)}; + Error -> + Error end. -maybe_filtered_json_doc(Doc, Opts, {selector, _Style, {_Selector, Fields}}) - when Fields =/= nil -> +maybe_filtered_json_doc(Doc, Opts, {selector, _Style, {_Selector, Fields}}) when + Fields =/= nil +-> mango_fields:extract(couch_doc:to_json_obj(Doc, Opts), Fields); maybe_filtered_json_doc(Doc, Opts, _Filter) -> couch_doc:to_json_obj(Doc, Opts). - possible_ancestors(_FullInfo, []) -> []; possible_ancestors(FullInfo, MissingRevs) -> - #doc_info{revs=RevsInfo} = couch_doc:to_doc_info(FullInfo), - LeafRevs = [Rev || #rev_info{rev=Rev} <- RevsInfo], + #doc_info{revs = RevsInfo} = couch_doc:to_doc_info(FullInfo), + LeafRevs = [Rev || #rev_info{rev = Rev} <- RevsInfo], % Find the revs that are possible parents of this rev - lists:foldl(fun({LeafPos, LeafRevId}, Acc) -> - % this leaf is a "possible ancenstor" of the missing - % revs if this LeafPos lessthan any of the missing revs - case lists:any(fun({MissingPos, _}) -> - LeafPos < MissingPos end, MissingRevs) of - true -> - [{LeafPos, LeafRevId} | Acc]; - false -> - Acc - end - end, [], LeafRevs). + lists:foldl( + fun({LeafPos, LeafRevId}, Acc) -> + % this leaf is a "possible ancenstor" of the missing + % revs if this LeafPos lessthan any of the missing revs + case + lists:any( + fun({MissingPos, _}) -> + LeafPos < MissingPos + end, + MissingRevs + ) + of + true -> + [{LeafPos, LeafRevId} | Acc]; + false -> + Acc + end + end, + [], + LeafRevs + ). make_att_readers([]) -> []; -make_att_readers([#doc{atts=Atts0} = Doc | Rest]) -> +make_att_readers([#doc{atts = Atts0} = Doc | Rest]) -> % % go through the attachments looking for 'follows' in the data, % % replace with function that reads the data from MIME stream. Atts = [couch_att:transform(data, fun make_att_reader/1, Att) || Att <- Atts0], @@ -577,14 +639,15 @@ make_att_readers([#doc{atts=Atts0} = Doc | Rest]) -> make_att_reader({follows, Parser, Ref}) -> fun() -> - ParserRef = case get(mp_parser_ref) of - undefined -> - PRef = erlang:monitor(process, Parser), - put(mp_parser_ref, PRef), - PRef; - Else -> - Else - end, + ParserRef = + case get(mp_parser_ref) of + undefined -> + PRef = erlang:monitor(process, Parser), + put(mp_parser_ref, PRef), + PRef; + Else -> + Else + end, Parser ! {get_bytes, Ref, self()}, receive {bytes, Ref, Bytes} -> @@ -600,14 +663,20 @@ make_att_reader(Else) -> Else. clean_stack(S) -> - lists:map(fun({M,F,A}) when is_list(A) -> {M,F,length(A)}; (X) -> X end, S). + lists:map( + fun + ({M, F, A}) when is_list(A) -> {M, F, length(A)}; + (X) -> X + end, + S + ). set_io_priority(DbName, Options) -> case lists:keyfind(io_priority, 1, Options) of - {io_priority, Pri} -> - erlang:put(io_priority, Pri); - false -> - erlang:put(io_priority, {interactive, DbName}) + {io_priority, Pri} -> + erlang:put(io_priority, Pri); + false -> + erlang:put(io_priority, {interactive, DbName}) end, case erlang:get(io_priority) of {interactive, _} -> @@ -623,7 +692,6 @@ set_io_priority(DbName, Options) -> ok end. - calculate_start_seq(Db, Node, Seq) -> case couch_db:calculate_start_seq(Db, Node, Seq) of N when is_integer(N) -> @@ -637,13 +705,11 @@ calculate_start_seq(Db, Node, Seq) -> mem3_rep:find_source_seq(Db, OriginalNode, Uuid, OriginalSeq) end. - uuid(Db) -> Uuid = couch_db:get_uuid(Db), Prefix = fabric_util:get_uuid_prefix_len(), binary:part(Uuid, {0, Prefix}). - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). diff --git a/src/fabric/src/fabric_streams.erl b/src/fabric/src/fabric_streams.erl index 59c8b8a6b..2a3a2b004 100644 --- a/src/fabric/src/fabric_streams.erl +++ b/src/fabric/src/fabric_streams.erl @@ -23,22 +23,17 @@ -include_lib("fabric/include/fabric.hrl"). -include_lib("mem3/include/mem3.hrl"). - -define(WORKER_CLEANER, fabric_worker_cleaner). - start(Workers, Keypos) -> start(Workers, Keypos, undefined, undefined). - start(Workers, Keypos, RingOpts) -> start(Workers, Keypos, undefined, undefined, RingOpts). - start(Workers, Keypos, StartFun, Replacements) -> start(Workers, Keypos, StartFun, Replacements, []). - start(Workers0, Keypos, StartFun, Replacements, RingOpts) -> Fun = fun handle_stream_start/3, Acc = #stream_acc{ @@ -52,16 +47,19 @@ start(Workers0, Keypos, StartFun, Replacements, RingOpts) -> Timeout = fabric_util:request_timeout(), case rexi_utils:recv(Workers0, Keypos, Fun, Acc, Timeout, infinity) of {ok, #stream_acc{ready = Workers}} -> - AckedWorkers = fabric_dict:fold(fun(Worker, From, WorkerAcc) -> - rexi:stream_start(From), - [Worker | WorkerAcc] - end, [], Workers), + AckedWorkers = fabric_dict:fold( + fun(Worker, From, WorkerAcc) -> + rexi:stream_start(From), + [Worker | WorkerAcc] + end, + [], + Workers + ), {ok, AckedWorkers}; Else -> Else end. - cleanup(Workers) -> % Stop the auxiliary cleaner process as we got to the point where cleanup % happesn in the regular fashion so we don't want to send 2x the number kill @@ -75,7 +73,6 @@ cleanup(Workers) -> end, fabric_util:cleanup(Workers). - handle_stream_start({rexi_DOWN, _, {_, NodeRef}, _}, _, St) -> #stream_acc{workers = Workers, ready = Ready, ring_opts = RingOpts} = St, case fabric_ring:node_down(NodeRef, Workers, Ready, RingOpts) of @@ -84,7 +81,6 @@ handle_stream_start({rexi_DOWN, _, {_, NodeRef}, _}, _, St) -> error -> {error, {nodedown, <<"progress not possible">>}} end; - handle_stream_start({rexi_EXIT, Reason}, Worker, St) -> #stream_acc{ workers = Workers, @@ -100,11 +96,15 @@ handle_stream_start({rexi_EXIT, Reason}, Worker, St) -> % and start the new workers if so. case lists:keytake(Worker#shard.range, 1, Replacements) of {value, {_Range, WorkerReplacements}, NewReplacements} -> - FinalWorkers = lists:foldl(fun(Repl, NewWorkers) -> - NewWorker = (St#stream_acc.start_fun)(Repl), - add_worker_to_cleaner(self(), NewWorker), - fabric_dict:store(NewWorker, waiting, NewWorkers) - end, Workers, WorkerReplacements), + FinalWorkers = lists:foldl( + fun(Repl, NewWorkers) -> + NewWorker = (St#stream_acc.start_fun)(Repl), + add_worker_to_cleaner(self(), NewWorker), + fabric_dict:store(NewWorker, waiting, NewWorkers) + end, + Workers, + WorkerReplacements + ), % Assert that our replaced worker provides us % the oppurtunity to make progress. Need to make sure % to include already processed responses, since we are @@ -115,8 +115,8 @@ handle_stream_start({rexi_EXIT, Reason}, Worker, St) -> true = fabric_ring:is_progress_possible(AllWorkers), NewRefs = fabric_dict:fetch_keys(FinalWorkers), {new_refs, NewRefs, St#stream_acc{ - workers=FinalWorkers, - replacements=NewReplacements + workers = FinalWorkers, + replacements = NewReplacements }}; false -> % If we progress isn't possible and we don't have any @@ -126,36 +126,32 @@ handle_stream_start({rexi_EXIT, Reason}, Worker, St) -> {error, _} -> {error, fabric_util:error_info(Reason)} end; - handle_stream_start(rexi_STREAM_INIT, {Worker, From}, St) -> #stream_acc{workers = Workers, ready = Ready, ring_opts = RingOpts} = St, case fabric_dict:lookup_element(Worker, Workers) of - undefined -> - % This worker lost the race with other partition copies, terminate - rexi:stream_cancel(From), - {ok, St}; - waiting -> - case fabric_ring:handle_response(Worker, From, Workers, Ready, RingOpts) of - {ok, {Workers1, Ready1}} -> - % Don't have a full ring yet. Keep getting responses - {ok, St#stream_acc{workers = Workers1, ready = Ready1}}; - {stop, Ready1} -> - % Have a full ring of workers. But don't ack the worker - % yet so they don't start sending us rows until we're ready - {stop, St#stream_acc{workers = [], ready = Ready1}} - end + undefined -> + % This worker lost the race with other partition copies, terminate + rexi:stream_cancel(From), + {ok, St}; + waiting -> + case fabric_ring:handle_response(Worker, From, Workers, Ready, RingOpts) of + {ok, {Workers1, Ready1}} -> + % Don't have a full ring yet. Keep getting responses + {ok, St#stream_acc{workers = Workers1, ready = Ready1}}; + {stop, Ready1} -> + % Have a full ring of workers. But don't ack the worker + % yet so they don't start sending us rows until we're ready + {stop, St#stream_acc{workers = [], ready = Ready1}} + end end; - handle_stream_start({ok, ddoc_updated}, _, St) -> WaitingWorkers = [W || {W, _} <- St#stream_acc.workers], ReadyWorkers = [W || {W, _} <- St#stream_acc.ready], cleanup(WaitingWorkers ++ ReadyWorkers), {stop, ddoc_updated}; - handle_stream_start(Else, _, _) -> exit({invalid_stream_start, Else}). - % Spawn an auxiliary rexi worker cleaner. This will be used in cases % when the coordinator (request) process is forceably killed and doesn't % get a chance to process its `after` fabric:clean/1 clause. @@ -168,10 +164,9 @@ spawn_worker_cleaner(Coordinator, Workers) -> end), put(?WORKER_CLEANER, Pid), Pid; - ExistingCleaner -> + ExistingCleaner -> ExistingCleaner - end. - + end. cleaner_loop(Pid, Workers) -> receive @@ -181,7 +176,6 @@ cleaner_loop(Pid, Workers) -> fabric_util:cleanup(Workers) end. - add_worker_to_cleaner(CoordinatorPid, Worker) -> case get(?WORKER_CLEANER) of CleanerPid when is_pid(CleanerPid) -> @@ -190,25 +184,27 @@ add_worker_to_cleaner(CoordinatorPid, Worker) -> ok end. - - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). worker_cleaner_test_() -> { - "Fabric spawn_worker_cleaner test", { - setup, fun setup/0, fun teardown/1, - fun(_) -> [ - should_clean_workers(), - does_not_fire_if_cleanup_called(), - should_clean_additional_worker_too() - ] end + "Fabric spawn_worker_cleaner test", + { + setup, + fun setup/0, + fun teardown/1, + fun(_) -> + [ + should_clean_workers(), + does_not_fire_if_cleanup_called(), + should_clean_additional_worker_too() + ] + end } }. - should_clean_workers() -> ?_test(begin meck:reset(rexi), @@ -217,15 +213,20 @@ should_clean_workers() -> #shard{node = 'n1', ref = make_ref()}, #shard{node = 'n2', ref = make_ref()} ], - {Coord, _} = spawn_monitor(fun() -> receive die -> ok end end), + {Coord, _} = spawn_monitor(fun() -> + receive + die -> ok + end + end), Cleaner = spawn_worker_cleaner(Coord, Workers), Ref = erlang:monitor(process, Cleaner), Coord ! die, - receive {'DOWN', Ref, _, Cleaner, _} -> ok end, + receive + {'DOWN', Ref, _, Cleaner, _} -> ok + end, ?assertEqual(1, meck:num_calls(rexi, kill_all, 1)) end). - does_not_fire_if_cleanup_called() -> ?_test(begin meck:reset(rexi), @@ -234,18 +235,23 @@ does_not_fire_if_cleanup_called() -> #shard{node = 'n1', ref = make_ref()}, #shard{node = 'n2', ref = make_ref()} ], - {Coord, _} = spawn_monitor(fun() -> receive die -> ok end end), + {Coord, _} = spawn_monitor(fun() -> + receive + die -> ok + end + end), Cleaner = spawn_worker_cleaner(Coord, Workers), Ref = erlang:monitor(process, Cleaner), cleanup(Workers), Coord ! die, - receive {'DOWN', Ref, _, _, _} -> ok end, + receive + {'DOWN', Ref, _, _, _} -> ok + end, % 2 calls would be from cleanup/1 function. If cleanup process fired % too it would have been 4 calls total. ?assertEqual(1, meck:num_calls(rexi, kill_all, 1)) end). - should_clean_additional_worker_too() -> ?_test(begin meck:reset(rexi), @@ -253,20 +259,24 @@ should_clean_additional_worker_too() -> Workers = [ #shard{node = 'n1', ref = make_ref()} ], - {Coord, _} = spawn_monitor(fun() -> receive die -> ok end end), + {Coord, _} = spawn_monitor(fun() -> + receive + die -> ok + end + end), Cleaner = spawn_worker_cleaner(Coord, Workers), add_worker_to_cleaner(Coord, #shard{node = 'n2', ref = make_ref()}), Ref = erlang:monitor(process, Cleaner), Coord ! die, - receive {'DOWN', Ref, _, Cleaner, _} -> ok end, + receive + {'DOWN', Ref, _, Cleaner, _} -> ok + end, ?assertEqual(1, meck:num_calls(rexi, kill_all, 1)) end). - setup() -> ok = meck:expect(rexi, kill_all, fun(_) -> ok end). - teardown(_) -> meck:unload(). diff --git a/src/fabric/src/fabric_util.erl b/src/fabric/src/fabric_util.erl index 4c4031627..30e82c29a 100644 --- a/src/fabric/src/fabric_util.erl +++ b/src/fabric/src/fabric_util.erl @@ -12,9 +12,19 @@ -module(fabric_util). --export([submit_jobs/3, submit_jobs/4, cleanup/1, recv/4, get_db/1, get_db/2, error_info/1, - update_counter/3, remove_ancestors/2, create_monitors/1, kv/2, - remove_down_workers/2, remove_down_workers/3, doc_id_and_rev/1]). +-export([ + submit_jobs/3, submit_jobs/4, + cleanup/1, + recv/4, + get_db/1, get_db/2, + error_info/1, + update_counter/3, + remove_ancestors/2, + create_monitors/1, + kv/2, + remove_down_workers/2, remove_down_workers/3, + doc_id_and_rev/1 +]). -export([request_timeout/0, attachments_timeout/0, all_docs_timeout/0, view_timeout/1]). -export([log_timeout/2, remove_done_workers/2]). -export([is_users_db/1, is_replicator_db/1]). @@ -26,8 +36,7 @@ -export([get_uuid_prefix_len/0]). -export([isolate/1, isolate/2]). - --compile({inline, [{doc_id_and_rev,1}]}). +-compile({inline, [{doc_id_and_rev, 1}]}). -include_lib("fabric/include/fabric.hrl"). -include_lib("mem3/include/mem3.hrl"). @@ -42,20 +51,23 @@ remove_down_workers(Workers, BadNode, RingOpts) -> Filter = fun(#shard{node = Node}, _) -> Node =/= BadNode end, NewWorkers = fabric_dict:filter(Filter, Workers), case fabric_ring:is_progress_possible(NewWorkers, RingOpts) of - true -> - {ok, NewWorkers}; - false -> - error + true -> + {ok, NewWorkers}; + false -> + error end. submit_jobs(Shards, EndPoint, ExtraArgs) -> submit_jobs(Shards, fabric_rpc, EndPoint, ExtraArgs). submit_jobs(Shards, Module, EndPoint, ExtraArgs) -> - lists:map(fun(#shard{node=Node, name=ShardName} = Shard) -> - Ref = rexi:cast(Node, {Module, EndPoint, [ShardName | ExtraArgs]}), - Shard#shard{ref = Ref} - end, Shards). + lists:map( + fun(#shard{node = Node, name = ShardName} = Shard) -> + Ref = rexi:cast(Node, {Module, EndPoint, [ShardName | ExtraArgs]}), + Shard#shard{ref = Ref} + end, + Shards + ). cleanup(Workers) -> rexi:kill_all([{Node, Ref} || #shard{node = Node, ref = Ref} <- Workers]). @@ -88,10 +100,13 @@ timeout(Type, Default) -> log_timeout(Workers, EndPoint) -> CounterKey = [fabric, worker, timeouts], couch_stats:increment_counter(CounterKey), - lists:map(fun(#shard{node=Dest, name=Name}) -> - Fmt = "fabric_worker_timeout ~s,~p,~p", - couch_log:error(Fmt, [EndPoint, Dest, Name]) - end, Workers). + lists:map( + fun(#shard{node = Dest, name = Name}) -> + Fmt = "fabric_worker_timeout ~s,~p,~p", + couch_log:error(Fmt, [EndPoint, Dest, Name]) + end, + Workers + ). remove_done_workers(Workers, WaitingIndicator) -> [W || {W, WI} <- fabric_dict:to_list(Workers), WI == WaitingIndicator]. @@ -103,9 +118,10 @@ get_db(DbName, Options) -> {Local, SameZone, DifferentZone} = mem3:group_by_proximity(mem3:shards(DbName)), % Prefer shards on the same node over other nodes, prefer shards in the same zone over % over zones and sort each remote list by name so that we don't repeatedly try the same node. - Shards = Local ++ lists:keysort(#shard.name, SameZone) ++ lists:keysort(#shard.name, DifferentZone), + Shards = + Local ++ lists:keysort(#shard.name, SameZone) ++ lists:keysort(#shard.name, DifferentZone), % suppress shards from down nodes - Nodes = [node()|erlang:nodes()], + Nodes = [node() | erlang:nodes()], Live = [S || #shard{node = N} = S <- Shards, lists:member(N, Nodes)], % Only accept factors > 1, otherwise our math breaks further down Factor = max(2, config:get_integer("fabric", "shard_timeout_factor", 2)), @@ -121,15 +137,16 @@ get_shard([#shard{node = Node, name = Name} | Rest], Opts, Timeout, Factor) -> MFA = {fabric_rpc, open_shard, [Name, [{timeout, Timeout} | Opts]]}, Ref = rexi:cast(Node, self(), MFA, [sync]), try - receive {Ref, {ok, Db}} -> - {ok, Db}; - {Ref, {'rexi_EXIT', {{unauthorized, _} = Error, _}}} -> - throw(Error); - {Ref, {'rexi_EXIT', {{forbidden, _} = Error, _}}} -> - throw(Error); - {Ref, Reason} -> - couch_log:debug("Failed to open shard ~p because: ~p", [Name, Reason]), - get_shard(Rest, Opts, Timeout, Factor) + receive + {Ref, {ok, Db}} -> + {ok, Db}; + {Ref, {'rexi_EXIT', {{unauthorized, _} = Error, _}}} -> + throw(Error); + {Ref, {'rexi_EXIT', {{forbidden, _} = Error, _}}} -> + throw(Error); + {Ref, Reason} -> + couch_log:debug("Failed to open shard ~p because: ~p", [Name, Reason]), + get_shard(Rest, Opts, Timeout, Factor) after Timeout -> couch_log:debug("Failed to open shard ~p after: ~p", [Name, Timeout]), get_shard(Rest, Opts, Factor * Timeout, Factor) @@ -169,18 +186,18 @@ error_info({Error, Stack}) -> {Error, nil, Stack}. update_counter(Item, Incr, D) -> - UpdateFun = fun ({Old, Count}) -> {Old, Count + Incr} end, + UpdateFun = fun({Old, Count}) -> {Old, Count + Incr} end, orddict:update(make_key(Item), UpdateFun, {Item, Incr}, D). make_key({ok, L}) when is_list(L) -> make_key(L); make_key([]) -> []; -make_key([{ok, #doc{revs= {Pos,[RevId | _]}}} | Rest]) -> +make_key([{ok, #doc{revs = {Pos, [RevId | _]}}} | Rest]) -> [{ok, {Pos, RevId}} | make_key(Rest)]; make_key([{{not_found, missing}, Rev} | Rest]) -> [{not_found, Rev} | make_key(Rest)]; -make_key({ok, #doc{id=Id,revs=Revs}}) -> +make_key({ok, #doc{id = Id, revs = Revs}}) -> {Id, Revs}; make_key(Else) -> Else. @@ -190,69 +207,88 @@ remove_ancestors([], Acc) -> lists:reverse(Acc); remove_ancestors([{_, {{not_found, _}, Count}} = Head | Tail], Acc) -> % any document is a descendant - case lists:filter(fun({_,{{ok, #doc{}}, _}}) -> true; (_) -> false end, Tail) of - [{_,{{ok, #doc{}} = Descendant, _}} | _] -> - remove_ancestors(update_counter(Descendant, Count, Tail), Acc); - [] -> - remove_ancestors(Tail, [Head | Acc]) + case + lists:filter( + fun + ({_, {{ok, #doc{}}, _}}) -> true; + (_) -> false + end, + Tail + ) + of + [{_, {{ok, #doc{}} = Descendant, _}} | _] -> + remove_ancestors(update_counter(Descendant, Count, Tail), Acc); + [] -> + remove_ancestors(Tail, [Head | Acc]) end; -remove_ancestors([{_,{{ok, #doc{revs = {Pos, Revs}}}, Count}} = Head | Tail], Acc) -> - Descendants = lists:dropwhile(fun - ({_,{{ok, #doc{revs = {Pos2, Revs2}}}, _}}) -> - case lists:nthtail(erlang:min(Pos2 - Pos, length(Revs2)), Revs2) of +remove_ancestors([{_, {{ok, #doc{revs = {Pos, Revs}}}, Count}} = Head | Tail], Acc) -> + Descendants = lists:dropwhile( + fun({_, {{ok, #doc{revs = {Pos2, Revs2}}}, _}}) -> + case lists:nthtail(erlang:min(Pos2 - Pos, length(Revs2)), Revs2) of + [] -> + % impossible to tell if Revs2 is a descendant - assume no + true; + History -> + % if Revs2 is a descendant, History is a prefix of Revs + not lists:prefix(History, Revs) + end + end, + Tail + ), + case Descendants of [] -> - % impossible to tell if Revs2 is a descendant - assume no - true; - History -> - % if Revs2 is a descendant, History is a prefix of Revs - not lists:prefix(History, Revs) - end - end, Tail), - case Descendants of [] -> - remove_ancestors(Tail, [Head | Acc]); - [{Descendant, _} | _] -> - remove_ancestors(update_counter(Descendant, Count, Tail), Acc) + remove_ancestors(Tail, [Head | Acc]); + [{Descendant, _} | _] -> + remove_ancestors(update_counter(Descendant, Count, Tail), Acc) end; remove_ancestors([Error | Tail], Acc) -> remove_ancestors(Tail, [Error | Acc]). create_monitors(Shards) -> - MonRefs = lists:usort([ - rexi_utils:server_pid(N) || #shard{node=N} <- Shards - ]), + MonRefs = lists:usort([rexi_utils:server_pid(N) || #shard{node = N} <- Shards]), rexi_monitor:start(MonRefs). %% verify only id and rev are used in key. update_counter_test() -> - Reply = {ok, #doc{id = <<"id">>, revs = <<"rev">>, - body = <<"body">>, atts = <<"atts">>}}, - ?assertEqual([{{<<"id">>,<<"rev">>}, {Reply, 1}}], - update_counter(Reply, 1, [])). + Reply = + {ok, #doc{ + id = <<"id">>, + revs = <<"rev">>, + body = <<"body">>, + atts = <<"atts">> + }}, + ?assertEqual( + [{{<<"id">>, <<"rev">>}, {Reply, 1}}], + update_counter(Reply, 1, []) + ). remove_ancestors_test() -> Foo1 = {ok, #doc{revs = {1, [<<"foo">>]}}}, Foo2 = {ok, #doc{revs = {2, [<<"foo2">>, <<"foo">>]}}}, Bar1 = {ok, #doc{revs = {1, [<<"bar">>]}}}, - Bar2 = {not_found, {1,<<"bar">>}}, + Bar2 = {not_found, {1, <<"bar">>}}, ?assertEqual( - [kv(Bar1,1), kv(Foo1,1)], - remove_ancestors([kv(Bar1,1), kv(Foo1,1)], []) + [kv(Bar1, 1), kv(Foo1, 1)], + remove_ancestors([kv(Bar1, 1), kv(Foo1, 1)], []) ), ?assertEqual( - [kv(Bar1,1), kv(Foo2,2)], - remove_ancestors([kv(Bar1,1), kv(Foo1,1), kv(Foo2,1)], []) + [kv(Bar1, 1), kv(Foo2, 2)], + remove_ancestors([kv(Bar1, 1), kv(Foo1, 1), kv(Foo2, 1)], []) ), ?assertEqual( - [kv(Bar1,2)], - remove_ancestors([kv(Bar2,1), kv(Bar1,1)], []) + [kv(Bar1, 2)], + remove_ancestors([kv(Bar2, 1), kv(Bar1, 1)], []) ). is_replicator_db(DbName) -> path_ends_with(DbName, <<"_replicator">>). is_users_db(DbName) -> - ConfigName = list_to_binary(config:get( - "chttpd_auth", "authentication_db", "_users")), + ConfigName = list_to_binary( + config:get( + "chttpd_auth", "authentication_db", "_users" + ) + ), DbName == ConfigName orelse path_ends_with(DbName, <<"_users">>). path_ends_with(Path, Suffix) -> @@ -269,79 +305,54 @@ open_cluster_db(#shard{dbname = DbName, opts = Options}) -> end. open_cluster_db(DbName, Opts) -> - {SecProps} = fabric:get_security(DbName), % as admin + % as admin + {SecProps} = fabric:get_security(DbName), UserCtx = couch_util:get_value(user_ctx, Opts, #user_ctx{}), {ok, Db} = couch_db:clustered_db(DbName, UserCtx, SecProps), Db. %% test function kv(Item, Count) -> - {make_key(Item), {Item,Count}}. + {make_key(Item), {Item, Count}}. -doc_id_and_rev(#doc{id=DocId, revs={RevNum, [RevHash|_]}}) -> +doc_id_and_rev(#doc{id = DocId, revs = {RevNum, [RevHash | _]}}) -> {DocId, {RevNum, RevHash}}. - is_partitioned(DbName0) when is_binary(DbName0) -> Shards = mem3:shards(fabric:dbname(DbName0)), is_partitioned(open_cluster_db(hd(Shards))); - is_partitioned(Db) -> couch_db:is_partitioned(Db). - validate_all_docs_args(DbName, Args) when is_binary(DbName) -> Shards = mem3:shards(fabric:dbname(DbName)), Db = open_cluster_db(hd(Shards)), validate_all_docs_args(Db, Args); - validate_all_docs_args(Db, Args) -> true = couch_db:is_clustered(Db), couch_mrview_util:validate_all_docs_args(Db, Args). - validate_args(DbName, DDoc, Args) when is_binary(DbName) -> Shards = mem3:shards(fabric:dbname(DbName)), Db = open_cluster_db(hd(Shards)), validate_args(Db, DDoc, Args); - validate_args(Db, DDoc, Args) -> true = couch_db:is_clustered(Db), couch_mrview_util:validate_args(Db, DDoc, Args). - upgrade_mrargs(#mrargs{} = Args) -> Args; - -upgrade_mrargs({mrargs, - ViewType, - Reduce, - PreflightFun, - StartKey, - StartKeyDocId, - EndKey, - EndKeyDocId, - Keys, - Direction, - Limit, - Skip, - GroupLevel, - Group, - Stale, - MultiGet, - InclusiveEnd, - IncludeDocs, - DocOptions, - UpdateSeq, - Conflicts, - Callback, - Sorted, - Extra}) -> - {Stable, Update} = case Stale of - ok -> {true, false}; - update_after -> {true, lazy}; - _ -> {false, true} - end, +upgrade_mrargs( + {mrargs, ViewType, Reduce, PreflightFun, StartKey, StartKeyDocId, EndKey, EndKeyDocId, Keys, + Direction, Limit, Skip, GroupLevel, Group, Stale, MultiGet, InclusiveEnd, IncludeDocs, + DocOptions, UpdateSeq, Conflicts, Callback, Sorted, Extra} +) -> + {Stable, Update} = + case Stale of + ok -> {true, false}; + update_after -> {true, lazy}; + _ -> {false, true} + end, #mrargs{ view_type = ViewType, reduce = Reduce, @@ -369,18 +380,19 @@ upgrade_mrargs({mrargs, extra = Extra }. - worker_ranges(Workers) -> - Ranges = fabric_dict:fold(fun(#shard{range=[X, Y]}, _, Acc) -> - [{X, Y} | Acc] - end, [], Workers), + Ranges = fabric_dict:fold( + fun(#shard{range = [X, Y]}, _, Acc) -> + [{X, Y} | Acc] + end, + [], + Workers + ), lists:usort(Ranges). - get_uuid_prefix_len() -> config:get_integer("fabric", "uuid_prefix_len", 7). - % If we issue multiple fabric calls from the same process we have to isolate % them so in case of error they don't pollute the processes dictionary or the % mailbox @@ -388,7 +400,6 @@ get_uuid_prefix_len() -> isolate(Fun) -> isolate(Fun, infinity). - isolate(Fun, Timeout) -> {Pid, Ref} = erlang:spawn_monitor(fun() -> exit(do_isolate(Fun)) end), receive @@ -402,22 +413,19 @@ isolate(Fun, Timeout) -> erlang:error(timeout) end. - % OTP_RELEASE is defined in OTP 21+ only -ifdef(OTP_RELEASE). - do_isolate(Fun) -> try {'$isolres', Fun()} - catch Tag:Reason:Stack -> - {'$isolerr', Tag, Reason, Stack} + catch + Tag:Reason:Stack -> + {'$isolerr', Tag, Reason, Stack} end. - -else. - do_isolate(Fun) -> try {'$isolres', Fun()} @@ -425,10 +433,8 @@ do_isolate(Fun) -> {'$isolerr', Tag, Reason, Stack} end. - -endif. - get_db_timeout_test() -> % Q=1, N=1 ?assertEqual(20000, get_db_timeout(1, 2, 100, 60000)), diff --git a/src/fabric/src/fabric_view.erl b/src/fabric/src/fabric_view.erl index 3048e8987..c2ef13392 100644 --- a/src/fabric/src/fabric_view.erl +++ b/src/fabric/src/fabric_view.erl @@ -12,10 +12,18 @@ -module(fabric_view). --export([remove_overlapping_shards/2, maybe_send_row/1, - transform_row/1, keydict/1, extract_view/4, get_shards/2, - check_down_shards/2, handle_worker_exit/3, - get_shard_replacements/2, maybe_update_others/5]). +-export([ + remove_overlapping_shards/2, + maybe_send_row/1, + transform_row/1, + keydict/1, + extract_view/4, + get_shards/2, + check_down_shards/2, + handle_worker_exit/3, + get_shard_replacements/2, + maybe_update_others/5 +]). -export([fix_skip_and_limit/1]). -include_lib("fabric/include/fabric.hrl"). @@ -27,7 +35,7 @@ -spec check_down_shards(#collector{}, node()) -> {ok, #collector{}} | {error, any()}. check_down_shards(Collector, BadNode) -> - #collector{callback=Callback, counters=Counters, user_acc=Acc} = Collector, + #collector{callback = Callback, counters = Counters, user_acc = Acc} = Collector, Filter = fun(#shard{node = Node}, _) -> Node == BadNode end, BadCounters = fabric_dict:filter(Filter, Counters), case fabric_dict:size(BadCounters) > 0 of @@ -42,24 +50,21 @@ check_down_shards(Collector, BadNode) -> %% @doc Handle a worker that dies during a stream -spec handle_worker_exit(#collector{}, #shard{}, any()) -> {error, any()}. handle_worker_exit(Collector, _Worker, Reason) -> - #collector{callback=Callback, user_acc=Acc} = Collector, + #collector{callback = Callback, user_acc = Acc} = Collector, {ok, Resp} = Callback({error, fabric_util:error_info(Reason)}, Acc), {error, Resp}. - -spec remove_overlapping_shards(#shard{}, [{#shard{}, any()}]) -> [{#shard{}, any()}]. remove_overlapping_shards(#shard{} = Shard, Counters) -> remove_overlapping_shards(Shard, Counters, fun stop_worker/1). - -spec remove_overlapping_shards(#shard{}, [{#shard{}, any()}], fun()) -> [{#shard{}, any()}]. remove_overlapping_shards(#shard{} = Shard, Counters, RemoveCb) -> Counters1 = filter_exact_copies(Shard, Counters, RemoveCb), filter_possible_overlaps(Shard, Counters1, RemoveCb). - filter_possible_overlaps(Shard, Counters, RemoveCb) -> Ranges0 = fabric_util:worker_ranges(Counters), #shard{range = [BShard, EShard]} = Shard, @@ -83,54 +88,59 @@ filter_possible_overlaps(Shard, Counters, RemoveCb) -> B1 =< B2 end, Ring = mem3_util:get_ring(Ranges, SortFun, MinB, MaxE), - fabric_dict:filter(fun - (S, _) when S =:= Shard -> - % Keep the original shard - true; - (#shard{range = [B, E]} = S, _) -> - case lists:member({B, E}, Ring) of - true -> - true; % Keep it - false -> - % Duplicate range, delete after calling callback function - case is_function(RemoveCb) of - true -> RemoveCb(S); - false -> ok - end, - false - end - end, Counters). - + fabric_dict:filter( + fun + (S, _) when S =:= Shard -> + % Keep the original shard + true; + (#shard{range = [B, E]} = S, _) -> + case lists:member({B, E}, Ring) of + true -> + % Keep it + true; + false -> + % Duplicate range, delete after calling callback function + case is_function(RemoveCb) of + true -> RemoveCb(S); + false -> ok + end, + false + end + end, + Counters + ). filter_exact_copies(#shard{range = Range0} = Shard0, Shards, Cb) -> - fabric_dict:filter(fun - (Shard, _) when Shard =:= Shard0 -> - true; % Don't remove ourselves - (#shard{range = Range} = Shard, _) when Range =:= Range0 -> - case is_function(Cb) of - true -> Cb(Shard); - false -> ok - end, - false; - (_, _) -> - true - end, Shards). - + fabric_dict:filter( + fun + (Shard, _) when Shard =:= Shard0 -> + % Don't remove ourselves + true; + (#shard{range = Range} = Shard, _) when Range =:= Range0 -> + case is_function(Cb) of + true -> Cb(Shard); + false -> ok + end, + false; + (_, _) -> + true + end, + Shards + ). stop_worker(#shard{ref = Ref, node = Node}) -> rexi:kill(Node, Ref). - -maybe_send_row(#collector{limit=0} = State) -> - #collector{counters=Counters, user_acc=AccIn, callback=Callback} = State, +maybe_send_row(#collector{limit = 0} = State) -> + #collector{counters = Counters, user_acc = AccIn, callback = Callback} = State, case fabric_dict:any(0, Counters) of - true -> - % we still need to send the total/offset header - {ok, State}; - false -> - erase(meta_sent), - {_, Acc} = Callback(complete, AccIn), - {stop, State#collector{user_acc=Acc}} + true -> + % we still need to send the total/offset header + {ok, State}; + false -> + erase(meta_sent), + {_, Acc} = Callback(complete, AccIn), + {stop, State#collector{user_acc = Acc}} end; maybe_send_row(State) -> #collector{ @@ -141,82 +151,95 @@ maybe_send_row(State) -> user_acc = AccIn } = State, case fabric_dict:any(0, Counters) of - true -> - {ok, State}; - false -> - try get_next_row(State) of - {_, NewState} when Skip > 0 -> - maybe_send_row(NewState#collector{skip=Skip-1}); - {Row0, NewState} -> - Row1 = possibly_embed_doc(NewState, Row0), - Row2 = detach_partition(Row1), - Row3 = transform_row(Row2), - case Callback(Row3, AccIn) of - {stop, Acc} -> - {stop, NewState#collector{user_acc=Acc, limit=Limit-1}}; - {ok, Acc} -> - maybe_send_row(NewState#collector{user_acc=Acc, limit=Limit-1}) + true -> + {ok, State}; + false -> + try get_next_row(State) of + {_, NewState} when Skip > 0 -> + maybe_send_row(NewState#collector{skip = Skip - 1}); + {Row0, NewState} -> + Row1 = possibly_embed_doc(NewState, Row0), + Row2 = detach_partition(Row1), + Row3 = transform_row(Row2), + case Callback(Row3, AccIn) of + {stop, Acc} -> + {stop, NewState#collector{user_acc = Acc, limit = Limit - 1}}; + {ok, Acc} -> + maybe_send_row(NewState#collector{user_acc = Acc, limit = Limit - 1}) + end + catch + complete -> + erase(meta_sent), + {_, Acc} = Callback(complete, AccIn), + {stop, State#collector{user_acc = Acc}} end - catch complete -> - erase(meta_sent), - {_, Acc} = Callback(complete, AccIn), - {stop, State#collector{user_acc=Acc}} - end end. %% if include_docs=true is used when keys and %% the values contain "_id" then use the "_id"s %% to retrieve documents and embed in result -possibly_embed_doc(_State, - #view_row{id=reduced}=Row) -> +possibly_embed_doc( + _State, + #view_row{id = reduced} = Row +) -> Row; -possibly_embed_doc(_State, - #view_row{value=undefined}=Row) -> +possibly_embed_doc( + _State, + #view_row{value = undefined} = Row +) -> Row; -possibly_embed_doc(#collector{db_name=DbName, query_args=Args}, - #view_row{key=_Key, id=_Id, value=Value, doc=_Doc}=Row) -> - #mrargs{include_docs=IncludeDocs} = Args, +possibly_embed_doc( + #collector{db_name = DbName, query_args = Args}, + #view_row{key = _Key, id = _Id, value = Value, doc = _Doc} = Row +) -> + #mrargs{include_docs = IncludeDocs} = Args, case IncludeDocs andalso is_tuple(Value) of - true -> - {Props} = Value, - Rev0 = couch_util:get_value(<<"_rev">>, Props), - case couch_util:get_value(<<"_id">>,Props) of - null -> Row#view_row{doc=null}; - undefined -> Row; - IncId -> - % use separate process to call fabric:open_doc - % to not interfere with current call - {Pid, Ref} = spawn_monitor(fun() -> - exit( - case Rev0 of + true -> + {Props} = Value, + Rev0 = couch_util:get_value(<<"_rev">>, Props), + case couch_util:get_value(<<"_id">>, Props) of + null -> + Row#view_row{doc = null}; undefined -> - case fabric:open_doc(DbName, IncId, []) of - {ok, NewDoc} -> - Row#view_row{doc=couch_doc:to_json_obj(NewDoc,[])}; - {not_found, _} -> - Row#view_row{doc=null}; - Else -> - Row#view_row{doc={error, Else}} - end; - Rev0 -> - Rev = couch_doc:parse_rev(Rev0), - case fabric:open_revs(DbName, IncId, [Rev], []) of - {ok, [{ok, NewDoc}]} -> - Row#view_row{doc=couch_doc:to_json_obj(NewDoc,[])}; - {ok, [{{not_found, _}, Rev}]} -> - Row#view_row{doc=null}; - Else -> - Row#view_row{doc={error, Else}} + Row; + IncId -> + % use separate process to call fabric:open_doc + % to not interfere with current call + {Pid, Ref} = spawn_monitor(fun() -> + exit( + case Rev0 of + undefined -> + case fabric:open_doc(DbName, IncId, []) of + {ok, NewDoc} -> + Row#view_row{doc = couch_doc:to_json_obj(NewDoc, [])}; + {not_found, _} -> + Row#view_row{doc = null}; + Else -> + Row#view_row{doc = {error, Else}} + end; + Rev0 -> + Rev = couch_doc:parse_rev(Rev0), + case fabric:open_revs(DbName, IncId, [Rev], []) of + {ok, [{ok, NewDoc}]} -> + Row#view_row{doc = couch_doc:to_json_obj(NewDoc, [])}; + {ok, [{{not_found, _}, Rev}]} -> + Row#view_row{doc = null}; + Else -> + Row#view_row{doc = {error, Else}} + end + end + ) + end), + receive + {'DOWN', Ref, process, Pid, Resp} -> + Resp end - end) end), - receive {'DOWN',Ref,process,Pid, Resp} -> - Resp - end - end; - _ -> Row + end; + _ -> + Row end. -detach_partition(#view_row{key={p, _Partition, Key}} = Row) -> +detach_partition(#view_row{key = {p, _Partition, Key}} = Row) -> Row#view_row{key = Key}; detach_partition(#view_row{} = Row) -> Row. @@ -224,8 +247,11 @@ detach_partition(#view_row{} = Row) -> keydict(undefined) -> undefined; keydict(Keys) -> - {Dict,_} = lists:foldl(fun(K, {D,I}) -> {dict:store(K,I,D), I+1} end, - {dict:new(),0}, Keys), + {Dict, _} = lists:foldl( + fun(K, {D, I}) -> {dict:store(K, I, D), I + 1} end, + {dict:new(), 0}, + Keys + ), Dict. %% internal %% @@ -243,30 +269,34 @@ get_next_row(#collector{reducer = RedSrc} = St) when RedSrc =/= undefined -> } = St, {Key, RestKeys} = find_next_key(Keys, Dir, Collation, RowDict), case reduce_row_dict_take(Key, RowDict, Collation) of - {Records, NewRowDict} -> - Counters = lists:foldl(fun(#view_row{worker={Worker,From}}, CntrsAcc) -> - case From of - {Pid, _} when is_pid(Pid) -> - gen_server:reply(From, ok); - Pid when is_pid(Pid) -> - rexi:stream_ack(From) - end, - fabric_dict:update_counter(Worker, -1, CntrsAcc) - end, Counters0, Records), - Wrapped = [[V] || #view_row{value=V} <- Records], - {ok, [Reduced]} = couch_query_servers:rereduce(Lang, [RedSrc], Wrapped), - {ok, Finalized} = couch_query_servers:finalize(RedSrc, Reduced), - NewSt = St#collector{keys=RestKeys, rows=NewRowDict, counters=Counters}, - {#view_row{key=Key, id=reduced, value=Finalized}, NewSt}; - error -> - get_next_row(St#collector{keys=RestKeys}) + {Records, NewRowDict} -> + Counters = lists:foldl( + fun(#view_row{worker = {Worker, From}}, CntrsAcc) -> + case From of + {Pid, _} when is_pid(Pid) -> + gen_server:reply(From, ok); + Pid when is_pid(Pid) -> + rexi:stream_ack(From) + end, + fabric_dict:update_counter(Worker, -1, CntrsAcc) + end, + Counters0, + Records + ), + Wrapped = [[V] || #view_row{value = V} <- Records], + {ok, [Reduced]} = couch_query_servers:rereduce(Lang, [RedSrc], Wrapped), + {ok, Finalized} = couch_query_servers:finalize(RedSrc, Reduced), + NewSt = St#collector{keys = RestKeys, rows = NewRowDict, counters = Counters}, + {#view_row{key = Key, id = reduced, value = Finalized}, NewSt}; + error -> + get_next_row(St#collector{keys = RestKeys}) end; get_next_row(State) -> - #collector{rows = [Row|Rest], counters = Counters0} = State, + #collector{rows = [Row | Rest], counters = Counters0} = State, {Worker, From} = Row#view_row.worker, rexi:stream_ack(From), Counters1 = fabric_dict:update_counter(Worker, -1, Counters0), - {Row, State#collector{rows = Rest, counters=Counters1}}. + {Row, State#collector{rows = Rest, counters = Counters1}}. reduce_row_dict_take(Key, Dict, <<"raw">>) -> dict:take(Key, Dict); @@ -278,9 +308,13 @@ reduce_row_dict_take(Key, Dict, _Collation) -> error; [_ | _] -> {Keys, Vals} = lists:unzip(KVs), - NewDict = lists:foldl(fun(K, Acc) -> - dict:erase(K, Acc) - end, Dict, Keys), + NewDict = lists:foldl( + fun(K, Acc) -> + dict:erase(K, Acc) + end, + Dict, + Keys + ), {lists:flatten(Vals), NewDict} end. @@ -290,28 +324,28 @@ find_next_key(nil, Dir, Collation, RowDict) -> find_next_key(undefined, Dir, Collation, RowDict) -> CmpFun = fun(A, B) -> compare(Dir, Collation, A, B) end, case lists:sort(CmpFun, dict:fetch_keys(RowDict)) of - [] -> - throw(complete); - [Key|_] -> - {Key, nil} + [] -> + throw(complete); + [Key | _] -> + {Key, nil} end; find_next_key([], _, _, _) -> throw(complete); -find_next_key([Key|Rest], _, _, _) -> +find_next_key([Key | Rest], _, _, _) -> {Key, Rest}. -transform_row(#view_row{value={[{reduce_overflow_error, Msg}]}}) -> - {row, [{key,null}, {id,error}, {value,reduce_overflow_error}, {reason,Msg}]}; -transform_row(#view_row{key=Key, id=reduced, value=Value}) -> - {row, [{key,Key}, {value,Value}]}; -transform_row(#view_row{key=Key, id=undefined}) -> - {row, [{key,Key}, {id,error}, {value,not_found}]}; -transform_row(#view_row{key=Key, id=Id, value=Value, doc=undefined}) -> - {row, [{id,Id}, {key,Key}, {value,Value}]}; -transform_row(#view_row{key=Key, id=_Id, value=_Value, doc={error,Reason}}) -> - {row, [{id,error}, {key,Key}, {value,Reason}]}; -transform_row(#view_row{key=Key, id=Id, value=Value, doc=Doc}) -> - {row, [{id,Id}, {key,Key}, {value,Value}, {doc,Doc}]}. +transform_row(#view_row{value = {[{reduce_overflow_error, Msg}]}}) -> + {row, [{key, null}, {id, error}, {value, reduce_overflow_error}, {reason, Msg}]}; +transform_row(#view_row{key = Key, id = reduced, value = Value}) -> + {row, [{key, Key}, {value, Value}]}; +transform_row(#view_row{key = Key, id = undefined}) -> + {row, [{key, Key}, {id, error}, {value, not_found}]}; +transform_row(#view_row{key = Key, id = Id, value = Value, doc = undefined}) -> + {row, [{id, Id}, {key, Key}, {value, Value}]}; +transform_row(#view_row{key = Key, id = _Id, value = _Value, doc = {error, Reason}}) -> + {row, [{id, error}, {key, Key}, {value, Reason}]}; +transform_row(#view_row{key = Key, id = Id, value = Value, doc = Doc}) -> + {row, [{id, Id}, {key, Key}, {value, Value}, {doc, Doc}]}. compare(fwd, <<"raw">>, A, B) -> A < B; compare(rev, <<"raw">>, A, B) -> B < A; @@ -322,16 +356,17 @@ extract_view(Pid, ViewName, [], _ViewType) -> couch_log:error("missing_named_view ~p", [ViewName]), exit(Pid, kill), exit(missing_named_view); -extract_view(Pid, ViewName, [View|Rest], ViewType) -> +extract_view(Pid, ViewName, [View | Rest], ViewType) -> case lists:member(ViewName, view_names(View, ViewType)) of - true -> - if ViewType == reduce -> - {index_of(ViewName, view_names(View, reduce)), View}; true -> - View - end; - false -> - extract_view(Pid, ViewName, Rest, ViewType) + if + ViewType == reduce -> + {index_of(ViewName, view_names(View, reduce)), View}; + true -> + View + end; + false -> + extract_view(Pid, ViewName, Rest, ViewType) end. view_names(View, Type) when Type == red_map; Type == reduce -> @@ -344,16 +379,17 @@ index_of(X, List) -> index_of(_X, [], _I) -> not_found; -index_of(X, [X|_Rest], I) -> +index_of(X, [X | _Rest], I) -> I; -index_of(X, [_|Rest], I) -> - index_of(X, Rest, I+1). +index_of(X, [_ | Rest], I) -> + index_of(X, Rest, I + 1). get_shards(Db, #mrargs{} = Args) -> DbPartitioned = fabric_util:is_partitioned(Db), Partition = couch_mrview_util:get_extra(Args, partition), - if DbPartitioned orelse Partition == undefined -> ok; true -> - throw({bad_request, <<"partition specified on non-partitioned db">>}) + if + DbPartitioned orelse Partition == undefined -> ok; + true -> throw({bad_request, <<"partition specified on non-partitioned db">>}) end, DbName = fabric:dbname(Db), % Decide which version of mem3:shards/1,2 or @@ -372,12 +408,20 @@ get_shards(Db, #mrargs{} = Args) -> {Shards, [{any, Shards}]} end. -maybe_update_others(DbName, DDoc, ShardsInvolved, ViewName, - #mrargs{update=lazy} = Args) -> +maybe_update_others( + DbName, + DDoc, + ShardsInvolved, + ViewName, + #mrargs{update = lazy} = Args +) -> ShardsNeedUpdated = mem3:shards(DbName) -- ShardsInvolved, - lists:foreach(fun(#shard{node=Node, name=ShardName}) -> - rpc:cast(Node, fabric_rpc, update_mrview, [ShardName, DDoc, ViewName, Args]) - end, ShardsNeedUpdated); + lists:foreach( + fun(#shard{node = Node, name = ShardName}) -> + rpc:cast(Node, fabric_rpc, update_mrview, [ShardName, DDoc, ViewName, Args]) + end, + ShardsNeedUpdated + ); maybe_update_others(_DbName, _DDoc, _ShardsInvolved, _ViewName, _Args) -> ok. @@ -385,48 +429,57 @@ get_shard_replacements(DbName, UsedShards0) -> % We only want to generate a replacements list from shards % that aren't already used. AllLiveShards = mem3:live_shards(DbName, [node() | nodes()]), - UsedShards = [S#shard{ref=undefined} || S <- UsedShards0], + UsedShards = [S#shard{ref = undefined} || S <- UsedShards0], get_shard_replacements_int(AllLiveShards -- UsedShards, UsedShards). get_shard_replacements_int(UnusedShards, UsedShards) -> % If we have more than one copy of a range then we don't % want to try and add a replacement to any copy. - RangeCounts = lists:foldl(fun(#shard{range=R}, Acc) -> - dict:update_counter(R, 1, Acc) - end, dict:new(), UsedShards), + RangeCounts = lists:foldl( + fun(#shard{range = R}, Acc) -> + dict:update_counter(R, 1, Acc) + end, + dict:new(), + UsedShards + ), % For each seq shard range with a count of 1, find any % possible replacements from the unused shards. The % replacement list is keyed by range. - lists:foldl(fun(#shard{range = [B, E] = Range}, Acc) -> - case dict:find(Range, RangeCounts) of - {ok, 1} -> - Repls = mem3_util:non_overlapping_shards(UnusedShards, B, E), - % Only keep non-empty lists of replacements - if Repls == [] -> Acc; true -> - [{Range, Repls} | Acc] - end; - _ -> - Acc - end - end, [], UsedShards). - --spec fix_skip_and_limit(#mrargs{}) -> {CoordArgs::#mrargs{}, WorkerArgs::#mrargs{}}. + lists:foldl( + fun(#shard{range = [B, E] = Range}, Acc) -> + case dict:find(Range, RangeCounts) of + {ok, 1} -> + Repls = mem3_util:non_overlapping_shards(UnusedShards, B, E), + % Only keep non-empty lists of replacements + if + Repls == [] -> Acc; + true -> [{Range, Repls} | Acc] + end; + _ -> + Acc + end + end, + [], + UsedShards + ). + +-spec fix_skip_and_limit(#mrargs{}) -> {CoordArgs :: #mrargs{}, WorkerArgs :: #mrargs{}}. fix_skip_and_limit(#mrargs{} = Args) -> - {CoordArgs, WorkerArgs} = case couch_mrview_util:get_extra(Args, partition) of - undefined -> - #mrargs{skip=Skip, limit=Limit}=Args, - {Args, Args#mrargs{skip=0, limit=Skip+Limit}}; - _Partition -> - {Args#mrargs{skip=0}, Args} - end, + {CoordArgs, WorkerArgs} = + case couch_mrview_util:get_extra(Args, partition) of + undefined -> + #mrargs{skip = Skip, limit = Limit} = Args, + {Args, Args#mrargs{skip = 0, limit = Skip + Limit}}; + _Partition -> + {Args#mrargs{skip = 0}, Args} + end, %% the coordinator needs to finalize each row, so make sure the shards don't {CoordArgs, remove_finalizer(WorkerArgs)}. remove_finalizer(Args) -> couch_mrview_util:set_extra(Args, finalizer, null). - remove_overlapping_shards_test() -> Cb = undefined, @@ -436,29 +489,42 @@ remove_overlapping_shards_test() -> Shard1 = mk_shard("node-3", [11, 20]), Shards1 = fabric_dict:store(Shard1, nil, Shards), R1 = remove_overlapping_shards(Shard1, Shards1, Cb), - ?assertEqual([{0, 10}, {11, 20}, {21, ?RING_END}], - fabric_util:worker_ranges(R1)), + ?assertEqual( + [{0, 10}, {11, 20}, {21, ?RING_END}], + fabric_util:worker_ranges(R1) + ), ?assert(fabric_dict:is_key(Shard1, R1)), % Split overlap (shard overlap multiple workers) Shard2 = mk_shard("node-3", [0, 20]), Shards2 = fabric_dict:store(Shard2, nil, Shards), R2 = remove_overlapping_shards(Shard2, Shards2, Cb), - ?assertEqual([{0, 20}, {21, ?RING_END}], - fabric_util:worker_ranges(R2)), + ?assertEqual( + [{0, 20}, {21, ?RING_END}], + fabric_util:worker_ranges(R2) + ), ?assert(fabric_dict:is_key(Shard2, R2)). - get_shard_replacements_test() -> - Unused = [mk_shard(N, [B, E]) || {N, B, E} <- [ - {"n1", 11, 20}, {"n1", 21, ?RING_END}, - {"n2", 0, 4}, {"n2", 5, 10}, {"n2", 11, 20}, - {"n3", 0, 21, ?RING_END} - ]], - Used = [mk_shard(N, [B, E]) || {N, B, E} <- [ - {"n2", 21, ?RING_END}, - {"n3", 0, 10}, {"n3", 11, 20} - ]], + Unused = [ + mk_shard(N, [B, E]) + || {N, B, E} <- [ + {"n1", 11, 20}, + {"n1", 21, ?RING_END}, + {"n2", 0, 4}, + {"n2", 5, 10}, + {"n2", 11, 20}, + {"n3", 0, 21, ?RING_END} + ] + ], + Used = [ + mk_shard(N, [B, E]) + || {N, B, E} <- [ + {"n2", 21, ?RING_END}, + {"n3", 0, 10}, + {"n3", 11, 20} + ] + ], Res = lists:sort(get_shard_replacements_int(Unused, Used)), % Notice that [0, 10] range can be replaced by spawning the [0, 4] and [5, % 10] workers on n1 @@ -469,22 +535,25 @@ get_shard_replacements_test() -> ], ?assertEqual(Expect, Res). - mk_cnts(Ranges, NoNodes) -> - orddict:from_list([{Shard,nil} - || Shard <- - lists:flatten(lists:map( - fun(Range) -> - mk_shards(NoNodes,Range,[]) - end, Ranges))] - ). - -mk_shards(0,_Range,Shards) -> - Shards; -mk_shards(NoNodes,Range,Shards) -> - Name ="node-" ++ integer_to_list(NoNodes), - mk_shards(NoNodes-1,Range, [mk_shard(Name, Range) | Shards]). + orddict:from_list([ + {Shard, nil} + || Shard <- + lists:flatten( + lists:map( + fun(Range) -> + mk_shards(NoNodes, Range, []) + end, + Ranges + ) + ) + ]). +mk_shards(0, _Range, Shards) -> + Shards; +mk_shards(NoNodes, Range, Shards) -> + Name = "node-" ++ integer_to_list(NoNodes), + mk_shards(NoNodes - 1, Range, [mk_shard(Name, Range) | Shards]). mk_shard(Name, Range) -> Node = list_to_atom(Name), diff --git a/src/fabric/src/fabric_view_all_docs.erl b/src/fabric/src/fabric_view_all_docs.erl index 1b1aa8a9c..4bd8e9522 100644 --- a/src/fabric/src/fabric_view_all_docs.erl +++ b/src/fabric/src/fabric_view_all_docs.erl @@ -13,19 +13,21 @@ -module(fabric_view_all_docs). -export([go/5]). --export([open_doc/4]). % exported for spawn +% exported for spawn +-export([open_doc/4]). -include_lib("fabric/include/fabric.hrl"). -include_lib("mem3/include/mem3.hrl"). -include_lib("couch/include/couch_db.hrl"). -include_lib("couch_mrview/include/couch_mrview.hrl"). -go(Db, Options, #mrargs{keys=undefined} = QueryArgs, Callback, Acc) -> +go(Db, Options, #mrargs{keys = undefined} = QueryArgs, Callback, Acc) -> {CoordArgs, WorkerArgs} = fabric_view:fix_skip_and_limit(QueryArgs), DbName = fabric:dbname(Db), {Shards, RingOpts} = shards(Db, QueryArgs), Workers0 = fabric_util:submit_jobs( - Shards, fabric_rpc, all_docs, [Options, WorkerArgs]), + Shards, fabric_rpc, all_docs, [Options, WorkerArgs] + ), RexiMon = fabric_util:create_monitors(Workers0), try case fabric_streams:start(Workers0, #shard.ref, RingOpts) of @@ -50,8 +52,6 @@ go(Db, Options, #mrargs{keys=undefined} = QueryArgs, Callback, Acc) -> after rexi_monitor:stop(RexiMon) end; - - go(DbName, Options, QueryArgs, Callback, Acc0) -> #mrargs{ direction = Dir, @@ -64,45 +64,51 @@ go(DbName, Options, QueryArgs, Callback, Acc0) -> extra = Extra, update_seq = UpdateSeq } = QueryArgs, - DocOptions1 = case Conflicts of - true -> [conflicts|DocOptions0]; - _ -> DocOptions0 - end, + DocOptions1 = + case Conflicts of + true -> [conflicts | DocOptions0]; + _ -> DocOptions0 + end, SpawnFun = fun(Key) -> spawn_monitor(?MODULE, open_doc, [DbName, Options ++ DocOptions1, Key, IncludeDocs]) end, MaxJobs = all_docs_concurrency(), - Keys1 = case Dir of - fwd -> Keys0; - _ -> lists:reverse(Keys0) - end, - Keys2 = case Skip < length(Keys1) of - true -> lists:nthtail(Skip, Keys1); - false -> [] - end, - Keys3 = case Limit < length(Keys2) of - true -> lists:sublist(Keys2, Limit); - false -> Keys2 - end, + Keys1 = + case Dir of + fwd -> Keys0; + _ -> lists:reverse(Keys0) + end, + Keys2 = + case Skip < length(Keys1) of + true -> lists:nthtail(Skip, Keys1); + false -> [] + end, + Keys3 = + case Limit < length(Keys2) of + true -> lists:sublist(Keys2, Limit); + false -> Keys2 + end, %% namespace can be _set_ to `undefined`, so we want simulate enum here - Namespace = case couch_util:get_value(namespace, Extra) of - <<"_all_docs">> -> <<"_all_docs">>; - <<"_design">> -> <<"_design">>; - <<"_local">> -> <<"_local">>; - _ -> <<"_all_docs">> - end, + Namespace = + case couch_util:get_value(namespace, Extra) of + <<"_all_docs">> -> <<"_all_docs">>; + <<"_design">> -> <<"_design">>; + <<"_local">> -> <<"_local">>; + _ -> <<"_all_docs">> + end, Timeout = fabric_util:all_docs_timeout(), {_, Ref} = spawn_monitor(fun() -> exit(fabric:get_doc_count(DbName, Namespace)) end), receive {'DOWN', Ref, _, _, {ok, TotalRows}} -> - Meta = case UpdateSeq of - false -> - [{total, TotalRows}, {offset, null}]; - true -> - [{total, TotalRows}, {offset, null}, {update_seq, null}] - end, + Meta = + case UpdateSeq of + false -> + [{total, TotalRows}, {offset, null}]; + true -> + [{total, TotalRows}, {offset, null}, {update_seq, null}] + end, {ok, Acc1} = Callback({meta, Meta}, Acc0), Resp = doc_receive_loop( Keys3, queue:new(), SpawnFun, MaxJobs, Callback, Acc1 @@ -129,50 +135,60 @@ go(DbName, _Options, Workers, QueryArgs, Callback, Acc0) -> skip = Skip, limit = Limit, user_acc = Acc0, - update_seq = case UpdateSeq of true -> []; false -> nil end + update_seq = + case UpdateSeq of + true -> []; + false -> nil + end }, - case rexi_utils:recv(Workers, #shard.ref, fun handle_message/3, - State, fabric_util:view_timeout(QueryArgs), 5000) of - {ok, NewState} -> - {ok, NewState#collector.user_acc}; - {timeout, NewState} -> - Callback({error, timeout}, NewState#collector.user_acc); - {error, Resp} -> - {ok, Resp} + case + rexi_utils:recv( + Workers, + #shard.ref, + fun handle_message/3, + State, + fabric_util:view_timeout(QueryArgs), + 5000 + ) + of + {ok, NewState} -> + {ok, NewState#collector.user_acc}; + {timeout, NewState} -> + Callback({error, timeout}, NewState#collector.user_acc); + {error, Resp} -> + {ok, Resp} end. shards(Db, Args) -> DbPartitioned = fabric_util:is_partitioned(Db), Partition = couch_mrview_util:get_extra(Args, partition), - NewArgs = case {DbPartitioned, Partition} of - {true, undefined} -> - % If a user specifies the same partition on both - % the start and end keys we can optimize the - % query by limiting to the partition shard. - Start = couch_partition:extract(Args#mrargs.start_key), - End = couch_partition:extract(Args#mrargs.end_key), - case {Start, End} of - {{Partition, SK}, {Partition, EK}} -> - A1 = Args#mrargs{ - start_key = SK, - end_key = EK - }, - couch_mrview_util:set_extra(A1, partition, Partition); - _ -> - Args - end; - _ -> - Args - end, + NewArgs = + case {DbPartitioned, Partition} of + {true, undefined} -> + % If a user specifies the same partition on both + % the start and end keys we can optimize the + % query by limiting to the partition shard. + Start = couch_partition:extract(Args#mrargs.start_key), + End = couch_partition:extract(Args#mrargs.end_key), + case {Start, End} of + {{Partition, SK}, {Partition, EK}} -> + A1 = Args#mrargs{ + start_key = SK, + end_key = EK + }, + couch_mrview_util:set_extra(A1, partition, Partition); + _ -> + Args + end; + _ -> + Args + end, fabric_view:get_shards(Db, NewArgs). - handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _, State) -> fabric_view:check_down_shards(State, NodeRef); - handle_message({rexi_EXIT, Reason}, Worker, State) -> fabric_view:handle_worker_exit(State, Worker, Reason); - handle_message({meta, Meta0}, {Worker, From}, State) -> Tot = couch_util:get_value(total, Meta0, 0), Off = couch_util:get_value(offset, Meta0, 0), @@ -190,62 +206,70 @@ handle_message({meta, Meta0}, {Worker, From}, State) -> 0 = fabric_dict:lookup_element(Worker, Counters0), rexi:stream_ack(From), Counters1 = fabric_dict:update_counter(Worker, 1, Counters0), - Total = if Tot == null -> null; true -> Total0 + Tot end, - Offset = if Off == null -> null; true -> Offset0 + Off end, - UpdateSeq = case {UpdateSeq0, Seq} of - {nil, _} -> nil; - {_, null} -> null; - _ -> [{Worker, Seq} | UpdateSeq0] - end, - case fabric_dict:any(0, Counters1) of - true -> - {ok, State#collector{ - counters = Counters1, - total_rows = Total, - update_seq = UpdateSeq, - offset = Offset - }}; - false -> - FinalOffset = case Offset of - null -> null; - _ -> erlang:min(Total, Offset+State#collector.skip) + Total = + if + Tot == null -> null; + true -> Total0 + Tot end, - Meta = [{total, Total}, {offset, FinalOffset}] ++ - case UpdateSeq of - nil -> - []; - null -> - [{update_seq, null}]; - _ -> - [{update_seq, fabric_view_changes:pack_seqs(UpdateSeq)}] - end, - {Go, Acc} = Callback({meta, Meta}, AccIn), - {Go, State#collector{ - counters = fabric_dict:decrement_all(Counters1), - total_rows = Total, - offset = FinalOffset, - user_acc = Acc, - update_seq = UpdateSeq0 - }} + Offset = + if + Off == null -> null; + true -> Offset0 + Off + end, + UpdateSeq = + case {UpdateSeq0, Seq} of + {nil, _} -> nil; + {_, null} -> null; + _ -> [{Worker, Seq} | UpdateSeq0] + end, + case fabric_dict:any(0, Counters1) of + true -> + {ok, State#collector{ + counters = Counters1, + total_rows = Total, + update_seq = UpdateSeq, + offset = Offset + }}; + false -> + FinalOffset = + case Offset of + null -> null; + _ -> erlang:min(Total, Offset + State#collector.skip) + end, + Meta = + [{total, Total}, {offset, FinalOffset}] ++ + case UpdateSeq of + nil -> + []; + null -> + [{update_seq, null}]; + _ -> + [{update_seq, fabric_view_changes:pack_seqs(UpdateSeq)}] + end, + {Go, Acc} = Callback({meta, Meta}, AccIn), + {Go, State#collector{ + counters = fabric_dict:decrement_all(Counters1), + total_rows = Total, + offset = FinalOffset, + user_acc = Acc, + update_seq = UpdateSeq0 + }} end; - handle_message(#view_row{} = Row, {Worker, From}, State) -> #collector{query_args = Args, counters = Counters0, rows = Rows0} = State, Dir = Args#mrargs.direction, - Rows = merge_row(Dir, Row#view_row{worker={Worker, From}}, Rows0), + Rows = merge_row(Dir, Row#view_row{worker = {Worker, From}}, Rows0), Counters1 = fabric_dict:update_counter(Worker, 1, Counters0), - State1 = State#collector{rows=Rows, counters=Counters1}, + State1 = State#collector{rows = Rows, counters = Counters1}, fabric_view:maybe_send_row(State1); - handle_message(complete, Worker, State) -> Counters = fabric_dict:update_counter(Worker, 1, State#collector.counters), fabric_view:maybe_send_row(State#collector{counters = Counters}); - -handle_message({execution_stats, _} = Msg, {_,From}, St) -> - #collector{callback=Callback, user_acc=AccIn} = St, +handle_message({execution_stats, _} = Msg, {_, From}, St) -> + #collector{callback = Callback, user_acc = AccIn} = St, {Go, Acc} = Callback(Msg, AccIn), rexi:stream_ack(From), - {Go, St#collector{user_acc=Acc}}. + {Go, St#collector{user_acc = Acc}}. merge_row(fwd, Row, Rows) -> lists:keymerge(#view_row.id, [Row], Rows); @@ -256,42 +280,43 @@ all_docs_concurrency() -> Value = config:get("fabric", "all_docs_concurrency", "10"), try list_to_integer(Value) - catch _:_ -> - 10 + catch + _:_ -> + 10 end. doc_receive_loop(Keys, Pids, SpawnFun, MaxJobs, Callback, AccIn) -> case {Keys, queue:len(Pids)} of - {[], 0} -> - {ok, AccIn}; - {[K | RKeys], Len} when Len < MaxJobs -> - Pids1 = queue:in(SpawnFun(K), Pids), - doc_receive_loop(RKeys, Pids1, SpawnFun, MaxJobs, Callback, AccIn); - _ -> - {{value, {Pid, Ref}}, RestPids} = queue:out(Pids), - Timeout = fabric_util:all_docs_timeout(), - receive {'DOWN', Ref, process, Pid, Row} -> - case Row of - #view_row{} -> - case Callback(fabric_view:transform_row(Row), AccIn) of - {ok, Acc} -> - doc_receive_loop( - Keys, RestPids, SpawnFun, MaxJobs, Callback, Acc - ); - {stop, Acc} -> - cancel_read_pids(RestPids), - {ok, Acc} - end; - Error -> - cancel_read_pids(RestPids), - Callback({error, Error}, AccIn) + {[], 0} -> + {ok, AccIn}; + {[K | RKeys], Len} when Len < MaxJobs -> + Pids1 = queue:in(SpawnFun(K), Pids), + doc_receive_loop(RKeys, Pids1, SpawnFun, MaxJobs, Callback, AccIn); + _ -> + {{value, {Pid, Ref}}, RestPids} = queue:out(Pids), + Timeout = fabric_util:all_docs_timeout(), + receive + {'DOWN', Ref, process, Pid, Row} -> + case Row of + #view_row{} -> + case Callback(fabric_view:transform_row(Row), AccIn) of + {ok, Acc} -> + doc_receive_loop( + Keys, RestPids, SpawnFun, MaxJobs, Callback, Acc + ); + {stop, Acc} -> + cancel_read_pids(RestPids), + {ok, Acc} + end; + Error -> + cancel_read_pids(RestPids), + Callback({error, Error}, AccIn) + end + after Timeout -> + timeout end - after Timeout -> - timeout - end end. - open_doc(DbName, Options, Id, IncludeDocs) -> try open_doc_int(DbName, Options, Id, IncludeDocs) of #view_row{} = Row -> @@ -303,22 +328,26 @@ open_doc(DbName, Options, Id, IncludeDocs) -> end. open_doc_int(DbName, Options, Id, IncludeDocs) -> - Row = case fabric:open_doc(DbName, Id, [deleted | Options]) of - {not_found, missing} -> - Doc = undefined, - #view_row{key=Id}; - {ok, #doc{deleted=true, revs=Revs}} -> - Doc = null, - {RevPos, [RevId|_]} = Revs, - Value = {[{rev,couch_doc:rev_to_str({RevPos, RevId})}, {deleted,true}]}, - #view_row{key=Id, id=Id, value=Value}; - {ok, #doc{revs=Revs} = Doc0} -> - Doc = couch_doc:to_json_obj(Doc0, Options), - {RevPos, [RevId|_]} = Revs, - Value = {[{rev,couch_doc:rev_to_str({RevPos, RevId})}]}, - #view_row{key=Id, id=Id, value=Value} - end, - if IncludeDocs -> Row#view_row{doc=Doc}; true -> Row end. + Row = + case fabric:open_doc(DbName, Id, [deleted | Options]) of + {not_found, missing} -> + Doc = undefined, + #view_row{key = Id}; + {ok, #doc{deleted = true, revs = Revs}} -> + Doc = null, + {RevPos, [RevId | _]} = Revs, + Value = {[{rev, couch_doc:rev_to_str({RevPos, RevId})}, {deleted, true}]}, + #view_row{key = Id, id = Id, value = Value}; + {ok, #doc{revs = Revs} = Doc0} -> + Doc = couch_doc:to_json_obj(Doc0, Options), + {RevPos, [RevId | _]} = Revs, + Value = {[{rev, couch_doc:rev_to_str({RevPos, RevId})}]}, + #view_row{key = Id, id = Id, value = Value} + end, + if + IncludeDocs -> Row#view_row{doc = Doc}; + true -> Row + end. cancel_read_pids(Pids) -> case queue:out(Pids) of diff --git a/src/fabric/src/fabric_view_changes.erl b/src/fabric/src/fabric_view_changes.erl index 9fdbf06df..b561da151 100644 --- a/src/fabric/src/fabric_view_changes.erl +++ b/src/fabric/src/fabric_view_changes.erl @@ -28,60 +28,66 @@ -import(fabric_db_update_listener, [wait_db_updated/1]). -go(DbName, Feed, Options, Callback, Acc0) when Feed == "continuous" orelse - Feed == "longpoll" orelse Feed == "eventsource" -> +go(DbName, Feed, Options, Callback, Acc0) when + Feed == "continuous" orelse + Feed == "longpoll" orelse Feed == "eventsource" +-> Args = make_changes_args(Options), Since = get_start_seq(DbName, Args), case validate_start_seq(DbName, Since) of - ok -> - {ok, Acc} = Callback(start, Acc0), - {Timeout, _} = couch_changes:get_changes_timeout(Args, Callback), - Ref = make_ref(), - Parent = self(), - UpdateListener = {spawn_link(fabric_db_update_listener, go, - [Parent, Ref, DbName, Timeout]), - Ref}, - put(changes_epoch, get_changes_epoch()), - try - keep_sending_changes( - DbName, - Args, - Callback, - Since, - Acc, - Timeout, - UpdateListener, - os:timestamp() - ) - after - fabric_db_update_listener:stop(UpdateListener) - end; - Error -> - Callback(Error, Acc0) + ok -> + {ok, Acc} = Callback(start, Acc0), + {Timeout, _} = couch_changes:get_changes_timeout(Args, Callback), + Ref = make_ref(), + Parent = self(), + UpdateListener = { + spawn_link( + fabric_db_update_listener, + go, + [Parent, Ref, DbName, Timeout] + ), + Ref + }, + put(changes_epoch, get_changes_epoch()), + try + keep_sending_changes( + DbName, + Args, + Callback, + Since, + Acc, + Timeout, + UpdateListener, + os:timestamp() + ) + after + fabric_db_update_listener:stop(UpdateListener) + end; + Error -> + Callback(Error, Acc0) end; - go(DbName, "normal", Options, Callback, Acc0) -> Args = make_changes_args(Options), Since = get_start_seq(DbName, Args), case validate_start_seq(DbName, Since) of - ok -> - {ok, Acc} = Callback(start, Acc0), - {ok, Collector} = send_changes( - DbName, - Args, - Callback, - Since, - Acc, - 5000 - ), - #collector{counters=Seqs, user_acc=AccOut, offset=Offset} = Collector, - Callback({stop, pack_seqs(Seqs), pending_count(Offset)}, AccOut); - Error -> - Callback(Error, Acc0) + ok -> + {ok, Acc} = Callback(start, Acc0), + {ok, Collector} = send_changes( + DbName, + Args, + Callback, + Since, + Acc, + 5000 + ), + #collector{counters = Seqs, user_acc = AccOut, offset = Offset} = Collector, + Callback({stop, pack_seqs(Seqs), pending_count(Offset)}, AccOut); + Error -> + Callback(Error, Acc0) end. keep_sending_changes(DbName, Args, Callback, Seqs, AccIn, Timeout, UpListen, T0) -> - #changes_args{limit=Limit, feed=Feed, heartbeat=Heartbeat} = Args, + #changes_args{limit = Limit, feed = Feed, heartbeat = Heartbeat} = Args, {ok, Collector} = send_changes(DbName, Args, Callback, Seqs, AccIn, Timeout), #collector{ limit = Limit2, @@ -92,39 +98,43 @@ keep_sending_changes(DbName, Args, Callback, Seqs, AccIn, Timeout, UpListen, T0) LastSeq = pack_seqs(NewSeqs), MaintenanceMode = config:get("couchdb", "maintenance_mode"), NewEpoch = get_changes_epoch() > erlang:get(changes_epoch), - if Limit > Limit2, Feed == "longpoll"; - MaintenanceMode == "true"; MaintenanceMode == "nolb"; NewEpoch -> - Callback({stop, LastSeq, pending_count(Offset)}, AccOut0); - true -> - {ok, AccOut} = Callback(waiting_for_updates, AccOut0), - WaitForUpdate = wait_db_updated(UpListen), - AccumulatedTime = timer:now_diff(os:timestamp(), T0) div 1000, - Max = case config:get("fabric", "changes_duration") of - undefined -> - infinity; - MaxStr -> - list_to_integer(MaxStr) - end, - case {Heartbeat, AccumulatedTime > Max, WaitForUpdate} of - {_, _, changes_feed_died} -> - Callback({stop, LastSeq, pending_count(Offset)}, AccOut); - {undefined, _, timeout} -> - Callback({stop, LastSeq, pending_count(Offset)}, AccOut); - {_, true, timeout} -> - Callback({stop, LastSeq, pending_count(Offset)}, AccOut); - _ -> - {ok, AccTimeout} = Callback(timeout, AccOut), - ?MODULE:keep_sending_changes( - DbName, - Args#changes_args{limit=Limit2}, - Callback, - LastSeq, - AccTimeout, - Timeout, - UpListen, - T0 - ) - end + if + Limit > Limit2, Feed == "longpoll"; + MaintenanceMode == "true"; + MaintenanceMode == "nolb"; + NewEpoch -> + Callback({stop, LastSeq, pending_count(Offset)}, AccOut0); + true -> + {ok, AccOut} = Callback(waiting_for_updates, AccOut0), + WaitForUpdate = wait_db_updated(UpListen), + AccumulatedTime = timer:now_diff(os:timestamp(), T0) div 1000, + Max = + case config:get("fabric", "changes_duration") of + undefined -> + infinity; + MaxStr -> + list_to_integer(MaxStr) + end, + case {Heartbeat, AccumulatedTime > Max, WaitForUpdate} of + {_, _, changes_feed_died} -> + Callback({stop, LastSeq, pending_count(Offset)}, AccOut); + {undefined, _, timeout} -> + Callback({stop, LastSeq, pending_count(Offset)}, AccOut); + {_, true, timeout} -> + Callback({stop, LastSeq, pending_count(Offset)}, AccOut); + _ -> + {ok, AccTimeout} = Callback(timeout, AccOut), + ?MODULE:keep_sending_changes( + DbName, + Args#changes_args{limit = Limit2}, + Callback, + LastSeq, + AccTimeout, + Timeout, + UpListen, + T0 + ) + end end. send_changes(DbName, ChangesArgs, Callback, PackedSeqs, AccIn, Timeout) -> @@ -133,35 +143,46 @@ send_changes(DbName, ChangesArgs, Callback, PackedSeqs, AccIn, Timeout) -> Seqs0 = unpack_seqs(PackedSeqs, DbName), {WSeqs0, Dead, Reps} = find_replacements(Seqs0, AllLiveShards), % Start workers which didn't need replacements - WSeqs = lists:map(fun({#shard{name = Name, node = N} = S, Seq}) -> - Ref = rexi:cast(N, {fabric_rpc, changes, [Name, ChangesArgs, Seq]}), - {S#shard{ref = Ref}, Seq} - end, WSeqs0), + WSeqs = lists:map( + fun({#shard{name = Name, node = N} = S, Seq}) -> + Ref = rexi:cast(N, {fabric_rpc, changes, [Name, ChangesArgs, Seq]}), + {S#shard{ref = Ref}, Seq} + end, + WSeqs0 + ), % For some dead workers see if they are a result of split shards. In that % case make a replacement argument so that local rexi workers can calculate % (hopefully) a > 0 update sequence. {WSplitSeqs0, Reps1} = find_split_shard_replacements(Dead, Reps), - WSplitSeqs = lists:map(fun({#shard{name = Name, node = N} = S, Seq}) -> - Arg = make_replacement_arg(N, Seq), - Ref = rexi:cast(N, {fabric_rpc, changes, [Name, ChangesArgs, Arg]}), - {S#shard{ref = Ref}, Seq} - end, WSplitSeqs0), + WSplitSeqs = lists:map( + fun({#shard{name = Name, node = N} = S, Seq}) -> + Arg = make_replacement_arg(N, Seq), + Ref = rexi:cast(N, {fabric_rpc, changes, [Name, ChangesArgs, Arg]}), + {S#shard{ref = Ref}, Seq} + end, + WSplitSeqs0 + ), % For ranges that were not split, look for a replacement on a different node - WReps = lists:map(fun(#shard{name = Name, node = NewNode, range = R} = S) -> - Arg = find_replacement_sequence(Dead, R), - case Arg =/= 0 of true -> ok; false -> - couch_log:warning("~p reset seq for ~p", [?MODULE, S]) - end, - Ref = rexi:cast(NewNode, {fabric_rpc, changes, [Name, ChangesArgs, Arg]}), - {S#shard{ref = Ref}, 0} - end, Reps1), + WReps = lists:map( + fun(#shard{name = Name, node = NewNode, range = R} = S) -> + Arg = find_replacement_sequence(Dead, R), + case Arg =/= 0 of + true -> ok; + false -> couch_log:warning("~p reset seq for ~p", [?MODULE, S]) + end, + Ref = rexi:cast(NewNode, {fabric_rpc, changes, [Name, ChangesArgs, Arg]}), + {S#shard{ref = Ref}, 0} + end, + Reps1 + ), Seqs = WSeqs ++ WSplitSeqs ++ WReps, {Workers0, _} = lists:unzip(Seqs), Repls = fabric_ring:get_shard_replacements(DbName, Workers0), - StartFun = fun(#shard{name=Name, node=N, range=R0}=Shard) -> + StartFun = fun(#shard{name = Name, node = N, range = R0} = Shard) -> SeqArg = find_replacement_sequence(Seqs, R0), - case SeqArg =/= 0 of true -> ok; false -> - couch_log:warning("~p StartFun reset seq for ~p", [?MODULE, Shard]) + case SeqArg =/= 0 of + true -> ok; + false -> couch_log:warning("~p StartFun reset seq for ~p", [?MODULE, Shard]) end, Ref = rexi:cast(N, {fabric_rpc, changes, [Name, ChangesArgs, SeqArg]}), Shard#shard{ref = Ref} @@ -171,14 +192,24 @@ send_changes(DbName, ChangesArgs, Callback, PackedSeqs, AccIn, Timeout) -> case fabric_streams:start(Workers0, #shard.ref, StartFun, Repls) of {ok, Workers} -> try - LiveSeqs = lists:map(fun(W) -> - case lists:keyfind(W, 1, Seqs) of - {W, Seq} -> {W, Seq}; - _ -> {W, 0} - end - end, Workers), - send_changes(DbName, Workers, LiveSeqs, ChangesArgs, - Callback, AccIn, Timeout) + LiveSeqs = lists:map( + fun(W) -> + case lists:keyfind(W, 1, Seqs) of + {W, Seq} -> {W, Seq}; + _ -> {W, 0} + end + end, + Workers + ), + send_changes( + DbName, + Workers, + LiveSeqs, + ChangesArgs, + Callback, + AccIn, + Timeout + ) after fabric_streams:cleanup(Workers) end; @@ -210,77 +241,84 @@ send_changes(DbName, Workers, Seqs, ChangesArgs, Callback, AccIn, Timeout) -> user_acc = AccIn, limit = ChangesArgs#changes_args.limit, offset = fabric_dict:init(Workers, null), - rows = Seqs % store sequence positions instead + % store sequence positions instead + rows = Seqs }, %% TODO: errors need to be handled here receive_results(Workers, State, Timeout, Callback). receive_results(Workers, State, Timeout, Callback) -> - case rexi_utils:recv(Workers, #shard.ref, fun handle_message/3, State, - Timeout, infinity) of - {timeout, NewState0} -> - {ok, AccOut} = Callback(timeout, NewState0#collector.user_acc), - NewState = NewState0#collector{user_acc = AccOut}, - receive_results(Workers, NewState, Timeout, Callback); - {_, NewState} -> - {ok, NewState} + case + rexi_utils:recv( + Workers, + #shard.ref, + fun handle_message/3, + State, + Timeout, + infinity + ) + of + {timeout, NewState0} -> + {ok, AccOut} = Callback(timeout, NewState0#collector.user_acc), + NewState = NewState0#collector{user_acc = AccOut}, + receive_results(Workers, NewState, Timeout, Callback); + {_, NewState} -> + {ok, NewState} end. handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _, State) -> fabric_view:check_down_shards(State, NodeRef); - handle_message({rexi_EXIT, Reason}, Worker, State) -> fabric_view:handle_worker_exit(State, Worker, Reason); - % Temporary upgrade clause - Case 24236 handle_message({complete, Key}, Worker, State) when is_tuple(Key) -> handle_message({complete, [{seq, Key}, {pending, 0}]}, Worker, State); - -handle_message({change, Props}, {Worker, _}, #collector{limit=0} = State) -> +handle_message({change, Props}, {Worker, _}, #collector{limit = 0} = State) -> O0 = State#collector.offset, - O1 = case fabric_dict:lookup_element(Worker, O0) of - null -> - % Use Pending+1 because we're ignoring this row in the response - Pending = couch_util:get_value(pending, Props, 0), - fabric_dict:store(Worker, Pending+1, O0); - _ -> - O0 - end, + O1 = + case fabric_dict:lookup_element(Worker, O0) of + null -> + % Use Pending+1 because we're ignoring this row in the response + Pending = couch_util:get_value(pending, Props, 0), + fabric_dict:store(Worker, Pending + 1, O0); + _ -> + O0 + end, maybe_stop(State#collector{offset = O1}); - -handle_message({complete, Props}, Worker, #collector{limit=0} = State) -> +handle_message({complete, Props}, Worker, #collector{limit = 0} = State) -> O0 = State#collector.offset, - O1 = case fabric_dict:lookup_element(Worker, O0) of - null -> - fabric_dict:store(Worker, couch_util:get_value(pending,Props), O0); - _ -> - O0 - end, + O1 = + case fabric_dict:lookup_element(Worker, O0) of + null -> + fabric_dict:store(Worker, couch_util:get_value(pending, Props), O0); + _ -> + O0 + end, maybe_stop(State#collector{offset = O1}); - -handle_message({no_pass, Props}, {Worker, From}, #collector{limit=0} = State) - when is_list(Props) -> +handle_message({no_pass, Props}, {Worker, From}, #collector{limit = 0} = State) when + is_list(Props) +-> #collector{counters = S0, offset = O0} = State, - O1 = case fabric_dict:lookup_element(Worker, O0) of - null -> - fabric_dict:store(Worker, couch_util:get_value(pending, Props), O0); - _ -> - O0 - end, + O1 = + case fabric_dict:lookup_element(Worker, O0) of + null -> + fabric_dict:store(Worker, couch_util:get_value(pending, Props), O0); + _ -> + O0 + end, S1 = fabric_dict:store(Worker, couch_util:get_value(seq, Props), S0), rexi:stream_ack(From), maybe_stop(State#collector{counters = S1, offset = O1}); - handle_message(#change{} = Row, {Worker, From}, St) -> - Change = {change, [ - {seq, Row#change.key}, - {id, Row#change.id}, - {changes, Row#change.value}, - {deleted, Row#change.deleted}, - {doc, Row#change.doc} - ]}, + Change = + {change, [ + {seq, Row#change.key}, + {id, Row#change.id}, + {changes, Row#change.value}, + {deleted, Row#change.deleted}, + {doc, Row#change.doc} + ]}, handle_message(Change, {Worker, From}, St); - handle_message({change, Props}, {Worker, From}, St) -> #collector{ callback = Callback, @@ -294,51 +332,50 @@ handle_message({change, Props}, {Worker, From}, St) -> O1 = fabric_dict:store(Worker, couch_util:get_value(pending, Props), O0), % Temporary hack for FB 23637 Interval = erlang:get(changes_seq_interval), - if (Interval == undefined) orelse (Limit rem Interval == 0) -> - Props2 = lists:keyreplace(seq, 1, Props, {seq, pack_seqs(S1)}); - true -> - Props2 = lists:keyreplace(seq, 1, Props, {seq, null}) + if + (Interval == undefined) orelse (Limit rem Interval == 0) -> + Props2 = lists:keyreplace(seq, 1, Props, {seq, pack_seqs(S1)}); + true -> + Props2 = lists:keyreplace(seq, 1, Props, {seq, null}) end, {Go, Acc} = Callback(changes_row(Props2), AccIn), rexi:stream_ack(From), - {Go, St#collector{counters=S1, offset=O1, limit=Limit-1, user_acc=Acc}}; - + {Go, St#collector{counters = S1, offset = O1, limit = Limit - 1, user_acc = Acc}}; %% upgrade clause handle_message({no_pass, Seq}, From, St) when is_integer(Seq) -> handle_message({no_pass, [{seq, Seq}]}, From, St); - handle_message({no_pass, Props}, {Worker, From}, St) -> Seq = couch_util:get_value(seq, Props), #collector{counters = S0} = St, true = fabric_dict:is_key(Worker, S0), S1 = fabric_dict:store(Worker, Seq, S0), rexi:stream_ack(From), - {ok, St#collector{counters=S1}}; - + {ok, St#collector{counters = S1}}; handle_message({complete, Props}, Worker, State) -> Key = couch_util:get_value(seq, Props), #collector{ counters = S0, offset = O0, - total_rows = Completed % override + % override + total_rows = Completed } = State, true = fabric_dict:is_key(Worker, S0), S1 = fabric_dict:store(Worker, Key, S0), O1 = fabric_dict:store(Worker, couch_util:get_value(pending, Props), O0), - NewState = State#collector{counters=S1, offset=O1, total_rows=Completed+1}, + NewState = State#collector{counters = S1, offset = O1, total_rows = Completed + 1}, % We're relying on S1 having exactly the numnber of workers that % are participtaing in this response. With the new stream_start % that's a bit more obvious but historically it wasn't quite % so clear. The Completed variable is just a hacky override % of the total_rows field in the #collector{} record. NumWorkers = fabric_dict:size(S1), - Go = case NumWorkers =:= (Completed+1) of - true -> stop; - false -> ok - end, + Go = + case NumWorkers =:= (Completed + 1) of + true -> stop; + false -> ok + end, {Go, NewState}. - make_replacement_arg(Node, {Seq, Uuid}) -> {replace, Node, Uuid, Seq}; make_replacement_arg(_Node, {Seq, Uuid, EpochNode}) -> @@ -358,36 +395,40 @@ maybe_stop(#collector{offset = Offset} = State) -> {ok, State} end. -make_changes_args(#changes_args{style=Style, filter_fun=undefined}=Args) -> +make_changes_args(#changes_args{style = Style, filter_fun = undefined} = Args) -> Args#changes_args{filter_fun = {default, Style}}; make_changes_args(Args) -> Args. -get_start_seq(DbName, #changes_args{dir=Dir, since=Since}) - when Dir == rev; Since == "now" -> +get_start_seq(DbName, #changes_args{dir = Dir, since = Since}) when + Dir == rev; Since == "now" +-> {ok, Info} = fabric:get_db_info(DbName), couch_util:get_value(update_seq, Info); -get_start_seq(_DbName, #changes_args{dir=fwd, since=Since}) -> +get_start_seq(_DbName, #changes_args{dir = fwd, since = Since}) -> Since. pending_count(Dict) -> - fabric_dict:fold(fun - (_Worker, Count, Acc) when is_integer(Count), is_integer(Acc) -> - Count + Acc; - (_Worker, _Count, _Acc) -> - null - end, 0, Dict). + fabric_dict:fold( + fun + (_Worker, Count, Acc) when is_integer(Count), is_integer(Acc) -> + Count + Acc; + (_Worker, _Count, _Acc) -> + null + end, + 0, + Dict + ). pack_seqs(Workers) -> - SeqList = [{N,R,S} || {#shard{node=N, range=R}, S} <- Workers], - SeqSum = lists:sum([seq(S) || {_,_,S} <- SeqList]), + SeqList = [{N, R, S} || {#shard{node = N, range = R}, S} <- Workers], + SeqSum = lists:sum([seq(S) || {_, _, S} <- SeqList]), Opaque = couch_util:encodeBase64Url(term_to_binary(SeqList, [compressed])), ?l2b([integer_to_list(SeqSum), $-, Opaque]). seq({Seq, _Uuid, _Node}) -> Seq; seq({Seq, _Uuid}) -> Seq; -seq(Seq) -> Seq. - +seq(Seq) -> Seq. unpack_seq_regex_match(Packed) -> NewPattern = "^\\[[0-9]+\s*,\s*\"(?<opaque>.*)\"\\]$", @@ -401,11 +442,9 @@ unpack_seq_regex_match(Packed) -> Match end. - unpack_seq_decode_term(Opaque) -> binary_to_term(couch_util:decodeBase64Url(Opaque)). - % This is used for testing and for remsh debugging % % Return the unpacked list of sequences from a raw update seq string. The input @@ -417,20 +456,17 @@ decode_seq(Packed) -> Opaque = unpack_seq_regex_match(Packed), unpack_seq_decode_term(Opaque). - % Returns fabric_dict with {Shard, Seq} entries % -spec unpack_seqs(pos_integer() | list() | binary(), binary()) -> - orddict:orddict(). + orddict:orddict(). unpack_seqs(0, DbName) -> fabric_dict:init(mem3:shards(DbName), 0); - unpack_seqs("0", DbName) -> fabric_dict:init(mem3:shards(DbName), 0); - -unpack_seqs([_SeqNum, Opaque], DbName) -> % deprecated +% deprecated +unpack_seqs([_SeqNum, Opaque], DbName) -> do_unpack_seqs(Opaque, DbName); - unpack_seqs(Packed, DbName) -> Opaque = unpack_seq_regex_match(Packed), do_unpack_seqs(Opaque, DbName). @@ -440,23 +476,33 @@ do_unpack_seqs(Opaque, DbName) -> % This just picks each unique shard and keeps the largest seq % value recorded. Decoded = unpack_seq_decode_term(Opaque), - DedupDict = lists:foldl(fun({Node, [A, B], Seq}, Acc) -> - dict:append({Node, [A, B]}, Seq, Acc) - end, dict:new(), Decoded), - Deduped = lists:map(fun({{Node, [A, B]}, SeqList}) -> - {Node, [A, B], lists:max(SeqList)} - end, dict:to_list(DedupDict)), + DedupDict = lists:foldl( + fun({Node, [A, B], Seq}, Acc) -> + dict:append({Node, [A, B]}, Seq, Acc) + end, + dict:new(), + Decoded + ), + Deduped = lists:map( + fun({{Node, [A, B]}, SeqList}) -> + {Node, [A, B], lists:max(SeqList)} + end, + dict:to_list(DedupDict) + ), % Create a fabric_dict of {Shard, Seq} entries % TODO relies on internal structure of fabric_dict as keylist - Unpacked = lists:flatmap(fun({Node, [A,B], Seq}) -> - case mem3:get_shard(DbName, Node, [A,B]) of - {ok, Shard} -> - [{Shard, Seq}]; - {error, not_found} -> - [] - end - end, Deduped), + Unpacked = lists:flatmap( + fun({Node, [A, B], Seq}) -> + case mem3:get_shard(DbName, Node, [A, B]) of + {ok, Shard} -> + [{Shard, Seq}]; + {error, not_found} -> + [] + end + end, + Deduped + ), % This just handles the case if the ring in the unpacked sequence % received is not complete and in that case tries to fill in the @@ -466,32 +512,40 @@ do_unpack_seqs(Opaque, DbName) -> Unpacked; false -> Uuids = get_db_uuid_shards(DbName), - PotentialWorkers = lists:map(fun({Node, [A, B], Seq}) -> - case mem3:get_shard(DbName, Node, [A, B]) of - {ok, Shard} -> - {Shard, Seq}; - {error, not_found} -> - Shard = replace_moved_shard(Node, [A, B], Seq, Uuids), - {Shard, Seq} - end - end, Deduped), + PotentialWorkers = lists:map( + fun({Node, [A, B], Seq}) -> + case mem3:get_shard(DbName, Node, [A, B]) of + {ok, Shard} -> + {Shard, Seq}; + {error, not_found} -> + Shard = replace_moved_shard(Node, [A, B], Seq, Uuids), + {Shard, Seq} + end + end, + Deduped + ), Shards = mem3:shards(DbName), {Unpacked1, Dead, Reps} = find_replacements(PotentialWorkers, Shards), {Splits, Reps1} = find_split_shard_replacements(Dead, Reps), - RepSeqs = lists:map(fun(#shard{} = S) -> - {S, get_old_seq(S, Deduped)} - end, Reps1), + RepSeqs = lists:map( + fun(#shard{} = S) -> + {S, get_old_seq(S, Deduped)} + end, + Reps1 + ), Unpacked1 ++ Splits ++ RepSeqs end. - -get_old_seq(#shard{range=R}=Shard, SinceSeqs) -> +get_old_seq(#shard{range = R} = Shard, SinceSeqs) -> case lists:keyfind(R, 2, SinceSeqs) of {Node, R, Seq} when is_number(Seq) -> % Unfortunately we don't have access to the db % uuid so we can't set a replacememnt here. - couch_log:warning("~p get_old_seq missing uuid " - "node: ~p, range: ~p, seq: ~p", [?MODULE, Node, R, Seq]), + couch_log:warning( + "~p get_old_seq missing uuid " + "node: ~p, range: ~p, seq: ~p", + [?MODULE, Node, R, Seq] + ), 0; {Node, R, {Seq, Uuid}} -> % This update seq is using the old format that @@ -503,12 +557,13 @@ get_old_seq(#shard{range=R}=Shard, SinceSeqs) -> % can use for replacement. {Seq, Uuid, EpochNode}; Error -> - couch_log:warning("~p get_old_seq error: ~p, shard: ~p, seqs: ~p", - [?MODULE, Error, Shard, SinceSeqs]), + couch_log:warning( + "~p get_old_seq error: ~p, shard: ~p, seqs: ~p", + [?MODULE, Error, Shard, SinceSeqs] + ), 0 end. - get_db_uuid_shards(DbName) -> % Need to use an isolated process as we are performing a fabric call from % another fabric call and there is a good chance we'd polute the mailbox @@ -522,10 +577,14 @@ get_db_uuid_shards(DbName) -> % sequence from the same cluster and we didn't tweak that % relatively obscure config option in the meantime. PrefixLen = fabric_util:get_uuid_prefix_len(), - maps:fold(fun(Uuid, Shard, Acc) -> - TrimmedUuid = binary:part(Uuid, {0, PrefixLen}), - Acc#{TrimmedUuid => Shard} - end, #{}, Uuids); + maps:fold( + fun(Uuid, Shard, Acc) -> + TrimmedUuid = binary:part(Uuid, {0, PrefixLen}), + Acc#{TrimmedUuid => Shard} + end, + #{}, + Uuids + ); {error, Error} -> % Since we are doing a best-effort approach to match moved shards, % tolerate and log errors. This should also handle cases when the @@ -541,7 +600,6 @@ get_db_uuid_shards(DbName) -> #{} end. - %% Determine if the missing shard moved to a new node. Do that by matching the %% uuids from the current shard map. If we cannot find a moved shard, we return %% the original node and range as a shard and hope for the best. @@ -561,29 +619,36 @@ replace_moved_shard(Node, Range, {_Seq, Uuid, _EpochNode}, #{} = UuidShards) -> #shard{node = Node, range = Range} end. - changes_row(Props0) -> - Props1 = case couch_util:get_value(deleted, Props0) of - true -> - Props0; - _ -> - lists:keydelete(deleted, 1, Props0) - end, + Props1 = + case couch_util:get_value(deleted, Props0) of + true -> + Props0; + _ -> + lists:keydelete(deleted, 1, Props0) + end, Allowed = [seq, id, changes, deleted, doc, error], - Props2 = lists:filter(fun({K,_V}) -> lists:member(K, Allowed) end, Props1), + Props2 = lists:filter(fun({K, _V}) -> lists:member(K, Allowed) end, Props1), {change, {Props2}}. - find_replacements(Workers, AllShards) -> % Build map [B, E] => [Worker1, Worker2, ...] for all workers - WrkMap = lists:foldl(fun({#shard{range = [B, E]}, _} = W, Acc) -> - maps:update_with({B, E}, fun(Ws) -> [W | Ws] end, [W], Acc) - end, #{}, fabric_dict:to_list(Workers)), + WrkMap = lists:foldl( + fun({#shard{range = [B, E]}, _} = W, Acc) -> + maps:update_with({B, E}, fun(Ws) -> [W | Ws] end, [W], Acc) + end, + #{}, + fabric_dict:to_list(Workers) + ), % Build map [B, E] => [Shard1, Shard2, ...] for all shards - AllMap = lists:foldl(fun(#shard{range = [B, E]} = S, Acc) -> - maps:update_with({B, E}, fun(Ss) -> [S | Ss] end, [S], Acc) - end, #{}, AllShards), + AllMap = lists:foldl( + fun(#shard{range = [B, E]} = S, Acc) -> + maps:update_with({B, E}, fun(Ss) -> [S | Ss] end, [S], Acc) + end, + #{}, + AllShards + ), % Custom sort function will prioritize workers over other shards. % The idea is to not unnecessarily kill workers if we don't have to @@ -610,59 +675,69 @@ find_replacements(Workers, AllShards) -> % Keep only workers in the ring and from one of the available nodes Keep = fun(#shard{range = [B, E], node = N}) -> - lists:member({B, E}, Ring) andalso lists:keyfind(N, #shard.node, - maps:get({B, E}, AllMap)) =/= false + lists:member({B, E}, Ring) andalso + lists:keyfind( + N, + #shard.node, + maps:get({B, E}, AllMap) + ) =/= false end, Workers1 = fabric_dict:filter(fun(S, _) -> Keep(S) end, Workers), Removed = fabric_dict:filter(fun(S, _) -> not Keep(S) end, Workers), - {Rep, _} = lists:foldl(fun(R, {RepAcc, AllMapAcc}) -> - case maps:is_key(R, WrkMap)of - true -> - % It's a worker and in the map of available shards. Make sure - % to keep it only if there is a range available on that node - % only (reuse Keep/1 predicate from above) - WorkersInRange = maps:get(R, WrkMap), - case lists:any(fun({S, _}) -> Keep(S) end, WorkersInRange) of - true -> - {RepAcc, AllMapAcc}; - false -> - [Shard | Rest] = maps:get(R, AllMapAcc), - {[Shard | RepAcc], AllMapAcc#{R := Rest}} - end; - false -> - % No worker for this range. Replace from available shards - [Shard | Rest] = maps:get(R, AllMapAcc), - {[Shard | RepAcc], AllMapAcc#{R := Rest}} - end - end, {[], AllMap}, Ring), + {Rep, _} = lists:foldl( + fun(R, {RepAcc, AllMapAcc}) -> + case maps:is_key(R, WrkMap) of + true -> + % It's a worker and in the map of available shards. Make sure + % to keep it only if there is a range available on that node + % only (reuse Keep/1 predicate from above) + WorkersInRange = maps:get(R, WrkMap), + case lists:any(fun({S, _}) -> Keep(S) end, WorkersInRange) of + true -> + {RepAcc, AllMapAcc}; + false -> + [Shard | Rest] = maps:get(R, AllMapAcc), + {[Shard | RepAcc], AllMapAcc#{R := Rest}} + end; + false -> + % No worker for this range. Replace from available shards + [Shard | Rest] = maps:get(R, AllMapAcc), + {[Shard | RepAcc], AllMapAcc#{R := Rest}} + end + end, + {[], AllMap}, + Ring + ), % Return the list of workers that are part of ring, list of removed workers % and a list of replacement shards that could be used to make sure the ring % completes. {Workers1, Removed, Rep}. - % From the list of dead workers determine if any are a result of a split shard. % In that case perhaps there is a way to not rewind the changes feed back to 0. % Returns {NewWorkers, Available} where NewWorkers is the list of % viable workers Available is the list of still unused input Shards find_split_shard_replacements(DeadWorkers, Shards) -> Acc0 = {[], Shards}, - AccF = fabric_dict:fold(fun(#shard{node = WN, range = R}, Seq, Acc) -> - [B, E] = R, - {SplitWorkers, Available} = Acc, - ShardsOnSameNode = [S || #shard{node = N} = S <- Available, N =:= WN], - SplitShards = mem3_util:non_overlapping_shards(ShardsOnSameNode, B, E), - RepCount = length(SplitShards), - NewWorkers = [{S, make_split_seq(Seq, RepCount)} || S <- SplitShards], - NewAvailable = [S || S <- Available, not lists:member(S, SplitShards)], - {NewWorkers ++ SplitWorkers, NewAvailable} - end, Acc0, DeadWorkers), + AccF = fabric_dict:fold( + fun(#shard{node = WN, range = R}, Seq, Acc) -> + [B, E] = R, + {SplitWorkers, Available} = Acc, + ShardsOnSameNode = [S || #shard{node = N} = S <- Available, N =:= WN], + SplitShards = mem3_util:non_overlapping_shards(ShardsOnSameNode, B, E), + RepCount = length(SplitShards), + NewWorkers = [{S, make_split_seq(Seq, RepCount)} || S <- SplitShards], + NewAvailable = [S || S <- Available, not lists:member(S, SplitShards)], + {NewWorkers ++ SplitWorkers, NewAvailable} + end, + Acc0, + DeadWorkers + ), {Workers, Available} = AccF, {fabric_dict:from_list(Workers), Available}. - find_replacement_sequence(OriginalSeqs, R0) -> %% Find the original shard copy in the Seqs array case lists:dropwhile(fun({S, _}) -> S#shard.range =/= R0 end, OriginalSeqs) of @@ -678,13 +753,11 @@ find_replacement_sequence(OriginalSeqs, R0) -> 0 end. - make_split_seq({Num, Uuid, Node}, RepCount) when RepCount > 1 -> {Num, {split, Uuid}, Node}; make_split_seq(Seq, _) -> Seq. - validate_start_seq(_DbName, "now") -> ok; validate_start_seq(_DbName, 0) -> @@ -719,7 +792,6 @@ get_changes_epoch() -> increment_changes_epoch() -> application:set_env(fabric, changes_epoch, os:timestamp()). - unpack_seq_setup() -> meck:new(mem3), meck:new(fabric_view), @@ -727,62 +799,80 @@ unpack_seq_setup() -> meck:expect(fabric_ring, is_progress_possible, fun(_) -> true end), ok. - unpack_seqs_test_() -> { setup, fun unpack_seq_setup/0, - fun (_) -> meck:unload() end, + fun(_) -> meck:unload() end, [ t_unpack_seqs() ] }. - t_unpack_seqs() -> ?_test(begin % BigCouch 0.3 style. - assert_shards("23423-g1AAAAE7eJzLYWBg4MhgTmHgS0ktM3QwND" - "LXMwBCwxygOFMiQ5L8____sxIZcKlIUgCSSfZgRUw4FTmAFMWDFTHiVJQAUlSPX1Ee" - "C5BkaABSQHXzsxKZ8StcAFG4H4_bIAoPQBTeJ2j1A4hCUJBkAQC7U1NA"), + assert_shards( + "23423-g1AAAAE7eJzLYWBg4MhgTmHgS0ktM3QwND" + "LXMwBCwxygOFMiQ5L8____sxIZcKlIUgCSSfZgRUw4FTmAFMWDFTHiVJQAUlSPX1Ee" + "C5BkaABSQHXzsxKZ8StcAFG4H4_bIAoPQBTeJ2j1A4hCUJBkAQC7U1NA" + ), % BigCouch 0.4 style. - assert_shards([23423,<<"g1AAAAE7eJzLYWBg4MhgTmHgS0ktM3QwND" - "LXMwBCwxygOFMiQ5L8____sxIZcKlIUgCSSfZgRUw4FTmAFMWDFTHiVJQAUlSPX1Ee" - "C5BkaABSQHXzsxKZ8StcAFG4H4_bIAoPQBTeJ2j1A4hCUJBkAQC7U1NA">>]), + assert_shards([ + 23423, + << + "g1AAAAE7eJzLYWBg4MhgTmHgS0ktM3QwND" + "LXMwBCwxygOFMiQ5L8____sxIZcKlIUgCSSfZgRUw4FTmAFMWDFTHiVJQAUlSPX1Ee" + "C5BkaABSQHXzsxKZ8StcAFG4H4_bIAoPQBTeJ2j1A4hCUJBkAQC7U1NA" + >> + ]), % BigCouch 0.4 style (as string). - assert_shards("[23423,\"g1AAAAE7eJzLYWBg4MhgTmHgS0ktM3QwND" - "LXMwBCwxygOFMiQ5L8____sxIZcKlIUgCSSfZgRUw4FTmAFMWDFTHiVJQAUlSPX1Ee" - "C5BkaABSQHXzsxKZ8StcAFG4H4_bIAoPQBTeJ2j1A4hCUJBkAQC7U1NA\"]"), - assert_shards("[23423 ,\"g1AAAAE7eJzLYWBg4MhgTmHgS0ktM3QwND" - "LXMwBCwxygOFMiQ5L8____sxIZcKlIUgCSSfZgRUw4FTmAFMWDFTHiVJQAUlSPX1Ee" - "C5BkaABSQHXzsxKZ8StcAFG4H4_bIAoPQBTeJ2j1A4hCUJBkAQC7U1NA\"]"), - assert_shards("[23423, \"g1AAAAE7eJzLYWBg4MhgTmHgS0ktM3QwND" - "LXMwBCwxygOFMiQ5L8____sxIZcKlIUgCSSfZgRUw4FTmAFMWDFTHiVJQAUlSPX1Ee" - "C5BkaABSQHXzsxKZ8StcAFG4H4_bIAoPQBTeJ2j1A4hCUJBkAQC7U1NA\"]"), - assert_shards("[23423 , \"g1AAAAE7eJzLYWBg4MhgTmHgS0ktM3QwND" - "LXMwBCwxygOFMiQ5L8____sxIZcKlIUgCSSfZgRUw4FTmAFMWDFTHiVJQAUlSPX1Ee" - "C5BkaABSQHXzsxKZ8StcAFG4H4_bIAoPQBTeJ2j1A4hCUJBkAQC7U1NA\"]"), + assert_shards( + "[23423,\"g1AAAAE7eJzLYWBg4MhgTmHgS0ktM3QwND" + "LXMwBCwxygOFMiQ5L8____sxIZcKlIUgCSSfZgRUw4FTmAFMWDFTHiVJQAUlSPX1Ee" + "C5BkaABSQHXzsxKZ8StcAFG4H4_bIAoPQBTeJ2j1A4hCUJBkAQC7U1NA\"]" + ), + assert_shards( + "[23423 ,\"g1AAAAE7eJzLYWBg4MhgTmHgS0ktM3QwND" + "LXMwBCwxygOFMiQ5L8____sxIZcKlIUgCSSfZgRUw4FTmAFMWDFTHiVJQAUlSPX1Ee" + "C5BkaABSQHXzsxKZ8StcAFG4H4_bIAoPQBTeJ2j1A4hCUJBkAQC7U1NA\"]" + ), + assert_shards( + "[23423, \"g1AAAAE7eJzLYWBg4MhgTmHgS0ktM3QwND" + "LXMwBCwxygOFMiQ5L8____sxIZcKlIUgCSSfZgRUw4FTmAFMWDFTHiVJQAUlSPX1Ee" + "C5BkaABSQHXzsxKZ8StcAFG4H4_bIAoPQBTeJ2j1A4hCUJBkAQC7U1NA\"]" + ), + assert_shards( + "[23423 , \"g1AAAAE7eJzLYWBg4MhgTmHgS0ktM3QwND" + "LXMwBCwxygOFMiQ5L8____sxIZcKlIUgCSSfZgRUw4FTmAFMWDFTHiVJQAUlSPX1Ee" + "C5BkaABSQHXzsxKZ8StcAFG4H4_bIAoPQBTeJ2j1A4hCUJBkAQC7U1NA\"]" + ), % with internal hypen - assert_shards("651-g1AAAAE7eJzLYWBg4MhgTmHgS0ktM3QwNDLXMwBCwxygOFMiQ" - "5L8____sxJTcalIUgCSSfZgReE4FTmAFMWDFYXgVJQAUlQPVuSKS1EeC5BkaABSQHXz8" - "VgJUbgAonB_VqIPfoUHIArvE7T6AUQh0I1-WQAzp1XB"), - assert_shards([651,"g1AAAAE7eJzLYWBg4MhgTmHgS0ktM3QwNDLXMwBCwxygOFMiQ" - "5L8____sxJTcalIUgCSSfZgReE4FTmAFMWDFYXgVJQAUlQPVuSKS1EeC5BkaABSQHXz8" - "VgJUbgAonB_VqIPfoUHIArvE7T6AUQh0I1-WQAzp1XB"]), + assert_shards( + "651-g1AAAAE7eJzLYWBg4MhgTmHgS0ktM3QwNDLXMwBCwxygOFMiQ" + "5L8____sxJTcalIUgCSSfZgReE4FTmAFMWDFYXgVJQAUlQPVuSKS1EeC5BkaABSQHXz8" + "VgJUbgAonB_VqIPfoUHIArvE7T6AUQh0I1-WQAzp1XB" + ), + assert_shards([ + 651, + "g1AAAAE7eJzLYWBg4MhgTmHgS0ktM3QwNDLXMwBCwxygOFMiQ" + "5L8____sxJTcalIUgCSSfZgReE4FTmAFMWDFYXgVJQAUlQPVuSKS1EeC5BkaABSQHXz8" + "VgJUbgAonB_VqIPfoUHIArvE7T6AUQh0I1-WQAzp1XB" + ]), % CouchDB 1.2 style - assert_shards("\"23423-g1AAAAE7eJzLYWBg4MhgTmHgS0ktM3QwND" - "LXMwBCwxygOFMiQ5L8____sxIZcKlIUgCSSfZgRUw4FTmAFMWDFTHiVJQAUlSPX1Ee" - "C5BkaABSQHXzsxKZ8StcAFG4H4_bIAoPQBTeJ2j1A4hCUJBkAQC7U1NA\"") + assert_shards( + "\"23423-g1AAAAE7eJzLYWBg4MhgTmHgS0ktM3QwND" + "LXMwBCwxygOFMiQ5L8____sxIZcKlIUgCSSfZgRUw4FTmAFMWDFTHiVJQAUlSPX1Ee" + "C5BkaABSQHXzsxKZ8StcAFG4H4_bIAoPQBTeJ2j1A4hCUJBkAQC7U1NA\"" + ) end). - assert_shards(Packed) -> - ?assertMatch([{#shard{},_}|_], unpack_seqs(Packed, <<"foo">>)). - + ?assertMatch([{#shard{}, _} | _], unpack_seqs(Packed, <<"foo">>)). find_replacements_test() -> % None of the workers are in the live list of shard but there is a @@ -811,11 +901,14 @@ find_replacements_test() -> {WorkersRes2, Dead2, Reps2} = find_replacements(Workers2, AllShards2), ?assertEqual([], WorkersRes2), ?assertEqual(Workers2, Dead2), - ?assertEqual([ - mk_shard("n1", 11, ?RING_END), - mk_shard("n2", 0, 4), - mk_shard("n2", 5, 10) - ], lists:sort(Reps2)), + ?assertEqual( + [ + mk_shard("n1", 11, ?RING_END), + mk_shard("n2", 0, 4), + mk_shard("n2", 5, 10) + ], + lists:sort(Reps2) + ), % One worker is available and one needs to be replaced. Replacement will be % from two split shards @@ -829,10 +922,13 @@ find_replacements_test() -> {WorkersRes3, Dead3, Reps3} = find_replacements(Workers3, AllShards3), ?assertEqual(mk_workers([{"n2", 11, ?RING_END}]), WorkersRes3), ?assertEqual(mk_workers([{"n1", 0, 10}]), Dead3), - ?assertEqual([ - mk_shard("n2", 0, 4), - mk_shard("n2", 5, 10) - ], lists:sort(Reps3)), + ?assertEqual( + [ + mk_shard("n2", 0, 4), + mk_shard("n2", 5, 10) + ], + lists:sort(Reps3) + ), % All workers are available. Make sure they are not killed even if there is % a longer (single) shard to replace them. @@ -849,20 +945,17 @@ find_replacements_test() -> ?assertEqual([], Dead4), ?assertEqual([], Reps4). - mk_workers(NodesRanges) -> mk_workers(NodesRanges, nil). mk_workers(NodesRanges, Val) -> orddict:from_list([{mk_shard(N, B, E), Val} || {N, B, E} <- NodesRanges]). - mk_shard(Name, B, E) -> Node = list_to_atom(Name), BName = list_to_binary(Name), #shard{name = BName, node = Node, range = [B, E]}. - find_split_shard_replacements_test() -> % One worker is can be replaced and one can't Dead1 = mk_workers([{"n1", 0, 10}, {"n2", 11, ?RING_END}], 42), @@ -884,12 +977,18 @@ find_split_shard_replacements_test() -> mk_shard("n2", 15, ?RING_END) ], {Workers2, ShardsLeft2} = find_split_shard_replacements(Dead2, Shards2), - ?assertEqual(mk_workers([ - {"n1", 0, 10}, - {"n2", 11, 12}, - {"n2", 13, 14}, - {"n2", 15, ?RING_END} - ], 42), Workers2), + ?assertEqual( + mk_workers( + [ + {"n1", 0, 10}, + {"n2", 11, 12}, + {"n2", 13, 14}, + {"n2", 15, ?RING_END} + ], + 42 + ), + Workers2 + ), ?assertEqual([], ShardsLeft2), % No workers can be replaced. Ranges match but they are on different nodes @@ -902,7 +1001,6 @@ find_split_shard_replacements_test() -> ?assertEqual([], Workers3), ?assertEqual(Shards3, ShardsLeft3). - find_replacement_sequence_test() -> Shards = [{"n2", 0, 10}, {"n3", 0, 5}], Uuid = <<"abc1234">>, @@ -915,20 +1013,28 @@ find_replacement_sequence_test() -> % {Seq, Uuid} should work Dead2 = mk_workers(Shards, {43, Uuid}), - ?assertEqual({replace, 'n2', Uuid, 43}, - find_replacement_sequence(Dead2, [0, 10])), - ?assertEqual({replace, 'n3', Uuid, 43}, - find_replacement_sequence(Dead2, [0, 5])), + ?assertEqual( + {replace, 'n2', Uuid, 43}, + find_replacement_sequence(Dead2, [0, 10]) + ), + ?assertEqual( + {replace, 'n3', Uuid, 43}, + find_replacement_sequence(Dead2, [0, 5]) + ), % Can't find the range at all ?assertEqual(0, find_replacement_sequence(Dead2, [0, 4])), % {Seq, Uuids, EpochNode} should work Dead3 = mk_workers(Shards, {44, Uuid, Epoch}), - ?assertEqual({replace, 'n1', Uuid, 44}, - find_replacement_sequence(Dead3, [0, 10])), - ?assertEqual({replace, 'n1', Uuid, 44}, - find_replacement_sequence(Dead3, [0, 5])), + ?assertEqual( + {replace, 'n1', Uuid, 44}, + find_replacement_sequence(Dead3, [0, 10]) + ), + ?assertEqual( + {replace, 'n1', Uuid, 44}, + find_replacement_sequence(Dead3, [0, 5]) + ), % Cannot replace a replacement Dead4 = mk_workers(Shards, {replace, 'n1', Uuid, 45}), diff --git a/src/fabric/src/fabric_view_map.erl b/src/fabric/src/fabric_view_map.erl index adfe7d648..104086d67 100644 --- a/src/fabric/src/fabric_view_map.erl +++ b/src/fabric/src/fabric_view_map.erl @@ -19,19 +19,20 @@ -include_lib("couch/include/couch_db.hrl"). -include_lib("couch_mrview/include/couch_mrview.hrl"). -go(DbName, Options, GroupId, View, Args, Callback, Acc, VInfo) - when is_binary(GroupId) -> +go(DbName, Options, GroupId, View, Args, Callback, Acc, VInfo) when + is_binary(GroupId) +-> {ok, DDoc} = fabric:open_doc(DbName, <<"_design/", GroupId/binary>>, []), go(DbName, Options, DDoc, View, Args, Callback, Acc, VInfo); - go(Db, Options, DDoc, View, Args0, Callback, Acc, VInfo) -> DbName = fabric:dbname(Db), - Args = case Args0 of - #mrargs{keys = Keys, direction = rev} when is_list(Keys) -> - Args0#mrargs{keys = lists:reverse(Keys)}; - #mrargs{} -> - Args0 - end, + Args = + case Args0 of + #mrargs{keys = Keys, direction = rev} when is_list(Keys) -> + Args0#mrargs{keys = lists:reverse(Keys)}; + #mrargs{} -> + Args0 + end, {Shards, RingOpts} = fabric_view:get_shards(Db, Args), {CoordArgs, WorkerArgs} = fabric_view:fix_skip_and_limit(Args), DocIdAndRev = fabric_util:doc_id_and_rev(DDoc), @@ -44,8 +45,15 @@ go(Db, Options, DDoc, View, Args0, Callback, Acc, VInfo) -> Workers0 = fabric_util:submit_jobs(Shards, fabric_rpc, map_view, RPCArgs), RexiMon = fabric_util:create_monitors(Workers0), try - case fabric_streams:start(Workers0, #shard.ref, StartFun, Repls, - RingOpts) of + case + fabric_streams:start( + Workers0, + #shard.ref, + StartFun, + Repls, + RingOpts + ) + of {ok, ddoc_updated} -> Callback({error, ddoc_updated}, Acc); {ok, Workers} -> @@ -72,10 +80,10 @@ go(Db, Options, DDoc, View, Args0, Callback, Acc, VInfo) -> end. go(DbName, Workers, {map, View, _}, Args, Callback, Acc0) -> - #mrargs{limit = Limit, skip = Skip, keys = Keys, update_seq=UpdateSeq} = Args, + #mrargs{limit = Limit, skip = Skip, keys = Keys, update_seq = UpdateSeq} = Args, Collation = couch_util:get_value(<<"collation">>, View#mrview.options), State = #collector{ - db_name=DbName, + db_name = DbName, query_args = Args, callback = Callback, counters = fabric_dict:init(Workers, 0), @@ -85,24 +93,34 @@ go(DbName, Workers, {map, View, _}, Args, Callback, Acc0) -> sorted = Args#mrargs.sorted, collation = Collation, user_acc = Acc0, - update_seq = case UpdateSeq of true -> []; false -> nil end + update_seq = + case UpdateSeq of + true -> []; + false -> nil + end }, - case rexi_utils:recv(Workers, #shard.ref, fun handle_message/3, - State, fabric_util:view_timeout(Args), 1000 * 60 * 60) of - {ok, NewState} -> - {ok, NewState#collector.user_acc}; - {timeout, NewState} -> - Callback({error, timeout}, NewState#collector.user_acc); - {error, Resp} -> - {ok, Resp} + case + rexi_utils:recv( + Workers, + #shard.ref, + fun handle_message/3, + State, + fabric_util:view_timeout(Args), + 1000 * 60 * 60 + ) + of + {ok, NewState} -> + {ok, NewState#collector.user_acc}; + {timeout, NewState} -> + Callback({error, timeout}, NewState#collector.user_acc); + {error, Resp} -> + {ok, Resp} end. handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _, State) -> fabric_view:check_down_shards(State, NodeRef); - handle_message({rexi_EXIT, Reason}, Worker, State) -> fabric_view:handle_worker_exit(State, Worker, Reason); - handle_message({meta, Meta0}, {Worker, From}, State) -> Tot = couch_util:get_value(total, Meta0, 0), Off = couch_util:get_value(offset, Meta0, 0), @@ -122,47 +140,46 @@ handle_message({meta, Meta0}, {Worker, From}, State) -> Counters1 = fabric_dict:update_counter(Worker, 1, Counters0), Total = Total0 + Tot, Offset = Offset0 + Off, - UpdateSeq = case UpdateSeq0 of - nil -> nil; - _ -> [{Worker, Seq} | UpdateSeq0] - end, + UpdateSeq = + case UpdateSeq0 of + nil -> nil; + _ -> [{Worker, Seq} | UpdateSeq0] + end, case fabric_dict:any(0, Counters1) of - true -> - {ok, State#collector{ - counters = Counters1, - total_rows = Total, - update_seq = UpdateSeq, - offset = Offset - }}; - false -> - FinalOffset = erlang:min(Total, Offset+State#collector.skip), - Meta = [{total, Total}, {offset, FinalOffset}] ++ - case UpdateSeq of - nil -> - []; - _ -> - [{update_seq, fabric_view_changes:pack_seqs(UpdateSeq)}] - end, - {Go, Acc} = Callback({meta, Meta}, AccIn), - {Go, State#collector{ - counters = fabric_dict:decrement_all(Counters1), - total_rows = Total, - offset = FinalOffset, - user_acc = Acc - }} + true -> + {ok, State#collector{ + counters = Counters1, + total_rows = Total, + update_seq = UpdateSeq, + offset = Offset + }}; + false -> + FinalOffset = erlang:min(Total, Offset + State#collector.skip), + Meta = + [{total, Total}, {offset, FinalOffset}] ++ + case UpdateSeq of + nil -> + []; + _ -> + [{update_seq, fabric_view_changes:pack_seqs(UpdateSeq)}] + end, + {Go, Acc} = Callback({meta, Meta}, AccIn), + {Go, State#collector{ + counters = fabric_dict:decrement_all(Counters1), + total_rows = Total, + offset = FinalOffset, + user_acc = Acc + }} end; - -handle_message(#view_row{}, {_, _}, #collector{sorted=false, limit=0} = State) -> - #collector{callback=Callback} = State, +handle_message(#view_row{}, {_, _}, #collector{sorted = false, limit = 0} = State) -> + #collector{callback = Callback} = State, {_, Acc} = Callback(complete, State#collector.user_acc), - {stop, State#collector{user_acc=Acc}}; - -handle_message(#view_row{} = Row, {_,From}, #collector{sorted=false} = St) -> - #collector{callback=Callback, user_acc=AccIn, limit=Limit} = St, + {stop, State#collector{user_acc = Acc}}; +handle_message(#view_row{} = Row, {_, From}, #collector{sorted = false} = St) -> + #collector{callback = Callback, user_acc = AccIn, limit = Limit} = St, {Go, Acc} = Callback(fabric_view:transform_row(Row), AccIn), rexi:stream_ack(From), - {Go, St#collector{user_acc=Acc, limit=Limit-1}}; - + {Go, St#collector{user_acc = Acc, limit = Limit - 1}}; handle_message(#view_row{} = Row, {Worker, From}, State) -> #collector{ query_args = #mrargs{direction = Dir}, @@ -175,29 +192,26 @@ handle_message(#view_row{} = Row, {Worker, From}, State) -> Dir, Collation, KeyDict0, - Row#view_row{worker={Worker, From}}, + Row#view_row{worker = {Worker, From}}, Rows0 ), Counters1 = fabric_dict:update_counter(Worker, 1, Counters0), - State1 = State#collector{rows=Rows, counters=Counters1, keys=KeyDict}, + State1 = State#collector{rows = Rows, counters = Counters1, keys = KeyDict}, fabric_view:maybe_send_row(State1); - handle_message(complete, Worker, State) -> Counters = fabric_dict:update_counter(Worker, 1, State#collector.counters), fabric_view:maybe_send_row(State#collector{counters = Counters}); - -handle_message({execution_stats, _} = Msg, {_,From}, St) -> - #collector{callback=Callback, user_acc=AccIn} = St, +handle_message({execution_stats, _} = Msg, {_, From}, St) -> + #collector{callback = Callback, user_acc = AccIn} = St, {Go, Acc} = Callback(Msg, AccIn), rexi:stream_ack(From), - {Go, St#collector{user_acc=Acc}}; - + {Go, St#collector{user_acc = Acc}}; handle_message(ddoc_updated, _Worker, State) -> {stop, State}. merge_row(Dir, Collation, undefined, Row, Rows0) -> Rows1 = lists:merge( - fun(#view_row{key=KeyA, id=IdA}, #view_row{key=KeyB, id=IdB}) -> + fun(#view_row{key = KeyA, id = IdA}, #view_row{key = KeyB, id = IdB}) -> compare(Dir, Collation, {KeyA, IdA}, {KeyB, IdB}) end, [Row], @@ -205,23 +219,27 @@ merge_row(Dir, Collation, undefined, Row, Rows0) -> ), {Rows1, undefined}; merge_row(Dir, Collation, KeyDict0, Row, Rows0) -> - CmpFun = case Collation of - <<"raw">> -> - fun (A, A) -> 0; - (A, B) -> case A < B of - true -> -1; - false -> 1 - end - end; - _ -> - fun couch_ejson_compare:less/2 - end, + CmpFun = + case Collation of + <<"raw">> -> + fun + (A, A) -> + 0; + (A, B) -> + case A < B of + true -> -1; + false -> 1 + end + end; + _ -> + fun couch_ejson_compare:less/2 + end, case maybe_update_keydict(Row#view_row.key, KeyDict0, CmpFun) of undefined -> {Rows0, KeyDict0}; KeyDict1 -> Rows1 = lists:merge( - fun(#view_row{key=A, id=IdA}, #view_row{key=B, id=IdB}) -> + fun(#view_row{key = A, id = IdA}, #view_row{key = B, id = IdB}) -> case {Dir, CmpFun(A, B)} of {fwd, 0} -> IdA < IdB; @@ -267,7 +285,7 @@ maybe_update_keydict(Key, KeyDict, CmpFun) -> key_index(_, [], _) -> undefined; -key_index(KeyA, [{KeyB, Value}|KVs], CmpFun) -> +key_index(KeyA, [{KeyB, Value} | KVs], CmpFun) -> case CmpFun(KeyA, KeyB) of 0 -> Value; _ -> key_index(KeyA, KVs, CmpFun) diff --git a/src/fabric/src/fabric_view_reduce.erl b/src/fabric/src/fabric_view_reduce.erl index a432b2cd5..600c8d01a 100644 --- a/src/fabric/src/fabric_view_reduce.erl +++ b/src/fabric/src/fabric_view_reduce.erl @@ -22,7 +22,6 @@ go(DbName, GroupId, View, Args, Callback, Acc0, VInfo) when is_binary(GroupId) -> {ok, DDoc} = fabric:open_doc(DbName, <<"_design/", GroupId/binary>>, []), go(DbName, DDoc, View, Args, Callback, Acc0, VInfo); - go(Db, DDoc, VName, Args, Callback, Acc, VInfo) -> DbName = fabric:dbname(Db), {Shards, RingOpts} = fabric_view:get_shards(Db, Args), @@ -34,11 +33,18 @@ go(Db, DDoc, VName, Args, Callback, Acc, VInfo) -> StartFun = fun(Shard) -> hd(fabric_util:submit_jobs([Shard], fabric_rpc, reduce_view, RPCArgs)) end, - Workers0 = fabric_util:submit_jobs(Shards,fabric_rpc,reduce_view,RPCArgs), + Workers0 = fabric_util:submit_jobs(Shards, fabric_rpc, reduce_view, RPCArgs), RexiMon = fabric_util:create_monitors(Workers0), try - case fabric_streams:start(Workers0, #shard.ref, StartFun, Repls, - RingOpts) of + case + fabric_streams:start( + Workers0, + #shard.ref, + StartFun, + Repls, + RingOpts + ) + of {ok, ddoc_updated} -> Callback({error, ddoc_updated}, Acc); {ok, Workers} -> @@ -64,13 +70,14 @@ go(Db, DDoc, VName, Args, Callback, Acc, VInfo) -> rexi_monitor:stop(RexiMon) end. -go2(DbName, Workers, {red, {_, Lang, View}, _}=VInfo, Args, Callback, Acc0) -> +go2(DbName, Workers, {red, {_, Lang, View}, _} = VInfo, Args, Callback, Acc0) -> #mrargs{limit = Limit, skip = Skip, keys = Keys, update_seq = UpdateSeq} = Args, RedSrc = couch_mrview_util:extract_view_reduce(VInfo), - OsProc = case os_proc_needed(RedSrc) of - true -> couch_query_servers:get_os_process(Lang); - _ -> nil - end, + OsProc = + case os_proc_needed(RedSrc) of + true -> couch_query_servers:get_os_process(Lang); + _ -> nil + end, State = #collector{ db_name = DbName, query_args = Args, @@ -85,28 +92,39 @@ go2(DbName, Workers, {red, {_, Lang, View}, _}=VInfo, Args, Callback, Acc0) -> collation = couch_util:get_value(<<"collation">>, View#mrview.options), rows = dict:new(), user_acc = Acc0, - update_seq = case UpdateSeq of true -> []; false -> nil end + update_seq = + case UpdateSeq of + true -> []; + false -> nil + end }, - try rexi_utils:recv(Workers, #shard.ref, fun handle_message/3, - State, fabric_util:view_timeout(Args), 1000 * 60 * 60) of - {ok, NewState} -> - {ok, NewState#collector.user_acc}; - {timeout, NewState} -> - Callback({error, timeout}, NewState#collector.user_acc); - {error, Resp} -> - {ok, Resp} + try + rexi_utils:recv( + Workers, + #shard.ref, + fun handle_message/3, + State, + fabric_util:view_timeout(Args), + 1000 * 60 * 60 + ) + of + {ok, NewState} -> + {ok, NewState#collector.user_acc}; + {timeout, NewState} -> + Callback({error, timeout}, NewState#collector.user_acc); + {error, Resp} -> + {ok, Resp} after - if OsProc == nil -> ok; true -> - catch couch_query_servers:ret_os_process(OsProc) + if + OsProc == nil -> ok; + true -> catch couch_query_servers:ret_os_process(OsProc) end end. handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _, State) -> fabric_view:check_down_shards(State, NodeRef); - handle_message({rexi_EXIT, Reason}, Worker, State) -> fabric_view:handle_worker_exit(State, Worker, Reason); - handle_message({meta, Meta0}, {Worker, From}, State) -> Seq = couch_util:get_value(update_seq, Meta0, 0), #collector{ @@ -120,46 +138,44 @@ handle_message({meta, Meta0}, {Worker, From}, State) -> 0 = fabric_dict:lookup_element(Worker, Counters0), rexi:stream_ack(From), Counters1 = fabric_dict:update_counter(Worker, 1, Counters0), - UpdateSeq = case UpdateSeq0 of - nil -> nil; - _ -> [{Worker, Seq} | UpdateSeq0] - end, - case fabric_dict:any(0, Counters1) of - true -> - {ok, State#collector{ - counters = Counters1, - update_seq = UpdateSeq - }}; - false -> - Meta = case UpdateSeq of - nil -> - []; - _ -> - [{update_seq, fabric_view_changes:pack_seqs(UpdateSeq)}] + UpdateSeq = + case UpdateSeq0 of + nil -> nil; + _ -> [{Worker, Seq} | UpdateSeq0] end, - {Go, Acc} = Callback({meta, Meta}, AccIn), - {Go, State#collector{ - counters = fabric_dict:decrement_all(Counters1), - user_acc = Acc - }} + case fabric_dict:any(0, Counters1) of + true -> + {ok, State#collector{ + counters = Counters1, + update_seq = UpdateSeq + }}; + false -> + Meta = + case UpdateSeq of + nil -> + []; + _ -> + [{update_seq, fabric_view_changes:pack_seqs(UpdateSeq)}] + end, + {Go, Acc} = Callback({meta, Meta}, AccIn), + {Go, State#collector{ + counters = fabric_dict:decrement_all(Counters1), + user_acc = Acc + }} end; - -handle_message(#view_row{key=Key} = Row, {Worker, From}, State) -> +handle_message(#view_row{key = Key} = Row, {Worker, From}, State) -> #collector{counters = Counters0, rows = Rows0} = State, true = fabric_dict:is_key(Worker, Counters0), - Rows = dict:append(Key, Row#view_row{worker={Worker, From}}, Rows0), + Rows = dict:append(Key, Row#view_row{worker = {Worker, From}}, Rows0), C1 = fabric_dict:update_counter(Worker, 1, Counters0), - State1 = State#collector{rows=Rows, counters=C1}, + State1 = State#collector{rows = Rows, counters = C1}, fabric_view:maybe_send_row(State1); - handle_message(complete, Worker, #collector{counters = Counters0} = State) -> true = fabric_dict:is_key(Worker, Counters0), C1 = fabric_dict:update_counter(Worker, 1, Counters0), fabric_view:maybe_send_row(State#collector{counters = C1}); - handle_message(ddoc_updated, _Worker, State) -> {stop, State}. os_proc_needed(<<"_", _/binary>>) -> false; os_proc_needed(_) -> true. - diff --git a/src/fabric/test/eunit/fabric_db_create_tests.erl b/src/fabric/test/eunit/fabric_db_create_tests.erl index 8e5b1085e..a7c98a96f 100644 --- a/src/fabric/test/eunit/fabric_db_create_tests.erl +++ b/src/fabric/test/eunit/fabric_db_create_tests.erl @@ -12,15 +12,12 @@ -module(fabric_db_create_tests). - -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). -include_lib("mem3/include/mem3.hrl"). - -define(TDEF(A), {atom_to_list(A), fun A/0}). - main_test_() -> { setup, @@ -31,16 +28,13 @@ main_test_() -> ] }. - setup() -> test_util:start_couch([fabric]). - teardown(Ctx) -> meck:unload(), test_util:stop_couch(Ctx). - t_handle_shard_doc_conflict() -> DbName = ?tempdb(), meck:new(mem3, [passthrough]), diff --git a/src/fabric/test/eunit/fabric_db_info_tests.erl b/src/fabric/test/eunit/fabric_db_info_tests.erl index ccdafe3ae..d4fb1e73f 100644 --- a/src/fabric/test/eunit/fabric_db_info_tests.erl +++ b/src/fabric/test/eunit/fabric_db_info_tests.erl @@ -12,15 +12,12 @@ -module(fabric_db_info_tests). - -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). -include_lib("mem3/include/mem3.hrl"). - -define(TDEF(A), {atom_to_list(A), fun A/0}). - main_test_() -> { setup, @@ -31,16 +28,13 @@ main_test_() -> ] }. - setup() -> test_util:start_couch([fabric]). - teardown(Ctx) -> meck:unload(), test_util:stop_couch(Ctx). - t_update_seq_has_uuids() -> DbName = ?tempdb(), ok = fabric:create_db(DbName, [{q, 1}, {n, 1}]), diff --git a/src/fabric/test/eunit/fabric_db_uuids_tests.erl b/src/fabric/test/eunit/fabric_db_uuids_tests.erl index 986945b52..1d99019a8 100644 --- a/src/fabric/test/eunit/fabric_db_uuids_tests.erl +++ b/src/fabric/test/eunit/fabric_db_uuids_tests.erl @@ -41,11 +41,15 @@ t_can_get_shard_uuids() -> Shards = mem3:shards(DbName), {ok, Uuids} = fabric:db_uuids(DbName), ?assertEqual(length(Shards), map_size(Uuids)), - UuidsFromShards = lists:foldl(fun(#shard{} = Shard, Acc) -> - Uuid = couch_util:with_db(Shard#shard.name, fun(Db) -> - couch_db:get_uuid(Db) - end), - Acc#{Uuid => Shard} - end, #{}, Shards), + UuidsFromShards = lists:foldl( + fun(#shard{} = Shard, Acc) -> + Uuid = couch_util:with_db(Shard#shard.name, fun(Db) -> + couch_db:get_uuid(Db) + end), + Acc#{Uuid => Shard} + end, + #{}, + Shards + ), ?assertEqual(UuidsFromShards, Uuids), ok = fabric:delete_db(DbName, []). diff --git a/src/fabric/test/eunit/fabric_moved_shards_seq_tests.erl b/src/fabric/test/eunit/fabric_moved_shards_seq_tests.erl index a78d17ab7..5463a5170 100644 --- a/src/fabric/test/eunit/fabric_moved_shards_seq_tests.erl +++ b/src/fabric/test/eunit/fabric_moved_shards_seq_tests.erl @@ -12,15 +12,12 @@ -module(fabric_moved_shards_seq_tests). - -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). -include_lib("mem3/include/mem3.hrl"). - -define(TDEF(A), {atom_to_list(A), fun A/0}). - main_test_() -> { setup, @@ -31,24 +28,24 @@ main_test_() -> ] }. - setup() -> test_util:start_couch([fabric]). - teardown(Ctx) -> meck:unload(), test_util:stop_couch(Ctx). - t_shard_moves_avoid_sequence_rewinds() -> DocCnt = 30, DbName = ?tempdb(), - ok = fabric:create_db(DbName, [{q,1}, {n,1}]), - lists:foreach(fun(I) -> - update_doc(DbName, #doc{id = erlang:integer_to_binary(I)}) - end, lists:seq(1, DocCnt)), + ok = fabric:create_db(DbName, [{q, 1}, {n, 1}]), + lists:foreach( + fun(I) -> + update_doc(DbName, #doc{id = erlang:integer_to_binary(I)}) + end, + lists:seq(1, DocCnt) + ), {ok, _, Seq1, 0} = changes(DbName, #changes_args{limit = 1, since = "now"}), [{_, Range, {Seq, Uuid, _}}] = seq_decode(Seq1), @@ -76,24 +73,19 @@ t_shard_moves_avoid_sequence_rewinds() -> ok = fabric:delete_db(DbName, []). - changes_callback(start, Acc) -> {ok, Acc}; - changes_callback({change, {Change}}, Acc) -> CM = maps:from_list(Change), {ok, [CM | Acc]}; - changes_callback({stop, EndSeq, Pending}, Acc) -> {ok, Acc, EndSeq, Pending}. - changes(DbName, #changes_args{} = Args) -> fabric_util:isolate(fun() -> fabric:changes(DbName, fun changes_callback/2, [], Args) end). - update_doc(DbName, #doc{} = Doc) -> fabric_util:isolate(fun() -> case fabric:update_doc(DbName, Doc, [?ADMIN_CTX]) of @@ -101,7 +93,6 @@ update_doc(DbName, #doc{} = Doc) -> end end). - seq_decode(Seq) -> % This is copied from fabric_view_changes Pattern = "^\"?([0-9]+-)?(?<opaque>.*?)\"?$", @@ -109,13 +100,11 @@ seq_decode(Seq) -> {match, Seq1} = re:run(Seq, Pattern, Options), binary_to_term(couch_util:decodeBase64Url(Seq1)). - seq_encode(Unpacked) -> % Copied from fabric_view_changes Opaque = couch_util:encodeBase64Url(term_to_binary(Unpacked, [compressed])), ?l2b(["30", $-, Opaque]). - mock_epochs(Epochs) -> % Since we made up a node name we'll have to mock epoch checking meck:new(couch_db_engine, [passthrough]), diff --git a/src/fabric/test/eunit/fabric_rpc_purge_tests.erl b/src/fabric/test/eunit/fabric_rpc_purge_tests.erl index 6db6a70aa..d3872410e 100644 --- a/src/fabric/test/eunit/fabric_rpc_purge_tests.erl +++ b/src/fabric/test/eunit/fabric_rpc_purge_tests.erl @@ -12,11 +12,9 @@ -module(fabric_rpc_purge_tests). - -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). - -define(TDEF(A), {A, fun A/1}). % TODO: Add tests: @@ -68,25 +66,20 @@ main_test_() -> ] }. - setup_all() -> test_util:start_couch(). - teardown_all(Ctx) -> test_util:stop_couch(Ctx). - setup_no_purge() -> {ok, Db} = create_db(), populate_db(Db), couch_db:name(Db). - teardown_no_purge(DbName) -> ok = couch_server:delete(DbName, []). - setup_single_purge() -> DbName = setup_no_purge(), DocId = <<"0003">>, @@ -94,26 +87,25 @@ setup_single_purge() -> purge_doc(DbName, DocId), {DbName, DocId, OldDoc, 1}. - teardown_single_purge({DbName, _, _, _}) -> teardown_no_purge(DbName). - setup_multi_purge() -> DbName = setup_no_purge(), DocId = <<"0003">>, {ok, OldDoc} = open_doc(DbName, DocId), - lists:foreach(fun(I) -> - PDocId = iolist_to_binary(io_lib:format("~4..0b", [I])), - purge_doc(DbName, PDocId) - end, lists:seq(1, 5)), + lists:foreach( + fun(I) -> + PDocId = iolist_to_binary(io_lib:format("~4..0b", [I])), + purge_doc(DbName, PDocId) + end, + lists:seq(1, 5) + ), {DbName, DocId, OldDoc, 3}. - teardown_multi_purge(Ctx) -> teardown_single_purge(Ctx). - t_no_purge_no_filter(DbName) -> DocId = <<"0003">>, @@ -126,7 +118,6 @@ t_no_purge_no_filter(DbName) -> ?assert(CurrDoc /= OldDoc), ?assert(CurrDoc == NewDoc). - t_filter({DbName, DocId, OldDoc, _PSeq}) -> ?assertEqual({not_found, missing}, open_doc(DbName, DocId)), create_purge_checkpoint(DbName, 0), @@ -135,7 +126,6 @@ t_filter({DbName, DocId, OldDoc, _PSeq}) -> ?assertEqual({not_found, missing}, open_doc(DbName, DocId)). - t_filter_unknown_node({DbName, DocId, OldDoc, _PSeq}) -> % Unknown nodes are assumed to start at PurgeSeq = 0 ?assertEqual({not_found, missing}, open_doc(DbName, DocId)), @@ -147,7 +137,6 @@ t_filter_unknown_node({DbName, DocId, OldDoc, _PSeq}) -> ?assertEqual({not_found, missing}, open_doc(DbName, DocId)). - t_no_filter_old_node({DbName, DocId, OldDoc, PSeq}) -> ?assertEqual({not_found, missing}, open_doc(DbName, DocId)), create_purge_checkpoint(DbName, PSeq), @@ -160,7 +149,6 @@ t_no_filter_old_node({DbName, DocId, OldDoc, PSeq}) -> ?assertEqual({ok, OldDoc}, open_doc(DbName, DocId)). - t_no_filter_different_node({DbName, DocId, OldDoc, PSeq}) -> ?assertEqual({not_found, missing}, open_doc(DbName, DocId)), create_purge_checkpoint(DbName, PSeq), @@ -173,7 +161,6 @@ t_no_filter_different_node({DbName, DocId, OldDoc, PSeq}) -> ?assertEqual({ok, OldDoc}, open_doc(DbName, DocId)). - t_filter_local_node({DbName, DocId, OldDoc, PSeq}) -> ?assertEqual({not_found, missing}, open_doc(DbName, DocId)), create_purge_checkpoint(DbName, PSeq), @@ -185,15 +172,16 @@ t_filter_local_node({DbName, DocId, OldDoc, PSeq}) -> % Add a local node rev to the list of node revs. It should % be filtered out {Pos, [Rev | _]} = OldDoc#doc.revs, - RROpts = [{read_repair, [ - {tgt_node(), [{Pos, Rev}]}, - {node(), [{1, <<"123">>}]} - ]}], + RROpts = [ + {read_repair, [ + {tgt_node(), [{Pos, Rev}]}, + {node(), [{1, <<"123">>}]} + ]} + ], rpc_update_doc(DbName, OldDoc, RROpts), ?assertEqual({ok, OldDoc}, open_doc(DbName, DocId)). - t_no_filter_after_repl({DbName, DocId, OldDoc, PSeq}) -> ?assertEqual({not_found, missing}, open_doc(DbName, DocId)), create_purge_checkpoint(DbName, PSeq), @@ -202,38 +190,37 @@ t_no_filter_after_repl({DbName, DocId, OldDoc, PSeq}) -> ?assertEqual({ok, OldDoc}, open_doc(DbName, DocId)). - wrap({Name, Fun}) -> fun(Arg) -> - {timeout, 60, {atom_to_list(Name), fun() -> - process_flag(trap_exit, true), - Fun(Arg) - end}} + {timeout, 60, + {atom_to_list(Name), fun() -> + process_flag(trap_exit, true), + Fun(Arg) + end}} end. - create_db() -> DbName = ?tempdb(), couch_db:create(DbName, [?ADMIN_CTX]). - populate_db(Db) -> - Docs = lists:map(fun(Idx) -> - DocId = lists:flatten(io_lib:format("~4..0b", [Idx])), - #doc{ - id = list_to_binary(DocId), - body = {[{<<"int">>, Idx}, {<<"vsn">>, 2}]} - } - end, lists:seq(1, 100)), + Docs = lists:map( + fun(Idx) -> + DocId = lists:flatten(io_lib:format("~4..0b", [Idx])), + #doc{ + id = list_to_binary(DocId), + body = {[{<<"int">>, Idx}, {<<"vsn">>, 2}]} + } + end, + lists:seq(1, 100) + ), {ok, _} = couch_db:update_docs(Db, Docs). - open_doc(DbName, DocId) -> couch_util:with_db(DbName, fun(Db) -> couch_db:open_doc(Db, DocId, []) end). - create_update(Doc, NewVsn) -> #doc{ id = DocId, @@ -247,7 +234,6 @@ create_update(Doc, NewVsn) -> body = {NewProps} }. - purge_doc(DbName, DocId) -> {ok, Doc} = open_doc(DbName, DocId), {Pos, [Rev | _]} = Doc#doc.revs, @@ -257,33 +243,30 @@ purge_doc(DbName, DocId) -> end), ?assertEqual({ok, [{ok, [{Pos, Rev}]}]}, Resp). - create_purge_checkpoint(DbName, PurgeSeq) -> create_purge_checkpoint(DbName, PurgeSeq, tgt_node_bin()). - create_purge_checkpoint(DbName, PurgeSeq, TgtNode) when is_binary(TgtNode) -> Resp = couch_util:with_db(DbName, fun(Db) -> SrcUUID = couch_db:get_uuid(Db), TgtUUID = couch_uuids:random(), CPDoc = #doc{ id = mem3_rep:make_purge_id(SrcUUID, TgtUUID), - body = {[ - {<<"target_node">>, TgtNode}, - {<<"purge_seq">>, PurgeSeq} - ]} + body = + {[ + {<<"target_node">>, TgtNode}, + {<<"purge_seq">>, PurgeSeq} + ]} }, couch_db:update_docs(Db, [CPDoc], []) end), ?assertMatch({ok, [_]}, Resp). - rpc_update_doc(DbName, Doc) -> {Pos, [Rev | _]} = Doc#doc.revs, RROpt = {read_repair, [{tgt_node(), [{Pos, Rev}]}]}, rpc_update_doc(DbName, Doc, [RROpt]). - rpc_update_doc(DbName, Doc, Opts) -> Ref = erlang:make_ref(), put(rexi_from, {self(), Ref}), @@ -298,10 +281,8 @@ rpc_update_doc(DbName, Doc, Opts) -> end), ?assertEqual({ok, []}, Reply). - tgt_node() -> 'foo@127.0.0.1'. - tgt_node_bin() -> iolist_to_binary(atom_to_list(tgt_node())). diff --git a/src/fabric/test/eunit/fabric_rpc_tests.erl b/src/fabric/test/eunit/fabric_rpc_tests.erl index b94caf659..f47d153ff 100644 --- a/src/fabric/test/eunit/fabric_rpc_tests.erl +++ b/src/fabric/test/eunit/fabric_rpc_tests.erl @@ -12,14 +12,11 @@ -module(fabric_rpc_tests). - -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). - -define(TDEF(A), {A, fun A/1}). - main_test_() -> { setup, @@ -53,69 +50,63 @@ main_test_() -> ?TDEF(t_db_create_with_config) ]) } - ] }. - setup_all() -> test_util:start_couch([rexi, mem3, fabric]). - teardown_all(Ctx) -> test_util:stop_couch(Ctx). - setup_no_db_or_config() -> ?tempdb(). - setup_shard() -> ?tempshard(). - teardown_noop(_DbName) -> ok. teardown_db(DbName) -> ok = couch_server:delete(DbName, []). - wrap({Name, Fun}) -> fun(Arg) -> - {timeout, 60, {atom_to_list(Name), fun() -> - process_flag(trap_exit, true), - Fun(Arg) - end}} + {timeout, 60, + {atom_to_list(Name), fun() -> + process_flag(trap_exit, true), + Fun(Arg) + end}} end. - t_no_db(DbName) -> ?assertEqual({not_found, no_db_file}, couch_db:open_int(DbName, [?ADMIN_CTX])). - t_no_config_non_shard_db_create_succeeds(DbName) -> ?assertEqual({not_found, no_db_file}, couch_db:open_int(DbName, [?ADMIN_CTX])), ?assertEqual(DbName, mem3:dbname(DbName)), ?assertMatch({ok, _}, mem3_util:get_or_create_db(DbName, [?ADMIN_CTX])). - t_no_config_db_create_fails_for_shard(DbName) -> ?assertEqual({not_found, no_db_file}, couch_db:open_int(DbName, [?ADMIN_CTX])), - ?assertException(throw, {error, missing_target}, mem3_util:get_or_create_db(DbName, [?ADMIN_CTX])). - + ?assertException( + throw, {error, missing_target}, mem3_util:get_or_create_db(DbName, [?ADMIN_CTX]) + ). t_no_config_db_create_fails_for_shard_rpc(DbName) -> ?assertEqual({not_found, no_db_file}, couch_db:open_int(DbName, [?ADMIN_CTX])), - ?assertException(throw, {error, missing_target}, mem3_util:get_or_create_db(DbName, [?ADMIN_CTX])), + ?assertException( + throw, {error, missing_target}, mem3_util:get_or_create_db(DbName, [?ADMIN_CTX]) + ), MFA = {fabric_rpc, get_db_info, [DbName]}, Ref = rexi:cast(node(), self(), MFA), - Resp = receive - Resp0 -> Resp0 - end, + Resp = + receive + Resp0 -> Resp0 + end, ?assertMatch({Ref, {'rexi_EXIT', {{error, missing_target}, _}}}, Resp). - t_db_create_with_config(DbName) -> MDbName = mem3:dbname(DbName), DbDoc = #doc{id = MDbName, body = test_db_doc()}, @@ -138,21 +129,22 @@ t_db_create_with_config(DbName) -> {Body} = test_db_doc(), DbProps = mem3_util:get_shard_opts(Body), - {Props} = case couch_db_engine:get_props(LDb) of - undefined -> {[]}; - Else -> {Else} - end, + {Props} = + case couch_db_engine:get_props(LDb) of + undefined -> {[]}; + Else -> {Else} + end, %% We don't normally store the default engine name - EngineProps = case couch_db_engine:get_engine(LDb) of - couch_bt_engine -> - []; - EngineName -> - [{engine, EngineName}] - end, + EngineProps = + case couch_db_engine:get_engine(LDb) of + couch_bt_engine -> + []; + EngineName -> + [{engine, EngineName}] + end, ?assertEqual([{props, Props} | EngineProps], DbProps) end). - test_db_doc() -> {[ {<<"shard_suffix">>, ".1584997648"}, @@ -164,18 +156,24 @@ test_db_doc() -> [<<"add">>, <<"80000000-ffffffff">>, <<"node2@127.0.0.1">>], [<<"add">>, <<"80000000-ffffffff">>, <<"node3@127.0.0.1">>] ]}, - {<<"by_node">>, {[ - {<<"node1@127.0.0.1">>, [<<"00000000-7fffffff">>, <<"80000000-ffffffff">>]}, - {<<"node2@127.0.0.1">>, [<<"00000000-7fffffff">>, <<"80000000-ffffffff">>]}, - {<<"node3@127.0.0.1">>, [<<"00000000-7fffffff">>, <<"80000000-ffffffff">>]} - ]}}, - {<<"by_range">>, {[ - {<<"00000000-7fffffff">>, [<<"node1@127.0.0.1">>, <<"node2@127.0.0.1">>, <<"node3@127.0.0.1">>]}, - {<<"80000000-ffffffff">>, [<<"node1@127.0.0.1">>, <<"node2@127.0.0.1">>, <<"node3@127.0.0.1">>]} - ]}}, - {<<"props">>, {[ - {partitioned, true}, - {hash, [couch_partition, hash, []]} - ]}} + {<<"by_node">>, + {[ + {<<"node1@127.0.0.1">>, [<<"00000000-7fffffff">>, <<"80000000-ffffffff">>]}, + {<<"node2@127.0.0.1">>, [<<"00000000-7fffffff">>, <<"80000000-ffffffff">>]}, + {<<"node3@127.0.0.1">>, [<<"00000000-7fffffff">>, <<"80000000-ffffffff">>]} + ]}}, + {<<"by_range">>, + {[ + {<<"00000000-7fffffff">>, [ + <<"node1@127.0.0.1">>, <<"node2@127.0.0.1">>, <<"node3@127.0.0.1">> + ]}, + {<<"80000000-ffffffff">>, [ + <<"node1@127.0.0.1">>, <<"node2@127.0.0.1">>, <<"node3@127.0.0.1">> + ]} + ]}}, + {<<"props">>, + {[ + {partitioned, true}, + {hash, [couch_partition, hash, []]} + ]}} ]}. - diff --git a/src/fabric/test/eunit/fabric_tests.erl b/src/fabric/test/eunit/fabric_tests.erl index 53995933b..c0e2b626b 100644 --- a/src/fabric/test/eunit/fabric_tests.erl +++ b/src/fabric/test/eunit/fabric_tests.erl @@ -12,23 +12,22 @@ -module(fabric_tests). - -include_lib("couch/include/couch_eunit.hrl"). - cleanup_index_files_test_() -> { setup, fun setup/0, fun teardown/1, - fun(Ctx) -> [ - t_cleanup_index_files(), - t_cleanup_index_files_with_existing_db(Ctx), - t_cleanup_index_files_with_deleted_db(Ctx) - ] end + fun(Ctx) -> + [ + t_cleanup_index_files(), + t_cleanup_index_files_with_existing_db(Ctx), + t_cleanup_index_files_with_deleted_db(Ctx) + ] + end }. - setup() -> Ctx = test_util:start_couch([fabric]), % TempDb is deleted in the test "t_cleanup_index_files_with_deleted_db". @@ -36,25 +35,25 @@ setup() -> fabric:create_db(TempDb), {Ctx, TempDb}. - teardown({Ctx, _TempDb}) -> test_util:stop_couch(Ctx). - t_cleanup_index_files() -> ?_assert( - lists:all(fun(Res) -> Res =:= ok end, fabric:cleanup_index_files())). - + lists:all(fun(Res) -> Res =:= ok end, fabric:cleanup_index_files()) + ). t_cleanup_index_files_with_existing_db({_Ctx, TempDb}) -> ?_assertEqual(ok, fabric:cleanup_index_files(TempDb)). - t_cleanup_index_files_with_deleted_db({_Ctx, TempDb}) -> ?_test( begin fabric:delete_db(TempDb, []), - ?assertError(database_does_not_exist, - fabric:inactive_index_files(TempDb)), + ?assertError( + database_does_not_exist, + fabric:inactive_index_files(TempDb) + ), ?assertEqual(ok, fabric:cleanup_index_files(TempDb)) - end). + end + ). diff --git a/src/global_changes/src/global_changes_app.erl b/src/global_changes/src/global_changes_app.erl index 03322a27e..aa0e5d3fd 100644 --- a/src/global_changes/src/global_changes_app.erl +++ b/src/global_changes/src/global_changes_app.erl @@ -13,16 +13,13 @@ -module(global_changes_app). -behavior(application). - -export([ start/2, stop/1 ]). - start(_StartType, _StartArgs) -> global_changes_sup:start_link(). - stop(_State) -> ok. diff --git a/src/global_changes/src/global_changes_epi.erl b/src/global_changes/src/global_changes_epi.erl index 5d8cbf928..25e204001 100644 --- a/src/global_changes/src/global_changes_epi.erl +++ b/src/global_changes/src/global_changes_epi.erl @@ -32,7 +32,6 @@ providers() -> {chttpd_handlers, global_changes_httpd_handlers} ]. - services() -> [ {global_changes, global_changes_plugin} diff --git a/src/global_changes/src/global_changes_httpd.erl b/src/global_changes/src/global_changes_httpd.erl index e579b09ea..cb4016b63 100644 --- a/src/global_changes/src/global_changes_httpd.erl +++ b/src/global_changes/src/global_changes_httpd.erl @@ -28,15 +28,16 @@ limit }). -handle_global_changes_req(#httpd{method='GET'}=Req) -> +handle_global_changes_req(#httpd{method = 'GET'} = Req) -> Db = global_changes_util:get_dbname(), Feed = chttpd:qs_value(Req, "feed", "normal"), Options = parse_global_changes_query(Req), - Heartbeat = case lists:keyfind(heartbeat, 1, Options) of - {heartbeat, true} -> 60000; - {heartbeat, Other} -> Other; - false -> false - end, + Heartbeat = + case lists:keyfind(heartbeat, 1, Options) of + {heartbeat, true} -> 60000; + {heartbeat, Other} -> Other; + false -> false + end, % Limit is handled in the changes callback, since the limit count needs to % only account for changes which happen after the filter. Limit = couch_util:get_value(limit, Options), @@ -44,11 +45,11 @@ handle_global_changes_req(#httpd{method='GET'}=Req) -> Options1 = Options, Owner = allowed_owner(Req), Acc = #acc{ - username=Owner, - feed=Feed, - resp=Req, - heartbeat_interval=Heartbeat, - limit=Limit + username = Owner, + feed = Feed, + resp = Req, + heartbeat_interval = Heartbeat, + limit = Limit }, case Feed of "normal" -> @@ -56,7 +57,7 @@ handle_global_changes_req(#httpd{method='GET'}=Req) -> Suffix = mem3:shard_suffix(Db), Etag = chttpd:make_etag({Info, Suffix}), chttpd:etag_respond(Req, Etag, fun() -> - fabric:changes(Db, fun changes_callback/2, Acc#acc{etag=Etag}, Options1) + fabric:changes(Db, fun changes_callback/2, Acc#acc{etag = Etag}, Options1) end); Feed when Feed =:= "continuous"; Feed =:= "longpoll"; Feed =:= "eventsource" -> fabric:changes(Db, fun changes_callback/2, Acc, Options1); @@ -68,18 +69,22 @@ handle_global_changes_req(Req) -> chttpd:send_method_not_allowed(Req, "GET"). transform_change(Username, Change) -> - global_changes_plugin:transform_change(Username, Change, - fun default_transform_change/2). + global_changes_plugin:transform_change( + Username, + Change, + fun default_transform_change/2 + ). default_transform_change(Username, {Props}) -> {id, Id} = lists:keyfind(id, 1, Props), {seq, Seq} = lists:keyfind(seq, 1, Props), - Info = case binary:split(Id, <<":">>) of - [Event0, DbName0] -> - {Event0, DbName0}; - _ -> - skip - end, + Info = + case binary:split(Id, <<":">>) of + [Event0, DbName0] -> + {Event0, DbName0}; + _ -> + skip + end, case Info of % Client is an admin, show them everything. {Event, DbName} when Username == admin -> @@ -94,19 +99,17 @@ default_transform_change(Username, {Props}) -> changes_callback(waiting_for_updates, Acc) -> {ok, Acc}; - % This clause is only hit when _db_updates is queried with limit=0. For % limit>0, the request is stopped by maybe_finish/1. -changes_callback({change, _}, #acc{limit=0}=Acc) -> +changes_callback({change, _}, #acc{limit = 0} = Acc) -> {stop, Acc}; - % callbacks for continuous feed (newline-delimited JSON Objects) -changes_callback(start, #acc{feed="continuous"}=Acc) -> - #acc{resp=Req} = Acc, +changes_callback(start, #acc{feed = "continuous"} = Acc) -> + #acc{resp = Req} = Acc, {ok, Resp} = chttpd:start_delayed_json_response(Req, 200), - {ok, Acc#acc{resp=Resp, last_data_sent_time=os:timestamp()}}; -changes_callback({change, Change0}, #acc{feed="continuous"}=Acc) -> - #acc{resp=Resp, username=Username} = Acc, + {ok, Acc#acc{resp = Resp, last_data_sent_time = os:timestamp()}}; +changes_callback({change, Change0}, #acc{feed = "continuous"} = Acc) -> + #acc{resp = Resp, username = Username} = Acc, case transform_change(Username, Change0) of skip -> {ok, maybe_send_heartbeat(Acc)}; @@ -114,20 +117,21 @@ changes_callback({change, Change0}, #acc{feed="continuous"}=Acc) -> Line = [?JSON_ENCODE(Change) | "\n"], {ok, Resp1} = chttpd:send_delayed_chunk(Resp, Line), Acc1 = Acc#acc{ - resp=Resp1, - last_data_sent_time=os:timestamp() + resp = Resp1, + last_data_sent_time = os:timestamp() }, maybe_finish(Acc1) end; -changes_callback({stop, EndSeq}, #acc{feed="continuous"}=Acc) -> +changes_callback({stop, EndSeq}, #acc{feed = "continuous"} = Acc) -> % Temporary upgrade clause - Case 24236 changes_callback({stop, EndSeq, null}, Acc); -changes_callback({stop, EndSeq, _Pending}, #acc{feed="continuous"}=Acc) -> - #acc{resp=Resp} = Acc, - {ok, Resp1} = chttpd:send_delayed_chunk(Resp, - [?JSON_ENCODE({[{<<"last_seq">>, EndSeq}]}) | "\n"]), +changes_callback({stop, EndSeq, _Pending}, #acc{feed = "continuous"} = Acc) -> + #acc{resp = Resp} = Acc, + {ok, Resp1} = chttpd:send_delayed_chunk( + Resp, + [?JSON_ENCODE({[{<<"last_seq">>, EndSeq}]}) | "\n"] + ), chttpd:end_delayed_json_response(Resp1); - % callbacks for eventsource feed (newline-delimited eventsource Objects) changes_callback(start, #acc{feed = "eventsource"} = Acc) -> #acc{resp = Req} = Acc, @@ -136,12 +140,15 @@ changes_callback(start, #acc{feed = "eventsource"} = Acc) -> {"Cache-Control", "no-cache"} ], {ok, Resp} = chttpd:start_delayed_json_response(Req, 200, Headers), - {ok, Acc#acc{resp = Resp, last_data_sent_time=os:timestamp()}}; -changes_callback({change, {ChangeProp}=Change}, #acc{resp = Resp, feed = "eventsource"} = Acc) -> + {ok, Acc#acc{resp = Resp, last_data_sent_time = os:timestamp()}}; +changes_callback({change, {ChangeProp} = Change}, #acc{resp = Resp, feed = "eventsource"} = Acc) -> Seq = proplists:get_value(seq, ChangeProp), Chunk = [ - "data: ", ?JSON_ENCODE(Change), - "\n", "id: ", ?JSON_ENCODE(Seq), + "data: ", + ?JSON_ENCODE(Change), + "\n", + "id: ", + ?JSON_ENCODE(Seq), "\n\n" ], {ok, Resp1} = chttpd:send_delayed_chunk(Resp, Chunk), @@ -155,37 +162,41 @@ changes_callback({stop, _EndSeq}, #acc{feed = "eventsource"} = Acc) -> #acc{resp = Resp} = Acc, % {ok, Resp1} = chttpd:send_delayed_chunk(Resp, Buf), chttpd:end_delayed_json_response(Resp); - % callbacks for longpoll and normal (single JSON Object) -changes_callback(start, #acc{feed="normal", etag=Etag}=Acc) - when Etag =/= undefined -> - #acc{resp=Req} = Acc, +changes_callback(start, #acc{feed = "normal", etag = Etag} = Acc) when + Etag =/= undefined +-> + #acc{resp = Req} = Acc, FirstChunk = "{\"results\":[\n", - {ok, Resp} = chttpd:start_delayed_json_response(Req, 200, - [{"Etag",Etag}], FirstChunk), - {ok, Acc#acc{resp=Resp, prepend="", last_data_sent_time=os:timestamp()}}; + {ok, Resp} = chttpd:start_delayed_json_response( + Req, + 200, + [{"Etag", Etag}], + FirstChunk + ), + {ok, Acc#acc{resp = Resp, prepend = "", last_data_sent_time = os:timestamp()}}; changes_callback(start, Acc) -> - #acc{resp=Req} = Acc, + #acc{resp = Req} = Acc, FirstChunk = "{\"results\":[\n", {ok, Resp} = chttpd:start_delayed_json_response(Req, 200, [], FirstChunk), {ok, Acc#acc{ - resp=Resp, - prepend="", - last_data_sent_time=os:timestamp() + resp = Resp, + prepend = "", + last_data_sent_time = os:timestamp() }}; changes_callback({change, Change0}, Acc) -> - #acc{resp=Resp, prepend=Prepend, username=Username} = Acc, + #acc{resp = Resp, prepend = Prepend, username = Username} = Acc, case transform_change(Username, Change0) of skip -> {ok, maybe_send_heartbeat(Acc)}; Change -> - #acc{resp=Resp, prepend=Prepend} = Acc, + #acc{resp = Resp, prepend = Prepend} = Acc, Line = [Prepend, ?JSON_ENCODE(Change)], {ok, Resp1} = chttpd:send_delayed_chunk(Resp, Line), Acc1 = Acc#acc{ - prepend=",\r\n", - resp=Resp1, - last_data_sent_time=os:timestamp() + prepend = ",\r\n", + resp = Resp1, + last_data_sent_time = os:timestamp() }, maybe_finish(Acc1) end; @@ -193,18 +204,18 @@ changes_callback({stop, EndSeq}, Acc) -> % Temporary upgrade clause - Case 24236 changes_callback({stop, EndSeq, null}, Acc); changes_callback({stop, EndSeq, _Pending}, Acc) -> - #acc{resp=Resp} = Acc, - {ok, Resp1} = chttpd:send_delayed_chunk(Resp, - ["\n],\n\"last_seq\":", ?JSON_ENCODE(EndSeq), "}\n"]), + #acc{resp = Resp} = Acc, + {ok, Resp1} = chttpd:send_delayed_chunk( + Resp, + ["\n],\n\"last_seq\":", ?JSON_ENCODE(EndSeq), "}\n"] + ), chttpd:end_delayed_json_response(Resp1); - changes_callback(timeout, Acc) -> {ok, maybe_send_heartbeat(Acc)}; - -changes_callback({error, Reason}, #acc{resp=Req=#httpd{}}) -> +changes_callback({error, Reason}, #acc{resp = Req = #httpd{}}) -> chttpd:send_error(Req, Reason); changes_callback({error, Reason}, Acc) -> - #acc{etag=Etag, feed=Feed, resp=Resp} = Acc, + #acc{etag = Etag, feed = Feed, resp = Resp} = Acc, case {Feed, Etag} of {"normal", Etag} when Etag =/= undefined -> chttpd:send_error(Resp, Reason); @@ -212,7 +223,6 @@ changes_callback({error, Reason}, Acc) -> chttpd:send_delayed_error(Resp, Reason) end. - maybe_finish(Acc) -> case Acc#acc.limit of 1 -> @@ -220,48 +230,50 @@ maybe_finish(Acc) -> undefined -> {ok, Acc}; Limit -> - {ok, Acc#acc{limit=Limit-1}} + {ok, Acc#acc{limit = Limit - 1}} end. - -maybe_send_heartbeat(#acc{heartbeat_interval=false}=Acc) -> +maybe_send_heartbeat(#acc{heartbeat_interval = false} = Acc) -> Acc; maybe_send_heartbeat(Acc) -> - #acc{last_data_sent_time=LastSentTime, heartbeat_interval=Interval, resp=Resp} = Acc, + #acc{last_data_sent_time = LastSentTime, heartbeat_interval = Interval, resp = Resp} = Acc, Now = os:timestamp(), case timer:now_diff(Now, LastSentTime) div 1000 > Interval of true -> {ok, Resp1} = chttpd:send_delayed_chunk(Resp, "\n"), - Acc#acc{last_data_sent_time=Now, resp=Resp1}; + Acc#acc{last_data_sent_time = Now, resp = Resp1}; false -> Acc end. - parse_global_changes_query(Req) -> - lists:foldl(fun({Key, Value}, Args) -> - case {Key, Value} of - {"feed", _} -> - [{feed, Value} | Args]; - {"descending", "true"} -> - [{dir, rev} | Args]; - {"since", _} -> - [{since, Value} | Args]; - {"limit", _} -> - [{limit, to_non_neg_int(Value)} | Args]; - {"heartbeat", "true"} -> - [{heartbeat, true} | Args]; - {"heartbeat", "false"} -> - Args; - {"heartbeat", _} -> - [{heartbeat, to_non_neg_int(Value)} | Args]; - {"timeout", _} -> - [{timeout, to_non_neg_int(Value)} | Args]; - _Else -> % unknown key value pair, ignore. - Args - end - end, [], chttpd:qs(Req)). - + lists:foldl( + fun({Key, Value}, Args) -> + case {Key, Value} of + {"feed", _} -> + [{feed, Value} | Args]; + {"descending", "true"} -> + [{dir, rev} | Args]; + {"since", _} -> + [{since, Value} | Args]; + {"limit", _} -> + [{limit, to_non_neg_int(Value)} | Args]; + {"heartbeat", "true"} -> + [{heartbeat, true} | Args]; + {"heartbeat", "false"} -> + Args; + {"heartbeat", _} -> + [{heartbeat, to_non_neg_int(Value)} | Args]; + {"timeout", _} -> + [{timeout, to_non_neg_int(Value)} | Args]; + % unknown key value pair, ignore. + _Else -> + Args + end + end, + [], + chttpd:qs(Req) + ). to_non_neg_int(Value) -> try list_to_integer(Value) of @@ -269,17 +281,18 @@ to_non_neg_int(Value) -> V; _ -> throw({bad_request, invalid_integer}) - catch error:badarg -> - throw({bad_request, invalid_integer}) + catch + error:badarg -> + throw({bad_request, invalid_integer}) end. allowed_owner(Req) -> case config:get("global_changes", "allowed_owner", undefined) of - undefined -> - chttpd:verify_is_server_admin(Req), - admin; - SpecStr -> - {ok, {M, F, A}} = couch_util:parse_term(SpecStr), - couch_util:validate_callback_exists(M, F, 2), - M:F(Req, A) + undefined -> + chttpd:verify_is_server_admin(Req), + admin; + SpecStr -> + {ok, {M, F, A}} = couch_util:parse_term(SpecStr), + couch_util:validate_callback_exists(M, F, 2), + M:F(Req, A) end. diff --git a/src/global_changes/src/global_changes_listener.erl b/src/global_changes/src/global_changes_listener.erl index 9adf0e13d..71d14e274 100644 --- a/src/global_changes/src/global_changes_listener.erl +++ b/src/global_changes/src/global_changes_listener.erl @@ -13,7 +13,6 @@ -module(global_changes_listener). -behavior(couch_event_listener). - -export([ start/0 ]). @@ -35,131 +34,142 @@ dbname }). - -include_lib("mem3/include/mem3.hrl"). - start() -> couch_event_listener:start(?MODULE, nil, [all_dbs]). - init(_) -> % get configs as strings UpdateDb0 = config:get("global_changes", "update_db", "true"), MaxEventDelay0 = config:get("global_changes", "max_event_delay", "25"), % make config strings into other data types - UpdateDb = case UpdateDb0 of "false" -> false; _ -> true end, + UpdateDb = + case UpdateDb0 of + "false" -> false; + _ -> true + end, MaxEventDelay = list_to_integer(MaxEventDelay0), State = #state{ - update_db=UpdateDb, - pending_update_count=0, - pending_updates=sets:new(), - max_event_delay=MaxEventDelay, - dbname=global_changes_util:get_dbname() + update_db = UpdateDb, + pending_update_count = 0, + pending_updates = sets:new(), + max_event_delay = MaxEventDelay, + dbname = global_changes_util:get_dbname() }, {ok, State}. - terminate(_Reason, _State) -> ok. - -handle_event(_ShardName, _Event, #state{update_db=false}=State) -> +handle_event(_ShardName, _Event, #state{update_db = false} = State) -> {ok, State}; -handle_event(ShardName, Event, State0) - when Event =:= updated orelse Event =:= deleted - orelse Event =:= created -> - #state{dbname=ChangesDbName} = State0, - State = case mem3:dbname(ShardName) of - ChangesDbName -> - State0; - DbName -> - #state{pending_update_count=Count} = State0, - EventBin = erlang:atom_to_binary(Event, latin1), - Key = <<EventBin/binary, <<":">>/binary, DbName/binary>>, - Pending = sets:add_element(Key, State0#state.pending_updates), - couch_stats:update_gauge( - [global_changes, listener_pending_updates], - Count + 1 - ), - State0#state{pending_updates=Pending, pending_update_count=Count+1} - end, +handle_event(ShardName, Event, State0) when + Event =:= updated orelse Event =:= deleted orelse + Event =:= created +-> + #state{dbname = ChangesDbName} = State0, + State = + case mem3:dbname(ShardName) of + ChangesDbName -> + State0; + DbName -> + #state{pending_update_count = Count} = State0, + EventBin = erlang:atom_to_binary(Event, latin1), + Key = <<EventBin/binary, <<":">>/binary, DbName/binary>>, + Pending = sets:add_element(Key, State0#state.pending_updates), + couch_stats:update_gauge( + [global_changes, listener_pending_updates], + Count + 1 + ), + State0#state{pending_updates = Pending, pending_update_count = Count + 1} + end, maybe_send_updates(State); handle_event(_DbName, _Event, State) -> maybe_send_updates(State). - handle_cast({set_max_event_delay, MaxEventDelay}, State) -> - maybe_send_updates(State#state{max_event_delay=MaxEventDelay}); + maybe_send_updates(State#state{max_event_delay = MaxEventDelay}); handle_cast({set_update_db, Boolean}, State0) -> % If turning update_db off, clear out server state - State = case {Boolean, State0#state.update_db} of - {false, true} -> - State0#state{ - update_db=Boolean, - pending_updates=sets:new(), - pending_update_count=0, - last_update_time=undefined - }; - _ -> - State0#state{update_db=Boolean} - end, + State = + case {Boolean, State0#state.update_db} of + {false, true} -> + State0#state{ + update_db = Boolean, + pending_updates = sets:new(), + pending_update_count = 0, + last_update_time = undefined + }; + _ -> + State0#state{update_db = Boolean} + end, maybe_send_updates(State); handle_cast(_Msg, State) -> maybe_send_updates(State). - -maybe_send_updates(#state{pending_update_count=0}=State) -> +maybe_send_updates(#state{pending_update_count = 0} = State) -> {ok, State}; -maybe_send_updates(#state{update_db=true}=State) -> - #state{max_event_delay=MaxEventDelay, last_update_time=LastUpdateTime} = State, +maybe_send_updates(#state{update_db = true} = State) -> + #state{max_event_delay = MaxEventDelay, last_update_time = LastUpdateTime} = State, Now = os:timestamp(), case LastUpdateTime of - undefined -> - {ok, State#state{last_update_time=Now}, MaxEventDelay}; - _ -> - Delta = timer:now_diff(Now, LastUpdateTime) div 1000, - if Delta >= MaxEventDelay -> - Updates = sets:to_list(State#state.pending_updates), - try group_updates_by_node(State#state.dbname, Updates) of - Grouped -> - dict:map(fun(Node, Docs) -> - couch_stats:increment_counter([global_changes, rpcs]), - global_changes_server:update_docs(Node, Docs) - end, Grouped) - catch error:database_does_not_exist -> - ok - end, - couch_stats:update_gauge( - [global_changes, listener_pending_updates], - 0 - ), - State1 = State#state{ - pending_updates=sets:new(), - pending_update_count=0, - last_update_time=undefined - }, - {ok, State1}; - true -> - {ok, State, MaxEventDelay-Delta} - end + undefined -> + {ok, State#state{last_update_time = Now}, MaxEventDelay}; + _ -> + Delta = timer:now_diff(Now, LastUpdateTime) div 1000, + if + Delta >= MaxEventDelay -> + Updates = sets:to_list(State#state.pending_updates), + try group_updates_by_node(State#state.dbname, Updates) of + Grouped -> + dict:map( + fun(Node, Docs) -> + couch_stats:increment_counter([global_changes, rpcs]), + global_changes_server:update_docs(Node, Docs) + end, + Grouped + ) + catch + error:database_does_not_exist -> + ok + end, + couch_stats:update_gauge( + [global_changes, listener_pending_updates], + 0 + ), + State1 = State#state{ + pending_updates = sets:new(), + pending_update_count = 0, + last_update_time = undefined + }, + {ok, State1}; + true -> + {ok, State, MaxEventDelay - Delta} + end end; maybe_send_updates(State) -> {ok, State}. - handle_info(_Msg, State) -> maybe_send_updates(State). - %% restore spec when R14 support is dropped %% -spec group_updates_by_node(binary(), [binary()]) -> dict:dict(). group_updates_by_node(DbName, Updates) -> - lists:foldl(fun(Key, OuterAcc) -> - Shards = mem3:shards(DbName, Key), - lists:foldl(fun(#shard{node=Node}, InnerAcc) -> - dict:append(Node, Key, InnerAcc) - end, OuterAcc, Shards) - end, dict:new(), Updates). + lists:foldl( + fun(Key, OuterAcc) -> + Shards = mem3:shards(DbName, Key), + lists:foldl( + fun(#shard{node = Node}, InnerAcc) -> + dict:append(Node, Key, InnerAcc) + end, + OuterAcc, + Shards + ) + end, + dict:new(), + Updates + ). diff --git a/src/global_changes/src/global_changes_plugin.erl b/src/global_changes/src/global_changes_plugin.erl index 96bb91eaa..8e42d56bd 100644 --- a/src/global_changes/src/global_changes_plugin.erl +++ b/src/global_changes/src/global_changes_plugin.erl @@ -18,7 +18,6 @@ -define(SERVICE_ID, global_changes). - %% ------------------------------------------------------------------ %% API Function Definitions %% ------------------------------------------------------------------ diff --git a/src/global_changes/src/global_changes_server.erl b/src/global_changes/src/global_changes_server.erl index 7e3062586..e4902e207 100644 --- a/src/global_changes/src/global_changes_server.erl +++ b/src/global_changes/src/global_changes_server.erl @@ -14,7 +14,6 @@ -behaviour(gen_server). -vsn(1). - -export([ start_link/0 ]). @@ -32,7 +31,6 @@ update_docs/2 ]). - -include_lib("couch/include/couch_db.hrl"). -include_lib("mem3/include/mem3.hrl"). @@ -45,11 +43,9 @@ handler_ref }). - start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - init([]) -> {ok, Handler} = global_changes_listener:start(), % get configs as strings @@ -57,32 +53,33 @@ init([]) -> MaxWriteDelay0 = config:get("global_changes", "max_write_delay", "500"), % make config strings into other data types - UpdateDb = case UpdateDb0 of "false" -> false; _ -> true end, + UpdateDb = + case UpdateDb0 of + "false" -> false; + _ -> true + end, MaxWriteDelay = list_to_integer(MaxWriteDelay0), % Start our write triggers erlang:send_after(MaxWriteDelay, self(), flush_updates), State = #state{ - update_db=UpdateDb, - pending_update_count=0, - pending_updates=sets:new(), - max_write_delay=MaxWriteDelay, - dbname=global_changes_util:get_dbname(), - handler_ref=erlang:monitor(process, Handler) + update_db = UpdateDb, + pending_update_count = 0, + pending_updates = sets:new(), + max_write_delay = MaxWriteDelay, + dbname = global_changes_util:get_dbname(), + handler_ref = erlang:monitor(process, Handler) }, {ok, State}. - terminate(_Reason, _Srv) -> ok. - handle_call(_Msg, _From, State) -> {reply, ok, State}. - -handle_cast(_Msg, #state{update_db=false}=State) -> +handle_cast(_Msg, #state{update_db = false} = State) -> {noreply, State}; handle_cast({update_docs, DocIds}, State) -> Pending = sets:union(sets:from_list(DocIds), State#state.pending_updates), @@ -92,35 +89,34 @@ handle_cast({update_docs, DocIds}, State) -> PendingCount ), NewState = State#state{ - pending_updates=Pending, - pending_update_count=PendingCount + pending_updates = Pending, + pending_update_count = PendingCount }, {noreply, NewState}; - handle_cast({set_max_write_delay, MaxWriteDelay}, State) -> - NewState = State#state{max_write_delay=MaxWriteDelay}, + NewState = State#state{max_write_delay = MaxWriteDelay}, {noreply, NewState}; handle_cast({set_update_db, Boolean}, State0) -> % If turning update_db off, clear out server state - State = case {Boolean, State0#state.update_db} of - {false, true} -> - State0#state{ - update_db=Boolean, - pending_updates=sets:new(), - pending_update_count=0 - }; - _ -> - State0#state{update_db=Boolean} - end, + State = + case {Boolean, State0#state.update_db} of + {false, true} -> + State0#state{ + update_db = Boolean, + pending_updates = sets:new(), + pending_update_count = 0 + }; + _ -> + State0#state{update_db = Boolean} + end, {noreply, State}; handle_cast(_Msg, State) -> {noreply, State}. - -handle_info(flush_updates, #state{pending_update_count=0}=State) -> +handle_info(flush_updates, #state{pending_update_count = 0} = State) -> erlang:send_after(State#state.max_write_delay, self(), flush_updates), {noreply, State}; -handle_info(flush_updates, #state{update_db=false}=State) -> +handle_info(flush_updates, #state{update_db = false} = State) -> erlang:send_after(State#state.max_write_delay, self(), flush_updates), {noreply, State}; handle_info(flush_updates, State) -> @@ -129,88 +125,97 @@ handle_info(flush_updates, State) -> handle_info(start_listener, State) -> {ok, Handler} = global_changes_listener:start(), NewState = State#state{ - handler_ref=erlang:monitor(process, Handler) + handler_ref = erlang:monitor(process, Handler) }, {noreply, NewState}; -handle_info({'DOWN', Ref, _, _, Reason}, #state{handler_ref=Ref}=State) -> +handle_info({'DOWN', Ref, _, _, Reason}, #state{handler_ref = Ref} = State) -> couch_log:error("global_changes_listener terminated: ~w", [Reason]), erlang:send_after(5000, self(), start_listener), {noreply, State}; handle_info(_, State) -> {noreply, State}. - code_change(_OldVsn, State, _Extra) -> {ok, State}. - - flush_updates(State) -> DocIds = sets:to_list(State#state.pending_updates), try group_ids_by_shard(State#state.dbname, DocIds) of - GroupedIds -> - Docs = dict:fold(fun(ShardName, Ids, DocInfoAcc) -> - {ok, Shard} = couch_db:open(ShardName, [?ADMIN_CTX]), - try - GroupedDocs = get_docs_locally(Shard, Ids), - GroupedDocs ++ DocInfoAcc - after - couch_db:close(Shard) - end - end, [], GroupedIds), - - spawn(fun() -> - fabric:update_docs(State#state.dbname, Docs, []) - end), - - Count = State#state.pending_update_count, - couch_stats:increment_counter( - [global_changes, db_writes], - Count - ) - catch error:database_does_not_exist -> - {noreply, State} + GroupedIds -> + Docs = dict:fold( + fun(ShardName, Ids, DocInfoAcc) -> + {ok, Shard} = couch_db:open(ShardName, [?ADMIN_CTX]), + try + GroupedDocs = get_docs_locally(Shard, Ids), + GroupedDocs ++ DocInfoAcc + after + couch_db:close(Shard) + end + end, + [], + GroupedIds + ), + + spawn(fun() -> + fabric:update_docs(State#state.dbname, Docs, []) + end), + + Count = State#state.pending_update_count, + couch_stats:increment_counter( + [global_changes, db_writes], + Count + ) + catch + error:database_does_not_exist -> + {noreply, State} end, couch_stats:update_gauge( [global_changes, server_pending_updates], 0 ), {noreply, State#state{ - pending_updates=sets:new(), - pending_update_count=0 + pending_updates = sets:new(), + pending_update_count = 0 }}. - update_docs(Node, Updates) -> gen_server:cast({?MODULE, Node}, {update_docs, Updates}). - group_ids_by_shard(DbName, DocIds) -> LocalNode = node(), - lists:foldl(fun(DocId, Acc) -> - Shards = mem3:shards(DbName, DocId), - lists:foldl(fun - (#shard{node=Node, name=Name}, Acc1) when Node == LocalNode -> - dict:append(Name, DocId, Acc1); - (_, Acc1) -> - Acc1 - end, Acc, Shards) - end, dict:new(), DocIds). - + lists:foldl( + fun(DocId, Acc) -> + Shards = mem3:shards(DbName, DocId), + lists:foldl( + fun + (#shard{node = Node, name = Name}, Acc1) when Node == LocalNode -> + dict:append(Name, DocId, Acc1); + (_, Acc1) -> + Acc1 + end, + Acc, + Shards + ) + end, + dict:new(), + DocIds + ). get_docs_locally(Shard, Ids) -> - lists:map(fun(Id) -> - DocInfo = couch_db:get_doc_info(Shard, Id), - #doc{id=Id, revs=get_rev(DocInfo)} - end, Ids). - + lists:map( + fun(Id) -> + DocInfo = couch_db:get_doc_info(Shard, Id), + #doc{id = Id, revs = get_rev(DocInfo)} + end, + Ids + ). get_rev(not_found) -> {0, []}; -get_rev({ok, #doc_info{revs=[RevInfo]}}) -> +get_rev({ok, #doc_info{revs = [RevInfo]}}) -> {Pos, Rev} = RevInfo#rev_info.rev, {Pos, [Rev]}; -get_rev({ok, #doc_info{revs=[RevInfo|_]}}) -> +get_rev({ok, #doc_info{revs = [RevInfo | _]}}) -> % couch_doc:to_doc_info/1 sorts things so that the first % #rev_info in the list is the "winning" revision which is % the one we'd want to base our edit off of. In theory diff --git a/src/global_changes/src/global_changes_sup.erl b/src/global_changes/src/global_changes_sup.erl index 59a40f26a..3229ac0d3 100644 --- a/src/global_changes/src/global_changes_sup.erl +++ b/src/global_changes/src/global_changes_sup.erl @@ -13,7 +13,6 @@ -module(global_changes_sup). -behavior(supervisor). - -export([start_link/0]). -export([init/1]). @@ -27,10 +26,10 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). - init([]) -> {ok, { - {one_for_one, 5, 10}, couch_epi:register_service(global_changes_epi, [ + {one_for_one, 5, 10}, + couch_epi:register_service(global_changes_epi, [ { config_listener_mon, {config_listener_mon, start_link, [?MODULE, nil]}, @@ -47,36 +46,35 @@ init([]) -> worker, [global_changes_server] } - ])}}. + ]) + }}. handle_config_change("global_changes", "max_event_delay", MaxDelayStr, _, _) -> try list_to_integer(MaxDelayStr) of MaxDelay -> gen_server:cast(?LISTENER, {set_max_event_delay, MaxDelay}) - catch error:badarg -> - ok + catch + error:badarg -> + ok end, {ok, nil}; - handle_config_change("global_changes", "max_write_delay", MaxDelayStr, _, _) -> try list_to_integer(MaxDelayStr) of MaxDelay -> gen_server:cast(?SERVER, {set_max_write_delay, MaxDelay}) - catch error:badarg -> - ok + catch + error:badarg -> + ok end, {ok, nil}; - handle_config_change("global_changes", "update_db", "false", _, _) -> gen_server:cast(?LISTENER, {set_update_db, false}), gen_server:cast(?SERVER, {set_update_db, false}), {ok, nil}; - handle_config_change("global_changes", "update_db", _, _, _) -> gen_server:cast(?LISTENER, {set_update_db, true}), gen_server:cast(?SERVER, {set_update_db, true}), {ok, nil}; - handle_config_change(_, _, _, _, _) -> {ok, nil}. diff --git a/src/global_changes/src/global_changes_util.erl b/src/global_changes/src/global_changes_util.erl index 0ca57a35f..910e0137f 100644 --- a/src/global_changes/src/global_changes_util.erl +++ b/src/global_changes/src/global_changes_util.erl @@ -12,10 +12,8 @@ -module(global_changes_util). - -export([get_dbname/0]). - get_dbname() -> case application:get_env(global_changes, dbname) of {ok, DbName} when is_binary(DbName) -> diff --git a/src/global_changes/test/eunit/global_changes_hooks_tests.erl b/src/global_changes/test/eunit/global_changes_hooks_tests.erl index 3c2fcc6e6..5d6bbd13d 100644 --- a/src/global_changes/test/eunit/global_changes_hooks_tests.erl +++ b/src/global_changes/test/eunit/global_changes_hooks_tests.erl @@ -38,8 +38,12 @@ setup(default) -> get_host(); setup(A) -> Host = setup(default), - ok = config:set("global_changes", "allowed_owner", - ?t2l({?MODULE, allowed_owner, A}), false), + ok = config:set( + "global_changes", + "allowed_owner", + ?t2l({?MODULE, allowed_owner, A}), + false + ), Host. teardown(_) -> @@ -57,7 +61,8 @@ allowed_owner_hook_test_() -> "Check allowed_owner hook", { setup, - fun start/0, fun stop/1, + fun start/0, + fun stop/1, [ disabled_allowed_owner_integration_point(), enabled_allowed_owner_integration_point() @@ -70,11 +75,12 @@ disabled_allowed_owner_integration_point() -> "disabled allowed_owner integration point", { foreach, - fun() -> setup(default) end, fun teardown/1, + fun() -> setup(default) end, + fun teardown/1, [ fun should_not_fail_for_admin/1, fun should_fail_for_non_admin/1 - ] + ] } }. @@ -84,12 +90,14 @@ enabled_allowed_owner_integration_point() -> [ { foreach, - fun() -> setup("throw") end, fun teardown/1, + fun() -> setup("throw") end, + fun teardown/1, [fun should_throw/1] }, { foreach, - fun() -> setup("pass") end, fun teardown/1, + fun() -> setup("pass") end, + fun teardown/1, [fun should_pass/1] } ] diff --git a/src/ioq/src/ioq.erl b/src/ioq/src/ioq.erl index c3f9365bf..51934d544 100644 --- a/src/ioq/src/ioq.erl +++ b/src/ioq/src/ioq.erl @@ -26,9 +26,9 @@ -record(state, { concurrency, ratio, - interactive=queue:new(), - background=queue:new(), - running=[] + interactive = queue:new(), + background = queue:new(), + running = [] }). -record(request, { @@ -85,11 +85,11 @@ io_class(_, _) -> other. queued_call(Fd, Msg, Priority) -> - Request = #request{fd=Fd, msg=Msg, priority=Priority, from=self()}, + Request = #request{fd = Fd, msg = Msg, priority = Priority, from = self()}, try gen_server:call(?MODULE, Request, infinity) catch - exit:{noproc,_} -> + exit:{noproc, _} -> gen_server:call(Fd, Msg, infinity) end. @@ -101,7 +101,7 @@ init(_) -> read_config(State) -> Ratio = config:get_float("ioq", "ratio", 0.01), Concurrency = config:get_integer("ioq", "concurrency", 10), - State#state{concurrency=Concurrency, ratio=Ratio}. + State#state{concurrency = Concurrency, ratio = Ratio}. handle_call(get_queue_lengths, _From, State) -> Response = #{ @@ -109,8 +109,8 @@ handle_call(get_queue_lengths, _From, State) -> background => queue:len(State#state.background) }, {reply, Response, State, 0}; -handle_call(#request{}=Request, From, State) -> - {noreply, enqueue_request(Request#request{from=From}, State), 0}. +handle_call(#request{} = Request, From, State) -> + {noreply, enqueue_request(Request#request{from = From}, State), 0}. handle_cast(change, State) -> {noreply, read_config(State)}; @@ -122,7 +122,7 @@ handle_info({Ref, Reply}, State) -> {value, Request, Remaining} -> erlang:demonitor(Ref, [flush]), gen_server:reply(Request#request.from, Reply), - {noreply, State#state{running=Remaining}, 0}; + {noreply, State#state{running = Remaining}, 0}; false -> {noreply, State, 0} end; @@ -130,7 +130,7 @@ handle_info({'DOWN', Ref, _, _, Reason}, State) -> case lists:keytake(Ref, #request.ref, State#state.running) of {value, Request, Remaining} -> gen_server:reply(Request#request.from, {'EXIT', Reason}), - {noreply, State#state{running=Remaining}, 0}; + {noreply, State#state{running = Remaining}, 0}; false -> {noreply, State, 0} end; @@ -156,15 +156,16 @@ code_change(_Vsn, State, _Extra) -> terminate(_Reason, _State) -> ok. -enqueue_request(#request{priority=compaction}=Request, #state{}=State) -> - State#state{background=queue:in(Request, State#state.background)}; -enqueue_request(#request{priority=shard_sync}=Request, #state{}=State) -> - State#state{background=queue:in(Request, State#state.background)}; -enqueue_request(#request{}=Request, #state{}=State) -> - State#state{interactive=queue:in(Request, State#state.interactive)}. +enqueue_request(#request{priority = compaction} = Request, #state{} = State) -> + State#state{background = queue:in(Request, State#state.background)}; +enqueue_request(#request{priority = shard_sync} = Request, #state{} = State) -> + State#state{background = queue:in(Request, State#state.background)}; +enqueue_request(#request{} = Request, #state{} = State) -> + State#state{interactive = queue:in(Request, State#state.interactive)}. -maybe_submit_request(#state{concurrency=Concurrency, running=Running}=State) - when length(Running) < Concurrency -> +maybe_submit_request(#state{concurrency = Concurrency, running = Running} = State) when + length(Running) < Concurrency +-> case make_next_request(State) of State -> State; @@ -176,7 +177,7 @@ maybe_submit_request(#state{concurrency=Concurrency, running=Running}=State) maybe_submit_request(State) -> State. -make_next_request(#state{}=State) -> +make_next_request(#state{} = State) -> case {queue:is_empty(State#state.background), queue:is_empty(State#state.interactive)} of {true, true} -> State; @@ -201,7 +202,7 @@ choose_next_request(Index, State) -> submit_request(Request, setelement(Index, State, Q)) end. -submit_request(#request{}=Request, #state{}=State) -> +submit_request(#request{} = Request, #state{} = State) -> Ref = erlang:monitor(process, Request#request.fd), Request#request.fd ! {'$gen_call', {self(), Ref}, Request#request.msg}, - State#state{running = [Request#request{ref=Ref} | State#state.running]}. + State#state{running = [Request#request{ref = Ref} | State#state.running]}. diff --git a/src/ioq/src/ioq_sup.erl b/src/ioq/src/ioq_sup.erl index c4d04a9e4..937e5a952 100644 --- a/src/ioq/src/ioq_sup.erl +++ b/src/ioq/src/ioq_sup.erl @@ -21,4 +21,4 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> - {ok, { {one_for_one, 5, 10}, [?CHILD(ioq, worker)]}}. + {ok, {{one_for_one, 5, 10}, [?CHILD(ioq, worker)]}}. diff --git a/src/jwtf/src/jwtf.erl b/src/jwtf/src/jwtf.erl index 4c4f80c70..d62789b0a 100644 --- a/src/jwtf/src/jwtf.erl +++ b/src/jwtf/src/jwtf.erl @@ -25,7 +25,8 @@ ]). -define(ALGS, [ - {<<"RS256">>, {public_key, sha256}}, % RSA PKCS#1 signature with SHA-256 + % RSA PKCS#1 signature with SHA-256 + {<<"RS256">>, {public_key, sha256}}, {<<"RS384">>, {public_key, sha384}}, {<<"RS512">>, {public_key, sha512}}, {<<"ES256">>, {public_key, sha256}}, @@ -33,7 +34,8 @@ {<<"ES512">>, {public_key, sha512}}, {<<"HS256">>, {hmac, sha256}}, {<<"HS384">>, {hmac, sha384}}, - {<<"HS512">>, {hmac, sha512}}]). + {<<"HS512">>, {hmac, sha512}} +]). -define(CHECKS, [ alg, @@ -43,8 +45,8 @@ kid, nbf, sig, - typ]). - + typ +]). % @doc encode % Encode the JSON Header and Claims using Key and Alg obtained from Header @@ -52,21 +54,23 @@ {ok, binary()} | no_return(). encode(Header = {HeaderProps}, Claims, Key) -> try - Alg = case prop(<<"alg">>, HeaderProps) of - undefined -> - throw({bad_request, <<"Missing alg header parameter">>}); - Val -> - Val - end, + Alg = + case prop(<<"alg">>, HeaderProps) of + undefined -> + throw({bad_request, <<"Missing alg header parameter">>}); + Val -> + Val + end, EncodedHeader = b64url:encode(jiffy:encode(Header)), EncodedClaims = b64url:encode(jiffy:encode(Claims)), Message = <<EncodedHeader/binary, $., EncodedClaims/binary>>, - SignatureOrMac = case verification_algorithm(Alg) of - {public_key, Algorithm} -> - public_key:sign(Message, Algorithm, Key); - {hmac, Algorithm} -> - hmac(Algorithm, Key, Message) - end, + SignatureOrMac = + case verification_algorithm(Alg) of + {public_key, Algorithm} -> + public_key:sign(Message, Algorithm, Key); + {hmac, Algorithm} -> + hmac(Algorithm, Key, Message) + end, EncodedSignatureOrMac = b64url:encode(SignatureOrMac), {ok, <<Message/binary, $., EncodedSignatureOrMac/binary>>} catch @@ -74,7 +78,6 @@ encode(Header = {HeaderProps}, Claims, Key) -> {error, Error} end. - % @doc decode % Decodes the supplied encoded token, checking % for the attributes defined in Checks and calling @@ -90,14 +93,12 @@ decode(EncodedToken, Checks, KS) -> {error, Error} end. - % @doc valid_algorithms % Return a list of supported algorithms -spec valid_algorithms() -> [binary()]. valid_algorithms() -> proplists:get_keys(?ALGS). - % @doc verification_algorithm % Return {VerificationMethod, Algorithm} tuple for the specified Alg -spec verification_algorithm(binary()) -> @@ -110,7 +111,6 @@ verification_algorithm(Alg) -> throw({bad_request, <<"Invalid alg header parameter">>}) end. - validate(Header0, Payload0, Signature, Checks, KS) -> validate_checks(Checks), Header1 = props(decode_b64url_json(Header0)), @@ -123,7 +123,6 @@ validate(Header0, Payload0, Signature, Checks, KS) -> Key = key(Header1, Checks, KS), verify(Alg, Header0, Payload0, Signature, Key). - validate_checks(Checks) when is_list(Checks) -> case {lists:usort(Checks), lists:sort(Checks)} of {L, L} -> @@ -139,22 +138,17 @@ validate_checks(Checks) when is_list(Checks) -> error({unknown_checks, UnknownChecks}) end. - valid_check(Check) when is_atom(Check) -> lists:member(Check, ?CHECKS); - valid_check({Check, _}) when is_atom(Check) -> lists:member(Check, ?CHECKS); - valid_check(_) -> false. - validate_header(Props, Checks) -> validate_typ(Props, Checks), validate_alg(Props, Checks). - validate_typ(Props, Checks) -> Required = prop(typ, Checks), TYP = prop(<<"typ">>, Props), @@ -169,7 +163,6 @@ validate_typ(Props, Checks) -> throw({bad_request, <<"Invalid typ header parameter">>}) end. - validate_alg(Props, Checks) -> Required = prop(alg, Checks), Alg = prop(<<"alg">>, Props), @@ -187,7 +180,6 @@ validate_alg(Props, Checks) -> end end. - %% Only validate required checks. validate_payload(Props, Checks) -> validate_iss(Props, Checks), @@ -195,13 +187,13 @@ validate_payload(Props, Checks) -> validate_nbf(Props, Checks), validate_exp(Props, Checks). - validate_iss(Props, Checks) -> ExpectedISS = prop(iss, Checks), ActualISS = prop(<<"iss">>, Props), case {ExpectedISS, ActualISS} of - {undefined, _} -> % ignore unrequired check + % ignore unrequired check + {undefined, _} -> ok; {ISS, undefined} when ISS /= undefined -> throw({bad_request, <<"Missing iss claim">>}); @@ -211,13 +203,13 @@ validate_iss(Props, Checks) -> throw({bad_request, <<"Invalid iss claim">>}) end. - validate_iat(Props, Checks) -> Required = prop(iat, Checks), IAT = prop(<<"iat">>, Props), case {Required, IAT} of - {undefined, _} -> % ignore unrequired check + % ignore unrequired check + {undefined, _} -> ok; {true, undefined} -> throw({bad_request, <<"Missing iat claim">>}); @@ -227,13 +219,13 @@ validate_iat(Props, Checks) -> throw({bad_request, <<"Invalid iat claim">>}) end. - validate_nbf(Props, Checks) -> Required = prop(nbf, Checks), NBF = prop(<<"nbf">>, Props), case {Required, NBF} of - {undefined, _} -> % ignore unrequired check + % ignore unrequired check + {undefined, _} -> ok; {true, undefined} -> throw({bad_request, <<"Missing nbf claim">>}); @@ -241,13 +233,13 @@ validate_nbf(Props, Checks) -> assert_past(<<"nbf">>, NBF) end. - validate_exp(Props, Checks) -> Required = prop(exp, Checks), EXP = prop(<<"exp">>, Props), case {Required, EXP} of - {undefined, _} -> % ignore unrequired check + % ignore unrequired check + {undefined, _} -> ok; {true, undefined} -> throw({bad_request, <<"Missing exp claim">>}); @@ -255,7 +247,6 @@ validate_exp(Props, Checks) -> assert_future(<<"exp">>, EXP) end. - key(Props, Checks, KS) -> Alg = prop(<<"alg">>, Props), Required = prop(kid, Checks), @@ -267,7 +258,6 @@ key(Props, Checks, KS) -> KS(Alg, KID) end. - verify(Alg, Header, Payload, SignatureOrMac0, Key) -> Message = <<Header/binary, $., Payload/binary>>, SignatureOrMac1 = b64url:decode(SignatureOrMac0), @@ -279,7 +269,6 @@ verify(Alg, Header, Payload, SignatureOrMac0, Key) -> hmac_verify(Algorithm, Message, SignatureOrMac1, Key) end. - public_key_verify(Algorithm, Message, Signature, PublicKey) -> case public_key:verify(Message, Algorithm, Signature, PublicKey) of true -> @@ -288,7 +277,6 @@ public_key_verify(Algorithm, Message, Signature, PublicKey) -> throw({bad_request, <<"Bad signature">>}) end. - hmac_verify(Algorithm, Message, HMAC, SecretKey) -> case hmac(Algorithm, SecretKey, Message) of HMAC -> @@ -297,14 +285,12 @@ hmac_verify(Algorithm, Message, HMAC, SecretKey) -> throw({bad_request, <<"Bad HMAC">>}) end. - split(EncodedToken) -> case binary:split(EncodedToken, <<$.>>, [global]) of [_, _, _] = Split -> Split; _ -> throw({bad_request, <<"Malformed token">>}) end. - decode_b64url_json(B64UrlEncoded) -> try case b64url:decode(B64UrlEncoded) of @@ -318,14 +304,11 @@ decode_b64url_json(B64UrlEncoded) -> throw({bad_request, Error}) end. - props({Props}) -> Props; - props(_) -> throw({bad_request, <<"Not an object">>}). - assert_past(Name, Time) -> case Time < now_seconds() of true -> @@ -342,16 +325,13 @@ assert_future(Name, Time) -> throw({unauthorized, <<Name/binary, " not in future">>}) end. - now_seconds() -> {MegaSecs, Secs, _MicroSecs} = os:timestamp(), MegaSecs * 1000000 + Secs. - prop(Prop, Props) -> proplists:get_value(Prop, Props). - -ifdef(OTP_RELEASE). -if(?OTP_RELEASE >= 22). @@ -366,7 +346,8 @@ hmac(Alg, Key, Message) -> hmac(Alg, Key, Message) -> crypto:hmac(Alg, Key, Message). --endif. % -if(?OTP_RELEASE >= 22) +% -if(?OTP_RELEASE >= 22) +-endif. -else. @@ -374,8 +355,8 @@ hmac(Alg, Key, Message) -> hmac(Alg, Key, Message) -> crypto:hmac(Alg, Key, Message). --endif. % -ifdef(OTP_RELEASE) - +% -ifdef(OTP_RELEASE) +-endif. -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). diff --git a/src/jwtf/src/jwtf_keystore.erl b/src/jwtf/src/jwtf_keystore.erl index be261e67c..4c2933264 100644 --- a/src/jwtf/src/jwtf_keystore.erl +++ b/src/jwtf/src/jwtf_keystore.erl @@ -23,8 +23,14 @@ ]). % gen_server api. --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - code_change/3, terminate/2]). +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + code_change/3, + terminate/2 +]). % config_listener api -export([handle_config_change/5, handle_config_terminate/3]). @@ -33,7 +39,6 @@ get(Alg, undefined) when is_binary(Alg) -> get(Alg, <<"_default">>); - get(Alg, KID0) when is_binary(Alg), is_binary(KID0) -> Kty = kty(Alg), KID = binary_to_list(KID0), @@ -43,10 +48,9 @@ get(Alg, KID0) when is_binary(Alg), is_binary(KID0) -> ok = gen_server:call(?MODULE, {set, Kty, KID, Key}), Key; [{{Kty, KID}, Key}] -> - Key + Key end. - start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). @@ -57,36 +61,28 @@ init(_) -> ets:new(?MODULE, [public, named_table]), {ok, nil}. - handle_call({set, Kty, KID, Key}, _From, State) -> true = ets:insert(?MODULE, {{Kty, KID}, Key}), {reply, ok, State}. - handle_cast({delete, Kty, KID}, State) -> true = ets:delete(?MODULE, {Kty, KID}), {noreply, State}; - handle_cast(_Msg, State) -> {noreply, State}. - handle_info(restart_config_listener, State) -> ok = config:listen_for_changes(?MODULE, nil), {noreply, State}; - handle_info(_Msg, State) -> {noreply, State}. - terminate(_Reason, _State) -> ok. - code_change(_OldVsn, State, _Extra) -> {ok, State}. - % config listener callback handle_config_change("jwt_keys", ConfigKey, _ConfigValue, _, _) -> @@ -97,13 +93,11 @@ handle_config_change("jwt_keys", ConfigKey, _ConfigValue, _, _) -> ignored end, {ok, nil}; - handle_config_change(_, _, _, _, _) -> {ok, nil}. handle_config_terminate(_Server, stop, _State) -> ok; - handle_config_terminate(_Server, _Reason, _State) -> erlang:send_after(100, whereis(?MODULE), restart_config_listener). @@ -150,12 +144,9 @@ pem_decode(PEM) -> kty(<<"HS", _/binary>>) -> "hmac"; - kty(<<"RS", _/binary>>) -> "rsa"; - kty(<<"ES", _/binary>>) -> "ec"; - kty(_) -> throw({bad_request, <<"Unknown kty">>}). diff --git a/src/jwtf/src/jwtf_sup.erl b/src/jwtf/src/jwtf_sup.erl index 6f44808de..98d354c96 100644 --- a/src/jwtf/src/jwtf_sup.erl +++ b/src/jwtf/src/jwtf_sup.erl @@ -35,4 +35,4 @@ start_link() -> %% =================================================================== init([]) -> - {ok, { {one_for_one, 5, 10}, [?CHILD(jwtf_keystore, worker)]} }. + {ok, {{one_for_one, 5, 10}, [?CHILD(jwtf_keystore, worker)]}}. diff --git a/src/jwtf/test/jwtf_keystore_tests.erl b/src/jwtf/test/jwtf_keystore_tests.erl index c0c55fb0b..c05d7f1b4 100644 --- a/src/jwtf/test/jwtf_keystore_tests.erl +++ b/src/jwtf/test/jwtf_keystore_tests.erl @@ -16,8 +16,12 @@ -include_lib("public_key/include/public_key.hrl"). -define(HMAC_SECRET, "aGVsbG8="). --define(RSA_SECRET, "-----BEGIN PUBLIC KEY-----\\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAztanwQtIx0sms+x7m1SF\\nh7EHJHkM2biTJ41jR89FsDE2gd3MChpaqxemS5GpNvfFKRvuHa4PUZ3JtRCBG1KM\\n/7EWIVTy1JQDr2mb8couGlQNqz4uXN2vkNQ0XszgjU4Wn6ZpvYxmqPFbmkRe8QSn\\nAy2Wf8jQgjsbez8eaaX0G9S1hgFZUN3KFu7SVmUDQNvWpQdaJPP+ms5Z0CqF7JLa\\nvJmSdsU49nlYw9VH/XmwlUBMye6HgR4ZGCLQS85frqF0xLWvi7CsMdchcIjHudXH\\nQK1AumD/VVZVdi8Q5Qew7F6VXeXqnhbw9n6Px25cCuNuh6u5+E6GUzXRrMpqo9vO\\nqQIDAQAB\\n-----END PUBLIC KEY-----\\n"). --define(EC_SECRET, "-----BEGIN PUBLIC KEY-----\\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEDsr0lz/Dg3luarb+Kua0Wcj9WrfR23os\\nwHzakglb8GhWRDn+oZT0Bt/26sX8uB4/ij9PEOLHPo+IHBtX4ELFFVr5GTzlqcJe\\nyctaTDd1OOAPXYuc67EWtGZ3pDAzztRs\\n-----END PUBLIC KEY-----\\n"). +-define(RSA_SECRET, + "-----BEGIN PUBLIC KEY-----\\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAztanwQtIx0sms+x7m1SF\\nh7EHJHkM2biTJ41jR89FsDE2gd3MChpaqxemS5GpNvfFKRvuHa4PUZ3JtRCBG1KM\\n/7EWIVTy1JQDr2mb8couGlQNqz4uXN2vkNQ0XszgjU4Wn6ZpvYxmqPFbmkRe8QSn\\nAy2Wf8jQgjsbez8eaaX0G9S1hgFZUN3KFu7SVmUDQNvWpQdaJPP+ms5Z0CqF7JLa\\nvJmSdsU49nlYw9VH/XmwlUBMye6HgR4ZGCLQS85frqF0xLWvi7CsMdchcIjHudXH\\nQK1AumD/VVZVdi8Q5Qew7F6VXeXqnhbw9n6Px25cCuNuh6u5+E6GUzXRrMpqo9vO\\nqQIDAQAB\\n-----END PUBLIC KEY-----\\n" +). +-define(EC_SECRET, + "-----BEGIN PUBLIC KEY-----\\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEDsr0lz/Dg3luarb+Kua0Wcj9WrfR23os\\nwHzakglb8GhWRDn+oZT0Bt/26sX8uB4/ij9PEOLHPo+IHBtX4ELFFVr5GTzlqcJe\\nyctaTDd1OOAPXYuc67EWtGZ3pDAzztRs\\n-----END PUBLIC KEY-----\\n" +). setup() -> test_util:start_applications([couch_log, config, jwtf]), @@ -38,20 +42,20 @@ teardown(_) -> jwtf_keystore_test_() -> { - setup, - fun setup/0, - fun teardown/1, - [ - ?_assertEqual(<<"hello">>, jwtf_keystore:get(<<"HS256">>, <<"hmac">>)), - ?_assertThrow({bad_request, _}, jwtf_keystore:get(<<"RS256">>, <<"hmac">>)), - ?_assertThrow({bad_request, _}, jwtf_keystore:get(<<"ES256">>, <<"hmac">>)), - - ?_assertThrow({bad_request, _}, jwtf_keystore:get(<<"HS256">>, <<"rsa">>)), - ?_assertMatch(#'RSAPublicKey'{}, jwtf_keystore:get(<<"RS256">>, <<"rsa">>)), - ?_assertThrow({bad_request, _}, jwtf_keystore:get(<<"ES256">>, <<"rsa">>)), - - ?_assertThrow({bad_request, _}, jwtf_keystore:get(<<"HS256">>, <<"ec">>)), - ?_assertThrow({bad_request, _}, jwtf_keystore:get(<<"RS256">>, <<"ec">>)), - ?_assertMatch({#'ECPoint'{}, _}, jwtf_keystore:get(<<"ES256">>, <<"ec">>)) - ] + setup, + fun setup/0, + fun teardown/1, + [ + ?_assertEqual(<<"hello">>, jwtf_keystore:get(<<"HS256">>, <<"hmac">>)), + ?_assertThrow({bad_request, _}, jwtf_keystore:get(<<"RS256">>, <<"hmac">>)), + ?_assertThrow({bad_request, _}, jwtf_keystore:get(<<"ES256">>, <<"hmac">>)), + + ?_assertThrow({bad_request, _}, jwtf_keystore:get(<<"HS256">>, <<"rsa">>)), + ?_assertMatch(#'RSAPublicKey'{}, jwtf_keystore:get(<<"RS256">>, <<"rsa">>)), + ?_assertThrow({bad_request, _}, jwtf_keystore:get(<<"ES256">>, <<"rsa">>)), + + ?_assertThrow({bad_request, _}, jwtf_keystore:get(<<"HS256">>, <<"ec">>)), + ?_assertThrow({bad_request, _}, jwtf_keystore:get(<<"RS256">>, <<"ec">>)), + ?_assertMatch({#'ECPoint'{}, _}, jwtf_keystore:get(<<"ES256">>, <<"ec">>)) + ] }. diff --git a/src/jwtf/test/jwtf_tests.erl b/src/jwtf/test/jwtf_tests.erl index ba944f7c7..f4685a54e 100644 --- a/src/jwtf/test/jwtf_tests.erl +++ b/src/jwtf/test/jwtf_tests.erl @@ -25,247 +25,286 @@ valid_header() -> {[{<<"typ">>, <<"JWT">>}, {<<"alg">>, <<"RS256">>}]}. jwt_io_pubkey() -> - PublicKeyPEM = <<"-----BEGIN PUBLIC KEY-----\n" - "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDdlatRjRjogo3WojgGH" - "FHYLugdUWAY9iR3fy4arWNA1KoS8kVw33cJibXr8bvwUAUparCwlvdbH6" - "dvEOfou0/gCFQsHUfQrSDv+MuSUMAe8jzKE4qW+jK+xQU9a03GUnKHkkl" - "e+Q0pX/g6jXZ7r1/xAK5Do2kQ+X5xK9cipRgEKwIDAQAB\n" - "-----END PUBLIC KEY-----\n">>, + PublicKeyPEM = << + "-----BEGIN PUBLIC KEY-----\n" + "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDdlatRjRjogo3WojgGH" + "FHYLugdUWAY9iR3fy4arWNA1KoS8kVw33cJibXr8bvwUAUparCwlvdbH6" + "dvEOfou0/gCFQsHUfQrSDv+MuSUMAe8jzKE4qW+jK+xQU9a03GUnKHkkl" + "e+Q0pX/g6jXZ7r1/xAK5Do2kQ+X5xK9cipRgEKwIDAQAB\n" + "-----END PUBLIC KEY-----\n" + >>, [PEMEntry] = public_key:pem_decode(PublicKeyPEM), public_key:pem_entry_decode(PEMEntry). - b64_badarg_test() -> Encoded = <<"0.0.0">>, - ?assertEqual({error, {bad_request,badarg}}, - jwtf:decode(Encoded, [], nil)). - + ?assertEqual( + {error, {bad_request, badarg}}, + jwtf:decode(Encoded, [], nil) + ). b64_bad_block_test() -> Encoded = <<" aGVsbG8. aGVsbG8. aGVsbG8">>, - ?assertEqual({error, {bad_request,{bad_block,0}}}, - jwtf:decode(Encoded, [], nil)). - + ?assertEqual( + {error, {bad_request, {bad_block, 0}}}, + jwtf:decode(Encoded, [], nil) + ). invalid_json_test() -> Encoded = <<"fQ.fQ.fQ">>, - ?assertEqual({error, {bad_request,{1,invalid_json}}}, - jwtf:decode(Encoded, [], nil)). - + ?assertEqual( + {error, {bad_request, {1, invalid_json}}}, + jwtf:decode(Encoded, [], nil) + ). truncated_json_test() -> Encoded = <<"ew.ew.ew">>, - ?assertEqual({error, {bad_request,{2,truncated_json}}}, - jwtf:decode(Encoded, [], nil)). - + ?assertEqual( + {error, {bad_request, {2, truncated_json}}}, + jwtf:decode(Encoded, [], nil) + ). missing_typ_test() -> Encoded = encode({[]}, []), - ?assertEqual({error, {bad_request,<<"Missing typ header parameter">>}}, - jwtf:decode(Encoded, [typ], nil)). - + ?assertEqual( + {error, {bad_request, <<"Missing typ header parameter">>}}, + jwtf:decode(Encoded, [typ], nil) + ). invalid_typ_test() -> Encoded = encode({[{<<"typ">>, <<"NOPE">>}]}, []), - ?assertEqual({error, {bad_request,<<"Invalid typ header parameter">>}}, - jwtf:decode(Encoded, [typ], nil)). - + ?assertEqual( + {error, {bad_request, <<"Invalid typ header parameter">>}}, + jwtf:decode(Encoded, [typ], nil) + ). missing_alg_test() -> Encoded = encode({[]}, []), - ?assertEqual({error, {bad_request,<<"Missing alg header parameter">>}}, - jwtf:decode(Encoded, [alg], nil)). - + ?assertEqual( + {error, {bad_request, <<"Missing alg header parameter">>}}, + jwtf:decode(Encoded, [alg], nil) + ). invalid_alg_test() -> Encoded = encode({[{<<"alg">>, <<"NOPE">>}]}, []), - ?assertEqual({error, {bad_request,<<"Invalid alg header parameter">>}}, - jwtf:decode(Encoded, [alg], nil)). - + ?assertEqual( + {error, {bad_request, <<"Invalid alg header parameter">>}}, + jwtf:decode(Encoded, [alg], nil) + ). missing_iss_test() -> Encoded = encode(valid_header(), {[]}), - ?assertEqual({error, {bad_request,<<"Missing iss claim">>}}, - jwtf:decode(Encoded, [{iss, right}], nil)). - + ?assertEqual( + {error, {bad_request, <<"Missing iss claim">>}}, + jwtf:decode(Encoded, [{iss, right}], nil) + ). invalid_iss_test() -> Encoded = encode(valid_header(), {[{<<"iss">>, <<"wrong">>}]}), - ?assertEqual({error, {bad_request,<<"Invalid iss claim">>}}, - jwtf:decode(Encoded, [{iss, right}], nil)). - + ?assertEqual( + {error, {bad_request, <<"Invalid iss claim">>}}, + jwtf:decode(Encoded, [{iss, right}], nil) + ). missing_iat_test() -> Encoded = encode(valid_header(), {[]}), - ?assertEqual({error, {bad_request,<<"Missing iat claim">>}}, - jwtf:decode(Encoded, [iat], nil)). - + ?assertEqual( + {error, {bad_request, <<"Missing iat claim">>}}, + jwtf:decode(Encoded, [iat], nil) + ). invalid_iat_test() -> Encoded = encode(valid_header(), {[{<<"iat">>, <<"hello">>}]}), - ?assertEqual({error, {bad_request,<<"Invalid iat claim">>}}, - jwtf:decode(Encoded, [iat], nil)). - + ?assertEqual( + {error, {bad_request, <<"Invalid iat claim">>}}, + jwtf:decode(Encoded, [iat], nil) + ). missing_nbf_test() -> Encoded = encode(valid_header(), {[]}), - ?assertEqual({error, {bad_request,<<"Missing nbf claim">>}}, - jwtf:decode(Encoded, [nbf], nil)). - + ?assertEqual( + {error, {bad_request, <<"Missing nbf claim">>}}, + jwtf:decode(Encoded, [nbf], nil) + ). invalid_nbf_test() -> Encoded = encode(valid_header(), {[{<<"nbf">>, 2 * now_seconds()}]}), - ?assertEqual({error, {unauthorized, <<"nbf not in past">>}}, - jwtf:decode(Encoded, [nbf], nil)). - + ?assertEqual( + {error, {unauthorized, <<"nbf not in past">>}}, + jwtf:decode(Encoded, [nbf], nil) + ). missing_exp_test() -> Encoded = encode(valid_header(), {[]}), - ?assertEqual({error, {bad_request, <<"Missing exp claim">>}}, - jwtf:decode(Encoded, [exp], nil)). - + ?assertEqual( + {error, {bad_request, <<"Missing exp claim">>}}, + jwtf:decode(Encoded, [exp], nil) + ). invalid_exp_test() -> Encoded = encode(valid_header(), {[{<<"exp">>, 0}]}), - ?assertEqual({error, {unauthorized, <<"exp not in future">>}}, - jwtf:decode(Encoded, [exp], nil)). - + ?assertEqual( + {error, {unauthorized, <<"exp not in future">>}}, + jwtf:decode(Encoded, [exp], nil) + ). missing_kid_test() -> Encoded = encode({[]}, {[]}), - ?assertEqual({error, {bad_request, <<"Missing kid claim">>}}, - jwtf:decode(Encoded, [kid], nil)). - + ?assertEqual( + {error, {bad_request, <<"Missing kid claim">>}}, + jwtf:decode(Encoded, [kid], nil) + ). public_key_not_found_test() -> Encoded = encode( {[{<<"alg">>, <<"RS256">>}, {<<"kid">>, <<"1">>}]}, - {[]}), + {[]} + ), KS = fun(_, _) -> throw(not_found) end, Expected = {error, not_found}, ?assertEqual(Expected, jwtf:decode(Encoded, [], KS)). - bad_rs256_sig_test() -> Encoded = encode( {[{<<"typ">>, <<"JWT">>}, {<<"alg">>, <<"RS256">>}]}, - {[]}), + {[]} + ), KS = fun(<<"RS256">>, undefined) -> jwt_io_pubkey() end, - ?assertEqual({error, {bad_request, <<"Bad signature">>}}, - jwtf:decode(Encoded, [], KS)). - + ?assertEqual( + {error, {bad_request, <<"Bad signature">>}}, + jwtf:decode(Encoded, [], KS) + ). bad_hs256_sig_test() -> Encoded = encode( {[{<<"typ">>, <<"JWT">>}, {<<"alg">>, <<"HS256">>}]}, - {[]}), + {[]} + ), KS = fun(<<"HS256">>, undefined) -> <<"bad">> end, - ?assertEqual({error, {bad_request, <<"Bad HMAC">>}}, - jwtf:decode(Encoded, [], KS)). - + ?assertEqual( + {error, {bad_request, <<"Bad HMAC">>}}, + jwtf:decode(Encoded, [], KS) + ). malformed_token_test() -> - ?assertEqual({error, {bad_request, <<"Malformed token">>}}, - jwtf:decode(<<"a.b.c.d">>, [], nil)). + ?assertEqual( + {error, {bad_request, <<"Malformed token">>}}, + jwtf:decode(<<"a.b.c.d">>, [], nil) + ). unknown_atom_check_test() -> - ?assertError({unknown_checks, [foo, bar]}, - jwtf:decode(<<"a.b.c">>, [exp, foo, iss, bar], nil)). + ?assertError( + {unknown_checks, [foo, bar]}, + jwtf:decode(<<"a.b.c">>, [exp, foo, iss, bar], nil) + ). unknown_binary_check_test() -> - ?assertError({unknown_checks, [<<"bar">>]}, - jwtf:decode(<<"a.b.c">>, [exp, iss, <<"bar">>], nil)). + ?assertError( + {unknown_checks, [<<"bar">>]}, + jwtf:decode(<<"a.b.c">>, [exp, iss, <<"bar">>], nil) + ). duplicate_check_test() -> - ?assertError({duplicate_checks, [exp]}, - jwtf:decode(<<"a.b.c">>, [exp, exp], nil)). - + ?assertError( + {duplicate_checks, [exp]}, + jwtf:decode(<<"a.b.c">>, [exp, exp], nil) + ). %% jwt.io generated hs256_test() -> - EncodedToken = <<"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IjEyMzQ1Ni" - "J9.eyJpc3MiOiJodHRwczovL2Zvby5jb20iLCJpYXQiOjAsImV4cCI" - "6MTAwMDAwMDAwMDAwMDAsImtpZCI6ImJhciJ9.iS8AH11QHHlczkBn" - "Hl9X119BYLOZyZPllOVhSBZ4RZs">>, + EncodedToken = << + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IjEyMzQ1Ni" + "J9.eyJpc3MiOiJodHRwczovL2Zvby5jb20iLCJpYXQiOjAsImV4cCI" + "6MTAwMDAwMDAwMDAwMDAsImtpZCI6ImJhciJ9.iS8AH11QHHlczkBn" + "Hl9X119BYLOZyZPllOVhSBZ4RZs" + >>, KS = fun(<<"HS256">>, <<"123456">>) -> <<"secret">> end, Checks = [{iss, <<"https://foo.com">>}, iat, exp, typ, alg, kid], ?assertMatch({ok, _}, catch jwtf:decode(EncodedToken, Checks, KS)). - %% pip install PyJWT %% > import jwt %% > jwt.encode({'foo':'bar'}, 'secret', algorithm='HS384') hs384_test() -> - EncodedToken = <<"eyJhbGciOiJIUzM4NCIsInR5cCI6IkpXVCJ9.eyJmb28iOiJiYXIif" - "Q.2quwghs6I56GM3j7ZQbn-ASZ53xdBqzPzTDHm_CtVec32LUy-Ezy" - "L3JjIe7WjL93">>, + EncodedToken = << + "eyJhbGciOiJIUzM4NCIsInR5cCI6IkpXVCJ9.eyJmb28iOiJiYXIif" + "Q.2quwghs6I56GM3j7ZQbn-ASZ53xdBqzPzTDHm_CtVec32LUy-Ezy" + "L3JjIe7WjL93" + >>, KS = fun(<<"HS384">>, _) -> <<"secret">> end, - ?assertMatch({ok, {[{<<"foo">>,<<"bar">>}]}}, - catch jwtf:decode(EncodedToken, [], KS)). - + ?assertMatch( + {ok, {[{<<"foo">>, <<"bar">>}]}}, + catch jwtf:decode(EncodedToken, [], KS) + ). %% pip install PyJWT %% > import jwt %% > jwt.encode({'foo':'bar'}, 'secret', algorithm='HS512') hs512_test() -> - EncodedToken = <<"eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJmb28iOiJiYX" - "IifQ.WePl7achkd0oGNB8XRF_LJwxlyiPZqpdNgdKpDboAjSTsW" - "q-aOGNynTp8TOv8KjonFym8vwFwppXOLoLXbkIaQ">>, + EncodedToken = << + "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJmb28iOiJiYX" + "IifQ.WePl7achkd0oGNB8XRF_LJwxlyiPZqpdNgdKpDboAjSTsW" + "q-aOGNynTp8TOv8KjonFym8vwFwppXOLoLXbkIaQ" + >>, KS = fun(<<"HS512">>, _) -> <<"secret">> end, - ?assertMatch({ok, {[{<<"foo">>,<<"bar">>}]}}, - catch jwtf:decode(EncodedToken, [], KS)). - + ?assertMatch( + {ok, {[{<<"foo">>, <<"bar">>}]}}, + catch jwtf:decode(EncodedToken, [], KS) + ). %% jwt.io generated rs256_test() -> - EncodedToken = <<"eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0N" - "TY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWV9.Ek" - "N-DOsnsuRjRO6BxXemmJDm3HbxrbRzXglbN2S4sOkopdU4IsDxTI8j" - "O19W_A4K8ZPJijNLis4EZsHeY559a4DFOd50_OqgHGuERTqYZyuhtF" - "39yxJPAjUESwxk2J5k_4zM3O-vtd1Ghyo4IbqKKSy6J9mTniYJPenn" - "5-HIirE">>, + EncodedToken = << + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0N" + "TY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWV9.Ek" + "N-DOsnsuRjRO6BxXemmJDm3HbxrbRzXglbN2S4sOkopdU4IsDxTI8j" + "O19W_A4K8ZPJijNLis4EZsHeY559a4DFOd50_OqgHGuERTqYZyuhtF" + "39yxJPAjUESwxk2J5k_4zM3O-vtd1Ghyo4IbqKKSy6J9mTniYJPenn" + "5-HIirE" + >>, Checks = [sig, alg], KS = fun(<<"RS256">>, undefined) -> jwt_io_pubkey() end, - ExpectedPayload = {[ - {<<"sub">>, <<"1234567890">>}, - {<<"name">>, <<"John Doe">>}, - {<<"admin">>, true} - ]}, + ExpectedPayload = + {[ + {<<"sub">>, <<"1234567890">>}, + {<<"name">>, <<"John Doe">>}, + {<<"admin">>, true} + ]}, ?assertMatch({ok, ExpectedPayload}, jwtf:decode(EncodedToken, Checks, KS)). - encode_missing_alg_test() -> - ?assertEqual({error, {bad_request, <<"Missing alg header parameter">>}}, - jwtf:encode({[]}, {[]}, <<"foo">>)). - + ?assertEqual( + {error, {bad_request, <<"Missing alg header parameter">>}}, + jwtf:encode({[]}, {[]}, <<"foo">>) + ). encode_invalid_alg_test() -> - ?assertEqual({error, {bad_request, <<"Invalid alg header parameter">>}}, - jwtf:encode({[{<<"alg">>, <<"BOGUS">>}]}, {[]}, <<"foo">>)). - + ?assertEqual( + {error, {bad_request, <<"Invalid alg header parameter">>}}, + jwtf:encode({[{<<"alg">>, <<"BOGUS">>}]}, {[]}, <<"foo">>) + ). encode_decode_test_() -> [{Alg, encode_decode(Alg)} || Alg <- jwtf:valid_algorithms()]. - encode_decode(Alg) -> - {EncodeKey, DecodeKey} = case jwtf:verification_algorithm(Alg) of - {public_key, _Algorithm} -> - create_keypair(); - {hmac, _Algorithm} -> - Key = <<"a-super-secret-key">>, - {Key, Key} - end, + {EncodeKey, DecodeKey} = + case jwtf:verification_algorithm(Alg) of + {public_key, _Algorithm} -> + create_keypair(); + {hmac, _Algorithm} -> + Key = <<"a-super-secret-key">>, + {Key, Key} + end, Claims = claims(), {ok, Encoded} = jwtf:encode(header(Alg), Claims, EncodeKey), KS = fun(_, _) -> DecodeKey end, {ok, Decoded} = jwtf:decode(Encoded, [], KS), ?_assertMatch(Claims, Decoded). - header(Alg) -> {[ {<<"typ">>, <<"JWT">>}, @@ -273,7 +312,6 @@ header(Alg) -> {<<"kid">>, <<"20170520-00:00:00">>} ]}. - claims() -> EpochSeconds = os:system_time(second), {[ @@ -283,19 +321,23 @@ claims() -> create_keypair() -> %% https://tools.ietf.org/html/rfc7517#appendix-C - N = decode(<<"t6Q8PWSi1dkJj9hTP8hNYFlvadM7DflW9mWepOJhJ66w7nyoK1gPNqFMSQRy" + N = decode(<< + "t6Q8PWSi1dkJj9hTP8hNYFlvadM7DflW9mWepOJhJ66w7nyoK1gPNqFMSQRy" "O125Gp-TEkodhWr0iujjHVx7BcV0llS4w5ACGgPrcAd6ZcSR0-Iqom-QFcNP" "8Sjg086MwoqQU_LYywlAGZ21WSdS_PERyGFiNnj3QQlO8Yns5jCtLCRwLHL0" "Pb1fEv45AuRIuUfVcPySBWYnDyGxvjYGDSM-AqWS9zIQ2ZilgT-GqUmipg0X" "OC0Cc20rgLe2ymLHjpHciCKVAbY5-L32-lSeZO-Os6U15_aXrk9Gw8cPUaX1" - "_I8sLGuSiVdt3C_Fn2PZ3Z8i744FPFGGcG1qs2Wz-Q">>), + "_I8sLGuSiVdt3C_Fn2PZ3Z8i744FPFGGcG1qs2Wz-Q" + >>), E = decode(<<"AQAB">>), - D = decode(<<"GRtbIQmhOZtyszfgKdg4u_N-R_mZGU_9k7JQ_jn1DnfTuMdSNprTeaSTyWfS" + D = decode(<< + "GRtbIQmhOZtyszfgKdg4u_N-R_mZGU_9k7JQ_jn1DnfTuMdSNprTeaSTyWfS" "NkuaAwnOEbIQVy1IQbWVV25NY3ybc_IhUJtfri7bAXYEReWaCl3hdlPKXy9U" "vqPYGR0kIXTQRqns-dVJ7jahlI7LyckrpTmrM8dWBo4_PMaenNnPiQgO0xnu" "ToxutRZJfJvG4Ox4ka3GORQd9CsCZ2vsUDmsXOfUENOyMqADC6p1M3h33tsu" "rY15k9qMSpG9OX_IJAXmxzAh_tWiZOwk2K4yxH9tS3Lq1yX8C1EWmeRDkK2a" - "hecG85-oLKQt5VEpWHKmjOi_gJSdSgqcN96X52esAQ">>), + "hecG85-oLKQt5VEpWHKmjOi_gJSdSgqcN96X52esAQ" + >>), RSAPrivateKey = #'RSAPrivateKey'{ modulus = N, publicExponent = E, @@ -307,11 +349,9 @@ create_keypair() -> }, {RSAPrivateKey, RSAPublicKey}. - decode(Goop) -> crypto:bytes_to_integer(b64url:decode(Goop)). - now_seconds() -> {MegaSecs, Secs, _MicroSecs} = os:timestamp(), MegaSecs * 1000000 + Secs. diff --git a/src/ken/src/ken_event_handler.erl b/src/ken/src/ken_event_handler.erl index 8f158f425..f45fec087 100644 --- a/src/ken/src/ken_event_handler.erl +++ b/src/ken/src/ken_event_handler.erl @@ -25,7 +25,6 @@ handle_info/2 ]). - start_link() -> couch_event_listener:start_link(?MODULE, nil, [all_dbs]). diff --git a/src/ken/src/ken_server.erl b/src/ken/src/ken_server.erl index b33d01f35..9f869b379 100644 --- a/src/ken/src/ken_server.erl +++ b/src/ken/src/ken_server.erl @@ -32,8 +32,10 @@ -export([update_db_indexes/2]). -record(job, { - name, % {DbName, GroupId} for view. {DbName, DDocId, IndexId} for search. - server, % Pid of either view group or search index + % {DbName, GroupId} for view. {DbName, DDocId, IndexId} for search. + name, + % Pid of either view group or search index + server, worker_pid = nil, seq = 0, lru = erlang:monotonic_time() @@ -78,11 +80,15 @@ remove(DbName) -> add_all_shards(DbName) -> try Shards = mem3:shards(mem3:dbname(DbName)), - lists:map(fun(Shard) -> - rexi:cast(Shard#shard.node, {ken_server, add, [Shard#shard.name]}) - end, Shards) - catch error:database_does_not_exist -> - ok + lists:map( + fun(Shard) -> + rexi:cast(Shard#shard.node, {ken_server, add, [Shard#shard.name]}) + end, + Shards + ) + catch + error:database_does_not_exist -> + ok end. %% @doc Changes the configured value for a batch size. @@ -124,66 +130,64 @@ terminate(_Reason, _State) -> handle_call({set_batch_size, BS}, _From, #state{batch_size = Old} = State) -> {reply, Old, State#state{batch_size = BS}, 0}; - handle_call({set_delay, Delay}, _From, #state{delay = Old} = State) -> {reply, Old, State#state{delay = Delay}, 0}; - handle_call({set_limit, Limit}, _From, #state{limit = Old} = State) -> {reply, Old, State#state{limit = Limit}, 0}; - -handle_call({set_prune_interval, Interval}, _From , State) -> +handle_call({set_prune_interval, Interval}, _From, State) -> Old = State#state.prune_interval, {reply, Old, State#state{prune_interval = Interval}, 0}; - handle_call(Msg, From, State) -> {stop, {unknown_call, Msg, From}, State}. % Queues a DB to (maybe) have indexing jobs spawned. handle_cast({add, DbName}, State) -> case ets:insert_new(ken_pending, {DbName}) of - true -> - {noreply, State#state{q = queue:in(DbName, State#state.q)}, 0}; - false -> - {noreply, State, 0} + true -> + {noreply, State#state{q = queue:in(DbName, State#state.q)}, 0}; + false -> + {noreply, State, 0} end; - handle_cast({remove, DbName}, State) -> Q2 = queue:filter(fun(X) -> X =/= DbName end, State#state.q), ets:delete(ken_pending, DbName), % Delete search index workers - ets:match_delete(ken_workers, #job{name={DbName,'_','_'}, _='_'}), + ets:match_delete(ken_workers, #job{name = {DbName, '_', '_'}, _ = '_'}), % Delete view index workers - ets:match_delete(ken_workers, #job{name={DbName,'_'}, _='_'}), + ets:match_delete(ken_workers, #job{name = {DbName, '_'}, _ = '_'}), % TODO kill off active jobs for this DB as well {noreply, State#state{q = Q2}, 0}; - handle_cast({resubmit, DbName}, State) -> ets:delete(ken_resubmit, DbName), handle_cast({add, DbName}, State); - % st index job names have 3 elements, 3rd being 'hastings'. See job record definition. -handle_cast({trigger_update, #job{name={_, _, hastings}, server=GPid, seq=Seq} = Job}, State) -> +handle_cast({trigger_update, #job{name = {_, _, hastings}, server = GPid, seq = Seq} = Job}, State) -> % hastings_index:await will trigger a hastings index update - {Pid, _} = erlang:spawn_monitor(hastings_index, await, - [GPid, Seq]), + {Pid, _} = erlang:spawn_monitor( + hastings_index, + await, + [GPid, Seq] + ), Now = erlang:monotonic_time(), ets:insert(ken_workers, Job#job{worker_pid = Pid, lru = Now}), {noreply, State, 0}; % search index job names have 3 elements. See job record definition. -handle_cast({trigger_update, #job{name={_,_,_}, server=GPid, seq=Seq} = Job}, State) -> +handle_cast({trigger_update, #job{name = {_, _, _}, server = GPid, seq = Seq} = Job}, State) -> % dreyfus_index:await will trigger a search index update. - {Pid, _} = erlang:spawn_monitor(dreyfus_index, await, - [GPid, Seq]), + {Pid, _} = erlang:spawn_monitor( + dreyfus_index, + await, + [GPid, Seq] + ), Now = erlang:monotonic_time(), ets:insert(ken_workers, Job#job{worker_pid = Pid, lru = Now}), {noreply, State, 0}; -handle_cast({trigger_update, #job{name={_,_}, server=SrvPid, seq=Seq} = Job}, State) -> +handle_cast({trigger_update, #job{name = {_, _}, server = SrvPid, seq = Seq} = Job}, State) -> % couch_index:get_state/2 will trigger a view group index update. {Pid, _} = erlang:spawn_monitor(couch_index, get_state, [SrvPid, Seq]), Now = erlang:monotonic_time(), ets:insert(ken_workers, Job#job{worker_pid = Pid, lru = Now}), {noreply, State, 0}; - handle_cast(Msg, State) -> {stop, {unknown_cast, Msg}, State}. @@ -191,37 +195,33 @@ handle_info({gen_event_EXIT, ken_event_handler, Reason}, State) -> couch_log:error("ken_event_handler terminated: ~w", [Reason]), erlang:send_after(5000, self(), start_event_handler), {ok, State, 0}; - handle_info(start_event_handler, State) -> case ken_event_handler:start_link() of - {ok, _Pid} -> - ok; - Error -> - couch_log:error("ken_event_handler init: ~w", [Error]), - erlang:send_after(5000, self(), start_event_handler) + {ok, _Pid} -> + ok; + Error -> + couch_log:error("ken_event_handler init: ~w", [Error]), + erlang:send_after(5000, self(), start_event_handler) end, {noreply, State, 0}; - handle_info(timeout, #state{prune_interval = I, pruned_last = Last} = State) -> Now = erlang:monotonic_time(), Interval = erlang:convert_time_unit( - State#state.delay, millisecond, native), + State#state.delay, millisecond, native + ), case Now - Last > Interval of - true -> - NewState = prune_worker_table(State); - _ -> - NewState = State + true -> + NewState = prune_worker_table(State); + _ -> + NewState = State end, {noreply, maybe_start_next_queued_job(NewState), I}; - -handle_info({'DOWN', _, _, Pid, Reason}, #state{dbworker = {Name,Pid}} = St) -> +handle_info({'DOWN', _, _, Pid, Reason}, #state{dbworker = {Name, Pid}} = St) -> maybe_resubmit(Name, Reason), - {noreply, St#state{dbworker=nil}, 0}; - + {noreply, St#state{dbworker = nil}, 0}; handle_info({'DOWN', _, _, Pid, Reason}, State) -> debrief_worker(Pid, Reason, State), {noreply, State, 0}; - handle_info(Msg, State) -> {stop, {unknown_info, Msg}, State}. @@ -230,31 +230,32 @@ code_change(_OldVsn, State, _Extra) -> %% private functions -maybe_start_next_queued_job(#state{dbworker = {_,_}} = State) -> +maybe_start_next_queued_job(#state{dbworker = {_, _}} = State) -> State; -maybe_start_next_queued_job(#state{q=Q} = State) -> +maybe_start_next_queued_job(#state{q = Q} = State) -> IncrementalChannels = list_to_integer(config("incremental_channels", "80")), BatchChannels = list_to_integer(config("batch_channels", "20")), TotalChannels = IncrementalChannels + BatchChannels, case queue:out(Q) of - {{value, DbName}, Q2} -> - case skip_job(DbName) of - true -> - % job is either being resubmitted or ignored, skip it - ets:delete(ken_pending, DbName), - maybe_start_next_queued_job(State#state{q = Q2}); - false -> - case get_active_count() of A when A < TotalChannels -> - Args = [DbName, State], - {Pid, _} = spawn_monitor(?MODULE, update_db_indexes, Args), - ets:delete(ken_pending, DbName), - State#state{dbworker = {DbName,Pid}, q = Q2}; - _ -> - State#state{q = queue:in_r(DbName, Q2)} - end - end; - {empty, Q} -> - State + {{value, DbName}, Q2} -> + case skip_job(DbName) of + true -> + % job is either being resubmitted or ignored, skip it + ets:delete(ken_pending, DbName), + maybe_start_next_queued_job(State#state{q = Q2}); + false -> + case get_active_count() of + A when A < TotalChannels -> + Args = [DbName, State], + {Pid, _} = spawn_monitor(?MODULE, update_db_indexes, Args), + ets:delete(ken_pending, DbName), + State#state{dbworker = {DbName, Pid}, q = Q2}; + _ -> + State#state{q = queue:in_r(DbName, Q2)} + end + end; + {empty, Q} -> + State end. skip_job(DbName) -> @@ -262,28 +263,35 @@ skip_job(DbName) -> ignore_db(DbName) -> case config:get("ken.ignore", ?b2l(DbName), false) of - "true" -> - true; - _ -> - false + "true" -> + true; + _ -> + false end. get_active_count() -> - MatchSpec = [{#job{worker_pid='$1', _='_'}, [{is_pid, '$1'}], [true]}], + MatchSpec = [{#job{worker_pid = '$1', _ = '_'}, [{is_pid, '$1'}], [true]}], ets:select_count(ken_workers, MatchSpec). % If any indexing job fails, resubmit requests for all indexes. update_db_indexes(Name, State) -> {ok, DDocs} = design_docs(Name), RandomSorted = lists:sort([{rand:uniform(), D} || D <- DDocs]), - Resubmit = lists:foldl(fun({_, DDoc}, Acc) -> - JsonDDoc = couch_doc:from_json_obj(DDoc), - case update_ddoc_indexes(Name, JsonDDoc, State) of - ok -> Acc; - _ -> true - end - end, false, RandomSorted), - if Resubmit -> exit(resubmit); true -> ok end. + Resubmit = lists:foldl( + fun({_, DDoc}, Acc) -> + JsonDDoc = couch_doc:from_json_obj(DDoc), + case update_ddoc_indexes(Name, JsonDDoc, State) of + ok -> Acc; + _ -> true + end + end, + false, + RandomSorted + ), + if + Resubmit -> exit(resubmit); + true -> ok + end. design_docs(Name) -> try @@ -293,27 +301,32 @@ design_docs(Name) -> Else -> Else end - catch error:database_does_not_exist -> - {ok, []} + catch + error:database_does_not_exist -> + {ok, []} end. % Returns an error if any job creation fails. -update_ddoc_indexes(Name, #doc{}=Doc, State) -> - {ok, Db} = case couch_db:open_int(Name, []) of - {ok, _} = Resp -> Resp; - Else -> exit(Else) - end, +update_ddoc_indexes(Name, #doc{} = Doc, State) -> + {ok, Db} = + case couch_db:open_int(Name, []) of + {ok, _} = Resp -> Resp; + Else -> exit(Else) + end, Seq = couch_db:get_update_seq(Db), couch_db:close(Db), - ViewUpdated = case should_update(Doc, <<"views">>) of true -> - try couch_mrview_util:ddoc_to_mrst(Name, Doc) of - {ok, MRSt} -> update_ddoc_views(Name, MRSt, Seq, State) - catch _:_ -> - ok - end; - false -> - ok - end, + ViewUpdated = + case should_update(Doc, <<"views">>) of + true -> + try couch_mrview_util:ddoc_to_mrst(Name, Doc) of + {ok, MRSt} -> update_ddoc_views(Name, MRSt, Seq, State) + catch + _:_ -> + ok + end; + false -> + ok + end, SearchUpdated = search_updated(Name, Doc, Seq, State), STUpdated = st_updated(Name, Doc, Seq, State), case {ViewUpdated, SearchUpdated, STUpdated} of @@ -323,14 +336,16 @@ update_ddoc_indexes(Name, #doc{}=Doc, State) -> -ifdef(HAVE_DREYFUS). search_updated(Name, Doc, Seq, State) -> - case should_update(Doc, <<"indexes">>) of true -> - try dreyfus_index:design_doc_to_indexes(Doc) of - SIndexes -> update_ddoc_search_indexes(Name, SIndexes, Seq, State) - catch _:_ -> + case should_update(Doc, <<"indexes">>) of + true -> + try dreyfus_index:design_doc_to_indexes(Doc) of + SIndexes -> update_ddoc_search_indexes(Name, SIndexes, Seq, State) + catch + _:_ -> + ok + end; + false -> ok - end; - false -> - ok end. -else. search_updated(_Name, _Doc, _Seq, _State) -> @@ -339,22 +354,23 @@ search_updated(_Name, _Doc, _Seq, _State) -> -ifdef(HAVE_HASTINGS). st_updated(Name, Doc, Seq, State) -> - case should_update(Doc, <<"st_indexes">>) of true -> - try - hastings_index:design_doc_to_indexes(Doc) of - STIndexes -> update_ddoc_st_indexes(Name, STIndexes, Seq, State) - catch _:_ -> + case should_update(Doc, <<"st_indexes">>) of + true -> + try hastings_index:design_doc_to_indexes(Doc) of + STIndexes -> update_ddoc_st_indexes(Name, STIndexes, Seq, State) + catch + _:_ -> + ok + end; + false -> ok - end; - false -> - ok end. -else. st_updated(_Name, _Doc, _Seq, _State) -> ok. -endif. -should_update(#doc{body={Props}}, IndexType) -> +should_update(#doc{body = {Props}}, IndexType) -> case couch_util:get_value(<<"autoupdate">>, Props) of false -> false; @@ -373,47 +389,66 @@ update_ddoc_views(Name, MRSt, Seq, State) -> Language = couch_mrview_index:get(language, MRSt), Allowed = lists:member(Language, allowed_languages()), Views = couch_mrview_index:get(views, MRSt), - if Allowed andalso Views =/= [] -> - {ok, Pid} = couch_index_server:get_index(couch_mrview_index, MRSt), - GroupName = couch_mrview_index:get(idx_name, MRSt), - maybe_start_job({Name, GroupName}, Pid, Seq, State); - true -> ok end. + if + Allowed andalso Views =/= [] -> + {ok, Pid} = couch_index_server:get_index(couch_mrview_index, MRSt), + GroupName = couch_mrview_index:get(idx_name, MRSt), + maybe_start_job({Name, GroupName}, Pid, Seq, State); + true -> + ok + end. -ifdef(HAVE_DREYFUS). update_ddoc_search_indexes(DbName, Indexes, Seq, State) -> - if Indexes =/= [] -> - % Spawn a job for each search index in the ddoc - lists:foldl(fun(#index{name=IName, ddoc_id=DDocName}=Index, Acc) -> - case dreyfus_index_manager:get_index(DbName, Index) of - {ok, Pid} -> - case maybe_start_job({DbName, DDocName, IName}, Pid, Seq, State) of - resubmit -> resubmit; - _ -> Acc - end; - _ -> - % If any job fails, retry the db. - resubmit - end end, ok, Indexes); - true -> ok end. + if + Indexes =/= [] -> + % Spawn a job for each search index in the ddoc + lists:foldl( + fun(#index{name = IName, ddoc_id = DDocName} = Index, Acc) -> + case dreyfus_index_manager:get_index(DbName, Index) of + {ok, Pid} -> + case maybe_start_job({DbName, DDocName, IName}, Pid, Seq, State) of + resubmit -> resubmit; + _ -> Acc + end; + _ -> + % If any job fails, retry the db. + resubmit + end + end, + ok, + Indexes + ); + true -> + ok + end. -endif. -ifdef(HAVE_HASTINGS). update_ddoc_st_indexes(DbName, Indexes, Seq, State) -> - if Indexes =/= [] -> - % The record name in hastings is #h_idx rather than #index as it is for dreyfus - % Spawn a job for each spatial index in the ddoc - lists:foldl(fun(#h_idx{ddoc_id=DDocName}=Index, Acc) -> - case hastings_index_manager:get_index(DbName, Index) of - {ok, Pid} -> - case maybe_start_job({DbName, DDocName, hastings}, Pid, Seq, State) of - resubmit -> resubmit; - _ -> Acc - end; - _ -> - % If any job fails, retry the db. - resubmit - end end, ok, Indexes); - true -> ok end. + if + Indexes =/= [] -> + % The record name in hastings is #h_idx rather than #index as it is for dreyfus + % Spawn a job for each spatial index in the ddoc + lists:foldl( + fun(#h_idx{ddoc_id = DDocName} = Index, Acc) -> + case hastings_index_manager:get_index(DbName, Index) of + {ok, Pid} -> + case maybe_start_job({DbName, DDocName, hastings}, Pid, Seq, State) of + resubmit -> resubmit; + _ -> Acc + end; + _ -> + % If any job fails, retry the db. + resubmit + end + end, + ok, + Indexes + ); + true -> + ok + end. -endif. should_start_job(#job{name = Name, seq = Seq, server = Pid}, State) -> @@ -424,48 +459,49 @@ should_start_job(#job{name = Name, seq = Seq, server = Pid}, State) -> A = get_active_count(), #state{delay = Delay, batch_size = BS} = State, case ets:lookup(ken_workers, Name) of - [] -> - if - A < BatchChannels -> - true; - A < TotalChannels -> - case Name of - % st_index name has three elements - {_, _, hastings} -> - {ok, CurrentSeq} = hastings_index:await(Pid, 0), - (Seq - CurrentSeq) < Threshold; - % View name has two elements. - {_,_} -> - % Since seq is 0, couch_index:get_state/2 won't - % spawn an index update. - {ok, MRSt} = couch_index:get_state(Pid, 0), - CurrentSeq = couch_mrview_index:get(update_seq, MRSt), - (Seq - CurrentSeq) < Threshold; - % Search name has three elements. - {_,_,_} -> - {ok, _IndexPid, CurrentSeq} = dreyfus_index:await(Pid, 0), - (Seq - CurrentSeq) < Threshold; - _ -> % Should never happen, but if it does, ignore. - false + [] -> + if + A < BatchChannels -> + true; + A < TotalChannels -> + case Name of + % st_index name has three elements + {_, _, hastings} -> + {ok, CurrentSeq} = hastings_index:await(Pid, 0), + (Seq - CurrentSeq) < Threshold; + % View name has two elements. + {_, _} -> + % Since seq is 0, couch_index:get_state/2 won't + % spawn an index update. + {ok, MRSt} = couch_index:get_state(Pid, 0), + CurrentSeq = couch_mrview_index:get(update_seq, MRSt), + (Seq - CurrentSeq) < Threshold; + % Search name has three elements. + {_, _, _} -> + {ok, _IndexPid, CurrentSeq} = dreyfus_index:await(Pid, 0), + (Seq - CurrentSeq) < Threshold; + % Should never happen, but if it does, ignore. + _ -> + false end; - true -> - false - end; - [#job{worker_pid = nil, lru = LRU, seq = OldSeq}] -> - Now = erlang:monotonic_time(), - DeltaT = erlang:convert_time_unit(Now - LRU, native, millisecond), - if - A < BatchChannels, (Seq - OldSeq) >= BS -> - true; - A < BatchChannels, DeltaT > Delay -> - true; - A < TotalChannels, (Seq - OldSeq) < Threshold, DeltaT > Delay -> - true; - true -> - false - end; - _ -> - false + true -> + false + end; + [#job{worker_pid = nil, lru = LRU, seq = OldSeq}] -> + Now = erlang:monotonic_time(), + DeltaT = erlang:convert_time_unit(Now - LRU, native, millisecond), + if + A < BatchChannels, (Seq - OldSeq) >= BS -> + true; + A < BatchChannels, DeltaT > Delay -> + true; + A < TotalChannels, (Seq - OldSeq) < Threshold, DeltaT > Delay -> + true; + true -> + false + end; + _ -> + false end. maybe_start_job(JobName, IndexPid, Seq, State) -> @@ -475,24 +511,25 @@ maybe_start_job(JobName, IndexPid, Seq, State) -> seq = Seq }, case should_start_job(Job, State) of - true -> - gen_server:cast(?MODULE, {trigger_update, Job}); - false -> - resubmit + true -> + gen_server:cast(?MODULE, {trigger_update, Job}); + false -> + resubmit end. debrief_worker(Pid, Reason, _State) -> - case ets:match_object(ken_workers, #job{worker_pid=Pid, _='_'}) of - [#job{name = Name} = Job] -> - case Name of - {DbName,_} -> - maybe_resubmit(DbName, Reason); - {DbName,_,_} -> - maybe_resubmit(DbName, Reason) - end, - ets:insert(ken_workers, Job#job{worker_pid = nil}); - [] -> % should never happen, but if it does, ignore - ok + case ets:match_object(ken_workers, #job{worker_pid = Pid, _ = '_'}) of + [#job{name = Name} = Job] -> + case Name of + {DbName, _} -> + maybe_resubmit(DbName, Reason); + {DbName, _, _} -> + maybe_resubmit(DbName, Reason) + end, + ets:insert(ken_workers, Job#job{worker_pid = nil}); + % should never happen, but if it does, ignore + [] -> + ok end. maybe_resubmit(_DbName, normal) -> @@ -519,14 +556,15 @@ prune_worker_table(State) -> Delay = erlang:convert_time_unit(State#state.delay, millisecond, native), C = erlang:monotonic_time() - Delay, %% fun(#job{worker_pid=nil, lru=A) when A < C -> true end - MatchHead = #job{worker_pid=nil, lru='$1', _='_'}, + MatchHead = #job{worker_pid = nil, lru = '$1', _ = '_'}, Guard = {'<', '$1', C}, ets:select_delete(ken_workers, [{MatchHead, [Guard], [true]}]), State#state{pruned_last = erlang:monotonic_time()}. allowed_languages() -> - Config = couch_proc_manager:get_servers_from_env("COUCHDB_QUERY_SERVER_") ++ - couch_proc_manager:get_servers_from_env("COUCHDB_NATIVE_QUERY_SERVER_"), + Config = + couch_proc_manager:get_servers_from_env("COUCHDB_QUERY_SERVER_") ++ + couch_proc_manager:get_servers_from_env("COUCHDB_NATIVE_QUERY_SERVER_"), Allowed = [list_to_binary(string:to_lower(Lang)) || {Lang, _Cmd} <- Config], [<<"query">> | Allowed]. @@ -536,8 +574,6 @@ config(Key, Default) -> -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). - - prune_old_entries_test() -> { setup, @@ -548,15 +584,19 @@ prune_old_entries_test() -> catch ets:delete(ken_workers) end, ?_test(begin - lists:foreach(fun(Idx) -> - ets:insert(ken_workers, #job{name=Idx}), - timer:sleep(100) - end, lists:seq(1, 3)), - prune_worker_table(#state{delay=250}), + lists:foreach( + fun(Idx) -> + ets:insert(ken_workers, #job{name = Idx}), + timer:sleep(100) + end, + lists:seq(1, 3) + ), + prune_worker_table(#state{delay = 250}), ?assertEqual( [2, 3], lists:usort( - [N || #job{name = N} <- ets:tab2list(ken_workers)]) + [N || #job{name = N} <- ets:tab2list(ken_workers)] + ) ), ok end) diff --git a/src/ken/src/ken_sup.erl b/src/ken/src/ken_sup.erl index fd08cfd11..8c06592e7 100644 --- a/src/ken/src/ken_sup.erl +++ b/src/ken/src/ken_sup.erl @@ -29,5 +29,4 @@ start_link() -> %% supervisor callbacks init([]) -> - {ok, { {one_for_one, 5, 10}, [?CHILD(ken_server, worker)]} }. - + {ok, {{one_for_one, 5, 10}, [?CHILD(ken_server, worker)]}}. diff --git a/src/ken/test/ken_server_test.erl b/src/ken/test/ken_server_test.erl index eed348422..090c5570a 100644 --- a/src/ken/test/ken_server_test.erl +++ b/src/ken/test/ken_server_test.erl @@ -16,10 +16,8 @@ %% hardcoded defaults: limit: 20; batch: 1; delay: 5000; prune: 60000 default_test_() -> - {inorder, {setup, - fun setup_default/0, - fun teardown/1, - [ + {inorder, + {setup, fun setup_default/0, fun teardown/1, [ set_builder("returns default", set_limit, 12, 20), set_builder("keeps set", set_limit, 6, 12), set_builder("returns default", set_batch_size, 3, 1), @@ -28,14 +26,11 @@ default_test_() -> set_builder("keeps set", set_delay, 10000, 7000), set_builder("returns default", set_prune_interval, 70000, 60000), set_builder("keeps set", set_prune_interval, 80000, 70000) - ] - }}. + ]}}. exception_test_() -> - {inorder, {foreach, - fun setup_default/0, - fun teardown/1, - [ + {inorder, + {foreach, fun setup_default/0, fun teardown/1, [ exception_builder("exception on zero", set_limit, 0), exception_builder("exception on negative", set_limit, -12), exception_builder("exception on zero", set_batch_size, 0), @@ -44,18 +39,14 @@ exception_test_() -> exception_builder("exception on negative", set_delay, -12), exception_builder("exception on zero", set_prune_interval, 0), exception_builder("exception on negative", set_prune_interval, -12) - ] - }}. + ]}}. config_test_() -> - {inorder, {setup, - fun setup_config/0, - fun teardown/1, - [ + {inorder, + {setup, fun setup_config/0, fun teardown/1, [ set_builder("reads config", set_limit, 24, 42), set_builder("keeps set", set_limit, 6, 24) - ] - }}. + ]}}. setup_default() -> {ok, EventPid} = start_server(couch_event_server), @@ -94,4 +85,6 @@ stop_server(Key, Cfg) -> {Key, Pid} = lists:keyfind(Key, 1, Cfg), MRef = erlang:monitor(process, Pid), true = exit(Pid, kill), - receive {'DOWN', MRef, _, _, _} -> ok end. + receive + {'DOWN', MRef, _, _, _} -> ok + end. diff --git a/src/mango/src/mango_crud.erl b/src/mango/src/mango_crud.erl index 41a4d143d..c13dbdcb9 100644 --- a/src/mango/src/mango_crud.erl +++ b/src/mango/src/mango_crud.erl @@ -24,14 +24,12 @@ collect_cb/2 ]). - -include_lib("couch/include/couch_db.hrl"). -include("mango.hrl"). - -insert(Db, #doc{}=Doc, Opts) -> +insert(Db, #doc{} = Doc, Opts) -> insert(Db, [Doc], Opts); -insert(Db, {_}=Doc, Opts) -> +insert(Db, {_} = Doc, Opts) -> insert(Db, [Doc], Opts); insert(Db, Docs, Opts0) when is_list(Docs) -> Opts1 = maybe_add_user_ctx(Db, Opts0), @@ -45,14 +43,12 @@ insert(Db, Docs, Opts0) when is_list(Docs) -> {error, lists:map(fun result_to_json/1, Errors)} end. - find(Db, Selector, Callback, UserAcc, Opts0) -> Opts1 = maybe_add_user_ctx(Db, Opts0), Opts2 = maybe_int_to_str(r, Opts1), {ok, Cursor} = mango_cursor:create(Db, Selector, Opts2), mango_cursor:execute(Cursor, Callback, UserAcc). - update(Db, Selector, Update, Options) -> Upsert = proplists:get_value(upsert, Options), case collect_docs(Db, Selector, Options) of @@ -65,47 +61,51 @@ update(Db, Selector, Update, Options) -> % Probably need to catch and rethrow errors from % this function. Doc = couch_doc:from_json_obj(InitDoc), - NewDoc = case Doc#doc.id of - <<"">> -> - Doc#doc{id=couch_uuids:new(), revs={0, []}}; - _ -> - Doc - end, + NewDoc = + case Doc#doc.id of + <<"">> -> + Doc#doc{id = couch_uuids:new(), revs = {0, []}}; + _ -> + Doc + end, insert(Db, NewDoc, Options) end; {ok, Docs} -> - NewDocs = lists:map(fun(Doc) -> - mango_doc:apply_update(Doc, Update) - end, Docs), + NewDocs = lists:map( + fun(Doc) -> + mango_doc:apply_update(Doc, Update) + end, + Docs + ), insert(Db, NewDocs, Options); Else -> Else end. - delete(Db, Selector, Options) -> case collect_docs(Db, Selector, Options) of {ok, Docs} -> - NewDocs = lists:map(fun({Props}) -> - {[ - {<<"_id">>, proplists:get_value(<<"_id">>, Props)}, - {<<"_rev">>, proplists:get_value(<<"_rev">>, Props)}, - {<<"_deleted">>, true} - ]} - end, Docs), + NewDocs = lists:map( + fun({Props}) -> + {[ + {<<"_id">>, proplists:get_value(<<"_id">>, Props)}, + {<<"_rev">>, proplists:get_value(<<"_rev">>, Props)}, + {<<"_deleted">>, true} + ]} + end, + Docs + ), insert(Db, NewDocs, Options); Else -> Else end. - explain(Db, Selector, Opts0) -> Opts1 = maybe_add_user_ctx(Db, Opts0), Opts2 = maybe_int_to_str(r, Opts1), {ok, Cursor} = mango_cursor:create(Db, Selector, Opts2), mango_cursor:explain(Cursor). - maybe_add_user_ctx(Db, Opts) -> case lists:keyfind(user_ctx, 1, Opts) of {user_ctx, _} -> @@ -114,7 +114,6 @@ maybe_add_user_ctx(Db, Opts) -> [{user_ctx, couch_db:get_user_ctx(Db)} | Opts] end. - maybe_int_to_str(_Key, []) -> []; maybe_int_to_str(Key, [{Key, Val} | Rest]) when is_integer(Val) -> @@ -122,8 +121,7 @@ maybe_int_to_str(Key, [{Key, Val} | Rest]) when is_integer(Val) -> maybe_int_to_str(Key, [KV | Rest]) -> [KV | maybe_int_to_str(Key, Rest)]. - -result_to_json(#doc{id=Id}, Result) -> +result_to_json(#doc{id = Id}, Result) -> result_to_json(Id, Result); result_to_json({Props}, Result) -> Id = couch_util:get_value(<<"_id">>, Props), @@ -149,7 +147,6 @@ result_to_json(DocId, Error) -> {reason, Reason} ]}. - % This is for errors because for some reason we % need a different return value for errors? Blargh. result_to_json({{Id, Rev}, Error}) -> @@ -161,7 +158,6 @@ result_to_json({{Id, Rev}, Error}) -> {reason, Reason} ]}. - collect_docs(Db, Selector, Options) -> Cb = fun ?MODULE:collect_cb/2, case find(Db, Selector, Cb, [], Options) of @@ -171,7 +167,5 @@ collect_docs(Db, Selector, Options) -> Else end. - collect_cb({row, Doc}, Acc) -> {ok, [Doc | Acc]}. - diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl index b1cb4148e..e9db4c3cf 100644 --- a/src/mango/src/mango_cursor.erl +++ b/src/mango/src/mango_cursor.erl @@ -12,7 +12,6 @@ -module(mango_cursor). - -export([ create/3, explain/1, @@ -23,13 +22,11 @@ maybe_noop_range/2 ]). - -include_lib("couch/include/couch_db.hrl"). -include("mango.hrl"). -include("mango_cursor.hrl"). -include("mango_idx.hrl"). - -ifdef(HAVE_DREYFUS). -define(CURSOR_MODULES, [ mango_cursor_view, @@ -45,7 +42,6 @@ -define(SUPERVISOR, mango_cursor_sup). - create(Db, Selector0, Opts) -> Selector = mango_selector:normalize(Selector0), UsableIndexes = mango_idx:get_usable_indexes(Db, Selector, Opts), @@ -57,8 +53,7 @@ create(Db, Selector0, Opts) -> create_cursor(Db, UserSpecifiedIndex, Selector, Opts) end. - -explain(#cursor{}=Cursor) -> +explain(#cursor{} = Cursor) -> #cursor{ index = Idx, selector = Selector, @@ -69,23 +64,23 @@ explain(#cursor{}=Cursor) -> } = Cursor, Mod = mango_idx:cursor_mod(Idx), Opts = lists:keydelete(user_ctx, 1, Opts0), - {[ - {dbname, mango_idx:dbname(Idx)}, - {index, mango_idx:to_json(Idx)}, - {partitioned, mango_idx:partitioned(Idx)}, - {selector, Selector}, - {opts, {Opts}}, - {limit, Limit}, - {skip, Skip}, - {fields, Fields} - ] ++ Mod:explain(Cursor)}. - - -execute(#cursor{index=Idx}=Cursor, UserFun, UserAcc) -> + { + [ + {dbname, mango_idx:dbname(Idx)}, + {index, mango_idx:to_json(Idx)}, + {partitioned, mango_idx:partitioned(Idx)}, + {selector, Selector}, + {opts, {Opts}}, + {limit, Limit}, + {skip, Skip}, + {fields, Fields} + ] ++ Mod:explain(Cursor) + }. + +execute(#cursor{index = Idx} = Cursor, UserFun, UserAcc) -> Mod = mango_idx:cursor_mod(Idx), Mod:execute(Cursor, UserFun, UserAcc). - maybe_filter_indexes_by_ddoc(Indexes, Opts) -> case lists:keyfind(use_index, 1, Opts) of {use_index, []} -> @@ -96,24 +91,22 @@ maybe_filter_indexes_by_ddoc(Indexes, Opts) -> filter_indexes(Indexes, DesignId, ViewName) end. - filter_indexes(Indexes, DesignId0) -> - DesignId = case DesignId0 of - <<"_design/", _/binary>> -> - DesignId0; - Else -> - <<"_design/", Else/binary>> - end, + DesignId = + case DesignId0 of + <<"_design/", _/binary>> -> + DesignId0; + Else -> + <<"_design/", Else/binary>> + end, FiltFun = fun(I) -> mango_idx:ddoc(I) == DesignId end, lists:filter(FiltFun, Indexes). - filter_indexes(Indexes0, DesignId, ViewName) -> Indexes = filter_indexes(Indexes0, DesignId), FiltFun = fun(I) -> mango_idx:name(I) == ViewName end, lists:filter(FiltFun, Indexes). - remove_indexes_with_partial_filter_selector(Indexes) -> FiltFun = fun(Idx) -> case mango_idx:get_partial_filter_selector(Idx) of @@ -123,7 +116,6 @@ remove_indexes_with_partial_filter_selector(Indexes) -> end, lists:filter(FiltFun, Indexes). - maybe_add_warning(UserFun, #cursor{index = Index, opts = Opts}, Stats, UserAcc) -> W0 = invalid_index_warning(Index, Opts), W1 = no_index_warning(Index), @@ -139,39 +131,41 @@ maybe_add_warning(UserFun, #cursor{index = Index, opts = Opts}, Stats, UserAcc) UserAcc1 end. - create_cursor(Db, Indexes, Selector, Opts) -> [{CursorMod, CursorModIndexes} | _] = group_indexes_by_type(Indexes), CursorMod:create(Db, CursorModIndexes, Selector, Opts). - group_indexes_by_type(Indexes) -> - IdxDict = lists:foldl(fun(I, D) -> - dict:append(mango_idx:cursor_mod(I), I, D) - end, dict:new(), Indexes), + IdxDict = lists:foldl( + fun(I, D) -> + dict:append(mango_idx:cursor_mod(I), I, D) + end, + dict:new(), + Indexes + ), % The first cursor module that has indexes will be % used to service this query. This is so that we % don't suddenly switch indexes for existing client % queries. - lists:flatmap(fun(CMod) -> - case dict:find(CMod, IdxDict) of - {ok, CModIndexes} -> - [{CMod, CModIndexes}]; - error -> - [] - end - end, ?CURSOR_MODULES). - + lists:flatmap( + fun(CMod) -> + case dict:find(CMod, IdxDict) of + {ok, CModIndexes} -> + [{CMod, CModIndexes}]; + error -> + [] + end + end, + ?CURSOR_MODULES + ). % warn if the _all_docs index was used to fulfil a query no_index_warning(#idx{type = Type}) when Type =:= <<"special">> -> couch_stats:increment_counter([mango, unindexed_queries]), [<<"No matching index found, create an index to optimize query time.">>]; - no_index_warning(_) -> []. - % warn if user specified an index which doesn't exist or isn't valid % for the selector. % In this scenario, Mango will ignore the index hint and auto-select an index. @@ -179,36 +173,42 @@ invalid_index_warning(Index, Opts) -> UseIndex = lists:keyfind(use_index, 1, Opts), invalid_index_warning_int(Index, UseIndex). - invalid_index_warning_int(Index, {use_index, [DesignId]}) -> Filtered = filter_indexes([Index], DesignId), - if Filtered /= [] -> []; true -> - couch_stats:increment_counter([mango, query_invalid_index]), - Reason = fmt("_design/~s was not used because it does not contain a valid index for this query.", - [ddoc_name(DesignId)]), - [Reason] + if + Filtered /= [] -> + []; + true -> + couch_stats:increment_counter([mango, query_invalid_index]), + Reason = fmt( + "_design/~s was not used because it does not contain a valid index for this query.", + [ddoc_name(DesignId)] + ), + [Reason] end; - invalid_index_warning_int(Index, {use_index, [DesignId, ViewName]}) -> Filtered = filter_indexes([Index], DesignId, ViewName), - if Filtered /= [] -> []; true -> - couch_stats:increment_counter([mango, query_invalid_index]), - Reason = fmt("_design/~s, ~s was not used because it is not a valid index for this query.", - [ddoc_name(DesignId), ViewName]), - [Reason] + if + Filtered /= [] -> + []; + true -> + couch_stats:increment_counter([mango, query_invalid_index]), + Reason = fmt( + "_design/~s, ~s was not used because it is not a valid index for this query.", + [ddoc_name(DesignId), ViewName] + ), + [Reason] end; - invalid_index_warning_int(_, _) -> []. - % warn if a large number of documents needed to be scanned per result % returned, implying a lot of in-memory filtering -index_scan_warning(#execution_stats { - totalDocsExamined = Docs, - totalQuorumDocsExamined = DocsQuorum, - resultsReturned = ResultCount - }) -> +index_scan_warning(#execution_stats{ + totalDocsExamined = Docs, + totalQuorumDocsExamined = DocsQuorum, + resultsReturned = ResultCount +}) -> % Docs and DocsQuorum are mutually exclusive so it's safe to sum them DocsScanned = Docs + DocsQuorum, Ratio = calculate_index_scan_ratio(DocsScanned, ResultCount), @@ -216,9 +216,11 @@ index_scan_warning(#execution_stats { case Threshold > 0 andalso Ratio > Threshold of true -> couch_stats:increment_counter([mango, too_many_docs_scanned]), - Reason = <<"The number of documents examined is high in proportion to the number of results returned. Consider adding a more specific index to improve this.">>, + Reason = + <<"The number of documents examined is high in proportion to the number of results returned. Consider adding a more specific index to improve this.">>, [Reason]; - false -> [] + false -> + [] end. % When there is an empty array for certain operators, we don't actually @@ -237,20 +239,15 @@ maybe_noop_range({[{Op, []}]}, IndexRanges) -> maybe_noop_range(_, IndexRanges) -> IndexRanges. - calculate_index_scan_ratio(DocsScanned, 0) -> DocsScanned; - calculate_index_scan_ratio(DocsScanned, ResultCount) -> DocsScanned / ResultCount. - fmt(Format, Args) -> iolist_to_binary(io_lib:format(Format, Args)). - ddoc_name(<<"_design/", Name/binary>>) -> Name; - ddoc_name(Name) -> Name. diff --git a/src/mango/src/mango_cursor_special.erl b/src/mango/src/mango_cursor_special.erl index df1f6d655..f20edebd1 100644 --- a/src/mango/src/mango_cursor_special.erl +++ b/src/mango/src/mango_cursor_special.erl @@ -22,12 +22,10 @@ handle_message/2 ]). - -include_lib("couch/include/couch_db.hrl"). -include_lib("couch_mrview/include/couch_mrview.hrl"). -include("mango_cursor.hrl"). - create(Db, Indexes, Selector, Opts) -> InitialRange = mango_idx_view:field_ranges(Selector), CatchAll = [{<<"_id">>, {'$gt', null, '$lt', mango_json_max}}], @@ -37,7 +35,7 @@ create(Db, Indexes, Selector, Opts) -> FieldRanges = InitialRange ++ CatchAll, Composited = mango_cursor_view:composite_indexes(Indexes, FieldRanges), {Index, IndexRanges} = mango_cursor_view:choose_best_index(Db, Composited), - + Limit = couch_util:get_value(limit, Opts, mango_opts:default_limit()), Skip = couch_util:get_value(skip, Opts, 0), Fields = couch_util:get_value(fields, Opts, all_fields), diff --git a/src/mango/src/mango_cursor_text.erl b/src/mango/src/mango_cursor_text.erl index 43ef84e4c..53bf63edb 100644 --- a/src/mango/src/mango_cursor_text.erl +++ b/src/mango/src/mango_cursor_text.erl @@ -20,13 +20,11 @@ execute/3 ]). - -include_lib("couch/include/couch_db.hrl"). -include_lib("dreyfus/include/dreyfus.hrl"). -include("mango_cursor.hrl"). -include("mango.hrl"). - -record(cacc, { selector, dbname, @@ -42,14 +40,14 @@ execution_stats }). - create(Db, Indexes, Selector, Opts0) -> - Index = case Indexes of - [Index0] -> - Index0; - _ -> - ?MANGO_ERROR(multiple_text_indexes) - end, + Index = + case Indexes of + [Index0] -> + Index0; + _ -> + ?MANGO_ERROR(multiple_text_indexes) + end, Opts = unpack_bookmark(couch_db:name(Db), Opts0), @@ -69,7 +67,6 @@ create(Db, Indexes, Selector, Opts0) -> fields = Fields }}. - explain(Cursor) -> #cursor{ selector = Selector, @@ -81,7 +78,6 @@ explain(Cursor) -> {sort, sort_query(Opts, Selector)} ]. - execute(Cursor, UserFun, UserAcc) -> #cursor{ db = Db, @@ -131,12 +127,13 @@ execute(Cursor, UserFun, UserAcc) -> JsonBM = dreyfus_bookmark:pack(FinalBM), Arg = {add_key, bookmark, JsonBM}, {_Go, FinalUserAcc} = UserFun(Arg, LastUserAcc), - FinalUserAcc0 = mango_execution_stats:maybe_add_stats(Opts, UserFun, Stats0, FinalUserAcc), + FinalUserAcc0 = mango_execution_stats:maybe_add_stats( + Opts, UserFun, Stats0, FinalUserAcc + ), FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, Stats0, FinalUserAcc0), {ok, FinalUserAcc1} end. - execute(CAcc) -> case search_docs(CAcc) of {ok, Bookmark, []} -> @@ -152,7 +149,6 @@ execute(CAcc) -> execute(FinalCAcc) end. - search_docs(CAcc) -> #cacc{ dbname = DbName, @@ -167,19 +163,15 @@ search_docs(CAcc) -> ?MANGO_ERROR({text_search_error, {error, Reason}}) end. - handle_hits(CAcc, []) -> {ok, CAcc}; - handle_hits(CAcc0, [{Sort, Doc} | Rest]) -> CAcc1 = handle_hit(CAcc0, Sort, Doc), handle_hits(CAcc1, Rest). - handle_hit(CAcc0, Sort, not_found) -> CAcc1 = update_bookmark(CAcc0, Sort), CAcc1; - handle_hit(CAcc0, Sort, Doc) -> #cacc{ limit = Limit, @@ -208,7 +200,6 @@ handle_hit(CAcc0, Sort, Doc) -> CAcc2 end. - apply_user_fun(CAcc, Doc) -> FinalDoc = mango_fields:extract(Doc, CAcc#cacc.fields), #cacc{ @@ -224,39 +215,40 @@ apply_user_fun(CAcc, Doc) -> throw({stop, CAcc#cacc{user_acc = NewUserAcc, execution_stats = Stats0}}) end. - %% Convert Query to Dreyfus sort specifications %% Covert <<"Field">>, <<"desc">> to <<"-Field">> %% and append to the dreyfus query sort_query(Opts, Selector) -> {sort, {Sort}} = lists:keyfind(sort, 1, Opts), - SortList = lists:map(fun(SortField) -> - {Dir, RawSortField} = case SortField of - {Field, <<"asc">>} -> {asc, Field}; - {Field, <<"desc">>} -> {desc, Field}; - Field when is_binary(Field) -> {asc, Field} + SortList = lists:map( + fun(SortField) -> + {Dir, RawSortField} = + case SortField of + {Field, <<"asc">>} -> {asc, Field}; + {Field, <<"desc">>} -> {desc, Field}; + Field when is_binary(Field) -> {asc, Field} + end, + SField = mango_selector_text:append_sort_type(RawSortField, Selector), + case Dir of + asc -> + SField; + desc -> + <<"-", SField/binary>> + end end, - SField = mango_selector_text:append_sort_type(RawSortField, Selector), - case Dir of - asc -> - SField; - desc -> - <<"-", SField/binary>> - end - end, Sort), + Sort + ), case SortList of [] -> relevance; _ -> SortList end. - get_partition(Opts, Default) -> case couch_util:get_value(partition, Opts) of <<>> -> Default; Else -> Else end. - get_bookmark(Opts) -> case lists:keyfind(bookmark, 1, Opts) of {_, BM} when is_list(BM), BM /= [] -> @@ -265,7 +257,6 @@ get_bookmark(Opts) -> nil end. - update_bookmark(CAcc, Sortable) -> BM = CAcc#cacc.bookmark, QueryArgs = CAcc#cacc.query_args, @@ -273,28 +264,27 @@ update_bookmark(CAcc, Sortable) -> NewBM = dreyfus_bookmark:update(Sort, BM, [Sortable]), CAcc#cacc{bookmark = NewBM}. - pack_bookmark(Bookmark) -> case dreyfus_bookmark:pack(Bookmark) of null -> nil; Enc -> Enc end. - unpack_bookmark(DbName, Opts) -> - NewBM = case lists:keyfind(bookmark, 1, Opts) of - {_, nil} -> - []; - {_, Bin} -> - try - dreyfus_bookmark:unpack(DbName, Bin) - catch _:_ -> - ?MANGO_ERROR({invalid_bookmark, Bin}) - end - end, + NewBM = + case lists:keyfind(bookmark, 1, Opts) of + {_, nil} -> + []; + {_, Bin} -> + try + dreyfus_bookmark:unpack(DbName, Bin) + catch + _:_ -> + ?MANGO_ERROR({invalid_bookmark, Bin}) + end + end, lists:keystore(bookmark, 1, Opts, {bookmark, NewBM}). - ddocid(Idx) -> case mango_idx:ddoc(Idx) of <<"_design/", Rest/binary>> -> @@ -303,7 +293,6 @@ ddocid(Idx) -> Else end. - update_query_args(CAcc) -> #cacc{ bookmark = Bookmark, @@ -314,29 +303,32 @@ update_query_args(CAcc) -> limit = get_limit(CAcc) }. - get_limit(CAcc) -> erlang:min(get_dreyfus_limit(), CAcc#cacc.limit + CAcc#cacc.skip). - get_dreyfus_limit() -> config:get_integer("dreyfus", "max_limit", 200). - get_json_docs(DbName, Hits) -> - Ids = lists:map(fun(#sortable{item = Item}) -> - couch_util:get_value(<<"_id">>, Item#hit.fields) - end, Hits), + Ids = lists:map( + fun(#sortable{item = Item}) -> + couch_util:get_value(<<"_id">>, Item#hit.fields) + end, + Hits + ), % TODO: respect R query parameter (same as json indexes) {ok, IdDocs} = dreyfus_fabric:get_json_docs(DbName, Ids), - lists:map(fun(#sortable{item = Item} = Sort) -> - Id = couch_util:get_value(<<"_id">>, Item#hit.fields), - case lists:keyfind(Id, 1, IdDocs) of - {Id, {doc, Doc}} -> - {Sort, Doc}; - false -> - {Sort, not_found} - end - end, Hits). + lists:map( + fun(#sortable{item = Item} = Sort) -> + Id = couch_util:get_value(<<"_id">>, Item#hit.fields), + case lists:keyfind(Id, 1, IdDocs) of + {Id, {doc, Doc}} -> + {Sort, Doc}; + false -> + {Sort, not_found} + end + end, + Hits + ). -endif. diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl index 68d7c3b62..5656ffc0b 100644 --- a/src/mango/src/mango_cursor_view.erl +++ b/src/mango/src/mango_cursor_view.erl @@ -26,7 +26,6 @@ choose_best_index/2 ]). - -include_lib("couch/include/couch_db.hrl"). -include_lib("couch_mrview/include/couch_mrview.hrl"). -include_lib("fabric/include/fabric.hrl"). @@ -60,7 +59,6 @@ create(Db, Indexes, Selector, Opts) -> bookmark = Bookmark }}. - explain(Cursor) -> #cursor{ opts = Opts @@ -69,47 +67,50 @@ explain(Cursor) -> BaseArgs = base_args(Cursor), Args = apply_opts(Opts, BaseArgs), - [{mrargs, {[ - {include_docs, Args#mrargs.include_docs}, - {view_type, Args#mrargs.view_type}, - {reduce, Args#mrargs.reduce}, - {partition, couch_mrview_util:get_extra(Args, partition, null)}, - {start_key, maybe_replace_max_json(Args#mrargs.start_key)}, - {end_key, maybe_replace_max_json(Args#mrargs.end_key)}, - {direction, Args#mrargs.direction}, - {stable, Args#mrargs.stable}, - {update, Args#mrargs.update}, - {conflicts, Args#mrargs.conflicts} - ]}}]. - + [ + {mrargs, + {[ + {include_docs, Args#mrargs.include_docs}, + {view_type, Args#mrargs.view_type}, + {reduce, Args#mrargs.reduce}, + {partition, couch_mrview_util:get_extra(Args, partition, null)}, + {start_key, maybe_replace_max_json(Args#mrargs.start_key)}, + {end_key, maybe_replace_max_json(Args#mrargs.end_key)}, + {direction, Args#mrargs.direction}, + {stable, Args#mrargs.stable}, + {update, Args#mrargs.update}, + {conflicts, Args#mrargs.conflicts} + ]}} + ]. % replace internal values that cannot % be represented as a valid UTF-8 string % with a token for JSON serialization maybe_replace_max_json([]) -> []; - maybe_replace_max_json(?MAX_STR) -> <<"<MAX>">>; - maybe_replace_max_json([H | T] = EndKey) when is_list(EndKey) -> - H1 = if H == ?MAX_JSON_OBJ -> <<"<MAX>">>; + H1 = + if + H == ?MAX_JSON_OBJ -> <<"<MAX>">>; true -> H - end, + end, [H1 | maybe_replace_max_json(T)]; - maybe_replace_max_json(EndKey) -> EndKey. - base_args(#cursor{index = Idx, selector = Selector} = Cursor) -> - {StartKey, EndKey} = case Cursor#cursor.ranges of - [empty] -> - {null, null}; - _ -> - {mango_idx:start_key(Idx, Cursor#cursor.ranges), - mango_idx:end_key(Idx, Cursor#cursor.ranges)} - end, + {StartKey, EndKey} = + case Cursor#cursor.ranges of + [empty] -> + {null, null}; + _ -> + { + mango_idx:start_key(Idx, Cursor#cursor.ranges), + mango_idx:end_key(Idx, Cursor#cursor.ranges) + } + end, #mrargs{ view_type = map, reduce = false, @@ -123,7 +124,6 @@ base_args(#cursor{index = Idx, selector = Selector} = Cursor) -> ] }. - execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFun, UserAcc) -> Cursor = Cursor0#cursor{ user_fun = UserFun, @@ -141,48 +141,55 @@ execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFu Args = mango_json_bookmark:update_args(Bookmark, Args0), UserCtx = couch_util:get_value(user_ctx, Opts, #user_ctx{}), DbOpts = [{user_ctx, UserCtx}], - Result = case mango_idx:def(Idx) of - all_docs -> - CB = fun ?MODULE:handle_all_docs_message/2, - fabric:all_docs(Db, DbOpts, CB, Cursor, Args); - _ -> - CB = fun ?MODULE:handle_message/2, - % Normal view - DDoc = ddocid(Idx), - Name = mango_idx:name(Idx), - fabric:query_view(Db, DbOpts, DDoc, Name, CB, Cursor, Args) - end, + Result = + case mango_idx:def(Idx) of + all_docs -> + CB = fun ?MODULE:handle_all_docs_message/2, + fabric:all_docs(Db, DbOpts, CB, Cursor, Args); + _ -> + CB = fun ?MODULE:handle_message/2, + % Normal view + DDoc = ddocid(Idx), + Name = mango_idx:name(Idx), + fabric:query_view(Db, DbOpts, DDoc, Name, CB, Cursor, Args) + end, case Result of {ok, LastCursor} -> NewBookmark = mango_json_bookmark:create(LastCursor), Arg = {add_key, bookmark, NewBookmark}, {_Go, FinalUserAcc} = UserFun(Arg, LastCursor#cursor.user_acc), Stats0 = LastCursor#cursor.execution_stats, - FinalUserAcc0 = mango_execution_stats:maybe_add_stats(Opts, UserFun, Stats0, FinalUserAcc), - FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, Stats0, FinalUserAcc0), + FinalUserAcc0 = mango_execution_stats:maybe_add_stats( + Opts, UserFun, Stats0, FinalUserAcc + ), + FinalUserAcc1 = mango_cursor:maybe_add_warning( + UserFun, Cursor, Stats0, FinalUserAcc0 + ), {ok, FinalUserAcc1}; {error, Reason} -> {error, Reason} end end. - % Any of these indexes may be a composite index. For each % index find the most specific set of fields for each % index. Ie, if an index has columns a, b, c, d, then % check FieldRanges for a, b, c, and d and return % the longest prefix of columns found. composite_indexes(Indexes, FieldRanges) -> - lists:foldl(fun(Idx, Acc) -> - Cols = mango_idx:columns(Idx), - Prefix = composite_prefix(Cols, FieldRanges), - % Calcuate the difference between the FieldRanges/Selector - % and the Prefix. We want to select the index with a prefix - % that is as close to the FieldRanges as possible - PrefixDifference = length(FieldRanges) - length(Prefix), - [{Idx, Prefix, PrefixDifference} | Acc] - end, [], Indexes). - + lists:foldl( + fun(Idx, Acc) -> + Cols = mango_idx:columns(Idx), + Prefix = composite_prefix(Cols, FieldRanges), + % Calcuate the difference between the FieldRanges/Selector + % and the Prefix. We want to select the index with a prefix + % that is as close to the FieldRanges as possible + PrefixDifference = length(FieldRanges) - length(Prefix), + [{Idx, Prefix, PrefixDifference} | Acc] + end, + [], + Indexes + ). composite_prefix([], _) -> []; @@ -194,7 +201,6 @@ composite_prefix([Col | Rest], Ranges) -> [] end. - % The query planner % First choose the index with the lowest difference between its % Prefix and the FieldRanges. If that is equal, then @@ -230,7 +236,6 @@ choose_best_index(_DbName, IndexRanges) -> {SelectedIndex, SelectedIndexRanges, _} = hd(lists:sort(Cmp, IndexRanges)), {SelectedIndex, SelectedIndexRanges}. - view_cb({meta, Meta}, Acc) -> % Map function starting put(mango_docs_examined, 0), @@ -238,7 +243,7 @@ view_cb({meta, Meta}, Acc) -> ok = rexi:stream2({meta, Meta}), {ok, Acc}; view_cb({row, Row}, #mrargs{extra = Options} = Acc) -> - ViewRow = #view_row{ + ViewRow = #view_row{ id = couch_util:get_value(id, Row), key = couch_util:get_value(key, Row), doc = couch_util:get_value(doc, Row) @@ -261,7 +266,7 @@ view_cb({row, Row}, #mrargs{extra = Options} = Acc) -> false -> maybe_send_mango_ping() end - end, + end, {ok, Acc}; view_cb(complete, Acc) -> % Send shard-level execution stats @@ -272,7 +277,6 @@ view_cb(complete, Acc) -> view_cb(ok, ddoc_updated) -> rexi:reply({ok, ddoc_updated}). - maybe_send_mango_ping() -> Current = os:timestamp(), LastPing = get(mango_last_msg_timestamp), @@ -286,24 +290,22 @@ maybe_send_mango_ping() -> set_mango_msg_timestamp() end. - set_mango_msg_timestamp() -> put(mango_last_msg_timestamp, os:timestamp()). - handle_message({meta, _}, Cursor) -> {ok, Cursor}; handle_message({row, Props}, Cursor) -> case doc_member(Cursor, Props) of {ok, Doc, {execution_stats, Stats}} -> - Cursor1 = Cursor#cursor { + Cursor1 = Cursor#cursor{ execution_stats = Stats }, Cursor2 = update_bookmark_keys(Cursor1, Props), FinalDoc = mango_fields:extract(Doc, Cursor2#cursor.fields), handle_doc(Cursor2, FinalDoc); {no_match, _, {execution_stats, Stats}} -> - Cursor1 = Cursor#cursor { + Cursor1 = Cursor#cursor{ execution_stats = Stats }, {ok, Cursor1}; @@ -322,7 +324,6 @@ handle_message(complete, Cursor) -> handle_message({error, Reason}, _Cursor) -> {error, Reason}. - handle_all_docs_message({row, Props}, Cursor) -> case is_design_doc(Props) of true -> {ok, Cursor}; @@ -331,7 +332,6 @@ handle_all_docs_message({row, Props}, Cursor) -> handle_all_docs_message(Message, Cursor) -> handle_message(Message, Cursor). - handle_doc(#cursor{skip = S} = C, _) when S > 0 -> {ok, C#cursor{skip = S - 1}}; handle_doc(#cursor{limit = L, execution_stats = Stats} = C, Doc) when L > 0 -> @@ -346,7 +346,6 @@ handle_doc(#cursor{limit = L, execution_stats = Stats} = C, Doc) when L > 0 -> handle_doc(C, _Doc) -> {stop, C}. - ddocid(Idx) -> case mango_idx:ddoc(Idx) of <<"_design/", Rest/binary>> -> @@ -355,19 +354,19 @@ ddocid(Idx) -> Else end. - apply_opts([], Args) -> Args; apply_opts([{r, RStr} | Rest], Args) -> - IncludeDocs = case list_to_integer(RStr) of - 1 -> - true; - R when R > 1 -> - % We don't load the doc in the view query because - % we have to do a quorum read in the coordinator - % so there's no point. - false - end, + IncludeDocs = + case list_to_integer(RStr) of + 1 -> + true; + R when R > 1 -> + % We don't load the doc in the view query because + % we have to do a quorum read in the coordinator + % so there's no point. + false + end, NewArgs = Args#mrargs{include_docs = IncludeDocs}, apply_opts(Rest, NewArgs); apply_opts([{conflicts, true} | Rest], Args) -> @@ -423,7 +422,6 @@ apply_opts([{_, _} | Rest], Args) -> % Ignore unknown options apply_opts(Rest, Args). - doc_member(Cursor, RowProps) -> Db = Cursor#cursor.db, Opts = Cursor#cursor.opts, @@ -441,7 +439,7 @@ doc_member(Cursor, RowProps) -> couch_stats:increment_counter([mango, quorum_docs_examined]), Id = couch_util:get_value(id, RowProps), case mango_util:defer(fabric, open_doc, [Db, Id, Opts]) of - {ok, #doc{}=DocProps} -> + {ok, #doc{} = DocProps} -> Doc = couch_doc:to_json_obj(DocProps, []), match_doc(Selector, Doc, ExecutionStats1); Else -> @@ -452,7 +450,6 @@ doc_member(Cursor, RowProps) -> {no_match, null, {execution_stats, ExecutionStats}} end. - match_doc(Selector, Doc, ExecutionStats) -> case mango_selector:match(Selector, Doc) of true -> @@ -461,51 +458,47 @@ match_doc(Selector, Doc, ExecutionStats) -> {no_match, Doc, {execution_stats, ExecutionStats}} end. - is_design_doc(RowProps) -> case couch_util:get_value(id, RowProps) of <<"_design/", _/binary>> -> true; _ -> false end. - update_bookmark_keys(#cursor{limit = Limit} = Cursor, Props) when Limit > 0 -> Id = couch_util:get_value(id, Props), Key = couch_util:get_value(key, Props), - Cursor#cursor { + Cursor#cursor{ bookmark_docid = Id, bookmark_key = Key }; update_bookmark_keys(Cursor, _Props) -> Cursor. - %%%%%%%% module tests below %%%%%%%% -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). - does_not_refetch_doc_with_value_test() -> - Cursor = #cursor { + Cursor = #cursor{ db = <<"db">>, opts = [], execution_stats = #execution_stats{}, selector = mango_selector:normalize({[{<<"user_id">>, <<"1234">>}]}) }, RowProps = [ - {id,<<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>}, - {key,<<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>}, - {doc,{ - [ - {<<"_id">>,<<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>}, - {<<"_rev">>,<<"1-a954fe2308f14307756067b0e18c2968">>}, - {<<"user_id">>,11} - ] - }} + {id, <<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>}, + {key, <<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>}, + {doc, + { + [ + {<<"_id">>, <<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>}, + {<<"_rev">>, <<"1-a954fe2308f14307756067b0e18c2968">>}, + {<<"user_id">>, 11} + ] + }} ], {Match, _, _} = doc_member(Cursor, RowProps), ?assertEqual(Match, ok). - -endif. diff --git a/src/mango/src/mango_doc.erl b/src/mango/src/mango_doc.erl index c22b15544..f8cb4c63b 100644 --- a/src/mango/src/mango_doc.erl +++ b/src/mango/src/mango_doc.erl @@ -12,7 +12,6 @@ -module(mango_doc). - -export([ from_bson/1, @@ -26,36 +25,35 @@ set_field/3 ]). - -include_lib("couch/include/couch_db.hrl"). -include("mango.hrl"). - from_bson({Props}) -> - DocProps = case lists:keytake(<<"_id">>, 1, Props) of - {value, {<<"_id">>, DocId0}, RestProps} -> - DocId = case DocId0 of - {[{<<"$id">>, Id}]} -> - Id; - Else -> - Else - end, - [{<<"_id">>, DocId} | RestProps]; - false -> - Props - end, + DocProps = + case lists:keytake(<<"_id">>, 1, Props) of + {value, {<<"_id">>, DocId0}, RestProps} -> + DocId = + case DocId0 of + {[{<<"$id">>, Id}]} -> + Id; + Else -> + Else + end, + [{<<"_id">>, DocId} | RestProps]; + false -> + Props + end, Doc = couch_doc:from_json_obj({DocProps}), case Doc#doc.id of <<"">> -> - Doc#doc{id=couch_uuids:new(), revs={0, []}}; + Doc#doc{id = couch_uuids:new(), revs = {0, []}}; _ -> Doc end. - -apply_update(#doc{body={Props}}=Doc, Update) -> +apply_update(#doc{body = {Props}} = Doc, Update) -> NewProps = apply_update(Props, Update), - Doc#doc{body={NewProps}}; + Doc#doc{body = {NewProps}}; apply_update({Props}, {Update}) -> Result = do_update({Props}, Update), case has_operators(Result) of @@ -66,13 +64,11 @@ apply_update({Props}, {Update}) -> end, Result. - update_as_insert({Update}) -> NewProps = do_update_to_insert(Update, {[]}), apply_update(NewProps, {Update}). - -has_operators(#doc{body=Body}) -> +has_operators(#doc{body = Body}) -> has_operators(Body); has_operators({Props}) when is_list(Props) -> has_operators_obj(Props); @@ -85,7 +81,6 @@ has_operators(Val) when is_number(Val) -> has_operators(Val) when is_binary(Val) -> false. - has_operators_obj([]) -> false; has_operators_obj([{K, V} | Rest]) -> @@ -101,7 +96,6 @@ has_operators_obj([{K, V} | Rest]) -> end end. - has_operators_arr([]) -> false; has_operators_arr([V | Rest]) -> @@ -112,25 +106,24 @@ has_operators_arr([V | Rest]) -> has_operators_arr(Rest) end. - do_update(Props, []) -> Props; do_update(Props, [{Op, Value} | Rest]) -> UpdateFun = update_operator_fun(Op), - NewProps = case UpdateFun of - undefined -> - lists:keystore(Op, 1, Props, {Op, Value}); - Fun when is_function(Fun, 2) -> - case Value of - {ValueProps} -> - Fun(Props, ValueProps); - _ -> - ?MANGO_ERROR({invalid_operand, Op, Value}) - end - end, + NewProps = + case UpdateFun of + undefined -> + lists:keystore(Op, 1, Props, {Op, Value}); + Fun when is_function(Fun, 2) -> + case Value of + {ValueProps} -> + Fun(Props, ValueProps); + _ -> + ?MANGO_ERROR({invalid_operand, Op, Value}) + end + end, do_update(NewProps, Rest). - update_operator_fun(<<"$", _/binary>> = Op) -> OperatorFuns = [ % Object operators @@ -160,217 +153,230 @@ update_operator_fun(<<"$", _/binary>> = Op) -> update_operator_fun(_) -> undefined. - do_update_inc(Props, []) -> Props; do_update_inc(Props, [{Field, Incr} | Rest]) -> - if is_number(Incr) -> ok; true -> - ?MANGO_ERROR({invalid_increment, Incr}) - end, - NewProps = case get_field(Props, Field, fun is_number/1) of - Value when is_number(Value) -> - set_field(Props, Field, Value + Incr); - not_found -> - set_field(Props, Field, Incr); - _ -> - Props + if + is_number(Incr) -> ok; + true -> ?MANGO_ERROR({invalid_increment, Incr}) end, + NewProps = + case get_field(Props, Field, fun is_number/1) of + Value when is_number(Value) -> + set_field(Props, Field, Value + Incr); + not_found -> + set_field(Props, Field, Incr); + _ -> + Props + end, do_update_inc(NewProps, Rest). - do_update_rename(Props, []) -> Props; do_update_rename(Props, [{OldField, NewField} | Rest]) -> - NewProps = case rem_field(Props, OldField) of - {RemProps, OldValue} -> - set_field(RemProps, NewField, OldValue); - _ -> - Props - end, + NewProps = + case rem_field(Props, OldField) of + {RemProps, OldValue} -> + set_field(RemProps, NewField, OldValue); + _ -> + Props + end, do_update_rename(NewProps, Rest). - do_update_set_on_insert(Props, _) -> % This is only called during calls to apply_update/2 % which means this isn't an insert, so drop it on % the floor. Props. - do_update_set(Props, []) -> Props; do_update_set(Props, [{Field, Value} | Rest]) -> NewProps = set_field(Props, Field, Value), do_update_set(NewProps, Rest). - do_update_unset(Props, []) -> Props; do_update_unset(Props, [{Field, _} | Rest]) -> - NewProps = case rem_field(Props, Field) of - {RemProps, _} -> - RemProps; - _ -> - Props - end, + NewProps = + case rem_field(Props, Field) of + {RemProps, _} -> + RemProps; + _ -> + Props + end, do_update_unset(NewProps, Rest). - do_update_add_to_set(Props, []) -> Props; do_update_add_to_set(Props, [{Field, NewValue} | Rest]) -> - ToAdd = case NewValue of - {[{<<"$each">>, NewValues}]} when is_list(NewValues) -> - NewValues; - {[{<<"$each">>, NewValue}]} -> - [NewValue]; - Else -> - [Else] - end, - NewProps = case get_field(Props, Field) of - OldValues when is_list(OldValues) -> - FinalValues = lists:foldl(fun(V, Acc) -> - lists:append(Acc, [V]) - end, OldValues, ToAdd), - set_field(Props, Field, FinalValues); - _ -> - Props - end, + ToAdd = + case NewValue of + {[{<<"$each">>, NewValues}]} when is_list(NewValues) -> + NewValues; + {[{<<"$each">>, NewValue}]} -> + [NewValue]; + Else -> + [Else] + end, + NewProps = + case get_field(Props, Field) of + OldValues when is_list(OldValues) -> + FinalValues = lists:foldl( + fun(V, Acc) -> + lists:append(Acc, [V]) + end, + OldValues, + ToAdd + ), + set_field(Props, Field, FinalValues); + _ -> + Props + end, do_update_add_to_set(NewProps, Rest). - do_update_pop(Props, []) -> Props; do_update_pop(Props, [{Field, Pos} | Rest]) -> - NewProps = case get_field(Props, Field) of - OldValues when is_list(OldValues) -> - NewValues = case Pos > 0 of - true -> - lists:sublist(OldValues, 1, length(OldValues) - 1); - false -> - lists:sublist(OldValues, 2, length(OldValues) - 1) - end, - set_field(Props, Field, NewValues); - _ -> - Props - end, + NewProps = + case get_field(Props, Field) of + OldValues when is_list(OldValues) -> + NewValues = + case Pos > 0 of + true -> + lists:sublist(OldValues, 1, length(OldValues) - 1); + false -> + lists:sublist(OldValues, 2, length(OldValues) - 1) + end, + set_field(Props, Field, NewValues); + _ -> + Props + end, do_update_pop(NewProps, Rest). - do_update_pull_all(Props, []) -> Props; do_update_pull_all(Props, [{Field, Values} | Rest]) -> - ToRem = case is_list(Values) of - true -> Values; - false -> [Values] - end, - NewProps = case get_field(Props, Field) of - OldValues when is_list(OldValues) -> - NewValues = lists:foldl(fun(ValToRem, Acc) -> - % The logic in these filter functions is a bit - % subtle. The way to think of this is that we - % return true for all elements we want to keep. - FilterFun = case has_operators(ValToRem) of - true -> - fun(A) -> - Sel = mango_selector:normalize(ValToRem), - not mango_selector:match(A, Sel) - end; - false -> - fun(A) -> A /= ValToRem end - end, - lists:filter(FilterFun, Acc) - end, OldValues, ToRem), - set_field(Props, Field, NewValues); - _ -> - Props - end, + ToRem = + case is_list(Values) of + true -> Values; + false -> [Values] + end, + NewProps = + case get_field(Props, Field) of + OldValues when is_list(OldValues) -> + NewValues = lists:foldl( + fun(ValToRem, Acc) -> + % The logic in these filter functions is a bit + % subtle. The way to think of this is that we + % return true for all elements we want to keep. + FilterFun = + case has_operators(ValToRem) of + true -> + fun(A) -> + Sel = mango_selector:normalize(ValToRem), + not mango_selector:match(A, Sel) + end; + false -> + fun(A) -> A /= ValToRem end + end, + lists:filter(FilterFun, Acc) + end, + OldValues, + ToRem + ), + set_field(Props, Field, NewValues); + _ -> + Props + end, do_update_add_to_set(NewProps, Rest). - do_update_pull(Props, []) -> Props; do_update_pull(Props, [{Field, Value} | Rest]) -> - ToRem = case Value of - {[{<<"$each">>, Values}]} when is_list(Values) -> - Values; - {[{<<"$each">>, Value}]} -> - [Value]; - Else -> - [Else] - end, + ToRem = + case Value of + {[{<<"$each">>, Values}]} when is_list(Values) -> + Values; + {[{<<"$each">>, Value}]} -> + [Value]; + Else -> + [Else] + end, NewProps = do_update_pull_all(Props, [{Field, ToRem}]), do_update_pull(NewProps, Rest). - do_update_push_all(_, []) -> []; do_update_push_all(Props, [{Field, Values} | Rest]) -> - ToAdd = case is_list(Values) of - true -> Values; - false -> [Values] - end, - NewProps = case get_field(Props, Field) of - OldValues when is_list(OldValues) -> - NewValues = OldValues ++ ToAdd, - set_field(Props, Field, NewValues); - _ -> - Props - end, + ToAdd = + case is_list(Values) of + true -> Values; + false -> [Values] + end, + NewProps = + case get_field(Props, Field) of + OldValues when is_list(OldValues) -> + NewValues = OldValues ++ ToAdd, + set_field(Props, Field, NewValues); + _ -> + Props + end, do_update_push_all(NewProps, Rest). - do_update_push(Props, []) -> Props; do_update_push(Props, [{Field, Value} | Rest]) -> - ToAdd = case Value of - {[{<<"$each">>, Values}]} when is_list(Values) -> - Values; - {[{<<"$each">>, Value}]} -> - [Value]; - Else -> - [Else] - end, + ToAdd = + case Value of + {[{<<"$each">>, Values}]} when is_list(Values) -> + Values; + {[{<<"$each">>, Value}]} -> + [Value]; + Else -> + [Else] + end, NewProps = do_update_push_all(Props, [{Field, ToAdd}]), do_update_push(NewProps, Rest). - - do_update_bitwise(Props, []) -> Props; do_update_bitwise(Props, [{Field, Value} | Rest]) -> - DoOp = case Value of - {[{<<"and">>, Val}]} when is_integer(Val) -> - fun(V) -> V band Val end; - {[{<<"or">>, Val}]} when is_integer(Val) -> - fun(V) -> V bor Val end; - _ -> - fun(V) -> V end - end, - NewProps = case get_field(Props, Field, fun is_number/1) of - Value when is_number(Value) -> - NewValue = DoOp(Value), - set_field(Props, Field, NewValue); - _ -> - Props - end, + DoOp = + case Value of + {[{<<"and">>, Val}]} when is_integer(Val) -> + fun(V) -> V band Val end; + {[{<<"or">>, Val}]} when is_integer(Val) -> + fun(V) -> V bor Val end; + _ -> + fun(V) -> V end + end, + NewProps = + case get_field(Props, Field, fun is_number/1) of + Value when is_number(Value) -> + NewValue = DoOp(Value), + set_field(Props, Field, NewValue); + _ -> + Props + end, do_update_bitwise(NewProps, Rest). - do_update_to_insert([], Doc) -> Doc; do_update_to_insert([{<<"$setOnInsert">>, {FieldProps}}], Doc) -> - lists:foldl(fun({Field, Value}, DocAcc) -> - set_field(DocAcc, Field, Value) - end, Doc, FieldProps); + lists:foldl( + fun({Field, Value}, DocAcc) -> + set_field(DocAcc, Field, Value) + end, + Doc, + FieldProps + ); do_update_to_insert([{_, _} | Rest], Doc) -> do_update_to_insert(Rest, Doc). - get_field(Props, Field) -> get_field(Props, Field, no_validation). - get_field(Props, Field, Validator) when is_binary(Field) -> {ok, Path} = mango_util:parse_field(Field), get_field(Props, Path, Validator); @@ -402,13 +408,13 @@ get_field(Values, [Name | Rest], Validator) when is_list(Values) -> false -> bad_path end - catch error:badarg -> - bad_path + catch + error:badarg -> + bad_path end; -get_field(_, [_|_], _) -> +get_field(_, [_ | _], _) -> bad_path. - rem_field(Props, Field) when is_binary(Field) -> {ok, Path} = mango_util:parse_field(Field), rem_field(Props, Path); @@ -443,8 +449,9 @@ rem_field(Values, [Name]) when is_list(Values) -> false -> bad_path end - catch error:badarg -> - bad_path + catch + error:badarg -> + bad_path end; rem_field(Values, [Name | Rest]) when is_list(Values) -> % Name might be an integer index into an array @@ -463,13 +470,13 @@ rem_field(Values, [Name | Rest]) when is_list(Values) -> false -> bad_path end - catch error:badarg -> - bad_path + catch + error:badarg -> + bad_path end; -rem_field(_, [_|_]) -> +rem_field(_, [_ | _]) -> bad_path. - set_field(Props, Field, Value) when is_binary(Field) -> {ok, Path} = mango_util:parse_field(Field), set_field(Props, Path, Value); @@ -495,8 +502,9 @@ set_field(Values, [Name], Value) when is_list(Values) -> false -> Values end - catch error:badarg -> - Values + catch + error:badarg -> + Values end; set_field(Values, [Name | Rest], Value) when is_list(Values) -> % Name might be an integer index into an array @@ -511,27 +519,25 @@ set_field(Values, [Name | Rest], Value) when is_list(Values) -> false -> Values end - catch error:badarg -> - Values + catch + error:badarg -> + Values end; -set_field(Value, [_|_], _) -> +set_field(Value, [_ | _], _) -> Value. - make_nested([], Value) -> Value; make_nested([Name | Rest], Value) -> {[{Name, make_nested(Rest, Value)}]}. - rem_elem(1, [Value | Rest]) -> {Rest, Value}; rem_elem(I, [Item | Rest]) when I > 1 -> - {Tail, Value} = rem_elem(I+1, Rest), + {Tail, Value} = rem_elem(I + 1, Rest), {[Item | Tail], Value}. - set_elem(1, [_ | Rest], Value) -> [Value | Rest]; set_elem(I, [Item | Rest], Value) when I > 1 -> - [Item | set_elem(I-1, Rest, Value)]. + [Item | set_elem(I - 1, Rest, Value)]. diff --git a/src/mango/src/mango_epi.erl b/src/mango/src/mango_epi.erl index 1fcd05b7f..b7ee68857 100644 --- a/src/mango/src/mango_epi.erl +++ b/src/mango/src/mango_epi.erl @@ -29,7 +29,7 @@ app() -> providers() -> [ - {chttpd_handlers, mango_httpd_handlers} + {chttpd_handlers, mango_httpd_handlers} ]. services() -> diff --git a/src/mango/src/mango_error.erl b/src/mango/src/mango_error.erl index bb545ad67..0301d079f 100644 --- a/src/mango/src/mango_error.erl +++ b/src/mango/src/mango_error.erl @@ -12,28 +12,29 @@ -module(mango_error). - -include_lib("couch/include/couch_db.hrl"). - -export([ info/2 ]). - info(mango_idx, {no_usable_index, missing_sort_index}) -> { 400, <<"no_usable_index">>, - <<"No index exists for this sort, " - "try indexing by the sort fields.">> + << + "No index exists for this sort, " + "try indexing by the sort fields." + >> }; info(mango_idx, {no_usable_index, missing_sort_index_partitioned}) -> { 400, <<"no_usable_index">>, - <<"No partitioned index exists for this sort, " - "try indexing by the sort fields.">> + << + "No partitioned index exists for this sort, " + "try indexing by the sort fields." + >> }; info(mango_idx, {no_usable_index, missing_sort_index_global}) -> { @@ -47,7 +48,6 @@ info(mango_json_bookmark, {invalid_bookmark, BadBookmark}) -> <<"invalid_bookmark">>, fmt("Invalid bookmark value: ~s", [?JSON_ENCODE(BadBookmark)]) }; - info(mango_cursor_text, {invalid_bookmark, BadBookmark}) -> { 400, @@ -60,8 +60,9 @@ info(mango_cursor_text, multiple_text_indexes) -> <<"multiple_text_indexes">>, <<"You must specify an index with the `use_index` parameter.">> }; -info(mango_cursor_text, {text_search_error, {error, {bad_request, Msg}}}) - when is_binary(Msg) -> +info(mango_cursor_text, {text_search_error, {error, {bad_request, Msg}}}) when + is_binary(Msg) +-> { 400, <<"text_search_error">>, @@ -73,7 +74,6 @@ info(mango_cursor_text, {text_search_error, {error, Error}}) -> <<"text_search_error">>, fmt("~p", [Error]) }; - info(mango_fields, {invalid_fields_json, BadFields}) -> { 400, @@ -86,7 +86,6 @@ info(mango_fields, {invalid_field_json, BadField}) -> <<"invalid_field">>, fmt("Invalid JSON for field spec: ~w", [BadField]) }; - info(mango_httpd, error_saving_ddoc) -> { 500, @@ -109,9 +108,8 @@ info(mango_httpd, invalid_list_index_params) -> { 500, <<"invalid_list_index_params">>, - <<"Index parameter ranges: limit > 1, skip > 0" >> + <<"Index parameter ranges: limit > 1, skip > 0">> }; - info(mango_idx, {invalid_index_type, BadType}) -> { 400, @@ -122,8 +120,11 @@ info(mango_idx, {partitioned_option_mismatch, BadDDoc}) -> { 400, <<"invalid_partitioned_option">>, - fmt("Requested partitioned option does not match existing value on" - " design document ~s", [BadDDoc]) + fmt( + "Requested partitioned option does not match existing value on" + " design document ~s", + [BadDDoc] + ) }; info(mango_idx, invalid_query_ddoc_language) -> { @@ -149,7 +150,6 @@ info(mango_idx, {index_service_unavailable, IndexName}) -> <<"required index service unavailable">>, fmt("~s", [IndexName]) }; - info(mango_idx_view, {invalid_index_json, BadIdx}) -> { 400, @@ -160,9 +160,12 @@ info(mango_idx_text, {invalid_index_fields_definition, Def}) -> { 400, <<"invalid_index_fields_definition">>, - fmt("Text Index field definitions must be of the form - {\"name\": \"non-empty fieldname\", \"type\": - \"boolean,number, or string\"}. Def: ~p", [Def]) + fmt( + "Text Index field definitions must be of the form\n" + " {\"name\": \"non-empty fieldname\", \"type\":\n" + " \"boolean,number, or string\"}. Def: ~p", + [Def] + ) }; info(mango_idx_view, {index_not_found, BadIdx}) -> { @@ -170,7 +173,6 @@ info(mango_idx_view, {index_not_found, BadIdx}) -> <<"invalid_index">>, fmt("JSON index ~s not found in this design doc.", [BadIdx]) }; - info(mango_idx_text, {invalid_index_text, BadIdx}) -> { 400, @@ -189,13 +191,14 @@ info(mango_idx_text, index_all_disabled) -> <<"index_all_disabled">>, <<"New text indexes are forbidden to index all fields.">> }; - info(mango_opts, {invalid_bulk_docs, Val}) -> { 400, <<"invalid_bulk_docs">>, - fmt("Bulk Delete requires an array of non-null docids. Docids: ~w", - [Val]) + fmt( + "Bulk Delete requires an array of non-null docids. Docids: ~w", + [Val] + ) }; info(mango_opts, {invalid_ejson, Val}) -> { @@ -269,15 +272,15 @@ info(mango_opts, {invalid_index_name, BadName}) -> <<"invalid_index_name">>, fmt("Invalid index name: ~w", [BadName]) }; - info(mango_opts, {multiple_text_operator, {invalid_selector, BadSel}}) -> { 400, <<"multiple_text_selector">>, - fmt("Selector cannot contain more than one $text operator: ~w", - [BadSel]) + fmt( + "Selector cannot contain more than one $text operator: ~w", + [BadSel] + ) }; - info(mango_selector, {invalid_selector, missing_field_name}) -> { 400, @@ -308,7 +311,6 @@ info(mango_selector, {bad_field, BadSel}) -> <<"bad_field">>, fmt("Invalid field normalization on selector: ~w", [BadSel]) }; - info(mango_selector_text, {invalid_operator, Op}) -> { 400, @@ -317,14 +319,14 @@ info(mango_selector_text, {invalid_operator, Op}) -> }; info(mango_selector_text, {text_sort_error, Field}) -> S = binary_to_list(Field), - Msg = "Unspecified or ambiguous sort type. Try appending :number or" + Msg = + "Unspecified or ambiguous sort type. Try appending :number or" " :string to the sort field. ~s", { 400, <<"text_sort_error">>, fmt(Msg, [S]) }; - info(mango_sort, {invalid_sort_json, BadSort}) -> { 400, @@ -349,7 +351,6 @@ info(mango_sort, {unsupported, mixed_sort}) -> <<"unsupported_mixed_sort">>, <<"Sorts currently only support a single direction for all fields.">> }; - info(mango_util, {error_loading_doc, DocId}) -> { 500, @@ -368,7 +369,6 @@ info(mango_util, {invalid_ddoc_lang, Lang}) -> <<"invalid_ddoc_lang">>, fmt("Existing design doc has an invalid language: ~w", [Lang]) }; - info(Module, Reason) -> { 500, @@ -376,6 +376,5 @@ info(Module, Reason) -> fmt("Unknown Error: ~s :: ~w", [Module, Reason]) }. - fmt(Format, Args) -> iolist_to_binary(io_lib:format(Format, Args)). diff --git a/src/mango/src/mango_execution_stats.erl b/src/mango/src/mango_execution_stats.erl index 5878a3190..0db3edf5f 100644 --- a/src/mango/src/mango_execution_stats.erl +++ b/src/mango/src/mango_execution_stats.erl @@ -12,7 +12,6 @@ -module(mango_execution_stats). - -export([ to_json/1, incr_keys_examined/1, @@ -25,10 +24,8 @@ maybe_add_stats/4 ]). - -include("mango_cursor.hrl"). - to_json(Stats) -> {[ {total_keys_examined, Stats#execution_stats.totalKeysExamined}, @@ -38,50 +35,42 @@ to_json(Stats) -> {execution_time_ms, Stats#execution_stats.executionTimeMs} ]}. - incr_keys_examined(Stats) -> - Stats#execution_stats { + Stats#execution_stats{ totalKeysExamined = Stats#execution_stats.totalKeysExamined + 1 }. - incr_docs_examined(Stats) -> incr_docs_examined(Stats, 1). - incr_docs_examined(Stats, N) -> - Stats#execution_stats { + Stats#execution_stats{ totalDocsExamined = Stats#execution_stats.totalDocsExamined + N }. - incr_quorum_docs_examined(Stats) -> - Stats#execution_stats { + Stats#execution_stats{ totalQuorumDocsExamined = Stats#execution_stats.totalQuorumDocsExamined + 1 }. - incr_results_returned(Stats) -> couch_stats:increment_counter([mango, results_returned]), - Stats#execution_stats { + Stats#execution_stats{ resultsReturned = Stats#execution_stats.resultsReturned + 1 }. - log_start(Stats) -> - Stats#execution_stats { + Stats#execution_stats{ executionStartTime = os:timestamp() }. - log_end(Stats) -> End = os:timestamp(), Diff = timer:now_diff(End, Stats#execution_stats.executionStartTime) / 1000, - Stats#execution_stats { + Stats#execution_stats{ executionTimeMs = Diff }. - maybe_add_stats(Opts, UserFun, Stats0, UserAcc) -> Stats1 = log_end(Stats0), couch_stats:update_histogram([mango, query_time], Stats1#execution_stats.executionTimeMs), diff --git a/src/mango/src/mango_fields.erl b/src/mango/src/mango_fields.erl index 273256025..1745cf9dd 100644 --- a/src/mango/src/mango_fields.erl +++ b/src/mango/src/mango_fields.erl @@ -17,10 +17,8 @@ extract/2 ]). - -include("mango.hrl"). - new([]) -> {ok, all_fields}; new(Fields) when is_list(Fields) -> @@ -28,24 +26,26 @@ new(Fields) when is_list(Fields) -> new(Else) -> ?MANGO_ERROR({invalid_fields_json, Else}). - extract(Doc, undefined) -> Doc; extract(Doc, all_fields) -> Doc; extract(Doc, Fields) -> - lists:foldl(fun(F, NewDoc) -> - {ok, Path} = mango_util:parse_field(F), - case mango_doc:get_field(Doc, Path) of - not_found -> - NewDoc; - bad_path -> - NewDoc; - Value -> - mango_doc:set_field(NewDoc, Path, Value) - end - end, {[]}, Fields). - + lists:foldl( + fun(F, NewDoc) -> + {ok, Path} = mango_util:parse_field(F), + case mango_doc:get_field(Doc, Path) of + not_found -> + NewDoc; + bad_path -> + NewDoc; + Value -> + mango_doc:set_field(NewDoc, Path, Value) + end + end, + {[]}, + Fields + ). field(Val) when is_binary(Val) -> Val; diff --git a/src/mango/src/mango_httpd.erl b/src/mango/src/mango_httpd.erl index 624691bb9..002c45b2f 100644 --- a/src/mango/src/mango_httpd.erl +++ b/src/mango/src/mango_httpd.erl @@ -12,12 +12,10 @@ -module(mango_httpd). - -export([ handle_req/2 ]). - -include_lib("couch/include/couch_db.hrl"). -include("mango.hrl"). -include("mango_idx.hrl"). @@ -46,41 +44,42 @@ handle_req(#httpd{} = Req, Db0) -> end end. - -handle_req_int(#httpd{path_parts=[_, <<"_index">> | _]} = Req, Db) -> +handle_req_int(#httpd{path_parts = [_, <<"_index">> | _]} = Req, Db) -> handle_index_req(Req, Db); -handle_req_int(#httpd{path_parts=[_, <<"_explain">> | _]} = Req, Db) -> +handle_req_int(#httpd{path_parts = [_, <<"_explain">> | _]} = Req, Db) -> handle_explain_req(Req, Db); -handle_req_int(#httpd{path_parts=[_, <<"_find">> | _]} = Req, Db) -> +handle_req_int(#httpd{path_parts = [_, <<"_find">> | _]} = Req, Db) -> handle_find_req(Req, Db); handle_req_int(_, _) -> throw({not_found, missing}). - -handle_index_req(#httpd{method='GET', path_parts=[_, _]}=Req, Db) -> - Params = lists:flatmap(fun({K, V}) -> parse_index_param(K, V) end, - chttpd:qs(Req)), +handle_index_req(#httpd{method = 'GET', path_parts = [_, _]} = Req, Db) -> + Params = lists:flatmap( + fun({K, V}) -> parse_index_param(K, V) end, + chttpd:qs(Req) + ), Idxs = lists:sort(mango_idx:list(Db)), JsonIdxs0 = lists:map(fun mango_idx:to_json/1, Idxs), TotalRows = length(JsonIdxs0), - Limit = case couch_util:get_value(limit, Params, TotalRows) of - Limit0 when Limit0 < 1 -> - ?MANGO_ERROR(invalid_list_index_params); - Limit0 -> - Limit0 - end, - Skip = case couch_util:get_value(skip, Params, 0) of - Skip0 when Skip0 < 0 -> - ?MANGO_ERROR(invalid_list_index_params); - Skip0 when Skip0 > TotalRows -> - TotalRows; - Skip0 -> - Skip0 - end, - JsonIdxs = lists:sublist(JsonIdxs0, Skip+1, Limit), - chttpd:send_json(Req, {[{total_rows, TotalRows}, {indexes, JsonIdxs}]}); - -handle_index_req(#httpd{method='POST', path_parts=[_, _]}=Req, Db) -> + Limit = + case couch_util:get_value(limit, Params, TotalRows) of + Limit0 when Limit0 < 1 -> + ?MANGO_ERROR(invalid_list_index_params); + Limit0 -> + Limit0 + end, + Skip = + case couch_util:get_value(skip, Params, 0) of + Skip0 when Skip0 < 0 -> + ?MANGO_ERROR(invalid_list_index_params); + Skip0 when Skip0 > TotalRows -> + TotalRows; + Skip0 -> + Skip0 + end, + JsonIdxs = lists:sublist(JsonIdxs0, Skip + 1, Limit), + chttpd:send_json(Req, {[{total_rows, TotalRows}, {indexes, JsonIdxs}]}); +handle_index_req(#httpd{method = 'POST', path_parts = [_, _]} = Req, Db) -> chttpd:validate_ctype(Req, "application/json"), {ok, Opts} = mango_opts:validate_idx_create(chttpd:json_body_obj(Req)), {ok, Idx0} = mango_idx:new(Db, Opts), @@ -89,62 +88,89 @@ handle_index_req(#httpd{method='POST', path_parts=[_, _]}=Req, Db) -> {ok, DDoc} = mango_util:load_ddoc(Db, mango_idx:ddoc(Idx), DbOpts), Id = Idx#idx.ddoc, Name = Idx#idx.name, - Status = case mango_idx:add(DDoc, Idx) of - {ok, DDoc} -> - <<"exists">>; - {ok, NewDDoc} -> - CreateOpts = get_idx_w_opts(Opts), - case mango_crud:insert(Db, NewDDoc, CreateOpts) of - {ok, [{RespProps}]} -> - case lists:keyfind(error, 1, RespProps) of - {error, Reason} -> - ?MANGO_ERROR({error_saving_ddoc, Reason}); - _ -> - <<"created">> - end; - _ -> - ?MANGO_ERROR(error_saving_ddoc) - end - end, - chttpd:send_json(Req, {[{result, Status}, {id, Id}, {name, Name}]}); - -handle_index_req(#httpd{path_parts=[_, _]}=Req, _Db) -> + Status = + case mango_idx:add(DDoc, Idx) of + {ok, DDoc} -> + <<"exists">>; + {ok, NewDDoc} -> + CreateOpts = get_idx_w_opts(Opts), + case mango_crud:insert(Db, NewDDoc, CreateOpts) of + {ok, [{RespProps}]} -> + case lists:keyfind(error, 1, RespProps) of + {error, Reason} -> + ?MANGO_ERROR({error_saving_ddoc, Reason}); + _ -> + <<"created">> + end; + _ -> + ?MANGO_ERROR(error_saving_ddoc) + end + end, + chttpd:send_json(Req, {[{result, Status}, {id, Id}, {name, Name}]}); +handle_index_req(#httpd{path_parts = [_, _]} = Req, _Db) -> chttpd:send_method_not_allowed(Req, "GET,POST"); - %% Essentially we just iterate through the list of ddoc ids passed in and %% delete one by one. If an error occurs, all previous documents will be %% deleted, but an error will be thrown for the current ddoc id. -handle_index_req(#httpd{method='POST', path_parts=[_, <<"_index">>, - <<"_bulk_delete">>]}=Req, Db) -> +handle_index_req( + #httpd{ + method = 'POST', + path_parts = [ + _, + <<"_index">>, + <<"_bulk_delete">> + ] + } = Req, + Db +) -> chttpd:validate_ctype(Req, "application/json"), {ok, Opts} = mango_opts:validate_bulk_delete(chttpd:json_body_obj(Req)), Idxs = mango_idx:list(Db), DDocs = get_bulk_delete_ddocs(Opts), DelOpts = get_idx_w_opts(Opts), - {Success, Fail} = lists:foldl(fun(DDocId0, {Success0, Fail0}) -> - DDocId = convert_to_design_id(DDocId0), - Filt = fun(Idx) -> mango_idx:ddoc(Idx) == DDocId end, - Id = {<<"id">>, DDocId}, - case mango_idx:delete(Filt, Db, Idxs, DelOpts) of - {ok, true} -> - {[{[Id, {<<"ok">>, true}]} | Success0], Fail0}; - {error, Error} -> - {Success0, [{[Id, {<<"error">>, Error}]} | Fail0]} - end - end, {[], []}, DDocs), + {Success, Fail} = lists:foldl( + fun(DDocId0, {Success0, Fail0}) -> + DDocId = convert_to_design_id(DDocId0), + Filt = fun(Idx) -> mango_idx:ddoc(Idx) == DDocId end, + Id = {<<"id">>, DDocId}, + case mango_idx:delete(Filt, Db, Idxs, DelOpts) of + {ok, true} -> + {[{[Id, {<<"ok">>, true}]} | Success0], Fail0}; + {error, Error} -> + {Success0, [{[Id, {<<"error">>, Error}]} | Fail0]} + end + end, + {[], []}, + DDocs + ), chttpd:send_json(Req, {[{<<"success">>, Success}, {<<"fail">>, Fail}]}); - -handle_index_req(#httpd{path_parts=[_, <<"_index">>, - <<"_bulk_delete">>]}=Req, _Db) -> +handle_index_req( + #httpd{ + path_parts = [ + _, + <<"_index">>, + <<"_bulk_delete">> + ] + } = Req, + _Db +) -> chttpd:send_method_not_allowed(Req, "POST"); - -handle_index_req(#httpd{method='DELETE', - path_parts=[A, B, <<"_design">>, DDocId0, Type, Name]}=Req, Db) -> +handle_index_req( + #httpd{ + method = 'DELETE', + path_parts = [A, B, <<"_design">>, DDocId0, Type, Name] + } = Req, + Db +) -> PathParts = [A, B, <<"_design/", DDocId0/binary>>, Type, Name], - handle_index_req(Req#httpd{path_parts=PathParts}, Db); - -handle_index_req(#httpd{method='DELETE', - path_parts=[_, _, DDocId0, Type, Name]}=Req, Db) -> + handle_index_req(Req#httpd{path_parts = PathParts}, Db); +handle_index_req( + #httpd{ + method = 'DELETE', + path_parts = [_, _, DDocId0, Type, Name] + } = Req, + Db +) -> Idxs = mango_idx:list(Db), DDocId = convert_to_design_id(DDocId0), DelOpts = get_idx_del_opts(Req), @@ -162,24 +188,20 @@ handle_index_req(#httpd{method='DELETE', {error, Error} -> ?MANGO_ERROR({error_saving_ddoc, Error}) end; - -handle_index_req(#httpd{path_parts=[_, _, _DDocId0, _Type, _Name]}=Req, _Db) -> +handle_index_req(#httpd{path_parts = [_, _, _DDocId0, _Type, _Name]} = Req, _Db) -> chttpd:send_method_not_allowed(Req, "DELETE"). - -handle_explain_req(#httpd{method='POST'}=Req, Db) -> +handle_explain_req(#httpd{method = 'POST'} = Req, Db) -> chttpd:validate_ctype(Req, "application/json"), Body = maybe_set_partition(Req), {ok, Opts0} = mango_opts:validate_find(Body), {value, {selector, Sel}, Opts} = lists:keytake(selector, 1, Opts0), Resp = mango_crud:explain(Db, Sel, Opts), chttpd:send_json(Req, Resp); - handle_explain_req(Req, _Db) -> chttpd:send_method_not_allowed(Req, "POST"). - -handle_find_req(#httpd{method='POST'}=Req, Db) -> +handle_find_req(#httpd{method = 'POST'} = Req, Db) -> chttpd:validate_ctype(Req, "application/json"), Body = maybe_set_partition(Req), {ok, Opts0} = mango_opts:validate_find(Body), @@ -191,17 +213,13 @@ handle_find_req(#httpd{method='POST'}=Req, Db) -> {error, Error} -> chttpd:send_error(Req, Error) end; - - handle_find_req(Req, _Db) -> chttpd:send_method_not_allowed(Req, "POST"). - -set_user_ctx(#httpd{user_ctx=Ctx}, Db) -> +set_user_ctx(#httpd{user_ctx = Ctx}, Db) -> {ok, NewDb} = couch_db:set_user_ctx(Db, Ctx), NewDb. - get_idx_w_opts(Opts) -> case lists:keyfind(w, 1, Opts) of {w, N} when is_integer(N), N > 0 -> @@ -210,7 +228,6 @@ get_idx_w_opts(Opts) -> [{w, "2"}] end. - get_bulk_delete_ddocs(Opts) -> case lists:keyfind(docids, 1, Opts) of {docids, DDocs} when is_list(DDocs) -> @@ -219,17 +236,16 @@ get_bulk_delete_ddocs(Opts) -> [] end. - get_idx_del_opts(Req) -> try WStr = chttpd:qs_value(Req, "w", "2"), _ = list_to_integer(WStr), [{w, WStr}] - catch _:_ -> - [{w, "2"}] + catch + _:_ -> + [{w, "2"}] end. - maybe_set_partition(Req) -> {Props} = chttpd:json_body_obj(Req), case chttpd:qs_value(Req, "partition", undefined) of @@ -246,31 +262,31 @@ maybe_set_partition(Req) -> end end. - convert_to_design_id(DDocId) -> case DDocId of <<"_design/", _/binary>> -> DDocId; _ -> <<"_design/", DDocId/binary>> end. - start_find_resp(Req) -> chttpd:start_delayed_json_response(Req, 200, [], "{\"docs\":["). - end_find_resp(Acc0) -> - #vacc{resp=Resp00, buffer=Buf, kvs=KVs, threshold=Max} = Acc0, + #vacc{resp = Resp00, buffer = Buf, kvs = KVs, threshold = Max} = Acc0, {ok, Resp0} = chttpd:close_delayed_json_object(Resp00, Buf, "\r\n]", Max), - FinalAcc = lists:foldl(fun({K, V}, Acc) -> - JK = ?JSON_ENCODE(K), - JV = ?JSON_ENCODE(V), - [JV, ": ", JK, ",\r\n" | Acc] - end, [], KVs), + FinalAcc = lists:foldl( + fun({K, V}, Acc) -> + JK = ?JSON_ENCODE(K), + JV = ?JSON_ENCODE(V), + [JV, ": ", JK, ",\r\n" | Acc] + end, + [], + KVs + ), Chunk = lists:reverse(FinalAcc, ["}\r\n"]), {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, Chunk), chttpd:end_delayed_json_response(Resp1). - run_find(Resp, Db, Sel, Opts) -> Acc0 = #vacc{ resp = Resp, @@ -280,18 +296,18 @@ run_find(Resp, Db, Sel, Opts) -> }, mango_crud:find(Db, Sel, fun handle_doc/2, Acc0, Opts). - handle_doc({add_key, Key, Value}, Acc0) -> - #vacc{kvs=KVs} = Acc0, + #vacc{kvs = KVs} = Acc0, NewKVs = lists:keystore(Key, 1, KVs, {Key, Value}), {ok, Acc0#vacc{kvs = NewKVs}}; handle_doc({row, Doc}, Acc0) -> - #vacc{prepend=Prepend} = Acc0, + #vacc{prepend = Prepend} = Acc0, Chunk = [Prepend, ?JSON_ENCODE(Doc)], maybe_flush_response(Acc0, Chunk, iolist_size(Chunk)). -maybe_flush_response(#vacc{bufsize=Size, threshold=Max} = Acc, Data, Len) - when Size > 0 andalso (Size + Len) > Max -> +maybe_flush_response(#vacc{bufsize = Size, threshold = Max} = Acc, Data, Len) when + Size > 0 andalso (Size + Len) > Max +-> #vacc{buffer = Buffer, resp = Resp} = Acc, {ok, R1} = chttpd:send_delayed_chunk(Resp, Buffer), {ok, Acc#vacc{prepend = ",\r\n", buffer = Data, bufsize = Len, resp = R1}}; @@ -304,18 +320,17 @@ maybe_flush_response(Acc0, Data, Len) -> }, {ok, Acc}. - parse_index_param("limit", Value) -> [{limit, parse_val(Value)}]; parse_index_param("skip", Value) -> [{skip, parse_val(Value)}]; parse_index_param(_Key, _Value) -> - []. + []. parse_val(Value) -> case (catch list_to_integer(Value)) of - IntVal when is_integer(IntVal) -> - IntVal; - _ -> - ?MANGO_ERROR(invalid_list_index_params) + IntVal when is_integer(IntVal) -> + IntVal; + _ -> + ?MANGO_ERROR(invalid_list_index_params) end. diff --git a/src/mango/src/mango_httpd_handlers.erl b/src/mango/src/mango_httpd_handlers.erl index 80e5e277e..feb693e94 100644 --- a/src/mango/src/mango_httpd_handlers.erl +++ b/src/mango/src/mango_httpd_handlers.erl @@ -16,9 +16,9 @@ url_handler(_) -> no_match. -db_handler(<<"_index">>) -> fun mango_httpd:handle_req/2; -db_handler(<<"_explain">>) -> fun mango_httpd:handle_req/2; -db_handler(<<"_find">>) -> fun mango_httpd:handle_req/2; +db_handler(<<"_index">>) -> fun mango_httpd:handle_req/2; +db_handler(<<"_explain">>) -> fun mango_httpd:handle_req/2; +db_handler(<<"_find">>) -> fun mango_httpd:handle_req/2; db_handler(_) -> no_match. design_handler(_) -> no_match. diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl index 5d06a8fe3..a2daa8b9e 100644 --- a/src/mango/src/mango_idx.erl +++ b/src/mango/src/mango_idx.erl @@ -16,7 +16,6 @@ -module(mango_idx). - -export([ list/1, recover/1, @@ -47,22 +46,19 @@ get_partial_filter_selector/1 ]). - -include_lib("couch/include/couch_db.hrl"). -include("mango.hrl"). -include("mango_idx.hrl"). - list(Db) -> {ok, Indexes} = ddoc_cache:open(db_to_name(Db), ?MODULE), Indexes. - get_usable_indexes(Db, Selector, Opts) -> ExistingIndexes = mango_idx:list(Db), GlobalIndexes = mango_cursor:remove_indexes_with_partial_filter_selector( - ExistingIndexes - ), + ExistingIndexes + ), UserSpecifiedIndex = mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts), UsableIndexes0 = lists:usort(GlobalIndexes ++ UserSpecifiedIndex), UsableIndexes1 = filter_partition_indexes(UsableIndexes0, Opts), @@ -77,7 +73,6 @@ get_usable_indexes(Db, Selector, Opts) -> UsableIndexes end. - mango_sort_error(Db, Opts) -> case {fabric_util:is_partitioned(Db), is_opts_partitioned(Opts)} of {false, _} -> @@ -88,7 +83,6 @@ mango_sort_error(Db, Opts) -> ?MANGO_ERROR({no_usable_index, missing_sort_index_global}) end. - recover(Db) -> {ok, DDocs0} = mango_util:open_ddocs(Db), Pred = fun({Props}) -> @@ -99,10 +93,14 @@ recover(Db) -> end, DDocs = lists:filter(Pred, DDocs0), Special = special(Db), - {ok, Special ++ lists:flatmap(fun(Doc) -> - from_ddoc(Db, Doc) - end, DDocs)}. - + {ok, + Special ++ + lists:flatmap( + fun(Doc) -> + from_ddoc(Db, Doc) + end, + DDocs + )}. get_sort_fields(Opts) -> case lists:keyfind(sort, 1, Opts) of @@ -112,7 +110,6 @@ get_sort_fields(Opts) -> [] end. - new(Db, Opts) -> Def = get_idx_def(Opts), Type = get_idx_type(Opts), @@ -128,12 +125,10 @@ new(Db, Opts) -> opts = filter_opts(Opts) }}. - validate_new(Idx, Db) -> Mod = idx_mod(Idx), Mod:validate_new(Idx, Db). - add(DDoc, Idx) -> Mod = idx_mod(Idx), {ok, NewDDoc1} = Mod:add(DDoc, Idx), @@ -142,7 +137,6 @@ add(DDoc, Idx) -> Body = ?JSON_DECODE(?JSON_ENCODE(NewDDoc2#doc.body)), {ok, NewDDoc2#doc{body = Body}}. - remove(DDoc, Idx) -> Mod = idx_mod(Idx), {ok, NewDDoc} = Mod:remove(DDoc, Idx), @@ -150,18 +144,18 @@ remove(DDoc, Idx) -> Body = ?JSON_DECODE(?JSON_ENCODE(NewDDoc#doc.body)), {ok, NewDDoc#doc{body = Body}}. - delete(Filt, Db, Indexes, DelOpts) -> case lists:filter(Filt, Indexes) of [Idx] -> {ok, DDoc} = mango_util:load_ddoc(Db, mango_idx:ddoc(Idx)), {ok, NewDDoc} = mango_idx:remove(DDoc, Idx), - FinalDDoc = case NewDDoc#doc.body of - {[{<<"language">>, <<"query">>}]} -> - NewDDoc#doc{deleted = true, body = {[]}}; - _ -> - NewDDoc - end, + FinalDDoc = + case NewDDoc#doc.body of + {[{<<"language">>, <<"query">>}]} -> + NewDDoc#doc{deleted = true, body = {[]}}; + _ -> + NewDDoc + end, case mango_crud:insert(Db, FinalDDoc, DelOpts) of {ok, _} -> {ok, true}; @@ -172,31 +166,32 @@ delete(Filt, Db, Indexes, DelOpts) -> {error, not_found} end. - from_ddoc(Db, {Props}) -> DbName = db_to_name(Db), DDoc = proplists:get_value(<<"_id">>, Props), case proplists:get_value(<<"language">>, Props) of <<"query">> -> ok; - _ -> - ?MANGO_ERROR(invalid_query_ddoc_language) - end, - IdxMods = case clouseau_rpc:connected() of - true -> - [mango_idx_view, mango_idx_text]; - false -> - [mango_idx_view] + _ -> ?MANGO_ERROR(invalid_query_ddoc_language) end, + IdxMods = + case clouseau_rpc:connected() of + true -> + [mango_idx_view, mango_idx_text]; + false -> + [mango_idx_view] + end, Idxs = lists:flatmap(fun(Mod) -> Mod:from_ddoc({Props}) end, IdxMods), - lists:map(fun(Idx) -> - Idx#idx{ - dbname = DbName, - ddoc = DDoc, - partitioned = get_idx_partitioned(Db, Props) - } - end, Idxs). - + lists:map( + fun(Idx) -> + Idx#idx{ + dbname = DbName, + ddoc = DDoc, + partitioned = get_idx_partitioned(Db, Props) + } + end, + Idxs + ). special(Db) -> AllDocs = #idx{ @@ -209,63 +204,50 @@ special(Db) -> % Add one for _update_seq [AllDocs]. - -dbname(#idx{dbname=DbName}) -> +dbname(#idx{dbname = DbName}) -> DbName. - -ddoc(#idx{ddoc=DDoc}) -> +ddoc(#idx{ddoc = DDoc}) -> DDoc. - -name(#idx{name=Name}) -> +name(#idx{name = Name}) -> Name. - -type(#idx{type=Type}) -> +type(#idx{type = Type}) -> Type. - -def(#idx{def=Def}) -> +def(#idx{def = Def}) -> Def. - -partitioned(#idx{partitioned=Partitioned}) -> +partitioned(#idx{partitioned = Partitioned}) -> Partitioned. - -opts(#idx{opts=Opts}) -> +opts(#idx{opts = Opts}) -> Opts. - -to_json(#idx{}=Idx) -> +to_json(#idx{} = Idx) -> Mod = idx_mod(Idx), Mod:to_json(Idx). - -columns(#idx{}=Idx) -> +columns(#idx{} = Idx) -> Mod = idx_mod(Idx), Mod:columns(Idx). - -is_usable(#idx{}=Idx, Selector, SortFields) -> +is_usable(#idx{} = Idx, Selector, SortFields) -> Mod = idx_mod(Idx), Mod:is_usable(Idx, Selector, SortFields). - -start_key(#idx{}=Idx, Ranges) -> +start_key(#idx{} = Idx, Ranges) -> Mod = idx_mod(Idx), Mod:start_key(Ranges). - -end_key(#idx{}=Idx, Ranges) -> +end_key(#idx{} = Idx, Ranges) -> Mod = idx_mod(Idx), Mod:end_key(Ranges). - cursor_mod(#idx{type = <<"json">>}) -> mango_cursor_view; -cursor_mod(#idx{def = all_docs, type= <<"special">>}) -> +cursor_mod(#idx{def = all_docs, type = <<"special">>}) -> mango_cursor_special; cursor_mod(#idx{type = <<"text">>}) -> case clouseau_rpc:connected() of @@ -275,7 +257,6 @@ cursor_mod(#idx{type = <<"text">>}) -> ?MANGO_ERROR({index_service_unavailable, <<"text">>}) end. - idx_mod(#idx{type = <<"json">>}) -> mango_idx_view; idx_mod(#idx{type = <<"special">>}) -> @@ -288,7 +269,6 @@ idx_mod(#idx{type = <<"text">>}) -> ?MANGO_ERROR({index_service_unavailable, <<"text">>}) end. - db_to_name(Name) when is_binary(Name) -> Name; db_to_name(Name) when is_list(Name) -> @@ -296,7 +276,6 @@ db_to_name(Name) when is_list(Name) -> db_to_name(Db) -> couch_db:name(Db). - get_idx_def(Opts) -> case proplists:get_value(def, Opts) of undefined -> @@ -305,23 +284,24 @@ get_idx_def(Opts) -> Def end. - get_idx_type(Opts) -> case proplists:get_value(type, Opts) of - <<"json">> -> <<"json">>; - <<"text">> -> case clouseau_rpc:connected() of - true -> - <<"text">>; - false -> - ?MANGO_ERROR({index_service_unavailable, <<"text">>}) + <<"json">> -> + <<"json">>; + <<"text">> -> + case clouseau_rpc:connected() of + true -> + <<"text">>; + false -> + ?MANGO_ERROR({index_service_unavailable, <<"text">>}) end; %<<"geo">> -> <<"geo">>; - undefined -> <<"json">>; + undefined -> + <<"json">>; BadType -> ?MANGO_ERROR({invalid_index_type, BadType}) end. - get_idx_ddoc(Idx, Opts) -> case proplists:get_value(ddoc, Opts) of <<"_design/", _Rest/binary>> = Name -> @@ -333,7 +313,6 @@ get_idx_ddoc(Idx, Opts) -> <<"_design/", Bin/binary>> end. - get_idx_name(Idx, Opts) -> case proplists:get_value(name, Opts) of Name when is_binary(Name) -> @@ -342,14 +321,12 @@ get_idx_name(Idx, Opts) -> gen_name(Idx, Opts) end. - gen_name(Idx, Opts0) -> Opts = lists:usort(Opts0), TermBin = term_to_binary({Idx, Opts}), Sha = crypto:hash(sha, TermBin), mango_util:enc_hex(Sha). - get_idx_partitioned(Opts) -> case proplists:get_value(partitioned, Opts) of B when is_boolean(B) -> @@ -360,7 +337,6 @@ get_idx_partitioned(Opts) -> undefined end. - set_ddoc_partitioned(DDoc, Idx) -> % We have to verify that the new index being added % to this design document either matches the current @@ -371,42 +347,45 @@ set_ddoc_partitioned(DDoc, Idx) -> body = {BodyProps} } = DDoc, OldDOpts = couch_util:get_value(<<"options">>, BodyProps), - OldOpt = case OldDOpts of - {OldDOptProps} when is_list(OldDOptProps) -> - couch_util:get_value(<<"partitioned">>, OldDOptProps); - _ -> - undefined - end, + OldOpt = + case OldDOpts of + {OldDOptProps} when is_list(OldDOptProps) -> + couch_util:get_value(<<"partitioned">>, OldDOptProps); + _ -> + undefined + end, % If new matches old we're done - if Idx#idx.partitioned == OldOpt -> DDoc; true -> - % If we're creating a ddoc then we can set the options - case Revs == {0, []} of - true when Idx#idx.partitioned /= undefined -> - set_ddoc_partitioned_option(DDoc, Idx#idx.partitioned); - true when Idx#idx.partitioned == undefined -> - DDoc; - false -> - ?MANGO_ERROR({partitioned_option_mismatch, DDocId}) - end + if + Idx#idx.partitioned == OldOpt -> + DDoc; + true -> + % If we're creating a ddoc then we can set the options + case Revs == {0, []} of + true when Idx#idx.partitioned /= undefined -> + set_ddoc_partitioned_option(DDoc, Idx#idx.partitioned); + true when Idx#idx.partitioned == undefined -> + DDoc; + false -> + ?MANGO_ERROR({partitioned_option_mismatch, DDocId}) + end end. - set_ddoc_partitioned_option(DDoc, Partitioned) -> #doc{ body = {BodyProps} } = DDoc, - NewProps = case couch_util:get_value(<<"options">>, BodyProps) of - {Existing} when is_list(Existing) -> - Opt = {<<"partitioned">>, Partitioned}, - New = lists:keystore(<<"partitioned">>, 1, Existing, Opt), - lists:keystore(<<"options">>, 1, BodyProps, {<<"options">>, New}); - undefined -> - New = {<<"options">>, {[{<<"partitioned">>, Partitioned}]}}, - lists:keystore(<<"options">>, 1, BodyProps, New) - end, + NewProps = + case couch_util:get_value(<<"options">>, BodyProps) of + {Existing} when is_list(Existing) -> + Opt = {<<"partitioned">>, Partitioned}, + New = lists:keystore(<<"partitioned">>, 1, Existing, Opt), + lists:keystore(<<"options">>, 1, BodyProps, {<<"options">>, New}); + undefined -> + New = {<<"options">>, {[{<<"partitioned">>, Partitioned}]}}, + lists:keystore(<<"options">>, 1, BodyProps, New) + end, DDoc#doc{body = {NewProps}}. - get_idx_partitioned(Db, DDocProps) -> Default = fabric_util:is_partitioned(Db), case couch_util:get_value(<<"options">>, DDocProps) of @@ -429,18 +408,17 @@ is_opts_partitioned(Opts) -> true end. - filter_partition_indexes(Indexes, Opts) -> - PFilt = case is_opts_partitioned(Opts) of - false -> - fun(#idx{partitioned = P}) -> not P end; - true -> - fun(#idx{partitioned = P}) -> P end - end, + PFilt = + case is_opts_partitioned(Opts) of + false -> + fun(#idx{partitioned = P}) -> not P end; + true -> + fun(#idx{partitioned = P}) -> P end + end, Filt = fun(Idx) -> type(Idx) == <<"special">> orelse PFilt(Idx) end, lists:filter(Filt, Indexes). - filter_opts([]) -> []; filter_opts([{user_ctx, _} | Rest]) -> @@ -458,7 +436,6 @@ filter_opts([{partitioned, _} | Rest]) -> filter_opts([Opt | Rest]) -> [Opt | filter_opts(Rest)]. - get_partial_filter_selector(#idx{def = Def}) when Def =:= all_docs; Def =:= undefined -> undefined; get_partial_filter_selector(#idx{def = {Def}}) -> @@ -468,7 +445,6 @@ get_partial_filter_selector(#idx{def = {Def}}) -> Selector -> Selector end. - % Partial filter selectors is supported in text indexes via the selector field % This adds backwards support for existing indexes that might have a selector in it get_legacy_selector(Def) -> @@ -483,13 +459,17 @@ get_legacy_selector(Def) -> index(SelectorName, Selector) -> { - idx,<<"mango_test_46418cd02081470d93290dc12306ebcb">>, - <<"_design/57e860dee471f40a2c74ea5b72997b81dda36a24">>, - <<"Selected">>,<<"json">>, - {[{<<"fields">>,{[{<<"location">>,<<"asc">>}]}}, - {SelectorName,{Selector}}]}, - false, - [{<<"def">>,{[{<<"fields">>,[<<"location">>]}]}}] + idx, + <<"mango_test_46418cd02081470d93290dc12306ebcb">>, + <<"_design/57e860dee471f40a2c74ea5b72997b81dda36a24">>, + <<"Selected">>, + <<"json">>, + {[ + {<<"fields">>, {[{<<"location">>, <<"asc">>}]}}, + {SelectorName, {Selector}} + ]}, + false, + [{<<"def">>, {[{<<"fields">>, [<<"location">>]}]}}] }. get_partial_filter_all_docs_test() -> @@ -509,12 +489,12 @@ get_partial_filter_selector_missing_test() -> ?assertEqual(undefined, get_partial_filter_selector(Idx)). get_partial_filter_selector_with_selector_test() -> - Selector = [{<<"location">>,{[{<<"$gt">>,<<"FRA">>}]}}], + Selector = [{<<"location">>, {[{<<"$gt">>, <<"FRA">>}]}}], Idx = index(<<"partial_filter_selector">>, Selector), ?assertEqual({Selector}, get_partial_filter_selector(Idx)). get_partial_filter_selector_with_legacy_selector_test() -> - Selector = [{<<"location">>,{[{<<"$gt">>,<<"FRA">>}]}}], + Selector = [{<<"location">>, {[{<<"$gt">>, <<"FRA">>}]}}], Idx = index(<<"selector">>, Selector), ?assertEqual({Selector}, get_partial_filter_selector(Idx)). @@ -522,7 +502,6 @@ get_partial_filter_selector_with_legacy_default_selector_test() -> Idx = index(<<"selector">>, []), ?assertEqual(undefined, get_partial_filter_selector(Idx)). - get_idx_ddoc_name_only_test() -> Opts = [{ddoc, <<"foo">>}], ?assertEqual(<<"_design/foo">>, get_idx_ddoc({}, Opts)). diff --git a/src/mango/src/mango_idx_special.erl b/src/mango/src/mango_idx_special.erl index ac6efc707..4c4001c80 100644 --- a/src/mango/src/mango_idx_special.erl +++ b/src/mango/src/mango_idx_special.erl @@ -12,7 +12,6 @@ -module(mango_idx_special). - -export([ validate/1, add/2, @@ -25,51 +24,45 @@ end_key/1 ]). - -include_lib("couch/include/couch_db.hrl"). -include("mango_idx.hrl"). - validate(_) -> erlang:exit(invalid_call). - add(_, _) -> erlang:exit(invalid_call). - remove(_, _) -> erlang:exit(invalid_call). - from_ddoc(_) -> erlang:exit(invalid_call). - -to_json(#idx{def=all_docs}) -> +to_json(#idx{def = all_docs}) -> {[ {ddoc, null}, {name, <<"_all_docs">>}, {type, <<"special">>}, - {def, {[ - {<<"fields">>, [{[ - {<<"_id">>, <<"asc">>} - ]}]} - ]}} + {def, + {[ + {<<"fields">>, [ + {[ + {<<"_id">>, <<"asc">>} + ]} + ]} + ]}} ]}. - -columns(#idx{def=all_docs}) -> +columns(#idx{def = all_docs}) -> [<<"_id">>]. - -is_usable(#idx{def=all_docs}, _Selector, []) -> +is_usable(#idx{def = all_docs}, _Selector, []) -> true; -is_usable(#idx{def=all_docs} = Idx, Selector, SortFields) -> +is_usable(#idx{def = all_docs} = Idx, Selector, SortFields) -> Fields = mango_idx_view:indexable_fields(Selector), lists:member(<<"_id">>, Fields) and can_use_sort(Idx, SortFields, Selector). - start_key([{'$gt', Key, _, _}]) -> case mango_json:special(Key) of true -> @@ -84,7 +77,6 @@ start_key([{'$eq', Key, '$eq', Key}]) -> false = mango_json:special(Key), Key. - end_key([{_, _, '$lt', Key}]) -> case mango_json:special(Key) of true -> @@ -99,7 +91,6 @@ end_key([{'$eq', Key, '$eq', Key}]) -> false = mango_json:special(Key), Key. - can_use_sort(_Idx, [], _Selector) -> true; can_use_sort(Idx, SortFields, _Selector) -> diff --git a/src/mango/src/mango_idx_text.erl b/src/mango/src/mango_idx_text.erl index 1d4becfb3..b4a46d688 100644 --- a/src/mango/src/mango_idx_text.erl +++ b/src/mango/src/mango_idx_text.erl @@ -12,7 +12,6 @@ -module(mango_idx_text). - -export([ validate_new/2, validate_fields/1, @@ -26,75 +25,74 @@ get_default_field_options/1 ]). - -include_lib("couch/include/couch_db.hrl"). -include("mango.hrl"). -include("mango_idx.hrl"). - -validate_new(#idx{}=Idx, Db) -> +validate_new(#idx{} = Idx, Db) -> {ok, Def} = do_validate(Idx#idx.def), maybe_reject_index_all_req(Def, Db), - {ok, Idx#idx{def=Def}}. - + {ok, Idx#idx{def = Def}}. validate_index_def(IndexInfo) -> do_validate(IndexInfo). - -add(#doc{body={Props0}}=DDoc, Idx) -> - Texts1 = case proplists:get_value(<<"indexes">>, Props0) of - {Texts0} -> Texts0; - _ -> [] - end, +add(#doc{body = {Props0}} = DDoc, Idx) -> + Texts1 = + case proplists:get_value(<<"indexes">>, Props0) of + {Texts0} -> Texts0; + _ -> [] + end, NewText = make_text(Idx), Texts2 = lists:keystore(element(1, NewText), 1, Texts1, NewText), - Props1 = lists:keystore(<<"indexes">>, 1, Props0, {<<"indexes">>, - {Texts2}}), - {ok, DDoc#doc{body={Props1}}}. - - -remove(#doc{body={Props0}}=DDoc, Idx) -> - Texts1 = case proplists:get_value(<<"indexes">>, Props0) of - {Texts0} -> - Texts0; - _ -> - ?MANGO_ERROR({index_not_found, Idx#idx.name}) - end, + Props1 = lists:keystore(<<"indexes">>, 1, Props0, {<<"indexes">>, {Texts2}}), + {ok, DDoc#doc{body = {Props1}}}. + +remove(#doc{body = {Props0}} = DDoc, Idx) -> + Texts1 = + case proplists:get_value(<<"indexes">>, Props0) of + {Texts0} -> + Texts0; + _ -> + ?MANGO_ERROR({index_not_found, Idx#idx.name}) + end, Texts2 = lists:keydelete(Idx#idx.name, 1, Texts1), - if Texts2 /= Texts1 -> ok; true -> - ?MANGO_ERROR({index_not_found, Idx#idx.name}) + if + Texts2 /= Texts1 -> ok; + true -> ?MANGO_ERROR({index_not_found, Idx#idx.name}) end, - Props1 = case Texts2 of - [] -> - lists:keydelete(<<"indexes">>, 1, Props0); - _ -> - lists:keystore(<<"indexes">>, 1, Props0, {<<"indexes">>, {Texts2}}) - end, - {ok, DDoc#doc{body={Props1}}}. - + Props1 = + case Texts2 of + [] -> + lists:keydelete(<<"indexes">>, 1, Props0); + _ -> + lists:keystore(<<"indexes">>, 1, Props0, {<<"indexes">>, {Texts2}}) + end, + {ok, DDoc#doc{body = {Props1}}}. from_ddoc({Props}) -> case lists:keyfind(<<"indexes">>, 1, Props) of {<<"indexes">>, {Texts}} when is_list(Texts) -> - lists:flatmap(fun({Name, {VProps}}) -> - case validate_ddoc(VProps) of - invalid_ddoc -> - []; - Def -> - I = #idx{ - type = <<"text">>, - name = Name, - def = Def - }, - [I] - end - end, Texts); + lists:flatmap( + fun({Name, {VProps}}) -> + case validate_ddoc(VProps) of + invalid_ddoc -> + []; + Def -> + I = #idx{ + type = <<"text">>, + name = Name, + def = Def + }, + [I] + end + end, + Texts + ); _ -> [] end. - to_json(Idx) -> {[ {ddoc, Idx#idx.ddoc}, @@ -104,7 +102,6 @@ to_json(Idx) -> {def, {def_to_json(Idx#idx.def)}} ]}. - columns(Idx) -> {Props} = Idx#idx.def, {<<"fields">>, Fields} = lists:keyfind(<<"fields">>, 1, Props), @@ -114,18 +111,22 @@ columns(Idx) -> _ -> {DFProps} = couch_util:get_value(<<"default_field">>, Props, {[]}), Enabled = couch_util:get_value(<<"enabled">>, DFProps, true), - Default = case Enabled of - true -> [<<"$default">>]; - false -> [] - end, - Default ++ lists:map(fun({FProps}) -> - {_, Name} = lists:keyfind(<<"name">>, 1, FProps), - {_, Type} = lists:keyfind(<<"type">>, 1, FProps), - iolist_to_binary([Name, ":", Type]) - end, Fields) + Default = + case Enabled of + true -> [<<"$default">>]; + false -> [] + end, + Default ++ + lists:map( + fun({FProps}) -> + {_, Name} = lists:keyfind(<<"name">>, 1, FProps), + {_, Type} = lists:keyfind(<<"type">>, 1, FProps), + iolist_to_binary([Name, ":", Type]) + end, + Fields + ) end. - is_usable(_, Selector, _) when Selector =:= {[]} -> false; is_usable(Idx, Selector, _) -> @@ -137,14 +138,12 @@ is_usable(Idx, Selector, _) -> sets:is_subset(sets:from_list(Fields), sets:from_list(Cols)) end. - do_validate({Props}) -> {ok, Opts} = mango_opts:validate(Props, opts()), {ok, {Opts}}; do_validate(Else) -> ?MANGO_ERROR({invalid_index_text, Else}). - def_to_json({Props}) -> def_to_json(Props); def_to_json([]) -> @@ -162,7 +161,6 @@ def_to_json([{<<"partial_filter_selector">>, {[]}} | Rest]) -> def_to_json([{Key, Value} | Rest]) -> [{Key, Value} | def_to_json(Rest)]. - fields_to_json([]) -> []; fields_to_json([{[{<<"name">>, Name}, {<<"type">>, Type0}]} | Rest]) -> @@ -174,17 +172,15 @@ fields_to_json([{[{<<"type">>, Type0}, {<<"name">>, Name}]} | Rest]) -> Type = validate_field_type(Type0), [{[{Name, Type}]} | fields_to_json(Rest)]. - %% In the future, we can possibly add more restrictive validation. %% For now, let's make sure the field name is not blank. validate_field_name(<<"">>) -> throw(invalid_field_name); -validate_field_name(Else) when is_binary(Else)-> +validate_field_name(Else) when is_binary(Else) -> ok; validate_field_name(_) -> throw(invalid_field_name). - validate_field_type(<<"string">>) -> <<"string">>; validate_field_type(<<"number">>) -> @@ -192,32 +188,33 @@ validate_field_type(<<"number">>) -> validate_field_type(<<"boolean">>) -> <<"boolean">>. - validate_fields(<<"all_fields">>) -> {ok, all_fields}; validate_fields(Fields) -> try fields_to_json(Fields) of _ -> mango_fields:new(Fields) - catch error:function_clause -> - ?MANGO_ERROR({invalid_index_fields_definition, Fields}); - throw:invalid_field_name -> - ?MANGO_ERROR({invalid_index_fields_definition, Fields}) + catch + error:function_clause -> + ?MANGO_ERROR({invalid_index_fields_definition, Fields}); + throw:invalid_field_name -> + ?MANGO_ERROR({invalid_index_fields_definition, Fields}) end. - validate_ddoc(VProps) -> try Def = proplists:get_value(<<"index">>, VProps), validate_index_def(Def), Def - catch Error:Reason -> - couch_log:error("Invalid Index Def ~p: Error. ~p, Reason: ~p", - [VProps, Error, Reason]), - invalid_ddoc + catch + Error:Reason -> + couch_log:error( + "Invalid Index Def ~p: Error. ~p, Reason: ~p", + [VProps, Error, Reason] + ), + invalid_ddoc end. - opts() -> [ {<<"default_analyzer">>, [ @@ -256,15 +253,14 @@ opts() -> ]} ]. - make_text(Idx) -> - Text= {[ - {<<"index">>, Idx#idx.def}, - {<<"analyzer">>, construct_analyzer(Idx#idx.def)} - ]}, + Text = + {[ + {<<"index">>, Idx#idx.def}, + {<<"analyzer">>, construct_analyzer(Idx#idx.def)} + ]}, {Idx#idx.name, Text}. - get_default_field_options(Props) -> Default = couch_util:get_value(default_field, Props, {[]}), case Default of @@ -272,24 +268,30 @@ get_default_field_options(Props) -> {Bool, <<"standard">>}; {[]} -> {true, <<"standard">>}; - {Opts}-> + {Opts} -> Enabled = couch_util:get_value(<<"enabled">>, Opts, true), - Analyzer = couch_util:get_value(<<"analyzer">>, Opts, - <<"standard">>), + Analyzer = couch_util:get_value( + <<"analyzer">>, + Opts, + <<"standard">> + ), {Enabled, Analyzer} end. - construct_analyzer({Props}) -> - DefaultAnalyzer = couch_util:get_value(default_analyzer, Props, - <<"keyword">>), + DefaultAnalyzer = couch_util:get_value( + default_analyzer, + Props, + <<"keyword">> + ), {DefaultField, DefaultFieldAnalyzer} = get_default_field_options(Props), - DefaultAnalyzerDef = case DefaultField of - true -> - [{<<"$default">>, DefaultFieldAnalyzer}]; - _ -> - [] - end, + DefaultAnalyzerDef = + case DefaultField of + true -> + [{<<"$default">>, DefaultFieldAnalyzer}]; + _ -> + [] + end, case DefaultAnalyzerDef of [] -> <<"keyword">>; @@ -301,23 +303,28 @@ construct_analyzer({Props}) -> ]} end. - indexable_fields(Selector) -> TupleTree = mango_selector_text:convert([], Selector), indexable_fields([], TupleTree). - indexable_fields(Fields, {op_and, Args}) when is_list(Args) -> - lists:foldl(fun(Arg, Fields0) -> indexable_fields(Fields0, Arg) end, - Fields, Args); - + lists:foldl( + fun(Arg, Fields0) -> indexable_fields(Fields0, Arg) end, + Fields, + Args + ); %% For queries that use array element access or $in operations, two %% fields get generated by mango_selector_text:convert. At index %% definition time, only one field gets defined. In this situation, we %% remove the extra generated field so that the index can be used. For %% all other situations, we include the fields as normal. -indexable_fields(Fields, {op_or, [{op_field, Field0}, - {op_field, {[Name | _], _}} = Field1]}) -> +indexable_fields( + Fields, + {op_or, [ + {op_field, Field0}, + {op_field, {[Name | _], _}} = Field1 + ]} +) -> case lists:member(<<"[]">>, Name) of true -> indexable_fields(Fields, {op_field, Field0}); @@ -326,40 +333,36 @@ indexable_fields(Fields, {op_or, [{op_field, Field0}, indexable_fields(Fields1, Field1) end; indexable_fields(Fields, {op_or, Args}) when is_list(Args) -> - lists:foldl(fun(Arg, Fields0) -> indexable_fields(Fields0, Arg) end, - Fields, Args); - + lists:foldl( + fun(Arg, Fields0) -> indexable_fields(Fields0, Arg) end, + Fields, + Args + ); indexable_fields(Fields, {op_not, {ExistsQuery, Arg}}) when is_tuple(Arg) -> Fields0 = indexable_fields(Fields, ExistsQuery), indexable_fields(Fields0, Arg); % forces "$exists" : false to use _all_docs indexable_fields(_, {op_not, {_, false}}) -> []; - indexable_fields(Fields, {op_insert, Arg}) when is_binary(Arg) -> Fields; - %% fieldname.[]:length is not a user defined field. indexable_fields(Fields, {op_field, {[_, <<":length">>], _}}) -> Fields; indexable_fields(Fields, {op_field, {Name, _}}) -> [iolist_to_binary(Name) | Fields]; - %% In this particular case, the lucene index is doing a field_exists query %% so it is looking at all sorts of combinations of field:* and field.* %% We don't add the field because we cannot pre-determine what field will exist. %% Hence we just return Fields and make it less restrictive. indexable_fields(Fields, {op_fieldname, {_, _}}) -> Fields; - %% Similar idea to op_fieldname but with fieldname:null indexable_fields(Fields, {op_null, {_, _}}) -> Fields; - indexable_fields(Fields, {op_default, _}) -> [<<"$default">> | Fields]. - maybe_reject_index_all_req({Def}, Db) -> DbName = couch_db:name(Db), #user_ctx{name = User} = couch_db:get_user_ctx(Db), @@ -368,48 +371,47 @@ maybe_reject_index_all_req({Def}, Db) -> {all_fields, "true"} -> ?MANGO_ERROR(index_all_disabled); {all_fields, "warn"} -> - couch_log:warning("User ~p is indexing all fields in db ~p", - [User, DbName]); + couch_log:warning( + "User ~p is indexing all fields in db ~p", + [User, DbName] + ); _ -> ok end. - forbid_index_all() -> config:get("mango", "index_all_disabled", "false"). - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). - setup_all() -> Ctx = test_util:start_couch(), - meck:expect(couch_log, warning, 2, - fun(_,_) -> + meck:expect( + couch_log, + warning, + 2, + fun(_, _) -> throw({test_error, logged_warning}) - end), + end + ), Ctx. - teardown_all(Ctx) -> meck:unload(), test_util:stop_couch(Ctx). - setup() -> %default index all def that generates {fields, all_fields} - Index = #idx{def={[]}}, + Index = #idx{def = {[]}}, DbName = <<"testdb">>, UserCtx = #user_ctx{name = <<"u1">>}, {ok, Db} = couch_db:clustered_db(DbName, UserCtx), {Index, Db}. - teardown(_) -> ok. - index_all_test_() -> { setup, @@ -427,34 +429,31 @@ index_all_test_() -> } }. - forbid_index_all({Idx, Db}) -> ?_test(begin ok = config:set("mango", "index_all_disabled", "true", false), - ?assertThrow({mango_error, ?MODULE, index_all_disabled}, + ?assertThrow( + {mango_error, ?MODULE, index_all_disabled}, validate_new(Idx, Db) ) end). - default_and_false_index_all({Idx, Db}) -> ?_test(begin config:delete("mango", "index_all_disabled", false), - {ok, #idx{def={Def}}} = validate_new(Idx, Db), + {ok, #idx{def = {Def}}} = validate_new(Idx, Db), Fields = couch_util:get_value(fields, Def), ?assertEqual(all_fields, Fields), ok = config:set("mango", "index_all_disabled", "false", false), - {ok, #idx{def={Def2}}} = validate_new(Idx, Db), + {ok, #idx{def = {Def2}}} = validate_new(Idx, Db), Fields2 = couch_util:get_value(fields, Def2), ?assertEqual(all_fields, Fields2) end). - warn_index_all({Idx, Db}) -> ?_test(begin ok = config:set("mango", "index_all_disabled", "warn", false), ?assertThrow({test_error, logged_warning}, validate_new(Idx, Db)) end). - -endif. diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl index 37911498c..ff8f6c6bb 100644 --- a/src/mango/src/mango_idx_view.erl +++ b/src/mango/src/mango_idx_view.erl @@ -12,7 +12,6 @@ -module(mango_idx_view). - -export([ validate_new/2, validate_index_def/1, @@ -30,75 +29,75 @@ field_ranges/2 ]). - -include_lib("couch/include/couch_db.hrl"). -include("mango.hrl"). -include("mango_idx.hrl"). -include("mango_idx_view.hrl"). - -validate_new(#idx{}=Idx, _Db) -> +validate_new(#idx{} = Idx, _Db) -> {ok, Def} = do_validate(Idx#idx.def), - {ok, Idx#idx{def=Def}}. - + {ok, Idx#idx{def = Def}}. validate_index_def(Def) -> def_to_json(Def). - -add(#doc{body={Props0}}=DDoc, Idx) -> - Views1 = case proplists:get_value(<<"views">>, Props0) of - {Views0} -> Views0; - _ -> [] - end, +add(#doc{body = {Props0}} = DDoc, Idx) -> + Views1 = + case proplists:get_value(<<"views">>, Props0) of + {Views0} -> Views0; + _ -> [] + end, NewView = make_view(Idx), Views2 = lists:keystore(element(1, NewView), 1, Views1, NewView), Props1 = lists:keystore(<<"views">>, 1, Props0, {<<"views">>, {Views2}}), - {ok, DDoc#doc{body={Props1}}}. - - -remove(#doc{body={Props0}}=DDoc, Idx) -> - Views1 = case proplists:get_value(<<"views">>, Props0) of - {Views0} -> - Views0; - _ -> - ?MANGO_ERROR({index_not_found, Idx#idx.name}) - end, + {ok, DDoc#doc{body = {Props1}}}. + +remove(#doc{body = {Props0}} = DDoc, Idx) -> + Views1 = + case proplists:get_value(<<"views">>, Props0) of + {Views0} -> + Views0; + _ -> + ?MANGO_ERROR({index_not_found, Idx#idx.name}) + end, Views2 = lists:keydelete(Idx#idx.name, 1, Views1), - if Views2 /= Views1 -> ok; true -> - ?MANGO_ERROR({index_not_found, Idx#idx.name}) - end, - Props1 = case Views2 of - [] -> - lists:keydelete(<<"views">>, 1, Props0); - _ -> - lists:keystore(<<"views">>, 1, Props0, {<<"views">>, {Views2}}) + if + Views2 /= Views1 -> ok; + true -> ?MANGO_ERROR({index_not_found, Idx#idx.name}) end, - {ok, DDoc#doc{body={Props1}}}. - + Props1 = + case Views2 of + [] -> + lists:keydelete(<<"views">>, 1, Props0); + _ -> + lists:keystore(<<"views">>, 1, Props0, {<<"views">>, {Views2}}) + end, + {ok, DDoc#doc{body = {Props1}}}. from_ddoc({Props}) -> case lists:keyfind(<<"views">>, 1, Props) of {<<"views">>, {Views}} when is_list(Views) -> - lists:flatmap(fun({Name, {VProps}}) -> - case validate_ddoc(VProps) of - invalid_view -> - []; - {Def, Opts} -> - I = #idx{ - type = <<"json">>, - name = Name, - def = Def, - opts = Opts - }, - [I] - end - end, Views); + lists:flatmap( + fun({Name, {VProps}}) -> + case validate_ddoc(VProps) of + invalid_view -> + []; + {Def, Opts} -> + I = #idx{ + type = <<"json">>, + name = Name, + def = Def, + opts = Opts + }, + [I] + end + end, + Views + ); _ -> [] end. - to_json(Idx) -> {[ {ddoc, Idx#idx.ddoc}, @@ -108,20 +107,18 @@ to_json(Idx) -> {def, {def_to_json(Idx#idx.def)}} ]}. - columns(Idx) -> {Props} = Idx#idx.def, {<<"fields">>, {Fields}} = lists:keyfind(<<"fields">>, 1, Props), [Key || {Key, _} <- Fields]. - is_usable(Idx, Selector, SortFields) -> % This index is usable if all of the columns are % restricted by the selector such that they are required to exist % and the selector is not a text search (so requires a text index) RequiredFields = columns(Idx), - % sort fields are required to exist in the results so + % sort fields are required to exist in the results so % we don't need to check the selector for these RequiredFields1 = ordsets:subtract(lists:usort(RequiredFields), lists:usort(SortFields)), @@ -129,31 +126,35 @@ is_usable(Idx, Selector, SortFields) -> % we don't need to check the selector for these either RequiredFields2 = ordsets:subtract( RequiredFields1, - [<<"_id">>, <<"_rev">>]), - - mango_selector:has_required_fields(Selector, RequiredFields2) - andalso not is_text_search(Selector) - andalso can_use_sort(RequiredFields, SortFields, Selector). + [<<"_id">>, <<"_rev">>] + ), + mango_selector:has_required_fields(Selector, RequiredFields2) andalso + not is_text_search(Selector) andalso + can_use_sort(RequiredFields, SortFields, Selector). is_text_search({[]}) -> false; is_text_search({[{<<"$default">>, _}]}) -> true; is_text_search({[{_Field, Cond}]}) when is_list(Cond) -> - lists:foldl(fun(C, Exists) -> - Exists orelse is_text_search(C) - end, false, Cond); + lists:foldl( + fun(C, Exists) -> + Exists orelse is_text_search(C) + end, + false, + Cond + ); is_text_search({[{_Field, Cond}]}) when is_tuple(Cond) -> is_text_search(Cond); is_text_search({[{_Field, _Cond}]}) -> false; %% we reached values, which should always be false -is_text_search(Val) - when is_number(Val); is_boolean(Val); is_binary(Val)-> +is_text_search(Val) when + is_number(Val); is_boolean(Val); is_binary(Val) +-> false. - start_key([]) -> []; start_key([{'$gt', Key, _, _} | Rest]) -> @@ -170,7 +171,6 @@ start_key([{'$eq', Key, '$eq', Key} | Rest]) -> false = mango_json:special(Key), [Key | start_key(Rest)]. - end_key([]) -> [?MAX_JSON_OBJ]; end_key([{_, _, '$lt', Key} | Rest]) -> @@ -187,14 +187,12 @@ end_key([{'$eq', Key, '$eq', Key} | Rest]) -> false = mango_json:special(Key), [Key | end_key(Rest)]. - do_validate({Props}) -> {ok, Opts} = mango_opts:validate(Props, opts()), {ok, {Opts}}; do_validate(Else) -> ?MANGO_ERROR({invalid_index_json, Else}). - def_to_json({Props}) -> def_to_json(Props); def_to_json([]) -> @@ -210,7 +208,6 @@ def_to_json([{<<"partial_filter_selector">>, {[]}} | Rest]) -> def_to_json([{Key, Value} | Rest]) -> [{Key, Value} | def_to_json(Rest)]. - opts() -> [ {<<"fields">>, [ @@ -225,16 +222,15 @@ opts() -> ]} ]. - make_view(Idx) -> - View = {[ - {<<"map">>, Idx#idx.def}, - {<<"reduce">>, <<"_count">>}, - {<<"options">>, {Idx#idx.opts}} - ]}, + View = + {[ + {<<"map">>, Idx#idx.def}, + {<<"reduce">>, <<"_count">>}, + {<<"options">>, {Idx#idx.opts}} + ]}, {Idx#idx.name, View}. - validate_ddoc(VProps) -> try Def = proplists:get_value(<<"map">>, VProps), @@ -242,13 +238,15 @@ validate_ddoc(VProps) -> {Opts0} = proplists:get_value(<<"options">>, VProps), Opts = lists:keydelete(<<"sort">>, 1, Opts0), {Def, Opts} - catch Error:Reason -> - couch_log:error("Invalid Index Def ~p. Error: ~p, Reason: ~p", - [VProps, Error, Reason]), - invalid_view + catch + Error:Reason -> + couch_log:error( + "Invalid Index Def ~p. Error: ~p, Reason: ~p", + [VProps, Error, Reason] + ), + invalid_view end. - % This function returns a list of indexes that % can be used to restrict this query. This works by % searching the selector looking for field names that @@ -268,11 +266,9 @@ validate_ddoc(VProps) -> % We can see through '$and' trivially indexable_fields({[{<<"$and">>, Args}]}) -> lists:usort(lists:flatten([indexable_fields(A) || A <- Args])); - % So far we can't see through any other operator indexable_fields({[{<<"$", _/binary>>, _}]}) -> []; - % If we have a field with a terminator that is locatable % using an index then the field is a possible index indexable_fields({[{Field, Cond}]}) -> @@ -282,12 +278,10 @@ indexable_fields({[{Field, Cond}]}) -> false -> [] end; - % An empty selector indexable_fields({[]}) -> []. - % Check if a condition is indexable. The logical % comparisons are mostly straight forward. We % currently don't understand '$in' which is @@ -304,24 +298,20 @@ indexable({[{<<"$gt">>, _}]}) -> true; indexable({[{<<"$gte">>, _}]}) -> true; - % All other operators are currently not indexable. % This is also a subtle assertion that we don't % call indexable/1 on a field name. indexable({[{<<"$", _/binary>>, _}]}) -> false. - % For each field, return {Field, Range} field_ranges(Selector) -> Fields = indexable_fields(Selector), field_ranges(Selector, Fields). - field_ranges(Selector, Fields) -> field_ranges(Selector, Fields, []). - field_ranges(_Selector, [], Acc) -> lists:reverse(Acc); field_ranges(Selector, [Field | Rest], Acc) -> @@ -332,7 +322,6 @@ field_ranges(Selector, [Field | Rest], Acc) -> field_ranges(Selector, Rest, [{Field, Range} | Acc]) end. - % Find the complete range for a given index in this % selector. This works by AND'ing logical comparisons % together so that we can define the start and end @@ -343,32 +332,31 @@ field_ranges(Selector, [Field | Rest], Acc) -> range(Selector, Index) -> range(Selector, Index, '$gt', mango_json:min(), '$lt', mango_json:max()). - % Adjust Low and High based on values found for the % givend Index in Selector. range({[{<<"$and">>, Args}]}, Index, LCmp, Low, HCmp, High) -> - lists:foldl(fun - (Arg, {LC, L, HC, H}) -> - range(Arg, Index, LC, L, HC, H); - (_Arg, empty) -> - empty - end, {LCmp, Low, HCmp, High}, Args); - + lists:foldl( + fun + (Arg, {LC, L, HC, H}) -> + range(Arg, Index, LC, L, HC, H); + (_Arg, empty) -> + empty + end, + {LCmp, Low, HCmp, High}, + Args + ); % We can currently only traverse '$and' operators range({[{<<"$", _/binary>>}]}, _Index, LCmp, Low, HCmp, High) -> {LCmp, Low, HCmp, High}; - % If the field name matches the index see if we can narrow % the acceptable range. range({[{Index, Cond}]}, Index, LCmp, Low, HCmp, High) -> range(Cond, LCmp, Low, HCmp, High); - % Else we have a field unrelated to this index so just % return the current values. range(_, _, LCmp, Low, HCmp, High) -> {LCmp, Low, HCmp, High}. - % The comments below are a bit cryptic at first but they show % where the Arg cand land in the current range. % @@ -425,7 +413,6 @@ range({[{<<"$lt">>, Arg}]}, LCmp, Low, HCmp, High) -> max -> {LCmp, Low, HCmp, High} end; - range({[{<<"$lte">>, Arg}]}, LCmp, Low, HCmp, High) -> case range_pos(Low, Arg, High) of min -> @@ -441,7 +428,6 @@ range({[{<<"$lte">>, Arg}]}, LCmp, Low, HCmp, High) -> max -> {LCmp, Low, HCmp, High} end; - range({[{<<"$eq">>, Arg}]}, LCmp, Low, HCmp, High) -> case range_pos(Low, Arg, High) of min -> @@ -459,7 +445,6 @@ range({[{<<"$eq">>, Arg}]}, LCmp, Low, HCmp, High) -> max -> empty end; - range({[{<<"$gte">>, Arg}]}, LCmp, Low, HCmp, High) -> case range_pos(Low, Arg, High) of min -> @@ -475,7 +460,6 @@ range({[{<<"$gte">>, Arg}]}, LCmp, Low, HCmp, High) -> max -> empty end; - range({[{<<"$gt">>, Arg}]}, LCmp, Low, HCmp, High) -> case range_pos(Low, Arg, High) of min -> @@ -489,14 +473,12 @@ range({[{<<"$gt">>, Arg}]}, LCmp, Low, HCmp, High) -> max -> empty end; - % There's some other un-indexable restriction on the index % that will be applied as a post-filter. Ignore it and % carry on our merry way. range({[{<<"$", _/binary>>, _}]}, LCmp, Low, HCmp, High) -> {LCmp, Low, HCmp, High}. - % Returns the value min | low | mid | high | max depending % on how Arg compares to Low and High. range_pos(Low, Arg, High) -> @@ -514,7 +496,6 @@ range_pos(Low, Arg, High) -> end end. - % Can_use_sort works as follows: % % * no sort fields then we can use this diff --git a/src/mango/src/mango_json.erl b/src/mango/src/mango_json.erl index 9584c2d7e..ca18d8898 100644 --- a/src/mango/src/mango_json.erl +++ b/src/mango/src/mango_json.erl @@ -12,7 +12,6 @@ -module(mango_json). - -export([ min/0, max/0, @@ -23,19 +22,15 @@ to_binary/1 ]). - -define(MIN_VAL, mango_json_min). -define(MAX_VAL, mango_json_max). - min() -> ?MIN_VAL. - max() -> ?MAX_VAL. - cmp(?MIN_VAL, ?MIN_VAL) -> 0; cmp(?MIN_VAL, _) -> @@ -51,7 +46,6 @@ cmp(_, ?MAX_VAL) -> cmp(A, B) -> couch_ejson_compare:less(A, B). - cmp_raw(?MIN_VAL, ?MIN_VAL) -> 0; cmp_raw(?MIN_VAL, _) -> @@ -77,7 +71,6 @@ cmp_raw(A, B) -> end end. - type(null) -> <<"null">>; type(Bool) when is_boolean(Bool) -> @@ -91,7 +84,6 @@ type({Props}) when is_list(Props) -> type(Vals) when is_list(Vals) -> <<"array">>. - special(?MIN_VAL) -> true; special(?MAX_VAL) -> @@ -99,7 +91,6 @@ special(?MAX_VAL) -> special(_) -> false. - to_binary({Props}) -> Pred = fun({Key, Value}) -> {to_binary(Key), to_binary(Value)} @@ -118,4 +109,4 @@ to_binary(Data) when is_atom(Data) -> to_binary(Data) when is_number(Data) -> Data; to_binary(Data) when is_binary(Data) -> - Data.
\ No newline at end of file + Data. diff --git a/src/mango/src/mango_json_bookmark.erl b/src/mango/src/mango_json_bookmark.erl index 83fd00f29..8446e0c8a 100644 --- a/src/mango/src/mango_json_bookmark.erl +++ b/src/mango/src/mango_json_bookmark.erl @@ -10,7 +10,6 @@ % License for the specific language governing permissions and limitations under % the License. - -module(mango_json_bookmark). -export([ @@ -18,15 +17,14 @@ create/1 ]). - -include_lib("couch_mrview/include/couch_mrview.hrl"). -include("mango_cursor.hrl"). -include("mango.hrl"). -update_args(EncodedBookmark, #mrargs{skip = Skip} = Args) -> +update_args(EncodedBookmark, #mrargs{skip = Skip} = Args) -> Bookmark = unpack(EncodedBookmark), case is_list(Bookmark) of - true -> + true -> {startkey, Startkey} = lists:keyfind(startkey, 1, Bookmark), {startkey_docid, StartkeyDocId} = lists:keyfind(startkey_docid, 1, Bookmark), Args#mrargs{ @@ -37,35 +35,36 @@ update_args(EncodedBookmark, #mrargs{skip = Skip} = Args) -> false -> Args end. - -create(#cursor{bookmark_docid = BookmarkDocId, bookmark_key = BookmarkKey}) when BookmarkKey =/= undefined -> +create(#cursor{bookmark_docid = BookmarkDocId, bookmark_key = BookmarkKey}) when + BookmarkKey =/= undefined +-> QueryArgs = [ {startkey_docid, BookmarkDocId}, {startkey, BookmarkKey} ], - Bin = term_to_binary(QueryArgs, [compressed, {minor_version,1}]), + Bin = term_to_binary(QueryArgs, [compressed, {minor_version, 1}]), couch_util:encodeBase64Url(Bin); create(#cursor{bookmark = Bookmark}) -> Bookmark. - unpack(nil) -> nil; unpack(Packed) -> try Bookmark = binary_to_term(couch_util:decodeBase64Url(Packed), [safe]), verify(Bookmark) - catch _:_ -> - ?MANGO_ERROR({invalid_bookmark, Packed}) + catch + _:_ -> + ?MANGO_ERROR({invalid_bookmark, Packed}) end. verify(Bookmark) when is_list(Bookmark) -> - case lists:keymember(startkey, 1, Bookmark) andalso lists:keymember(startkey_docid, 1, Bookmark) of + case + lists:keymember(startkey, 1, Bookmark) andalso lists:keymember(startkey_docid, 1, Bookmark) + of true -> Bookmark; _ -> throw(invalid_bookmark) end; verify(_Bookmark) -> throw(invalid_bookmark). - -
\ No newline at end of file diff --git a/src/mango/src/mango_native_proc.erl b/src/mango/src/mango_native_proc.erl index 274ae11de..48c78c4c4 100644 --- a/src/mango/src/mango_native_proc.erl +++ b/src/mango/src/mango_native_proc.erl @@ -13,10 +13,8 @@ -module(mango_native_proc). -behavior(gen_server). - -include("mango_idx.hrl"). - -export([ start_link/0, set_timeout/2, @@ -32,117 +30,94 @@ code_change/3 ]). - -record(st, { indexes = [], timeout = 5000 }). - -record(tacc, { index_array_lengths = true, fields = all_fields, path = [] }). - start_link() -> gen_server:start_link(?MODULE, [], []). - set_timeout(Pid, TimeOut) when is_integer(TimeOut), TimeOut > 0 -> gen_server:call(Pid, {set_timeout, TimeOut}). - prompt(Pid, Data) -> gen_server:call(Pid, {prompt, Data}). - init(_) -> {ok, #st{}}. - terminate(_Reason, _St) -> ok. - handle_call({set_timeout, TimeOut}, _From, St) -> - {reply, ok, St#st{timeout=TimeOut}}; - + {reply, ok, St#st{timeout = TimeOut}}; handle_call({prompt, [<<"reset">>]}, _From, St) -> - {reply, true, St#st{indexes=[]}}; - + {reply, true, St#st{indexes = []}}; handle_call({prompt, [<<"reset">>, _QueryConfig]}, _From, St) -> - {reply, true, St#st{indexes=[]}}; - + {reply, true, St#st{indexes = []}}; handle_call({prompt, [<<"add_fun">>, IndexInfo]}, _From, St) -> - Indexes = case validate_index_info(IndexInfo) of - true -> - St#st.indexes ++ [IndexInfo]; - false -> - couch_log:error("No Valid Indexes For: ~p", [IndexInfo]), - St#st.indexes - end, + Indexes = + case validate_index_info(IndexInfo) of + true -> + St#st.indexes ++ [IndexInfo]; + false -> + couch_log:error("No Valid Indexes For: ~p", [IndexInfo]), + St#st.indexes + end, NewSt = St#st{indexes = Indexes}, {reply, true, NewSt}; - handle_call({prompt, [<<"map_doc">>, Doc]}, _From, St) -> {reply, map_doc(St, mango_json:to_binary(Doc)), St}; - handle_call({prompt, [<<"reduce">>, RedSrcs, _]}, _From, St) -> {reply, [true, [null || _ <- RedSrcs]], St}; - handle_call({prompt, [<<"rereduce">>, RedSrcs, _]}, _From, St) -> {reply, [true, [null || _ <- RedSrcs]], St}; - handle_call({prompt, [<<"index_doc">>, Doc]}, _From, St) -> - Vals = case index_doc(St, mango_json:to_binary(Doc)) of - [] -> - [[]]; - Else -> - Else - end, + Vals = + case index_doc(St, mango_json:to_binary(Doc)) of + [] -> + [[]]; + Else -> + Else + end, {reply, Vals, St}; - - handle_call(Msg, _From, St) -> {stop, {invalid_call, Msg}, {invalid_call, Msg}, St}. - handle_cast(garbage_collect, St) -> erlang:garbage_collect(), {noreply, St}; - handle_cast(stop, St) -> {stop, normal, St}; - handle_cast(Msg, St) -> {stop, {invalid_cast, Msg}, St}. - handle_info(Msg, St) -> {stop, {invalid_info, Msg}, St}. - code_change(_OldVsn, St, _Extra) -> {ok, St}. - -map_doc(#st{indexes=Indexes}, Doc) -> +map_doc(#st{indexes = Indexes}, Doc) -> lists:map(fun(Idx) -> get_index_entries(Idx, Doc) end, Indexes). - -index_doc(#st{indexes=Indexes}, Doc) -> +index_doc(#st{indexes = Indexes}, Doc) -> lists:map(fun(Idx) -> get_text_entries(Idx, Doc) end, Indexes). - get_index_entries({IdxProps}, Doc) -> {Fields} = couch_util:get_value(<<"fields">>, IdxProps), Selector = get_index_partial_filter_selector(IdxProps), case should_index(Selector, Doc) of - false -> + false -> []; - true -> + true -> Values = get_index_values(Fields, Doc), case lists:member(not_found, Values) of true -> []; @@ -150,16 +125,17 @@ get_index_entries({IdxProps}, Doc) -> end end. - get_index_values(Fields, Doc) -> - lists:map(fun({Field, _Dir}) -> - case mango_doc:get_field(Doc, Field) of - not_found -> not_found; - bad_path -> not_found; - Value -> Value - end - end, Fields). - + lists:map( + fun({Field, _Dir}) -> + case mango_doc:get_field(Doc, Field) of + not_found -> not_found; + bad_path -> not_found; + Value -> Value + end + end, + Fields + ). get_text_entries({IdxProps}, Doc) -> Selector = get_index_partial_filter_selector(IdxProps), @@ -170,7 +146,6 @@ get_text_entries({IdxProps}, Doc) -> [] end. - get_index_partial_filter_selector(IdxProps) -> case couch_util:get_value(<<"partial_filter_selector">>, IdxProps, {[]}) of {[]} -> @@ -181,7 +156,6 @@ get_index_partial_filter_selector(IdxProps) -> Else end. - get_text_entries0(IdxProps, Doc) -> DefaultEnabled = get_default_enabled(IdxProps), IndexArrayLengths = get_index_array_lengths(IdxProps), @@ -191,22 +165,22 @@ get_text_entries0(IdxProps, Doc) -> fields = FieldsList }, Fields0 = get_text_field_values(Doc, TAcc), - Fields = if not DefaultEnabled -> Fields0; true -> - add_default_text_field(Fields0) - end, + Fields = + if + not DefaultEnabled -> Fields0; + true -> add_default_text_field(Fields0) + end, FieldNames = get_field_names(Fields), Converted = convert_text_fields(Fields), FieldNames ++ Converted. - get_text_field_values({Props}, TAcc) when is_list(Props) -> get_text_field_values_obj(Props, TAcc, []); - get_text_field_values(Values, TAcc) when is_list(Values) -> IndexArrayLengths = TAcc#tacc.index_array_lengths, NewPath = ["[]" | TAcc#tacc.path], NewTAcc = TAcc#tacc{path = NewPath}, - case IndexArrayLengths of + case IndexArrayLengths of true -> % We bypass make_text_field and directly call make_text_field_name % because the length field name is not part of the path. @@ -216,20 +190,15 @@ get_text_field_values(Values, TAcc) when is_list(Values) -> _ -> get_text_field_values_arr(Values, NewTAcc, []) end; - get_text_field_values(Bin, TAcc) when is_binary(Bin) -> make_text_field(TAcc, <<"string">>, Bin); - get_text_field_values(Num, TAcc) when is_number(Num) -> make_text_field(TAcc, <<"number">>, Num); - get_text_field_values(Bool, TAcc) when is_boolean(Bool) -> make_text_field(TAcc, <<"boolean">>, Bool); - get_text_field_values(null, TAcc) -> make_text_field(TAcc, <<"null">>, true). - get_text_field_values_obj([], _, FAcc) -> FAcc; get_text_field_values_obj([{Key, Val} | Rest], TAcc, FAcc) -> @@ -238,34 +207,29 @@ get_text_field_values_obj([{Key, Val} | Rest], TAcc, FAcc) -> Fields = get_text_field_values(Val, NewTAcc), get_text_field_values_obj(Rest, TAcc, Fields ++ FAcc). - get_text_field_values_arr([], _, FAcc) -> FAcc; get_text_field_values_arr([Value | Rest], TAcc, FAcc) -> Fields = get_text_field_values(Value, TAcc), get_text_field_values_arr(Rest, TAcc, Fields ++ FAcc). - get_default_enabled(Props) -> case couch_util:get_value(<<"default_field">>, Props, {[]}) of Bool when is_boolean(Bool) -> Bool; {[]} -> true; - {Opts}-> + {Opts} -> couch_util:get_value(<<"enabled">>, Opts, true) end. - get_index_array_lengths(Props) -> couch_util:get_value(<<"index_array_lengths">>, Props, true). - add_default_text_field(Fields) -> DefaultFields = add_default_text_field(Fields, []), DefaultFields ++ Fields. - add_default_text_field([], Acc) -> Acc; add_default_text_field([{_Name, <<"string">>, Value} | Rest], Acc) -> @@ -274,32 +238,33 @@ add_default_text_field([{_Name, <<"string">>, Value} | Rest], Acc) -> add_default_text_field([_ | Rest], Acc) -> add_default_text_field(Rest, Acc). - %% index of all field names get_field_names(Fields) -> - FieldNameSet = lists:foldl(fun({Name, _, _}, Set) -> - gb_sets:add([<<"$fieldnames">>, Name, []], Set) - end, gb_sets:new(), Fields), + FieldNameSet = lists:foldl( + fun({Name, _, _}, Set) -> + gb_sets:add([<<"$fieldnames">>, Name, []], Set) + end, + gb_sets:new(), + Fields + ), gb_sets:to_list(FieldNameSet). - convert_text_fields([]) -> []; convert_text_fields([{Name, _Type, Value} | Rest]) -> [[Name, Value, []] | convert_text_fields(Rest)]. - should_index(Selector, Doc) -> % We should do this NormSelector = mango_selector:normalize(Selector), Matches = mango_selector:match(NormSelector, Doc), - IsDesign = case mango_doc:get_field(Doc, <<"_id">>) of - <<"_design/", _/binary>> -> true; - _ -> false - end, + IsDesign = + case mango_doc:get_field(Doc, <<"_id">>) of + <<"_design/", _/binary>> -> true; + _ -> false + end, Matches and not IsDesign. - get_text_field_list(IdxProps) -> case couch_util:get_value(<<"fields">>, IdxProps) of Fields when is_list(Fields) -> @@ -309,16 +274,17 @@ get_text_field_list(IdxProps) -> all_fields end. - get_text_field_info({Props}) -> Name = couch_util:get_value(<<"name">>, Props), Type0 = couch_util:get_value(<<"type">>, Props), - if not is_binary(Name) -> []; true -> - Type = get_text_field_type(Type0), - [iolist_to_binary([Name, ":", Type])] + if + not is_binary(Name) -> + []; + true -> + Type = get_text_field_type(Type0), + [iolist_to_binary([Name, ":", Type])] end. - get_text_field_type(<<"number">>) -> <<"number">>; get_text_field_type(<<"boolean">>) -> @@ -326,7 +292,6 @@ get_text_field_type(<<"boolean">>) -> get_text_field_type(_) -> <<"string">>. - make_text_field(TAcc, Type, Value) -> FieldName = make_text_field_name(TAcc#tacc.path, Type), Fields = TAcc#tacc.fields, @@ -337,31 +302,34 @@ make_text_field(TAcc, Type, Value) -> [] end. - make_text_field_name([P | Rest], Type) -> Parts = lists:reverse(Rest, [iolist_to_binary([P, ":", Type])]), Escaped = [mango_util:lucene_escape_field(N) || N <- Parts], iolist_to_binary(mango_util:join(".", Escaped)). - validate_index_info(IndexInfo) -> - IdxTypes = case clouseau_rpc:connected() of - true -> - [mango_idx_view, mango_idx_text]; - false -> - [mango_idx_view] - end, - Results = lists:foldl(fun(IdxType, Results0) -> - try - IdxType:validate_index_def(IndexInfo), - [valid_index | Results0] - catch _:_ -> - [invalid_index | Results0] - end - end, [], IdxTypes), + IdxTypes = + case clouseau_rpc:connected() of + true -> + [mango_idx_view, mango_idx_text]; + false -> + [mango_idx_view] + end, + Results = lists:foldl( + fun(IdxType, Results0) -> + try + IdxType:validate_index_def(IndexInfo), + [valid_index | Results0] + catch + _:_ -> + [invalid_index | Results0] + end + end, + [], + IdxTypes + ), lists:member(valid_index, Results). - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). diff --git a/src/mango/src/mango_opts.erl b/src/mango/src/mango_opts.erl index 92c07f743..04fe5bbf8 100644 --- a/src/mango/src/mango_opts.erl +++ b/src/mango/src/mango_opts.erl @@ -39,10 +39,8 @@ default_limit/0 ]). - -include("mango.hrl"). - validate_idx_create({Props}) -> Opts = [ {<<"index">>, [ @@ -81,7 +79,6 @@ validate_idx_create({Props}) -> ], validate(Props, Opts). - validate_find({Props}) -> Opts = [ {<<"selector">>, [ @@ -169,7 +166,6 @@ validate_find({Props}) -> ], validate(Props, Opts). - validate_bulk_delete({Props}) -> Opts = [ {<<"docids">>, [ @@ -185,7 +181,6 @@ validate_bulk_delete({Props}) -> ], validate(Props, Opts). - validate(Props, Opts) -> case mango_util:assert_ejson({Props}) of true -> @@ -202,13 +197,11 @@ validate(Props, Opts) -> end, {ok, Acc}. - is_string(Val) when is_binary(Val) -> {ok, Val}; is_string(Else) -> ?MANGO_ERROR({invalid_string, Else}). - is_boolean(true) -> {ok, true}; is_boolean(false) -> @@ -216,19 +209,16 @@ is_boolean(false) -> is_boolean(Else) -> ?MANGO_ERROR({invalid_boolean, Else}). - is_pos_integer(V) when is_integer(V), V > 0 -> {ok, V}; is_pos_integer(Else) -> ?MANGO_ERROR({invalid_pos_integer, Else}). - is_non_neg_integer(V) when is_integer(V), V >= 0 -> {ok, V}; is_non_neg_integer(Else) -> ?MANGO_ERROR({invalid_non_neg_integer, Else}). - is_object({Props}) -> true = mango_util:assert_ejson({Props}), {ok, {Props}}; @@ -236,27 +226,26 @@ is_object(Else) -> ?MANGO_ERROR({invalid_object, Else}). is_ok_or_false(<<"ok">>) -> - {ok, ok}; -is_ok_or_false(<<"false">>) -> % convenience - {ok, false}; + {ok, ok}; +% convenience +is_ok_or_false(<<"false">>) -> + {ok, false}; is_ok_or_false(false) -> - {ok, false}; + {ok, false}; is_ok_or_false(Else) -> - ?MANGO_ERROR({invalid_ok_or_false_value, Else}). + ?MANGO_ERROR({invalid_ok_or_false_value, Else}). validate_idx_name(auto_name) -> {ok, auto_name}; validate_idx_name(Else) -> is_string(Else). - validate_selector({Props}) -> Norm = mango_selector:normalize({Props}), {ok, Norm}; validate_selector(Else) -> ?MANGO_ERROR({invalid_selector_json, Else}). - %% We re-use validate_use_index to make sure the index names are valid validate_bulk_docs(Docs) when is_list(Docs) -> lists:foreach(fun validate_use_index/1, Docs), @@ -264,7 +253,6 @@ validate_bulk_docs(Docs) when is_list(Docs) -> validate_bulk_docs(Else) -> ?MANGO_ERROR({invalid_bulk_docs, Else}). - validate_use_index(IndexName) when is_binary(IndexName) -> case binary:split(IndexName, <<"/">>) of [DesignId] -> @@ -284,13 +272,13 @@ validate_use_index([]) -> {ok, []}; validate_use_index([DesignId]) when is_binary(DesignId) -> {ok, [DesignId]}; -validate_use_index([DesignId, ViewName]) - when is_binary(DesignId), is_binary(ViewName) -> +validate_use_index([DesignId, ViewName]) when + is_binary(DesignId), is_binary(ViewName) +-> {ok, [DesignId, ViewName]}; validate_use_index(Else) -> ?MANGO_ERROR({invalid_index_name, Else}). - validate_bookmark(null) -> {ok, nil}; validate_bookmark(<<>>) -> @@ -300,15 +288,12 @@ validate_bookmark(Bin) when is_binary(Bin) -> validate_bookmark(Else) -> ?MANGO_ERROR({invalid_bookmark, Else}). - validate_sort(Value) -> mango_sort:new(Value). - validate_fields(Value) -> mango_fields:new(Value). - validate_partitioned(true) -> {ok, true}; validate_partitioned(false) -> @@ -318,14 +303,12 @@ validate_partitioned(db_default) -> validate_partitioned(Else) -> ?MANGO_ERROR({invalid_partitioned_value, Else}). - validate_partition(<<>>) -> {ok, <<>>}; validate_partition(Partition) -> couch_partition:validate_partition(Partition), {ok, Partition}. - validate_opts([], Props, Acc) -> {Props, lists:reverse(Acc)}; validate_opts([{Name, Desc} | Rest], Props, Acc) -> @@ -339,7 +322,6 @@ validate_opts([{Name, Desc} | Rest], Props, Acc) -> validate_opts(Rest, Props, NewAcc) end. - validate_opt(_Name, [], Value) -> Value; validate_opt(Name, Desc0, undefined) -> @@ -372,6 +354,5 @@ validate_opt(Name, [{validator, Fun} | Rest], Value) -> ?MANGO_ERROR({invalid_value, Name, Value}) end. - default_limit() -> config:get_integer("mango", "default_limit", 25). diff --git a/src/mango/src/mango_selector.erl b/src/mango/src/mango_selector.erl index fc6a6d1a7..be2616ff5 100644 --- a/src/mango/src/mango_selector.erl +++ b/src/mango/src/mango_selector.erl @@ -12,7 +12,6 @@ -module(mango_selector). - -export([ normalize/1, match/2, @@ -20,11 +19,9 @@ is_constant_field/2 ]). - -include_lib("couch/include/couch_db.hrl"). -include("mango.hrl"). - % Validate and normalize each operator. This translates % every selector operator into a consistent version that % we can then rely on for all other selector functions. @@ -48,7 +45,6 @@ normalize(Selector) -> end, {NProps}. - % Match a selector against a #doc{} or EJSON value. % This assumes that the Selector has been normalized. % Returns true or false. @@ -56,14 +52,11 @@ match(Selector, D) -> couch_stats:increment_counter([mango, evaluate_selector]), match_int(Selector, D). - % An empty selector matches any value. match_int({[]}, _) -> true; - -match_int(Selector, #doc{body=Body}) -> +match_int(Selector, #doc{body = Body}) -> match(Selector, Body, fun mango_json:cmp/2); - match_int(Selector, {Props}) -> match(Selector, {Props}, fun mango_json:cmp/2). @@ -74,47 +67,38 @@ norm_ops({[{<<"$and">>, Args}]}) when is_list(Args) -> {[{<<"$and">>, [norm_ops(A) || A <- Args]}]}; norm_ops({[{<<"$and">>, Arg}]}) -> ?MANGO_ERROR({bad_arg, '$and', Arg}); - norm_ops({[{<<"$or">>, Args}]}) when is_list(Args) -> {[{<<"$or">>, [norm_ops(A) || A <- Args]}]}; norm_ops({[{<<"$or">>, Arg}]}) -> ?MANGO_ERROR({bad_arg, '$or', Arg}); - -norm_ops({[{<<"$not">>, {_}=Arg}]}) -> +norm_ops({[{<<"$not">>, {_} = Arg}]}) -> {[{<<"$not">>, norm_ops(Arg)}]}; norm_ops({[{<<"$not">>, Arg}]}) -> ?MANGO_ERROR({bad_arg, '$not', Arg}); - norm_ops({[{<<"$nor">>, Args}]}) when is_list(Args) -> {[{<<"$nor">>, [norm_ops(A) || A <- Args]}]}; norm_ops({[{<<"$nor">>, Arg}]}) -> ?MANGO_ERROR({bad_arg, '$nor', Arg}); - norm_ops({[{<<"$in">>, Args}]} = Cond) when is_list(Args) -> Cond; norm_ops({[{<<"$in">>, Arg}]}) -> ?MANGO_ERROR({bad_arg, '$in', Arg}); - norm_ops({[{<<"$nin">>, Args}]} = Cond) when is_list(Args) -> Cond; norm_ops({[{<<"$nin">>, Arg}]}) -> ?MANGO_ERROR({bad_arg, '$nin', Arg}); - norm_ops({[{<<"$exists">>, Arg}]} = Cond) when is_boolean(Arg) -> Cond; norm_ops({[{<<"$exists">>, Arg}]}) -> ?MANGO_ERROR({bad_arg, '$exists', Arg}); - norm_ops({[{<<"$type">>, Arg}]} = Cond) when is_binary(Arg) -> Cond; norm_ops({[{<<"$type">>, Arg}]}) -> ?MANGO_ERROR({bad_arg, '$type', Arg}); - norm_ops({[{<<"$mod">>, [D, R]}]} = Cond) when is_integer(D), is_integer(R) -> Cond; norm_ops({[{<<"$mod">>, Arg}]}) -> ?MANGO_ERROR({bad_arg, '$mod', Arg}); - norm_ops({[{<<"$regex">>, Regex}]} = Cond) when is_binary(Regex) -> case re:compile(Regex) of {ok, _} -> @@ -122,45 +106,40 @@ norm_ops({[{<<"$regex">>, Regex}]} = Cond) when is_binary(Regex) -> _ -> ?MANGO_ERROR({bad_arg, '$regex', Regex}) end; - norm_ops({[{<<"$all">>, Args}]}) when is_list(Args) -> {[{<<"$all">>, Args}]}; norm_ops({[{<<"$all">>, Arg}]}) -> ?MANGO_ERROR({bad_arg, '$all', Arg}); - -norm_ops({[{<<"$elemMatch">>, {_}=Arg}]}) -> +norm_ops({[{<<"$elemMatch">>, {_} = Arg}]}) -> {[{<<"$elemMatch">>, norm_ops(Arg)}]}; norm_ops({[{<<"$elemMatch">>, Arg}]}) -> ?MANGO_ERROR({bad_arg, '$elemMatch', Arg}); - -norm_ops({[{<<"$allMatch">>, {_}=Arg}]}) -> +norm_ops({[{<<"$allMatch">>, {_} = Arg}]}) -> {[{<<"$allMatch">>, norm_ops(Arg)}]}; norm_ops({[{<<"$allMatch">>, Arg}]}) -> ?MANGO_ERROR({bad_arg, '$allMatch', Arg}); - -norm_ops({[{<<"$keyMapMatch">>, {_}=Arg}]}) -> +norm_ops({[{<<"$keyMapMatch">>, {_} = Arg}]}) -> {[{<<"$keyMapMatch">>, norm_ops(Arg)}]}; norm_ops({[{<<"$keyMapMatch">>, Arg}]}) -> ?MANGO_ERROR({bad_arg, '$keyMapMatch', Arg}); - norm_ops({[{<<"$size">>, Arg}]}) when is_integer(Arg), Arg >= 0 -> {[{<<"$size">>, Arg}]}; norm_ops({[{<<"$size">>, Arg}]}) -> ?MANGO_ERROR({bad_arg, '$size', Arg}); - -norm_ops({[{<<"$text">>, Arg}]}) when is_binary(Arg); is_number(Arg); - is_boolean(Arg) -> +norm_ops({[{<<"$text">>, Arg}]}) when + is_binary(Arg); + is_number(Arg); + is_boolean(Arg) +-> {[{<<"$default">>, {[{<<"$text">>, Arg}]}}]}; norm_ops({[{<<"$text">>, Arg}]}) -> ?MANGO_ERROR({bad_arg, '$text', Arg}); - % Not technically an operator but we pass it through here % so that this function accepts its own output. This exists % so that $text can have a field name value which simplifies % logic elsewhere. norm_ops({[{<<"$default">>, _}]} = Selector) -> Selector; - % Terminals where we can't perform any validation % on the value because any value is acceptable. norm_ops({[{<<"$lt">>, _}]} = Cond) -> @@ -175,7 +154,6 @@ norm_ops({[{<<"$gte">>, _}]} = Cond) -> Cond; norm_ops({[{<<"$gt">>, _}]} = Cond) -> Cond; - % Known but unsupported operators norm_ops({[{<<"$where">>, _}]}) -> ?MANGO_ERROR({not_supported, '$where'}); @@ -187,24 +165,19 @@ norm_ops({[{<<"$near">>, _}]}) -> ?MANGO_ERROR({not_supported, '$near'}); norm_ops({[{<<"$nearSphere">>, _}]}) -> ?MANGO_ERROR({not_supported, '$nearSphere'}); - % Unknown operator -norm_ops({[{<<"$", _/binary>>=Op, _}]}) -> +norm_ops({[{<<"$", _/binary>> = Op, _}]}) -> ?MANGO_ERROR({invalid_operator, Op}); - % A {Field: Cond} pair norm_ops({[{Field, Cond}]}) -> {[{Field, norm_ops(Cond)}]}; - % An implicit $and norm_ops({[_, _ | _] = Props}) -> {[{<<"$and">>, [norm_ops({[P]}) || P <- Props]}]}; - % A bare value condition means equality norm_ops(Value) -> {[{<<"$eq">>, Value}]}. - % This takes a selector and normalizes all of the % field names as far as possible. For instance: % @@ -233,52 +206,40 @@ norm_fields({[]}) -> norm_fields(Selector) -> norm_fields(Selector, <<>>). - % Operators where we can push the field names further % down the operator tree norm_fields({[{<<"$and">>, Args}]}, Path) -> {[{<<"$and">>, [norm_fields(A, Path) || A <- Args]}]}; - norm_fields({[{<<"$or">>, Args}]}, Path) -> {[{<<"$or">>, [norm_fields(A, Path) || A <- Args]}]}; - norm_fields({[{<<"$not">>, Arg}]}, Path) -> {[{<<"$not">>, norm_fields(Arg, Path)}]}; - norm_fields({[{<<"$nor">>, Args}]}, Path) -> {[{<<"$nor">>, [norm_fields(A, Path) || A <- Args]}]}; - % Fields where we can normalize fields in the % operator arguments independently. norm_fields({[{<<"$elemMatch">>, Arg}]}, Path) -> Cond = {[{<<"$elemMatch">>, norm_fields(Arg)}]}, {[{Path, Cond}]}; - norm_fields({[{<<"$allMatch">>, Arg}]}, Path) -> Cond = {[{<<"$allMatch">>, norm_fields(Arg)}]}, {[{Path, Cond}]}; - norm_fields({[{<<"$keyMapMatch">>, Arg}]}, Path) -> Cond = {[{<<"$keyMapMatch">>, norm_fields(Arg)}]}, {[{Path, Cond}]}; - - % The text operator operates against the internal % $default field. This also asserts that the $default % field is at the root as well as that it only has % a $text operator applied. -norm_fields({[{<<"$default">>, {[{<<"$text">>, _Arg}]}}]}=Sel, <<>>) -> +norm_fields({[{<<"$default">>, {[{<<"$text">>, _Arg}]}}]} = Sel, <<>>) -> Sel; norm_fields({[{<<"$default">>, _}]} = Selector, _) -> ?MANGO_ERROR({bad_field, Selector}); - - % Any other operator is a terminal below which no % field names should exist. Set the path to this % terminal and return it. norm_fields({[{<<"$", _/binary>>, _}]} = Cond, Path) -> {[{Path, Cond}]}; - % We've found a field name. Append it to the path % and skip this node as we unroll the stack as % the full path will be further down the branch. @@ -288,16 +249,13 @@ norm_fields({[{Field, Cond}]}, <<>>) -> norm_fields(Cond, Field); norm_fields({[{Field, Cond}]}, Path) -> norm_fields(Cond, <<Path/binary, ".", Field/binary>>); - % An empty selector norm_fields({[]}, Path) -> {Path, {[]}}; - % Else we have an invalid selector norm_fields(BadSelector, _) -> ?MANGO_ERROR({bad_field, BadSelector}). - % Take all the negation operators and move the logic % as far down the branch as possible. This does things % like: @@ -325,33 +283,25 @@ norm_fields(BadSelector, _) -> % Operators that cause a negation norm_negations({[{<<"$not">>, Arg}]}) -> negate(Arg); - norm_negations({[{<<"$nor">>, Args}]}) -> {[{<<"$and">>, [negate(A) || A <- Args]}]}; - % Operators that we merely seek through as we look for % negations. norm_negations({[{<<"$and">>, Args}]}) -> {[{<<"$and">>, [norm_negations(A) || A <- Args]}]}; - norm_negations({[{<<"$or">>, Args}]}) -> {[{<<"$or">>, [norm_negations(A) || A <- Args]}]}; - norm_negations({[{<<"$elemMatch">>, Arg}]}) -> {[{<<"$elemMatch">>, norm_negations(Arg)}]}; - norm_negations({[{<<"$allMatch">>, Arg}]}) -> {[{<<"$allMatch">>, norm_negations(Arg)}]}; - norm_negations({[{<<"$keyMapMatch">>, Arg}]}) -> {[{<<"$keyMapMatch">>, norm_negations(Arg)}]}; - % All other conditions can't introduce negations anywhere % further down the operator tree. norm_negations(Cond) -> Cond. - % Actually negate an expression. Make sure and read up % on DeMorgan's laws if you're trying to read this, but % in a nutshell: @@ -368,20 +318,15 @@ norm_negations(Cond) -> % norm_negations/1 negate({[{<<"$not">>, Arg}]}) -> norm_negations(Arg); - negate({[{<<"$nor">>, Args}]}) -> {[{<<"$or">>, [norm_negations(A) || A <- Args]}]}; - % DeMorgan Negations negate({[{<<"$and">>, Args}]}) -> {[{<<"$or">>, [negate(A) || A <- Args]}]}; - negate({[{<<"$or">>, Args}]}) -> {[{<<"$and">>, [negate(A) || A <- Args]}]}; - negate({[{<<"$default">>, _}]} = Arg) -> ?MANGO_ERROR({bad_arg, '$not', Arg}); - % Negating comparison operators is straight forward negate({[{<<"$lt">>, Arg}]}) -> {[{<<"$gte">>, Arg}]}; @@ -399,22 +344,18 @@ negate({[{<<"$in">>, Args}]}) -> {[{<<"$nin">>, Args}]}; negate({[{<<"$nin">>, Args}]}) -> {[{<<"$in">>, Args}]}; - % We can also trivially negate the exists operator negate({[{<<"$exists">>, Arg}]}) -> {[{<<"$exists">>, not Arg}]}; - % Anything else we have to just terminate the % negation by reinserting the negation operator negate({[{<<"$", _/binary>>, _}]} = Cond) -> {[{<<"$not">>, Cond}]}; - % Finally, negating a field just means we negate its % condition. negate({[{Field, Cond}]}) -> {[{Field, negate(Cond)}]}. - % We need to treat an empty array as always true. This will be applied % for $or, $in, $all, $nin as well. match({[{<<"$and">>, []}]}, _, _) -> @@ -422,16 +363,13 @@ match({[{<<"$and">>, []}]}, _, _) -> match({[{<<"$and">>, Args}]}, Value, Cmp) -> Pred = fun(SubSel) -> match(SubSel, Value, Cmp) end, lists:all(Pred, Args); - match({[{<<"$or">>, []}]}, _, _) -> true; match({[{<<"$or">>, Args}]}, Value, Cmp) -> Pred = fun(SubSel) -> match(SubSel, Value, Cmp) end, lists:any(Pred, Args); - match({[{<<"$not">>, Arg}]}, Value, Cmp) -> not match(Arg, Value, Cmp); - match({[{<<"$all">>, []}]}, _, _) -> false; % All of the values in Args must exist in Values or @@ -440,16 +378,16 @@ match({[{<<"$all">>, []}]}, _, _) -> match({[{<<"$all">>, Args}]}, Values, _Cmp) when is_list(Values) -> Pred = fun(A) -> lists:member(A, Values) end, HasArgs = lists:all(Pred, Args), - IsArgs = case Args of - [A] when is_list(A) -> - A == Values; - _ -> - false - end, + IsArgs = + case Args of + [A] when is_list(A) -> + A == Values; + _ -> + false + end, HasArgs orelse IsArgs; match({[{<<"$all">>, _Args}]}, _Values, _Cmp) -> false; - %% This is for $elemMatch, $allMatch, and possibly $in because of our normalizer. %% A selector such as {"field_name": {"$elemMatch": {"$gte": 80, "$lt": 85}}} %% gets normalized to: @@ -464,17 +402,19 @@ match({[{<<"$all">>, _Args}]}, _Values, _Cmp) -> %% So we filter out the <<>>. match({[{<<>>, Arg}]}, Values, Cmp) -> match(Arg, Values, Cmp); - % Matches when any element in values matches the % sub-selector Arg. match({[{<<"$elemMatch">>, Arg}]}, Values, Cmp) when is_list(Values) -> try - lists:foreach(fun(V) -> - case match(Arg, V, Cmp) of - true -> throw(matched); - _ -> ok - end - end, Values), + lists:foreach( + fun(V) -> + case match(Arg, V, Cmp) of + true -> throw(matched); + _ -> ok + end + end, + Values + ), false catch throw:matched -> @@ -484,17 +424,19 @@ match({[{<<"$elemMatch">>, Arg}]}, Values, Cmp) when is_list(Values) -> end; match({[{<<"$elemMatch">>, _Arg}]}, _Value, _Cmp) -> false; - % Matches when all elements in values match the % sub-selector Arg. match({[{<<"$allMatch">>, Arg}]}, [_ | _] = Values, Cmp) -> try - lists:foreach(fun(V) -> - case match(Arg, V, Cmp) of - false -> throw(unmatched); - _ -> ok - end - end, Values), + lists:foreach( + fun(V) -> + case match(Arg, V, Cmp) of + false -> throw(unmatched); + _ -> ok + end + end, + Values + ), true catch _:_ -> @@ -502,17 +444,19 @@ match({[{<<"$allMatch">>, Arg}]}, [_ | _] = Values, Cmp) -> end; match({[{<<"$allMatch">>, _Arg}]}, _Value, _Cmp) -> false; - % Matches when any key in the map value matches the % sub-selector Arg. match({[{<<"$keyMapMatch">>, Arg}]}, Value, Cmp) when is_tuple(Value) -> try - lists:foreach(fun(V) -> - case match(Arg, V, Cmp) of - true -> throw(matched); - _ -> ok - end - end, [Key || {Key, _} <- element(1, Value)]), + lists:foreach( + fun(V) -> + case match(Arg, V, Cmp) of + true -> throw(matched); + _ -> ok + end + end, + [Key || {Key, _} <- element(1, Value)] + ), false catch throw:matched -> @@ -522,7 +466,6 @@ match({[{<<"$keyMapMatch">>, Arg}]}, Value, Cmp) when is_tuple(Value) -> end; match({[{<<"$keyMapMatch">>, _Arg}]}, _Value, _Cmp) -> false; - % Our comparison operators are fairly straight forward match({[{<<"$lt">>, Arg}]}, Value, Cmp) -> Cmp(Value, Arg) < 0; @@ -536,67 +479,62 @@ match({[{<<"$gte">>, Arg}]}, Value, Cmp) -> Cmp(Value, Arg) >= 0; match({[{<<"$gt">>, Arg}]}, Value, Cmp) -> Cmp(Value, Arg) > 0; - match({[{<<"$in">>, []}]}, _, _) -> false; -match({[{<<"$in">>, Args}]}, Values, Cmp) when is_list(Values)-> +match({[{<<"$in">>, Args}]}, Values, Cmp) when is_list(Values) -> Pred = fun(Arg) -> - lists:foldl(fun(Value,Match) -> - (Cmp(Value, Arg) == 0) or Match - end, false, Values) + lists:foldl( + fun(Value, Match) -> + (Cmp(Value, Arg) == 0) or Match + end, + false, + Values + ) end, lists:any(Pred, Args); match({[{<<"$in">>, Args}]}, Value, Cmp) -> Pred = fun(Arg) -> Cmp(Value, Arg) == 0 end, lists:any(Pred, Args); - match({[{<<"$nin">>, []}]}, _, _) -> true; -match({[{<<"$nin">>, Args}]}, Values, Cmp) when is_list(Values)-> +match({[{<<"$nin">>, Args}]}, Values, Cmp) when is_list(Values) -> not match({[{<<"$in">>, Args}]}, Values, Cmp); match({[{<<"$nin">>, Args}]}, Value, Cmp) -> Pred = fun(Arg) -> Cmp(Value, Arg) /= 0 end, lists:all(Pred, Args); - % This logic is a bit subtle. Basically, if value is % not undefined, then it exists. match({[{<<"$exists">>, ShouldExist}]}, Value, _Cmp) -> Exists = Value /= undefined, ShouldExist andalso Exists; - match({[{<<"$type">>, Arg}]}, Value, _Cmp) when is_binary(Arg) -> Arg == mango_json:type(Value); - match({[{<<"$mod">>, [D, R]}]}, Value, _Cmp) when is_integer(Value) -> Value rem D == R; match({[{<<"$mod">>, _}]}, _Value, _Cmp) -> false; - match({[{<<"$regex">>, Regex}]}, Value, _Cmp) when is_binary(Value) -> try match == re:run(Value, Regex, [{capture, none}]) - catch _:_ -> - false + catch + _:_ -> + false end; match({[{<<"$regex">>, _}]}, _Value, _Cmp) -> false; - match({[{<<"$size">>, Arg}]}, Values, _Cmp) when is_list(Values) -> length(Values) == Arg; match({[{<<"$size">>, _}]}, _Value, _Cmp) -> false; - % We don't have any choice but to believe that the text % index returned valid matches match({[{<<"$default">>, _}]}, _Value, _Cmp) -> true; - % All other operators are internal assertion errors for % matching because we either should've removed them during % normalization or something else broke. -match({[{<<"$", _/binary>>=Op, _}]}, _, _) -> +match({[{<<"$", _/binary>> = Op, _}]}, _, _) -> ?MANGO_ERROR({invalid_operator, Op}); - % We need to traverse value to find field. The call to % mango_doc:get_field/2 may return either not_found or % bad_path in which case matching fails. @@ -613,11 +551,9 @@ match({[{Field, Cond}]}, Value, Cmp) -> SubValue -> match(Cond, SubValue, Cmp) end; - match({[_, _ | _] = _Props} = Sel, _Value, _Cmp) -> erlang:error({unnormalized_selector, Sel}). - % Returns true if Selector requires all % fields in RequiredFields to exist in any matching documents. @@ -634,48 +570,48 @@ has_required_fields(Selector, RequiredFields) -> % Empty selector has_required_fields_int({[]}, Remainder) -> Remainder; - % No more required fields has_required_fields_int(_, []) -> []; - % No more selector has_required_fields_int([], Remainder) -> Remainder; - has_required_fields_int(Selector, RequiredFields) when not is_list(Selector) -> has_required_fields_int([Selector], RequiredFields); - % We can "see" through $and operator. Iterate % through the list of child operators. -has_required_fields_int([{[{<<"$and">>, Args}]}], RequiredFields) - when is_list(Args) -> +has_required_fields_int([{[{<<"$and">>, Args}]}], RequiredFields) when + is_list(Args) +-> has_required_fields_int(Args, RequiredFields); - % We can "see" through $or operator. Required fields % must be covered by all children. -has_required_fields_int([{[{<<"$or">>, Args}]} | Rest], RequiredFields) - when is_list(Args) -> - Remainder0 = lists:foldl(fun(Arg, Acc) -> - % for each child test coverage against the full - % set of required fields - Remainder = has_required_fields_int(Arg, RequiredFields), - - % collect the remaining fields across all children - Acc ++ Remainder - end, [], Args), +has_required_fields_int([{[{<<"$or">>, Args}]} | Rest], RequiredFields) when + is_list(Args) +-> + Remainder0 = lists:foldl( + fun(Arg, Acc) -> + % for each child test coverage against the full + % set of required fields + Remainder = has_required_fields_int(Arg, RequiredFields), + + % collect the remaining fields across all children + Acc ++ Remainder + end, + [], + Args + ), % remove duplicate fields Remainder1 = lists:usort(Remainder0), has_required_fields_int(Rest, Remainder1); - % Handle $and operator where it has peers. Required fields % can be covered by any child. -has_required_fields_int([{[{<<"$and">>, Args}]} | Rest], RequiredFields) - when is_list(Args) -> +has_required_fields_int([{[{<<"$and">>, Args}]} | Rest], RequiredFields) when + is_list(Args) +-> Remainder = has_required_fields_int(Args, RequiredFields), has_required_fields_int(Rest, Remainder); - has_required_fields_int([{[{Field, Cond}]} | Rest], RequiredFields) -> case Cond of % $exists:false is a special case - this is the only operator @@ -686,30 +622,22 @@ has_required_fields_int([{[{Field, Cond}]} | Rest], RequiredFields) -> has_required_fields_int(Rest, lists:delete(Field, RequiredFields)) end. - % Returns true if a field in the selector is a constant value e.g. {a: {$eq: 1}} is_constant_field({[]}, _Field) -> false; - is_constant_field(Selector, Field) when not is_list(Selector) -> is_constant_field([Selector], Field); - is_constant_field([], _Field) -> false; - is_constant_field([{[{<<"$and">>, Args}]}], Field) when is_list(Args) -> lists:any(fun(Arg) -> is_constant_field(Arg, Field) end, Args); - is_constant_field([{[{<<"$and">>, Args}]}], Field) -> is_constant_field(Args, Field); - is_constant_field([{[{Field, {[{Cond, _Val}]}}]} | _Rest], Field) -> Cond =:= <<"$eq">>; - is_constant_field([{[{_UnMatched, _}]} | Rest], Field) -> is_constant_field(Rest, Field). - %%%%%%%% module tests below %%%%%%%% -ifdef(TEST). @@ -721,42 +649,49 @@ is_constant_field_basic_test() -> ?assertEqual(true, is_constant_field(Selector, Field)). is_constant_field_basic_two_test() -> - Selector = normalize({[{<<"$and">>, - [ - {[{<<"cars">>,{[{<<"$eq">>,<<"2">>}]}}]}, - {[{<<"age">>,{[{<<"$gt">>,10}]}}]} - ] - }]}), + Selector = normalize( + {[ + {<<"$and">>, [ + {[{<<"cars">>, {[{<<"$eq">>, <<"2">>}]}}]}, + {[{<<"age">>, {[{<<"$gt">>, 10}]}}]} + ]} + ]} + ), Field = <<"cars">>, ?assertEqual(true, is_constant_field(Selector, Field)). is_constant_field_not_eq_test() -> - Selector = normalize({[{<<"$and">>, - [ - {[{<<"cars">>,{[{<<"$eq">>,<<"2">>}]}}]}, - {[{<<"age">>,{[{<<"$gt">>,10}]}}]} - ] - }]}), + Selector = normalize( + {[ + {<<"$and">>, [ + {[{<<"cars">>, {[{<<"$eq">>, <<"2">>}]}}]}, + {[{<<"age">>, {[{<<"$gt">>, 10}]}}]} + ]} + ]} + ), Field = <<"age">>, ?assertEqual(false, is_constant_field(Selector, Field)). is_constant_field_missing_field_test() -> - Selector = normalize({[{<<"$and">>, - [ - {[{<<"cars">>,{[{<<"$eq">>,<<"2">>}]}}]}, - {[{<<"age">>,{[{<<"$gt">>,10}]}}]} - ] - }]}), + Selector = normalize( + {[ + {<<"$and">>, [ + {[{<<"cars">>, {[{<<"$eq">>, <<"2">>}]}}]}, + {[{<<"age">>, {[{<<"$gt">>, 10}]}}]} + ]} + ]} + ), Field = <<"wrong">>, ?assertEqual(false, is_constant_field(Selector, Field)). is_constant_field_or_field_test() -> - Selector = {[{<<"$or">>, - [ - {[{<<"A">>, <<"foo">>}]}, - {[{<<"B">>, <<"foo">>}]} - ] - }]}, + Selector = + {[ + {<<"$or">>, [ + {[{<<"A">>, <<"foo">>}]}, + {[{<<"B">>, <<"foo">>}]} + ]} + ]}, Normalized = normalize(Selector), Field = <<"A">>, ?assertEqual(false, is_constant_field(Normalized, Field)). @@ -767,37 +702,40 @@ is_constant_field_empty_selector_test() -> ?assertEqual(false, is_constant_field(Selector, Field)). is_constant_nested_and_test() -> - Selector1 = {[{<<"$and">>, - [ - {[{<<"A">>, <<"foo">>}]} - ] - }]}, - Selector2 = {[{<<"$and">>, - [ - {[{<<"B">>, {[{<<"$gt">>,10}]}}]} - ] - }]}, - Selector = {[{<<"$and">>, - [ - Selector1, - Selector2 - ] - }]}, + Selector1 = + {[ + {<<"$and">>, [ + {[{<<"A">>, <<"foo">>}]} + ]} + ]}, + Selector2 = + {[ + {<<"$and">>, [ + {[{<<"B">>, {[{<<"$gt">>, 10}]}}]} + ]} + ]}, + Selector = + {[ + {<<"$and">>, [ + Selector1, + Selector2 + ]} + ]}, Normalized = normalize(Selector), ?assertEqual(true, is_constant_field(Normalized, <<"A">>)), ?assertEqual(false, is_constant_field(Normalized, <<"B">>)). is_constant_combined_or_and_equals_test() -> - Selector = {[{<<"A">>, "foo"}, - {<<"$or">>, - [ - {[{<<"B">>, <<"bar">>}]}, - {[{<<"B">>, <<"baz">>}]} - ] - }, - {<<"C">>, "qux"} - ]}, + Selector = + {[ + {<<"A">>, "foo"}, + {<<"$or">>, [ + {[{<<"B">>, <<"bar">>}]}, + {[{<<"B">>, <<"baz">>}]} + ]}, + {<<"C">>, "qux"} + ]}, Normalized = normalize(Selector), ?assertEqual(true, is_constant_field(Normalized, <<"C">>)), ?assertEqual(false, is_constant_field(Normalized, <<"B">>)). @@ -822,202 +760,225 @@ has_required_fields_empty_selector_test() -> has_required_fields_exists_false_test() -> RequiredFields = [<<"A">>], - Selector = {[{<<"A">>,{[{<<"$exists">>, false}]}}]}, + Selector = {[{<<"A">>, {[{<<"$exists">>, false}]}}]}, Normalized = normalize(Selector), ?assertEqual(false, has_required_fields(Normalized, RequiredFields)). has_required_fields_and_true_test() -> RequiredFields = [<<"A">>], - Selector = {[{<<"$and">>, - [ - {[{<<"A">>, <<"foo">>}]}, - {[{<<"B">>, <<"foo">>}]} - ] - }]}, + Selector = + {[ + {<<"$and">>, [ + {[{<<"A">>, <<"foo">>}]}, + {[{<<"B">>, <<"foo">>}]} + ]} + ]}, Normalized = normalize(Selector), ?assertEqual(true, has_required_fields(Normalized, RequiredFields)). has_required_fields_nested_and_true_test() -> RequiredFields = [<<"A">>, <<"B">>], - Selector1 = {[{<<"$and">>, - [ - {[{<<"A">>, <<"foo">>}]} - ] - }]}, - Selector2 = {[{<<"$and">>, - [ - {[{<<"B">>, <<"foo">>}]} - ] - }]}, - Selector = {[{<<"$and">>, - [ - Selector1, - Selector2 - ] - }]}, + Selector1 = + {[ + {<<"$and">>, [ + {[{<<"A">>, <<"foo">>}]} + ]} + ]}, + Selector2 = + {[ + {<<"$and">>, [ + {[{<<"B">>, <<"foo">>}]} + ]} + ]}, + Selector = + {[ + {<<"$and">>, [ + Selector1, + Selector2 + ]} + ]}, Normalized = normalize(Selector), ?assertEqual(true, has_required_fields(Normalized, RequiredFields)). has_required_fields_and_false_test() -> RequiredFields = [<<"A">>, <<"C">>], - Selector = {[{<<"$and">>, - [ - {[{<<"A">>, <<"foo">>}]}, - {[{<<"B">>, <<"foo">>}]} - ] - }]}, + Selector = + {[ + {<<"$and">>, [ + {[{<<"A">>, <<"foo">>}]}, + {[{<<"B">>, <<"foo">>}]} + ]} + ]}, Normalized = normalize(Selector), ?assertEqual(false, has_required_fields(Normalized, RequiredFields)). has_required_fields_or_false_test() -> RequiredFields = [<<"A">>], - Selector = {[{<<"$or">>, - [ - {[{<<"A">>, <<"foo">>}]}, - {[{<<"B">>, <<"foo">>}]} - ] - }]}, + Selector = + {[ + {<<"$or">>, [ + {[{<<"A">>, <<"foo">>}]}, + {[{<<"B">>, <<"foo">>}]} + ]} + ]}, Normalized = normalize(Selector), ?assertEqual(false, has_required_fields(Normalized, RequiredFields)). has_required_fields_or_true_test() -> RequiredFields = [<<"A">>, <<"B">>, <<"C">>], - Selector = {[{<<"A">>, "foo"}, - {<<"$or">>, - [ - {[{<<"B">>, <<"bar">>}]}, - {[{<<"B">>, <<"baz">>}]} - ] - }, - {<<"C">>, "qux"} - ]}, + Selector = + {[ + {<<"A">>, "foo"}, + {<<"$or">>, [ + {[{<<"B">>, <<"bar">>}]}, + {[{<<"B">>, <<"baz">>}]} + ]}, + {<<"C">>, "qux"} + ]}, Normalized = normalize(Selector), ?assertEqual(true, has_required_fields(Normalized, RequiredFields)). has_required_fields_and_nested_or_true_test() -> RequiredFields = [<<"A">>, <<"B">>], - Selector1 = {[{<<"$and">>, - [ - {[{<<"A">>, <<"foo">>}]} - ] - }]}, - Selector2 = {[{<<"$or">>, - [ - {[{<<"B">>, <<"foo">>}]}, - {[{<<"B">>, <<"foo">>}]} - ] - }]}, - Selector = {[{<<"$and">>, - [ - Selector1, - Selector2 - ] - }]}, + Selector1 = + {[ + {<<"$and">>, [ + {[{<<"A">>, <<"foo">>}]} + ]} + ]}, + Selector2 = + {[ + {<<"$or">>, [ + {[{<<"B">>, <<"foo">>}]}, + {[{<<"B">>, <<"foo">>}]} + ]} + ]}, + Selector = + {[ + {<<"$and">>, [ + Selector1, + Selector2 + ]} + ]}, Normalized = normalize(Selector), ?assertEqual(true, has_required_fields(Normalized, RequiredFields)), - SelectorReverse = {[{<<"$and">>, - [ - Selector2, - Selector1 - ] - }]}, + SelectorReverse = + {[ + {<<"$and">>, [ + Selector2, + Selector1 + ]} + ]}, NormalizedReverse = normalize(SelectorReverse), ?assertEqual(true, has_required_fields(NormalizedReverse, RequiredFields)). has_required_fields_and_nested_or_false_test() -> RequiredFields = [<<"A">>, <<"B">>], - Selector1 = {[{<<"$and">>, - [ - {[{<<"A">>, <<"foo">>}]} - ] - }]}, - Selector2 = {[{<<"$or">>, - [ - {[{<<"A">>, <<"foo">>}]}, - {[{<<"B">>, <<"foo">>}]} - ] - }]}, - Selector = {[{<<"$and">>, - [ - Selector1, - Selector2 - ] - }]}, + Selector1 = + {[ + {<<"$and">>, [ + {[{<<"A">>, <<"foo">>}]} + ]} + ]}, + Selector2 = + {[ + {<<"$or">>, [ + {[{<<"A">>, <<"foo">>}]}, + {[{<<"B">>, <<"foo">>}]} + ]} + ]}, + Selector = + {[ + {<<"$and">>, [ + Selector1, + Selector2 + ]} + ]}, Normalized = normalize(Selector), ?assertEqual(false, has_required_fields(Normalized, RequiredFields)), - SelectorReverse = {[{<<"$and">>, - [ - Selector2, - Selector1 - ] - }]}, + SelectorReverse = + {[ + {<<"$and">>, [ + Selector2, + Selector1 + ]} + ]}, NormalizedReverse = normalize(SelectorReverse), ?assertEqual(false, has_required_fields(NormalizedReverse, RequiredFields)). has_required_fields_or_nested_and_true_test() -> RequiredFields = [<<"A">>], - Selector1 = {[{<<"$and">>, - [ - {[{<<"A">>, <<"foo">>}]} - ] - }]}, - Selector2 = {[{<<"$and">>, - [ - {[{<<"A">>, <<"foo">>}]} - ] - }]}, - Selector = {[{<<"$or">>, - [ - Selector1, - Selector2 - ] - }]}, + Selector1 = + {[ + {<<"$and">>, [ + {[{<<"A">>, <<"foo">>}]} + ]} + ]}, + Selector2 = + {[ + {<<"$and">>, [ + {[{<<"A">>, <<"foo">>}]} + ]} + ]}, + Selector = + {[ + {<<"$or">>, [ + Selector1, + Selector2 + ]} + ]}, Normalized = normalize(Selector), ?assertEqual(true, has_required_fields(Normalized, RequiredFields)). has_required_fields_or_nested_or_true_test() -> RequiredFields = [<<"A">>], - Selector1 = {[{<<"$or">>, - [ - {[{<<"A">>, <<"foo">>}]} - ] - }]}, - Selector2 = {[{<<"$or">>, - [ - {[{<<"A">>, <<"bar">>}]} - ] - }]}, - Selector = {[{<<"$or">>, - [ - Selector1, - Selector2 - ] - }]}, + Selector1 = + {[ + {<<"$or">>, [ + {[{<<"A">>, <<"foo">>}]} + ]} + ]}, + Selector2 = + {[ + {<<"$or">>, [ + {[{<<"A">>, <<"bar">>}]} + ]} + ]}, + Selector = + {[ + {<<"$or">>, [ + Selector1, + Selector2 + ]} + ]}, Normalized = normalize(Selector), ?assertEqual(true, has_required_fields(Normalized, RequiredFields)). has_required_fields_or_nested_or_false_test() -> RequiredFields = [<<"A">>], - Selector1 = {[{<<"$or">>, - [ - {[{<<"A">>, <<"foo">>}]} - ] - }]}, - Selector2 = {[{<<"$or">>, - [ - {[{<<"B">>, <<"bar">>}]} - ] - }]}, - Selector = {[{<<"$or">>, - [ - Selector1, - Selector2 - ] - }]}, + Selector1 = + {[ + {<<"$or">>, [ + {[{<<"A">>, <<"foo">>}]} + ]} + ]}, + Selector2 = + {[ + {<<"$or">>, [ + {[{<<"B">>, <<"bar">>}]} + ]} + ]}, + Selector = + {[ + {<<"$or">>, [ + Selector1, + Selector2 + ]} + ]}, Normalized = normalize(Selector), ?assertEqual(false, has_required_fields(Normalized, RequiredFields)). diff --git a/src/mango/src/mango_selector_text.erl b/src/mango/src/mango_selector_text.erl index b3b61ff26..aaa1e3329 100644 --- a/src/mango/src/mango_selector_text.erl +++ b/src/mango/src/mango_selector_text.erl @@ -12,7 +12,6 @@ -module(mango_selector_text). - -export([ convert/1, convert/2, @@ -20,20 +19,16 @@ append_sort_type/2 ]). - -include_lib("couch/include/couch_db.hrl"). -include("mango.hrl"). - %% Regex for <<"\\.">> -define(PERIOD, "\\."). - convert(Object) -> TupleTree = convert([], Object), iolist_to_binary(to_query(TupleTree)). - convert(Path, {[{<<"$and">>, Args}]}) -> Parts = [convert(Path, Arg) || Arg <- Args], {op_and, Parts}; @@ -45,12 +40,10 @@ convert(Path, {[{<<"$not">>, Arg}]}) -> convert(Path, {[{<<"$default">>, Arg}]}) -> {op_field, {_, Query}} = convert(Path, Arg), {op_default, Query}; - % The $text operator specifies a Lucene syntax query % so we just pull it in directly. convert(Path, {[{<<"$text">>, Query}]}) when is_binary(Query) -> {op_field, {make_field(Path, Query), value_str(Query)}}; - % The MongoDB docs for $all are super confusing and read more % like they screwed up the implementation of this operator % and then just documented it as a feature. @@ -68,15 +61,14 @@ convert(Path, {[{<<"$all">>, Args}]}) -> % that means we just need to search for each value in % Path.[] and Path.[].[] and rely on our filtering to limit % the results properly. - Fields1 = convert(Path, {[{<<"$eq">> , Values}]}), - Fields2 = convert([<<"[]">>| Path], {[{<<"$eq">> , Values}]}), + Fields1 = convert(Path, {[{<<"$eq">>, Values}]}), + Fields2 = convert([<<"[]">> | Path], {[{<<"$eq">>, Values}]}), {op_or, [Fields1, Fields2]}; _ -> % Otherwise the $all operator is equivalent to an $and % operator so we treat it as such. convert([<<"[]">> | Path], {[{<<"$and">>, Args}]}) end; - % The $elemMatch Lucene query is not an exact translation % as we can't enforce that the matches are all for the same % item in an array. We just rely on the final selector match @@ -85,18 +77,22 @@ convert(Path, {[{<<"$all">>, Args}]}) -> % say this has to match against an array. convert(Path, {[{<<"$elemMatch">>, Arg}]}) -> convert([<<"[]">> | Path], Arg); - convert(Path, {[{<<"$allMatch">>, Arg}]}) -> convert([<<"[]">> | Path], Arg); - % Our comparison operators are fairly straight forward -convert(Path, {[{<<"$lt">>, Arg}]}) when is_list(Arg); is_tuple(Arg); - Arg =:= null -> +convert(Path, {[{<<"$lt">>, Arg}]}) when + is_list(Arg); + is_tuple(Arg); + Arg =:= null +-> field_exists_query(Path); convert(Path, {[{<<"$lt">>, Arg}]}) -> {op_field, {make_field(Path, Arg), range(lt, Arg)}}; -convert(Path, {[{<<"$lte">>, Arg}]}) when is_list(Arg); is_tuple(Arg); - Arg =:= null-> +convert(Path, {[{<<"$lte">>, Arg}]}) when + is_list(Arg); + is_tuple(Arg); + Arg =:= null +-> field_exists_query(Path); convert(Path, {[{<<"$lte">>, Arg}]}) -> {op_field, {make_field(Path, Arg), range(lte, Arg)}}; @@ -115,66 +111,64 @@ convert(Path, {[{<<"$eq">>, Arg}]}) -> {op_field, {make_field(Path, Arg), value_str(Arg)}}; convert(Path, {[{<<"$ne">>, Arg}]}) -> {op_not, {field_exists_query(Path), convert(Path, {[{<<"$eq">>, Arg}]})}}; -convert(Path, {[{<<"$gte">>, Arg}]}) when is_list(Arg); is_tuple(Arg); - Arg =:= null -> +convert(Path, {[{<<"$gte">>, Arg}]}) when + is_list(Arg); + is_tuple(Arg); + Arg =:= null +-> field_exists_query(Path); convert(Path, {[{<<"$gte">>, Arg}]}) -> {op_field, {make_field(Path, Arg), range(gte, Arg)}}; -convert(Path, {[{<<"$gt">>, Arg}]}) when is_list(Arg); is_tuple(Arg); - Arg =:= null-> +convert(Path, {[{<<"$gt">>, Arg}]}) when + is_list(Arg); + is_tuple(Arg); + Arg =:= null +-> field_exists_query(Path); convert(Path, {[{<<"$gt">>, Arg}]}) -> {op_field, {make_field(Path, Arg), range(gt, Arg)}}; - convert(Path, {[{<<"$in">>, Args}]}) -> {op_or, convert_in(Path, Args)}; - convert(Path, {[{<<"$nin">>, Args}]}) -> {op_not, {field_exists_query(Path), convert(Path, {[{<<"$in">>, Args}]})}}; - convert(Path, {[{<<"$exists">>, ShouldExist}]}) -> FieldExists = field_exists_query(Path), case ShouldExist of true -> FieldExists; false -> {op_not, {FieldExists, false}} end; - % We're not checking the actual type here, just looking for % anything that has a possibility of matching by checking % for the field name. We use the same logic for $exists on % the actual query. convert(Path, {[{<<"$type">>, _}]}) -> field_exists_query(Path); - convert(Path, {[{<<"$mod">>, _}]}) -> field_exists_query(Path, "number"); - % The lucene regular expression engine does not use java's regex engine but % instead a custom implementation. The syntax is therefore different, so we do % would get different behavior than our view indexes. To be consistent, we will % simply return docs for fields that exist and then run our match filter. convert(Path, {[{<<"$regex">>, _}]}) -> field_exists_query(Path, "string"); - convert(Path, {[{<<"$size">>, Arg}]}) -> {op_field, {make_field([<<"[]">> | Path], length), value_str(Arg)}}; - % All other operators are internal assertion errors for % matching because we either should've removed them during % normalization or something else broke. -convert(_Path, {[{<<"$", _/binary>>=Op, _}]}) -> +convert(_Path, {[{<<"$", _/binary>> = Op, _}]}) -> ?MANGO_ERROR({invalid_operator, Op}); - % We've hit a field name specifier. Check if the field name is accessing % arrays. Convert occurrences of element position references to .[]. Then we % need to break the name into path parts and continue our conversion. convert(Path, {[{Field0, Cond}]}) -> - {ok, PP0} = case Field0 of - <<>> -> - {ok, []}; - _ -> - mango_util:parse_field(Field0) - end, + {ok, PP0} = + case Field0 of + <<>> -> + {ok, []}; + _ -> + mango_util:parse_field(Field0) + end, % Later on, we perform a lucene_escape_user call on the % final Path, which calls parse_field again. Calling the function % twice converts <<"a\\.b">> to [<<"a">>,<<"b">>]. This leads to @@ -182,8 +176,15 @@ convert(Path, {[{Field0, Cond}]}) -> % our escaping mechanism, we simply revert this first parse_field % effect and replace instances of "." to "\\.". MP = mango_util:cached_re(mango_period, ?PERIOD), - PP1 = [re:replace(P, MP, <<"\\\\.">>, - [global,{return,binary}]) || P <- PP0], + PP1 = [ + re:replace( + P, + MP, + <<"\\\\.">>, + [global, {return, binary}] + ) + || P <- PP0 + ], {PP2, HasInteger} = replace_array_indexes(PP1, [], false), NewPath = PP2 ++ Path, case HasInteger of @@ -195,101 +196,88 @@ convert(Path, {[{Field0, Cond}]}) -> false -> convert(NewPath, Cond) end; - %% For $in convert(Path, Val) when is_binary(Val); is_number(Val); is_boolean(Val) -> {op_field, {make_field(Path, Val), value_str(Val)}}; - % Anything else is a bad selector. convert(_Path, {Props} = Sel) when length(Props) > 1 -> erlang:error({unnormalized_selector, Sel}). - to_query_nested(Args) -> QueryArgs = lists:map(fun to_query/1, Args), % removes empty queries that result from selectors with empty arrays FilterFun = fun(A) -> A =/= [] andalso A =/= "()" end, lists:filter(FilterFun, QueryArgs). - to_query({op_and, []}) -> []; - to_query({op_and, Args}) when is_list(Args) -> case to_query_nested(Args) of [] -> []; - QueryArgs -> ["(", mango_util:join(<<" AND ">>, QueryArgs), ")"] + QueryArgs -> ["(", mango_util:join(<<" AND ">>, QueryArgs), ")"] end; - to_query({op_or, []}) -> []; - to_query({op_or, Args}) when is_list(Args) -> case to_query_nested(Args) of [] -> []; QueryArgs -> ["(", mango_util:join(" OR ", QueryArgs), ")"] end; - to_query({op_not, {ExistsQuery, Arg}}) when is_tuple(Arg) -> case to_query(Arg) of [] -> ["(", to_query(ExistsQuery), ")"]; Query -> ["(", to_query(ExistsQuery), " AND NOT (", Query, "))"] end; - %% For $exists:false to_query({op_not, {ExistsQuery, false}}) -> ["($fieldnames:/.*/ ", " AND NOT (", to_query(ExistsQuery), "))"]; - to_query({op_insert, Arg}) when is_binary(Arg) -> ["(", Arg, ")"]; - %% We escape : and / for now for values and all lucene chars for fieldnames %% This needs to be resolved. to_query({op_field, {Name, Value}}) -> NameBin = iolist_to_binary(Name), ["(", mango_util:lucene_escape_user(NameBin), ":", Value, ")"]; - %% This is for indexable_fields to_query({op_null, {Name, Value}}) -> NameBin = iolist_to_binary(Name), ["(", mango_util:lucene_escape_user(NameBin), ":", Value, ")"]; - to_query({op_fieldname, {Name, Wildcard}}) -> NameBin = iolist_to_binary(Name), ["($fieldnames:", mango_util:lucene_escape_user(NameBin), Wildcard, ")"]; - to_query({op_default, Value}) -> ["($default:", Value, ")"]. - %% We match on fieldname and fieldname.[] convert_in(Path, Args) -> Path0 = [<<"[]">> | Path], - lists:map(fun(Arg) -> - case Arg of - {Object} -> - Parts = lists:map(fun (SubObject) -> - Fields1 = convert(Path, {[SubObject]}), - Fields2 = convert(Path0, {[SubObject]}), + lists:map( + fun(Arg) -> + case Arg of + {Object} -> + Parts = lists:map( + fun(SubObject) -> + Fields1 = convert(Path, {[SubObject]}), + Fields2 = convert(Path0, {[SubObject]}), + {op_or, [Fields1, Fields2]} + end, + Object + ), + {op_or, Parts}; + SingleVal -> + Fields1 = {op_field, {make_field(Path, SingleVal), value_str(SingleVal)}}, + Fields2 = {op_field, {make_field(Path0, SingleVal), value_str(SingleVal)}}, {op_or, [Fields1, Fields2]} - end, Object), - {op_or, Parts}; - SingleVal -> - Fields1 = {op_field, {make_field(Path, SingleVal), - value_str(SingleVal)}}, - Fields2 = {op_field, {make_field(Path0, SingleVal), - value_str(SingleVal)}}, - {op_or, [Fields1, Fields2]} - end - end, Args). - + end + end, + Args + ). make_field(Path, length) -> [path_str(Path), <<":length">>]; make_field(Path, Arg) -> [path_str(Path), <<":">>, type_str(Arg)]. - range(lt, Arg) -> Min = get_range(min, Arg), [<<"[", Min/binary, " TO ">>, value_str(Arg), <<"}">>]; @@ -312,7 +300,6 @@ get_range(max, Arg) when is_number(Arg) -> get_range(max, _Arg) -> <<"\u0x10FFFF">>. - field_exists_query(Path) -> % We specify two here for :* and .* so that we don't incorrectly % match a path foo.name against foo.name_first (if were to just @@ -326,15 +313,12 @@ field_exists_query(Path) -> ], {op_or, Parts}. - field_exists_query(Path, Type) -> {op_fieldname, {[path_str(Path), ":"], Type}}. - path_str(Path) -> path_str(Path, []). - path_str([], Acc) -> Acc; path_str([Part], Acc) -> @@ -350,7 +334,6 @@ path_str([Part | Rest], Acc) -> path_str(Rest, [<<".">>, Part | Acc]) end. - type_str(Value) when is_number(Value) -> <<"number">>; type_str(Value) when is_boolean(Value) -> @@ -360,7 +343,6 @@ type_str(Value) when is_binary(Value) -> type_str(null) -> <<"null">>. - value_str(Value) when is_binary(Value) -> case mango_util:is_number_string(Value) of true -> @@ -380,7 +362,6 @@ value_str(false) -> value_str(null) -> <<"true">>. - append_sort_type(RawSortField, Selector) -> EncodeField = mango_util:lucene_escape_user(RawSortField), String = mango_util:has_suffix(EncodeField, <<"_3astring">>), @@ -395,7 +376,6 @@ append_sort_type(RawSortField, Selector) -> <<EncodeField/binary, Type/binary>> end. - get_sort_type(Field, Selector) -> Types = get_sort_types(Field, Selector, []), case lists:usort(Types) of @@ -404,35 +384,40 @@ get_sort_type(Field, Selector) -> _ -> ?MANGO_ERROR({text_sort_error, Field}) end. - -get_sort_types(Field, {[{Field, {[{<<"$", _/binary>>, Cond}]}}]}, Acc) - when is_binary(Cond) -> +get_sort_types(Field, {[{Field, {[{<<"$", _/binary>>, Cond}]}}]}, Acc) when + is_binary(Cond) +-> [str | Acc]; - -get_sort_types(Field, {[{Field, {[{<<"$", _/binary>>, Cond}]}}]}, Acc) - when is_number(Cond) -> +get_sort_types(Field, {[{Field, {[{<<"$", _/binary>>, Cond}]}}]}, Acc) when + is_number(Cond) +-> [num | Acc]; - get_sort_types(Field, {[{_, Cond}]}, Acc) when is_list(Cond) -> - lists:foldl(fun(Arg, InnerAcc) -> - get_sort_types(Field, Arg, InnerAcc) - end, Acc, Cond); - -get_sort_types(Field, {[{_, Cond}]}, Acc) when is_tuple(Cond)-> + lists:foldl( + fun(Arg, InnerAcc) -> + get_sort_types(Field, Arg, InnerAcc) + end, + Acc, + Cond + ); +get_sort_types(Field, {[{_, Cond}]}, Acc) when is_tuple(Cond) -> get_sort_types(Field, Cond, Acc); - -get_sort_types(_Field, _, Acc) -> +get_sort_types(_Field, _, Acc) -> Acc. - replace_array_indexes([], NewPartsAcc, HasIntAcc) -> {NewPartsAcc, HasIntAcc}; replace_array_indexes([Part | Rest], NewPartsAcc, HasIntAcc) -> - {NewPart, HasInt} = try - _ = list_to_integer(binary_to_list(Part)), - {<<"[]">>, true} - catch _:_ -> - {Part, false} - end, - replace_array_indexes(Rest, [NewPart | NewPartsAcc], - HasInt or HasIntAcc). + {NewPart, HasInt} = + try + _ = list_to_integer(binary_to_list(Part)), + {<<"[]">>, true} + catch + _:_ -> + {Part, false} + end, + replace_array_indexes( + Rest, + [NewPart | NewPartsAcc], + HasInt or HasIntAcc + ). diff --git a/src/mango/src/mango_sort.erl b/src/mango/src/mango_sort.erl index 17249c297..808b6e7f2 100644 --- a/src/mango/src/mango_sort.erl +++ b/src/mango/src/mango_sort.erl @@ -19,10 +19,8 @@ directions/1 ]). - -include("mango.hrl"). - new(Fields) when is_list(Fields) -> Sort = {[sort_field(Field) || Field <- Fields]}, validate(Sort), @@ -30,7 +28,6 @@ new(Fields) when is_list(Fields) -> new(Else) -> ?MANGO_ERROR({invalid_sort_json, Else}). - to_json({Fields}) -> to_json(Fields); to_json([]) -> @@ -38,15 +35,12 @@ to_json([]) -> to_json([{Name, Dir} | Rest]) -> [{[{Name, Dir}]} | to_json(Rest)]. - fields({Props}) -> [Name || {Name, _Dir} <- Props]. - directions({Props}) -> [Dir || {_Name, Dir} <- Props]. - sort_field(<<"">>) -> ?MANGO_ERROR({invalid_sort_field, <<"">>}); sort_field(Field) when is_binary(Field) -> @@ -60,7 +54,6 @@ sort_field({Name, BadDir}) when is_binary(Name) -> sort_field(Else) -> ?MANGO_ERROR({invalid_sort_field, Else}). - validate({Props}) -> % Assert each field is in the same direction % until we support mixed direction sorts. diff --git a/src/mango/src/mango_sup.erl b/src/mango/src/mango_sup.erl index b0dedf125..c0b04d9c9 100644 --- a/src/mango/src/mango_sup.erl +++ b/src/mango/src/mango_sup.erl @@ -16,9 +16,8 @@ -export([start_link/1]). - start_link(Args) -> - supervisor:start_link({local,?MODULE}, ?MODULE, Args). + supervisor:start_link({local, ?MODULE}, ?MODULE, Args). init([]) -> {ok, {{one_for_one, 3, 10}, couch_epi:register_service(mango_epi, [])}}. diff --git a/src/mango/src/mango_util.erl b/src/mango/src/mango_util.erl index 8257e841f..609a9dbc0 100644 --- a/src/mango/src/mango_util.erl +++ b/src/mango/src/mango_util.erl @@ -12,7 +12,6 @@ -module(mango_util). - -export([ open_doc/2, open_ddocs/1, @@ -46,44 +45,41 @@ cached_re/2 ]). - -include_lib("couch/include/couch_db.hrl"). -include("mango.hrl"). - -define(DIGITS, "(\\p{N}+)"). -define(HEXDIGITS, "([0-9a-fA-F]+)"). -define(EXP, "[eE][+-]?" ++ ?DIGITS). -define(NUMSTRING, -"[\\x00-\\x20]*" ++ "[+-]?(" ++ "NaN|" - ++ "Infinity|" ++ "(((" - ++ ?DIGITS - ++ "(\\.)?(" - ++ ?DIGITS - ++ "?)(" - ++ ?EXP - ++ ")?)|" - ++ "(\\.(" - ++ ?DIGITS - ++ ")(" - ++ ?EXP - ++ ")?)|" - ++ "((" - ++ "(0[xX]" - ++ ?HEXDIGITS - ++ "(\\.)?)|" - ++ "(0[xX]" - ++ ?HEXDIGITS - ++ "?(\\.)" - ++ ?HEXDIGITS - ++ ")" - ++ ")[pP][+-]?" ++ ?DIGITS ++ "))" ++ "[fFdD]?))" ++ "[\\x00-\\x20]*"). - + "[\\x00-\\x20]*" ++ "[+-]?(" ++ "NaN|" ++ + "Infinity|" ++ "(((" ++ + ?DIGITS ++ + "(\\.)?(" ++ + ?DIGITS ++ + "?)(" ++ + ?EXP ++ + ")?)|" ++ + "(\\.(" ++ + ?DIGITS ++ + ")(" ++ + ?EXP ++ + ")?)|" ++ + "((" ++ + "(0[xX]" ++ + ?HEXDIGITS ++ + "(\\.)?)|" ++ + "(0[xX]" ++ + ?HEXDIGITS ++ + "?(\\.)" ++ + ?HEXDIGITS ++ + ")" ++ + ")[pP][+-]?" ++ ?DIGITS ++ "))" ++ "[fFdD]?))" ++ "[\\x00-\\x20]*" +). open_doc(Db, DocId) -> open_doc(Db, DocId, [deleted, ejson_body]). - open_doc(Db, DocId, Options) -> case mango_util:defer(fabric, open_doc, [Db, DocId, Options]) of {ok, Doc} -> @@ -94,7 +90,6 @@ open_doc(Db, DocId, Options) -> ?MANGO_ERROR({error_loading_doc, DocId}) end. - open_ddocs(Db) -> case mango_util:defer(fabric, design_docs, [Db]) of {ok, Docs} -> @@ -103,7 +98,6 @@ open_ddocs(Db) -> ?MANGO_ERROR(error_loading_ddocs) end. - load_ddoc(Db, DDocId) -> load_ddoc(Db, DDocId, [deleted, ejson_body]). @@ -112,13 +106,13 @@ load_ddoc(Db, DDocId, DbOpts) -> {ok, Doc} -> {ok, check_lang(Doc)}; not_found -> - Body = {[ - {<<"language">>, <<"query">>} - ]}, + Body = + {[ + {<<"language">>, <<"query">>} + ]}, {ok, #doc{id = DDocId, body = Body}} end. - defer(Mod, Fun, Args) -> {Pid, Ref} = erlang:spawn_monitor(?MODULE, do_defer, [Mod, Fun, Args]), receive @@ -132,7 +126,6 @@ defer(Mod, Fun, Args) -> erlang:exit(Value) end. - do_defer(Mod, Fun, Args) -> try erlang:apply(Mod, Fun, Args) of Resp -> @@ -149,7 +142,6 @@ do_defer(Mod, Fun, Args) -> erlang:exit({mango_defer_exit, Error}) end. - assert_ejson({Props}) -> assert_ejson_obj(Props); assert_ejson(Vals) when is_list(Vals) -> @@ -167,7 +159,6 @@ assert_ejson(Number) when is_number(Number) -> assert_ejson(_Else) -> false. - assert_ejson_obj([]) -> true; assert_ejson_obj([{Key, Val} | Rest]) when is_binary(Key) -> @@ -180,7 +171,6 @@ assert_ejson_obj([{Key, Val} | Rest]) when is_binary(Key) -> assert_ejson_obj(_Else) -> false. - assert_ejson_arr([]) -> true; assert_ejson_arr([Val | Rest]) -> @@ -191,11 +181,11 @@ assert_ejson_arr([Val | Rest]) -> false end. - check_lang(#doc{id = Id, deleted = true}) -> - Body = {[ - {<<"language">>, <<"query">>} - ]}, + Body = + {[ + {<<"language">>, <<"query">>} + ]}, #doc{id = Id, body = Body}; check_lang(#doc{body = {Props}} = Doc) -> case lists:keyfind(<<"language">>, 1, Props) of @@ -205,13 +195,11 @@ check_lang(#doc{body = {Props}} = Doc) -> ?MANGO_ERROR({invalid_ddoc_lang, Else}) end. - to_lower(Key) when is_binary(Key) -> KStr = binary_to_list(Key), KLower = string:to_lower(KStr), list_to_binary(KLower). - enc_dbname(<<>>) -> <<>>; enc_dbname(<<A:8/integer, Rest/binary>>) -> @@ -219,7 +207,6 @@ enc_dbname(<<A:8/integer, Rest/binary>>) -> Tail = enc_dbname(Rest), <<Bytes/binary, Tail/binary>>. - enc_db_byte(N) when N >= $a, N =< $z -> <<N>>; enc_db_byte(N) when N >= $0, N =< $9 -> <<N>>; enc_db_byte(N) when N == $/; N == $_; N == $- -> <<N>>; @@ -228,7 +215,6 @@ enc_db_byte(N) -> L = enc_hex_byte(N rem 16), <<$$, H:8/integer, L:8/integer>>. - dec_dbname(<<>>) -> <<>>; dec_dbname(<<$$, _:8/integer>>) -> @@ -241,7 +227,6 @@ dec_dbname(<<N:8/integer, Rest/binary>>) -> Tail = dec_dbname(Rest), <<N:8/integer, Tail/binary>>. - enc_hex(<<>>) -> <<>>; enc_hex(<<V:8/integer, Rest/binary>>) -> @@ -250,12 +235,10 @@ enc_hex(<<V:8/integer, Rest/binary>>) -> Tail = enc_hex(Rest), <<H:8/integer, L:8/integer, Tail/binary>>. - enc_hex_byte(N) when N >= 0, N < 10 -> $0 + N; enc_hex_byte(N) when N >= 10, N < 16 -> $a + (N - 10); enc_hex_byte(N) -> throw({invalid_hex_value, N}). - dec_hex(<<>>) -> <<>>; dec_hex(<<_:8/integer>>) -> @@ -265,14 +248,11 @@ dec_hex(<<H:8/integer, L:8/integer, Rest/binary>>) -> Tail = dec_hex(Rest), <<Byte:8/integer, Tail/binary>>. - dec_hex_byte(N) when N >= $0, N =< $9 -> (N - $0); dec_hex_byte(N) when N >= $a, N =< $f -> (N - $a) + 10; dec_hex_byte(N) when N >= $A, N =< $F -> (N - $A) + 10; dec_hex_byte(N) -> throw({invalid_hex_character, N}). - - lucene_escape_field(Bin) when is_binary(Bin) -> Str = binary_to_list(Bin), Enc = lucene_escape_field(Str), @@ -289,58 +269,58 @@ lucene_escape_field([H | T]) when is_number(H), H >= 0, H =< 255 -> Hi = enc_hex_byte(H div 16), Lo = enc_hex_byte(H rem 16), [$_, Hi, Lo | lucene_escape_field(T)] - end; + end; lucene_escape_field([]) -> []. - lucene_escape_query_value(IoList) when is_list(IoList) -> lucene_escape_query_value(iolist_to_binary(IoList)); lucene_escape_query_value(Bin) when is_binary(Bin) -> IoList = lucene_escape_qv(Bin), iolist_to_binary(IoList). - % This escapes the special Lucene query characters % listed below as well as any whitespace. % % + - && || ! ( ) { } [ ] ^ ~ * ? : \ " / % -lucene_escape_qv(<<>>) -> []; +lucene_escape_qv(<<>>) -> + []; lucene_escape_qv(<<"&&", Rest/binary>>) -> ["\\&&" | lucene_escape_qv(Rest)]; lucene_escape_qv(<<"||", Rest/binary>>) -> ["\\||" | lucene_escape_qv(Rest)]; lucene_escape_qv(<<C, Rest/binary>>) -> NeedsEscape = "+-(){}[]!^~*?:/\\\" \t\r\n", - Out = case lists:member(C, NeedsEscape) of - true -> ["\\", C]; - false -> [C] - end, + Out = + case lists:member(C, NeedsEscape) of + true -> ["\\", C]; + false -> [C] + end, Out ++ lucene_escape_qv(Rest). - lucene_escape_user(Field) -> {ok, Path} = parse_field(Field), Escaped = [mango_util:lucene_escape_field(P) || P <- Path], iolist_to_binary(join(".", Escaped)). - has_suffix(Bin, Suffix) when is_binary(Bin), is_binary(Suffix) -> SBin = size(Bin), SSuffix = size(Suffix), - if SBin < SSuffix -> false; true -> - PSize = SBin - SSuffix, - case Bin of - <<_:PSize/binary, Suffix/binary>> -> - true; - _ -> - false - end + if + SBin < SSuffix -> + false; + true -> + PSize = SBin - SSuffix, + case Bin of + <<_:PSize/binary, Suffix/binary>> -> + true; + _ -> + false + end end. - join(_Sep, []) -> []; join(_Sep, [Item]) -> @@ -348,10 +328,9 @@ join(_Sep, [Item]) -> join(Sep, [Item | Rest]) -> [Item, Sep | join(Sep, Rest)]. - is_number_string(Value) when is_binary(Value) -> is_number_string(binary_to_list(Value)); -is_number_string(Value) when is_list(Value)-> +is_number_string(Value) when is_list(Value) -> MP = cached_re(mango_numstring_re, ?NUMSTRING), case re:run(Value, MP) of nomatch -> @@ -360,7 +339,6 @@ is_number_string(Value) when is_list(Value)-> true end. - cached_re(Name, RE) -> case mochiglobal:get(Name) of undefined -> @@ -371,7 +349,6 @@ cached_re(Name, RE) -> MP end. - parse_field(Field) -> case binary:match(Field, <<"\\">>, []) of nomatch -> @@ -382,12 +359,15 @@ parse_field(Field) -> end. parse_field_slow(Field) -> - Path = lists:map(fun - (P) when P =:= <<>> -> - ?MANGO_ERROR({invalid_field_name, Field}); - (P) -> - re:replace(P, <<"\\\\">>, <<>>, [global, {return, binary}]) - end, re:split(Field, <<"(?<!\\\\)\\.">>)), + Path = lists:map( + fun + (P) when P =:= <<>> -> + ?MANGO_ERROR({invalid_field_name, Field}); + (P) -> + re:replace(P, <<"\\\\">>, <<>>, [global, {return, binary}]) + end, + re:split(Field, <<"(?<!\\\\)\\.">>) + ), {ok, Path}. check_non_empty(Field, Parts) -> @@ -398,7 +378,6 @@ check_non_empty(Field, Parts) -> Parts end. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). diff --git a/src/mem3/src/mem3.erl b/src/mem3/src/mem3.erl index 86eb3bb45..7151a3ec1 100644 --- a/src/mem3/src/mem3.erl +++ b/src/mem3/src/mem3.erl @@ -12,8 +12,18 @@ -module(mem3). --export([start/0, stop/0, restart/0, nodes/0, node_info/2, shards/1, shards/2, - choose_shards/2, n/1, n/2, dbname/1, ushards/1, ushards/2]). +-export([ + start/0, + stop/0, + restart/0, + nodes/0, + node_info/2, + shards/1, shards/2, + choose_shards/2, + n/1, n/2, + dbname/1, + ushards/1, ushards/2 +]). -export([get_shard/3, local_shards/1, shard_suffix/1, fold_shards/2]). -export([sync_security/0, sync_security/1]). -export([compare_nodelists/0, compare_shards/1]). @@ -47,30 +57,45 @@ restart() -> %% key and the nodes holding that state as the value. Also reports member %% nodes which fail to respond and nodes which are connected but are not %% cluster members. Useful for debugging. --spec compare_nodelists() -> [{{cluster_nodes, [node()]} | bad_nodes - | non_member_nodes, [node()]}]. +-spec compare_nodelists() -> + [ + { + {cluster_nodes, [node()]} + | bad_nodes + | non_member_nodes, + [node()] + } + ]. compare_nodelists() -> Nodes = mem3:nodes(), AllNodes = erlang:nodes([this, visible]), {Replies, BadNodes} = gen_server:multi_call(Nodes, mem3_nodes, get_nodelist), - Dict = lists:foldl(fun({Node, Nodelist}, D) -> - orddict:append({cluster_nodes, Nodelist}, Node, D) - end, orddict:new(), Replies), + Dict = lists:foldl( + fun({Node, Nodelist}, D) -> + orddict:append({cluster_nodes, Nodelist}, Node, D) + end, + orddict:new(), + Replies + ), [{non_member_nodes, AllNodes -- Nodes}, {bad_nodes, BadNodes} | Dict]. --spec compare_shards(DbName::iodata()) -> [{bad_nodes | [#shard{}], [node()]}]. +-spec compare_shards(DbName :: iodata()) -> [{bad_nodes | [#shard{}], [node()]}]. compare_shards(DbName) when is_list(DbName) -> compare_shards(list_to_binary(DbName)); compare_shards(DbName) -> Nodes = mem3:nodes(), {Replies, BadNodes} = rpc:multicall(mem3, shards, [DbName]), GoodNodes = [N || N <- Nodes, not lists:member(N, BadNodes)], - Dict = lists:foldl(fun({Shards, Node}, D) -> - orddict:append(Shards, Node, D) - end, orddict:new(), lists:zip(Replies, GoodNodes)), + Dict = lists:foldl( + fun({Shards, Node}, D) -> + orddict:append(Shards, Node, D) + end, + orddict:new(), + lists:zip(Replies, GoodNodes) + ), [{bad_nodes, BadNodes} | Dict]. --spec n(DbName::iodata()) -> integer(). +-spec n(DbName :: iodata()) -> integer(). n(DbName) -> % Use _design to avoid issues with % partition validation @@ -86,7 +111,7 @@ nodes() -> node_info(Node, Key) -> mem3_nodes:get_node_info(Node, Key). --spec shards(DbName::iodata()) -> [#shard{}]. +-spec shards(DbName :: iodata()) -> [#shard{}]. shards(DbName) -> shards_int(DbName, []). @@ -97,30 +122,36 @@ shards_int(DbName, Options) -> ShardDbName = list_to_binary(config:get("mem3", "shards_db", "_dbs")), case DbName of - ShardDbName when Ordered -> - %% shard_db is treated as a single sharded db to support calls to db_info - %% and view_all_docs - [#ordered_shard{ - node = node(), - name = ShardDbName, - dbname = ShardDbName, - range = [0, (2 bsl 31)-1], - order = undefined, - opts = []}]; - ShardDbName -> - %% shard_db is treated as a single sharded db to support calls to db_info - %% and view_all_docs - [#shard{ - node = node(), - name = ShardDbName, - dbname = ShardDbName, - range = [0, (2 bsl 31)-1], - opts = []}]; - _ -> - mem3_shards:for_db(DbName, Options) + ShardDbName when Ordered -> + %% shard_db is treated as a single sharded db to support calls to db_info + %% and view_all_docs + [ + #ordered_shard{ + node = node(), + name = ShardDbName, + dbname = ShardDbName, + range = [0, (2 bsl 31) - 1], + order = undefined, + opts = [] + } + ]; + ShardDbName -> + %% shard_db is treated as a single sharded db to support calls to db_info + %% and view_all_docs + [ + #shard{ + node = node(), + name = ShardDbName, + dbname = ShardDbName, + range = [0, (2 bsl 31) - 1], + opts = [] + } + ]; + _ -> + mem3_shards:for_db(DbName, Options) end. --spec shards(DbName::iodata(), DocId::binary()) -> [#shard{}]. +-spec shards(DbName :: iodata(), DocId :: binary()) -> [#shard{}]. shards(DbName, DocId) -> shards_int(DbName, DocId, []). @@ -131,22 +162,21 @@ shards_int(DbName, DocId, Options) when is_list(DocId) -> shards_int(DbName, DocId, Options) -> mem3_shards:for_docid(DbName, DocId, Options). - --spec ushards(DbName::iodata()) -> [#shard{}]. +-spec ushards(DbName :: iodata()) -> [#shard{}]. ushards(DbName) -> - Nodes = [node()|erlang:nodes()], + Nodes = [node() | erlang:nodes()], ZoneMap = zone_map(Nodes), Shards = ushards(DbName, live_shards(DbName, Nodes, [ordered]), ZoneMap), mem3_util:downcast(Shards). --spec ushards(DbName::iodata(), DocId::binary()) -> [#shard{}]. +-spec ushards(DbName :: iodata(), DocId :: binary()) -> [#shard{}]. ushards(DbName, DocId) -> Shards = shards_int(DbName, DocId, [ordered]), Shard = hd(Shards), mem3_util:downcast([Shard]). ushards(DbName, Shards0, ZoneMap) -> - {L,S,D} = group_by_proximity(Shards0, ZoneMap), + {L, S, D} = group_by_proximity(Shards0, ZoneMap), % Prefer shards in the local zone over shards in a different zone, % but sort each zone separately to ensure a consistent choice between % nodes in the same zone. @@ -177,34 +207,45 @@ sync_security() -> sync_security(Db) -> mem3_sync_security:go(dbname(Db)). --spec choose_shards(DbName::iodata(), Options::list()) -> [#shard{}]. +-spec choose_shards(DbName :: iodata(), Options :: list()) -> [#shard{}]. choose_shards(DbName, Options) when is_list(DbName) -> choose_shards(list_to_binary(DbName), Options); choose_shards(DbName, Options) -> - try shards(DbName) - catch error:E when E==database_does_not_exist; E==badarg -> - Nodes = allowed_nodes(), - case get_placement(Options) of - undefined -> - choose_shards(DbName, Nodes, Options); - Placement -> - lists:flatmap(fun({Zone, N}) -> - NodesInZone = nodes_in_zone(Nodes, Zone), - Options1 = lists:keymerge(1, [{n,N}], Options), - choose_shards(DbName, NodesInZone, Options1) - end, Placement) - end + try + shards(DbName) + catch + error:E when E == database_does_not_exist; E == badarg -> + Nodes = allowed_nodes(), + case get_placement(Options) of + undefined -> + choose_shards(DbName, Nodes, Options); + Placement -> + lists:flatmap( + fun({Zone, N}) -> + NodesInZone = nodes_in_zone(Nodes, Zone), + Options1 = lists:keymerge(1, [{n, N}], Options), + choose_shards(DbName, NodesInZone, Options1) + end, + Placement + ) + end end. choose_shards(DbName, Nodes, Options) -> NodeCount = length(Nodes), Suffix = couch_util:get_value(shard_suffix, Options, ""), N = mem3_util:n_val(couch_util:get_value(n, Options), NodeCount), - if N =:= 0 -> erlang:error(no_nodes_in_zone); - true -> ok + if + N =:= 0 -> erlang:error(no_nodes_in_zone); + true -> ok end, - Q = mem3_util:q_val(couch_util:get_value(q, Options, - config:get_integer("cluster", "q", 2))), + Q = mem3_util:q_val( + couch_util:get_value( + q, + Options, + config:get_integer("cluster", "q", 2) + ) + ), %% rotate to a random entry in the nodelist for even distribution RotatedNodes = rotate_rand(Nodes), mem3_util:create_partition_map(DbName, N, Q, RotatedNodes, Suffix). @@ -227,10 +268,13 @@ get_placement(Options) -> end. decode_placement_string(PlacementStr) -> - [begin - [Zone, N] = string:tokens(Rule, ":"), - {list_to_binary(Zone), list_to_integer(N)} - end || Rule <- string:tokens(PlacementStr, ",")]. + [ + begin + [Zone, N] = string:tokens(Rule, ":"), + {list_to_binary(Zone), list_to_integer(N)} + end + || Rule <- string:tokens(PlacementStr, ",") + ]. -spec dbname(#shard{} | iodata()) -> binary(). dbname(#shard{dbname = DbName}) -> @@ -245,7 +289,7 @@ dbname(_) -> erlang:error(badarg). %% @doc Determine if DocId belongs in shard (identified by record or filename) -belongs(#shard{}=Shard, DocId) when is_binary(DocId) -> +belongs(#shard{} = Shard, DocId) when is_binary(DocId) -> [Begin, End] = range(Shard), belongs(Begin, End, Shard, DocId); belongs(<<"shards/", _/binary>> = ShardName, DocId) when is_binary(DocId) -> @@ -263,14 +307,19 @@ range(#shard{range = Range}) -> range(#ordered_shard{range = Range}) -> Range; range(<<"shards/", Start:8/binary, "-", End:8/binary, "/", _/binary>>) -> - [httpd_util:hexlist_to_integer(binary_to_list(Start)), - httpd_util:hexlist_to_integer(binary_to_list(End))]. + [ + httpd_util:hexlist_to_integer(binary_to_list(Start)), + httpd_util:hexlist_to_integer(binary_to_list(End)) + ]. allowed_nodes() -> - lists:filter(fun(Node) -> - Decom = mem3:node_info(Node, <<"decom">>), - (Decom =/= true) andalso (Decom =/= <<"true">>) - end, mem3:nodes()). + lists:filter( + fun(Node) -> + Decom = mem3:node_info(Node, <<"decom">>), + (Decom =/= true) andalso (Decom =/= <<"true">>) + end, + mem3:nodes() + ). nodes_in_zone(Nodes, Zone) -> [Node || Node <- Nodes, Zone == mem3:node_info(Node, <<"zone">>)]. @@ -289,8 +338,10 @@ group_by_proximity(Shards) -> group_by_proximity(Shards, zone_map(Nodes)). group_by_proximity(Shards, ZoneMap) -> - {Local, Remote} = lists:partition(fun(S) -> mem3:node(S) =:= node() end, - Shards), + {Local, Remote} = lists:partition( + fun(S) -> mem3:node(S) =:= node() end, + Shards + ), LocalZone = proplists:get_value(node(), ZoneMap), Fun = fun(S) -> proplists:get_value(mem3:node(S), ZoneMap) =:= LocalZone end, {SameZone, DifferentZone} = lists:partition(Fun, Remote), @@ -298,18 +349,25 @@ group_by_proximity(Shards, ZoneMap) -> choose_ushards(DbName, Shards) -> Groups0 = group_by_range(Shards), - Groups1 = [mem3_util:rotate_list({DbName, R}, order_shards(G)) - || {R, G} <- Groups0], + Groups1 = [ + mem3_util:rotate_list({DbName, R}, order_shards(G)) + || {R, G} <- Groups0 + ], [hd(G) || G <- Groups1]. -order_shards([#ordered_shard{}|_]=OrderedShards) -> +order_shards([#ordered_shard{} | _] = OrderedShards) -> lists:keysort(#ordered_shard.order, OrderedShards); order_shards(UnorderedShards) -> UnorderedShards. group_by_range(Shards) -> - lists:foldl(fun(Shard, Dict) -> - orddict:append(mem3:range(Shard), Shard, Dict) end, orddict:new(), Shards). + lists:foldl( + fun(Shard, Dict) -> + orddict:append(mem3:range(Shard), Shard, Dict) + end, + orddict:new(), + Shards + ). % quorum functions @@ -318,15 +376,14 @@ quorum(DbName) when is_binary(DbName) -> quorum(Db) -> quorum(couch_db:name(Db)). - -node(#shard{node=Node}) -> +node(#shard{node = Node}) -> Node; -node(#ordered_shard{node=Node}) -> +node(#ordered_shard{node = Node}) -> Node. -name(#shard{name=Name}) -> +name(#shard{name = Name}) -> Name; -name(#ordered_shard{name=Name}) -> +name(#ordered_shard{name = Name}) -> Name. % Direct calculation of node membership. This is the algorithm part. It @@ -335,9 +392,9 @@ name(#ordered_shard{name=Name}) -> owner(DbName, DocId, Nodes) -> hd(mem3_util:rotate_list({DbName, DocId}, lists:usort(Nodes))). -engine(#shard{opts=Opts}) -> +engine(#shard{opts = Opts}) -> engine(Opts); -engine(#ordered_shard{opts=Opts}) -> +engine(#ordered_shard{opts = Opts}) -> engine(Opts); engine(Opts) when is_list(Opts) -> case couch_util:get_value(engine, Opts) of @@ -360,18 +417,23 @@ ping(Node) -> ping(Node, Timeout) when is_atom(Node) -> %% The implementation of the function is copied from %% lib/kernel/src/net_adm.erl with addition of a Timeout - case catch gen:call({net_kernel, Node}, - '$gen_call', {is_auth, node()}, Timeout) of - {ok, yes} -> pong; + case + catch gen:call( + {net_kernel, Node}, + '$gen_call', + {is_auth, node()}, + Timeout + ) + of + {ok, yes} -> + pong; _ -> erlang:disconnect_node(Node), pang end. - db_is_current(#shard{name = Name}) -> db_is_current(Name); - db_is_current(<<"shards/", _/binary>> = Name) -> try Shards = mem3:shards(mem3:dbname(Name)), @@ -380,13 +442,11 @@ db_is_current(<<"shards/", _/binary>> = Name) -> error:database_does_not_exist -> false end; - db_is_current(Name) when is_binary(Name) -> % This accounts for local (non-sharded) dbs, and is mostly % for unit tests that either test or use mem3_rep logic couch_server:exists(Name). - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -394,25 +454,34 @@ db_is_current(Name) when is_binary(Name) -> -define(ALLOWED_NODE, 'node1@127.0.0.1'). allowed_nodes_test_() -> - {"allowed_nodes test", [{ - setup, - fun () -> - Props = [ - {?ALLOWED_NODE, []}, - {'node2@127.0.0.1', [{<<"decom">>,<<"true">>}]}, - {'node3@127.0.0.1', [{<<"decom">>,true}]}], - ok = meck:expect(mem3_nodes, get_nodelist, - fun() -> proplists:get_keys(Props) end), - ok = meck:expect(mem3_nodes, get_node_info, - fun(Node, Key) -> - couch_util:get_value(Key, proplists:get_value(Node, Props)) - end) - end, - fun (_) -> meck:unload() end, - [ - ?_assertMatch([?ALLOWED_NODE], allowed_nodes()) - ] - }]}. + {"allowed_nodes test", [ + { + setup, + fun() -> + Props = [ + {?ALLOWED_NODE, []}, + {'node2@127.0.0.1', [{<<"decom">>, <<"true">>}]}, + {'node3@127.0.0.1', [{<<"decom">>, true}]} + ], + ok = meck:expect( + mem3_nodes, + get_nodelist, + fun() -> proplists:get_keys(Props) end + ), + ok = meck:expect( + mem3_nodes, + get_node_info, + fun(Node, Key) -> + couch_util:get_value(Key, proplists:get_value(Node, Props)) + end + ) + end, + fun(_) -> meck:unload() end, + [ + ?_assertMatch([?ALLOWED_NODE], allowed_nodes()) + ] + } + ]}. rotate_rand_degenerate_test() -> ?assertEqual([1], rotate_rand([1])). diff --git a/src/mem3/src/mem3_bdu.erl b/src/mem3/src/mem3_bdu.erl index bf84d4470..84eda2397 100644 --- a/src/mem3/src/mem3_bdu.erl +++ b/src/mem3/src/mem3_bdu.erl @@ -12,35 +12,28 @@ -module(mem3_bdu). - -export([ before_doc_update/3 ]). - -include_lib("couch/include/couch_db.hrl"). - --spec before_doc_update(#doc{}, Db::any(), couch_db:update_type()) -> #doc{}. +-spec before_doc_update(#doc{}, Db :: any(), couch_db:update_type()) -> #doc{}. before_doc_update(#doc{id = <<?DESIGN_DOC_PREFIX, _/binary>>} = Doc, _Db, _UpdateType) -> % Skip design docs Doc; - before_doc_update(#doc{deleted = true} = Doc, _Db, _UpdateType) -> % Skip deleted Doc; - before_doc_update(#doc{} = Doc, _Db, replicated_changes) -> % Skip internal replicator updates Doc; - before_doc_update(#doc{} = Doc, _Db, _UpdateType) -> Body1 = couch_util:json_encode(Doc#doc.body), Body2 = couch_util:json_decode(Body1, [return_maps]), validate(Body2), Doc. - validate(#{} = Body) -> validate_key(<<"by_node">>, Body, ["by_node is mandatory"]), validate_key(<<"by_range">>, Body, ["by_range is mandatory"]), @@ -57,48 +50,55 @@ validate(#{} = Body) -> % "by_node": { % "node1@xxx.xxx.xxx.xxx": ["00000000-1fffffff",...] % ]} - maps:map(fun(Node, Ranges) -> - validate_by_node(Node, Ranges, ByRange) - end, ByNode), + maps:map( + fun(Node, Ranges) -> + validate_by_node(Node, Ranges, ByRange) + end, + ByNode + ), % "by_range": { % "00000000-1fffffff": ["node1@xxx.xxx.xxx.xxx", ...] % ]} - maps:map(fun(Range, Nodes) -> - validate_by_range(Range, Nodes, ByNode) - end, ByRange). - + maps:map( + fun(Range, Nodes) -> + validate_by_range(Range, Nodes, ByNode) + end, + ByRange + ). validate_by_node(Node, Ranges, ByRange) -> validate_array(Ranges, ["by_node", Ranges, "value not an array"]), - lists:foreach(fun(Range) -> - validate_key(Range, ByRange, ["by_range for", Range, "missing"]), - Nodes = maps:get(Range, ByRange), - validate_member(Node, Nodes, ["by_range for", Range, "missing", Node]) - end, Ranges). - + lists:foreach( + fun(Range) -> + validate_key(Range, ByRange, ["by_range for", Range, "missing"]), + Nodes = maps:get(Range, ByRange), + validate_member(Node, Nodes, ["by_range for", Range, "missing", Node]) + end, + Ranges + ). validate_by_range(Range, Nodes, ByNode) -> validate_array(Nodes, ["by_range", Nodes, "value not an array"]), - lists:foreach(fun(Node) -> - validate_key(Node, ByNode, ["by_node for", Node, "missing"]), - Ranges = maps:get(Node, ByNode), - validate_member(Range, Ranges, ["by_node for", Node, "missing", Range]) - end, Nodes). - + lists:foreach( + fun(Node) -> + validate_key(Node, ByNode, ["by_node for", Node, "missing"]), + Ranges = maps:get(Node, ByNode), + validate_member(Range, Ranges, ["by_node for", Node, "missing", Range]) + end, + Nodes + ). validate_array(Val, _ErrMsg) when is_list(Val) -> ok; validate_array(_Val, ErrMsg) -> throw({forbidden, errmsg(ErrMsg)}). - validate_key(Key, #{} = Map, ErrMsg) -> case maps:is_key(Key, Map) of true -> ok; false -> throw({forbidden, errmsg(ErrMsg)}) end. - validate_member(Val, Array, ErrMsg) when is_list(Array) -> case lists:member(Val, Array) of true -> ok; @@ -107,6 +107,5 @@ validate_member(Val, Array, ErrMsg) when is_list(Array) -> validate_member(_Val, _Array, ErrMsg) -> throw({forbidden, errmsg(ErrMsg)}). - errmsg(ErrMsg) when is_list(ErrMsg) -> list_to_binary(lists:join(" ", ErrMsg)). diff --git a/src/mem3/src/mem3_cluster.erl b/src/mem3/src/mem3_cluster.erl index 7e3d477cb..974b2cbef 100644 --- a/src/mem3/src/mem3_cluster.erl +++ b/src/mem3/src/mem3_cluster.erl @@ -25,7 +25,6 @@ % That can be configured via the StartPeriod argument. If the time since start % is less than a full period, then the StartPeriod is used as the period. - -module(mem3_cluster). -behaviour(gen_server). @@ -44,11 +43,9 @@ code_change/3 ]). - -callback cluster_stable(Context :: term()) -> NewContext :: term(). -callback cluster_unstable(Context :: term()) -> NewContext :: term(). - -record(state, { mod :: atom(), ctx :: term(), @@ -59,19 +56,17 @@ timer :: reference() }). - -spec start_link(module(), term(), integer(), integer()) -> {ok, pid()} | ignore | {error, term()}. -start_link(Module, Context, StartPeriod, Period) - when is_atom(Module), is_integer(StartPeriod), is_integer(Period) -> +start_link(Module, Context, StartPeriod, Period) when + is_atom(Module), is_integer(StartPeriod), is_integer(Period) +-> gen_server:start_link(?MODULE, [Module, Context, StartPeriod, Period], []). - -spec set_period(pid(), integer()) -> ok. set_period(Server, Period) when is_pid(Server), is_integer(Period) -> gen_server:cast(Server, {set_period, Period}). - % gen_server callbacks init([Module, Context, StartPeriod, Period]) -> @@ -84,8 +79,7 @@ init([Module, Context, StartPeriod, Period]) -> period = Period, start_period = StartPeriod, timer = new_timer(StartPeriod) - }}. - + }}. terminate(_Reason, _State) -> ok. @@ -93,32 +87,26 @@ terminate(_Reason, _State) -> handle_call(_Msg, _From, State) -> {reply, ignored, State}. - handle_cast({set_period, Period}, State) -> {noreply, State#state{period = Period}}. - handle_info({nodeup, _Node}, State) -> {noreply, cluster_changed(State)}; - handle_info({nodedown, _Node}, State) -> {noreply, cluster_changed(State)}; - handle_info(stability_check, #state{mod = Mod, ctx = Ctx} = State) -> - erlang:cancel_timer(State#state.timer), - case now_diff_sec(State#state.last_change) > interval(State) of - true -> - {noreply, State#state{ctx = Mod:cluster_stable(Ctx)}}; - false -> - Timer = new_timer(interval(State)), - {noreply, State#state{timer = Timer}} - end. - + erlang:cancel_timer(State#state.timer), + case now_diff_sec(State#state.last_change) > interval(State) of + true -> + {noreply, State#state{ctx = Mod:cluster_stable(Ctx)}}; + false -> + Timer = new_timer(interval(State)), + {noreply, State#state{timer = Timer}} + end. code_change(_OldVsn, State, _Extra) -> {ok, State}. - %% Internal functions -spec cluster_changed(#state{}) -> #state{}. @@ -129,18 +117,19 @@ cluster_changed(#state{mod = Mod, ctx = Ctx} = State) -> ctx = Mod:cluster_unstable(Ctx) }. - -spec new_timer(non_neg_integer()) -> reference(). new_timer(IntervalSec) -> erlang:send_after(IntervalSec * 1000, self(), stability_check). - % For the first Period seconds after node boot we check cluster stability every % StartPeriod seconds. Once the initial Period seconds have passed we continue % to monitor once every Period seconds -spec interval(#state{}) -> non_neg_integer(). -interval(#state{period = Period, start_period = StartPeriod, - start_time = T0}) -> +interval(#state{ + period = Period, + start_period = StartPeriod, + start_time = T0 +}) -> case now_diff_sec(T0) > Period of true -> % Normal operation @@ -150,12 +139,11 @@ interval(#state{period = Period, start_period = StartPeriod, StartPeriod end. - -spec now_diff_sec(erlang:timestamp()) -> non_neg_integer(). now_diff_sec(Time) -> case timer:now_diff(os:timestamp(), Time) of USec when USec < 0 -> 0; USec when USec >= 0 -> - USec / 1000000 + USec / 1000000 end. diff --git a/src/mem3/src/mem3_epi.erl b/src/mem3/src/mem3_epi.erl index 4bf2bf5d2..7bfc74dcf 100644 --- a/src/mem3/src/mem3_epi.erl +++ b/src/mem3/src/mem3_epi.erl @@ -10,7 +10,6 @@ % License for the specific language governing permissions and limitations under % the License. - -module(mem3_epi). -behaviour(couch_epi_plugin). @@ -34,7 +33,6 @@ providers() -> {chttpd_handlers, mem3_httpd_handlers} ]. - services() -> []. diff --git a/src/mem3/src/mem3_hash.erl b/src/mem3/src/mem3_hash.erl index 665c61cb1..ccaab7223 100644 --- a/src/mem3/src/mem3_hash.erl +++ b/src/mem3/src/mem3_hash.erl @@ -20,50 +20,41 @@ crc32/1 ]). - -include_lib("mem3/include/mem3.hrl"). - calculate(#shard{opts = Opts}, DocId) -> Props = couch_util:get_value(props, Opts, []), MFA = get_hash_fun_int(Props), calculate(MFA, DocId); - calculate(#ordered_shard{opts = Opts}, DocId) -> Props = couch_util:get_value(props, Opts, []), MFA = get_hash_fun_int(Props), calculate(MFA, DocId); - calculate(DbName, DocId) when is_binary(DbName) -> MFA = get_hash_fun(DbName), calculate(MFA, DocId); - calculate({Mod, Fun, Args}, DocId) -> erlang:apply(Mod, Fun, [DocId | Args]). - get_hash_fun(#shard{opts = Opts}) -> get_hash_fun_int(Opts); - get_hash_fun(#ordered_shard{opts = Opts}) -> get_hash_fun_int(Opts); - get_hash_fun(DbName0) when is_binary(DbName0) -> DbName = mem3:dbname(DbName0), try - [#shard{opts=Opts} | _] = mem3_shards:for_db(DbName), + [#shard{opts = Opts} | _] = mem3_shards:for_db(DbName), get_hash_fun_int(couch_util:get_value(props, Opts, [])) - catch error:database_does_not_exist -> - {?MODULE, crc32, []} + catch + error:database_does_not_exist -> + {?MODULE, crc32, []} end. - crc32(Item) when is_binary(Item) -> erlang:crc32(Item); crc32(Item) -> erlang:crc32(term_to_binary(Item)). - get_hash_fun_int(Opts) when is_list(Opts) -> case lists:keyfind(hash, 1, Opts) of {hash, [Mod, Fun, Args]} -> diff --git a/src/mem3/src/mem3_httpd.erl b/src/mem3/src/mem3_httpd.erl index 3df7e1876..745fe815c 100644 --- a/src/mem3/src/mem3_httpd.erl +++ b/src/mem3/src/mem3_httpd.erl @@ -12,57 +12,90 @@ -module(mem3_httpd). --export([handle_membership_req/1, handle_shards_req/2, - handle_sync_req/2]). +-export([ + handle_membership_req/1, + handle_shards_req/2, + handle_sync_req/2 +]). %% includes -include_lib("mem3/include/mem3.hrl"). -include_lib("couch/include/couch_db.hrl"). - -handle_membership_req(#httpd{method='GET', - path_parts=[<<"_membership">>]} = Req) -> - ClusterNodes = try mem3:nodes() - catch _:_ -> {ok,[]} end, - couch_httpd:send_json(Req, {[ - {all_nodes, lists:sort([node()|nodes()])}, - {cluster_nodes, lists:sort(ClusterNodes)} - ]}); -handle_membership_req(#httpd{path_parts=[<<"_membership">>]}=Req) -> +handle_membership_req( + #httpd{ + method = 'GET', + path_parts = [<<"_membership">>] + } = Req +) -> + ClusterNodes = + try + mem3:nodes() + catch + _:_ -> {ok, []} + end, + couch_httpd:send_json( + Req, + {[ + {all_nodes, lists:sort([node() | nodes()])}, + {cluster_nodes, lists:sort(ClusterNodes)} + ]} + ); +handle_membership_req(#httpd{path_parts = [<<"_membership">>]} = Req) -> chttpd:send_method_not_allowed(Req, "GET"). -handle_shards_req(#httpd{method='GET', - path_parts=[_DbName, <<"_shards">>]} = Req, Db) -> +handle_shards_req( + #httpd{ + method = 'GET', + path_parts = [_DbName, <<"_shards">>] + } = Req, + Db +) -> DbName = mem3:dbname(couch_db:name(Db)), Shards = mem3:shards(DbName), JsonShards = json_shards(Shards, dict:new()), - couch_httpd:send_json(Req, {[ - {shards, JsonShards} - ]}); -handle_shards_req(#httpd{method='GET', - path_parts=[_DbName, <<"_shards">>, DocId]} = Req, Db) -> + couch_httpd:send_json( + Req, + {[ + {shards, JsonShards} + ]} + ); +handle_shards_req( + #httpd{ + method = 'GET', + path_parts = [_DbName, <<"_shards">>, DocId] + } = Req, + Db +) -> DbName = mem3:dbname(couch_db:name(Db)), Shards = mem3:shards(DbName, DocId), {[{Shard, Dbs}]} = json_shards(Shards, dict:new()), - couch_httpd:send_json(Req, {[ - {range, Shard}, - {nodes, Dbs} - ]}); -handle_shards_req(#httpd{path_parts=[_DbName, <<"_shards">>]}=Req, _Db) -> + couch_httpd:send_json( + Req, + {[ + {range, Shard}, + {nodes, Dbs} + ]} + ); +handle_shards_req(#httpd{path_parts = [_DbName, <<"_shards">>]} = Req, _Db) -> chttpd:send_method_not_allowed(Req, "GET"); -handle_shards_req(#httpd{path_parts=[_DbName, <<"_shards">>, _DocId]}=Req, _Db) -> +handle_shards_req(#httpd{path_parts = [_DbName, <<"_shards">>, _DocId]} = Req, _Db) -> chttpd:send_method_not_allowed(Req, "GET"). -handle_sync_req(#httpd{method='POST', - path_parts=[_DbName, <<"_sync_shards">>]} = Req, Db) -> +handle_sync_req( + #httpd{ + method = 'POST', + path_parts = [_DbName, <<"_sync_shards">>] + } = Req, + Db +) -> DbName = mem3:dbname(couch_db:name(Db)), ShardList = [S#shard.name || S <- mem3:ushards(DbName)], - [ sync_shard(S) || S <- ShardList ], + [sync_shard(S) || S <- ShardList], chttpd:send_json(Req, 202, {[{ok, true}]}); handle_sync_req(Req, _) -> chttpd:send_method_not_allowed(Req, "POST"). - %% %% internal %% @@ -70,7 +103,7 @@ handle_sync_req(Req, _) -> json_shards([], AccIn) -> List = dict:to_list(AccIn), {lists:sort(List)}; -json_shards([#shard{node=Node, range=[B,E]} | Rest], AccIn) -> +json_shards([#shard{node = Node, range = [B, E]} | Rest], AccIn) -> HexBeg = couch_util:to_hex(<<B:32/integer>>), HexEnd = couch_util:to_hex(<<E:32/integer>>), Range = list_to_binary(HexBeg ++ "-" ++ HexEnd), @@ -78,7 +111,8 @@ json_shards([#shard{node=Node, range=[B,E]} | Rest], AccIn) -> sync_shard(ShardName) -> Shards = mem3_shards:for_shard_range(ShardName), - [rpc:call(S1#shard.node, mem3_sync, push, [S1, S2#shard.node]) || - S1 <- Shards, S2 <- Shards, S1 =/= S2], + [ + rpc:call(S1#shard.node, mem3_sync, push, [S1, S2#shard.node]) + || S1 <- Shards, S2 <- Shards, S1 =/= S2 + ], ok. - diff --git a/src/mem3/src/mem3_httpd_handlers.erl b/src/mem3/src/mem3_httpd_handlers.erl index 7dd6ab052..ca6893e98 100644 --- a/src/mem3/src/mem3_httpd_handlers.erl +++ b/src/mem3/src/mem3_httpd_handlers.erl @@ -19,7 +19,7 @@ url_handler(<<"_reshard">>) -> fun mem3_reshard_httpd:handle_reshard_req/1; url_handler(_) -> no_match. db_handler(<<"_shards">>) -> fun mem3_httpd:handle_shards_req/2; -db_handler(<<"_sync_shards">>) -> fun mem3_httpd:handle_sync_req/2; +db_handler(<<"_sync_shards">>) -> fun mem3_httpd:handle_sync_req/2; db_handler(_) -> no_match. design_handler(_) -> no_match. diff --git a/src/mem3/src/mem3_nodes.erl b/src/mem3/src/mem3_nodes.erl index dd5be1a72..b46b3bb64 100644 --- a/src/mem3/src/mem3_nodes.erl +++ b/src/mem3/src/mem3_nodes.erl @@ -13,8 +13,14 @@ -module(mem3_nodes). -behaviour(gen_server). -vsn(1). --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3]). +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3 +]). -export([start_link/0, get_nodelist/0, get_node_info/2]). @@ -28,16 +34,18 @@ start_link() -> get_nodelist() -> try - lists:sort([N || {N,_} <- ets:tab2list(?MODULE)]) - catch error:badarg -> - gen_server:call(?MODULE, get_nodelist) + lists:sort([N || {N, _} <- ets:tab2list(?MODULE)]) + catch + error:badarg -> + gen_server:call(?MODULE, get_nodelist) end. get_node_info(Node, Key) -> try couch_util:get_value(Key, ets:lookup_element(?MODULE, Node, 2)) - catch error:badarg -> - gen_server:call(?MODULE, {get_node_info, Node, Key}) + catch + error:badarg -> + gen_server:call(?MODULE, {get_node_info, Node, Key}) end. init([]) -> @@ -47,13 +55,15 @@ init([]) -> {ok, #state{changes_pid = Pid, update_seq = UpdateSeq}}. handle_call(get_nodelist, _From, State) -> - {reply, lists:sort([N || {N,_} <- ets:tab2list(?MODULE)]), State}; + {reply, lists:sort([N || {N, _} <- ets:tab2list(?MODULE)]), State}; handle_call({get_node_info, Node, Key}, _From, State) -> - Resp = try - couch_util:get_value(Key, ets:lookup_element(?MODULE, Node, 2)) - catch error:badarg -> - error - end, + Resp = + try + couch_util:get_value(Key, ets:lookup_element(?MODULE, Node, 2)) + catch + error:badarg -> + error + end, {reply, Resp, State}; handle_call({add_node, Node, NodeInfo}, _From, State) -> gen_event:notify(mem3_events, {add_node, Node}), @@ -69,22 +79,26 @@ handle_call(_Call, _From, State) -> handle_cast(_Msg, State) -> {noreply, State}. -handle_info({'DOWN', _, _, Pid, Reason}, #state{changes_pid=Pid} = State) -> +handle_info({'DOWN', _, _, Pid, Reason}, #state{changes_pid = Pid} = State) -> couch_log:notice("~p changes listener died ~p", [?MODULE, Reason]), StartSeq = State#state.update_seq, - Seq = case Reason of {seq, EndSeq} -> EndSeq; _ -> StartSeq end, + Seq = + case Reason of + {seq, EndSeq} -> EndSeq; + _ -> StartSeq + end, erlang:send_after(5000, self(), start_listener), {noreply, State#state{update_seq = Seq}}; handle_info(start_listener, #state{update_seq = Seq} = State) -> {NewPid, _} = spawn_monitor(fun() -> listen_for_changes(Seq) end), - {noreply, State#state{changes_pid=NewPid}}; + {noreply, State#state{changes_pid = NewPid}}; handle_info(_Info, State) -> {noreply, State}. terminate(_Reason, _State) -> ok. -code_change(_OldVsn, #state{}=State, _Extra) -> +code_change(_OldVsn, #state{} = State, _Extra) -> {ok, State}. %% internal functions @@ -100,10 +114,10 @@ initialize_nodelist() -> first_fold(#full_doc_info{id = <<"_design/", _/binary>>}, Acc) -> {ok, Acc}; -first_fold(#full_doc_info{deleted=true}, Acc) -> +first_fold(#full_doc_info{deleted = true}, Acc) -> {ok, Acc}; -first_fold(#full_doc_info{id=Id}=DocInfo, Db) -> - {ok, #doc{body={Props}}} = couch_db:open_doc(Db, DocInfo, [ejson_body]), +first_fold(#full_doc_info{id = Id} = DocInfo, Db) -> + {ok, #doc{body = {Props}}} = couch_db:open_doc(Db, DocInfo, [ejson_body]), ets:insert(?MODULE, {mem3_util:to_atom(Id), Props}), {ok, Db}. @@ -125,31 +139,39 @@ changes_callback({stop, EndSeq}, _) -> exit({seq, EndSeq}); changes_callback({change, {Change}, _}, _) -> Node = couch_util:get_value(<<"id">>, Change), - case Node of <<"_design/", _/binary>> -> ok; _ -> - case mem3_util:is_deleted(Change) of - false -> - {Props} = couch_util:get_value(doc, Change), - gen_server:call(?MODULE, {add_node, mem3_util:to_atom(Node), Props}); - true -> - gen_server:call(?MODULE, {remove_node, mem3_util:to_atom(Node)}) - end + case Node of + <<"_design/", _/binary>> -> + ok; + _ -> + case mem3_util:is_deleted(Change) of + false -> + {Props} = couch_util:get_value(doc, Change), + gen_server:call(?MODULE, {add_node, mem3_util:to_atom(Node), Props}); + true -> + gen_server:call(?MODULE, {remove_node, mem3_util:to_atom(Node)}) + end end, {ok, couch_util:get_value(<<"seq">>, Change)}; changes_callback(timeout, _) -> {ok, nil}. insert_if_missing(Db, Nodes) -> - Docs = lists:foldl(fun(Node, Acc) -> - case ets:lookup(?MODULE, Node) of - [_] -> - Acc; - [] -> - ets:insert(?MODULE, {Node, []}), - [#doc{id = couch_util:to_binary(Node)} | Acc] - end - end, [], Nodes), - if Docs =/= [] -> - {ok, _} = couch_db:update_docs(Db, Docs, []); - true -> - {ok, []} + Docs = lists:foldl( + fun(Node, Acc) -> + case ets:lookup(?MODULE, Node) of + [_] -> + Acc; + [] -> + ets:insert(?MODULE, {Node, []}), + [#doc{id = couch_util:to_binary(Node)} | Acc] + end + end, + [], + Nodes + ), + if + Docs =/= [] -> + {ok, _} = couch_db:update_docs(Db, Docs, []); + true -> + {ok, []} end. diff --git a/src/mem3/src/mem3_plugin_couch_db.erl b/src/mem3/src/mem3_plugin_couch_db.erl index 8cb5d7898..ca6a2e570 100644 --- a/src/mem3/src/mem3_plugin_couch_db.erl +++ b/src/mem3/src/mem3_plugin_couch_db.erl @@ -16,6 +16,5 @@ is_valid_purge_client/2 ]). - is_valid_purge_client(DbName, Props) -> mem3_rep:verify_purge_checkpoint(DbName, Props). diff --git a/src/mem3/src/mem3_rep.erl b/src/mem3/src/mem3_rep.erl index 2487e6a98..1ffc70fc1 100644 --- a/src/mem3/src/mem3_rep.erl +++ b/src/mem3/src/mem3_rep.erl @@ -12,7 +12,6 @@ -module(mem3_rep). - -export([ go/2, go/3, @@ -29,7 +28,6 @@ changes_enumerator/2 ]). - -include_lib("mem3/include/mem3.hrl"). -include_lib("couch/include/couch_db.hrl"). @@ -59,10 +57,8 @@ go(Source, Target) -> go(Source, Target, []). - go(DbName, Node, Opts) when is_binary(DbName), is_atom(Node) -> - go(#shard{name=DbName, node=node()}, #shard{name=DbName, node=Node}, Opts); - + go(#shard{name = DbName, node = node()}, #shard{name = DbName, node = Node}, Opts); go(#shard{} = Source, #shard{} = Target, Opts) -> case mem3:db_is_current(Source) of true -> @@ -71,23 +67,27 @@ go(#shard{} = Source, #shard{} = Target, Opts) -> % Database could have been recreated {error, missing_source} end; - go(#shard{} = Source, #{} = Targets0, Opts) when map_size(Targets0) > 0 -> Targets = maps:map(fun(_, T) -> #tgt{shard = T} end, Targets0), case couch_server:exists(Source#shard.name) of true -> sync_security(Source, Targets), - BatchSize = case proplists:get_value(batch_size, Opts) of - BS when is_integer(BS), BS > 0 -> BS; - _ -> 100 - end, - BatchCount = case proplists:get_value(batch_count, Opts) of - all -> all; - BC when is_integer(BC), BC > 0 -> BC; - _ -> 1 - end, - IncompleteRanges = config:get_boolean("mem3", "incomplete_ranges", - false), + BatchSize = + case proplists:get_value(batch_size, Opts) of + BS when is_integer(BS), BS > 0 -> BS; + _ -> 100 + end, + BatchCount = + case proplists:get_value(batch_count, Opts) of + all -> all; + BC when is_integer(BC), BC > 0 -> BC; + _ -> 1 + end, + IncompleteRanges = config:get_boolean( + "mem3", + "incomplete_ranges", + false + ), Filter = proplists:get_value(filter, Opts), Acc = #acc{ batch_size = BatchSize, @@ -102,99 +102,97 @@ go(#shard{} = Source, #{} = Targets0, Opts) when map_size(Targets0) > 0 -> {error, missing_source} end. - -go(#acc{source=Source, batch_count=BC}=Acc) -> +go(#acc{source = Source, batch_count = BC} = Acc) -> case couch_db:open(Source#shard.name, [?ADMIN_CTX]) of - {ok, Db} -> - Resp = try - HashFun = mem3_hash:get_hash_fun(couch_db:name(Db)), - repl(Acc#acc{db = Db, hashfun = HashFun}) - catch - error:{error, missing_source} -> - {error, missing_source}; - error:{not_found, no_db_file} -> - {error, missing_target} - after - couch_db:close(Db) - end, - case Resp of - {ok, P} when P > 0, BC == all -> - go(Acc); - {ok, P} when P > 0, BC > 1 -> - go(Acc#acc{batch_count=BC-1}); - Else -> - Else - end; - {not_found, no_db_file} -> - {error, missing_source} + {ok, Db} -> + Resp = + try + HashFun = mem3_hash:get_hash_fun(couch_db:name(Db)), + repl(Acc#acc{db = Db, hashfun = HashFun}) + catch + error:{error, missing_source} -> + {error, missing_source}; + error:{not_found, no_db_file} -> + {error, missing_target} + after + couch_db:close(Db) + end, + case Resp of + {ok, P} when P > 0, BC == all -> + go(Acc); + {ok, P} when P > 0, BC > 1 -> + go(Acc#acc{batch_count = BC - 1}); + Else -> + Else + end; + {not_found, no_db_file} -> + {error, missing_source} end. - make_local_id(Source, Target) -> make_local_id(Source, Target, undefined). - -make_local_id(#shard{node=SourceNode}, #shard{node=TargetNode}, Filter) -> +make_local_id(#shard{node = SourceNode}, #shard{node = TargetNode}, Filter) -> make_local_id(SourceNode, TargetNode, Filter); - make_local_id(SourceThing, TargetThing, F) when is_binary(F) -> S = local_id_hash(SourceThing), T = local_id_hash(TargetThing), <<"_local/shard-sync-", S/binary, "-", T/binary, F/binary>>; - make_local_id(SourceThing, TargetThing, Filter) -> S = local_id_hash(SourceThing), T = local_id_hash(TargetThing), F = filter_hash(Filter), <<"_local/shard-sync-", S/binary, "-", T/binary, F/binary>>. - filter_hash(Filter) when is_function(Filter) -> {new_uniq, Hash} = erlang:fun_info(Filter, new_uniq), B = couch_util:encodeBase64Url(Hash), <<"-", B/binary>>; - filter_hash(_) -> <<>>. - local_id_hash(Thing) -> couch_util:encodeBase64Url(couch_hash:md5_hash(term_to_binary(Thing))). - make_purge_id(SourceUUID, TargetUUID) -> <<"_local/purge-mem3-", SourceUUID/binary, "-", TargetUUID/binary>>. - verify_purge_checkpoint(DbName, Props) -> try Type = couch_util:get_value(<<"type">>, Props), - if Type =/= <<"internal_replication">> -> false; true -> - SourceBin = couch_util:get_value(<<"source">>, Props), - TargetBin = couch_util:get_value(<<"target">>, Props), - Range = couch_util:get_value(<<"range">>, Props), - - Source = binary_to_existing_atom(SourceBin, latin1), - Target = binary_to_existing_atom(TargetBin, latin1), - - try - Nodes = lists:foldl(fun(Shard, Acc) -> - case Shard#shard.range == Range of - true -> [Shard#shard.node | Acc]; - false -> Acc - end - end, [], mem3:shards(DbName)), - lists:member(Source, Nodes) andalso lists:member(Target, Nodes) - catch - error:database_does_not_exist -> - false - end + if + Type =/= <<"internal_replication">> -> + false; + true -> + SourceBin = couch_util:get_value(<<"source">>, Props), + TargetBin = couch_util:get_value(<<"target">>, Props), + Range = couch_util:get_value(<<"range">>, Props), + + Source = binary_to_existing_atom(SourceBin, latin1), + Target = binary_to_existing_atom(TargetBin, latin1), + + try + Nodes = lists:foldl( + fun(Shard, Acc) -> + case Shard#shard.range == Range of + true -> [Shard#shard.node | Acc]; + false -> Acc + end + end, + [], + mem3:shards(DbName) + ), + lists:member(Source, Nodes) andalso lists:member(Target, Nodes) + catch + error:database_does_not_exist -> + false + end end - catch _:_ -> - false + catch + _:_ -> + false end. - %% @doc Find and return the largest update_seq in SourceDb %% that the client has seen from TargetNode. %% @@ -209,35 +207,41 @@ verify_purge_checkpoint(DbName, Props) -> %% largest source_seq that has a target_seq =< TgtSeq. find_source_seq(SrcDb, TgtNode, TgtUUIDPrefix, TgtSeq) -> case find_repl_doc(SrcDb, TgtUUIDPrefix) of - {ok, TgtUUID, Doc} -> - SrcNode = atom_to_binary(node(), utf8), - find_source_seq_int(Doc, SrcNode, TgtNode, TgtUUID, TgtSeq); - {not_found, _} -> - couch_log:warning("~p find_source_seq repl doc not_found " - "src_db: ~p, tgt_node: ~p, tgt_uuid_prefix: ~p, tgt_seq: ~p", - [?MODULE, SrcDb, TgtNode, TgtUUIDPrefix, TgtSeq]), - 0 + {ok, TgtUUID, Doc} -> + SrcNode = atom_to_binary(node(), utf8), + find_source_seq_int(Doc, SrcNode, TgtNode, TgtUUID, TgtSeq); + {not_found, _} -> + couch_log:warning( + "~p find_source_seq repl doc not_found " + "src_db: ~p, tgt_node: ~p, tgt_uuid_prefix: ~p, tgt_seq: ~p", + [?MODULE, SrcDb, TgtNode, TgtUUIDPrefix, TgtSeq] + ), + 0 end. - -find_source_seq_int(#doc{body={Props}}, SrcNode0, TgtNode0, TgtUUID, TgtSeq) -> - SrcNode = case is_atom(SrcNode0) of - true -> atom_to_binary(SrcNode0, utf8); - false -> SrcNode0 - end, - TgtNode = case is_atom(TgtNode0) of - true -> atom_to_binary(TgtNode0, utf8); - false -> TgtNode0 - end, +find_source_seq_int(#doc{body = {Props}}, SrcNode0, TgtNode0, TgtUUID, TgtSeq) -> + SrcNode = + case is_atom(SrcNode0) of + true -> atom_to_binary(SrcNode0, utf8); + false -> SrcNode0 + end, + TgtNode = + case is_atom(TgtNode0) of + true -> atom_to_binary(TgtNode0, utf8); + false -> TgtNode0 + end, % This is split off purely for the ability to run unit tests % against this bit of code without requiring all sorts of mocks. {History} = couch_util:get_value(<<"history">>, Props, {[]}), SrcHistory = couch_util:get_value(SrcNode, History, []), - UseableHistory = lists:filter(fun({Entry}) -> - couch_util:get_value(<<"target_node">>, Entry) =:= TgtNode andalso - couch_util:get_value(<<"target_uuid">>, Entry) =:= TgtUUID andalso - couch_util:get_value(<<"target_seq">>, Entry) =< TgtSeq - end, SrcHistory), + UseableHistory = lists:filter( + fun({Entry}) -> + couch_util:get_value(<<"target_node">>, Entry) =:= TgtNode andalso + couch_util:get_value(<<"target_uuid">>, Entry) =:= TgtUUID andalso + couch_util:get_value(<<"target_seq">>, Entry) =< TgtSeq + end, + SrcHistory + ), % This relies on SrcHistory being ordered desceding by source % sequence. @@ -245,19 +249,21 @@ find_source_seq_int(#doc{body={Props}}, SrcNode0, TgtNode0, TgtUUID, TgtSeq) -> [{Entry} | _] -> couch_util:get_value(<<"source_seq">>, Entry); [] -> - couch_log:warning("~p find_source_seq_int nil useable history " + couch_log:warning( + "~p find_source_seq_int nil useable history " "src_node: ~p, tgt_node: ~p, tgt_uuid: ~p, tgt_seq: ~p, " "src_history: ~p", - [?MODULE, SrcNode, TgtNode, TgtUUID, TgtSeq, SrcHistory]), + [?MODULE, SrcNode, TgtNode, TgtUUID, TgtSeq, SrcHistory] + ), 0 end. - find_split_target_seq(TgtDb, SrcNode0, SrcUUIDPrefix, SrcSeq) -> - SrcNode = case is_atom(SrcNode0) of - true -> atom_to_binary(SrcNode0, utf8); - false -> SrcNode0 - end, + SrcNode = + case is_atom(SrcNode0) of + true -> atom_to_binary(SrcNode0, utf8); + false -> SrcNode0 + end, case find_split_target_seq_int(TgtDb, SrcNode, SrcUUIDPrefix) of {ok, [{BulkCopySeq, BulkCopySeq} | _]} when SrcSeq =< BulkCopySeq -> % Check if source sequence is at or below the initial bulk copy @@ -266,65 +272,77 @@ find_split_target_seq(TgtDb, SrcNode0, SrcUUIDPrefix, SrcSeq) -> % extra safety we assert that the initial source and target % sequences are the same value SrcSeq; - {ok, Seqs= [{_, _} | _]} -> + {ok, Seqs = [{_, _} | _]} -> % Pick the target sequence for the greatest source sequence that is % less than `SrcSeq`. case lists:takewhile(fun({Seq, _}) -> Seq < SrcSeq end, Seqs) of [] -> - couch_log:warning("~p find_split_target_seq target seq not found " - "tgt_db: ~p, src_uuid_prefix: ~p, src_seq: ~p", - [?MODULE, couch_db:name(TgtDb), SrcUUIDPrefix, SrcSeq]), + couch_log:warning( + "~p find_split_target_seq target seq not found " + "tgt_db: ~p, src_uuid_prefix: ~p, src_seq: ~p", + [?MODULE, couch_db:name(TgtDb), SrcUUIDPrefix, SrcSeq] + ), 0; [{_, _} | _] = Seqs1 -> {_, TSeq} = lists:last(Seqs1), TSeq end; {not_found, _} -> - couch_log:warning("~p find_split_target_seq target seq not found " + couch_log:warning( + "~p find_split_target_seq target seq not found " "tgt_db: ~p, src_uuid_prefix: ~p, src_seq: ~p", - [?MODULE, couch_db:name(TgtDb), SrcUUIDPrefix, SrcSeq]), + [?MODULE, couch_db:name(TgtDb), SrcUUIDPrefix, SrcSeq] + ), 0 end. - repl(#acc{db = Db0} = Acc0) -> erlang:put(io_priority, {internal_repl, couch_db:name(Db0)}), Acc1 = calculate_start_seq_multi(Acc0), try - Acc3 = case config:get_boolean("mem3", "replicate_purges", false) of - true -> - Acc2 = pull_purges_multi(Acc1), - push_purges_multi(Acc2); - false -> - Acc1 - end, + Acc3 = + case config:get_boolean("mem3", "replicate_purges", false) of + true -> + Acc2 = pull_purges_multi(Acc1), + push_purges_multi(Acc2); + false -> + Acc1 + end, push_changes(Acc3) catch throw:{finished, Count} -> {ok, Count} end. - pull_purges_multi(#acc{source = Source} = Acc0) -> #acc{batch_size = Count, seq = UpdateSeq, targets = Targets0} = Acc0, with_src_db(Acc0, fun(Db) -> - Targets = maps:map(fun(_, #tgt{} = T) -> - pull_purges(Db, Count, Source, T) - end, reset_remaining(Targets0)), - Remaining = maps:fold(fun(_, #tgt{remaining = R}, Sum) -> - Sum + R - end, 0, Targets), - if Remaining == 0 -> Acc0#acc{targets = Targets}; true -> - PurgeSeq = couch_db:get_purge_seq(Db), - OldestPurgeSeq = couch_db:get_oldest_purge_seq(Db), - PurgesToPush = PurgeSeq - OldestPurgeSeq, - Changes = couch_db:count_changes_since(Db, UpdateSeq), - Pending = Remaining + PurgesToPush + Changes, - throw({finished, Pending}) + Targets = maps:map( + fun(_, #tgt{} = T) -> + pull_purges(Db, Count, Source, T) + end, + reset_remaining(Targets0) + ), + Remaining = maps:fold( + fun(_, #tgt{remaining = R}, Sum) -> + Sum + R + end, + 0, + Targets + ), + if + Remaining == 0 -> + Acc0#acc{targets = Targets}; + true -> + PurgeSeq = couch_db:get_purge_seq(Db), + OldestPurgeSeq = couch_db:get_oldest_purge_seq(Db), + PurgesToPush = PurgeSeq - OldestPurgeSeq, + Changes = couch_db:count_changes_since(Db, UpdateSeq), + Pending = Remaining + PurgesToPush + Changes, + throw({finished, Pending}) end end). - pull_purges(Db, Count, SrcShard, #tgt{} = Tgt0) -> #tgt{shard = TgtShard} = Tgt0, SrcUUID = couch_db:get_uuid(Db), @@ -332,76 +350,101 @@ pull_purges(Db, Count, SrcShard, #tgt{} = Tgt0) -> {LocalPurgeId, Infos, ThroughSeq, Remaining} = mem3_rpc:load_purge_infos(TgtNode, TgtDbName, SrcUUID, Count), Tgt = Tgt0#tgt{purgeid = LocalPurgeId}, - if Infos == [] -> ok; true -> - {ok, _} = couch_db:purge_docs(Db, Infos, [replicated_edits]), - Body = purge_cp_body(SrcShard, TgtShard, ThroughSeq), - mem3_rpc:save_purge_checkpoint(TgtNode, TgtDbName, LocalPurgeId, Body) + if + Infos == [] -> + ok; + true -> + {ok, _} = couch_db:purge_docs(Db, Infos, [replicated_edits]), + Body = purge_cp_body(SrcShard, TgtShard, ThroughSeq), + mem3_rpc:save_purge_checkpoint(TgtNode, TgtDbName, LocalPurgeId, Body) end, Tgt#tgt{remaining = max(0, Remaining)}. - push_purges_multi(#acc{source = SrcShard} = Acc) -> #acc{batch_size = BatchSize, seq = UpdateSeq, targets = Targets0} = Acc, with_src_db(Acc, fun(Db) -> - Targets = maps:map(fun(_, #tgt{} = T) -> - push_purges(Db, BatchSize, SrcShard, T) - end, reset_remaining(Targets0)), - Remaining = maps:fold(fun(_, #tgt{remaining = R}, Sum) -> - Sum + R - end, 0, Targets), - if Remaining == 0 -> Acc#acc{targets = Targets}; true -> - Changes = couch_db:count_changes_since(Db, UpdateSeq), - throw({finished, Remaining + Changes}) + Targets = maps:map( + fun(_, #tgt{} = T) -> + push_purges(Db, BatchSize, SrcShard, T) + end, + reset_remaining(Targets0) + ), + Remaining = maps:fold( + fun(_, #tgt{remaining = R}, Sum) -> + Sum + R + end, + 0, + Targets + ), + if + Remaining == 0 -> + Acc#acc{targets = Targets}; + true -> + Changes = couch_db:count_changes_since(Db, UpdateSeq), + throw({finished, Remaining + Changes}) end end). - push_purges(Db, BatchSize, SrcShard, Tgt) -> #tgt{shard = TgtShard, purgeid = LocalPurgeId} = Tgt, #shard{node = TgtNode, name = TgtDbName} = TgtShard, - StartSeq = case couch_db:open_doc(Db, LocalPurgeId, []) of - {ok, #doc{body = {Props}}} -> - couch_util:get_value(<<"purge_seq">>, Props); - {not_found, _} -> - Oldest = couch_db:get_oldest_purge_seq(Db), - erlang:max(0, Oldest - 1) - end, + StartSeq = + case couch_db:open_doc(Db, LocalPurgeId, []) of + {ok, #doc{body = {Props}}} -> + couch_util:get_value(<<"purge_seq">>, Props); + {not_found, _} -> + Oldest = couch_db:get_oldest_purge_seq(Db), + erlang:max(0, Oldest - 1) + end, FoldFun = fun({PSeq, UUID, Id, Revs}, {Count, Infos, _}) -> NewCount = Count + length(Revs), NewInfos = [{UUID, Id, Revs} | Infos], - Status = if NewCount < BatchSize -> ok; true -> stop end, + Status = + if + NewCount < BatchSize -> ok; + true -> stop + end, {Status, {NewCount, NewInfos, PSeq}} end, InitAcc = {0, [], StartSeq}, {ok, {_, Infos, ThroughSeq}} = couch_db:fold_purge_infos(Db, StartSeq, FoldFun, InitAcc), - if Infos == [] -> ok; true -> - ok = purge_on_target(TgtNode, TgtDbName, Infos), - Body = purge_cp_body(SrcShard, TgtShard, ThroughSeq), - Doc = #doc{id = LocalPurgeId, body = Body}, - {ok, _} = couch_db:update_doc(Db, Doc, []) + if + Infos == [] -> + ok; + true -> + ok = purge_on_target(TgtNode, TgtDbName, Infos), + Body = purge_cp_body(SrcShard, TgtShard, ThroughSeq), + Doc = #doc{id = LocalPurgeId, body = Body}, + {ok, _} = couch_db:update_doc(Db, Doc, []) end, Tgt#tgt{remaining = max(0, couch_db:get_purge_seq(Db) - ThroughSeq)}. - calculate_start_seq_multi(#acc{} = Acc) -> #acc{db = Db, targets = Targets0, filter = Filter} = Acc, FilterHash = filter_hash(Filter), - Targets = maps:map(fun(_, #tgt{} = T) -> - calculate_start_seq(Db, FilterHash, T) - end, Targets0), + Targets = maps:map( + fun(_, #tgt{} = T) -> + calculate_start_seq(Db, FilterHash, T) + end, + Targets0 + ), % There will always be at least one target #tgt{seq = Seq0} = hd(maps:values(Targets)), - Seq = maps:fold(fun(_, #tgt{seq = S}, M) -> min(S, M) end, Seq0, Targets), + Seq = maps:fold(fun(_, #tgt{seq = S}, M) -> min(S, M) end, Seq0, Targets), Acc#acc{seq = Seq, targets = Targets}. - calculate_start_seq(Db, FilterHash, #tgt{shard = TgtShard} = Tgt) -> UUID = couch_db:get_uuid(Db), #shard{node = Node, name = Name} = TgtShard, - {NewDocId, Doc} = mem3_rpc:load_checkpoint(Node, Name, node(), UUID, - FilterHash), - #doc{id=FoundId, body={TProps}} = Doc, + {NewDocId, Doc} = mem3_rpc:load_checkpoint( + Node, + Name, + node(), + UUID, + FilterHash + ), + #doc{id = FoundId, body = {TProps}} = Doc, Tgt1 = Tgt#tgt{localid = NewDocId}, % NewDocId and FoundId may be different the first time % this code runs to save our newly named internal replication @@ -429,7 +472,6 @@ calculate_start_seq(Db, FilterHash, #tgt{shard = TgtShard} = Tgt) -> compare_epochs(Db, Tgt1) end. - push_changes(#acc{} = Acc0) -> #acc{ db = Db0, @@ -439,8 +481,9 @@ push_changes(#acc{} = Acc0) -> % Avoid needless rewriting the internal replication % checkpoint document if nothing is replicated. UpdateSeq = couch_db:get_update_seq(Db0), - if Seq < UpdateSeq -> ok; true -> - throw({finished, 0}) + if + Seq < UpdateSeq -> ok; + true -> throw({finished, 0}) end, with_src_db(Acc0, fun(Db) -> @@ -451,7 +494,6 @@ push_changes(#acc{} = Acc0) -> {ok, couch_db:count_changes_since(Db, LastSeq)} end). - compare_epochs(Db, #tgt{shard = TgtShard} = Tgt) -> #shard{node = Node, name = Name} = TgtShard, UUID = couch_db:get_uuid(Db), @@ -459,33 +501,44 @@ compare_epochs(Db, #tgt{shard = TgtShard} = Tgt) -> Seq = mem3_rpc:find_common_seq(Node, Name, UUID, Epochs), Tgt#tgt{seq = Seq, history = {[]}}. - -changes_enumerator(#doc_info{id=DocId}, #acc{db=Db}=Acc) -> +changes_enumerator(#doc_info{id = DocId}, #acc{db = Db} = Acc) -> {ok, FDI} = couch_db:get_full_doc_info(Db, DocId), changes_enumerator(FDI, Acc); -changes_enumerator(#full_doc_info{}=FDI, #acc{}=Acc0) -> +changes_enumerator(#full_doc_info{} = FDI, #acc{} = Acc0) -> #acc{ revcount = C, targets = Targets0, hashfun = HashFun, incomplete_ranges = IncompleteRanges } = Acc0, - #doc_info{high_seq=Seq, revs=Revs} = couch_doc:to_doc_info(FDI), - {Count, Targets} = case filter_doc(Acc0#acc.filter, FDI) of - keep -> - NewTargets = changes_append_fdi(FDI, Targets0, HashFun, - IncompleteRanges), - {C + length(Revs), NewTargets}; - discard -> - {C, Targets0} - end, + #doc_info{high_seq = Seq, revs = Revs} = couch_doc:to_doc_info(FDI), + {Count, Targets} = + case filter_doc(Acc0#acc.filter, FDI) of + keep -> + NewTargets = changes_append_fdi( + FDI, + Targets0, + HashFun, + IncompleteRanges + ), + {C + length(Revs), NewTargets}; + discard -> + {C, Targets0} + end, Acc1 = Acc0#acc{seq = Seq, revcount = Count, targets = Targets}, - Go = if Count < Acc1#acc.batch_size -> ok; true -> stop end, + Go = + if + Count < Acc1#acc.batch_size -> ok; + true -> stop + end, {Go, Acc1}. - -changes_append_fdi(#full_doc_info{id = Id} = FDI, Targets, HashFun, - IncompleteRanges) -> +changes_append_fdi( + #full_doc_info{id = Id} = FDI, + Targets, + HashFun, + IncompleteRanges +) -> case mem3_reshard_job:pickfun(Id, maps:keys(Targets), HashFun) of not_in_range when IncompleteRanges -> Targets; @@ -496,49 +549,61 @@ changes_append_fdi(#full_doc_info{id = Id} = FDI, Targets, HashFun, couch_log:error(ErrMsg, [?MODULE, Id, TNames]), error({error, {Id, not_in_target_ranges}}); Key -> - maps:update_with(Key, fun(#tgt{infos = Infos} = T) -> - T#tgt{infos = [FDI | Infos]} - end, Targets) + maps:update_with( + Key, + fun(#tgt{infos = Infos} = T) -> + T#tgt{infos = [FDI | Infos]} + end, + Targets + ) end. - replicate_batch_multi(#acc{targets = Targets0, seq = Seq, db = Db} = Acc) -> - Targets = maps:map(fun(_, #tgt{} = T) -> - replicate_batch(T, Db, Seq) - end, Targets0), + Targets = maps:map( + fun(_, #tgt{} = T) -> + replicate_batch(T, Db, Seq) + end, + Targets0 + ), {ok, Acc#acc{targets = Targets, revcount = 0}}. - replicate_batch(#tgt{shard = TgtShard, infos = Infos} = Target, Db, Seq) -> #shard{node = Node, name = Name} = TgtShard, case find_missing_revs(Target) of [] -> ok; Missing -> - lists:map(fun(Chunk) -> - Docs = open_docs(Db, Infos, Chunk), - ok = save_on_target(Node, Name, Docs) - end, chunk_revs(Missing)) + lists:map( + fun(Chunk) -> + Docs = open_docs(Db, Infos, Chunk), + ok = save_on_target(Node, Name, Docs) + end, + chunk_revs(Missing) + ) end, update_locals(Target, Db, Seq), Target#tgt{infos = []}. - find_missing_revs(#tgt{shard = TgtShard, infos = Infos}) -> #shard{node = Node, name = Name} = TgtShard, - IdsRevs = lists:map(fun(FDI) -> - #doc_info{id=Id, revs=RevInfos} = couch_doc:to_doc_info(FDI), - {Id, [R || #rev_info{rev=R} <- RevInfos]} - end, Infos), + IdsRevs = lists:map( + fun(FDI) -> + #doc_info{id = Id, revs = RevInfos} = couch_doc:to_doc_info(FDI), + {Id, [R || #rev_info{rev = R} <- RevInfos]} + end, + Infos + ), Missing = mem3_rpc:get_missing_revs(Node, Name, IdsRevs, [ {io_priority, {internal_repl, Name}}, ?ADMIN_CTX ]), - lists:filter(fun - ({_Id, [], _Ancestors}) -> false; - ({_Id, _Revs, _Ancestors}) -> true - end, Missing). - + lists:filter( + fun + ({_Id, [], _Ancestors}) -> false; + ({_Id, _Revs, _Ancestors}) -> true + end, + Missing + ). chunk_revs(Revs) -> Limit = list_to_integer(config:get("mem3", "rev_chunk_size", "5000")), @@ -548,34 +613,38 @@ chunk_revs(Revs, Limit) -> chunk_revs(Revs, {0, []}, [], Limit). chunk_revs([], {_Count, Chunk}, Chunks, _Limit) -> - [Chunk|Chunks]; -chunk_revs([{Id, R, A}|Revs], {Count, Chunk}, Chunks, Limit) when length(R) =< Limit - Count -> + [Chunk | Chunks]; +chunk_revs([{Id, R, A} | Revs], {Count, Chunk}, Chunks, Limit) when length(R) =< Limit - Count -> chunk_revs( Revs, - {Count + length(R), [{Id, R, A}|Chunk]}, + {Count + length(R), [{Id, R, A} | Chunk]}, Chunks, Limit ); -chunk_revs([{Id, R, A}|Revs], {Count, Chunk}, Chunks, Limit) -> +chunk_revs([{Id, R, A} | Revs], {Count, Chunk}, Chunks, Limit) -> {This, Next} = lists:split(Limit - Count, R), chunk_revs( - [{Id, Next, A}|Revs], + [{Id, Next, A} | Revs], {0, []}, - [[{Id, This, A}|Chunk]|Chunks], + [[{Id, This, A} | Chunk] | Chunks], Limit ). - open_docs(Db, Infos, Missing) -> - lists:flatmap(fun({Id, Revs, _}) -> - FDI = lists:keyfind(Id, #full_doc_info.id, Infos), - #full_doc_info{rev_tree=RevTree} = FDI, - {FoundRevs, _} = couch_key_tree:get_key_leafs(RevTree, Revs), - lists:map(fun({#leaf{deleted=IsDel, ptr=SummaryPtr}, FoundRevPath}) -> - couch_db:make_doc(Db, Id, IsDel, SummaryPtr, FoundRevPath) - end, FoundRevs) - end, Missing). - + lists:flatmap( + fun({Id, Revs, _}) -> + FDI = lists:keyfind(Id, #full_doc_info.id, Infos), + #full_doc_info{rev_tree = RevTree} = FDI, + {FoundRevs, _} = couch_key_tree:get_key_leafs(RevTree, Revs), + lists:map( + fun({#leaf{deleted = IsDel, ptr = SummaryPtr}, FoundRevPath}) -> + couch_db:make_doc(Db, Id, IsDel, SummaryPtr, FoundRevPath) + end, + FoundRevs + ) + end, + Missing + ). save_on_target(Node, Name, Docs) -> mem3_rpc:update_docs(Node, Name, Docs, [ @@ -586,7 +655,6 @@ save_on_target(Node, Name, Docs) -> ]), ok. - purge_on_target(Node, Name, PurgeInfos) -> mem3_rpc:purge_docs(Node, Name, PurgeInfos, [ replicated_changes, @@ -596,7 +664,6 @@ purge_on_target(Node, Name, PurgeInfos) -> ]), ok. - update_locals(Target, Db, Seq) -> #tgt{shard = TgtShard, localid = Id, history = History} = Target, #shard{node = Node, name = Name} = TgtShard, @@ -609,7 +676,6 @@ update_locals(Target, Db, Seq) -> NewBody = mem3_rpc:save_checkpoint(Node, Name, Id, Seq, NewEntry, History), {ok, _} = couch_db:update_doc(Db, #doc{id = Id, body = NewBody}, []). - purge_cp_body(#shard{} = Source, #shard{} = Target, PurgeSeq) -> {Mega, Secs, _} = os:timestamp(), NowSecs = Mega * 1000000 + Secs, @@ -622,7 +688,6 @@ purge_cp_body(#shard{} = Source, #shard{} = Target, PurgeSeq) -> {<<"range">>, Source#shard.range} ]}. - find_repl_doc(SrcDb, TgtUUIDPrefix) -> SrcUUID = couch_db:get_uuid(SrcDb), S = local_id_hash(SrcUUID), @@ -652,7 +717,6 @@ find_repl_doc(SrcDb, TgtUUIDPrefix) -> {not_found, missing} end. - find_split_target_seq_int(TgtDb, Node, SrcUUIDPrefix) -> TgtUUID = couch_db:get_uuid(TgtDb), FoldFun = fun(#doc{body = {Props}}, _) -> @@ -685,29 +749,27 @@ find_split_target_seq_int(TgtDb, Node, SrcUUIDPrefix) -> {not_found, missing} end. - % Get target sequences for each checkpoint when source replicated to the target % The "target" is the current db where the history entry was read from and "source" % is another, now possibly deleted, database. get_target_seqs([], _TgtUUID, _Node, _SrcUUIDPrefix, Acc) -> lists:reverse(Acc); - get_target_seqs([{Entry} | HProps], TgtUUID, Node, SrcUUIDPrefix, Acc) -> SameTgt = couch_util:get_value(<<"target_uuid">>, Entry) =:= TgtUUID, SameNode = couch_util:get_value(<<"target_node">>, Entry) =:= Node, SrcUUID = couch_util:get_value(<<"source_uuid">>, Entry), IsPrefix = is_prefix(SrcUUIDPrefix, SrcUUID), - Acc1 = case SameTgt andalso SameNode andalso IsPrefix of - true -> - EntrySourceSeq = couch_util:get_value(<<"source_seq">>, Entry), - EntryTargetSeq = couch_util:get_value(<<"target_seq">>, Entry), - [{EntrySourceSeq, EntryTargetSeq} | Acc]; - false -> - Acc - end, + Acc1 = + case SameTgt andalso SameNode andalso IsPrefix of + true -> + EntrySourceSeq = couch_util:get_value(<<"source_seq">>, Entry), + EntryTargetSeq = couch_util:get_value(<<"target_seq">>, Entry), + [{EntrySourceSeq, EntryTargetSeq} | Acc]; + false -> + Acc + end, get_target_seqs(HProps, TgtUUID, Node, SrcUUIDPrefix, Acc1). - with_src_db(#acc{source = Source}, Fun) -> case couch_db:open(Source#shard.name, [?ADMIN_CTX]) of {ok, Db} -> @@ -716,34 +778,36 @@ with_src_db(#acc{source = Source}, Fun) -> after couch_db:close(Db) end; - {not_found, _} -> + {not_found, _} -> error({error, missing_source}) end. - is_prefix(Prefix, Subject) -> binary:longest_common_prefix([Prefix, Subject]) == size(Prefix). - filter_doc(Filter, FullDocInfo) when is_function(Filter) -> try Filter(FullDocInfo) of discard -> discard; _ -> keep - catch _:_ -> - keep + catch + _:_ -> + keep end; filter_doc(_, _) -> keep. - sync_security(#shard{} = Source, #{} = Targets) -> - maps:map(fun(_, #tgt{shard = Target}) -> - mem3_sync_security:maybe_sync(Source, Target) - end, Targets). - + maps:map( + fun(_, #tgt{shard = Target}) -> + mem3_sync_security:maybe_sync(Source, Target) + end, + Targets + ). -targets_map(#shard{name = <<"shards/", _/binary>> = SrcName} = Src, - #shard{name = <<"shards/", _/binary>>, node = TgtNode} = Tgt) -> +targets_map( + #shard{name = <<"shards/", _/binary>> = SrcName} = Src, + #shard{name = <<"shards/", _/binary>>, node = TgtNode} = Tgt +) -> % Parse range from name in case the passed shard is built with a name only SrcRange = mem3:range(SrcName), Shards0 = mem3:shards(mem3:dbname(SrcName)), @@ -756,41 +820,35 @@ targets_map(#shard{name = <<"shards/", _/binary>> = SrcName} = Src, % moving / copying shards using mem3:go/2,3 before the % shards are present in the shard map #{mem3:range(SrcName) => Tgt}; - [_ | _] = TMapList-> + [_ | _] = TMapList -> maps:from_list(TMapList) end; - - targets_map(_Src, Tgt) -> #{[0, ?RING_END] => Tgt}. - shard_eq(#shard{name = Name, node = Node}, #shard{name = Name, node = Node}) -> true; - shard_eq(_, _) -> false. - check_overlap(SrcRange, Node, #shard{node = Node, range = TgtRange}) -> mem3_util:range_overlap(SrcRange, TgtRange); - check_overlap([_, _], _, #shard{}) -> false. - reset_remaining(#{} = Targets) -> - maps:map(fun(_, #tgt{} = T) -> - T#tgt{remaining = 0} - end, Targets). - + maps:map( + fun(_, #tgt{} = T) -> + T#tgt{remaining = 0} + end, + Targets + ). -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). -define(TDEF(A), {atom_to_list(A), fun A/0}). - find_source_seq_int_test_() -> { setup, @@ -805,42 +863,36 @@ find_source_seq_int_test_() -> ] }. - t_unknown_node() -> ?assertEqual( find_source_seq_int(doc_(), <<"foo">>, <<"bing">>, <<"bar_uuid">>, 10), 0 ). - t_unknown_uuid() -> ?assertEqual( find_source_seq_int(doc_(), <<"foo">>, <<"bar">>, <<"teapot">>, 10), 0 ). - t_ok() -> ?assertEqual( find_source_seq_int(doc_(), <<"foo">>, <<"bar">>, <<"bar_uuid">>, 100), 100 ). - t_old_ok() -> ?assertEqual( find_source_seq_int(doc_(), <<"foo">>, <<"bar">>, <<"bar_uuid">>, 84), 50 ). - t_different_node() -> ?assertEqual( find_source_seq_int(doc_(), <<"foo2">>, <<"bar">>, <<"bar_uuid">>, 92), 31 ). - -define(SNODE, <<"source_node">>). -define(SUUID, <<"source_uuid">>). -define(SSEQ, <<"source_seq">>). @@ -851,49 +903,81 @@ t_different_node() -> doc_() -> Foo_Bar = [ {[ - {?SNODE, <<"foo">>}, {?SUUID, <<"foo_uuid">>}, {?SSEQ, 100}, - {?TNODE, <<"bar">>}, {?TUUID, <<"bar_uuid">>}, {?TSEQ, 100} + {?SNODE, <<"foo">>}, + {?SUUID, <<"foo_uuid">>}, + {?SSEQ, 100}, + {?TNODE, <<"bar">>}, + {?TUUID, <<"bar_uuid">>}, + {?TSEQ, 100} ]}, {[ - {?SNODE, <<"foo">>}, {?SUUID, <<"foo_uuid">>}, {?SSEQ, 90}, - {?TNODE, <<"bar">>}, {?TUUID, <<"bar_uuid">>}, {?TSEQ, 85} + {?SNODE, <<"foo">>}, + {?SUUID, <<"foo_uuid">>}, + {?SSEQ, 90}, + {?TNODE, <<"bar">>}, + {?TUUID, <<"bar_uuid">>}, + {?TSEQ, 85} ]}, {[ - {?SNODE, <<"foo">>}, {?SUUID, <<"foo_uuid">>}, {?SSEQ, 50}, - {?TNODE, <<"bar">>}, {?TUUID, <<"bar_uuid">>}, {?TSEQ, 51} + {?SNODE, <<"foo">>}, + {?SUUID, <<"foo_uuid">>}, + {?SSEQ, 50}, + {?TNODE, <<"bar">>}, + {?TUUID, <<"bar_uuid">>}, + {?TSEQ, 51} ]}, {[ - {?SNODE, <<"foo">>}, {?SUUID, <<"foo_uuid">>}, {?SSEQ, 40}, - {?TNODE, <<"bar">>}, {?TUUID, <<"bar_uuid">>}, {?TSEQ, 45} + {?SNODE, <<"foo">>}, + {?SUUID, <<"foo_uuid">>}, + {?SSEQ, 40}, + {?TNODE, <<"bar">>}, + {?TUUID, <<"bar_uuid">>}, + {?TSEQ, 45} ]}, {[ - {?SNODE, <<"foo">>}, {?SUUID, <<"foo_uuid">>}, {?SSEQ, 2}, - {?TNODE, <<"bar">>}, {?TUUID, <<"bar_uuid">>}, {?TSEQ, 2} + {?SNODE, <<"foo">>}, + {?SUUID, <<"foo_uuid">>}, + {?SSEQ, 2}, + {?TNODE, <<"bar">>}, + {?TUUID, <<"bar_uuid">>}, + {?TSEQ, 2} ]} ], Foo2_Bar = [ {[ - {?SNODE, <<"foo2">>}, {?SUUID, <<"foo_uuid">>}, {?SSEQ, 100}, - {?TNODE, <<"bar">>}, {?TUUID, <<"bar_uuid">>}, {?TSEQ, 100} + {?SNODE, <<"foo2">>}, + {?SUUID, <<"foo_uuid">>}, + {?SSEQ, 100}, + {?TNODE, <<"bar">>}, + {?TUUID, <<"bar_uuid">>}, + {?TSEQ, 100} ]}, {[ - {?SNODE, <<"foo2">>}, {?SUUID, <<"foo_uuid">>}, {?SSEQ, 92}, - {?TNODE, <<"bar">>}, {?TUUID, <<"bar_uuid">>}, {?TSEQ, 93} + {?SNODE, <<"foo2">>}, + {?SUUID, <<"foo_uuid">>}, + {?SSEQ, 92}, + {?TNODE, <<"bar">>}, + {?TUUID, <<"bar_uuid">>}, + {?TSEQ, 93} ]}, {[ - {?SNODE, <<"foo2">>}, {?SUUID, <<"foo_uuid">>}, {?SSEQ, 31}, - {?TNODE, <<"bar">>}, {?TUUID, <<"bar_uuid">>}, {?TSEQ, 30} + {?SNODE, <<"foo2">>}, + {?SUUID, <<"foo_uuid">>}, + {?SSEQ, 31}, + {?TNODE, <<"bar">>}, + {?TUUID, <<"bar_uuid">>}, + {?TSEQ, 30} ]} ], - History = {[ - {<<"foo">>, Foo_Bar}, - {<<"foo2">>, Foo2_Bar} - ]}, + History = + {[ + {<<"foo">>, Foo_Bar}, + {<<"foo2">>, Foo2_Bar} + ]}, #doc{ - body={[{<<"history">>, History}]} + body = {[{<<"history">>, History}]} }. - targets_map_test_() -> { setup, @@ -908,11 +992,9 @@ targets_map_test_() -> ] }. - target_not_a_shard() -> ?_assertEqual(#{[0, ?RING_END] => <<"t">>}, targets_map(<<"s">>, <<"t">>)). - source_contained_in_target() -> ?_test(begin R07 = [16#00000000, 16#7fffffff], @@ -937,13 +1019,12 @@ source_contained_in_target() -> ?assertEqual(1, map_size(Map1)), ?assertMatch(#{R07 := #shard{node = 'n2'}}, Map1), - Tgt2 = #shard{name = TgtName1, node = 'n3'}, - Map2 = targets_map(Src1, Tgt2), + Tgt2 = #shard{name = TgtName1, node = 'n3'}, + Map2 = targets_map(Src1, Tgt2), ?assertEqual(1, map_size(Map2)), ?assertMatch(#{R0f := #shard{node = 'n3'}}, Map2) end). - multiple_targets() -> ?_test(begin R07 = [16#00000000, 16#7fffffff], @@ -968,7 +1049,6 @@ multiple_targets() -> ?assertMatch(#{R8f := #shard{node = 'n1'}}, Map) end). - uneven_overlap() -> ?_test(begin R04 = [16#00000000, 16#4fffffff], @@ -995,7 +1075,6 @@ uneven_overlap() -> ?assertMatch(#{R58 := #shard{node = 'n1'}}, Map) end). - target_not_in_shard_map() -> ?_test(begin R0f = [16#00000000, 16#ffffffff], diff --git a/src/mem3/src/mem3_reshard.erl b/src/mem3/src/mem3_reshard.erl index 620b1bc73..ec08c72cd 100644 --- a/src/mem3/src/mem3_reshard.erl +++ b/src/mem3/src/mem3_reshard.erl @@ -12,10 +12,8 @@ -module(mem3_reshard). - -behaviour(gen_server). - -export([ start_link/0, @@ -50,10 +48,8 @@ code_change/3 ]). - -include("mem3_reshard.hrl"). - -define(JOB_ID_VERSION, 1). -define(JOB_STATE_VERSION, 1). -define(DEFAULT_MAX_JOBS, 48). @@ -61,14 +57,12 @@ -define(JOB_PREFIX, <<"reshard-job-">>). -define(STATE_PREFIX, <<"reshard-state-">>). - %% Public API -spec start_link() -> {ok, pid()} | ignore | {error, term()}. start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - -spec start() -> ok | {error, any()}. start() -> case is_disabled() of @@ -76,7 +70,6 @@ start() -> false -> gen_server:call(?MODULE, start, infinity) end. - -spec stop(binary()) -> ok | {error, any()}. stop(Reason) -> case is_disabled() of @@ -84,15 +77,12 @@ stop(Reason) -> false -> gen_server:call(?MODULE, {stop, Reason}, infinity) end. - -spec start_split_job(#shard{} | binary()) -> {ok, binary()} | {error, term()}. start_split_job(#shard{} = Shard) -> start_split_job(Shard, 2); - start_split_job(ShardName) when is_binary(ShardName) -> start_split_job(shard_from_name(ShardName), 2). - -spec start_split_job(#shard{}, split()) -> {ok, binary()} | {error, any()}. start_split_job(#shard{} = Source, Split) -> case is_disabled() of @@ -100,7 +90,6 @@ start_split_job(#shard{} = Source, Split) -> false -> validate_and_start_job(Source, Split) end. - -spec stop_job(binary(), binary()) -> ok | {error, any()}. stop_job(JobId, Reason) when is_binary(JobId), is_binary(Reason) -> case is_disabled() of @@ -108,7 +97,6 @@ stop_job(JobId, Reason) when is_binary(JobId), is_binary(Reason) -> false -> gen_server:call(?MODULE, {stop_job, JobId, Reason}, infinity) end. - -spec resume_job(binary()) -> ok | {error, any()}. resume_job(JobId) when is_binary(JobId) -> case is_disabled() of @@ -116,7 +104,6 @@ resume_job(JobId) when is_binary(JobId) -> false -> gen_server:call(?MODULE, {resume_job, JobId}, infinity) end. - -spec remove_job(binary()) -> ok | {error, any()}. remove_job(JobId) when is_binary(JobId) -> case is_disabled() of @@ -124,20 +111,21 @@ remove_job(JobId) when is_binary(JobId) -> false -> gen_server:call(?MODULE, {remove_job, JobId}, infinity) end. - -spec get_state() -> {[_ | _]}. get_state() -> gen_server:call(?MODULE, get_state, infinity). - -spec jobs() -> [[tuple()]]. jobs() -> - ets:foldl(fun(Job, Acc) -> - Opts = [iso8601], - Props = mem3_reshard_store:job_to_ejson_props(Job, Opts), - [{Props} | Acc] - end, [], ?MODULE). - + ets:foldl( + fun(Job, Acc) -> + Opts = [iso8601], + Props = mem3_reshard_store:job_to_ejson_props(Job, Opts), + [{Props} | Acc] + end, + [], + ?MODULE + ). -spec job(job_id()) -> {ok, {[_ | _]}} | {error, not_found}. job(JobId) -> @@ -150,7 +138,6 @@ job(JobId) -> {error, not_found} end. - % Return true resharding is disabled in the application level settings -spec is_disabled() -> boolean(). is_disabled() -> @@ -160,19 +147,16 @@ is_disabled() -> _ -> false end. - % State reporting callbacks. Used by mem3_reshard_job module. -spec report(pid(), #job{}) -> ok. report(Server, #job{} = Job) when is_pid(Server) -> gen_server:cast(Server, {report, Job}). - -spec checkpoint(pid(), #job{}) -> ok. checkpoint(Server, #job{} = Job) -> couch_log:notice("~p checkpointing ~p ~p", [?MODULE, Server, jobfmt(Job)]), gen_server:cast(Server, {checkpoint, Job}). - % Utility functions used from other mem3_reshard modules -spec now_sec() -> non_neg_integer(). @@ -180,12 +164,10 @@ now_sec() -> {Mega, Sec, _Micro} = os:timestamp(), Mega * 1000000 + Sec. - -spec update_history(atom(), binary() | null, time_sec(), list()) -> list(). update_history(State, State, Ts, History) -> % State is the same as detail. Make the detail null to avoid duplication update_history(State, null, Ts, History); - update_history(State, Detail, Ts, History) -> % Reverse, so we can process the last event as the head using % head matches, then after append and trimming, reserse again @@ -194,22 +176,18 @@ update_history(State, Detail, Ts, History) -> TrimmedRev = lists:sublist(UpdatedRev, max_history()), lists:reverse(TrimmedRev). - -spec shard_from_name(binary()) -> #shard{}. -shard_from_name(<<"shards/", _:8/binary, "-", _:8/binary, "/", - Rest/binary>> = Shard) -> +shard_from_name(<<"shards/", _:8/binary, "-", _:8/binary, "/", Rest/binary>> = Shard) -> Range = mem3:range(Shard), [DbName, Suffix] = binary:split(Rest, <<".">>), build_shard(Range, DbName, Suffix). - % For debugging only -spec reset_state() -> ok. reset_state() -> gen_server:call(?MODULE, reset_state, infinity). - % Gen server functions init(_) -> @@ -235,14 +213,12 @@ init(_) -> gen_server:cast(self(), reload_jobs), {ok, State3}. - terminate(Reason, State) -> couch_log:notice("~p terminate ~p ~p", [?MODULE, Reason, statefmt(State)]), catch unlink(State#state.db_monitor), catch exit(State#state.db_monitor, kill), lists:foreach(fun(Job) -> kill_job_int(Job) end, running_jobs()). - handle_call(start, _From, #state{state = stopped} = State) -> State1 = State#state{ state = running, @@ -253,10 +229,8 @@ handle_call(start, _From, #state{state = stopped} = State) -> State2 = maybe_disable(State1), State3 = reload_jobs(State2), {reply, ok, State3}; - handle_call(start, _From, State) -> {reply, ok, State}; - handle_call({stop, Reason}, _From, #state{state = running} = State) -> State1 = State#state{ state = stopped, @@ -266,15 +240,13 @@ handle_call({stop, Reason}, _From, #state{state = running} = State) -> ok = mem3_reshard_store:store_state(State1), lists:foreach(fun(Job) -> temporarily_stop_job(Job) end, running_jobs()), {reply, ok, State1}; - handle_call({stop, _}, _From, State) -> {reply, ok, State}; - handle_call({start_job, #job{id = Id, source = Source} = Job}, _From, State) -> couch_log:notice("~p start_job call ~p", [?MODULE, jobfmt(Job)]), Total = ets:info(?MODULE, size), SourceOk = mem3_reshard_validate:source(Source), - case {job_by_id(Id), Total + 1 =< get_max_jobs(), SourceOk} of + case {job_by_id(Id), Total + 1 =< get_max_jobs(), SourceOk} of {not_found, true, ok} -> handle_start_job(Job, State); {#job{}, _, _} -> @@ -284,7 +256,6 @@ handle_call({start_job, #job{id = Id, source = Source} = Job}, _From, State) -> {_, _, {error, _} = SourceError} -> {reply, SourceError, State} end; - handle_call({resume_job, _}, _From, #state{state = stopped} = State) -> case couch_util:get_value(reason, State#state.state_info) of undefined -> @@ -292,7 +263,6 @@ handle_call({resume_job, _}, _From, #state{state = stopped} = State) -> Reason -> {reply, {error, {stopped, Reason}}, State} end; - handle_call({resume_job, Id}, _From, State) -> couch_log:notice("~p resume_job call ~p", [?MODULE, Id]), case job_by_id(Id) of @@ -308,12 +278,13 @@ handle_call({resume_job, Id}, _From, State) -> not_found -> {reply, {error, not_found}, State} end; - handle_call({stop_job, Id, Reason}, _From, State) -> couch_log:notice("~p stop_job Id:~p Reason:~p", [?MODULE, Id, Reason]), case job_by_id(Id) of - #job{job_state = JSt} = Job when JSt =:= running orelse JSt =:= new - orelse JSt =:= stopped -> + #job{job_state = JSt} = Job when + JSt =:= running orelse JSt =:= new orelse + JSt =:= stopped + -> ok = stop_job_int(Job, stopped, Reason, State), {reply, ok, State}; #job{} -> @@ -321,62 +292,57 @@ handle_call({stop_job, Id, Reason}, _From, State) -> not_found -> {reply, {error, not_found}, State} end; - handle_call({remove_job, Id}, _From, State) -> {reply, remove_job_int(Id, State), State}; - handle_call(get_state, _From, #state{state = GlobalState} = State) -> StateProps = mem3_reshard_store:state_to_ejson_props(State), - Stats0 = #{running => 0, completed => 0, failed => 0, stopped => 0}, - StateStats = ets:foldl(fun(#job{job_state = JS}, Acc) -> - % When jobs are disabled globally their state is not checkpointed as - % "stopped", but it stays as "running". But when returning the state we - % don't want to mislead and indicate that there are "N running jobs" - % when the global state is "stopped". - JS1 = case GlobalState =:= stopped andalso JS =:= running of - true -> stopped; - false -> JS + Stats0 = #{running => 0, completed => 0, failed => 0, stopped => 0}, + StateStats = ets:foldl( + fun(#job{job_state = JS}, Acc) -> + % When jobs are disabled globally their state is not checkpointed as + % "stopped", but it stays as "running". But when returning the state we + % don't want to mislead and indicate that there are "N running jobs" + % when the global state is "stopped". + JS1 = + case GlobalState =:= stopped andalso JS =:= running of + true -> stopped; + false -> JS + end, + Acc#{JS1 => maps:get(JS1, Acc, 0) + 1} end, - Acc#{JS1 => maps:get(JS1, Acc, 0) + 1} - end, Stats0, ?MODULE), + Stats0, + ?MODULE + ), Total = ets:info(?MODULE, size), StateStats1 = maps:to_list(StateStats) ++ [{total, Total}], Result = {lists:sort(StateProps ++ StateStats1)}, {reply, Result, State}; - handle_call(reset_state, _From, State) -> {reply, ok, reset_state(State)}; - handle_call(Call, From, State) -> couch_log:error("~p unknown call ~p from: ~p", [?MODULE, Call, From]), {noreply, State}. - handle_cast({db_deleted, DbName}, State) -> % Remove only completed jobs. Other running states would `fail` but % job result would stick around so users can inspect them. JobIds = jobs_by_db_and_state(DbName, completed), [remove_job_int(JobId, State) || JobId <- JobIds], {noreply, State}; - handle_cast({report, Job}, State) -> report_int(Job), {noreply, State}; - handle_cast({checkpoint, Job}, State) -> {noreply, checkpoint_int(Job, State)}; - handle_cast(reload_jobs, State) -> couch_log:notice("~p starting reloading jobs", [?MODULE]), State1 = reload_jobs(State), couch_log:notice("~p finished reloading jobs", [?MODULE]), {noreply, State1}; - handle_cast(Cast, State) -> couch_log:error("~p unexpected cast ~p", [?MODULE, Cast]), {noreply, State}. - handle_info({'DOWN', _Ref, process, Pid, Info}, State) -> case job_by_pid(Pid) of {ok, Job} -> @@ -386,16 +352,13 @@ handle_info({'DOWN', _Ref, process, Pid, Info}, State) -> couch_log:error("~p job not found: ~p ~p", [?MODULE, Pid, Info]) end, {noreply, State}; - handle_info(Info, State) -> couch_log:error("~p unexpected info ~p", [?MODULE, Info]), {noreply, State}. - code_change(_OldVsn, State, _Extra) -> {ok, State}. - %% Private API validate_and_start_job(#shard{} = Source, Split) -> @@ -425,7 +388,6 @@ validate_and_start_job(#shard{} = Source, Split) -> {error, Error} end. - handle_start_job(#job{} = Job, #state{state = running} = State) -> case start_job_int(Job, State) of ok -> @@ -433,7 +395,6 @@ handle_start_job(#job{} = Job, #state{state = running} = State) -> {error, Error} -> {reply, {error, Error}, State} end; - handle_start_job(#job{} = Job, #state{state = stopped} = State) -> ok = mem3_reshard_store:store_job(State, Job), % Since resharding is stopped on this node, the job is temporarily marked @@ -442,7 +403,6 @@ handle_start_job(#job{} = Job, #state{state = stopped} = State) -> temporarily_stop_job(Job), {reply, {ok, Job#job.id}, State}. - % Insert job in the ets table as a temporarily stopped job. This would happen % when a job is reloaded or added when node-wide resharding is stopped. -spec temporarily_stop_job(#job{}) -> #job{}. @@ -462,25 +422,24 @@ temporarily_stop_job(Job) -> true = ets:insert(?MODULE, Job3), Job3. - -spec reload_jobs(#state{}) -> #state{}. reload_jobs(State) -> Jobs = mem3_reshard_store:get_jobs(State), lists:foldl(fun reload_job/2, State, Jobs). - % This is a case when main application is stopped but a job is reloaded that % was checkpointed in running state. Set that state to stopped to avoid the API % results looking odd. -spec reload_job(#job{}, #state{}) -> #state{}. -reload_job(#job{job_state = JS} = Job, #state{state = stopped} = State) - when JS =:= running orelse JS =:= new -> +reload_job(#job{job_state = JS} = Job, #state{state = stopped} = State) when + JS =:= running orelse JS =:= new +-> temporarily_stop_job(Job), State; - % This is a case when a job process should be spawend -reload_job(#job{job_state = JS} = Job, #state{state = running} = State) - when JS =:= running orelse JS =:= new -> +reload_job(#job{job_state = JS} = Job, #state{state = running} = State) when + JS =:= running orelse JS =:= new +-> case start_job_int(Job, State) of ok -> State; @@ -489,20 +448,18 @@ reload_job(#job{job_state = JS} = Job, #state{state = running} = State) couch_log:error(Msg, [?MODULE, jobfmt(Job), Error]), State end; - % If job is disabled individually (stopped by the user), is completed or failed % then simply load it into the ets table -reload_job(#job{job_state = JS} = Job, #state{} = State) - when JS =:= failed orelse JS =:= completed orelse JS =:= stopped -> +reload_job(#job{job_state = JS} = Job, #state{} = State) when + JS =:= failed orelse JS =:= completed orelse JS =:= stopped +-> true = ets:insert(?MODULE, Job), State. - -spec get_max_jobs() -> integer(). get_max_jobs() -> config:get_integer("reshard", "max_jobs", ?DEFAULT_MAX_JOBS). - -spec start_job_int(#job{}, #state{}) -> ok | {error, term()}. start_job_int(Job, State) -> case spawn_job(Job) of @@ -515,7 +472,6 @@ start_job_int(Job, State) -> {error, Error} end. - -spec spawn_job(#job{}) -> {ok, pid()} | {error, term()}. spawn_job(#job{} = Job0) -> Job = Job0#job{ @@ -535,11 +491,14 @@ spawn_job(#job{} = Job0) -> {error, Reason} end. - -spec stop_job_int(#job{}, job_state(), term(), #state{}) -> ok. stop_job_int(#job{} = Job, JobState, Reason, State) -> - couch_log:info("~p stop_job_int ~p newstate: ~p reason:~p", [?MODULE, - jobfmt(Job), JobState, Reason]), + couch_log:info("~p stop_job_int ~p newstate: ~p reason:~p", [ + ?MODULE, + jobfmt(Job), + JobState, + Reason + ]), Job1 = kill_job_int(Job), Job2 = Job1#job{ job_state = JobState, @@ -551,11 +510,9 @@ stop_job_int(#job{} = Job, JobState, Reason, State) -> couch_log:info("~p stop_job_int stopped ~p", [?MODULE, jobfmt(Job2)]), ok. - -spec kill_job_int(#job{}) -> #job{}. kill_job_int(#job{pid = undefined} = Job) -> Job; - kill_job_int(#job{pid = Pid, ref = Ref} = Job) -> couch_log:info("~p kill_job_int ~p", [?MODULE, jobfmt(Job)]), demonitor(Ref, [flush]), @@ -569,7 +526,6 @@ kill_job_int(#job{pid = Pid, ref = Ref} = Job) -> true = ets:insert(?MODULE, Job1), Job1. - -spec handle_job_exit(#job{}, term(), #state{}) -> ok. handle_job_exit(#job{split_state = completed} = Job, normal, State) -> couch_log:notice("~p completed job ~s exited", [?MODULE, Job#job.id]), @@ -584,7 +540,6 @@ handle_job_exit(#job{split_state = completed} = Job, normal, State) -> ok = mem3_reshard_store:store_job(State, Job2), true = ets:insert(?MODULE, Job2), ok; - handle_job_exit(#job{job_state = running} = Job, normal, _State) -> couch_log:notice("~p running job ~s stopped", [?MODULE, Job#job.id]), OldInfo = Job#job.state_info, @@ -597,7 +552,6 @@ handle_job_exit(#job{job_state = running} = Job, normal, _State) -> }, true = ets:insert(?MODULE, update_job_history(Job1)), ok; - handle_job_exit(#job{job_state = running} = Job, shutdown, _State) -> couch_log:notice("~p job ~s shutdown", [?MODULE, Job#job.id]), OldInfo = Job#job.state_info, @@ -610,7 +564,6 @@ handle_job_exit(#job{job_state = running} = Job, shutdown, _State) -> }, true = ets:insert(?MODULE, update_job_history(Job1)), ok; - handle_job_exit(#job{job_state = running} = Job, {shutdown, Msg}, _State) -> couch_log:notice("~p job ~s shutdown ~p", [?MODULE, Job#job.id, Msg]), OldInfo = Job#job.state_info, @@ -623,7 +576,6 @@ handle_job_exit(#job{job_state = running} = Job, {shutdown, Msg}, _State) -> }, true = ets:insert(?MODULE, update_job_history(Job1)), ok; - handle_job_exit(#job{} = Job, Error, State) -> couch_log:notice("~p job ~s failed ~p", [?MODULE, Job#job.id, Error]), OldInfo = Job#job.state_info, @@ -639,7 +591,6 @@ handle_job_exit(#job{} = Job, Error, State) -> true = ets:insert(?MODULE, Job2), ok. - -spec job_by_id(job_id()) -> #job{} | not_found. job_by_id(Id) -> case ets:lookup(?MODULE, Id) of @@ -649,23 +600,20 @@ job_by_id(Id) -> Job end. - -spec job_by_pid(pid()) -> {ok, #job{}} | {error, not_found}. job_by_pid(Pid) when is_pid(Pid) -> - case ets:match_object(?MODULE, #job{pid=Pid, _='_'}) of + case ets:match_object(?MODULE, #job{pid = Pid, _ = '_'}) of [] -> {error, not_found}; [#job{} = Job] -> {ok, Job} end. - -spec state_id() -> binary(). state_id() -> Ver = iolist_to_binary(io_lib:format("~3..0B", [?JOB_STATE_VERSION])), <<?STATE_PREFIX/binary, Ver/binary>>. - -spec job_id(#job{}) -> binary(). job_id(#job{source = #shard{name = SourceName}}) -> HashInput = [SourceName, atom_to_binary(node(), utf8)], @@ -674,20 +622,22 @@ job_id(#job{source = #shard{name = SourceName}}) -> Prefix = iolist_to_binary(io_lib:format("~3..0B", [?JOB_ID_VERSION])), <<Prefix/binary, "-", IdHash/binary>>. - -spec target_shards(#shard{}, split()) -> [#shard{}]. target_shards(#shard{name = Name, range = [B, E], dbname = DbName}, Split) when - is_integer(Split), Split >= 2, (E - B + 1) >= Split -> + is_integer(Split), Split >= 2, (E - B + 1) >= Split +-> Ranges = target_ranges([B, E], Split), <<"shards/", _:8/binary, "-", _:8/binary, "/", DbAndSuffix/binary>> = Name, [DbName, Suffix] = binary:split(DbAndSuffix, <<".">>), [build_shard(R, DbName, Suffix) || R <- Ranges]. - -spec target_ranges([range_pos()], split()) -> [[range_pos()]]. -target_ranges([Begin, End], Split) when (End - Begin + 1) >= Split, - Split >=2 -> - Len = End - Begin + 1, % + 1 since intervals are inclusive +target_ranges([Begin, End], Split) when + (End - Begin + 1) >= Split, + Split >= 2 +-> + % + 1 since intervals are inclusive + Len = End - Begin + 1, NewLen = Len div Split, Rem = Len rem Split, Ranges = [[I, I + NewLen - 1] || I <- lists:seq(Begin, End - Rem, NewLen)], @@ -698,29 +648,24 @@ target_ranges([Begin, End], Split) when (End - Begin + 1) >= Split, {BeforeLast, [[BeginLast, _]]} = lists:split(Split - 1, Ranges), BeforeLast ++ [[BeginLast, End]]. - -spec build_shard([non_neg_integer()], binary(), binary()) -> #shard{}. build_shard(Range, DbName, Suffix) -> Shard = #shard{dbname = DbName, range = Range, node = node()}, mem3_util:name_shard(Shard, <<".", Suffix/binary>>). - -spec running_jobs() -> [#job{}]. running_jobs() -> Pat = #job{job_state = running, _ = '_'}, ets:match_object(?MODULE, Pat). - -spec info_update(atom(), any(), [tuple()]) -> [tuple()]. info_update(Key, Val, StateInfo) -> lists:keystore(Key, 1, StateInfo, {Key, Val}). - -spec info_delete(atom(), [tuple()]) -> [tuple()]. info_delete(Key, StateInfo) -> lists:keydelete(Key, 1, StateInfo). - -spec checkpoint_int(#job{}, #state{}) -> #state{}. checkpoint_int(#job{} = Job, State) -> couch_log:debug("~p checkpoint ~s", [?MODULE, jobfmt(Job)]), @@ -734,7 +679,6 @@ checkpoint_int(#job{} = Job, State) -> State end. - -spec report_int(#job{}) -> ok | not_found. report_int(Job) -> case ets:lookup(?MODULE, Job#job.id) of @@ -756,7 +700,6 @@ report_int(Job) -> not_found end. - -spec remove_job_int(#job{}, #state{}) -> ok | {error, not_found}. remove_job_int(Id, State) -> couch_log:notice("~p call remove_job Id:~p", [?MODULE, Id]), @@ -770,7 +713,6 @@ remove_job_int(Id, State) -> {error, not_found} end. - % This function is for testing and debugging only -spec reset_state(#state{}) -> #state{}. reset_state(#state{} = State) -> @@ -781,11 +723,15 @@ reset_state(#state{} = State) -> ets:delete_all_objects(?MODULE), couch_log:warning("~p resetting all job states", [?MODULE]), Jobs = mem3_reshard_store:get_jobs(State), - lists:foldl(fun(#job{id = Id}, StateAcc) -> - couch_log:warning("~p resetting job state ~p", [?MODULE, Id]), - ok = mem3_reshard_store:delete_job(StateAcc, Id), - StateAcc - end, State, Jobs), + lists:foldl( + fun(#job{id = Id}, StateAcc) -> + couch_log:warning("~p resetting job state ~p", [?MODULE, Id]), + ok = mem3_reshard_store:delete_job(StateAcc, Id), + StateAcc + end, + State, + Jobs + ), couch_log:warning("~p resetting state done", [?MODULE]), State#state{ state = running, @@ -793,38 +739,32 @@ reset_state(#state{} = State) -> update_time = now_sec() }. - -spec update_job_history(#job{}) -> #job{}. update_job_history(#job{job_state = St, update_time = Ts} = Job) -> Hist = Job#job.history, - Reason = case couch_util:get_value(reason, Job#job.state_info) of - undefined -> null; - Val -> couch_util:to_binary(Val) - end, + Reason = + case couch_util:get_value(reason, Job#job.state_info) of + undefined -> null; + Val -> couch_util:to_binary(Val) + end, Job#job{history = update_history(St, Reason, Ts, Hist)}. - update_history_rev(State, null, Ts, [{_, State, Detail} | Rest]) -> % Just updated the detail, state stays the same, no new entry added [{Ts, State, Detail} | Rest]; - update_history_rev(State, Detail, Ts, [{_, State, Detail} | Rest]) -> % State and detail were same as last event, just update the timestamp [{Ts, State, Detail} | Rest]; - update_history_rev(State, Detail, Ts, [{_, State, Detail} | Rest]) -> % State and detail were same as last event, just update the timestamp [{Ts, State, Detail} | Rest]; - update_history_rev(State, Detail, Ts, History) -> [{Ts, State, Detail} | History]. - -spec max_history() -> non_neg_integer(). max_history() -> config:get_integer("reshard", "max_history", ?DEFAULT_MAX_HISTORY). - -spec maybe_disable(#state{}) -> #state{}. maybe_disable(#state{} = State) -> case is_disabled() of @@ -839,19 +779,17 @@ maybe_disable(#state{} = State) -> State end. - -spec jobs_by_db_and_state(binary(), split_state() | '_') -> [job_id()]. jobs_by_db_and_state(Db, State) -> DbName = mem3:dbname(Db), Pat = #job{ id = '$1', - source =#shard{dbname = DbName, _ = '_'}, + source = #shard{dbname = DbName, _ = '_'}, job_state = State, _ = '_' }, [JobId || [JobId] <- ets:match(?MODULE, Pat)]. - -spec db_exists(binary()) -> boolean(). db_exists(Name) -> try @@ -862,7 +800,6 @@ db_exists(Name) -> false end. - -spec db_monitor(pid()) -> no_return(). db_monitor(Server) -> couch_log:notice("~p db monitor ~p starting", [?MODULE, self()]), @@ -870,7 +807,6 @@ db_monitor(Server) -> couch_event:register_all(self()), db_monitor_loop(Server, EvtRef). - -spec db_monitor_loop(pid(), reference()) -> no_return(). db_monitor_loop(Server, EvtRef) -> receive @@ -899,7 +835,6 @@ db_monitor_loop(Server, EvtRef) -> db_monitor_loop(Server, EvtRef) end. - -spec statefmt(#state{} | term()) -> string(). statefmt(#state{state = StateName}) -> Total = ets:info(?MODULE, size), @@ -907,12 +842,10 @@ statefmt(#state{state = StateName}) -> Msg = "#state{~s total:~B active:~B}", Fmt = io_lib:format(Msg, [StateName, Total, Active]), lists:flatten(Fmt); - statefmt(State) -> Fmt = io_lib:format("<Unknown split state:~p>", [State]), lists:flatten(Fmt). - -spec jobfmt(#job{}) -> string(). jobfmt(#job{} = Job) -> mem3_reshard_job:jobfmt(Job). diff --git a/src/mem3/src/mem3_reshard_api.erl b/src/mem3/src/mem3_reshard_api.erl index 0d3377db7..a4d395461 100644 --- a/src/mem3/src/mem3_reshard_api.erl +++ b/src/mem3/src/mem3_reshard_api.erl @@ -24,28 +24,28 @@ get_shard_splitting_state/0 ]). - create_jobs(Node, Shard, Db, Range, split) -> - lists:map(fun(S) -> - N = mem3:node(S), - Name = mem3:name(S), - case rpc:call(N, mem3_reshard, start_split_job, [Name]) of - {badrpc, Error} -> - {error, Error, N, Name}; - {ok, JobId} -> - {ok, JobId, N, Name}; - {error, Error} -> - {error, Error, N, Name} - end - end, pick_shards(Node, Shard, Db, Range)). - + lists:map( + fun(S) -> + N = mem3:node(S), + Name = mem3:name(S), + case rpc:call(N, mem3_reshard, start_split_job, [Name]) of + {badrpc, Error} -> + {error, Error, N, Name}; + {ok, JobId} -> + {ok, JobId, N, Name}; + {error, Error} -> + {error, Error, N, Name} + end + end, + pick_shards(Node, Shard, Db, Range) + ). get_jobs() -> Nodes = mem3_util:live_nodes(), {Replies, _Bad} = rpc:multicall(Nodes, mem3_reshard, jobs, []), lists:flatten(Replies). - get_job(JobId) -> Nodes = mem3_util:live_nodes(), {Replies, _Bad} = rpc:multicall(Nodes, mem3_reshard, job, [JobId]), @@ -56,26 +56,40 @@ get_job(JobId) -> {error, not_found} end. - get_summary() -> Nodes = mem3_util:live_nodes(), {Replies, _Bad} = rpc:multicall(Nodes, mem3_reshard, get_state, []), - Stats0 = #{running => 0, total => 0, completed => 0, failed => 0, - stopped => 0}, - StatsF = lists:foldl(fun({Res}, Stats) -> - maps:map(fun(Stat, OldVal) -> - OldVal + couch_util:get_value(Stat, Res, 0) - end, Stats) - end, Stats0, Replies), + Stats0 = #{ + running => 0, + total => 0, + completed => 0, + failed => 0, + stopped => 0 + }, + StatsF = lists:foldl( + fun({Res}, Stats) -> + maps:map( + fun(Stat, OldVal) -> + OldVal + couch_util:get_value(Stat, Res, 0) + end, + Stats + ) + end, + Stats0, + Replies + ), {State, Reason} = state_and_reason(Replies), StateReasonProps = [{state, State}, {state_reason, Reason}], {StateReasonProps ++ lists:sort(maps:to_list(StatsF))}. - resume_job(JobId) -> Nodes = mem3_util:live_nodes(), - {Replies, _Bad} = rpc:multicall(Nodes, mem3_reshard, resume_job, - [JobId]), + {Replies, _Bad} = rpc:multicall( + Nodes, + mem3_reshard, + resume_job, + [JobId] + ), WithoutNotFound = [R || R <- Replies, R =/= {error, not_found}], case lists:usort(WithoutNotFound) of [ok] -> @@ -86,11 +100,14 @@ resume_job(JobId) -> {error, not_found} end. - stop_job(JobId, Reason) -> Nodes = mem3_util:live_nodes(), - {Replies, _Bad} = rpc:multicall(Nodes, mem3_reshard, stop_job, - [JobId, Reason]), + {Replies, _Bad} = rpc:multicall( + Nodes, + mem3_reshard, + stop_job, + [JobId, Reason] + ), WithoutNotFound = [R || R <- Replies, R =/= {error, not_found}], case lists:usort(WithoutNotFound) of [ok] -> @@ -101,7 +118,6 @@ stop_job(JobId, Reason) -> {error, not_found} end. - start_shard_splitting() -> {Replies, _Bad} = rpc:multicall(mem3_reshard, start, []), case lists:usort(lists:flatten(Replies)) of @@ -111,7 +127,6 @@ start_shard_splitting() -> {error, {[{error, couch_util:to_binary(Error)}]}} end. - stop_shard_splitting(Reason) -> {Replies, _Bad} = rpc:multicall(mem3_reshard, stop, [Reason]), case lists:usort(lists:flatten(Replies)) of @@ -121,22 +136,24 @@ stop_shard_splitting(Reason) -> {error, {[{error, couch_util:to_binary(Error)}]}} end. - get_shard_splitting_state() -> Nodes = mem3_util:live_nodes(), {Replies, _Bad} = rpc:multicall(Nodes, mem3_reshard, get_state, []), state_and_reason(Replies). - state_and_reason(StateReplies) -> - AccF = lists:foldl(fun({ResProps}, Acc) -> - Reason = get_reason(ResProps), - case couch_util:get_value(state, ResProps) of - <<"running">> -> orddict:append(running, Reason, Acc); - <<"stopped">> -> orddict:append(stopped, Reason, Acc); - undefined -> Acc - end - end, orddict:from_list([{running, []}, {stopped, []}]), StateReplies), + AccF = lists:foldl( + fun({ResProps}, Acc) -> + Reason = get_reason(ResProps), + case couch_util:get_value(state, ResProps) of + <<"running">> -> orddict:append(running, Reason, Acc); + <<"stopped">> -> orddict:append(stopped, Reason, Acc); + undefined -> Acc + end + end, + orddict:from_list([{running, []}, {stopped, []}]), + StateReplies + ), Running = orddict:fetch(running, AccF), case length(Running) > 0 of true -> @@ -147,7 +164,6 @@ state_and_reason(StateReplies) -> {stopped, Reason} end. - pick_reason(Reasons) -> Reasons1 = lists:usort(Reasons), Reasons2 = [R || R <- Reasons1, R =/= undefined], @@ -156,7 +172,6 @@ pick_reason(Reasons) -> [R1 | _] -> R1 end. - get_reason(StateProps) when is_list(StateProps) -> case couch_util:get_value(state_info, StateProps) of [] -> undefined; @@ -164,42 +179,39 @@ get_reason(StateProps) when is_list(StateProps) -> {SInfoProps} -> couch_util:get_value(reason, SInfoProps) end. - pick_shards(undefined, undefined, Db, undefined) when is_binary(Db) -> check_node_required(), check_range_required(), mem3:shards(Db); - -pick_shards(Node, undefined, Db, undefined) when is_atom(Node), - is_binary(Db) -> +pick_shards(Node, undefined, Db, undefined) when + is_atom(Node), + is_binary(Db) +-> check_range_required(), [S || S <- mem3:shards(Db), mem3:node(S) == Node]; - -pick_shards(undefined, undefined, Db, [_B, _E] = Range) when is_binary(Db) -> +pick_shards(undefined, undefined, Db, [_B, _E] = Range) when is_binary(Db) -> check_node_required(), [S || S <- mem3:shards(Db), mem3:range(S) == Range]; - -pick_shards(Node, undefined, Db, [_B, _E] = Range) when is_atom(Node), - is_binary(Db) -> +pick_shards(Node, undefined, Db, [_B, _E] = Range) when + is_atom(Node), + is_binary(Db) +-> [S || S <- mem3:shards(Db), mem3:node(S) == Node, mem3:range(S) == Range]; - pick_shards(undefined, Shard, undefined, undefined) when is_binary(Shard) -> check_node_required(), Db = mem3:dbname(Shard), [S || S <- mem3:shards(Db), mem3:name(S) == Shard]; - -pick_shards(Node, Shard, undefined, undefined) when is_atom(Node), - is_binary(Shard) -> +pick_shards(Node, Shard, undefined, undefined) when + is_atom(Node), + is_binary(Shard) +-> Db = mem3:dbname(Shard), [S || S <- mem3:shards(Db), mem3:name(S) == Shard, mem3:node(S) == Node]; - pick_shards(_, undefined, undefined, _) -> throw({bad_request, <<"Must specify at least `db` or `shard`">>}); - pick_shards(_, Db, Shard, _) when is_binary(Db), is_binary(Shard) -> throw({bad_request, <<"`db` and `shard` are mutually exclusive">>}). - check_node_required() -> case config:get_boolean("reshard", "require_node_param", false) of true -> diff --git a/src/mem3/src/mem3_reshard_dbdoc.erl b/src/mem3/src/mem3_reshard_dbdoc.erl index 4a0a35c1f..7fb69598e 100644 --- a/src/mem3/src/mem3_reshard_dbdoc.erl +++ b/src/mem3/src/mem3_reshard_dbdoc.erl @@ -27,11 +27,9 @@ code_change/3 ]). - -include_lib("couch/include/couch_db.hrl"). -include("mem3_reshard.hrl"). - -spec update_shard_map(#job{}) -> no_return | ok. update_shard_map(#job{source = Source, target = Target} = Job) -> Node = hd(mem3_util:live_nodes()), @@ -58,93 +56,82 @@ update_shard_map(#job{source = Source, target = Target} = Job) -> false -> exit(shard_update_did_not_propagate) end. - -spec start_link() -> {ok, pid()} | ignore | {error, term()}. start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - init(_) -> couch_log:notice("~p start init()", [?MODULE]), {ok, nil}. - terminate(_Reason, _State) -> ok. - handle_call({update_shard_map, Source, Target}, _From, State) -> - Res = try - update_shard_map(Source, Target) - catch - throw:{error, Error} -> - {error, Error} - end, + Res = + try + update_shard_map(Source, Target) + catch + throw:{error, Error} -> + {error, Error} + end, {reply, Res, State}; - handle_call(Call, From, State) -> couch_log:error("~p unknown call ~p from: ~p", [?MODULE, Call, From]), {noreply, State}. - handle_cast(Cast, State) -> couch_log:error("~p unexpected cast ~p", [?MODULE, Cast]), {noreply, State}. - handle_info(Info, State) -> couch_log:error("~p unexpected info ~p", [?MODULE, Info]), {noreply, State}. - code_change(_OldVsn, State, _Extra) -> {ok, State}. - % Private update_shard_map(Source, Target) -> ok = validate_coordinator(), ok = replicate_from_all_nodes(shard_update_timeout_msec()), DocId = mem3:dbname(Source#shard.name), - OldDoc = case mem3_util:open_db_doc(DocId) of - {ok, #doc{deleted = true}} -> - throw({error, missing_source}); - {ok, #doc{} = Doc} -> - Doc; - {not_found, deleted} -> - throw({error, missing_source}); - OpenErr -> - throw({error, {shard_doc_open_error, OpenErr}}) - end, + OldDoc = + case mem3_util:open_db_doc(DocId) of + {ok, #doc{deleted = true}} -> + throw({error, missing_source}); + {ok, #doc{} = Doc} -> + Doc; + {not_found, deleted} -> + throw({error, missing_source}); + OpenErr -> + throw({error, {shard_doc_open_error, OpenErr}}) + end, #doc{body = OldBody} = OldDoc, NewBody = update_shard_props(OldBody, Source, Target), - {ok, _} = write_shard_doc(OldDoc, NewBody), + {ok, _} = write_shard_doc(OldDoc, NewBody), ok = replicate_to_all_nodes(shard_update_timeout_msec()), {ok, NewBody}. - validate_coordinator() -> case hd(mem3_util:live_nodes()) =:= node() of true -> ok; false -> throw({error, coordinator_changed}) end. - replicate_from_all_nodes(TimeoutMSec) -> case mem3_util:replicate_dbs_from_all_nodes(TimeoutMSec) of ok -> ok; Error -> throw({error, Error}) end. - replicate_to_all_nodes(TimeoutMSec) -> case mem3_util:replicate_dbs_to_all_nodes(TimeoutMSec) of ok -> ok; Error -> throw({error, Error}) end. - write_shard_doc(#doc{id = Id} = Doc, Body) -> UpdatedDoc = Doc#doc{body = Body}, couch_util:with_db(mem3_sync:shards_db(), fun(Db) -> @@ -156,9 +143,8 @@ write_shard_doc(#doc{id = Id} = Doc, Body) -> end end). - update_shard_props({Props0}, #shard{} = Source, [#shard{} | _] = Targets) -> - {ByNode0} = couch_util:get_value(<<"by_node">>, Props0, {[]}), + {ByNode0} = couch_util:get_value(<<"by_node">>, Props0, {[]}), ByNodeKV = {<<"by_node">>, {update_by_node(ByNode0, Source, Targets)}}, Props1 = lists:keyreplace(<<"by_node">>, 1, Props0, ByNodeKV), @@ -175,7 +161,6 @@ update_shard_props({Props0}, #shard{} = Source, [#shard{} | _] = Targets) -> {Props3}. - update_by_node(ByNode, #shard{} = Source, [#shard{} | _] = Targets) -> {NodeKey, SKey} = {node_key(Source), range_key(Source)}, {_, Ranges} = lists:keyfind(NodeKey, 1, ByNode), @@ -183,12 +168,10 @@ update_by_node(ByNode, #shard{} = Source, [#shard{} | _] = Targets) -> Ranges2 = Ranges1 ++ [range_key(T) || T <- Targets], lists:keyreplace(NodeKey, 1, ByNode, {NodeKey, lists:sort(Ranges2)}). - update_by_range(ByRange, Source, Targets) -> ByRange1 = remove_node_from_source(ByRange, Source), lists:foldl(fun add_node_to_target_foldl/2, ByRange1, Targets). - remove_node_from_source(ByRange, Source) -> {NodeKey, SKey} = {node_key(Source), range_key(Source)}, {_, SourceNodes} = lists:keyfind(SKey, 1, ByRange), @@ -208,7 +191,6 @@ remove_node_from_source(ByRange, Source) -> lists:keyreplace(SKey, 1, ByRange, {SKey, SourceNodes1}) end. - add_node_to_target_foldl(#shard{} = Target, ByRange) -> {NodeKey, TKey} = {node_key(Target), range_key(Target)}, case lists:keyfind(TKey, 1, ByRange) of @@ -227,21 +209,17 @@ add_node_to_target_foldl(#shard{} = Target, ByRange) -> lists:sort([{TKey, [NodeKey]} | ByRange]) end. - node_key(#shard{node = Node}) -> couch_util:to_binary(Node). - range_key(#shard{range = [B, E]}) -> BHex = couch_util:to_hex(<<B:32/integer>>), EHex = couch_util:to_hex(<<E:32/integer>>), list_to_binary([BHex, "-", EHex]). - shard_update_timeout_msec() -> config:get_integer("reshard", "shard_upate_timeout_msec", 300000). - wait_source_removed(#shard{name = Name} = Source, SleepSec, UntilSec) -> case check_source_removed(Source) of true -> @@ -258,7 +236,6 @@ wait_source_removed(#shard{name = Name} = Source, SleepSec, UntilSec) -> end end. - check_source_removed(#shard{name = Name}) -> DbName = mem3:dbname(Name), Live = mem3_util:live_nodes(), @@ -266,9 +243,13 @@ check_source_removed(#shard{name = Name}) -> Nodes = lists:usort([N || N <- ShardNodes, lists:member(N, Live)]), {Responses, _} = rpc:multicall(Nodes, mem3, shards, [DbName]), Shards = lists:usort(lists:flatten(Responses)), - SourcePresent = [S || S = #shard{name = S, node = N} <- Shards, S =:= Name, - N =:= node()], + SourcePresent = [ + S + || S = #shard{name = S, node = N} <- Shards, + S =:= Name, + N =:= node() + ], case SourcePresent of - [] -> true; + [] -> true; [_ | _] -> false end. diff --git a/src/mem3/src/mem3_reshard_httpd.erl b/src/mem3/src/mem3_reshard_httpd.erl index 3d0f77f39..5abe8025c 100644 --- a/src/mem3/src/mem3_reshard_httpd.erl +++ b/src/mem3/src/mem3_reshard_httpd.erl @@ -22,35 +22,37 @@ send_method_not_allowed/2 ]). - -include_lib("couch/include/couch_db.hrl"). - -define(JOBS, <<"jobs">>). -define(STATE, <<"state">>). -define(S_RUNNING, <<"running">>). -define(S_STOPPED, <<"stopped">>). - % GET /_reshard -handle_reshard_req(#httpd{method='GET', path_parts=[_]} = Req) -> +handle_reshard_req(#httpd{method = 'GET', path_parts = [_]} = Req) -> reject_if_disabled(), State = mem3_reshard_api:get_summary(), send_json(Req, State); - -handle_reshard_req(#httpd{path_parts=[_]} = Req) -> +handle_reshard_req(#httpd{path_parts = [_]} = Req) -> send_method_not_allowed(Req, "GET,HEAD"); - % GET /_reshard/state -handle_reshard_req(#httpd{method='GET', - path_parts=[_, ?STATE]} = Req) -> +handle_reshard_req( + #httpd{ + method = 'GET', + path_parts = [_, ?STATE] + } = Req +) -> reject_if_disabled(), {State, Reason} = mem3_reshard_api:get_shard_splitting_state(), send_json(Req, {[{state, State}, {reason, Reason}]}); - % PUT /_reshard/state -handle_reshard_req(#httpd{method='PUT', - path_parts=[_, ?STATE]} = Req) -> +handle_reshard_req( + #httpd{ + method = 'PUT', + path_parts = [_, ?STATE] + } = Req +) -> reject_if_disabled(), couch_httpd:validate_ctype(Req, "application/json"), {Props} = couch_httpd:json_body_obj(Req), @@ -67,10 +69,11 @@ handle_reshard_req(#httpd{method='PUT', send_json(Req, 500, JsonError) end; {?S_STOPPED, Reason} -> - Reason1 = case Reason =:= undefined of - false -> Reason; - true -> <<"Cluster-wide resharding stopped by the user">> - end, + Reason1 = + case Reason =:= undefined of + false -> Reason; + true -> <<"Cluster-wide resharding stopped by the user">> + end, case mem3_reshard_api:stop_shard_splitting(Reason1) of {ok, JsonResult} -> send_json(Req, 200, JsonResult); @@ -80,23 +83,23 @@ handle_reshard_req(#httpd{method='PUT', {_, _} -> throw({bad_request, <<"State field not `running` or `stopped`">>}) end; - -handle_reshard_req(#httpd{path_parts=[_, ?STATE]} = Req) -> +handle_reshard_req(#httpd{path_parts = [_, ?STATE]} = Req) -> send_method_not_allowed(Req, "GET,HEAD,PUT"); - -handle_reshard_req(#httpd{path_parts=[_, ?STATE | _]} = _Req) -> +handle_reshard_req(#httpd{path_parts = [_, ?STATE | _]} = _Req) -> throw(not_found); - % GET /_reshard/jobs -handle_reshard_req(#httpd{method='GET', path_parts=[_, ?JOBS]}=Req) -> +handle_reshard_req(#httpd{method = 'GET', path_parts = [_, ?JOBS]} = Req) -> reject_if_disabled(), Jobs = mem3_reshard_api:get_jobs(), Total = length(Jobs), send_json(Req, {[{total_rows, Total}, {offset, 0}, {jobs, Jobs}]}); - % POST /_reshard/jobs {"node": "...", "shard": "..."} -handle_reshard_req(#httpd{method = 'POST', - path_parts=[_, ?JOBS]} = Req) -> +handle_reshard_req( + #httpd{ + method = 'POST', + path_parts = [_, ?JOBS] + } = Req +) -> reject_if_disabled(), couch_httpd:validate_ctype(Req, "application/json"), {Props} = couch_httpd:json_body_obj(Req), @@ -111,28 +114,33 @@ handle_reshard_req(#httpd{method = 'POST', _ -> ok end, Oks = length([R || {ok, _, _, _} = R <- Res]), - Code = case {Oks, length(Res)} of - {Oks, Oks} -> 201; - {Oks, _} when Oks > 0 -> 202; - {0, _} -> 500 - end, - EJson = lists:map(fun - ({ok, Id, N, S}) -> - {[{ok, true}, {id, Id}, {node, N}, {shard, S}]}; - ({error, E, N, S}) -> - {[{error, couch_util:to_binary(E)}, {node, N}, {shard, S}]} - end, Res), + Code = + case {Oks, length(Res)} of + {Oks, Oks} -> 201; + {Oks, _} when Oks > 0 -> 202; + {0, _} -> 500 + end, + EJson = lists:map( + fun + ({ok, Id, N, S}) -> + {[{ok, true}, {id, Id}, {node, N}, {shard, S}]}; + ({error, E, N, S}) -> + {[{error, couch_util:to_binary(E)}, {node, N}, {shard, S}]} + end, + Res + ), send_json(Req, Code, EJson); - -handle_reshard_req(#httpd{path_parts=[_, ?JOBS]} = Req) -> +handle_reshard_req(#httpd{path_parts = [_, ?JOBS]} = Req) -> send_method_not_allowed(Req, "GET,HEAD,POST"); - -handle_reshard_req(#httpd{path_parts=[_, _]}) -> +handle_reshard_req(#httpd{path_parts = [_, _]}) -> throw(not_found); - % GET /_reshard/jobs/$jobid -handle_reshard_req(#httpd{method='GET', - path_parts=[_, ?JOBS, JobId]}=Req) -> +handle_reshard_req( + #httpd{ + method = 'GET', + path_parts = [_, ?JOBS, JobId] + } = Req +) -> reject_if_disabled(), case mem3_reshard_api:get_job(JobId) of {ok, JobInfo} -> @@ -140,10 +148,13 @@ handle_reshard_req(#httpd{method='GET', {error, not_found} -> throw(not_found) end; - % DELETE /_reshard/jobs/$jobid -handle_reshard_req(#httpd{method='DELETE', - path_parts=[_, ?JOBS, JobId]}=Req) -> +handle_reshard_req( + #httpd{ + method = 'DELETE', + path_parts = [_, ?JOBS, JobId] + } = Req +) -> reject_if_disabled(), case mem3_reshard_api:get_job(JobId) of {ok, {Props}} -> @@ -158,30 +169,36 @@ handle_reshard_req(#httpd{method='DELETE', {error, not_found} -> throw(not_found) end; - -handle_reshard_req(#httpd{path_parts=[_, ?JOBS, _]} = Req) -> +handle_reshard_req(#httpd{path_parts = [_, ?JOBS, _]} = Req) -> send_method_not_allowed(Req, "GET,HEAD,DELETE"); - % GET /_reshard/jobs/$jobid/state -handle_reshard_req(#httpd{method='GET', - path_parts=[_, ?JOBS, JobId, ?STATE]} = Req) -> +handle_reshard_req( + #httpd{ + method = 'GET', + path_parts = [_, ?JOBS, JobId, ?STATE] + } = Req +) -> reject_if_disabled(), case mem3_reshard_api:get_job(JobId) of {ok, {Props}} -> JobState = couch_util:get_value(job_state, Props), {SIProps} = couch_util:get_value(state_info, Props), - Reason = case couch_util:get_value(reason, SIProps) of - undefined -> null; - Val -> couch_util:to_binary(Val) - end, + Reason = + case couch_util:get_value(reason, SIProps) of + undefined -> null; + Val -> couch_util:to_binary(Val) + end, send_json(Req, 200, {[{state, JobState}, {reason, Reason}]}); {error, not_found} -> throw(not_found) end; - % PUT /_reshard/jobs/$jobid/state -handle_reshard_req(#httpd{method='PUT', - path_parts=[_, ?JOBS, JobId, ?STATE]} = Req) -> +handle_reshard_req( + #httpd{ + method = 'PUT', + path_parts = [_, ?JOBS, JobId, ?STATE] + } = Req +) -> reject_if_disabled(), couch_httpd:validate_ctype(Req, "application/json"), {Props} = couch_httpd:json_body_obj(Req), @@ -200,10 +217,11 @@ handle_reshard_req(#httpd{method='PUT', send_json(Req, 500, JsonError) end; {?S_STOPPED, Reason} -> - Reason1 = case Reason =:= undefined of - false -> Reason; - true -> <<"Stopped by user">> - end, + Reason1 = + case Reason =:= undefined of + false -> Reason; + true -> <<"Stopped by user">> + end, case mem3_reshard_api:stop_job(JobId, Reason1) of ok -> send_json(Req, 200, {[{ok, true}]}); @@ -215,28 +233,22 @@ handle_reshard_req(#httpd{method='PUT', {_, _} -> throw({bad_request, <<"State field not `running` or `stopped`">>}) end; - -handle_reshard_req(#httpd{path_parts=[_, ?JOBS, _, ?STATE]} = Req) -> +handle_reshard_req(#httpd{path_parts = [_, ?JOBS, _, ?STATE]} = Req) -> send_method_not_allowed(Req, "GET,HEAD,PUT"). - reject_if_disabled() -> case mem3_reshard:is_disabled() of true -> throw(not_implemented); false -> ok end. - validate_type(<<"split">>) -> split; - validate_type(_Type) -> throw({bad_request, <<"`job type must be `split`">>}). - validate_node(undefined) -> undefined; - validate_node(Node0) when is_binary(Node0) -> Nodes = mem3_util:live_nodes(), try binary_to_existing_atom(Node0, utf8) of @@ -249,14 +261,11 @@ validate_node(Node0) when is_binary(Node0) -> error:badarg -> throw({bad_request, <<"`node` is not a valid node name">>}) end; - validate_node(_Node) -> throw({bad_request, <<"Invalid `node`">>}). - validate_shard(undefined) -> undefined; - validate_shard(Shard) when is_binary(Shard) -> case Shard of <<"shards/", _:8/binary, "-", _:8/binary, "/", _/binary>> -> @@ -264,40 +273,35 @@ validate_shard(Shard) when is_binary(Shard) -> _ -> throw({bad_request, <<"`shard` is invalid">>}) end; - validate_shard(_Shard) -> throw({bad_request, <<"Invalid `shard`">>}). - validate_db(undefined) -> undefined; - validate_db(DbName) when is_binary(DbName) -> try mem3:shards(DbName) of [_ | _] -> DbName; - _ -> throw({bad_request, <<"`No shards in `db`">>}) + _ -> throw({bad_request, <<"`No shards in `db`">>}) catch _:_ -> throw({bad_request, <<"Invalid `db`">>}) end; - validate_db(_bName) -> throw({bad_request, <<"Invalid `db`">>}). - validate_range(undefined) -> undefined; - validate_range(<<BBin:8/binary, "-", EBin:8/binary>>) -> - {B, E} = try - { - httpd_util:hexlist_to_integer(binary_to_list(BBin)), - httpd_util:hexlist_to_integer(binary_to_list(EBin)) - } - catch - _:_ -> - invalid_range() - end, + {B, E} = + try + { + httpd_util:hexlist_to_integer(binary_to_list(BBin)), + httpd_util:hexlist_to_integer(binary_to_list(EBin)) + } + catch + _:_ -> + invalid_range() + end, if B < 0 -> invalid_range(); E < 0 -> invalid_range(); @@ -308,10 +312,8 @@ validate_range(<<BBin:8/binary, "-", EBin:8/binary>>) -> end, % Use a list format here to make it look the same as #shard's range [B, E]; - validate_range(_Range) -> invalid_range(). - invalid_range() -> throw({bad_request, <<"Invalid `range`">>}). diff --git a/src/mem3/src/mem3_reshard_index.erl b/src/mem3/src/mem3_reshard_index.erl index d4cb7caa1..fef25d52c 100644 --- a/src/mem3/src/mem3_reshard_index.erl +++ b/src/mem3/src/mem3_reshard_index.erl @@ -12,18 +12,15 @@ -module(mem3_reshard_index). - -export([ design_docs/1, target_indices/2, spawn_builders/1 ]). - -include_lib("mem3/include/mem3.hrl"). -include_lib("couch/include/couch_db.hrl"). - %% Public API design_docs(DbName) -> @@ -37,16 +34,15 @@ design_docs(DbName) -> Else -> Else end - catch error:database_does_not_exist -> - {ok, []} + catch + error:database_does_not_exist -> + {ok, []} end. - target_indices(Docs, Targets) -> Indices = [[indices(N, D) || D <- Docs] || #shard{name = N} <- Targets], lists:flatten(Indices). - spawn_builders(Indices) -> Results = [build_index(Index) || Index <- Indices], Oks = [{ok, Pid} || {ok, Pid} <- Results, is_pid(Pid)], @@ -58,14 +54,16 @@ spawn_builders(Indices) -> % spawned, kill the spawned ones and and return the error. ErrMsg = "~p failed to spawn index builders: ~p ~p", couch_log:error(ErrMsg, [?MODULE, Error, Indices]), - lists:foreach(fun({ok, Pid}) -> - catch unlink(Pid), - catch exit(Pid, kill) - end, Oks), + lists:foreach( + fun({ok, Pid}) -> + catch unlink(Pid), + catch exit(Pid, kill) + end, + Oks + ), {error, Error} end. - %% Private API fabric_design_docs(DbName) -> @@ -74,12 +72,10 @@ fabric_design_docs(DbName) -> {error, Error} -> Error end. - indices(DbName, Doc) -> - mrview_indices(DbName, Doc) - ++ [dreyfus_indices(DbName, Doc) || has_app(dreyfus)] - ++ [hastings_indices(DbName, Doc) || has_app(hastings)]. - + mrview_indices(DbName, Doc) ++ + [dreyfus_indices(DbName, Doc) || has_app(dreyfus)] ++ + [hastings_indices(DbName, Doc) || has_app(hastings)]. mrview_indices(DbName, Doc) -> try @@ -98,7 +94,6 @@ mrview_indices(DbName, Doc) -> [] end. - dreyfus_indices(DbName, Doc) -> try Indices = dreyfus_index:design_doc_to_indexes(Doc), @@ -110,7 +105,6 @@ dreyfus_indices(DbName, Doc) -> [] end. - hastings_indices(DbName, Doc) -> try Indices = hastings_index:design_doc_to_indexes(Doc), @@ -122,7 +116,6 @@ hastings_indices(DbName, Doc) -> [] end. - build_index({mrview, DbName, MRSt}) -> case couch_index_server:get_index(couch_mrview_index, MRSt) of {ok, Pid} -> @@ -132,8 +125,7 @@ build_index({mrview, DbName, MRSt}) -> Error -> Error end; - -build_index({dreyfus, DbName, Index})-> +build_index({dreyfus, DbName, Index}) -> case dreyfus_index_manager:get_index(DbName, Index) of {ok, Pid} -> Args = [Pid, get_update_seq(DbName)], @@ -142,7 +134,6 @@ build_index({dreyfus, DbName, Index})-> Error -> Error end; - build_index({hastings, DbName, Index}) -> case hastings_index_manager:get_index(DbName, Index) of {ok, Pid} -> @@ -153,11 +144,9 @@ build_index({hastings, DbName, Index}) -> Error end. - has_app(App) -> code:lib_dir(App) /= {error, bad_name}. - get_update_seq(DbName) -> couch_util:with_db(DbName, fun(Db) -> couch_db:get_update_seq(Db) diff --git a/src/mem3/src/mem3_reshard_job.erl b/src/mem3/src/mem3_reshard_job.erl index aedca21bb..aff5c2648 100644 --- a/src/mem3/src/mem3_reshard_job.erl +++ b/src/mem3/src/mem3_reshard_job.erl @@ -12,7 +12,6 @@ -module(mem3_reshard_job). - -export([ start_link/1, @@ -46,11 +45,9 @@ completed/1 ]). - -include_lib("couch/include/couch_db.hrl"). -include("mem3_reshard.hrl"). - % Batch size for internal replication topoffs -define(INTERNAL_REP_BATCH_SIZE, 2000). @@ -73,7 +70,6 @@ completed ]). - % When a job starts it may be resuming from a partially % completed state. These state pairs list the state % we have to restart from for each possible state. @@ -91,7 +87,6 @@ completed => completed }). - % If we have a worker failing during any of these % states we need to clean up the targets -define(CLEAN_TARGET_STATES, [ @@ -102,11 +97,9 @@ copy_local_docs ]). - start_link(#job{} = Job) -> proc_lib:start_link(?MODULE, init, [Job]). - % This is called by the main proces after it has checkpointed the progress % of the job. After the new state is checkpointed, we signal the job to start % executing that state. @@ -115,7 +108,6 @@ checkpoint_done(#job{pid = Pid} = Job) -> Pid ! checkpoint_done, ok. - % Formatting function, used for logging mostly jobfmt(#job{} = Job) -> #job{ @@ -131,13 +123,13 @@ jobfmt(#job{} = Job) -> Fmt = io_lib:format(Msg, [Id, Source, TargetCount, JobState, State, Pid]), lists:flatten(Fmt). - % This is the function which picks between various targets. It is used here as % well as in mem3_rep internal replicator and couch_db_split bulk copy logic. % Given a document id and list of ranges, and a hash function, it will pick one % of the range or return not_in_range atom. pickfun(DocId, [[B, E] | _] = Ranges, {_M, _F, _A} = HashFun) when - is_integer(B), is_integer(E), B =< E -> + is_integer(B), is_integer(E), B =< E +-> HashKey = mem3_hash:calculate(HashFun, DocId), Pred = fun([Begin, End]) -> Begin =< HashKey andalso HashKey =< End @@ -147,7 +139,6 @@ pickfun(DocId, [[B, E] | _] = Ranges, {_M, _F, _A} = HashFun) when [Key] -> Key end. - init(#job{} = Job0) -> process_flag(trap_exit, true), Job1 = set_start_state(Job0#job{ @@ -162,52 +153,47 @@ init(#job{} = Job0) -> ok = checkpoint(Job2), run(Job2). - run(#job{split_state = CurrState} = Job) -> - StateFun = case CurrState of - topoff1 -> topoff; - topoff2 -> topoff; - topoff3 -> topoff; - _ -> CurrState - end, - NewJob = try - Job1 = ?MODULE:StateFun(Job), - Job2 = wait_for_workers(Job1), - Job3 = switch_to_next_state(Job2), - ok = checkpoint(Job3), - Job3 - catch - throw:{retry, RetryJob} -> - RetryJob - end, + StateFun = + case CurrState of + topoff1 -> topoff; + topoff2 -> topoff; + topoff3 -> topoff; + _ -> CurrState + end, + NewJob = + try + Job1 = ?MODULE:StateFun(Job), + Job2 = wait_for_workers(Job1), + Job3 = switch_to_next_state(Job2), + ok = checkpoint(Job3), + Job3 + catch + throw:{retry, RetryJob} -> + RetryJob + end, run(NewJob). - set_start_state(#job{split_state = State} = Job) -> case maps:get(State, ?STATE_RESTART, undefined) of undefined -> Fmt1 = "~p recover : unknown state ~s", couch_log:error(Fmt1, [?MODULE, jobfmt(Job)]), erlang:error({invalid_split_job_recover_state, Job}); - StartState-> + StartState -> Job#job{split_state = StartState} end. - get_next_state(#job{split_state = State}) -> get_next_state(State, ?SPLIT_STATES). - get_next_state(completed, _) -> completed; - get_next_state(CurrState, [CurrState, NextState | _]) -> NextState; - get_next_state(CurrState, [_ | Rest]) -> get_next_state(CurrState, Rest). - switch_to_next_state(#job{} = Job0) -> Info0 = Job0#job.state_info, Info1 = info_delete(error, Info0), @@ -222,7 +208,6 @@ switch_to_next_state(#job{} = Job0) -> Job2 = update_split_history(Job1), check_state(Job2). - checkpoint(Job) -> % Ask main process to checkpoint. When it has finished it will notify us % by calling by checkpoint_done/1. The reason not to call the main process @@ -241,10 +226,8 @@ checkpoint(Job) -> handle_unknown_msg(Job, "checkpoint", Other) end. - wait_for_workers(#job{workers = []} = Job) -> Job; - wait_for_workers(#job{workers = Workers} = Job) -> Parent = parent(), receive @@ -262,10 +245,8 @@ wait_for_workers(#job{workers = Workers} = Job) -> handle_unknown_msg(Job, "wait_for_workers", Other) end. - handle_worker_exit(#job{workers = Workers} = Job, Pid, normal) -> Job#job{workers = Workers -- [Pid]}; - handle_worker_exit(#job{} = Job, _Pid, {error, missing_source}) -> Msg1 = "~p stopping worker due to source missing ~p", couch_log:error(Msg1, [?MODULE, jobfmt(Job)]), @@ -279,13 +260,11 @@ handle_worker_exit(#job{} = Job, _Pid, {error, missing_source}) -> false -> exit({error, missing_source}) end; - handle_worker_exit(#job{} = Job, _Pid, {error, missing_target}) -> Msg = "~p stopping worker due to target db missing ~p", couch_log:error(Msg, [?MODULE, jobfmt(Job)]), kill_workers(Job), exit({error, missing_target}); - handle_worker_exit(#job{} = Job0, _Pid, Reason) -> couch_log:error("~p worker error ~p ~p", [?MODULE, jobfmt(Job0), Reason]), kill_workers(Job0), @@ -297,11 +276,12 @@ handle_worker_exit(#job{} = Job0, _Pid, Reason) -> exit(Reason) end. - % Cleanup and exit when we receive an 'EXIT' message from our parent. In case % the shard map is being updated, try to wait some time for it to finish. -handle_exit(#job{split_state = update_shardmap, workers = [WPid]} = Job, - Reason) -> +handle_exit( + #job{split_state = update_shardmap, workers = [WPid]} = Job, + Reason +) -> Timeout = update_shard_map_timeout_sec(), Msg1 = "~p job exit ~s ~p while shard map is updating, waiting ~p sec", couch_log:warning(Msg1, [?MODULE, jobfmt(Job), Reason, Timeout]), @@ -314,18 +294,16 @@ handle_exit(#job{split_state = update_shardmap, workers = [WPid]} = Job, Msg3 = "~p ~s shard map update failed with error ~p", couch_log:error(Msg3, [?MODULE, jobfmt(Job), Error]), exit(Reason) - after Timeout * 1000-> + after Timeout * 1000 -> Msg4 = "~p ~s shard map update timeout exceeded ~p sec", couch_log:error(Msg4, [?MODULE, jobfmt(Job), Timeout]), kill_workers(Job), exit(Reason) end; - handle_exit(#job{} = Job, Reason) -> kill_workers(Job), exit(Reason). - retry_state(#job{retries = Retries, state_info = Info} = Job0, Error) -> Job1 = Job0#job{ retries = Retries + 1, @@ -345,21 +323,21 @@ retry_state(#job{retries = Retries, state_info = Info} = Job0, Error) -> end, throw({retry, Job2}). - report(#job{manager = ManagerPid} = Job) -> Job1 = Job#job{update_time = mem3_reshard:now_sec()}, ok = mem3_reshard:report(ManagerPid, Job1), Job1. - kill_workers(#job{workers = Workers}) -> - lists:foreach(fun(Worker) -> - unlink(Worker), - exit(Worker, kill) - end, Workers), + lists:foreach( + fun(Worker) -> + unlink(Worker), + exit(Worker, kill) + end, + Workers + ), flush_worker_messages(). - flush_worker_messages() -> Parent = parent(), receive @@ -369,7 +347,6 @@ flush_worker_messages() -> ok end. - parent() -> case get('$ancestors') of [Pid | _] when is_pid(Pid) -> Pid; @@ -377,18 +354,15 @@ parent() -> _ -> undefined end. - handle_unknown_msg(Job, When, RMsg) -> LogMsg = "~p ~s received an unknown message ~p when in ~s", couch_log:error(LogMsg, [?MODULE, jobfmt(Job), RMsg, When]), erlang:error({invalid_split_job_message, Job#job.id, When, RMsg}). - initial_copy(#job{} = Job) -> Pid = spawn_link(?MODULE, initial_copy_impl, [Job]), report(Job#job{workers = [Pid]}). - initial_copy_impl(#job{source = Source, target = Targets0} = Job) -> #shard{name = SourceName} = Source, Targets = [{R, N} || #shard{range = R, name = N} <- Targets0], @@ -410,18 +384,16 @@ initial_copy_impl(#job{source = Source, target = Targets0} = Job) -> exit({error, Error}) end. - topoff(#job{} = Job) -> Pid = spawn_link(?MODULE, topoff_impl, [Job]), report(Job#job{workers = [Pid]}). - topoff_impl(#job{source = #shard{} = Source, target = Targets}) -> couch_log:notice("~p topoff ~p", [?MODULE, shardsstr(Source, Targets)]), check_source_exists(Source, topoff), check_targets_exist(Targets, topoff), TMap = maps:from_list([{R, T} || #shard{range = R} = T <- Targets]), - Opts = [{batch_size, ?INTERNAL_REP_BATCH_SIZE}, {batch_count, all}], + Opts = [{batch_size, ?INTERNAL_REP_BATCH_SIZE}, {batch_count, all}], case mem3_rep:go(Source, TMap, Opts) of {ok, Count} -> Args = [?MODULE, shardsstr(Source, Targets), Count], @@ -433,7 +405,6 @@ topoff_impl(#job{source = #shard{} = Source, target = Targets}) -> exit({error, Error}) end. - build_indices(#job{} = Job) -> #job{ source = #shard{name = SourceName} = Source, @@ -462,12 +433,10 @@ build_indices(#job{} = Job) -> end end. - copy_local_docs(#job{split_state = copy_local_docs} = Job) -> Pid = spawn_link(?MODULE, copy_local_docs_impl, [Job]), report(Job#job{workers = [Pid]}). - copy_local_docs_impl(#job{source = Source, target = Targets0}) -> #shard{name = SourceName} = Source, Targets = [{R, N} || #shard{range = R, name = N} <- Targets0], @@ -484,18 +453,15 @@ copy_local_docs_impl(#job{source = Source, target = Targets0}) -> exit({error, Error}) end. - update_shardmap(#job{} = Job) -> Pid = spawn_link(mem3_reshard_dbdoc, update_shard_map, [Job]), report(Job#job{workers = [Pid]}). - wait_source_close(#job{source = #shard{name = Name}} = Job) -> couch_event:notify(Name, deleted), Pid = spawn_link(?MODULE, wait_source_close_impl, [Job]), report(Job#job{workers = [Pid]}). - wait_source_close_impl(#job{source = #shard{name = Name}, target = Targets}) -> Timeout = config:get_integer("reshard", "source_close_timeout_sec", 600), check_targets_exist(Targets, wait_source_close), @@ -513,7 +479,6 @@ wait_source_close_impl(#job{source = #shard{name = Name}, target = Targets}) -> ok end. - wait_source_close(Db, SleepSec, UntilSec) -> case couch_db:monitored_by(Db) -- [self()] of [] -> @@ -531,23 +496,25 @@ wait_source_close(Db, SleepSec, UntilSec) -> end end. - source_delete(#job{} = Job) -> Pid = spawn_link(?MODULE, source_delete_impl, [Job]), report(Job#job{workers = [Pid]}). - source_delete_impl(#job{source = #shard{name = Name}, target = Targets}) -> check_targets_exist(Targets, source_delete), case config:get_boolean("mem3_reshard", "delete_source", true) of true -> case couch_server:delete(Name, [?ADMIN_CTX]) of ok -> - couch_log:notice("~p : deleted source shard ~p", - [?MODULE, Name]); + couch_log:notice( + "~p : deleted source shard ~p", + [?MODULE, Name] + ); not_found -> - couch_log:warning("~p : source was already deleted ~p", - [?MODULE, Name]) + couch_log:warning( + "~p : source was already deleted ~p", + [?MODULE, Name] + ) end; false -> % Emit deleted event even when not actually deleting the files this @@ -562,12 +529,10 @@ source_delete_impl(#job{source = #shard{name = Name}, target = Targets}) -> TNames = [TName || #shard{name = TName} <- Targets], lists:foreach(fun(TName) -> couch_event:notify(TName, updated) end, TNames). - completed(#job{} = Job) -> couch_log:notice("~p : ~p completed, exit normal", [?MODULE, jobfmt(Job)]), exit(normal). - % This is for belt and suspenders really. Call periodically to validate the % state is one of the expected states. -spec check_state(#job{}) -> #job{} | no_return(). @@ -579,45 +544,47 @@ check_state(#job{split_state = State} = Job) -> erlang:error({invalid_shard_split_state, State, Job}) end. - create_artificial_mem3_rep_checkpoints(#job{} = Job, Seq) -> #job{source = Source = #shard{name = SourceName}, target = Targets} = Job, check_source_exists(Source, initial_copy), TNames = [TN || #shard{name = TN} <- Targets], Timestamp = list_to_binary(mem3_util:iso8601_timestamp()), couch_util:with_db(SourceName, fun(SDb) -> - [couch_util:with_db(TName, fun(TDb) -> - Doc = mem3_rep_checkpoint_doc(SDb, TDb, Timestamp, Seq), - {ok, _} = couch_db:update_doc(SDb, Doc, []), - {ok, _} = couch_db:update_doc(TDb, Doc, []), - ok - end) || TName <- TNames] + [ + couch_util:with_db(TName, fun(TDb) -> + Doc = mem3_rep_checkpoint_doc(SDb, TDb, Timestamp, Seq), + {ok, _} = couch_db:update_doc(SDb, Doc, []), + {ok, _} = couch_db:update_doc(TDb, Doc, []), + ok + end) + || TName <- TNames + ] end), ok. - mem3_rep_checkpoint_doc(SourceDb, TargetDb, Timestamp, Seq) -> Node = atom_to_binary(node(), utf8), - SourceUUID = couch_db:get_uuid(SourceDb), + SourceUUID = couch_db:get_uuid(SourceDb), TargetUUID = couch_db:get_uuid(TargetDb), - History = {[ - {<<"source_node">>, Node}, - {<<"source_uuid">>, SourceUUID}, - {<<"source_seq">>, Seq}, - {<<"timestamp">>, Timestamp}, - {<<"target_node">>, Node}, - {<<"target_uuid">>, TargetUUID}, - {<<"target_seq">>, Seq} - ]}, - Body = {[ - {<<"seq">>, Seq}, - {<<"target_uuid">>, TargetUUID}, - {<<"history">>, {[{Node, [History]}]}} - ]}, + History = + {[ + {<<"source_node">>, Node}, + {<<"source_uuid">>, SourceUUID}, + {<<"source_seq">>, Seq}, + {<<"timestamp">>, Timestamp}, + {<<"target_node">>, Node}, + {<<"target_uuid">>, TargetUUID}, + {<<"target_seq">>, Seq} + ]}, + Body = + {[ + {<<"seq">>, Seq}, + {<<"target_uuid">>, TargetUUID}, + {<<"history">>, {[{Node, [History]}]}} + ]}, Id = mem3_rep:make_local_id(SourceUUID, TargetUUID), #doc{id = Id, body = Body}. - check_source_exists(#shard{name = Name}, StateName) -> case couch_server:exists(Name) of true -> @@ -628,89 +595,87 @@ check_source_exists(#shard{name = Name}, StateName) -> exit({error, missing_source}) end. - check_targets_exist(Targets, StateName) -> - lists:foreach(fun(#shard{name = Name}) -> - case couch_server:exists(Name) of - true -> - ok; - false -> - ErrMsg = "~p target ~p is unexpectedly missing in ~p", - couch_log:error(ErrMsg, [?MODULE, Name, StateName]), - exit({error, missing_target}) - end - end, Targets). - + lists:foreach( + fun(#shard{name = Name}) -> + case couch_server:exists(Name) of + true -> + ok; + false -> + ErrMsg = "~p target ~p is unexpectedly missing in ~p", + couch_log:error(ErrMsg, [?MODULE, Name, StateName]), + exit({error, missing_target}) + end + end, + Targets + ). -spec max_retries() -> integer(). max_retries() -> config:get_integer("reshard", "max_retries", 1). - -spec retry_interval_sec() -> integer(). retry_interval_sec() -> config:get_integer("reshard", "retry_interval_sec", 10). - -spec update_shard_map_timeout_sec() -> integer(). update_shard_map_timeout_sec() -> config:get_integer("reshard", "update_shardmap_timeout_sec", 60). - -spec info_update(atom(), any(), [tuple()]) -> [tuple()]. info_update(Key, Val, StateInfo) -> lists:keystore(Key, 1, StateInfo, {Key, Val}). - -spec info_delete(atom(), [tuple()]) -> [tuple()]. info_delete(Key, StateInfo) -> lists:keydelete(Key, 1, StateInfo). - -spec shardsstr(#shard{}, #shard{} | [#shard{}]) -> string(). shardsstr(#shard{name = SourceName}, #shard{name = TargetName}) -> lists:flatten(io_lib:format("~s -> ~s", [SourceName, TargetName])); - shardsstr(#shard{name = SourceName}, Targets) -> TNames = [TN || #shard{name = TN} <- Targets], TargetsStr = string:join([binary_to_list(T) || T <- TNames], ","), lists:flatten(io_lib:format("~s -> ~s", [SourceName, TargetsStr])). - -spec reset_target(#job{}) -> #job{}. reset_target(#job{source = Source, target = Targets} = Job) -> - ShardNames = try - [N || #shard{name = N} <- mem3:local_shards(mem3:dbname(Source))] - catch - error:database_does_not_exist -> - [] - end, - lists:map(fun(#shard{name = Name}) -> - case {couch_server:exists(Name), lists:member(Name, ShardNames)} of - {_, true} -> - % Should never get here but if we do crash and don't continue - LogMsg = "~p : ~p target unexpectedly found in shard map ~p", - couch_log:error(LogMsg, [?MODULE, jobfmt(Job), Name]), - erlang:error({target_present_in_shard_map, Name}); - {true, false} -> - LogMsg = "~p : ~p resetting ~p target", - couch_log:warning(LogMsg, [?MODULE, jobfmt(Job), Name]), - couch_db_split:cleanup_target(Source#shard.name, Name); - {false, false} -> - ok - end - end, Targets), + ShardNames = + try + [N || #shard{name = N} <- mem3:local_shards(mem3:dbname(Source))] + catch + error:database_does_not_exist -> + [] + end, + lists:map( + fun(#shard{name = Name}) -> + case {couch_server:exists(Name), lists:member(Name, ShardNames)} of + {_, true} -> + % Should never get here but if we do crash and don't continue + LogMsg = "~p : ~p target unexpectedly found in shard map ~p", + couch_log:error(LogMsg, [?MODULE, jobfmt(Job), Name]), + erlang:error({target_present_in_shard_map, Name}); + {true, false} -> + LogMsg = "~p : ~p resetting ~p target", + couch_log:warning(LogMsg, [?MODULE, jobfmt(Job), Name]), + couch_db_split:cleanup_target(Source#shard.name, Name); + {false, false} -> + ok + end + end, + Targets + ), Job. - -spec update_split_history(#job{}) -> #job{}. update_split_history(#job{split_state = St, update_time = Ts} = Job) -> Hist = Job#job.history, - JobSt = case St of - completed -> completed; - failed -> failed; - new -> new; - stopped -> stopped; - _ -> running - end, + JobSt = + case St of + completed -> completed; + failed -> failed; + new -> new; + stopped -> stopped; + _ -> running + end, Job#job{history = mem3_reshard:update_history(JobSt, St, Ts, Hist)}. diff --git a/src/mem3/src/mem3_reshard_job_sup.erl b/src/mem3/src/mem3_reshard_job_sup.erl index 3f1b3bfb4..e98cb5a08 100644 --- a/src/mem3/src/mem3_reshard_job_sup.erl +++ b/src/mem3/src/mem3_reshard_job_sup.erl @@ -22,34 +22,25 @@ init/1 ]). - -include("mem3_reshard.hrl"). - start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). - start_child(Job) -> supervisor:start_child(?MODULE, [Job]). - terminate_child(Pid) -> supervisor:terminate_child(?MODULE, Pid). - count_children() -> Props = supervisor:count_children(?MODULE), proplists:get_value(active, Props). - init(_Args) -> Children = [ - {mem3_reshard_job, - {mem3_reshard_job, start_link, []}, - temporary, - 60000, - worker, - [mem3_reshard_job]} + {mem3_reshard_job, {mem3_reshard_job, start_link, []}, temporary, 60000, worker, [ + mem3_reshard_job + ]} ], {ok, {{simple_one_for_one, 10, 3}, Children}}. diff --git a/src/mem3/src/mem3_reshard_store.erl b/src/mem3/src/mem3_reshard_store.erl index c3534b374..140cc5bd7 100644 --- a/src/mem3/src/mem3_reshard_store.erl +++ b/src/mem3/src/mem3_reshard_store.erl @@ -12,7 +12,6 @@ -module(mem3_reshard_store). - -export([ init/3, @@ -23,17 +22,16 @@ store_state/1, load_state/2, - delete_state/1, % for debugging + % for debugging + delete_state/1, job_to_ejson_props/2, state_to_ejson_props/1 ]). - -include_lib("couch/include/couch_db.hrl"). -include("mem3_reshard.hrl"). - -spec init(#state{}, binary(), binary()) -> #state{}. init(#state{} = State, JobPrefix, StateDocId) -> State#state{ @@ -41,7 +39,6 @@ init(#state{} = State, JobPrefix, StateDocId) -> state_id = <<?LOCAL_DOC_PREFIX, StateDocId/binary>> }. - -spec store_job(#state{}, #job{}) -> ok. store_job(#state{job_prefix = Prefix}, #job{id = Id} = Job) -> with_shards_db(fun(Db) -> @@ -49,7 +46,6 @@ store_job(#state{job_prefix = Prefix}, #job{id = Id} = Job) -> ok = update_doc(Db, DocId, job_to_ejson_props(Job)) end). - -spec load_job(#state{}, binary()) -> {ok, {[_]}} | not_found. load_job(#state{job_prefix = Prefix}, Id) -> with_shards_db(fun(Db) -> @@ -61,7 +57,6 @@ load_job(#state{job_prefix = Prefix}, Id) -> end end). - -spec delete_job(#state{}, binary()) -> ok. delete_job(#state{job_prefix = Prefix}, Id) -> with_shards_db(fun(Db) -> @@ -69,7 +64,6 @@ delete_job(#state{job_prefix = Prefix}, Id) -> ok = delete_doc(Db, DocId) end). - -spec get_jobs(#state{}) -> [#job{}]. get_jobs(#state{job_prefix = Prefix}) -> with_shards_db(fun(Db) -> @@ -87,14 +81,12 @@ get_jobs(#state{job_prefix = Prefix}) -> lists:reverse(Jobs) end). - -spec store_state(#state{}) -> ok. store_state(#state{state_id = DocId} = State) -> with_shards_db(fun(Db) -> ok = update_doc(Db, DocId, state_to_ejson_props(State)) end). - -spec load_state(#state{}, atom()) -> #state{}. load_state(#state{state_id = DocId} = State, Default) -> with_shards_db(fun(Db) -> @@ -106,25 +98,25 @@ load_state(#state{state_id = DocId} = State, Default) -> end end). - -spec delete_state(#state{}) -> ok. delete_state(#state{state_id = DocId}) -> with_shards_db(fun(Db) -> ok = delete_doc(Db, DocId) end). - job_to_ejson_props(#job{source = Source, target = Targets} = Job, Opts) -> Iso8601 = proplists:get_value(iso8601, Opts), History = history_to_ejson(Job#job.history, Iso8601), - StartTime = case Iso8601 of - true -> iso8601(Job#job.start_time); - _ -> Job#job.start_time - end, - UpdateTime = case Iso8601 of - true -> iso8601(Job#job.update_time); - _ -> Job#job.update_time - end, + StartTime = + case Iso8601 of + true -> iso8601(Job#job.start_time); + _ -> Job#job.start_time + end, + UpdateTime = + case Iso8601 of + true -> iso8601(Job#job.update_time); + _ -> Job#job.update_time + end, [ {id, Job#job.id}, {type, Job#job.type}, @@ -139,7 +131,6 @@ job_to_ejson_props(#job{source = Source, target = Targets} = Job, Opts) -> {history, History} ]. - state_to_ejson_props(#state{} = State) -> [ {state, atom_to_binary(State#state.state, utf8)}, @@ -148,7 +139,6 @@ state_to_ejson_props(#state{} = State) -> {node, atom_to_binary(State#state.node, utf8)} ]. - % Private API with_shards_db(Fun) -> @@ -164,7 +154,6 @@ with_shards_db(Fun) -> throw(Else) end. - delete_doc(Db, DocId) -> case couch_db:open_doc(Db, DocId, []) of {ok, #doc{revs = {_, Revs}}} -> @@ -174,17 +163,17 @@ delete_doc(Db, DocId) -> ok end. - update_doc(Db, DocId, Body) -> DocProps = [{<<"_id">>, DocId}] ++ Body, Body1 = ?JSON_DECODE(?JSON_ENCODE({DocProps})), BaseDoc = couch_doc:from_json_obj(Body1), - Doc = case couch_db:open_doc(Db, DocId, []) of - {ok, #doc{revs = Revs}} -> - BaseDoc#doc{revs = Revs}; - {not_found, _} -> - BaseDoc - end, + Doc = + case couch_db:open_doc(Db, DocId, []) of + {ok, #doc{revs = Revs}} -> + BaseDoc#doc{revs = Revs}; + {not_found, _} -> + BaseDoc + end, case store_state() of true -> {ok, _} = couch_db:update_doc(Db, Doc, []), @@ -195,7 +184,6 @@ update_doc(Db, DocId, Body) -> ok end. - load_doc(Db, DocId) -> case couch_db:open_doc(Db, DocId, [ejson_body]) of {ok, #doc{body = Body}} -> @@ -205,11 +193,9 @@ load_doc(Db, DocId) -> not_found end. - job_to_ejson_props(#job{} = Job) -> job_to_ejson_props(Job, []). - job_from_ejson({Props}) -> Id = couch_util:get_value(<<"id">>, Props), Type = couch_util:get_value(<<"type">>, Props), @@ -235,7 +221,6 @@ job_from_ejson({Props}) -> history = history_from_ejson(History) }. - state_from_ejson(#state{} = State, {Props}) -> StateVal = couch_util:get_value(<<"state">>, Props), StateInfo = couch_util:get_value(<<"state_info">>, Props), @@ -247,37 +232,35 @@ state_from_ejson(#state{} = State, {Props}) -> update_time = TUpdated }. - state_info_from_ejson({Props}) -> - Props1 = [{binary_to_atom(K, utf8), couch_util:to_binary(V)} - || {K, V} <- Props], + Props1 = [ + {binary_to_atom(K, utf8), couch_util:to_binary(V)} + || {K, V} <- Props + ], lists:sort(Props1). - history_to_ejson(Hist, true) when is_list(Hist) -> [{[{timestamp, iso8601(T)}, {type, S}, {detail, D}]} || {T, S, D} <- Hist]; - history_to_ejson(Hist, _) when is_list(Hist) -> [{[{timestamp, T}, {type, S}, {detail, D}]} || {T, S, D} <- Hist]. - history_from_ejson(HistoryEJson) when is_list(HistoryEJson) -> - lists:map(fun({EventProps}) -> - Timestamp = couch_util:get_value(<<"timestamp">>, EventProps), - State = couch_util:get_value(<<"type">>, EventProps), - Detail = couch_util:get_value(<<"detail">>, EventProps), - {Timestamp, binary_to_atom(State, utf8), Detail} - end, HistoryEJson). - + lists:map( + fun({EventProps}) -> + Timestamp = couch_util:get_value(<<"timestamp">>, EventProps), + State = couch_util:get_value(<<"type">>, EventProps), + Detail = couch_util:get_value(<<"detail">>, EventProps), + {Timestamp, binary_to_atom(State, utf8), Detail} + end, + HistoryEJson + ). state_info_to_ejson(Props) -> {lists:sort([{K, couch_util:to_binary(V)} || {K, V} <- Props])}. - store_state() -> config:get_boolean("reshard", "store_state", true). - iso8601(UnixSec) -> Mega = UnixSec div 1000000, Sec = UnixSec rem 1000000, diff --git a/src/mem3/src/mem3_reshard_sup.erl b/src/mem3/src/mem3_reshard_sup.erl index 6349a4041..5a28359fb 100644 --- a/src/mem3/src/mem3_reshard_sup.erl +++ b/src/mem3/src/mem3_reshard_sup.erl @@ -22,26 +22,15 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). - init(_Args) -> Children = [ - {mem3_reshard_dbdoc, - {mem3_reshard_dbdoc, start_link, []}, - permanent, - infinity, - worker, - [mem3_reshard_dbdoc]}, - {mem3_reshard_job_sup, - {mem3_reshard_job_sup, start_link, []}, - permanent, - infinity, - supervisor, - [mem3_reshard_job_sup]}, - {mem3_reshard, - {mem3_reshard, start_link, []}, - permanent, - brutal_kill, - worker, - [mem3_reshard]} + {mem3_reshard_dbdoc, {mem3_reshard_dbdoc, start_link, []}, permanent, infinity, worker, [ + mem3_reshard_dbdoc + ]}, + {mem3_reshard_job_sup, {mem3_reshard_job_sup, start_link, []}, permanent, infinity, + supervisor, [mem3_reshard_job_sup]}, + {mem3_reshard, {mem3_reshard, start_link, []}, permanent, brutal_kill, worker, [ + mem3_reshard + ]} ], {ok, {{one_for_all, 5, 5}, Children}}. diff --git a/src/mem3/src/mem3_reshard_validate.erl b/src/mem3/src/mem3_reshard_validate.erl index aa8df3e16..fca1617ce 100644 --- a/src/mem3/src/mem3_reshard_validate.erl +++ b/src/mem3/src/mem3_reshard_validate.erl @@ -20,7 +20,6 @@ -include_lib("mem3/include/mem3.hrl"). - -spec start_args(#shard{}, any()) -> ok | {error, term()}. start_args(Source, Split) -> first_error([ @@ -31,8 +30,7 @@ start_args(Source, Split) -> check_shard_map(Source) ]). - --spec source(#shard{}) -> ok | {error, term()}. +-spec source(#shard{}) -> ok | {error, term()}. source(#shard{name = Name}) -> case couch_server:exists(Name) of true -> @@ -41,8 +39,7 @@ source(#shard{name = Name}) -> {error, {source_shard_not_found, Name}} end. - --spec check_shard_map(#shard{}) -> ok | {error, term()}. +-spec check_shard_map(#shard{}) -> ok | {error, term()}. check_shard_map(#shard{name = Name}) -> DbName = mem3:dbname(Name), AllShards = mem3:shards(DbName), @@ -53,22 +50,19 @@ check_shard_map(#shard{name = Name}) -> {error, {not_enough_shard_copies, DbName}} end. - -spec targets(#shard{}, [#shard{}]) -> ok | {error, term()}. targets(#shard{} = Source, Targets) -> first_error([ target_ranges(Source, Targets) ]). - --spec check_split(any()) -> ok | {error, term()}. +-spec check_split(any()) -> ok | {error, term()}. check_split(Split) when is_integer(Split), Split > 1 -> ok; check_split(Split) -> {error, {invalid_split_parameter, Split}}. - --spec check_range(#shard{}, any()) -> ok | {error, term()}. +-spec check_range(#shard{}, any()) -> ok | {error, term()}. check_range(#shard{range = Range = [B, E]}, Split) -> case (E + 1 - B) >= Split of true -> @@ -77,33 +71,33 @@ check_range(#shard{range = Range = [B, E]}, Split) -> {error, {shard_range_cannot_be_split, Range, Split}} end. - --spec check_node(#shard{}) -> ok | {error, term()}. +-spec check_node(#shard{}) -> ok | {error, term()}. check_node(#shard{node = undefined}) -> ok; - check_node(#shard{node = Node}) when Node =:= node() -> ok; - check_node(#shard{node = Node}) -> {error, {source_shard_node_is_not_current_node, Node}}. - -spec target_ranges(#shard{}, [#shard{}]) -> ok | {error, any()}. target_ranges(#shard{range = [Begin, End]}, Targets) -> Ranges = [R || #shard{range = R} <- Targets], SortFun = fun([B1, _], [B2, _]) -> B1 =< B2 end, [First | RestRanges] = lists:sort(SortFun, Ranges), try - TotalRange = lists:foldl(fun([B2, E2], [B1, E1]) -> - case B2 =:= E1 + 1 of - true -> - ok; - false -> - throw({range_error, {B2, E1}}) + TotalRange = lists:foldl( + fun([B2, E2], [B1, E1]) -> + case B2 =:= E1 + 1 of + true -> + ok; + false -> + throw({range_error, {B2, E1}}) + end, + [B1, E2] end, - [B1, E2] - end, First, RestRanges), + First, + RestRanges + ), case [Begin, End] =:= TotalRange of true -> ok; @@ -115,7 +109,6 @@ target_ranges(#shard{range = [Begin, End]}, Targets) -> {error, {shard_range_error, Error}} end. - -spec first_error([ok | {error, term()}]) -> ok | {error, term()}. first_error(Results) -> case [Res || Res <- Results, Res =/= ok] of diff --git a/src/mem3/src/mem3_rpc.erl b/src/mem3/src/mem3_rpc.erl index 9e0f42a8e..468bdee21 100644 --- a/src/mem3/src/mem3_rpc.erl +++ b/src/mem3/src/mem3_rpc.erl @@ -14,7 +14,6 @@ -module(mem3_rpc). - -export([ find_common_seq/4, get_missing_revs/4, @@ -43,18 +42,14 @@ save_purge_checkpoint_rpc/3, replicate_rpc/2 - ]). - -include("mem3.hrl"). -include_lib("couch/include/couch_db.hrl"). - -define(BATCH_SIZE, 1000). -define(REXI_CALL_TIMEOUT_MSEC, 600000). - % "Pull" is a bit of a misnomer here, as what we're actually doing is % issuing an RPC request and telling the remote node to push updates to % us. This lets us reuse all of the battle-tested machinery of mem3_rpc. @@ -64,7 +59,6 @@ pull_replication(Seed) -> get_missing_revs(Node, DbName, IdsRevs, Options) -> rexi_call(Node, {fabric_rpc, get_missing_revs, [DbName, IdsRevs, Options]}). - update_docs(Node, DbName, Docs, Options) -> rexi_call(Node, {fabric_rpc, update_docs, [DbName, Docs, Options]}). @@ -77,95 +71,91 @@ load_checkpoint(Node, DbName, SourceNode, SourceUUID, FilterHash) -> Args = [DbName, SourceNode, SourceUUID, FilterHash], rexi_call(Node, {mem3_rpc, load_checkpoint_rpc, Args}). - load_checkpoint(Node, DbName, SourceNode, SourceUUID) -> Args = [DbName, SourceNode, SourceUUID], rexi_call(Node, {mem3_rpc, load_checkpoint_rpc, Args}). - save_checkpoint(Node, DbName, DocId, Seq, Entry, History) -> Args = [DbName, DocId, Seq, Entry, History], rexi_call(Node, {mem3_rpc, save_checkpoint_rpc, Args}). - find_common_seq(Node, DbName, SourceUUID, SourceEpochs) -> Args = [DbName, SourceUUID, SourceEpochs], rexi_call(Node, {mem3_rpc, find_common_seq_rpc, Args}). - load_purge_infos(Node, DbName, SourceUUID, Count) -> Args = [DbName, SourceUUID, Count], rexi_call(Node, {mem3_rpc, load_purge_infos_rpc, Args}). - save_purge_checkpoint(Node, DbName, PurgeDocId, Body) -> Args = [DbName, PurgeDocId, Body], rexi_call(Node, {mem3_rpc, save_purge_checkpoint_rpc, Args}). - purge_docs(Node, DbName, PurgeInfos, Options) -> rexi_call(Node, {fabric_rpc, purge_docs, [DbName, PurgeInfos, Options]}). - -replicate(Source, Target, DbName, Timeout) - when is_atom(Source), is_atom(Target), is_binary(DbName) -> +replicate(Source, Target, DbName, Timeout) when + is_atom(Source), is_atom(Target), is_binary(DbName) +-> Args = [DbName, Target], rexi_call(Source, {mem3_rpc, replicate_rpc, Args}, Timeout). - load_checkpoint_rpc(DbName, SourceNode, SourceUUID) -> load_checkpoint_rpc(DbName, SourceNode, SourceUUID, <<>>). - load_checkpoint_rpc(DbName, SourceNode, SourceUUID, FilterHash) -> erlang:put(io_priority, {internal_repl, DbName}), case get_or_create_db(DbName, [?ADMIN_CTX]) of - {ok, Db} -> - TargetUUID = couch_db:get_uuid(Db), - NewId = mem3_rep:make_local_id(SourceUUID, TargetUUID, FilterHash), - case couch_db:open_doc(Db, NewId, []) of - {ok, Doc} -> - rexi:reply({ok, {NewId, Doc}}); - {not_found, _} -> - OldId = mem3_rep:make_local_id(SourceNode, node()), - case couch_db:open_doc(Db, OldId, []) of - {ok, Doc} -> - rexi:reply({ok, {NewId, Doc}}); - {not_found, _} -> - rexi:reply({ok, {NewId, #doc{id = NewId}}}) - end - end; - Error -> - rexi:reply(Error) + {ok, Db} -> + TargetUUID = couch_db:get_uuid(Db), + NewId = mem3_rep:make_local_id(SourceUUID, TargetUUID, FilterHash), + case couch_db:open_doc(Db, NewId, []) of + {ok, Doc} -> + rexi:reply({ok, {NewId, Doc}}); + {not_found, _} -> + OldId = mem3_rep:make_local_id(SourceNode, node()), + case couch_db:open_doc(Db, OldId, []) of + {ok, Doc} -> + rexi:reply({ok, {NewId, Doc}}); + {not_found, _} -> + rexi:reply({ok, {NewId, #doc{id = NewId}}}) + end + end; + Error -> + rexi:reply(Error) end. - save_checkpoint_rpc(DbName, Id, SourceSeq, NewEntry0, History0) -> erlang:put(io_priority, {internal_repl, DbName}), case get_or_create_db(DbName, [?ADMIN_CTX]) of {ok, Db} -> - NewEntry = {[ - {<<"target_node">>, atom_to_binary(node(), utf8)}, - {<<"target_uuid">>, couch_db:get_uuid(Db)}, - {<<"target_seq">>, couch_db:get_update_seq(Db)} - ] ++ NewEntry0}, - Body = {[ - {<<"seq">>, SourceSeq}, - {<<"target_uuid">>, couch_db:get_uuid(Db)}, - {<<"history">>, add_checkpoint(NewEntry, History0)} - ]}, + NewEntry = { + [ + {<<"target_node">>, atom_to_binary(node(), utf8)}, + {<<"target_uuid">>, couch_db:get_uuid(Db)}, + {<<"target_seq">>, couch_db:get_update_seq(Db)} + ] ++ NewEntry0 + }, + Body = + {[ + {<<"seq">>, SourceSeq}, + {<<"target_uuid">>, couch_db:get_uuid(Db)}, + {<<"history">>, add_checkpoint(NewEntry, History0)} + ]}, Doc = #doc{id = Id, body = Body}, - rexi:reply(try couch_db:update_doc(Db, Doc, []) of - {ok, _} -> - {ok, Body}; - Else -> - {error, Else} - catch - Exception -> - Exception; - error:Reason -> - {error, Reason} - end); + rexi:reply( + try couch_db:update_doc(Db, Doc, []) of + {ok, _} -> + {ok, Body}; + Else -> + {error, Else} + catch + Exception -> + Exception; + error:Reason -> + {error, Reason} + end + ); Error -> rexi:reply(Error) end. @@ -173,17 +163,17 @@ save_checkpoint_rpc(DbName, Id, SourceSeq, NewEntry0, History0) -> find_common_seq_rpc(DbName, SourceUUID, SourceEpochs) -> erlang:put(io_priority, {internal_repl, DbName}), case get_or_create_db(DbName, [?ADMIN_CTX]) of - {ok, Db} -> - case couch_db:get_uuid(Db) of - SourceUUID -> - TargetEpochs = couch_db:get_epochs(Db), - Seq = compare_epochs(SourceEpochs, TargetEpochs), - rexi:reply({ok, Seq}); - _Else -> - rexi:reply({ok, 0}) - end; - Error -> - rexi:reply(Error) + {ok, Db} -> + case couch_db:get_uuid(Db) of + SourceUUID -> + TargetEpochs = couch_db:get_epochs(Db), + Seq = compare_epochs(SourceEpochs, TargetEpochs), + rexi:reply({ok, Seq}); + _Else -> + rexi:reply({ok, 0}) + end; + Error -> + rexi:reply(Error) end. pull_replication_rpc(Target) -> @@ -192,29 +182,33 @@ pull_replication_rpc(Target) -> Repl = fun(Db) -> {Db, mem3_rep:go(Db, Target, Opts)} end, rexi:reply({ok, lists:map(Repl, Dbs)}). - load_purge_infos_rpc(DbName, SrcUUID, BatchSize) -> erlang:put(io_priority, {internal_repl, DbName}), case get_or_create_db(DbName, [?ADMIN_CTX]) of {ok, Db} -> TgtUUID = couch_db:get_uuid(Db), PurgeDocId = mem3_rep:make_purge_id(SrcUUID, TgtUUID), - StartSeq = case couch_db:open_doc(Db, PurgeDocId, []) of - {ok, #doc{body = {Props}}} -> - couch_util:get_value(<<"purge_seq">>, Props); - {not_found, _} -> - Oldest = couch_db:get_oldest_purge_seq(Db), - erlang:max(0, Oldest - 1) - end, + StartSeq = + case couch_db:open_doc(Db, PurgeDocId, []) of + {ok, #doc{body = {Props}}} -> + couch_util:get_value(<<"purge_seq">>, Props); + {not_found, _} -> + Oldest = couch_db:get_oldest_purge_seq(Db), + erlang:max(0, Oldest - 1) + end, FoldFun = fun({PSeq, UUID, Id, Revs}, {Count, Infos, _}) -> NewCount = Count + length(Revs), NewInfos = [{UUID, Id, Revs} | Infos], - Status = if NewCount < BatchSize -> ok; true -> stop end, + Status = + if + NewCount < BatchSize -> ok; + true -> stop + end, {Status, {NewCount, NewInfos, PSeq}} end, InitAcc = {0, [], StartSeq}, {ok, {_, PurgeInfos, ThroughSeq}} = - couch_db:fold_purge_infos(Db, StartSeq, FoldFun, InitAcc), + couch_db:fold_purge_infos(Db, StartSeq, FoldFun, InitAcc), PurgeSeq = couch_db:get_purge_seq(Db), Remaining = PurgeSeq - ThroughSeq, rexi:reply({ok, {PurgeDocId, PurgeInfos, ThroughSeq, Remaining}}); @@ -222,32 +216,33 @@ load_purge_infos_rpc(DbName, SrcUUID, BatchSize) -> rexi:reply(Else) end. - save_purge_checkpoint_rpc(DbName, PurgeDocId, Body) -> erlang:put(io_priority, {internal_repl, DbName}), case get_or_create_db(DbName, [?ADMIN_CTX]) of {ok, Db} -> Doc = #doc{id = PurgeDocId, body = Body}, - Resp = try couch_db:update_doc(Db, Doc, []) of - Resp0 -> Resp0 - catch T:R -> - {T, R} - end, + Resp = + try couch_db:update_doc(Db, Doc, []) of + Resp0 -> Resp0 + catch + T:R -> + {T, R} + end, rexi:reply(Resp); Error -> rexi:reply(Error) end. - replicate_rpc(DbName, Target) -> - rexi:reply(try - Opts = [{batch_size, ?BATCH_SIZE}, {batch_count, all}], - {ok, mem3_rep:go(DbName, Target, Opts)} - catch - Tag:Error -> - {Tag, Error} - end). - + rexi:reply( + try + Opts = [{batch_size, ?BATCH_SIZE}, {batch_count, all}], + {ok, mem3_rep:go(DbName, Target, Opts)} + catch + Tag:Error -> + {Tag, Error} + end + ). %% @doc Return the sequence where two files with the same UUID diverged. compare_epochs(SourceEpochs, TargetEpochs) -> @@ -256,7 +251,6 @@ compare_epochs(SourceEpochs, TargetEpochs) -> lists:reverse(TargetEpochs) ). - compare_rev_epochs([{Node, Seq} | SourceRest], [{Node, Seq} | TargetRest]) -> % Common history, fast-forward compare_epochs(SourceRest, TargetRest); @@ -270,7 +264,6 @@ compare_rev_epochs([{_, SourceSeq} | _], [{_, TargetSeq} | _]) -> % The source was moved to a new location independently, take the minimum erlang:min(SourceSeq, TargetSeq) - 1. - %% @doc This adds a new update sequence checkpoint to the replication %% history. Checkpoints are keyed by the source node so that we %% aren't mixing history between source shard moves. @@ -298,7 +291,6 @@ add_checkpoint({Props}, {History}) -> NodeRemoved = lists:keydelete(SourceNode, 1, History), {[{SourceNode, NewSourceHistory} | NodeRemoved]}. - filter_history(SourceSeqThresh, TargetSeqThresh, History) -> SourceFilter = fun({Entry}) -> SourceSeq = couch_util:get_value(<<"source_seq">>, Entry), @@ -311,7 +303,6 @@ filter_history(SourceSeqThresh, TargetSeqThresh, History) -> SourceFiltered = lists:filter(SourceFilter, History), lists:filter(TargetFilter, SourceFiltered). - %% @doc This function adjusts our history to maintain a %% history of checkpoints that follow an exponentially %% increasing age from the most recent checkpoint. @@ -335,7 +326,7 @@ filter_history(SourceSeqThresh, TargetSeqThresh, History) -> %% This function expects the provided history to be sorted %% in descending order of source_seq values. rebucket([], _NewSeq, Bucket) -> - {Bucket+1, []}; + {Bucket + 1, []}; rebucket([{Entry} | RestHistory], NewSeq, Bucket) -> CurSeq = couch_util:get_value(<<"source_seq">>, Entry), case find_bucket(NewSeq, CurSeq, Bucket) of @@ -363,7 +354,6 @@ rebucket([{Entry} | RestHistory], NewSeq, Bucket) -> {NextBucket, [{Entry} | NewHistory]} end. - %% @doc Find the bucket id for the given sequence pair. find_bucket(NewSeq, CurSeq, Bucket) -> % The +1 constant in this comparison is a bit subtle. The @@ -371,27 +361,27 @@ find_bucket(NewSeq, CurSeq, Bucket) -> % the history is guaranteed to have a BucketId of 1. This % also relies on never having a duplicated update % sequence so adding 1 here guarantees a difference >= 2. - if (NewSeq - CurSeq + 1) > (2 bsl Bucket) -> - find_bucket(NewSeq, CurSeq, Bucket+1); - true -> - Bucket + if + (NewSeq - CurSeq + 1) > (2 bsl Bucket) -> + find_bucket(NewSeq, CurSeq, Bucket + 1); + true -> + Bucket end. - rexi_call(Node, MFA) -> rexi_call(Node, MFA, ?REXI_CALL_TIMEOUT_MSEC). - rexi_call(Node, MFA, Timeout) -> Mon = rexi_monitor:start([rexi_utils:server_pid(Node)]), Ref = rexi:cast(Node, self(), MFA, [sync]), try - receive {Ref, {ok, Reply}} -> - Reply; - {Ref, Error} -> - erlang:error(Error); - {rexi_DOWN, Mon, _, Reason} -> - erlang:error({rexi_DOWN, {Node, Reason}}) + receive + {Ref, {ok, Reply}} -> + Reply; + {Ref, Error} -> + erlang:error(Error); + {rexi_DOWN, Mon, _, Reason} -> + erlang:error({rexi_DOWN, {Node, Reason}}) after Timeout -> erlang:error(timeout) end @@ -399,15 +389,12 @@ rexi_call(Node, MFA, Timeout) -> rexi_monitor:stop(Mon) end. - get_or_create_db(DbName, Options) -> mem3_util:get_or_create_db_int(DbName, Options). - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). - -define(SNODE, <<"src@localhost">>). -define(TNODE, <<"tgt@localhost">>). -define(SNODE_KV, {<<"source_node">>, ?SNODE}). @@ -416,7 +403,6 @@ get_or_create_db(DbName, Options) -> -define(TSEQ, <<"target_seq">>). -define(ENTRY(S, T), {[?SNODE_KV, {?SSEQ, S}, ?TNODE_KV, {?TSEQ, T}]}). - filter_history_data() -> [ ?ENTRY(13, 15), @@ -424,7 +410,6 @@ filter_history_data() -> ?ENTRY(2, 3) ]. - filter_history_remove_none_test() -> ?assertEqual(filter_history(20, 20, filter_history_data()), [ ?ENTRY(13, 15), @@ -432,11 +417,9 @@ filter_history_remove_none_test() -> ?ENTRY(2, 3) ]). - filter_history_remove_all_test() -> ?assertEqual(filter_history(1, 1, filter_history_data()), []). - filter_history_remove_equal_test() -> ?assertEqual(filter_history(10, 10, filter_history_data()), [ ?ENTRY(2, 3) @@ -445,7 +428,6 @@ filter_history_remove_equal_test() -> ?ENTRY(2, 3) ]). - filter_history_remove_for_source_and_target_test() -> ?assertEqual(filter_history(11, 20, filter_history_data()), [ ?ENTRY(10, 9), @@ -456,129 +438,176 @@ filter_history_remove_for_source_and_target_test() -> ?ENTRY(2, 3) ]). - filter_history_remove_for_both_test() -> ?assertEqual(filter_history(11, 11, filter_history_data()), [ ?ENTRY(10, 9), ?ENTRY(2, 3) ]). - filter_history_remove_for_both_again_test() -> ?assertEqual(filter_history(3, 4, filter_history_data()), [ ?ENTRY(2, 3) ]). - add_first_checkpoint_test() -> History = {[]}, - ?assertEqual(add_checkpoint(?ENTRY(2, 3), History), {[ - {?SNODE, [ - ?ENTRY(2, 3) + ?assertEqual( + add_checkpoint(?ENTRY(2, 3), History), + {[ + {?SNODE, [ + ?ENTRY(2, 3) + ]} ]} - ]}). - + ). add_first_checkpoint_to_empty_test() -> History = {[{?SNODE, []}]}, - ?assertEqual(add_checkpoint(?ENTRY(2, 3), History), {[ - {?SNODE, [ - ?ENTRY(2, 3) + ?assertEqual( + add_checkpoint(?ENTRY(2, 3), History), + {[ + {?SNODE, [ + ?ENTRY(2, 3) + ]} ]} - ]}). - + ). add_second_checkpoint_test() -> History = {[{?SNODE, [?ENTRY(2, 3)]}]}, - ?assertEqual(add_checkpoint(?ENTRY(10, 9), History), {[ - {?SNODE, [ - ?ENTRY(10, 9), - ?ENTRY(2, 3) + ?assertEqual( + add_checkpoint(?ENTRY(10, 9), History), + {[ + {?SNODE, [ + ?ENTRY(10, 9), + ?ENTRY(2, 3) + ]} ]} - ]}). - + ). add_third_checkpoint_test() -> - History = {[{?SNODE, [ - ?ENTRY(10, 9), - ?ENTRY(2, 3) - ]}]}, - ?assertEqual(add_checkpoint(?ENTRY(11, 10), History), {[ - {?SNODE, [ - ?ENTRY(11, 10), - ?ENTRY(10, 9), - ?ENTRY(2, 3) + History = + {[ + {?SNODE, [ + ?ENTRY(10, 9), + ?ENTRY(2, 3) + ]} + ]}, + ?assertEqual( + add_checkpoint(?ENTRY(11, 10), History), + {[ + {?SNODE, [ + ?ENTRY(11, 10), + ?ENTRY(10, 9), + ?ENTRY(2, 3) + ]} ]} - ]}). - + ). add_fourth_checkpoint_test() -> - History = {[{?SNODE, [ - ?ENTRY(11, 10), - ?ENTRY(10, 9), - ?ENTRY(2, 3) - ]}]}, - ?assertEqual(add_checkpoint(?ENTRY(12, 13), History), {[ - {?SNODE, [ - ?ENTRY(12, 13), - ?ENTRY(11, 10), - ?ENTRY(10, 9), - ?ENTRY(2, 3) + History = + {[ + {?SNODE, [ + ?ENTRY(11, 10), + ?ENTRY(10, 9), + ?ENTRY(2, 3) + ]} + ]}, + ?assertEqual( + add_checkpoint(?ENTRY(12, 13), History), + {[ + {?SNODE, [ + ?ENTRY(12, 13), + ?ENTRY(11, 10), + ?ENTRY(10, 9), + ?ENTRY(2, 3) + ]} ]} - ]}). - + ). add_checkpoint_with_replacement_test() -> - History = {[{?SNODE, [ - ?ENTRY(12, 13), - ?ENTRY(11, 10), - ?ENTRY(10, 9), - ?ENTRY(2, 3) - ]}]}, + History = + {[ + {?SNODE, [ + ?ENTRY(12, 13), + ?ENTRY(11, 10), + ?ENTRY(10, 9), + ?ENTRY(2, 3) + ]} + ]}, % Picking a source_seq of 16 to force 10, 11, and 12 % into the same bucket to show we drop the 11 entry. - ?assertEqual(add_checkpoint(?ENTRY(16, 16), History), {[ - {?SNODE, [ - ?ENTRY(16, 16), - ?ENTRY(12, 13), - ?ENTRY(10, 9), - ?ENTRY(2, 3) + ?assertEqual( + add_checkpoint(?ENTRY(16, 16), History), + {[ + {?SNODE, [ + ?ENTRY(16, 16), + ?ENTRY(12, 13), + ?ENTRY(10, 9), + ?ENTRY(2, 3) + ]} ]} - ]}). + ). add_checkpoint_drops_redundant_checkpoints_test() -> % I've added comments showing the bucket ID based % on the ?ENTRY passed to add_checkpoint - History = {[{?SNODE, [ - ?ENTRY(15, 15), % Bucket 0 - ?ENTRY(14, 14), % Bucket 1 - ?ENTRY(13, 13), % Bucket 1 - ?ENTRY(12, 12), % Bucket 2 - ?ENTRY(11, 11), % Bucket 2 - ?ENTRY(10, 10), % Bucket 2 - ?ENTRY(9, 9), % Bucket 2 - ?ENTRY(8, 8), % Bucket 3 - ?ENTRY(7, 7), % Bucket 3 - ?ENTRY(6, 6), % Bucket 3 - ?ENTRY(5, 5), % Bucket 3 - ?ENTRY(4, 4), % Bucket 3 - ?ENTRY(3, 3), % Bucket 3 - ?ENTRY(2, 2), % Bucket 3 - ?ENTRY(1, 1) % Bucket 3 - ]}]}, - ?assertEqual(add_checkpoint(?ENTRY(16, 16), History), {[ - {?SNODE, [ - ?ENTRY(16, 16), % Bucket 0 - ?ENTRY(15, 15), % Bucket 0 - ?ENTRY(14, 14), % Bucket 1 - ?ENTRY(13, 13), % Bucket 1 - ?ENTRY(12, 12), % Bucket 2 - ?ENTRY(9, 9), % Bucket 2 - ?ENTRY(8, 8), % Bucket 3 - ?ENTRY(1, 1) % Bucket 3 + History = + {[ + {?SNODE, [ + % Bucket 0 + ?ENTRY(15, 15), + % Bucket 1 + ?ENTRY(14, 14), + % Bucket 1 + ?ENTRY(13, 13), + % Bucket 2 + ?ENTRY(12, 12), + % Bucket 2 + ?ENTRY(11, 11), + % Bucket 2 + ?ENTRY(10, 10), + % Bucket 2 + ?ENTRY(9, 9), + % Bucket 3 + ?ENTRY(8, 8), + % Bucket 3 + ?ENTRY(7, 7), + % Bucket 3 + ?ENTRY(6, 6), + % Bucket 3 + ?ENTRY(5, 5), + % Bucket 3 + ?ENTRY(4, 4), + % Bucket 3 + ?ENTRY(3, 3), + % Bucket 3 + ?ENTRY(2, 2), + % Bucket 3 + ?ENTRY(1, 1) + ]} + ]}, + ?assertEqual( + add_checkpoint(?ENTRY(16, 16), History), + {[ + {?SNODE, [ + % Bucket 0 + ?ENTRY(16, 16), + % Bucket 0 + ?ENTRY(15, 15), + % Bucket 1 + ?ENTRY(14, 14), + % Bucket 1 + ?ENTRY(13, 13), + % Bucket 2 + ?ENTRY(12, 12), + % Bucket 2 + ?ENTRY(9, 9), + % Bucket 3 + ?ENTRY(8, 8), + % Bucket 3 + ?ENTRY(1, 1) + ]} ]} - ]}). - + ). add_checkpoint_show_not_always_a_drop_test() -> % Depending on the edge conditions of buckets we @@ -588,124 +617,181 @@ add_checkpoint_show_not_always_a_drop_test() -> % % I've added comments showing the bucket ID based % on the ?ENTRY passed to add_checkpoint - History = {[{?SNODE, [ - ?ENTRY(16, 16), % Bucket 0 - ?ENTRY(15, 15), % Bucket 1 - ?ENTRY(14, 14), % Bucket 1 - ?ENTRY(13, 13), % Bucket 2 - ?ENTRY(12, 12), % Bucket 2 - ?ENTRY(9, 9), % Bucket 3 - ?ENTRY(8, 8), % Bucket 3 - ?ENTRY(1, 1) % Bucket 4 - ]}]}, - ?assertEqual(add_checkpoint(?ENTRY(17, 17), History), {[ - {?SNODE, [ - ?ENTRY(17, 17), % Bucket 0 - ?ENTRY(16, 16), % Bucket 0 - ?ENTRY(15, 15), % Bucket 1 - ?ENTRY(14, 14), % Bucket 1 - ?ENTRY(13, 13), % Bucket 2 - ?ENTRY(12, 12), % Bucket 2 - ?ENTRY(9, 9), % Bucket 3 - ?ENTRY(8, 8), % Bucket 3 - ?ENTRY(1, 1) % Bucket 4 + History = + {[ + {?SNODE, [ + % Bucket 0 + ?ENTRY(16, 16), + % Bucket 1 + ?ENTRY(15, 15), + % Bucket 1 + ?ENTRY(14, 14), + % Bucket 2 + ?ENTRY(13, 13), + % Bucket 2 + ?ENTRY(12, 12), + % Bucket 3 + ?ENTRY(9, 9), + % Bucket 3 + ?ENTRY(8, 8), + % Bucket 4 + ?ENTRY(1, 1) + ]} + ]}, + ?assertEqual( + add_checkpoint(?ENTRY(17, 17), History), + {[ + {?SNODE, [ + % Bucket 0 + ?ENTRY(17, 17), + % Bucket 0 + ?ENTRY(16, 16), + % Bucket 1 + ?ENTRY(15, 15), + % Bucket 1 + ?ENTRY(14, 14), + % Bucket 2 + ?ENTRY(13, 13), + % Bucket 2 + ?ENTRY(12, 12), + % Bucket 3 + ?ENTRY(9, 9), + % Bucket 3 + ?ENTRY(8, 8), + % Bucket 4 + ?ENTRY(1, 1) + ]} ]} - ]}). - + ). add_checkpoint_big_jump_show_lots_drop_test() -> % I've added comments showing the bucket ID based % on the ?ENTRY passed to add_checkpoint - History = {[{?SNODE, [ - ?ENTRY(16, 16), % Bucket 4 - ?ENTRY(15, 15), % Bucket 4 - ?ENTRY(14, 14), % Bucket 4 - ?ENTRY(13, 13), % Bucket 4 - ?ENTRY(12, 12), % Bucket 4 - ?ENTRY(9, 9), % Bucket 4 - ?ENTRY(8, 8), % Bucket 4 - ?ENTRY(1, 1) % Bucket 4 - ]}]}, - ?assertEqual(add_checkpoint(?ENTRY(32, 32), History), {[ - {?SNODE, [ - ?ENTRY(32, 32), % Bucket 0 - ?ENTRY(16, 16), % Bucket 4 - ?ENTRY(1, 1) % Bucket 4 + History = + {[ + {?SNODE, [ + % Bucket 4 + ?ENTRY(16, 16), + % Bucket 4 + ?ENTRY(15, 15), + % Bucket 4 + ?ENTRY(14, 14), + % Bucket 4 + ?ENTRY(13, 13), + % Bucket 4 + ?ENTRY(12, 12), + % Bucket 4 + ?ENTRY(9, 9), + % Bucket 4 + ?ENTRY(8, 8), + % Bucket 4 + ?ENTRY(1, 1) + ]} + ]}, + ?assertEqual( + add_checkpoint(?ENTRY(32, 32), History), + {[ + {?SNODE, [ + % Bucket 0 + ?ENTRY(32, 32), + % Bucket 4 + ?ENTRY(16, 16), + % Bucket 4 + ?ENTRY(1, 1) + ]} ]} - ]}). - + ). add_checkpoint_show_filter_history_test() -> - History = {[{?SNODE, [ - ?ENTRY(16, 16), - ?ENTRY(15, 15), - ?ENTRY(14, 14), - ?ENTRY(13, 13), - ?ENTRY(12, 12), - ?ENTRY(9, 9), - ?ENTRY(8, 8), - ?ENTRY(1, 1) - ]}]}, + History = + {[ + {?SNODE, [ + ?ENTRY(16, 16), + ?ENTRY(15, 15), + ?ENTRY(14, 14), + ?ENTRY(13, 13), + ?ENTRY(12, 12), + ?ENTRY(9, 9), + ?ENTRY(8, 8), + ?ENTRY(1, 1) + ]} + ]}, % Drop for both - ?assertEqual(add_checkpoint(?ENTRY(10, 10), History), {[ - {?SNODE, [ - ?ENTRY(10, 10), - ?ENTRY(9, 9), - ?ENTRY(8, 8), - ?ENTRY(1, 1) + ?assertEqual( + add_checkpoint(?ENTRY(10, 10), History), + {[ + {?SNODE, [ + ?ENTRY(10, 10), + ?ENTRY(9, 9), + ?ENTRY(8, 8), + ?ENTRY(1, 1) + ]} ]} - ]}), + ), % Drop four source - ?assertEqual(add_checkpoint(?ENTRY(10, 200), History), {[ - {?SNODE, [ - ?ENTRY(10, 200), - ?ENTRY(9, 9), - ?ENTRY(8, 8), - ?ENTRY(1, 1) + ?assertEqual( + add_checkpoint(?ENTRY(10, 200), History), + {[ + {?SNODE, [ + ?ENTRY(10, 200), + ?ENTRY(9, 9), + ?ENTRY(8, 8), + ?ENTRY(1, 1) + ]} ]} - ]}), + ), % Drop for target. Obviously a source_seq of 200 % will end up droping the 8 entry. - ?assertEqual(add_checkpoint(?ENTRY(200, 10), History), {[ - {?SNODE, [ - ?ENTRY(200, 10), - ?ENTRY(9, 9), - ?ENTRY(1, 1) + ?assertEqual( + add_checkpoint(?ENTRY(200, 10), History), + {[ + {?SNODE, [ + ?ENTRY(200, 10), + ?ENTRY(9, 9), + ?ENTRY(1, 1) + ]} ]} - ]}). - + ). add_checkpoint_from_other_node_test() -> - History = {[{<<"not_the_source">>, [ - ?ENTRY(12, 13), - ?ENTRY(11, 10), - ?ENTRY(10, 9), - ?ENTRY(2, 3) - ]}]}, - % No filtering - ?assertEqual(add_checkpoint(?ENTRY(1, 1), History), {[ - {?SNODE, [ - ?ENTRY(1, 1) + History = + {[ + {<<"not_the_source">>, [ + ?ENTRY(12, 13), + ?ENTRY(11, 10), + ?ENTRY(10, 9), + ?ENTRY(2, 3) + ]} ]}, - {<<"not_the_source">>, [ - ?ENTRY(12, 13), - ?ENTRY(11, 10), - ?ENTRY(10, 9), - ?ENTRY(2, 3) + % No filtering + ?assertEqual( + add_checkpoint(?ENTRY(1, 1), History), + {[ + {?SNODE, [ + ?ENTRY(1, 1) + ]}, + {<<"not_the_source">>, [ + ?ENTRY(12, 13), + ?ENTRY(11, 10), + ?ENTRY(10, 9), + ?ENTRY(2, 3) + ]} ]} - ]}), + ), % No dropping - ?assertEqual(add_checkpoint(?ENTRY(200, 200), History), {[ - {?SNODE, [ - ?ENTRY(200, 200) - ]}, - {<<"not_the_source">>, [ - ?ENTRY(12, 13), - ?ENTRY(11, 10), - ?ENTRY(10, 9), - ?ENTRY(2, 3) + ?assertEqual( + add_checkpoint(?ENTRY(200, 200), History), + {[ + {?SNODE, [ + ?ENTRY(200, 200) + ]}, + {<<"not_the_source">>, [ + ?ENTRY(12, 13), + ?ENTRY(11, 10), + ?ENTRY(10, 9), + ?ENTRY(2, 3) + ]} ]} - ]}). - + ). -endif. diff --git a/src/mem3/src/mem3_seeds.erl b/src/mem3/src/mem3_seeds.erl index f1aceb996..6d74398e7 100644 --- a/src/mem3/src/mem3_seeds.erl +++ b/src/mem3/src/mem3_seeds.erl @@ -32,7 +32,8 @@ ready = false, seeds = [], jobref = nil, - status = [] % nested proplist keyed on node name + % nested proplist keyed on node name + status = [] }). -define(REPLICATION_INTERVAL, 60000). @@ -58,29 +59,36 @@ init([]) -> InitStatus = [{Seed, {[]}} || Seed <- Seeds], State = #st{ seeds = Seeds, - ready = case Seeds of [] -> true; _ -> false end, + ready = + case Seeds of + [] -> true; + _ -> false + end, jobref = start_replication(Seeds), status = InitStatus }, {ok, State}. handle_call(get_status, _From, St) -> - Status = {[ - {status, case St#st.ready of true -> ok; false -> seeding end}, - {seeds, {St#st.status}} - ]}, + Status = + {[ + {status, + case St#st.ready of + true -> ok; + false -> seeding + end}, + {seeds, {St#st.status}} + ]}, {reply, {ok, Status}, St}. handle_cast(_Msg, St) -> {noreply, St}. -handle_info(start_replication, #st{jobref=nil} = St) -> +handle_info(start_replication, #st{jobref = nil} = St) -> JobRef = start_replication(St#st.seeds), {noreply, St#st{jobref = JobRef}}; - handle_info({'DOWN', Ref, _, Pid, Output}, #st{jobref = {Pid, Ref}} = St) -> {noreply, update_state(St, Output)}; - handle_info(_Msg, St) -> {noreply, St}. @@ -102,11 +110,12 @@ start_replication([Seed | _]) -> update_state(State, {ok, Data}) -> #st{seeds = [Current | Tail], status = Status} = State, - Report = {[ - {timestamp, list_to_binary(mem3_util:iso8601_timestamp())}, - {last_replication_status, ok}, - format_data(Data) - ]}, + Report = + {[ + {timestamp, list_to_binary(mem3_util:iso8601_timestamp())}, + {last_replication_status, ok}, + format_data(Data) + ]}, NewStatus = lists:ukeymerge(1, [{Current, Report}], Status), Ready = is_ready(State#st.ready, Data), case Ready of @@ -126,16 +135,18 @@ update_state(State, {ok, Data}) -> }; update_state(State, {_Error, _Stack}) -> #st{seeds = [Current | Tail], status = Status} = State, - Report = {[ - {timestamp, list_to_binary(mem3_util:iso8601_timestamp())}, - {last_replication_status, error} - ]}, + Report = + {[ + {timestamp, list_to_binary(mem3_util:iso8601_timestamp())}, + {last_replication_status, error} + ]}, NewStatus = lists:ukeymerge(1, [{Current, Report}], Status), Seeds = Tail ++ [Current], - if not State#st.ready -> - erlang:send_after(1000, self(), start_replication); - true -> - ok + if + not State#st.ready -> + erlang:send_after(1000, self(), start_replication); + true -> + ok end, State#st{ seeds = Seeds, @@ -149,14 +160,17 @@ is_ready(false, Data) -> lists:all(fun({_DbName, Pending}) -> Pending =:= {ok, 0} end, Data). format_data(Data) -> - Formatted = lists:map(fun({DbName, Status}) -> - case Status of - {ok, Pending} when is_number(Pending) -> - {DbName, Pending}; - {error, Tag} -> - {DbName, list_to_binary(io_lib:format("~p", [Tag]))}; - _Else -> - {DbName, unknown_error} - end - end, Data), + Formatted = lists:map( + fun({DbName, Status}) -> + case Status of + {ok, Pending} when is_number(Pending) -> + {DbName, Pending}; + {error, Tag} -> + {DbName, list_to_binary(io_lib:format("~p", [Tag]))}; + _Else -> + {DbName, unknown_error} + end + end, + Data + ), {pending_updates, {Formatted}}. diff --git a/src/mem3/src/mem3_shards.erl b/src/mem3/src/mem3_shards.erl index fd1894abe..8bbc92411 100644 --- a/src/mem3/src/mem3_shards.erl +++ b/src/mem3/src/mem3_shards.erl @@ -60,17 +60,19 @@ for_db(DbName) -> for_db(DbName, []). for_db(DbName, Options) -> - Shards = try ets:lookup(?SHARDS, DbName) of - [] -> - load_shards_from_disk(DbName); - Else -> - gen_server:cast(?MODULE, {cache_hit, DbName}), - Else - catch error:badarg -> - load_shards_from_disk(DbName) - end, + Shards = + try ets:lookup(?SHARDS, DbName) of + [] -> + load_shards_from_disk(DbName); + Else -> + gen_server:cast(?MODULE, {cache_hit, DbName}), + Else + catch + error:badarg -> + load_shards_from_disk(DbName) + end, case lists:member(ordered, Options) of - true -> Shards; + true -> Shards; false -> mem3_util:downcast(Shards) end. @@ -92,17 +94,19 @@ for_docid(DbName, DocId, Options) -> Conditions = [{'=<', '$1', HashKey}, {'=<', HashKey, '$2'}], ShardSpec = {ShardHead, Conditions, ['$_']}, OrderedShardSpec = {OrderedShardHead, Conditions, ['$_']}, - Shards = try ets:select(?SHARDS, [ShardSpec, OrderedShardSpec]) of - [] -> - load_shards_from_disk(DbName, DocId); - Else -> - gen_server:cast(?MODULE, {cache_hit, DbName}), - Else - catch error:badarg -> - load_shards_from_disk(DbName, DocId) - end, + Shards = + try ets:select(?SHARDS, [ShardSpec, OrderedShardSpec]) of + [] -> + load_shards_from_disk(DbName, DocId); + Else -> + gen_server:cast(?MODULE, {cache_hit, DbName}), + Else + catch + error:badarg -> + load_shards_from_disk(DbName, DocId) + end, case lists:member(ordered, Options) of - true -> Shards; + true -> Shards; false -> mem3_util:downcast(Shards) end. @@ -123,35 +127,43 @@ for_shard_range(ShardName) -> Conditions = [{'=<', '$1', E}, {'=<', B, '$2'}], ShardSpec = {ShardHead, Conditions, ['$_']}, OrderedShardSpec = {OrderedShardHead, Conditions, ['$_']}, - Shards = try ets:select(?SHARDS, [ShardSpec, OrderedShardSpec]) of - [] -> - filter_shards_by_range([B, E], load_shards_from_disk(DbName)); - Else -> - gen_server:cast(?MODULE, {cache_hit, DbName}), - Else - catch error:badarg -> - filter_shards_by_range([B, E], load_shards_from_disk(DbName)) - end, + Shards = + try ets:select(?SHARDS, [ShardSpec, OrderedShardSpec]) of + [] -> + filter_shards_by_range([B, E], load_shards_from_disk(DbName)); + Else -> + gen_server:cast(?MODULE, {cache_hit, DbName}), + Else + catch + error:badarg -> + filter_shards_by_range([B, E], load_shards_from_disk(DbName)) + end, mem3_util:downcast(Shards). - get(DbName, Node, Range) -> - Res = lists:foldl(fun(#shard{node=N, range=R}=S, Acc) -> - case {N, R} of - {Node, Range} -> [S | Acc]; - _ -> Acc - end - end, [], for_db(DbName)), + Res = lists:foldl( + fun(#shard{node = N, range = R} = S, Acc) -> + case {N, R} of + {Node, Range} -> [S | Acc]; + _ -> Acc + end + end, + [], + for_db(DbName) + ), case Res of [] -> {error, not_found}; [Shard] -> {ok, Shard}; - [_|_] -> {error, duplicates} + [_ | _] -> {error, duplicates} end. local(DbName) when is_list(DbName) -> local(list_to_binary(DbName)); local(DbName) -> - Pred = fun(#shard{node=Node}) when Node == node() -> true; (_) -> false end, + Pred = fun + (#shard{node = Node}) when Node == node() -> true; + (_) -> false + end, lists:filter(Pred, for_db(DbName)). fold(Fun, Acc) -> @@ -177,11 +189,13 @@ handle_config_change("mem3", "shard_cache_size", SizeList, _, _) -> handle_config_change("mem3", "shards_db", _DbName, _, _) -> {ok, gen_server:call(?MODULE, shard_db_changed, infinity)}; handle_config_change("mem3", "shard_write_timeout", Timeout, _, _) -> - Timeout = try - list_to_integer(Timeout) - catch _:_ -> - 1000 - end, + Timeout = + try + list_to_integer(Timeout) + catch + _:_ -> + 1000 + end, {ok, gen_server:call(?MODULE, {set_write_timeout, Timeout})}; handle_config_change(_, _, _, _, _) -> {ok, nil}. @@ -197,7 +211,7 @@ init([]) -> bag, public, named_table, - {keypos,#shard.dbname}, + {keypos, #shard.dbname}, {read_concurrency, true} ]), ets:new(?DBS, [set, protected, named_table]), @@ -216,7 +230,7 @@ init([]) -> }}. handle_call({set_max_size, Size}, _From, St) -> - {reply, ok, cache_free(St#st{max_size=Size})}; + {reply, ok, cache_free(St#st{max_size = Size})}; handle_call(shard_db_changed, _From, St) -> exit(St#st.changes_pid, shard_db_changed), {reply, ok, St}; @@ -237,13 +251,14 @@ handle_cast({cache_insert, DbName, Writer, UpdateSeq}, St) -> % to think of when a _dbs db doesn't change. If it used % `=<` it would be impossible to insert anything into % the cache. - NewSt = case UpdateSeq < St#st.update_seq of - true -> - Writer ! cancel, - St; - false -> - cache_free(cache_insert(St, DbName, Writer, St#st.write_timeout)) - end, + NewSt = + case UpdateSeq < St#st.update_seq of + true -> + Writer ! cancel, + St; + false -> + cache_free(cache_insert(St, DbName, Writer, St#st.write_timeout)) + end, {noreply, NewSt}; handle_cast({cache_remove, DbName}, St) -> couch_stats:increment_counter([mem3, shard_cache, eviction]), @@ -258,18 +273,19 @@ handle_cast({cache_remove_change, DbName, UpdateSeq}, St) -> handle_cast(_Msg, St) -> {noreply, St}. -handle_info({'DOWN', _, _, Pid, Reason}, #st{changes_pid=Pid}=St) -> - {NewSt, Seq} = case Reason of - {seq, EndSeq} -> - {St, EndSeq}; - shard_db_changed -> - {cache_clear(St), get_update_seq()}; - _ -> - couch_log:notice("~p changes listener died ~p", [?MODULE, Reason]), - {St, get_update_seq()} - end, +handle_info({'DOWN', _, _, Pid, Reason}, #st{changes_pid = Pid} = St) -> + {NewSt, Seq} = + case Reason of + {seq, EndSeq} -> + {St, EndSeq}; + shard_db_changed -> + {cache_clear(St), get_update_seq()}; + _ -> + couch_log:notice("~p changes listener died ~p", [?MODULE, Reason]), + {St, get_update_seq()} + end, erlang:send_after(5000, self(), {start_listener, Seq}), - {noreply, NewSt#st{changes_pid=undefined}}; + {noreply, NewSt#st{changes_pid = undefined}}; handle_info({start_listener, Seq}, St) -> {noreply, St#st{ changes_pid = start_changes_listener(Seq) @@ -280,11 +296,11 @@ handle_info(restart_config_listener, State) -> handle_info(_Msg, St) -> {noreply, St}. -terminate(_Reason, #st{changes_pid=Pid}) -> +terminate(_Reason, #st{changes_pid = Pid}) -> exit(Pid, kill), ok. -code_change(_OldVsn, #st{}=St, _Extra) -> +code_change(_OldVsn, #st{} = St, _Extra) -> {ok, St}. %% internal functions @@ -304,10 +320,10 @@ start_changes_listener(SinceSeq) -> end), Pid. -fold_fun(#full_doc_info{}=FDI, Acc) -> +fold_fun(#full_doc_info{} = FDI, Acc) -> DI = couch_doc:to_doc_info(FDI), fold_fun(DI, Acc); -fold_fun(#doc_info{}=DI, {Db, UFun, UAcc}) -> +fold_fun(#doc_info{} = DI, {Db, UFun, UAcc}) -> case couch_db:open_doc(Db, DI, [ejson_body, conflicts]) of {ok, Doc} -> {Props} = Doc#doc.body, @@ -342,27 +358,37 @@ changes_callback({stop, EndSeq}, _) -> changes_callback({change, {Change}, _}, _) -> DbName = couch_util:get_value(<<"id">>, Change), Seq = couch_util:get_value(<<"seq">>, Change), - case DbName of <<"_design/", _/binary>> -> ok; _Else -> - case mem3_util:is_deleted(Change) of - true -> - gen_server:cast(?MODULE, {cache_remove_change, DbName, Seq}); - false -> - case couch_util:get_value(doc, Change) of - {error, Reason} -> - couch_log:error("missing partition table for ~s: ~p", - [DbName, Reason]); - {Doc} -> - Shards = mem3_util:build_ordered_shards(DbName, Doc), - IdleTimeout = config:get_integer( - "mem3", "writer_idle_timeout", 30000), - Writer = spawn_shard_writer(DbName, Shards, IdleTimeout), - ets:insert(?OPENERS, {DbName, Writer}), - Msg = {cache_insert_change, DbName, Writer, Seq}, - gen_server:cast(?MODULE, Msg), - [create_if_missing(mem3:name(S)) || S - <- Shards, mem3:node(S) =:= node()] + case DbName of + <<"_design/", _/binary>> -> + ok; + _Else -> + case mem3_util:is_deleted(Change) of + true -> + gen_server:cast(?MODULE, {cache_remove_change, DbName, Seq}); + false -> + case couch_util:get_value(doc, Change) of + {error, Reason} -> + couch_log:error( + "missing partition table for ~s: ~p", + [DbName, Reason] + ); + {Doc} -> + Shards = mem3_util:build_ordered_shards(DbName, Doc), + IdleTimeout = config:get_integer( + "mem3", "writer_idle_timeout", 30000 + ), + Writer = spawn_shard_writer(DbName, Shards, IdleTimeout), + ets:insert(?OPENERS, {DbName, Writer}), + Msg = {cache_insert_change, DbName, Writer, Seq}, + gen_server:cast(?MODULE, Msg), + [ + create_if_missing(mem3:name(S)) + || S <- + Shards, + mem3:node(S) =:= node() + ] + end end - end end, {ok, Seq}; changes_callback(timeout, _) -> @@ -379,28 +405,28 @@ load_shards_from_disk(DbName) when is_binary(DbName) -> load_shards_from_db(ShardDb, DbName) -> case couch_db:open_doc(ShardDb, DbName, [ejson_body]) of - {ok, #doc{body = {Props}}} -> - Seq = couch_db:get_update_seq(ShardDb), - Shards = mem3_util:build_ordered_shards(DbName, Props), - IdleTimeout = config:get_integer("mem3", "writer_idle_timeout", 30000), - case maybe_spawn_shard_writer(DbName, Shards, IdleTimeout) of - Writer when is_pid(Writer) -> - case ets:insert_new(?OPENERS, {DbName, Writer}) of - true -> - Msg = {cache_insert, DbName, Writer, Seq}, - gen_server:cast(?MODULE, Msg); - false -> - Writer ! cancel - end; - ignore -> - ok - end, - Shards; - {not_found, _} -> - erlang:error(database_does_not_exist, ?b2l(DbName)) + {ok, #doc{body = {Props}}} -> + Seq = couch_db:get_update_seq(ShardDb), + Shards = mem3_util:build_ordered_shards(DbName, Props), + IdleTimeout = config:get_integer("mem3", "writer_idle_timeout", 30000), + case maybe_spawn_shard_writer(DbName, Shards, IdleTimeout) of + Writer when is_pid(Writer) -> + case ets:insert_new(?OPENERS, {DbName, Writer}) of + true -> + Msg = {cache_insert, DbName, Writer, Seq}, + gen_server:cast(?MODULE, Msg); + false -> + Writer ! cancel + end; + ignore -> + ok + end, + Shards; + {not_found, _} -> + erlang:error(database_does_not_exist, ?b2l(DbName)) end. -load_shards_from_disk(DbName, DocId)-> +load_shards_from_disk(DbName, DocId) -> Shards = load_shards_from_disk(DbName), HashKey = mem3_hash:calculate(hd(Shards), DocId), [S || S <- Shards, in_range(S, HashKey)]. @@ -416,15 +442,17 @@ create_if_missing(ShardName) -> false -> Options = opts_for_db(ShardName), case couch_server:create(ShardName, [?ADMIN_CTX] ++ Options) of - {ok, Db} -> - couch_db:close(Db); - Error -> - couch_log:error("~p tried to create ~s, got ~p", - [?MODULE, ShardName, Error]) + {ok, Db} -> + couch_db:close(Db); + Error -> + couch_log:error( + "~p tried to create ~s, got ~p", + [?MODULE, ShardName, Error] + ) end end. -cache_insert(#st{cur_size=Cur}=St, DbName, Writer, Timeout) -> +cache_insert(#st{cur_size = Cur} = St, DbName, Writer, Timeout) -> NewATime = couch_util:unique_monotonic_integer(), true = ets:delete(?SHARDS, DbName), flush_write(DbName, Writer, Timeout), @@ -437,16 +465,16 @@ cache_insert(#st{cur_size=Cur}=St, DbName, Writer, Timeout) -> [] -> true = ets:insert(?ATIMES, {NewATime, DbName}), true = ets:insert(?DBS, {DbName, NewATime}), - St#st{cur_size=Cur + 1} + St#st{cur_size = Cur + 1} end. -cache_remove(#st{cur_size=Cur}=St, DbName) -> +cache_remove(#st{cur_size = Cur} = St, DbName) -> true = ets:delete(?SHARDS, DbName), case ets:lookup(?DBS, DbName) of [{DbName, ATime}] -> true = ets:delete(?DBS, DbName), true = ets:delete(?ATIMES, ATime), - St#st{cur_size=Cur-1}; + St#st{cur_size = Cur - 1}; [] -> St end. @@ -462,13 +490,13 @@ cache_hit(DbName) -> ok end. -cache_free(#st{max_size=Max, cur_size=Cur}=St) when Max =< Cur -> +cache_free(#st{max_size = Max, cur_size = Cur} = St) when Max =< Cur -> ATime = ets:first(?ATIMES), [{ATime, DbName}] = ets:lookup(?ATIMES, ATime), true = ets:delete(?ATIMES, ATime), true = ets:delete(?DBS, DbName), true = ets:delete(?SHARDS, DbName), - cache_free(St#st{cur_size=Cur-1}); + cache_free(St#st{cur_size = Cur - 1}); cache_free(St) -> St. @@ -476,7 +504,7 @@ cache_clear(St) -> true = ets:delete_all_objects(?DBS), true = ets:delete_all_objects(?SHARDS), true = ets:delete_all_objects(?ATIMES), - St#st{cur_size=0}. + St#st{cur_size = 0}. maybe_spawn_shard_writer(DbName, Shards, IdleTimeout) -> case ets:member(?OPENERS, DbName) of @@ -515,13 +543,14 @@ flush_write(DbName, Writer, WriteTimeout) -> erlang:exit({mem3_shards_write_timeout, DbName}) end. - -filter_shards_by_range(Range, Shards)-> - lists:filter(fun - (#ordered_shard{range = R}) -> mem3_util:range_overlap(Range, R); - (#shard{range = R}) -> mem3_util:range_overlap(Range, R) - end, Shards). - +filter_shards_by_range(Range, Shards) -> + lists:filter( + fun + (#ordered_shard{range = R}) -> mem3_util:range_overlap(Range, R); + (#shard{range = R}) -> mem3_util:range_overlap(Range, R) + end, + Shards + ). -ifdef(TEST). @@ -530,7 +559,6 @@ filter_shards_by_range(Range, Shards)-> -define(DB, <<"eunit_db_name">>). -define(INFINITY, 99999999). - mem3_shards_test_() -> { setup, @@ -555,7 +583,6 @@ mem3_shards_test_() -> } }. - setup_all() -> ets:new(?SHARDS, [bag, public, named_table, {keypos, #shard.dbname}]), ets:new(?OPENERS, [bag, public, named_table]), @@ -564,7 +591,6 @@ setup_all() -> meck:expect(config, get, ["mem3", "shards_db", '_'], "_dbs"), ok. - teardown_all(_) -> meck:unload(), ets:delete(?ATIMES), @@ -572,18 +598,15 @@ teardown_all(_) -> ets:delete(?OPENERS), ets:delete(?SHARDS). - setup() -> ets:delete_all_objects(?ATIMES), ets:delete_all_objects(?DBS), ets:delete_all_objects(?OPENERS), ets:delete_all_objects(?SHARDS). - teardown(_) -> ok. - t_maybe_spawn_shard_writer_already_exists() -> ?_test(begin ets:insert(?OPENERS, {?DB, self()}), @@ -592,7 +615,6 @@ t_maybe_spawn_shard_writer_already_exists() -> ?assertEqual(ignore, WRes) end). - t_maybe_spawn_shard_writer_new() -> ?_test(begin Shards = mock_shards(), @@ -605,7 +627,6 @@ t_maybe_spawn_shard_writer_new() -> ?assertEqual(Shards, ets:tab2list(?SHARDS)) end). - t_flush_writer_exists_normal() -> ?_test(begin Shards = mock_shards(), @@ -614,24 +635,29 @@ t_flush_writer_exists_normal() -> ?assertEqual(Shards, ets:tab2list(?SHARDS)) end). - t_flush_writer_times_out() -> ?_test(begin - WPid = spawn(fun() -> receive will_never_receive_this -> ok end end), + WPid = spawn(fun() -> + receive + will_never_receive_this -> ok + end + end), Error = {mem3_shards_write_timeout, ?DB}, ?assertExit(Error, flush_write(?DB, WPid, 100)), exit(WPid, kill) end). - t_flush_writer_crashes() -> ?_test(begin - WPid = spawn(fun() -> receive write -> exit('kapow!') end end), + WPid = spawn(fun() -> + receive + write -> exit('kapow!') + end + end), Error = {mem3_shards_bad_write, 'kapow!'}, ?assertExit(Error, flush_write(?DB, WPid, 1000)) end). - t_writer_deletes_itself_when_done() -> ?_test(begin Shards = mock_shards(), @@ -644,14 +670,14 @@ t_writer_deletes_itself_when_done() -> ?assertEqual([], ets:tab2list(?OPENERS)) end). - t_writer_does_not_delete_other_writers_for_same_shard() -> ?_test(begin Shards = mock_shards(), WPid = spawn_link_mock_writer(?DB, Shards, ?INFINITY), WRef = erlang:monitor(process, WPid), ets:insert(?OPENERS, {?DB, WPid}), - ets:insert(?OPENERS, {?DB, self()}), % should not be deleted + % should not be deleted + ets:insert(?OPENERS, {?DB, self()}), WPid ! write, ?assertEqual(normal, wait_writer_result(WRef)), ?assertEqual(Shards, ets:tab2list(?SHARDS)), @@ -659,21 +685,22 @@ t_writer_does_not_delete_other_writers_for_same_shard() -> ?assertEqual([{?DB, self()}], ets:tab2list(?OPENERS)) end). - t_spawn_writer_in_load_shards_from_db() -> ?_test(begin meck:expect(couch_db, open_doc, 3, {ok, #doc{body = {[]}}}), meck:expect(couch_db, get_update_seq, 1, 1), meck:expect(mem3_util, build_ordered_shards, 2, mock_shards()), - erlang:register(?MODULE, self()), % register to get cache_insert cast + % register to get cache_insert cast + erlang:register(?MODULE, self()), load_shards_from_db(test_util:fake_db([{name, <<"testdb">>}]), ?DB), meck:validate(couch_db), meck:validate(mem3_util), - Cast = receive + Cast = + receive {'$gen_cast', Msg} -> Msg after 1000 -> timeout - end, + end, ?assertMatch({cache_insert, ?DB, Pid, 1} when is_pid(Pid), Cast), {cache_insert, _, WPid, _} = Cast, exit(WPid, kill), @@ -682,7 +709,6 @@ t_spawn_writer_in_load_shards_from_db() -> meck:unload(mem3_util) end). - t_cache_insert_takes_new_update() -> ?_test(begin Shards = mock_shards(), @@ -694,7 +720,6 @@ t_cache_insert_takes_new_update() -> ?assertEqual([], ets:tab2list(?OPENERS)) end). - t_cache_insert_ignores_stale_update_and_kills_worker() -> ?_test(begin Shards = mock_shards(), @@ -708,7 +733,6 @@ t_cache_insert_ignores_stale_update_and_kills_worker() -> ?assertEqual([], ets:tab2list(?OPENERS)) end). - mock_state(UpdateSeq) -> #st{ update_seq = UpdateSeq, @@ -716,44 +740,40 @@ mock_state(UpdateSeq) -> write_timeout = 1000 }. - mock_shards() -> [ #ordered_shard{ name = <<"testshardname">>, node = node(), dbname = ?DB, - range = [0,1], + range = [0, 1], order = 1 } ]. - wait_writer_result(WRef) -> receive {'DOWN', WRef, _, _, Result} -> Result - after 1000 -> - timeout + after 1000 -> + timeout end. - spawn_link_mock_writer(Db, Shards, Timeout) -> erlang:spawn_link(fun() -> shard_writer(Db, Shards, Timeout) end). - - -mem3_shards_changes_test_() -> { - "Test mem3_shards changes listener", +mem3_shards_changes_test_() -> { - setup, - fun test_util:start_couch/0, fun test_util:stop_couch/1, - [ - fun should_kill_changes_listener_on_shutdown/0 - ] - } -}. - + "Test mem3_shards changes listener", + { + setup, + fun test_util:start_couch/0, + fun test_util:stop_couch/1, + [ + fun should_kill_changes_listener_on_shutdown/0 + ] + } + }. should_kill_changes_listener_on_shutdown() -> {ok, Pid} = ?MODULE:start_link(), @@ -761,7 +781,8 @@ should_kill_changes_listener_on_shutdown() -> ?assert(is_process_alive(ChangesPid)), true = erlang:unlink(Pid), true = test_util:stop_sync_throw( - ChangesPid, fun() -> exit(Pid, shutdown) end, wait_timeout), + ChangesPid, fun() -> exit(Pid, shutdown) end, wait_timeout + ), ?assertNot(is_process_alive(ChangesPid)), exit(Pid, shutdown). diff --git a/src/mem3/src/mem3_sup.erl b/src/mem3/src/mem3_sup.erl index 3a1a3ca5a..a2dc5ba8d 100644 --- a/src/mem3/src/mem3_sup.erl +++ b/src/mem3/src/mem3_sup.erl @@ -22,13 +22,14 @@ init(_Args) -> child(mem3_events), child(mem3_nodes), child(mem3_seeds), - child(mem3_sync_nodes), % Order important? + % Order important? + child(mem3_sync_nodes), child(mem3_sync), child(mem3_shards), child(mem3_sync_event_listener), child(mem3_reshard_sup) ], - {ok, {{one_for_one,10,1}, couch_epi:register_service(mem3_epi, Children)}}. + {ok, {{one_for_one, 10, 1}, couch_epi:register_service(mem3_epi, Children)}}. child(mem3_events) -> MFA = {gen_event, start_link, [{local, mem3_events}]}, diff --git a/src/mem3/src/mem3_sync.erl b/src/mem3/src/mem3_sync.erl index cfed6a445..3d1c18420 100644 --- a/src/mem3/src/mem3_sync.erl +++ b/src/mem3/src/mem3_sync.erl @@ -13,12 +13,29 @@ -module(mem3_sync). -behaviour(gen_server). -vsn(1). --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3]). +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3 +]). --export([start_link/0, get_active/0, get_queue/0, push/1, push/2, - remove_node/1, remove_shard/1, initial_sync/1, get_backlog/0, nodes_db/0, - shards_db/0, users_db/0, find_next_node/0]). +-export([ + start_link/0, + get_active/0, + get_queue/0, + push/1, push/2, + remove_node/1, + remove_shard/1, + initial_sync/1, + get_backlog/0, + nodes_db/0, + shards_db/0, + users_db/0, + find_next_node/0 +]). -export([ local_dbs/0 ]). @@ -36,7 +53,7 @@ waiting = queue:new() }). --record(job, {name, node, count=nil, pid=nil}). +-record(job, {name, node, count = nil, pid = nil}). start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). @@ -52,7 +69,7 @@ get_backlog() -> push(#shard{name = Name}, Target) -> push(Name, Target); -push(Name, #shard{node=Node}) -> +push(Name, #shard{node = Node}) -> push(Name, Node); push(Name, Node) -> push(#job{name = Name, node = Node}). @@ -77,137 +94,147 @@ init([]) -> handle_call({push, Job}, From, State) -> handle_cast({push, Job#job{pid = From}}, State); - handle_call(get_active, _From, State) -> {reply, State#state.active, State}; - handle_call(get_queue, _From, State) -> {reply, to_list(State#state.waiting), State}; - -handle_call(get_backlog, _From, #state{active=A, waiting=WQ} = State) -> - CA = lists:sum([C || #job{count=C} <- A, is_integer(C)]), - CW = lists:sum([C || #job{count=C} <- to_list(WQ), is_integer(C)]), - {reply, CA+CW, State}. +handle_call(get_backlog, _From, #state{active = A, waiting = WQ} = State) -> + CA = lists:sum([C || #job{count = C} <- A, is_integer(C)]), + CW = lists:sum([C || #job{count = C} <- to_list(WQ), is_integer(C)]), + {reply, CA + CW, State}. handle_cast({push, DbName, Node}, State) -> handle_cast({push, #job{name = DbName, node = Node}}, State); - -handle_cast({push, Job}, #state{count=Count, limit=Limit} = State) - when Count >= Limit -> +handle_cast({push, Job}, #state{count = Count, limit = Limit} = State) when + Count >= Limit +-> {noreply, add_to_queue(State, Job)}; - handle_cast({push, Job}, State) -> #state{active = L, count = C} = State, #job{name = DbName, node = Node} = Job, case is_running(DbName, Node, L) of - true -> - {noreply, add_to_queue(State, Job)}; - false -> - Pid = start_push_replication(Job), - {noreply, State#state{active=[Job#job{pid=Pid}|L], count=C+1}} + true -> + {noreply, add_to_queue(State, Job)}; + false -> + Pid = start_push_replication(Job), + {noreply, State#state{active = [Job#job{pid = Pid} | L], count = C + 1}} end; - handle_cast({remove_node, Node}, #state{waiting = W0} = State) -> - {Alive, Dead} = lists:partition(fun(#job{node=N}) -> N =/= Node end, to_list(W0)), + {Alive, Dead} = lists:partition(fun(#job{node = N}) -> N =/= Node end, to_list(W0)), Dict = remove_entries(State#state.dict, Dead), - [exit(Pid, die_now) || #job{node=N, pid=Pid} <- State#state.active, - N =:= Node], + [ + exit(Pid, die_now) + || #job{node = N, pid = Pid} <- State#state.active, + N =:= Node + ], {noreply, State#state{dict = Dict, waiting = from_list(Alive)}}; - handle_cast({remove_shard, Shard}, #state{waiting = W0} = State) -> - {Alive, Dead} = lists:partition(fun(#job{name=S}) -> - S =/= Shard end, to_list(W0)), + {Alive, Dead} = lists:partition( + fun(#job{name = S}) -> + S =/= Shard + end, + to_list(W0) + ), Dict = remove_entries(State#state.dict, Dead), - [exit(Pid, die_now) || #job{name=S, pid=Pid} <- State#state.active, - S =:= Shard], + [ + exit(Pid, die_now) + || #job{name = S, pid = Pid} <- State#state.active, + S =:= Shard + ], {noreply, State#state{dict = Dict, waiting = from_list(Alive)}}. handle_info({'EXIT', Active, normal}, State) -> handle_replication_exit(State, Active); - handle_info({'EXIT', Active, die_now}, State) -> % we forced this one ourselves, do not retry handle_replication_exit(State, Active); - handle_info({'EXIT', Active, {{not_found, no_db_file}, _Stack}}, State) -> % target doesn't exist, do not retry handle_replication_exit(State, Active); - handle_info({'EXIT', Active, Reason}, State) -> - NewState = case lists:keyfind(Active, #job.pid, State#state.active) of - #job{name=OldDbName, node=OldNode} = Job -> - couch_log:warning("~s ~s ~s ~w", [?MODULE, OldDbName, OldNode, Reason]), - case Reason of {pending_changes, Count} -> - maybe_resubmit(State, Job#job{pid = nil, count = Count}); - _ -> - case mem3:db_is_current(Job#job.name) of - true -> - timer:apply_after(5000, ?MODULE, push, [Job#job{pid=nil}]); - false -> - % no need to retry (db deleted or recreated) - ok - end, - State - end; - false -> State end, + NewState = + case lists:keyfind(Active, #job.pid, State#state.active) of + #job{name = OldDbName, node = OldNode} = Job -> + couch_log:warning("~s ~s ~s ~w", [?MODULE, OldDbName, OldNode, Reason]), + case Reason of + {pending_changes, Count} -> + maybe_resubmit(State, Job#job{pid = nil, count = Count}); + _ -> + case mem3:db_is_current(Job#job.name) of + true -> + timer:apply_after(5000, ?MODULE, push, [Job#job{pid = nil}]); + false -> + % no need to retry (db deleted or recreated) + ok + end, + State + end; + false -> + State + end, handle_replication_exit(NewState, Active); - handle_info(Msg, State) -> couch_log:notice("unexpected msg at replication manager ~p", [Msg]), {noreply, State}. terminate(_Reason, State) -> - [exit(Pid, shutdown) || #job{pid=Pid} <- State#state.active], + [exit(Pid, shutdown) || #job{pid = Pid} <- State#state.active], ok. code_change(_, #state{waiting = WaitingList} = State, _) when is_list(WaitingList) -> {ok, State#state{waiting = from_list(WaitingList)}}; - code_change(_, State, _) -> {ok, State}. -maybe_resubmit(State, #job{name=DbName, node=Node} = Job) -> +maybe_resubmit(State, #job{name = DbName, node = Node} = Job) -> case lists:member(DbName, local_dbs()) of - true -> - case find_next_node() of - Node -> - add_to_queue(State, Job); - _ -> - State % don't resubmit b/c we have a new replication target - end; - false -> - add_to_queue(State, Job) + true -> + case find_next_node() of + Node -> + add_to_queue(State, Job); + _ -> + % don't resubmit b/c we have a new replication target + State + end; + false -> + add_to_queue(State, Job) end. handle_replication_exit(State, Pid) -> - #state{active=Active, limit=Limit, dict=D, waiting=Waiting} = State, + #state{active = Active, limit = Limit, dict = D, waiting = Waiting} = State, Active1 = lists:keydelete(Pid, #job.pid, Active), case is_empty(Waiting) of - true -> - {noreply, State#state{active=Active1, count=length(Active1)}}; - _ -> - Count = length(Active1), - NewState = if Count < Limit -> - case next_replication(Active1, Waiting, queue:new()) of - nil -> % all waiting replications are also active - State#state{active = Active1, count = Count}; - {#job{name=DbName, node=Node} = Job, StillWaiting} -> - NewPid = start_push_replication(Job), - State#state{ - active = [Job#job{pid = NewPid} | Active1], - count = Count+1, - dict = dict:erase({DbName,Node}, D), - waiting = StillWaiting - } - end; true -> - State#state{active = Active1, count=Count} - end, - {noreply, NewState} + {noreply, State#state{active = Active1, count = length(Active1)}}; + _ -> + Count = length(Active1), + NewState = + if + Count < Limit -> + case next_replication(Active1, Waiting, queue:new()) of + % all waiting replications are also active + nil -> + State#state{active = Active1, count = Count}; + {#job{name = DbName, node = Node} = Job, StillWaiting} -> + NewPid = start_push_replication(Job), + State#state{ + active = [Job#job{pid = NewPid} | Active1], + count = Count + 1, + dict = dict:erase({DbName, Node}, D), + waiting = StillWaiting + } + end; + true -> + State#state{active = Active1, count = Count} + end, + {noreply, NewState} end. -start_push_replication(#job{name=Name, node=Node, pid=From}) -> - if From =/= nil -> gen_server:reply(From, ok); true -> ok end, +start_push_replication(#job{name = Name, node = Node, pid = From}) -> + if + From =/= nil -> gen_server:reply(From, ok); + true -> ok + end, spawn_link(fun() -> case mem3_rep:go(Name, maybe_redirect(Node)) of {ok, Pending} when Pending > 0 -> @@ -217,18 +244,21 @@ start_push_replication(#job{name=Name, node=Node, pid=From}) -> end end). -add_to_queue(State, #job{name=DbName, node=Node, pid=From} = Job) -> - #state{dict=D, waiting=WQ} = State, +add_to_queue(State, #job{name = DbName, node = Node, pid = From} = Job) -> + #state{dict = D, waiting = WQ} = State, case dict:is_key({DbName, Node}, D) of - true -> - if From =/= nil -> gen_server:reply(From, ok); true -> ok end, - State; - false -> - couch_log:debug("adding ~s -> ~p to mem3_sync queue", [DbName, Node]), - State#state{ - dict = dict:store({DbName,Node}, ok, D), - waiting = in(Job, WQ) - } + true -> + if + From =/= nil -> gen_server:reply(From, ok); + true -> ok + end, + State; + false -> + couch_log:debug("adding ~s -> ~p to mem3_sync queue", [DbName, Node]), + State#state{ + dict = dict:store({DbName, Node}, ok, D), + waiting = in(Job, WQ) + } end. sync_nodes_and_dbs() -> @@ -247,32 +277,37 @@ initial_sync(Live) -> initial_sync_fold(#shard{dbname = Db} = Shard, {LocalNode, Live, AccShards}) -> case AccShards of - [#shard{dbname = AccDb} | _] when Db =/= AccDb -> - submit_replication_tasks(LocalNode, Live, AccShards), - {LocalNode, Live, [Shard]}; - _ -> - {LocalNode, Live, [Shard|AccShards]} + [#shard{dbname = AccDb} | _] when Db =/= AccDb -> + submit_replication_tasks(LocalNode, Live, AccShards), + {LocalNode, Live, [Shard]}; + _ -> + {LocalNode, Live, [Shard | AccShards]} end. submit_replication_tasks(LocalNode, Live, Shards) -> SplitFun = fun(#shard{node = Node}) -> Node =:= LocalNode end, {Local, Remote} = lists:partition(SplitFun, Shards), - lists:foreach(fun(#shard{name = ShardName}) -> - [sync_push(ShardName, N) || #shard{node=N, name=Name} <- Remote, - Name =:= ShardName, lists:member(N, Live)] - end, Local). + lists:foreach( + fun(#shard{name = ShardName}) -> + [ + sync_push(ShardName, N) + || #shard{node = N, name = Name} <- Remote, + Name =:= ShardName, + lists:member(N, Live) + ] + end, + Local + ). sync_push(ShardName, N) -> - gen_server:call(mem3_sync, {push, #job{name=ShardName, node=N}}, infinity). - - + gen_server:call(mem3_sync, {push, #job{name = ShardName, node = N}}, infinity). find_next_node() -> - LiveNodes = [node()|nodes()], + LiveNodes = [node() | nodes()], AllNodes0 = lists:sort(mem3:nodes()), AllNodes1 = [X || X <- AllNodes0, lists:member(X, LiveNodes)], AllNodes = AllNodes1 ++ [hd(AllNodes1)], - [_Self, Next| _] = lists:dropwhile(fun(N) -> N =/= node() end, AllNodes), + [_Self, Next | _] = lists:dropwhile(fun(N) -> N =/= node() end, AllNodes), Next. %% @doc Finds the next {DbName,Node} pair in the list of waiting replications @@ -281,25 +316,29 @@ find_next_node() -> {#job{}, queue:queue(_)} | nil. next_replication(Active, Waiting, WaitingAndRunning) -> case is_empty(Waiting) of - true -> - nil; - false -> - {{value, #job{name=S, node=N} = Job}, RemQ} = out(Waiting), - case is_running(S,N,Active) of true -> - next_replication(Active, RemQ, in(Job, WaitingAndRunning)); + nil; false -> - {Job, join(RemQ, WaitingAndRunning)} - end + {{value, #job{name = S, node = N} = Job}, RemQ} = out(Waiting), + case is_running(S, N, Active) of + true -> + next_replication(Active, RemQ, in(Job, WaitingAndRunning)); + false -> + {Job, join(RemQ, WaitingAndRunning)} + end end. is_running(DbName, Node, ActiveList) -> - [] =/= [true || #job{name=S, node=N} <- ActiveList, S=:=DbName, N=:=Node]. + [] =/= [true || #job{name = S, node = N} <- ActiveList, S =:= DbName, N =:= Node]. remove_entries(Dict, Entries) -> - lists:foldl(fun(#job{name=S, node=N}, D) -> - dict:erase({S, N}, D) - end, Dict, Entries). + lists:foldl( + fun(#job{name = S, node = N}, D) -> + dict:erase({S, N}, D) + end, + Dict, + Entries + ). local_dbs() -> UsersDb = users_db(), diff --git a/src/mem3/src/mem3_sync_event.erl b/src/mem3/src/mem3_sync_event.erl index 7bca23086..ec6debb45 100644 --- a/src/mem3/src/mem3_sync_event.erl +++ b/src/mem3/src/mem3_sync_event.erl @@ -14,8 +14,14 @@ -behaviour(gen_event). -vsn(1). --export([init/1, handle_event/2, handle_call/2, handle_info/2, terminate/2, - code_change/3]). +-export([ + init/1, + handle_event/2, + handle_call/2, + handle_info/2, + terminate/2, + code_change/3 +]). init(_) -> net_kernel:monitor_nodes(true), @@ -25,11 +31,9 @@ handle_event({add_node, Node}, State) when Node =/= node() -> net_kernel:connect_node(Node), mem3_sync_nodes:add([Node]), {ok, State}; - -handle_event({remove_node, Node}, State) -> +handle_event({remove_node, Node}, State) -> mem3_sync:remove_node(Node), {ok, State}; - handle_event(_Event, State) -> {ok, State}. @@ -41,11 +45,9 @@ handle_info({nodeup, Node}, State) -> Nodes = lists:filter(fun(N) -> lists:member(N, mem3:nodes()) end, Nodes0), wait_for_rexi(Nodes, 5), {ok, State}; - handle_info({nodedown, Node}, State) -> mem3_sync:remove_node(Node), {ok, State}; - handle_info(_Info, State) -> {ok, State}. @@ -75,12 +77,13 @@ wait_for_rexi(Waiting, Retries) -> case length(Up) > 0 of true -> mem3_sync_nodes:add(Up); - false -> ok + false -> + ok end, case length(NotUp) > 0 andalso Retries > 0 of true -> timer:sleep(1000), - wait_for_rexi(NotUp, Retries-1); + wait_for_rexi(NotUp, Retries - 1); false -> ok end. diff --git a/src/mem3/src/mem3_sync_event_listener.erl b/src/mem3/src/mem3_sync_event_listener.erl index 5a8d162d2..a01921f85 100644 --- a/src/mem3/src/mem3_sync_event_listener.erl +++ b/src/mem3/src/mem3_sync_event_listener.erl @@ -90,7 +90,7 @@ handle_event(UsersDb, updated, #state{users = UsersDb} = St) -> maybe_push_shards(St); handle_event(<<"shards/", _/binary>> = ShardName, updated, St) -> Buckets = bucket_shard(ShardName, St#state.buckets), - maybe_push_shards(St#state{buckets=Buckets}); + maybe_push_shards(St#state{buckets = Buckets}); handle_event(<<"shards/", _:18/binary, _/binary>> = ShardName, deleted, St) -> mem3_sync:remove_shard(ShardName), maybe_push_shards(St); @@ -100,11 +100,11 @@ handle_event(_DbName, _Event, St) -> handle_cast({set_frequency, Frequency}, St) -> #state{delay = Delay, buckets = Buckets0} = St, Buckets1 = rebucket_shards(Delay, Frequency, Buckets0), - maybe_push_shards(St#state{frequency=Frequency, buckets=Buckets1}); + maybe_push_shards(St#state{frequency = Frequency, buckets = Buckets1}); handle_cast({set_delay, Delay}, St) -> #state{frequency = Frequency, buckets = Buckets0} = St, Buckets1 = rebucket_shards(Delay, Frequency, Buckets0), - maybe_push_shards(St#state{delay=Delay, buckets=Buckets1}); + maybe_push_shards(St#state{delay = Delay, buckets = Buckets1}); handle_cast(Msg, St) -> couch_log:notice("unexpected cast to mem3_sync_event_listener: ~p", [Msg]), maybe_push_shards(St). @@ -134,19 +134,20 @@ set_config(Cmd, Value, Error) -> try list_to_integer(Value) of IntegerValue -> couch_event_listener:cast(self(), {Cmd, IntegerValue}) - catch error:badarg -> - couch_log:warning("~s: ~p", [Error, Value]) + catch + error:badarg -> + couch_log:warning("~s: ~p", [Error, Value]) end. -bucket_shard(ShardName, [B|Bs]=Buckets0) -> +bucket_shard(ShardName, [B | Bs] = Buckets0) -> case waiting(ShardName, Buckets0) of true -> Buckets0; - false -> [sets:add_element(ShardName, B)|Bs] + false -> [sets:add_element(ShardName, B) | Bs] end. waiting(_, []) -> false; -waiting(ShardName, [B|Bs]) -> +waiting(ShardName, [B | Bs]) -> case sets:is_element(ShardName, B) of true -> true; false -> waiting(ShardName, Bs) @@ -158,8 +159,8 @@ rebucket_shards(Frequency, Delay, Buckets0) -> Buckets0; N when N < 0 -> %% Reduce the number of buckets by merging the last N + 1 together - {ToMerge, [B|Buckets1]} = lists:split(abs(N), Buckets0), - [sets:union([B|ToMerge])|Buckets1]; + {ToMerge, [B | Buckets1]} = lists:split(abs(N), Buckets0), + [sets:union([B | ToMerge]) | Buckets1]; M -> %% Extend the number of buckets by M lists:duplicate(M, sets:new()) ++ Buckets0 @@ -170,42 +171,43 @@ rebucket_shards(Frequency, Delay, Buckets0) -> %% to maybe_push_shards/1 rather than directly. All timing coordination - i.e., %% calling mem3_sync:push/2 or setting a proper timeout to ensure that pending %% messages aren't dropped in case no further messages arrive - is handled here. -maybe_push_shards(#state{last_push=undefined} = St) -> - {ok, St#state{last_push=os:timestamp()}, St#state.frequency}; +maybe_push_shards(#state{last_push = undefined} = St) -> + {ok, St#state{last_push = os:timestamp()}, St#state.frequency}; maybe_push_shards(St) -> - #state{frequency=Frequency, last_push=LastPush, buckets=Buckets0} = St, + #state{frequency = Frequency, last_push = LastPush, buckets = Buckets0} = St, Now = os:timestamp(), Delta = timer:now_diff(Now, LastPush) div 1000, case Delta > Frequency of true -> {Buckets1, [ToPush]} = lists:split(length(Buckets0) - 1, Buckets0), - Buckets2 = [sets:new()|Buckets1], + Buckets2 = [sets:new() | Buckets1], %% There's no sets:map/2! sets:fold( fun(ShardName, _) -> push_shard(ShardName) end, undefined, ToPush ), - {ok, St#state{last_push=Now, buckets=Buckets2}, Frequency}; + {ok, St#state{last_push = Now, buckets = Buckets2}, Frequency}; false -> {ok, St, Frequency - Delta} end. push_shard(ShardName) -> try mem3_shards:for_shard_range(ShardName) of - Shards -> - Live = nodes(), - lists:foreach( - fun(#shard{node=N}) -> - case lists:member(N, Live) of - true -> mem3_sync:push(ShardName, N); - false -> ok - end - end, - Shards - ) - catch error:database_does_not_exist -> - ok + Shards -> + Live = nodes(), + lists:foreach( + fun(#shard{node = N}) -> + case lists:member(N, Live) of + true -> mem3_sync:push(ShardName, N); + false -> ok + end + end, + Shards + ) + catch + error:database_does_not_exist -> + ok end. subscribe_for_config() -> @@ -320,7 +322,6 @@ should_terminate(Pid) -> ok end). - get_state(Pid) -> Ref = make_ref(), Pid ! {get_state, Ref, self()}, @@ -330,7 +331,6 @@ get_state(Pid) -> timeout end. - wait_state(Pid, Field, Val) when is_pid(Pid), is_integer(Field) -> WaitFun = fun() -> case get_state(Pid) of @@ -342,7 +342,6 @@ wait_state(Pid, Field, Val) when is_pid(Pid), is_integer(Field) -> end, test_util:wait(WaitFun). - wait_config_subscribed(Pid) -> WaitFun = fun() -> Handlers = gen_event:which_handlers(config_event), diff --git a/src/mem3/src/mem3_sync_nodes.erl b/src/mem3/src/mem3_sync_nodes.erl index 0a4bffcd2..43ca8b756 100644 --- a/src/mem3/src/mem3_sync_nodes.erl +++ b/src/mem3/src/mem3_sync_nodes.erl @@ -14,7 +14,6 @@ -behaviour(gen_server). -vsn(1). - -export([start_link/0]). -export([add/1]). @@ -23,86 +22,71 @@ -export([monitor_sync/1]). - -record(st, { tid }). - -record(job, { nodes, pid, retry }). - start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - add(Nodes) -> gen_server:cast(?MODULE, {add, Nodes}). - init([]) -> {ok, #st{ tid = ets:new(?MODULE, [set, protected, {keypos, #job.nodes}]) }}. - terminate(_Reason, St) -> - [exit(Pid, kill) || #job{pid=Pid} <- ets:tab2list(St#st.tid)], + [exit(Pid, kill) || #job{pid = Pid} <- ets:tab2list(St#st.tid)], ok. - handle_call(Msg, _From, St) -> {stop, {invalid_call, Msg}, invalid_call, St}. - -handle_cast({add, Nodes}, #st{tid=Tid}=St) -> +handle_cast({add, Nodes}, #st{tid = Tid} = St) -> case ets:lookup(Tid, Nodes) of [] -> Pid = start_sync(Nodes), - ets:insert(Tid, #job{nodes=Nodes, pid=Pid, retry=false}); - [#job{retry=false}=Job] -> - ets:insert(Tid, Job#job{retry=true}); + ets:insert(Tid, #job{nodes = Nodes, pid = Pid, retry = false}); + [#job{retry = false} = Job] -> + ets:insert(Tid, Job#job{retry = true}); _ -> ok end, {noreply, St}; - handle_cast(Msg, St) -> {stop, {invalid_cast, Msg}, St}. - -handle_info({'DOWN', _, _, _, {sync_done, Nodes}}, #st{tid=Tid}=St) -> +handle_info({'DOWN', _, _, _, {sync_done, Nodes}}, #st{tid = Tid} = St) -> case ets:lookup(Tid, Nodes) of - [#job{retry=true}=Job] -> + [#job{retry = true} = Job] -> Pid = start_sync(Nodes), - ets:insert(Tid, Job#job{pid=Pid, retry=false}); + ets:insert(Tid, Job#job{pid = Pid, retry = false}); _ -> ets:delete(Tid, Nodes) end, {noreply, St}; - -handle_info({'DOWN', _, _, _, {sync_error, Nodes}}, #st{tid=Tid}=St) -> +handle_info({'DOWN', _, _, _, {sync_error, Nodes}}, #st{tid = Tid} = St) -> Pid = start_sync(Nodes), - ets:insert(Tid, #job{nodes=Nodes, pid=Pid, retry=false}), + ets:insert(Tid, #job{nodes = Nodes, pid = Pid, retry = false}), {noreply, St}; - handle_info(Msg, St) -> {stop, {invalid_info, Msg}, St}. - code_change(_OldVsn, St, _Extra) -> {ok, St}. - start_sync(Nodes) -> {Pid, _} = spawn_monitor(?MODULE, monitor_sync, [Nodes]), Pid. - monitor_sync(Nodes) -> process_flag(trap_exit, true), Pid = spawn_link(mem3_sync, initial_sync, [Nodes]), @@ -112,4 +96,3 @@ monitor_sync(Nodes) -> _ -> exit({sync_error, Nodes}) end. - diff --git a/src/mem3/src/mem3_sync_security.erl b/src/mem3/src/mem3_sync_security.erl index 291e4e085..fc1726901 100644 --- a/src/mem3/src/mem3_sync_security.erl +++ b/src/mem3/src/mem3_sync_security.erl @@ -17,8 +17,7 @@ -include_lib("mem3/include/mem3.hrl"). - -maybe_sync(#shard{}=Src, #shard{}=Dst) -> +maybe_sync(#shard{} = Src, #shard{} = Dst) -> case is_local(Src#shard.name) of false -> erlang:spawn(?MODULE, maybe_sync_int, [Src, Dst]); @@ -26,7 +25,7 @@ maybe_sync(#shard{}=Src, #shard{}=Dst) -> ok end. -maybe_sync_int(#shard{name=Name}=Src, Dst) -> +maybe_sync_int(#shard{name = Name} = Src, Dst) -> DbName = mem3:dbname(Name), case fabric:get_all_security(DbName, [{shards, [Src, Dst]}]) of {ok, WorkerObjs} -> @@ -53,44 +52,55 @@ handle_existing_db(DbName) -> try handle_db(DbName) of _ -> ok catch - error:database_does_not_exist-> - couch_log:error("Db was deleted while getting security" - " object. DbName: ~p", [DbName]), + error:database_does_not_exist -> + couch_log:error( + "Db was deleted while getting security" + " object. DbName: ~p", + [DbName] + ), ok end. handle_db(DbName) -> ShardCount = length(mem3:shards(DbName)), case get_all_security(DbName) of - {ok, SecObjs} -> - case is_ok(SecObjs, ShardCount) of - ok -> - ok; - {fixable, SecObj} -> - couch_log:info("Sync security object for ~p: ~p", [DbName, SecObj]), - case fabric:set_security(DbName, SecObj) of - ok -> ok; - Error -> - couch_log:error("Error setting security object in ~p: ~p", - [DbName, Error]) + {ok, SecObjs} -> + case is_ok(SecObjs, ShardCount) of + ok -> + ok; + {fixable, SecObj} -> + couch_log:info("Sync security object for ~p: ~p", [DbName, SecObj]), + case fabric:set_security(DbName, SecObj) of + ok -> + ok; + Error -> + couch_log:error( + "Error setting security object in ~p: ~p", + [DbName, Error] + ) + end; + broken -> + couch_log:error("Bad security object in ~p: ~p", [DbName, SecObjs]) end; - broken -> - couch_log:error("Bad security object in ~p: ~p", [DbName, SecObjs]) - end; - Error -> - couch_log:error("Error getting security objects for ~p: ~p", [ - DbName, Error]) + Error -> + couch_log:error("Error getting security objects for ~p: ~p", [ + DbName, Error + ]) end. get_all_security(DbName) -> case fabric:get_all_security(DbName) of - {ok, SecObjs} -> - SecObjsDict = lists:foldl(fun({_, SO}, Acc) -> - dict:update_counter(SO, 1, Acc) - end, dict:new(), SecObjs), - {ok, dict:to_list(SecObjsDict)}; - Error -> - Error + {ok, SecObjs} -> + SecObjsDict = lists:foldl( + fun({_, SO}, Acc) -> + dict:update_counter(SO, 1, Acc) + end, + dict:new(), + SecObjs + ), + {ok, dict:to_list(SecObjsDict)}; + Error -> + Error end. is_ok([_], _) -> @@ -100,7 +110,7 @@ is_ok([_, _] = SecObjs0, ShardCount) -> % Figure out if we have a simple majority of security objects % and if so, use that as the correct value. Otherwise we abort % and rely on human intervention. - {Count, SecObj} = lists:max([{C, O} || {O, C} <- SecObjs0]), + {Count, SecObj} = lists:max([{C, O} || {O, C} <- SecObjs0]), case Count >= ((ShardCount div 2) + 1) of true -> {fixable, SecObj}; false -> broken @@ -109,9 +119,7 @@ is_ok(_, _) -> % Anything else requires human intervention broken. - is_local(<<"shards/", _/binary>>) -> false; is_local(_) -> true. - diff --git a/src/mem3/src/mem3_util.erl b/src/mem3/src/mem3_util.erl index 005a6b1bc..8547fc071 100644 --- a/src/mem3/src/mem3_util.erl +++ b/src/mem3/src/mem3_util.erl @@ -12,9 +12,21 @@ -module(mem3_util). --export([name_shard/2, create_partition_map/5, build_shards/2, - n_val/2, q_val/1, to_atom/1, to_integer/1, write_db_doc/1, delete_db_doc/1, - shard_info/1, ensure_exists/1, open_db_doc/1, update_db_doc/1]). +-export([ + name_shard/2, + create_partition_map/5, + build_shards/2, + n_val/2, + q_val/1, + to_atom/1, + to_integer/1, + write_db_doc/1, + delete_db_doc/1, + shard_info/1, + ensure_exists/1, + open_db_doc/1, + update_db_doc/1 +]). -export([get_or_create_db/2, get_or_create_db_int/2]). -export([is_deleted/1, rotate_list/2]). -export([get_shard_opts/1, get_engine_opt/1, get_props_opt/1]). @@ -41,26 +53,32 @@ -deprecated({create_partition_map, 4, eventually}). -deprecated({name_shard, 1, eventually}). --define(RINGTOP, 2 bsl 31). % CRC32 space +% CRC32 space +-define(RINGTOP, 2 bsl 31). -include_lib("mem3/include/mem3.hrl"). -include_lib("couch/include/couch_db.hrl"). - name_shard(Shard) -> name_shard(Shard, ""). -name_shard(#shard{dbname = DbName, range=Range} = Shard, Suffix) -> +name_shard(#shard{dbname = DbName, range = Range} = Shard, Suffix) -> Name = make_name(DbName, Range, Suffix), Shard#shard{name = ?l2b(Name)}; - -name_shard(#ordered_shard{dbname = DbName, range=Range} = Shard, Suffix) -> +name_shard(#ordered_shard{dbname = DbName, range = Range} = Shard, Suffix) -> Name = make_name(DbName, Range, Suffix), Shard#ordered_shard{name = ?l2b(Name)}. -make_name(DbName, [B,E], Suffix) -> - ["shards/", couch_util:to_hex(<<B:32/integer>>), "-", - couch_util:to_hex(<<E:32/integer>>), "/", DbName, Suffix]. +make_name(DbName, [B, E], Suffix) -> + [ + "shards/", + couch_util:to_hex(<<B:32/integer>>), + "-", + couch_util:to_hex(<<E:32/integer>>), + "/", + DbName, + Suffix + ]. create_partition_map(DbName, N, Q, Nodes) -> create_partition_map(DbName, N, Q, Nodes, ""). @@ -69,50 +87,55 @@ create_partition_map(DbName, N, Q, Nodes, Suffix) when Q > 0 -> UniqueShards = make_key_ranges((?RINGTOP) div Q, 0, []), Shards0 = lists:flatten([lists:duplicate(N, S) || S <- UniqueShards]), Shards1 = attach_nodes(Shards0, [], Nodes, []), - [name_shard(S#shard{dbname=DbName}, Suffix) || S <- Shards1]. + [name_shard(S#shard{dbname = DbName}, Suffix) || S <- Shards1]. make_key_ranges(I, CurrentPos, Acc) when I > 0, CurrentPos >= ?RINGTOP -> Acc; make_key_ranges(Increment, Start, Acc) when Increment > 0 -> - case Start + 2*Increment of - X when X > ?RINGTOP -> - End = ?RINGTOP - 1; - _ -> - End = Start + Increment - 1 + case Start + 2 * Increment of + X when X > ?RINGTOP -> + End = ?RINGTOP - 1; + _ -> + End = Start + Increment - 1 end, - make_key_ranges(Increment, End+1, [#shard{range=[Start, End]} | Acc]). + make_key_ranges(Increment, End + 1, [#shard{range = [Start, End]} | Acc]). attach_nodes([], Acc, _, _) -> lists:reverse(Acc); attach_nodes(Shards, Acc, [], UsedNodes) -> attach_nodes(Shards, Acc, lists:reverse(UsedNodes), []); attach_nodes([S | Rest], Acc, [Node | Nodes], UsedNodes) -> - attach_nodes(Rest, [S#shard{node=Node} | Acc], Nodes, [Node | UsedNodes]). + attach_nodes(Rest, [S#shard{node = Node} | Acc], Nodes, [Node | UsedNodes]). open_db_doc(DocId) -> {ok, Db} = couch_db:open(mem3_sync:shards_db(), [?ADMIN_CTX]), - try couch_db:open_doc(Db, DocId, [ejson_body]) after couch_db:close(Db) end. + try + couch_db:open_doc(Db, DocId, [ejson_body]) + after + couch_db:close(Db) + end. write_db_doc(Doc) -> write_db_doc(mem3_sync:shards_db(), Doc, true). -write_db_doc(DbName, #doc{id=Id, body=Body} = Doc, ShouldMutate) -> +write_db_doc(DbName, #doc{id = Id, body = Body} = Doc, ShouldMutate) -> {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]), try couch_db:open_doc(Db, Id, [ejson_body]) of - {ok, #doc{body = Body}} -> - % the doc is already in the desired state, we're done here - ok; - {not_found, _} when ShouldMutate -> - try couch_db:update_doc(Db, Doc, []) of - {ok, _} -> - ok - catch conflict -> - % check to see if this was a replication race or a different edit - write_db_doc(DbName, Doc, false) - end; - _ -> - % the doc already exists in a different state - conflict + {ok, #doc{body = Body}} -> + % the doc is already in the desired state, we're done here + ok; + {not_found, _} when ShouldMutate -> + try couch_db:update_doc(Db, Doc, []) of + {ok, _} -> + ok + catch + conflict -> + % check to see if this was a replication race or a different edit + write_db_doc(DbName, Doc, false) + end; + _ -> + % the doc already exists in a different state + conflict after couch_db:close(Db) end. @@ -120,27 +143,28 @@ write_db_doc(DbName, #doc{id=Id, body=Body} = Doc, ShouldMutate) -> update_db_doc(Doc) -> update_db_doc(mem3_sync:shards_db(), Doc, true). -update_db_doc(DbName, #doc{id=Id, body=Body} = Doc, ShouldMutate) -> +update_db_doc(DbName, #doc{id = Id, body = Body} = Doc, ShouldMutate) -> {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]), try couch_db:open_doc(Db, Id, [ejson_body]) of - {ok, #doc{body = Body}} -> - % the doc is already in the desired state, we're done here - ok; - {ok, #doc{body = Body1}} -> - % the doc has a new body to be written - {ok, _} = couch_db:update_doc(Db, Doc#doc{body=Body1}, []), - ok; - {not_found, _} when ShouldMutate -> - try couch_db:update_doc(Db, Doc, []) of - {ok, _} -> - ok - catch conflict -> - % check to see if this was a replication race or a different edit - update_db_doc(DbName, Doc, false) - end; - _ -> - % the doc already exists in a different state - conflict + {ok, #doc{body = Body}} -> + % the doc is already in the desired state, we're done here + ok; + {ok, #doc{body = Body1}} -> + % the doc has a new body to be written + {ok, _} = couch_db:update_doc(Db, Doc#doc{body = Body1}, []), + ok; + {not_found, _} when ShouldMutate -> + try couch_db:update_doc(Db, Doc, []) of + {ok, _} -> + ok + catch + conflict -> + % check to see if this was a replication race or a different edit + update_db_doc(DbName, Doc, false) + end; + _ -> + % the doc already exists in a different state + conflict after couch_db:close(Db) end. @@ -152,20 +176,21 @@ delete_db_doc(DocId) -> delete_db_doc(DbName, DocId, ShouldMutate) -> {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]), {ok, Revs} = couch_db:open_doc_revs(Db, DocId, all, []), - try [Doc#doc{deleted=true} || {ok, #doc{deleted=false}=Doc} <- Revs] of - [] -> - not_found; - Docs when ShouldMutate -> - try couch_db:update_docs(Db, Docs, []) of - {ok, _} -> - ok - catch conflict -> - % check to see if this was a replication race or if leafs survived - delete_db_doc(DbName, DocId, false) - end; - _ -> - % we have live leafs that we aren't allowed to delete. let's bail - conflict + try [Doc#doc{deleted = true} || {ok, #doc{deleted = false} = Doc} <- Revs] of + [] -> + not_found; + Docs when ShouldMutate -> + try couch_db:update_docs(Db, Docs, []) of + {ok, _} -> + ok + catch + conflict -> + % check to see if this was a replication race or if leafs survived + delete_db_doc(DbName, DocId, false) + end; + _ -> + % we have live leafs that we aren't allowed to delete. let's bail + conflict after couch_db:close(Db) end. @@ -184,44 +209,62 @@ build_ordered_shards(DbName, DocProps) -> ByRange = build_shards_by_range(DbName, DocProps), Symmetrical = lists:sort(ByNode) =:= lists:sort(downcast(ByRange)), case Symmetrical of - true -> ByRange; + true -> ByRange; false -> ByNode end. build_shards_by_node(DbName, DocProps) -> {ByNode} = couch_util:get_value(<<"by_node">>, DocProps, {[]}), Suffix = couch_util:get_value(<<"shard_suffix">>, DocProps, ""), - lists:flatmap(fun({Node, Ranges}) -> - lists:map(fun(Range) -> - [B,E] = string:tokens(?b2l(Range), "-"), - Beg = httpd_util:hexlist_to_integer(B), - End = httpd_util:hexlist_to_integer(E), - name_shard(#shard{ - dbname = DbName, - node = to_atom(Node), - range = [Beg, End], - opts = get_shard_opts(DocProps) - }, Suffix) - end, Ranges) - end, ByNode). + lists:flatmap( + fun({Node, Ranges}) -> + lists:map( + fun(Range) -> + [B, E] = string:tokens(?b2l(Range), "-"), + Beg = httpd_util:hexlist_to_integer(B), + End = httpd_util:hexlist_to_integer(E), + name_shard( + #shard{ + dbname = DbName, + node = to_atom(Node), + range = [Beg, End], + opts = get_shard_opts(DocProps) + }, + Suffix + ) + end, + Ranges + ) + end, + ByNode + ). build_shards_by_range(DbName, DocProps) -> {ByRange} = couch_util:get_value(<<"by_range">>, DocProps, {[]}), Suffix = couch_util:get_value(<<"shard_suffix">>, DocProps, ""), - lists:flatmap(fun({Range, Nodes}) -> - lists:map(fun({Node, Order}) -> - [B,E] = string:tokens(?b2l(Range), "-"), - Beg = httpd_util:hexlist_to_integer(B), - End = httpd_util:hexlist_to_integer(E), - name_shard(#ordered_shard{ - dbname = DbName, - node = to_atom(Node), - range = [Beg, End], - order = Order, - opts = get_shard_opts(DocProps) - }, Suffix) - end, lists:zip(Nodes, lists:seq(1, length(Nodes)))) - end, ByRange). + lists:flatmap( + fun({Range, Nodes}) -> + lists:map( + fun({Node, Order}) -> + [B, E] = string:tokens(?b2l(Range), "-"), + Beg = httpd_util:hexlist_to_integer(B), + End = httpd_util:hexlist_to_integer(E), + name_shard( + #ordered_shard{ + dbname = DbName, + node = to_atom(Node), + range = [Beg, End], + order = Order, + opts = get_shard_opts(DocProps) + }, + Suffix + ) + end, + lists:zip(Nodes, lists:seq(1, length(Nodes))) + ) + end, + ByRange + ). to_atom(Node) when is_binary(Node) -> list_to_atom(binary_to_list(Node)); @@ -256,15 +299,12 @@ get_props_opt(DocProps) -> db_props_from_json([]) -> []; - db_props_from_json([{<<"partitioned">>, Value} | Rest]) -> [{partitioned, Value} | db_props_from_json(Rest)]; - db_props_from_json([{<<"hash">>, [MBin, FBin, A]} | Rest]) -> M = binary_to_existing_atom(MBin, utf8), F = binary_to_existing_atom(FBin, utf8), [{hash, [M, F, A]} | db_props_from_json(Rest)]; - db_props_from_json([{K, V} | Rest]) -> [{K, V} | db_props_from_json(Rest)]. @@ -288,28 +328,29 @@ q_val(_) -> throw({error, invalid_q_value}). shard_info(DbName) -> - [{n, mem3:n(DbName)}, - {q, length(mem3:shards(DbName)) div mem3:n(DbName)}]. + [ + {n, mem3:n(DbName)}, + {q, length(mem3:shards(DbName)) div mem3:n(DbName)} + ]. ensure_exists(DbName) when is_list(DbName) -> ensure_exists(list_to_binary(DbName)); ensure_exists(DbName) -> Options = [nologifmissing, sys_db, {create_if_missing, true}, ?ADMIN_CTX], case couch_db:open(DbName, Options) of - {ok, Db} -> - {ok, Db}; - file_exists -> - couch_db:open(DbName, [sys_db, ?ADMIN_CTX]) + {ok, Db} -> + {ok, Db}; + file_exists -> + couch_db:open(DbName, [sys_db, ?ADMIN_CTX]) end. - is_deleted(Change) -> case couch_util:get_value(<<"deleted">>, Change) of - undefined -> - % keep backwards compatibility for a while - couch_util:get_value(deleted, Change, false); - Else -> - Else + undefined -> + % keep backwards compatibility for a while + couch_util:get_value(deleted, Change, false); + Else -> + Else end. rotate_list(_Key, []) -> @@ -320,44 +361,41 @@ rotate_list(Key, List) -> {H, T} = lists:split(erlang:crc32(Key) rem length(List), List), T ++ H. -downcast(#shard{}=S) -> +downcast(#shard{} = S) -> S; -downcast(#ordered_shard{}=S) -> +downcast(#ordered_shard{} = S) -> #shard{ - name = S#ordered_shard.name, - node = S#ordered_shard.node, - dbname = S#ordered_shard.dbname, - range = S#ordered_shard.range, - ref = S#ordered_shard.ref, - opts = S#ordered_shard.opts - }; + name = S#ordered_shard.name, + node = S#ordered_shard.node, + dbname = S#ordered_shard.dbname, + range = S#ordered_shard.range, + ref = S#ordered_shard.ref, + opts = S#ordered_shard.opts + }; downcast(Shards) when is_list(Shards) -> [downcast(Shard) || Shard <- Shards]. iso8601_timestamp() -> - {_,_,Micro} = Now = os:timestamp(), - {{Year,Month,Date},{Hour,Minute,Second}} = calendar:now_to_datetime(Now), + {_, _, Micro} = Now = os:timestamp(), + {{Year, Month, Date}, {Hour, Minute, Second}} = calendar:now_to_datetime(Now), Format = "~4.10.0B-~2.10.0B-~2.10.0BT~2.10.0B:~2.10.0B:~2.10.0B.~6.10.0BZ", io_lib:format(Format, [Year, Month, Date, Hour, Minute, Second, Micro]). - live_nodes() -> LiveNodes = [node() | nodes()], Mem3Nodes = lists:sort(mem3:nodes()), [N || N <- Mem3Nodes, lists:member(N, LiveNodes)]. - % Replicate "dbs" db to all nodes. Basically push the changes to all the live % mem3:nodes(). Returns only after all current changes have been replicated, % which could be a while. % replicate_dbs_to_all_nodes(Timeout) -> DbName = mem3_sync:shards_db(), - Targets= mem3_util:live_nodes() -- [node()], - Res = [start_replication(node(), T, DbName, Timeout) || T <- Targets], + Targets = mem3_util:live_nodes() -- [node()], + Res = [start_replication(node(), T, DbName, Timeout) || T <- Targets], collect_replication_results(Res, Timeout). - % Replicate "dbs" db from all nodes to this node. Basically make an rpc call % to all the nodes an have them push their changes to this node. Then monitor % them until they are all done. @@ -368,7 +406,6 @@ replicate_dbs_from_all_nodes(Timeout) -> Res = [start_replication(S, node(), DbName, Timeout) || S <- Sources], collect_replication_results(Res, Timeout). - % Spawn and monitor a single replication of a database to a target node. % Returns {ok, PidRef}. This function could be called locally or remotely from % mem3_rpc, for instance when replicating other nodes' data to this node. @@ -383,7 +420,6 @@ start_replication(Source, Target, DbName, Timeout) -> end end). - collect_replication_results(Replications, Timeout) -> Res = [collect_replication_result(R, Timeout) || R <- Replications], case [R || R <- Res, R =/= ok] of @@ -393,7 +429,6 @@ collect_replication_results(Replications, Timeout) -> {error, Errors} end. - collect_replication_result({Pid, Ref}, Timeout) when is_pid(Pid) -> receive {'DOWN', Ref, _, _, Res} -> @@ -403,11 +438,9 @@ collect_replication_result({Pid, Ref}, Timeout) when is_pid(Pid) -> exit(Pid, kill), {error, {timeout, Timeout, node(Pid)}} end; - collect_replication_result(Error, _) -> {error, Error}. - % Consider these cases: % % A-------B @@ -423,64 +456,77 @@ collect_replication_result(Error, _) -> % X-Y because X !=< B % range_overlap([A, B], [X, Y]) when - is_integer(A), is_integer(B), - is_integer(X), is_integer(Y), - A =< B, X =< Y -> + is_integer(A), + is_integer(B), + is_integer(X), + is_integer(Y), + A =< B, + X =< Y +-> A =< Y andalso X =< B. - non_overlapping_shards(Shards) -> - {Start, End} = lists:foldl(fun(Shard, {Min, Max}) -> - [B, E] = mem3:range(Shard), - {min(B, Min), max(E, Max)} - end, {0, ?RING_END}, Shards), + {Start, End} = lists:foldl( + fun(Shard, {Min, Max}) -> + [B, E] = mem3:range(Shard), + {min(B, Min), max(E, Max)} + end, + {0, ?RING_END}, + Shards + ), non_overlapping_shards(Shards, Start, End). - non_overlapping_shards([], _, _) -> []; - non_overlapping_shards(Shards, Start, End) -> - Ranges = lists:map(fun(Shard) -> - [B, E] = mem3:range(Shard), - {B, E} - end, Shards), + Ranges = lists:map( + fun(Shard) -> + [B, E] = mem3:range(Shard), + {B, E} + end, + Shards + ), Ring = get_ring(Ranges, fun sort_ranges_fun/2, Start, End), - lists:filter(fun(Shard) -> - [B, E] = mem3:range(Shard), - lists:member({B, E}, Ring) - end, Shards). - + lists:filter( + fun(Shard) -> + [B, E] = mem3:range(Shard), + lists:member({B, E}, Ring) + end, + Shards + ). % Given a list of shards, return the maximum number of copies % across all the ranges. If the ring is incomplete it will return 0. % If there it is an n = 1 database, it should return 1, etc. calculate_max_n(Shards) -> - Ranges = lists:map(fun(Shard) -> - [B, E] = mem3:range(Shard), - {B, E} - end, Shards), + Ranges = lists:map( + fun(Shard) -> + [B, E] = mem3:range(Shard), + {B, E} + end, + Shards + ), calculate_max_n(Ranges, get_ring(Ranges), 0). - calculate_max_n(_Ranges, [], N) -> N; - calculate_max_n(Ranges, Ring, N) -> NewRanges = Ranges -- Ring, calculate_max_n(NewRanges, get_ring(NewRanges), N + 1). - get_ring(Ranges) -> get_ring(Ranges, fun sort_ranges_fun/2, 0, ?RING_END). - get_ring(Ranges, SortFun) when is_function(SortFun, 2) -> get_ring(Ranges, SortFun, 0, ?RING_END). - -get_ring(Ranges, Start, End) when is_integer(Start), is_integer(End), - Start >= 0, End >= 0, Start =< End -> +get_ring(Ranges, Start, End) when + is_integer(Start), + is_integer(End), + Start >= 0, + End >= 0, + Start =< End +-> get_ring(Ranges, fun sort_ranges_fun/2, Start, End). % Build a ring out of a list of possibly overlapping ranges. If a ring cannot @@ -493,19 +539,22 @@ get_ring(Ranges, Start, End) when is_integer(Start), is_integer(End), % get_ring([], _SortFun, _Start, _End) -> []; -get_ring(Ranges, SortFun, Start, End) when is_function(SortFun, 2), - is_integer(Start), is_integer(End), - Start >= 0, End >= 0, Start =< End -> +get_ring(Ranges, SortFun, Start, End) when + is_function(SortFun, 2), + is_integer(Start), + is_integer(End), + Start >= 0, + End >= 0, + Start =< End +-> Sorted = lists:usort(SortFun, Ranges), case get_subring_int(Start, End, Sorted) of fail -> []; Ring -> Ring end. - get_subring_int(_, _, []) -> fail; - get_subring_int(Start, EndMax, [{Start, End} = Range | Tail]) -> case End =:= EndMax of true -> @@ -518,33 +567,28 @@ get_subring_int(Start, EndMax, [{Start, End} = Range | Tail]) -> [Range | Acc] end end; - get_subring_int(Start1, _, [{Start2, _} | _]) when Start2 > Start1 -> % Found a gap, this attempt is done fail; - get_subring_int(Start1, EndMax, [{Start2, _} | Rest]) when Start2 < Start1 -> % We've overlapped the head, skip the shard get_subring_int(Start1, EndMax, Rest). - % Sort ranges by starting point, then sort so that % the longest range comes first sort_ranges_fun({B, E1}, {B, E2}) -> E2 =< E1; - sort_ranges_fun({B1, _}, {B2, _}) -> B1 =< B2. - add_db_config_options(DbName, Options) -> - DbOpts = case mem3:dbname(DbName) of - DbName -> []; - MDbName -> mem3_shards:opts_for_db(MDbName) - end, + DbOpts = + case mem3:dbname(DbName) of + DbName -> []; + MDbName -> mem3_shards:opts_for_db(MDbName) + end, merge_opts(DbOpts, Options). - get_or_create_db(DbName, Options) -> case couch_db:open(DbName, Options) of {ok, _} = OkDb -> @@ -554,14 +598,14 @@ get_or_create_db(DbName, Options) -> Options1 = [{create_if_missing, true} | Options], Options2 = add_db_config_options(DbName, Options1), couch_db:open(DbName, Options2) - catch error:database_does_not_exist -> - throw({error, missing_target}) + catch + error:database_does_not_exist -> + throw({error, missing_target}) end; Else -> Else end. - get_or_create_db_int(DbName, Options) -> case couch_db:open_int(DbName, Options) of {ok, _} = OkDb -> @@ -571,35 +615,40 @@ get_or_create_db_int(DbName, Options) -> Options1 = [{create_if_missing, true} | Options], Options2 = add_db_config_options(DbName, Options1), couch_db:open_int(DbName, Options2) - catch error:database_does_not_exist -> - throw({error, missing_target}) + catch + error:database_does_not_exist -> + throw({error, missing_target}) end; Else -> Else end. - %% merge two proplists, atom options only valid in Old merge_opts(New, Old) -> - lists:foldl(fun({Key, Val}, Acc) -> - lists:keystore(Key, 1, Acc, {Key, Val}) - end, Old, New). - + lists:foldl( + fun({Key, Val}, Acc) -> + lists:keystore(Key, 1, Acc, {Key, Val}) + end, + Old, + New + ). get_shard_props(ShardName) -> case couch_db:open_int(ShardName, []) of {ok, Db} -> - Props = case couch_db_engine:get_props(Db) of - undefined -> []; - Else -> Else - end, + Props = + case couch_db_engine:get_props(Db) of + undefined -> []; + Else -> Else + end, %% We don't normally store the default engine name - EngineProps = case couch_db_engine:get_engine(Db) of - couch_bt_engine -> - []; - EngineName -> - [{engine, EngineName}] - end, + EngineProps = + case couch_db_engine:get_engine(Db) of + couch_bt_engine -> + []; + EngineName -> + [{engine, EngineName}] + end, [{props, Props} | EngineProps]; {not_found, _} -> not_found; @@ -607,92 +656,98 @@ get_shard_props(ShardName) -> Else end. - find_dirty_shards() -> - mem3_shards:fold(fun(#shard{node=Node, name=Name, opts=Opts}=Shard, Acc) -> - case Opts of - [] -> - Acc; - [{props, []}] -> - Acc; - _ -> - Props = rpc:call(Node, ?MODULE, get_shard_props, [Name]), - case Props =:= Opts of - true -> - Acc; - false -> - [{Shard, Props} | Acc] - end - end - end, []). - + mem3_shards:fold( + fun(#shard{node = Node, name = Name, opts = Opts} = Shard, Acc) -> + case Opts of + [] -> + Acc; + [{props, []}] -> + Acc; + _ -> + Props = rpc:call(Node, ?MODULE, get_shard_props, [Name]), + case Props =:= Opts of + true -> + Acc; + false -> + [{Shard, Props} | Acc] + end + end + end, + [] + ). -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). range_overlap_test_() -> - [?_assertEqual(Res, range_overlap(R1, R2)) || {R1, R2, Res} <- [ - {[2, 6], [1, 3], true}, - {[2, 6], [3, 4], true}, - {[2, 6], [4, 8], true}, - {[2, 6], [1, 9], true}, - {[2, 6], [1, 2], true}, - {[2, 6], [6, 7], true}, - {[2, 6], [0, 1], false}, - {[2, 6], [7, 9], false} - ]]. - + [ + ?_assertEqual(Res, range_overlap(R1, R2)) + || {R1, R2, Res} <- [ + {[2, 6], [1, 3], true}, + {[2, 6], [3, 4], true}, + {[2, 6], [4, 8], true}, + {[2, 6], [1, 9], true}, + {[2, 6], [1, 2], true}, + {[2, 6], [6, 7], true}, + {[2, 6], [0, 1], false}, + {[2, 6], [7, 9], false} + ] + ]. non_overlapping_shards_test() -> - [?_assertEqual(Res, non_overlapping_shards(Shards)) || {Shards, Res} <- [ - { - [shard(0, ?RING_END)], - [shard(0, ?RING_END)] - }, - { - [shard(0, 1)], - [shard(0, 1)] - }, - { - [shard(0, 1), shard(0, 1)], - [shard(0, 1)] - }, - { - [shard(0, 1), shard(3, 4)], - [] - }, - { - [shard(0, 1), shard(1, 2), shard(2, 3)], - [shard(0, 1), shard(2, 3)] - }, - { - [shard(1, 2), shard(0, 1)], - [shard(0, 1), shard(1, 2)] - }, - { - [shard(0, 1), shard(0, 2), shard(2, 5), shard(3, 5)], - [shard(0, 2), shard(2, 5)] - }, - { - [shard(0, 2), shard(4, 5), shard(1, 3)], - [] - } - - ]]. - + [ + ?_assertEqual(Res, non_overlapping_shards(Shards)) + || {Shards, Res} <- [ + { + [shard(0, ?RING_END)], + [shard(0, ?RING_END)] + }, + { + [shard(0, 1)], + [shard(0, 1)] + }, + { + [shard(0, 1), shard(0, 1)], + [shard(0, 1)] + }, + { + [shard(0, 1), shard(3, 4)], + [] + }, + { + [shard(0, 1), shard(1, 2), shard(2, 3)], + [shard(0, 1), shard(2, 3)] + }, + { + [shard(1, 2), shard(0, 1)], + [shard(0, 1), shard(1, 2)] + }, + { + [shard(0, 1), shard(0, 2), shard(2, 5), shard(3, 5)], + [shard(0, 2), shard(2, 5)] + }, + { + [shard(0, 2), shard(4, 5), shard(1, 3)], + [] + } + ] + ]. calculate_max_n_test_() -> - [?_assertEqual(Res, calculate_max_n(Shards)) || {Res, Shards} <- [ - {0, []}, - {0, [shard(1, ?RING_END)]}, - {1, [shard(0, ?RING_END)]}, - {1, [shard(0, ?RING_END), shard(1, ?RING_END)]}, - {2, [shard(0, ?RING_END), shard(0, ?RING_END)]}, - {2, [shard(0, 1), shard(2, ?RING_END), shard(0, ?RING_END)]}, - {0, [shard(0, 3), shard(5, ?RING_END), shard(1, ?RING_END)]} - ]]. - + [ + ?_assertEqual(Res, calculate_max_n(Shards)) + || {Res, Shards} <- [ + {0, []}, + {0, [shard(1, ?RING_END)]}, + {1, [shard(0, ?RING_END)]}, + {1, [shard(0, ?RING_END), shard(1, ?RING_END)]}, + {2, [shard(0, ?RING_END), shard(0, ?RING_END)]}, + {2, [shard(0, 1), shard(2, ?RING_END), shard(0, ?RING_END)]}, + {0, [shard(0, 3), shard(5, ?RING_END), shard(1, ?RING_END)]} + ] + ]. shard(Begin, End) -> #shard{range = [Begin, End]}. diff --git a/src/mem3/test/eunit/mem3_bdu_test.erl b/src/mem3/test/eunit/mem3_bdu_test.erl index ad047f6e9..849295691 100644 --- a/src/mem3/test/eunit/mem3_bdu_test.erl +++ b/src/mem3/test/eunit/mem3_bdu_test.erl @@ -12,11 +12,9 @@ -module(mem3_bdu_test). - -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). - -define(TDEF_FE(Name), fun(Arg) -> {atom_to_list(Name), ?_test(Name(Arg))} end). -define(USER, "mem3_bdu_test_admin"). @@ -24,10 +22,9 @@ -define(AUTH, {basic_auth, {?USER, ?PASS}}). -define(JSON, {"Content-Type", "application/json"}). - setup() -> Hashed = couch_passwords:hash_admin_password(?PASS), - ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist=false), + ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist = false), Addr = config:get("chttpd", "bind_address", "127.0.0.1"), Db = ?tempdb(), Port = mochiweb_socket_server:get(chttpd, port), @@ -35,29 +32,27 @@ setup() -> ShardsDb = "_node/_local/" ++ config:get("mem3", "shards_db", "_dbs"), {Url, Db, ShardsDb}. - teardown({Url, Db, _}) -> sync_delete_db(Url, Db), - ok = config:delete("admins", ?USER, _Persist=false). - + ok = config:delete("admins", ?USER, _Persist = false). start_couch() -> test_util:start_couch([mem3, chttpd]). - stop_couch(Ctx) -> test_util:stop_couch(Ctx). - mem3_bdu_shard_doc_test_() -> { "mem3 bdu shard doc tests", { setup, - fun start_couch/0, fun stop_couch/1, + fun start_couch/0, + fun stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ ?TDEF_FE(t_can_insert_shard_map_doc), ?TDEF_FE(t_missing_by_node_section), @@ -77,7 +72,6 @@ mem3_bdu_shard_doc_test_() -> } }. - t_can_insert_shard_map_doc({Top, Db, ShardsDb}) -> Node = atom_to_binary(node(), utf8), Range = <<"00000000-ffffffff">>, @@ -91,7 +85,6 @@ t_can_insert_shard_map_doc({Top, Db, ShardsDb}) -> ?assertEqual(201, Code), ?assertMatch(#{<<"ok">> := true}, Res). - t_missing_by_node_section({Top, Db, ShardsDb}) -> Node = atom_to_binary(node(), utf8), Range = <<"00000000-ffffffff">>, @@ -101,7 +94,6 @@ t_missing_by_node_section({Top, Db, ShardsDb}) -> }, ?assertMatch({403, _}, req(post, Top ++ ShardsDb, ShardMap)). - t_by_node_not_a_map({Top, Db, ShardsDb}) -> Node = atom_to_binary(node(), utf8), Range = <<"00000000-ffffffff">>, @@ -112,7 +104,6 @@ t_by_node_not_a_map({Top, Db, ShardsDb}) -> }, ?assertMatch({403, _}, req(post, Top ++ ShardsDb, ShardMap)). - t_missing_by_range_section({Top, Db, ShardsDb}) -> Node = atom_to_binary(node(), utf8), Range = <<"00000000-ffffffff">>, @@ -122,7 +113,6 @@ t_missing_by_range_section({Top, Db, ShardsDb}) -> }, ?assertMatch({403, _}, req(post, Top ++ ShardsDb, ShardMap)). - t_by_range_not_a_map({Top, Db, ShardsDb}) -> Node = atom_to_binary(node(), utf8), Range = <<"00000000-ffffffff">>, @@ -133,7 +123,6 @@ t_by_range_not_a_map({Top, Db, ShardsDb}) -> }, ?assertMatch({403, _}, req(post, Top ++ ShardsDb, ShardMap)). - t_missing_range_in_by_range({Top, Db, ShardsDb}) -> Node = atom_to_binary(node(), utf8), Range = <<"00000000-ffffffff">>, @@ -144,7 +133,6 @@ t_missing_range_in_by_range({Top, Db, ShardsDb}) -> }, ?assertMatch({403, _}, req(post, Top ++ ShardsDb, ShardMap)). - t_missing_node_in_by_range_node_list({Top, Db, ShardsDb}) -> Node = atom_to_binary(node(), utf8), Range = <<"00000000-ffffffff">>, @@ -155,7 +143,6 @@ t_missing_node_in_by_range_node_list({Top, Db, ShardsDb}) -> }, ?assertMatch({403, _}, req(post, Top ++ ShardsDb, ShardMap)). - t_missing_node_in_by_node({Top, Db, ShardsDb}) -> Node = atom_to_binary(node(), utf8), Range = <<"00000000-ffffffff">>, @@ -166,7 +153,6 @@ t_missing_node_in_by_node({Top, Db, ShardsDb}) -> }, ?assertMatch({403, _}, req(post, Top ++ ShardsDb, ShardMap)). - t_missing_range_in_by_node_range_list({Top, Db, ShardsDb}) -> Node = atom_to_binary(node(), utf8), Range = <<"00000000-ffffffff">>, @@ -177,7 +163,6 @@ t_missing_range_in_by_node_range_list({Top, Db, ShardsDb}) -> }, ?assertMatch({403, _}, req(post, Top ++ ShardsDb, ShardMap)). - t_by_node_val_not_array({Top, Db, ShardsDb}) -> Node = atom_to_binary(node(), utf8), Range = <<"00000000-ffffffff">>, @@ -188,7 +173,6 @@ t_by_node_val_not_array({Top, Db, ShardsDb}) -> }, ?assertMatch({403, _}, req(post, Top ++ ShardsDb, ShardMap)). - t_by_range_val_not_array({Top, Db, ShardsDb}) -> Node = atom_to_binary(node(), utf8), Range = <<"00000000-ffffffff">>, @@ -199,7 +183,6 @@ t_by_range_val_not_array({Top, Db, ShardsDb}) -> }, ?assertMatch({403, _}, req(post, Top ++ ShardsDb, ShardMap)). - t_design_docs_are_not_validated({Top, _, ShardsDb}) -> Suffix = integer_to_list(erlang:system_time() + rand:uniform(1000)), DDocId = list_to_binary("_design/ddoc_bdu_test-" ++ Suffix), @@ -214,7 +197,6 @@ t_design_docs_are_not_validated({Top, _, ShardsDb}) -> }, ?assertMatch({200, _}, req(post, Top ++ ShardsDb, Deleted)). - t_replicated_changes_not_validated({Top, Db, ShardsDb}) -> Node = atom_to_binary(node(), utf8), Range = <<"00000000-ffffffff">>, @@ -244,7 +226,6 @@ t_replicated_changes_not_validated({Top, Db, ShardsDb}) -> }, ?assertMatch({200, _}, req(post, Top ++ ShardsDb, Deleted)). - delete_db(Top, Db) when is_binary(Db) -> Url = Top ++ binary_to_list(Db), case test_request:get(Url, [?AUTH]) of @@ -255,7 +236,6 @@ delete_db(Top, Db) when is_binary(Db) -> ok end. - sync_delete_db(Top, Db) when is_binary(Db) -> delete_db(Top, Db), try @@ -268,15 +248,12 @@ sync_delete_db(Top, Db) when is_binary(Db) -> ok end. - req(Method, Url, #{} = Body) -> req(Method, Url, jiffy:encode(Body)); - req(Method, Url, Body) -> Headers = [?JSON, ?AUTH], {ok, Code, _, Res} = test_request:request(Method, Url, Headers, Body), {Code, jiffy:decode(Res, [return_maps])}. - suffix() -> integer_to_list(erlang:system_time(second)). diff --git a/src/mem3/test/eunit/mem3_cluster_test.erl b/src/mem3/test/eunit/mem3_cluster_test.erl index 4610d64bd..d1a0fcd38 100644 --- a/src/mem3/test/eunit/mem3_cluster_test.erl +++ b/src/mem3/test/eunit/mem3_cluster_test.erl @@ -21,7 +21,6 @@ cluster_stable/1 ]). - % Mem3 cluster callbacks cluster_unstable(Server) -> @@ -32,7 +31,6 @@ cluster_stable(Server) -> Server ! cluster_stable, Server. - mem3_cluster_test_test_() -> { foreach, @@ -43,81 +41,79 @@ mem3_cluster_test_test_() -> t_cluster_unstable_delivered_on_nodeup(), t_cluster_unstable_delivered_on_nodedown(), t_wait_period_is_reset_after_last_change() - ] + ] }. - t_cluster_stable_during_startup_period() -> - ?_test(begin + ?_test(begin {ok, Pid} = mem3_cluster:start_link(?MODULE, self(), 1, 2), register(?MODULE, Pid), receive cluster_stable -> ?assert(true) - after 1500 -> - throw(timeout) + after 1500 -> + throw(timeout) end, unlink(Pid), exit(Pid, kill) end). - t_cluster_unstable_delivered_on_nodeup() -> - ?_test(begin + ?_test(begin {ok, Pid} = mem3_cluster:start_link(?MODULE, self(), 1, 2), register(?MODULE, Pid), Pid ! {nodeup, node()}, receive cluster_unstable -> ?assert(true) - after 1000 -> - throw(timeout) + after 1000 -> + throw(timeout) end, unlink(Pid), exit(Pid, kill) end). - t_cluster_unstable_delivered_on_nodedown() -> - ?_test(begin + ?_test(begin {ok, Pid} = mem3_cluster:start_link(?MODULE, self(), 1, 2), register(?MODULE, Pid), Pid ! {nodedown, node()}, receive cluster_unstable -> ?assert(true) - after 1000 -> - throw(timeout) + after 1000 -> + throw(timeout) end, unlink(Pid), exit(Pid, kill) end). - t_wait_period_is_reset_after_last_change() -> - ?_test(begin + ?_test(begin {ok, Pid} = mem3_cluster:start_link(?MODULE, self(), 1, 1), register(?MODULE, Pid), timer:sleep(800), - Pid ! {nodeup, node()}, % after 800 sec send a nodeup + % after 800 sec send a nodeup + Pid ! {nodeup, node()}, receive cluster_stable -> ?assert(false) - after 400 -> - ?assert(true) % stability check should have been reset + after 400 -> + % stability check should have been reset + ?assert(true) end, timer:sleep(1000), receive cluster_stable -> ?assert(true) - after 0 -> - ?assert(false) % cluster_stable arrives after enough quiet time + after 0 -> + % cluster_stable arrives after enough quiet time + ?assert(false) end, unlink(Pid), exit(Pid, kill) end). - % Test helper functions setup() -> diff --git a/src/mem3/test/eunit/mem3_hash_test.erl b/src/mem3/test/eunit/mem3_hash_test.erl index 7a40c5366..beeb0ac63 100644 --- a/src/mem3/test/eunit/mem3_hash_test.erl +++ b/src/mem3/test/eunit/mem3_hash_test.erl @@ -15,9 +15,9 @@ -include_lib("eunit/include/eunit.hrl"). hash_test() -> - ?assertEqual(1624516141,mem3_hash:crc32(0)), - ?assertEqual(3816901808,mem3_hash:crc32("0")), - ?assertEqual(3523407757,mem3_hash:crc32(<<0>>)), - ?assertEqual(4108050209,mem3_hash:crc32(<<"0">>)), - ?assertEqual(3094724072,mem3_hash:crc32(zero)), + ?assertEqual(1624516141, mem3_hash:crc32(0)), + ?assertEqual(3816901808, mem3_hash:crc32("0")), + ?assertEqual(3523407757, mem3_hash:crc32(<<0>>)), + ?assertEqual(4108050209, mem3_hash:crc32(<<"0">>)), + ?assertEqual(3094724072, mem3_hash:crc32(zero)), ok. diff --git a/src/mem3/test/eunit/mem3_rep_test.erl b/src/mem3/test/eunit/mem3_rep_test.erl index 4a46e7b93..31a6d9b77 100644 --- a/src/mem3/test/eunit/mem3_rep_test.erl +++ b/src/mem3/test/eunit/mem3_rep_test.erl @@ -12,15 +12,14 @@ -module(mem3_rep_test). - -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). -include_lib("couch_mrview/include/couch_mrview.hrl"). -include_lib("mem3/include/mem3.hrl"). - -define(ID, <<"_id">>). --define(TIMEOUT, 60). % seconds +% seconds +-define(TIMEOUT, 60). setup() -> {AllSrc, AllTgt} = {?tempdb(), ?tempdb()}, @@ -32,28 +31,26 @@ setup() -> create_db(PartTgt, [{q, 2}, {n, 1}, {props, PartProps}]), #{allsrc => AllSrc, alltgt => AllTgt, partsrc => PartSrc, parttgt => PartTgt}. - teardown(#{} = Dbs) -> maps:map(fun(_, Db) -> delete_db(Db) end, Dbs). - start_couch() -> test_util:start_couch([mem3, fabric]). - stop_couch(Ctx) -> test_util:stop_couch(Ctx). - mem3_reshard_db_test_() -> { "mem3 rep db tests", { setup, - fun start_couch/0, fun stop_couch/1, + fun start_couch/0, + fun stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun replicate_basics/1, fun replicate_small_batches/1, @@ -64,150 +61,148 @@ mem3_reshard_db_test_() -> } }. - replicate_basics(#{allsrc := AllSrc, alltgt := AllTgt}) -> - {timeout, ?TIMEOUT, ?_test(begin - DocSpec = #{docs => 10, delete => [5, 9]}, - add_test_docs(AllSrc, DocSpec), - SDocs = get_all_docs(AllSrc), - - [Src] = lists:sort(mem3:local_shards(AllSrc)), - [Tgt1, Tgt2] = lists:sort(mem3:local_shards(AllTgt)), - #shard{range = R1} = Tgt1, - #shard{range = R2} = Tgt2, - TMap = #{R1 => Tgt1, R2 => Tgt2}, - Opts = [{batch_size, 1000}, {batch_count, all}], - ?assertMatch({ok, 0}, mem3_rep:go(Src, TMap, Opts)), - - ?assertEqual(SDocs, get_all_docs(AllTgt)) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + DocSpec = #{docs => 10, delete => [5, 9]}, + add_test_docs(AllSrc, DocSpec), + SDocs = get_all_docs(AllSrc), + + [Src] = lists:sort(mem3:local_shards(AllSrc)), + [Tgt1, Tgt2] = lists:sort(mem3:local_shards(AllTgt)), + #shard{range = R1} = Tgt1, + #shard{range = R2} = Tgt2, + TMap = #{R1 => Tgt1, R2 => Tgt2}, + Opts = [{batch_size, 1000}, {batch_count, all}], + ?assertMatch({ok, 0}, mem3_rep:go(Src, TMap, Opts)), + + ?assertEqual(SDocs, get_all_docs(AllTgt)) + end)}. replicate_small_batches(#{allsrc := AllSrc, alltgt := AllTgt}) -> - {timeout, ?TIMEOUT, ?_test(begin - DocSpec = #{docs => 10, delete => [5, 9]}, - add_test_docs(AllSrc, DocSpec), - SDocs = get_all_docs(AllSrc), - - [Src] = lists:sort(mem3:local_shards(AllSrc)), - [Tgt1, Tgt2] = lists:sort(mem3:local_shards(AllTgt)), - #shard{range = R1} = Tgt1, - #shard{range = R2} = Tgt2, - TMap = #{R1 => Tgt1, R2 => Tgt2}, - Opts = [{batch_size, 2}, {batch_count, all}], - ?assertMatch({ok, 0}, mem3_rep:go(Src, TMap, Opts)), - - ?assertEqual(SDocs, get_all_docs(AllTgt)) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + DocSpec = #{docs => 10, delete => [5, 9]}, + add_test_docs(AllSrc, DocSpec), + SDocs = get_all_docs(AllSrc), + + [Src] = lists:sort(mem3:local_shards(AllSrc)), + [Tgt1, Tgt2] = lists:sort(mem3:local_shards(AllTgt)), + #shard{range = R1} = Tgt1, + #shard{range = R2} = Tgt2, + TMap = #{R1 => Tgt1, R2 => Tgt2}, + Opts = [{batch_size, 2}, {batch_count, all}], + ?assertMatch({ok, 0}, mem3_rep:go(Src, TMap, Opts)), + + ?assertEqual(SDocs, get_all_docs(AllTgt)) + end)}. replicate_low_batch_count(#{allsrc := AllSrc, alltgt := AllTgt}) -> - {timeout, ?TIMEOUT, ?_test(begin - DocSpec = #{docs => 10, delete => [5, 9]}, - add_test_docs(AllSrc, DocSpec), - SDocs = get_all_docs(AllSrc), - - [Src] = lists:sort(mem3:local_shards(AllSrc)), - [Tgt1, Tgt2] = lists:sort(mem3:local_shards(AllTgt)), - #shard{range = R1} = Tgt1, - #shard{range = R2} = Tgt2, - TMap = #{R1 => Tgt1, R2 => Tgt2}, + {timeout, ?TIMEOUT, + ?_test(begin + DocSpec = #{docs => 10, delete => [5, 9]}, + add_test_docs(AllSrc, DocSpec), + SDocs = get_all_docs(AllSrc), - Opts1 = [{batch_size, 2}, {batch_count, 1}], - ?assertMatch({ok, 8}, mem3_rep:go(Src, TMap, Opts1)), + [Src] = lists:sort(mem3:local_shards(AllSrc)), + [Tgt1, Tgt2] = lists:sort(mem3:local_shards(AllTgt)), + #shard{range = R1} = Tgt1, + #shard{range = R2} = Tgt2, + TMap = #{R1 => Tgt1, R2 => Tgt2}, - Opts2 = [{batch_size, 1}, {batch_count, 2}], - ?assertMatch({ok, 6}, mem3_rep:go(Src, TMap, Opts2)), + Opts1 = [{batch_size, 2}, {batch_count, 1}], + ?assertMatch({ok, 8}, mem3_rep:go(Src, TMap, Opts1)), - Opts3 = [{batch_size, 1000}, {batch_count, all}], - ?assertMatch({ok, 0}, mem3_rep:go(Src, TMap, Opts3)), + Opts2 = [{batch_size, 1}, {batch_count, 2}], + ?assertMatch({ok, 6}, mem3_rep:go(Src, TMap, Opts2)), - ?assertEqual(SDocs, get_all_docs(AllTgt)) - end)}. + Opts3 = [{batch_size, 1000}, {batch_count, all}], + ?assertMatch({ok, 0}, mem3_rep:go(Src, TMap, Opts3)), + ?assertEqual(SDocs, get_all_docs(AllTgt)) + end)}. replicate_with_partitions(#{partsrc := PartSrc, parttgt := PartTgt}) -> - {timeout, ?TIMEOUT, ?_test(begin - DocSpec = #{ - pdocs => #{ - <<"PX">> => 15, - <<"PY">> => 19 - } - }, - add_test_docs(PartSrc, DocSpec), - SDocs = get_all_docs(PartSrc), - PXSrc = get_partition_info(PartSrc, <<"PX">>), - PYSrc = get_partition_info(PartSrc, <<"PY">>), - - [Src] = lists:sort(mem3:local_shards(PartSrc)), - [Tgt1, Tgt2] = lists:sort(mem3:local_shards(PartTgt)), - #shard{range = R1} = Tgt1, - #shard{range = R2} = Tgt2, - TMap = #{R1 => Tgt1, R2 => Tgt2}, - Opts = [{batch_size, 1000}, {batch_count, all}], - ?assertMatch({ok, 0}, mem3_rep:go(Src, TMap, Opts)), - - ?assertEqual(PXSrc, get_partition_info(PartTgt, <<"PX">>)), - ?assertEqual(PYSrc, get_partition_info(PartTgt, <<"PY">>)), - ?assertEqual(SDocs, get_all_docs(PartTgt)) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + DocSpec = #{ + pdocs => #{ + <<"PX">> => 15, + <<"PY">> => 19 + } + }, + add_test_docs(PartSrc, DocSpec), + SDocs = get_all_docs(PartSrc), + PXSrc = get_partition_info(PartSrc, <<"PX">>), + PYSrc = get_partition_info(PartSrc, <<"PY">>), + + [Src] = lists:sort(mem3:local_shards(PartSrc)), + [Tgt1, Tgt2] = lists:sort(mem3:local_shards(PartTgt)), + #shard{range = R1} = Tgt1, + #shard{range = R2} = Tgt2, + TMap = #{R1 => Tgt1, R2 => Tgt2}, + Opts = [{batch_size, 1000}, {batch_count, all}], + ?assertMatch({ok, 0}, mem3_rep:go(Src, TMap, Opts)), + + ?assertEqual(PXSrc, get_partition_info(PartTgt, <<"PX">>)), + ?assertEqual(PYSrc, get_partition_info(PartTgt, <<"PY">>)), + ?assertEqual(SDocs, get_all_docs(PartTgt)) + end)}. get_partition_info(DbName, Partition) -> with_proc(fun() -> {ok, PInfo} = fabric:get_partition_info(DbName, Partition), - maps:with([ - <<"doc_count">>, <<"doc_del_count">>, <<"partition">> - ], to_map(PInfo)) + maps:with( + [ + <<"doc_count">>, <<"doc_del_count">>, <<"partition">> + ], + to_map(PInfo) + ) end). - get_all_docs(DbName) -> get_all_docs(DbName, #mrargs{}). - get_all_docs(DbName, #mrargs{} = QArgs0) -> GL = erlang:group_leader(), - with_proc(fun() -> - Cb = fun - ({row, Props}, Acc) -> - Doc = to_map(couch_util:get_value(doc, Props)), - #{?ID := Id} = Doc, - {ok, Acc#{Id => Doc}}; - ({meta, _}, Acc) -> {ok, Acc}; - (complete, Acc) -> {ok, Acc} + with_proc( + fun() -> + Cb = fun + ({row, Props}, Acc) -> + Doc = to_map(couch_util:get_value(doc, Props)), + #{?ID := Id} = Doc, + {ok, Acc#{Id => Doc}}; + ({meta, _}, Acc) -> + {ok, Acc}; + (complete, Acc) -> + {ok, Acc} + end, + QArgs = QArgs0#mrargs{include_docs = true}, + {ok, Docs} = fabric:all_docs(DbName, Cb, #{}, QArgs), + Docs end, - QArgs = QArgs0#mrargs{include_docs = true}, - {ok, Docs} = fabric:all_docs(DbName, Cb, #{}, QArgs), - Docs - end, GL). - + GL + ). to_map([_ | _] = Props) -> to_map({Props}); - to_map({[_ | _]} = EJson) -> jiffy:decode(jiffy:encode(EJson), [return_maps]). - create_db(DbName, Opts) -> GL = erlang:group_leader(), with_proc(fun() -> fabric:create_db(DbName, Opts) end, GL). - delete_db(DbName) -> GL = erlang:group_leader(), with_proc(fun() -> fabric:delete_db(DbName, [?ADMIN_CTX]) end, GL). - with_proc(Fun) -> with_proc(Fun, undefined, 30000). - with_proc(Fun, GroupLeader) -> with_proc(Fun, GroupLeader, 30000). - with_proc(Fun, GroupLeader, Timeout) -> {Pid, Ref} = spawn_monitor(fun() -> case GroupLeader of @@ -227,21 +222,23 @@ with_proc(Fun, GroupLeader, Timeout) -> error({with_proc_timeout, Fun, Timeout}) end. - add_test_docs(DbName, #{} = DocSpec) -> - Docs = docs(maps:get(docs, DocSpec, [])) - ++ pdocs(maps:get(pdocs, DocSpec, #{})), + Docs = + docs(maps:get(docs, DocSpec, [])) ++ + pdocs(maps:get(pdocs, DocSpec, #{})), Res = update_docs(DbName, Docs), - Docs1 = lists:map(fun({Doc, {ok, {RevPos, Rev}}}) -> - Doc#doc{revs = {RevPos, [Rev]}} - end, lists:zip(Docs, Res)), + Docs1 = lists:map( + fun({Doc, {ok, {RevPos, Rev}}}) -> + Doc#doc{revs = {RevPos, [Rev]}} + end, + lists:zip(Docs, Res) + ), case delete_docs(maps:get(delete, DocSpec, []), Docs1) of [] -> ok; [_ | _] = Deleted -> update_docs(DbName, Deleted) end, ok. - update_docs(DbName, Docs) -> with_proc(fun() -> case fabric:update_docs(DbName, Docs, [?ADMIN_CTX]) of @@ -250,29 +247,32 @@ update_docs(DbName, Docs) -> end end). - delete_docs([S, E], Docs) when E >= S -> ToDelete = [doc_id(<<"">>, I) || I <- lists:seq(S, E)], - lists:filtermap(fun(#doc{id = Id} = Doc) -> - case lists:member(Id, ToDelete) of - true -> {true, Doc#doc{deleted = true}}; - false -> false - end - end, Docs); + lists:filtermap( + fun(#doc{id = Id} = Doc) -> + case lists:member(Id, ToDelete) of + true -> {true, Doc#doc{deleted = true}}; + false -> false + end + end, + Docs + ); delete_docs(_, _) -> []. - pdocs(#{} = PMap) -> - maps:fold(fun(Part, DocSpec, DocsAcc) -> - docs(DocSpec, <<Part/binary, ":">>) ++ DocsAcc - end, [], PMap). - + maps:fold( + fun(Part, DocSpec, DocsAcc) -> + docs(DocSpec, <<Part/binary, ":">>) ++ DocsAcc + end, + [], + PMap + ). docs(DocSpec) -> docs(DocSpec, <<"">>). - docs(N, Prefix) when is_integer(N), N > 0 -> docs([0, N - 1], Prefix); docs([S, E], Prefix) when E >= S -> @@ -280,12 +280,10 @@ docs([S, E], Prefix) when E >= S -> docs(_, _) -> []. - doc(Pref, Id) -> Body = bodyprops(), doc(Pref, Id, Body, 42). - doc(Pref, Id, BodyProps, AttSize) -> #doc{ id = doc_id(Pref, Id), @@ -293,29 +291,28 @@ doc(Pref, Id, BodyProps, AttSize) -> atts = atts(AttSize) }. - doc_id(Pref, Id) -> IdBin = iolist_to_binary(io_lib:format("~5..0B", [Id])), <<Pref/binary, IdBin/binary>>. - bodyprops() -> [ - {<<"g">>, {[ - {<<"type">>, <<"Polygon">>}, - {<<"coordinates">>, [[[-71.0, 48.4], [-70.0, 48.4], [-71.0, 48.4]]]} - ]}} + {<<"g">>, + {[ + {<<"type">>, <<"Polygon">>}, + {<<"coordinates">>, [[[-71.0, 48.4], [-70.0, 48.4], [-71.0, 48.4]]]} + ]}} ]. - atts(0) -> []; - atts(Size) when is_integer(Size), Size >= 1 -> - Data = << <<"x">> || _ <- lists:seq(1, Size) >>, - [couch_att:new([ - {name, <<"att">>}, - {type, <<"app/binary">>}, - {att_len, Size}, - {data, Data} - ])]. + Data = <<<<"x">> || _ <- lists:seq(1, Size)>>, + [ + couch_att:new([ + {name, <<"att">>}, + {type, <<"app/binary">>}, + {att_len, Size}, + {data, Data} + ]) + ]. diff --git a/src/mem3/test/eunit/mem3_reshard_api_test.erl b/src/mem3/test/eunit/mem3_reshard_api_test.erl index c4df24ad3..6e4107a5c 100644 --- a/src/mem3/test/eunit/mem3_reshard_api_test.erl +++ b/src/mem3/test/eunit/mem3_reshard_api_test.erl @@ -12,12 +12,10 @@ -module(mem3_reshard_api_test). - -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). -include_lib("mem3/src/mem3_reshard.hrl"). - -define(USER, "mem3_reshard_api_test_admin"). -define(PASS, "pass"). -define(AUTH, {basic_auth, {?USER, ?PASS}}). @@ -27,12 +25,12 @@ -define(STATE, "_reshard/state"). -define(ID, <<"id">>). -define(OK, <<"ok">>). --define(TIMEOUT, 60). % seconds - +% seconds +-define(TIMEOUT, 60). setup() -> Hashed = couch_passwords:hash_admin_password(?PASS), - ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist=false), + ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist = false), Addr = config:get("chttpd", "bind_address", "127.0.0.1"), Port = mochiweb_socket_server:get(chttpd, port), Url = lists:concat(["http://", Addr, ":", Port, "/"]), @@ -42,37 +40,35 @@ setup() -> create_db(Url, Db3, "?q=2&n=1"), {Url, {Db1, Db2, Db3}}. - teardown({Url, {Db1, Db2, Db3}}) -> mem3_reshard:reset_state(), application:unset_env(mem3, reshard_disabled), delete_db(Url, Db1), delete_db(Url, Db2), delete_db(Url, Db3), - ok = config:delete("reshard", "max_jobs", _Persist=false), - ok = config:delete("reshard", "require_node_param", _Persist=false), - ok = config:delete("reshard", "require_range_param", _Persist=false), - ok = config:delete("admins", ?USER, _Persist=false), + ok = config:delete("reshard", "max_jobs", _Persist = false), + ok = config:delete("reshard", "require_node_param", _Persist = false), + ok = config:delete("reshard", "require_range_param", _Persist = false), + ok = config:delete("admins", ?USER, _Persist = false), meck:unload(). - start_couch() -> test_util:start_couch([mem3, chttpd]). - stop_couch(Ctx) -> test_util:stop_couch(Ctx). - mem3_reshard_api_test_() -> { "mem3 shard split api tests", { setup, - fun start_couch/0, fun stop_couch/1, + fun start_couch/0, + fun stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun basics/1, fun create_job_basic/1, @@ -109,632 +105,768 @@ mem3_reshard_api_test_() -> } }. - basics({Top, _}) -> - {timeout, ?TIMEOUT, ?_test(begin - % GET /_reshard - ?assertMatch({200, #{ - <<"state">> := <<"running">>, - <<"state_reason">> := null, - <<"completed">> := 0, - <<"failed">> := 0, - <<"running">> := 0, - <<"stopped">> := 0, - <<"total">> := 0 - }}, req(get, Top ++ ?RESHARD)), - - % GET _reshard/state - ?assertMatch({200, #{<<"state">> := <<"running">>}}, - req(get, Top ++ ?STATE)), - - % GET _reshard/jobs - ?assertMatch({200, #{ - <<"jobs">> := [], - <<"offset">> := 0, - <<"total_rows">> := 0 - }}, req(get, Top ++ ?JOBS)), - - % Some invalid paths and methods - ?assertMatch({404, _}, req(get, Top ++ ?RESHARD ++ "/invalidpath")), - ?assertMatch({405, _}, req(put, Top ++ ?RESHARD, #{dont => thinkso})), - ?assertMatch({405, _}, req(post, Top ++ ?RESHARD, #{nope => nope})) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + % GET /_reshard + ?assertMatch( + {200, #{ + <<"state">> := <<"running">>, + <<"state_reason">> := null, + <<"completed">> := 0, + <<"failed">> := 0, + <<"running">> := 0, + <<"stopped">> := 0, + <<"total">> := 0 + }}, + req(get, Top ++ ?RESHARD) + ), + + % GET _reshard/state + ?assertMatch( + {200, #{<<"state">> := <<"running">>}}, + req(get, Top ++ ?STATE) + ), + + % GET _reshard/jobs + ?assertMatch( + {200, #{ + <<"jobs">> := [], + <<"offset">> := 0, + <<"total_rows">> := 0 + }}, + req(get, Top ++ ?JOBS) + ), + + % Some invalid paths and methods + ?assertMatch({404, _}, req(get, Top ++ ?RESHARD ++ "/invalidpath")), + ?assertMatch({405, _}, req(put, Top ++ ?RESHARD, #{dont => thinkso})), + ?assertMatch({405, _}, req(post, Top ++ ?RESHARD, #{nope => nope})) + end)}. create_job_basic({Top, {Db1, _, _}}) -> - {timeout, ?TIMEOUT, ?_test(begin - % POST /_reshard/jobs - {C1, R1} = req(post, Top ++ ?JOBS, #{type => split, db => Db1}), - ?assertEqual(201, C1), - ?assertMatch([#{?OK := true, ?ID := J, <<"shard">> := S}] - when is_binary(J) andalso is_binary(S), R1), - [#{?ID := Id, <<"shard">> := Shard}] = R1, - - % GET /_reshard/jobs - ?assertMatch({200, #{ - <<"jobs">> := [#{?ID := Id, <<"type">> := <<"split">>}], - <<"offset">> := 0, - <<"total_rows">> := 1 - }}, req(get, Top ++ ?JOBS)), - - % GET /_reshard/job/$jobid - {C2, R2} = req(get, Top ++ ?JOBS ++ ?b2l(Id)), - ?assertEqual(200, C2), - ThisNode = atom_to_binary(node(), utf8), - ?assertMatch(#{?ID := Id}, R2), - ?assertMatch(#{<<"type">> := <<"split">>}, R2), - ?assertMatch(#{<<"source">> := Shard}, R2), - ?assertMatch(#{<<"history">> := History} when length(History) > 1, R2), - ?assertMatch(#{<<"node">> := ThisNode}, R2), - ?assertMatch(#{<<"split_state">> := SSt} when is_binary(SSt), R2), - ?assertMatch(#{<<"job_state">> := JSt} when is_binary(JSt), R2), - ?assertMatch(#{<<"state_info">> := #{}}, R2), - ?assertMatch(#{<<"target">> := Target} when length(Target) == 2, R2), - - % GET /_reshard/job/$jobid/state - ?assertMatch({200, #{<<"state">> := S, <<"reason">> := R}} - when is_binary(S) andalso (is_binary(R) orelse R =:= null), - req(get, Top ++ ?JOBS ++ ?b2l(Id) ++ "/state")), - - % GET /_reshard - ?assertMatch({200, #{<<"state">> := <<"running">>, <<"total">> := 1}}, - req(get, Top ++ ?RESHARD)), - - % DELETE /_reshard/jobs/$jobid - ?assertMatch({200, #{?OK := true}}, - req(delete, Top ++ ?JOBS ++ ?b2l(Id))), - - % GET _reshard/jobs - ?assertMatch({200, #{<<"jobs">> := [], <<"total_rows">> := 0}}, - req(get, Top ++ ?JOBS)), - - % GET /_reshard/job/$jobid should be a 404 - ?assertMatch({404, #{}}, req(get, Top ++ ?JOBS ++ ?b2l(Id))), - - % DELETE /_reshard/jobs/$jobid should be a 404 as well - ?assertMatch({404, #{}}, req(delete, Top ++ ?JOBS ++ ?b2l(Id))) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + % POST /_reshard/jobs + {C1, R1} = req(post, Top ++ ?JOBS, #{type => split, db => Db1}), + ?assertEqual(201, C1), + ?assertMatch( + [#{?OK := true, ?ID := J, <<"shard">> := S}] when + is_binary(J) andalso is_binary(S), + R1 + ), + [#{?ID := Id, <<"shard">> := Shard}] = R1, + + % GET /_reshard/jobs + ?assertMatch( + {200, #{ + <<"jobs">> := [#{?ID := Id, <<"type">> := <<"split">>}], + <<"offset">> := 0, + <<"total_rows">> := 1 + }}, + req(get, Top ++ ?JOBS) + ), + + % GET /_reshard/job/$jobid + {C2, R2} = req(get, Top ++ ?JOBS ++ ?b2l(Id)), + ?assertEqual(200, C2), + ThisNode = atom_to_binary(node(), utf8), + ?assertMatch(#{?ID := Id}, R2), + ?assertMatch(#{<<"type">> := <<"split">>}, R2), + ?assertMatch(#{<<"source">> := Shard}, R2), + ?assertMatch(#{<<"history">> := History} when length(History) > 1, R2), + ?assertMatch(#{<<"node">> := ThisNode}, R2), + ?assertMatch(#{<<"split_state">> := SSt} when is_binary(SSt), R2), + ?assertMatch(#{<<"job_state">> := JSt} when is_binary(JSt), R2), + ?assertMatch(#{<<"state_info">> := #{}}, R2), + ?assertMatch(#{<<"target">> := Target} when length(Target) == 2, R2), + + % GET /_reshard/job/$jobid/state + ?assertMatch( + {200, #{<<"state">> := S, <<"reason">> := R}} when + is_binary(S) andalso (is_binary(R) orelse R =:= null), + req(get, Top ++ ?JOBS ++ ?b2l(Id) ++ "/state") + ), + + % GET /_reshard + ?assertMatch( + {200, #{<<"state">> := <<"running">>, <<"total">> := 1}}, + req(get, Top ++ ?RESHARD) + ), + + % DELETE /_reshard/jobs/$jobid + ?assertMatch( + {200, #{?OK := true}}, + req(delete, Top ++ ?JOBS ++ ?b2l(Id)) + ), + + % GET _reshard/jobs + ?assertMatch( + {200, #{<<"jobs">> := [], <<"total_rows">> := 0}}, + req(get, Top ++ ?JOBS) + ), + + % GET /_reshard/job/$jobid should be a 404 + ?assertMatch({404, #{}}, req(get, Top ++ ?JOBS ++ ?b2l(Id))), + + % DELETE /_reshard/jobs/$jobid should be a 404 as well + ?assertMatch({404, #{}}, req(delete, Top ++ ?JOBS ++ ?b2l(Id))) + end)}. create_two_jobs({Top, {Db1, Db2, _}}) -> - {timeout, ?TIMEOUT, ?_test(begin - Jobs = Top ++ ?JOBS, - - ?assertMatch({201, [#{?OK := true}]}, - req(post, Jobs, #{type => split, db => Db1})), - ?assertMatch({201, [#{?OK := true}]}, - req(post, Jobs, #{type => split, db => Db2})), - - ?assertMatch({200, #{<<"total">> := 2}}, req(get, Top ++ ?RESHARD)), - - ?assertMatch({200, #{ - <<"jobs">> := [#{?ID := Id1}, #{?ID := Id2}], - <<"offset">> := 0, - <<"total_rows">> := 2 - }} when Id1 =/= Id2, req(get, Jobs)), - - {200, #{<<"jobs">> := [#{?ID := Id1}, #{?ID := Id2}]}} = req(get, Jobs), - - {200, #{?OK := true}} = req(delete, Jobs ++ ?b2l(Id1)), - ?assertMatch({200, #{<<"total">> := 1}}, req(get, Top ++ ?RESHARD)), - {200, #{?OK := true}} = req(delete, Jobs ++ ?b2l(Id2)), - ?assertMatch({200, #{<<"total">> := 0}}, req(get, Top ++ ?RESHARD)) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + Jobs = Top ++ ?JOBS, + + ?assertMatch( + {201, [#{?OK := true}]}, + req(post, Jobs, #{type => split, db => Db1}) + ), + ?assertMatch( + {201, [#{?OK := true}]}, + req(post, Jobs, #{type => split, db => Db2}) + ), + + ?assertMatch({200, #{<<"total">> := 2}}, req(get, Top ++ ?RESHARD)), + + ?assertMatch( + {200, #{ + <<"jobs">> := [#{?ID := Id1}, #{?ID := Id2}], + <<"offset">> := 0, + <<"total_rows">> := 2 + }} when Id1 =/= Id2, + req(get, Jobs) + ), + + {200, #{<<"jobs">> := [#{?ID := Id1}, #{?ID := Id2}]}} = req(get, Jobs), + + {200, #{?OK := true}} = req(delete, Jobs ++ ?b2l(Id1)), + ?assertMatch({200, #{<<"total">> := 1}}, req(get, Top ++ ?RESHARD)), + {200, #{?OK := true}} = req(delete, Jobs ++ ?b2l(Id2)), + ?assertMatch({200, #{<<"total">> := 0}}, req(get, Top ++ ?RESHARD)) + end)}. create_multiple_jobs_from_one_post({Top, {_, _, Db3}}) -> - {timeout, ?TIMEOUT, ?_test(begin - Jobs = Top ++ ?JOBS, - {C1, R1} = req(post, Jobs, #{type => split, db => Db3}), - ?assertMatch({201, [#{?OK := true}, #{?OK := true}]}, {C1, R1}), - ?assertMatch({200, #{<<"total">> := 2}}, req(get, Top ++ ?RESHARD)) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + Jobs = Top ++ ?JOBS, + {C1, R1} = req(post, Jobs, #{type => split, db => Db3}), + ?assertMatch({201, [#{?OK := true}, #{?OK := true}]}, {C1, R1}), + ?assertMatch({200, #{<<"total">> := 2}}, req(get, Top ++ ?RESHARD)) + end)}. start_stop_cluster_basic({Top, _}) -> - {timeout, ?TIMEOUT, ?_test(begin - Url = Top ++ ?STATE, - - ?assertMatch({200, #{ - <<"state">> := <<"running">>, - <<"reason">> := null - }}, req(get, Url)), - - ?assertMatch({200, _}, req(put, Url, #{state => stopped})), - ?assertMatch({200, #{ - <<"state">> := <<"stopped">>, - <<"reason">> := R - }} when is_binary(R), req(get, Url)), - - ?assertMatch({200, _}, req(put, Url, #{state => running})), - - % Make sure the reason shows in the state GET request - Reason = <<"somereason">>, - ?assertMatch({200, _}, req(put, Url, #{state => stopped, - reason => Reason})), - ?assertMatch({200, #{<<"state">> := <<"stopped">>, - <<"reason">> := Reason}}, req(get, Url)), - - % Top level summary also shows the reason - ?assertMatch({200, #{ - <<"state">> := <<"stopped">>, - <<"state_reason">> := Reason - }}, req(get, Top ++ ?RESHARD)), - ?assertMatch({200, _}, req(put, Url, #{state => running})), - ?assertMatch({200, #{<<"state">> := <<"running">>}}, req(get, Url)) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + Url = Top ++ ?STATE, + + ?assertMatch( + {200, #{ + <<"state">> := <<"running">>, + <<"reason">> := null + }}, + req(get, Url) + ), + + ?assertMatch({200, _}, req(put, Url, #{state => stopped})), + ?assertMatch( + {200, #{ + <<"state">> := <<"stopped">>, + <<"reason">> := R + }} when is_binary(R), + req(get, Url) + ), + + ?assertMatch({200, _}, req(put, Url, #{state => running})), + + % Make sure the reason shows in the state GET request + Reason = <<"somereason">>, + ?assertMatch( + {200, _}, + req(put, Url, #{ + state => stopped, + reason => Reason + }) + ), + ?assertMatch( + {200, #{ + <<"state">> := <<"stopped">>, + <<"reason">> := Reason + }}, + req(get, Url) + ), + + % Top level summary also shows the reason + ?assertMatch( + {200, #{ + <<"state">> := <<"stopped">>, + <<"state_reason">> := Reason + }}, + req(get, Top ++ ?RESHARD) + ), + ?assertMatch({200, _}, req(put, Url, #{state => running})), + ?assertMatch({200, #{<<"state">> := <<"running">>}}, req(get, Url)) + end)}. test_disabled({Top, _}) -> - {timeout, ?TIMEOUT, ?_test(begin - application:set_env(mem3, reshard_disabled, true), - ?assertMatch({501, _}, req(get, Top ++ ?RESHARD)), - ?assertMatch({501, _}, req(put, Top ++ ?STATE, #{state => running})), - - application:unset_env(mem3, reshard_disabled), - ?assertMatch({200, _}, req(get, Top ++ ?RESHARD)), - ?assertMatch({200, _}, req(put, Top ++ ?STATE, #{state => running})) - end)}. + {timeout, ?TIMEOUT, + ?_test(begin + application:set_env(mem3, reshard_disabled, true), + ?assertMatch({501, _}, req(get, Top ++ ?RESHARD)), + ?assertMatch({501, _}, req(put, Top ++ ?STATE, #{state => running})), + application:unset_env(mem3, reshard_disabled), + ?assertMatch({200, _}, req(get, Top ++ ?RESHARD)), + ?assertMatch({200, _}, req(put, Top ++ ?STATE, #{state => running})) + end)}. start_stop_cluster_with_a_job({Top, {Db1, _, _}}) -> - {timeout, ?TIMEOUT, ?_test(begin - Url = Top ++ ?STATE, - - ?assertMatch({200, _}, req(put, Url, #{state => stopped})), - ?assertMatch({200, #{<<"state">> := <<"stopped">>}}, req(get, Url)), - - % Can add jobs with global state stopped, they just won't be running - {201, R1} = req(post, Top ++ ?JOBS, #{type => split, db => Db1}), - ?assertMatch([#{?OK := true}], R1), - [#{?ID := Id1}] = R1, - {200, J1} = req(get, Top ++ ?JOBS ++ ?b2l(Id1)), - ?assertMatch(#{?ID := Id1, <<"job_state">> := <<"stopped">>}, J1), - % Check summary stats - ?assertMatch({200, #{ - <<"state">> := <<"stopped">>, - <<"running">> := 0, - <<"stopped">> := 1, - <<"total">> := 1 - }}, req(get, Top ++ ?RESHARD)), - - % Can delete the job when stopped - {200, #{?OK := true}} = req(delete, Top ++ ?JOBS ++ ?b2l(Id1)), - ?assertMatch({200, #{ - <<"state">> := <<"stopped">>, - <<"running">> := 0, - <<"stopped">> := 0, - <<"total">> := 0 - }}, req(get, Top ++ ?RESHARD)), - - % Add same job again - {201, [#{?ID := Id2}]} = req(post, Top ++ ?JOBS, #{type => split, - db => Db1}), - ?assertMatch({200, #{?ID := Id2, <<"job_state">> := <<"stopped">>}}, - req(get, Top ++ ?JOBS ++ ?b2l(Id2))), - - % Job should start after resharding is started on the cluster - ?assertMatch({200, _}, req(put, Url, #{state => running})), - ?assertMatch({200, #{?ID := Id2, <<"job_state">> := JSt}} - when JSt =/= <<"stopped">>, req(get, Top ++ ?JOBS ++ ?b2l(Id2))) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + Url = Top ++ ?STATE, + + ?assertMatch({200, _}, req(put, Url, #{state => stopped})), + ?assertMatch({200, #{<<"state">> := <<"stopped">>}}, req(get, Url)), + + % Can add jobs with global state stopped, they just won't be running + {201, R1} = req(post, Top ++ ?JOBS, #{type => split, db => Db1}), + ?assertMatch([#{?OK := true}], R1), + [#{?ID := Id1}] = R1, + {200, J1} = req(get, Top ++ ?JOBS ++ ?b2l(Id1)), + ?assertMatch(#{?ID := Id1, <<"job_state">> := <<"stopped">>}, J1), + % Check summary stats + ?assertMatch( + {200, #{ + <<"state">> := <<"stopped">>, + <<"running">> := 0, + <<"stopped">> := 1, + <<"total">> := 1 + }}, + req(get, Top ++ ?RESHARD) + ), + + % Can delete the job when stopped + {200, #{?OK := true}} = req(delete, Top ++ ?JOBS ++ ?b2l(Id1)), + ?assertMatch( + {200, #{ + <<"state">> := <<"stopped">>, + <<"running">> := 0, + <<"stopped">> := 0, + <<"total">> := 0 + }}, + req(get, Top ++ ?RESHARD) + ), + + % Add same job again + {201, [#{?ID := Id2}]} = req(post, Top ++ ?JOBS, #{ + type => split, + db => Db1 + }), + ?assertMatch( + {200, #{?ID := Id2, <<"job_state">> := <<"stopped">>}}, + req(get, Top ++ ?JOBS ++ ?b2l(Id2)) + ), + + % Job should start after resharding is started on the cluster + ?assertMatch({200, _}, req(put, Url, #{state => running})), + ?assertMatch( + {200, #{?ID := Id2, <<"job_state">> := JSt}} when + JSt =/= <<"stopped">>, + req(get, Top ++ ?JOBS ++ ?b2l(Id2)) + ) + end)}. individual_job_start_stop({Top, {Db1, _, _}}) -> - {timeout, ?TIMEOUT, ?_test(begin - intercept_state(topoff1), - - Body = #{type => split, db => Db1}, - {201, [#{?ID := Id}]} = req(post, Top ++ ?JOBS, Body), - - JobUrl = Top ++ ?JOBS ++ ?b2l(Id), - StUrl = JobUrl ++ "/state", - - % Wait for the the job to start running and intercept it in topoff1 state - receive {JobPid, topoff1} -> ok end, - % Tell the intercept to never finish checkpointing so job is left hanging - % forever in running state - JobPid ! cancel, - ?assertMatch({200, #{<<"state">> := <<"running">>}}, req(get, StUrl)), - - {200, _} = req(put, StUrl, #{state => stopped}), - wait_state(StUrl, <<"stopped">>), - - % Stop/start resharding globally and job should still stay stopped - ?assertMatch({200, _}, req(put, Top ++ ?STATE, #{state => stopped})), - ?assertMatch({200, _}, req(put, Top ++ ?STATE, #{state => running})), - ?assertMatch({200, #{<<"state">> := <<"stopped">>}}, req(get, StUrl)), - - % Start the job again - ?assertMatch({200, _}, req(put, StUrl, #{state => running})), - % Wait for the the job to start running and intercept it in topoff1 state - receive {JobPid2, topoff1} -> ok end, - ?assertMatch({200, #{<<"state">> := <<"running">>}}, req(get, StUrl)), - % Let it continue running and it should complete eventually - JobPid2 ! continue, - wait_state(StUrl, <<"completed">>) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + intercept_state(topoff1), + + Body = #{type => split, db => Db1}, + {201, [#{?ID := Id}]} = req(post, Top ++ ?JOBS, Body), + + JobUrl = Top ++ ?JOBS ++ ?b2l(Id), + StUrl = JobUrl ++ "/state", + + % Wait for the the job to start running and intercept it in topoff1 state + receive + {JobPid, topoff1} -> ok + end, + % Tell the intercept to never finish checkpointing so job is left hanging + % forever in running state + JobPid ! cancel, + ?assertMatch({200, #{<<"state">> := <<"running">>}}, req(get, StUrl)), + + {200, _} = req(put, StUrl, #{state => stopped}), + wait_state(StUrl, <<"stopped">>), + + % Stop/start resharding globally and job should still stay stopped + ?assertMatch({200, _}, req(put, Top ++ ?STATE, #{state => stopped})), + ?assertMatch({200, _}, req(put, Top ++ ?STATE, #{state => running})), + ?assertMatch({200, #{<<"state">> := <<"stopped">>}}, req(get, StUrl)), + + % Start the job again + ?assertMatch({200, _}, req(put, StUrl, #{state => running})), + % Wait for the the job to start running and intercept it in topoff1 state + receive + {JobPid2, topoff1} -> ok + end, + ?assertMatch({200, #{<<"state">> := <<"running">>}}, req(get, StUrl)), + % Let it continue running and it should complete eventually + JobPid2 ! continue, + wait_state(StUrl, <<"completed">>) + end)}. individual_job_stop_when_cluster_stopped({Top, {Db1, _, _}}) -> - {timeout, ?TIMEOUT, ?_test(begin - intercept_state(topoff1), - - Body = #{type => split, db => Db1}, - {201, [#{?ID := Id}]} = req(post, Top ++ ?JOBS, Body), - - JobUrl = Top ++ ?JOBS ++ ?b2l(Id), - StUrl = JobUrl ++ "/state", - - % Wait for the the job to start running and intercept in topoff1 - receive {JobPid, topoff1} -> ok end, - % Tell the intercept to never finish checkpointing so job is left - % hanging forever in running state - JobPid ! cancel, - ?assertMatch({200, #{<<"state">> := <<"running">>}}, req(get, StUrl)), - - % Stop resharding globally - ?assertMatch({200, _}, req(put, Top ++ ?STATE, #{state => stopped})), - wait_state(StUrl, <<"stopped">>), - - % Stop the job specifically - {200, _} = req(put, StUrl, #{state => stopped}), - % Job stays stopped - ?assertMatch({200, #{<<"state">> := <<"stopped">>}}, req(get, StUrl)), - - % Set cluster to running again - ?assertMatch({200, _}, req(put, Top ++ ?STATE, #{state => running})), - - % The job should stay stopped - ?assertMatch({200, #{<<"state">> := <<"stopped">>}}, req(get, StUrl)), - - % It should be possible to resume job and it should complete - ?assertMatch({200, _}, req(put, StUrl, #{state => running})), - - % Wait for the the job to start running and intercept in topoff1 state - receive {JobPid2, topoff1} -> ok end, - ?assertMatch({200, #{<<"state">> := <<"running">>}}, req(get, StUrl)), - - % Let it continue running and it should complete eventually - JobPid2 ! continue, - wait_state(StUrl, <<"completed">>) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + intercept_state(topoff1), + + Body = #{type => split, db => Db1}, + {201, [#{?ID := Id}]} = req(post, Top ++ ?JOBS, Body), + + JobUrl = Top ++ ?JOBS ++ ?b2l(Id), + StUrl = JobUrl ++ "/state", + + % Wait for the the job to start running and intercept in topoff1 + receive + {JobPid, topoff1} -> ok + end, + % Tell the intercept to never finish checkpointing so job is left + % hanging forever in running state + JobPid ! cancel, + ?assertMatch({200, #{<<"state">> := <<"running">>}}, req(get, StUrl)), + + % Stop resharding globally + ?assertMatch({200, _}, req(put, Top ++ ?STATE, #{state => stopped})), + wait_state(StUrl, <<"stopped">>), + + % Stop the job specifically + {200, _} = req(put, StUrl, #{state => stopped}), + % Job stays stopped + ?assertMatch({200, #{<<"state">> := <<"stopped">>}}, req(get, StUrl)), + + % Set cluster to running again + ?assertMatch({200, _}, req(put, Top ++ ?STATE, #{state => running})), + + % The job should stay stopped + ?assertMatch({200, #{<<"state">> := <<"stopped">>}}, req(get, StUrl)), + + % It should be possible to resume job and it should complete + ?assertMatch({200, _}, req(put, StUrl, #{state => running})), + + % Wait for the the job to start running and intercept in topoff1 state + receive + {JobPid2, topoff1} -> ok + end, + ?assertMatch({200, #{<<"state">> := <<"running">>}}, req(get, StUrl)), + + % Let it continue running and it should complete eventually + JobPid2 ! continue, + wait_state(StUrl, <<"completed">>) + end)}. create_job_with_invalid_arguments({Top, {Db1, _, _}}) -> - {timeout, ?TIMEOUT, ?_test(begin - Jobs = Top ++ ?JOBS, - - % Nothing in the body - ?assertMatch({400, _}, req(post, Jobs, #{})), - - % Missing type - ?assertMatch({400, _}, req(post, Jobs, #{db => Db1})), - - % Have type but no db and no shard - ?assertMatch({400, _}, req(post, Jobs, #{type => split})), - - % Have type and db but db is invalid - ?assertMatch({400, _}, req(post, Jobs, #{db => <<"baddb">>, - type => split})), - - % Have type and shard but shard is not an existing database - ?assertMatch({404, _}, req(post, Jobs, #{type => split, - shard => <<"shards/80000000-ffffffff/baddb.1549492084">>})), - - % Bad range values, too large, different types, inverted - ?assertMatch({400, _}, req(post, Jobs, #{db => Db1, range => 42, - type => split})), - ?assertMatch({400, _}, req(post, Jobs, #{db => Db1, - range => <<"x">>, type => split})), - ?assertMatch({400, _}, req(post, Jobs, #{db => Db1, - range => <<"ffffffff-80000000">>, type => split})), - ?assertMatch({400, _}, req(post, Jobs, #{db => Db1, - range => <<"00000000-fffffffff">>, type => split})), - - % Can't have both db and shard - ?assertMatch({400, _}, req(post, Jobs, #{type => split, db => Db1, - shard => <<"blah">>})) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + Jobs = Top ++ ?JOBS, + + % Nothing in the body + ?assertMatch({400, _}, req(post, Jobs, #{})), + + % Missing type + ?assertMatch({400, _}, req(post, Jobs, #{db => Db1})), + + % Have type but no db and no shard + ?assertMatch({400, _}, req(post, Jobs, #{type => split})), + + % Have type and db but db is invalid + ?assertMatch( + {400, _}, + req(post, Jobs, #{ + db => <<"baddb">>, + type => split + }) + ), + + % Have type and shard but shard is not an existing database + ?assertMatch( + {404, _}, + req(post, Jobs, #{ + type => split, + shard => <<"shards/80000000-ffffffff/baddb.1549492084">> + }) + ), + + % Bad range values, too large, different types, inverted + ?assertMatch( + {400, _}, + req(post, Jobs, #{ + db => Db1, + range => 42, + type => split + }) + ), + ?assertMatch( + {400, _}, + req(post, Jobs, #{ + db => Db1, + range => <<"x">>, + type => split + }) + ), + ?assertMatch( + {400, _}, + req(post, Jobs, #{ + db => Db1, + range => <<"ffffffff-80000000">>, + type => split + }) + ), + ?assertMatch( + {400, _}, + req(post, Jobs, #{ + db => Db1, + range => <<"00000000-fffffffff">>, + type => split + }) + ), + + % Can't have both db and shard + ?assertMatch( + {400, _}, + req(post, Jobs, #{ + type => split, + db => Db1, + shard => <<"blah">> + }) + ) + end)}. create_job_with_db({Top, {Db1, _, _}}) -> - {timeout, ?TIMEOUT, ?_test(begin - Jobs = Top ++ ?JOBS, - Body1 = #{type => split, db => Db1}, - - % Node with db - N = atom_to_binary(node(), utf8), - {C1, R1} = req(post, Jobs, Body1#{node => N}), - ?assertMatch({201, [#{?OK := true}]}, {C1, R1}), - wait_to_complete_then_cleanup(Top, R1), - - % Range and db - {C2, R2} = req(post, Jobs, Body1#{range => <<"00000000-7fffffff">>}), - ?assertMatch({201, [#{?OK := true}]}, {C2, R2}), - wait_to_complete_then_cleanup(Top, R2), - - % Node, range and db - Range = <<"80000000-ffffffff">>, - {C3, R3} = req(post, Jobs, Body1#{range => Range, node => N}), - ?assertMatch({201, [#{?OK := true}]}, {C3, R3}), - wait_to_complete_then_cleanup(Top, R3), - - ?assertMatch([ - [16#00000000, 16#3fffffff], - [16#40000000, 16#7fffffff], - [16#80000000, 16#bfffffff], - [16#c0000000, 16#ffffffff] - ], [mem3:range(S) || S <- lists:sort(mem3:shards(Db1))]) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + Jobs = Top ++ ?JOBS, + Body1 = #{type => split, db => Db1}, + + % Node with db + N = atom_to_binary(node(), utf8), + {C1, R1} = req(post, Jobs, Body1#{node => N}), + ?assertMatch({201, [#{?OK := true}]}, {C1, R1}), + wait_to_complete_then_cleanup(Top, R1), + + % Range and db + {C2, R2} = req(post, Jobs, Body1#{range => <<"00000000-7fffffff">>}), + ?assertMatch({201, [#{?OK := true}]}, {C2, R2}), + wait_to_complete_then_cleanup(Top, R2), + + % Node, range and db + Range = <<"80000000-ffffffff">>, + {C3, R3} = req(post, Jobs, Body1#{range => Range, node => N}), + ?assertMatch({201, [#{?OK := true}]}, {C3, R3}), + wait_to_complete_then_cleanup(Top, R3), + + ?assertMatch( + [ + [16#00000000, 16#3fffffff], + [16#40000000, 16#7fffffff], + [16#80000000, 16#bfffffff], + [16#c0000000, 16#ffffffff] + ], + [mem3:range(S) || S <- lists:sort(mem3:shards(Db1))] + ) + end)}. create_job_with_shard_name({Top, {_, _, Db3}}) -> - {timeout, ?TIMEOUT, ?_test(begin - Jobs = Top ++ ?JOBS, - [S1, S2] = [mem3:name(S) || S <- lists:sort(mem3:shards(Db3))], - - % Shard only - {C1, R1} = req(post, Jobs, #{type => split, shard => S1}), - ?assertMatch({201, [#{?OK := true}]}, {C1, R1}), - wait_to_complete_then_cleanup(Top, R1), - - % Shard with a node - N = atom_to_binary(node(), utf8), - {C2, R2} = req(post, Jobs, #{type => split, shard => S2, node => N}), - ?assertMatch({201, [#{?OK := true}]}, {C2, R2}), - wait_to_complete_then_cleanup(Top, R2), - - ?assertMatch([ - [16#00000000, 16#3fffffff], - [16#40000000, 16#7fffffff], - [16#80000000, 16#bfffffff], - [16#c0000000, 16#ffffffff] - ], [mem3:range(S) || S <- lists:sort(mem3:shards(Db3))]) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + Jobs = Top ++ ?JOBS, + [S1, S2] = [mem3:name(S) || S <- lists:sort(mem3:shards(Db3))], + + % Shard only + {C1, R1} = req(post, Jobs, #{type => split, shard => S1}), + ?assertMatch({201, [#{?OK := true}]}, {C1, R1}), + wait_to_complete_then_cleanup(Top, R1), + + % Shard with a node + N = atom_to_binary(node(), utf8), + {C2, R2} = req(post, Jobs, #{type => split, shard => S2, node => N}), + ?assertMatch({201, [#{?OK := true}]}, {C2, R2}), + wait_to_complete_then_cleanup(Top, R2), + + ?assertMatch( + [ + [16#00000000, 16#3fffffff], + [16#40000000, 16#7fffffff], + [16#80000000, 16#bfffffff], + [16#c0000000, 16#ffffffff] + ], + [mem3:range(S) || S <- lists:sort(mem3:shards(Db3))] + ) + end)}. completed_job_handling({Top, {Db1, _, _}}) -> - {timeout, ?TIMEOUT, ?_test(begin - Jobs = Top ++ ?JOBS, - - % Run job to completion - {C1, R1} = req(post, Jobs, #{type => split, db => Db1}), - ?assertMatch({201, [#{?OK := true}]}, {C1, R1}), - [#{?ID := Id}] = R1, - wait_to_complete(Top, R1), - - % Check top level stats - ?assertMatch({200, #{ - <<"state">> := <<"running">>, - <<"state_reason">> := null, - <<"completed">> := 1, - <<"failed">> := 0, - <<"running">> := 0, - <<"stopped">> := 0, - <<"total">> := 1 - }}, req(get, Top ++ ?RESHARD)), - - % Job state itself - JobUrl = Jobs ++ ?b2l(Id), - ?assertMatch({200, #{ - <<"split_state">> := <<"completed">>, - <<"job_state">> := <<"completed">> - }}, req(get, JobUrl)), - - % Job's state endpoint - StUrl = Jobs ++ ?b2l(Id) ++ "/state", - ?assertMatch({200, #{<<"state">> := <<"completed">>}}, req(get, StUrl)), - - % Try to stop it and it should stay completed - {200, _} = req(put, StUrl, #{state => stopped}), - ?assertMatch({200, #{<<"state">> := <<"completed">>}}, req(get, StUrl)), - - % Try to resume it and it should stay completed - {200, _} = req(put, StUrl, #{state => running}), - ?assertMatch({200, #{<<"state">> := <<"completed">>}}, req(get, StUrl)), - - % Stop resharding globally and job should still stay completed - ?assertMatch({200, _}, req(put, Top ++ ?STATE, #{state => stopped})), - ?assertMatch({200, #{<<"state">> := <<"completed">>}}, req(get, StUrl)), - - % Start resharding and job stays completed - ?assertMatch({200, _}, req(put, Top ++ ?STATE, #{state => running})), - ?assertMatch({200, #{<<"state">> := <<"completed">>}}, req(get, StUrl)), - - ?assertMatch({200, #{?OK := true}}, req(delete, JobUrl)) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + Jobs = Top ++ ?JOBS, + + % Run job to completion + {C1, R1} = req(post, Jobs, #{type => split, db => Db1}), + ?assertMatch({201, [#{?OK := true}]}, {C1, R1}), + [#{?ID := Id}] = R1, + wait_to_complete(Top, R1), + + % Check top level stats + ?assertMatch( + {200, #{ + <<"state">> := <<"running">>, + <<"state_reason">> := null, + <<"completed">> := 1, + <<"failed">> := 0, + <<"running">> := 0, + <<"stopped">> := 0, + <<"total">> := 1 + }}, + req(get, Top ++ ?RESHARD) + ), + + % Job state itself + JobUrl = Jobs ++ ?b2l(Id), + ?assertMatch( + {200, #{ + <<"split_state">> := <<"completed">>, + <<"job_state">> := <<"completed">> + }}, + req(get, JobUrl) + ), + + % Job's state endpoint + StUrl = Jobs ++ ?b2l(Id) ++ "/state", + ?assertMatch({200, #{<<"state">> := <<"completed">>}}, req(get, StUrl)), + + % Try to stop it and it should stay completed + {200, _} = req(put, StUrl, #{state => stopped}), + ?assertMatch({200, #{<<"state">> := <<"completed">>}}, req(get, StUrl)), + + % Try to resume it and it should stay completed + {200, _} = req(put, StUrl, #{state => running}), + ?assertMatch({200, #{<<"state">> := <<"completed">>}}, req(get, StUrl)), + + % Stop resharding globally and job should still stay completed + ?assertMatch({200, _}, req(put, Top ++ ?STATE, #{state => stopped})), + ?assertMatch({200, #{<<"state">> := <<"completed">>}}, req(get, StUrl)), + + % Start resharding and job stays completed + ?assertMatch({200, _}, req(put, Top ++ ?STATE, #{state => running})), + ?assertMatch({200, #{<<"state">> := <<"completed">>}}, req(get, StUrl)), + + ?assertMatch({200, #{?OK := true}}, req(delete, JobUrl)) + end)}. handle_db_deletion_in_topoff1({Top, {Db1, _, _}}) -> - {timeout, ?TIMEOUT, ?_test(begin - JobId = delete_source_in_state(Top, Db1, topoff1), - wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"failed">>) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + JobId = delete_source_in_state(Top, Db1, topoff1), + wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"failed">>) + end)}. handle_db_deletion_in_initial_copy({Top, {Db1, _, _}}) -> - {timeout, ?TIMEOUT, ?_test(begin - JobId = delete_source_in_state(Top, Db1, initial_copy), - wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"failed">>) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + JobId = delete_source_in_state(Top, Db1, initial_copy), + wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"failed">>) + end)}. handle_db_deletion_in_copy_local_docs({Top, {Db1, _, _}}) -> - {timeout, ?TIMEOUT, ?_test(begin - JobId = delete_source_in_state(Top, Db1, copy_local_docs), - wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"failed">>) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + JobId = delete_source_in_state(Top, Db1, copy_local_docs), + wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"failed">>) + end)}. handle_db_deletion_in_build_indices({Top, {Db1, _, _}}) -> - {timeout, ?TIMEOUT, ?_test(begin - JobId = delete_source_in_state(Top, Db1, build_indices), - wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"failed">>) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + JobId = delete_source_in_state(Top, Db1, build_indices), + wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"failed">>) + end)}. handle_db_deletion_in_update_shard_map({Top, {Db1, _, _}}) -> - {timeout, ?TIMEOUT, ?_test(begin - JobId = delete_source_in_state(Top, Db1, update_shardmap), - wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"failed">>) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + JobId = delete_source_in_state(Top, Db1, update_shardmap), + wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"failed">>) + end)}. handle_db_deletion_in_wait_source_close({Top, {Db1, _, _}}) -> - {timeout, ?TIMEOUT, ?_test(begin - JobId = delete_source_in_state(Top, Db1, wait_source_close), - wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"failed">>) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + JobId = delete_source_in_state(Top, Db1, wait_source_close), + wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"failed">>) + end)}. recover_in_topoff1({Top, {Db1, _, _}}) -> - {timeout, ?TIMEOUT, ?_test(begin - JobId = recover_in_state(Top, Db1, topoff1), - wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"completed">>) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + JobId = recover_in_state(Top, Db1, topoff1), + wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"completed">>) + end)}. recover_in_initial_copy({Top, {Db1, _, _}}) -> - {timeout, ?TIMEOUT, ?_test(begin - JobId = recover_in_state(Top, Db1, initial_copy), - wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"completed">>) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + JobId = recover_in_state(Top, Db1, initial_copy), + wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"completed">>) + end)}. recover_in_copy_local_docs({Top, {Db1, _, _}}) -> - {timeout, ?TIMEOUT, ?_test(begin - JobId = recover_in_state(Top, Db1, copy_local_docs), - wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"completed">>) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + JobId = recover_in_state(Top, Db1, copy_local_docs), + wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"completed">>) + end)}. recover_in_build_indices({Top, {Db1, _, _}}) -> - {timeout, ?TIMEOUT, ?_test(begin - JobId = recover_in_state(Top, Db1, build_indices), - wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"completed">>) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + JobId = recover_in_state(Top, Db1, build_indices), + wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"completed">>) + end)}. recover_in_update_shard_map({Top, {Db1, _, _}}) -> - {timeout, ?TIMEOUT, ?_test(begin - JobId = recover_in_state(Top, Db1, update_shardmap), - wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"completed">>) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + JobId = recover_in_state(Top, Db1, update_shardmap), + wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"completed">>) + end)}. recover_in_wait_source_close({Top, {Db1, _, _}}) -> - {timeout, ?TIMEOUT, ?_test(begin - JobId = recover_in_state(Top, Db1, wait_source_close), - wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"completed">>) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + JobId = recover_in_state(Top, Db1, wait_source_close), + wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"completed">>) + end)}. recover_in_topoff3({Top, {Db1, _, _}}) -> - {timeout, ?TIMEOUT, ?_test(begin - JobId = recover_in_state(Top, Db1, topoff3), - wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"completed">>) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + JobId = recover_in_state(Top, Db1, topoff3), + wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"completed">>) + end)}. recover_in_source_delete({Top, {Db1, _, _}}) -> - {timeout, ?TIMEOUT, ?_test(begin - JobId = recover_in_state(Top, Db1, source_delete), - wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"completed">>) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + JobId = recover_in_state(Top, Db1, source_delete), + wait_state(Top ++ ?JOBS ++ ?b2l(JobId) ++ "/state", <<"completed">>) + end)}. check_max_jobs({Top, {Db1, Db2, _}}) -> - {timeout, ?TIMEOUT, ?_test(begin - Jobs = Top ++ ?JOBS, - - config:set("reshard", "max_jobs", "0", _Persist=false), - {C1, R1} = req(post, Jobs, #{type => split, db => Db1}), - ?assertMatch({500, [#{<<"error">> := <<"max_jobs_exceeded">>}]}, {C1, R1}), + {timeout, ?TIMEOUT, + ?_test(begin + Jobs = Top ++ ?JOBS, - config:set("reshard", "max_jobs", "1", _Persist=false), - {201, R2} = req(post, Jobs, #{type => split, db => Db1}), - wait_to_complete(Top, R2), + config:set("reshard", "max_jobs", "0", _Persist = false), + {C1, R1} = req(post, Jobs, #{type => split, db => Db1}), + ?assertMatch({500, [#{<<"error">> := <<"max_jobs_exceeded">>}]}, {C1, R1}), - % Stop clustering so jobs are not started anymore and ensure max jobs - % is enforced even if jobs are stopped - ?assertMatch({200, _}, req(put, Top ++ ?STATE, #{state => stopped})), + config:set("reshard", "max_jobs", "1", _Persist = false), + {201, R2} = req(post, Jobs, #{type => split, db => Db1}), + wait_to_complete(Top, R2), - {C3, R3} = req(post, Jobs, #{type => split, db => Db2}), - ?assertMatch({500, [#{<<"error">> := <<"max_jobs_exceeded">>}]}, - {C3, R3}), + % Stop clustering so jobs are not started anymore and ensure max jobs + % is enforced even if jobs are stopped + ?assertMatch({200, _}, req(put, Top ++ ?STATE, #{state => stopped})), - % Allow the job to be created by raising max_jobs - config:set("reshard", "max_jobs", "2", _Persist=false), + {C3, R3} = req(post, Jobs, #{type => split, db => Db2}), + ?assertMatch( + {500, [#{<<"error">> := <<"max_jobs_exceeded">>}]}, + {C3, R3} + ), - {C4, R4} = req(post, Jobs, #{type => split, db => Db2}), - ?assertEqual(201, C4), + % Allow the job to be created by raising max_jobs + config:set("reshard", "max_jobs", "2", _Persist = false), - % Lower max_jobs after job is created but it's not running - config:set("reshard", "max_jobs", "1", _Persist=false), + {C4, R4} = req(post, Jobs, #{type => split, db => Db2}), + ?assertEqual(201, C4), - % Start resharding again - ?assertMatch({200, _}, req(put, Top ++ ?STATE, #{state => running})), + % Lower max_jobs after job is created but it's not running + config:set("reshard", "max_jobs", "1", _Persist = false), - % Jobs that have been created already are not removed if max jobs is lowered - % so make sure the job completes - wait_to_complete(Top, R4) - end)}. + % Start resharding again + ?assertMatch({200, _}, req(put, Top ++ ?STATE, #{state => running})), + % Jobs that have been created already are not removed if max jobs is lowered + % so make sure the job completes + wait_to_complete(Top, R4) + end)}. check_node_and_range_required_params({Top, {Db1, _, _}}) -> - {timeout, ?TIMEOUT, ?_test(begin - Jobs = Top ++ ?JOBS, - - Node = atom_to_binary(node(), utf8), - Range = <<"00000000-ffffffff">>, - - config:set("reshard", "require_node_param", "true", _Persist=false), - {C1, R1} = req(post, Jobs, #{type => split, db => Db1}), - NodeRequiredErr = <<"`node` prameter is required">>, - ?assertEqual({400, #{<<"error">> => <<"bad_request">>, - <<"reason">> => NodeRequiredErr}}, {C1, R1}), - - config:set("reshard", "require_range_param", "true", _Persist=false), - {C2, R2} = req(post, Jobs, #{type => split, db => Db1, node => Node}), - RangeRequiredErr = <<"`range` prameter is required">>, - ?assertEqual({400, #{<<"error">> => <<"bad_request">>, - <<"reason">> => RangeRequiredErr}}, {C2, R2}), - - Body = #{type => split, db => Db1, range => Range, node => Node}, - {C3, R3} = req(post, Jobs, Body), - ?assertMatch({201, [#{?OK := true}]}, {C3, R3}), - wait_to_complete_then_cleanup(Top, R3) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + Jobs = Top ++ ?JOBS, + + Node = atom_to_binary(node(), utf8), + Range = <<"00000000-ffffffff">>, + + config:set("reshard", "require_node_param", "true", _Persist = false), + {C1, R1} = req(post, Jobs, #{type => split, db => Db1}), + NodeRequiredErr = <<"`node` prameter is required">>, + ?assertEqual( + {400, #{ + <<"error">> => <<"bad_request">>, + <<"reason">> => NodeRequiredErr + }}, + {C1, R1} + ), + + config:set("reshard", "require_range_param", "true", _Persist = false), + {C2, R2} = req(post, Jobs, #{type => split, db => Db1, node => Node}), + RangeRequiredErr = <<"`range` prameter is required">>, + ?assertEqual( + {400, #{ + <<"error">> => <<"bad_request">>, + <<"reason">> => RangeRequiredErr + }}, + {C2, R2} + ), + + Body = #{type => split, db => Db1, range => Range, node => Node}, + {C3, R3} = req(post, Jobs, Body), + ?assertMatch({201, [#{?OK := true}]}, {C3, R3}), + wait_to_complete_then_cleanup(Top, R3) + end)}. cleanup_completed_jobs({Top, {Db1, _, _}}) -> - {timeout, ?TIMEOUT, ?_test(begin - Body = #{type => split, db => Db1}, - {201, [#{?ID := Id}]} = req(post, Top ++ ?JOBS, Body), - JobUrl = Top ++ ?JOBS ++ ?b2l(Id), - wait_state(JobUrl ++ "/state", <<"completed">>), - delete_db(Top, Db1), - wait_for_http_code(JobUrl, 404) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + Body = #{type => split, db => Db1}, + {201, [#{?ID := Id}]} = req(post, Top ++ ?JOBS, Body), + JobUrl = Top ++ ?JOBS ++ ?b2l(Id), + wait_state(JobUrl ++ "/state", <<"completed">>), + delete_db(Top, Db1), + wait_for_http_code(JobUrl, 404) + end)}. % Test help functions wait_to_complete_then_cleanup(Top, Jobs) -> JobsUrl = Top ++ ?JOBS, - lists:foreach(fun(#{?ID := Id}) -> - wait_state(JobsUrl ++ ?b2l(Id) ++ "/state", <<"completed">>), - {200, _} = req(delete, JobsUrl ++ ?b2l(Id)) - end, Jobs). - + lists:foreach( + fun(#{?ID := Id}) -> + wait_state(JobsUrl ++ ?b2l(Id) ++ "/state", <<"completed">>), + {200, _} = req(delete, JobsUrl ++ ?b2l(Id)) + end, + Jobs + ). wait_to_complete(Top, Jobs) -> JobsUrl = Top ++ ?JOBS, - lists:foreach(fun(#{?ID := Id}) -> - wait_state(JobsUrl ++ ?b2l(Id) ++ "/state", <<"completed">>) - end, Jobs). - + lists:foreach( + fun(#{?ID := Id}) -> + wait_state(JobsUrl ++ ?b2l(Id) ++ "/state", <<"completed">>) + end, + Jobs + ). intercept_state(State) -> TestPid = self(), @@ -752,46 +884,57 @@ intercept_state(State) -> end end). - cancel_intercept() -> meck:expect(mem3_reshard_job, checkpoint_done, fun(Job) -> meck:passthrough([Job]) end). - wait_state(Url, State) -> - test_util:wait(fun() -> + test_util:wait( + fun() -> case req(get, Url) of - {200, #{<<"state">> := State}} -> ok; - {200, #{}} -> timer:sleep(100), wait + {200, #{<<"state">> := State}} -> + ok; + {200, #{}} -> + timer:sleep(100), + wait end - end, 30000). - + end, + 30000 + ). wait_for_http_code(Url, Code) when is_integer(Code) -> - test_util:wait(fun() -> + test_util:wait( + fun() -> case req(get, Url) of - {Code, _} -> ok; - {_, _} -> timer:sleep(100), wait + {Code, _} -> + ok; + {_, _} -> + timer:sleep(100), + wait end - end, 30000). - + end, + 30000 + ). delete_source_in_state(Top, Db, State) when is_atom(State), is_binary(Db) -> intercept_state(State), Body = #{type => split, db => Db}, {201, [#{?ID := Id}]} = req(post, Top ++ ?JOBS, Body), - receive {JobPid, State} -> ok end, + receive + {JobPid, State} -> ok + end, sync_delete_db(Top, Db), JobPid ! continue, Id. - recover_in_state(Top, Db, State) when is_atom(State) -> intercept_state(State), Body = #{type => split, db => Db}, {201, [#{?ID := Id}]} = req(post, Top ++ ?JOBS, Body), - receive {JobPid, State} -> ok end, + receive + {JobPid, State} -> ok + end, % Job is now stuck in running we prevented it from executing % the given state JobPid ! cancel, @@ -801,13 +944,11 @@ recover_in_state(Top, Db, State) when is_atom(State) -> ?assertMatch({200, _}, req(put, Top ++ ?STATE, #{state => running})), Id. - create_db(Top, Db, QArgs) when is_binary(Db) -> Url = Top ++ binary_to_list(Db) ++ QArgs, {ok, Status, _, _} = test_request:put(Url, [?JSON, ?AUTH], "{}"), ?assert(Status =:= 201 orelse Status =:= 202). - delete_db(Top, Db) when is_binary(Db) -> Url = Top ++ binary_to_list(Db), case test_request:get(Url, [?AUTH]) of @@ -818,7 +959,6 @@ delete_db(Top, Db) when is_binary(Db) -> ok end. - sync_delete_db(Top, Db) when is_binary(Db) -> delete_db(Top, Db), try @@ -831,16 +971,13 @@ sync_delete_db(Top, Db) when is_binary(Db) -> ok end. - req(Method, Url) -> Headers = [?AUTH], {ok, Code, _, Res} = test_request:request(Method, Url, Headers), {Code, jiffy:decode(Res, [return_maps])}. - req(Method, Url, #{} = Body) -> req(Method, Url, jiffy:encode(Body)); - req(Method, Url, Body) -> Headers = [?JSON, ?AUTH], {ok, Code, _, Res} = test_request:request(Method, Url, Headers, Body), diff --git a/src/mem3/test/eunit/mem3_reshard_changes_feed_test.erl b/src/mem3/test/eunit/mem3_reshard_changes_feed_test.erl index 4b9e2a34a..b9cafd75c 100644 --- a/src/mem3/test/eunit/mem3_reshard_changes_feed_test.erl +++ b/src/mem3/test/eunit/mem3_reshard_changes_feed_test.erl @@ -12,51 +12,47 @@ -module(mem3_reshard_changes_feed_test). - -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). -include_lib("mem3/src/mem3_reshard.hrl"). --define(TIMEOUT, 60). % seconds - --define(assertChanges(Expected, Received), - begin - ((fun() -> - ExpectedIDs = lists:sort([I || #{id := I} <- Expected]), - ReceivedIDs = lists:sort([I || #{id := I} <- Received]), - ?assertEqual(ExpectedIDs, ReceivedIDs) - end)()) - end). +% seconds +-define(TIMEOUT, 60). +-define(assertChanges(Expected, Received), begin + ((fun() -> + ExpectedIDs = lists:sort([I || #{id := I} <- Expected]), + ReceivedIDs = lists:sort([I || #{id := I} <- Received]), + ?assertEqual(ExpectedIDs, ReceivedIDs) + end)()) +end). setup() -> Db1 = ?tempdb(), create_db(Db1, [{q, 1}, {n, 1}]), #{db1 => Db1}. - teardown(#{} = Dbs) -> mem3_reshard:reset_state(), maps:map(fun(_, Db) -> delete_db(Db) end, Dbs). - start_couch() -> test_util:start_couch(?CONFIG_CHAIN, [mem3, fabric]). - stop_couch(Ctx) -> test_util:stop_couch(Ctx). - mem3_reshard_changes_feed_test_() -> { "mem3 shard split changes feed tests", { setup, - fun start_couch/0, fun stop_couch/1, + fun start_couch/0, + fun stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun normal_feed_should_work_after_split/1, fun continuous_feed_should_work_during_split/1 @@ -65,177 +61,193 @@ mem3_reshard_changes_feed_test_() -> } }. - normal_feed_should_work_after_split(#{db1 := Db}) -> - {timeout, ?TIMEOUT, ?_test(begin - DocSpec = #{ - docs => [1, 10], - delete => [5, 6] - }, - add_test_docs(Db, DocSpec), - - % gather pre-shard changes - BaseArgs = #changes_args{feed = "normal", dir = fwd, since = 0}, - {ok, OldChanges, OldEndSeq} = get_changes_feed(Db, BaseArgs), - - % Split the shard - split_and_wait(Db), - - % verify changes list consistent for all the old seqs - lists:foldl(fun(#{seq := Seq} = C, ExpectedChanges) -> - Args = BaseArgs#changes_args{since = Seq}, - {ok, Changes, _EndSeq} = get_changes_feed(Db, Args), - ?assertChanges(ExpectedChanges, Changes), - [C | ExpectedChanges] - end, [], OldChanges), - - % confirm that old LastSeq respected - Args1 = BaseArgs#changes_args{since = OldEndSeq}, - {ok, Changes1, EndSeq1} = get_changes_feed(Db, Args1), - ?assertChanges([], Changes1), - - % confirm that new LastSeq also respected - Args2 = BaseArgs#changes_args{since = EndSeq1}, - {ok, Changes2, EndSeq2} = get_changes_feed(Db, Args2), - ?assertChanges([], Changes2), - ?assertEqual(EndSeq2, EndSeq1), - - % confirm we didn't lost any changes and have consistent last seq - {ok, Changes3, EndSeq3} = get_changes_feed(Db, BaseArgs), - ?assertChanges(OldChanges, Changes3), - - % add some docs - add_test_docs(Db, #{docs => [11, 15]}), - Args4 = BaseArgs#changes_args{since = EndSeq3}, - {ok, Changes4, EndSeq4} = get_changes_feed(Db, Args4), - AddedChanges = [#{id => ID} || #doc{id = ID} <- docs([11, 15])], - ?assertChanges(AddedChanges, Changes4), - - % confirm include_docs and deleted works - Args5 = BaseArgs#changes_args{include_docs = true}, - {ok, Changes5, EndSeq5} = get_changes_feed(Db, Args5), - ?assertEqual(EndSeq4, EndSeq5), - [SampleChange] = [C || #{id := ID} = C <- Changes5, ID == <<"00005">>], - ?assertMatch(#{deleted := true}, SampleChange), - ?assertMatch(#{doc := {Body}} when is_list(Body), SampleChange), - - % update and delete some pre and post split docs - AllDocs = [couch_doc:from_json_obj(Doc) || #{doc := Doc} <- Changes5], - UpdateDocs = lists:filtermap(fun - (#doc{id = <<"00002">>}) -> true; - (#doc{id = <<"00012">>}) -> true; - (#doc{id = <<"00004">>} = Doc) -> {true, Doc#doc{deleted = true}}; - (#doc{id = <<"00014">>} = Doc) -> {true, Doc#doc{deleted = true}}; - (_) -> false - end, AllDocs), - update_docs(Db, UpdateDocs), - - Args6 = BaseArgs#changes_args{since = EndSeq5}, - {ok, Changes6, EndSeq6} = get_changes_feed(Db, Args6), - UpdatedChanges = [#{id => ID} || #doc{id = ID} <- UpdateDocs], - ?assertChanges(UpdatedChanges, Changes6), - [#{seq := Seq6} | _] = Changes6, - ?assertEqual(EndSeq6, Seq6), - - Args7 = Args6#changes_args{dir = rev, limit = 4}, - {ok, Changes7, EndSeq7} = get_changes_feed(Db, Args7), - ?assertEqual(4, length(Changes7)), - [#{seq := Seq7} | _] = Changes7, - ?assertEqual(EndSeq7, Seq7) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + DocSpec = #{ + docs => [1, 10], + delete => [5, 6] + }, + add_test_docs(Db, DocSpec), + + % gather pre-shard changes + BaseArgs = #changes_args{feed = "normal", dir = fwd, since = 0}, + {ok, OldChanges, OldEndSeq} = get_changes_feed(Db, BaseArgs), + + % Split the shard + split_and_wait(Db), + + % verify changes list consistent for all the old seqs + lists:foldl( + fun(#{seq := Seq} = C, ExpectedChanges) -> + Args = BaseArgs#changes_args{since = Seq}, + {ok, Changes, _EndSeq} = get_changes_feed(Db, Args), + ?assertChanges(ExpectedChanges, Changes), + [C | ExpectedChanges] + end, + [], + OldChanges + ), + + % confirm that old LastSeq respected + Args1 = BaseArgs#changes_args{since = OldEndSeq}, + {ok, Changes1, EndSeq1} = get_changes_feed(Db, Args1), + ?assertChanges([], Changes1), + + % confirm that new LastSeq also respected + Args2 = BaseArgs#changes_args{since = EndSeq1}, + {ok, Changes2, EndSeq2} = get_changes_feed(Db, Args2), + ?assertChanges([], Changes2), + ?assertEqual(EndSeq2, EndSeq1), + + % confirm we didn't lost any changes and have consistent last seq + {ok, Changes3, EndSeq3} = get_changes_feed(Db, BaseArgs), + ?assertChanges(OldChanges, Changes3), + + % add some docs + add_test_docs(Db, #{docs => [11, 15]}), + Args4 = BaseArgs#changes_args{since = EndSeq3}, + {ok, Changes4, EndSeq4} = get_changes_feed(Db, Args4), + AddedChanges = [#{id => ID} || #doc{id = ID} <- docs([11, 15])], + ?assertChanges(AddedChanges, Changes4), + + % confirm include_docs and deleted works + Args5 = BaseArgs#changes_args{include_docs = true}, + {ok, Changes5, EndSeq5} = get_changes_feed(Db, Args5), + ?assertEqual(EndSeq4, EndSeq5), + [SampleChange] = [C || #{id := ID} = C <- Changes5, ID == <<"00005">>], + ?assertMatch(#{deleted := true}, SampleChange), + ?assertMatch(#{doc := {Body}} when is_list(Body), SampleChange), + + % update and delete some pre and post split docs + AllDocs = [couch_doc:from_json_obj(Doc) || #{doc := Doc} <- Changes5], + UpdateDocs = lists:filtermap( + fun + (#doc{id = <<"00002">>}) -> true; + (#doc{id = <<"00012">>}) -> true; + (#doc{id = <<"00004">>} = Doc) -> {true, Doc#doc{deleted = true}}; + (#doc{id = <<"00014">>} = Doc) -> {true, Doc#doc{deleted = true}}; + (_) -> false + end, + AllDocs + ), + update_docs(Db, UpdateDocs), + + Args6 = BaseArgs#changes_args{since = EndSeq5}, + {ok, Changes6, EndSeq6} = get_changes_feed(Db, Args6), + UpdatedChanges = [#{id => ID} || #doc{id = ID} <- UpdateDocs], + ?assertChanges(UpdatedChanges, Changes6), + [#{seq := Seq6} | _] = Changes6, + ?assertEqual(EndSeq6, Seq6), + + Args7 = Args6#changes_args{dir = rev, limit = 4}, + {ok, Changes7, EndSeq7} = get_changes_feed(Db, Args7), + ?assertEqual(4, length(Changes7)), + [#{seq := Seq7} | _] = Changes7, + ?assertEqual(EndSeq7, Seq7) + end)}. continuous_feed_should_work_during_split(#{db1 := Db}) -> - {timeout, ?TIMEOUT, ?_test(begin - {UpdaterPid, UpdaterRef} = spawn_monitor(fun() -> - Updater = fun U({State, I}) -> - receive - {get_state, {Pid, Ref}} -> - Pid ! {state, Ref, {State, I}}, - U({State, I}); - add -> - DocSpec = #{docs => [I, I]}, - add_test_docs(Db, DocSpec), - U({State, I + 1}); - split -> - spawn_monitor(fun() -> split_and_wait(Db) end), - U({"in_process", I}); - stop -> - receive {'DOWN', _, process, _, _} -> ok end, - ok - end + {timeout, ?TIMEOUT, + ?_test(begin + {UpdaterPid, UpdaterRef} = spawn_monitor(fun() -> + Updater = fun U({State, I}) -> + receive + {get_state, {Pid, Ref}} -> + Pid ! {state, Ref, {State, I}}, + U({State, I}); + add -> + DocSpec = #{docs => [I, I]}, + add_test_docs(Db, DocSpec), + U({State, I + 1}); + split -> + spawn_monitor(fun() -> split_and_wait(Db) end), + U({"in_process", I}); + stop -> + receive + {'DOWN', _, process, _, _} -> ok + end, + ok + end + end, + Updater({"before", 1}) + end), + + Callback = fun + (start, Acc) -> + {ok, Acc}; + (waiting_for_updates, Acc) -> + Ref = make_ref(), + UpdaterPid ! {get_state, {self(), Ref}}, + receive + {state, Ref, {State, _}} -> ok + end, + case {State, length(Acc)} of + {"before", N} when N < 5 -> + UpdaterPid ! add, + {ok, Acc}; + {"before", _} -> + UpdaterPid ! split, + {ok, Acc}; + {"in_process", N} when N < 10 -> + UpdaterPid ! add, + {ok, Acc}; + {"in_process", _} -> + {ok, Acc} + end; + (timeout, Acc) -> + {ok, Acc}; + ({change, {Change}}, Acc) -> + CM = maps:from_list(Change), + {ok, [CM | Acc]}; + ({stop, EndSeq, _Pending}, Acc) -> + % Notice updater is still running + {stop, EndSeq, Acc} end, - Updater({"before", 1}) - end), - - Callback = fun - (start, Acc) -> - {ok, Acc}; - (waiting_for_updates, Acc) -> - Ref = make_ref(), - UpdaterPid ! {get_state, {self(), Ref}}, - receive {state, Ref, {State, _}} -> ok end, - case {State, length(Acc)} of - {"before", N} when N < 5 -> - UpdaterPid ! add, - {ok, Acc}; - {"before", _} -> - UpdaterPid ! split, - {ok, Acc}; - {"in_process", N} when N < 10 -> - UpdaterPid ! add, - {ok, Acc}; - {"in_process", _} -> - {ok, Acc} - end; - (timeout, Acc) -> - {ok, Acc}; - ({change, {Change}}, Acc) -> - CM = maps:from_list(Change), - {ok, [CM | Acc]}; - ({stop, EndSeq, _Pending}, Acc) -> - % Notice updater is still running - {stop, EndSeq, Acc} - end, - BaseArgs = #changes_args{ - feed = "continuous", - heartbeat = 100, - timeout = 1000 - }, - StopResult = get_changes_feed(Db, BaseArgs, Callback), - - % Changes feed stopped when source shard was deleted - ?assertMatch({stop, _, _}, StopResult), - {stop, StopEndSeq, StopChanges} = StopResult, - - % Add 5 extra docs to the db right after changes feed was stopped - [UpdaterPid ! add || _ <- lists:seq(1, 5)], - - % The the number of documents that updater had added - Ref = make_ref(), - UpdaterPid ! {get_state, {self(), Ref}}, - DocCount = receive {state, Ref, {_, I}} -> I - 1 end, - - UpdaterPid ! stop, - receive - {'DOWN', UpdaterRef, process, UpdaterPid, normal} -> - ok; - {'DOWN', UpdaterRef, process, UpdaterPid, Error} -> - erlang:error({test_context_failed, [ - {module, ?MODULE}, - {line, ?LINE}, - {value, Error}, - {reason, "Updater died"}]}) - end, + BaseArgs = #changes_args{ + feed = "continuous", + heartbeat = 100, + timeout = 1000 + }, + StopResult = get_changes_feed(Db, BaseArgs, Callback), + + % Changes feed stopped when source shard was deleted + ?assertMatch({stop, _, _}, StopResult), + {stop, StopEndSeq, StopChanges} = StopResult, - AfterArgs = #changes_args{feed = "normal", since = StopEndSeq}, - {ok, AfterChanges, _} = get_changes_feed(Db, AfterArgs), - DocIDs = [Id || #{id := Id} <- StopChanges ++ AfterChanges], - ExpectedDocIDs = [doc_id(<<>>, N) || N <- lists:seq(1, DocCount)], - ?assertEqual(ExpectedDocIDs, lists:usort(DocIDs)) - end)}. + % Add 5 extra docs to the db right after changes feed was stopped + [UpdaterPid ! add || _ <- lists:seq(1, 5)], + + % The the number of documents that updater had added + Ref = make_ref(), + UpdaterPid ! {get_state, {self(), Ref}}, + DocCount = + receive + {state, Ref, {_, I}} -> I - 1 + end, + + UpdaterPid ! stop, + receive + {'DOWN', UpdaterRef, process, UpdaterPid, normal} -> + ok; + {'DOWN', UpdaterRef, process, UpdaterPid, Error} -> + erlang:error( + {test_context_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {value, Error}, + {reason, "Updater died"} + ]} + ) + end, + AfterArgs = #changes_args{feed = "normal", since = StopEndSeq}, + {ok, AfterChanges, _} = get_changes_feed(Db, AfterArgs), + DocIDs = [Id || #{id := Id} <- StopChanges ++ AfterChanges], + ExpectedDocIDs = [doc_id(<<>>, N) || N <- lists:seq(1, DocCount)], + ?assertEqual(ExpectedDocIDs, lists:usort(DocIDs)) + end)}. split_and_wait(Db) -> [#shard{name = Shard}] = lists:sort(mem3:local_shards(Db)), @@ -244,30 +256,34 @@ split_and_wait(Db) -> ResultShards = lists:sort(mem3:local_shards(Db)), ?assertEqual(2, length(ResultShards)). - wait_state(JobId, State) -> - test_util:wait(fun() -> - case mem3_reshard:job(JobId) of - {ok, {Props}} -> - case couch_util:get_value(job_state, Props) of - State -> ok; - _ -> timer:sleep(100), wait - end; - {error, not_found} -> timer:sleep(100), wait - end - end, 30000). - + test_util:wait( + fun() -> + case mem3_reshard:job(JobId) of + {ok, {Props}} -> + case couch_util:get_value(job_state, Props) of + State -> + ok; + _ -> + timer:sleep(100), + wait + end; + {error, not_found} -> + timer:sleep(100), + wait + end + end, + 30000 + ). get_changes_feed(Db, Args) -> get_changes_feed(Db, Args, fun changes_callback/2). - get_changes_feed(Db, Args, Callback) -> with_proc(fun() -> fabric:changes(Db, Callback, [], Args) end). - changes_callback(start, Acc) -> {ok, Acc}; changes_callback({change, {Change}}, Acc) -> @@ -276,28 +292,22 @@ changes_callback({change, {Change}}, Acc) -> changes_callback({stop, EndSeq, _Pending}, Acc) -> {ok, Acc, EndSeq}. - %% common helpers from here - create_db(DbName, Opts) -> GL = erlang:group_leader(), with_proc(fun() -> fabric:create_db(DbName, Opts) end, GL). - delete_db(DbName) -> GL = erlang:group_leader(), with_proc(fun() -> fabric:delete_db(DbName, [?ADMIN_CTX]) end, GL). - with_proc(Fun) -> with_proc(Fun, undefined, 30000). - with_proc(Fun, GroupLeader) -> with_proc(Fun, GroupLeader, 30000). - with_proc(Fun, GroupLeader, Timeout) -> {Pid, Ref} = spawn_monitor(fun() -> case GroupLeader of @@ -317,20 +327,21 @@ with_proc(Fun, GroupLeader, Timeout) -> error({with_proc_timeout, Fun, Timeout}) end. - add_test_docs(DbName, #{} = DocSpec) -> Docs = docs(maps:get(docs, DocSpec, [])), Res = update_docs(DbName, Docs), - Docs1 = lists:map(fun({Doc, {ok, {RevPos, Rev}}}) -> - Doc#doc{revs = {RevPos, [Rev]}} - end, lists:zip(Docs, Res)), + Docs1 = lists:map( + fun({Doc, {ok, {RevPos, Rev}}}) -> + Doc#doc{revs = {RevPos, [Rev]}} + end, + lists:zip(Docs, Res) + ), case delete_docs(maps:get(delete, DocSpec, []), Docs1) of [] -> ok; [_ | _] = Deleted -> update_docs(DbName, Deleted) end, ok. - update_docs(DbName, Docs) -> with_proc(fun() -> case fabric:update_docs(DbName, Docs, [?ADMIN_CTX]) of @@ -339,30 +350,29 @@ update_docs(DbName, Docs) -> end end). - delete_docs([S, E], Docs) when E >= S -> ToDelete = [doc_id(<<"">>, I) || I <- lists:seq(S, E)], - lists:filtermap(fun(#doc{id = Id} = Doc) -> - case lists:member(Id, ToDelete) of - true -> {true, Doc#doc{deleted = true}}; - false -> false - end - end, Docs); + lists:filtermap( + fun(#doc{id = Id} = Doc) -> + case lists:member(Id, ToDelete) of + true -> {true, Doc#doc{deleted = true}}; + false -> false + end + end, + Docs + ); delete_docs(_, _) -> []. - docs([S, E]) when E >= S -> [doc(<<"">>, I) || I <- lists:seq(S, E)]; docs(_) -> []. - doc(Pref, Id) -> Body = [{<<"a">>, <<"b">>}], doc(Pref, Id, Body, 42). - doc(Pref, Id, BodyProps, AttSize) -> #doc{ id = doc_id(Pref, Id), @@ -370,20 +380,19 @@ doc(Pref, Id, BodyProps, AttSize) -> atts = atts(AttSize) }. - doc_id(Pref, Id) -> IdBin = iolist_to_binary(io_lib:format("~5..0B", [Id])), <<Pref/binary, IdBin/binary>>. - atts(0) -> []; - atts(Size) when is_integer(Size), Size >= 1 -> - Data = << <<"x">> || _ <- lists:seq(1, Size) >>, - [couch_att:new([ - {name, <<"att">>}, - {type, <<"app/binary">>}, - {att_len, Size}, - {data, Data} - ])]. + Data = <<<<"x">> || _ <- lists:seq(1, Size)>>, + [ + couch_att:new([ + {name, <<"att">>}, + {type, <<"app/binary">>}, + {att_len, Size}, + {data, Data} + ]) + ]. diff --git a/src/mem3/test/eunit/mem3_reshard_test.erl b/src/mem3/test/eunit/mem3_reshard_test.erl index 65f2b4bb0..1929242bb 100644 --- a/src/mem3/test/eunit/mem3_reshard_test.erl +++ b/src/mem3/test/eunit/mem3_reshard_test.erl @@ -12,57 +12,57 @@ -module(mem3_reshard_test). - -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). -include_lib("mem3/src/mem3_reshard.hrl"). --include_lib("couch_mrview/include/couch_mrview.hrl"). % for all_docs function +% for all_docs function +-include_lib("couch_mrview/include/couch_mrview.hrl"). -define(ID, <<"_id">>). -define(TIMEOUT, 60). setup() -> HaveDreyfus = code:lib_dir(dreyfus) /= {error, bad_name}, - case HaveDreyfus of false -> ok; true -> - mock_dreyfus_indices() + case HaveDreyfus of + false -> ok; + true -> mock_dreyfus_indices() end, HaveHastings = code:lib_dir(hastings) /= {error, bad_name}, - case HaveHastings of false -> ok; true -> - mock_hastings_indices() + case HaveHastings of + false -> ok; + true -> mock_hastings_indices() end, {Db1, Db2} = {?tempdb(), ?tempdb()}, create_db(Db1, [{q, 1}, {n, 1}]), PartProps = [{partitioned, true}, {hash, [couch_partition, hash, []]}], create_db(Db2, [{q, 1}, {n, 1}, {props, PartProps}]), - config:set("reshard", "retry_interval_sec", "0", _Persist=false), + config:set("reshard", "retry_interval_sec", "0", _Persist = false), #{db1 => Db1, db2 => Db2}. - teardown(#{} = Dbs) -> mem3_reshard:reset_state(), maps:map(fun(_, Db) -> delete_db(Db) end, Dbs), - config:delete("reshard", "retry_interval_sec", _Persist=false), + config:delete("reshard", "retry_interval_sec", _Persist = false), meck:unload(). - start_couch() -> test_util:start_couch(?CONFIG_CHAIN, [mem3, fabric]). - stop_couch(Ctx) -> test_util:stop_couch(Ctx). - mem3_reshard_db_test_() -> { "mem3 shard split db tests", { setup, - fun start_couch/0, fun stop_couch/1, + fun start_couch/0, + fun stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun split_one_shard/1, fun split_shard_with_lots_of_purges/1, @@ -80,187 +80,198 @@ mem3_reshard_db_test_() -> } }. - % This is a basic test to check that shard splitting preserves documents, and % db meta props like revs limits and security. split_one_shard(#{db1 := Db}) -> - {timeout, ?TIMEOUT, ?_test(begin - DocSpec = #{docs => 10, delete => [5, 9], mrview => 1, local => 1}, - add_test_docs(Db, DocSpec), - - % Save documents before the split - Docs0 = get_all_docs(Db), - Local0 = get_local_docs(Db), - - % Set some custom metadata properties - set_revs_limit(Db, 942), - set_purge_infos_limit(Db, 943), - SecObj = {[{<<"foo">>, <<"bar">>}]}, - set_security(Db, SecObj), - - % DbInfo is saved after setting metadata bits - % as those could bump the update sequence - DbInfo0 = get_db_info(Db), - - % Split the one shard - [#shard{name=Shard}] = lists:sort(mem3:local_shards(Db)), - {ok, JobId} = mem3_reshard:start_split_job(Shard), - wait_state(JobId, completed), - - % Perform some basic checks that the shard was split - Shards1 = lists:sort(mem3:local_shards(Db)), - ?assertEqual(2, length(Shards1)), - [#shard{range = R1}, #shard{range = R2}] = Shards1, - ?assertEqual([16#00000000, 16#7fffffff], R1), - ?assertEqual([16#80000000, 16#ffffffff], R2), - - % Check metadata bits after the split - ?assertEqual(942, get_revs_limit(Db)), - ?assertEqual(943, get_purge_infos_limit(Db)), - ?assertEqual(SecObj, get_security(Db)), - - DbInfo1 = get_db_info(Db), - Docs1 = get_all_docs(Db), - Local1 = get_local_docs(Db), - - % When comparing db infos, ignore update sequences they won't be the - % same since they are more shards involved after the split - ?assertEqual(without_seqs(DbInfo0), without_seqs(DbInfo1)), - - % Update seq prefix number is a sum of all shard update sequences - #{<<"update_seq">> := UpdateSeq0} = update_seq_to_num(DbInfo0), - #{<<"update_seq">> := UpdateSeq1} = update_seq_to_num(DbInfo1), - ?assertEqual(UpdateSeq0 * 2, UpdateSeq1), - - % Finally compare that the documents are still there after the split - ?assertEqual(Docs0, Docs1), - - % Don't forget about the local but don't include internal checkpoints - % as some of those are munged and transformed during the split - ?assertEqual(without_meta_locals(Local0), without_meta_locals(Local1)) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + DocSpec = #{docs => 10, delete => [5, 9], mrview => 1, local => 1}, + add_test_docs(Db, DocSpec), + + % Save documents before the split + Docs0 = get_all_docs(Db), + Local0 = get_local_docs(Db), + + % Set some custom metadata properties + set_revs_limit(Db, 942), + set_purge_infos_limit(Db, 943), + SecObj = {[{<<"foo">>, <<"bar">>}]}, + set_security(Db, SecObj), + + % DbInfo is saved after setting metadata bits + % as those could bump the update sequence + DbInfo0 = get_db_info(Db), + + % Split the one shard + [#shard{name = Shard}] = lists:sort(mem3:local_shards(Db)), + {ok, JobId} = mem3_reshard:start_split_job(Shard), + wait_state(JobId, completed), + + % Perform some basic checks that the shard was split + Shards1 = lists:sort(mem3:local_shards(Db)), + ?assertEqual(2, length(Shards1)), + [#shard{range = R1}, #shard{range = R2}] = Shards1, + ?assertEqual([16#00000000, 16#7fffffff], R1), + ?assertEqual([16#80000000, 16#ffffffff], R2), + + % Check metadata bits after the split + ?assertEqual(942, get_revs_limit(Db)), + ?assertEqual(943, get_purge_infos_limit(Db)), + ?assertEqual(SecObj, get_security(Db)), + + DbInfo1 = get_db_info(Db), + Docs1 = get_all_docs(Db), + Local1 = get_local_docs(Db), + + % When comparing db infos, ignore update sequences they won't be the + % same since they are more shards involved after the split + ?assertEqual(without_seqs(DbInfo0), without_seqs(DbInfo1)), + + % Update seq prefix number is a sum of all shard update sequences + #{<<"update_seq">> := UpdateSeq0} = update_seq_to_num(DbInfo0), + #{<<"update_seq">> := UpdateSeq1} = update_seq_to_num(DbInfo1), + ?assertEqual(UpdateSeq0 * 2, UpdateSeq1), + + % Finally compare that the documents are still there after the split + ?assertEqual(Docs0, Docs1), + + % Don't forget about the local but don't include internal checkpoints + % as some of those are munged and transformed during the split + ?assertEqual(without_meta_locals(Local0), without_meta_locals(Local1)) + end)}. % Test to check that shard with high number of purges can be split split_shard_with_lots_of_purges(#{db1 := Db}) -> - {timeout, ?TIMEOUT, ?_test(begin - % Set a low purge infos limit, we are planning to overrun it - set_purge_infos_limit(Db, 10), - - % Add docs 1..20 and purge them - add_test_docs(Db, #{docs => [1, 20]}), - IdRevs = maps:fold(fun(Id, #{<<"_rev">> := Rev}, Acc) -> - [{Id, [Rev]} | Acc] - end, [], get_all_docs(Db)), - ?assertMatch({ok, _}, purge_docs(Db, IdRevs)), - - % Compact to trim the purge sequence - ok = compact(Db), - - % Add some extra docs, these won't be purged - add_test_docs(Db, #{docs => [21, 30]}), - Docs0 = get_all_docs(Db), - - % Save db info before splitting - DbInfo0 = get_db_info(Db), - - % Split the one shard - [#shard{name=Shard}] = lists:sort(mem3:local_shards(Db)), - {ok, JobId} = mem3_reshard:start_split_job(Shard), - wait_state(JobId, completed), - - % Perform some basic checks that the shard was split - Shards1 = lists:sort(mem3:local_shards(Db)), - ?assertEqual(2, length(Shards1)), - [#shard{range = R1}, #shard{range = R2}] = Shards1, - ?assertEqual([16#00000000, 16#7fffffff], R1), - ?assertEqual([16#80000000, 16#ffffffff], R2), - - % Check metadata bits after the split - ?assertEqual(10, get_purge_infos_limit(Db)), - - DbInfo1 = get_db_info(Db), - Docs1 = get_all_docs(Db), - - % When comparing db infos, ignore update sequences they won't be the - % same since they are more shards involved after the split - ?assertEqual(without_seqs(DbInfo0), without_seqs(DbInfo1)), - - % Finally compare that the documents are still there after the split - ?assertEqual(Docs0, Docs1) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + % Set a low purge infos limit, we are planning to overrun it + set_purge_infos_limit(Db, 10), + + % Add docs 1..20 and purge them + add_test_docs(Db, #{docs => [1, 20]}), + IdRevs = maps:fold( + fun(Id, #{<<"_rev">> := Rev}, Acc) -> + [{Id, [Rev]} | Acc] + end, + [], + get_all_docs(Db) + ), + ?assertMatch({ok, _}, purge_docs(Db, IdRevs)), + + % Compact to trim the purge sequence + ok = compact(Db), + + % Add some extra docs, these won't be purged + add_test_docs(Db, #{docs => [21, 30]}), + Docs0 = get_all_docs(Db), + + % Save db info before splitting + DbInfo0 = get_db_info(Db), + + % Split the one shard + [#shard{name = Shard}] = lists:sort(mem3:local_shards(Db)), + {ok, JobId} = mem3_reshard:start_split_job(Shard), + wait_state(JobId, completed), + + % Perform some basic checks that the shard was split + Shards1 = lists:sort(mem3:local_shards(Db)), + ?assertEqual(2, length(Shards1)), + [#shard{range = R1}, #shard{range = R2}] = Shards1, + ?assertEqual([16#00000000, 16#7fffffff], R1), + ?assertEqual([16#80000000, 16#ffffffff], R2), + + % Check metadata bits after the split + ?assertEqual(10, get_purge_infos_limit(Db)), + + DbInfo1 = get_db_info(Db), + Docs1 = get_all_docs(Db), + + % When comparing db infos, ignore update sequences they won't be the + % same since they are more shards involved after the split + ?assertEqual(without_seqs(DbInfo0), without_seqs(DbInfo1)), + + % Finally compare that the documents are still there after the split + ?assertEqual(Docs0, Docs1) + end)}. % This test checks that document added while the shard is being split are not % lost. Topoff1 state happens before indices are built update_docs_before_topoff1(#{db1 := Db}) -> - {timeout, ?TIMEOUT, ?_test(begin - add_test_docs(Db, #{docs => 10}), + {timeout, ?TIMEOUT, + ?_test(begin + add_test_docs(Db, #{docs => 10}), - intercept_state(topoff1), + intercept_state(topoff1), - [#shard{name=Shard}] = lists:sort(mem3:local_shards(Db)), - {ok, JobId} = mem3_reshard:start_split_job(Shard), + [#shard{name = Shard}] = lists:sort(mem3:local_shards(Db)), + {ok, JobId} = mem3_reshard:start_split_job(Shard), - receive {JobPid, topoff1} -> ok end, - add_test_docs(Db, #{docs => [10, 19], local => 1}), - Docs0 = get_all_docs(Db), - Local0 = get_local_docs(Db), - DbInfo0 = get_db_info(Db), - JobPid ! continue, - - wait_state(JobId, completed), + receive + {JobPid, topoff1} -> ok + end, + add_test_docs(Db, #{docs => [10, 19], local => 1}), + Docs0 = get_all_docs(Db), + Local0 = get_local_docs(Db), + DbInfo0 = get_db_info(Db), + JobPid ! continue, - % Perform some basic checks that the shard was split - Shards1 = lists:sort(mem3:local_shards(Db)), - ?assertEqual(2, length(Shards1)), + wait_state(JobId, completed), - DbInfo1 = get_db_info(Db), - Docs1 = get_all_docs(Db), - Local1 = get_local_docs(Db), + % Perform some basic checks that the shard was split + Shards1 = lists:sort(mem3:local_shards(Db)), + ?assertEqual(2, length(Shards1)), - ?assertEqual(without_seqs(DbInfo0), without_seqs(DbInfo1)), + DbInfo1 = get_db_info(Db), + Docs1 = get_all_docs(Db), + Local1 = get_local_docs(Db), - % Update sequence after initial copy with 10 docs would be 10 on each - % target shard (to match the source) and the total update sequence - % would have been 20. But then 10 more docs were added (3 might have - % ended up on one target and 7 on another) so the final update sequence - % would then be 20 + 10 = 30. - ?assertMatch(#{<<"update_seq">> := 30}, update_seq_to_num(DbInfo1)), + ?assertEqual(without_seqs(DbInfo0), without_seqs(DbInfo1)), - ?assertEqual(Docs0, Docs1), - ?assertEqual(without_meta_locals(Local0), without_meta_locals(Local1)) - end)}. + % Update sequence after initial copy with 10 docs would be 10 on each + % target shard (to match the source) and the total update sequence + % would have been 20. But then 10 more docs were added (3 might have + % ended up on one target and 7 on another) so the final update sequence + % would then be 20 + 10 = 30. + ?assertMatch(#{<<"update_seq">> := 30}, update_seq_to_num(DbInfo1)), + ?assertEqual(Docs0, Docs1), + ?assertEqual(without_meta_locals(Local0), without_meta_locals(Local1)) + end)}. % This test that indices are built during shard splitting. indices_are_built(#{db1 := Db}) -> - {timeout, ?TIMEOUT, ?_test(begin - HaveDreyfus = code:lib_dir(dreyfus) /= {error, bad_name}, - HaveHastings = code:lib_dir(hastings) /= {error, bad_name}, - - add_test_docs(Db, #{docs => 10, mrview => 2, search => 2, geo => 2}), - [#shard{name=Shard}] = lists:sort(mem3:local_shards(Db)), - {ok, JobId} = mem3_reshard:start_split_job(Shard), - wait_state(JobId, completed), - Shards1 = lists:sort(mem3:local_shards(Db)), - ?assertEqual(2, length(Shards1)), - MRViewGroupInfo = get_group_info(Db, <<"_design/mrview00000">>), - ?assertMatch(#{<<"update_seq">> := 32}, MRViewGroupInfo), - - HaveDreyfus = code:lib_dir(dreyfus) /= {error, bad_name}, - case HaveDreyfus of false -> ok; true -> - % 4 because there are 2 indices and 2 target shards - ?assertEqual(4, meck:num_calls(dreyfus_index, await, 2)) - end, - - HaveHastings = code:lib_dir(hastings) /= {error, bad_name}, - case HaveHastings of false -> ok; true -> - % 4 because there are 2 indices and 2 target shards - ?assertEqual(4, meck:num_calls(hastings_index, await, 2)) - end - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + HaveDreyfus = code:lib_dir(dreyfus) /= {error, bad_name}, + HaveHastings = code:lib_dir(hastings) /= {error, bad_name}, + + add_test_docs(Db, #{docs => 10, mrview => 2, search => 2, geo => 2}), + [#shard{name = Shard}] = lists:sort(mem3:local_shards(Db)), + {ok, JobId} = mem3_reshard:start_split_job(Shard), + wait_state(JobId, completed), + Shards1 = lists:sort(mem3:local_shards(Db)), + ?assertEqual(2, length(Shards1)), + MRViewGroupInfo = get_group_info(Db, <<"_design/mrview00000">>), + ?assertMatch(#{<<"update_seq">> := 32}, MRViewGroupInfo), + + HaveDreyfus = code:lib_dir(dreyfus) /= {error, bad_name}, + case HaveDreyfus of + false -> + ok; + true -> + % 4 because there are 2 indices and 2 target shards + ?assertEqual(4, meck:num_calls(dreyfus_index, await, 2)) + end, + + HaveHastings = code:lib_dir(hastings) /= {error, bad_name}, + case HaveHastings of + false -> + ok; + true -> + % 4 because there are 2 indices and 2 target shards + ?assertEqual(4, meck:num_calls(hastings_index, await, 2)) + end + end)}. mock_dreyfus_indices() -> meck:expect(dreyfus_index, design_doc_to_indexes, fun(Doc) -> @@ -275,7 +286,6 @@ mock_dreyfus_indices() -> meck:expect(dreyfus_index_manager, get_index, fun(_, _) -> {ok, pid} end), meck:expect(dreyfus_index, await, fun(_, _) -> ok end). - mock_hastings_indices() -> meck:expect(hastings_index, design_doc_to_indexes, fun(Doc) -> #doc{body = {BodyProps}} = Doc, @@ -291,465 +301,484 @@ mock_hastings_indices() -> % Split partitioned database split_partitioned_db(#{db2 := Db}) -> - {timeout, ?TIMEOUT, ?_test(begin - DocSpec = #{ - pdocs => #{ - <<"PX">> => 5, - <<"PY">> => 5 + {timeout, ?TIMEOUT, + ?_test(begin + DocSpec = #{ + pdocs => #{ + <<"PX">> => 5, + <<"PY">> => 5 + }, + mrview => 1, + local => 1 }, - mrview => 1, - local => 1 - }, - add_test_docs(Db, DocSpec), - - % Save documents before the split - Docs0 = get_all_docs(Db), - Local0 = get_local_docs(Db), - - % Set some custom metadata properties - set_revs_limit(Db, 942), - set_purge_infos_limit(Db, 943), - SecObj = {[{<<"foo">>, <<"bar">>}]}, - set_security(Db, SecObj), - - % DbInfo is saved after setting metadata bits - % as those could bump the update sequence - DbInfo0 = get_db_info(Db), - PX0 = get_partition_info(Db, <<"PX">>), - PY0 = get_partition_info(Db, <<"PY">>), - - % Split the one shard - [#shard{name=Shard}] = lists:sort(mem3:local_shards(Db)), - {ok, JobId} = mem3_reshard:start_split_job(Shard), - wait_state(JobId, completed), - - % Perform some basic checks that the shard was split - Shards1 = lists:sort(mem3:local_shards(Db)), - ?assertEqual(2, length(Shards1)), - [#shard{range = R1}, #shard{range = R2}] = Shards1, - ?assertEqual([16#00000000, 16#7fffffff], R1), - ?assertEqual([16#80000000, 16#ffffffff], R2), - - % Check metadata bits after the split - ?assertEqual(942, get_revs_limit(Db)), - ?assertEqual(943, get_purge_infos_limit(Db)), - ?assertEqual(SecObj, get_security(Db)), - - DbInfo1 = get_db_info(Db), - Docs1 = get_all_docs(Db), - Local1 = get_local_docs(Db), - - % When comparing db infos, ignore update sequences they won't be the - % same since they are more shards involved after the split - ?assertEqual(without_seqs(DbInfo0), without_seqs(DbInfo1)), - - % Update seq prefix number is a sum of all shard update sequences - #{<<"update_seq">> := UpdateSeq0} = update_seq_to_num(DbInfo0), - #{<<"update_seq">> := UpdateSeq1} = update_seq_to_num(DbInfo1), - ?assertEqual(UpdateSeq0 * 2, UpdateSeq1), - - % Finally compare that documents are still there after the split - ?assertEqual(Docs0, Docs1), - - ?assertEqual(PX0, get_partition_info(Db, <<"PX">>)), - ?assertEqual(PY0, get_partition_info(Db, <<"PY">>)), - - % Don't forget about the local but don't include internal checkpoints - % as some of those are munged and transformed during the split - ?assertEqual(without_meta_locals(Local0), without_meta_locals(Local1)) - end)}. - + add_test_docs(Db, DocSpec), + + % Save documents before the split + Docs0 = get_all_docs(Db), + Local0 = get_local_docs(Db), + + % Set some custom metadata properties + set_revs_limit(Db, 942), + set_purge_infos_limit(Db, 943), + SecObj = {[{<<"foo">>, <<"bar">>}]}, + set_security(Db, SecObj), + + % DbInfo is saved after setting metadata bits + % as those could bump the update sequence + DbInfo0 = get_db_info(Db), + PX0 = get_partition_info(Db, <<"PX">>), + PY0 = get_partition_info(Db, <<"PY">>), + + % Split the one shard + [#shard{name = Shard}] = lists:sort(mem3:local_shards(Db)), + {ok, JobId} = mem3_reshard:start_split_job(Shard), + wait_state(JobId, completed), + + % Perform some basic checks that the shard was split + Shards1 = lists:sort(mem3:local_shards(Db)), + ?assertEqual(2, length(Shards1)), + [#shard{range = R1}, #shard{range = R2}] = Shards1, + ?assertEqual([16#00000000, 16#7fffffff], R1), + ?assertEqual([16#80000000, 16#ffffffff], R2), + + % Check metadata bits after the split + ?assertEqual(942, get_revs_limit(Db)), + ?assertEqual(943, get_purge_infos_limit(Db)), + ?assertEqual(SecObj, get_security(Db)), + + DbInfo1 = get_db_info(Db), + Docs1 = get_all_docs(Db), + Local1 = get_local_docs(Db), + + % When comparing db infos, ignore update sequences they won't be the + % same since they are more shards involved after the split + ?assertEqual(without_seqs(DbInfo0), without_seqs(DbInfo1)), + + % Update seq prefix number is a sum of all shard update sequences + #{<<"update_seq">> := UpdateSeq0} = update_seq_to_num(DbInfo0), + #{<<"update_seq">> := UpdateSeq1} = update_seq_to_num(DbInfo1), + ?assertEqual(UpdateSeq0 * 2, UpdateSeq1), + + % Finally compare that documents are still there after the split + ?assertEqual(Docs0, Docs1), + + ?assertEqual(PX0, get_partition_info(Db, <<"PX">>)), + ?assertEqual(PY0, get_partition_info(Db, <<"PY">>)), + + % Don't forget about the local but don't include internal checkpoints + % as some of those are munged and transformed during the split + ?assertEqual(without_meta_locals(Local0), without_meta_locals(Local1)) + end)}. % Make sure a shard can be split again after it was split once. This checks that % too many got added to some range, such that on next split they'd fail to fit % in to any of the new target ranges. split_twice(#{db1 := Db}) -> - {timeout, ?TIMEOUT, ?_test(begin - DocSpec = #{docs => 100, delete => [80, 99], mrview => 2, local => 100}, - add_test_docs(Db, DocSpec), - - % Save documents before the split - Docs0 = get_all_docs(Db), - Local0 = get_local_docs(Db), - - % Set some custom metadata properties - set_revs_limit(Db, 942), - set_purge_infos_limit(Db, 943), - SecObj = {[{<<"foo">>, <<"bar">>}]}, - set_security(Db, SecObj), - - % DbInfo is saved after setting metadata bits - % as those could bump the update sequence - DbInfo0 = get_db_info(Db), - - % Split the one shard - [#shard{name=Shard1}] = lists:sort(mem3:local_shards(Db)), - {ok, JobId1} = mem3_reshard:start_split_job(Shard1), - wait_state(JobId1, completed), - - % Perform some basic checks that the shard was split - Shards1 = lists:sort(mem3:local_shards(Db)), - ?assertEqual(2, length(Shards1)), - [#shard{range = R1}, #shard{range = R2}] = Shards1, - ?assertEqual([16#00000000, 16#7fffffff], R1), - ?assertEqual([16#80000000, 16#ffffffff], R2), - - % Check metadata bits after the split - ?assertEqual(942, get_revs_limit(Db)), - ?assertEqual(943, get_purge_infos_limit(Db)), - ?assertEqual(SecObj, get_security(Db)), - - DbInfo1 = get_db_info(Db), - Docs1 = get_all_docs(Db), - Local1 = get_local_docs(Db), - - % When comparing db infos, ignore update sequences they won't be the - % same since they are more shards involved after the split - ?assertEqual(without_seqs(DbInfo0), without_seqs(DbInfo1)), - - % Update seq prefix number is a sum of all shard update sequences - #{<<"update_seq">> := UpdateSeq0} = update_seq_to_num(DbInfo0), - #{<<"update_seq">> := UpdateSeq1} = update_seq_to_num(DbInfo1), - ?assertEqual(UpdateSeq0 * 2, UpdateSeq1), - - ?assertEqual(Docs0, Docs1), - ?assertEqual(without_meta_locals(Local0), without_meta_locals(Local1)), - - % Split the first range again - [#shard{name=Shard2}, _] = lists:sort(mem3:local_shards(Db)), - {ok, JobId2} = mem3_reshard:start_split_job(Shard2), - wait_state(JobId2, completed), - - Shards2 = lists:sort(mem3:local_shards(Db)), - ?assertEqual(3, length(Shards2)), - [R3, R4, R5] = [R || #shard{range = R} <- Shards2], - ?assertEqual([16#00000000, 16#3fffffff], R3), - ?assertEqual([16#40000000, 16#7fffffff], R4), - ?assertEqual([16#80000000, 16#ffffffff], R5), - - % Check metadata bits after the second split - ?assertEqual(942, get_revs_limit(Db)), - ?assertEqual(943, get_purge_infos_limit(Db)), - ?assertEqual(SecObj, get_security(Db)), - - DbInfo2 = get_db_info(Db), - Docs2 = get_all_docs(Db), - Local2 = get_local_docs(Db), - - ?assertEqual(without_seqs(DbInfo1), without_seqs(DbInfo2)), - % Update seq prefix number is a sum of all shard update sequences - % But only 1 shard out of 2 was split - #{<<"update_seq">> := UpdateSeq2} = update_seq_to_num(DbInfo2), - ?assertEqual(trunc(UpdateSeq1 * 1.5), UpdateSeq2), - ?assertEqual(Docs1, Docs2), - ?assertEqual(without_meta_locals(Local1), without_meta_locals(Local2)) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + DocSpec = #{docs => 100, delete => [80, 99], mrview => 2, local => 100}, + add_test_docs(Db, DocSpec), + + % Save documents before the split + Docs0 = get_all_docs(Db), + Local0 = get_local_docs(Db), + + % Set some custom metadata properties + set_revs_limit(Db, 942), + set_purge_infos_limit(Db, 943), + SecObj = {[{<<"foo">>, <<"bar">>}]}, + set_security(Db, SecObj), + + % DbInfo is saved after setting metadata bits + % as those could bump the update sequence + DbInfo0 = get_db_info(Db), + + % Split the one shard + [#shard{name = Shard1}] = lists:sort(mem3:local_shards(Db)), + {ok, JobId1} = mem3_reshard:start_split_job(Shard1), + wait_state(JobId1, completed), + + % Perform some basic checks that the shard was split + Shards1 = lists:sort(mem3:local_shards(Db)), + ?assertEqual(2, length(Shards1)), + [#shard{range = R1}, #shard{range = R2}] = Shards1, + ?assertEqual([16#00000000, 16#7fffffff], R1), + ?assertEqual([16#80000000, 16#ffffffff], R2), + + % Check metadata bits after the split + ?assertEqual(942, get_revs_limit(Db)), + ?assertEqual(943, get_purge_infos_limit(Db)), + ?assertEqual(SecObj, get_security(Db)), + + DbInfo1 = get_db_info(Db), + Docs1 = get_all_docs(Db), + Local1 = get_local_docs(Db), + + % When comparing db infos, ignore update sequences they won't be the + % same since they are more shards involved after the split + ?assertEqual(without_seqs(DbInfo0), without_seqs(DbInfo1)), + + % Update seq prefix number is a sum of all shard update sequences + #{<<"update_seq">> := UpdateSeq0} = update_seq_to_num(DbInfo0), + #{<<"update_seq">> := UpdateSeq1} = update_seq_to_num(DbInfo1), + ?assertEqual(UpdateSeq0 * 2, UpdateSeq1), + + ?assertEqual(Docs0, Docs1), + ?assertEqual(without_meta_locals(Local0), without_meta_locals(Local1)), + + % Split the first range again + [#shard{name = Shard2}, _] = lists:sort(mem3:local_shards(Db)), + {ok, JobId2} = mem3_reshard:start_split_job(Shard2), + wait_state(JobId2, completed), + + Shards2 = lists:sort(mem3:local_shards(Db)), + ?assertEqual(3, length(Shards2)), + [R3, R4, R5] = [R || #shard{range = R} <- Shards2], + ?assertEqual([16#00000000, 16#3fffffff], R3), + ?assertEqual([16#40000000, 16#7fffffff], R4), + ?assertEqual([16#80000000, 16#ffffffff], R5), + + % Check metadata bits after the second split + ?assertEqual(942, get_revs_limit(Db)), + ?assertEqual(943, get_purge_infos_limit(Db)), + ?assertEqual(SecObj, get_security(Db)), + + DbInfo2 = get_db_info(Db), + Docs2 = get_all_docs(Db), + Local2 = get_local_docs(Db), + + ?assertEqual(without_seqs(DbInfo1), without_seqs(DbInfo2)), + % Update seq prefix number is a sum of all shard update sequences + % But only 1 shard out of 2 was split + #{<<"update_seq">> := UpdateSeq2} = update_seq_to_num(DbInfo2), + ?assertEqual(trunc(UpdateSeq1 * 1.5), UpdateSeq2), + ?assertEqual(Docs1, Docs2), + ?assertEqual(without_meta_locals(Local1), without_meta_locals(Local2)) + end)}. couch_events_are_emitted(#{db1 := Db}) -> - {timeout, ?TIMEOUT, ?_test(begin - couch_event:register_all(self()), - - % Split the one shard - [#shard{name=Shard}] = lists:sort(mem3:local_shards(Db)), - {ok, JobId} = mem3_reshard:start_split_job(Shard), - wait_state(JobId, completed), - - % Perform some basic checks that the shard was split - Shards1 = lists:sort(mem3:local_shards(Db)), - ?assertEqual(2, length(Shards1)), - [#shard{range = R1}, #shard{range = R2}] = Shards1, - ?assertEqual([16#00000000, 16#7fffffff], R1), - ?assertEqual([16#80000000, 16#ffffffff], R2), - - Flush = fun F(Events) -> - receive - {'$couch_event', DbName, Event} when Event =:= deleted - orelse Event =:= updated -> - case binary:match(DbName, Db) of - nomatch -> F(Events); - {_, _} -> F([Event | Events]) - end - after 0 -> + {timeout, ?TIMEOUT, + ?_test(begin + couch_event:register_all(self()), + + % Split the one shard + [#shard{name = Shard}] = lists:sort(mem3:local_shards(Db)), + {ok, JobId} = mem3_reshard:start_split_job(Shard), + wait_state(JobId, completed), + + % Perform some basic checks that the shard was split + Shards1 = lists:sort(mem3:local_shards(Db)), + ?assertEqual(2, length(Shards1)), + [#shard{range = R1}, #shard{range = R2}] = Shards1, + ?assertEqual([16#00000000, 16#7fffffff], R1), + ?assertEqual([16#80000000, 16#ffffffff], R2), + + Flush = fun F(Events) -> + receive + {'$couch_event', DbName, Event} when + Event =:= deleted orelse + Event =:= updated + -> + case binary:match(DbName, Db) of + nomatch -> F(Events); + {_, _} -> F([Event | Events]) + end + after 0 -> lists:reverse(Events) - end - end, - Events = Flush([]), - StartAtDeleted = lists:dropwhile(fun(E) -> E =/= deleted end, Events), - ?assertMatch([deleted, deleted, updated, updated | _], StartAtDeleted), - couch_event:unregister(self()) - end)}. - + end + end, + Events = Flush([]), + StartAtDeleted = lists:dropwhile(fun(E) -> E =/= deleted end, Events), + ?assertMatch([deleted, deleted, updated, updated | _], StartAtDeleted), + couch_event:unregister(self()) + end)}. retries_work(#{db1 := Db}) -> - {timeout, ?TIMEOUT, ?_test(begin - meck:expect(couch_db_split, split, fun(_, _, _) -> - error(kapow) - end), + {timeout, ?TIMEOUT, + ?_test(begin + meck:expect(couch_db_split, split, fun(_, _, _) -> + error(kapow) + end), - [#shard{name=Shard}] = lists:sort(mem3:local_shards(Db)), - {ok, JobId} = mem3_reshard:start_split_job(Shard), - - wait_state(JobId, failed), - ?assertEqual(3, meck:num_calls(couch_db_split, split, 3)) - end)}. + [#shard{name = Shard}] = lists:sort(mem3:local_shards(Db)), + {ok, JobId} = mem3_reshard:start_split_job(Shard), + wait_state(JobId, failed), + ?assertEqual(3, meck:num_calls(couch_db_split, split, 3)) + end)}. target_reset_in_initial_copy(#{db1 := Db}) -> - {timeout, ?TIMEOUT, ?_test(begin - [#shard{} = Src] = lists:sort(mem3:local_shards(Db)), - Job = #job{ - source = Src, - target = [#shard{name= <<"t1">>}, #shard{name = <<"t2">>}], - job_state = running, - split_state = initial_copy - }, - meck:expect(couch_db_split, cleanup_target, 2, ok), - meck:expect(couch_server, exists, fun - (<<"t1">>) -> true; - (<<"t2">>) -> true; - (DbName) -> meck:passthrough([DbName]) - end), - JobPid = spawn(fun() -> mem3_reshard_job:initial_copy_impl(Job) end), - meck:wait(2, couch_db_split, cleanup_target, ['_', '_'], 5000), - exit(JobPid, kill), - ?assertEqual(2, meck:num_calls(couch_db_split, cleanup_target, 2)) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + [#shard{} = Src] = lists:sort(mem3:local_shards(Db)), + Job = #job{ + source = Src, + target = [#shard{name = <<"t1">>}, #shard{name = <<"t2">>}], + job_state = running, + split_state = initial_copy + }, + meck:expect(couch_db_split, cleanup_target, 2, ok), + meck:expect(couch_server, exists, fun + (<<"t1">>) -> true; + (<<"t2">>) -> true; + (DbName) -> meck:passthrough([DbName]) + end), + JobPid = spawn(fun() -> mem3_reshard_job:initial_copy_impl(Job) end), + meck:wait(2, couch_db_split, cleanup_target, ['_', '_'], 5000), + exit(JobPid, kill), + ?assertEqual(2, meck:num_calls(couch_db_split, cleanup_target, 2)) + end)}. split_an_incomplete_shard_map(#{db1 := Db}) -> - {timeout, ?TIMEOUT, ?_test(begin - [#shard{name=Shard}] = lists:sort(mem3:local_shards(Db)), - meck:expect(mem3_util, calculate_max_n, 1, 0), - ?assertMatch({error, {not_enough_shard_copies, _}}, - mem3_reshard:start_split_job(Shard)) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + [#shard{name = Shard}] = lists:sort(mem3:local_shards(Db)), + meck:expect(mem3_util, calculate_max_n, 1, 0), + ?assertMatch( + {error, {not_enough_shard_copies, _}}, + mem3_reshard:start_split_job(Shard) + ) + end)}. % Opening a db target db in initial copy phase will throw an error target_shards_are_locked(#{db1 := Db}) -> - {timeout, ?TIMEOUT, ?_test(begin - add_test_docs(Db, #{docs => 10}), - - % Make the job stops right when it was about to copy the docs - TestPid = self(), - meck:new(couch_db, [passthrough]), - meck:expect(couch_db, start_link, fun(Engine, TName, FilePath, Opts) -> - TestPid ! {start_link, self(), TName}, - receive - continue -> - meck:passthrough([Engine, TName, FilePath, Opts]) - end - end), - - [#shard{name=Shard}] = lists:sort(mem3:local_shards(Db)), - {ok, JobId} = mem3_reshard:start_split_job(Shard), - {Target0, JobPid} = receive - {start_link, Pid, TName} -> {TName, Pid} - end, - ?assertEqual({error, {locked, <<"shard splitting">>}}, - couch_db:open_int(Target0, [])), - - % Send two continues for two targets - JobPid ! continue, - JobPid ! continue, - - wait_state(JobId, completed) - end)}. - + {timeout, ?TIMEOUT, + ?_test(begin + add_test_docs(Db, #{docs => 10}), + + % Make the job stops right when it was about to copy the docs + TestPid = self(), + meck:new(couch_db, [passthrough]), + meck:expect(couch_db, start_link, fun(Engine, TName, FilePath, Opts) -> + TestPid ! {start_link, self(), TName}, + receive + continue -> + meck:passthrough([Engine, TName, FilePath, Opts]) + end + end), + + [#shard{name = Shard}] = lists:sort(mem3:local_shards(Db)), + {ok, JobId} = mem3_reshard:start_split_job(Shard), + {Target0, JobPid} = + receive + {start_link, Pid, TName} -> {TName, Pid} + end, + ?assertEqual( + {error, {locked, <<"shard splitting">>}}, + couch_db:open_int(Target0, []) + ), + + % Send two continues for two targets + JobPid ! continue, + JobPid ! continue, + + wait_state(JobId, completed) + end)}. intercept_state(State) -> TestPid = self(), meck:new(mem3_reshard_job, [passthrough]), meck:expect(mem3_reshard_job, checkpoint_done, fun(Job) -> - case Job#job.split_state of - State -> - TestPid ! {self(), State}, - receive - continue -> meck:passthrough([Job]); - cancel -> ok - end; - _ -> - meck:passthrough([Job]) - end - end). - + case Job#job.split_state of + State -> + TestPid ! {self(), State}, + receive + continue -> meck:passthrough([Job]); + cancel -> ok + end; + _ -> + meck:passthrough([Job]) + end + end). wait_state(JobId, State) -> - test_util:wait(fun() -> + test_util:wait( + fun() -> case mem3_reshard:job(JobId) of {ok, {Props}} -> case couch_util:get_value(job_state, Props) of - State -> ok; - _ -> timer:sleep(100), wait + State -> + ok; + _ -> + timer:sleep(100), + wait end; - {error, not_found} -> timer:sleep(100), wait + {error, not_found} -> + timer:sleep(100), + wait end - end, 30000). - + end, + 30000 + ). set_revs_limit(DbName, Limit) -> with_proc(fun() -> fabric:set_revs_limit(DbName, Limit, [?ADMIN_CTX]) end). - get_revs_limit(DbName) -> with_proc(fun() -> fabric:get_revs_limit(DbName) end). - get_purge_infos_limit(DbName) -> with_proc(fun() -> fabric:get_purge_infos_limit(DbName) end). - set_purge_infos_limit(DbName, Limit) -> with_proc(fun() -> fabric:set_purge_infos_limit(DbName, Limit, [?ADMIN_CTX]) end). - purge_docs(DbName, DocIdRevs) -> with_proc(fun() -> fabric:purge_docs(DbName, DocIdRevs, []) end). - compact(DbName) -> InitFileSize = get_db_file_size(DbName), ok = with_proc(fun() -> fabric:compact(DbName) end), - test_util:wait(fun() -> - case {compact_running(DbName), get_db_file_size(DbName)} of - {true, _} -> wait; - {false, FileSize} when FileSize == InitFileSize -> wait; - {false, FileSize} when FileSize < InitFileSize -> ok - end - end, 5000, 200). - + test_util:wait( + fun() -> + case {compact_running(DbName), get_db_file_size(DbName)} of + {true, _} -> wait; + {false, FileSize} when FileSize == InitFileSize -> wait; + {false, FileSize} when FileSize < InitFileSize -> ok + end + end, + 5000, + 200 + ). compact_running(DbName) -> {ok, DbInfo} = with_proc(fun() -> fabric:get_db_info(DbName) end), #{<<"compact_running">> := CompactRunning} = to_map(DbInfo), CompactRunning. - get_db_file_size(DbName) -> {ok, DbInfo} = with_proc(fun() -> fabric:get_db_info(DbName) end), #{<<"sizes">> := #{<<"file">> := FileSize}} = to_map(DbInfo), FileSize. - set_security(DbName, SecObj) -> with_proc(fun() -> fabric:set_security(DbName, SecObj) end). - get_security(DbName) -> with_proc(fun() -> fabric:get_security(DbName, [?ADMIN_CTX]) end). - get_db_info(DbName) -> with_proc(fun() -> {ok, Info} = fabric:get_db_info(DbName), - maps:with([ - <<"db_name">>, <<"doc_count">>, <<"props">>, <<"doc_del_count">>, - <<"update_seq">>, <<"purge_seq">>, <<"disk_format_version">> - ], to_map(Info)) + maps:with( + [ + <<"db_name">>, + <<"doc_count">>, + <<"props">>, + <<"doc_del_count">>, + <<"update_seq">>, + <<"purge_seq">>, + <<"disk_format_version">> + ], + to_map(Info) + ) end). - get_group_info(DbName, DesignId) -> with_proc(fun() -> {ok, GInfo} = fabric:get_view_group_info(DbName, DesignId), - maps:with([ - <<"language">>, <<"purge_seq">>, <<"signature">>, <<"update_seq">> - ], to_map(GInfo)) + maps:with( + [ + <<"language">>, <<"purge_seq">>, <<"signature">>, <<"update_seq">> + ], + to_map(GInfo) + ) end). - get_partition_info(DbName, Partition) -> with_proc(fun() -> {ok, PInfo} = fabric:get_partition_info(DbName, Partition), - maps:with([ - <<"db_name">>, <<"doc_count">>, <<"doc_del_count">>, <<"partition">> - ], to_map(PInfo)) + maps:with( + [ + <<"db_name">>, <<"doc_count">>, <<"doc_del_count">>, <<"partition">> + ], + to_map(PInfo) + ) end). - get_all_docs(DbName) -> get_all_docs(DbName, #mrargs{}). - get_all_docs(DbName, #mrargs{} = QArgs0) -> GL = erlang:group_leader(), - with_proc(fun() -> - Cb = fun - ({row, Props}, Acc) -> - Doc = to_map(couch_util:get_value(doc, Props)), - #{?ID := Id} = Doc, - {ok, Acc#{Id => Doc}}; - ({meta, _}, Acc) -> {ok, Acc}; - (complete, Acc) -> {ok, Acc} + with_proc( + fun() -> + Cb = fun + ({row, Props}, Acc) -> + Doc = to_map(couch_util:get_value(doc, Props)), + #{?ID := Id} = Doc, + {ok, Acc#{Id => Doc}}; + ({meta, _}, Acc) -> + {ok, Acc}; + (complete, Acc) -> + {ok, Acc} + end, + QArgs = QArgs0#mrargs{include_docs = true}, + {ok, Docs} = fabric:all_docs(DbName, Cb, #{}, QArgs), + Docs end, - QArgs = QArgs0#mrargs{include_docs = true}, - {ok, Docs} = fabric:all_docs(DbName, Cb, #{}, QArgs), - Docs - end, GL). - + GL + ). get_local_docs(DbName) -> LocalNS = {namespace, <<"_local">>}, - maps:map(fun(_, Doc) -> - maps:without([<<"_rev">>], Doc) - end, get_all_docs(DbName, #mrargs{extra = [LocalNS]})). - + maps:map( + fun(_, Doc) -> + maps:without([<<"_rev">>], Doc) + end, + get_all_docs(DbName, #mrargs{extra = [LocalNS]}) + ). without_seqs(#{} = InfoMap) -> maps:without([<<"update_seq">>, <<"purge_seq">>], InfoMap). - without_meta_locals(#{} = Local) -> - maps:filter(fun - (<<"_local/purge-mrview-", _/binary>>, _) -> false; - (<<"_local/shard-sync-", _/binary>>, _) -> false; - (_, _) -> true - end, Local). - + maps:filter( + fun + (<<"_local/purge-mrview-", _/binary>>, _) -> false; + (<<"_local/shard-sync-", _/binary>>, _) -> false; + (_, _) -> true + end, + Local + ). update_seq_to_num(#{} = InfoMap) -> - maps:map(fun - (<<"update_seq">>, Seq) -> seq_to_num(Seq); - (<<"purge_seq">>, PSeq) -> seq_to_num(PSeq); - (_, V) -> V - end, InfoMap). - + maps:map( + fun + (<<"update_seq">>, Seq) -> seq_to_num(Seq); + (<<"purge_seq">>, PSeq) -> seq_to_num(PSeq); + (_, V) -> V + end, + InfoMap + ). seq_to_num(Seq) -> [SeqNum, _] = binary:split(Seq, <<"-">>), binary_to_integer(SeqNum). - to_map([_ | _] = Props) -> to_map({Props}); - to_map({[_ | _]} = EJson) -> - jiffy:decode(jiffy:encode(EJson), [return_maps]). - + jiffy:decode(jiffy:encode(EJson), [return_maps]). create_db(DbName, Opts) -> GL = erlang:group_leader(), with_proc(fun() -> fabric:create_db(DbName, Opts) end, GL). - delete_db(DbName) -> GL = erlang:group_leader(), with_proc(fun() -> fabric:delete_db(DbName, [?ADMIN_CTX]) end, GL). - with_proc(Fun) -> with_proc(Fun, undefined, 30000). - with_proc(Fun, GroupLeader) -> with_proc(Fun, GroupLeader, 30000). - with_proc(Fun, GroupLeader, Timeout) -> {Pid, Ref} = spawn_monitor(fun() -> case GroupLeader of @@ -767,27 +796,29 @@ with_proc(Fun, GroupLeader, Timeout) -> erlang:demonitor(Ref, [flush]), exit(Pid, kill), error({with_proc_timeout, Fun, Timeout}) - end. - + end. add_test_docs(DbName, #{} = DocSpec) -> - Docs = docs(maps:get(docs, DocSpec, [])) - ++ pdocs(maps:get(pdocs, DocSpec, #{})) - ++ ddocs(mrview, maps:get(mrview, DocSpec, [])) - ++ ddocs(search, maps:get(search, DocSpec, [])) - ++ ddocs(geo, maps:get(geo, DocSpec, [])) - ++ ldocs(maps:get(local, DocSpec, [])), + Docs = + docs(maps:get(docs, DocSpec, [])) ++ + pdocs(maps:get(pdocs, DocSpec, #{})) ++ + ddocs(mrview, maps:get(mrview, DocSpec, [])) ++ + ddocs(search, maps:get(search, DocSpec, [])) ++ + ddocs(geo, maps:get(geo, DocSpec, [])) ++ + ldocs(maps:get(local, DocSpec, [])), Res = update_docs(DbName, Docs), - Docs1 = lists:map(fun({Doc, {ok, {RevPos, Rev}}}) -> - Doc#doc{revs = {RevPos, [Rev]}} - end, lists:zip(Docs, Res)), + Docs1 = lists:map( + fun({Doc, {ok, {RevPos, Rev}}}) -> + Doc#doc{revs = {RevPos, [Rev]}} + end, + lists:zip(Docs, Res) + ), case delete_docs(maps:get(delete, DocSpec, []), Docs1) of [] -> ok; [_ | _] = Deleted -> update_docs(DbName, Deleted) end, ok. - update_docs(DbName, Docs) -> with_proc(fun() -> case fabric:update_docs(DbName, Docs, [?ADMIN_CTX]) of @@ -796,29 +827,32 @@ update_docs(DbName, Docs) -> end end). - delete_docs([S, E], Docs) when E >= S -> ToDelete = [doc_id(<<"">>, I) || I <- lists:seq(S, E)], - lists:filtermap(fun(#doc{id = Id} = Doc) -> - case lists:member(Id, ToDelete) of - true -> {true, Doc#doc{deleted = true}}; - false -> false - end - end, Docs); + lists:filtermap( + fun(#doc{id = Id} = Doc) -> + case lists:member(Id, ToDelete) of + true -> {true, Doc#doc{deleted = true}}; + false -> false + end + end, + Docs + ); delete_docs(_, _) -> []. - pdocs(#{} = PMap) -> - maps:fold(fun(Part, DocSpec, DocsAcc) -> - docs(DocSpec, <<Part/binary, ":">>) ++ DocsAcc - end, [], PMap). - + maps:fold( + fun(Part, DocSpec, DocsAcc) -> + docs(DocSpec, <<Part/binary, ":">>) ++ DocsAcc + end, + [], + PMap + ). docs(DocSpec) -> docs(DocSpec, <<"">>). - docs(N, Prefix) when is_integer(N), N > 0 -> docs([0, N - 1], Prefix); docs([S, E], Prefix) when E >= S -> @@ -835,7 +869,6 @@ ddocs(Type, [S, E]) when E >= S -> ddocs(_, _) -> []. - ldocs(N) when is_integer(N), N > 0 -> ldocs([0, N - 1]); ldocs([S, E]) when E >= S -> @@ -843,13 +876,10 @@ ldocs([S, E]) when E >= S -> ldocs(_) -> []. - - doc(Pref, Id) -> Body = bodyprops(), doc(Pref, Id, Body, 42). - doc(Pref, Id, BodyProps, AttSize) -> #doc{ id = doc_id(Pref, Id), @@ -857,58 +887,60 @@ doc(Pref, Id, BodyProps, AttSize) -> atts = atts(AttSize) }. - doc_id(Pref, Id) -> IdBin = iolist_to_binary(io_lib:format("~5..0B", [Id])), <<Pref/binary, IdBin/binary>>. - ddprop(mrview) -> [ - {<<"views">>, {[ - {<<"v1">>, {[ - {<<"map">>, <<"function(d){emit(d);}">>} - ]}} - ]}} + {<<"views">>, + {[ + {<<"v1">>, + {[ + {<<"map">>, <<"function(d){emit(d);}">>} + ]}} + ]}} ]; - ddprop(geo) -> [ - {<<"st_indexes">>, {[ - {<<"area">>, {[ - {<<"analyzer">>, <<"standard">>}, - {<<"index">>, <<"function(d){if(d.g){st_index(d.g)}}">> } + {<<"st_indexes">>, + {[ + {<<"area">>, + {[ + {<<"analyzer">>, <<"standard">>}, + {<<"index">>, <<"function(d){if(d.g){st_index(d.g)}}">>} + ]}} ]}} - ]}} ]; - ddprop(search) -> [ - {<<"indexes">>, {[ - {<<"types">>, {[ - {<<"index">>, <<"function(d){if(d.g){st_index(d.g.type)}}">>} + {<<"indexes">>, + {[ + {<<"types">>, + {[ + {<<"index">>, <<"function(d){if(d.g){st_index(d.g.type)}}">>} + ]}} ]}} - ]}} - ]. - + ]. bodyprops() -> [ - {<<"g">>, {[ - {<<"type">>, <<"Polygon">>}, - {<<"coordinates">>, [[[-71.0, 48.4], [-70.0, 48.4], [-71.0, 48.4]]]} - ]}} - ]. - + {<<"g">>, + {[ + {<<"type">>, <<"Polygon">>}, + {<<"coordinates">>, [[[-71.0, 48.4], [-70.0, 48.4], [-71.0, 48.4]]]} + ]}} + ]. atts(0) -> []; - atts(Size) when is_integer(Size), Size >= 1 -> - Data = << <<"x">> || _ <- lists:seq(1, Size) >>, - [couch_att:new([ - {name, <<"att">>}, - {type, <<"app/binary">>}, - {att_len, Size}, - {data, Data} - ])]. + Data = <<<<"x">> || _ <- lists:seq(1, Size)>>, + [ + couch_att:new([ + {name, <<"att">>}, + {type, <<"app/binary">>}, + {att_len, Size}, + {data, Data} + ]) + ]. diff --git a/src/mem3/test/eunit/mem3_ring_prop_tests.erl b/src/mem3/test/eunit/mem3_ring_prop_tests.erl index 51d8f10bf..3e2310b21 100644 --- a/src/mem3/test/eunit/mem3_ring_prop_tests.erl +++ b/src/mem3/test/eunit/mem3_ring_prop_tests.erl @@ -12,30 +12,32 @@ -module(mem3_ring_prop_tests). - -ifdef(WITH_PROPER). -include_lib("couch/include/couch_eunit_proper.hrl"). - property_test_() -> ?EUNIT_QUICKCHECK(60). - % Properties prop_get_ring_with_connected_intervals() -> - ?FORALL({Start, End}, oneof(ranges()), - ?FORALL(Intervals, g_connected_intervals(Start, End), + ?FORALL( + {Start, End}, + oneof(ranges()), + ?FORALL( + Intervals, + g_connected_intervals(Start, End), mem3_util:get_ring(Intervals, Start, End) =:= lists:sort(Intervals) ) ). - prop_get_ring_connected_plus_random_intervals() -> - ?FORALL({Intervals, Extra}, {g_connected_intervals(1, 100), - g_random_intervals(1, 100)}, - ?IMPLIES(sets:is_disjoint(endpoints(Intervals), endpoints(Extra)), + ?FORALL( + {Intervals, Extra}, + {g_connected_intervals(1, 100), g_random_intervals(1, 100)}, + ?IMPLIES( + sets:is_disjoint(endpoints(Intervals), endpoints(Extra)), begin AllInts = Intervals ++ Extra, Ring = mem3_util:get_ring(AllInts, 1, 100), @@ -44,10 +46,13 @@ prop_get_ring_connected_plus_random_intervals() -> ) ). - prop_get_ring_connected_with_sub_intervals() -> - ?FORALL(Intervals, g_connected_intervals(1, 100), - ?FORALL(SubIntervals, g_subintervals(Intervals), + ?FORALL( + Intervals, + g_connected_intervals(1, 100), + ?FORALL( + SubIntervals, + g_subintervals(Intervals), begin AllInts = Intervals ++ SubIntervals, Ring = mem3_util:get_ring(AllInts, 1, 100), @@ -56,94 +61,98 @@ prop_get_ring_connected_with_sub_intervals() -> ) ). - prop_get_ring_with_disconnected_intervals() -> - ?FORALL({Start, End}, oneof(ranges()), - ?FORALL(Intervals, g_disconnected_intervals(Start, End), + ?FORALL( + {Start, End}, + oneof(ranges()), + ?FORALL( + Intervals, + g_disconnected_intervals(Start, End), mem3_util:get_ring(Intervals, Start, End) =:= [] ) ). - % Generators ranges() -> [{1, 10}, {0, 2 bsl 31 - 1}, {2 bsl 31 - 10, 2 bsl 31 - 1}]. - g_connected_intervals(Begin, End) -> ?SIZED(Size, g_connected_intervals(Begin, End, 5 * Size)). - g_connected_intervals(Begin, End, Split) when Begin =< End -> - ?LET(N, choose(0, Split), - begin - if - N == 0 -> - [{Begin, End}]; - N > 0 -> - Ns = lists:seq(1, N - 1), - Bs = lists:usort([rand_range(Begin, End) || _ <- Ns]), - Es = [B - 1 || B <- Bs], - shuffle(lists:zip([Begin] ++ Bs, Es ++ [End])) + ?LET( + N, + choose(0, Split), + begin + if + N == 0 -> + [{Begin, End}]; + N > 0 -> + Ns = lists:seq(1, N - 1), + Bs = lists:usort([rand_range(Begin, End) || _ <- Ns]), + Es = [B - 1 || B <- Bs], + shuffle(lists:zip([Begin] ++ Bs, Es ++ [End])) + end end - end). - + ). g_non_trivial_connected_intervals(Begin, End, Split) -> - ?SUCHTHAT(Connected, g_connected_intervals(Begin, End, Split), - length(Connected) > 1). - + ?SUCHTHAT( + Connected, + g_connected_intervals(Begin, End, Split), + length(Connected) > 1 + ). g_disconnected_intervals(Begin, End) -> ?SIZED(Size, g_disconnected_intervals(Begin, End, Size)). - g_disconnected_intervals(Begin, End, Split) when Begin =< End -> - ?LET(Connected, g_non_trivial_connected_intervals(Begin, End, Split), - begin - I = rand:uniform(length(Connected)) - 1, - {Before, [_ | After]} = lists:split(I, Connected), - Before ++ After - end). - + ?LET( + Connected, + g_non_trivial_connected_intervals(Begin, End, Split), + begin + I = rand:uniform(length(Connected)) - 1, + {Before, [_ | After]} = lists:split(I, Connected), + Before ++ After + end + ). g_subintervals(Intervals) -> lists:foldl(fun(R, Acc) -> split_interval(R) ++ Acc end, [], Intervals). - split_interval({B, E}) when E - B >= 2 -> - E1 = rand_range(B, E) - 1, - B1 = E1 + 1, - [{B, E1}, {B1, E}]; - + E1 = rand_range(B, E) - 1, + B1 = E1 + 1, + [{B, E1}, {B1, E}]; split_interval(_Range) -> []. - g_random_intervals(Start, End) -> - ?LET(N, choose(1, 10), - begin - [begin - B = rand_range(Start, End), - E = rand_range(B, End), - {B, E} - end || _ <- lists:seq(1, N)] - end). - + ?LET( + N, + choose(1, 10), + begin + [ + begin + B = rand_range(Start, End), + E = rand_range(B, End), + {B, E} + end + || _ <- lists:seq(1, N) + ] + end + ). rand_range(B, B) -> B; - rand_range(B, E) -> B + rand:uniform(E - B). - shuffle(L) -> Tagged = [{rand:uniform(), X} || X <- L], [X || {_, X} <- lists:sort(Tagged)]. - endpoints(Ranges) -> {Begins, Ends} = lists:unzip(Ranges), sets:from_list(Begins ++ Ends). diff --git a/src/mem3/test/eunit/mem3_seeds_test.erl b/src/mem3/test/eunit/mem3_seeds_test.erl index 1c1e2fed6..2c9c1d383 100644 --- a/src/mem3/test/eunit/mem3_seeds_test.erl +++ b/src/mem3/test/eunit/mem3_seeds_test.erl @@ -61,8 +61,10 @@ check_nodelist() -> check_local_dbs() -> LocalDbs = mem3_sync:local_dbs(), {ok, _} = couch_server:create(<<"_users">>, []), - ?assertEqual(lists:append(LocalDbs, [<<"_users">>]), - mem3_sync:local_dbs()). + ?assertEqual( + lists:append(LocalDbs, [<<"_users">>]), + mem3_sync:local_dbs() + ). cleanup() -> application:stop(mem3), diff --git a/src/mem3/test/eunit/mem3_shards_test.erl b/src/mem3/test/eunit/mem3_shards_test.erl index 9c9bbb402..6d2766fa2 100644 --- a/src/mem3/test/eunit/mem3_shards_test.erl +++ b/src/mem3/test/eunit/mem3_shards_test.erl @@ -12,11 +12,11 @@ -module(mem3_shards_test). - -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). -include_lib("mem3/src/mem3_reshard.hrl"). --include_lib("couch_mrview/include/couch_mrview.hrl"). % for all_docs function +% for all_docs function +-include_lib("couch_mrview/include/couch_mrview.hrl"). -define(ID, <<"_id">>). -define(TIMEOUT, 60). @@ -28,28 +28,26 @@ setup() -> {ok, DbDoc} = mem3_util:open_db_doc(DbName), #{dbname => DbName, dbdoc => DbDoc}. - teardown(#{dbname := DbName}) -> delete_db(DbName). - start_couch() -> test_util:start_couch(?CONFIG_CHAIN, [mem3, fabric]). - stop_couch(Ctx) -> test_util:stop_couch(Ctx). - mem3_shards_db_create_props_test_() -> { "mem3 shards partition query database properties tests", { setup, - fun start_couch/0, fun stop_couch/1, + fun start_couch/0, + fun stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun partitioned_shards_recreated_properly/1 ] @@ -57,57 +55,61 @@ mem3_shards_db_create_props_test_() -> } }. - % This asserts that when the mem3_shards's changes listener on the shards db % encounters a db doc update for a db that has a missing shard on the local % instance, the shard creation logic will properly propagate the db's config % properties. % SEE: apache/couchdb#3631 partitioned_shards_recreated_properly(#{dbname := DbName, dbdoc := DbDoc}) -> - {timeout, ?TIMEOUT, ?_test(begin - #doc{body = {Body0}} = DbDoc, - Body1 = [{<<"foo">>, <<"bar">>} | Body0], - Shards = [Shard|_] = lists:sort(mem3:shards(DbName)), - ShardName = Shard#shard.name, - ?assert(is_partitioned(Shards)), - ok = with_proc(fun() -> couch_server:delete(ShardName, []) end), - ?assertThrow({not_found, no_db_file}, is_partitioned(Shard)), - ok = mem3_util:update_db_doc(DbDoc#doc{body = {Body1}}), - Shards = [Shard|_] = test_util:wait_value(fun() -> - lists:sort(mem3:shards(DbName)) - end, Shards), - ?assertEqual(true, test_util:wait_value(fun() -> - catch is_partitioned(Shard) - end, true)) - end)}. - - -is_partitioned([#shard{}|_]=Shards) -> + {timeout, ?TIMEOUT, + ?_test(begin + #doc{body = {Body0}} = DbDoc, + Body1 = [{<<"foo">>, <<"bar">>} | Body0], + Shards = [Shard | _] = lists:sort(mem3:shards(DbName)), + ShardName = Shard#shard.name, + ?assert(is_partitioned(Shards)), + ok = with_proc(fun() -> couch_server:delete(ShardName, []) end), + ?assertThrow({not_found, no_db_file}, is_partitioned(Shard)), + ok = mem3_util:update_db_doc(DbDoc#doc{body = {Body1}}), + Shards = + [Shard | _] = test_util:wait_value( + fun() -> + lists:sort(mem3:shards(DbName)) + end, + Shards + ), + ?assertEqual( + true, + test_util:wait_value( + fun() -> + catch is_partitioned(Shard) + end, + true + ) + ) + end)}. + +is_partitioned([#shard{} | _] = Shards) -> lists:all(fun is_partitioned/1, Shards); -is_partitioned(#shard{name=Name}) -> +is_partitioned(#shard{name = Name}) -> couch_util:with_db(Name, fun couch_db:is_partitioned/1); is_partitioned(Db) -> couch_db:is_partitioned(Db). - create_db(DbName, Opts) -> GL = erlang:group_leader(), with_proc(fun() -> fabric:create_db(DbName, Opts) end, GL). - delete_db(DbName) -> GL = erlang:group_leader(), with_proc(fun() -> fabric:delete_db(DbName, [?ADMIN_CTX]) end, GL). - with_proc(Fun) -> with_proc(Fun, undefined, 30000). - with_proc(Fun, GroupLeader) -> with_proc(Fun, GroupLeader, 30000). - with_proc(Fun, GroupLeader, Timeout) -> {Pid, Ref} = spawn_monitor(fun() -> case GroupLeader of @@ -125,5 +127,4 @@ with_proc(Fun, GroupLeader, Timeout) -> erlang:demonitor(Ref, [flush]), exit(Pid, kill), error({with_proc_timeout, Fun, Timeout}) - end. - + end. diff --git a/src/mem3/test/eunit/mem3_sync_security_test.erl b/src/mem3/test/eunit/mem3_sync_security_test.erl index e67a72017..7f4b7b699 100644 --- a/src/mem3/test/eunit/mem3_sync_security_test.erl +++ b/src/mem3/test/eunit/mem3_sync_security_test.erl @@ -17,17 +17,20 @@ -include("mem3.hrl"). -include_lib("eunit/include/eunit.hrl"). --define(TIMEOUT, 5). % seconds +% seconds +-define(TIMEOUT, 5). go_test_() -> { "security property sync test", { setup, - fun start_couch/0, fun stop_couch/1, + fun start_couch/0, + fun stop_couch/1, { foreach, - fun setup/0, fun teardown/1, + fun setup/0, + fun teardown/1, [ fun sync_security_ok/1 ] diff --git a/src/mem3/test/eunit/mem3_util_test.erl b/src/mem3/test/eunit/mem3_util_test.erl index a8103bfd0..9cab89f67 100644 --- a/src/mem3/test/eunit/mem3_util_test.erl +++ b/src/mem3/test/eunit/mem3_util_test.erl @@ -19,19 +19,20 @@ name_shard_test() -> Shard1 = #shard{}, ?assertError(function_clause, mem3_util:name_shard(Shard1, ".1234")), - Shard2 = #shard{dbname = <<"testdb">>, range = [0,100]}, - #shard{name=Name2} = mem3_util:name_shard(Shard2, ".1234"), + Shard2 = #shard{dbname = <<"testdb">>, range = [0, 100]}, + #shard{name = Name2} = mem3_util:name_shard(Shard2, ".1234"), ?assertEqual(<<"shards/00000000-00000064/testdb.1234">>, Name2), ok. create_partition_map_test() -> - {DbName1, N1, Q1, Nodes1} = {<<"testdb1">>, 3, 4, [a,b,c,d]}, + {DbName1, N1, Q1, Nodes1} = {<<"testdb1">>, 3, 4, [a, b, c, d]}, Map1 = mem3_util:create_partition_map(DbName1, N1, Q1, Nodes1), ?assertEqual(12, length(Map1)), - {DbName2, N2, Q2, Nodes2} = {<<"testdb2">>, 1, 1, [a,b,c,d]}, - [#shard{name=Name2,node=Node2}] = Map2 = + {DbName2, N2, Q2, Nodes2} = {<<"testdb2">>, 1, 1, [a, b, c, d]}, + [#shard{name = Name2, node = Node2}] = + Map2 = mem3_util:create_partition_map(DbName2, N2, Q2, Nodes2, ".1234"), ?assertEqual(1, length(Map2)), ?assertEqual(<<"shards/00000000-ffffffff/testdb2.1234">>, Name2), @@ -40,76 +41,97 @@ create_partition_map_test() -> build_shards_test() -> DocProps1 = - [{<<"changelog">>, - [[<<"add">>,<<"00000000-1fffffff">>, - <<"bigcouch@node.local">>], - [<<"add">>,<<"20000000-3fffffff">>, - <<"bigcouch@node.local">>], - [<<"add">>,<<"40000000-5fffffff">>, - <<"bigcouch@node.local">>], - [<<"add">>,<<"60000000-7fffffff">>, - <<"bigcouch@node.local">>], - [<<"add">>,<<"80000000-9fffffff">>, - <<"bigcouch@node.local">>], - [<<"add">>,<<"a0000000-bfffffff">>, - <<"bigcouch@node.local">>], - [<<"add">>,<<"c0000000-dfffffff">>, - <<"bigcouch@node.local">>], - [<<"add">>,<<"e0000000-ffffffff">>, - <<"bigcouch@node.local">>]]}, - {<<"by_node">>, - {[{<<"bigcouch@node.local">>, - [<<"00000000-1fffffff">>,<<"20000000-3fffffff">>, - <<"40000000-5fffffff">>,<<"60000000-7fffffff">>, - <<"80000000-9fffffff">>,<<"a0000000-bfffffff">>, - <<"c0000000-dfffffff">>,<<"e0000000-ffffffff">>]}]}}, - {<<"by_range">>, - {[{<<"00000000-1fffffff">>,[<<"bigcouch@node.local">>]}, - {<<"20000000-3fffffff">>,[<<"bigcouch@node.local">>]}, - {<<"40000000-5fffffff">>,[<<"bigcouch@node.local">>]}, - {<<"60000000-7fffffff">>,[<<"bigcouch@node.local">>]}, - {<<"80000000-9fffffff">>,[<<"bigcouch@node.local">>]}, - {<<"a0000000-bfffffff">>,[<<"bigcouch@node.local">>]}, - {<<"c0000000-dfffffff">>,[<<"bigcouch@node.local">>]}, - {<<"e0000000-ffffffff">>,[<<"bigcouch@node.local">>]}]}}], + [ + {<<"changelog">>, [ + [ + <<"add">>, + <<"00000000-1fffffff">>, + <<"bigcouch@node.local">> + ], + [ + <<"add">>, + <<"20000000-3fffffff">>, + <<"bigcouch@node.local">> + ], + [ + <<"add">>, + <<"40000000-5fffffff">>, + <<"bigcouch@node.local">> + ], + [ + <<"add">>, + <<"60000000-7fffffff">>, + <<"bigcouch@node.local">> + ], + [ + <<"add">>, + <<"80000000-9fffffff">>, + <<"bigcouch@node.local">> + ], + [ + <<"add">>, + <<"a0000000-bfffffff">>, + <<"bigcouch@node.local">> + ], + [ + <<"add">>, + <<"c0000000-dfffffff">>, + <<"bigcouch@node.local">> + ], + [ + <<"add">>, + <<"e0000000-ffffffff">>, + <<"bigcouch@node.local">> + ] + ]}, + {<<"by_node">>, + {[ + {<<"bigcouch@node.local">>, [ + <<"00000000-1fffffff">>, + <<"20000000-3fffffff">>, + <<"40000000-5fffffff">>, + <<"60000000-7fffffff">>, + <<"80000000-9fffffff">>, + <<"a0000000-bfffffff">>, + <<"c0000000-dfffffff">>, + <<"e0000000-ffffffff">> + ]} + ]}}, + {<<"by_range">>, + {[ + {<<"00000000-1fffffff">>, [<<"bigcouch@node.local">>]}, + {<<"20000000-3fffffff">>, [<<"bigcouch@node.local">>]}, + {<<"40000000-5fffffff">>, [<<"bigcouch@node.local">>]}, + {<<"60000000-7fffffff">>, [<<"bigcouch@node.local">>]}, + {<<"80000000-9fffffff">>, [<<"bigcouch@node.local">>]}, + {<<"a0000000-bfffffff">>, [<<"bigcouch@node.local">>]}, + {<<"c0000000-dfffffff">>, [<<"bigcouch@node.local">>]}, + {<<"e0000000-ffffffff">>, [<<"bigcouch@node.local">>]} + ]}} + ], Shards1 = mem3_util:build_shards(<<"testdb1">>, DocProps1), ExpectedShards1 = - [{shard,<<"shards/00000000-1fffffff/testdb1">>, - 'bigcouch@node.local',<<"testdb1">>, - [0,536870911], - undefined,[]}, - {shard,<<"shards/20000000-3fffffff/testdb1">>, - 'bigcouch@node.local',<<"testdb1">>, - [536870912,1073741823], - undefined,[]}, - {shard,<<"shards/40000000-5fffffff/testdb1">>, - 'bigcouch@node.local',<<"testdb1">>, - [1073741824,1610612735], - undefined,[]}, - {shard,<<"shards/60000000-7fffffff/testdb1">>, - 'bigcouch@node.local',<<"testdb1">>, - [1610612736,2147483647], - undefined,[]}, - {shard,<<"shards/80000000-9fffffff/testdb1">>, - 'bigcouch@node.local',<<"testdb1">>, - [2147483648,2684354559], - undefined,[]}, - {shard,<<"shards/a0000000-bfffffff/testdb1">>, - 'bigcouch@node.local',<<"testdb1">>, - [2684354560,3221225471], - undefined,[]}, - {shard,<<"shards/c0000000-dfffffff/testdb1">>, - 'bigcouch@node.local',<<"testdb1">>, - [3221225472,3758096383], - undefined,[]}, - {shard,<<"shards/e0000000-ffffffff/testdb1">>, - 'bigcouch@node.local',<<"testdb1">>, - [3758096384,4294967295], - undefined,[]}], + [ + {shard, <<"shards/00000000-1fffffff/testdb1">>, 'bigcouch@node.local', <<"testdb1">>, + [0, 536870911], undefined, []}, + {shard, <<"shards/20000000-3fffffff/testdb1">>, 'bigcouch@node.local', <<"testdb1">>, + [536870912, 1073741823], undefined, []}, + {shard, <<"shards/40000000-5fffffff/testdb1">>, 'bigcouch@node.local', <<"testdb1">>, + [1073741824, 1610612735], undefined, []}, + {shard, <<"shards/60000000-7fffffff/testdb1">>, 'bigcouch@node.local', <<"testdb1">>, + [1610612736, 2147483647], undefined, []}, + {shard, <<"shards/80000000-9fffffff/testdb1">>, 'bigcouch@node.local', <<"testdb1">>, + [2147483648, 2684354559], undefined, []}, + {shard, <<"shards/a0000000-bfffffff/testdb1">>, 'bigcouch@node.local', <<"testdb1">>, + [2684354560, 3221225471], undefined, []}, + {shard, <<"shards/c0000000-dfffffff/testdb1">>, 'bigcouch@node.local', <<"testdb1">>, + [3221225472, 3758096383], undefined, []}, + {shard, <<"shards/e0000000-ffffffff/testdb1">>, 'bigcouch@node.local', <<"testdb1">>, + [3758096384, 4294967295], undefined, []} + ], ?assertEqual(ExpectedShards1, Shards1), ok. - %% n_val tests nval_test_() -> diff --git a/src/rexi/src/rexi.erl b/src/rexi/src/rexi.erl index 170503b7c..fb1763e51 100644 --- a/src/rexi/src/rexi.erl +++ b/src/rexi/src/rexi.erl @@ -30,8 +30,8 @@ stop() -> application:stop(rexi). restart() -> - stop(), start(). - + stop(), + start(). %% @equiv cast(Node, self(), MFA) -spec cast(node(), {atom(), atom(), list()}) -> reference(). @@ -86,22 +86,34 @@ kill_all(NodeRefs) when is_list(NodeRefs) -> %% configure the cluster to send kill_all messages. case config:get_boolean("rexi", "use_kill_all", false) of true -> - PerNodeMap = lists:foldl(fun({Node, Ref}, Acc) -> - maps:update_with(Node, fun(Refs) -> - [Ref | Refs] - end, [Ref], Acc) - end, #{}, NodeRefs), - maps:map(fun(Node, Refs) -> - ServerPid = rexi_utils:server_pid(Node), - rexi_utils:send(ServerPid, cast_msg({kill_all, Refs})) - end, PerNodeMap); + PerNodeMap = lists:foldl( + fun({Node, Ref}, Acc) -> + maps:update_with( + Node, + fun(Refs) -> + [Ref | Refs] + end, + [Ref], + Acc + ) + end, + #{}, + NodeRefs + ), + maps:map( + fun(Node, Refs) -> + ServerPid = rexi_utils:server_pid(Node), + rexi_utils:send(ServerPid, cast_msg({kill_all, Refs})) + end, + PerNodeMap + ); false -> lists:foreach(fun({Node, Ref}) -> kill(Node, Ref) end, NodeRefs) end, ok. %% @equiv async_server_call(Server, self(), Request) --spec async_server_call(pid() | {atom(),node()}, any()) -> reference(). +-spec async_server_call(pid() | {atom(), node()}, any()) -> reference(). async_server_call(Server, Request) -> async_server_call(Server, self(), Request). @@ -110,17 +122,17 @@ async_server_call(Server, Request) -> %% function acts more like cast() than call() in that the server process %% is not monitored. Clients who want to know if the server is alive should %% monitor it themselves before calling this function. --spec async_server_call(pid() | {atom(),node()}, pid(), any()) -> reference(). +-spec async_server_call(pid() | {atom(), node()}, pid(), any()) -> reference(). async_server_call(Server, Caller, Request) -> Ref = make_ref(), - rexi_utils:send(Server, {'$gen_call', {Caller,Ref}, Request}), + rexi_utils:send(Server, {'$gen_call', {Caller, Ref}, Request}), Ref. %% @doc convenience function to reply to the original rexi Caller. -spec reply(any()) -> any(). reply(Reply) -> {Caller, Ref} = get(rexi_from), - erlang:send(Caller, {Ref,Reply}). + erlang:send(Caller, {Ref, Reply}). %% @equiv sync_reply(Reply, 300000) sync_reply(Reply) -> @@ -133,9 +145,10 @@ sync_reply(Reply) -> sync_reply(Reply, Timeout) -> {Caller, Ref} = get(rexi_from), Tag = make_ref(), - erlang:send(Caller, {Ref, {self(),Tag}, Reply}), - receive {Tag, Response} -> - Response + erlang:send(Caller, {Ref, {self(), Tag}, Reply}), + receive + {Tag, Response} -> + Response after Timeout -> timeout end. @@ -174,7 +187,7 @@ stream_init(Timeout) -> %% sending messages. The `From` should be the value provided by %% the worker in the rexi_STREAM_INIT message. -spec stream_start({pid(), any()}) -> ok. -stream_start({Pid, _Tag}=From) when is_pid(Pid) -> +stream_start({Pid, _Tag} = From) when is_pid(Pid) -> gen_server:reply(From, rexi_STREAM_START). %% @doc Cancel a worker stream @@ -184,7 +197,7 @@ stream_start({Pid, _Tag}=From) when is_pid(Pid) -> %% The `From` should be the value provided by the worker in the %% rexi_STREAM_INIT message. -spec stream_cancel({pid(), any()}) -> ok. -stream_cancel({Pid, _Tag}=From) when is_pid(Pid) -> +stream_cancel({Pid, _Tag} = From) when is_pid(Pid) -> gen_server:reply(From, rexi_STREAM_CANCEL). %% @equiv stream(Msg, 100, 300000) @@ -202,13 +215,14 @@ stream(Msg, Limit) -> stream(Msg, Limit, Timeout) -> try maybe_wait(Limit, Timeout) of {ok, Count} -> - put(rexi_unacked, Count+1), + put(rexi_unacked, Count + 1), {Caller, Ref} = get(rexi_from), erlang:send(Caller, {Ref, self(), Msg}), ok - catch throw:timeout -> - couch_stats:increment_counter([rexi, streams, timeout, stream]), - exit(timeout) + catch + throw:timeout -> + couch_stats:increment_counter([rexi, streams, timeout, stream]), + exit(timeout) end. %% @equiv stream2(Msg, 5, 300000) @@ -230,13 +244,14 @@ stream2(Msg, Limit, Timeout) -> maybe_init_stream(Timeout), try maybe_wait(Limit, Timeout) of {ok, Count} -> - put(rexi_unacked, Count+1), + put(rexi_unacked, Count + 1), {Caller, Ref} = get(rexi_from), erlang:send(Caller, {Ref, self(), Msg}), ok - catch throw:timeout -> - couch_stats:increment_counter([rexi, streams, timeout, stream]), - exit(timeout) + catch + throw:timeout -> + couch_stats:increment_counter([rexi, streams, timeout, stream]), + exit(timeout) end. %% @equiv stream_last(Msg, 300000) @@ -259,14 +274,12 @@ stream_ack(Client) -> stream_ack(Client, N) -> erlang:send(Client, {rexi_ack, N}). - %% Sends a ping message to the coordinator. This is for long running %% operations on a node that could exceed the rexi timeout -ping() -> +ping() -> {Caller, _} = get(rexi_from), erlang:send(Caller, {rexi, '$rexi_ping'}). - %% internal functions %% cast_msg(Msg) -> {'$gen_cast', Msg}. @@ -304,7 +317,7 @@ maybe_wait(Limit, Timeout) -> wait_for_ack(Count, Timeout) -> receive - {rexi_ack, N} -> drain_acks(Count-N) + {rexi_ack, N} -> drain_acks(Count - N) after Timeout -> couch_stats:increment_counter([rexi, streams, timeout, wait_for_ack]), throw(timeout) @@ -314,7 +327,7 @@ drain_acks(Count) when Count < 0 -> erlang:error(mismatched_rexi_ack); drain_acks(Count) -> receive - {rexi_ack, N} -> drain_acks(Count-N) + {rexi_ack, N} -> drain_acks(Count - N) after 0 -> {ok, Count} end. diff --git a/src/rexi/src/rexi_app.erl b/src/rexi/src/rexi_app.erl index 0f1e892b5..61e7886e1 100644 --- a/src/rexi/src/rexi_app.erl +++ b/src/rexi/src/rexi_app.erl @@ -14,7 +14,6 @@ -behaviour(application). -export([start/2, stop/1]). - start(_Type, StartArgs) -> rexi_sup:start_link(StartArgs). diff --git a/src/rexi/src/rexi_buffer.erl b/src/rexi/src/rexi_buffer.erl index d16dc8ba3..7f0079f03 100644 --- a/src/rexi/src/rexi_buffer.erl +++ b/src/rexi/src/rexi_buffer.erl @@ -15,10 +15,16 @@ -vsn(1). % gen_server callbacks --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3 +]). --export ([ +-export([ send/2, start_link/1 ]). @@ -37,7 +43,6 @@ send(Dest, Msg) -> Server = list_to_atom(lists:concat([rexi_buffer, "_", get_node(Dest)])), gen_server:cast(Server, {deliver, Dest, Msg}). - init(_) -> %% TODO Leverage os_mon to discover available memory in the system Max = list_to_integer(config:get("rexi", "buffer_count", "2000")), @@ -45,7 +50,6 @@ init(_) -> handle_call(erase_buffer, _From, State) -> {reply, ok, State#state{buffer = queue:new(), count = 0}, 0}; - handle_call(get_buffered_count, _From, State) -> {reply, State#state.count, State, 0}. @@ -53,19 +57,19 @@ handle_cast({deliver, Dest, Msg}, #state{buffer = Q, count = C} = State) -> couch_stats:increment_counter([rexi, buffered]), Q2 = queue:in({Dest, Msg}, Q), case should_drop(State) of - true -> - couch_stats:increment_counter([rexi, dropped]), + true -> + couch_stats:increment_counter([rexi, dropped]), {noreply, State#state{buffer = queue:drop(Q2)}, 0}; - false -> - {noreply, State#state{buffer = Q2, count = C+1}, 0} + false -> + {noreply, State#state{buffer = Q2, count = C + 1}, 0} end. -handle_info(timeout, #state{sender = nil, buffer = {[],[]}, count = 0}=State) -> +handle_info(timeout, #state{sender = nil, buffer = {[], []}, count = 0} = State) -> {noreply, State}; handle_info(timeout, #state{sender = nil, count = C} = State) when C > 0 -> #state{buffer = Q, count = C} = State, {{value, {Dest, Msg}}, Q2} = queue:out_r(Q), - NewState = State#state{buffer = Q2, count = C-1}, + NewState = State#state{buffer = Q2, count = C - 1}, case erlang:send(Dest, Msg, [noconnect, nosuspend]) of ok when C =:= 1 -> % We just sent the last queued messsage, we'll use this opportunity @@ -82,7 +86,6 @@ handle_info(timeout, #state{sender = nil, count = C} = State) when C > 0 -> handle_info(timeout, State) -> % Waiting on a sender to return {noreply, State}; - handle_info({'DOWN', Ref, _, Pid, _}, #state{sender = {Pid, Ref}} = State) -> {noreply, State#state{sender = nil}, 0}. @@ -91,7 +94,7 @@ terminate(_Reason, _State) -> code_change(_OldVsn, {state, Buffer, Sender, Count}, _Extra) -> Max = list_to_integer(config:get("rexi", "buffer_count", "2000")), - {ok, #state{buffer=Buffer, sender=Sender, count=Count, max_count=Max}}; + {ok, #state{buffer = Buffer, sender = Sender, count = Count, max_count = Max}}; code_change(_OldVsn, State, _Extra) -> {ok, State}. diff --git a/src/rexi/src/rexi_monitor.erl b/src/rexi/src/rexi_monitor.erl index f90ec5160..7fe66db71 100644 --- a/src/rexi/src/rexi_monitor.erl +++ b/src/rexi/src/rexi_monitor.erl @@ -14,16 +14,17 @@ -export([start/1, stop/1]). -export([wait_monitors/1]). - %% @doc spawn_links a process which monitors the supplied list of items and %% returns the process ID. If a monitored process exits, the caller will %% receive a {rexi_DOWN, MonitoringPid, DeadPid, Reason} message. --spec start([pid() | atom() | {atom(),node()}]) -> pid(). +-spec start([pid() | atom() | {atom(), node()}]) -> pid(). start(Procs) -> Parent = self(), Nodes = [node() | nodes()], - {Mon, Skip} = lists:partition(fun(P) -> should_monitor(P, Nodes) end, - Procs), + {Mon, Skip} = lists:partition( + fun(P) -> should_monitor(P, Nodes) end, + Procs + ), spawn_link(fun() -> [notify_parent(Parent, P, noconnect) || P <- Skip], [erlang:monitor(process, P) || P <- Mon], @@ -50,16 +51,17 @@ should_monitor({_, Node}, Nodes) -> wait_monitors(Parent) -> receive - {'DOWN', _, process, Pid, Reason} -> - notify_parent(Parent, Pid, Reason), - ?MODULE:wait_monitors(Parent); - {Parent, shutdown} -> - ok + {'DOWN', _, process, Pid, Reason} -> + notify_parent(Parent, Pid, Reason), + ?MODULE:wait_monitors(Parent); + {Parent, shutdown} -> + ok end. flush_down_messages() -> - receive {rexi_DOWN, _, _, _} -> - flush_down_messages() + receive + {rexi_DOWN, _, _, _} -> + flush_down_messages() after 0 -> ok end. diff --git a/src/rexi/src/rexi_server.erl b/src/rexi/src/rexi_server.erl index 4cd9ce66e..47c128d7b 100644 --- a/src/rexi/src/rexi_server.erl +++ b/src/rexi/src/rexi_server.erl @@ -13,8 +13,14 @@ -module(rexi_server). -behaviour(gen_server). -vsn(1). --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3]). +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3 +]). -export([start_link/1, init_p/2, init_p/3]). @@ -24,10 +30,10 @@ -include_lib("rexi/include/rexi.hrl"). -record(job, { - client::reference(), - worker::reference(), - client_pid::pid(), - worker_pid::pid() + client :: reference(), + worker :: reference(), + client_pid :: pid(), + worker_pid :: pid() }). -record(st, { @@ -47,30 +53,27 @@ init([]) -> handle_call(get_errors, _From, #st{errors = Errors} = St) -> {reply, {ok, lists:reverse(queue:to_list(Errors))}, St}; - handle_call(get_last_error, _From, #st{errors = Errors} = St) -> try {reply, {ok, queue:get_r(Errors)}, St} - catch error:empty -> - {reply, {error, empty}, St} + catch + error:empty -> + {reply, {error, empty}, St} end; - -handle_call({set_error_limit, N}, _From, #st{error_count=Len, errors=Q} = St) -> - if N < Len -> - {NewQ, _} = queue:split(N, Q); - true -> - NewQ = Q +handle_call({set_error_limit, N}, _From, #st{error_count = Len, errors = Q} = St) -> + if + N < Len -> + {NewQ, _} = queue:split(N, Q); + true -> + NewQ = Q end, NewLen = queue:len(NewQ), - {reply, ok, St#st{error_limit=N, error_count=NewLen, errors=NewQ}}; - + {reply, ok, St#st{error_limit = N, error_count = NewLen, errors = NewQ}}; handle_call(_Request, _From, St) -> {reply, ignored, St}. - handle_cast({doit, From, MFA}, St) -> handle_cast({doit, From, undefined, MFA}, St); - handle_cast({doit, {ClientPid, ClientRef} = From, Nonce, MFA}, State) -> {LocalPid, Ref} = spawn_monitor(?MODULE, init_p, [From, MFA, Nonce]), Job = #job{ @@ -80,60 +83,61 @@ handle_cast({doit, {ClientPid, ClientRef} = From, Nonce, MFA}, State) -> worker_pid = LocalPid }, {noreply, add_job(Job, State)}; - - handle_cast({kill, FromRef}, St) -> kill_worker(FromRef, St), {noreply, St}; - handle_cast({kill_all, FromRefs}, St) -> lists:foreach(fun(FromRef) -> kill_worker(FromRef, St) end, FromRefs), {noreply, St}; - handle_cast(_, St) -> couch_log:notice("rexi_server ignored_cast", []), {noreply, St}. -handle_info({'DOWN', Ref, process, _, normal}, #st{workers=Workers} = St) -> +handle_info({'DOWN', Ref, process, _, normal}, #st{workers = Workers} = St) -> case find_worker(Ref, Workers) of - #job{} = Job -> - {noreply, remove_job(Job, St)}; - false -> - {noreply, St} + #job{} = Job -> + {noreply, remove_job(Job, St)}; + false -> + {noreply, St} end; - -handle_info({'DOWN', Ref, process, Pid, Error}, #st{workers=Workers} = St) -> +handle_info({'DOWN', Ref, process, Pid, Error}, #st{workers = Workers} = St) -> case find_worker(Ref, Workers) of - #job{worker_pid=Pid, worker=Ref, client_pid=CPid, client=CRef} =Job -> - case Error of #error{reason = {_Class, Reason}, stack = Stack} -> - notify_caller({CPid, CRef}, {Reason, Stack}), - St1 = save_error(Error, St), - {noreply, remove_job(Job, St1)}; - _ -> - notify_caller({CPid, CRef}, Error), - {noreply, remove_job(Job, St)} - end; - false -> - {noreply, St} + #job{worker_pid = Pid, worker = Ref, client_pid = CPid, client = CRef} = Job -> + case Error of + #error{reason = {_Class, Reason}, stack = Stack} -> + notify_caller({CPid, CRef}, {Reason, Stack}), + St1 = save_error(Error, St), + {noreply, remove_job(Job, St1)}; + _ -> + notify_caller({CPid, CRef}, Error), + {noreply, remove_job(Job, St)} + end; + false -> + {noreply, St} end; - handle_info(_Info, St) -> {noreply, St}. terminate(_Reason, St) -> - ets:foldl(fun(#job{worker_pid=Pid},_) -> exit(Pid,kill) end, nil, - St#st.workers), + ets:foldl( + fun(#job{worker_pid = Pid}, _) -> exit(Pid, kill) end, + nil, + St#st.workers + ), ok. -code_change(_OldVsn, #st{}=State, _Extra) -> +code_change(_OldVsn, #st{} = State, _Extra) -> {ok, State}. init_p(From, MFA) -> init_p(From, MFA, undefined). %% @doc initializes a process started by rexi_server. --spec init_p({pid(), reference()}, {atom(), atom(), list()}, - string() | undefined) -> any(). +-spec init_p( + {pid(), reference()}, + {atom(), atom(), list()}, + string() | undefined +) -> any(). init_p(From, {M,F,A}, Nonce) -> put(rexi_from, From), put('$initial_call', {M,F,length(A)}), @@ -158,13 +162,19 @@ init_p(From, {M,F,A}, Nonce) -> save_error(_E, #st{error_limit = 0} = St) -> St; -save_error(E, #st{errors=Q, error_limit=L, error_count=C} = St) when C >= L -> +save_error(E, #st{errors = Q, error_limit = L, error_count = C} = St) when C >= L -> St#st{errors = queue:in(E, queue:drop(Q))}; -save_error(E, #st{errors=Q, error_count=C} = St) -> - St#st{errors = queue:in(E, Q), error_count = C+1}. +save_error(E, #st{errors = Q, error_count = C} = St) -> + St#st{errors = queue:in(E, Q), error_count = C + 1}. clean_stack(S) -> - lists:map(fun({M,F,A}) when is_list(A) -> {M,F,length(A)}; (X) -> X end, S). + lists:map( + fun + ({M, F, A}) when is_list(A) -> {M, F, length(A)}; + (X) -> X + end, + S + ). add_job(Job, #st{workers = Workers, clients = Clients} = State) -> ets:insert(Workers, Job), @@ -177,19 +187,21 @@ remove_job(Job, #st{workers = Workers, clients = Clients} = State) -> State. find_worker(Ref, Tab) -> - case ets:lookup(Tab, Ref) of [] -> false; [Worker] -> Worker end. + case ets:lookup(Tab, Ref) of + [] -> false; + [Worker] -> Worker + end. notify_caller({Caller, Ref}, Reason) -> rexi_utils:send(Caller, {Ref, {rexi_EXIT, Reason}}). - kill_worker(FromRef, #st{clients = Clients} = St) -> case find_worker(FromRef, Clients) of - #job{worker = KeyRef, worker_pid = Pid} = Job -> - erlang:demonitor(KeyRef), - exit(Pid, kill), - remove_job(Job, St), - ok; - false -> - ok + #job{worker = KeyRef, worker_pid = Pid} = Job -> + erlang:demonitor(KeyRef), + exit(Pid, kill), + remove_job(Job, St), + ok; + false -> + ok end. diff --git a/src/rexi/src/rexi_server_mon.erl b/src/rexi/src/rexi_server_mon.erl index cfe1144ce..9057807e6 100644 --- a/src/rexi/src/rexi_server_mon.erl +++ b/src/rexi/src/rexi_server_mon.erl @@ -1,5 +1,5 @@ % Copyright 2010-2013 Cloudant -% +% % Licensed under the Apache License, Version 2.0 (the "License"); you may not % use this file except in compliance with the License. You may obtain a copy of % the License at @@ -17,13 +17,11 @@ -behaviour(mem3_cluster). -vsn(1). - -export([ start_link/1, status/0 ]). - -export([ init/1, terminate/2, @@ -34,23 +32,19 @@ ]). -export([ - cluster_stable/1, - cluster_unstable/1 + cluster_stable/1, + cluster_unstable/1 ]). - -define(CLUSTER_STABILITY_PERIOD_SEC, 15). - start_link(ChildMod) -> Name = list_to_atom(lists:concat([ChildMod, "_mon"])), gen_server:start_link({local, Name}, ?MODULE, ChildMod, []). - status() -> gen_server:call(?MODULE, status). - % Mem3 cluster callbacks cluster_unstable(Server) -> @@ -62,21 +56,22 @@ cluster_stable(Server) -> gen_server:cast(Server, cluster_stable), Server. - % gen_server callbacks init(ChildMod) -> - {ok, _Mem3Cluster} = mem3_cluster:start_link(?MODULE, self(), - ?CLUSTER_STABILITY_PERIOD_SEC, ?CLUSTER_STABILITY_PERIOD_SEC), + {ok, _Mem3Cluster} = mem3_cluster:start_link( + ?MODULE, + self(), + ?CLUSTER_STABILITY_PERIOD_SEC, + ?CLUSTER_STABILITY_PERIOD_SEC + ), start_servers(ChildMod), couch_log:notice("~s : started servers", [ChildMod]), {ok, ChildMod}. - terminate(_Reason, _St) -> ok. - handle_call(status, _From, ChildMod) -> case missing_servers(ChildMod) of [] -> @@ -84,7 +79,6 @@ handle_call(status, _From, ChildMod) -> Missing -> {reply, {waiting, length(Missing)}, ChildMod} end; - handle_call(Msg, _From, St) -> couch_log:notice("~s ignored_call ~w", [?MODULE, Msg]), {reply, ignored, St}. @@ -96,7 +90,6 @@ handle_cast(cluster_unstable, ChildMod) -> couch_log:notice("~s : cluster unstable", [ChildMod]), start_servers(ChildMod), {noreply, ChildMod}; - % When cluster is stable, start any servers for new nodes and stop servers for % the ones that disconnected. handle_cast(cluster_stable, ChildMod) -> @@ -104,51 +97,48 @@ handle_cast(cluster_stable, ChildMod) -> start_servers(ChildMod), stop_servers(ChildMod), {noreply, ChildMod}; - handle_cast(Msg, St) -> couch_log:notice("~s ignored_cast ~w", [?MODULE, Msg]), {noreply, St}. - handle_info(Msg, St) -> couch_log:notice("~s ignored_info ~w", [?MODULE, Msg]), {noreply, St}. - code_change(_OldVsn, nil, _Extra) -> {ok, rexi_server}; code_change(_OldVsn, St, _Extra) -> {ok, St}. - start_servers(ChildMod) -> - lists:foreach(fun(Id) -> - {ok, _} = start_server(ChildMod, Id) - end, missing_servers(ChildMod)). + lists:foreach( + fun(Id) -> + {ok, _} = start_server(ChildMod, Id) + end, + missing_servers(ChildMod) + ). stop_servers(ChildMod) -> - lists:foreach(fun(Id) -> - ok = stop_server(ChildMod, Id) - end, extra_servers(ChildMod)). - + lists:foreach( + fun(Id) -> + ok = stop_server(ChildMod, Id) + end, + extra_servers(ChildMod) + ). server_ids(ChildMod) -> Nodes = [node() | nodes()], [list_to_atom(lists:concat([ChildMod, "_", Node])) || Node <- Nodes]. - running_servers(ChildMod) -> [Id || {Id, _, _, _} <- supervisor:which_children(sup_module(ChildMod))]. - missing_servers(ChildMod) -> server_ids(ChildMod) -- running_servers(ChildMod). - extra_servers(ChildMod) -> running_servers(ChildMod) -- server_ids(ChildMod). - start_server(ChildMod, ChildId) -> ChildSpec = { ChildId, @@ -165,12 +155,10 @@ start_server(ChildMod, ChildId) -> erlang:error(Else) end. - stop_server(ChildMod, ChildId) -> SupMod = sup_module(ChildMod), ok = supervisor:terminate_child(SupMod, ChildId), ok = supervisor:delete_child(SupMod, ChildId). - sup_module(ChildMod) -> list_to_atom(lists:concat([ChildMod, "_sup"])). diff --git a/src/rexi/src/rexi_server_sup.erl b/src/rexi/src/rexi_server_sup.erl index 29c6ad60c..53497197f 100644 --- a/src/rexi/src/rexi_server_sup.erl +++ b/src/rexi/src/rexi_server_sup.erl @@ -15,15 +15,12 @@ -module(rexi_server_sup). -behaviour(supervisor). - -export([init/1]). -export([start_link/1]). - start_link(Name) -> supervisor:start_link({local, Name}, ?MODULE, []). - init([]) -> {ok, {{one_for_one, 1, 1}, []}}. diff --git a/src/rexi/src/rexi_sup.erl b/src/rexi/src/rexi_sup.erl index 3d9aa2a16..3bea0ed15 100644 --- a/src/rexi/src/rexi_sup.erl +++ b/src/rexi/src/rexi_sup.erl @@ -17,48 +17,49 @@ -export([init/1]). start_link(Args) -> - supervisor:start_link({local,?MODULE}, ?MODULE, Args). + supervisor:start_link({local, ?MODULE}, ?MODULE, Args). init([]) -> - {ok, {{rest_for_one, 3, 10}, [ - { - rexi_server, - {rexi_server, start_link, [rexi_server]}, - permanent, - 100, - worker, - [rexi_server] - }, - { - rexi_server_sup, - {rexi_server_sup, start_link, [rexi_server_sup]}, - permanent, - 100, - supervisor, - [rexi_server_sup] - }, - { - rexi_server_mon, - {rexi_server_mon, start_link, [rexi_server]}, - permanent, - 100, - worker, - [rexi_server_mon] - }, - { - rexi_buffer_sup, - {rexi_server_sup, start_link, [rexi_buffer_sup]}, - permanent, - 100, - supervisor, - [rexi_server_sup] - }, - { - rexi_buffer_mon, - {rexi_server_mon, start_link, [rexi_buffer]}, - permanent, - 100, - worker, - [rexi_server_mon] - } - ]}}. + {ok, + {{rest_for_one, 3, 10}, [ + { + rexi_server, + {rexi_server, start_link, [rexi_server]}, + permanent, + 100, + worker, + [rexi_server] + }, + { + rexi_server_sup, + {rexi_server_sup, start_link, [rexi_server_sup]}, + permanent, + 100, + supervisor, + [rexi_server_sup] + }, + { + rexi_server_mon, + {rexi_server_mon, start_link, [rexi_server]}, + permanent, + 100, + worker, + [rexi_server_mon] + }, + { + rexi_buffer_sup, + {rexi_server_sup, start_link, [rexi_buffer_sup]}, + permanent, + 100, + supervisor, + [rexi_server_sup] + }, + { + rexi_buffer_mon, + {rexi_server_mon, start_link, [rexi_buffer]}, + permanent, + 100, + worker, + [rexi_server_mon] + } + ]}}. diff --git a/src/rexi/src/rexi_utils.erl b/src/rexi/src/rexi_utils.erl index 960318418..d59c5ea0f 100644 --- a/src/rexi/src/rexi_utils.erl +++ b/src/rexi/src/rexi_utils.erl @@ -17,10 +17,10 @@ %% @doc Return a rexi_server id for the given node. server_id(Node) -> case config:get_boolean("rexi", "server_per_node", true) of - true -> - list_to_atom("rexi_server_" ++ atom_to_list(Node)); - _ -> - rexi_server + true -> + list_to_atom("rexi_server_" ++ atom_to_list(Node)); + _ -> + rexi_server end. %% @doc Return a {server_id(node()), Node} Pid name for the given Node. @@ -30,11 +30,11 @@ server_pid(Node) -> %% @doc send a message as quickly as possible send(Dest, Msg) -> case erlang:send(Dest, Msg, [noconnect, nosuspend]) of - ok -> - ok; - _ -> - % treat nosuspend and noconnect the same - rexi_buffer:send(Dest, Msg) + ok -> + ok; + _ -> + % treat nosuspend and noconnect the same + rexi_buffer:send(Dest, Msg) end. %% @doc set up the receive loop with an overall timeout @@ -53,53 +53,53 @@ recv(Refs, Keypos, Fun, Acc0, GlobalTimeout, PerMsgTO) -> process_mailbox(RefList, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) -> case process_message(RefList, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) of - {ok, Acc} -> - process_mailbox(RefList, Keypos, Fun, Acc, TimeoutRef, PerMsgTO); - {new_refs, NewRefList, Acc} -> - process_mailbox(NewRefList, Keypos, Fun, Acc, TimeoutRef, PerMsgTO); - {stop, Acc} -> - {ok, Acc}; - Error -> - Error + {ok, Acc} -> + process_mailbox(RefList, Keypos, Fun, Acc, TimeoutRef, PerMsgTO); + {new_refs, NewRefList, Acc} -> + process_mailbox(NewRefList, Keypos, Fun, Acc, TimeoutRef, PerMsgTO); + {stop, Acc} -> + {ok, Acc}; + Error -> + Error end. process_message(RefList, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) -> receive - {timeout, TimeoutRef} -> - {timeout, Acc0}; - {rexi, Ref, Msg} -> - case lists:keyfind(Ref, Keypos, RefList) of - false -> + {timeout, TimeoutRef} -> + {timeout, Acc0}; + {rexi, Ref, Msg} -> + case lists:keyfind(Ref, Keypos, RefList) of + false -> + {ok, Acc0}; + Worker -> + Fun(Msg, Worker, Acc0) + end; + {rexi, Ref, From, Msg} -> + case lists:keyfind(Ref, Keypos, RefList) of + false -> + {ok, Acc0}; + Worker -> + Fun(Msg, {Worker, From}, Acc0) + end; + {rexi, '$rexi_ping'} -> {ok, Acc0}; - Worker -> - Fun(Msg, Worker, Acc0) - end; - {rexi, Ref, From, Msg} -> - case lists:keyfind(Ref, Keypos, RefList) of - false -> - {ok, Acc0}; - Worker -> - Fun(Msg, {Worker, From}, Acc0) - end; - {rexi, '$rexi_ping'} -> - {ok, Acc0}; - {Ref, Msg} -> - case lists:keyfind(Ref, Keypos, RefList) of - false -> - % this was some non-matching message which we will ignore - {ok, Acc0}; - Worker -> - Fun(Msg, Worker, Acc0) - end; - {Ref, From, Msg} -> - case lists:keyfind(Ref, Keypos, RefList) of - false -> - {ok, Acc0}; - Worker -> - Fun(Msg, {Worker, From}, Acc0) - end; - {rexi_DOWN, _, _, _} = Msg -> - Fun(Msg, nil, Acc0) + {Ref, Msg} -> + case lists:keyfind(Ref, Keypos, RefList) of + false -> + % this was some non-matching message which we will ignore + {ok, Acc0}; + Worker -> + Fun(Msg, Worker, Acc0) + end; + {Ref, From, Msg} -> + case lists:keyfind(Ref, Keypos, RefList) of + false -> + {ok, Acc0}; + Worker -> + Fun(Msg, {Worker, From}, Acc0) + end; + {rexi_DOWN, _, _, _} = Msg -> + Fun(Msg, nil, Acc0) after PerMsgTO -> {timeout, Acc0} end. diff --git a/src/setup/src/setup.erl b/src/setup/src/setup.erl index 56773e4da..1757a43e7 100644 --- a/src/setup/src/setup.erl +++ b/src/setup/src/setup.erl @@ -18,11 +18,10 @@ -include_lib("../couch/include/couch_db.hrl"). - require_admins(undefined, {undefined, undefined}) -> % no admin in CouchDB, no admin in request throw({error, "Cluster setup requires admin account to be configured"}); -require_admins(_,_) -> +require_admins(_, _) -> ok. require_node_count(undefined) -> @@ -34,8 +33,14 @@ error_local_bind_address() -> throw({error, "Cluster setup requires a remote bind_address (not 127.0.0.1 nor ::1)"}). error_invalid_bind_address(InvalidBindAddress) -> - throw({error, io:format("Setup requires a valid IP bind_address. " ++ - "~p is invalid.", [InvalidBindAddress])}). + throw( + {error, + io:format( + "Setup requires a valid IP bind_address. " ++ + "~p is invalid.", + [InvalidBindAddress] + )} + ). require_remote_bind_address(OldBindAddress, NewBindAddress) -> case {OldBindAddress, NewBindAddress} of @@ -60,8 +65,8 @@ is_cluster_enabled() -> Admins = config:get("admins"), case {BindAddress, Admins} of {"127.0.0.1", _} -> false; - {_,[]} -> false; - {_,_} -> true + {_, []} -> false; + {_, _} -> true end. is_single_node_enabled(Dbs) -> @@ -71,23 +76,21 @@ is_single_node_enabled(Dbs) -> case {Admins, HasDbs} of {[], _} -> false; {_, false} -> false; - {_,_} -> true + {_, _} -> true end. cluster_system_dbs() -> ["_users", "_replicator"]. - has_cluster_system_dbs([]) -> true; -has_cluster_system_dbs([Db|Dbs]) -> +has_cluster_system_dbs([Db | Dbs]) -> case catch fabric:get_db_info(Db) of {ok, _} -> has_cluster_system_dbs(Dbs); _ -> false end. enable_cluster(Options) -> - case couch_util:get_value(remote_node, Options, undefined) of undefined -> enable_cluster_int(Options, is_cluster_enabled()); @@ -114,17 +117,19 @@ enable_cluster_http(Options) -> AdminUsername = couch_util:get_value(username, Options), AdminPasswordHash = config:get("admins", binary_to_list(AdminUsername)), - Body = ?JSON_ENCODE({[ - {<<"action">>, <<"enable_cluster">>}, - {<<"username">>, AdminUsername}, - {<<"password_hash">>, ?l2b(AdminPasswordHash)}, - {<<"bind_address">>, couch_util:get_value(bind_address, Options)}, - {<<"port">>, couch_util:get_value(port, Options)}, - {<<"node_count">>, couch_util:get_value(node_count, Options)} - ]}), + Body = ?JSON_ENCODE( + {[ + {<<"action">>, <<"enable_cluster">>}, + {<<"username">>, AdminUsername}, + {<<"password_hash">>, ?l2b(AdminPasswordHash)}, + {<<"bind_address">>, couch_util:get_value(bind_address, Options)}, + {<<"port">>, couch_util:get_value(port, Options)}, + {<<"node_count">>, couch_util:get_value(node_count, Options)} + ]} + ), Headers = [ - {"Content-Type","application/json"} + {"Content-Type", "application/json"} ], RemoteNode = couch_util:get_value(remote_node, Options), @@ -142,19 +147,18 @@ enable_cluster_http(Options) -> enable_cluster_int(_Options, true) -> {error, cluster_enabled}; enable_cluster_int(Options, false) -> - % if no admin in config and no admin in req -> error CurrentAdmins = config:get("admins"), NewCredentials = { proplists:get_value(username, Options), case proplists:get_value(password_hash, Options) of - undefined -> proplists:get_value(password, Options); - Pw -> Pw + undefined -> proplists:get_value(password, Options); + Pw -> Pw end }, ok = require_admins(CurrentAdmins, NewCredentials), % if bind_address == 127.0.0.1 and no bind_address in req -> error - CurrentBindAddress = config:get("chttpd","bind_address"), + CurrentBindAddress = config:get("chttpd", "bind_address"), NewBindAddress = proplists:get_value(bind_address, Options), ok = require_remote_bind_address(CurrentBindAddress, NewBindAddress), NodeCount = couch_util:get_value(node_count, Options), @@ -196,7 +200,6 @@ setup_node(NewCredentials, NewBindAddress, NodeCount, Port) -> config:set_integer("chttpd", "port", Port) end. - finish_cluster(Options) -> % ensure that uuid is set couch_server:get_uuid(), @@ -208,7 +211,6 @@ finish_cluster(Options) -> Dbs = proplists:get_value(ensure_dbs_exist, Options, cluster_system_dbs()), finish_cluster_int(Dbs, has_cluster_system_dbs(Dbs)). - wait_connected() -> Nodes = other_nodes(), Result = test_util:wait(fun() -> @@ -225,30 +227,31 @@ wait_connected() -> ok end. - other_nodes() -> mem3:nodes() -- [node()]. - disconnected(Nodes) -> - lists:filter(fun(Node) -> - case net_adm:ping(Node) of - pong -> false; - pang -> true - end - end, Nodes). - + lists:filter( + fun(Node) -> + case net_adm:ping(Node) of + pong -> false; + pang -> true + end + end, + Nodes + ). sync_admins() -> - ok = lists:foreach(fun({User, Pass}) -> - sync_admin(User, Pass) - end, config:get("admins")). - + ok = lists:foreach( + fun({User, Pass}) -> + sync_admin(User, Pass) + end, + config:get("admins") + ). sync_admin(User, Pass) -> sync_config("admins", User, Pass). - sync_uuid() -> Uuid = config:get("couchdb", "uuid"), sync_config("couchdb", "uuid", Uuid). @@ -257,26 +260,33 @@ sync_auth_secret() -> Secret = config:get("chttpd_auth", "secret"), sync_config("chttpd_auth", "secret", Secret). - sync_config(Section, Key, Value) -> - {Results, Errors} = rpc:multicall(other_nodes(), config, set, - [Section, Key, Value]), + {Results, Errors} = rpc:multicall( + other_nodes(), + config, + set, + [Section, Key, Value] + ), case validate_multicall(Results, Errors) of ok -> ok; error -> - couch_log:error("~p sync_admin results ~p errors ~p", - [?MODULE, Results, Errors]), + couch_log:error( + "~p sync_admin results ~p errors ~p", + [?MODULE, Results, Errors] + ), Reason = "Cluster setup unable to sync admin passwords", throw({setup_error, Reason}) end. - validate_multicall(Results, Errors) -> - AllOk = lists:all(fun - (ok) -> true; - (_) -> false - end, Results), + AllOk = lists:all( + fun + (ok) -> true; + (_) -> false + end, + Results + ), case AllOk andalso Errors == [] of true -> ok; @@ -284,21 +294,19 @@ validate_multicall(Results, Errors) -> error end. - finish_cluster_int(_Dbs, true) -> {error, cluster_finished}; finish_cluster_int(Dbs, false) -> lists:foreach(fun fabric:create_db/1, Dbs). - enable_single_node(Options) -> % if no admin in config and no admin in req -> error CurrentAdmins = config:get("admins"), NewCredentials = { proplists:get_value(username, Options), case proplists:get_value(password_hash, Options) of - undefined -> proplists:get_value(password, Options); - Pw -> Pw + undefined -> proplists:get_value(password, Options); + Pw -> Pw end }, ok = require_admins(CurrentAdmins, NewCredentials), @@ -311,7 +319,6 @@ enable_single_node(Options) -> finish_cluster_int(Dbs, has_cluster_system_dbs(Dbs)), couch_log:debug("Enable Single Node: ~p~n", [Options]). - add_node(Options) -> add_node_int(Options, is_cluster_enabled()). @@ -329,13 +336,15 @@ add_node_int(Options, true) -> }} ], - Body = ?JSON_ENCODE({[ - {<<"action">>, <<"receive_cookie">>}, - {<<"cookie">>, atom_to_binary(ErlangCookie, utf8)} - ]}), + Body = ?JSON_ENCODE( + {[ + {<<"action">>, <<"receive_cookie">>}, + {<<"cookie">>, atom_to_binary(ErlangCookie, utf8)} + ]} + ), Headers = [ - {"Content-Type","application/json"} + {"Content-Type", "application/json"} ], Host = proplists:get_value(host, Options), diff --git a/src/setup/src/setup_epi.erl b/src/setup/src/setup_epi.erl index c3f2636f0..df717dbc3 100644 --- a/src/setup/src/setup_epi.erl +++ b/src/setup/src/setup_epi.erl @@ -10,7 +10,6 @@ % License for the specific language governing permissions and limitations under % the License. - -module(setup_epi). -behaviour(couch_epi_plugin). @@ -30,7 +29,7 @@ app() -> providers() -> [ - {chttpd_handlers, setup_httpd_handlers} + {chttpd_handlers, setup_httpd_handlers} ]. services() -> diff --git a/src/setup/src/setup_httpd.erl b/src/setup/src/setup_httpd.erl index 6d17186fb..418a72845 100644 --- a/src/setup/src/setup_httpd.erl +++ b/src/setup/src/setup_httpd.erl @@ -15,19 +15,19 @@ -export([handle_setup_req/1]). -handle_setup_req(#httpd{method='POST'}=Req) -> +handle_setup_req(#httpd{method = 'POST'} = Req) -> ok = chttpd:verify_is_server_admin(Req), couch_httpd:validate_ctype(Req, "application/json"), Setup = get_body(Req), couch_log:notice("Setup: ~p~n", [remove_sensitive(Setup)]), Action = binary_to_list(couch_util:get_value(<<"action">>, Setup, <<"missing">>)), case handle_action(Action, Setup) of - ok -> - chttpd:send_json(Req, 201, {[{ok, true}]}); - {error, Message} -> - couch_httpd:send_error(Req, 400, <<"bad_request">>, Message) + ok -> + chttpd:send_json(Req, 201, {[{ok, true}]}); + {error, Message} -> + couch_httpd:send_error(Req, 400, <<"bad_request">>, Message) end; -handle_setup_req(#httpd{method='GET'}=Req) -> +handle_setup_req(#httpd{method = 'GET'} = Req) -> ok = chttpd:verify_is_server_admin(Req), Dbs = chttpd:qs_json_value(Req, "ensure_dbs_exist", setup:cluster_system_dbs()), couch_log:notice("Dbs: ~p~n", [Dbs]), @@ -58,10 +58,9 @@ handle_setup_req(#httpd{method='GET'}=Req) -> end end end; -handle_setup_req(#httpd{}=Req) -> +handle_setup_req(#httpd{} = Req) -> chttpd:send_method_not_allowed(Req, "GET,POST"). - get_options(Options, Setup) -> ExtractValues = fun({Tag, Option}, OptionsAcc) -> case couch_util:get_value(Option, Setup) of @@ -72,30 +71,35 @@ get_options(Options, Setup) -> lists:foldl(ExtractValues, [], Options). handle_action("enable_cluster", Setup) -> - Options = get_options([ - {username, <<"username">>}, - {password, <<"password">>}, - {password_hash, <<"password_hash">>}, - {bind_address, <<"bind_address">>}, - {port, <<"port">>}, - {remote_node, <<"remote_node">>}, - {remote_current_user, <<"remote_current_user">>}, - {remote_current_password, <<"remote_current_password">>}, - {node_count, <<"node_count">>} - ], Setup), + Options = get_options( + [ + {username, <<"username">>}, + {password, <<"password">>}, + {password_hash, <<"password_hash">>}, + {bind_address, <<"bind_address">>}, + {port, <<"port">>}, + {remote_node, <<"remote_node">>}, + {remote_current_user, <<"remote_current_user">>}, + {remote_current_password, <<"remote_current_password">>}, + {node_count, <<"node_count">>} + ], + Setup + ), case setup:enable_cluster(Options) of {error, cluster_enabled} -> {error, <<"Cluster is already enabled">>}; - _ -> ok + _ -> + ok end; - - handle_action("finish_cluster", Setup) -> couch_log:notice("finish_cluster: ~p~n", [remove_sensitive(Setup)]), - Options = get_options([ - {ensure_dbs_exist, <<"ensure_dbs_exist">>} - ], Setup), + Options = get_options( + [ + {ensure_dbs_exist, <<"ensure_dbs_exist">>} + ], + Setup + ), case setup:finish_cluster(Options) of {error, cluster_finished} -> {error, <<"Cluster is already finished">>}; @@ -103,18 +107,20 @@ handle_action("finish_cluster", Setup) -> couch_log:notice("finish_cluster: ~p~n", [Else]), ok end; - handle_action("enable_single_node", Setup) -> couch_log:notice("enable_single_node: ~p~n", [remove_sensitive(Setup)]), - Options = get_options([ - {ensure_dbs_exist, <<"ensure_dbs_exist">>}, - {username, <<"username">>}, - {password, <<"password">>}, - {password_hash, <<"password_hash">>}, - {bind_address, <<"bind_address">>}, - {port, <<"port">>} - ], Setup), + Options = get_options( + [ + {ensure_dbs_exist, <<"ensure_dbs_exist">>}, + {username, <<"username">>}, + {password, <<"password">>}, + {password_hash, <<"password_hash">>}, + {bind_address, <<"bind_address">>}, + {port, <<"port">>} + ], + Setup + ), case setup:enable_single_node(Options) of {error, cluster_finished} -> {error, <<"Cluster is already finished">>}; @@ -122,18 +128,19 @@ handle_action("enable_single_node", Setup) -> couch_log:notice("Else: ~p~n", [Else]), ok end; - - handle_action("add_node", Setup) -> couch_log:notice("add_node: ~p~n", [remove_sensitive(Setup)]), - Options = get_options([ - {username, <<"username">>}, - {password, <<"password">>}, - {host, <<"host">>}, - {port, <<"port">>}, - {name, <<"name">>} - ], Setup), + Options = get_options( + [ + {username, <<"username">>}, + {password, <<"password">>}, + {host, <<"host">>}, + {port, <<"port">>}, + {name, <<"name">>} + ], + Setup + ), case setup:add_node(Options) of {error, cluster_not_enabled} -> {error, <<"Cluster is not enabled.">>}; @@ -143,35 +150,36 @@ handle_action("add_node", Setup) -> {error, <<"Add node failed. Invalid admin credentials,">>}; {error, Message} -> {error, Message}; - _ -> ok + _ -> + ok end; - handle_action("remove_node", Setup) -> couch_log:notice("remove_node: ~p~n", [remove_sensitive(Setup)]); - handle_action("receive_cookie", Setup) -> couch_log:notice("receive_cookie: ~p~n", [remove_sensitive(Setup)]), - Options = get_options([ - {cookie, <<"cookie">>} - ], Setup), + Options = get_options( + [ + {cookie, <<"cookie">>} + ], + Setup + ), case setup:receive_cookie(Options) of {error, Error} -> {error, Error}; - _ -> ok + _ -> + ok end; - handle_action(_, _) -> couch_log:notice("invalid_action: ~n", []), {error, <<"Invalid Action'">>}. - get_body(Req) -> case catch couch_httpd:json_body_obj(Req) of - {Body} -> - Body; - Else -> - couch_log:notice("Body Fail: ~p~n", [Else]), - couch_httpd:send_error(Req, 400, <<"bad_request">>, <<"Missing JSON body'">>) + {Body} -> + Body; + Else -> + couch_log:notice("Body Fail: ~p~n", [Else]), + couch_httpd:send_error(Req, 400, <<"bad_request">>, <<"Missing JSON body'">>) end. remove_sensitive(KVList) -> diff --git a/src/setup/src/setup_sup.erl b/src/setup/src/setup_sup.erl index 4670a0a59..e80ad8ef0 100644 --- a/src/setup/src/setup_sup.erl +++ b/src/setup/src/setup_sup.erl @@ -35,8 +35,8 @@ start_link() -> %% =================================================================== init([]) -> - case config:get_boolean("couchdb", "single_node", false) of - true -> + case config:get_boolean("couchdb", "single_node", false) of + true -> setup:finish_cluster([]); false -> ok diff --git a/src/smoosh/src/smoosh.erl b/src/smoosh/src/smoosh.erl index 676e7faad..950500ffa 100644 --- a/src/smoosh/src/smoosh.erl +++ b/src/smoosh/src/smoosh.erl @@ -37,33 +37,48 @@ status() -> smoosh_server:status(). enqueue_all_dbs() -> - fold_local_shards(fun(#shard{name=Name}, _Acc) -> - sync_enqueue(Name) end, ok). + fold_local_shards( + fun(#shard{name = Name}, _Acc) -> + sync_enqueue(Name) + end, + ok + ). enqueue_all_dbs(Timeout) -> - fold_local_shards(fun(#shard{name=Name}, _Acc) -> - sync_enqueue(Name, Timeout) end, ok). + fold_local_shards( + fun(#shard{name = Name}, _Acc) -> + sync_enqueue(Name, Timeout) + end, + ok + ). enqueue_all_views() -> - fold_local_shards(fun(#shard{name=Name}, _Acc) -> - catch enqueue_views(Name) end, ok). + fold_local_shards( + fun(#shard{name = Name}, _Acc) -> + catch enqueue_views(Name) + end, + ok + ). fold_local_shards(Fun, Acc0) -> - mem3:fold_shards(fun(Shard, Acc1) -> - case node() == Shard#shard.node of - true -> - Fun(Shard, Acc1); - false -> - Acc1 - end - end, Acc0). + mem3:fold_shards( + fun(Shard, Acc1) -> + case node() == Shard#shard.node of + true -> + Fun(Shard, Acc1); + false -> + Acc1 + end + end, + Acc0 + ). enqueue_views(ShardName) -> DbName = mem3:dbname(ShardName), {ok, DDocs} = fabric:design_docs(DbName), [sync_enqueue({ShardName, id(DDoc)}) || DDoc <- DDocs]. -id(#doc{id=Id}) -> +id(#doc{id = Id}) -> Id; id({Props}) -> couch_util:get_value(<<"_id">>, Props). diff --git a/src/smoosh/src/smoosh_channel.erl b/src/smoosh/src/smoosh_channel.erl index 2bc98be9d..06849ac3a 100644 --- a/src/smoosh/src/smoosh_channel.erl +++ b/src/smoosh/src/smoosh_channel.erl @@ -20,17 +20,23 @@ -export([enqueue/3, last_updated/2, flush/1]). % gen_server api. --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - code_change/3, terminate/2]). +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + code_change/3, + terminate/2 +]). % records. -record(state, { - active=[], + active = [], name, - waiting=smoosh_priority_queue:new(), - paused=true, - starting=[] + waiting = smoosh_priority_queue:new(), + paused = true, + starting = [] }). % public functions. @@ -64,41 +70,40 @@ flush(ServerRef) -> init(Name) -> schedule_unpause(), erlang:send_after(60 * 1000, self(), check_window), - {ok, #state{name=Name}}. + {ok, #state{name = Name}}. handle_call({last_updated, Object}, _From, State0) -> {ok, State} = code_change(nil, State0, nil), LastUpdated = smoosh_priority_queue:last_updated(Object, State#state.waiting), {reply, LastUpdated, State}; - handle_call(suspend, _From, State0) -> {ok, State} = code_change(nil, State0, nil), #state{active = Active} = State, - [catch erlang:suspend_process(Pid, [unless_suspending]) - || {_,Pid} <- Active], - {reply, ok, State#state{paused=true}}; - + [ + catch erlang:suspend_process(Pid, [unless_suspending]) + || {_, Pid} <- Active + ], + {reply, ok, State#state{paused = true}}; handle_call(resume, _From, State0) -> {ok, State} = code_change(nil, State0, nil), #state{active = Active} = State, - [catch erlang:resume_process(Pid) || {_,Pid} <- Active], - {reply, ok, State#state{paused=false}}; - + [catch erlang:resume_process(Pid) || {_, Pid} <- Active], + {reply, ok, State#state{paused = false}}; handle_call(status, _From, State0) -> {ok, State} = code_change(nil, State0, nil), - {reply, {ok, [ - {active, length(State#state.active)}, - {starting, length(State#state.starting)}, - {waiting, smoosh_priority_queue:info(State#state.waiting)} - ]}, State}; - + {reply, + {ok, [ + {active, length(State#state.active)}, + {starting, length(State#state.starting)}, + {waiting, smoosh_priority_queue:info(State#state.waiting)} + ]}, + State}; handle_call(close, _From, State0) -> {ok, State} = code_change(nil, State0, nil), {stop, normal, ok, State}; - handle_call(flush, _From, State0) -> {ok, State} = code_change(nil, State0, nil), - {reply, ok, State#state{waiting=smoosh_priority_queue:new()}}. + {reply, ok, State#state{waiting = smoosh_priority_queue:new()}}. handle_cast({enqueue, _Object, 0}, State0) -> {ok, State} = code_change(nil, State0, nil), @@ -109,206 +114,247 @@ handle_cast({enqueue, Object, Priority}, State0) -> % We accept noproc here due to possibly having monitored a restarted compaction % pid after it finished. -handle_info({'DOWN', Ref, _, Job, Reason}, State0) when Reason == normal; - Reason == noproc -> +handle_info({'DOWN', Ref, _, Job, Reason}, State0) when + Reason == normal; + Reason == noproc +-> {ok, State} = code_change(nil, State0, nil), - #state{active=Active, starting=Starting} = State, - {noreply, maybe_start_compaction( - State#state{active=lists:keydelete(Job, 2, Active), - starting=lists:keydelete(Ref, 1, Starting)})}; - + #state{active = Active, starting = Starting} = State, + {noreply, + maybe_start_compaction( + State#state{ + active = lists:keydelete(Job, 2, Active), + starting = lists:keydelete(Ref, 1, Starting) + } + )}; handle_info({'DOWN', Ref, _, Job, Reason}, State0) -> {ok, State} = code_change(nil, State0, nil), - #state{active=Active0, starting=Starting0} = State, + #state{active = Active0, starting = Starting0} = State, case lists:keytake(Job, 2, Active0) of {value, {Key, _Pid}, Active1} -> - State1 = maybe_remonitor_cpid(State#state{active=Active1}, Key, - Reason), + State1 = maybe_remonitor_cpid( + State#state{active = Active1}, + Key, + Reason + ), {noreply, maybe_start_compaction(State1)}; false -> case lists:keytake(Ref, 1, Starting0) of {value, {_, Key}, Starting1} -> couch_log:warning("failed to start compaction of ~p: ~p", [ - smoosh_utils:stringify(Key), Reason]), + smoosh_utils:stringify(Key), Reason + ]), {ok, _} = timer:apply_after(5000, smoosh_server, enqueue, [Key]), - {noreply, maybe_start_compaction(State#state{starting=Starting1})}; + {noreply, maybe_start_compaction(State#state{starting = Starting1})}; false -> {noreply, State} end end; - handle_info({Ref, {ok, Pid}}, State0) when is_reference(Ref) -> {ok, State} = code_change(nil, State0, nil), case lists:keytake(Ref, 1, State#state.starting) of {value, {_, Key}, Starting1} -> - couch_log:notice("~s: Started compaction for ~s", - [State#state.name, smoosh_utils:stringify(Key)]), + couch_log:notice( + "~s: Started compaction for ~s", + [State#state.name, smoosh_utils:stringify(Key)] + ), erlang:monitor(process, Pid), erlang:demonitor(Ref, [flush]), - {noreply, State#state{active=[{Key, Pid}|State#state.active], - starting=Starting1}}; + {noreply, State#state{ + active = [{Key, Pid} | State#state.active], + starting = Starting1 + }}; false -> {noreply, State} end; - handle_info(check_window, State0) -> {ok, State} = code_change(nil, State0, nil), #state{paused = Paused, name = Name} = State, StrictWindow = smoosh_utils:get(Name, "strict_window", "false"), - FinalState = case {not Paused, smoosh_utils:in_allowed_window(Name)} of - {false, false} -> - % already in desired state - State; - {true, true} -> - % already in desired state - State; - {false, true} -> - % resume is always safe even if we did not previously suspend - {reply, ok, NewState} = handle_call(resume, nil, State), - NewState; - {true, false} -> - if StrictWindow =:= "true" -> - {reply, ok, NewState} = handle_call(suspend, nil, State), + FinalState = + case {not Paused, smoosh_utils:in_allowed_window(Name)} of + {false, false} -> + % already in desired state + State; + {true, true} -> + % already in desired state + State; + {false, true} -> + % resume is always safe even if we did not previously suspend + {reply, ok, NewState} = handle_call(resume, nil, State), NewState; - true -> - State#state{paused=true} - end - end, + {true, false} -> + if + StrictWindow =:= "true" -> + {reply, ok, NewState} = handle_call(suspend, nil, State), + NewState; + true -> + State#state{paused = true} + end + end, erlang:send_after(60 * 1000, self(), check_window), {noreply, FinalState}; - handle_info(pause, State0) -> {ok, State} = code_change(nil, State0, nil), - {noreply, State#state{paused=true}}; + {noreply, State#state{paused = true}}; handle_info(unpause, State0) -> {ok, State} = code_change(nil, State0, nil), - {noreply, maybe_start_compaction(State#state{paused=false})}. + {noreply, maybe_start_compaction(State#state{paused = false})}. terminate(_Reason, _State) -> ok. -code_change(_OldVsn, #state{}=State, _Extra) -> +code_change(_OldVsn, #state{} = State, _Extra) -> {ok, State}. % private functions. add_to_queue(Key, Priority, State) -> - #state{active=Active,waiting=Q} = State, + #state{active = Active, waiting = Q} = State, case lists:keymember(Key, 1, Active) of - true -> - State; - false -> - Capacity = list_to_integer(smoosh_utils:get(State#state.name, "capacity", "9999")), - couch_log:notice( - "~s: adding ~p to internal compactor queue with priority ~p", - [State#state.name, Key, Priority]), - State#state{ - waiting=smoosh_priority_queue:in(Key, Priority, Priority, Capacity, Q) - } + true -> + State; + false -> + Capacity = list_to_integer(smoosh_utils:get(State#state.name, "capacity", "9999")), + couch_log:notice( + "~s: adding ~p to internal compactor queue with priority ~p", + [State#state.name, Key, Priority] + ), + State#state{ + waiting = smoosh_priority_queue:in(Key, Priority, Priority, Capacity, Q) + } end. -maybe_start_compaction(#state{paused=true}=State) -> +maybe_start_compaction(#state{paused = true} = State) -> State; maybe_start_compaction(State) -> - Concurrency = list_to_integer(smoosh_utils:get(State#state.name, - "concurrency", "1")), - if length(State#state.active) + length(State#state.starting) < Concurrency -> - case smoosh_priority_queue:out(State#state.waiting) of - false -> - State; - {Key, Priority, Q} -> - try - State2 = case start_compact(State, Key) of + Concurrency = list_to_integer( + smoosh_utils:get( + State#state.name, + "concurrency", + "1" + ) + ), + if + length(State#state.active) + length(State#state.starting) < Concurrency -> + case smoosh_priority_queue:out(State#state.waiting) of false -> State; - State1 -> - couch_log:notice( - "~s: Starting compaction for ~s (priority ~p)", - [State#state.name, smoosh_utils:stringify(Key), Priority]), - State1 - end, - maybe_start_compaction(State2#state{waiting=Q}) - catch Class:Exception -> - couch_log:notice("~s: ~p ~p for ~s", - [State#state.name, Class, Exception, - smoosh_utils:stringify(Key)]), - maybe_start_compaction(State#state{waiting=Q}) - end - end; - true -> - State + {Key, Priority, Q} -> + try + State2 = + case start_compact(State, Key) of + false -> + State; + State1 -> + couch_log:notice( + "~s: Starting compaction for ~s (priority ~p)", + [State#state.name, smoosh_utils:stringify(Key), Priority] + ), + State1 + end, + maybe_start_compaction(State2#state{waiting = Q}) + catch + Class:Exception -> + couch_log:notice( + "~s: ~p ~p for ~s", + [ + State#state.name, + Class, + Exception, + smoosh_utils:stringify(Key) + ] + ), + maybe_start_compaction(State#state{waiting = Q}) + end + end; + true -> + State end. start_compact(State, {schema, DbName, GroupId}) -> case smoosh_utils:ignore_db({DbName, GroupId}) of false -> - {ok, Pid} = couch_md_index_manager:get_group_pid(DbName, - GroupId), + {ok, Pid} = couch_md_index_manager:get_group_pid( + DbName, + GroupId + ), Ref = erlang:monitor(process, Pid), Pid ! {'$gen_call', {self(), Ref}, compact}, - State#state{starting=[{Ref, {schema, DbName, - GroupId}} | State#state.starting]}; + State#state{starting = [{Ref, {schema, DbName, GroupId}} | State#state.starting]}; _ -> false end; - start_compact(State, DbName) when is_list(DbName) -> start_compact(State, ?l2b(DbName)); start_compact(State, DbName) when is_binary(DbName) -> {ok, Db} = couch_db:open_int(DbName, []), - try start_compact(State, Db) after couch_db:close(Db) end; -start_compact(State, {Shard,GroupId}) -> + try + start_compact(State, Db) + after + couch_db:close(Db) + end; +start_compact(State, {Shard, GroupId}) -> case smoosh_utils:ignore_db({Shard, GroupId}) of - false -> - DbName = mem3:dbname(Shard), - {ok, Pid} = couch_index_server:get_index( - couch_mrview_index, Shard, GroupId), - spawn(fun() -> cleanup_index_files(DbName, Shard) end), - Ref = erlang:monitor(process, Pid), - Pid ! {'$gen_call', {self(), Ref}, compact}, - State#state{starting=[{Ref, {Shard, GroupId}}|State#state.starting]}; - _ -> - false + false -> + DbName = mem3:dbname(Shard), + {ok, Pid} = couch_index_server:get_index( + couch_mrview_index, Shard, GroupId + ), + spawn(fun() -> cleanup_index_files(DbName, Shard) end), + Ref = erlang:monitor(process, Pid), + Pid ! {'$gen_call', {self(), Ref}, compact}, + State#state{starting = [{Ref, {Shard, GroupId}} | State#state.starting]}; + _ -> + false end; start_compact(State, Db) -> case smoosh_utils:ignore_db(Db) of - false -> - DbPid = couch_db:get_pid(Db), - Key = couch_db:name(Db), - case couch_db:get_compactor_pid(Db) of - nil -> - Ref = erlang:monitor(process, DbPid), - DbPid ! {'$gen_call', {self(), Ref}, start_compact}, - State#state{starting=[{Ref, Key}|State#state.starting]}; - % Compaction is already running, so monitor existing compaction pid. - CPid -> - couch_log:notice("Db ~s continuing compaction", - [smoosh_utils:stringify(Key)]), - erlang:monitor(process, CPid), - State#state{active=[{Key, CPid}|State#state.active]} - end; - _ -> - false + false -> + DbPid = couch_db:get_pid(Db), + Key = couch_db:name(Db), + case couch_db:get_compactor_pid(Db) of + nil -> + Ref = erlang:monitor(process, DbPid), + DbPid ! {'$gen_call', {self(), Ref}, start_compact}, + State#state{starting = [{Ref, Key} | State#state.starting]}; + % Compaction is already running, so monitor existing compaction pid. + CPid -> + couch_log:notice( + "Db ~s continuing compaction", + [smoosh_utils:stringify(Key)] + ), + erlang:monitor(process, CPid), + State#state{active = [{Key, CPid} | State#state.active]} + end; + _ -> + false end. maybe_remonitor_cpid(State, DbName, Reason) when is_binary(DbName) -> {ok, Db} = couch_db:open_int(DbName, []), case couch_db:get_compactor_pid_sync(Db) of nil -> - couch_log:warning("exit for compaction of ~p: ~p", - [smoosh_utils:stringify(DbName), Reason]), + couch_log:warning( + "exit for compaction of ~p: ~p", + [smoosh_utils:stringify(DbName), Reason] + ), {ok, _} = timer:apply_after(5000, smoosh_server, enqueue, [DbName]), State; CPid -> - couch_log:notice("~s compaction already running. Re-monitor Pid ~p", - [smoosh_utils:stringify(DbName), CPid]), + couch_log:notice( + "~s compaction already running. Re-monitor Pid ~p", + [smoosh_utils:stringify(DbName), CPid] + ), erlang:monitor(process, CPid), - State#state{active=[{DbName, CPid}|State#state.active]} + State#state{active = [{DbName, CPid} | State#state.active]} end; % not a database compaction, so ignore the pid check maybe_remonitor_cpid(State, Key, Reason) -> - couch_log:warning("exit for compaction of ~p: ~p", - [smoosh_utils:stringify(Key), Reason]), + couch_log:warning( + "exit for compaction of ~p: ~p", + [smoosh_utils:stringify(Key), Reason] + ), {ok, _} = timer:apply_after(5000, smoosh_server, enqueue, [Key]), State. @@ -318,8 +364,8 @@ schedule_unpause() -> cleanup_index_files(DbName, _Shard) -> case config:get("smoosh", "cleanup_index_files", "false") of - "true" -> - fabric:cleanup_index_files(DbName); - _ -> - ok + "true" -> + fabric:cleanup_index_files(DbName); + _ -> + ok end. diff --git a/src/smoosh/src/smoosh_priority_queue.erl b/src/smoosh/src/smoosh_priority_queue.erl index 6376103d9..b6f4b6dd8 100644 --- a/src/smoosh/src/smoosh_priority_queue.erl +++ b/src/smoosh/src/smoosh_priority_queue.erl @@ -15,14 +15,14 @@ -export([new/0, last_updated/2, is_key/2, in/4, in/5, out/1, size/1, info/1]). -record(priority_queue, { - dict=dict:new(), - tree=gb_trees:empty() + dict = dict:new(), + tree = gb_trees:empty() }). new() -> #priority_queue{}. -last_updated(Key, #priority_queue{dict=Dict}) -> +last_updated(Key, #priority_queue{dict = Dict}) -> case dict:find(Key, Dict) of {ok, {_Priority, {LastUpdatedMTime, _MInt}}} -> LastUpdatedMTime; @@ -30,48 +30,51 @@ last_updated(Key, #priority_queue{dict=Dict}) -> false end. -is_key(Key, #priority_queue{dict=Dict}) -> +is_key(Key, #priority_queue{dict = Dict}) -> dict:is_key(Key, Dict). in(Key, Value, Priority, Q) -> in(Key, Value, Priority, infinity, Q). -in(Key, Value, Priority, Capacity, #priority_queue{dict=Dict, tree=Tree}) -> - Tree1 = case dict:find(Key, Dict) of - {ok, TreeKey} -> - gb_trees:delete_any(TreeKey, Tree); - error -> - Tree - end, +in(Key, Value, Priority, Capacity, #priority_queue{dict = Dict, tree = Tree}) -> + Tree1 = + case dict:find(Key, Dict) of + {ok, TreeKey} -> + gb_trees:delete_any(TreeKey, Tree); + error -> + Tree + end, Now = {erlang:monotonic_time(), erlang:unique_integer([monotonic])}, TreeKey1 = {Priority, Now}, Tree2 = gb_trees:enter(TreeKey1, {Key, Value}, Tree1), Dict1 = dict:store(Key, TreeKey1, Dict), - truncate(Capacity, #priority_queue{dict=Dict1, tree=Tree2}). + truncate(Capacity, #priority_queue{dict = Dict1, tree = Tree2}). -out(#priority_queue{dict=Dict,tree=Tree}) -> +out(#priority_queue{dict = Dict, tree = Tree}) -> case gb_trees:is_empty(Tree) of - true -> - false; - false -> - {_, {Key, Value}, Tree1} = gb_trees:take_largest(Tree), - Dict1 = dict:erase(Key, Dict), - {Key, Value, #priority_queue{dict=Dict1, tree=Tree1}} + true -> + false; + false -> + {_, {Key, Value}, Tree1} = gb_trees:take_largest(Tree), + Dict1 = dict:erase(Key, Dict), + {Key, Value, #priority_queue{dict = Dict1, tree = Tree1}} end. -size(#priority_queue{tree=Tree}) -> +size(#priority_queue{tree = Tree}) -> gb_trees:size(Tree). -info(#priority_queue{tree=Tree}=Q) -> - [{size, ?MODULE:size(Q)}| - case gb_trees:is_empty(Tree) of - true -> - []; - false -> - {Min, _, _} = gb_trees:take_smallest(Tree), - {Max, _, _} = gb_trees:take_largest(Tree), - [{min, Min}, {max, Max}] - end]. +info(#priority_queue{tree = Tree} = Q) -> + [ + {size, ?MODULE:size(Q)} + | case gb_trees:is_empty(Tree) of + true -> + []; + false -> + {Min, _, _} = gb_trees:take_smallest(Tree), + {Max, _, _} = gb_trees:take_largest(Tree), + [{min, Min}, {max, Max}] + end + ]. truncate(infinity, Q) -> Q; @@ -80,7 +83,7 @@ truncate(Capacity, Q) when Capacity > 0 -> truncate(Capacity, Size, Q) when Size =< Capacity -> Q; -truncate(Capacity, Size, #priority_queue{dict=Dict, tree=Tree}) when Size > 0 -> +truncate(Capacity, Size, #priority_queue{dict = Dict, tree = Tree}) when Size > 0 -> {_, {Key, _}, Tree1} = gb_trees:take_smallest(Tree), - Q1 = #priority_queue{dict=dict:erase(Key, Dict), tree=Tree1}, + Q1 = #priority_queue{dict = dict:erase(Key, Dict), tree = Tree1}, truncate(Capacity, ?MODULE:size(Q1), Q1). diff --git a/src/smoosh/src/smoosh_server.erl b/src/smoosh/src/smoosh_server.erl index 6aab19183..0526625ff 100644 --- a/src/smoosh/src/smoosh_server.erl +++ b/src/smoosh/src/smoosh_server.erl @@ -31,8 +31,14 @@ -define(SECONDS_PER_MINUTE, 60). % gen_server api. --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - code_change/3, terminate/2]). +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + code_change/3, + terminate/2 +]). % config_listener api -export([handle_config_change/5, handle_config_terminate/3]). @@ -49,12 +55,12 @@ % private records. -record(state, { - db_channels=[], - view_channels=[], - schema_channels=[], + db_channels = [], + view_channels = [], + schema_channels = [], tab, event_listener, - waiting=dict:new() + waiting = dict:new() }). -record(channel, { @@ -107,19 +113,27 @@ init([]) -> ok = config:listen_for_changes(?MODULE, nil), {ok, Pid} = start_event_listener(), DbChannels = smoosh_utils:split( - config:get("smoosh", "db_channels", "upgrade_dbs,ratio_dbs,slack_dbs")), + config:get("smoosh", "db_channels", "upgrade_dbs,ratio_dbs,slack_dbs") + ), ViewChannels = smoosh_utils:split( - config:get("smoosh", "view_channels", "upgrade_views,ratio_views,slack_views")), - SchemaChannels = smoosh_utils:split(config:get("smoosh", - "schema_channels", "ratio_schemas,slack_schemas")), + config:get("smoosh", "view_channels", "upgrade_views,ratio_views,slack_views") + ), + SchemaChannels = smoosh_utils:split( + config:get( + "smoosh", + "schema_channels", + "ratio_schemas,slack_schemas" + ) + ), Tab = ets:new(channels, [{keypos, #channel.name}]), - {ok, create_missing_channels(#state{ - db_channels=DbChannels, - view_channels=ViewChannels, - schema_channels=SchemaChannels, - event_listener=Pid, - tab=Tab - })}. + {ok, + create_missing_channels(#state{ + db_channels = DbChannels, + view_channels = ViewChannels, + schema_channels = SchemaChannels, + event_listener = Pid, + tab = Tab + })}. handle_config_change("smoosh", "db_channels", L, _, _) -> {ok, gen_server:cast(?MODULE, {new_db_channels, smoosh_utils:split(L)})}; @@ -133,105 +147,123 @@ handle_config_change(_, _, _, _, _) -> handle_config_terminate(_Server, stop, _State) -> ok; handle_config_terminate(_Server, _Reason, _State) -> - erlang:send_after(?RELISTEN_DELAY, - whereis(?MODULE), restart_config_listener). + erlang:send_after( + ?RELISTEN_DELAY, + whereis(?MODULE), + restart_config_listener + ). handle_call(status, _From, State) -> Acc = ets:foldl(fun get_channel_status/2, [], State#state.tab), {reply, {ok, Acc}, State}; - handle_call({enqueue, Object}, _From, State) -> {noreply, NewState} = handle_cast({enqueue, Object}, State), {reply, ok, NewState}; - handle_call(suspend, _From, State) -> - ets:foldl(fun(#channel{name=Name, pid=P}, _) -> - couch_log:notice("Suspending ~p", [Name]), - smoosh_channel:suspend(P) end, 0, - State#state.tab), + ets:foldl( + fun(#channel{name = Name, pid = P}, _) -> + couch_log:notice("Suspending ~p", [Name]), + smoosh_channel:suspend(P) + end, + 0, + State#state.tab + ), {reply, ok, State}; - handle_call(resume, _From, State) -> - ets:foldl(fun(#channel{name=Name, pid=P}, _) -> - couch_log:notice("Resuming ~p", [Name]), - smoosh_channel:resume(P) end, 0, - State#state.tab), + ets:foldl( + fun(#channel{name = Name, pid = P}, _) -> + couch_log:notice("Resuming ~p", [Name]), + smoosh_channel:resume(P) + end, + 0, + State#state.tab + ), {reply, ok, State}. handle_cast({new_db_channels, Channels}, State) -> - [smoosh_channel:close(channel_pid(State#state.tab, C)) || - C <- State#state.db_channels -- Channels], - {noreply, create_missing_channels(State#state{db_channels=Channels})}; - + [ + smoosh_channel:close(channel_pid(State#state.tab, C)) + || C <- State#state.db_channels -- Channels + ], + {noreply, create_missing_channels(State#state{db_channels = Channels})}; handle_cast({new_view_channels, Channels}, State) -> - [smoosh_channel:close(channel_pid(State#state.tab, C)) || - C <- State#state.view_channels -- Channels], - {noreply, create_missing_channels(State#state{view_channels=Channels})}; - + [ + smoosh_channel:close(channel_pid(State#state.tab, C)) + || C <- State#state.view_channels -- Channels + ], + {noreply, create_missing_channels(State#state{view_channels = Channels})}; handle_cast({new_schema_channels, Channels}, State) -> - [smoosh_channel:close(channel_pid(State#state.tab, C)) || - C <- State#state.schema_channels -- Channels], - {noreply, create_missing_channels(State#state{view_channels=Channels})}; - + [ + smoosh_channel:close(channel_pid(State#state.tab, C)) + || C <- State#state.schema_channels -- Channels + ], + {noreply, create_missing_channels(State#state{view_channels = Channels})}; handle_cast({enqueue, Object}, State) -> - #state{waiting=Waiting}=State, + #state{waiting = Waiting} = State, case dict:is_key(Object, Waiting) of true -> {noreply, State}; false -> {_Pid, Ref} = spawn_monitor(?MODULE, enqueue_request, [State, Object]), - {noreply, State#state{waiting=dict:store(Object, Ref, Waiting)}} + {noreply, State#state{waiting = dict:store(Object, Ref, Waiting)}} end. -handle_info({'EXIT', Pid, Reason}, #state{event_listener=Pid}=State) -> - couch_log:notice("update notifier died ~p", [Reason]), - {ok, Pid1} = start_event_listener(), - {noreply, State#state{event_listener=Pid1}}; +handle_info({'EXIT', Pid, Reason}, #state{event_listener = Pid} = State) -> + couch_log:notice("update notifier died ~p", [Reason]), + {ok, Pid1} = start_event_listener(), + {noreply, State#state{event_listener = Pid1}}; handle_info({'EXIT', Pid, Reason}, State) -> couch_log:notice("~p ~p died ~p", [?MODULE, Pid, Reason]), - case ets:match_object(State#state.tab, #channel{pid=Pid, _='_'}) of - [#channel{name=Name}] -> - ets:delete(State#state.tab, Name); - _ -> - ok + case ets:match_object(State#state.tab, #channel{pid = Pid, _ = '_'}) of + [#channel{name = Name}] -> + ets:delete(State#state.tab, Name); + _ -> + ok end, {noreply, create_missing_channels(State)}; - handle_info({'DOWN', Ref, _, _, _}, State) -> - Waiting = dict:filter(fun(_Key, Value) -> Value =/= Ref end, - State#state.waiting), - {noreply, State#state{waiting=Waiting}}; - + Waiting = dict:filter( + fun(_Key, Value) -> Value =/= Ref end, + State#state.waiting + ), + {noreply, State#state{waiting = Waiting}}; handle_info(restart_config_listener, State) -> ok = config:listen_for_changes(?MODULE, nil), {noreply, State}; - handle_info(_Msg, State) -> {noreply, State}. terminate(_Reason, State) -> - ets:foldl(fun(#channel{pid=P}, _) -> smoosh_channel:close(P) end, 0, - State#state.tab), + ets:foldl( + fun(#channel{pid = P}, _) -> smoosh_channel:close(P) end, + 0, + State#state.tab + ), ok. -code_change(_OldVsn, {state, DbChannels, ViewChannels, Tab, - EventListener, Waiting}, _Extra) -> - {ok, #state{db_channels=DbChannels, view_channels=ViewChannels, - schema_channels=[], tab=Tab, event_listener = EventListener, - waiting=Waiting}}; +code_change(_OldVsn, {state, DbChannels, ViewChannels, Tab, EventListener, Waiting}, _Extra) -> + {ok, #state{ + db_channels = DbChannels, + view_channels = ViewChannels, + schema_channels = [], + tab = Tab, + event_listener = EventListener, + waiting = Waiting + }}; code_change(_OldVsn, State, _Extra) -> {ok, State}. % private functions. -get_channel_status(#channel{name=Name, pid=P}, Acc0) when is_pid(P) -> +get_channel_status(#channel{name = Name, pid = P}, Acc0) when is_pid(P) -> try gen_server:call(P, status) of - {ok, Status} -> - [{Name, Status} | Acc0]; - _ -> - Acc0 - catch _:_ -> - Acc0 + {ok, Status} -> + [{Name, Status} | Acc0]; + _ -> + Acc0 + catch + _:_ -> + Acc0 end; get_channel_status(_, Acc0) -> Acc0. @@ -254,41 +286,42 @@ enqueue_request(State, Object) -> smoosh_utils:stringify(Object), Stack]) end. -find_channel(#state{}=State, {schema, DbName, GroupId}) -> +find_channel(#state{} = State, {schema, DbName, GroupId}) -> find_channel(State#state.tab, State#state.schema_channels, {schema, DbName, GroupId}); -find_channel(#state{}=State, {Shard, GroupId}) -> +find_channel(#state{} = State, {Shard, GroupId}) -> find_channel(State#state.tab, State#state.view_channels, {Shard, GroupId}); -find_channel(#state{}=State, DbName) -> +find_channel(#state{} = State, DbName) -> find_channel(State#state.tab, State#state.db_channels, DbName). find_channel(_Tab, [], _Object) -> false; -find_channel(Tab, [Channel|Rest], Object) -> +find_channel(Tab, [Channel | Rest], Object) -> Pid = channel_pid(Tab, Channel), LastUpdated = smoosh_channel:last_updated(Pid, Object), - StalenessInSec = config:get_integer("smoosh", "staleness", 5) - * ?SECONDS_PER_MINUTE, + StalenessInSec = + config:get_integer("smoosh", "staleness", 5) * + ?SECONDS_PER_MINUTE, Staleness = erlang:convert_time_unit(StalenessInSec, seconds, native), Now = erlang:monotonic_time(), case LastUpdated =:= false orelse Now - LastUpdated > Staleness of - true -> - case smoosh_utils:ignore_db(Object) of true -> - find_channel(Tab, Rest, Object); - _ -> - case get_priority(Channel, Object) of - 0 -> - find_channel(Tab, Rest, Object); - Priority -> - {ok, Pid, Priority} - end - end; - false -> - find_channel(Tab, Rest, Object) + case smoosh_utils:ignore_db(Object) of + true -> + find_channel(Tab, Rest, Object); + _ -> + case get_priority(Channel, Object) of + 0 -> + find_channel(Tab, Rest, Object); + Priority -> + {ok, Pid, Priority} + end + end; + false -> + find_channel(Tab, Rest, Object) end. channel_pid(Tab, Channel) -> - [#channel{pid=Pid}] = ets:lookup(Tab, Channel), + [#channel{pid = Pid}] = ets:lookup(Tab, Channel), Pid. create_missing_channels(State) -> @@ -299,11 +332,11 @@ create_missing_channels(State) -> create_missing_channels(_Tab, []) -> ok; -create_missing_channels(Tab, [Channel|Rest]) -> +create_missing_channels(Tab, [Channel | Rest]) -> case ets:lookup(Tab, Channel) of [] -> {ok, Pid} = smoosh_channel:start_link(Channel), - true = ets:insert(Tab, [#channel{name=Channel, pid=Pid}]); + true = ets:insert(Tab, [#channel{name = Channel, pid = Pid}]); _ -> ok end, @@ -311,44 +344,50 @@ create_missing_channels(Tab, [Channel|Rest]) -> get_priority(Channel, {Shard, GroupId}) -> case couch_index_server:get_index(couch_mrview_index, Shard, GroupId) of - {ok, Pid} -> - try - {ok, ViewInfo} = couch_index:get_info(Pid), - {SizeInfo} = couch_util:get_value(sizes, ViewInfo), - DiskSize = couch_util:get_value(file, SizeInfo), - ActiveSize = couch_util:get_value(active, SizeInfo), - NeedsUpgrade = needs_upgrade(ViewInfo), - get_priority(Channel, DiskSize, ActiveSize, NeedsUpgrade) - catch - exit:{timeout, _} -> - 0 - end; - {not_found, _Reason} -> - 0; - {error, Reason} -> - couch_log:warning("Failed to get group_pid for ~p ~p ~p: ~p", - [Channel, Shard, GroupId, Reason]), - 0 + {ok, Pid} -> + try + {ok, ViewInfo} = couch_index:get_info(Pid), + {SizeInfo} = couch_util:get_value(sizes, ViewInfo), + DiskSize = couch_util:get_value(file, SizeInfo), + ActiveSize = couch_util:get_value(active, SizeInfo), + NeedsUpgrade = needs_upgrade(ViewInfo), + get_priority(Channel, DiskSize, ActiveSize, NeedsUpgrade) + catch + exit:{timeout, _} -> + 0 + end; + {not_found, _Reason} -> + 0; + {error, Reason} -> + couch_log:warning( + "Failed to get group_pid for ~p ~p ~p: ~p", + [Channel, Shard, GroupId, Reason] + ), + 0 end; - get_priority(Channel, {schema, DbName, DDocId}) -> case couch_md_index_manager:get_group_pid(DbName, DDocId) of - {ok, Pid} -> - {ok, SchemaInfo} = couch_md_index:get_info(Pid), - DiskSize = couch_util:get_value(disk_size, SchemaInfo), - DataSize = couch_util:get_value(data_size, SchemaInfo), - get_priority(Channel, DiskSize, DataSize, false); - {error, Reason} -> - couch_log:warning("Failed to get group_pid for ~p ~p ~p: ~p", - [Channel, DbName, DDocId, Reason]), - 0 + {ok, Pid} -> + {ok, SchemaInfo} = couch_md_index:get_info(Pid), + DiskSize = couch_util:get_value(disk_size, SchemaInfo), + DataSize = couch_util:get_value(data_size, SchemaInfo), + get_priority(Channel, DiskSize, DataSize, false); + {error, Reason} -> + couch_log:warning( + "Failed to get group_pid for ~p ~p ~p: ~p", + [Channel, DbName, DDocId, Reason] + ), + 0 end; - get_priority(Channel, DbName) when is_list(DbName) -> get_priority(Channel, ?l2b(DbName)); get_priority(Channel, DbName) when is_binary(DbName) -> {ok, Db} = couch_db:open_int(DbName, []), - try get_priority(Channel, Db) after couch_db:close(Db) end; + try + get_priority(Channel, Db) + after + couch_db:close(Db) + end; get_priority(Channel, Db) -> {ok, DocInfo} = couch_db:get_db_info(Db), {SizeInfo} = couch_util:get_value(sizes, DocInfo), @@ -356,7 +395,7 @@ get_priority(Channel, Db) -> ActiveSize = couch_util:get_value(active, SizeInfo), NeedsUpgrade = needs_upgrade(DocInfo), case db_changed(Channel, DocInfo) of - true -> get_priority(Channel, DiskSize, ActiveSize, NeedsUpgrade); + true -> get_priority(Channel, DiskSize, ActiveSize, NeedsUpgrade); false -> 0 end. @@ -364,30 +403,35 @@ get_priority(Channel, DiskSize, DataSize, NeedsUpgrade) -> Priority = get_priority(Channel), MinSize = to_number(Channel, "min_size", "1048576"), MaxSize = to_number(Channel, "max_size", "infinity"), - DefaultMinPriority = case Priority of "slack" -> "536870912"; _ -> "2.0" end, + DefaultMinPriority = + case Priority of + "slack" -> "536870912"; + _ -> "2.0" + end, MinPriority = to_number(Channel, "min_priority", DefaultMinPriority), MaxPriority = to_number(Channel, "max_priority", "infinity"), - if Priority =:= "upgrade", NeedsUpgrade -> + if + Priority =:= "upgrade", NeedsUpgrade -> 1; - DiskSize =< MinSize -> + DiskSize =< MinSize -> 0; - DiskSize > MaxSize -> + DiskSize > MaxSize -> 0; - DataSize =:= 0 -> + DataSize =:= 0 -> MinPriority; - Priority =:= "ratio", DiskSize/DataSize =< MinPriority -> + Priority =:= "ratio", DiskSize / DataSize =< MinPriority -> 0; - Priority =:= "ratio", DiskSize/DataSize > MaxPriority -> + Priority =:= "ratio", DiskSize / DataSize > MaxPriority -> 0; - Priority =:= "ratio" -> - DiskSize/DataSize; - Priority =:= "slack", DiskSize-DataSize =< MinPriority -> + Priority =:= "ratio" -> + DiskSize / DataSize; + Priority =:= "slack", DiskSize - DataSize =< MinPriority -> 0; - Priority =:= "slack", DiskSize-DataSize > MaxPriority -> + Priority =:= "slack", DiskSize - DataSize > MaxPriority -> 0; - Priority =:= "slack" -> - DiskSize-DataSize; - true -> + Priority =:= "slack" -> + DiskSize - DataSize; + true -> 0 end. @@ -397,19 +441,22 @@ db_changed(Channel, Info) -> true; CompactedSeq -> MinChanges = list_to_integer( - smoosh_utils:get(Channel, "min_changes", "0")), + smoosh_utils:get(Channel, "min_changes", "0") + ), UpdateSeq = couch_util:get_value(update_seq, Info), UpdateSeq - CompactedSeq >= MinChanges end. to_number(Channel, Name, Default) -> case smoosh_utils:get(Channel, Name, Default) of - "infinity" -> infinity; + "infinity" -> + infinity; Value -> try list_to_float(Value) - catch error:badarg -> - list_to_integer(Value) + catch + error:badarg -> + list_to_integer(Value) end end. @@ -441,11 +488,9 @@ needs_upgrade(Props) -> false end. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). - setup_all() -> Ctx = test_util:start_couch([couch_log]), meck:new([config, couch_index, couch_index_server], [passthrough]), @@ -477,7 +522,7 @@ config_change_test_() -> fun t_restart_config_listener/1 ] } -}. + }. get_priority_test_() -> { diff --git a/src/smoosh/src/smoosh_sup.erl b/src/smoosh/src/smoosh_sup.erl index 158498cd5..abd55a2eb 100644 --- a/src/smoosh/src/smoosh_sup.erl +++ b/src/smoosh/src/smoosh_sup.erl @@ -35,4 +35,4 @@ start_link() -> %% =================================================================== init([]) -> - {ok, { {one_for_one, 5, 10}, [?CHILD(smoosh_server, worker)]} }. + {ok, {{one_for_one, 5, 10}, [?CHILD(smoosh_server, worker)]}}. diff --git a/src/smoosh/src/smoosh_utils.erl b/src/smoosh/src/smoosh_utils.erl index fcd0fcd6f..882d3ec84 100644 --- a/src/smoosh/src/smoosh_utils.erl +++ b/src/smoosh/src/smoosh_utils.erl @@ -20,14 +20,15 @@ group_pid({Shard, GroupId}) -> case couch_view_group:open_db_group(Shard, GroupId) of - {ok, Group} -> - try - gen_server:call(couch_view, {get_group_server, Shard, Group}) - catch _:Error -> - {error, Error} - end; - Else -> - Else + {ok, Group} -> + try + gen_server:call(couch_view, {get_group_server, Shard, Group}) + catch + _:Error -> + {error, Error} + end; + Else -> + Else end. get(Channel, Key) -> @@ -37,7 +38,7 @@ get(Channel, Key, Default) -> config:get("smoosh." ++ Channel, Key, Default). split(CSV) -> - re:split(CSV, "\\s*,\\s*", [{return,list}, trim]). + re:split(CSV, "\\s*,\\s*", [{return, list}, trim]). stringify({DbName, GroupId}) -> io_lib:format("~s ~s", [DbName, GroupId]); @@ -48,14 +49,14 @@ stringify(DbName) -> ignore_db({DbName, _GroupName}) -> ignore_db(DbName); -ignore_db(DbName) when is_binary(DbName)-> +ignore_db(DbName) when is_binary(DbName) -> ignore_db(?b2l(DbName)); ignore_db(DbName) when is_list(DbName) -> case config:get("smoosh.ignore", DbName, false) of - "true" -> - true; - _ -> - false + "true" -> + true; + _ -> + false end; ignore_db(Db) -> ignore_db(couch_db:name(Db)). @@ -68,13 +69,12 @@ in_allowed_window(Channel) -> in_allowed_window(From, To) -> {_, {HH, MM, _}} = calendar:universal_time(), case From < To of - true -> - ({HH, MM} >= From) andalso ({HH, MM} < To); - false -> - ({HH, MM} >= From) orelse ({HH, MM} < To) + true -> + ({HH, MM} >= From) andalso ({HH, MM} < To); + false -> + ({HH, MM} >= From) orelse ({HH, MM} < To) end. - parse_time(undefined, Default) -> Default; parse_time(String, Default) -> @@ -82,9 +82,10 @@ parse_time(String, Default) -> [HH, MM] -> try {list_to_integer(HH), list_to_integer(MM)} - catch error:badarg -> - couch_log:error("Malformed compaction schedule configuration: ~s", [String]), - Default + catch + error:badarg -> + couch_log:error("Malformed compaction schedule configuration: ~s", [String]), + Default end; _Else -> couch_log:error("Malformed compaction schedule configuration: ~s", [String]), diff --git a/src/weatherreport/src/weatherreport.erl b/src/weatherreport/src/weatherreport.erl index fd4b1e450..8a46b4a87 100644 --- a/src/weatherreport/src/weatherreport.erl +++ b/src/weatherreport/src/weatherreport.erl @@ -56,20 +56,23 @@ -export([main/1]). -define(OPTS, [ - {etc, $c, "etc", string, "Path to the CouchDB configuration directory"}, - {level, $d, "level", {atom, notice}, "Minimum message severity level (default: notice)"}, - {expert, $e, "expert", undefined, "Perform more detailed diagnostics" }, - {usage, $h, "help", undefined, "Display help/usage" }, - {list, $l, "list", undefined, "Describe available diagnostic tasks" }, - {all_nodes, $a, "all-nodes", undefined, "Run weatherreport on all cluster nodes" }, - {timeout, $t, "timeout", integer, "Timeout value (in ms) for each diagnostic check" } - ]). + {etc, $c, "etc", string, "Path to the CouchDB configuration directory"}, + {level, $d, "level", {atom, notice}, "Minimum message severity level (default: notice)"}, + {expert, $e, "expert", undefined, "Perform more detailed diagnostics"}, + {usage, $h, "help", undefined, "Display help/usage"}, + {list, $l, "list", undefined, "Describe available diagnostic tasks"}, + {all_nodes, $a, "all-nodes", undefined, "Run weatherreport on all cluster nodes"}, + {timeout, $t, "timeout", integer, "Timeout value (in ms) for each diagnostic check"} +]). --define(USAGE_OPTS, [ O || O <- ?OPTS, - element(5,O) =/= undefined]). +-define(USAGE_OPTS, [ + O + || O <- ?OPTS, + element(5, O) =/= undefined +]). %% @doc The main entry point for the weatherreport escript. --spec main(CommandLineArguments::[string()]) -> any(). +-spec main(CommandLineArguments :: [string()]) -> any(). main(Args) -> application:load(weatherreport), @@ -86,15 +89,22 @@ main(Args) -> end. list_checks() -> - Descriptions = [ {weatherreport_util:short_name(Mod), Mod:description()} || - Mod <- weatherreport_check:modules() ], + Descriptions = [ + {weatherreport_util:short_name(Mod), Mod:description()} + || Mod <- weatherreport_check:modules() + ], io:format("Available diagnostic checks:~n~n"), - lists:foreach(fun({Mod, Desc}) -> - io:format(" ~.20s ~s~n", [Mod, Desc]) - end, lists:sort(Descriptions)). + lists:foreach( + fun({Mod, Desc}) -> + io:format(" ~.20s ~s~n", [Mod, Desc]) + end, + lists:sort(Descriptions) + ). usage() -> - weatherreport_getopt:usage(?USAGE_OPTS, "weatherreport ", "[check_name ...]", [{"check_name", "A specific check to run"}]). + weatherreport_getopt:usage(?USAGE_OPTS, "weatherreport ", "[check_name ...]", [ + {"check_name", "A specific check to run"} + ]). run(InputChecks) -> case weatherreport_config:prepare() of @@ -104,47 +114,58 @@ run(InputChecks) -> _ -> ok end, - Checks = case InputChecks of - [] -> - weatherreport_check:modules(); - _ -> - ShortNames = [{weatherreport_util:short_name(Mod), Mod} || Mod <- weatherreport_check:modules() ], - element(1, lists:foldr(fun validate_checks/2, {[], ShortNames}, InputChecks)) - end, - Messages = case application:get_env(weatherreport, all_nodes) of - {ok, true} -> - weatherreport_runner:run(Checks, all); - _ -> - weatherreport_runner:run(Checks) - end, - case Messages of - [] -> - io:format("No diagnostic messages to report.~n"), - halt(0); - _ -> - %% Print the most critical messages first - FilteredMessages = lists:filter(fun({_,Level,_,_}) -> - weatherreport_log:should_log(Level) - end, Messages), - SortedMessages = lists:sort(fun({_, ALevel, _, _}, {_, BLevel, _, _}) -> - weatherreport_log:level(ALevel) =< weatherreport_log:level(BLevel) - end, FilteredMessages), - case SortedMessages of + Checks = + case InputChecks of [] -> - io:format("No diagnostic messages to report.~n"), - halt(0); + weatherreport_check:modules(); _ -> - lists:foreach(fun weatherreport_check:print/1, SortedMessages), - weatherreport_util:flush_stdout(), - halt(1) + ShortNames = [ + {weatherreport_util:short_name(Mod), Mod} + || Mod <- weatherreport_check:modules() + ], + element(1, lists:foldr(fun validate_checks/2, {[], ShortNames}, InputChecks)) end, - halt(1) + Messages = + case application:get_env(weatherreport, all_nodes) of + {ok, true} -> + weatherreport_runner:run(Checks, all); + _ -> + weatherreport_runner:run(Checks) + end, + case Messages of + [] -> + io:format("No diagnostic messages to report.~n"), + halt(0); + _ -> + %% Print the most critical messages first + FilteredMessages = lists:filter( + fun({_, Level, _, _}) -> + weatherreport_log:should_log(Level) + end, + Messages + ), + SortedMessages = lists:sort( + fun({_, ALevel, _, _}, {_, BLevel, _, _}) -> + weatherreport_log:level(ALevel) =< weatherreport_log:level(BLevel) + end, + FilteredMessages + ), + case SortedMessages of + [] -> + io:format("No diagnostic messages to report.~n"), + halt(0); + _ -> + lists:foreach(fun weatherreport_check:print/1, SortedMessages), + weatherreport_util:flush_stdout(), + halt(1) + end, + halt(1) end. validate_checks(Check, {Mods, SNames}) -> case lists:keyfind(Check, 1, SNames) of {Check, Mod} -> - {[Mod|Mods], lists:delete({Check, Mod}, SNames)}; + {[Mod | Mods], lists:delete({Check, Mod}, SNames)}; _ -> io:format("Unknown check '~s' specified, skipping.~n", [Check]), {Mods, SNames} @@ -155,10 +176,10 @@ process_opts(Opts) -> process_opts([], Result) -> Result; -process_opts([H|T], Result) -> +process_opts([H | T], Result) -> process_opts(T, process_option(H, Result)). -process_option({etc,Path}, Result) -> +process_option({etc, Path}, Result) -> application:set_env(weatherreport, etc, filename:absname(Path)), Result; process_option({level, Level}, Result) -> @@ -173,7 +194,8 @@ process_option(expert, Result) -> process_option(all_nodes, Result) -> application:set_env(weatherreport, all_nodes, true), Result; -process_option(list, usage) -> %% Help should have precedence over listing checks +%% Help should have precedence over listing checks +process_option(list, usage) -> usage; process_option(list, _) -> list; diff --git a/src/weatherreport/src/weatherreport_check.erl b/src/weatherreport/src/weatherreport_check.erl index ffac2da0a..65ce1a416 100644 --- a/src/weatherreport/src/weatherreport_check.erl +++ b/src/weatherreport/src/weatherreport_check.erl @@ -58,27 +58,31 @@ -module(weatherreport_check). -export([behaviour_info/1]). --export([check/2, - modules/0, - print/1]). +-export([ + check/2, + modules/0, + print/1 +]). %% @doc The behaviour definition for diagnostic modules. -spec behaviour_info(atom()) -> 'undefined' | [{atom(), arity()}]. behaviour_info(callbacks) -> - [{description, 0}, - {valid, 0}, - {check, 1}, - {format, 1}]; + [ + {description, 0}, + {valid, 0}, + {check, 1}, + {format, 1} + ]; behaviour_info(_) -> undefined. %% @doc Runs the diagnostic in the given module, if it is valid. Returns a %% list of messages that will be printed later using print/1. --spec check(Module::module(), list()) -> [{atom(), module(), term()}]. +-spec check(Module :: module(), list()) -> [{atom(), module(), term()}]. check(Module, Opts) -> case Module:valid() of true -> - [ {Level, Module, Message} || {Level, Message} <- Module:check(Opts) ]; + [{Level, Module, Message} || {Level, Message} <- Module:check(Opts)]; _ -> [] end. @@ -88,16 +92,18 @@ check(Module, Opts) -> -spec modules() -> [module()]. modules() -> {ok, Mods} = application:get_key(weatherreport, modules), - [ M || M <- Mods, - Attr <- M:module_info(attributes), - {behaviour, [?MODULE]} =:= Attr orelse {behavior, [?MODULE]} =:= Attr ]. - + [ + M + || M <- Mods, + Attr <- M:module_info(attributes), + {behaviour, [?MODULE]} =:= Attr orelse {behavior, [?MODULE]} =:= Attr + ]. %% @doc Formats and prints the given message. The diagnostic %% module's format/1 function will be called to provide a %% human-readable message. It should return an iolist() or a 2-tuple %% consisting of a format string and a list of terms. --spec print({Node::atom(), Level::atom(), Module::module(), Data::term()}) -> ok. +-spec print({Node :: atom(), Level :: atom(), Module :: module(), Data :: term()}) -> ok. print({Node, Level, Mod, Data}) -> case Mod:format(Data) of {Format, Terms} -> diff --git a/src/weatherreport/src/weatherreport_check_custodian.erl b/src/weatherreport/src/weatherreport_check_custodian.erl index 36a6a4c8b..924d1c94f 100644 --- a/src/weatherreport/src/weatherreport_check_custodian.erl +++ b/src/weatherreport/src/weatherreport_check_custodian.erl @@ -33,10 +33,12 @@ -module(weatherreport_check_custodian). -behaviour(weatherreport_check). --export([description/0, - valid/0, - check/1, - format/1]). +-export([ + description/0, + valid/0, + check/1, + format/1 +]). -include_lib("eunit/include/eunit.hrl"). diff --git a/src/weatherreport/src/weatherreport_check_disk.erl b/src/weatherreport/src/weatherreport_check_disk.erl index 187db86b7..5361ae632 100644 --- a/src/weatherreport/src/weatherreport_check_disk.erl +++ b/src/weatherreport/src/weatherreport_check_disk.erl @@ -38,18 +38,22 @@ -define(TEST_FILE, "weatherreport.tmp"). %% A dependent chain of permissions checking functions. --define(CHECKPERMFUNS, [fun check_is_dir/1, - fun check_is_writeable/1, - fun check_is_readable/1, - fun check_is_file_readable/1, - fun check_atime/1]). +-define(CHECKPERMFUNS, [ + fun check_is_dir/1, + fun check_is_writeable/1, + fun check_is_readable/1, + fun check_is_file_readable/1, + fun check_atime/1 +]). -include_lib("kernel/include/file.hrl"). --export([description/0, - valid/0, - check/1, - format/1]). +-export([ + description/0, + valid/0, + check/1, + format/1 +]). -spec description() -> string(). description() -> @@ -63,29 +67,41 @@ valid() -> check(_Opts) -> DataDirs = weatherreport_config:data_directories(), %% Add additional disk checks in the function below - lists:flatmap(fun(Dir) -> - check_directory_permissions(Dir) - end, - DataDirs). + lists:flatmap( + fun(Dir) -> + check_directory_permissions(Dir) + end, + DataDirs + ). -spec format(term()) -> {io:format(), [term()]}. format({disk_full, DataDir}) -> - {"Disk containing data directory ~s is full! " - "Please check that it is set to the correct location and that there are not " - "other files using up space intended for Riak.", [DataDir]}; + { + "Disk containing data directory ~s is full! " + "Please check that it is set to the correct location and that there are not " + "other files using up space intended for Riak.", + [DataDir] + }; format({no_data_dir, DataDir}) -> {"Data directory ~s does not exist. Please create it.", [DataDir]}; format({no_write, DataDir}) -> User = weatherreport_config:user(), - {"No write access to data directory ~s. Please make it writeable by the '~s' user.", [DataDir, User]}; + {"No write access to data directory ~s. Please make it writeable by the '~s' user.", [ + DataDir, User + ]}; format({no_read, DataDir}) -> User = weatherreport_config:user(), - {"No read access to data directory ~s. Please make it readable by the '~s' user.", [DataDir, User]}; + {"No read access to data directory ~s. Please make it readable by the '~s' user.", [ + DataDir, User + ]}; format({write_check, File}) -> {"Write-test file ~s is a directory! Please remove it so this test can continue.", [File]}; format({atime, Dir}) -> - {"Data directory ~s is not mounted with 'noatime'. " - "Please remount its disk with the 'noatime' flag to improve performance.", [Dir]}. + { + "Data directory ~s is not mounted with 'noatime'. " + "Please remount its disk with the 'noatime' flag to improve performance.", + [Dir] + }. %%% Private functions @@ -96,12 +112,12 @@ check_directory_permissions(Directory) -> %% returning the first non-ok result. check_directory(_, []) -> []; -check_directory(Directory, [Check|Checks]) -> +check_directory(Directory, [Check | Checks]) -> case Check(Directory) of ok -> check_directory(Directory, Checks); Message -> - [ Message ] + [Message] end. %% Check if the path is actually a directory @@ -130,13 +146,17 @@ check_is_writeable(Directory) -> %% Check if the directory is readable check_is_readable(Directory) -> case file:read_file_info(Directory) of - {ok, #file_info{access=Access}} when Access == read orelse - Access == read_write -> + {ok, #file_info{access = Access}} when + Access == read orelse + Access == read_write + -> ok; {error, eacces} -> {error, {no_read, Directory}}; - {error, Error} when Error == enoent orelse - Error == enotdir -> + {error, Error} when + Error == enoent orelse + Error == enotdir + -> {error, {no_data_dir, Directory}}; _ -> {error, {no_read, Directory}} @@ -146,12 +166,15 @@ check_is_readable(Directory) -> check_is_file_readable(Directory) -> File = filename:join([Directory, ?TEST_FILE]), case file:read_file(File) of - {error, Error} when Error == eacces orelse - Error == enotdir -> + {error, Error} when + Error == eacces orelse + Error == enotdir + -> {error, {no_read, Directory}}; {error, enoent} -> {error, {write_check, File}}; - _ -> ok + _ -> + ok end. %% Check if the directory is mounted with 'noatime' diff --git a/src/weatherreport/src/weatherreport_check_internal_replication.erl b/src/weatherreport/src/weatherreport_check_internal_replication.erl index 7cfdea09e..5dc0bfa64 100644 --- a/src/weatherreport/src/weatherreport_check_internal_replication.erl +++ b/src/weatherreport/src/weatherreport_check_internal_replication.erl @@ -21,15 +21,17 @@ %% ------------------------------------------------------------------- %% @doc Diagnostic that checks the current size of the mem3_sync -%% backlog. The size is printed as an info message if under a defined +%% backlog. The size is printed as an info message if under a defined %% threshold, or as a warning if above the threshold. -module(weatherreport_check_internal_replication). -behaviour(weatherreport_check). --export([description/0, - valid/0, - check/1, - format/1]). +-export([ + description/0, + valid/0, + check/1, + format/1 +]). -define(THRESHOLD, 1000000). @@ -49,8 +51,8 @@ total_to_level(_Total) -> -spec check(list()) -> [{atom(), term()}]. check(_Opts) -> - Backlog = mem3_sync:get_backlog(), - [{total_to_level(Backlog), Backlog}]. + Backlog = mem3_sync:get_backlog(), + [{total_to_level(Backlog), Backlog}]. -spec format(term()) -> {io:format(), [term()]}. format(Backlog) -> diff --git a/src/weatherreport/src/weatherreport_check_ioq.erl b/src/weatherreport/src/weatherreport_check_ioq.erl index 2c25964ef..a0e0b0e60 100644 --- a/src/weatherreport/src/weatherreport_check_ioq.erl +++ b/src/weatherreport/src/weatherreport_check_ioq.erl @@ -26,10 +26,12 @@ -module(weatherreport_check_ioq). -behaviour(weatherreport_check). --export([description/0, - valid/0, - check/1, - format/1]). +-export([ + description/0, + valid/0, + check/1, + format/1 +]). -define(THRESHOLD, 500). @@ -67,9 +69,13 @@ check(Opts) -> true -> case ioq:get_queue_lengths() of Queues when is_map(Queues) -> - Total = maps:fold(fun(_Key, Val, Acc) -> - Val + Acc - end, 0, Queues), + Total = maps:fold( + fun(_Key, Val, Acc) -> + Val + Acc + end, + 0, + Queues + ), [{total_to_level(Total), {ioq_requests, Total, Queues}}]; Error -> [{warning, {ioq_requests_unknown, Error}}] diff --git a/src/weatherreport/src/weatherreport_check_mem3_sync.erl b/src/weatherreport/src/weatherreport_check_mem3_sync.erl index 8dfe41c02..cabca5d50 100644 --- a/src/weatherreport/src/weatherreport_check_mem3_sync.erl +++ b/src/weatherreport/src/weatherreport_check_mem3_sync.erl @@ -26,10 +26,12 @@ -module(weatherreport_check_mem3_sync). -behaviour(weatherreport_check). --export([description/0, - valid/0, - check/1, - format/1]). +-export([ + description/0, + valid/0, + check/1, + format/1 +]). -spec description() -> string(). description() -> diff --git a/src/weatherreport/src/weatherreport_check_membership.erl b/src/weatherreport/src/weatherreport_check_membership.erl index 1ba104116..8fff33c10 100644 --- a/src/weatherreport/src/weatherreport_check_membership.erl +++ b/src/weatherreport/src/weatherreport_check_membership.erl @@ -34,10 +34,12 @@ -module(weatherreport_check_membership). -behaviour(weatherreport_check). --export([description/0, - valid/0, - check/1, - format/1]). +-export([ + description/0, + valid/0, + check/1, + format/1 +]). -include_lib("eunit/include/eunit.hrl"). @@ -62,4 +64,5 @@ check(_Opts) -> -spec format(term()) -> {io:format(), [term()]}. format({not_ring_member, Nodename}) -> - {"Local node ~w is not a member of the cluster. Please check that the -name setting in vm.args is correct.", [Nodename]}. + {"Local node ~w is not a member of the cluster. Please check that the -name setting in vm.args is correct.", + [Nodename]}. diff --git a/src/weatherreport/src/weatherreport_check_memory_use.erl b/src/weatherreport/src/weatherreport_check_memory_use.erl index 5e49ab0d8..04c021381 100644 --- a/src/weatherreport/src/weatherreport_check_memory_use.erl +++ b/src/weatherreport/src/weatherreport_check_memory_use.erl @@ -34,10 +34,12 @@ -module(weatherreport_check_memory_use). -behaviour(weatherreport_check). --export([description/0, - valid/0, - check/1, - format/1]). +-export([ + description/0, + valid/0, + check/1, + format/1 +]). -spec description() -> string(). description() -> @@ -51,7 +53,7 @@ valid() -> check(_Opts) -> Pid = weatherreport_node:pid(), Output = weatherreport_util:run_command("ps -o pmem,rss -p " ++ Pid), - [_,_,Percent, RealSize| _] = string:tokens(Output, "/n \n"), + [_, _, Percent, RealSize | _] = string:tokens(Output, "/n \n"), Messages = [{info, {process_usage, Percent, RealSize}}], case weatherreport_util:binary_to_float(list_to_binary(Percent)) >= 90 of false -> diff --git a/src/weatherreport/src/weatherreport_check_message_queues.erl b/src/weatherreport/src/weatherreport_check_message_queues.erl index ae99ff4dc..e55e9eb52 100644 --- a/src/weatherreport/src/weatherreport_check_message_queues.erl +++ b/src/weatherreport/src/weatherreport_check_message_queues.erl @@ -26,10 +26,12 @@ -module(weatherreport_check_message_queues). -behaviour(weatherreport_check). --export([description/0, - valid/0, - check/1, - format/1]). +-export([ + description/0, + valid/0, + check/1, + format/1 +]). -define(THRESHOLD, 1000). @@ -46,7 +48,8 @@ check(Opts) -> weatherreport_util:check_proc_count( message_queue_len, ?THRESHOLD, - Opts). + Opts + ). -spec format(term()) -> {io:format(), [term()]}. format({high, {Pid, MBoxSize, Info, Pinfo}}) -> diff --git a/src/weatherreport/src/weatherreport_check_node_stats.erl b/src/weatherreport/src/weatherreport_check_node_stats.erl index 27b77cefd..6c3353dc6 100644 --- a/src/weatherreport/src/weatherreport_check_node_stats.erl +++ b/src/weatherreport/src/weatherreport_check_node_stats.erl @@ -26,10 +26,12 @@ -module(weatherreport_check_node_stats). -behaviour(weatherreport_check). --export([description/0, - valid/0, - check/1, - format/1]). +-export([ + description/0, + valid/0, + check/1, + format/1 +]). -define(SAMPLES, 10). -define(T_RUN_QUEUE, 40). @@ -57,9 +59,9 @@ mean_to_message({Statistic, Mean}) -> -spec check(list()) -> [{atom(), term()}]. check(_Opts) -> - SumOfStats = recon:node_stats(?SAMPLES, 100, fun sum_absolute_stats/2, []), - MeanStats = [{K, erlang:round(V / ?SAMPLES)} || {K, V} <- SumOfStats], - lists:map(fun mean_to_message/1, MeanStats). + SumOfStats = recon:node_stats(?SAMPLES, 100, fun sum_absolute_stats/2, []), + MeanStats = [{K, erlang:round(V / ?SAMPLES)} || {K, V} <- SumOfStats], + lists:map(fun mean_to_message/1, MeanStats). -spec format(term()) -> {io:format(), [term()]}. format({Statistic, Value}) -> diff --git a/src/weatherreport/src/weatherreport_check_nodes_connected.erl b/src/weatherreport/src/weatherreport_check_nodes_connected.erl index 7b47de8aa..389054209 100644 --- a/src/weatherreport/src/weatherreport_check_nodes_connected.erl +++ b/src/weatherreport/src/weatherreport_check_nodes_connected.erl @@ -32,10 +32,12 @@ -module(weatherreport_check_nodes_connected). -behaviour(weatherreport_check). --export([description/0, - valid/0, - check/1, - format/1]). +-export([ + description/0, + valid/0, + check/1, + format/1 +]). -spec description() -> string(). description() -> @@ -50,9 +52,12 @@ check(_Opts) -> NodeName = node(), ConnectedNodes = [NodeName | erlang:nodes()], Members = mem3:nodes(), - [{warning, {node_disconnected, N}} || N <- Members, - N =/= NodeName, - lists:member(N, ConnectedNodes) == false]. + [ + {warning, {node_disconnected, N}} + || N <- Members, + N =/= NodeName, + lists:member(N, ConnectedNodes) == false + ]. -spec format(term()) -> {io:format(), [term()]}. format({node_disconnected, Node}) -> diff --git a/src/weatherreport/src/weatherreport_check_process_calls.erl b/src/weatherreport/src/weatherreport_check_process_calls.erl index a12fa61e4..b6a228aeb 100644 --- a/src/weatherreport/src/weatherreport_check_process_calls.erl +++ b/src/weatherreport/src/weatherreport_check_process_calls.erl @@ -25,10 +25,12 @@ -module(weatherreport_check_process_calls). -behaviour(weatherreport_check). --export([description/0, - valid/0, - check/1, - format/1]). +-export([ + description/0, + valid/0, + check/1, + format/1 +]). -define(THRESHOLD, 1000). @@ -56,18 +58,22 @@ fold_processes([{Count, undefined} | T], Acc, Lim, CallType, Opts) -> fold_processes(T, [Message | Acc], Lim - 1, CallType, Opts); fold_processes([{Count, {M, F, A}} | T], Acc, Lim, CallType, Opts) -> Level = total_to_level(Count), - Message = case proplists:get_value(expert, Opts) of - true -> - PidFun = list_to_atom("find_by_" ++ CallType ++ "_call"), - Pids = erlang:apply(recon, PidFun, [M, F]), - Pinfos = lists:map(fun(Pid) -> - Pinfo = recon:info(Pid), - {Pid, Pinfo} - end, lists:sublist(Pids, 10)), - {Level, {process_count, {CallType, Count, M, F, A, Pinfos}}}; - _ -> - {Level, {process_count, {CallType, Count, M, F, A}}} - end, + Message = + case proplists:get_value(expert, Opts) of + true -> + PidFun = list_to_atom("find_by_" ++ CallType ++ "_call"), + Pids = erlang:apply(recon, PidFun, [M, F]), + Pinfos = lists:map( + fun(Pid) -> + Pinfo = recon:info(Pid), + {Pid, Pinfo} + end, + lists:sublist(Pids, 10) + ), + {Level, {process_count, {CallType, Count, M, F, A, Pinfos}}}; + _ -> + {Level, {process_count, {CallType, Count, M, F, A}}} + end, fold_processes(T, [Message | Acc], Lim - 1, CallType, Opts). -spec check(list()) -> [{atom(), term()}]. @@ -81,13 +87,15 @@ check(Opts) -> Opts ), FirstCallCounts = show_first_call_counts(), - lists:reverse(fold_processes( - FirstCallCounts, - CurrentCallMessages, - 10, - "first", - Opts - )). + lists:reverse( + fold_processes( + FirstCallCounts, + CurrentCallMessages, + 10, + "first", + Opts + ) + ). -spec format(term()) -> {io:format(), [term()]}. format({process_count, {CallType, Count, undefined}}) -> @@ -97,57 +105,64 @@ format({process_count, {CallType, Count, M, F, A}}) -> format({process_count, {CallType, Count, M, F, A, Pinfos}}) -> {"~w processes with ~s call ~w:~w/~w ~w", [Count, CallType, M, F, A, Pinfos]}. - %% @doc Show the list of first calls sorted by the number of %% processes that had that initial call. -spec show_first_call_counts() -> [{Count, {Module, Function, Arity}}] when - Count :: pos_integer(), - Module :: atom(), - Function :: atom(), - Arity :: non_neg_integer(). + Count :: pos_integer(), + Module :: atom(), + Function :: atom(), + Arity :: non_neg_integer(). show_first_call_counts() -> - Res = lists:foldl(fun(Pid, Acc) -> - dict:update_counter(first_call(Pid), 1, Acc) - end, dict:new(), processes()), + Res = lists:foldl( + fun(Pid, Acc) -> + dict:update_counter(first_call(Pid), 1, Acc) + end, + dict:new(), + processes() + ), Rev = [{Count, Call} || {Call, Count} <- dict:to_list(Res)], lists:reverse(lists:sort(Rev)). - %% @doc Show the list of current calls sorted by the number of %% processes that had that current call. -spec show_current_call_counts() -> [{Count, {Module, Function, Arity}}] when - Count :: pos_integer(), - Module :: atom(), - Function :: atom(), - Arity :: non_neg_integer(). + Count :: pos_integer(), + Module :: atom(), + Function :: atom(), + Arity :: non_neg_integer(). show_current_call_counts() -> - Res = lists:foldl(fun(Pid, Acc) -> - case process_info(Pid, current_function) of - {current_function, Call} -> - dict:update_counter(Call, 1, Acc); - undefined -> - Acc - end - end, dict:new(), processes()), + Res = lists:foldl( + fun(Pid, Acc) -> + case process_info(Pid, current_function) of + {current_function, Call} -> + dict:update_counter(Call, 1, Acc); + undefined -> + Acc + end + end, + dict:new(), + processes() + ), Rev = [{Count, Call} || {Call, Count} <- dict:to_list(Res)], lists:reverse(lists:sort(Rev)). - %% @doc Find the first function call for a Pid taking into account cases %% where '$initial_call' is set in the process dictionary. -spec first_call(Pid) -> {Module, Function, Arity} when - Pid :: pid(), - Module :: atom(), - Function :: atom(), - Arity :: non_neg_integer(). + Pid :: pid(), + Module :: atom(), + Function :: atom(), + Arity :: non_neg_integer(). first_call(Pid) -> - IC = case process_info(Pid, initial_call) of - {initial_call, IC0} -> IC0; - undefined -> undefined - end, - Dict = case process_info(Pid, dictionary) of - {dictionary, Dict0} -> Dict0; - undefined -> [] - end, + IC = + case process_info(Pid, initial_call) of + {initial_call, IC0} -> IC0; + undefined -> undefined + end, + Dict = + case process_info(Pid, dictionary) of + {dictionary, Dict0} -> Dict0; + undefined -> [] + end, MaybeCall = proplists:get_value('$initial_call', Dict, IC), proplists:get_value(initial_call, Dict, MaybeCall). diff --git a/src/weatherreport/src/weatherreport_check_process_memory.erl b/src/weatherreport/src/weatherreport_check_process_memory.erl index 2f766cdfe..4c7b2c76f 100644 --- a/src/weatherreport/src/weatherreport_check_process_memory.erl +++ b/src/weatherreport/src/weatherreport_check_process_memory.erl @@ -26,10 +26,12 @@ -module(weatherreport_check_process_memory). -behaviour(weatherreport_check). --export([description/0, - valid/0, - check/1, - format/1]). +-export([ + description/0, + valid/0, + check/1, + format/1 +]). -define(THRESHOLD, 104857600). @@ -46,7 +48,8 @@ check(Opts) -> weatherreport_util:check_proc_count( memory, ?THRESHOLD, - Opts). + Opts + ). -spec format(term()) -> {io:format(), [term()]}. format({high, {Pid, Memory, Info, Pinfo}}) -> diff --git a/src/weatherreport/src/weatherreport_check_safe_to_rebuild.erl b/src/weatherreport/src/weatherreport_check_safe_to_rebuild.erl index a7c46c979..86bb1f9c7 100644 --- a/src/weatherreport/src/weatherreport_check_safe_to_rebuild.erl +++ b/src/weatherreport/src/weatherreport_check_safe_to_rebuild.erl @@ -25,10 +25,12 @@ -module(weatherreport_check_safe_to_rebuild). -behaviour(weatherreport_check). --export([description/0, - valid/0, - check/1, - format/1]). +-export([ + description/0, + valid/0, + check/1, + format/1 +]). -spec description() -> string(). description() -> @@ -42,15 +44,17 @@ valid() -> %% that no shard would end up with N<Threshold when the node is offline -spec safe_to_rebuild(atom(), integer()) -> [list()]. safe_to_rebuild(Node, RawThreshold) -> - Threshold = case config:get("couchdb", "maintenance_mode") of - "true" -> - RawThreshold - 1; - _ -> - RawThreshold - end, + Threshold = + case config:get("couchdb", "maintenance_mode") of + "true" -> + RawThreshold - 1; + _ -> + RawThreshold + end, BelowThreshold = fun ({_, _, {_, C}}) when C =< Threshold -> true; - (_) -> false end, + (_) -> false + end, ToKV = fun({Db, Range, Status}) -> {[Db, Range], Status} end, ShardsInDanger = dict:from_list( @@ -77,7 +81,8 @@ safe_to_rebuild(Node, RawThreshold) -> _ -> Acc end; - _ -> Acc + _ -> + Acc end end, [] diff --git a/src/weatherreport/src/weatherreport_check_search.erl b/src/weatherreport/src/weatherreport_check_search.erl index c04c86d63..b7986db2b 100644 --- a/src/weatherreport/src/weatherreport_check_search.erl +++ b/src/weatherreport/src/weatherreport_check_search.erl @@ -27,10 +27,12 @@ -module(weatherreport_check_search). -behaviour(weatherreport_check). --export([description/0, - valid/0, - check/1, - format/1]). +-export([ + description/0, + valid/0, + check/1, + format/1 +]). -spec description() -> string(). description() -> diff --git a/src/weatherreport/src/weatherreport_check_tcp_queues.erl b/src/weatherreport/src/weatherreport_check_tcp_queues.erl index 8e161487f..cc502031b 100644 --- a/src/weatherreport/src/weatherreport_check_tcp_queues.erl +++ b/src/weatherreport/src/weatherreport_check_tcp_queues.erl @@ -26,10 +26,12 @@ -module(weatherreport_check_tcp_queues). -behaviour(weatherreport_check). --export([description/0, - valid/0, - check/1, - format/1]). +-export([ + description/0, + valid/0, + check/1, + format/1 +]). -define(THRESHOLD, 1000000). @@ -55,13 +57,14 @@ sum_queues(Netstats) -> sum_queues([], Acc) -> Acc; sum_queues([Row | Rest], {SumRecvQ, SumSendQ}) -> - {RecvQ, SendQ} = case string:tokens(Row, " ") of - [[$t, $c, $p | _] | _]=Cols -> - {Rq, Sq} = {lists:nth(2, Cols), lists:nth(3, Cols)}, - {list_to_integer(Rq), list_to_integer(Sq)}; - _ -> - {0, 0} - end, + {RecvQ, SendQ} = + case string:tokens(Row, " ") of + [[$t, $c, $p | _] | _] = Cols -> + {Rq, Sq} = {lists:nth(2, Cols), lists:nth(3, Cols)}, + {list_to_integer(Rq), list_to_integer(Sq)}; + _ -> + {0, 0} + end, sum_queues(Rest, {RecvQ + SumRecvQ, SendQ + SumSendQ}). %% @doc Converts the sum of queue lengths to a log message at the approriate @@ -77,7 +80,7 @@ check(_Opts) -> Netstats = weatherreport_util:run_command("netstat"), {SumRecvQ, SumSendQ} = sum_queues(Netstats), [sum_to_message(SumRecvQ, "recv_q"), sum_to_message(SumSendQ, "send_q")]. - + -spec format(term()) -> {io:format(), [term()]}. format({recv_q_high, QLen}) -> {"Total TCP Recv-Q is HIGH: ~w", [QLen]}; diff --git a/src/weatherreport/src/weatherreport_config.erl b/src/weatherreport/src/weatherreport_config.erl index 0538365be..6cf9fd533 100644 --- a/src/weatherreport/src/weatherreport_config.erl +++ b/src/weatherreport/src/weatherreport_config.erl @@ -34,14 +34,16 @@ -module(weatherreport_config). --export([prepare/0, - data_directories/0, - get_vm_env/1, - etc_dir/0, - timeout/0, - node_name/0, - cookie/0, - user/0]). +-export([ + prepare/0, + data_directories/0, + get_vm_env/1, + etc_dir/0, + timeout/0, + node_name/0, + cookie/0, + user/0 +]). %% @doc Prepares appropriate configuration to the weatherreport script %% can run. This is called by the weaterreport module and you do @@ -52,7 +54,7 @@ prepare() -> prepare([]) -> ok; -prepare([Fun|T]) -> +prepare([Fun | T]) -> case Fun() of {error, Reason} -> {error, Reason}; @@ -62,9 +64,9 @@ prepare([Fun|T]) -> %% @doc Determines where CouchDB is configured to store data. Returns a %% list of paths to directories defined by storage backends. --spec data_directories() -> [ file:filename() ]. +-spec data_directories() -> [file:filename()]. data_directories() -> - [config:get("couchdb","view_index_dir"), config:get("couchdb","database_dir")]. + [config:get("couchdb", "view_index_dir"), config:get("couchdb", "database_dir")]. %% @doc Get an -env flag out of the vm.args file. -spec get_vm_env(string()) -> string() | undefined. @@ -110,7 +112,7 @@ etc_dir() -> %% @doc The local node name. Includes whether the node uses short %% or long nodenames for distributed Erlang. --spec node_name() -> {shortnames | longnames, Name::string()}. +-spec node_name() -> {shortnames | longnames, Name :: string()}. node_name() -> case application:get_env(weatherreport, node_name) of undefined -> @@ -140,13 +142,15 @@ load_app_config() -> weatherreport_log:log(node(), debug, "Local node config: ~p~n", [config:all()]). load_vm_args() -> - VmArgs = case init:get_argument(vm_args) of - {ok, [[X]]} -> X; - _ -> - %% This is a backup. If for some reason -vm_args isn't specified - %% then assume it lives in the same dir as app.config - filename:absname("./vm.args", ?MODULE:etc_dir()) - end, + VmArgs = + case init:get_argument(vm_args) of + {ok, [[X]]} -> + X; + _ -> + %% This is a backup. If for some reason -vm_args isn't specified + %% then assume it lives in the same dir as app.config + filename:absname("./vm.args", ?MODULE:etc_dir()) + end, case file:read_file(VmArgs) of {error, Reason} -> @@ -159,32 +163,32 @@ load_vm_args(Bin) when is_binary(Bin) -> load_vm_args(re:split(Bin, "\s*\r?\n\s*", [{return, list}, trim])); load_vm_args([]) -> ok; -load_vm_args([[$#|_]|T]) -> +load_vm_args([[$# | _] | T]) -> load_vm_args(T); -load_vm_args([""|T]) -> +load_vm_args(["" | T]) -> load_vm_args(T); -load_vm_args(["-sname " ++ NodeName|T]) -> +load_vm_args(["-sname " ++ NodeName | T]) -> application:set_env(weatherreport, node_name, {shortnames, string:strip(NodeName)}), load_vm_args(T); -load_vm_args(["-name " ++ NodeName|T]) -> +load_vm_args(["-name " ++ NodeName | T]) -> application:set_env(weatherreport, node_name, {longnames, string:strip(NodeName)}), load_vm_args(T); -load_vm_args(["-setcookie " ++ Cookie|T]) -> +load_vm_args(["-setcookie " ++ Cookie | T]) -> application:set_env(weatherreport, cookie, string:strip(Cookie)), load_vm_args(T); -load_vm_args(["-env " ++ Env|T]) -> +load_vm_args(["-env " ++ Env | T]) -> [Key, Value] = re:split(Env, "\s+", [{return, list}, trim]), add_or_insert_env(vm_env, {Key, Value}), load_vm_args(T); -load_vm_args([[$+|EmuFlags]|T]) -> - [Flag|Rest] = re:split(EmuFlags, "\s+", [{return,list}, trim]), - add_or_insert_env(emu_flags, {[$+|Flag], Rest}), +load_vm_args([[$+ | EmuFlags] | T]) -> + [Flag | Rest] = re:split(EmuFlags, "\s+", [{return, list}, trim]), + add_or_insert_env(emu_flags, {[$+ | Flag], Rest}), load_vm_args(T); -load_vm_args([[$-|InitFlags]|T]) -> - [Flag|Rest] = re:split(InitFlags, "\s+", [{return,list}, trim]), - add_or_insert_env(init_flags, {[$-|Flag], Rest}), +load_vm_args([[$- | InitFlags] | T]) -> + [Flag | Rest] = re:split(InitFlags, "\s+", [{return, list}, trim]), + add_or_insert_env(init_flags, {[$- | Flag], Rest}), load_vm_args(T); -load_vm_args([Line|_]) -> +load_vm_args([Line | _]) -> {error, io_lib:format("Erroneous line in vm.args: ~s", [Line])}. add_or_insert_env(Key, Value) -> @@ -192,5 +196,5 @@ add_or_insert_env(Key, Value) -> undefined -> application:set_env(weatherreport, Key, [Value]); {ok, List} -> - application:set_env(weatherreport, Key, [Value|List]) + application:set_env(weatherreport, Key, [Value | List]) end. diff --git a/src/weatherreport/src/weatherreport_getopt.erl b/src/weatherreport/src/weatherreport_getopt.erl index cbee63cd6..736112630 100644 --- a/src/weatherreport/src/weatherreport_getopt.erl +++ b/src/weatherreport/src/weatherreport_getopt.erl @@ -13,13 +13,15 @@ -export([parse/2, usage/2, usage/3, usage/4]). --export_type([arg_type/0, - arg_value/0, - arg_spec/0, - simple_option/0, - compound_option/0, - option/0, - option_spec/0]). +-export_type([ + arg_type/0, + arg_value/0, + arg_spec/0, + simple_option/0, + compound_option/0, + option/0, + option_spec/0 +]). -define(TAB_LENGTH, 8). %% Indentation of the help messages in number of tabs. @@ -34,7 +36,6 @@ -define(IS_OPT_SPEC(Opt), (tuple_size(Opt) =:= ?OPT_HELP)). - %% Atom indicating the data type that an argument can be converted to. -type arg_type() :: 'atom' | 'binary' | 'boolean' | 'float' | 'integer' | 'string'. %% Data type that an argument can be converted to. @@ -47,16 +48,15 @@ -type option() :: simple_option() | compound_option(). %% Command line option specification. -type option_spec() :: { - Name :: atom(), - Short :: char() | undefined, - Long :: string() | undefined, - ArgSpec :: arg_spec(), - Help :: string() | undefined - }. + Name :: atom(), + Short :: char() | undefined, + Long :: string() | undefined, + ArgSpec :: arg_spec(), + Help :: string() | undefined +}. %% Output streams -type output_stream() :: 'standard_io' | 'standard_error'. - %% @doc Parse the command line options and arguments returning a list of tuples %% and/or atoms using the Erlang convention for sending options to a %% function. @@ -64,19 +64,19 @@ {ok, {[option()], [string()]}} | {error, {Reason :: atom(), Data :: any()}}. parse(OptSpecList, CmdLine) -> try - Args = if - is_integer(hd(CmdLine)) -> - string:tokens(CmdLine, " \t\n"); - true -> - CmdLine - end, + Args = + if + is_integer(hd(CmdLine)) -> + string:tokens(CmdLine, " \t\n"); + true -> + CmdLine + end, parse(OptSpecList, [], [], 0, Args) catch - throw: {error, {_Reason, _Data}} = Error -> + throw:{error, {_Reason, _Data}} = Error -> Error end. - -spec parse([option_spec()], [option()], [string()], integer(), [string()]) -> {ok, {[option()], [string()]}}. %% Process the option terminator. @@ -102,27 +102,28 @@ parse(OptSpecList, OptAcc, ArgAcc, _ArgPos, []) -> %% not present but had default arguments in the specification. {ok, {lists:reverse(append_default_options(OptSpecList, OptAcc)), lists:reverse(ArgAcc)}}. - %% @doc Parse a long option, add it to the option accumulator and continue %% parsing the rest of the arguments recursively. %% A long option can have the following syntax: %% --foo Single option 'foo', no argument %% --foo=bar Single option 'foo', argument "bar" %% --foo bar Single option 'foo', argument "bar" --spec parse_long_option([option_spec()], [option()], [string()], integer(), [string()], string(), string()) -> - {ok, {[option()], [string()]}}. +-spec parse_long_option( + [option_spec()], [option()], [string()], integer(), [string()], string(), string() +) -> + {ok, {[option()], [string()]}}. parse_long_option(OptSpecList, OptAcc, ArgAcc, ArgPos, Args, OptStr, OptArg) -> case split_assigned_arg(OptArg) of {Long, Arg} -> %% Get option that has its argument within the same string %% separated by an equal ('=') character (e.g. "--port=1000"). - parse_long_option_assigned_arg(OptSpecList, OptAcc, ArgAcc, ArgPos, Args, OptStr, Long, Arg); - + parse_long_option_assigned_arg( + OptSpecList, OptAcc, ArgAcc, ArgPos, Args, OptStr, Long, Arg + ); Long -> case lists:keyfind(Long, ?OPT_LONG, OptSpecList) of {Name, _Short, Long, undefined, _Help} -> parse(OptSpecList, [Name | OptAcc], ArgAcc, ArgPos, Args); - {_Name, _Short, Long, _ArgSpec, _Help} = OptSpec -> %% The option argument string is empty, but the option requires %% an argument, so we look into the next string in the list. @@ -133,13 +134,20 @@ parse_long_option(OptSpecList, OptAcc, ArgAcc, ArgPos, Args, OptStr, OptArg) -> end end. - %% @doc Parse an option where the argument is 'assigned' in the same string using %% the '=' character, add it to the option accumulator and continue parsing the %% rest of the arguments recursively. This syntax is only valid for long options. --spec parse_long_option_assigned_arg([option_spec()], [option()], [string()], integer(), - [string()], string(), string(), string()) -> - {ok, {[option()], [string()]}}. +-spec parse_long_option_assigned_arg( + [option_spec()], + [option()], + [string()], + integer(), + [string()], + string(), + string(), + string() +) -> + {ok, {[option()], [string()]}}. parse_long_option_assigned_arg(OptSpecList, OptAcc, ArgAcc, ArgPos, Args, OptStr, Long, Arg) -> case lists:keyfind(Long, ?OPT_LONG, OptSpecList) of {_Name, _Short, Long, ArgSpec, _Help} = OptSpec -> @@ -147,13 +155,18 @@ parse_long_option_assigned_arg(OptSpecList, OptAcc, ArgAcc, ArgPos, Args, OptStr undefined -> throw({error, {invalid_option_arg, OptStr}}); _ -> - parse(OptSpecList, add_option_with_assigned_arg(OptSpec, Arg, OptAcc), ArgAcc, ArgPos, Args) + parse( + OptSpecList, + add_option_with_assigned_arg(OptSpec, Arg, OptAcc), + ArgAcc, + ArgPos, + Args + ) end; false -> throw({error, {invalid_option, OptStr}}) end. - %% @doc Split an option string that may contain an option with its argument %% separated by an equal ('=') character (e.g. "port=1000"). -spec split_assigned_arg(string()) -> {Name :: string(), Arg :: string()} | string(). @@ -167,11 +180,12 @@ split_assigned_arg(OptStr, [Char | Tail], Acc) -> split_assigned_arg(OptStr, [], _Acc) -> OptStr. - %% @doc Retrieve the argument for an option from the next string in the list of %% command-line parameters or set the value of the argument from the argument %% specification (for boolean and integer arguments), if possible. -parse_long_option_next_arg(OptSpecList, OptAcc, ArgAcc, ArgPos, Args, {Name, _Short, _Long, ArgSpec, _Help} = OptSpec) -> +parse_long_option_next_arg( + OptSpecList, OptAcc, ArgAcc, ArgPos, Args, {Name, _Short, _Long, ArgSpec, _Help} = OptSpec +) -> ArgSpecType = arg_spec_type(ArgSpec), case Args =:= [] orelse is_implicit_arg(ArgSpecType, hd(Args)) of true -> @@ -179,14 +193,15 @@ parse_long_option_next_arg(OptSpecList, OptAcc, ArgAcc, ArgPos, Args, {Name, _Sh false -> [Arg | Tail] = Args, try - parse(OptSpecList, [{Name, to_type(ArgSpecType, Arg)} | OptAcc], ArgAcc, ArgPos, Tail) + parse( + OptSpecList, [{Name, to_type(ArgSpecType, Arg)} | OptAcc], ArgAcc, ArgPos, Tail + ) catch error:_ -> throw({error, {invalid_option_arg, {Name, Arg}}}) end end. - %% @doc Parse a short option, add it to the option accumulator and continue %% parsing the rest of the arguments recursively. %% A short option can have the following syntax: @@ -196,7 +211,9 @@ parse_long_option_next_arg(OptSpecList, OptAcc, ArgAcc, ArgPos, Args, {Name, _Sh %% -abc Multiple options: 'a'; 'b'; 'c' %% -bcafoo Multiple options: 'b'; 'c'; 'a' with argument "foo" %% -aaa Multiple repetitions of option 'a' (only valid for options with integer arguments) --spec parse_short_option([option_spec()], [option()], [string()], integer(), [string()], string(), string()) -> +-spec parse_short_option( + [option_spec()], [option()], [string()], integer(), [string()], string(), string() +) -> {ok, {[option()], [string()]}}. parse_short_option(OptSpecList, OptAcc, ArgAcc, ArgPos, Args, OptStr, OptArg) -> parse_short_option(OptSpecList, OptAcc, ArgAcc, ArgPos, Args, OptStr, first, OptArg). @@ -204,45 +221,68 @@ parse_short_option(OptSpecList, OptAcc, ArgAcc, ArgPos, Args, OptStr, OptArg) -> parse_short_option(OptSpecList, OptAcc, ArgAcc, ArgPos, Args, OptStr, OptPos, [Short | Arg]) -> case lists:keyfind(Short, ?OPT_SHORT, OptSpecList) of {Name, Short, _Long, undefined, _Help} -> - parse_short_option(OptSpecList, [Name | OptAcc], ArgAcc, ArgPos, Args, OptStr, first, Arg); - + parse_short_option( + OptSpecList, [Name | OptAcc], ArgAcc, ArgPos, Args, OptStr, first, Arg + ); {_Name, Short, _Long, ArgSpec, _Help} = OptSpec -> %% The option has a specification, so it requires an argument. case Arg of [] -> %% The option argument string is empty, but the option requires %% an argument, so we look into the next string in the list. - parse_short_option_next_arg(OptSpecList, OptAcc, ArgAcc, ArgPos, Args, OptSpec, OptPos); - + parse_short_option_next_arg( + OptSpecList, OptAcc, ArgAcc, ArgPos, Args, OptSpec, OptPos + ); _ -> case is_valid_arg(ArgSpec, Arg) of true -> - parse(OptSpecList, add_option_with_arg(OptSpec, Arg, OptAcc), ArgAcc, ArgPos, Args); + parse( + OptSpecList, + add_option_with_arg(OptSpec, Arg, OptAcc), + ArgAcc, + ArgPos, + Args + ); _ -> - NewOptAcc = case OptPos of - first -> add_option_with_implicit_arg(OptSpec, OptAcc); - _ -> add_option_with_implicit_incrementable_arg(OptSpec, OptAcc) - end, - parse_short_option(OptSpecList, NewOptAcc, ArgAcc, ArgPos, Args, OptStr, next, Arg) + NewOptAcc = + case OptPos of + first -> add_option_with_implicit_arg(OptSpec, OptAcc); + _ -> add_option_with_implicit_incrementable_arg(OptSpec, OptAcc) + end, + parse_short_option( + OptSpecList, NewOptAcc, ArgAcc, ArgPos, Args, OptStr, next, Arg + ) end end; - false -> throw({error, {invalid_option, OptStr}}) end; parse_short_option(OptSpecList, OptAcc, ArgAcc, ArgPos, Args, _OptStr, _OptPos, []) -> parse(OptSpecList, OptAcc, ArgAcc, ArgPos, Args). - %% @doc Retrieve the argument for an option from the next string in the list of %% command-line parameters or set the value of the argument from the argument %% specification (for boolean and integer arguments), if possible. -parse_short_option_next_arg(OptSpecList, OptAcc, ArgAcc, ArgPos, Args, {Name, _Short, _Long, ArgSpec, _Help} = OptSpec, OptPos) -> +parse_short_option_next_arg( + OptSpecList, + OptAcc, + ArgAcc, + ArgPos, + Args, + {Name, _Short, _Long, ArgSpec, _Help} = OptSpec, + OptPos +) -> case Args =:= [] orelse is_implicit_arg(ArgSpec, hd(Args)) of true when OptPos =:= first -> parse(OptSpecList, add_option_with_implicit_arg(OptSpec, OptAcc), ArgAcc, ArgPos, Args); true -> - parse(OptSpecList, add_option_with_implicit_incrementable_arg(OptSpec, OptAcc), ArgAcc, ArgPos, Args); + parse( + OptSpecList, + add_option_with_implicit_incrementable_arg(OptSpec, OptAcc), + ArgAcc, + ArgPos, + Args + ); false -> [Arg | Tail] = Args, try @@ -253,12 +293,11 @@ parse_short_option_next_arg(OptSpecList, OptAcc, ArgAcc, ArgPos, Args, {Name, _S end end. - %% @doc Find the option for the discrete argument in position specified in the %% Pos argument. -spec find_non_option_arg([option_spec()], integer()) -> {value, option_spec()} | false. find_non_option_arg([{_Name, undefined, undefined, _ArgSpec, _Help} = OptSpec | _Tail], 0) -> - {value, OptSpec}; + {value, OptSpec}; find_non_option_arg([{_Name, undefined, undefined, _ArgSpec, _Help} | Tail], Pos) -> find_non_option_arg(Tail, Pos - 1); find_non_option_arg([_Head | Tail], Pos) -> @@ -266,25 +305,25 @@ find_non_option_arg([_Head | Tail], Pos) -> find_non_option_arg([], _Pos) -> false. - %% @doc Append options that were not present in the command line arguments with %% their default arguments. -spec append_default_options([option_spec()], [option()]) -> [option()]. append_default_options([{Name, _Short, _Long, {_Type, DefaultArg}, _Help} | Tail], OptAcc) -> - append_default_options(Tail, - case lists:keymember(Name, 1, OptAcc) of - false -> - [{Name, DefaultArg} | OptAcc]; - _ -> - OptAcc - end); + append_default_options( + Tail, + case lists:keymember(Name, 1, OptAcc) of + false -> + [{Name, DefaultArg} | OptAcc]; + _ -> + OptAcc + end + ); %% For options with no default argument. append_default_options([_Head | Tail], OptAcc) -> append_default_options(Tail, OptAcc); append_default_options([], OptAcc) -> OptAcc. - %% @doc Add an option with argument converting it to the data type indicated by the %% argument specification. -spec add_option_with_arg(option_spec(), string(), [option()]) -> [option()]. @@ -301,7 +340,6 @@ add_option_with_arg({Name, _Short, _Long, ArgSpec, _Help} = OptSpec, Arg, OptAcc add_option_with_implicit_arg(OptSpec, OptAcc) end. - %% @doc Add an option with argument that was part of an assignment expression %% (e.g. "--verbose=3") converting it to the data type indicated by the %% argument specification. @@ -314,7 +352,6 @@ add_option_with_assigned_arg({Name, _Short, _Long, ArgSpec, _Help}, Arg, OptAcc) throw({error, {invalid_option_arg, {Name, Arg}}}) end. - %% @doc Add an option that required an argument but did not have one. Some data %% types (boolean, integer) allow implicit or assumed arguments. -spec add_option_with_implicit_arg(option_spec(), [option()]) -> [option()]. @@ -333,9 +370,9 @@ add_option_with_implicit_arg({Name, _Short, _Long, ArgSpec, _Help}, OptAcc) -> throw({error, {missing_option_arg, Name}}) end. - %% @doc Add an option with an implicit or assumed argument. --spec add_option_with_implicit_incrementable_arg(option_spec() | arg_spec(), [option()]) -> [option()]. +-spec add_option_with_implicit_incrementable_arg(option_spec() | arg_spec(), [option()]) -> + [option()]. add_option_with_implicit_incrementable_arg({Name, _Short, _Long, ArgSpec, _Help}, OptAcc) -> case arg_spec_type(ArgSpec) of boolean -> @@ -357,7 +394,6 @@ add_option_with_implicit_incrementable_arg({Name, _Short, _Long, ArgSpec, _Help} throw({error, {missing_option_arg, Name}}) end. - %% @doc Retrieve the data type form an argument specification. -spec arg_spec_type(arg_spec()) -> arg_type() | undefined. arg_spec_type({Type, _DefaultArg}) -> @@ -365,7 +401,6 @@ arg_spec_type({Type, _DefaultArg}) -> arg_spec_type(Type) when is_atom(Type) -> Type. - %% @doc Convert an argument string to its corresponding data type. -spec to_type(arg_spec() | arg_type(), string()) -> arg_value(). to_type({Type, _DefaultArg}, Arg) -> @@ -394,22 +429,19 @@ to_type(boolean, Arg) -> to_type(_Type, Arg) -> Arg. - -spec is_arg_true(string()) -> boolean(). is_arg_true(Arg) -> (Arg =:= "true") orelse (Arg =:= "t") orelse - (Arg =:= "yes") orelse (Arg =:= "y") orelse - (Arg =:= "on") orelse (Arg =:= "enabled") orelse - (Arg =:= "1"). - + (Arg =:= "yes") orelse (Arg =:= "y") orelse + (Arg =:= "on") orelse (Arg =:= "enabled") orelse + (Arg =:= "1"). -spec is_arg_false(string()) -> boolean(). is_arg_false(Arg) -> (Arg =:= "false") orelse (Arg =:= "f") orelse - (Arg =:= "no") orelse (Arg =:= "n") orelse - (Arg =:= "off") orelse (Arg =:= "disabled") orelse - (Arg =:= "0"). - + (Arg =:= "no") orelse (Arg =:= "n") orelse + (Arg =:= "off") orelse (Arg =:= "disabled") orelse + (Arg =:= "0"). -spec is_valid_arg(arg_spec(), nonempty_string()) -> boolean(). is_valid_arg({Type, _DefaultArg}, Arg) -> @@ -423,7 +455,6 @@ is_valid_arg(float, Arg) -> is_valid_arg(_Type, _Arg) -> true. - -spec is_implicit_arg(arg_spec(), nonempty_string()) -> boolean(). is_implicit_arg({Type, _DefaultArg}, Arg) -> is_implicit_arg(Type, Arg); @@ -434,20 +465,17 @@ is_implicit_arg(integer, Arg) -> is_implicit_arg(_Type, _Arg) -> false. - -spec is_boolean_arg(string()) -> boolean(). is_boolean_arg(Arg) -> LowerArg = string:to_lower(Arg), is_arg_true(LowerArg) orelse is_arg_false(LowerArg). - -spec is_integer_arg(string()) -> boolean(). is_integer_arg("-" ++ Tail) -> is_non_neg_integer_arg(Tail); is_integer_arg(Arg) -> is_non_neg_integer_arg(Arg). - -spec is_non_neg_integer_arg(string()) -> boolean(). is_non_neg_integer_arg([Head | Tail]) when Head >= $0, Head =< $9 -> is_non_neg_integer_arg(Tail); @@ -456,7 +484,6 @@ is_non_neg_integer_arg([_Head | _Tail]) -> is_non_neg_integer_arg([]) -> true. - -spec is_non_neg_float_arg(string()) -> boolean(). is_non_neg_float_arg([Head | Tail]) when (Head >= $0 andalso Head =< $9) orelse Head =:= $. -> is_non_neg_float_arg(Tail); @@ -465,41 +492,43 @@ is_non_neg_float_arg([_Head | _Tail]) -> is_non_neg_float_arg([]) -> true. - %% @doc Show a message on standard_error indicating the command line options and %% arguments that are supported by the program. -spec usage([option_spec()], string()) -> ok. usage(OptSpecList, ProgramName) -> - usage(OptSpecList, ProgramName, standard_error). - + usage(OptSpecList, ProgramName, standard_error). %% @doc Show a message on standard_error or standard_io indicating the command line options and %% arguments that are supported by the program. -spec usage([option_spec()], string(), output_stream() | string()) -> ok. usage(OptSpecList, ProgramName, OutputStream) when is_atom(OutputStream) -> - io:format(OutputStream, "Usage: ~s~s~n~n~s~n", - [ProgramName, usage_cmd_line(OptSpecList), usage_options(OptSpecList)]); + io:format( + OutputStream, + "Usage: ~s~s~n~n~s~n", + [ProgramName, usage_cmd_line(OptSpecList), usage_options(OptSpecList)] + ); %% @doc Show a message on standard_error indicating the command line options and %% arguments that are supported by the program. The CmdLineTail argument %% is a string that is added to the end of the usage command line. usage(OptSpecList, ProgramName, CmdLineTail) -> - usage(OptSpecList, ProgramName, CmdLineTail, standard_error). - + usage(OptSpecList, ProgramName, CmdLineTail, standard_error). %% @doc Show a message on standard_error or standard_io indicating the command line options and %% arguments that are supported by the program. The CmdLineTail argument %% is a string that is added to the end of the usage command line. -spec usage([option_spec()], string(), string(), output_stream() | [{string(), string()}]) -> ok. usage(OptSpecList, ProgramName, CmdLineTail, OutputStream) when is_atom(OutputStream) -> - io:format(OutputStream, "Usage: ~s~s ~s~n~n~s~n", - [ProgramName, usage_cmd_line(OptSpecList), CmdLineTail, usage_options(OptSpecList)]); + io:format( + OutputStream, + "Usage: ~s~s ~s~n~n~s~n", + [ProgramName, usage_cmd_line(OptSpecList), CmdLineTail, usage_options(OptSpecList)] + ); %% @doc Show a message on standard_error indicating the command line options and %% arguments that are supported by the program. The CmdLineTail and OptionsTail %% arguments are a string that is added to the end of the usage command line %% and a list of tuples that are added to the end of the options' help lines. usage(OptSpecList, ProgramName, CmdLineTail, OptionsTail) -> - usage(OptSpecList, ProgramName, CmdLineTail, OptionsTail, standard_error). - + usage(OptSpecList, ProgramName, CmdLineTail, OptionsTail, standard_error). %% @doc Show a message on standard_error or standard_io indicating the command line options and %% arguments that are supported by the program. The CmdLineTail and OptionsTail @@ -508,13 +537,22 @@ usage(OptSpecList, ProgramName, CmdLineTail, OptionsTail) -> -spec usage([option_spec()], string(), string(), [{string(), string()}], output_stream()) -> ok. usage(OptSpecList, ProgramName, CmdLineTail, OptionsTail, OutputStream) -> UsageOptions = lists:foldl( - fun ({Prefix, Help}, Acc) -> - add_option_help(Prefix, Help, Acc) - end, usage_options_reverse(OptSpecList, []), OptionsTail), - io:format(OutputStream, "Usage: ~s~s ~s~n~n~s~n", - [ProgramName, usage_cmd_line(OptSpecList), CmdLineTail, - lists:flatten(lists:reverse(UsageOptions))]). - + fun({Prefix, Help}, Acc) -> + add_option_help(Prefix, Help, Acc) + end, + usage_options_reverse(OptSpecList, []), + OptionsTail + ), + io:format( + OutputStream, + "Usage: ~s~s ~s~n~n~s~n", + [ + ProgramName, + usage_cmd_line(OptSpecList), + CmdLineTail, + lists:flatten(lists:reverse(UsageOptions)) + ] + ). %% @doc Return a string with the syntax for the command line options and %% arguments. @@ -553,7 +591,6 @@ usage_cmd_line([{Name, Short, Long, ArgSpec, _Help} | Tail], Acc) -> usage_cmd_line([], Acc) -> lists:flatten(lists:reverse(Acc)). - %% @doc Return a string with the help message for each of the options and %% arguments. -spec usage_options([option_spec()]) -> string(). @@ -586,7 +623,6 @@ usage_options_reverse([{Name, Short, Long, _ArgSpec, Help} | Tail], Acc) -> usage_options_reverse([], Acc) -> Acc. - %% @doc Add the help message corresponding to an option specification to a list %% with the correct indentation. -spec add_option_help(Prefix :: string(), Help :: string(), Acc :: string()) -> string(). @@ -605,8 +641,6 @@ add_option_help(Prefix, Help, Acc) when is_list(Help), Help =/= [] -> add_option_help(_Opt, _Prefix, Acc) -> Acc. - - %% @doc Return the smallest integral value not less than the argument. -spec ceiling(float()) -> integer(). ceiling(X) -> diff --git a/src/weatherreport/src/weatherreport_log.erl b/src/weatherreport/src/weatherreport_log.erl index 29547da6f..7a511aa1c 100644 --- a/src/weatherreport/src/weatherreport_log.erl +++ b/src/weatherreport/src/weatherreport_log.erl @@ -20,25 +20,33 @@ should_log/1 ]). - -level(debug) -> 7; -level(info) -> 6; -level(notice) -> 5; -level(warn) -> 4; -level(warning) -> 4; -level(err) -> 3; -level(error) -> 3; -level(crit) -> 2; -level(alert) -> 1; -level(emerg) -> 0; -level(panic) -> 0; - +level(debug) -> + 7; +level(info) -> + 6; +level(notice) -> + 5; +level(warn) -> + 4; +level(warning) -> + 4; +level(err) -> + 3; +level(error) -> + 3; +level(crit) -> + 2; +level(alert) -> + 1; +level(emerg) -> + 0; +level(panic) -> + 0; level(I) when is_integer(I), I >= 0, I =< 7 -> I; level(_BadLevel) -> 3. - log(Node, Level, Format, Terms) -> case should_log(Level) of true -> @@ -59,10 +67,11 @@ log(Node, Level, String) -> end. should_log(Level) -> - AppLevel = case application:get_env(weatherreport, log_level) of - undefined -> info; - {ok, L0} -> L0 - end, + AppLevel = + case application:get_env(weatherreport, log_level) of + undefined -> info; + {ok, L0} -> L0 + end, level(AppLevel) >= level(Level). get_prefix(Node, Level) -> diff --git a/src/weatherreport/src/weatherreport_node.erl b/src/weatherreport/src/weatherreport_node.erl index bb5cef007..81d868c39 100644 --- a/src/weatherreport/src/weatherreport_node.erl +++ b/src/weatherreport/src/weatherreport_node.erl @@ -31,21 +31,22 @@ %% node or other members of the cluster. -module(weatherreport_node). --export([can_connect/0, - can_connect_all/0, - pid/0, - local_command/2, - local_command/3, - local_command/4, - multicall/5, - nodename/0 - ]). +-export([ + can_connect/0, + can_connect_all/0, + pid/0, + local_command/2, + local_command/3, + local_command/4, + multicall/5, + nodename/0 +]). %% @doc Calls the given 0-arity module and function on the local %% node and returns the result of that call. %% @equiv local_command(Module, Function, []) %% @see can_connect/0. --spec local_command(Module::atom(), Function::atom()) -> term(). +-spec local_command(Module :: atom(), Function :: atom()) -> term(). local_command(Module, Function) -> local_command(Module, Function, []). @@ -53,7 +54,7 @@ local_command(Module, Function) -> %% on the local node and returns the result of that call. %% @equiv local_command(Module, Function, Args, 5000) %% @see can_connect/0 --spec local_command(Module::atom(), Function::atom(), Args::[term()]) -> term(). +-spec local_command(Module :: atom(), Function :: atom(), Args :: [term()]) -> term(). local_command(Module, Function, Args) -> local_command(Module, Function, Args, weatherreport_config:timeout()). @@ -63,7 +64,8 @@ local_command(Module, Function, Args) -> %% timeout. %% @equiv rpc:call(NodeName, Module, Function, Args, Timeout) %% @see can_connect/0 --spec local_command(Module::atom(), Function::atom(), Args::[term()], Timeout::integer()) -> term(). +-spec local_command(Module :: atom(), Function :: atom(), Args :: [term()], Timeout :: integer()) -> + term(). local_command(Module, Function, Args, Timeout) -> case is_cluster_node() of true -> @@ -86,7 +88,9 @@ local_command(Module, Function, Args, Timeout) -> %% @doc Call rpc:multicall/5 from the local cluster node rather than the %% escript. --spec multicall([node()], Module::atom(), Function::atom(), Args::[term()], Timeout::integer()) -> term(). +-spec multicall( + [node()], Module :: atom(), Function :: atom(), Args :: [term()], Timeout :: integer() +) -> term(). multicall(Nodes, Module, Function, Args, Timeout) -> case local_command(rpc, multicall, [Nodes, Module, Function, Args, Timeout]) of {badrpc, Reason} -> @@ -108,7 +112,8 @@ pid() -> -spec can_connect() -> true | false. can_connect() -> case is_connected() or is_cluster_node() of - true -> true; + true -> + true; false -> weatherreport_log:log( node(), @@ -127,16 +132,18 @@ can_connect_all() -> [] -> true; _ -> false end; - false -> false + false -> + false end. nodename() -> - Name = case weatherreport_config:node_name() of - undefined -> - atom_to_list(node()); - {_, NodeName} -> - NodeName - end, + Name = + case weatherreport_config:node_name() of + undefined -> + atom_to_list(node()); + {_, NodeName} -> + NodeName + end, case string:tokens(Name, "@") of [_Node, _Host] -> list_to_atom(Name); diff --git a/src/weatherreport/src/weatherreport_runner.erl b/src/weatherreport/src/weatherreport_runner.erl index e67940ace..77518d690 100644 --- a/src/weatherreport/src/weatherreport_runner.erl +++ b/src/weatherreport/src/weatherreport_runner.erl @@ -29,13 +29,13 @@ -export([run/1, run/2, format/1]). %% @doc Run the supplied list of checks on the local node --spec run([Module::atom()]) -> [tuple()]. +-spec run([Module :: atom()]) -> [tuple()]. run(Checks) -> weatherreport_node:can_connect(), run(Checks, [weatherreport_node:nodename()]). %% @doc Run the supplied list of checks on the supplied list of cluster nodes --spec run([Module::atom()], [node()] | all) -> [tuple()]. +-spec run([Module :: atom()], [node()] | all) -> [tuple()]. run(Checks, all) -> weatherreport_node:can_connect(), case weatherreport_node:local_command(mem3, nodes, []) of @@ -46,27 +46,33 @@ run(Checks, all) -> end; run(Checks, Nodes) -> CheckOpts = get_check_options(), - lists:flatten(lists:foldl(fun(Mod, Acc) -> - {Resps, BadNodes} = weatherreport_node:multicall( - Nodes, - erlang, - apply, - [fun() -> {node(), weatherreport_check:check(Mod, CheckOpts)} end, []], - weatherreport_config:timeout() - ), - TransformFailedCheck = fun(Node) -> - {node(), crit, weatherreport_runner, {check_failed, Mod, Node}} - end, - FailedChecks = [TransformFailedCheck(Node) || Node <- BadNodes], - TransformResponse = fun - ({badrpc, Error}) -> - [{node(), crit, weatherreport_runner, {badrpc, Mod, Error}}]; - ({Node, Messages}) -> - [{Node, Lvl, Module, Msg} || {Lvl, Module, Msg} <- Messages] - end, - Responses = [TransformResponse(Resp) || Resp <- Resps], - [Responses ++ FailedChecks | Acc] - end, [], Checks)). + lists:flatten( + lists:foldl( + fun(Mod, Acc) -> + {Resps, BadNodes} = weatherreport_node:multicall( + Nodes, + erlang, + apply, + [fun() -> {node(), weatherreport_check:check(Mod, CheckOpts)} end, []], + weatherreport_config:timeout() + ), + TransformFailedCheck = fun(Node) -> + {node(), crit, weatherreport_runner, {check_failed, Mod, Node}} + end, + FailedChecks = [TransformFailedCheck(Node) || Node <- BadNodes], + TransformResponse = fun + ({badrpc, Error}) -> + [{node(), crit, weatherreport_runner, {badrpc, Mod, Error}}]; + ({Node, Messages}) -> + [{Node, Lvl, Module, Msg} || {Lvl, Module, Msg} <- Messages] + end, + Responses = [TransformResponse(Resp) || Resp <- Resps], + [Responses ++ FailedChecks | Acc] + end, + [], + Checks + ) + ). %% @doc Part of the weatherreport_check behaviour. This means that any messages %% returned by this module can be handled via the existing message reporting @@ -80,10 +86,11 @@ format({badrpc, Check, Error}) -> %% Private functions get_check_options() -> - Expert = case application:get_env(weatherreport, expert) of - {ok, true} -> - true; - _ -> - false - end, + Expert = + case application:get_env(weatherreport, expert) of + {ok, true} -> + true; + _ -> + false + end, [{expert, Expert}]. diff --git a/src/weatherreport/src/weatherreport_util.erl b/src/weatherreport/src/weatherreport_util.erl index 450475d42..ef42505e9 100644 --- a/src/weatherreport/src/weatherreport_util.erl +++ b/src/weatherreport/src/weatherreport_util.erl @@ -28,11 +28,13 @@ %% @doc Utility functions for weatherreport. %% @end -module(weatherreport_util). --export([short_name/1, - run_command/1, - binary_to_float/1, - flush_stdout/0, - check_proc_count/3]). +-export([ + short_name/1, + run_command/1, + binary_to_float/1, + flush_stdout/0, + check_proc_count/3 +]). %% @doc Converts a check module name into a short name that can be %% used to refer to a check on the command line. For example, @@ -44,7 +46,7 @@ short_name(Mod) when is_atom(Mod) -> %% @doc Runs a shell command and returns the output. stderr is %% redirected to stdout so its output will be included. --spec run_command(Command::iodata()) -> StdOut::iodata(). +-spec run_command(Command :: iodata()) -> StdOut :: iodata(). run_command(Command) -> weatherreport_log:log( node(), @@ -52,7 +54,7 @@ run_command(Command) -> "Running shell command: ~s", [Command] ), - Port = erlang:open_port({spawn,Command},[exit_status, stderr_to_stdout]), + Port = erlang:open_port({spawn, Command}, [exit_status, stderr_to_stdout]), do_read(Port, []). do_read(Port, Acc) -> @@ -65,14 +67,14 @@ do_read(Port, Acc) -> [StdOut] ), do_read(Port, Acc ++ StdOut); - {Port, {exit_status, _}} -> + {Port, {exit_status, _}} -> %%port_close(Port), Acc; - Other -> + Other -> io:format("~w", [Other]), do_read(Port, Acc) end. - + %% @doc Converts a binary containing a text representation of a float %% into a float type. -spec binary_to_float(binary()) -> float(). @@ -95,17 +97,19 @@ check_proc_count(Key, Threshold, Opts) -> procs_to_messages([], _Threshold, Acc, _Opts) -> Acc; procs_to_messages([{Pid, Value, Info} | T], Threshold, Acc, Opts) -> - Level = case Value > Threshold of - true -> warning; - _ -> info - end, - Message = case {Level, proplists:get_value(expert, Opts)} of - {warning, true} -> - Pinfo = recon:info(Pid), - {warning, {high, {Pid, Value, Info, Pinfo}}}; - {warning, _} -> - {warning, {high, {Pid, Value, Info}}}; - {info, _} -> - {info, {ok, {Pid, Value, Info}}} - end, + Level = + case Value > Threshold of + true -> warning; + _ -> info + end, + Message = + case {Level, proplists:get_value(expert, Opts)} of + {warning, true} -> + Pinfo = recon:info(Pid), + {warning, {high, {Pid, Value, Info, Pinfo}}}; + {warning, _} -> + {warning, {high, {Pid, Value, Info}}}; + {info, _} -> + {info, {ok, {Pid, Value, Info}}} + end, procs_to_messages(T, Threshold, [Message | Acc], Opts). |