diff options
Diffstat (limited to 'src/couch_replicator/src/couch_replicator_utils.erl')
| -rw-r--r-- | src/couch_replicator/src/couch_replicator_utils.erl | 445 |
1 files changed, 239 insertions, 206 deletions
diff --git a/src/couch_replicator/src/couch_replicator_utils.erl b/src/couch_replicator/src/couch_replicator_utils.erl index dbadb3787..b2bc34078 100644 --- a/src/couch_replicator/src/couch_replicator_utils.erl +++ b/src/couch_replicator/src/couch_replicator_utils.erl @@ -13,24 +13,23 @@ -module(couch_replicator_utils). -export([ - parse_rep_doc/2, - replication_id/2, - sum_stats/2, - is_deleted/1, - rep_error_to_binary/1, - get_json_value/2, - get_json_value/3, - pp_rep_id/1, - iso8601/1, - filter_state/3, - normalize_rep/1, - ejson_state_info/1, - get_basic_auth_creds/1, - remove_basic_auth_creds/1, - normalize_basic_auth/1 + parse_rep_doc/2, + replication_id/2, + sum_stats/2, + is_deleted/1, + rep_error_to_binary/1, + get_json_value/2, + get_json_value/3, + pp_rep_id/1, + iso8601/1, + filter_state/3, + normalize_rep/1, + ejson_state_info/1, + get_basic_auth_creds/1, + remove_basic_auth_creds/1, + normalize_basic_auth/1 ]). - -include_lib("ibrowse/include/ibrowse.hrl"). -include_lib("couch/include/couch_db.hrl"). -include("couch_replicator.hrl"). @@ -41,22 +40,20 @@ get_value/3 ]). - rep_error_to_binary(Error) -> couch_util:to_binary(error_reason(Error)). - error_reason({shutdown, Error}) -> error_reason(Error); -error_reason({error, {Error, Reason}}) - when is_atom(Error), is_binary(Reason) -> +error_reason({error, {Error, Reason}}) when + is_atom(Error), is_binary(Reason) +-> io_lib:format("~s: ~s", [Error, Reason]); error_reason({error, Reason}) -> Reason; error_reason(Reason) -> Reason. - get_json_value(Key, Props) -> get_json_value(Key, Props, undefined). @@ -77,7 +74,6 @@ get_json_value(Key, Props, Default) when is_binary(Key) -> Else end. - % pretty-print replication id -spec pp_rep_id(#rep{} | rep_id()) -> string(). pp_rep_id(#rep{id = RepId}) -> @@ -85,34 +81,28 @@ pp_rep_id(#rep{id = RepId}) -> pp_rep_id({Base, Extension}) -> Base ++ Extension. - % NV: TODO: this function is not used outside api wrap module % consider moving it there during final cleanup is_deleted(Change) -> get_json_value(<<"deleted">>, Change, false). - % NV: TODO: proxy some functions which used to be here, later remove % these and replace calls to their respective modules replication_id(Rep, Version) -> couch_replicator_ids:replication_id(Rep, Version). - sum_stats(S1, S2) -> couch_replicator_stats:sum_stats(S1, S2). - parse_rep_doc(Props, UserCtx) -> couch_replicator_docs:parse_rep_doc(Props, UserCtx). - -spec iso8601(erlang:timestamp()) -> binary(). iso8601({_Mega, _Sec, _Micro} = Timestamp) -> {{Y, Mon, D}, {H, Min, S}} = calendar:now_to_universal_time(Timestamp), Format = "~B-~2..0B-~2..0BT~2..0B:~2..0B:~2..0BZ", iolist_to_binary(io_lib:format(Format, [Y, Mon, D, H, Min, S])). - %% Filter replication info ejson by state provided. If it matches return %% the input value, if it doesn't return 'skip'. This is used from replicator %% fabric coordinator and worker. @@ -129,7 +119,6 @@ filter_state(State, States, Info) -> skip end. - remove_basic_auth_from_headers(Headers) -> Headers1 = mochiweb_headers:make(Headers), case mochiweb_headers:get_value("Authorization", Headers1) of @@ -140,14 +129,12 @@ remove_basic_auth_from_headers(Headers) -> maybe_remove_basic_auth(string:to_lower(Basic), Base64, Headers1) end. - maybe_remove_basic_auth("basic", " " ++ Base64, Headers) -> Headers1 = mochiweb_headers:delete_any("Authorization", Headers), {decode_basic_creds(Base64), mochiweb_headers:to_list(Headers1)}; maybe_remove_basic_auth(_, _, Headers) -> {{undefined, undefined}, mochiweb_headers:to_list(Headers)}. - decode_basic_creds(Base64) -> try re:split(base64:decode(Base64), ":", [{return, list}, {parts, 2}]) of [User, Pass] -> @@ -160,47 +147,45 @@ decode_basic_creds(Base64) -> {undefined, undefined} end. - % Normalize a #rep{} record such that it doesn't contain time dependent fields % pids (like httpc pools), and options / props are sorted. This function would % used during comparisons. -spec normalize_rep(#rep{} | nil) -> #rep{} | nil. normalize_rep(nil) -> nil; - -normalize_rep(#rep{} = Rep)-> +normalize_rep(#rep{} = Rep) -> #rep{ source = couch_replicator_api_wrap:normalize_db(Rep#rep.source), target = couch_replicator_api_wrap:normalize_db(Rep#rep.target), - options = Rep#rep.options, % already sorted in make_options/1 + % already sorted in make_options/1 + options = Rep#rep.options, type = Rep#rep.type, view = Rep#rep.view, doc_id = Rep#rep.doc_id, db_name = Rep#rep.db_name }. - -spec ejson_state_info(binary() | nil) -> binary() | null. ejson_state_info(nil) -> null; ejson_state_info(Info) when is_binary(Info) -> {[{<<"error">>, Info}]}; ejson_state_info([]) -> - null; % Status not set yet => null for compatibility reasons + % Status not set yet => null for compatibility reasons + null; ejson_state_info([{_, _} | _] = Info) -> {Info}; ejson_state_info(Info) -> ErrMsg = couch_replicator_utils:rep_error_to_binary(Info), {[{<<"error">>, ErrMsg}]}. - -spec get_basic_auth_creds(#httpdb{}) -> {string(), string()} | {undefined, undefined}. get_basic_auth_creds(#httpdb{auth_props = AuthProps}) -> case couch_util:get_value(<<"basic">>, AuthProps) of undefined -> {undefined, undefined}; - {UserPass} when is_list(UserPass) -> + {UserPass} when is_list(UserPass) -> User = couch_util:get_value(<<"username">>, UserPass), Pass = couch_util:get_value(<<"password">>, UserPass), case {User, Pass} of @@ -213,31 +198,30 @@ get_basic_auth_creds(#httpdb{auth_props = AuthProps}) -> {undefined, undefined} end. - -spec remove_basic_auth_creds(#httpd{}) -> #httpdb{}. remove_basic_auth_creds(#httpdb{auth_props = Props} = HttpDb) -> Props1 = lists:keydelete(<<"basic">>, 1, Props), HttpDb#httpdb{auth_props = Props1}. - -spec set_basic_auth_creds(string(), string(), #httpd{}) -> #httpdb{}. set_basic_auth_creds(undefined, undefined, #httpdb{} = HttpDb) -> HttpDb; -set_basic_auth_creds(User, Pass, #httpdb{} = HttpDb) - when is_list(User), is_list(Pass) -> +set_basic_auth_creds(User, Pass, #httpdb{} = HttpDb) when + is_list(User), is_list(Pass) +-> HttpDb1 = remove_basic_auth_creds(HttpDb), Props = HttpDb1#httpdb.auth_props, - UserPass = {[ - {<<"username">>, list_to_binary(User)}, - {<<"password">>, list_to_binary(Pass)} - ]}, + UserPass = + {[ + {<<"username">>, list_to_binary(User)}, + {<<"password">>, list_to_binary(Pass)} + ]}, Props1 = lists:keystore(<<"basic">>, 1, Props, {<<"basic">>, UserPass}), HttpDb1#httpdb{auth_props = Props1}. - -spec extract_creds_from_url(string()) -> - {ok, {string() | undefined, string() | undefined}, string()} | - {error, term()}. + {ok, {string() | undefined, string() | undefined}, string()} + | {error, term()}. extract_creds_from_url(Url) -> case ibrowse_lib:parse_url(Url) of {error, Error} -> @@ -253,7 +237,6 @@ extract_creds_from_url(Url) -> {ok, {User, Pass}, NoCreds} end. - % Normalize basic auth credentials so they are set only in the auth props % object. If multiple basic auth credentials are provided, the resulting % credentials are picked in the following order. @@ -265,35 +248,39 @@ extract_creds_from_url(Url) -> normalize_basic_auth(#httpdb{} = HttpDb) -> #httpdb{url = Url, headers = Headers} = HttpDb, {HeaderCreds, HeadersNoCreds} = remove_basic_auth_from_headers(Headers), - {UrlCreds, UrlWithoutCreds} = case extract_creds_from_url(Url) of - {ok, Creds = {_, _}, UrlNoCreds} -> - {Creds, UrlNoCreds}; - {error, _Error} -> - % Don't crash replicator if user provided an invalid - % userinfo part - {undefined, undefined} - end, + {UrlCreds, UrlWithoutCreds} = + case extract_creds_from_url(Url) of + {ok, Creds = {_, _}, UrlNoCreds} -> + {Creds, UrlNoCreds}; + {error, _Error} -> + % Don't crash replicator if user provided an invalid + % userinfo part + {undefined, undefined} + end, AuthCreds = {_, _} = get_basic_auth_creds(HttpDb), HttpDb1 = remove_basic_auth_creds(HttpDb#httpdb{ url = UrlWithoutCreds, headers = HeadersNoCreds }), - {User, Pass} = case {AuthCreds, UrlCreds, HeaderCreds} of - {{U, P}, {_, _}, {_, _}} when is_list(U), is_list(P) -> {U, P}; - {{_, _}, {U, P}, {_, _}} when is_list(U), is_list(P) -> {U, P}; - {{_, _}, {_, _}, {U, P}} -> {U, P} - end, + {User, Pass} = + case {AuthCreds, UrlCreds, HeaderCreds} of + {{U, P}, {_, _}, {_, _}} when is_list(U), is_list(P) -> {U, P}; + {{_, _}, {U, P}, {_, _}} when is_list(U), is_list(P) -> {U, P}; + {{_, _}, {_, _}, {U, P}} -> {U, P} + end, set_basic_auth_creds(User, Pass, HttpDb1). - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). remove_basic_auth_from_headers_test_() -> - [?_assertMatch({{User, Pass}, NoAuthHeaders}, - remove_basic_auth_from_headers(Headers)) || - {{User, Pass, NoAuthHeaders}, Headers} <- [ + [ + ?_assertMatch( + {{User, Pass}, NoAuthHeaders}, + remove_basic_auth_from_headers(Headers) + ) + || {{User, Pass, NoAuthHeaders}, Headers} <- [ { {undefined, undefined, []}, [] @@ -332,39 +319,42 @@ remove_basic_auth_from_headers_test_() -> ] ]. - b64creds(User, Pass) -> base64:encode_to_string(User ++ ":" ++ Pass). - normalize_rep_test_() -> { setup, - fun() -> meck:expect(config, get, - fun(_, _, Default) -> Default end) + fun() -> + meck:expect( + config, + get, + fun(_, _, Default) -> Default end + ) end, fun(_) -> meck:unload() end, ?_test(begin - EJson1 = {[ - {<<"source">>, <<"http://host.com/source_db">>}, - {<<"target">>, <<"http://target.local/db">>}, - {<<"doc_ids">>, [<<"a">>, <<"c">>, <<"b">>]}, - {<<"other_field">>, <<"some_value">>} - ]}, + EJson1 = + {[ + {<<"source">>, <<"http://host.com/source_db">>}, + {<<"target">>, <<"http://target.local/db">>}, + {<<"doc_ids">>, [<<"a">>, <<"c">>, <<"b">>]}, + {<<"other_field">>, <<"some_value">>} + ]}, Rep1 = couch_replicator_docs:parse_rep_doc_without_id(EJson1), - EJson2 = {[ - {<<"other_field">>, <<"unrelated">>}, - {<<"target">>, <<"http://target.local/db">>}, - {<<"source">>, <<"http://host.com/source_db">>}, - {<<"doc_ids">>, [<<"c">>, <<"a">>, <<"b">>]}, - {<<"other_field2">>, <<"unrelated2">>} - ]}, + EJson2 = + {[ + {<<"other_field">>, <<"unrelated">>}, + {<<"target">>, <<"http://target.local/db">>}, + {<<"source">>, <<"http://host.com/source_db">>}, + {<<"doc_ids">>, [<<"c">>, <<"a">>, <<"b">>]}, + {<<"other_field2">>, <<"unrelated2">>} + ]}, Rep2 = couch_replicator_docs:parse_rep_doc_without_id(EJson2), ?assertEqual(normalize_rep(Rep1), normalize_rep(Rep2)) end) }. - get_basic_auth_creds_test() -> Check = fun(Props) -> get_basic_auth_creds(#httpdb{auth_props = Props}) @@ -384,7 +374,6 @@ get_basic_auth_creds_test() -> UserPass3 = {[{<<"username">>, <<"u">>}, {<<"password">>, null}]}, ?assertEqual({undefined, undefined}, Check([{<<"basic">>, UserPass3}])). - remove_basic_auth_creds_test() -> Check = fun(Props) -> HttpDb = remove_basic_auth_creds(#httpdb{auth_props = Props}), @@ -395,21 +384,28 @@ remove_basic_auth_creds_test() -> ?assertEqual([{<<"other">>, {[]}}], Check([{<<"other">>, {[]}}])), - ?assertEqual([], Check([ - {<<"basic">>, {[ - {<<"username">>, <<"u">>}, - {<<"password">>, <<"p">>} - ]}} - ])), - - ?assertEqual([{<<"other">>, {[]}}], Check([ - {<<"basic">>, {[ - {<<"username">>, <<"u">>}, - {<<"password">>, <<"p">>} - ]}}, - {<<"other">>, {[]}} - ])). - + ?assertEqual( + [], + Check([ + {<<"basic">>, + {[ + {<<"username">>, <<"u">>}, + {<<"password">>, <<"p">>} + ]}} + ]) + ), + + ?assertEqual( + [{<<"other">>, {[]}}], + Check([ + {<<"basic">>, + {[ + {<<"username">>, <<"u">>}, + {<<"password">>, <<"p">>} + ]}}, + {<<"other">>, {[]}} + ]) + ). set_basic_auth_creds_test() -> Check = fun(User, Pass, Props) -> @@ -419,121 +415,158 @@ set_basic_auth_creds_test() -> ?assertEqual([], Check(undefined, undefined, [])), - ?assertEqual([{<<"other">>, {[]}}], Check(undefined, undefined, - [{<<"other">>, {[]}}])), - - ?assertEqual([ - {<<"basic">>, {[ - {<<"username">>, <<"u">>}, - {<<"password">>, <<"p">>} - ]}} - ], Check("u", "p", [])), - - ?assertEqual([ - {<<"other">>, {[]}}, - {<<"basic">>, {[ - {<<"username">>, <<"u">>}, - {<<"password">>, <<"p">>} - ]}} - ], Check("u", "p", [{<<"other">>, {[]}}])). - + ?assertEqual( + [{<<"other">>, {[]}}], + Check( + undefined, + undefined, + [{<<"other">>, {[]}}] + ) + ), + + ?assertEqual( + [ + {<<"basic">>, + {[ + {<<"username">>, <<"u">>}, + {<<"password">>, <<"p">>} + ]}} + ], + Check("u", "p", []) + ), + + ?assertEqual( + [ + {<<"other">>, {[]}}, + {<<"basic">>, + {[ + {<<"username">>, <<"u">>}, + {<<"password">>, <<"p">>} + ]}} + ], + Check("u", "p", [{<<"other">>, {[]}}]) + ). normalize_basic_creds_test_() -> DefaultHeaders = (#httpdb{})#httpdb.headers, - [?_assertEqual(Expect, normalize_basic_auth(Input)) || {Input, Expect} <- [ - { - #httpdb{url = "http://u:p@x.y/db"}, - #httpdb{url = "http://x.y/db", auth_props = auth_props("u", "p")} - }, - { - #httpdb{url = "http://u:p@h:80/db"}, - #httpdb{url = "http://h:80/db", auth_props = auth_props("u", "p")} - }, - { - #httpdb{url = "https://u:p@h/db"}, - #httpdb{url = "https://h/db", auth_props = auth_props("u", "p")} - }, - { - #httpdb{url = "http://u:p@[2001:db8:a1b:12f9::1]/db"}, - #httpdb{url = "http://[2001:db8:a1b:12f9::1]/db", - auth_props = auth_props("u", "p")} - }, - { - #httpdb{ - url = "http://h/db", - headers = DefaultHeaders ++ [ - {"Authorization", "Basic " ++ b64creds("u", "p")} - ] + [ + ?_assertEqual(Expect, normalize_basic_auth(Input)) + || {Input, Expect} <- [ + { + #httpdb{url = "http://u:p@x.y/db"}, + #httpdb{url = "http://x.y/db", auth_props = auth_props("u", "p")} }, - #httpdb{url = "http://h/db", auth_props = auth_props("u", "p")} - }, - { - #httpdb{ - url = "http://h/db", - headers = DefaultHeaders ++ [ - {"Authorization", "Basic " ++ b64creds("u", "p@")} - ] + { + #httpdb{url = "http://u:p@h:80/db"}, + #httpdb{url = "http://h:80/db", auth_props = auth_props("u", "p")} }, - #httpdb{url = "http://h/db", auth_props = auth_props("u", "p@")} - }, - { - #httpdb{ - url = "http://h/db", - headers = DefaultHeaders ++ [ - {"Authorization", "Basic " ++ b64creds("u", "p@%40")} - ] + { + #httpdb{url = "https://u:p@h/db"}, + #httpdb{url = "https://h/db", auth_props = auth_props("u", "p")} }, - #httpdb{url = "http://h/db", auth_props = auth_props("u", "p@%40")} - }, - { - #httpdb{ - url = "http://h/db", - headers = DefaultHeaders ++ [ - {"aUthoriZation", "bASIC " ++ b64creds("U", "p")} - ] + { + #httpdb{url = "http://u:p@[2001:db8:a1b:12f9::1]/db"}, + #httpdb{ + url = "http://[2001:db8:a1b:12f9::1]/db", + auth_props = auth_props("u", "p") + } }, - #httpdb{url = "http://h/db", auth_props = auth_props("U", "p")} - }, - { - #httpdb{ - url = "http://u1:p1@h/db", - headers = DefaultHeaders ++ [ - {"Authorization", "Basic " ++ b64creds("u2", "p2")} - ] + { + #httpdb{ + url = "http://h/db", + headers = + DefaultHeaders ++ + [ + {"Authorization", "Basic " ++ b64creds("u", "p")} + ] + }, + #httpdb{url = "http://h/db", auth_props = auth_props("u", "p")} }, - #httpdb{url ="http://h/db", auth_props = auth_props("u1", "p1")} - }, - { - #httpdb{ - url = "http://u1:p1@h/db", - auth_props = [{<<"basic">>, {[ - {<<"username">>, <<"u2">>}, - {<<"password">>, <<"p2">>} - ]}}] + { + #httpdb{ + url = "http://h/db", + headers = + DefaultHeaders ++ + [ + {"Authorization", "Basic " ++ b64creds("u", "p@")} + ] + }, + #httpdb{url = "http://h/db", auth_props = auth_props("u", "p@")} }, - #httpdb{url = "http://h/db", auth_props = auth_props("u2", "p2")} - }, - { - #httpdb{ - url = "http://u1:p1@h/db", - auth_props = [{<<"basic">>, {[ - {<<"username">>, <<"u2">>}, - {<<"password">>, <<"p2">>} - ]}}], - headers = DefaultHeaders ++ [ - {"Authorization", "Basic " ++ b64creds("u3", "p3")} - ] + { + #httpdb{ + url = "http://h/db", + headers = + DefaultHeaders ++ + [ + {"Authorization", "Basic " ++ b64creds("u", "p@%40")} + ] + }, + #httpdb{url = "http://h/db", auth_props = auth_props("u", "p@%40")} }, - #httpdb{url = "http://h/db", auth_props = auth_props("u2", "p2")} - } - ]]. - + { + #httpdb{ + url = "http://h/db", + headers = + DefaultHeaders ++ + [ + {"aUthoriZation", "bASIC " ++ b64creds("U", "p")} + ] + }, + #httpdb{url = "http://h/db", auth_props = auth_props("U", "p")} + }, + { + #httpdb{ + url = "http://u1:p1@h/db", + headers = + DefaultHeaders ++ + [ + {"Authorization", "Basic " ++ b64creds("u2", "p2")} + ] + }, + #httpdb{url = "http://h/db", auth_props = auth_props("u1", "p1")} + }, + { + #httpdb{ + url = "http://u1:p1@h/db", + auth_props = [ + {<<"basic">>, + {[ + {<<"username">>, <<"u2">>}, + {<<"password">>, <<"p2">>} + ]}} + ] + }, + #httpdb{url = "http://h/db", auth_props = auth_props("u2", "p2")} + }, + { + #httpdb{ + url = "http://u1:p1@h/db", + auth_props = [ + {<<"basic">>, + {[ + {<<"username">>, <<"u2">>}, + {<<"password">>, <<"p2">>} + ]}} + ], + headers = + DefaultHeaders ++ + [ + {"Authorization", "Basic " ++ b64creds("u3", "p3")} + ] + }, + #httpdb{url = "http://h/db", auth_props = auth_props("u2", "p2")} + } + ] + ]. auth_props(User, Pass) when is_list(User), is_list(Pass) -> - [{<<"basic">>, {[ - {<<"username">>, list_to_binary(User)}, - {<<"password">>, list_to_binary(Pass)} - ]}}]. - + [ + {<<"basic">>, + {[ + {<<"username">>, list_to_binary(User)}, + {<<"password">>, list_to_binary(Pass)} + ]}} + ]. -endif. |