Improve basic auth credentials handling in replicator

This is a port of commit ecd266b0e87f44e1080cabdb4c28e4758f5a4406 from 3.x to
main. Including the same commit message from there for completeness and then
towards the end, there is a description of changes required to port the PR to
main.

Previously, there were two ways to pass in basic auth credentials for endpoints
-- using URL's userinfo part, and encoding them in an `"Authorization": "basic
..."` header. Neither one is ideal for these reasons:

 * Passwords in userinfo don't allow using ":", "@" and other characters.
   However, even after switching to always unquoting them like we did recently
   [1], it could break authentication for usernames or passwords previously
   containing "+" or "%HH" patterns, as "+" might now be decoded to a " ".

 * Base64 encoded headers need an extra step to encode them. Also, quite often
   these encoded headers are confused as being "encrypted" and shared in a
   clear channel.

To improve this, revert the recent commit to unquote URL userinfo parts to
restore backwards compatibility, and introduce a way to pass in basic auth
credentials in the "auth" object. The "auth" object was already added a while
back to allow authentication plugins to store their credentials in it. The
format is:

```
   "source": {
       "url": "https://host/db",
       "auth": {
           "basic": {
               "username":"myuser",
               "password":"mypassword"
           }
       }
   }
```

{"auth" : "basic" : {...}} object is checked first, and if credentials are
provided, they will be used. If they are not then userinfo and basic auth
header will be parsed.

Internally, there was a good amount duplication related to parsing credentials
from userinfo and headers in replication ID generation logic and in the auth
session plugin. As a cleanup, consolidate that logic in the
`couch_replicator_parse` module.

The commit is quite different from the 3.x one for these two reasons:

 * `main` uses two types of replication endpoint "objects": `#httpdb` records
   and `HttpDb` maps. In most cases it uses maps which can be serialized and
   deserialized to and from json. But in lower level, connection handling code
   in couch_replicator_httpc, it uses `#httpdb` records. This explain the need
   to still handle both representations. Auth session plugin, for instance,
   uses the lower level #httpdb records while replicator ID handling code uses
   the map based one.

 * `main` has all the parsing of replication documents and `_replicate` request
   bodies in a separate `couch_replicator_parse`. So, most of the code which
   handles normalizing basic auth creds is there instead of
   `couch_replicator_docs` or `couch_replicator_utils` like it is in 3.x

6 files changed, 521 insertions, 319 deletions

diff --git a/src/couch_replicator/src/couch_replicator_auth_session.erl b/src/couch_replicator/src/couch_replicator_auth_session.erl
index 6ca30c806..b2bf31722 100644
--- a/src/couch_replicator/src/couch_replicator_auth_session.erl
+++ b/src/couch_replicator/src/couch_replicator_auth_session.erl
@@ -79,7 +79,6 @@
 
 -type headers() :: [{string(), string()}].
 -type code() :: non_neg_integer().
--type creds() :: {string() | undefined, string() | undefined}.
 -type time_sec() :: non_neg_integer().
 -type age() :: time_sec() | undefined.
 
@@ -246,59 +245,15 @@ init_state(#httpdb{} = HttpDb) ->
 
 -spec extract_creds(#httpdb{}) ->
     {ok, string(), string(), #httpdb{}} | {error, term()}.
-extract_creds(#httpdb{url = Url, headers = Headers} = HttpDb) ->
-    {{HeadersUser, HeadersPass}, HeadersNoCreds} =
-            couch_replicator_utils:remove_basic_auth_from_headers(Headers),
-    case extract_creds_from_url(Url) of
-        {ok, UrlUser, UrlPass, UrlNoCreds} ->
-            case pick_creds({UrlUser, UrlPass}, {HeadersUser, HeadersPass}) of
-                {ok, User, Pass} ->
-                    HttpDb1 = HttpDb#httpdb{
-                        url = UrlNoCreds,
-                        headers = HeadersNoCreds
-                    },
-                    {ok, User, Pass, HttpDb1};
-                {error, Error} ->
-                    {error, Error}
-            end;
-        {error, Error} ->
-            {error, Error}
-    end.
-
-
-% Credentials could be specified in the url and/or in the headers.
-%  * If no credentials specified return error.
-%  * If specified in url but not in headers, pick url creds.
-%  * Otherwise pick headers creds.
-%
--spec pick_creds(creds(), creds()) ->
-    {ok, string(), string()} | {error, missing_credentials}.
-pick_creds({undefined, _}, {undefined, _}) ->
-    {error, missing_credentials};
-pick_creds({UrlUser, UrlPass}, {undefined, _}) ->
-    {ok, UrlUser, UrlPass};
-pick_creds({_, _}, {HeadersUser, HeadersPass}) ->
-    {ok, HeadersUser, HeadersPass}.
-
-
--spec extract_creds_from_url(string()) ->
-    {ok, string() | undefined, string() | undefined, string()} |
-    {error, term()}.
-extract_creds_from_url(Url) ->
-    case ibrowse_lib:parse_url(Url) of
-        {error, Error} ->
-            {error, Error};
-        #url{username = undefined, password = undefined} ->
-            {ok, undefined, undefined, Url};
-        #url{protocol = Proto, username = User, password = Pass} ->
-            % Excise user and pass parts from the url. Try to keep the host,
-            % port and path as they were in the original.
-            Prefix = lists:concat([Proto, "://", User, ":", Pass, "@"]),
-            Suffix = lists:sublist(Url, length(Prefix) + 1, length(Url) + 1),
-            NoCreds = lists:concat([Proto, "://", Suffix]),
-            User1 = chttpd:unquote(User),
-            Pass1 = chttpd:unquote(Pass),
-            {ok, User1, Pass1, NoCreds}
+extract_creds(#httpdb{} = HttpDb) ->
+    case couch_replicator_utils:get_basic_auth_creds(HttpDb) of
+        {undefined, undefined} ->
+            % Return error. Session plugin should ignore this replication
+            % endpoint as there are no valid creds which can be used
+            {error, missing_credentials};
+        {User, Pass} when is_list(User), is_list(Pass) ->
+            HttpDb1 = remove_basic_auth_creds(HttpDb),
+            {ok, User, Pass, HttpDb1}
     end.
 
 
@@ -567,6 +522,11 @@ b64creds(User, Pass) ->
     base64:encode_to_string(User ++ ":" ++ Pass).
 
 
+remove_basic_auth_creds(#httpdb{auth_props = Props} = HttpDb) ->
+    Props1 = maps:remove(<<"basic">>, Props),
+    HttpDb#httpdb{auth_props = Props1}.
+
+
 -ifdef(TEST).
 
 -include_lib("eunit/include/eunit.hrl").
@@ -582,116 +542,15 @@ get_session_url_test_() ->
     ]].
 
 
-extract_creds_success_test_() ->
-    DefaultHeaders = (#httpdb{})#httpdb.headers,
-    [?_assertEqual({ok, User, Pass, HttpDb2}, extract_creds(HttpDb1)) ||
-        {HttpDb1, {User, Pass, HttpDb2}} <- [
-        {
-            #httpdb{url = "http://u:p@x.y/db"},
-            {"u", "p", #httpdb{url = "http://x.y/db"}}
-        },
-        {
-            #httpdb{url = "http://u%40:p%40@x.y/db"},
-            {"u@", "p@", #httpdb{url = "http://x.y/db"}}
-        },
-        {
-            #httpdb{url = "http://u%40u:p%40p@x.y/db"},
-            {"u@u", "p@p", #httpdb{url = "http://x.y/db"}}
-        },
-        {
-            #httpdb{url = "http://u%40%401:p%40%401@x.y/db"},
-            {"u@@1", "p@@1", #httpdb{url = "http://x.y/db"}}
-        },
-        {
-            #httpdb{url = "http://u%40%2540:p%40%2540@x.y/db"},
-            {"u@%40", "p@%40", #httpdb{url = "http://x.y/db"}}
-        },
-        {
-            #httpdb{url = "http://u:p@h:80/db"},
-            {"u", "p", #httpdb{url = "http://h:80/db"}}
-        },
-        {
-            #httpdb{url = "http://u%3A:p%3A@h:80/db"},
-            {"u:", "p:", #httpdb{url = "http://h:80/db"}}
-        },
-        {
-            #httpdb{url = "https://u:p@h/db"},
-            {"u", "p", #httpdb{url = "https://h/db"}}
-        },
-        {
-            #httpdb{url = "https://u%2F:p%2F@h/db"},
-            {"u/", "p/", #httpdb{url = "https://h/db"}}
-        },
-        {
-            #httpdb{url = "http://u:p@127.0.0.1:5984/db"},
-            {"u", "p", #httpdb{url = "http://127.0.0.1:5984/db"}}
-        },
-        {
-            #httpdb{url = "http://u:p@[2001:db8:a1b:12f9::1]/db"},
-            {"u", "p", #httpdb{url = "http://[2001:db8:a1b:12f9::1]/db"}}
-        },
-        {
-            #httpdb{url = "http://u:p@[2001:db8:a1b:12f9::1]:81/db"},
-            {"u", "p", #httpdb{url = "http://[2001:db8:a1b:12f9::1]:81/db"}}
-        },
-        {
-            #httpdb{url = "http://u:p%3A%2F%5B%5D%40@[2001:db8:a1b:12f9::1]:81/db"},
-            {"u", "p:/[]@", #httpdb{url = "http://[2001:db8:a1b:12f9::1]:81/db"}}
-        },
-        {
-            #httpdb{url = "http://u:p@x.y/db/other?query=Z&query=w"},
-            {"u", "p", #httpdb{url = "http://x.y/db/other?query=Z&query=w"}}
-        },
-        {
-            #httpdb{url = "http://u:p%3F@x.y/db/other?query=Z&query=w"},
-            {"u", "p?", #httpdb{url = "http://x.y/db/other?query=Z&query=w"}}
-        },
-        {
-            #httpdb{
-                url = "http://h/db",
-                headers = DefaultHeaders ++ [
-                    {"Authorization", "Basic " ++ b64creds("u", "p")}
-                ]
-            },
-            {"u", "p", #httpdb{url = "http://h/db"}}
-        },
-        {
-            #httpdb{
-                url = "http://h/db",
-                headers = DefaultHeaders ++ [
-                    {"Authorization", "Basic " ++ b64creds("u", "p@")}
-                ]
-            },
-            {"u", "p@", #httpdb{url = "http://h/db"}}
-        },
-        {
-            #httpdb{
-                url = "http://h/db",
-                headers = DefaultHeaders ++ [
-                    {"Authorization", "Basic " ++ b64creds("u", "p@%40")}
-                ]
-            },
-            {"u", "p@%40", #httpdb{url = "http://h/db"}}
-        },
-        {
-            #httpdb{
-                url = "http://h/db",
-                headers = DefaultHeaders ++ [
-                    {"aUthoriZation", "bASIC " ++ b64creds("U", "p")}
-                ]
-            },
-            {"U", "p", #httpdb{url = "http://h/db"}}
-        },
-        {
-            #httpdb{
-                url = "http://u1:p1@h/db",
-                headers = DefaultHeaders ++ [
-                    {"Authorization", "Basic " ++ b64creds("u2", "p2")}
-                ]
-            },
-            {"u2", "p2", #httpdb{url = "http://h/db"}}
+extract_creds_success_test() ->
+    HttpDb = #httpdb{auth_props = #{
+        <<"basic">> => #{
+            <<"username">> => <<"u2">>,
+            <<"password">> => <<"p2">>
         }
-    ]].
+    }},
+    ?assertEqual({ok, "u2", "p2", #httpdb{}}, extract_creds(HttpDb)),
+    ?assertEqual({error, missing_credentials}, extract_creds(#httpdb{})).
 
 
 cookie_update_test_() ->
@@ -808,7 +667,7 @@ t_process_ok_no_cookie() ->
 t_init_state_fails_on_401() ->
     ?_test(begin
         mock_http_401_response(),
-        {error, Error} = init_state(#httpdb{url = "http://u:p@h"}),
+        {error, Error} = init_state(httpdb("http://u:p@h")),
         SessionUrl =  "http://h/_session",
         ?assertEqual({session_request_unauthorized, SessionUrl, "u"}, Error)
     end).
@@ -818,32 +677,42 @@ t_init_state_401_with_require_valid_user() ->
     ?_test(begin
         mock_http_401_response_with_require_valid_user(),
         ?assertMatch({ok, #httpdb{}, #state{cookie = "Cookie"}},
-            init_state(#httpdb{url = "http://u:p@h"}))
+            init_state(httpdb("http://u:p@h")))
     end).
 
 
 t_init_state_404() ->
     ?_test(begin
         mock_http_404_response(),
-        ?assertEqual(ignore, init_state(#httpdb{url = "http://u:p@h"}))
+        ?assertEqual(ignore, init_state(httpdb("http://u:p@h")))
     end).
 
 
 t_init_state_no_creds() ->
     ?_test(begin
-        ?_assertEqual(ignore, init_state(#httpdb{url = "http://h"}))
+        ?_assertEqual(ignore, init_state(httpdb("http://h")))
     end).
 
 
 t_init_state_http_error() ->
     ?_test(begin
         mock_http_error_response(),
-        {error, Error} = init_state(#httpdb{url = "http://u:p@h"}),
+        {error, Error} = init_state(httpdb("http://u:p@h")),
         SessionUrl = "http://h/_session",
         ?assertEqual({session_request_failed, SessionUrl, "u", x}, Error)
     end).
 
 
+httpdb(Url) when is_list(Url) ->
+    FakeDoc = #{
+        <<"source">> => list_to_binary(Url),
+        <<"target">> => <<"http://somehost">>
+    },
+    {ok, Rep} = couch_replicator_parse:parse_rep(FakeDoc, null),
+    HttpDb = maps:get(<<"source">>, Rep),
+    couch_replicator_api_wrap:db_from_json(HttpDb).
+
+
 setup_all() ->
     meck:expect(couch_replicator_httpc_pool, get_worker, 1, {ok, worker}),
     meck:expect(couch_replicator_httpc_pool, release_worker_sync, 2, ok),
@@ -898,14 +767,6 @@ mock_http_error_response() ->
     meck:expect(ibrowse, send_req_direct, 7, {error, x}).
 
 
-extract_creds_error_test_() ->
-    [?_assertMatch({error, Error}, extract_creds(HttpDb)) ||
-        {HttpDb, Error} <- [
-        {#httpdb{url = "some_junk"}, invalid_uri},
-        {#httpdb{url = "http://h/db"}, missing_credentials}
-    ]].
-
-
 parse_max_age_test_() ->
     [?_assertEqual(R, parse_max_age(mochiweb_headers:make([{"Max-Age", A}])))
         ||  {A, R} <- [
@@ -922,4 +783,29 @@ parse_max_age_test_() ->
     ].
 
 
+remove_basic_auth_creds_test() ->
+    Check = fun(Props) ->
+        HttpDb = remove_basic_auth_creds(#httpdb{auth_props = Props}),
+        HttpDb#httpdb.auth_props
+    end,
+
+    ?assertEqual(#{}, Check(#{})),
+
+    ?assertEqual(#{<<"other">> => #{}}, Check(#{<<"other">> => #{}})),
+
+    ?assertEqual(#{}, Check(#{
+        <<"basic">> => #{
+            <<"username">> => <<"u">>,
+            <<"password">> => <<"p">>
+        }
+    })),
+
+    ?assertEqual(#{<<"other">> => #{}}, Check(#{
+        <<"basic">> => #{
+            <<"username">> => <<"u">>,
+            <<"password">> => <<"p">>
+        },
+        <<"other">> => #{}
+    })).
+
 -endif.
diff --git a/src/couch_replicator/src/couch_replicator_httpc.erl b/src/couch_replicator/src/couch_replicator_httpc.erl
index 28b0f3811..59a79cf93 100644
--- a/src/couch_replicator/src/couch_replicator_httpc.erl
+++ b/src/couch_replicator/src/couch_replicator_httpc.erl
@@ -119,7 +119,13 @@ send_ibrowse_req(#httpdb{headers = BaseHeaders} = HttpDb0, Params) ->
         end
     end,
     {ok, Worker} = couch_replicator_httpc_pool:get_worker(HttpDb#httpdb.httpc_pool),
-    IbrowseOptions = [
+    BasicAuthOpts = case couch_replicator_utils:get_basic_auth_creds(HttpDb) of
+        {undefined, undefined} ->
+            [];
+        {User, Pass} when is_list(User), is_list(Pass) ->
+            [{basic_auth, {User, Pass}}]
+    end,
+    IbrowseOptions = BasicAuthOpts ++ [
         {response_format, binary}, {inactivity_timeout, HttpDb#httpdb.timeout} |
         lists:ukeymerge(1, get_value(ibrowse_options, Params, []),
             HttpDb#httpdb.ibrowse_options)
diff --git a/src/couch_replicator/src/couch_replicator_ids.erl b/src/couch_replicator/src/couch_replicator_ids.erl
index 44b9e47e6..b78140432 100644
--- a/src/couch_replicator/src/couch_replicator_ids.erl
+++ b/src/couch_replicator/src/couch_replicator_ids.erl
@@ -174,22 +174,12 @@ get_rep_endpoint(#{<<"url">> := Url0, <<"headers">> := Headers0}) ->
 
 get_v4_endpoint(#{} = HttpDb) ->
     {remote, Url, Headers} = get_rep_endpoint(HttpDb),
-    {{UserFromHeaders, _}, HeadersWithoutBasicAuth} =
-        couch_replicator_utils:remove_basic_auth_from_headers(Headers),
-    {UserFromUrl, Host, NonDefaultPort, Path} = get_v4_url_info(Url),
-    User = pick_defined_value([UserFromUrl, UserFromHeaders]),
+    {User, _} = couch_replicator_utils:get_basic_auth_creds(HttpDb),
+    {Host, NonDefaultPort, Path} = get_v4_url_info(Url),
     OAuth = undefined, % Keep this to ensure checkpoints don't change
-    {remote, User, Host, NonDefaultPort, Path, HeadersWithoutBasicAuth, OAuth}.
+    {remote, User, Host, NonDefaultPort, Path, Headers, OAuth}.
 
 
-pick_defined_value(Values) ->
-    case [V || V <- Values, V /= undefined] of
-        [] ->
-            undefined;
-        DefinedValues ->
-            hd(DefinedValues)
-    end.
-
 
 get_v4_url_info(Url) when is_binary(Url) ->
     get_v4_url_info(binary_to_list(Url));
@@ -198,16 +188,16 @@ get_v4_url_info(Url) ->
         {error, invalid_uri} ->
             % Tolerate errors here to avoid a bad user document
             % crashing the replicator
-            {undefined, Url, undefined, undefined};
+            {Url, undefined, undefined};
         #url{
             protocol = Schema,
-            username = User,
+
             host = Host,
             port = Port,
             path = Path
         } ->
             NonDefaultPort = get_non_default_port(Schema, Port),
-            {User, Host, NonDefaultPort, Path}
+            {Host, NonDefaultPort, Path}
     end.
 
 
@@ -239,82 +229,82 @@ replication_id_convert_test_() ->
 
 http_v4_endpoint_test_() ->
     [?_assertMatch({remote, User, Host, Port, Path, HeadersNoAuth, undefined},
-        get_v4_endpoint(#{<<"url">> => Url, <<"headers">> => Headers})) ||
-            {{User, Host, Port, Path, HeadersNoAuth}, {Url, Headers}} <- [
+        get_v4_endpoint(HttpDb)) ||
+            {{User, Host, Port, Path, HeadersNoAuth}, HttpDb} <- [
                 {
                     {undefined, "host", default, "/", []},
-                    {<<"http://host">>, #{}}
+                    httpdb("http://host")
                 },
                 {
                     {undefined, "host", default, "/", []},
-                    {<<"https://host">>, #{}}
+                    httpdb("https://host")
                 },
                 {
                     {undefined, "host", default, "/", []},
-                    {<<"http://host:5984">>, #{}}
+                    httpdb("http://host:5984")
                 },
                 {
                     {undefined, "host", 1, "/", []},
-                    {<<"http://host:1">>, #{}}
+                    httpdb("http://host:1")
                 },
                 {
                     {undefined, "host", 2, "/", []},
-                    {<<"https://host:2">>, #{}}
+                    httpdb("https://host:2")
                 },
                 {
                     {undefined, "host", default, "/", [{"h", "v"}]},
-                    {<<"http://host">>, #{<<"h">> => <<"v">>}}
+                    httpdb("http://host", undefined, undefined, #{"h" => "v"})
                 },
                 {
                     {undefined, "host", default, "/a/b", []},
-                    {<<"http://host/a/b">>, #{}}
-                },
-                {
-                    {"user", "host", default, "/", []},
-                    {<<"http://user:pass@host">>, #{}}
-                },
-                {
-                    {"user", "host", 3, "/", []},
-                    {<<"http://user:pass@host:3">>, #{}}
+                    httpdb("http://host/a/b")
                 },
                 {
                     {"user", "host", default, "/", []},
-                    {<<"http://user:newpass@host">>, #{}}
+                    httpdb("http://host", "user", "pass")
                 },
                 {
                     {"user", "host", default, "/", []},
-                    {<<"http://host">>, basic_auth(<<"user">>, <<"pass">>)}
-                },
-                {
-                    {"user", "host", default, "/", []},
-                    {<<"http://host">>, basic_auth(<<"user">>, <<"newpass">>)}
-                },
-                {
-                    {"user1", "host", default, "/", []},
-                    {<<"http://user1:pass1@host">>, basic_auth(<<"user2">>,
-                        <<"pass2">>)}
-                },
-                {
-                    {"user", "host", default, "/", [{"h", "v"}]},
-                    {<<"http://host">>, maps:merge(#{<<"h">> => <<"v">>},
-                        basic_auth(<<"user">>, <<"pass">>))}
+                    httpdb("http://host", "user", "newpass")
                 },
                 {
-                    {undefined, "random_junk", undefined, undefined},
-                    {<<"random_junk">>, #{}}
-                },
-                {
-                    {undefined, "host", default, "/", []},
-                    {<<"http://host">>, #{<<"Authorization">> =>
-                        <<"Basic bad">>}}
+                    {"user2", "host", default, "/", [{"h", "v"}]},
+                    httpdb("http://host", "user2", "pass2", #{"h" => "v"})
                 }
         ]
     ].
 
 
-basic_auth(User, Pass) ->
-    B64Auth = base64:encode(<<User/binary, ":", Pass/binary>>),
-    #{<<"Authorization">> => <<"Basic ", B64Auth/binary>>}.
+httpdb(Url) ->
+    #{
+        <<"url">> => list_to_binary(Url),
+        <<"auth_props">> => #{},
+        <<"headers">> => #{}
+    }.
+
+
+httpdb(Url, User, Pass) ->
+    #{
+        <<"url">> => list_to_binary(Url),
+        <<"auth_props">> => #{
+            <<"basic">> => #{
+                <<"username">> => list_to_binary(User),
+                <<"password">> => list_to_binary(Pass)
+            }
+        },
+        <<"headers">> => #{}
+    }.
+
+
+httpdb(Url, User, Pass, #{} = Headers) ->
+    HttpDb1 = case {User, Pass} of
+        {undefined, undefined} -> httpdb(Url);
+        {User, Pass} -> httpdb(Url, User, Pass)
+    end,
+    Headers1 = maps:fold(fun(K, V, Acc) ->
+        Acc#{list_to_binary(K) => list_to_binary(V)}
+    end, #{}, Headers),
+    HttpDb1#{<<"headers">> => Headers1}.
 
 
 version4_matches_couchdb3_test_() ->
@@ -352,4 +342,10 @@ id_matches_couchdb3(_) ->
     ?assertEqual(BaseId3x, BaseId).
 
 
+auth_props(User, Pass) when is_list(User), is_list(Pass) ->
+    [{<<"basic">>, {[
+       {<<"username">>, list_to_binary(User)},
+       {<<"password">>, list_to_binary(Pass)}
+    ]}}].
+
 -endif.
diff --git a/src/couch_replicator/src/couch_replicator_job.erl b/src/couch_replicator/src/couch_replicator_job.erl
index 951471a14..381fe5739 100644
--- a/src/couch_replicator/src/couch_replicator_job.erl
+++ b/src/couch_replicator/src/couch_replicator_job.erl
@@ -1727,8 +1727,8 @@ t_format_status(_) ->
     },
     Format = format_status(opts_ignored, [pdict, State]),
     FmtOptions = proplists:get_value(options, Format),
-    ?assertEqual("http://u:*****@h1/d1/", proplists:get_value(source, Format)),
-    ?assertEqual("http://u:*****@h2/d2/", proplists:get_value(target, Format)),
+    ?assertEqual("http://h1/d1/", proplists:get_value(source, Format)),
+    ?assertEqual("http://h2/d2/", proplists:get_value(target, Format)),
     ?assertEqual(<<"base+ext">>, proplists:get_value(rep_id, Format)),
     ?assertEqual(true, maps:get(<<"create_target">>, FmtOptions)),
     ?assertEqual(<<"mydoc">>, proplists:get_value(doc_id, Format)),
diff --git a/src/couch_replicator/src/couch_replicator_parse.erl b/src/couch_replicator/src/couch_replicator_parse.erl
index ac25bee41..38d50d437 100644
--- a/src/couch_replicator/src/couch_replicator_parse.erl
+++ b/src/couch_replicator/src/couch_replicator_parse.erl
@@ -205,7 +205,7 @@ parse_rep_db(#{} = Endpoint, #{} = ProxyParams, #{} = Options) ->
     }, ProxyParams),
     SslParams = ssl_params(Url),
 
-    #{
+    HttpDb = #{
         <<"url">> => Url,
         <<"auth_props">> => AuthProps,
         <<"headers">> => Headers,
@@ -214,7 +214,8 @@ parse_rep_db(#{} = Endpoint, #{} = ProxyParams, #{} = Options) ->
         <<"http_connections">> => maps:get(<<"http_connections">>, Options),
         <<"retries">> => maps:get(<<"retries_per_request">>, Options),
         <<"proxy_url">> => ProxyUrl
-    };
+    },
+    normalize_basic_auth(HttpDb);
 
 parse_rep_db(<<"http://", _/binary>> = Url, Proxy, Options) ->
     parse_rep_db(#{<<"url">> => Url}, Proxy, Options);
@@ -475,6 +476,117 @@ ssl_verify_options(false) ->
     }.
 
 
+-spec set_basic_auth_creds(string(), string(), map()) -> map().
+set_basic_auth_creds(undefined, undefined, #{}= HttpDb) ->
+    HttpDb;
+set_basic_auth_creds(User, Pass, #{} = HttpDb)
+        when is_list(User), is_list(Pass) ->
+    #{<<"auth_props">> := AuthProps}  = HttpDb,
+    UserPass = #{
+        <<"username">> => list_to_binary(User),
+        <<"password">> => list_to_binary(Pass)
+    },
+    AuthProps1 = AuthProps#{<<"basic">> => UserPass},
+    HttpDb#{<<"auth_props">> := AuthProps1}.
+
+
+-spec extract_creds_from_url(binary()) ->
+    {ok, {string() | undefined, string() | undefined}, string()} |
+    {error, term()}.
+extract_creds_from_url(Url0) ->
+    Url = binary_to_list(Url0),
+    case ibrowse_lib:parse_url(Url) of
+        {error, Error} ->
+            {error, Error};
+        #url{username = undefined, password = undefined} ->
+            {ok, {undefined, undefined}, Url0};
+        #url{protocol = Proto, username = User, password = Pass} ->
+            % Excise user and pass parts from the url. Try to keep the host,
+            % port and path as they were in the original.
+            Prefix = lists:concat([Proto, "://", User, ":", Pass, "@"]),
+            Suffix = lists:sublist(Url, length(Prefix) + 1, length(Url) + 1),
+            NoCreds = lists:concat([Proto, "://", Suffix]),
+            {ok, {User, Pass}, list_to_binary(NoCreds)}
+    end.
+
+
+% Normalize basic auth credentials so they are set only in the auth props
+% object. If multiple basic auth credentials are provided, the resulting
+% credentials are picked in the following order.
+%  1) {"auth": "basic": {"username":.., "password": ...} ...}
+%  2) URL userinfo part
+%  3) "Authentication" : "basic $base64" headers
+%
+-spec normalize_basic_auth(map()) -> map().
+normalize_basic_auth(#{} = HttpDb) ->
+    #{
+        <<"url">> := Url,
+        <<"headers">> := Headers
+    } = HttpDb,
+    {HeaderCreds, HeadersNoCreds} = remove_basic_auth_from_headers(Headers),
+    {UrlCreds, UrlWithoutCreds} = case extract_creds_from_url(Url) of
+        {ok, Creds = {_, _}, UrlNoCreds} ->
+            {Creds, UrlNoCreds};
+        {error, _Error} ->
+            % Don't crash replicator if user provided an invalid
+            % userinfo part
+            {undefined, undefined}
+    end,
+    AuthCreds = {_, _} = couch_replicator_utils:get_basic_auth_creds(HttpDb),
+    HttpDb1 = HttpDb#{
+        <<"url">> := UrlWithoutCreds,
+        <<"headers">> := HeadersNoCreds
+    },
+    {User, Pass} = case {AuthCreds, UrlCreds, HeaderCreds} of
+        {{U, P}, {_, _}, {_, _}} when is_list(U), is_list(P) -> {U, P};
+        {{_, _}, {U, P}, {_, _}} when is_list(U), is_list(P) -> {U, P};
+        {{_, _}, {_, _}, {U, P}} -> {U, P}
+    end,
+    set_basic_auth_creds(User, Pass, HttpDb1).
+
+
+remove_basic_auth_from_headers(#{} = HeadersMap) ->
+    % Headers are passed in a map however mochiweb_headers expects them to be
+    % lists so we transform them to lists first, then back to maps
+    Headers = maps:fold(fun(K, V, Acc) ->
+        [{binary_to_list(K), binary_to_list(V)} | Acc]
+    end, [], HeadersMap),
+    Headers1 = mochiweb_headers:make(Headers),
+    case mochiweb_headers:get_value("Authorization", Headers1) of
+        undefined ->
+            {{undefined, undefined}, HeadersMap};
+        Auth ->
+            {Basic, B64} = lists:splitwith(fun(X) -> X =/= $\s end, Auth),
+            BasicLower = string:to_lower(Basic),
+            Result = maybe_remove_basic_auth(BasicLower, B64, Headers1),
+            {{User, Pass}, Headers2} = Result,
+            HeadersMapResult = lists:foldl(fun({K, V}, Acc) ->
+                Acc#{list_to_binary(K) => list_to_binary(V)}
+            end, #{}, Headers2),
+            {{User, Pass}, HeadersMapResult}
+    end.
+
+
+maybe_remove_basic_auth("basic", " " ++ Base64, Headers) ->
+    Headers1 = mochiweb_headers:delete_any("Authorization", Headers),
+    {decode_basic_creds(Base64), mochiweb_headers:to_list(Headers1)};
+maybe_remove_basic_auth(_, _, Headers) ->
+    {{undefined, undefined}, mochiweb_headers:to_list(Headers)}.
+
+
+decode_basic_creds(Base64) ->
+    try re:split(base64:decode(Base64), ":", [{return, list}, {parts, 2}]) of
+        [User, Pass] ->
+            {User, Pass};
+        _ ->
+            {undefined, undefined}
+    catch
+        % Tolerate invalid B64 values here to avoid crashing replicator
+        error:function_clause ->
+            {undefined, undefined}
+    end.
+
+
 -ifdef(TEST).
 
 -include_lib("couch/include/couch_eunit.hrl").
@@ -565,4 +677,243 @@ t_error_on_local_endpoint(_) ->
     ?assertThrow({bad_rep_doc, Expect}, parse_rep_doc(RepDoc)).
 
 
+remove_basic_auth_from_headers_test_() ->
+    B64 = list_to_binary(b64creds("user", "pass")),
+    [?_assertEqual({{User, Pass}, NoAuthHeaders},
+        remove_basic_auth_from_headers(Headers)) ||
+        {{User, Pass, NoAuthHeaders}, Headers} <- [
+            {
+                {undefined, undefined, #{}},
+                #{}
+            },
+            {
+                {undefined, undefined, #{<<"h">> => <<"v">>}},
+                #{<<"h">> => <<"v">>}
+            },
+            {
+                {undefined, undefined, #{<<"Authorization">> => <<"junk">>}},
+                #{<<"Authorization">> => <<"junk">>}
+            },
+            {
+                {undefined, undefined, #{}},
+                #{<<"Authorization">> => <<"basic X">>}
+            },
+            {
+                {"user", "pass", #{}},
+                #{<<"Authorization">> => <<"Basic ", B64/binary>>}
+            },
+            {
+                {"user", "pass", #{}},
+                #{<<"AuThorization">> => <<"Basic ", B64/binary>>}
+            },
+            {
+                {"user", "pass", #{}},
+                #{<<"Authorization">> => <<"bAsIc ", B64/binary>>}
+            },
+            {
+                {"user", "pass", #{<<"h">> => <<"v">>}},
+                #{
+                    <<"Authorization">> => <<"Basic ", B64/binary>>,
+                    <<"h">> => <<"v">>
+                }
+            }
+        ]
+    ].
+
+
+b64creds(User, Pass) ->
+    base64:encode_to_string(User ++ ":" ++ Pass).
+
+
+set_basic_auth_creds_test() ->
+    Check = fun(User, Pass, Props) ->
+        HttpDb = #{<<"auth_props">> => Props},
+        HttpDb1 = set_basic_auth_creds(User, Pass, HttpDb),
+        maps:get(<<"auth_props">>, HttpDb1)
+    end,
+
+    ?assertEqual(#{}, Check(undefined, undefined, #{})),
+
+    ?assertEqual(#{<<"other">> => #{}}, Check(undefined, undefined,
+        #{<<"other">> => #{}})),
+
+    ?assertEqual(#{
+        <<"basic">> => #{
+            <<"username">> => <<"u">>,
+            <<"password">> => <<"p">>
+        }
+    }, Check("u", "p", #{})),
+
+    ?assertEqual(#{
+        <<"other">> => #{},
+        <<"basic">> => #{
+            <<"username">> => <<"u">>,
+            <<"password">> => <<"p">>
+        }
+    }, Check("u", "p", #{<<"other">> => #{}})).
+
+
+normalize_basic_creds_test_() ->
+    DefaultHeaders = couch_replicator_utils:default_headers_map(),
+    [?_assertEqual(Expect, normalize_basic_auth(Input)) || {Input, Expect} <- [
+        {
+            #{
+                <<"url">> => <<"http://u:p@x.y/db">>,
+                <<"auth_props">> => #{},
+                <<"headers">> => DefaultHeaders
+            },
+            #{
+                <<"url">> => <<"http://x.y/db">>,
+                <<"auth_props">> => auth_props("u", "p"),
+                <<"headers">> => DefaultHeaders
+            }
+        },
+        {
+            #{
+                <<"url">> => <<"http://u:p@h:80/db">>,
+                <<"auth_props">> => #{},
+                <<"headers">> => DefaultHeaders
+            },
+            #{
+                <<"url">> => <<"http://h:80/db">>,
+                <<"auth_props">> => auth_props("u", "p"),
+                <<"headers">> => DefaultHeaders
+            }
+        },
+        {
+            #{
+                <<"url">> => <<"https://u:p@h/db">>,
+                <<"auth_props">> => #{},
+                <<"headers">> => DefaultHeaders
+            },
+            #{
+                <<"url">> => <<"https://h/db">>,
+                <<"auth_props">> => auth_props("u", "p"),
+                <<"headers">> => DefaultHeaders
+            }
+        },
+        {
+            #{
+                <<"url">> => <<"http://u:p@[2001:db8:a1b:12f9::1]/db">>,
+                <<"auth_props">> => #{},
+                <<"headers">> => DefaultHeaders
+            },
+            #{
+                <<"url">> => <<"http://[2001:db8:a1b:12f9::1]/db">>,
+                <<"auth_props">> => auth_props("u", "p"),
+                <<"headers">> => DefaultHeaders
+            }
+        },
+        {
+            #{
+                <<"url">> => <<"http://h/db">>,
+                <<"auth_props">> => #{},
+                <<"headers">> => maps:merge(DefaultHeaders, #{
+                    <<"authorization">> => basic_b64("u", "p")
+                })
+            },
+            #{
+                <<"url">> => <<"http://h/db">>,
+                <<"auth_props">> => auth_props("u", "p"),
+                <<"headers">> => DefaultHeaders
+            }
+        },
+        {
+            #{
+                <<"url">> => <<"http://h/db">>,
+                <<"auth_props">> => #{},
+                <<"headers">> => maps:merge(DefaultHeaders, #{
+                    <<"authorization">> => basic_b64("u", "p@")
+                })
+            },
+            #{
+                <<"url">> => <<"http://h/db">>,
+                <<"auth_props">> => auth_props("u", "p@"),
+                <<"headers">> => DefaultHeaders
+            }
+        },
+        {
+            #{
+                <<"url">> => <<"http://h/db">>,
+                <<"auth_props">> => #{},
+                <<"headers">> => maps:merge(DefaultHeaders, #{
+                    <<"authorization">> => basic_b64("u", "p@%40")
+                })
+            },
+            #{
+                <<"url">> => <<"http://h/db">>,
+                <<"auth_props">> => auth_props("u", "p@%40"),
+                <<"headers">> => DefaultHeaders
+            }
+        },
+        {
+            #{
+                <<"url">> => <<"http://h/db">>,
+                <<"auth_props">> => #{},
+                <<"headers">> => maps:merge(DefaultHeaders, #{
+                    <<"aUthoriZation">> => basic_b64("U", "p")
+                })
+            },
+            #{
+                <<"url">> => <<"http://h/db">>,
+                <<"auth_props">> => auth_props("U", "p"),
+                <<"headers">> => DefaultHeaders
+            }
+        },
+        {
+            #{
+                <<"url">> => <<"http://u1:p1@h/db">>,
+                <<"auth_props">> => #{},
+                <<"headers">> => maps:merge(DefaultHeaders, #{
+                    <<"Authorization">> => basic_b64("u2", "p2")
+                })
+            },
+            #{
+                <<"url">> => <<"http://h/db">>,
+                <<"auth_props">> => auth_props("u1", "p1"),
+                <<"headers">> => DefaultHeaders
+            }
+        },
+        {
+            #{
+                <<"url">> => <<"http://u1:p1@h/db">>,
+                <<"auth_props">> => auth_props("u2", "p2"),
+                <<"headers">> => DefaultHeaders
+            },
+            #{
+                <<"url">> => <<"http://h/db">>,
+                <<"auth_props">> => auth_props("u2", "p2"),
+                <<"headers">> => DefaultHeaders
+            }
+        },
+        {
+            #{
+                <<"url">> => <<"http://u1:p1@h/db">>,
+                <<"auth_props">> => auth_props("u2", "p2"),
+                <<"headers">> => maps:merge(DefaultHeaders, #{
+                    <<"Authorization">> => basic_b64("u3", "p3")
+                })
+            },
+            #{
+                <<"url">> => <<"http://h/db">>,
+                <<"auth_props">> => auth_props("u2", "p2"),
+                <<"headers">> => DefaultHeaders
+            }
+        }
+    ]].
+
+
+basic_b64(User, Pass) when is_list(User), is_list(Pass) ->
+    B64Creds = list_to_binary(b64creds(User, Pass)),
+    <<"basic ", B64Creds/binary>>.
+
+
+auth_props(User, Pass) when is_list(User), is_list(Pass) ->
+    #{
+        <<"basic">> => #{
+            <<"username">> => list_to_binary(User),
+            <<"password">> => list_to_binary(Pass)
+        }
+    }.
+
 -endif.
diff --git a/src/couch_replicator/src/couch_replicator_utils.erl b/src/couch_replicator/src/couch_replicator_utils.erl
index 523de5f54..c60cf5682 100644
--- a/src/couch_replicator/src/couch_replicator_utils.erl
+++ b/src/couch_replicator/src/couch_replicator_utils.erl
@@ -18,12 +18,12 @@
    iso8601/1,
    rfc1123_local/0,
    rfc1123_local/1,
-   remove_basic_auth_from_headers/1,
    normalize_rep/1,
    compare_reps/2,
    default_headers_map/0,
    parse_replication_states/1,
    parse_int_param/5,
+   get_basic_auth_creds/1,
    proplist_options/1
 ]).
 
@@ -71,37 +71,6 @@ rfc1123_local(Sec) ->
     list_to_binary(httpd_util:rfc1123_date(Local)).
 
 
-remove_basic_auth_from_headers(Headers) ->
-    Headers1 = mochiweb_headers:make(Headers),
-    case mochiweb_headers:get_value("Authorization", Headers1) of
-        undefined ->
-            {{undefined, undefined}, Headers};
-        Auth ->
-            {Basic, Base64} = lists:splitwith(fun(X) -> X =/= $\s end, Auth),
-            maybe_remove_basic_auth(string:to_lower(Basic), Base64, Headers1)
-    end.
-
-
-maybe_remove_basic_auth("basic", " " ++ Base64, Headers) ->
-    Headers1 = mochiweb_headers:delete_any("Authorization", Headers),
-    {decode_basic_creds(Base64), mochiweb_headers:to_list(Headers1)};
-maybe_remove_basic_auth(_, _, Headers) ->
-    {{undefined, undefined}, mochiweb_headers:to_list(Headers)}.
-
-
-decode_basic_creds(Base64) ->
-    try re:split(base64:decode(Base64), ":", [{return, list}, {parts, 2}]) of
-        [User, Pass] ->
-            {User, Pass};
-        _ ->
-            {undefined, undefined}
-    catch
-        % Tolerate invalid B64 values here to avoid crashing replicator
-        error:function_clause ->
-            {undefined, undefined}
-    end.
-
-
 -spec compare_reps(#{} | null, #{} | null) -> boolean().
 compare_reps(Rep1, Rep2) ->
     NormRep1 = normalize_rep(Rep1),
@@ -199,56 +168,30 @@ unix_sec_to_timestamp(Sec) when is_integer(Sec) ->
     {MegaSecPart, SecPart, 0}.
 
 
--ifdef(TEST).
+-spec get_basic_auth_creds(#httpdb{} | map()) ->
+    {string(), string()} | {undefined, undefined}.
+get_basic_auth_creds(#httpdb{auth_props = AuthProps}) ->
+    get_basic_auth_creds(#{<<"auth_props">> => AuthProps});
+
+get_basic_auth_creds(#{<<"auth_props">> := Props}) ->
+    case Props of
+        #{<<"basic">> := Basic} ->
+            User = maps:get(<<"username">>, Basic, undefined),
+            Pass = maps:get(<<"password">>, Basic, undefined),
+            case {User, Pass} of
+                _ when is_binary(User), is_binary(Pass) ->
+                    {binary_to_list(User), binary_to_list(Pass)};
+                _Other ->
+                    {undefined, undefined}
+            end;
+        _Other ->
+            {undefined, undefined}
+    end.
 
--include_lib("eunit/include/eunit.hrl").
 
-remove_basic_auth_from_headers_test_() ->
-    [?_assertMatch({{User, Pass}, NoAuthHeaders},
-        remove_basic_auth_from_headers(Headers)) ||
-        {{User, Pass, NoAuthHeaders}, Headers} <- [
-            {
-                {undefined, undefined, []},
-                []
-            },
-            {
-                {undefined, undefined, [{"h", "v"}]},
-                [{"h", "v"}]
-            },
-            {
-                {undefined, undefined, [{"Authorization", "junk"}]},
-                [{"Authorization", "junk"}]
-            },
-            {
-                {undefined, undefined, []},
-                [{"Authorization", "basic X"}]
-            },
-            {
-                {"user", "pass", []},
-                [{"Authorization", "Basic " ++ b64creds("user", "pass")}]
-            },
-            {
-                {"user", "pass", []},
-                [{"AuThorization", "Basic " ++ b64creds("user", "pass")}]
-            },
-            {
-                {"user", "pass", []},
-                [{"Authorization", "bAsIc " ++ b64creds("user", "pass")}]
-            },
-            {
-                {"user", "pass", [{"h", "v"}]},
-                [
-                    {"Authorization", "Basic " ++ b64creds("user", "pass")},
-                    {"h", "v"}
-                ]
-            }
-        ]
-    ].
-
-
-b64creds(User, Pass) ->
-    base64:encode_to_string(User ++ ":" ++ Pass).
+-ifdef(TEST).
 
+-include_lib("eunit/include/eunit.hrl").
 
 normalize_rep_test_() ->
     {
@@ -299,4 +242,24 @@ normalize_endpoint() ->
     ?assertEqual(<<"local">>, normalize_endpoint(<<"local">>)).
 
 
+get_basic_auth_creds_from_httpdb_test() ->
+    Check = fun(Props) ->
+        get_basic_auth_creds(#{<<"auth_props">> => Props})
+    end,
+
+    ?assertEqual({undefined, undefined}, Check(#{})),
+
+    ?assertEqual({undefined, undefined}, Check(#{a => b})),
+
+    ?assertEqual({undefined, undefined}, Check(#{<<"other">> => <<"x">>})),
+
+    ?assertEqual({undefined, undefined}, Check(#{<<"basic">> => #{}})),
+
+    UserPass1 = #{<<"username">> => <<"u">>, <<"password">> => <<"p">>},
+    ?assertEqual({"u", "p"}, Check(#{<<"basic">> => UserPass1})),
+
+    UserPass2 = #{<<"username">> => <<"u">>, <<"password">> => null},
+    ?assertEqual({undefined, undefined}, Check(#{<<"basic">> => UserPass2})).
+
+
 -endif.