Implement `_users` db authentication

This changes `chttpd_auth_cache` to use FoundationDB to back the
`_users` database including the `before_doc_update` and `after_doc_read`
features.

6 files changed, 212 insertions, 26 deletions

diff --git a/src/chttpd/src/chttpd_auth_cache.erl b/src/chttpd/src/chttpd_auth_cache.erl
index 638d8c748..d947fe62e 100644
--- a/src/chttpd/src/chttpd_auth_cache.erl
+++ b/src/chttpd/src/chttpd_auth_cache.erl
@@ -52,7 +52,8 @@ get_user_creds(_Req, UserName) when is_binary(UserName) ->
 
 update_user_creds(_Req, UserDoc, _Ctx) ->
     {_, Ref} = spawn_monitor(fun() ->
-        case fabric:update_doc(dbname(), UserDoc, []) of
+        {ok, Db} = fabric2_db:open(dbname(), [?ADMIN_CTX]),
+        case fabric2_db:update_doc(Db, UserDoc) of
             {ok, _} ->
                 exit(ok);
             Else ->
@@ -100,6 +101,14 @@ maybe_increment_auth_cache_miss(UserName) ->
 %% gen_server callbacks
 
 init([]) ->
+    try
+        fabric2_db:open(dbname(), [?ADMIN_CTX])
+    catch error:database_does_not_exist ->
+        case fabric2_db:create(dbname(), [?ADMIN_CTX]) of
+            {ok, _} -> ok;
+            {error, file_exists} -> ok
+        end
+    end,
     self() ! {start_listener, 0},
     {ok, #state{}}.
 
@@ -139,7 +148,8 @@ spawn_changes(Since) ->
     Pid.
 
 listen_for_changes(Since) ->
-    ensure_auth_ddoc_exists(dbname(), <<"_design/_auth">>),
+    {ok, Db} = fabric2_db:open(dbname(), [?ADMIN_CTX]),
+    ensure_auth_ddoc_exists(Db, <<"_design/_auth">>),
     CBFun = fun ?MODULE:changes_callback/2,
     Args = #changes_args{
         feed = "continuous",
@@ -147,7 +157,8 @@ listen_for_changes(Since) ->
         heartbeat = true,
         filter = {default, main_only}
     },
-    fabric:changes(dbname(), CBFun, Since, Args).
+    ChangesFun = chttpd_changes:handle_db_changes(Args, nil, Db),
+    ChangesFun({CBFun, Since}).
 
 changes_callback(waiting_for_updates, Acc) ->
     {ok, Acc};
@@ -156,7 +167,7 @@ changes_callback(start, Since) ->
 changes_callback({stop, EndSeq, _Pending}, _) ->
     exit({seq, EndSeq});
 changes_callback({change, {Change}}, _) ->
-    case couch_util:get_value(id, Change) of
+    case couch_util:get_value(<<"id">>, Change) of
         <<"_design/", _/binary>> ->
             ok;
         DocId ->
@@ -171,7 +182,8 @@ changes_callback({error, _}, EndSeq) ->
     exit({seq, EndSeq}).
 
 load_user_from_db(UserName) ->
-    try fabric:open_doc(dbname(), docid(UserName), [?ADMIN_CTX, ejson_body, conflicts]) of
+    {ok, Db} = fabric2_db:open(dbname(), [?ADMIN_CTX]),
+    try fabric2_db:open_doc(Db, docid(UserName), [conflicts]) of
     {ok, Doc} ->
         {Props} = couch_doc:to_json_obj(Doc, []),
         Props;
@@ -183,7 +195,8 @@ load_user_from_db(UserName) ->
     end.
 
 dbname() ->
-    config:get("chttpd_auth", "authentication_db", "_users").
+    DbNameStr = config:get("chttpd_auth", "authentication_db", "_users"),
+    iolist_to_binary(DbNameStr).
 
 docid(UserName) ->
     <<"org.couchdb.user:", UserName/binary>>.
@@ -191,11 +204,11 @@ docid(UserName) ->
 username(<<"org.couchdb.user:", UserName/binary>>) ->
     UserName.
 
-ensure_auth_ddoc_exists(DbName, DDocId) ->
-    case fabric:open_doc(DbName, DDocId, [?ADMIN_CTX, ejson_body]) of
+ensure_auth_ddoc_exists(Db, DDocId) ->
+    case fabric2_db:open_doc(Db, DDocId) of
     {not_found, _Reason} ->
         {ok, AuthDesign} = couch_auth_cache:auth_design_doc(DDocId),
-        update_doc_ignoring_conflict(DbName, AuthDesign, [?ADMIN_CTX]);
+        update_doc_ignoring_conflict(Db, AuthDesign);
     {ok, Doc} ->
         {Props} = couch_doc:to_json_obj(Doc, []),
         case couch_util:get_value(<<"validate_doc_update">>, Props, []) of
@@ -205,17 +218,18 @@ ensure_auth_ddoc_exists(DbName, DDocId) ->
                 Props1 = lists:keyreplace(<<"validate_doc_update">>, 1, Props,
                     {<<"validate_doc_update">>,
                     ?AUTH_DB_DOC_VALIDATE_FUNCTION}),
-                update_doc_ignoring_conflict(DbName, couch_doc:from_json_obj({Props1}), [?ADMIN_CTX])
+                NewDoc = couch_doc:from_json_obj({Props1}),
+                update_doc_ignoring_conflict(Db, NewDoc)
         end;
     {error, Reason} ->
-        couch_log:notice("Failed to ensure auth ddoc ~s/~s exists for reason: ~p", [DbName, DDocId, Reason]),
+        couch_log:notice("Failed to ensure auth ddoc ~s/~s exists for reason: ~p", [dbname(), DDocId, Reason]),
         ok
     end,
     ok.
 
-update_doc_ignoring_conflict(DbName, Doc, Options) ->
+update_doc_ignoring_conflict(DbName, Doc) ->
     try
-        fabric:update_doc(DbName, Doc, Options)
+        fabric2_db:update_doc(DbName, Doc)
     catch
         throw:conflict ->
             ok
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index 40c1a1e38..4337041f1 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -724,10 +724,9 @@ db_req(#httpd{method='POST',path_parts=[_,<<"_revs_diff">>]}=Req, Db) ->
 db_req(#httpd{path_parts=[_,<<"_revs_diff">>]}=Req, _Db) ->
     send_method_not_allowed(Req, "POST");
 
-db_req(#httpd{method='PUT',path_parts=[_,<<"_security">>],user_ctx=Ctx}=Req,
-        Db) ->
+db_req(#httpd{method = 'PUT',path_parts = [_, <<"_security">>]} = Req, Db) ->
     SecObj = chttpd:json_body(Req),
-    case fabric:set_security(Db, SecObj, [{user_ctx, Ctx}]) of
+    case fabric2_db:set_security(Db, SecObj) of
         ok ->
             send_json(Req, {[{<<"ok">>, true}]});
         Else ->
diff --git a/src/fabric/src/fabric2_db.erl b/src/fabric/src/fabric2_db.erl
index 48e50f11c..80028a645 100644
--- a/src/fabric/src/fabric2_db.erl
+++ b/src/fabric/src/fabric2_db.erl
@@ -149,9 +149,10 @@ create(DbName, Options) ->
     % We cache outside of the transaction so that we're sure
     % that the transaction was committed.
     case Result of
-        #{} = Db ->
-            ok = fabric2_server:store(Db),
-            {ok, Db#{tx := undefined}};
+        #{} = Db0 ->
+            Db1 = maybe_add_sys_db_callbacks(Db0),
+            ok = fabric2_server:store(Db1),
+            {ok, Db1#{tx := undefined}};
         Error ->
             Error
     end.
@@ -167,9 +168,10 @@ open(DbName, Options) ->
             end),
             % Cache outside the transaction retry loop
             case Result of
-                #{} = Db ->
-                    ok = fabric2_server:store(Db),
-                    {ok, Db#{tx := undefined}};
+                #{} = Db0 ->
+                    Db1 = maybe_add_sys_db_callbacks(Db0),
+                    ok = fabric2_server:store(Db1),
+                    {ok, Db1#{tx := undefined}};
                 Error ->
                     Error
             end
@@ -552,18 +554,19 @@ update_docs(Db, Docs) ->
     update_docs(Db, Docs, []).
 
 
-update_docs(Db, Docs, Options) ->
+update_docs(Db, Docs0, Options) ->
+    Docs1 = apply_before_doc_update(Db, Docs0, Options),
     Resps0 = case lists:member(replicated_changes, Options) of
         false ->
             fabric2_fdb:transactional(Db, fun(TxDb) ->
-                update_docs_interactive(TxDb, Docs, Options)
+                update_docs_interactive(TxDb, Docs1, Options)
             end);
         true ->
             lists:map(fun(Doc) ->
                 fabric2_fdb:transactional(Db, fun(TxDb) ->
                     update_doc_int(TxDb, Doc, Options)
                 end)
-            end, Docs)
+            end, Docs1)
     end,
     % Convert errors
     Resps1 = lists:map(fun(Resp) ->
@@ -882,6 +885,19 @@ find_possible_ancestors(RevInfos, MissingRevs) ->
     end, RevInfos).
 
 
+apply_before_doc_update(Db, Docs, Options) ->
+    #{before_doc_update := BDU} = Db,
+    UpdateType = case lists:member(replicated_changes, Options) of
+        true -> replicated_changes;
+        false -> interactive_edit
+    end,
+    if BDU == undefined -> Docs; true ->
+        lists:map(fun(Doc) ->
+            BDU(Doc, Db, UpdateType)
+        end, Docs)
+    end.
+
+
 update_doc_int(#{} = Db, #doc{} = Doc, Options) ->
     IsLocal = case Doc#doc.id of
         <<?LOCAL_DOC_PREFIX, _/binary>> -> true;
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index d179387f6..4b0182646 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -944,7 +944,13 @@ fdb_to_doc(Db, DocId, Pos, Path, Bin) when is_binary(Bin) ->
         body = Body,
         atts = Atts,
         deleted = Deleted
-    };
+    },
+
+    case Db of
+        #{after_doc_read := undefined} -> Doc0;
+        #{after_doc_read := ADR} -> ADR(Doc0, Db)
+    end;
+
 fdb_to_doc(_Db, _DocId, _Pos, _Path, not_found) ->
     {not_found, missing}.
 
diff --git a/src/fabric/src/fabric2_users_db.erl b/src/fabric/src/fabric2_users_db.erl
new file mode 100644
index 000000000..9a8a462c3
--- /dev/null
+++ b/src/fabric/src/fabric2_users_db.erl
@@ -0,0 +1,144 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric2_users_db).
+
+-export([
+    before_doc_update/3,
+    after_doc_read/2,
+    strip_non_public_fields/1
+]).
+
+-include_lib("couch/include/couch_db.hrl").
+
+-define(NAME, <<"name">>).
+-define(PASSWORD, <<"password">>).
+-define(DERIVED_KEY, <<"derived_key">>).
+-define(PASSWORD_SCHEME, <<"password_scheme">>).
+-define(SIMPLE, <<"simple">>).
+-define(PASSWORD_SHA, <<"password_sha">>).
+-define(PBKDF2, <<"pbkdf2">>).
+-define(ITERATIONS, <<"iterations">>).
+-define(SALT, <<"salt">>).
+-define(replace(L, K, V), lists:keystore(K, 1, L, {K, V})).
+
+-define(
+    DDOCS_ADMIN_ONLY,
+    <<"Only administrators can view design docs in the users database.">>
+).
+
+% If the request's userCtx identifies an admin
+%   -> save_doc (see below)
+%
+% If the request's userCtx.name is null:
+%   -> save_doc
+%   // this is an anonymous user registering a new document
+%   // in case a user doc with the same id already exists, the anonymous
+%   // user will get a regular doc update conflict.
+% If the request's userCtx.name doesn't match the doc's name
+%   -> 404 // Not Found
+% Else
+%   -> save_doc
+before_doc_update(Doc, Db, _UpdateType) ->
+    #user_ctx{name = Name} = fabric2_db:get_user_ctx(Db),
+    DocName = get_doc_name(Doc),
+    case (catch fabric2_db:check_is_admin(Db)) of
+    ok ->
+        save_doc(Doc);
+    _ when Name =:= DocName orelse Name =:= null ->
+        save_doc(Doc);
+    _ ->
+        throw(not_found)
+    end.
+
+% If newDoc.password == null || newDoc.password == undefined:
+%   ->
+%   noop
+% Else -> // calculate password hash server side
+%    newDoc.password_sha = hash_pw(newDoc.password + salt)
+%    newDoc.salt = salt
+%    newDoc.password = null
+save_doc(#doc{body={Body}} = Doc) ->
+    %% Support both schemes to smooth migration from legacy scheme
+    Scheme = config:get("couch_httpd_auth", "password_scheme", "pbkdf2"),
+    case {fabric2_util:get_value(?PASSWORD, Body), Scheme} of
+    {null, _} -> % server admins don't have a user-db password entry
+        Doc;
+    {undefined, _} ->
+        Doc;
+    {ClearPassword, "simple"} -> % deprecated
+        Salt = couch_uuids:random(),
+        PasswordSha = couch_passwords:simple(ClearPassword, Salt),
+        Body0 = ?replace(Body, ?PASSWORD_SCHEME, ?SIMPLE),
+        Body1 = ?replace(Body0, ?SALT, Salt),
+        Body2 = ?replace(Body1, ?PASSWORD_SHA, PasswordSha),
+        Body3 = proplists:delete(?PASSWORD, Body2),
+        Doc#doc{body={Body3}};
+    {ClearPassword, "pbkdf2"} ->
+        Iterations = list_to_integer(config:get("couch_httpd_auth", "iterations", "1000")),
+        Salt = couch_uuids:random(),
+        DerivedKey = couch_passwords:pbkdf2(ClearPassword, Salt, Iterations),
+        Body0 = ?replace(Body, ?PASSWORD_SCHEME, ?PBKDF2),
+        Body1 = ?replace(Body0, ?ITERATIONS, Iterations),
+        Body2 = ?replace(Body1, ?DERIVED_KEY, DerivedKey),
+        Body3 = ?replace(Body2, ?SALT, Salt),
+        Body4 = proplists:delete(?PASSWORD, Body3),
+        Doc#doc{body={Body4}};
+    {_ClearPassword, Scheme} ->
+        couch_log:error("[couch_httpd_auth] password_scheme value of '~p' is invalid.", [Scheme]),
+        throw({forbidden, "Server cannot hash passwords at this time."})
+    end.
+
+
+% If the doc is a design doc
+%   If the request's userCtx identifies an admin
+%     -> return doc
+%   Else
+%     -> 403 // Forbidden
+% If the request's userCtx identifies an admin
+%   -> return doc
+% If the request's userCtx.name doesn't match the doc's name
+%   -> 404 // Not Found
+% Else
+%   -> return doc
+after_doc_read(#doc{id = <<?DESIGN_DOC_PREFIX, _/binary>>} = Doc, Db) ->
+    case (catch fabric2_db:check_is_admin(Db)) of
+        ok -> Doc;
+        _ -> throw({forbidden, ?DDOCS_ADMIN_ONLY})
+    end;
+after_doc_read(Doc, Db) ->
+    #user_ctx{name = Name} = fabric2_db:get_user_ctx(Db),
+    DocName = get_doc_name(Doc),
+    case (catch fabric2_db:check_is_admin(Db)) of
+        ok ->
+            Doc;
+        _ when Name =:= DocName ->
+            Doc;
+        _ ->
+            Doc1 = strip_non_public_fields(Doc),
+            case Doc1 of
+                #doc{body={[]}} -> throw(not_found);
+                _ -> Doc1
+            end
+    end.
+
+
+get_doc_name(#doc{id= <<"org.couchdb.user:", Name/binary>>}) ->
+    Name;
+get_doc_name(_) ->
+    undefined.
+
+
+strip_non_public_fields(#doc{body={Props}}=Doc) ->
+    PublicFields = config:get("couch_httpd_auth", "public_fields", ""),
+    Public = re:split(PublicFields, "\\s*,\\s*", [{return, binary}]),
+    Doc#doc{body={[{K, V} || {K, V} <- Props, lists:member(K, Public)]}}.
diff --git a/src/fabric/src/fabric2_util.erl b/src/fabric/src/fabric2_util.erl
index 6e2df67c2..fb59d5923 100644
--- a/src/fabric/src/fabric2_util.erl
+++ b/src/fabric/src/fabric2_util.erl
@@ -24,6 +24,8 @@
 
     validate_security_object/1,
 
+    dbname_ends_with/2,
+
     get_value/2,
     get_value/3,
     to_hex/1,
@@ -113,6 +115,11 @@ validate_json_list_of_strings(Member, Props) ->
     end.
 
 
+dbname_ends_with(#{} = Db, Suffix) when is_binary(Suffix) ->
+    DbName = fabric2_db:name(Db),
+    Suffix == filename:basename(DbName).
+
+
 get_value(Key, List) ->
     get_value(Key, List, undefined).