Support soft-deletion in chttpd leveldb-softdeletion

Co-Authored-By: Paul J. Davis <paul.joseph.davis@gmail.com>

3 files changed, 332 insertions, 29 deletions

diff --git a/src/chttpd/src/chttpd_httpd_handlers.erl b/src/chttpd/src/chttpd_httpd_handlers.erl
index 831c014b3..de66712c9 100644
--- a/src/chttpd/src/chttpd_httpd_handlers.erl
+++ b/src/chttpd/src/chttpd_httpd_handlers.erl
@@ -21,6 +21,7 @@ url_handler(<<>>)                  -> fun chttpd_misc:handle_welcome_req/1;
 url_handler(<<"favicon.ico">>)     -> fun chttpd_misc:handle_favicon_req/1;
 url_handler(<<"_utils">>)          -> fun chttpd_misc:handle_utils_dir_req/1;
 url_handler(<<"_all_dbs">>)        -> fun chttpd_misc:handle_all_dbs_req/1;
+url_handler(<<"_deleted_dbs">>)    -> fun chttpd_misc:handle_deleted_dbs_req/1;
 url_handler(<<"_dbs_info">>)       -> fun chttpd_misc:handle_dbs_info_req/1;
 url_handler(<<"_active_tasks">>)   -> fun chttpd_misc:handle_task_status_req/1;
 url_handler(<<"_scheduler">>)      -> fun couch_replicator_httpd:handle_scheduler_req/1;
@@ -58,6 +59,15 @@ handler_info('GET', [<<"_active_tasks">>], _) ->
 handler_info('GET', [<<"_all_dbs">>], _) ->
     {'all_dbs.read', #{}};
 
+handler_info('GET', [<<"_deleted_dbs">>], _) ->
+    {'account-deleted-dbs.read', #{}};
+
+handler_info('POST', [<<"_deleted_dbs">>], _) ->
+    {'account-deleted-dbs.undelete', #{}};
+
+handler_info('DELETE', [<<"_deleted_dbs">>, Db], _) ->
+    {'account-deleted-dbs.delete', #{'db.name' => Db}};
+
 handler_info('POST', [<<"_dbs_info">>], _) ->
     {'dbs_info.read', #{}};
 
diff --git a/src/chttpd/src/chttpd_misc.erl b/src/chttpd/src/chttpd_misc.erl
index ca1e58ad2..843c3fe7e 100644
--- a/src/chttpd/src/chttpd_misc.erl
+++ b/src/chttpd/src/chttpd_misc.erl
@@ -15,6 +15,7 @@
 -export([
     handle_all_dbs_req/1,
     handle_dbs_info_req/1,
+    handle_deleted_dbs_req/1,
     handle_favicon_req/1,
     handle_favicon_req/2,
     handle_replicate_req/1,
@@ -164,35 +165,7 @@ all_dbs_callback({error, Reason}, #vacc{resp=Resp0}=Acc) ->
 
 handle_dbs_info_req(#httpd{method = 'GET'} = Req) ->
     ok = chttpd:verify_is_server_admin(Req),
-
-    #mrargs{
-        start_key = StartKey,
-        end_key = EndKey,
-        direction = Dir,
-        limit = Limit,
-        skip = Skip
-    } = couch_mrview_http:parse_params(Req, undefined),
-
-    Options = [
-        {start_key, StartKey},
-        {end_key, EndKey},
-        {dir, Dir},
-        {limit, Limit},
-        {skip, Skip}
-    ],
-
-    % TODO: Figure out if we can't calculate a valid
-    % ETag for this request. \xFFmetadataVersion won't
-    % work as we don't bump versions on size changes
-
-    {ok, Resp} = chttpd:start_delayed_json_response(Req, 200, []),
-    Callback = fun dbs_info_callback/2,
-    Acc = #vacc{req = Req, resp = Resp},
-    {ok, Resp} = fabric2_db:list_dbs_info(Callback, Acc, Options),
-    case is_record(Resp, vacc) of
-        true -> {ok, Resp#vacc.resp};
-        _ -> {ok, Resp}
-    end;
+    send_db_infos(Req, list_dbs_info);
 handle_dbs_info_req(#httpd{method='POST', user_ctx=UserCtx}=Req) ->
     chttpd:validate_ctype(Req, "application/json"),
     Props = chttpd:json_body_obj(Req),
@@ -226,6 +199,92 @@ handle_dbs_info_req(#httpd{method='POST', user_ctx=UserCtx}=Req) ->
 handle_dbs_info_req(Req) ->
     send_method_not_allowed(Req, "GET,HEAD,POST").
 
+handle_deleted_dbs_req(#httpd{method='GET', path_parts=[_]}=Req) ->
+    ok = chttpd:verify_is_server_admin(Req),
+    send_db_infos(Req, list_deleted_dbs_info);
+handle_deleted_dbs_req(#httpd{method='POST', user_ctx=Ctx, path_parts=[_]}=Req) ->
+    couch_httpd:verify_is_server_admin(Req),
+    chttpd:validate_ctype(Req, "application/json"),
+    GetJSON = fun(Key, Props, Default) ->
+        case couch_util:get_value(Key, Props) of
+            undefined when Default == error ->
+                Fmt = "POST body must include `~s` parameter.",
+                Msg = io_lib:format(Fmt, [Key]),
+                throw({bad_request, iolist_to_binary(Msg)});
+            undefined ->
+                Default;
+            Value ->
+                Value
+        end
+    end,
+    {BodyProps} = chttpd:json_body_obj(Req),
+    {UndeleteProps} = GetJSON(<<"undelete">>, BodyProps, error),
+    DbName = GetJSON(<<"source">>, UndeleteProps, error),
+    TimeStamp = GetJSON(<<"timestamp">>, UndeleteProps, error),
+    TgtDbName = GetJSON(<<"target">>, UndeleteProps, DbName),
+    case fabric2_db:undelete(DbName, TgtDbName, TimeStamp, [{user_ctx, Ctx}]) of
+        ok ->
+            send_json(Req, 200, {[{ok, true}]});
+        {error, file_exists} ->
+            chttpd:send_error(Req, file_exists);
+        {error, not_found} ->
+            chttpd:send_error(Req, not_found);
+        Error ->
+            throw(Error)
+    end;
+handle_deleted_dbs_req(#httpd{path_parts = PP}=Req) when length(PP) == 1 ->
+    send_method_not_allowed(Req, "GET,HEAD,POST");
+handle_deleted_dbs_req(#httpd{method='DELETE', user_ctx=Ctx, path_parts=[_, DbName]}=Req) ->
+    couch_httpd:verify_is_server_admin(Req),
+    TS = case ?JSON_DECODE(couch_httpd:qs_value(Req, "timestamp", "null")) of
+        null ->
+            throw({bad_request, "`timestamp` parameter is not provided."});
+        TS0 ->
+            TS0
+    end,
+    case fabric2_db:delete(DbName, [{user_ctx, Ctx}, {deleted_at, TS}]) of
+        ok ->
+            send_json(Req, 200, {[{ok, true}]});
+        {error, not_found} ->
+            chttpd:send_error(Req, not_found);
+        Error ->
+            throw(Error)
+    end;
+handle_deleted_dbs_req(#httpd{path_parts = PP}=Req) when length(PP) == 2 ->
+    send_method_not_allowed(Req, "HEAD,DELETE");
+handle_deleted_dbs_req(Req) ->
+    chttpd:send_error(Req, not_found).
+
+send_db_infos(Req, ListFunctionName) ->
+    #mrargs{
+        start_key = StartKey,
+        end_key = EndKey,
+        direction = Dir,
+        limit = Limit,
+        skip = Skip
+    } = couch_mrview_http:parse_params(Req, undefined),
+
+    Options = [
+        {start_key, StartKey},
+        {end_key, EndKey},
+        {dir, Dir},
+        {limit, Limit},
+        {skip, Skip}
+    ],
+
+    % TODO: Figure out if we can't calculate a valid
+    % ETag for this request. \xFFmetadataVersion won't
+    % work as we don't bump versions on size changes
+
+    {ok, Resp1} = chttpd:start_delayed_json_response(Req, 200, []),
+    Callback = fun dbs_info_callback/2,
+    Acc = #vacc{req = Req, resp = Resp1},
+    {ok, Resp2} = fabric2_db:ListFunctionName(Callback, Acc, Options),
+    case is_record(Resp2, vacc) of
+        true -> {ok, Resp2#vacc.resp};
+        _ -> {ok, Resp2}
+    end.
+
 dbs_info_callback({meta, _Meta}, #vacc{resp = Resp0} = Acc) ->
     {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, "["),
     {ok, Acc#vacc{resp = Resp1}};
diff --git a/src/chttpd/test/eunit/chttpd_deleted_dbs_test.erl b/src/chttpd/test/eunit/chttpd_deleted_dbs_test.erl
new file mode 100644
index 000000000..d6375c048
--- /dev/null
+++ b/src/chttpd/test/eunit/chttpd_deleted_dbs_test.erl
@@ -0,0 +1,234 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(chttpd_deleted_dbs_test).
+
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-define(USER, "chttpd_db_test_admin").
+-define(PASS, "pass").
+-define(AUTH, {basic_auth, {?USER, ?PASS}}).
+-define(CONTENT_JSON, {"Content-Type", "application/json"}).
+
+
+setup() ->
+    Hashed = couch_passwords:hash_admin_password(?PASS),
+    ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist=false),
+    Addr = config:get("chttpd", "bind_address", "127.0.0.1"),
+    Port = mochiweb_socket_server:get(chttpd, port),
+    lists:concat(["http://", Addr, ":", Port, "/"]).
+
+
+teardown(_Url) ->
+    ok = config:delete("couchdb", "enable_database_recovery", false),
+    ok = config:delete("admins", ?USER, _Persist=false).
+
+
+create_db(Url) ->
+    {ok, Status, _, _} = http(put, Url, ""),
+    ?assert(Status =:= 201 orelse Status =:= 202).
+
+
+delete_db(Url) ->
+    {ok, 200, _, _} = http(delete, Url).
+
+
+deleted_dbs_test_() ->
+    {
+        "chttpd deleted dbs tests",
+        {
+            setup,
+            fun chttpd_test_util:start_couch/0,
+            fun chttpd_test_util:stop_couch/1,
+            {
+                foreach,
+                fun setup/0,
+                fun teardown/1,
+                [
+                    fun should_return_error_for_unsupported_method/1,
+                    fun should_list_deleted_dbs/1,
+                    fun should_list_deleted_dbs_info/1,
+                    fun should_undelete_db/1,
+                    fun should_remove_deleted_db/1,
+                    fun should_undelete_db_to_target_db/1,
+                    fun should_not_undelete_db_to_existing_db/1
+                ]
+            }
+        }
+    }.
+
+
+should_return_error_for_unsupported_method(Url) ->
+    ?_test(begin
+        {ok, Code, _, Body} = http(delete, mk_url(Url)),
+
+        ?assertEqual(405, Code),
+        ?assertEqual(<<"method_not_allowed">>, get_json(<<"error">>, Body))
+    end).
+
+
+should_list_deleted_dbs(Url) ->
+    ?_test(begin
+        DbName1 = create_and_delete_db(Url),
+        DbName2 = create_and_delete_db(Url),
+        {ok, Code, _, Body} = http(get, mk_url(Url)),
+        DeletedDbs = get_db_names(Body),
+
+        ?assertEqual(200, Code),
+        ?assertEqual(true, lists:member(DbName1, DeletedDbs)),
+        ?assertEqual(true, lists:member(DbName2, DeletedDbs))
+    end).
+
+
+should_list_deleted_dbs_info(Url) ->
+    ?_test(begin
+         DbName = create_and_delete_db(Url),
+         {ok, _, _, Body} = http(get, mk_url(Url, DbName)),
+         [{Props}] = jiffy:decode(Body),
+
+         ?assertEqual(DbName, couch_util:get_value(<<"db_name">>, Props))
+     end).
+
+
+should_undelete_db(Url) ->
+    ?_test(begin
+        DbName = create_and_delete_db(Url),
+        {ok, _, _, ResultBody} = http(get, mk_url(Url, DbName)),
+        [{Props}] = jiffy:decode(ResultBody),
+        TimeStamp = couch_util:get_value(<<"timestamp">>, Props),
+
+        ErlJSON = {[
+            {undelete, {[
+                {source, DbName},
+                {timestamp, TimeStamp}
+            ]}}
+        ]},
+
+        {ok, Code1, _, _} = http(get, Url ++ DbName),
+        ?assertEqual(404, Code1),
+
+        {ok, Code2, _, _} = http(post, mk_url(Url), ErlJSON),
+        ?assertEqual(200, Code2),
+
+        {ok, Code3, _, _} = http(get, Url ++ DbName),
+        ?assertEqual(200, Code3)
+    end).
+
+
+should_remove_deleted_db(Url) ->
+    ?_test(begin
+        DbName = create_and_delete_db(Url),
+        {ok, _, _, Body1} = http(get, mk_url(Url, DbName)),
+        [{Props}] = jiffy:decode(Body1),
+        TimeStamp = couch_util:get_value(<<"timestamp">>, Props),
+
+        {ok, Code, _, _} = http(delete, mk_url(Url, DbName, TimeStamp)),
+        ?assertEqual(200, Code),
+
+        {ok, _, _, Body2} = http(get, mk_url(Url, DbName)),
+        ?assertEqual([], jiffy:decode(Body2))
+    end).
+
+
+should_undelete_db_to_target_db(Url) ->
+    ?_test(begin
+        DbName = create_and_delete_db(Url),
+        {ok, _, _, Body} = http(get, mk_url(Url, DbName)),
+        [{Props}] = jiffy:decode(Body),
+        TimeStamp = couch_util:get_value(<<"timestamp">>, Props),
+
+        NewDbName = ?tempdb(),
+        ErlJSON = {[
+            {undelete, {[
+                {source, DbName},
+                {timestamp, TimeStamp},
+                {target, NewDbName}
+            ]}}
+        ]},
+
+        {ok, Code1, _, _} = http(get, Url ++ NewDbName),
+        ?assertEqual(404, Code1),
+
+        {ok, Code2, _, _} = http(post, mk_url(Url), ErlJSON),
+        ?assertEqual(200, Code2),
+
+        {ok, Code3, _, _} = http(get, Url ++ NewDbName),
+        ?assertEqual(200, Code3)
+    end).
+
+
+should_not_undelete_db_to_existing_db(Url) ->
+    ?_test(begin
+        DbName = create_and_delete_db(Url),
+        {ok, _, _, ResultBody} = http(get, mk_url(Url, DbName)),
+        [{Props}] = jiffy:decode(ResultBody),
+        TimeStamp = couch_util:get_value(<<"timestamp">>, Props),
+
+        NewDbName = ?tempdb(),
+        create_db(Url ++ NewDbName),
+        ErlJSON = {[
+            {undelete, {[
+                {source, DbName},
+                {timestamp, TimeStamp},
+                {target, NewDbName}
+            ]}}
+        ]},
+        {ok, Code2, _, ResultBody2} = http(post, mk_url(Url), ErlJSON),
+        ?assertEqual(412, Code2),
+        ?assertEqual(<<"file_exists">>, get_json(<<"error">>, ResultBody2))
+    end).
+
+
+create_and_delete_db(BaseUrl) ->
+    DbName = ?tempdb(),
+    DbUrl = BaseUrl ++ DbName,
+    create_db(DbUrl),
+    ok = config:set("couchdb", "enable_database_recovery", "true", false),
+    delete_db(DbUrl),
+    DbName.
+
+
+http(Verb, Url) ->
+    Headers = [?CONTENT_JSON, ?AUTH],
+    test_request:Verb(Url, Headers).
+
+
+http(Verb, Url, Body) ->
+    Headers = [?CONTENT_JSON, ?AUTH],
+    test_request:Verb(Url, Headers, jiffy:encode(Body)).
+
+
+mk_url(Url) ->
+    Url ++ "/_deleted_dbs".
+
+
+mk_url(Url, DbName) ->
+    Url ++ "/_deleted_dbs?key=\"" ++ ?b2l(DbName) ++ "\"".
+
+
+mk_url(Url, DbName, TimeStamp) ->
+    Url ++ "/_deleted_dbs/" ++ ?b2l(DbName) ++ "?timestamp=\"" ++
+        ?b2l(TimeStamp) ++ "\"".
+
+
+get_json(Key, Body) ->
+    {Props} = jiffy:decode(Body),
+    couch_util:get_value(Key, Props).
+
+
+get_db_names(Body) ->
+    RevDbNames = lists:foldl(fun({DbInfo}, Acc) ->
+        DbName = couch_util:get_value(<<"db_name">>, DbInfo),
+        [DbName | Acc]
+    end, [], jiffy:decode(Body)),
+    lists:reverse(RevDbNames).