Improve fabric index cleanup

 * Clean-up stale view purge checkpoints. Previously we didn't and purge
   progress could have stalled by keeping around inactive(lagging) lagging
   purge checkpoints.

 * couch_mrview_cleanup attempted to clean purge checkpoints but that didn't
   work for clustered databases, only for local ones. Nowadays most dbs
   are clustered so make sure those work as well.

 * DRY-out code from both fabric inactive index cleanup and
   couch_mrview_cleanup modules. Move some of the common code to
   couch_mrview_util module. couch_mrvew_cleanup is the only place in charge
   the cleanup logic now.

 * Consolidate and improve tests. Utility functions to get all index files,
   purge checkpoint and signatures are now tested with couch_mrview_util tests,
   and end-to-end fabric cleanup tests are in fabric_tests. Since fabirc_tests
   covers all the test scenarios from fabric_test.exs, remove fabric_test.exs
   so we don't have test duplicated and get same coverage.

7 files changed, 500 insertions, 246 deletions

diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl
index 969b93636..cb4936193 100644
--- a/src/couch/src/couch_db.erl
+++ b/src/couch/src/couch_db.erl
@@ -687,6 +687,11 @@ get_design_doc(#db{} = Db, DDocId0) ->
     DDocId = couch_util:normalize_ddoc_id(DDocId0),
     couch_db:open_doc_int(Db, DDocId, [ejson_body]).
 
+% Note: get_design_docs/1 for clustered db shards returns docs as ejson {[_ |
+% _]} and for simple node-local dbs it returns #full_docs_info{} records. To
+% obtain ejson docs in that case would need to call couch_db:open_doc_int(Db,
+% FDI, [ejson_body]) for each one of them.
+%
 get_design_docs(#db{name = <<"shards/", _/binary>> = ShardDbName}) ->
     DbName = mem3:dbname(ShardDbName),
     {_, Ref} = spawn_monitor(fun() -> exit(fabric:design_docs(DbName)) end),
diff --git a/src/couch/test/exunit/fabric_test.exs b/src/couch/test/exunit/fabric_test.exs
deleted file mode 100644
index bdb84e9a2..000000000
--- a/src/couch/test/exunit/fabric_test.exs
+++ /dev/null
@@ -1,101 +0,0 @@
-defmodule Couch.Test.Fabric do
-  use Couch.Test.ExUnit.Case
-  alias Couch.Test.Utils
-
-  alias Couch.Test.Setup
-
-  alias Couch.Test.Setup.Step
-
-  import Couch.DBTest
-
-  import Utils
-
-  @admin {:user_ctx, user_ctx(roles: ["_admin"])}
-
-  def with_db(context, setup) do
-    setup =
-      setup
-      |> Setup.Common.with_db()
-      |> Setup.run()
-
-    context =
-      Map.merge(context, %{
-        db_name: setup |> Setup.get(:db) |> Step.Create.DB.name()
-      })
-
-    {context, setup}
-  end
-
-  describe "Fabric miscellaneous API" do
-    @describetag setup: &__MODULE__.with_db/2
-    test "Get inactive_index_files", ctx do
-      {:ok, _rev} = update_doc(ctx.db_name, %{"_id" => "doc1"})
-
-      design_doc = %{
-        "_id" => "_design/test",
-        "language" => "javascript",
-        "views" => %{
-          "view" => %{
-            "map" => "function(doc){emit(doc._id, doc._rev)}"
-          }
-        }
-      }
-
-      {:ok, rev1} = update_doc(ctx.db_name, design_doc)
-      wait_sig_update(ctx.db_name, "test", "")
-      prev_active = get_active_sig(ctx.db_name, "test")
-
-      updated_design_doc =
-        put_in(design_doc, ["views", "view", "map"], "function(doc){emit(doc._id, null)}")
-
-      {:ok, rev2} =
-        update_doc(
-          ctx.db_name,
-          Map.put(updated_design_doc, "_rev", rev1)
-        )
-
-      assert rev1 != rev2
-      wait_sig_update(ctx.db_name, "test", prev_active)
-
-      {:ok, info} = :fabric.get_view_group_info(ctx.db_name, "_design/test")
-      active = info[:signature]
-
-      files = Enum.map(:fabric.inactive_index_files(ctx.db_name), &List.to_string/1)
-
-      assert [] != files, "We should have some inactive"
-
-      assert not Enum.any?(files, fn
-               file_path -> String.contains?(file_path, active)
-             end),
-             "We are not suppose to return active views"
-
-      assert Enum.all?(files, fn
-               file_path -> String.contains?(file_path, prev_active)
-             end),
-             "We expect all files to contain previous active signature"
-    end
-  end
-
-  defp update_doc(db_name, body) do
-    json_body = :jiffy.decode(:jiffy.encode(body))
-
-    case :fabric.update_doc(db_name, json_body, [@admin]) do
-      {:ok, rev} ->
-        {:ok, :couch_doc.rev_to_str(rev)}
-
-      error ->
-        error
-    end
-  end
-
-  defp get_active_sig(db_name, ddoc_id) do
-    {:ok, info} = :fabric.get_view_group_info(db_name, "_design/#{ddoc_id}")
-    info[:signature]
-  end
-
-  defp wait_sig_update(db_name, ddoc_id, prev_active) do
-    retry_until(fn ->
-      get_active_sig(db_name, ddoc_id) != prev_active
-    end)
-  end
-end
diff --git a/src/couch_mrview/src/couch_mrview_cleanup.erl b/src/couch_mrview/src/couch_mrview_cleanup.erl
index 417605c55..5b5afbdce 100644
--- a/src/couch_mrview/src/couch_mrview_cleanup.erl
+++ b/src/couch_mrview/src/couch_mrview_cleanup.erl
@@ -12,57 +12,62 @@
 
 -module(couch_mrview_cleanup).
 
--export([run/1]).
+-export([
+    run/1,
+    cleanup_purges/3,
+    cleanup_indices/2
+]).
 
 -include_lib("couch/include/couch_db.hrl").
--include_lib("couch_mrview/include/couch_mrview.hrl").
 
 run(Db) ->
-    RootDir = couch_index_util:root_dir(),
-    DbName = couch_db:name(Db),
+    Indices = couch_mrview_util:get_index_files(Db),
+    Checkpoints = couch_mrview_util:get_purge_checkpoints(Db),
+    {ok, Db1} = couch_db:reopen(Db),
+    Sigs = couch_mrview_util:get_signatures(Db1),
+    ok = cleanup_purges(Db1, Sigs, Checkpoints),
+    ok = cleanup_indices(Sigs, Indices).
 
-    {ok, DesignDocs} = couch_db:get_design_docs(Db),
-    SigFiles = lists:foldl(
-        fun(DDocInfo, SFAcc) ->
-            {ok, DDoc} = couch_db:open_doc_int(Db, DDocInfo, [ejson_body]),
-            {ok, InitState} = couch_mrview_util:ddoc_to_mrst(DbName, DDoc),
-            Sig = InitState#mrst.sig,
-            IFName = couch_mrview_util:index_file(DbName, Sig),
-            CFName = couch_mrview_util:compaction_file(DbName, Sig),
-            [IFName, CFName | SFAcc]
-        end,
-        [],
-        [DD || DD <- DesignDocs, DD#full_doc_info.deleted == false]
-    ),
+cleanup_purges(DbName, Sigs, Checkpoints) when is_binary(DbName) ->
+    couch_util:with_db(DbName, fun(Db) ->
+        cleanup_purges(Db, Sigs, Checkpoints)
+    end);
+cleanup_purges(Db, #{} = Sigs, #{} = CheckpointsMap) ->
+    InactiveMap = maps:without(maps:keys(Sigs), CheckpointsMap),
+    InactiveCheckpoints = maps:values(InactiveMap),
+    DeleteFun = fun(DocId) -> delete_checkpoint(Db, DocId) end,
+    lists:foreach(DeleteFun, InactiveCheckpoints).
 
-    IdxDir = couch_index_util:index_dir(mrview, DbName),
-    DiskFiles = filelib:wildcard(filename:join(IdxDir, "*")),
+cleanup_indices(#{} = Sigs, #{} = IndexMap) ->
+    Fun = fun(_, Files) -> lists:foreach(fun delete_file/1, Files) end,
+    maps:map(Fun, maps:without(maps:keys(Sigs), IndexMap)),
+    ok.
 
-    % We need to delete files that have no ddoc.
-    ToDelete = DiskFiles -- SigFiles,
+delete_file(File) ->
+    RootDir = couch_index_util:root_dir(),
+    couch_log:debug("~p : deleting inactive index : ~s", [?MODULE, File]),
+    try
+        couch_file:delete(RootDir, File, [sync])
+    catch
+        Tag:Error ->
+            ErrLog = "~p : error deleting inactive index file ~s ~p:~p",
+            couch_log:error(ErrLog, [?MODULE, File, Tag, Error]),
+            ok
+    end.
 
-    lists:foreach(
-        fun(FN) ->
-            couch_log:debug("Deleting stale view file: ~s", [FN]),
-            couch_file:delete(RootDir, FN, [sync]),
-            case couch_mrview_util:verify_view_filename(FN) of
-                true ->
-                    Sig = couch_mrview_util:get_signature_from_filename(FN),
-                    DocId = couch_mrview_util:get_local_purge_doc_id(Sig),
-                    case couch_db:open_doc(Db, DocId, []) of
-                        {ok, LocalPurgeDoc} ->
-                            couch_db:update_doc(
-                                Db,
-                                LocalPurgeDoc#doc{deleted = true},
-                                [?ADMIN_CTX]
-                            );
-                        {not_found, _} ->
-                            ok
-                    end;
-                false ->
-                    ok
-            end
-        end,
-        ToDelete
-    ),
-    ok.
+delete_checkpoint(Db, DocId) ->
+    DbName = couch_db:name(Db),
+    LogMsg = "~p : deleting inactive purge checkpoint ~s : ~s",
+    couch_log:debug(LogMsg, [?MODULE, DbName, DocId]),
+    try couch_db:open_doc(Db, DocId, []) of
+        {ok, Doc = #doc{}} ->
+            Deleted = Doc#doc{deleted = true, body = {[]}},
+            couch_db:update_doc(Db, Deleted, [?ADMIN_CTX]);
+        {not_found, _} ->
+            ok
+    catch
+        Tag:Error ->
+            ErrLog = "~p : error deleting checkpoint ~s : ~s error: ~p:~p",
+            couch_log:error(ErrLog, [?MODULE, DbName, DocId, Tag, Error]),
+            ok
+    end.
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index 9e3d292ed..e1e75f34f 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -15,6 +15,7 @@
 -export([get_view/4, get_view_index_pid/4]).
 -export([get_local_purge_doc_id/1, get_value_from_options/2]).
 -export([verify_view_filename/1, get_signature_from_filename/1]).
+-export([get_signatures/1, get_purge_checkpoints/1, get_index_files/1]).
 -export([ddoc_to_mrst/2, init_state/4, reset_index/3]).
 -export([make_header/1]).
 -export([index_file/2, compaction_file/2, open_file/1]).
@@ -53,6 +54,11 @@
         true -> B
     end)
 ).
+-define(IS_HEX(C),
+    ((C >= $0 andalso C =< $9) orelse
+        (C >= $a andalso C =< $f) orelse
+        (C >= $A andalso C =< $F))
+).
 
 -include_lib("couch/include/couch_db.hrl").
 -include_lib("couch_mrview/include/couch_mrview.hrl").
@@ -70,31 +76,78 @@ get_value_from_options(Key, Options) ->
     end.
 
 verify_view_filename(FileName) ->
-    FilePathList = filename:split(FileName),
-    PureFN = lists:last(FilePathList),
-    case filename:extension(PureFN) of
+    case filename:extension(FileName) of
         ".view" ->
-            Sig = filename:basename(PureFN),
-            case
-                [
-                    Ch
-                 || Ch <- Sig,
-                    not (((Ch >= $0) and (Ch =< $9)) orelse
-                        ((Ch >= $a) and (Ch =< $f)) orelse
-                        ((Ch >= $A) and (Ch =< $F)))
-                ] == []
-            of
-                true -> true;
-                false -> false
-            end;
+            Sig = get_signature_from_filename(FileName),
+            lists:all(fun(C) -> ?IS_HEX(C) end, Sig);
         _ ->
             false
     end.
 
-get_signature_from_filename(FileName) ->
-    FilePathList = filename:split(FileName),
-    PureFN = lists:last(FilePathList),
-    filename:basename(PureFN, ".view").
+get_signature_from_filename(Path) ->
+    filename:basename(filename:basename(Path, ".view"), ".compact").
+
+% Returns map of `Sig => true` elements with all the active signatures.
+% Sig is a hex-encoded binary.
+%
+get_signatures(DbName) when is_binary(DbName) ->
+    couch_util:with_db(DbName, fun get_signatures/1);
+get_signatures(Db) ->
+    DbName = couch_db:name(Db),
+    % get_design_docs/1 returns ejson for clustered shards, and
+    % #full_doc_info{}'s for other cases.
+    {ok, DDocs} = couch_db:get_design_docs(Db),
+    FoldFun = fun
+        ({[_ | _]} = EJsonDoc, Acc) ->
+            Doc = couch_doc:from_json_obj(EJsonDoc),
+            {ok, Mrst} = ddoc_to_mrst(DbName, Doc),
+            Sig = couch_util:to_hex_bin(Mrst#mrst.sig),
+            Acc#{Sig => true};
+        (#full_doc_info{} = FDI, Acc) ->
+            {ok, Doc} = couch_db:open_doc_int(Db, FDI, [ejson_body]),
+            {ok, Mrst} = ddoc_to_mrst(DbName, Doc),
+            Sig = couch_util:to_hex_bin(Mrst#mrst.sig),
+            Acc#{Sig => true}
+    end,
+    lists:foldl(FoldFun, #{}, DDocs).
+
+% Returns a map of `Sig => DocId` elements for all the purge view
+% checkpoint docs. Sig is a hex-encoded binary.
+%
+get_purge_checkpoints(DbName) when is_binary(DbName) ->
+    couch_util:with_db(DbName, fun get_purge_checkpoints/1);
+get_purge_checkpoints(Db) ->
+    FoldFun = fun(#doc{id = Id}, Acc) ->
+        case Id of
+            <<?LOCAL_DOC_PREFIX, "purge-mrview-", Sig/binary>> ->
+                {ok, Acc#{Sig => Id}};
+            _ ->
+                {stop, Acc}
+        end
+    end,
+    Opts = [{start_key, <<?LOCAL_DOC_PREFIX, "purge-mrview-">>}],
+    {ok, Signatures = #{}} = couch_db:fold_local_docs(Db, FoldFun, #{}, Opts),
+    Signatures.
+
+% Returns a map of `Sig => [FilePath, ...]` elements. Sig is a hex-encoded
+% binary and FilePaths are lists as they intended to be passed to couch_file
+% and file module functions.
+%
+get_index_files(DbName) when is_binary(DbName) ->
+    IdxDir = couch_index_util:index_dir(mrview, DbName),
+    WildcardPath = filename:join(IdxDir, "*"),
+    FoldFun = fun(F, Acc) ->
+        case verify_view_filename(F) of
+            true ->
+                Sig = ?l2b(get_signature_from_filename(F)),
+                maps:update_with(Sig, fun(Fs) -> [F | Fs] end, [F], Acc);
+            false ->
+                Acc
+        end
+    end,
+    lists:foldl(FoldFun, #{}, filelib:wildcard(WildcardPath));
+get_index_files(Db) ->
+    get_index_files(couch_db:name(Db)).
 
 get_view(Db, DDoc, ViewName, Args0) ->
     case get_view_index_state(Db, DDoc, ViewName, Args0) of
diff --git a/src/couch_mrview/test/eunit/couch_mrview_util_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_util_tests.erl
index a495fd82c..2562bb511 100644
--- a/src/couch_mrview/test/eunit/couch_mrview_util_tests.erl
+++ b/src/couch_mrview/test/eunit/couch_mrview_util_tests.erl
@@ -13,8 +13,11 @@
 -module(couch_mrview_util_tests).
 
 -include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
 -include_lib("couch_mrview/include/couch_mrview.hrl").
 
+-define(DDOC_ID, <<"_design/bar">>).
+
 couch_mrview_util_test_() ->
     [
         ?_assertEqual(0, validate_group_level(undefined, undefined)),
@@ -35,3 +38,139 @@ validate_group_level(Group, GroupLevel) ->
     Args0 = #mrargs{group = Group, group_level = GroupLevel, view_type = red},
     Args1 = couch_mrview_util:validate_args(Args0),
     Args1#mrargs.group_level.
+
+get_signature_from_filename_test() ->
+    Sig = "da817c3d3f7413c1a610f25635a0c521",
+    P1 = "/x.1667618375_design/mrview/da817c3d3f7413c1a610f25635a0c521.view",
+    P2 = "/x.1667618375_design/mrview/da817c3d3f7413c1a610f25635a0c521.compact.view",
+    P3 = "/x.1667618375_design/mrview/da817c3d3f7413c1a610f25635a0c521",
+    ?assertEqual(Sig, couch_mrview_util:get_signature_from_filename(P1)),
+    ?assertEqual(Sig, couch_mrview_util:get_signature_from_filename(P2)),
+    ?assertEqual(Sig, couch_mrview_util:get_signature_from_filename(P3)).
+
+verify_view_filename_test() ->
+    P1 = "/x.1667618375_design/mrview/da817c3d3f7413c1a610f25635a0c521.view",
+    P2 = "/x.1667618375_design/mrview/da817c3d3f7413c1a610f25635a0c521.compact.view",
+    P3 = "/x.1667618375_design/mrview/da817c3d3f7413c1a610f25635a0c521",
+    ?assert(couch_mrview_util:verify_view_filename(P1)),
+    ?assert(couch_mrview_util:verify_view_filename(P2)),
+    ?assertNot(couch_mrview_util:verify_view_filename(P3)),
+    ?assertNot(couch_mrview_util:verify_view_filename("")),
+    ?assertNot(couch_mrview_util:verify_view_filename("foo.view")).
+
+setup() ->
+    DbName = ?tempdb(),
+    ok = fabric:create_db(DbName, [?ADMIN_CTX, {q, 2}]),
+    DDoc = couch_mrview_test_util:ddoc(map),
+    {ok, _} = fabric:update_doc(DbName, DDoc, [?ADMIN_CTX]),
+    {ok, _} = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}),
+    {ok, Db} = couch_mrview_test_util:init_db(?tempdb(), map),
+    {ok, _} = couch_mrview:query_view(Db, ?DDOC_ID, <<"baz">>),
+    {DbName, Db}.
+
+teardown({DbName, Db}) ->
+    couch_db:close(Db),
+    couch_server:delete(couch_db:name(Db), [?ADMIN_CTX]),
+    ok = fabric:delete_db(DbName, [?ADMIN_CTX]).
+
+get_signatures_test_() ->
+    {
+        setup,
+        fun() -> test_util:start_couch([fabric]) end,
+        fun test_util:stop_couch/1,
+        {
+            foreach,
+            fun setup/0,
+            fun teardown/1,
+            [
+                ?TDEF_FE(t_get_signatures_local),
+                ?TDEF_FE(t_get_signatures_clustered),
+                ?TDEF_FE(t_get_purge_checkpoints_local),
+                ?TDEF_FE(t_get_purge_checkpoints_clustered),
+                ?TDEF_FE(t_get_index_files_local),
+                ?TDEF_FE(t_get_index_files_clustered)
+            ]
+        }
+    }.
+
+t_get_signatures_local({_, Db}) ->
+    DbName = couch_db:name(Db),
+    Sigs = couch_mrview_util:get_signatures(DbName),
+    ?assert(is_map(Sigs)),
+    ?assertEqual(1, map_size(Sigs)),
+    [{Sig, true}] = maps:to_list(Sigs),
+    {ok, Info} = couch_mrview:get_info(Db, ?DDOC_ID),
+    ?assertEqual(proplists:get_value(signature, Info), Sig),
+
+    {ok, DDoc} = couch_db:open_doc(Db, ?DDOC_ID, [?ADMIN_CTX]),
+    Deleted = DDoc#doc{deleted = true, body = {[]}},
+    {ok, _} = couch_db:update_doc(Db, Deleted, []),
+    ?assertEqual(#{}, couch_mrview_util:get_signatures(DbName)).
+
+t_get_signatures_clustered({DbName, _Db}) ->
+    [Shard1, Shard2] = mem3:local_shards(DbName),
+    ShardName1 = mem3:name(Shard1),
+    ShardName2 = mem3:name(Shard2),
+    Sigs = couch_mrview_util:get_signatures(ShardName1),
+    ?assertEqual(Sigs, couch_mrview_util:get_signatures(ShardName2)),
+    ?assert(is_map(Sigs)),
+    ?assertEqual(1, map_size(Sigs)),
+    [{Sig, true}] = maps:to_list(Sigs),
+    {ok, Info} = couch_mrview:get_info(ShardName1, ?DDOC_ID),
+    ?assertEqual(proplists:get_value(signature, Info), Sig),
+
+    {ok, DDoc} = fabric:open_doc(DbName, ?DDOC_ID, [?ADMIN_CTX]),
+    Deleted = DDoc#doc{deleted = true, body = {[]}},
+    {ok, _} = fabric:update_doc(DbName, Deleted, [?ADMIN_CTX]),
+    ?assertEqual(#{}, couch_mrview_util:get_signatures(ShardName1)),
+    ?assertEqual(#{}, couch_mrview_util:get_signatures(ShardName2)).
+
+t_get_purge_checkpoints_local({_, Db}) ->
+    DbName = couch_db:name(Db),
+    Checkpoints = couch_mrview_util:get_purge_checkpoints(DbName),
+    ?assert(is_map(Checkpoints)),
+    ?assertEqual(1, map_size(Checkpoints)),
+    [{Sig, <<"_local/", _/binary>>}] = maps:to_list(Checkpoints),
+    {ok, Info} = couch_mrview:get_info(Db, ?DDOC_ID),
+    ?assertEqual(proplists:get_value(signature, Info), Sig).
+
+t_get_purge_checkpoints_clustered({DbName, _Db}) ->
+    {ok, _} = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}),
+    [Shard1, Shard2] = mem3:local_shards(DbName),
+    ShardName1 = mem3:name(Shard1),
+    ShardName2 = mem3:name(Shard2),
+    Sigs1 = couch_mrview_util:get_purge_checkpoints(ShardName1),
+    Sigs2 = couch_mrview_util:get_purge_checkpoints(ShardName2),
+    ?assertEqual(lists:sort(maps:keys(Sigs1)), lists:sort(maps:keys(Sigs2))),
+    ?assert(is_map(Sigs1)),
+    ?assertEqual(1, map_size(Sigs1)),
+    [{Sig, <<"_local/", _/binary>>}] = maps:to_list(Sigs1),
+    {ok, Info} = couch_mrview:get_info(ShardName1, ?DDOC_ID),
+    ?assertEqual(proplists:get_value(signature, Info), Sig).
+
+t_get_index_files_local({_, Db}) ->
+    DbName = couch_db:name(Db),
+    SigFilesMap = couch_mrview_util:get_index_files(DbName),
+    ?assert(is_map(SigFilesMap)),
+    ?assertEqual(1, map_size(SigFilesMap)),
+    [{Sig, [File]}] = maps:to_list(SigFilesMap),
+    ?assertMatch({ok, _}, file:read_file_info(File)),
+    {ok, Info} = couch_mrview:get_info(Db, ?DDOC_ID),
+    ?assertEqual(proplists:get_value(signature, Info), Sig).
+
+t_get_index_files_clustered({DbName, _Db}) ->
+    {ok, _} = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}),
+    [Shard1, Shard2] = mem3:local_shards(DbName),
+    ShardName1 = mem3:name(Shard1),
+    ShardName2 = mem3:name(Shard2),
+    SigFilesMap1 = couch_mrview_util:get_index_files(ShardName1),
+    SigFilesMap2 = couch_mrview_util:get_index_files(ShardName2),
+    SigKeys1 = lists:sort(maps:keys(SigFilesMap1)),
+    SigKeys2 = lists:sort(maps:keys(SigFilesMap2)),
+    ?assertEqual(SigKeys1, SigKeys2),
+    ?assert(is_map(SigFilesMap1)),
+    ?assertEqual(1, map_size(SigFilesMap1)),
+    [{Sig, [File]}] = maps:to_list(SigFilesMap1),
+    ?assertMatch({ok, _}, file:read_file_info(File)),
+    {ok, Info} = couch_mrview:get_info(ShardName1, ?DDOC_ID),
+    ?assertEqual(proplists:get_value(signature, Info), Sig).
diff --git a/src/fabric/src/fabric.erl b/src/fabric/src/fabric.erl
index e61521da0..acd7f1190 100644
--- a/src/fabric/src/fabric.erl
+++ b/src/fabric/src/fabric.erl
@@ -65,7 +65,6 @@
     cleanup_index_files/1,
     cleanup_index_files_all_nodes/1,
     dbname/1,
-    inactive_index_files/1,
     db_uuids/1
 ]).
 
@@ -586,58 +585,34 @@ cleanup_index_files() ->
 -spec cleanup_index_files(dbname()) -> ok.
 cleanup_index_files(DbName) ->
     try
-        lists:foreach(
-            fun(File) ->
-                file:delete(File)
-            end,
-            inactive_index_files(DbName)
-        )
+        ShardNames = [mem3:name(S) || S <- mem3:local_shards(dbname(DbName))],
+        cleanup_local_indices_and_purge_checkpoints(ShardNames)
     catch
-        error:Error ->
-            couch_log:error(
-                "~p:cleanup_index_files. Error: ~p",
-                [?MODULE, Error]
-            ),
+        error:database_does_not_exist ->
             ok
     end.
 
-%% @doc inactive index files for a specific db
--spec inactive_index_files(dbname()) -> ok.
-inactive_index_files(DbName) ->
-    {ok, DesignDocs} = fabric:design_docs(DbName),
-
-    ActiveSigs = maps:from_list(
-        lists:map(
-            fun(#doc{id = GroupId}) ->
-                {ok, Info} = fabric:get_view_group_info(DbName, GroupId),
-                {binary_to_list(couch_util:get_value(signature, Info)), nil}
-            end,
-            [couch_doc:from_json_obj(DD) || DD <- DesignDocs]
-        )
-    ),
-
-    FileList = lists:flatmap(
-        fun(#shard{name = ShardName}) ->
-            IndexDir = couch_index_util:index_dir(mrview, ShardName),
-            filelib:wildcard([IndexDir, "/*"])
-        end,
-        mem3:local_shards(dbname(DbName))
-    ),
+cleanup_local_indices_and_purge_checkpoints([]) ->
+    ok;
+cleanup_local_indices_and_purge_checkpoints([_ | _] = Dbs) ->
+    AllIndices = lists:map(fun couch_mrview_util:get_index_files/1, Dbs),
+    AllPurges = lists:map(fun couch_mrview_util:get_purge_checkpoints/1, Dbs),
+    Sigs = couch_mrview_util:get_signatures(hd(Dbs)),
+    ok = cleanup_purges(Sigs, AllPurges, Dbs),
+    ok = cleanup_indices(Sigs, AllIndices).
+
+cleanup_purges(Sigs, AllPurges, Dbs) ->
+    Fun = fun(DbPurges, Db) ->
+        couch_mrview_cleanup:cleanup_purges(Db, Sigs, DbPurges)
+    end,
+    lists:zipwith(Fun, AllPurges, Dbs),
+    ok.
 
-    if
-        ActiveSigs =:= [] ->
-            FileList;
-        true ->
-            %% <sig>.view and <sig>.compact.view where <sig> is in ActiveSigs
-            %% will be excluded from FileList because they are active view
-            %% files and should not be deleted.
-            lists:filter(
-                fun(FilePath) ->
-                    not maps:is_key(get_view_sig_from_filename(FilePath), ActiveSigs)
-                end,
-                FileList
-            )
-    end.
+cleanup_indices(Sigs, AllIndices) ->
+    Fun = fun(DbIndices) ->
+        couch_mrview_cleanup:cleanup_indices(Sigs, DbIndices)
+    end,
+    lists:foreach(Fun, AllIndices).
 
 %% @doc clean up index files for a specific db on all nodes
 -spec cleanup_index_files_all_nodes(dbname()) -> [reference()].
@@ -791,9 +766,6 @@ kl_to_record(KeyList, RecName) ->
 set_namespace(NS, #mrargs{extra = Extra} = Args) ->
     Args#mrargs{extra = [{namespace, NS} | Extra]}.
 
-get_view_sig_from_filename(FilePath) ->
-    filename:basename(filename:basename(FilePath, ".view"), ".compact").
-
 -ifdef(TEST).
 -include_lib("eunit/include/eunit.hrl").
 
diff --git a/src/fabric/test/eunit/fabric_tests.erl b/src/fabric/test/eunit/fabric_tests.erl
index c0e2b626b..76400a9fe 100644
--- a/src/fabric/test/eunit/fabric_tests.erl
+++ b/src/fabric/test/eunit/fabric_tests.erl
@@ -12,48 +12,229 @@
 
 -module(fabric_tests).
 
+-include_lib("couch/include/couch_db.hrl").
 -include_lib("couch/include/couch_eunit.hrl").
 
 cleanup_index_files_test_() ->
     {
-        setup,
+        foreach,
         fun setup/0,
         fun teardown/1,
-        fun(Ctx) ->
-            [
-                t_cleanup_index_files(),
-                t_cleanup_index_files_with_existing_db(Ctx),
-                t_cleanup_index_files_with_deleted_db(Ctx)
-            ]
-        end
+        [
+            ?TDEF_FE(t_cleanup_index_files),
+            ?TDEF_FE(t_cleanup_index_files_with_existing_db),
+            ?TDEF_FE(t_cleanup_index_files_with_view_data),
+            ?TDEF_FE(t_cleanup_index_files_with_deleted_db),
+            ?TDEF_FE(t_cleanup_index_file_after_ddoc_update),
+            ?TDEF_FE(t_cleanup_index_file_after_ddoc_delete)
+        ]
     }.
 
 setup() ->
     Ctx = test_util:start_couch([fabric]),
-    % TempDb is deleted in the test "t_cleanup_index_files_with_deleted_db".
-    TempDb = ?tempdb(),
-    fabric:create_db(TempDb),
-    {Ctx, TempDb}.
+    DbName = ?tempdb(),
+    fabric:create_db(DbName, [{q, 1}]),
+    create_ddoc(DbName, <<"_design/foo">>, <<"bar">>),
+    {ok, _} = fabric:query_view(DbName, <<"foo">>, <<"bar">>),
+    create_ddoc(DbName, <<"_design/boo">>, <<"baz">>),
+    {ok, _} = fabric:query_view(DbName, <<"boo">>, <<"baz">>),
+    {Ctx, DbName}.
 
-teardown({Ctx, _TempDb}) ->
+teardown({Ctx, DbName}) ->
+    fabric:delete_db(DbName),
     test_util:stop_couch(Ctx).
 
-t_cleanup_index_files() ->
-    ?_assert(
-        lists:all(fun(Res) -> Res =:= ok end, fabric:cleanup_index_files())
+t_cleanup_index_files(_) ->
+    CheckFun = fun(Res) -> Res =:= ok end,
+    ?assert(lists:all(CheckFun, fabric:cleanup_index_files())).
+
+t_cleanup_index_files_with_existing_db({_, DbName}) ->
+    ?assertEqual(ok, fabric:cleanup_index_files(DbName)).
+
+t_cleanup_index_files_with_view_data({_, DbName}) ->
+    Sigs = sigs(DbName),
+    Indices = indices(DbName),
+    Purges = purges(DbName),
+    ok = fabric:cleanup_index_files(DbName),
+    % We haven't inadvertently removed any active index bits
+    ?assertEqual(Sigs, sigs(DbName)),
+    ?assertEqual(Indices, indices(DbName)),
+    ?assertEqual(Purges, purges(DbName)).
+
+t_cleanup_index_files_with_deleted_db(_) ->
+    SomeDb = ?tempdb(),
+    ?assertEqual(ok, fabric:cleanup_index_files(SomeDb)).
+
+t_cleanup_index_file_after_ddoc_update({_, DbName}) ->
+    ?assertEqual(
+        [
+            "4bcdf852098ff6b0578ddf472c320e9c.view",
+            "da817c3d3f7413c1a610f25635a0c521.view"
+        ],
+        indices(DbName)
+    ),
+    ?assertEqual(
+        [
+            <<"_local/purge-mrview-4bcdf852098ff6b0578ddf472c320e9c">>,
+            <<"_local/purge-mrview-da817c3d3f7413c1a610f25635a0c521">>
+        ],
+        purges(DbName)
+    ),
+
+    update_ddoc(DbName, <<"_design/foo">>, <<"bar1">>),
+    ok = fabric:cleanup_index_files(DbName),
+    {ok, _} = fabric:query_view(DbName, <<"foo">>, <<"bar1">>),
+
+    % One 4bc stays, da8 should  gone and 9e3 is added
+    ?assertEqual(
+        [
+            "4bcdf852098ff6b0578ddf472c320e9c.view",
+            "9e355b0fee411b4257036b8fca56f263.view"
+        ],
+        indices(DbName)
+    ),
+    ?assertEqual(
+        [
+            <<"_local/purge-mrview-4bcdf852098ff6b0578ddf472c320e9c">>,
+            <<"_local/purge-mrview-9e355b0fee411b4257036b8fca56f263">>
+        ],
+        purges(DbName)
     ).
 
-t_cleanup_index_files_with_existing_db({_Ctx, TempDb}) ->
-    ?_assertEqual(ok, fabric:cleanup_index_files(TempDb)).
+t_cleanup_index_file_after_ddoc_delete({_, DbName}) ->
+    ?assertEqual(
+        [
+            "4bcdf852098ff6b0578ddf472c320e9c.view",
+            "da817c3d3f7413c1a610f25635a0c521.view"
+        ],
+        indices(DbName)
+    ),
+    ?assertEqual(
+        [
+            <<"_local/purge-mrview-4bcdf852098ff6b0578ddf472c320e9c">>,
+            <<"_local/purge-mrview-da817c3d3f7413c1a610f25635a0c521">>
+        ],
+        purges(DbName)
+    ),
+
+    delete_ddoc(DbName, <<"_design/foo">>),
+    ok = fabric:cleanup_index_files(DbName),
+
+    % 4bc stays the same, da8 should be gone
+    ?assertEqual(
+        [
+            "4bcdf852098ff6b0578ddf472c320e9c.view"
+        ],
+        indices(DbName)
+    ),
+    ?assertEqual(
+        [
+            <<"_local/purge-mrview-4bcdf852098ff6b0578ddf472c320e9c">>
+        ],
+        purges(DbName)
+    ),
+
+    delete_ddoc(DbName, <<"_design/boo">>),
+    ok = fabric:cleanup_index_files(DbName),
+
+    ?assertEqual([], indices(DbName)),
+    ?assertEqual([], purges(DbName)),
+
+    % cleaning a db with all deleted indices should still work
+    ok = fabric:cleanup_index_files(DbName),
 
-t_cleanup_index_files_with_deleted_db({_Ctx, TempDb}) ->
-    ?_test(
-        begin
-            fabric:delete_db(TempDb, []),
-            ?assertError(
-                database_does_not_exist,
-                fabric:inactive_index_files(TempDb)
+    ?assertEqual([], indices(DbName)),
+    ?assertEqual([], purges(DbName)).
+
+shard_names(DbName) ->
+    [mem3:name(S) || S <- mem3:local_shards(DbName)].
+
+% Sorted list of sigs
+%
+sigs(DbName) ->
+    case shard_names(DbName) of
+        [] ->
+            [];
+        [SomeDb | _] ->
+            Sigs = couch_mrview_util:get_signatures(SomeDb),
+            lists:sort(maps:keys(Sigs))
+    end.
+
+% Sorted list of index files
+%
+indices(DbName) ->
+    case shard_names(DbName) of
+        [] ->
+            [];
+        [_ | _] = Dbs ->
+            AllIndices = lists:map(fun couch_mrview_util:get_index_files/1, Dbs),
+            AsList = lists:sort(
+                lists:foldl(
+                    fun(Indices, Acc) ->
+                        maps:values(Indices) ++ Acc
+                    end,
+                    [],
+                    AllIndices
+                )
             ),
-            ?assertEqual(ok, fabric:cleanup_index_files(TempDb))
-        end
-    ).
+            % Keep only file names and extensions. Since we use q=1, we shouldn't
+            % have any duplicates
+            [filename:basename(F) || F <- AsList]
+    end.
+
+% Sorted list of purge checkpoint doc ids
+%
+purges(DbName) ->
+    case shard_names(DbName) of
+        [] ->
+            [];
+        [_ | _] = Dbs ->
+            AllPurges = lists:map(fun couch_mrview_util:get_purge_checkpoints/1, Dbs),
+            lists:sort(
+                lists:foldl(
+                    fun(Purges, Acc) ->
+                        maps:values(Purges) ++ Acc
+                    end,
+                    [],
+                    AllPurges
+                )
+            )
+    end.
+
+create_ddoc(DbName, DDocId, ViewName) ->
+    DDoc = couch_doc:from_json_obj(
+        {[
+            {<<"_id">>, DDocId},
+            {<<"language">>, <<"javascript">>},
+            {<<"views">>,
+                {[
+                    {ViewName,
+                        {[
+                            {<<"map">>, <<"function(doc) { emit(doc.value, null); }">>}
+                        ]}}
+                ]}}
+        ]}
+    ),
+    fabric:update_doc(DbName, DDoc, [?ADMIN_CTX]).
+
+update_ddoc(DbName, DDocId, ViewName) ->
+    {ok, DDoc0} = fabric:open_doc(DbName, DDocId, [?ADMIN_CTX]),
+    DDoc = DDoc0#doc{
+        body =
+            {[
+                {<<"language">>, <<"javascript">>},
+                {<<"views">>,
+                    {[
+                        {ViewName,
+                            {[
+                                {<<"map">>, <<"function(doc) { emit(doc.value, 1); }">>}
+                            ]}}
+                    ]}}
+            ]}
+    },
+    fabric:update_doc(DbName, DDoc, [?ADMIN_CTX]).
+
+delete_ddoc(DbName, DDocId) ->
+    {ok, DDoc0} = fabric:open_doc(DbName, DDocId, [?ADMIN_CTX]),
+    DDoc = DDoc0#doc{deleted = true, body = {[]}},
+    fabric:update_doc(DbName, DDoc, [?ADMIN_CTX]).