Add _nouveau_cleanup

11 files changed, 110 insertions, 23 deletions

diff --git a/nouveau/base/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java b/nouveau/base/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java
index 4103bb043..a4c01df5d 100644
--- a/nouveau/base/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java
+++ b/nouveau/base/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java
@@ -20,6 +20,7 @@ import java.io.IOException;
 import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.List;
 import java.util.Map.Entry;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
@@ -115,20 +116,26 @@ public final class IndexManager implements Managed {
         return Files.exists(indexDefinitionPath(name));
     }
 
-    public void deleteAll(final String path) throws IOException {
-        final Path rootPath = indexRootPath(path);
-        LOGGER.info("deleting indexes below {}", rootPath);
-        if (!rootPath.toFile().exists()) {
+    public void deleteAll(final String path, final List<String> exclusions) throws IOException {
+        LOGGER.info("deleting indexes below {} (excluding {})", path,
+            exclusions == null ? "nothing" : exclusions);
+
+        final Path indexRootPath = indexRootPath(path);
+        if (!indexRootPath.toFile().exists()) {
             return;
         }
-        Stream<Path> stream = Files.find(rootPath, 100,
-                (p, attr) -> attr.isDirectory() && isIndex(p));
+        Stream<Path> stream = Files.find(indexRootPath, 100,
+            (p, attr) -> attr.isDirectory() && isIndex(p));
         try {
             stream.forEach((p) -> {
+                final String relativeToExclusions = indexRootPath.relativize(p).toString();
+                if (exclusions != null && exclusions.indexOf(relativeToExclusions) != -1) {
+                    return;
+                }
+                final String relativeName = rootDir.relativize(p).toString();
                 try {
-                    final String relativeName = rootDir.relativize(p).toString();
                     deleteIndex(relativeName);
-                } catch (Exception e) {
+                } catch (final IOException e) {
                     LOGGER.error("I/O exception deleting " + p, e);
                 }
             });
@@ -137,7 +144,7 @@ public final class IndexManager implements Managed {
         }
 
         // Clean any newly empty directories.
-        Path p = rootPath;
+        Path p = indexRootPath;
         do {
             final File f = p.toFile();
             if (f.isDirectory() && f.list().length == 0) {
diff --git a/nouveau/base/src/main/java/org/apache/couchdb/nouveau/health/BaseIndexHealthCheck.java b/nouveau/base/src/main/java/org/apache/couchdb/nouveau/health/BaseIndexHealthCheck.java
index 9ff772372..b11e669c3 100644
--- a/nouveau/base/src/main/java/org/apache/couchdb/nouveau/health/BaseIndexHealthCheck.java
+++ b/nouveau/base/src/main/java/org/apache/couchdb/nouveau/health/BaseIndexHealthCheck.java
@@ -41,7 +41,7 @@ public abstract class BaseIndexHealthCheck<T> extends HealthCheck {
     protected Result check() throws Exception {
         final String name = generateIndexName();
         try {
-            indexResource.deletePath(name);
+            indexResource.deletePath(name, null);
         } catch (IOException e) {
             // Ignored, index might not exist yet.
         }
@@ -59,7 +59,7 @@ public abstract class BaseIndexHealthCheck<T> extends HealthCheck {
                 return Result.healthy();
             }
         } finally {
-            indexResource.deletePath(name);
+            indexResource.deletePath(name, null);
         }
         return Result.unhealthy(name);
     }
diff --git a/nouveau/base/src/main/java/org/apache/couchdb/nouveau/resources/BaseIndexResource.java b/nouveau/base/src/main/java/org/apache/couchdb/nouveau/resources/BaseIndexResource.java
index 72202e5c5..159086d68 100644
--- a/nouveau/base/src/main/java/org/apache/couchdb/nouveau/resources/BaseIndexResource.java
+++ b/nouveau/base/src/main/java/org/apache/couchdb/nouveau/resources/BaseIndexResource.java
@@ -14,6 +14,8 @@
 package org.apache.couchdb.nouveau.resources;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
 
 import javax.validation.Valid;
 import javax.validation.constraints.NotNull;
@@ -42,8 +44,8 @@ public abstract class BaseIndexResource<T> {
         });
     }
 
-    public void deletePath(String path) throws IOException {
-        indexManager.deleteAll(path);
+    public void deletePath(String path, @Valid List<String> exclusions) throws IOException {
+        indexManager.deleteAll(path, exclusions);
     }
 
     public void createIndex(String name, @NotNull @Valid IndexDefinition indexDefinition)
diff --git a/nouveau/lucene4/src/main/java/org/apache/couchdb/nouveau/lucene4/resources/IndexResource.java b/nouveau/lucene4/src/main/java/org/apache/couchdb/nouveau/lucene4/resources/IndexResource.java
index 5be2342a7..8e2697046 100644
--- a/nouveau/lucene4/src/main/java/org/apache/couchdb/nouveau/lucene4/resources/IndexResource.java
+++ b/nouveau/lucene4/src/main/java/org/apache/couchdb/nouveau/lucene4/resources/IndexResource.java
@@ -14,6 +14,7 @@
 package org.apache.couchdb.nouveau.lucene4.resources;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.Map;
 
 import javax.validation.Valid;
@@ -85,8 +86,8 @@ public class IndexResource extends BaseIndexResource<IndexableField> {
 
     @DELETE
     @Override
-    public void deletePath(@PathParam("name") String path) throws IOException {
-        super.deletePath(path);
+    public void deletePath(@PathParam("name") String path, @Valid List<String> exclusions) throws IOException {
+        super.deletePath(path, exclusions);
     }
 
     @GET
diff --git a/nouveau/lucene9/src/main/java/org/apache/couchdb/nouveau/lucene9/resources/IndexResource.java b/nouveau/lucene9/src/main/java/org/apache/couchdb/nouveau/lucene9/resources/IndexResource.java
index d0cb5219b..ccffc73d9 100644
--- a/nouveau/lucene9/src/main/java/org/apache/couchdb/nouveau/lucene9/resources/IndexResource.java
+++ b/nouveau/lucene9/src/main/java/org/apache/couchdb/nouveau/lucene9/resources/IndexResource.java
@@ -14,6 +14,7 @@
 package org.apache.couchdb.nouveau.lucene9.resources;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.Map;
 
 import javax.validation.Valid;
@@ -85,8 +86,8 @@ public class IndexResource extends BaseIndexResource<IndexableField> {
 
     @DELETE
     @Override
-    public void deletePath(@PathParam("name") String path) throws IOException {
-        super.deletePath(path);
+    public void deletePath(@PathParam("name") String path, @Valid final List<String> exclusions) throws IOException {
+        super.deletePath(path, exclusions);
     }
 
     @GET
diff --git a/nouveau/server/nouveau.yaml b/nouveau/server/nouveau.yaml
index f47e44602..85e74a7e3 100644
--- a/nouveau/server/nouveau.yaml
+++ b/nouveau/server/nouveau.yaml
@@ -19,3 +19,7 @@ server:
     includedMethods:
       - GET
       - POST
+  requestLog:
+    appenders:
+      - type: console
+        target: stderr
diff --git a/src/nouveau/src/nouveau_api.erl b/src/nouveau/src/nouveau_api.erl
index 8d385be54..8ebd6d5b8 100644
--- a/src/nouveau/src/nouveau_api.erl
+++ b/src/nouveau/src/nouveau_api.erl
@@ -22,6 +22,7 @@
     index_info/1,
     create_index/2,
     delete_path/2,
+    delete_path/3,
     delete_doc/3,
     update_doc/4,
     search/2
@@ -77,10 +78,15 @@ create_index(#index{} = Index, IndexDefinition) ->
             send_error(Reason)
     end.
 
-delete_path(LuceneMajor, Path) when
-    is_integer(LuceneMajor), is_binary(Path)
+delete_path(LuceneMajor, Path) ->
+    delete_path(LuceneMajor, Path, []).
+
+delete_path(LuceneMajor, Path, Exclusions) when
+    is_integer(LuceneMajor), is_binary(Path), is_list(Exclusions)
 ->
-    Resp = send_if_enabled(index_path(LuceneMajor, Path), [?JSON_CONTENT_TYPE], delete, []),
+    Resp = send_if_enabled(
+        index_path(LuceneMajor, Path), [?JSON_CONTENT_TYPE], delete, jiffy:encode(Exclusions)
+    ),
     case Resp of
         {ok, "204", _, _} ->
             ok;
diff --git a/src/nouveau/src/nouveau_fabric_cleanup.erl b/src/nouveau/src/nouveau_fabric_cleanup.erl
new file mode 100644
index 000000000..cd4128fb1
--- /dev/null
+++ b/src/nouveau/src/nouveau_fabric_cleanup.erl
@@ -0,0 +1,43 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
+
+-module(nouveau_fabric_cleanup).
+
+-include_lib("couch/include/couch_db.hrl").
+
+-include("nouveau.hrl").
+-include_lib("mem3/include/mem3.hrl").
+
+-export([go/1]).
+
+go(DbName) ->
+    {ok, DesignDocs} = fabric:design_docs(DbName),
+    ActiveSigs =
+        lists:usort(
+            lists:flatmap(
+                fun(Doc) -> active_sigs(DbName, Doc) end,
+                [couch_doc:from_json_obj(DD) || DD <- DesignDocs]
+            )
+        ),
+    Shards = mem3:shards(DbName),
+    lists:foreach(
+        fun(Shard) ->
+            rexi:cast(Shard#shard.node, {nouveau_rpc, cleanup, [Shard#shard.name, ActiveSigs]})
+        end,
+        Shards
+    ).
+
+active_sigs(DbName, #doc{} = Doc) ->
+    Indexes = nouveau_util:design_doc_to_indexes(DbName, Doc),
+    lists:map(fun(Index) -> Index#index.sig end, Indexes).
diff --git a/src/nouveau/src/nouveau_httpd.erl b/src/nouveau/src/nouveau_httpd.erl
index 6ebce7394..623addd5f 100644
--- a/src/nouveau/src/nouveau_httpd.erl
+++ b/src/nouveau/src/nouveau_httpd.erl
@@ -18,7 +18,12 @@
 -include_lib("couch/include/couch_db.hrl").
 -include("nouveau.hrl").
 
--export([handle_analyze_req/1, handle_search_req/3, handle_info_req/3]).
+-export([
+    handle_analyze_req/1,
+    handle_search_req/3,
+    handle_info_req/3,
+    handle_cleanup_req/2
+]).
 
 -import(chttpd, [
     send_method_not_allowed/2,
@@ -125,6 +130,13 @@ handle_info_req(Req, _Db, _DDoc) ->
     check_if_enabled(),
     send_error(Req, {bad_request, "path not recognized"}).
 
+handle_cleanup_req(#httpd{method = 'POST'} = Req, Db) ->
+    couch_httpd:validate_ctype(Req, "application/json"),
+    ok = nouveau_fabric_cleanup:go(couch_db:name(Db)),
+    send_json(Req, 202, {[{ok, true}]});
+handle_cleanup_req(Req, _Db) ->
+    send_method_not_allowed(Req, "POST").
+
 include_docs(_DbName, Hits, false) ->
     Hits;
 include_docs(DbName, Hits, true) ->
diff --git a/src/nouveau/src/nouveau_httpd_handlers.erl b/src/nouveau/src/nouveau_httpd_handlers.erl
index 16999e103..971833d8c 100644
--- a/src/nouveau/src/nouveau_httpd_handlers.erl
+++ b/src/nouveau/src/nouveau_httpd_handlers.erl
@@ -22,6 +22,8 @@ url_handler(<<"_nouveau_analyze">>) ->
 url_handler(_) ->
     no_match.
 
+db_handler(<<"_nouveau_cleanup">>) ->
+    fun nouveau_httpd:handle_cleanup_req/2;
 db_handler(_) ->
     no_match.
 
diff --git a/src/nouveau/src/nouveau_rpc.erl b/src/nouveau/src/nouveau_rpc.erl
index a17b1c2ef..9612628c6 100644
--- a/src/nouveau/src/nouveau_rpc.erl
+++ b/src/nouveau/src/nouveau_rpc.erl
@@ -15,7 +15,11 @@
 
 -module(nouveau_rpc).
 
--export([search/3, info/2]).
+-export([
+    search/3,
+    info/2,
+    cleanup/2
+]).
 
 -include("nouveau.hrl").
 -import(nouveau_util, [index_path/1]).
@@ -46,4 +50,9 @@ search(DbName, #index{} = Index0, QueryArgs) ->
 info(DbName, #index{} = Index0) ->
     %% Incorporate the shard name into the record.
     Index1 = Index0#index{dbname = DbName},
-    rexi:reply(nouveau_api:index_info(Index1)).
\ No newline at end of file
+    rexi:reply(nouveau_api:index_info(Index1)).
+
+cleanup(DbName, Exclusions) ->
+    nouveau_api:delete_path(4, nouveau_util:index_name(DbName), Exclusions),
+    nouveau_api:delete_path(9, nouveau_util:index_name(DbName), Exclusions),
+    rexi:reply(ok).