Configurable FDB directory prefixes for CouchDB instances

Specifying a custom prefix allows having multiple CouchDB instances on a single
FDB cluster. This can be used for one form of multi-tenancy. It can also be
used for integration testing by creating a temporary prefix then deleting all
data in that directory when the test has finished.

6 files changed, 111 insertions, 6 deletions

diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index 59c89b062..9f892b64e 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -200,6 +200,11 @@ port = 6984
 ; attachments_timeout = 60000
 ; view_timeout = 3600000
 ; partition_view_timeout = 3600000
+;
+; Custom FDB directory prefix. All the nodes of the same CouchDB instance
+; should have a matching directory prefix in order to read and write the same
+; data. Changes to this value take effect only on node start-up.
+;fdb_directory = couchdb
 
 ; [rexi]
 ; buffer_count = 2000
diff --git a/src/couch_jobs/src/couch_jobs_fdb.erl b/src/couch_jobs/src/couch_jobs_fdb.erl
index 1317d03df..6903801a2 100644
--- a/src/couch_jobs/src/couch_jobs_fdb.erl
+++ b/src/couch_jobs/src/couch_jobs_fdb.erl
@@ -616,7 +616,8 @@ init_jtx(undefined) ->
 
 init_jtx({erlfdb_transaction, _} = Tx) ->
     Root = erlfdb_directory:root(),
-    CouchDB = erlfdb_directory:create_or_open(Tx, Root, [<<"couchdb">>]),
+    Dir = fabric2_server:fdb_directory(),
+    CouchDB = erlfdb_directory:create_or_open(Tx, Root, Dir),
     LayerPrefix = erlfdb_directory:get_name(CouchDB),
     Jobs = erlfdb_tuple:pack({?JOBS}, LayerPrefix),
     Version = erlfdb:wait(erlfdb:get(Tx, ?METADATA_VERSION_KEY)),
diff --git a/src/fabric/src/fabric2_dir_prefix_tests.erl b/src/fabric/src/fabric2_dir_prefix_tests.erl
new file mode 100644
index 000000000..c7bc8bba4
--- /dev/null
+++ b/src/fabric/src/fabric2_dir_prefix_tests.erl
@@ -0,0 +1,69 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric2_dir_prefix_tests).
+
+
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("eunit/include/eunit.hrl").
+
+
+-define(TDEF(A), {atom_to_list(A), fun A/0}).
+
+
+dir_prefix_test_() ->
+    {
+        "Test couchdb fdb directory prefix",
+        foreach,
+        fun() ->
+            % erlfdb, rexi and mem3 are all dependent apps for fabric. We make
+            % sure to start them so when fabric is started during the test it
+            % already has its dependencies
+            test_util:start_couch([erlfdb, rexi, mem3])
+        end,
+        fun(Ctx) ->
+            config:delete("fabric", "fdb_directory"),
+            ok = application:stop(fabric),
+            test_util:stop_couch(Ctx)
+        end,
+        [
+            ?TDEF(default_prefix),
+            ?TDEF(custom_prefix)
+        ]
+    }.
+
+
+default_prefix() ->
+    ok = application:start(fabric),
+
+    ?assertEqual([<<"couchdb">>], fabric2_server:fdb_directory()),
+
+    % Try again to test pdict caching code
+    ?assertEqual([<<"couchdb">>], fabric2_server:fdb_directory()),
+
+    % Check that we can create dbs
+    DbName = ?tempdb(),
+    ?assertMatch({ok, _}, fabric2_db:create(DbName, [])).
+
+
+custom_prefix() ->
+    ok = config:set("fabric", "fdb_directory", "couchdb_foo"),
+    ok = application:start(fabric),
+
+    ?assertEqual([<<"couchdb_foo">>], fabric2_server:fdb_directory()),
+
+    % Try again to test pdict caching code
+    ?assertEqual([<<"couchdb_foo">>], fabric2_server:fdb_directory()),
+
+    % Check that we can create dbs
+    DbName = ?tempdb(),
+    ?assertMatch({ok, _}, fabric2_db:create(DbName, [])).
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 3f02e6ac4..c58b5f674 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -258,7 +258,8 @@ exists(#{name := DbName} = Db) when is_binary(DbName) ->
 
 list_dbs(Tx, Callback, AccIn, Options) ->
     Root = erlfdb_directory:root(),
-    CouchDB = erlfdb_directory:create_or_open(Tx, Root, [<<"couchdb">>]),
+    Dir = fabric2_server:fdb_directory(),
+    CouchDB = erlfdb_directory:create_or_open(Tx, Root, Dir),
     LayerPrefix = erlfdb_directory:get_name(CouchDB),
     Prefix = erlfdb_tuple:pack({?ALL_DBS}, LayerPrefix),
     fold_range({tx, Tx}, Prefix, fun({K, _V}, Acc) ->
@@ -737,7 +738,8 @@ debug_cluster(Start, End) ->
 
 init_db(Tx, DbName, Options) ->
     Root = erlfdb_directory:root(),
-    CouchDB = erlfdb_directory:create_or_open(Tx, Root, [<<"couchdb">>]),
+    Dir = fabric2_server:fdb_directory(),
+    CouchDB = erlfdb_directory:create_or_open(Tx, Root, Dir),
     Prefix = erlfdb_directory:get_name(CouchDB),
     Version = erlfdb:wait(erlfdb:get(Tx, ?METADATA_VERSION_KEY)),
     #{
diff --git a/src/fabric/src/fabric2_server.erl b/src/fabric/src/fabric2_server.erl
index 5b826cd14..f88ceb643 100644
--- a/src/fabric/src/fabric2_server.erl
+++ b/src/fabric/src/fabric2_server.erl
@@ -19,7 +19,8 @@
     start_link/0,
     fetch/1,
     store/1,
-    remove/1
+    remove/1,
+    fdb_directory/0
 ]).
 
 
@@ -37,6 +38,8 @@
 
 
 -define(CLUSTER_FILE, "/usr/local/etc/foundationdb/fdb.cluster").
+-define(FDB_DIRECTORY, fdb_directory).
+-define(DEFAULT_FDB_DIRECTORY, <<"couchdb">>).
 
 
 start_link() ->
@@ -81,6 +84,14 @@ init(_) ->
     end,
     application:set_env(fabric, db, Db),
 
+    Dir = case config:get("fabric", "fdb_directory") of
+        Val when is_list(Val), length(Val) > 0 ->
+            [?l2b(Val)];
+        _ ->
+            [?DEFAULT_FDB_DIRECTORY]
+    end,
+    application:set_env(fabric, ?FDB_DIRECTORY, Dir),
+
     {ok, nil}.
 
 
@@ -102,3 +113,18 @@ handle_info(Msg, St) ->
 
 code_change(_OldVsn, St, _Extra) ->
     {ok, St}.
+
+
+fdb_directory() ->
+    case get(?FDB_DIRECTORY) of
+        undefined ->
+            case application:get_env(fabric, ?FDB_DIRECTORY) of
+                undefined ->
+                    erlang:error(fabric_application_not_started);
+                {ok, Dir} ->
+                    put(?FDB_DIRECTORY, Dir),
+                    Dir
+            end;
+        Dir ->
+            Dir
+    end.
diff --git a/src/fabric/src/fabric2_txids.erl b/src/fabric/src/fabric2_txids.erl
index ba427415d..06704f021 100644
--- a/src/fabric/src/fabric2_txids.erl
+++ b/src/fabric/src/fabric2_txids.erl
@@ -45,7 +45,8 @@ start_link() ->
 
 create(Tx, undefined) ->
     Root = erlfdb_directory:root(),
-    CouchDB = erlfdb_directory:create_or_open(Tx, Root, [<<"couchdb">>]),
+    Dir = fabric2_server:fdb_directory(),
+    CouchDB = erlfdb_directory:create_or_open(Tx, Root, Dir),
     Prefix = erlfdb_directory:get_name(CouchDB),
     create(Tx, Prefix);
 
@@ -136,7 +137,8 @@ clean(St, NeedsSweep) ->
 
 sweep(Tx, {Mega, Secs, Micro}) ->
     Root = erlfdb_directory:root(),
-    CouchDB = erlfdb_directory:create_or_open(Tx, Root, [<<"couchdb">>]),
+    Dir = fabric2_server:fdb_directory(),
+    CouchDB = erlfdb_directory:create_or_open(Tx, Root, Dir),
     Prefix = erlfdb_directory:get_name(CouchDB),
     StartKey = erlfdb_tuple:pack({?TX_IDS}, Prefix),
     EndKey = erlfdb_tuple:pack({?TX_IDS, Mega, Secs, Micro}, Prefix),