Handle race condition during concurrent db creationfix-concurrent-db-creation-2613-3.0.x

Previously concurrent db creation requests could get past db_exists/1 and try
to create the shard document. The first one would succeed but the others would
fail with a `500 {"error":"error", "reason":"conflict"}` error instead of
`412`.

Handle the race condition by checking if db exists after getting a conflict
error and return a 412 response.

Fixes: #2613

2 files changed, 59 insertions, 0 deletions

diff --git a/src/fabric/src/fabric_db_create.erl b/src/fabric/src/fabric_db_create.erl
index 03fabb4ea..1048cb0bb 100644
--- a/src/fabric/src/fabric_db_create.erl
+++ b/src/fabric/src/fabric_db_create.erl
@@ -37,6 +37,12 @@ go(DbName, Options) ->
                 case {CreateShardResult, create_shard_db_doc(Doc)} of
                 {ok, {ok, Status}} ->
                     Status;
+                {ok, {error, conflict} = ShardDocError} ->
+                    % Check if it is just a race to create the shard doc
+                    case db_exists(DbName) of
+                        true -> {error, file_exists};
+                        false -> ShardDocError
+                    end;
                 {file_exists, {ok, _}} ->
                     {error, file_exists};
                 {_, Error} ->
diff --git a/src/fabric/test/eunit/fabric_db_create_tests.erl b/src/fabric/test/eunit/fabric_db_create_tests.erl
new file mode 100644
index 000000000..8e5b1085e
--- /dev/null
+++ b/src/fabric/test/eunit/fabric_db_create_tests.erl
@@ -0,0 +1,53 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric_db_create_tests).
+
+
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("mem3/include/mem3.hrl").
+
+
+-define(TDEF(A), {atom_to_list(A), fun A/0}).
+
+
+main_test_() ->
+    {
+        setup,
+        fun setup/0,
+        fun teardown/1,
+        [
+            ?TDEF(t_handle_shard_doc_conflict)
+        ]
+    }.
+
+
+setup() ->
+    test_util:start_couch([fabric]).
+
+
+teardown(Ctx) ->
+    meck:unload(),
+    test_util:stop_couch(Ctx).
+
+
+t_handle_shard_doc_conflict() ->
+    DbName = ?tempdb(),
+    meck:new(mem3, [passthrough]),
+    meck:new(fabric_util, [passthrough]),
+    ok = meck:sequence(mem3, shards, 1, [
+        fun(_) -> meck:raise(error, database_does_not_exist) end,
+        [#shard{dbname = DbName}]
+    ]),
+    meck:expect(fabric_util, recv, 4, {error, conflict}),
+    ?assertEqual({error, file_exists}, fabric_db_create:go(DbName, [])).