Sleep and retry after insert conflicthandle-insert-conflict

Too many parallel attempts to insert the same keys can result in
`{erlfdb_error, 1020}`, which translates to:
"Transaction not committed due to conflict with another transaction"

This attempts to mitigate the problem by lowering the transaction
retry limit, catching transaction conflict errors, and sleeping for
a random time before retrying a final time.

1 files changed, 20 insertions, 2 deletions

diff --git a/src/couch_expiring_cache/src/couch_expiring_cache.erl b/src/couch_expiring_cache/src/couch_expiring_cache.erl
index f1ce20276..efabf5d9c 100644
--- a/src/couch_expiring_cache/src/couch_expiring_cache.erl
+++ b/src/couch_expiring_cache/src/couch_expiring_cache.erl
@@ -23,6 +23,9 @@
 -include_lib("couch_expiring_cache/include/couch_expiring_cache.hrl").
 
 
+-define(DEFAULT_MAX_RETRY_DELAY_MS, 100).
+
+
 -spec insert(Name :: binary(), Key :: binary(), Value :: binary(),
     StaleTS :: ?TIME_UNIT(), ExpiresTS :: ?TIME_UNIT()) -> ok.
 insert(Name, Key, Value, StaleTS, ExpiresTS)
@@ -38,8 +41,18 @@ insert(Tx, Name, Key, Value, StaleTS, ExpiresTS)
         when is_binary(Name), is_binary(Key), is_binary(Value),
         is_integer(StaleTS), is_integer(ExpiresTS) ->
     couch_jobs_fdb:tx(couch_jobs_fdb:get_jtx(Tx), fun(JTx) ->
-        couch_expiring_cache_fdb:insert(
-            JTx, Name, Key, Value, StaleTS, ExpiresTS)
+        try
+            #{tx := Tx0} = JTx,
+            erlfdb:set_option(Tx0, retry_limit, 1),
+            couch_expiring_cache_fdb:insert(
+                JTx, Name, Key, Value, StaleTS, ExpiresTS)
+        catch
+            error:{erlfdb_error, 1020} -> % conflict with another transaction
+                Delay = rand:uniform(max_retry_delay()),
+                timer:sleep(Delay),
+                couch_expiring_cache_fdb:insert(
+                    JTx, Name, Key, Value, StaleTS, ExpiresTS)
+        end
     end).
 
 
@@ -55,3 +68,8 @@ lookup(Tx, Name, Key) when is_binary(Name), is_binary(Key) ->
     couch_jobs_fdb:tx(couch_jobs_fdb:get_jtx(Tx), fun(JTx) ->
         couch_expiring_cache_fdb:lookup(JTx, Name, Key)
     end).
+
+
+max_retry_delay() ->
+    config:get_integer(
+        "couch_expiring_cache", "max_retry_delay", ?DEFAULT_MAX_RETRY_DELAY_MS).