Add `COUCH_JOBS_RETRYABLE` macro to couch_jobs.hrl

This macro can be used to simplify retryable error checks throughout couch_jobs
app. It checks for erlfdb retryable errors (1007, 1009, etc), for the 1031
(`transaction_timed_out`) error and for `{timeout, _}`.

5 files changed, 20 insertions, 30 deletions

diff --git a/src/couch_jobs/src/couch_jobs.erl b/src/couch_jobs/src/couch_jobs.erl
index 8d2fcd807..ca7bc4001 100644
--- a/src/couch_jobs/src/couch_jobs.erl
+++ b/src/couch_jobs/src/couch_jobs.erl
@@ -349,11 +349,7 @@ accept_loop(Type, NoSched, MaxSchedTime, Timeout) ->
     AcceptResult = try
         couch_jobs_fdb:tx(couch_jobs_fdb:get_jtx(), TxFun)
     catch
-        error:{timeout, _} ->
-            retry;
-        error:{erlfdb_error, ?ERLFDB_TRANSACTION_TIMED_OUT} ->
-            retry;
-        error:{erlfdb_error, Err} when ?ERLFDB_IS_RETRYABLE(Err) ->
+        error:{Tag, Err} when ?COUCH_JOBS_RETRYABLE(Tag, Err) ->
             retry
     end,
     case AcceptResult of
@@ -389,15 +385,12 @@ wait_pending(PendingWatch, MaxSTime, UserTimeout, NoSched) ->
         erlfdb:wait(PendingWatch, [{timeout, Timeout}]),
         ok
     catch
-        error:{erlfdb_error, ?ERLFDB_TRANSACTION_TIMED_OUT} ->
-            erlfdb:cancel(PendingWatch, [flush]),
-            retry;
-        error:{erlfdb_error, Error} when ?ERLFDB_IS_RETRYABLE(Error) ->
-            erlfdb:cancel(PendingWatch, [flush]),
-            retry;
         error:{timeout, _} ->
             erlfdb:cancel(PendingWatch, [flush]),
-            {error, not_found}
+            {error, not_found};
+        error:{Err, Tag} when ?COUCH_JOBS_RETRYABLE(Err, Tag) ->
+            erlfdb:cancel(PendingWatch, [flush]),
+            retry
     end.
 
 
diff --git a/src/couch_jobs/src/couch_jobs.hrl b/src/couch_jobs/src/couch_jobs.hrl
index bb561b136..a160605a0 100644
--- a/src/couch_jobs/src/couch_jobs.hrl
+++ b/src/couch_jobs/src/couch_jobs.hrl
@@ -40,6 +40,12 @@
 -define(COUCH_JOBS_CURRENT, '$couch_jobs_current').
 -define(UNDEFINED_MAX_SCHEDULED_TIME, 1 bsl 36).
 
+-define(COUCH_JOBS_RETRYABLE(Tag, Err), (
+    (Tag == timeout) orelse (
+        (Tag == erlfdb_error andalso ?ERLFDB_IS_RETRYABLE(Err)) orelse
+        (Tag == erlfdb_error andalso Err =:= ?ERLFDB_TRANSACTION_TIMED_OUT))
+)).
+
 
 -type jtx() :: map() | undefined | tuple().
 -type job_id() :: binary().
diff --git a/src/couch_jobs/src/couch_jobs_activity_monitor.erl b/src/couch_jobs/src/couch_jobs_activity_monitor.erl
index d5dfa41d3..7281ef626 100644
--- a/src/couch_jobs/src/couch_jobs_activity_monitor.erl
+++ b/src/couch_jobs/src/couch_jobs_activity_monitor.erl
@@ -72,8 +72,7 @@ handle_info(check_activity, St) ->
     St1 = try
         check_activity(St)
     catch
-        error:{erlfdb_error, Err} when ?ERLFDB_IS_RETRYABLE(Err) orelse
-                Err =:= ?ERLFDB_TRANSACTION_TIMED_OUT ->
+        error:{Tag, Err} when ?COUCH_JOBS_RETRYABLE(Tag, Err) ->
             LogMsg = "~p : type:~p got ~p error, possibly from overload",
             couch_log:error(LogMsg, [?MODULE, St#st.type, Err]),
             St
diff --git a/src/couch_jobs/src/couch_jobs_server.erl b/src/couch_jobs/src/couch_jobs_server.erl
index 2e03c7dcf..5b087ef4b 100644
--- a/src/couch_jobs/src/couch_jobs_server.erl
+++ b/src/couch_jobs/src/couch_jobs_server.erl
@@ -15,6 +15,9 @@
 -behaviour(gen_server).
 
 
+-include("couch_jobs.hrl").
+
+
 -export([
     start_link/0,
     get_notifier_server/1,
@@ -170,8 +173,9 @@ fdb_types() ->
             couch_jobs_fdb:get_types(JTx)
         end)
     catch
-        error:{timeout, _} ->
-            couch_log:warning("~p : Timed out connecting to FDB", [?MODULE]),
+        error:{Tag, Err} when ?COUCH_JOBS_RETRYABLE(Tag, Err) ->
+            LogMsg = "~p : Error ~p:~p connecting to FDB",
+            couch_log:warning(LogMsg, [?MODULE, Tag, Err]),
             []
     end.
 
diff --git a/src/couch_jobs/src/couch_jobs_type_monitor.erl b/src/couch_jobs/src/couch_jobs_type_monitor.erl
index b58f34ecf..95aee4e7a 100644
--- a/src/couch_jobs/src/couch_jobs_type_monitor.erl
+++ b/src/couch_jobs/src/couch_jobs_type_monitor.erl
@@ -55,13 +55,7 @@ loop(#st{vs = VS, timeout = Timeout} = St) ->
     try
         erlfdb:wait(Watch, [{timeout, Timeout}])
     catch
-        error:{erlfdb_error, ?ERLFDB_TRANSACTION_TIMED_OUT} ->
-            erlfdb:cancel(Watch, [flush]),
-            ok;
-        error:{erlfdb_error, Code} when ?ERLFDB_IS_RETRYABLE(Code) ->
-            erlfdb:cancel(Watch, [flush]),
-            ok;
-        error:{timeout, _} ->
+        error:{Tag, Err} when ?COUCH_JOBS_RETRYABLE(Tag, Err) ->
             erlfdb:cancel(Watch, [flush]),
             ok
     end,
@@ -88,13 +82,7 @@ get_vs_and_watch(#st{} = St) ->
             couch_jobs_fdb:get_activity_vs_and_watch(JTx1, Type)
         end)
     catch
-        error:{erlfdb_error, ?ERLFDB_TRANSACTION_TIMED_OUT} ->
-            timer:sleep(HoldOff),
-            get_vs_and_watch(St);
-        error:{erlfdb_error, Code} when ?ERLFDB_IS_RETRYABLE(Code) ->
-            timer:sleep(HoldOff),
-            get_vs_and_watch(St);
-        error:{timeout, _} ->
+        error:{Tag, Err} when ?COUCH_JOBS_RETRYABLE(Tag, Err) ->
             timer:sleep(HoldOff),
             get_vs_and_watch(St)
     end.