diff options
author | Nick Vatamaniuc <vatamane@gmail.com> | 2021-04-22 13:17:19 -0400 |
---|---|---|
committer | Nick Vatamaniuc <nickva@users.noreply.github.com> | 2021-04-22 16:16:16 -0400 |
commit | cb8e066e2e731a245a74c0b93a6b51500691dc0d (patch) | |
tree | 01f5cc0a4a602ddda2452e290b19e02f91dd0025 | |
parent | bf9b451689482ea3f046a4551300af73fbcba616 (diff) | |
download | couchdb-cb8e066e2e731a245a74c0b93a6b51500691dc0d.tar.gz |
Add `COUCH_JOBS_RETRYABLE` macro to couch_jobs.hrl
This macro can be used to simplify retryable error checks throughout couch_jobs
app. It checks for erlfdb retryable errors (1007, 1009, etc), for the 1031
(`transaction_timed_out`) error and for `{timeout, _}`.
-rw-r--r-- | src/couch_jobs/src/couch_jobs.erl | 17 | ||||
-rw-r--r-- | src/couch_jobs/src/couch_jobs.hrl | 6 | ||||
-rw-r--r-- | src/couch_jobs/src/couch_jobs_activity_monitor.erl | 3 | ||||
-rw-r--r-- | src/couch_jobs/src/couch_jobs_server.erl | 8 | ||||
-rw-r--r-- | src/couch_jobs/src/couch_jobs_type_monitor.erl | 16 |
5 files changed, 20 insertions, 30 deletions
diff --git a/src/couch_jobs/src/couch_jobs.erl b/src/couch_jobs/src/couch_jobs.erl index 8d2fcd807..ca7bc4001 100644 --- a/src/couch_jobs/src/couch_jobs.erl +++ b/src/couch_jobs/src/couch_jobs.erl @@ -349,11 +349,7 @@ accept_loop(Type, NoSched, MaxSchedTime, Timeout) -> AcceptResult = try couch_jobs_fdb:tx(couch_jobs_fdb:get_jtx(), TxFun) catch - error:{timeout, _} -> - retry; - error:{erlfdb_error, ?ERLFDB_TRANSACTION_TIMED_OUT} -> - retry; - error:{erlfdb_error, Err} when ?ERLFDB_IS_RETRYABLE(Err) -> + error:{Tag, Err} when ?COUCH_JOBS_RETRYABLE(Tag, Err) -> retry end, case AcceptResult of @@ -389,15 +385,12 @@ wait_pending(PendingWatch, MaxSTime, UserTimeout, NoSched) -> erlfdb:wait(PendingWatch, [{timeout, Timeout}]), ok catch - error:{erlfdb_error, ?ERLFDB_TRANSACTION_TIMED_OUT} -> - erlfdb:cancel(PendingWatch, [flush]), - retry; - error:{erlfdb_error, Error} when ?ERLFDB_IS_RETRYABLE(Error) -> - erlfdb:cancel(PendingWatch, [flush]), - retry; error:{timeout, _} -> erlfdb:cancel(PendingWatch, [flush]), - {error, not_found} + {error, not_found}; + error:{Err, Tag} when ?COUCH_JOBS_RETRYABLE(Err, Tag) -> + erlfdb:cancel(PendingWatch, [flush]), + retry end. diff --git a/src/couch_jobs/src/couch_jobs.hrl b/src/couch_jobs/src/couch_jobs.hrl index bb561b136..a160605a0 100644 --- a/src/couch_jobs/src/couch_jobs.hrl +++ b/src/couch_jobs/src/couch_jobs.hrl @@ -40,6 +40,12 @@ -define(COUCH_JOBS_CURRENT, '$couch_jobs_current'). -define(UNDEFINED_MAX_SCHEDULED_TIME, 1 bsl 36). +-define(COUCH_JOBS_RETRYABLE(Tag, Err), ( + (Tag == timeout) orelse ( + (Tag == erlfdb_error andalso ?ERLFDB_IS_RETRYABLE(Err)) orelse + (Tag == erlfdb_error andalso Err =:= ?ERLFDB_TRANSACTION_TIMED_OUT)) +)). + -type jtx() :: map() | undefined | tuple(). -type job_id() :: binary(). diff --git a/src/couch_jobs/src/couch_jobs_activity_monitor.erl b/src/couch_jobs/src/couch_jobs_activity_monitor.erl index d5dfa41d3..7281ef626 100644 --- a/src/couch_jobs/src/couch_jobs_activity_monitor.erl +++ b/src/couch_jobs/src/couch_jobs_activity_monitor.erl @@ -72,8 +72,7 @@ handle_info(check_activity, St) -> St1 = try check_activity(St) catch - error:{erlfdb_error, Err} when ?ERLFDB_IS_RETRYABLE(Err) orelse - Err =:= ?ERLFDB_TRANSACTION_TIMED_OUT -> + error:{Tag, Err} when ?COUCH_JOBS_RETRYABLE(Tag, Err) -> LogMsg = "~p : type:~p got ~p error, possibly from overload", couch_log:error(LogMsg, [?MODULE, St#st.type, Err]), St diff --git a/src/couch_jobs/src/couch_jobs_server.erl b/src/couch_jobs/src/couch_jobs_server.erl index 2e03c7dcf..5b087ef4b 100644 --- a/src/couch_jobs/src/couch_jobs_server.erl +++ b/src/couch_jobs/src/couch_jobs_server.erl @@ -15,6 +15,9 @@ -behaviour(gen_server). +-include("couch_jobs.hrl"). + + -export([ start_link/0, get_notifier_server/1, @@ -170,8 +173,9 @@ fdb_types() -> couch_jobs_fdb:get_types(JTx) end) catch - error:{timeout, _} -> - couch_log:warning("~p : Timed out connecting to FDB", [?MODULE]), + error:{Tag, Err} when ?COUCH_JOBS_RETRYABLE(Tag, Err) -> + LogMsg = "~p : Error ~p:~p connecting to FDB", + couch_log:warning(LogMsg, [?MODULE, Tag, Err]), [] end. diff --git a/src/couch_jobs/src/couch_jobs_type_monitor.erl b/src/couch_jobs/src/couch_jobs_type_monitor.erl index b58f34ecf..95aee4e7a 100644 --- a/src/couch_jobs/src/couch_jobs_type_monitor.erl +++ b/src/couch_jobs/src/couch_jobs_type_monitor.erl @@ -55,13 +55,7 @@ loop(#st{vs = VS, timeout = Timeout} = St) -> try erlfdb:wait(Watch, [{timeout, Timeout}]) catch - error:{erlfdb_error, ?ERLFDB_TRANSACTION_TIMED_OUT} -> - erlfdb:cancel(Watch, [flush]), - ok; - error:{erlfdb_error, Code} when ?ERLFDB_IS_RETRYABLE(Code) -> - erlfdb:cancel(Watch, [flush]), - ok; - error:{timeout, _} -> + error:{Tag, Err} when ?COUCH_JOBS_RETRYABLE(Tag, Err) -> erlfdb:cancel(Watch, [flush]), ok end, @@ -88,13 +82,7 @@ get_vs_and_watch(#st{} = St) -> couch_jobs_fdb:get_activity_vs_and_watch(JTx1, Type) end) catch - error:{erlfdb_error, ?ERLFDB_TRANSACTION_TIMED_OUT} -> - timer:sleep(HoldOff), - get_vs_and_watch(St); - error:{erlfdb_error, Code} when ?ERLFDB_IS_RETRYABLE(Code) -> - timer:sleep(HoldOff), - get_vs_and_watch(St); - error:{timeout, _} -> + error:{Tag, Err} when ?COUCH_JOBS_RETRYABLE(Tag, Err) -> timer:sleep(HoldOff), get_vs_and_watch(St) end. |