From 3dd5b0f724fdda39c2dabccbaeac28191970ebfe Mon Sep 17 00:00:00 2001 From: Nick Vatamaniuc Date: Wed, 3 Mar 2021 14:59:37 -0500 Subject: Lower the view indexer transaction retry limit Previously, the view indexer used the default retry limit (100) from the `fdb_tx_options` config section. However, since the batching algorithm relies on sensing errors and reacting to them, retrying the batch 100 times before erroring out was not optimal. So value is lowered down to 5 and it's also made configurable. --- rel/overlay/etc/default.ini | 13 +++++++++++++ src/couch_views/src/couch_views_indexer.erl | 14 ++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini index ac4e2fa13..2668c3e81 100644 --- a/rel/overlay/etc/default.ini +++ b/rel/overlay/etc/default.ini @@ -351,6 +351,19 @@ iterations = 10 ; iterations for password hashing ; batch_max_tx_size_bytes = 9000000 ; Maximum transaction size in bytes ; batch_max_tx_time_msec = 4500 ; Maximum transaction time in milliseconds ; batch_thresold_penalty = 0.2 ; Amount to reduce batch size when crossing a threshold +; +; Set the FDB transaction retry limit applied to the main indexing transaction +; for retriable errors. These errors include 1007 (transaction_too_old), 1020 +; (commit conflict) and others. This value overrides the retry_limit setting +; from fdb transaction settings in the [fdb_tx_options] section. Setting this +; value may interfere with the batching algorithm as it's designed to adjust +; the batch size in response to erlfdb errors. A too large a value, would +; re-run the same transaction too many times in a row wasting resources. +; However, the value shouldn't be too small as in some cases the errors are +; recoverable such as when the FDB server fast-forwards time during the +; transaction system recovery. +; +;indexer_tx_retry_limit = 5 ; CSP (Content Security Policy) Support for _utils [csp] diff --git a/src/couch_views/src/couch_views_indexer.erl b/src/couch_views/src/couch_views_indexer.erl index 5a88cc606..3a4da34a7 100644 --- a/src/couch_views/src/couch_views_indexer.erl +++ b/src/couch_views/src/couch_views_indexer.erl @@ -37,6 +37,9 @@ -define(KEY_SIZE_LIMIT, 8000). -define(VALUE_SIZE_LIMIT, 64000). +-define(DEFAULT_TX_RETRY_LIMIT, 5). + + % These are all of the errors that we can fix by using % a smaller batch size. -define(IS_RECOVERABLE_ERROR(Code), ( @@ -106,7 +109,8 @@ init() -> changes_done => 0, doc_acc => [], design_opts => Mrst#mrst.design_opts, - update_stats => #{} + update_stats => #{}, + tx_retry_limit => tx_retry_limit() }, try @@ -204,7 +208,8 @@ update(#{} = Db, Mrst0, State0) -> do_update(Db, Mrst0, State0) -> - fabric2_fdb:transactional(Db, fun(TxDb) -> + TxOpts = #{retry_limit => maps:get(tx_retry_limit, State0)}, + fabric2_fdb:transactional(Db, TxOpts, fun(TxDb) -> #{ tx := Tx } = TxDb, @@ -647,3 +652,8 @@ key_size_limit() -> value_size_limit() -> config:get_integer("couch_views", "value_size_limit", ?VALUE_SIZE_LIMIT). + + +tx_retry_limit() -> + config:get_integer("couch_views", "indexer_tx_retry_limit", + ?DEFAULT_TX_RETRY_LIMIT). -- cgit v1.2.1