Import wiredtiger: eb1b72848c33e5c19a31a316ba6831c76c3a0241 from branch mongodb-4.2

ref: 58afeb82a6..eb1b72848c
for: 4.2.11

WT-6160       Fix format failure caused by stack overwrite
WT-6507       Exit cache eviction worker after our operation has timed out
WT-6602       Allow operation timeout ms to be passed to commit and rollback
WT-6666       Start op timer when we configure it in rollback and commit

9 files changed, 165 insertions, 61 deletions

diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py
index 473c6a62a36..93fc5cdb002 100644
--- a/src/third_party/wiredtiger/dist/api_data.py
+++ b/src/third_party/wiredtiger/dist/api_data.py
@@ -1401,11 +1401,11 @@ methods = {
     Config('name', '', r'''
         name of the transaction for tracing and debugging'''),
     Config('operation_timeout_ms', '0', r'''
-        when non-zero, a requested limit on the number of elapsed real time milliseconds taken
-        to complete database operations in this transaction.  Time is measured from the start
-        of each WiredTiger API call.  There is no guarantee any operation will not take longer
-        than this amount of time. If WiredTiger notices the limit has been exceeded, an operation
-        may return a WT_ROLLBACK error. Default is to have no limit''',
+        when non-zero, a requested limit on the time taken to complete operations in this
+        transaction. Time is measured in real time milliseconds from the start of each WiredTiger
+        API call. There is no guarantee any operation will not take longer than this amount of time.
+        If WiredTiger notices the limit has been exceeded, an operation may return a WT_ROLLBACK
+        error. Default is to have no limit''',
         min=1),
     Config('priority', 0, r'''
         priority of the transaction for resolving conflicts.
@@ -1455,6 +1455,13 @@ methods = {
         current transaction.  The value must also not be older than the
         current stable timestamp.  See
         @ref transaction_timestamps'''),
+    Config('operation_timeout_ms', '0', r'''
+        when non-zero, a requested limit on the time taken to complete operations in this
+        transaction. Time is measured in real time milliseconds from the start of each WiredTiger
+        API call. There is no guarantee any operation will not take longer than this amount of time.
+        If WiredTiger notices the limit has been exceeded, an operation may return a WT_ROLLBACK
+        error. Default is to have no limit''',
+        min=1),
     Config('sync', '', r'''
         override whether to sync log records when the transaction commits,
         inherited from ::wiredtiger_open \c transaction_sync.
@@ -1496,7 +1503,15 @@ methods = {
         for a transaction. See @ref transaction_timestamps'''),
 ]),
 
-'WT_SESSION.rollback_transaction' : Method([]),
+'WT_SESSION.rollback_transaction' : Method([
+    Config('operation_timeout_ms', '0', r'''
+        when non-zero, a requested limit on the time taken to complete operations in this
+        transaction. Time is measured in real time milliseconds from the start of each WiredTiger
+        API call. There is no guarantee any operation will not take longer than this amount of time.
+        If WiredTiger notices the limit has been exceeded, an operation may return a WT_ROLLBACK
+        error. Default is to have no limit''',
+        min=1),
+]),
 
 'WT_SESSION.checkpoint' : Method([
     Config('drop', '', r'''
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index ddd8c152f5f..d44307b3485 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-4.2",
-    "commit": "58afeb82a68fde5a3c3eb47cd6e652edb0f62b3a"
+    "commit": "eb1b72848c33e5c19a31a316ba6831c76c3a0241"
 }
diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c
index 548ad69061c..d6cae821ce1 100644
--- a/src/third_party/wiredtiger/src/config/config_def.c
+++ b/src/third_party/wiredtiger/src/config/config_def.c
@@ -208,6 +208,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_checkpoint[] = {
 static const WT_CONFIG_CHECK confchk_WT_SESSION_commit_transaction[] = {
   {"commit_timestamp", "string", NULL, NULL, NULL, 0},
   {"durable_timestamp", "string", NULL, NULL, NULL, 0},
+  {"operation_timeout_ms", "int", NULL, "min=1", NULL, 0},
   {"sync", "string", NULL, "choices=[\"background\",\"off\",\"on\"]", NULL, 0},
   {NULL, NULL, NULL, NULL, NULL, 0}};
 
@@ -338,6 +339,9 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_reconfigure[] = {
     NULL, 0},
   {NULL, NULL, NULL, NULL, NULL, 0}};
 
+static const WT_CONFIG_CHECK confchk_WT_SESSION_rollback_transaction[] = {
+  {"operation_timeout_ms", "int", NULL, "min=1", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}};
+
 static const WT_CONFIG_CHECK confchk_WT_SESSION_salvage[] = {
   {"force", "boolean", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}};
 
@@ -869,7 +873,9 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator",
     confchk_WT_SESSION_checkpoint, 5},
   {"WT_SESSION.close", "", NULL, 0},
   {"WT_SESSION.commit_transaction",
-    "commit_timestamp=,durable_timestamp=,sync=", confchk_WT_SESSION_commit_transaction, 3},
+    "commit_timestamp=,durable_timestamp=,operation_timeout_ms=0,"
+    "sync=",
+    confchk_WT_SESSION_commit_transaction, 4},
   {"WT_SESSION.compact", "timeout=1200", confchk_WT_SESSION_compact, 1},
   {"WT_SESSION.create",
     "access_pattern_hint=none,allocation_size=4KB,app_metadata=,"
@@ -921,7 +927,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator",
     "isolation=read-committed",
     confchk_WT_SESSION_reconfigure, 3},
   {"WT_SESSION.rename", "", NULL, 0}, {"WT_SESSION.reset", "", NULL, 0},
-  {"WT_SESSION.rollback_transaction", "", NULL, 0},
+  {"WT_SESSION.rollback_transaction", "operation_timeout_ms=0",
+    confchk_WT_SESSION_rollback_transaction, 1},
   {"WT_SESSION.salvage", "force=false", confchk_WT_SESSION_salvage, 1},
   {"WT_SESSION.snapshot", "drop=(all=false,before=,names=,to=),include_updates=false,name=",
     confchk_WT_SESSION_snapshot, 3},
diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c
index b3f0a40e176..40d432322b5 100644
--- a/src/third_party/wiredtiger/src/evict/evict_lru.c
+++ b/src/third_party/wiredtiger/src/evict/evict_lru.c
@@ -2307,6 +2307,16 @@ __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, bool readonly, d
         }
 
         /*
+         * Check if we've exceeded our operation timeout, this would also get called from the
+         * previous txn is blocking call, however it won't pickup transactions that have been
+         * committed or rolled back as their mod count is 0, and that txn needs to be the oldest.
+         *
+         * Additionally we don't return rollback which could confuse the caller.
+         */
+        if (__wt_op_timer_fired(session))
+            break;
+
+        /*
          * Check if we have become busy.
          *
          * If we're busy (because of the transaction check we just did or because our caller is
diff --git a/src/third_party/wiredtiger/src/include/txn.i b/src/third_party/wiredtiger/src/include/txn.i
index aaf6ca7dc71..0e40382a158 100644
--- a/src/third_party/wiredtiger/src/include/txn.i
+++ b/src/third_party/wiredtiger/src/include/txn.i
@@ -806,8 +806,7 @@ __wt_txn_begin(WT_SESSION_IMPL *session, const char *cfg[])
     txn->isolation = session->isolation;
     txn->txn_logsync = S2C(session)->txn_logsync;
 
-    if (cfg != NULL)
-        WT_RET(__wt_txn_config(session, cfg));
+    WT_RET(__wt_txn_config(session, cfg));
 
     /*
      * Allocate a snapshot if required. Named snapshot transactions already have an ID setup.
diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in
index 6309776842b..24633fe7ab0 100644
--- a/src/third_party/wiredtiger/src/include/wiredtiger.in
+++ b/src/third_party/wiredtiger/src/include/wiredtiger.in
@@ -1751,12 +1751,12 @@ struct __wt_session {
 	 * \c "read-committed"\, \c "snapshot"; default empty.}
 	 * @config{name, name of the transaction for tracing and debugging., a string; default
 	 * empty.}
-	 * @config{operation_timeout_ms, when non-zero\, a requested limit on the number of elapsed
-	 * real time milliseconds taken to complete database operations in this transaction.  Time
-	 * is measured from the start of each WiredTiger API call.  There is no guarantee any
-	 * operation will not take longer than this amount of time.  If WiredTiger notices the limit
-	 * has been exceeded\, an operation may return a WT_ROLLBACK error.  Default is to have no
-	 * limit., an integer greater than or equal to 1; default \c 0.}
+	 * @config{operation_timeout_ms, when non-zero\, a requested limit on the time taken to
+	 * complete operations in this transaction.  Time is measured in real time milliseconds from
+	 * the start of each WiredTiger API call.  There is no guarantee any operation will not take
+	 * longer than this amount of time.  If WiredTiger notices the limit has been exceeded\, an
+	 * operation may return a WT_ROLLBACK error.  Default is to have no limit., an integer
+	 * greater than or equal to 1; default \c 0.}
 	 * @config{priority, priority of the transaction for resolving conflicts.  Transactions with
 	 * higher values are less likely to abort., an integer between -100 and 100; default \c 0.}
 	 * @config{read_timestamp, read using the specified timestamp.  The supplied value must not
@@ -1807,6 +1807,12 @@ struct __wt_session {
 	 * supplied value must not be older than the commit timestamp set for the current
 	 * transaction.  The value must also not be older than the current stable timestamp.  See
 	 * @ref transaction_timestamps., a string; default empty.}
+	 * @config{operation_timeout_ms, when non-zero\, a requested limit on the time taken to
+	 * complete operations in this transaction.  Time is measured in real time milliseconds from
+	 * the start of each WiredTiger API call.  There is no guarantee any operation will not take
+	 * longer than this amount of time.  If WiredTiger notices the limit has been exceeded\, an
+	 * operation may return a WT_ROLLBACK error.  Default is to have no limit., an integer
+	 * greater than or equal to 1; default \c 0.}
 	 * @config{sync, override whether to sync log records when the transaction commits\,
 	 * inherited from ::wiredtiger_open \c transaction_sync.  The \c background setting
 	 * initiates a background synchronization intended to be used with a later call to
@@ -1856,7 +1862,14 @@ struct __wt_session {
 	 * @snippet ex_all.c transaction commit/rollback
 	 *
 	 * @param session the session handle
-	 * @configempty{WT_SESSION.rollback_transaction, see dist/api_data.py}
+	 * @configstart{WT_SESSION.rollback_transaction, see dist/api_data.py}
+	 * @config{operation_timeout_ms, when non-zero\, a requested limit on the time taken to
+	 * complete operations in this transaction.  Time is measured in real time milliseconds from
+	 * the start of each WiredTiger API call.  There is no guarantee any operation will not take
+	 * longer than this amount of time.  If WiredTiger notices the limit has been exceeded\, an
+	 * operation may return a WT_ROLLBACK error.  Default is to have no limit., an integer
+	 * greater than or equal to 1; default \c 0.}
+	 * @configend
 	 * @errors
 	 */
 	int __F(rollback_transaction)(WT_SESSION *session, const char *config);
diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c
index 4cebdfc9a87..119bbc13987 100644
--- a/src/third_party/wiredtiger/src/txn/txn.c
+++ b/src/third_party/wiredtiger/src/txn/txn.c
@@ -446,6 +446,40 @@ done:
 }
 
 /*
+ * __txn_config_operation_timeout --
+ *     Configure a transactions operation timeout duration.
+ */
+static int
+__txn_config_operation_timeout(WT_SESSION_IMPL *session, const char *cfg[], bool start_timer)
+{
+    WT_CONFIG_ITEM cval;
+    WT_TXN *txn;
+
+    txn = &session->txn;
+
+    if (cfg == NULL)
+        return (0);
+
+    /* Retrieve the maximum operation time, defaulting to the database-wide configuration. */
+    WT_RET(__wt_config_gets(session, cfg, "operation_timeout_ms", &cval));
+
+    /*
+     * The default configuration value is 0, we can't tell if they're setting it back to 0 or, if
+     * the default was automatically passed in.
+     */
+    if (cval.val != 0) {
+        txn->operation_timeout_us = (uint64_t)(cval.val * WT_THOUSAND);
+        /*
+         * The op timer will generally be started on entry to the API call however when we configure
+         * it internally we need to start it separately.
+         */
+        if (start_timer)
+            __wt_op_timer_start(session);
+    }
+    return (0);
+}
+
+/*
  * __wt_txn_config --
  *     Configure a transaction.
  */
@@ -458,6 +492,9 @@ __wt_txn_config(WT_SESSION_IMPL *session, const char *cfg[])
 
     txn = &session->txn;
 
+    if (cfg == NULL)
+        return (0);
+
     WT_RET(__wt_config_gets_def(session, cfg, "isolation", 0, &cval));
     if (cval.len != 0)
         txn->isolation = WT_STRING_MATCH("snapshot", cval.str, cval.len) ?
@@ -465,9 +502,7 @@ __wt_txn_config(WT_SESSION_IMPL *session, const char *cfg[])
           WT_STRING_MATCH("read-committed", cval.str, cval.len) ? WT_ISO_READ_COMMITTED :
                                                                   WT_ISO_READ_UNCOMMITTED;
 
-    /* Retrieve the maximum operation time, defaulting to the database-wide configuration. */
-    WT_RET(__wt_config_gets(session, cfg, "operation_timeout_ms", &cval));
-    txn->operation_timeout_us = (uint64_t)(cval.val * WT_THOUSAND);
+    WT_RET(__txn_config_operation_timeout(session, cfg, false));
 
     /*
      * The default sync setting is inherited from the connection, but can be overridden by an
@@ -928,6 +963,9 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[])
     WT_ASSERT(session, F_ISSET(txn, WT_TXN_RUNNING));
     WT_ASSERT(session, !F_ISSET(txn, WT_TXN_ERROR) || txn->mod_count == 0);
 
+    /* Configure the timeout for this commit operation. */
+    WT_ERR(__txn_config_operation_timeout(session, cfg, true));
+
     /*
      * Clear the prepared round up flag if the transaction is not prepared. There is no rounding up
      * to do in that case.
@@ -1305,8 +1343,6 @@ __wt_txn_rollback(WT_SESSION_IMPL *session, const char *cfg[])
     u_int i;
     bool prepare, readonly;
 
-    WT_UNUSED(cfg);
-
     cursor = NULL;
     txn = &session->txn;
     prepare = F_ISSET(txn, WT_TXN_PREPARE);
@@ -1318,6 +1354,9 @@ __wt_txn_rollback(WT_SESSION_IMPL *session, const char *cfg[])
     if (txn->notify != NULL)
         WT_TRET(txn->notify->notify(txn->notify, (WT_SESSION *)session, txn->id, 0));
 
+    /* Configure the timeout for this rollback operation. */
+    WT_RET(__txn_config_operation_timeout(session, cfg, true));
+
     /*
      * Resolving prepared updates is expensive. Sort prepared modifications so all updates for each
      * page within each file are done at the same time.
diff --git a/src/third_party/wiredtiger/test/format/backup.c b/src/third_party/wiredtiger/test/format/backup.c
index 56f48c6888d..9454032160c 100644
--- a/src/third_party/wiredtiger/test/format/backup.c
+++ b/src/third_party/wiredtiger/test/format/backup.c
@@ -242,54 +242,57 @@ static void
 copy_blocks(WT_SESSION *session, WT_CURSOR *bkup_c, const char *name)
 {
     WT_CURSOR *incr_cur;
+    WT_DECL_RET;
     size_t len, tmp_sz;
     ssize_t rdsize;
-    uint64_t offset, type;
-    u_int size;
-    int ret, rfd, wfd1, wfd2;
-    char buf[512], config[512], *first, *second, *tmp;
+    uint64_t offset, size, type;
+    int rfd, wfd1, wfd2;
+    char config[512], *tmp;
     bool first_pass;
 
-    /*
-     * We need to prepend the home directory name here because we are not using the WiredTiger
-     * internal functions that would prepend it for us.
-     */
-    len = strlen(g.home) + strlen("BACKUP") + strlen(name) + 10;
-    first = dmalloc(len);
-
-    /*
-     * Save another copy of the original file to make debugging recovery errors easier.
-     */
-    len = strlen(g.home) + strlen("BACKUP.copy") + strlen(name) + 10;
-    second = dmalloc(len);
-    testutil_check(__wt_snprintf(config, sizeof(config), "incremental=(file=%s)", name));
-
-    /* Open the duplicate incremental backup cursor with the file name given. */
     tmp_sz = 0;
     tmp = NULL;
     first_pass = true;
     rfd = wfd1 = wfd2 = -1;
+
+    /* Open the duplicate incremental backup cursor with the file name given. */
+    testutil_check(__wt_snprintf(config, sizeof(config), "incremental=(file=%s)", name));
     testutil_check(session->open_cursor(session, NULL, bkup_c, config, &incr_cur));
     while ((ret = incr_cur->next(incr_cur)) == 0) {
-        testutil_check(incr_cur->get_key(incr_cur, &offset, (uint64_t *)&size, &type));
+        testutil_check(incr_cur->get_key(incr_cur, &offset, &size, &type));
         if (type == WT_BACKUP_RANGE) {
             /*
              * Since we are using system calls below instead of a WiredTiger function, we have to
              * prepend the home directory to the file names ourselves.
              */
-            testutil_check(__wt_snprintf(first, len, "%s/BACKUP/%s", g.home, name));
-            testutil_check(__wt_snprintf(second, len, "%s/BACKUP.copy/%s", g.home, name));
+            if (first_pass) {
+                len = strlen(g.home) + strlen(name) + 10;
+                tmp = dmalloc(len);
+                testutil_check(__wt_snprintf(tmp, len, "%s/%s", g.home, name));
+                error_sys_check(rfd = open(tmp, O_RDONLY, 0));
+                free(tmp);
+                tmp = NULL;
+
+                len = strlen(g.home) + strlen("BACKUP") + strlen(name) + 10;
+                tmp = dmalloc(len);
+                testutil_check(__wt_snprintf(tmp, len, "%s/BACKUP/%s", g.home, name));
+                error_sys_check(wfd1 = open(tmp, O_WRONLY | O_CREAT, 0));
+                free(tmp);
+                tmp = NULL;
+
+                len = strlen(g.home) + strlen("BACKUP.copy") + strlen(name) + 10;
+                tmp = dmalloc(len);
+                testutil_check(__wt_snprintf(tmp, len, "%s/BACKUP.copy/%s", g.home, name));
+                error_sys_check(wfd2 = open(tmp, O_WRONLY | O_CREAT, 0));
+                free(tmp);
+                tmp = NULL;
+
+                first_pass = false;
+            }
             if (tmp_sz < size) {
                 tmp = drealloc(tmp, size);
                 tmp_sz = size;
             }
-            if (first_pass) {
-                testutil_check(__wt_snprintf(buf, sizeof(buf), "%s/%s", g.home, name));
-                error_sys_check(rfd = open(buf, O_RDONLY, 0));
-                error_sys_check(wfd1 = open(first, O_WRONLY | O_CREAT, 0));
-                error_sys_check(wfd2 = open(second, O_WRONLY | O_CREAT, 0));
-                first_pass = false;
-            }
             error_sys_check(lseek(rfd, (wt_off_t)offset, SEEK_SET));
             error_sys_check(rdsize = read(rfd, tmp, size));
             error_sys_check(lseek(wfd1, (wt_off_t)offset, SEEK_SET));
@@ -298,17 +301,27 @@ copy_blocks(WT_SESSION *session, WT_CURSOR *bkup_c, const char *name)
             error_sys_check(write(wfd1, tmp, (size_t)rdsize));
             error_sys_check(write(wfd2, tmp, (size_t)rdsize));
         } else {
+            testutil_assert(type == WT_BACKUP_FILE);
+            testutil_assert(first_pass == true);
+            testutil_assert(rfd == -1);
+
             /*
              * These operations are using a WiredTiger function so it will prepend the home
              * directory to the name for us.
              */
-            testutil_check(__wt_snprintf(first, len, "BACKUP/%s", name));
-            testutil_check(__wt_snprintf(second, len, "BACKUP.copy/%s", name));
-            testutil_assert(type == WT_BACKUP_FILE);
-            testutil_assert(rfd == -1);
-            testutil_assert(first_pass == true);
-            testutil_check(__wt_copy_and_sync(session, name, first));
-            testutil_check(__wt_copy_and_sync(session, first, second));
+            len = strlen("BACKUP") + strlen(name) + 10;
+            tmp = dmalloc(len);
+            testutil_check(__wt_snprintf(tmp, len, "BACKUP/%s", name));
+            testutil_check(__wt_copy_and_sync(session, name, tmp));
+            free(tmp);
+            tmp = NULL;
+
+            len = strlen("BACKUP.copy") + strlen(name) + 10;
+            tmp = dmalloc(len);
+            testutil_check(__wt_snprintf(tmp, len, "BACKUP.copy/%s", name));
+            testutil_check(__wt_copy_and_sync(session, name, tmp));
+            free(tmp);
+            tmp = NULL;
         }
     }
     testutil_check(incr_cur->close(incr_cur));
@@ -317,8 +330,6 @@ copy_blocks(WT_SESSION *session, WT_CURSOR *bkup_c, const char *name)
         error_sys_check(close(wfd1));
         error_sys_check(close(wfd2));
     }
-    free(first);
-    free(second);
     free(tmp);
 }
 /*
diff --git a/src/third_party/wiredtiger/test/suite/test_txn21.py b/src/third_party/wiredtiger/test/suite/test_txn21.py
index 5abf907ac84..06db7ce76aa 100644
--- a/src/third_party/wiredtiger/test/suite/test_txn21.py
+++ b/src/third_party/wiredtiger/test/suite/test_txn21.py
@@ -43,7 +43,17 @@ class test_txn21(wttest.WiredTigerTestCase):
 
     # Transaction-level configuration.
     def test_operation_timeout_txn(self):
+        # Test during begin.
         self.session.begin_transaction('operation_timeout_ms=2000')
+        self.session.rollback_transaction()
+
+        # Test during rollback.
+        self.session.begin_transaction()
+        self.session.rollback_transaction('operation_timeout_ms=2000')
+
+        # Test during commit.
+        self.session.begin_transaction()
+        self.session.commit_transaction('operation_timeout_ms=2000')
 
 if __name__ == '__main__':
     wttest.run()