SERVER-67787 Retry ops in auth concurrency tests

(cherry picked from commit 51b7a812dd006d2df91c649a3d71b28770a5a69c)

5 files changed, 150 insertions, 42 deletions

diff --git a/jstests/concurrency/fsm_workloads/auth_create_role.js b/jstests/concurrency/fsm_workloads/auth_create_role.js
index 1cc041b876a..05ec4a9fb32 100644
--- a/jstests/concurrency/fsm_workloads/auth_create_role.js
+++ b/jstests/concurrency/fsm_workloads/auth_create_role.js
@@ -7,6 +7,9 @@
  */
 load('jstests/concurrency/fsm_workload_helpers/drop_utils.js');  // for dropRoles
 
+// UMC commands are not supported in transactions.
+TestData.runInsideTransaction = false;
+
 var $config = (function() {
     var data = {
         // Use the workload name as a prefix for the role name,
@@ -24,13 +27,29 @@ var $config = (function() {
         }
 
         function createRole(db, collName) {
-            var roleName = uniqueRoleName(this.prefix, this.tid, this.num++);
-            db.createRole({
-                role: roleName,
-                privileges:
-                    [{resource: {db: db.getName(), collection: collName}, actions: ['update']}],
-                roles: [{role: 'read', db: db.getName()}]
-            });
+            const roleName = uniqueRoleName(this.prefix, this.tid, this.num++);
+            const kCreateRoleRetries = 5;
+            const kCreateRoleRetryInterval = 5 * 1000;
+            assert.retry(
+                function() {
+                    try {
+                        db.createRole({
+                            role: roleName,
+                            privileges: [{
+                                resource: {db: db.getName(), collection: collName},
+                                actions: ['update']
+                            }],
+                            roles: [{role: 'read', db: db.getName()}]
+                        });
+                        return true;
+                    } catch (e) {
+                        jsTest.log("Caught createRole exception: " + tojson(e));
+                        return false;
+                    }
+                },
+                "Failed creating role '" + roleName + "'",
+                kCreateRoleRetries,
+                kCreateRoleRetryInterval);
 
             // Verify the newly created role exists, as well as all previously created roles
             for (var i = 0; i < this.num; ++i) {
diff --git a/jstests/concurrency/fsm_workloads/auth_create_user.js b/jstests/concurrency/fsm_workloads/auth_create_user.js
index 2a703c78833..26c327dd0cd 100644
--- a/jstests/concurrency/fsm_workloads/auth_create_user.js
+++ b/jstests/concurrency/fsm_workloads/auth_create_user.js
@@ -7,6 +7,9 @@
  */
 load('jstests/concurrency/fsm_workload_helpers/drop_utils.js');  // for dropUsers
 
+// UMC commands are not supported in transactions.
+TestData.runInsideTransaction = false;
+
 var $config = (function() {
     var data = {
         // Use the workload name as a prefix for the username,
@@ -24,8 +27,23 @@ var $config = (function() {
         }
 
         function createUser(db, collName) {
-            var username = uniqueUsername(this.prefix, this.tid, this.num++);
-            db.createUser({user: username, pwd: 'password', roles: ['readWrite', 'dbAdmin']});
+            const username = uniqueUsername(this.prefix, this.tid, this.num++);
+            const kCreateUserRetries = 5;
+            const kCreateUserRetryInterval = 5 * 1000;
+            assert.retry(
+                function() {
+                    try {
+                        db.createUser(
+                            {user: username, pwd: 'password', roles: ['readWrite', 'dbAdmin']});
+                        return true;
+                    } catch (e) {
+                        jsTest.log("Caught createUser exception: " + tojson(e));
+                        return false;
+                    }
+                },
+                "Failed creating user: '" + username + "'",
+                kCreateUserRetries,
+                kCreateUserRetryInterval);
 
             // Verify the newly created user exists, as well as all previously created users
             for (var i = 0; i < this.num; ++i) {
diff --git a/jstests/concurrency/fsm_workloads/auth_drop_role.js b/jstests/concurrency/fsm_workloads/auth_drop_role.js
index 5cff820b3e9..c8c322ce0ce 100644
--- a/jstests/concurrency/fsm_workloads/auth_drop_role.js
+++ b/jstests/concurrency/fsm_workloads/auth_drop_role.js
@@ -8,6 +8,9 @@
  */
 load('jstests/concurrency/fsm_workload_helpers/drop_utils.js');  // for dropRoles
 
+// UMC commands are not supported in transactions.
+TestData.runInsideTransaction = false;
+
 var $config = (function() {
     const kMaxCmdTimeMs = 60000;
     const kMaxTxnLockReqTimeMs = 100;
@@ -31,13 +34,29 @@ var $config = (function() {
         function createAndDropRole(db, collName) {
             var roleName = uniqueRoleName(this.prefix, this.tid, this.num++);
 
-            db.runCommand({
-                createRole: roleName,
-                privileges:
-                    [{resource: {db: db.getName(), collection: collName}, actions: ['remove']}],
-                roles: [{role: 'read', db: db.getName()}],
-                maxTimeMS: kMaxCmdTimeMs
-            });
+            const kCreateRoleRetries = 5;
+            const kCreateRoleRetryInterval = 5 * 1000;
+            assert.retry(
+                function() {
+                    try {
+                        assert.commandWorked(db.runCommand({
+                            createRole: roleName,
+                            privileges: [{
+                                resource: {db: db.getName(), collection: collName},
+                                actions: ['remove']
+                            }],
+                            roles: [{role: 'read', db: db.getName()}],
+                            maxTimeMS: kMaxCmdTimeMs
+                        }));
+                        return true;
+                    } catch (e) {
+                        jsTest.log("Caught createRole exception: " + tojson(e));
+                        return false;
+                    }
+                },
+                "Failed creating role '" + roleName + "'",
+                kCreateRoleRetries,
+                kCreateRoleRetryInterval);
 
             var res = db.getRole(roleName);
 
@@ -47,20 +66,24 @@ var $config = (function() {
 
             // Some test machines may hit high contention during these concurrency tests
             // allow for occaisional failure with retries.
-            for (var i = 3; i >= 0; --i) {
-                let dropResult = db.runCommand({dropRole: roleName, maxTimeMS: kMaxCmdTimeMs});
-
-                if (dropResult === true) {
-                    // Success
-                    break;
-                } else if (i > 0) {
-                    // Failure, try again
-                    print("Retrying a dropRole() which resulted in: " + tojson(dropResult));
-                } else {
-                    // Out of do-overs, just die.
-                    assertAlways(dropResult);
+            const kDropRoleRetries = 5;
+            const kDropRoleSnapshotUnavailableIntervalMS = 5 * 1000;
+            const kDropRoleRetryInterval = 0;
+            assert.retry(function() {
+                let cmdResult;
+                try {
+                    cmdResult = db.runCommand({dropRole: roleName, maxTimeMS: kMaxCmdTimeMs});
+                    assert.commandWorked(cmdResult);
+                    return true;
+                } catch (e) {
+                    jsTest.log("Caught dropRole exception: " + tojson(e));
+                    if (cmdResult.code == ErrorCodes.SnapshotUnavailable) {
+                        // Give pending catalog changes a chance to catch up.
+                        sleep(kDropRoleSnapshotUnavailableIntervalMS);
+                    }
+                    return false;
                 }
-            }
+            }, "Failed dropping role '" + roleName + "'", kDropRoleRetries, kDropRoleRetryInterval);
 
             assertAlways.isnull(db.getRole(roleName), "role '" + roleName + "' should not exist");
         }
diff --git a/jstests/concurrency/fsm_workloads/auth_drop_user.js b/jstests/concurrency/fsm_workloads/auth_drop_user.js
index e8b9c1db4de..ee14ffa4221 100644
--- a/jstests/concurrency/fsm_workloads/auth_drop_user.js
+++ b/jstests/concurrency/fsm_workloads/auth_drop_user.js
@@ -1,5 +1,8 @@
 'use strict';
 
+// UMC commands are not supported in transactions.
+TestData.runInsideTransaction = false;
+
 /**
  * auth_drop_user.js
  *
@@ -23,15 +26,41 @@ var $config = (function() {
         }
 
         function createAndDropUser(db, collName) {
-            var username = uniqueUsername(this.prefix, this.tid, this.num++);
-            db.createUser({user: username, pwd: 'password', roles: ['readWrite', 'dbAdmin']});
+            const username = uniqueUsername(this.prefix, this.tid, this.num++);
+
+            const kCreateUserRetries = 5;
+            const kCreateUserRetryInterval = 5 * 1000;
+            assert.retry(
+                function() {
+                    try {
+                        db.createUser(
+                            {user: username, pwd: 'password', roles: ['readWrite', 'dbAdmin']});
+                        return true;
+                    } catch (e) {
+                        jsTest.log("Caught createUser exception: " + tojson(e));
+                        return false;
+                    }
+                },
+                "Failed creating user '" + username + "'",
+                kCreateUserRetries,
+                kCreateUserRetryInterval);
 
-            var res = db.getUser(username);
+            const res = db.getUser(username);
             assertAlways(res !== null, "user '" + username + "' should exist");
             assertAlways.eq(username, res.user);
             assertAlways.eq(db.getName(), res.db);
 
-            assertAlways(db.dropUser(username));
+            const kDropUserRetries = 5;
+            const kDropUserRetryInterval = 5 * 1000;
+            assert.retry(function() {
+                try {
+                    db.dropUser(username);
+                    return true;
+                } catch (e) {
+                    jsTest.log("Caught dropUser exception: " + tojson(e));
+                    return false;
+                }
+            }, "Failed dropping user '" + username + "'", kDropUserRetries, kDropUserRetryInterval);
             assertAlways.isnull(db.getUser(username), "user '" + username + "' should not exist");
         }
 
diff --git a/jstests/concurrency/fsm_workloads/auth_privilege_consistency.js b/jstests/concurrency/fsm_workloads/auth_privilege_consistency.js
index f3494f835c3..79cfc2cc29d 100644
--- a/jstests/concurrency/fsm_workloads/auth_privilege_consistency.js
+++ b/jstests/concurrency/fsm_workloads/auth_privilege_consistency.js
@@ -10,12 +10,31 @@
  */
 load('jstests/concurrency/fsm_workload_helpers/drop_utils.js');  // for dropRoles
 
+// UMC commands are not supported in transactions.
+TestData.runInsideTransaction = false;
+
 var $config = (function() {
     const kTestUserPassword = 'secret';
     const kMaxCmdTimeMs = 60000;
     const kMaxTxnLockReqTimeMs = 100;
     const kDefaultTxnLockReqTimeMs = 5;
 
+    function doRetry(cb) {
+        const kNumRetries = 5;
+        const kRetryInterval = 5 * 1000;
+
+        assert.retry(function() {
+            try {
+                cb();
+                return true;
+            } catch (e) {
+                jsTest.log("Caught exception performing: " + tojson(cb) +
+                           ", exception was: " + tojson(e));
+                return false;
+            }
+        }, "Failed performing: " + tojson(cb), kNumRetries, kRetryInterval);
+    }
+
     const states = (function() {
         let roleWithDB = {};
         let privilege = {actions: ['insert', 'update', 'remove', 'find']};
@@ -38,7 +57,7 @@ var $config = (function() {
                 privilege.resource = {db: db.getName(), collection: ''};
                 const roleName = this.getRoleName(this.tid);
                 roleWithDB = {role: roleName, db: db.getName()};
-                db.createRole({role: roleName, privileges: [privilege], roles: []});
+                doRetry(() => db.createRole({role: roleName, privileges: [privilege], roles: []}));
             },
 
             mutate: function(db, collName) {
@@ -46,31 +65,31 @@ var $config = (function() {
                 // then give that, now empty, role to the user.
                 const roleName = this.getRoleName(this.tid);
 
-                db.runCommand({
+                doRetry(() => assert.commandWorked(db.runCommand({
                     revokePrivilegesFromRole: roleName,
                     privileges: [privilege],
                     maxTimeMS: kMaxCmdTimeMs
-                });
+                })));
 
-                db.runCommand({
+                doRetry(() => assert.commandWorked(db.runCommand({
                     grantRolesToUser: this.getUserName(),
                     roles: [roleWithDB],
                     maxTimeMS: kMaxCmdTimeMs
-                });
+                })));
 
                 // Take the role away from the user, and give it privs.
 
-                db.runCommand({
+                doRetry(() => assert.commandWorked(db.runCommand({
                     revokeRolesFromUser: this.getUserName(),
                     roles: [roleWithDB],
                     maxTimeMS: kMaxCmdTimeMs
-                });
+                })));
 
-                db.runCommand({
+                doRetry(() => assert.commandWorked(db.runCommand({
                     grantPrivilegesToRole: roleName,
                     privileges: [privilege],
                     maxTimeMS: kMaxCmdTimeMs
-                });
+                })));
             },
 
             observeInit: function(db, collName) {