diff options
author | Sara Golemon <sara.golemon@mongodb.com> | 2022-07-05 11:31:02 -0500 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-07-14 16:51:20 +0000 |
commit | 51b7a812dd006d2df91c649a3d71b28770a5a69c (patch) | |
tree | 5d273a4c08a7ecea5ec914045008d56b2af84ff9 | |
parent | 5a595559bf4e2a764c6afe7314409dd75383cd42 (diff) | |
download | mongo-51b7a812dd006d2df91c649a3d71b28770a5a69c.tar.gz |
SERVER-67787 Retry ops in auth concurrency tests
5 files changed, 144 insertions, 43 deletions
diff --git a/jstests/concurrency/fsm_workloads/auth_create_role.js b/jstests/concurrency/fsm_workloads/auth_create_role.js index 1cc041b876a..05ec4a9fb32 100644 --- a/jstests/concurrency/fsm_workloads/auth_create_role.js +++ b/jstests/concurrency/fsm_workloads/auth_create_role.js @@ -7,6 +7,9 @@ */ load('jstests/concurrency/fsm_workload_helpers/drop_utils.js'); // for dropRoles +// UMC commands are not supported in transactions. +TestData.runInsideTransaction = false; + var $config = (function() { var data = { // Use the workload name as a prefix for the role name, @@ -24,13 +27,29 @@ var $config = (function() { } function createRole(db, collName) { - var roleName = uniqueRoleName(this.prefix, this.tid, this.num++); - db.createRole({ - role: roleName, - privileges: - [{resource: {db: db.getName(), collection: collName}, actions: ['update']}], - roles: [{role: 'read', db: db.getName()}] - }); + const roleName = uniqueRoleName(this.prefix, this.tid, this.num++); + const kCreateRoleRetries = 5; + const kCreateRoleRetryInterval = 5 * 1000; + assert.retry( + function() { + try { + db.createRole({ + role: roleName, + privileges: [{ + resource: {db: db.getName(), collection: collName}, + actions: ['update'] + }], + roles: [{role: 'read', db: db.getName()}] + }); + return true; + } catch (e) { + jsTest.log("Caught createRole exception: " + tojson(e)); + return false; + } + }, + "Failed creating role '" + roleName + "'", + kCreateRoleRetries, + kCreateRoleRetryInterval); // Verify the newly created role exists, as well as all previously created roles for (var i = 0; i < this.num; ++i) { diff --git a/jstests/concurrency/fsm_workloads/auth_create_user.js b/jstests/concurrency/fsm_workloads/auth_create_user.js index 2a703c78833..26c327dd0cd 100644 --- a/jstests/concurrency/fsm_workloads/auth_create_user.js +++ b/jstests/concurrency/fsm_workloads/auth_create_user.js @@ -7,6 +7,9 @@ */ load('jstests/concurrency/fsm_workload_helpers/drop_utils.js'); // for dropUsers +// UMC commands are not supported in transactions. +TestData.runInsideTransaction = false; + var $config = (function() { var data = { // Use the workload name as a prefix for the username, @@ -24,8 +27,23 @@ var $config = (function() { } function createUser(db, collName) { - var username = uniqueUsername(this.prefix, this.tid, this.num++); - db.createUser({user: username, pwd: 'password', roles: ['readWrite', 'dbAdmin']}); + const username = uniqueUsername(this.prefix, this.tid, this.num++); + const kCreateUserRetries = 5; + const kCreateUserRetryInterval = 5 * 1000; + assert.retry( + function() { + try { + db.createUser( + {user: username, pwd: 'password', roles: ['readWrite', 'dbAdmin']}); + return true; + } catch (e) { + jsTest.log("Caught createUser exception: " + tojson(e)); + return false; + } + }, + "Failed creating user: '" + username + "'", + kCreateUserRetries, + kCreateUserRetryInterval); // Verify the newly created user exists, as well as all previously created users for (var i = 0; i < this.num; ++i) { diff --git a/jstests/concurrency/fsm_workloads/auth_drop_role.js b/jstests/concurrency/fsm_workloads/auth_drop_role.js index 275e56906d6..c8c322ce0ce 100644 --- a/jstests/concurrency/fsm_workloads/auth_drop_role.js +++ b/jstests/concurrency/fsm_workloads/auth_drop_role.js @@ -8,6 +8,9 @@ */ load('jstests/concurrency/fsm_workload_helpers/drop_utils.js'); // for dropRoles +// UMC commands are not supported in transactions. +TestData.runInsideTransaction = false; + var $config = (function() { const kMaxCmdTimeMs = 60000; const kMaxTxnLockReqTimeMs = 100; @@ -31,13 +34,29 @@ var $config = (function() { function createAndDropRole(db, collName) { var roleName = uniqueRoleName(this.prefix, this.tid, this.num++); - db.runCommand({ - createRole: roleName, - privileges: - [{resource: {db: db.getName(), collection: collName}, actions: ['remove']}], - roles: [{role: 'read', db: db.getName()}], - maxTimeMS: kMaxCmdTimeMs - }); + const kCreateRoleRetries = 5; + const kCreateRoleRetryInterval = 5 * 1000; + assert.retry( + function() { + try { + assert.commandWorked(db.runCommand({ + createRole: roleName, + privileges: [{ + resource: {db: db.getName(), collection: collName}, + actions: ['remove'] + }], + roles: [{role: 'read', db: db.getName()}], + maxTimeMS: kMaxCmdTimeMs + })); + return true; + } catch (e) { + jsTest.log("Caught createRole exception: " + tojson(e)); + return false; + } + }, + "Failed creating role '" + roleName + "'", + kCreateRoleRetries, + kCreateRoleRetryInterval); var res = db.getRole(roleName); @@ -47,27 +66,24 @@ var $config = (function() { // Some test machines may hit high contention during these concurrency tests // allow for occaisional failure with retries. - for (var i = 5; i >= 0; --i) { + const kDropRoleRetries = 5; + const kDropRoleSnapshotUnavailableIntervalMS = 5 * 1000; + const kDropRoleRetryInterval = 0; + assert.retry(function() { let cmdResult; try { cmdResult = db.runCommand({dropRole: roleName, maxTimeMS: kMaxCmdTimeMs}); assert.commandWorked(cmdResult); - break; + return true; } catch (e) { - if (i > 0) { - // Failure, try again - print("Retrying dropRole(" + roleName + "), previous call resulted in " + - tojson(cmdResult)); - if (cmdResult.code == ErrorCodes.SnapshotUnavailable) { - // Give pending catalog changes a chance to catch up. - sleep(5000); - } - } else { - // Out of do-overs, just die. - throw e; + jsTest.log("Caught dropRole exception: " + tojson(e)); + if (cmdResult.code == ErrorCodes.SnapshotUnavailable) { + // Give pending catalog changes a chance to catch up. + sleep(kDropRoleSnapshotUnavailableIntervalMS); } + return false; } - } + }, "Failed dropping role '" + roleName + "'", kDropRoleRetries, kDropRoleRetryInterval); assertAlways.isnull(db.getRole(roleName), "role '" + roleName + "' should not exist"); } diff --git a/jstests/concurrency/fsm_workloads/auth_drop_user.js b/jstests/concurrency/fsm_workloads/auth_drop_user.js index e8b9c1db4de..ee14ffa4221 100644 --- a/jstests/concurrency/fsm_workloads/auth_drop_user.js +++ b/jstests/concurrency/fsm_workloads/auth_drop_user.js @@ -1,5 +1,8 @@ 'use strict'; +// UMC commands are not supported in transactions. +TestData.runInsideTransaction = false; + /** * auth_drop_user.js * @@ -23,15 +26,41 @@ var $config = (function() { } function createAndDropUser(db, collName) { - var username = uniqueUsername(this.prefix, this.tid, this.num++); - db.createUser({user: username, pwd: 'password', roles: ['readWrite', 'dbAdmin']}); + const username = uniqueUsername(this.prefix, this.tid, this.num++); + + const kCreateUserRetries = 5; + const kCreateUserRetryInterval = 5 * 1000; + assert.retry( + function() { + try { + db.createUser( + {user: username, pwd: 'password', roles: ['readWrite', 'dbAdmin']}); + return true; + } catch (e) { + jsTest.log("Caught createUser exception: " + tojson(e)); + return false; + } + }, + "Failed creating user '" + username + "'", + kCreateUserRetries, + kCreateUserRetryInterval); - var res = db.getUser(username); + const res = db.getUser(username); assertAlways(res !== null, "user '" + username + "' should exist"); assertAlways.eq(username, res.user); assertAlways.eq(db.getName(), res.db); - assertAlways(db.dropUser(username)); + const kDropUserRetries = 5; + const kDropUserRetryInterval = 5 * 1000; + assert.retry(function() { + try { + db.dropUser(username); + return true; + } catch (e) { + jsTest.log("Caught dropUser exception: " + tojson(e)); + return false; + } + }, "Failed dropping user '" + username + "'", kDropUserRetries, kDropUserRetryInterval); assertAlways.isnull(db.getUser(username), "user '" + username + "' should not exist"); } diff --git a/jstests/concurrency/fsm_workloads/auth_privilege_consistency.js b/jstests/concurrency/fsm_workloads/auth_privilege_consistency.js index f3494f835c3..79cfc2cc29d 100644 --- a/jstests/concurrency/fsm_workloads/auth_privilege_consistency.js +++ b/jstests/concurrency/fsm_workloads/auth_privilege_consistency.js @@ -10,12 +10,31 @@ */ load('jstests/concurrency/fsm_workload_helpers/drop_utils.js'); // for dropRoles +// UMC commands are not supported in transactions. +TestData.runInsideTransaction = false; + var $config = (function() { const kTestUserPassword = 'secret'; const kMaxCmdTimeMs = 60000; const kMaxTxnLockReqTimeMs = 100; const kDefaultTxnLockReqTimeMs = 5; + function doRetry(cb) { + const kNumRetries = 5; + const kRetryInterval = 5 * 1000; + + assert.retry(function() { + try { + cb(); + return true; + } catch (e) { + jsTest.log("Caught exception performing: " + tojson(cb) + + ", exception was: " + tojson(e)); + return false; + } + }, "Failed performing: " + tojson(cb), kNumRetries, kRetryInterval); + } + const states = (function() { let roleWithDB = {}; let privilege = {actions: ['insert', 'update', 'remove', 'find']}; @@ -38,7 +57,7 @@ var $config = (function() { privilege.resource = {db: db.getName(), collection: ''}; const roleName = this.getRoleName(this.tid); roleWithDB = {role: roleName, db: db.getName()}; - db.createRole({role: roleName, privileges: [privilege], roles: []}); + doRetry(() => db.createRole({role: roleName, privileges: [privilege], roles: []})); }, mutate: function(db, collName) { @@ -46,31 +65,31 @@ var $config = (function() { // then give that, now empty, role to the user. const roleName = this.getRoleName(this.tid); - db.runCommand({ + doRetry(() => assert.commandWorked(db.runCommand({ revokePrivilegesFromRole: roleName, privileges: [privilege], maxTimeMS: kMaxCmdTimeMs - }); + }))); - db.runCommand({ + doRetry(() => assert.commandWorked(db.runCommand({ grantRolesToUser: this.getUserName(), roles: [roleWithDB], maxTimeMS: kMaxCmdTimeMs - }); + }))); // Take the role away from the user, and give it privs. - db.runCommand({ + doRetry(() => assert.commandWorked(db.runCommand({ revokeRolesFromUser: this.getUserName(), roles: [roleWithDB], maxTimeMS: kMaxCmdTimeMs - }); + }))); - db.runCommand({ + doRetry(() => assert.commandWorked(db.runCommand({ grantPrivilegesToRole: roleName, privileges: [privilege], maxTimeMS: kMaxCmdTimeMs - }); + }))); }, observeInit: function(db, collName) { |