SERVER-48594 Add concurrency workload using slow user acquisition

7 files changed, 142 insertions, 0 deletions

diff --git a/jstests/concurrency/fsm_workloads/auth_privilege_cache_miss.js b/jstests/concurrency/fsm_workloads/auth_privilege_cache_miss.js
new file mode 100644
index 00000000000..fe14a4e32d1
--- /dev/null
+++ b/jstests/concurrency/fsm_workloads/auth_privilege_cache_miss.js
@@ -0,0 +1,40 @@
+'use strict';
+
+/**
+ * auth_privilege_cache_miss.js
+ *
+ * Validate user permission consistency during cache miss and slow load.
+ *
+ * @tags: [requires_fcv_47]
+ */
+
+// Use the auth_privilege_consistency workload as a base.
+load('jstests/concurrency/fsm_libs/extend_workload.js');
+load('jstests/concurrency/fsm_workloads/auth_privilege_consistency.js');
+
+var $config = extendWorkload($config, function($config, $super) {
+    // Override setup() to also set cache-miss and slow load failpoints.
+    const kResolveRolesDelayMS = 250;
+
+    const originalSetup = $config.setup;
+    $config.setup = function(db, collName, cluster) {
+        originalSetup(db, collName, cluster);
+
+        const cacheBypass = {configureFailPoint: 'authUserCacheBypass', mode: 'alwaysOn'};
+
+        cluster.executeOnMongosNodes(function(nodeAdminDB) {
+            assert.commandWorked(nodeAdminDB.runCommand(cacheBypass));
+        });
+
+        cluster.executeOnMongodNodes(function(nodeAdminDB) {
+            assert.commandWorked(nodeAdminDB.runCommand(cacheBypass));
+            assert.commandWorked(nodeAdminDB.runCommand({
+                configureFailPoint: 'authLocalGetUser',
+                mode: 'alwaysOn',
+                data: {resolveRolesDelayMS: NumberInt(kResolveRolesDelayMS)}
+            }));
+        });
+    };
+
+    return $config;
+});
diff --git a/src/mongo/db/auth/SConscript b/src/mongo/db/auth/SConscript
index 2bae6734174..41654b1dd87 100644
--- a/src/mongo/db/auth/SConscript
+++ b/src/mongo/db/auth/SConscript
@@ -68,6 +68,7 @@ env.Library(
 env.Library(
     target='auth_impl_internal_local',
     source=[
+        env.Idlc('auth_types.idl')[0],
         'authz_manager_external_state_local.cpp',
     ],
     LIBDEPS=[
diff --git a/src/mongo/db/auth/auth_types.idl b/src/mongo/db/auth/auth_types.idl
index 683688591a0..a653d0152ab 100644
--- a/src/mongo/db/auth/auth_types.idl
+++ b/src/mongo/db/auth/auth_types.idl
@@ -65,3 +65,12 @@ types:
     cpp_type: "Privilege"
     deserializer: "mongo::Privilege::fromBSON"
     serializer: "mongo::Privilege::toBSON"
+
+structs:
+  authLocalGetUserFailPoint:
+    description: Data for authLocalGetUser failpoint
+    fields:
+      resolveRolesDelayMS:
+        type: int
+        default: 0
+        validator: { gte: 0 }
diff --git a/src/mongo/db/auth/authorization_manager_impl.cpp b/src/mongo/db/auth/authorization_manager_impl.cpp
index e853b254ee3..66084c34f87 100644
--- a/src/mongo/db/auth/authorization_manager_impl.cpp
+++ b/src/mongo/db/auth/authorization_manager_impl.cpp
@@ -55,6 +55,7 @@
 #include "mongo/db/mongod_options.h"
 #include "mongo/logv2/log.h"
 #include "mongo/util/assert_util.h"
+#include "mongo/util/fail_point.h"
 #include "mongo/util/net/ssl_peer_info.h"
 #include "mongo/util/net/ssl_types.h"
 #include "mongo/util/str.h"
@@ -410,6 +411,10 @@ Status AuthorizationManagerImpl::getRoleDescriptionsForDB(
         opCtx, dbname, privileges, restrictions, showBuiltinRoles, result);
 }
 
+namespace {
+MONGO_FAIL_POINT_DEFINE(authUserCacheBypass);
+}  // namespace
+
 StatusWith<UserHandle> AuthorizationManagerImpl::acquireUser(OperationContext* opCtx,
                                                              const UserName& userName) try {
     if (userName == internalSecurity.user->getName()) {
@@ -433,6 +438,17 @@ StatusWith<UserHandle> AuthorizationManagerImpl::acquireUser(OperationContext* o
     }
 #endif
 
+    if (authUserCacheBypass.shouldFail()) {
+        // Bypass cache and force a fresh load of the user.
+        auto loadedUser = uassertStatusOK(_externalState->getUserObject(opCtx, request));
+        // We have to inject into the cache in order to get a UserHandle.
+        auto userHandle =
+            _userCache.insertOrAssignAndGet(request, std::move(loadedUser), Date_t::now());
+        invariant(userHandle);
+        LOGV2_DEBUG(4859401, 1, "Bypassing user cache to load user", "user"_attr = userName);
+        return userHandle;
+    }
+
     auto cachedUser = _userCache.acquire(opCtx, request);
     invariant(cachedUser);
 
diff --git a/src/mongo/db/auth/authz_manager_external_state_local.cpp b/src/mongo/db/auth/authz_manager_external_state_local.cpp
index 82bd535d26b..0e5211cd505 100644
--- a/src/mongo/db/auth/authz_manager_external_state_local.cpp
+++ b/src/mongo/db/auth/authz_manager_external_state_local.cpp
@@ -40,11 +40,15 @@
 #include "mongo/bson/util/bson_extract.h"
 #include "mongo/db/auth/address_restriction.h"
 #include "mongo/db/auth/auth_options_gen.h"
+#include "mongo/db/auth/auth_types_gen.h"
 #include "mongo/db/auth/privilege_parser.h"
 #include "mongo/db/auth/user_document_parser.h"
 #include "mongo/db/operation_context.h"
 #include "mongo/db/server_options.h"
+#include "mongo/db/storage/snapshot_manager.h"
 #include "mongo/logv2/log.h"
+#include "mongo/util/duration.h"
+#include "mongo/util/fail_point.h"
 #include "mongo/util/net/ssl_types.h"
 #include "mongo/util/str.h"
 
@@ -192,6 +196,28 @@ ResolveRoleOption makeResolveRoleOption(PrivilegeFormat showPrivileges,
 
     return option;
 }
+
+MONGO_FAIL_POINT_DEFINE(authLocalGetUser);
+void handleAuthLocalGetUserFailPoint(const std::vector<RoleName>& directRoles) {
+    auto sfp = authLocalGetUser.scoped();
+    if (!sfp.isActive()) {
+        return;
+    }
+
+    IDLParserErrorContext ctx("authLocalGetUser");
+    auto delay = AuthLocalGetUserFailPoint::parse(ctx, sfp.getData()).getResolveRolesDelayMS();
+
+    if (delay <= 0) {
+        return;
+    }
+
+    LOGV2_DEBUG(4859400,
+                3,
+                "Sleeping prior to merging direct roles, after user acquisition",
+                "duration"_attr = Milliseconds(delay),
+                "directRoles"_attr = directRoles);
+    sleepmillis(delay);
+}
 }  // namespace
 
 bool AuthzManagerExternalStateLocal::hasAnyPrivilegeDocuments(OperationContext* opCtx) {
@@ -219,12 +245,31 @@ bool AuthzManagerExternalStateLocal::hasAnyPrivilegeDocuments(OperationContext*
     return false;
 }
 
+AuthzManagerExternalStateLocal::RolesLocks::RolesLocks(OperationContext* opCtx) {
+    _adminLock =
+        std::make_unique<Lock::DBLock>(opCtx, NamespaceString::kAdminDb, LockMode::MODE_IS);
+    _rolesLock = std::make_unique<Lock::CollectionLock>(
+        opCtx, AuthorizationManager::rolesCollectionNamespace, LockMode::MODE_S);
+}
+
+AuthzManagerExternalStateLocal::RolesLocks::~RolesLocks() {
+    _rolesLock.reset(nullptr);
+    _adminLock.reset(nullptr);
+}
+
+AuthzManagerExternalStateLocal::RolesLocks AuthzManagerExternalStateLocal::_lockRoles(
+    OperationContext* opCtx) {
+    return AuthzManagerExternalStateLocal::RolesLocks(opCtx);
+}
+
 StatusWith<User> AuthzManagerExternalStateLocal::getUserObject(OperationContext* opCtx,
                                                                const UserRequest& userReq) try {
     const UserName& userName = userReq.name;
     std::vector<RoleName> directRoles;
     User user(userReq.name);
 
+    auto rolesLock = _lockRoles(opCtx);
+
     if (!userReq.roles) {
         // Normal path: Acquire a user from the local store by UserName.
         BSONObj userDoc;
@@ -255,6 +300,8 @@ StatusWith<User> AuthzManagerExternalStateLocal::getUserObject(OperationContext*
         user.setRoles(makeRoleNameIteratorForContainer(directRoles));
     }
 
+    handleAuthLocalGetUserFailPoint(directRoles);
+
     auto data = uassertStatusOK(resolveRoles(opCtx, directRoles, ResolveRoleOption::kAll));
     data.roles->insert(directRoles.cbegin(), directRoles.cend());
     user.setIndirectRoles(makeRoleNameIteratorForContainer(data.roles.get()));
@@ -273,6 +320,8 @@ Status AuthzManagerExternalStateLocal::getUserDescription(OperationContext* opCt
     std::vector<RoleName> directRoles;
     BSONObjBuilder resultBuilder;
 
+    auto rolesLock = _lockRoles(opCtx);
+
     if (!userReq.roles) {
         BSONObj userDoc;
         auto status = findOne(
@@ -302,6 +351,8 @@ Status AuthzManagerExternalStateLocal::getUserDescription(OperationContext* opCt
         rolesBuilder.doneFast();
     }
 
+    handleAuthLocalGetUserFailPoint(directRoles);
+
     auto data = uassertStatusOK(resolveRoles(opCtx, directRoles, ResolveRoleOption::kAll));
     data.roles->insert(directRoles.cbegin(), directRoles.cend());
     serializeResolvedRoles(&resultBuilder, data);
diff --git a/src/mongo/db/auth/authz_manager_external_state_local.h b/src/mongo/db/auth/authz_manager_external_state_local.h
index 8a9b4f871ab..2e489bf559f 100644
--- a/src/mongo/db/auth/authz_manager_external_state_local.h
+++ b/src/mongo/db/auth/authz_manager_external_state_local.h
@@ -37,6 +37,7 @@
 #include "mongo/db/auth/builtin_roles.h"
 #include "mongo/db/auth/role_name.h"
 #include "mongo/db/auth/user_name.h"
+#include "mongo/db/concurrency/d_concurrency.h"
 #include "mongo/platform/mutex.h"
 
 namespace mongo {
@@ -118,6 +119,25 @@ public:
 protected:
     AuthzManagerExternalStateLocal() = default;
 
+    class RolesLocks {
+    public:
+        RolesLocks() = default;
+        RolesLocks(OperationContext*);
+        ~RolesLocks();
+
+    private:
+        std::unique_ptr<Lock::DBLock> _adminLock;
+        std::unique_ptr<Lock::CollectionLock> _rolesLock;
+    };
+
+    /**
+     * Set an auto-releasing shared lock on the roles database.
+     * This allows us to maintain a consistent state during user acquisiiton.
+     *
+     * virtual to allow Mock to not lock anything.
+     */
+    virtual RolesLocks _lockRoles(OperationContext* opCtx);
+
 private:
     /**
      * Once *any* privilege document is observed we cache the state forever,
diff --git a/src/mongo/db/auth/authz_manager_external_state_mock.h b/src/mongo/db/auth/authz_manager_external_state_mock.h
index 709cede8024..19390add9a3 100644
--- a/src/mongo/db/auth/authz_manager_external_state_mock.h
+++ b/src/mongo/db/auth/authz_manager_external_state_mock.h
@@ -114,6 +114,11 @@ public:
 
     std::vector<BSONObj> getCollectionContents(const NamespaceString& collectionName);
 
+protected:
+    RolesLocks _lockRoles(OperationContext* opCtx) override {
+        return RolesLocks();
+    }
+
 private:
     typedef std::vector<BSONObj> BSONObjCollection;
     typedef std::map<NamespaceString, BSONObjCollection> NamespaceDocumentMap;