SERVER-59148: Periodically refresh LDAP users in authorization user cache

11 files changed, 293 insertions, 11 deletions

diff --git a/src/mongo/db/auth/authorization_manager.h b/src/mongo/db/auth/authorization_manager.h
index 0c92156d70f..1459d72078c 100644
--- a/src/mongo/db/auth/authorization_manager.h
+++ b/src/mongo/db/auth/authorization_manager.h
@@ -320,6 +320,13 @@ public:
     virtual void invalidateUsersFromDB(OperationContext* opCtx, StringData dbname) = 0;
 
     /**
+     * Retrieves all users whose source is "$external" and checks if the corresponding user in the
+     * backing store has a different set of roles now. If so, it updates the cache entry with the
+     * new UserHandle.
+     */
+    virtual Status refreshExternalUsers(OperationContext* opCtx) = 0;
+
+    /**
      * Initializes the authorization manager.  Depending on what version the authorization
      * system is at, this may involve building up the user cache and/or the roles graph.
      * Call this function at startup and after resynchronizing a secondary.
diff --git a/src/mongo/db/auth/authorization_manager_impl.cpp b/src/mongo/db/auth/authorization_manager_impl.cpp
index 5ddc068c1d2..2804a04de40 100644
--- a/src/mongo/db/auth/authorization_manager_impl.cpp
+++ b/src/mongo/db/auth/authorization_manager_impl.cpp
@@ -667,6 +667,39 @@ void AuthorizationManagerImpl::invalidateUserCache(OperationContext* opCtx) {
     _userCache.invalidateAll();
 }
 
+Status AuthorizationManagerImpl::refreshExternalUsers(OperationContext* opCtx) {
+    LOGV2_DEBUG(5914801, 2, "Refreshing all users from the $external database");
+    // First, get a snapshot of the UserHandles in the cache.
+    std::vector<UserHandle> cachedUsers = _userCache.getValueHandlesIfKey(
+        [&](const UserRequest& userRequest) { return userRequest.name.getDB() == "$external"_sd; });
+
+    // Then, retrieve the corresponding Users from the backing store for users in the $external
+    // database. Compare each of these user objects with the cached user object and call
+    // insertOrAssign if they differ.
+    bool isRefreshed{false};
+    for (const auto& cachedUser : cachedUsers) {
+        UserRequest request(cachedUser->getName(), boost::none);
+        auto storedUserStatus = _externalState->getUserObject(opCtx, request);
+        if (!storedUserStatus.isOK()) {
+            return storedUserStatus.getStatus();
+        }
+
+        if (cachedUser->hasDifferentRoles(storedUserStatus.getValue())) {
+            _userCache.insertOrAssign(
+                request, std::move(storedUserStatus.getValue()), Date_t::now());
+            isRefreshed = true;
+        }
+    }
+
+    // If any entries were refreshed, then the cache generation must be bumped for mongos to refresh
+    // its cache.
+    if (isRefreshed) {
+        _updateCacheGeneration();
+    }
+
+    return Status::OK();
+}
+
 Status AuthorizationManagerImpl::initialize(OperationContext* opCtx) {
     Status status = _externalState->initialize(opCtx);
     if (!status.isOK())
diff --git a/src/mongo/db/auth/authorization_manager_impl.h b/src/mongo/db/auth/authorization_manager_impl.h
index 6b6d059d790..a569a6f9723 100644
--- a/src/mongo/db/auth/authorization_manager_impl.h
+++ b/src/mongo/db/auth/authorization_manager_impl.h
@@ -107,6 +107,12 @@ public:
 
     void invalidateUsersFromDB(OperationContext* opCtx, StringData dbname) override;
 
+    /**
+     * Verify role information for users in the $external database and insert updated information
+     * into the cache if necessary. Currently, this is only used to refresh LDAP users.
+     */
+    Status refreshExternalUsers(OperationContext* opCtx) override;
+
     Status initialize(OperationContext* opCtx) override;
 
     /**
diff --git a/src/mongo/db/auth/authorization_manager_test.cpp b/src/mongo/db/auth/authorization_manager_test.cpp
index 001433e82f9..f5e5598af4b 100644
--- a/src/mongo/db/auth/authorization_manager_test.cpp
+++ b/src/mongo/db/auth/authorization_manager_test.cpp
@@ -258,5 +258,127 @@ TEST_F(AuthorizationManagerTest, testAcquireV2UserWithUnrecognizedActions) {
     ASSERT(actions.empty());
 }
 
+TEST_F(AuthorizationManagerTest, testRefreshExternalV2User) {
+    constexpr auto kUserFieldName = "user"_sd;
+    constexpr auto kDbFieldName = "db"_sd;
+    constexpr auto kRoleFieldName = "role"_sd;
+
+    // Insert one user on db test and two users on db $external.
+    BSONObj externalCredentials = BSON("external" << true);
+    std::vector<BSONObj> userDocs{BSON("_id"
+                                       << "admin.v2read"
+                                       << "user"
+                                       << "v2read"
+                                       << "db"
+                                       << "test"
+                                       << "credentials" << credentials << "roles"
+                                       << BSON_ARRAY(BSON("role"
+                                                          << "read"
+                                                          << "db"
+                                                          << "test"))),
+                                  BSON("_id"
+                                       << "admin.v2externalOne"
+                                       << "user"
+                                       << "v2externalOne"
+                                       << "db"
+                                       << "$external"
+                                       << "credentials" << externalCredentials << "roles"
+                                       << BSON_ARRAY(BSON("role"
+                                                          << "read"
+                                                          << "db"
+                                                          << "test"))),
+                                  BSON("_id"
+                                       << "admin.v2externalTwo"
+                                       << "user"
+                                       << "v2externalTwo"
+                                       << "db"
+                                       << "$external"
+                                       << "credentials" << externalCredentials << "roles"
+                                       << BSON_ARRAY(BSON("role"
+                                                          << "read"
+                                                          << "db"
+                                                          << "test")))};
+
+    std::vector<BSONObj> initialRoles{BSON("role"
+                                           << "read"
+                                           << "db"
+                                           << "test")};
+    std::vector<BSONObj> updatedRoles{BSON("role"
+                                           << "readWrite"
+                                           << "db"
+                                           << "test")};
+
+    for (const auto& userDoc : userDocs) {
+        ASSERT_OK(externalState->insertPrivilegeDocument(opCtx.get(), userDoc, BSONObj()));
+    }
+
+    // Acquire these users to force the AuthorizationManager to load these users into the user
+    // cache.
+    for (const auto& userDoc : userDocs) {
+        auto swUser = authzManager->acquireUser(
+            opCtx.get(),
+            UserName(userDoc.getStringField(kUserFieldName), userDoc.getStringField(kDbFieldName)));
+        ASSERT_OK(swUser.getStatus());
+        auto user = std::move(swUser.getValue());
+        ASSERT_EQUALS(
+            UserName(userDoc.getStringField(kUserFieldName), userDoc.getStringField(kDbFieldName)),
+            user->getName());
+        ASSERT(user.isValid());
+
+        RoleNameIterator cachedUserRoles = user->getRoles();
+        for (const auto& userDocRole : initialRoles) {
+            ASSERT_EQUALS(cachedUserRoles.next(),
+                          RoleName(userDocRole.getStringField(kRoleFieldName),
+                                   userDocRole.getStringField(kDbFieldName)));
+        }
+        ASSERT_FALSE(cachedUserRoles.more());
+    }
+
+    // Update each of the users added into the external state so that they gain the readWrite role.
+    for (const auto& userDoc : userDocs) {
+        BSONObj updateQuery = BSON("user" << userDoc.getStringField(kUserFieldName));
+        ASSERT_OK(
+            externalState->updateOne(opCtx.get(),
+                                     AuthorizationManager::usersCollectionNamespace,
+                                     updateQuery,
+                                     BSON("$set" << BSON("roles" << BSON_ARRAY(updatedRoles[0]))),
+                                     true,
+                                     BSONObj()));
+    }
+
+    // Refresh all external entries in the authorization manager's cache.
+    ASSERT_OK(authzManager->refreshExternalUsers(opCtx.get()));
+
+    // Retrieve all users from the cache and verify that only the external ones contain the newly
+    // added role.
+    for (const auto& userDoc : userDocs) {
+        auto swUser = authzManager->acquireUser(
+            opCtx.get(),
+            UserName(userDoc.getStringField(kUserFieldName), userDoc.getStringField(kDbFieldName)));
+        ASSERT_OK(swUser.getStatus());
+        auto user = std::move(swUser.getValue());
+        ASSERT_EQUALS(
+            UserName(userDoc.getStringField(kUserFieldName), userDoc.getStringField(kDbFieldName)),
+            user->getName());
+        ASSERT(user.isValid());
+
+        RoleNameIterator cachedUserRolesIt = user->getRoles();
+        if (userDoc.getStringField(kDbFieldName) == "$external"_sd) {
+            for (const auto& userDocRole : updatedRoles) {
+                ASSERT_EQUALS(cachedUserRolesIt.next(),
+                              RoleName(userDocRole.getStringField(kRoleFieldName),
+                                       userDocRole.getStringField(kDbFieldName)));
+            }
+        } else {
+            for (const auto& userDocRole : initialRoles) {
+                ASSERT_EQUALS(cachedUserRolesIt.next(),
+                              RoleName(userDocRole.getStringField(kRoleFieldName),
+                                       userDocRole.getStringField(kDbFieldName)));
+            }
+        }
+        ASSERT_FALSE(cachedUserRolesIt.more());
+    }
+}
+
 }  // namespace
 }  // namespace mongo
diff --git a/src/mongo/db/auth/authz_manager_external_state_mock.cpp b/src/mongo/db/auth/authz_manager_external_state_mock.cpp
index cb684ed4f84..00dd9b421c7 100644
--- a/src/mongo/db/auth/authz_manager_external_state_mock.cpp
+++ b/src/mongo/db/auth/authz_manager_external_state_mock.cpp
@@ -226,10 +226,11 @@ Status AuthzManagerExternalStateMock::updateOne(OperationContext* opCtx,
             return status;
         BSONObj newObj = document.getObject().copy();
         *iter = newObj;
-        BSONObj idQuery = newObj["_id"_sd].Obj();
+        BSONElement idQuery = newObj["_id"_sd];
+        BSONObj idQueryObj = idQuery.isABSONObj() ? idQuery.Obj() : BSON("_id" << idQuery);
 
         if (_authzManager) {
-            _authzManager->logOp(opCtx, "u", collectionName, logObj, &idQuery);
+            _authzManager->logOp(opCtx, "u", collectionName, logObj, &idQueryObj);
         }
 
         return Status::OK();
diff --git a/src/mongo/db/auth/user.cpp b/src/mongo/db/auth/user.cpp
index 6ed98aae6f3..36e2c26ff93 100644
--- a/src/mongo/db/auth/user.cpp
+++ b/src/mongo/db/auth/user.cpp
@@ -117,6 +117,8 @@ void User::setIndirectRoles(RoleNameIterator indirectRoles) {
     while (indirectRoles.more()) {
         _indirectRoles.push_back(indirectRoles.next());
     }
+    // Keep indirectRoles sorted for more efficient comparison against other users.
+    std::sort(_indirectRoles.begin(), _indirectRoles.end());
 }
 
 void User::setPrivileges(const PrivilegeVector& privileges) {
@@ -181,4 +183,29 @@ Status User::validateRestrictions(OperationContext* opCtx) const {
     return Status::OK();
 }
 
+bool User::hasDifferentRoles(const User& otherUser) const {
+    // If the number of direct or indirect roles in the users' are not the same, they have
+    // different roles.
+    if (_roles.size() != otherUser._roles.size() ||
+        _indirectRoles.size() != otherUser._indirectRoles.size()) {
+        return true;
+    }
+
+    // At this point, it is known that the users have the same number of direct roles. The
+    // direct roles sets are equivalent if all of the roles in the first user's directRoles are
+    // also in the other user's directRoles.
+    for (const auto& role : _roles) {
+        if (otherUser._roles.find(role) == otherUser._roles.end()) {
+            return true;
+        }
+    }
+
+    // Indirect roles should always be sorted.
+    dassert(std::is_sorted(_indirectRoles.begin(), _indirectRoles.end()));
+    dassert(std::is_sorted(otherUser._indirectRoles.begin(), otherUser._indirectRoles.end()));
+
+    return !std::equal(
+        _indirectRoles.begin(), _indirectRoles.end(), otherUser._indirectRoles.begin());
+}
+
 }  // namespace mongo
diff --git a/src/mongo/db/auth/user.h b/src/mongo/db/auth/user.h
index 4a6b204ca76..53e5e2a7574 100644
--- a/src/mongo/db/auth/user.h
+++ b/src/mongo/db/auth/user.h
@@ -245,6 +245,8 @@ public:
      */
     Status validateRestrictions(OperationContext* opCtx) const;
 
+    bool hasDifferentRoles(const User& otherUser) const;
+
 private:
     // Unique ID (often UUID) for this user. May be empty for legacy users.
     UserId _id;
diff --git a/src/mongo/embedded/embedded_auth_manager.cpp b/src/mongo/embedded/embedded_auth_manager.cpp
index beb23608acd..fd89bb01d34 100644
--- a/src/mongo/embedded/embedded_auth_manager.cpp
+++ b/src/mongo/embedded/embedded_auth_manager.cpp
@@ -124,6 +124,10 @@ public:
         UASSERT_NOT_IMPLEMENTED;
     }
 
+    Status refreshExternalUsers(OperationContext* opCtx) override {
+        UASSERT_NOT_IMPLEMENTED;
+    }
+
     Status initialize(OperationContext* opCtx) override {
         UASSERT_NOT_IMPLEMENTED;
     }
diff --git a/src/mongo/shell/check_log.js b/src/mongo/shell/check_log.js
index 4db75d72d5d..6169f6ad035 100644
--- a/src/mongo/shell/check_log.js
+++ b/src/mongo/shell/check_log.js
@@ -67,7 +67,7 @@ checkLog = (function() {
                 throw ex;
             }
 
-            if (_compareLogs(obj, id, severity, attrsDict)) {
+            if (_compareLogs(obj, id, severity, null, attrsDict)) {
                 return true;
             }
         }
@@ -83,8 +83,17 @@ checkLog = (function() {
      * complete equality. In addition, the `expectedCount` param ensures that the log appears
      * exactly as many times as expected.
      */
-    const checkContainsWithCountJson = function(
-        conn, id, attrsDict, expectedCount, severity = null, isRelaxed = false) {
+    const checkContainsWithCountJson = function(conn,
+                                                id,
+                                                attrsDict,
+                                                expectedCount,
+                                                severity = null,
+                                                isRelaxed = false,
+                                                comparator =
+                                                    (actual, expected) => {
+                                                        return actual === expected;
+                                                    },
+                                                context = null) {
         const logMessages = getGlobalLog(conn);
         if (logMessages === null) {
             return false;
@@ -101,11 +110,12 @@ checkLog = (function() {
                 throw ex;
             }
 
-            if (_compareLogs(obj, id, severity, attrsDict, isRelaxed)) {
+            if (_compareLogs(obj, id, severity, context, attrsDict, isRelaxed)) {
                 count++;
             }
         }
-        return count === expectedCount;
+
+        return comparator(count, expectedCount);
     };
 
     /*
@@ -165,12 +175,21 @@ checkLog = (function() {
      * intervals on the provided connection 'conn' until a log with id 'id' and all of the
      * attributes in 'attrsDict' is found `expectedCount` times or the timeout (in ms) is reached.
      */
-    let containsRelaxedJson = function(
-        conn, id, attrsDict, expectedCount = 1, timeoutMillis = 5 * 60 * 1000) {
+    let containsRelaxedJson = function(conn,
+                                       id,
+                                       attrsDict,
+                                       expectedCount = 1,
+                                       timeoutMillis = 5 * 60 * 1000,
+                                       comparator =
+                                           (actual, expected) => {
+                                               return actual === expected;
+                                           },
+                                       context = null) {
         // Don't run the hang analyzer because we don't expect contains() to always succeed.
         assert.soon(
             function() {
-                return checkContainsWithCountJson(conn, id, attrsDict, expectedCount, null, true);
+                return checkContainsWithCountJson(
+                    conn, id, attrsDict, expectedCount, null, true, comparator, context);
             },
             'Could not find log entries containing the following id: ' + id +
                 ', and attrs: ' + tojson(attrsDict),
@@ -333,13 +352,22 @@ checkLog = (function() {
      * fields specified in the attrsDict attribute are equal to those in the corresponding attribute
      * of obj. Otherwise, `_deepEqual()` checks that both subobjects are identical.
      */
-    const _compareLogs = function(obj, id, severity, attrsDict, isRelaxed = false) {
+    const _compareLogs = function(obj, id, severity, context, attrsDict, isRelaxed = false) {
         if (obj.id !== id) {
             return false;
         }
         if (severity !== null && obj.s !== severity) {
             return false;
         }
+        if (context !== null) {
+            if (context instanceof RegExp) {
+                if (!context.test(obj.ctx)) {
+                    return false;
+                }
+            } else if (context !== obj.ctx) {
+                return false;
+            }
+        }
 
         for (let attrKey in attrsDict) {
             const attrValue = attrsDict[attrKey];
@@ -347,6 +375,10 @@ checkLog = (function() {
                 if (!attrValue(obj.attr[attrKey])) {
                     return false;
                 }
+            } else if (attrValue instanceof RegExp) {
+                if (!attrValue.test(obj.attr[attrKey])) {
+                    return false;
+                }
             } else if (obj.attr[attrKey] !== attrValue && typeof obj.attr[attrKey] == "object") {
                 if (!_deepEqual(obj.attr[attrKey], attrValue, isRelaxed)) {
                     return false;
diff --git a/src/mongo/util/invalidating_lru_cache.h b/src/mongo/util/invalidating_lru_cache.h
index f2307dc007f..d07eb12e0ec 100644
--- a/src/mongo/util/invalidating_lru_cache.h
+++ b/src/mongo/util/invalidating_lru_cache.h
@@ -535,6 +535,32 @@ public:
         }
     }
 
+    /**
+     * Returns a vector of ValueHandles for all of the entries that satisfy matchPredicate.
+     */
+    template <typename Pred>
+    std::vector<ValueHandle> getEntriesIf(Pred matchPredicate) {
+        std::vector<ValueHandle> entries;
+        entries.reserve(_cache.size() + _evictedCheckedOutValues.size());
+        {
+            stdx::lock_guard lg(_mutex);
+            for (const auto& entry : _cache) {
+                if (matchPredicate(entry.first, &entry.second->value)) {
+                    entries.push_back(ValueHandle(entry.second));
+                }
+            }
+
+            for (const auto& entry : _evictedCheckedOutValues) {
+                if (auto storedValue = entry.second.lock()) {
+                    if (matchPredicate(entry.first, &storedValue->value)) {
+                        entries.push_back(ValueHandle(std::move(storedValue)));
+                    }
+                }
+            }
+        }
+        return entries;
+    }
+
     struct CachedItemInfo {
         Key key;            // The key of the item in the cache
         long int useCount;  // The number of callers of 'get', which still have the item checked-out
diff --git a/src/mongo/util/read_through_cache.h b/src/mongo/util/read_through_cache.h
index 2fe230bdd41..06ce434114e 100644
--- a/src/mongo/util/read_through_cache.h
+++ b/src/mongo/util/read_through_cache.h
@@ -430,6 +430,28 @@ public:
     }
 
     /**
+     * Returns a vector of ValueHandles for all of the keys that satisfy matchPredicate.
+     */
+    template <typename Pred>
+    std::vector<ValueHandle> getValueHandlesIfKey(const Pred& matchPredicate) {
+        stdx::unique_lock ul(_mutex);
+        auto invalidatingCacheValues = _cache.getEntriesIf(
+            [&](const Key& key, const StoredValue*) { return matchPredicate(key); });
+        ul.unlock();
+
+        std::vector<ValueHandle> valueHandles;
+        valueHandles.reserve(invalidatingCacheValues.size());
+        std::transform(invalidatingCacheValues.begin(),
+                       invalidatingCacheValues.end(),
+                       std::back_inserter(valueHandles),
+                       [](auto& invalidatingCacheValue) {
+                           return ValueHandle(std::move(invalidatingCacheValue));
+                       });
+
+        return valueHandles;
+    }
+
+    /**
      * Returns statistics information about the cache for reporting purposes.
      */
     std::vector<typename Cache::CachedItemInfo> getCacheInfo() const {