SERVER-40529 Refresh pinned users in background thread

12 files changed, 254 insertions, 328 deletions

diff --git a/jstests/auth/pinned_users.js b/jstests/auth/pinned_users.js
index 758905f7503..f57bfa85f74 100644
--- a/jstests/auth/pinned_users.js
+++ b/jstests/auth/pinned_users.js
@@ -30,10 +30,10 @@
     // the deadlock
     assert.commandWorked(admin.runCommand({
         setParameter: 1,
+        logLevel: 2,
         authorizationManagerPinnedUsers: [
             {user: "admin2", db: "admin"},
         ],
-        logLevel: 1
     }));
 
     admin.createUser({user: "admin2", pwd: "admin", roles: ["root"]});
@@ -44,8 +44,13 @@
 
     // Invalidate the user cache so we know only "admin" is in there
     assert.commandWorked(admin.runCommand({invalidateUserCache: 1}));
-    print("User cache after initialization: ",
-          tojson(admin.aggregate([{$listCachedAndActiveUsers: {}}]).toArray()));
+    assert.soon(function() {
+        let cacheContents = admin.aggregate([{$listCachedAndActiveUsers: {}}]).toArray();
+        print("User cache after initialization: ", tojson(cacheContents));
+
+        const admin2Doc = sortDoc({"username": "admin2", "db": "admin", "active": true});
+        return cacheContents.some((doc) => friendlyEqual(admin2Doc, sortDoc(doc)));
+    });
 
     const waitForCommand = function(waitingFor, opFilter) {
         let opId = -1;
@@ -123,7 +128,7 @@
     // Mark the "admin2" user as pinned in memory
     assert.commandWorked(admin.runCommand({
         setParameter: 1,
-        logLevel: 1,
+        logLevel: 2,
         authorizationManagerPinnedUsers: [
             {user: "admin2", db: "admin"},
         ],
@@ -144,3 +149,50 @@
     assert.eq(admin.auth("admin2", "admin"), 0);
     MongoRunner.stopMongod(mongod);
 })();
+
+// This checks that clearing the pinned user list actually unpins a user.
+(function() {
+    'use strict';
+    jsTest.setOption("enableTestCommands", true);
+    // Start a mongod with the user cache size set to zero, so we know that users who have
+    // logged out always get fetched cleanly from disk.
+    const mongod =
+        MongoRunner.runMongod({auth: "", setParameter: "authorizationManagerCacheSize=0"});
+    let admin = mongod.getDB("admin");
+
+    admin.createUser({user: "admin", pwd: "admin", roles: ["root"]});
+    admin.auth("admin", "admin");
+
+    // Mark the "admin2" user as pinned in memory
+    assert.commandWorked(admin.runCommand({
+        setParameter: 1,
+        logLevel: 2,
+        authorizationManagerPinnedUsers: [
+            {user: "admin2", db: "admin"},
+        ],
+    }));
+
+    admin.createUser({user: "admin2", pwd: "admin", roles: ["root"]});
+    assert.soon(function() {
+        let cacheContents = admin.aggregate([{$listCachedAndActiveUsers: {}}]).toArray();
+        print("User cache after initialization: ", tojson(cacheContents));
+
+        const admin2Doc = sortDoc({"username": "admin2", "db": "admin", "active": true});
+        return cacheContents.some((doc) => friendlyEqual(admin2Doc, sortDoc(doc)));
+    });
+
+    // Clear the pinned users list
+    assert.commandWorked(admin.runCommand({setParameter: 1, authorizationManagerPinnedUsers: []}));
+
+    // Check that admin2 gets removed from the cache
+    assert.commandWorked(admin.runCommand({invalidateUserCache: 1}));
+    assert.soon(function() {
+        let cacheContents = admin.aggregate([{$listCachedAndActiveUsers: {}}]).toArray();
+        print("User cache after initialization: ", tojson(cacheContents));
+
+        const admin2Doc = sortDoc({"username": "admin2", "db": "admin", "active": true});
+        return !cacheContents.some((doc) => friendlyEqual(admin2Doc, sortDoc(doc)));
+    });
+
+    MongoRunner.stopMongod(mongod);
+})();
diff --git a/src/mongo/db/auth/authorization_manager.h b/src/mongo/db/auth/authorization_manager.h
index b1b6a6e0583..0646954b6ae 100644
--- a/src/mongo/db/auth/authorization_manager.h
+++ b/src/mongo/db/auth/authorization_manager.h
@@ -107,7 +107,6 @@ public:
     static constexpr StringData V1_USER_NAME_FIELD_NAME = "user"_sd;
     static constexpr StringData V1_USER_SOURCE_FIELD_NAME = "userSource"_sd;
 
-
     static const NamespaceString adminCommandNamespace;
     static const NamespaceString rolesCollectionNamespace;
     static const NamespaceString usersAltCollectionNamespace;
@@ -117,6 +116,7 @@ public:
     static const NamespaceString defaultTempUsersCollectionNamespace;  // for mongorestore
     static const NamespaceString defaultTempRolesCollectionNamespace;  // for mongorestore
 
+
     /**
      * Status to be returned when authentication fails. Being consistent about our returned Status
      * prevents information leakage.
@@ -296,6 +296,13 @@ public:
     virtual void invalidateUserCache(OperationContext* opCtx) = 0;
 
     /**
+     * Sets the list of users that should be pinned in memory.
+     *
+     * This will start the PinnedUserTracker thread if it hasn't been started already.
+     */
+    virtual void updatePinnedUsersList(std::vector<UserName> names) = 0;
+
+    /**
      * Parses privDoc and fully initializes the user object (credentials, roles, and privileges)
      * with the information extracted from the privilege document.
      * This should never be called from outside the AuthorizationManager - the only reason it's
@@ -323,8 +330,6 @@ public:
     };
 
     virtual std::vector<CachedUserInfo> getUserCacheInfo() const = 0;
-
-    virtual void setInUserManagementCommand(OperationContext* opCtx, bool val) = 0;
 };
 
 }  // namespace mongo
diff --git a/src/mongo/db/auth/authorization_manager_impl.cpp b/src/mongo/db/auth/authorization_manager_impl.cpp
index d97b1c5d4b9..3342212c020 100644
--- a/src/mongo/db/auth/authorization_manager_impl.cpp
+++ b/src/mongo/db/auth/authorization_manager_impl.cpp
@@ -57,7 +57,7 @@
 #include "mongo/db/auth/user_document_parser.h"
 #include "mongo/db/auth/user_management_commands_parser.h"
 #include "mongo/db/auth/user_name.h"
-
+#include "mongo/db/concurrency/d_concurrency.h"
 #include "mongo/db/global_settings.h"
 #include "mongo/db/jsobj.h"
 #include "mongo/db/mongod_options.h"
@@ -116,7 +116,8 @@ class PinnedUserSetParameter {
 public:
     void append(OperationContext* opCtx, BSONObjBuilder& b, const std::string& name) const {
         BSONArrayBuilder sub(b.subarrayStart(name));
-        for (const auto& username : _userNames) {
+        stdx::lock_guard<stdx::mutex> lk(_mutex);
+        for (const auto& username : _pinnedUsersList) {
             BSONObjBuilder nameObj(sub.subobjStart());
             nameObj << AuthorizationManager::USER_NAME_FIELD_NAME << username.getUser()
                     << AuthorizationManager::USER_DB_FIELD_NAME << username.getDB();
@@ -138,16 +139,14 @@ public:
                 return status;
             }
 
-            stdx::unique_lock<stdx::mutex> lk(_mutex);
-            _userNames = std::move(out);
+            stdx::lock_guard<stdx::mutex> lk(_mutex);
+            _pinnedUsersList = out;
             auto authzManager = _authzManager;
             if (!authzManager) {
                 return Status::OK();
             }
 
-            lk.unlock();
-
-            authzManager->invalidateUserCache(Client::getCurrent()->getOperationContext());
+            authzManager->updatePinnedUsersList(std::move(out));
             return Status::OK();
         } else {
             return {ErrorCodes::BadValue,
@@ -173,28 +172,21 @@ public:
             return status;
         }
 
+        stdx::lock_guard<stdx::mutex> lk(_mutex);
+        _pinnedUsersList = out;
         auto authzManager = _authzManager;
         if (!authzManager) {
             return Status::OK();
         }
 
-        {
-            stdx::lock_guard<stdx::mutex> lk(_mutex);
-            _userNames = std::move(out);
-        }
-
-        authzManager->invalidateUserCache(Client::getCurrent()->getOperationContext());
+        authzManager->updatePinnedUsersList(std::move(out));
         return Status::OK();
     }
 
     void setAuthzManager(AuthorizationManager* authzManager) {
         stdx::lock_guard<stdx::mutex> lk(_mutex);
         _authzManager = authzManager;
-    }
-
-    std::vector<UserName> getUserNames() {
-        stdx::lock_guard<stdx::mutex> lk(_mutex);
-        return _userNames;
+        _authzManager->updatePinnedUsersList(std::move(_pinnedUsersList));
     }
 
 private:
@@ -207,9 +199,10 @@ private:
         }
         return Status::OK();
     }
-    stdx::mutex _mutex;
-    std::vector<UserName> _userNames;
     AuthorizationManager* _authzManager = nullptr;
+
+    mutable stdx::mutex _mutex;
+    std::vector<UserName> _pinnedUsersList;
 } authorizationManagerPinnedUsers;
 
 }  // namespace
@@ -271,19 +264,18 @@ public:
     explicit CacheGuard(OperationContext* opCtx, AuthorizationManagerImpl* authzManager)
         : _isThisGuardInFetchPhase(false),
           _authzManager(authzManager),
-          _stateLock(_authzManager->_externalState->lock(opCtx)),
-          _lock(authzManager->_cacheWriteMutex) {}
+          _cacheLock(authzManager->_cacheWriteMutex) {}
 
     /**
      * Releases the mutex that synchronizes user cache access, if held, and notifies
      * any threads waiting for their own opportunity to update the user cache.
      */
     ~CacheGuard() {
-        if (!_lock.owns_lock()) {
-            _lock.lock();
+        if (!_cacheLock.owns_lock()) {
+            _cacheLock.lock();
         }
         if (_isThisGuardInFetchPhase) {
-            fassert(17190, _authzManager->_isFetchPhaseBusy);
+            fassert(17190, otherUpdateInFetchPhase());
             _authzManager->_isFetchPhaseBusy = false;
             _authzManager->_fetchPhaseIsReady.notify_all();
         }
@@ -301,7 +293,8 @@ public:
      */
     void wait() {
         fassert(17222, !_isThisGuardInFetchPhase);
-        _authzManager->_fetchPhaseIsReady.wait(_lock);
+        _authzManager->_fetchPhaseIsReady.wait(_cacheLock,
+                                               [&] { return !otherUpdateInFetchPhase(); });
     }
 
     /**
@@ -309,26 +302,18 @@ public:
      * cache generation.
      */
     void beginFetchPhase() {
-        beginFetchPhaseNoYield();
-        _lock.unlock();
-    }
-
-    /**
-     * Sets up the fetch phase without releasing _authzManager->_cacheMutex
-     */
-    void beginFetchPhaseNoYield() {
-        fassert(17191, !_authzManager->_isFetchPhaseBusy);
+        fassert(17191, !otherUpdateInFetchPhase());
         _isThisGuardInFetchPhase = true;
         _authzManager->_isFetchPhaseBusy = true;
         _startGeneration = _authzManager->_fetchGeneration;
+        _cacheLock.unlock();
     }
 
     /**
      * Exits the fetch phase, reacquiring the _authzManager->_cacheMutex.
      */
     void endFetchPhase() {
-        if (!_lock.owns_lock())
-            _lock.lock();
+        _cacheLock.lock();
         // We do not clear _authzManager->_isFetchPhaseBusy or notify waiters until
         // ~CacheGuard(), for two reasons.  First, there's no value to notifying the waiters
         // before you're ready to release the mutex, because they'll just go to sleep on the
@@ -346,7 +331,7 @@ public:
      */
     bool isSameCacheGeneration() const {
         fassert(17223, _isThisGuardInFetchPhase);
-        fassert(17231, _lock.owns_lock());
+        fassert(17231, _cacheLock.owns_lock());
         return _startGeneration == _authzManager->_fetchGeneration;
     }
 
@@ -354,8 +339,8 @@ private:
     OID _startGeneration;
     bool _isThisGuardInFetchPhase;
     AuthorizationManagerImpl* _authzManager;
-    std::unique_ptr<AuthzManagerExternalState::StateLock> _stateLock;
-    stdx::unique_lock<stdx::mutex> _lock;
+
+    stdx::unique_lock<stdx::mutex> _cacheLock;
 };
 
 AuthorizationManagerImpl::AuthorizationManagerImpl()
@@ -369,8 +354,7 @@ AuthorizationManagerImpl::AuthorizationManagerImpl(
       _externalState(std::move(externalState)),
       _version(schemaVersionInvalid),
       _userCache(authorizationManagerCacheSize, UserCacheInvalidator()),
-      _fetchGeneration(OID::gen()),
-      _isFetchPhaseBusy(false) {}
+      _fetchGeneration(OID::gen()) {}
 
 AuthorizationManagerImpl::~AuthorizationManagerImpl() {}
 
@@ -431,7 +415,6 @@ bool AuthorizationManagerImpl::hasAnyPrivilegeDocuments(OperationContext* opCtx)
         return true;
     }
 
-    auto stateLk = _externalState->lock(opCtx);
     bool privDocsExist = _externalState->hasAnyPrivilegeDocuments(opCtx);
 
     if (privDocsExist) {
@@ -482,7 +465,6 @@ Status AuthorizationManagerImpl::_initializeUserFromPrivilegeDocument(User* user
 Status AuthorizationManagerImpl::getUserDescription(OperationContext* opCtx,
                                                     const UserName& userName,
                                                     BSONObj* result) {
-    auto lk = _externalState->lock(opCtx);
     return _externalState->getUserDescription(opCtx, userName, result);
 }
 
@@ -491,7 +473,6 @@ Status AuthorizationManagerImpl::getRoleDescription(OperationContext* opCtx,
                                                     PrivilegeFormat privileges,
                                                     AuthenticationRestrictionsFormat restrictions,
                                                     BSONObj* result) {
-    auto lk = _externalState->lock(opCtx);
     return _externalState->getRoleDescription(opCtx, roleName, privileges, restrictions, result);
 }
 
@@ -500,7 +481,6 @@ Status AuthorizationManagerImpl::getRolesDescription(OperationContext* opCtx,
                                                      PrivilegeFormat privileges,
                                                      AuthenticationRestrictionsFormat restrictions,
                                                      BSONObj* result) {
-    auto lk = _externalState->lock(opCtx);
     return _externalState->getRolesDescription(opCtx, roleName, privileges, restrictions, result);
 }
 
@@ -512,7 +492,6 @@ Status AuthorizationManagerImpl::getRoleDescriptionsForDB(
     AuthenticationRestrictionsFormat restrictions,
     bool showBuiltinRoles,
     vector<BSONObj>* result) {
-    auto lk = _externalState->lock(opCtx);
     return _externalState->getRoleDescriptionsForDB(
         opCtx, dbname, privileges, restrictions, showBuiltinRoles, result);
 }
@@ -559,41 +538,7 @@ StatusWith<UserHandle> AuthorizationManagerImpl::acquireUser(OperationContext* o
     guard.beginFetchPhase();
     // If there's still no user in the cache, then we need to go to disk. Take the slow path.
     LOG(1) << "Getting user " << userName << " from disk";
-    auto ret = _acquireUserSlowPath(guard, opCtx, userName);
-
-    guard.endFetchPhase();
-    if (!ret.isOK()) {
-        return ret.getStatus();
-    }
 
-    auto user = std::move(ret.getValue());
-    if (user->isValid()) {
-        _updateCacheGeneration_inlock(guard);
-    }
-
-    return user;
-}
-
-StatusWith<UserHandle> AuthorizationManagerImpl::acquireUserForSessionRefresh(
-    OperationContext* opCtx, const UserName& userName, const User::UserId& uid) {
-    auto swUserHandle = acquireUser(opCtx, userName);
-    if (!swUserHandle.isOK()) {
-        return swUserHandle.getStatus();
-    }
-
-    auto ret = std::move(swUserHandle.getValue());
-    if (uid != ret->getID()) {
-        return {ErrorCodes::UserNotFound,
-                str::stream() << "User id from privilege document '" << userName.toString()
-                              << "' does not match user id in session."};
-    }
-
-    return ret;
-}
-
-StatusWith<UserHandle> AuthorizationManagerImpl::_acquireUserSlowPath(CacheGuard& guard,
-                                                                      OperationContext* opCtx,
-                                                                      const UserName& userName) {
     int authzVersion = _version;
 
     // Number of times to retry a user document that fetches due to transient
@@ -643,83 +588,38 @@ StatusWith<UserHandle> AuthorizationManagerImpl::_acquireUserSlowPath(CacheGuard
     // may also call endFetchPhase() after this returns.
     guard.endFetchPhase();
 
+    UserHandle ret;
     if (guard.isSameCacheGeneration()) {
         if (_version == schemaVersionInvalid)
             _version = authzVersion;
-        return _userCache.insertOrAssignAndGet(userName, std::move(user));
+        ret = _userCache.insertOrAssignAndGet(userName, std::move(user));
+        _updateCacheGeneration_inlock(guard);
     } else {
         // If the cache generation changed while this thread was in fetch mode, the data
         // associated with the user may now be invalid, so we must mark it as such.  The caller
         // may still opt to use the information for a short while, but not indefinitely.
         user->_invalidate();
-        return UserHandle(std::move(user));
-    }
-}
-
-std::vector<UserHandle> AuthorizationManagerImpl::_fetchPinnedUsers(CacheGuard& guard,
-                                                                    OperationContext* opCtx) {
-    // Get the list of users to pin
-    const auto usersToPin = authorizationManagerPinnedUsers.getUserNames();
-    if (usersToPin.empty()) {
-        return {};
-    }
-
-    // Remove any users that shouldn't be pinned anymore or that are invalid.
-    std::vector<UserHandle> newPinnedUsers;
-    std::copy_if(_pinnedUsers.begin(),
-                 _pinnedUsers.end(),
-                 std::back_inserter(newPinnedUsers),
-                 [&](const UserHandle& user) {
-                     if (!user->isValid())
-                         return false;
-                     return std::any_of(
-                         usersToPin.begin(), usersToPin.end(), [&](const UserName& userName) {
-                             return (user->getName() == userName);
-                         });
-                 });
-
-    const auto findPinnedUser = [&](const auto& userName) {
-        return std::find_if(
-            newPinnedUsers.begin(), newPinnedUsers.end(), [&](const auto& userHandle) {
-                return (userHandle->getName() == userName);
-            });
-    };
-
-
-    while (guard.otherUpdateInFetchPhase()) {
-        guard.wait();
+        ret = UserHandle(std::move(user));
     }
 
-    // Begin the fetch phase but don't yield any locks. We want re-pinning of users to block other
-    // operations until all the users are refreshed.
-    guard.beginFetchPhaseNoYield();
-
-    bool cacheUpdated = false;
-    for (const auto& userName : usersToPin) {
-        auto existingPin = findPinnedUser(userName);
-        if (existingPin != newPinnedUsers.end()) {
-            newPinnedUsers.push_back(*existingPin);
-            continue;
-        }
+    return ret;
+}
 
-        cacheUpdated = true;
-        auto swUser = _acquireUserSlowPath(guard, opCtx, userName);
-        if (swUser.isOK()) {
-            newPinnedUsers.emplace(newPinnedUsers.end(), std::move(swUser.getValue()));
-        } else {
-            const auto& status = swUser.getStatus();
-            // If the user is not found, then it might just not exist yet. Skip this user for now.
-            if (status != ErrorCodes::UserNotFound) {
-                warning() << "Unable to fetch pinned user " << userName.toString() << ": "
-                          << status;
-            }
-        }
+StatusWith<UserHandle> AuthorizationManagerImpl::acquireUserForSessionRefresh(
+    OperationContext* opCtx, const UserName& userName, const User::UserId& uid) {
+    auto swUserHandle = acquireUser(opCtx, userName);
+    if (!swUserHandle.isOK()) {
+        return swUserHandle.getStatus();
     }
 
-    if (cacheUpdated)
-        _updateCacheGeneration_inlock(guard);
+    auto ret = std::move(swUserHandle.getValue());
+    if (uid != ret->getID()) {
+        return {ErrorCodes::UserNotFound,
+                str::stream() << "User id from privilege document '" << userName.toString()
+                              << "' does not match user id in session."};
+    }
 
-    return newPinnedUsers;
+    return ret;
 }
 
 Status AuthorizationManagerImpl::_fetchUserV2(OperationContext* opCtx,
@@ -742,53 +642,118 @@ Status AuthorizationManagerImpl::_fetchUserV2(OperationContext* opCtx,
     return Status::OK();
 }
 
-void AuthorizationManagerImpl::invalidateUserByName(OperationContext* opCtx,
-                                                    const UserName& userName) {
-    setPinnedUsers(invalidateUserByNameNoPin(opCtx, userName));
+void AuthorizationManagerImpl::updatePinnedUsersList(std::vector<UserName> names) {
+    stdx::unique_lock<stdx::mutex> lk(_pinnedUsersMutex);
+    _usersToPin = std::move(names);
+    bool noUsersToPin = _usersToPin->empty();
+    _pinnedUsersCond.notify_one();
+    if (noUsersToPin) {
+        LOG(1) << "There were no users to pin, not starting tracker thread";
+        return;
+    }
+
+    std::call_once(_pinnedThreadTrackerStarted, [this] {
+        stdx::thread thread(&AuthorizationManagerImpl::_pinnedUsersThreadRoutine, this);
+        thread.detach();
+    });
 }
 
-std::vector<UserHandle> AuthorizationManagerImpl::invalidateUserByNameNoPin(
-    OperationContext* opCtx, const UserName& userName) {
-    CacheGuard guard(opCtx, this);
-    _updateCacheGeneration_inlock(guard);
-    _userCache.invalidate(userName);
+void AuthorizationManagerImpl::_pinnedUsersThreadRoutine() noexcept try {
+    Client::initThread("PinnedUsersTracker");
+    std::list<UserHandle> pinnedUsers;
+    std::vector<UserName> usersToPin;
+    LOG(1) << "Starting pinned users tracking thread";
+    while (true) {
+        auto opCtx = cc().makeOperationContext();
 
-    return _fetchPinnedUsers(guard, opCtx);
-}
+        stdx::unique_lock<stdx::mutex> lk(_pinnedUsersMutex);
+        const Milliseconds timeout(authorizationManagerPinnedUsersRefreshIntervalMillis.load());
+        auto waitRes = opCtx->waitForConditionOrInterruptFor(
+            _pinnedUsersCond, lk, timeout, [&] { return _usersToPin.has_value(); });
 
-void AuthorizationManagerImpl::setPinnedUsers_inlock(const CacheGuard& guard,
-                                                     std::vector<UserHandle> refreshedUsers) {
-    _pinnedUsers = std::move(refreshedUsers);
+        if (waitRes) {
+            usersToPin = std::move(_usersToPin.get());
+            _usersToPin = boost::none;
+        }
+        lk.unlock();
+        if (usersToPin.empty()) {
+            pinnedUsers.clear();
+            continue;
+        }
+
+        // Remove any users that shouldn't be pinned anymore or that are invalid.
+        for (auto it = pinnedUsers.begin(); it != pinnedUsers.end();) {
+            const auto& user = *it;
+            const auto shouldPin =
+                std::any_of(usersToPin.begin(), usersToPin.end(), [&](const UserName& userName) {
+                    return (user->getName() == userName);
+                });
+
+            if (!user->isValid() || !shouldPin) {
+                if (!shouldPin) {
+                    LOG(2) << "Unpinning user " << user->getName();
+                } else {
+                    LOG(2) << "Pinned user no longer valid, will re-pin " << user->getName();
+                }
+                it = pinnedUsers.erase(it);
+            } else {
+                LOG(3) << "Pinned user is still valid and pinned " << user->getName();
+                ++it;
+            }
+        }
+
+        for (const auto& userName : usersToPin) {
+            if (std::any_of(pinnedUsers.begin(), pinnedUsers.end(), [&](const auto& user) {
+                    return user->getName() == userName;
+                })) {
+                continue;
+            }
+
+            auto swUser = acquireUser(opCtx.get(), userName);
+
+            if (swUser.isOK()) {
+                LOG(2) << "Pinned user " << userName;
+                pinnedUsers.emplace_back(std::move(swUser.getValue()));
+            } else {
+                const auto& status = swUser.getStatus();
+                // If the user is not found, then it might just not exist yet. Skip this user for
+                // now.
+                if (status != ErrorCodes::UserNotFound) {
+                    warning() << "Unable to fetch pinned user " << userName.toString() << ": "
+                              << status;
+                } else {
+                    LOG(2) << "Pinned user not found: " << userName;
+                }
+            }
+        }
+    }
+} catch (const ExceptionFor<ErrorCodes::InterruptedAtShutdown>&) {
+    LOG(1) << "Ending pinned users tracking thread";
+    return;
 }
 
-void AuthorizationManagerImpl::setPinnedUsers(std::vector<UserHandle> refreshedUsers) {
-    stdx::lock_guard<stdx::mutex> lock(_cacheWriteMutex);
-    _pinnedUsers = std::move(refreshedUsers);
+void AuthorizationManagerImpl::invalidateUserByName(OperationContext* opCtx,
+                                                    const UserName& userName) {
+    CacheGuard guard(opCtx, this);
+    _updateCacheGeneration_inlock(guard);
+    LOG(2) << "Invalidating user " << userName;
+    _userCache.invalidate(userName);
 }
 
 void AuthorizationManagerImpl::invalidateUsersFromDB(OperationContext* opCtx, StringData dbname) {
     CacheGuard guard(opCtx, this);
     _updateCacheGeneration_inlock(guard);
+    LOG(2) << "Invalidating all users from database " << dbname;
     _userCache.invalidateIf(
         [&](const UserName& user, const User*) { return user.getDB() == dbname; });
-
-
-    setPinnedUsers_inlock(guard, _fetchPinnedUsers(guard, opCtx));
 }
 
 void AuthorizationManagerImpl::invalidateUserCache(OperationContext* opCtx) {
-    setPinnedUsers(invalidateUserCacheNoPin(opCtx));
-}
-
-std::vector<UserHandle> AuthorizationManagerImpl::invalidateUserCacheNoPin(
-    OperationContext* opCtx) {
     CacheGuard guard(opCtx, this);
+    LOG(2) << "Invalidating user cache";
     _invalidateUserCache_inlock(guard);
-
-    return _fetchPinnedUsers(guard, opCtx);
 }
 
-
 void AuthorizationManagerImpl::_invalidateUserCache_inlock(const CacheGuard& guard) {
     _updateCacheGeneration_inlock(guard);
     _userCache.invalidateIf([](const UserName& a, const User*) { return true; });
@@ -887,8 +852,4 @@ std::vector<AuthorizationManager::CachedUserInfo> AuthorizationManagerImpl::getU
     return ret;
 }
 
-void AuthorizationManagerImpl::setInUserManagementCommand(OperationContext* opCtx, bool val) {
-    _externalState->setInUserManagementCommand(opCtx, val);
-}
-
 }  // namespace mongo
diff --git a/src/mongo/db/auth/authorization_manager_impl.h b/src/mongo/db/auth/authorization_manager_impl.h
index a0b82fecade..830e6094bce 100644
--- a/src/mongo/db/auth/authorization_manager_impl.h
+++ b/src/mongo/db/auth/authorization_manager_impl.h
@@ -32,6 +32,7 @@
 #include "mongo/db/auth/authorization_manager.h"
 
 #include <memory>
+#include <mutex>
 #include <string>
 
 #include "mongo/base/secure_allocator.h"
@@ -125,12 +126,6 @@ public:
      * Invalidate a user, and repin it if necessary.
      */
     void invalidateUserByName(OperationContext* opCtx, const UserName& user) override;
-    /**
-     * Invalidate a user, and return a full set of pinned users to be later attached to
-     * the AuthorizationManager.
-     */
-    std::vector<UserHandle> invalidateUserByNameNoPin(OperationContext* opCtx,
-                                                      const UserName& user);
 
     void invalidateUsersFromDB(OperationContext* opCtx, StringData dbname) override;
 
@@ -141,17 +136,7 @@ public:
      */
     void invalidateUserCache(OperationContext* opCtx) override;
 
-    /**
-     * Invalidate the user cache, without repinning users. Instead return the set
-     * of users which should be later pinned.
-     */
-    std::vector<UserHandle> invalidateUserCacheNoPin(OperationContext* opCtx);
-
-    /**
-     * Attach a set of previously acquired Users to the AuthorizationManager
-     * as pinned users.
-     */
-    void setPinnedUsers(std::vector<UserHandle>);
+    void updatePinnedUsersList(std::vector<UserName> names) override;
 
     Status _initializeUserFromPrivilegeDocument(User* user, const BSONObj& privDoc) override;
 
@@ -163,8 +148,6 @@ public:
 
     std::vector<CachedUserInfo> getUserCacheInfo() const override;
 
-    void setInUserManagementCommand(OperationContext* opCtx, bool val) override;
-
 private:
     /**
      * Type used to guard accesses and updates to the user cache.
@@ -173,12 +156,6 @@ private:
     friend class AuthorizationManagerImpl::CacheGuard;
 
     /**
-     * Attach a set of previously acquired Users to the AuthorizationManager
-     * as pinned users, under the protection of an existing CacheGuard.
-     */
-    void setPinnedUsers_inlock(const CacheGuard&, std::vector<UserHandle>);
-
-    /**
      * Invalidates all User objects in the cache and removes them from the cache.
      * Should only be called when already holding _cacheMutex.
      */
@@ -200,12 +177,7 @@ private:
      */
     void _updateCacheGeneration_inlock(const CacheGuard&);
 
-
-    std::vector<UserHandle> _fetchPinnedUsers(CacheGuard& guard, OperationContext* opCtx);
-
-    StatusWith<UserHandle> _acquireUserSlowPath(CacheGuard& guard,
-                                                OperationContext* opCtx,
-                                                const UserName& userName);
+    void _pinnedUsersThreadRoutine() noexcept;
 
     /**
      * Fetches user information from a v2-schema user document for the named user,
@@ -259,7 +231,11 @@ private:
     };
 
     InvalidatingLRUCache<UserName, User, UserCacheInvalidator> _userCache;
-    std::vector<UserHandle> _pinnedUsers;
+
+    stdx::mutex _pinnedUsersMutex;
+    stdx::condition_variable _pinnedUsersCond;
+    std::once_flag _pinnedThreadTrackerStarted;
+    boost::optional<std::vector<UserName>> _usersToPin;
 
     /**
      * Protects _cacheGeneration, _version and _isFetchPhaseBusy.  Manipulated
@@ -279,15 +255,13 @@ private:
      *
      * Manipulated via CacheGuard.
      */
-    bool _isFetchPhaseBusy;
+    bool _isFetchPhaseBusy = false;
 
     /**
      * Condition used to signal that it is OK for another CacheGuard to enter a fetch phase.
      * Manipulated via CacheGuard.
      */
     stdx::condition_variable _fetchPhaseIsReady;
-
-    AtomicWord<bool> _inUserManagementCommand{false};
 };
 
 extern int authorizationManagerCacheSize;
diff --git a/src/mongo/db/auth/authorization_manager_impl_parameters.idl b/src/mongo/db/auth/authorization_manager_impl_parameters.idl
index c6ae14726ab..795c83f219d 100644
--- a/src/mongo/db/auth/authorization_manager_impl_parameters.idl
+++ b/src/mongo/db/auth/authorization_manager_impl_parameters.idl
@@ -51,3 +51,12 @@ server_parameters:
     cpp_class:
       name: AuthorizationManagerPinnedUsersServerParameter
       override_set: true
+
+  authorizationManagerPinnedUsersRefreshIntervalMillis:
+    description: The number of milliseconds between refreshing pinned users.
+    set_at:
+      - startup
+      - runtime
+    cpp_vartype: AtomicWord<long long>
+    cpp_varname: authorizationManagerPinnedUsersRefreshIntervalMillis
+    default: 1000
diff --git a/src/mongo/db/auth/authz_manager_external_state.h b/src/mongo/db/auth/authz_manager_external_state.h
index f7733da4e70..d4f2539c7f5 100644
--- a/src/mongo/db/auth/authz_manager_external_state.h
+++ b/src/mongo/db/auth/authz_manager_external_state.h
@@ -167,35 +167,6 @@ public:
                        const NamespaceString& ns,
                        const BSONObj& o,
                        const BSONObj* o2) {}
-    /**
-     * Represents a lock_guard on the storage for this implementation of the external state.
-     */
-    class StateLock {
-        StateLock(StateLock&) = delete;
-        StateLock& operator=(StateLock&) = delete;
-
-    public:
-        StateLock() = default;
-        virtual ~StateLock() = default;
-    };
-
-    /**
-     * Returns a Lock on the external state for the given operation context.
-     *
-     * By default this returns an empty/noop StateLock.
-     */
-    virtual std::unique_ptr<StateLock> lock(OperationContext* opCtx) {
-        return std::make_unique<StateLock>();
-    };
-
-    /**
-     * Returns true if you must acquire a StateLock before fetching a user description.
-     *
-     * By default this returns false since only mongod actually needs to do locking at this level.
-     */
-    virtual bool needsLockForUserName(OperationContext* opCtx, const UserName& user) {
-        return false;
-    }
 
     virtual void setInUserManagementCommand(OperationContext* opCtx, bool val) {}
 
diff --git a/src/mongo/db/auth/authz_manager_external_state_d.cpp b/src/mongo/db/auth/authz_manager_external_state_d.cpp
index 751ce2e2d57..1f58ef1ecf4 100644
--- a/src/mongo/db/auth/authz_manager_external_state_d.cpp
+++ b/src/mongo/db/auth/authz_manager_external_state_d.cpp
@@ -58,36 +58,6 @@ std::unique_ptr<AuthzSessionExternalState>
 AuthzManagerExternalStateMongod::makeAuthzSessionExternalState(AuthorizationManager* authzManager) {
     return stdx::make_unique<AuthzSessionExternalStateMongod>(authzManager);
 }
-
-class AuthzLock : public AuthzManagerExternalState::StateLock {
-public:
-    explicit AuthzLock(OperationContext* opCtx)
-        : _lock(opCtx,
-                AuthorizationManager::usersCollectionNamespace.db(),
-                MODE_S,
-                opCtx->getDeadline()) {}
-
-    static bool isLocked(OperationContext* opCtx);
-
-private:
-    Lock::DBLock _lock;
-};
-
-bool AuthzLock::isLocked(OperationContext* opCtx) {
-    return opCtx->lockState()->isDbLockedForMode(
-        AuthorizationManager::usersCollectionNamespace.db(), MODE_S);
-}
-
-std::unique_ptr<AuthzManagerExternalState::StateLock> AuthzManagerExternalStateMongod::lock(
-    OperationContext* opCtx) {
-    return std::make_unique<AuthzLock>(opCtx);
-}
-
-bool AuthzManagerExternalStateMongod::needsLockForUserName(OperationContext* opCtx,
-                                                           const UserName& name) {
-    return (shouldUseRolesFromConnection(opCtx, name) == false);
-}
-
 Status AuthzManagerExternalStateMongod::query(
     OperationContext* opCtx,
     const NamespaceString& collectionName,
diff --git a/src/mongo/db/auth/authz_manager_external_state_d.h b/src/mongo/db/auth/authz_manager_external_state_d.h
index 38d1c1df8b7..97cbc1b074b 100644
--- a/src/mongo/db/auth/authz_manager_external_state_d.h
+++ b/src/mongo/db/auth/authz_manager_external_state_d.h
@@ -53,10 +53,6 @@ public:
     std::unique_ptr<AuthzSessionExternalState> makeAuthzSessionExternalState(
         AuthorizationManager* authzManager) override;
 
-    std::unique_ptr<StateLock> lock(OperationContext* opCtx) final;
-
-    bool needsLockForUserName(OperationContext* opCtx, const UserName& name) final;
-
     virtual Status findOne(OperationContext* opCtx,
                            const NamespaceString& collectionName,
                            const BSONObj& query,
diff --git a/src/mongo/db/auth/authz_manager_external_state_local.cpp b/src/mongo/db/auth/authz_manager_external_state_local.cpp
index 386a82e9f1e..e3184bef814 100644
--- a/src/mongo/db/auth/authz_manager_external_state_local.cpp
+++ b/src/mongo/db/auth/authz_manager_external_state_local.cpp
@@ -49,11 +49,6 @@
 
 namespace mongo {
 
-namespace {
-
-const auto inUserManagementCommandsFlag = OperationContext::declareDecoration<bool>();
-}
-
 using std::vector;
 
 Status AuthzManagerExternalStateLocal::initialize(OperationContext* opCtx) {
@@ -545,18 +540,16 @@ public:
           _op(op),
           _nss(nss),
           _o(o.getOwned()),
-          _o2(o2 ? boost::optional<BSONObj>(o2->getOwned()) : boost::none),
-          _refreshedPinnedUsers(_invalidateRelevantCacheData()) {}
+          _o2(o2 ? boost::optional<BSONObj>(o2->getOwned()) : boost::none) {
 
-    void commit(boost::optional<Timestamp>) final {
+        _invalidateRelevantCacheData();
+    }
+
+    void commit(boost::optional<Timestamp> timestamp) final {
         if (_nss == AuthorizationManager::rolesCollectionNamespace ||
             _nss == AuthorizationManager::adminCommandNamespace) {
             _refreshRoleGraph();
         }
-
-        if (_refreshedPinnedUsers) {
-            _authzManager->setPinnedUsers(std::move(*_refreshedPinnedUsers));
-        }
     }
 
     void rollback() final {}
@@ -612,18 +605,11 @@ private:
         }
     }
 
-    boost::optional<std::vector<UserHandle>> _invalidateRelevantCacheData() {
-        // When we're doing a user management command we lock the admin DB for the duration
-        // of the command and invalidate the cache at the end of the command, so we don't need
-        // to invalidate it based on calls to logOp().
-        if (inUserManagementCommandsFlag(_opCtx)) {
-            LOG(1) << "Skipping cache invalidation in opObserver because of active user command";
-            return boost::none;
-        }
-
+    void _invalidateRelevantCacheData() {
         if (_nss == AuthorizationManager::rolesCollectionNamespace ||
             _nss == AuthorizationManager::versionCollectionNamespace) {
-            return _authzManager->invalidateUserCacheNoPin(_opCtx);
+            _authzManager->invalidateUserCache(_opCtx);
+            return;
         }
 
         if (_op == "i" || _op == "d" || _op == "u") {
@@ -639,11 +625,12 @@ private:
                 warning() << "Invalidating user cache based on user being updated failed, will "
                              "invalidate the entire cache instead: "
                           << userName.getStatus();
-                return _authzManager->invalidateUserCacheNoPin(_opCtx);
+                _authzManager->invalidateUserCache(_opCtx);
+                return;
             }
-            return _authzManager->invalidateUserByNameNoPin(_opCtx, userName.getValue());
+            _authzManager->invalidateUserByName(_opCtx, userName.getValue());
         } else {
-            return _authzManager->invalidateUserCacheNoPin(_opCtx);
+            _authzManager->invalidateUserCache(_opCtx);
         }
     }
 
@@ -655,8 +642,6 @@ private:
     const NamespaceString _nss;
     const BSONObj _o;
     const boost::optional<BSONObj> _o2;
-
-    boost::optional<std::vector<UserHandle>> _refreshedPinnedUsers;
 };
 
 void AuthzManagerExternalStateLocal::logOp(OperationContext* opCtx,
@@ -681,8 +666,4 @@ void AuthzManagerExternalStateLocal::logOp(OperationContext* opCtx,
     }
 }
 
-void AuthzManagerExternalStateLocal::setInUserManagementCommand(OperationContext* opCtx, bool val) {
-    inUserManagementCommandsFlag(opCtx) = val;
-}
-
 }  // namespace mongo
diff --git a/src/mongo/db/auth/authz_manager_external_state_local.h b/src/mongo/db/auth/authz_manager_external_state_local.h
index b3dd05cefb3..5ddb737b4f5 100644
--- a/src/mongo/db/auth/authz_manager_external_state_local.h
+++ b/src/mongo/db/auth/authz_manager_external_state_local.h
@@ -110,9 +110,6 @@ public:
                const BSONObj& o,
                const BSONObj* o2) final;
 
-
-    void setInUserManagementCommand(OperationContext* opCtx, bool val) final;
-
     /**
      * Takes a user document, and processes it with the RoleGraph, in order to recursively
      * resolve roles and add the 'inheritedRoles', 'inheritedPrivileges',
diff --git a/src/mongo/db/commands/user_management_commands.cpp b/src/mongo/db/commands/user_management_commands.cpp
index ee93eafdf49..623cffb0367 100644
--- a/src/mongo/db/commands/user_management_commands.cpp
+++ b/src/mongo/db/commands/user_management_commands.cpp
@@ -562,31 +562,41 @@ Status writeAuthSchemaVersionIfNeeded(OperationContext* opCtx,
     return status;
 }
 
+auto getUMCMutex = ServiceContext::declareDecoration<stdx::mutex>();
+
 class AuthzLockGuard {
     AuthzLockGuard(AuthzLockGuard&) = delete;
     AuthzLockGuard& operator=(AuthzLockGuard&) = delete;
 
 public:
-    AuthzLockGuard(OperationContext* opCtx, LockMode mode)
+    enum InvalidationMode { kInvalidate, kReadOnly };
+    AuthzLockGuard(OperationContext* opCtx, InvalidationMode mode)
         : _opCtx(opCtx),
-          _lock(_opCtx,
-                AuthorizationManager::usersCollectionNamespace.db(),
-                mode,
-                _opCtx->getDeadline()) {
-        auto authzMgr = AuthorizationManager::get(_opCtx->getServiceContext());
-        authzMgr->setInUserManagementCommand(_opCtx, true);
-    }
+          _authzManager(AuthorizationManager::get(_opCtx->getServiceContext())),
+          _lock(getUMCMutex(opCtx->getServiceContext())),
+          _mode(mode),
+          _cacheGeneration(_authzManager->getCacheGeneration()) {}
 
     ~AuthzLockGuard() {
-        auto authzMgr = AuthorizationManager::get(_opCtx->getServiceContext());
-        authzMgr->setInUserManagementCommand(_opCtx, false);
+        if (!_lock.owns_lock() || _mode == kReadOnly) {
+            return;
+        }
+
+        if (_authzManager->getCacheGeneration() == _cacheGeneration) {
+            LOG(1) << "User management command did not invalidate the user cache.";
+            _authzManager->invalidateUserCache(_opCtx);
+        }
     }
 
     AuthzLockGuard(AuthzLockGuard&&) = default;
+    AuthzLockGuard& operator=(AuthzLockGuard&&) = default;
 
 private:
     OperationContext* _opCtx;
-    Lock::DBLock _lock;
+    AuthorizationManager* _authzManager;
+    stdx::unique_lock<stdx::mutex> _lock;
+    InvalidationMode _mode;
+    OID _cacheGeneration;
 };
 
 /**
@@ -600,7 +610,7 @@ StatusWith<AuthzLockGuard> requireWritableAuthSchema28SCRAM(OperationContext* op
     // We take a MODE_X lock during writes because we want to be sure that we can read any pinned
     // user documents back out of the database after writing them during the user management
     // commands, and to ensure only one user management command is running at a time.
-    AuthzLockGuard lk(opCtx, MODE_X);
+    AuthzLockGuard lk(opCtx, AuthzLockGuard::kInvalidate);
     Status status = authzManager->getAuthorizationVersion(opCtx, &foundSchemaVersion);
     if (!status.isOK()) {
         return status;
@@ -638,7 +648,7 @@ StatusWith<AuthzLockGuard> requireWritableAuthSchema28SCRAM(OperationContext* op
 StatusWith<AuthzLockGuard> requireReadableAuthSchema26Upgrade(OperationContext* opCtx,
                                                               AuthorizationManager* authzManager) {
     int foundSchemaVersion;
-    AuthzLockGuard lk(opCtx, MODE_IS);
+    AuthzLockGuard lk(opCtx, AuthzLockGuard::kReadOnly);
     Status status = authzManager->getAuthorizationVersion(opCtx, &foundSchemaVersion);
     if (!status.isOK()) {
         return status;
diff --git a/src/mongo/embedded/embedded_auth_manager.cpp b/src/mongo/embedded/embedded_auth_manager.cpp
index dc3a3ffd1eb..4701be9c91a 100644
--- a/src/mongo/embedded/embedded_auth_manager.cpp
+++ b/src/mongo/embedded/embedded_auth_manager.cpp
@@ -139,7 +139,7 @@ public:
         UASSERT_NOT_IMPLEMENTED;
     }
 
-    void setInUserManagementCommand(OperationContext*, bool) override {
+    void updatePinnedUsersList(std::vector<UserName>) override {
         UASSERT_NOT_IMPLEMENTED;
     }