SERVER-11980 Only invalidate the user cache on mongos if authorization information actually changed

(cherry picked from commit a51c7bcac0b93ea0a0da73974bac6c469075864d)

Signed-off-by: Spencer T Brody <spencer@mongodb.com>

6 files changed, 171 insertions, 40 deletions

diff --git a/jstests/auth/mongos_cache_invalidation.js b/jstests/auth/mongos_cache_invalidation.js
index a392975d645..19be18fdcbb 100644
--- a/jstests/auth/mongos_cache_invalidation.js
+++ b/jstests/auth/mongos_cache_invalidation.js
@@ -12,11 +12,11 @@ var hasAuthzError = function (result) {
 var st = new ShardingTest({ shards: 2,
                             config: 3,
                             mongos: [{},
-                                     {setParameter: "userCacheInvalidationIntervalSecs=30"},
+                                     {setParameter: "userCacheInvalidationIntervalSecs=5"},
                                      {}],
                             keyFile: 'jstests/libs/key1' });
 
-var res = st.s1.getDB('admin').runCommand({setParameter: 1, userCacheInvalidationIntervalSecs: 29});
+var res = st.s1.getDB('admin').runCommand({setParameter: 1, userCacheInvalidationIntervalSecs: 0});
 assert.commandFailed(res, "Setting the invalidation interval to an disallowed value should fail");
 
 res = st.s1.getDB('admin').runCommand({setParameter: 1, userCacheInvalidationIntervalSecs: 100000});
@@ -24,7 +24,7 @@ assert.commandFailed(res, "Setting the invalidation interval to an disallowed va
 
 res = st.s1.getDB('admin').runCommand({getParameter: 1, userCacheInvalidationIntervalSecs: 1});
 
-assert.eq(30, res.userCacheInvalidationIntervalSecs);
+assert.eq(5, res.userCacheInvalidationIntervalSecs);
 st.s0.getDB('test').foo.insert({a:1}); // initial data
 
 st.s0.getDB('admin').createUser({user: 'admin', pwd: 'pwd', roles: ['userAdminAnyDatabase']});
@@ -50,11 +50,9 @@ db3.auth('spencer', 'pwd');
 /**
  * At this point we have 3 handles to the "test" database, each of which are on connections to
  * different mongoses.  "db1", "db2", and "db3" are all auth'd as spencer@test and will be used
- * to verify that user and role data changes get propaged to their mongoses. "db1" is *additionally*
- * auth'd as a user with the "userAdminAnyDatabase" role.  This is the mongos that will be used to
- * modify user and role data.
- * "db2" is connected to a mongos with a 30 second user cache invalidation interval,
- * while "db3" is connected to a mongos with the default 10 minute cache invalidation interval.
+ * to verify that user and role data changes get propaged to their mongoses.
+ * "db2" is connected to a mongos with a 5 second user cache invalidation interval,
+ * while "db3" is connected to a mongos with the default 30 second cache invalidation interval.
  */
 
 (function testGrantingPrivileges() {
@@ -76,7 +74,7 @@ db3.auth('spencer', 'pwd');
      assert.writeOK(db1.foo.update({}, { $inc: { a: 1 }}));
      assert.eq(2, db1.foo.findOne().a);
 
-     // s1/db2 should update its cache in 30 seconds.
+     // s1/db2 should update its cache in 5 seconds.
      assert.soon(function() {
                      var res = db2.foo.update({}, { $inc: { a: 1 }});
                      if (res.hasWriteError()) {
@@ -84,8 +82,8 @@ db3.auth('spencer', 'pwd');
                      }
                      return db2.foo.findOne().a == 3;
                  },
-                 "Mongos did not update its user cache after 30 seconds",
-                 31 * 1000); // Give an extra 1 second to avoid races
+                 "Mongos did not update its user cache after 5 seconds",
+                 6 * 1000); // Give an extra 1 second to avoid races
 
      // We manually invalidate the cache on s2/db3.
      db3.adminCommand("invalidateUserCache");
@@ -104,13 +102,13 @@ db3.auth('spencer', 'pwd');
      // s0/db1 should update its cache instantly
      hasAuthzError(db1.foo.update({}, { $inc: { a: 1 }}));
 
-     // s1/db2 should update its cache in 30 seconds.
+     // s1/db2 should update its cache in 5 seconds.
      assert.soon(function() {
                      var res = db2.foo.update({}, { $inc: { a: 1 }});
                      return res.hasWriteError() && res.getWriteError().code == authzErrorCode;
                  },
-                 "Mongos did not update its user cache after 30 seconds",
-                 31 * 1000); // Give an extra 1 second to avoid races
+                 "Mongos did not update its user cache after 5 seconds",
+                 6 * 1000); // Give an extra 1 second to avoid races
 
      // We manually invalidate the cache on s1/db3.
      db3.adminCommand("invalidateUserCache");
@@ -129,12 +127,12 @@ db3.auth('spencer', 'pwd');
      // s0/db1 should update its cache instantly
      assert.writeOK(db1.foo.update({}, { $inc: { a: 1 }}));
 
-     // s1/db2 should update its cache in 30 seconds.
+     // s1/db2 should update its cache in 5 seconds.
      assert.soon(function() {
                      return !db2.foo.update({}, { $inc: { a: 1 }}).hasWriteError();
                  },
-                 "Mongos did not update its user cache after 30 seconds",
-                 31 * 1000); // Give an extra 1 second to avoid races
+                 "Mongos did not update its user cache after 5 seconds",
+                 6 * 1000); // Give an extra 1 second to avoid races
 
      // We manually invalidate the cache on s1/db3.
      db3.adminCommand("invalidateUserCache");
@@ -179,12 +177,12 @@ db3.auth('spencer', 'pwd');
      // s0/db1 should update its cache instantly
      assert.commandFailedWithCode(db1.foo.runCommand("collStats"), authzErrorCode);
 
-     // s1/db2 should update its cache in 30 seconds.
+     // s1/db2 should update its cache in 5 seconds.
      assert.soon(function() {
                      return db2.foo.runCommand("collStats").code == authzErrorCode;
                  },
-                 "Mongos did not update its user cache after 30 seconds",
-                 31 * 1000); // Give an extra 1 second to avoid races
+                 "Mongos did not update its user cache after 5 seconds",
+                 6 * 1000); // Give an extra 1 second to avoid races
 
      // We manually invalidate the cache on s2/db3.
      db3.adminCommand("invalidateUserCache");
diff --git a/src/mongo/db/auth/authorization_manager.cpp b/src/mongo/db/auth/authorization_manager.cpp
index 70ea803579c..d85b4841804 100644
--- a/src/mongo/db/auth/authorization_manager.cpp
+++ b/src/mongo/db/auth/authorization_manager.cpp
@@ -238,18 +238,18 @@ namespace mongo {
             _authzManager->_isFetchPhaseBusy = true;
         }
 
-        uint64_t _startGeneration;
+        OID _startGeneration;
         bool _isThisGuardInFetchPhase;
         AuthorizationManager* _authzManager;
         boost::unique_lock<boost::mutex> _lock;
     };
 
     AuthorizationManager::AuthorizationManager(AuthzManagerExternalState* externalState) :
-        _authEnabled(false),
-        _externalState(externalState),
-        _version(schemaVersionInvalid),
-        _cacheGeneration(0),
-        _isFetchPhaseBusy(false) {
+            _authEnabled(false),
+            _externalState(externalState),
+            _version(schemaVersionInvalid),
+            _isFetchPhaseBusy(false) {
+        _updateCacheGeneration_inlock();
     }
 
     AuthorizationManager::~AuthorizationManager() {
@@ -292,6 +292,11 @@ namespace mongo {
         return _doesSupportOldStylePrivileges;
     }
 
+    OID AuthorizationManager::getCacheGeneration() {
+        CacheGuard guard(this, CacheGuard::fetchSynchronizationManual);
+        return _cacheGeneration;
+    }
+
     void AuthorizationManager::setAuthEnabled(bool enabled) {
         _authEnabled = enabled;
     }
@@ -844,7 +849,7 @@ namespace mongo {
 
     void AuthorizationManager::invalidateUserByName(const UserName& userName) {
         CacheGuard guard(this, CacheGuard::fetchSynchronizationManual);
-        ++_cacheGeneration;
+        _updateCacheGeneration_inlock();
         unordered_map<UserName, User*>::iterator it = _userCache.find(userName);
         if (it == _userCache.end()) {
             return;
@@ -857,7 +862,7 @@ namespace mongo {
 
     void AuthorizationManager::invalidateUsersFromDB(const std::string& dbname) {
         CacheGuard guard(this, CacheGuard::fetchSynchronizationManual);
-        ++_cacheGeneration;
+        _updateCacheGeneration_inlock();
         unordered_map<UserName, User*>::iterator it = _userCache.begin();
         while (it != _userCache.end()) {
             User* user = it->second;
@@ -876,7 +881,7 @@ namespace mongo {
     }
 
     void AuthorizationManager::_invalidateUserCache_inlock() {
-        ++_cacheGeneration;
+        _updateCacheGeneration_inlock();
         for (unordered_map<UserName, User*>::iterator it = _userCache.begin();
                 it != _userCache.end(); ++it) {
             fassert(17266, it->second != internalSecurity.user);
@@ -1407,8 +1412,13 @@ namespace {
             return true;
         }
     }
+
 }  // namespace
 
+    void AuthorizationManager::_updateCacheGeneration_inlock() {
+        _cacheGeneration = OID::gen();
+    }
+
     void AuthorizationManager::logOp(
             const char* op,
             const char* ns,
diff --git a/src/mongo/db/auth/authorization_manager.h b/src/mongo/db/auth/authorization_manager.h
index 3ad54dd6106..8b6a078dd8f 100644
--- a/src/mongo/db/auth/authorization_manager.h
+++ b/src/mongo/db/auth/authorization_manager.h
@@ -38,6 +38,7 @@
 #include "mongo/base/disallow_copying.h"
 #include "mongo/base/status.h"
 #include "mongo/bson/mutable/element.h"
+#include "mongo/bson/oid.h"
 #include "mongo/db/auth/action_set.h"
 #include "mongo/db/auth/resource_pattern.h"
 #include "mongo/db/auth/role_graph.h"
@@ -176,6 +177,11 @@ namespace mongo {
          */
         Status getAuthorizationVersion(int* version);
 
+        /**
+         * Returns the user cache generation identifier.
+         */
+        OID getCacheGeneration();
+
         // Returns true if there exists at least one privilege document in the system.
         bool hasAnyPrivilegeDocuments() const;
 
@@ -445,6 +451,11 @@ namespace mongo {
         void _invalidateUserCache_inlock();
 
         /**
+         * Updates _cacheGeneration to a new OID
+         */
+        void _updateCacheGeneration_inlock();
+
+        /**
          * Fetches user information from a v2-schema user document for the named user,
          * and stores a pointer to a new user object into *acquiredUser on success.
          */
@@ -488,10 +499,10 @@ namespace mongo {
         unordered_map<UserName, User*> _userCache;
 
         /**
-         * Current generation of cached data.  Bumped every time part of the cache gets
-         * invalidated.
+         * Current generation of cached data.  Updated every time part of the cache gets
+         * invalidated.  Protected by CacheGuard.
          */
-        uint64_t _cacheGeneration;
+        OID _cacheGeneration;
 
         /**
          * True if there is an update to the _userCache in progress, and that update is currently in
diff --git a/src/mongo/db/auth/user_cache_invalidator_job.cpp b/src/mongo/db/auth/user_cache_invalidator_job.cpp
index 5eeada66ea5..209a143b65e 100644
--- a/src/mongo/db/auth/user_cache_invalidator_job.cpp
+++ b/src/mongo/db/auth/user_cache_invalidator_job.cpp
@@ -19,16 +19,21 @@
 
 #include <string>
 
+#include "mongo/base/status.h"
+#include "mongo/base/status_with.h"
 #include "mongo/db/auth/authorization_manager.h"
 #include "mongo/db/client.h"
+#include "mongo/db/commands.h"
 #include "mongo/db/server_parameters.h"
+#include "mongo/s/config.h"
 #include "mongo/util/background.h"
 #include "mongo/util/log.h"
 
 namespace mongo {
 namespace {
 
-    int userCacheInvalidationIntervalSecs = 60 * 10; // 10 minute default
+    // How often to check with the config servers whether authorization information has changed.
+    int userCacheInvalidationIntervalSecs = 30; // 30 second default
 
     class ExportedInvalidationIntervalParameter : public ExportedServerParameter<int> {
     public:
@@ -41,26 +46,75 @@ namespace {
 
         virtual Status validate( const int& potentialNewValue )
         {
-            if (potentialNewValue < 30 || potentialNewValue > 86400) {
+            if (potentialNewValue < 1 || potentialNewValue > 86400) {
                 return Status(ErrorCodes::BadValue,
-                              "userCacheInvalidationIntervalSecs must be between 30 " 
+                              "userCacheInvalidationIntervalSecs must be between 1 "
                               "and 86400 (24 hours)");
             }
             return Status::OK();
         }
     } exportedIntervalParam;
 
+    StatusWith<OID> getCurrentCacheGeneration() {
+        try {
+            ConnectionString config = configServer.getConnectionString();
+            ScopedDbConnection conn(config.toString(), 30);
+
+            BSONObj result;
+            conn->runCommand("admin", BSON("_getUserCacheGeneration" << 1), result);
+            conn.done();
+
+            Status status = Command::getStatusFromCommandResult(result);
+            if (!status.isOK()) {
+                return StatusWith<OID>(status);
+            }
+
+            return StatusWith<OID>(result["cacheGeneration"].OID());
+        } catch (const DBException& e) {
+            return StatusWith<OID>(e.toStatus());
+        } catch (const std::exception& e) {
+            return StatusWith<OID>(ErrorCodes::UnknownError, e.what());
+        }
+    }
+
 } // namespace
 
+    UserCacheInvalidator::UserCacheInvalidator(AuthorizationManager* authzManager) :
+            _authzManager(authzManager) {
+        _previousCacheGeneration = _authzManager->getCacheGeneration();
+    }
+
     void UserCacheInvalidator::run() {
         Client::initThread("UserCacheInvalidatorThread");
+
         while (true) {
             sleepsecs(userCacheInvalidationIntervalSecs);
             if (inShutdown()) {
                 break;
             }
-            LOG(1) << "Invalidating user cache" << endl;
-            _authzManager->invalidateUserCache();
+
+            StatusWith<OID> currentGeneration = getCurrentCacheGeneration();
+            if (!currentGeneration.isOK()) {
+                if (currentGeneration.getStatus().code() == ErrorCodes::CommandNotFound) {
+                    warning() << "_getUserCacheGeneration command not found on config server(s), "
+                            "this most likely means you are running an outdated version of mongod "
+                            "on the config servers" << std::endl;
+                } else {
+                    warning() << "An error occurred while fetching current user cache generation "
+                            "to check if user cache needs invalidation: " <<
+                            currentGeneration.getStatus() << std::endl;
+                }
+                // When in doubt, invalidate the cache
+                _authzManager->invalidateUserCache();
+            }
+
+            if (currentGeneration.getValue() != _previousCacheGeneration) {
+                log() << "User cache generation changed from " << _previousCacheGeneration <<
+                        " to " << currentGeneration.getValue() << "; invalidating user cache" <<
+                        std::endl;
+                _authzManager->invalidateUserCache();
+                _previousCacheGeneration = currentGeneration.getValue();
+            }
         }
     }
 
diff --git a/src/mongo/db/auth/user_cache_invalidator_job.h b/src/mongo/db/auth/user_cache_invalidator_job.h
index 23d2367d3d6..5dc9403272d 100644
--- a/src/mongo/db/auth/user_cache_invalidator_job.h
+++ b/src/mongo/db/auth/user_cache_invalidator_job.h
@@ -13,6 +13,7 @@
  *    limitations under the License.
  */
 
+#include "mongo/bson/oid.h"
 #include "mongo/util/background.h"
 
 #include <string>
@@ -21,11 +22,15 @@ namespace mongo {
 
     class AuthorizationManager;
 
-    // Background job that periodically causes the AuthorizationManager to throw out its in-memory
-    // cache of User objects (which contains the users' credentials, roles, privileges, etc).
+    /**
+     * Background job that runs only in mongos and periodically checks in with the config servers
+     * to determine whether any authorization information has changed, and if so causes the
+     * AuthorizationManager to throw out its in-memory cache of User objects (which contains the
+     * users' credentials, roles, privileges, etc).
+     */
     class UserCacheInvalidator : public BackgroundJob {
     public:
-        UserCacheInvalidator(AuthorizationManager* authzManager) : _authzManager(authzManager) {}
+        explicit UserCacheInvalidator(AuthorizationManager* authzManager);
 
     protected:
         virtual std::string name() const;
@@ -33,6 +38,7 @@ namespace mongo {
 
     private:
         AuthorizationManager* _authzManager;
+        OID _previousCacheGeneration;
     };
 
 } // namespace mongo
diff --git a/src/mongo/db/commands/user_management_commands.cpp b/src/mongo/db/commands/user_management_commands.cpp
index 78e08565d62..61cb9f7326f 100644
--- a/src/mongo/db/commands/user_management_commands.cpp
+++ b/src/mongo/db/commands/user_management_commands.cpp
@@ -2437,6 +2437,12 @@ namespace mongo {
             return NONE;
         }
 
+        virtual bool adminOnly() const {
+            return true;
+        }
+
+        virtual bool isWriteCommandForConfigServer() const { return false; }
+
         CmdInvalidateUserCache() : Command("invalidateUserCache") {}
 
         virtual void help(stringstream& ss) const {
@@ -2468,6 +2474,52 @@ namespace mongo {
 
     } cmdInvalidateUserCache;
 
+    class CmdGetCacheGeneration: public Command {
+    public:
+
+        virtual bool slaveOk() const {
+            return true;
+        }
+
+        virtual bool adminOnly() const {
+            return true;
+        }
+
+        virtual LockType locktype() const { return NONE; }
+
+        virtual bool isWriteCommandForConfigServer() const { return false; }
+
+        CmdGetCacheGeneration() : Command("_getUserCacheGeneration") {}
+
+        virtual void help(stringstream& ss) const {
+            ss << "internal" << endl;
+        }
+
+        virtual Status checkAuthForCommand(ClientBasic* client,
+                                           const std::string& dbname,
+                                           const BSONObj& cmdObj) {
+            AuthorizationSession* authzSession = client->getAuthorizationSession();
+            if (!authzSession->isAuthorizedForActionsOnResource(
+                    ResourcePattern::forClusterResource(), ActionType::internal)) {
+                return Status(ErrorCodes::Unauthorized, "Not authorized to get cache generation");
+            }
+            return Status::OK();
+        }
+
+        bool run(const string& dbname,
+                 BSONObj& cmdObj,
+                 int options,
+                 string& errmsg,
+                 BSONObjBuilder& result,
+                 bool fromRepl) {
+
+            AuthorizationManager* authzManager = getGlobalAuthorizationManager();
+            result.append("cacheGeneration", authzManager->getCacheGeneration());
+            return true;
+        }
+
+    } CmdGetCacheGeneration;
+
     /**
      * This command is used only by mongorestore to handle restoring users/roles.  We do this so
      * that mongorestore doesn't do direct inserts into the admin.system.users and