SERVER-10596 Globalize formerly per-thread Pool of JS Scopes

This ensures that the limit of 10 pooled scopes is actually enforced.
With a per-thread Pool, long-lived connections could cause very high
memory usage (both real and virtual) even if they haven't used JS in a
long time.

15 files changed, 98 insertions, 161 deletions

diff --git a/src/mongo/db/auth/authorization_session.cpp b/src/mongo/db/auth/authorization_session.cpp
index 656d6b148e7..f3728318c93 100644
--- a/src/mongo/db/auth/authorization_session.cpp
+++ b/src/mongo/db/auth/authorization_session.cpp
@@ -72,7 +72,6 @@ namespace {
             getAuthorizationManager().releaseUser(replacedUser);
         }
 
-        _externalState->onAddAuthorizedUser();
         return Status::OK();
     }
 
@@ -85,14 +84,24 @@ namespace {
         if (removedUser) {
             getAuthorizationManager().releaseUser(removedUser);
         }
-
-        _externalState->onLogoutDatabase(dbname);
     }
 
     UserSet::NameIterator AuthorizationSession::getAuthenticatedUserNames() {
         return _authenticatedUsers.getNames();
     }
 
+    std::string AuthorizationSession::getAuthenticatedUserNamesToken() {
+        std::string ret;
+        for (UserSet::NameIterator nameIter = getAuthenticatedUserNames();
+                nameIter.more();
+                nameIter.next()) {
+            ret += '\0'; // Using a NUL byte which isn't valid in usernames to separate them.
+            ret += nameIter->getFullName();
+        }
+
+        return ret;
+    }
+
     void AuthorizationSession::grantInternalAuthorization() {
         _authenticatedUsers.add(internalSecurity.user);
     }
diff --git a/src/mongo/db/auth/authorization_session.h b/src/mongo/db/auth/authorization_session.h
index b8f587b8091..f4844fae471 100644
--- a/src/mongo/db/auth/authorization_session.h
+++ b/src/mongo/db/auth/authorization_session.h
@@ -68,6 +68,10 @@ namespace mongo {
         // Gets an iterator over the names of all authenticated users stored in this manager.
         UserSet::NameIterator getAuthenticatedUserNames();
 
+        // Returns a string representing all logged-in users on the current session.
+        // WARNING: this string will contain NUL bytes so don't call c_str()!
+        std::string getAuthenticatedUserNamesToken();
+
         // Removes any authenticated principals whose authorization credentials came from the given
         // database, and revokes any privileges that were granted via that principal.
         void logoutDatabase(const std::string& dbname);
diff --git a/src/mongo/db/auth/authz_session_external_state.h b/src/mongo/db/auth/authz_session_external_state.h
index 063181615f0..4b17edef12f 100644
--- a/src/mongo/db/auth/authz_session_external_state.h
+++ b/src/mongo/db/auth/authz_session_external_state.h
@@ -54,14 +54,6 @@ namespace mongo {
         // necessary to determine if localhost connections should be given full access.
         virtual void startRequest() = 0;
 
-        // Authorization event hooks
-
-        // Handle any global state which needs to be updated when a new user has been authorized
-        virtual void onAddAuthorizedUser() = 0;
-
-        // Handle any global state which needs to be updated when a user logs out
-        virtual void onLogoutDatabase(const std::string& dbname) = 0;
-
     protected:
         // This class should never be instantiated directly.
         AuthzSessionExternalState(AuthorizationManager* authzManager);
diff --git a/src/mongo/db/auth/authz_session_external_state_d.cpp b/src/mongo/db/auth/authz_session_external_state_d.cpp
index 1736e827bc2..021a47376cc 100644
--- a/src/mongo/db/auth/authz_session_external_state_d.cpp
+++ b/src/mongo/db/auth/authz_session_external_state_d.cpp
@@ -43,16 +43,4 @@ namespace mongo {
         return cc().isGod() || AuthzSessionExternalStateServerCommon::shouldIgnoreAuthChecks();
     }
 
-    void AuthzSessionExternalStateMongod::onAddAuthorizedUser() {
-        // invalidate all thread-local JS scopes due to new user authentication
-        if (globalScriptEngine)
-            globalScriptEngine->threadDone();
-    }
-
-    void AuthzSessionExternalStateMongod::onLogoutDatabase(const std::string&) {
-        // invalidate all thread-local JS scopes due to logout
-        if (globalScriptEngine)
-            globalScriptEngine->threadDone();
-    }
-
 } // namespace mongo
diff --git a/src/mongo/db/auth/authz_session_external_state_d.h b/src/mongo/db/auth/authz_session_external_state_d.h
index 9cdae29bc1f..91f94fe2cd5 100644
--- a/src/mongo/db/auth/authz_session_external_state_d.h
+++ b/src/mongo/db/auth/authz_session_external_state_d.h
@@ -36,11 +36,6 @@ namespace mongo {
         virtual bool shouldIgnoreAuthChecks() const;
 
         virtual void startRequest();
-
-        virtual void onAddAuthorizedUser();
-
-        virtual void onLogoutDatabase(const std::string&);
-
     };
 
 } // namespace mongo
diff --git a/src/mongo/db/auth/authz_session_external_state_mock.h b/src/mongo/db/auth/authz_session_external_state_mock.h
index 32b76db5bd1..969ce74ed71 100644
--- a/src/mongo/db/auth/authz_session_external_state_mock.h
+++ b/src/mongo/db/auth/authz_session_external_state_mock.h
@@ -49,10 +49,6 @@ namespace mongo {
 
         virtual void startRequest() {}
 
-        virtual void onAddAuthorizedUser() {}
-
-        virtual void onLogoutDatabase(const std::string& dbname) {}
-
     private:
         bool _returnValue;
     };
diff --git a/src/mongo/db/auth/authz_session_external_state_s.cpp b/src/mongo/db/auth/authz_session_external_state_s.cpp
index a7c8bb7cbd4..3e155aa4562 100644
--- a/src/mongo/db/auth/authz_session_external_state_s.cpp
+++ b/src/mongo/db/auth/authz_session_external_state_s.cpp
@@ -31,10 +31,6 @@ namespace mongo {
                     AuthzSessionExternalStateServerCommon(authzManager) {}
     AuthzSessionExternalStateMongos::~AuthzSessionExternalStateMongos() {}
 
-    void AuthzSessionExternalStateMongos::onAddAuthorizedUser() { }
-
-    void AuthzSessionExternalStateMongos::onLogoutDatabase(const std::string&) { }
-
     void AuthzSessionExternalStateMongos::startRequest() {
         _checkShouldAllowLocalhost();
     }
diff --git a/src/mongo/db/auth/authz_session_external_state_s.h b/src/mongo/db/auth/authz_session_external_state_s.h
index c91d2826438..ecb6ad4b291 100644
--- a/src/mongo/db/auth/authz_session_external_state_s.h
+++ b/src/mongo/db/auth/authz_session_external_state_s.h
@@ -34,11 +34,6 @@ namespace mongo {
         virtual ~AuthzSessionExternalStateMongos();
 
         virtual void startRequest();
-
-        virtual void onAddAuthorizedUser();
-
-        virtual void onLogoutDatabase(const std::string&);
-
     };
 
 } // namespace mongo
diff --git a/src/mongo/db/commands/group.cpp b/src/mongo/db/commands/group.cpp
index 979fcd4f9aa..f3c5025b6df 100644
--- a/src/mongo/db/commands/group.cpp
+++ b/src/mongo/db/commands/group.cpp
@@ -20,9 +20,11 @@
 
 #include <vector>
 
+#include "mongo/db/auth/authorization_session.h"
 #include "mongo/db/auth/action_set.h"
 #include "mongo/db/auth/action_type.h"
 #include "mongo/db/auth/privilege.h"
+#include "mongo/db/client_basic.h"
 #include "mongo/db/clientcursor.h"
 #include "mongo/db/commands.h"
 #include "mongo/db/instance.h"
@@ -73,7 +75,9 @@ namespace mongo {
                     string& errmsg,
                     BSONObjBuilder& result ) {
 
-            auto_ptr<Scope> s = globalScriptEngine->getPooledScope( realdbname, "group");
+            const string userToken = ClientBasic::getCurrent()->getAuthorizationSession()
+                                                              ->getAuthenticatedUserNamesToken();
+            auto_ptr<Scope> s = globalScriptEngine->getPooledScope(realdbname, "group" + userToken);
 
             if ( reduceScope )
                 s->init( reduceScope );
diff --git a/src/mongo/db/commands/mr.cpp b/src/mongo/db/commands/mr.cpp
index 3e66c57e04d..a1247360cf3 100644
--- a/src/mongo/db/commands/mr.cpp
+++ b/src/mongo/db/commands/mr.cpp
@@ -22,6 +22,7 @@
 
 #include "mongo/client/connpool.h"
 #include "mongo/client/parallel.h"
+#include "mongo/db/auth/authorization_session.h"
 #include "mongo/db/clientcursor.h"
 #include "mongo/db/commands.h"
 #include "mongo/db/db.h"
@@ -628,7 +629,10 @@ namespace mongo {
          */
         void State::init() {
             // setup js
-            _scope.reset(globalScriptEngine->getPooledScope( _config.dbname, "mapreduce" ).release() );
+            const string userToken = ClientBasic::getCurrent()->getAuthorizationSession()
+                                                              ->getAuthenticatedUserNamesToken();
+            _scope.reset(globalScriptEngine->getPooledScope(
+                            _config.dbname, "mapreduce" + userToken).release());
 
             if ( ! _config.scopeSetup.isEmpty() )
                 _scope->init( &_config.scopeSetup );
diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp
index 4f081e64ae1..3043125399f 100644
--- a/src/mongo/db/db.cpp
+++ b/src/mongo/db/db.cpp
@@ -242,7 +242,6 @@ namespace mongo {
         virtual void disconnected( AbstractMessagingPort* p ) {
             Client * c = currentClient.get();
             if( c ) c->shutdown();
-            globalScriptEngine->threadDone();
         }
 
     };
diff --git a/src/mongo/db/dbeval.cpp b/src/mongo/db/dbeval.cpp
index 94b80c796f3..f619c6adaf9 100644
--- a/src/mongo/db/dbeval.cpp
+++ b/src/mongo/db/dbeval.cpp
@@ -60,7 +60,9 @@ namespace mongo {
             return false;
         }
 
-        auto_ptr<Scope> s = globalScriptEngine->getPooledScope( dbName, "dbeval" );
+        const string userToken = ClientBasic::getCurrent()->getAuthorizationSession()
+                                                          ->getAuthenticatedUserNamesToken();
+        auto_ptr<Scope> s = globalScriptEngine->getPooledScope( dbName, "dbeval" + userToken );
         ScriptingFunction f = s->createFunction(code);
         if ( f == 0 ) {
             errmsg = (string)"compile failed: " + s->getError();
diff --git a/src/mongo/db/matcher/expression_where.cpp b/src/mongo/db/matcher/expression_where.cpp
index 4ebf5343dc4..1e784ec8fb7 100644
--- a/src/mongo/db/matcher/expression_where.cpp
+++ b/src/mongo/db/matcher/expression_where.cpp
@@ -18,6 +18,7 @@
 
 #include "mongo/pch.h"
 #include "mongo/base/init.h"
+#include "mongo/db/auth/authorization_session.h"
 #include "mongo/db/namespace_string.h"
 #include "mongo/db/client.h"
 #include "mongo/db/jsobj.h"
@@ -67,7 +68,10 @@ namespace mongo {
         _userScope = scope.getOwned();
 
         NamespaceString nswrapper( _ns );
-        _scope = globalScriptEngine->getPooledScope( nswrapper.db().toString(), "where" );
+        const string userToken = ClientBasic::getCurrent()->getAuthorizationSession()
+                                                          ->getAuthenticatedUserNamesToken();
+        _scope = globalScriptEngine->getPooledScope( nswrapper.db().toString(),
+                                                     "where" + userToken );
         _func = _scope->createFunction( _code.c_str() );
 
         if ( !_func )
diff --git a/src/mongo/scripting/engine.cpp b/src/mongo/scripting/engine.cpp
index 24e9d3b5ae7..d0f537dfa36 100644
--- a/src/mongo/scripting/engine.cpp
+++ b/src/mongo/scripting/engine.cpp
@@ -41,7 +41,7 @@ namespace mongo {
 
     Scope::Scope() : _localDBName(""),
                      _loadedVersion(0),
-                     _numTimeUsed(0),
+                     _numTimesUsed(0),
                      _lastRetIsNativeCode(false) {
     }
 
@@ -268,119 +268,80 @@ namespace mongo {
         injectNative("benchFinish", BenchRunner::benchFinish);
     }
 
-    typedef map<string, list<Scope*> > PoolToScopes;
-    typedef unordered_set<Scope*> ActiveScopes;
-
+namespace {
     class ScopeCache {
     public:
-        ScopeCache() : _mutex("ScopeCache") {
-        }
-
-        ~ScopeCache() {
-            if (inShutdown())
-                return;
-            clear();
-        }
+        ScopeCache() : _mutex("ScopeCache") {}
 
-        void done(const string& pool, Scope* s) {
+        void release(const string& poolName, const boost::shared_ptr<Scope>& scope) {
             scoped_lock lk(_mutex);
-            list<Scope*>& l = _pools[pool];
-            bool oom = s->hasOutOfMemoryException();
-
-            const unsigned kMaxPoolSize = 10;
-            const int kMaxScopeReuse = 10;
-
-            // 'orphaned' scopes were in-use while the ScopeCache was clear()ed.  they should not
-            // be returned to any pool since authentication credentials may have changed.
-            bool orphaned = !_active.erase(s);
-
-            // do not keep too many contexts, use them for too long, or reuse after error
-            if (l.size() > kMaxPoolSize ||
-                s->getTimeUsed() > kMaxScopeReuse ||
-                !s->getError().empty() ||
-                oom ||
-                orphaned) {
-                delete s;
-            }
-            else {
-                l.push_back(s);
-                s->reset();
-            }
 
-            if (oom) {
-                // out of mem, make some room
+            if (scope->hasOutOfMemoryException()) {
+                // make some room
                 log() << "Clearing all idle JS contexts due to out of memory" << endl;
-                clear();
+                _pools.clear();
+                return;
             }
+
+            if (scope->getTimesUsed() > kMaxScopeReuse)
+                return; // used too many times to save
+
+            if (!scope->getError().empty())
+                return; // not saving errored scopes
+
+            if (_pools.size() >= kMaxPoolSize) {
+                // prefer to keep recently-used scopes
+                _pools.pop_back();
+            }
+
+            ScopeAndPool toStore = {scope, poolName};
+            _pools.push_front(toStore);
         }
 
-        Scope* get(const string& pool) {
+        boost::shared_ptr<Scope> tryAcquire(const string& poolName) {
             scoped_lock lk(_mutex);
-            list<Scope*>& l = _pools[pool];
-            if (l.size() == 0)
-                return NULL;
-
-            Scope* s = l.back();
-            l.pop_back();
-            _active.insert(s);
-            s->reset();
-            s->incTimeUsed();
-            return s;
-        }
 
-        void clear() {
-            set<Scope*> seen;
-            for (PoolToScopes::iterator i = _pools.begin(); i != _pools.end(); ++i) {
-                for (list<Scope*>::iterator j = i->second.begin(); j != i->second.end(); ++j) {
-                    Scope* s = *j;
-                    fassert(16652, seen.insert(s).second);
-                    delete s;
+            for (Pools::iterator it = _pools.begin(); it != _pools.end(); ++it) {
+                if (it->poolName == poolName) {
+                    boost::shared_ptr<Scope> scope = it->scope;
+                    _pools.erase(it);
+                    scope->incTimesUsed();
+                    scope->reset();
+                    return scope;
                 }
             }
-            _pools.clear();
-            _active.clear();
 
-        }
-
-        /**
-         * Add a scope to the active (or in-use) set of scopes.
-         * A scope is considered active if it is not in the pool.  If the pool is cleared while an
-         * active scope is still running, the active scope will become orphaned.  When the orphaned
-         * scope calls done(), it will be freed instead of being placed back in the pool for reuse.
-         *
-         * This should only be called when ScriptEngine::getPooledScope() must create a new scope
-         * for the pool.
-         */
-        void addActive(Scope* scope) {
-            _active.insert(scope);
+            return boost::shared_ptr<Scope>();
         }
 
     private:
-        PoolToScopes _pools;    // protected by _mutex
-        ActiveScopes _active;   // protected by _mutex
+        struct ScopeAndPool {
+            boost::shared_ptr<Scope> scope;
+            string poolName;
+        };
+
+        // Note: if these numbers change, reconsider choice of datastructure for _pools
+        static const unsigned kMaxPoolSize = 10;
+        static const int kMaxScopeReuse = 10;
+
+        typedef deque<ScopeAndPool> Pools; // More-recently used Scopes are kept at the front.
+        Pools _pools;    // protected by _mutex
         mongo::mutex _mutex;
     };
 
-    thread_specific_ptr<ScopeCache> scopeCache;
+    ScopeCache scopeCache;
+} // anonymous namespace
 
     class PooledScope : public Scope {
     public:
-        PooledScope(const std::string& pool, Scope* real) : _pool(pool), _real(real) {
+        PooledScope(const std::string& pool, const boost::shared_ptr<Scope>& real)
+            : _pool(pool)
+            , _real(real) {
             _real->loadStored(true);
-        };
+        }
+
         virtual ~PooledScope() {
-            ScopeCache* sc = scopeCache.get();
-            if (sc) {
-                sc->done(_pool, _real);
-                _real = NULL;
-            }
-            else {
-                // this means that the Scope was killed from a different thread
-                // for example a cursor got timed out that has a $where clause
-                LOG(3) << "warning: scopeCache is empty!" << endl;
-                delete _real;
-                _real = 0;
-            }
+            scopeCache.release(_pool, _real);
         }
 
         // wrappers for the derived (_real) scope
@@ -442,33 +403,24 @@ namespace mongo {
 
     private:
         string _pool;
-        Scope* _real;
+        boost::shared_ptr<Scope> _real;
     };
 
     /** Get a scope from the pool of scopes matching the supplied pool name */
-    auto_ptr<Scope> ScriptEngine::getPooledScope(const string& pool, const string& scopeType) {
-        if (!scopeCache.get())
-            scopeCache.reset(new ScopeCache());
-
-        Scope* s = scopeCache->get(pool + scopeType);
+    auto_ptr<Scope> ScriptEngine::getPooledScope(const string& db, const string& scopeType) {
+        const string fullPoolName = db + scopeType;
+        boost::shared_ptr<Scope> s = scopeCache.tryAcquire(fullPoolName);
         if (!s) {
-            s = newScope();
-            scopeCache->addActive(s);
+            s.reset(newScope());
         }
 
         auto_ptr<Scope> p;
-        p.reset(new PooledScope(pool + scopeType, s));
-        p->setLocalDB(pool);
+        p.reset(new PooledScope(fullPoolName, s));
+        p->setLocalDB(db);
         p->loadStored(true);
         return p;
     }
 
-    void ScriptEngine::threadDone() {
-        ScopeCache* sc = scopeCache.get();
-        if (sc)
-            sc->clear();
-    }
-
     void (*ScriptEngine::_connectCallback)(DBClientWithCommands&) = 0;
     const char* (*ScriptEngine::_checkInterruptCallback)() = 0;
     unsigned (*ScriptEngine::_getCurrentOpIdCallback)() = 0;
diff --git a/src/mongo/scripting/engine.h b/src/mongo/scripting/engine.h
index 0f269e93498..ecbc44754bd 100644
--- a/src/mongo/scripting/engine.h
+++ b/src/mongo/scripting/engine.h
@@ -135,10 +135,10 @@ namespace mongo {
         static void validateObjectIdString(const string& str);
 
         /** increments the number of times a scope was used */
-        void incTimeUsed() { ++_numTimeUsed; }
+        void incTimesUsed() { ++_numTimesUsed; }
 
         /** gets the number of times a scope was used */
-        int getTimeUsed() { return _numTimeUsed; }
+        int getTimesUsed() { return _numTimesUsed; }
 
         /** return true if last invoke() return'd native code */
         virtual bool isLastRetNativeCode() { return _lastRetIsNativeCode; }
@@ -168,7 +168,7 @@ namespace mongo {
         set<string> _storedNames;
         static long long _lastVersion;
         FunctionCacheMap _cachedFunctions;
-        int _numTimeUsed;
+        int _numTimesUsed;
         bool _lastRetIsNativeCode; // v8 only: set to true if eval'd script returns a native func
     };
 
@@ -188,15 +188,12 @@ namespace mongo {
         static void setup();
 
         /** gets a scope from the pool or a new one if pool is empty
-         * @param pool An identifier for the pool, usually the db name
+         * @param db The db name
+         * @param scopeType A unique id to limit scope sharing.
+         *                  This must include authenticated users.
          * @return the scope
          */
-        auto_ptr<Scope> getPooledScope(const string& pool, const string& scopeType);
-
-        /**
-         * call this method to release some JS resources when a thread is done
-         */
-        void threadDone();
+        auto_ptr<Scope> getPooledScope(const string& db, const string& scopeType);
 
         void setScopeInitCallback(void (*func)(Scope&)) { _scopeInitCallback = func; }
         static void setConnectCallback(void (*func)(DBClientWithCommands&)) {