SERVER-10172 Refactor RamLog to make it thread-safe.

Also prevent accidental creation of duplicate ramlogs of the same name.

11 files changed, 189 insertions, 92 deletions

diff --git a/src/mongo/db/commands/server_status.cpp b/src/mongo/db/commands/server_status.cpp
index 689dac9aac2..2059ddeb551 100644
--- a/src/mongo/db/commands/server_status.cpp
+++ b/src/mongo/db/commands/server_status.cpp
@@ -141,20 +141,16 @@ namespace mongo {
             // --- some hard coded global things hard to pull out
 
             {
-                RamLog* rl = RamLog::get( "warnings" );
-                massert(15880, "no ram log for warnings?" , rl);
-                
-                if (rl->lastWrite() >= time(0)-(10*60)){ // only show warnings from last 10 minutes
-                    vector<const char*> lines;
-                    rl->get( lines );
-                    
-                    BSONArrayBuilder arr( result.subarrayStart( "warnings" ) );
-                    for ( unsigned i=std::max(0,(int)lines.size()-10); i<lines.size(); i++ )
-                        arr.append( lines[i] );
+                RamLog::LineIterator rl(RamLog::get("warnings"));
+                if (rl.lastWrite() >= time(0)-(10*60)){  // only show warnings from last 10 minutes
+                    BSONArrayBuilder arr(result.subarrayStart("warnings"));
+                    while (rl.more()) {
+                        arr.append(rl.next());
+                    }
                     arr.done();
                 }
             }
-            
+
             timeBuilder.appendNumber( "at end" , Listener::getElapsedTimeMillis() - start );
             if ( Listener::getElapsedTimeMillis() - start > 1000 ) {
                 BSONObj t = timeBuilder.obj();
diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp
index 448b631dec0..8bd7efb37fb 100644
--- a/src/mongo/db/db.cpp
+++ b/src/mongo/db/db.cpp
@@ -581,10 +581,6 @@ namespace mongo {
 
         Client::initThread("initandlisten");
 
-        logger::globalLogDomain()->attachAppender(
-                logger::MessageLogDomain::AppenderAutoPtr(
-                        new RamLogAppender(new RamLog("global"))));
-
         bool is32bit = sizeof(int*) == 4;
 
         {
diff --git a/src/mongo/db/dbcommands_generic.cpp b/src/mongo/db/dbcommands_generic.cpp
index 1a78890a971..22666d7a5c5 100644
--- a/src/mongo/db/dbcommands_generic.cpp
+++ b/src/mongo/db/dbcommands_generic.cpp
@@ -354,20 +354,18 @@ namespace mongo {
                 result.appendArray( "names" , arr.arr() );
             }
             else {
-                RamLog* rl = RamLog::get( p );
-                if ( ! rl ) {
+                RamLog* ramlog = RamLog::getIfExists(p);
+                if ( ! ramlog ) {
                     errmsg = str::stream() << "no RamLog named: " << p;
                     return false;
                 }
+                RamLog::LineIterator rl(ramlog);
 
-                result.appendNumber( "totalLinesWritten", rl->getTotalLinesWritten() );
-
-                vector<const char*> lines;
-                rl->get( lines );
+                result.appendNumber( "totalLinesWritten", rl.getTotalLinesWritten() );
 
                 BSONArrayBuilder arr( result.subarrayStart( "log" ) );
-                for ( unsigned i=0; i<lines.size(); i++ )
-                    arr.append( lines[i] );
+                while (rl.more())
+                    arr.append(rl.next());
                 arr.done();
             }
             return true;
diff --git a/src/mongo/db/dbwebserver.cpp b/src/mongo/db/dbwebserver.cpp
index 4754d2a57ad..506945575fe 100644
--- a/src/mongo/db/dbwebserver.cpp
+++ b/src/mongo/db/dbwebserver.cpp
@@ -324,21 +324,15 @@ namespace mongo {
 
     vector<WebStatusPlugin*> * WebStatusPlugin::_plugins = 0;
 
-    // -- basic statuc plugins --
+    // -- basic status plugins --
 
     class LogPlugin : public WebStatusPlugin {
     public:
         LogPlugin() : WebStatusPlugin( "Log" , 100 ), _log(0) {
             _log = RamLog::get( "global" );
-            if ( ! _log ) {
-                _log = new RamLog("global");
-                logger::globalLogDomain()->attachAppender(logger::MessageLogDomain::AppenderAutoPtr(
-                                                                  new RamLogAppender(_log)));
-            }
         }
 
-        virtual void init() {
-        }
+        virtual void init() {}
 
         virtual void run( stringstream& ss ) {
             _log->toHTML( ss );
@@ -346,9 +340,7 @@ namespace mongo {
         RamLog * _log;
     };
 
-    MONGO_INITIALIZER_GENERAL(WebStatusLogPlugin, ("ServerLogRedirection"), ("default"))(
-            InitializerContext*) {
-
+    MONGO_INITIALIZER(WebStatusLogPlugin)(InitializerContext*) {
         if (cmdLine.isHttpInterfaceEnabled) {
             new LogPlugin;
         }
diff --git a/src/mongo/db/initialize_server_global_state.cpp b/src/mongo/db/initialize_server_global_state.cpp
index 084109d11b7..71798bcc238 100644
--- a/src/mongo/db/initialize_server_global_state.cpp
+++ b/src/mongo/db/initialize_server_global_state.cpp
@@ -35,6 +35,7 @@
 #include "mongo/logger/logger.h"
 #include "mongo/logger/message_event.h"
 #include "mongo/logger/message_event_utf8_encoder.h"
+#include "mongo/logger/ramlog.h"
 #include "mongo/logger/rotatable_file_appender.h"
 #include "mongo/logger/rotatable_file_manager.h"
 #include "mongo/logger/rotatable_file_writer.h"
@@ -264,6 +265,10 @@ namespace mongo {
             }
         }
 
+        logger::globalLogDomain()->attachAppender(
+                logger::MessageLogDomain::AppenderAutoPtr(
+                        new RamLogAppender(RamLog::get("global"))));
+
         return Status::OK();
     }
 
diff --git a/src/mongo/db/repl/health.cpp b/src/mongo/db/repl/health.cpp
index 3a2681128f8..7dd9f528d27 100644
--- a/src/mongo/db/repl/health.cpp
+++ b/src/mongo/db/repl/health.cpp
@@ -46,7 +46,7 @@ namespace mongo {
     using namespace mongoutils::html;
     using namespace bson;
 
-    static RamLog * _rsLog = new RamLog( "rs" );
+    static RamLog * _rsLog = RamLog::get("rs");
     Tee *rsLog = _rsLog;
     extern bool replSetBlind; // for testing
 
diff --git a/src/mongo/logger/ramlog.cpp b/src/mongo/logger/ramlog.cpp
index 6bb8e9df40d..d2a5f507e76 100644
--- a/src/mongo/logger/ramlog.cpp
+++ b/src/mongo/logger/ramlog.cpp
@@ -19,43 +19,39 @@
 
 #include "mongo/logger/ramlog.h"
 
+#include "mongo/base/init.h"
+#include "mongo/base/status.h"
 #include "mongo/logger/message_event_utf8_encoder.h"
 #include "mongo/util/mongoutils/html.h"
+#include "mongo/util/map_util.h"
 #include "mongo/util/mongoutils/str.h"
 
 namespace mongo {
+namespace {
+    typedef std::map<string,RamLog*> RM;
+    mongo::mutex* _namedLock = NULL;
+    RM*  _named = NULL;
+
+}  // namespace
 
     using namespace mongoutils;
 
     RamLog::RamLog( const std::string& name ) : _name(name), _totalLinesWritten(0), _lastWrite(0) {
-        h = 0; n = 0;
+        h = 0;
+        n = 0;
         for( int i = 0; i < N; i++ )
             lines[i][C-1] = 0;
-
-        if ( name.size() ) {
-            
-            if ( ! _namedLock )
-                _namedLock = new mongo::mutex("RamLog::_namedLock");
-
-            scoped_lock lk( *_namedLock );
-            if ( ! _named )
-                _named = new RM();
-            (*_named)[name] = this;
-        }
-        
     }
 
-    RamLog::~RamLog() {
-        
-    }
+    RamLog::~RamLog() {}
 
     void RamLog::write(const std::string& str) {
-        boost::unique_lock<boost::mutex> lk(_mutex);
+        boost::lock_guard<boost::mutex> lk(_mutex);
         _lastWrite = time(0);
         _totalLinesWritten++;
 
         char *p = lines[(h+n)%N];
-        
+
         unsigned sz = str.size();
         if (0 == sz) return;
         if( sz < C ) {
@@ -74,20 +70,18 @@ namespace mongo {
         else h = (h+1) % N;
     }
 
-    time_t RamLog::lastWrite() {
-        boost::unique_lock<boost::mutex> lk(_mutex);
-        return _lastWrite;
+    time_t RamLog::LineIterator::lastWrite() {
+        return _ramlog->_lastWrite;
     }
 
-    long long RamLog::getTotalLinesWritten() {
-        boost::unique_lock<boost::mutex> lk(_mutex);
-        return _totalLinesWritten;
+    long long RamLog::LineIterator::getTotalLinesWritten() {
+        return _ramlog->_totalLinesWritten;
     }
 
-    void RamLog::get( std::vector<const char*>& v) {
-        boost::unique_lock<boost::mutex> lk(_mutex);
-        for( unsigned x=0, i=h; x++ < n; i=(i+1)%N )
-            v.push_back(lines[i]);
+    const char* RamLog::getLine_inlock(unsigned lineNumber) const {
+        if (lineNumber >= n)
+            return "";
+        return lines[(lineNumber + h) % N]; // h = 0 unless n == N, hence modulo N.
     }
 
     int RamLog::repeats(const std::vector<const char *>& v, int i) {
@@ -143,8 +137,10 @@ namespace mongo {
     }
 
     void RamLog::toHTML(std::stringstream& s) {
+        LineIterator iter(this);
         std::vector<const char*> v;
-        get( v );
+        while (iter.more())
+            v.push_back(iter.next());
 
         s << "<pre>\n";
         for( int i = 0; i < (int)v.size(); i++ ) {
@@ -173,6 +169,12 @@ namespace mongo {
         s << "</pre>\n";
     }
 
+    RamLog::LineIterator::LineIterator(RamLog* ramlog) :
+        _ramlog(ramlog),
+        _lock(ramlog->_mutex),
+        _nextLineIndex(0) {
+    }
+
     RamLogAppender::RamLogAppender(RamLog* ramlog) : _ramlog(ramlog) {}
     RamLogAppender::~RamLogAppender() {}
 
@@ -188,15 +190,31 @@ namespace mongo {
     // static things
     // ---------------
 
-    RamLog* RamLog::get( const std::string& name ) {
-        if ( ! _named )
-            return 0;
+    RamLog* RamLog::get(const std::string& name) {
+        if (!_namedLock) {
+            // Guaranteed to happen before multi-threaded operation.
+            _namedLock = new mongo::mutex("RamLog::_namedLock");
+        }
 
         scoped_lock lk( *_namedLock );
-        RM::iterator i = _named->find( name );
-        if ( i == _named->end() )
-            return 0;
-        return i->second;
+        if (!_named) {
+            // Guaranteed to happen before multi-threaded operation.
+            _named = new RM();
+        }
+
+        RamLog* result = mapFindWithDefault(*_named, name, static_cast<RamLog*>(NULL));
+        if (!result) {
+            result = new RamLog(name);
+            (*_named)[name] = result;
+        }
+        return result;
+    }
+
+    RamLog* RamLog::getIfExists(const std::string& name) {
+        if (!_named)
+            return NULL;
+        scoped_lock lk(*_namedLock);
+        return mapFindWithDefault(*_named, name, static_cast<RamLog*>(NULL));
     }
 
     void RamLog::getNames( std::vector<string>& names ) {
@@ -210,6 +228,20 @@ namespace mongo {
         }
     }
 
-    mongo::mutex* RamLog::_namedLock;
-    RamLog::RM*  RamLog::_named = 0;
+    /**
+     * Ensures that RamLog::get() is called at least once during single-threaded operation,
+     * ensuring that _namedLock and _named are initialized safely.
+     */
+    MONGO_INITIALIZER(RamLogCatalog)(InitializerContext*) {
+        if (!_namedLock) {
+            if (_named) {
+                return Status(ErrorCodes::InternalError,
+                              "Inconsistent intiailization of RamLogCatalog.");
+            }
+            _namedLock = new mongo::mutex("RamLog::_namedLock");
+            _named = new RM();
+        }
+
+        return Status::OK();
+    }
 }
diff --git a/src/mongo/logger/ramlog.h b/src/mongo/logger/ramlog.h
index 2b78d91daaa..16263055c38 100644
--- a/src/mongo/logger/ramlog.h
+++ b/src/mongo/logger/ramlog.h
@@ -17,7 +17,6 @@
 
 #pragma once
 
-#include <map>
 #include <sstream>
 #include <string>
 #include <vector>
@@ -32,23 +31,61 @@
 
 namespace mongo {
 
+    /**
+     * Fixed-capacity log of line-oriented messages.
+     *
+     * Holds up to RamLog::N lines of up to RamLog::C bytes, each.
+     *
+     * RamLogs are stored in a global registry, accessed via RamLog::get() and
+     * RamLog::getIfExists().
+     *
+     * RamLogs and their registry are self-synchronizing.  See documentary comments.
+     * To read a RamLog, instantiate a RamLog::LineIterator, documented below.
+     */
     class RamLog : public logger::Tee {
         MONGO_DISALLOW_COPYING(RamLog);
     public:
-        RamLog( const std::string& name );
+        class LineIterator;
+        friend class RamLog::LineIterator;
+
+        /**
+         * Returns a pointer to the ramlog named "name", creating one if it did not already exist.
+         *
+         * Synchronizes on the RamLog catalog lock, _namedLock.
+         */
+        static RamLog* get(const std::string& name);
+
+        /**
+         * Returns a pointer to the ramlog named "name", or NULL if no such ramlog exists.
+         *
+         * Synchronizes on the RamLog catalog lock, _namedLock.
+         */
+        static RamLog* getIfExists(const std::string& name);
+
+        /**
+         * Writes the names of all existing ramlogs into "names".
+         *
+         * Synchronizes on the RamLog catalog lock, _namedLock.
+         */
+        static void getNames( std::vector<std::string>& names );
 
+        /**
+         * Writes "str" as a line into the RamLog.  If "str" is longer than the maximum
+         * line size, RamLog::C, truncates the line to the first C bytes.  If "str"
+         * is shorter than RamLog::C and has a terminal '\n', it omits that character.
+         *
+         * Synchronized on the instance's own mutex, _mutex.
+         */
         void write(const std::string& str);
 
-        void get( std::vector<const char*>& v);
 
+        /**
+         * Writes an HTML representation of the log to "s".
+         *
+         * Synchronized on the instance's own mutex, _mutex.
+         */
         void toHTML(std::stringstream& s);
 
-        static RamLog* get( const std::string& name );
-        static void getNames( std::vector<std::string>& names );
-
-        time_t lastWrite();
-        long long getTotalLinesWritten();
-
     private:
         static int repeats(const std::vector<const char *>& v, int i);
         static string clean(const std::vector<const char *>& v, int i, string line="");
@@ -57,6 +94,7 @@ namespace mongo {
         /* turn http:... into an anchor */
         static string linkify(const char *s);
 
+        explicit RamLog( const std::string& name );
         ~RamLog(); // want this private as we want to leak so we can use them till the very end
 
         enum {
@@ -64,6 +102,8 @@ namespace mongo {
             C = 512 // max size of line
         };
 
+        const char* getLine_inlock(unsigned lineNumber) const;
+
         boost::mutex _mutex;  // Guards all non-static data.
         char lines[N][C];
         unsigned h; // current position
@@ -71,12 +111,53 @@ namespace mongo {
         string _name;
         long long _totalLinesWritten;
 
-        typedef std::map<string,RamLog*> RM;
-        static mongo::mutex* _namedLock;
-        static RM*  _named;
         time_t _lastWrite;
     };
 
+    /**
+     * Iterator over the lines of a RamLog.
+     *
+     * Also acts as a means of inspecting other properites of a ramlog consistently.
+     *
+     * Instances of LineIterator hold the lock for the underlying RamLog for their whole lifetime,
+     * and so should not be kept around.
+     */
+    class RamLog::LineIterator {
+        MONGO_DISALLOW_COPYING(LineIterator);
+    public:
+        explicit LineIterator(RamLog* ramlog);
+
+        /**
+         * Returns true if there are more lines available to return by calls to next().
+         */
+        bool more() const { return _nextLineIndex < _ramlog->n; }
+
+        /**
+         * Returns the next line and advances the iterator.
+         */
+        const char* next() {
+            return _ramlog->getLine_inlock(_nextLineIndex++);  // Postfix increment.
+        }
+
+        /**
+         * Returns the time of the last write to the ramlog.
+         */
+        time_t lastWrite();
+
+        /**
+         * Returns the total number of lines ever written to the ramlog.
+         */
+        long long getTotalLinesWritten();
+
+    private:
+        const RamLog* _ramlog;
+        boost::lock_guard<boost::mutex> _lock;
+        unsigned _nextLineIndex;
+    };
+
+    /**
+     * Appender for appending MessageEvents to a RamLog.
+     */
     class RamLogAppender : public logger::Appender<logger::MessageEventEphemeral> {
     public:
         explicit RamLogAppender(RamLog* ramlog);
diff --git a/src/mongo/s/d_migrate.cpp b/src/mongo/s/d_migrate.cpp
index 27b238c921c..04122d23008 100644
--- a/src/mongo/s/d_migrate.cpp
+++ b/src/mongo/s/d_migrate.cpp
@@ -54,6 +54,7 @@
 #include "mongo/db/repl/rs.h"
 #include "mongo/db/repl/rs_config.h"
 #include "mongo/db/repl/write_concern.h"
+#include "mongo/logger/ramlog.h"
 #include "mongo/s/chunk.h"
 #include "mongo/s/chunk_version.h"
 #include "mongo/s/config.h"
@@ -64,14 +65,13 @@
 #include "mongo/util/elapsed_tracker.h"
 #include "mongo/util/processinfo.h"
 #include "mongo/util/queue.h"
-#include "mongo/util/ramlog.h"
 #include "mongo/util/startup_test.h"
 
 using namespace std;
 
 namespace mongo {
 
-    Tee* migrateLog = new RamLog( "migrate" );
+    Tee* migrateLog = RamLog::get("migrate");
 
     class MoveTimingHelper {
     public:
diff --git a/src/mongo/s/server.cpp b/src/mongo/s/server.cpp
index 1d90c204ac1..58995b63055 100644
--- a/src/mongo/s/server.cpp
+++ b/src/mongo/s/server.cpp
@@ -228,9 +228,6 @@ namespace mongo {
 
     void init() {
         serverID.init();
-
-        logger::globalLogDomain()->attachAppender(
-                logger::MessageLogDomain::AppenderAutoPtr(new RamLogAppender(new RamLog("global"))));
     }
 
     void start( const MessageServer::Options& opts ) {
diff --git a/src/mongo/util/log.cpp b/src/mongo/util/log.cpp
index 34421f3aecb..b778b8cc006 100644
--- a/src/mongo/util/log.cpp
+++ b/src/mongo/util/log.cpp
@@ -137,7 +137,7 @@ namespace mongo {
     LogIndentLevel::~LogIndentLevel() {
     }
 
-    Tee* const warnings = new RamLog("warnings"); // Things put here go in serverStatus
-    Tee* const startupWarningsLog = new RamLog("startupWarnings");  // intentionally leaked
+    Tee* const warnings = RamLog::get("warnings"); // Things put here go in serverStatus
+    Tee* const startupWarningsLog = RamLog::get("startupWarnings");  // intentionally leaked
 
 }  // namespace mongo