SERVER-14395 Clean up StorageEngine initialization and shutdown

* Added StorageEngine::cleanShutdown() and commented that the destructor will
  never be called.

* MMapV1StorageEngine specific operations were pulled into its constructor
  and cleanShutdown implementation.

* StorageEngines are now constructed at a point where it is safe to spawn
  threads.

10 files changed, 343 insertions, 309 deletions

diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp
index 50657a2ebf5..7d067fea53f 100644
--- a/src/mongo/db/db.cpp
+++ b/src/mongo/db/db.cpp
@@ -80,7 +80,6 @@
 #include "mongo/db/startup_warnings.h"
 #include "mongo/db/stats/counters.h"
 #include "mongo/db/stats/snapshots.h"
-#include "mongo/db/storage/mmap_v1/dur.h"
 #include "mongo/db/storage/storage_engine.h"
 #include "mongo/db/storage_options.h"
 #include "mongo/db/ttl.h"
@@ -93,7 +92,6 @@
 #include "mongo/util/concurrency/thread_name.h"
 #include "mongo/util/exception_filter_win32.h"
 #include "mongo/util/exit.h"
-#include "mongo/util/file_allocator.h"
 #include "mongo/util/log.h"
 #include "mongo/util/net/message_server.h"
 #include "mongo/util/net/ssl_manager.h"
@@ -118,7 +116,6 @@ namespace mongo {
     void (*snmpInit)() = NULL;
 
     extern int diagLogging;
-    extern int lockFile;
 
 #ifdef _WIN32
     ntservice::NtServiceDefaultStrings defaultServiceStrings = {
@@ -350,7 +347,7 @@ namespace mongo {
             if (shouldClearNonLocalTmpCollections || dbName == "local")
                 ctx.db()->clearTmpCollections(&txn);
 
-            if ( mongodGlobalParams.repair ) {
+            if ( storageGlobalParams.repair ) {
                 fassert(18506, globalStorageEngine->repairDatabase(&txn, dbName));
             }
             else if (!ctx.db()->getDatabaseCatalogEntry()->currentFilesCompatible(&txn)) {
@@ -406,17 +403,6 @@ namespace mongo {
         LOG(1) << "done repairDatabases" << endl;
     }
 
-    void clearTmpFiles() {
-        boost::filesystem::path path(storageGlobalParams.dbpath);
-        for ( boost::filesystem::directory_iterator i( path );
-                i != boost::filesystem::directory_iterator(); ++i ) {
-            string fileName = boost::filesystem::path(*i).leaf().string();
-            if ( boost::filesystem::is_directory( *i ) &&
-                    fileName.length() && fileName[ 0 ] == '$' )
-                boost::filesystem::remove_all( *i );
-        }
-    }
-
     /**
      * Checks if this server was started without --replset but has a config in local.system.replset
      * (meaning that this is probably a replica set member started in stand-alone mode).
@@ -548,53 +534,6 @@ namespace mongo {
     }
 #endif
 
-    /// warn if readahead > 256KB (gridfs chunk size)
-    static void checkReadAhead(const string& dir) {
-#ifdef __linux__
-        try {
-            const dev_t dev = getPartition(dir);
-
-            // This path handles the case where the filesystem uses the whole device (including LVM)
-            string path = str::stream() <<
-                "/sys/dev/block/" << major(dev) << ':' << minor(dev) << "/queue/read_ahead_kb";
-
-            if (!boost::filesystem::exists(path)){
-                // This path handles the case where the filesystem is on a partition.
-                path = str::stream()
-                    << "/sys/dev/block/" << major(dev) << ':' << minor(dev) // this is a symlink
-                    << "/.." // parent directory of a partition is for the whole device
-                    << "/queue/read_ahead_kb";
-            }
-
-            if (boost::filesystem::exists(path)) {
-                ifstream file (path.c_str());
-                if (file.is_open()) {
-                    int kb;
-                    file >> kb;
-                    if (kb > 256) {
-                        log() << startupWarningsLog;
-
-                        log() << "** WARNING: Readahead for " << dir << " is set to " << kb << "KB"
-                                << startupWarningsLog;
-
-                        log() << "**          We suggest setting it to 256KB (512 sectors) or less"
-                                << startupWarningsLog;
-
-                        log() << "**          http://dochub.mongodb.org/core/readahead"
-                                << startupWarningsLog;
-                    }
-                }
-            }
-        }
-        catch (const std::exception& e) {
-            log() << "unable to validate readahead settings due to error: " << e.what()
-                  << startupWarningsLog;
-            log() << "for more information, see http://dochub.mongodb.org/core/readahead"
-                  << startupWarningsLog;
-        }
-#endif // __linux__
-    }
-
     static void _initAndListen(int listenPort ) {
         Client::initThread("initandlisten");
 
@@ -635,23 +574,15 @@ namespace mongo {
                     boost::filesystem::exists(storageGlobalParams.repairpath));
         }
 
-        // TODO check non-journal subdirs if using directory-per-db
-        checkReadAhead(storageGlobalParams.dbpath);
-
-        acquirePathLock(mongodGlobalParams.repair);
-        boost::filesystem::remove_all(storageGlobalParams.dbpath + "/_tmp/");
-
-        FileAllocator::get()->start();
-
         // TODO:  This should go into a MONGO_INITIALIZER once we have figured out the correct
         // dependencies.
         if (snmpInit) {
             snmpInit();
         }
 
-        MONGO_ASSERT_ON_EXCEPTION_WITH_MSG( clearTmpFiles(), "clear tmp files" );
+        initGlobalStorageEngine();
 
-        dur::startup();
+        boost::filesystem::remove_all(storageGlobalParams.dbpath + "/_tmp/");
 
         if (storageGlobalParams.durOptions & StorageGlobalParams::DurRecoverOnly)
             return;
@@ -684,7 +615,7 @@ namespace mongo {
                                          || replSettings.slave == repl::SimpleSlave);
         repairDatabasesAndCheckVersion(shouldClearNonLocalTmpCollections);
 
-        if (mongodGlobalParams.upgrade) {
+        if (storageGlobalParams.upgrade) {
             log() << "finished checking dbs" << endl;
             cc().shutdown();
             exitCleanly(EXIT_CLEAN);
diff --git a/src/mongo/db/instance.cpp b/src/mongo/db/instance.cpp
index c10267261a7..4625693ade5 100644
--- a/src/mongo/db/instance.cpp
+++ b/src/mongo/db/instance.cpp
@@ -30,14 +30,8 @@
 
 #include "mongo/platform/basic.h"
 
-#include <boost/filesystem/operations.hpp>
 #include <boost/thread/thread.hpp>
 #include <fstream>
-#if defined(_WIN32)
-#include <io.h>
-#else
-#include <sys/file.h>
-#endif
 
 #include "mongo/base/status.h"
 #include "mongo/bson/util/atomic_int.h"
@@ -53,9 +47,6 @@
 #include "mongo/db/dbhelpers.h"
 #include "mongo/db/dbmessage.h"
 #include "mongo/db/storage/storage_engine.h"
-#include "mongo/db/storage/mmap_v1/dur_commitjob.h"
-#include "mongo/db/storage/mmap_v1/dur_journal.h"
-#include "mongo/db/storage/mmap_v1/dur_recover.h"
 #include "mongo/db/operation_context_impl.h"
 #include "mongo/db/global_optime.h"
 #include "mongo/db/global_environment_experiment.h"
@@ -85,7 +76,6 @@
 #include "mongo/scripting/engine.h"
 #include "mongo/util/exit.h"
 #include "mongo/util/fail_point_service.h"
-#include "mongo/util/file_allocator.h"
 #include "mongo/util/gcov.h"
 #include "mongo/util/goodies.h"
 #include "mongo/util/log.h"
@@ -111,10 +101,6 @@ namespace mongo {
 
     string dbExecCommand;
 
-    int lockFile = 0;
-#ifdef _WIN32
-    HANDLE lockFileHandle;
-#endif
 
     MONGO_FP_DECLARE(rsStopGetMore);
 
@@ -1052,45 +1038,7 @@ namespace {
         log() << "shutdown: going to close sockets..." << endl;
         boost::thread close_socket_thread( stdx::bind(MessagingPort::closeAllSockets, 0) );
 
-        // wait until file preallocation finishes
-        // we would only hang here if the file_allocator code generates a
-        // synchronous signal, which we don't expect
-        log() << "shutdown: waiting for fs preallocator..." << endl;
-        FileAllocator::get()->waitUntilFinished();
-
-        if (storageGlobalParams.dur) {
-            log() << "shutdown: final commit..." << endl;
-            getDur().commitNow(txn);
-
-            globalStorageEngine->flushAllFiles(true);
-        }
-
-        log() << "shutdown: closing all files..." << endl;
-        stringstream ss3;
-        MemoryMappedFile::closeAllFiles( ss3 );
-        log() << ss3.str() << endl;
-
-        if (storageGlobalParams.dur) {
-            dur::journalCleanup(true);
-        }
-
-#if !defined(__sunos__)
-        if ( lockFile ) {
-            log() << "shutdown: removing fs lock..." << endl;
-            /* This ought to be an unlink(), but Eliot says the last
-               time that was attempted, there was a race condition
-               with acquirePathLock().  */
-#ifdef _WIN32
-            if( _chsize( lockFile , 0 ) )
-                log() << "couldn't remove fs lock " << errnoWithDescription(_doserrno) << endl;
-            CloseHandle(lockFileHandle);
-#else
-            if( ftruncate( lockFile , 0 ) )
-                log() << "couldn't remove fs lock " << errnoWithDescription() << endl;
-            flock( lockFile, LOCK_UN );
-#endif
-        }
-#endif
+        globalStorageEngine->cleanShutdown(txn);
     }
 
     void exitCleanly( ExitCode code ) {
@@ -1127,8 +1075,7 @@ namespace {
     NOINLINE_DECL void dbexit( ExitCode rc, const char *why ) {
         flushForGcov();
 
-        Client * c = currentClient.get();
-        audit::logShutdown(c);
+        audit::logShutdown(currentClient.get());
         {
             scoped_lock lk( exitMutex );
             if ( numExitCalls++ > 0 ) {
@@ -1137,7 +1084,6 @@ namespace {
                     ::_exit( rc );
                 }
                 log() << "dbexit: " << why << "; exiting immediately";
-                if ( c ) c->shutdown();
                 ::_exit( rc );
             }
         }
@@ -1151,175 +1097,18 @@ namespace {
         catch (...) { }
 #endif
 
-        // block the dur thread from doing any work for the rest of the run
-        LOG(2) << "shutdown: groupCommitMutex" << endl;
-        SimpleMutex::scoped_lock lk(dur::commitJob.groupCommitMutex);
-
 #ifdef _WIN32
         // Windows Service Controller wants to be told when we are down,
         //  so don't call ::_exit() yet, or say "really exiting now"
         //
         if ( rc == EXIT_WINDOWS_SERVICE_STOP ) {
-            if ( c ) c->shutdown();
             return;
         }
 #endif
         log() << "dbexit: really exiting now";
-        if ( c ) c->shutdown();
         ::_exit(rc);
     }
 
-#if !defined(__sunos__)
-    void writePid(int fd) {
-        stringstream ss;
-        ss << ProcessId::getCurrent() << endl;
-        string s = ss.str();
-        const char * data = s.c_str();
-#ifdef _WIN32
-        verify( _write( fd, data, strlen( data ) ) );
-#else
-        verify( write( fd, data, strlen( data ) ) );
-#endif
-    }
-
-    void acquirePathLock(bool doingRepair) {
-        string name = (boost::filesystem::path(storageGlobalParams.dbpath) / "mongod.lock").string();
-
-        bool oldFile = false;
-
-        if ( boost::filesystem::exists( name ) && boost::filesystem::file_size( name ) > 0 ) {
-            oldFile = true;
-        }
-
-#ifdef _WIN32
-        lockFileHandle = CreateFileA( name.c_str(), GENERIC_READ | GENERIC_WRITE,
-            0 /* do not allow anyone else access */, NULL, 
-            OPEN_ALWAYS /* success if fh can open */, 0, NULL );
-
-        if (lockFileHandle == INVALID_HANDLE_VALUE) {
-            DWORD code = GetLastError();
-            char *msg;
-            FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM,
-                NULL, code, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
-                (LPSTR)&msg, 0, NULL);
-            string m = msg;
-            str::stripTrailing(m, "\r\n");
-            uasserted( 13627 , str::stream() << "Unable to create/open lock file: " << name << ' ' << m << ". Is a mongod instance already running?" );
-        }
-        lockFile = _open_osfhandle((intptr_t)lockFileHandle, 0);
-#else
-        lockFile = open( name.c_str(), O_RDWR | O_CREAT , S_IRWXU | S_IRWXG | S_IRWXO );
-        if( lockFile <= 0 ) {
-            uasserted( 10309 , str::stream() << "Unable to create/open lock file: " << name << ' ' << errnoWithDescription() << " Is a mongod instance already running?" );
-        }
-        if (flock( lockFile, LOCK_EX | LOCK_NB ) != 0) {
-            close ( lockFile );
-            lockFile = 0;
-            uassert( 10310 ,  "Unable to lock file: " + name + ". Is a mongod instance already running?",  0 );
-        }
-#endif
-
-        if ( oldFile ) {
-            // we check this here because we want to see if we can get the lock
-            // if we can't, then its probably just another mongod running
-            
-            string errmsg;
-            if (doingRepair && dur::haveJournalFiles()) {
-                errmsg = "************** \n"
-                         "You specified --repair but there are dirty journal files. Please\n"
-                         "restart without --repair to allow the journal files to be replayed.\n"
-                         "If you wish to repair all databases, please shutdown cleanly and\n"
-                         "run with --repair again.\n"
-                         "**************";
-            }
-            else if (storageGlobalParams.dur) {
-                if (!dur::haveJournalFiles(/*anyFiles=*/true)) {
-                    // Passing anyFiles=true as we are trying to protect against starting in an
-                    // unclean state with the journal directory unmounted. If there are any files,
-                    // even prealloc files, then it means that it is mounted so we can continue.
-                    // Previously there was an issue (SERVER-5056) where we would fail to start up
-                    // if killed during prealloc.
-
-                    vector<string> dbnames;
-                    globalStorageEngine->listDatabases( &dbnames );
-
-                    if ( dbnames.size() == 0 ) {
-                        // this means that mongod crashed
-                        // between initial startup and when journaling was initialized
-                        // it is safe to continue
-                    }
-                    else {
-                        errmsg = str::stream()
-                            << "************** \n"
-                            << "old lock file: " << name << ".  probably means unclean shutdown,\n"
-                            << "but there are no journal files to recover.\n"
-                            << "this is likely human error or filesystem corruption.\n"
-                            << "please make sure that your journal directory is mounted.\n"
-                            << "found " << dbnames.size() << " dbs.\n"
-                            << "see: http://dochub.mongodb.org/core/repair for more information\n"
-                            << "*************";
-                    }
-
-                }
-            }
-            else {
-                if (!dur::haveJournalFiles() && !doingRepair) {
-                    errmsg = str::stream()
-                             << "************** \n"
-                             << "Unclean shutdown detected.\n"
-                             << "Please visit http://dochub.mongodb.org/core/repair for recovery instructions.\n"
-                             << "*************";
-                }
-            }
-
-            if (!errmsg.empty()) {
-                cout << errmsg << endl;
-#ifdef _WIN32
-                CloseHandle( lockFileHandle );
-#else
-                close ( lockFile );
-#endif
-                lockFile = 0;
-                uassert( 12596 , "old lock file" , 0 );
-            }
-        }
-
-        // Not related to lock file, but this is where we handle unclean shutdown
-        if (!storageGlobalParams.dur && dur::haveJournalFiles()) {
-            cout << "**************" << endl;
-            cout << "Error: journal files are present in journal directory, yet starting without journaling enabled." << endl;
-            cout << "It is recommended that you start with journaling enabled so that recovery may occur." << endl;
-            cout << "**************" << endl;
-            uasserted(13597, "can't start without --journal enabled when journal/ files are present");
-        }
-
-#ifdef _WIN32
-        uassert( 13625, "Unable to truncate lock file", _chsize(lockFile, 0) == 0);
-        writePid( lockFile );
-        _commit( lockFile );
-#else
-        uassert( 13342, "Unable to truncate lock file", ftruncate(lockFile, 0) == 0);
-        writePid( lockFile );
-        fsync( lockFile );
-        flushMyDirectory(name);
-#endif
-    }
-#else
-    void acquirePathLock(bool) {
-        // TODO - this is very bad that the code above not running here.
-
-        // Not related to lock file, but this is where we handle unclean shutdown
-        if (!storageGlobalParams.dur && dur::haveJournalFiles()) {
-            cout << "**************" << endl;
-            cout << "Error: journal files are present in journal directory, yet starting without --journal enabled." << endl;
-            cout << "It is recommended that you start with journaling enabled so that recovery may occur." << endl;
-            cout << "Alternatively (not recommended), you can backup everything, then delete the journal files, and run --repair" << endl;
-            cout << "**************" << endl;
-            uasserted(13618, "can't start without --journal enabled when journal/ files are present");
-        }
-    }
-#endif
-
     // ----- BEGIN Diaglog -----
     DiagLog::DiagLog() : f(0) , level(0), mutex("DiagLog") { 
     }
diff --git a/src/mongo/db/instance.h b/src/mongo/db/instance.h
index d5fef449645..a8abac7e309 100644
--- a/src/mongo/db/instance.h
+++ b/src/mongo/db/instance.h
@@ -144,11 +144,6 @@ namespace mongo {
         OperationContext* _txn; // Points either to _txnOwned or a passed-in transaction.
     };
 
-    extern int lockFile;
-#ifdef _WIN32
-    extern HANDLE lockFileHandle;
-#endif
-    void acquirePathLock(bool doingRepair=false); // if doingRepair=true don't consider unclean shutdown an error
     void maybeCreatePidFile();
 
     void exitCleanly( ExitCode code );
diff --git a/src/mongo/db/mongod_options.cpp b/src/mongo/db/mongod_options.cpp
index bbfe400dfa4..5b56c79c525 100644
--- a/src/mongo/db/mongod_options.cpp
+++ b/src/mongo/db/mongod_options.cpp
@@ -989,12 +989,12 @@ namespace mongo {
         }
 
         if (params.count("repair") && params["repair"].as<bool>() == true) {
-            mongodGlobalParams.upgrade = 1; // --repair implies --upgrade
-            mongodGlobalParams.repair = 1;
+            storageGlobalParams.upgrade = 1; // --repair implies --upgrade
+            storageGlobalParams.repair = 1;
             storageGlobalParams.dur = false;
         }
         if (params.count("upgrade") && params["upgrade"].as<bool>() == true) {
-            mongodGlobalParams.upgrade = 1;
+            storageGlobalParams.upgrade = 1;
         }
         if (params.count("notablescan")) {
             storageGlobalParams.noTableScan = params["notablescan"].as<bool>();
diff --git a/src/mongo/db/mongod_options.h b/src/mongo/db/mongod_options.h
index f8ed7d86dcb..0843175533b 100644
--- a/src/mongo/db/mongod_options.h
+++ b/src/mongo/db/mongod_options.h
@@ -45,13 +45,9 @@ namespace mongo {
     namespace moe = mongo::optionenvironment;
 
     struct MongodGlobalParams {
-        bool upgrade;
-        bool repair;
         bool scriptingEnabled; // --noscripting
 
         MongodGlobalParams() :
-            upgrade(0),
-            repair(0),
             scriptingEnabled(true)
         { }
     };
diff --git a/src/mongo/db/storage/mmap_v1/mmap_v1_engine.cpp b/src/mongo/db/storage/mmap_v1/mmap_v1_engine.cpp
index d30d4f3d679..e8ef1e285c5 100644
--- a/src/mongo/db/storage/mmap_v1/mmap_v1_engine.cpp
+++ b/src/mongo/db/storage/mmap_v1/mmap_v1_engine.cpp
@@ -32,14 +32,274 @@
 
 #include <boost/filesystem/path.hpp>
 #include <boost/filesystem/operations.hpp>
+#include <fstream>
 
-#include "mongo/db/storage_options.h"
-#include "mongo/db/storage/mmap_v1/mmap_v1_database_catalog_entry.h"
+#if defined(_WIN32)
+#include <io.h>
+#else
+#include <sys/file.h>
+#endif
+
+#include "mongo/db/mongod_options.h"
+#include "mongo/db/storage/mmap_v1/dur.h"
+#include "mongo/db/storage/mmap_v1/dur_commitjob.h"
+#include "mongo/db/storage/mmap_v1/dur_journal.h"
+#include "mongo/db/storage/mmap_v1/dur_recover.h"
 #include "mongo/db/storage/mmap_v1/dur_recovery_unit.h"
+#include "mongo/db/storage/mmap_v1/mmap_v1_database_catalog_entry.h"
+#include "mongo/db/storage_options.h"
+#include "mongo/platform/process_id.h"
+#include "mongo/util/file_allocator.h"
 #include "mongo/util/mmap.h"
 
 namespace mongo {
 
+namespace {
+#ifdef _WIN32
+    HANDLE lockFileHandle;
+#endif
+
+    // This is used by everyone, including windows.
+    int lockFile = 0;
+
+#if !defined(__sunos__)
+    void writePid(int fd) {
+        stringstream ss;
+        ss << ProcessId::getCurrent() << endl;
+        string s = ss.str();
+        const char * data = s.c_str();
+#ifdef _WIN32
+        verify( _write( fd, data, strlen( data ) ) );
+#else
+        verify( write( fd, data, strlen( data ) ) );
+#endif
+    }
+
+    // if doingRepair is true don't consider unclean shutdown an error
+    void acquirePathLock(MMAPV1Engine* storageEngine, bool doingRepair) {
+        string name = (boost::filesystem::path(storageGlobalParams.dbpath) / "mongod.lock").string();
+
+        bool oldFile = false;
+
+        if ( boost::filesystem::exists( name ) && boost::filesystem::file_size( name ) > 0 ) {
+            oldFile = true;
+        }
+
+#ifdef _WIN32
+        lockFileHandle = CreateFileA( name.c_str(), GENERIC_READ | GENERIC_WRITE,
+            0 /* do not allow anyone else access */, NULL, 
+            OPEN_ALWAYS /* success if fh can open */, 0, NULL );
+
+        if (lockFileHandle == INVALID_HANDLE_VALUE) {
+            DWORD code = GetLastError();
+            char *msg;
+            FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM,
+                NULL, code, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+                (LPSTR)&msg, 0, NULL);
+            string m = msg;
+            str::stripTrailing(m, "\r\n");
+            uasserted(ErrorCodes::DBPathInUse,
+                      str::stream() << "Unable to create/open lock file: "
+                                    << name << ' ' << m
+                                    << ". Is a mongod instance already running?");
+        }
+        lockFile = _open_osfhandle((intptr_t)lockFileHandle, 0);
+#else
+        lockFile = open( name.c_str(), O_RDWR | O_CREAT , S_IRWXU | S_IRWXG | S_IRWXO );
+        if( lockFile <= 0 ) {
+            uasserted( ErrorCodes::DBPathInUse,
+                       str::stream() << "Unable to create/open lock file: "
+                                     << name << ' ' << errnoWithDescription()
+                                     << " Is a mongod instance already running?" );
+        }
+        if (flock( lockFile, LOCK_EX | LOCK_NB ) != 0) {
+            close ( lockFile );
+            lockFile = 0;
+            uasserted(ErrorCodes::DBPathInUse,
+                      "Unable to lock file: " + name + ". Is a mongod instance already running?");
+        }
+#endif
+
+        if ( oldFile ) {
+            // we check this here because we want to see if we can get the lock
+            // if we can't, then its probably just another mongod running
+            
+            string errmsg;
+            if (doingRepair && dur::haveJournalFiles()) {
+                errmsg = "************** \n"
+                         "You specified --repair but there are dirty journal files. Please\n"
+                         "restart without --repair to allow the journal files to be replayed.\n"
+                         "If you wish to repair all databases, please shutdown cleanly and\n"
+                         "run with --repair again.\n"
+                         "**************";
+            }
+            else if (storageGlobalParams.dur) {
+                if (!dur::haveJournalFiles(/*anyFiles=*/true)) {
+                    // Passing anyFiles=true as we are trying to protect against starting in an
+                    // unclean state with the journal directory unmounted. If there are any files,
+                    // even prealloc files, then it means that it is mounted so we can continue.
+                    // Previously there was an issue (SERVER-5056) where we would fail to start up
+                    // if killed during prealloc.
+
+                    vector<string> dbnames;
+                    storageEngine->listDatabases( &dbnames );
+
+                    if ( dbnames.size() == 0 ) {
+                        // this means that mongod crashed
+                        // between initial startup and when journaling was initialized
+                        // it is safe to continue
+                    }
+                    else {
+                        errmsg = str::stream()
+                            << "************** \n"
+                            << "old lock file: " << name << ".  probably means unclean shutdown,\n"
+                            << "but there are no journal files to recover.\n"
+                            << "this is likely human error or filesystem corruption.\n"
+                            << "please make sure that your journal directory is mounted.\n"
+                            << "found " << dbnames.size() << " dbs.\n"
+                            << "see: http://dochub.mongodb.org/core/repair for more information\n"
+                            << "*************";
+                    }
+
+                }
+            }
+            else {
+                if (!dur::haveJournalFiles() && !doingRepair) {
+                    errmsg = str::stream()
+                             << "************** \n"
+                             << "Unclean shutdown detected.\n"
+                             << "Please visit http://dochub.mongodb.org/core/repair for recovery instructions.\n"
+                             << "*************";
+                }
+            }
+
+            if (!errmsg.empty()) {
+                log() << errmsg << endl;
+#ifdef _WIN32
+                CloseHandle( lockFileHandle );
+#else
+                close ( lockFile );
+#endif
+                lockFile = 0;
+                uassert( 12596 , "old lock file" , 0 );
+            }
+        }
+
+        // Not related to lock file, but this is where we handle unclean shutdown
+        if (!storageGlobalParams.dur && dur::haveJournalFiles()) {
+            log() << "**************" << endl;
+            log() << "Error: journal files are present in journal directory, yet starting without journaling enabled." << endl;
+            log() << "It is recommended that you start with journaling enabled so that recovery may occur." << endl;
+            log() << "**************" << endl;
+            uasserted(13597, "can't start without --journal enabled when journal/ files are present");
+        }
+
+#ifdef _WIN32
+        uassert( 13625, "Unable to truncate lock file", _chsize(lockFile, 0) == 0);
+        writePid( lockFile );
+        _commit( lockFile );
+#else
+        uassert( 13342, "Unable to truncate lock file", ftruncate(lockFile, 0) == 0);
+        writePid( lockFile );
+        fsync( lockFile );
+        flushMyDirectory(name);
+#endif
+    }
+#else
+    void acquirePathLock(MMAPV1Engine* storageEngine, bool) {
+        // TODO - this is very bad that the code above not running here.
+
+        // Not related to lock file, but this is where we handle unclean shutdown
+        if (!storageGlobalParams.dur && dur::haveJournalFiles()) {
+            log() << "**************" << endl;
+            log() << "Error: journal files are present in journal directory, yet starting without --journal enabled." << endl;
+            log() << "It is recommended that you start with journaling enabled so that recovery may occur." << endl;
+            log() << "Alternatively (not recommended), you can backup everything, then delete the journal files, and run --repair" << endl;
+            log() << "**************" << endl;
+            uasserted(13618, "can't start without --journal enabled when journal/ files are present");
+        }
+    }
+#endif
+
+
+    /// warn if readahead > 256KB (gridfs chunk size)
+    void checkReadAhead(const string& dir) {
+#ifdef __linux__
+        try {
+            const dev_t dev = getPartition(dir);
+
+            // This path handles the case where the filesystem uses the whole device (including LVM)
+            string path = str::stream() <<
+                "/sys/dev/block/" << major(dev) << ':' << minor(dev) << "/queue/read_ahead_kb";
+
+            if (!boost::filesystem::exists(path)){
+                // This path handles the case where the filesystem is on a partition.
+                path = str::stream()
+                    << "/sys/dev/block/" << major(dev) << ':' << minor(dev) // this is a symlink
+                    << "/.." // parent directory of a partition is for the whole device
+                    << "/queue/read_ahead_kb";
+            }
+
+            if (boost::filesystem::exists(path)) {
+                ifstream file (path.c_str());
+                if (file.is_open()) {
+                    int kb;
+                    file >> kb;
+                    if (kb > 256) {
+                        log() << startupWarningsLog;
+
+                        log() << "** WARNING: Readahead for " << dir << " is set to " << kb << "KB"
+                                << startupWarningsLog;
+
+                        log() << "**          We suggest setting it to 256KB (512 sectors) or less"
+                                << startupWarningsLog;
+
+                        log() << "**          http://dochub.mongodb.org/core/readahead"
+                                << startupWarningsLog;
+                    }
+                }
+            }
+        }
+        catch (const std::exception& e) {
+            log() << "unable to validate readahead settings due to error: " << e.what()
+                  << startupWarningsLog;
+            log() << "for more information, see http://dochub.mongodb.org/core/readahead"
+                  << startupWarningsLog;
+        }
+#endif // __linux__
+    }
+
+    // This is unrelated to the _tmp directory in dbpath.
+    void clearTmpFiles() {
+        boost::filesystem::path path(storageGlobalParams.dbpath);
+        for ( boost::filesystem::directory_iterator i( path );
+                i != boost::filesystem::directory_iterator(); ++i ) {
+            string fileName = boost::filesystem::path(*i).leaf().string();
+            if ( boost::filesystem::is_directory( *i ) &&
+                    fileName.length() && fileName[ 0 ] == '$' )
+                boost::filesystem::remove_all( *i );
+        }
+    }
+} // namespace
+
+    MMAPV1Engine::MMAPV1Engine() {
+        // TODO check non-journal subdirs if using directory-per-db
+        checkReadAhead(storageGlobalParams.dbpath);
+
+        acquirePathLock(this, storageGlobalParams.repair);
+
+        FileAllocator::get()->start();
+
+        MONGO_ASSERT_ON_EXCEPTION_WITH_MSG( clearTmpFiles(), "clear tmp files" );
+
+        // dur::startup() depends on globalStorageEngine being set before calling.
+        // TODO clean up dur::startup() so this isn't needed.
+        invariant(!globalStorageEngine);
+        globalStorageEngine = this;
+
+        dur::startup();
+    }
+
     MMAPV1Engine::~MMAPV1Engine() {
     }
 
@@ -85,4 +345,45 @@ namespace mongo {
         return MongoFile::flushAll( sync );
     }
 
+    void MMAPV1Engine::cleanShutdown(OperationContext* txn) {
+        // wait until file preallocation finishes
+        // we would only hang here if the file_allocator code generates a
+        // synchronous signal, which we don't expect
+        log() << "shutdown: waiting for fs preallocator..." << endl;
+        FileAllocator::get()->waitUntilFinished();
+
+        if (storageGlobalParams.dur) {
+            log() << "shutdown: final commit..." << endl;
+            getDur().commitNow(txn);
+
+            flushAllFiles(true);
+        }
+
+        log() << "shutdown: closing all files..." << endl;
+        stringstream ss3;
+        MemoryMappedFile::closeAllFiles( ss3 );
+        log() << ss3.str() << endl;
+
+        if (storageGlobalParams.dur) {
+            dur::journalCleanup(true);
+        }
+
+#if !defined(__sunos__)
+        if ( lockFile ) {
+            log() << "shutdown: removing fs lock..." << endl;
+            /* This ought to be an unlink(), but Eliot says the last
+               time that was attempted, there was a race condition
+               with acquirePathLock().  */
+#ifdef _WIN32
+            if( _chsize( lockFile , 0 ) )
+                log() << "couldn't remove fs lock " << errnoWithDescription(_doserrno) << endl;
+            CloseHandle(lockFileHandle);
+#else
+            if( ftruncate( lockFile , 0 ) )
+                log() << "couldn't remove fs lock " << errnoWithDescription() << endl;
+            flock( lockFile, LOCK_UN );
+#endif
+        }
+#endif
+    }
 }
diff --git a/src/mongo/db/storage/mmap_v1/mmap_v1_engine.h b/src/mongo/db/storage/mmap_v1/mmap_v1_engine.h
index 2ea73273f2e..ca021729171 100644
--- a/src/mongo/db/storage/mmap_v1/mmap_v1_engine.h
+++ b/src/mongo/db/storage/mmap_v1/mmap_v1_engine.h
@@ -36,6 +36,7 @@ namespace mongo {
 
     class MMAPV1Engine : public StorageEngine {
     public:
+        MMAPV1Engine();
         virtual ~MMAPV1Engine();
 
         RecoveryUnit* newRecoveryUnit( OperationContext* opCtx );
@@ -50,6 +51,8 @@ namespace mongo {
         DatabaseCatalogEntry* getDatabaseCatalogEntry( OperationContext* opCtx,
                                                        const StringData& db );
 
+        void cleanShutdown(OperationContext* txn);
+
     private:
         static void _listDatabases( const std::string& directory,
                                     std::vector<std::string>* out );
diff --git a/src/mongo/db/storage/storage_engine.cpp b/src/mongo/db/storage/storage_engine.cpp
index 42d2a434372..26a8f036f7b 100644
--- a/src/mongo/db/storage/storage_engine.cpp
+++ b/src/mongo/db/storage/storage_engine.cpp
@@ -30,7 +30,6 @@
 
 #include "mongo/db/storage/storage_engine.h"
 
-#include "mongo/base/init.h"
 #include "mongo/db/storage_options.h"
 #include "mongo/db/storage/heap1/heap1_engine.h"
 #include "mongo/db/storage/mmap_v1/mmap_v1_engine.h"
@@ -50,7 +49,8 @@ namespace mongo {
         factorys[name] = factory;
     }
 
-    MONGO_INITIALIZER(StorageEngineInit) (InitializerContext* context) {
+    void initGlobalStorageEngine() {
+        // TODO these should use the StorageEngine::Factory system
         if ( storageGlobalParams.engine == "mmapv1" ) {
             globalStorageEngine = new MMAPV1Engine();
         }
@@ -59,13 +59,10 @@ namespace mongo {
         }
         else {
             const StorageEngine::Factory* factory = factorys[storageGlobalParams.engine];
-            if ( !factory ) {
-                error() << "unknown storage engine: " << storageGlobalParams.engine;
-                return Status( ErrorCodes::BadValue, "unknown storage engine" );
-            }
+            uassert(18525, "unknown storage engine: " + storageGlobalParams.engine,
+                    factory);
             globalStorageEngine = factory->create( storageGlobalParams );
         }
-        return Status::OK();
     }
 }
 
diff --git a/src/mongo/db/storage/storage_engine.h b/src/mongo/db/storage/storage_engine.h
index f75fde343e4..66fa954d8fa 100644
--- a/src/mongo/db/storage/storage_engine.h
+++ b/src/mongo/db/storage/storage_engine.h
@@ -44,7 +44,6 @@ namespace mongo {
 
     class StorageEngine {
     public:
-        virtual ~StorageEngine() {}
 
         virtual RecoveryUnit* newRecoveryUnit( OperationContext* opCtx ) = 0;
 
@@ -61,11 +60,18 @@ namespace mongo {
          */
         virtual int flushAllFiles( bool sync ) = 0;
 
-        virtual Status repairDatabase( OperationContext* tnx,
+        virtual Status repairDatabase( OperationContext* txn,
                                        const std::string& dbName,
                                        bool preserveClonedFilesOnFailure = false,
                                        bool backupOriginalFiles = false ) = 0;
 
+        /**
+         * Will be called before a clean shutdown.
+         * Override if you have clean-up to do that is different from unclean shutdown.
+         * There is intentionally no uncleanShutdown().
+         */
+        virtual void cleanShutdown(OperationContext* txn) {}
+
         class Factory {
         public:
             virtual ~Factory(){}
@@ -73,8 +79,20 @@ namespace mongo {
         };
 
         static void registerFactory( const std::string& name, const Factory* factory );
+
+    protected:
+        /**
+         * The destructor will never be called. See cleanShutdown instead.
+         */
+        virtual ~StorageEngine() {}
     };
 
+    /**
+     * Sets up the globalStorageEngine pointer and performs any startup work needed by the selected
+     * storage engine. This must be called at a point where it is safe to spawn worker threads.
+     */
+    void initGlobalStorageEngine();
+
     // TODO: this is temporary
     extern StorageEngine* globalStorageEngine;
 }
diff --git a/src/mongo/db/storage_options.h b/src/mongo/db/storage_options.h
index 70908e06800..b2e4de0d99c 100644
--- a/src/mongo/db/storage_options.h
+++ b/src/mongo/db/storage_options.h
@@ -49,6 +49,8 @@ namespace mongo {
             dbpath("/data/db/"),
 #endif
             directoryperdb(false),
+            upgrade(false),
+            repair(false),
             lenForNewNsFiles(16 * 1024 * 1024),
             preallocj(true),
             journalCommitInterval(0), // 0 means use default
@@ -70,6 +72,8 @@ namespace mongo {
         std::string engine;
         std::string dbpath;
         bool directoryperdb;
+        bool upgrade;
+        bool repair;
         std::string repairpath;
         unsigned lenForNewNsFiles;