SERVER-2500 private map was using too much memory on windows

7 files changed, 207 insertions, 39 deletions

diff --git a/db/dur.cpp b/db/dur.cpp
index cfc31643db4..4f19b105c80 100644
--- a/db/dur.cpp
+++ b/db/dur.cpp
@@ -123,10 +123,13 @@ namespace mongo {
         }
 
         void DurableImpl::setNoJournal(void *dst, void *src, unsigned len) {
+            MemoryMappedFile::makeWritable(dst, len);
+
             // we stay in this mutex for everything to work with DurParanoid/validateSingleMapMatches
             //
             // this also makes setNoJournal threadsafe, which is good as we call it from a read (not a write) lock 
             // in class SlaveTracking
+            //
             scoped_lock lk( privateViews._mutex() );
             size_t ofs;
             MongoMMF *f = privateViews.find_inlock(dst, ofs);
diff --git a/db/dur_commitjob.cpp b/db/dur_commitjob.cpp
index fdd61889517..aed38e8d5ef 100644
--- a/db/dur_commitjob.cpp
+++ b/db/dur_commitjob.cpp
@@ -21,6 +21,7 @@
 #include "taskqueue.h"
 
 namespace mongo {
+
     namespace dur {
 
         BOOST_STATIC_ASSERT( UncommittedBytesLimit > BSONObjMaxInternalSize * 3 );
@@ -144,6 +145,8 @@ namespace mongo {
             DEV dbMutex.assertWriteLocked();
             dassert( cmdLine.dur );
             if( !_wi._alreadyNoted.checkAndSet(p, len) ) {
+                MemoryMappedFile::makeWritable(p, len);
+
                 if( !_hasWritten ) {
                     // you can't be writing if one of these is pending, so this is a verification.
                     assert( !dbMutex._remapPrivateViewRequested );
@@ -183,8 +186,8 @@ namespace mongo {
                 assert(  _wi._writes.size() < 20000000 );
 
                 {
-                    // a bit over conservative
-                    static size_t lastPos;
+                    // a bit over conservative in counting pagebytes used
+                    static size_t lastPos; // note this doesn't reset with each commit, but that is ok we aren't being that precise
                     size_t x = ((size_t) p) & ~0xfff; // round off to page address (4KB)
                     if( x != lastPos ) { 
                         lastPos = x;
diff --git a/db/mongommf.cpp b/db/mongommf.cpp
index 4afb38c5c5a..d4be7d38361 100644
--- a/db/mongommf.cpp
+++ b/db/mongommf.cpp
@@ -31,6 +31,131 @@ using namespace mongoutils;
 
 namespace mongo {
 
+#if defined(_WIN32)

+    extern mutex mapViewMutex;
+
+    __declspec(noinline) void makeChunkWritable(size_t chunkno) { 

+        scoped_lock lk(mapViewMutex);
+

+        if( writable.get(chunkno) )

+            return;

+

+        size_t loc = chunkno * MemoryMappedFile::ChunkSize;

+        void *Loc = (void*) loc;

+        size_t ofs;

+        MongoMMF *mmf = privateViews.find( (void *) (loc), ofs );

+        MemoryMappedFile *f = (MemoryMappedFile*) mmf;

+        assert(f);

+

+        size_t len = MemoryMappedFile::ChunkSize;

+        assert( mmf->getView() <= Loc );

+        if( ofs + len > f->length() ) {

+            // at the very end of the map

+            len = f->length() - ofs;

+        }

+        else { 

+            ;

+        }

+

+        // todo: check this goes away on remap

+        DWORD old;

+        bool ok = VirtualProtect(Loc, len, PAGE_WRITECOPY, &old);

+        if( !ok ) {

+            DWORD e = GetLastError();

+            cout << "virtualprotect " << Loc << ' ' << len << ' ' << e << endl;

+            assert(false);

+        }

+

+        writable.set(chunkno);

+    }

+

+    // align so that there is only one map per chunksize so our bitset works right
+    void* mapaligned(HANDLE h, unsigned long long _len) {
+        void *loc = 0;
+        int n = 0;
+        while( 1 ) { 
+            n++;
+            void *m = MapViewOfFileEx(h, FILE_MAP_READ, 0, 0, 0, loc);
+            if( m == 0 ) {
+                DWORD e = GetLastError();
+                if( n == 0 ) { 
+                    // if first fails, it isn't going to work
+                    log() << "mapaligned errno: " << e << endl;
+                    break;
+                }
+                if( debug && n == 1 ) { 
+                    log() << "mapaligned info e:" << e << " at n=1" << endl;
+                }
+                if( n > 98 ) {
+                    log() << "couldn't align mapped view of file len:" << _len/1024.0/1024.0 << "MB errno:" << e << endl;
+                    break;
+                }
+                loc = (void*) (((size_t)loc)+MemoryMappedFile::ChunkSize);
+                continue;
+            }
+
+            size_t x = (size_t) m;
+            if( x % MemoryMappedFile::ChunkSize == 0 ) {
+                void *end = (void*) (x+_len);
+                log() << "mapaligned " << m << '-' << end << " len:" << _len << endl;
+                return m;
+            }
+
+            UnmapViewOfFile(m);
+            x = ((x+MemoryMappedFile::ChunkSize-1) / MemoryMappedFile::ChunkSize) * MemoryMappedFile::ChunkSize;
+            loc = (void*) x;
+            if( n % 20 == 0 ) { 
+                log() << "warning mapaligned n=20" << endl;
+            }
+            if( n > 100 ) {
+                log() << "couldn't align mapped view of file len:" << _len/1024.0/1024.0 << "MB" << endl;
+                break;
+            }
+        }
+        return 0;
+    }
+
+    void* MemoryMappedFile::createPrivateMap() {
+        assert( maphandle );
+        scoped_lock lk(mapViewMutex);
+        void *p = mapaligned(maphandle, len);
+        if ( p == 0 ) {
+            DWORD e = GetLastError();
+            log() << "createPrivateMap failed " << filename() << " " << errnoWithDescription(e) << endl;
+        }
+        else {
+            views.push_back(p);
+        }
+        return p;
+    }
+
+    void* MemoryMappedFile::remapPrivateView(void *oldPrivateAddr) {
+        // the mutex is to assure we get the same address on the remap
+        dbMutex.assertWriteLocked();
+
+        scoped_lock lk(mapViewMutex);
+
+        unmapped(oldPrivateAddr);
+
+        cout << "unmapping " << oldPrivateAddr << ' ' << len << endl;
+        bool ok = UnmapViewOfFile(oldPrivateAddr);
+        assert(ok);
+
+        // we want the new address to be the same as the old address in case things keep pointers around (as namespaceindex does).
+        void *p = MapViewOfFileEx(maphandle, FILE_MAP_READ, 0, 0,
+                                  /*dwNumberOfBytesToMap 0 means to eof*/0 /*len*/,
+                                  oldPrivateAddr);
+        
+        if ( p == 0 ) {
+            DWORD e = GetLastError();
+            log() << "MapViewOfFileEx failed " << filename() << " " << errnoWithDescription(e) << endl;
+            assert(p);
+        }
+        assert(p == oldPrivateAddr);
+        return p;
+    }
+#endif
+
     void MongoMMF::remapThePrivateView() {
         assert( cmdLine.dur );
 
@@ -183,7 +308,9 @@ namespace mongo {
         if( _view_write ) {
             if( cmdLine.dur ) {
                 _view_private = createPrivateMap();
-                massert( 13636 , "createPrivateMap failed (look in log for error)" , _view_private );
+                if( _view_private == 0 ) {
+                    massert( 13636 , "createPrivateMap failed (look in log for error)" , false );
+                }
                 privateViews.add(_view_private, this); // note that testIntent builds use this, even though it points to view_write then...
             }
             else {
diff --git a/db/namespace.h b/db/namespace.h
index f1ef0d6d047..4ec1eddabe1 100644
--- a/db/namespace.h
+++ b/db/namespace.h
@@ -293,7 +293,9 @@ namespace mongo {
          */
         IndexDetails& addIndex(const char *thisns, bool resetTransient=true);
 
-        void aboutToDeleteAnIndex() { flags &= ~Flag_HaveIdIndex;  }
+        void aboutToDeleteAnIndex() { 
+            *getDur().writing(&flags) = flags & ~Flag_HaveIdIndex;
+        }
 
         /* returns index of the first index in which the field is present. -1 if not present. */
         int fieldIsIndexed(const char *fieldName);
diff --git a/dbtests/mmaptests.cpp b/dbtests/mmaptests.cpp
index 88dacb8f5a5..7fb6eee98fc 100644
--- a/dbtests/mmaptests.cpp
+++ b/dbtests/mmaptests.cpp
@@ -52,6 +52,8 @@ namespace MMapTests {
                     char *p = (char *) f.getView();
                     assert(p);
                     // write something to the private view as a test
+                    if( cmdLine.dur ) 
+                        MemoryMappedFile::makeWritable(p, 6);
                     strcpy(p, "hello");
                 }
                 if( cmdLine.dur ) {
@@ -81,6 +83,8 @@ namespace MMapTests {
                 {
                     char *p = (char *) f.getView();
                     assert(p);
+                    if( cmdLine.dur ) 
+                        MemoryMappedFile::makeWritable(p, 4);
                     strcpy(p, "zzz");
                 }
                 if( cmdLine.dur ) {
diff --git a/util/mmap.h b/util/mmap.h
index 393d67183ea..4903d92cfa8 100644
--- a/util/mmap.h
+++ b/util/mmap.h
@@ -172,6 +172,14 @@ namespace mongo {
         void* createReadOnlyMap();
         void* createPrivateMap();
 
+        /** make the private map range writable (necessary for our windows implementation) */
+        static void makeWritable(void *, unsigned len)
+#if defined(_WIN32)
+            ;
+#else
+        { }
+#endif
+
     private:
         static void updateLength( const char *filename, unsigned long long &length );
 
@@ -182,6 +190,12 @@ namespace mongo {
 
 #ifdef _WIN32
         boost::shared_ptr<mutex> _flushMutex;
+        void unmapped(void *privateView);
+    public:
+        static const unsigned ChunkSize = 64 * 1024 * 1024;
+        static const unsigned NChunks = 1024 * 1024;
+#else
+        void unmapped(void *privateView) { }
 #endif
 
     protected:
@@ -203,4 +217,42 @@ namespace mongo {
             p(*i);
     }
 
+#if defined(_WIN32)    
+    class ourbitset {
+        volatile unsigned bits[MemoryMappedFile::NChunks]; // volatile as we are doing double check locking
+    public:
+        ourbitset() { 
+            memset((void*) bits, 0, sizeof(bits));
+        }
+        bool get(unsigned i) const { 
+            unsigned x = i / 32;
+            assert( x < MemoryMappedFile::NChunks );
+            return bits[x] & (1 << (i%32));
+        }
+        void set(unsigned i) { 
+            unsigned x = i / 32;
+            assert( x < MemoryMappedFile::NChunks );
+            bits[x] |= (1 << (i%32));
+        }
+        void clear(unsigned i) { 
+            unsigned x = i / 32;
+            assert( x < MemoryMappedFile::NChunks );
+            bits[x] &= ~(1 << (i%32));
+        }
+    };
+    extern ourbitset writable;
+    void makeChunkWritable(size_t chunkno);

+    inline void MemoryMappedFile::makeWritable(void *_p, unsigned len) {

+        size_t p = (size_t) _p;

+        unsigned a = p/ChunkSize;

+        unsigned b = (p+len)/ChunkSize;

+        for( unsigned i = a; i <= b; i++ ) {

+            if( !writable.get(a) ) {

+                makeChunkWritable(i);

+            }

+        }

+    }
+
+#endif
+
 } // namespace mongo
diff --git a/util/mmap_win.cpp b/util/mmap_win.cpp
index 5bbf49b5649..76837edd5c4 100644
--- a/util/mmap_win.cpp
+++ b/util/mmap_win.cpp
@@ -19,10 +19,21 @@
 #include "mmap.h"
 #include "text.h"
 #include <windows.h>
+#include "../db/mongommf.h"
+#include "../db/concurrency.h"
 
 namespace mongo {
 
     mutex mapViewMutex("mapView");
+    ourbitset writable;
+
+    /** notification on unmapping so we can clear writable bits */
+    void MemoryMappedFile::unmapped(void *p) {
+        for( unsigned i = ((size_t)p)/ChunkSize; i <= (((size_t)p)+len)/ChunkSize; i++ ) {
+            writable.clear(i);
+            assert( !writable.get(i) );
+        }
+    }
 
     MemoryMappedFile::MemoryMappedFile()
         : _flushMutex(new mutex("flushMutex")) {
@@ -34,6 +45,7 @@ namespace mongo {
 
     void MemoryMappedFile::close() {
         for( vector<void*>::iterator i = views.begin(); i != views.end(); i++ ) {
+            unmapped(*i);
             UnmapViewOfFile(*i);
         }
         views.clear();
@@ -47,41 +59,6 @@ namespace mongo {
 
     unsigned long long mapped = 0;
 
-    void* MemoryMappedFile::remapPrivateView(void *oldPrivateAddr) {
-        // the mutex is to assure we get the same address on the remap
-        scoped_lock lk(mapViewMutex);
-
-        bool ok = UnmapViewOfFile(oldPrivateAddr);
-        assert(ok);
-
-        // we want the new address to be the same as the old address in case things keep pointers around (as namespaceindex does).
-        void *p = MapViewOfFileEx(maphandle, FILE_MAP_COPY, 0, 0,
-                                  /*dwNumberOfBytesToMap 0 means to eof*/0 /*len*/,
-                                  oldPrivateAddr);
-        
-        if ( p == 0 ) {
-            DWORD e = GetLastError();
-            log() << "MapViewOfFileEx failed " << filename() << " " << errnoWithDescription(e) << endl;
-            assert(p);
-        }
-        assert(p == oldPrivateAddr);
-        return p;
-    }
-
-    void* MemoryMappedFile::createPrivateMap() {
-        assert( maphandle );
-        scoped_lock lk(mapViewMutex);
-        void *p = MapViewOfFile(maphandle, FILE_MAP_COPY, /*f ofs hi*/0, /*f ofs lo*/ 0, /*dwNumberOfBytesToMap 0 means to eof*/0);
-        if ( p == 0 ) {
-            DWORD e = GetLastError();
-            log() << "createPrivateMap failed " << filename() << " " << errnoWithDescription(e) << endl;
-        }
-        else {
-            views.push_back(p);
-        }
-        return p;
-    }
-
     void* MemoryMappedFile::createReadOnlyMap() {
         assert( maphandle );
         scoped_lock lk(mapViewMutex);