Merge branch 'master' of git@github.com:mongodb/mongo

8 files changed, 159 insertions, 66 deletions

diff --git a/db/clientcursor.cpp b/db/clientcursor.cpp
index cc7f3aea190..5abb24ff552 100644
--- a/db/clientcursor.cpp
+++ b/db/clientcursor.cpp
@@ -31,8 +31,6 @@
 
 namespace mongo {
 
-    /* TODO: FIX cleanup of clientCursors when hit the end. (ntoreturn insufficient) */
-
     CCById clientCursorsById;
 
     /* ------------------------------------------- */
diff --git a/db/clientcursor.h b/db/clientcursor.h
index 8a48fa36a38..531c16e4280 100644
--- a/db/clientcursor.h
+++ b/db/clientcursor.h
@@ -1,3 +1,5 @@
+/* clientcursor.h */
+
 /**
 *    Copyright (C) 2008 10gen Inc.
 *
@@ -14,9 +16,7 @@
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-/* clientcursor.h
-
-   Cursor -- and its derived classes -- are our internal cursors.
+/* Cursor -- and its derived classes -- are our internal cursors.
 
    ClientCursor is a wrapper that represents a cursorid from our database
    application's perspective.
@@ -28,14 +28,13 @@
 
 namespace mongo {
 
-    typedef long long CursorId;
-    class Cursor;
+    typedef long long CursorId; /* passed to the client so it can send back on getMore */
+    class Cursor; /* internal server cursor base class */
     class ClientCursor;
     typedef map<CursorId, ClientCursor*> CCById;
     extern CCById clientCursorsById;
 
     class ClientCursor {
-        friend class CursInspector;
         DiskLoc _lastLoc; // use getter and setter not this.
         static CursorId allocCursorId();
     public:
@@ -45,10 +44,9 @@ namespace mongo {
         ~ClientCursor();
         const CursorId cursorid;
         string ns;
-        //BSONObj pattern; // the query object
         auto_ptr<KeyValJSMatcher> matcher;
         auto_ptr<Cursor> c;
-        int pos;
+        int pos; /* # objects into the cursor so far */
         DiskLoc lastLoc() const {
             return _lastLoc;
         }
diff --git a/db/database.h b/db/database.h
index f71200fbd3b..4090917f008 100644
--- a/db/database.h
+++ b/db/database.h
@@ -49,6 +49,7 @@ namespace mongo {
             profileName = name + ".system.profile";
         }
         ~Database() {
+            btreeStore->closeFiles(name, path);
             int n = files.size();
             for ( int i = 0; i < n; i++ )
                 delete files[i];
diff --git a/db/rec.h b/db/rec.h
index ce5a5331753..2f506534621 100644
--- a/db/rec.h
+++ b/db/rec.h
@@ -34,6 +34,15 @@ public:
     virtual void drop(const char *ns) { 
         dropNS(ns);
     }
+
+    /* close datafiles associated with the db specified. */
+    virtual void closeFiles(string dbname, string path) {
+        /* as this is only used for indexes so far, and we are in the same 
+           PDFiles as the nonindex data, we just rely on them having been closed 
+           at the same time.  one day this may need to change.
+        */
+    }
+
 };
 
 /* An in memory RecStoreInterface implementation ----------------------------
diff --git a/db/reccache.cpp b/db/reccache.cpp
index 41fbd8f13a4..dd6b281bcdc 100644
--- a/db/reccache.cpp
+++ b/db/reccache.cpp
@@ -1,9 +1,13 @@
 // storage.cpp

 

 #include "stdafx.h"

+int aaa;

 #include "pdfile.h"

+

 #include "reccache.h"

+

 #include "rec.h"

+

 #include "db.h"

 

 namespace mongo {

@@ -32,6 +36,40 @@ void recCacheCloseAll() {
 

 int ndirtywritten;

 

+inline static string escape(const char *ns) {

+    char buf[256];

+    char *p = buf;

+    while( 1 ) {

+        if( *ns == '$' ) *p = '~';

+        else

+            *p = *ns;

+        if( *ns == 0 )

+            break;

+        p++; ns++;

+    }

+    assert( p - buf < (int) sizeof(buf) );

+    return buf;

+}

+

+inline static string unescape(const char *ns) {

+    char buf[256];

+    char *p = buf;

+    while( 1 ) {

+        if( *ns == '~' ) *p = '$';

+        else

+            *p = *ns;

+        if( *ns == 0 )

+            break;

+        p++; ns++;

+    }

+    assert( p - buf < (int) sizeof(buf) );

+    return buf;

+}

+

+string RecCache::directory() { 

+    return database->path;

+}

+

 /* filename format is 

 

      <n>-<ns>.idx

@@ -54,7 +92,7 @@ BasicRecStore* RecCache::_initStore(string fname) {
     const char *p = rest.c_str();

     const char *q = strstr(p, ".idx");

     assert( q );

-    string ns(p, q-p);

+    string escaped_ns(p, q-p);

 

     // arbitrary limit.  if you are hitting, we should use fewer files and put multiple 

     // indexes in a single file (which is easy to do)

@@ -64,12 +102,13 @@ BasicRecStore* RecCache::_initStore(string fname) {
         stores.resize(n+1);

     assert( stores[n] == 0 );

     BasicRecStore *rs = new BasicRecStore(n);

-    path pf(dbpath);

+    path pf(directory());

     pf /= fname;

     string full = pf.string();

     rs->init(full.c_str(), recsize);

     stores[n] = rs;

-    storesByNs[ns] = rs;

+    string ns = unescape(escaped_ns.c_str());

+    storesByNsKey[mknskey(ns.c_str())] = rs;

     return rs;

 }

 

@@ -82,7 +121,7 @@ BasicRecStore* RecCache::initStore(int n) {
     }

 

     /* this will be slow if there are thousands of files */

-    path dir(dbpath);

+    path dir(directory());

     directory_iterator end;
     try {
         directory_iterator i(dir);
@@ -101,7 +140,7 @@ BasicRecStore* RecCache::initStore(int n) {
         throw;
     }
     catch (...) {
-        string s = string("i/o error looking for .idx file in ") + dbpath;
+        string s = string("i/o error looking for .idx file in ") + directory();
         massert(s, false);
     }
     stringstream ss;
@@ -121,13 +160,12 @@ string RecCache::findStoreFilename(const char *_ns, bool& found) {
     { 

         stringstream ss;

         ss << '-';

-        ss << _ns;

-        assert( strchr(_ns, '$') == 0); // $ not good for filenames

+        ss << escape(_ns);

         ss << ".idx";

         namefrag = ss.str();

     }

 

-    path dir(dbpath);

+    path dir(directory());

     directory_iterator end;
     int nmax = -1;
     try {
@@ -150,7 +188,7 @@ string RecCache::findStoreFilename(const char *_ns, bool& found) {
         }
     }
     catch (...) {
-        string s = string("i/o error looking for .idx file in ") + dbpath;
+        string s = string("i/o error looking for .idx file in ") + directory();
         massert(s, false);
     }
 
@@ -161,7 +199,7 @@ string RecCache::findStoreFilename(const char *_ns, bool& found) {
     return ss.str();
 }

 

-void RecCache::initStoreByNs(const char *_ns) {

+void RecCache::initStoreByNs(const char *_ns, const string& nskey) {

     bool found;

     string fn = findStoreFilename(_ns, found);

     _initStore(fn);
@@ -175,6 +213,28 @@ inline void RecCache::writeIfDirty(Node *n) {
     }

 }

 

+void RecCache::closeFiles(string dbname, string path) { 

+    dassert( dbMutexInfo.isLocked() );

+    boostlock lk(rcmutex);

+

+    // first we write all dirty pages.  it is not easy to check which Nodes are for a particular

+    // db, so we just write them all.

+    writeDirty( dirtyl.begin(), true );

+

+    string key = path + dbname + '.';

+    unsigned sz = key.size();

+    for( map<string, BasicRecStore*>::iterator i = storesByNsKey.begin(); i != storesByNsKey.end(); i++ ) { 

+        map<string, BasicRecStore*>::iterator j = i;

+        i++;

+        if( strncmp(j->first.c_str(), key.c_str(), sz) == 0 ) {

+            assert( stores[j->second->fileNumber] != 0 );

+            stores[j->second->fileNumber] = 0;

+            delete j->second;

+            storesByNsKey.erase(j);

+        }

+    }

+}

+

 void RecCache::closing() { 

     boostlock lk(rcmutex);

     (cout << "TEMP: recCacheCloseAll() writing dirty pages...\n").flush();

@@ -274,26 +334,33 @@ void RecCache::dump() {
 //    cout << endl;

 }

 

+/* cleans up everything EXCEPT storesByNsKey.

+   note this function is slow should not be invoked often

+*/

 void RecCache::closeStore(BasicRecStore *rs) { 

+    int n = rs->fileNumber + Base;

     for( set<DiskLoc>::iterator i = dirtyl.begin(); i != dirtyl.end(); ) { 

         DiskLoc k = *i++;

-        if( k.a() == rs->fileNumber )

+        if( k.a() == n )

             dirtyl.erase(k);

     }

 

     for( map<DiskLoc,Node*>::iterator i = m.begin(); i != m.end(); ) { 

         DiskLoc k = i->first;

         i++;

-        if( k.a() == rs->fileNumber )

+        if( k.a() == n )

             m.erase(k);

     }

 

+    assert( stores[rs->fileNumber] != 0 );

+    stores[rs->fileNumber] = 0;

+/*

     for( unsigned i = 0; i < stores.size(); i++ ) { 

         if( stores[i] == rs ) { 

             stores[i] = 0;

             break;

         }

-    }

+    }*/

     delete rs; // closes file

 }

 

@@ -301,39 +368,27 @@ void RecCache::drop(const char *_ns) {
     // todo: test with a non clean shutdown file

     boostlock lk(rcmutex);

 

-    char buf[256];

-    {

-        const char *ns = _ns;

-        char *p = buf;

-        while( 1 ) {

-            if( *ns == '$' ) *p = '_';

-            else

-                *p = *ns;

-            if( *ns == 0 )

-                break;

-            p++; ns++;

-        }

-        assert( p - buf < (int) sizeof(buf) );

-    }

-    BasicRecStore *&rs = storesByNs[buf];

+    map<string, BasicRecStore*>::iterator it = storesByNsKey.find(mknskey(_ns));

     string fname;

-    if( rs ) {

-        fname = rs->filename;

-        closeStore(rs);

-        rs = 0;

+    if( it != storesByNsKey.end() ) {

+        fname = it->second->filename;

+        closeStore(it->second); // cleans up stores[] etc.

+        storesByNsKey.erase(it);

     }

     else { 

         bool found;

-        fname = findStoreFilename(buf, found);

+        fname = findStoreFilename(_ns, found);

         if( !found ) { 

             log() << "RecCache::drop: no idx file found for " << _ns << endl;

             return;

         }

-        path pf(dbpath);

+        path pf(directory());

         pf /= fname;

         fname = pf.string();

     }

-    try { 

+    try {

+        if( !boost::filesystem::exists(fname) ) 

+            log() << "RecCache::drop: can't find file to remove " << fname << endl;

         boost::filesystem::remove(fname);

     } 

     catch(...) { 

diff --git a/db/reccache.h b/db/reccache.h
index 871ab582935..3e12a6fab85 100644
--- a/db/reccache.h
+++ b/db/reccache.h
@@ -31,51 +31,48 @@ class RecCache {
         char *data;

         DiskLoc loc;

         bool dirty;

-        Node *older, *newer;

+        Node *older, *newer; // lru

     };

     boost::mutex rcmutex; // mainly to coordinate with the lazy writer thread

     unsigned recsize;

-    map<DiskLoc, Node*> m;

+    map<DiskLoc, Node*> m; // the cache

     Node *newest, *oldest;

     unsigned nnodes;

     set<DiskLoc> dirtyl;

-    vector<BasicRecStore*> stores;

-    map<string, BasicRecStore*> storesByNs;

+    vector<BasicRecStore*> stores; // DiskLoc::a() indicates the index into this vector

+    map<string, BasicRecStore*> storesByNsKey; // nskey -> BasicRecStore*

 public:

     enum { Base = 10000 };

 private:

     BasicRecStore* _initStore(string fname);

     BasicRecStore* initStore(int n);

     string findStoreFilename(const char *_ns, bool& found);

-    void initStoreByNs(const char *escaped_ns);

+    void initStoreByNs(const char *ns, const string& nskey);

     void closeStore(BasicRecStore *rs);

 

+    static string directory();

+    static string mknskey(const char *ns) { 

+        return directory() + ns;

+    }

+

     /* get the right file for a given diskloc */

     BasicRecStore& store(DiskLoc& d) { 

         int n = d.a() - Base;

         if( (int) stores.size() > n ) { 

             BasicRecStore *rs = stores[n];

-            if( rs ) 

+            if( rs ) {

+                assert( rs->fileNumber == n );

                 return *rs;

+            }

         }

         return *initStore(n);

     }

     BasicRecStore& store(const char *ns) {

-        char buf[256];

-        char *p = buf;

-        while( 1 ) {

-            if( *ns == '$' ) *p = '_';

-            else

-                *p = *ns;

-            if( *ns == 0 )

-                break;

-            p++; ns++;

-        }

-        assert( p - buf < (int) sizeof(buf) );

-        BasicRecStore *&rs = storesByNs[buf];

+        string nskey = mknskey(ns);

+        BasicRecStore *&rs = storesByNsKey[nskey];

         if( rs )

             return *rs;

-        initStoreByNs(buf);

+        initStoreByNs(ns, nskey);

         return *rs;

     }

 

@@ -117,9 +114,9 @@ private:
     void dump();

 

 public:

-    /* all public functions (except constructor) use the mutex */

+    /* all public functions (except constructor) should use the mutex */

 

-    RecCache(unsigned sz) : recsize(sz) { 

+    RecCache(unsigned recsz) : recsize(recsz) { 

         nnodes = 0;

         newest = oldest = 0;

     }

@@ -184,6 +181,9 @@ public:
         return d;

     }

 

+    void closeFiles(string dbname, string path);

+

+    // at termination: write dirty pages and close all files

     void closing();

 };

 

@@ -207,6 +207,11 @@ public:
     virtual void drop(const char *ns) { 
         theRecCache.drop(ns);
     }
+
+    /* close datafiles associated with the db specified. */
+    virtual void closeFiles(string dbname, string path) {
+        theRecCache.closeFiles(dbname, dbpath);
+    }
 };
 

 } /*namespace*/

diff --git a/db/reci.h b/db/reci.h
index a93aed7e24a..4c870f6c434 100644
--- a/db/reci.h
+++ b/db/reci.h
@@ -27,6 +27,9 @@ public:
     /* drop the collection */
     virtual void drop(const char *ns) = 0;
 
+    /* close datafiles associated with the db specified. */
+    virtual void closeFiles(string dbname, string path) = 0;
+
     /* todo add: 
        closeFiles(dbname)
        eraseFiles(dbname)
diff --git a/jstests/recstore.js b/jstests/recstore.js
new file mode 100644
index 00000000000..f2e78e20731
--- /dev/null
+++ b/jstests/recstore.js
@@ -0,0 +1,24 @@
+// recstore.js
+// this is a simple test for new recstores (see reci.h)
+// it is probably redundant with other tests but is a convenient starting point
+// for testing such things.
+
+t = db.storetest;
+
+t.drop();
+
+t.save({z:3});
+t.save({z:2});
+
+t.ensureIndex({z:1});
+t.ensureIndex({q:1});
+assert( t.find().sort({z:1})[0].z == 2 );
+
+t.dropIndexes();
+
+assert( t.find().sort({z:1})[0].z == 2 );
+
+t.ensureIndex({z:1});
+t.ensureIndex({q:1});
+
+db.getSisterDB('admin').$cmd.findOne({closeAllDatabases:1});