SERVER-9333 clear ghostcache at reconfig time

5 files changed, 101 insertions, 2 deletions

diff --git a/jstests/replsets/tags_with_reconfig.js b/jstests/replsets/tags_with_reconfig.js
new file mode 100644
index 00000000000..df04f10067a
--- /dev/null
+++ b/jstests/replsets/tags_with_reconfig.js
@@ -0,0 +1,80 @@
+// Test for SERVER-9333
+// Previously, we were not clearing the cache of slaves in the primary at reconfig
+// time.  This would cause us to update stale items in the cache when secondaries
+// reported their progress to a primary.
+
+
+// Start a replica set with 3 nodes
+var host = getHostName();
+var replTest = new ReplSetTest( {name: "server9333", nodes: 3, startPort: 32000} );
+var nodes = replTest.startSet();
+var ports = replTest.ports;
+
+// Set tags and getLastErrorModes
+var conf = {_id : "server9333", version: 1, members : [
+    {_id : 0, host : host+":"+ports[0], tags : {"dc" : "bbb"}},
+    {_id : 1, host : host+":"+ports[1], tags : {"dc" : "bbb"}},
+    {_id : 2, host : host+":"+ports[2], tags : {"dc" : "ccc"}}],
+            settings : {getLastErrorModes : {
+                anydc : {dc : 1},
+                alldc : {dc : 2}, }} };
+
+
+replTest.initiate( conf );
+replTest.awaitReplication();
+
+
+master = replTest.getMaster();
+var db = master.getDB("test");
+
+// Insert a document and getLastError with write concern : anydc
+db.foo.insert( {x:1} );
+var result = db.runCommand( {getLastError:1, w:"anydc", wtimeout:20000} );
+printjson (result)
+assert.eq (result.err, null);
+
+// Insert a document and getLastError with write concern : alldc
+db.foo.insert( {x:2} );
+var result = db.runCommand( {getLastError:1, w:"alldc", wtimeout:20000} );
+printjson (result)
+assert.eq (result.err, null);
+
+
+// Add a new tag to the replica set
+var config = master.getDB("local").system.replset.findOne();
+printjson(config);
+var modes = config.settings.getLastErrorModes;
+config.version++;
+config.members[0].tags.newtag = "newtag";
+
+try {
+    master.getDB("admin").runCommand({replSetReconfig : config});
+}
+catch(e) {
+    print(e);
+}
+
+replTest.awaitReplication();
+
+// Print the new config for replica set
+var config = master.getDB("local").system.replset.findOne();
+printjson(config);
+
+
+master = replTest.getMaster();
+var db = master.getDB("test");
+
+// Insert a document and getLastError with write concern : anydc
+db.foo.insert( {x:3} );
+var result = db.runCommand( {getLastError:1, w:"anydc", wtimeout:20000} );
+printjson (result)
+assert.eq (result.err, null);
+
+// Insert a document and getLastError with write concern : alldc
+db.foo.insert( {x:4} );
+var result = db.runCommand( {getLastError:1, w:"alldc", wtimeout:1000} );
+printjson (result)
+assert.eq (result.err, null);
+
+
+replTest.stopSet();
diff --git a/src/mongo/db/client.cpp b/src/mongo/db/client.cpp
index 713e8da07a4..a3ec1c5c674 100644
--- a/src/mongo/db/client.cpp
+++ b/src/mongo/db/client.cpp
@@ -388,7 +388,7 @@ namespace mongo {
         _handshake = b.obj();
 
         if (theReplSet && o.hasField("member")) {
-            theReplSet->ghost->associateSlave(_remoteId, o["member"].Int());
+            theReplSet->registerSlave(_remoteId, o["member"].Int());
         }
     }
 
diff --git a/src/mongo/db/repl/rs.cpp b/src/mongo/db/repl/rs.cpp
index f6ef3dca558..1c135556c6f 100644
--- a/src/mongo/db/repl/rs.cpp
+++ b/src/mongo/db/repl/rs.cpp
@@ -584,7 +584,6 @@ namespace mongo {
         verify( _name.empty() || _name == config()._id );
         _name = config()._id;
         verify( !_name.empty() );
-
         // this is a shortcut for simple changes
         if( additive ) {
             log() << "replSet info : additive change to configuration" << rsLog;
@@ -613,6 +612,10 @@ namespace mongo {
         _members.orphanAll();
 
         endOldHealthTasks();
+        
+        // Clear out our memory of who might have been syncing from us.
+        // Any incoming handshake connections after this point will be newly registered.
+        ghost->clearCache();
 
         int oldPrimaryId = -1;
         {
@@ -907,6 +910,13 @@ namespace mongo {
         return OpTime();
     }
 
+    void ReplSetImpl::registerSlave(const BSONObj& rid, const int memberId) {
+        // To prevent race conditions with clearing the cache at reconfig time,
+        // we lock the replset mutex here.
+        lock lk(this);
+        ghost->associateSlave(rid, memberId);
+    }
+
     class ReplIndexPrefetch : public ServerParameter {
     public:
         ReplIndexPrefetch()
diff --git a/src/mongo/db/repl/rs.h b/src/mongo/db/repl/rs.h
index cd353752e82..1e7c86596c9 100644
--- a/src/mongo/db/repl/rs.h
+++ b/src/mongo/db/repl/rs.h
@@ -149,6 +149,7 @@ namespace mongo {
         void percolate(const BSONObj& rid, const OpTime& last);
         void associateSlave(const BSONObj& rid, const int memberId);
         void updateSlave(const mongo::OID& id, const OpTime& last);
+        void clearCache();
     };
 
     class Consensus {
@@ -489,6 +490,9 @@ namespace mongo {
          * have called it again, passing in false.
          */
         bool setMaintenanceMode(const bool inc);
+
+        // Records a new slave's id in the GhostSlave map, at handshake time.
+        void registerSlave(const BSONObj& rid, const int memberId);
     private:
         Member* head() const { return _members.head(); }
     public:
diff --git a/src/mongo/db/repl/rs_sync.cpp b/src/mongo/db/repl/rs_sync.cpp
index 7d43b3ac466..548b646c342 100644
--- a/src/mongo/db/repl/rs_sync.cpp
+++ b/src/mongo/db/repl/rs_sync.cpp
@@ -784,6 +784,11 @@ namespace replset {
         }
     }
 
+    void GhostSync::clearCache() {
+        rwlock lk(_lock, true);
+        _ghostCache.clear();
+    }
+
     void GhostSync::associateSlave(const BSONObj& id, const int memberId) {
         const OID rid = id["_id"].OID();
         rwlock lk( _lock , true );