Tests for replica set and cluster upgrade

8 files changed, 751 insertions, 0 deletions

diff --git a/jstests/libs/test_background_ops.js b/jstests/libs/test_background_ops.js
new file mode 100644
index 00000000000..91f50aaa362
--- /dev/null
+++ b/jstests/libs/test_background_ops.js
@@ -0,0 +1,340 @@
+//
+// Utilities related to background operations while other operations are working
+//
+
+/**
+ * Allows synchronization between background ops and the test operations
+ */
+var waitForLock = function( mongo, name ){
+    
+    var ts = new ObjectId()
+    var lockColl = mongo.getCollection( "config.testLocks" )
+    
+    lockColl.update({ _id : name, state : 0 }, { $set : { state : 0 } }, true)
+    
+    //
+    // Wait until we can set the state to 1 with our id
+    //
+    
+    var startTime = new Date().getTime()
+    
+    assert.soon( function() {
+        lockColl.update({ _id : name, state : 0 }, { $set : { ts : ts, state : 1 } })
+        var gleObj = lockColl.getDB().getLastErrorObj()
+        
+        if( new Date().getTime() - startTime > 20 * 1000 ){
+            print( "Waiting for..." )
+            printjson( gleObj )
+            printjson( lockColl.findOne() )
+            printjson( ts )
+        }
+        
+        return gleObj.n == 1 || gleObj.updatedExisting
+    }, "could not acquire lock", 30 * 1000, 100 )
+    
+    print( "Acquired lock " + tojson( { _id : name, ts : ts } ) + " curr : " + 
+           tojson( lockColl.findOne({ _id : name }) ) )
+    
+    // Set the state back to 0
+    var unlock = function(){
+        print( "Releasing lock " + tojson( { _id : name, ts : ts } ) + " curr : " + 
+                tojson( lockColl.findOne({ _id : name }) ) )
+        lockColl.update({ _id : name, ts : ts }, { $set : { state : 0 } })
+    }
+    
+    // Return an object we can invoke unlock on
+    return { unlock : unlock }
+}
+
+/**
+ * Allows a test or background op to say it's finished
+ */
+var setFinished = function( mongo, name, finished ){
+    if( finished || finished == undefined )
+        mongo.getCollection( "config.testFinished" ).update({ _id : name }, { _id : name }, true )
+    else
+        mongo.getCollection( "config.testFinished" ).remove({ _id : name })
+}
+
+/**
+ * Checks whether a test or background op is finished
+ */
+var isFinished = function( mongo, name ){
+    return mongo.getCollection( "config.testFinished" ).findOne({ _id : name }) != null 
+}
+
+/**
+ * Sets the result of a background op
+ */
+var setResult = function( mongo, name, result, err ){
+    mongo.getCollection( "config.testResult" ).update({ _id : name }, { _id : name, result : result, err : err }, true )
+}
+
+/**
+ * Gets the result for a background op
+ */
+var getResult = function( mongo, name ){
+    return mongo.getCollection( "config.testResult" ).findOne({ _id : name })
+}
+
+/**
+ * Overrides the parallel shell code in mongo
+ */
+function startParallelShell( jsCode, port ){
+    
+    var x;
+    if ( port ) {
+        x = startMongoProgramNoConnect( "mongo" , "--port" , port , "--eval" , jsCode );
+    } else {
+        x = startMongoProgramNoConnect( "mongo" , "--eval" , jsCode , db ? db.getMongo().host : null );        
+    }
+    
+    return function(){
+        jsTestLog( "Waiting for shell " + x + "..." )
+        waitProgram( x );
+        jsTestLog( "Shell " + x + " finished." )
+    };
+}
+
+startParallelOps = function( mongo, proc, args, context ){
+                 
+    var procName = proc.name + "-" + new ObjectId()
+    var seed = new ObjectId( new ObjectId().valueOf().split("").reverse().join("") )
+                                                                           .getTimestamp().getTime()
+       
+    // Make sure we aren't finished before we start
+    setFinished( mongo, procName, false )
+    setResult( mongo, procName, undefined, undefined )
+    
+    // TODO: Make this a context of its own
+    var procContext = { procName : procName,
+                        seed : seed,
+                        waitForLock : waitForLock,
+                        setFinished : setFinished,
+                        isFinished : isFinished,
+                        setResult : setResult,
+                        
+                        setup : function( context, stored ){
+                            
+                            waitForLock = function(){
+                                return context.waitForLock( db.getMongo(), context.procName )
+                            }
+                            setFinished = function( finished ){
+                                return context.setFinished( db.getMongo(), context.procName, finished )
+                            }
+                            isFinished = function(){
+                                return context.isFinished( db.getMongo(), context.procName )
+                            }
+                            setResult = function( result, err ){
+                                return context.setResult( db.getMongo(), context.procName, result, err )
+                            }
+                        }}
+                    
+    var bootstrapper = function( stored ){
+                
+        var procContext = stored.procContext        
+        procContext.setup( procContext, stored )
+        
+        var contexts = stored.contexts                                            
+        eval( "contexts = " + contexts )
+        
+        for( var i = 0; i < contexts.length; i++ ){
+            if( typeof( contexts[i] ) != "undefined" ){                
+                // Evaluate all contexts
+                contexts[i]( procContext )
+            }
+        }
+        
+        var operation = stored.operation
+        eval( "operation = " + operation )
+        
+        var args = stored.args
+        eval( "args = " + args )
+        
+        result = undefined
+        err = undefined
+        
+        try{
+            result = operation.apply( null, args )
+        }
+        catch( e ){
+            err = e 
+        }
+        
+        setResult( result, err )
+    }
+    
+    var contexts = [ RandomFunctionContext, context ]
+    
+    var testDataColl = mongo.getCollection( "config.parallelTest" )
+    
+    testDataColl.insert({ _id : procName,
+                          bootstrapper : tojson( bootstrapper ),
+                          operation : tojson( proc ),
+                          args : tojson( args ),
+                          procContext : procContext,
+                          contexts : tojson( contexts ) })
+                          
+    assert.eq( null, testDataColl.getDB().getLastError() )
+    
+    var bootstrapStartup =
+        "{ var procName = '" + procName + "'; " +    
+          "var stored = db.getMongo().getCollection( '" + testDataColl + "' )" + 
+                         ".findOne({ _id : procName }); " +
+          "var bootstrapper = stored.bootstrapper; " +
+          "eval( 'bootstrapper = ' + bootstrapper ); " +
+          "bootstrapper( stored ); " +
+        "}"
+    
+          
+    var oldDB = db
+    db = mongo.getDB( "test" )
+    
+    jsTest.log( "Starting " + proc.name + " operations..." )
+    
+    var rawJoin = startParallelShell( bootstrapStartup )
+                               
+    db = oldDB
+          
+    
+    var join = function(){ 
+        setFinished( mongo, procName, true )
+        
+        rawJoin();
+        result = getResult( mongo, procName )
+                           
+        assert.neq( result, null )                 
+                           
+        if( result.err ) throw "Error in parallel ops " + procName + " : " 
+                                                        + tojson( result.err )
+                                                        
+        else return result.result
+    }
+    
+    join.isFinished = function(){
+        return isFinished( mongo, procName )
+    }
+    
+    join.setFinished = function( finished ){
+        return setFinished( mongo, procName, finished )
+    }
+    
+    join.waitForLock = function( name ){
+        return waitForLock( mongo, name )
+    }
+    
+    return join
+}
+
+var RandomFunctionContext = function( context ){
+    
+    Random.srand( context.seed );
+    
+    Random.randBool = function(){ return Random.rand() > 0.5 }
+    
+    Random.randInt = function( min, max ){ 
+        
+        if( max == undefined ){
+            max = min
+            min = 0
+        }
+        
+        return min + Math.floor( Random.rand() * max ) 
+    }
+    
+    Random.randShardKey = function(){
+        
+        var numFields = 2 //Random.randInt(1, 3)
+        
+        var key = {}
+        for( var i = 0; i < numFields; i++ ){
+            var field = String.fromCharCode( "a".charCodeAt() + i )
+            key[ field ] = 1
+        }
+        
+        return key
+    }
+    
+    Random.randShardKeyValue = function( shardKey ){
+        
+        var keyValue = {}
+        for( field in shardKey ){
+            keyValue[ field ] = Random.randInt(1, 100)
+        }
+        
+        return keyValue
+    }
+    
+    Random.randCluster = function(){
+
+        var numShards = 2 //Random.randInt( 1, 10 )
+        var rs = false //Random.randBool()
+        var st = new ShardingTest({ shards : numShards, 
+                                    mongos : 4, 
+                                    other : { separateConfig : true, rs : rs } })
+        
+        return st
+    }
+}
+
+
+//
+// Some utility operations
+//
+
+function moveOps( collName, options ){
+    
+    options = options || {}
+    
+    var admin = db.getMongo().getDB( "admin" )
+    var config = db.getMongo().getDB( "config" )
+    var shards = config.shards.find().toArray()
+    var shardKey = config.collections.findOne({ _id : collName }).key
+    
+    while( ! isFinished() ){
+        
+        var findKey = Random.randShardKeyValue( shardKey )
+        var toShard = shards[ Random.randInt( shards.length ) ]._id
+        
+        try {
+            printjson( admin.runCommand({ moveChunk : collName, 
+                                          find : findKey,
+                                          to : toShard }) )
+        }
+        catch( e ){
+            printjson( e )
+        }
+        
+        sleep( 1000 )
+    }
+    
+    jsTest.log( "Stopping moveOps..." )
+}
+
+function splitOps( collName, options ){
+        
+    options = options || {}
+    
+    var admin = db.getMongo().getDB( "admin" )
+    var config = db.getMongo().getDB( "config" )
+    var shards = config.shards.find().toArray()
+    var shardKey = config.collections.findOne({ _id : collName }).key
+    
+    while( ! isFinished() ){
+        
+        var middleKey = Random.randShardKeyValue( shardKey )
+        
+        try {
+            printjson( admin.runCommand({ split : collName, 
+                                          middle : middleKey }) )
+        }
+        catch( e ){
+            printjson( e )
+        }
+        
+        sleep( 1000 )
+    }
+    
+    jsTest.log( "Stopping splitOps..." )
+}
+
diff --git a/jstests/multiVersion/1_test_launching.js b/jstests/multiVersion/0_test_launching.js
index 061ee2223d0..061ee2223d0 100644
--- a/jstests/multiVersion/1_test_launching.js
+++ b/jstests/multiVersion/0_test_launching.js
diff --git a/jstests/multiVersion/2_test_launching_replset.js b/jstests/multiVersion/1_test_launching_replset.js
index a71e4b276c0..a71e4b276c0 100644
--- a/jstests/multiVersion/2_test_launching_replset.js
+++ b/jstests/multiVersion/1_test_launching_replset.js
diff --git a/jstests/multiVersion/0_test_launching_cluster.js b/jstests/multiVersion/2_test_launching_cluster.js
index ed23ff27eff..ed23ff27eff 100644
--- a/jstests/multiVersion/0_test_launching_cluster.js
+++ b/jstests/multiVersion/2_test_launching_cluster.js
diff --git a/jstests/multiVersion/3_upgrade_replset.js b/jstests/multiVersion/3_upgrade_replset.js
new file mode 100644
index 00000000000..287477ba5c6
--- /dev/null
+++ b/jstests/multiVersion/3_upgrade_replset.js
@@ -0,0 +1,86 @@
+//
+// Tests upgrading a replica set
+//
+
+load( './jstests/multiVersion/libs/multi_rs.js' )
+load( './jstests/libs/test_background_ops.js' )
+
+var oldVersion = "2.0.6"
+var newVersion = "latest"
+
+var nodes = { n1 : { binVersion : oldVersion }, 
+              n2 : { binVersion : oldVersion },
+              a3 : { binVersion : oldVersion } }
+
+var rst = new ReplSetTest({ nodes : nodes })
+
+rst.startSet()
+rst.initiate()
+
+// Wait for a primary node...
+var primary = rst.getPrimary()
+var otherOpConn = new Mongo( rst.getURL() )
+var insertNS = "test.foo"
+
+
+jsTest.log( "Starting parallel operations during upgrade..." )
+
+function findAndInsert( rsURL, coll ){
+    
+    var coll = new Mongo( rsURL ).getCollection( coll + "" )
+    var count = 0
+    
+    jsTest.log( "Starting finds and inserts..." )
+    
+    while( ! isFinished() ){
+        
+        try{
+            
+            coll.insert({ _id : count, hello : "world" })
+            assert.eq( null, coll.getDB().getLastError() )
+            assert.neq( null, coll.findOne({ _id : count }) )
+        }
+        catch( e ){
+            printjson( e )
+        }
+        
+        count++
+    }
+    
+    jsTest.log( "Finished finds and inserts..." )
+    return count
+}
+
+var joinFindInsert = 
+    startParallelOps( primary, // The connection where the test info is passed and stored
+                      findAndInsert,
+                      [ rst.getURL(), insertNS ] )
+
+
+jsTest.log( "Upgrading replica set..." )
+
+rst.upgradeSet( "latest" )
+
+jsTest.log( "Replica set upgraded." )
+
+// Wait for primary
+var primary = rst.getPrimary()
+
+printjson( rst.status() )
+
+
+// Allow more valid writes to go through
+sleep( 10 * 1000 )
+
+
+joinFindInsert()
+
+var totalInserts = primary.getCollection( insertNS ).find().sort({ _id : -1 }).next()._id
+var dataFound = primary.getCollection( insertNS ).count()
+
+jsTest.log( "Found " + dataFound + " docs out of " + tojson( totalInserts ) + " inserted." )
+
+assert.gt( dataFound / totalInserts, 0.5 )
+
+rst.stopSet()
+
diff --git a/jstests/multiVersion/4_upgrade_cluster.js b/jstests/multiVersion/4_upgrade_cluster.js
new file mode 100644
index 00000000000..7f7a60d1536
--- /dev/null
+++ b/jstests/multiVersion/4_upgrade_cluster.js
@@ -0,0 +1,145 @@
+//
+// Upgrades a cluster to a newer version
+//
+
+load( './jstests/multiVersion/libs/multi_rs.js' )
+load( './jstests/multiVersion/libs/multi_cluster.js' )
+load( './jstests/libs/test_background_ops.js' )
+
+var oldVersion = "2.0.6"
+var newVersion = "latest"
+
+// BIG OUTER LOOP, RS CLUSTER OR NOT!
+for( var test = 0; test < 1; test++ ){
+    
+// TODO: RS Test messes up here
+var isRSCluster = test == 1
+
+
+
+jsTest.log( "Starting " + ( isRSCluster ? "(replica set)" : "" ) + " cluster..." )
+
+var options = {
+    
+    mongosOptions : { binVersion : oldVersion },
+    configOptions : { binVersion : oldVersion },
+    shardOptions : { binVersion : oldVersion },
+    
+    separateConfig : true,
+    sync : true,
+    rs : isRSCluster
+}
+
+var st = new ShardingTest({ shards : 2, mongos : 2, other : options })
+
+
+jsTest.log( "Starting parallel operations during upgrade..." )
+
+var insertNS = "test.foo"
+var shardedInsertNS = "test.bar"
+    
+var admin = st.s.getDB( "admin" )
+var shards = st.s.getDB( "config" ).shards.find().toArray()
+
+printjson( admin.runCommand({ enableSharding : shardedInsertNS }) )
+printjson( admin.runCommand({ movePrimary : shardedInsertNS, to : shards[0]._id }) )
+printjson( admin.runCommand({ shardCollection : shardedInsertNS, key : { _id : 1 } }) )
+
+st.stopBalancer()
+
+for( var i = 0; i < 5; i++ ){
+    printjson( admin.runCommand({ split : shardedInsertNS, middle : { _id : i * 50 } }) )
+    printjson( admin.runCommand({ moveChunk : shardedInsertNS, 
+                                  find : { _id : i * 50 }, 
+                                  to : shards[ i % shards.length ]._id }) )
+}
+
+function findAndInsert( mongosURL, ns ){
+    
+    var coll = null
+    
+    // Make sure we can eventually connect to the mongos
+    assert.soon( function(){
+        try{ 
+            coll = new Mongo( mongosURL ).getCollection( ns + "" )
+            return true
+        }
+        catch( e ){
+            printjson( e )
+            return false
+        }
+    })
+    
+    var count = 0
+    
+    jsTest.log( "Starting finds and inserts..." )
+    
+    while( ! isFinished() ){
+        
+        try{
+            
+            coll.insert({ _id : count, hello : "world" })
+            assert.eq( null, coll.getDB().getLastError() )
+            assert.neq( null, coll.findOne({ _id : count }) )
+        }
+        catch( e ){
+            printjson( e )
+        }
+        
+        count++
+    }
+    
+    jsTest.log( "Finished finds and inserts..." )
+    return count
+}
+
+var staticMongod = MongoRunner.runMongod({})
+
+printjson( staticMongod )
+
+var joinFindInsert = 
+    startParallelOps( staticMongod, // The connection where the test info is passed and stored
+                      findAndInsert,
+                      [ st.s0.host, insertNS ] )
+                      
+var joinShardedFindInsert = 
+    startParallelOps( staticMongod, // The connection where the test info is passed and stored
+                      findAndInsert,
+                      [ st.s1.host, shardedInsertNS ] )
+
+
+jsTest.log( "Upgrading cluster..." )
+
+st.upgradeCluster( newVersion )
+
+jsTest.log( "Cluster upgraded." )
+
+st.printShardingStatus()
+
+
+// Allow more valid writes to go through
+sleep( 10 * 1000 )
+
+
+joinFindInsert()
+joinShardedFindInsert()
+
+var totalInserts = st.s.getCollection( insertNS ).find().sort({ _id : -1 }).next()._id
+var dataFound = st.s.getCollection( insertNS ).count()
+
+jsTest.log( "Found " + dataFound + " docs out of " + totalInserts + " inserted." )
+
+assert.gt( dataFound / totalInserts, 0.5 )
+
+var totalInserts = st.s.getCollection( shardedInsertNS ).find().sort({ _id : -1 }).next()._id
+var dataFound = st.s.getCollection( shardedInsertNS ).find().itcount()
+
+jsTest.log( "Found " + dataFound + " sharded docs out of " + tojson( totalInserts ) + " inserted." )
+
+assert.gt( dataFound / totalInserts, 0.5 )
+
+jsTest.log( "DONE!" )
+
+st.stop()
+
+} // END OUTER LOOP FOR RS CLUSTER
\ No newline at end of file
diff --git a/jstests/multiVersion/libs/multi_cluster.js b/jstests/multiVersion/libs/multi_cluster.js
new file mode 100644
index 00000000000..ae789cb38e6
--- /dev/null
+++ b/jstests/multiVersion/libs/multi_cluster.js
@@ -0,0 +1,90 @@
+//
+// MultiVersion utility functions for clusters
+//
+
+ShardingTest.prototype.upgradeCluster = function( binVersion, options ){
+    
+    options = options || {}
+    if( options.upgradeShards == undefined ) options.upgradeShards = true
+    if( options.upgradeConfigs == undefined ) options.upgradeConfigs = true
+    if( options.upgradeMongos == undefined ) options.upgradeMongos = true
+    
+    if( options.upgradeMongos ){
+        
+        // Upgrade all mongos hosts if specified
+        
+        var numMongoses = this._mongos.length
+        
+        for( var i = 0; i < numMongoses; i++ ){
+            
+            var mongos = this._mongos[i]
+            
+            MongoRunner.stopMongos( mongos )
+            
+            mongos = MongoRunner.runMongos({ restart : mongos, binVersion : binVersion })
+            
+            this[ "s" + i ] = this._mongos[i] = mongos
+            if( i == 0 ) this.s = mongos
+        }
+        
+        this.config = this.s.getDB( "config" )
+        this.admin = this.s.getDB( "admin" )
+    }
+    
+    var upgradedSingleShards = []
+    
+    if( options.upgradeShards ){
+        
+        var numShards = this._connections.length
+        
+        // Upgrade shards
+        for( var i = 0; i < numShards; i++ ){
+            
+            if( this._rs && this._rs[i] ){
+                                
+                // Upgrade replica set
+                var rst = this._rs[i].test
+                
+                rst.upgradeSet( binVersion )
+            }
+            else {
+                
+                // Upgrade shard
+                var shard = this._connections[i]
+                
+                MongoRunner.stopMongod( shard )
+                
+                shard = MongoRunner.runMongod({ restart : shard, binVersion : binVersion })
+                    
+                upgradedSingleShards[ shard.host ] = shard
+                
+                this[ "shard" + i ] = this[ "d" + i ] = this._connections[i] = shard
+            }        
+        }
+    }
+    
+    if( options.upgradeConfigs ){
+        
+        // Upgrade config servers if they aren't already upgraded shards 
+        var numConfigs = this._configServers.length
+        
+        for( var i = 0; i < numConfigs; i++ ){
+            
+            var configSvr = this._configServers[i]
+            
+            if( configSvr.host in upgradedSingleShards ){
+                
+                configSvr = upgradedSingleShards[ configSvr.host ]
+            }
+            else{
+                
+                MongoRunner.stopMongod( configSvr )
+                
+                configSvr = MongoRunner.runMongod({ restart : configSvr, binVersion : binVersion })
+            }
+            
+            this[ "config" + i ] = this[ "c" + i ] = this._configServers[i] = configSvr
+        }
+    }
+    
+}
\ No newline at end of file
diff --git a/jstests/multiVersion/libs/multi_rs.js b/jstests/multiVersion/libs/multi_rs.js
new file mode 100644
index 00000000000..6ff098201f7
--- /dev/null
+++ b/jstests/multiVersion/libs/multi_rs.js
@@ -0,0 +1,90 @@
+//
+// Utility functions for multi-version replica sets
+// 
+
+ReplSetTest.prototype.upgradeSet = function( binVersion, options ){
+    
+    options = options || {}
+    if( options.primaryStepdown == undefined ) options.primaryStepdown = true
+    
+    var nodes = this.nodes
+    var primary = this.getPrimary()
+    
+    // Upgrade secondaries first
+    var nodesToUpgrade = this.getSecondaries()
+    
+    // Then upgrade primaries
+    nodesToUpgrade.push( primary )
+    
+    // We can upgrade with no primary downtime if we have enough nodes
+    var noDowntimePossible = nodes.length > 2
+    
+    for( var i = 0; i < nodesToUpgrade.length; i++ ){
+        
+        var node = nodesToUpgrade[ i ]
+        
+        if( node == primary && options.primaryStepdown ){
+            
+            node = this.stepdown( node )
+            primary = this.getPrimary()
+        }
+        
+        var prevPrimaryId = this.getNodeId( primary )
+        
+        this.upgradeNode( node, binVersion, true )
+        
+        if( noDowntimePossible )
+            assert.eq( this.getNodeId( primary ), prevPrimaryId )
+    }
+}
+
+ReplSetTest.prototype.upgradeNode = function( node, binVersion, waitForState ){
+    
+    var node = this.restart( node, { binVersion : binVersion } )
+    
+    // By default, wait for primary or secondary state
+    if( waitForState == undefined ) waitForState = true
+    if( waitForState == true ) waitForState = [ ReplSetTest.State.PRIMARY, 
+                                                ReplSetTest.State.SECONDARY,
+                                                ReplSetTest.State.ARBITER ]
+    if( waitForState )
+        this.waitForState( node, waitForState )
+    
+    return node
+}
+
+ReplSetTest.prototype.stepdown = function( nodeId ){
+        
+    nodeId = this.getNodeId( nodeId )
+    
+    assert.eq( this.getNodeId( this.getPrimary() ), nodeId )    
+    
+    var node = this.nodes[ nodeId ]
+    
+    try {
+        node.getDB("admin").runCommand({ replSetStepDown: 50, force : true })
+        assert( false )
+    }
+    catch( e ){
+        printjson( e );
+    }
+    
+    return this.reconnect( node )
+}
+
+ReplSetTest.prototype.reconnect = function( node ){
+    
+    var nodeId = this.getNodeId( node )
+    
+    this.nodes[ nodeId ] = new Mongo( node.host )
+    
+    // TODO
+    var except = {}
+    
+    for( var i in node ){
+        if( typeof( node[i] ) == "function" ) continue
+        this.nodes[ nodeId ][ i ] = node[ i ]
+    }
+    
+    return this.nodes[ nodeId ]
+}