diff options
author | Kaloian Manassiev <kaloian.manassiev@mongodb.com> | 2015-10-26 10:09:34 -0400 |
---|---|---|
committer | Kaloian Manassiev <kaloian.manassiev@mongodb.com> | 2015-10-27 08:24:24 -0400 |
commit | 38e5c4febb788441f19044a4d3d2b89e1ab70bf0 (patch) | |
tree | 19da129e382ed59e845838f67a4711d558c3f1c3 /src/mongo/shell/shardingtest.js | |
parent | ba8757e0afd1577a090785f014fc07b55e219d9a (diff) | |
download | mongo-38e5c4febb788441f19044a4d3d2b89e1ab70bf0.tar.gz |
SERVER-21009 Get rid of some unused/unnecessary methods in ShardingTest
Removes shardGo in lieu of shardColl and removes setBalancer because it
duplicates functionality, which is already in sh.
Diffstat (limited to 'src/mongo/shell/shardingtest.js')
-rw-r--r-- | src/mongo/shell/shardingtest.js | 1276 |
1 files changed, 627 insertions, 649 deletions
diff --git a/src/mongo/shell/shardingtest.js b/src/mongo/shell/shardingtest.js index eee3bdbc791..ad93c6977b0 100644 --- a/src/mongo/shell/shardingtest.js +++ b/src/mongo/shell/shardingtest.js @@ -80,85 +80,606 @@ * c0, c1, ... {Mongo} - same as config0, config1, ... * configRS - If the config servers are a replset, this will contain the config ReplSetTest object */ -ShardingTest = function( testName , numShards , verboseLevel , numMongos , otherParams ) { - this._startTime = new Date(); +var ShardingTest = function(params) { + + /** + * Attempts to open a connection to the specified connection string or throws if unable to + * connect. + */ + function _connectWithRetry(url) { + var conn; + assert.soon(function() { + try { + conn = new Mongo(url); + return true; + } catch (e) { + print("Error connecting to " + url + ": " + e); + return false; + } + }); - // Check if testName is an object, if so, pull params from there - var keyFile = undefined - var numConfigs = 3; - otherParams = Object.merge( otherParams || {}, {} ) + return conn; + } + + /** + * Constructs a human-readable string representing a chunk's range. + */ + function _rangeToString(r) { + return tojsononeline(r.min) + " -> " + tojsononeline(r.max); + } + + // Used for counting the test duration + var _startTime = new Date(); + + // ShardingTest API + + this.getRSEntry = function(setName) { + for (var i=0; i<this._rs.length; i++) + if (this._rs[i].setName == setName) + return this._rs[i]; + throw Error("can't find rs: " + setName); + }; + + this.getDB = function(name) { + return this.s.getDB(name); + }; + + this.getServerName = function(dbname) { + var x = this.config.databases.findOne({ _id : "" + dbname }); + if (x) + return x.primary; + this.config.databases.find().forEach(printjson); + throw Error("couldn't find dbname: " + dbname + " total: " + this.config.databases.count()); + }; + + this.getNonPrimaries = function(dbname) { + var x = this.config.databases.findOne({ _id : dbname }); + if (! x) { + this.config.databases.find().forEach(printjson); + throw Error("couldn't find dbname: " + dbname + " total: " + this.config.databases.count()); + } - if( isObject( testName ) ) { - var params = Object.merge( testName, {} ) + return this.config.shards.find({ _id : { $ne : x.primary } }).map(function(z) { return z._id; }) + }; - testName = params.name || "test" - otherParams = Object.merge(otherParams, params); - otherParams = Object.merge(params.other || {}, otherParams); + this.getConnNames = function() { + var names = []; + for (var i=0; i<this._connections.length; i++) { + names.push(this._connections[i].name); + } + return names; + }; - numShards = otherParams.hasOwnProperty('shards') ? otherParams.shards : 2; - verboseLevel = otherParams.hasOwnProperty('verbose') ? otherParams.verbose : 0; - numMongos = otherParams.hasOwnProperty('mongos') ? otherParams.mongos : 1; - numConfigs = otherParams.hasOwnProperty('config') ? otherParams.config : numConfigs; + this.getServer = function(dbname) { + var name = this.getServerName(dbname); - var tempCount = 0; + var x = this.config.shards.findOne({ _id : name }); + if (x) + name = x.host; + + var rsName = null; + if (name.indexOf("/") > 0) + rsName = name.substring(0 , name.indexOf("/")); + + for (var i=0; i<this._connections.length; i++) { + var c = this._connections[i]; + if (connectionURLTheSame(name , c.name) || + connectionURLTheSame(rsName , c.name)) + return c; + } + + throw Error("can't find server for: " + dbname + " name:" + name); + }; + + this.normalize = function(x) { + var z = this.config.shards.findOne({ host : x }); + if (z) + return z._id; + return x; + }; + + this.getOther = function(one) { + if (this._connections.length < 2) + throw Error("getOther only works with 2 servers"); + + if (one._mongo) + one = one._mongo + + for(var i = 0; i < this._connections.length; i++) { + if (this._connections[i] != one) return this._connections[i] + } - // Allow specifying options like : - // { mongos : [ { noprealloc : "" } ], config : [ { smallfiles : "" } ], shards : { rs : true, d : true } } - if( Array.isArray( numShards ) ){ - for( var i = 0; i < numShards.length; i++ ){ - otherParams[ "d" + i ] = numShards[i]; + return null; + }; + + this.getAnother = function(one) { + if (this._connections.length < 2) + throw Error("getAnother() only works with multiple servers"); + + if (one._mongo) + one = one._mongo + + for(var i = 0; i < this._connections.length; i++) { + if (this._connections[i] == one) + return this._connections[(i + 1) % this._connections.length]; + } + }; + + this.getFirstOther = function(one) { + for (var i=0; i<this._connections.length; i++) { + if (this._connections[i] != one) + return this._connections[i]; + } + throw Error("impossible"); + }; + + this.stop = function() { + for (var i = 0; i < this._mongos.length; i++) { + MongoRunner.stopMongos(this._mongos[i].port); + } + + for (var i = 0; i < this._connections.length; i++) { + if (this._rs[i]) { + this._rs[i].test.stopSet(15); + } else { + MongoRunner.stopMongod(this._connections[i].port); } + } + + if (this.configRS) { + this.configRS.stopSet(); + } else { + // Old style config triplet + for (var i = 0; i < this._configServers.length; i++) { + MongoRunner.stopMongod(this._configServers[i]); + } + } - numShards = numShards.length; + for (var i = 0; i < this._alldbpaths.length; i++) { + resetDbpath(MongoRunner.dataPath + this._alldbpaths[i]); } - else if( isObject( numShards ) ){ - tempCount = 0; - for( var i in numShards ) { - otherParams[ i ] = numShards[i]; - tempCount++; + + var timeMillis = new Date().getTime() - _startTime.getTime(); + + print('*** ShardingTest ' + this._testName + " completed successfully in " + (timeMillis / 1000) + " seconds ***"); + }; + + this.adminCommand = function(cmd) { + var res = this.admin.runCommand(cmd); + if (res && res.ok == 1) + return true; + + throw _getErrorWithCode(res, "command " + tojson(cmd) + " failed: " + tojson(res)); + }; + + this.printChangeLog = function() { + this.config.changelog.find().forEach(function(z) { + var msg = z.server + "\t" + z.time + "\t" + z.what; + for (var i = z.what.length; i < 15; i++) + msg += " "; + + msg += " " + z.ns + "\t"; + if (z.what == "split") { + msg += _rangeToString(z.details.before) + " -->> (" + _rangeToString(z.details.left) + "), (" + _rangeToString(z.details.right) + ")"; } - - numShards = tempCount; + else if (z.what == "multi-split") { + msg += _rangeToString(z.details.before) + " -->> (" + z.details.number + "/" + z.details.of + " " + _rangeToString(z.details.chunk) + ")"; + } + else { + msg += tojsononeline(z.details); + } + + print("ShardingTest " + msg); + }); + }; + + this.getChunksString = function(ns) { + var q = {} + if (ns) + q.ns = ns; + + var s = ""; + this.config.chunks.find(q).sort({ ns : 1 , min : 1 }).forEach( + function(z) { + s += " " + z._id + "\t" + z.lastmod.t + "|" + z.lastmod.i + "\t" + tojson(z.min) + " -> " + tojson(z.max) + " " + z.shard + " " + z.ns + "\n"; + } + ); + + return s; + }; + + this.printChunks = function(ns) { + print("ShardingTest " + this.getChunksString(ns)); + }; + + this.printShardingStatus = function() { + printShardingStatus(this.config); + }; + + this.printCollectionInfo = function(ns , msg) { + var out = ""; + if (msg) + out += msg + "\n"; + out += "sharding collection info: " + ns + "\n"; + for (var i=0; i<this._connections.length; i++) { + var c = this._connections[i]; + out += " mongod " + c + " " + tojson(c.getCollection(ns).getShardVersion() , " " , true) + "\n"; + } + for (var i=0; i<this._mongos.length; i++) { + var c = this._mongos[i]; + out += " mongos " + c + " " + tojson(c.getCollection(ns).getShardVersion() , " " , true) + "\n"; } - if( Array.isArray( numMongos ) ){ - for( var i = 0; i < numMongos.length; i++ ) { - otherParams[ "s" + i ] = numMongos[i]; + out += this.getChunksString(ns); + + print("ShardingTest " + out); + }; + + this.sync = function() { + this.adminCommand("connpoolsync"); + }; + + this.onNumShards = function(collName , dbName) { + this.sync(); // we should sync since we're going directly to mongod here + dbName = dbName || "test"; + var num=0; + for (var i=0; i<this._connections.length; i++) + if (this._connections[i].getDB(dbName).getCollection(collName).count() > 0) + num++; + return num; + }; + + this.shardCounts = function(collName , dbName) { + this.sync(); // we should sync since we're going directly to mongod here + dbName = dbName || "test"; + var counts = {} + for (var i=0; i<this._connections.length; i++) + counts[i] = this._connections[i].getDB(dbName).getCollection(collName).count(); + return counts; + }; + + this.chunkCounts = function(collName , dbName) { + dbName = dbName || "test"; + var x = {} + + this.config.shards.find().forEach( + function(z) { + x[z._id] = 0; + } + ); + + this.config.chunks.find({ ns : dbName + "." + collName }).forEach( + function(z) { + if (x[z.shard]) + x[z.shard]++ + else + x[z.shard] = 1; } - - numMongos = numMongos.length; + ); + + return x; + }; + + this.chunkDiff = function(collName , dbName) { + var c = this.chunkCounts(collName , dbName); + var min = 100000000; + var max = 0; + for (var s in c) { + if (c[s] < min) + min = c[s]; + if (c[s] > max) + max = c[s]; } - else if( isObject( numMongos ) ){ - tempCount = 0; - for( var i in numMongos ) { - otherParams[ i ] = numMongos[i]; - tempCount++; + print("ShardingTest input: " + tojson(c) + " min: " + min + " max: " + max ); + return max - min; + }; + + // Waits up to one minute for the difference in chunks between the most loaded shard and least + // loaded shard to be 0 or 1, indicating that the collection is well balanced. + // This should only be called after creating a big enough chunk difference to trigger balancing. + this.awaitBalance = function(collName , dbName , timeToWait) { + timeToWait = timeToWait || 60000; + var shardingTest = this; + assert.soon(function() { + var x = shardingTest.chunkDiff(collName , dbName); + print("chunk diff: " + x); + return x < 2; + } , "no balance happened", 60000); + }; + + this.getShardNames = function() { + var shards = []; + this.s.getCollection("config.shards").find().forEach(function(shardDoc) { + shards.push(shardDoc._id); + }); + return shards; + }; + + this.getShard = function(coll, query, includeEmpty) { + var shards = this.getShardsForQuery(coll, query, includeEmpty) + assert.eq(shards.length, 1) + return shards[0] + }; + + /** + * Returns the shards on which documents matching a particular query reside. + */ + this.getShardsForQuery = function(coll, query, includeEmpty) { + if (! coll.getDB) + coll = this.s.getCollection(coll) + + var explain = coll.find(query).explain("executionStats") + var shards = [] + + var execStages = explain.executionStats.executionStages; + var plannerShards = explain.queryPlanner.winningPlan.shards; + + if (execStages.shards) { + for(var i = 0; i < execStages.shards.length; i++) { + var hasResults = execStages.shards[i].executionStages.nReturned && + execStages.shards[i].executionStages.nReturned > 0; + if (includeEmpty || hasResults) { + shards.push(plannerShards[i].connectionString); + } } - - numMongos = tempCount; } - - if( Array.isArray( numConfigs ) ){ - for( var i = 0; i < numConfigs.length; i++ ){ - otherParams[ "c" + i ] = numConfigs[i]; + + for(var i = 0; i < shards.length; i++) { + for(var j = 0; j < this._connections.length; j++) { + if (connectionURLTheSame( this._connections[j] , shards[i])) { + shards[i] = this._connections[j] + break; + } } + } + + return shards; + }; - numConfigs = numConfigs.length + this.isSharded = function(collName) { + var collName = "" + collName + var dbName = undefined + + if (typeof collName.getCollectionNames == 'function') { + dbName = "" + collName + collName = undefined } - else if( isObject( numConfigs ) ){ - tempCount = 0; - for( var i in numConfigs ) { - otherParams[ i ] = numConfigs[i]; - tempCount++; - } - numConfigs = tempCount; + + if (dbName) { + var x = this.config.databases.findOne({ _id : dbname }) + if (x) return x.partitioned + else return false + } + + if (collName) { + var x = this.config.collections.findOne({ _id : collName }) + if (x) return true + else return false + } + }; + + this.shardColl = function(collName , key , split , move , dbName, waitForDelete) { + split = (split != false ? (split || key) : split) + move = (split != false && move != false ? (move || split) : false) + + if (collName.getDB) + dbName = "" + collName.getDB() + else + dbName = dbName || "test"; + + var c = dbName + "." + collName; + if (collName.getDB) + c = "" + collName + + var isEmpty = (this.s.getCollection(c).count() == 0); + + if (! this.isSharded(dbName)) + this.s.adminCommand({ enableSharding : dbName }) + + var result = this.s.adminCommand({ shardcollection : c , key : key }) + if (! result.ok) { + printjson(result) + assert(false) + } + + if (split == false) return; + + result = this.s.adminCommand({ split : c , middle : split }); + if (! result.ok) { + printjson(result) + assert(false) + } + + if (move == false) return; + + var result = null + for(var i = 0; i < 5; i++) { + result = this.s.adminCommand({ movechunk : c , find : move , to : this.getOther(this.getServer(dbName)).name, _waitForDelete: waitForDelete }); + if (result.ok) break; + sleep(5 * 1000); + } + + printjson(result); + assert(result.ok); + }; + + this.stopBalancer = function(timeout, interval) { + if (typeof db == "undefined") db = undefined; + var oldDB = db; + db = this.config; + + try { + sh.setBalancerState(false); + sh.waitForBalancer(false, timeout, interval); + } + finally { + db = oldDB; + } + }; + + this.startBalancer = function(timeout, interval) { + if (typeof db == "undefined") db = undefined; + var oldDB = db; + db = this.config; + + try { + sh.setBalancerState(true); + sh.waitForBalancer(true, timeout, interval); + } + finally { + db = oldDB; + } + }; + + this.isAnyBalanceInFlight = function() { + if (this.config.locks.find({ _id : { $ne : "balancer" }, state : 2 }).count() > 0) + return true; + + var allCurrent = this.s.getDB("admin").currentOp().inprog; + for (var i = 0; i < allCurrent.length; i++) { + if (allCurrent[i].desc && + allCurrent[i].desc.indexOf("cleanupOldData") == 0) + return true; + } + return false; + }; + + /** + * Kills the mongos with index n. + */ + this.stopMongos = function(n) { + MongoRunner.stopMongos(this['s' + n].port); + }; + + /** + * Kills the mongod with index n. + */ + this.stopMongod = function(n) { + MongoRunner.stopMongod(this['d' + n].port); + }; + + /** + * Restarts a previously stopped mongos. + * + * If opts is specified, the new mongos is started using those options. Otherwise, it is started + * with its previous parameters. + * + * Warning: Overwrites the old s (if n = 0) admin, config, and sn member variables. + */ + this.restartMongos = function(n, opts) { + var mongos = this['s' + n]; + + if (opts === undefined) { + opts = this['s' + n]; + opts.restart = true; + } + + MongoRunner.stopMongos(mongos); + + var newConn = MongoRunner.runMongos(opts); + + this['s' + n] = newConn; + if (n == 0) { + this.s = newConn; + this.admin = newConn.getDB('admin'); + this.config = newConn.getDB('config'); + } + }; + + /** + * Restarts a previously stopped mongod using the same parameters as before. + * + * Warning: Overwrites the old dn member variables. + */ + this.restartMongod = function(n) { + var mongod = this['d' + n]; + MongoRunner.stopMongod(mongod); + mongod.restart = true; + + var newConn = MongoRunner.runMongod(mongod); + + this['d' + n] = newConn; + }; + + /** + * Helper method for setting primary shard of a database and making sure that it was successful. + * Note: first mongos needs to be up. + */ + this.ensurePrimaryShard = function(dbName, shardName) { + var db = this.s0.getDB('admin'); + var res = db.adminCommand({ movePrimary: dbName, to: shardName }); + assert(res.ok || res.errmsg == "it is already the primary", tojson(res)); + }; + + // ShardingTest initialization + + assert(isObject(params), 'ShardingTest configuration must be a JSON object'); + + var testName = params.name || "test"; + var otherParams = Object.merge(params, params.other || {}); + + var numShards = otherParams.hasOwnProperty('shards') ? otherParams.shards : 2; + var verboseLevel = otherParams.hasOwnProperty('verbose') ? otherParams.verbose : 0; + var numMongos = otherParams.hasOwnProperty('mongos') ? otherParams.mongos : 1; + var numConfigs = otherParams.hasOwnProperty('config') ? otherParams.config : 3; + + // Allow specifying options like : + // { mongos : [ { noprealloc : "" } ], config : [ { smallfiles : "" } ], shards : { rs : true, d : true } } + if (Array.isArray(numShards)) { + for(var i = 0; i < numShards.length; i++) { + otherParams[ "d" + i ] = numShards[i]; + } + + numShards = numShards.length; + } + else if (isObject(numShards)) { + var tempCount = 0; + for(var i in numShards) { + otherParams[ i ] = numShards[i]; + tempCount++; } + + numShards = tempCount; + } + + if (Array.isArray(numMongos)) { + for(var i = 0; i < numMongos.length; i++) { + otherParams[ "s" + i ] = numMongos[i]; + } + + numMongos = numMongos.length; + } + else if (isObject(numMongos)) { + var tempCount = 0; + for(var i in numMongos) { + otherParams[ i ] = numMongos[i]; + tempCount++; + } + + numMongos = tempCount; + } + + if (Array.isArray(numConfigs)) { + for(var i = 0; i < numConfigs.length; i++) { + otherParams[ "c" + i ] = numConfigs[i]; + } + + numConfigs = numConfigs.length + } + else if (isObject(numConfigs)) { + var tempCount = 0; + for(var i in numConfigs) { + otherParams[ i ] = numConfigs[i]; + tempCount++; + } + + numConfigs = tempCount; } otherParams.extraOptions = otherParams.extraOptions || {}; otherParams.useHostname = otherParams.useHostname == undefined ? true : otherParams.useHostname; - keyFile = otherParams.keyFile || otherParams.extraOptions.keyFile + var keyFile = otherParams.keyFile || otherParams.extraOptions.keyFile this._testName = testName this._otherParams = otherParams @@ -166,8 +687,8 @@ ShardingTest = function( testName , numShards , verboseLevel , numMongos , other var pathOpts = this.pathOpts = { testName : testName } var hasRS = false - for( var k in otherParams ){ - if( k.startsWith( "rs" ) && otherParams[k] != undefined ){ + for(var k in otherParams) { + if (k.startsWith("rs") && otherParams[k] != undefined) { hasRS = true break } @@ -180,19 +701,19 @@ ShardingTest = function( testName , numShards , verboseLevel , numMongos , other this._rsObjects = [] // Start the MongoD servers (shards) - for ( var i = 0; i < numShards; i++ ) { - if( otherParams.rs || otherParams["rs" + i] ){ + for (var i = 0; i < numShards; i++) { + if (otherParams.rs || otherParams["rs" + i]) { var setName = testName + "-rs" + i; rsDefaults = { useHostname : otherParams.useHostname, noJournalPrealloc : otherParams.nopreallocj, oplogSize : 16, - pathOpts : Object.merge( pathOpts, { shard : i } )} + pathOpts : Object.merge(pathOpts, { shard : i })} - rsDefaults = Object.merge( rsDefaults, ShardingTest.rsOptions || {} ) - rsDefaults = Object.merge( rsDefaults, otherParams.rs ) - rsDefaults = Object.merge( rsDefaults, otherParams.rsOptions ) - rsDefaults = Object.merge( rsDefaults, otherParams["rs" + i] ) + rsDefaults = Object.merge(rsDefaults, ShardingTest.rsOptions || {}) + rsDefaults = Object.merge(rsDefaults, otherParams.rs) + rsDefaults = Object.merge(rsDefaults, otherParams.rsOptions) + rsDefaults = Object.merge(rsDefaults, otherParams["rs" + i]) rsDefaults.nodes = rsDefaults.nodes || otherParams.numReplicas var numReplicas = rsDefaults.nodes || 3; @@ -200,7 +721,7 @@ ShardingTest = function( testName , numShards , verboseLevel , numMongos , other var protocolVersion = rsDefaults.protocolVersion; delete rsDefaults.protocolVersion; - print( "Replica set test!" ) + print("Replica set test!") var rs = new ReplSetTest({ name : setName, nodes : numReplicas, @@ -219,8 +740,8 @@ ShardingTest = function( testName , numShards , verboseLevel , numMongos , other this._rsObjects[i] = rs - this._alldbpaths.push( null ) - this._connections.push( null ) + this._alldbpaths.push(null) + this._connections.push(null) } else { var options = { @@ -231,20 +752,20 @@ ShardingTest = function( testName , numShards , verboseLevel , numMongos , other keyFile: keyFile }; - options = Object.merge( options, ShardingTest.shardOptions || {} ) + options = Object.merge(options, ShardingTest.shardOptions || {}) - if( otherParams.shardOptions && otherParams.shardOptions.binVersion ){ + if (otherParams.shardOptions && otherParams.shardOptions.binVersion) { otherParams.shardOptions.binVersion = - MongoRunner.versionIterator( otherParams.shardOptions.binVersion ) + MongoRunner.versionIterator(otherParams.shardOptions.binVersion) } - options = Object.merge( options, otherParams.shardOptions ) - options = Object.merge( options, otherParams["d" + i] ) + options = Object.merge(options, otherParams.shardOptions) + options = Object.merge(options, otherParams["d" + i]) - var conn = MongoRunner.runMongod( options ); + var conn = MongoRunner.runMongod(options); - this._alldbpaths.push( testName +i ) - this._connections.push( conn ); + this._alldbpaths.push(testName +i) + this._connections.push(conn); this["shard" + i] = conn this["d" + i] = conn @@ -255,19 +776,19 @@ ShardingTest = function( testName , numShards , verboseLevel , numMongos , other // Do replication on replica sets if required for (var i = 0; i < numShards; i++) { - if(!otherParams.rs && !otherParams["rs" + i]) { + if (!otherParams.rs && !otherParams["rs" + i]) { continue; } var rs = this._rs[i].test; - rs.getMaster().getDB( "admin" ).foo.save( { x : 1 } ) + rs.getMaster().getDB("admin").foo.save({ x : 1 }) if (keyFile) { authutil.asCluster(rs.nodes, keyFile, function() { rs.awaitReplication(); }); } rs.awaitSecondaryNodes(); - var rsConn = new Mongo( rs.getURL() ); + var rsConn = new Mongo(rs.getURL()); rsConn.name = rs.getURL(); this._connections[i] = rsConn this["shard" + i] = rsConn @@ -282,8 +803,7 @@ ShardingTest = function( testName , numShards , verboseLevel , numMongos , other otherParams.sync = true; } - this._configServers = [] - this._configServersAreRS = !otherParams.sync; + this._configServers = []; // Start the config servers if (otherParams.sync) { @@ -295,26 +815,26 @@ ShardingTest = function( testName , numShards , verboseLevel , numMongos , other for (var i = 0; i < numConfigs; i++) { var options = { useHostname : otherParams.useHostname, noJournalPrealloc : otherParams.nopreallocj, - pathOpts : Object.merge( pathOpts, { config : i } ), + pathOpts : Object.merge(pathOpts, { config : i }), dbpath : "$testName-config$config", keyFile : keyFile, // Ensure that journaling is always enabled for config servers. journal : "", configsvr : "" }; - options = Object.merge( options, ShardingTest.configOptions || {} ) + options = Object.merge(options, ShardingTest.configOptions || {}) if (otherParams.configOptions && otherParams.configOptions.binVersion) { otherParams.configOptions.binVersion = - MongoRunner.versionIterator( otherParams.configOptions.binVersion ) + MongoRunner.versionIterator(otherParams.configOptions.binVersion) } - options = Object.merge( options, otherParams.configOptions ) - options = Object.merge( options, otherParams["c" + i] ) + options = Object.merge(options, otherParams.configOptions) + options = Object.merge(options, otherParams["c" + i]) - var conn = MongoRunner.runMongod( options ) + var conn = MongoRunner.runMongod(options) - this._alldbpaths.push( testName + "-config" + i ) + this._alldbpaths.push(testName + "-config" + i) this._configServers.push(conn); configNames.push(conn.name); @@ -327,7 +847,6 @@ ShardingTest = function( testName , numShards , verboseLevel , numMongos , other } else { // Using replica set for config servers - var rstOptions = { useHostName : otherParams.useHostname, keyFile : keyFile, name: testName + "-configRS", @@ -342,14 +861,14 @@ ShardingTest = function( testName , numShards , verboseLevel , numMongos , other storageEngine : "wiredTiger", }; - startOptions = Object.merge( startOptions, ShardingTest.configOptions || {} ) + startOptions = Object.merge(startOptions, ShardingTest.configOptions || {}) - if ( otherParams.configOptions && otherParams.configOptions.binVersion ) { + if (otherParams.configOptions && otherParams.configOptions.binVersion) { otherParams.configOptions.binVersion = - MongoRunner.versionIterator( otherParams.configOptions.binVersion ) + MongoRunner.versionIterator(otherParams.configOptions.binVersion) } - startOptions = Object.merge( startOptions, otherParams.configOptions ) + startOptions = Object.merge(startOptions, otherParams.configOptions) var nodeOptions = []; for (var i = 0; i < numConfigs; ++i) { nodeOptions.push(otherParams["c" + i] || {}); @@ -377,36 +896,23 @@ ShardingTest = function( testName , numShards , verboseLevel , numMongos , other printjson("config servers: " + this._configDB); - var connectWithRetry = function(url) { - var conn = null; - assert.soon( function() { - try { - conn = new Mongo(url); - return true; - } catch (e) { - print("Error connecting to " + url + ": " + e); - return false; - } - }); - return conn; - } - this._configConnection = connectWithRetry(this._configDB); + this._configConnection = _connectWithRetry(this._configDB); - print( "ShardingTest " + this._testName + " :\n" + tojson( { config : this._configDB, shards : this._connections } ) ); + print("ShardingTest " + this._testName + " :\n" + tojson({ config : this._configDB, shards : this._connections })); - if ( numMongos == 0 && !otherParams.noChunkSize ) { - if ( keyFile ) { + if (numMongos == 0 && !otherParams.noChunkSize) { + if (keyFile) { throw Error("Cannot set chunk size without any mongos when using auth"); } else { - this._configConnection.getDB( "config" ).settings.insert( - { _id : "chunksize" , value : otherParams.chunksize || otherParams.chunkSize || 50 } ); + this._configConnection.getDB("config").settings.insert( + { _id : "chunksize" , value : otherParams.chunksize || otherParams.chunkSize || 50 }); } } this._mongos = [] // Start the MongoS servers - for (var i = 0; i < ( ( numMongos == 0 ? -1 : numMongos ) || 1 ); i++ ){ + for (var i = 0; i < ((numMongos == 0 ? -1 : numMongos) || 1); i++) { options = { useHostname: otherParams.useHostname, pathOpts: Object.merge(pathOpts, {mongos: i}), @@ -419,16 +925,16 @@ ShardingTest = function( testName , numShards , verboseLevel , numMongos , other options.chunkSize = otherParams.chunksize || otherParams.chunkSize || 50; } - options = Object.merge( options, ShardingTest.mongosOptions || {} ) + options = Object.merge(options, ShardingTest.mongosOptions || {}) if (otherParams.mongosOptions && otherParams.mongosOptions.binVersion) { otherParams.mongosOptions.binVersion = MongoRunner.versionIterator(otherParams.mongosOptions.binVersion); } - options = Object.merge( options, otherParams.mongosOptions ) - options = Object.merge( options, otherParams.extraOptions ) - options = Object.merge( options, otherParams["s" + i] ) + options = Object.merge(options, otherParams.mongosOptions) + options = Object.merge(options, otherParams.extraOptions) + options = Object.merge(options, otherParams["s" + i]) conn = MongoRunner.runMongos(options); @@ -444,7 +950,7 @@ ShardingTest = function( testName , numShards , verboseLevel , numMongos , other } // Disable the balancer unless it is explicitly turned on - if ( !otherParams.enableBalancer ) { + if (!otherParams.enableBalancer) { if (keyFile) { authutil.assertAuthenticate(this._mongos, 'admin', { user: '__system', @@ -474,7 +980,7 @@ ShardingTest = function( testName , numShards , verboseLevel , numMongos , other this._connections.forEach( function(z) { var n = z.name; - if (!n){ + if (!n) { n = z.host; if (!n) { n = z; @@ -489,540 +995,12 @@ ShardingTest = function( testName , numShards , verboseLevel , numMongos , other shardNames.push(result.shardAdded); z.shardName = result.shardAdded; } - ); + ); } if (jsTestOptions().keyFile) { - jsTest.authenticate( this._configConnection ); - jsTest.authenticateNodes( this._configServers ); - jsTest.authenticateNodes( this._mongos ); - } -} - -ShardingTest.prototype.getRSEntry = function( setName ){ - for ( var i=0; i<this._rs.length; i++ ) - if ( this._rs[i].setName == setName ) - return this._rs[i]; - throw Error( "can't find rs: " + setName ); -} - -ShardingTest.prototype.getConfigIndex = function( config ){ - - // Assume config is a # if not a conn object - if( ! isObject( config ) ) config = getHostName() + ":" + config - - for( var i = 0; i < this._configServers.length; i++ ){ - if( connectionURLTheSame( this._configServers[i], config ) ) return i - } - - return -1 -} - -ShardingTest.prototype.getDB = function( name ){ - return this.s.getDB( name ); -} - -ShardingTest.prototype.getServerName = function( dbname ){ - var x = this.config.databases.findOne( { _id : "" + dbname } ); - if ( x ) - return x.primary; - this.config.databases.find().forEach( printjson ); - throw Error( "couldn't find dbname: " + dbname + " total: " + this.config.databases.count() ); -} - - -ShardingTest.prototype.getNonPrimaries = function( dbname ){ - var x = this.config.databases.findOne( { _id : dbname } ); - if ( ! x ){ - this.config.databases.find().forEach( printjson ); - throw Error( "couldn't find dbname: " + dbname + " total: " + this.config.databases.count() ); + jsTest.authenticate(this._configConnection); + jsTest.authenticateNodes(this._configServers); + jsTest.authenticateNodes(this._mongos); } - - return this.config.shards.find( { _id : { $ne : x.primary } } ).map( function(z){ return z._id; } ) -} - - -ShardingTest.prototype.getConnNames = function(){ - var names = []; - for ( var i=0; i<this._connections.length; i++ ){ - names.push( this._connections[i].name ); - } - return names; -} - -ShardingTest.prototype.getServer = function( dbname ){ - var name = this.getServerName( dbname ); - - var x = this.config.shards.findOne( { _id : name } ); - if ( x ) - name = x.host; - - var rsName = null; - if ( name.indexOf( "/" ) > 0 ) - rsName = name.substring( 0 , name.indexOf( "/" ) ); - - for ( var i=0; i<this._connections.length; i++ ){ - var c = this._connections[i]; - if ( connectionURLTheSame( name , c.name ) || - connectionURLTheSame( rsName , c.name ) ) - return c; - } - - throw Error( "can't find server for: " + dbname + " name:" + name ); - -} - -ShardingTest.prototype.normalize = function( x ){ - var z = this.config.shards.findOne( { host : x } ); - if ( z ) - return z._id; - return x; -} - -ShardingTest.prototype.getOther = function( one ){ - if ( this._connections.length < 2 ) - throw Error("getOther only works with 2 servers"); - - if ( one._mongo ) - one = one._mongo - - for( var i = 0; i < this._connections.length; i++ ){ - if( this._connections[i] != one ) return this._connections[i] - } - - return null -} - -ShardingTest.prototype.getAnother = function( one ){ - if(this._connections.length < 2) - throw Error("getAnother() only works with multiple servers"); - - if ( one._mongo ) - one = one._mongo - - for(var i = 0; i < this._connections.length; i++){ - if(this._connections[i] == one) - return this._connections[(i + 1) % this._connections.length]; - } -} - -ShardingTest.prototype.getFirstOther = function( one ){ - for ( var i=0; i<this._connections.length; i++ ){ - if ( this._connections[i] != one ) - return this._connections[i]; - } - throw Error("impossible"); -} - -ShardingTest.prototype.stop = function(){ - for (var i = 0; i < this._mongos.length; i++) { - MongoRunner.stopMongos(this._mongos[i].port); - } - - for (var i = 0; i < this._connections.length; i++) { - if (this._rs[i]) { - this._rs[i].test.stopSet(15); - } else { - MongoRunner.stopMongod(this._connections[i].port); - } - } - - if (this._configServersAreRS) { - this.configRS.stopSet(); - } else { - // Old style config triplet - for (var i = 0; i < this._configServers.length; i++) { - MongoRunner.stopMongod(this._configServers[i]); - } - } - - for (var i = 0; i < this._alldbpaths.length; i++) { - resetDbpath(MongoRunner.dataPath + this._alldbpaths[i]); - } - - var timeMillis = new Date().getTime() - this._startTime.getTime(); - - print('*** ShardingTest ' + this._testName + " completed successfully in " + ( timeMillis / 1000 ) + " seconds ***"); -} - -ShardingTest.prototype.adminCommand = function(cmd){ - var res = this.admin.runCommand( cmd ); - if ( res && res.ok == 1 ) - return true; - - throw _getErrorWithCode(res, "command " + tojson(cmd) + " failed: " + tojson(res)); -} - -ShardingTest.prototype._rangeToString = function(r){ - return tojsononeline( r.min ) + " -> " + tojsononeline( r.max ); -} - -ShardingTest.prototype.printChangeLog = function(){ - var s = this; - this.config.changelog.find().forEach( - function(z){ - var msg = z.server + "\t" + z.time + "\t" + z.what; - for ( i=z.what.length; i<15; i++ ) - msg += " "; - msg += " " + z.ns + "\t"; - if ( z.what == "split" ){ - msg += s._rangeToString( z.details.before ) + " -->> (" + s._rangeToString( z.details.left ) + "),(" + s._rangeToString( z.details.right ) + ")"; - } - else if (z.what == "multi-split" ){ - msg += s._rangeToString( z.details.before ) + " -->> (" + z.details.number + "/" + z.details.of + " " + s._rangeToString( z.details.chunk ) + ")"; - } - else { - msg += tojsononeline( z.details ); - } - - print( "ShardingTest " + msg ) - } - ); - -} - -ShardingTest.prototype.getChunksString = function( ns ){ - var q = {} - if ( ns ) - q.ns = ns; - - var s = ""; - this.config.chunks.find( q ).sort( { ns : 1 , min : 1 } ).forEach( - function(z){ - s += " " + z._id + "\t" + z.lastmod.t + "|" + z.lastmod.i + "\t" + tojson(z.min) + " -> " + tojson(z.max) + " " + z.shard + " " + z.ns + "\n"; - } - ); - - return s; -} - -ShardingTest.prototype.printChunks = function( ns ){ - print( "ShardingTest " + this.getChunksString( ns ) ); -} - -ShardingTest.prototype.printShardingStatus = function(){ - printShardingStatus( this.config ); -} - -ShardingTest.prototype.printCollectionInfo = function( ns , msg ){ - var out = ""; - if ( msg ) - out += msg + "\n"; - out += "sharding collection info: " + ns + "\n"; - for ( var i=0; i<this._connections.length; i++ ){ - var c = this._connections[i]; - out += " mongod " + c + " " + tojson( c.getCollection( ns ).getShardVersion() , " " , true ) + "\n"; - } - for ( var i=0; i<this._mongos.length; i++ ){ - var c = this._mongos[i]; - out += " mongos " + c + " " + tojson( c.getCollection( ns ).getShardVersion() , " " , true ) + "\n"; - } - - out += this.getChunksString( ns ); - - print( "ShardingTest " + out ); -} - -ShardingTest.prototype.sync = function(){ - this.adminCommand( "connpoolsync" ); -} - -ShardingTest.prototype.onNumShards = function( collName , dbName ){ - this.sync(); // we should sync since we're going directly to mongod here - dbName = dbName || "test"; - var num=0; - for ( var i=0; i<this._connections.length; i++ ) - if ( this._connections[i].getDB( dbName ).getCollection( collName ).count() > 0 ) - num++; - return num; -} - - -ShardingTest.prototype.shardCounts = function( collName , dbName ){ - this.sync(); // we should sync since we're going directly to mongod here - dbName = dbName || "test"; - var counts = {} - for ( var i=0; i<this._connections.length; i++ ) - counts[i] = this._connections[i].getDB( dbName ).getCollection( collName ).count(); - return counts; -} - -ShardingTest.prototype.chunkCounts = function( collName , dbName ){ - dbName = dbName || "test"; - var x = {} - - this.config.shards.find().forEach( - function(z){ - x[z._id] = 0; - } - ); - - this.config.chunks.find( { ns : dbName + "." + collName } ).forEach( - function(z){ - if ( x[z.shard] ) - x[z.shard]++ - else - x[z.shard] = 1; - } - ); - return x; - -} - -ShardingTest.prototype.chunkDiff = function( collName , dbName ){ - var c = this.chunkCounts( collName , dbName ); - var min = 100000000; - var max = 0; - for ( var s in c ){ - if ( c[s] < min ) - min = c[s]; - if ( c[s] > max ) - max = c[s]; - } - print( "ShardingTest input: " + tojson( c ) + " min: " + min + " max: " + max ); - return max - min; -} - -// Waits up to one minute for the difference in chunks between the most loaded shard and least -// loaded shard to be 0 or 1, indicating that the collection is well balanced. -// This should only be called after creating a big enough chunk difference to trigger balancing. -ShardingTest.prototype.awaitBalance = function( collName , dbName , timeToWait ) { - timeToWait = timeToWait || 60000; - var shardingTest = this; - assert.soon( function() { - var x = shardingTest.chunkDiff( collName , dbName ); - print( "chunk diff: " + x ); - return x < 2; - } , "no balance happened", 60000 ); - -} - -ShardingTest.prototype.getShardNames = function() { - var shards = []; - this.s.getCollection("config.shards").find().forEach(function(shardDoc) { - shards.push(shardDoc._id); - }); - return shards; -} - - -ShardingTest.prototype.getShard = function( coll, query, includeEmpty ){ - var shards = this.getShardsForQuery( coll, query, includeEmpty ) - assert.eq( shards.length, 1 ) - return shards[0] -} - -// Returns the shards on which documents matching a particular query reside -ShardingTest.prototype.getShardsForQuery = function( coll, query, includeEmpty ){ - if( ! coll.getDB ) - coll = this.s.getCollection( coll ) - - var explain = coll.find( query ).explain("executionStats") - var shards = [] - - var execStages = explain.executionStats.executionStages; - var plannerShards = explain.queryPlanner.winningPlan.shards; - - if( execStages.shards ){ - for( var i = 0; i < execStages.shards.length; i++ ){ - var hasResults = execStages.shards[i].executionStages.nReturned && - execStages.shards[i].executionStages.nReturned > 0; - if( includeEmpty || hasResults ){ - shards.push(plannerShards[i].connectionString); - } - } - } - - for( var i = 0; i < shards.length; i++ ){ - for( var j = 0; j < this._connections.length; j++ ){ - if ( connectionURLTheSame( this._connections[j] , shards[i] ) ){ - shards[i] = this._connections[j] - break; - } - } - } - - return shards -} - -ShardingTest.prototype.isSharded = function( collName ){ - - var collName = "" + collName - var dbName = undefined - - if( typeof collName.getCollectionNames == 'function' ){ - dbName = "" + collName - collName = undefined - } - - if( dbName ){ - var x = this.config.databases.findOne( { _id : dbname } ) - if( x ) return x.partitioned - else return false - } - - if( collName ){ - var x = this.config.collections.findOne( { _id : collName } ) - if( x ) return true - else return false - } - -} - -ShardingTest.prototype.shardGo = function( collName , key , split , move , dbName, waitForDelete ){ - - split = ( split != false ? ( split || key ) : split ) - move = ( split != false && move != false ? ( move || split ) : false ) - - if( collName.getDB ) - dbName = "" + collName.getDB() - else dbName = dbName || "test"; - - var c = dbName + "." + collName; - if( collName.getDB ) - c = "" + collName - - var isEmpty = this.s.getCollection( c ).count() == 0 - - if( ! this.isSharded( dbName ) ) - this.s.adminCommand( { enableSharding : dbName } ) - - var result = this.s.adminCommand( { shardcollection : c , key : key } ) - if( ! result.ok ){ - printjson( result ) - assert( false ) - } - - if( split == false ) return - - result = this.s.adminCommand( { split : c , middle : split } ); - if( ! result.ok ){ - printjson( result ) - assert( false ) - } - - if( move == false ) return - - var result = null - for( var i = 0; i < 5; i++ ){ - result = this.s.adminCommand( { movechunk : c , find : move , to : this.getOther( this.getServer( dbName ) ).name, _waitForDelete: waitForDelete } ); - if( result.ok ) break; - sleep( 5 * 1000 ); - } - printjson( result ) - assert( result.ok ) - -}; - -ShardingTest.prototype.shardColl = ShardingTest.prototype.shardGo - -ShardingTest.prototype.setBalancer = function( balancer ){ - if( balancer || balancer == undefined ){ - this.config.settings.update( { _id: "balancer" }, { $set : { stopped: false } } , true ) - } - else if( balancer == false ){ - this.config.settings.update( { _id: "balancer" }, { $set : { stopped: true } } , true ) - } -} - -ShardingTest.prototype.stopBalancer = function( timeout, interval ) { - this.setBalancer( false ) - - if( typeof db == "undefined" ) db = undefined - var oldDB = db - - db = this.config - sh.waitForBalancer( false, timeout, interval ) - db = oldDB -} - -ShardingTest.prototype.startBalancer = function( timeout, interval ) { - this.setBalancer( true ) - - if( typeof db == "undefined" ) db = undefined - var oldDB = db - - db = this.config - sh.waitForBalancer( true, timeout, interval ) - db = oldDB -} - -ShardingTest.prototype.isAnyBalanceInFlight = function() { - if ( this.config.locks.find({ _id : { $ne : "balancer" }, state : 2 }).count() > 0 ) - return true; - - var allCurrent = this.s.getDB( "admin" ).currentOp().inprog; - for ( var i = 0; i < allCurrent.length; i++ ) { - if ( allCurrent[i].desc && - allCurrent[i].desc.indexOf( "cleanupOldData" ) == 0 ) - return true; - } - return false; -} - -/** - * Kills the mongos with index n. - */ -ShardingTest.prototype.stopMongos = function(n) { - MongoRunner.stopMongos(this['s' + n].port); -}; - -/** - * Kills the mongod with index n. - */ -ShardingTest.prototype.stopMongod = function(n) { - MongoRunner.stopMongod(this['d' + n].port); -}; - -/** - * Restarts a previously stopped mongos. - * - * If opts is specified, the new mongos is started using those options. Otherwise, it is started - * with its previous parameters. - * - * Warning: Overwrites the old s (if n = 0) admin, config, and sn member variables. - */ -ShardingTest.prototype.restartMongos = function(n, opts) { - var mongos = this['s' + n]; - - if (opts === undefined) { - opts = this['s' + n]; - opts.restart = true; - } - - MongoRunner.stopMongos(mongos); - - var newConn = MongoRunner.runMongos(opts); - - this['s' + n] = newConn; - if (n == 0) { - this.s = newConn; - this.admin = newConn.getDB('admin'); - this.config = newConn.getDB('config'); - } -}; - -/** - * Restarts a previously stopped mongod using the same parameters as before. - * - * Warning: Overwrites the old dn member variables. - */ -ShardingTest.prototype.restartMongod = function(n) { - var mongod = this['d' + n]; - MongoRunner.stopMongod(mongod); - mongod.restart = true; - - var newConn = MongoRunner.runMongod(mongod); - - this['d' + n] = newConn; -}; - -/** - * Helper method for setting primary shard of a database and making sure that it was successful. - * Note: first mongos needs to be up. - */ -ShardingTest.prototype.ensurePrimaryShard = function(dbName, shardName) { - var db = this.s0.getDB('admin'); - var res = db.adminCommand({ movePrimary: dbName, to: shardName }); - assert(res.ok || res.errmsg == "it is already the primary", tojson(res)); }; |