// // Utility functions for multi-version replica sets // ReplSetTest.prototype._stablePrimaryOnRestarts = function() { // In a 2-node replica set the secondary can step up after a restart. In fact while the // secondary is being restarted, the primary may end up stepping down (due to heartbeats not // being received) and for the restarted node to run for and win the election. return this.nodes.length > 2; }; /** * Upgrade or downgrade replica sets. * * @param options {Object} see ReplSetTest.start & MongoRunner.runMongod. * @param user {string} optional, user name for authentication. * @param pwd {string} optional, password for authentication. Must be set if user is set. */ ReplSetTest.prototype.upgradeSet = function(options, user, pwd) { this.awaitNodesAgreeOnPrimary(); let primary = this.getPrimary(); this.upgradeSecondaries(Object.assign({}, options), user, pwd); this.upgradeArbiters(Object.assign({}, options), user, pwd); if (!this._stablePrimaryOnRestarts()) { this.awaitNodesAgreeOnPrimary(); if (this.getPrimary() != primary) { this.upgradeMembers([primary], Object.assign({}, options), user, pwd); return; } } if (this.getPrimary() == primary) { this.upgradePrimary(primary, Object.assign({}, options), user, pwd); } else { // An election occured during upgrade, old primary is now a secondary. this.upgradeMembers([primary], Object.assign({}, options), user, pwd); } }; function mergeNodeOptions(nodeOptions, options) { for (let nodeName in nodeOptions) { nodeOptions[nodeName] = Object.merge(nodeOptions[nodeName], options); } return nodeOptions; } ReplSetTest.prototype.upgradeMembers = function(members, options, user, pwd) { // Merge new options into node settings. this.nodeOptions = mergeNodeOptions(this.nodeOptions, options); for (let member of members) { this.upgradeNode(member, options, user, pwd); } }; ReplSetTest.prototype.getNonArbiterSecondaries = function() { let secs = this.getSecondaries(); let arbiters = this.getArbiters(); let nonArbiters = secs.filter(x => !arbiters.includes(x)); return nonArbiters; }; ReplSetTest.prototype.upgradeSecondaries = function(options, user, pwd) { this.upgradeMembers(this.getNonArbiterSecondaries(), options, user, pwd); }; ReplSetTest.prototype.upgradeArbiters = function(options, user, pwd) { // We don't support downgrading data files for arbiters. We need to instead delete the dbpath. const oldStartClean = {startClean: (options && !!options["startClean"])}; if (options && options.binVersion == "last-lts") { options["startClean"] = true; } this.upgradeMembers(this.getArbiters(), options, user, pwd); // Make sure we don't set {startClean:true} on other nodes unless the user explicitly requested. this.nodeOptions = mergeNodeOptions(this.nodeOptions, oldStartClean); }; ReplSetTest.prototype.upgradePrimary = function(primary, options, user, pwd) { function authNode(node) { if (user !== undefined) { assert(node.getDB('admin').auth(user, pwd)); } else { jsTest.authenticate(node); } } // Merge new options into node settings. this.nodeOptions = mergeNodeOptions(this.nodeOptions, options); authNode(primary); let oldPrimary = this.stepdown(primary); this.waitForState(oldPrimary, ReplSetTest.State.SECONDARY, undefined, authNode); // waitForState() runs the logout command via asCluster() on either the current primary or the // first node in the replica set so we re-authenticate on all connections before calling // awaitNodesAgreeOnPrimary(). for (const node of this.nodes) { const connStatus = assert.commandWorked(node.adminCommand({connectionStatus: 1, showPrivileges: true})); const connIsAuthenticated = connStatus.authInfo.authenticatedUsers.length > 0; if (connIsAuthenticated) { continue; } authNode(node); } this.awaitNodesAgreeOnPrimary(); primary = this.getPrimary(); this.upgradeNode(oldPrimary, options, user, pwd); let newPrimary = this.getPrimary(); if (this._stablePrimaryOnRestarts()) { assert.eq( newPrimary, primary, "Primary changed unexpectedly after upgrading old primary node"); } return newPrimary; }; ReplSetTest.prototype.upgradeNode = function(node, opts = {}, user, pwd) { if (user !== undefined) { assert.eq(1, node.getDB("admin").auth(user, pwd)); } jsTest.authenticate(node); var isMaster = node.getDB('admin').runCommand({isMaster: 1}); if (!isMaster.arbiterOnly) { // Must retry this command, as it might return "currently running for election" and fail. // Node might still be running for an election that will fail because it lost the election // race with another node, at test initialization. See SERVER-23133. assert.soonNoExcept(function() { assert.commandWorked(node.adminCommand("replSetMaintenance")); return true; }); this.waitForState(node, ReplSetTest.State.RECOVERING); } var newNode = this.restart(node, opts); if (user !== undefined) { newNode.getDB("admin").auth(user, pwd); } var waitForStates = [ReplSetTest.State.PRIMARY, ReplSetTest.State.SECONDARY, ReplSetTest.State.ARBITER]; this.waitForState(newNode, waitForStates); if (user !== undefined) { newNode.getDB('admin').logout(); } return newNode; }; ReplSetTest.prototype.stepdown = function(nodeId) { nodeId = this.getNodeId(nodeId); assert.eq(this.getNodeId(this.getPrimary()), nodeId, "Trying to stepdown a non primary node"); var node = this.nodes[nodeId]; assert.soonNoExcept(function() { // Due to a rare race condition in stepdown, it's possible the secondary just replicated // the most recent write and sent replSetUpdatePosition to the primary, and that // replSetUpdatePosition command gets interrupted by the stepdown. In that case, // the secondary will clear its sync source, but will be unable to re-connect to the // primary that is trying to step down, because they are at the same OpTime. The primary // will then get stuck waiting forever for the secondary to catch up so it can complete // stepdown. Adding a garbage write here ensures that the secondary will be able to // resume syncing from the primary in this case, which in turn will let the primary // finish stepping down successfully. node.getDB('admin').garbageWriteToAdvanceOpTime.insert({a: 1}); assert.adminCommandWorkedAllowingNetworkError( node, {replSetStepDown: 5 * 60, secondaryCatchUpPeriodSecs: 60}); return true; }); return this.reconnect(node); }; ReplSetTest.prototype.reconnect = function(node) { var nodeId = this.getNodeId(node); this.nodes[nodeId] = new Mongo(node.host); // Skip the 'authenticated' property because the new connection hasn't been authenticated even // if the original one was. This ensures Mongo.prototype.getDB() will attempt to authenticate // automatically if TestData is configured appropriately. // // Skip the '_defaultSession' property because the DriverSession object is bound to the original // connection object. Copying the '_defaultSession' property would cause commands to go through // the original connection despite methods being called on DB objects from the new connection. const except = new Set(["authenticated", "_defaultSession"]); for (var i in node) { if (typeof (node[i]) == "function" || except.has(i)) continue; this.nodes[nodeId][i] = node[i]; } return this.nodes[nodeId]; }; ReplSetTest.prototype.conf = function() { var admin = this.getPrimary().getDB('admin'); var resp = admin.runCommand({replSetGetConfig: 1}); if (resp.ok && !(resp.errmsg) && resp.config) return resp.config; else if (resp.errmsg && resp.errmsg.startsWith("no such cmd")) return admin.getSiblingDB("local").system.replset.findOne(); throw new Error("Could not retrieve replica set config: " + tojson(resp)); };