diff options
author | Kaloian Manassiev <kaloian.manassiev@mongodb.com> | 2017-11-29 10:21:19 -0500 |
---|---|---|
committer | Kaloian Manassiev <kaloian.manassiev@mongodb.com> | 2017-12-04 13:24:55 -0500 |
commit | 95e95613412c33b52bb8a514751550c2447526d4 (patch) | |
tree | b0f86dd04b6c6f8978b6a77f6d29adffab79eddf | |
parent | 7920e242c0def907b502265ca14ddf3d86c98025 (diff) | |
download | mongo-95e95613412c33b52bb8a514751550c2447526d4.tar.gz |
SERVER-31056 Remove usages of ScopedCollectionMetadata default constructor
-rw-r--r-- | jstests/sharding/stale_mongos_updates_and_removes.js | 396 | ||||
-rw-r--r-- | src/mongo/db/commands/mr.cpp | 12 | ||||
-rw-r--r-- | src/mongo/db/range_arithmetic.cpp | 54 | ||||
-rw-r--r-- | src/mongo/db/range_arithmetic.h | 35 | ||||
-rw-r--r-- | src/mongo/db/range_arithmetic_test.cpp | 35 | ||||
-rw-r--r-- | src/mongo/db/s/collection_metadata.cpp | 36 | ||||
-rw-r--r-- | src/mongo/db/s/collection_metadata_test.cpp | 3 | ||||
-rw-r--r-- | src/mongo/db/s/collection_sharding_state.cpp | 5 | ||||
-rw-r--r-- | src/mongo/db/s/get_shard_version_command.cpp | 15 | ||||
-rw-r--r-- | src/mongo/db/s/merge_chunks_command.cpp | 69 | ||||
-rw-r--r-- | src/mongo/db/s/metadata_manager.cpp | 15 | ||||
-rw-r--r-- | src/mongo/db/s/metadata_manager_test.cpp | 2 | ||||
-rw-r--r-- | src/mongo/db/s/migration_destination_manager_legacy_commands.cpp | 7 | ||||
-rw-r--r-- | src/mongo/db/s/migration_source_manager.cpp | 80 | ||||
-rw-r--r-- | src/mongo/db/s/migration_source_manager.h | 19 | ||||
-rw-r--r-- | src/mongo/db/s/sharding_state.cpp | 29 | ||||
-rw-r--r-- | src/mongo/db/s/split_chunk.cpp | 11 |
17 files changed, 370 insertions, 453 deletions
diff --git a/jstests/sharding/stale_mongos_updates_and_removes.js b/jstests/sharding/stale_mongos_updates_and_removes.js index a58d1656590..8291e637ed2 100644 --- a/jstests/sharding/stale_mongos_updates_and_removes.js +++ b/jstests/sharding/stale_mongos_updates_and_removes.js @@ -13,199 +13,213 @@ * * This test is labeled resource intensive because its total io_write is 31MB compared to a median * of 5MB across all sharding tests in wiredTiger. + * * @tags: [resource_intensive] */ -// Create a new sharded collection with numDocs documents, with two docs sharing each shard key -// (used for testing *multi* removes to a *specific* shard key). -var resetCollection = function() { - assert(staleMongos.getCollection(collNS).drop()); - assert.commandWorked(staleMongos.adminCommand({shardCollection: collNS, key: {x: 1}})); - for (var i = 0; i < numShardKeys; i++) { - assert.writeOK(staleMongos.getCollection(collNS).insert({x: i, fieldToUpdate: 0})); - assert.writeOK(staleMongos.getCollection(collNS).insert({x: i, fieldToUpdate: 0})); - } +(function() { + 'use strict'; - // Make sure data has replicated to all config servers so freshMongos finds a sharded - // collection: freshMongos has an older optime and won't wait to see what staleMongos did - // (shardCollection). - st.configRS.awaitLastOpCommitted(); -}; - -// Create a new sharded collection, and split data into two chunks on different shards using the -// stale mongos. Then use the fresh mongos to consolidate the chunks onto one of the shards. -// In the end: -// staleMongos will see: -// shard0: (-inf, splitPoint] -// shard1: (splitPoint, inf] -// freshMongos will see: -// shard0: (-inf, splitPoint], (splitPoint, inf] -// shard1: -var makeStaleMongosTargetMultipleShards = function() { - resetCollection(); - - // Make sure staleMongos sees all data on first shard. - var chunk = - staleMongos.getCollection("config.chunks").findOne({min: {x: MinKey}, max: {x: MaxKey}}); - assert(chunk.shard === st.shard0.shardName); - - // Make sure staleMongos sees two chunks on two different shards. - assert.commandWorked(staleMongos.adminCommand({split: collNS, middle: {x: splitPoint}})); - assert.commandWorked(staleMongos.adminCommand( - {moveChunk: collNS, find: {x: 0}, to: st.shard1.shardName, _waitForDelete: true})); - - st.configRS.awaitLastOpCommitted(); - - // Use freshMongos to consolidate the chunks on one shard. - assert.commandWorked(freshMongos.adminCommand( - {moveChunk: collNS, find: {x: 0}, to: st.shard0.shardName, _waitForDelete: true})); -}; - -// Create a new sharded collection and move a chunk from one shard to another. In the end, -// staleMongos will see: -// shard0: (-inf, inf] -// shard1: -// freshMongos will see: -// shard0: -// shard1: (-inf, inf] -var makeStaleMongosTargetSingleShard = function() { - resetCollection(); - // Make sure staleMongos sees all data on first shard. - var chunk = - staleMongos.getCollection("config.chunks").findOne({min: {x: MinKey}, max: {x: MaxKey}}); - assert(chunk.shard === st.shard0.shardName); - - // Use freshMongos to move chunk to another shard. - assert.commandWorked(freshMongos.adminCommand( - {moveChunk: collNS, find: {x: 0}, to: st.shard1.shardName, _waitForDelete: true})); -}; - -var checkAllRemoveQueries = function(makeMongosStaleFunc) { - var multi = {justOne: false}; - var single = {justOne: true}; - - var doRemove = function(query, multiOption, makeMongosStaleFunc) { - makeMongosStaleFunc(); - assert.writeOK(staleMongos.getCollection(collNS).remove(query, multiOption)); - if (multiOption.justOne) { - // A total of one document should have been removed from the collection. - assert.eq(numDocs - 1, staleMongos.getCollection(collNS).find().itcount()); - } else { - // All documents matching the query should have been removed. - assert.eq(0, staleMongos.getCollection(collNS).find(query).itcount()); - } - }; - - var checkRemoveIsInvalid = function(query, multiOption, makeMongosStaleFunc) { - makeMongosStaleFunc(); - var res = staleMongos.getCollection(collNS).remove(query, multiOption); - assert.writeError(res); - }; - - // Not possible because single remove requires equality match on shard key. - checkRemoveIsInvalid(emptyQuery, single, makeMongosStaleFunc); - doRemove(emptyQuery, multi, makeMongosStaleFunc); - - doRemove(pointQuery, single, makeMongosStaleFunc); - doRemove(pointQuery, multi, makeMongosStaleFunc); - - // Not possible because can't do range query on a single remove. - checkRemoveIsInvalid(rangeQuery, single, makeMongosStaleFunc); - doRemove(rangeQuery, multi, makeMongosStaleFunc); - - // Not possible because single remove must contain _id or shard key at top level - // (not within $or). - checkRemoveIsInvalid(multiPointQuery, single, makeMongosStaleFunc); - doRemove(multiPointQuery, multi, makeMongosStaleFunc); -}; - -var checkAllUpdateQueries = function(makeMongosStaleFunc) { - var oUpdate = {$inc: {fieldToUpdate: 1}}; // op-style update (non-idempotent) - var rUpdate = {x: 0, fieldToUpdate: 1}; // replacement-style update (idempotent) - var queryAfterUpdate = {fieldToUpdate: 1}; - - var multi = {multi: true}; - var single = {multi: false}; - - var doUpdate = function(query, update, multiOption, makeMongosStaleFunc) { - makeMongosStaleFunc(); - assert.writeOK(staleMongos.getCollection(collNS).update(query, update, multiOption)); - if (multiOption.multi) { - // All documents matching the query should have been updated. - assert.eq(staleMongos.getCollection(collNS).find(query).itcount(), - staleMongos.getCollection(collNS).find(queryAfterUpdate).itcount()); - } else { - // A total of one document should have been updated. - assert.eq(1, staleMongos.getCollection(collNS).find(queryAfterUpdate).itcount()); + // Create a new sharded collection with numDocs documents, with two docs sharing each shard key + // (used for testing *multi* removes to a *specific* shard key). + function resetCollection() { + assert(staleMongos.getCollection(collNS).drop()); + assert.commandWorked(staleMongos.adminCommand({shardCollection: collNS, key: {x: 1}})); + + for (var i = 0; i < numShardKeys; i++) { + assert.writeOK(staleMongos.getCollection(collNS).insert({x: i, fieldToUpdate: 0})); + assert.writeOK(staleMongos.getCollection(collNS).insert({x: i, fieldToUpdate: 0})); } - }; - - var checkUpdateIsInvalid = function(query, update, multiOption, makeMongosStaleFunc, err) { - makeMongosStaleFunc(); - var res = staleMongos.getCollection(collNS).update(query, update, multiOption); - assert.writeError(res); - }; - - // This update has inconsistent behavior as explained in SERVER-22895. - // doUpdate(emptyQuery, rUpdate, single, makeMongosStaleFunc); - // Not possible because replacement-style requires equality match on shard key. - checkUpdateIsInvalid(emptyQuery, rUpdate, multi, makeMongosStaleFunc); - // Not possible because op-style requires equality match on shard key if single update. - checkUpdateIsInvalid(emptyQuery, oUpdate, single, makeMongosStaleFunc); - doUpdate(emptyQuery, oUpdate, multi, makeMongosStaleFunc); - - doUpdate(pointQuery, rUpdate, single, makeMongosStaleFunc); - // Not possible because replacement-style requires multi=false. - checkUpdateIsInvalid(pointQuery, rUpdate, multi, makeMongosStaleFunc); - doUpdate(pointQuery, oUpdate, single, makeMongosStaleFunc); - doUpdate(pointQuery, oUpdate, multi, makeMongosStaleFunc); - - doUpdate(rangeQuery, rUpdate, single, makeMongosStaleFunc); - // Not possible because replacement-style requires multi=false. - checkUpdateIsInvalid(rangeQuery, rUpdate, multi, makeMongosStaleFunc); - // Not possible because can't do range query on a single update. - checkUpdateIsInvalid(rangeQuery, oUpdate, single, makeMongosStaleFunc); - doUpdate(rangeQuery, oUpdate, multi, makeMongosStaleFunc); - - doUpdate(multiPointQuery, rUpdate, single, makeMongosStaleFunc); - // Not possible because replacement-style requires multi=false. - checkUpdateIsInvalid(multiPointQuery, rUpdate, multi, makeMongosStaleFunc); - // Not possible because single remove must contain _id or shard key at top level - // (not within $or). - checkUpdateIsInvalid(multiPointQuery, oUpdate, single, makeMongosStaleFunc); - doUpdate(multiPointQuery, oUpdate, multi, makeMongosStaleFunc); -}; - -var st = new ShardingTest({shards: 2, mongos: 2}); - -var dbName = 'test'; -var collNS = dbName + '.foo'; -var numShardKeys = 10; -var numDocs = numShardKeys * 2; -var splitPoint = numShardKeys / 2; - -assert.commandWorked(st.s.adminCommand({enableSharding: dbName})); -assert.commandWorked(st.s.adminCommand({shardCollection: collNS, key: {x: 1}})); - -st.ensurePrimaryShard(dbName, st.shard0.shardName); - -var freshMongos = st.s0; -var staleMongos = st.s1; - -var emptyQuery = {}; -var pointQuery = {x: 0}; - -// Choose a range that would fall on only one shard. -// Use (splitPoint - 1) because of SERVER-20768. -var rangeQuery = {x: {$gte: 0, $lt: splitPoint - 1}}; - -// Choose points that would fall on two different shards. -var multiPointQuery = {$or: [{x: 0}, {x: numShardKeys}]}; - -checkAllRemoveQueries(makeStaleMongosTargetSingleShard); -checkAllRemoveQueries(makeStaleMongosTargetMultipleShards); - -checkAllUpdateQueries(makeStaleMongosTargetSingleShard); -checkAllUpdateQueries(makeStaleMongosTargetMultipleShards); - -st.stop(); + + // Make sure data has replicated to all config servers so freshMongos finds a sharded + // collection: freshMongos has an older optime and won't wait to see what staleMongos did + // (shardCollection). + st.configRS.awaitLastOpCommitted(); + } + + // Create a new sharded collection, and split data into two chunks on different shards using the + // stale mongos. Then use the fresh mongos to consolidate the chunks onto one of the shards. + // In the end: + // staleMongos will see: + // shard0: (-inf, splitPoint] + // shard1: (splitPoint, inf] + // freshMongos will see: + // shard0: (-inf, splitPoint], (splitPoint, inf] + // shard1: + function makeStaleMongosTargetMultipleShards() { + resetCollection(); + + // Make sure staleMongos sees all data on first shard. + var chunk = staleMongos.getCollection("config.chunks") + .findOne({min: {x: MinKey}, max: {x: MaxKey}}); + assert(chunk.shard === st.shard0.shardName); + + // Make sure staleMongos sees two chunks on two different shards. + assert.commandWorked(staleMongos.adminCommand({split: collNS, middle: {x: splitPoint}})); + assert.commandWorked(staleMongos.adminCommand( + {moveChunk: collNS, find: {x: 0}, to: st.shard1.shardName, _waitForDelete: true})); + + st.configRS.awaitLastOpCommitted(); + + // Use freshMongos to consolidate the chunks on one shard. + assert.commandWorked(freshMongos.adminCommand( + {moveChunk: collNS, find: {x: 0}, to: st.shard0.shardName, _waitForDelete: true})); + } + + // Create a new sharded collection and move a chunk from one shard to another. In the end: + // staleMongos will see: + // shard0: (-inf, inf] + // shard1: + // freshMongos will see: + // shard0: + // shard1: (-inf, inf] + function makeStaleMongosTargetSingleShard() { + resetCollection(); + + // Make sure staleMongos sees all data on first shard. + var chunk = staleMongos.getCollection("config.chunks") + .findOne({min: {x: MinKey}, max: {x: MaxKey}}); + assert(chunk.shard === st.shard0.shardName); + + // Use freshMongos to move chunk to another shard. + assert.commandWorked(freshMongos.adminCommand( + {moveChunk: collNS, find: {x: 0}, to: st.shard1.shardName, _waitForDelete: true})); + } + + function checkAllRemoveQueries(makeMongosStaleFunc) { + var multi = {justOne: false}; + var single = {justOne: true}; + + var doRemove = function(query, multiOption, makeMongosStaleFunc) { + makeMongosStaleFunc(); + assert.writeOK(staleMongos.getCollection(collNS).remove(query, multiOption)); + if (multiOption.justOne) { + // A total of one document should have been removed from the collection. + assert.eq(numDocs - 1, staleMongos.getCollection(collNS).find().itcount()); + } else { + // All documents matching the query should have been removed. + assert.eq(0, staleMongos.getCollection(collNS).find(query).itcount()); + } + }; + + var checkRemoveIsInvalid = function(query, multiOption, makeMongosStaleFunc) { + makeMongosStaleFunc(); + var res = staleMongos.getCollection(collNS).remove(query, multiOption); + assert.writeError(res); + }; + + // Not possible because single remove requires equality match on shard key. + checkRemoveIsInvalid(emptyQuery, single, makeMongosStaleFunc); + doRemove(emptyQuery, multi, makeMongosStaleFunc); + + doRemove(pointQuery, single, makeMongosStaleFunc); + doRemove(pointQuery, multi, makeMongosStaleFunc); + + // Not possible because can't do range query on a single remove. + checkRemoveIsInvalid(rangeQuery, single, makeMongosStaleFunc); + doRemove(rangeQuery, multi, makeMongosStaleFunc); + + // Not possible because single remove must contain _id or shard key at top level + // (not within $or). + checkRemoveIsInvalid(multiPointQuery, single, makeMongosStaleFunc); + doRemove(multiPointQuery, multi, makeMongosStaleFunc); + } + + function checkAllUpdateQueries(makeMongosStaleFunc) { + var oUpdate = {$inc: {fieldToUpdate: 1}}; // op-style update (non-idempotent) + var rUpdate = {x: 0, fieldToUpdate: 1}; // replacement-style update (idempotent) + var queryAfterUpdate = {fieldToUpdate: 1}; + + var multi = {multi: true}; + var single = {multi: false}; + + var doUpdate = function(query, update, multiOption, makeMongosStaleFunc) { + makeMongosStaleFunc(); + assert.writeOK(staleMongos.getCollection(collNS).update(query, update, multiOption)); + if (multiOption.multi) { + // All documents matching the query should have been updated. + assert.eq(staleMongos.getCollection(collNS).find(query).itcount(), + staleMongos.getCollection(collNS).find(queryAfterUpdate).itcount()); + } else { + // A total of one document should have been updated. + assert.eq(1, staleMongos.getCollection(collNS).find(queryAfterUpdate).itcount()); + } + }; + + var checkUpdateIsInvalid = function(query, update, multiOption, makeMongosStaleFunc, err) { + makeMongosStaleFunc(); + var res = staleMongos.getCollection(collNS).update(query, update, multiOption); + assert.writeError(res); + }; + + // This update has inconsistent behavior as explained in SERVER-22895. + // doUpdate(emptyQuery, rUpdate, single, makeMongosStaleFunc); + + // Not possible because replacement-style requires equality match on shard key. + checkUpdateIsInvalid(emptyQuery, rUpdate, multi, makeMongosStaleFunc); + + // Not possible because op-style requires equality match on shard key if single update. + checkUpdateIsInvalid(emptyQuery, oUpdate, single, makeMongosStaleFunc); + doUpdate(emptyQuery, oUpdate, multi, makeMongosStaleFunc); + + doUpdate(pointQuery, rUpdate, single, makeMongosStaleFunc); + + // Not possible because replacement-style requires multi=false. + checkUpdateIsInvalid(pointQuery, rUpdate, multi, makeMongosStaleFunc); + doUpdate(pointQuery, oUpdate, single, makeMongosStaleFunc); + doUpdate(pointQuery, oUpdate, multi, makeMongosStaleFunc); + + doUpdate(rangeQuery, rUpdate, single, makeMongosStaleFunc); + + // Not possible because replacement-style requires multi=false. + checkUpdateIsInvalid(rangeQuery, rUpdate, multi, makeMongosStaleFunc); + + // Not possible because can't do range query on a single update. + checkUpdateIsInvalid(rangeQuery, oUpdate, single, makeMongosStaleFunc); + doUpdate(rangeQuery, oUpdate, multi, makeMongosStaleFunc); + + doUpdate(multiPointQuery, rUpdate, single, makeMongosStaleFunc); + + // Not possible because replacement-style requires multi=false. + checkUpdateIsInvalid(multiPointQuery, rUpdate, multi, makeMongosStaleFunc); + + // Not possible because single remove must contain _id or shard key at top level (not within + // $or). + checkUpdateIsInvalid(multiPointQuery, oUpdate, single, makeMongosStaleFunc); + doUpdate(multiPointQuery, oUpdate, multi, makeMongosStaleFunc); + } + + var st = new ShardingTest({shards: 2, mongos: 2}); + + var dbName = 'test'; + var collNS = dbName + '.foo'; + var numShardKeys = 10; + var numDocs = numShardKeys * 2; + var splitPoint = numShardKeys / 2; + + assert.commandWorked(st.s.adminCommand({enableSharding: dbName})); + assert.commandWorked(st.s.adminCommand({shardCollection: collNS, key: {x: 1}})); + + st.ensurePrimaryShard(dbName, st.shard0.shardName); + + var freshMongos = st.s0; + var staleMongos = st.s1; + + var emptyQuery = {}; + var pointQuery = {x: 0}; + + // Choose a range that would fall on only one shard. + // Use (splitPoint - 1) because of SERVER-20768. + var rangeQuery = {x: {$gte: 0, $lt: splitPoint - 1}}; + + // Choose points that would fall on two different shards. + var multiPointQuery = {$or: [{x: 0}, {x: numShardKeys}]}; + + checkAllRemoveQueries(makeStaleMongosTargetSingleShard); + checkAllRemoveQueries(makeStaleMongosTargetMultipleShards); + + checkAllUpdateQueries(makeStaleMongosTargetSingleShard); + checkAllUpdateQueries(makeStaleMongosTargetMultipleShards); + + st.stop(); +})(); diff --git a/src/mongo/db/commands/mr.cpp b/src/mongo/db/commands/mr.cpp index 6abe8ceba72..90ee22d85a0 100644 --- a/src/mongo/db/commands/mr.cpp +++ b/src/mongo/db/commands/mr.cpp @@ -1409,16 +1409,12 @@ public: uassert(16149, "cannot run map reduce without the js engine", getGlobalScriptEngine()); // Prevent sharding state from changing during the MR. - ScopedCollectionMetadata collMetadata; - { + const auto collMetadata = [&] { // Get metadata before we check our version, to make sure it doesn't increment in the - // meantime. + // meantime AutoGetCollectionForReadCommand autoColl(opCtx, config.nss); - auto collection = autoColl.getCollection(); - if (collection) { - collMetadata = CollectionShardingState::get(opCtx, config.nss)->getMetadata(); - } - } + return CollectionShardingState::get(opCtx, config.nss)->getMetadata(); + }(); bool shouldHaveData = false; diff --git a/src/mongo/db/range_arithmetic.cpp b/src/mongo/db/range_arithmetic.cpp index 4335f668f21..6e89578c7b9 100644 --- a/src/mongo/db/range_arithmetic.cpp +++ b/src/mongo/db/range_arithmetic.cpp @@ -31,31 +31,10 @@ #include "mongo/db/range_arithmetic.h" namespace mongo { - -using std::make_pair; -using std::pair; -using std::string; -using std::stringstream; - -CachedChunkInfo::CachedChunkInfo(BSONObj maxKey, ChunkVersion version) - : _maxKey(std::move(maxKey)) {} - -bool rangeContains(const BSONObj& inclusiveLower, - const BSONObj& exclusiveUpper, - const BSONObj& point) { - return point.woCompare(inclusiveLower) >= 0 && point.woCompare(exclusiveUpper) < 0; -} - -bool rangeOverlaps(const BSONObj& inclusiveLower1, - const BSONObj& exclusiveUpper1, - const BSONObj& inclusiveLower2, - const BSONObj& exclusiveUpper2) { - return (exclusiveUpper1.woCompare(inclusiveLower2) > 0) && - (exclusiveUpper2.woCompare(inclusiveLower1) > 0); -} +namespace { // Represents the start and end of an overlap of a tested range -typedef pair<RangeMap::const_iterator, RangeMap::const_iterator> OverlapBounds; +typedef std::pair<RangeMap::const_iterator, RangeMap::const_iterator> OverlapBounds; // Internal-only, shared functionality OverlapBounds rangeMapOverlapBounds(const RangeMap& ranges, @@ -71,7 +50,7 @@ OverlapBounds rangeMapOverlapBounds(const RangeMap& ranges, --low; // If the previous range's max value is lte our min value - if (low->second.getMaxKey().woCompare(inclusiveLower) < 1) { + if (low->second.woCompare(inclusiveLower) < 1) { low = next; } } @@ -83,22 +62,27 @@ OverlapBounds rangeMapOverlapBounds(const RangeMap& ranges, return OverlapBounds(low, high); } -bool rangeMapOverlaps(const RangeMap& ranges, - const BSONObj& inclusiveLower, - const BSONObj& exclusiveUpper) { - OverlapBounds bounds = rangeMapOverlapBounds(ranges, inclusiveLower, exclusiveUpper); - return bounds.first != bounds.second; +} // namespace + +bool rangeContains(const BSONObj& inclusiveLower, + const BSONObj& exclusiveUpper, + const BSONObj& point) { + return point.woCompare(inclusiveLower) >= 0 && point.woCompare(exclusiveUpper) < 0; +} + +bool rangeOverlaps(const BSONObj& inclusiveLower1, + const BSONObj& exclusiveUpper1, + const BSONObj& inclusiveLower2, + const BSONObj& exclusiveUpper2) { + return (exclusiveUpper1.woCompare(inclusiveLower2) > 0) && + (exclusiveUpper2.woCompare(inclusiveLower1) > 0); } -bool rangeMapContains(const RangeMap& ranges, +bool rangeMapOverlaps(const RangeMap& ranges, const BSONObj& inclusiveLower, const BSONObj& exclusiveUpper) { OverlapBounds bounds = rangeMapOverlapBounds(ranges, inclusiveLower, exclusiveUpper); - if (bounds.first == ranges.end()) - return false; - - return bounds.first->first.woCompare(inclusiveLower) == 0 && - bounds.first->second.getMaxKey().woCompare(exclusiveUpper) == 0; + return bounds.first != bounds.second; } } // namespace mongo diff --git a/src/mongo/db/range_arithmetic.h b/src/mongo/db/range_arithmetic.h index 7bfdbe1cbd3..75707ca9b44 100644 --- a/src/mongo/db/range_arithmetic.h +++ b/src/mongo/db/range_arithmetic.h @@ -30,7 +30,6 @@ #include <map> #include <string> -#include <vector> #include "mongo/bson/simple_bsonobj_comparator.h" #include "mongo/db/jsobj.h" @@ -94,32 +93,12 @@ bool rangeOverlaps(const BSONObj& inclusiveLower1, const BSONObj& exclusiveUpper2); /** - * Represents a cached chunk information on the shard. - */ -class CachedChunkInfo { -public: - CachedChunkInfo(BSONObj maxKey, ChunkVersion version); - - const BSONObj& getMaxKey() const { - return _maxKey; - } - -private: - BSONObj _maxKey; -}; - -/** - * A RangeMap is a mapping of an inclusive lower BSON key to an upper key and chunk version, using - * standard BSON woCompare. The upper bound is exclusive. + * A RangeMap is a mapping of an inclusive lower BSON key to an exclusive upper key, using standard + * BSON woCompare. * * NOTE: For overlap testing to work correctly, there may be no overlaps present in the map itself. */ -typedef BSONObjIndexedMap<CachedChunkInfo> RangeMap; - -/** - * A RangeVector is a list of [lower,upper) ranges. - */ -typedef std::vector<std::pair<BSONObj, BSONObj>> RangeVector; +typedef BSONObjIndexedMap<BSONObj> RangeMap; /** * Returns true if the provided range map has ranges which overlap the provided range @@ -129,12 +108,4 @@ bool rangeMapOverlaps(const RangeMap& ranges, const BSONObj& inclusiveLower, const BSONObj& exclusiveUpper); -/** - * Returns true if the provided range map exactly contains the provided range - * [inclusiveLower, exclusiveUpper). - */ -bool rangeMapContains(const RangeMap& ranges, - const BSONObj& inclusiveLower, - const BSONObj& exclusiveUpper); - } // namespace mongo diff --git a/src/mongo/db/range_arithmetic_test.cpp b/src/mongo/db/range_arithmetic_test.cpp index e9555062d0a..f4d5f3fb14f 100644 --- a/src/mongo/db/range_arithmetic_test.cpp +++ b/src/mongo/db/range_arithmetic_test.cpp @@ -26,14 +26,14 @@ * then also delete it in the license file. */ +#include "mongo/platform/basic.h" + #include "mongo/db/range_arithmetic.h" #include "mongo/unittest/unittest.h" namespace mongo { namespace { -using std::make_pair; - TEST(BSONRange, SmallerLowerRangeNonSubset) { ASSERT_TRUE( rangeOverlaps(BSON("x" << 100), BSON("x" << 200), BSON("x" << 50), BSON("x" << 200))); @@ -71,11 +71,8 @@ TEST(BSONRange, EqualRange) { } TEST(RangeMap, RangeMapOverlaps) { - const OID epoch = OID::gen(); - - RangeMap rangeMap = SimpleBSONObjComparator::kInstance.makeBSONObjIndexedMap<CachedChunkInfo>(); - rangeMap.insert( - make_pair(BSON("x" << 100), CachedChunkInfo(BSON("x" << 200), ChunkVersion(1, 0, epoch)))); + RangeMap rangeMap = SimpleBSONObjComparator::kInstance.makeBSONObjIndexedMap<BSONObj>(); + rangeMap.insert(std::make_pair(BSON("x" << 100), BSON("x" << 200))); ASSERT(rangeMapOverlaps(rangeMap, BSON("x" << 100), BSON("x" << 200))); ASSERT(rangeMapOverlaps(rangeMap, BSON("x" << 99), BSON("x" << 200))); @@ -85,29 +82,5 @@ TEST(RangeMap, RangeMapOverlaps) { ASSERT(!rangeMapOverlaps(rangeMap, BSON("x" << 200), BSON("x" << 201))); } -TEST(RangeMap, RangeMapContains) { - const OID epoch = OID::gen(); - - RangeMap rangeMap = SimpleBSONObjComparator::kInstance.makeBSONObjIndexedMap<CachedChunkInfo>(); - rangeMap.insert( - make_pair(BSON("x" << 100), CachedChunkInfo(BSON("x" << 200), ChunkVersion(1, 0, epoch)))); - - ASSERT(rangeMapContains(rangeMap, BSON("x" << 100), BSON("x" << 200))); - ASSERT(!rangeMapContains(rangeMap, BSON("x" << 99), BSON("x" << 200))); - ASSERT(!rangeMapContains(rangeMap, BSON("x" << 100), BSON("x" << 201))); -} - -TEST(RangeMap, RangeMapContainsMinMax) { - const OID epoch = OID::gen(); - - RangeMap rangeMap = SimpleBSONObjComparator::kInstance.makeBSONObjIndexedMap<CachedChunkInfo>(); - rangeMap.insert(make_pair(BSON("x" << MINKEY), - CachedChunkInfo(BSON("x" << MAXKEY), ChunkVersion(1, 0, epoch)))); - - ASSERT(rangeMapContains(rangeMap, BSON("x" << MINKEY), BSON("x" << MAXKEY))); - ASSERT(!rangeMapContains(rangeMap, BSON("x" << 1), BSON("x" << MAXKEY))); - ASSERT(!rangeMapContains(rangeMap, BSON("x" << MINKEY), BSON("x" << 1))); -} - } // namespace } // namespace mongo diff --git a/src/mongo/db/s/collection_metadata.cpp b/src/mongo/db/s/collection_metadata.cpp index c9edf1f8194..964370cc4e8 100644 --- a/src/mongo/db/s/collection_metadata.cpp +++ b/src/mongo/db/s/collection_metadata.cpp @@ -45,8 +45,8 @@ CollectionMetadata::CollectionMetadata(std::shared_ptr<ChunkManager> cm, const S : _cm(std::move(cm)), _thisShardId(thisShardId), _shardVersion(_cm->getVersion(_thisShardId)), - _chunksMap(SimpleBSONObjComparator::kInstance.makeBSONObjIndexedMap<CachedChunkInfo>()), - _rangesMap(SimpleBSONObjComparator::kInstance.makeBSONObjIndexedMap<CachedChunkInfo>()) { + _chunksMap(SimpleBSONObjComparator::kInstance.makeBSONObjIndexedMap<BSONObj>()), + _rangesMap(SimpleBSONObjComparator::kInstance.makeBSONObjIndexedMap<BSONObj>()) { invariant(_cm->getVersion().isSet()); invariant(_cm->getVersion() >= _shardVersion); @@ -55,9 +55,7 @@ CollectionMetadata::CollectionMetadata(std::shared_ptr<ChunkManager> cm, const S if (chunk->getShardId() != _thisShardId) continue; - _chunksMap.emplace_hint(_chunksMap.end(), - chunk->getMin(), - CachedChunkInfo(chunk->getMax(), chunk->getLastmod())); + _chunksMap.emplace_hint(_chunksMap.end(), chunk->getMin(), chunk->getMax()); } if (_chunksMap.empty()) { @@ -81,7 +79,7 @@ void CollectionMetadata::_buildRangesMap() { for (const auto& entry : _chunksMap) { BSONObj const& currMin = entry.first; - BSONObj const& currMax = entry.second.getMaxKey(); + BSONObj const& currMax = entry.second; // Coalesce the chunk's bounds in ranges if they are adjacent chunks if (min.isEmpty()) { @@ -95,8 +93,7 @@ void CollectionMetadata::_buildRangesMap() { continue; } - _rangesMap.emplace_hint( - _rangesMap.end(), min, CachedChunkInfo(max, ChunkVersion::IGNORED())); + _rangesMap.emplace_hint(_rangesMap.end(), min, max); min = currMin; max = currMax; @@ -105,7 +102,7 @@ void CollectionMetadata::_buildRangesMap() { invariant(!min.isEmpty()); invariant(!max.isEmpty()); - _rangesMap.emplace(min, CachedChunkInfo(max, ChunkVersion::IGNORED())); + _rangesMap.emplace(min, max); } bool CollectionMetadata::keyBelongsToMe(const BSONObj& key) const { @@ -117,7 +114,7 @@ bool CollectionMetadata::keyBelongsToMe(const BSONObj& key) const { if (it != _rangesMap.begin()) it--; - return rangeContains(it->first, it->second.getMaxKey(), key); + return rangeContains(it->first, it->second, key); } bool CollectionMetadata::getNextChunk(const BSONObj& lookupKey, ChunkType* chunk) const { @@ -130,16 +127,15 @@ bool CollectionMetadata::getNextChunk(const BSONObj& lookupKey, ChunkType* chunk lowerChunkIt = _chunksMap.end(); } - if (lowerChunkIt != _chunksMap.end() && - lowerChunkIt->second.getMaxKey().woCompare(lookupKey) > 0) { + if (lowerChunkIt != _chunksMap.end() && lowerChunkIt->second.woCompare(lookupKey) > 0) { chunk->setMin(lowerChunkIt->first); - chunk->setMax(lowerChunkIt->second.getMaxKey()); + chunk->setMax(lowerChunkIt->second); return true; } if (upperChunkIt != _chunksMap.end()) { chunk->setMin(upperChunkIt->first); - chunk->setMax(upperChunkIt->second.getMaxKey()); + chunk->setMax(upperChunkIt->second); return true; } @@ -154,7 +150,7 @@ bool CollectionMetadata::getDifferentChunk(const BSONObj& chunkMinKey, while (lowerChunkIt != upperChunkIt) { if (lowerChunkIt->first.woCompare(chunkMinKey) != 0) { differentChunk->setMin(lowerChunkIt->first); - differentChunk->setMax(lowerChunkIt->second.getMaxKey()); + differentChunk->setMax(lowerChunkIt->second); return true; } ++lowerChunkIt; @@ -202,7 +198,7 @@ void CollectionMetadata::toBSONChunks(BSONArrayBuilder& bb) const { for (RangeMap::const_iterator it = _chunksMap.begin(); it != _chunksMap.end(); ++it) { BSONArrayBuilder chunkBB(bb.subarrayStart()); chunkBB.append(it->first); - chunkBB.append(it->second.getMaxKey()); + chunkBB.append(it->second); chunkBB.done(); } } @@ -235,8 +231,8 @@ boost::optional<KeyRange> CollectionMetadata::getNextOrphanRange( // If we overlap, continue after the overlap // TODO: Could optimize slightly by finding next non-contiguous chunk - if (lowerIt != map.end() && lowerIt->second.getMaxKey().woCompare(lookupKey) > 0) { - lookupKey = lowerIt->second.getMaxKey(); // note side effect + if (lowerIt != map.end() && lowerIt->second.woCompare(lookupKey) > 0) { + lookupKey = lowerIt->second; // note side effect return boost::none; } else { return Its(lowerIt, upperIt); @@ -258,8 +254,8 @@ boost::optional<KeyRange> CollectionMetadata::getNextOrphanRange( // bounds of the surrounding ranges in both maps. auto lowerIt = its.first, upperIt = its.second; - if (lowerIt != map.end() && lowerIt->second.getMaxKey().woCompare(range->minKey) > 0) { - range->minKey = lowerIt->second.getMaxKey(); + if (lowerIt != map.end() && lowerIt->second.woCompare(range->minKey) > 0) { + range->minKey = lowerIt->second; } if (upperIt != map.end() && upperIt->first.woCompare(range->maxKey) < 0) { range->maxKey = upperIt->first; diff --git a/src/mongo/db/s/collection_metadata_test.cpp b/src/mongo/db/s/collection_metadata_test.cpp index 20bf7c2b40e..14229bc3965 100644 --- a/src/mongo/db/s/collection_metadata_test.cpp +++ b/src/mongo/db/s/collection_metadata_test.cpp @@ -83,8 +83,7 @@ protected: }; struct stRangeMap : public RangeMap { - stRangeMap() - : RangeMap(SimpleBSONObjComparator::kInstance.makeBSONObjIndexedMap<CachedChunkInfo>()) {} + stRangeMap() : RangeMap(SimpleBSONObjComparator::kInstance.makeBSONObjIndexedMap<BSONObj>()) {} }; TEST_F(NoChunkFixture, BasicBelongsToMe) { diff --git a/src/mongo/db/s/collection_sharding_state.cpp b/src/mongo/db/s/collection_sharding_state.cpp index 93b54051160..3ead8021941 100644 --- a/src/mongo/db/s/collection_sharding_state.cpp +++ b/src/mongo/db/s/collection_sharding_state.cpp @@ -213,10 +213,7 @@ void CollectionShardingState::checkShardVersionOrThrow(OperationContext* opCtx) ChunkVersion wanted; if (!_checkShardVersionOk(opCtx, &errmsg, &received, &wanted)) { throw StaleConfigException( - _nss.ns(), - str::stream() << "[" << _nss.ns() << "] shard version not ok: " << errmsg, - received, - wanted); + _nss.ns(), str::stream() << "shard version not ok: " << errmsg, received, wanted); } } diff --git a/src/mongo/db/s/get_shard_version_command.cpp b/src/mongo/db/s/get_shard_version_command.cpp index f400d9f31bc..91c7707eae6 100644 --- a/src/mongo/db/s/get_shard_version_command.cpp +++ b/src/mongo/db/s/get_shard_version_command.cpp @@ -87,13 +87,10 @@ public: const BSONObj& cmdObj, BSONObjBuilder& result) override { const NamespaceString nss(parseNs(dbname, cmdObj)); - uassert(ErrorCodes::InvalidNamespace, - str::stream() << nss.ns() << " is not a valid namespace", - nss.isValid()); - ShardingState* const gss = ShardingState::get(opCtx); - if (gss->enabled()) { - result.append("configServer", gss->getConfigServer(opCtx).toString()); + ShardingState* const shardingState = ShardingState::get(opCtx); + if (shardingState->enabled()) { + result.append("configServer", shardingState->getConfigServer(opCtx).toString()); } else { result.append("configServer", ""); } @@ -109,11 +106,7 @@ public: AutoGetCollection autoColl(opCtx, nss, MODE_IS); CollectionShardingState* const css = CollectionShardingState::get(opCtx, nss); - ScopedCollectionMetadata metadata; - if (css) { - metadata = css->getMetadata(); - } - + const auto metadata = css->getMetadata(); if (metadata) { result.appendTimestamp("global", metadata->getShardVersion().toLong()); } else { diff --git a/src/mongo/db/s/merge_chunks_command.cpp b/src/mongo/db/s/merge_chunks_command.cpp index c6281ce9bdb..f86073a55da 100644 --- a/src/mongo/db/s/merge_chunks_command.cpp +++ b/src/mongo/db/s/merge_chunks_command.cpp @@ -57,17 +57,18 @@ using std::vector; namespace { -bool _checkMetadataForSuccess(OperationContext* opCtx, - const NamespaceString& nss, - const BSONObj& minKey, - const BSONObj& maxKey) { - ScopedCollectionMetadata metadataAfterMerge; - { +bool checkMetadataForSuccess(OperationContext* opCtx, + const NamespaceString& nss, + const BSONObj& minKey, + const BSONObj& maxKey) { + const auto metadataAfterMerge = [&] { AutoGetCollection autoColl(opCtx, nss, MODE_IS); + return CollectionShardingState::get(opCtx, nss.ns())->getMetadata(); + }(); - // Get collection metadata - metadataAfterMerge = CollectionShardingState::get(opCtx, nss.ns())->getMetadata(); - } + uassert(ErrorCodes::StaleConfig, + str::stream() << "Collection " << nss.ns() << " became unsharded", + metadataAfterMerge); ChunkType chunk; if (!metadataAfterMerge->getNextChunk(minKey, &chunk)) { @@ -100,15 +101,14 @@ Status mergeChunks(OperationContext* opCtx, return Status(scopedDistLock.getStatus().code(), errmsg); } - ShardingState* shardingState = ShardingState::get(opCtx); + auto const shardingState = ShardingState::get(opCtx); // // We now have the collection lock, refresh metadata to latest version and sanity check // - ChunkVersion shardVersion; - Status refreshStatus = shardingState->refreshMetadataNow(opCtx, nss, &shardVersion); - + ChunkVersion unusedShardVersion; + Status refreshStatus = shardingState->refreshMetadataNow(opCtx, nss, &unusedShardVersion); if (!refreshStatus.isOK()) { std::string errmsg = str::stream() << "could not merge chunks, failed to refresh metadata for " << nss.ns() @@ -118,33 +118,31 @@ Status mergeChunks(OperationContext* opCtx, return Status(refreshStatus.code(), errmsg); } - if (epoch.isSet() && shardVersion.epoch() != epoch) { - std::string errmsg = stream() - << "could not merge chunks, collection " << nss.ns() << " has changed" - << " since merge was sent" - << "(sent epoch : " << epoch.toString() - << ", current epoch : " << shardVersion.epoch().toString() << ")"; + const auto metadata = [&] { + AutoGetCollection autoColl(opCtx, nss, MODE_IS); + return CollectionShardingState::get(opCtx, nss.ns())->getMetadata(); + }(); + + if (!metadata) { + std::string errmsg = stream() << "could not merge chunks, collection " << nss.ns() + << " is not sharded"; warning() << errmsg; - return Status(ErrorCodes::StaleEpoch, errmsg); + return {ErrorCodes::StaleEpoch, errmsg}; } - ScopedCollectionMetadata metadata; - { - AutoGetCollection autoColl(opCtx, nss, MODE_IS); + const auto shardVersion = metadata->getShardVersion(); - metadata = CollectionShardingState::get(opCtx, nss.ns())->getMetadata(); - if (!metadata) { - std::string errmsg = stream() << "could not merge chunks, collection " << nss.ns() - << " is not sharded"; + if (epoch.isSet() && shardVersion.epoch() != epoch) { + std::string errmsg = stream() + << "could not merge chunks, collection " << nss.ns() + << " has changed since merge was sent (sent epoch: " << epoch.toString() + << ", current epoch: " << shardVersion.epoch() << ")"; - warning() << errmsg; - return Status(ErrorCodes::IllegalOperation, errmsg); - } + warning() << errmsg; + return {ErrorCodes::StaleEpoch, errmsg}; } - dassert(metadata->getShardVersion().equals(shardVersion)); - if (!metadata->isValidKey(minKey) || !metadata->isValidKey(maxKey)) { std::string errmsg = stream() << "could not merge chunks, the range " << redact(ChunkRange(minKey, maxKey).toString()) @@ -156,7 +154,6 @@ Status mergeChunks(OperationContext* opCtx, return Status(ErrorCodes::IllegalOperation, errmsg); } - // // Get merged chunk information // @@ -277,8 +274,8 @@ Status mergeChunks(OperationContext* opCtx, // running _configsvrCommitChunkMerge). // { - ChunkVersion shardVersionAfterMerge; - refreshStatus = shardingState->refreshMetadataNow(opCtx, nss, &shardVersionAfterMerge); + ChunkVersion unusedShardVersion; + refreshStatus = shardingState->refreshMetadataNow(opCtx, nss, &unusedShardVersion); if (!refreshStatus.isOK()) { std::string errmsg = str::stream() << "failed to refresh metadata for merge chunk [" @@ -304,7 +301,7 @@ Status mergeChunks(OperationContext* opCtx, auto writeConcernStatus = std::move(cmdResponseStatus.getValue().writeConcernStatus); if ((!commandStatus.isOK() || !writeConcernStatus.isOK()) && - _checkMetadataForSuccess(opCtx, nss, minKey, maxKey)) { + checkMetadataForSuccess(opCtx, nss, minKey, maxKey)) { LOG(1) << "mergeChunk [" << redact(minKey) << "," << redact(maxKey) << ") has already been committed."; diff --git a/src/mongo/db/s/metadata_manager.cpp b/src/mongo/db/s/metadata_manager.cpp index ad26fe22a10..c98a506f05b 100644 --- a/src/mongo/db/s/metadata_manager.cpp +++ b/src/mongo/db/s/metadata_manager.cpp @@ -160,8 +160,7 @@ MetadataManager::MetadataManager(ServiceContext* serviceContext, : _serviceContext(serviceContext), _nss(std::move(nss)), _executor(executor), - _receivingChunks( - SimpleBSONObjComparator::kInstance.makeBSONObjIndexedMap<CachedChunkInfo>()) {} + _receivingChunks(SimpleBSONObjComparator::kInstance.makeBSONObjIndexedMap<BSONObj>()) {} MetadataManager::~MetadataManager() { stdx::lock_guard<stdx::mutex> lg(_managerLock); @@ -265,7 +264,7 @@ void MetadataManager::refreshActiveMetadata(std::unique_ptr<CollectionMetadata> // Should be no more than one. for (auto it = _receivingChunks.begin(); it != _receivingChunks.end();) { BSONObj const& min = it->first; - BSONObj const& max = it->second.getMaxKey(); + BSONObj const& max = it->second; if (!remoteMetadata->rangeOverlapsChunk(ChunkRange(min, max))) { ++it; @@ -308,7 +307,7 @@ void MetadataManager::toBSONPending(BSONArrayBuilder& bb) const { for (auto it = _receivingChunks.begin(); it != _receivingChunks.end(); ++it) { BSONArrayBuilder pendingBB(bb.subarrayStart()); pendingBB.append(it->first); - pendingBB.append(it->second.getMaxKey()); + pendingBB.append(it->second); pendingBB.done(); } } @@ -321,7 +320,7 @@ void MetadataManager::append(BSONObjBuilder* builder) const { BSONArrayBuilder pcArr(builder->subarrayStart("pendingChunks")); for (const auto& entry : _receivingChunks) { BSONObjBuilder obj; - ChunkRange r = ChunkRange(entry.first, entry.second.getMaxKey()); + ChunkRange r = ChunkRange(entry.first, entry.second); r.append(&obj); pcArr.append(obj.done()); } @@ -334,7 +333,7 @@ void MetadataManager::append(BSONObjBuilder* builder) const { BSONArrayBuilder amrArr(builder->subarrayStart("activeMetadataRanges")); for (const auto& entry : _metadata.back()->metadata.getChunks()) { BSONObjBuilder obj; - ChunkRange r = ChunkRange(entry.first, entry.second.getMaxKey()); + ChunkRange r = ChunkRange(entry.first, entry.second); r.append(&obj); amrArr.append(obj.done()); } @@ -360,9 +359,7 @@ void MetadataManager::_pushListToClean(WithLock, std::list<Deletion> ranges) { } void MetadataManager::_addToReceiving(WithLock, ChunkRange const& range) { - _receivingChunks.insert( - std::make_pair(range.getMin().getOwned(), - CachedChunkInfo(range.getMax().getOwned(), ChunkVersion::IGNORED()))); + _receivingChunks.insert(std::make_pair(range.getMin().getOwned(), range.getMax().getOwned())); } auto MetadataManager::beginReceive(ChunkRange const& range) -> CleanupNotification { diff --git a/src/mongo/db/s/metadata_manager_test.cpp b/src/mongo/db/s/metadata_manager_test.cpp index 7dc36756544..eb0999b6b00 100644 --- a/src/mongo/db/s/metadata_manager_test.cpp +++ b/src/mongo/db/s/metadata_manager_test.cpp @@ -304,7 +304,7 @@ TEST_F(MetadataManagerTest, RefreshMetadataAfterDropAndRecreate) { const auto chunkEntry = _manager->getActiveMetadata(_manager)->getChunks().begin(); ASSERT_BSONOBJ_EQ(BSON("key" << 20), chunkEntry->first); - ASSERT_BSONOBJ_EQ(BSON("key" << 30), chunkEntry->second.getMaxKey()); + ASSERT_BSONOBJ_EQ(BSON("key" << 30), chunkEntry->second); } // Tests membership functions for _rangesToClean diff --git a/src/mongo/db/s/migration_destination_manager_legacy_commands.cpp b/src/mongo/db/s/migration_destination_manager_legacy_commands.cpp index d7d1a351973..1024433b5d6 100644 --- a/src/mongo/db/s/migration_destination_manager_legacy_commands.cpp +++ b/src/mongo/db/s/migration_destination_manager_legacy_commands.cpp @@ -102,9 +102,8 @@ public: // Refresh our collection manager from the config server, we need a collection manager to // start registering pending chunks. We force the remote refresh here to make the behavior // consistent and predictable, generally we'd refresh anyway, and to be paranoid. - ChunkVersion currentVersion; - - Status status = shardingState->refreshMetadataNow(opCtx, nss, ¤tVersion); + ChunkVersion shardVersion; + Status status = shardingState->refreshMetadataNow(opCtx, nss, &shardVersion); if (!status.isOK()) { errmsg = str::stream() << "cannot start receiving chunk " << redact(chunkRange.toString()) << causedBy(redact(status)); @@ -147,7 +146,7 @@ public: chunkRange.getMin(), chunkRange.getMax(), shardKeyPattern, - currentVersion.epoch(), + shardVersion.epoch(), writeConcern)); result.appendBool("started", true); diff --git a/src/mongo/db/s/migration_source_manager.cpp b/src/mongo/db/s/migration_source_manager.cpp index b26dfb91b2e..9d9d424bf24 100644 --- a/src/mongo/db/s/migration_source_manager.cpp +++ b/src/mongo/db/s/migration_source_manager.cpp @@ -36,8 +36,6 @@ #include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/db_raii.h" #include "mongo/db/operation_context.h" -#include "mongo/db/s/collection_metadata.h" -#include "mongo/db/s/collection_sharding_state.h" #include "mongo/db/s/migration_chunk_cloner_source_legacy.h" #include "mongo/db/s/migration_util.h" #include "mongo/db/s/shard_metadata_util.h" @@ -126,8 +124,7 @@ MigrationSourceManager::MigrationSourceManager(OperationContext* opCtx, HostAndPort recipientHost) : _args(std::move(request)), _donorConnStr(std::move(donorConnStr)), - _recipientHost(std::move(recipientHost)), - _startTime() { + _recipientHost(std::move(recipientHost)) { invariant(!opCtx->lockState()->isLocked()); // Disallow moving a chunk to ourselves @@ -226,10 +223,10 @@ Status MigrationSourceManager::startClone(OperationContext* opCtx) { << "to" << _args.getToShardId()), ShardingCatalogClient::kMajorityWriteConcern) - .transitional_ignore(); + .ignore(); _cloneDriver = stdx::make_unique<MigrationChunkClonerSourceLegacy>( - _args, _collectionMetadata->getKeyPattern(), _donorConnStr, _recipientHost); + _args, _keyPattern, _donorConnStr, _recipientHost); { // Register for notifications from the replication subsystem @@ -271,38 +268,13 @@ Status MigrationSourceManager::enterCriticalSection(OperationContext* opCtx) { invariant(_state == kCloneCaughtUp); auto scopedGuard = MakeGuard([&] { cleanupOnError(opCtx); }); - const ShardId& recipientId = _args.getToShardId(); - if (!_collectionMetadata->getChunkManager()->getVersion(recipientId).isSet() && - (serverGlobalParams.featureCompatibility.getVersion() == - ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo36)) { - // The recipient didn't have any chunks of this collection. Write the no-op message so that - // change stream will notice that and close cursor to notify mongos to target to the new - // shard. - std::stringstream ss; - // The message for debugging. - ss << "Migrating chunk from shard " << _args.getFromShardId() << " to shard " - << _args.getToShardId() << " with no chunks for this collection"; - // The message expected by change streams. - auto message = BSON("type" - << "migrateChunkToNewShard" - << "from" - << _args.getFromShardId() - << "to" - << _args.getToShardId()); - AutoGetCollection autoColl(opCtx, NamespaceString::kRsOplogNamespace, MODE_IX); - writeConflictRetry( - opCtx, "migrateChunkToNewShard", NamespaceString::kRsOplogNamespace.ns(), [&] { - WriteUnitOfWork uow(opCtx); - opCtx->getClient()->getServiceContext()->getOpObserver()->onInternalOpMessage( - opCtx, getNss(), _collectionUuid, BSON("msg" << ss.str()), message); - uow.commit(); - }); - } + _notifyChangeStreamsOnRecipientFirstChunk(opCtx); // Mark the shard as running critical operation, which requires recovery on crash. // - // Note: the 'migrateChunkToNewShard' oplog message written above depends on this - // majority write to carry its local write to majority committed. + // NOTE: The 'migrateChunkToNewShard' oplog message written by the above call to + // '_notifyChangeStreamsOnRecipientFirstChunk' depends on this majority write to carry its local + // write to majority committed. Status status = ShardingStateRecovery::startMetadataOp(opCtx); if (!status.isOK()) { return status; @@ -547,7 +519,7 @@ Status MigrationSourceManager::commitChunkMetadataOnConfig(OperationContext* opC << "to" << _args.getToShardId()), ShardingCatalogClient::kMajorityWriteConcern) - .transitional_ignore(); + .ignore(); // Wait for the metadata update to be persisted before attempting to delete orphaned documents // so that metadata changes propagate to secondaries first @@ -605,11 +577,45 @@ void MigrationSourceManager::cleanupOnError(OperationContext* opCtx) { << "to" << _args.getToShardId()), ShardingCatalogClient::kMajorityWriteConcern) - .transitional_ignore(); + .ignore(); _cleanup(opCtx); } +void MigrationSourceManager::_notifyChangeStreamsOnRecipientFirstChunk(OperationContext* opCtx) { + // Change streams are only supported in 3.6 and above + if (serverGlobalParams.featureCompatibility.getVersion() != + ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo36) + return; + + // If this is not the first donation, there is nothing to be done + if (_collectionMetadata->getChunkManager()->getVersion(_args.getToShardId()).isSet()) + return; + + const std::string dbgMessage = str::stream() + << "Migrating chunk from shard " << _args.getFromShardId() << " to shard " + << _args.getToShardId() << " with no chunks for this collection"; + + // The message expected by change streams + const auto o2Message = BSON("type" + << "migrateChunkToNewShard" + << "from" + << _args.getFromShardId() + << "to" + << _args.getToShardId()); + + auto const serviceContext = opCtx->getClient()->getServiceContext(); + + AutoGetCollection autoColl(opCtx, NamespaceString::kRsOplogNamespace, MODE_IX); + writeConflictRetry( + opCtx, "migrateChunkToNewShard", NamespaceString::kRsOplogNamespace.ns(), [&] { + WriteUnitOfWork uow(opCtx); + serviceContext->getOpObserver()->onInternalOpMessage( + opCtx, getNss(), _collectionUuid, BSON("msg" << dbgMessage), o2Message); + uow.commit(); + }); +} + void MigrationSourceManager::_cleanup(OperationContext* opCtx) { invariant(_state != kDone); diff --git a/src/mongo/db/s/migration_source_manager.h b/src/mongo/db/s/migration_source_manager.h index 0f90ac96f2b..255aa984024 100644 --- a/src/mongo/db/s/migration_source_manager.h +++ b/src/mongo/db/s/migration_source_manager.h @@ -28,12 +28,9 @@ #pragma once -#include <string> - #include "mongo/base/disallow_copying.h" -#include "mongo/db/s/metadata_manager.h" +#include "mongo/db/s/collection_sharding_state.h" #include "mongo/s/move_chunk_request.h" -#include "mongo/s/shard_key_pattern.h" #include "mongo/util/concurrency/notification.h" #include "mongo/util/timer.h" @@ -157,13 +154,6 @@ public: void cleanupOnError(OperationContext* opCtx); /** - * Returns the key pattern object for the stored committed metadata. - */ - BSONObj getKeyPattern() const { - return _keyPattern; - } - - /** * Returns the cloner which is being used for this migration. This value is available only if * the migration source manager is currently in the clone phase (i.e. the previous call to * startClone has succeeded). @@ -197,6 +187,13 @@ private: enum State { kCreated, kCloning, kCloneCaughtUp, kCriticalSection, kCloneCompleted, kDone }; /** + * If this donation moves the first chunk to the recipient (i.e., the recipient didn't have any + * chunks), this function writes a no-op message to the oplog, so that change stream will notice + * that and close the cursor in order to notify mongos to target the new shard as well. + */ + void _notifyChangeStreamsOnRecipientFirstChunk(OperationContext* opCtx); + + /** * Called when any of the states fails. May only be called once and will put the migration * manager into the kDone state. */ diff --git a/src/mongo/db/s/sharding_state.cpp b/src/mongo/db/s/sharding_state.cpp index 2f46a11552e..608a4bac233 100644 --- a/src/mongo/db/s/sharding_state.cpp +++ b/src/mongo/db/s/sharding_state.cpp @@ -243,26 +243,23 @@ Status ShardingState::onStaleShardVersion(OperationContext* opCtx, auto& oss = OperationShardingState::get(opCtx); oss.waitForMigrationCriticalSectionSignal(opCtx); - ChunkVersion collectionShardVersion; - - // Fast path - check if the requested version is at a higher version than the current metadata - // version or a different epoch before verifying against config server. - ScopedCollectionMetadata currentMetadata; - - { + const auto collectionShardVersion = [&] { + // Fast path - check if the requested version is at a higher version than the current + // metadata version or a different epoch before verifying against config server AutoGetCollection autoColl(opCtx, nss, MODE_IS); - - currentMetadata = CollectionShardingState::get(opCtx, nss)->getMetadata(); + const auto currentMetadata = CollectionShardingState::get(opCtx, nss)->getMetadata(); if (currentMetadata) { - collectionShardVersion = currentMetadata->getShardVersion(); + return currentMetadata->getShardVersion(); } - if (collectionShardVersion.epoch() == expectedVersion.epoch() && - collectionShardVersion >= expectedVersion) { - // Don't need to remotely reload if we're in the same epoch and the requested version is - // smaller than the one we know about. This means that the remote side is behind. - return Status::OK(); - } + return ChunkVersion::UNSHARDED(); + }(); + + if (collectionShardVersion.epoch() == expectedVersion.epoch() && + collectionShardVersion >= expectedVersion) { + // Don't need to remotely reload if we're in the same epoch and the requested version is + // smaller than the one we know about. This means that the remote side is behind. + return Status::OK(); } try { diff --git a/src/mongo/db/s/split_chunk.cpp b/src/mongo/db/s/split_chunk.cpp index 3dea2009f94..77120fb2674 100644 --- a/src/mongo/db/s/split_chunk.cpp +++ b/src/mongo/db/s/split_chunk.cpp @@ -95,13 +95,14 @@ bool checkMetadataForSuccessfulSplitChunk(OperationContext* opCtx, const NamespaceString& nss, const ChunkRange& chunkRange, const std::vector<BSONObj>& splitKeys) { - ScopedCollectionMetadata metadataAfterSplit; - { + const auto metadataAfterSplit = [&] { AutoGetCollection autoColl(opCtx, nss, MODE_IS); + return CollectionShardingState::get(opCtx, nss.ns())->getMetadata(); + }(); - // Get collection metadata - metadataAfterSplit = CollectionShardingState::get(opCtx, nss.ns())->getMetadata(); - } + uassert(ErrorCodes::StaleConfig, + str::stream() << "Collection " << nss.ns() << " became unsharded", + metadataAfterSplit); auto newChunkBounds(splitKeys); auto startKey = chunkRange.getMin(); |