diff options
Diffstat (limited to 'src/mongo/s/chunk_diff.cpp')
| -rw-r--r-- | src/mongo/s/chunk_diff.cpp | 354 |
1 files changed, 173 insertions, 181 deletions
diff --git a/src/mongo/s/chunk_diff.cpp b/src/mongo/s/chunk_diff.cpp index 488ea85728b..b270a54d9d3 100644 --- a/src/mongo/s/chunk_diff.cpp +++ b/src/mongo/s/chunk_diff.cpp @@ -42,223 +42,215 @@ namespace mongo { - template <class ValType, class ShardType> - ConfigDiffTracker<ValType, ShardType>::ConfigDiffTracker() { - _ns.clear(); - _currMap = NULL; - _maxVersion = NULL; - _maxShardVersions = NULL; - _validDiffs = 0; - } +template <class ValType, class ShardType> +ConfigDiffTracker<ValType, ShardType>::ConfigDiffTracker() { + _ns.clear(); + _currMap = NULL; + _maxVersion = NULL; + _maxShardVersions = NULL; + _validDiffs = 0; +} + +template <class ValType, class ShardType> +ConfigDiffTracker<ValType, ShardType>::~ConfigDiffTracker() = default; + +template <class ValType, class ShardType> +void ConfigDiffTracker<ValType, ShardType>::attach(const std::string& ns, + RangeMap& currMap, + ChunkVersion& maxVersion, + MaxChunkVersionMap& maxShardVersions) { + _ns = ns; + _currMap = &currMap; + _maxVersion = &maxVersion; + _maxShardVersions = &maxShardVersions; + _validDiffs = 0; +} + +template <class ValType, class ShardType> +bool ConfigDiffTracker<ValType, ShardType>::isOverlapping(const BSONObj& min, const BSONObj& max) { + RangeOverlap overlap = overlappingRange(min, max); + + return overlap.first != overlap.second; +} + +template <class ValType, class ShardType> +void ConfigDiffTracker<ValType, ShardType>::removeOverlapping(const BSONObj& min, + const BSONObj& max) { + _assertAttached(); - template <class ValType, class ShardType> - ConfigDiffTracker<ValType, ShardType>::~ConfigDiffTracker() = default; - - template <class ValType, class ShardType> - void ConfigDiffTracker<ValType, ShardType>::attach(const std::string& ns, - RangeMap& currMap, - ChunkVersion& maxVersion, - MaxChunkVersionMap& maxShardVersions) { - _ns = ns; - _currMap = &currMap; - _maxVersion = &maxVersion; - _maxShardVersions = &maxShardVersions; - _validDiffs = 0; - } + RangeOverlap overlap = overlappingRange(min, max); - template <class ValType, class ShardType> - bool ConfigDiffTracker<ValType, ShardType>::isOverlapping(const BSONObj& min, - const BSONObj& max) { - RangeOverlap overlap = overlappingRange(min, max); + _currMap->erase(overlap.first, overlap.second); +} - return overlap.first != overlap.second; - } +template <class ValType, class ShardType> +typename ConfigDiffTracker<ValType, ShardType>::RangeOverlap +ConfigDiffTracker<ValType, ShardType>::overlappingRange(const BSONObj& min, const BSONObj& max) { + _assertAttached(); + + typename RangeMap::iterator low; + typename RangeMap::iterator high; - template <class ValType, class ShardType> - void ConfigDiffTracker<ValType, ShardType>::removeOverlapping(const BSONObj& min, - const BSONObj& max) { - _assertAttached(); + if (isMinKeyIndexed()) { + // Returns the first chunk with a min key that is >= min - implies the + // previous chunk cannot overlap min + low = _currMap->lower_bound(min); - RangeOverlap overlap = overlappingRange(min, max); + // Returns the first chunk with a min key that is >= max - implies the + // chunk does not overlap max + high = _currMap->lower_bound(max); + } else { + // Returns the first chunk with a max key that is > min - implies that + // the chunk overlaps min + low = _currMap->upper_bound(min); - _currMap->erase(overlap.first, overlap.second); + // Returns the first chunk with a max key that is > max - implies that + // the next chunk cannot not overlap max + high = _currMap->upper_bound(max); } - template <class ValType, class ShardType> - typename ConfigDiffTracker<ValType, ShardType>::RangeOverlap - ConfigDiffTracker<ValType, ShardType>::overlappingRange(const BSONObj& min, - const BSONObj& max) { - _assertAttached(); + return RangeOverlap(low, high); +} - typename RangeMap::iterator low; - typename RangeMap::iterator high; +template <class ValType, class ShardType> +int ConfigDiffTracker<ValType, ShardType>::calculateConfigDiff(CatalogManager* catalogManager) { + _assertAttached(); - if (isMinKeyIndexed()) { - // Returns the first chunk with a min key that is >= min - implies the - // previous chunk cannot overlap min - low = _currMap->lower_bound(min); + // Get the diff query required + Query diffQuery = configDiffQuery(); - // Returns the first chunk with a min key that is >= max - implies the - // chunk does not overlap max - high = _currMap->lower_bound(max); - } - else { - // Returns the first chunk with a max key that is > min - implies that - // the chunk overlaps min - low = _currMap->upper_bound(min); - - // Returns the first chunk with a max key that is > max - implies that - // the next chunk cannot not overlap max - high = _currMap->upper_bound(max); - } + try { + std::vector<ChunkType> chunks; + uassertStatusOK(catalogManager->getChunks(diffQuery, 0, &chunks)); - return RangeOverlap(low, high); + return calculateConfigDiff(chunks); + } catch (DBException& e) { + // Should only happen on connection errors + e.addContext(str::stream() << "could not calculate config difference for ns " << _ns); + throw; } +} + +template <class ValType, class ShardType> +int ConfigDiffTracker<ValType, ShardType>::calculateConfigDiff( + const std::vector<ChunkType>& chunks) { + _assertAttached(); + + // Apply the chunk changes to the ranges and versions + // + // Overall idea here is to work in two steps : + // 1. For all the new chunks we find, increment the maximum version per-shard and + // per-collection, and remove any conflicting chunks from the ranges. + // 2. For all the new chunks we're interested in (all of them for mongos, just chunks on + // the shard for mongod) add them to the ranges. - template<class ValType, class ShardType> - int ConfigDiffTracker<ValType, ShardType>::calculateConfigDiff( - CatalogManager* catalogManager) { - _assertAttached(); + std::vector<ChunkType> newTracked; - // Get the diff query required - Query diffQuery = configDiffQuery(); + // Store epoch now so it doesn't change when we change max + OID currEpoch = _maxVersion->epoch(); - try { - std::vector<ChunkType> chunks; - uassertStatusOK(catalogManager->getChunks(diffQuery, 0, &chunks)); + _validDiffs = 0; - return calculateConfigDiff(chunks); + for (const ChunkType& chunk : chunks) { + ChunkVersion chunkVersion = + ChunkVersion::fromBSON(chunk.toBSON(), ChunkType::DEPRECATED_lastmod()); + + if (!chunkVersion.isSet() || !chunkVersion.hasEqualEpoch(currEpoch)) { + warning() << "got invalid chunk version " << chunkVersion << " in document " + << chunk.toString() << " when trying to load differing chunks at version " + << ChunkVersion( + _maxVersion->majorVersion(), _maxVersion->minorVersion(), currEpoch); + + // Don't keep loading, since we know we'll be broken here + return -1; } - catch (DBException& e) { - // Should only happen on connection errors - e.addContext(str::stream() << "could not calculate config difference for ns " << _ns); - throw; + + _validDiffs++; + + // Get max changed version and chunk version + if (chunkVersion > *_maxVersion) { + *_maxVersion = chunkVersion; } - } - template<class ValType, class ShardType> - int ConfigDiffTracker<ValType, ShardType>::calculateConfigDiff( - const std::vector<ChunkType>& chunks) { - _assertAttached(); + // Chunk version changes + ShardType shard = shardFor(chunk.getShard()); - // Apply the chunk changes to the ranges and versions - // - // Overall idea here is to work in two steps : - // 1. For all the new chunks we find, increment the maximum version per-shard and - // per-collection, and remove any conflicting chunks from the ranges. - // 2. For all the new chunks we're interested in (all of them for mongos, just chunks on - // the shard for mongod) add them to the ranges. - - std::vector<ChunkType> newTracked; - - // Store epoch now so it doesn't change when we change max - OID currEpoch = _maxVersion->epoch(); - - _validDiffs = 0; - - for (const ChunkType& chunk : chunks) { - ChunkVersion chunkVersion = ChunkVersion::fromBSON(chunk.toBSON(), - ChunkType::DEPRECATED_lastmod()); - - if (!chunkVersion.isSet() || !chunkVersion.hasEqualEpoch(currEpoch)) { - warning() << "got invalid chunk version " << chunkVersion - << " in document " << chunk.toString() - << " when trying to load differing chunks at version " - << ChunkVersion(_maxVersion->majorVersion(), - _maxVersion->minorVersion(), - currEpoch); - - // Don't keep loading, since we know we'll be broken here - return -1; - } - - _validDiffs++; - - // Get max changed version and chunk version - if (chunkVersion > *_maxVersion) { - *_maxVersion = chunkVersion; - } - - // Chunk version changes - ShardType shard = shardFor(chunk.getShard()); - - typename MaxChunkVersionMap::const_iterator shardVersionIt = _maxShardVersions->find(shard); - if (shardVersionIt == _maxShardVersions->end() || - shardVersionIt->second < chunkVersion) { - (*_maxShardVersions)[shard] = chunkVersion; - } - - // See if we need to remove any chunks we are currently tracking because of this - // chunk's changes - removeOverlapping(chunk.getMin(), chunk.getMax()); - - // Figure out which of the new chunks we need to track - // Important - we need to actually own this doc, in case the cursor decides to getMore - // or unbuffer. - if (isTracked(chunk)) { - newTracked.push_back(chunk); - } + typename MaxChunkVersionMap::const_iterator shardVersionIt = _maxShardVersions->find(shard); + if (shardVersionIt == _maxShardVersions->end() || shardVersionIt->second < chunkVersion) { + (*_maxShardVersions)[shard] = chunkVersion; } - LOG(3) << "found " << _validDiffs << " new chunks for collection " << _ns - << " (tracking " << newTracked.size() << "), new version is " << *_maxVersion; + // See if we need to remove any chunks we are currently tracking because of this + // chunk's changes + removeOverlapping(chunk.getMin(), chunk.getMax()); - for (const ChunkType& chunk : newTracked) { - // Invariant enforced by sharding - it's possible to read inconsistent state due to - // getMore and yielding, so we want to detect it as early as possible. - // - // TODO: This checks for overlap, we also should check for holes here iff we're - // tracking all chunks. - if (isOverlapping(chunk.getMin(), chunk.getMax())) { - return -1; - } + // Figure out which of the new chunks we need to track + // Important - we need to actually own this doc, in case the cursor decides to getMore + // or unbuffer. + if (isTracked(chunk)) { + newTracked.push_back(chunk); + } + } + + LOG(3) << "found " << _validDiffs << " new chunks for collection " << _ns << " (tracking " + << newTracked.size() << "), new version is " << *_maxVersion; - _currMap->insert(rangeFor(chunk)); + for (const ChunkType& chunk : newTracked) { + // Invariant enforced by sharding - it's possible to read inconsistent state due to + // getMore and yielding, so we want to detect it as early as possible. + // + // TODO: This checks for overlap, we also should check for holes here iff we're + // tracking all chunks. + if (isOverlapping(chunk.getMin(), chunk.getMax())) { + return -1; } - return _validDiffs; + _currMap->insert(rangeFor(chunk)); } - template<class ValType, class ShardType> - Query ConfigDiffTracker<ValType, ShardType>::configDiffQuery() const { - _assertAttached(); + return _validDiffs; +} - // Basic idea behind the query is to find all the chunks $gte the current max version. - // Currently, any splits and merges will increment the current max version. - BSONObjBuilder queryB; - queryB.append(ChunkType::ns(), _ns); +template <class ValType, class ShardType> +Query ConfigDiffTracker<ValType, ShardType>::configDiffQuery() const { + _assertAttached(); - { - BSONObjBuilder tsBuilder(queryB.subobjStart(ChunkType::DEPRECATED_lastmod())); - tsBuilder.appendTimestamp("$gte", _maxVersion->toLong()); - tsBuilder.done(); - } + // Basic idea behind the query is to find all the chunks $gte the current max version. + // Currently, any splits and merges will increment the current max version. + BSONObjBuilder queryB; + queryB.append(ChunkType::ns(), _ns); - // NOTE: IT IS IMPORTANT FOR CONSISTENCY THAT WE SORT BY ASC VERSION, IN ORDER TO HANDLE - // CURSOR YIELDING BETWEEN CHUNKS BEING MIGRATED. - // - // This ensures that changes to chunk version (which will always be higher) will always - // come *after* our current position in the chunk cursor. + { + BSONObjBuilder tsBuilder(queryB.subobjStart(ChunkType::DEPRECATED_lastmod())); + tsBuilder.appendTimestamp("$gte", _maxVersion->toLong()); + tsBuilder.done(); + } - Query queryObj(queryB.obj()); - queryObj.sort(BSON("lastmod" << 1)); + // NOTE: IT IS IMPORTANT FOR CONSISTENCY THAT WE SORT BY ASC VERSION, IN ORDER TO HANDLE + // CURSOR YIELDING BETWEEN CHUNKS BEING MIGRATED. + // + // This ensures that changes to chunk version (which will always be higher) will always + // come *after* our current position in the chunk cursor. - LOG(2) << "major version query from " << *_maxVersion - << " and over " << _maxShardVersions->size() << " shards is " << queryObj; + Query queryObj(queryB.obj()); + queryObj.sort(BSON("lastmod" << 1)); - return queryObj; - } + LOG(2) << "major version query from " << *_maxVersion << " and over " + << _maxShardVersions->size() << " shards is " << queryObj; - template <class ValType, class ShardType> - void ConfigDiffTracker<ValType, ShardType>::_assertAttached() const { - invariant(_currMap); - invariant(_maxVersion); - invariant(_maxShardVersions); - } + return queryObj; +} + +template <class ValType, class ShardType> +void ConfigDiffTracker<ValType, ShardType>::_assertAttached() const { + invariant(_currMap); + invariant(_maxVersion); + invariant(_maxShardVersions); +} - // Ensures that these instances of the template are compiled - template class ConfigDiffTracker<BSONObj, std::string>; - template class ConfigDiffTracker<std::shared_ptr<Chunk>, std::string>; +// Ensures that these instances of the template are compiled +template class ConfigDiffTracker<BSONObj, std::string>; +template class ConfigDiffTracker<std::shared_ptr<Chunk>, std::string>; -} // namespace mongo +} // namespace mongo |