diff options
author | Esha Maharishi <esha.maharishi@mongodb.com> | 2016-03-08 17:56:34 -0500 |
---|---|---|
committer | Esha Maharishi <esha.maharishi@mongodb.com> | 2016-03-14 19:05:31 -0400 |
commit | 50f37a888d2d4f0a53196648b4ef9ada4a039be9 (patch) | |
tree | b9f43bdc078adfce417a2db8b54da4e2df470ad1 | |
parent | 9ffa6bc05261012b9df16b62a2915a61dcb8f51d (diff) | |
download | mongo-50f37a888d2d4f0a53196648b4ef9ada4a039be9.tar.gz |
SERVER-22483 convert disabled SCCC dbtests to jstests or unittests that use CSRS
18 files changed, 463 insertions, 1481 deletions
diff --git a/jstests/sharding/merge_chunks_compound_shard_key.js b/jstests/sharding/merge_chunks_compound_shard_key.js new file mode 100644 index 00000000000..f6a2ef81ceb --- /dev/null +++ b/jstests/sharding/merge_chunks_compound_shard_key.js @@ -0,0 +1,93 @@ +// +// Tests that merging chunks via mongos works/doesn't work with different chunk configurations +// with a compound shard key. +// + +(function() { +'use strict'; + +var getShardVersion = function() { + var res = st.shard0.adminCommand({ getShardVersion: coll + "" }) + assert.commandWorked(res); + var version = res.global; + assert(version); + return version; +} + +// Merge two neighboring chunks and check post conditions. +var checkMergeWorked = function(lowerBound, upperBound) { + var oldVersion = getShardVersion(); + var numChunksBefore = chunks.find().itcount(); + + assert.commandWorked(admin.runCommand({ mergeChunks: coll + "", + bounds: [lowerBound, upperBound] })); + + assert.eq(numChunksBefore - 1, chunks.find().itcount()); + assert.eq(1, chunks.find({ min: lowerBound, max: upperBound }).itcount()); + + var newVersion = getShardVersion(); + assert.eq(newVersion.t, oldVersion.t); + assert.gt(newVersion.i, oldVersion.i); +} + +var st = new ShardingTest({ shards: 2, mongos: 1 }); + +var mongos = st.s; +var admin = mongos.getDB( "admin" ); +var shards = mongos.getCollection( "config.shards" ).find().toArray(); +var chunks = mongos.getCollection( "config.chunks" ); +var coll = mongos.getCollection( "foo.bar" ); + +jsTest.log("Create a sharded collection with a compound shard key."); +assert.commandWorked(admin.runCommand({ enableSharding: coll.getDB() + "" })); +printjson( admin.runCommand({ movePrimary: coll.getDB() + "", to: st.shard0.shardName }) ); +assert.commandWorked(admin.runCommand({ shardCollection: coll + "", key: { x: 1, y: 1 } })); + +// Chunks after splits: +// (MinKey, { x: 0, y: 1 }) +// ({ x: 0, y: 1 }, { x: 1, y: 0 }) +// ({ x: 1, y: 0 }, { x: 2, y: 0 }) +// ({ x: 2, y: 0 }, { x: 2, y: 1 }) +// ({ x: 2, y: 1 }, MaxKey) +jsTest.log("Create chunks."); +assert.commandWorked(admin.runCommand({ split: coll + "", middle: { x: 0, y: 1 } })); +assert.commandWorked(admin.runCommand({ split: coll + "", middle: { x: 1, y: 0 } })); +assert.commandWorked(admin.runCommand({ split: coll + "", middle: { x: 2, y: 0 } })); +assert.commandWorked(admin.runCommand({ split: coll + "", middle: { x: 2, y: 1 } })); + +jsTest.log("Insert some data into each of the chunk ranges."); +assert.writeOK(coll.insert({ x: -1, y: 2 })); +assert.writeOK(coll.insert({ x: 0, y: 2 })); +assert.writeOK(coll.insert({ x: 1, y: 2 })); +assert.writeOK(coll.insert({ x: 2, y: 1 })); +assert.writeOK(coll.insert({ x: 2, y: 3 })); + +// Chunks after merge: +// (MinKey, { x: 0, y: 1 }) +// ({ x: 0, y: 1 }, { x: 2, y: 0 }) +// ({ x: 2, y: 0 }, { x: 2, y: 1 }) +// ({ x: 2, y: 1 }, MaxKey) +jsTest.log("Merge chunks whose upper and lower bounds are compound shard keys."); +checkMergeWorked({ x: 0, y: 1 }, { x: 2, y: 0 }); + +// Chunks after merge: +// (MinKey, { x: 2, y: 0 }) +// ({ x: 2, y: 0 }, { x: 2, y: 1 }) +// ({ x: 2, y: 1 }, MaxKey) +jsTest.log("Merge chunks whose upper bound contains a compound shard key, lower bound is MinKey"); +checkMergeWorked({ x: MinKey, y: MinKey }, { x: 2, y: 0 }); + +// Chunks after merge: +// (MinKey, { x: 2, y: 0 }) +// ({ x: 2, y: 0 }, MaxKey) +jsTest.log("Merge chunks whose lower bound contains a compound shard key, upper bound is MaxKey"); +checkMergeWorked({ x: 2, y: 0 }, { x: MaxKey, y: MaxKey }); + +// Chunks after merge: +// (MinKey, MaxKey) +jsTest.log("Merge chunks whos bounds are MinKey/MaxKey, but which have a compound shard key"); +checkMergeWorked({ x: MinKey, y: MinKey }, { x: MaxKey, y: MaxKey }); + +st.stop(); + +})(); diff --git a/src/mongo/db/s/collection_metadata_test.cpp b/src/mongo/db/s/collection_metadata_test.cpp index 7a8c59eafbf..bb3ab6b2ba9 100644 --- a/src/mongo/db/s/collection_metadata_test.cpp +++ b/src/mongo/db/s/collection_metadata_test.cpp @@ -27,9 +27,14 @@ */ #include "mongo/base/status.h" +#include "mongo/client/remote_command_targeter_mock.h" +#include "mongo/client/remote_command_targeter_factory_mock.h" #include "mongo/db/commands.h" #include "mongo/db/s/collection_metadata.h" -#include "mongo/db/s/metadata_loader_fixture.h" +#include "mongo/db/s/metadata_loader.h" +#include "mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.h" +#include "mongo/s/catalog/type_chunk.h" +#include "mongo/s/catalog/type_collection.h" #include "mongo/s/chunk_version.h" #include "mongo/s/write_ops/batched_command_response.h" @@ -42,17 +47,21 @@ using std::vector; using executor::RemoteCommandResponse; -class NoChunkFixture : public MetadataLoaderFixture { +class NoChunkFixture : public CatalogManagerReplSetTestFixture { protected: void setUp() { - MetadataLoaderFixture::setUp(); + CatalogManagerReplSetTestFixture::setUp(); + getMessagingPort()->setRemote(HostAndPort("FakeRemoteClient:34567")); + configTargeter()->setFindHostReturnValue(configHost); + + OID epoch = OID::gen(); CollectionType collType; collType.setNs(NamespaceString{"test.foo"}); collType.setKeyPattern(BSON("a" << 1)); collType.setUnique(false); collType.setUpdatedAt(Date_t::fromMillisSinceEpoch(1)); - collType.setEpoch(_epoch); + collType.setEpoch(epoch); ASSERT_OK(collType.validate()); // Need a chunk on another shard, otherwise the chunks are invalid in general and we @@ -62,7 +71,7 @@ protected: chunkType.setShard("shard0001"); chunkType.setMin(BSON("a" << MINKEY)); chunkType.setMax(BSON("a" << MAXKEY)); - chunkType.setVersion(ChunkVersion(1, 0, _epoch)); + chunkType.setVersion(ChunkVersion(1, 0, epoch)); chunkType.setName(OID::gen().toString()); ASSERT_OK(chunkType.validate()); std::vector<BSONObj> chunksToSend{chunkType.toBSON()}; @@ -91,6 +100,7 @@ protected: private: CollectionMetadata _metadata; + const HostAndPort configHost{HostAndPort(CONFIG_HOST_PORT)}; }; TEST_F(NoChunkFixture, BasicBelongsToMe) { @@ -426,26 +436,30 @@ TEST_F(NoChunkFixture, PendingOrphanedDataRanges) { * Fixture with single chunk containing: * [10->20) */ -class SingleChunkFixture : public MetadataLoaderFixture { +class SingleChunkFixture : public CatalogManagerReplSetTestFixture { protected: void setUp() { - MetadataLoaderFixture::setUp(); + CatalogManagerReplSetTestFixture::setUp(); + getMessagingPort()->setRemote(HostAndPort("FakeRemoteClient:34567")); + configTargeter()->setFindHostReturnValue(configHost); + + OID epoch = OID::gen(); - ChunkVersion chunkVersion = ChunkVersion(1, 0, _epoch); + ChunkVersion chunkVersion = ChunkVersion(1, 0, epoch); CollectionType collType; collType.setNs(NamespaceString{"test.foo"}); collType.setKeyPattern(BSON("a" << 1)); collType.setUnique(false); collType.setUpdatedAt(Date_t::fromMillisSinceEpoch(1)); - collType.setEpoch(_epoch); + collType.setEpoch(epoch); BSONObj fooSingle = BSON( ChunkType::name("test.foo-a_10") << ChunkType::ns("test.foo") << ChunkType::min(BSON("a" << 10)) << ChunkType::max(BSON("a" << 20)) << ChunkType::DEPRECATED_lastmod(Date_t::fromMillisSinceEpoch(chunkVersion.toLong())) - << ChunkType::DEPRECATED_epoch(_epoch) << ChunkType::shard("shard0000")); + << ChunkType::DEPRECATED_epoch(epoch) << ChunkType::shard("shard0000")); std::vector<BSONObj> chunksToSend{fooSingle}; auto future = launchAsync([this] { @@ -471,6 +485,7 @@ protected: private: CollectionMetadata _metadata; + const HostAndPort configHost{HostAndPort(CONFIG_HOST_PORT)}; }; TEST_F(SingleChunkFixture, BasicBelongsToMe) { @@ -737,26 +752,30 @@ TEST_F(SingleChunkFixture, ChunkOrphanedDataRanges) { * Fixture with single chunk containing: * [(min, min)->(max, max)) */ -class SingleChunkMinMaxCompoundKeyFixture : public MetadataLoaderFixture { +class SingleChunkMinMaxCompoundKeyFixture : public CatalogManagerReplSetTestFixture { protected: void setUp() { - MetadataLoaderFixture::setUp(); + CatalogManagerReplSetTestFixture::setUp(); + getMessagingPort()->setRemote(HostAndPort("FakeRemoteClient:34567")); + configTargeter()->setFindHostReturnValue(configHost); + + OID epoch = OID::gen(); - ChunkVersion chunkVersion = ChunkVersion(1, 0, _epoch); + ChunkVersion chunkVersion = ChunkVersion(1, 0, epoch); CollectionType collType; collType.setNs(NamespaceString{"test.foo"}); collType.setKeyPattern(BSON("a" << 1)); collType.setUnique(false); collType.setUpdatedAt(Date_t::fromMillisSinceEpoch(1)); - collType.setEpoch(_epoch); + collType.setEpoch(epoch); BSONObj fooSingle = BSON( ChunkType::name("test.foo-a_MinKey") << ChunkType::ns("test.foo") << ChunkType::min(BSON("a" << MINKEY << "b" << MINKEY)) << ChunkType::max(BSON("a" << MAXKEY << "b" << MAXKEY)) << ChunkType::DEPRECATED_lastmod(Date_t::fromMillisSinceEpoch(chunkVersion.toLong())) - << ChunkType::DEPRECATED_epoch(_epoch) << ChunkType::shard("shard0000")); + << ChunkType::DEPRECATED_epoch(epoch) << ChunkType::shard("shard0000")); std::vector<BSONObj> chunksToSend{fooSingle}; auto future = launchAsync([this] { @@ -782,6 +801,7 @@ protected: private: CollectionMetadata _metadata; + const HostAndPort configHost{HostAndPort(CONFIG_HOST_PORT)}; }; // Note: no tests for single key belongsToMe because they are not allowed @@ -798,19 +818,23 @@ TEST_F(SingleChunkMinMaxCompoundKeyFixture, CompoudKeyBelongsToMe) { * Fixture with chunks: * [(10, 0)->(20, 0)), [(30, 0)->(40, 0)) */ -class TwoChunksWithGapCompoundKeyFixture : public MetadataLoaderFixture { +class TwoChunksWithGapCompoundKeyFixture : public CatalogManagerReplSetTestFixture { protected: void setUp() { - MetadataLoaderFixture::setUp(); + CatalogManagerReplSetTestFixture::setUp(); + getMessagingPort()->setRemote(HostAndPort("FakeRemoteClient:34567")); + configTargeter()->setFindHostReturnValue(configHost); - ChunkVersion chunkVersion = ChunkVersion(1, 0, _epoch); + OID epoch = OID::gen(); + + ChunkVersion chunkVersion = ChunkVersion(1, 0, epoch); CollectionType collType; collType.setNs(NamespaceString{"test.foo"}); collType.setKeyPattern(BSON("a" << 1)); collType.setUnique(false); collType.setUpdatedAt(Date_t::fromMillisSinceEpoch(1)); - collType.setEpoch(_epoch); + collType.setEpoch(epoch); std::vector<BSONObj> chunksToSend; chunksToSend.push_back(BSON( @@ -818,13 +842,13 @@ protected: << ChunkType::ns("test.foo") << ChunkType::min(BSON("a" << 10 << "b" << 0)) << ChunkType::max(BSON("a" << 20 << "b" << 0)) << ChunkType::DEPRECATED_lastmod(Date_t::fromMillisSinceEpoch(chunkVersion.toLong())) - << ChunkType::DEPRECATED_epoch(_epoch) << ChunkType::shard("shard0000"))); + << ChunkType::DEPRECATED_epoch(epoch) << ChunkType::shard("shard0000"))); chunksToSend.push_back(BSON( ChunkType::name("test.foo-a_10") << ChunkType::ns("test.foo") << ChunkType::min(BSON("a" << 30 << "b" << 0)) << ChunkType::max(BSON("a" << 40 << "b" << 0)) << ChunkType::DEPRECATED_lastmod(Date_t::fromMillisSinceEpoch(chunkVersion.toLong())) - << ChunkType::DEPRECATED_epoch(_epoch) << ChunkType::shard("shard0000"))); + << ChunkType::DEPRECATED_epoch(epoch) << ChunkType::shard("shard0000"))); auto future = launchAsync([this] { MetadataLoader loader; @@ -849,6 +873,7 @@ protected: private: CollectionMetadata _metadata; + const HostAndPort configHost{HostAndPort(CONFIG_HOST_PORT)}; }; TEST_F(TwoChunksWithGapCompoundKeyFixture, ClonePlusBasic) { @@ -1041,21 +1066,25 @@ TEST_F(TwoChunksWithGapCompoundKeyFixture, ChunkGapAndPendingOrphanedDataRanges) * Fixture with chunk containing: * [min->10) , [10->20) , <gap> , [30->max) */ -class ThreeChunkWithRangeGapFixture : public MetadataLoaderFixture { +class ThreeChunkWithRangeGapFixture : public CatalogManagerReplSetTestFixture { protected: void setUp() { - MetadataLoaderFixture::setUp(); + CatalogManagerReplSetTestFixture::setUp(); + getMessagingPort()->setRemote(HostAndPort("FakeRemoteClient:34567")); + configTargeter()->setFindHostReturnValue(configHost); + + OID epoch = OID::gen(); CollectionType collType; collType.setNs(NamespaceString{"x.y"}); collType.setKeyPattern(BSON("a" << 1)); collType.setUnique(false); collType.setUpdatedAt(Date_t::fromMillisSinceEpoch(1)); - collType.setEpoch(_epoch); + collType.setEpoch(epoch); std::vector<BSONObj> chunksToSend; { - ChunkVersion version(1, 1, _epoch); + ChunkVersion version(1, 1, epoch); chunksToSend.push_back(BSON( ChunkType::name("x.y-a_MinKey") << ChunkType::ns("x.y") << ChunkType::min(BSON("a" << MINKEY)) @@ -1065,7 +1094,7 @@ protected: } { - ChunkVersion version(1, 3, _epoch); + ChunkVersion version(1, 3, epoch); chunksToSend.push_back(BSON( ChunkType::name("x.y-a_10") << ChunkType::ns("x.y") << ChunkType::min(BSON("a" << 10)) @@ -1075,7 +1104,7 @@ protected: } { - ChunkVersion version(1, 2, _epoch); + ChunkVersion version(1, 2, epoch); chunksToSend.push_back(BSON( ChunkType::name("x.y-a_30") << ChunkType::ns("x.y") << ChunkType::min(BSON("a" << 30)) @@ -1107,6 +1136,7 @@ protected: private: CollectionMetadata _metadata; + const HostAndPort configHost{HostAndPort(CONFIG_HOST_PORT)}; }; TEST_F(ThreeChunkWithRangeGapFixture, ShardOwnsDoc) { diff --git a/src/mongo/db/s/metadata_loader_fixture.h b/src/mongo/db/s/metadata_loader_fixture.h index 8d9ba9bc948..1f12caa90fe 100644 --- a/src/mongo/db/s/metadata_loader_fixture.h +++ b/src/mongo/db/s/metadata_loader_fixture.h @@ -28,8 +28,6 @@ #pragma once -#include <vector> - #include "mongo/base/owned_pointer_vector.h" #include "mongo/db/s/metadata_loader.h" #include "mongo/platform/basic.h" @@ -47,13 +45,8 @@ public: ~MetadataLoaderFixture(); protected: - static const std::string CONFIG_HOST_PORT; - void setUp() override; - void expectFindOnConfigSendErrorCode(ErrorCodes::Error code); - void expectFindOnConfigSendBSONObjVector(std::vector<BSONObj> obj); - void expectFindOnConfigSendCollectionDefault(); void expectFindOnConfigSendChunksDefault(); diff --git a/src/mongo/db/s/metadata_loader_test.cpp b/src/mongo/db/s/metadata_loader_test.cpp index d766f613c9a..0d5b93cc7fc 100644 --- a/src/mongo/db/s/metadata_loader_test.cpp +++ b/src/mongo/db/s/metadata_loader_test.cpp @@ -42,8 +42,6 @@ using std::vector; using executor::RemoteCommandRequest; -const string MetadataLoaderFixture::CONFIG_HOST_PORT{"$dummy_config:27017"}; - MetadataLoaderFixture::MetadataLoaderFixture() = default; MetadataLoaderFixture::~MetadataLoaderFixture() = default; @@ -56,24 +54,6 @@ void MetadataLoaderFixture::setUp() { _loader.reset(new MetadataLoader); } -void MetadataLoaderFixture::expectFindOnConfigSendErrorCode(ErrorCodes::Error code) { - onCommand([&, code](const RemoteCommandRequest& request) { - ASSERT_EQ(request.target, HostAndPort(CONFIG_HOST_PORT)); - ASSERT_EQ(request.dbname, "config"); - BSONObjBuilder responseBuilder; - Command::appendCommandStatus(responseBuilder, Status(code, "")); - return responseBuilder.obj(); - }); -} - -void MetadataLoaderFixture::expectFindOnConfigSendBSONObjVector(std::vector<BSONObj> obj) { - onFindCommand([&, obj](const RemoteCommandRequest& request) { - ASSERT_EQ(request.target, HostAndPort(CONFIG_HOST_PORT)); - ASSERT_EQ(request.dbname, "config"); - return obj; - }); -} - void MetadataLoaderFixture::expectFindOnConfigSendCollectionDefault() { CollectionType collType; collType.setNs(NamespaceString{"test.foo"}); diff --git a/src/mongo/dbtests/SConscript b/src/mongo/dbtests/SConscript index f239d5b8477..a7dd427ed0e 100644 --- a/src/mongo/dbtests/SConscript +++ b/src/mongo/dbtests/SConscript @@ -60,7 +60,6 @@ dbtest = env.Program( 'chunktests.cpp', 'clienttests.cpp', 'commandtests.cpp', - 'config_server_fixture.cpp', 'counttests.cpp', 'dbhelper_tests.cpp', 'dbtests.cpp', diff --git a/src/mongo/dbtests/chunk_manager_tests.cpp b/src/mongo/dbtests/chunk_manager_tests.cpp deleted file mode 100644 index 1d87e53d59b..00000000000 --- a/src/mongo/dbtests/chunk_manager_tests.cpp +++ /dev/null @@ -1,208 +0,0 @@ -/** - * Copyright (C) 2009 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kDefault - -#include "mongo/platform/basic.h" - -#include "mongo/db/operation_context_impl.h" -#include "mongo/dbtests/config_server_fixture.h" -#include "mongo/dbtests/dbtests.h" -#include "mongo/s/catalog/type_chunk.h" -#include "mongo/s/catalog/type_collection.h" -#include "mongo/s/catalog/type_shard.h" -#include "mongo/s/chunk_manager.h" - -namespace mongo { - -using std::unique_ptr; -using std::set; -using std::string; -using std::vector; - -namespace { - -static int rand(int max = -1) { - static unsigned seed = 1337; - -#if !defined(_WIN32) - int r = rand_r(&seed); -#else - int r = ::rand(); // seed not used in this case -#endif - - // Modding is bad, but don't really care in this case - return max > 0 ? r % max : r; -} - -/** - * Sets up a basic environment for loading chunks to/from the direct database connection. Redirects - * connections to the direct database for the duration of the test. - */ -class ChunkManagerFixture : public ConfigServerFixture { -public: - void setUp() override { - ConfigServerFixture::setUp(); - - _client.dropDatabase(nsGetDB(_collName)); - _client.insert(_collName, - BSON("hello" - << "world")); - _client.dropCollection(_collName); - - // Add dummy shard to config DB - _client.insert(ShardType::ConfigNS, - BSON(ShardType::name() - << _shardId << ShardType::host() - << ConnectionString(HostAndPort("$hostFooBar:27017")).toString())); - } - -protected: - static const ShardId _shardId; - static const string _collName; - - static const int numSplitPoints = 100; - - void genRandomSplitPoints(vector<int>* splitPoints) { - for (int i = 0; i < numSplitPoints; i++) { - splitPoints->push_back(rand(numSplitPoints * 10)); - } - } - - void genRandomSplitKeys(const string& keyName, vector<BSONObj>* splitKeys) { - vector<int> splitPoints; - genRandomSplitPoints(&splitPoints); - - for (vector<int>::iterator it = splitPoints.begin(); it != splitPoints.end(); ++it) { - splitKeys->push_back(BSON(keyName << *it)); - } - } - - // Uses a chunk manager to create chunks - void createChunks(const string& keyName) { - vector<BSONObj> splitKeys; - genRandomSplitKeys(keyName, &splitKeys); - - ShardKeyPattern shardKeyPattern(BSON(keyName << 1)); - ChunkManager manager(_collName, shardKeyPattern, false); - - uassertStatusOK(manager.createFirstChunks(&_txn, _shardId, &splitKeys, NULL)); - } -}; - -const ShardId ChunkManagerFixture::_shardId{"shard0000"}; -const string ChunkManagerFixture::_collName{"foo.bar"}; - -// Rename the fixture so that our tests have a useful name in the executable -typedef ChunkManagerFixture ChunkManagerTests; - -/** - * Tests that chunks are loaded correctly from the db with no a-priori info and also that they can - * be reloaded on top of an old chunk manager with changes. - */ -TEST_F(ChunkManagerTests, Basic) { - string keyName = "_id"; - createChunks(keyName); - int numChunks = - static_cast<int>(_client.count(ChunkType::ConfigNS, BSON(ChunkType::ns(_collName)))); - - BSONObj firstChunk = _client.findOne(ChunkType::ConfigNS, BSONObj()).getOwned(); - - ChunkVersion version = ChunkVersion::fromBSON(firstChunk, ChunkType::DEPRECATED_lastmod()); - - // Make manager load existing chunks - CollectionType collType; - collType.setNs(NamespaceString{_collName}); - collType.setEpoch(version.epoch()); - collType.setUpdatedAt(jsTime()); - collType.setKeyPattern(BSON("_id" << 1)); - collType.setUnique(false); - collType.setDropped(false); - - ChunkManager manager(collType); - manager.loadExistingRanges(&_txn, nullptr); - - ASSERT_EQ(version.epoch(), manager.getVersion().epoch()); - ASSERT_EQ(numChunks - 1, manager.getVersion().minorVersion()); - ASSERT_EQ(numChunks, static_cast<int>(manager.getChunkMap().size())); - - // Modify chunks collection - BSONObjBuilder b; - ChunkVersion laterVersion = ChunkVersion(2, 1, version.epoch()); - laterVersion.addToBSON(b, ChunkType::DEPRECATED_lastmod()); - - _client.update(ChunkType::ConfigNS, BSONObj(), BSON("$set" << b.obj())); - - // Make new manager load chunk diff - ChunkManager newManager(manager.getns(), manager.getShardKeyPattern(), manager.isUnique()); - newManager.loadExistingRanges(&_txn, &manager); - - ASSERT_EQ(numChunks, static_cast<int>(manager.getChunkMap().size())); - ASSERT_EQ(laterVersion.toString(), newManager.getVersion().toString()); -} - -/** - * Tests creating a new chunk manager with random split points. Creating chunks on multiple shards - * is not tested here since there are unresolved race conditions there and probably should be - * avoided if at all possible. - */ -TEST_F(ChunkManagerTests, FullTest) { - string keyName = "_id"; - createChunks(keyName); - - unique_ptr<DBClientCursor> cursor = - _client.query(ChunkType::ConfigNS, QUERY(ChunkType::ns(_collName))); - - set<int> minorVersions; - OID epoch; - - // Check that all chunks were created with version 1|x with consistent epoch and unique - // minor versions - while (cursor->more()) { - BSONObj chunk = cursor->next(); - - ChunkVersion version = ChunkVersion::fromBSON(chunk, ChunkType::DEPRECATED_lastmod()); - - ASSERT(version.majorVersion() == 1); - ASSERT(version.epoch().isSet()); - - if (!epoch.isSet()) { - epoch = version.epoch(); - } - - ASSERT(version.epoch() == epoch); - - ASSERT(minorVersions.find(version.minorVersion()) == minorVersions.end()); - minorVersions.insert(version.minorVersion()); - - ASSERT(chunk[ChunkType::shard()].String() == _shardId); - } -} - -} // namespace -} // namespace mongo diff --git a/src/mongo/dbtests/config_server_fixture.cpp b/src/mongo/dbtests/config_server_fixture.cpp deleted file mode 100644 index fe2fb4d4044..00000000000 --- a/src/mongo/dbtests/config_server_fixture.cpp +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Copyright (C) 2012 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kDefault - -#include "mongo/platform/basic.h" - -#include "mongo/dbtests/config_server_fixture.h" - -#include <list> - -#include "mongo/dbtests/dbtests.h" -#include "mongo/db/service_context.h" -#include "mongo/db/s/sharding_state.h" -#include "mongo/s/catalog/type_chunk.h" -#include "mongo/s/catalog/type_config_version.h" -#include "mongo/s/client/shard_connection.h" -#include "mongo/stdx/memory.h" -#include "mongo/util/log.h" - -namespace mongo { - -using std::unique_ptr; -using std::list; -using std::string; - -ConfigServerFixture::ConfigServerFixture() : _client(&_txn), _connectHook(NULL) {} - -string ConfigServerFixture::shardName() { - return "TestShardName"; -} - -void ConfigServerFixture::setUp() { - shardConnectionPool.clear(); - DBException::traceExceptions = true; - - // Make all connections redirect to the direct client - _connectHook = new CustomConnectHook(&_txn); - ConnectionString::setConnectionHook(_connectHook); - - // Create the default config database before querying, necessary for direct connections - clearServer(); - _client.insert("config.test", - BSON("hello" - << "world")); - _client.dropCollection("config.test"); - - // Create an index over the chunks, to allow correct diffing - ASSERT_OK( - dbtests::createIndex(&_txn, - ChunkType::ConfigNS, - BSON(ChunkType::ns() << 1 << ChunkType::DEPRECATED_lastmod() << 1))); - - const ConnectionString connStr(uassertStatusOK(ConnectionString::parse("$dummy:10000"))); - - ShardingState::get(&_txn)->initialize(&_txn, connStr.toString()); - ShardingState::get(&_txn)->setShardName(shardName()); -} - -void ConfigServerFixture::clearServer() { - _client.dropDatabase("config"); -} - -void ConfigServerFixture::clearVersion() { - _client.dropCollection(VersionType::ConfigNS); -} - -void ConfigServerFixture::dumpServer() { - log() << "Dumping virtual config server to log..."; - - list<string> collectionNames(_client.getCollectionNames("config")); - - for (list<string>::iterator it = collectionNames.begin(); it != collectionNames.end(); ++it) { - const string& collection = *it; - - unique_ptr<DBClientCursor> cursor(_client.query(collection, BSONObj()).release()); - ASSERT(cursor.get() != NULL); - - log() << "Dumping collection " << collection; - - while (cursor->more()) { - BSONObj obj = cursor->nextSafe(); - log() << obj.toString(); - } - } -} - -void ConfigServerFixture::tearDown() { - ShardingState::get(&_txn)->clearCollectionMetadata(); - clearServer(); - - // Make all connections redirect to the direct client - ConnectionString::setConnectionHook(NULL); - delete _connectHook; - _connectHook = NULL; - - DBException::traceExceptions = false; -} - -} // namespace mongo diff --git a/src/mongo/dbtests/config_server_fixture.h b/src/mongo/dbtests/config_server_fixture.h deleted file mode 100644 index 62cbedae4e7..00000000000 --- a/src/mongo/dbtests/config_server_fixture.h +++ /dev/null @@ -1,123 +0,0 @@ -/** - * Copyright (C) 2012 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#pragma once - -#include "mongo/client/dbclientinterface.h" -#include "mongo/db/dbdirectclient.h" -#include "mongo/db/operation_context_impl.h" -#include "mongo/db/wire_version.h" -#include "mongo/unittest/unittest.h" -#include "mongo/util/assert_util.h" - -namespace mongo { - -class CustomDirectClient : public DBDirectClient { -public: - CustomDirectClient(OperationContext* txn) : DBDirectClient(txn) {} - - virtual ConnectionString::ConnectionType type() const { - return ConnectionString::CUSTOM; - } - - virtual bool recv(Message& m) { - // This is tailored to act as a dummy response for write commands. - - BufBuilder bb; - bb.skip(sizeof(QueryResult::Value)); - - BSONObj cmdResult(BSON("ok" << 1)); - - bb.appendBuf(cmdResult.objdata(), cmdResult.objsize()); - - QueryResult::View qr = bb.buf(); - bb.decouple(); - qr.setResultFlagsToOk(); - qr.msgdata().setLen(bb.len()); - qr.msgdata().setOperation(opReply); - qr.setCursorId(0); - qr.setStartingFrom(0); - qr.setNReturned(1); - m.setData(qr.view2ptr(), true); - - return true; - } -}; - -class CustomConnectHook : public ConnectionString::ConnectionHook { -public: - CustomConnectHook(OperationContext* txn) : _txn(txn) {} - - virtual DBClientBase* connect(const ConnectionString& connStr, - std::string& errmsg, - double socketTimeout) { - // Note - must be new, since it gets owned elsewhere - return new CustomDirectClient(_txn); - } - -private: - OperationContext* const _txn; -}; - -/** - * Fixture for testing complicated operations against a "virtual" config server. - * - * Use this if your test requires complex commands and writing to many collections, - * otherwise a unit test in the mock framework may be a better option. - */ -class ConfigServerFixture : public mongo::unittest::Test { -public: - ConfigServerFixture(); - - /** - * Returns a uniform shard name to use throughout the tests. - */ - static std::string shardName(); - - /** - * Clears all data on the server - */ - void clearServer(); - - void clearVersion(); - - /** - * Dumps the contents of the config server to the log. - */ - void dumpServer(); - -protected: - OperationContextImpl _txn; - CustomDirectClient _client; - CustomConnectHook* _connectHook; - - virtual void setUp(); - virtual void tearDown(); -}; - -} // namespace mongo diff --git a/src/mongo/dbtests/config_upgrade_tests.cpp b/src/mongo/dbtests/config_upgrade_tests.cpp deleted file mode 100644 index 88db3071a1a..00000000000 --- a/src/mongo/dbtests/config_upgrade_tests.cpp +++ /dev/null @@ -1,249 +0,0 @@ -/** - * Copyright (C) 2012 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/dbtests/config_server_fixture.h" -#include "mongo/s/catalog/config_server_version.h" -#include "mongo/s/catalog/legacy/cluster_client_internal.h" -#include "mongo/s/catalog/legacy/config_upgrade.h" -#include "mongo/s/catalog/type_chunk.h" -#include "mongo/s/catalog/type_config_version.h" -#include "mongo/s/catalog/type_mongos.h" -#include "mongo/s/catalog/type_settings.h" -#include "mongo/s/catalog/type_shard.h" -#include "mongo/s/chunk_version.h" -#include "mongo/s/grid.h" -#include "mongo/unittest/unittest.h" -#include "mongo/util/version.h" - -namespace mongo { - -using std::string; - -namespace { - -/** - * Specialization of the config server fixture with helpers for the tests below. - */ -class ConfigUpgradeFixture : public ConfigServerFixture { -public: - void stopBalancer() { - // Note: The balancer key is needed in the update portion, for some reason related to - // DBDirectClient - DBDirectClient client(&_txn); - client.update(SettingsType::ConfigNS, - BSON(SettingsType::key(SettingsType::BalancerDocKey)), - BSON(SettingsType::key(SettingsType::BalancerDocKey) - << SettingsType::balancerStopped(true)), - true, - false); - } - - /** - * Stores a legacy { version : X } config server entry - */ - void storeLegacyConfigVersion(int version) { - if (version == 0) - return; - - DBDirectClient client(&_txn); - - if (version == 1) { - ShardType shard; - shard.setName("test"); - shard.setHost("$dummy:10000"); - client.insert(ShardType::ConfigNS, shard.toBSON()); - return; - } - - client.insert(VersionType::ConfigNS, BSON("_id" << 1 << "version" << version)); - } - - VersionType loadLegacyConfigVersion() { - DBDirectClient client(&_txn); - return unittest::assertGet( - VersionType::fromBSON(client.findOne(VersionType::ConfigNS, BSONObj()))); - } - - /** - * Stores a newer { version, minVersion, currentVersion, clusterId } config server entry - */ - void storeConfigVersion(const VersionType& versionInfo) { - DBDirectClient client(&_txn); - client.insert(VersionType::ConfigNS, versionInfo.toBSON()); - } - - /** - * Stores a newer { version, minVersion, currentVersion, clusterId } config server entry. - * - * @return clusterId - */ - OID storeConfigVersion(int configVersion) { - if (configVersion < CURRENT_CONFIG_VERSION) { - storeLegacyConfigVersion(configVersion); - return OID(); - } - - VersionType version; - version.setMinCompatibleVersion(configVersion); - version.setCurrentVersion(configVersion); - - OID clusterId = OID::gen(); - - version.setClusterId(clusterId); - - storeConfigVersion(version); - return clusterId; - } - - /** - * Stores sample shard and ping information at the current version. - */ - void storeShardsAndPings(int numShards, int numPings) { - DBDirectClient client(&_txn); - - for (int i = 0; i < numShards; i++) { - ShardType shard; - shard.setName(OID::gen().toString()); - shard.setHost((string)(str::stream() << "$dummyShard:" << (i + 1) << "0000")); - - client.insert(ShardType::ConfigNS, shard.toBSON()); - } - - time_t started = time(0); - for (int i = 0; i < numPings; i++) { - MongosType ping; - ping.setName((string)(str::stream() << "$dummyMongos:" << (i + 1) << "0000")); - ping.setPing(jsTime()); - ping.setUptime(time(0) - started); - ping.setWaiting(false); - ping.setMongoVersion(versionString); - ping.setConfigVersion(CURRENT_CONFIG_VERSION); - - if (i % 2 == 0) { - ping.setPing(ping.getPing() - Minutes(10)); - } - - client.insert(MongosType::ConfigNS, ping.toBSON()); - } - } -}; - -// -// Tests for upgrading the config server between versions. -// -// In general these tests do pretty minimal validation of the config server data itself, but -// do ensure that the upgrade mechanism is working correctly w.r.t the config.version -// collection. -// - -// Rename the fixture so that our tests have a useful name in the executable -typedef ConfigUpgradeFixture ConfigUpgradeTests; - -TEST_F(ConfigUpgradeTests, EmptyVersion) { - // - // Tests detection of empty config version - // - - // Zero version (no version doc) - VersionType oldVersion; - Status status = getConfigVersion(grid.catalogManager(&_txn), &oldVersion); - ASSERT(status.isOK()); - - ASSERT_EQUALS(oldVersion.getMinCompatibleVersion(), 0); - ASSERT_EQUALS(oldVersion.getCurrentVersion(), 0); -} - -TEST_F(ConfigUpgradeTests, ClusterIDVersion) { - // - // Tests detection of newer config versions - // - - VersionType newVersion; - newVersion.setMinCompatibleVersion(MIN_COMPATIBLE_CONFIG_VERSION); - newVersion.setCurrentVersion(CURRENT_CONFIG_VERSION); - storeConfigVersion(newVersion); - - newVersion.clear(); - - // Current Version w/o clusterId (invalid!) - Status status = getConfigVersion(grid.catalogManager(&_txn), &newVersion); - ASSERT(!status.isOK()); - - newVersion.clear(); - - OID clusterId = OID::gen(); - newVersion.setClusterId(clusterId); - newVersion.setMinCompatibleVersion(MIN_COMPATIBLE_CONFIG_VERSION); - newVersion.setCurrentVersion(CURRENT_CONFIG_VERSION); - - clearVersion(); - storeConfigVersion(newVersion); - - newVersion.clear(); - - // Current version w/ clusterId (valid!) - status = getConfigVersion(grid.catalogManager(&_txn), &newVersion); - ASSERT(status.isOK()); - - ASSERT_EQUALS(newVersion.getMinCompatibleVersion(), MIN_COMPATIBLE_CONFIG_VERSION); - ASSERT_EQUALS(newVersion.getCurrentVersion(), CURRENT_CONFIG_VERSION); - ASSERT_EQUALS(newVersion.getClusterId(), clusterId); -} - -TEST_F(ConfigUpgradeTests, InitialUpgrade) { - // - // Tests initializing the config server to the initial version - // - - string errMsg; - ASSERT_OK(grid.catalogManager(&_txn)->initConfigVersion(&_txn)); - - VersionType version; - ASSERT_OK(getConfigVersion(grid.catalogManager(&_txn), &version)); - - ASSERT_EQUALS(MIN_COMPATIBLE_CONFIG_VERSION, version.getMinCompatibleVersion()); - ASSERT_EQUALS(CURRENT_CONFIG_VERSION, version.getCurrentVersion()); - ASSERT_TRUE(version.getClusterId().isSet()); -} - -TEST_F(ConfigUpgradeTests, BadVersionUpgrade) { - // - // Tests that we can't upgrade from a config version we don't have an upgrade path for - // - - stopBalancer(); - - storeLegacyConfigVersion(1); - - // Default version (not upgradeable) - ASSERT_EQ(ErrorCodes::IncompatibleShardingMetadata, - grid.catalogManager(&_txn)->initConfigVersion(&_txn)); -} - -} // namespace -} // namespace mongo diff --git a/src/mongo/dbtests/framework.cpp b/src/mongo/dbtests/framework.cpp index c9cc9d2028c..6ba11c289e0 100644 --- a/src/mongo/dbtests/framework.cpp +++ b/src/mongo/dbtests/framework.cpp @@ -41,7 +41,6 @@ #include "mongo/db/service_context.h" #include "mongo/db/service_context_d.h" #include "mongo/db/s/sharding_state.h" -#include "mongo/dbtests/config_server_fixture.h" #include "mongo/dbtests/dbtests.h" #include "mongo/dbtests/framework_options.h" #include "mongo/s/catalog/catalog_manager.h" diff --git a/src/mongo/dbtests/merge_chunk_tests.cpp b/src/mongo/dbtests/merge_chunk_tests.cpp deleted file mode 100644 index a2de2c89e2c..00000000000 --- a/src/mongo/dbtests/merge_chunk_tests.cpp +++ /dev/null @@ -1,338 +0,0 @@ -/** - * Copyright (C) 2013 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/platform/basic.h" - -#include "mongo/db/range_arithmetic.h" -#include "mongo/db/service_context.h" -#include "mongo/db/s/collection_metadata.h" -#include "mongo/db/s/sharding_state.h" -#include "mongo/dbtests/config_server_fixture.h" -#include "mongo/s/catalog/type_chunk.h" -#include "mongo/s/catalog/type_collection.h" -#include "mongo/s/chunk.h" -#include "mongo/s/chunk_version.h" -#include "mongo/s/d_merge.h" -#include "mongo/unittest/unittest.h" - -namespace mongo { - -using std::string; -using std::vector; - -namespace { - -/** - * Specialization of the config server fixture with helpers for the tests below. - */ -class MergeChunkFixture : public ConfigServerFixture { -public: - /** - * Stores ranges for a particular collection and shard starting from some version - */ - void storeCollectionRanges(const NamespaceString& nss, - const string& shardName, - const vector<KeyRange>& ranges, - const ChunkVersion& startVersion) { - // Get key pattern from first range - ASSERT_GREATER_THAN(ranges.size(), 0u); - - CollectionType coll; - coll.setNs(nss); - coll.setKeyPattern(ranges.begin()->keyPattern); - coll.setEpoch(startVersion.epoch()); - coll.setUpdatedAt(Date_t::fromMillisSinceEpoch(1)); - ASSERT_OK(coll.validate()); - - DBDirectClient client(&_txn); - - client.update(CollectionType::ConfigNS, - BSON(CollectionType::fullNs(coll.getNs().ns())), - coll.toBSON(), - true, - false); - - ChunkVersion nextVersion = startVersion; - for (vector<KeyRange>::const_iterator it = ranges.begin(); it != ranges.end(); ++it) { - ChunkType chunk; - // TODO: We should not rely on the serialized ns, minkey being unique in the future, - // causes problems since it links string serialization to correctness. - chunk.setName(Chunk::genID(nss.ns(), it->minKey)); - chunk.setShard(shardName); - chunk.setNS(nss.ns()); - chunk.setVersion(nextVersion); - chunk.setMin(it->minKey); - chunk.setMax(it->maxKey); - nextVersion.incMajor(); - - client.insert(ChunkType::ConfigNS, chunk.toBSON()); - } - } - - /** - * Makes sure that all the ranges here no longer exist on disk but the merged range does - */ - void assertWrittenAsMerged(const vector<KeyRange>& ranges) { - dumpServer(); - - BSONObj rangeMin; - BSONObj rangeMax; - - DBDirectClient client(&_txn); - - // Ensure written - for (vector<KeyRange>::const_iterator it = ranges.begin(); it != ranges.end(); ++it) { - Query query(BSON(ChunkType::min(it->minKey) << ChunkType::max(it->maxKey) - << ChunkType::shard(shardName()))); - ASSERT(client.findOne(ChunkType::ConfigNS, query).isEmpty()); - - if (rangeMin.isEmpty() || rangeMin.woCompare(it->minKey) > 0) { - rangeMin = it->minKey; - } - - if (rangeMax.isEmpty() || rangeMax.woCompare(it->maxKey) < 0) { - rangeMax = it->maxKey; - } - } - - Query query(BSON(ChunkType::min(rangeMin) << ChunkType::max(rangeMax) - << ChunkType::shard(shardName()))); - ASSERT(!client.findOne(ChunkType::ConfigNS, query).isEmpty()); - } -}; - -// -// Tests for upgrading the config server between versions. -// -// In general these tests do pretty minimal validation of the config server data itself, but -// do ensure that the upgrade mechanism is working correctly w.r.t the config.version -// collection. -// - -// Rename the fixture so that our tests have a useful name in the executable -typedef MergeChunkFixture MergeChunkTests; - -TEST_F(MergeChunkTests, FailedMerge) { - const NamespaceString nss("foo.bar"); - const BSONObj kp = BSON("x" << 1); - const OID epoch = OID::gen(); - vector<KeyRange> ranges; - - // Setup chunk metadata - ranges.push_back(KeyRange(nss.ns(), BSON("x" << 0), BSON("x" << 10), kp)); - ranges.push_back(KeyRange(nss.ns(), BSON("x" << 10), BSON("x" << 20), kp)); - storeCollectionRanges(nss, shardName(), ranges, ChunkVersion(1, 0, epoch)); - - // Do bad merges - string errMsg; - bool result; - - result = mergeChunks(&_txn, nss, BSON("x" << 5), BSON("x" << 20), epoch, &errMsg); - ASSERT_NOT_EQUALS(errMsg, ""); - ASSERT(!result); - - result = mergeChunks(&_txn, nss, BSON("x" << 0), BSON("x" << 15), epoch, &errMsg); - ASSERT_NOT_EQUALS(errMsg, ""); - ASSERT(!result); - - result = mergeChunks(&_txn, nss, BSON("x" << -10), BSON("x" << 20), epoch, &errMsg); - ASSERT_NOT_EQUALS(errMsg, ""); - ASSERT(!result); - - result = mergeChunks(&_txn, nss, BSON("x" << 0), BSON("x" << 30), epoch, &errMsg); - ASSERT_NOT_EQUALS(errMsg, ""); - ASSERT(!result); - - result = mergeChunks(&_txn, nss, BSON("x" << 0), BSON("x" << 10), epoch, &errMsg); - ASSERT_NOT_EQUALS(errMsg, ""); - ASSERT(!result); - - // Wrong epoch - result = mergeChunks(&_txn, nss, BSON("x" << 0), BSON("x" << 10), OID::gen(), &errMsg); - ASSERT_NOT_EQUALS(errMsg, ""); - ASSERT(!result); -} - -TEST_F(MergeChunkTests, FailedMergeHole) { - const NamespaceString nss("foo.bar"); - const BSONObj kp = BSON("x" << 1); - const OID epoch = OID::gen(); - vector<KeyRange> ranges; - - // Setup chunk metadata - ranges.push_back(KeyRange(nss.ns(), BSON("x" << 0), BSON("x" << 10), kp)); - ranges.push_back(KeyRange(nss.ns(), BSON("x" << 11), BSON("x" << 20), kp)); - storeCollectionRanges(nss, shardName(), ranges, ChunkVersion(1, 0, epoch)); - - // Do bad merge with hole - string errMsg; - bool result; - result = mergeChunks(&_txn, nss, BSON("x" << 0), BSON("x" << 20), epoch, &errMsg); - ASSERT_NOT_EQUALS(errMsg, ""); - ASSERT(!result); -} - -TEST_F(MergeChunkTests, FailedMergeMinMax) { - const NamespaceString nss("foo.bar"); - const BSONObj kp = BSON("x" << 1); - const OID epoch = OID::gen(); - vector<KeyRange> ranges; - - // Setup chunk metadata - ranges.push_back(KeyRange(nss.ns(), BSON("x" << MINKEY), BSON("x" << 0), kp)); - ranges.push_back(KeyRange(nss.ns(), BSON("x" << 0), BSON("x" << MAXKEY), kp)); - storeCollectionRanges(nss, shardName(), ranges, ChunkVersion(1, 0, epoch)); - - // Do bad merge with hole - string errMsg; - bool result; - result = mergeChunks(&_txn, nss, BSON("x" << -1), BSON("x" << MAXKEY), epoch, &errMsg); - ASSERT_NOT_EQUALS(errMsg, ""); - ASSERT(!result); - - result = mergeChunks(&_txn, nss, BSON("x" << MINKEY), BSON("x" << 1), epoch, &errMsg); - ASSERT_NOT_EQUALS(errMsg, ""); - ASSERT(!result); -} - -TEST_F(MergeChunkTests, BasicMerge) { - const NamespaceString nss("foo.bar"); - const BSONObj kp = BSON("x" << 1); - const OID epoch = OID::gen(); - vector<KeyRange> ranges; - - // Setup chunk metadata - ranges.push_back(KeyRange(nss.ns(), BSON("x" << 0), BSON("x" << 1), kp)); - ranges.push_back(KeyRange(nss.ns(), BSON("x" << 1), BSON("x" << 2), kp)); - storeCollectionRanges(nss, shardName(), ranges, ChunkVersion(1, 0, epoch)); - - // Get latest version - ChunkVersion latestVersion; - ShardingState::get(&_txn)->refreshMetadataNow(&_txn, nss.ns(), &latestVersion); - ShardingState::get(&_txn)->resetMetadata(nss.ns()); - - // Do merge - string errMsg; - bool result = mergeChunks(&_txn, nss, BSON("x" << 0), BSON("x" << 2), epoch, &errMsg); - ASSERT_EQUALS(errMsg, ""); - ASSERT(result); - - // Verify result - CollectionMetadataPtr metadata = ShardingState::get(&_txn)->getCollectionMetadata(nss.ns()); - - ChunkType chunk; - ASSERT(metadata->getNextChunk(BSON("x" << 0), &chunk)); - ASSERT(chunk.getMin().woCompare(BSON("x" << 0)) == 0); - ASSERT(chunk.getMax().woCompare(BSON("x" << 2)) == 0); - ASSERT_EQUALS(metadata->getNumChunks(), 1u); - - ASSERT_EQUALS(metadata->getShardVersion().majorVersion(), latestVersion.majorVersion()); - ASSERT_GREATER_THAN(metadata->getShardVersion().minorVersion(), latestVersion.minorVersion()); - - assertWrittenAsMerged(ranges); -} - -TEST_F(MergeChunkTests, BasicMergeMinMax) { - const NamespaceString nss("foo.bar"); - const BSONObj kp = BSON("x" << 1); - const OID epoch = OID::gen(); - vector<KeyRange> ranges; - - // Setup chunk metadata - ranges.push_back(KeyRange(nss.ns(), BSON("x" << MINKEY), BSON("x" << 0), kp)); - ranges.push_back(KeyRange(nss.ns(), BSON("x" << 0), BSON("x" << MAXKEY), kp)); - storeCollectionRanges(nss, shardName(), ranges, ChunkVersion(1, 0, epoch)); - - // Get latest version - ChunkVersion latestVersion; - ShardingState::get(&_txn)->refreshMetadataNow(&_txn, nss.ns(), &latestVersion); - ShardingState::get(&_txn)->resetMetadata(nss.ns()); - - // Do merge - string errMsg; - bool result = mergeChunks(&_txn, nss, BSON("x" << MINKEY), BSON("x" << MAXKEY), epoch, &errMsg); - ASSERT_EQUALS(errMsg, ""); - ASSERT(result); - - // Verify result - CollectionMetadataPtr metadata = ShardingState::get(&_txn)->getCollectionMetadata(nss.ns()); - - ChunkType chunk; - ASSERT(metadata->getNextChunk(BSON("x" << MINKEY), &chunk)); - ASSERT(chunk.getMin().woCompare(BSON("x" << MINKEY)) == 0); - ASSERT(chunk.getMax().woCompare(BSON("x" << MAXKEY)) == 0); - ASSERT_EQUALS(metadata->getNumChunks(), 1u); - - ASSERT_EQUALS(metadata->getShardVersion().majorVersion(), latestVersion.majorVersion()); - ASSERT_GREATER_THAN(metadata->getShardVersion().minorVersion(), latestVersion.minorVersion()); - - assertWrittenAsMerged(ranges); -} - -TEST_F(MergeChunkTests, CompoundMerge) { - const NamespaceString nss("foo.bar"); - const BSONObj kp = BSON("x" << 1 << "y" << 1); - const OID epoch = OID::gen(); - vector<KeyRange> ranges; - - // Setup chunk metadata - ranges.push_back( - KeyRange(nss.ns(), BSON("x" << 0 << "y" << 1), BSON("x" << 1 << "y" << 0), kp)); - ranges.push_back( - KeyRange(nss.ns(), BSON("x" << 1 << "y" << 0), BSON("x" << 2 << "y" << 1), kp)); - storeCollectionRanges(nss, shardName(), ranges, ChunkVersion(1, 0, epoch)); - - // Get latest version - ChunkVersion latestVersion; - ShardingState::get(&_txn)->refreshMetadataNow(&_txn, nss.ns(), &latestVersion); - ShardingState::get(&_txn)->resetMetadata(nss.ns()); - - // Do merge - string errMsg; - bool result = mergeChunks( - &_txn, nss, BSON("x" << 0 << "y" << 1), BSON("x" << 2 << "y" << 1), epoch, &errMsg); - ASSERT_EQUALS(errMsg, ""); - ASSERT(result); - - // Verify result - CollectionMetadataPtr metadata = ShardingState::get(&_txn)->getCollectionMetadata(nss.ns()); - - ChunkType chunk; - ASSERT(metadata->getNextChunk(BSON("x" << 0 << "y" << 1), &chunk)); - ASSERT(chunk.getMin().woCompare(BSON("x" << 0 << "y" << 1)) == 0); - ASSERT(chunk.getMax().woCompare(BSON("x" << 2 << "y" << 1)) == 0); - ASSERT_EQUALS(metadata->getNumChunks(), 1u); - - ASSERT_EQUALS(metadata->getShardVersion().majorVersion(), latestVersion.majorVersion()); - ASSERT_GREATER_THAN(metadata->getShardVersion().minorVersion(), latestVersion.minorVersion()); - - assertWrittenAsMerged(ranges); -} - -} // namespace -} // namespace mongo diff --git a/src/mongo/s/SConscript b/src/mongo/s/SConscript index bfd6eead67e..a99be665ed3 100644 --- a/src/mongo/s/SConscript +++ b/src/mongo/s/SConscript @@ -149,6 +149,16 @@ env.CppUnitTest( ] ) +env.CppUnitTest( + target='chunk_manager_tests', + source=[ + 'chunk_manager_tests.cpp' + ], + LIBDEPS=[ + 'coreshard', + '$BUILD_DIR/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture'] +) + # This library contains sharding functionality used by both mongod and mongos. Certain tests, # which exercise this functionality also link against it. env.Library( diff --git a/src/mongo/s/catalog/legacy/config_upgrade.cpp b/src/mongo/s/catalog/legacy/config_upgrade.cpp deleted file mode 100644 index ea09db1dc00..00000000000 --- a/src/mongo/s/catalog/legacy/config_upgrade.cpp +++ /dev/null @@ -1,323 +0,0 @@ -/** - * Copyright (C) 2012 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kSharding - -#include "mongo/platform/basic.h" - -#include "mongo/s/catalog/legacy/config_upgrade.h" - -#include "mongo/client/connpool.h" -#include "mongo/client/dbclientcursor.h" -#include "mongo/rpc/get_status_from_command_result.h" -#include "mongo/s/catalog/config_server_version.h" -#include "mongo/s/catalog/dist_lock_manager.h" -#include "mongo/s/catalog/legacy/cluster_client_internal.h" -#include "mongo/s/catalog/mongo_version_range.h" -#include "mongo/s/catalog/type_collection.h" -#include "mongo/s/catalog/type_config_version.h" -#include "mongo/s/catalog/type_database.h" -#include "mongo/s/catalog/type_settings.h" -#include "mongo/s/catalog/type_shard.h" -#include "mongo/s/client/shard_registry.h" -#include "mongo/s/grid.h" -#include "mongo/stdx/functional.h" -#include "mongo/util/assert_util.h" -#include "mongo/util/log.h" -#include "mongo/util/version.h" - -namespace mongo { - -using std::unique_ptr; -using std::make_pair; -using std::map; -using std::string; -using std::vector; -using str::stream; - -namespace { - -Status makeConfigVersionDocument(OperationContext* txn, CatalogManager* catalogManager) { - // - // Even though the initial config write is a single-document update, that single document - // is on multiple config servers and requests can interleave. The upgrade lock prevents - // this. - // - - log() << "writing initial config version at v" << CURRENT_CONFIG_VERSION; - - OID newClusterId = OID::gen(); - - VersionType versionInfo; - - // Upgrade to new version - versionInfo.setMinCompatibleVersion(MIN_COMPATIBLE_CONFIG_VERSION); - versionInfo.setCurrentVersion(CURRENT_CONFIG_VERSION); - versionInfo.setClusterId(newClusterId); - - invariantOK(versionInfo.validate()); - - // If the cluster has not previously been initialized, we need to set the version before - // using so subsequent mongoses use the config data the same way. This requires all three - // config servers online initially. - auto status = catalogManager->updateConfigDocument( - txn, VersionType::ConfigNS, BSON("_id" << 1), versionInfo.toBSON(), true); - return status.getStatus(); -} - -struct VersionRange { - VersionRange(int _minCompatibleVersion, int _currentVersion) - : minCompatibleVersion(_minCompatibleVersion), currentVersion(_currentVersion) {} - - bool operator==(const VersionRange& other) const { - return (other.minCompatibleVersion == minCompatibleVersion) && - (other.currentVersion == currentVersion); - } - - bool operator!=(const VersionRange& other) const { - return !(*this == other); - } - - int minCompatibleVersion; - int currentVersion; -}; - -enum VersionStatus { - // No way to upgrade the test version to be compatible with current version - VersionStatus_Incompatible, - - // Current version is compatible with test version - VersionStatus_Compatible, - - // Test version must be upgraded to be compatible with current version - VersionStatus_NeedUpgrade -}; - -/** - * Checks whether or not a particular cluster version is compatible with our current - * version and mongodb version. The version is compatible if it falls between the - * MIN_COMPATIBLE_CONFIG_VERSION and CURRENT_CONFIG_VERSION and is not explicitly excluded. - * - * @return a VersionStatus enum indicating compatibility - */ -VersionStatus isConfigVersionCompatible(const VersionType& versionInfo, string* whyNot) { - string dummy; - if (!whyNot) { - whyNot = &dummy; - } - - // Check if we're empty - if (versionInfo.getCurrentVersion() == UpgradeHistory_EmptyVersion) { - return VersionStatus_NeedUpgrade; - } - - // Check that we aren't too old - if (CURRENT_CONFIG_VERSION < versionInfo.getMinCompatibleVersion()) { - *whyNot = stream() << "the config version " << CURRENT_CONFIG_VERSION - << " of our process is too old " - << "for the detected config version " - << versionInfo.getMinCompatibleVersion(); - - return VersionStatus_Incompatible; - } - - // versionString is the global version of this process - if (isInMongoVersionRanges(versionString, versionInfo.getExcludingMongoVersions())) { - // Cast needed here for MSVC compiler issue - *whyNot = stream() << "not compatible with current config version, version " - << reinterpret_cast<const char*>(versionString) << "has been excluded."; - - return VersionStatus_Incompatible; - } - - // Check if we need to upgrade - if (versionInfo.getCurrentVersion() >= CURRENT_CONFIG_VERSION) { - return VersionStatus_Compatible; - } - - return VersionStatus_NeedUpgrade; -} - -// Checks that all config servers are online -Status _checkConfigServersAlive(const ConnectionString& configLoc) { - BSONObj result; - try { - ScopedDbConnection conn(configLoc, 30); - conn->runCommand("admin", BSON("fsync" << 1), result); - conn.done(); - return getStatusFromCommandResult(result); - } catch (const DBException& e) { - return e.toStatus(); - } -} - -} // namespace - - -/** - * Returns the config version of the cluster pointed at by the connection string. - * - * @return OK if version found successfully, error status if something bad happened. - */ -Status getConfigVersion(CatalogManager* catalogManager, VersionType* versionInfo) { - try { - versionInfo->clear(); - - ScopedDbConnection conn(grid.shardRegistry()->getConfigServerConnectionString(), 30); - - unique_ptr<DBClientCursor> cursor(_safeCursor(conn->query("config.version", BSONObj()))); - - bool hasConfigData = conn->count(ShardType::ConfigNS) || - conn->count(DatabaseType::ConfigNS) || conn->count(CollectionType::ConfigNS); - - if (!cursor->more()) { - // Version is 1 if we have data, 0 if we're completely empty - if (hasConfigData) { - versionInfo->setMinCompatibleVersion(UpgradeHistory_UnreportedVersion); - versionInfo->setCurrentVersion(UpgradeHistory_UnreportedVersion); - } else { - versionInfo->setMinCompatibleVersion(UpgradeHistory_EmptyVersion); - versionInfo->setCurrentVersion(UpgradeHistory_EmptyVersion); - } - - conn.done(); - return Status::OK(); - } - - BSONObj versionDoc = cursor->next(); - auto versionInfoResult = VersionType::fromBSON(versionDoc); - if (!versionInfoResult.isOK()) { - conn.done(); - - return Status(ErrorCodes::UnsupportedFormat, - stream() << "invalid config version document " << versionDoc - << versionInfoResult.getStatus().toString()); - } - *versionInfo = versionInfoResult.getValue(); - - if (cursor->more()) { - conn.done(); - - return Status(ErrorCodes::RemoteValidationError, - stream() << "should only have 1 document " - << "in config.version collection"); - } - conn.done(); - } catch (const DBException& e) { - return e.toStatus(); - } - - return Status::OK(); -} - -Status checkAndInitConfigVersion(OperationContext* txn, - CatalogManager* catalogManager, - DistLockManager* distLockManager) { - VersionType versionInfo; - Status status = getConfigVersion(catalogManager, &versionInfo); - if (!status.isOK()) { - return status; - } - - string errMsg; - VersionStatus comp = isConfigVersionCompatible(versionInfo, &errMsg); - - if (comp == VersionStatus_Incompatible) - return {ErrorCodes::IncompatibleShardingMetadata, errMsg}; - if (comp == VersionStatus_Compatible) - return Status::OK(); - - invariant(comp == VersionStatus_NeedUpgrade); - - if (versionInfo.getCurrentVersion() != UpgradeHistory_EmptyVersion) { - return {ErrorCodes::IncompatibleShardingMetadata, - stream() << "newer version " << CURRENT_CONFIG_VERSION - << " of mongo config metadata is required, " - << "current version is " << versionInfo.getCurrentVersion()}; - } - - // Contact the config servers to make sure all are online - otherwise we wait a long time - // for locks. - status = _checkConfigServersAlive(grid.shardRegistry()->getConfigServerConnectionString()); - if (!status.isOK()) { - return status; - } - - // - // Acquire a lock for the upgrade process. - // - // We want to ensure that only a single mongo process is upgrading the config server at a - // time. - // - - string whyMessage(stream() << "initializing config database to new format v" - << CURRENT_CONFIG_VERSION); - auto lockTimeout = stdx::chrono::minutes(20); - auto scopedDistLock = distLockManager->lock(txn, "configUpgrade", whyMessage, lockTimeout); - if (!scopedDistLock.isOK()) { - return scopedDistLock.getStatus(); - } - - // - // Double-check compatibility inside the upgrade lock - // Another process may have won the lock earlier and done the upgrade for us, check - // if this is the case. - // - - status = getConfigVersion(catalogManager, &versionInfo); - if (!status.isOK()) { - return status; - } - - comp = isConfigVersionCompatible(versionInfo, &errMsg); - - if (comp == VersionStatus_Incompatible) { - return {ErrorCodes::IncompatibleShardingMetadata, errMsg}; - } - if (comp == VersionStatus_Compatible) - return Status::OK(); - - invariant(comp == VersionStatus_NeedUpgrade); - - // - // Run through the upgrade steps necessary to bring our config version to the current - // version - // - - log() << "initializing config server version to " << CURRENT_CONFIG_VERSION; - - status = makeConfigVersionDocument(txn, catalogManager); - if (!status.isOK()) - return status; - - log() << "initialization of config server to v" << CURRENT_CONFIG_VERSION << " successful"; - - return Status::OK(); -} - -} // namespace mongo diff --git a/src/mongo/s/catalog/legacy/config_upgrade.h b/src/mongo/s/catalog/legacy/config_upgrade.h deleted file mode 100644 index de4f9afcf6a..00000000000 --- a/src/mongo/s/catalog/legacy/config_upgrade.h +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright (C) 2012 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#pragma once - -#include <string> - -namespace mongo { - -class CatalogManager; -class DistLockManager; -class OperationContext; -class Status; -class VersionType; - -/** - * Returns the config version of the cluster pointed at by the connection string. - * - * @return OK if version found successfully, error status if something bad happened. - */ -Status getConfigVersion(CatalogManager* catalogManager, VersionType* versionInfo); - -/** - * Checks the config version and ensures it's the latest version, otherwise tries to update. - * - * Returns Status::OK() on success, or an error status indicating the source of failure. - */ -Status checkAndInitConfigVersion(OperationContext* txn, - CatalogManager* catalogManager, - DistLockManager* distLockManager); - -} // namespace mongo diff --git a/src/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.cpp b/src/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.cpp index d3bcef468d1..cc88467e662 100644 --- a/src/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.cpp +++ b/src/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.cpp @@ -28,12 +28,38 @@ #include "mongo/platform/basic.h" +#include "mongo/db/commands.h" #include "mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.h" namespace mongo { +using executor::RemoteCommandRequest; + +using std::string; + +const string CatalogManagerReplSetTestFixture::CONFIG_HOST_PORT{"$dummy_config:27017"}; + CatalogManagerReplSetTestFixture::CatalogManagerReplSetTestFixture() = default; CatalogManagerReplSetTestFixture::~CatalogManagerReplSetTestFixture() = default; +void CatalogManagerReplSetTestFixture::expectFindOnConfigSendErrorCode(ErrorCodes::Error code) { + onCommand([&, code](const RemoteCommandRequest& request) { + ASSERT_EQ(request.target, HostAndPort(CONFIG_HOST_PORT)); + ASSERT_EQ(request.dbname, "config"); + BSONObjBuilder responseBuilder; + Command::appendCommandStatus(responseBuilder, Status(code, "")); + return responseBuilder.obj(); + }); +} + +void CatalogManagerReplSetTestFixture::expectFindOnConfigSendBSONObjVector( + std::vector<BSONObj> obj) { + onFindCommand([&, obj](const RemoteCommandRequest& request) { + ASSERT_EQ(request.target, HostAndPort(CONFIG_HOST_PORT)); + ASSERT_EQ(request.dbname, "config"); + return obj; + }); +} + } // namespace mongo diff --git a/src/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.h b/src/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.h index 1b22b8c0e82..620c2ee7473 100644 --- a/src/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.h +++ b/src/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.h @@ -28,6 +28,8 @@ #pragma once +#include <vector> + #include "mongo/s/sharding_test_fixture.h" namespace mongo { @@ -39,6 +41,11 @@ class CatalogManagerReplSetTestFixture : public ShardingTestFixture { public: CatalogManagerReplSetTestFixture(); ~CatalogManagerReplSetTestFixture(); -}; +protected: + static const std::string CONFIG_HOST_PORT; + + void expectFindOnConfigSendErrorCode(ErrorCodes::Error code); + void expectFindOnConfigSendBSONObjVector(std::vector<BSONObj> obj); +}; } // namespace mongo diff --git a/src/mongo/s/catalog/type_config_version.cpp b/src/mongo/s/catalog/type_config_version.cpp index e90b8019e09..92710ca7638 100644 --- a/src/mongo/s/catalog/type_config_version.cpp +++ b/src/mongo/s/catalog/type_config_version.cpp @@ -33,7 +33,6 @@ #include "mongo/bson/bsonobjbuilder.h" #include "mongo/bson/util/bson_extract.h" #include "mongo/s/catalog/config_server_version.h" -#include "mongo/s/catalog/legacy/config_upgrade.h" #include "mongo/util/assert_util.h" #include "mongo/util/mongoutils/str.h" diff --git a/src/mongo/s/chunk_manager_tests.cpp b/src/mongo/s/chunk_manager_tests.cpp new file mode 100644 index 00000000000..de46653617a --- /dev/null +++ b/src/mongo/s/chunk_manager_tests.cpp @@ -0,0 +1,269 @@ +/** + * Copyright (C) 2009 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kDefault + +#include "mongo/client/remote_command_targeter_mock.h" +#include "mongo/client/remote_command_targeter_factory_mock.h" +#include "mongo/platform/basic.h" +#include "mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.h" +#include "mongo/s/catalog/type_chunk.h" +#include "mongo/s/catalog/type_collection.h" +#include "mongo/s/catalog/type_shard.h" +#include "mongo/s/chunk_manager.h" +#include "mongo/s/write_ops/batched_command_request.h" +#include "mongo/s/write_ops/batched_command_response.h" + +namespace mongo { + +using std::unique_ptr; +using std::set; +using std::string; +using std::vector; + +using executor::RemoteCommandResponse; +using executor::RemoteCommandRequest; + +namespace { + +static int rand(int max = -1) { + static unsigned seed = 1337; + +#if !defined(_WIN32) + int r = rand_r(&seed); +#else + int r = ::rand(); // seed not used in this case +#endif + + // Modding is bad, but don't really care in this case + return max > 0 ? r % max : r; +} + +class ChunkManagerFixture : public CatalogManagerReplSetTestFixture { +public: + void setUp() override { + CatalogManagerReplSetTestFixture::setUp(); + getMessagingPort()->setRemote(HostAndPort("FakeRemoteClient:34567")); + configTargeter()->setFindHostReturnValue(configHost); + } + +protected: + const HostAndPort configHost{HostAndPort(CONFIG_HOST_PORT)}; + static const ShardId _shardId; + static const string _collName; + static const string _dbName; + + static const int numSplitPoints = 100; + + void genUniqueRandomSplitKeys(const string& keyName, vector<BSONObj>* splitKeys) { + std::unordered_set<int> uniquePoints; + while (static_cast<int>(uniquePoints.size()) < numSplitPoints) { + uniquePoints.insert(rand(numSplitPoints * 10)); + } + for (auto it = uniquePoints.begin(); it != uniquePoints.end(); ++it) { + splitKeys->push_back(BSON(keyName << *it)); + } + } + + void expectInsertOnConfigSaveChunkAndReturnOk(std::vector<BSONObj>& chunks) { + onCommandWithMetadata([&](const RemoteCommandRequest& request) mutable { + ASSERT_EQ(request.target, HostAndPort(CONFIG_HOST_PORT)); + ASSERT_EQ(request.dbname, "config"); + + // Get "inserted" chunk doc from RemoteCommandRequest. + BatchedCommandRequest batchedCommandRequest(BatchedCommandRequest::BatchType_Insert); + string errmsg; + batchedCommandRequest.parseBSON(_dbName, request.cmdObj, &errmsg); + vector<BSONObj> docs = batchedCommandRequest.getInsertRequest()->getDocuments(); + BSONObj chunk = docs.front(); + + // Save chunk (mimic "insertion"). + chunks.push_back(chunk); + + return RemoteCommandResponse(BSON("ok" << 1), BSONObj(), Milliseconds(1)); + }); + } + + void expectInsertOnConfigCheckMetadataAndReturnOk(set<int>& minorVersions, OID& epoch) { + onCommandWithMetadata([&](const RemoteCommandRequest& request) mutable { + ASSERT_EQ(request.target, HostAndPort(CONFIG_HOST_PORT)); + ASSERT_EQ(request.dbname, "config"); + + // Get "inserted" chunk doc from RemoteCommandRequest. + BatchedCommandRequest batchedCommandRequest(BatchedCommandRequest::BatchType_Insert); + string errmsg; + batchedCommandRequest.parseBSON(_dbName, request.cmdObj, &errmsg); + vector<BSONObj> docs = batchedCommandRequest.getInsertRequest()->getDocuments(); + BSONObj chunk = docs.front(); + + ChunkVersion version = ChunkVersion::fromBSON(chunk, ChunkType::DEPRECATED_lastmod()); + + // Check chunk's major version. + ASSERT(version.majorVersion() == 1); + + // Check chunk's minor version is unique. + ASSERT(minorVersions.find(version.minorVersion()) == minorVersions.end()); + minorVersions.insert(version.minorVersion()); + + // Check chunk's epoch is consistent. + ASSERT(version.epoch().isSet()); + if (!epoch.isSet()) { + epoch = version.epoch(); + } + ASSERT(version.epoch() == epoch); + + // Check chunk's shard id. + ASSERT(chunk[ChunkType::shard()].String() == _shardId); + + return RemoteCommandResponse(BSON("ok" << 1), BSONObj(), Milliseconds(1)); + }); + } +}; + +const ShardId ChunkManagerFixture::_shardId{"shard0000"}; +const string ChunkManagerFixture::_collName{"foo.bar"}; +const string ChunkManagerFixture::_dbName{"foo"}; + +// Rename the fixture so that our tests have a useful name in the executable +typedef ChunkManagerFixture ChunkManagerTests; + +/** + * Tests loading chunks into a ChunkManager with or without an old ChunkManager. + */ +TEST_F(ChunkManagerTests, Basic) { + string keyName = "_id"; + vector<BSONObj> splitKeys; + genUniqueRandomSplitKeys(keyName, &splitKeys); + ShardKeyPattern shardKeyPattern(BSON(keyName << 1)); + + std::vector<BSONObj> shards{ + BSON(ShardType::name() << _shardId << ShardType::host() + << ConnectionString(HostAndPort("$hostFooBar:27017")).toString())}; + + // Generate and save a set of chunks with metadata using a temporary ChunkManager. + + std::vector<BSONObj> chunks; + auto future = launchAsync([&] { + ChunkManager manager(_collName, shardKeyPattern, false); + auto status = manager.createFirstChunks(operationContext(), _shardId, &splitKeys, NULL); + ASSERT_OK(status); + }); + + // Call the expect() one extra time since numChunks = numSplits + 1. + for (int i = 0; i < static_cast<int>(splitKeys.size()) + 1; i++) { + expectInsertOnConfigSaveChunkAndReturnOk(chunks); + } + + future.timed_get(kFutureTimeout); + + // Test that a *new* ChunkManager correctly loads the chunks with *no prior info*. + + int numChunks = static_cast<int>(chunks.size()); + BSONObj firstChunk = chunks.back(); + ChunkVersion version = ChunkVersion::fromBSON(firstChunk, ChunkType::DEPRECATED_lastmod()); + + CollectionType collType; + collType.setNs(NamespaceString{_collName}); + collType.setEpoch(version.epoch()); + collType.setUpdatedAt(jsTime()); + collType.setKeyPattern(BSON(keyName << 1)); + collType.setUnique(false); + collType.setDropped(false); + + ChunkManager manager(collType); + future = launchAsync([&] { + manager.loadExistingRanges(operationContext(), nullptr); + + ASSERT_EQ(version.epoch(), manager.getVersion().epoch()); + ASSERT_EQ(numChunks - 1, manager.getVersion().minorVersion()); + ASSERT_EQ(numChunks, static_cast<int>(manager.getChunkMap().size())); + }); + expectFindOnConfigSendBSONObjVector(chunks); + expectFindOnConfigSendBSONObjVector(shards); + future.timed_get(kFutureTimeout); + + // Test that a *new* ChunkManager correctly loads modified chunks *given an old ChunkManager*. + + // Simulate modified chunks collection + ChunkVersion laterVersion = ChunkVersion(2, 1, version.epoch()); + BSONObj oldChunk = chunks.front(); + BSONObjBuilder newChunk; + newChunk.append("_id", oldChunk.getStringField("_id")); + newChunk.append("ns", oldChunk.getStringField("ns")); + newChunk.append("min", oldChunk.getObjectField("min")); + newChunk.append("max", oldChunk.getObjectField("min")); + newChunk.append("shard", oldChunk.getStringField("shard")); + laterVersion.addToBSON(newChunk, ChunkType::DEPRECATED_lastmod()); + newChunk.append("lastmodEpoch", oldChunk.getField("lastmodEpoch").OID()); + + // Make new manager load chunk diff + future = launchAsync([&] { + ChunkManager newManager(manager.getns(), manager.getShardKeyPattern(), manager.isUnique()); + newManager.loadExistingRanges(operationContext(), &manager); + + ASSERT_EQ(numChunks, static_cast<int>(manager.getChunkMap().size())); + ASSERT_EQ(laterVersion.toString(), newManager.getVersion().toString()); + }); + expectFindOnConfigSendBSONObjVector(std::vector<BSONObj>{chunks.back(), newChunk.obj()}); + + std::cout << "done"; + future.timed_get(kFutureTimeout); + std::cout << "completely done"; +} + +/** + * Tests that chunk metadata is created correctly when using ChunkManager to create chunks for the + * first time. Creating chunks on multiple shards is not tested here since there are unresolved + * race conditions there and probably should be avoided if at all possible. + */ +TEST_F(ChunkManagerTests, FullTest) { + string keyName = "_id"; + vector<BSONObj> splitKeys; + genUniqueRandomSplitKeys(keyName, &splitKeys); + ShardKeyPattern shardKeyPattern(BSON(keyName << 1)); + + auto future = launchAsync([&] { + ChunkManager manager(_collName, shardKeyPattern, false); + auto status = manager.createFirstChunks(operationContext(), _shardId, &splitKeys, NULL); + ASSERT_OK(status); + }); + + // Check that config server receives chunks with the expected metadata. + // Call expectInsertOnConfigCheckMetadataAndReturnOk one extra time since numChunks = numSplits + // + 1 + set<int> minorVersions; + OID epoch; + for (auto it = splitKeys.begin(); it != splitKeys.end(); ++it) { + expectInsertOnConfigCheckMetadataAndReturnOk(minorVersions, epoch); + } + expectInsertOnConfigCheckMetadataAndReturnOk(minorVersions, epoch); + future.timed_get(kFutureTimeout); +} + +} // namespace +} // namespace mongo |