summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorEsha Maharishi <esha.maharishi@mongodb.com>2016-03-08 17:56:34 -0500
committerEsha Maharishi <esha.maharishi@mongodb.com>2016-03-14 19:05:31 -0400
commit50f37a888d2d4f0a53196648b4ef9ada4a039be9 (patch)
treeb9f43bdc078adfce417a2db8b54da4e2df470ad1
parent9ffa6bc05261012b9df16b62a2915a61dcb8f51d (diff)
downloadmongo-50f37a888d2d4f0a53196648b4ef9ada4a039be9.tar.gz
SERVER-22483 convert disabled SCCC dbtests to jstests or unittests that use CSRS
-rw-r--r--jstests/sharding/merge_chunks_compound_shard_key.js93
-rw-r--r--src/mongo/db/s/collection_metadata_test.cpp84
-rw-r--r--src/mongo/db/s/metadata_loader_fixture.h7
-rw-r--r--src/mongo/db/s/metadata_loader_test.cpp20
-rw-r--r--src/mongo/dbtests/SConscript1
-rw-r--r--src/mongo/dbtests/chunk_manager_tests.cpp208
-rw-r--r--src/mongo/dbtests/config_server_fixture.cpp125
-rw-r--r--src/mongo/dbtests/config_server_fixture.h123
-rw-r--r--src/mongo/dbtests/config_upgrade_tests.cpp249
-rw-r--r--src/mongo/dbtests/framework.cpp1
-rw-r--r--src/mongo/dbtests/merge_chunk_tests.cpp338
-rw-r--r--src/mongo/s/SConscript10
-rw-r--r--src/mongo/s/catalog/legacy/config_upgrade.cpp323
-rw-r--r--src/mongo/s/catalog/legacy/config_upgrade.h57
-rw-r--r--src/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.cpp26
-rw-r--r--src/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.h9
-rw-r--r--src/mongo/s/catalog/type_config_version.cpp1
-rw-r--r--src/mongo/s/chunk_manager_tests.cpp269
18 files changed, 463 insertions, 1481 deletions
diff --git a/jstests/sharding/merge_chunks_compound_shard_key.js b/jstests/sharding/merge_chunks_compound_shard_key.js
new file mode 100644
index 00000000000..f6a2ef81ceb
--- /dev/null
+++ b/jstests/sharding/merge_chunks_compound_shard_key.js
@@ -0,0 +1,93 @@
+//
+// Tests that merging chunks via mongos works/doesn't work with different chunk configurations
+// with a compound shard key.
+//
+
+(function() {
+'use strict';
+
+var getShardVersion = function() {
+ var res = st.shard0.adminCommand({ getShardVersion: coll + "" })
+ assert.commandWorked(res);
+ var version = res.global;
+ assert(version);
+ return version;
+}
+
+// Merge two neighboring chunks and check post conditions.
+var checkMergeWorked = function(lowerBound, upperBound) {
+ var oldVersion = getShardVersion();
+ var numChunksBefore = chunks.find().itcount();
+
+ assert.commandWorked(admin.runCommand({ mergeChunks: coll + "",
+ bounds: [lowerBound, upperBound] }));
+
+ assert.eq(numChunksBefore - 1, chunks.find().itcount());
+ assert.eq(1, chunks.find({ min: lowerBound, max: upperBound }).itcount());
+
+ var newVersion = getShardVersion();
+ assert.eq(newVersion.t, oldVersion.t);
+ assert.gt(newVersion.i, oldVersion.i);
+}
+
+var st = new ShardingTest({ shards: 2, mongos: 1 });
+
+var mongos = st.s;
+var admin = mongos.getDB( "admin" );
+var shards = mongos.getCollection( "config.shards" ).find().toArray();
+var chunks = mongos.getCollection( "config.chunks" );
+var coll = mongos.getCollection( "foo.bar" );
+
+jsTest.log("Create a sharded collection with a compound shard key.");
+assert.commandWorked(admin.runCommand({ enableSharding: coll.getDB() + "" }));
+printjson( admin.runCommand({ movePrimary: coll.getDB() + "", to: st.shard0.shardName }) );
+assert.commandWorked(admin.runCommand({ shardCollection: coll + "", key: { x: 1, y: 1 } }));
+
+// Chunks after splits:
+// (MinKey, { x: 0, y: 1 })
+// ({ x: 0, y: 1 }, { x: 1, y: 0 })
+// ({ x: 1, y: 0 }, { x: 2, y: 0 })
+// ({ x: 2, y: 0 }, { x: 2, y: 1 })
+// ({ x: 2, y: 1 }, MaxKey)
+jsTest.log("Create chunks.");
+assert.commandWorked(admin.runCommand({ split: coll + "", middle: { x: 0, y: 1 } }));
+assert.commandWorked(admin.runCommand({ split: coll + "", middle: { x: 1, y: 0 } }));
+assert.commandWorked(admin.runCommand({ split: coll + "", middle: { x: 2, y: 0 } }));
+assert.commandWorked(admin.runCommand({ split: coll + "", middle: { x: 2, y: 1 } }));
+
+jsTest.log("Insert some data into each of the chunk ranges.");
+assert.writeOK(coll.insert({ x: -1, y: 2 }));
+assert.writeOK(coll.insert({ x: 0, y: 2 }));
+assert.writeOK(coll.insert({ x: 1, y: 2 }));
+assert.writeOK(coll.insert({ x: 2, y: 1 }));
+assert.writeOK(coll.insert({ x: 2, y: 3 }));
+
+// Chunks after merge:
+// (MinKey, { x: 0, y: 1 })
+// ({ x: 0, y: 1 }, { x: 2, y: 0 })
+// ({ x: 2, y: 0 }, { x: 2, y: 1 })
+// ({ x: 2, y: 1 }, MaxKey)
+jsTest.log("Merge chunks whose upper and lower bounds are compound shard keys.");
+checkMergeWorked({ x: 0, y: 1 }, { x: 2, y: 0 });
+
+// Chunks after merge:
+// (MinKey, { x: 2, y: 0 })
+// ({ x: 2, y: 0 }, { x: 2, y: 1 })
+// ({ x: 2, y: 1 }, MaxKey)
+jsTest.log("Merge chunks whose upper bound contains a compound shard key, lower bound is MinKey");
+checkMergeWorked({ x: MinKey, y: MinKey }, { x: 2, y: 0 });
+
+// Chunks after merge:
+// (MinKey, { x: 2, y: 0 })
+// ({ x: 2, y: 0 }, MaxKey)
+jsTest.log("Merge chunks whose lower bound contains a compound shard key, upper bound is MaxKey");
+checkMergeWorked({ x: 2, y: 0 }, { x: MaxKey, y: MaxKey });
+
+// Chunks after merge:
+// (MinKey, MaxKey)
+jsTest.log("Merge chunks whos bounds are MinKey/MaxKey, but which have a compound shard key");
+checkMergeWorked({ x: MinKey, y: MinKey }, { x: MaxKey, y: MaxKey });
+
+st.stop();
+
+})();
diff --git a/src/mongo/db/s/collection_metadata_test.cpp b/src/mongo/db/s/collection_metadata_test.cpp
index 7a8c59eafbf..bb3ab6b2ba9 100644
--- a/src/mongo/db/s/collection_metadata_test.cpp
+++ b/src/mongo/db/s/collection_metadata_test.cpp
@@ -27,9 +27,14 @@
*/
#include "mongo/base/status.h"
+#include "mongo/client/remote_command_targeter_mock.h"
+#include "mongo/client/remote_command_targeter_factory_mock.h"
#include "mongo/db/commands.h"
#include "mongo/db/s/collection_metadata.h"
-#include "mongo/db/s/metadata_loader_fixture.h"
+#include "mongo/db/s/metadata_loader.h"
+#include "mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.h"
+#include "mongo/s/catalog/type_chunk.h"
+#include "mongo/s/catalog/type_collection.h"
#include "mongo/s/chunk_version.h"
#include "mongo/s/write_ops/batched_command_response.h"
@@ -42,17 +47,21 @@ using std::vector;
using executor::RemoteCommandResponse;
-class NoChunkFixture : public MetadataLoaderFixture {
+class NoChunkFixture : public CatalogManagerReplSetTestFixture {
protected:
void setUp() {
- MetadataLoaderFixture::setUp();
+ CatalogManagerReplSetTestFixture::setUp();
+ getMessagingPort()->setRemote(HostAndPort("FakeRemoteClient:34567"));
+ configTargeter()->setFindHostReturnValue(configHost);
+
+ OID epoch = OID::gen();
CollectionType collType;
collType.setNs(NamespaceString{"test.foo"});
collType.setKeyPattern(BSON("a" << 1));
collType.setUnique(false);
collType.setUpdatedAt(Date_t::fromMillisSinceEpoch(1));
- collType.setEpoch(_epoch);
+ collType.setEpoch(epoch);
ASSERT_OK(collType.validate());
// Need a chunk on another shard, otherwise the chunks are invalid in general and we
@@ -62,7 +71,7 @@ protected:
chunkType.setShard("shard0001");
chunkType.setMin(BSON("a" << MINKEY));
chunkType.setMax(BSON("a" << MAXKEY));
- chunkType.setVersion(ChunkVersion(1, 0, _epoch));
+ chunkType.setVersion(ChunkVersion(1, 0, epoch));
chunkType.setName(OID::gen().toString());
ASSERT_OK(chunkType.validate());
std::vector<BSONObj> chunksToSend{chunkType.toBSON()};
@@ -91,6 +100,7 @@ protected:
private:
CollectionMetadata _metadata;
+ const HostAndPort configHost{HostAndPort(CONFIG_HOST_PORT)};
};
TEST_F(NoChunkFixture, BasicBelongsToMe) {
@@ -426,26 +436,30 @@ TEST_F(NoChunkFixture, PendingOrphanedDataRanges) {
* Fixture with single chunk containing:
* [10->20)
*/
-class SingleChunkFixture : public MetadataLoaderFixture {
+class SingleChunkFixture : public CatalogManagerReplSetTestFixture {
protected:
void setUp() {
- MetadataLoaderFixture::setUp();
+ CatalogManagerReplSetTestFixture::setUp();
+ getMessagingPort()->setRemote(HostAndPort("FakeRemoteClient:34567"));
+ configTargeter()->setFindHostReturnValue(configHost);
+
+ OID epoch = OID::gen();
- ChunkVersion chunkVersion = ChunkVersion(1, 0, _epoch);
+ ChunkVersion chunkVersion = ChunkVersion(1, 0, epoch);
CollectionType collType;
collType.setNs(NamespaceString{"test.foo"});
collType.setKeyPattern(BSON("a" << 1));
collType.setUnique(false);
collType.setUpdatedAt(Date_t::fromMillisSinceEpoch(1));
- collType.setEpoch(_epoch);
+ collType.setEpoch(epoch);
BSONObj fooSingle = BSON(
ChunkType::name("test.foo-a_10")
<< ChunkType::ns("test.foo") << ChunkType::min(BSON("a" << 10))
<< ChunkType::max(BSON("a" << 20))
<< ChunkType::DEPRECATED_lastmod(Date_t::fromMillisSinceEpoch(chunkVersion.toLong()))
- << ChunkType::DEPRECATED_epoch(_epoch) << ChunkType::shard("shard0000"));
+ << ChunkType::DEPRECATED_epoch(epoch) << ChunkType::shard("shard0000"));
std::vector<BSONObj> chunksToSend{fooSingle};
auto future = launchAsync([this] {
@@ -471,6 +485,7 @@ protected:
private:
CollectionMetadata _metadata;
+ const HostAndPort configHost{HostAndPort(CONFIG_HOST_PORT)};
};
TEST_F(SingleChunkFixture, BasicBelongsToMe) {
@@ -737,26 +752,30 @@ TEST_F(SingleChunkFixture, ChunkOrphanedDataRanges) {
* Fixture with single chunk containing:
* [(min, min)->(max, max))
*/
-class SingleChunkMinMaxCompoundKeyFixture : public MetadataLoaderFixture {
+class SingleChunkMinMaxCompoundKeyFixture : public CatalogManagerReplSetTestFixture {
protected:
void setUp() {
- MetadataLoaderFixture::setUp();
+ CatalogManagerReplSetTestFixture::setUp();
+ getMessagingPort()->setRemote(HostAndPort("FakeRemoteClient:34567"));
+ configTargeter()->setFindHostReturnValue(configHost);
+
+ OID epoch = OID::gen();
- ChunkVersion chunkVersion = ChunkVersion(1, 0, _epoch);
+ ChunkVersion chunkVersion = ChunkVersion(1, 0, epoch);
CollectionType collType;
collType.setNs(NamespaceString{"test.foo"});
collType.setKeyPattern(BSON("a" << 1));
collType.setUnique(false);
collType.setUpdatedAt(Date_t::fromMillisSinceEpoch(1));
- collType.setEpoch(_epoch);
+ collType.setEpoch(epoch);
BSONObj fooSingle = BSON(
ChunkType::name("test.foo-a_MinKey")
<< ChunkType::ns("test.foo") << ChunkType::min(BSON("a" << MINKEY << "b" << MINKEY))
<< ChunkType::max(BSON("a" << MAXKEY << "b" << MAXKEY))
<< ChunkType::DEPRECATED_lastmod(Date_t::fromMillisSinceEpoch(chunkVersion.toLong()))
- << ChunkType::DEPRECATED_epoch(_epoch) << ChunkType::shard("shard0000"));
+ << ChunkType::DEPRECATED_epoch(epoch) << ChunkType::shard("shard0000"));
std::vector<BSONObj> chunksToSend{fooSingle};
auto future = launchAsync([this] {
@@ -782,6 +801,7 @@ protected:
private:
CollectionMetadata _metadata;
+ const HostAndPort configHost{HostAndPort(CONFIG_HOST_PORT)};
};
// Note: no tests for single key belongsToMe because they are not allowed
@@ -798,19 +818,23 @@ TEST_F(SingleChunkMinMaxCompoundKeyFixture, CompoudKeyBelongsToMe) {
* Fixture with chunks:
* [(10, 0)->(20, 0)), [(30, 0)->(40, 0))
*/
-class TwoChunksWithGapCompoundKeyFixture : public MetadataLoaderFixture {
+class TwoChunksWithGapCompoundKeyFixture : public CatalogManagerReplSetTestFixture {
protected:
void setUp() {
- MetadataLoaderFixture::setUp();
+ CatalogManagerReplSetTestFixture::setUp();
+ getMessagingPort()->setRemote(HostAndPort("FakeRemoteClient:34567"));
+ configTargeter()->setFindHostReturnValue(configHost);
- ChunkVersion chunkVersion = ChunkVersion(1, 0, _epoch);
+ OID epoch = OID::gen();
+
+ ChunkVersion chunkVersion = ChunkVersion(1, 0, epoch);
CollectionType collType;
collType.setNs(NamespaceString{"test.foo"});
collType.setKeyPattern(BSON("a" << 1));
collType.setUnique(false);
collType.setUpdatedAt(Date_t::fromMillisSinceEpoch(1));
- collType.setEpoch(_epoch);
+ collType.setEpoch(epoch);
std::vector<BSONObj> chunksToSend;
chunksToSend.push_back(BSON(
@@ -818,13 +842,13 @@ protected:
<< ChunkType::ns("test.foo") << ChunkType::min(BSON("a" << 10 << "b" << 0))
<< ChunkType::max(BSON("a" << 20 << "b" << 0))
<< ChunkType::DEPRECATED_lastmod(Date_t::fromMillisSinceEpoch(chunkVersion.toLong()))
- << ChunkType::DEPRECATED_epoch(_epoch) << ChunkType::shard("shard0000")));
+ << ChunkType::DEPRECATED_epoch(epoch) << ChunkType::shard("shard0000")));
chunksToSend.push_back(BSON(
ChunkType::name("test.foo-a_10")
<< ChunkType::ns("test.foo") << ChunkType::min(BSON("a" << 30 << "b" << 0))
<< ChunkType::max(BSON("a" << 40 << "b" << 0))
<< ChunkType::DEPRECATED_lastmod(Date_t::fromMillisSinceEpoch(chunkVersion.toLong()))
- << ChunkType::DEPRECATED_epoch(_epoch) << ChunkType::shard("shard0000")));
+ << ChunkType::DEPRECATED_epoch(epoch) << ChunkType::shard("shard0000")));
auto future = launchAsync([this] {
MetadataLoader loader;
@@ -849,6 +873,7 @@ protected:
private:
CollectionMetadata _metadata;
+ const HostAndPort configHost{HostAndPort(CONFIG_HOST_PORT)};
};
TEST_F(TwoChunksWithGapCompoundKeyFixture, ClonePlusBasic) {
@@ -1041,21 +1066,25 @@ TEST_F(TwoChunksWithGapCompoundKeyFixture, ChunkGapAndPendingOrphanedDataRanges)
* Fixture with chunk containing:
* [min->10) , [10->20) , <gap> , [30->max)
*/
-class ThreeChunkWithRangeGapFixture : public MetadataLoaderFixture {
+class ThreeChunkWithRangeGapFixture : public CatalogManagerReplSetTestFixture {
protected:
void setUp() {
- MetadataLoaderFixture::setUp();
+ CatalogManagerReplSetTestFixture::setUp();
+ getMessagingPort()->setRemote(HostAndPort("FakeRemoteClient:34567"));
+ configTargeter()->setFindHostReturnValue(configHost);
+
+ OID epoch = OID::gen();
CollectionType collType;
collType.setNs(NamespaceString{"x.y"});
collType.setKeyPattern(BSON("a" << 1));
collType.setUnique(false);
collType.setUpdatedAt(Date_t::fromMillisSinceEpoch(1));
- collType.setEpoch(_epoch);
+ collType.setEpoch(epoch);
std::vector<BSONObj> chunksToSend;
{
- ChunkVersion version(1, 1, _epoch);
+ ChunkVersion version(1, 1, epoch);
chunksToSend.push_back(BSON(
ChunkType::name("x.y-a_MinKey")
<< ChunkType::ns("x.y") << ChunkType::min(BSON("a" << MINKEY))
@@ -1065,7 +1094,7 @@ protected:
}
{
- ChunkVersion version(1, 3, _epoch);
+ ChunkVersion version(1, 3, epoch);
chunksToSend.push_back(BSON(
ChunkType::name("x.y-a_10")
<< ChunkType::ns("x.y") << ChunkType::min(BSON("a" << 10))
@@ -1075,7 +1104,7 @@ protected:
}
{
- ChunkVersion version(1, 2, _epoch);
+ ChunkVersion version(1, 2, epoch);
chunksToSend.push_back(BSON(
ChunkType::name("x.y-a_30")
<< ChunkType::ns("x.y") << ChunkType::min(BSON("a" << 30))
@@ -1107,6 +1136,7 @@ protected:
private:
CollectionMetadata _metadata;
+ const HostAndPort configHost{HostAndPort(CONFIG_HOST_PORT)};
};
TEST_F(ThreeChunkWithRangeGapFixture, ShardOwnsDoc) {
diff --git a/src/mongo/db/s/metadata_loader_fixture.h b/src/mongo/db/s/metadata_loader_fixture.h
index 8d9ba9bc948..1f12caa90fe 100644
--- a/src/mongo/db/s/metadata_loader_fixture.h
+++ b/src/mongo/db/s/metadata_loader_fixture.h
@@ -28,8 +28,6 @@
#pragma once
-#include <vector>
-
#include "mongo/base/owned_pointer_vector.h"
#include "mongo/db/s/metadata_loader.h"
#include "mongo/platform/basic.h"
@@ -47,13 +45,8 @@ public:
~MetadataLoaderFixture();
protected:
- static const std::string CONFIG_HOST_PORT;
-
void setUp() override;
- void expectFindOnConfigSendErrorCode(ErrorCodes::Error code);
- void expectFindOnConfigSendBSONObjVector(std::vector<BSONObj> obj);
-
void expectFindOnConfigSendCollectionDefault();
void expectFindOnConfigSendChunksDefault();
diff --git a/src/mongo/db/s/metadata_loader_test.cpp b/src/mongo/db/s/metadata_loader_test.cpp
index d766f613c9a..0d5b93cc7fc 100644
--- a/src/mongo/db/s/metadata_loader_test.cpp
+++ b/src/mongo/db/s/metadata_loader_test.cpp
@@ -42,8 +42,6 @@ using std::vector;
using executor::RemoteCommandRequest;
-const string MetadataLoaderFixture::CONFIG_HOST_PORT{"$dummy_config:27017"};
-
MetadataLoaderFixture::MetadataLoaderFixture() = default;
MetadataLoaderFixture::~MetadataLoaderFixture() = default;
@@ -56,24 +54,6 @@ void MetadataLoaderFixture::setUp() {
_loader.reset(new MetadataLoader);
}
-void MetadataLoaderFixture::expectFindOnConfigSendErrorCode(ErrorCodes::Error code) {
- onCommand([&, code](const RemoteCommandRequest& request) {
- ASSERT_EQ(request.target, HostAndPort(CONFIG_HOST_PORT));
- ASSERT_EQ(request.dbname, "config");
- BSONObjBuilder responseBuilder;
- Command::appendCommandStatus(responseBuilder, Status(code, ""));
- return responseBuilder.obj();
- });
-}
-
-void MetadataLoaderFixture::expectFindOnConfigSendBSONObjVector(std::vector<BSONObj> obj) {
- onFindCommand([&, obj](const RemoteCommandRequest& request) {
- ASSERT_EQ(request.target, HostAndPort(CONFIG_HOST_PORT));
- ASSERT_EQ(request.dbname, "config");
- return obj;
- });
-}
-
void MetadataLoaderFixture::expectFindOnConfigSendCollectionDefault() {
CollectionType collType;
collType.setNs(NamespaceString{"test.foo"});
diff --git a/src/mongo/dbtests/SConscript b/src/mongo/dbtests/SConscript
index f239d5b8477..a7dd427ed0e 100644
--- a/src/mongo/dbtests/SConscript
+++ b/src/mongo/dbtests/SConscript
@@ -60,7 +60,6 @@ dbtest = env.Program(
'chunktests.cpp',
'clienttests.cpp',
'commandtests.cpp',
- 'config_server_fixture.cpp',
'counttests.cpp',
'dbhelper_tests.cpp',
'dbtests.cpp',
diff --git a/src/mongo/dbtests/chunk_manager_tests.cpp b/src/mongo/dbtests/chunk_manager_tests.cpp
deleted file mode 100644
index 1d87e53d59b..00000000000
--- a/src/mongo/dbtests/chunk_manager_tests.cpp
+++ /dev/null
@@ -1,208 +0,0 @@
-/**
- * Copyright (C) 2009 10gen Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License, version 3,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the GNU Affero General Public License in all respects
- * for all of the code used other than as permitted herein. If you modify
- * file(s) with this exception, you may extend this exception to your
- * version of the file(s), but you are not obligated to do so. If you do not
- * wish to do so, delete this exception statement from your version. If you
- * delete this exception statement from all source files in the program,
- * then also delete it in the license file.
- */
-
-#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kDefault
-
-#include "mongo/platform/basic.h"
-
-#include "mongo/db/operation_context_impl.h"
-#include "mongo/dbtests/config_server_fixture.h"
-#include "mongo/dbtests/dbtests.h"
-#include "mongo/s/catalog/type_chunk.h"
-#include "mongo/s/catalog/type_collection.h"
-#include "mongo/s/catalog/type_shard.h"
-#include "mongo/s/chunk_manager.h"
-
-namespace mongo {
-
-using std::unique_ptr;
-using std::set;
-using std::string;
-using std::vector;
-
-namespace {
-
-static int rand(int max = -1) {
- static unsigned seed = 1337;
-
-#if !defined(_WIN32)
- int r = rand_r(&seed);
-#else
- int r = ::rand(); // seed not used in this case
-#endif
-
- // Modding is bad, but don't really care in this case
- return max > 0 ? r % max : r;
-}
-
-/**
- * Sets up a basic environment for loading chunks to/from the direct database connection. Redirects
- * connections to the direct database for the duration of the test.
- */
-class ChunkManagerFixture : public ConfigServerFixture {
-public:
- void setUp() override {
- ConfigServerFixture::setUp();
-
- _client.dropDatabase(nsGetDB(_collName));
- _client.insert(_collName,
- BSON("hello"
- << "world"));
- _client.dropCollection(_collName);
-
- // Add dummy shard to config DB
- _client.insert(ShardType::ConfigNS,
- BSON(ShardType::name()
- << _shardId << ShardType::host()
- << ConnectionString(HostAndPort("$hostFooBar:27017")).toString()));
- }
-
-protected:
- static const ShardId _shardId;
- static const string _collName;
-
- static const int numSplitPoints = 100;
-
- void genRandomSplitPoints(vector<int>* splitPoints) {
- for (int i = 0; i < numSplitPoints; i++) {
- splitPoints->push_back(rand(numSplitPoints * 10));
- }
- }
-
- void genRandomSplitKeys(const string& keyName, vector<BSONObj>* splitKeys) {
- vector<int> splitPoints;
- genRandomSplitPoints(&splitPoints);
-
- for (vector<int>::iterator it = splitPoints.begin(); it != splitPoints.end(); ++it) {
- splitKeys->push_back(BSON(keyName << *it));
- }
- }
-
- // Uses a chunk manager to create chunks
- void createChunks(const string& keyName) {
- vector<BSONObj> splitKeys;
- genRandomSplitKeys(keyName, &splitKeys);
-
- ShardKeyPattern shardKeyPattern(BSON(keyName << 1));
- ChunkManager manager(_collName, shardKeyPattern, false);
-
- uassertStatusOK(manager.createFirstChunks(&_txn, _shardId, &splitKeys, NULL));
- }
-};
-
-const ShardId ChunkManagerFixture::_shardId{"shard0000"};
-const string ChunkManagerFixture::_collName{"foo.bar"};
-
-// Rename the fixture so that our tests have a useful name in the executable
-typedef ChunkManagerFixture ChunkManagerTests;
-
-/**
- * Tests that chunks are loaded correctly from the db with no a-priori info and also that they can
- * be reloaded on top of an old chunk manager with changes.
- */
-TEST_F(ChunkManagerTests, Basic) {
- string keyName = "_id";
- createChunks(keyName);
- int numChunks =
- static_cast<int>(_client.count(ChunkType::ConfigNS, BSON(ChunkType::ns(_collName))));
-
- BSONObj firstChunk = _client.findOne(ChunkType::ConfigNS, BSONObj()).getOwned();
-
- ChunkVersion version = ChunkVersion::fromBSON(firstChunk, ChunkType::DEPRECATED_lastmod());
-
- // Make manager load existing chunks
- CollectionType collType;
- collType.setNs(NamespaceString{_collName});
- collType.setEpoch(version.epoch());
- collType.setUpdatedAt(jsTime());
- collType.setKeyPattern(BSON("_id" << 1));
- collType.setUnique(false);
- collType.setDropped(false);
-
- ChunkManager manager(collType);
- manager.loadExistingRanges(&_txn, nullptr);
-
- ASSERT_EQ(version.epoch(), manager.getVersion().epoch());
- ASSERT_EQ(numChunks - 1, manager.getVersion().minorVersion());
- ASSERT_EQ(numChunks, static_cast<int>(manager.getChunkMap().size()));
-
- // Modify chunks collection
- BSONObjBuilder b;
- ChunkVersion laterVersion = ChunkVersion(2, 1, version.epoch());
- laterVersion.addToBSON(b, ChunkType::DEPRECATED_lastmod());
-
- _client.update(ChunkType::ConfigNS, BSONObj(), BSON("$set" << b.obj()));
-
- // Make new manager load chunk diff
- ChunkManager newManager(manager.getns(), manager.getShardKeyPattern(), manager.isUnique());
- newManager.loadExistingRanges(&_txn, &manager);
-
- ASSERT_EQ(numChunks, static_cast<int>(manager.getChunkMap().size()));
- ASSERT_EQ(laterVersion.toString(), newManager.getVersion().toString());
-}
-
-/**
- * Tests creating a new chunk manager with random split points. Creating chunks on multiple shards
- * is not tested here since there are unresolved race conditions there and probably should be
- * avoided if at all possible.
- */
-TEST_F(ChunkManagerTests, FullTest) {
- string keyName = "_id";
- createChunks(keyName);
-
- unique_ptr<DBClientCursor> cursor =
- _client.query(ChunkType::ConfigNS, QUERY(ChunkType::ns(_collName)));
-
- set<int> minorVersions;
- OID epoch;
-
- // Check that all chunks were created with version 1|x with consistent epoch and unique
- // minor versions
- while (cursor->more()) {
- BSONObj chunk = cursor->next();
-
- ChunkVersion version = ChunkVersion::fromBSON(chunk, ChunkType::DEPRECATED_lastmod());
-
- ASSERT(version.majorVersion() == 1);
- ASSERT(version.epoch().isSet());
-
- if (!epoch.isSet()) {
- epoch = version.epoch();
- }
-
- ASSERT(version.epoch() == epoch);
-
- ASSERT(minorVersions.find(version.minorVersion()) == minorVersions.end());
- minorVersions.insert(version.minorVersion());
-
- ASSERT(chunk[ChunkType::shard()].String() == _shardId);
- }
-}
-
-} // namespace
-} // namespace mongo
diff --git a/src/mongo/dbtests/config_server_fixture.cpp b/src/mongo/dbtests/config_server_fixture.cpp
deleted file mode 100644
index fe2fb4d4044..00000000000
--- a/src/mongo/dbtests/config_server_fixture.cpp
+++ /dev/null
@@ -1,125 +0,0 @@
-/**
- * Copyright (C) 2012 10gen Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License, version 3,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the GNU Affero General Public License in all respects
- * for all of the code used other than as permitted herein. If you modify
- * file(s) with this exception, you may extend this exception to your
- * version of the file(s), but you are not obligated to do so. If you do not
- * wish to do so, delete this exception statement from your version. If you
- * delete this exception statement from all source files in the program,
- * then also delete it in the license file.
- */
-
-#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kDefault
-
-#include "mongo/platform/basic.h"
-
-#include "mongo/dbtests/config_server_fixture.h"
-
-#include <list>
-
-#include "mongo/dbtests/dbtests.h"
-#include "mongo/db/service_context.h"
-#include "mongo/db/s/sharding_state.h"
-#include "mongo/s/catalog/type_chunk.h"
-#include "mongo/s/catalog/type_config_version.h"
-#include "mongo/s/client/shard_connection.h"
-#include "mongo/stdx/memory.h"
-#include "mongo/util/log.h"
-
-namespace mongo {
-
-using std::unique_ptr;
-using std::list;
-using std::string;
-
-ConfigServerFixture::ConfigServerFixture() : _client(&_txn), _connectHook(NULL) {}
-
-string ConfigServerFixture::shardName() {
- return "TestShardName";
-}
-
-void ConfigServerFixture::setUp() {
- shardConnectionPool.clear();
- DBException::traceExceptions = true;
-
- // Make all connections redirect to the direct client
- _connectHook = new CustomConnectHook(&_txn);
- ConnectionString::setConnectionHook(_connectHook);
-
- // Create the default config database before querying, necessary for direct connections
- clearServer();
- _client.insert("config.test",
- BSON("hello"
- << "world"));
- _client.dropCollection("config.test");
-
- // Create an index over the chunks, to allow correct diffing
- ASSERT_OK(
- dbtests::createIndex(&_txn,
- ChunkType::ConfigNS,
- BSON(ChunkType::ns() << 1 << ChunkType::DEPRECATED_lastmod() << 1)));
-
- const ConnectionString connStr(uassertStatusOK(ConnectionString::parse("$dummy:10000")));
-
- ShardingState::get(&_txn)->initialize(&_txn, connStr.toString());
- ShardingState::get(&_txn)->setShardName(shardName());
-}
-
-void ConfigServerFixture::clearServer() {
- _client.dropDatabase("config");
-}
-
-void ConfigServerFixture::clearVersion() {
- _client.dropCollection(VersionType::ConfigNS);
-}
-
-void ConfigServerFixture::dumpServer() {
- log() << "Dumping virtual config server to log...";
-
- list<string> collectionNames(_client.getCollectionNames("config"));
-
- for (list<string>::iterator it = collectionNames.begin(); it != collectionNames.end(); ++it) {
- const string& collection = *it;
-
- unique_ptr<DBClientCursor> cursor(_client.query(collection, BSONObj()).release());
- ASSERT(cursor.get() != NULL);
-
- log() << "Dumping collection " << collection;
-
- while (cursor->more()) {
- BSONObj obj = cursor->nextSafe();
- log() << obj.toString();
- }
- }
-}
-
-void ConfigServerFixture::tearDown() {
- ShardingState::get(&_txn)->clearCollectionMetadata();
- clearServer();
-
- // Make all connections redirect to the direct client
- ConnectionString::setConnectionHook(NULL);
- delete _connectHook;
- _connectHook = NULL;
-
- DBException::traceExceptions = false;
-}
-
-} // namespace mongo
diff --git a/src/mongo/dbtests/config_server_fixture.h b/src/mongo/dbtests/config_server_fixture.h
deleted file mode 100644
index 62cbedae4e7..00000000000
--- a/src/mongo/dbtests/config_server_fixture.h
+++ /dev/null
@@ -1,123 +0,0 @@
-/**
- * Copyright (C) 2012 10gen Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License, version 3,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the GNU Affero General Public License in all respects
- * for all of the code used other than as permitted herein. If you modify
- * file(s) with this exception, you may extend this exception to your
- * version of the file(s), but you are not obligated to do so. If you do not
- * wish to do so, delete this exception statement from your version. If you
- * delete this exception statement from all source files in the program,
- * then also delete it in the license file.
- */
-
-#pragma once
-
-#include "mongo/client/dbclientinterface.h"
-#include "mongo/db/dbdirectclient.h"
-#include "mongo/db/operation_context_impl.h"
-#include "mongo/db/wire_version.h"
-#include "mongo/unittest/unittest.h"
-#include "mongo/util/assert_util.h"
-
-namespace mongo {
-
-class CustomDirectClient : public DBDirectClient {
-public:
- CustomDirectClient(OperationContext* txn) : DBDirectClient(txn) {}
-
- virtual ConnectionString::ConnectionType type() const {
- return ConnectionString::CUSTOM;
- }
-
- virtual bool recv(Message& m) {
- // This is tailored to act as a dummy response for write commands.
-
- BufBuilder bb;
- bb.skip(sizeof(QueryResult::Value));
-
- BSONObj cmdResult(BSON("ok" << 1));
-
- bb.appendBuf(cmdResult.objdata(), cmdResult.objsize());
-
- QueryResult::View qr = bb.buf();
- bb.decouple();
- qr.setResultFlagsToOk();
- qr.msgdata().setLen(bb.len());
- qr.msgdata().setOperation(opReply);
- qr.setCursorId(0);
- qr.setStartingFrom(0);
- qr.setNReturned(1);
- m.setData(qr.view2ptr(), true);
-
- return true;
- }
-};
-
-class CustomConnectHook : public ConnectionString::ConnectionHook {
-public:
- CustomConnectHook(OperationContext* txn) : _txn(txn) {}
-
- virtual DBClientBase* connect(const ConnectionString& connStr,
- std::string& errmsg,
- double socketTimeout) {
- // Note - must be new, since it gets owned elsewhere
- return new CustomDirectClient(_txn);
- }
-
-private:
- OperationContext* const _txn;
-};
-
-/**
- * Fixture for testing complicated operations against a "virtual" config server.
- *
- * Use this if your test requires complex commands and writing to many collections,
- * otherwise a unit test in the mock framework may be a better option.
- */
-class ConfigServerFixture : public mongo::unittest::Test {
-public:
- ConfigServerFixture();
-
- /**
- * Returns a uniform shard name to use throughout the tests.
- */
- static std::string shardName();
-
- /**
- * Clears all data on the server
- */
- void clearServer();
-
- void clearVersion();
-
- /**
- * Dumps the contents of the config server to the log.
- */
- void dumpServer();
-
-protected:
- OperationContextImpl _txn;
- CustomDirectClient _client;
- CustomConnectHook* _connectHook;
-
- virtual void setUp();
- virtual void tearDown();
-};
-
-} // namespace mongo
diff --git a/src/mongo/dbtests/config_upgrade_tests.cpp b/src/mongo/dbtests/config_upgrade_tests.cpp
deleted file mode 100644
index 88db3071a1a..00000000000
--- a/src/mongo/dbtests/config_upgrade_tests.cpp
+++ /dev/null
@@ -1,249 +0,0 @@
-/**
- * Copyright (C) 2012 10gen Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License, version 3,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the GNU Affero General Public License in all respects
- * for all of the code used other than as permitted herein. If you modify
- * file(s) with this exception, you may extend this exception to your
- * version of the file(s), but you are not obligated to do so. If you do not
- * wish to do so, delete this exception statement from your version. If you
- * delete this exception statement from all source files in the program,
- * then also delete it in the license file.
- */
-
-#include "mongo/dbtests/config_server_fixture.h"
-#include "mongo/s/catalog/config_server_version.h"
-#include "mongo/s/catalog/legacy/cluster_client_internal.h"
-#include "mongo/s/catalog/legacy/config_upgrade.h"
-#include "mongo/s/catalog/type_chunk.h"
-#include "mongo/s/catalog/type_config_version.h"
-#include "mongo/s/catalog/type_mongos.h"
-#include "mongo/s/catalog/type_settings.h"
-#include "mongo/s/catalog/type_shard.h"
-#include "mongo/s/chunk_version.h"
-#include "mongo/s/grid.h"
-#include "mongo/unittest/unittest.h"
-#include "mongo/util/version.h"
-
-namespace mongo {
-
-using std::string;
-
-namespace {
-
-/**
- * Specialization of the config server fixture with helpers for the tests below.
- */
-class ConfigUpgradeFixture : public ConfigServerFixture {
-public:
- void stopBalancer() {
- // Note: The balancer key is needed in the update portion, for some reason related to
- // DBDirectClient
- DBDirectClient client(&_txn);
- client.update(SettingsType::ConfigNS,
- BSON(SettingsType::key(SettingsType::BalancerDocKey)),
- BSON(SettingsType::key(SettingsType::BalancerDocKey)
- << SettingsType::balancerStopped(true)),
- true,
- false);
- }
-
- /**
- * Stores a legacy { version : X } config server entry
- */
- void storeLegacyConfigVersion(int version) {
- if (version == 0)
- return;
-
- DBDirectClient client(&_txn);
-
- if (version == 1) {
- ShardType shard;
- shard.setName("test");
- shard.setHost("$dummy:10000");
- client.insert(ShardType::ConfigNS, shard.toBSON());
- return;
- }
-
- client.insert(VersionType::ConfigNS, BSON("_id" << 1 << "version" << version));
- }
-
- VersionType loadLegacyConfigVersion() {
- DBDirectClient client(&_txn);
- return unittest::assertGet(
- VersionType::fromBSON(client.findOne(VersionType::ConfigNS, BSONObj())));
- }
-
- /**
- * Stores a newer { version, minVersion, currentVersion, clusterId } config server entry
- */
- void storeConfigVersion(const VersionType& versionInfo) {
- DBDirectClient client(&_txn);
- client.insert(VersionType::ConfigNS, versionInfo.toBSON());
- }
-
- /**
- * Stores a newer { version, minVersion, currentVersion, clusterId } config server entry.
- *
- * @return clusterId
- */
- OID storeConfigVersion(int configVersion) {
- if (configVersion < CURRENT_CONFIG_VERSION) {
- storeLegacyConfigVersion(configVersion);
- return OID();
- }
-
- VersionType version;
- version.setMinCompatibleVersion(configVersion);
- version.setCurrentVersion(configVersion);
-
- OID clusterId = OID::gen();
-
- version.setClusterId(clusterId);
-
- storeConfigVersion(version);
- return clusterId;
- }
-
- /**
- * Stores sample shard and ping information at the current version.
- */
- void storeShardsAndPings(int numShards, int numPings) {
- DBDirectClient client(&_txn);
-
- for (int i = 0; i < numShards; i++) {
- ShardType shard;
- shard.setName(OID::gen().toString());
- shard.setHost((string)(str::stream() << "$dummyShard:" << (i + 1) << "0000"));
-
- client.insert(ShardType::ConfigNS, shard.toBSON());
- }
-
- time_t started = time(0);
- for (int i = 0; i < numPings; i++) {
- MongosType ping;
- ping.setName((string)(str::stream() << "$dummyMongos:" << (i + 1) << "0000"));
- ping.setPing(jsTime());
- ping.setUptime(time(0) - started);
- ping.setWaiting(false);
- ping.setMongoVersion(versionString);
- ping.setConfigVersion(CURRENT_CONFIG_VERSION);
-
- if (i % 2 == 0) {
- ping.setPing(ping.getPing() - Minutes(10));
- }
-
- client.insert(MongosType::ConfigNS, ping.toBSON());
- }
- }
-};
-
-//
-// Tests for upgrading the config server between versions.
-//
-// In general these tests do pretty minimal validation of the config server data itself, but
-// do ensure that the upgrade mechanism is working correctly w.r.t the config.version
-// collection.
-//
-
-// Rename the fixture so that our tests have a useful name in the executable
-typedef ConfigUpgradeFixture ConfigUpgradeTests;
-
-TEST_F(ConfigUpgradeTests, EmptyVersion) {
- //
- // Tests detection of empty config version
- //
-
- // Zero version (no version doc)
- VersionType oldVersion;
- Status status = getConfigVersion(grid.catalogManager(&_txn), &oldVersion);
- ASSERT(status.isOK());
-
- ASSERT_EQUALS(oldVersion.getMinCompatibleVersion(), 0);
- ASSERT_EQUALS(oldVersion.getCurrentVersion(), 0);
-}
-
-TEST_F(ConfigUpgradeTests, ClusterIDVersion) {
- //
- // Tests detection of newer config versions
- //
-
- VersionType newVersion;
- newVersion.setMinCompatibleVersion(MIN_COMPATIBLE_CONFIG_VERSION);
- newVersion.setCurrentVersion(CURRENT_CONFIG_VERSION);
- storeConfigVersion(newVersion);
-
- newVersion.clear();
-
- // Current Version w/o clusterId (invalid!)
- Status status = getConfigVersion(grid.catalogManager(&_txn), &newVersion);
- ASSERT(!status.isOK());
-
- newVersion.clear();
-
- OID clusterId = OID::gen();
- newVersion.setClusterId(clusterId);
- newVersion.setMinCompatibleVersion(MIN_COMPATIBLE_CONFIG_VERSION);
- newVersion.setCurrentVersion(CURRENT_CONFIG_VERSION);
-
- clearVersion();
- storeConfigVersion(newVersion);
-
- newVersion.clear();
-
- // Current version w/ clusterId (valid!)
- status = getConfigVersion(grid.catalogManager(&_txn), &newVersion);
- ASSERT(status.isOK());
-
- ASSERT_EQUALS(newVersion.getMinCompatibleVersion(), MIN_COMPATIBLE_CONFIG_VERSION);
- ASSERT_EQUALS(newVersion.getCurrentVersion(), CURRENT_CONFIG_VERSION);
- ASSERT_EQUALS(newVersion.getClusterId(), clusterId);
-}
-
-TEST_F(ConfigUpgradeTests, InitialUpgrade) {
- //
- // Tests initializing the config server to the initial version
- //
-
- string errMsg;
- ASSERT_OK(grid.catalogManager(&_txn)->initConfigVersion(&_txn));
-
- VersionType version;
- ASSERT_OK(getConfigVersion(grid.catalogManager(&_txn), &version));
-
- ASSERT_EQUALS(MIN_COMPATIBLE_CONFIG_VERSION, version.getMinCompatibleVersion());
- ASSERT_EQUALS(CURRENT_CONFIG_VERSION, version.getCurrentVersion());
- ASSERT_TRUE(version.getClusterId().isSet());
-}
-
-TEST_F(ConfigUpgradeTests, BadVersionUpgrade) {
- //
- // Tests that we can't upgrade from a config version we don't have an upgrade path for
- //
-
- stopBalancer();
-
- storeLegacyConfigVersion(1);
-
- // Default version (not upgradeable)
- ASSERT_EQ(ErrorCodes::IncompatibleShardingMetadata,
- grid.catalogManager(&_txn)->initConfigVersion(&_txn));
-}
-
-} // namespace
-} // namespace mongo
diff --git a/src/mongo/dbtests/framework.cpp b/src/mongo/dbtests/framework.cpp
index c9cc9d2028c..6ba11c289e0 100644
--- a/src/mongo/dbtests/framework.cpp
+++ b/src/mongo/dbtests/framework.cpp
@@ -41,7 +41,6 @@
#include "mongo/db/service_context.h"
#include "mongo/db/service_context_d.h"
#include "mongo/db/s/sharding_state.h"
-#include "mongo/dbtests/config_server_fixture.h"
#include "mongo/dbtests/dbtests.h"
#include "mongo/dbtests/framework_options.h"
#include "mongo/s/catalog/catalog_manager.h"
diff --git a/src/mongo/dbtests/merge_chunk_tests.cpp b/src/mongo/dbtests/merge_chunk_tests.cpp
deleted file mode 100644
index a2de2c89e2c..00000000000
--- a/src/mongo/dbtests/merge_chunk_tests.cpp
+++ /dev/null
@@ -1,338 +0,0 @@
-/**
- * Copyright (C) 2013 10gen Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License, version 3,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the GNU Affero General Public License in all respects
- * for all of the code used other than as permitted herein. If you modify
- * file(s) with this exception, you may extend this exception to your
- * version of the file(s), but you are not obligated to do so. If you do not
- * wish to do so, delete this exception statement from your version. If you
- * delete this exception statement from all source files in the program,
- * then also delete it in the license file.
- */
-
-#include "mongo/platform/basic.h"
-
-#include "mongo/db/range_arithmetic.h"
-#include "mongo/db/service_context.h"
-#include "mongo/db/s/collection_metadata.h"
-#include "mongo/db/s/sharding_state.h"
-#include "mongo/dbtests/config_server_fixture.h"
-#include "mongo/s/catalog/type_chunk.h"
-#include "mongo/s/catalog/type_collection.h"
-#include "mongo/s/chunk.h"
-#include "mongo/s/chunk_version.h"
-#include "mongo/s/d_merge.h"
-#include "mongo/unittest/unittest.h"
-
-namespace mongo {
-
-using std::string;
-using std::vector;
-
-namespace {
-
-/**
- * Specialization of the config server fixture with helpers for the tests below.
- */
-class MergeChunkFixture : public ConfigServerFixture {
-public:
- /**
- * Stores ranges for a particular collection and shard starting from some version
- */
- void storeCollectionRanges(const NamespaceString& nss,
- const string& shardName,
- const vector<KeyRange>& ranges,
- const ChunkVersion& startVersion) {
- // Get key pattern from first range
- ASSERT_GREATER_THAN(ranges.size(), 0u);
-
- CollectionType coll;
- coll.setNs(nss);
- coll.setKeyPattern(ranges.begin()->keyPattern);
- coll.setEpoch(startVersion.epoch());
- coll.setUpdatedAt(Date_t::fromMillisSinceEpoch(1));
- ASSERT_OK(coll.validate());
-
- DBDirectClient client(&_txn);
-
- client.update(CollectionType::ConfigNS,
- BSON(CollectionType::fullNs(coll.getNs().ns())),
- coll.toBSON(),
- true,
- false);
-
- ChunkVersion nextVersion = startVersion;
- for (vector<KeyRange>::const_iterator it = ranges.begin(); it != ranges.end(); ++it) {
- ChunkType chunk;
- // TODO: We should not rely on the serialized ns, minkey being unique in the future,
- // causes problems since it links string serialization to correctness.
- chunk.setName(Chunk::genID(nss.ns(), it->minKey));
- chunk.setShard(shardName);
- chunk.setNS(nss.ns());
- chunk.setVersion(nextVersion);
- chunk.setMin(it->minKey);
- chunk.setMax(it->maxKey);
- nextVersion.incMajor();
-
- client.insert(ChunkType::ConfigNS, chunk.toBSON());
- }
- }
-
- /**
- * Makes sure that all the ranges here no longer exist on disk but the merged range does
- */
- void assertWrittenAsMerged(const vector<KeyRange>& ranges) {
- dumpServer();
-
- BSONObj rangeMin;
- BSONObj rangeMax;
-
- DBDirectClient client(&_txn);
-
- // Ensure written
- for (vector<KeyRange>::const_iterator it = ranges.begin(); it != ranges.end(); ++it) {
- Query query(BSON(ChunkType::min(it->minKey) << ChunkType::max(it->maxKey)
- << ChunkType::shard(shardName())));
- ASSERT(client.findOne(ChunkType::ConfigNS, query).isEmpty());
-
- if (rangeMin.isEmpty() || rangeMin.woCompare(it->minKey) > 0) {
- rangeMin = it->minKey;
- }
-
- if (rangeMax.isEmpty() || rangeMax.woCompare(it->maxKey) < 0) {
- rangeMax = it->maxKey;
- }
- }
-
- Query query(BSON(ChunkType::min(rangeMin) << ChunkType::max(rangeMax)
- << ChunkType::shard(shardName())));
- ASSERT(!client.findOne(ChunkType::ConfigNS, query).isEmpty());
- }
-};
-
-//
-// Tests for upgrading the config server between versions.
-//
-// In general these tests do pretty minimal validation of the config server data itself, but
-// do ensure that the upgrade mechanism is working correctly w.r.t the config.version
-// collection.
-//
-
-// Rename the fixture so that our tests have a useful name in the executable
-typedef MergeChunkFixture MergeChunkTests;
-
-TEST_F(MergeChunkTests, FailedMerge) {
- const NamespaceString nss("foo.bar");
- const BSONObj kp = BSON("x" << 1);
- const OID epoch = OID::gen();
- vector<KeyRange> ranges;
-
- // Setup chunk metadata
- ranges.push_back(KeyRange(nss.ns(), BSON("x" << 0), BSON("x" << 10), kp));
- ranges.push_back(KeyRange(nss.ns(), BSON("x" << 10), BSON("x" << 20), kp));
- storeCollectionRanges(nss, shardName(), ranges, ChunkVersion(1, 0, epoch));
-
- // Do bad merges
- string errMsg;
- bool result;
-
- result = mergeChunks(&_txn, nss, BSON("x" << 5), BSON("x" << 20), epoch, &errMsg);
- ASSERT_NOT_EQUALS(errMsg, "");
- ASSERT(!result);
-
- result = mergeChunks(&_txn, nss, BSON("x" << 0), BSON("x" << 15), epoch, &errMsg);
- ASSERT_NOT_EQUALS(errMsg, "");
- ASSERT(!result);
-
- result = mergeChunks(&_txn, nss, BSON("x" << -10), BSON("x" << 20), epoch, &errMsg);
- ASSERT_NOT_EQUALS(errMsg, "");
- ASSERT(!result);
-
- result = mergeChunks(&_txn, nss, BSON("x" << 0), BSON("x" << 30), epoch, &errMsg);
- ASSERT_NOT_EQUALS(errMsg, "");
- ASSERT(!result);
-
- result = mergeChunks(&_txn, nss, BSON("x" << 0), BSON("x" << 10), epoch, &errMsg);
- ASSERT_NOT_EQUALS(errMsg, "");
- ASSERT(!result);
-
- // Wrong epoch
- result = mergeChunks(&_txn, nss, BSON("x" << 0), BSON("x" << 10), OID::gen(), &errMsg);
- ASSERT_NOT_EQUALS(errMsg, "");
- ASSERT(!result);
-}
-
-TEST_F(MergeChunkTests, FailedMergeHole) {
- const NamespaceString nss("foo.bar");
- const BSONObj kp = BSON("x" << 1);
- const OID epoch = OID::gen();
- vector<KeyRange> ranges;
-
- // Setup chunk metadata
- ranges.push_back(KeyRange(nss.ns(), BSON("x" << 0), BSON("x" << 10), kp));
- ranges.push_back(KeyRange(nss.ns(), BSON("x" << 11), BSON("x" << 20), kp));
- storeCollectionRanges(nss, shardName(), ranges, ChunkVersion(1, 0, epoch));
-
- // Do bad merge with hole
- string errMsg;
- bool result;
- result = mergeChunks(&_txn, nss, BSON("x" << 0), BSON("x" << 20), epoch, &errMsg);
- ASSERT_NOT_EQUALS(errMsg, "");
- ASSERT(!result);
-}
-
-TEST_F(MergeChunkTests, FailedMergeMinMax) {
- const NamespaceString nss("foo.bar");
- const BSONObj kp = BSON("x" << 1);
- const OID epoch = OID::gen();
- vector<KeyRange> ranges;
-
- // Setup chunk metadata
- ranges.push_back(KeyRange(nss.ns(), BSON("x" << MINKEY), BSON("x" << 0), kp));
- ranges.push_back(KeyRange(nss.ns(), BSON("x" << 0), BSON("x" << MAXKEY), kp));
- storeCollectionRanges(nss, shardName(), ranges, ChunkVersion(1, 0, epoch));
-
- // Do bad merge with hole
- string errMsg;
- bool result;
- result = mergeChunks(&_txn, nss, BSON("x" << -1), BSON("x" << MAXKEY), epoch, &errMsg);
- ASSERT_NOT_EQUALS(errMsg, "");
- ASSERT(!result);
-
- result = mergeChunks(&_txn, nss, BSON("x" << MINKEY), BSON("x" << 1), epoch, &errMsg);
- ASSERT_NOT_EQUALS(errMsg, "");
- ASSERT(!result);
-}
-
-TEST_F(MergeChunkTests, BasicMerge) {
- const NamespaceString nss("foo.bar");
- const BSONObj kp = BSON("x" << 1);
- const OID epoch = OID::gen();
- vector<KeyRange> ranges;
-
- // Setup chunk metadata
- ranges.push_back(KeyRange(nss.ns(), BSON("x" << 0), BSON("x" << 1), kp));
- ranges.push_back(KeyRange(nss.ns(), BSON("x" << 1), BSON("x" << 2), kp));
- storeCollectionRanges(nss, shardName(), ranges, ChunkVersion(1, 0, epoch));
-
- // Get latest version
- ChunkVersion latestVersion;
- ShardingState::get(&_txn)->refreshMetadataNow(&_txn, nss.ns(), &latestVersion);
- ShardingState::get(&_txn)->resetMetadata(nss.ns());
-
- // Do merge
- string errMsg;
- bool result = mergeChunks(&_txn, nss, BSON("x" << 0), BSON("x" << 2), epoch, &errMsg);
- ASSERT_EQUALS(errMsg, "");
- ASSERT(result);
-
- // Verify result
- CollectionMetadataPtr metadata = ShardingState::get(&_txn)->getCollectionMetadata(nss.ns());
-
- ChunkType chunk;
- ASSERT(metadata->getNextChunk(BSON("x" << 0), &chunk));
- ASSERT(chunk.getMin().woCompare(BSON("x" << 0)) == 0);
- ASSERT(chunk.getMax().woCompare(BSON("x" << 2)) == 0);
- ASSERT_EQUALS(metadata->getNumChunks(), 1u);
-
- ASSERT_EQUALS(metadata->getShardVersion().majorVersion(), latestVersion.majorVersion());
- ASSERT_GREATER_THAN(metadata->getShardVersion().minorVersion(), latestVersion.minorVersion());
-
- assertWrittenAsMerged(ranges);
-}
-
-TEST_F(MergeChunkTests, BasicMergeMinMax) {
- const NamespaceString nss("foo.bar");
- const BSONObj kp = BSON("x" << 1);
- const OID epoch = OID::gen();
- vector<KeyRange> ranges;
-
- // Setup chunk metadata
- ranges.push_back(KeyRange(nss.ns(), BSON("x" << MINKEY), BSON("x" << 0), kp));
- ranges.push_back(KeyRange(nss.ns(), BSON("x" << 0), BSON("x" << MAXKEY), kp));
- storeCollectionRanges(nss, shardName(), ranges, ChunkVersion(1, 0, epoch));
-
- // Get latest version
- ChunkVersion latestVersion;
- ShardingState::get(&_txn)->refreshMetadataNow(&_txn, nss.ns(), &latestVersion);
- ShardingState::get(&_txn)->resetMetadata(nss.ns());
-
- // Do merge
- string errMsg;
- bool result = mergeChunks(&_txn, nss, BSON("x" << MINKEY), BSON("x" << MAXKEY), epoch, &errMsg);
- ASSERT_EQUALS(errMsg, "");
- ASSERT(result);
-
- // Verify result
- CollectionMetadataPtr metadata = ShardingState::get(&_txn)->getCollectionMetadata(nss.ns());
-
- ChunkType chunk;
- ASSERT(metadata->getNextChunk(BSON("x" << MINKEY), &chunk));
- ASSERT(chunk.getMin().woCompare(BSON("x" << MINKEY)) == 0);
- ASSERT(chunk.getMax().woCompare(BSON("x" << MAXKEY)) == 0);
- ASSERT_EQUALS(metadata->getNumChunks(), 1u);
-
- ASSERT_EQUALS(metadata->getShardVersion().majorVersion(), latestVersion.majorVersion());
- ASSERT_GREATER_THAN(metadata->getShardVersion().minorVersion(), latestVersion.minorVersion());
-
- assertWrittenAsMerged(ranges);
-}
-
-TEST_F(MergeChunkTests, CompoundMerge) {
- const NamespaceString nss("foo.bar");
- const BSONObj kp = BSON("x" << 1 << "y" << 1);
- const OID epoch = OID::gen();
- vector<KeyRange> ranges;
-
- // Setup chunk metadata
- ranges.push_back(
- KeyRange(nss.ns(), BSON("x" << 0 << "y" << 1), BSON("x" << 1 << "y" << 0), kp));
- ranges.push_back(
- KeyRange(nss.ns(), BSON("x" << 1 << "y" << 0), BSON("x" << 2 << "y" << 1), kp));
- storeCollectionRanges(nss, shardName(), ranges, ChunkVersion(1, 0, epoch));
-
- // Get latest version
- ChunkVersion latestVersion;
- ShardingState::get(&_txn)->refreshMetadataNow(&_txn, nss.ns(), &latestVersion);
- ShardingState::get(&_txn)->resetMetadata(nss.ns());
-
- // Do merge
- string errMsg;
- bool result = mergeChunks(
- &_txn, nss, BSON("x" << 0 << "y" << 1), BSON("x" << 2 << "y" << 1), epoch, &errMsg);
- ASSERT_EQUALS(errMsg, "");
- ASSERT(result);
-
- // Verify result
- CollectionMetadataPtr metadata = ShardingState::get(&_txn)->getCollectionMetadata(nss.ns());
-
- ChunkType chunk;
- ASSERT(metadata->getNextChunk(BSON("x" << 0 << "y" << 1), &chunk));
- ASSERT(chunk.getMin().woCompare(BSON("x" << 0 << "y" << 1)) == 0);
- ASSERT(chunk.getMax().woCompare(BSON("x" << 2 << "y" << 1)) == 0);
- ASSERT_EQUALS(metadata->getNumChunks(), 1u);
-
- ASSERT_EQUALS(metadata->getShardVersion().majorVersion(), latestVersion.majorVersion());
- ASSERT_GREATER_THAN(metadata->getShardVersion().minorVersion(), latestVersion.minorVersion());
-
- assertWrittenAsMerged(ranges);
-}
-
-} // namespace
-} // namespace mongo
diff --git a/src/mongo/s/SConscript b/src/mongo/s/SConscript
index bfd6eead67e..a99be665ed3 100644
--- a/src/mongo/s/SConscript
+++ b/src/mongo/s/SConscript
@@ -149,6 +149,16 @@ env.CppUnitTest(
]
)
+env.CppUnitTest(
+ target='chunk_manager_tests',
+ source=[
+ 'chunk_manager_tests.cpp'
+ ],
+ LIBDEPS=[
+ 'coreshard',
+ '$BUILD_DIR/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture']
+)
+
# This library contains sharding functionality used by both mongod and mongos. Certain tests,
# which exercise this functionality also link against it.
env.Library(
diff --git a/src/mongo/s/catalog/legacy/config_upgrade.cpp b/src/mongo/s/catalog/legacy/config_upgrade.cpp
deleted file mode 100644
index ea09db1dc00..00000000000
--- a/src/mongo/s/catalog/legacy/config_upgrade.cpp
+++ /dev/null
@@ -1,323 +0,0 @@
-/**
- * Copyright (C) 2012 10gen Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License, version 3,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the GNU Affero General Public License in all respects
- * for all of the code used other than as permitted herein. If you modify
- * file(s) with this exception, you may extend this exception to your
- * version of the file(s), but you are not obligated to do so. If you do not
- * wish to do so, delete this exception statement from your version. If you
- * delete this exception statement from all source files in the program,
- * then also delete it in the license file.
- */
-
-#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kSharding
-
-#include "mongo/platform/basic.h"
-
-#include "mongo/s/catalog/legacy/config_upgrade.h"
-
-#include "mongo/client/connpool.h"
-#include "mongo/client/dbclientcursor.h"
-#include "mongo/rpc/get_status_from_command_result.h"
-#include "mongo/s/catalog/config_server_version.h"
-#include "mongo/s/catalog/dist_lock_manager.h"
-#include "mongo/s/catalog/legacy/cluster_client_internal.h"
-#include "mongo/s/catalog/mongo_version_range.h"
-#include "mongo/s/catalog/type_collection.h"
-#include "mongo/s/catalog/type_config_version.h"
-#include "mongo/s/catalog/type_database.h"
-#include "mongo/s/catalog/type_settings.h"
-#include "mongo/s/catalog/type_shard.h"
-#include "mongo/s/client/shard_registry.h"
-#include "mongo/s/grid.h"
-#include "mongo/stdx/functional.h"
-#include "mongo/util/assert_util.h"
-#include "mongo/util/log.h"
-#include "mongo/util/version.h"
-
-namespace mongo {
-
-using std::unique_ptr;
-using std::make_pair;
-using std::map;
-using std::string;
-using std::vector;
-using str::stream;
-
-namespace {
-
-Status makeConfigVersionDocument(OperationContext* txn, CatalogManager* catalogManager) {
- //
- // Even though the initial config write is a single-document update, that single document
- // is on multiple config servers and requests can interleave. The upgrade lock prevents
- // this.
- //
-
- log() << "writing initial config version at v" << CURRENT_CONFIG_VERSION;
-
- OID newClusterId = OID::gen();
-
- VersionType versionInfo;
-
- // Upgrade to new version
- versionInfo.setMinCompatibleVersion(MIN_COMPATIBLE_CONFIG_VERSION);
- versionInfo.setCurrentVersion(CURRENT_CONFIG_VERSION);
- versionInfo.setClusterId(newClusterId);
-
- invariantOK(versionInfo.validate());
-
- // If the cluster has not previously been initialized, we need to set the version before
- // using so subsequent mongoses use the config data the same way. This requires all three
- // config servers online initially.
- auto status = catalogManager->updateConfigDocument(
- txn, VersionType::ConfigNS, BSON("_id" << 1), versionInfo.toBSON(), true);
- return status.getStatus();
-}
-
-struct VersionRange {
- VersionRange(int _minCompatibleVersion, int _currentVersion)
- : minCompatibleVersion(_minCompatibleVersion), currentVersion(_currentVersion) {}
-
- bool operator==(const VersionRange& other) const {
- return (other.minCompatibleVersion == minCompatibleVersion) &&
- (other.currentVersion == currentVersion);
- }
-
- bool operator!=(const VersionRange& other) const {
- return !(*this == other);
- }
-
- int minCompatibleVersion;
- int currentVersion;
-};
-
-enum VersionStatus {
- // No way to upgrade the test version to be compatible with current version
- VersionStatus_Incompatible,
-
- // Current version is compatible with test version
- VersionStatus_Compatible,
-
- // Test version must be upgraded to be compatible with current version
- VersionStatus_NeedUpgrade
-};
-
-/**
- * Checks whether or not a particular cluster version is compatible with our current
- * version and mongodb version. The version is compatible if it falls between the
- * MIN_COMPATIBLE_CONFIG_VERSION and CURRENT_CONFIG_VERSION and is not explicitly excluded.
- *
- * @return a VersionStatus enum indicating compatibility
- */
-VersionStatus isConfigVersionCompatible(const VersionType& versionInfo, string* whyNot) {
- string dummy;
- if (!whyNot) {
- whyNot = &dummy;
- }
-
- // Check if we're empty
- if (versionInfo.getCurrentVersion() == UpgradeHistory_EmptyVersion) {
- return VersionStatus_NeedUpgrade;
- }
-
- // Check that we aren't too old
- if (CURRENT_CONFIG_VERSION < versionInfo.getMinCompatibleVersion()) {
- *whyNot = stream() << "the config version " << CURRENT_CONFIG_VERSION
- << " of our process is too old "
- << "for the detected config version "
- << versionInfo.getMinCompatibleVersion();
-
- return VersionStatus_Incompatible;
- }
-
- // versionString is the global version of this process
- if (isInMongoVersionRanges(versionString, versionInfo.getExcludingMongoVersions())) {
- // Cast needed here for MSVC compiler issue
- *whyNot = stream() << "not compatible with current config version, version "
- << reinterpret_cast<const char*>(versionString) << "has been excluded.";
-
- return VersionStatus_Incompatible;
- }
-
- // Check if we need to upgrade
- if (versionInfo.getCurrentVersion() >= CURRENT_CONFIG_VERSION) {
- return VersionStatus_Compatible;
- }
-
- return VersionStatus_NeedUpgrade;
-}
-
-// Checks that all config servers are online
-Status _checkConfigServersAlive(const ConnectionString& configLoc) {
- BSONObj result;
- try {
- ScopedDbConnection conn(configLoc, 30);
- conn->runCommand("admin", BSON("fsync" << 1), result);
- conn.done();
- return getStatusFromCommandResult(result);
- } catch (const DBException& e) {
- return e.toStatus();
- }
-}
-
-} // namespace
-
-
-/**
- * Returns the config version of the cluster pointed at by the connection string.
- *
- * @return OK if version found successfully, error status if something bad happened.
- */
-Status getConfigVersion(CatalogManager* catalogManager, VersionType* versionInfo) {
- try {
- versionInfo->clear();
-
- ScopedDbConnection conn(grid.shardRegistry()->getConfigServerConnectionString(), 30);
-
- unique_ptr<DBClientCursor> cursor(_safeCursor(conn->query("config.version", BSONObj())));
-
- bool hasConfigData = conn->count(ShardType::ConfigNS) ||
- conn->count(DatabaseType::ConfigNS) || conn->count(CollectionType::ConfigNS);
-
- if (!cursor->more()) {
- // Version is 1 if we have data, 0 if we're completely empty
- if (hasConfigData) {
- versionInfo->setMinCompatibleVersion(UpgradeHistory_UnreportedVersion);
- versionInfo->setCurrentVersion(UpgradeHistory_UnreportedVersion);
- } else {
- versionInfo->setMinCompatibleVersion(UpgradeHistory_EmptyVersion);
- versionInfo->setCurrentVersion(UpgradeHistory_EmptyVersion);
- }
-
- conn.done();
- return Status::OK();
- }
-
- BSONObj versionDoc = cursor->next();
- auto versionInfoResult = VersionType::fromBSON(versionDoc);
- if (!versionInfoResult.isOK()) {
- conn.done();
-
- return Status(ErrorCodes::UnsupportedFormat,
- stream() << "invalid config version document " << versionDoc
- << versionInfoResult.getStatus().toString());
- }
- *versionInfo = versionInfoResult.getValue();
-
- if (cursor->more()) {
- conn.done();
-
- return Status(ErrorCodes::RemoteValidationError,
- stream() << "should only have 1 document "
- << "in config.version collection");
- }
- conn.done();
- } catch (const DBException& e) {
- return e.toStatus();
- }
-
- return Status::OK();
-}
-
-Status checkAndInitConfigVersion(OperationContext* txn,
- CatalogManager* catalogManager,
- DistLockManager* distLockManager) {
- VersionType versionInfo;
- Status status = getConfigVersion(catalogManager, &versionInfo);
- if (!status.isOK()) {
- return status;
- }
-
- string errMsg;
- VersionStatus comp = isConfigVersionCompatible(versionInfo, &errMsg);
-
- if (comp == VersionStatus_Incompatible)
- return {ErrorCodes::IncompatibleShardingMetadata, errMsg};
- if (comp == VersionStatus_Compatible)
- return Status::OK();
-
- invariant(comp == VersionStatus_NeedUpgrade);
-
- if (versionInfo.getCurrentVersion() != UpgradeHistory_EmptyVersion) {
- return {ErrorCodes::IncompatibleShardingMetadata,
- stream() << "newer version " << CURRENT_CONFIG_VERSION
- << " of mongo config metadata is required, "
- << "current version is " << versionInfo.getCurrentVersion()};
- }
-
- // Contact the config servers to make sure all are online - otherwise we wait a long time
- // for locks.
- status = _checkConfigServersAlive(grid.shardRegistry()->getConfigServerConnectionString());
- if (!status.isOK()) {
- return status;
- }
-
- //
- // Acquire a lock for the upgrade process.
- //
- // We want to ensure that only a single mongo process is upgrading the config server at a
- // time.
- //
-
- string whyMessage(stream() << "initializing config database to new format v"
- << CURRENT_CONFIG_VERSION);
- auto lockTimeout = stdx::chrono::minutes(20);
- auto scopedDistLock = distLockManager->lock(txn, "configUpgrade", whyMessage, lockTimeout);
- if (!scopedDistLock.isOK()) {
- return scopedDistLock.getStatus();
- }
-
- //
- // Double-check compatibility inside the upgrade lock
- // Another process may have won the lock earlier and done the upgrade for us, check
- // if this is the case.
- //
-
- status = getConfigVersion(catalogManager, &versionInfo);
- if (!status.isOK()) {
- return status;
- }
-
- comp = isConfigVersionCompatible(versionInfo, &errMsg);
-
- if (comp == VersionStatus_Incompatible) {
- return {ErrorCodes::IncompatibleShardingMetadata, errMsg};
- }
- if (comp == VersionStatus_Compatible)
- return Status::OK();
-
- invariant(comp == VersionStatus_NeedUpgrade);
-
- //
- // Run through the upgrade steps necessary to bring our config version to the current
- // version
- //
-
- log() << "initializing config server version to " << CURRENT_CONFIG_VERSION;
-
- status = makeConfigVersionDocument(txn, catalogManager);
- if (!status.isOK())
- return status;
-
- log() << "initialization of config server to v" << CURRENT_CONFIG_VERSION << " successful";
-
- return Status::OK();
-}
-
-} // namespace mongo
diff --git a/src/mongo/s/catalog/legacy/config_upgrade.h b/src/mongo/s/catalog/legacy/config_upgrade.h
deleted file mode 100644
index de4f9afcf6a..00000000000
--- a/src/mongo/s/catalog/legacy/config_upgrade.h
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Copyright (C) 2012 10gen Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License, version 3,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the GNU Affero General Public License in all respects
- * for all of the code used other than as permitted herein. If you modify
- * file(s) with this exception, you may extend this exception to your
- * version of the file(s), but you are not obligated to do so. If you do not
- * wish to do so, delete this exception statement from your version. If you
- * delete this exception statement from all source files in the program,
- * then also delete it in the license file.
- */
-
-#pragma once
-
-#include <string>
-
-namespace mongo {
-
-class CatalogManager;
-class DistLockManager;
-class OperationContext;
-class Status;
-class VersionType;
-
-/**
- * Returns the config version of the cluster pointed at by the connection string.
- *
- * @return OK if version found successfully, error status if something bad happened.
- */
-Status getConfigVersion(CatalogManager* catalogManager, VersionType* versionInfo);
-
-/**
- * Checks the config version and ensures it's the latest version, otherwise tries to update.
- *
- * Returns Status::OK() on success, or an error status indicating the source of failure.
- */
-Status checkAndInitConfigVersion(OperationContext* txn,
- CatalogManager* catalogManager,
- DistLockManager* distLockManager);
-
-} // namespace mongo
diff --git a/src/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.cpp b/src/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.cpp
index d3bcef468d1..cc88467e662 100644
--- a/src/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.cpp
+++ b/src/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.cpp
@@ -28,12 +28,38 @@
#include "mongo/platform/basic.h"
+#include "mongo/db/commands.h"
#include "mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.h"
namespace mongo {
+using executor::RemoteCommandRequest;
+
+using std::string;
+
+const string CatalogManagerReplSetTestFixture::CONFIG_HOST_PORT{"$dummy_config:27017"};
+
CatalogManagerReplSetTestFixture::CatalogManagerReplSetTestFixture() = default;
CatalogManagerReplSetTestFixture::~CatalogManagerReplSetTestFixture() = default;
+void CatalogManagerReplSetTestFixture::expectFindOnConfigSendErrorCode(ErrorCodes::Error code) {
+ onCommand([&, code](const RemoteCommandRequest& request) {
+ ASSERT_EQ(request.target, HostAndPort(CONFIG_HOST_PORT));
+ ASSERT_EQ(request.dbname, "config");
+ BSONObjBuilder responseBuilder;
+ Command::appendCommandStatus(responseBuilder, Status(code, ""));
+ return responseBuilder.obj();
+ });
+}
+
+void CatalogManagerReplSetTestFixture::expectFindOnConfigSendBSONObjVector(
+ std::vector<BSONObj> obj) {
+ onFindCommand([&, obj](const RemoteCommandRequest& request) {
+ ASSERT_EQ(request.target, HostAndPort(CONFIG_HOST_PORT));
+ ASSERT_EQ(request.dbname, "config");
+ return obj;
+ });
+}
+
} // namespace mongo
diff --git a/src/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.h b/src/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.h
index 1b22b8c0e82..620c2ee7473 100644
--- a/src/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.h
+++ b/src/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.h
@@ -28,6 +28,8 @@
#pragma once
+#include <vector>
+
#include "mongo/s/sharding_test_fixture.h"
namespace mongo {
@@ -39,6 +41,11 @@ class CatalogManagerReplSetTestFixture : public ShardingTestFixture {
public:
CatalogManagerReplSetTestFixture();
~CatalogManagerReplSetTestFixture();
-};
+protected:
+ static const std::string CONFIG_HOST_PORT;
+
+ void expectFindOnConfigSendErrorCode(ErrorCodes::Error code);
+ void expectFindOnConfigSendBSONObjVector(std::vector<BSONObj> obj);
+};
} // namespace mongo
diff --git a/src/mongo/s/catalog/type_config_version.cpp b/src/mongo/s/catalog/type_config_version.cpp
index e90b8019e09..92710ca7638 100644
--- a/src/mongo/s/catalog/type_config_version.cpp
+++ b/src/mongo/s/catalog/type_config_version.cpp
@@ -33,7 +33,6 @@
#include "mongo/bson/bsonobjbuilder.h"
#include "mongo/bson/util/bson_extract.h"
#include "mongo/s/catalog/config_server_version.h"
-#include "mongo/s/catalog/legacy/config_upgrade.h"
#include "mongo/util/assert_util.h"
#include "mongo/util/mongoutils/str.h"
diff --git a/src/mongo/s/chunk_manager_tests.cpp b/src/mongo/s/chunk_manager_tests.cpp
new file mode 100644
index 00000000000..de46653617a
--- /dev/null
+++ b/src/mongo/s/chunk_manager_tests.cpp
@@ -0,0 +1,269 @@
+/**
+ * Copyright (C) 2009 10gen Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the GNU Affero General Public License in all respects
+ * for all of the code used other than as permitted herein. If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so. If you do not
+ * wish to do so, delete this exception statement from your version. If you
+ * delete this exception statement from all source files in the program,
+ * then also delete it in the license file.
+ */
+
+#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kDefault
+
+#include "mongo/client/remote_command_targeter_mock.h"
+#include "mongo/client/remote_command_targeter_factory_mock.h"
+#include "mongo/platform/basic.h"
+#include "mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.h"
+#include "mongo/s/catalog/type_chunk.h"
+#include "mongo/s/catalog/type_collection.h"
+#include "mongo/s/catalog/type_shard.h"
+#include "mongo/s/chunk_manager.h"
+#include "mongo/s/write_ops/batched_command_request.h"
+#include "mongo/s/write_ops/batched_command_response.h"
+
+namespace mongo {
+
+using std::unique_ptr;
+using std::set;
+using std::string;
+using std::vector;
+
+using executor::RemoteCommandResponse;
+using executor::RemoteCommandRequest;
+
+namespace {
+
+static int rand(int max = -1) {
+ static unsigned seed = 1337;
+
+#if !defined(_WIN32)
+ int r = rand_r(&seed);
+#else
+ int r = ::rand(); // seed not used in this case
+#endif
+
+ // Modding is bad, but don't really care in this case
+ return max > 0 ? r % max : r;
+}
+
+class ChunkManagerFixture : public CatalogManagerReplSetTestFixture {
+public:
+ void setUp() override {
+ CatalogManagerReplSetTestFixture::setUp();
+ getMessagingPort()->setRemote(HostAndPort("FakeRemoteClient:34567"));
+ configTargeter()->setFindHostReturnValue(configHost);
+ }
+
+protected:
+ const HostAndPort configHost{HostAndPort(CONFIG_HOST_PORT)};
+ static const ShardId _shardId;
+ static const string _collName;
+ static const string _dbName;
+
+ static const int numSplitPoints = 100;
+
+ void genUniqueRandomSplitKeys(const string& keyName, vector<BSONObj>* splitKeys) {
+ std::unordered_set<int> uniquePoints;
+ while (static_cast<int>(uniquePoints.size()) < numSplitPoints) {
+ uniquePoints.insert(rand(numSplitPoints * 10));
+ }
+ for (auto it = uniquePoints.begin(); it != uniquePoints.end(); ++it) {
+ splitKeys->push_back(BSON(keyName << *it));
+ }
+ }
+
+ void expectInsertOnConfigSaveChunkAndReturnOk(std::vector<BSONObj>& chunks) {
+ onCommandWithMetadata([&](const RemoteCommandRequest& request) mutable {
+ ASSERT_EQ(request.target, HostAndPort(CONFIG_HOST_PORT));
+ ASSERT_EQ(request.dbname, "config");
+
+ // Get "inserted" chunk doc from RemoteCommandRequest.
+ BatchedCommandRequest batchedCommandRequest(BatchedCommandRequest::BatchType_Insert);
+ string errmsg;
+ batchedCommandRequest.parseBSON(_dbName, request.cmdObj, &errmsg);
+ vector<BSONObj> docs = batchedCommandRequest.getInsertRequest()->getDocuments();
+ BSONObj chunk = docs.front();
+
+ // Save chunk (mimic "insertion").
+ chunks.push_back(chunk);
+
+ return RemoteCommandResponse(BSON("ok" << 1), BSONObj(), Milliseconds(1));
+ });
+ }
+
+ void expectInsertOnConfigCheckMetadataAndReturnOk(set<int>& minorVersions, OID& epoch) {
+ onCommandWithMetadata([&](const RemoteCommandRequest& request) mutable {
+ ASSERT_EQ(request.target, HostAndPort(CONFIG_HOST_PORT));
+ ASSERT_EQ(request.dbname, "config");
+
+ // Get "inserted" chunk doc from RemoteCommandRequest.
+ BatchedCommandRequest batchedCommandRequest(BatchedCommandRequest::BatchType_Insert);
+ string errmsg;
+ batchedCommandRequest.parseBSON(_dbName, request.cmdObj, &errmsg);
+ vector<BSONObj> docs = batchedCommandRequest.getInsertRequest()->getDocuments();
+ BSONObj chunk = docs.front();
+
+ ChunkVersion version = ChunkVersion::fromBSON(chunk, ChunkType::DEPRECATED_lastmod());
+
+ // Check chunk's major version.
+ ASSERT(version.majorVersion() == 1);
+
+ // Check chunk's minor version is unique.
+ ASSERT(minorVersions.find(version.minorVersion()) == minorVersions.end());
+ minorVersions.insert(version.minorVersion());
+
+ // Check chunk's epoch is consistent.
+ ASSERT(version.epoch().isSet());
+ if (!epoch.isSet()) {
+ epoch = version.epoch();
+ }
+ ASSERT(version.epoch() == epoch);
+
+ // Check chunk's shard id.
+ ASSERT(chunk[ChunkType::shard()].String() == _shardId);
+
+ return RemoteCommandResponse(BSON("ok" << 1), BSONObj(), Milliseconds(1));
+ });
+ }
+};
+
+const ShardId ChunkManagerFixture::_shardId{"shard0000"};
+const string ChunkManagerFixture::_collName{"foo.bar"};
+const string ChunkManagerFixture::_dbName{"foo"};
+
+// Rename the fixture so that our tests have a useful name in the executable
+typedef ChunkManagerFixture ChunkManagerTests;
+
+/**
+ * Tests loading chunks into a ChunkManager with or without an old ChunkManager.
+ */
+TEST_F(ChunkManagerTests, Basic) {
+ string keyName = "_id";
+ vector<BSONObj> splitKeys;
+ genUniqueRandomSplitKeys(keyName, &splitKeys);
+ ShardKeyPattern shardKeyPattern(BSON(keyName << 1));
+
+ std::vector<BSONObj> shards{
+ BSON(ShardType::name() << _shardId << ShardType::host()
+ << ConnectionString(HostAndPort("$hostFooBar:27017")).toString())};
+
+ // Generate and save a set of chunks with metadata using a temporary ChunkManager.
+
+ std::vector<BSONObj> chunks;
+ auto future = launchAsync([&] {
+ ChunkManager manager(_collName, shardKeyPattern, false);
+ auto status = manager.createFirstChunks(operationContext(), _shardId, &splitKeys, NULL);
+ ASSERT_OK(status);
+ });
+
+ // Call the expect() one extra time since numChunks = numSplits + 1.
+ for (int i = 0; i < static_cast<int>(splitKeys.size()) + 1; i++) {
+ expectInsertOnConfigSaveChunkAndReturnOk(chunks);
+ }
+
+ future.timed_get(kFutureTimeout);
+
+ // Test that a *new* ChunkManager correctly loads the chunks with *no prior info*.
+
+ int numChunks = static_cast<int>(chunks.size());
+ BSONObj firstChunk = chunks.back();
+ ChunkVersion version = ChunkVersion::fromBSON(firstChunk, ChunkType::DEPRECATED_lastmod());
+
+ CollectionType collType;
+ collType.setNs(NamespaceString{_collName});
+ collType.setEpoch(version.epoch());
+ collType.setUpdatedAt(jsTime());
+ collType.setKeyPattern(BSON(keyName << 1));
+ collType.setUnique(false);
+ collType.setDropped(false);
+
+ ChunkManager manager(collType);
+ future = launchAsync([&] {
+ manager.loadExistingRanges(operationContext(), nullptr);
+
+ ASSERT_EQ(version.epoch(), manager.getVersion().epoch());
+ ASSERT_EQ(numChunks - 1, manager.getVersion().minorVersion());
+ ASSERT_EQ(numChunks, static_cast<int>(manager.getChunkMap().size()));
+ });
+ expectFindOnConfigSendBSONObjVector(chunks);
+ expectFindOnConfigSendBSONObjVector(shards);
+ future.timed_get(kFutureTimeout);
+
+ // Test that a *new* ChunkManager correctly loads modified chunks *given an old ChunkManager*.
+
+ // Simulate modified chunks collection
+ ChunkVersion laterVersion = ChunkVersion(2, 1, version.epoch());
+ BSONObj oldChunk = chunks.front();
+ BSONObjBuilder newChunk;
+ newChunk.append("_id", oldChunk.getStringField("_id"));
+ newChunk.append("ns", oldChunk.getStringField("ns"));
+ newChunk.append("min", oldChunk.getObjectField("min"));
+ newChunk.append("max", oldChunk.getObjectField("min"));
+ newChunk.append("shard", oldChunk.getStringField("shard"));
+ laterVersion.addToBSON(newChunk, ChunkType::DEPRECATED_lastmod());
+ newChunk.append("lastmodEpoch", oldChunk.getField("lastmodEpoch").OID());
+
+ // Make new manager load chunk diff
+ future = launchAsync([&] {
+ ChunkManager newManager(manager.getns(), manager.getShardKeyPattern(), manager.isUnique());
+ newManager.loadExistingRanges(operationContext(), &manager);
+
+ ASSERT_EQ(numChunks, static_cast<int>(manager.getChunkMap().size()));
+ ASSERT_EQ(laterVersion.toString(), newManager.getVersion().toString());
+ });
+ expectFindOnConfigSendBSONObjVector(std::vector<BSONObj>{chunks.back(), newChunk.obj()});
+
+ std::cout << "done";
+ future.timed_get(kFutureTimeout);
+ std::cout << "completely done";
+}
+
+/**
+ * Tests that chunk metadata is created correctly when using ChunkManager to create chunks for the
+ * first time. Creating chunks on multiple shards is not tested here since there are unresolved
+ * race conditions there and probably should be avoided if at all possible.
+ */
+TEST_F(ChunkManagerTests, FullTest) {
+ string keyName = "_id";
+ vector<BSONObj> splitKeys;
+ genUniqueRandomSplitKeys(keyName, &splitKeys);
+ ShardKeyPattern shardKeyPattern(BSON(keyName << 1));
+
+ auto future = launchAsync([&] {
+ ChunkManager manager(_collName, shardKeyPattern, false);
+ auto status = manager.createFirstChunks(operationContext(), _shardId, &splitKeys, NULL);
+ ASSERT_OK(status);
+ });
+
+ // Check that config server receives chunks with the expected metadata.
+ // Call expectInsertOnConfigCheckMetadataAndReturnOk one extra time since numChunks = numSplits
+ // + 1
+ set<int> minorVersions;
+ OID epoch;
+ for (auto it = splitKeys.begin(); it != splitKeys.end(); ++it) {
+ expectInsertOnConfigCheckMetadataAndReturnOk(minorVersions, epoch);
+ }
+ expectInsertOnConfigCheckMetadataAndReturnOk(minorVersions, epoch);
+ future.timed_get(kFutureTimeout);
+}
+
+} // namespace
+} // namespace mongo