From 4777fb87f36dc4c1f808570490eab01a1c12095f Mon Sep 17 00:00:00 2001 From: Kaloian Manassiev Date: Mon, 13 Apr 2015 15:47:20 -0400 Subject: SERVER-18024 Move collection metadata loading under the catalog manager Cleans up the CollectionType parsing code and moves all the collection metadata retrieval logic under the catalog manager. Also moves type_collection.* under the catalog manager library. --- src/mongo/dbtests/config_upgrade_tests.cpp | 1 - src/mongo/dbtests/merge_chunk_tests.cpp | 9 +- src/mongo/dbtests/sharding.cpp | 22 +- src/mongo/s/SConscript | 39 ++- src/mongo/s/balance.cpp | 79 +++--- src/mongo/s/balance.h | 2 +- src/mongo/s/catalog/SConscript | 4 +- src/mongo/s/catalog/catalog_manager.h | 43 ++- .../s/catalog/legacy/catalog_manager_legacy.cpp | 74 ++++- .../s/catalog/legacy/catalog_manager_legacy.h | 7 + .../s/catalog/legacy/cluster_client_internal.h | 1 - src/mongo/s/catalog/legacy/config_upgrade.cpp | 1 + src/mongo/s/catalog/type_changelog_test.cpp | 20 +- src/mongo/s/catalog/type_chunk_test.cpp | 14 +- src/mongo/s/catalog/type_collection.cpp | 242 +++++++++++++++++ src/mongo/s/catalog/type_collection.h | 133 +++++++++ src/mongo/s/catalog/type_collection_test.cpp | 80 ++++++ src/mongo/s/catalog/type_shard_test.cpp | 12 +- src/mongo/s/chunk.cpp | 14 +- src/mongo/s/chunk_manager.cpp | 44 +-- src/mongo/s/chunk_manager.h | 5 +- src/mongo/s/client/SConscript | 4 +- src/mongo/s/collection_metadata_test.cpp | 188 +++++++------ src/mongo/s/config.cpp | 87 ++---- src/mongo/s/config.h | 5 +- src/mongo/s/d_state.cpp | 9 +- src/mongo/s/grid.cpp | 20 -- src/mongo/s/grid.h | 6 - src/mongo/s/metadata_loader.cpp | 124 ++------- src/mongo/s/metadata_loader.h | 15 +- src/mongo/s/metadata_loader_test.cpp | 290 ++++++++++---------- src/mongo/s/type_collection.cpp | 252 ----------------- src/mongo/s/type_collection.h | 299 --------------------- src/mongo/s/type_collection_test.cpp | 158 ----------- 34 files changed, 1011 insertions(+), 1292 deletions(-) create mode 100644 src/mongo/s/catalog/type_collection.cpp create mode 100644 src/mongo/s/catalog/type_collection.h create mode 100644 src/mongo/s/catalog/type_collection_test.cpp delete mode 100644 src/mongo/s/type_collection.cpp delete mode 100644 src/mongo/s/type_collection.h delete mode 100644 src/mongo/s/type_collection_test.cpp (limited to 'src') diff --git a/src/mongo/dbtests/config_upgrade_tests.cpp b/src/mongo/dbtests/config_upgrade_tests.cpp index 9c097beeace..b500c082dbe 100644 --- a/src/mongo/dbtests/config_upgrade_tests.cpp +++ b/src/mongo/dbtests/config_upgrade_tests.cpp @@ -33,7 +33,6 @@ #include "mongo/s/catalog/type_shard.h" #include "mongo/s/chunk_version.h" #include "mongo/s/type_mongos.h" -#include "mongo/s/type_collection.h" #include "mongo/s/type_settings.h" #include "mongo/s/type_config_version.h" #include "mongo/unittest/unittest.h" diff --git a/src/mongo/dbtests/merge_chunk_tests.cpp b/src/mongo/dbtests/merge_chunk_tests.cpp index 99334c786bf..2a80126c03d 100644 --- a/src/mongo/dbtests/merge_chunk_tests.cpp +++ b/src/mongo/dbtests/merge_chunk_tests.cpp @@ -29,12 +29,12 @@ #include "mongo/db/range_arithmetic.h" #include "mongo/dbtests/config_server_fixture.h" #include "mongo/s/catalog/type_chunk.h" +#include "mongo/s/catalog/type_collection.h" #include "mongo/s/chunk.h" // for genID #include "mongo/s/chunk_version.h" #include "mongo/s/collection_metadata.h" #include "mongo/s/d_state.h" #include "mongo/s/d_merge.h" -#include "mongo/s/type_collection.h" #include "mongo/unittest/unittest.h" namespace mongo { @@ -64,17 +64,16 @@ namespace mongo { ASSERT_GREATER_THAN( ranges.size(), 0u ); CollectionType coll; - coll.setNS( nss.ns() ); + coll.setNs( nss.ns() ); coll.setKeyPattern( ranges.begin()->keyPattern ); coll.setEpoch( startVersion.epoch() ); coll.setUpdatedAt( 1ULL ); - string errMsg; - ASSERT( coll.isValid( &errMsg ) ); + ASSERT_OK(coll.validate()); DBDirectClient client(&_txn); client.update( CollectionType::ConfigNS, - BSON( CollectionType::ns( coll.getNS() ) ), + BSON( CollectionType::fullNs( coll.getNs() ) ), coll.toBSON(), true, false ); ChunkVersion nextVersion = startVersion; diff --git a/src/mongo/dbtests/sharding.cpp b/src/mongo/dbtests/sharding.cpp index 1d68fb9ce20..632471629ee 100644 --- a/src/mongo/dbtests/sharding.cpp +++ b/src/mongo/dbtests/sharding.cpp @@ -39,12 +39,12 @@ #include "mongo/dbtests/config_server_fixture.h" #include "mongo/dbtests/dbtests.h" #include "mongo/s/catalog/type_chunk.h" +#include "mongo/s/catalog/type_collection.h" #include "mongo/s/catalog/type_shard.h" #include "mongo/s/chunk_diff.h" #include "mongo/s/chunk_manager.h" #include "mongo/s/chunk_version.h" #include "mongo/s/config.h" -#include "mongo/s/type_collection.h" #include "mongo/util/log.h" namespace ShardingTests { @@ -259,17 +259,15 @@ namespace ShardingTests { ChunkType::DEPRECATED_lastmod()); // Make manager load existing chunks - BSONObjBuilder collDocBuilder; - collDocBuilder << CollectionType::ns(collName()); - collDocBuilder << CollectionType::keyPattern(BSON( "_id" << 1 )); - collDocBuilder << CollectionType::unique(false); - collDocBuilder << CollectionType::dropped(false); - collDocBuilder << CollectionType::DEPRECATED_lastmod(jsTime()); - collDocBuilder << CollectionType::DEPRECATED_lastmodEpoch(version.epoch()); - - BSONObj collDoc(collDocBuilder.done()); - - ChunkManager manager(collDoc); + CollectionType collType; + collType.setNs(collName()); + collType.setEpoch(version.epoch()); + collType.setUpdatedAt(jsTime()); + collType.setKeyPattern(BSON("_id" << 1)); + collType.setUnique(false); + collType.setDropped(false); + + ChunkManager manager(collType); manager.loadExistingRanges(shard().getConnString(), NULL); ASSERT(manager.getVersion().epoch() == version.epoch()); diff --git a/src/mongo/s/SConscript b/src/mongo/s/SConscript index ad2193212ad..06018a72e22 100644 --- a/src/mongo/s/SConscript +++ b/src/mongo/s/SConscript @@ -7,7 +7,6 @@ Import("env") # env.Library('base', ['mongo_version_range.cpp', - 'type_collection.cpp', 'type_database.cpp', 'type_locks.cpp', 'type_lockpings.cpp', @@ -27,10 +26,6 @@ env.CppUnitTest('mongo_version_range_test', 'mongo_version_range_test.cpp', '$BUILD_DIR/mongo/bson', '$BUILD_DIR/mongo/db/common']) -env.CppUnitTest('type_collection_test', 'type_collection_test.cpp', - LIBDEPS=['base', - '$BUILD_DIR/mongo/db/common']) - env.CppUnitTest('type_config_version_test', 'type_config_version_test.cpp', LIBDEPS=['base', '$BUILD_DIR/mongo/db/common']) @@ -79,22 +74,24 @@ env.Library( ] ) -env.CppUnitTest('chunk_diff_test', - 'chunk_diff_test.cpp', - LIBDEPS=['metadata', - '$BUILD_DIR/mongo/db/common']) - -env.CppUnitTest('collection_metadata_test', - 'collection_metadata_test.cpp', - LIBDEPS=['metadata', - '$BUILD_DIR/mongo/mocklib', - '$BUILD_DIR/mongo/db/common']) - -env.CppUnitTest('metadata_loader_test', - 'metadata_loader_test.cpp', - LIBDEPS=['metadata', - '$BUILD_DIR/mongo/mocklib', - '$BUILD_DIR/mongo/db/common']) +env.CppUnitTest( + target='metadata_test', + source=[ + 'chunk_diff_test.cpp', + 'metadata_loader_test.cpp', + 'collection_metadata_test.cpp', + ], + LIBDEPS=[ + 'catalog/legacy/catalog_manager_legacy', + 'metadata', + '$BUILD_DIR/mongo/coredb', + '$BUILD_DIR/mongo/coreserver', + '$BUILD_DIR/mongo/coreshard', + '$BUILD_DIR/mongo/mocklib', + '$BUILD_DIR/mongo/mongocommon', + '$BUILD_DIR/mongo/mongoscore', + '$BUILD_DIR/mongo/db/common', + ]) # # Write Operations diff --git a/src/mongo/s/balance.cpp b/src/mongo/s/balance.cpp index d3fe09dade1..f4970f872bd 100644 --- a/src/mongo/s/balance.cpp +++ b/src/mongo/s/balance.cpp @@ -45,6 +45,7 @@ #include "mongo/s/catalog/catalog_manager.h" #include "mongo/s/catalog/type_actionlog.h" #include "mongo/s/catalog/type_chunk.h" +#include "mongo/s/catalog/type_collection.h" #include "mongo/s/chunk_manager.h" #include "mongo/s/config.h" #include "mongo/s/config_server_checker_service.h" @@ -52,7 +53,6 @@ #include "mongo/s/grid.h" #include "mongo/s/server.h" #include "mongo/s/client/shard.h" -#include "mongo/s/type_collection.h" #include "mongo/s/type_mongos.h" #include "mongo/s/type_settings.h" #include "mongo/s/type_tags.h" @@ -292,55 +292,29 @@ namespace mongo { } } - void Balancer::_doBalanceRound( DBClientBase& conn, vector* candidateChunks ) { - verify( candidateChunks ); + void Balancer::_doBalanceRound(DBClientBase& conn, vector* candidateChunks) { + invariant(candidateChunks); - // - // 1. Check whether there is any sharded collection to be balanced by querying - // the ShardsNS::collections collection - // - - auto_ptr cursor = conn.query(CollectionType::ConfigNS, BSONObj()); - - if ( NULL == cursor.get() ) { - warning() << "could not query " << CollectionType::ConfigNS - << " while trying to balance" << endl; + vector collections; + Status collsStatus = grid.catalogManager()->getCollections(nullptr, &collections); + if (!collsStatus.isOK()) { + warning() << "Failed to retrieve the set of collections during balancing round " + << collsStatus; return; } - vector< string > collections; - while ( cursor->more() ) { - BSONObj col = cursor->nextSafe(); - - // sharded collections will have a shard "key". - if ( ! col[CollectionType::keyPattern()].eoo() && - ! col[CollectionType::noBalance()].trueValue() ){ - collections.push_back( col[CollectionType::ns()].String() ); - } - else if( col[CollectionType::noBalance()].trueValue() ){ - LOG(1) << "not balancing collection " << col[CollectionType::ns()].String() - << ", explicitly disabled" << endl; - } - - } - cursor.reset(); - - if ( collections.empty() ) { - LOG(1) << "no collections to balance" << endl; + if (collections.empty()) { + LOG(1) << "no collections to balance"; return; } - // - // 2. Get a list of all the shards that are participating in this balance round - // along with any maximum allowed quotas and current utilization. We get the - // latter by issuing db.serverStatus() (mem.mapped) to all shards. + // Get a list of all the shards that are participating in this balance round along with any + // maximum allowed quotas and current utilization. We get the latter by issuing + // db.serverStatus() (mem.mapped) to all shards. // // TODO: skip unresponsive shards and mark information as stale. - // - ShardInfoMap shardInfo; Status loadStatus = DistributionStatus::populateShardInfoMap(&shardInfo); - if (!loadStatus.isOK()) { warning() << "failed to load shard metadata" << causedBy(loadStatus); return; @@ -353,16 +327,20 @@ namespace mongo { OCCASIONALLY warnOnMultiVersion( shardInfo ); - // - // 3. For each collection, check if the balancing policy recommends moving anything around. - // + // For each collection, check if the balancing policy recommends moving anything around. + for (const auto& coll : collections) { + // Skip collections for which balancing is disabled + const string& ns = coll.getNs(); - for (vector::const_iterator it = collections.begin(); it != collections.end(); ++it ) { - const string& ns = *it; + if (!coll.getAllowBalance()) { + LOG(1) << "Not balancing collection " << ns << "; explicitly disabled."; + continue; + } OwnedPointerMap > shardToChunksMap; - cursor = conn.query(ChunkType::ConfigNS, - QUERY(ChunkType::ns(ns)).sort(ChunkType::min())); + auto_ptr cursor = + conn.query(ChunkType::ConfigNS, + QUERY(ChunkType::ns(ns)).sort(ChunkType::min())); set allChunkMinimums; @@ -375,6 +353,7 @@ namespace mongo { << ": " << chunkRes.getStatus().reason(); return; } + auto_ptr chunk(new ChunkType()); chunkRes.getValue().cloneTo(chunk.get()); @@ -473,13 +452,15 @@ namespace mongo { break; } - if ( didAnySplits ) { + if (didAnySplits) { // state change, just wait till next round continue; } - CandidateChunk* p = _policy->balance( ns, status, _balancedLastTime ); - if ( p ) candidateChunks->push_back( CandidateChunkPtr( p ) ); + CandidateChunk* p = _policy->balance(ns, status, _balancedLastTime); + if (p) { + candidateChunks->push_back(CandidateChunkPtr(p)); + } } } diff --git a/src/mongo/s/balance.h b/src/mongo/s/balance.h index 8161af39293..0bd41054ea2 100644 --- a/src/mongo/s/balance.h +++ b/src/mongo/s/balance.h @@ -93,7 +93,7 @@ namespace mongo { * @param conn is the connection with the config server(s) * @param candidateChunks (IN/OUT) filled with candidate chunks, one per collection, that could possibly be moved */ - void _doBalanceRound( DBClientBase& conn, std::vector* candidateChunks ); + void _doBalanceRound(DBClientBase& conn, std::vector* candidateChunks); /** * Issues chunk migration request, one at a time. diff --git a/src/mongo/s/catalog/SConscript b/src/mongo/s/catalog/SConscript index 0df0ecb300c..c0b21710cef 100644 --- a/src/mongo/s/catalog/SConscript +++ b/src/mongo/s/catalog/SConscript @@ -8,6 +8,7 @@ env.Library( 'type_actionlog.cpp', 'type_changelog.cpp', 'type_chunk.cpp', + 'type_collection.cpp', 'type_shard.cpp' ], LIBDEPS=[ @@ -29,7 +30,7 @@ env.Library( '$BUILD_DIR/mongo/db/common', '$BUILD_DIR/mongo/s/batch_write_types', '$BUILD_DIR/mongo/s/catalog/catalog_types' - ], + ] ) env.CppUnitTest( @@ -37,6 +38,7 @@ env.CppUnitTest( source=[ 'type_changelog_test.cpp', 'type_chunk_test.cpp', + 'type_collection_test.cpp', 'type_shard_test.cpp' ], LIBDEPS=[ diff --git a/src/mongo/s/catalog/catalog_manager.h b/src/mongo/s/catalog/catalog_manager.h index 711f6c4b1fe..291f174ba16 100644 --- a/src/mongo/s/catalog/catalog_manager.h +++ b/src/mongo/s/catalog/catalog_manager.h @@ -42,6 +42,7 @@ namespace mongo { struct BSONArray; class BSONObj; class ChunkType; + class CollectionType; class ConnectionString; class DatabaseType; class OperationContext; @@ -92,7 +93,7 @@ namespace mongo { * @param fieldsAndOrder: shardKey pattern * @param unique: if true, ensure underlying index enforces a unique constraint. * @param initPoints: create chunks based on a set of specified split points. - * @param initShards: if NULL, use primary shard as lone shard for DB. + * @param initShards: if nullptr, use primary shard as lone shard for DB. * * WARNING: It's not completely safe to place initial chunks onto non-primary * shards using this method because a conflict may result if multiple map-reduce @@ -103,7 +104,7 @@ namespace mongo { const ShardKeyPattern& fieldsAndOrder, bool unique, std::vector* initPoints, - std::vector* initShards = NULL) = 0; + std::vector* initShards = nullptr) = 0; /** * @@ -137,8 +138,8 @@ namespace mongo { * metadata and sets the specified shard as primary. * * @param dbName name of the database (case sensitive) - * @param shard Optional shard to use as primary. If NULL is specified, one will be picked - * by the system. + * @param shard Optional shard to use as primary. If nullptr is specified, one will be + * picked by the system. * * Returns Status::OK on success or any error code indicating the failure. These are some * of the known failures: @@ -149,8 +150,7 @@ namespace mongo { virtual Status createDatabase(const std::string& dbName, const Shard* shard) = 0; /** - * Updates the metadata for a given database. Currently, if the specified DB entry does - * not exist, it will be created. + * Updates or creates the metadata for a given database. */ virtual Status updateDatabase(const std::string& dbName, const DatabaseType& db) = 0; @@ -165,6 +165,34 @@ namespace mongo { */ virtual StatusWith getDatabase(const std::string& dbName) = 0; + /** + * Updates or creates the metadata for a given collection. + */ + virtual Status updateCollection(const std::string& collNs, const CollectionType& coll) = 0; + + /** + * Retrieves the metadata for a given collection, if it exists. + * + * @param collectionNs fully qualified name of the collection (case sensitive) + * + * Returns Status::OK along with the collection information or any error code indicating + * the failure. These are some of the known failures: + * - NamespaceNotFound - collection does not exist + */ + virtual StatusWith getCollection(const std::string& collNs) = 0; + + /** + * Retrieves all collections undera specified database (or in the system). + * + * @param dbName an optional database name. Must be nullptr or non-empty. If nullptr is + * specified, all collections on the system are returned. + * @param collections variable to receive the set of collections. + * + * Returns a !OK status if an error occurs. + */ + virtual Status getCollections(const std::string* dbName, + std::vector* collections) = 0; + /** * Drops the specified collection from the collection metadata store. * @@ -176,6 +204,7 @@ namespace mongo { /** * Retrieves all databases for a shard. + * * Returns a !OK status if an error occurs. */ virtual void getDatabasesForShard(const std::string& shardName, @@ -245,7 +274,7 @@ namespace mongo { * class and externally for writes to the admin/config namespaces. * * @param request Request to be sent to the config server. - * @param response Out parameter to receive the response. Can be NULL. + * @param response Out parameter to receive the response. Can be nullptr. */ virtual void writeConfigServerDirect(const BatchedCommandRequest& request, BatchedCommandResponse* response) = 0; diff --git a/src/mongo/s/catalog/legacy/catalog_manager_legacy.cpp b/src/mongo/s/catalog/legacy/catalog_manager_legacy.cpp index 7f058e6c3e8..bb9cc4007ae 100644 --- a/src/mongo/s/catalog/legacy/catalog_manager_legacy.cpp +++ b/src/mongo/s/catalog/legacy/catalog_manager_legacy.cpp @@ -49,6 +49,7 @@ #include "mongo/s/catalog/type_actionlog.h" #include "mongo/s/catalog/type_changelog.h" #include "mongo/s/catalog/type_chunk.h" +#include "mongo/s/catalog/type_collection.h" #include "mongo/s/catalog/type_shard.h" #include "mongo/s/chunk_manager.h" #include "mongo/s/client/dbclient_multi_command.h" @@ -67,6 +68,7 @@ namespace mongo { + using std::auto_ptr; using std::map; using std::pair; using std::set; @@ -758,8 +760,8 @@ namespace { &response); if (!status.isOK()) { return Status(status.code(), - str::stream() << "database metadata write failed: " - << response.toBSON()); + str::stream() << "database metadata write failed: " + << response.toBSON() << "; status: " << status.toString()); } return Status::OK(); @@ -787,9 +789,75 @@ namespace { stream() << "database " << dbName << " not found"); } + conn.done(); return DatabaseType::fromBSON(dbObj); } + Status CatalogManagerLegacy::updateCollection(const std::string& collNs, + const CollectionType& coll) { + fassert(28634, coll.validate()); + + BatchedCommandResponse response; + Status status = update(CollectionType::ConfigNS, + BSON(CollectionType::fullNs(collNs)), + coll.toBSON(), + true, // upsert + false, // multi + NULL); + if (!status.isOK()) { + return Status(status.code(), + str::stream() << "collection metadata write failed: " + << response.toBSON() << "; status: " << status.toString()); + } + + return Status::OK(); + } + + StatusWith CatalogManagerLegacy::getCollection(const std::string& collNs) { + ScopedDbConnection conn(_configServerConnectionString, 30.0); + + BSONObj collObj = conn->findOne(CollectionType::ConfigNS, + BSON(CollectionType::fullNs(collNs))); + if (collObj.isEmpty()) { + conn.done(); + return Status(ErrorCodes::NamespaceNotFound, + stream() << "collection " << collNs << " not found"); + } + + conn.done(); + return CollectionType::fromBSON(collObj); + } + + Status CatalogManagerLegacy::getCollections(const std::string* dbName, + std::vector* collections) { + collections->empty(); + + BSONObjBuilder b; + if (dbName) { + invariant(!dbName->empty()); + b.appendRegex(CollectionType::fullNs(), + (string)"^" + pcrecpp::RE::QuoteMeta(*dbName) + "\\."); + } + + ScopedDbConnection conn(_configServerConnectionString, 30.0); + + auto_ptr cursor = conn->query(CollectionType::ConfigNS, b.obj()); + while (cursor->more()) { + const BSONObj collObj = cursor->next(); + + auto status = CollectionType::fromBSON(collObj); + if (!status.isOK()) { + conn.done(); + return status.getStatus(); + } + + collections->push_back(status.getValue()); + } + + conn.done(); + return Status::OK(); + } + Status CatalogManagerLegacy::dropCollection(const std::string& collectionNs) { logChange(NULL, "dropCollection.start", collectionNs, BSONObj()); @@ -1024,6 +1092,7 @@ namespace { StatusWith shardRes = ShardType::fromBSON(shardObj); if (!shardRes.isOK()) { + conn.done(); return Status(ErrorCodes::FailedToParse, str::stream() << "Failed to parse chunk BSONObj: " << shardRes.getStatus().reason()); @@ -1058,6 +1127,7 @@ namespace { catch (const DBException& ex) { return ex.toStatus(); } + if (!ok) { string errMsg(str::stream() << "Unable to save chunk ops. Command: " << cmd << ". Result: " << cmdResult); diff --git a/src/mongo/s/catalog/legacy/catalog_manager_legacy.h b/src/mongo/s/catalog/legacy/catalog_manager_legacy.h index e0ef3ae6577..f0ff435e606 100644 --- a/src/mongo/s/catalog/legacy/catalog_manager_legacy.h +++ b/src/mongo/s/catalog/legacy/catalog_manager_legacy.h @@ -72,6 +72,13 @@ namespace mongo { virtual StatusWith getDatabase(const std::string& dbName); + virtual Status updateCollection(const std::string& collNs, const CollectionType& coll); + + virtual StatusWith getCollection(const std::string& collNs); + + virtual Status getCollections(const std::string* dbName, + std::vector* collections); + virtual Status dropCollection(const std::string& collectionNs); virtual void getDatabasesForShard(const std::string& shardName, diff --git a/src/mongo/s/catalog/legacy/cluster_client_internal.h b/src/mongo/s/catalog/legacy/cluster_client_internal.h index 7641f1e5333..216fc4579ef 100644 --- a/src/mongo/s/catalog/legacy/cluster_client_internal.h +++ b/src/mongo/s/catalog/legacy/cluster_client_internal.h @@ -39,7 +39,6 @@ #include "mongo/base/owned_pointer_vector.h" #include "mongo/client/dbclientinterface.h" #include "mongo/s/catalog/type_chunk.h" -#include "mongo/s/type_collection.h" namespace mongo { diff --git a/src/mongo/s/catalog/legacy/config_upgrade.cpp b/src/mongo/s/catalog/legacy/config_upgrade.cpp index 36fc7a13f58..6ce128d2764 100644 --- a/src/mongo/s/catalog/legacy/config_upgrade.cpp +++ b/src/mongo/s/catalog/legacy/config_upgrade.cpp @@ -36,6 +36,7 @@ #include "mongo/client/dbclientcursor.h" #include "mongo/s/catalog/catalog_manager.h" #include "mongo/s/catalog/legacy/cluster_client_internal.h" +#include "mongo/s/catalog/type_collection.h" #include "mongo/s/catalog/type_shard.h" #include "mongo/s/distlock.h" #include "mongo/s/grid.h" diff --git a/src/mongo/s/catalog/type_changelog_test.cpp b/src/mongo/s/catalog/type_changelog_test.cpp index b2d45126acf..5777a3b7e37 100644 --- a/src/mongo/s/catalog/type_changelog_test.cpp +++ b/src/mongo/s/catalog/type_changelog_test.cpp @@ -39,7 +39,7 @@ namespace { using mongo::BSONObj; using mongo::Date_t; - TEST(Validity, Empty) { + TEST(ChangelogType, Empty) { ChangelogType logEntry; BSONObj emptyObj = BSONObj(); string errMsg; @@ -48,7 +48,7 @@ namespace { ASSERT_FALSE(logEntry.isValid(NULL)); } - TEST(Validity, Valid) { + TEST(ChangelogType, Valid) { ChangelogType logEntry; BSONObj obj = BSON(ChangelogType::changeID("host.local-2012-11-21T19:14:10-8") << ChangelogType::server("host.local") << @@ -70,7 +70,7 @@ namespace { ASSERT_EQUALS(logEntry.getDetails(), BSON("dummy" << "info")); } - TEST(Validity, MissingChangeID) { + TEST(ChangelogType, MissingChangeID) { ChangelogType logEntry; BSONObj obj = BSON(ChangelogType::server("host.local") << ChangelogType::clientAddr("192.168.0.189:51128") << @@ -84,7 +84,7 @@ namespace { ASSERT_FALSE(logEntry.isValid(NULL)); } - TEST(Validity, MissingServer) { + TEST(ChangelogType, MissingServer) { ChangelogType logEntry; BSONObj obj = BSON(ChangelogType::changeID("host.local-2012-11-21T19:14:10-8") << ChangelogType::clientAddr("192.168.0.189:51128") << @@ -98,7 +98,7 @@ namespace { ASSERT_FALSE(logEntry.isValid(NULL)); } - TEST(Validity, MissingClientAddr) { + TEST(ChangelogType, MissingClientAddr) { ChangelogType logEntry; BSONObj obj = BSON(ChangelogType::changeID("host.local-2012-11-21T19:14:10-8") << ChangelogType::server("host.local") << @@ -112,7 +112,7 @@ namespace { ASSERT_FALSE(logEntry.isValid(NULL)); } - TEST(Validity, MissingTime) { + TEST(ChangelogType, MissingTime) { ChangelogType logEntry; BSONObj obj = BSON(ChangelogType::changeID("host.local-2012-11-21T19:14:10-8") << ChangelogType::server("host.local") << @@ -126,7 +126,7 @@ namespace { ASSERT_FALSE(logEntry.isValid(NULL)); } - TEST(Validity, MissingWhat) { + TEST(ChangelogType, MissingWhat) { ChangelogType logEntry; BSONObj obj = BSON(ChangelogType::changeID("host.local-2012-11-21T19:14:10-8") << ChangelogType::server("host.local") << @@ -140,7 +140,7 @@ namespace { ASSERT_FALSE(logEntry.isValid(NULL)); } - TEST(Validity, MissingNS) { + TEST(ChangelogType, MissingNS) { ChangelogType logEntry; BSONObj obj = BSON(ChangelogType::changeID("host.local-2012-11-21T19:14:10-8") << ChangelogType::server("host.local") << @@ -154,7 +154,7 @@ namespace { ASSERT_FALSE(logEntry.isValid(NULL)); } - TEST(Validity, MissingDetails) { + TEST(ChangelogType, MissingDetails) { ChangelogType logEntry; BSONObj obj = BSON(ChangelogType::changeID("host.local-2012-11-21T19:14:10-8") << ChangelogType::server("host.local") << @@ -168,7 +168,7 @@ namespace { ASSERT_FALSE(logEntry.isValid(NULL)); } - TEST(Validity, BadType) { + TEST(ChangelogType, BadType) { ChangelogType logEntry; BSONObj obj = BSON(ChangelogType::changeID() << 0); string errMsg; diff --git a/src/mongo/s/catalog/type_chunk_test.cpp b/src/mongo/s/catalog/type_chunk_test.cpp index b415606eded..98c474020a4 100644 --- a/src/mongo/s/catalog/type_chunk_test.cpp +++ b/src/mongo/s/catalog/type_chunk_test.cpp @@ -42,7 +42,7 @@ namespace { using std::string; - TEST(Validity, MissingRequiredFields) { + TEST(ChunkType, MissingRequiredFields) { ChunkType chunk; BSONArray version = BSON_ARRAY(Date_t(1) << OID::gen()); @@ -78,7 +78,7 @@ namespace { ASSERT_FALSE(chunkRes.isOK()); } - TEST(MinMaxValidity, DifferentNumberOfColumns) { + TEST(ChunkType, DifferentNumberOfColumns) { BSONArray version = BSON_ARRAY(Date_t(1) << OID::gen()); BSONObj obj = BSON(ChunkType::name("test.mycol-a_MinKey") << ChunkType::ns("test.mycol") << @@ -91,7 +91,7 @@ namespace { ASSERT_FALSE(chunkRes.getValue().validate().isOK()); } - TEST(MinMaxValidity, DifferentColumns) { + TEST(ChunkType, DifferentColumns) { BSONArray version = BSON_ARRAY(Date_t(1) << OID::gen()); BSONObj obj = BSON(ChunkType::name("test.mycol-a_MinKey") << ChunkType::ns("test.mycol") << @@ -104,7 +104,7 @@ namespace { ASSERT_FALSE(chunkRes.getValue().validate().isOK()); } - TEST(MinMaxValidity, NotAscending) { + TEST(ChunkType, NotAscending) { BSONArray version = BSON_ARRAY(Date_t(1) << OID::gen()); BSONObj obj = BSON(ChunkType::name("test.mycol-a_MinKey") << ChunkType::ns("test.mycol") << @@ -117,7 +117,7 @@ namespace { ASSERT_FALSE(chunkRes.getValue().validate().isOK()); } - TEST(Compatibility, NewFormatVersion) { + TEST(ChunkType, NewFormatVersion) { ChunkType chunk; OID epoch = OID::gen(); BSONArray version = BSON_ARRAY(Date_t(1) << epoch); @@ -142,7 +142,7 @@ namespace { ASSERT_TRUE(chunk.validate().isOK()); } - TEST(Compatibility, OldFormatVersion) { + TEST(ChunkType, OldFormatVersion) { ChunkType chunk; OID epoch = OID::gen(); BSONObj obj = BSON(ChunkType::name("test.mycol-a_MinKey") << @@ -167,7 +167,7 @@ namespace { ASSERT_TRUE(chunk.validate().isOK()); } - TEST(Validity, BadType) { + TEST(ChunkType, BadType) { BSONObj obj = BSON(ChunkType::name() << 0); StatusWith chunkRes = ChunkType::fromBSON(obj); ASSERT_FALSE(chunkRes.isOK()); diff --git a/src/mongo/s/catalog/type_collection.cpp b/src/mongo/s/catalog/type_collection.cpp new file mode 100644 index 00000000000..9656daf9886 --- /dev/null +++ b/src/mongo/s/catalog/type_collection.cpp @@ -0,0 +1,242 @@ +/** + * Copyright (C) 2012 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/s/catalog/type_collection.h" + +#include "mongo/base/status_with.h" +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/bson/util/bson_extract.h" +#include "mongo/db/namespace_string.h" +#include "mongo/util/assert_util.h" + +namespace mongo { + + const std::string CollectionType::ConfigNS = "config.collections"; + + const BSONField CollectionType::fullNs("_id"); + const BSONField CollectionType::epoch("lastmodEpoch"); + const BSONField CollectionType::updatedAt("lastmod"); + const BSONField CollectionType::keyPattern("key"); + const BSONField CollectionType::unique("unique"); + const BSONField CollectionType::noBalance("noBalance"); + const BSONField CollectionType::dropped("dropped"); + + + CollectionType::CollectionType() { + clear(); + } + + StatusWith CollectionType::fromBSON(const BSONObj& source) { + CollectionType coll; + + { + std::string collFullNs; + Status status = bsonExtractStringField(source, fullNs.name(), &collFullNs); + if (!status.isOK()) return status; + + coll._fullNs = collFullNs; + } + + { + OID collEpoch; + Status status = bsonExtractOIDField(source, epoch.name(), &collEpoch); + if (!status.isOK()) return status; + + coll._epoch = collEpoch; + } + + { + BSONElement collUpdatedAt; + Status status = bsonExtractTypedField(source, updatedAt.name(), Date, &collUpdatedAt); + if (!status.isOK()) return status; + + coll._updatedAt = collUpdatedAt.Date(); + } + + { + bool collDropped; + + // Dropped can be missing in which case it is presumed false + Status status = + bsonExtractBooleanFieldWithDefault(source, dropped.name(), false, &collDropped); + if (!status.isOK()) return status; + + coll._dropped = collDropped; + } + + { + BSONElement collKeyPattern; + Status status = + bsonExtractTypedField(source, keyPattern.name(), Object, &collKeyPattern); + if (status.isOK()) { + BSONObj obj = collKeyPattern.Obj(); + if (obj.isEmpty()) { + return Status(ErrorCodes::ShardKeyNotFound, "invalid shard key"); + } + + coll._keyPattern = obj.getOwned(); + } + else if ((status == ErrorCodes::NoSuchKey) && coll.getDropped()) { + // Sharding key can be missing if the collection is dropped + } + else { + return status; + } + } + + { + bool collUnique; + + // Key uniqueness can be missing in which case it is presumed false + Status status = + bsonExtractBooleanFieldWithDefault(source, unique.name(), false, &collUnique); + if (!status.isOK()) return status; + + coll._unique = collUnique; + } + + { + bool collNoBalance; + + // No balance can be missing in which case it is presumed as false + Status status = bsonExtractBooleanFieldWithDefault(source, + noBalance.name(), + false, + &collNoBalance); + if (!status.isOK()) return status; + + coll._allowBalance = !collNoBalance; + } + + return StatusWith(coll); + } + + Status CollectionType::validate() const { + // These fields must always be set + if (!_fullNs.is_initialized() || _fullNs->empty()) { + return Status(ErrorCodes::NoSuchKey, "missing ns"); + } + + const NamespaceString nss(_fullNs.get()); + if (!nss.isValid()) { + return Status(ErrorCodes::BadValue, "invalid namespace " + nss.toString()); + } + + if (!_epoch.is_initialized()) { + return Status(ErrorCodes::NoSuchKey, "missing epoch"); + } + + if (!_updatedAt.is_initialized()) { + return Status(ErrorCodes::NoSuchKey, "missing updated at timestamp"); + } + + if (!_dropped.get_value_or(false)) { + if (!_epoch->isSet()) { + return Status(ErrorCodes::BadValue, "invalid epoch"); + } + + if (!_updatedAt.get()) { + return Status(ErrorCodes::BadValue, "invalid updated at timestamp"); + } + + if (!_keyPattern.is_initialized()) { + return Status(ErrorCodes::NoSuchKey, "missing key pattern"); + } + else { + invariant(!_keyPattern->isEmpty()); + } + } + + return Status::OK(); + } + + BSONObj CollectionType::toBSON() const { + BSONObjBuilder builder; + + builder.append(fullNs.name(), _fullNs.get_value_or("")); + builder.append(epoch.name(), _epoch.get_value_or(OID())); + builder.append(updatedAt.name(), _updatedAt.get_value_or(0)); + + // These fields are optional, so do not include them in the metadata for the purposes of + // consuming less space on the config servers. + + if (_dropped.is_initialized()) { + builder.append(dropped.name(), _dropped.get()); + } + + if (_keyPattern.is_initialized()) { + builder.append(keyPattern.name(), _keyPattern.get()); + } + + if (_unique.is_initialized()) { + builder.append(unique.name(), _unique.get()); + } + + if (_allowBalance.is_initialized()) { + builder.append(noBalance.name(), !_allowBalance.get()); + } + + return builder.obj(); + } + + void CollectionType::clear() { + _fullNs.reset(); + _epoch.reset(); + _updatedAt.reset(); + _keyPattern.reset(); + _unique.reset(); + _allowBalance.reset(); + _dropped.reset(); + } + + std::string CollectionType::toString() const { + return toBSON().toString(); + } + + void CollectionType::setNs(const std::string& fullNs) { + invariant(!fullNs.empty()); + _fullNs = fullNs; + } + + void CollectionType::setEpoch(OID epoch) { + _epoch = epoch; + } + + void CollectionType::setUpdatedAt(Date_t updatedAt) { + _updatedAt = updatedAt; + } + + void CollectionType::setKeyPattern(const BSONObj& keyPattern) { + invariant(!keyPattern.isEmpty()); + _keyPattern = keyPattern; + } + +} // namespace mongo diff --git a/src/mongo/s/catalog/type_collection.h b/src/mongo/s/catalog/type_collection.h new file mode 100644 index 00000000000..5f05733fbba --- /dev/null +++ b/src/mongo/s/catalog/type_collection.h @@ -0,0 +1,133 @@ +/** + * Copyright (C) 2012 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/db/jsobj.h" + +namespace mongo { + + class BSONObj; + class Status; + template class StatusWith; + + + /** + * This class represents the layout and contents of documents contained in the + * config.collections collection. All manipulation of documents coming from that collection + * should be done with this class. + */ + class CollectionType { + public: + // Name of the collections collection in the config server. + static const std::string ConfigNS; + + static const BSONField fullNs; + static const BSONField epoch; + static const BSONField updatedAt; + static const BSONField keyPattern; + static const BSONField unique; + static const BSONField noBalance; + static const BSONField dropped; + + + CollectionType(); + + /** + * Constructs a new DatabaseType object from BSON. Also does validation of the contents. + */ + static StatusWith fromBSON(const BSONObj& source); + + /** + * Returns OK if all fields have been set. Otherwise returns NoSuchKey and information + * about what is the first field which is missing. + */ + Status validate() const; + + /** + * Returns the BSON representation of the entry. + */ + BSONObj toBSON() const; + + /** + * Clears the internal state. + */ + void clear(); + + /** + * Returns a std::string representation of the current internal state. + */ + std::string toString() const; + + const std::string& getNs() const { return _fullNs.get(); } + void setNs(const std::string& fullNs); + + OID getEpoch() const { return _epoch.get(); } + void setEpoch(OID epoch); + + Date_t getUpdatedAt() const { return _updatedAt.get(); } + void setUpdatedAt(Date_t updatedAt); + + bool getDropped() const { return _dropped.get_value_or(false); } + void setDropped(bool dropped) { _dropped = dropped; } + + const BSONObj& getKeyPattern() const { return _keyPattern.get(); } + void setKeyPattern(const BSONObj& keyPattern); + + bool getUnique() const { return _unique.get_value_or(false); } + void setUnique(bool unique) { _unique = unique; } + + bool getAllowBalance() const { return _allowBalance.get_value_or(true); } + + private: + // Required full namespace (with the database prefix). + boost::optional _fullNs; + + // Required to disambiguate collection namespace incarnations. + boost::optional _epoch; + + // Required last updated time. + boost::optional _updatedAt; + + // Optional, whether the collection has been dropped. If missing, implies false. + boost::optional _dropped; + + // Sharding key. Required, if collection is not dropped. + boost::optional _keyPattern; + + // Optional uniqueness of the sharding key. If missing, implies false. + boost::optional _unique; + + // Optional whether balancing is allowed for this collection. If missing, implies true. + boost::optional _allowBalance; + }; + +} // namespace mongo diff --git a/src/mongo/s/catalog/type_collection_test.cpp b/src/mongo/s/catalog/type_collection_test.cpp new file mode 100644 index 00000000000..d8ac849d879 --- /dev/null +++ b/src/mongo/s/catalog/type_collection_test.cpp @@ -0,0 +1,80 @@ +/** + * Copyright (C) 2012 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/bson/oid.h" +#include "mongo/base/status_with.h" +#include "mongo/s/catalog/type_collection.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/time_support.h" + +namespace { + + using namespace mongo; + + + TEST(CollectionType, Empty) { + StatusWith status = CollectionType::fromBSON(BSONObj()); + ASSERT_FALSE(status.isOK()); + } + + TEST(CollectionType, Basic) { + const OID oid = OID::gen(); + StatusWith status = CollectionType::fromBSON( + BSON(CollectionType::fullNs("db.coll") << + CollectionType::epoch(oid) << + CollectionType::updatedAt(1ULL) << + CollectionType::keyPattern(BSON("a" << 1)) << + CollectionType::unique(true))); + ASSERT_TRUE(status.isOK()); + + CollectionType coll = status.getValue(); + ASSERT_TRUE(coll.validate().isOK()); + ASSERT_EQUALS(coll.getNs(), "db.coll"); + ASSERT_EQUALS(coll.getEpoch(), oid); + ASSERT_EQUALS(coll.getUpdatedAt(), 1ULL); + ASSERT_EQUALS(coll.getKeyPattern(), BSON("a" << 1)); + ASSERT_EQUALS(coll.getUnique(), true); + ASSERT_EQUALS(coll.getAllowBalance(), true); + ASSERT_EQUALS(coll.getDropped(), false); + } + + TEST(CollectionType, BadType) { + const OID oid = OID::gen(); + StatusWith status = CollectionType::fromBSON( + BSON(CollectionType::fullNs() << 1 << + CollectionType::epoch(oid) << + CollectionType::updatedAt(1ULL) << + CollectionType::keyPattern(BSON("a" << 1)) << + CollectionType::unique(true))); + + ASSERT_FALSE(status.isOK()); + } + +} // namespace diff --git a/src/mongo/s/catalog/type_shard_test.cpp b/src/mongo/s/catalog/type_shard_test.cpp index ec72a4d5371..75364804b65 100644 --- a/src/mongo/s/catalog/type_shard_test.cpp +++ b/src/mongo/s/catalog/type_shard_test.cpp @@ -40,19 +40,19 @@ namespace { using std::string; - TEST(Validity, MissingName) { + TEST(ShardType, MissingName) { BSONObj obj = BSON(ShardType::host("localhost:27017")); StatusWith shardRes = ShardType::fromBSON(obj); ASSERT_FALSE(shardRes.isOK()); } - TEST(Validity, MissingHost) { + TEST(ShardType, MissingHost) { BSONObj obj = BSON(ShardType::name("shard0000")); StatusWith shardRes = ShardType::fromBSON(obj); ASSERT_FALSE(shardRes.isOK()); } - TEST(Validity, OnlyMandatory) { + TEST(ShardType, OnlyMandatory) { BSONObj obj = BSON(ShardType::name("shard0000") << ShardType::host("localhost:27017")); StatusWith shardRes = ShardType::fromBSON(obj); @@ -61,7 +61,7 @@ namespace { ASSERT(shard.validate().isOK()); } - TEST(Validity, AllOptionalsPresent) { + TEST(ShardType, AllOptionalsPresent) { BSONObj obj = BSON(ShardType::name("shard0000") << ShardType::host("localhost:27017") << ShardType::draining(true) << @@ -72,7 +72,7 @@ namespace { ASSERT(shard.validate().isOK()); } - TEST(Validity, MaxSizeAsFloat) { + TEST(ShardType, MaxSizeAsFloat) { BSONObj obj = BSON(ShardType::name("shard0000") << ShardType::host("localhost:27017") << ShardType::maxSize() << 100.0); @@ -82,7 +82,7 @@ namespace { ASSERT(shard.validate().isOK()); } - TEST(Validity, BadType) { + TEST(ShardType, BadType) { BSONObj obj = BSON(ShardType::name() << 0); StatusWith shardRes = ShardType::fromBSON(obj); ASSERT_FALSE(shardRes.isOK()); diff --git a/src/mongo/s/chunk.cpp b/src/mongo/s/chunk.cpp index 1b07e788af5..dcdd528a548 100644 --- a/src/mongo/s/chunk.cpp +++ b/src/mongo/s/chunk.cpp @@ -45,6 +45,7 @@ #include "mongo/platform/random.h" #include "mongo/s/balancer_policy.h" #include "mongo/s/catalog/catalog_manager.h" +#include "mongo/s/catalog/type_collection.h" #include "mongo/s/chunk_manager.h" #include "mongo/s/config.h" #include "mongo/s/config_server_checker_service.h" @@ -584,8 +585,17 @@ namespace { _dataWritten = 0; } - const bool shouldBalance = grid.getConfigShouldBalance() && - grid.getCollShouldBalance(_manager->getns()); + bool shouldBalance = grid.getConfigShouldBalance(); + if (shouldBalance) { + auto status = grid.catalogManager()->getCollection(_manager->getns()); + if (!status.isOK()) { + log() << "Auto-split for " << _manager->getns() + << " failed to load collection metadata due to " << status.getStatus(); + return false; + } + + shouldBalance = status.getValue().getAllowBalance(); + } log() << "autosplitted " << _manager->getns() << " shard: " << toString() diff --git a/src/mongo/s/chunk_manager.cpp b/src/mongo/s/chunk_manager.cpp index 51034ba0218..68b833fc455 100644 --- a/src/mongo/s/chunk_manager.cpp +++ b/src/mongo/s/chunk_manager.cpp @@ -41,12 +41,12 @@ #include "mongo/db/query/query_planner_common.h" #include "mongo/s/catalog/catalog_cache.h" #include "mongo/s/catalog/catalog_manager.h" +#include "mongo/s/catalog/type_collection.h" #include "mongo/s/chunk_diff.h" #include "mongo/s/client/shard_connection.h" #include "mongo/s/config.h" #include "mongo/s/distlock.h" #include "mongo/s/grid.h" -#include "mongo/s/type_collection.h" #include "mongo/util/log.h" #include "mongo/util/timer.h" @@ -126,31 +126,17 @@ namespace { // } - ChunkManager::ChunkManager( const BSONObj& collDoc ) : - // Need the ns early, to construct the lock - // TODO: Construct lock on demand? Not sure why we need to keep it around - _ns(collDoc[CollectionType::ns()].type() == String ? - collDoc[CollectionType::ns()].String() : - ""), - _keyPattern(collDoc[CollectionType::keyPattern()].type() == Object ? - collDoc[CollectionType::keyPattern()].Obj().getOwned() : - BSONObj()), - _unique(collDoc[CollectionType::unique()].trueValue()), - _chunkRanges(), - // The shard versioning mechanism hinges on keeping track of the number of times we reloaded ChunkManager's. - // Increasing this number here will prompt checkShardVersion() to refresh the connection-level versions to - // the most up to date value. - _sequenceNumber(NextSequenceNumber.addAndFetch(1)) - { - - // - // Sets up a chunk manager from an existing sharded collection document - // - - verify( _ns != "" ); - verify( ! _keyPattern.toBSON().isEmpty() ); - - _version = ChunkVersion::fromBSON( collDoc ); + ChunkManager::ChunkManager(const CollectionType& coll) + : _ns(coll.getNs()), + _keyPattern(coll.getKeyPattern()), + _unique(coll.getUnique()), + _chunkRanges(), + // The shard versioning mechanism hinges on keeping track of the number of times we + // reload ChunkManagers. Increasing this number here will prompt checkShardVersion to + // refresh the connection-level versions to the most up to date value. + _sequenceNumber(NextSequenceNumber.addAndFetch(1)) { + + _version = ChunkVersion::fromBSON(coll.toBSON()); } void ChunkManager::loadExistingRanges( const string& config, const ChunkManager* oldManager ) { @@ -692,12 +678,6 @@ namespace { return _version; } - void ChunkManager::getInfo( BSONObjBuilder& b ) const { - b.append(CollectionType::keyPattern(), _keyPattern.toBSON()); - b.appendBool(CollectionType::unique(), _unique); - _version.addEpochToBSON(b, CollectionType::DEPRECATED_lastmod()); - } - string ChunkManager::toString() const { StringBuilder sb; sb << "ChunkManager: " << _ns << " key:" << _keyPattern.toString() << '\n'; diff --git a/src/mongo/s/chunk_manager.h b/src/mongo/s/chunk_manager.h index 23582488d98..6233ea24c9a 100644 --- a/src/mongo/s/chunk_manager.h +++ b/src/mongo/s/chunk_manager.h @@ -40,6 +40,7 @@ namespace mongo { class CanonicalQuery; class ChunkManager; + class CollectionType; struct QuerySolutionNode; typedef boost::shared_ptr ChunkManagerPtr; @@ -114,7 +115,7 @@ namespace mongo { typedef std::map ShardVersionMap; // Loads a new chunk manager from a collection document - ChunkManager( const BSONObj& collDoc ); + explicit ChunkManager(const CollectionType& coll); // Creates an empty chunk manager for the namespace ChunkManager( const std::string& ns, const ShardKeyPattern& pattern, bool unique ); @@ -203,8 +204,6 @@ namespace mongo { ChunkVersion getVersion(const std::string& shardName) const; ChunkVersion getVersion() const; - void getInfo( BSONObjBuilder& b ) const; - void _printChunks() const; int getCurrentDesiredChunkSize() const; diff --git a/src/mongo/s/client/SConscript b/src/mongo/s/client/SConscript index 414b3071eac..a39ed198084 100644 --- a/src/mongo/s/client/SConscript +++ b/src/mongo/s/client/SConscript @@ -41,8 +41,8 @@ env.CppUnitTest( '$BUILD_DIR/mongo/coredb', '$BUILD_DIR/mongo/coreserver', '$BUILD_DIR/mongo/coreshard', - '$BUILD_DIR/mongo/mongoscore', - '$BUILD_DIR/mongo/mongocommon', '$BUILD_DIR/mongo/mocklib', + '$BUILD_DIR/mongo/mongocommon', + '$BUILD_DIR/mongo/mongoscore', ] ) diff --git a/src/mongo/s/collection_metadata_test.cpp b/src/mongo/s/collection_metadata_test.cpp index 0ba8af618dd..c60e9605058 100644 --- a/src/mongo/s/collection_metadata_test.cpp +++ b/src/mongo/s/collection_metadata_test.cpp @@ -26,6 +26,8 @@ * then also delete it in the license file. */ +#include "mongo/platform/basic.h" + #include #include #include @@ -33,35 +35,20 @@ #include "mongo/db/jsobj.h" #include "mongo/dbtests/mock/mock_conn_registry.h" #include "mongo/dbtests/mock/mock_remote_db_server.h" +#include "mongo/s/catalog/legacy/catalog_manager_legacy.h" #include "mongo/s/catalog/type_chunk.h" +#include "mongo/s/catalog/type_collection.h" #include "mongo/s/chunk_version.h" #include "mongo/s/collection_metadata.h" #include "mongo/s/metadata_loader.h" -#include "mongo/s/type_collection.h" #include "mongo/unittest/unittest.h" #include "mongo/util/net/hostandport.h" namespace { + using namespace mongo; + using boost::scoped_ptr; - using mongo::BSONObj; - using mongo::BSONArray; - using mongo::ChunkType; - using mongo::ConnectionString; - using mongo::CollectionMetadata; - using mongo::CollectionType; - using mongo::Date_t; - using mongo::HostAndPort; - using mongo::KeyRange; - using mongo::MAXKEY; - using mongo::MetadataLoader; - using mongo::MINKEY; - using mongo::OID; - using mongo::ChunkVersion; - using mongo::MockConnRegistry; - using mongo::MockRemoteDBServer; - using mongo::RangeVector; - using mongo::Status; using std::auto_ptr; using std::make_pair; using std::string; @@ -79,13 +66,12 @@ namespace { OID epoch = OID::gen(); CollectionType collType; - collType.setNS( "test.foo" ); + collType.setNs( "test.foo" ); collType.setKeyPattern( BSON("a" << 1) ); collType.setUnique( false ); collType.setUpdatedAt( 1ULL ); collType.setEpoch( epoch ); - string errMsg; - ASSERT( collType.isValid( &errMsg ) ); + ASSERT_OK(collType.validate()); _dummyConfig->insert( CollectionType::ConfigNS, collType.toBSON() ); @@ -98,18 +84,22 @@ namespace { chunkType.setMax( BSON( "a" << MAXKEY ) ); chunkType.setVersion( ChunkVersion( 1, 0, epoch ) ); chunkType.setName( OID::gen().toString() ); - ASSERT(chunkType.validate().isOK()); + ASSERT_OK(collType.validate()); _dummyConfig->insert( ChunkType::ConfigNS, chunkType.toBSON() ); - ConnectionString configLoc = ConnectionString( HostAndPort(CONFIG_HOST_PORT) ); - MetadataLoader loader( configLoc ); - - Status status = loader.makeCollectionMetadata( "test.foo", - "shard0000", - NULL, - &_metadata ); - ASSERT( status.isOK() ); + ConnectionString configLoc = ConnectionString(HostAndPort(CONFIG_HOST_PORT)); + ASSERT(configLoc.isValid()); + CatalogManagerLegacy catalogManager; + catalogManager.init(configLoc); + + MetadataLoader loader(configLoc); + Status status = loader.makeCollectionMetadata(&catalogManager, + "test.foo", + "shard0000", + NULL, + &_metadata); + ASSERT_OK(status); ASSERT_EQUALS( 0u, _metadata.getNumChunks() ); } @@ -483,12 +473,13 @@ namespace { OID epoch = OID::gen(); ChunkVersion chunkVersion = ChunkVersion( 1, 0, epoch ); - BSONObj collFoo = BSON(CollectionType::ns("test.foo") << - CollectionType::keyPattern(BSON("a" << 1)) << - CollectionType::unique(false) << - CollectionType::updatedAt(1ULL) << - CollectionType::epoch(epoch)); - _dummyConfig->insert( CollectionType::ConfigNS, collFoo ); + CollectionType collType; + collType.setNs("test.foo"); + collType.setKeyPattern(BSON("a" << 1)); + collType.setUnique(false); + collType.setUpdatedAt(1ULL); + collType.setEpoch(epoch); + _dummyConfig->insert(CollectionType::ConfigNS, collType.toBSON()); BSONObj fooSingle = BSON(ChunkType::name("test.foo-a_10") << ChunkType::ns("test.foo") << @@ -499,14 +490,18 @@ namespace { ChunkType::shard("shard0000")); _dummyConfig->insert( ChunkType::ConfigNS, fooSingle ); - ConnectionString configLoc( (HostAndPort(CONFIG_HOST_PORT)) ); - MetadataLoader loader( configLoc ); - - Status status = loader.makeCollectionMetadata( "test.foo", - "shard0000", - NULL, - &_metadata ); - ASSERT( status.isOK() ); + ConnectionString configLoc = ConnectionString(HostAndPort(CONFIG_HOST_PORT)); + ASSERT(configLoc.isValid()); + CatalogManagerLegacy catalogManager; + catalogManager.init(configLoc); + + MetadataLoader loader(configLoc); + Status status = loader.makeCollectionMetadata(&catalogManager, + "test.foo", + "shard0000", + NULL, + &_metadata); + ASSERT_OK(status); } void tearDown() { @@ -826,12 +821,13 @@ namespace { OID epoch = OID::gen(); ChunkVersion chunkVersion = ChunkVersion( 1, 0, epoch ); - BSONObj collFoo = BSON(CollectionType::ns("test.foo") << - CollectionType::keyPattern(BSON("a" << 1)) << - CollectionType::unique(false) << - CollectionType::updatedAt(1ULL) << - CollectionType::epoch(epoch)); - _dummyConfig->insert( CollectionType::ConfigNS, collFoo ); + CollectionType collType; + collType.setNs("test.foo"); + collType.setKeyPattern(BSON("a" << 1)); + collType.setUnique(false); + collType.setUpdatedAt(1ULL); + collType.setEpoch(epoch); + _dummyConfig->insert(CollectionType::ConfigNS, collType.toBSON()); BSONObj fooSingle = BSON(ChunkType::name("test.foo-a_MinKey") << ChunkType::ns("test.foo") << @@ -842,14 +838,18 @@ namespace { ChunkType::shard("shard0000")); _dummyConfig->insert( ChunkType::ConfigNS, fooSingle ); - ConnectionString configLoc((HostAndPort(CONFIG_HOST_PORT))); - MetadataLoader loader( configLoc ); - - Status status = loader.makeCollectionMetadata( "test.foo", - "shard0000", - NULL, - &_metadata ); - ASSERT( status.isOK() ); + ConnectionString configLoc = ConnectionString(HostAndPort(CONFIG_HOST_PORT)); + ASSERT(configLoc.isValid()); + CatalogManagerLegacy catalogManager; + catalogManager.init(configLoc); + + MetadataLoader loader(configLoc); + Status status = loader.makeCollectionMetadata(&catalogManager, + "test.foo", + "shard0000", + NULL, + &_metadata); + ASSERT_OK(status); } void tearDown() { @@ -889,12 +889,13 @@ namespace { OID epoch = OID::gen(); ChunkVersion chunkVersion = ChunkVersion( 1, 0, epoch ); - BSONObj collFoo = BSON(CollectionType::ns("test.foo") << - CollectionType::keyPattern(BSON("a" << 1)) << - CollectionType::unique(false) << - CollectionType::updatedAt(1ULL) << - CollectionType::epoch(epoch)); - _dummyConfig->insert( CollectionType::ConfigNS, collFoo ); + CollectionType collType; + collType.setNs("test.foo"); + collType.setKeyPattern(BSON("a" << 1)); + collType.setUnique(false); + collType.setUpdatedAt(1ULL); + collType.setEpoch(epoch); + _dummyConfig->insert(CollectionType::ConfigNS, collType.toBSON()); _dummyConfig->insert( ChunkType::ConfigNS, BSON(ChunkType::name("test.foo-a_10") << ChunkType::ns("test.foo") << @@ -912,14 +913,18 @@ namespace { ChunkType::DEPRECATED_epoch(epoch) << ChunkType::shard("shard0000")) ); - ConnectionString configLoc((HostAndPort(CONFIG_HOST_PORT))); - MetadataLoader loader( configLoc ); - - Status status = loader.makeCollectionMetadata( "test.foo", - "shard0000", - NULL, - &_metadata ); - ASSERT( status.isOK() ); + ConnectionString configLoc = ConnectionString(HostAndPort(CONFIG_HOST_PORT)); + ASSERT(configLoc.isValid()); + CatalogManagerLegacy catalogManager; + catalogManager.init(configLoc); + + MetadataLoader loader(configLoc); + Status status = loader.makeCollectionMetadata(&catalogManager, + "test.foo", + "shard0000", + NULL, + &_metadata); + ASSERT_OK(status); } void tearDown() { @@ -1153,14 +1158,17 @@ namespace { mongo::ConnectionString::setConnectionHook( MockConnRegistry::get()->getConnStrHook() ); MockConnRegistry::get()->addServer( _dummyConfig.get() ); - OID epoch( OID::gen() ); + OID epoch(OID::gen()); - _dummyConfig->insert( CollectionType::ConfigNS, BSON(CollectionType::ns("x.y") << - CollectionType::dropped(false) << - CollectionType::keyPattern(BSON("a" << 1)) << - CollectionType::unique(false) << - CollectionType::updatedAt(1ULL) << - CollectionType::epoch(epoch)) ); + { + CollectionType collType; + collType.setNs("x.y"); + collType.setKeyPattern(BSON("a" << 1)); + collType.setUnique(false); + collType.setUpdatedAt(1ULL); + collType.setEpoch(epoch); + _dummyConfig->insert(CollectionType::ConfigNS, collType.toBSON()); + } { ChunkVersion version( 1, 1, epoch ); @@ -1195,14 +1203,18 @@ namespace { ChunkType::shard("shard0000")) ); } - ConnectionString configLoc((HostAndPort(CONFIG_HOST_PORT))); - MetadataLoader loader( configLoc ); - - Status status = loader.makeCollectionMetadata( "test.foo", - "shard0000", - NULL, - &_metadata ); - ASSERT( status.isOK() ); + ConnectionString configLoc = ConnectionString(HostAndPort(CONFIG_HOST_PORT)); + ASSERT(configLoc.isValid()); + CatalogManagerLegacy catalogManager; + catalogManager.init(configLoc); + + MetadataLoader loader(configLoc); + Status status = loader.makeCollectionMetadata(&catalogManager, + "test.foo", + "shard0000", + NULL, + &_metadata); + ASSERT_OK(status); } void tearDown() { diff --git a/src/mongo/s/config.cpp b/src/mongo/s/config.cpp index 501012e6375..66444be1d10 100644 --- a/src/mongo/s/config.cpp +++ b/src/mongo/s/config.cpp @@ -43,6 +43,7 @@ #include "mongo/db/write_concern.h" #include "mongo/s/catalog/catalog_cache.h" #include "mongo/s/catalog/catalog_manager.h" +#include "mongo/s/catalog/type_collection.h" #include "mongo/s/catalog/type_chunk.h" #include "mongo/s/catalog/type_shard.h" #include "mongo/s/chunk_manager.h" @@ -51,7 +52,6 @@ #include "mongo/s/cluster_write.h" #include "mongo/s/grid.h" #include "mongo/s/server.h" -#include "mongo/s/type_collection.h" #include "mongo/s/type_database.h" #include "mongo/s/type_locks.h" #include "mongo/s/type_lockpings.h" @@ -76,14 +76,10 @@ namespace mongo { Shard Shard::EMPTY; - CollectionInfo::CollectionInfo(const BSONObj& in) { - _dirty = false; - _dropped = in[CollectionType::dropped()].trueValue(); - - if (in[CollectionType::keyPattern()].isABSONObj()) { - shard(new ChunkManager(in)); - } + CollectionInfo::CollectionInfo(const CollectionType& coll) { + _dropped = coll.getDropped(); + shard(new ChunkManager(coll)); _dirty = false; } @@ -133,33 +129,24 @@ namespace mongo { } void CollectionInfo::save(const string& ns) { - BSONObj key = BSON( "_id" << ns ); - - BSONObjBuilder val; - val.append(CollectionType::ns(), ns); - val.appendDate(CollectionType::DEPRECATED_lastmod(), jsTime()); - val.appendBool(CollectionType::dropped(), _dropped); - if ( _cm ) { - // This also appends the lastmodEpoch. - _cm->getInfo( val ); + CollectionType coll; + coll.setNs(ns); + + if (_cm) { + invariant(!_dropped); + coll.setEpoch(_cm->getVersion().epoch()); + coll.setUpdatedAt(_cm->getVersion().toLong()); + coll.setKeyPattern(_cm->getShardKeyPattern().toBSON()); + coll.setUnique(_cm->isUnique()); } else { - // lastmodEpoch is a required field so we also need to do it here. - val.append(CollectionType::DEPRECATED_lastmodEpoch(), ChunkVersion::DROPPED().epoch()); - } - - Status result = grid.catalogManager()->update(CollectionType::ConfigNS, - key, - val.obj(), - true, // upsert - false, // multi - NULL); - if (!result.isOK()) { - uasserted(13473, - str::stream() << "failed to save collection (" << ns << "): " - << result.reason()); + invariant(_dropped); + coll.setDropped(true); + coll.setEpoch(ChunkVersion::DROPPED().epoch()); + coll.setUpdatedAt(jsTime()); } + uassertStatusOK(grid.catalogManager()->updateCollection(ns, coll)); _dirty = false; } @@ -471,43 +458,25 @@ namespace mongo { _shardingEnabled = dbt.getSharded(); // Load all collections - BSONObjBuilder b; - b.appendRegex(CollectionType::ns(), - (string)"^" + pcrecpp::RE::QuoteMeta( _name ) + "\\." ); + vector collections; + uassertStatusOK(grid.catalogManager()->getCollections(&_name, &collections)); int numCollsErased = 0; int numCollsSharded = 0; - ScopedDbConnection conn(configServer.modelServer(), 30.0); - auto_ptr cursor = conn->query(CollectionType::ConfigNS, b.obj()); - verify( cursor.get() ); - while ( cursor->more() ) { - - BSONObj collObj = cursor->next(); - string collName = collObj[CollectionType::ns()].String(); - - if( collObj[CollectionType::dropped()].trueValue() ){ - _collections.erase( collName ); - numCollsErased++; - } - else if( !collObj[CollectionType::primary()].eoo() ){ - // For future compatibility, explicitly ignore any collection with the - // "primary" field set. - - // Erased in case it was previously sharded, dropped, then init'd as unsharded - _collections.erase( collName ); + for (const auto& coll : collections) { + if (coll.getDropped()) { + _collections.erase(coll.getNs()); numCollsErased++; } - else{ - _collections[ collName ] = CollectionInfo( collObj ); - if( _collections[ collName ].isSharded() ) numCollsSharded++; + else { + _collections[coll.getNs()] = CollectionInfo(coll); + numCollsSharded++; } } - LOG(2) << "found " << numCollsErased << " dropped collections and " - << numCollsSharded << " sharded collections for database " << _name << endl; - - conn.done(); + LOG(2) << "found " << numCollsSharded << " collections left and " + << numCollsErased << " collections dropped for database " << _name; return true; } diff --git a/src/mongo/s/config.h b/src/mongo/s/config.h index 9c89e7d93ff..d451354031c 100644 --- a/src/mongo/s/config.h +++ b/src/mongo/s/config.h @@ -38,6 +38,7 @@ namespace mongo { class ChunkManager; + class CollectionType; class ConfigServer; class DatabaseType; class DBConfig; @@ -46,13 +47,15 @@ namespace mongo { extern ConfigServer& configServer; + struct CollectionInfo { + CollectionInfo() { _dirty = false; _dropped = false; } - CollectionInfo(const BSONObj& in); + CollectionInfo(const CollectionType& in); ~CollectionInfo(); bool isSharded() const { diff --git a/src/mongo/s/d_state.cpp b/src/mongo/s/d_state.cpp index f4ada947354..09197770db1 100644 --- a/src/mongo/s/d_state.cpp +++ b/src/mongo/s/d_state.cpp @@ -582,10 +582,11 @@ namespace mongo { Timer refreshTimer; Status status = - mdLoader.makeCollectionMetadata( ns, - getShardName(), - ( fullReload ? NULL : beforeMetadata.get() ), - remoteMetadataRaw ); + mdLoader.makeCollectionMetadata(grid.catalogManager(), + ns, + getShardName(), + fullReload ? NULL : beforeMetadata.get(), + remoteMetadataRaw); long long refreshMillis = refreshTimer.millis(); if ( status.code() == ErrorCodes::NamespaceNotFound ) { diff --git a/src/mongo/s/grid.cpp b/src/mongo/s/grid.cpp index 846b7244902..5cce580cc76 100644 --- a/src/mongo/s/grid.cpp +++ b/src/mongo/s/grid.cpp @@ -38,7 +38,6 @@ #include "mongo/s/catalog/catalog_manager.h" #include "mongo/s/catalog/type_shard.h" #include "mongo/s/config.h" -#include "mongo/s/type_collection.h" #include "mongo/s/type_settings.h" #include "mongo/util/fail_point_service.h" #include "mongo/util/log.h" @@ -149,25 +148,6 @@ namespace mongo { return shouldBalance(balSettings); } - bool Grid::getCollShouldBalance(const std::string& ns) const { - BSONObj collDoc; - ScopedDbConnection conn(configServer.getPrimary().getConnString(), 30); - - try { - collDoc = conn->findOne(CollectionType::ConfigNS, BSON(CollectionType::ns(ns))); - conn.done(); - } - catch (const DBException& e){ - conn.kill(); - warning() << "could not determine whether balancer should be running, error getting" - << "config data from " << conn.getHost() << causedBy(e) << endl; - // if anything goes wrong, we shouldn't try balancing - return false; - } - - return !collDoc[CollectionType::noBalance()].trueValue(); - } - bool Grid::_inBalancingWindow( const BSONObj& balancerDoc , const boost::posix_time::ptime& now ) { // check the 'activeWindow' marker // if present, it is an interval during the day when the balancer should be active diff --git a/src/mongo/s/grid.h b/src/mongo/s/grid.h index 828a59b2c1f..21b4be76fc9 100644 --- a/src/mongo/s/grid.h +++ b/src/mongo/s/grid.h @@ -93,12 +93,6 @@ namespace mongo { */ bool getConfigShouldBalance() const; - /** - * Returns true if the given collection can be balanced based on the config.collections - * document. - */ - bool getCollShouldBalance(const std::string& ns) const; - CatalogManager* catalogManager() const { return _catalogManager.get(); } CatalogCache* catalogCache() const { return _catalogCache.get(); } diff --git a/src/mongo/s/metadata_loader.cpp b/src/mongo/s/metadata_loader.cpp index 123a5eca7d8..58fe43acf68 100644 --- a/src/mongo/s/metadata_loader.cpp +++ b/src/mongo/s/metadata_loader.cpp @@ -35,11 +35,12 @@ #include "mongo/client/connpool.h" #include "mongo/client/dbclientcursor.h" #include "mongo/client/dbclientmockcursor.h" +#include "mongo/s/catalog/catalog_manager.h" #include "mongo/s/catalog/type_chunk.h" +#include "mongo/s/catalog/type_collection.h" #include "mongo/s/chunk_diff.h" #include "mongo/s/chunk_version.h" #include "mongo/s/collection_metadata.h" -#include "mongo/s/type_collection.h" #include "mongo/util/log.h" namespace mongo { @@ -102,112 +103,43 @@ namespace mongo { MetadataLoader::~MetadataLoader() { } - Status MetadataLoader::makeCollectionMetadata( const string& ns, - const string& shard, - const CollectionMetadata* oldMetadata, - CollectionMetadata* metadata ) const + Status MetadataLoader::makeCollectionMetadata(CatalogManager* catalogManager, + const string& ns, + const string& shard, + const CollectionMetadata* oldMetadata, + CollectionMetadata* metadata) const { - Status status = initCollection( ns, shard, metadata ); - if ( !status.isOK() || metadata->getKeyPattern().isEmpty() ) return status; + Status status = _initCollection(catalogManager, ns, shard, metadata); + if (!status.isOK() || metadata->getKeyPattern().isEmpty()) { + return status; + } + return initChunks( ns, shard, oldMetadata, metadata ); } - Status MetadataLoader::initCollection( const string& ns, + Status MetadataLoader::_initCollection(CatalogManager* catalogManager, + const string& ns, const string& shard, - CollectionMetadata* metadata ) const - { - // - // Bring collection entry from the config server. - // - - BSONObj collDoc; - { - try { - ScopedDbConnection conn( _configLoc.toString(), 30 ); - collDoc = conn->findOne( CollectionType::ConfigNS, QUERY(CollectionType::ns()<getCollection(ns); + if (!coll.isOK()) { + return coll.getStatus(); } - if ( collInfo.isDroppedSet() && collInfo.getDropped() ) { - - errMsg = str::stream() << "could not load metadata, collection " << ns - << " was dropped"; - warning() << errMsg << endl; - - return Status( ErrorCodes::NamespaceNotFound, errMsg ); + CollectionType collInfo = coll.getValue(); + if (collInfo.getDropped()) { + return Status(ErrorCodes::NamespaceNotFound, + str::stream() << "could not load metadata, collection " + << ns << " was dropped"); } - if ( collInfo.isKeyPatternSet() && !collInfo.getKeyPattern().isEmpty() ) { - - // Sharded collection, need to load chunks - - metadata->_keyPattern = collInfo.getKeyPattern(); - metadata->fillKeyPatternFields(); - metadata->_shardVersion = ChunkVersion( 0, 0, collInfo.getEpoch() ); - metadata->_collVersion = ChunkVersion( 0, 0, collInfo.getEpoch() ); - - return Status::OK(); - } - else if ( collInfo.isPrimarySet() && collInfo.getPrimary() == shard ) { - - // A collection with a non-default primary - - // Empty primary field not allowed if set - dassert( collInfo.getPrimary() != "" ); - - metadata->_keyPattern = BSONObj(); - metadata->fillKeyPatternFields(); - metadata->_shardVersion = ChunkVersion( 1, 0, collInfo.getEpoch() ); - metadata->_collVersion = metadata->_shardVersion; + metadata->_keyPattern = collInfo.getKeyPattern(); + metadata->fillKeyPatternFields(); + metadata->_shardVersion = ChunkVersion(0, 0, collInfo.getEpoch()); + metadata->_collVersion = ChunkVersion(0, 0, collInfo.getEpoch()); - return Status::OK(); - } - else { - - // A collection with a primary that doesn't match this shard or is empty, the primary - // may have changed before we loaded. - - errMsg = // br - str::stream() << "collection " << ns << " does not have a shard key " - << "and primary " - << ( collInfo.isPrimarySet() ? collInfo.getPrimary() : "" ) - << " does not match this shard " << shard; - - warning() << errMsg << endl; - - metadata->_collVersion = ChunkVersion( 0, 0, OID() ); - - return Status( ErrorCodes::RemoteChangeDetected, errMsg ); - } + return Status::OK(); } Status MetadataLoader::initChunks( const string& ns, diff --git a/src/mongo/s/metadata_loader.h b/src/mongo/s/metadata_loader.h index 7cf97f4c73e..5db5a5bcc32 100644 --- a/src/mongo/s/metadata_loader.h +++ b/src/mongo/s/metadata_loader.h @@ -36,6 +36,7 @@ namespace mongo { + class CatalogManager; class CollectionMetadata; class CollectionType; class DBClientCursor; @@ -94,10 +95,11 @@ namespace mongo { * @return HostUnreachable if there was an error contacting the config servers * @return RemoteChangeDetected if the data loaded was modified by another operation */ - Status makeCollectionMetadata( const std::string& ns, - const std::string& shard, - const CollectionMetadata* oldMetadata, - CollectionMetadata* metadata ) const; + Status makeCollectionMetadata(CatalogManager* catalogManager, + const std::string& ns, + const std::string& shard, + const CollectionMetadata* oldMetadata, + CollectionMetadata* metadata) const; /** * Replaces the pending chunks of the remote metadata with the more up-to-date pending @@ -137,9 +139,10 @@ namespace mongo { * @return RemoteChangeDetected if the collection doc loaded is unexpectedly different * */ - Status initCollection( const std::string& ns, + Status _initCollection(CatalogManager* catalogManager, + const std::string& ns, const std::string& shard, - CollectionMetadata* metadata ) const; + CollectionMetadata* metadata) const; /** * Returns OK and fills in the chunk state of 'metadata' to portray the chunks of the diff --git a/src/mongo/s/metadata_loader_test.cpp b/src/mongo/s/metadata_loader_test.cpp index 9a9902a2320..54ae860cbe5 100644 --- a/src/mongo/s/metadata_loader_test.cpp +++ b/src/mongo/s/metadata_loader_test.cpp @@ -26,6 +26,8 @@ * then also delete it in the license file. */ +#include "mongo/platform/basic.h" + #include #include @@ -36,37 +38,19 @@ #include "mongo/db/jsobj.h" #include "mongo/dbtests/mock/mock_conn_registry.h" #include "mongo/dbtests/mock/mock_remote_db_server.h" +#include "mongo/s/catalog/legacy/catalog_manager_legacy.h" #include "mongo/s/catalog/type_chunk.h" +#include "mongo/s/catalog/type_collection.h" #include "mongo/s/collection_metadata.h" #include "mongo/s/metadata_loader.h" -#include "mongo/s/type_collection.h" #include "mongo/unittest/unittest.h" #include "mongo/util/net/hostandport.h" namespace { + using namespace mongo; + using boost::scoped_ptr; - using mongo::BSONObj; - using mongo::BSONArray; - using mongo::BSONObjBuilder; - using mongo::BSONObjIterator; - using mongo::ChunkType; - using mongo::ChunkVersion; - using mongo::CollectionMetadata; - using mongo::CollectionType; - using mongo::ConnectionString; - using mongo::Date_t; - using mongo::ErrorCodes; - using mongo::HostAndPort; - using mongo::MAXKEY; - using mongo::MINKEY; - using mongo::MetadataLoader; - using mongo::OID; - using mongo::OwnedPointerVector; - using mongo::MockConnRegistry; - using mongo::MockRemoteDBServer; - using mongo::ScopedDbConnection; - using mongo::Status; using std::auto_ptr; using std::string; using std::vector; @@ -78,20 +62,34 @@ namespace { // TODO: Test read of chunks with new epoch // TODO: Test that you can properly load config using format with deprecated fields? - TEST(MetadataLoader, DroppedColl) { + class MetadataLoaderFixture : public mongo::unittest::Test { + public: + void setUp() { + ConnectionString configLoc = ConnectionString(HostAndPort(CONFIG_HOST_PORT)); + ASSERT(configLoc.isValid()); + ASSERT_OK(_catalogManager.init(configLoc)); + } + + protected: + CatalogManager* catalogManager() { return &_catalogManager; } + + private: + CatalogManagerLegacy _catalogManager; + }; + + TEST_F(MetadataLoaderFixture, DroppedColl) { MockRemoteDBServer dummyConfig( CONFIG_HOST_PORT ); mongo::ConnectionString::setConnectionHook( MockConnRegistry::get()->getConnStrHook() ); MockConnRegistry::get()->addServer( &dummyConfig ); CollectionType collInfo; - collInfo.setNS( "test.foo" ); + collInfo.setNs( "test.foo" ); + collInfo.setKeyPattern(BSON("a" << 1)); collInfo.setUpdatedAt( 0 ); collInfo.setEpoch( OID() ); collInfo.setDropped( true ); - - string errMsg; - ASSERT( collInfo.isValid( &errMsg ) ); + ASSERT_OK(collInfo.validate()); dummyConfig.insert( CollectionType::ConfigNS, collInfo.toBSON() ); @@ -99,10 +97,11 @@ namespace { string errmsg; CollectionMetadata metadata; - Status status = loader.makeCollectionMetadata( "test.foo", // br - "shard0000", - NULL, /* no old metadata */ - &metadata ); + Status status = loader.makeCollectionMetadata(catalogManager(), + "test.foo", + "shard0000", + NULL, /* no old metadata */ + &metadata); ASSERT_EQUALS( status.code(), ErrorCodes::NamespaceNotFound ); @@ -110,8 +109,7 @@ namespace { ScopedDbConnection::clearPool(); } - TEST(MetadataLoader, EmptyColl) { - + TEST_F(MetadataLoaderFixture, EmptyColl) { MockRemoteDBServer dummyConfig( CONFIG_HOST_PORT ); mongo::ConnectionString::setConnectionHook( MockConnRegistry::get()->getConnStrHook() ); MockConnRegistry::get()->addServer( &dummyConfig ); @@ -120,10 +118,11 @@ namespace { string errmsg; CollectionMetadata metadata; - Status status = loader.makeCollectionMetadata( "test.foo", // br - "shard0000", - NULL, /* no old metadata */ - &metadata ); + Status status = loader.makeCollectionMetadata(catalogManager(), + "test.foo", + "shard0000", + NULL, /* no old metadata */ + &metadata); ASSERT_EQUALS( status.code(), ErrorCodes::NamespaceNotFound ); @@ -131,43 +130,40 @@ namespace { ScopedDbConnection::clearPool(); } - TEST(MetadataLoader, BadColl) { - + TEST_F(MetadataLoaderFixture, BadColl) { MockRemoteDBServer dummyConfig( CONFIG_HOST_PORT ); mongo::ConnectionString::setConnectionHook( MockConnRegistry::get()->getConnStrHook() ); MockConnRegistry::get()->addServer( &dummyConfig ); - dummyConfig.insert( CollectionType::ConfigNS, BSON( CollectionType::ns("test.foo") ) ); + dummyConfig.insert(CollectionType::ConfigNS, BSON(CollectionType::fullNs("test.foo"))); MetadataLoader loader( CONFIG_LOC ); string errmsg; CollectionMetadata metadata; - Status status = loader.makeCollectionMetadata( "test.foo", // br - "shard0000", - NULL, /* no old metadata */ - &metadata ); + Status status = loader.makeCollectionMetadata(catalogManager(), + "test.foo", + "shard0000", + NULL, /* no old metadata */ + &metadata); - ASSERT_EQUALS( status.code(), ErrorCodes::FailedToParse ); + ASSERT_EQUALS(status.code(), ErrorCodes::NoSuchKey); MockConnRegistry::get()->clear(); ScopedDbConnection::clearPool(); } - TEST(MetadataLoader, BadChunk) { - + TEST_F(MetadataLoaderFixture, BadChunk) { MockRemoteDBServer dummyConfig( CONFIG_HOST_PORT ); mongo::ConnectionString::setConnectionHook( MockConnRegistry::get()->getConnStrHook() ); MockConnRegistry::get()->addServer( &dummyConfig ); CollectionType collInfo; - collInfo.setNS( "test.foo" ); - collInfo.setUpdatedAt( 0 ); + collInfo.setNs( "test.foo" ); + collInfo.setUpdatedAt(1ULL); collInfo.setKeyPattern( BSON("a" << 1) ); collInfo.setEpoch( OID::gen() ); - - string errMsg; - ASSERT( collInfo.isValid( &errMsg ) ); + ASSERT_OK(collInfo.validate()); dummyConfig.insert( CollectionType::ConfigNS, collInfo.toBSON() ); @@ -182,10 +178,11 @@ namespace { string errmsg; CollectionMetadata metadata; - Status status = loader.makeCollectionMetadata( "test.foo", // br - "shard0000", - NULL, /* no old metadata */ - &metadata ); + Status status = loader.makeCollectionMetadata(catalogManager(), + "test.foo", + "shard0000", + NULL, /* no old metadata */ + &metadata); // For now, since the differ doesn't have parsing errors, we get this kind of status // NamespaceNotFound since we aren't refreshing off known metadata @@ -196,21 +193,24 @@ namespace { ScopedDbConnection::clearPool(); } - class NoChunkFixture : public mongo::unittest::Test { + class NoChunkFixture : public MetadataLoaderFixture { protected: void setUp() { + MetadataLoaderFixture::setUp(); + _dummyConfig.reset( new MockRemoteDBServer( CONFIG_HOST_PORT ) ); mongo::ConnectionString::setConnectionHook( MockConnRegistry::get()->getConnStrHook() ); MockConnRegistry::get()->addServer( _dummyConfig.get() ); OID epoch = OID::gen(); - BSONObj collFoo = BSON(CollectionType::ns("test.foo") << - CollectionType::keyPattern(BSON("a" << 1)) << - CollectionType::unique(false) << - CollectionType::updatedAt(1ULL) << - CollectionType::epoch(epoch)); - _dummyConfig->insert( CollectionType::ConfigNS, collFoo ); + CollectionType collType; + collType.setNs("test.foo"); + collType.setKeyPattern(BSON("a" << 1)); + collType.setUnique(false); + collType.setUpdatedAt(1ULL); + collType.setEpoch(epoch); + _dummyConfig->insert(CollectionType::ConfigNS, collType.toBSON()); } void tearDown() { @@ -223,22 +223,24 @@ namespace { }; TEST_F(NoChunkFixture, NoChunksIsDropped) { - MetadataLoader loader( CONFIG_LOC ); CollectionMetadata metadata; - Status status = loader.makeCollectionMetadata( "test.foo", // br - "shard0000", - NULL, /* no old metadata */ - &metadata ); + Status status = loader.makeCollectionMetadata(catalogManager(), + "test.foo", + "shard0000", + NULL, /* no old metadata */ + &metadata); // This is interpreted as a dropped ns, since we drop the chunks first ASSERT_EQUALS( status.code(), ErrorCodes::NamespaceNotFound ); } - class NoChunkHereFixture : public mongo::unittest::Test { + class NoChunkHereFixture : public MetadataLoaderFixture { protected: void setUp() { + MetadataLoaderFixture::setUp(); + _dummyConfig.reset( new MockRemoteDBServer( CONFIG_HOST_PORT ) ); mongo::ConnectionString::setConnectionHook( MockConnRegistry::get()->getConnStrHook() ); MockConnRegistry::get()->addServer( _dummyConfig.get() ); @@ -246,13 +248,12 @@ namespace { OID epoch = OID::gen(); CollectionType collType; - collType.setNS( "test.foo" ); + collType.setNs( "test.foo" ); collType.setKeyPattern( BSON("a" << 1) ); collType.setUnique( false ); collType.setUpdatedAt( 1ULL ); collType.setEpoch( epoch ); - string errMsg; - ASSERT( collType.isValid( &errMsg ) ); + ASSERT_OK(collType.validate()); _dummyConfig->insert( CollectionType::ConfigNS, collType.toBSON() ); @@ -284,15 +285,14 @@ namespace { }; TEST_F(NoChunkHereFixture, CheckNumChunk) { - ConnectionString confServerStr((HostAndPort(CONFIG_HOST_PORT))); - ConnectionString configLoc( confServerStr ); - MetadataLoader loader( configLoc ); + MetadataLoader loader(CONFIG_LOC); CollectionMetadata metadata; - Status status = loader.makeCollectionMetadata( "test.foo", // br - "shard0000", - NULL, /* no old metadata */ - &metadata ); + Status status = loader.makeCollectionMetadata(catalogManager(), + "test.foo", + "shard0000", + NULL, /* no old metadata */ + &metadata); ASSERT( status.isOK() ); ASSERT_EQUALS( 0U, metadata.getNumChunks() ); @@ -303,14 +303,14 @@ namespace { } TEST_F(NoChunkHereFixture, BadChunkNotDropped) { - - MetadataLoader loader( CONFIG_LOC ); + MetadataLoader loader(CONFIG_LOC); CollectionMetadata metadata; - Status status = loader.makeCollectionMetadata( "test.foo", // br - "shard0000", - NULL, /* no old metadata */ - &metadata ); + Status status = loader.makeCollectionMetadata(catalogManager(), + "test.foo", + "shard0000", + NULL, /* no old metadata */ + &metadata); ASSERT( status.isOK() ); @@ -324,10 +324,11 @@ namespace { getDummyConfig()->insert( ChunkType::ConfigNS, chunkInfo.toBSON() ); CollectionMetadata nextMetadata; - status = loader.makeCollectionMetadata( "test.foo", // br - "shard0000", - &metadata, /* using old metadata */ - &nextMetadata ); + status = loader.makeCollectionMetadata(catalogManager(), + "test.foo", + "shard0000", + &metadata, /* using old metadata */ + &nextMetadata); // Remote change error, since there's not an epoch change and we reloaded no chunks ASSERT_EQUALS( status.code(), ErrorCodes::RemoteChangeDetected ); @@ -336,9 +337,11 @@ namespace { ScopedDbConnection::clearPool(); } - class ConfigServerFixture : public mongo::unittest::Test { + class ConfigServerFixture : public MetadataLoaderFixture { protected: void setUp() { + MetadataLoaderFixture::setUp(); + _dummyConfig.reset( new MockRemoteDBServer( CONFIG_HOST_PORT ) ); mongo::ConnectionString::setConnectionHook( MockConnRegistry::get()->getConnStrHook() ); MockConnRegistry::get()->addServer( _dummyConfig.get() ); @@ -346,12 +349,13 @@ namespace { OID epoch = OID::gen(); _maxCollVersion = ChunkVersion( 1, 0, epoch ); - BSONObj collFoo = BSON(CollectionType::ns("test.foo") << - CollectionType::keyPattern(BSON("a" << 1)) << - CollectionType::unique(false) << - CollectionType::updatedAt(1ULL) << - CollectionType::epoch(epoch)); - _dummyConfig->insert( CollectionType::ConfigNS, collFoo ); + CollectionType collType; + collType.setNs("test.foo"); + collType.setKeyPattern(BSON("a" << 1)); + collType.setUnique(false); + collType.setUpdatedAt(1ULL); + collType.setEpoch(epoch); + _dummyConfig->insert(CollectionType::ConfigNS, collType.toBSON()); BSONObj fooSingle = BSON(ChunkType::name("test.foo-a_MinKey") << ChunkType::ns("test.foo") << @@ -385,15 +389,13 @@ namespace { }; TEST_F(ConfigServerFixture, SingleChunkCheckNumChunk) { - // Load from mock server. - ConnectionString confServerStr((HostAndPort(CONFIG_HOST_PORT))); - ConnectionString configLoc( confServerStr ); - MetadataLoader loader( configLoc ); + MetadataLoader loader(CONFIG_LOC); CollectionMetadata metadata; - Status status = loader.makeCollectionMetadata( "test.foo", // br - "shard0000", - NULL, /* no old metadata */ - &metadata ); + Status status = loader.makeCollectionMetadata(catalogManager(), + "test.foo", + "shard0000", + NULL, /* no old metadata */ + &metadata); ASSERT( status.isOK() ); ASSERT_EQUALS( 1U, metadata.getNumChunks() ); } @@ -403,40 +405,46 @@ namespace { ConnectionString configLoc( confServerStr ); MetadataLoader loader( configLoc ); CollectionMetadata metadata; - loader.makeCollectionMetadata( "test.foo", "shard0000", NULL, /* no old metadata */ - &metadata ); + loader.makeCollectionMetadata(catalogManager(), + "test.foo", + "shard0000", + NULL, /* no old metadata */ + &metadata); ChunkType chunkInfo; ASSERT_TRUE(metadata.getNextChunk(metadata.getMinKey(), &chunkInfo)); } TEST_F(ConfigServerFixture, SingleChunkGetShardKey) { - ConnectionString confServerStr((HostAndPort(CONFIG_HOST_PORT))); - ConnectionString configLoc( confServerStr ); - MetadataLoader loader( configLoc ); + MetadataLoader loader(CONFIG_LOC); CollectionMetadata metadata; - loader.makeCollectionMetadata( "test.foo", "shard0000", NULL, /* no old metadata */ - &metadata ); + loader.makeCollectionMetadata(catalogManager(), + "test.foo", + "shard0000", + NULL, /* no old metadata */ + &metadata); ASSERT_TRUE( metadata.getKeyPattern().equal(BSON("a" << 1)) ); } TEST_F(ConfigServerFixture, SingleChunkGetMaxCollVersion) { - ConnectionString confServerStr((HostAndPort(CONFIG_HOST_PORT))); - ConnectionString configLoc( confServerStr ); - MetadataLoader loader( configLoc ); + MetadataLoader loader(CONFIG_LOC); CollectionMetadata metadata; - loader.makeCollectionMetadata( "test.foo", "shard0000", NULL, /* no old metadata */ - &metadata ); + loader.makeCollectionMetadata(catalogManager(), + "test.foo", + "shard0000", + NULL, /* no old metadata */ + &metadata); ASSERT_TRUE( getMaxCollVersion().equals( metadata.getCollVersion() ) ); } TEST_F(ConfigServerFixture, SingleChunkGetMaxShardVersion) { - ConnectionString confServerStr((HostAndPort(CONFIG_HOST_PORT))); - ConnectionString configLoc( confServerStr ); - MetadataLoader loader( configLoc ); + MetadataLoader loader(CONFIG_LOC); CollectionMetadata metadata; - loader.makeCollectionMetadata( "test.foo", "shard0000", NULL, /* no old metadata */ - &metadata ); + loader.makeCollectionMetadata(catalogManager(), + "test.foo", + "shard0000", + NULL, /* no old metadata */ + &metadata); ASSERT_TRUE( getMaxShardVersion().equals( metadata.getShardVersion() ) ); } @@ -444,22 +452,23 @@ namespace { TEST_F(ConfigServerFixture, NoChunks) { getConfigServer()->remove( ChunkType::ConfigNS, BSONObj() ); - ConnectionString confServerStr((HostAndPort(CONFIG_HOST_PORT))); - ConnectionString configLoc( confServerStr ); - MetadataLoader loader( configLoc ); + MetadataLoader loader(CONFIG_LOC); CollectionMetadata metadata; - Status status = loader.makeCollectionMetadata( "test.foo", // br - "shard0000", - NULL, /* no old metadata */ - &metadata ); + Status status = loader.makeCollectionMetadata(catalogManager(), + "test.foo", + "shard0000", + NULL, /* no old metadata */ + &metadata); // NSNotFound because we're reloading with no old metadata ASSERT_EQUALS( status.code(), ErrorCodes::NamespaceNotFound ); } - class MultipleMetadataFixture : public mongo::unittest::Test { + class MultipleMetadataFixture : public MetadataLoaderFixture { protected: void setUp() { + MetadataLoaderFixture::setUp(); + _dummyConfig.reset( new MockRemoteDBServer( CONFIG_HOST_PORT ) ); mongo::ConnectionString::setConnectionHook( MockConnRegistry::get()->getConnStrHook() ); MockConnRegistry::get()->addServer( _dummyConfig.get() ); @@ -492,13 +501,12 @@ namespace { _dummyConfig->remove( ChunkType::ConfigNS, BSONObj() ); CollectionType coll; - coll.setNS( ns ); + coll.setNs( ns ); coll.setKeyPattern( BSON( "a" << 1 ) ); coll.setUpdatedAt( 1ULL ); coll.setEpoch( epoch ); + ASSERT_OK(coll.validate()); - string errMsg; - ASSERT( coll.isValid( &errMsg ) ); _dummyConfig->insert( CollectionType::ConfigNS, coll.toBSON() ); ChunkVersion version( 1, 0, epoch ); @@ -520,8 +528,12 @@ namespace { _dummyConfig->insert( ChunkType::ConfigNS, chunk.toBSON() ); } - Status status = loader().makeCollectionMetadata( ns, shardName, NULL, metadata ); - ASSERT( status.isOK() ); + Status status = loader().makeCollectionMetadata(catalogManager(), + ns, + shardName, + NULL, + metadata); + ASSERT(status.isOK()); } void tearDown() { @@ -534,7 +546,6 @@ namespace { }; TEST_F(MultipleMetadataFixture, PromotePendingNA) { - auto_ptr chunk( new ChunkType() ); chunk->setMin( BSON( "x" << MINKEY ) ); chunk->setMax( BSON( "x" << 0 ) ); @@ -569,7 +580,6 @@ namespace { } TEST_F(MultipleMetadataFixture, PromotePendingNAVersion) { - OID epoch = OID::gen(); auto_ptr chunk( new ChunkType() ); chunk->setMin( BSON( "x" << MINKEY ) ); @@ -606,7 +616,6 @@ namespace { } TEST_F(MultipleMetadataFixture, PromotePendingGoodOverlap) { - OID epoch = OID::gen(); // @@ -683,7 +692,6 @@ namespace { } TEST_F(MultipleMetadataFixture, PromotePendingBadOverlap) { - OID epoch = OID::gen(); // @@ -740,7 +748,7 @@ namespace { ConnectionString configLoc( confServerStr ); MetadataLoader loader( configLoc ); CollectionMetadata metadata; - Status status = loader.makeCollectionMetadata( "not.sharded", // br + Status status = loader.makeCollectionMetadata( "not.sharded", "shard0000", NULL, /* no old metadata */ &metadata ); @@ -779,7 +787,7 @@ namespace { ConnectionString confServerStr(HostAndPort(CONFIG_HOST_PORT)); ConnectionString configLoc( confServerStr ); MetadataLoader loader( configLoc ); - Status status = loader.makeCollectionMetadata( "not.sharded", // br + Status status = loader.makeCollectionMetadata( "not.sharded", "shard0000", NULL, /* no old metadata */ &_oldMetadata ); @@ -876,7 +884,7 @@ namespace { ConnectionString configLoc( confServerStr ); MetadataLoader loader( configLoc ); CollectionMetadata metadata; - Status status = loader.makeCollectionMetadata( "not.sharded", // br + Status status = loader.makeCollectionMetadata( "not.sharded", "shard0000", NULL, /* no old metadata */ &metadata ); diff --git a/src/mongo/s/type_collection.cpp b/src/mongo/s/type_collection.cpp deleted file mode 100644 index 7665b36b9be..00000000000 --- a/src/mongo/s/type_collection.cpp +++ /dev/null @@ -1,252 +0,0 @@ -/** - * Copyright (C) 2012 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ -#include "mongo/s/type_collection.h" - -#include "mongo/db/field_parser.h" -#include "mongo/util/mongoutils/str.h" - -namespace mongo { - - using std::string; - - using mongoutils::str::stream; - - const std::string CollectionType::ConfigNS = "config.collections"; - - const BSONField CollectionType::ns("_id"); - const BSONField CollectionType::primary("primary"); - const BSONField CollectionType::keyPattern("key"); - const BSONField CollectionType::unique("unique"); - const BSONField CollectionType::updatedAt("updatedAt"); - const BSONField CollectionType::noBalance("noBalance"); - const BSONField CollectionType::epoch("epoch"); - const BSONField CollectionType::dropped("dropped"); - const BSONField CollectionType::DEPRECATED_lastmodEpoch("lastmodEpoch"); - const BSONField CollectionType::DEPRECATED_lastmod("lastmod"); - - CollectionType::CollectionType() { - clear(); - } - - CollectionType::~CollectionType() { - } - - bool CollectionType::isValid(std::string* errMsg) const { - std::string dummy; - if (errMsg == NULL) { - errMsg = &dummy; - } - - // All the mandatory fields must be present. - if (!_isNsSet) { - *errMsg = stream() << "missing " << ns.name() << " field"; - return false; - } - if (!_isUpdatedAtSet) { - *errMsg = stream() << "missing " << updatedAt.name() << " field"; - return false; - } - if (!_isEpochSet) { - *errMsg = stream() << "missing " << epoch.name() << " field"; - return false; - } - - // Either sharding or primary information or dropped should be filled. - int numSet = 0; - if (_isPrimarySet && !_primary.empty()) numSet++; - if (_isKeyPatternSet && !(_keyPattern.nFields() == 0)) numSet++; - if (_isDroppedSet && _dropped) numSet++; - if (numSet != 1) { - *errMsg = stream() << "one of " << primary.name() << " or " << keyPattern.name() - << " or " << dropped.name() << " should be filled"; - return false; - } - - // Sharding related fields may only be set if the sharding key pattern is present, unless - // we're dropped. - if ( ( _unique || _noBalance ) && ( !_isDroppedSet || !_dropped ) - && ( _keyPattern.nFields() == 0 ) ) - { - *errMsg = stream() << "missing " << keyPattern.name() << " field"; - return false; - } - - return true; - } - - BSONObj CollectionType::toBSON() const { - BSONObjBuilder builder; - - if (_isNsSet) builder.append(ns(), _ns); - if (_isPrimarySet) builder.append(primary(), _primary); - if (_isKeyPatternSet) builder.append(keyPattern(), _keyPattern); - if (_isUniqueSet) builder.append(unique(), _unique); - if (_isUpdatedAtSet) builder.append(updatedAt(), _updatedAt); - if (_isNoBalanceSet) builder.append(noBalance(), _noBalance); - - if (_isUpdatedAtSet) builder.append(DEPRECATED_lastmod(), _updatedAt); - if (_isEpochSet) { - builder.append(epoch(), _epoch); - builder.append(DEPRECATED_lastmodEpoch(), _epoch); - } - - // Always need to write dropped for compatibility w/ 2.0/2.2 - builder.append(dropped(), _dropped); - - return builder.obj(); - } - - bool CollectionType::parseBSON(const BSONObj& source, string* errMsg) { - clear(); - - std::string dummy; - if (!errMsg) errMsg = &dummy; - - FieldParser::FieldState fieldState; - fieldState = FieldParser::extract(source, ns, &_ns, errMsg); - if (fieldState == FieldParser::FIELD_INVALID) return false; - _isNsSet = fieldState == FieldParser::FIELD_SET; - - fieldState = FieldParser::extract(source, primary, &_primary, errMsg); - if (fieldState == FieldParser::FIELD_INVALID) return false; - _isPrimarySet = fieldState == FieldParser::FIELD_SET; - - fieldState = FieldParser::extract(source, keyPattern, &_keyPattern, errMsg); - if (fieldState == FieldParser::FIELD_INVALID) return false; - _isKeyPatternSet = fieldState == FieldParser::FIELD_SET; - - fieldState = FieldParser::extract(source, unique, &_unique, errMsg); - if (fieldState == FieldParser::FIELD_INVALID) return false; - _isUniqueSet = fieldState == FieldParser::FIELD_SET; - - fieldState = FieldParser::extract(source, updatedAt, &_updatedAt, errMsg); - if (fieldState == FieldParser::FIELD_INVALID) return false; - _isUpdatedAtSet = fieldState == FieldParser::FIELD_SET; - - fieldState = FieldParser::extract(source, noBalance, &_noBalance, errMsg); - if (fieldState == FieldParser::FIELD_INVALID) return false; - _isNoBalanceSet = fieldState == FieldParser::FIELD_SET; - - fieldState = FieldParser::extract(source, epoch, &_epoch, errMsg); - if (fieldState == FieldParser::FIELD_INVALID) return false; - _isEpochSet = fieldState == FieldParser::FIELD_SET; - - fieldState = FieldParser::extract(source, dropped, &_dropped, errMsg); - if (fieldState == FieldParser::FIELD_INVALID) return false; - _isDroppedSet = fieldState == FieldParser::FIELD_SET; - - // - // backward compatibility - // - - // 'updatedAt' used to be called 'lastmod' up to 2.2. - - Date_t lastmod; - fieldState = FieldParser::extract(source, DEPRECATED_lastmod, &lastmod, errMsg); - if (fieldState == FieldParser::FIELD_INVALID) return false; - - if (fieldState == FieldParser::FIELD_SET && _isUpdatedAtSet == false) { - _updatedAt = lastmod; - _isUpdatedAtSet = true; - } - - // 'lastmodEpoch' was a transition format to 'epoch', up to 2.2 - OID lastmodEpoch; - fieldState = FieldParser::extract(source, DEPRECATED_lastmodEpoch, &lastmodEpoch, errMsg); - if (fieldState == FieldParser::FIELD_INVALID) return false; - - if (fieldState == FieldParser::FIELD_SET && _isEpochSet == false) { - _epoch = lastmodEpoch; - _isEpochSet = true; - } - - return true; - } - - void CollectionType::clear() { - - _ns.clear(); - _isNsSet = false; - - _primary.clear(); - _isPrimarySet = false; - - _keyPattern = BSONObj(); - _isKeyPatternSet = false; - - _unique = false; - _isUniqueSet = false; - - _updatedAt = 0ULL; - _isUpdatedAtSet = false; - - _noBalance = false; - _isNoBalanceSet = false; - - _epoch = OID(); - _isEpochSet = false; - - _dropped = false; - _isDroppedSet = false; - - } - - void CollectionType::cloneTo(CollectionType* other) const { - other->clear(); - - other->_ns = _ns; - other->_isNsSet = _isNsSet; - - other->_primary = _primary; - other->_isPrimarySet = _isPrimarySet; - - other->_keyPattern = _keyPattern; - other->_isKeyPatternSet = _isKeyPatternSet; - - other->_unique = _unique; - other->_isUniqueSet = _isUniqueSet; - - other->_updatedAt = _updatedAt; - other->_isUpdatedAtSet = _isUpdatedAtSet; - - other->_noBalance = _noBalance; - other->_isNoBalanceSet = _isNoBalanceSet; - - other->_epoch = _epoch; - other->_isEpochSet = _isEpochSet; - - other->_dropped = _dropped; - other->_isDroppedSet = _isDroppedSet; - - } - - std::string CollectionType::toString() const { - return toBSON().toString(); - } - -} // namespace mongo diff --git a/src/mongo/s/type_collection.h b/src/mongo/s/type_collection.h deleted file mode 100644 index dcbedfa8436..00000000000 --- a/src/mongo/s/type_collection.h +++ /dev/null @@ -1,299 +0,0 @@ -/** - * Copyright (C) 2012 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#pragma once - -#include - -#include "mongo/base/disallow_copying.h" -#include "mongo/base/string_data.h" -#include "mongo/db/jsobj.h" - -namespace mongo { - - /** - * This class represents the layout and contents of documents contained in the - * config.collections collection. All manipulation of documents coming from that - * collection should be done with this class. - * - * Usage Example: - * - * // Contact the config. 'conn' has been obtained before. - * DBClientBase* conn; - * BSONObj query = QUERY(CollectionType::exampleField("exampleFieldName")); - * exampleDoc = conn->findOne(CollectionType::ConfigNS, query); - * - * // Process the response. - * CollectionType exampleType; - * std::string errMsg; - * if (!exampleType.parseBSON(exampleDoc, &errMsg) || !exampleType.isValid(&errMsg)) { - * // Can't use 'exampleType'. Take action. - * } - * // use 'exampleType' - * - */ - class CollectionType { - MONGO_DISALLOW_COPYING(CollectionType); - public: - - // - // schema declarations - // - - // Name of the collections collection in the config server. - static const std::string ConfigNS; - - // Field names and types in the collections collection type. - static const BSONField ns; - static const BSONField primary; - static const BSONField keyPattern; - static const BSONField unique; - static const BSONField updatedAt; - static const BSONField noBalance; - static const BSONField epoch; - static const BSONField dropped; - static const BSONField DEPRECATED_lastmodEpoch; - static const BSONField DEPRECATED_lastmod; - - // - // collections type methods - // - - CollectionType(); - ~CollectionType(); - - /** - * Returns true if all the mandatory fields are present and have valid - * representations. Otherwise returns false and fills in the optional 'errMsg' string. - */ - bool isValid(std::string* errMsg) const; - - /** - * Returns the BSON representation of the entry. - */ - BSONObj toBSON() const; - - /** - * Clears and populates the internal state using the 'source' BSON object if the - * latter contains valid values. Otherwise sets errMsg and returns false. - */ - bool parseBSON(const BSONObj& source, std::string* errMsg); - - /** - * Clears the internal state. - */ - void clear(); - - /** - * Copies all the fields present in 'this' to 'other'. - */ - void cloneTo(CollectionType* other) const; - - /** - * Returns a std::string representation of the current internal state. - */ - std::string toString() const; - - // - // individual field accessors - // - - // Mandatory Fields - void setNS(StringData ns) { - _ns = ns.toString(); - _isNsSet = true; - } - - void unsetNS() { _isNsSet = false; } - - bool isNSSet() const { return _isNsSet; } - - // Calling get*() methods when the member is not set results in undefined behavior - const std::string getNS() const { - dassert(_isNsSet); - return _ns; - } - - void setUpdatedAt(const Date_t updatedAt) { - _updatedAt = updatedAt; - _isUpdatedAtSet = true; - } - - void unsetUpdatedAt() { _isUpdatedAtSet = false; } - - bool isUpdatedAtSet() const { return _isUpdatedAtSet; } - - // Calling get*() methods when the member is not set results in undefined behavior - const Date_t getUpdatedAt() const { - dassert(_isUpdatedAtSet); - return _updatedAt; - } - - void setEpoch(const OID epoch) { - _epoch = epoch; - _isEpochSet = true; - } - - void unsetEpoch() { _isEpochSet = false; } - - bool isEpochSet() const { return _isEpochSet; } - - // Calling get*() methods when the member is not set results in undefined behavior - const OID getEpoch() const { - dassert(_isEpochSet); - return _epoch; - } - - // Optional Fields - void setPrimary(StringData& primary) { - _primary = primary.toString(); - _isPrimarySet = true; - } - - void unsetPrimary() { _isPrimarySet = false; } - - bool isPrimarySet() const { - return _isPrimarySet || primary.hasDefault(); - } - - // Calling get*() methods when the member is not set and has no default results in undefined - // behavior - std::string getPrimary() const { - if (_isPrimarySet) { - return _primary; - } else { - dassert(primary.hasDefault()); - return primary.getDefault(); - } - } - void setKeyPattern(const BSONObj& keyPattern) { - _keyPattern = keyPattern.getOwned(); - _isKeyPatternSet = true; - } - - void unsetKeyPattern() { _isKeyPatternSet = false; } - - bool isKeyPatternSet() const { - return _isKeyPatternSet || keyPattern.hasDefault(); - } - - // Calling get*() methods when the member is not set and has no default results in undefined - // behavior - BSONObj getKeyPattern() const { - if (_isKeyPatternSet) { - return _keyPattern; - } else { - dassert(keyPattern.hasDefault()); - return keyPattern.getDefault(); - } - } - void setUnique(bool unique) { - _unique = unique; - _isUniqueSet = true; - } - - void unsetUnique() { _isUniqueSet = false; } - - bool isUniqueSet() const { - return _isUniqueSet || unique.hasDefault(); - } - - // Calling get*() methods when the member is not set and has no default results in undefined - // behavior - bool getUnique() const { - if (_isUniqueSet) { - return _unique; - } else { - dassert(unique.hasDefault()); - return unique.getDefault(); - } - } - void setNoBalance(bool noBalance) { - _noBalance = noBalance; - _isNoBalanceSet = true; - } - - void unsetNoBalance() { _isNoBalanceSet = false; } - - bool isNoBalanceSet() const { - return _isNoBalanceSet || noBalance.hasDefault(); - } - - // Calling get*() methods when the member is not set and has no default results in undefined - // behavior - bool getNoBalance() const { - if (_isNoBalanceSet) { - return _noBalance; - } else { - dassert(noBalance.hasDefault()); - return noBalance.getDefault(); - } - } - void setDropped(bool dropped) { - _dropped = dropped; - _isDroppedSet = true; - } - - void unsetDropped() { _isDroppedSet = false; } - - bool isDroppedSet() const { - return _isDroppedSet || dropped.hasDefault(); - } - - // Calling get*() methods when the member is not set and has no default results in undefined - // behavior - bool getDropped() const { - if (_isDroppedSet) { - return _dropped; - } else { - dassert(dropped.hasDefault()); - return dropped.getDefault(); - } - } - - private: - // Convention: (M)andatory, (O)ptional, (S)pecial rule. - std::string _ns; // (M) namespace - bool _isNsSet; - std::string _primary; // (O) either/or with _keyPattern - bool _isPrimarySet; - BSONObj _keyPattern; // (O) sharding pattern if sharded - bool _isKeyPatternSet; - bool _unique; // (O) mandatory if sharded, index is unique - bool _isUniqueSet; - Date_t _updatedAt; // (M) last updated time - bool _isUpdatedAtSet; - bool _noBalance; // (O) optional if sharded, disable balancing - bool _isNoBalanceSet; - OID _epoch; // (M) disambiguates collection incarnations - bool _isEpochSet; - bool _dropped; // (O) if true, ignore this entry - bool _isDroppedSet; - }; - -} // namespace mongo diff --git a/src/mongo/s/type_collection_test.cpp b/src/mongo/s/type_collection_test.cpp deleted file mode 100644 index 843b19d71f8..00000000000 --- a/src/mongo/s/type_collection_test.cpp +++ /dev/null @@ -1,158 +0,0 @@ -/** - * Copyright (C) 2012 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/platform/basic.h" - -#include "mongo/bson/oid.h" -#include "mongo/s/type_collection.h" -#include "mongo/unittest/unittest.h" -#include "mongo/util/time_support.h" - -namespace { - - using std::string; - using mongo::CollectionType; - using mongo::BSONObj; - using mongo::OID; - using mongo::Date_t; - - TEST(Validity, Empty) { - CollectionType coll; - BSONObj emptyObj = BSONObj(); - string errMsg; - ASSERT(coll.parseBSON(emptyObj, &errMsg)); - ASSERT_EQUALS(errMsg, ""); - ASSERT_FALSE(coll.isValid(NULL)); - } - - TEST(Validity, ShardedCollection) { - CollectionType coll; - BSONObj obj = BSON(CollectionType::ns("db.coll") << - CollectionType::keyPattern(BSON("a" << 1)) << - CollectionType::updatedAt(1ULL) << - CollectionType::epoch(OID::gen())); - string errMsg; - ASSERT(coll.parseBSON(obj, &errMsg)); - ASSERT_EQUALS(errMsg, ""); - ASSERT_TRUE(coll.isValid(NULL)); - } - - TEST(Validity, UnshardedCollection) { - CollectionType coll; - BSONObj obj = BSON(CollectionType::ns("db.coll") << - CollectionType::primary("my_primary_shard") << - CollectionType::updatedAt(1ULL) << - CollectionType::epoch(OID::gen())); - string errMsg; - ASSERT(coll.parseBSON(obj, &errMsg)); - ASSERT_EQUALS(errMsg, ""); - ASSERT_TRUE(coll.isValid(NULL)); - } - - TEST(Validity, MixingOptionals) { - CollectionType coll; - BSONObj obj = BSON(CollectionType::ns("db.coll") << - CollectionType::updatedAt(time(0)) << - CollectionType::unique(true)); - string errMsg; - ASSERT(coll.parseBSON(obj, &errMsg)); - ASSERT_EQUALS(errMsg, ""); - ASSERT_FALSE(coll.isValid(NULL)); - } - - TEST(Compatibility, OldLastmod ) { - CollectionType coll; - Date_t creation(time(0)); - BSONObj obj = BSON(CollectionType::ns("db.coll") << - CollectionType::primary("my_primary_shard") << - CollectionType::DEPRECATED_lastmod(creation) << - CollectionType::epoch(OID::gen())); - string errMsg; - ASSERT(coll.parseBSON(obj, &errMsg)); - ASSERT_EQUALS(errMsg, ""); - ASSERT_TRUE(coll.isValid(NULL)); - ASSERT_EQUALS(coll.getUpdatedAt(), creation); - } - - TEST(Compatibility, OldEpoch) { - CollectionType coll; - OID epoch = OID::gen(); - BSONObj obj = BSON(CollectionType::ns("db.coll") << - CollectionType::primary("my_primary_shard") << - CollectionType::updatedAt(1ULL) << - CollectionType::DEPRECATED_lastmodEpoch(epoch)); - string errMsg; - ASSERT(coll.parseBSON(obj, &errMsg)); - ASSERT_EQUALS(errMsg, ""); - ASSERT_TRUE(coll.isValid(NULL)); - ASSERT_EQUALS(coll.getEpoch(), epoch); - } - - TEST(Compatibility, OldDroppedTrue) { - // The 'dropped' field creates a special case. We still validly parse the document - // containing it but we need to ignore dropped collections in code which uses this. - // Dropped collections should not have sharding information. - CollectionType coll; - BSONObj obj = BSON(CollectionType::ns("db.coll") << - CollectionType::DEPRECATED_lastmod(1ULL) << - CollectionType::DEPRECATED_lastmodEpoch(OID::gen()) << - CollectionType::dropped(true)); - string errMsg; - ASSERT(coll.parseBSON(obj, &errMsg)); - ASSERT_EQUALS(errMsg, ""); - ASSERT_TRUE(coll.isValid(NULL)); - } - - TEST(Compatibility, OldDroppedFalse) { - CollectionType coll; - OID epoch = OID::gen(); - BSONObj obj = BSON(CollectionType::ns("db.coll") << - CollectionType::keyPattern(BSON("a" << 1)) << - CollectionType::unique(true) << - CollectionType::DEPRECATED_lastmod(1ULL) << - CollectionType::DEPRECATED_lastmodEpoch(epoch) << - CollectionType::dropped(false)); - string errMsg; - ASSERT(coll.parseBSON(obj, &errMsg)); - ASSERT_EQUALS(errMsg, ""); - ASSERT_EQUALS(coll.getNS(), "db.coll"); - ASSERT_EQUALS(coll.getKeyPattern(), BSON("a" << 1)); - ASSERT_EQUALS(coll.getUnique(), true); - ASSERT_EQUALS(coll.getUpdatedAt(), 1ULL); - ASSERT_EQUALS(coll.getEpoch(), epoch); - ASSERT_TRUE(coll.isValid(NULL)); - } - - TEST(Validity, BadType) { - CollectionType coll; - BSONObj obj = BSON(CollectionType::ns() << 0); - string errMsg; - ASSERT((!coll.parseBSON(obj, &errMsg)) && (errMsg != "")); - } - -} // unnamed namespace -- cgit v1.2.1