SERVER-28120 persist config.collections entries on shards for the collections that the shards possess

6 files changed, 467 insertions, 3 deletions

diff --git a/src/mongo/s/SConscript b/src/mongo/s/SConscript
index f802a27739d..f6ef30c691d 100644
--- a/src/mongo/s/SConscript
+++ b/src/mongo/s/SConscript
@@ -73,6 +73,7 @@ env.Library(
         'catalog/type_locks.cpp',
         'catalog/type_mongos.cpp',
         'catalog/type_shard.cpp',
+        'catalog/type_shard_collection.cpp',
         'catalog/type_tags.cpp',
         'request_types/add_shard_request_type.cpp',
         'request_types/add_shard_to_zone_request_type.cpp',
@@ -192,13 +193,13 @@ env.CppUnitTest(
         'catalog/type_locks_test.cpp',
         'catalog/type_mongos_test.cpp',
         'catalog/type_shard_test.cpp',
+        'catalog/type_shard_collection_test.cpp',
         'catalog/type_tags_test.cpp',
         'chunk_version_test.cpp',
         'migration_secondary_throttle_options_test.cpp',
         'move_chunk_request_test.cpp',
         'set_shard_version_request_test.cpp',
         'shard_key_pattern_test.cpp',
-#        'shard_id_test.cpp',
     ],
     LIBDEPS=[
         '$BUILD_DIR/mongo/db/query/query_test_service_context',
diff --git a/src/mongo/s/catalog/sharding_catalog_client_impl.cpp b/src/mongo/s/catalog/sharding_catalog_client_impl.cpp
index f541dde581f..8c36fccae77 100644
--- a/src/mongo/s/catalog/sharding_catalog_client_impl.cpp
+++ b/src/mongo/s/catalog/sharding_catalog_client_impl.cpp
@@ -854,7 +854,13 @@ StatusWith<repl::OpTimeWith<CollectionType>> ShardingCatalogClientImpl::getColle
         return parseStatus.getStatus();
     }
 
-    return repl::OpTimeWith<CollectionType>(parseStatus.getValue(), retOpTimePair.opTime);
+    auto collType = parseStatus.getValue();
+    if (collType.getDropped()) {
+        return Status(ErrorCodes::NamespaceNotFound,
+                      stream() << "collection " << collNs << " was dropped");
+    }
+
+    return repl::OpTimeWith<CollectionType>(collType, retOpTimePair.opTime);
 }
 
 Status ShardingCatalogClientImpl::getCollections(OperationContext* opCtx,
diff --git a/src/mongo/s/catalog/type_collection.h b/src/mongo/s/catalog/type_collection.h
index 06c51be3b55..b46937266e1 100644
--- a/src/mongo/s/catalog/type_collection.h
+++ b/src/mongo/s/catalog/type_collection.h
@@ -43,9 +43,26 @@ class StatusWith;
 
 
 /**
- * This class represents the layout and contents of documents contained in the
+ * This class represents the layout and contents of documents contained in the config server's
  * config.collections collection. All manipulation of documents coming from that collection
  * should be done with this class.
+ *
+ * Expected config server config.collections collection format:
+ *   {
+ *      "_id" : "foo.bar",
+ *      "lastmodEpoch" : ObjectId("58b6fd76132358839e409e47"),
+ *      "lastmod" : ISODate("1970-02-19T17:02:47.296Z"),
+ *      "dropped" : false,
+ *      "key" : {
+ *          "_id" : 1
+ *      },
+ *      "defaultCollation" : {
+ *          "locale" : "fr_CA"
+ *      },
+ *      "unique" : false,
+ *      "noBalance" : false
+ *   }
+ *
  */
 class CollectionType {
 public:
diff --git a/src/mongo/s/catalog/type_shard_collection.cpp b/src/mongo/s/catalog/type_shard_collection.cpp
new file mode 100644
index 00000000000..e1c2a067be6
--- /dev/null
+++ b/src/mongo/s/catalog/type_shard_collection.cpp
@@ -0,0 +1,160 @@
+/**
+ *    Copyright (C) 2017 10gen Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects
+ *    for all of the code used other than as permitted herein. If you modify
+ *    file(s) with this exception, you may extend this exception to your
+ *    version of the file(s), but you are not obligated to do so. If you do not
+ *    wish to do so, delete this exception statement from your version. If you
+ *    delete this exception statement from all source files in the program,
+ *    then also delete it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/s/catalog/type_shard_collection.h"
+
+#include "mongo/base/status_with.h"
+#include "mongo/bson/bsonobj.h"
+#include "mongo/bson/bsonobjbuilder.h"
+#include "mongo/bson/util/bson_extract.h"
+#include "mongo/s/catalog/type_collection.h"
+#include "mongo/util/assert_util.h"
+
+namespace mongo {
+
+const std::string ShardCollectionType::ConfigNS = "config.collections";
+
+const BSONField<std::string> ShardCollectionType::uuid("_id");
+const BSONField<std::string> ShardCollectionType::ns("ns");
+const BSONField<BSONObj> ShardCollectionType::keyPattern("key");
+const BSONField<OID> ShardCollectionType::lastConsistentCollectionVersionEpoch(
+    "lastConsistentCollectionVersionEpoch");
+const BSONField<Date_t> ShardCollectionType::lastConsistentCollectionVersion(
+    "lastConsistentCollectionVersion");
+
+ShardCollectionType::ShardCollectionType(const CollectionType& collectionType)
+    : ShardCollectionType(
+          collectionType.getNs(), collectionType.getNs(), collectionType.getKeyPattern()) {}
+
+ShardCollectionType::ShardCollectionType(const NamespaceString& uuid,
+                                         const NamespaceString& ns,
+                                         const KeyPattern& keyPattern)
+    : _uuid(uuid), _ns(ns), _keyPattern(keyPattern.toBSON()) {}
+
+StatusWith<ShardCollectionType> ShardCollectionType::fromBSON(const BSONObj& source) {
+    NamespaceString uuidNss;
+    {
+        std::string uuidString;
+        Status status = bsonExtractStringField(source, uuid.name(), &uuidString);
+        if (!status.isOK()) {
+            return status;
+        }
+        uuidNss = NamespaceString{uuidString};
+    }
+
+    NamespaceString nsNss;
+    {
+        std::string nsString;
+        Status status = bsonExtractStringField(source, ns.name(), &nsString);
+        if (!status.isOK()) {
+            return status;
+        }
+        nsNss = NamespaceString{nsString};
+    }
+
+    BSONElement collKeyPattern;
+    Status status = bsonExtractTypedField(source, keyPattern.name(), Object, &collKeyPattern);
+    if (!status.isOK()) {
+        return status;
+    }
+    BSONObj obj = collKeyPattern.Obj();
+    if (obj.isEmpty()) {
+        return Status(ErrorCodes::ShardKeyNotFound,
+                      str::stream() << "Empty shard key. Failed to parse: " << source.toString());
+    }
+    KeyPattern pattern(obj.getOwned());
+
+    ShardCollectionType shardCollectionType(uuidNss, nsNss, pattern);
+
+    {
+        auto statusWithChunkVersion = ChunkVersion::parseFromBSONWithFieldForCommands(
+            source, lastConsistentCollectionVersion.name());
+        if (statusWithChunkVersion.isOK()) {
+            ChunkVersion collVersion = std::move(statusWithChunkVersion.getValue());
+            shardCollectionType.setLastConsistentCollectionVersion(std::move(collVersion));
+        } else if (statusWithChunkVersion == ErrorCodes::NoSuchKey) {
+            // May not be set yet, which is okay.
+        } else {
+            return statusWithChunkVersion.getStatus();
+        }
+    }
+
+    return shardCollectionType;
+}
+
+bool ShardCollectionType::isLastConsistentCollectionVersionSet() const {
+    return _lastConsistentCollectionVersion.is_initialized();
+}
+
+BSONObj ShardCollectionType::toBSON() const {
+    BSONObjBuilder builder;
+
+    builder.append(uuid.name(), _uuid.toString());
+    builder.append(ns.name(), _ns.toString());
+    builder.append(keyPattern.name(), _keyPattern.toBSON());
+
+    if (_lastConsistentCollectionVersion) {
+        _lastConsistentCollectionVersion->appendWithFieldForCommands(
+            &builder, lastConsistentCollectionVersion.name());
+    }
+
+    return builder.obj();
+}
+
+std::string ShardCollectionType::toString() const {
+    return toBSON().toString();
+}
+
+void ShardCollectionType::setUUID(const NamespaceString& uuid) {
+    invariant(uuid.isValid());
+    _uuid = uuid;
+}
+
+void ShardCollectionType::setNs(const NamespaceString& ns) {
+    invariant(ns.isValid());
+    _ns = ns;
+}
+
+void ShardCollectionType::setKeyPattern(const KeyPattern& keyPattern) {
+    invariant(!keyPattern.toBSON().isEmpty());
+    _keyPattern = keyPattern;
+}
+
+void ShardCollectionType::setLastConsistentCollectionVersion(
+    const ChunkVersion& lastConsistentCollectionVersion) {
+    _lastConsistentCollectionVersion = lastConsistentCollectionVersion;
+}
+
+const OID ShardCollectionType::getLastConsistentCollectionVersionEpoch() const {
+    invariant(_lastConsistentCollectionVersion);
+    return _lastConsistentCollectionVersion->epoch();
+}
+
+}  // namespace mongo
diff --git a/src/mongo/s/catalog/type_shard_collection.h b/src/mongo/s/catalog/type_shard_collection.h
new file mode 100644
index 00000000000..ca9fa00d5b5
--- /dev/null
+++ b/src/mongo/s/catalog/type_shard_collection.h
@@ -0,0 +1,147 @@
+/**
+ *    Copyright (C) 2017 10gen Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects
+ *    for all of the code used other than as permitted herein. If you modify
+ *    file(s) with this exception, you may extend this exception to your
+ *    version of the file(s), but you are not obligated to do so. If you do not
+ *    wish to do so, delete this exception statement from your version. If you
+ *    delete this exception statement from all source files in the program,
+ *    then also delete it in the license file.
+ */
+
+#pragma once
+
+#include <boost/optional.hpp>
+#include <string>
+
+#include "mongo/db/jsobj.h"
+#include "mongo/db/keypattern.h"
+#include "mongo/db/namespace_string.h"
+#include "mongo/s/chunk_version.h"
+
+namespace mongo {
+
+class CollectionType;
+class Status;
+template <typename T>
+class StatusWith;
+
+/**
+ * This class represents the layout and contents of documents contained in the shard server's
+ * config.collections collections. All manipulation of documents coming from that collection should
+ * be done with this class.
+ *
+ * Expected shard server config.collections collection format:
+ *   {
+ *      "_id" : "foo.bar",  // will eventually become a UUID field, when it becomes available
+ *      "ns" : "foo.bar",
+ *      "key" : {
+ *          "_id" : 1
+ *      },
+ *      "lastConsistentCollectionVersionEpoch" : ObjectId("58b6fd76132358839e409e47"),
+ *      "lastConsistentCollectionVersion" : ISODate("1970-02-19T17:02:47.296Z")
+ *   }
+ *
+ * The 'lastConsistentCollectionVersion' is written by shard primaries and used by shard
+ * secondaries. A secondary uses the value to refresh chunk metadata up to the chunk with that
+ * chunk version. Chunk metadata updates on the shard involve multiple chunks collection document
+ * writes, during which time the data can be inconsistent and should not be loaded.
+ */
+class ShardCollectionType {
+public:
+    // Name of the collections collection in the config server.
+    static const std::string ConfigNS;
+
+    static const BSONField<std::string> uuid;
+    static const BSONField<std::string> ns;
+    static const BSONField<BSONObj> keyPattern;
+    static const BSONField<OID> lastConsistentCollectionVersionEpoch;
+    static const BSONField<Date_t> lastConsistentCollectionVersion;
+
+    /**
+     * The MetadataLoader fetches CollectionType format documents from the config server and then
+     * writes them in ShardCollectionType format to the shard. This constructor facilitates the
+     * conversion from config to shard config.collections schema.
+     */
+    explicit ShardCollectionType(const CollectionType& collType);
+
+    /**
+     * Constructs a new ShardCollectionType object from BSON retrieved from a shard server. Also
+     * does
+     * validation of the contents.
+     */
+    static StatusWith<ShardCollectionType> fromBSON(const BSONObj& source);
+
+    /**
+     * Returns the BSON representation of the entry for the shard collection schema.
+     *
+     * This function only appends the fields and values relevant to shards that are SET on the
+     * ShardCollectionType object. No field is guaranteed to be appended.
+     */
+    BSONObj toBSON() const;
+
+    /**
+     * Returns a std::string representation of the current internal state.
+     */
+    std::string toString() const;
+
+    const NamespaceString& getUUID() const {
+        return _uuid;
+    }
+    void setUUID(const NamespaceString& uuid);
+
+    const NamespaceString& getNs() const {
+        return _ns;
+    }
+    void setNs(const NamespaceString& ns);
+
+    const KeyPattern& getKeyPattern() const {
+        return _keyPattern;
+    }
+    void setKeyPattern(const KeyPattern& keyPattern);
+
+    const ChunkVersion& getLastConsistentCollectionVersion() const {
+        return _lastConsistentCollectionVersion.get();
+    }
+    void setLastConsistentCollectionVersion(const ChunkVersion& lastConsistentCollectionVersion);
+
+    bool isLastConsistentCollectionVersionSet() const;
+
+    const OID getLastConsistentCollectionVersionEpoch() const;
+
+private:
+    ShardCollectionType(const NamespaceString& uuid,
+                        const NamespaceString& ns,
+                        const KeyPattern& keyPattern);
+
+    NamespaceString _uuid;
+
+    // The full namespace (with the database prefix).
+    NamespaceString _ns;
+
+    // Sharding key. If collection is dropped, this is no longer required.
+    KeyPattern _keyPattern;
+
+    // used by shard secondaries to safely refresh chunk metadata up to this version: higher
+    // versions may put the chunk metadata into an inconsistent state.
+    boost::optional<ChunkVersion> _lastConsistentCollectionVersion;
+};
+
+}  // namespace mongo
diff --git a/src/mongo/s/catalog/type_shard_collection_test.cpp b/src/mongo/s/catalog/type_shard_collection_test.cpp
new file mode 100644
index 00000000000..d77d7fa4996
--- /dev/null
+++ b/src/mongo/s/catalog/type_shard_collection_test.cpp
@@ -0,0 +1,133 @@
+/**
+ *    Copyright (C) 2017 10gen Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects
+ *    for all of the code used other than as permitted herein. If you modify
+ *    file(s) with this exception, you may extend this exception to your
+ *    version of the file(s), but you are not obligated to do so. If you do not
+ *    wish to do so, delete this exception statement from your version. If you
+ *    delete this exception statement from all source files in the program,
+ *    then also delete it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/s/catalog/type_shard_collection.h"
+
+#include "mongo/base/status_with.h"
+#include "mongo/bson/oid.h"
+#include "mongo/s/catalog/type_collection.h"
+#include "mongo/unittest/unittest.h"
+#include "mongo/util/time_support.h"
+
+namespace mongo {
+namespace {
+
+using unittest::assertGet;
+
+const NamespaceString kNss = NamespaceString("db.coll");
+const BSONObj kKeyPattern = BSON("a" << 1);
+
+TEST(ShardCollectionType, ToFromShardBSONWithLastConsistentCollectionVersion) {
+    const ChunkVersion lastConsistent(1, 0, OID::gen());
+
+    BSONObjBuilder builder;
+    builder.append(ShardCollectionType::uuid.name(), kNss.ns());
+    builder.append(ShardCollectionType::ns.name(), kNss.ns());
+    builder.append(ShardCollectionType::keyPattern.name(), kKeyPattern);
+    lastConsistent.appendWithFieldForCommands(
+        &builder, ShardCollectionType::lastConsistentCollectionVersion.name());
+    BSONObj obj = builder.obj();
+
+    ShardCollectionType shardCollectionType = assertGet(ShardCollectionType::fromBSON(obj));
+
+    ASSERT_EQUALS(shardCollectionType.getUUID(), kNss);
+    ASSERT_EQUALS(shardCollectionType.getNs(), kNss);
+    ASSERT_BSONOBJ_EQ(shardCollectionType.getKeyPattern().toBSON(), kKeyPattern);
+    ASSERT_EQUALS(shardCollectionType.getLastConsistentCollectionVersion(), lastConsistent);
+
+    ASSERT_BSONOBJ_EQ(obj, shardCollectionType.toBSON());
+}
+
+TEST(ShardCollectionType, ToFromShardBSONWithoutLastConsistentCollectionVersion) {
+    BSONObjBuilder builder;
+    builder.append(ShardCollectionType::uuid.name(), kNss.ns());
+    builder.append(ShardCollectionType::ns.name(), kNss.ns());
+    builder.append(ShardCollectionType::keyPattern.name(), kKeyPattern);
+    BSONObj obj = builder.obj();
+
+    ShardCollectionType shardCollectionType = assertGet(ShardCollectionType::fromBSON(obj));
+
+    ASSERT_EQUALS(shardCollectionType.getUUID(), kNss);
+    ASSERT_EQUALS(shardCollectionType.getNs(), kNss);
+    ASSERT_BSONOBJ_EQ(shardCollectionType.getKeyPattern().toBSON(), kKeyPattern);
+    ASSERT_FALSE(shardCollectionType.isLastConsistentCollectionVersionSet());
+
+    ASSERT_BSONOBJ_EQ(obj, shardCollectionType.toBSON());
+}
+
+TEST(ShardCollectionType, FromEmptyBSON) {
+    StatusWith<ShardCollectionType> status = ShardCollectionType::fromBSON(BSONObj());
+    ASSERT_FALSE(status.isOK());
+}
+
+TEST(ShardCollectionType, FromBSONNoUUIDFails) {
+    BSONObj obj =
+        BSON(ShardCollectionType::ns(kNss.ns()) << ShardCollectionType::keyPattern(kKeyPattern));
+
+    StatusWith<ShardCollectionType> status = ShardCollectionType::fromBSON(obj);
+    ASSERT_EQUALS(status.getStatus().code(), ErrorCodes::NoSuchKey);
+}
+
+TEST(ShardCollectionType, FromBSONNoNSFails) {
+    BSONObj obj =
+        BSON(ShardCollectionType::uuid(kNss.ns()) << ShardCollectionType::keyPattern(kKeyPattern));
+
+    StatusWith<ShardCollectionType> status = ShardCollectionType::fromBSON(obj);
+    ASSERT_EQUALS(status.getStatus().code(), ErrorCodes::NoSuchKey);
+}
+
+TEST(ShardCollectionType, FromBSONNoShardKeyFails) {
+    BSONObj obj = BSON(ShardCollectionType::uuid(kNss.ns()) << ShardCollectionType::ns(kNss.ns()));
+
+    StatusWith<ShardCollectionType> status = ShardCollectionType::fromBSON(obj);
+    ASSERT_EQUALS(status.getStatus().code(), ErrorCodes::NoSuchKey);
+}
+
+TEST(ShardCollectionType, ConstructFromCollectionType) {
+    const OID oid = OID::gen();
+
+    BSONObj obj = BSON(CollectionType::fullNs(kNss.ns())
+                       << CollectionType::epoch(oid)
+                       << CollectionType::updatedAt(Date_t::fromMillisSinceEpoch(1))
+                       << CollectionType::keyPattern(kKeyPattern));
+    CollectionType collectionType = assertGet(CollectionType::fromBSON(obj));
+    ASSERT_TRUE(collectionType.validate().isOK());
+
+    BSONObjBuilder shardCollectionTypeBuilder;
+    shardCollectionTypeBuilder.append(ShardCollectionType::uuid.name(), kNss.ns());
+    shardCollectionTypeBuilder.append(ShardCollectionType::ns.name(), kNss.ns());
+    shardCollectionTypeBuilder.append(ShardCollectionType::keyPattern.name(), kKeyPattern);
+
+    ASSERT_BSONOBJ_EQ(ShardCollectionType(collectionType).toBSON(),
+                      shardCollectionTypeBuilder.obj());
+}
+
+}  // namespace
+}  // namespace mongo