summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorGregory Wlodarek <gregory.wlodarek@mongodb.com>2019-02-09 19:45:56 -0500
committerGregory Wlodarek <gregory.wlodarek@mongodb.com>2019-02-09 19:46:58 -0500
commiteb1e30426a83b276f5bbdb89ac21d7b7571a6293 (patch)
tree38aaa1dc509c1031c2c30125929f995097dc3539 /src
parentfd5f4c12ac1c1403fbdf816c1650c97ad4606fdc (diff)
downloadmongo-eb1e30426a83b276f5bbdb89ac21d7b7571a6293.tar.gz
SERVER-37979 Add writing utilities to the index build interface
Diffstat (limited to 'src')
-rw-r--r--src/mongo/db/SConscript32
-rw-r--r--src/mongo/db/catalog/index_build_entry_test.cpp38
-rw-r--r--src/mongo/db/index_build_entry_helpers.cpp376
-rw-r--r--src/mongo/db/index_build_entry_helpers.h173
-rw-r--r--src/mongo/db/index_build_entry_helpers_test.cpp292
-rw-r--r--src/mongo/db/namespace_string.cpp2
-rw-r--r--src/mongo/db/namespace_string.h3
7 files changed, 903 insertions, 13 deletions
diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript
index 846be350302..6b46e5e9fb9 100644
--- a/src/mongo/db/SConscript
+++ b/src/mongo/db/SConscript
@@ -986,6 +986,38 @@ env.CppUnitTest(
]
)
+
+env.Library(
+ target='index_build_entry_helpers',
+ source=[
+ "index_build_entry_helpers.cpp",
+ ],
+ LIBDEPS_PRIVATE=[
+ "catalog_raii",
+ "dbhelpers",
+ "namespace_string",
+ "$BUILD_DIR/mongo/base",
+ "$BUILD_DIR/mongo/db/catalog/commit_quorum_options",
+ "$BUILD_DIR/mongo/db/catalog/index_build_entry_idl",
+ "$BUILD_DIR/mongo/db/storage/write_unit_of_work",
+ "$BUILD_DIR/mongo/db/query_exec",
+ "$BUILD_DIR/mongo/util/net/network",
+ ],
+)
+
+env.CppUnitTest(
+ target='index_build_entry_helpers_test',
+ source=[
+ 'index_build_entry_helpers_test.cpp',
+ ],
+ LIBDEPS=[
+ 'index_build_entry_helpers',
+ '$BUILD_DIR/mongo/db/catalog/catalog_test_fixture',
+ '$BUILD_DIR/mongo/db/catalog/index_build_entry_idl',
+ '$BUILD_DIR/mongo/util/net/network',
+ ],
+)
+
env.Library(
target="cloner",
source=[
diff --git a/src/mongo/db/catalog/index_build_entry_test.cpp b/src/mongo/db/catalog/index_build_entry_test.cpp
index 10e5221af46..7daf4400d46 100644
--- a/src/mongo/db/catalog/index_build_entry_test.cpp
+++ b/src/mongo/db/catalog/index_build_entry_test.cpp
@@ -48,7 +48,7 @@ namespace {
const std::vector<std::string> generateIndexes(size_t numIndexes) {
std::vector<std::string> indexes;
for (size_t i = 0; i < numIndexes; i++) {
- indexes.push_back("Index" + std::to_string(i));
+ indexes.push_back("index_" + std::to_string(i));
}
return indexes;
}
@@ -61,6 +61,28 @@ const std::vector<HostAndPort> generateCommitReadyMembers(size_t numMembers) {
return members;
}
+void checkIfEqual(IndexBuildEntry lhs, IndexBuildEntry rhs) {
+ ASSERT_EQ(lhs.getBuildUUID(), rhs.getBuildUUID());
+ ASSERT_EQ(lhs.getCollectionUUID(), rhs.getCollectionUUID());
+
+ BSONObj commitQuorumOptionsBsonLHS = lhs.getCommitQuorum().toBSON();
+ BSONObj commitQuorumOptionsBsonRHS = rhs.getCommitQuorum().toBSON();
+ ASSERT_BSONOBJ_EQ(commitQuorumOptionsBsonLHS, commitQuorumOptionsBsonRHS);
+
+ auto lhsIndexNames = lhs.getIndexNames();
+ auto rhsIndexNames = rhs.getIndexNames();
+ ASSERT_TRUE(std::equal(lhsIndexNames.begin(), lhsIndexNames.end(), rhsIndexNames.begin()));
+
+ if (lhs.getCommitReadyMembers() && rhs.getCommitReadyMembers()) {
+ auto lhsMembers = lhs.getCommitReadyMembers().get();
+ auto rhsMembers = rhs.getCommitReadyMembers().get();
+ ASSERT_TRUE(std::equal(lhsMembers.begin(), lhsMembers.end(), rhsMembers.begin()));
+ } else {
+ ASSERT_FALSE(lhs.getCommitReadyMembers());
+ ASSERT_FALSE(rhs.getCommitReadyMembers());
+ }
+}
+
TEST(IndexBuildEntryTest, IndexBuildEntryWithRequiredFields) {
const UUID id = UUID::gen();
const UUID collectionUUID = UUID::gen();
@@ -73,7 +95,6 @@ TEST(IndexBuildEntryTest, IndexBuildEntryWithRequiredFields) {
ASSERT_EQUALS(entry.getCollectionUUID(), collectionUUID);
ASSERT_EQUALS(entry.getCommitQuorum().numNodes, 1);
ASSERT_EQUALS(entry.getIndexNames().size(), indexes.size());
- ASSERT_FALSE(entry.getPrepareIndexBuild());
}
TEST(IndexBuildEntryTest, IndexBuildEntryWithOptionalFields) {
@@ -84,16 +105,13 @@ TEST(IndexBuildEntryTest, IndexBuildEntryWithOptionalFields) {
IndexBuildEntry entry(id, collectionUUID, commitQuorum, indexes);
- ASSERT_FALSE(entry.getPrepareIndexBuild());
- entry.setPrepareIndexBuild(true);
entry.setCommitReadyMembers(generateCommitReadyMembers(2));
ASSERT_EQUALS(entry.getBuildUUID(), id);
ASSERT_EQUALS(entry.getCollectionUUID(), collectionUUID);
ASSERT_EQUALS(entry.getCommitQuorum().mode, CommitQuorumOptions::kMajority);
ASSERT_EQUALS(entry.getIndexNames().size(), indexes.size());
- ASSERT_TRUE(entry.getPrepareIndexBuild());
- ASSERT_TRUE(entry.getCommitReadyMembers()->size() == 2);
+ ASSERT_EQ(entry.getCommitReadyMembers()->size(), 2U);
}
TEST(IndexBuildEntryTest, SerializeAndDeserialize) {
@@ -103,7 +121,6 @@ TEST(IndexBuildEntryTest, SerializeAndDeserialize) {
const std::vector<std::string> indexes = generateIndexes(1);
IndexBuildEntry entry(id, collectionUUID, commitQuorum, indexes);
- entry.setPrepareIndexBuild(false);
entry.setCommitReadyMembers(generateCommitReadyMembers(3));
BSONObj obj = entry.toBSON();
@@ -112,12 +129,7 @@ TEST(IndexBuildEntryTest, SerializeAndDeserialize) {
IDLParserErrorContext ctx("IndexBuildsEntry Parser");
IndexBuildEntry rebuiltEntry = IndexBuildEntry::parse(ctx, obj);
- ASSERT_EQUALS(rebuiltEntry.getBuildUUID(), id);
- ASSERT_EQUALS(rebuiltEntry.getCollectionUUID(), collectionUUID);
- ASSERT_EQUALS(rebuiltEntry.getCommitQuorum().mode, "someTag");
- ASSERT_EQUALS(rebuiltEntry.getIndexNames().size(), indexes.size());
- ASSERT_FALSE(rebuiltEntry.getPrepareIndexBuild());
- ASSERT_TRUE(rebuiltEntry.getCommitReadyMembers()->size() == 3);
+ checkIfEqual(entry, rebuiltEntry);
}
} // namespace
diff --git a/src/mongo/db/index_build_entry_helpers.cpp b/src/mongo/db/index_build_entry_helpers.cpp
new file mode 100644
index 00000000000..c243a3977af
--- /dev/null
+++ b/src/mongo/db/index_build_entry_helpers.cpp
@@ -0,0 +1,376 @@
+/**
+ * Copyright (C) 2019-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kStorage
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/index_build_entry_helpers.h"
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "mongo/db/catalog/commit_quorum_options.h"
+#include "mongo/db/catalog/create_collection.h"
+#include "mongo/db/catalog/database_impl.h"
+#include "mongo/db/catalog/index_build_entry_gen.h"
+#include "mongo/db/catalog_raii.h"
+#include "mongo/db/concurrency/write_conflict_exception.h"
+#include "mongo/db/db_raii.h"
+#include "mongo/db/dbhelpers.h"
+#include "mongo/db/matcher/extensions_callback_real.h"
+#include "mongo/db/namespace_string.h"
+#include "mongo/db/operation_context.h"
+#include "mongo/db/query/canonical_query.h"
+#include "mongo/db/query/get_executor.h"
+#include "mongo/db/query/query_request.h"
+#include "mongo/db/record_id.h"
+#include "mongo/db/storage/write_unit_of_work.h"
+#include "mongo/util/log.h"
+#include "mongo/util/mongoutils/str.h"
+#include "mongo/util/uuid.h"
+
+namespace mongo {
+
+namespace {
+
+Status upsert(OperationContext* opCtx, IndexBuildEntry indexBuildEntry) {
+ return writeConflictRetry(opCtx,
+ "upsertIndexBuildEntry",
+ NamespaceString::kIndexBuildEntryNamespace.ns(),
+ [&]() -> Status {
+ AutoGetCollection autoCollection(
+ opCtx, NamespaceString::kIndexBuildEntryNamespace, MODE_IX);
+ Collection* collection = autoCollection.getCollection();
+ if (!collection) {
+ mongoutils::str::stream ss;
+ ss << "Collection not found: "
+ << NamespaceString::kIndexBuildEntryNamespace.ns();
+ return Status(ErrorCodes::NamespaceNotFound, ss);
+ }
+
+ WriteUnitOfWork wuow(opCtx);
+ Helpers::upsert(opCtx,
+ NamespaceString::kIndexBuildEntryNamespace.ns(),
+ indexBuildEntry.toBSON(),
+ /*fromMigrate=*/false);
+ wuow.commit();
+ return Status::OK();
+ });
+}
+
+} // namespace
+
+namespace indexbuildentryhelpers {
+
+void ensureIndexBuildEntriesNamespaceExists(OperationContext* opCtx) {
+ writeConflictRetry(
+ opCtx,
+ "createIndexBuildCollection",
+ NamespaceString::kIndexBuildEntryNamespace.ns(),
+ [&]() -> void {
+ AutoGetOrCreateDb autoDb(
+ opCtx, NamespaceString::kIndexBuildEntryNamespace.db(), MODE_X);
+ Database* db = autoDb.getDb();
+
+ // Ensure the database exists.
+ invariant(db);
+
+ // Create the collection if it doesn't exist.
+ if (!db->getCollection(opCtx, NamespaceString::kIndexBuildEntryNamespace)) {
+ WriteUnitOfWork wuow(opCtx);
+ CollectionOptions options;
+ Collection* collection = db->createCollection(
+ opCtx, NamespaceString::kIndexBuildEntryNamespace.ns(), options);
+
+ // Ensure the collection exists.
+ invariant(collection);
+ wuow.commit();
+ }
+ });
+}
+
+Status addIndexBuildEntry(OperationContext* opCtx, IndexBuildEntry indexBuildEntry) {
+ return writeConflictRetry(opCtx,
+ "addIndexBuildEntry",
+ NamespaceString::kIndexBuildEntryNamespace.ns(),
+ [&]() -> Status {
+ AutoGetCollection autoCollection(
+ opCtx, NamespaceString::kIndexBuildEntryNamespace, MODE_IX);
+ Collection* collection = autoCollection.getCollection();
+ if (!collection) {
+ mongoutils::str::stream ss;
+ ss << "Collection not found: "
+ << NamespaceString::kIndexBuildEntryNamespace.ns();
+ return Status(ErrorCodes::NamespaceNotFound, ss);
+ }
+
+ WriteUnitOfWork wuow(opCtx);
+ Status status = collection->insertDocument(
+ opCtx, InsertStatement(indexBuildEntry.toBSON()), nullptr);
+ if (!status.isOK()) {
+ return status;
+ }
+ wuow.commit();
+ return Status::OK();
+ });
+}
+
+Status removeIndexBuildEntry(OperationContext* opCtx, UUID indexBuildUUID) {
+ return writeConflictRetry(
+ opCtx,
+ "removeIndexBuildEntry",
+ NamespaceString::kIndexBuildEntryNamespace.ns(),
+ [&]() -> Status {
+ AutoGetCollection autoCollection(
+ opCtx, NamespaceString::kIndexBuildEntryNamespace, MODE_IX);
+ Collection* collection = autoCollection.getCollection();
+ if (!collection) {
+ mongoutils::str::stream ss;
+ ss << "Collection not found: " << NamespaceString::kIndexBuildEntryNamespace.ns();
+ return Status(ErrorCodes::NamespaceNotFound, ss);
+ }
+
+ RecordId rid = Helpers::findOne(
+ opCtx, collection, BSON("_id" << indexBuildUUID), /*requireIndex=*/true);
+ if (rid.isNull()) {
+ mongoutils::str::stream ss;
+ ss << "No matching IndexBuildEntry found with indexBuildUUID: " << indexBuildUUID;
+ return Status(ErrorCodes::NoMatchingDocument, ss);
+ }
+
+ WriteUnitOfWork wuow(opCtx);
+ OpDebug opDebug;
+ collection->deleteDocument(opCtx, kUninitializedStmtId, rid, &opDebug);
+ wuow.commit();
+ return Status::OK();
+ });
+}
+
+StatusWith<IndexBuildEntry> getIndexBuildEntry(OperationContext* opCtx, UUID indexBuildUUID) {
+ AutoGetCollectionForRead autoCollection(opCtx, NamespaceString::kIndexBuildEntryNamespace);
+ Collection* collection = autoCollection.getCollection();
+ if (!collection) {
+ mongoutils::str::stream ss;
+ ss << "Collection not found: " << NamespaceString::kIndexBuildEntryNamespace.ns();
+ return Status(ErrorCodes::NamespaceNotFound, ss);
+ }
+
+ BSONObj obj;
+ bool foundObj = Helpers::findOne(
+ opCtx, collection, BSON("_id" << indexBuildUUID), obj, /*requireIndex=*/true);
+ if (!foundObj) {
+ mongoutils::str::stream ss;
+ ss << "No matching IndexBuildEntry found with indexBuildUUID: " << indexBuildUUID;
+ return Status(ErrorCodes::NoMatchingDocument, ss);
+ }
+
+ try {
+ IDLParserErrorContext ctx("IndexBuildsEntry Parser");
+ IndexBuildEntry indexBuildEntry = IndexBuildEntry::parse(ctx, obj);
+ return indexBuildEntry;
+ } catch (const DBException& ex) {
+ mongoutils::str::stream ss;
+ ss << "Invalid BSON found for matching document with indexBuildUUID: " << indexBuildUUID;
+ return Status(ErrorCodes::InvalidBSON, ss);
+ }
+}
+
+StatusWith<std::vector<IndexBuildEntry>> getIndexBuildEntries(OperationContext* opCtx,
+ UUID collectionUUID) {
+ AutoGetCollectionForRead autoCollection(opCtx, NamespaceString::kIndexBuildEntryNamespace);
+ Collection* collection = autoCollection.getCollection();
+ if (!collection) {
+ mongoutils::str::stream ss;
+ ss << "Collection not found: " << NamespaceString::kIndexBuildEntryNamespace.ns();
+ return Status(ErrorCodes::NamespaceNotFound, ss);
+ }
+
+ BSONObj collectionQuery = BSON("collectionUUID" << collectionUUID);
+ std::vector<IndexBuildEntry> indexBuildEntries;
+
+ auto qr = std::make_unique<QueryRequest>(collection->ns());
+ qr->setFilter(collectionQuery);
+
+ const ExtensionsCallbackReal extensionsCallback(opCtx, &collection->ns());
+ const boost::intrusive_ptr<ExpressionContext> expCtx;
+ auto statusWithCQ =
+ CanonicalQuery::canonicalize(opCtx,
+ std::move(qr),
+ expCtx,
+ extensionsCallback,
+ MatchExpressionParser::kAllowAllSpecialFeatures);
+
+ if (!statusWithCQ.isOK()) {
+ return statusWithCQ.getStatus();
+ }
+
+ std::unique_ptr<CanonicalQuery> cq = std::move(statusWithCQ.getValue());
+
+ auto statusWithExecutor = getExecutor(
+ opCtx, collection, std::move(cq), PlanExecutor::NO_YIELD, QueryPlannerParams::DEFAULT);
+ if (!statusWithExecutor.isOK()) {
+ return statusWithExecutor.getStatus();
+ }
+
+ auto exec = std::move(statusWithExecutor.getValue());
+ PlanExecutor::ExecState state;
+ BSONObj obj;
+ RecordId loc;
+ while (PlanExecutor::ADVANCED == (state = exec->getNext(&obj, &loc))) {
+ try {
+ IDLParserErrorContext ctx("IndexBuildsEntry Parser");
+ IndexBuildEntry indexBuildEntry = IndexBuildEntry::parse(ctx, obj);
+ indexBuildEntries.push_back(indexBuildEntry);
+ } catch (const DBException& ex) {
+ mongoutils::str::stream ss;
+ ss << "Invalid BSON found for RecordId " << loc << " in collection "
+ << collection->ns();
+ return Status(ErrorCodes::InvalidBSON, ss);
+ }
+ }
+
+ return indexBuildEntries;
+}
+
+StatusWith<CommitQuorumOptions> getCommitQuorum(OperationContext* opCtx, UUID indexBuildUUID) {
+ StatusWith<IndexBuildEntry> status = getIndexBuildEntry(opCtx, indexBuildUUID);
+ if (!status.isOK()) {
+ return status.getStatus();
+ }
+
+ IndexBuildEntry indexBuildEntry = status.getValue();
+ return indexBuildEntry.getCommitQuorum();
+}
+
+Status setCommitQuorum(OperationContext* opCtx,
+ UUID indexBuildUUID,
+ CommitQuorumOptions commitQuorumOptions) {
+ StatusWith<IndexBuildEntry> status = getIndexBuildEntry(opCtx, indexBuildUUID);
+ if (!status.isOK()) {
+ return status.getStatus();
+ }
+
+ IndexBuildEntry indexBuildEntry = status.getValue();
+ indexBuildEntry.setCommitQuorum(commitQuorumOptions);
+ return upsert(opCtx, indexBuildEntry);
+}
+
+Status addCommitReadyMember(OperationContext* opCtx, UUID indexBuildUUID, HostAndPort hostAndPort) {
+ StatusWith<IndexBuildEntry> status = getIndexBuildEntry(opCtx, indexBuildUUID);
+ if (!status.isOK()) {
+ return status.getStatus();
+ }
+
+ IndexBuildEntry indexBuildEntry = status.getValue();
+
+ std::vector<HostAndPort> newCommitReadyMembers;
+ if (indexBuildEntry.getCommitReadyMembers()) {
+ newCommitReadyMembers = indexBuildEntry.getCommitReadyMembers().get();
+ }
+
+ if (std::find(newCommitReadyMembers.begin(), newCommitReadyMembers.end(), hostAndPort) ==
+ newCommitReadyMembers.end()) {
+ newCommitReadyMembers.push_back(hostAndPort);
+ indexBuildEntry.setCommitReadyMembers(newCommitReadyMembers);
+ return upsert(opCtx, indexBuildEntry);
+ }
+
+ return Status::OK();
+}
+
+Status removeCommitReadyMember(OperationContext* opCtx,
+ UUID indexBuildUUID,
+ HostAndPort hostAndPort) {
+ StatusWith<IndexBuildEntry> status = getIndexBuildEntry(opCtx, indexBuildUUID);
+ if (!status.isOK()) {
+ return status.getStatus();
+ }
+
+ IndexBuildEntry indexBuildEntry = status.getValue();
+
+ std::vector<HostAndPort> newCommitReadyMembers;
+ if (indexBuildEntry.getCommitReadyMembers()) {
+ newCommitReadyMembers = indexBuildEntry.getCommitReadyMembers().get();
+ }
+
+ if (std::find(newCommitReadyMembers.begin(), newCommitReadyMembers.end(), hostAndPort) !=
+ newCommitReadyMembers.end()) {
+ newCommitReadyMembers.erase(
+ std::remove(newCommitReadyMembers.begin(), newCommitReadyMembers.end(), hostAndPort));
+ indexBuildEntry.setCommitReadyMembers(newCommitReadyMembers);
+ return upsert(opCtx, indexBuildEntry);
+ }
+
+ return Status::OK();
+}
+
+StatusWith<std::vector<HostAndPort>> getCommitReadyMembers(OperationContext* opCtx,
+ UUID indexBuildUUID) {
+ StatusWith<IndexBuildEntry> status = getIndexBuildEntry(opCtx, indexBuildUUID);
+ if (!status.isOK()) {
+ return status.getStatus();
+ }
+
+ IndexBuildEntry indexBuildEntry = status.getValue();
+ if (indexBuildEntry.getCommitReadyMembers()) {
+ return indexBuildEntry.getCommitReadyMembers().get();
+ }
+
+ return std::vector<HostAndPort>();
+}
+
+Status clearAllIndexBuildEntries(OperationContext* opCtx) {
+ return writeConflictRetry(opCtx,
+ "truncateIndexBuildEntries",
+ NamespaceString::kIndexBuildEntryNamespace.ns(),
+ [&]() -> Status {
+ AutoGetCollection autoCollection(
+ opCtx, NamespaceString::kIndexBuildEntryNamespace, MODE_X);
+ Collection* collection = autoCollection.getCollection();
+ if (!collection) {
+ mongoutils::str::stream ss;
+ ss << "Collection not found: "
+ << NamespaceString::kIndexBuildEntryNamespace.ns();
+ return Status(ErrorCodes::NamespaceNotFound, ss);
+ }
+
+ WriteUnitOfWork wuow(opCtx);
+ Status status = collection->truncate(opCtx);
+ if (!status.isOK()) {
+ return status;
+ }
+ wuow.commit();
+ return Status::OK();
+ });
+}
+
+} // namespace indexbuildentryhelpers
+} // namespace mongo
diff --git a/src/mongo/db/index_build_entry_helpers.h b/src/mongo/db/index_build_entry_helpers.h
new file mode 100644
index 00000000000..e0591077784
--- /dev/null
+++ b/src/mongo/db/index_build_entry_helpers.h
@@ -0,0 +1,173 @@
+/**
+ * Copyright (C) 2019-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#pragma once
+
+#include <vector>
+
+namespace mongo {
+
+class IndexBuildEntry;
+class CommitQuorumOptions;
+class OperationContext;
+class Status;
+template <typename T>
+class StatusWith;
+class UUID;
+struct HostAndPort;
+
+/**
+ * Format of IndexBuildEntry:
+ * {
+ * _id : indexBuildUUID,
+ * collectionUUID : <UUID>,
+ * commitQuorum : <BSON>,
+ * indexes : [<index_name1>, <index_name2>, ...],
+ * commitReadyMembers : [
+ * <hostAndPort1>,
+ * <hostAndPort2>,
+ * ...
+ * ]
+ * }
+ */
+
+namespace indexbuildentryhelpers {
+
+/**
+ * Creates the "config.system.indexBuilds" collection if it does not already exist.
+ * This is the collection where the IndexBuildEntries will be stored on disk for active index
+ * builds.
+ *
+ * The collection should exist before calling any other helper functions to prevent them from
+ * failing.
+ */
+void ensureIndexBuildEntriesNamespaceExists(OperationContext* opCtx);
+
+/**
+ * Writes the 'indexBuildEntry' to the disk.
+ *
+ * An IndexBuildEntry should be stored on the disk during the duration of the index build process
+ * for the 'indexBuildEntry'.
+ *
+ * Returns 'DuplicateKey' error code if a document already exists on the disk with the same
+ * 'indexBuildUUID'.
+ */
+Status addIndexBuildEntry(OperationContext* opCtx, IndexBuildEntry indexBuildEntry);
+
+/**
+ * Removes the IndexBuildEntry from the disk.
+ *
+ * An IndexBuildEntry should be removed from the disk when the index build either succeeds or fails
+ * for the given 'indexBuildUUID'.
+ *
+ * Returns 'NoMatchingDocument' error code if no document with 'indexBuildUUID' is found.
+ */
+Status removeIndexBuildEntry(OperationContext* opCtx, UUID indexBuildUUID);
+
+/**
+ * Returns the IndexBuildEntry matching the document with 'indexBuildUUID' from the disk if it
+ * exists.
+ *
+ * If the stored IndexBuildEntry on disk contains invalid BSON, the 'InvalidBSON' error code is
+ * returned.
+ *
+ * Returns 'NoMatchingDocument' error code if no document with 'indexBuildUUID' is found.
+ */
+StatusWith<IndexBuildEntry> getIndexBuildEntry(OperationContext* opCtx, UUID indexBuildUUID);
+
+/**
+ * Returns a vector of matching IndexBuildEntries matching the documents with 'collectionUUID'
+ * from disk.
+ *
+ * Can be used to get all the unfinished index builds on the collection if the indexBuildUUID is
+ * unknown.
+ */
+StatusWith<std::vector<IndexBuildEntry>> getIndexBuildEntries(OperationContext* opCtx,
+ UUID collectionUUID);
+
+/**
+ * Returns the 'commitQuorum' matching the document with 'indexBuildUUID' from disk if it
+ * exists.
+ *
+ * Returns 'NoMatchingDocument' error code if no document with 'indexBuildUUID' is found.
+ */
+StatusWith<CommitQuorumOptions> getCommitQuorum(OperationContext* opCtx, UUID indexBuildUUID);
+
+/**
+ * Sets the documents 'commitQuorum' field matching the document with 'indexBuildUUID'.
+ *
+ * Since the commit quorum is configurable until the index build is committed, this should be called
+ * whenever the commit quorum is changed.
+ *
+ * Returns 'NoMatchingDocument' error code if no document with 'indexBuildUUID' is found.
+ */
+Status setCommitQuorum(OperationContext* opCtx,
+ UUID indexBuildUUID,
+ CommitQuorumOptions commitQuorumOptions);
+
+/**
+ * Adds 'hostAndPort' to the 'commitReadyMembers' field for the document with 'indexBuildUUID'.
+ * If the 'hostAndPort' is already in the 'commitReadyMembers' field, nothing is done.
+ *
+ * When a replica set member is ready to commit the index build, we need to record this.
+ *
+ * Returns 'NoMatchingDocument' error code if no document with 'indexBuildUUID' is found.
+ */
+Status addCommitReadyMember(OperationContext* opCtx, UUID indexBuildUUID, HostAndPort hostAndPort);
+
+/**
+ * Removes 'hostAndPort' from the 'commitReadyMembers' field for the document with
+ * 'indexBuildUUID' if it exists.
+ *
+ * If a replica set member is removed during a reconfig and it was a commit ready member, we need to
+ * remove its entry from the 'commitReadyMembers' field.
+ *
+ * Returns 'NoMatchingDocument' error code if no document with 'indexBuildUUID' is found.
+ */
+Status removeCommitReadyMember(OperationContext* opCtx,
+ UUID indexBuildUUID,
+ HostAndPort hostAndPort);
+
+/**
+ * Returns a vector of HostAndPorts of all the 'commitReadyMembers' for the document with
+ * 'indexBuildUUID'.
+ *
+ * Returns 'NoMatchingDocument' error code if no document with 'indexBuildUUID' is found.
+ */
+StatusWith<std::vector<HostAndPort>> getCommitReadyMembers(OperationContext* opCtx,
+ UUID indexBuildUUID);
+
+/**
+ * Truncates all the documents in the "config.system.indexBuilds" collection.
+ * Can be used during recovery to remove unfinished index builds to restart them.
+ */
+Status clearAllIndexBuildEntries(OperationContext* opCtx);
+
+} // namespace indexbuildentryhelpers
+} // namespace mongo
diff --git a/src/mongo/db/index_build_entry_helpers_test.cpp b/src/mongo/db/index_build_entry_helpers_test.cpp
new file mode 100644
index 00000000000..4e58c794429
--- /dev/null
+++ b/src/mongo/db/index_build_entry_helpers_test.cpp
@@ -0,0 +1,292 @@
+/**
+ * Copyright (C) 2019-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include <string>
+#include <vector>
+
+#include "mongo/bson/bsonobj.h"
+#include "mongo/bson/bsonobjbuilder.h"
+#include "mongo/bson/bsontypes.h"
+#include "mongo/db/catalog/catalog_test_fixture.h"
+#include "mongo/db/catalog/commit_quorum_options.h"
+#include "mongo/db/catalog/index_build_entry_gen.h"
+#include "mongo/db/client.h"
+#include "mongo/db/index_build_entry_helpers.h"
+#include "mongo/db/service_context.h"
+#include "mongo/unittest/unittest.h"
+#include "mongo/util/assert_util.h"
+#include "mongo/util/net/hostandport.h"
+#include "mongo/util/uuid.h"
+
+namespace mongo {
+
+namespace {
+using namespace indexbuildentryhelpers;
+
+const std::vector<std::string> generateIndexes(size_t numIndexes) {
+ std::vector<std::string> indexes;
+ for (size_t i = 0; i < numIndexes; i++) {
+ indexes.push_back("index_" + std::to_string(i));
+ }
+ return indexes;
+}
+
+const std::vector<HostAndPort> generateCommitReadyMembers(size_t numMembers) {
+ std::vector<HostAndPort> members;
+ for (size_t i = 0; i < numMembers; i++) {
+ members.push_back(HostAndPort("localhost:27017"));
+ }
+ return members;
+}
+
+void checkIfEqual(IndexBuildEntry lhs, IndexBuildEntry rhs) {
+ ASSERT_EQ(lhs.getBuildUUID(), rhs.getBuildUUID());
+ ASSERT_EQ(lhs.getCollectionUUID(), rhs.getCollectionUUID());
+
+ BSONObj commitQuorumOptionsBsonLHS = lhs.getCommitQuorum().toBSON();
+ BSONObj commitQuorumOptionsBsonRHS = rhs.getCommitQuorum().toBSON();
+ ASSERT_BSONOBJ_EQ(commitQuorumOptionsBsonLHS, commitQuorumOptionsBsonRHS);
+
+ auto lhsIndexNames = lhs.getIndexNames();
+ auto rhsIndexNames = rhs.getIndexNames();
+ ASSERT_TRUE(std::equal(lhsIndexNames.begin(), lhsIndexNames.end(), rhsIndexNames.begin()));
+
+ if (lhs.getCommitReadyMembers() && rhs.getCommitReadyMembers()) {
+ auto lhsMembers = lhs.getCommitReadyMembers().get();
+ auto rhsMembers = rhs.getCommitReadyMembers().get();
+ ASSERT_TRUE(std::equal(lhsMembers.begin(), lhsMembers.end(), rhsMembers.begin()));
+ } else {
+ ASSERT_FALSE(lhs.getCommitReadyMembers());
+ ASSERT_FALSE(rhs.getCommitReadyMembers());
+ }
+}
+
+class IndexBuildEntryHelpersTest : public CatalogTestFixture {
+public:
+ void setUp() {
+ CatalogTestFixture::setUp();
+ operationContext()->lockState()->setShouldConflictWithSecondaryBatchApplication(false);
+
+ const UUID collectionUUID = UUID::gen();
+ const CommitQuorumOptions commitQuorum(CommitQuorumOptions::kMajority);
+
+ // `_firstEntry` and `_secondEntry` are index builds on the same collection.
+ _firstEntry = IndexBuildEntry(UUID::gen(),
+ collectionUUID,
+ CommitQuorumOptions(CommitQuorumOptions::kMajority),
+ generateIndexes(3));
+ _secondEntry = IndexBuildEntry(
+ UUID::gen(), collectionUUID, CommitQuorumOptions(5), generateIndexes(6));
+
+ _thirdEntry = IndexBuildEntry(
+ UUID::gen(), UUID::gen(), CommitQuorumOptions("someTag"), generateIndexes(10));
+
+ ensureIndexBuildEntriesNamespaceExists(operationContext());
+ }
+
+protected:
+ IndexBuildEntry _firstEntry;
+ IndexBuildEntry _secondEntry;
+ IndexBuildEntry _thirdEntry;
+};
+
+TEST_F(IndexBuildEntryHelpersTest, AddIndexBuildEntry) {
+ // Insert an entry twice. The second time we should get a DuplicateKey error.
+ ASSERT_OK(addIndexBuildEntry(operationContext(), _firstEntry));
+
+ Status status = addIndexBuildEntry(operationContext(), _firstEntry);
+ ASSERT_EQUALS(status.code(), ErrorCodes::DuplicateKey);
+
+ ASSERT_OK(addIndexBuildEntry(operationContext(), _secondEntry));
+ ASSERT_OK(addIndexBuildEntry(operationContext(), _thirdEntry));
+
+ unittest::assertGet(getIndexBuildEntry(operationContext(), _firstEntry.getBuildUUID()));
+ unittest::assertGet(getIndexBuildEntry(operationContext(), _secondEntry.getBuildUUID()));
+ unittest::assertGet(getIndexBuildEntry(operationContext(), _thirdEntry.getBuildUUID()));
+}
+
+TEST_F(IndexBuildEntryHelpersTest, RemoveIndexBuildEntry) {
+ ASSERT_OK(addIndexBuildEntry(operationContext(), _firstEntry));
+ ASSERT_OK(addIndexBuildEntry(operationContext(), _secondEntry));
+
+ // Remove an entry with an incorrect index build UUID.
+ Status status = removeIndexBuildEntry(operationContext(), UUID::gen());
+ ASSERT_EQUALS(status, ErrorCodes::NoMatchingDocument);
+
+ ASSERT_OK(removeIndexBuildEntry(operationContext(), _firstEntry.getBuildUUID()));
+ status = removeIndexBuildEntry(operationContext(), _firstEntry.getBuildUUID());
+ ASSERT_EQUALS(status, ErrorCodes::NoMatchingDocument);
+
+ ASSERT_OK(removeIndexBuildEntry(operationContext(), _secondEntry.getBuildUUID()));
+}
+
+TEST_F(IndexBuildEntryHelpersTest, GetIndexBuildEntries) {
+ ASSERT_OK(addIndexBuildEntry(operationContext(), _firstEntry));
+ ASSERT_OK(addIndexBuildEntry(operationContext(), _secondEntry));
+ ASSERT_OK(addIndexBuildEntry(operationContext(), _thirdEntry));
+
+ // Fail to find a document with an incorrect index build UUID.
+ StatusWith<IndexBuildEntry> status = getIndexBuildEntry(operationContext(), UUID::gen());
+ ASSERT_EQUALS(status.getStatus(), ErrorCodes::NoMatchingDocument);
+
+ // Find a document with the correct index build UUID.
+ IndexBuildEntry fetchedEntry =
+ unittest::assertGet(getIndexBuildEntry(operationContext(), _firstEntry.getBuildUUID()));
+
+ checkIfEqual(_firstEntry, fetchedEntry);
+
+ // Search for index build entries by collection UUID.
+ {
+ std::vector<IndexBuildEntry> entries = unittest::assertGet(
+ getIndexBuildEntries(operationContext(), _secondEntry.getCollectionUUID()));
+ ASSERT_EQ(2U, entries.size());
+ }
+
+ {
+ std::vector<IndexBuildEntry> entries = unittest::assertGet(
+ getIndexBuildEntries(operationContext(), _thirdEntry.getCollectionUUID()));
+ ASSERT_EQ(1U, entries.size());
+ }
+
+ {
+ std::vector<IndexBuildEntry> entries =
+ unittest::assertGet(getIndexBuildEntries(operationContext(), UUID::gen()));
+ ASSERT_EQ(0U, entries.size());
+ }
+}
+
+TEST_F(IndexBuildEntryHelpersTest, CommitQuorum) {
+ ASSERT_OK(addIndexBuildEntry(operationContext(), _firstEntry));
+
+ {
+ StatusWith<CommitQuorumOptions> statusWith =
+ getCommitQuorum(operationContext(), UUID::gen());
+ ASSERT_EQUALS(statusWith.getStatus(), ErrorCodes::NoMatchingDocument);
+
+ Status status = setCommitQuorum(operationContext(), UUID::gen(), CommitQuorumOptions(1));
+ ASSERT_EQUALS(status.code(), ErrorCodes::NoMatchingDocument);
+ }
+
+ {
+ CommitQuorumOptions opts =
+ unittest::assertGet(getCommitQuorum(operationContext(), _firstEntry.getBuildUUID()));
+ ASSERT_BSONOBJ_EQ(opts.toBSON(), _firstEntry.getCommitQuorum().toBSON());
+
+ CommitQuorumOptions newCommitQuorum(0);
+ ASSERT_OK(setCommitQuorum(operationContext(), _firstEntry.getBuildUUID(), newCommitQuorum));
+
+ opts = unittest::assertGet(getCommitQuorum(operationContext(), _firstEntry.getBuildUUID()));
+ ASSERT_BSONOBJ_EQ(opts.toBSON(), newCommitQuorum.toBSON());
+ }
+}
+
+TEST_F(IndexBuildEntryHelpersTest, CommitReadyMembers) {
+ ASSERT_OK(addIndexBuildEntry(operationContext(), _firstEntry));
+
+ HostAndPort first("localhost:27017");
+ HostAndPort second("localhost:27018");
+
+ {
+ StatusWith<std::vector<HostAndPort>> statusWith =
+ getCommitReadyMembers(operationContext(), UUID::gen());
+ ASSERT_EQUALS(statusWith.getStatus(), ErrorCodes::NoMatchingDocument);
+
+ Status status = addCommitReadyMember(operationContext(), UUID::gen(), first);
+ ASSERT_EQUALS(status.code(), ErrorCodes::NoMatchingDocument);
+
+ status = removeCommitReadyMember(operationContext(), UUID::gen(), first);
+ ASSERT_EQUALS(status.code(), ErrorCodes::NoMatchingDocument);
+ }
+
+ {
+ std::vector<HostAndPort> entries = unittest::assertGet(
+ getCommitReadyMembers(operationContext(), _firstEntry.getBuildUUID()));
+ ASSERT_EQ(entries.size(), 0U);
+
+ ASSERT_OK(addCommitReadyMember(operationContext(), _firstEntry.getBuildUUID(), first));
+ ASSERT_OK(addCommitReadyMember(operationContext(), _firstEntry.getBuildUUID(), second));
+
+ entries = unittest::assertGet(
+ getCommitReadyMembers(operationContext(), _firstEntry.getBuildUUID()));
+ ASSERT_EQ(entries.size(), 2U);
+ ASSERT_EQ(entries.at(0), first);
+ ASSERT_EQ(entries.at(1), second);
+
+ ASSERT_OK(removeCommitReadyMember(operationContext(), _firstEntry.getBuildUUID(), first));
+ entries = unittest::assertGet(
+ getCommitReadyMembers(operationContext(), _firstEntry.getBuildUUID()));
+ ASSERT_EQ(entries.size(), 1U);
+ ASSERT_EQ(entries.at(0), second);
+
+ ASSERT_OK(removeCommitReadyMember(operationContext(), _firstEntry.getBuildUUID(), second));
+ }
+
+ {
+ // Adding the same HostAndPort pair twice should only register it once.
+ ASSERT_OK(addCommitReadyMember(operationContext(), _firstEntry.getBuildUUID(), first));
+ ASSERT_OK(addCommitReadyMember(operationContext(), _firstEntry.getBuildUUID(), first));
+
+ std::vector<HostAndPort> entries = unittest::assertGet(
+ getCommitReadyMembers(operationContext(), _firstEntry.getBuildUUID()));
+ ASSERT_EQ(entries.size(), 1U);
+
+ // Removing HostAndPort pair not in array should have no affect.
+ ASSERT_OK(removeCommitReadyMember(operationContext(), _firstEntry.getBuildUUID(), second));
+ entries = unittest::assertGet(
+ getCommitReadyMembers(operationContext(), _firstEntry.getBuildUUID()));
+ ASSERT_EQ(entries.size(), 1U);
+ }
+}
+
+TEST_F(IndexBuildEntryHelpersTest, ClearAllIndexBuildEntries) {
+ ASSERT_OK(addIndexBuildEntry(operationContext(), _firstEntry));
+ ASSERT_OK(addIndexBuildEntry(operationContext(), _secondEntry));
+ ASSERT_OK(addIndexBuildEntry(operationContext(), _thirdEntry));
+
+ unittest::assertGet(getIndexBuildEntry(operationContext(), _firstEntry.getBuildUUID()));
+ unittest::assertGet(getIndexBuildEntry(operationContext(), _secondEntry.getBuildUUID()));
+ unittest::assertGet(getIndexBuildEntry(operationContext(), _thirdEntry.getBuildUUID()));
+
+ ASSERT_OK(clearAllIndexBuildEntries(operationContext()));
+
+ StatusWith<IndexBuildEntry> status =
+ getIndexBuildEntry(operationContext(), _firstEntry.getBuildUUID());
+ ASSERT_EQUALS(status.getStatus(), ErrorCodes::NoMatchingDocument);
+
+ status = getIndexBuildEntry(operationContext(), _secondEntry.getBuildUUID());
+ ASSERT_EQUALS(status.getStatus(), ErrorCodes::NoMatchingDocument);
+
+ status = getIndexBuildEntry(operationContext(), _thirdEntry.getBuildUUID());
+ ASSERT_EQUALS(status.getStatus(), ErrorCodes::NoMatchingDocument);
+}
+
+} // namespace
+} // namespace mongo
diff --git a/src/mongo/db/namespace_string.cpp b/src/mongo/db/namespace_string.cpp
index f13a31e3ecd..4fb37b3e9d3 100644
--- a/src/mongo/db/namespace_string.cpp
+++ b/src/mongo/db/namespace_string.cpp
@@ -75,6 +75,8 @@ const NamespaceString NamespaceString::kSystemKeysNamespace(NamespaceString::kAd
const NamespaceString NamespaceString::kRsOplogNamespace(NamespaceString::kLocalDb, "oplog.rs");
const NamespaceString NamespaceString::kSystemReplSetNamespace(NamespaceString::kLocalDb,
"system.replset");
+const NamespaceString NamespaceString::kIndexBuildEntryNamespace(NamespaceString::kConfigDb,
+ "system.indexBuilds");
bool NamespaceString::isListCollectionsCursorNS() const {
return coll() == listCollectionsCursorCol;
diff --git a/src/mongo/db/namespace_string.h b/src/mongo/db/namespace_string.h
index 80766e9564d..b9e810378c9 100644
--- a/src/mongo/db/namespace_string.h
+++ b/src/mongo/db/namespace_string.h
@@ -99,6 +99,9 @@ public:
// Namespace for replica set configuration settings.
static const NamespaceString kSystemReplSetNamespace;
+ // Namespace for index build entries.
+ static const NamespaceString kIndexBuildEntryNamespace;
+
/**
* Constructs an empty NamespaceString.
*/