summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGeert Bosch <geert@mongodb.com>2017-06-15 00:20:33 -0400
committerGeert Bosch <geert@mongodb.com>2017-06-23 15:11:09 -0400
commita69ae445303fc4821c6745866b3902623a385c1c (patch)
tree177cab03d8078defe675fd0dff15349cc32401c0
parent3bb0f6030b5609002049ea2156e97fe4c6c05d5d (diff)
downloadmongo-a69ae445303fc4821c6745866b3902623a385c1c.tar.gz
SERVER-27992 Use UUIDs for replication
-rw-r--r--jstests/aggregation/sources/changeNotification/changeNotification.js2
-rw-r--r--jstests/replsets/apply_ops_idempotency.js153
-rw-r--r--src/mongo/bson/bson_obj_test.cpp20
-rw-r--r--src/mongo/bson/bsonobj.cpp20
-rw-r--r--src/mongo/bson/bsonobj.h7
-rw-r--r--src/mongo/db/SConscript6
-rw-r--r--src/mongo/db/catalog/SConscript38
-rw-r--r--src/mongo/db/catalog/capped_utils.cpp10
-rw-r--r--src/mongo/db/catalog/collection_impl.cpp13
-rw-r--r--src/mongo/db/catalog/collection_impl.h1
-rw-r--r--src/mongo/db/catalog/create_collection.cpp143
-rw-r--r--src/mongo/db/catalog/create_collection.h14
-rw-r--r--src/mongo/db/catalog/database_impl.cpp30
-rw-r--r--src/mongo/db/catalog/drop_indexes.h2
-rw-r--r--src/mongo/db/catalog/namespace_uuid_cache.cpp (renamed from src/mongo/util/namespace_uuid_cache.cpp)44
-rw-r--r--src/mongo/db/catalog/namespace_uuid_cache.h (renamed from src/mongo/util/namespace_uuid_cache.h)31
-rw-r--r--src/mongo/db/catalog/namespace_uuid_cache_test.cpp (renamed from src/mongo/util/namespace_uuid_cache_test.cpp)11
-rw-r--r--src/mongo/db/catalog/rename_collection.cpp32
-rw-r--r--src/mongo/db/catalog/rename_collection.h11
-rw-r--r--src/mongo/db/catalog/uuid_catalog.cpp (renamed from src/mongo/util/uuid_catalog.cpp)62
-rw-r--r--src/mongo/db/catalog/uuid_catalog.h (renamed from src/mongo/util/uuid_catalog.h)31
-rw-r--r--src/mongo/db/catalog/uuid_catalog_test.cpp (renamed from src/mongo/util/uuid_catalog_test.cpp)2
-rw-r--r--src/mongo/db/commands.cpp4
-rw-r--r--src/mongo/db/commands.h3
-rw-r--r--src/mongo/db/op_observer.h3
-rw-r--r--src/mongo/db/op_observer_impl.cpp58
-rw-r--r--src/mongo/db/op_observer_impl.h3
-rw-r--r--src/mongo/db/op_observer_noop.cpp72
-rw-r--r--src/mongo/db/op_observer_noop.h3
-rw-r--r--src/mongo/db/repair_database.cpp12
-rw-r--r--src/mongo/db/repl/oplog.cpp326
-rw-r--r--src/mongo/db/repl/oplog_entry.cpp4
-rw-r--r--src/mongo/db/repl/oplog_entry.h2
-rw-r--r--src/mongo/db/repl/rollback_fix_up_info_test.cpp30
-rw-r--r--src/mongo/db/repl/storage_interface_impl.cpp11
-rw-r--r--src/mongo/db/repl/sync_tail.cpp73
-rw-r--r--src/mongo/db/repl/sync_tail_test.cpp9
-rw-r--r--src/mongo/db/storage/mmap_v1/repair_database.cpp9
-rw-r--r--src/mongo/dbtests/repltests.cpp15
-rw-r--r--src/mongo/dbtests/rollbacktests.cpp29
-rw-r--r--src/mongo/util/SConscript39
-rw-r--r--src/mongo/util/uuid.cpp4
-rw-r--r--src/mongo/util/uuid.h15
43 files changed, 1039 insertions, 368 deletions
diff --git a/jstests/aggregation/sources/changeNotification/changeNotification.js b/jstests/aggregation/sources/changeNotification/changeNotification.js
index b7d0777c58a..aabbfb1b5f6 100644
--- a/jstests/aggregation/sources/changeNotification/changeNotification.js
+++ b/jstests/aggregation/sources/changeNotification/changeNotification.js
@@ -7,7 +7,7 @@
// Helper for testing that pipeline returns correct set of results.
function testPipeline(pipeline, expectedResult, collection) {
// Strip the oplog fields we aren't testing.
- pipeline.push({$project: {ts: 0, t: 0, h: 0, v: 0, wall: 0}});
+ pipeline.push({$project: {ts: 0, t: 0, h: 0, v: 0, wall: 0, ui: 0}});
assert.docEq(collection.aggregate(pipeline).toArray(), expectedResult);
}
diff --git a/jstests/replsets/apply_ops_idempotency.js b/jstests/replsets/apply_ops_idempotency.js
new file mode 100644
index 00000000000..92319302a42
--- /dev/null
+++ b/jstests/replsets/apply_ops_idempotency.js
@@ -0,0 +1,153 @@
+(function() {
+ 'use strict';
+ const debug = 0;
+
+ let rst = new ReplSetTest({
+ name: "applyOpsIdempotency",
+ nodes: 1,
+ nodeOptions: {setParameter: "enableCollectionUUIDs=1"}
+ });
+ rst.startSet();
+ rst.initiate();
+
+ /**
+ * Apply ops on mydb, asserting success.
+ */
+ function assertApplyOpsWorks(mydb, ops) {
+ // Remaining operations in ops must still be applied
+ while (ops.length) {
+ let cmd = {applyOps: ops};
+ let res = mydb.adminCommand(cmd);
+ // If the entire operation succeeded, we're done.
+ if (res.ok == 1)
+ return res;
+
+ // Skip any operations that succeeded.
+ while (res.applied-- && res.results.shift())
+ ops.shift();
+
+ // These errors are expected when replaying operations and should be ignored.
+ if (res.code == ErrorCodes.NamespaceNotFound || res.code == ErrorCodes.DuplicateKey) {
+ ops.shift();
+ continue;
+ }
+
+ // Generate the appropriate error message.
+ assert.commandWorked(res, tojson(cmd));
+ }
+ }
+
+ /**
+ * Run the dbHash command on mydb, assert it worked and return the md5.
+ */
+ function dbHash(mydb) {
+ let cmd = {dbHash: 1};
+ let res = mydb.runCommand(cmd);
+ assert.commandWorked(res, tojson(cmd));
+ return res.md5;
+ }
+
+ var getCollections = (mydb, prefixes) => prefixes.map((prefix) => mydb[prefix]);
+
+ /**
+ * Test functions to run and test using replay of oplog.
+ */
+ var tests = {
+ crud: (mydb) => {
+ let [x, y, z] = getCollections(mydb, ['x', 'y', 'z']);
+ assert.writeOK(x.insert({_id: 1}));
+ assert.writeOK(x.update({_id: 1}, {$set: {x: 1}}));
+ assert.writeOK(x.remove({_id: 1}));
+
+ assert.writeOK(y.update({_id: 1}, {y: 1}));
+ assert.writeOK(y.insert({_id: 2, y: false, z: false}));
+ assert.writeOK(y.update({_id: 2}, {y: 2}));
+
+ assert.writeOK(z.insert({_id: 1, z: 1}));
+ assert.writeOK(z.remove({_id: 1}));
+ assert.writeOK(z.insert({_id: 1}));
+ assert.writeOK(z.insert({_id: 2, z: 2}));
+ },
+ renameCollectionWithinDatabase: (mydb) => {
+ let [x, y, z] = getCollections(mydb, ['x', 'y', 'z']);
+ assert.writeOK(x.insert({_id: 1, x: 1}));
+ assert.writeOK(y.insert({_id: 1, y: 1}));
+
+ assert.commandWorked(x.renameCollection(z.getName()));
+ assert.writeOK(z.insert({_id: 2, x: 2}));
+ assert.writeOK(x.insert({_id: 2, x: false}));
+ assert.writeOK(y.insert({y: 2}));
+
+ assert.commandWorked(y.renameCollection(x.getName(), true));
+ assert.commandWorked(z.renameCollection(y.getName()));
+ },
+ renameCollectionAcrossDatabase: (mydb) => {
+ let otherdb = mydb.getSiblingDB(mydb + '_');
+ let [x, y] = getCollections(mydb, ['x', 'y']);
+ let [z] = getCollections(otherdb, ['z']);
+ assert.writeOK(x.insert({_id: 1, x: 1}));
+ assert.writeOK(y.insert({_id: 1, y: 1}));
+
+ assert.commandWorked(x.renameCollection(z.getName()));
+ assert.writeOK(z.insert({_id: 2, x: 2}));
+ assert.writeOK(x.insert({_id: 2, x: false}));
+ assert.writeOK(y.insert({y: 2}));
+
+ assert.commandWorked(y.renameCollection(x.getName(), true));
+ assert.commandWorked(z.renameCollection(y.getName()));
+
+ },
+ createIndex: (mydb) => {
+ let [x, y] = getCollections(mydb, ['x', 'y']);
+ assert.commandWorked(x.createIndex({x: 1}));
+ assert.writeOK(x.insert({_id: 1, x: 1}));
+ assert.writeOK(y.insert({_id: 1, y: 1}));
+ assert.commandWorked(y.createIndex({y: 1}));
+ assert.writeOK(y.insert({_id: 2, y: 2}));
+ },
+ };
+
+ /**
+ * Create a new uniquely named database, execute testFun and compute the dbHash. Then replay
+ * all different suffixes of the oplog and check for the correct hash.
+ */
+ function testIdempotency(primary, testFun, testName) {
+ // Create a new database name, so it's easier to filter out our oplog records later.
+ let dbname = (new Date()).toISOString().match(/[-0-9T]/g).join(''); // 2017-05-30T155055713
+ let mydb = primary.getDB(dbname);
+ testFun(mydb);
+ let expectedMD5 = dbHash(mydb);
+ let expectedInfos = mydb.getCollectionInfos();
+
+ let oplog = mydb.getSiblingDB('local').oplog.rs;
+ let ops = oplog
+ .find({op: {$ne: 'n'}, ns: new RegExp('^' + mydb.getName())},
+ {ts: 0, t: 0, h: 0, v: 0})
+ .toArray();
+ assert.gt(ops.length, 0, 'Could not find any matching ops in the oplog');
+ assert.commandWorked(mydb.dropDatabase());
+
+ if (debug) {
+ print(testName + ': replaying suffixes of ' + ops.length + ' operations');
+ ops.forEach((op) => printjsononeline({op}));
+ }
+
+ for (let j = 0; j < ops.length; j++) {
+ let replayOps = ops.slice(j);
+ assertApplyOpsWorks(mydb, replayOps);
+ let actualMD5 = dbHash(mydb);
+ assert.eq(
+ actualMD5,
+ expectedMD5,
+ 'unexpected dbHash result after replaying final ' + replayOps.length + ' ops');
+ let actualInfos = mydb.getCollectionInfos();
+ assert.eq(actualInfos,
+ expectedInfos,
+ 'unexpected differences in collection information after replaying final ' +
+ replayOps.length + ' ops');
+ }
+ }
+
+ for (let f in tests)
+ testIdempotency(rst.getPrimary(), tests[f], f);
+})();
diff --git a/src/mongo/bson/bson_obj_test.cpp b/src/mongo/bson/bson_obj_test.cpp
index 890d3c4580f..5a9360605d5 100644
--- a/src/mongo/bson/bson_obj_test.cpp
+++ b/src/mongo/bson/bson_obj_test.cpp
@@ -608,4 +608,24 @@ TEST(BSONObj, ShareOwnershipWith) {
ASSERT_BSONOBJ_EQ(obj, BSON("a" << 1));
}
+TEST(BSONObj, addField) {
+ auto obj = BSON("a" << 1 << "b" << 2);
+
+ // Check that replacing a field maintains the same ordering and doesn't add a field.
+ auto objA2 = BSON("a" << 2);
+ auto elemA2 = objA2.firstElement();
+ auto addFieldA2 = obj.addField(elemA2);
+ ASSERT_EQ(addFieldA2.nFields(), 2);
+ ASSERT_BSONOBJ_EQ(addFieldA2, BSON("a" << 2 << "b" << 2));
+
+ // Check that adding a new field places it at the end.
+ auto objC3 = BSON("c" << 3);
+ auto elemC3 = objC3.firstElement();
+ auto addFieldC3 = obj.addField(elemC3);
+ ASSERT_BSONOBJ_EQ(addFieldC3, BSON("a" << 1 << "b" << 2 << "c" << 3));
+
+ // Check that after all this obj is unchanged.
+ ASSERT_BSONOBJ_EQ(obj, BSON("a" << 1 << "b" << 2));
+}
+
} // unnamed namespace
diff --git a/src/mongo/bson/bsonobj.cpp b/src/mongo/bson/bsonobj.cpp
index 2f7133a2a63..06f446c9c05 100644
--- a/src/mongo/bson/bsonobj.cpp
+++ b/src/mongo/bson/bsonobj.cpp
@@ -503,6 +503,26 @@ bool BSONObj::getObjectID(BSONElement& e) const {
return false;
}
+BSONObj BSONObj::addField(const BSONElement& field) const {
+ if (!field.ok())
+ return copy();
+ BSONObjBuilder b;
+ StringData name = field.fieldNameStringData();
+ bool added = false;
+ for (auto e : *this) {
+ if (e.fieldNameStringData() == name) {
+ if (!added)
+ b.append(field);
+ added = true;
+ } else {
+ b.append(e);
+ }
+ }
+ if (!added)
+ b.append(field);
+ return b.obj();
+}
+
BSONObj BSONObj::removeField(StringData name) const {
BSONObjBuilder b;
BSONObjIterator i(*this);
diff --git a/src/mongo/bson/bsonobj.h b/src/mongo/bson/bsonobj.h
index 8d62bc8fd3d..15e2326c0db 100644
--- a/src/mongo/bson/bsonobj.h
+++ b/src/mongo/bson/bsonobj.h
@@ -245,6 +245,13 @@ public:
/** note: addFields always adds _id even if not specified */
int addFields(BSONObj& from, std::set<std::string>& fields); /* returns n added */
+ /**
+ * Add specific field to the end of the object if it did not exist, otherwise replace it
+ * preserving original field order. Returns newly built object. Returns copy of this for empty
+ * field.
+ */
+ BSONObj addField(const BSONElement& field) const;
+
/** remove specified field and return a new object with the remaining fields.
slowish as builds a full new object
*/
diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript
index 83fd4b3ec40..2ce2c26873a 100644
--- a/src/mongo/db/SConscript
+++ b/src/mongo/db/SConscript
@@ -504,7 +504,7 @@ env.Library(
'commands/test_commands_enabled',
'service_context',
'$BUILD_DIR/mongo/util/uuid',
- '$BUILD_DIR/mongo/util/uuid_catalog',
+ '$BUILD_DIR/mongo/db/catalog/uuid_catalog',
],
)
@@ -608,7 +608,7 @@ env.Library(
'$BUILD_DIR/mongo/base',
'repl/serveronly',
'views/views_mongod',
- '$BUILD_DIR/mongo/util/uuid_catalog',
+ '$BUILD_DIR/mongo/db/catalog/uuid_catalog',
],
)
@@ -1230,6 +1230,8 @@ env.Library(
],
LIBDEPS= [
'$BUILD_DIR/mongo/db/repl/optime',
+ '$BUILD_DIR/mongo/db/catalog/uuid_catalog',
+ '$BUILD_DIR/mongo/db/catalog/database_holder',
],
)
diff --git a/src/mongo/db/catalog/SConscript b/src/mongo/db/catalog/SConscript
index 96796d1ff23..8e62bb4b74b 100644
--- a/src/mongo/db/catalog/SConscript
+++ b/src/mongo/db/catalog/SConscript
@@ -149,6 +149,44 @@ env.Library(
)
env.Library(
+ target='uuid_catalog',
+ source=[
+ 'namespace_uuid_cache.cpp',
+ 'uuid_catalog.cpp',
+ ],
+ LIBDEPS=[
+ 'collection',
+ 'database',
+ '$BUILD_DIR/mongo/base',
+ '$BUILD_DIR/mongo/db/namespace_string',
+ '$BUILD_DIR/mongo/db/storage/storage_options',
+ '$BUILD_DIR/mongo/util/decorable',
+ '$BUILD_DIR/mongo/util/uuid',
+ ],
+)
+
+env.CppUnitTest(
+ target='namespace_uuid_cache_test',
+ source=[
+ 'namespace_uuid_cache_test.cpp'
+ ],
+ LIBDEPS=[
+ 'uuid_catalog',
+ ],
+)
+
+env.CppUnitTest(
+ target='uuid_catalog_test',
+ source=[
+ 'uuid_catalog_test.cpp',
+ ],
+ LIBDEPS=[
+ 'uuid_catalog',
+ '$BUILD_DIR/mongo/db/service_context',
+ ]
+ )
+
+env.Library(
target='catalog',
source=[
"collection_compact.cpp",
diff --git a/src/mongo/db/catalog/capped_utils.cpp b/src/mongo/db/catalog/capped_utils.cpp
index 69bac1c04ff..3f26e0c5bd6 100644
--- a/src/mongo/db/catalog/capped_utils.cpp
+++ b/src/mongo/db/catalog/capped_utils.cpp
@@ -262,15 +262,18 @@ mongo::Status mongo::convertToCapped(OperationContext* opCtx,
}
BackgroundOperation::assertNoBgOpInProgForDb(dbname);
+ auto opObserver = getGlobalServiceContext()->getOpObserver();
std::string shortTmpName = str::stream() << "tmp.convertToCapped." << shortSource;
NamespaceString longTmpName(dbname, shortTmpName);
- if (db->getCollection(opCtx, longTmpName)) {
+ if (auto existingTmpColl = db->getCollection(opCtx, longTmpName)) {
WriteUnitOfWork wunit(opCtx);
Status status = db->dropCollection(opCtx, longTmpName.ns());
if (!status.isOK())
return status;
+ opObserver->onDropCollection(opCtx, longTmpName, existingTmpColl->uuid());
+ wunit.commit();
}
{
@@ -283,7 +286,8 @@ mongo::Status mongo::convertToCapped(OperationContext* opCtx,
}
}
- OptionalCollectionUUID uuid = db->getCollection(opCtx, longTmpName)->uuid();
+ OptionalCollectionUUID origUUID = db->getCollection(opCtx, collectionName)->uuid();
+ OptionalCollectionUUID cappedUUID = db->getCollection(opCtx, longTmpName)->uuid();
{
WriteUnitOfWork wunit(opCtx);
@@ -299,7 +303,7 @@ mongo::Status mongo::convertToCapped(OperationContext* opCtx,
return status;
getGlobalServiceContext()->getOpObserver()->onConvertToCapped(
- opCtx, collectionName, uuid, size);
+ opCtx, collectionName, origUUID, cappedUUID, size);
wunit.commit();
}
diff --git a/src/mongo/db/catalog/collection_impl.cpp b/src/mongo/db/catalog/collection_impl.cpp
index 8e7316c0fa8..da81803a1b0 100644
--- a/src/mongo/db/catalog/collection_impl.cpp
+++ b/src/mongo/db/catalog/collection_impl.cpp
@@ -45,6 +45,8 @@
#include "mongo/db/catalog/database_catalog_entry.h"
#include "mongo/db/catalog/document_validation.h"
#include "mongo/db/catalog/index_create.h"
+#include "mongo/db/catalog/namespace_uuid_cache.h"
+#include "mongo/db/catalog/uuid_catalog.h"
#include "mongo/db/clientcursor.h"
#include "mongo/db/commands/server_status_metric.h"
#include "mongo/db/concurrency/d_concurrency.h"
@@ -206,6 +208,17 @@ CollectionImpl::~CollectionImpl() {
_recordStore->setCappedCallback(nullptr);
_cappedNotifier->kill();
}
+
+ if (_uuid) {
+ if (auto opCtx = cc().getOperationContext()) {
+ auto& uuidCatalog = UUIDCatalog::get(opCtx);
+ invariant(uuidCatalog.lookupCollectionByUUID(_uuid.get()) != _this);
+ auto& cache = NamespaceUUIDCache::get(opCtx);
+ // TODO(geert): cache.verifyNotCached(ns(), uuid().get());
+ cache.evictNamespace(ns());
+ }
+ LOG(2) << "destructed collection " << ns() << " with UUID " << uuid()->toString();
+ }
_magic = 0;
}
diff --git a/src/mongo/db/catalog/collection_impl.h b/src/mongo/db/catalog/collection_impl.h
index 043b98ef424..c65983d39e6 100644
--- a/src/mongo/db/catalog/collection_impl.h
+++ b/src/mongo/db/catalog/collection_impl.h
@@ -32,6 +32,7 @@
#include "mongo/db/catalog/collection_catalog_entry.h"
namespace mongo {
+class UUIDCatalog;
class CollectionImpl final : virtual public Collection::Impl,
virtual CappedCallback,
virtual UpdateNotifier {
diff --git a/src/mongo/db/catalog/create_collection.cpp b/src/mongo/db/catalog/create_collection.cpp
index 8ef6b79ad00..9af83620ab6 100644
--- a/src/mongo/db/catalog/create_collection.cpp
+++ b/src/mongo/db/catalog/create_collection.cpp
@@ -31,6 +31,9 @@
#include "mongo/db/catalog/create_collection.h"
#include "mongo/bson/bsonobj.h"
+#include "mongo/db/catalog/database_holder.h"
+#include "mongo/db/catalog/uuid_catalog.h"
+#include "mongo/db/commands.h"
#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/curop.h"
#include "mongo/db/db_raii.h"
@@ -39,26 +42,28 @@
#include "mongo/db/ops/insert.h"
#include "mongo/db/repl/replication_coordinator_global.h"
-mongo::Status mongo::createCollection(OperationContext* opCtx,
- const std::string& dbName,
- const BSONObj& cmdObj,
- const BSONObj& idIndex) {
+namespace mongo {
+namespace {
+/**
+ * Shared part of the implementation of the createCollection versions for replicated and regular
+ * collection creation.
+ */
+Status createCollection(OperationContext* opCtx,
+ const NamespaceString& nss,
+ const BSONObj& cmdObj,
+ const BSONObj& idIndex,
+ CollectionOptions::ParseKind kind) {
BSONObjIterator it(cmdObj);
- // Extract ns from first cmdObj element.
+ // Skip the first cmdObj element.
BSONElement firstElt = it.next();
- uassert(ErrorCodes::TypeMismatch,
- str::stream() << "Expected first element to be of type String in: " << cmdObj,
- firstElt.type() == BSONType::String);
- uassert(15888, "must pass name of collection to create", !firstElt.valueStringData().empty());
+ invariant(firstElt.fieldNameStringData() == "create");
- Status status = userAllowedCreateNS(dbName, firstElt.valueStringData());
+ Status status = userAllowedCreateNS(nss.db(), nss.coll());
if (!status.isOK()) {
return status;
}
- const NamespaceString nss(dbName, firstElt.valueStringData());
-
// Build options object from remaining cmdObj elements.
BSONObjBuilder optionsBuilder;
while (it.more()) {
@@ -74,7 +79,7 @@ mongo::Status mongo::createCollection(OperationContext* opCtx,
options.hasField("$nExtents"));
MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN {
- Lock::DBLock dbXLock(opCtx, dbName, MODE_X);
+ Lock::DBLock dbXLock(opCtx, nss.db(), MODE_X);
OldClientContext ctx(opCtx, nss.ns());
if (opCtx->writesAreReplicated() &&
!repl::getGlobalReplicationCoordinator()->canAcceptWritesFor(opCtx, nss)) {
@@ -86,13 +91,8 @@ mongo::Status mongo::createCollection(OperationContext* opCtx,
// Create collection.
const bool createDefaultIndexes = true;
- status = userCreateNS(opCtx,
- ctx.db(),
- nss.ns(),
- options,
- CollectionOptions::parseForCommand,
- createDefaultIndexes,
- idIndex);
+ status =
+ userCreateNS(opCtx, ctx.db(), nss.ns(), options, kind, createDefaultIndexes, idIndex);
if (!status.isOK()) {
return status;
}
@@ -102,3 +102,106 @@ mongo::Status mongo::createCollection(OperationContext* opCtx,
MONGO_WRITE_CONFLICT_RETRY_LOOP_END(opCtx, "create", nss.ns());
return Status::OK();
}
+} // namespace
+
+Status createCollection(OperationContext* opCtx,
+ const std::string& dbName,
+ const BSONObj& cmdObj,
+ const BSONObj& idIndex) {
+ return createCollection(opCtx,
+ Command::parseNsCollectionRequired(dbName, cmdObj),
+ cmdObj,
+ idIndex,
+ CollectionOptions::parseForCommand);
+}
+
+Status createCollectionForApplyOps(OperationContext* opCtx,
+ const std::string& dbName,
+ const BSONElement& ui,
+ const BSONObj& cmdObj,
+ const BSONObj& idIndex) {
+ invariant(opCtx->lockState()->isDbLockedForMode(dbName, MODE_X));
+ auto db = dbHolder().get(opCtx, dbName);
+ const NamespaceString newCollName(Command::parseNsCollectionRequired(dbName, cmdObj));
+ auto newCmd = cmdObj;
+
+ // If a UUID is given, see if we need to rename a collection out of the way, and whether the
+ // collection already exists under a different name. If so, rename it into place. As this is
+ // done during replay of the oplog, the operations do not need to be atomic, just idempotent.
+ // We need to do the renaming part in a separate transaction, as we cannot transactionally
+ // create a database on MMAPv1, which could result in createCollection failing if the database
+ // does not yet exist.
+ if (ui.ok()) {
+ MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN {
+ WriteUnitOfWork wunit(opCtx);
+ // Options need the field to be named "uuid", so parse/recreate.
+ auto uuid = uassertStatusOK(UUID::parse(ui));
+ uassert(ErrorCodes::InvalidUUID,
+ "Invalid UUID in applyOps create command: " + uuid.toString(),
+ uuid.isRFC4122v4());
+
+ auto& catalog = UUIDCatalog::get(opCtx);
+ auto currentName = catalog.lookupNSSByUUID(uuid);
+ OpObserver* opObserver = getGlobalServiceContext()->getOpObserver();
+ if (currentName == newCollName)
+ return Status::OK();
+
+ // In the case of oplog replay, a future command may have created or renamed a
+ // collection with that same name. In that case, renaming this future collection to a
+ // random temporary name is correct: once all entries are replayed no temporary names
+ // will remain.
+ // On MMAPv1 the rename can result in index names that are too long. However this should
+ // only happen for initial sync and "resync collection" for rollback, so we can let the
+ // error propagate resulting in an abort and restart of the initial sync or result in
+ // rollback to fassert, requiring a resync of that node.
+ const bool stayTemp = true;
+ if (auto futureColl = db ? db->getCollection(opCtx, newCollName) : nullptr) {
+ auto tmpName = NamespaceString(newCollName.db(), "tmp" + UUID::gen().toString());
+ Status status =
+ db->renameCollection(opCtx, newCollName.ns(), tmpName.ns(), stayTemp);
+ if (!status.isOK())
+ return status;
+ opObserver->onRenameCollection(opCtx,
+ newCollName,
+ tmpName,
+ futureColl->uuid(),
+ /*dropTarget*/ false,
+ /*dropTargetUUID*/ {},
+ /*dropSourceUUID*/ {},
+ stayTemp);
+ }
+
+ // If the collection with the requested UUID already exists, but with a different name,
+ // just rename it to 'newCollName'.
+ if (catalog.lookupCollectionByUUID(uuid)) {
+ Status status =
+ db->renameCollection(opCtx, currentName.ns(), newCollName.ns(), stayTemp);
+ if (!status.isOK())
+ return status;
+ opObserver->onRenameCollection(opCtx,
+ currentName,
+ newCollName,
+ uuid,
+ /*dropTarget*/ false,
+ /*dropTargetUUID*/ {},
+ /*dropSourceUUID*/ {},
+ stayTemp);
+
+ wunit.commit();
+ return Status::OK();
+ }
+
+ // A new collection with the specific UUID must be created, so add the UUID to the
+ // creation options. Regular user collection creation commands cannot do this.
+ auto uuidObj = uuid.toBSON();
+ newCmd = cmdObj.addField(uuidObj.firstElement());
+ wunit.commit();
+ }
+ MONGO_WRITE_CONFLICT_RETRY_LOOP_END(opCtx, "createCollectionForApplyOps", newCollName.ns());
+ }
+
+ return createCollection(
+ opCtx, newCollName, newCmd, idIndex, CollectionOptions::parseForStorage);
+}
+
+} // namespace mongo
diff --git a/src/mongo/db/catalog/create_collection.h b/src/mongo/db/catalog/create_collection.h
index 73dd82bebd8..8b8316f8629 100644
--- a/src/mongo/db/catalog/create_collection.h
+++ b/src/mongo/db/catalog/create_collection.h
@@ -34,6 +34,7 @@
namespace mongo {
class BSONObj;
class OperationContext;
+class BSONElement;
/**
* Creates a collection as described in "cmdObj" on the database "dbName". Creates the collection's
@@ -44,4 +45,17 @@ Status createCollection(OperationContext* opCtx,
const std::string& dbName,
const BSONObj& cmdObj,
const BSONObj& idIndex = BSONObj());
+
+/**
+ * As above, but only used by replication to apply operations. This allows recreating collections
+ * with specific UUIDs (if ui is given), and in that case will rename any existing collections with
+ * the same name and a UUID to a temporary name. If ui is not given, an existing collection will
+ * result in an error.
+ */
+Status createCollectionForApplyOps(OperationContext* opCtx,
+ const std::string& dbName,
+ const BSONElement& ui,
+ const BSONObj& cmdObj,
+ const BSONObj& idIndex = BSONObj());
+
} // namespace mongo
diff --git a/src/mongo/db/catalog/database_impl.cpp b/src/mongo/db/catalog/database_impl.cpp
index 387d96cf98b..9933ea6f36c 100644
--- a/src/mongo/db/catalog/database_impl.cpp
+++ b/src/mongo/db/catalog/database_impl.cpp
@@ -44,6 +44,8 @@
#include "mongo/db/catalog/collection_options.h"
#include "mongo/db/catalog/database_catalog_entry.h"
#include "mongo/db/catalog/database_holder.h"
+#include "mongo/db/catalog/namespace_uuid_cache.h"
+#include "mongo/db/catalog/uuid_catalog.h"
#include "mongo/db/clientcursor.h"
#include "mongo/db/concurrency/d_concurrency.h"
#include "mongo/db/concurrency/write_conflict_exception.h"
@@ -66,8 +68,6 @@
#include "mongo/db/views/view_catalog.h"
#include "mongo/util/assert_util.h"
#include "mongo/util/log.h"
-#include "mongo/util/namespace_uuid_cache.h"
-#include "mongo/util/uuid_catalog.h"
namespace mongo {
namespace {
@@ -584,11 +584,8 @@ Collection* DatabaseImpl::getCollection(OperationContext* opCtx, const Namespace
Collection* found = it->second;
if (enableCollectionUUIDs) {
NamespaceUUIDCache& cache = NamespaceUUIDCache::get(opCtx);
- CollectionOptions found_options = found->getCatalogEntry()->getCollectionOptions(opCtx);
- if (found_options.uuid) {
- CollectionUUID uuid = found_options.uuid.get();
- cache.ensureNamespaceInCache(nss, uuid);
- }
+ if (auto uuid = found->uuid())
+ cache.ensureNamespaceInCache(nss, uuid.get());
}
return found;
}
@@ -698,11 +695,16 @@ Collection* DatabaseImpl::createCollection(OperationContext* opCtx,
invariant(opCtx->lockState()->isDbLockedForMode(name(), MODE_X));
invariant(!options.isView());
+ CollectionOptions optionsWithUUID = options;
+ if (enableCollectionUUIDs && !optionsWithUUID.uuid)
+ optionsWithUUID.uuid.emplace(CollectionUUID::gen());
+
NamespaceString nss(ns);
- _checkCanCreateCollection(opCtx, nss, options);
+ _checkCanCreateCollection(opCtx, nss, optionsWithUUID);
audit::logCreateCollection(&cc(), ns);
- Status status = _dbEntry->createCollection(opCtx, ns, options, true /*allocateDefaultSpace*/);
+ Status status =
+ _dbEntry->createCollection(opCtx, ns, optionsWithUUID, true /*allocateDefaultSpace*/);
massertNoTraceStatusOK(status);
opCtx->recoveryUnit()->registerChange(new AddCollectionChange(opCtx, this, ns));
@@ -714,8 +716,8 @@ Collection* DatabaseImpl::createCollection(OperationContext* opCtx,
if (createIdIndex) {
if (collection->requiresIdIndex()) {
- if (options.autoIndexId == CollectionOptions::YES ||
- options.autoIndexId == CollectionOptions::DEFAULT) {
+ if (optionsWithUUID.autoIndexId == CollectionOptions::YES ||
+ optionsWithUUID.autoIndexId == CollectionOptions::DEFAULT) {
const auto featureCompatibilityVersion =
serverGlobalParams.featureCompatibility.version.load();
IndexCatalog* ic = collection->getIndexCatalog();
@@ -732,7 +734,7 @@ Collection* DatabaseImpl::createCollection(OperationContext* opCtx,
}
getGlobalServiceContext()->getOpObserver()->onCreateCollection(
- opCtx, collection, nss, options, fullIdIndexSpec);
+ opCtx, collection, nss, optionsWithUUID, fullIdIndexSpec);
return collection;
}
@@ -754,8 +756,10 @@ void DatabaseImpl::dropDatabase(OperationContext* opCtx, Database* db) {
audit::logDropDatabase(opCtx->getClient(), name);
+ UUIDCatalog::get(opCtx).onCloseDatabase(db);
for (auto&& coll : *db) {
Top::get(opCtx->getClient()->getServiceContext()).collectionDropped(coll->ns().ns(), true);
+ NamespaceUUIDCache::get(opCtx).evictNamespace(coll->ns());
}
dbHolder().close(opCtx, name, "database dropped");
@@ -893,8 +897,6 @@ auto mongo::userCreateNSImpl(OperationContext* opCtx,
invariant(parseKind == CollectionOptions::parseForCommand);
uassertStatusOK(db->createView(opCtx, ns, collectionOptions));
} else {
- if (enableCollectionUUIDs && !collectionOptions.uuid)
- collectionOptions.uuid.emplace(CollectionUUID::gen());
invariant(
db->createCollection(opCtx, ns, collectionOptions, createDefaultIndexes, idIndex));
}
diff --git a/src/mongo/db/catalog/drop_indexes.h b/src/mongo/db/catalog/drop_indexes.h
index bd46f1ebe40..4c288c08b3f 100644
--- a/src/mongo/db/catalog/drop_indexes.h
+++ b/src/mongo/db/catalog/drop_indexes.h
@@ -35,7 +35,7 @@ class NamespaceString;
class OperationContext;
/**
- * Drops the index from collection "ns" that matches the "idxDescriptor" and populates
+ * Drops the index from collection "nss" that matches the "idxDescriptor" and populates
* "result" with some statistics about the dropped index.
*/
Status dropIndexes(OperationContext* opCtx,
diff --git a/src/mongo/util/namespace_uuid_cache.cpp b/src/mongo/db/catalog/namespace_uuid_cache.cpp
index 59e40b73166..8919865e4b8 100644
--- a/src/mongo/util/namespace_uuid_cache.cpp
+++ b/src/mongo/db/catalog/namespace_uuid_cache.cpp
@@ -26,12 +26,24 @@
* it in the license file.
*/
+#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kStorage
+
+#include "mongo/platform/basic.h"
+
+#include <algorithm>
+
#include "namespace_uuid_cache.h"
+#include "mongo/db/catalog/database.h"
+#include "mongo/db/server_parameters.h"
#include "mongo/util/assert_util.h"
+#include "mongo/util/log.h"
namespace mongo {
+// TODO(geert): Enable checks by default
+MONGO_EXPORT_STARTUP_SERVER_PARAMETER(debugCollectionUUIDs, bool, false);
+
const OperationContext::Decoration<NamespaceUUIDCache> NamespaceUUIDCache::get =
OperationContext::declareDecoration<NamespaceUUIDCache>();
@@ -41,23 +53,31 @@ void NamespaceUUIDCache::ensureNamespaceInCache(const NamespaceString& nss, Coll
if (it == _cache.end()) {
// Add ns, uuid pair to the cache if it does not yet exist.
invariant(_cache.try_emplace(ns, uuid).second == true);
- } else {
+ LOG(2) << "NamespaceUUIDCache: registered namespace " << nss.ns() << " with UUID " << uuid;
+
+ } else if (it->second != uuid) {
// If ns exists in the cache, make sure it does not correspond to another uuid.
- uassert(40418,
- "Cannot continue operation on namespace " + ns + ": it now resolves " +
- uuid.toString() + " instead of " + it->second.toString(),
- it->second == uuid);
+ auto msg = "Namespace " + ns + " now resolves to UUID " + uuid.toString() +
+ " instead of UUID " + it->second.toString();
+ LOG(1) << msg;
+ uassert(40418, "Cannot continue operation: " + msg, !debugCollectionUUIDs);
}
}
-void NamespaceUUIDCache::onDropCollection(const NamespaceString& nss) {
- _evictNamespace(nss);
-}
-void NamespaceUUIDCache::onRenameCollection(const NamespaceString& nss) {
- _evictNamespace(nss);
+void NamespaceUUIDCache::evictNamespace(const NamespaceString& nss) {
+ size_t evicted = _cache.erase(nss.ns());
+ if (evicted) {
+ LOG(2) << "NamespaceUUIDCache: evicted namespace " << nss.ns();
+ }
+ invariant(evicted <= 1);
}
-void NamespaceUUIDCache::_evictNamespace(const NamespaceString& nss) {
- invariant(_cache.erase(nss.ns()) <= 1);
+void NamespaceUUIDCache::evictNamespacesInDatabase(StringData dbname) {
+ for (auto&& it = _cache.begin(); it != _cache.end();) {
+ auto entry = it++;
+ if (entry->first.empty() || nsToDatabaseSubstring(entry->first) == dbname)
+ _cache.erase(entry);
+ }
}
+
} // namespace mongo
diff --git a/src/mongo/util/namespace_uuid_cache.h b/src/mongo/db/catalog/namespace_uuid_cache.h
index 10f4bd75206..6f10d3bf80a 100644
--- a/src/mongo/util/namespace_uuid_cache.h
+++ b/src/mongo/db/catalog/namespace_uuid_cache.h
@@ -36,11 +36,14 @@
namespace mongo {
+extern bool debugCollectionUUIDs;
+
/**
* This class comprises the NamespaceString to UUID cache that prevents given namespaces
* from resolving to multiple UUIDs.
*/
using CollectionUUID = UUID;
+class Database;
class NamespaceUUIDCache {
MONGO_DISALLOW_COPYING(NamespaceUUIDCache);
@@ -50,36 +53,36 @@ public:
NamespaceUUIDCache() = default;
/**
- * This function adds the pair nss.ns(), uuid to the namespace uuid cache
+ * This function adds the pair nss.ns(), uuid to the namespace UUID cache
* if it does not yet exist. If nss.ns() already exists in the cache with
- * a different uuid, a UserException is thrown, so we can guarantee that
+ * a different UUID, a UserException is thrown, so we can guarantee that
* an operation will always resolve the same name to the same collection,
* even in presence of drops and renames.
*/
void ensureNamespaceInCache(const NamespaceString& nss, CollectionUUID uuid);
/**
- * This function removes the entry for nss.ns() from the namespace uuid
+ * This function removes the entry for nss.ns() from the namespace UUID
* cache. Does nothing if the entry doesn't exist. It is called via the
- * op observer when a collection is dropped.
+ * op observer when a collection is dropped or renamed.
*/
- void onDropCollection(const NamespaceString& nss);
+ void evictNamespace(const NamespaceString& nss);
/**
- * This function removes the entry for nss.ns() from the namespace uuid
- * cache. Does nothing if the entry doesn't exist. It is called via the
- * op observer when a collection is renamed.
+ * Same as above, but for all registered namespaces in the given dbname.
*/
- void onRenameCollection(const NamespaceString& nss);
+ void evictNamespacesInDatabase(StringData dbname);
-private:
/**
- * This function removes the entry for nss.ns() from the namespace uuid
- * cache. Does nothing if the entry doesn't exist.
+ * For testing only: verify that 'nss' is not cached.
*/
- void _evictNamespace(const NamespaceString& nss);
+ void verifyNotCached(const NamespaceString& nss, CollectionUUID uuid) {
+ auto it = _cache.find(nss.ns());
+ invariant(it == _cache.end() || it->second != uuid);
+ }
+
+private:
using CollectionUUIDMap = StringMap<CollectionUUID>;
CollectionUUIDMap _cache;
};
-
} // namespace mongo
diff --git a/src/mongo/util/namespace_uuid_cache_test.cpp b/src/mongo/db/catalog/namespace_uuid_cache_test.cpp
index 7834dcb4968..22034f8ee43 100644
--- a/src/mongo/util/namespace_uuid_cache_test.cpp
+++ b/src/mongo/db/catalog/namespace_uuid_cache_test.cpp
@@ -26,7 +26,7 @@
* it in the license file.
*/
-#include "mongo/util/namespace_uuid_cache.h"
+#include "mongo/db/catalog/namespace_uuid_cache.h"
#include "mongo/unittest/unittest.h"
@@ -43,8 +43,11 @@ TEST(NamespaceUUIDCache, ensureNamespaceInCache) {
cache.ensureNamespaceInCache(nss, uuid);
// Do nothing if we query for existing nss, uuid pairing.
cache.ensureNamespaceInCache(nss, uuid);
- // Uassert if we query for existing nss and uuid that does not match.
- ASSERT_THROWS(cache.ensureNamespaceInCache(nss, uuidConflict), UserException);
+
+ if (debugCollectionUUIDs) {
+ // Uassert if we query for existing nss and uuid that does not match.
+ ASSERT_THROWS(cache.ensureNamespaceInCache(nss, uuidConflict), UserException);
+ }
}
TEST(NamespaceUUIDCache, onDropCollection) {
@@ -53,7 +56,7 @@ TEST(NamespaceUUIDCache, onDropCollection) {
CollectionUUID newUuid = CollectionUUID::gen();
NamespaceString nss("test", "test_collection_ns");
cache.ensureNamespaceInCache(nss, uuid);
- cache.onDropCollection(nss);
+ cache.evictNamespace(nss);
// Add nss to the cache with a different uuid. This should not throw since
// we evicted the previous entry from the cache.
cache.ensureNamespaceInCache(nss, newUuid);
diff --git a/src/mongo/db/catalog/rename_collection.cpp b/src/mongo/db/catalog/rename_collection.cpp
index 8fdf3551b7b..2e71e2be1a0 100644
--- a/src/mongo/db/catalog/rename_collection.cpp
+++ b/src/mongo/db/catalog/rename_collection.cpp
@@ -40,6 +40,7 @@
#include "mongo/db/catalog/document_validation.h"
#include "mongo/db/catalog/index_catalog.h"
#include "mongo/db/catalog/index_create.h"
+#include "mongo/db/catalog/uuid_catalog.h"
#include "mongo/db/client.h"
#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/curop.h"
@@ -63,6 +64,37 @@ static void dropCollection(OperationContext* opCtx, Database* db, StringData col
}
} // namespace
+Status renameCollectionForApplyOps(OperationContext* opCtx,
+ const std::string& dbName,
+ const BSONElement& ui,
+ const BSONObj& cmd) {
+
+ const auto sourceNsElt = cmd.firstElement();
+ const auto targetNsElt = cmd["to"];
+ uassert(ErrorCodes::TypeMismatch,
+ "'renameCollection' must be of type String",
+ sourceNsElt.type() == BSONType::String);
+ uassert(ErrorCodes::TypeMismatch,
+ "'to' must be of type String",
+ targetNsElt.type() == BSONType::String);
+
+ NamespaceString sourceNss(sourceNsElt.valueStringData());
+ NamespaceString targetNss(targetNsElt.valueStringData());
+ if (!ui.eoo()) {
+ auto statusWithUUID = UUID::parse(ui);
+ uassertStatusOK(statusWithUUID);
+ auto uuid = statusWithUUID.getValue();
+ Collection* source = UUIDCatalog::get(opCtx).lookupCollectionByUUID(uuid);
+ uassert(ErrorCodes::NamespaceNotFound,
+ "cannot find collection with UUID " + uuid.toString(),
+ source);
+ sourceNss = source->ns();
+ }
+
+ return renameCollection(
+ opCtx, sourceNss, targetNss, cmd["dropTarget"].trueValue(), cmd["stayTemp"].trueValue());
+}
+
Status renameCollection(OperationContext* opCtx,
const NamespaceString& source,
const NamespaceString& target,
diff --git a/src/mongo/db/catalog/rename_collection.h b/src/mongo/db/catalog/rename_collection.h
index c6a3f56b380..2c59831922a 100644
--- a/src/mongo/db/catalog/rename_collection.h
+++ b/src/mongo/db/catalog/rename_collection.h
@@ -27,6 +27,8 @@
*/
#include "mongo/base/status.h"
+#include "mongo/bson/bsonelement.h"
+#include "mongo/bson/bsonobj.h"
namespace mongo {
class NamespaceString;
@@ -43,4 +45,13 @@ Status renameCollection(OperationContext* opCtx,
bool dropTarget,
bool stayTemp);
+/**
+ * As above, but may only be called from applyCommand_inlock. This allows creating a collection
+ * with a specific UUID for cross-database renames.
+ */
+Status renameCollectionForApplyOps(OperationContext* opCtx,
+ const std::string& dbName,
+ const BSONElement& ui,
+ const BSONObj& cmd);
+
} // namespace mongo
diff --git a/src/mongo/util/uuid_catalog.cpp b/src/mongo/db/catalog/uuid_catalog.cpp
index a73b1288b7a..ef5727a4bbc 100644
--- a/src/mongo/util/uuid_catalog.cpp
+++ b/src/mongo/db/catalog/uuid_catalog.cpp
@@ -31,14 +31,23 @@
#include "uuid_catalog.h"
+#include "mongo/db/catalog/database.h"
#include "mongo/db/storage/recovery_unit.h"
#include "mongo/util/log.h"
#include "mongo/util/uuid.h"
namespace mongo {
-
-const ServiceContext::Decoration<UUIDCatalog> UUIDCatalog::get =
+namespace {
+const ServiceContext::Decoration<UUIDCatalog> getCatalog =
ServiceContext::declareDecoration<UUIDCatalog>();
+} // namespace
+
+UUIDCatalog& UUIDCatalog::get(ServiceContext* svcCtx) {
+ return getCatalog(svcCtx);
+}
+UUIDCatalog& UUIDCatalog::get(OperationContext* opCtx) {
+ return getCatalog(opCtx->getServiceContext());
+}
void UUIDCatalog::onCreateCollection(OperationContext* opCtx,
Collection* coll,
@@ -47,6 +56,33 @@ void UUIDCatalog::onCreateCollection(OperationContext* opCtx,
opCtx->recoveryUnit()->onRollback([this, uuid] { _removeUUIDCatalogEntry(uuid); });
}
+void UUIDCatalog::onDropCollection(OperationContext* opCtx, CollectionUUID uuid) {
+ Collection* foundColl = _removeUUIDCatalogEntry(uuid);
+ opCtx->recoveryUnit()->onRollback(
+ [this, foundColl, uuid] { _registerUUIDCatalogEntry(uuid, foundColl); });
+}
+
+void UUIDCatalog::onRenameCollection(OperationContext* opCtx,
+ Collection* newColl,
+ CollectionUUID uuid) {
+ Collection* oldColl = _removeUUIDCatalogEntry(uuid);
+ _registerUUIDCatalogEntry(uuid, newColl);
+ opCtx->recoveryUnit()->onRollback([this, oldColl, uuid] {
+ _removeUUIDCatalogEntry(uuid);
+ _registerUUIDCatalogEntry(uuid, oldColl);
+ });
+}
+
+void UUIDCatalog::onCloseDatabase(Database* db) {
+ for (auto&& coll : *db) {
+ if (coll->uuid()) {
+ // While the collection does not actually get dropped, we're going to destroy the
+ // Collection object, so for purposes of the UUIDCatalog it looks the same.
+ _removeUUIDCatalogEntry(coll->uuid().get());
+ }
+ }
+}
+
Collection* UUIDCatalog::lookupCollectionByUUID(CollectionUUID uuid) const {
stdx::lock_guard<stdx::mutex> lock(_catalogLock);
auto foundIt = _catalog.find(uuid);
@@ -56,28 +92,28 @@ Collection* UUIDCatalog::lookupCollectionByUUID(CollectionUUID uuid) const {
NamespaceString UUIDCatalog::lookupNSSByUUID(CollectionUUID uuid) const {
stdx::lock_guard<stdx::mutex> lock(_catalogLock);
auto foundIt = _catalog.find(uuid);
- return foundIt == _catalog.end() ? NamespaceString() : foundIt->second->ns();
-}
-
-void UUIDCatalog::onDropCollection(OperationContext* opCtx, CollectionUUID uuid) {
- Collection* foundCol = _removeUUIDCatalogEntry(uuid);
- opCtx->recoveryUnit()->onRollback(
- [this, foundCol, uuid] { _registerUUIDCatalogEntry(uuid, foundCol); });
+ Collection* coll = foundIt == _catalog.end() ? nullptr : foundIt->second;
+ return foundIt == _catalog.end() ? NamespaceString() : coll->ns();
}
void UUIDCatalog::_registerUUIDCatalogEntry(CollectionUUID uuid, Collection* coll) {
stdx::lock_guard<stdx::mutex> lock(_catalogLock);
if (coll) {
std::pair<CollectionUUID, Collection*> entry = std::make_pair(uuid, coll);
- LOG(2) << "registering collection " << coll->ns() << " as having UUID " << uuid.toString();
- invariant(_catalog.insert(entry).second);
+ LOG(2) << "registering collection " << coll->ns() << " with UUID " << uuid.toString();
+ invariant(_catalog.insert(entry).second == true);
}
}
Collection* UUIDCatalog::_removeUUIDCatalogEntry(CollectionUUID uuid) {
stdx::lock_guard<stdx::mutex> lock(_catalogLock);
- Collection* foundCol = _catalog[uuid];
- invariant(_catalog.erase(uuid) <= 1);
+ auto foundIt = _catalog.find(uuid);
+ if (foundIt == _catalog.end())
+ return nullptr;
+
+ auto foundCol = foundIt->second;
+ LOG(2) << "unregistering collection " << foundCol->ns() << " with UUID " << uuid.toString();
+ _catalog.erase(foundIt);
return foundCol;
}
} // namespace mongo
diff --git a/src/mongo/util/uuid_catalog.h b/src/mongo/db/catalog/uuid_catalog.h
index 6f532745ba7..84ce57cb9b0 100644
--- a/src/mongo/util/uuid_catalog.h
+++ b/src/mongo/db/catalog/uuid_catalog.h
@@ -42,20 +42,38 @@ namespace mongo {
* collection lookup by UUID.
*/
using CollectionUUID = UUID;
+class Database;
class UUIDCatalog {
MONGO_DISALLOW_COPYING(UUIDCatalog);
public:
- static const ServiceContext::Decoration<UUIDCatalog> get;
+ static UUIDCatalog& get(ServiceContext* svcCtx);
+ static UUIDCatalog& get(OperationContext* opCtx);
UUIDCatalog() = default;
- /* This function inserts the entry for uuid, coll into the UUID
- * Collection. It is called by the op observer when a collection
- * is created.
+ /**
+ * This function inserts the entry for uuid, coll into the UUID Collection. It is called by
+ * the op observer when a collection is created.
*/
void onCreateCollection(OperationContext* opCtx, Collection* coll, CollectionUUID uuid);
+ /**
+ * This function removes the entry for uuid from the UUID catalog. It is called by the op
+ * observer when a collection is dropped.
+ */
+ void onDropCollection(OperationContext* opCtx, CollectionUUID uuid);
+
+ /**
+ * Combination of onDropCollection and onCreateCollection.
+ */
+ void onRenameCollection(OperationContext* opCtx, Collection* newColl, CollectionUUID uuid);
+
+ /**
+ * Implies onDropCollection for all collections in db, but is not transactional.
+ */
+ void onCloseDatabase(Database* db);
+
/* This function gets the Collection* pointer that corresponds to
* CollectionUUID uuid. The required locks should be obtained prior
* to calling this function, or else the found Collection pointer
@@ -69,11 +87,6 @@ public:
*/
NamespaceString lookupNSSByUUID(CollectionUUID uuid) const;
- /* This function removes the entry for uuid from the UUID catalog. It
- * is called by the op observer when a collection is dropped.
- */
- void onDropCollection(OperationContext* opCtx, CollectionUUID uuid);
-
private:
mutable mongo::stdx::mutex _catalogLock;
mongo::stdx::unordered_map<CollectionUUID, Collection*, CollectionUUID::Hash> _catalog;
diff --git a/src/mongo/util/uuid_catalog_test.cpp b/src/mongo/db/catalog/uuid_catalog_test.cpp
index 65720343f45..cbcf9c2ec13 100644
--- a/src/mongo/util/uuid_catalog_test.cpp
+++ b/src/mongo/db/catalog/uuid_catalog_test.cpp
@@ -26,7 +26,7 @@
* it in the license file.
*/
-#include "mongo/util/uuid_catalog.h"
+#include "mongo/db/catalog/uuid_catalog.h"
#include "mongo/db/catalog/collection_mock.h"
#include "mongo/db/operation_context_noop.h"
diff --git a/src/mongo/db/commands.cpp b/src/mongo/db/commands.cpp
index dd75b69c6be..ebf796282e6 100644
--- a/src/mongo/db/commands.cpp
+++ b/src/mongo/db/commands.cpp
@@ -43,6 +43,7 @@
#include "mongo/db/auth/authorization_manager.h"
#include "mongo/db/auth/authorization_session.h"
#include "mongo/db/auth/privilege.h"
+#include "mongo/db/catalog/uuid_catalog.h"
#include "mongo/db/client.h"
#include "mongo/db/curop.h"
#include "mongo/db/jsobj.h"
@@ -50,7 +51,6 @@
#include "mongo/db/server_parameters.h"
#include "mongo/rpc/write_concern_error_detail.h"
#include "mongo/util/log.h"
-#include "mongo/util/uuid_catalog.h"
namespace mongo {
@@ -108,7 +108,7 @@ NamespaceString Command::parseNsOrUUID(OperationContext* opCtx,
if (first.type() == BinData && first.binDataType() == BinDataType::newUUID) {
StatusWith<UUID> uuidRes = UUID::parse(first);
uassertStatusOK(uuidRes);
- UUIDCatalog& catalog = UUIDCatalog::get(opCtx->getServiceContext());
+ UUIDCatalog& catalog = UUIDCatalog::get(opCtx);
return catalog.lookupNSSByUUID(uuidRes.getValue());
} else {
// Ensure collection identifier is not a Command or specialCommand
diff --git a/src/mongo/db/commands.h b/src/mongo/db/commands.h
index 6a7abae5879..6ad9e536b47 100644
--- a/src/mongo/db/commands.h
+++ b/src/mongo/db/commands.h
@@ -284,7 +284,7 @@ private:
* Serves as a base for server commands. See the constructor for more details.
*/
class Command : public CommandInterface {
-protected:
+public:
// The type of the first field in 'cmdObj' must be mongo::String. The first field is
// interpreted as a collection name.
static std::string parseNsFullyQualified(const std::string& dbname, const BSONObj& cmdObj);
@@ -297,7 +297,6 @@ protected:
const std::string& dbname,
const BSONObj& cmdObj);
-public:
typedef StringMap<Command*> CommandMap;
/**
diff --git a/src/mongo/db/op_observer.h b/src/mongo/db/op_observer.h
index 00dc3ec324b..742ada49432 100644
--- a/src/mongo/db/op_observer.h
+++ b/src/mongo/db/op_observer.h
@@ -192,7 +192,8 @@ public:
OptionalCollectionUUID uuid) = 0;
virtual void onConvertToCapped(OperationContext* opCtx,
const NamespaceString& collectionName,
- OptionalCollectionUUID uuid,
+ OptionalCollectionUUID origUUID,
+ OptionalCollectionUUID cappedUUID,
double size) = 0;
};
diff --git a/src/mongo/db/op_observer_impl.cpp b/src/mongo/db/op_observer_impl.cpp
index d7b47edeb2f..bafe2bd07f6 100644
--- a/src/mongo/db/op_observer_impl.cpp
+++ b/src/mongo/db/op_observer_impl.cpp
@@ -33,6 +33,10 @@
#include "mongo/bson/bsonobjbuilder.h"
#include "mongo/db/auth/authorization_manager_global.h"
#include "mongo/db/catalog/collection_options.h"
+#include "mongo/db/catalog/database.h"
+#include "mongo/db/catalog/database_holder.h"
+#include "mongo/db/catalog/namespace_uuid_cache.h"
+#include "mongo/db/catalog/uuid_catalog.h"
#include "mongo/db/commands/feature_compatibility_version.h"
#include "mongo/db/index/index_descriptor.h"
#include "mongo/db/namespace_string.h"
@@ -42,8 +46,6 @@
#include "mongo/db/server_options.h"
#include "mongo/db/views/durable_view_catalog.h"
#include "mongo/scripting/engine.h"
-#include "mongo/util/namespace_uuid_cache.h"
-#include "mongo/util/uuid_catalog.h"
namespace mongo {
@@ -55,7 +57,7 @@ void OpObserverImpl::onCreateIndex(OperationContext* opCtx,
NamespaceString systemIndexes{nss.getSystemIndexesCollection()};
if (uuid) {
BSONObjBuilder builder;
- builder.append("createIndex", nss.coll());
+ builder.append("createIndexes", nss.coll());
for (const auto& e : indexDoc) {
if (e.fieldNameStringData() != "ns"_sd)
@@ -215,8 +217,11 @@ void OpObserverImpl::onCreateCollection(OperationContext* opCtx,
getGlobalAuthorizationManager()->logOp(opCtx, "c", dbName, cmdObj, nullptr);
if (options.uuid) {
- UUIDCatalog& catalog = UUIDCatalog::get(opCtx->getServiceContext());
+ UUIDCatalog& catalog = UUIDCatalog::get(opCtx);
catalog.onCreateCollection(opCtx, coll, options.uuid.get());
+ opCtx->recoveryUnit()->onRollback([opCtx, collectionName]() {
+ NamespaceUUIDCache::get(opCtx).evictNamespace(collectionName);
+ });
}
}
@@ -292,6 +297,8 @@ void OpObserverImpl::onDropDatabase(OperationContext* opCtx, const std::string&
FeatureCompatibilityVersion::onDropCollection();
}
+ NamespaceUUIDCache::get(opCtx).evictNamespacesInDatabase(dbName);
+
getGlobalAuthorizationManager()->logOp(opCtx, "c", cmdNss, cmdObj, nullptr);
}
@@ -321,12 +328,11 @@ repl::OpTime OpObserverImpl::onDropCollection(OperationContext* opCtx,
css->onDropCollection(opCtx, collectionName);
// Evict namespace entry from the namespace/uuid cache if it exists.
- NamespaceUUIDCache& cache = NamespaceUUIDCache::get(opCtx);
- cache.onDropCollection(collectionName);
+ NamespaceUUIDCache::get(opCtx).evictNamespace(collectionName);
// Remove collection from the uuid catalog.
if (uuid) {
- UUIDCatalog& catalog = UUIDCatalog::get(opCtx->getServiceContext());
+ UUIDCatalog& catalog = UUIDCatalog::get(opCtx);
catalog.onDropCollection(opCtx, uuid.get());
}
@@ -379,7 +385,20 @@ void OpObserverImpl::onRenameCollection(OperationContext* opCtx,
// Evict namespace entry from the namespace/uuid cache if it exists.
NamespaceUUIDCache& cache = NamespaceUUIDCache::get(opCtx);
- cache.onRenameCollection(fromCollection);
+ cache.evictNamespace(fromCollection);
+ cache.evictNamespace(toCollection);
+ opCtx->recoveryUnit()->onRollback(
+ [&cache, toCollection]() { cache.evictNamespace(toCollection); });
+
+
+ // Finally update the UUID Catalog.
+ if (uuid) {
+ auto db = dbHolder().get(opCtx, toCollection.db());
+ auto newColl = db->getCollection(opCtx, toCollection);
+ invariant(newColl);
+ UUIDCatalog& catalog = UUIDCatalog::get(opCtx);
+ catalog.onRenameCollection(opCtx, newColl, uuid.get());
+ }
}
void OpObserverImpl::onApplyOps(OperationContext* opCtx,
@@ -393,14 +412,33 @@ void OpObserverImpl::onApplyOps(OperationContext* opCtx,
void OpObserverImpl::onConvertToCapped(OperationContext* opCtx,
const NamespaceString& collectionName,
- OptionalCollectionUUID uuid,
+ OptionalCollectionUUID origUUID,
+ OptionalCollectionUUID cappedUUID,
double size) {
const NamespaceString cmdNss = collectionName.getCommandNS();
BSONObj cmdObj = BSON("convertToCapped" << collectionName.coll() << "size" << size);
if (!collectionName.isSystemDotProfile()) {
// do not replicate system.profile modifications
- repl::logOp(opCtx, "c", cmdNss, uuid, cmdObj, nullptr, false);
+ repl::logOp(opCtx, "c", cmdNss, cappedUUID, cmdObj, nullptr, false);
+ }
+
+ // Evict namespace entry from the namespace/uuid cache if it exists.
+ NamespaceUUIDCache& cache = NamespaceUUIDCache::get(opCtx);
+ cache.evictNamespace(collectionName);
+ opCtx->recoveryUnit()->onRollback(
+ [&cache, collectionName]() { cache.evictNamespace(collectionName); });
+
+ // Finally update the UUID Catalog.
+ UUIDCatalog& catalog = UUIDCatalog::get(opCtx);
+ if (origUUID) {
+ catalog.onDropCollection(opCtx, origUUID.get());
+ }
+ if (cappedUUID) {
+ auto db = dbHolder().get(opCtx, collectionName.db());
+ auto newColl = db->getCollection(opCtx, collectionName);
+ invariant(newColl);
+ catalog.onRenameCollection(opCtx, newColl, cappedUUID.get());
}
getGlobalAuthorizationManager()->logOp(opCtx, "c", cmdNss, cmdObj, nullptr);
diff --git a/src/mongo/db/op_observer_impl.h b/src/mongo/db/op_observer_impl.h
index 730ccfb1f84..476711a86f5 100644
--- a/src/mongo/db/op_observer_impl.h
+++ b/src/mongo/db/op_observer_impl.h
@@ -96,7 +96,8 @@ public:
OptionalCollectionUUID uuid);
void onConvertToCapped(OperationContext* opCtx,
const NamespaceString& collectionName,
- OptionalCollectionUUID uuid,
+ OptionalCollectionUUID origUUID,
+ OptionalCollectionUUID cappedUUID,
double size) override;
};
diff --git a/src/mongo/db/op_observer_noop.cpp b/src/mongo/db/op_observer_noop.cpp
index 310df4c5424..10901aa4cd0 100644
--- a/src/mongo/db/op_observer_noop.cpp
+++ b/src/mongo/db/op_observer_noop.cpp
@@ -28,6 +28,11 @@
#include "mongo/platform/basic.h"
+#include "mongo/db/catalog/collection.h"
+#include "mongo/db/catalog/database.h"
+#include "mongo/db/catalog/database_holder.h"
+#include "mongo/db/catalog/namespace_uuid_cache.h"
+#include "mongo/db/catalog/uuid_catalog.h"
#include "mongo/db/op_observer_noop.h"
namespace mongo {
@@ -58,11 +63,19 @@ void OpObserverNoop::onDelete(OperationContext*,
void OpObserverNoop::onOpMessage(OperationContext*, const BSONObj&) {}
-void OpObserverNoop::onCreateCollection(OperationContext*,
- Collection*,
- const NamespaceString&,
- const CollectionOptions&,
- const BSONObj&) {}
+void OpObserverNoop::onCreateCollection(OperationContext* opCtx,
+ Collection* coll,
+ const NamespaceString& collectionName,
+ const CollectionOptions& options,
+ const BSONObj& idIndex) {
+ if (options.uuid) {
+ UUIDCatalog& catalog = UUIDCatalog::get(opCtx);
+ catalog.onCreateCollection(opCtx, coll, options.uuid.get());
+ NamespaceUUIDCache& cache = NamespaceUUIDCache::get(opCtx);
+ opCtx->recoveryUnit()->onRollback(
+ [&cache, collectionName]() { cache.evictNamespace(collectionName); });
+ }
+}
void OpObserverNoop::onCollMod(OperationContext*,
const NamespaceString&,
@@ -73,9 +86,19 @@ void OpObserverNoop::onCollMod(OperationContext*,
void OpObserverNoop::onDropDatabase(OperationContext*, const std::string&) {}
-repl::OpTime OpObserverNoop::onDropCollection(OperationContext*,
- const NamespaceString&,
- OptionalCollectionUUID) {
+repl::OpTime OpObserverNoop::onDropCollection(OperationContext* opCtx,
+ const NamespaceString& collectionName,
+ OptionalCollectionUUID uuid) {
+ // Evict namespace entry from the namespace/uuid cache if it exists.
+ NamespaceUUIDCache& cache = NamespaceUUIDCache::get(opCtx);
+ cache.evictNamespace(collectionName);
+
+ // Remove collection from the uuid catalog.
+ if (uuid) {
+ UUIDCatalog& catalog = UUIDCatalog::get(opCtx);
+ catalog.onDropCollection(opCtx, uuid.get());
+ }
+
return {};
}
@@ -85,20 +108,37 @@ void OpObserverNoop::onDropIndex(OperationContext*,
const std::string&,
const BSONObj&) {}
-void OpObserverNoop::onRenameCollection(OperationContext*,
- const NamespaceString&,
- const NamespaceString&,
- OptionalCollectionUUID,
- bool,
- OptionalCollectionUUID,
- OptionalCollectionUUID,
- bool) {}
+void OpObserverNoop::onRenameCollection(OperationContext* opCtx,
+ const NamespaceString& fromCollection,
+ const NamespaceString& toCollection,
+ OptionalCollectionUUID uuid,
+ bool dropTarget,
+ OptionalCollectionUUID dropTargetUUID,
+ OptionalCollectionUUID dropSourceUUID,
+ bool stayTemp) {
+ // Evict namespace entry from the namespace/uuid cache if it exists.
+ NamespaceUUIDCache& cache = NamespaceUUIDCache::get(opCtx);
+ cache.evictNamespace(fromCollection);
+ cache.evictNamespace(toCollection);
+ opCtx->recoveryUnit()->onRollback(
+ [&cache, toCollection]() { cache.evictNamespace(toCollection); });
+
+ // Finally update the UUID Catalog.
+ if (uuid) {
+ auto db = dbHolder().get(opCtx, toCollection.db());
+ auto newColl = db->getCollection(opCtx, toCollection);
+ invariant(newColl);
+ UUIDCatalog& catalog = UUIDCatalog::get(opCtx);
+ catalog.onRenameCollection(opCtx, newColl, uuid.get());
+ }
+}
void OpObserverNoop::onApplyOps(OperationContext*, const std::string&, const BSONObj&) {}
void OpObserverNoop::onConvertToCapped(OperationContext*,
const NamespaceString&,
OptionalCollectionUUID,
+ OptionalCollectionUUID,
double) {}
void OpObserverNoop::onEmptyCapped(OperationContext*,
diff --git a/src/mongo/db/op_observer_noop.h b/src/mongo/db/op_observer_noop.h
index d7e9e3dee24..393b707f2c9 100644
--- a/src/mongo/db/op_observer_noop.h
+++ b/src/mongo/db/op_observer_noop.h
@@ -96,7 +96,8 @@ public:
OptionalCollectionUUID uuid);
void onConvertToCapped(OperationContext* opCtx,
const NamespaceString& collectionName,
- OptionalCollectionUUID uuid,
+ OptionalCollectionUUID origUUID,
+ OptionalCollectionUUID cappedUUID,
double size) override;
};
diff --git a/src/mongo/db/repair_database.cpp b/src/mongo/db/repair_database.cpp
index 70405da7928..771c6278137 100644
--- a/src/mongo/db/repair_database.cpp
+++ b/src/mongo/db/repair_database.cpp
@@ -44,6 +44,8 @@
#include "mongo/db/catalog/document_validation.h"
#include "mongo/db/catalog/index_create.h"
#include "mongo/db/catalog/index_key_validate.h"
+#include "mongo/db/catalog/namespace_uuid_cache.h"
+#include "mongo/db/catalog/uuid_catalog.h"
#include "mongo/db/index/index_descriptor.h"
#include "mongo/db/repl/replication_coordinator.h"
#include "mongo/db/storage/mmap_v1/mmap_v1_engine.h"
@@ -242,7 +244,15 @@ Status repairDatabase(OperationContext* opCtx,
return Status(ErrorCodes::BadValue, "backupOriginalFiles not supported");
}
- // Close the db to invalidate all current users and caches.
+ // Close the db and invalidate all current users and caches.
+ {
+ Database* db = dbHolder().get(opCtx, dbName);
+ if (db) {
+ UUIDCatalog::get(opCtx).onCloseDatabase(db);
+ for (auto&& coll : *db)
+ NamespaceUUIDCache::get(opCtx).evictNamespace(coll->ns());
+ }
+ }
dbHolder().close(opCtx, dbName, "database closed for repair");
ON_BLOCK_EXIT([&dbName, &opCtx] {
try {
diff --git a/src/mongo/db/repl/oplog.cpp b/src/mongo/db/repl/oplog.cpp
index b6d7249e6dc..9baa4b211cc 100644
--- a/src/mongo/db/repl/oplog.cpp
+++ b/src/mongo/db/repl/oplog.cpp
@@ -54,6 +54,7 @@
#include "mongo/db/catalog/drop_database.h"
#include "mongo/db/catalog/drop_indexes.h"
#include "mongo/db/catalog/rename_collection.h"
+#include "mongo/db/catalog/uuid_catalog.h"
#include "mongo/db/client.h"
#include "mongo/db/commands.h"
#include "mongo/db/concurrency/write_conflict_exception.h"
@@ -567,8 +568,72 @@ NamespaceString parseNs(const string& ns, const BSONObj& cmdObj) {
return NamespaceString(NamespaceString(ns).db().toString(), coll);
}
-using OpApplyFn = stdx::function<Status(
- OperationContext* opCtx, const char* ns, BSONObj& cmd, const OpTime& opTime)>;
+NamespaceString parseUUID(OperationContext* opCtx, const BSONElement& ui) {
+ auto statusWithUUID = UUID::parse(ui);
+ uassertStatusOK(statusWithUUID);
+ auto uuid = statusWithUUID.getValue();
+ auto& catalog = UUIDCatalog::get(opCtx);
+ auto nss = catalog.lookupNSSByUUID(uuid);
+ uassert(
+ ErrorCodes::NamespaceNotFound, "No namespace with UUID " + uuid.toString(), !nss.isEmpty());
+ return nss;
+}
+
+NamespaceString parseUUIDorNs(OperationContext* opCtx,
+ const char* ns,
+ const BSONElement& ui,
+ BSONObj& cmd) {
+ return ui.ok() ? parseUUID(opCtx, ui) : parseNs(ns, cmd);
+}
+
+void createIndexForApplyOps(OperationContext* opCtx,
+ const BSONObj& indexSpec,
+ const NamespaceString& indexNss,
+ IncrementOpsAppliedStatsFn incrementOpsAppliedStats) {
+ // Check if collection exists.
+ Database* db = dbHolder().get(opCtx, indexNss.ns());
+ auto indexCollection = db ? db->getCollection(opCtx, indexNss) : nullptr;
+ uassert(ErrorCodes::NamespaceNotFound,
+ str::stream() << "Failed to create index due to missing collection: " << indexNss.ns(),
+ indexCollection);
+
+ OpCounters* opCounters = opCtx->writesAreReplicated() ? &globalOpCounters : &replOpCounters;
+ opCounters->gotInsert();
+
+ bool relaxIndexConstraints =
+ ReplicationCoordinator::get(opCtx)->shouldRelaxIndexConstraints(opCtx, indexNss);
+ if (indexSpec["background"].trueValue()) {
+ Lock::TempRelease release(opCtx->lockState());
+ if (opCtx->lockState()->isLocked()) {
+ // If TempRelease fails, background index build will deadlock.
+ LOG(3) << "apply op: building background index " << indexSpec
+ << " in the foreground because temp release failed";
+ IndexBuilder builder(indexSpec, relaxIndexConstraints);
+ Status status = builder.buildInForeground(opCtx, db);
+ uassertStatusOK(status);
+ } else {
+ IndexBuilder* builder = new IndexBuilder(indexSpec, relaxIndexConstraints);
+ // This spawns a new thread and returns immediately.
+ builder->go();
+ // Wait for thread to start and register itself
+ IndexBuilder::waitForBgIndexStarting();
+ }
+ opCtx->recoveryUnit()->abandonSnapshot();
+ } else {
+ IndexBuilder builder(indexSpec, relaxIndexConstraints);
+ Status status = builder.buildInForeground(opCtx, db);
+ uassertStatusOK(status);
+ }
+ if (incrementOpsAppliedStats) {
+ incrementOpsAppliedStats();
+ }
+}
+
+using OpApplyFn = stdx::function<Status(OperationContext* opCtx,
+ const char* ns,
+ const BSONElement& ui,
+ BSONObj& cmd,
+ const OpTime& opTime)>;
struct ApplyOpMetadata {
OpApplyFn applyFunc;
@@ -586,13 +651,17 @@ struct ApplyOpMetadata {
std::map<std::string, ApplyOpMetadata> opsMap = {
{"create",
- {[](OperationContext* opCtx, const char* ns, BSONObj& cmd, const OpTime& opTime) -> Status {
+ {[](OperationContext* opCtx,
+ const char* ns,
+ const BSONElement& ui,
+ BSONObj& cmd,
+ const OpTime& opTime) -> Status {
const NamespaceString nss(parseNs(ns, cmd));
if (auto idIndexElem = cmd["idIndex"]) {
// Remove "idIndex" field from command.
auto cmdWithoutIdIndex = cmd.removeField("idIndex");
- return createCollection(
- opCtx, nss.db().toString(), cmdWithoutIdIndex, idIndexElem.Obj());
+ return createCollectionForApplyOps(
+ opCtx, nss.db().toString(), ui, cmdWithoutIdIndex, idIndexElem.Obj());
}
// No _id index spec was provided, so we should build a v:1 _id index.
@@ -602,25 +671,59 @@ std::map<std::string, ApplyOpMetadata> opsMap = {
idIndexSpecBuilder.append(IndexDescriptor::kIndexNameFieldName, "_id_");
idIndexSpecBuilder.append(IndexDescriptor::kNamespaceFieldName, nss.ns());
idIndexSpecBuilder.append(IndexDescriptor::kKeyPatternFieldName, BSON("_id" << 1));
- return createCollection(opCtx, nss.db().toString(), cmd, idIndexSpecBuilder.done());
+ return createCollectionForApplyOps(
+ opCtx, nss.db().toString(), ui, cmd, idIndexSpecBuilder.done());
},
{ErrorCodes::NamespaceExists}}},
+ {"createIndexes",
+ {[](OperationContext* opCtx,
+ const char* ns,
+ const BSONElement& ui,
+ BSONObj& cmd,
+ const OpTime& opTime) -> Status {
+ const NamespaceString nss(parseUUID(opCtx, ui));
+ BSONElement first = cmd.firstElement();
+ invariant(first.fieldNameStringData() == "createIndexes");
+ uassert(ErrorCodes::InvalidNamespace,
+ "createIndexes value must be a string",
+ first.type() == mongo::String);
+ BSONObj indexSpec = cmd.removeField("createIndexes");
+ // The UUID determines the collection to build the index on, so create new 'ns' field.
+ BSONObj nsObj = BSON("ns" << nss.ns());
+ indexSpec = indexSpec.addField(nsObj.firstElement());
+
+ createIndexForApplyOps(opCtx, indexSpec, nss, {});
+ return Status::OK();
+ },
+ {ErrorCodes::IndexAlreadyExists}}},
{"collMod",
- {[](OperationContext* opCtx, const char* ns, BSONObj& cmd, const OpTime& opTime) -> Status {
+ {[](OperationContext* opCtx,
+ const char* ns,
+ const BSONElement& ui,
+ BSONObj& cmd,
+ const OpTime& opTime) -> Status {
BSONObjBuilder resultWeDontCareAbout;
- return collMod(opCtx, parseNs(ns, cmd), cmd, &resultWeDontCareAbout);
+ return collMod(opCtx, parseUUIDorNs(opCtx, ns, ui, cmd), cmd, &resultWeDontCareAbout);
},
{ErrorCodes::IndexNotFound, ErrorCodes::NamespaceNotFound}}},
{"dropDatabase",
- {[](OperationContext* opCtx, const char* ns, BSONObj& cmd, const OpTime& opTime) -> Status {
+ {[](OperationContext* opCtx,
+ const char* ns,
+ const BSONElement& ui,
+ BSONObj& cmd,
+ const OpTime& opTime) -> Status {
return dropDatabase(opCtx, NamespaceString(ns).db().toString());
},
{ErrorCodes::NamespaceNotFound}}},
{"drop",
- {[](OperationContext* opCtx, const char* ns, BSONObj& cmd, const OpTime& opTime) -> Status {
+ {[](OperationContext* opCtx,
+ const char* ns,
+ const BSONElement& ui,
+ BSONObj& cmd,
+ const OpTime& opTime) -> Status {
BSONObjBuilder resultWeDontCareAbout;
return dropCollection(opCtx,
- parseNs(ns, cmd),
+ parseUUIDorNs(opCtx, ns, ui, cmd),
resultWeDontCareAbout,
opTime,
DropCollectionSystemCollectionMode::kAllowSystemCollectionDrops);
@@ -628,59 +731,79 @@ std::map<std::string, ApplyOpMetadata> opsMap = {
{ErrorCodes::NamespaceNotFound}}},
// deleteIndex(es) is deprecated but still works as of April 10, 2015
{"deleteIndex",
- {[](OperationContext* opCtx, const char* ns, BSONObj& cmd, const OpTime& opTime) -> Status {
+ {[](OperationContext* opCtx,
+ const char* ns,
+ const BSONElement& ui,
+ BSONObj& cmd,
+ const OpTime& opTime) -> Status {
BSONObjBuilder resultWeDontCareAbout;
- return dropIndexes(opCtx, parseNs(ns, cmd), cmd, &resultWeDontCareAbout);
+ return dropIndexes(opCtx, parseUUIDorNs(opCtx, ns, ui, cmd), cmd, &resultWeDontCareAbout);
},
{ErrorCodes::NamespaceNotFound, ErrorCodes::IndexNotFound}}},
{"deleteIndexes",
- {[](OperationContext* opCtx, const char* ns, BSONObj& cmd, const OpTime& opTime) -> Status {
+ {[](OperationContext* opCtx,
+ const char* ns,
+ const BSONElement& ui,
+ BSONObj& cmd,
+ const OpTime& opTime) -> Status {
BSONObjBuilder resultWeDontCareAbout;
- return dropIndexes(opCtx, parseNs(ns, cmd), cmd, &resultWeDontCareAbout);
+ return dropIndexes(opCtx, parseUUIDorNs(opCtx, ns, ui, cmd), cmd, &resultWeDontCareAbout);
},
{ErrorCodes::NamespaceNotFound, ErrorCodes::IndexNotFound}}},
{"dropIndex",
- {[](OperationContext* opCtx, const char* ns, BSONObj& cmd, const OpTime& opTime) -> Status {
+ {[](OperationContext* opCtx,
+ const char* ns,
+ const BSONElement& ui,
+ BSONObj& cmd,
+ const OpTime& opTime) -> Status {
BSONObjBuilder resultWeDontCareAbout;
- return dropIndexes(opCtx, parseNs(ns, cmd), cmd, &resultWeDontCareAbout);
+ return dropIndexes(opCtx, parseUUIDorNs(opCtx, ns, ui, cmd), cmd, &resultWeDontCareAbout);
},
{ErrorCodes::NamespaceNotFound, ErrorCodes::IndexNotFound}}},
{"dropIndexes",
- {[](OperationContext* opCtx, const char* ns, BSONObj& cmd, const OpTime& opTime) -> Status {
+ {[](OperationContext* opCtx,
+ const char* ns,
+ const BSONElement& ui,
+ BSONObj& cmd,
+ const OpTime& opTime) -> Status {
BSONObjBuilder resultWeDontCareAbout;
- return dropIndexes(opCtx, parseNs(ns, cmd), cmd, &resultWeDontCareAbout);
+ return dropIndexes(opCtx, parseUUIDorNs(opCtx, ns, ui, cmd), cmd, &resultWeDontCareAbout);
},
{ErrorCodes::NamespaceNotFound, ErrorCodes::IndexNotFound}}},
{"renameCollection",
- {[](OperationContext* opCtx, const char* ns, BSONObj& cmd, const OpTime& opTime) -> Status {
- const auto sourceNsElt = cmd.firstElement();
- const auto targetNsElt = cmd["to"];
- uassert(ErrorCodes::TypeMismatch,
- "'renameCollection' must be of type String",
- sourceNsElt.type() == BSONType::String);
- uassert(ErrorCodes::TypeMismatch,
- "'to' must be of type String",
- targetNsElt.type() == BSONType::String);
- return renameCollection(opCtx,
- NamespaceString(sourceNsElt.valueStringData()),
- NamespaceString(targetNsElt.valueStringData()),
- cmd["dropTarget"].trueValue(),
- cmd["stayTemp"].trueValue());
+ {[](OperationContext* opCtx,
+ const char* ns,
+ const BSONElement& ui,
+ BSONObj& cmd,
+ const OpTime& opTime) -> Status {
+ return renameCollectionForApplyOps(opCtx, nsToDatabase(ns), ui, cmd);
},
{ErrorCodes::NamespaceNotFound, ErrorCodes::NamespaceExists}}},
{"applyOps",
- {[](OperationContext* opCtx, const char* ns, BSONObj& cmd, const OpTime& opTime) -> Status {
+ {[](OperationContext* opCtx,
+ const char* ns,
+ const BSONElement& ui,
+ BSONObj& cmd,
+ const OpTime& opTime) -> Status {
BSONObjBuilder resultWeDontCareAbout;
return applyOps(opCtx, nsToDatabase(ns), cmd, &resultWeDontCareAbout);
},
{ErrorCodes::UnknownError}}},
{"convertToCapped",
- {[](OperationContext* opCtx, const char* ns, BSONObj& cmd, const OpTime& opTime) -> Status {
- return convertToCapped(opCtx, parseNs(ns, cmd), cmd["size"].number());
+ {[](OperationContext* opCtx,
+ const char* ns,
+ const BSONElement& ui,
+ BSONObj& cmd,
+ const OpTime& opTime) -> Status {
+ return convertToCapped(opCtx, parseUUIDorNs(opCtx, ns, ui, cmd), cmd["size"].number());
}}},
{"emptycapped",
- {[](OperationContext* opCtx, const char* ns, BSONObj& cmd, const OpTime& opTime) -> Status {
- return emptyCapped(opCtx, parseNs(ns, cmd));
+ {[](OperationContext* opCtx,
+ const char* ns,
+ const BSONElement& ui,
+ BSONObj& cmd,
+ const OpTime& opTime) -> Status {
+ return emptyCapped(opCtx, parseUUIDorNs(opCtx, ns, ui, cmd));
}}},
};
@@ -736,24 +859,54 @@ Status applyOperation_inlock(OperationContext* opCtx,
OpCounters* opCounters = opCtx->writesAreReplicated() ? &globalOpCounters : &replOpCounters;
- const char* names[] = {"o", "ns", "op", "b", "o2"};
- BSONElement fields[5];
- op.getFields(5, names, fields);
+ std::array<StringData, 6> names = {"o", "ui", "ns", "op", "b", "o2"};
+ std::array<BSONElement, 6> fields;
+ op.getFields(names, &fields);
BSONElement& fieldO = fields[0];
- BSONElement& fieldNs = fields[1];
- BSONElement& fieldOp = fields[2];
- BSONElement& fieldB = fields[3];
- BSONElement& fieldO2 = fields[4];
+ BSONElement& fieldUI = fields[1];
+ BSONElement& fieldNs = fields[2];
+ BSONElement& fieldOp = fields[3];
+ BSONElement& fieldB = fields[4];
+ BSONElement& fieldO2 = fields[5];
BSONObj o;
if (fieldO.isABSONObj())
o = fieldO.embeddedObject();
- uassert(ErrorCodes::InvalidNamespace,
- "'ns' must be of type String",
- fieldNs.type() == BSONType::String);
- const StringData ns = fieldNs.valueStringData();
- NamespaceString requestNss{ns};
+ // operation type -- see logOp() comments for types
+ const char* opType = fieldOp.valuestrsafe();
+
+ NamespaceString requestNss;
+ Collection* collection = nullptr;
+ if (fieldUI) {
+ UUIDCatalog& catalog = UUIDCatalog::get(opCtx);
+ auto uuid = UUID::parse(fieldUI);
+ uassertStatusOK(uuid);
+ collection = catalog.lookupCollectionByUUID(uuid.getValue());
+ if (collection) {
+ requestNss = collection->ns();
+ dassert(opCtx->lockState()->isCollectionLockedForMode(
+ requestNss.ns(), supportsDocLocking() ? MODE_IX : MODE_X));
+ }
+ } else {
+ uassert(ErrorCodes::InvalidNamespace,
+ "'ns' must be of type String",
+ fieldNs.type() == BSONType::String);
+ const StringData ns = fieldNs.valueStringData();
+ requestNss = NamespaceString(ns);
+ if (nsIsFull(ns)) {
+ if (supportsDocLocking()) {
+ // WiredTiger, and others requires MODE_IX since the applier threads driving
+ // this allow writes to the same collection on any thread.
+ dassert(opCtx->lockState()->isCollectionLockedForMode(ns, MODE_IX));
+ } else {
+ // mmapV1 ensures that all operations to the same collection are executed from
+ // the same worker thread, so it takes an exclusive lock (MODE_X)
+ dassert(opCtx->lockState()->isCollectionLockedForMode(ns, MODE_X));
+ }
+ }
+ collection = db->getCollection(opCtx, requestNss);
+ }
BSONObj o2;
if (fieldO2.isABSONObj())
@@ -761,27 +914,11 @@ Status applyOperation_inlock(OperationContext* opCtx,
bool valueB = fieldB.booleanSafe();
- if (nsIsFull(ns)) {
- if (supportsDocLocking()) {
- // WiredTiger, and others requires MODE_IX since the applier threads driving
- // this allow writes to the same collection on any thread.
- dassert(opCtx->lockState()->isCollectionLockedForMode(ns, MODE_IX));
- } else {
- // mmapV1 ensures that all operations to the same collection are executed from
- // the same worker thread, so it takes an exclusive lock (MODE_X)
- dassert(opCtx->lockState()->isCollectionLockedForMode(ns, MODE_X));
- }
- }
- Collection* collection = db->getCollection(opCtx, requestNss);
IndexCatalog* indexCatalog = collection == nullptr ? nullptr : collection->getIndexCatalog();
const bool haveWrappingWriteUnitOfWork = opCtx->lockState()->inAWriteUnitOfWork();
uassert(ErrorCodes::CommandNotSupportedOnView,
- str::stream() << "applyOps not supported on view: " << ns,
- collection || !db->getViewCatalog()->lookup(opCtx, ns));
-
- // operation type -- see logOp() comments for types
- const char* opType = fieldOp.valuestrsafe();
- invariant(*opType != 'c'); // commands are processed in applyCommand_inlock()
+ str::stream() << "applyOps not supported on view: " << requestNss.ns(),
+ collection || !db->getViewCatalog()->lookup(opCtx, requestNss.ns()));
if (*opType == 'i') {
if (requestNss.isSystemDotIndexes()) {
@@ -789,44 +926,7 @@ Status applyOperation_inlock(OperationContext* opCtx,
NamespaceString indexNss;
std::tie(indexSpec, indexNss) =
repl::prepForApplyOpsIndexInsert(fieldO, op, requestNss);
-
- // Check if collection exists.
- auto indexCollection = db->getCollection(opCtx, indexNss);
- uassert(ErrorCodes::NamespaceNotFound,
- str::stream() << "Failed to create index due to missing collection: "
- << op.toString(),
- indexCollection);
-
- opCounters->gotInsert();
-
- bool relaxIndexConstraints =
- ReplicationCoordinator::get(opCtx)->shouldRelaxIndexConstraints(opCtx, indexNss);
- if (indexSpec["background"].trueValue()) {
- Lock::TempRelease release(opCtx->lockState());
- if (opCtx->lockState()->isLocked()) {
- // If TempRelease fails, background index build will deadlock.
- LOG(3) << "apply op: building background index " << indexSpec
- << " in the foreground because temp release failed";
- IndexBuilder builder(indexSpec, relaxIndexConstraints);
- Status status = builder.buildInForeground(opCtx, db);
- uassertStatusOK(status);
- } else {
- IndexBuilder* builder = new IndexBuilder(indexSpec, relaxIndexConstraints);
- // This spawns a new thread and returns immediately.
- builder->go();
- // Wait for thread to start and register itself
- IndexBuilder::waitForBgIndexStarting();
- }
- opCtx->recoveryUnit()->abandonSnapshot();
- } else {
- IndexBuilder builder(indexSpec, relaxIndexConstraints);
- Status status = builder.buildInForeground(opCtx, db);
- uassertStatusOK(status);
- }
- // Since this is an index operation we can return without falling through.
- if (incrementOpsAppliedStats) {
- incrementOpsAppliedStats();
- }
+ createIndexForApplyOps(opCtx, indexSpec, indexNss, incrementOpsAppliedStats);
return Status::OK();
}
uassert(ErrorCodes::NamespaceNotFound,
@@ -1005,6 +1105,7 @@ Status applyOperation_inlock(OperationContext* opCtx,
incrementOpsAppliedStats();
}
} else {
+ invariant(*opType != 'c'); // commands are processed in applyCommand_inlock()
throw MsgAssertionException(
14825, str::stream() << "error in applyOperation : unknown opType " << *opType);
}
@@ -1015,12 +1116,13 @@ Status applyOperation_inlock(OperationContext* opCtx,
Status applyCommand_inlock(OperationContext* opCtx,
const BSONObj& op,
bool inSteadyStateReplication) {
- const char* names[] = {"o", "ns", "op"};
- BSONElement fields[3];
- op.getFields(3, names, fields);
+ std::array<StringData, 4> names = {"o", "ui", "ns", "op"};
+ std::array<BSONElement, 4> fields;
+ op.getFields(names, &fields);
BSONElement& fieldO = fields[0];
- BSONElement& fieldNs = fields[1];
- BSONElement& fieldOp = fields[2];
+ BSONElement& fieldUI = fields[1];
+ BSONElement& fieldNs = fields[2];
+ BSONElement& fieldOp = fields[3];
const char* opType = fieldOp.valuestrsafe();
invariant(*opType == 'c'); // only commands are processed here
@@ -1084,7 +1186,7 @@ Status applyCommand_inlock(OperationContext* opCtx,
ApplyOpMetadata curOpToApply = op->second;
Status status = Status::OK();
try {
- status = curOpToApply.applyFunc(opCtx, nss.ns().c_str(), o, opTime);
+ status = curOpToApply.applyFunc(opCtx, nss.ns().c_str(), fieldUI, o, opTime);
} catch (...) {
status = exceptionToStatus();
}
diff --git a/src/mongo/db/repl/oplog_entry.cpp b/src/mongo/db/repl/oplog_entry.cpp
index 130543160bf..d2e9416e407 100644
--- a/src/mongo/db/repl/oplog_entry.cpp
+++ b/src/mongo/db/repl/oplog_entry.cpp
@@ -58,8 +58,8 @@ OplogEntry::CommandType parseCommandType(const BSONObj& objectField) {
return OplogEntry::CommandType::kEmptyCapped;
} else if (commandString == "convertToCapped") {
return OplogEntry::CommandType::kConvertToCapped;
- } else if (commandString == "createIndex") {
- return OplogEntry::CommandType::kCreateIndex;
+ } else if (commandString == "createIndexes") {
+ return OplogEntry::CommandType::kCreateIndexes;
} else if (commandString == "dropIndexes") {
return OplogEntry::CommandType::kDropIndexes;
} else if (commandString == "deleteIndexes") {
diff --git a/src/mongo/db/repl/oplog_entry.h b/src/mongo/db/repl/oplog_entry.h
index 7e7d5e03603..51ae9df3d01 100644
--- a/src/mongo/db/repl/oplog_entry.h
+++ b/src/mongo/db/repl/oplog_entry.h
@@ -51,7 +51,7 @@ public:
kDropDatabase,
kEmptyCapped,
kConvertToCapped,
- kCreateIndex,
+ kCreateIndexes,
kDropIndexes
};
diff --git a/src/mongo/db/repl/rollback_fix_up_info_test.cpp b/src/mongo/db/repl/rollback_fix_up_info_test.cpp
index 54600e51fa5..cc72e9bd4cb 100644
--- a/src/mongo/db/repl/rollback_fix_up_info_test.cpp
+++ b/src/mongo/db/repl/rollback_fix_up_info_test.cpp
@@ -407,7 +407,7 @@ TEST_F(
RollbackFixUpInfoTest,
ProcessCreateCollectionOplogEntryInsertsDocumentIntoRollbackCollectionUuidCollectionWithEmptyNamespace) {
// State of oplog:
- // {create: mynewcoll}, {createIndex: myindex}, {collMod: mynewcoll}, {op: 'i'}, ....
+ // {create: mynewcoll}, {createIndexes: myindex}, {collMod: mynewcoll}, {op: 'i'}, ....
// (earliest optime) ---> (latest optime)
//
// Oplog entries are processed in reverse optime order.
@@ -685,12 +685,12 @@ TEST_F(RollbackFixUpInfoTest,
<< "ui"
<< UUID::gen()
<< "o"
- << BSON("createIndex" << 1 << "v" << 2 << "key" << BSON("b" << 1) << "name"
- << "b_1"
- << "ns"
- << "mydb.mycoll"
- << "expireAfterSeconds"
- << 60));
+ << BSON("createIndexes" << 1 << "v" << 2 << "key" << BSON("b" << 1) << "name"
+ << "b_1"
+ << "ns"
+ << "mydb.mycoll"
+ << "expireAfterSeconds"
+ << 60));
auto collectionUuid = unittest::assertGet(UUID::parse(operation["ui"]));
auto indexName = operation["o"].Obj()["name"].String();
@@ -716,7 +716,7 @@ TEST_F(RollbackFixUpInfoTest,
ProcessCreateIndexOplogEntryWhenExistingDocumentHasDropOpTypeRemovesExistingDocument) {
// State of oplog:
- // {createIndex: indexA}, ...., {dropIndexes: indexA}, ....
+ // {createIndexes: indexA}, ...., {dropIndexes: indexA}, ....
// (earliest optime) ---> (latest optime)
//
// Oplog entries are processed in reverse optime order.
@@ -741,7 +741,7 @@ TEST_F(RollbackFixUpInfoTest,
<< "infoObj"
<< infoObj)});
- // Next, process createIndex. This should cancel out the existing 'drop' operation and remove
+ // Next, process createIndexes. This should cancel out the existing 'drop' operation and remove
// existing document from the collection.
ASSERT_OK(
rollbackFixUpInfo.processCreateIndexOplogEntry(opCtx.get(), collectionUuid, indexName));
@@ -752,7 +752,7 @@ TEST_F(RollbackFixUpInfoTest,
ProcessCreateIndexOplogEntryWhenExistingDocumentHasUpdateTTLOpTypeReplacesExistingDocument) {
// State of oplog:
- // {createIndex: indexA}, ...., {collMod: indexA}, ....
+ // {createIndexes: indexA}, ...., {collMod: indexA}, ....
// (earliest optime) ---> (latest optime)
//
// Oplog entries are processed in reverse optime order.
@@ -775,7 +775,7 @@ TEST_F(RollbackFixUpInfoTest,
<< "infoObj"
<< BSON("expireAfterSeconds" << 60))});
- // Next, process createIndex. This should replace the existing 'updateTTL' operation so that
+ // Next, process createIndexes. This should replace the existing 'updateTTL' operation so that
// we drop the index when it's time to apply the fix up info.
ASSERT_OK(
rollbackFixUpInfo.processCreateIndexOplogEntry(opCtx.get(), collectionUuid, indexName));
@@ -812,7 +812,7 @@ TEST_F(
_assertDocumentsInCollectionEquals(
opCtx.get(), RollbackFixUpInfo::kRollbackIndexNamespace, {malformedDoc});
- // Process createIndex. This should log an error when checking the operation type on the
+ // Process createIndexes. This should log an error when checking the operation type on the
// existing document. The malformed document should be replaced.
ASSERT_OK(
rollbackFixUpInfo.processCreateIndexOplogEntry(opCtx.get(), collectionUuid, indexName));
@@ -1001,13 +1001,13 @@ TEST_F(RollbackFixUpInfoTest,
ProcessDropIndexOplogEntryWhenExistingDocumentHasCreateOpTypeReplacesExistingDocument) {
// State of oplog:
- // {dropIndexes: indexA}, ...., {createIndex: indexA}, ....
+ // {dropIndexes: indexA}, ...., {createIndexes: indexA}, ....
// (earliest optime) ---> (latest optime)
//
// Oplog entries are processed in reverse optime order.
- // First, process createIndex. This should insert a document into the collection with a 'create'
- // op type.
+ // First, process createIndexes. This should insert a document into the collection with a
+ // 'create' op type.
auto collectionUuid = UUID::gen();
std::string indexName = "b_1";
auto infoObj = BSON("v" << 2 << "key" << BSON("b" << 1) << "name" << indexName << "ns"
diff --git a/src/mongo/db/repl/storage_interface_impl.cpp b/src/mongo/db/repl/storage_interface_impl.cpp
index 8f1346f5d0b..7422b8c913d 100644
--- a/src/mongo/db/repl/storage_interface_impl.cpp
+++ b/src/mongo/db/repl/storage_interface_impl.cpp
@@ -49,6 +49,7 @@
#include "mongo/db/catalog/document_validation.h"
#include "mongo/db/catalog/index_catalog.h"
#include "mongo/db/catalog/index_create.h"
+#include "mongo/db/catalog/uuid_catalog.h"
#include "mongo/db/client.h"
#include "mongo/db/concurrency/d_concurrency.h"
#include "mongo/db/concurrency/write_conflict_exception.h"
@@ -432,9 +433,15 @@ Status StorageInterfaceImpl::renameCollection(OperationContext* opCtx,
WriteUnitOfWork wunit(opCtx);
const auto status =
autoDB.getDb()->renameCollection(opCtx, fromNS.ns(), toNS.ns(), stayTemp);
- if (status.isOK()) {
- wunit.commit();
+ if (!status.isOK()) {
+ return status;
+ }
+
+ auto newColl = autoDB.getDb()->getCollection(opCtx, toNS);
+ if (newColl->uuid()) {
+ UUIDCatalog::get(opCtx).onRenameCollection(opCtx, newColl, newColl->uuid().get());
}
+ wunit.commit();
return status;
}
MONGO_WRITE_CONFLICT_RETRY_LOOP_END(
diff --git a/src/mongo/db/repl/sync_tail.cpp b/src/mongo/db/repl/sync_tail.cpp
index c3d5bd30efb..a44591e130d 100644
--- a/src/mongo/db/repl/sync_tail.cpp
+++ b/src/mongo/db/repl/sync_tail.cpp
@@ -290,38 +290,10 @@ Status SyncTail::syncApply(OperationContext* opCtx,
// Count each log op application as a separate operation, for reporting purposes
CurOp individualOp(opCtx);
- const char* ns = op.getStringField("ns");
- verify(ns);
+ const NamespaceString nss(op.getStringField("ns"));
const char* opType = op["op"].valuestrsafe();
- bool isCommand(opType[0] == 'c');
- bool isNoOp(opType[0] == 'n');
-
- if ((*ns == '\0') || (*ns == '.')) {
- // this is ugly
- // this is often a no-op
- // but can't be 100% sure
- if (!isNoOp) {
- error() << "skipping bad op in oplog: " << redact(op);
- }
- return Status::OK();
- }
-
- if (isCommand) {
- MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN {
- // a command may need a global write lock. so we will conservatively go
- // ahead and grab one here. suboptimal. :-(
- Lock::GlobalWrite globalWriteLock(opCtx);
-
- // special case apply for commands to avoid implicit database creation
- Status status = applyCommandInLock(opCtx, op, inSteadyStateReplication);
- incrementOpsAppliedStats();
- return status;
- }
- MONGO_WRITE_CONFLICT_RETRY_LOOP_END(opCtx, "syncApply_command", ns);
- }
-
auto applyOp = [&](Database* db) {
// For non-initial-sync, we convert updates to upserts
// to suppress errors when replaying oplog entries.
@@ -336,14 +308,17 @@ Status SyncTail::syncApply(OperationContext* opCtx,
return status;
};
- if (isNoOp || (opType[0] == 'i' && nsToCollectionSubstring(ns) == "system.indexes")) {
+ bool isNoOp = opType[0] == 'n';
+ if (isNoOp || (opType[0] == 'i' && nss.isSystemDotIndexes())) {
auto opStr = isNoOp ? "syncApply_noop" : "syncApply_indexBuild";
+ if (isNoOp && nss.db() == "")
+ return Status::OK();
MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN {
- Lock::DBLock dbLock(opCtx, nsToDatabaseSubstring(ns), MODE_X);
- OldClientContext ctx(opCtx, ns);
+ Lock::DBLock dbLock(opCtx, nss.db(), MODE_X);
+ OldClientContext ctx(opCtx, nss.ns());
return applyOp(ctx.db());
}
- MONGO_WRITE_CONFLICT_RETRY_LOOP_END(opCtx, opStr, ns);
+ MONGO_WRITE_CONFLICT_RETRY_LOOP_END(opCtx, opStr, nss.ns());
}
if (isCrudOpType(opType)) {
@@ -353,7 +328,6 @@ Status SyncTail::syncApply(OperationContext* opCtx,
std::unique_ptr<Lock::CollectionLock> collectionLock;
std::unique_ptr<OldClientContext> ctx;
- NamespaceString nss(ns);
auto dbName = nss.db();
auto resetLocks = [&](LockMode mode) {
@@ -363,28 +337,42 @@ Status SyncTail::syncApply(OperationContext* opCtx,
// the upgraded one.
dbLock.reset();
dbLock.reset(new Lock::DBLock(opCtx, dbName, mode));
- collectionLock.reset(new Lock::CollectionLock(opCtx->lockState(), ns, mode));
+ collectionLock.reset(new Lock::CollectionLock(opCtx->lockState(), nss.ns(), mode));
};
resetLocks(MODE_IX);
if (!dbHolder().get(opCtx, dbName)) {
// Need to create database, so reset lock to stronger mode.
resetLocks(MODE_X);
- ctx.reset(new OldClientContext(opCtx, ns));
+ ctx.reset(new OldClientContext(opCtx, nss.ns()));
} else {
- ctx.reset(new OldClientContext(opCtx, ns));
+ ctx.reset(new OldClientContext(opCtx, nss.ns()));
if (!ctx->db()->getCollection(opCtx, nss)) {
// Need to implicitly create collection. This occurs for 'u' opTypes,
// but not for 'i' nor 'd'.
ctx.reset();
resetLocks(MODE_X);
- ctx.reset(new OldClientContext(opCtx, ns));
+ ctx.reset(new OldClientContext(opCtx, nss.ns()));
}
}
return applyOp(ctx->db());
}
- MONGO_WRITE_CONFLICT_RETRY_LOOP_END(opCtx, "syncApply_CRUD", ns);
+ MONGO_WRITE_CONFLICT_RETRY_LOOP_END(opCtx, "syncApply_CRUD", nss.ns());
+ }
+
+ if (opType[0] == 'c') {
+ MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN {
+ // a command may need a global write lock. so we will conservatively go
+ // ahead and grab one here. suboptimal. :-(
+ Lock::GlobalWrite globalWriteLock(opCtx);
+
+ // special case apply for commands to avoid implicit database creation
+ Status status = applyCommandInLock(opCtx, op, inSteadyStateReplication);
+ incrementOpsAppliedStats();
+ return status;
+ }
+ MONGO_WRITE_CONFLICT_RETRY_LOOP_END(opCtx, "syncApply_command", nss.ns());
}
// unknown opType
@@ -756,14 +744,17 @@ private:
void SyncTail::oplogApplication(ReplicationCoordinator* replCoord) {
OpQueueBatcher batcher(this);
- const ServiceContext::UniqueOperationContext opCtxPtr = cc().makeOperationContext();
- OperationContext& opCtx = *opCtxPtr;
std::unique_ptr<ApplyBatchFinalizer> finalizer{
getGlobalServiceContext()->getGlobalStorageEngine()->isDurable()
? new ApplyBatchFinalizerForJournal(replCoord)
: new ApplyBatchFinalizer(replCoord)};
while (true) { // Exits on message from OpQueueBatcher.
+ // Use a new operation context each iteration, as otherwise we may appear to use a single
+ // collection name to refer to collections with different UUIDs.
+ const ServiceContext::UniqueOperationContext opCtxPtr = cc().makeOperationContext();
+ OperationContext& opCtx = *opCtxPtr;
+
// For pausing replication in tests.
while (MONGO_FAIL_POINT(rsSyncApplyStop)) {
// Tests should not trigger clean shutdown while that failpoint is active. If we
diff --git a/src/mongo/db/repl/sync_tail_test.cpp b/src/mongo/db/repl/sync_tail_test.cpp
index 888640f96a3..9e1e8178bd7 100644
--- a/src/mongo/db/repl/sync_tail_test.cpp
+++ b/src/mongo/db/repl/sync_tail_test.cpp
@@ -368,7 +368,8 @@ Status failedApplyCommand(OperationContext* opCtx, const BSONObj& theOperation,
TEST_F(SyncTailTest, SyncApplyNoNamespaceBadOp) {
const BSONObj op = BSON("op"
<< "x");
- ASSERT_OK(SyncTail::syncApply(_opCtx.get(), op, false, _applyOp, _applyCmd, _incOps));
+ ASSERT_EQUALS(ErrorCodes::BadValue,
+ SyncTail::syncApply(_opCtx.get(), op, false, _applyOp, _applyCmd, _incOps));
ASSERT_EQUALS(0U, _opsApplied);
}
@@ -1137,6 +1138,11 @@ OplogEntry IdempotencyTest::dropIndex(const std::string& indexName) {
}
CollectionState IdempotencyTest::validate() {
+ // We check that a given operation will not resolve the same NamespaceString to different UUIDs,
+ // make sure to use a new operation here.
+ _opCtx.reset();
+ _opCtx = cc().makeOperationContext();
+
AutoGetCollectionForReadCommand autoColl(_opCtx.get(), nss);
auto collection = autoColl.getCollection();
@@ -1178,6 +1184,7 @@ CollectionState IdempotencyTest::validate() {
auto collectionCatalog = collection->getCatalogEntry();
auto collectionOptions = collectionCatalog->getCollectionOptions(_opCtx.get());
+ collectionOptions.uuid.reset();
std::vector<std::string> allIndexes;
BSONObjSet indexSpecs = SimpleBSONObjComparator::kInstance.makeBSONObjSet();
collectionCatalog->getAllIndexes(_opCtx.get(), &allIndexes);
diff --git a/src/mongo/db/storage/mmap_v1/repair_database.cpp b/src/mongo/db/storage/mmap_v1/repair_database.cpp
index 175afb53298..b70d6844d39 100644
--- a/src/mongo/db/storage/mmap_v1/repair_database.cpp
+++ b/src/mongo/db/storage/mmap_v1/repair_database.cpp
@@ -41,6 +41,7 @@
#include "mongo/db/catalog/database.h"
#include "mongo/db/catalog/database_holder.h"
#include "mongo/db/catalog/index_create.h"
+#include "mongo/db/catalog/uuid_catalog.h"
#include "mongo/db/client.h"
#include "mongo/db/db_raii.h"
#include "mongo/db/index/index_descriptor.h"
@@ -319,8 +320,9 @@ Status MMAPV1Engine::repairDatabase(OperationContext* opCtx,
unique_ptr<Database> tempDatabase;
// Must call this before MMAPV1DatabaseCatalogEntry's destructor closes the DB files
- ON_BLOCK_EXIT([&dbEntry, &opCtx] {
+ ON_BLOCK_EXIT([&dbEntry, &opCtx, &tempDatabase] {
getDur().syncDataAndTruncateJournal(opCtx);
+ UUIDCatalog::get(opCtx).onCloseDatabase(tempDatabase.get());
dbEntry->close(opCtx);
});
@@ -362,7 +364,7 @@ Status MMAPV1Engine::repairDatabase(OperationContext* opCtx,
CollectionOptions options;
if (obj["options"].isABSONObj()) {
Status status =
- options.parse(obj["options"].Obj(), CollectionOptions::parseForCommand);
+ options.parse(obj["options"].Obj(), CollectionOptions::parseForStorage);
if (!status.isOK())
return status;
}
@@ -381,6 +383,9 @@ Status MMAPV1Engine::repairDatabase(OperationContext* opCtx,
Collection* tempCollection = NULL;
{
WriteUnitOfWork wunit(opCtx);
+ if (options.uuid) {
+ UUIDCatalog::get(opCtx).onDropCollection(opCtx, options.uuid.get());
+ }
tempCollection = tempDatabase->createCollection(opCtx, ns, options, false);
wunit.commit();
}
diff --git a/src/mongo/dbtests/repltests.cpp b/src/mongo/dbtests/repltests.cpp
index 9cb4b18af93..48fb0381342 100644
--- a/src/mongo/dbtests/repltests.cpp
+++ b/src/mongo/dbtests/repltests.cpp
@@ -891,20 +891,6 @@ public:
}
};
-class EmptyPushSparseIndex : public EmptyPush {
-public:
- EmptyPushSparseIndex() {
- _client.insert("unittests.system.indexes",
- BSON("ns" << ns() << "key" << BSON("a" << 1) << "name"
- << "foo"
- << "sparse"
- << true));
- }
- ~EmptyPushSparseIndex() {
- _client.dropIndexes(ns());
- }
-};
-
class PushAll : public Base {
public:
void doIt() const {
@@ -1465,7 +1451,6 @@ public:
add<Idempotence::PushUpsert>();
add<Idempotence::MultiPush>();
add<Idempotence::EmptyPush>();
- add<Idempotence::EmptyPushSparseIndex>();
add<Idempotence::PushAll>();
add<Idempotence::PushSlice>();
add<Idempotence::PushSliceInitiallyInexistent>();
diff --git a/src/mongo/dbtests/rollbacktests.cpp b/src/mongo/dbtests/rollbacktests.cpp
index 71f1a89a8ff..3cab6c741ab 100644
--- a/src/mongo/dbtests/rollbacktests.cpp
+++ b/src/mongo/dbtests/rollbacktests.cpp
@@ -32,8 +32,10 @@
#include "mongo/db/catalog/collection.h"
#include "mongo/db/catalog/database_catalog_entry.h"
#include "mongo/db/catalog/database_holder.h"
+#include "mongo/db/catalog/drop_collection.h"
#include "mongo/db/catalog/head_manager.h"
#include "mongo/db/catalog/index_create.h"
+#include "mongo/db/catalog/rename_collection.h"
#include "mongo/db/client.h"
#include "mongo/db/db_raii.h"
#include "mongo/db/index/index_descriptor.h"
@@ -80,8 +82,7 @@ Status renameCollection(OperationContext* opCtx,
const NamespaceString& source,
const NamespaceString& target) {
ASSERT_EQ(source.db(), target.db());
- Database* db = dbHolder().get(opCtx, source.db());
- return db->renameCollection(opCtx, source.ns(), target.ns(), false);
+ return renameCollection(opCtx, source, target, false, false);
}
Status truncateCollection(OperationContext* opCtx, const NamespaceString& nss) {
Collection* coll = dbHolder().get(opCtx, nss.db())->getCollection(opCtx, nss);
@@ -317,7 +318,13 @@ public:
{
WriteUnitOfWork uow(&opCtx);
- ASSERT_OK(ctx.db()->dropCollection(&opCtx, target.ns()));
+ BSONObjBuilder result;
+ ASSERT_OK(
+ dropCollection(&opCtx,
+ target,
+ result,
+ {},
+ DropCollectionSystemCollectionMode::kDisallowSystemCollectionDrops));
ASSERT_OK(renameCollection(&opCtx, source, target));
ASSERT(!collectionExists(&ctx, source.ns()));
ASSERT(collectionExists(&ctx, target.ns()));
@@ -375,7 +382,13 @@ public:
{
WriteUnitOfWork uow(&opCtx);
- ASSERT_OK(ctx.db()->dropCollection(&opCtx, nss.ns()));
+ BSONObjBuilder result;
+ ASSERT_OK(
+ dropCollection(&opCtx,
+ nss,
+ result,
+ {},
+ DropCollectionSystemCollectionMode::kDisallowSystemCollectionDrops));
ASSERT(!collectionExists(&ctx, nss.ns()));
ASSERT_OK(userCreateNS(&opCtx,
ctx.db(),
@@ -428,7 +441,13 @@ public:
insertRecord(&opCtx, nss, doc);
assertOnlyRecord(&opCtx, nss, doc);
- ASSERT_OK(ctx.db()->dropCollection(&opCtx, nss.ns()));
+ BSONObjBuilder result;
+ ASSERT_OK(
+ dropCollection(&opCtx,
+ nss,
+ result,
+ {},
+ DropCollectionSystemCollectionMode::kDisallowSystemCollectionDrops));
ASSERT(!collectionExists(&ctx, nss.ns()));
if (!rollback) {
diff --git a/src/mongo/util/SConscript b/src/mongo/util/SConscript
index 2bb79104f3c..604743801c4 100644
--- a/src/mongo/util/SConscript
+++ b/src/mongo/util/SConscript
@@ -101,26 +101,9 @@ env.Library(
target='uuid',
source=[
'uuid.cpp',
- 'namespace_uuid_cache.cpp',
],
LIBDEPS=[
'$BUILD_DIR/mongo/base',
- '$BUILD_DIR/mongo/db/catalog/collection',
- '$BUILD_DIR/mongo/util/decorable',
- '$BUILD_DIR/third_party/murmurhash3/murmurhash3',
- ],
-)
-
-env.Library(
- target='uuid_catalog',
- source=[
- 'uuid_catalog.cpp'
- ],
- LIBDEPS=[
- 'uuid',
- '$BUILD_DIR/mongo/base',
- '$BUILD_DIR/mongo/db/catalog/collection',
- '$BUILD_DIR/mongo/db/namespace_string',
'$BUILD_DIR/mongo/util/decorable',
'$BUILD_DIR/third_party/murmurhash3/murmurhash3',
],
@@ -137,28 +120,6 @@ env.CppUnitTest(
],
)
-env.CppUnitTest(
- target='namespace_uuid_cache_test',
- source=[
- 'namespace_uuid_cache_test.cpp'
- ],
- LIBDEPS=[
- 'uuid',
- ],
-)
-
-env.CppUnitTest(
- target='uuid_catalog_test',
- source=[
- 'uuid_catalog_test.cpp',
- ],
- LIBDEPS=[
- 'uuid_catalog',
- 'uuid',
- '$BUILD_DIR/mongo/db/service_context',
- ]
- )
-
env.Library(
target='summation',
source=[
diff --git a/src/mongo/util/uuid.cpp b/src/mongo/util/uuid.cpp
index 3c041e64010..fdd0798abb1 100644
--- a/src/mongo/util/uuid.cpp
+++ b/src/mongo/util/uuid.cpp
@@ -91,6 +91,10 @@ bool UUID::isUUIDString(const std::string& s) {
return std::regex_match(s, uuidRegex);
}
+bool UUID::isRFC4122v4() const {
+ return (_uuid[6] & ~0x0f) == 0x40 && (_uuid[8] & ~0x3f) == 0x80; // See RFC 4122, section 4.4.
+}
+
UUID UUID::gen() {
int64_t randomWords[2];
diff --git a/src/mongo/util/uuid.h b/src/mongo/util/uuid.h
index 9e1caa88b6c..0049fe9b665 100644
--- a/src/mongo/util/uuid.h
+++ b/src/mongo/util/uuid.h
@@ -64,7 +64,7 @@ public:
static constexpr int kNumBytes = sizeof(UUIDStorage);
/**
- * Generate a new random v4 UUID per RFC 4122.
+ * Generates a new random v4 UUID per RFC 4122.
*/
static UUID gen();
@@ -81,7 +81,7 @@ public:
static StatusWith<UUID> parse(BSONElement from);
/**
- * Parse a BSON document of the form { uuid: BinData(4, "...") }.
+ * Parses a BSON document of the form { uuid: BinData(4, "...") }.
*
* For IDL.
*/
@@ -107,17 +107,17 @@ public:
}
/**
- * Append to builder as BinData(4, "...") element with the given name.
+ * Appends to builder as BinData(4, "...") element with the given name.
*/
void appendToBuilder(BSONObjBuilder* builder, StringData name) const;
/**
- * Return a BSON object of the form { uuid: BinData(4, "...") }.
+ * Returns a BSON object of the form { uuid: BinData(4, "...") }.
*/
BSONObj toBSON() const;
/**
- * Return a string representation of this UUID, in hexadecimal,
+ * Returns a string representation of this UUID, in hexadecimal,
* as per RFC 4122:
*
* 4 Octets - 2 Octets - 2 Octets - 2 Octets - 6 Octets
@@ -133,6 +133,11 @@ public:
}
/**
+ * Returns true only if the UUID is the RFC 4122 variant, v4 (random).
+ */
+ bool isRFC4122v4() const;
+
+ /**
* Custom hasher so UUIDs can be used in unordered data structures.
*
* ex: std::unordered_set<UUID, UUID::Hash> uuidSet;