diff options
18 files changed, 153 insertions, 118 deletions
diff --git a/buildscripts/resmokeconfig/suites/replica_sets_kill_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_kill_primary_jscore_passthrough.yml index 7afe3e36772..6865443484a 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_kill_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_kill_primary_jscore_passthrough.yml @@ -57,9 +57,6 @@ selector: # Inserts enough data that recovery takes more than 8 seconds, so we never get a working primary. - jstests/core/geo_s2ordering.js - # TODO: Remove when SERVER-55559 is fixed. - - jstests/core/timeseries/timeseries_union_with.js - exclude_with_any_tags: - assumes_standalone_mongod ## diff --git a/buildscripts/resmokeconfig/suites/retryable_writes_jscore_stepdown_passthrough.yml b/buildscripts/resmokeconfig/suites/retryable_writes_jscore_stepdown_passthrough.yml index ea45563b543..5d218b3f65a 100644 --- a/buildscripts/resmokeconfig/suites/retryable_writes_jscore_stepdown_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/retryable_writes_jscore_stepdown_passthrough.yml @@ -55,11 +55,6 @@ selector: - jstests/core/startup_log.js # Checks pid, which is different on each server. - jstests/core/validate_cmd_ns.js # Calls _exec() directly, not retryable. - # TODO: Remove when SERVER-55559 is fixed. - - jstests/core/timeseries/timeseries_union_with.js - - jstests/core/timeseries/timeseries_lookup.js - - jstests/core/timeseries/timeseries_graph_lookup.js - exclude_with_any_tags: - assumes_standalone_mongod ## diff --git a/jstests/replsets/rollback_clustered_indexes.js b/jstests/replsets/rollback_clustered_indexes.js new file mode 100644 index 00000000000..5c20f841d60 --- /dev/null +++ b/jstests/replsets/rollback_clustered_indexes.js @@ -0,0 +1,72 @@ +/** + * Tests that writes on collections clustered by _id can be rolled back. + * @tags: [ + * requires_fcv_49, + * requires_replication, + * requires_wiredtiger, + * ] + */ +(function() { +'use strict'; + +load('jstests/replsets/libs/rollback_test.js'); +load('jstests/replsets/libs/rollback_files.js'); +load("jstests/libs/uuid_util.js"); + +// Operations that will be present on both nodes, before the common point. +const dbName = 'test'; +const collName = 'test.system.buckets.t'; +const collNameShort = 'system.buckets.t'; +let commonOps = (node) => { + const db = node.getDB(dbName); + assert.commandWorked(db.createCollection(collNameShort, {clusteredIndex: {}})); + const coll = node.getCollection(collName); + assert.commandWorked(coll.createIndex({a: 1, b: -1})); + assert.commandWorked(coll.insert({a: 0, b: 0})); +}; + +// Operations that will be performed on the rollback node past the common point. +let rollbackDocs = []; +let rollbackOps = (node) => { + const coll = node.getCollection(collName); + let doc; + doc = {_id: new ObjectId(), a: 1, b: 3}; + assert.commandWorked(coll.insert(doc)); + rollbackDocs.push(doc); + + doc = {_id: new ObjectId(), a: 2, b: 2}; + assert.commandWorked(coll.insert(doc)); + rollbackDocs.push(doc); + + doc = {_id: new ObjectId(), a: 3, b: 1}; + assert.commandWorked(coll.insert(doc)); + rollbackDocs.push(doc); +}; + +// Set up Rollback Test. +const rollbackTest = new RollbackTest(); + +commonOps(rollbackTest.getPrimary()); + +const rollbackNode = rollbackTest.transitionToRollbackOperations(); +rollbackOps(rollbackNode); + +// Wait for rollback to finish. +rollbackTest.transitionToSyncSourceOperationsBeforeRollback(); +rollbackTest.transitionToSyncSourceOperationsDuringRollback(); +rollbackTest.transitionToSteadyStateOperations(); + +// Check collection count. +const primary = rollbackTest.getPrimary(); +const coll = primary.getCollection(collName); +assert.eq(1, coll.find().itcount()); +assert.eq(1, coll.count()); + +// Confirm that the rollback wrote deleted documents to a file. +const replTest = rollbackTest.getTestFixture(); + +const uuid = getUUIDFromListCollections(rollbackTest.getPrimary().getDB(dbName), collNameShort); +checkRollbackFiles(replTest.getDbPath(rollbackNode), collName, uuid, rollbackDocs); + +rollbackTest.stop(); +})(); diff --git a/src/mongo/db/catalog/capped_utils.cpp b/src/mongo/db/catalog/capped_utils.cpp index 6fdd90d1de2..129d044857c 100644 --- a/src/mongo/db/catalog/capped_utils.cpp +++ b/src/mongo/db/catalog/capped_utils.cpp @@ -175,7 +175,6 @@ void cloneCollectionAsCapped(OperationContext* opCtx, auto exec = InternalPlanner::collectionScan(opCtx, - fromNss.ns(), &fromCollection, PlanYieldPolicy::YieldPolicy::WRITE_CONFLICT_RETRY_ONLY, InternalPlanner::FORWARD); diff --git a/src/mongo/db/catalog/collection_impl.cpp b/src/mongo/db/catalog/collection_impl.cpp index 69ce18cfd3b..4d13f2ea779 100644 --- a/src/mongo/db/catalog/collection_impl.cpp +++ b/src/mongo/db/catalog/collection_impl.cpp @@ -1497,12 +1497,8 @@ std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> CollectionImpl::makePlanExe boost::optional<RecordId> resumeAfterRecordId) const { auto isForward = scanDirection == ScanDirection::kForward; auto direction = isForward ? InternalPlanner::FORWARD : InternalPlanner::BACKWARD; - return InternalPlanner::collectionScan(opCtx, - yieldableCollection->ns().ns(), - &yieldableCollection, - yieldPolicy, - direction, - resumeAfterRecordId); + return InternalPlanner::collectionScan( + opCtx, &yieldableCollection, yieldPolicy, direction, resumeAfterRecordId); } void CollectionImpl::setNs(NamespaceString nss) { diff --git a/src/mongo/db/catalog/collection_test.cpp b/src/mongo/db/catalog/collection_test.cpp index 807282a60c1..c405161d4b6 100644 --- a/src/mongo/db/catalog/collection_test.cpp +++ b/src/mongo/db/catalog/collection_test.cpp @@ -214,57 +214,6 @@ TEST_F(CollectionTest, AsynchronouslyNotifyCappedWaitersIfNeeded) { ASSERT_EQ(notifier->getVersion(), thisVersion); } -TEST_F(CollectionTest, CreateTimeseriesBucketCollection) { - NamespaceString nss("test.system.buckets.a"); - invariant(nss.isTimeseriesBucketsCollection()); - - AutoGetOrCreateDb databaseWriteGuard(operationContext(), nss.db(), MODE_IX); - auto db = databaseWriteGuard.getDb(); - invariant(db); - - Lock::CollectionLock lk(operationContext(), nss, MODE_IX); - - const BSONObj idxSpec = BSON("v" << IndexDescriptor::getDefaultIndexVersion() << "name" - << "_id_" - << "key" << BSON("_id" << 1)); - - CollectionOptions options; - options.clusteredIndex = ClusteredIndexOptions{}; - { - WriteUnitOfWork wuow(operationContext()); - - // Database::createCollection() ignores the index spec if the _id index is not required on - // the collection. - Collection* collection = db->createCollection(operationContext(), - nss, - options, - /*createIdIndex=*/true, - /*idIndex=*/ - idxSpec); - ASSERT(collection); - ASSERT_EQ(0, collection->getIndexCatalog()->numIndexesTotal(operationContext())); - - StatusWith<BSONObj> swSpec = collection->getIndexCatalog()->createIndexOnEmptyCollection( - operationContext(), idxSpec); - ASSERT_NOT_OK(swSpec.getStatus()); - ASSERT_EQ(swSpec.getStatus().code(), ErrorCodes::CannotCreateIndex); - ASSERT_STRING_CONTAINS( - swSpec.getStatus().reason(), - "cannot create an _id index on a collection already clustered by _id"); - - // Rollback. - } - - { - WriteUnitOfWork wuow(operationContext()); - auto collection = - db->createCollection(operationContext(), nss, options, /*createIdIndex=*/false); - ASSERT(collection); - ASSERT_EQ(0, collection->getIndexCatalog()->numIndexesTotal(operationContext())); - wuow.commit(); - } -} - TEST_F(CatalogTestFixture, CollectionPtrNoYieldTag) { CollectionMock mock(NamespaceString("test.t")); diff --git a/src/mongo/db/commands/dbcommands.cpp b/src/mongo/db/commands/dbcommands.cpp index e4db162abde..e20b53a358d 100644 --- a/src/mongo/db/commands/dbcommands.cpp +++ b/src/mongo/db/commands/dbcommands.cpp @@ -398,7 +398,7 @@ public: return 1; } exec = InternalPlanner::collectionScan( - opCtx, ns, &collection.getCollection(), PlanYieldPolicy::YieldPolicy::NO_YIELD); + opCtx, &collection.getCollection(), PlanYieldPolicy::YieldPolicy::NO_YIELD); } else if (min.isEmpty() || max.isEmpty()) { errmsg = "only one of min or max specified"; return false; diff --git a/src/mongo/db/commands/dbhash.cpp b/src/mongo/db/commands/dbhash.cpp index f68f511fff7..6ba7a8638dc 100644 --- a/src/mongo/db/commands/dbhash.cpp +++ b/src/mongo/db/commands/dbhash.cpp @@ -358,7 +358,7 @@ private: InternalPlanner::IXSCAN_FETCH); } else if (collection->isCapped() || collection->isClustered()) { exec = InternalPlanner::collectionScan( - opCtx, nss.ns(), &collection, PlanYieldPolicy::YieldPolicy::NO_YIELD); + opCtx, &collection, PlanYieldPolicy::YieldPolicy::NO_YIELD); } else { LOGV2(20455, "Can't find _id index for namespace", "namespace"_attr = nss); return "no _id _index"; diff --git a/src/mongo/db/commands/test_commands.cpp b/src/mongo/db/commands/test_commands.cpp index 6afc6d6f19e..6ee4a5b7849 100644 --- a/src/mongo/db/commands/test_commands.cpp +++ b/src/mongo/db/commands/test_commands.cpp @@ -169,7 +169,6 @@ public: // We will remove 'n' documents, so start truncating from the (n + 1)th document to the // end. auto exec = InternalPlanner::collectionScan(opCtx, - fullNs.ns(), &collection.getCollection(), PlanYieldPolicy::YieldPolicy::NO_YIELD, InternalPlanner::BACKWARD); diff --git a/src/mongo/db/dbhelpers.cpp b/src/mongo/db/dbhelpers.cpp index 95fa749e364..dc91d74c7d4 100644 --- a/src/mongo/db/dbhelpers.cpp +++ b/src/mongo/db/dbhelpers.cpp @@ -187,9 +187,12 @@ bool Helpers::getSingleton(OperationContext* opCtx, const char* ns, BSONObj& res boost::optional<AutoGetCollectionForReadCommand> autoColl; boost::optional<AutoGetOplog> autoOplog; const auto& collection = getCollectionForRead(opCtx, NamespaceString(ns), autoColl, autoOplog); + if (!collection) { + return false; + } - auto exec = InternalPlanner::collectionScan( - opCtx, ns, &collection, PlanYieldPolicy::YieldPolicy::NO_YIELD); + auto exec = + InternalPlanner::collectionScan(opCtx, &collection, PlanYieldPolicy::YieldPolicy::NO_YIELD); PlanExecutor::ExecState state = exec->getNext(&result, nullptr); CurOp::get(opCtx)->done(); @@ -209,9 +212,12 @@ bool Helpers::getLast(OperationContext* opCtx, const char* ns, BSONObj& result) boost::optional<AutoGetCollectionForReadCommand> autoColl; boost::optional<AutoGetOplog> autoOplog; const auto& collection = getCollectionForRead(opCtx, NamespaceString(ns), autoColl, autoOplog); + if (!collection) { + return false; + } auto exec = InternalPlanner::collectionScan( - opCtx, ns, &collection, PlanYieldPolicy::YieldPolicy::NO_YIELD, InternalPlanner::BACKWARD); + opCtx, &collection, PlanYieldPolicy::YieldPolicy::NO_YIELD, InternalPlanner::BACKWARD); PlanExecutor::ExecState state = exec->getNext(&result, nullptr); // Non-yielding collection scans from InternalPlanner will never error. diff --git a/src/mongo/db/query/internal_plans.cpp b/src/mongo/db/query/internal_plans.cpp index be2e74f05ab..26864580277 100644 --- a/src/mongo/db/query/internal_plans.cpp +++ b/src/mongo/db/query/internal_plans.cpp @@ -50,36 +50,21 @@ namespace mongo { std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> InternalPlanner::collectionScan( OperationContext* opCtx, - StringData ns, const CollectionPtr* coll, PlanYieldPolicy::YieldPolicy yieldPolicy, const Direction direction, - boost::optional<RecordId> resumeAfterRecordId) { + boost::optional<RecordId> resumeAfterRecordId, + boost::optional<RecordId> minRecord, + boost::optional<RecordId> maxRecord) { const auto& collection = *coll; + invariant(collection); std::unique_ptr<WorkingSet> ws = std::make_unique<WorkingSet>(); auto expCtx = make_intrusive<ExpressionContext>( - opCtx, std::unique_ptr<CollatorInterface>(nullptr), NamespaceString(ns)); - - if (!collection) { - auto eof = std::make_unique<EOFStage>(expCtx.get()); - // Takes ownership of 'ws' and 'eof'. - auto statusWithPlanExecutor = - plan_executor_factory::make(expCtx, - std::move(ws), - std::move(eof), - &CollectionPtr::null, - yieldPolicy, - false, /* whether owned BSON must be returned */ - NamespaceString(ns)); - invariant(statusWithPlanExecutor.isOK()); - return std::move(statusWithPlanExecutor.getValue()); - } - - invariant(ns == collection->ns().ns()); - - auto cs = _collectionScan(expCtx, ws.get(), &collection, direction, resumeAfterRecordId); + opCtx, std::unique_ptr<CollatorInterface>(nullptr), collection->ns()); + auto cs = _collectionScan( + expCtx, ws.get(), &collection, direction, resumeAfterRecordId, minRecord, maxRecord); // Takes ownership of 'ws' and 'cs'. auto statusWithPlanExecutor = diff --git a/src/mongo/db/query/internal_plans.h b/src/mongo/db/query/internal_plans.h index b077e663204..920f964868b 100644 --- a/src/mongo/db/query/internal_plans.h +++ b/src/mongo/db/query/internal_plans.h @@ -68,15 +68,17 @@ public: }; /** - * Returns a collection scan. Caller owns pointer. + * Returns a collection scan. Refer to CollectionScanParams for usage of 'minRecord' and + * 'maxRecord'. */ static std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> collectionScan( OperationContext* opCtx, - StringData ns, const CollectionPtr* collection, PlanYieldPolicy::YieldPolicy yieldPolicy, const Direction direction = FORWARD, - boost::optional<RecordId> resumeAfterRecordId = boost::none); + boost::optional<RecordId> resumeAfterRecordId = boost::none, + boost::optional<RecordId> minRecord = boost::none, + boost::optional<RecordId> maxRecord = boost::none); /** * Returns a FETCH => DELETE plan. diff --git a/src/mongo/db/repl/oplog_applier_impl_test_fixture.cpp b/src/mongo/db/repl/oplog_applier_impl_test_fixture.cpp index b7a745c70b4..6465720aeda 100644 --- a/src/mongo/db/repl/oplog_applier_impl_test_fixture.cpp +++ b/src/mongo/db/repl/oplog_applier_impl_test_fixture.cpp @@ -334,7 +334,6 @@ void checkTxnTable(OperationContext* opCtx, CollectionReader::CollectionReader(OperationContext* opCtx, const NamespaceString& nss) : _collToScan(opCtx, nss), _exec(InternalPlanner::collectionScan(opCtx, - nss.ns(), &_collToScan.getCollection(), PlanYieldPolicy::YieldPolicy::NO_YIELD, InternalPlanner::FORWARD)) {} diff --git a/src/mongo/db/repl/oplog_interface_local.cpp b/src/mongo/db/repl/oplog_interface_local.cpp index 77ee3f4cea0..8f5e589aa6c 100644 --- a/src/mongo/db/repl/oplog_interface_local.cpp +++ b/src/mongo/db/repl/oplog_interface_local.cpp @@ -58,18 +58,19 @@ private: OplogIteratorLocal::OplogIteratorLocal(OperationContext* opCtx) : _oplogRead(opCtx, OplogAccessMode::kRead), _ctx(opCtx, NamespaceString::kRsOplogNamespace.ns()), - _exec(InternalPlanner::collectionScan(opCtx, - NamespaceString::kRsOplogNamespace.ns(), - &_oplogRead.getCollection(), - PlanYieldPolicy::YieldPolicy::NO_YIELD, - InternalPlanner::BACKWARD)) {} + _exec(_oplogRead.getCollection() + ? InternalPlanner::collectionScan(opCtx, + &_oplogRead.getCollection(), + PlanYieldPolicy::YieldPolicy::NO_YIELD, + InternalPlanner::BACKWARD) + : nullptr) {} StatusWith<OplogInterface::Iterator::Value> OplogIteratorLocal::next() { BSONObj obj; RecordId recordId; PlanExecutor::ExecState state; - if (PlanExecutor::ADVANCED != (state = _exec->getNext(&obj, &recordId))) { + if (!_exec || PlanExecutor::ADVANCED != (state = _exec->getNext(&obj, &recordId))) { return StatusWith<Value>(ErrorCodes::CollectionIsEmpty, "no more operations in local oplog"); } diff --git a/src/mongo/db/repl/rs_rollback.cpp b/src/mongo/db/repl/rs_rollback.cpp index c50626cc3f9..87db2d9fe9f 100644 --- a/src/mongo/db/repl/rs_rollback.cpp +++ b/src/mongo/db/repl/rs_rollback.cpp @@ -1032,7 +1032,7 @@ void dropCollection(OperationContext* opCtx, // Performs a collection scan and writes all documents in the collection to disk // in order to keep an archive of items that were rolled back. auto exec = InternalPlanner::collectionScan( - opCtx, nss.toString(), &collection, PlanYieldPolicy::YieldPolicy::YIELD_AUTO); + opCtx, &collection, PlanYieldPolicy::YieldPolicy::YIELD_AUTO); PlanExecutor::ExecState execState; try { BSONObj curObj; diff --git a/src/mongo/db/repl/storage_interface_impl.cpp b/src/mongo/db/repl/storage_interface_impl.cpp index 4bc7a881232..6eecb902615 100644 --- a/src/mongo/db/repl/storage_interface_impl.cpp +++ b/src/mongo/db/repl/storage_interface_impl.cpp @@ -697,17 +697,57 @@ StatusWith<std::vector<BSONObj>> _findOrDeleteDocuments( } // Use collection scan. planExecutor = isFind + ? InternalPlanner::collectionScan( + opCtx, &collection, PlanYieldPolicy::YieldPolicy::NO_YIELD, direction) + : InternalPlanner::deleteWithCollectionScan( + opCtx, + &collection, + makeDeleteStageParamsForDeleteDocuments(), + PlanYieldPolicy::YieldPolicy::NO_YIELD, + direction); + } else if (*indexName == kIdIndexName && collection->isClustered()) { + // This collection is clustered by _id. Use a bounded collection scan, since a + // separate _id index is likely not available. + if (boundInclusion != BoundInclusion::kIncludeBothStartAndEndKeys) { + return Result( + ErrorCodes::InvalidOptions, + "bound inclusion must be BoundInclusion::kIncludeBothStartAndEndKeys for " + "bounded collection scan"); + } + + // Note: this is a limitation of this helper, not bounded collection scans. + if (direction != InternalPlanner::FORWARD) { + return Result(ErrorCodes::InvalidOptions, + "bounded collection scans only support forward scans"); + } + + boost::optional<RecordId> minRecord, maxRecord; + if (!startKey.isEmpty()) { + auto oid = startKey.firstElement().OID(); + minRecord = RecordId(oid.view().view(), OID::kOIDSize); + } + + if (!endKey.isEmpty()) { + auto oid = endKey.firstElement().OID(); + maxRecord = RecordId(oid.view().view(), OID::kOIDSize); + } + + planExecutor = isFind ? InternalPlanner::collectionScan(opCtx, - nsOrUUID.toString(), &collection, PlanYieldPolicy::YieldPolicy::NO_YIELD, - direction) + direction, + boost::none /* resumeAfterId */, + minRecord, + maxRecord) : InternalPlanner::deleteWithCollectionScan( opCtx, &collection, makeDeleteStageParamsForDeleteDocuments(), PlanYieldPolicy::YieldPolicy::NO_YIELD, - direction); + direction, + minRecord, + maxRecord); } else { // Use index scan. auto indexCatalog = collection->getIndexCatalog(); @@ -1118,12 +1158,8 @@ boost::optional<BSONObj> StorageInterfaceImpl::findOplogEntryLessThanOrEqualToTi invariant(oplog); invariant(opCtx->lockState()->isLocked()); - std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> exec = - InternalPlanner::collectionScan(opCtx, - NamespaceString::kRsOplogNamespace.ns(), - &oplog, - PlanYieldPolicy::YieldPolicy::NO_YIELD, - InternalPlanner::BACKWARD); + std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> exec = InternalPlanner::collectionScan( + opCtx, &oplog, PlanYieldPolicy::YieldPolicy::NO_YIELD, InternalPlanner::BACKWARD); // A record id in the oplog collection is equivalent to the document's timestamp field. RecordId desiredRecordId = RecordId(timestamp.asULL()); diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_kv_engine.cpp b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_kv_engine.cpp index 9fb3be20719..bf7f2e5a943 100644 --- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_kv_engine.cpp +++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_kv_engine.cpp @@ -70,6 +70,7 @@ Status KVEngine::createRecordStore(OperationContext* opCtx, StringData ns, StringData ident, const CollectionOptions& options) { + uassert(5555900, "The 'clusteredIndex' option is not supported", !options.clusteredIndex); stdx::lock_guard lock(_identsLock); _idents[ident.toString()] = true; return Status::OK(); diff --git a/src/mongo/dbtests/deferred_writer.cpp b/src/mongo/dbtests/deferred_writer.cpp index d45d9bedaa2..90f4b29a366 100644 --- a/src/mongo/dbtests/deferred_writer.cpp +++ b/src/mongo/dbtests/deferred_writer.cpp @@ -117,10 +117,8 @@ public: AutoGetCollection agc(_opCtx.get(), kTestNamespace, MODE_IS); ASSERT_TRUE(agc.getCollection()); - auto plan = InternalPlanner::collectionScan(_opCtx.get(), - kTestNamespace.ns(), - &agc.getCollection(), - PlanYieldPolicy::YieldPolicy::NO_YIELD); + auto plan = InternalPlanner::collectionScan( + _opCtx.get(), &agc.getCollection(), PlanYieldPolicy::YieldPolicy::NO_YIELD); std::vector<BSONObj> result; BSONObj i; |