diff options
| -rw-r--r-- | src/mongo/db/catalog/SConscript | 12 | ||||
| -rw-r--r-- | src/mongo/db/catalog/index_catalog.h | 2 | ||||
| -rw-r--r-- | src/mongo/db/catalog/index_create.cpp | 456 | ||||
| -rw-r--r-- | src/mongo/db/catalog/index_create.h | 201 | ||||
| -rw-r--r-- | src/mongo/db/catalog/index_create_impl.cpp | 468 | ||||
| -rw-r--r-- | src/mongo/db/catalog/index_create_impl.h | 220 |
6 files changed, 861 insertions, 498 deletions
diff --git a/src/mongo/db/catalog/SConscript b/src/mongo/db/catalog/SConscript index 0776c567bc3..846a62202eb 100644 --- a/src/mongo/db/catalog/SConscript +++ b/src/mongo/db/catalog/SConscript @@ -112,6 +112,15 @@ env.Library( ) env.Library( + target='index_create', + source=[ + "index_create.cpp", + ], + LIBDEPS=[ + ], +) + +env.Library( target='catalog', source=[ "apply_ops.cpp", @@ -129,7 +138,7 @@ env.Library( "drop_indexes.cpp", "index_catalog_impl.cpp", "index_catalog_entry_impl.cpp", - "index_create.cpp", + "index_create_impl.cpp", "rename_collection.cpp", ], LIBDEPS=[ @@ -140,6 +149,7 @@ env.Library( 'database_holder', 'index_catalog', 'index_catalog_entry', + 'index_create', 'index_key_validate', '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/concurrency/lock_manager', diff --git a/src/mongo/db/catalog/index_catalog.h b/src/mongo/db/catalog/index_catalog.h index 606220fc8d0..86e98e061dd 100644 --- a/src/mongo/db/catalog/index_catalog.h +++ b/src/mongo/db/catalog/index_catalog.h @@ -623,6 +623,6 @@ private: friend IndexCatalogEntry; friend class IndexCatalogImpl; - friend class MultiIndexBlock; + friend class MultiIndexBlockImpl; }; } // namespace mongo diff --git a/src/mongo/db/catalog/index_create.cpp b/src/mongo/db/catalog/index_create.cpp index c86db6b13ee..678e4ae6d0f 100644 --- a/src/mongo/db/catalog/index_create.cpp +++ b/src/mongo/db/catalog/index_create.cpp @@ -1,32 +1,30 @@ -// index_create.cpp - /** -* Copyright (C) 2008-2014 MongoDB Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see <http://www.gnu.org/licenses/>. -* -* As a special exception, the copyright holders give permission to link the -* code of portions of this program with the OpenSSL library under certain -* conditions as described in each individual source file and distribute -* linked combinations including the program with the OpenSSL library. You -* must comply with the GNU Affero General Public License in all respects for -* all of the code used other than as permitted herein. If you modify file(s) -* with this exception, you may extend this exception to your version of the -* file(s), but you are not obligated to do so. If you do not wish to do so, -* delete this exception statement from your version. If you delete this -* exception statement from all source files in the program, then also delete -* it in the license file. -*/ + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ #define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kIndex @@ -48,6 +46,7 @@ #include "mongo/db/query/internal_plans.h" #include "mongo/db/repl/replication_coordinator_global.h" #include "mongo/db/server_parameters.h" +#include "mongo/stdx/functional.h" #include "mongo/stdx/mutex.h" #include "mongo/util/fail_point.h" #include "mongo/util/fail_point_service.h" @@ -57,402 +56,21 @@ #include "mongo/util/quick_exit.h" namespace mongo { +MultiIndexBlock::Impl::~Impl() = default; -using std::unique_ptr; -using std::string; -using std::endl; - -MONGO_FP_DECLARE(crashAfterStartingIndexBuild); -MONGO_FP_DECLARE(hangAfterStartingIndexBuild); -MONGO_FP_DECLARE(hangAfterStartingIndexBuildUnlocked); - -AtomicInt32 maxIndexBuildMemoryUsageMegabytes(500); - -class ExportedMaxIndexBuildMemoryUsageParameter - : public ExportedServerParameter<std::int32_t, ServerParameterType::kStartupAndRuntime> { -public: - ExportedMaxIndexBuildMemoryUsageParameter() - : ExportedServerParameter<std::int32_t, ServerParameterType::kStartupAndRuntime>( - ServerParameterSet::getGlobal(), - "maxIndexBuildMemoryUsageMegabytes", - &maxIndexBuildMemoryUsageMegabytes) {} - - virtual Status validate(const std::int32_t& potentialNewValue) { - if (potentialNewValue < 100) { - return Status( - ErrorCodes::BadValue, - "maxIndexBuildMemoryUsageMegabytes must be greater than or equal to 100 MB"); - } - - return Status::OK(); - } - -} exportedMaxIndexBuildMemoryUsageParameter; - - -/** - * On rollback sets MultiIndexBlock::_needToCleanup to true. - */ -class MultiIndexBlock::SetNeedToCleanupOnRollback : public RecoveryUnit::Change { -public: - explicit SetNeedToCleanupOnRollback(MultiIndexBlock* indexer) : _indexer(indexer) {} - - virtual void commit() {} - virtual void rollback() { - _indexer->_needToCleanup = true; - } - -private: - MultiIndexBlock* const _indexer; -}; - -/** - * On rollback in init(), cleans up _indexes so that ~MultiIndexBlock doesn't try to clean - * up _indexes manually (since the changes were already rolled back). - * Due to this, it is thus legal to call init() again after it fails. - */ -class MultiIndexBlock::CleanupIndexesVectorOnRollback : public RecoveryUnit::Change { -public: - explicit CleanupIndexesVectorOnRollback(MultiIndexBlock* indexer) : _indexer(indexer) {} - - virtual void commit() {} - virtual void rollback() { - _indexer->_indexes.clear(); - } - -private: - MultiIndexBlock* const _indexer; -}; +namespace { +stdx::function<MultiIndexBlock::factory_function_type> factory; +} // namespace -MultiIndexBlock::MultiIndexBlock(OperationContext* opCtx, Collection* collection) - : _collection(collection), - _opCtx(opCtx), - _buildInBackground(false), - _allowInterruption(false), - _ignoreUnique(false), - _needToCleanup(true) {} - -MultiIndexBlock::~MultiIndexBlock() { - if (!_needToCleanup || _indexes.empty()) - return; - while (true) { - try { - WriteUnitOfWork wunit(_opCtx); - // This cleans up all index builds. - // Because that may need to write, it is done inside - // of a WUOW. Nothing inside this block can fail, and it is made fatal if it does. - for (size_t i = 0; i < _indexes.size(); i++) { - _indexes[i].block->fail(); - } - wunit.commit(); - return; - } catch (const WriteConflictException& e) { - continue; - } catch (const DBException& e) { - if (e.toStatus() == ErrorCodes::ExceededMemoryLimit) - continue; - error() << "Caught exception while cleaning up partially built indexes: " << redact(e); - } catch (const std::exception& e) { - error() << "Caught exception while cleaning up partially built indexes: " << e.what(); - } catch (...) { - error() << "Caught unknown exception while cleaning up partially built indexes."; - } - fassertFailed(18644); - } -} - -void MultiIndexBlock::removeExistingIndexes(std::vector<BSONObj>* specs) const { - for (size_t i = 0; i < specs->size(); i++) { - Status status = - _collection->getIndexCatalog()->prepareSpecForCreate(_opCtx, (*specs)[i]).getStatus(); - if (status.code() == ErrorCodes::IndexAlreadyExists) { - specs->erase(specs->begin() + i); - i--; - } - // intentionally ignoring other error codes - } +void MultiIndexBlock::registerFactory(decltype(factory) newFactory) { + factory = std::move(newFactory); } -StatusWith<std::vector<BSONObj>> MultiIndexBlock::init(const BSONObj& spec) { - const auto indexes = std::vector<BSONObj>(1, spec); - return init(indexes); +auto MultiIndexBlock::makeImpl(OperationContext* const opCtx, Collection* const collection) + -> std::unique_ptr<Impl> { + return factory(opCtx, collection); } -StatusWith<std::vector<BSONObj>> MultiIndexBlock::init(const std::vector<BSONObj>& indexSpecs) { - WriteUnitOfWork wunit(_opCtx); - - invariant(_indexes.empty()); - _opCtx->recoveryUnit()->registerChange(new CleanupIndexesVectorOnRollback(this)); - - const string& ns = _collection->ns().ns(); - - const auto idxCat = _collection->getIndexCatalog(); - invariant(idxCat); - invariant(idxCat->ok()); - Status status = idxCat->checkUnfinished(); - if (!status.isOK()) - return status; - - for (size_t i = 0; i < indexSpecs.size(); i++) { - BSONObj info = indexSpecs[i]; - - string pluginName = IndexNames::findPluginName(info["key"].Obj()); - if (pluginName.size()) { - Status s = _collection->getIndexCatalog()->_upgradeDatabaseMinorVersionIfNeeded( - _opCtx, pluginName); - if (!s.isOK()) - return s; - } - - // Any foreground indexes make all indexes be built in the foreground. - _buildInBackground = (_buildInBackground && info["background"].trueValue()); - } - - std::vector<BSONObj> indexInfoObjs; - indexInfoObjs.reserve(indexSpecs.size()); - std::size_t eachIndexBuildMaxMemoryUsageBytes = 0; - if (!indexSpecs.empty()) { - eachIndexBuildMaxMemoryUsageBytes = - static_cast<std::size_t>(maxIndexBuildMemoryUsageMegabytes.load()) * 1024 * 1024 / - indexSpecs.size(); - } - - for (size_t i = 0; i < indexSpecs.size(); i++) { - BSONObj info = indexSpecs[i]; - StatusWith<BSONObj> statusWithInfo = - _collection->getIndexCatalog()->prepareSpecForCreate(_opCtx, info); - Status status = statusWithInfo.getStatus(); - if (!status.isOK()) - return status; - info = statusWithInfo.getValue(); - indexInfoObjs.push_back(info); - - IndexToBuild index; - index.block.reset(new IndexCatalogImpl::IndexBuildBlock(_opCtx, _collection, info)); - status = index.block->init(); - if (!status.isOK()) - return status; - - index.real = index.block->getEntry()->accessMethod(); - status = index.real->initializeAsEmpty(_opCtx); - if (!status.isOK()) - return status; - - if (!_buildInBackground) { - // Bulk build process requires foreground building as it assumes nothing is changing - // under it. - index.bulk = index.real->initiateBulk(eachIndexBuildMaxMemoryUsageBytes); - } - - const IndexDescriptor* descriptor = index.block->getEntry()->descriptor(); - - IndexCatalog::prepareInsertDeleteOptions(_opCtx, descriptor, &index.options); - index.options.dupsAllowed = index.options.dupsAllowed || _ignoreUnique; - if (_ignoreUnique) { - index.options.getKeysMode = IndexAccessMethod::GetKeysMode::kRelaxConstraints; - } - - log() << "build index on: " << ns << " properties: " << descriptor->toString(); - if (index.bulk) - log() << "\t building index using bulk method; build may temporarily use up to " - << eachIndexBuildMaxMemoryUsageBytes / 1024 / 1024 << " megabytes of RAM"; - - index.filterExpression = index.block->getEntry()->getFilterExpression(); - - // TODO SERVER-14888 Suppress this in cases we don't want to audit. - audit::logCreateIndex(_opCtx->getClient(), &info, descriptor->indexName(), ns); - - _indexes.push_back(std::move(index)); - } - - if (_buildInBackground) - _backgroundOperation.reset(new BackgroundOperation(ns)); - - wunit.commit(); - - if (MONGO_FAIL_POINT(crashAfterStartingIndexBuild)) { - log() << "Index build interrupted due to 'crashAfterStartingIndexBuild' failpoint. Exiting " - "after waiting for changes to become durable."; - Locker::LockSnapshot lockInfo; - _opCtx->lockState()->saveLockStateAndUnlock(&lockInfo); - if (_opCtx->recoveryUnit()->waitUntilDurable()) { - quickExit(EXIT_TEST); - } - } - - return indexInfoObjs; -} - -Status MultiIndexBlock::insertAllDocumentsInCollection(std::set<RecordId>* dupsOut) { - const char* curopMessage = _buildInBackground ? "Index Build (background)" : "Index Build"; - const auto numRecords = _collection->numRecords(_opCtx); - stdx::unique_lock<Client> lk(*_opCtx->getClient()); - ProgressMeterHolder progress( - *_opCtx->setMessage_inlock(curopMessage, curopMessage, numRecords)); - lk.unlock(); - - Timer t; - - unsigned long long n = 0; - - PlanExecutor::YieldPolicy yieldPolicy; - if (_buildInBackground) { - invariant(_allowInterruption); - yieldPolicy = PlanExecutor::YIELD_AUTO; - } else { - yieldPolicy = PlanExecutor::WRITE_CONFLICT_RETRY_ONLY; - } - auto exec = - InternalPlanner::collectionScan(_opCtx, _collection->ns().ns(), _collection, yieldPolicy); - - Snapshotted<BSONObj> objToIndex; - RecordId loc; - PlanExecutor::ExecState state; - int retries = 0; // non-zero when retrying our last document. - while (retries || - (PlanExecutor::ADVANCED == (state = exec->getNextSnapshotted(&objToIndex, &loc)))) { - try { - if (_allowInterruption) - _opCtx->checkForInterrupt(); - - // Make sure we are working with the latest version of the document. - if (objToIndex.snapshotId() != _opCtx->recoveryUnit()->getSnapshotId() && - !_collection->findDoc(_opCtx, loc, &objToIndex)) { - // doc was deleted so don't index it. - retries = 0; - continue; - } - - // Done before insert so we can retry document if it WCEs. - progress->setTotalWhileRunning(_collection->numRecords(_opCtx)); - - WriteUnitOfWork wunit(_opCtx); - Status ret = insert(objToIndex.value(), loc); - if (_buildInBackground) - exec->saveState(); - if (ret.isOK()) { - wunit.commit(); - } else if (dupsOut && ret.code() == ErrorCodes::DuplicateKey) { - // If dupsOut is non-null, we should only fail the specific insert that - // led to a DuplicateKey rather than the whole index build. - dupsOut->insert(loc); - } else { - // Fail the index build hard. - return ret; - } - if (_buildInBackground) - exec->restoreState(); // Handles any WCEs internally. - - // Go to the next document - progress->hit(); - n++; - retries = 0; - } catch (const WriteConflictException& wce) { - CurOp::get(_opCtx)->debug().writeConflicts++; - retries++; // logAndBackoff expects this to be 1 on first call. - wce.logAndBackoff(retries, "index creation", _collection->ns().ns()); - - // Can't use WRITE_CONFLICT_RETRY_LOOP macros since we need to save/restore exec - // around call to abandonSnapshot. - exec->saveState(); - _opCtx->recoveryUnit()->abandonSnapshot(); - exec->restoreState(); // Handles any WCEs internally. - } - } - - uassert(28550, - "Unable to complete index build due to collection scan failure: " + - WorkingSetCommon::toStatusString(objToIndex.value()), - state == PlanExecutor::IS_EOF); - - if (MONGO_FAIL_POINT(hangAfterStartingIndexBuild)) { - // Need the index build to hang before the progress meter is marked as finished so we can - // reliably check that the index build has actually started in js tests. - while (MONGO_FAIL_POINT(hangAfterStartingIndexBuild)) { - log() << "Hanging index build due to 'hangAfterStartingIndexBuild' failpoint"; - sleepmillis(1000); - } - - // Check for interrupt to allow for killop prior to index build completion. - _opCtx->checkForInterrupt(); - } - - if (MONGO_FAIL_POINT(hangAfterStartingIndexBuildUnlocked)) { - // Unlock before hanging so replication recognizes we've completed. - Locker::LockSnapshot lockInfo; - _opCtx->lockState()->saveLockStateAndUnlock(&lockInfo); - while (MONGO_FAIL_POINT(hangAfterStartingIndexBuildUnlocked)) { - log() << "Hanging index build with no locks due to " - "'hangAfterStartingIndexBuildUnlocked' failpoint"; - sleepmillis(1000); - } - // If we want to support this, we'd need to regrab the lock and be sure that all callers are - // ok with us yielding. They should be for BG indexes, but not for foreground. - invariant(!"the hangAfterStartingIndexBuildUnlocked failpoint can't be turned off"); - } - - progress->finished(); - - Status ret = doneInserting(dupsOut); - if (!ret.isOK()) - return ret; - - log() << "build index done. scanned " << n << " total records. " << t.seconds() << " secs"; - - return Status::OK(); -} - -Status MultiIndexBlock::insert(const BSONObj& doc, const RecordId& loc) { - for (size_t i = 0; i < _indexes.size(); i++) { - if (_indexes[i].filterExpression && !_indexes[i].filterExpression->matchesBSON(doc)) { - continue; - } - - int64_t unused; - Status idxStatus(ErrorCodes::InternalError, ""); - if (_indexes[i].bulk) { - idxStatus = _indexes[i].bulk->insert(_opCtx, doc, loc, _indexes[i].options, &unused); - } else { - idxStatus = _indexes[i].real->insert(_opCtx, doc, loc, _indexes[i].options, &unused); - } - - if (!idxStatus.isOK()) - return idxStatus; - } - return Status::OK(); -} - -Status MultiIndexBlock::doneInserting(std::set<RecordId>* dupsOut) { - for (size_t i = 0; i < _indexes.size(); i++) { - if (_indexes[i].bulk == NULL) - continue; - LOG(1) << "\t bulk commit starting for index: " - << _indexes[i].block->getEntry()->descriptor()->indexName(); - Status status = _indexes[i].real->commitBulk(_opCtx, - std::move(_indexes[i].bulk), - _allowInterruption, - _indexes[i].options.dupsAllowed, - dupsOut); - if (!status.isOK()) { - return status; - } - } - - return Status::OK(); -} - -void MultiIndexBlock::abortWithoutCleanup() { - _indexes.clear(); - _needToCleanup = false; -} - -void MultiIndexBlock::commit() { - for (size_t i = 0; i < _indexes.size(); i++) { - _indexes[i].block->success(); - } - - _opCtx->recoveryUnit()->registerChange(new SetNeedToCleanupOnRollback(this)); - _needToCleanup = false; -} +void MultiIndexBlock::TUHook::hook() noexcept {} } // namespace mongo diff --git a/src/mongo/db/catalog/index_create.h b/src/mongo/db/catalog/index_create.h index c7800659ced..56c85609a1f 100644 --- a/src/mongo/db/catalog/index_create.h +++ b/src/mongo/db/catalog/index_create.h @@ -1,32 +1,30 @@ -// index_create.h - /** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see <http://www.gnu.org/licenses/>. -* -* As a special exception, the copyright holders give permission to link the -* code of portions of this program with the OpenSSL library under certain -* conditions as described in each individual source file and distribute -* linked combinations including the program with the OpenSSL library. You -* must comply with the GNU Affero General Public License in all respects for -* all of the code used other than as permitted herein. If you modify file(s) -* with this exception, you may extend this exception to your version of the -* file(s), but you are not obligated to do so. If you do not wish to do so, -* delete this exception statement from your version. If you delete this -* exception statement from all source files in the program, then also delete -* it in the license file. -*/ + * Copyright (C) 2017 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ #pragma once @@ -41,9 +39,9 @@ #include "mongo/db/catalog/index_catalog_impl.h" #include "mongo/db/index/index_access_method.h" #include "mongo/db/record_id.h" +#include "mongo/stdx/functional.h" namespace mongo { - class BackgroundOperation; class BSONObj; class Collection; @@ -61,14 +59,75 @@ class OperationContext; * (as it is itself essentially a form of rollback, you don't want to "rollback the rollback"). */ class MultiIndexBlock { - MONGO_DISALLOW_COPYING(MultiIndexBlock); +public: + class Impl { + public: + virtual ~Impl() = 0; + + virtual void allowBackgroundBuilding() = 0; + + virtual void allowInterruption() = 0; + + virtual void ignoreUniqueConstraint() = 0; + + virtual void removeExistingIndexes(std::vector<BSONObj>* specs) const = 0; + + virtual StatusWith<std::vector<BSONObj>> init(const std::vector<BSONObj>& specs) = 0; + + virtual StatusWith<std::vector<BSONObj>> init(const BSONObj& spec) = 0; + + virtual Status insertAllDocumentsInCollection(std::set<RecordId>* dupsOut = NULL) = 0; + + virtual Status insert(const BSONObj& wholeDocument, const RecordId& loc) = 0; + + virtual Status doneInserting(std::set<RecordId>* dupsOut = NULL) = 0; + + virtual void commit() = 0; + + virtual void abortWithoutCleanup() = 0; + + virtual bool getBuildInBackground() const = 0; + }; + +private: + std::unique_ptr<Impl> _pimpl; + + // This structure exists to give us a customization point to decide how to force users of this + // class to depend upon the corresponding `index_create.cpp` Translation Unit (TU). All public + // forwarding functions call `_impl(), and `_impl` creates an instance of this structure. + struct TUHook { + static void hook() noexcept; + + explicit inline TUHook() noexcept { + if (kDebugBuild) + this->hook(); + } + }; + + inline const Impl& _impl() const { + TUHook{}; + return *this->_pimpl; + } + + inline Impl& _impl() { + TUHook{}; + return *this->_pimpl; + } + + static std::unique_ptr<Impl> makeImpl(OperationContext* opCtx, Collection* collection); public: + using factory_function_type = decltype(makeImpl); + + static void registerFactory(stdx::function<factory_function_type> factory); + + inline ~MultiIndexBlock() = default; + /** * Neither pointer is owned. */ - MultiIndexBlock(OperationContext* opCtx, Collection* collection); - ~MultiIndexBlock(); + inline explicit MultiIndexBlock(OperationContext* const opCtx, Collection* const collection) + : _pimpl(makeImpl(opCtx, collection)) {} /** * By default we ignore the 'background' flag in specs when building an index. If this is @@ -78,16 +137,16 @@ public: * indexes in the background, but there is a performance benefit to building all in the * foreground. */ - void allowBackgroundBuilding() { - _buildInBackground = true; + inline void allowBackgroundBuilding() { + return this->_impl().allowBackgroundBuilding(); } /** * Call this before init() to allow the index build to be interrupted. * This only affects builds using the insertAllDocumentsInCollection helper. */ - void allowInterruption() { - _allowInterruption = true; + inline void allowInterruption() { + return this->_impl().allowInterruption(); } /** @@ -97,15 +156,17 @@ public: * * If this is called, any dupsOut sets passed in will never be filled. */ - void ignoreUniqueConstraint() { - _ignoreUnique = true; + inline void ignoreUniqueConstraint() { + return this->_impl().ignoreUniqueConstraint(); } /** * Removes pre-existing indexes from 'specs'. If this isn't done, init() may fail with * IndexAlreadyExists. */ - void removeExistingIndexes(std::vector<BSONObj>* specs) const; + inline void removeExistingIndexes(std::vector<BSONObj>* const specs) const { + return this->_impl().removeExistingIndexes(specs); + } /** * Prepares the index(es) for building and returns the canonicalized form of the requested index @@ -115,8 +176,13 @@ public: * * Requires holding an exclusive database lock. */ - StatusWith<std::vector<BSONObj>> init(const std::vector<BSONObj>& specs); - StatusWith<std::vector<BSONObj>> init(const BSONObj& spec); + inline StatusWith<std::vector<BSONObj>> init(const std::vector<BSONObj>& specs) { + return this->_impl().init(specs); + } + + inline StatusWith<std::vector<BSONObj>> init(const BSONObj& spec) { + return this->_impl().init(spec); + } /** * Inserts all documents in the Collection into the indexes and logs with timing info. @@ -132,7 +198,9 @@ public: * * Should not be called inside of a WriteUnitOfWork. */ - Status insertAllDocumentsInCollection(std::set<RecordId>* dupsOut = NULL); + inline Status insertAllDocumentsInCollection(std::set<RecordId>* const dupsOut = nullptr) { + return this->_impl().insertAllDocumentsInCollection(dupsOut); + } /** * Call this after init() for each document in the collection. @@ -141,7 +209,9 @@ public: * * Should be called inside of a WriteUnitOfWork. */ - Status insert(const BSONObj& wholeDocument, const RecordId& loc); + inline Status insert(const BSONObj& wholeDocument, const RecordId& loc) { + return this->_impl().insert(wholeDocument, loc); + } /** * Call this after the last insert(). This gives the index builder a chance to do any @@ -155,7 +225,9 @@ public: * * Should not be called inside of a WriteUnitOfWork. */ - Status doneInserting(std::set<RecordId>* dupsOut = NULL); + inline Status doneInserting(std::set<RecordId>* const dupsOut = nullptr) { + return this->_impl().doneInserting(dupsOut); + } /** * Marks the index ready for use. Should only be called as the last method after @@ -166,7 +238,9 @@ public: * * Requires holding an exclusive database lock. */ - void commit(); + inline void commit() { + return this->_impl().commit(); + } /** * May be called at any time after construction but before a successful commit(). Suppresses @@ -182,39 +256,12 @@ public: * Does not matter whether it is called inside of a WriteUnitOfWork. Will not be rolled * back. */ - void abortWithoutCleanup(); - - bool getBuildInBackground() const { - return _buildInBackground; + inline void abortWithoutCleanup() { + return this->_impl().abortWithoutCleanup(); } -private: - class SetNeedToCleanupOnRollback; - class CleanupIndexesVectorOnRollback; - - struct IndexToBuild { - std::unique_ptr<IndexCatalogImpl::IndexBuildBlock> block; - - IndexAccessMethod* real = NULL; // owned elsewhere - const MatchExpression* filterExpression; // might be NULL, owned elsewhere - std::unique_ptr<IndexAccessMethod::BulkBuilder> bulk; - - InsertDeleteOptions options; - }; - - std::vector<IndexToBuild> _indexes; - - std::unique_ptr<BackgroundOperation> _backgroundOperation; - - // Pointers not owned here and must outlive 'this' - Collection* _collection; - OperationContext* _opCtx; - - bool _buildInBackground; - bool _allowInterruption; - bool _ignoreUnique; - - bool _needToCleanup; + inline bool getBuildInBackground() const { + return this->_impl().getBuildInBackground(); + } }; - } // namespace mongo diff --git a/src/mongo/db/catalog/index_create_impl.cpp b/src/mongo/db/catalog/index_create_impl.cpp new file mode 100644 index 00000000000..b93bcf8f01e --- /dev/null +++ b/src/mongo/db/catalog/index_create_impl.cpp @@ -0,0 +1,468 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kIndex + +#include "mongo/platform/basic.h" + +#include "mongo/db/catalog/index_create_impl.h" + +#include "mongo/base/error_codes.h" +#include "mongo/base/init.h" +#include "mongo/client/dbclientinterface.h" +#include "mongo/db/audit.h" +#include "mongo/db/background.h" +#include "mongo/db/catalog/collection.h" +#include "mongo/db/client.h" +#include "mongo/db/clientcursor.h" +#include "mongo/db/concurrency/write_conflict_exception.h" +#include "mongo/db/curop.h" +#include "mongo/db/exec/working_set_common.h" +#include "mongo/db/operation_context.h" +#include "mongo/db/query/internal_plans.h" +#include "mongo/db/repl/replication_coordinator_global.h" +#include "mongo/db/server_parameters.h" +#include "mongo/stdx/memory.h" +#include "mongo/stdx/mutex.h" +#include "mongo/util/fail_point.h" +#include "mongo/util/fail_point_service.h" +#include "mongo/util/log.h" +#include "mongo/util/processinfo.h" +#include "mongo/util/progress_meter.h" +#include "mongo/util/quick_exit.h" + +namespace mongo { + +namespace { +MONGO_INITIALIZER(InitializeMultiIndexBlockFactory)(InitializerContext* const) { + MultiIndexBlock::registerFactory( + [](OperationContext* const opCtx, Collection* const collection) { + return stdx::make_unique<MultiIndexBlockImpl>(opCtx, collection); + }); + return Status::OK(); +} +} // namespace + +using std::unique_ptr; +using std::string; +using std::endl; + +MONGO_FP_DECLARE(crashAfterStartingIndexBuild); +MONGO_FP_DECLARE(hangAfterStartingIndexBuild); +MONGO_FP_DECLARE(hangAfterStartingIndexBuildUnlocked); + +AtomicInt32 maxIndexBuildMemoryUsageMegabytes(500); + +class ExportedMaxIndexBuildMemoryUsageParameter + : public ExportedServerParameter<std::int32_t, ServerParameterType::kStartupAndRuntime> { +public: + ExportedMaxIndexBuildMemoryUsageParameter() + : ExportedServerParameter<std::int32_t, ServerParameterType::kStartupAndRuntime>( + ServerParameterSet::getGlobal(), + "maxIndexBuildMemoryUsageMegabytes", + &maxIndexBuildMemoryUsageMegabytes) {} + + virtual Status validate(const std::int32_t& potentialNewValue) { + if (potentialNewValue < 100) { + return Status( + ErrorCodes::BadValue, + "maxIndexBuildMemoryUsageMegabytes must be greater than or equal to 100 MB"); + } + + return Status::OK(); + } + +} exportedMaxIndexBuildMemoryUsageParameter; + + +/** + * On rollback sets MultiIndexBlockImpl::_needToCleanup to true. + */ +class MultiIndexBlockImpl::SetNeedToCleanupOnRollback : public RecoveryUnit::Change { +public: + explicit SetNeedToCleanupOnRollback(MultiIndexBlockImpl* indexer) : _indexer(indexer) {} + + virtual void commit() {} + virtual void rollback() { + _indexer->_needToCleanup = true; + } + +private: + MultiIndexBlockImpl* const _indexer; +}; + +/** + * On rollback in init(), cleans up _indexes so that ~MultiIndexBlock doesn't try to clean + * up _indexes manually (since the changes were already rolled back). + * Due to this, it is thus legal to call init() again after it fails. + */ +class MultiIndexBlockImpl::CleanupIndexesVectorOnRollback : public RecoveryUnit::Change { +public: + explicit CleanupIndexesVectorOnRollback(MultiIndexBlockImpl* indexer) : _indexer(indexer) {} + + virtual void commit() {} + virtual void rollback() { + _indexer->_indexes.clear(); + } + +private: + MultiIndexBlockImpl* const _indexer; +}; + +MultiIndexBlockImpl::MultiIndexBlockImpl(OperationContext* opCtx, Collection* collection) + : _collection(collection), + _opCtx(opCtx), + _buildInBackground(false), + _allowInterruption(false), + _ignoreUnique(false), + _needToCleanup(true) {} + +MultiIndexBlockImpl::~MultiIndexBlockImpl() { + if (!_needToCleanup || _indexes.empty()) + return; + while (true) { + try { + WriteUnitOfWork wunit(_opCtx); + // This cleans up all index builds. + // Because that may need to write, it is done inside + // of a WUOW. Nothing inside this block can fail, and it is made fatal if it does. + for (size_t i = 0; i < _indexes.size(); i++) { + _indexes[i].block->fail(); + } + wunit.commit(); + return; + } catch (const WriteConflictException& e) { + continue; + } catch (const DBException& e) { + if (e.toStatus() == ErrorCodes::ExceededMemoryLimit) + continue; + error() << "Caught exception while cleaning up partially built indexes: " << redact(e); + } catch (const std::exception& e) { + error() << "Caught exception while cleaning up partially built indexes: " << e.what(); + } catch (...) { + error() << "Caught unknown exception while cleaning up partially built indexes."; + } + fassertFailed(18644); + } +} + +void MultiIndexBlockImpl::removeExistingIndexes(std::vector<BSONObj>* specs) const { + for (size_t i = 0; i < specs->size(); i++) { + Status status = + _collection->getIndexCatalog()->prepareSpecForCreate(_opCtx, (*specs)[i]).getStatus(); + if (status.code() == ErrorCodes::IndexAlreadyExists) { + specs->erase(specs->begin() + i); + i--; + } + // intentionally ignoring other error codes + } +} + +StatusWith<std::vector<BSONObj>> MultiIndexBlockImpl::init(const BSONObj& spec) { + const auto indexes = std::vector<BSONObj>(1, spec); + return init(indexes); +} + +StatusWith<std::vector<BSONObj>> MultiIndexBlockImpl::init(const std::vector<BSONObj>& indexSpecs) { + WriteUnitOfWork wunit(_opCtx); + + invariant(_indexes.empty()); + _opCtx->recoveryUnit()->registerChange(new CleanupIndexesVectorOnRollback(this)); + + const string& ns = _collection->ns().ns(); + + const auto idxCat = _collection->getIndexCatalog(); + invariant(idxCat); + invariant(idxCat->ok()); + Status status = idxCat->checkUnfinished(); + if (!status.isOK()) + return status; + + for (size_t i = 0; i < indexSpecs.size(); i++) { + BSONObj info = indexSpecs[i]; + + string pluginName = IndexNames::findPluginName(info["key"].Obj()); + if (pluginName.size()) { + Status s = _collection->getIndexCatalog()->_upgradeDatabaseMinorVersionIfNeeded( + _opCtx, pluginName); + if (!s.isOK()) + return s; + } + + // Any foreground indexes make all indexes be built in the foreground. + _buildInBackground = (_buildInBackground && info["background"].trueValue()); + } + + std::vector<BSONObj> indexInfoObjs; + indexInfoObjs.reserve(indexSpecs.size()); + std::size_t eachIndexBuildMaxMemoryUsageBytes = 0; + if (!indexSpecs.empty()) { + eachIndexBuildMaxMemoryUsageBytes = + static_cast<std::size_t>(maxIndexBuildMemoryUsageMegabytes.load()) * 1024 * 1024 / + indexSpecs.size(); + } + + for (size_t i = 0; i < indexSpecs.size(); i++) { + BSONObj info = indexSpecs[i]; + StatusWith<BSONObj> statusWithInfo = + _collection->getIndexCatalog()->prepareSpecForCreate(_opCtx, info); + Status status = statusWithInfo.getStatus(); + if (!status.isOK()) + return status; + info = statusWithInfo.getValue(); + indexInfoObjs.push_back(info); + + IndexToBuild index; + index.block.reset(new IndexCatalogImpl::IndexBuildBlock(_opCtx, _collection, info)); + status = index.block->init(); + if (!status.isOK()) + return status; + + index.real = index.block->getEntry()->accessMethod(); + status = index.real->initializeAsEmpty(_opCtx); + if (!status.isOK()) + return status; + + if (!_buildInBackground) { + // Bulk build process requires foreground building as it assumes nothing is changing + // under it. + index.bulk = index.real->initiateBulk(eachIndexBuildMaxMemoryUsageBytes); + } + + const IndexDescriptor* descriptor = index.block->getEntry()->descriptor(); + + IndexCatalog::prepareInsertDeleteOptions(_opCtx, descriptor, &index.options); + index.options.dupsAllowed = index.options.dupsAllowed || _ignoreUnique; + if (_ignoreUnique) { + index.options.getKeysMode = IndexAccessMethod::GetKeysMode::kRelaxConstraints; + } + + log() << "build index on: " << ns << " properties: " << descriptor->toString(); + if (index.bulk) + log() << "\t building index using bulk method; build may temporarily use up to " + << eachIndexBuildMaxMemoryUsageBytes / 1024 / 1024 << " megabytes of RAM"; + + index.filterExpression = index.block->getEntry()->getFilterExpression(); + + // TODO SERVER-14888 Suppress this in cases we don't want to audit. + audit::logCreateIndex(_opCtx->getClient(), &info, descriptor->indexName(), ns); + + _indexes.push_back(std::move(index)); + } + + if (_buildInBackground) + _backgroundOperation.reset(new BackgroundOperation(ns)); + + wunit.commit(); + + if (MONGO_FAIL_POINT(crashAfterStartingIndexBuild)) { + log() << "Index build interrupted due to 'crashAfterStartingIndexBuild' failpoint. Exiting " + "after waiting for changes to become durable."; + Locker::LockSnapshot lockInfo; + _opCtx->lockState()->saveLockStateAndUnlock(&lockInfo); + if (_opCtx->recoveryUnit()->waitUntilDurable()) { + quickExit(EXIT_TEST); + } + } + + return indexInfoObjs; +} + +Status MultiIndexBlockImpl::insertAllDocumentsInCollection(std::set<RecordId>* dupsOut) { + const char* curopMessage = _buildInBackground ? "Index Build (background)" : "Index Build"; + const auto numRecords = _collection->numRecords(_opCtx); + stdx::unique_lock<Client> lk(*_opCtx->getClient()); + ProgressMeterHolder progress( + *_opCtx->setMessage_inlock(curopMessage, curopMessage, numRecords)); + lk.unlock(); + + Timer t; + + unsigned long long n = 0; + + PlanExecutor::YieldPolicy yieldPolicy; + if (_buildInBackground) { + invariant(_allowInterruption); + yieldPolicy = PlanExecutor::YIELD_AUTO; + } else { + yieldPolicy = PlanExecutor::WRITE_CONFLICT_RETRY_ONLY; + } + auto exec = + InternalPlanner::collectionScan(_opCtx, _collection->ns().ns(), _collection, yieldPolicy); + + Snapshotted<BSONObj> objToIndex; + RecordId loc; + PlanExecutor::ExecState state; + int retries = 0; // non-zero when retrying our last document. + while (retries || + (PlanExecutor::ADVANCED == (state = exec->getNextSnapshotted(&objToIndex, &loc)))) { + try { + if (_allowInterruption) + _opCtx->checkForInterrupt(); + + // Make sure we are working with the latest version of the document. + if (objToIndex.snapshotId() != _opCtx->recoveryUnit()->getSnapshotId() && + !_collection->findDoc(_opCtx, loc, &objToIndex)) { + // doc was deleted so don't index it. + retries = 0; + continue; + } + + // Done before insert so we can retry document if it WCEs. + progress->setTotalWhileRunning(_collection->numRecords(_opCtx)); + + WriteUnitOfWork wunit(_opCtx); + Status ret = insert(objToIndex.value(), loc); + if (_buildInBackground) + exec->saveState(); + if (ret.isOK()) { + wunit.commit(); + } else if (dupsOut && ret.code() == ErrorCodes::DuplicateKey) { + // If dupsOut is non-null, we should only fail the specific insert that + // led to a DuplicateKey rather than the whole index build. + dupsOut->insert(loc); + } else { + // Fail the index build hard. + return ret; + } + if (_buildInBackground) + exec->restoreState(); // Handles any WCEs internally. + + // Go to the next document + progress->hit(); + n++; + retries = 0; + } catch (const WriteConflictException& wce) { + CurOp::get(_opCtx)->debug().writeConflicts++; + retries++; // logAndBackoff expects this to be 1 on first call. + wce.logAndBackoff(retries, "index creation", _collection->ns().ns()); + + // Can't use WRITE_CONFLICT_RETRY_LOOP macros since we need to save/restore exec + // around call to abandonSnapshot. + exec->saveState(); + _opCtx->recoveryUnit()->abandonSnapshot(); + exec->restoreState(); // Handles any WCEs internally. + } + } + + uassert(28550, + "Unable to complete index build due to collection scan failure: " + + WorkingSetCommon::toStatusString(objToIndex.value()), + state == PlanExecutor::IS_EOF); + + if (MONGO_FAIL_POINT(hangAfterStartingIndexBuild)) { + // Need the index build to hang before the progress meter is marked as finished so we can + // reliably check that the index build has actually started in js tests. + while (MONGO_FAIL_POINT(hangAfterStartingIndexBuild)) { + log() << "Hanging index build due to 'hangAfterStartingIndexBuild' failpoint"; + sleepmillis(1000); + } + + // Check for interrupt to allow for killop prior to index build completion. + _opCtx->checkForInterrupt(); + } + + if (MONGO_FAIL_POINT(hangAfterStartingIndexBuildUnlocked)) { + // Unlock before hanging so replication recognizes we've completed. + Locker::LockSnapshot lockInfo; + _opCtx->lockState()->saveLockStateAndUnlock(&lockInfo); + while (MONGO_FAIL_POINT(hangAfterStartingIndexBuildUnlocked)) { + log() << "Hanging index build with no locks due to " + "'hangAfterStartingIndexBuildUnlocked' failpoint"; + sleepmillis(1000); + } + // If we want to support this, we'd need to regrab the lock and be sure that all callers are + // ok with us yielding. They should be for BG indexes, but not for foreground. + invariant(!"the hangAfterStartingIndexBuildUnlocked failpoint can't be turned off"); + } + + progress->finished(); + + Status ret = doneInserting(dupsOut); + if (!ret.isOK()) + return ret; + + log() << "build index done. scanned " << n << " total records. " << t.seconds() << " secs"; + + return Status::OK(); +} + +Status MultiIndexBlockImpl::insert(const BSONObj& doc, const RecordId& loc) { + for (size_t i = 0; i < _indexes.size(); i++) { + if (_indexes[i].filterExpression && !_indexes[i].filterExpression->matchesBSON(doc)) { + continue; + } + + int64_t unused; + Status idxStatus(ErrorCodes::InternalError, ""); + if (_indexes[i].bulk) { + idxStatus = _indexes[i].bulk->insert(_opCtx, doc, loc, _indexes[i].options, &unused); + } else { + idxStatus = _indexes[i].real->insert(_opCtx, doc, loc, _indexes[i].options, &unused); + } + + if (!idxStatus.isOK()) + return idxStatus; + } + return Status::OK(); +} + +Status MultiIndexBlockImpl::doneInserting(std::set<RecordId>* dupsOut) { + for (size_t i = 0; i < _indexes.size(); i++) { + if (_indexes[i].bulk == NULL) + continue; + LOG(1) << "\t bulk commit starting for index: " + << _indexes[i].block->getEntry()->descriptor()->indexName(); + Status status = _indexes[i].real->commitBulk(_opCtx, + std::move(_indexes[i].bulk), + _allowInterruption, + _indexes[i].options.dupsAllowed, + dupsOut); + if (!status.isOK()) { + return status; + } + } + + return Status::OK(); +} + +void MultiIndexBlockImpl::abortWithoutCleanup() { + _indexes.clear(); + _needToCleanup = false; +} + +void MultiIndexBlockImpl::commit() { + for (size_t i = 0; i < _indexes.size(); i++) { + _indexes[i].block->success(); + } + + _opCtx->recoveryUnit()->registerChange(new SetNeedToCleanupOnRollback(this)); + _needToCleanup = false; +} + +} // namespace mongo diff --git a/src/mongo/db/catalog/index_create_impl.h b/src/mongo/db/catalog/index_create_impl.h new file mode 100644 index 00000000000..40e3ead87b3 --- /dev/null +++ b/src/mongo/db/catalog/index_create_impl.h @@ -0,0 +1,220 @@ +/** + * Copyright (C) 2017 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/catalog/index_create.h" + +#include <memory> +#include <set> +#include <string> +#include <vector> + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/status.h" +#include "mongo/db/catalog/index_catalog.h" +#include "mongo/db/catalog/index_catalog_impl.h" +#include "mongo/db/index/index_access_method.h" +#include "mongo/db/record_id.h" + +namespace mongo { + +class BackgroundOperation; +class BSONObj; +class Collection; +class OperationContext; + +/** + * Builds one or more indexes. + * + * If any method other than insert() returns a not-ok Status, this MultiIndexBlock should be + * considered failed and must be destroyed. + * + * If a MultiIndexBlock is destroyed before commit() or if commit() is rolled back, it will + * clean up all traces of the indexes being constructed. MultiIndexBlocks should not be + * destructed from inside of a WriteUnitOfWork as any cleanup needed should never be rolled back + * (as it is itself essentially a form of rollback, you don't want to "rollback the rollback"). + */ +class MultiIndexBlockImpl : public MultiIndexBlock::Impl { + MONGO_DISALLOW_COPYING(MultiIndexBlockImpl); + +public: + /** + * Neither pointer is owned. + */ + MultiIndexBlockImpl(OperationContext* opCtx, Collection* collection); + ~MultiIndexBlockImpl() override; + + /** + * By default we ignore the 'background' flag in specs when building an index. If this is + * called before init(), we will build the indexes in the background as long as *all* specs + * call for background indexing. If any spec calls for foreground indexing all indexes will + * be built in the foreground, as there is no concurrency benefit to building a subset of + * indexes in the background, but there is a performance benefit to building all in the + * foreground. + */ + void allowBackgroundBuilding() override { + _buildInBackground = true; + } + + /** + * Call this before init() to allow the index build to be interrupted. + * This only affects builds using the insertAllDocumentsInCollection helper. + */ + void allowInterruption() override { + _allowInterruption = true; + } + + /** + * By default we enforce the 'unique' flag in specs when building an index by failing. + * If this is called before init(), we will ignore unique violations. This has no effect if + * no specs are unique. + * + * If this is called, any dupsOut sets passed in will never be filled. + */ + void ignoreUniqueConstraint() override { + _ignoreUnique = true; + } + + /** + * Removes pre-existing indexes from 'specs'. If this isn't done, init() may fail with + * IndexAlreadyExists. + */ + void removeExistingIndexes(std::vector<BSONObj>* specs) const override; + + /** + * Prepares the index(es) for building and returns the canonicalized form of the requested index + * specifications. + * + * Does not need to be called inside of a WriteUnitOfWork (but can be due to nesting). + * + * Requires holding an exclusive database lock. + */ + StatusWith<std::vector<BSONObj>> init(const std::vector<BSONObj>& specs) override; + StatusWith<std::vector<BSONObj>> init(const BSONObj& spec) override; + + /** + * Inserts all documents in the Collection into the indexes and logs with timing info. + * + * This is a simplified replacement for insert and doneInserting. Do not call this if you + * are calling either of them. + * + * If dupsOut is passed as non-NULL, violators of uniqueness constraints will be added to + * the set rather than failing the build. Documents added to this set are not indexed, so + * callers MUST either fail this index build or delete the documents from the collection. + * + * Can throw an exception if interrupted. + * + * Should not be called inside of a WriteUnitOfWork. + */ + Status insertAllDocumentsInCollection(std::set<RecordId>* dupsOut = nullptr) override; + + /** + * Call this after init() for each document in the collection. + * + * Do not call if you called insertAllDocumentsInCollection(); + * + * Should be called inside of a WriteUnitOfWork. + */ + Status insert(const BSONObj& wholeDocument, const RecordId& loc) override; + + /** + * Call this after the last insert(). This gives the index builder a chance to do any + * long-running operations in separate units of work from commit(). + * + * Do not call if you called insertAllDocumentsInCollection(); + * + * If dupsOut is passed as non-NULL, violators of uniqueness constraints will be added to + * the set. Documents added to this set are not indexed, so callers MUST either fail this + * index build or delete the documents from the collection. + * + * Should not be called inside of a WriteUnitOfWork. + */ + Status doneInserting(std::set<RecordId>* dupsOut = nullptr) override; + + /** + * Marks the index ready for use. Should only be called as the last method after + * doneInserting() or insertAllDocumentsInCollection() return success. + * + * Should be called inside of a WriteUnitOfWork. If the index building is to be logOp'd, + * logOp() should be called from the same unit of work as commit(). + * + * Requires holding an exclusive database lock. + */ + void commit() override; + + /** + * May be called at any time after construction but before a successful commit(). Suppresses + * the default behavior on destruction of removing all traces of uncommitted index builds. + * + * The most common use of this is if the indexes were already dropped via some other + * mechanism such as the whole collection being dropped. In that case, it would be invalid + * to try to remove the indexes again. Also, replication uses this to ensure that indexes + * that are being built on shutdown are resumed on startup. + * + * Do not use this unless you are really sure you need to. + * + * Does not matter whether it is called inside of a WriteUnitOfWork. Will not be rolled + * back. + */ + void abortWithoutCleanup() override; + + bool getBuildInBackground() const override { + return _buildInBackground; + } + +private: + class SetNeedToCleanupOnRollback; + class CleanupIndexesVectorOnRollback; + + struct IndexToBuild { + std::unique_ptr<IndexCatalogImpl::IndexBuildBlock> block; + + IndexAccessMethod* real = NULL; // owned elsewhere + const MatchExpression* filterExpression; // might be NULL, owned elsewhere + std::unique_ptr<IndexAccessMethod::BulkBuilder> bulk; + + InsertDeleteOptions options; + }; + + std::vector<IndexToBuild> _indexes; + + std::unique_ptr<BackgroundOperation> _backgroundOperation; + + // Pointers not owned here and must outlive 'this' + Collection* _collection; + OperationContext* _opCtx; + + bool _buildInBackground; + bool _allowInterruption; + bool _ignoreUnique; + + bool _needToCleanup; +}; + +} // namespace mongo |