diff options
| -rw-r--r-- | src/mongo/db/catalog/index_create.cpp | 35 | ||||
| -rw-r--r-- | src/mongo/db/index/index_access_method.cpp | 10 | ||||
| -rw-r--r-- | src/mongo/db/index/index_access_method.h | 9 |
3 files changed, 46 insertions, 8 deletions
diff --git a/src/mongo/db/catalog/index_create.cpp b/src/mongo/db/catalog/index_create.cpp index 823dacff550..b9ca4f87a3f 100644 --- a/src/mongo/db/catalog/index_create.cpp +++ b/src/mongo/db/catalog/index_create.cpp @@ -47,6 +47,7 @@ #include "mongo/db/operation_context.h" #include "mongo/db/query/internal_plans.h" #include "mongo/db/repl/replication_coordinator_global.h" +#include "mongo/db/server_parameters.h" #include "mongo/stdx/mutex.h" #include "mongo/util/fail_point.h" #include "mongo/util/fail_point_service.h" @@ -65,6 +66,30 @@ MONGO_FP_DECLARE(crashAfterStartingIndexBuild); MONGO_FP_DECLARE(hangAfterStartingIndexBuild); MONGO_FP_DECLARE(hangAfterStartingIndexBuildUnlocked); +std::atomic<std::int32_t> maxIndexBuildMemoryUsageMegabytes(500); // NOLINT + +class ExportedMaxIndexBuildMemoryUsageParameter + : public ExportedServerParameter<std::int32_t, ServerParameterType::kStartupAndRuntime> { +public: + ExportedMaxIndexBuildMemoryUsageParameter() + : ExportedServerParameter<std::int32_t, ServerParameterType::kStartupAndRuntime>( + ServerParameterSet::getGlobal(), + "maxIndexBuildMemoryUsageMegabytes", + &maxIndexBuildMemoryUsageMegabytes) {} + + virtual Status validate(const std::int32_t& potentialNewValue) { + if (potentialNewValue < 100) { + return Status( + ErrorCodes::BadValue, + "maxIndexBuildMemoryUsageMegabytes must be greater than or equal to 100 MB"); + } + + return Status::OK(); + } + +} exportedMaxIndexBuildMemoryUsageParameter; + + /** * On rollback sets MultiIndexBlock::_needToCleanup to true. */ @@ -185,6 +210,11 @@ StatusWith<std::vector<BSONObj>> MultiIndexBlock::init(const std::vector<BSONObj std::vector<BSONObj> indexInfoObjs; indexInfoObjs.reserve(indexSpecs.size()); + std::size_t eachIndexBuildMaxMemoryUsageBytes = 0; + if (!indexSpecs.empty()) { + eachIndexBuildMaxMemoryUsageBytes = + std::size_t(maxIndexBuildMemoryUsageMegabytes) * 1024 * 1024 / indexSpecs.size(); + } for (size_t i = 0; i < indexSpecs.size(); i++) { BSONObj info = indexSpecs[i]; @@ -210,7 +240,7 @@ StatusWith<std::vector<BSONObj>> MultiIndexBlock::init(const std::vector<BSONObj if (!_buildInBackground) { // Bulk build process requires foreground building as it assumes nothing is changing // under it. - index.bulk = index.real->initiateBulk(); + index.bulk = index.real->initiateBulk(eachIndexBuildMaxMemoryUsageBytes); } const IndexDescriptor* descriptor = index.block->getEntry()->descriptor(); @@ -221,7 +251,8 @@ StatusWith<std::vector<BSONObj>> MultiIndexBlock::init(const std::vector<BSONObj log() << "build index on: " << ns << " properties: " << descriptor->toString(); if (index.bulk) - log() << "\t building index using bulk method"; + log() << "\t building index using bulk method; build may temporarily use up to " + << eachIndexBuildMaxMemoryUsageBytes / 1024 / 1024 << " megabytes of RAM"; index.filterExpression = index.block->getEntry()->getFilterExpression(); diff --git a/src/mongo/db/index/index_access_method.cpp b/src/mongo/db/index/index_access_method.cpp index a4268c12055..9ca00a16f3e 100644 --- a/src/mongo/db/index/index_access_method.cpp +++ b/src/mongo/db/index/index_access_method.cpp @@ -402,17 +402,19 @@ Status IndexAccessMethod::compact(OperationContext* txn) { return this->_newInterface->compact(txn); } -std::unique_ptr<IndexAccessMethod::BulkBuilder> IndexAccessMethod::initiateBulk() { - return std::unique_ptr<BulkBuilder>(new BulkBuilder(this, _descriptor)); +std::unique_ptr<IndexAccessMethod::BulkBuilder> IndexAccessMethod::initiateBulk( + size_t maxMemoryUsageBytes) { + return std::unique_ptr<BulkBuilder>(new BulkBuilder(this, _descriptor, maxMemoryUsageBytes)); } IndexAccessMethod::BulkBuilder::BulkBuilder(const IndexAccessMethod* index, - const IndexDescriptor* descriptor) + const IndexDescriptor* descriptor, + size_t maxMemoryUsageBytes) : _sorter(Sorter::make( SortOptions() .TempDir(storageGlobalParams.dbpath + "/_tmp") .ExtSortAllowed() - .MaxMemoryUsageBytes(100 * 1024 * 1024), + .MaxMemoryUsageBytes(maxMemoryUsageBytes), BtreeExternalSortComparison(descriptor->keyPattern(), descriptor->version()))), _real(index) {} diff --git a/src/mongo/db/index/index_access_method.h b/src/mongo/db/index/index_access_method.h index b3647ed3a09..7847baf4ecb 100644 --- a/src/mongo/db/index/index_access_method.h +++ b/src/mongo/db/index/index_access_method.h @@ -212,7 +212,9 @@ public: using Sorter = mongo::Sorter<BSONObj, RecordId>; - BulkBuilder(const IndexAccessMethod* index, const IndexDescriptor* descriptor); + BulkBuilder(const IndexAccessMethod* index, + const IndexDescriptor* descriptor, + size_t maxMemoryUsageBytes); std::unique_ptr<Sorter> _sorter; const IndexAccessMethod* _real; @@ -233,8 +235,11 @@ public: * This can return NULL, meaning bulk mode is not available. * * It is only legal to initiate bulk when the index is new and empty. + * + * maxMemoryUsageBytes: amount of memory consumed before the external sorter starts spilling to + * disk */ - std::unique_ptr<BulkBuilder> initiateBulk(); + std::unique_ptr<BulkBuilder> initiateBulk(size_t maxMemoryUsageBytes); /** * Call this when you are ready to finish your bulk work. |