diff options
author | David Storch <david.storch@mongodb.com> | 2019-10-23 23:11:41 +0000 |
---|---|---|
committer | evergreen <evergreen@mongodb.com> | 2019-10-23 23:11:41 +0000 |
commit | ae45a26bf49febf006516e484bbc9f421003e0ec (patch) | |
tree | 3bef4031741944dcc810dd873b7a2942c7c10ca5 | |
parent | 4cf93b3802fa90ba02cdeed36b4f6d615b6d6393 (diff) | |
download | mongo-ae45a26bf49febf006516e484bbc9f421003e0ec.tar.gz |
SERVER-44053 Rename blocking sort memory limit to "internalQueryMaxBlockingSortMemoryUsageBytes".
-rw-r--r-- | buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml | 27 | ||||
-rw-r--r-- | jstests/noPassthrough/external_sort_find.js | 2 | ||||
-rw-r--r-- | jstests/noPassthrough/log_format_slowms_samplerate_loglevel.js | 12 | ||||
-rw-r--r-- | jstests/noPassthrough/plan_cache_list_failed_plans.js | 4 | ||||
-rw-r--r-- | jstests/noPassthrough/plan_cache_list_plans_new_format.js | 4 | ||||
-rw-r--r-- | jstests/noPassthrough/query_knobs_validation.js | 8 | ||||
-rw-r--r-- | jstests/noPassthrough/use_disk.js | 4 | ||||
-rw-r--r-- | jstests/noPassthroughWithMongod/find_and_modify_server16469.js | 12 | ||||
-rw-r--r-- | jstests/sharding/in_memory_sort_limit.js | 5 | ||||
-rw-r--r-- | jstests/sharding/sharding_balance2.js | 3 | ||||
-rw-r--r-- | jstests/sharding/sharding_balance3.js | 3 | ||||
-rw-r--r-- | src/mongo/db/pipeline/document_source_sort.cpp | 5 | ||||
-rw-r--r-- | src/mongo/db/pipeline/document_source_sort.h | 2 | ||||
-rw-r--r-- | src/mongo/db/query/query_knobs.idl | 4 | ||||
-rw-r--r-- | src/mongo/db/query/stage_builder.cpp | 2 | ||||
-rw-r--r-- | src/mongo/dbtests/plan_ranking.cpp | 14 | ||||
-rw-r--r-- | src/mongo/dbtests/query_stage_sort.cpp | 2 |
17 files changed, 62 insertions, 51 deletions
diff --git a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml index 11f855555f9..0d2dd16af55 100644 --- a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml +++ b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml @@ -23,36 +23,39 @@ selector: # Enable after SERVER-40258 gets backported and available in the official 4.2 binaries. - jstests/sharding/prepared_txn_metadata_refresh.js # Enable when 4.4 becomes last stable + - jstests/sharding/banned_txn_databases_sharded.js - jstests/sharding/bulk_insert.js + - jstests/sharding/change_streams_establishment_finds_new_shards.js - jstests/sharding/change_streams_new_shard_new_database.js - jstests/sharding/clear_jumbo.js + - jstests/sharding/clone_catalog_data.js - jstests/sharding/comment_field.js - jstests/sharding/covered_shard_key_indexes.js + - jstests/sharding/database_and_shard_versioning_all_commands.js + - jstests/sharding/database_versioning_safe_secondary_reads.js - jstests/sharding/explain_agg_read_pref.js - jstests/sharding/explain_exec_stats_on_shards.js - jstests/sharding/extract_shard_key_values.js - jstests/sharding/features1.js + - jstests/sharding/in_memory_sort_limit.js - jstests/sharding/map_reduce_invalid_output_collection.js - jstests/sharding/mongos_query_comment.js + - jstests/sharding/move_primary_clone_test.js - jstests/sharding/prefix_shard_key.js + - jstests/sharding/refine_collection_shard_key_atomic.js - jstests/sharding/refine_collection_shard_key_basic.js - - jstests/sharding/refine_collection_shard_key_jumbo.js - jstests/sharding/refine_collection_shard_key_drops_chunks.js - - jstests/sharding/refine_collection_shard_key_atomic.js - - jstests/sharding/update_compound_shard_key.js - - jstests/sharding/update_replace_id.js - - jstests/sharding/move_primary_clone_test.js - - jstests/sharding/database_versioning_safe_secondary_reads.js - - jstests/sharding/clone_catalog_data.js - - jstests/sharding/database_and_shard_versioning_all_commands.js + - jstests/sharding/refine_collection_shard_key_jumbo.js - jstests/sharding/shard1.js - jstests/sharding/shard_removal_triggers_catalog_cache_invalidation.js - - jstests/sharding/track_unsharded_collections_create_collection.js + - jstests/sharding/sharding_balance2.js + - jstests/sharding/sharding_balance3.js + - jstests/sharding/split_large_key.js - jstests/sharding/track_unsharded_collections_check_shard_version.js + - jstests/sharding/track_unsharded_collections_create_collection.js - jstests/sharding/track_unsharded_collections_rename_collection.js - - jstests/sharding/banned_txn_databases_sharded.js - - jstests/sharding/split_large_key.js - - jstests/sharding/change_streams_establishment_finds_new_shards.js + - jstests/sharding/update_compound_shard_key.js + - jstests/sharding/update_replace_id.js # Enable if SERVER-41813 is backported or 4.4 becomes last-stable - jstests/sharding/invalid_system_views_sharded_collection.js diff --git a/jstests/noPassthrough/external_sort_find.js b/jstests/noPassthrough/external_sort_find.js index fee636eb89b..e3803e3d170 100644 --- a/jstests/noPassthrough/external_sort_find.js +++ b/jstests/noPassthrough/external_sort_find.js @@ -23,7 +23,7 @@ const kNumDocsExceedingMemLimit = 50; const kMemoryLimitExceededErrCode = 16819; const options = { - setParameter: "internalQueryExecMaxBlockingSortBytes=" + kMaxMemoryUsageBytes + setParameter: "internalQueryMaxBlockingSortMemoryUsageBytes=" + kMaxMemoryUsageBytes }; const conn = MongoRunner.runMongod(options); assert.neq(null, conn, "mongod was unable to start up with options: " + tojson(options)); diff --git a/jstests/noPassthrough/log_format_slowms_samplerate_loglevel.js b/jstests/noPassthrough/log_format_slowms_samplerate_loglevel.js index 56c8c3b7f47..0d48ac404d7 100644 --- a/jstests/noPassthrough/log_format_slowms_samplerate_loglevel.js +++ b/jstests/noPassthrough/log_format_slowms_samplerate_loglevel.js @@ -324,18 +324,18 @@ function runLoggingTests({db, readWriteMode, slowMs, logLevel, sampleRate}) { }, { test: function(db) { - const originalSortBytes = - db.adminCommand({getParameter: 1, internalQueryExecMaxBlockingSortBytes: 1}); + const originalSortBytes = db.adminCommand( + {getParameter: 1, internalQueryMaxBlockingSortMemoryUsageBytes: 1}); assert.commandWorked(originalSortBytes); - assert.commandWorked( - db.adminCommand({setParameter: 1, internalQueryExecMaxBlockingSortBytes: 10})); + assert.commandWorked(db.adminCommand( + {setParameter: 1, internalQueryMaxBlockingSortMemoryUsageBytes: 10})); assert.eq(coll.aggregate([{$match: {a: 1}}, {$sort: {a: 1}}], {allowDiskUse: true}) .itcount(), 1); assert.commandWorked(db.adminCommand({ setParameter: 1, - internalQueryExecMaxBlockingSortBytes: - originalSortBytes.internalQueryExecMaxBlockingSortBytes + internalQueryMaxBlockingSortMemoryUsageBytes: + originalSortBytes.internalQueryMaxBlockingSortMemoryUsageBytes })); }, logFields: diff --git a/jstests/noPassthrough/plan_cache_list_failed_plans.js b/jstests/noPassthrough/plan_cache_list_failed_plans.js index 778a8854fa5..13101d7247c 100644 --- a/jstests/noPassthrough/plan_cache_list_failed_plans.js +++ b/jstests/noPassthrough/plan_cache_list_failed_plans.js @@ -12,8 +12,8 @@ coll.drop(); // Setup the database such that it will generate a failing plan and a succeeding plan. const numDocs = 32; const smallNumber = 10; -assert.commandWorked( - testDB.adminCommand({setParameter: 1, internalQueryExecMaxBlockingSortBytes: smallNumber})); +assert.commandWorked(testDB.adminCommand( + {setParameter: 1, internalQueryMaxBlockingSortMemoryUsageBytes: smallNumber})); for (let i = 0; i < numDocs * 2; ++i) assert.commandWorked(coll.insert({a: ((i >= (numDocs * 2) - smallNumber) ? 1 : 0), d: i})); diff --git a/jstests/noPassthrough/plan_cache_list_plans_new_format.js b/jstests/noPassthrough/plan_cache_list_plans_new_format.js index 0b95b880d5b..b687529e5ee 100644 --- a/jstests/noPassthrough/plan_cache_list_plans_new_format.js +++ b/jstests/noPassthrough/plan_cache_list_plans_new_format.js @@ -81,8 +81,8 @@ assert(coll.drop()); // Setup the database such that it will generate a failing plan and a succeeding plan. const numDocs = 32; const smallNumber = 10; -assert.commandWorked( - testDB.adminCommand({setParameter: 1, internalQueryExecMaxBlockingSortBytes: smallNumber})); +assert.commandWorked(testDB.adminCommand( + {setParameter: 1, internalQueryMaxBlockingSortMemoryUsageBytes: smallNumber})); for (let i = 0; i < numDocs * 2; ++i) assert.commandWorked(coll.insert({a: ((i >= (numDocs * 2) - smallNumber) ? 1 : 0), d: i})); diff --git a/jstests/noPassthrough/query_knobs_validation.js b/jstests/noPassthrough/query_knobs_validation.js index 6f3e43834f4..1523187eba1 100644 --- a/jstests/noPassthrough/query_knobs_validation.js +++ b/jstests/noPassthrough/query_knobs_validation.js @@ -28,7 +28,7 @@ const expectedParamDefaults = { internalQueryPlannerEnableHashIntersection: false, internalQueryPlanOrChildrenIndependently: true, internalQueryMaxScansToExplode: 200, - internalQueryExecMaxBlockingSortBytes: 100 * 1024 * 1024, + internalQueryMaxBlockingSortMemoryUsageBytes: 100 * 1024 * 1024, internalQueryExecYieldIterations: 128, internalQueryExecYieldPeriodMS: 10, internalQueryFacetBufferSizeBytes: 100 * 1024 * 1024, @@ -117,9 +117,9 @@ assertSetParameterSucceeds("internalQueryMaxScansToExplode", 11); assertSetParameterSucceeds("internalQueryMaxScansToExplode", 0); assertSetParameterFails("internalQueryMaxScansToExplode", -1); -assertSetParameterSucceeds("internalQueryExecMaxBlockingSortBytes", 11); -assertSetParameterSucceeds("internalQueryExecMaxBlockingSortBytes", 0); -assertSetParameterFails("internalQueryExecMaxBlockingSortBytes", -1); +assertSetParameterSucceeds("internalQueryMaxBlockingSortMemoryUsageBytes", 11); +assertSetParameterSucceeds("internalQueryMaxBlockingSortMemoryUsageBytes", 0); +assertSetParameterFails("internalQueryMaxBlockingSortMemoryUsageBytes", -1); assertSetParameterSucceeds("internalQueryExecYieldIterations", 10); assertSetParameterSucceeds("internalQueryExecYieldIterations", 0); diff --git a/jstests/noPassthrough/use_disk.js b/jstests/noPassthrough/use_disk.js index f4323bf3d35..61232cea24c 100644 --- a/jstests/noPassthrough/use_disk.js +++ b/jstests/noPassthrough/use_disk.js @@ -43,7 +43,7 @@ assert(!profileObj.hasOwnProperty("usedDisk"), tojson(profileObj)); assert.eq(profileObj.hasSortStage, true, tojson(profileObj)); assert.commandWorked( - testDB.adminCommand({setParameter: 1, internalQueryExecMaxBlockingSortBytes: 10})); + testDB.adminCommand({setParameter: 1, internalQueryMaxBlockingSortMemoryUsageBytes: 10})); assert.eq( 8, coll.aggregate([{$match: {a: {$gte: 2}}}, {$sort: {a: 1}}], {allowDiskUse: true}).itcount()); profileObj = getLatestProfilerEntry(testDB); @@ -136,7 +136,7 @@ assert.eq(profileObj.usedDisk, true, tojson(profileObj)); // Test that usedDisk is not set for a $lookup with a pipeline that does not use disk. // assert.commandWorked(testDB.adminCommand( - {setParameter: 1, internalQueryExecMaxBlockingSortBytes: 100 * 1024 * 1024})); + {setParameter: 1, internalQueryMaxBlockingSortMemoryUsageBytes: 100 * 1024 * 1024})); resetCollection(); resetForeignCollection(); coll.aggregate( diff --git a/jstests/noPassthroughWithMongod/find_and_modify_server16469.js b/jstests/noPassthroughWithMongod/find_and_modify_server16469.js index 70fe0651748..b1b7ceae613 100644 --- a/jstests/noPassthroughWithMongod/find_and_modify_server16469.js +++ b/jstests/noPassthroughWithMongod/find_and_modify_server16469.js @@ -1,7 +1,7 @@ // SERVER-16469: test that the findAndModify command will correctly sort a large amount of data // without hitting the internal sort memory limit (using a "top-K sort", where K=1). // -// Note that this test sets the server parameter "internalQueryExecMaxBlockingSortBytes", and +// Note that this test sets the server parameter "internalQueryMaxBlockingSortMemoryUsageBytes", and // restores the original value of the parameter before exiting. As a result, this test cannot run // in the sharding passthrough (because mongos does not have this parameter), and cannot run in the // parallel suite (because the change of the parameter value would interfere with other tests). @@ -10,12 +10,12 @@ var coll = db.find_and_modify_server16469; coll.drop(); // Set the internal sort memory limit to 1MB. -var result = db.adminCommand({getParameter: 1, internalQueryExecMaxBlockingSortBytes: 1}); +var result = db.adminCommand({getParameter: 1, internalQueryMaxBlockingSortMemoryUsageBytes: 1}); assert.commandWorked(result); -var oldSortLimit = result.internalQueryExecMaxBlockingSortBytes; +var oldSortLimit = result.internalQueryMaxBlockingSortMemoryUsageBytes; var newSortLimit = 1024 * 1024; assert.commandWorked( - db.adminCommand({setParameter: 1, internalQueryExecMaxBlockingSortBytes: newSortLimit})); + db.adminCommand({setParameter: 1, internalQueryMaxBlockingSortMemoryUsageBytes: newSortLimit})); try { // Insert ~3MB of data. @@ -41,6 +41,6 @@ try { assert.eq(result.value.b, 0); } finally { // Restore the orginal sort memory limit. - assert.commandWorked( - db.adminCommand({setParameter: 1, internalQueryExecMaxBlockingSortBytes: oldSortLimit})); + assert.commandWorked(db.adminCommand( + {setParameter: 1, internalQueryMaxBlockingSortMemoryUsageBytes: oldSortLimit})); } diff --git a/jstests/sharding/in_memory_sort_limit.js b/jstests/sharding/in_memory_sort_limit.js index d93392537af..edc5d1ac7c0 100644 --- a/jstests/sharding/in_memory_sort_limit.js +++ b/jstests/sharding/in_memory_sort_limit.js @@ -6,7 +6,10 @@ var st = new ShardingTest({ shards: 2, - other: {shardOptions: {setParameter: {internalQueryExecMaxBlockingSortBytes: 32 * 1024 * 1024}}} + other: { + shardOptions: + {setParameter: {internalQueryMaxBlockingSortMemoryUsageBytes: 32 * 1024 * 1024}} + } }); assert.commandWorked(st.s.adminCommand({enableSharding: 'test'})); st.ensurePrimaryShard('test', st.shard0.shardName); diff --git a/jstests/sharding/sharding_balance2.js b/jstests/sharding/sharding_balance2.js index 50365ace129..a8eaad6ad71 100644 --- a/jstests/sharding/sharding_balance2.js +++ b/jstests/sharding/sharding_balance2.js @@ -11,7 +11,8 @@ var s = new ShardingTest({ other: { chunkSize: 1, manualAddShard: true, - shardOptions: {setParameter: {internalQueryExecMaxBlockingSortBytes: 32 * 1024 * 1024}} + shardOptions: + {setParameter: {internalQueryMaxBlockingSortMemoryUsageBytes: 32 * 1024 * 1024}} } }); var db = s.getDB("test"); diff --git a/jstests/sharding/sharding_balance3.js b/jstests/sharding/sharding_balance3.js index 4600861ede5..7639dbb8762 100644 --- a/jstests/sharding/sharding_balance3.js +++ b/jstests/sharding/sharding_balance3.js @@ -9,7 +9,8 @@ var s = new ShardingTest({ other: { chunkSize: 1, enableBalancer: true, - shardOptions: {setParameter: {internalQueryExecMaxBlockingSortBytes: 32 * 1024 * 1024}} + shardOptions: + {setParameter: {internalQueryMaxBlockingSortMemoryUsageBytes: 32 * 1024 * 1024}} } }); diff --git a/src/mongo/db/pipeline/document_source_sort.cpp b/src/mongo/db/pipeline/document_source_sort.cpp index b9a3efa3a42..538a5751649 100644 --- a/src/mongo/db/pipeline/document_source_sort.cpp +++ b/src/mongo/db/pipeline/document_source_sort.cpp @@ -193,8 +193,9 @@ intrusive_ptr<DocumentSourceSort> DocumentSourceSort::create( BSONObj sortOrder, uint64_t limit, boost::optional<uint64_t> maxMemoryUsageBytes) { - auto resolvedMaxBytes = - maxMemoryUsageBytes ? *maxMemoryUsageBytes : internalQueryExecMaxBlockingSortBytes.load(); + auto resolvedMaxBytes = maxMemoryUsageBytes + ? *maxMemoryUsageBytes + : internalQueryMaxBlockingSortMemoryUsageBytes.load(); intrusive_ptr<DocumentSourceSort> pSort( new DocumentSourceSort(pExpCtx, sortOrder.getOwned(), limit, resolvedMaxBytes)); return pSort; diff --git a/src/mongo/db/pipeline/document_source_sort.h b/src/mongo/db/pipeline/document_source_sort.h index 116bbe57af2..9defe4da947 100644 --- a/src/mongo/db/pipeline/document_source_sort.h +++ b/src/mongo/db/pipeline/document_source_sort.h @@ -105,7 +105,7 @@ public: /** * Convenience method for creating a $sort stage. If maxMemoryUsageBytes is boost::none, - * then it will actually use the value of internalQueryExecMaxBlockingSortBytes. + * then it will actually use the value of 'internalQueryMaxBlockingSortMemoryUsageBytes'. */ static boost::intrusive_ptr<DocumentSourceSort> create( const boost::intrusive_ptr<ExpressionContext>& pExpCtx, diff --git a/src/mongo/db/query/query_knobs.idl b/src/mongo/db/query/query_knobs.idl index 6995b4e5e4b..62a47e4767c 100644 --- a/src/mongo/db/query/query_knobs.idl +++ b/src/mongo/db/query/query_knobs.idl @@ -202,12 +202,12 @@ server_parameters: # # Query execution # - internalQueryExecMaxBlockingSortBytes: + internalQueryMaxBlockingSortMemoryUsageBytes: description: "The maximum amount of memory a query (e.g. a find or aggregate command) is willing to use to execute a blocking sort, measured in bytes. If disk use is allowed, then it may be possible to sort more data, but this limit will still constrain the memory consumption." set_at: [ startup, runtime ] - cpp_varname: "internalQueryExecMaxBlockingSortBytes" + cpp_varname: "internalQueryMaxBlockingSortMemoryUsageBytes" cpp_vartype: AtomicWord<int> default: expr: 100 * 1024 * 1024 diff --git a/src/mongo/db/query/stage_builder.cpp b/src/mongo/db/query/stage_builder.cpp index a3cc6aee999..d7afdebefbb 100644 --- a/src/mongo/db/query/stage_builder.cpp +++ b/src/mongo/db/query/stage_builder.cpp @@ -126,7 +126,7 @@ std::unique_ptr<PlanStage> buildStages(OperationContext* opCtx, ws, SortPattern{sn->pattern, cq.getExpCtx()}, sn->limit, - internalQueryExecMaxBlockingSortBytes.load(), + internalQueryMaxBlockingSortMemoryUsageBytes.load(), std::move(childStage)); } case STAGE_SORT_KEY_GENERATOR: { diff --git a/src/mongo/dbtests/plan_ranking.cpp b/src/mongo/dbtests/plan_ranking.cpp index 18f4795f13c..c425164beca 100644 --- a/src/mongo/dbtests/plan_ranking.cpp +++ b/src/mongo/dbtests/plan_ranking.cpp @@ -61,7 +61,7 @@ extern AtomicWord<bool> internalQueryForceIntersectionPlans; extern AtomicWord<bool> internalQueryPlannerEnableHashIntersection; -extern AtomicWord<int> internalQueryExecMaxBlockingSortBytes; +extern AtomicWord<int> internalQueryMaxBlockingSortMemoryUsageBytes; extern AtomicWord<int> internalQueryPlanEvaluationMaxResults; @@ -189,17 +189,19 @@ class PlanRankingPreferNonFailed : public PlanRankingTestBase { public: PlanRankingPreferNonFailed() : PlanRankingTestBase(), - _internalQueryExecMaxBlockingSortBytes(internalQueryExecMaxBlockingSortBytes.load()), + _internalQueryMaxBlockingSortMemoryUsageBytes( + internalQueryMaxBlockingSortMemoryUsageBytes.load()), // We set the max results to decrease the amount of work that is done during the trial // period. We want it to do less work than there are docs to ensure that no plan reaches // EOF. _internalQueryPlanEvaluationMaxResults(internalQueryPlanEvaluationMaxResults.load()) { - internalQueryExecMaxBlockingSortBytes.store(10); + internalQueryMaxBlockingSortMemoryUsageBytes.store(10); internalQueryPlanEvaluationMaxResults.store(100); } ~PlanRankingPreferNonFailed() { - internalQueryExecMaxBlockingSortBytes.store(_internalQueryExecMaxBlockingSortBytes); + internalQueryMaxBlockingSortMemoryUsageBytes.store( + _internalQueryMaxBlockingSortMemoryUsageBytes); internalQueryPlanEvaluationMaxResults.store(_internalQueryPlanEvaluationMaxResults); } @@ -253,9 +255,9 @@ public: } private: - // Holds the value of global "internalQueryExecMaxBlockingSortBytes" setParameter flag. + // Holds the value of global "internalQueryMaxBlockingSortMemoryUsageBytes" setParameter flag. // Restored at end of test invocation regardless of test result. - int _internalQueryExecMaxBlockingSortBytes; + int _internalQueryMaxBlockingSortMemoryUsageBytes; int _internalQueryPlanEvaluationMaxResults; }; diff --git a/src/mongo/dbtests/query_stage_sort.cpp b/src/mongo/dbtests/query_stage_sort.cpp index 3e52f8e622a..69f8582b54c 100644 --- a/src/mongo/dbtests/query_stage_sort.cpp +++ b/src/mongo/dbtests/query_stage_sort.cpp @@ -221,7 +221,7 @@ public: }; uint64_t maxMemoryUsageBytes() const { - return internalQueryExecMaxBlockingSortBytes.load(); + return internalQueryMaxBlockingSortMemoryUsageBytes.load(); } static const char* ns() { |