diff options
author | Mark Benvenuto <mark.benvenuto@mongodb.com> | 2015-03-31 15:41:01 -0400 |
---|---|---|
committer | Ramon Fernandez <ramon.fernandez@mongodb.com> | 2015-04-17 11:32:05 -0400 |
commit | cbdaff1efd3c0ba1450da45cf7ac0c6dabf62518 (patch) | |
tree | 41db23a10c8826bfcfef8a619ced4d414ae76380 | |
parent | 437125df84156433d08ba70821a981dea8b6c36e (diff) | |
download | mongo-cbdaff1efd3c0ba1450da45cf7ac0c6dabf62518.tar.gz |
SERVER-17745: Improve dirty page estimation in mmapv1 on Windows
(cherry picked from commit fe358a4fd67fc48188eb062ae026001ff74082e9)
-rw-r--r-- | src/mongo/db/storage/mmap_v1/dur.cpp | 19 | ||||
-rw-r--r-- | src/mongo/util/processinfo.h | 7 | ||||
-rw-r--r-- | src/mongo/util/processinfo_darwin.cpp | 4 | ||||
-rw-r--r-- | src/mongo/util/processinfo_freebsd.cpp | 4 | ||||
-rw-r--r-- | src/mongo/util/processinfo_linux2.cpp | 4 | ||||
-rw-r--r-- | src/mongo/util/processinfo_none.cpp | 4 | ||||
-rw-r--r-- | src/mongo/util/processinfo_openbsd5.cpp | 4 | ||||
-rw-r--r-- | src/mongo/util/processinfo_sunos5.cpp | 4 | ||||
-rw-r--r-- | src/mongo/util/processinfo_win32.cpp | 59 |
9 files changed, 101 insertions, 8 deletions
diff --git a/src/mongo/db/storage/mmap_v1/dur.cpp b/src/mongo/db/storage/mmap_v1/dur.cpp index b22aae79d9c..29ac53e80d2 100644 --- a/src/mongo/db/storage/mmap_v1/dur.cpp +++ b/src/mongo/db/storage/mmap_v1/dur.cpp @@ -769,10 +769,20 @@ namespace { // would wipe out their changes without ever being committed. commitJob.committingReset(); + double systemMemoryPressurePercentage = + ProcessInfo::getSystemMemoryPressurePercentage(); + // Now that the in-memory modifications have been collected, we can potentially // release the flush lock if remap is not necessary. + // When we remap due to memory pressure, we look at two criteria + // 1. If the amount of 4k pages touched exceeds 512 MB, + // a reasonable estimate of memory pressure on Linux. + // 2. Check if the amount of free memory on the machine is running low, + // since #1 is underestimates the memory pressure on Windows since + // commits in 64MB chunks. const bool shouldRemap = (estimatedPrivateMapSize >= UncommittedBytesLimit) || + (systemMemoryPressurePercentage > 0.0) || (commitCounter % NumCommitsBeforeRemap == 0) || (mmapv1GlobalOptions.journalOptions & MMAPV1Options::JournalAlwaysRemap); @@ -794,11 +804,12 @@ namespace { } else { // We don't want to get close to the UncommittedBytesLimit - const double f = + const double remapMemFraction = estimatedPrivateMapSize / ((double)UncommittedBytesLimit); - if (f > remapFraction) { - remapFraction = f; - } + + remapFraction = std::max(remapMemFraction, remapFraction); + + remapFraction = std::max(systemMemoryPressurePercentage, remapFraction); } } else { diff --git a/src/mongo/util/processinfo.h b/src/mongo/util/processinfo.h index d415bf2eea7..18f6956d9b7 100644 --- a/src/mongo/util/processinfo.h +++ b/src/mongo/util/processinfo.h @@ -126,6 +126,13 @@ namespace mongo { static bool blockInMemory(const void* start); /** + * Returns a positive floating point number between 0.0 and 1.0 to inform MMapV1 how much it + * must remap pages to bring the system page file implementation back below a certain + * threshold. A number of 1.0 means remap everything. + */ + static double getSystemMemoryPressurePercentage(); + + /** * @return a pointer aligned to the start of the page the provided pointer belongs to. * * NOTE requires blockCheckSupported() == true diff --git a/src/mongo/util/processinfo_darwin.cpp b/src/mongo/util/processinfo_darwin.cpp index a88aadbbd34..994b4759e81 100644 --- a/src/mongo/util/processinfo_darwin.cpp +++ b/src/mongo/util/processinfo_darwin.cpp @@ -109,6 +109,10 @@ namespace mongo { return (int)( ti.resident_size / (1024 * 1024 ) ); } + double ProcessInfo::getSystemMemoryPressurePercentage() { + return 0.0; + } + void ProcessInfo::getExtraInfo(BSONObjBuilder& info) { struct task_events_info taskInfo; mach_msg_type_number_t taskInfoCount = TASK_EVENTS_INFO_COUNT; diff --git a/src/mongo/util/processinfo_freebsd.cpp b/src/mongo/util/processinfo_freebsd.cpp index f9ade4d34d6..8b7de762394 100644 --- a/src/mongo/util/processinfo_freebsd.cpp +++ b/src/mongo/util/processinfo_freebsd.cpp @@ -122,6 +122,10 @@ namespace mongo { return task->ki_rssize * sysconf( _SC_PAGESIZE ) / 1024 / 1024; // convert from pages to MB } + double ProcessInfo::getSystemMemoryPressurePercentage() { + return 0.0; + } + void ProcessInfo::SystemInfo::collectSystemInfo() { osType = "BSD"; osName = "FreeBSD"; diff --git a/src/mongo/util/processinfo_linux2.cpp b/src/mongo/util/processinfo_linux2.cpp index 2691fb7f054..1d0c6e585fb 100644 --- a/src/mongo/util/processinfo_linux2.cpp +++ b/src/mongo/util/processinfo_linux2.cpp @@ -396,6 +396,10 @@ namespace mongo { return (int)( p.getResidentSize() / ( 1024.0 * 1024 ) ); } + double ProcessInfo::getSystemMemoryPressurePercentage() { + return 0.0; + } + void ProcessInfo::getExtraInfo( BSONObjBuilder& info ) { // [dm] i don't think mallinfo works. (64 bit.) ?? struct mallinfo malloc_info = mallinfo(); // structure has same name as function that returns it. (see malloc.h) diff --git a/src/mongo/util/processinfo_none.cpp b/src/mongo/util/processinfo_none.cpp index 8d4216971a3..e39a422f909 100644 --- a/src/mongo/util/processinfo_none.cpp +++ b/src/mongo/util/processinfo_none.cpp @@ -55,6 +55,10 @@ namespace mongo { return -1; } + double ProcessInfo::getSystemMemoryPressurePercentage() { + return 0.0; + } + bool ProcessInfo::checkNumaEnabled() { return false; } diff --git a/src/mongo/util/processinfo_openbsd5.cpp b/src/mongo/util/processinfo_openbsd5.cpp index f36bc0cab54..cd21882497c 100644 --- a/src/mongo/util/processinfo_openbsd5.cpp +++ b/src/mongo/util/processinfo_openbsd5.cpp @@ -131,6 +131,10 @@ namespace mongo { return (task->p_vm_rssize * sysconf( _SC_PAGESIZE )) / 1048576; // convert from pages to MB } + double ProcessInfo::getSystemMemoryPressurePercentage() { + return 0.0; + } + void ProcessInfo::SystemInfo::collectSystemInfo() { osType = "BSD"; osName = "OpenBSD"; diff --git a/src/mongo/util/processinfo_sunos5.cpp b/src/mongo/util/processinfo_sunos5.cpp index 1a71f26bafb..5cbb51740b4 100644 --- a/src/mongo/util/processinfo_sunos5.cpp +++ b/src/mongo/util/processinfo_sunos5.cpp @@ -117,6 +117,10 @@ namespace mongo { return static_cast<int>(p.psinfo.pr_rssize / 1024); } + double ProcessInfo::getSystemMemoryPressurePercentage() { + return 0.0; + } + void ProcessInfo::getExtraInfo(BSONObjBuilder& info) { ProcUsage p; info.appendNumber("page_faults", static_cast<long long>(p.prusage.pr_majf)); diff --git a/src/mongo/util/processinfo_win32.cpp b/src/mongo/util/processinfo_win32.cpp index 1747b980225..e370076fec9 100644 --- a/src/mongo/util/processinfo_win32.cpp +++ b/src/mongo/util/processinfo_win32.cpp @@ -85,18 +85,69 @@ namespace mongo { int ProcessInfo::getVirtualMemorySize() { MEMORYSTATUSEX mse; mse.dwLength = sizeof(mse); - verify( GlobalMemoryStatusEx( &mse ) ); - DWORDLONG x = (mse.ullTotalVirtual - mse.ullAvailVirtual) / (1024 * 1024) ; - verify( x <= 0x7fffffff ); + BOOL status = GlobalMemoryStatusEx(&mse); + if (!status) { + DWORD gle = GetLastError(); + error() << "GlobalMemoryStatusEx failed with " << errnoWithDescription(gle); + fassert(28621, status); + } + + DWORDLONG x = (mse.ullTotalVirtual - mse.ullAvailVirtual) / (1024 * 1024); + invariant( x <= 0x7fffffff ); return (int) x; } int ProcessInfo::getResidentSize() { PROCESS_MEMORY_COUNTERS pmc; - verify( GetProcessMemoryInfo( GetCurrentProcess() , &pmc, sizeof(pmc) ) ); + BOOL status = GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc)); + if (!status) { + DWORD gle = GetLastError(); + error() << "GetProcessMemoryInfo failed with " << errnoWithDescription(gle); + fassert(28622, status); + } + return _wconvertmtos( pmc.WorkingSetSize ); } + double ProcessInfo::getSystemMemoryPressurePercentage() { + MEMORYSTATUSEX mse; + mse.dwLength = sizeof(mse); + BOOL status = GlobalMemoryStatusEx( &mse ); + if (!status) { + DWORD gle = GetLastError(); + error() << "GlobalMemoryStatusEx failed with " << errnoWithDescription(gle); + fassert(28623, status); + } + + DWORDLONG totalPageFile = mse.ullTotalPageFile; + if (totalPageFile == 0) { + return false; + } + + // If the page file is >= 50%, say we are low on system memory + // If the page file is >= 75%, we are running very low on system memory + // + DWORDLONG highWatermark = totalPageFile / 2; + DWORDLONG veryHighWatermark = 3 * (totalPageFile / 4); + + DWORDLONG usedPageFile = mse.ullTotalPageFile - mse.ullAvailPageFile; + + // Below the watermark, we are fine + // Also check we will not do a divide by zero below + if (usedPageFile < highWatermark || + veryHighWatermark <= highWatermark) { + return 0.0; + } + + // Above the high watermark, we tell MMapV1 how much to remap + // < 1.0, we have some pressure, but not much so do not be very aggressive + // 1.0 = we are at very high watermark, remap everything + // > 1.0, the user may run out of memory, remap everything + // i.e., Example (N - 50) / (75 - 50) + return static_cast<double>(usedPageFile - highWatermark) / + (veryHighWatermark - highWatermark); + } + void ProcessInfo::getExtraInfo(BSONObjBuilder& info) { MEMORYSTATUSEX mse; mse.dwLength = sizeof(mse); |