diff options
author | Giampaolo Rodola <g.rodola@gmail.com> | 2016-09-21 12:47:47 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2016-09-21 12:47:47 +0200 |
commit | ec7957c7bfd2abef1042bf4f2f72f586400c0d24 (patch) | |
tree | 53c92201fd08769dc783254d012b387962313b7d | |
parent | 1e707020f116627d5e3d060164c32b56257a1938 (diff) | |
parent | e0cea3bc9162d1083c0198d0731c8f734d2eacc0 (diff) | |
download | psutil-ec7957c7bfd2abef1042bf4f2f72f586400c0d24.tar.gz |
Merge pull request #890 from giampaolo/887-linux-free-mem-standardization
887 linux free mem standardization
-rw-r--r-- | HISTORY.rst | 5 | ||||
-rw-r--r-- | docs/index.rst | 36 | ||||
-rw-r--r-- | psutil/__init__.py | 2 | ||||
-rw-r--r-- | psutil/_pslinux.py | 201 | ||||
-rw-r--r-- | psutil/tests/test_linux.py | 185 | ||||
-rwxr-xr-x | scripts/meminfo.py | 1 | ||||
-rwxr-xr-x | scripts/top.py | 3 |
7 files changed, 350 insertions, 83 deletions
diff --git a/HISTORY.rst b/HISTORY.rst index b5775bfd..c5ef6d72 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,7 +1,7 @@ Bug tracker at https://github.com/giampaolo/psutil/issues -4.3.2 - XXXX-XX-XX +4.4.0 - XXXX-XX-XX ================== **Bug fixes** @@ -10,6 +10,9 @@ Bug tracker at https://github.com/giampaolo/psutil/issues - #880: [Windows] Handle race condition inside psutil_net_connections. - #885: ValueError is raised if a negative integer is passed to cpu_percent() functions. +- #887: [Linux] virtual_memory()'s 'available' and 'used' values are more + precise and match "free" cmdline utility. "available" also takes into + account LCX containers preventing "available" to overflow "total". 4.3.1 - 2016-09-01 diff --git a/docs/index.rst b/docs/index.rst index fa83aefa..76a93558 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -172,24 +172,24 @@ Memory .. function:: virtual_memory() Return statistics about system memory usage as a namedtuple including the - following fields, expressed in bytes: - - - **total**: total physical memory available. - - **available**: the actual amount of available memory that can be given - instantly to processes that request more memory in bytes; this is - calculated by summing different memory values depending on the platform - (e.g. ``(free + buffers + cached)`` on Linux) and it is supposed to be used - to monitor actual memory usage in a cross platform fashion. - - **percent**: the percentage usage calculated as - ``(total - available) / total * 100``. - - **used**: memory used, calculated differently depending on the platform and - designed for informational purposes only. - - **free**: memory not being used at all (zeroed) that is readily available; - note that this doesn't reflect the actual memory available (use 'available' - instead). + following fields, expressed in bytes. Main metrics: - Platform-specific fields: + - **total**: total physical memory. + - **available**: the memory that can be given instantly to processes without + the system going into swap. + This is calculated by summing different memory values depending on the + platform and it is supposed to be used to monitor actual memory usage in a + cross platform fashion. + + Other metrics: + - **used**: memory used, calculated differently depending on the platform and + designed for informational purposes only. **total - free** does not + necessarily match **used**. + - **free**: memory not being used at all (zeroed) that is readily available; + note that this doesn't reflect the actual memory available (use + **available** instead). **total - used** does not necessarily match + **free**. - **active** *(UNIX)*: memory currently in use or very recently used, and so it is in RAM. - **inactive** *(UNIX)*: memory that is marked as not used. @@ -221,6 +221,10 @@ Memory .. versionchanged:: 4.2.0 added *shared* metrics on Linux. + .. versionchanged:: 4.4.0 *available* and *used* values on Linux are more + precise and match "free" cmdline utility. + + .. function:: swap_memory() Return system swap memory statistics as a namedtuple including the following diff --git a/psutil/__init__.py b/psutil/__init__.py index 0a6f3ec6..020a0bdd 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -187,7 +187,7 @@ __all__ = [ ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "4.3.2" +__version__ = "4.4.0" version_info = tuple([int(num) for num in __version__.split('.')]) AF_LINK = _psplatform.AF_LINK _TOTAL_PHYMEM = None diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 7f6e0405..c3750fec 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -288,62 +288,167 @@ except Exception: # ===================================================================== +def calculate_avail_vmem(mems): + """Fallback for kernels < 3.14 where /proc/meminfo does not provide + "MemAvailable:" column (see: https://blog.famzah.net/2014/09/24/). + This code reimplements the algorithm outlined here: + https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/ + commit/?id=34e431b0ae398fc54ea69ff85ec700722c9da773 + + XXX: on recent kernels this calculation differs by ~1.5% than + "MemAvailable:" as it's calculated slightly differently, see: + https://gitlab.com/procps-ng/procps/issues/42 + https://github.com/famzah/linux-memavailable-procfs/issues/2 + It is still way more realistic than doing (free + cached) though. + """ + # Fallback for very old distros. According to + # https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/ + # commit/?id=34e431b0ae398fc54ea69ff85ec700722c9da773 + # ...long ago "avail" was calculated as (free + cached). + # We might fallback in such cases: + # "Active(file)" not available: 2.6.28 / Dec 2008 + # "Inactive(file)" not available: 2.6.28 / Dec 2008 + # "SReclaimable:" not available: 2.6.19 / Nov 2006 + # /proc/zoneinfo not available: 2.6.13 / Aug 2005 + free = mems[b'MemFree:'] + fallback = free + mems.get(b"Cached:", 0) + try: + lru_active_file = mems[b'Active(file):'] + lru_inactive_file = mems[b'Inactive(file):'] + slab_reclaimable = mems[b'SReclaimable:'] + except KeyError: + return fallback + try: + f = open_binary('%s/zoneinfo' % get_procfs_path()) + except IOError: + return fallback # kernel 2.6.13 + + watermark_low = 0 + with f: + for line in f: + line = line.strip() + if line.startswith(b'low'): + watermark_low += int(line.split()[1]) + watermark_low *= PAGESIZE + watermark_low = watermark_low + + avail = free - watermark_low + pagecache = lru_active_file + lru_inactive_file + pagecache -= min(pagecache / 2, watermark_low) + avail += pagecache + avail += slab_reclaimable - min(slab_reclaimable / 2.0, watermark_low) + return int(avail) + + def virtual_memory(): - total, free, buffers, shared, _, _, unit_multiplier = cext.linux_sysinfo() - total *= unit_multiplier - free *= unit_multiplier - buffers *= unit_multiplier - # Note: this (on my Ubuntu 14.04, kernel 3.13 at least) may be 0. - # If so, it will be determined from /proc/meminfo. - shared *= unit_multiplier or None - if shared == 0: - shared = None - - cached = active = inactive = None + """Report virtual memory stats. + This implementation matches "free" and "vmstat -s" cmdline + utility values and procps-ng-3.3.12 source was used as a reference + (2016-09-18): + https://gitlab.com/procps-ng/procps/blob/ + 24fd2605c51fccc375ab0287cec33aa767f06718/proc/sysinfo.c + For reference, procps-ng-3.3.10 is the version available on Ubuntu + 16.04. + + Note about "available" memory: up until psutil 4.3 it was + calculated as "avail = (free + buffers + cached)". Now + "MemAvailable:" column (kernel 3.14) from /proc/meminfo is used as + it's more accurate. + That matches "available" column in newer versions of "free". + """ + missing_fields = [] + mems = {} with open_binary('%s/meminfo' % get_procfs_path()) as f: for line in f: - if cached is None and line.startswith(b"Cached:"): - cached = int(line.split()[1]) * 1024 - elif active is None and line.startswith(b"Active:"): - active = int(line.split()[1]) * 1024 - elif inactive is None and line.startswith(b"Inactive:"): - inactive = int(line.split()[1]) * 1024 - # From "man free": - # The shared memory column represents either the MemShared - # value (2.4 kernels) or the Shmem value (2.6+ kernels) taken - # from the /proc/meminfo file. The value is zero if none of - # the entries is exported by the kernel. - elif shared is None and \ - line.startswith(b"MemShared:") or \ - line.startswith(b"Shmem:"): - shared = int(line.split()[1]) * 1024 - - missing = [] - if cached is None: - missing.append('cached') + fields = line.split() + mems[fields[0]] = int(fields[1]) * 1024 + + # /proc doc states that the available fields in /proc/meminfo vary + # by architecture and compile options, but these 3 values are also + # returned by sysinfo(2); as such we assume they are always there. + total = mems[b'MemTotal:'] + free = mems[b'MemFree:'] + buffers = mems[b'Buffers:'] + + try: + cached = mems[b"Cached:"] + except KeyError: cached = 0 - if active is None: - missing.append('active') + missing_fields.append('cached') + else: + # "free" cmdline utility sums reclaimable to cached. + # Older versions of procps used to add slab memory instead. + # This got changed in: + # https://gitlab.com/procps-ng/procps/commit/ + # 05d751c4f076a2f0118b914c5e51cfbb4762ad8e + cached += mems.get(b"SReclaimable:", 0) # since kernel 2.6.19 + + try: + shared = mems[b'Shmem:'] # since kernel 2.6.32 + except KeyError: + try: + shared = mems[b'MemShared:'] # kernels 2.4 + except KeyError: + shared = 0 + missing_fields.append('shared') + + try: + active = mems[b"Active:"] + except KeyError: active = 0 - if inactive is None: - missing.append('inactive') - inactive = 0 - if shared is None: - missing.append('shared') - shared = 0 - if missing: + missing_fields.append('active') + + try: + inactive = mems[b"Inactive:"] + except KeyError: + try: + inactive = \ + mems[b"Inact_dirty:"] + \ + mems[b"Inact_clean:"] + \ + mems[b"Inact_laundry:"] + except KeyError: + inactive = 0 + missing_fields.append('inactive') + + used = total - free - cached - buffers + if used < 0: + # May be symptomatic of running within a LCX container where such + # values will be dramatically distorted over those of the host. + used = total - free + + # - starting from 4.4.0 we match free's "available" column. + # Before 4.4.0 we calculated it as (free + buffers + cached) + # which matched htop. + # - free and htop available memory differs as per: + # http://askubuntu.com/a/369589 + # http://unix.stackexchange.com/a/65852/168884 + # - MemAvailable has been introduced in kernel 3.14 + try: + avail = mems[b'MemAvailable:'] + except KeyError: + avail = calculate_avail_vmem(mems) + + if avail < 0: + avail = 0 + missing_fields.append('available') + + # If avail is greater than total or our calculation overflows, + # that's symptomatic of running within a LCX container where such + # values will be dramatically distorted over those of the host. + # https://gitlab.com/procps-ng/procps/blob/ + # 24fd2605c51fccc375ab0287cec33aa767f06718/proc/sysinfo.c#L764 + if avail > total: + avail = free + + percent = usage_percent((total - avail), total, _round=1) + + # Warn about missing metrics which are set to 0. + if missing_fields: msg = "%s memory stats couldn't be determined and %s set to 0" % ( - ", ".join(missing), - "was" if len(missing) == 1 else "were") + ", ".join(missing_fields), + "was" if len(missing_fields) == 1 else "were") warnings.warn(msg, RuntimeWarning) - # Note: this value matches "htop" perfectly. - avail = free + buffers + cached - # Note: this value matches "free", but not all the time, see: - # https://github.com/giampaolo/psutil/issues/685#issuecomment-202914057 - used = total - free - # Note: this value matches "htop" perfectly. - percent = usage_percent((total - avail), total, _round=1) return svmem(total, avail, percent, used, free, active, inactive, buffers, cached, shared) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 9f7c25fb..7ee92597 100644 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -6,6 +6,7 @@ """Linux specific tests.""" +from __future__ import division import collections import contextlib import errno @@ -117,8 +118,9 @@ def free_physmem(): if line.startswith('Mem'): total, used, free, shared = \ [int(x) for x in line.split()[1:5]] - nt = collections.namedtuple('free', 'total used free shared') - return nt(total, used, free, shared) + nt = collections.namedtuple( + 'free', 'total used free shared output') + return nt(total, used, free, shared, out) raise ValueError( "can't find 'Mem' in 'free' output:\n%s" % '\n'.join(lines)) @@ -132,6 +134,11 @@ def vmstat(stat): raise ValueError("can't find %r in 'vmstat' output" % stat) +def get_free_version_info(): + out = sh("free -V").strip() + return tuple(map(int, out.split()[-1].split('.'))) + + # ===================================================================== # system virtual memory # ===================================================================== @@ -148,12 +155,20 @@ class TestSystemVirtualMemory(unittest.TestCase): psutil_value = psutil.virtual_memory().total self.assertAlmostEqual(vmstat_value, psutil_value) + # Older versions of procps used slab memory to calculate used memory. + # This got changed in: + # https://gitlab.com/procps-ng/procps/commit/ + # 05d751c4f076a2f0118b914c5e51cfbb4762ad8e + @unittest.skipUnless( + LINUX and get_free_version_info() >= (3, 3, 12), "old free version") @retry_before_failing() def test_used(self): - free_value = free_physmem().used + free = free_physmem() + free_value = free.used psutil_value = psutil.virtual_memory().used self.assertAlmostEqual( - free_value, psutil_value, delta=MEMORY_TOLERANCE) + free_value, psutil_value, delta=MEMORY_TOLERANCE, + msg='%s %s \n%s' % (free_value, psutil_value, free.output)) @retry_before_failing() def test_free(self): @@ -188,31 +203,174 @@ class TestSystemVirtualMemory(unittest.TestCase): vmstat_value, psutil_value, delta=MEMORY_TOLERANCE) @retry_before_failing() - @unittest.skipIf(TRAVIS, "fails on travis") def test_shared(self): - free_value = free_physmem().shared + free = free_physmem() + free_value = free.shared if free_value == 0: raise unittest.SkipTest("free does not support 'shared' column") psutil_value = psutil.virtual_memory().shared self.assertAlmostEqual( - free_value, psutil_value, delta=MEMORY_TOLERANCE) + free_value, psutil_value, delta=MEMORY_TOLERANCE, + msg='%s %s \n%s' % (free_value, psutil_value, free.output)) - # --- mocked tests + @retry_before_failing() + def test_available(self): + # "free" output format has changed at some point: + # https://github.com/giampaolo/psutil/issues/538#issuecomment-147192098 + out = sh("free -b") + lines = out.split('\n') + if 'available' not in lines[0]: + raise unittest.SkipTest("free does not support 'available' column") + else: + free_value = int(lines[1].split()[-1]) + psutil_value = psutil.virtual_memory().available + self.assertAlmostEqual( + free_value, psutil_value, delta=MEMORY_TOLERANCE, + msg='%s %s \n%s' % (free_value, psutil_value, out)) def test_warnings_mocked(self): - with mock.patch('psutil._pslinux.open', create=True) as m: + def open_mock(name, *args, **kwargs): + if name == '/proc/meminfo': + return io.BytesIO(textwrap.dedent("""\ + Active(anon): 6145416 kB + Active(file): 2950064 kB + Buffers: 287952 kB + Inactive(anon): 574764 kB + Inactive(file): 1567648 kB + MemAvailable: 6574984 kB + MemFree: 2057400 kB + MemTotal: 16325648 kB + SReclaimable: 346648 kB + """).encode()) + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, create=True, side_effect=open_mock) as m: with warnings.catch_warnings(record=True) as ws: warnings.simplefilter("always") - ret = psutil._pslinux.virtual_memory() + ret = psutil.virtual_memory() assert m.called self.assertEqual(len(ws), 1) w = ws[0] self.assertTrue(w.filename.endswith('psutil/_pslinux.py')) self.assertIn( "memory stats couldn't be determined", str(w.message)) + self.assertIn("cached", str(w.message)) + self.assertIn("shared", str(w.message)) + self.assertIn("active", str(w.message)) + self.assertIn("inactive", str(w.message)) self.assertEqual(ret.cached, 0) self.assertEqual(ret.active, 0) self.assertEqual(ret.inactive, 0) + self.assertEqual(ret.shared, 0) + + def test_avail_old_percent(self): + # Make sure that our calculation of avail mem for old kernels + # is off by max 2%. + from psutil._pslinux import calculate_avail_vmem + from psutil._pslinux import open_binary + + mems = {} + with open_binary('/proc/meminfo') as f: + for line in f: + fields = line.split() + mems[fields[0]] = int(fields[1]) * 1024 + + a = calculate_avail_vmem(mems) + if b'MemAvailable:' in mems: + b = mems[b'MemAvailable:'] + diff_percent = abs(a - b) / a * 100 + self.assertLess(diff_percent, 2) + + def test_avail_old_comes_from_kernel(self): + # Make sure "MemAvailable:" coluimn is used instead of relying + # on our internal algorithm to calculate avail mem. + def open_mock(name, *args, **kwargs): + if name == "/proc/meminfo": + return io.BytesIO(textwrap.dedent("""\ + Active: 9444728 kB + Active(anon): 6145416 kB + Active(file): 2950064 kB + Buffers: 287952 kB + Cached: 4818144 kB + Inactive(file): 1578132 kB + Inactive(anon): 574764 kB + Inactive(file): 1567648 kB + MemAvailable: 6574984 kB + MemFree: 2057400 kB + MemTotal: 16325648 kB + Shmem: 577588 kB + SReclaimable: 346648 kB + """).encode()) + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, create=True, side_effect=open_mock) as m: + ret = psutil.virtual_memory() + assert m.called + self.assertEqual(ret.available, 6574984 * 1024) + + def test_avail_old_missing_fields(self): + # Remove Active(file), Inactive(file) and SReclaimable + # from /proc/meminfo and make sure the fallback is used + # (free + cached), + def open_mock(name, *args, **kwargs): + if name == "/proc/meminfo": + return io.BytesIO(textwrap.dedent("""\ + Active: 9444728 kB + Active(anon): 6145416 kB + Buffers: 287952 kB + Cached: 4818144 kB + Inactive(file): 1578132 kB + Inactive(anon): 574764 kB + MemFree: 2057400 kB + MemTotal: 16325648 kB + Shmem: 577588 kB + """).encode()) + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, create=True, side_effect=open_mock) as m: + ret = psutil.virtual_memory() + assert m.called + self.assertEqual(ret.available, 2057400 * 1024 + 4818144 * 1024) + + def test_avail_old_missing_zoneinfo(self): + # Remove /proc/zoneinfo file. Make sure fallback is used + # (free + cached). + def open_mock(name, *args, **kwargs): + if name == "/proc/meminfo": + return io.BytesIO(textwrap.dedent("""\ + Active: 9444728 kB + Active(anon): 6145416 kB + Active(file): 2950064 kB + Buffers: 287952 kB + Cached: 4818144 kB + Inactive(file): 1578132 kB + Inactive(anon): 574764 kB + Inactive(file): 1567648 kB + MemFree: 2057400 kB + MemTotal: 16325648 kB + Shmem: 577588 kB + SReclaimable: 346648 kB + """).encode()) + elif name == "/proc/zoneinfo": + raise IOError(errno.ENOENT, 'no such file or directory') + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, create=True, side_effect=open_mock) as m: + ret = psutil.virtual_memory() + assert m.called + self.assertEqual(ret.available, 2057400 * 1024 + 4818144 * 1024) # ===================================================================== @@ -247,7 +405,7 @@ class TestSystemSwapMemory(unittest.TestCase): with mock.patch('psutil._pslinux.open', create=True) as m: with warnings.catch_warnings(record=True) as ws: warnings.simplefilter("always") - ret = psutil._pslinux.swap_memory() + ret = psutil.swap_memory() assert m.called self.assertEqual(len(ws), 1) w = ws[0] @@ -370,6 +528,7 @@ class TestSystemCPU(unittest.TestCase): @unittest.skipUnless(LINUX, "not a Linux system") class TestSystemCPUStats(unittest.TestCase): + @unittest.skipIf(TRAVIS, "fails on Travis") def test_ctx_switches(self): vmstat_value = vmstat("context switches") psutil_value = psutil.cpu_stats().ctx_switches @@ -483,7 +642,6 @@ class TestSystemNetwork(unittest.TestCase): """)) else: return orig_open(name, *args, **kwargs) - return orig_open(name, *args) orig_open = open patch_point = 'builtins.open' if PY3 else '__builtin__.open' @@ -566,7 +724,6 @@ class TestSystemDisks(unittest.TestCase): u(" 3 0 1 hda 2 3 4 5 6 7 8 9 10 11 12")) else: return orig_open(name, *args, **kwargs) - return orig_open(name, *args) orig_open = open patch_point = 'builtins.open' if PY3 else '__builtin__.open' @@ -599,7 +756,6 @@ class TestSystemDisks(unittest.TestCase): u(" 3 0 hda 1 2 3 4 5 6 7 8 9 10 11")) else: return orig_open(name, *args, **kwargs) - return orig_open(name, *args) orig_open = open patch_point = 'builtins.open' if PY3 else '__builtin__.open' @@ -634,7 +790,6 @@ class TestSystemDisks(unittest.TestCase): u(" 3 1 hda 1 2 3 4")) else: return orig_open(name, *args, **kwargs) - return orig_open(name, *args) orig_open = open patch_point = 'builtins.open' if PY3 else '__builtin__.open' diff --git a/scripts/meminfo.py b/scripts/meminfo.py index 3546960b..88c3a937 100755 --- a/scripts/meminfo.py +++ b/scripts/meminfo.py @@ -64,5 +64,6 @@ def main(): print('\nSWAP\n----') pprint_ntuple(psutil.swap_memory()) + if __name__ == '__main__': main() diff --git a/scripts/top.py b/scripts/top.py index 1caa8136..0c99047e 100755 --- a/scripts/top.py +++ b/scripts/top.py @@ -137,11 +137,10 @@ def print_header(procs_status, num_procs): perc)) mem = psutil.virtual_memory() dashes, empty_dashes = get_dashes(mem.percent) - used = mem.total - mem.available line = " Mem [%s%s] %5s%% %6s/%s" % ( dashes, empty_dashes, mem.percent, - str(int(used / 1024 / 1024)) + "M", + str(int(mem.used / 1024 / 1024)) + "M", str(int(mem.total / 1024 / 1024)) + "M" ) print_line(line) |