diff options
author | Giampaolo Rodola <g.rodola@gmail.com> | 2022-05-18 21:20:37 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-05-18 21:20:37 +0200 |
commit | f1f299527634a425cb34b621d6201fa9172d3529 (patch) | |
tree | 7cb80b97b27aa56c830986c533b9f5bb255264b3 /psutil | |
parent | 33981759a67601b91d925a14403077e8d218c015 (diff) | |
download | psutil-f1f299527634a425cb34b621d6201fa9172d3529.tar.gz |
[Linux] Speedup `Process.full_memory_info()` (#2108)
`Process.memory_full_info()` (reporting proecss USS/PSS/Swap memory) now reads ``/proc/pid/smaps_rollup`` instead of ``/proc/pids/smaps`` which makes it 5 times faster.
Without patch:
```
~/svn/psutil {linux-smaps-rollup}$ python3 -m timeit -s "import psutil; p = psutil.Process()" "p.memory_full_info()"
500 loops, best of 5: 518 usec per loop
```
With patch (5 times faster):
```
~/svn/psutil {linux-smaps-rollup}$ python3 -m timeit -s "import psutil; p = psutil.Process()" "p.memory_full_info()"
2000 loops, best of 5: 111 usec per loop
```
----
`make test-memleaks` suite, who heavily rely on `Process.memory_full_info()`, also received a nice speedup:
Before patch:
```
$ make test-memleaks
----------------------------------------------------------------------
Ran 99 tests in 1.646s
OK (skipped=9)
SUCCESS
```
After patch:
```
$ make test-memleaks
----------------------------------------------------------------------
Ran 99 tests in 1.195s
OK (skipped=9)
SUCCESS
```
Diffstat (limited to 'psutil')
-rw-r--r-- | psutil/_pslinux.py | 47 | ||||
-rwxr-xr-x | psutil/tests/test_linux.py | 35 |
2 files changed, 53 insertions, 29 deletions
diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index afcb78f5..e7cef439 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -77,7 +77,8 @@ __extra__all__ = [ POWER_SUPPLY_PATH = "/sys/class/power_supply" -HAS_SMAPS = os.path.exists('/proc/%s/smaps' % os.getpid()) +HAS_PROC_SMAPS = os.path.exists('/proc/%s/smaps' % os.getpid()) +HAS_PROC_SMAPS_ROLLUP = os.path.exists('/proc/%s/smaps_rollup' % os.getpid()) HAS_PROC_IO_PRIORITY = hasattr(cext, "proc_ioprio_get") HAS_CPU_AFFINITY = hasattr(cext, "proc_cpu_affinity_get") @@ -1875,18 +1876,42 @@ class Process(object): [int(x) * PAGESIZE for x in f.readline().split()[:7]] return pmem(rss, vms, shared, text, lib, data, dirty) - # /proc/pid/smaps does not exist on kernels < 2.6.14 or if - # CONFIG_MMU kernel configuration option is not enabled. - if HAS_SMAPS: + if HAS_PROC_SMAPS_ROLLUP or HAS_PROC_SMAPS: @wrap_exceptions - def memory_full_info( + def _parse_smaps_rollup(self): + # /proc/pid/smaps_rollup was added to Linux in 2017. Faster + # than /proc/pid/smaps. It reports higher PSS than */smaps + # (from 1k up to 200k higher; tested against all processes). + uss = pss = swap = 0 + try: + with open_binary("{}/{}/smaps_rollup".format( + self._procfs_path, self.pid)) as f: + for line in f: + if line.startswith(b"Private_"): + # Private_Clean, Private_Dirty, Private_Hugetlb + uss += int(line.split()[1]) * 1024 + elif line.startswith(b"Pss:"): + pss = int(line.split()[1]) * 1024 + elif line.startswith(b"Swap:"): + swap = int(line.split()[1]) * 1024 + except ProcessLookupError: # happens on readline() + if not pid_exists(self.pid): + raise NoSuchProcess(self.pid, self._name) + else: + raise ZombieProcess(self.pid, self._name, self._ppid) + return (uss, pss, swap) + + @wrap_exceptions + def _parse_smaps( self, # Gets Private_Clean, Private_Dirty, Private_Hugetlb. _private_re=re.compile(br"\nPrivate.*:\s+(\d+)"), _pss_re=re.compile(br"\nPss\:\s+(\d+)"), _swap_re=re.compile(br"\nSwap\:\s+(\d+)")): - basic_mem = self.memory_info() + # /proc/pid/smaps does not exist on kernels < 2.6.14 or if + # CONFIG_MMU kernel configuration option is not enabled. + # Note: using 3 regexes is faster than reading the file # line by line. # XXX: on Python 3 the 2 regexes are 30% slower than on @@ -1905,12 +1930,20 @@ class Process(object): uss = sum(map(int, _private_re.findall(smaps_data))) * 1024 pss = sum(map(int, _pss_re.findall(smaps_data))) * 1024 swap = sum(map(int, _swap_re.findall(smaps_data))) * 1024 + return (uss, pss, swap) + + def memory_full_info(self): + if HAS_PROC_SMAPS_ROLLUP: # faster + uss, pss, swap = self._parse_smaps_rollup() + else: + uss, pss, swap = self._parse_smaps() + basic_mem = self.memory_info() return pfullmem(*basic_mem + (uss, pss, swap)) else: memory_full_info = memory_info - if HAS_SMAPS: + if HAS_PROC_SMAPS: @wrap_exceptions def memory_maps(self): diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 8dd4caae..51e8be51 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1775,28 +1775,19 @@ class TestSensorsFans(PsutilTestCase): class TestProcess(PsutilTestCase): @retry_on_failure() - def test_memory_full_info(self): - testfn = self.get_testfn() - src = textwrap.dedent(""" - import time - with open("%s", "w") as f: - time.sleep(10) - """ % testfn) - sproc = self.pyrun(src) - call_until(lambda: os.listdir('.'), "'%s' not in ret" % testfn) - p = psutil.Process(sproc.pid) - time.sleep(.1) - mem = p.memory_full_info() - maps = p.memory_maps(grouped=False) + def test_parse_smaps_vs_memory_maps(self): + sproc = self.spawn_testproc() + uss, pss, swap = psutil._pslinux.Process(sproc.pid)._parse_smaps() + maps = psutil.Process(sproc.pid).memory_maps(grouped=False) self.assertAlmostEqual( - mem.uss, sum([x.private_dirty + x.private_clean for x in maps]), + uss, sum([x.private_dirty + x.private_clean for x in maps]), delta=4096) self.assertAlmostEqual( - mem.pss, sum([x.pss for x in maps]), delta=4096) + pss, sum([x.pss for x in maps]), delta=4096) self.assertAlmostEqual( - mem.swap, sum([x.swap for x in maps]), delta=4096) + swap, sum([x.swap for x in maps]), delta=4096) - def test_memory_full_info_mocked(self): + def test_parse_smaps_mocked(self): # See: https://github.com/giampaolo/psutil/issues/1222 with mock_open_content( "/proc/%s/smaps" % os.getpid(), @@ -1823,12 +1814,12 @@ class TestProcess(PsutilTestCase): Locked: 19 kB VmFlags: rd ex """).encode()) as m: - p = psutil.Process() - mem = p.memory_full_info() + p = psutil._pslinux.Process(os.getpid()) + uss, pss, swap = p._parse_smaps() assert m.called - self.assertEqual(mem.uss, (6 + 7 + 14) * 1024) - self.assertEqual(mem.pss, 3 * 1024) - self.assertEqual(mem.swap, 15 * 1024) + self.assertEqual(uss, (6 + 7 + 14) * 1024) + self.assertEqual(pss, 3 * 1024) + self.assertEqual(swap, 15 * 1024) # On PYPY file descriptors are not closed fast enough. @unittest.skipIf(PYPY, "unreliable on PYPY") |