summaryrefslogtreecommitdiff
path: root/psutil
diff options
context:
space:
mode:
authorGiampaolo Rodola <g.rodola@gmail.com>2022-05-18 21:20:37 +0200
committerGitHub <noreply@github.com>2022-05-18 21:20:37 +0200
commitf1f299527634a425cb34b621d6201fa9172d3529 (patch)
tree7cb80b97b27aa56c830986c533b9f5bb255264b3 /psutil
parent33981759a67601b91d925a14403077e8d218c015 (diff)
downloadpsutil-f1f299527634a425cb34b621d6201fa9172d3529.tar.gz
[Linux] Speedup `Process.full_memory_info()` (#2108)
`Process.memory_full_info()` (reporting proecss USS/PSS/Swap memory) now reads ``/proc/pid/smaps_rollup`` instead of ``/proc/pids/smaps`` which makes it 5 times faster. Without patch: ``` ~/svn/psutil {linux-smaps-rollup}$ python3 -m timeit -s "import psutil; p = psutil.Process()" "p.memory_full_info()" 500 loops, best of 5: 518 usec per loop ``` With patch (5 times faster): ``` ~/svn/psutil {linux-smaps-rollup}$ python3 -m timeit -s "import psutil; p = psutil.Process()" "p.memory_full_info()" 2000 loops, best of 5: 111 usec per loop ``` ---- `make test-memleaks` suite, who heavily rely on `Process.memory_full_info()`, also received a nice speedup: Before patch: ``` $ make test-memleaks ---------------------------------------------------------------------- Ran 99 tests in 1.646s OK (skipped=9) SUCCESS ``` After patch: ``` $ make test-memleaks ---------------------------------------------------------------------- Ran 99 tests in 1.195s OK (skipped=9) SUCCESS ```
Diffstat (limited to 'psutil')
-rw-r--r--psutil/_pslinux.py47
-rwxr-xr-xpsutil/tests/test_linux.py35
2 files changed, 53 insertions, 29 deletions
diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py
index afcb78f5..e7cef439 100644
--- a/psutil/_pslinux.py
+++ b/psutil/_pslinux.py
@@ -77,7 +77,8 @@ __extra__all__ = [
POWER_SUPPLY_PATH = "/sys/class/power_supply"
-HAS_SMAPS = os.path.exists('/proc/%s/smaps' % os.getpid())
+HAS_PROC_SMAPS = os.path.exists('/proc/%s/smaps' % os.getpid())
+HAS_PROC_SMAPS_ROLLUP = os.path.exists('/proc/%s/smaps_rollup' % os.getpid())
HAS_PROC_IO_PRIORITY = hasattr(cext, "proc_ioprio_get")
HAS_CPU_AFFINITY = hasattr(cext, "proc_cpu_affinity_get")
@@ -1875,18 +1876,42 @@ class Process(object):
[int(x) * PAGESIZE for x in f.readline().split()[:7]]
return pmem(rss, vms, shared, text, lib, data, dirty)
- # /proc/pid/smaps does not exist on kernels < 2.6.14 or if
- # CONFIG_MMU kernel configuration option is not enabled.
- if HAS_SMAPS:
+ if HAS_PROC_SMAPS_ROLLUP or HAS_PROC_SMAPS:
@wrap_exceptions
- def memory_full_info(
+ def _parse_smaps_rollup(self):
+ # /proc/pid/smaps_rollup was added to Linux in 2017. Faster
+ # than /proc/pid/smaps. It reports higher PSS than */smaps
+ # (from 1k up to 200k higher; tested against all processes).
+ uss = pss = swap = 0
+ try:
+ with open_binary("{}/{}/smaps_rollup".format(
+ self._procfs_path, self.pid)) as f:
+ for line in f:
+ if line.startswith(b"Private_"):
+ # Private_Clean, Private_Dirty, Private_Hugetlb
+ uss += int(line.split()[1]) * 1024
+ elif line.startswith(b"Pss:"):
+ pss = int(line.split()[1]) * 1024
+ elif line.startswith(b"Swap:"):
+ swap = int(line.split()[1]) * 1024
+ except ProcessLookupError: # happens on readline()
+ if not pid_exists(self.pid):
+ raise NoSuchProcess(self.pid, self._name)
+ else:
+ raise ZombieProcess(self.pid, self._name, self._ppid)
+ return (uss, pss, swap)
+
+ @wrap_exceptions
+ def _parse_smaps(
self,
# Gets Private_Clean, Private_Dirty, Private_Hugetlb.
_private_re=re.compile(br"\nPrivate.*:\s+(\d+)"),
_pss_re=re.compile(br"\nPss\:\s+(\d+)"),
_swap_re=re.compile(br"\nSwap\:\s+(\d+)")):
- basic_mem = self.memory_info()
+ # /proc/pid/smaps does not exist on kernels < 2.6.14 or if
+ # CONFIG_MMU kernel configuration option is not enabled.
+
# Note: using 3 regexes is faster than reading the file
# line by line.
# XXX: on Python 3 the 2 regexes are 30% slower than on
@@ -1905,12 +1930,20 @@ class Process(object):
uss = sum(map(int, _private_re.findall(smaps_data))) * 1024
pss = sum(map(int, _pss_re.findall(smaps_data))) * 1024
swap = sum(map(int, _swap_re.findall(smaps_data))) * 1024
+ return (uss, pss, swap)
+
+ def memory_full_info(self):
+ if HAS_PROC_SMAPS_ROLLUP: # faster
+ uss, pss, swap = self._parse_smaps_rollup()
+ else:
+ uss, pss, swap = self._parse_smaps()
+ basic_mem = self.memory_info()
return pfullmem(*basic_mem + (uss, pss, swap))
else:
memory_full_info = memory_info
- if HAS_SMAPS:
+ if HAS_PROC_SMAPS:
@wrap_exceptions
def memory_maps(self):
diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py
index 8dd4caae..51e8be51 100755
--- a/psutil/tests/test_linux.py
+++ b/psutil/tests/test_linux.py
@@ -1775,28 +1775,19 @@ class TestSensorsFans(PsutilTestCase):
class TestProcess(PsutilTestCase):
@retry_on_failure()
- def test_memory_full_info(self):
- testfn = self.get_testfn()
- src = textwrap.dedent("""
- import time
- with open("%s", "w") as f:
- time.sleep(10)
- """ % testfn)
- sproc = self.pyrun(src)
- call_until(lambda: os.listdir('.'), "'%s' not in ret" % testfn)
- p = psutil.Process(sproc.pid)
- time.sleep(.1)
- mem = p.memory_full_info()
- maps = p.memory_maps(grouped=False)
+ def test_parse_smaps_vs_memory_maps(self):
+ sproc = self.spawn_testproc()
+ uss, pss, swap = psutil._pslinux.Process(sproc.pid)._parse_smaps()
+ maps = psutil.Process(sproc.pid).memory_maps(grouped=False)
self.assertAlmostEqual(
- mem.uss, sum([x.private_dirty + x.private_clean for x in maps]),
+ uss, sum([x.private_dirty + x.private_clean for x in maps]),
delta=4096)
self.assertAlmostEqual(
- mem.pss, sum([x.pss for x in maps]), delta=4096)
+ pss, sum([x.pss for x in maps]), delta=4096)
self.assertAlmostEqual(
- mem.swap, sum([x.swap for x in maps]), delta=4096)
+ swap, sum([x.swap for x in maps]), delta=4096)
- def test_memory_full_info_mocked(self):
+ def test_parse_smaps_mocked(self):
# See: https://github.com/giampaolo/psutil/issues/1222
with mock_open_content(
"/proc/%s/smaps" % os.getpid(),
@@ -1823,12 +1814,12 @@ class TestProcess(PsutilTestCase):
Locked: 19 kB
VmFlags: rd ex
""").encode()) as m:
- p = psutil.Process()
- mem = p.memory_full_info()
+ p = psutil._pslinux.Process(os.getpid())
+ uss, pss, swap = p._parse_smaps()
assert m.called
- self.assertEqual(mem.uss, (6 + 7 + 14) * 1024)
- self.assertEqual(mem.pss, 3 * 1024)
- self.assertEqual(mem.swap, 15 * 1024)
+ self.assertEqual(uss, (6 + 7 + 14) * 1024)
+ self.assertEqual(pss, 3 * 1024)
+ self.assertEqual(swap, 15 * 1024)
# On PYPY file descriptors are not closed fast enough.
@unittest.skipIf(PYPY, "unreliable on PYPY")