diff options
| -rw-r--r-- | docs/index.rst | 55 | ||||
| -rw-r--r-- | psutil/__init__.py | 87 | ||||
| -rw-r--r-- | psutil/_psbsd.py | 50 | ||||
| -rw-r--r-- | psutil/_pslinux.py | 17 | ||||
| -rw-r--r-- | psutil/_psosx.py | 5 | ||||
| -rw-r--r-- | psutil/_psutil_common.c | 15 | ||||
| -rw-r--r-- | psutil/_psutil_common.h | 1 | ||||
| -rw-r--r-- | psutil/_psutil_osx.c | 2 | ||||
| -rw-r--r-- | psutil/_psutil_windows.c | 5 | ||||
| -rw-r--r-- | psutil/_pswindows.py | 12 | ||||
| -rw-r--r-- | psutil/arch/osx/cpu.c | 13 | ||||
| -rw-r--r-- | psutil/arch/osx/cpu.h | 1 | ||||
| -rw-r--r-- | psutil/arch/windows/cpu.c | 45 | ||||
| -rw-r--r-- | psutil/arch/windows/cpu.h | 2 | ||||
| -rwxr-xr-x | psutil/tests/test_aix.py | 2 | ||||
| -rwxr-xr-x | psutil/tests/test_bsd.py | 4 | ||||
| -rwxr-xr-x | psutil/tests/test_contracts.py | 7 | ||||
| -rwxr-xr-x | psutil/tests/test_linux.py | 96 | ||||
| -rwxr-xr-x | psutil/tests/test_memleaks.py | 18 | ||||
| -rwxr-xr-x | psutil/tests/test_osx.py | 28 | ||||
| -rwxr-xr-x | psutil/tests/test_system.py | 98 | ||||
| -rwxr-xr-x | psutil/tests/test_windows.py | 15 | ||||
| -rwxr-xr-x | scripts/internal/print_api_speed.py | 8 |
23 files changed, 424 insertions, 162 deletions
diff --git a/docs/index.rst b/docs/index.rst index d7d32632..595130cd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -194,34 +194,59 @@ CPU .. versionchanged:: 4.1.0 two new *interrupt* and *dpc* fields are returned on Windows. -.. function:: cpu_count(logical=True) - - Return the number of logical CPUs in the system (same as `os.cpu_count`_ - in Python 3.4) or ``None`` if undetermined. - "logical CPUs" means the number of physical cores multiplied by the number - of threads that can run on each core (this is known as Hyper Threading). - If *logical* is ``False`` return the number of physical cores only, or - ``None`` if undetermined. - On OpenBSD and NetBSD ``psutil.cpu_count(logical=False)`` always return - ``None``. +.. function:: cpu_count(kind="logical") + + Return the number of CPUs in the system (various kinds) or ``None`` if + undetermined (same as `os.cpu_count`_). + The *kind* parameter dictates the desired CPU count to get: + + - **logical**: the total number of CPUs in the system (same as `os.cpu_count`_). + Usually this is the number of cores multiplied by the number of threads + that can run on each core (this is known as Hyper Threading). + - **cores**: the number of physical CPU cores + - **sockets**: the number of physical CPU sockets on the motherboard + - **numa**: the number of CPU NUMA nodes + + ======== =========== ========== ========== ======== ======== ======== ======= ======= + Type Linux Windows macOS FreeBSD OpenBSD NetBSD SunOS AIX + ======== =========== ========== ========== ======== ======== ======== ======= ======= + logical X X X X X X X X + cores X X X X X X + sockets X X X X + numa X X + ======== =========== ========== ========== ======== ======== ======== ======= ======= + Example on a system having 2 cores + Hyper Threading: + .. code-block:: python + >>> import psutil - >>> psutil.cpu_count() + >>> psutil.cpu_count(kind="logical") 4 - >>> psutil.cpu_count(logical=False) + >>> psutil.cpu_count(kind="cores") 2 + >>> psutil.cpu_count(kind="sockets") + 1 - Note that ``psutil.cpu_count()`` may not necessarily be equivalent to the - actual number of CPUs the current process can use. + Note that "logical" count may not necessarily be equivalent to the actual + number of CPUs the current process can use. That can vary in case process CPU affinity has been changed, Linux cgroups are being used or (in case of Windows) on systems using processor groups or having more than 64 CPUs. - The number of usable CPUs can be obtained with: + A reasonably reliable number of usable CPUs can be obtained with: >>> len(psutil.Process().cpu_affinity()) 1 + .. versionchanged:: 5.8.1 "kind" parameter was added, "logical" parameter was + deprecated, "sockets" and "numa" CPU counts were added. + + .. warning:: + the original signature of this function was ``cpu_count(logical=True)``. + That signature got deprecated in 5.8.1. The old function invocations + ``cpu_count(<bool>)`` and ``cpu_count(logical=<str>)`` still work though, + but will raise a ``DeprecationWarning``. + .. function:: cpu_stats() Return various CPU statistics as a named tuple: diff --git a/psutil/__init__.py b/psutil/__init__.py index 44efb7ff..7e6afada 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -31,6 +31,7 @@ import subprocess import sys import threading import time +import warnings try: import pwd except ImportError: @@ -1558,27 +1559,81 @@ def wait_procs(procs, timeout=None, callback=None): # ===================================================================== -def cpu_count(logical=True): - """Return the number of logical CPUs in the system (same as - os.cpu_count() in Python 3.4). +def cpu_count(kind="logical", **_kwargs): + """Return the number of CPUs in the system (various kinds). - If *logical* is False return the number of physical cores only - (e.g. hyper thread CPUs are excluded). + DEPRECATION WARNING: before, the signature of this function was + cpu_count(logical=True), then it was replaced with: + cpu_count(kind="logical"). - Return None if undetermined. + The old argument is still supported, even if it's not part of the + signature anymore (it's hidden in the private **_kwargs). + The followings statements are equivalent and should be replaced in + new code: - The return value is cached after first call. - If desired cache can be cleared like this: - - >>> psutil.cpu_count.cache_clear() + cpu_count(logical=True) == cpu_count(kind="logical") + cpu_count(logical=False) == cpu_count(kind="cores") + cpu_count(True) == cpu_count("logical") + cpu_count(False) == cpu_count("cores") """ - if logical: - ret = _psplatform.cpu_count_logical() + if isinstance(kind, bool): + msg = "cpu_count(%s) function invocation is deprecated; use " \ + "cpu_count(%r) instead" % (kind, "logical" if kind else "cores") + warnings.warn(msg, category=DeprecationWarning, stacklevel=2) + kind = "logical" if kind else "cores" + if _kwargs: + if list(_kwargs.keys()) == ["logical"]: + msg = "cpu_count(logical=%s) function invocation is deprecated; " \ + "use cpu_count(kind=%r) instead" + msg = msg % (_kwargs["logical"], "logical" if _kwargs["logical"] + else "cores") + kind = "logical" if _kwargs["logical"] else "cores" + warnings.warn(msg, category=DeprecationWarning, stacklevel=2) + else: + raise TypeError("cpu_count() got an unexpected keyword argument " + "'%s'" % (list(_kwargs.keys()).pop())) + + if kind == "logical": + # Availability: all + count = _psplatform.cpu_count_logical() + elif kind == "cores": + # Availability: all except OpenBSD and NetBSD + if not hasattr(_psplatform, "cpu_count_cores"): + return None + count = _psplatform.cpu_count_cores() + elif kind == "sockets": + # Availability: Linux, Windows, macOS, FreeBSD + if not hasattr(_psplatform, "cpu_count_sockets"): + return None + count = _psplatform.cpu_count_sockets() + elif kind == "numa": + # Availability: Linux, Windows + if not hasattr(_psplatform, "cpu_count_numa"): + return None + count = _psplatform.cpu_count_numa() + # XXX: this is more complicated than this, see: + # https://github.com/giampaolo/psutil/issues/1122 + # https://bugs.python.org/issue36054 + # elif kind == "usable": + # if hasattr(os, "sched_getaffinity"): + # # Availability: some UNIXes (definitively Linux), Python >= 3.3 + # count = len(os.sched_getaffinity(0)) + # elif hasattr(Process, "cpu_affinity"): + # # Availability: Linux, Windows, FreeBSD + # count = len(Process().cpu_affinity()) + # else: + # # Note that this may not necessarily be correct in case: + # # * the process CPU affinity has been changed + # # * Linux cgroups are in use with CPU affinity configured + # # * Windows systems using processor groups or having more + # # than 64 CPUs + # count = _psplatform.cpu_count_logical() else: - ret = _psplatform.cpu_count_cores() - if ret is not None and ret < 1: - ret = None - return ret + valid = ("logical", "cores", "sockets", "numa") + raise ValueError("invalid kind %r; choose between %s" % (kind, valid)) + if count is not None and count < 1: + count = None + return count def cpu_times(percpu=False): diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index bdcfc1e6..19e636fd 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -248,36 +248,28 @@ def cpu_count_logical(): return cext.cpu_count_logical() -if OPENBSD or NETBSD: - def cpu_count_cores(): - # OpenBSD and NetBSD do not implement this. - return 1 if cpu_count_logical() == 1 else None -else: +if FREEBSD: + def cpu_count_cores(): - """Return the number of CPU cores in the system.""" - # From the C module we'll get an XML string similar to this: - # http://manpages.ubuntu.com/manpages/precise/man4/smp.4freebsd.html - # We may get None in case "sysctl kern.sched.topology_spec" - # is not supported on this BSD version, in which case we'll mimic - # os.cpu_count() and return None. - ret = None - s = cext.cpu_topology() - if s is not None: - # get rid of padding chars appended at the end of the string - index = s.rfind("</groups>") - if index != -1: - s = s[:index + 9] - root = ET.fromstring(s) - try: - ret = len(root.findall('group/children/group/cpu')) or None - finally: - # needed otherwise it will memleak - root.clear() - if not ret: - # If logical CPUs == 1 it's obvious we' have only 1 core. - if cpu_count_logical() == 1: - return 1 - return ret + # https://manpages.ubuntu.com/manpages/precise/man4/smp.4freebsd.html + xmlstr = cext.cpu_topology() + if xmlstr is not None: + root = ET.fromstring(xmlstr) + try: + count = len(root.findall('group/children/group/cpu')) + return count if count != 0 else None + finally: + root.clear() + + def cpu_count_sockets(): + xmlstr = cext.cpu_topology() + if xmlstr is not None: + root = ET.fromstring(xmlstr) + try: + count = len(root.findall('group')) + return count if count != 0 else None + finally: + root.clear() def cpu_stats(): diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 640a0f3d..69b175c5 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -698,6 +698,23 @@ def cpu_count_cores(): return result or None # mimic os.cpu_count() +def cpu_count_sockets(): + """Return the number of physical CPU sockets on the motherboard.""" + found = set() + with open_binary('%s/cpuinfo' % get_procfs_path()) as f: + for line in f: + line = line.strip().lower() + if line.startswith(b'physical id'): + key, value = line.split(b'\t:', 1) + found.add(int(value)) + return len(found) or None + + +def cpu_count_numa(): + """Return the number of CPU NUMA nodes.""" + return len(glob.glob("/sys/devices/system/node/node[0-9]*")) + + def cpu_stats(): """Return various CPU stats as a named tuple.""" with open_binary('%s/stat' % get_procfs_path()) as f: diff --git a/psutil/_psosx.py b/psutil/_psosx.py index d948cc15..9cfb2d15 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -164,6 +164,11 @@ def cpu_count_cores(): return cext.cpu_count_cores() +def cpu_count_sockets(): + """Return the number of physical sockets on the motherboard.""" + return cext.cpu_count_sockets() + + def cpu_stats(): ctx_switches, interrupts, soft_interrupts, syscalls, traps = \ cext.cpu_stats() diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index ff060a51..b82d5e89 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -178,6 +178,21 @@ psutil_setup(void) { } +int +psutil_add_to_dict(PyObject *py_dict, char *keyname, PyObject *py_obj) { + // Add a new python object to an existing dict, DECREFing that object + // and setting it to NULL both in case of success or failure. + if (!py_obj) + return 1; + if (PyDict_SetItemString(py_dict, keyname, py_obj)) { + Py_CLEAR(py_obj); + return 1; + } + Py_CLEAR(py_obj); + return 0; +} + + // ============================================================================ // Utility functions (BSD) // ============================================================================ diff --git a/psutil/_psutil_common.h b/psutil/_psutil_common.h index cb0b399d..636f783c 100644 --- a/psutil/_psutil_common.h +++ b/psutil/_psutil_common.h @@ -103,6 +103,7 @@ PyObject* PyErr_SetFromOSErrnoWithSyscall(const char *syscall); PyObject* psutil_set_testing(PyObject *self, PyObject *args); void psutil_debug(const char* format, ...); int psutil_setup(void); +int psutil_add_to_dict(PyObject *py_dict, char *keyname, PyObject *py_obj); // ==================================================================== // --- BSD diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index 5a77de14..a12679ec 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -1650,6 +1650,8 @@ static PyMethodDef mod_methods[] = { "Return number of logical CPUs on the system"}, {"cpu_count_cores", psutil_cpu_count_cores, METH_VARARGS, "Return number of CPU cores on the system"}, + {"cpu_count_sockets", psutil_cpu_count_sockets, METH_VARARGS, + "Return number of CPU sockets on the motherboard"}, {"virtual_mem", psutil_virtual_mem, METH_VARARGS, "Return system virtual memory stats"}, {"swap_mem", psutil_swap_mem, METH_VARARGS, diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 13cf58c4..1b813f99 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -1582,8 +1582,6 @@ PsutilMethods[] = { "Determine if the process exists in the current process list."}, {"cpu_count_logical", psutil_cpu_count_logical, METH_VARARGS, "Returns the number of logical CPUs on the system"}, - {"cpu_count_cores", psutil_cpu_count_cores, METH_VARARGS, - "Returns the number of CPU cores on the system"}, {"boot_time", psutil_boot_time, METH_VARARGS, "Return the system boot time expressed in seconds since the epoch."}, {"virtual_mem", psutil_virtual_mem, METH_VARARGS, @@ -1639,6 +1637,9 @@ PsutilMethods[] = { // --- windows API bindings {"QueryDosDevice", psutil_QueryDosDevice, METH_VARARGS, "QueryDosDevice binding"}, + {"GetLogicalProcessorInformationEx", + psutil_GetLogicalProcessorInformationEx, METH_VARARGS, + "GetLogicalProcessorInformationEx win API call as a dict"}, // --- others {"set_testing", psutil_set_testing, METH_NOARGS, diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 0ad60c4a..30902297 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -306,7 +306,17 @@ def cpu_count_logical(): def cpu_count_cores(): """Return the number of CPU cores in the system.""" - return cext.cpu_count_cores() + return cext.GetLogicalProcessorInformationEx()['cores'] + + +def cpu_count_sockets(): + """Return the number of CPU sockets on the motherboard.""" + return cext.GetLogicalProcessorInformationEx()['sockets'] + + +def cpu_count_numa(): + """Return the number of CPU NUMA nodes.""" + return cext.GetLogicalProcessorInformationEx()['numa'] def cpu_stats(): diff --git a/psutil/arch/osx/cpu.c b/psutil/arch/osx/cpu.c index 37141a2d..e6f72a8a 100644 --- a/psutil/arch/osx/cpu.c +++ b/psutil/arch/osx/cpu.c @@ -30,7 +30,6 @@ For reference, here's the git history with original implementations: #include "../../_psutil_posix.h" - PyObject * psutil_cpu_count_logical(PyObject *self, PyObject *args) { int num; @@ -56,6 +55,18 @@ psutil_cpu_count_cores(PyObject *self, PyObject *args) { PyObject * +psutil_cpu_count_sockets(PyObject *self, PyObject *args) { + int value; + size_t size = sizeof(value); + + if (sysctlbyname("hw.packages", &value, &size, NULL, 2)) + Py_RETURN_NONE; // mimic os.cpu_count() + else + return Py_BuildValue("i", value); +} + + +PyObject * psutil_cpu_times(PyObject *self, PyObject *args) { mach_msg_type_number_t count = HOST_CPU_LOAD_INFO_COUNT; kern_return_t error; diff --git a/psutil/arch/osx/cpu.h b/psutil/arch/osx/cpu.h index aac0f809..40be1117 100644 --- a/psutil/arch/osx/cpu.h +++ b/psutil/arch/osx/cpu.h @@ -8,6 +8,7 @@ PyObject *psutil_cpu_count_logical(PyObject *self, PyObject *args); PyObject *psutil_cpu_count_cores(PyObject *self, PyObject *args); +PyObject *psutil_cpu_count_sockets(PyObject *self, PyObject *args); PyObject *psutil_cpu_times(PyObject *self, PyObject *args); PyObject *psutil_cpu_freq(PyObject *self, PyObject *args); PyObject *psutil_cpu_stats(PyObject *self, PyObject *args); diff --git a/psutil/arch/windows/cpu.c b/psutil/arch/windows/cpu.c index 355de6df..020a9727 100644 --- a/psutil/arch/windows/cpu.c +++ b/psutil/arch/windows/cpu.c @@ -177,17 +177,25 @@ psutil_cpu_count_logical(PyObject *self, PyObject *args) { /* - * Return the number of CPU cores (non hyper-threading). + * Re-adapted from: + * https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/ + * nf-sysinfoapi-getlogicalprocessorinformation?redirectedfrom=MSDN */ PyObject * -psutil_cpu_count_cores(PyObject *self, PyObject *args) { +psutil_GetLogicalProcessorInformationEx(PyObject *self, PyObject *args) { DWORD rc; PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX buffer = NULL; PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX ptr = NULL; DWORD length = 0; DWORD offset = 0; - DWORD ncpus = 0; - DWORD prev_processor_info_size = 0; + DWORD coresCount = 0; + DWORD socketsCount = 0; + DWORD numaNodesCount = 0; + DWORD prevProcInfoSize = 0; + PyObject *py_retdict = PyDict_New(); + + if (py_retdict == NULL) + return NULL; // GetLogicalProcessorInformationEx() is available from Windows 7 // onward. Differently from GetLogicalProcessorInformation() @@ -230,26 +238,39 @@ psutil_cpu_count_cores(PyObject *self, PyObject *args) { // Advance ptr by the size of the previous // SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX struct. ptr = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX*) \ - (((char*)ptr) + prev_processor_info_size); + (((char*)ptr) + prevProcInfoSize); if (ptr->Relationship == RelationProcessorCore) { - ncpus += 1; + coresCount += 1; + } + else if (ptr->Relationship == RelationNumaNode) { + numaNodesCount += 1; + } + else if (ptr->Relationship == RelationProcessorPackage) { + socketsCount += 1; } // When offset == length, we've reached the last processor // info struct in the buffer. offset += ptr->Size; - prev_processor_info_size = ptr->Size; + prevProcInfoSize = ptr->Size; } free(buffer); - if (ncpus != 0) { - return Py_BuildValue("I", ncpus); + + if (psutil_add_to_dict(py_retdict, "cores", + Py_BuildValue("I", coresCount)) == 1) { + return NULL; } - else { - psutil_debug("GetLogicalProcessorInformationEx() count was 0"); - Py_RETURN_NONE; // mimick os.cpu_count() + if (psutil_add_to_dict(py_retdict, "sockets", + Py_BuildValue("I", socketsCount)) == 1) { + return NULL; + } + if (psutil_add_to_dict(py_retdict, "numa", + Py_BuildValue("I", numaNodesCount)) == 1) { + return NULL; } + return py_retdict; return_none: if (buffer != NULL) diff --git a/psutil/arch/windows/cpu.h b/psutil/arch/windows/cpu.h index 1ef3ff1f..8ebcf80c 100644 --- a/psutil/arch/windows/cpu.h +++ b/psutil/arch/windows/cpu.h @@ -7,7 +7,7 @@ #include <Python.h> PyObject *psutil_cpu_count_logical(PyObject *self, PyObject *args); -PyObject *psutil_cpu_count_cores(PyObject *self, PyObject *args); +PyObject *psutil_GetLogicalProcessorInformationEx(PyObject *self, PyObject *args); PyObject *psutil_cpu_freq(PyObject *self, PyObject *args); PyObject *psutil_cpu_stats(PyObject *self, PyObject *args); PyObject *psutil_cpu_times(PyObject *self, PyObject *args); diff --git a/psutil/tests/test_aix.py b/psutil/tests/test_aix.py index a32c3f6a..b1133833 100755 --- a/psutil/tests/test_aix.py +++ b/psutil/tests/test_aix.py @@ -107,7 +107,7 @@ class AIXSpecificTestCase(PsutilTestCase): def test_cpu_count_logical(self): out = sh('/usr/bin/mpstat -a') mpstat_lcpu = int(re.search(r"lcpu=(\d+)", out).group(1)) - psutil_lcpu = psutil.cpu_count(logical=True) + psutil_lcpu = psutil.cpu_count("logical") self.assertEqual(mpstat_lcpu, psutil_lcpu) def test_net_if_addrs_names(self): diff --git a/psutil/tests/test_bsd.py b/psutil/tests/test_bsd.py index b0bff87f..48b118ab 100755 --- a/psutil/tests/test_bsd.py +++ b/psutil/tests/test_bsd.py @@ -124,7 +124,7 @@ class BSDTestCase(PsutilTestCase): @unittest.skipIf(not which('sysctl'), "sysctl cmd not available") def test_cpu_count_logical(self): syst = sysctl("hw.ncpu") - self.assertEqual(psutil.cpu_count(logical=True), syst) + self.assertEqual(psutil.cpu_count("logical"), syst) @unittest.skipIf(not which('sysctl'), "sysctl cmd not available") def test_virtual_memory_total(self): @@ -457,7 +457,7 @@ class FreeBSDSystemTestCase(PsutilTestCase): # --- sensors_temperatures def test_sensors_temperatures_against_sysctl(self): - num_cpus = psutil.cpu_count(True) + num_cpus = psutil.cpu_count() for cpu in range(num_cpus): sensor = "dev.cpu.%s.temperature" % cpu # sysctl returns a string in the format 46.0C diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index 32c75fd7..3cb2c97f 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -233,6 +233,13 @@ class TestSystemAPITypes(PsutilTestCase): def test_cpu_count(self): self.assertIsInstance(psutil.cpu_count(), int) + self.assertIsNotNone(psutil.cpu_count("logical")) + if not OPENBSD or NETBSD: + self.assertIsNotNone(psutil.cpu_count("cores")) + if LINUX or MACOS or WINDOWS or FREEBSD: + self.assertIsNotNone(psutil.cpu_count("sockets")) + if LINUX or WINDOWS: + self.assertIsNotNone(psutil.cpu_count("numa")) @unittest.skipIf(not HAS_CPU_FREQ, "not supported") def test_cpu_freq(self): diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 0c6d498c..e03490b3 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -58,6 +58,7 @@ if LINUX: SECTOR_SIZE = 512 EMPTY_TEMPERATURES = not glob.glob('/sys/class/hwmon/hwmon*') + # ===================================================================== # --- utils # ===================================================================== @@ -182,6 +183,14 @@ def vmstat(stat): raise ValueError("can't find %r in 'vmstat' output" % stat) +def lscpu(field): + out = sh("lscpu") + for line in out.splitlines(): + key, _, value = line.partition(':') + if field.lower() == key.lower(): + return value.strip() + + def get_free_version_info(): out = sh("free -V").strip() if 'UNKNOWN' in out: @@ -625,32 +634,11 @@ class TestSystemSwapMemory(PsutilTestCase): # ===================================================================== -# --- system CPU +# --- system CPU counts # ===================================================================== @unittest.skipIf(not LINUX, "LINUX only") -class TestSystemCPUTimes(PsutilTestCase): - - def test_fields(self): - fields = psutil.cpu_times()._fields - kernel_ver = re.findall(r'\d+\.\d+\.\d+', os.uname()[2])[0] - kernel_ver_info = tuple(map(int, kernel_ver.split('.'))) - if kernel_ver_info >= (2, 6, 11): - self.assertIn('steal', fields) - else: - self.assertNotIn('steal', fields) - if kernel_ver_info >= (2, 6, 24): - self.assertIn('guest', fields) - else: - self.assertNotIn('guest', fields) - if kernel_ver_info >= (3, 2, 0): - self.assertIn('guest_nice', fields) - else: - self.assertNotIn('guest_nice', fields) - - -@unittest.skipIf(not LINUX, "LINUX only") class TestSystemCPUCountLogical(PsutilTestCase): @unittest.skipIf(not os.path.exists("/sys/devices/system/cpu/online"), @@ -660,25 +648,24 @@ class TestSystemCPUCountLogical(PsutilTestCase): value = f.read().strip() if "-" in str(value): value = int(value.split('-')[1]) + 1 - self.assertEqual(psutil.cpu_count(), value) + self.assertEqual(psutil.cpu_count("logical"), value) @unittest.skipIf(not os.path.exists("/sys/devices/system/cpu"), "/sys/devices/system/cpu does not exist") def test_against_sysdev_cpu_num(self): ls = os.listdir("/sys/devices/system/cpu") count = len([x for x in ls if re.search(r"cpu\d+$", x) is not None]) - self.assertEqual(psutil.cpu_count(), count) + self.assertEqual(psutil.cpu_count("logical"), count) @unittest.skipIf(not which("nproc"), "nproc utility not available") def test_against_nproc(self): num = int(sh("nproc --all")) - self.assertEqual(psutil.cpu_count(logical=True), num) + self.assertEqual(psutil.cpu_count("logical"), num) @unittest.skipIf(not which("lscpu"), "lscpu utility not available") def test_against_lscpu(self): - out = sh("lscpu -p") - num = len([x for x in out.split('\n') if not x.startswith('#')]) - self.assertEqual(psutil.cpu_count(logical=True), num) + num = int(lscpu("cpu(s)")) + self.assertEqual(psutil.cpu_count("logical"), num) def test_emulate_fallbacks(self): import psutil._pslinux @@ -719,13 +706,8 @@ class TestSystemCPUCountCores(PsutilTestCase): @unittest.skipIf(not which("lscpu"), "lscpu utility not available") def test_against_lscpu(self): - out = sh("lscpu -p") - core_ids = set() - for line in out.split('\n'): - if not line.startswith('#'): - fields = line.split(',') - core_ids.add(fields[1]) - self.assertEqual(psutil.cpu_count(logical=False), len(core_ids)) + num = int(lscpu("core(s) per socket")) + self.assertEqual(psutil.cpu_count("cores"), num) def test_method_2(self): meth_1 = psutil._pslinux.cpu_count_cores() @@ -744,6 +726,50 @@ class TestSystemCPUCountCores(PsutilTestCase): @unittest.skipIf(not LINUX, "LINUX only") +class TestSystemCPUCountSockets(PsutilTestCase): + + @unittest.skipIf(not which("lscpu"), "lscpu utility not available") + def test_against_lscpu(self): + num = int(lscpu("socket(s)")) + self.assertEqual(psutil.cpu_count(kind="sockets"), num) + + +@unittest.skipIf(not LINUX, "LINUX only") +class TestSystemCPUCountNuma(PsutilTestCase): + + @unittest.skipIf(not which("lscpu"), "lscpu utility not available") + def test_against_lscpu(self): + num = int(lscpu("numa node(s)")) + self.assertEqual(psutil.cpu_count(kind="numa"), num) + + +# ===================================================================== +# --- system CPU (others) +# ===================================================================== + + +@unittest.skipIf(not LINUX, "LINUX only") +class TestSystemCPUTimes(PsutilTestCase): + + def test_fields(self): + fields = psutil.cpu_times()._fields + kernel_ver = re.findall(r'\d+\.\d+\.\d+', os.uname()[2])[0] + kernel_ver_info = tuple(map(int, kernel_ver.split('.'))) + if kernel_ver_info >= (2, 6, 11): + self.assertIn('steal', fields) + else: + self.assertNotIn('steal', fields) + if kernel_ver_info >= (2, 6, 24): + self.assertIn('guest', fields) + else: + self.assertNotIn('guest', fields) + if kernel_ver_info >= (3, 2, 0): + self.assertIn('guest_nice', fields) + else: + self.assertNotIn('guest_nice', fields) + + +@unittest.skipIf(not LINUX, "LINUX only") class TestSystemCPUFrequency(PsutilTestCase): @unittest.skipIf(not HAS_CPU_FREQ, "not supported") diff --git a/psutil/tests/test_memleaks.py b/psutil/tests/test_memleaks.py index e6940a30..38628478 100755 --- a/psutil/tests/test_memleaks.py +++ b/psutil/tests/test_memleaks.py @@ -342,13 +342,25 @@ class TestModuleFunctionsLeaks(TestMemoryLeak): # --- cpu + def test_cpu_count(self): + # here just to make ns.test_class_coverage happy + pass + @fewtimes_if_linux() - def test_cpu_count(self): # logical - self.execute(lambda: psutil.cpu_count(logical=True)) + def test_cpu_count_logical(self): + self.execute(lambda: psutil.cpu_count("logical")) @fewtimes_if_linux() def test_cpu_count_cores(self): - self.execute(lambda: psutil.cpu_count(logical=False)) + self.execute(lambda: psutil.cpu_count("cores")) + + @fewtimes_if_linux() + def test_cpu_count_sockets(self): + self.execute(lambda: psutil.cpu_count("sockets")) + + @fewtimes_if_linux() + def test_cpu_count_numa(self): + self.execute(lambda: psutil.cpu_count("numa")) @fewtimes_if_linux() def test_cpu_times(self): diff --git a/psutil/tests/test_osx.py b/psutil/tests/test_osx.py index b7a0b088..a3d97c31 100755 --- a/psutil/tests/test_osx.py +++ b/psutil/tests/test_osx.py @@ -22,12 +22,10 @@ from psutil.tests import TOLERANCE_SYS_MEM from psutil.tests import unittest -def sysctl(cmdline): - """Expects a sysctl command with an argument and parse the result - returning only the value of interest. - """ - out = sh(cmdline) - result = out.split()[1] +def sysctl(param): + """Expects a sysctl sub-command and return its stripped result.""" + out = sh("sysctl " + param) + result = out.partition(' ')[2] try: return int(result) except ValueError: @@ -134,21 +132,25 @@ class TestSystemAPIs(PsutilTestCase): # --- cpu def test_cpu_count_logical(self): - num = sysctl("sysctl hw.logicalcpu") - self.assertEqual(num, psutil.cpu_count(logical=True)) + num = sysctl("hw.logicalcpu") + self.assertEqual(num, psutil.cpu_count("logical")) def test_cpu_count_cores(self): - num = sysctl("sysctl hw.physicalcpu") - self.assertEqual(num, psutil.cpu_count(logical=False)) + num = sysctl("hw.physicalcpu") + self.assertEqual(num, psutil.cpu_count("cores")) + + def test_cpu_count_sockets(self): + num = sysctl("hw.packages") + self.assertEqual(num, psutil.cpu_count("sockets")) def test_cpu_freq(self): freq = psutil.cpu_freq() self.assertEqual( - freq.current * 1000 * 1000, sysctl("sysctl hw.cpufrequency")) + freq.current * 1000 * 1000, sysctl("hw.cpufrequency")) self.assertEqual( - freq.min * 1000 * 1000, sysctl("sysctl hw.cpufrequency_min")) + freq.min * 1000 * 1000, sysctl("hw.cpufrequency_min")) self.assertEqual( - freq.max * 1000 * 1000, sysctl("sysctl hw.cpufrequency_max")) + freq.max * 1000 * 1000, sysctl("hw.cpufrequency_max")) # --- virtual mem diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 4e3ac3e4..145d0ffe 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -16,6 +16,7 @@ import signal import socket import sys import time +import warnings import psutil from psutil import AIX @@ -302,23 +303,69 @@ class TestMemoryAPIs(PsutilTestCase): assert mem.sout >= 0, mem -class TestCpuAPIs(PsutilTestCase): - - def test_cpu_count_logical(self): - logical = psutil.cpu_count() - self.assertIsNotNone(logical) - self.assertEqual(logical, len(psutil.cpu_times(percpu=True))) - self.assertGreaterEqual(logical, 1) - # - if os.path.exists("/proc/cpuinfo"): - with open("/proc/cpuinfo") as fd: - cpuinfo_data = fd.read() - if "physical id" not in cpuinfo_data: - raise unittest.SkipTest("cpuinfo doesn't include physical id") - - def test_cpu_count_cores(self): +class TestCpuCount(PsutilTestCase): + + def test_base(self): + kinds = ("logical", "cores", "sockets", "numa") + for kind in kinds: + n = psutil.cpu_count(kind=kind) + if n is not None: + self.assertIsInstance(n, int) + with self.subTest(kind): + self.assertGreaterEqual(n, 1) + + self.assertEqual(psutil.cpu_count(), psutil.cpu_count("logical")) + + with self.assertRaises(ValueError) as cm: + psutil.cpu_count(kind='xxx') + self.assertIn(str(kinds), str(cm.exception)) + + def test_consistency(self): + logical = psutil.cpu_count("logical") + cores = psutil.cpu_count("cores") + sockets = psutil.cpu_count("sockets") + numa = psutil.cpu_count(kind='numa') + + # logical (always supposed to be the highest) + if logical is not None: + if cores is not None: + self.assertGreaterEqual(logical, cores) + if sockets is not None: + self.assertGreaterEqual(logical, sockets) + + # cores (at least >= sockets) + if cores is not None: + if sockets is not None: + self.assertGreaterEqual(cores, sockets) + + if numa is not None: + self.assertGreaterEqual(numa, 0) + + def test_deprecation(self): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + self.assertEqual(psutil.cpu_count(logical=True), + psutil.cpu_count(kind="logical")) + self.assertEqual(psutil.cpu_count(logical=False), + psutil.cpu_count(kind="cores")) + self.assertEqual(psutil.cpu_count(True), + psutil.cpu_count("logical")) + self.assertEqual(psutil.cpu_count(False), + psutil.cpu_count("cores")) + + with warnings.catch_warnings(record=True) as ws: + warnings.simplefilter("always") + psutil.cpu_count(logical=True) + assert ws + + with warnings.catch_warnings(record=True) as ws: + warnings.simplefilter("always") + psutil.cpu_count(True) + assert ws + + def test_cores(self): logical = psutil.cpu_count() - cores = psutil.cpu_count(logical=False) + cores = psutil.cpu_count("cores") if cores is None: raise self.skipTest("cpu_count_cores() is None") if WINDOWS and sys.getwindowsversion()[:2] <= (6, 1): # <= Vista @@ -327,19 +374,18 @@ class TestCpuAPIs(PsutilTestCase): self.assertGreaterEqual(cores, 1) self.assertGreaterEqual(logical, cores) - def test_cpu_count_none(self): + def test_return_none(self): # https://github.com/giampaolo/psutil/issues/1085 for val in (-1, 0, None): with mock.patch('psutil._psplatform.cpu_count_logical', return_value=val) as m: self.assertIsNone(psutil.cpu_count()) assert m.called - with mock.patch('psutil._psplatform.cpu_count_cores', - return_value=val) as m: - self.assertIsNone(psutil.cpu_count(logical=False)) - assert m.called - def test_cpu_times(self): + +class TestCpuTimes(PsutilTestCase): + + def test_base(self): # Check type, value >= 0, str(). total = 0 times = psutil.cpu_times() @@ -368,7 +414,7 @@ class TestCpuAPIs(PsutilTestCase): # msg="%s %s" % (new_t, last_t)) # last = new - def test_cpu_times_time_increases(self): + def test_time_increases(self): # Make sure time increases between calls. t1 = sum(psutil.cpu_times()) stop_at = time.time() + GLOBAL_TIMEOUT @@ -436,6 +482,9 @@ class TestCpuAPIs(PsutilTestCase): self.assertAlmostEqual( getattr(base, field), getattr(summed_values, field), delta=1) + +class TestCpuPercent(PsutilTestCase): + def _test_cpu_percent(self, percent, last_ret, new_ret): try: self.assertIsInstance(percent, float) @@ -498,6 +547,9 @@ class TestCpuAPIs(PsutilTestCase): for percent in cpu: self._test_cpu_percent(percent, None, None) + +class TestOtherCPUFunctions(PsutilTestCase): + def test_cpu_stats(self): # Tested more extensively in per-platform test modules. infos = psutil.cpu_stats() diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index aeb282c8..815771bf 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -82,28 +82,29 @@ class TestCpuAPIs(WindowsTestCase): @unittest.skipIf('NUMBER_OF_PROCESSORS' not in os.environ, 'NUMBER_OF_PROCESSORS env var is not available') - def test_cpu_count_vs_NUMBER_OF_PROCESSORS(self): + def test_cpu_count_logical_vs_NUMBER_OF_PROCESSORS(self): # Will likely fail on many-cores systems: # https://stackoverflow.com/questions/31209256 num_cpus = int(os.environ['NUMBER_OF_PROCESSORS']) - self.assertEqual(num_cpus, psutil.cpu_count()) + self.assertEqual(num_cpus, psutil.cpu_count("logical")) - def test_cpu_count_vs_GetSystemInfo(self): + def test_cpu_count_logical_vs_GetSystemInfo(self): # Will likely fail on many-cores systems: # https://stackoverflow.com/questions/31209256 sys_value = win32api.GetSystemInfo()[5] - psutil_value = psutil.cpu_count() + psutil_value = psutil.cpu_count("logical") self.assertEqual(sys_value, psutil_value) def test_cpu_count_logical_vs_wmi(self): w = wmi.WMI() proc = w.Win32_Processor()[0] - self.assertEqual(psutil.cpu_count(), proc.NumberOfLogicalProcessors) + self.assertEqual(psutil.cpu_count("logical"), + proc.NumberOfLogicalProcessors) def test_cpu_count_cores_vs_wmi(self): w = wmi.WMI() proc = w.Win32_Processor()[0] - self.assertEqual(psutil.cpu_count(logical=False), proc.NumberOfCores) + self.assertEqual(psutil.cpu_count("cores"), proc.NumberOfCores) def test_cpu_count_vs_cpu_times(self): self.assertEqual(psutil.cpu_count(), @@ -725,7 +726,7 @@ class RemoteProcessTestCase(PsutilTestCase): p = psutil.Process(self.proc32.pid) e = p.environ() self.assertIn("THINK_OF_A_NUMBER", e) - self.assertEquals(e["THINK_OF_A_NUMBER"], str(os.getpid())) + self.assertEqual(e["THINK_OF_A_NUMBER"], str(os.getpid())) def test_environ_64(self): p = psutil.Process(self.proc64.pid) diff --git a/scripts/internal/print_api_speed.py b/scripts/internal/print_api_speed.py index e39a1baa..f312f86b 100755 --- a/scripts/internal/print_api_speed.py +++ b/scripts/internal/print_api_speed.py @@ -81,7 +81,13 @@ def main(): elif name == 'disk_usage': args = (os.getcwd(), ) timecall(name, fun, *args) - timecall('cpu_count (cores)', psutil.cpu_count, logical=False) + ("logical", "cores", "sockets", "numa", "usable") + + timecall('cpu_count (logical)', psutil.cpu_count, "logical") + timecall('cpu_count (cores)', psutil.cpu_count, "cores") + timecall('cpu_count (sockets)', psutil.cpu_count, "sockets") + timecall('cpu_count (numa)', psutil.cpu_count, "numa") + timecall('cpu_count (usable)', psutil.cpu_count, "usable") timecall('process_iter (all)', lambda: list(psutil.process_iter())) print_timings() |