diff options
| author | Giampaolo Rodola <g.rodola@gmail.com> | 2019-10-17 08:44:09 -0700 |
|---|---|---|
| committer | Giampaolo Rodola <g.rodola@gmail.com> | 2019-10-17 08:44:09 -0700 |
| commit | 75fe258fc515907e0095dbaa51b49c9914385c3a (patch) | |
| tree | 773e988e18209bd43aa8460d1549ebeadd818e14 | |
| parent | 6dc8c8d6ee3da0b4979520b7c1a3dc4a269d44f0 (diff) | |
| parent | c20d734ed387476cd79b711684b5a295baf8b8b0 (diff) | |
| download | psutil-75fe258fc515907e0095dbaa51b49c9914385c3a.tar.gz | |
merge from master
44 files changed, 1117 insertions, 784 deletions
diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..c39b2b61 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,9 @@ +# These are supported funding model platforms + +tidelift: "pypi/psutil" +github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +patreon: # Replace with a single Patreon username +open_collective: # Replace with a single Open Collective username +ko_fi: # Replace with a single Ko-fi username +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +custom: # Replace with a single custom sponsorship URL diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md index 283d8dc6..ba4a026c 100644 --- a/.github/ISSUE_TEMPLATE/bug.md +++ b/.github/ISSUE_TEMPLATE/bug.md @@ -3,7 +3,6 @@ name: Bug about: Report a bug title: "[OS] title" labels: 'bug' -assignees: 'giampaolo' --- **Platform** diff --git a/.github/ISSUE_TEMPLATE/enhancement.md b/.github/ISSUE_TEMPLATE/enhancement.md index 1a5e14e5..7e7159f2 100644 --- a/.github/ISSUE_TEMPLATE/enhancement.md +++ b/.github/ISSUE_TEMPLATE/enhancement.md @@ -2,7 +2,6 @@ name: Enhancement about: Propose an enhancement labels: 'enhancement' -assignees: 'giampaolo' title: "[OS] title" --- diff --git a/.travis.yml b/.travis.yml index fdab64f6..b3b0102d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,3 @@ -sudo: false language: python cache: pip matrix: @@ -10,7 +9,6 @@ matrix: - python: 3.6 - python: 3.7 dist: xenial - sudo: true # macOS - language: generic os: osx @@ -57,7 +57,7 @@ N: Arnon Yaari (wiggin15) W: https://github.com/wiggin15 D: AIX implementation, expert on multiple fronts I: 517, 607, 610, 1131, 1123, 1130, 1154, 1164, 1174, 1177, 1210, 1214, 1408, - 1329, 1276, 1494. + 1329, 1276, 1494, 1528. N: Alex Manuskin W: https://github.com/amanusk @@ -617,3 +617,16 @@ N: agnewee W: https://github.com/Agnewee C: China I: 1491 + +N: Kamil Rytarowski +W: https://github.com/krytarowski +C: Poland +I: 1526, 1530 + +N: Athos Ribeiro +W: https://github.com/athos-ribeiro +I: 1585 + +N: Erwan Le Pape +W: https://github.com/erwan-le-pape +I: 1570 diff --git a/HISTORY.rst b/HISTORY.rst index 6b517766..1e8cd0ba 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,12 +1,39 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* -5.6.3 +5.6.4 ===== XXXX-XX-XX **Enhancements** +- 1527_: [Linux] added Process.cpu_times().iowait counter, which is the time + spent waiting for blocking I/O to complete. + +**Bug fixes** + +- 1126_: [Linux] cpu_affinity() segfaults on CentOS 5 / manylinux. + cpu_affinity() support for CentOS 5 was removed. +- 1528_: [AIX] compilation error on AIX 7.2 due to 32 vs 64 bit differences. + (patch by Arnon Yaari) +- 1535_: 'type' and 'family' fields returned by net_connections() are not + always turned into enums. +- 1536_: [NetBSD] process cmdline() erroneously raise ZombieProcess error if + cmdline has non encodable chars. +- 1546_: usage percent may be rounded to 0 on Python 2. +- 1552_: [Windows] getloadavg() math for calculating 5 and 15 mins values is + incorrect. +- 1570_: [Windows] NtWow64* syscalls fail to raise the proper error code +- 1585_: [OSX] calling close() (in C) on possible negative integers. (patch + by Athos Ribeiro) + +5.6.3 +===== + +2019-06-11 + +**Enhancements** + - 1494_: [AIX] added support for Process.environ(). (patch by Arnon Yaari) **Bug fixes** @@ -15,6 +42,8 @@ XXXX-XX-XX - 1501_: [Windows] Process cmdline() and exe() raise unhandled "WinError 1168 element not found" exceptions for "Registry" and "Memory Compression" psuedo processes on Windows 10. +- 1526_: [NetBSD] process cmdline() could raise MemoryError. (patch by + Kamil Rytarowski) 5.6.2 ===== @@ -1,4 +1,4 @@ -psutil is distributed under BSD license reproduced below. +BSD 3-Clause License Copyright (c) 2009, Jay Loden, Dave Daeschler, Giampaolo Rodola' All rights reserved. @@ -8,9 +8,11 @@ are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * Neither the name of the psutil authors nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -14,7 +14,8 @@ DEPS = \ futures \ ipaddress \ mock==1.0.1 \ - perf \ + pyperf \ + readline \ requests \ setuptools \ sphinx \ @@ -54,8 +55,7 @@ clean: ## Remove all build files. build/ \ dist/ \ docs/_build/ \ - htmlcov/ \ - tmp/ + htmlcov/ _: @@ -67,13 +67,11 @@ build: _ ## Compile without installing. @# "import psutil" when using the interactive interpreter from within @# this directory. PYTHONWARNINGS=all $(PYTHON) setup.py build_ext -i - rm -rf tmp $(PYTHON) -c "import psutil" # make sure it actually worked install: ## Install this package as current user in "edit" mode. ${MAKE} build PYTHONWARNINGS=all $(PYTHON) setup.py develop $(INSTALL_OPTS) - rm -rf tmp uninstall: ## Uninstall this package via pip. cd ..; $(PYTHON) -m pip uninstall -y -v psutil || true @@ -101,8 +99,8 @@ install-pip: ## Install pip (no-op if already installed). setup-dev-env: ## Install GIT hooks, pip, test deps (also upgrades them). ${MAKE} install-git-hooks ${MAKE} install-pip - $(PYTHON) -m pip install $(INSTALL_OPTS) --upgrade pip - $(PYTHON) -m pip install $(INSTALL_OPTS) --upgrade $(DEPS) + $(PYTHON) -m pip install $(INSTALL_OPTS) --upgrade --trusted-host files.pythonhosted.org pip + $(PYTHON) -m pip install $(INSTALL_OPTS) --upgrade --trusted-host files.pythonhosted.org $(DEPS) # =================================================================== # Tests @@ -173,6 +171,9 @@ test-coverage: ## Run test coverage. flake8: ## flake8 linter. @git ls-files | grep \\.py$ | xargs $(PYTHON) -m flake8 +fix-flake8: ## Attempt to automaticall fix some flake8 issues. + @git ls-files | grep \\.py$ | xargs $(PYTHON) -m flake8 --exit-zero | $(PYTHON) scripts/internal/fix_flake8.py + # =================================================================== # GIT # =================================================================== @@ -1,47 +1,67 @@ -.. image:: https://pepy.tech/badge/psutil/month +| |downloads| |stars| |forks| |contributors| |coverage| |quality| +| |version| |py-versions| |packages| |license| +| |travis| |appveyor| |doc| |twitter| |tidelift| + +.. |downloads| image:: https://img.shields.io/pypi/dm/psutil.svg :target: https://pepy.tech/project/psutil :alt: Downloads -.. image:: https://img.shields.io/github/stars/giampaolo/psutil.svg +.. |stars| image:: https://img.shields.io/github/stars/giampaolo/psutil.svg :target: https://github.com/giampaolo/psutil/stargazers :alt: Github stars -.. image:: https://img.shields.io/github/forks/giampaolo/psutil.svg +.. |forks| image:: https://img.shields.io/github/forks/giampaolo/psutil.svg :target: https://github.com/giampaolo/psutil/network/members :alt: Github forks -.. image:: https://img.shields.io/github/contributors/giampaolo/psutil.svg +.. |contributors| image:: https://img.shields.io/github/contributors/giampaolo/psutil.svg :target: https://github.com/giampaolo/psutil/graphs/contributors :alt: Contributors -.. image:: https://img.shields.io/travis/giampaolo/psutil/master.svg?maxAge=3600&label=Linux%20/%20macOS +.. |quality| image:: https://img.shields.io/codacy/grade/ce63e7f7f69d44b5b59682196e6fbfca.svg + :target: https://www.codacy.com/app/g-rodola/psutil?utm_source=github.com&utm_medium=referral&utm_content=giampaolo/psutil&utm_campaign=Badge_Grade + :alt: Code quality + +.. |travis| image:: https://img.shields.io/travis/giampaolo/psutil/master.svg?maxAge=3600&label=linux%20/%20osx :target: https://travis-ci.org/giampaolo/psutil :alt: Linux tests (Travis) -.. image:: https://img.shields.io/appveyor/ci/giampaolo/psutil/master.svg?maxAge=3600&label=Windows +.. |appveyor| image:: https://img.shields.io/appveyor/ci/giampaolo/psutil/master.svg?maxAge=3600&label=windows :target: https://ci.appveyor.com/project/giampaolo/psutil :alt: Windows tests (Appveyor) -.. image:: https://coveralls.io/repos/github/giampaolo/psutil/badge.svg?branch=master +.. |coverage| image:: https://img.shields.io/coveralls/github/giampaolo/psutil.svg?label=test%20coverage :target: https://coveralls.io/github/giampaolo/psutil?branch=master :alt: Test coverage (coverall.io) -.. image:: https://readthedocs.org/projects/psutil/badge/?version=latest +.. |doc| image:: https://readthedocs.org/projects/psutil/badge/?version=latest :target: http://psutil.readthedocs.io/en/latest/?badge=latest :alt: Documentation Status -.. image:: https://img.shields.io/pypi/v/psutil.svg?label=pypi +.. |version| image:: https://img.shields.io/pypi/v/psutil.svg?label=pypi :target: https://pypi.org/project/psutil :alt: Latest version -.. image:: https://img.shields.io/pypi/pyversions/psutil.svg +.. |py-versions| image:: https://img.shields.io/pypi/pyversions/psutil.svg :target: https://pypi.org/project/psutil :alt: Supported Python versions -.. image:: https://img.shields.io/pypi/l/psutil.svg - :target: https://pypi.org/project/psutil +.. |packages| image:: https://repology.org/badge/tiny-repos/python:psutil.svg + :target: https://repology.org/metapackage/python:psutil/versions + :alt: Binary packages + +.. |license| image:: https://img.shields.io/pypi/l/psutil.svg + :target: https://github.com/giampaolo/psutil/blob/master/LICENSE :alt: License +.. |twitter| image:: https://img.shields.io/twitter/follow/grodola.svg?label=follow&style=flat&logo=twitter&logoColor=4FADFF + :target: https://twitter.com/grodola + :alt: Twitter Follow + +.. |tidelift| image:: https://tidelift.com/badges/github/giampaolo/psutil?style=flat + :target: https://tidelift.com/subscription/pkg/pypi-psutil?utm_source=pypi-psutil&utm_medium=referral&utm_campaign=readme + :alt: Tidelift + ----- Quick links @@ -57,7 +77,6 @@ Quick links - `Development guide <https://github.com/giampaolo/psutil/blob/master/docs/DEVGUIDE.rst>`_ - `What's new <https://github.com/giampaolo/psutil/blob/master/HISTORY.rst>`_ - Summary ======= @@ -66,9 +85,8 @@ retrieving information on **running processes** and **system utilization** (CPU, memory, disks, network, sensors) in Python. It is useful mainly for **system monitoring**, **profiling and limiting process resources** and **management of running processes**. -It implements many functionalities offered by UNIX command line tools such as: -ps, top, lsof, netstat, ifconfig, who, df, kill, free, nice, ionice, iostat, -iotop, uptime, pidof, tty, taskset, pmap. +It implements many functionalities offered by classic UNIX command line tools +such as *ps, top, iotop, lsof, netstat, ifconfig, free* and others. psutil currently supports the following platforms: - **Linux** @@ -80,24 +98,28 @@ psutil currently supports the following platforms: ...both **32-bit** and **64-bit** architectures. Supported Python versions are **2.6**, **2.7** and **3.4+**. `PyPy <http://pypy.org/>`__ is also known to work. +Professional support +==================== -Author -====== - -psutil was created and is maintained by -`Giampaolo Rodola <http://grodola.blogspot.com/p/about.html>`__ and it -received many useful `contributions <https://github.com/giampaolo/psutil/blob/master/CREDITS>`__ -over the years. -A lot of time and effort went into making psutil as it is right now. -If you feel psutil is useful to you or your business and want to support its -future development consider making a small donation: +.. |tideliftlogo| image:: https://nedbatchelder.com/pix/Tidelift_Logos_RGB_Tidelift_Shorthand_On-White_small.png + :width: 100 + :alt: Tidelift + :target: https://tidelift.com/subscription/pkg/pypi-psutil?utm_source=pypi-psutil&utm_medium=referral&utm_campaign=readme -.. image:: http://www.paypal.com/en_US/i/btn/x-click-but04.gif - :target: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=A9ZS7PKKRM3S8 - :alt: Donate via PayPal +.. list-table:: + :widths: 10 100 -Don't want to donate money? Then maybe you could `write me a recommendation on Linkedin <https://www.linkedin.com/in/grodola>`_. + * - |tideliftlogo| + - Professional support for psutil is available as part of the + `Tidelift Subscription`_. + Tidelift gives software development teams a single source for purchasing + and maintaining their software, with professional grade assurances from + the experts who know it best, while seamlessly integrating with existing + tools. + By subscribing you will help me (`Giampaolo Rodola`_) support psutil + future development. Alternatively consider making a small `donation`_. +.. _Tidelift Subscription: https://tidelift.com/subscription/pkg/pypi-psutil?utm_source=pypi-psutil&utm_medium=referral&utm_campaign=readme Example applications ==================== @@ -118,7 +140,7 @@ Projects using psutil psutil has roughly the following monthly downloads: -.. image:: https://pepy.tech/badge/psutil/month +.. image:: https://img.shields.io/pypi/dm/psutil.svg :target: https://pepy.tech/project/psutil :alt: Downloads @@ -337,7 +359,7 @@ Process management pgids(real=1000, effective=1000, saved=1000) >>> >>> p.cpu_times() - pcputimes(user=1.02, system=0.31, children_user=0.32, children_system=0.1) + pcputimes(user=1.02, system=0.31, children_user=0.32, children_system=0.1, iowait=0.0) >>> p.cpu_percent(interval=1.0) 12.1 >>> p.cpu_affinity() @@ -363,8 +385,8 @@ Process management pio(read_count=478001, write_count=59371, read_bytes=700416, write_bytes=69632, read_chars=456232, write_chars=517543) >>> >>> p.open_files() - [popenfile(path='/home/giampaolo/svn/psutil/setup.py', fd=3, position=0, mode='r', flags=32768), - popenfile(path='/var/log/monitd', fd=4, position=235542, mode='a', flags=33793)] + [popenfile(path='/home/giampaolo/monit.py', fd=3, position=0, mode='r', flags=32768), + popenfile(path='/var/log/monit.log', fd=4, position=235542, mode='a', flags=33793)] >>> >>> p.connections() [pconn(fd=115, family=<AddressFamily.AF_INET: 2>, type=<SocketType.SOCK_STREAM: 1>, laddr=addr(ip='10.0.0.1', port=48776), raddr=addr(ip='93.186.135.91', port=80), status='ESTABLISHED'), @@ -406,6 +428,7 @@ Process management >>> p.resume() >>> >>> p.terminate() + >>> p.kill() >>> p.wait(timeout=3) 0 >>> @@ -481,3 +504,7 @@ Windows services 'start_type': 'manual', 'status': 'stopped', 'username': 'NT AUTHORITY\\LocalService'} + + +.. _`Giampaolo Rodola`: http://grodola.blogspot.com/p/about.html +.. _`donation`: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=A9ZS7PKKRM3S8 diff --git a/docs/index.rst b/docs/index.rst index ede2b3f9..e5e40737 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,7 +13,7 @@ Quick links - `Blog <http://grodola.blogspot.com/search/label/psutil>`__ - `Forum <http://groups.google.com/group/psutil/topics>`__ - `Download <https://pypi.org/project/psutil/#files>`__ -- `Development guide <https://github.com/giampaolo/psutil/blob/master/DEVGUIDE.rst>`_ +- `Development guide <https://github.com/giampaolo/psutil/blob/master/docs/DEVGUIDE.rst>`_ - `What's new <https://github.com/giampaolo/psutil/blob/master/HISTORY.rst>`__ About @@ -78,7 +78,8 @@ CPU - **nice** *(UNIX)*: time spent by niced (prioritized) processes executing in user mode; on Linux this also includes **guest_nice** time - - **iowait** *(Linux)*: time spent waiting for I/O to complete + - **iowait** *(Linux)*: time spent waiting for I/O to complete. This is *not* + accounted in **idle** time counter. - **irq** *(Linux, BSD)*: time spent for servicing hardware interrupts - **softirq** *(Linux)*: time spent for servicing software interrupts - **steal** *(Linux 2.6.11+)*: time spent by other operating systems running @@ -241,19 +242,27 @@ CPU .. function:: getloadavg() Return the average system load over the last 1, 5 and 15 minutes as a tuple. - The load represents how many processes are waiting to be run by the - operating system. - On UNIX systems this relies on `os.getloadavg`_. On Windows this is - emulated by using a Windows API that spawns a thread which updates the - average every 5 seconds, mimicking the UNIX behavior. Thus, the first time - this is called and for the next 5 seconds it will return a meaningless - ``(0.0, 0.0, 0.0)`` tuple. Example: + The load represents the processes which are in a runnable state, either + using the CPU or waiting to use the CPU (e.g. waiting for disk I/O). + On UNIX systems this relies on `os.getloadavg`_. On Windows this is emulated + by using a Windows API that spawns a thread which keeps running in + background and updates the load average every 5 seconds, mimicking the UNIX + behavior. Thus, the first time this is called and for the next 5 seconds + it will return a meaningless ``(0.0, 0.0, 0.0)`` tuple. + The numbers returned only make sense if related to the number of CPU cores + installed on the system. So, for instance, `3.14` on a system with 10 CPU + cores means that the system load was 31.4% percent over the last N minutes. .. code-block:: python >>> import psutil >>> psutil.getloadavg() (3.14, 3.89, 4.67) + >>> psutil.cpu_count() + 10 + >>> # percentage representation + >>> [x / psutil.cpu_count() * 100 for x in psutil.getloadavg()] + [31.4, 38.9, 46.7] Availability: Unix, Windows @@ -513,7 +522,8 @@ Network to obtain a usable socket object. On Windows and SunOS this is always set to ``-1``. - **family**: the address family, either `AF_INET`_, `AF_INET6`_ or `AF_UNIX`_. - - **type**: the address type, either `SOCK_STREAM`_ or `SOCK_DGRAM`_. + - **type**: the address type, either `SOCK_STREAM`_, `SOCK_DGRAM`_ or + `SOCK_SEQPACKET`_. - **laddr**: the local address as a ``(ip, port)`` named tuple or a ``path`` in case of AF_UNIX sockets. For UNIX sockets see notes below. - **raddr**: the remote address as a ``(ip, port)`` named tuple or an @@ -1169,6 +1179,10 @@ Process class >>> p = psutil.Process() >>> p.as_dict(attrs=['pid', 'name', 'username']) {'username': 'giampaolo', 'pid': 12366, 'name': 'python'} + >>> + >>> # get a list of valid attrs names + >>> list(psutil.Process().as_dict().keys()) + ['status', 'cpu_num', 'num_ctx_switches', 'pid', 'memory_full_info', 'connections', 'cmdline', 'create_time', 'ionice', 'num_fds', 'memory_maps', 'cpu_percent', 'terminal', 'ppid', 'cwd', 'nice', 'username', 'cpu_times', 'io_counters', 'memory_info', 'threads', 'open_files', 'name', 'num_threads', 'exe', 'uids', 'gids', 'cpu_affinity', 'memory_percent', 'environ'] .. versionchanged:: 3.0.0 *ad_value* is used also when incurring into @@ -1201,6 +1215,8 @@ Process class The process current working directory as an absolute path. + .. versionchanged:: 5.6.4 added support for NetBSD + .. method:: username() The name of the user that owns the process. On UNIX this is calculated by @@ -1324,7 +1340,7 @@ Process class Return process I/O statistics as a named tuple. For Linux you can refer to - `/proc filesysem documentation <https://stackoverflow.com/questions/3633286/>`__. + `/proc filesystem documentation <https://stackoverflow.com/questions/3633286/>`__. - **read_count**: the number of read operations performed (cumulative). This is supposed to count the number of read-related syscalls such as @@ -1397,16 +1413,33 @@ Process class .. method:: cpu_times() - Return a `(user, system, children_user, children_system)` named tuple - representing the accumulated process time, in seconds (see - `explanation <http://stackoverflow.com/questions/556405/>`__). - On Windows and macOS only *user* and *system* are filled, the others are - set to ``0``. + Return a named tuple representing the accumulated process times, in seconds + (see `explanation <http://stackoverflow.com/questions/556405/>`__). This is similar to `os.times`_ but can be used for any process PID. + - **user**: time spent in user mode. + - **system**: time spent in kernel mode. + - **children_user**: user time of all child processes (always ``0`` on + Windows and macOS). + - **system_user**: user time of all child processes (always ``0`` on + Windows and macOS). + - **iowait**: (Linux) time spent waiting for blocking I/O to complete. + This value is excluded from `user` and `system` times count (because the + CPU is not doing any work). + + >>> import psutil + >>> p = psutil.Process() + >>> p.cpu_times() + pcputimes(user=0.03, system=0.67, children_user=0.0, children_system=0.0, iowait=0.08) + >>> sum(p.cpu_times()[:2]) # cumulative, excluding children and iowait + 0.70 + .. versionchanged:: 4.1.0 return two extra fields: *children_user* and *children_system*. + .. versionchanged:: + 5.6.4 added *iowait* on Linux. + .. method:: cpu_percent(interval=None) Return a float representing the process CPU utilization as a percentage @@ -1778,7 +1811,8 @@ Process class always set to ``-1``. - **family**: the address family, either `AF_INET`_, `AF_INET6`_ or `AF_UNIX`_. - - **type**: the address type, either `SOCK_STREAM`_ or `SOCK_DGRAM`_. + - **type**: the address type, either `SOCK_STREAM`_, `SOCK_DGRAM`_ or + `SOCK_SEQPACKET`_. . - **laddr**: the local address as a ``(ip, port)`` named tuple or a ``path`` in case of AF_UNIX sockets. For UNIX sockets see notes below. - **raddr**: the remote address as a ``(ip, port)`` named tuple or an @@ -2185,7 +2219,7 @@ Process priority constants .. data:: IOPRIO_NORMAL .. data:: IOPRIO_HIGH - A set of integers representing the I/O priority of a process on Linux. + A set of integers representing the I/O priority of a process on Windows. They can be used in conjunction with :meth:`psutil.Process.ionice()` to get or set process I/O priority. @@ -2572,7 +2606,7 @@ FAQs especially on macOS (see `issue #883`_) and Windows. Unfortunately there's not much you can do about this except running the Python process with higher privileges. - On Unix you may run the the Python process as root or use the SUID bit + On Unix you may run the Python process as root or use the SUID bit (this is the trick used by tools such as ``ps`` and ``netstat``). On Windows you may run the Python process as NT AUTHORITY\\SYSTEM or install the Python script as a Windows service (this is the trick used by tools @@ -2599,7 +2633,11 @@ take a look at the `development guide`_. Timeline ======== -- 2019-0426: +- 2019-06-11: + `5.6.3 <https://pypi.org/project/psutil/5.6.3/#files>`__ - + `what's new <https://github.com/giampaolo/psutil/blob/master/HISTORY.rst#563>`__ - + `diff <https://github.com/giampaolo/psutil/compare/release-5.6.2...release-5.6.3#files_bucket>`__ +- 2019-04-26: `5.6.2 <https://pypi.org/project/psutil/5.6.2/#files>`__ - `what's new <https://github.com/giampaolo/psutil/blob/master/HISTORY.rst#562>`__ - `diff <https://github.com/giampaolo/psutil/compare/release-5.6.1...release-5.6.2#files_bucket>`__ @@ -2898,7 +2936,7 @@ Timeline .. _`BPO-6973`: https://bugs.python.org/issue6973 .. _`CPU affinity`: https://www.linuxjournal.com/article/6799?page=0,0 .. _`cpu_distribution.py`: https://github.com/giampaolo/psutil/blob/master/scripts/cpu_distribution.py -.. _`development guide`: https://github.com/giampaolo/psutil/blob/master/DEVGUIDE.rst +.. _`development guide`: https://github.com/giampaolo/psutil/blob/master/docs/DEVGUIDE.rst .. _`disk_usage.py`: https://github.com/giampaolo/psutil/blob/master/scripts/disk_usage.py .. _`enums`: https://docs.python.org/3/library/enum.html#module-enum .. _`fans.py`: https://github.com/giampaolo/psutil/blob/master/scripts/fans.py @@ -2915,7 +2953,7 @@ Timeline .. _`issue #883`: https://github.com/giampaolo/psutil/issues/883 .. _`man prlimit`: https://linux.die.net/man/2/prlimit .. _`meminfo.py`: https://github.com/giampaolo/psutil/blob/master/scripts/meminfo.py -.. _`netstat.py`: https://github.com/giampaolo/psutil/blob/master/scripts/netstat.py. +.. _`netstat.py`: https://github.com/giampaolo/psutil/blob/master/scripts/netstat.py .. _`nettop.py`: https://github.com/giampaolo/psutil/blob/master/scripts/nettop.py .. _`open`: https://docs.python.org/3/library/functions.html#open .. _`os.cpu_count`: https://docs.python.org/3/library/os.html#os.cpu_count @@ -2940,6 +2978,7 @@ Timeline .. _`shutil.disk_usage`: https://docs.python.org/3/library/shutil.html#shutil.disk_usage. .. _`signal module`: https://docs.python.org//library/signal.html .. _`SOCK_DGRAM`: https://docs.python.org/3/library/socket.html#socket.SOCK_DGRAM +.. _`SOCK_SEQPACKET`: https://docs.python.org/3/library/socket.html#socket.SOCK_SEQPACKET .. _`SOCK_STREAM`: https://docs.python.org/3/library/socket.html#socket.SOCK_STREAM .. _`socket.fromfd`: https://docs.python.org/3/library/socket.html#socket.fromfd .. _`subprocess.Popen`: https://docs.python.org/3/library/subprocess.html#subprocess.Popen diff --git a/psutil/__init__.py b/psutil/__init__.py index 0e4a8de9..62dadc90 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -25,7 +25,6 @@ from __future__ import division import collections import contextlib import datetime -import errno import functools import os import signal @@ -44,6 +43,8 @@ from ._common import memoize from ._common import memoize_when_activated from ._common import wrap_numbers as _wrap_numbers from ._compat import long +from ._compat import PermissionError +from ._compat import ProcessLookupError from ._compat import PY3 as _PY3 from ._common import STATUS_DEAD @@ -221,7 +222,7 @@ __all__ = [ __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "5.6.3" +__version__ = "5.6.4" version_info = tuple([int(num) for num in __version__.split('.')]) _timer = getattr(time, 'monotonic', time.time) @@ -1290,18 +1291,16 @@ class Process(object): "calling process (os.getpid()) instead of PID 0") try: os.kill(self.pid, sig) - except OSError as err: - if err.errno == errno.ESRCH: - if OPENBSD and pid_exists(self.pid): - # We do this because os.kill() lies in case of - # zombie processes. - raise ZombieProcess(self.pid, self._name, self._ppid) - else: - self._gone = True - raise NoSuchProcess(self.pid, self._name) - if err.errno in (errno.EPERM, errno.EACCES): - raise AccessDenied(self.pid, self._name) - raise + except ProcessLookupError: + if OPENBSD and pid_exists(self.pid): + # We do this because os.kill() lies in case of + # zombie processes. + raise ZombieProcess(self.pid, self._name, self._ppid) + else: + self._gone = True + raise NoSuchProcess(self.pid, self._name) + except PermissionError: + raise AccessDenied(self.pid, self._name) @_assert_pid_not_reused def send_signal(self, sig): @@ -2053,7 +2052,7 @@ def virtual_memory(): - used: memory used, calculated differently depending on the platform and designed for informational purposes only: - macOS: active + inactive + wired + macOS: active + wired BSD: active + wired + cached Linux: total - free diff --git a/psutil/_common.py b/psutil/_common.py index e3b45417..126d9d6f 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -64,7 +64,7 @@ __all__ = [ 'conn_tmap', 'deprecated_method', 'isfile_strict', 'memoize', 'parse_environ_block', 'path_exists_strict', 'usage_percent', 'supports_ipv6', 'sockfam_to_enum', 'socktype_to_enum', "wrap_numbers", - 'bytes2human', + 'bytes2human', 'conn_to_ntuple', ] @@ -257,8 +257,6 @@ if AF_UNIX is not None: "unix": ([AF_UNIX], [SOCK_STREAM, SOCK_DGRAM]), }) -del AF_INET, AF_UNIX, SOCK_STREAM, SOCK_DGRAM - # =================================================================== # --- utils @@ -268,12 +266,12 @@ del AF_INET, AF_UNIX, SOCK_STREAM, SOCK_DGRAM def usage_percent(used, total, round_=None): """Calculate percentage usage of 'used' against 'total'.""" try: - ret = (used / total) * 100 + ret = (float(used) / total) * 100 except ZeroDivisionError: - ret = 0.0 if isinstance(used, float) or isinstance(total, float) else 0 - if round_ is not None: - return round(ret, round_) + return 0.0 else: + if round_ is not None: + ret = round(ret, round_) return ret @@ -447,7 +445,7 @@ def sockfam_to_enum(num): else: # pragma: no cover try: return socket.AddressFamily(num) - except (ValueError, AttributeError): + except ValueError: return num @@ -459,11 +457,30 @@ def socktype_to_enum(num): return num else: # pragma: no cover try: - return socket.AddressType(num) - except (ValueError, AttributeError): + return socket.SocketKind(num) + except ValueError: return num +def conn_to_ntuple(fd, fam, type_, laddr, raddr, status, status_map, pid=None): + """Convert a raw connection tuple to a proper ntuple.""" + if fam in (socket.AF_INET, AF_INET6): + if laddr: + laddr = addr(*laddr) + if raddr: + raddr = addr(*raddr) + if type_ == socket.SOCK_STREAM and fam in (AF_INET, AF_INET6): + status = status_map.get(status, CONN_NONE) + else: + status = CONN_NONE # ignore whatever C returned to us + fam = sockfam_to_enum(fam) + type_ = socktype_to_enum(type_) + if pid is None: + return pconn(fd, fam, type_, laddr, raddr, status) + else: + return sconn(fd, fam, type_, laddr, raddr, status, pid) + + def deprecated_method(replacement): """A decorator which can be used to mark a method as deprecated 'replcement' is the method name which will be called instead. diff --git a/psutil/_compat.py b/psutil/_compat.py index c772f61d..07ab909a 100644 --- a/psutil/_compat.py +++ b/psutil/_compat.py @@ -5,12 +5,15 @@ """Module which provides compatibility with older Python versions.""" import collections +import errno import functools import os import sys __all__ = ["PY3", "long", "xrange", "unicode", "basestring", "u", "b", - "lru_cache", "which", "get_terminal_size"] + "lru_cache", "which", "get_terminal_size", + "FileNotFoundError", "PermissionError", "ProcessLookupError", + "InterruptedError", "ChildProcessError", "FileExistsError"] PY3 = sys.version_info[0] == 3 @@ -38,6 +41,73 @@ else: return s +# --- exceptions + + +if PY3: + FileNotFoundError = FileNotFoundError # NOQA + PermissionError = PermissionError # NOQA + ProcessLookupError = ProcessLookupError # NOQA + InterruptedError = InterruptedError # NOQA + ChildProcessError = ChildProcessError # NOQA + FileExistsError = FileExistsError # NOQA +else: + # https://github.com/PythonCharmers/python-future/blob/exceptions/ + # src/future/types/exceptions/pep3151.py + + def instance_checking_exception(base_exception=Exception): + def wrapped(instance_checker): + class TemporaryClass(base_exception): + + def __init__(self, *args, **kwargs): + if len(args) == 1 and isinstance(args[0], TemporaryClass): + unwrap_me = args[0] + for attr in dir(unwrap_me): + if not attr.startswith('__'): + setattr(self, attr, getattr(unwrap_me, attr)) + else: + super(TemporaryClass, self).__init__(*args, **kwargs) + + class __metaclass__(type): + def __instancecheck__(cls, inst): + return instance_checker(inst) + + def __subclasscheck__(cls, classinfo): + value = sys.exc_info()[1] + return isinstance(value, cls) + + TemporaryClass.__name__ = instance_checker.__name__ + TemporaryClass.__doc__ = instance_checker.__doc__ + return TemporaryClass + + return wrapped + + @instance_checking_exception(EnvironmentError) + def FileNotFoundError(inst): + return getattr(inst, 'errno', object()) == errno.ENOENT + + @instance_checking_exception(EnvironmentError) + def ProcessLookupError(inst): + return getattr(inst, 'errno', object()) == errno.ESRCH + + @instance_checking_exception(EnvironmentError) + def PermissionError(inst): + return getattr(inst, 'errno', object()) in ( + errno.EACCES, errno.EPERM) + + @instance_checking_exception(EnvironmentError) + def InterruptedError(inst): + return getattr(inst, 'errno', object()) == errno.EINTR + + @instance_checking_exception(EnvironmentError) + def ChildProcessError(inst): + return getattr(inst, 'errno', object()) == errno.ECHILD + + @instance_checking_exception(EnvironmentError) + def FileExistsError(inst): + return getattr(inst, 'errno', object()) == errno.EEXIST + + # --- stdlib additions diff --git a/psutil/_psaix.py b/psutil/_psaix.py index b24325d1..79e3be15 100644 --- a/psutil/_psaix.py +++ b/psutil/_psaix.py @@ -6,7 +6,6 @@ """AIX platform implementation.""" -import errno import functools import glob import os @@ -14,21 +13,21 @@ import re import subprocess import sys from collections import namedtuple -from socket import AF_INET from . import _common from . import _psposix from . import _psutil_aix as cext from . import _psutil_posix as cext_posix -from ._common import AF_INET6 +from ._common import conn_to_ntuple from ._common import get_procfs_path from ._common import memoize_when_activated from ._common import NIC_DUPLEX_FULL from ._common import NIC_DUPLEX_HALF from ._common import NIC_DUPLEX_UNKNOWN -from ._common import sockfam_to_enum -from ._common import socktype_to_enum from ._common import usage_percent +from ._compat import FileNotFoundError +from ._compat import PermissionError +from ._compat import ProcessLookupError from ._compat import PY3 @@ -220,27 +219,17 @@ def net_connections(kind, _pid=-1): % (kind, ', '.join([repr(x) for x in cmap]))) families, types = _common.conn_tmap[kind] rawlist = cext.net_connections(_pid) - ret = set() + ret = [] for item in rawlist: fd, fam, type_, laddr, raddr, status, pid = item if fam not in families: continue if type_ not in types: continue - status = TCP_STATUSES[status] - if fam in (AF_INET, AF_INET6): - if laddr: - laddr = _common.addr(*laddr) - if raddr: - raddr = _common.addr(*raddr) - fam = sockfam_to_enum(fam) - type_ = socktype_to_enum(type_) - if _pid == -1: - nt = _common.sconn(fd, fam, type_, laddr, raddr, status, pid) - else: - nt = _common.pconn(fd, fam, type_, laddr, raddr, status) - ret.add(nt) - return list(ret) + nt = conn_to_ntuple(fd, fam, type_, laddr, raddr, status, + TCP_STATUSES, pid=pid if _pid == -1 else None) + ret.append(nt) + return ret def net_if_stats(): @@ -327,22 +316,16 @@ def wrap_exceptions(fun): def wrapper(self, *args, **kwargs): try: return fun(self, *args, **kwargs) - except EnvironmentError as err: - # support for private module import - if (NoSuchProcess is None or AccessDenied is None or - ZombieProcess is None): - raise + except (FileNotFoundError, ProcessLookupError): # ENOENT (no such file or directory) gets raised on open(). # ESRCH (no such process) can get raised on read() if # process is gone in meantime. - if err.errno in (errno.ENOENT, errno.ESRCH): - if not pid_exists(self.pid): - raise NoSuchProcess(self.pid, self._name) - else: - raise ZombieProcess(self.pid, self._name, self._ppid) - if err.errno in (errno.EPERM, errno.EACCES): - raise AccessDenied(self.pid, self._name) - raise + if not pid_exists(self.pid): + raise NoSuchProcess(self.pid, self._name) + else: + raise ZombieProcess(self.pid, self._name, self._ppid) + except PermissionError: + raise AccessDenied(self.pid, self._name) return wrapper @@ -501,11 +484,9 @@ class Process(object): try: result = os.readlink("%s/%s/cwd" % (procfs_path, self.pid)) return result.rstrip('/') - except OSError as err: - if err.errno == errno.ENOENT: - os.stat("%s/%s" % (procfs_path, self.pid)) # raise NSP or AD - return None - raise + except FileNotFoundError: + os.stat("%s/%s" % (procfs_path, self.pid)) # raise NSP or AD + return None @wrap_exceptions def memory_info(self): diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index 3d9dfdab..2f41dc0b 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -10,25 +10,26 @@ import functools import os import xml.etree.ElementTree as ET from collections import namedtuple -from socket import AF_INET from collections import defaultdict from . import _common from . import _psposix from . import _psutil_bsd as cext from . import _psutil_posix as cext_posix -from ._common import AF_INET6 from ._common import conn_tmap +from ._common import conn_to_ntuple from ._common import FREEBSD from ._common import memoize from ._common import memoize_when_activated from ._common import NETBSD from ._common import OPENBSD -from ._common import sockfam_to_enum -from ._common import socktype_to_enum from ._common import usage_percent +from ._compat import FileNotFoundError +from ._compat import PermissionError +from ._compat import ProcessLookupError from ._compat import which + __extra__all__ = [] @@ -399,22 +400,8 @@ def net_connections(kind): fd, fam, type, laddr, raddr, status, pid = item # TODO: apply filter at C level if fam in families and type in types: - try: - status = TCP_STATUSES[status] - except KeyError: - # XXX: Not sure why this happens. I saw this occurring - # with IPv6 sockets opened by 'vim'. Those sockets - # have a very short lifetime so maybe the kernel - # can't initialize their status? - status = TCP_STATUSES[cext.PSUTIL_CONN_NONE] - if fam in (AF_INET, AF_INET6): - if laddr: - laddr = _common.addr(*laddr) - if raddr: - raddr = _common.addr(*raddr) - fam = sockfam_to_enum(fam) - type = socktype_to_enum(type) - nt = _common.sconn(fd, fam, type, laddr, raddr, status, pid) + nt = conn_to_ntuple(fd, fam, type, laddr, raddr, status, + TCP_STATUSES, pid) ret.add(nt) return list(ret) @@ -551,6 +538,14 @@ else: pid_exists = _psposix.pid_exists +def is_zombie(pid): + try: + st = cext.proc_oneshot_info(pid)[kinfo_proc_map['status']] + return st == cext.SZOMB + except Exception: + return False + + def wrap_exceptions(fun): """Decorator which translates bare OSError exceptions into NoSuchProcess and AccessDenied. @@ -559,19 +554,19 @@ def wrap_exceptions(fun): def wrapper(self, *args, **kwargs): try: return fun(self, *args, **kwargs) - except OSError as err: + except ProcessLookupError: + if not pid_exists(self.pid): + raise NoSuchProcess(self.pid, self._name) + else: + raise ZombieProcess(self.pid, self._name, self._ppid) + except PermissionError: + raise AccessDenied(self.pid, self._name) + except OSError: if self.pid == 0: if 0 in pids(): raise AccessDenied(self.pid, self._name) else: raise - if err.errno == errno.ESRCH: - if not pid_exists(self.pid): - raise NoSuchProcess(self.pid, self._name) - else: - raise ZombieProcess(self.pid, self._name, self._ppid) - if err.errno in (errno.EPERM, errno.EACCES): - raise AccessDenied(self.pid, self._name) raise return wrapper @@ -581,18 +576,16 @@ def wrap_exceptions_procfs(inst): """Same as above, for routines relying on reading /proc fs.""" try: yield - except EnvironmentError as err: + except (ProcessLookupError, FileNotFoundError): # ENOENT (no such file or directory) gets raised on open(). # ESRCH (no such process) can get raised on read() if # process is gone in meantime. - if err.errno in (errno.ENOENT, errno.ESRCH): - if not pid_exists(inst.pid): - raise NoSuchProcess(inst.pid, inst._name) - else: - raise ZombieProcess(inst.pid, inst._name, inst._ppid) - if err.errno in (errno.EPERM, errno.EACCES): - raise AccessDenied(inst.pid, inst._name) - raise + if not pid_exists(inst.pid): + raise NoSuchProcess(inst.pid, inst._name) + else: + raise ZombieProcess(inst.pid, inst._name, inst._ppid) + except PermissionError: + raise AccessDenied(inst.pid, inst._name) class Process(object): @@ -665,10 +658,14 @@ class Process(object): return cext.proc_cmdline(self.pid) except OSError as err: if err.errno == errno.EINVAL: - if not pid_exists(self.pid): - raise NoSuchProcess(self.pid, self._name) - else: + if is_zombie(self.pid): raise ZombieProcess(self.pid, self._name, self._ppid) + elif not pid_exists(self.pid): + raise NoSuchProcess(self.pid, self._name, self._ppid) + else: + # XXX: this happens with unicode tests. It means the C + # routine is unable to decode invalid unicode chars. + return [] else: raise else: @@ -769,25 +766,15 @@ class Process(object): if NETBSD: families, types = conn_tmap[kind] - ret = set() + ret = [] rawlist = cext.net_connections(self.pid) for item in rawlist: fd, fam, type, laddr, raddr, status, pid = item assert pid == self.pid if fam in families and type in types: - try: - status = TCP_STATUSES[status] - except KeyError: - status = TCP_STATUSES[cext.PSUTIL_CONN_NONE] - if fam in (AF_INET, AF_INET6): - if laddr: - laddr = _common.addr(*laddr) - if raddr: - raddr = _common.addr(*raddr) - fam = sockfam_to_enum(fam) - type = socktype_to_enum(type) - nt = _common.pconn(fd, fam, type, laddr, raddr, status) - ret.add(nt) + nt = conn_to_ntuple(fd, fam, type, laddr, raddr, status, + TCP_STATUSES) + ret.append(nt) self._assert_alive() return list(ret) @@ -796,18 +783,13 @@ class Process(object): ret = [] for item in rawlist: fd, fam, type, laddr, raddr, status = item - if fam in (AF_INET, AF_INET6): - if laddr: - laddr = _common.addr(*laddr) - if raddr: - raddr = _common.addr(*raddr) - fam = sockfam_to_enum(fam) - type = socktype_to_enum(type) - status = TCP_STATUSES[status] - nt = _common.pconn(fd, fam, type, laddr, raddr, status) + nt = conn_to_ntuple(fd, fam, type, laddr, raddr, status, + TCP_STATUSES) ret.append(nt) + if OPENBSD: self._assert_alive() + return ret @wrap_exceptions @@ -844,10 +826,7 @@ class Process(object): # it into None if OPENBSD and self.pid == 0: return None # ...else it would raise EINVAL - elif NETBSD: - with wrap_exceptions_procfs(self): - return os.readlink("/proc/%s/cwd" % self.pid) - elif HAS_PROC_OPEN_FILES: + elif NETBSD or HAS_PROC_OPEN_FILES: # FreeBSD < 8 does not support functions based on # kinfo_getfile() and kinfo_getvmmap() return cext.proc_cwd(self.pid) or None diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 33bafd28..d29ccc85 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -41,6 +41,9 @@ from ._common import supports_ipv6 from ._common import usage_percent from ._compat import b from ._compat import basestring +from ._compat import FileNotFoundError +from ._compat import PermissionError +from ._compat import ProcessLookupError from ._compat import PY3 if sys.version_info >= (3, 4): @@ -70,6 +73,7 @@ POWER_SUPPLY_PATH = "/sys/class/power_supply" HAS_SMAPS = os.path.exists('/proc/%s/smaps' % os.getpid()) HAS_PRLIMIT = hasattr(cext, "linux_prlimit") HAS_PROC_IO_PRIORITY = hasattr(cext, "proc_ioprio_get") +HAS_CPU_AFFINITY = hasattr(cext, "proc_cpu_affinity_get") _DEFAULT = object() # RLIMIT_* constants, not guaranteed to be present on all kernels @@ -200,6 +204,10 @@ pmmap_ext = namedtuple( pio = namedtuple('pio', ['read_count', 'write_count', 'read_bytes', 'write_bytes', 'read_chars', 'write_chars']) +# psutil.Process.cpu_times() +pcputimes = namedtuple('pcputimes', + ['user', 'system', 'children_user', 'children_system', + 'iowait']) # ===================================================================== @@ -772,17 +780,16 @@ class Connections: for fd in os.listdir("%s/%s/fd" % (self._procfs_path, pid)): try: inode = readlink("%s/%s/fd/%s" % (self._procfs_path, pid, fd)) - except OSError as err: + except (FileNotFoundError, ProcessLookupError): # ENOENT == file which is gone in the meantime; # os.stat('/proc/%s' % self.pid) will be done later # to force NSP (if it's the case) - if err.errno in (errno.ENOENT, errno.ESRCH): - continue - elif err.errno == errno.EINVAL: + continue + except OSError as err: + if err.errno == errno.EINVAL: # not a link continue - else: - raise + raise else: if inode.startswith('socket:['): # the process is using a socket @@ -795,7 +802,7 @@ class Connections: for pid in pids(): try: inodes.update(self.get_proc_inodes(pid)) - except OSError as err: + except (FileNotFoundError, ProcessLookupError, PermissionError): # os.listdir() is gonna raise a lot of access denied # exceptions in case of unprivileged user; that's fine # as we'll just end up returning a connection with PID @@ -803,9 +810,7 @@ class Connections: # Both netstat -an and lsof does the same so it's # unlikely we can do any better. # ENOENT just means a PID disappeared on us. - if err.errno not in ( - errno.ENOENT, errno.ESRCH, errno.EPERM, errno.EACCES): - raise + continue return inodes @staticmethod @@ -933,7 +938,7 @@ class Connections: path = tokens[-1] else: path = "" - type_ = int(type_) + type_ = _common.socktype_to_enum(int(type_)) # XXX: determining the remote endpoint of a # UNIX socket on Linux is not possible, see: # https://serverfault.com/questions/252723/ @@ -1490,11 +1495,10 @@ def ppid_map(): try: with open_binary("%s/%s/stat" % (procfs_path, pid)) as f: data = f.read() - except EnvironmentError as err: + except (FileNotFoundError, ProcessLookupError): # Note: we should be able to access /stat for all processes # aka it's unlikely we'll bump into EPERM, which is good. - if err.errno not in (errno.ENOENT, errno.ESRCH): - raise + pass else: rpar = data.rfind(b')') dset = data[rpar + 2:].split() @@ -1511,16 +1515,12 @@ def wrap_exceptions(fun): def wrapper(self, *args, **kwargs): try: return fun(self, *args, **kwargs) - except EnvironmentError as err: - if err.errno in (errno.EPERM, errno.EACCES): - raise AccessDenied(self.pid, self._name) - # ESRCH (no such process) can be raised on read() if - # process is gone in the meantime. - if err.errno == errno.ESRCH: - raise NoSuchProcess(self.pid, self._name) - # ENOENT (no such file or directory) can be raised on open(). - if err.errno == errno.ENOENT and not os.path.exists("%s/%s" % ( - self._procfs_path, self.pid)): + except PermissionError: + raise AccessDenied(self.pid, self._name) + except ProcessLookupError: + raise NoSuchProcess(self.pid, self._name) + except FileNotFoundError: + if not os.path.exists("%s/%s" % (self._procfs_path, self.pid)): raise NoSuchProcess(self.pid, self._name) # Note: zombies will keep existing under /proc until they're # gone so there's no way to distinguish them in here. @@ -1576,6 +1576,7 @@ class Process(object): ret['children_stime'] = fields[14] ret['create_time'] = fields[19] ret['cpu_num'] = fields[36] + ret['blkio_ticks'] = fields[39] # aka 'delayacct_blkio_ticks' return ret @@ -1617,21 +1618,19 @@ class Process(object): def exe(self): try: return readlink("%s/%s/exe" % (self._procfs_path, self.pid)) - except OSError as err: - if err.errno in (errno.ENOENT, errno.ESRCH): - # no such file error; might be raised also if the - # path actually exists for system processes with - # low pids (about 0-20) - if os.path.lexists("%s/%s" % (self._procfs_path, self.pid)): - return "" + except (FileNotFoundError, ProcessLookupError): + # no such file error; might be raised also if the + # path actually exists for system processes with + # low pids (about 0-20) + if os.path.lexists("%s/%s" % (self._procfs_path, self.pid)): + return "" + else: + if not pid_exists(self.pid): + raise NoSuchProcess(self.pid, self._name) else: - if not pid_exists(self.pid): - raise NoSuchProcess(self.pid, self._name) - else: - raise ZombieProcess(self.pid, self._name, self._ppid) - if err.errno in (errno.EPERM, errno.EACCES): - raise AccessDenied(self.pid, self._name) - raise + raise ZombieProcess(self.pid, self._name, self._ppid) + except PermissionError: + raise AccessDenied(self.pid, self._name) @wrap_exceptions def cmdline(self): @@ -1650,7 +1649,7 @@ class Process(object): sep = '\x00' if data.endswith('\x00') else ' ' if data.endswith(sep): data = data[:-1] - return [x for x in data.split(sep)] + return data.split(sep) @wrap_exceptions def environ(self): @@ -1707,7 +1706,8 @@ class Process(object): stime = float(values['stime']) / CLOCK_TICKS children_utime = float(values['children_utime']) / CLOCK_TICKS children_stime = float(values['children_stime']) / CLOCK_TICKS - return _common.pcputimes(utime, stime, children_utime, children_stime) + iowait = float(values['blkio_ticks']) / CLOCK_TICKS + return pcputimes(utime, stime, children_utime, children_stime, iowait) @wrap_exceptions def cpu_num(self): @@ -1856,14 +1856,12 @@ class Process(object): def cwd(self): try: return readlink("%s/%s/cwd" % (self._procfs_path, self.pid)) - except OSError as err: + except (FileNotFoundError, ProcessLookupError): # https://github.com/giampaolo/psutil/issues/986 - if err.errno in (errno.ENOENT, errno.ESRCH): - if not pid_exists(self.pid): - raise NoSuchProcess(self.pid, self._name) - else: - raise ZombieProcess(self.pid, self._name, self._ppid) - raise + if not pid_exists(self.pid): + raise NoSuchProcess(self.pid, self._name) + else: + raise ZombieProcess(self.pid, self._name, self._ppid) @wrap_exceptions def num_ctx_switches(self, @@ -1899,13 +1897,11 @@ class Process(object): try: with open_binary(fname) as f: st = f.read().strip() - except IOError as err: - if err.errno == errno.ENOENT: - # no such file or directory; it means thread - # disappeared on us - hit_enoent = True - continue - raise + except FileNotFoundError: + # no such file or directory; it means thread + # disappeared on us + hit_enoent = True + continue # ignore the first two values ("pid (exe)") st = st[st.find(b')') + 2:] values = st.split(b' ') @@ -1930,38 +1926,41 @@ class Process(object): def nice_set(self, value): return cext_posix.setpriority(self.pid, value) - @wrap_exceptions - def cpu_affinity_get(self): - return cext.proc_cpu_affinity_get(self.pid) + # starting from CentOS 6. + if HAS_CPU_AFFINITY: - def _get_eligible_cpus( - self, _re=re.compile(br"Cpus_allowed_list:\t(\d+)-(\d+)")): - # See: https://github.com/giampaolo/psutil/issues/956 - data = self._read_status_file() - match = _re.findall(data) - if match: - return list(range(int(match[0][0]), int(match[0][1]) + 1)) - else: - return list(range(len(per_cpu_times()))) + @wrap_exceptions + def cpu_affinity_get(self): + return cext.proc_cpu_affinity_get(self.pid) + + def _get_eligible_cpus( + self, _re=re.compile(br"Cpus_allowed_list:\t(\d+)-(\d+)")): + # See: https://github.com/giampaolo/psutil/issues/956 + data = self._read_status_file() + match = _re.findall(data) + if match: + return list(range(int(match[0][0]), int(match[0][1]) + 1)) + else: + return list(range(len(per_cpu_times()))) - @wrap_exceptions - def cpu_affinity_set(self, cpus): - try: - cext.proc_cpu_affinity_set(self.pid, cpus) - except (OSError, ValueError) as err: - if isinstance(err, ValueError) or err.errno == errno.EINVAL: - eligible_cpus = self._get_eligible_cpus() - all_cpus = tuple(range(len(per_cpu_times()))) - for cpu in cpus: - if cpu not in all_cpus: - raise ValueError( - "invalid CPU number %r; choose between %s" % ( - cpu, eligible_cpus)) - if cpu not in eligible_cpus: - raise ValueError( - "CPU number %r is not eligible; choose " - "between %s" % (cpu, eligible_cpus)) - raise + @wrap_exceptions + def cpu_affinity_set(self, cpus): + try: + cext.proc_cpu_affinity_set(self.pid, cpus) + except (OSError, ValueError) as err: + if isinstance(err, ValueError) or err.errno == errno.EINVAL: + eligible_cpus = self._get_eligible_cpus() + all_cpus = tuple(range(len(per_cpu_times()))) + for cpu in cpus: + if cpu not in all_cpus: + raise ValueError( + "invalid CPU number %r; choose between %s" % ( + cpu, eligible_cpus)) + if cpu not in eligible_cpus: + raise ValueError( + "CPU number %r is not eligible; choose " + "between %s" % (cpu, eligible_cpus)) + raise # only starting from kernel 2.6.13 if HAS_PROC_IO_PRIORITY: @@ -2029,16 +2028,15 @@ class Process(object): file = "%s/%s/fd/%s" % (self._procfs_path, self.pid, fd) try: path = readlink(file) - except OSError as err: + except (FileNotFoundError, ProcessLookupError): # ENOENT == file which is gone in the meantime - if err.errno in (errno.ENOENT, errno.ESRCH): - hit_enoent = True - continue - elif err.errno == errno.EINVAL: + hit_enoent = True + continue + except OSError as err: + if err.errno == errno.EINVAL: # not a link continue - else: - raise + raise else: # If path is not an absolute there's no way to tell # whether it's a regular file or not, so we skip it. @@ -2052,13 +2050,10 @@ class Process(object): with open_binary(file) as f: pos = int(f.readline().split()[1]) flags = int(f.readline().split()[1], 8) - except IOError as err: - if err.errno == errno.ENOENT: - # fd gone in the meantime; process may - # still be alive - hit_enoent = True - else: - raise + except FileNotFoundError: + # fd gone in the meantime; process may + # still be alive + hit_enoent = True else: mode = file_flags_to_mode(flags) ntuple = popenfile( diff --git a/psutil/_psosx.py b/psutil/_psosx.py index 7459a0f3..7f28447b 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -8,20 +8,19 @@ import contextlib import errno import functools import os -from socket import AF_INET from collections import namedtuple from . import _common from . import _psposix from . import _psutil_osx as cext from . import _psutil_posix as cext_posix -from ._common import AF_INET6 from ._common import conn_tmap +from ._common import conn_to_ntuple from ._common import isfile_strict from ._common import memoize_when_activated from ._common import parse_environ_block -from ._common import sockfam_to_enum -from ._common import socktype_to_enum +from ._compat import PermissionError +from ._compat import ProcessLookupError from ._common import usage_percent @@ -337,12 +336,10 @@ def wrap_exceptions(fun): def wrapper(self, *args, **kwargs): try: return fun(self, *args, **kwargs) - except OSError as err: - if err.errno == errno.ESRCH: - raise NoSuchProcess(self.pid, self._name) - if err.errno in (errno.EPERM, errno.EACCES): - raise AccessDenied(self.pid, self._name) - raise + except ProcessLookupError: + raise NoSuchProcess(self.pid, self._name) + except PermissionError: + raise AccessDenied(self.pid, self._name) except cext.ZombieProcessError: raise ZombieProcess(self.pid, self._name, self._ppid) return wrapper @@ -529,15 +526,8 @@ class Process(object): ret = [] for item in rawlist: fd, fam, type, laddr, raddr, status = item - status = TCP_STATUSES[status] - fam = sockfam_to_enum(fam) - type = socktype_to_enum(type) - if fam in (AF_INET, AF_INET6): - if laddr: - laddr = _common.addr(*laddr) - if raddr: - raddr = _common.addr(*raddr) - nt = _common.pconn(fd, fam, type, laddr, raddr, status) + nt = conn_to_ntuple(fd, fam, type, laddr, raddr, status, + TCP_STATUSES) ret.append(nt) return ret diff --git a/psutil/_psposix.py b/psutil/_psposix.py index d362143f..24570224 100644 --- a/psutil/_psposix.py +++ b/psutil/_psposix.py @@ -4,7 +4,6 @@ """Routines common to all posix systems.""" -import errno import glob import os import sys @@ -13,6 +12,11 @@ import time from ._common import memoize from ._common import sdiskusage from ._common import usage_percent +from ._compat import ChildProcessError +from ._compat import FileNotFoundError +from ._compat import InterruptedError +from ._compat import PermissionError +from ._compat import ProcessLookupError from ._compat import PY3 from ._compat import unicode @@ -36,19 +40,13 @@ def pid_exists(pid): return True try: os.kill(pid, 0) - except OSError as err: - if err.errno == errno.ESRCH: - # ESRCH == No such process - return False - elif err.errno == errno.EPERM: - # EPERM clearly means there's a process to deny access to - return True - else: - # According to "man 2 kill" possible error values are - # (EINVAL, EPERM, ESRCH) therefore we should never get - # here. If we do let's be explicit in considering this - # an error. - raise err + except ProcessLookupError: + return False + except PermissionError: + # EPERM clearly means there's a process to deny access to + return True + # According to "man 2 kill" possible error values are + # (EINVAL, EPERM, ESRCH) else: return True @@ -84,24 +82,20 @@ def wait_pid(pid, timeout=None, proc_name=None): while True: try: retpid, status = waitcall() - except OSError as err: - if err.errno == errno.EINTR: - delay = check_timeout(delay) - continue - elif err.errno == errno.ECHILD: - # This has two meanings: - # - pid is not a child of os.getpid() in which case - # we keep polling until it's gone - # - pid never existed in the first place - # In both cases we'll eventually return None as we - # can't determine its exit status code. - while True: - if pid_exists(pid): - delay = check_timeout(delay) - else: - return - else: - raise + except InterruptedError: + delay = check_timeout(delay) + except ChildProcessError: + # This has two meanings: + # - pid is not a child of os.getpid() in which case + # we keep polling until it's gone + # - pid never existed in the first place + # In both cases we'll eventually return None as we + # can't determine its exit status code. + while True: + if pid_exists(pid): + delay = check_timeout(delay) + else: + return else: if retpid == 0: # WNOHANG was used, pid is still running @@ -180,7 +174,6 @@ def get_terminal_map(): assert name not in ret, name try: ret[os.stat(name).st_rdev] = name - except OSError as err: - if err.errno != errno.ENOENT: - raise + except FileNotFoundError: + pass return ret diff --git a/psutil/_pssunos.py b/psutil/_pssunos.py index 6d7fda85..2aa2a866 100644 --- a/psutil/_pssunos.py +++ b/psutil/_pssunos.py @@ -25,6 +25,9 @@ from ._common import sockfam_to_enum from ._common import socktype_to_enum from ._common import usage_percent from ._compat import b +from ._compat import FileNotFoundError +from ._compat import PermissionError +from ._compat import ProcessLookupError from ._compat import PY3 @@ -262,6 +265,7 @@ def net_connections(kind, _pid=-1): continue if type_ not in types: continue + # TODO: refactor and use _common.conn_to_ntuple. if fam in (AF_INET, AF_INET6): if laddr: laddr = _common.addr(*laddr) @@ -341,22 +345,22 @@ def wrap_exceptions(fun): def wrapper(self, *args, **kwargs): try: return fun(self, *args, **kwargs) - except EnvironmentError as err: + except (FileNotFoundError, ProcessLookupError): + # ENOENT (no such file or directory) gets raised on open(). + # ESRCH (no such process) can get raised on read() if + # process is gone in meantime. + if not pid_exists(self.pid): + raise NoSuchProcess(self.pid, self._name) + else: + raise ZombieProcess(self.pid, self._name, self._ppid) + except PermissionError: + raise AccessDenied(self.pid, self._name) + except OSError: if self.pid == 0: if 0 in pids(): raise AccessDenied(self.pid, self._name) else: raise - # ENOENT (no such file or directory) gets raised on open(). - # ESRCH (no such process) can get raised on read() if - # process is gone in meantime. - if err.errno in (errno.ENOENT, errno.ESRCH): - if not pid_exists(self.pid): - raise NoSuchProcess(self.pid, self._name) - else: - raise ZombieProcess(self.pid, self._name, self._ppid) - if err.errno in (errno.EPERM, errno.EACCES): - raise AccessDenied(self.pid, self._name) raise return wrapper @@ -514,11 +518,9 @@ class Process(object): try: return os.readlink( '%s/%d/path/%d' % (procfs_path, self.pid, x)) - except OSError as err: - if err.errno == errno.ENOENT: - hit_enoent = True - continue - raise + except FileNotFoundError: + hit_enoent = True + continue if hit_enoent: self._assert_alive() @@ -531,11 +533,9 @@ class Process(object): procfs_path = self._procfs_path try: return os.readlink("%s/%s/path/cwd" % (procfs_path, self.pid)) - except OSError as err: - if err.errno == errno.ENOENT: - os.stat("%s/%s" % (procfs_path, self.pid)) # raise NSP or AD - return None - raise + except FileNotFoundError: + os.stat("%s/%s" % (procfs_path, self.pid)) # raise NSP or AD + return None @wrap_exceptions def memory_info(self): @@ -596,12 +596,9 @@ class Process(object): if os.path.islink(path): try: file = os.readlink(path) - except OSError as err: - # ENOENT == file which is gone in the meantime - if err.errno == errno.ENOENT: - hit_enoent = True - continue - raise + except FileNotFoundError: + hit_enoent = True + continue else: if isfile_strict(file): retlist.append(_common.popenfile(file, int(fd))) diff --git a/psutil/_psutil_aix.c b/psutil/_psutil_aix.c index 723d159d..8c055a43 100644 --- a/psutil/_psutil_aix.c +++ b/psutil/_psutil_aix.c @@ -180,7 +180,7 @@ psutil_proc_args(PyObject *self, PyObject *args) { } procbuf.pi_pid = pid; - ret = getargs(&procbuf, sizeof(struct procinfo), argbuf, ARG_MAX); + ret = getargs(&procbuf, sizeof(procbuf), argbuf, ARG_MAX); if (ret == -1) { PyErr_SetFromErrno(PyExc_OSError); goto error; @@ -241,7 +241,7 @@ psutil_proc_environ(PyObject *self, PyObject *args) { } procbuf.pi_pid = pid; - ret = getevars(&procbuf, sizeof(struct procinfo), envbuf, ARG_MAX); + ret = getevars(&procbuf, sizeof(procbuf), envbuf, ARG_MAX); if (ret == -1) { PyErr_SetFromErrno(PyExc_OSError); goto error; diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index efb933fb..74fe5922 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -30,7 +30,9 @@ #include <sys/types.h> #include <sys/param.h> #include <sys/sysctl.h> +#if !defined(__NetBSD__) #include <sys/user.h> +#endif #include <sys/proc.h> #include <sys/file.h> #include <sys/socket.h> @@ -919,9 +921,9 @@ PsutilMethods[] = { #if defined(PSUTIL_FREEBSD) || defined(PSUTIL_OPENBSD) {"proc_connections", psutil_proc_connections, METH_VARARGS, "Return connections opened by process"}, +#endif {"proc_cwd", psutil_proc_cwd, METH_VARARGS, "Return process current working directory."}, -#endif #if defined(__FreeBSD_version) && __FreeBSD_version >= 800000 || PSUTIL_OPENBSD || defined(PSUTIL_NETBSD) {"proc_num_fds", psutil_proc_num_fds, METH_VARARGS, "Return the number of file descriptors opened by this process"}, diff --git a/psutil/_psutil_linux.c b/psutil/_psutil_linux.c index 4bf53b85..8151b75d 100644 --- a/psutil/_psutil_linux.c +++ b/psutil/_psutil_linux.c @@ -54,6 +54,11 @@ static const int NCPUS_START = sizeof(unsigned long) * CHAR_BIT; #include <sys/resource.h> #endif +// Should exist starting from CentOS 6 (year 2011). +#ifdef CPU_ALLOC + #define PSUTIL_HAVE_CPU_AFFINITY +#endif + #include "_psutil_common.h" #include "_psutil_posix.h" @@ -279,11 +284,8 @@ psutil_linux_sysinfo(PyObject *self, PyObject *args) { /* * Return process CPU affinity as a Python list - * The dual implementation exists because of: - * https://github.com/giampaolo/psutil/issues/536 */ - -#ifdef CPU_ALLOC +#ifdef PSUTIL_HAVE_CPU_AFFINITY static PyObject * psutil_proc_cpu_affinity_get(PyObject *self, PyObject *args) { @@ -347,51 +349,9 @@ error: Py_XDECREF(py_list); return NULL; } -#else /* - * Alternative implementation in case CPU_ALLOC is not defined. - */ -static PyObject * -psutil_proc_cpu_affinity_get(PyObject *self, PyObject *args) { - cpu_set_t cpuset; - unsigned int len = sizeof(cpu_set_t); - long pid; - int i; - PyObject* py_retlist = NULL; - PyObject *py_cpu_num = NULL; - - if (!PyArg_ParseTuple(args, "l", &pid)) - return NULL; - CPU_ZERO(&cpuset); - if (sched_getaffinity(pid, len, &cpuset) < 0) - return PyErr_SetFromErrno(PyExc_OSError); - - py_retlist = PyList_New(0); - if (py_retlist == NULL) - goto error; - for (i = 0; i < CPU_SETSIZE; ++i) { - if (CPU_ISSET(i, &cpuset)) { - py_cpu_num = Py_BuildValue("i", i); - if (py_cpu_num == NULL) - goto error; - if (PyList_Append(py_retlist, py_cpu_num)) - goto error; - Py_DECREF(py_cpu_num); - } - } - - return py_retlist; - -error: - Py_XDECREF(py_cpu_num); - Py_XDECREF(py_retlist); - return NULL; -} -#endif - -/* * Set process CPU affinity; expects a bitmask */ static PyObject * @@ -432,7 +392,6 @@ psutil_proc_cpu_affinity_set(PyObject *self, PyObject *args) { CPU_SET(value, &cpu_set); } - len = sizeof(cpu_set); if (sched_setaffinity(pid, len, &cpu_set)) { PyErr_SetFromErrno(PyExc_OSError); @@ -447,6 +406,7 @@ error: Py_DECREF(py_cpu_seq); return NULL; } +#endif /* PSUTIL_HAVE_CPU_AFFINITY */ /* @@ -583,16 +543,18 @@ static PyMethodDef PsutilMethods[] = { // --- per-process functions -#if PSUTIL_HAVE_IOPRIO +#ifdef PSUTIL_HAVE_IOPRIO {"proc_ioprio_get", psutil_proc_ioprio_get, METH_VARARGS, "Get process I/O priority"}, {"proc_ioprio_set", psutil_proc_ioprio_set, METH_VARARGS, "Set process I/O priority"}, #endif +#ifdef PSUTIL_HAVE_CPU_AFFINITY {"proc_cpu_affinity_get", psutil_proc_cpu_affinity_get, METH_VARARGS, "Return process CPU affinity as a Python long (the bitmask)."}, {"proc_cpu_affinity_set", psutil_proc_cpu_affinity_set, METH_VARARGS, "Set process CPU affinity; expects a bitmask."}, +#endif // --- system related functions diff --git a/psutil/_psutil_posix.c b/psutil/_psutil_posix.c index d9a8f6d1..5d20b21e 100644 --- a/psutil/_psutil_posix.c +++ b/psutil/_psutil_posix.c @@ -354,7 +354,7 @@ error: static PyObject * psutil_net_if_mtu(PyObject *self, PyObject *args) { char *nic_name; - int sock = 0; + int sock = -1; int ret; #ifdef PSUTIL_SUNOS10 struct lifreq lifr; @@ -387,7 +387,7 @@ psutil_net_if_mtu(PyObject *self, PyObject *args) { #endif error: - if (sock != 0) + if (sock != -1) close(sock); return PyErr_SetFromErrno(PyExc_OSError); } @@ -401,7 +401,7 @@ error: static PyObject * psutil_net_if_flags(PyObject *self, PyObject *args) { char *nic_name; - int sock = 0; + int sock = -1; int ret; struct ifreq ifr; @@ -424,7 +424,7 @@ psutil_net_if_flags(PyObject *self, PyObject *args) { return Py_BuildValue("O", Py_False); error: - if (sock != 0) + if (sock != -1) close(sock); return PyErr_SetFromErrno(PyExc_OSError); } @@ -579,7 +579,7 @@ int psutil_get_nic_speed(int ifm_active) { static PyObject * psutil_net_if_duplex_speed(PyObject *self, PyObject *args) { char *nic_name; - int sock = 0; + int sock = -1; int ret; int duplex; int speed; @@ -591,7 +591,7 @@ psutil_net_if_duplex_speed(PyObject *self, PyObject *args) { sock = socket(AF_INET, SOCK_DGRAM, 0); if (sock == -1) - goto error; + return PyErr_SetFromErrno(PyExc_OSError); strncpy(ifr.ifr_name, nic_name, sizeof(ifr.ifr_name)); // speed / duplex @@ -614,11 +614,6 @@ psutil_net_if_duplex_speed(PyObject *self, PyObject *args) { close(sock); return Py_BuildValue("[ii]", duplex, speed); - -error: - if (sock != 0) - close(sock); - return PyErr_SetFromErrno(PyExc_OSError); } #endif // net_if_stats() macOS/BSD implementation diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 3f131980..cf938a6f 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -33,14 +33,13 @@ except ImportError as err: raise from ._common import conn_tmap +from ._common import conn_to_ntuple from ._common import ENCODING from ._common import ENCODING_ERRS from ._common import isfile_strict from ._common import memoize from ._common import memoize_when_activated from ._common import parse_environ_block -from ._common import sockfam_to_enum -from ._common import socktype_to_enum from ._common import usage_percent from ._compat import long from ._compat import lru_cache @@ -388,17 +387,8 @@ def net_connections(kind, _pid=-1): ret = set() for item in rawlist: fd, fam, type, laddr, raddr, status, pid = item - if laddr: - laddr = _common.addr(*laddr) - if raddr: - raddr = _common.addr(*raddr) - status = TCP_STATUSES[status] - fam = sockfam_to_enum(fam) - type = socktype_to_enum(type) - if _pid == -1: - nt = _common.sconn(fd, fam, type, laddr, raddr, status, pid) - else: - nt = _common.pconn(fd, fam, type, laddr, raddr, status) + nt = conn_to_ntuple(fd, fam, type, laddr, raddr, status, TCP_STATUSES, + pid=pid if _pid == -1 else None) ret.add(nt) return list(ret) diff --git a/psutil/arch/netbsd/specific.c b/psutil/arch/netbsd/specific.c index cab60d60..25adffcd 100644 --- a/psutil/arch/netbsd/specific.c +++ b/psutil/arch/netbsd/specific.c @@ -22,7 +22,6 @@ #include <sys/types.h> #include <sys/param.h> #include <sys/sysctl.h> -#include <sys/user.h> #include <sys/proc.h> #include <sys/swap.h> // for swap_mem #include <signal.h> @@ -112,6 +111,44 @@ kinfo_getfile(pid_t pid, int* cnt) { return kf; } +PyObject * +psutil_proc_cwd(PyObject *self, PyObject *args) { + long pid; + + char path[MAXPATHLEN]; + size_t pathlen = sizeof path; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + +#ifdef KERN_PROC_CWD + int name[] = { CTL_KERN, KERN_PROC_ARGS, pid, KERN_PROC_CWD}; + if (sysctl(name, 4, path, &pathlen, NULL, 0) != 0) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } +#else + char *buf; + if (asprintf(&buf, "/proc/%d/cwd", (int)pid) < 0) { + PyErr_NoMemory(); + return NULL; + } + + ssize_t len = readlink(buf, path, sizeof(path) - 1); + free(buf); + if (len == -1) { + if (errno == ENOENT) + NoSuchProcess(""); + else + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + path[len] = '\0'; +#endif + + return PyUnicode_DecodeFSDefault(path); +} + // XXX: This is no longer used as per // https://github.com/giampaolo/psutil/pull/557#issuecomment-171912820 @@ -313,40 +350,35 @@ psutil_get_proc_list(kinfo_proc **procList, size_t *procCount) { char * psutil_get_cmd_args(pid_t pid, size_t *argsize) { int mib[4]; - ssize_t st; - size_t argmax; - size_t size; - char *procargs = NULL; + int st; + size_t len; + char *procargs; mib[0] = CTL_KERN; - mib[1] = KERN_ARGMAX; + mib[1] = KERN_PROC_ARGS; + mib[2] = pid; + mib[3] = KERN_PROC_ARGV; + len = 0; - size = sizeof(argmax); - st = sysctl(mib, 2, &argmax, &size, NULL, 0); + st = sysctl(mib, __arraycount(mib), NULL, &len, NULL, 0); if (st == -1) { PyErr_SetFromErrno(PyExc_OSError); return NULL; } - procargs = (char *)malloc(argmax); + procargs = (char *)malloc(len); if (procargs == NULL) { PyErr_NoMemory(); return NULL; } - - mib[0] = CTL_KERN; - mib[1] = KERN_PROC_ARGS; - mib[2] = pid; - mib[3] = KERN_PROC_ARGV; - - st = sysctl(mib, 4, procargs, &argmax, NULL, 0); + st = sysctl(mib, __arraycount(mib), procargs, &len, NULL, 0); if (st == -1) { free(procargs); PyErr_SetFromErrno(PyExc_OSError); return NULL; } - *argsize = argmax; + *argsize = len; return procargs; } diff --git a/psutil/arch/netbsd/specific.h b/psutil/arch/netbsd/specific.h index 96ad9f7d..391ed164 100644 --- a/psutil/arch/netbsd/specific.h +++ b/psutil/arch/netbsd/specific.h @@ -26,3 +26,4 @@ PyObject* psutil_disk_io_counters(PyObject* self, PyObject* args); PyObject* psutil_proc_exe(PyObject* self, PyObject* args); PyObject* psutil_proc_num_threads(PyObject* self, PyObject* args); PyObject* psutil_cpu_stats(PyObject* self, PyObject* args); +PyObject *psutil_proc_cwd(PyObject *self, PyObject *args); diff --git a/psutil/arch/windows/process_info.c b/psutil/arch/windows/process_info.c index ba9966bb..69da28bd 100644 --- a/psutil/arch/windows/process_info.c +++ b/psutil/arch/windows/process_info.c @@ -576,41 +576,44 @@ psutil_get_process_data(long pid, } } - if (! NT_SUCCESS( - NtWow64QueryInformationProcess64( + status = NtWow64QueryInformationProcess64( hProcess, ProcessBasicInformation, &pbi64, sizeof(pbi64), - NULL))) - { - PyErr_SetFromOSErrnoWithSyscall( - "NtWow64QueryInformationProcess64(ProcessBasicInformation)"); + NULL); + if (!NT_SUCCESS(status)) { + psutil_SetFromNTStatusErr( + status, + "NtWow64QueryInformationProcess64(ProcessBasicInformation)" + ); goto error; } // read peb - if (! NT_SUCCESS(NtWow64ReadVirtualMemory64( - hProcess, - pbi64.PebBaseAddress, - &peb64, - sizeof(peb64), - NULL))) - { - PyErr_SetFromOSErrnoWithSyscall("NtWow64ReadVirtualMemory64"); + status = NtWow64ReadVirtualMemory64( + hProcess, + pbi64.PebBaseAddress, + &peb64, + sizeof(peb64), + NULL); + if (!NT_SUCCESS(status)) { + psutil_SetFromNTStatusErr(status, "NtWow64ReadVirtualMemory64"); goto error; } // read process parameters - if (! NT_SUCCESS(NtWow64ReadVirtualMemory64( + status = NtWow64ReadVirtualMemory64( hProcess, peb64.ProcessParameters, &procParameters64, sizeof(procParameters64), - NULL))) - { - PyErr_SetFromOSErrnoWithSyscall( - "NtWow64ReadVirtualMemory64(ProcessParameters)"); + NULL); + if (!NT_SUCCESS(status)) { + psutil_SetFromNTStatusErr( + status, + "NtWow64ReadVirtualMemory64(ProcessParameters)" + ); goto error; } @@ -705,14 +708,14 @@ psutil_get_process_data(long pid, #ifndef _WIN64 if (weAreWow64 && !theyAreWow64) { - if (! NT_SUCCESS(NtWow64ReadVirtualMemory64( + status = NtWow64ReadVirtualMemory64( hProcess, src64, buffer, size, - NULL))) - { - PyErr_SetFromOSErrnoWithSyscall("NtWow64ReadVirtualMemory64"); + NULL); + if (!NT_SUCCESS(status)) { + psutil_SetFromNTStatusErr(status, "NtWow64ReadVirtualMemory64"); goto error; } } else diff --git a/psutil/arch/windows/wmi.c b/psutil/arch/windows/wmi.c index 5858a9e0..f43d790c 100644 --- a/psutil/arch/windows/wmi.c +++ b/psutil/arch/windows/wmi.c @@ -22,8 +22,8 @@ // This formula comes from linux's include/linux/sched/loadavg.h // https://github.com/torvalds/linux/blob/345671ea0f9258f410eb057b9ced9cefbbe5dc78/include/linux/sched/loadavg.h#L20-L23 #define LOADAVG_FACTOR_1F 0.9200444146293232478931553241 -#define LOADAVG_FACTOR_5F 0.6592406302004437462547604110 -#define LOADAVG_FACTOR_15F 0.2865047968601901003248854266 +#define LOADAVG_FACTOR_5F 0.9834714538216174894737477501 +#define LOADAVG_FACTOR_15F 0.9944598480048967508795473394 // The time interval in seconds between taking load counts, same as Linux #define SAMPLING_INTERVAL 5 @@ -112,4 +112,4 @@ error: PyObject * psutil_get_loadavg(PyObject *self, PyObject *args) { return Py_BuildValue("(ddd)", load_avg_1m, load_avg_5m, load_avg_15m); -}
\ No newline at end of file +} diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 79681719..6c629368 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -32,15 +32,18 @@ import traceback import warnings from socket import AF_INET from socket import AF_INET6 -from socket import SOCK_DGRAM from socket import SOCK_STREAM import psutil +from psutil import AIX from psutil import MACOS from psutil import POSIX from psutil import SUNOS from psutil import WINDOWS from psutil._common import supports_ipv6 +from psutil._compat import ChildProcessError +from psutil._compat import FileExistsError +from psutil._compat import FileNotFoundError from psutil._compat import PY3 from psutil._compat import u from psutil._compat import unicode @@ -91,7 +94,7 @@ __all__ = [ # sync primitives 'call_until', 'wait_for_pid', 'wait_for_file', # network - 'check_connection_ntuple', 'check_net_address', + 'check_net_address', 'get_free_port', 'unix_socket_path', 'bind_socket', 'bind_unix_socket', 'tcp_socketpair', 'unix_socketpair', 'create_sockets', # compat @@ -175,6 +178,7 @@ except Exception: HAS_SENSORS_FANS = hasattr(psutil, "sensors_fans") HAS_SENSORS_TEMPERATURES = hasattr(psutil, "sensors_temperatures") HAS_THREADS = hasattr(psutil.Process, "threads") +SKIP_SYSCONS = (MACOS or AIX) and os.getuid() != 0 # --- misc @@ -212,7 +216,6 @@ DEVNULL = open(os.devnull, 'r+') VALID_PROC_STATUSES = [getattr(psutil, x) for x in dir(psutil) if x.startswith('STATUS_')] AF_UNIX = getattr(socket, "AF_UNIX", object()) -SOCK_SEQPACKET = getattr(socket, "SOCK_SEQPACKET", object()) _subprocesses_started = set() _pids_started = set() @@ -403,7 +406,7 @@ def create_zombie_proc(): with contextlib.closing(socket.socket(socket.AF_UNIX)) as sock: sock.settimeout(GLOBAL_TIMEOUT) sock.bind(unix_file) - sock.listen(1) + sock.listen(5) pyrun(src) conn, _ = sock.accept() try: @@ -514,9 +517,8 @@ def reap_children(recursive=False): # Wait for the process to terminate, to avoid zombies. try: subp.wait() - except OSError as err: - if err.errno != errno.ECHILD: - raise + except ChildProcessError: + pass # Terminate started pids. while _pids_started: @@ -710,13 +712,12 @@ def safe_rmpath(path): while time.time() < stop_at: try: return fun() + except FileNotFoundError: + pass except WindowsError as _: err = _ - if err.errno != errno.ENOENT: - raise - else: - warn("ignoring %s" % (str(err))) - time.sleep(0.01) + warn("ignoring %s" % (str(err))) + time.sleep(0.01) raise err try: @@ -729,18 +730,16 @@ def safe_rmpath(path): fun() else: retry_fun(fun) - except OSError as err: - if err.errno != errno.ENOENT: - raise + except FileNotFoundError: + pass def safe_mkdir(dir): "Convenience function for creating a directory" try: os.mkdir(dir) - except OSError as err: - if err.errno != errno.EEXIST: - raise + except FileExistsError: + pass @contextlib.contextmanager @@ -868,7 +867,6 @@ def skip_on_not_implemented(only_if=None): def get_free_port(host='127.0.0.1'): """Return an unused TCP port.""" with contextlib.closing(socket.socket()) as sock: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.bind((host, 0)) return sock.getsockname()[1] @@ -895,10 +893,11 @@ def bind_socket(family=AF_INET, type=SOCK_STREAM, addr=None): addr = ("", 0) sock = socket.socket(family, type) try: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if os.name not in ('nt', 'cygwin'): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.bind(addr) if type == socket.SOCK_STREAM: - sock.listen(10) + sock.listen(5) return sock except Exception: sock.close() @@ -913,7 +912,7 @@ def bind_unix_socket(name, type=socket.SOCK_STREAM): try: sock.bind(name) if type == socket.SOCK_STREAM: - sock.listen(10) + sock.listen(5) except Exception: sock.close() raise @@ -926,7 +925,7 @@ def tcp_socketpair(family, addr=("", 0)): """ with contextlib.closing(socket.socket(family, SOCK_STREAM)) as ll: ll.bind(addr) - ll.listen(10) + ll.listen(5) addr = ll.getsockname() c = socket.socket(family, SOCK_STREAM) try: @@ -1022,77 +1021,6 @@ def check_net_address(addr, family): raise ValueError("unknown family %r", family) -def check_connection_ntuple(conn): - """Check validity of a connection namedtuple.""" - # check ntuple - assert len(conn) in (6, 7), conn - has_pid = len(conn) == 7 - has_fd = getattr(conn, 'fd', -1) != -1 - assert conn[0] == conn.fd - assert conn[1] == conn.family - assert conn[2] == conn.type - assert conn[3] == conn.laddr - assert conn[4] == conn.raddr - assert conn[5] == conn.status - if has_pid: - assert conn[6] == conn.pid - - # check fd - if has_fd: - assert conn.fd >= 0, conn - if hasattr(socket, 'fromfd') and not WINDOWS: - try: - dupsock = socket.fromfd(conn.fd, conn.family, conn.type) - except (socket.error, OSError) as err: - if err.args[0] != errno.EBADF: - raise - else: - with contextlib.closing(dupsock): - assert dupsock.family == conn.family - assert dupsock.type == conn.type - - # check family - assert conn.family in (AF_INET, AF_INET6, AF_UNIX), repr(conn.family) - if conn.family in (AF_INET, AF_INET6): - # actually try to bind the local socket; ignore IPv6 - # sockets as their address might be represented as - # an IPv4-mapped-address (e.g. "::127.0.0.1") - # and that's rejected by bind() - if conn.family == AF_INET: - s = socket.socket(conn.family, conn.type) - with contextlib.closing(s): - try: - s.bind((conn.laddr[0], 0)) - except socket.error as err: - if err.errno != errno.EADDRNOTAVAIL: - raise - elif conn.family == AF_UNIX: - assert conn.status == psutil.CONN_NONE, conn.status - - # check type (SOCK_SEQPACKET may happen in case of AF_UNIX socks) - assert conn.type in (SOCK_STREAM, SOCK_DGRAM, SOCK_SEQPACKET), \ - repr(conn.type) - if conn.type == SOCK_DGRAM: - assert conn.status == psutil.CONN_NONE, conn.status - - # check laddr (IP address and port sanity) - for addr in (conn.laddr, conn.raddr): - if conn.family in (AF_INET, AF_INET6): - assert isinstance(addr, tuple), addr - if not addr: - continue - assert isinstance(addr.port, int), addr.port - assert 0 <= addr.port <= 65535, addr.port - check_net_address(addr.ip, conn.family) - elif conn.family == AF_UNIX: - assert isinstance(addr, str), addr - - # check status - assert isinstance(conn.status, str), conn - valids = [getattr(psutil, x) for x in dir(psutil) if x.startswith('CONN_')] - assert conn.status in valids, conn - - # =================================================================== # --- compatibility # =================================================================== diff --git a/psutil/tests/test_bsd.py b/psutil/tests/test_bsd.py index 5df8ad29..7cba4b78 100755 --- a/psutil/tests/test_bsd.py +++ b/psutil/tests/test_bsd.py @@ -72,7 +72,7 @@ def muse(field): @unittest.skipIf(not BSD, "BSD only") -class BSDSpecificTestCase(unittest.TestCase): +class BSDTestCase(unittest.TestCase): """Generic tests common to all BSD variants.""" @classmethod @@ -148,7 +148,7 @@ class BSDSpecificTestCase(unittest.TestCase): @unittest.skipIf(not FREEBSD, "FREEBSD only") -class FreeBSDSpecificTestCase(unittest.TestCase): +class FreeBSDProcessTestCase(unittest.TestCase): @classmethod def setUpClass(cls): @@ -158,21 +158,8 @@ class FreeBSDSpecificTestCase(unittest.TestCase): def tearDownClass(cls): reap_children() - @staticmethod - def parse_swapinfo(): - # the last line is always the total - output = sh("swapinfo -k").splitlines()[-1] - parts = re.split(r'\s+', output) - - if not parts: - raise ValueError("Can't parse swapinfo: %s" % output) - - # the size is in 1k units, so multiply by 1024 - total, used, free = (int(p) * 1024 for p in parts[1:4]) - return total, used, free - @retry_on_failure() - def test_proc_memory_maps(self): + def test_memory_maps(self): out = sh('procstat -v %s' % self.pid) maps = psutil.Process(self.pid).memory_maps(grouped=False) lines = out.split('\n')[1:] @@ -186,17 +173,17 @@ class FreeBSDSpecificTestCase(unittest.TestCase): if not map.path.startswith('['): self.assertEqual(fields[10], map.path) - def test_proc_exe(self): + def test_exe(self): out = sh('procstat -b %s' % self.pid) self.assertEqual(psutil.Process(self.pid).exe(), out.split('\n')[1].split()[-1]) - def test_proc_cmdline(self): + def test_cmdline(self): out = sh('procstat -c %s' % self.pid) self.assertEqual(' '.join(psutil.Process(self.pid).cmdline()), ' '.join(out.split('\n')[1].split()[2:])) - def test_proc_uids_gids(self): + def test_uids_gids(self): out = sh('procstat -s %s' % self.pid) euid, ruid, suid, egid, rgid, sgid = out.split('\n')[1].split()[2:8] p = psutil.Process(self.pid) @@ -210,7 +197,7 @@ class FreeBSDSpecificTestCase(unittest.TestCase): self.assertEqual(gids.saved, int(sgid)) @retry_on_failure() - def test_proc_ctx_switches(self): + def test_ctx_switches(self): tested = [] out = sh('procstat -r %s' % self.pid) p = psutil.Process(self.pid) @@ -230,7 +217,7 @@ class FreeBSDSpecificTestCase(unittest.TestCase): raise RuntimeError("couldn't find lines match in procstat out") @retry_on_failure() - def test_proc_cpu_times(self): + def test_cpu_times(self): tested = [] out = sh('procstat -r %s' % self.pid) p = psutil.Process(self.pid) @@ -249,11 +236,31 @@ class FreeBSDSpecificTestCase(unittest.TestCase): if len(tested) != 2: raise RuntimeError("couldn't find lines match in procstat out") + +@unittest.skipIf(not FREEBSD, "FREEBSD only") +class FreeBSDSystemTestCase(unittest.TestCase): + + @staticmethod + def parse_swapinfo(): + # the last line is always the total + output = sh("swapinfo -k").splitlines()[-1] + parts = re.split(r'\s+', output) + + if not parts: + raise ValueError("Can't parse swapinfo: %s" % output) + + # the size is in 1k units, so multiply by 1024 + total, used, free = (int(p) * 1024 for p in parts[1:4]) + return total, used, free + def test_cpu_frequency_against_sysctl(self): # Currently only cpu 0 is frequency is supported in FreeBSD # All other cores use the same frequency. sensor = "dev.cpu.0.freq" - sysctl_result = int(sysctl(sensor)) + try: + sysctl_result = int(sysctl(sensor)) + except RuntimeError: + self.skipTest("frequencies not supported by kernel") self.assertEqual(psutil.cpu_freq().current, sysctl_result) sensor = "dev.cpu.0.freq_levels" @@ -366,8 +373,9 @@ class FreeBSDSpecificTestCase(unittest.TestCase): sysctl('vm.stats.sys.v_soft'), delta=1000) def test_cpu_stats_syscalls(self): + # pretty high tolerance but it looks like it's OK. self.assertAlmostEqual(psutil.cpu_stats().syscalls, - sysctl('vm.stats.sys.v_syscall'), delta=1000) + sysctl('vm.stats.sys.v_syscall'), delta=100000) # def test_cpu_stats_traps(self): # self.assertAlmostEqual(psutil.cpu_stats().traps, @@ -450,7 +458,10 @@ class FreeBSDSpecificTestCase(unittest.TestCase): for cpu in range(num_cpus): sensor = "dev.cpu.%s.temperature" % cpu # sysctl returns a string in the format 46.0C - sysctl_result = int(float(sysctl(sensor)[:-1])) + try: + sysctl_result = int(float(sysctl(sensor)[:-1])) + except RuntimeError: + self.skipTest("temperatures not supported by kernel") self.assertAlmostEqual( psutil.sensors_temperatures()["coretemp"][cpu].current, sysctl_result, delta=10) @@ -467,7 +478,7 @@ class FreeBSDSpecificTestCase(unittest.TestCase): @unittest.skipIf(not OPENBSD, "OPENBSD only") -class OpenBSDSpecificTestCase(unittest.TestCase): +class OpenBSDTestCase(unittest.TestCase): def test_boot_time(self): s = sysctl('kern.boottime') @@ -482,11 +493,11 @@ class OpenBSDSpecificTestCase(unittest.TestCase): @unittest.skipIf(not NETBSD, "NETBSD only") -class NetBSDSpecificTestCase(unittest.TestCase): +class NetBSDTestCase(unittest.TestCase): @staticmethod def parse_meminfo(look_for): - with open('/proc/meminfo', 'rb') as f: + with open('/proc/meminfo', 'rt') as f: for line in f: if line.startswith(look_for): return int(line.split()[1]) * 1024 diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 68eea784..5fda78cb 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -6,6 +6,8 @@ """Tests for net_connections() and Process.connections() APIs.""" +import contextlib +import errno import os import socket import textwrap @@ -29,14 +31,16 @@ from psutil._compat import PY3 from psutil.tests import AF_UNIX from psutil.tests import bind_socket from psutil.tests import bind_unix_socket -from psutil.tests import check_connection_ntuple +from psutil.tests import check_net_address from psutil.tests import create_sockets +from psutil.tests import enum from psutil.tests import get_free_port from psutil.tests import HAS_CONNECTIONS_UNIX from psutil.tests import pyrun from psutil.tests import reap_children from psutil.tests import safe_rmpath from psutil.tests import skip_on_access_denied +from psutil.tests import SKIP_SYSCONS from psutil.tests import tcp_socketpair from psutil.tests import TESTFN from psutil.tests import TRAVIS @@ -47,6 +51,7 @@ from psutil.tests import wait_for_file thisproc = psutil.Process() +SOCK_SEQPACKET = getattr(socket, "SOCK_SEQPACKET", object()) class Base(object): @@ -67,6 +72,122 @@ class Base(object): cons = thisproc.connections(kind='all') assert not cons, cons + def compare_procsys_connections(self, pid, proc_cons, kind='all'): + """Given a process PID and its list of connections compare + those against system-wide connections retrieved via + psutil.net_connections. + """ + try: + sys_cons = psutil.net_connections(kind=kind) + except psutil.AccessDenied: + # On MACOS, system-wide connections are retrieved by iterating + # over all processes + if MACOS: + return + else: + raise + # Filter for this proc PID and exlucde PIDs from the tuple. + sys_cons = [c[:-1] for c in sys_cons if c.pid == pid] + sys_cons.sort() + proc_cons.sort() + self.assertEqual(proc_cons, sys_cons) + + def check_connection_ntuple(self, conn): + """Check validity of a connection namedtuple.""" + def check_ntuple(conn): + has_pid = len(conn) == 7 + self.assertIn(len(conn), (6, 7)) + self.assertEqual(conn[0], conn.fd) + self.assertEqual(conn[1], conn.family) + self.assertEqual(conn[2], conn.type) + self.assertEqual(conn[3], conn.laddr) + self.assertEqual(conn[4], conn.raddr) + self.assertEqual(conn[5], conn.status) + if has_pid: + self.assertEqual(conn[6], conn.pid) + + def check_family(conn): + self.assertIn(conn.family, (AF_INET, AF_INET6, AF_UNIX)) + if enum is not None: + assert isinstance(conn.family, enum.IntEnum), conn + else: + assert isinstance(conn.family, int), conn + if conn.family == AF_INET: + # actually try to bind the local socket; ignore IPv6 + # sockets as their address might be represented as + # an IPv4-mapped-address (e.g. "::127.0.0.1") + # and that's rejected by bind() + s = socket.socket(conn.family, conn.type) + with contextlib.closing(s): + try: + s.bind((conn.laddr[0], 0)) + except socket.error as err: + if err.errno != errno.EADDRNOTAVAIL: + raise + elif conn.family == AF_UNIX: + self.assertEqual(conn.status, psutil.CONN_NONE) + + def check_type(conn): + # SOCK_SEQPACKET may happen in case of AF_UNIX socks + self.assertIn(conn.type, (SOCK_STREAM, SOCK_DGRAM, SOCK_SEQPACKET)) + if enum is not None: + assert isinstance(conn.type, enum.IntEnum), conn + else: + assert isinstance(conn.type, int), conn + if conn.type == SOCK_DGRAM: + self.assertEqual(conn.status, psutil.CONN_NONE) + + def check_addrs(conn): + # check IP address and port sanity + for addr in (conn.laddr, conn.raddr): + if conn.family in (AF_INET, AF_INET6): + self.assertIsInstance(addr, tuple) + if not addr: + continue + self.assertIsInstance(addr.port, int) + assert 0 <= addr.port <= 65535, addr.port + check_net_address(addr.ip, conn.family) + elif conn.family == AF_UNIX: + self.assertIsInstance(addr, str) + + def check_status(conn): + self.assertIsInstance(conn.status, str) + valids = [getattr(psutil, x) for x in dir(psutil) + if x.startswith('CONN_')] + self.assertIn(conn.status, valids) + if conn.family in (AF_INET, AF_INET6) and conn.type == SOCK_STREAM: + self.assertNotEqual(conn.status, psutil.CONN_NONE) + else: + self.assertEqual(conn.status, psutil.CONN_NONE) + + check_ntuple(conn) + check_family(conn) + check_type(conn) + check_addrs(conn) + check_status(conn) + + +class TestBase(Base, unittest.TestCase): + + @unittest.skipIf(SKIP_SYSCONS, "requires root") + def test_system(self): + with create_sockets(): + for conn in psutil.net_connections(kind='all'): + self.check_connection_ntuple(conn) + + def test_process(self): + with create_sockets(): + for conn in psutil.Process().connections(kind='all'): + self.check_connection_ntuple(conn) + + def test_invalid_kind(self): + self.assertRaises(ValueError, thisproc.connections, kind='???') + self.assertRaises(ValueError, psutil.net_connections, kind='???') + + +class TestUnconnectedSockets(Base, unittest.TestCase): + """Tests sockets which are open but not connected to anything.""" + def get_conn_from_sock(self, sock): cons = thisproc.connections(kind='all') smap = dict([(c.fd, c) for c in cons]) @@ -80,14 +201,13 @@ class Base(object): self.assertEqual(smap[sock.fileno()].fd, sock.fileno()) return cons[0] - def check_socket(self, sock, conn=None): + def check_socket(self, sock): """Given a socket, makes sure it matches the one obtained via psutil. It assumes this process created one connection only (the one supposed to be checked). """ - if conn is None: - conn = self.get_conn_from_sock(sock) - check_connection_ntuple(conn) + conn = self.get_conn_from_sock(sock) + self.check_connection_ntuple(conn) # fd, family, type if conn.fd != -1: @@ -113,38 +233,9 @@ class Base(object): # XXX Solaris can't retrieve system-wide UNIX sockets if sock.family == AF_UNIX and HAS_CONNECTIONS_UNIX: cons = thisproc.connections(kind='all') - self.compare_procsys_connections(os.getpid(), cons) + self.compare_procsys_connections(os.getpid(), cons, kind='all') return conn - def compare_procsys_connections(self, pid, proc_cons, kind='all'): - """Given a process PID and its list of connections compare - those against system-wide connections retrieved via - psutil.net_connections. - """ - try: - sys_cons = psutil.net_connections(kind=kind) - except psutil.AccessDenied: - # On MACOS, system-wide connections are retrieved by iterating - # over all processes - if MACOS: - return - else: - raise - # Filter for this proc PID and exlucde PIDs from the tuple. - sys_cons = [c[:-1] for c in sys_cons if c.pid == pid] - sys_cons.sort() - proc_cons.sort() - self.assertEqual(proc_cons, sys_cons) - - -# ===================================================================== -# --- Test unconnected sockets -# ===================================================================== - - -class TestUnconnectedSockets(Base, unittest.TestCase): - """Tests sockets which are open but not connected to anything.""" - def test_tcp_v4(self): addr = ("127.0.0.1", get_free_port()) with closing(bind_socket(AF_INET, SOCK_STREAM, addr=addr)) as sock: @@ -192,12 +283,7 @@ class TestUnconnectedSockets(Base, unittest.TestCase): self.assertEqual(conn.status, psutil.CONN_NONE) -# ===================================================================== -# --- Test connected sockets -# ===================================================================== - - -class TestConnectedSocketPairs(Base, unittest.TestCase): +class TestConnectedSocket(Base, unittest.TestCase): """Test socket pairs which are are actually connected to each other. """ @@ -257,12 +343,58 @@ class TestConnectedSocketPairs(Base, unittest.TestCase): server.close() client.close() + +class TestFilters(Base, unittest.TestCase): + + def test_filters(self): + def check(kind, families, types): + for conn in thisproc.connections(kind=kind): + self.assertIn(conn.family, families) + self.assertIn(conn.type, types) + if not SKIP_SYSCONS: + for conn in psutil.net_connections(kind=kind): + self.assertIn(conn.family, families) + self.assertIn(conn.type, types) + + with create_sockets(): + check('all', + [AF_INET, AF_INET6, AF_UNIX], + [SOCK_STREAM, SOCK_DGRAM, SOCK_SEQPACKET]) + check('inet', + [AF_INET, AF_INET6], + [SOCK_STREAM, SOCK_DGRAM]) + check('inet4', + [AF_INET], + [SOCK_STREAM, SOCK_DGRAM]) + check('tcp', + [AF_INET, AF_INET6], + [SOCK_STREAM]) + check('tcp4', + [AF_INET], + [SOCK_STREAM]) + check('tcp6', + [AF_INET6], + [SOCK_STREAM]) + check('udp', + [AF_INET, AF_INET6], + [SOCK_DGRAM]) + check('udp4', + [AF_INET], + [SOCK_DGRAM]) + check('udp6', + [AF_INET6], + [SOCK_DGRAM]) + if HAS_CONNECTIONS_UNIX: + check('unix', + [AF_UNIX], + [SOCK_STREAM, SOCK_DGRAM, SOCK_SEQPACKET]) + @skip_on_access_denied(only_if=MACOS) def test_combos(self): def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): all_kinds = ("all", "inet", "inet4", "inet6", "tcp", "tcp4", "tcp6", "udp", "udp4", "udp6") - check_connection_ntuple(conn) + self.check_connection_ntuple(conn) self.assertEqual(conn.family, family) self.assertEqual(conn.type, type) self.assertEqual(conn.laddr, laddr) @@ -284,7 +416,7 @@ class TestConnectedSocketPairs(Base, unittest.TestCase): import socket, time s = socket.socket($family, socket.SOCK_STREAM) s.bind(('$addr', 0)) - s.listen(1) + s.listen(5) with open('$testfn', 'w') as f: f.write(str(s.getsockname()[:2])) time.sleep(60) @@ -352,13 +484,8 @@ class TestConnectedSocketPairs(Base, unittest.TestCase): psutil.CONN_NONE, ("all", "inet", "inet6", "udp", "udp6")) - # err - self.assertRaises(ValueError, p.connections, kind='???') - - def test_multi_sockets_filtering(self): - with create_sockets() as socks: - cons = thisproc.connections(kind='all') - self.assertEqual(len(cons), len(socks)) + def test_count(self): + with create_sockets(): # tcp cons = thisproc.connections(kind='tcp') self.assertEqual(len(cons), 2 if supports_ipv6() else 1) @@ -406,8 +533,9 @@ class TestConnectedSocketPairs(Base, unittest.TestCase): for conn in cons: self.assertEqual(conn.family, AF_INET6) self.assertIn(conn.type, (SOCK_STREAM, SOCK_DGRAM)) - # unix - if HAS_CONNECTIONS_UNIX: + # unix (skipped on NetBSD because by default the Python process + # creates a connection to '/var/run/log' UNIX socket) + if HAS_CONNECTIONS_UNIX and not NETBSD: cons = thisproc.connections(kind='unix') self.assertEqual(len(cons), 3) for conn in cons: @@ -415,23 +543,17 @@ class TestConnectedSocketPairs(Base, unittest.TestCase): self.assertIn(conn.type, (SOCK_STREAM, SOCK_DGRAM)) -# ===================================================================== -# --- Miscellaneous tests -# ===================================================================== - - +@unittest.skipIf(SKIP_SYSCONS, "requires root") class TestSystemWideConnections(Base, unittest.TestCase): """Tests for net_connections().""" - @skip_on_access_denied() def test_it(self): def check(cons, families, types_): - AF_UNIX = getattr(socket, 'AF_UNIX', object()) for conn in cons: self.assertIn(conn.family, families, msg=conn) if conn.family != AF_UNIX: self.assertIn(conn.type, types_, msg=conn) - check_connection_ntuple(conn) + self.check_connection_ntuple(conn) with create_sockets(): from psutil._common import conn_tmap @@ -444,16 +566,6 @@ class TestSystemWideConnections(Base, unittest.TestCase): self.assertEqual(len(cons), len(set(cons))) check(cons, families, types_) - self.assertRaises(ValueError, psutil.net_connections, kind='???') - - @skip_on_access_denied() - def test_multi_socks(self): - with create_sockets() as socks: - cons = [x for x in psutil.net_connections(kind='all') - if x.pid == os.getpid()] - self.assertEqual(len(cons), len(socks)) - - @skip_on_access_denied() # See: https://travis-ci.org/giampaolo/psutil/jobs/237566297 @unittest.skipIf(MACOS and TRAVIS, "unreliable on MACOS + TRAVIS") def test_multi_sockets_procs(self): @@ -495,11 +607,6 @@ class TestSystemWideConnections(Base, unittest.TestCase): self.assertEqual(len(p.connections('all')), expected) -# ===================================================================== -# --- Miscellaneous tests -# ===================================================================== - - class TestMisc(unittest.TestCase): def test_connection_constants(self): diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index 20da8241..cb4a2b96 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -15,7 +15,6 @@ import stat import time import traceback import warnings -from contextlib import closing from psutil import AIX from psutil import BSD @@ -29,20 +28,17 @@ from psutil import POSIX from psutil import SUNOS from psutil import WINDOWS from psutil._compat import long -from psutil.tests import bind_unix_socket -from psutil.tests import check_connection_ntuple +from psutil.tests import create_sockets from psutil.tests import get_kernel_version -from psutil.tests import HAS_CONNECTIONS_UNIX from psutil.tests import HAS_NET_IO_COUNTERS from psutil.tests import HAS_RLIMIT from psutil.tests import HAS_SENSORS_FANS from psutil.tests import HAS_SENSORS_TEMPERATURES from psutil.tests import is_namedtuple from psutil.tests import safe_rmpath -from psutil.tests import skip_on_access_denied +from psutil.tests import SKIP_SYSCONS from psutil.tests import TESTFN from psutil.tests import unittest -from psutil.tests import unix_socket_path from psutil.tests import VALID_PROC_STATUSES from psutil.tests import warn import psutil @@ -226,16 +222,13 @@ class TestSystem(unittest.TestCase): self.assertIsInstance(disk.fstype, str) self.assertIsInstance(disk.opts, str) - @unittest.skipIf(not POSIX, 'POSIX only') - @unittest.skipIf(not HAS_CONNECTIONS_UNIX, "can't list UNIX sockets") - @skip_on_access_denied(only_if=MACOS) + @unittest.skipIf(SKIP_SYSCONS, "requires root") def test_net_connections(self): - with unix_socket_path() as name: - with closing(bind_unix_socket(name)): - cons = psutil.net_connections(kind='unix') - assert cons - for conn in cons: - self.assertIsInstance(conn.laddr, str) + with create_sockets(): + ret = psutil.net_connections('all') + self.assertEqual(len(ret), len(set(ret))) + for conn in ret: + assert is_namedtuple(conn) def test_net_if_addrs(self): # Duplicate of test_system.py. Keep it anyway. @@ -560,9 +553,10 @@ class TestFetchAllProcesses(unittest.TestCase): self.assertGreaterEqual(ret, 0) def connections(self, ret, proc): - self.assertEqual(len(ret), len(set(ret))) - for conn in ret: - check_connection_ntuple(conn) + with create_sockets(): + self.assertEqual(len(ret), len(set(ret))) + for conn in ret: + assert is_namedtuple(conn) def cwd(self, ret, proc): if ret: # 'ret' can be None or empty diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index d732e90d..09fed4e4 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -10,6 +10,7 @@ from __future__ import division import collections import contextlib import errno +import glob import io import os import re @@ -24,6 +25,7 @@ import warnings import psutil from psutil import LINUX from psutil._compat import basestring +from psutil._compat import FileNotFoundError from psutil._compat import PY3 from psutil._compat import u from psutil.tests import call_until @@ -54,7 +56,7 @@ SIOCGIFCONF = 0x8912 SIOCGIFHWADDR = 0x8927 if LINUX: SECTOR_SIZE = 512 - +EMPTY_TEMPERATURES = not glob.glob('/sys/class/hwmon/hwmon*') # ===================================================================== # --- utils @@ -704,15 +706,12 @@ class TestSystemCPUFrequency(unittest.TestCase): if path.startswith("/sys/devices/system/cpu/cpufreq/policy"): return False else: - flags.append(None) return orig_exists(path) - flags = [] orig_exists = os.path.exists with mock.patch("os.path.exists", side_effect=path_exists_mock, create=True): assert psutil.cpu_freq() - self.assertEqual(len(flags), psutil.cpu_count(logical=True)) @unittest.skipIf(not HAS_CPU_FREQ, "not supported") def test_emulate_use_cpuinfo(self): @@ -843,25 +842,6 @@ class TestSystemCPUFrequency(unittest.TestCase): freq = psutil.cpu_freq() self.assertEqual(freq.current, 200) - # Also test that NotImplementedError is raised in case no - # current freq file is present. - - def open_mock(name, *args, **kwargs): - if name.endswith('/scaling_cur_freq'): - raise IOError(errno.ENOENT, "") - elif name.endswith('/cpuinfo_cur_freq'): - raise IOError(errno.ENOENT, "") - elif name == '/proc/cpuinfo': - raise IOError(errno.ENOENT, "") - else: - return orig_open(name, *args, **kwargs) - - orig_open = open - patch_point = 'builtins.open' if PY3 else '__builtin__.open' - with mock.patch(patch_point, side_effect=open_mock): - with mock.patch('os.path.exists', return_value=True): - self.assertRaises(NotImplementedError, psutil.cpu_freq) - @unittest.skipIf(not LINUX, "LINUX only") class TestSystemCPUStats(unittest.TestCase): @@ -1080,10 +1060,9 @@ class TestSystemDiskPartitions(unittest.TestCase): try: with mock.patch('os.path.realpath', return_value='/non/existent') as m: - with self.assertRaises(OSError) as cm: + with self.assertRaises(FileNotFoundError): psutil.disk_partitions() assert m.called - self.assertEqual(cm.exception.errno, errno.ENOENT) finally: psutil.PROCFS_PATH = "/proc" @@ -1569,6 +1548,7 @@ class TestSensorsBattery(unittest.TestCase): class TestSensorsTemperatures(unittest.TestCase): @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") + @unittest.skipIf(LINUX and EMPTY_TEMPERATURES, "no temperatures") def test_emulate_eio_error(self): def open_mock(name, *args, **kwargs): if name.endswith("_input"): @@ -1920,9 +1900,8 @@ class TestProcess(unittest.TestCase): '/proc/%s/smaps' % os.getpid(), IOError(errno.ENOENT, "")) as m: p = psutil.Process() - with self.assertRaises(IOError) as err: + with self.assertRaises(FileNotFoundError): p.memory_maps() - self.assertEqual(err.exception.errno, errno.ENOENT) assert m.called @unittest.skipIf(not HAS_RLIMIT, "not supported") @@ -1994,6 +1973,9 @@ class TestProcess(unittest.TestCase): "0", # cnswap "0", # exit_signal "6", # processor + "0", # rt priority + "0", # policy + "7", # delayacct_blkio_ticks ] content = " ".join(args).encode() with mock_open_content('/proc/%s/stat' % os.getpid(), content): @@ -2008,6 +1990,7 @@ class TestProcess(unittest.TestCase): self.assertEqual(cpu.system, 3 / CLOCK_TICKS) self.assertEqual(cpu.children_user, 4 / CLOCK_TICKS) self.assertEqual(cpu.children_system, 5 / CLOCK_TICKS) + self.assertEqual(cpu.iowait, 7 / CLOCK_TICKS) self.assertEqual(p.cpu_num(), 6) def test_status_file_parsing(self): diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index dde50a57..ba75eef0 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -14,7 +14,6 @@ for some reason). """ from __future__ import print_function -import errno import functools import gc import os @@ -31,6 +30,7 @@ from psutil import POSIX from psutil import SUNOS from psutil import WINDOWS from psutil._common import bytes2human +from psutil._compat import ProcessLookupError from psutil._compat import xrange from psutil.tests import create_sockets from psutil.tests import get_test_subprocess @@ -423,9 +423,8 @@ class TestTerminatedProcessLeaks(TestProcessObjectLeaks): def call(): try: return cext.proc_info(self.proc.pid) - except OSError as err: - if err.errno != errno.ESRCH: - raise + except ProcessLookupError: + pass self.execute(call) @@ -467,6 +466,7 @@ class TestModuleFunctionsLeaks(TestMemLeak): def test_per_cpu_times(self): self.execute(psutil.cpu_times, percpu=True) + @skip_if_linux() def test_cpu_stats(self): self.execute(psutil.cpu_stats) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 29e1e41e..4e476871 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -20,6 +20,7 @@ import socket import stat from psutil import LINUX +from psutil import NETBSD from psutil import POSIX from psutil import WINDOWS from psutil._common import memoize @@ -179,7 +180,8 @@ class TestMisc(unittest.TestCase): for name in dir_psutil: if name in ('callable', 'error', 'namedtuple', 'tests', 'long', 'test', 'NUM_CPUS', 'BOOT_TIME', - 'TOTAL_PHYMEM'): + 'TOTAL_PHYMEM', 'PermissionError', + 'ProcessLookupError'): continue if not name.startswith('_'): try: @@ -765,11 +767,15 @@ class TestScripts(unittest.TestCase): @unittest.skipIf(not HAS_SENSORS_TEMPERATURES, "not supported") @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") def test_temperatures(self): + if not psutil.sensors_temperatures(): + self.skipTest("no temperatures") self.assert_stdout('temperatures.py') @unittest.skipIf(not HAS_SENSORS_FANS, "not supported") @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") def test_fans(self): + if not psutil.sensors_fans(): + self.skipTest("no fans") self.assert_stdout('fans.py') @unittest.skipIf(not HAS_SENSORS_BATTERY, "not supported") @@ -1011,6 +1017,7 @@ class TestNetUtils(unittest.TestCase): self.assertNotEqual(client.getsockname(), addr) @unittest.skipIf(not POSIX, "POSIX only") + @unittest.skipIf(NETBSD, "/var/run/log UNIX socket opened by default") def test_unix_socketpair(self): p = psutil.Process() num_fds = p.num_fds() diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 512a8437..24a29b5a 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -24,6 +24,7 @@ import psutil from psutil import AIX from psutil import BSD +from psutil import FREEBSD from psutil import LINUX from psutil import MACOS from psutil import NETBSD @@ -251,6 +252,8 @@ class TestProcess(unittest.TestCase): assert (times.user > 0.0) or (times.system > 0.0), times assert (times.children_user >= 0.0), times assert (times.children_system >= 0.0), times + if LINUX: + assert times.iowait >= 0.0, times # make sure returned values can be pretty printed with strftime for name in times._fields: time.strftime("%H:%M:%S", time.localtime(getattr(times, name))) @@ -937,6 +940,8 @@ class TestProcess(unittest.TestCase): self.addCleanup(p.cpu_affinity, initial) # All possible CPU set combinations. + if len(initial) > 12: + initial = initial[:12] # ...otherwise it will take forever combos = [] for l in range(0, len(initial) + 1): for subset in itertools.combinations(initial, l): @@ -1084,7 +1089,8 @@ class TestProcess(unittest.TestCase): self.assertEqual(p1.parents()[0], psutil.Process()) self.assertEqual(p2.parents()[0], p1) self.assertEqual(p2.parents()[1], psutil.Process()) - if POSIX: + if POSIX and not FREEBSD: + # On FreeBSD PID 1 has an older/smaller time than PID 0 (?) lowest_pid = psutil.pids()[0] self.assertEqual(p1.parents()[-1].pid, lowest_pid) self.assertEqual(p2.parents()[-1].pid, lowest_pid) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index eb9016f0..e04e120b 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -29,6 +29,7 @@ from psutil import OPENBSD from psutil import POSIX from psutil import SUNOS from psutil import WINDOWS +from psutil._compat import FileNotFoundError from psutil._compat import long from psutil.tests import APPVEYOR from psutil.tests import ASCII_FS @@ -363,17 +364,16 @@ class TestSystemAPIs(unittest.TestCase): # Simulate some work load then make sure time have increased # between calls. tot1 = psutil.cpu_times(percpu=True) - stop_at = time.time() + 0.1 + giveup_at = time.time() + 1 while True: - if time.time() >= stop_at: - break - tot2 = psutil.cpu_times(percpu=True) - for t1, t2 in zip(tot1, tot2): - t1, t2 = sum(t1), sum(t2) - difference = t2 - t1 - if difference >= 0.05: - return - self.fail() + if time.time() >= giveup_at: + return self.fail("timeout") + tot2 = psutil.cpu_times(percpu=True) + for t1, t2 in zip(tot1, tot2): + t1, t2 = psutil._cpu_busy_time(t1), psutil._cpu_busy_time(t2) + difference = t2 - t1 + if difference >= 0.05: + return def test_cpu_times_comparison(self): # Make sure the sum of all per cpu times is almost equal to @@ -468,9 +468,8 @@ class TestSystemAPIs(unittest.TestCase): # if path does not exist OSError ENOENT is expected across # all platforms fname = tempfile.mktemp() - with self.assertRaises(OSError) as exc: + with self.assertRaises(FileNotFoundError): psutil.disk_usage(fname) - self.assertEqual(exc.exception.errno, errno.ENOENT) def test_disk_usage_unicode(self): # See: https://github.com/giampaolo/psutil/issues/416 @@ -501,11 +500,8 @@ class TestSystemAPIs(unittest.TestCase): # we cannot make any assumption about this, see: # http://goo.gl/p9c43 disk.device - if SUNOS or TRAVIS: - # on solaris apparently mount points can also be files - assert os.path.exists(disk.mountpoint), disk - else: - assert os.path.isdir(disk.mountpoint), disk + # on modern systems mount points can also be files + assert os.path.exists(disk.mountpoint), disk assert disk.fstype, disk # all = True diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 5a998dd4..51b53f0e 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -21,6 +21,7 @@ import warnings import psutil from psutil import WINDOWS +from psutil._compat import FileNotFoundError from psutil.tests import APPVEYOR from psutil.tests import get_test_subprocess from psutil.tests import HAS_BATTERY @@ -161,12 +162,9 @@ class TestSystemAPIs(unittest.TestCase): break try: usage = psutil.disk_usage(ps_part.mountpoint) - except OSError as err: - if err.errno == errno.ENOENT: - # usually this is the floppy - break - else: - raise + except FileNotFoundError: + # usually this is the floppy + break self.assertEqual(usage.total, int(wmi_part.Size)) wmi_free = int(wmi_part.FreeSpace) self.assertEqual(usage.free, wmi_free) diff --git a/scripts/internal/bench_oneshot_2.py b/scripts/internal/bench_oneshot_2.py index a25d1806..becf930c 100644 --- a/scripts/internal/bench_oneshot_2.py +++ b/scripts/internal/bench_oneshot_2.py @@ -11,7 +11,7 @@ supposed to be more precise. import sys -import perf # requires "pip install perf" +import pyperf # requires "pip install pyperf" import psutil from bench_oneshot import names @@ -37,7 +37,7 @@ def add_cmdline_args(cmd, args): def main(): - runner = perf.Runner() + runner = pyperf.Runner() args = runner.parse_args() if not args.worker: diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index 3d108d81..73134818 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -160,7 +160,7 @@ def parse_c(fname): def parse_generic(fname): - with open(fname) as f: + with open(fname, 'rt', errors='ignore') as f: text = f.read() return find_urls(text) @@ -174,7 +174,7 @@ def get_urls(fname): elif fname.endswith('.c') or fname.endswith('.h'): return parse_c(fname) else: - with open(fname) as f: + with open(fname, 'rt', errors='ignore') as f: if f.readline().strip().startswith('#!/usr/bin/env python'): return parse_py(fname) return parse_generic(fname) diff --git a/scripts/internal/fix_flake8.py b/scripts/internal/fix_flake8.py new file mode 100644 index 00000000..5aa84db5 --- /dev/null +++ b/scripts/internal/fix_flake8.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Fix some flake8 errors by rewriting files for which flake8 emitted +an error/warning. Usage (from the root dir): + + $ python3 -m flake8 --exit-zero | python3 scripts/fix_flake8.py +""" + +import sys +import tempfile +import shutil +from collections import defaultdict +from collections import namedtuple +from pprint import pprint as pp # NOQA + + +ntentry = namedtuple('ntentry', 'msg, issue, lineno, pos, descr') + + +# ===================================================================== +# utils +# ===================================================================== + + +def enter_pdb(): + from pdb import set_trace as st # trick GIT commit hook + sys.stdin = open('/dev/tty') + st() + + +def read_lines(fname): + with open(fname, 'rt') as f: + return f.readlines() + + +def read_line(fname, lineno): + with open(fname, 'rt') as f: + for i, line in enumerate(f, 1): + if i == lineno: + return line + raise ValueError("lineno too big") + + +def write_file(fname, newlines): + with tempfile.NamedTemporaryFile('wt', delete=False) as f: + for line in newlines: + f.write(line) + shutil.move(f.name, fname) + + +# ===================================================================== +# handlers +# ===================================================================== + + +def handle_f401(fname, lineno): + """ This is 'module imported but not used' + Able to handle this case: + import os + + ...but not this: + from os import listdir + """ + line = read_line(fname, lineno).strip() + if line.startswith('import '): + return True + else: + mods = line.partition(' import ')[2] # everything after import + return ',' not in mods + + +# ===================================================================== +# converters +# ===================================================================== + + +def remove_lines(fname, entries): + """Check if we should remove lines, then do it. + Return the numner of lines removed. + """ + to_remove = [] + for entry in entries: + msg, issue, lineno, pos, descr = entry + # 'module imported but not used' + if issue == 'F401' and handle_f401(fname, lineno): + to_remove.append(lineno) + # 'blank line(s) at end of file' + elif issue == 'W391': + lines = read_lines(fname) + i = len(lines) - 1 + while lines[i] == '\n': + to_remove.append(i + 1) + i -= 1 + # 'too many blank lines' + elif issue == 'E303': + howmany = descr.replace('(', '').replace(')', '') + howmany = int(howmany[-1]) + for x in range(lineno - howmany, lineno): + to_remove.append(x) + + if to_remove: + newlines = [] + for i, line in enumerate(read_lines(fname), 1): + if i not in to_remove: + newlines.append(line) + print("removing line(s) from %s" % fname) + write_file(fname, newlines) + + return len(to_remove) + + +def add_lines(fname, entries): + """Check if we should remove lines, then do it. + Return the numner of lines removed. + """ + EXPECTED_BLANK_LINES = ( + 'E302', # 'expected 2 blank limes, found 1' + 'E305') # ìexpected 2 blank lines after class or fun definition' + to_add = {} + for entry in entries: + msg, issue, lineno, pos, descr = entry + if issue in EXPECTED_BLANK_LINES: + howmany = 2 if descr.endswith('0') else 1 + to_add[lineno] = howmany + + if to_add: + newlines = [] + for i, line in enumerate(read_lines(fname), 1): + if i in to_add: + newlines.append('\n' * to_add[i]) + newlines.append(line) + print("adding line(s) to %s" % fname) + write_file(fname, newlines) + + return len(to_add) + + +# ===================================================================== +# main +# ===================================================================== + + +def build_table(): + table = defaultdict(list) + for line in sys.stdin: + line = line.strip() + if not line: + break + fields = line.split(':') + fname, lineno, pos = fields[:3] + issue = fields[3].split()[0] + descr = fields[3].strip().partition(' ')[2] + lineno, pos = int(lineno), int(pos) + table[fname].append(ntentry(line, issue, lineno, pos, descr)) + return table + + +def main(): + table = build_table() + + # remove lines (unused imports) + removed = 0 + for fname, entries in table.items(): + removed += remove_lines(fname, entries) + if removed: + print("%s lines were removed from some file(s); please re-run" % + removed) + return + + # add lines (missing \n between functions/classes) + added = 0 + for fname, entries in table.items(): + added += add_lines(fname, entries) + if added: + print("%s lines were added from some file(s); please re-run" % + added) + return + + +main() diff --git a/scripts/internal/print_timeline.py b/scripts/internal/print_timeline.py index 4bfe76b3..9bf6e9d1 100644 --- a/scripts/internal/print_timeline.py +++ b/scripts/internal/print_timeline.py @@ -30,7 +30,7 @@ def get_tag_date(tag): def main(): releases = [] - out = sh("git tags") + out = sh("git tag") for line in out.split('\n'): tag = line.split(' ')[0] ver = tag.replace('release-', '') diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index 75b4c348..116809ca 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -39,8 +39,8 @@ DEPS = [ "flake8", "nose", "pdbpp", - "perf", "pip", + "pyperf", "pypiwin32==219" if sys.version_info[:2] <= (3, 4) else "pypiwin32", "pyreadline", "setuptools", @@ -10,6 +10,7 @@ import contextlib import io import os import platform +import shutil import sys import tempfile import warnings @@ -205,20 +206,19 @@ elif LINUX: suffix='.c', delete=False, mode="wt") as f: f.write("#include <linux/ethtool.h>") + output_dir = tempfile.mkdtemp() try: compiler = UnixCCompiler() with silenced_output('stderr'): with silenced_output('stdout'): - compiler.compile([f.name]) + compiler.compile([f.name], output_dir=output_dir) except CompileError: return ("PSUTIL_ETHTOOL_MISSING_TYPES", 1) else: return None finally: - try: - os.remove(f.name) - except OSError: - pass + os.remove(f.name) + shutil.rmtree(output_dir) macros.append(("PSUTIL_LINUX", 1)) ETHTOOL_MACRO = get_ethtool_macro() @@ -240,7 +240,7 @@ elif SUNOS: ], define_macros=macros, libraries=['kstat', 'nsl', 'socket']) -# AIX + elif AIX: macros.append(("PSUTIL_AIX", 1)) ext = Extension( @@ -320,10 +320,6 @@ def main(): 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Programming Language :: Python', |