Merge branch 'master' into virtualization

15 files changed, 262 insertions, 173 deletions

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index d8940cd0..f9c0a435 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -37,7 +37,7 @@ jobs:
         PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_DEBUG=1 python {project}/psutil/tests/runner.py &&
         PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_DEBUG=1 python {project}/psutil/tests/test_memleaks.py
       CIBW_TEST_EXTRAS: test
-      CIBW_BUILD: 'cp37-* cp38-* cp39-* cp310-*'
+      CIBW_BUILD: 'cp36-* cp37-* cp38-* cp39-* cp310-*'
       CIBW_SKIP: '*-musllinux_*'
 
     steps:
diff --git a/HISTORY.rst b/HISTORY.rst
index 6b16affa..b19c7142 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -1,5 +1,25 @@
 *Bug tracker at https://github.com/giampaolo/psutil/issues*
 
+5.9.1 (IN DEVELOPMENT)
+======================
+
+XXXX-XX-XX
+
+**Enhancements**
+
+- 2050_, [Linux]: increase ``read(2)`` buffer size from 1k to 32k when reading
+  ``/proc`` pseudo files line by line. This should help having more consistent
+  results.
+
+**Bug fixes**
+
+- 2048_: ``AttributeError`` is raised if ``psutil.Error`` class is raised
+  manually and passed through ``str``.
+- 2049_, [Linux]: `cpu_freq`_ erroneously returns ``curr`` value in GHz while
+  ``min`` and ``max`` are in MHz.
+- 2050_, [Linux]: `virtual_memory()`_ may raise ``ValueError`` if running in a
+  LCX container.
+
 5.9.0
 =====
 
diff --git a/README.rst b/README.rst
index 6ad07db2..e6f5cb2d 100644
--- a/README.rst
+++ b/README.rst
@@ -144,6 +144,7 @@ Supporters
       <a href="https://opencollective.com/alexey-vazhnov"><img height="40" width="40" title="Alexey Vazhnov" src="https://images.opencollective.com/alexey-vazhnov/daed334/avatar/40.png" /></a>
       <a href="https://github.com/indeedeng"><img height="40" width="40" title="indeedeng" src="https://avatars.githubusercontent.com/u/2905043?s=200&v=4" /></a>
       <a href="https://github.com/PySimpleGUI"><img height="40" width="40" title="PySimpleGUI" src="https://avatars.githubusercontent.com/u/46163555?v=4" /></a>
+      <a href="https://github.com/u93"><img height="40" width="40" title="Eugenio E Breijo" src="https://avatars.githubusercontent.com/u/16807302?v=4" /></a>
 
     </div>
     <sup><a href="https://github.com/sponsors/giampaolo">add your avatar</a></sup>
diff --git a/appveyor.yml b/appveyor.yml
index 485182d3..83e570ae 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -24,6 +24,10 @@ environment:
       PYTHON_VERSION: "2.7.x"
       PYTHON_ARCH: "32"
 
+    - PYTHON: "C:\\Python36"
+      PYTHON_VERSION: "3.6.x"
+      PYTHON_ARCH: "32"
+
     - PYTHON: "C:\\Python37"
       PYTHON_VERSION: "3.7.x"
       PYTHON_ARCH: "32"
@@ -48,6 +52,10 @@ environment:
       PYTHON_VERSION: "2.7.x"
       PYTHON_ARCH: "64"
 
+    - PYTHON: "C:\\Python36-x64"
+      PYTHON_VERSION: "3.6.x"
+      PYTHON_ARCH: "64"
+
     - PYTHON: "C:\\Python37-x64"
       PYTHON_VERSION: "3.7.x"
       PYTHON_ARCH: "64"
diff --git a/docs/index.rst b/docs/index.rst
index ef9d3c26..7c9e8ae3 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -88,6 +88,7 @@ Supporters
       <a href="https://opencollective.com/alexey-vazhnov"><img height="40" width="40" title="Alexey Vazhnov" src="https://images.opencollective.com/alexey-vazhnov/daed334/avatar/40.png" /></a>
       <a href="https://github.com/indeedeng"><img height="40" width="40" title="indeedeng" src="https://avatars.githubusercontent.com/u/2905043?s=200&v=4" /></a>
       <a href="https://github.com/PySimpleGUI"><img height="40" width="40" title="PySimpleGUI" src="https://avatars.githubusercontent.com/u/46163555?v=4" /></a>
+      <a href="https://github.com/u93"><img height="40" width="40" title="Eugenio E Breijo" src="https://avatars.githubusercontent.com/u/16807302?v=4" /></a>
     </div>
     <br />
     <sup><a href="https://github.com/sponsors/giampaolo">add your avatar</a></sup>
diff --git a/psutil/__init__.py b/psutil/__init__.py
index cd943de4..3d045776 100644
--- a/psutil/__init__.py
+++ b/psutil/__init__.py
@@ -236,7 +236,7 @@ if hasattr(_psplatform.Process, "rlimit"):
 AF_LINK = _psplatform.AF_LINK
 
 __author__ = "Giampaolo Rodola'"
-__version__ = "5.9.0"
+__version__ = "5.9.1"
 version_info = tuple([int(num) for num in __version__.split('.')])
 
 _timer = getattr(time, 'monotonic', time.time)
diff --git a/psutil/_common.py b/psutil/_common.py
index 16d3b3b5..1e33699c 100644
--- a/psutil/_common.py
+++ b/psutil/_common.py
@@ -44,6 +44,9 @@ else:
 # can't take it from _common.py as this script is imported by setup.py
 PY3 = sys.version_info[0] == 3
 PSUTIL_DEBUG = bool(os.getenv('PSUTIL_DEBUG', 0))
+if PSUTIL_DEBUG:
+    import inspect
+_DEFAULT = object()
 
 __all__ = [
     # OS constants
@@ -70,6 +73,7 @@ __all__ = [
     'conn_tmap', 'deprecated_method', 'isfile_strict', 'memoize',
     'parse_environ_block', 'path_exists_strict', 'usage_percent',
     'supports_ipv6', 'sockfam_to_enum', 'socktype_to_enum', "wrap_numbers",
+    'open_text', 'open_binary', 'cat', 'bcat',
     'bytes2human', 'conn_to_ntuple', 'debug',
     # shell utils
     'hilite', 'term_supports_colors', 'print_color',
@@ -297,7 +301,7 @@ class Error(Exception):
                 ["%s=%r" % (k, v) for k, v in info.items()])
         else:
             details = None
-        return " ".join([x for x in (self.msg, details) if x])
+        return " ".join([x for x in (getattr(self, "msg", ""), details) if x])
 
     def __repr__(self):
         # invoked on `repr(Error)`
@@ -707,22 +711,71 @@ wrap_numbers.cache_clear = _wn.cache_clear
 wrap_numbers.cache_info = _wn.cache_info
 
 
-def open_binary(fname, **kwargs):
-    return open(fname, "rb", **kwargs)
+# The read buffer size for open() builtin. This (also) dictates how
+# much data we read(2) when iterating over file lines as in:
+#   >>> with open(file) as f:
+#   ...    for line in f:
+#   ...        ...
+# Default per-line buffer size for binary files is 1K. For text files
+# is 8K. We use a bigger buffer (32K) in order to have more consistent
+# results when reading /proc pseudo files on Linux, see:
+# https://github.com/giampaolo/psutil/issues/2050
+# On Python 2 this also speeds up the reading of big files:
+# (namely /proc/{pid}/smaps and /proc/net/*):
+# https://github.com/giampaolo/psutil/issues/708
+FILE_READ_BUFFER_SIZE = 32 * 1024
 
 
-def open_text(fname, **kwargs):
+def open_binary(fname):
+    return open(fname, "rb", buffering=FILE_READ_BUFFER_SIZE)
+
+
+def open_text(fname):
     """On Python 3 opens a file in text mode by using fs encoding and
     a proper en/decoding errors handler.
     On Python 2 this is just an alias for open(name, 'rt').
     """
-    if PY3:
-        # See:
-        # https://github.com/giampaolo/psutil/issues/675
-        # https://github.com/giampaolo/psutil/pull/733
-        kwargs.setdefault('encoding', ENCODING)
-        kwargs.setdefault('errors', ENCODING_ERRS)
-    return open(fname, "rt", **kwargs)
+    if not PY3:
+        return open(fname, "rt", buffering=FILE_READ_BUFFER_SIZE)
+
+    # See:
+    # https://github.com/giampaolo/psutil/issues/675
+    # https://github.com/giampaolo/psutil/pull/733
+    fobj = open(fname, "rt", buffering=FILE_READ_BUFFER_SIZE,
+                encoding=ENCODING, errors=ENCODING_ERRS)
+    try:
+        # Dictates per-line read(2) buffer size. Defaults is 8k. See:
+        # https://github.com/giampaolo/psutil/issues/2050#issuecomment-1013387546
+        fobj._CHUNK_SIZE = FILE_READ_BUFFER_SIZE
+    except AttributeError:
+        pass
+    except Exception:
+        fobj.close()
+        raise
+
+    return fobj
+
+
+def cat(fname, fallback=_DEFAULT, _open=open_text):
+    """Read entire file content and return it as a string. File is
+    opened in text mode. If specified, `fallback` is the value
+    returned in case of error, either if the file does not exist or
+    it can't be read().
+    """
+    if fallback is _DEFAULT:
+        with _open(fname) as f:
+            return f.read()
+    else:
+        try:
+            with _open(fname) as f:
+                return f.read()
+        except (IOError, OSError):
+            return fallback
+
+
+def bcat(fname, fallback=_DEFAULT):
+    """Same as above but opens file in binary mode."""
+    return cat(fname, fallback=fallback, _open=open_binary)
 
 
 def bytes2human(n, format="%(value).1f%(symbol)s"):
@@ -835,7 +888,6 @@ def print_color(
 def debug(msg):
     """If PSUTIL_DEBUG env var is set, print a debug message to stderr."""
     if PSUTIL_DEBUG:
-        import inspect
         fname, lineno, func_name, lines, index = inspect.getframeinfo(
             inspect.currentframe().f_back)
         if isinstance(msg, Exception):
diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py
index ac0a64b6..118e928c 100644
--- a/psutil/_pslinux.py
+++ b/psutil/_pslinux.py
@@ -31,6 +31,8 @@ from ._common import NIC_DUPLEX_UNKNOWN
 from ._common import AccessDenied
 from ._common import NoSuchProcess
 from ._common import ZombieProcess
+from ._common import bcat
+from ._common import cat
 from ._common import debug
 from ._common import decode
 from ._common import get_procfs_path
@@ -91,16 +93,11 @@ POWER_SUPPLY_PATH = "/sys/class/power_supply"
 HAS_SMAPS = os.path.exists('/proc/%s/smaps' % os.getpid())
 HAS_PROC_IO_PRIORITY = hasattr(cext, "proc_ioprio_get")
 HAS_CPU_AFFINITY = hasattr(cext, "proc_cpu_affinity_get")
-_DEFAULT = object()
 
 # Number of clock ticks per second
 CLOCK_TICKS = os.sysconf("SC_CLK_TCK")
 PAGESIZE = cext_posix.getpagesize()
 BOOT_TIME = None  # set later
-# Used when reading "big" files, namely /proc/{pid}/smaps and /proc/net/*.
-# On Python 2, using a buffer with open() for such files may result in a
-# speedup, see: https://github.com/giampaolo/psutil/issues/708
-BIGFILE_BUFFERING = -1 if PY3 else 8192
 LITTLE_ENDIAN = sys.byteorder == 'little'
 
 # "man iostat" states that sectors are equivalent with blocks and have
@@ -337,22 +334,6 @@ def set_scputimes_ntuple(procfs_path):
     scputimes = namedtuple('scputimes', fields)
 
 
-def cat(fname, fallback=_DEFAULT, binary=True):
-    """Return file content.
-    fallback: the value returned in case the file does not exist or
-              cannot be read
-    binary: whether to open the file in binary or text mode.
-    """
-    try:
-        with open_binary(fname) if binary else open_text(fname) as f:
-            return f.read().strip()
-    except (IOError, OSError):
-        if fallback is not _DEFAULT:
-            return fallback
-        else:
-            raise
-
-
 try:
     set_scputimes_ntuple("/proc")
 except Exception:  # pragma: no cover
@@ -803,19 +784,19 @@ if os.path.exists("/sys/devices/system/cpu/cpufreq/policy0") or \
             if len(paths) == len(cpuinfo_freqs):
                 # take cached value from cpuinfo if available, see:
                 # https://github.com/giampaolo/psutil/issues/1851
-                curr = cpuinfo_freqs[i]
+                curr = cpuinfo_freqs[i] * 1000
             else:
-                curr = cat(pjoin(path, "scaling_cur_freq"), fallback=None)
+                curr = bcat(pjoin(path, "scaling_cur_freq"), fallback=None)
             if curr is None:
                 # Likely an old RedHat, see:
                 # https://github.com/giampaolo/psutil/issues/1071
-                curr = cat(pjoin(path, "cpuinfo_cur_freq"), fallback=None)
+                curr = bcat(pjoin(path, "cpuinfo_cur_freq"), fallback=None)
                 if curr is None:
                     raise NotImplementedError(
                         "can't find current frequency file")
             curr = int(curr) / 1000
-            max_ = int(cat(pjoin(path, "scaling_max_freq"))) / 1000
-            min_ = int(cat(pjoin(path, "scaling_min_freq"))) / 1000
+            max_ = int(bcat(pjoin(path, "scaling_max_freq"))) / 1000
+            min_ = int(bcat(pjoin(path, "scaling_min_freq"))) / 1000
             ret.append(_common.scpufreq(curr, min_, max_))
         return ret
 
@@ -973,7 +954,7 @@ class Connections:
         if file.endswith('6') and not os.path.exists(file):
             # IPv6 not supported
             return
-        with open_text(file, buffering=BIGFILE_BUFFERING) as f:
+        with open_text(file) as f:
             f.readline()  # skip the first line
             for lineno, line in enumerate(f, 1):
                 try:
@@ -1010,7 +991,7 @@ class Connections:
     @staticmethod
     def process_unix(file, family, inodes, filter_pid=None):
         """Parse /proc/net/unix files."""
-        with open_text(file, buffering=BIGFILE_BUFFERING) as f:
+        with open_text(file) as f:
             f.readline()  # skip the first line
             for line in f:
                 tokens = line.split()
@@ -1403,9 +1384,9 @@ def sensors_temperatures():
     for base in basenames:
         try:
             path = base + '_input'
-            current = float(cat(path)) / 1000.0
+            current = float(bcat(path)) / 1000.0
             path = os.path.join(os.path.dirname(base), 'name')
-            unit_name = cat(path, binary=False)
+            unit_name = cat(path).strip()
         except (IOError, OSError, ValueError):
             # A lot of things can go wrong here, so let's just skip the
             # whole entry. Sure thing is Linux's /sys/class/hwmon really
@@ -1417,9 +1398,9 @@ def sensors_temperatures():
             # https://github.com/giampaolo/psutil/issues/1323
             continue
 
-        high = cat(base + '_max', fallback=None)
-        critical = cat(base + '_crit', fallback=None)
-        label = cat(base + '_label', fallback='', binary=False)
+        high = bcat(base + '_max', fallback=None)
+        critical = bcat(base + '_crit', fallback=None)
+        label = cat(base + '_label', fallback='').strip()
 
         if high is not None:
             try:
@@ -1442,9 +1423,9 @@ def sensors_temperatures():
         for base in basenames:
             try:
                 path = os.path.join(base, 'temp')
-                current = float(cat(path)) / 1000.0
+                current = float(bcat(path)) / 1000.0
                 path = os.path.join(base, 'type')
-                unit_name = cat(path, binary=False)
+                unit_name = cat(path).strip()
             except (IOError, OSError, ValueError) as err:
                 debug(err)
                 continue
@@ -1456,13 +1437,13 @@ def sensors_temperatures():
             high = None
             for trip_point in trip_points:
                 path = os.path.join(base, trip_point + "_type")
-                trip_type = cat(path, fallback='', binary=False)
+                trip_type = cat(path, fallback='').strip()
                 if trip_type == 'critical':
-                    critical = cat(os.path.join(base, trip_point + "_temp"),
-                                   fallback=None)
+                    critical = bcat(os.path.join(base, trip_point + "_temp"),
+                                    fallback=None)
                 elif trip_type == 'high':
-                    high = cat(os.path.join(base, trip_point + "_temp"),
-                               fallback=None)
+                    high = bcat(os.path.join(base, trip_point + "_temp"),
+                                fallback=None)
 
                 if high is not None:
                     try:
@@ -1500,13 +1481,12 @@ def sensors_fans():
     basenames = sorted(set([x.split('_')[0] for x in basenames]))
     for base in basenames:
         try:
-            current = int(cat(base + '_input'))
+            current = int(bcat(base + '_input'))
         except (IOError, OSError) as err:
             debug(err)
             continue
-        unit_name = cat(os.path.join(os.path.dirname(base), 'name'),
-                        binary=False)
-        label = cat(base + '_label', fallback='', binary=False)
+        unit_name = cat(os.path.join(os.path.dirname(base), 'name')).strip()
+        label = cat(base + '_label', fallback='').strip()
         ret[unit_name].append(_common.sfan(label, current))
 
     return dict(ret)
@@ -1521,17 +1501,17 @@ def sensors_battery():
     """
     null = object()
 
-    def multi_cat(*paths):
+    def multi_bcat(*paths):
         """Attempt to read the content of multiple files which may
         not exist. If none of them exist return None.
         """
         for path in paths:
-            ret = cat(path, fallback=null)
+            ret = bcat(path, fallback=null)
             if ret != null:
                 try:
                     return int(ret)
                 except ValueError:
-                    return ret
+                    return ret.strip()
         return None
 
     bats = [x for x in os.listdir(POWER_SUPPLY_PATH) if x.startswith('BAT') or
@@ -1544,16 +1524,16 @@ def sensors_battery():
     root = os.path.join(POWER_SUPPLY_PATH, sorted(bats)[0])
 
     # Base metrics.
-    energy_now = multi_cat(
+    energy_now = multi_bcat(
         root + "/energy_now",
         root + "/charge_now")
-    power_now = multi_cat(
+    power_now = multi_bcat(
         root + "/power_now",
         root + "/current_now")
-    energy_full = multi_cat(
+    energy_full = multi_bcat(
         root + "/energy_full",
         root + "/charge_full")
-    time_to_empty = multi_cat(root + "/time_to_empty_now")
+    time_to_empty = multi_bcat(root + "/time_to_empty_now")
 
     # Percent. If we have energy_full the percentage will be more
     # accurate compared to reading /capacity file (float vs. int).
@@ -1571,13 +1551,13 @@ def sensors_battery():
     # Note: AC0 is not always available and sometimes (e.g. CentOS7)
     # it's called "AC".
     power_plugged = None
-    online = multi_cat(
+    online = multi_bcat(
         os.path.join(POWER_SUPPLY_PATH, "AC0/online"),
         os.path.join(POWER_SUPPLY_PATH, "AC/online"))
     if online is not None:
         power_plugged = online == 1
     else:
-        status = cat(root + "/status", fallback="", binary=False).lower()
+        status = cat(root + "/status", fallback="").strip().lower()
         if status == "discharging":
             power_plugged = False
         elif status in ("charging", "full"):
@@ -2007,8 +1987,7 @@ class Process(object):
         The return value is cached in case oneshot() ctx manager is
         in use.
         """
-        with open_binary("%s/%s/stat" % (self._procfs_path, self.pid)) as f:
-            data = f.read()
+        data = bcat("%s/%s/stat" % (self._procfs_path, self.pid))
         # Process name is between parentheses. It can contain spaces and
         # other parentheses. This is taken into account by looking for
         # the first occurrence of "(" and the last occurence of ")".
@@ -2044,8 +2023,7 @@ class Process(object):
     @wrap_exceptions
     @memoize_when_activated
     def _read_smaps_file(self):
-        with open_binary("%s/%s/smaps" % (self._procfs_path, self.pid),
-                         buffering=BIGFILE_BUFFERING) as f:
+        with open_binary("%s/%s/smaps" % (self._procfs_path, self.pid)) as f:
             return f.read().strip()
 
     def oneshot_enter(self):
diff --git a/psutil/tests/test_bsd.py b/psutil/tests/test_bsd.py
index 54b488bc..1ae810f1 100755
--- a/psutil/tests/test_bsd.py
+++ b/psutil/tests/test_bsd.py
@@ -117,9 +117,9 @@ class BSDTestCase(PsutilTestCase):
             self.assertEqual(usage.total, total)
             # 10 MB tollerance
             if abs(usage.free - free) > 10 * 1024 * 1024:
-                self.fail("psutil=%s, df=%s" % (usage.free, free))
+                raise self.fail("psutil=%s, df=%s" % (usage.free, free))
             if abs(usage.used - used) > 10 * 1024 * 1024:
-                self.fail("psutil=%s, df=%s" % (usage.used, used))
+                raise self.fail("psutil=%s, df=%s" % (usage.used, used))
 
     @unittest.skipIf(not which('sysctl'), "sysctl cmd not available")
     def test_cpu_count_logical(self):
diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py
index c25e9075..d1cf1736 100755
--- a/psutil/tests/test_linux.py
+++ b/psutil/tests/test_linux.py
@@ -1120,7 +1120,7 @@ class TestSystemDiskPartitions(PsutilTestCase):
                 if part.fstype == 'zfs':
                     break
             else:
-                self.fail("couldn't find any ZFS partition")
+                raise self.fail("couldn't find any ZFS partition")
         else:
             # No ZFS partitions on this system. Let's fake one.
             fake_file = io.StringIO(u("nodev\tzfs\n"))
@@ -1927,9 +1927,10 @@ class TestProcess(PsutilTestCase):
             patch_point = 'psutil._pslinux.os.readlink'
             with mock.patch(patch_point,
                             side_effect=OSError(errno.ENAMETOOLONG, "")) as m:
-                files = p.open_files()
-                assert not files
-                assert m.called
+                with mock.patch("psutil._common.debug"):
+                    files = p.open_files()
+                    assert not files
+                    assert m.called
 
     # --- mocked tests
 
@@ -2171,8 +2172,9 @@ class TestProcess(PsutilTestCase):
         with mock.patch('psutil._pslinux.os.readlink',
                         side_effect=OSError(errno.ENAMETOOLONG, "")) as m:
             p = psutil.Process()
-            assert not p.connections()
-            assert m.called
+            with mock.patch("psutil._common.debug"):
+                assert not p.connections()
+                assert m.called
 
 
 @unittest.skipIf(not LINUX, "LINUX only")
@@ -2463,15 +2465,6 @@ class TestUtils(PsutilTestCase):
             self.assertEqual(psutil._psplatform.readlink("bar"), "foo")
             assert m.called
 
-    def test_cat(self):
-        testfn = self.get_testfn()
-        with open(testfn, "wt") as f:
-            f.write("foo ")
-        self.assertEqual(psutil._psplatform.cat(testfn, binary=False), "foo")
-        self.assertEqual(psutil._psplatform.cat(testfn, binary=True), b"foo")
-        self.assertEqual(
-            psutil._psplatform.cat(testfn + '??', fallback="bar"), "bar")
-
 
 if __name__ == '__main__':
     from psutil.tests.runner import run_from_name
diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py
index d946eb62..aa30cbd6 100755
--- a/psutil/tests/test_misc.py
+++ b/psutil/tests/test_misc.py
@@ -23,12 +23,17 @@ import psutil.tests
 from psutil import LINUX
 from psutil import POSIX
 from psutil import WINDOWS
+from psutil._common import bcat
+from psutil._common import cat
 from psutil._common import debug
+from psutil._common import isfile_strict
 from psutil._common import memoize
 from psutil._common import memoize_when_activated
+from psutil._common import parse_environ_block
 from psutil._common import supports_ipv6
 from psutil._common import wrap_numbers
 from psutil._compat import PY3
+from psutil._compat import FileNotFoundError
 from psutil._compat import redirect_stderr
 from psutil.tests import APPVEYOR
 from psutil.tests import CI_TESTING
@@ -50,11 +55,11 @@ from psutil.tests import unittest
 
 
 # ===================================================================
-# --- Misc / generic tests.
+# --- Test classes' repr(), str(), ...
 # ===================================================================
 
 
-class TestMisc(PsutilTestCase):
+class TestSpecialMethods(PsutilTestCase):
 
     def test_process__repr__(self, func=repr):
         p = psutil.Process(self.spawn_testproc().pid)
@@ -95,6 +100,12 @@ class TestMisc(PsutilTestCase):
     def test_process__str__(self):
         self.test_process__repr__(func=str)
 
+    def test_error__repr__(self):
+        self.assertEqual(repr(psutil.Error()), "psutil.Error()")
+
+    def test_error__str__(self):
+        self.assertEqual(str(psutil.Error()), "")
+
     def test_no_such_process__repr__(self):
         self.assertEqual(
             repr(psutil.NoSuchProcess(321)),
@@ -173,6 +184,14 @@ class TestMisc(PsutilTestCase):
         s = set([psutil.Process(), psutil.Process()])
         self.assertEqual(len(s), 1)
 
+
+# ===================================================================
+# --- Misc, generic, corner cases
+# ===================================================================
+
+
+class TestMisc(PsutilTestCase):
+
     def test__all__(self):
         dir_psutil = dir(psutil)
         for name in dir_psutil:
@@ -189,7 +208,7 @@ class TestMisc(PsutilTestCase):
                             continue
                         if (fun.__doc__ is not None and
                                 'deprecated' not in fun.__doc__.lower()):
-                            self.fail('%r not in psutil.__all__' % name)
+                            raise self.fail('%r not in psutil.__all__' % name)
 
         # Import 'star' will break if __all__ is inconsistent, see:
         # https://github.com/giampaolo/psutil/issues/656
@@ -208,6 +227,72 @@ class TestMisc(PsutilTestCase):
         p.foo = '1'
         self.assertNotIn('foo', p.as_dict())
 
+    def test_serialization(self):
+        def check(ret):
+            if json is not None:
+                json.loads(json.dumps(ret))
+            a = pickle.dumps(ret)
+            b = pickle.loads(a)
+            self.assertEqual(ret, b)
+
+        check(psutil.Process().as_dict())
+        check(psutil.virtual_memory())
+        check(psutil.swap_memory())
+        check(psutil.cpu_times())
+        check(psutil.cpu_times_percent(interval=0))
+        check(psutil.net_io_counters())
+        if LINUX and not os.path.exists('/proc/diskstats'):
+            pass
+        else:
+            if not APPVEYOR:
+                check(psutil.disk_io_counters())
+        check(psutil.disk_partitions())
+        check(psutil.disk_usage(os.getcwd()))
+        check(psutil.users())
+
+    # XXX: https://github.com/pypa/setuptools/pull/2896
+    @unittest.skipIf(APPVEYOR, "temporarily disabled due to setuptools bug")
+    def test_setup_script(self):
+        setup_py = os.path.join(ROOT_DIR, 'setup.py')
+        if CI_TESTING and not os.path.exists(setup_py):
+            return self.skipTest("can't find setup.py")
+        module = import_module_by_path(setup_py)
+        self.assertRaises(SystemExit, module.setup)
+        self.assertEqual(module.get_version(), psutil.__version__)
+
+    def test_ad_on_process_creation(self):
+        # We are supposed to be able to instantiate Process also in case
+        # of zombie processes or access denied.
+        with mock.patch.object(psutil.Process, 'create_time',
+                               side_effect=psutil.AccessDenied) as meth:
+            psutil.Process()
+            assert meth.called
+        with mock.patch.object(psutil.Process, 'create_time',
+                               side_effect=psutil.ZombieProcess(1)) as meth:
+            psutil.Process()
+            assert meth.called
+        with mock.patch.object(psutil.Process, 'create_time',
+                               side_effect=ValueError) as meth:
+            with self.assertRaises(ValueError):
+                psutil.Process()
+            assert meth.called
+
+    def test_sanity_version_check(self):
+        # see: https://github.com/giampaolo/psutil/issues/564
+        with mock.patch(
+                "psutil._psplatform.cext.version", return_value="0.0.0"):
+            with self.assertRaises(ImportError) as cm:
+                reload_module(psutil)
+            self.assertIn("version conflict", str(cm.exception).lower())
+
+
+# ===================================================================
+# --- psutil/_common.py utils
+# ===================================================================
+
+
+class TestCommonModule(PsutilTestCase):
+
     def test_memoize(self):
         @memoize
         def foo(*args, **kwargs):
@@ -271,8 +356,6 @@ class TestMisc(PsutilTestCase):
         self.assertEqual(len(calls), 2)
 
     def test_parse_environ_block(self):
-        from psutil._common import parse_environ_block
-
         def k(s):
             return s.upper() if WINDOWS else s
 
@@ -327,7 +410,6 @@ class TestMisc(PsutilTestCase):
                     sock.close()
 
     def test_isfile_strict(self):
-        from psutil._common import isfile_strict
         this_file = os.path.abspath(__file__)
         assert isfile_strict(this_file)
         assert not isfile_strict(os.path.dirname(this_file))
@@ -343,64 +425,6 @@ class TestMisc(PsutilTestCase):
         with mock.patch('psutil._common.stat.S_ISREG', return_value=False):
             assert not isfile_strict(this_file)
 
-    def test_serialization(self):
-        def check(ret):
-            if json is not None:
-                json.loads(json.dumps(ret))
-            a = pickle.dumps(ret)
-            b = pickle.loads(a)
-            self.assertEqual(ret, b)
-
-        check(psutil.Process().as_dict())
-        check(psutil.virtual_memory())
-        check(psutil.swap_memory())
-        check(psutil.cpu_times())
-        check(psutil.cpu_times_percent(interval=0))
-        check(psutil.net_io_counters())
-        if LINUX and not os.path.exists('/proc/diskstats'):
-            pass
-        else:
-            if not APPVEYOR:
-                check(psutil.disk_io_counters())
-        check(psutil.disk_partitions())
-        check(psutil.disk_usage(os.getcwd()))
-        check(psutil.users())
-
-    # XXX: https://github.com/pypa/setuptools/pull/2896
-    @unittest.skipIf(APPVEYOR, "temporarily disabled due to setuptools bug")
-    def test_setup_script(self):
-        setup_py = os.path.join(ROOT_DIR, 'setup.py')
-        if CI_TESTING and not os.path.exists(setup_py):
-            return self.skipTest("can't find setup.py")
-        module = import_module_by_path(setup_py)
-        self.assertRaises(SystemExit, module.setup)
-        self.assertEqual(module.get_version(), psutil.__version__)
-
-    def test_ad_on_process_creation(self):
-        # We are supposed to be able to instantiate Process also in case
-        # of zombie processes or access denied.
-        with mock.patch.object(psutil.Process, 'create_time',
-                               side_effect=psutil.AccessDenied) as meth:
-            psutil.Process()
-            assert meth.called
-        with mock.patch.object(psutil.Process, 'create_time',
-                               side_effect=psutil.ZombieProcess(1)) as meth:
-            psutil.Process()
-            assert meth.called
-        with mock.patch.object(psutil.Process, 'create_time',
-                               side_effect=ValueError) as meth:
-            with self.assertRaises(ValueError):
-                psutil.Process()
-            assert meth.called
-
-    def test_sanity_version_check(self):
-        # see: https://github.com/giampaolo/psutil/issues/564
-        with mock.patch(
-                "psutil._psplatform.cext.version", return_value="0.0.0"):
-            with self.assertRaises(ImportError) as cm:
-                reload_module(psutil)
-            self.assertIn("version conflict", str(cm.exception).lower())
-
     def test_debug(self):
         if PY3:
             from io import StringIO
@@ -430,6 +454,17 @@ class TestMisc(PsutilTestCase):
         self.assertIn("no such file", msg)
         self.assertIn("/foo", msg)
 
+    def test_cat_bcat(self):
+        testfn = self.get_testfn()
+        with open(testfn, "wt") as f:
+            f.write("foo")
+        self.assertEqual(cat(testfn), "foo")
+        self.assertEqual(bcat(testfn), b"foo")
+        self.assertRaises(FileNotFoundError, cat, testfn + '-invalid')
+        self.assertRaises(FileNotFoundError, bcat, testfn + '-invalid')
+        self.assertEqual(cat(testfn + '-invalid', fallback="bar"), "bar")
+        self.assertEqual(bcat(testfn + '-invalid', fallback="bar"), "bar")
+
 
 # ===================================================================
 # --- Tests for wrap_numbers() function.
@@ -717,8 +752,8 @@ class TestScripts(PsutilTestCase):
             if name.endswith('.py'):
                 if 'test_' + os.path.splitext(name)[0] not in meths:
                     # self.assert_stdout(name)
-                    self.fail('no test defined for %r script'
-                              % os.path.join(SCRIPTS_DIR, name))
+                    raise self.fail('no test defined for %r script'
+                                    % os.path.join(SCRIPTS_DIR, name))
 
     @unittest.skipIf(not POSIX, "POSIX only")
     def test_executable(self):
diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py
index 31b81926..edef7e7d 100755
--- a/psutil/tests/test_posix.py
+++ b/psutil/tests/test_posix.py
@@ -308,7 +308,7 @@ class TestSystemAPIs(PsutilTestCase):
         if len(pids_ps) - len(pids_psutil) > 1:
             difference = [x for x in pids_psutil if x not in pids_ps] + \
                          [x for x in pids_ps if x not in pids_psutil]
-            self.fail("difference: " + str(difference))
+            raise self.fail("difference: " + str(difference))
 
     # for some reason ifconfig -a does not report all interfaces
     # returned by psutil
@@ -322,7 +322,7 @@ class TestSystemAPIs(PsutilTestCase):
                 if line.startswith(nic):
                     break
             else:
-                self.fail(
+                raise self.fail(
                     "couldn't find %s nic in 'ifconfig -a' output\n%s" % (
                         nic, output))
 
diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py
index c9059e33..2a2af93c 100755
--- a/psutil/tests/test_process.py
+++ b/psutil/tests/test_process.py
@@ -261,10 +261,10 @@ class TestProcess(PsutilTestCase):
         # using a tolerance  of +/- 0.1 seconds.
         # It will fail if the difference between the values is > 0.1s.
         if (max([user_time, utime]) - min([user_time, utime])) > 0.1:
-            self.fail("expected: %s, found: %s" % (utime, user_time))
+            raise self.fail("expected: %s, found: %s" % (utime, user_time))
 
         if (max([kernel_time, ktime]) - min([kernel_time, ktime])) > 0.1:
-            self.fail("expected: %s, found: %s" % (ktime, kernel_time))
+            raise self.fail("expected: %s, found: %s" % (ktime, kernel_time))
 
     @unittest.skipIf(not HAS_PROC_CPU_NUM, "not supported")
     def test_cpu_num(self):
@@ -285,8 +285,8 @@ class TestProcess(PsutilTestCase):
         # It will fail if the difference between the values is > 2s.
         difference = abs(create_time - now)
         if difference > 2:
-            self.fail("expected: %s, found: %s, difference: %s"
-                      % (now, create_time, difference))
+            raise self.fail("expected: %s, found: %s, difference: %s"
+                            % (now, create_time, difference))
 
         # make sure returned value can be pretty printed with strftime
         time.strftime("%Y %m %d %H:%M:%S", time.localtime(p.create_time()))
@@ -983,7 +983,8 @@ class TestProcess(PsutilTestCase):
                         file.fd == fileobj.fileno():
                     break
             else:
-                self.fail("no file found; files=%s" % repr(p.open_files()))
+                raise self.fail("no file found; files=%s" % (
+                                repr(p.open_files())))
             self.assertEqual(normcase(file.path), normcase(fileobj.name))
             if WINDOWS:
                 self.assertEqual(file.fd, -1)
@@ -1020,7 +1021,8 @@ class TestProcess(PsutilTestCase):
             after = sum(p.num_ctx_switches())
             if after > before:
                 return
-        self.fail("num ctx switches still the same after 50.000 iterations")
+        raise self.fail(
+            "num ctx switches still the same after 50.000 iterations")
 
     def test_ppid(self):
         p = psutil.Process()
@@ -1495,7 +1497,7 @@ if POSIX and os.getuid() == 0:
             except psutil.AccessDenied:
                 pass
             else:
-                self.fail("exception not raised")
+                raise self.fail("exception not raised")
 
         @unittest.skipIf(1, "causes problem as root")
         def test_zombie_process(self):
diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py
index db2cb348..ed328d01 100755
--- a/psutil/tests/test_system.py
+++ b/psutil/tests/test_system.py
@@ -184,8 +184,7 @@ class TestProcessAPIs(PsutilTestCase):
                 # in case the process disappeared in meantime fail only
                 # if it is no longer in psutil.pids()
                 time.sleep(.1)
-                if pid in psutil.pids():
-                    self.fail(pid)
+                self.assertNotIn(pid, psutil.pids())
         pids = range(max(pids) + 5000, max(pids) + 6000)
         for pid in pids:
             self.assertFalse(psutil.pid_exists(pid), msg=pid)
@@ -280,10 +279,10 @@ class TestMemoryAPIs(PsutilTestCase):
                 self.assertIsInstance(value, (int, long))
             if name != 'total':
                 if not value >= 0:
-                    self.fail("%r < 0 (%s)" % (name, value))
+                    raise self.fail("%r < 0 (%s)" % (name, value))
                 if value > mem.total:
-                    self.fail("%r > total (total=%s, %s=%s)"
-                              % (name, mem.total, name, value))
+                    raise self.fail("%r > total (total=%s, %s=%s)"
+                                    % (name, mem.total, name, value))
 
     def test_swap_memory(self):
         mem = psutil.swap_memory()
@@ -376,7 +375,7 @@ class TestCpuAPIs(PsutilTestCase):
             t2 = sum(psutil.cpu_times())
             if t2 > t1:
                 return
-        self.fail("time remained the same")
+        raise self.fail("time remained the same")
 
     def test_per_cpu_times(self):
         # Check type, value >= 0, str().
diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py
index ea694be4..55b6bc7b 100755
--- a/psutil/tests/test_windows.py
+++ b/psutil/tests/test_windows.py
@@ -114,7 +114,7 @@ class TestSystemAPIs(WindowsTestCase):
             if "pseudo-interface" in nic.replace(' ', '-').lower():
                 continue
             if nic not in out:
-                self.fail(
+                raise self.fail(
                     "%r nic wasn't found in 'ipconfig /all' output" % nic)
 
     def test_total_phymem(self):
@@ -168,11 +168,11 @@ class TestSystemAPIs(WindowsTestCase):
                     self.assertEqual(usage.free, wmi_free)
                     # 10 MB tollerance
                     if abs(usage.free - wmi_free) > 10 * 1024 * 1024:
-                        self.fail("psutil=%s, wmi=%s" % (
+                        raise self.fail("psutil=%s, wmi=%s" % (
                             usage.free, wmi_free))
                     break
             else:
-                self.fail("can't find partition %s" % repr(ps_part))
+                raise self.fail("can't find partition %s" % repr(ps_part))
 
     @retry_on_failure()
     def test_disk_usage(self):
@@ -539,7 +539,7 @@ class TestProcessWMI(WindowsTestCase):
         # returned instead.
         wmi_usage = int(w.PageFileUsage)
         if (vms != wmi_usage) and (vms != wmi_usage * 1024):
-            self.fail("wmi=%s, psutil=%s" % (wmi_usage, vms))
+            raise self.fail("wmi=%s, psutil=%s" % (wmi_usage, vms))
 
     def test_create_time(self):
         w = wmi.WMI().Win32_Process(ProcessId=self.pid)[0]