[Linux]: increase `read(2)` buffer size when reading /proc files lines (#2054)

This should help having more consistent results.

1 files changed, 39 insertions, 12 deletions

diff --git a/psutil/_common.py b/psutil/_common.py
index 8030e2e0..1e33699c 100644
--- a/psutil/_common.py
+++ b/psutil/_common.py
@@ -711,22 +711,49 @@ wrap_numbers.cache_clear = _wn.cache_clear
 wrap_numbers.cache_info = _wn.cache_info
 
 
-def open_binary(fname, **kwargs):
-    return open(fname, "rb", **kwargs)
-
-
-def open_text(fname, **kwargs):
+# The read buffer size for open() builtin. This (also) dictates how
+# much data we read(2) when iterating over file lines as in:
+#   >>> with open(file) as f:
+#   ...    for line in f:
+#   ...        ...
+# Default per-line buffer size for binary files is 1K. For text files
+# is 8K. We use a bigger buffer (32K) in order to have more consistent
+# results when reading /proc pseudo files on Linux, see:
+# https://github.com/giampaolo/psutil/issues/2050
+# On Python 2 this also speeds up the reading of big files:
+# (namely /proc/{pid}/smaps and /proc/net/*):
+# https://github.com/giampaolo/psutil/issues/708
+FILE_READ_BUFFER_SIZE = 32 * 1024
+
+
+def open_binary(fname):
+    return open(fname, "rb", buffering=FILE_READ_BUFFER_SIZE)
+
+
+def open_text(fname):
     """On Python 3 opens a file in text mode by using fs encoding and
     a proper en/decoding errors handler.
     On Python 2 this is just an alias for open(name, 'rt').
     """
-    if PY3:
-        # See:
-        # https://github.com/giampaolo/psutil/issues/675
-        # https://github.com/giampaolo/psutil/pull/733
-        kwargs.setdefault('encoding', ENCODING)
-        kwargs.setdefault('errors', ENCODING_ERRS)
-    return open(fname, "rt", **kwargs)
+    if not PY3:
+        return open(fname, "rt", buffering=FILE_READ_BUFFER_SIZE)
+
+    # See:
+    # https://github.com/giampaolo/psutil/issues/675
+    # https://github.com/giampaolo/psutil/pull/733
+    fobj = open(fname, "rt", buffering=FILE_READ_BUFFER_SIZE,
+                encoding=ENCODING, errors=ENCODING_ERRS)
+    try:
+        # Dictates per-line read(2) buffer size. Defaults is 8k. See:
+        # https://github.com/giampaolo/psutil/issues/2050#issuecomment-1013387546
+        fobj._CHUNK_SIZE = FILE_READ_BUFFER_SIZE
+    except AttributeError:
+        pass
+    except Exception:
+        fobj.close()
+        raise
+
+    return fobj
 
 
 def cat(fname, fallback=_DEFAULT, _open=open_text):