PSUTIL_DEBUG: print file + line number for C ext modules (#2005)

20 files changed, 98 insertions, 61 deletions

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 84f3e749..0559acd2 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -32,8 +32,8 @@ jobs:
           - {name: Linux, python: '3.9', os: ubuntu-latest}
     env:
       CIBW_TEST_COMMAND:
-        PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_TESTING=1 PSUTIL_DEBUG=1 python {project}/psutil/tests/runner.py &&
-        PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_TESTING=1 PSUTIL_DEBUG=1 python {project}/psutil/tests/test_memleaks.py
+        PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_DEBUG=1 python {project}/psutil/tests/runner.py &&
+        PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_DEBUG=1 python {project}/psutil/tests/test_memleaks.py
       CIBW_TEST_EXTRAS: test
       CIBW_SKIP: cp35-* pp*
 
@@ -94,7 +94,6 @@ jobs:
           export \
             PYTHONUNBUFFERED=1 \
             PYTHONWARNINGS=always \
-            PSUTIL_TESTING=1 \
             PSUTIL_DEBUG=1
           python3 -m pip install --user setuptools
           python3 setup.py install
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 69c9dd94..481793fc 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -14,6 +14,8 @@ Issues
   editing the default issue **template**. There is a bot which automatically
   assigns **labels** based on issue's title and body format. Labels help
   keeping the issues properly organized and searchable (by OS, issue type, etc.).
+* When reporting a malfunction, consider enabling
+  [debug mode](https://psutil.readthedocs.io/en/latest/#debug-mode) first.
 * To report a **security vulnerability**, use the
   [Tidelift security contact](https://tidelift.com/security).
   Tidelift will coordinate the fix and the disclosure of the reported problem.
diff --git a/HISTORY.rst b/HISTORY.rst
index 9319ba1c..5531f9d1 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -16,6 +16,8 @@ XXXX-XX-XX
 - 1996_: add support for MidnightBSD.  (patch by Saeed Rasooli)
 - 1999_: [Linux] disk_partitions(): convert "/dev/root" device (an alias used
   on some Linux distros) to real root device path.
+- 2005_: PSUTIL_DEBUG mode now prints file name and line number of the debug
+  messages coming from C extension modules.
 
 **Bug fixes**
 
diff --git a/Makefile b/Makefile
index 1f4f27d4..f255b651 100644
--- a/Makefile
+++ b/Makefile
@@ -39,7 +39,7 @@ BUILD_OPTS = `$(PYTHON) -c \
 # In not in a virtualenv, add --user options for install commands.
 INSTALL_OPTS = `$(PYTHON) -c \
 	"import sys; print('' if hasattr(sys, 'real_prefix') else '--user')"`
-TEST_PREFIX = PYTHONWARNINGS=always PSUTIL_TESTING=1 PSUTIL_DEBUG=1
+TEST_PREFIX = PYTHONWARNINGS=always PSUTIL_DEBUG=1
 
 all: test
 
diff --git a/appveyor.yml b/appveyor.yml
index aaf7de30..59c47f0a 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -16,7 +16,6 @@ environment:
     WITH_COMPILER: "cmd /E:ON /V:ON /C .\\.ci\\appveyor\\run_with_compiler.cmd"
     PYTHONWARNINGS: always
     PYTHONUNBUFFERED: 1
-    PSUTIL_TESTING: 1
     PSUTIL_DEBUG: 1
   matrix:
     # 32 bits
diff --git a/docs/index.rst b/docs/index.rst
index 4ddddba5..6738f5a2 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -2553,6 +2553,30 @@ Running tests
 
     $ python3 -m psutil.tests
 
+Debug mode
+==========
+
+If you want to debug unusual situations or want to report a bug, it may be
+useful to enable debug mode via ``PSUTIL_DEBUG`` environment variable.
+In this mode, psutil may (or may not) print additional information to stderr.
+Usually these are error conditions which are not severe, and hence are ignored
+(instead of crashing).
+Unit tests automatically run with debug mode enabled.
+On UNIX:
+
+::
+
+  $ PSUTIL_DEBUG=1 python3 script.py
+  psutil-debug [psutil/_psutil_linux.c:150]> setmntent() failed (ignored)
+
+On Windows:
+
+::
+
+  set PSUTIL_DEBUG=1 python.exe script.py
+  psutil-debug [psutil/arch/windows/process_info.c:90]> NtWow64ReadVirtualMemory64(pbi64.PebBaseAddress) -> 998 (Unknown error) (ignored)
+
+
 Security
 ========
 
diff --git a/psutil/__init__.py b/psutil/__init__.py
index e52fdcda..2760c76b 100644
--- a/psutil/__init__.py
+++ b/psutil/__init__.py
@@ -2345,6 +2345,15 @@ if WINDOWS:
 # =====================================================================
 
 
+def _set_debug(value):
+    """Enable or disable PSUTIL_DEBUG option, which prints debugging
+    messages to stderr.
+    """
+    import psutil._common
+    psutil._common.PSUTIL_DEBUG = bool(value)
+    _psplatform.cext.set_debug(bool(value))
+
+
 def test():  # pragma: no cover
     from ._common import bytes2human
     from ._compat import get_terminal_size
diff --git a/psutil/_common.py b/psutil/_common.py
index 6a2f38e1..5e6dbb6d 100644
--- a/psutil/_common.py
+++ b/psutil/_common.py
@@ -41,6 +41,7 @@ else:
 
 # can't take it from _common.py as this script is imported by setup.py
 PY3 = sys.version_info[0] == 3
+PSUTIL_DEBUG = bool(os.getenv('PSUTIL_DEBUG', 0))
 
 __all__ = [
     # OS constants
@@ -829,11 +830,10 @@ def print_color(
             SetConsoleTextAttribute(handle, DEFAULT_COLOR)
 
 
-if bool(os.getenv('PSUTIL_DEBUG', 0)):
-    import inspect
-
-    def debug(msg):
-        """If PSUTIL_DEBUG env var is set, print a debug message to stderr."""
+def debug(msg):
+    """If PSUTIL_DEBUG env var is set, print a debug message to stderr."""
+    if PSUTIL_DEBUG:
+        import inspect
         fname, lineno, func_name, lines, index = inspect.getframeinfo(
             inspect.currentframe().f_back)
         if isinstance(msg, Exception):
@@ -844,6 +844,3 @@ if bool(os.getenv('PSUTIL_DEBUG', 0)):
                 msg = "ignoring %r" % msg
         print("psutil-debug [%s:%s]> %s" % (fname, lineno, msg),  # NOQA
               file=sys.stderr)
-else:
-    def debug(msg):
-        pass
diff --git a/psutil/_psutil_aix.c b/psutil/_psutil_aix.c
index a80bed70..ba638641 100644
--- a/psutil/_psutil_aix.c
+++ b/psutil/_psutil_aix.c
@@ -1039,8 +1039,8 @@ PsutilMethods[] =
      "Return CPU statistics"},
 
     // --- others
-    {"set_testing", psutil_set_testing, METH_NOARGS,
-     "Set psutil in testing mode"},
+    {"set_debug", psutil_set_debug, METH_VARARGS,
+     "Enable or disable PSUTIL_DEBUG messages"},
 
     {NULL, NULL, 0, NULL}
 };
diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c
index 69ce6e8e..873ebbc9 100644
--- a/psutil/_psutil_bsd.c
+++ b/psutil/_psutil_bsd.c
@@ -1144,8 +1144,8 @@ static PyMethodDef mod_methods[] = {
 #endif
 
     // --- others
-    {"set_testing", psutil_set_testing, METH_NOARGS,
-     "Set psutil in testing mode"},
+    {"set_debug", psutil_set_debug, METH_VARARGS,
+     "Enable or disable PSUTIL_DEBUG messages"},
 
     {NULL, NULL, 0, NULL}
 };
diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c
index ff060a51..6ed14658 100644
--- a/psutil/_psutil_common.c
+++ b/psutil/_psutil_common.c
@@ -14,8 +14,6 @@
 // ====================================================================
 
 int PSUTIL_DEBUG = 0;
-int PSUTIL_TESTING = 0;
-// PSUTIL_CONN_NONE
 
 
 // ====================================================================
@@ -135,33 +133,26 @@ AccessDenied(const char *syscall) {
 // --- Global utils
 // ====================================================================
 
-/*
- * Enable testing mode. This has the same effect as setting PSUTIL_TESTING
- * env var. This dual method exists because updating os.environ on
- * Windows has no effect. Called on unit tests setup.
- */
-PyObject *
-psutil_set_testing(PyObject *self, PyObject *args) {
-    PSUTIL_TESTING = 1;
-    PSUTIL_DEBUG = 1;
-    Py_INCREF(Py_None);
-    return Py_None;
-}
 
+// Enable or disable PSUTIL_DEBUG messages.
+PyObject *
+psutil_set_debug(PyObject *self, PyObject *args) {
+    PyObject *value;
+    int x;
 
-/*
- * Print a debug message on stderr. No-op if PSUTIL_DEBUG env var is not set.
- */
-void
-psutil_debug(const char* format, ...) {
-    va_list argptr;
-    if (PSUTIL_DEBUG) {
-        va_start(argptr, format);
-        fprintf(stderr, "psutil-debug> ");
-        vfprintf(stderr, format, argptr);
-        fprintf(stderr, "\n");
-        va_end(argptr);
+    if (!PyArg_ParseTuple(args, "O", &value))
+        return NULL;
+    x = PyObject_IsTrue(value);
+    if (x < 0) {
+        return NULL;
+    }
+    else if (x == 0) {
+        PSUTIL_DEBUG = 0;
+    }
+    else {
+        PSUTIL_DEBUG = 1;
     }
+    Py_RETURN_NONE;
 }
 
 
@@ -172,8 +163,6 @@ int
 psutil_setup(void) {
     if (getenv("PSUTIL_DEBUG") != NULL)
         PSUTIL_DEBUG = 1;
-    if (getenv("PSUTIL_TESTING") != NULL)
-        PSUTIL_TESTING = 1;
     return 0;
 }
 
diff --git a/psutil/_psutil_common.h b/psutil/_psutil_common.h
index cb0b399d..6cf19d65 100644
--- a/psutil/_psutil_common.h
+++ b/psutil/_psutil_common.h
@@ -10,7 +10,6 @@
 // --- Global vars / constants
 // ====================================================================
 
-extern int PSUTIL_TESTING;
 extern int PSUTIL_DEBUG;
 // a signaler for connections without an actual status
 static const int PSUTIL_CONN_NONE = 128;
@@ -100,10 +99,19 @@ PyObject* PyErr_SetFromOSErrnoWithSyscall(const char *syscall);
 // --- Global utils
 // ====================================================================
 
-PyObject* psutil_set_testing(PyObject *self, PyObject *args);
-void psutil_debug(const char* format, ...);
+PyObject* psutil_set_debug(PyObject *self, PyObject *args);
 int psutil_setup(void);
 
+
+// Print a debug message on stderr.
+#define psutil_debug(...) do { \
+    if (! PSUTIL_DEBUG) \
+        break; \
+    fprintf(stderr, "psutil-debug [%s:%d]> ", __FILE__, __LINE__); \
+    fprintf(stderr, __VA_ARGS__); \
+    fprintf(stderr, "\n");} while(0)
+
+
 // ====================================================================
 // --- BSD
 // ====================================================================
diff --git a/psutil/_psutil_linux.c b/psutil/_psutil_linux.c
index 5836cd6b..bb213610 100644
--- a/psutil/_psutil_linux.c
+++ b/psutil/_psutil_linux.c
@@ -503,8 +503,8 @@ static PyMethodDef mod_methods[] = {
     {"linux_sysinfo", psutil_linux_sysinfo, METH_VARARGS,
      "A wrapper around sysinfo(), return system memory usage statistics"},
     // --- others
-    {"set_testing", psutil_set_testing, METH_NOARGS,
-     "Set psutil in testing mode"},
+    {"set_debug", psutil_set_debug, METH_VARARGS,
+     "Enable or disable PSUTIL_DEBUG messages"},
 
     {NULL, NULL, 0, NULL}
 };
diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c
index 6f824eb2..8c5990ae 100644
--- a/psutil/_psutil_osx.c
+++ b/psutil/_psutil_osx.c
@@ -1699,8 +1699,8 @@ static PyMethodDef mod_methods[] = {
      "Return battery information."},
 
     // --- others
-    {"set_testing", psutil_set_testing, METH_NOARGS,
-     "Set psutil in testing mode"},
+    {"set_debug", psutil_set_debug, METH_VARARGS,
+     "Enable or disable PSUTIL_DEBUG messages"},
 
     {NULL, NULL, 0, NULL}
 };
diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c
index 342798a8..2e0bd943 100644
--- a/psutil/_psutil_sunos.c
+++ b/psutil/_psutil_sunos.c
@@ -1679,8 +1679,8 @@ PsutilMethods[] = {
      "Return CPU statistics"},
 
     // --- others
-    {"set_testing", psutil_set_testing, METH_NOARGS,
-     "Set psutil in testing mode"},
+    {"set_debug", psutil_set_debug, METH_VARARGS,
+     "Enable or disable PSUTIL_DEBUG messages"},
 
     {NULL, NULL, 0, NULL}
 };
diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c
index 6a128ae2..596aa5c0 100644
--- a/psutil/_psutil_windows.c
+++ b/psutil/_psutil_windows.c
@@ -1674,8 +1674,8 @@ PsutilMethods[] = {
      "QueryDosDevice binding"},
 
     // --- others
-    {"set_testing", psutil_set_testing, METH_NOARGS,
-     "Set psutil in testing mode"},
+    {"set_debug", psutil_set_debug, METH_VARARGS,
+     "Enable or disable PSUTIL_DEBUG messages"},
 
     {NULL, NULL, 0, NULL}
 };
diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py
index 245ff2f3..d871abe4 100644
--- a/psutil/tests/__init__.py
+++ b/psutil/tests/__init__.py
@@ -952,6 +952,15 @@ class TestMemoryLeak(PsutilTestCase):
     retries = 10 if CI_TESTING else 5
     verbose = True
     _thisproc = psutil.Process()
+    _psutil_debug_orig = bool(os.getenv('PSUTIL_DEBUG', 0))
+
+    @classmethod
+    def setUpClass(cls):
+        psutil._set_debug(False)  # avoid spamming to stderr
+
+    @classmethod
+    def tearDownClass(cls):
+        psutil._set_debug(cls._psutil_debug_orig)
 
     def _get_mem(self):
         # USS is the closest thing we have to "real" memory usage and it
diff --git a/psutil/tests/runner.py b/psutil/tests/runner.py
index d761cd21..65f222c4 100755
--- a/psutil/tests/runner.py
+++ b/psutil/tests/runner.py
@@ -304,10 +304,7 @@ def run_from_name(name):
 
 
 def setup():
-    # Note: doc states that altering os.environment may cause memory
-    # leaks on some platforms.
-    # Sets PSUTIL_TESTING and PSUTIL_DEBUG in the C module.
-    psutil._psplatform.cext.set_testing()
+    psutil._set_debug(True)
 
 
 def main():
diff --git a/psutil/tests/test_memleaks.py b/psutil/tests/test_memleaks.py
index e6940a30..f0bd59ee 100755
--- a/psutil/tests/test_memleaks.py
+++ b/psutil/tests/test_memleaks.py
@@ -456,6 +456,9 @@ class TestModuleFunctionsLeaks(TestMemoryLeak):
     def test_users(self):
         self.execute(psutil.users)
 
+    def test_set_debug(self):
+        self.execute(lambda: psutil._set_debug(False))
+
     if WINDOWS:
 
         # --- win services
diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py
index 002c58de..fe74e601 100755
--- a/psutil/tests/test_process.py
+++ b/psutil/tests/test_process.py
@@ -1389,7 +1389,6 @@ class TestProcess(PsutilTestCase):
     def test_environ(self):
         def clean_dict(d):
             # Most of these are problematic on Travis.
-            d.pop("PSUTIL_TESTING", None)
             d.pop("PLAT", None)
             d.pop("HOME", None)
             if MACOS: