PSUTIL_TESTING env var (#1083)

* Introduce PSUTIL_TESTING env var

...so that we can make stricter assertions in C and py code during tests
only.

* define a C function in _common.c which returns whether the var is set
* set PSUTIL_TESTING from the Makefile

* cache psutil_testing() result

* winmake: set PSUTIL_TESTING env var for tests

12 files changed, 118 insertions, 40 deletions

diff --git a/Makefile b/Makefile
index 8b40d8c2..c44c6c02 100644
--- a/Makefile
+++ b/Makefile
@@ -118,65 +118,65 @@ setup-dev-env:
 # Run all tests.
 test:
 	${MAKE} install
-	PYTHONWARNINGS=all $(PYTHON) $(TSCRIPT)
+	PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) $(TSCRIPT)
 
 # Run process-related API tests.
 test-process:
 	${MAKE} install
-	PYTHONWARNINGS=all $(PYTHON) -m unittest -v psutil.tests.test_process
+	PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) -m unittest -v psutil.tests.test_process
 
 # Run system-related API tests.
 test-system:
 	${MAKE} install
-	PYTHONWARNINGS=all $(PYTHON) -m unittest -v psutil.tests.test_system
+	PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) -m unittest -v psutil.tests.test_system
 
 # Run miscellaneous tests.
 test-misc:
 	${MAKE} install
-	PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_misc.py
+	PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_misc.py
 
 # Test APIs dealing with strings.
 test-unicode:
 	${MAKE} install
-	PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_unicode.py
+	PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_unicode.py
 
 # APIs sanity tests.
 test-contracts:
 	${MAKE} install
-	PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_contracts.py
+	PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_contracts.py
 
 # Test net_connections() and Process.connections().
 test-connections:
 	${MAKE} install
-	PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_connections.py
+	PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_connections.py
 
 # POSIX specific tests.
 test-posix:
 	${MAKE} install
-	PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_posix.py
+	PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_posix.py
 
 # Run specific platform tests only.
 test-platform:
 	${MAKE} install
-	PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS") if getattr(psutil, x)][0])'`.py
+	PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS") if getattr(psutil, x)][0])'`.py
 
 # Memory leak tests.
 test-memleaks:
 	${MAKE} install
-	PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_memory_leaks.py
+	PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_memory_leaks.py
 
 # Run a specific test by name, e.g.
 # make test-by-name psutil.tests.test_system.TestSystemAPIs.test_cpu_times
 test-by-name:
 	${MAKE} install
-	@PYTHONWARNINGS=all $(PYTHON) -m unittest -v $(ARGS)
+	@PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) -m unittest -v $(ARGS)
 
 # Run test coverage.
 coverage:
 	${MAKE} install
 	# Note: coverage options are controlled by .coveragerc file
 	rm -rf .coverage htmlcov
-	PYTHONWARNINGS=all $(PYTHON) -m coverage run $(TSCRIPT)
+	PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) -m coverage run $(TSCRIPT)
 	$(PYTHON) -m coverage report
 	@echo "writing results to htmlcov/index.html"
 	$(PYTHON) -m coverage html
diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c
index 217a95de..3527b666 100644
--- a/psutil/_psutil_bsd.c
+++ b/psutil/_psutil_bsd.c
@@ -908,7 +908,6 @@ error:
  */
 static PyMethodDef
 PsutilMethods[] = {
-
     // --- per-process functions
 
     {"proc_oneshot_info", psutil_proc_oneshot_info, METH_VARARGS,
@@ -983,6 +982,11 @@ PsutilMethods[] = {
     {"sensors_battery", psutil_sensors_battery, METH_VARARGS,
      "Return battery information."},
 #endif
+
+    // --- others
+    {"py_psutil_testing", py_psutil_testing, METH_VARARGS,
+     "Return True if PSUTIL_TESTING env var is set"},
+
     {NULL, NULL, 0, NULL}
 };
 
diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c
index c757c725..dace4724 100644
--- a/psutil/_psutil_common.c
+++ b/psutil/_psutil_common.c
@@ -7,6 +7,7 @@
  */
 
 #include <Python.h>
+#include <stdio.h>
 
 /*
  * Set OSError(errno=ESRCH, strerror="No such process") Python exception.
@@ -36,6 +37,36 @@ AccessDenied(void) {
 }
 
 
+static int _psutil_testing = -1;
+
+
+/*
+ * Return 1 if PSUTIL_TESTING env var is set else 0.
+ */
+int
+psutil_testing(void) {
+    if (_psutil_testing == -1) {
+        if (getenv("PSUTIL_TESTING") != NULL)
+            _psutil_testing = 1;
+        else
+            _psutil_testing = 0;
+    }
+    return _psutil_testing;
+}
+
+
+/*
+ * Return True if PSUTIL_TESTING env var is set else False.
+ */
+PyObject *
+py_psutil_testing(PyObject *self, PyObject *args) {
+    PyObject *res;
+    res = psutil_testing() ? Py_True : Py_False;
+    Py_INCREF(res);
+    return res;
+}
+
+
 /*
  * Backport of unicode FS APIs from Python 3.
  * On Python 2 we just return a plain byte string
diff --git a/psutil/_psutil_common.h b/psutil/_psutil_common.h
index aa634ad3..13404532 100644
--- a/psutil/_psutil_common.h
+++ b/psutil/_psutil_common.h
@@ -11,7 +11,9 @@ static const int PSUTIL_CONN_NONE = 128;
 
 PyObject* AccessDenied(void);
 PyObject* NoSuchProcess(void);
+int psutil_testing(void);
+PyObject* py_psutil_testing(PyObject *self, PyObject *args);
 #if PY_MAJOR_VERSION < 3
 PyObject* PyUnicode_DecodeFSDefault(char *s);
 PyObject* PyUnicode_DecodeFSDefaultAndSize(char *s, Py_ssize_t size);
-#endif
\ No newline at end of file
+#endif
diff --git a/psutil/_psutil_linux.c b/psutil/_psutil_linux.c
index 1a96fea0..e262ac70 100644
--- a/psutil/_psutil_linux.c
+++ b/psutil/_psutil_linux.c
@@ -203,6 +203,8 @@ psutil_disk_partitions(PyObject *self, PyObject *args) {
     if (py_retlist == NULL)
         return NULL;
 
+    psutil_testing();
+
     // MOUNTED constant comes from mntent.h and it's == '/etc/mtab'
     Py_BEGIN_ALLOW_THREADS
     file = setmntent(MOUNTED, "r");
@@ -574,7 +576,6 @@ error:
  */
 static PyMethodDef
 PsutilMethods[] = {
-
     // --- per-process functions
 
 #if PSUTIL_HAVE_IOPRIO
@@ -607,6 +608,9 @@ PsutilMethods[] = {
      "Get or set process resource limits."},
 #endif
 
+    // --- others
+    {"py_psutil_testing", py_psutil_testing, METH_VARARGS,
+     "Return True if PSUTIL_TESTING env var is set"},
 
     {NULL, NULL, 0, NULL}
 };
diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c
index 20ece694..7d762a1c 100644
--- a/psutil/_psutil_osx.c
+++ b/psutil/_psutil_osx.c
@@ -1779,7 +1779,6 @@ psutil_cpu_stats(PyObject *self, PyObject *args) {
  */
 static PyMethodDef
 PsutilMethods[] = {
-
     // --- per-process functions
 
     {"proc_kinfo_oneshot", psutil_proc_kinfo_oneshot, METH_VARARGS,
@@ -1841,6 +1840,10 @@ PsutilMethods[] = {
     {"cpu_stats", psutil_cpu_stats, METH_VARARGS,
      "Return CPU statistics"},
 
+    // --- others
+    {"py_psutil_testing", py_psutil_testing, METH_VARARGS,
+     "Return True if PSUTIL_TESTING env var is set"},
+
     {NULL, NULL, 0, NULL}
 };
 
diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c
index e3eb2560..785805d4 100644
--- a/psutil/_psutil_sunos.c
+++ b/psutil/_psutil_sunos.c
@@ -1470,7 +1470,6 @@ error:
  */
 static PyMethodDef
 PsutilMethods[] = {
-
     // --- process-related functions
     {"proc_basic_info", psutil_proc_basic_info, METH_VARARGS,
      "Return process ppid, rss, vms, ctime, nice, nthreads, status and tty"},
@@ -1513,6 +1512,10 @@ PsutilMethods[] = {
     {"cpu_stats", psutil_cpu_stats, METH_VARARGS,
      "Return CPU statistics"},
 
+    // --- others
+    {"py_psutil_testing", py_psutil_testing, METH_VARARGS,
+     "Return True if PSUTIL_TESTING env var is set"},
+
     {NULL, NULL, 0, NULL}
 };
 
diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c
index 436dd76b..795ee9cd 100644
--- a/psutil/_psutil_windows.c
+++ b/psutil/_psutil_windows.c
@@ -3479,7 +3479,6 @@ psutil_sensors_battery(PyObject *self, PyObject *args) {
 
 static PyMethodDef
 PsutilMethods[] = {
-
     // --- per-process functions
 
     {"proc_cmdline", psutil_proc_cmdline, METH_VARARGS,
@@ -3602,6 +3601,10 @@ PsutilMethods[] = {
     {"win32_QueryDosDevice", psutil_win32_QueryDosDevice, METH_VARARGS,
      "QueryDosDevice binding"},
 
+    // --- others
+    {"py_psutil_testing", py_psutil_testing, METH_VARARGS,
+     "Return True if PSUTIL_TESTING env var is set"},
+
     {NULL, NULL, 0, NULL}
 };
 
diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py
index ba7e1ce4..0ba95b18 100644
--- a/psutil/tests/__init__.py
+++ b/psutil/tests/__init__.py
@@ -77,7 +77,8 @@ __all__ = [
     'ThreadTask'
     # test utils
     'unittest', 'skip_on_access_denied', 'skip_on_not_implemented',
-    'retry_before_failing', 'run_test_module_by_name',
+    'retry_before_failing', 'run_test_module_by_name', 'get_suite',
+    'run_suite',
     # install utils
     'install_pip', 'install_test_deps',
     # fs utils
@@ -141,6 +142,7 @@ ASCII_FS = sys.getfilesystemencoding().lower() in ('ascii', 'us-ascii')
 
 ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
 SCRIPTS_DIR = os.path.join(ROOT_DIR, 'scripts')
+HERE = os.path.abspath(os.path.dirname(__file__))
 
 # --- support
 
@@ -383,6 +385,9 @@ def reap_children(recursive=False):
     # https://ci.appveyor.com/project/giampaolo/psutil/build/job/
     #     jiq2cgd6stsbtn60
     def assert_gone(pid):
+        # XXX
+        if WINDOWS:
+            return
         assert not psutil.pid_exists(pid), pid
         assert pid not in psutil.pids(), pid
         try:
@@ -699,9 +704,34 @@ class TestCase(unittest.TestCase):
 unittest.TestCase = TestCase
 
 
+def _setup_tests():
+    assert 'PSUTIL_TESTING' in os.environ
+    assert psutil._psplatform.cext.py_psutil_testing()
+
+
+def get_suite():
+    testmodules = [os.path.splitext(x)[0] for x in os.listdir(HERE)
+                   if x.endswith('.py') and x.startswith('test_') and not
+                   x.startswith('test_memory_leaks')]
+    suite = unittest.TestSuite()
+    for tm in testmodules:
+        # ...so that the full test paths are printed on screen
+        tm = "psutil.tests.%s" % tm
+        suite.addTest(unittest.defaultTestLoader.loadTestsFromName(tm))
+    return suite
+
+
+def run_suite():
+    _setup_tests()
+    result = unittest.TextTestRunner(verbosity=VERBOSITY).run(get_suite())
+    success = result.wasSuccessful()
+    sys.exit(0 if success else 1)
+
+
 def run_test_module_by_name(name):
     # testmodules = [os.path.splitext(x)[0] for x in os.listdir(HERE)
     #                if x.endswith('.py') and x.startswith('test_')]
+    _setup_tests()
     name = os.path.splitext(os.path.basename(name))[0]
     suite = unittest.TestSuite()
     suite.addTest(unittest.defaultTestLoader.loadTestsFromName(name))
diff --git a/psutil/tests/__main__.py b/psutil/tests/__main__.py
index 896b00cc..475e6b81 100755
--- a/psutil/tests/__main__.py
+++ b/psutil/tests/__main__.py
@@ -21,8 +21,7 @@ try:
 except ImportError:
     from urllib2 import urlopen
 
-from psutil.tests import unittest
-from psutil.tests import VERBOSITY
+from psutil.tests import run_suite
 
 
 HERE = os.path.abspath(os.path.dirname(__file__))
@@ -73,24 +72,6 @@ def install_test_deps(deps=None):
         return code
 
 
-def get_suite():
-    testmodules = [os.path.splitext(x)[0] for x in os.listdir(HERE)
-                   if x.endswith('.py') and x.startswith('test_') and not
-                   x.startswith('test_memory_leaks')]
-    suite = unittest.TestSuite()
-    for tm in testmodules:
-        # ...so that the full test paths are printed on screen
-        tm = "psutil.tests.%s" % tm
-        suite.addTest(unittest.defaultTestLoader.loadTestsFromName(tm))
-    return suite
-
-
-def run_suite():
-    result = unittest.TextTestRunner(verbosity=VERBOSITY).run(get_suite())
-    success = result.wasSuccessful()
-    sys.exit(0 if success else 1)
-
-
 def main():
     usage = "%s -m psutil.tests [opts]" % PYTHON
     parser = optparse.OptionParser(usage=usage, description="run unit tests")
diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py
index b1f2508f..3410ec0b 100755
--- a/psutil/tests/test_process.py
+++ b/psutil/tests/test_process.py
@@ -1391,6 +1391,8 @@ class TestProcess(unittest.TestCase):
         d2 = os.environ.copy()
 
         removes = []
+        if 'PSUTIL_TESTING' in os.environ:
+            removes.append('PSUTIL_TESTING')
         if OSX:
             removes.extend([
                 "__CF_USER_TEXT_ENCODING",
diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py
index d0c2c0a1..40ba3742 100755
--- a/scripts/internal/winmake.py
+++ b/scripts/internal/winmake.py
@@ -77,7 +77,10 @@ def safe_print(text, file=sys.stdout, flush=False):
 def sh(cmd, nolog=False):
     if not nolog:
         safe_print("cmd: " + cmd)
-    subprocess.check_call(cmd, shell=True, env=os.environ, cwd=os.getcwd())
+    p = subprocess.Popen(cmd, shell=True, env=os.environ, cwd=os.getcwd())
+    p.communicate()
+    if p.returncode != 0:
+        sys.exit(p.returncode)
 
 
 def cmd(fun):
@@ -327,6 +330,7 @@ def flake8():
 def test():
     """Run tests"""
     install()
+    os.environ['PSUTIL_TESTING'] = '1'
     sh("%s -Wa %s" % (PYTHON, TSCRIPT))
 
 
@@ -335,6 +339,7 @@ def coverage():
     """Run coverage tests."""
     # Note: coverage options are controlled by .coveragerc file
     install()
+    os.environ['PSUTIL_TESTING'] = '1'
     sh("%s -Wa -m coverage run %s" % (PYTHON, TSCRIPT))
     sh("%s -m coverage report" % PYTHON)
     sh("%s -m coverage html" % PYTHON)
@@ -345,6 +350,7 @@ def coverage():
 def test_process():
     """Run process tests"""
     install()
+    os.environ['PSUTIL_TESTING'] = '1'
     sh("%s -Wa -m unittest -v psutil.tests.test_process" % PYTHON)
 
 
@@ -352,6 +358,7 @@ def test_process():
 def test_system():
     """Run system tests"""
     install()
+    os.environ['PSUTIL_TESTING'] = '1'
     sh("%s -Wa -m unittest -v psutil.tests.test_system" % PYTHON)
 
 
@@ -359,6 +366,7 @@ def test_system():
 def test_platform():
     """Run windows only tests"""
     install()
+    os.environ['PSUTIL_TESTING'] = '1'
     sh("%s -Wa -m unittest -v psutil.tests.test_windows" % PYTHON)
 
 
@@ -366,6 +374,7 @@ def test_platform():
 def test_misc():
     """Run misc tests"""
     install()
+    os.environ['PSUTIL_TESTING'] = '1'
     sh("%s -Wa -m unittest -v psutil.tests.test_misc" % PYTHON)
 
 
@@ -373,6 +382,7 @@ def test_misc():
 def test_unicode():
     """Run unicode tests"""
     install()
+    os.environ['PSUTIL_TESTING'] = '1'
     sh("%s -Wa -m unittest -v psutil.tests.test_unicode" % PYTHON)
 
 
@@ -380,6 +390,7 @@ def test_unicode():
 def test_connections():
     """Run connections tests"""
     install()
+    os.environ['PSUTIL_TESTING'] = '1'
     sh("%s -Wa -m unittest -v psutil.tests.test_connections" % PYTHON)
 
 
@@ -387,6 +398,7 @@ def test_connections():
 def test_contracts():
     """Run contracts tests"""
     install()
+    os.environ['PSUTIL_TESTING'] = '1'
     sh("%s -Wa -m unittest -v psutil.tests.test_contracts" % PYTHON)
 
 
@@ -399,6 +411,7 @@ def test_by_name():
     except IndexError:
         sys.exit('second arg missing')
     install()
+    os.environ['PSUTIL_TESTING'] = '1'
     sh("%s -Wa -m unittest -v %s" % (PYTHON, name))
 
 
@@ -411,6 +424,7 @@ def test_script():
     except IndexError:
         sys.exit('second arg missing')
     install()
+    os.environ['PSUTIL_TESTING'] = '1'
     sh("%s -Wa %s" % (PYTHON, name))
 
 
@@ -418,6 +432,7 @@ def test_script():
 def test_memleaks():
     """Run memory leaks tests"""
     install()
+    os.environ['PSUTIL_TESTING'] = '1'
     sh("%s -Wa psutil\\tests\\test_memory_leaks.py" % PYTHON)