ENH: support parallel compilation of extensions

Allow extensions using numpy.distutils to compile in parallel.
By passing `--jobs=n` or `-j n` to `setup.py build` the compilation of
extensions is now performed in `n` parallel processes.
Additionally the environment variable NPY_NUM_BUILD_JOBS is used as
the default value, if its unset the default is serial compilation.

The parallelization is limited to within the files of an extension, so
only numpy multiarraymodule really profits but its still a nice
improvement when you have 2-4 cores.
Unfortunately Cython will not profit at all as it tends to build one
module per file.

1 files changed, 42 insertions, 7 deletions

diff --git a/numpy/distutils/misc_util.py b/numpy/distutils/misc_util.py
index c146178f0..cab0a4cba 100644
--- a/numpy/distutils/misc_util.py
+++ b/numpy/distutils/misc_util.py
@@ -13,6 +13,10 @@ import shutil
 
 import distutils
 from distutils.errors import DistutilsError
+try:
+    from threading import local as tlocal
+except ImportError:
+    from dummy_threading import local as tlocal
 
 try:
     set
@@ -31,7 +35,8 @@ __all__ = ['Configuration', 'get_numpy_include_dirs', 'default_config_dict',
            'get_script_files', 'get_lib_source_files', 'get_data_files',
            'dot_join', 'get_frame', 'minrelpath', 'njoin',
            'is_sequence', 'is_string', 'as_list', 'gpaths', 'get_language',
-           'quote_args', 'get_build_architecture', 'get_info', 'get_pkg_info']
+           'quote_args', 'get_build_architecture', 'get_info', 'get_pkg_info',
+           'get_num_build_jobs']
 
 class InstallableLib(object):
     """
@@ -60,6 +65,36 @@ class InstallableLib(object):
         self.build_info = build_info
         self.target_dir = target_dir
 
+
+def get_num_build_jobs():
+    """
+    Get number of parallel build jobs set by the --jobs command line argument
+    of setup.py
+    If the command did not receive a setting the environment variable
+    NPY_NUM_BUILD_JOBS checked and if that is unset it returns 1.
+
+    Returns
+    -------
+    out : int
+        number of parallel jobs that can be run
+
+    """
+    from numpy.distutils.core import get_distribution
+    envjobs = int(os.environ.get("NPY_NUM_BUILD_JOBS", 1))
+    dist = get_distribution()
+    # may be None during configuration
+    if dist is None:
+        return envjobs
+
+    # any of these three may have the job set, take the largest
+    cmdattr = (getattr(dist.get_command_obj('build'), 'jobs'),
+               getattr(dist.get_command_obj('build_ext'), 'jobs'),
+               getattr(dist.get_command_obj('build_clib'), 'jobs'))
+    if all(x is None for x in cmdattr):
+        return envjobs
+    else:
+        return max(x for x in cmdattr if x is not None)
+
 def quote_args(args):
     # don't used _nt_quote_args as it does not check if
     # args items already have quotes or not.
@@ -249,9 +284,9 @@ def gpaths(paths, local_path='', include_non_existing=True):
     return _fix_paths(paths, local_path, include_non_existing)
 
 
-_temporary_directory = None
 def clean_up_temporary_directory():
-    global _temporary_directory
+    tdata = tlocal()
+    _temporary_directory = getattr(tdata, 'tempdir', None)
     if not _temporary_directory:
         return
     try:
@@ -261,13 +296,13 @@ def clean_up_temporary_directory():
     _temporary_directory = None
 
 def make_temp_file(suffix='', prefix='', text=True):
-    global _temporary_directory
-    if not _temporary_directory:
-        _temporary_directory = tempfile.mkdtemp()
+    tdata = tlocal()
+    if not hasattr(tdata, 'tempdir'):
+        tdata.tempdir = tempfile.mkdtemp()
         atexit.register(clean_up_temporary_directory)
     fid, name = tempfile.mkstemp(suffix=suffix,
                                  prefix=prefix,
-                                 dir=_temporary_directory,
+                                 dir=tdata.tempdir,
                                  text=text)
     fo = os.fdopen(fid, 'w')
     return fo, name