"""Build extension modules.""" import os import re import sys import site import sysconfig from packaging.util import get_platform from packaging.command.cmd import Command from packaging.errors import (CCompilerError, CompileError, PackagingError, PackagingPlatformError, PackagingSetupError) from packaging.compiler import customize_compiler, show_compilers from packaging.util import newer_group from packaging.compiler.extension import Extension from packaging import logger if os.name == 'nt': from packaging.compiler.msvccompiler import get_build_version MSVC_VERSION = int(get_build_version()) # An extension name is just a dot-separated list of Python NAMEs (ie. # the same as a fully-qualified module name). extension_name_re = re.compile \ (r'^[a-zA-Z_][a-zA-Z_0-9]*(\.[a-zA-Z_][a-zA-Z_0-9]*)*$') class build_ext(Command): description = "build C/C++ extension modules (compile/link to build directory)" # XXX thoughts on how to deal with complex command-line options like # these, i.e. how to make it so fancy_getopt can suck them off the # command line and turn them into the appropriate # lists of tuples of what-have-you. # - each command needs a callback to process its command-line options # - Command.__init__() needs access to its share of the whole # command line (must ultimately come from # Distribution.parse_command_line()) # - it then calls the current command class' option-parsing # callback to deal with weird options like -D, which have to # parse the option text and churn out some custom data # structure # - that data structure (in this case, a list of 2-tuples) # will then be present in the command object by the time # we get to finalize_options() (i.e. the constructor # takes care of both command-line and client options # in between initialize_options() and finalize_options()) sep_by = " (separated by '%s')" % os.pathsep user_options = [ ('build-lib=', 'b', "directory for compiled extension modules"), ('build-temp=', 't', "directory for temporary files (build by-products)"), ('plat-name=', 'p', "platform name to cross-compile for, if supported " "(default: %s)" % get_platform()), ('inplace', 'i', "ignore build-lib and put compiled extensions into the source " + "directory alongside your pure Python modules"), ('user', None, "add user include, library and rpath"), ('include-dirs=', 'I', "list of directories to search for header files" + sep_by), ('define=', 'D', "C preprocessor macros to define"), ('undef=', 'U', "C preprocessor macros to undefine"), ('libraries=', 'l', "external C libraries to link with"), ('library-dirs=', 'L', "directories to search for external C libraries" + sep_by), ('rpath=', 'R', "directories to search for shared C libraries at runtime"), ('link-objects=', 'O', "extra explicit link objects to include in the link"), ('debug', 'g', "compile/link with debugging information"), ('force', 'f', "forcibly build everything (ignore file timestamps)"), ('compiler=', 'c', "specify the compiler type"), ('swig-opts=', None, "list of SWIG command-line options"), ('swig=', None, "path to the SWIG executable"), ] boolean_options = ['inplace', 'debug', 'force', 'user'] help_options = [ ('help-compiler', None, "list available compilers", show_compilers), ] def initialize_options(self): self.extensions = None self.build_lib = None self.plat_name = None self.build_temp = None self.inplace = False self.package = None self.include_dirs = None self.define = None self.undef = None self.libraries = None self.library_dirs = None self.rpath = None self.link_objects = None self.debug = None self.force = None self.compiler = None self.swig = None self.swig_opts = None self.user = None def finalize_options(self): self.set_undefined_options('build', 'build_lib', 'build_temp', 'compiler', 'debug', 'force', 'plat_name') if self.package is None: self.package = self.distribution.ext_package # Ensure that the list of extensions is valid, i.e. it is a list of # Extension objects. self.extensions = self.distribution.ext_modules if self.extensions: if not isinstance(self.extensions, (list, tuple)): type_name = (self.extensions is None and 'None' or type(self.extensions).__name__) raise PackagingSetupError( "'ext_modules' must be a sequence of Extension instances," " not %s" % (type_name,)) for i, ext in enumerate(self.extensions): if isinstance(ext, Extension): continue # OK! (assume type-checking done # by Extension constructor) type_name = (ext is None and 'None' or type(ext).__name__) raise PackagingSetupError( "'ext_modules' item %d must be an Extension instance," " not %s" % (i, type_name)) # Make sure Python's include directories (for Python.h, pyconfig.h, # etc.) are in the include search path. py_include = sysconfig.get_path('include') plat_py_include = sysconfig.get_path('platinclude') if self.include_dirs is None: self.include_dirs = self.distribution.include_dirs or [] if isinstance(self.include_dirs, str): self.include_dirs = self.include_dirs.split(os.pathsep) # Put the Python "system" include dir at the end, so that # any local include dirs take precedence. self.include_dirs.append(py_include) if plat_py_include != py_include: self.include_dirs.append(plat_py_include) self.ensure_string_list('libraries') # Life is easier if we're not forever checking for None, so # simplify these options to empty lists if unset if self.libraries is None: self.libraries = [] if self.library_dirs is None: self.library_dirs = [] elif isinstance(self.library_dirs, str): self.library_dirs = self.library_dirs.split(os.pathsep) if self.rpath is None: self.rpath = [] elif isinstance(self.rpath, str): self.rpath = self.rpath.split(os.pathsep) # for extensions under windows use different directories # for Release and Debug builds. # also Python's library directory must be appended to library_dirs if os.name == 'nt': # the 'libs' directory is for binary installs - we assume that # must be the *native* platform. But we don't really support # cross-compiling via a binary install anyway, so we let it go. self.library_dirs.append(os.path.join(sys.exec_prefix, 'libs')) if self.debug: self.build_temp = os.path.join(self.build_temp, "Debug") else: self.build_temp = os.path.join(self.build_temp, "Release") # Append the source distribution include and library directories, # this allows distutils on windows to work in the source tree self.include_dirs.append(os.path.join(sys.exec_prefix, 'PC')) if MSVC_VERSION == 9: # Use the .lib files for the correct architecture if self.plat_name == 'win32': suffix = '' else: # win-amd64 or win-ia64 suffix = self.plat_name[4:] new_lib = os.path.join(sys.exec_prefix, 'PCbuild') if suffix: new_lib = os.path.join(new_lib, suffix) self.library_dirs.append(new_lib) elif MSVC_VERSION == 8: self.library_dirs.append(os.path.join(sys.exec_prefix, 'PC', 'VS8.0')) elif MSVC_VERSION == 7: self.library_dirs.append(os.path.join(sys.exec_prefix, 'PC', 'VS7.1')) else: self.library_dirs.append(os.path.join(sys.exec_prefix, 'PC', 'VC6')) # OS/2 (EMX) doesn't support Debug vs Release builds, but has the # import libraries in its "Config" subdirectory if os.name == 'os2': self.library_dirs.append(os.path.join(sys.exec_prefix, 'Config')) # for extensions under Cygwin and AtheOS Python's library directory must be # appended to library_dirs if sys.platform[:6] == 'cygwin' or sys.platform[:6] == 'atheos': if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): # building third party extensions self.library_dirs.append(os.path.join(sys.prefix, "lib", "python" + sysconfig.get_python_version(), "config")) else: # building python standard extensions self.library_dirs.append(os.curdir) # for extensions under Linux or Solaris with a shared Python library, # Python's library directory must be appended to library_dirs sysconfig.get_config_var('Py_ENABLE_SHARED') if (sys.platform.startswith(('linux', 'gnu', 'sunos')) and sysconfig.get_config_var('Py_ENABLE_SHARED')): if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): # building third party extensions self.library_dirs.append(sysconfig.get_config_var('LIBDIR')) else: # building python standard extensions self.library_dirs.append(os.curdir) # The argument parsing will result in self.define being a string, but # it has to be a list of 2-tuples. All the preprocessor symbols # specified by the 'define' option will be set to '1'. Multiple # symbols can be separated with commas. if self.define: defines = self.define.split(',') self.define = [(symbol, '1') for symbol in defines] # The option for macros to undefine is also a string from the # option parsing, but has to be a list. Multiple symbols can also # be separated with commas here. if self.undef: self.undef = self.undef.split(',') if self.swig_opts is None: self.swig_opts = [] else: self.swig_opts = self.swig_opts.split(' ') # Finally add the user include and library directories if requested if self.user: user_include = os.path.join(site.USER_BASE, "include") user_lib = os.path.join(site.USER_BASE, "lib") if os.path.isdir(user_include): self.include_dirs.append(user_include) if os.path.isdir(user_lib): self.library_dirs.append(user_lib) self.rpath.append(user_lib) def run(self): from packaging.compiler import new_compiler if not self.extensions: return # If we were asked to build any C/C++ libraries, make sure that the # directory where we put them is in the library search path for # linking extensions. if self.distribution.has_c_libraries(): build_clib = self.get_finalized_command('build_clib') self.libraries.extend(build_clib.get_library_names() or []) self.library_dirs.append(build_clib.build_clib) # Setup the CCompiler object that we'll use to do all the # compiling and linking self.compiler_obj = new_compiler(compiler=self.compiler, dry_run=self.dry_run, force=self.force) customize_compiler(self.compiler_obj) # If we are cross-compiling, init the compiler now (if we are not # cross-compiling, init would not hurt, but people may rely on # late initialization of compiler even if they shouldn't...) if os.name == 'nt' and self.plat_name != get_platform(): self.compiler_obj.initialize(self.plat_name) # And make sure that any compile/link-related options (which might # come from the command line or from the setup script) are set in # that CCompiler object -- that way, they automatically apply to # all compiling and linking done here. if self.include_dirs is not None: self.compiler_obj.set_include_dirs(self.include_dirs) if self.define is not None: # 'define' option is a list of (name,value) tuples for name, value in self.define: self.compiler_obj.define_macro(name, value) if self.undef is not None: for macro in self.undef: self.compiler_obj.undefine_macro(macro) if self.libraries is not None: self.compiler_obj.set_libraries(self.libraries) if self.library_dirs is not None: self.compiler_obj.set_library_dirs(self.library_dirs) if self.rpath is not None: self.compiler_obj.set_runtime_library_dirs(self.rpath) if self.link_objects is not None: self.compiler_obj.set_link_objects(self.link_objects) # Now actually compile and link everything. self.build_extensions() def get_source_files(self): filenames = [] # Wouldn't it be neat if we knew the names of header files too... for ext in self.extensions: filenames.extend(ext.sources) return filenames def get_outputs(self): # And build the list of output (built) filenames. Note that this # ignores the 'inplace' flag, and assumes everything goes in the # "build" tree. outputs = [] for ext in self.extensions: outputs.append(self.get_ext_fullpath(ext.name)) return outputs def build_extensions(self): for ext in self.extensions: try: self.build_extension(ext) except (CCompilerError, PackagingError, CompileError) as e: if not ext.optional: raise logger.warning('%s: building extension %r failed: %s', self.get_command_name(), ext.name, e) def build_extension(self, ext): sources = ext.sources if sources is None or not isinstance(sources, (list, tuple)): raise PackagingSetupError(("in 'ext_modules' option (extension '%s'), " + "'sources' must be present and must be " + "a list of source filenames") % ext.name) sources = list(sources) ext_path = self.get_ext_fullpath(ext.name) depends = sources + ext.depends if not (self.force or newer_group(depends, ext_path, 'newer')): logger.debug("skipping '%s' extension (up-to-date)", ext.name) return else: logger.info("building '%s' extension", ext.name) # First, scan the sources for SWIG definition files (.i), run # SWIG on 'em to create .c files, and modify the sources list # accordingly. sources = self.swig_sources(sources, ext) # Next, compile the source code to object files. # XXX not honouring 'define_macros' or 'undef_macros' -- the # CCompiler API needs to change to accommodate this, and I # want to do one thing at a time! # Two possible sources for extra compiler arguments: # - 'extra_compile_args' in Extension object # - CFLAGS environment variable (not particularly # elegant, but people seem to expect it and I # guess it's useful) # The environment variable should take precedence, and # any sensible compiler will give precedence to later # command-line args. Hence we combine them in order: extra_args = ext.extra_compile_args or [] macros = ext.define_macros[:] for undef in ext.undef_macros: macros.append((undef,)) objects = self.compiler_obj.compile(sources, output_dir=self.build_temp, macros=macros, include_dirs=ext.include_dirs, debug=self.debug, extra_postargs=extra_args, depends=ext.depends) # XXX -- this is a Vile HACK! # # The setup.py script for Python on Unix needs to be able to # get this list so it can perform all the clean up needed to # avoid keeping object files around when cleaning out a failed # build of an extension module. Since Packaging does not # track dependencies, we have to get rid of intermediates to # ensure all the intermediates will be properly re-built. # self._built_objects = objects[:] # Now link the object files together into a "shared object" -- # of course, first we have to figure out all the other things # that go into the mix. if ext.extra_objects: objects.extend(ext.extra_objects) extra_args = ext.extra_link_args or [] # Detect target language, if not provided language = ext.language or self.compiler_obj.detect_language(sources) self.compiler_obj.link_shared_object( objects, ext_path, libraries=self.get_libraries(ext), library_dirs=ext.library_dirs, runtime_library_dirs=ext.runtime_library_dirs, extra_postargs=extra_args, export_symbols=self.get_export_symbols(ext), debug=self.debug, build_temp=self.build_temp, target_lang=language) def swig_sources(self, sources, extension): """Walk the list of source files in 'sources', looking for SWIG interface (.i) files. Run SWIG on all that are found, and return a modified 'sources' list with SWIG source files replaced by the generated C (or C++) files. """ new_sources = [] swig_sources = [] swig_targets = {} # XXX this drops generated C/C++ files into the source tree, which # is fine for developers who want to distribute the generated # source -- but there should be an option to put SWIG output in # the temp dir. if ('-c++' in self.swig_opts or '-c++' in extension.swig_opts): target_ext = '.cpp' else: target_ext = '.c' for source in sources: base, ext = os.path.splitext(source) if ext == ".i": # SWIG interface file new_sources.append(base + '_wrap' + target_ext) swig_sources.append(source) swig_targets[source] = new_sources[-1] else: new_sources.append(source) if not swig_sources: return new_sources swig = self.swig or self.find_swig() swig_cmd = [swig, "-python"] swig_cmd.extend(self.swig_opts) # Do not override commandline arguments if not self.swig_opts: for o in extension.swig_opts: swig_cmd.append(o) for source in swig_sources: target = swig_targets[source] logger.info("swigging %s to %s", source, target) self.spawn(swig_cmd + ["-o", target, source]) return new_sources def find_swig(self): """Return the name of the SWIG executable. On Unix, this is just "swig" -- it should be in the PATH. Tries a bit harder on Windows. """ if os.name == "posix": return "swig" elif os.name == "nt": # Look for SWIG in its standard installation directory on # Windows (or so I presume!). If we find it there, great; # if not, act like Unix and assume it's in the PATH. for vers in ("1.3", "1.2", "1.1"): fn = os.path.join("c:\\swig%s" % vers, "swig.exe") if os.path.isfile(fn): return fn else: return "swig.exe" elif os.name == "os2": # assume swig available in the PATH. return "swig.exe" else: raise PackagingPlatformError(("I don't know how to find (much less run) SWIG " "on platform '%s'") % os.name) # -- Name generators ----------------------------------------------- # (extension names, filenames, whatever) def get_ext_fullpath(self, ext_name): """Returns the path of the filename for a given extension. The file is located in `build_lib` or directly in the package (inplace option). """ fullname = self.get_ext_fullname(ext_name) modpath = fullname.split('.') filename = self.get_ext_filename(modpath[-1]) if not self.inplace: # no further work needed # returning : # build_dir/package/path/filename filename = os.path.join(*modpath[:-1]+[filename]) return os.path.join(self.build_lib, filename) # the inplace option requires to find the package directory # using the build_py command for that package = '.'.join(modpath[0:-1]) build_py = self.get_finalized_command('build_py') package_dir = os.path.abspath(build_py.get_package_dir(package)) # returning # package_dir/filename return os.path.join(package_dir, filename) def get_ext_fullname(self, ext_name): """Returns the fullname of a given extension name. Adds the `package.` prefix""" if self.package is None: return ext_name else: return self.package + '.' + ext_name def get_ext_filename(self, ext_name): r"""Convert the name of an extension (eg. "foo.bar") into the name of the file from which it will be loaded (eg. "foo/bar.so", or "foo\bar.pyd"). """ ext_path = ext_name.split('.') # OS/2 has an 8 character module (extension) limit :-( if os.name == "os2": ext_path[len(ext_path) - 1] = ext_path[len(ext_path) - 1][:8] # extensions in debug_mode are named 'module_d.pyd' under windows so_ext = sysconfig.get_config_var('SO') if os.name == 'nt' and self.debug: return os.path.join(*ext_path) + '_d' + so_ext return os.path.join(*ext_path) + so_ext def get_export_symbols(self, ext): """Return the list of symbols that a shared extension has to export. This either uses 'ext.export_symbols' or, if it's not provided, "init" + module_name. Only relevant on Windows, where the .pyd file (DLL) must export the module "init" function. """ initfunc_name = "PyInit_" + ext.name.split('.')[-1] if initfunc_name not in ext.export_symbols: ext.export_symbols.append(initfunc_name) return ext.export_symbols def get_libraries(self, ext): """Return the list of libraries to link against when building a shared extension. On most platforms, this is just 'ext.libraries'; on Windows and OS/2, we add the Python library (eg. python20.dll). """ # The python library is always needed on Windows. For MSVC, this # is redundant, since the library is mentioned in a pragma in # pyconfig.h that MSVC groks. The other Windows compilers all seem # to need it mentioned explicitly, though, so that's what we do. # Append '_d' to the python import library on debug builds. if sys.platform == "win32": from packaging.compiler.msvccompiler import MSVCCompiler if not isinstance(self.compiler_obj, MSVCCompiler): template = "python%d%d" if self.debug: template = template + '_d' pythonlib = template % sys.version_info[:2] # don't extend ext.libraries, it may be shared with other # extensions, it is a reference to the original list return ext.libraries + [pythonlib] else: return ext.libraries elif sys.platform == "os2emx": # EMX/GCC requires the python library explicitly, and I # believe VACPP does as well (though not confirmed) - AIM Apr01 template = "python%d%d" # debug versions of the main DLL aren't supported, at least # not at this time - AIM Apr01 #if self.debug: # template = template + '_d' pythonlib = template % sys.version_info[:2] # don't extend ext.libraries, it may be shared with other # extensions, it is a reference to the original list return ext.libraries + [pythonlib] elif sys.platform[:6] == "cygwin": template = "python%d.%d" pythonlib = template % sys.version_info[:2] # don't extend ext.libraries, it may be shared with other # extensions, it is a reference to the original list return ext.libraries + [pythonlib] elif sys.platform[:6] == "atheos": template = "python%d.%d" pythonlib = template % sys.version_info[:2] # Get SHLIBS from Makefile extra = [] for lib in sysconfig.get_config_var('SHLIBS').split(): if lib.startswith('-l'): extra.append(lib[2:]) else: extra.append(lib) # don't extend ext.libraries, it may be shared with other # extensions, it is a reference to the original list return ext.libraries + [pythonlib, "m"] + extra elif sys.platform == 'darwin': # Don't use the default code below return ext.libraries else: if sysconfig.get_config_var('Py_ENABLE_SHARED'): template = 'python%d.%d' + sys.abiflags pythonlib = template % sys.version_info[:2] return ext.libraries + [pythonlib] else: return ext.libraries