diff options
author | Pearu Peterson <pearu.peterson@gmail.com> | 2007-05-18 16:41:44 +0000 |
---|---|---|
committer | Pearu Peterson <pearu.peterson@gmail.com> | 2007-05-18 16:41:44 +0000 |
commit | b3caec68294618d217bdb26872b3c9d235c6ade6 (patch) | |
tree | af56df505e0cbb0ba2a4b29ebdb85024d9f2e5df /numpy/distutils | |
parent | 5fcecff23893db08dd325501cdfa61cccd55688d (diff) | |
download | numpy-b3caec68294618d217bdb26872b3c9d235c6ade6.tar.gz |
Extension modules and libraries are built with suitable compilers/linkers. Improved failure handling.
Diffstat (limited to 'numpy/distutils')
-rw-r--r-- | numpy/distutils/command/build_clib.py | 108 | ||||
-rw-r--r-- | numpy/distutils/command/build_ext.py | 364 |
2 files changed, 295 insertions, 177 deletions
diff --git a/numpy/distutils/command/build_clib.py b/numpy/distutils/command/build_clib.py index 427b52dc3..fd4d41cde 100644 --- a/numpy/distutils/command/build_clib.py +++ b/numpy/distutils/command/build_clib.py @@ -1,8 +1,9 @@ """ Modified version of build_clib that handles fortran source files. """ +import os from distutils.command.build_clib import build_clib as old_build_clib -from distutils.errors import DistutilsSetupError +from distutils.errors import DistutilsSetupError, DistutilsError from numpy.distutils import log from distutils.dep_util import newer_group @@ -89,6 +90,7 @@ class build_clib(old_build_clib): self.fcompiler.show_customization() self.build_libraries(self.libraries) + return def get_source_files(self): self.check_library_list(self.libraries) @@ -111,9 +113,21 @@ class build_clib(old_build_clib): "a list of source filenames") % lib_name sources = list(sources) + c_sources, cxx_sources, f_sources, fmodule_sources \ + = filter_sources(sources) + requiref90 = not not fmodule_sources or \ + build_info.get('language','c')=='f90' + + # save source type information so that build_ext can use it. + source_languages = [] + if c_sources: source_languages.append('c') + if cxx_sources: source_languages.append('c++') + if requiref90: source_languages.append('f90') + elif f_sources: source_languages.append('f77') + build_info['source_languages'] = source_languages + lib_file = compiler.library_filename(lib_name, output_dir=self.build_clib) - depends = sources + build_info.get('depends',[]) if not (self.force or newer_group(depends, lib_file, 'newer')): log.debug("skipping '%s' library (up-to-date)", lib_name) @@ -121,15 +135,13 @@ class build_clib(old_build_clib): else: log.info("building '%s' library", lib_name) - config_fc = build_info.get('config_fc',{}) if fcompiler is not None and config_fc: log.info('using additional config_fc from setup script '\ 'for fortran compiler: %s' \ % (config_fc,)) from numpy.distutils.fcompiler import new_fcompiler - requiref90 = build_info.get('language','c')=='f90' - fcompiler = new_fcompiler(compiler=self.fcompiler.compiler_type, + fcompiler = new_fcompiler(compiler=fcompiler.compiler_type, verbose=self.verbose, dry_run=self.dry_run, force=self.force, @@ -139,23 +151,26 @@ class build_clib(old_build_clib): base_config_fc.update(config_fc) fcompiler.customize(base_config_fc) + # check availability of Fortran compilers + if (f_sources or fmodule_sources) and fcompiler is None: + raise DistutilsError, "library %s has Fortran sources"\ + " but no Fortran compiler found" % (lib_name) + macros = build_info.get('macros') include_dirs = build_info.get('include_dirs') extra_postargs = build_info.get('extra_compiler_args') or [] - c_sources, cxx_sources, f_sources, fmodule_sources \ - = filter_sources(sources) - - if self.compiler.compiler_type=='msvc': + # where compiled F90 module files are: + module_dirs = build_info.get('module_dirs') or [] + module_build_dir = os.path.dirname(lib_file) + if requiref90: self.mkpath(module_build_dir) + + if compiler.compiler_type=='msvc': # this hack works around the msvc compiler attributes # problem, msvc uses its own convention :( c_sources += cxx_sources cxx_sources = [] - if fmodule_sources: - print 'XXX: Fortran 90 module support not implemented or tested' - f_sources.extend(fmodule_sources) - objects = [] if c_sources: log.info("compiling C sources") @@ -176,24 +191,67 @@ class build_clib(old_build_clib): debug=self.debug, extra_postargs=extra_postargs) objects.extend(cxx_objects) + + if f_sources or fmodule_sources: + extra_postargs = [] + f_objects = [] + + if requiref90: + if fcompiler.module_dir_switch is None: + existing_modules = glob('*.mod') + extra_postargs += fcompiler.module_options(\ + module_dirs,module_build_dir) + + if fmodule_sources: + log.info("compiling Fortran 90 module sources") + f_objects += fcompiler.compile(fmodule_sources, + output_dir=self.build_temp, + macros=macros, + include_dirs=include_dirs, + debug=self.debug, + extra_postargs=extra_postargs) + + if requiref90 and self.fcompiler.module_dir_switch is None: + # move new compiled F90 module files to module_build_dir + for f in glob('*.mod'): + if f in existing_modules: + continue + t = os.path.join(module_build_dir, f) + if os.path.abspath(f)==os.path.abspath(t): + continue + if os.path.isfile(t): + os.remove(t) + try: + self.move_file(f, module_build_dir) + except DistutilsFileError: + log.warn('failed to move %r to %r' \ + % (f, module_build_dir)) + + if f_sources: + log.info("compiling Fortran sources") + f_objects += fcompiler.compile(f_sources, + output_dir=self.build_temp, + macros=macros, + include_dirs=include_dirs, + debug=self.debug, + extra_postargs=extra_postargs) + else: + f_objects = [] - if f_sources: - log.info("compiling Fortran sources") - f_objects = fcompiler.compile(f_sources, - output_dir=self.build_temp, - macros=macros, - include_dirs=include_dirs, - debug=self.debug, - extra_postargs=[]) - objects.extend(f_objects) + objects.extend(f_objects) - self.compiler.create_static_lib(objects, lib_name, - output_dir=self.build_clib, - debug=self.debug) + # assume that default linker is suitable for + # linking Fortran object files + compiler.create_static_lib(objects, lib_name, + output_dir=self.build_clib, + debug=self.debug) + # fix library dependencies clib_libraries = build_info.get('libraries',[]) for lname, binfo in libraries: if lname in clib_libraries: clib_libraries.extend(binfo[1].get('libraries',[])) if clib_libraries: build_info['libraries'] = clib_libraries + return +#EOF diff --git a/numpy/distutils/command/build_ext.py b/numpy/distutils/command/build_ext.py index 466462c56..17e60e1ab 100644 --- a/numpy/distutils/command/build_ext.py +++ b/numpy/distutils/command/build_ext.py @@ -8,7 +8,8 @@ from glob import glob from distutils.dep_util import newer_group from distutils.command.build_ext import build_ext as old_build_ext -from distutils.errors import DistutilsFileError, DistutilsSetupError +from distutils.errors import DistutilsFileError, DistutilsSetupError,\ + DistutilsError from distutils.file_util import copy_file from numpy.distutils import log @@ -52,7 +53,7 @@ class build_ext (old_build_ext): if self.distribution.has_c_libraries(): self.run_command('build_clib') build_clib = self.get_finalized_command('build_clib') - self.library_dirs.append(build_clib.build_clib) + self.library_dirs.append(build_clib.build_clib) else: build_clib = None @@ -61,60 +62,142 @@ class build_ext (old_build_ext): # bogus linking commands. Extensions must # explicitly specify the C libraries that they use. - # Determine if Fortran compiler is needed. - if build_clib and build_clib.fcompiler is not None: - need_f_compiler = 1 - else: - need_f_compiler = 0 - for ext in self.extensions: - if has_f_sources(ext.sources): - need_f_compiler = 1 - break - if getattr(ext,'language','c') in ['f77','f90']: - need_f_compiler = 1 - break - - requiref90 = 0 - if need_f_compiler: - for ext in self.extensions: - if getattr(ext,'language','c')=='f90': - requiref90 = 1 - break - - # Determine if C++ compiler is needed. - need_cxx_compiler = 0 - for ext in self.extensions: - if has_cxx_sources(ext.sources): - need_cxx_compiler = 1 - break - if getattr(ext,'language','c')=='c++': - need_cxx_compiler = 1 - break - from distutils.ccompiler import new_compiler - self.compiler = new_compiler(compiler=self.compiler, + from numpy.distutils.fcompiler import new_fcompiler + + compiler_type = self.compiler + # Initialize C compiler: + self.compiler = new_compiler(compiler=compiler_type, verbose=self.verbose, dry_run=self.dry_run, force=self.force) - self.compiler.customize(self.distribution,need_cxx=need_cxx_compiler) + self.compiler.customize(self.distribution) self.compiler.customize_cmd(self) self.compiler.show_customization() - # Initialize Fortran/C++ compilers if needed. - if need_f_compiler: - from numpy.distutils.fcompiler import new_fcompiler - self.fcompiler = new_fcompiler(compiler=self.fcompiler, - verbose=self.verbose, - dry_run=self.dry_run, - force=self.force, - requiref90=requiref90) - if self.fcompiler.get_version(): - self.fcompiler.customize(self.distribution) - self.fcompiler.customize_cmd(self) - self.fcompiler.show_customization() + # Create mapping of libraries built by build_clib: + clibs = {} + if build_clib is not None: + for libname,build_info in build_clib.libraries or []: + if clibs.has_key(libname): + log.warn('library %r defined more than once,'\ + ' overwriting build_info %r with %r.' \ + % (libname, clibs[libname], build_info)) + clibs[libname] = build_info + # .. and distribution libraries: + for libname,build_info in self.distribution.libraries or []: + if clibs.has_key(libname): + # build_clib libraries have a precedence before distribution ones + continue + clibs[libname] = build_info + + # Determine if C++/Fortran 77/Fortran 90 compilers are needed. + # Update extension libraries, library_dirs, and macros. + all_languages = [] + for ext in self.extensions: + ext_languages = [] + c_libs = [] + c_lib_dirs = [] + macros = [] + for libname in ext.libraries: + if clibs.has_key(libname): + binfo = clibs[libname] + c_libs += binfo.get('libraries',[]) + c_lib_dirs += binfo.get('library_dirs',[]) + for m in binfo.get('macros',[]): + if m not in macros: macros.append(m) + for l in clibs.get(libname,{}).get('source_languages',[]): + if l not in ext_languages: ext_languages.append(l) + if c_libs: + new_c_libs = ext.libraries + c_libs + log.info('updating extension %r libraries from %r to %r' \ + % (ext.name, ext.libraries, new_c_libs)) + ext.libraries = new_c_libs + ext.library_dirs = ext.library_dirs + c_lib_dirs + if macros: + log.info('extending extension %r defined_macros with %r' \ + % (ext.name, macros)) + ext.define_macros = ext.define_macros + macros + + # determine extension languages + if 'f77' not in ext_languages and has_f_sources(ext.sources): + ext_languages.append('f77') + if 'c++' not in ext_languages and has_cxx_sources(ext.sources): + ext_languages.append('c++') + if sys.version[:3]>='2.3': + l = ext.language or self.compiler.detect_language(ext.sources) + else: + l = ext.language + if l and l not in ext_languages: ext_languages.append(l) + # reset language attribute for choosing proper linker + if 'c++' in ext_languages: + ext_language = 'c++' + elif 'f90' in ext_languages: + ext_language = 'f90' + elif 'f77' in ext_languages: + ext_language = 'f77' + else: + ext_language = 'c' # default + if l and l!=ext_language: + log.warn('resetting extension %r language from %r to %r.' % (ext.name,l,ext_language)) + ext.language = ext_language + # global language + for l in ext_languages: + if l not in all_languages: all_languages.append(l) + + need_f90_compiler = 'f90' in all_languages + need_f77_compiler = 'f77' in all_languages + need_cxx_compiler = 'c++' in all_languages + + # Initialize C++ compiler: + if need_cxx_compiler: + self._cxx_compiler = new_compiler(compiler=compiler_type, + verbose=self.verbose, + dry_run=self.dry_run, + force=self.force) + compiler = self._cxx_compiler + compiler.customize(self.distribution,need_cxx=need_cxx_compiler) + compiler.customize_cmd(self) + compiler.show_customization() + self._cxx_compiler = compiler.cxx_compiler() + else: + self._cxx_compiler = None + + # Initialize Fortran 77 compiler: + if need_f77_compiler: + self._f77_compiler = new_fcompiler(compiler=self.fcompiler, + verbose=self.verbose, + dry_run=self.dry_run, + force=self.force, + requiref90=False) + fcompiler = self._f77_compiler + if fcompiler.get_version(): + fcompiler.customize(self.distribution) + fcompiler.customize_cmd(self) + fcompiler.show_customization() + else: + self.warn('f77_compiler=%s is not available.' % (fcompiler.compiler_type)) + self._f77_compiler = None + else: + self._f77_compiler = None + + # Initialize Fortran 90 compiler: + if need_f90_compiler: + self._f90_compiler = new_fcompiler(compiler=self.fcompiler, + verbose=self.verbose, + dry_run=self.dry_run, + force=self.force, + requiref90=True) + fcompiler = self._f90_compiler + if fcompiler.get_version(): + fcompiler.customize(self.distribution) + fcompiler.customize_cmd(self) + fcompiler.show_customization() else: - self.warn('fcompiler=%s is not available.' % (self.fcompiler.compiler_type)) - self.fcompiler = None + self.warn('f90_compiler=%s is not available.' % (fcompiler.compiler_type)) + self._f90_compiler = None + else: + self._f90_compiler = None # Build extensions self.build_extensions() @@ -162,17 +245,11 @@ class build_ext (old_build_ext): for undef in ext.undef_macros: macros.append((undef,)) - clib_libraries = [] - clib_library_dirs = [] - if self.distribution.libraries: - for libname,build_info in self.distribution.libraries: - if libname in ext.libraries: - macros.extend(build_info.get('macros',[])) - clib_libraries.extend(build_info.get('libraries',[])) - clib_library_dirs.extend(build_info.get('library_dirs',[])) - c_sources, cxx_sources, f_sources, fmodule_sources = \ filter_sources(ext.sources) + + + if self.compiler.compiler_type=='msvc': if cxx_sources: # Needed to compile kiva.agg._agg extension. @@ -182,6 +259,28 @@ class build_ext (old_build_ext): c_sources += cxx_sources cxx_sources = [] + # Set Fortran/C++ compilers for compilation and linking. + if ext.language=='f90': + fcompiler = self._f90_compiler + elif ext.language=='f77': + fcompiler = self._f77_compiler + else: # in case ext.language is c++, for instance + fcompiler = self._f90_compiler or self._f77_compiler + cxx_compiler = self._cxx_compiler + + # check for the availability of required compilers + if cxx_sources and cxx_compiler is None: + raise DistutilsError, "extension %r has C++ sources" \ + "but no C++ compiler found" % (ext.name) + if (f_sources or fmodule_sources) and fcompiler is None: + raise DistutilsError, "extension %r has Fortran sources " \ + "but no Fortran compiler found" % (ext.name) + if ext.language in ['f77','f90'] and fcompiler is None: + self.warn("extension %r has Fortran libraries " \ + "but no Fortran linker found, using default linker" % (ext.name)) + if ext.language=='c++' and cxx_compiler is None: + self.warn("extension %r has C++ libraries " \ + "but no C++ linker found, using default linker" % (ext.name)) kws = {'depends':ext.depends} output_dir = self.build_temp @@ -198,11 +297,9 @@ class build_ext (old_build_ext): debug=self.debug, extra_postargs=extra_args, **kws) - if cxx_sources: - log.info("compiling C++ sources") - - cxx_compiler = self.compiler.cxx_compiler() + if cxx_sources: + log.info("compiling C++ sources") c_objects += cxx_compiler.compile(cxx_sources, output_dir=output_dir, macros=macros, @@ -211,127 +308,89 @@ class build_ext (old_build_ext): extra_postargs=extra_args, **kws) - check_for_f90_modules = not not fmodule_sources - - if f_sources or fmodule_sources: - extra_postargs = [] + extra_postargs = [] + f_objects = [] + if fmodule_sources: + log.info("compiling Fortran 90 module sources") module_dirs = ext.module_dirs[:] - - #if self.fcompiler.compiler_type=='ibm': - macros = [] - - if check_for_f90_modules: - module_build_dir = os.path.join(\ + module_build_dir = os.path.join(\ self.build_temp,os.path.dirname(\ self.get_ext_filename(fullname))) - - self.mkpath(module_build_dir) - if self.fcompiler.module_dir_switch is None: - existing_modules = glob('*.mod') - extra_postargs += self.fcompiler.module_options(\ + + self.mkpath(module_build_dir) + if fcompiler.module_dir_switch is None: + existing_modules = glob('*.mod') + extra_postargs += fcompiler.module_options(\ module_dirs,module_build_dir) - - f_objects = [] - if fmodule_sources: - log.info("compiling Fortran 90 module sources") - f_objects = self.fcompiler.compile(fmodule_sources, - output_dir=self.build_temp, - macros=macros, - include_dirs=include_dirs, - debug=self.debug, - extra_postargs=extra_postargs, - depends=ext.depends) - - if check_for_f90_modules \ - and self.fcompiler.module_dir_switch is None: + f_objects += fcompiler.compile(fmodule_sources, + output_dir=self.build_temp, + macros=macros, + include_dirs=include_dirs, + debug=self.debug, + extra_postargs=extra_postargs, + depends=ext.depends) + + if fcompiler.module_dir_switch is None: for f in glob('*.mod'): if f in existing_modules: continue + t = os.path.join(module_build_dir, f) + if os.path.abspath(f)==os.path.abspath(t): + continue + if os.path.isfile(t): + os.remove(t) try: self.move_file(f, module_build_dir) - except DistutilsFileError: # already exists in destination - os.remove(f) - - if f_sources: - log.info("compiling Fortran sources") - f_objects += self.fcompiler.compile(f_sources, - output_dir=self.build_temp, - macros=macros, - include_dirs=include_dirs, - debug=self.debug, - extra_postargs=extra_postargs, - depends=ext.depends) - else: - f_objects = [] + except DistutilsFileError: + log.warn('failed to move %r to %r' % (f, module_build_dir)) + if f_sources: + log.info("compiling Fortran sources") + f_objects += fcompiler.compile(f_sources, + output_dir=self.build_temp, + macros=macros, + include_dirs=include_dirs, + debug=self.debug, + extra_postargs=extra_postargs, + depends=ext.depends) objects = c_objects + f_objects if ext.extra_objects: objects.extend(ext.extra_objects) extra_args = ext.extra_link_args or [] - + libraries = self.get_libraries(ext)[:] + library_dirs = ext.library_dirs[:] + linker = self.compiler.link_shared_object - - use_fortran_linker = getattr(ext,'language','c') in ['f77','f90'] \ - and self.fcompiler is not None - c_libraries = [] - c_library_dirs = [] - if use_fortran_linker or f_sources: - use_fortran_linker = 1 - elif self.distribution.has_c_libraries(): - build_clib = self.get_finalized_command('build_clib') - f_libs = [] - for (lib_name, build_info) in build_clib.libraries: - if has_f_sources(build_info.get('sources',[])): - f_libs.append(lib_name) - if lib_name in ext.libraries: - # XXX: how to determine if c_libraries contain - # fortran compiled sources? - c_libraries.extend(build_info.get('libraries',[])) - c_library_dirs.extend(build_info.get('library_dirs',[])) - for l in ext.libraries: - if l in f_libs: - use_fortran_linker = 1 - break - # Always use system linker when using MSVC compiler. - if self.compiler.compiler_type=='msvc' and use_fortran_linker: - self._libs_with_msvc_and_fortran(c_libraries, c_library_dirs) - use_fortran_linker = False - - if use_fortran_linker: - if cxx_sources: - # XXX: Which linker should be used, Fortran or C++? - log.warn('mixing Fortran and C++ is untested') - linker = self.fcompiler.link_shared_object - language = ext.language or self.fcompiler.detect_language(f_sources) - else: - linker = self.compiler.link_shared_object - if sys.version[:3]>='2.3': - language = ext.language or self.compiler.detect_language(sources) - else: - language = ext.language - if cxx_sources: - linker = self.compiler.cxx_compiler().link_shared_object + if self.compiler.compiler_type=='msvc': + # expand libraries with fcompiler libraries as we are + # not using fcompiler linker + self._libs_with_msvc_and_fortran(fcompiler, libraries, library_dirs) + elif ext.language in ['f77','f90'] and fcompiler is not None: + linker = fcompiler.link_shared_object + if ext.language=='c++' and cxx_compiler is not None: + linker = cxx_compiler.link_shared_object if sys.version[:3]>='2.3': - kws = {'target_lang':language} + kws = {'target_lang':ext.language} else: kws = {} linker(objects, ext_filename, - libraries=self.get_libraries(ext) + c_libraries + clib_libraries, - library_dirs=ext.library_dirs+c_library_dirs+clib_library_dirs, + libraries=libraries, + library_dirs=library_dirs, runtime_library_dirs=ext.runtime_library_dirs, extra_postargs=extra_args, export_symbols=self.get_export_symbols(ext), debug=self.debug, build_temp=self.build_temp,**kws) + return - def _libs_with_msvc_and_fortran(self, c_libraries, c_library_dirs): + def _libs_with_msvc_and_fortran(self, fcompiler, c_libraries, c_library_dirs): # Always use system linker when using MSVC compiler. f_lib_dirs = [] - for dir in self.fcompiler.library_dirs: + for dir in fcompiler.library_dirs: # correct path when compiling in Cygwin but with normal Win # Python if dir.startswith('/usr/lib'): @@ -343,7 +402,7 @@ class build_ext (old_build_ext): # make g77-compiled static libs available to MSVC lib_added = False - for lib in self.fcompiler.libraries: + for lib in fcompiler.libraries: if not lib.startswith('msvcr'): c_libraries.append(lib) p = combine_paths(f_lib_dirs, 'lib' + lib + '.a') @@ -353,6 +412,7 @@ class build_ext (old_build_ext): if not lib_added: c_library_dirs.append(self.build_temp) lib_added = True + return def get_source_files (self): self.check_extensions_list(self.extensions) |