giscanner: Use Distutils to Build Dumper Program

Add compile() and link() functions in ccompiler.py to call
distutils.ccompiler's compiler() and link() functions, so that we
can in turn call them from dumper.py as needed.  Note that for
linking the dumper program when building with libtool, we are still
using the original method of constructing the link command line
and running that command line, as distutils and libtool do not get
along well with each other.  For non-libtool builds, such as MSVC
builds, we would link the dumper program using distutils.

For MSVC builds, we need to ignore mt.exe failing to find a
.exe.manifest file as Visual Studio 2010+ do not generate such files
during linking, and it is done by distutils as Python 2.7.x is built
with Visual Studio 2008, which do generate such manifest files during
the build.

https://bugzilla.gnome.org/show_bug.cgi?id=753428

2 files changed, 172 insertions, 119 deletions

diff --git a/giscanner/ccompiler.py b/giscanner/ccompiler.py
index 438f847e..85ff0268 100644
--- a/giscanner/ccompiler.py
+++ b/giscanner/ccompiler.py
@@ -72,7 +72,7 @@ class CCompiler(object):
             self.compiler = distutils.ccompiler.new_compiler(compiler=compiler_name)
         customize_compiler(self.compiler)
 
-        # customize_compiler from distutils only does customization
+        # customize_compiler() from distutils only does customization
         # for 'unix' compiler type.  Also, avoid linking to msvcrxx.dll
         # for MinGW builds as the dumper binary does not link to the
         # Python DLL, but link to msvcrt.dll if necessary.
@@ -110,45 +110,60 @@ class CCompiler(object):
         # An "internal" link is where the library to be introspected
         # is being built in the current directory.
 
-        # Search the current directory first
-        # (This flag is not supported nor needed for Visual C++)
-        if not self.check_is_msvc():
-            args.append('-L.')
+        if not libtool:
+            # non-libtool case: prepare distutils use
+            if self.check_is_msvc():
+                for library in libraries:
+                    # MSVC Builds don't use libtool, so no .la libraries,
+                    # so just add the library directly.
+                    self.compiler.add_library(library)
+                    for libpath in libpaths:
+                        self.compiler.add_library_dir(libpath)
+            else:
+                # Search the current directory first
+                # (This flag is not supported nor needed for Visual C++)
+                self.compiler.add_library_dir('.')
+                if os.name != 'nt':
+                    self.compiler.add_runtime_library_dir('.')
 
-            # https://bugzilla.gnome.org/show_bug.cgi?id=625195
-            if not libtool:
+                # https://bugzilla.gnome.org/show_bug.cgi?id=625195
                 args.append('-Wl,-rpath=.')
+
+                # Ensure libraries are always linked as we are going to use ldd to work
+                # out their names later
                 args.append('-Wl,--no-as-needed')
 
-        for library in libraries:
-            if self.check_is_msvc():
-                args.append(library + '.lib')
-            else:
+            for library in libraries:
+                self.compiler.add_library(library)
+            if not self.check_is_msvc():
+                for library_path in libpaths:
+                    args.append('-L' + library_path)
+                    if os.path.isabs(library_path):
+                        args.append('-Wl,-rpath=' + library_path)
+
+        else:
+            # libtool case: assemble linker command arguments, like we did before
+            args.append('-L.')
+            for library in libraries:
                 if library.endswith(".la"):  # explicitly specified libtool library
                     args.append(library)
                 else:
                     args.append('-l' + library)
 
-        for library_path in libpaths:
-            # Not used/needed on Visual C++, and -Wl,-rpath options
-            # will cause grief
-            if not self.check_is_msvc():
+            for library_path in libpaths:
                 args.append('-L' + library_path)
                 if os.path.isabs(library_path):
-                    if libtool:
-                        args.append('-rpath')
-                        args.append(library_path)
-                    else:
-                        args.append('-Wl,-rpath=' + library_path)
+                    args.append('-rpath')
+                    args.append(library_path)
 
-    def get_external_link_flags(self, args, libraries):
+    def get_external_link_flags(self, args, libtool, libraries):
         # An "external" link is where the library to be introspected
         # is installed on the system; this case is used for the scanning
         # of GLib in gobject-introspection itself.
 
         for library in libraries:
-            if self.check_is_msvc():
-                args.append(library + '.lib')
+            if not libtool:
+                self.compiler.add_library(library)
             else:
                 if library.endswith(".la"):  # explicitly specified libtool library
                     args.append(library)
@@ -180,6 +195,49 @@ class CCompiler(object):
                                  include_dirs=include_dirs,
                                  extra_postargs=extra_postargs)
 
+    def compile(self, pkg_config_cflags, cpp_includes, source, init_sections):
+        extra_postargs = []
+        includes = []
+        source_str = ''.join(source)
+        tmpdir_idx = source_str.rfind(os.sep, 0, source_str.rfind(os.sep))
+        (include_paths, macros, extra_args) = \
+            self._set_cpp_options(pkg_config_cflags)
+
+        for include in cpp_includes:
+            includes.append(include)
+
+        # Do not add -Wall when using init code as we do not include any
+        # header of the library being introspected
+        if self.compiler_cmd == 'gcc' and not init_sections:
+            extra_postargs.append('-Wall')
+        extra_postargs.append(self._cflags_no_deprecation_warnings)
+
+        includes.extend(include_paths)
+        extra_postargs.extend(extra_args)
+
+        return self.compiler.compile(sources=source,
+                                     macros=macros,
+                                     include_dirs=includes,
+                                     extra_postargs=extra_postargs,
+                                     output_dir=source_str[tmpdir_idx + 1:
+                                                           source_str.rfind(os.sep)])
+
+    def link(self, output, objects, lib_args):
+        # Note: This is used for non-libtool builds only!
+        extra_preargs = []
+        extra_postargs = []
+        library_dirs = []
+        libraries = []
+
+        for arg in lib_args:
+            extra_postargs.append(arg)
+
+        self.compiler.link(target_desc=self.compiler.EXECUTABLE,
+                           objects=objects,
+                           output_filename=output,
+                           extra_preargs=extra_preargs,
+                           extra_postargs=extra_postargs)
+
     def resolve_windows_libs(self, libraries, options):
         args = []
         libsearch = []
diff --git a/giscanner/dumper.py b/giscanner/dumper.py
index 3a7ced6f..18725c8a 100644
--- a/giscanner/dumper.py
+++ b/giscanner/dumper.py
@@ -24,6 +24,7 @@ import sys
 import subprocess
 import shutil
 import tempfile
+from distutils.errors import LinkError
 
 from .gdumpparser import IntrospectionBinary
 from . import utils
@@ -92,7 +93,10 @@ class DumpCompiler(object):
         self._uninst_srcdir = os.environ.get('UNINSTALLED_INTROSPECTION_SRCDIR')
         self._packages = ['gio-2.0 gmodule-2.0']
         self._packages.extend(options.packages)
-        self._linker_cmd = os.environ.get('CC', 'cc')
+        if hasattr(self._compiler.compiler, 'linker_exe'):
+            self._linker_cmd = self._compiler.compiler.linker_exe
+        else:
+            self._linker_cmd = []
 
     # Public API
 
@@ -147,29 +151,22 @@ class DumpCompiler(object):
                     f.write("  " + func)
                 f.write("\n};\n")
 
-        # Microsoft compilers generate intermediate .obj files
-        # during compilation, unlike .o files like GCC and others
-        if self._compiler.check_is_msvc():
-            o_path = self._generate_tempfile(tmpdir, '.obj')
-        else:
-            o_path = self._generate_tempfile(tmpdir, '.o')
-
-        if os.name == 'nt':
-            ext = '.exe'
+        if self._compiler.compiler.exe_extension:
+            ext = self._compiler.compiler.exe_extension
         else:
             ext = ''
 
         bin_path = self._generate_tempfile(tmpdir, ext)
 
         try:
-            self._compile(o_path, c_path)
+            introspection_obj = self._compile(c_path)
         except CompilerError as e:
             if not utils.have_debug_flag('save-temps'):
                 shutil.rmtree(tmpdir)
             raise SystemExit('compilation of temporary binary failed:' + str(e))
 
         try:
-            self._link(bin_path, o_path)
+            self._link(bin_path, introspection_obj)
         except LinkerError as e:
             if not utils.have_debug_flag('save-temps'):
                 shutil.rmtree(tmpdir)
@@ -196,91 +193,59 @@ class DumpCompiler(object):
             stdout=subprocess.PIPE)
         return proc.communicate()[0].split()
 
-    def _compile(self, output, *sources):
-        # Not strictly speaking correct, but easier than parsing shell
-        args = self._compiler.compiler_cmd.split()
-        # Do not add -Wall when using init code as we do not include any
-        # header of the library being introspected
-        if self._compiler.compiler_cmd == 'gcc' and not self._options.init_sections:
-            args.append('-Wall')
-        # The Microsoft compiler uses different option flags for
-        # silencing warnings on deprecated function usage
-        if self._compiler.check_is_msvc():
-            args.append("-wd4996")
-        else:
-            args.append("-Wno-deprecated-declarations")
+    def _compile(self, *sources):
         pkgconfig_flags = self._run_pkgconfig('--cflags')
-        args.extend([utils.cflag_real_include_path(f) for f in pkgconfig_flags])
-        cppflags = os.environ.get('CPPFLAGS', '')
-        for cppflag in cppflags.split():
-            args.append(cppflag)
-        cflags = os.environ.get('CFLAGS', '')
-        for cflag in cflags.split():
-            args.append(cflag)
-        for include in self._options.cpp_includes:
-            args.append('-I' + include)
-        # The Microsoft compiler uses different option flags for
-        # compilation result output
-        if self._compiler.check_is_msvc():
-            args.extend(['-c', '-Fe' + output, '-Fo' + output])
-        else:
-            args.extend(['-c', '-o', output])
-        for source in sources:
-            if not os.path.exists(source):
-                raise CompilerError(
-                    "Could not find c source file: %s" % (source, ))
-        args.extend(list(sources))
-        if not self._options.quiet:
-            print "g-ir-scanner: compile: %s" % (
-                subprocess.list2cmdline(args), )
-            sys.stdout.flush()
-        try:
-            subprocess.check_call(args)
-        except subprocess.CalledProcessError as e:
-            raise CompilerError(e)
+        return self._compiler.compile(pkgconfig_flags,
+                                      self._options.cpp_includes,
+                                      sources,
+                                      self._options.init_sections)
 
-    def _link(self, output, *sources):
+    def _link(self, output, sources):
         args = []
         libtool = utils.get_libtool_command(self._options)
         if libtool:
+            # Note: MSVC Builds do not use libtool!
+            # In the libtool case, put together the linker command, as we did before.
+            # We aren't using distutils to link in this case.
             args.extend(libtool)
             args.append('--mode=link')
             args.append('--tag=CC')
             if self._options.quiet:
                 args.append('--silent')
 
-        args.extend(self._linker_cmd.split())
-        # We can use -o for the Microsoft compiler/linker,
-        # but it is considered deprecated usage with that
-        if self._compiler.check_is_msvc():
-            args.extend(['-Fe' + output])
-        else:
+            args.extend(self._linker_cmd)
+
             args.extend(['-o', output])
-        if libtool:
             if os.name == 'nt':
                 args.append('-Wl,--export-all-symbols')
             else:
                 args.append('-export-dynamic')
 
-        cppflags = os.environ.get('CPPFLAGS', '')
-        for cppflag in cppflags.split():
-            args.append(cppflag)
-        cflags = os.environ.get('CFLAGS', '')
-        for cflag in cflags.split():
-            args.append(cflag)
-        ldflags = os.environ.get('LDFLAGS', '')
-        for ldflag in ldflags.split():
-            args.append(ldflag)
+        if not self._compiler.check_is_msvc():
+            # These envvars are not used for MSVC Builds!
+            # MSVC Builds use the INCLUDE, LIB envvars,
+            # which are automatically picked up during
+            # compilation and linking
+            cppflags = os.environ.get('CPPFLAGS', '')
+            for cppflag in cppflags.split():
+                args.append(cppflag)
+            cflags = os.environ.get('CFLAGS', '')
+            for cflag in cflags.split():
+                args.append(cflag)
+            ldflags = os.environ.get('LDFLAGS', '')
+            for ldflag in ldflags.split():
+                args.append(ldflag)
 
         # Make sure to list the library to be introspected first since it's
         # likely to be uninstalled yet and we want the uninstalled RPATHs have
         # priority (or we might run with installed library that is older)
-
         for source in sources:
             if not os.path.exists(source):
                 raise CompilerError(
                     "Could not find object file: %s" % (source, ))
-        args.extend(list(sources))
+
+        if libtool:
+            args.extend(sources)
 
         pkg_config_libs = self._run_pkgconfig('--libs')
 
@@ -293,31 +258,61 @@ class DumpCompiler(object):
 
         else:
             args.extend(pkg_config_libs)
-            self._compiler.get_external_link_flags(args, self._options.libraries)
-
-        if not self._options.quiet:
-            print "g-ir-scanner: link: %s" % (
-                subprocess.list2cmdline(args), )
-            sys.stdout.flush()
-        msys = os.environ.get('MSYSTEM', None)
-        if msys:
-            shell = os.environ.get('SHELL', 'sh.exe')
-            # Create a temporary script file that
-            # runs the command we want
-            tf, tf_name = tempfile.mkstemp()
-            with os.fdopen(tf, 'wb') as f:
-                shellcontents = ' '.join([x.replace('\\', '/') for x in args])
-                fcontents = '#!/bin/sh\nunset PWD\n{}\n'.format(shellcontents)
-                f.write(fcontents)
-            shell = utils.which(shell)
-            args = [shell, tf_name.replace('\\', '/')]
-        try:
-            subprocess.check_call(args)
-        except subprocess.CalledProcessError as e:
-            raise LinkerError(e)
-        finally:
+            self._compiler.get_external_link_flags(args,
+                                                   libtool,
+                                                   self._options.libraries)
+
+        if not libtool:
+            # non-libtool: prepare distutils for linking the introspection
+            # dumper program...
+            try:
+                self._compiler.link(output,
+                                    sources,
+                                    args)
+
+            # Ignore failing to embed the manifest files, when the manifest
+            # file does not exist, especially for MSVC 2010 and later builds.
+            # If we are on Visual C++ 2005/2008, where
+            # this embedding is required, the build will fail anyway, as
+            # the dumper program will likely fail to run, and this means
+            # something went wrong with the build.
+            except LinkError, e:
+                if self._compiler.check_is_msvc():
+                    msg = str(e)
+
+                    if msg[msg.rfind('mt.exe'):] == 'mt.exe\' failed with exit status 31':
+                        sys.exc_clear()
+                        pass
+                    else:
+                        raise LinkError(e)
+                else:
+                    raise LinkError(e)
+        else:
+            # libtool: Run the assembled link command, we don't use distutils
+            # for linking here.
+            if not self._options.quiet:
+                print "g-ir-scanner: link: %s" % (
+                    subprocess.list2cmdline(args), )
+                sys.stdout.flush()
+            msys = os.environ.get('MSYSTEM', None)
             if msys:
-                os.remove(tf_name)
+                shell = os.environ.get('SHELL', 'sh.exe')
+                # Create a temporary script file that
+                # runs the command we want
+                tf, tf_name = tempfile.mkstemp()
+                with os.fdopen(tf, 'wb') as f:
+                    shellcontents = ' '.join([x.replace('\\', '/') for x in args])
+                    fcontents = '#!/bin/sh\nunset PWD\n{}\n'.format(shellcontents)
+                    f.write(fcontents)
+                shell = utils.which(shell)
+                args = [shell, tf_name.replace('\\', '/')]
+            try:
+                subprocess.check_call(args)
+            except subprocess.CalledProcessError as e:
+                raise LinkerError(e)
+            finally:
+                if msys:
+                    os.remove(tf_name)
 
 
 def compile_introspection_binary(options, get_type_functions,