Merge pull request #2697 from mesonbuild/custom-target-depends-serialize

custom target: Consider all build depends while serializing

13 files changed, 267 insertions, 42 deletions

diff --git a/.appveyor.yml b/.appveyor.yml
index 46edbd4d9..2ef722f83 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -8,6 +8,14 @@ environment:
       compiler: msys2-mingw
       backend: ninja
 
+    - arch: x64
+      compiler: msys2-mingw
+      backend: ninja
+
+    - arch: x64
+      compiler: cygwin
+      backend: ninja
+
     - arch: x86
       compiler: msvc2010
       backend: ninja
@@ -27,14 +35,6 @@ environment:
       BOOST_ROOT: C:\Libraries\Boost_1_59_0
 
     - arch: x64
-      compiler: cygwin
-      backend: ninja
-
-    - arch: x64
-      compiler: msys2-mingw
-      backend: ninja
-
-    - arch: x64
       compiler: msvc2017
       backend: ninja
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
@@ -78,27 +78,27 @@ install:
   - cmd: if %compiler%==msvc2010 ( call "C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat" %arch% )
   - cmd: if %compiler%==msvc2015 ( call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %arch% )
   - cmd: if %compiler%==msvc2017 ( call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=%arch% )
+  - cmd: if %compiler%==cygwin ( set PYTHON=python3 ) else ( set PYTHON=python )
   - ps: |
         If($Env:compiler -eq 'msys2-mingw') {
           If($Env:arch -eq 'x86') {
             $env:Path = 'C:\msys64\mingw32\bin;' + $env:Path
+            $env:MESON_PYTHON_PATH = 'C:\msys64\mingw32\bin'
+            $env:PYTHON = 'python3'
+            C:\msys64\usr\bin\pacman -S --noconfirm mingw32/mingw-w64-i686-python3
           } Else {
             $env:Path = 'C:\msys64\mingw64\bin;' + $env:Path
+            $env:MESON_PYTHON_PATH = 'C:\msys64\mingw64\bin'
+            $env:PYTHON = 'python3'
+            C:\msys64\usr\bin\pacman -S --noconfirm mingw64/mingw-w64-x86_64-python3
           }
         }
   - cmd: if not %compiler%==cygwin ( set "PATH=%cd%;%MESON_PYTHON_PATH%;%PATH%;" )
-  - cmd: if %compiler%==cygwin ( set PYTHON=python3 ) else ( set PYTHON=python )
   - cmd: if %compiler%==cygwin ( set WRAPPER=ci\run-in-cygwin.bat )
   - cmd: if %compiler%==cygwin ( %WRAPPER% which %PYTHON% ) else ( where %PYTHON% )
 
-  # Install additional packages needed for specific builds.
-  # This was originally downloaded from http://bootstrap.pypa.io/get-pip.py,
-  # but they have a broken certificate (on Windows) and an HSTS policy that
-  # forces usage of HTTPS, so we mirror our own copy.
-  - ps: If($Env:compiler -eq 'msys2-mingw') {(new-object Net.WebClient).DownloadFile('https://nirbheek.in/files/meson/get-pip.py', 'C:\projects\meson\get-pip.py')}
   # pkg-config is needed for the pkg-config tests on msvc
   - ps: If($Env:compiler.StartsWith('msvc')) {(new-object net.webclient).DownloadFile('http://nirbheek.in/files/binaries/pkg-config/win32/pkg-config.exe', 'C:\projects\meson\pkg-config.exe')}
-  - cmd: if %compiler%==msys2-mingw ( %PYTHON% "C:\projects\meson\get-pip.py" )
   - cmd: if %compiler%==cygwin ( call ci\appveyor-install.bat )
   - ps: |
         If($Env:compiler -like 'msvc*') {
diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py
index 067b71936..f7b56bdef 100644
--- a/mesonbuild/backend/backends.py
+++ b/mesonbuild/backend/backends.py
@@ -238,8 +238,14 @@ class Backend:
         return obj_list
 
     def serialize_executable(self, exe, cmd_args, workdir, env={},
-                             capture=None):
+                             extra_paths=None, capture=None):
         import hashlib
+        if extra_paths is None:
+            # The callee didn't check if we needed extra paths, so check it here
+            if mesonlib.is_windows() or mesonlib.is_cygwin():
+                extra_paths = self.determine_windows_extra_paths(exe, [])
+            else:
+                extra_paths = []
         # Can't just use exe.name here; it will likely be run more than once
         if isinstance(exe, (dependencies.ExternalProgram,
                             build.BuildTarget, build.CustomTarget)):
@@ -272,10 +278,6 @@ class Backend:
                 exe_wrapper = self.environment.cross_info.config['binaries'].get('exe_wrapper', None)
             else:
                 exe_wrapper = None
-            if mesonlib.is_windows() or mesonlib.is_cygwin():
-                extra_paths = self.determine_windows_extra_paths(exe)
-            else:
-                extra_paths = []
             es = ExecutableSerialisation(basename, exe_cmd, cmd_args, env,
                                          is_cross_built, exe_wrapper, workdir,
                                          extra_paths, capture)
@@ -531,23 +533,27 @@ class Backend:
             args.append(d_arg)
         return args
 
-    def determine_windows_extra_paths(self, target):
+    def determine_windows_extra_paths(self, target, extra_bdeps):
         '''On Windows there is no such thing as an rpath.
         We must determine all locations of DLLs that this exe
         links to and return them so they can be used in unit
         tests.'''
-        if not isinstance(target, build.Executable):
-            return []
-        prospectives = target.get_transitive_link_deps()
         result = []
+        prospectives = []
+        if isinstance(target, build.Executable):
+            prospectives = target.get_transitive_link_deps()
+            # External deps
+            for deppath in self.rpaths_for_bundled_shared_libraries(target):
+                result.append(os.path.normpath(os.path.join(self.environment.get_build_dir(), deppath)))
+        for bdep in extra_bdeps:
+            prospectives += bdep.get_transitive_link_deps()
+        # Internal deps
         for ld in prospectives:
             if ld == '' or ld == '.':
                 continue
             dirseg = os.path.join(self.environment.get_build_dir(), self.get_target_dir(ld))
             if dirseg not in result:
                 result.append(dirseg)
-        for deppath in self.rpaths_for_bundled_shared_libraries(target):
-            result.append(os.path.normpath(os.path.join(self.environment.get_build_dir(), deppath)))
         return result
 
     def write_benchmark_file(self, datafile):
@@ -578,7 +584,7 @@ class Backend:
             else:
                 exe_wrapper = None
             if mesonlib.is_windows() or mesonlib.is_cygwin():
-                extra_paths = self.determine_windows_extra_paths(exe)
+                extra_paths = self.determine_windows_extra_paths(exe, [])
             else:
                 extra_paths = []
             cmd_args = []
diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py
index bcda603ed..e35a391dc 100644
--- a/mesonbuild/backend/ninjabackend.py
+++ b/mesonbuild/backend/ninjabackend.py
@@ -504,22 +504,30 @@ int dummy;
             # Add a dependency on all the outputs of this target
             for output in d.get_outputs():
                 elem.add_dep(os.path.join(self.get_target_dir(d), output))
+        serialize = False
+        extra_paths = []
         # If the target requires capturing stdout, then use the serialized
         # executable wrapper to capture that output and save it to a file.
-        #
+        if target.capture:
+            serialize = True
         # If the command line requires a newline, also use the wrapper, as
         # ninja does not support them in its build rule syntax.
-        #
+        if any('\n' in c for c in cmd):
+            serialize = True
         # Windows doesn't have -rpath, so for EXEs that need DLLs built within
         # the project, we need to set PATH so the DLLs are found. We use
         # a serialized executable wrapper for that and check if the
         # CustomTarget command needs extra paths first.
-        if (target.capture or any('\n' in c for c in cmd) or
-                ((mesonlib.is_windows() or mesonlib.is_cygwin()) and
-                 self.determine_windows_extra_paths(target.command[0]))):
+        if mesonlib.is_windows() or mesonlib.is_cygwin():
+            extra_bdeps = target.get_transitive_build_target_deps()
+            extra_paths = self.determine_windows_extra_paths(target.command[0], extra_bdeps)
+            if extra_paths:
+                serialize = True
+        if serialize:
             exe_data = self.serialize_executable(target.command[0], cmd[1:],
                                                  # All targets are built from the build dir
                                                  self.environment.get_build_dir(),
+                                                 extra_paths=extra_paths,
                                                  capture=ofilenames[0] if target.capture else None)
             cmd = self.environment.get_build_command() + ['--internal', 'exe', exe_data]
             cmd_type = 'meson_exe.py custom'
diff --git a/mesonbuild/backend/vs2010backend.py b/mesonbuild/backend/vs2010backend.py
index ea02580c0..e872a0474 100644
--- a/mesonbuild/backend/vs2010backend.py
+++ b/mesonbuild/backend/vs2010backend.py
@@ -429,9 +429,12 @@ class Vs2010Backend(backends.Backend):
         # Always use a wrapper because MSBuild eats random characters when
         # there are many arguments.
         tdir_abs = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target))
+        extra_bdeps = target.get_transitive_build_target_deps()
+        extra_paths = self.determine_windows_extra_paths(target.command[0], extra_bdeps)
         exe_data = self.serialize_executable(target.command[0], cmd[1:],
                                              # All targets run from the target dir
                                              tdir_abs,
+                                             extra_paths=extra_paths,
                                              capture=ofilenames[0] if target.capture else None)
         wrapper_cmd = self.environment.get_build_command() + ['--internal', 'exe', exe_data]
         ET.SubElement(customstep, 'Command').text = ' '.join(self.quote_arguments(wrapper_cmd))
diff --git a/mesonbuild/build.py b/mesonbuild/build.py
index 8eb95dc13..7757300c5 100644
--- a/mesonbuild/build.py
+++ b/mesonbuild/build.py
@@ -1549,6 +1549,24 @@ class CustomTarget(Target):
                 deps.append(c)
         return deps
 
+    def get_transitive_build_target_deps(self):
+        '''
+        Recursively fetch the build targets that this custom target depends on,
+        whether through `command:`, `depends:`, or `sources:` The recursion is
+        only performed on custom targets.
+        This is useful for setting PATH on Windows for finding required DLLs.
+        F.ex, if you have a python script that loads a C module that links to
+        other DLLs in your project.
+        '''
+        bdeps = set()
+        deps = self.get_target_dependencies()
+        for d in deps:
+            if isinstance(d, BuildTarget):
+                bdeps.add(d)
+            elif isinstance(d, CustomTarget):
+                bdeps.update(d.get_transitive_build_target_deps())
+        return bdeps
+
     def flatten_command(self, cmd):
         cmd = listify(cmd, unholder=True)
         final_cmd = []
diff --git a/mesonbuild/dependencies/misc.py b/mesonbuild/dependencies/misc.py
index 61dd953de..ce9313e80 100644
--- a/mesonbuild/dependencies/misc.py
+++ b/mesonbuild/dependencies/misc.py
@@ -21,6 +21,8 @@ import shlex
 import shutil
 import sysconfig
 
+from pathlib import Path
+
 from .. import mlog
 from .. import mesonlib
 from ..mesonlib import Popen_safe, extract_as_list
@@ -603,6 +605,7 @@ class Python3Dependency(ExternalDependency):
     def __init__(self, environment, kwargs):
         super().__init__('python3', environment, None, kwargs)
         self.name = 'python3'
+        self.static = kwargs.get('static', False)
         # We can only be sure that it is Python 3 at this point
         self.version = '3'
         self.pkgdep = None
@@ -638,12 +641,55 @@ class Python3Dependency(ExternalDependency):
         else:
             mlog.log('Dependency', mlog.bold(self.name), 'found:', mlog.red('NO'))
 
+    @staticmethod
+    def get_windows_python_arch():
+        pyplat = sysconfig.get_platform()
+        if pyplat == 'mingw':
+            pycc = sysconfig.get_config_var('CC')
+            if pycc.startswith('x86_64'):
+                return '64'
+            elif pycc.startswith(('i686', 'i386')):
+                return '32'
+            else:
+                mlog.log('MinGW Python built with unknown CC {!r}, please file'
+                         'a bug'.format(pycc))
+                return None
+        elif pyplat == 'win32':
+            return '32'
+        elif pyplat in ('win64', 'win-amd64'):
+            return '64'
+        mlog.log('Unknown Windows Python platform {!r}'.format(pyplat))
+        return None
+
+    def get_windows_link_args(self):
+        pyplat = sysconfig.get_platform()
+        if pyplat.startswith('win'):
+            vernum = sysconfig.get_config_var('py_version_nodot')
+            if self.static:
+                libname = 'libpython{}.a'.format(vernum)
+            else:
+                libname = 'python{}.lib'.format(vernum)
+            lib = Path(sysconfig.get_config_var('base')) / 'libs' / libname
+        elif pyplat == 'mingw':
+            if self.static:
+                libname = sysconfig.get_config_var('LIBRARY')
+            else:
+                libname = sysconfig.get_config_var('LDLIBRARY')
+            lib = Path(sysconfig.get_config_var('LIBDIR')) / libname
+        if not lib.exists():
+            mlog.log('Could not find Python3 library {!r}'.format(str(lib)))
+            return None
+        return [str(lib)]
+
     def _find_libpy3_windows(self, env):
         '''
         Find python3 libraries on Windows and also verify that the arch matches
         what we are building for.
         '''
-        pyarch = sysconfig.get_platform()
+        pyarch = self.get_windows_python_arch()
+        if pyarch is None:
+            self.is_found = False
+            return
         arch = detect_cpu_family(env.coredata.compilers)
         if arch == 'x86':
             arch = '32'
@@ -656,22 +702,24 @@ class Python3Dependency(ExternalDependency):
             self.is_found = False
             return
         # Pyarch ends in '32' or '64'
-        if arch != pyarch[-2:]:
-            mlog.log('Need', mlog.bold(self.name),
-                     'for {}-bit, but found {}-bit'.format(arch, pyarch[-2:]))
+        if arch != pyarch:
+            mlog.log('Need', mlog.bold(self.name), 'for {}-bit, but '
+                     'found {}-bit'.format(arch, pyarch))
+            self.is_found = False
+            return
+        # This can fail if the library is not found
+        largs = self.get_windows_link_args()
+        if largs is None:
             self.is_found = False
             return
+        self.link_args = largs
+        # Compile args
         inc = sysconfig.get_path('include')
         platinc = sysconfig.get_path('platinclude')
         self.compile_args = ['-I' + inc]
         if inc != platinc:
             self.compile_args.append('-I' + platinc)
-        # Nothing exposes this directly that I coulf find
-        basedir = sysconfig.get_config_var('base')
-        vernum = sysconfig.get_config_var('py_version_nodot')
-        self.link_args = ['-L{}/libs'.format(basedir),
-                          '-lpython{}'.format(vernum)]
-        self.version = sysconfig.get_config_var('py_version_short')
+        self.version = sysconfig.get_config_var('py_version')
         self.is_found = True
 
     def get_methods(self):
diff --git a/test cases/python3/4 custom target depends extmodule/blaster.py b/test cases/python3/4 custom target depends extmodule/blaster.py
new file mode 100644
index 000000000..49be46644
--- /dev/null
+++ b/test cases/python3/4 custom target depends extmodule/blaster.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import argparse
+
+from pathlib import Path
+
+filedir = Path(os.path.dirname(__file__)).resolve()
+if list(filedir.glob('ext/*tachyon.*')):
+    sys.path.insert(0, (filedir / 'ext').as_posix())
+
+import tachyon
+
+parser = argparse.ArgumentParser()
+parser.add_argument('-o', dest='output', default=None)
+
+options = parser.parse_args(sys.argv[1:])
+
+result = tachyon.phaserize('shoot')
+
+if options.output:
+    with open(options.output, 'w') as f:
+        f.write('success')
+
+if not isinstance(result, int):
+    print('Returned result not an integer.')
+    sys.exit(1)
+
+if result != 1:
+    print('Returned result {} is not 1.'.format(result))
+    sys.exit(1)
diff --git a/test cases/python3/4 custom target depends extmodule/ext/lib/meson-tachyonlib.c b/test cases/python3/4 custom target depends extmodule/ext/lib/meson-tachyonlib.c
new file mode 100644
index 000000000..aeff29655
--- /dev/null
+++ b/test cases/python3/4 custom target depends extmodule/ext/lib/meson-tachyonlib.c
@@ -0,0 +1,8 @@
+#ifdef _MSC_VER
+__declspec(dllexport)
+#endif
+const char*
+tachyon_phaser_command (void)
+{
+    return "shoot";
+}
diff --git a/test cases/python3/4 custom target depends extmodule/ext/lib/meson-tachyonlib.h b/test cases/python3/4 custom target depends extmodule/ext/lib/meson-tachyonlib.h
new file mode 100644
index 000000000..dca71d311
--- /dev/null
+++ b/test cases/python3/4 custom target depends extmodule/ext/lib/meson-tachyonlib.h
@@ -0,0 +1,6 @@
+#pragma once
+
+#ifdef _MSC_VER
+__declspec(dllimport)
+#endif
+const char* tachyon_phaser_command (void);
diff --git a/test cases/python3/4 custom target depends extmodule/ext/lib/meson.build b/test cases/python3/4 custom target depends extmodule/ext/lib/meson.build
new file mode 100644
index 000000000..b1f893849
--- /dev/null
+++ b/test cases/python3/4 custom target depends extmodule/ext/lib/meson.build
@@ -0,0 +1,4 @@
+libtachyon = shared_library('tachyonlib', 'meson-tachyonlib.c')
+
+libtachyon_dep = declare_dependency(link_with : libtachyon,
+                                    include_directories : include_directories('.'))
diff --git a/test cases/python3/4 custom target depends extmodule/ext/meson.build b/test cases/python3/4 custom target depends extmodule/ext/meson.build
new file mode 100644
index 000000000..5ccbe223c
--- /dev/null
+++ b/test cases/python3/4 custom target depends extmodule/ext/meson.build
@@ -0,0 +1,6 @@
+subdir('lib')
+
+pylib = py3_mod.extension_module('tachyon',
+  'tachyon_module.c',
+  dependencies : [libtachyon_dep, py3_dep],
+)
diff --git a/test cases/python3/4 custom target depends extmodule/ext/tachyon_module.c b/test cases/python3/4 custom target depends extmodule/ext/tachyon_module.c
new file mode 100644
index 000000000..b48032bbb
--- /dev/null
+++ b/test cases/python3/4 custom target depends extmodule/ext/tachyon_module.c
@@ -0,0 +1,51 @@
+/*
+  Copyright 2016 The Meson development team
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+/* A very simple Python extension module. */
+
+#include <Python.h>
+#include <string.h>
+
+#include "meson-tachyonlib.h"
+
+static PyObject* phaserize(PyObject *self, PyObject *args) {
+    const char *message;
+    int result;
+
+    if(!PyArg_ParseTuple(args, "s", &message))
+        return NULL;
+
+    result = strcmp(message, tachyon_phaser_command()) ? 0 : 1;
+    return PyLong_FromLong(result);
+}
+
+static PyMethodDef TachyonMethods[] = {
+    {"phaserize",  phaserize, METH_VARARGS,
+     "Shoot tachyon cannons."},
+    {NULL, NULL, 0, NULL}
+};
+
+static struct PyModuleDef tachyonmodule = {
+   PyModuleDef_HEAD_INIT,
+   "tachyon",
+   NULL,
+   -1,
+   TachyonMethods
+};
+
+PyMODINIT_FUNC PyInit_tachyon(void) {
+    return PyModule_Create(&tachyonmodule);
+}
diff --git a/test cases/python3/4 custom target depends extmodule/meson.build b/test cases/python3/4 custom target depends extmodule/meson.build
new file mode 100644
index 000000000..dd6fed09a
--- /dev/null
+++ b/test cases/python3/4 custom target depends extmodule/meson.build
@@ -0,0 +1,35 @@
+project('Python extension module', 'c',
+  default_options : ['buildtype=release'])
+# Because Windows Python ships only with optimized libs,
+# we must build this project the same way.
+
+py3_mod = import('python3')
+py3 = py3_mod.find_python()
+py3_dep = dependency('python3', required : false)
+
+# Copy to the builddir so that blaster.py can find the built tachyon module
+# FIXME: We should automatically detect this case and append the correct paths
+# to PYTHONLIBDIR
+blaster_py = configure_file(input : 'blaster.py',
+                            output : 'blaster.py',
+                            configuration : configuration_data())
+
+check_exists = '''
+import os, sys
+with open(sys.argv[1], 'rb') as f:
+  assert(f.read() == b'success')
+'''
+if py3_dep.found()
+  subdir('ext')
+
+  out_txt = custom_target('tachyon flux',
+    input : blaster_py,
+    output : 'out.txt',
+    command : [py3, '@INPUT@', '-o', '@OUTPUT@'],
+    depends : pylib,
+    build_by_default: true)
+
+  test('flux', py3, args : ['-c', check_exists, out_txt])
+else
+  error('MESON_SKIP_TEST: Python3 libraries not found, skipping test.')
+endif