Merge pull request #3060 from jon-turney/always-run-framework-tests

Always run all framework tests on all platforms

18 files changed, 140 insertions, 26 deletions

diff --git a/mesonbuild/dependencies/base.py b/mesonbuild/dependencies/base.py
index 3357e8ebc..7a652a4cf 100644
--- a/mesonbuild/dependencies/base.py
+++ b/mesonbuild/dependencies/base.py
@@ -495,7 +495,13 @@ class PkgConfigDependency(ExternalDependency):
         return converted
 
     def _set_cargs(self):
-        ret, out = self._call_pkgbin(['--cflags', self.name])
+        env = None
+        if self.language == 'fortran':
+            # gfortran doesn't appear to look in system paths for INCLUDE files,
+            # so don't allow pkg-config to suppress -I flags for system paths
+            env = os.environ.copy()
+            env['PKG_CONFIG_ALLOW_SYSTEM_CFLAGS'] = '1'
+        ret, out = self._call_pkgbin(['--cflags', self.name], env=env)
         if ret != 0:
             raise DependencyException('Could not generate cargs for %s:\n\n%s' %
                                       (self.name, out))
diff --git a/run_project_tests.py b/run_project_tests.py
index 9599155cd..c2c3efe76 100755
--- a/run_project_tests.py
+++ b/run_project_tests.py
@@ -77,7 +77,8 @@ class AutoDeletedDir:
 
 failing_logs = []
 print_debug = 'MESON_PRINT_TEST_OUTPUT' in os.environ
-do_debug = not {'MESON_PRINT_TEST_OUTPUT', 'TRAVIS', 'APPVEYOR'}.isdisjoint(os.environ)
+under_ci = not {'TRAVIS', 'APPVEYOR'}.isdisjoint(os.environ)
+do_debug = under_ci or print_debug
 no_meson_log_msg = 'No meson-log.txt found.'
 
 system_compiler = None
@@ -437,6 +438,33 @@ def have_java():
         return True
     return False
 
+def skippable(suite, test):
+    if not under_ci:
+        return True
+
+    if not suite.endswith('frameworks'):
+        return True
+
+    # gtk-doc test is always skipped pending upstream fixes for spaces in
+    # filenames landing in distros
+    if test.endswith('10 gtk-doc'):
+        return True
+
+    # No frameworks test should be skipped on linux CI, as we expect all
+    # prerequisites to be installed
+    if mesonlib.is_linux():
+        return False
+
+    # Boost test should only be skipped for windows CI build matrix entries
+    # which don't define BOOST_ROOT
+    if test.endswith('1 boost'):
+        if mesonlib.is_windows():
+            return 'BOOST_ROOT' not in os.environ
+        return False
+
+    # Other framework tests are allowed to be skipped on other platforms
+    return True
+
 def detect_tests_to_run():
     # Name, subdirectory, skip condition.
     all_tests = [
@@ -460,20 +488,9 @@ def detect_tests_to_run():
         ('swift', 'swift', backend not in (Backend.ninja, Backend.xcode) or not shutil.which('swiftc')),
         ('python3', 'python3', backend is not Backend.ninja),
         ('fpga', 'fpga', shutil.which('yosys') is None),
+        ('frameworks', 'frameworks', False),
     ]
     gathered_tests = [(name, gather_tests('test cases/' + subdir), skip) for name, subdir, skip in all_tests]
-    if mesonlib.is_windows():
-        # TODO: Set BOOST_ROOT in .appveyor.yml
-        gathered_tests += [('framework', ['test cases/frameworks/1 boost'], 'BOOST_ROOT' not in os.environ)]
-    elif mesonlib.is_osx():
-        if os.path.exists('/usr/local/include/boost'):
-            # Just do the BOOST test
-            gathered_tests += [('framework', ['test cases/frameworks/1 boost'], False)]
-    elif mesonlib.is_cygwin():
-        # Just do the BOOST test
-        gathered_tests += [('framework', ['test cases/frameworks/1 boost'], False)]
-    else:
-        gathered_tests += [('framework', gather_tests('test cases/frameworks'), False)]
     return gathered_tests
 
 def run_tests(all_tests, log_name_base, extra_args):
@@ -532,7 +549,7 @@ def _run_tests(all_tests, log_name_base, extra_args):
         for (testname, t, result) in futures:
             sys.stdout.flush()
             result = result.result()
-            if result is None or 'MESON_SKIP_TEST' in result.stdo:
+            if (result is None) or (('MESON_SKIP_TEST' in result.stdo) and (skippable(name, t))):
                 print(yellow('Skipping:'), t)
                 current_test = ET.SubElement(current_suite, 'testcase', {'name': testname,
                                                                          'classname': name})
diff --git a/test cases/frameworks/1 boost/meson.build b/test cases/frameworks/1 boost/meson.build
index 771ecbc6f..df55b3050 100644
--- a/test cases/frameworks/1 boost/meson.build
+++ b/test cases/frameworks/1 boost/meson.build
@@ -5,6 +5,11 @@ add_project_arguments(['-DBOOST_LOG_DYN_LINK'],
   language : 'cpp'
 )
 
+dep = dependency('boost', required: false)
+if not dep.found()
+  error('MESON_SKIP_TEST boost not found.')
+endif
+
 # We want to have multiple separate configurations of Boost
 # within one project. The need to be independent of each other.
 # Use one without a library dependency and one with it.
diff --git a/test cases/frameworks/10 gtk-doc/meson.build b/test cases/frameworks/10 gtk-doc/meson.build
index e5e77058d..71f341cfd 100644
--- a/test cases/frameworks/10 gtk-doc/meson.build
+++ b/test cases/frameworks/10 gtk-doc/meson.build
@@ -1,5 +1,10 @@
 project('gtkdoctest', 'c', version : '1.0.0')
 
+gtkdoc = find_program('gtkdoc-scan', required: false)
+if not gtkdoc.found()
+  error('MESON_SKIP_TEST gtkdoc not found.')
+endif
+
 gnome = import('gnome')
 
 assert(gnome.gtkdoc_html_dir('foobar') == 'share/gtk-doc/html/foobar', 'Gtkdoc install dir is incorrect.')
diff --git a/test cases/frameworks/11 gir subproject/meson.build b/test cases/frameworks/11 gir subproject/meson.build
index f3bde4042..a599ae9eb 100644
--- a/test cases/frameworks/11 gir subproject/meson.build
+++ b/test cases/frameworks/11 gir subproject/meson.build
@@ -1,5 +1,16 @@
 project('gobject-introspection-with-subproject', 'c')
 
+gir = find_program('g-ir-scanner', required: false)
+if not gir.found()
+  error('MESON_SKIP_TEST g-ir-scanner not found.')
+endif
+
+python3 = import('python3')
+py3 = python3.find_python()
+if run_command(py3, '-c', 'import gi;').returncode() != 0
+    error('MESON_SKIP_TEST python3-gi not found')
+endif
+
 gnome = import('gnome')
 gobj = dependency('gobject-2.0')
 
@@ -7,4 +18,3 @@ add_global_arguments('-DMESON_TEST', language : 'c')
 meson_gir = dependency('meson-gir', fallback : ['mesongir', 'meson_gir'])
 
 subdir('gir')
-
diff --git a/test cases/frameworks/12 multiple gir/meson.build b/test cases/frameworks/12 multiple gir/meson.build
index 794abc5bc..ddc9830d5 100644
--- a/test cases/frameworks/12 multiple gir/meson.build
+++ b/test cases/frameworks/12 multiple gir/meson.build
@@ -1,5 +1,10 @@
 project('multiple-gobject-introspection', 'c')
 
+gir = find_program('g-ir-scanner', required: false)
+if not gir.found()
+  error('MESON_SKIP_TEST g-ir-scanner not found.')
+endif
+
 gnome = import('gnome')
 gobj = dependency('gobject-2.0')
 
diff --git a/test cases/frameworks/13 yelp/meson.build b/test cases/frameworks/13 yelp/meson.build
index 725ff7b1b..9fdde25ae 100644
--- a/test cases/frameworks/13 yelp/meson.build
+++ b/test cases/frameworks/13 yelp/meson.build
@@ -1,2 +1,8 @@
 project('yelp', 'c')
+
+itstool = find_program('itstool', required: false)
+if not itstool.found()
+  error('MESON_SKIP_TEST itstool not found.')
+endif
+
 subdir('help')
diff --git a/test cases/frameworks/15 llvm/meson.build b/test cases/frameworks/15 llvm/meson.build
index 549adce90..b5505eb57 100644
--- a/test cases/frameworks/15 llvm/meson.build
+++ b/test cases/frameworks/15 llvm/meson.build
@@ -1,5 +1,10 @@
 project('llvmtest', ['c', 'cpp'], default_options : ['c_std=c99'])
 
+d = dependency('llvm', required : false)
+if not d.found()
+  error('MESON_SKIP_TEST llvm not found.')
+endif
+
 d = dependency('llvm', modules : 'not-found', required : false)
 assert(d.found() == false, 'not-found llvm module found')
 
@@ -12,7 +17,7 @@ assert(d.found() == true, 'optional module stopped llvm from being found.')
 dep_tinfo = dependency('tinfo', required : false)
 if not dep_tinfo.found()
   cpp = meson.get_compiler('cpp')
-  dep_tinfo = cpp.find_library('tinfo')
+  dep_tinfo = cpp.find_library('tinfo', required: false)
 endif
 
 foreach static : [true, false]
diff --git a/test cases/frameworks/16 sdl2/meson.build b/test cases/frameworks/16 sdl2/meson.build
index 61a34efa0..1bbf09f71 100644
--- a/test cases/frameworks/16 sdl2/meson.build
+++ b/test cases/frameworks/16 sdl2/meson.build
@@ -1,6 +1,10 @@
 project('sdl2 test', 'c')
 
-sdl2_dep = dependency('sdl2', version : '>=2.0.0')
+sdl2_dep = dependency('sdl2', version : '>=2.0.0', required: false)
+
+if not sdl2_dep.found()
+  error('MESON_SKIP_TEST sdl2 not found.')
+endif
 
 e = executable('sdl2prog', 'sdl2prog.c', dependencies : sdl2_dep)
 
diff --git a/test cases/frameworks/19 pcap/meson.build b/test cases/frameworks/19 pcap/meson.build
index f02f4114b..eb6fc2c14 100644
--- a/test cases/frameworks/19 pcap/meson.build
+++ b/test cases/frameworks/19 pcap/meson.build
@@ -1,6 +1,10 @@
 project('pcap test', 'c')
 
-pcap_dep = dependency('pcap', version : '>=1.0')
+pcap_dep = dependency('pcap', version : '>=1.0', required: false)
+
+if not pcap_dep.found()
+  error('MESON_SKIP_TEST pcap not found.')
+endif
 
 pcap_ver = pcap_dep.version()
 assert(pcap_ver.split('.').length() > 1, 'pcap version is "@0@"'.format(pcap_ver))
@@ -9,6 +13,6 @@ e = executable('pcap_prog', 'pcap_prog.c', dependencies : pcap_dep)
 
 test('pcaptest', e)
 
-# Ensure discovery bia the configuration tools work also
+# Ensure discovery via the configuration tools work also
 pcap_dep = dependency('pcap', version : '>=1.0', method : 'pcap-config')
 pcap_dep = dependency('pcap', version : '>=1.0', method : 'config-tool')
diff --git a/test cases/frameworks/19 pcap/pcap_prog.c b/test cases/frameworks/19 pcap/pcap_prog.c
index 18e0ad895..0fca16cca 100644
--- a/test cases/frameworks/19 pcap/pcap_prog.c
+++ b/test cases/frameworks/19 pcap/pcap_prog.c
@@ -4,6 +4,12 @@ int
 main()
 {
     char errbuf[PCAP_ERRBUF_SIZE];
-    pcap_t *p = pcap_create(NULL, errbuf);
+#ifdef __APPLE__
+    // source = NULL for "any" doesn't work on macOS (linux only?)
+    char *source = "en0";
+#else
+    char *source = NULL;
+#endif
+    pcap_t *p = pcap_create(source, errbuf);
     return p == NULL;
 }
diff --git a/test cases/frameworks/20 cups/meson.build b/test cases/frameworks/20 cups/meson.build
index 11f6f631b..9040de6e1 100644
--- a/test cases/frameworks/20 cups/meson.build
+++ b/test cases/frameworks/20 cups/meson.build
@@ -1,6 +1,10 @@
 project('cups test', 'c')
 
-cups_dep = dependency('cups', version : '>=1.4')
+cups_dep = dependency('cups', version : '>=1.4', required: false)
+
+if not cups_dep.found()
+  error('MESON_SKIP_TEST cups not found.')
+endif
 
 e = executable('cups_prog', 'cups_prog.c', dependencies : cups_dep)
 
diff --git a/test cases/frameworks/4 qt/meson.build b/test cases/frameworks/4 qt/meson.build
index b0e848d26..b508df303 100644
--- a/test cases/frameworks/4 qt/meson.build
+++ b/test cases/frameworks/4 qt/meson.build
@@ -21,6 +21,14 @@ foreach qt : ['qt4', 'qt5']
     error('Invalid qt dep incorrectly found!')
   endif
 
+  # This test should be skipped if qt5 isn't found
+  if qt == 'qt5'
+    dep = dependency(qt, modules : ['Core'], required : false, method : get_option('method'))
+    if not dep.found()
+      error('MESON_SKIP_TEST qt5 not found.')
+    endif
+  endif
+
   # Ensure that the "no-Core-module-specified" code branch is hit
   nocoredep = dependency(qt, modules : ['Gui'], required : qt == 'qt5', method : get_option('method'))
 
diff --git a/test cases/frameworks/6 gettext/installed_files.txt b/test cases/frameworks/6 gettext/installed_files.txt
index ffe543ffe..879f56bc5 100644
--- a/test cases/frameworks/6 gettext/installed_files.txt
+++ b/test cases/frameworks/6 gettext/installed_files.txt
@@ -1,4 +1,4 @@
-usr/bin/intlprog
+usr/bin/intlprog?exe
 usr/share/locale/de/LC_MESSAGES/intltest.mo
 usr/share/locale/fi/LC_MESSAGES/intltest.mo
 usr/share/applications/test.desktop
diff --git a/test cases/frameworks/6 gettext/meson.build b/test cases/frameworks/6 gettext/meson.build
index 6b517a490..e02234b56 100644
--- a/test cases/frameworks/6 gettext/meson.build
+++ b/test cases/frameworks/6 gettext/meson.build
@@ -1,5 +1,14 @@
 project('gettext example', 'c')
 
+gettext = find_program('gettext', required: false)
+if not gettext.found()
+  error('MESON_SKIP_TEST gettext not found.')
+endif
+
+if not meson.get_compiler('c').has_header('libintl.h')
+  error('MESON_SKIP_TEST libintl.h not found.')
+endif
+
 i18n = import('i18n')
 
 subdir('po')
diff --git a/test cases/frameworks/7 gnome/meson.build b/test cases/frameworks/7 gnome/meson.build
index 795f458e9..03335b885 100644
--- a/test cases/frameworks/7 gnome/meson.build
+++ b/test cases/frameworks/7 gnome/meson.build
@@ -1,5 +1,16 @@
 project('gobject-introspection', 'c')
 
+glib = dependency('glib-2.0', required: false)
+if not glib.found()
+  error('MESON_SKIP_TEST glib not found.')
+endif
+
+python3 = import('python3')
+py3 = python3.find_python()
+if run_command(py3, '-c', 'import gi;').returncode() != 0
+    error('MESON_SKIP_TEST python3-gi not found')
+endif
+
 cc = meson.get_compiler('c')
 
 add_global_arguments('-DMESON_TEST', language : 'c')
diff --git a/test cases/frameworks/8 flex/meson.build b/test cases/frameworks/8 flex/meson.build
index 13ac9f634..cb5efdeb4 100644
--- a/test cases/frameworks/8 flex/meson.build
+++ b/test cases/frameworks/8 flex/meson.build
@@ -4,8 +4,16 @@ project('flex and bison', 'c')
 # may output headers that are necessary to build
 # the sources of a different generator.
 
-flex = find_program('flex')
-bison = find_program('bison')
+flex = find_program('flex', required: false)
+bison = find_program('bison', required: false)
+
+if not flex.found()
+  error('MESON_SKIP_TEST flex not found.')
+endif
+
+if not bison.found()
+  error('MESON_SKIP_TEST bison not found.')
+endif
 
 lgen = generator(flex,
 output : '@PLAINNAME@.yy.c',
@@ -23,4 +31,3 @@ e = executable('pgen', 'prog.c',
 lfiles, pfiles)
 
 test('parsertest', e)
-
diff --git a/test cases/frameworks/8 flex/prog.c b/test cases/frameworks/8 flex/prog.c
index 1e48f611c..0b84d1823 100644
--- a/test cases/frameworks/8 flex/prog.c
+++ b/test cases/frameworks/8 flex/prog.c
@@ -6,6 +6,8 @@
 #include<stdio.h>
 #include<stdlib.h>
 
+extern int yyparse();
+
 int main(int argc, char **argv) {
     /*
     int input;