Add Windows to TravisCI

Implement the generation of Travis jobs for Windows. Currently, the
generated jobs replicate Appveyor setup and complete successfully. There
is support for MinGW GCC and MSVC compilers as well as 64 and 32 bit
compilation. Linux and MacOS jobs behave identically, but some
environment variables change - CROSS_COMPILE_32BIT=yes is added for
builds with cross compilation, empty COMPILER_FLAGS are not set anymore.

5 files changed, 226 insertions, 53 deletions

diff --git a/scripts/gen_travis.py b/scripts/gen_travis.py
index e98ebeb6..63e00549 100755
--- a/scripts/gen_travis.py
+++ b/scripts/gen_travis.py
@@ -6,6 +6,7 @@ from enum import Enum, auto
 
 LINUX = 'linux'
 OSX = 'osx'
+WINDOWS = 'windows'
 
 
 AMD64 = 'amd64'
@@ -13,28 +14,48 @@ ARM64 = 'arm64'
 PPC64LE = 'ppc64le'
 
 
-TRAVIS_TEMPLATE = """
+TRAVIS_TEMPLATE = """\
 # This config file is generated by ./scripts/gen_travis.py.
 # Do not edit by hand.
 
-language: generic
+# We use 'minimal', because 'generic' makes Windows VMs hang at startup. Also
+# the software provided by 'generic' is simply not needed for our tests.
+# Differences are explained here:
+# https://docs.travis-ci.com/user/languages/minimal-and-generic/
+language: minimal
 dist: focal
 
 jobs:
   include:
 {jobs}
 
+before_install:
+  - |-
+    if test -f "./scripts/$TRAVIS_OS_NAME/before_install.sh"; then
+      source ./scripts/$TRAVIS_OS_NAME/before_install.sh
+    fi
+
 before_script:
-  - autoconf
-  - scripts/gen_travis.py > travis_script && diff .travis.yml travis_script
-  # If COMPILER_FLAGS are not empty, add them to CC and CXX
-  - ./configure ${{COMPILER_FLAGS:+ CC="$CC $COMPILER_FLAGS" \
+  - |-
+    if test -f "./scripts/$TRAVIS_OS_NAME/before_script.sh"; then
+      source ./scripts/$TRAVIS_OS_NAME/before_script.sh
+    else
+      scripts/gen_travis.py > travis_script && diff .travis.yml travis_script
+      autoconf
+      # If COMPILER_FLAGS are not empty, add them to CC and CXX
+      ./configure ${{COMPILER_FLAGS:+ CC="$CC $COMPILER_FLAGS" \
 CXX="$CXX $COMPILER_FLAGS"}} $CONFIGURE_FLAGS
-  - make -j3
-  - make -j3 tests
+      make -j3
+      make -j3 tests
+    fi
 
 script:
-  - make check
+  - |-
+    if test -f "./scripts/$TRAVIS_OS_NAME/script.sh"; then
+      source ./scripts/$TRAVIS_OS_NAME/script.sh
+    else
+      make check
+    fi
 """
 
 
@@ -44,6 +65,7 @@ class Option(object):
         COMPILER_FLAG = auto()
         CONFIGURE_FLAG = auto()
         MALLOC_CONF = auto()
+        FEATURE = auto()
 
     def __init__(self, type, value):
         self.type = type
@@ -65,6 +87,10 @@ class Option(object):
     def as_malloc_conf(value):
         return Option(Option.Type.MALLOC_CONF, value)
 
+    @staticmethod
+    def as_feature(value):
+        return Option(Option.Type.FEATURE, value)
+
     def __eq__(self, obj):
         return (isinstance(obj, Option) and obj.type == self.type
                 and obj.value == self.value)
@@ -81,13 +107,14 @@ MAX_UNUSUAL_OPTIONS = 2
 
 GCC = Option.as_compiler('CC=gcc CXX=g++')
 CLANG = Option.as_compiler('CC=clang CXX=clang++')
+CL = Option.as_compiler('CC=cl.exe CXX=cl.exe')
 
 
-compiler_default = GCC
 compilers_unusual = [CLANG,]
 
 
-compiler_flag_unusuals = [Option.as_compiler_flag(opt) for opt in ('-m32',)]
+CROSS_COMPILE_32BIT = Option.as_feature('CROSS_COMPILE_32BIT')
+feature_unusuals = [CROSS_COMPILE_32BIT]
 
 
 configure_flag_unusuals = [Option.as_configure_flag(opt) for opt in (
@@ -108,73 +135,75 @@ malloc_conf_unusuals = [Option.as_malloc_conf(opt) for opt in (
 )]
 
 
-all_unusuals = (compilers_unusual + compiler_flag_unusuals
+all_unusuals = (compilers_unusual + feature_unusuals
     + configure_flag_unusuals + malloc_conf_unusuals)
 
 
-gcc_multilib_set = False
-
-
 def get_extra_cflags(os, compiler):
+    if os == WINDOWS:
+        # For non-CL compilers under Windows (for now it's only MinGW-GCC),
+        # -fcommon needs to be specified to correctly handle multiple
+        # 'malloc_conf' symbols and such, which are declared weak under Linux.
+        # Weak symbols don't work with MinGW-GCC.
+        if compiler != CL.value:
+            return ['-fcommon']
+        else:
+            return []
+
     # We get some spurious errors when -Warray-bounds is enabled.
     extra_cflags = ['-Werror', '-Wno-array-bounds']
     if compiler == CLANG.value or os == OSX:
         extra_cflags += [
-	    '-Wno-unknown-warning-option',
-	    '-Wno-ignored-attributes'
-	]
+            '-Wno-unknown-warning-option',
+            '-Wno-ignored-attributes'
+        ]
     if os == OSX:
         extra_cflags += [
-	    '-Wno-deprecated-declarations',
-	]
+            '-Wno-deprecated-declarations',
+        ]
     return extra_cflags
 
 
 # Formats a job from a combination of flags
 def format_job(os, arch, combination):
-    global gcc_multilib_set
-
-    compiler = [x.value for x in combination if x.type == Option.Type.COMPILER]
-    assert(len(compiler) <= 1)
-    if not compiler:
-        compiler = compiler_default.value
-    else:
-        compiler = compiler[0]
+    compilers = [x.value for x in combination if x.type == Option.Type.COMPILER]
+    assert(len(compilers) <= 1)
     compiler_flags = [x.value for x in combination if x.type == Option.Type.COMPILER_FLAG]
     configure_flags = [x.value for x in combination if x.type == Option.Type.CONFIGURE_FLAG]
     malloc_conf = [x.value for x in combination if x.type == Option.Type.MALLOC_CONF]
+    features = [x.value for x in combination if x.type == Option.Type.FEATURE]
 
     if len(malloc_conf) > 0:
         configure_flags.append('--with-malloc-conf=' + ','.join(malloc_conf))
 
-    job = ""
-    job += '    - os: {}\n'.format(os)
-    job += '      arch: {}\n'.format(arch)
+    if not compilers:
+        compiler = GCC.value
+    else:
+        compiler = compilers[0]
 
-    if '-m32' in compiler_flags and os == 'linux':
-        job += '      addons:'
-        if gcc_multilib_set:
-            job += ' *gcc_multilib\n'
-        else:
-            job += ' &gcc_multilib\n'
-            job += '        apt:\n'
-            job += '          packages:\n'
-            job += '            - gcc-multilib\n'
-            job += '            - g++-multilib\n'
-            gcc_multilib_set = True
-
-    env_string = ('{} COMPILER_FLAGS="{}" CONFIGURE_FLAGS="{}" '
-        'EXTRA_CFLAGS="{}"'.format(
+    extra_environment_vars = ''
+    cross_compile = CROSS_COMPILE_32BIT.value in features
+    if os == LINUX and cross_compile:
+        compiler_flags.append('-m32')
+
+    features_str = ' '.join([' {}=yes'.format(feature) for feature in features])
+
+    stringify = lambda arr, name: ' {}="{}"'.format(name, ' '.join(arr)) if arr else ''
+    env_string = '{}{}{}{}{}{}'.format(
             compiler,
-            ' '.join(compiler_flags),
-            ' '.join(configure_flags),
-            ' '.join(get_extra_cflags(os, compiler))))
+            features_str,
+            stringify(compiler_flags, 'COMPILER_FLAGS'),
+            stringify(configure_flags, 'CONFIGURE_FLAGS'),
+            stringify(get_extra_cflags(os, compiler), 'EXTRA_CFLAGS'),
+            extra_environment_vars)
 
+    job = '    - os: {}\n'.format(os)
+    job += '      arch: {}\n'.format(arch)
     job += '      env: {}'.format(env_string)
     return job
 
 
-def generate_unusual_combinations(max_unusual_opts):
+def generate_unusual_combinations(unusuals, max_unusual_opts):
     """
     Generates different combinations of non-standard compilers, compiler flags,
     configure flags and malloc_conf settings.
@@ -182,20 +211,22 @@ def generate_unusual_combinations(max_unusual_opts):
     @param max_unusual_opts: Limit of unusual options per combination.
     """
     return chain.from_iterable(
-            [combinations(all_unusuals, i) for i in range(max_unusual_opts + 1)])
+            [combinations(unusuals, i) for i in range(max_unusual_opts + 1)])
 
 
 def included(combination, exclude):
     """
     Checks if the combination of options should be included in the Travis
     testing matrix.
+
+    @param exclude: A list of options to be avoided.
     """
     return not any(excluded in combination for excluded in exclude)
 
 
-def generate_jobs(os, arch, exclude, max_unusual_opts):
+def generate_jobs(os, arch, exclude, max_unusual_opts, unusuals=all_unusuals):
     jobs = []
-    for combination in generate_unusual_combinations(max_unusual_opts):
+    for combination in generate_unusual_combinations(unusuals, max_unusual_opts):
         if included(combination, exclude):
             jobs.append(format_job(os, arch, combination))
     return '\n'.join(jobs)
@@ -210,7 +241,7 @@ def generate_linux(arch):
     exclude = []
     if arch == PPC64LE:
         # Avoid 32 bit builds and clang on PowerPC
-        exclude = [Option.as_compiler_flag('-m32')] + compilers_unusual
+        exclude = (CROSS_COMPILE_32BIT, CLANG,)
 
     return generate_jobs(os, arch, exclude, max_unusual_opts)
 
@@ -230,6 +261,19 @@ def generate_macos(arch):
     return generate_jobs(os, arch, exclude, max_unusual_opts)
 
 
+def generate_windows(arch):
+    os = WINDOWS
+
+    max_unusual_opts = 3
+    unusuals = (
+        Option.as_configure_flag('--enable-debug'),
+        CL,
+        CROSS_COMPILE_32BIT,
+    )
+    return generate_jobs(os, arch, (), max_unusual_opts, unusuals)
+
+
+
 def get_manual_jobs():
     return """\
     # Development build
@@ -251,6 +295,9 @@ def main():
         generate_linux(PPC64LE),
 
         generate_macos(AMD64),
+
+        generate_windows(AMD64),
+
         get_manual_jobs()
     ))
 
diff --git a/scripts/linux/before_install.sh b/scripts/linux/before_install.sh
new file mode 100644
index 00000000..67417463
--- /dev/null
+++ b/scripts/linux/before_install.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -ev
+
+if [[ "$TRAVIS_OS_NAME" != "linux" ]]; then
+    echo "Incorrect \$TRAVIS_OS_NAME: expected linux, got $TRAVIS_OS_NAME"
+    exit 1
+fi
+
+if [[ "$CROSS_COMPILE_32BIT" == "yes" ]]; then
+    sudo apt-get update
+    sudo apt-get -y install gcc-multilib g++-multilib
+fi
diff --git a/scripts/windows/before_install.sh b/scripts/windows/before_install.sh
new file mode 100644
index 00000000..2740c458
--- /dev/null
+++ b/scripts/windows/before_install.sh
@@ -0,0 +1,83 @@
+#!/bin/bash
+
+set -e
+
+# The purpose of this script is to install build dependencies and set
+# $build_env to a function that sets appropriate environment variables,
+# to enable (mingw32|mingw64) environment if we want to compile with gcc, or
+# (mingw32|mingw64) + vcvarsall.bat if we want to compile with cl.exe
+
+if [[ "$TRAVIS_OS_NAME" != "windows" ]]; then
+    echo "Incorrect \$TRAVIS_OS_NAME: expected windows, got $TRAVIS_OS_NAME"
+    exit 1
+fi
+
+[[ ! -f C:/tools/msys64/msys2_shell.cmd ]] && rm -rf C:/tools/msys64
+choco uninstall -y mingw
+choco upgrade --no-progress -y msys2
+
+msys_shell_cmd="cmd //C RefreshEnv.cmd && set MSYS=winsymlinks:nativestrict && C:\\tools\\msys64\\msys2_shell.cmd"
+
+msys2() { $msys_shell_cmd -defterm -no-start -msys2 -c "$*"; }
+mingw32() { $msys_shell_cmd -defterm -no-start -mingw32 -c "$*"; }
+mingw64() { $msys_shell_cmd -defterm -no-start -mingw64 -c "$*"; }
+
+if [[ "$CROSS_COMPILE_32BIT" == "yes" ]]; then
+    mingw=mingw32
+    mingw_gcc_package_arch=i686
+else
+    mingw=mingw64
+    mingw_gcc_package_arch=x86_64
+fi
+
+if [[ "$CC" == *"gcc"* ]]; then
+    $mingw pacman -S --noconfirm --needed \
+        autotools \
+        git \
+        mingw-w64-${mingw_gcc_package_arch}-make \
+	    mingw-w64-${mingw_gcc_package_arch}-gcc \
+	    mingw-w64-${mingw_gcc_package_arch}-binutils
+    build_env=$mingw
+elif [[ "$CC" == *"cl"* ]]; then
+    $mingw pacman -S --noconfirm --needed \
+        autotools \
+	    git \
+	    mingw-w64-${mingw_gcc_package_arch}-make \
+	    mingw-w64-${mingw_gcc_package_arch}-binutils
+
+    # In order to use MSVC compiler (cl.exe), we need to correctly set some environment
+    # variables, namely PATH, INCLUDE, LIB and LIBPATH. The correct values of these
+    # variables are set by a batch script "vcvarsall.bat". The code below generates
+    # a batch script that calls "vcvarsall.bat" and prints the environment variables.
+    #
+    # Then, those environment variables are transformed from cmd to bash format and put
+    # into a script $apply_vsenv. If cl.exe needs to be used from bash, one can
+    # 'source $apply_vsenv' and it will apply the environment variables needed for cl.exe
+    # to be located and function correctly.
+    #
+    # At last, a function "mingw_with_msvc_vars" is generated which forwards user input
+    # into a correct mingw (32 or 64) subshell that automatically performs 'source $apply_vsenv',
+    # making it possible for autotools to discover and use cl.exe.
+    vcvarsall="vcvarsall.tmp.bat"
+    echo "@echo off" > $vcvarsall
+    echo "call \"c:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\\\vcvarsall.bat\" $USE_MSVC" >> $vcvarsall
+    echo "set" >> $vcvarsall
+
+    apply_vsenv="./apply_vsenv.sh"
+    cmd //C $vcvarsall | grep -E "^PATH=" | sed -n -e 's/\(.*\)=\(.*\)/export \1=$PATH:"\2"/g' \
+        -e 's/\([a-zA-Z]\):[\\\/]/\/\1\//g' \
+        -e 's/\\/\//g' \
+        -e 's/;\//:\//gp' > $apply_vsenv
+    cmd //C $vcvarsall | grep -E "^(INCLUDE|LIB|LIBPATH)=" | sed -n -e 's/\(.*\)=\(.*\)/export \1="\2"/gp' >> $apply_vsenv
+
+    cat $apply_vsenv
+    mingw_with_msvc_vars() { $msys_shell_cmd -defterm -no-start -$mingw -c "source $apply_vsenv && ""$*"; }
+    build_env=mingw_with_msvc_vars
+
+    rm -f $vcvarsall
+else
+    echo "Unknown C compiler: $CC"
+    exit 1
+fi
+
+echo "Build environment function: $build_env"
diff --git a/scripts/windows/before_script.sh b/scripts/windows/before_script.sh
new file mode 100644
index 00000000..9d30abab
--- /dev/null
+++ b/scripts/windows/before_script.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -e
+
+if [[ "$TRAVIS_OS_NAME" != "windows" ]]; then
+    echo "Incorrect \$TRAVIS_OS_NAME: expected windows, got $TRAVIS_OS_NAME"
+    exit 1
+fi
+
+$build_env autoconf
+$build_env ./configure $CONFIGURE_FLAGS
+# mingw32-make simply means "make", unrelated to mingw32 vs mingw64.
+# Simply disregard the prefix and treat is as "make".
+$build_env mingw32-make -j3
+# At the moment, it's impossible to make tests in parallel,
+# seemingly due to concurrent writes to '.pdb' file. I don't know why
+# that happens, because we explicitly supply '/Fs' to the compiler.
+# Until we figure out how to fix it, we should build tests sequentially
+# on Windows.
+$build_env mingw32-make tests
diff --git a/scripts/windows/script.sh b/scripts/windows/script.sh
new file mode 100644
index 00000000..3a27f70a
--- /dev/null
+++ b/scripts/windows/script.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -e
+
+if [[ "$TRAVIS_OS_NAME" != "windows" ]]; then
+    echo "Incorrect \$TRAVIS_OS_NAME: expected windows, got $TRAVIS_OS_NAME"
+    exit 1
+fi
+
+$build_env mingw32-make -k check