ci: Use self hosted CI data

3 files changed, 75 insertions, 243 deletions

diff --git a/ci/azure-steps.yml b/ci/azure-steps.yml
index 83c2e4ddc..233bbfa36 100644
--- a/ci/azure-steps.yml
+++ b/ci/azure-steps.yml
@@ -1,172 +1,8 @@
 steps:
-- powershell: |
-     echo "=== Check skip CI ==="
-     python ./skip_ci.py --base-branch-env=SYSTEM_PULLREQUEST_TARGETBRANCH --is-pull-env=SYSTEM_PULLREQUEST_PULLREQUESTID --base-branch-origin
-     if ($LastExitCode -ne 0) {
-        exit 0
-     }
-
-     echo "=== BEGIN CI setup ==="
-     # remove Chocolately, MinGW, Strawberry Perl from path, so we don't find gcc/gfortran and try to use it
-     $env:Path = ($env:Path.Split(';') | Where-Object { $_ -notmatch 'mingw|Strawberry|Chocolatey' }) -join ';'
-
-     # download and install prerequisites
-     function DownloadFile([String] $Source, [String] $Destination) {
-       $retries = 10
-       echo ('Downloading {0} ...' -f $Source)
-       for ($i = 1; $i -le $retries; $i++) {
-           try {
-               (New-Object net.webclient).DownloadFile($Source, $Destination)
-               echo '... DONE'
-               break # succeeded
-           } catch [net.WebException] {
-               if ($i -eq $retries) {
-                   throw # fail on last retry
-               }
-               $backoff = (10 * $i) # backoff 10s, 20s, 30s...
-               echo ('{0}: {1}' -f $Source, $_.Exception.Message)
-               echo ('Retrying in {0}s...' -f $backoff)
-               Start-Sleep -m ($backoff * 1000)
-              }
-          }
-     }
-
-     DownloadFile -Source 'https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-win.zip' -Destination $(System.WorkFolder)\ninja-win.zip
-     # ExpandArchive is only available in Powershell 5+
-     Add-Type -AssemblyName System.IO.Compression.FileSystem
-     function unzip {
-         param( [string]$ziparchive, [string]$extractpath )
-         [System.IO.Compression.ZipFile]::ExtractToDirectory( $ziparchive, $extractpath )
-     }
-     unzip $(System.WorkFolder)\ninja-win.zip $(System.WorkFolder)
-     DownloadFile -Source 'https://github.com/mesonbuild/cidata/raw/master/win32/pkg-config.exe' -Destination $(System.WorkFolder)\pkg-config.exe
-     DownloadFile -Source 'https://download.microsoft.com/download/D/B/B/DBB64BA1-7B51-43DB-8BF1-D1FB45EACF7A/msmpisdk.msi' -Destination msmpisdk.msi
-     DownloadFile -Source 'https://download.microsoft.com/download/D/B/B/DBB64BA1-7B51-43DB-8BF1-D1FB45EACF7A/MSMpiSetup.exe' -Destination MSMpiSetup.exe
-     Start-Process msiexec.exe -ArgumentList '/i msmpisdk.msi /quiet' -Wait
-     Start-Process .\MSMpiSetup.exe -ArgumentList '-unattend -full' -Wait
-
-     # import ms-mpi env vars (set by installer)
-     foreach ($p in "MSMPI_INC", "MSMPI_LIB32", "MSMPI_LIB64") {
-       $v = [Environment]::GetEnvironmentVariable($p, "Machine")
-       Set-Content "env:$p" "$v"
-     }
-
-     # install boost (except for clang-cl)
-     if ($env:arch -eq 'x86') { $boost_bitness = '32' } else { $boost_bitness = '64' }
-     if ($env:compiler -eq 'msvc2017' -Or $env:compiler -eq 'msvc2019' -Or $env:compiler -eq 'clang-cl') {
-       $boost_version = '1.72.0' ; $boost_abi_tag = '14.1'
-     }
-     if ($boost_version) {
-       $boost_filename = $boost_version.Replace('.', '_')
-       Downloadfile -Source "https://sourceforge.net/projects/boost/files/boost-binaries/$boost_version/boost_$boost_filename-msvc-$boost_abi_tag-$boost_bitness.exe" -Destination boost_$boost_filename-msvc-$boost_abi_tag-$boost_bitness.exe
-       Start-Process "boost_$boost_filename-msvc-$boost_abi_tag-$boost_bitness.exe" -ArgumentList "/dir=$(System.WorkFolder)\boost_$boost_filename /silent" -Wait
-       $env:BOOST_ROOT = "$(System.WorkFolder)\boost_$boost_filename"
-       $env:Path = "$env:Path;$env:BOOST_ROOT\lib$boost_bitness-msvc-$boost_abi_tag"
-     } else {
-       $env:BOOST_ROOT = ""
-     }
-
-     # install D compiler and dub packages
-     if ($env:backend -eq 'ninja') {
-          & .\ci\install-dmd.ps1
-          $arch = 'x86_mscoff'
-          if ($Env:arch -eq 'x64') {
-            $arch = 'x86_64'
-          }
-          & dub fetch urld
-          & dub build urld --compiler=dmd --arch=$arch
-          & dub fetch dubtestproject
-          & dub build dubtestproject:test1 --compiler=dmd --arch=$arch
-          & dub build dubtestproject:test2 --compiler=dmd --arch=$arch
-        }
-
-     # test_find_program exercises some behaviour which relies on .py being in PATHEXT
-     $env:PATHEXT += ';.py'
-
-     # add downloads to PATH
-     $env:Path = "$env:SYSTEM_WORKFOLDER;$env:Path"
-
-     $origPath = $env:Path
-     # import visual studio variables
-     echo "source vcvarsall.bat"
-     if ($env:compiler -eq 'msvc2019') {
-       $vcvars = "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat"
-     } else {
-       # Note: this is also for clangcl
-       $vcvars = "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvarsall.bat"
-     }
-
-     ## A multiline commit message containing "=" can interact badly with this
-     ## hack to extract the environment from vcvarsall.bat
-     Remove-Item env:BUILD_SOURCEVERSIONMESSAGE
-
-     ## ask cmd.exe to output the environment table after the batch file completes
-     $tempFile = [IO.Path]::GetTempFileName()
-     cmd /c " `"$vcvars`" $env:arch && set > `"$tempFile`" "
-
-     ## go through the environment variables in the temp file.
-     ## for each of them, set the variable in our local environment.
-     Get-Content $tempFile | Foreach-Object {
-       if($_ -match "^(.*?)=(.*)$") {
-         Set-Content "env:\$($matches[1])" $matches[2]
-       }
-     }
-     Remove-Item $tempFile
-
-     if ($env:compiler -eq 'clang-cl') {
-       # drop visual studio from PATH
-       # (but leave INCLUDE, LIB and WindowsSdkDir environment variables set)
-       $env:Path = $origPath
-
-       # install llvm for clang-cl builds
-       DownloadFile -Source 'http://releases.llvm.org/9.0.0/LLVM-9.0.0-win64.exe' -Destination LLVM-9.0.0-win64.exe
-       Start-Process .\LLVM-9.0.0-win64.exe -ArgumentList '/S' -Wait
-       $env:Path = "C:\Program Files\LLVM\bin;$env:Path"
-       $env:CC = "clang-cl"
-       $env:CXX = "clang-cl"
-
-       # and use Windows SDK tools
-       $env:Path = "$env:WindowsSdkDir\bin\$env:Arch;$env:Path"
-     }
-
-     # add .NET framework tools to path for resgen for C# tests
-     # (always use 32-bit tool, as there doesn't seem to be a 64-bit tool)
-     if ((Get-Command "resgen.exe" -ErrorAction SilentlyContinue) -eq $null) {
-       $env:Path = "$env:WindowsSDK_ExecutablePath_x86;$env:Path"
-     }
-
-     echo "=== END CI setup ==="
-
-     if ($env:backend -eq 'ninja') {
-       ninja --version
-     } else {
-       MSBuild /version
-     }
-
-     echo "=== PATH BEGIN ==="
-     echo ($env:Path).Replace(';',"`n")
-     echo "=== PATH END ==="
-     echo ""
-     echo "Locating Python:"
-     where.exe python
-     python --version
-
-     # Needed for running unit tests in parallel.
-     python -m pip --disable-pip-version-check install --upgrade pefile pytest-xdist
-
-
-     echo ""
-     echo "Locating cl, rc, link:"
-     where.exe cl
-     where.exe rc
-     where.exe link
-
-     echo ""
-     echo "=== Start running tests ==="
-     # Starting from VS2019 Powershell(?) will fail the test run
-     # if it prints anything to stderr. Python's test runner
-     # does that by default so we need to forward it.
-     cmd /c 'python 2>&1' run_tests.py --backend $(backend)
+- task: PowerShell@2
+  inputs:
+    targetType: 'filePath'
+    filePath: .\ci\run.ps1
 
 - task: PublishTestResults@2
   inputs:
diff --git a/ci/install-dmd.ps1 b/ci/install-dmd.ps1
deleted file mode 100644
index ebfce16bc..000000000
--- a/ci/install-dmd.ps1
+++ /dev/null
@@ -1,75 +0,0 @@
-param (
-    [string]$Version = $null
-)
-Set-StrictMode -Version latest
-$ErrorActionPreference = "Stop"
-$ProgressPreference = "SilentlyContinue"
-
-echo "=== Installing DMD ==="
-
-# default installation directory
-$dmd_install = "C:\D"
-$dmd_version_file = "C:\cache\DMD_LATEST"
-
-if (!$Version) {
-    #echo "Fetching latest DMD version..."
-    $dmd_latest_url = "http://downloads.dlang.org/releases/LATEST"
-    $retries = 10
-    echo ('Downloading {0} ...' -f $dmd_latest_url)
-    for ($i = 1; $i -le $retries; $i++) {
-        try {
-            [system.io.directory]::CreateDirectory((Split-Path -parent $dmd_version_file)) > $null
-            Invoke-WebRequest -URI $dmd_latest_url -OutFile $dmd_version_file
-            echo '... DONE'
-            break
-        } catch [net.WebException] {
-            if ($i -eq $retries) {
-                break
-            }
-            $backoff = (10 * $i) # backoff 10s, 20s, 30s...
-            echo ('{0}: {1}' -f $dmd_latest_url, $_.Exception.Message)
-            echo ('Retrying in {0}s...' -f $backoff)
-            Start-Sleep -m ($backoff * 1000)
-        } catch {
-            throw
-        }
-    }
-    if (Test-Path $dmd_version_file) {
-        $dmd_version = Get-Content -Path $dmd_version_file
-    } else {
-        throw "Failed to resolve latest DMD version"
-    }
-} else {
-    $dmd_version = $Version
-}
-$dmd_url = "http://downloads.dlang.org/releases/2.x/$dmd_version/dmd.$dmd_version.windows.zip"
-$dmd_filename = [System.IO.Path]::GetFileName($dmd_url)
-$dmd_archive = Join-Path ($env:temp) $dmd_filename
-
-echo "Downloading $dmd_filename..."
-$retries = 10
-for ($i = 1; $i -le $retries; $i++) {
-    try {
-        (New-Object net.webclient).DownloadFile($dmd_url, $dmd_archive)
-        break
-    } catch [net.WebException] {
-        if ($i -eq $retries) {
-            throw # fail on last retry
-        }
-        $backoff = (10 * $i) # backoff 10s, 20s, 30s...
-        echo ('{0}: {1}' -f $dmd_url, $_.Exception.Message)
-        echo ('Retrying in {0}s...' -f $backoff)
-        Start-Sleep -m ($backoff * 1000)
-    }
-}
-
-echo "Extracting $dmd_filename..."
-Expand-Archive $dmd_archive -Force -DestinationPath $dmd_install
-
-# add to environment path
-echo "Installing DMD..."
-$dmd_bin = Join-Path $dmd_install "dmd2\windows\bin"
-$Env:Path = $Env:Path + ";" + $dmd_bin
-
-echo "Testing DMD..."
-& dmd.exe --version
diff --git a/ci/run.ps1 b/ci/run.ps1
new file mode 100644
index 000000000..8729185fe
--- /dev/null
+++ b/ci/run.ps1
@@ -0,0 +1,71 @@
+python ./skip_ci.py --base-branch-env=SYSTEM_PULLREQUEST_TARGETBRANCH --is-pull-env=SYSTEM_PULLREQUEST_PULLREQUESTID --base-branch-origin
+if ($LastExitCode -ne 0) {
+  exit 0
+}
+
+# remove Chocolately, MinGW, Strawberry Perl from path, so we don't find gcc/gfortran and try to use it
+$env:Path = ($env:Path.Split(';') | Where-Object { $_ -notmatch 'mingw|Strawberry|Chocolatey' }) -join ';'
+
+# download and install prerequisites
+function DownloadFile([String] $Source, [String] $Destination) {
+  $retries = 10
+  echo "Downloading $Source"
+  for ($i = 1; $i -le $retries; $i++) {
+      try {
+          (New-Object net.webclient).DownloadFile($Source, $Destination)
+          break # succeeded
+      } catch [net.WebException] {
+          if ($i -eq $retries) {
+              throw # fail on last retry
+          }
+          $backoff = (10 * $i) # backoff 10s, 20s, 30s...
+          echo ('{0}: {1}' -f $Source, $_.Exception.Message)
+          echo ('Retrying in {0}s...' -f $backoff)
+          Start-Sleep -m ($backoff * 1000)
+        }
+    }
+}
+
+
+if ($env:backend -eq 'ninja') { $dmd = $true } else { $dmd = $false }
+
+DownloadFile -Source https://github.com/mesonbuild/cidata/releases/download/ci1/ci_data.zip -Destination $env:AGENT_WORKFOLDER\ci_data.zip
+echo "Extracting  ci_data.zip"
+Expand-Archive $env:AGENT_WORKFOLDER\ci_data.zip -DestinationPath $env:AGENT_WORKFOLDER\ci_data
+& "$env:AGENT_WORKFOLDER\ci_data\install.ps1" -Arch $env:arch -Compiler $env:compiler -Boost $true -DMD $dmd
+
+
+echo "=== PATH BEGIN ==="
+echo ($env:Path).Replace(';',"`n")
+echo "=== PATH END ==="
+echo ""
+
+$progs = @("python","ninja","pkg-config","cl","rc","link")
+foreach ($prog in $progs) {
+  echo ""
+  echo "Locating ${prog}:"
+  where.exe $prog
+}
+
+echo ""
+echo "Ninja / MSBuld version:"
+if ($env:backend -eq 'ninja') {
+  ninja --version
+} else {
+  MSBuild /version
+}
+
+echo ""
+echo "Python version:"
+python --version
+
+# Needed for running unit tests in parallel.
+echo ""
+python -m pip --disable-pip-version-check install --upgrade pefile pytest-xdist
+
+echo ""
+echo "=== Start running tests ==="
+# Starting from VS2019 Powershell(?) will fail the test run
+# if it prints anything to stderr. Python's test runner
+# does that by default so we need to forward it.
+cmd /c 'python 2>&1' run_tests.py --backend $env:backend