Merge pull request #39998 from thaJeztah/carry_39971_unittests_junit_1

Windows: Build and use gotestsum for running all tests

4 files changed, 120 insertions, 24 deletions

diff --git a/Dockerfile.windows b/Dockerfile.windows
index 3043e1619e..95d6a89ea4 100644
--- a/Dockerfile.windows
+++ b/Dockerfile.windows
@@ -166,6 +166,7 @@ FROM microsoft/windowsservercore
 SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
 
 ARG GO_VERSION=1.13.4
+ARG GOTESTSUM_COMMIT=v0.3.5
 
 # Environment variable notes:
 #  - GO_VERSION must be consistent with 'Dockerfile' used by Linux.
@@ -174,7 +175,8 @@ ENV GO_VERSION=${GO_VERSION} `
     GIT_VERSION=2.11.1 `
     GOPATH=C:\gopath `
     GO111MODULE=off `
-    FROM_DOCKERFILE=1
+    FROM_DOCKERFILE=1 `
+    GOTESTSUM_COMMIT=${GOTESTSUM_COMMIT}
 
 RUN `
   Function Test-Nano() { `
@@ -249,13 +251,46 @@ RUN `
   Remove-Item C:\binutils.zip; `
   Remove-Item C:\gitsetup.zip; `
   `
-  Write-Host INFO: Creating source directory...; `
-  New-Item -ItemType Directory -Path ${GOPATH}\src\github.com\docker\docker | Out-Null; `
+  # Ensure all directories exist that we will require below....
+  $srcDir = """$Env:GOPATH`\src\github.com\docker\docker\bundles"""; `
+  Write-Host INFO: Ensuring existence of directory $srcDir...; `
+  New-Item -Force -ItemType Directory -Path $srcDir | Out-Null; `
   `
   Write-Host INFO: Configuring git core.autocrlf...; `
-  C:\git\cmd\git config --global core.autocrlf true; `
+  C:\git\cmd\git config --global core.autocrlf true;
+
+RUN `
+  Function Build-GoTestSum() { `
+    Write-Host "INFO: Building gotestsum version $Env:GOTESTSUM_COMMIT in $Env:GOPATH"; `
+    $optsForGet = @('"get"', '"-d"', '"gotest.tools/gotestsum"'); `
+    &go $optsForGet; `
+    $savedExitCode = $LASTEXITCODE; `
+    if ($savedExitCode -ne 0) {  `
+      Throw '"Failed getting gotestsum sources..."'  `
+    }; `
+    Write-Host "INFO:     Sources obtained for gotestsum..."; `
+    $GotestsumPath=Join-Path -Path $Env:GOPATH -ChildPath "src\gotest.tools\gotestsum"; `
+    Push-Location $GotestsumPath; `
+    $optsForCheckout = @('"checkout"', '"-q"', """$Env:GOTESTSUM_COMMIT"""); `
+    &git $optsForCheckout; `
+    $savedExitCode = $LASTEXITCODE; `
+    if ($savedExitCode -eq 0) { `
+      Write-Host "INFO:     Checkout done for gotestsum..."; `
+      $optsForBuild = @('"build"', '"-buildmode=exe"'); `
+      &go $optsForBuild; `
+      $savedExitCode = $LASTEXITCODE; `
+    } else { `
+      Throw '"gotestsum checkout failed..."'; `
+    } `
+    Pop-Location; `
+    `
+    if ($savedExitCode -ne 0) {  `
+      Throw '"gotestsum build failed..."'; `
+    } `
+    Write-Host "INFO:     Build done for gotestsum..."; `
+  } `
   `
-  Write-Host INFO: Completed
+  Build-GoTestSum
 
 # Make PowerShell the default entrypoint
 ENTRYPOINT ["powershell.exe"]
diff --git a/Jenkinsfile b/Jenkinsfile
index cdb27e3e37..f3b7ab9f13 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -928,13 +928,14 @@ pipeline {
                     }
                     post {
                         always {
+                            junit testResults: 'bundles/junit-report-*.xml', allowEmptyResults: true
                             catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') {
                                 powershell '''
                                 $bundleName="windowsRS1-integration"
                                 Write-Host -ForegroundColor Green "Creating ${bundleName}-bundles.zip"
 
                                 # archiveArtifacts does not support env-vars to , so save the artifacts in a fixed location
-                                Compress-Archive -Path "${env:TEMP}/CIDUT.out", "${env:TEMP}/CIDUT.err" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip"
+                                Compress-Archive -Path "${env:TEMP}/CIDUT.out", "${env:TEMP}/CIDUT.err", "${env:TEMP}/testresults/unittests/junit-report-unit-tests.xml" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip"
                                 '''
 
                                 archiveArtifacts artifacts: '*-bundles.zip', allowEmptyArchive: true
@@ -988,13 +989,14 @@ pipeline {
                     }
                     post {
                         always {
+                            junit testResults: 'bundles/junit-report-*.xml', allowEmptyResults: true
                             catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') {
                                 powershell '''
                                 $bundleName="windowsRS5-integration"
                                 Write-Host -ForegroundColor Green "Creating ${bundleName}-bundles.zip"
 
                                 # archiveArtifacts does not support env-vars to , so save the artifacts in a fixed location
-                                Compress-Archive -Path "${env:TEMP}/CIDUT.out", "${env:TEMP}/CIDUT.err" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip"
+                                Compress-Archive -Path "${env:TEMP}/CIDUT.out", "${env:TEMP}/CIDUT.err", "${env:TEMP}/junit-report-*.xml" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip"
                                 '''
 
                                 archiveArtifacts artifacts: '*-bundles.zip', allowEmptyArchive: true
diff --git a/hack/ci/windows.ps1 b/hack/ci/windows.ps1
index 3bf951a4cb..554940d1df 100644
--- a/hack/ci/windows.ps1
+++ b/hack/ci/windows.ps1
@@ -447,6 +447,8 @@ Try {
     $errorActionPreference='Stop'
     New-Item -ItemType Directory "$env:TEMP" -ErrorAction SilentlyContinue | Out-Null
     New-Item -ItemType Directory "$env:TEMP\userprofile" -ErrorAction SilentlyContinue  | Out-Null
+    New-Item -ItemType Directory "$env:TEMP\testresults" -ErrorAction SilentlyContinue  | Out-Null
+    New-Item -ItemType Directory "$env:TEMP\testresults\unittests" -ErrorAction SilentlyContinue  | Out-Null
     New-Item -ItemType Directory "$env:TEMP\localappdata" -ErrorAction SilentlyContinue | Out-Null
     New-Item -ItemType Directory "$env:TEMP\binary" -ErrorAction SilentlyContinue | Out-Null
     New-Item -ItemType Directory "$env:TEMP\installer" -ErrorAction SilentlyContinue | Out-Null
@@ -525,6 +527,16 @@ Try {
         if (-not($LastExitCode -eq 0)) {
             Throw "ERROR: Failed to docker cp the daemon binary (dockerd.exe) to $env:TEMP\binary"
         }
+
+        $GotestsumBinRelLocationInContainer="src\gotest.tools\gotestsum\gotestsum.exe"
+        if (-not($LastExitCode -eq 0)) {
+            Throw "ERROR: Failed to docker cp the gotestsum binary (gotestsum.exe) to $env:TEMP\binary"
+        }
+        docker cp "$COMMITHASH`:c`:\gopath\$GotestsumBinRelLocationInContainer" $env:TEMP\binary\
+        if (-not (Test-Path "$env:TEMP\binary\gotestsum.exe")) {
+            Throw "ERROR: gotestsum.exe not found...." `
+        }
+
         $ErrorActionPreference = "Stop"
 
         # Copy the built dockerd.exe to dockerd-$COMMITHASH.exe so that easily spotted in task manager.
@@ -774,11 +786,36 @@ Try {
     # Run the unit tests inside a container unless SKIP_UNIT_TESTS is defined
     if (($null -eq $env:LCOW_MODE) -and ($null -eq $env:LCOW_BASIC_MODE)) {
         if ($null -eq $env:SKIP_UNIT_TESTS) {
+            $ContainerNameForUnitTests = $COMMITHASH + "_UnitTests"
             Write-Host -ForegroundColor Cyan "INFO: Running unit tests at $(Get-Date)..."
             $ErrorActionPreference = "SilentlyContinue"
-            $Duration=$(Measure-Command {docker run -e DOCKER_GITCOMMIT=$COMMITHASH$CommitUnsupported docker hack\make.ps1 -TestUnit | Out-Host })
+            $Duration=$(Measure-Command {docker run --name $ContainerNameForUnitTests -e DOCKER_GITCOMMIT=$COMMITHASH$CommitUnsupported docker hack\make.ps1 -TestUnit | Out-Host })
+            $TestRunExitCode = $LastExitCode
             $ErrorActionPreference = "Stop"
+
+            # Saving for artifacts......
+            $unitTestsContPath="$ContainerNameForUnitTests`:c`:\gopath\src\github.com\docker\docker\bundles"
+            $JunitExpectedContFilePath = "$unitTestsContPath\junit-report-unit-tests.xml"
+            docker cp $JunitExpectedContFilePath "$TEMPORIG"
+            if (-not($LastExitCode -eq 0)) {
+                Throw "ERROR: Failed to docker cp the unit tests report ($JunitExpectedContFilePath) to $TEMPORIG"
+            }
+
+            if (Test-Path "$TEMPORIG\junit-report-unit-tests.xml") {
+                Write-Host -ForegroundColor Magenta "INFO: Unit tests results($TEMPORIG\junit-report-unit-tests.xml) exist. pwd=$pwd"
+            } else {
+                Write-Host -ForegroundColor Magenta "ERROR: Unit tests results($TEMPORIG\junit-report-unit-tests.xml) do not exist. pwd=$pwd"
+            }
+            
+            # Saving where jenkins will take a look at.....
+            $bundlesDir = "bundles"
+            New-Item -Force -ItemType Directory $bundlesDir | Out-Null
+            docker cp $JunitExpectedContFilePath "$bundlesDir"
             if (-not($LastExitCode -eq 0)) {
+                Throw "ERROR: Failed to docker cp the unit tests report ($JunitExpectedContFilePath) to $bundlesDir"
+            }
+
+            if (-not($TestRunExitCode -eq 0)) {
                 Throw "ERROR: Unit tests failed"
             }
             Write-Host  -ForegroundColor Green "INFO: Unit tests ended at $(Get-Date). Duration`:$Duration"
@@ -830,7 +867,9 @@ Try {
             $env:OrigDOCKER_HOST="$env:DOCKER_HOST"
     
             #https://blogs.technet.microsoft.com/heyscriptingguy/2011/09/20/solve-problems-with-external-command-lines-in-powershell/ is useful to see tokenising
-            $c = "go test "
+            $jsonFilePath = "..\\bundles\\go-test-report-intcli-tests.json"
+            $xmlFilePath = "..\\bundles\\junit-report-intcli-tests.xml"
+            $c = "gotestsum --format=standard-quiet --jsonfile=$jsonFilePath --junitfile=$xmlFilePath -- "
             $c += "`"-test.v`" "
             if ($null -ne $env:INTEGRATION_TEST_NAME) { # Makes is quicker for debugging to be able to run only a subset of the integration tests
                 $c += "`"-test.run`" "
@@ -864,10 +903,12 @@ Try {
                 if (!($env:INTEGRATION_TESTFLAGS)) {
                     $env:INTEGRATION_TESTFLAGS = "-test.v"
                 }
-                Set-Location "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker"
                 $start=(Get-Date); Invoke-Expression ".\hack\make.ps1 -TestIntegration"; $Duration=New-Timespan -Start $start -End (Get-Date)
+                $IntTestsRunResult = $LastExitCode
                 $ErrorActionPreference = "Stop"
-                if (-not($LastExitCode -eq 0)) {
+                # Copy all the test results to TEMPORIG for archival
+                Copy-Item -Path "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker\bundles\junit-report*xml" -Destination $TEMPORIG
+                if (-not($IntTestsRunResult -eq 0)) {
                     Throw "ERROR: Integration API tests failed at $(Get-Date). Duration`:$Duration"
                 }
 
@@ -877,6 +918,7 @@ Try {
                 Set-Location "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker\integration-cli"
                 # Explicit to not use measure-command otherwise don't get output as it goes
                 $start=(Get-Date); Invoke-Expression $c; $Duration=New-Timespan -Start $start -End (Get-Date)
+                Copy-Item -Path $xmlFilePath -Destination $TEMPORIG
             }
             $ErrorActionPreference = "Stop"
             if (-not($LastExitCode -eq 0)) {
diff --git a/hack/make.ps1 b/hack/make.ps1
index e8ea35d798..e613e2360e 100644
--- a/hack/make.ps1
+++ b/hack/make.ps1
@@ -94,6 +94,7 @@ param(
 $ErrorActionPreference = "Stop"
 $ProgressPreference = "SilentlyContinue"
 $pushed=$False  # To restore the directory if we have temporarily pushed to one.
+Set-Variable GOTESTSUM_LOCATION -option Constant -value "$env:GOPATH/src/gotest.tools/gotestsum"
 
 # Utility function to get the commit ID of the repository
 Function Get-GitCommit() {
@@ -319,31 +320,42 @@ Function Run-UnitTests() {
     $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/man"
     $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/integration"
     $pkgList = $pkgList -replace "`r`n", " "
-    $goTestCommand = "go test" + $raceParm + " -cover -ldflags -w -tags """ + "autogen daemon" + """ -a """ + "-test.timeout=10m" + """ $pkgList"
+    $goTestCommand = "$GOTESTSUM_LOCATION\gotestsum.exe --format=standard-quiet --jsonfile=bundles\go-test-report-unit-tests.json --junitfile=bundles\junit-report-unit-tests.xml -- " + $raceParm + " -cover -ldflags -w -a """ + "-test.timeout=10m" + """ $pkgList"
+    Write-Host "INFO: Invoking unit tests run with $goTestCommand"
     Invoke-Expression $goTestCommand
     if ($LASTEXITCODE -ne 0) { Throw "Unit tests failed" }
 }
 
 # Run the integration tests
 Function Run-IntegrationTests() {
-    $env:DOCKER_INTEGRATION_DAEMON_DEST = $root + "\bundles\tmp"
+    $escRoot = [Regex]::Escape($root)
+    $env:DOCKER_INTEGRATION_DAEMON_DEST = $bundlesDir + "\tmp"
     $dirs = go list -test -f '{{- if ne .ForTest `"`" -}}{{- .Dir -}}{{- end -}}' .\integration\...
-    $integration_api_dirs = @()
     ForEach($dir in $dirs) {
-        $integration_api_dirs += $dir
-        Write-Host "Building test suite binary $dir"
-        go test -c -o "$dir\test.exe" $dir
-    }
-
-    ForEach($dir in $integration_api_dirs) {
+        # Normalize directory name for using in the test results files.
+        $normDir = $dir.Trim()
+        $normDir = $normDir -replace $escRoot, ""
+        $normDir = $normDir -replace "\\", "-"
+        $normDir = $normDir -replace "\/", "-"
+        $normDir = $normDir -replace "\.", "-"
+        if ($normDir.StartsWith("-"))
+        {
+            $normDir = $normDir.TrimStart("-")
+        }
+        if ($normDir.EndsWith("-"))
+        {
+            $normDir = $normDir.TrimEnd("-")
+        }
+        $jsonFilePath = $bundlesDir + "\go-test-report-int-tests-$normDir" + ".json"
+        $xmlFilePath = $bundlesDir + "\junit-report-int-tests-$normDir" + ".xml"
         Set-Location $dir
         Write-Host "Running $($PWD.Path)"
         $pinfo = New-Object System.Diagnostics.ProcessStartInfo
-        $pinfo.FileName = "$($PWD.Path)\test.exe"
+        $pinfo.FileName = "gotestsum.exe"
         $pinfo.WorkingDirectory = "$($PWD.Path)"
         $pinfo.RedirectStandardError = $true
         $pinfo.UseShellExecute = $false
-        $pinfo.Arguments = $env:INTEGRATION_TESTFLAGS
+        $pinfo.Arguments = "--format=standard-quiet --jsonfile=$jsonFilePath --junitfile=$xmlFilePath -- $env:INTEGRATION_TESTFLAGS"
         $p = New-Object System.Diagnostics.Process
         $p.StartInfo = $pinfo
         $p.Start() | Out-Null
@@ -351,6 +363,8 @@ Function Run-IntegrationTests() {
         $err = $p.StandardError.ReadToEnd()
         if (($LASTEXITCODE -ne 0) -and ($err -notlike "*warning: no tests to run*")) {
             Throw "Integration tests failed: $err"
+        } else {
+            Write-Host "$err"
         }
     }
 }
@@ -363,6 +377,11 @@ Try {
     $root = $(Split-Path $MyInvocation.MyCommand.Definition -Parent | Split-Path -Parent)
     Push-Location $root
 
+    # Ensure the bundles directory exists
+    $bundlesDir = $root + "\bundles"
+    Set-Variable bundlesDir -option ReadOnly
+    New-Item -Force $bundlesDir -ItemType Directory | Out-Null
+
     # Handle the "-All" shortcut to turn on all things we can handle.
     # Note we expressly only include the items which can run in a container - the validations tests cannot
     # as they require the .git directory which is excluded from the image by .dockerignore
@@ -424,8 +443,6 @@ Try {
 
     # Build the binaries
     if ($Client -or $Daemon) {
-        # Create the bundles directory if it doesn't exist
-        if (-not (Test-Path ".\bundles")) { New-Item ".\bundles" -ItemType Directory | Out-Null }
 
         # Perform the actual build
         if ($Daemon) { Execute-Build "daemon" "daemon" "dockerd" }