ci: updates for CI

14 files changed, 170 insertions, 20 deletions

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 74bab53f3..3435556af 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -10,7 +10,7 @@ on:
   workflow_dispatch:
 
 env:
-  docker-registry: docker.pkg.github.com
+  docker-registry: ghcr.io
   docker-config-path: source/ci/docker
 
 jobs:
@@ -69,7 +69,7 @@ jobs:
         if [ "${{ matrix.container.base }}" != "" ]; then
           BASE_ARG="--build-arg BASE=${{ matrix.container.base }}"
         fi
-        docker build -t ${{ env.docker-registry-container-sha }} ${BASE_ARG} -f ${{ env.dockerfile }} .
+        docker build -t ${{ env.docker-registry-container-sha }} --build-arg UID=$(id -u) --build-arg GID=$(id -g) ${BASE_ARG} -f ${{ env.dockerfile }} .
         docker tag ${{ env.docker-registry-container-sha }} ${{ env.docker-registry-container-latest }}
         docker push ${{ env.docker-registry-container-sha }}
         docker push ${{ env.docker-registry-container-latest }}
@@ -85,6 +85,7 @@ jobs:
       matrix:
         platform:
         - name: "Linux (Xenial, GCC, OpenSSL)"
+          id: xenial-gcc-openssl
           container:
             name: xenial
           env:
@@ -93,6 +94,7 @@ jobs:
             CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DUSE_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=ON -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON
           os: ubuntu-latest
         - name: Linux (Xenial, GCC, mbedTLS)
+          id: xenial-gcc-mbedtls
           container:
             name: xenial
           env:
@@ -101,6 +103,7 @@ jobs:
             CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DDEPRECATE_HARD=ON -DUSE_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=ON
           os: ubuntu-latest
         - name: "Linux (Xenial, Clang, OpenSSL)"
+          id: xenial-clang-openssl
           container:
             name: xenial
           env:
@@ -109,6 +112,7 @@ jobs:
             CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DDEPRECATE_HARD=ON -DUSE_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=ON
           os: ubuntu-latest
         - name: "Linux (Xenial, Clang, mbedTLS)"
+          id: xenial-clang-mbedtls
           container:
             name: xenial
           env:
@@ -117,6 +121,7 @@ jobs:
             CMAKE_GENERATOR: Ninja
           os: ubuntu-latest
         - name: "Linux (MemorySanitizer)"
+          id: memorysanitizer
           container:
             name: focal
           env:
@@ -130,6 +135,7 @@ jobs:
             UBSAN_OPTIONS: print_stacktrace=1
           os: ubuntu-latest
         - name: "Linux (UndefinedBehaviorSanitizer)"
+          id: ubsanitizer
           container:
             name: focal
           env:
@@ -143,6 +149,7 @@ jobs:
             UBSAN_OPTIONS: print_stacktrace=1
           os: ubuntu-latest
         - name: "Linux (ThreadSanitizer)"
+          id: threadsanitizer
           container:
             name: focal
           env:
@@ -157,6 +164,7 @@ jobs:
             TSAN_OPTIONS: suppressions=/home/libgit2/source/script/thread-sanitizer.supp second_deadlock_stack=1
           os: ubuntu-latest
         - name: "macOS"
+          id: macos
           os: macos-10.15
           env:
             CC: clang
@@ -166,6 +174,7 @@ jobs:
             SKIP_NEGOTIATE_TESTS: true
           setup-script: osx
         - name: "Windows (amd64, Visual Studio)"
+          id: windows-amd64-vs
           os: windows-2019
           env:
             ARCH: amd64
@@ -174,6 +183,7 @@ jobs:
             SKIP_SSH_TESTS: true
             SKIP_NEGOTIATE_TESTS: true
         - name: "Windows (x86, Visual Studio)"
+          id: windows-x86-vs
           os: windows-2019
           env:
             ARCH: x86
@@ -182,6 +192,7 @@ jobs:
             SKIP_SSH_TESTS: true
             SKIP_NEGOTIATE_TESTS: true
         - name: "Windows (amd64, mingw)"
+          id: windows-amd64-mingw
           os: windows-2019
           setup-script: mingw
           env:
@@ -193,6 +204,7 @@ jobs:
             SKIP_SSH_TESTS: true
             SKIP_NEGOTIATE_TESTS: true
         - name: "Windows (x86, mingw)"
+          id: windows-x86-mingw
           os: windows-2019
           setup-script: mingw
           env:
@@ -214,7 +226,7 @@ jobs:
         path: source
         fetch-depth: 0
     - name: Set up build environment
-      run: source/ci/setup-${{ matrix.platform.setup-script }}.sh
+      run: source/ci/setup-${{ matrix.platform.setup-script }}-build.sh
       shell: bash
       if: matrix.platform.setup-script != ''
     - name: Setup QEMU
@@ -237,10 +249,12 @@ jobs:
         export GITTEST_NEGOTIATE_PASSWORD="${{ secrets.GITTEST_NEGOTIATE_PASSWORD }}"
 
         if [ -n "${{ matrix.platform.container.name }}" ]; then
+          mkdir build
           docker run \
               --rm \
-              --user libgit2:libgit2 \
+              --user "$(id -u):$(id -g)" \
               -v "$(pwd)/source:/home/libgit2/source" \
+              -v "$(pwd)/build:/home/libgit2/build" \
               -w /home/libgit2 \
               -e ASAN_SYMBOLIZER_PATH \
               -e CC \
@@ -248,19 +262,40 @@ jobs:
               -e CMAKE_GENERATOR \
               -e CMAKE_OPTIONS \
               -e GITTEST_NEGOTIATE_PASSWORD \
+              -e GITTEST_FLAKY_STAT \
               -e PKG_CONFIG_PATH \
               -e SKIP_NEGOTIATE_TESTS \
               -e SKIP_SSH_TESTS \
               -e TSAN_OPTIONS \
               -e UBSAN_OPTIONS \
               ${{ env.docker-registry-container-sha }} \
-              /bin/bash -c "mkdir build && cd build && ../source/ci/build.sh && ../source/ci/test.sh"
+              /bin/bash -c "cd build && ../source/ci/build.sh && ../source/ci/test.sh"
         else
-          mkdir build && cd build
+          mkdir build
+          cd build
           ../source/ci/build.sh
           ../source/ci/test.sh
         fi
       shell: bash
+    - name: Upload test results
+      uses: actions/upload-artifact@v3
+      if: success() || failure()
+      with:
+        name: test-results-${{ matrix.platform.id }}
+        path: build/results_*.xml
+
+  test_results:
+    name: Test results
+    needs: [ build ]
+    runs-on: ubuntu-latest
+    steps:
+    - name: Download test results
+      uses: actions/download-artifact@v3
+    - name: Generate test summary
+      uses: test-summary/action@v1
+      with:
+        paths: 'test-results-*/*.xml'
+
 
   # Generate documentation using docurium.  We'll upload the documentation
   # as a build artifact so that it can be reviewed as part of a pull
@@ -270,6 +305,7 @@ jobs:
   documentation:
     name: Generate documentation
     needs: [ containers ]
+    if: success() || failure()
     runs-on: ubuntu-latest
     steps:
     - name: Check out repository
diff --git a/ci/build.sh b/ci/build.sh
index 5a51f925a..21a45af5f 100755
--- a/ci/build.sh
+++ b/ci/build.sh
@@ -59,7 +59,7 @@ echo "##########################################################################
 echo "## Configuring build environment"
 echo "##############################################################################"
 
-echo cmake -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -G \"${CMAKE_GENERATOR}\" ${CMAKE_OPTIONS} -S \"${SOURCE_DIR}\"
+echo "${CMAKE}" -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -G \"${CMAKE_GENERATOR}\" ${CMAKE_OPTIONS} -S \"${SOURCE_DIR}\"
 env PATH="${BUILD_PATH}" "${CMAKE}" -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -G "${CMAKE_GENERATOR}" ${CMAKE_OPTIONS} -S "${SOURCE_DIR}"
 
 echo ""
@@ -69,10 +69,11 @@ echo "##########################################################################
 
 # Determine parallelism; newer cmake supports `--build --parallel` but
 # we cannot yet rely on that.
-if [ "${CMAKE_GENERATOR}" = "Unix Makefiles" -a "${CORES}" != "" ]; then
+if [ "${CMAKE_GENERATOR}" = "Unix Makefiles" -a "${CORES}" != "" -a "${CMAKE_BUILD_OPTIONS}" = "" ]; then
 	BUILDER=(make -j ${CORES})
 else
-	BUILDER=("${CMAKE}" --build .)
+	BUILDER=("${CMAKE}" --build . ${CMAKE_BUILD_OPTIONS})
 fi
 
+echo "${BUILDER[@]}"
 env PATH="${BUILD_PATH}" "${BUILDER[@]}"
diff --git a/ci/docker/bionic b/ci/docker/bionic
index 51af5c01c..52832f3d3 100644
--- a/ci/docker/bionic
+++ b/ci/docker/bionic
@@ -38,7 +38,12 @@ RUN cd /tmp && \
     rm -rf mbedtls-2.16.2
 
 FROM mbedtls AS adduser
-RUN useradd --shell /bin/bash libgit2 --create-home
+ARG UID=""
+ARG GID=""
+RUN if [ "${UID}" != "" ]; then USER_ARG="--uid ${UID}"; fi && \
+    if [ "${GID}" != "" ]; then GROUP_ARG="--gid ${GID}"; fi && \
+    groupadd ${GROUP_ARG} libgit2 && \
+    useradd ${USER_ARG} --gid libgit2 --shell /bin/bash --create-home libgit2
 
 FROM adduser AS configure
 RUN mkdir /var/run/sshd
diff --git a/ci/docker/centos7 b/ci/docker/centos7
index 8105f1442..28ed65081 100644
--- a/ci/docker/centos7
+++ b/ci/docker/centos7
@@ -48,7 +48,12 @@ RUN cd /tmp && \
     rm -rf cmake-3.21.1
 
 FROM cmake AS adduser
-RUN useradd --shell /bin/bash libgit2 --create-home
+ARG UID=""
+ARG GID=""
+RUN if [ "${UID}" != "" ]; then USER_ARG="--uid ${UID}"; fi && \
+    if [ "${GID}" != "" ]; then GROUP_ARG="--gid ${GID}"; fi && \
+    groupadd ${GROUP_ARG} libgit2 && \
+    useradd ${USER_ARG} --gid libgit2 --shell /bin/bash --create-home libgit2
 
 FROM adduser AS configure
 ENV PKG_CONFIG_PATH /usr/local/lib/pkgconfig
diff --git a/ci/docker/centos8 b/ci/docker/centos8
index cca088103..81f0c3c76 100644
--- a/ci/docker/centos8
+++ b/ci/docker/centos8
@@ -1,6 +1,10 @@
 ARG BASE=centos:8
 
-FROM ${BASE} AS yum
+FROM ${BASE} AS stream
+RUN dnf -y --disablerepo '*' --enablerepo=extras swap centos-linux-repos centos-stream-repos && \
+    dnf -y distro-sync
+
+FROM stream AS yum
 RUN yum install -y \
 	which \
 	bzip2 \
@@ -40,7 +44,12 @@ RUN cd /tmp && \
     rm -rf valgrind-3.15.0
 
 FROM valgrind AS adduser
-RUN useradd --shell /bin/bash libgit2 --create-home
+ARG UID=""
+ARG GID=""
+RUN if [ "${UID}" != "" ]; then USER_ARG="--uid ${UID}"; fi && \
+    if [ "${GID}" != "" ]; then GROUP_ARG="--gid ${GID}"; fi && \
+    groupadd ${GROUP_ARG} libgit2 && \
+    useradd ${USER_ARG} --gid libgit2 --shell /bin/bash --create-home libgit2
 
 FROM adduser AS configure
 ENV PKG_CONFIG_PATH /usr/local/lib/pkgconfig
diff --git a/ci/docker/focal b/ci/docker/focal
index 37d7d6356..8061237d7 100644
--- a/ci/docker/focal
+++ b/ci/docker/focal
@@ -73,7 +73,13 @@ RUN cd /tmp && \
     rm -rf valgrind-3.15.0
 
 FROM valgrind AS adduser
-RUN useradd --shell /bin/bash libgit2 --create-home
+ARG UID=""
+ARG GID=""
+RUN if [ "${UID}" != "" ]; then USER_ARG="--uid ${UID}"; fi && \
+    if [ "${GID}" != "" ]; then GROUP_ARG="--gid ${GID}"; fi && \
+    groupadd ${GROUP_ARG} libgit2 && \
+    useradd ${USER_ARG} --gid libgit2 --shell /bin/bash --create-home libgit2
+
 
 FROM adduser AS configure
 RUN mkdir /var/run/sshd
diff --git a/ci/docker/xenial b/ci/docker/xenial
index c19fe421d..c9791462f 100644
--- a/ci/docker/xenial
+++ b/ci/docker/xenial
@@ -60,7 +60,13 @@ RUN cd /tmp && \
     rm -rf valgrind-3.15.0
 
 FROM valgrind AS adduser
-RUN useradd --shell /bin/bash libgit2 --create-home
+ARG UID=""
+ARG GID=""
+RUN if [ "${UID}" != "" ]; then USER_ARG="--uid ${UID}"; fi && \
+    if [ "${GID}" != "" ]; then GROUP_ARG="--gid ${GID}"; fi && \
+    groupadd ${GROUP_ARG} libgit2 && \
+    useradd ${USER_ARG} --gid libgit2 --shell /bin/bash --create-home libgit2
+
 
 FROM adduser AS configure
 RUN mkdir /var/run/sshd
diff --git a/ci/getcontainer.sh b/ci/getcontainer.sh
index 07ef7b8ea..81d0c1d92 100755
--- a/ci/getcontainer.sh
+++ b/ci/getcontainer.sh
@@ -37,9 +37,13 @@ DOCKER_REGISTRY_CONTAINER_SHA="${DOCKER_REGISTRY_CONTAINER}:${DOCKER_SHA}"
 echo "docker-registry-container-sha=${DOCKER_REGISTRY_CONTAINER_SHA}" >> $GITHUB_ENV
 echo "docker-registry-container-latest=${DOCKER_REGISTRY_CONTAINER}:latest" >> $GITHUB_ENV
 
+echo "::: logging in to ${DOCKER_REGISTRY} as ${GITHUB_ACTOR}"
+
 exists="true"
 docker login https://${DOCKER_REGISTRY} -u ${GITHUB_ACTOR} -p ${GITHUB_TOKEN} || exists="false"
 
+echo "::: pulling ${DOCKER_REGISTRY_CONTAINER_SHA}"
+
 if [ "${exists}" != "false" ]; then
 	docker pull ${DOCKER_REGISTRY_CONTAINER_SHA} || exists="false"
 fi
diff --git a/ci/setup-mingw-build.sh b/ci/setup-mingw-build.sh
new file mode 100755
index 000000000..3d72b24eb
--- /dev/null
+++ b/ci/setup-mingw-build.sh
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+set -ex
+
+echo "##############################################################################"
+echo "## Downloading mingw"
+echo "##############################################################################"
+
+BUILD_TEMP=${BUILD_TEMP:=$TEMP}
+BUILD_TEMP=$(cygpath $BUILD_TEMP)
+
+case "$ARCH" in
+	amd64)
+		MINGW_URI="https://github.com/libgit2/ci-dependencies/releases/download/2021-05-04/mingw-x86_64-8.1.0-release-win32-sjlj-rt_v6-rev0.zip";;
+	x86)
+		MINGW_URI="https://github.com/libgit2/ci-dependencies/releases/download/2021-05-04/mingw-i686-8.1.0-release-win32-sjlj-rt_v6-rev0.zip";;
+esac
+
+if [ -z "$MINGW_URI" ]; then
+	echo "No URL"
+	exit 1
+fi
+
+mkdir -p "$BUILD_TEMP"
+
+curl -s -L "$MINGW_URI" -o "$BUILD_TEMP"/mingw-"$ARCH".zip
+unzip -q "$BUILD_TEMP"/mingw-"$ARCH".zip -d "$BUILD_TEMP"
diff --git a/ci/setup-osx-benchmark.sh b/ci/setup-osx-benchmark.sh
new file mode 100755
index 000000000..80d87682b
--- /dev/null
+++ b/ci/setup-osx-benchmark.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+set -ex
+
+brew update
+brew install hyperfine
diff --git a/ci/setup-osx-build.sh b/ci/setup-osx-build.sh
new file mode 100755
index 000000000..0b95e7629
--- /dev/null
+++ b/ci/setup-osx-build.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -ex
+
+brew update
+brew install pkgconfig zlib curl openssl libssh2 ninja
+
+ln -s /Applications/Xcode.app/Contents/Developer/usr/lib/libLeaksAtExit.dylib /usr/local/lib
diff --git a/ci/setup-ubuntu-benchmark.sh b/ci/setup-ubuntu-benchmark.sh
new file mode 100755
index 000000000..561a18fd9
--- /dev/null
+++ b/ci/setup-ubuntu-benchmark.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -ex
+
+sudo apt-get update
+sudo apt-get install -y --no-install-recommends \
+	cargo \
+	cmake \
+	gcc \
+	git \
+	krb5-user \
+	libkrb5-dev \
+	libssl-dev \
+	libz-dev \
+	make \
+	ninja-build \
+	pkgconf
+
+wget https://github.com/sharkdp/hyperfine/releases/download/v1.12.0/hyperfine_1.12.0_amd64.deb
+sudo dpkg -i hyperfine_1.12.0_amd64.deb
diff --git a/ci/setup-win32-benchmark.sh b/ci/setup-win32-benchmark.sh
new file mode 100755
index 000000000..0eac2f666
--- /dev/null
+++ b/ci/setup-win32-benchmark.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -ex
+
+choco install hyperfine zip
+
+CHOCO_PATH=$(mktemp -d)
+curl -L https://github.com/ethomson/PurgeStandbyList/releases/download/v1.0/purgestandbylist.1.0.0.nupkg -o "${CHOCO_PATH}/purgestandbylist.1.0.0.nupkg"
+choco install purgestandbylist -s $(cygpath -w "${CHOCO_PATH}")
diff --git a/ci/test.sh b/ci/test.sh
index a94839778..0815522a9 100755
--- a/ci/test.sh
+++ b/ci/test.sh
@@ -159,10 +159,18 @@ fi
 if [ -z "$SKIP_OFFLINE_TESTS" ]; then
 	echo ""
 	echo "##############################################################################"
-	echo "## Running (offline) tests"
+	echo "## Running core tests"
 	echo "##############################################################################"
 
+	echo ""
+	echo "Running libgit2 integration (offline) tests"
+	echo ""
 	run_test offline
+
+	echo ""
+	echo "Running utility tests"
+	echo ""
+	run_test util
 fi
 
 if [ -n "$RUN_INVASIVE_TESTS" ]; then
@@ -186,7 +194,7 @@ if [ -z "$SKIP_ONLINE_TESTS" ]; then
 
 	echo ""
 	echo "##############################################################################"
-	echo "## Running (online) tests"
+	echo "## Running networking (online) tests"
 	echo "##############################################################################"
 
 	export GITTEST_REMOTE_REDIRECT_INITIAL="http://localhost:9000/initial-redirect/libgit2/TestGitRepository"
@@ -198,9 +206,9 @@ if [ -z "$SKIP_ONLINE_TESTS" ]; then
 	# Run the online tests that immutably change global state separately
 	# to avoid polluting the test environment.
 	echo ""
-	echo "##############################################################################"
-	echo "## Running (online_customcert) tests"
-	echo "##############################################################################"
+	echo "Running custom certificate (online_customcert) tests"
+	echo ""
+
 	run_test online_customcert
 fi