From 7b66f9604ddb4743276f38b6a17f10bf4d446583 Mon Sep 17 00:00:00 2001 From: Emmanuele Bassi Date: Tue, 14 Apr 2020 11:48:55 +0100 Subject: Improve the Pango CI pipeline on Linux Use a scaled down version of the GTK and GLib CI set ups, with a custom Docker image and multiple stages. The CI pipeline now generates a JUnit report, which integrates with the GitLab merge requests; builds the documentation as a separate stage; and publishes the API reference for the master branch at the end. --- .gitlab-ci/README.md | 38 +++++++++++ .gitlab-ci/fedora.Dockerfile | 49 ++++++++++++++ .gitlab-ci/meson-junit-report.py | 112 ++++++++++++++++++++++++++++++++ .gitlab-ci/run-docker.sh | 135 +++++++++++++++++++++++++++++++++++++++ .gitlab-ci/run-tests.sh | 29 +++++++++ 5 files changed, 363 insertions(+) create mode 100644 .gitlab-ci/README.md create mode 100644 .gitlab-ci/fedora.Dockerfile create mode 100755 .gitlab-ci/meson-junit-report.py create mode 100755 .gitlab-ci/run-docker.sh create mode 100755 .gitlab-ci/run-tests.sh (limited to '.gitlab-ci') diff --git a/.gitlab-ci/README.md b/.gitlab-ci/README.md new file mode 100644 index 00000000..a5c7736d --- /dev/null +++ b/.gitlab-ci/README.md @@ -0,0 +1,38 @@ +## Pango CI infrastructure + +Pango uses different CI images depending on platform and jobs. + +The CI images are Docker containers, generated either using `docker` or +`podman`, and pushed to the GitLab [container registry][registry]. + +Each Docker image has a tag composed of two parts: + + - `${image}`: the base image for a given platform, like "fedora" or + "debian-stable" + - `${number}`: an incremental version number, or `latest` + +See the [container registry][registry] for the available images for each +branch, as well as their available versions. + +### Checklist for Updating a CI image + + - [ ] Update the `${image}.Dockerfile` file with the dependencies + - [ ] Run `./run-docker.sh build --base ${image} --base-version ${number}` + - [ ] Run `./run-docker.sh push --base ${image} --base-version ${number}` + once the Docker image is built; you may need to log in by using + `docker login` or `podman login` + - [ ] Update the `image` keys in the `.gitlab-ci.yml` file with the new + image tag + - [ ] Open a merge request with your changes and let it run + +### Checklist for Adding a new CI image + + - [ ] Write a new `${image}.Dockerfile` with the instructions to set up + a build environment + - [ ] Add the `pip3 install meson` incantation + - [ ] Run `./run-docker.sh build --base ${image} --base-version ${number}` + - [ ] Run `./run-docker.sh push --base ${image} --base-version ${number}` + - [ ] Add the new job to `.gitlab-ci.yml` referencing the image + - [ ] Open a merge request with your changes and let it run + +[registry]: https://gitlab.gnome.org/GNOME/pango/container_registry diff --git a/.gitlab-ci/fedora.Dockerfile b/.gitlab-ci/fedora.Dockerfile new file mode 100644 index 00000000..c1aadc93 --- /dev/null +++ b/.gitlab-ci/fedora.Dockerfile @@ -0,0 +1,49 @@ +FROM fedora:31 + +RUN dnf -y install \ + abattis-cantarell-fonts \ + cairo-devel \ + cairo-gobject-devel \ + ccache \ + clang \ + clang-analyzer \ + desktop-file-utils \ + diffutils \ + fribidi-devel \ + gcc \ + gcc-c++ \ + gettext \ + git \ + glib2-devel \ + glib2-static \ + glibc-devel \ + glibc-headers \ + glibc-langpack-en \ + gobject-introspection-devel \ + google-droid-sans-fonts \ + gtk-doc \ + harfbuzz-devel \ + hicolor-icon-theme \ + itstool \ + lcov \ + libthai-devel \ + ninja-build \ + python3 \ + python3-jinja2 \ + python3-pip \ + python3-pygments \ + python3-wheel \ + redhat-rpm-config \ + thai-scalable-waree-fonts \ + && dnf clean all + +RUN pip3 install meson==0.53.1 + +ARG HOST_USER_ID=5555 +ENV HOST_USER_ID ${HOST_USER_ID} +RUN useradd -u $HOST_USER_ID -ms /bin/bash user + +USER user +WORKDIR /home/user + +ENV LANG C.UTF-8 diff --git a/.gitlab-ci/meson-junit-report.py b/.gitlab-ci/meson-junit-report.py new file mode 100755 index 00000000..fc96efba --- /dev/null +++ b/.gitlab-ci/meson-junit-report.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 + +# Turns a Meson testlog.json file into a JUnit XML report +# +# Copyright 2019 GNOME Foundation +# +# SPDX-License-Identifier: LGPL-2.1-or-later +# +# Original author: Emmanuele Bassi + +import argparse +import datetime +import json +import os +import sys +import xml.etree.ElementTree as ET + +aparser = argparse.ArgumentParser(description='Turns a Meson test log into a JUnit report') +aparser.add_argument('--project-name', metavar='NAME', + help='The project name', + default='unknown') +aparser.add_argument('--job-id', metavar='ID', + help='The job ID for the report', + default='Unknown') +aparser.add_argument('--branch', metavar='NAME', + help='Branch of the project being tested', + default='master') +aparser.add_argument('--output', metavar='FILE', + help='The output file, stdout by default', + type=argparse.FileType('w', encoding='UTF-8'), + default=sys.stdout) +aparser.add_argument('infile', metavar='FILE', + help='The input testlog.json, stdin by default', + type=argparse.FileType('r', encoding='UTF-8'), + default=sys.stdin) + +args = aparser.parse_args() + +outfile = args.output + +testsuites = ET.Element('testsuites') +testsuites.set('id', '{}/{}'.format(args.job_id, args.branch)) +testsuites.set('package', args.project_name) +testsuites.set('timestamp', datetime.datetime.utcnow().isoformat(timespec='minutes')) + +suites = {} +for line in args.infile: + data = json.loads(line) + unit_name = data['name'] + suite_name = args.project_name + project_name = args.project_name + + duration = data['duration'] + return_code = data['returncode'] + result = data['result'] + log = data['stdout'] + + unit = { + 'suite': suite_name, + 'name': unit_name, + 'duration': duration, + 'returncode': return_code, + 'result': result, + 'stdout': log, + } + + units = suites.setdefault(suite_name, []) + units.append(unit) + +for name, units in suites.items(): + print('Processing suite {} (units: {})'.format(name, len(units))) + + def if_failed(unit): + if unit['result'] in ['FAIL', 'UNEXPECTEDPASS', 'TIMEOUT']: + return True + return False + + def if_succeded(unit): + if unit['result'] in ['OK', 'EXPECTEDFAIL', 'SKIP']: + return True + return False + + successes = list(filter(if_succeded, units)) + failures = list(filter(if_failed, units)) + print(' - {}: {} pass, {} fail'.format(name, len(successes), len(failures))) + + testsuite = ET.SubElement(testsuites, 'testsuite') + testsuite.set('name', '{}/{}'.format(args.project_name, name)) + testsuite.set('tests', str(len(units))) + testsuite.set('errors', str(len(failures))) + testsuite.set('failures', str(len(failures))) + + for unit in successes: + testcase = ET.SubElement(testsuite, 'testcase') + testcase.set('classname', '{}/{}'.format(args.project_name, unit['suite'])) + testcase.set('name', unit['name']) + testcase.set('time', str(unit['duration'])) + + for unit in failures: + testcase = ET.SubElement(testsuite, 'testcase') + testcase.set('classname', '{}/{}'.format(args.project_name, unit['suite'])) + testcase.set('name', unit['name']) + testcase.set('time', str(unit['duration'])) + + failure = ET.SubElement(testcase, 'failure') + failure.set('classname', '{}/{}'.format(args.project_name, unit['suite'])) + failure.set('name', unit['name']) + failure.set('type', 'error') + failure.text = unit['stdout'] + +output = ET.tostring(testsuites, encoding='unicode') +outfile.write(output) diff --git a/.gitlab-ci/run-docker.sh b/.gitlab-ci/run-docker.sh new file mode 100755 index 00000000..06c4f63d --- /dev/null +++ b/.gitlab-ci/run-docker.sh @@ -0,0 +1,135 @@ +#!/bin/bash + +read_arg() { + # $1 = arg name + # $2 = arg value + # $3 = arg parameter + local rematch='^[^=]*=(.*)$' + if [[ $2 =~ $rematch ]]; then + read "$1" <<< "${BASH_REMATCH[1]}" + else + read "$1" <<< "$3" + # There is no way to shift our callers args, so + # return 1 to indicate they should do it instead. + return 1 + fi +} + +set -e + +build=0 +run=0 +push=0 +list=0 +print_help=0 +no_login=0 + +while (($# > 0)); do + case "${1%%=*}" in + build) build=1;; + run) run=1;; + push) push=1;; + list) list=1;; + help) print_help=1;; + --base|-b) read_arg base "$@" || shift;; + --base-version) read_arg base_version "$@" || shift;; + --no-login) no_login=1;; + *) echo -e "\e[1;31mERROR\e[0m: Unknown option '$1'"; exit 1;; + esac + shift +done + +if [ $print_help == 1 ]; then + echo "$0 - Build and run Docker images" + echo "" + echo "Usage: $0 [options] [basename]" + echo "" + echo "Available commands" + echo "" + echo " build --base= - Build Docker image .Dockerfile" + echo " run --base= - Run Docker image " + echo " push --base= - Push Docker image to the registry" + echo " list - List available images" + echo " help - This help message" + echo "" + exit 0 +fi + +cd "$(dirname "$0")" + +if [ $list == 1 ]; then + echo "Available Docker images:" + for f in *.Dockerfile; do + filename=$( basename -- "$f" ) + basename="${filename%.*}" + + echo -e " \e[1;39m$basename\e[0m" + done + exit 0 +fi + +# All commands after this require --base to be set +if [ -z $base ]; then + echo "Usage: $0 " + exit 1 +fi + +if [ ! -f "$base.Dockerfile" ]; then + echo -e "\e[1;31mERROR\e[0m: Dockerfile for '$base' not found" + exit 1 +fi + +if [ -z $base_version ]; then + base_version="latest" +else + base_version="v$base_version" +fi + +if [ ! -x "$(command -v docker)" ] || [ docker --help |& grep -q podman ]; then + # Docker is actually implemented by podman, and its OCI output + # is incompatible with some of the dockerd instances on GitLab + # CI runners. + echo "Using: Podman" + format="--format docker" + CMD="podman" +else + echo "Using: Docker" + format="" + CMD="sudo socker" +fi + +REGISTRY="registry.gitlab.gnome.org" +TAG="${REGISTRY}/gnome/pango/${base}:${base_version}" + +if [ $build == 1 ]; then + echo -e "\e[1;32mBUILDING\e[0m: ${base} as ${TAG}" + ${CMD} build \ + ${format} \ + --build-arg HOST_USER_ID="$UID" \ + --tag "${TAG}" \ + --file "${base}.Dockerfile" . + exit $? +fi + +if [ $push == 1 ]; then + echo -e "\e[1;32mPUSHING\e[0m: ${base} as ${TAG}" + + if [ $no_login == 0 ]; then + ${CMD} login ${REGISTRY} + fi + + ${CMD} push ${TAG} + exit $? +fi + +if [ $run == 1 ]; then + echo -e "\e[1;32mRUNNING\e[0m: ${base} as ${TAG}" + ${CMD} run \ + --rm \ + --volume "$(pwd)/..:/home/user/app" \ + --workdir "/home/user/app" \ + --tty \ + --interactive "${TAG}" \ + bash + exit $? +fi diff --git a/.gitlab-ci/run-tests.sh b/.gitlab-ci/run-tests.sh new file mode 100755 index 00000000..cca589b4 --- /dev/null +++ b/.gitlab-ci/run-tests.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set +x +set +e + +srcdir=$( pwd ) +builddir=$1 + +meson test -C ${builddir} \ + --print-errorlogs \ + --suite=pango + +# Store the exit code for the CI run, but always +# generate the reports +exit_code=$? + +cd ${builddir} + +./utils/pango-list --verbose > fontlist.txt +./tests/test-font -p /pango/font/metrics --verbose +./utils/pango-view --no-display --output hello.png ${srcdir}/utils/HELLO.txt + +$srcdir/.gitlab-ci/meson-junit-report.py \ + --project-name=pango \ + --job-id="${CI_JOB_NAME}" \ + --output=report.xml \ + meson-logs/testlog.json + +exit $exit_code -- cgit v1.2.1