diff options
Diffstat (limited to '.gitlab-ci.yml')
| -rw-r--r-- | .gitlab-ci.yml | 242 |
1 files changed, 137 insertions, 105 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a6e7ad4d5a..b7025dc99b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,90 +1,121 @@ -# Copyright 2021 The Chromium OS Authors. All rights reserved. +# Copyright 2021 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -image: sjg20/ubuntu-25feb22c +image: jbettis/ubuntu-07sep22 # You can update that image using this repo: # https://gitlab.com/zephyr-ec/gitlab-ci-runner/-/tree/main -# Change pip's cache directory to be inside the project directory since we can -# only cache local items. -variables: - PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" - -# Pip's cache doesn't store the python packages -# https://pip.pypa.io/en/stable/reference/pip_install/#caching -# -# If you want to also cache the installed packages, you have to install -# them in a virtualenv and cache it as well. -cache: - key: ${CI_COMMIT_SHA} - policy: pull - paths: - - .cache/pip - - venv/ - - modules/ - - zephyr/ - +# Only run on main and coverage branches +workflow: + rules: + - if: ($CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH == "coverage") # The directory structure is: # -# /zephyr -# /builds/zephyr-ec/ec EC_DIR +# ${CI_PROJECT_DIR} +# |_ ec_dir/ - EC repo (${EC_DIR}) +# |_ build/ - make and zmake artifacts (${BUILD_DIR}) +# |_ twister-out/ - Twister artifacts (${TWISTER_OUT_DIR}) +# |_ modules/ - Additional Zephyr modules (${MODULES_DIR}) +# | |_ cmsis/ +# | |_ hal_stm32/ +# | |_ nanopb/ +# | |_ cryptoc/ +# |_ zephyr/ +# |_ main/ - Zephyr repo (${ZEPHYR_BASE}) +# +# On Gitlab builds, ${CI_PROJECT_DIR} is at /builds/zephyr-ec/ec. On local +# builds, it will be /builds/0/project-0. + before_script: + - echo "CI_PROJECT_DIR is at ${CI_PROJECT_DIR}" + # Move freshly-checked out ec code to subdirectory + - cd ${CI_PROJECT_DIR} + - mkdir -p ec_dir + - find . -mindepth 1 -maxdepth 1 -name ec_dir -o -print0 | xargs -0 -I {} mv {} ec_dir + - export EC_DIR="${CI_PROJECT_DIR}/ec_dir" - export MODULES_DIR="$CI_PROJECT_DIR/modules" - - mkdir -p "${MODULES_DIR}" - export ZEPHYR_BASE="${CI_PROJECT_DIR}/zephyr/main" - - mkdir -p "${ZEPHYR_BASE}" - - test -d "${ZEPHYR_BASE}/.git" || git clone --depth 1 -b main https://chromium.googlesource.com/chromiumos/third_party/zephyr "${ZEPHYR_BASE}" - - test -d "${MODULES_DIR}/cmsis" || git clone --depth 1 -b chromeos-main https://chromium.googlesource.com/chromiumos/third_party/zephyr/cmsis "${MODULES_DIR}/cmsis" - - test -d "${MODULES_DIR}/hal_stm32" || git clone --depth 1 -b chromeos-main https://chromium.googlesource.com/chromiumos/third_party/zephyr/hal_stm32 "${MODULES_DIR}/hal_stm32" - - test -d "${MODULES_DIR}/nanopb" || git clone --depth 1 -b main https://chromium.googlesource.com/chromiumos/third_party/zephyr/nanopb "${MODULES_DIR}/nanopb" - - test -d "${MODULES_DIR}/cryptoc" || git clone --depth 1 -b main https://chromium.googlesource.com/chromiumos/third_party/cryptoc "${MODULES_DIR}/cryptoc" - - ln -s "$(pwd)" "${MODULES_DIR}/ec" + - git config --global --add safe.directory '*' + # Get Zephyr repo and modules + - checkout_at_date() { + local url="$1" ; + local branch="$2" ; + local path="$3" ; + local date="$4" ; + local retries=5 ; + + while [ ${retries} -gt 0 ] ; do + rm -rf "${path}" ; + mkdir -p "${path}" ; + if git clone --depth 1 -b "${branch}" "${url}" "${path}"; then + break ; + else + echo "git clone failed will retry ${retries} times"; + fi ; + retries="$(( ${retries} - 1 ))" ; + done ; + cd "${path}" ; + rev="$(git rev-list -n 1 --before="${date}" "${branch}")" ; + depth=1 ; + while [ -z "${rev}" ] ; do + depth="$(( ${depth} + 100 ))" ; + git fetch --depth="${depth}" ; + rev="$(git rev-list -n 1 --before="${date}" "${branch}")" ; + done ; + git checkout -d "${rev}" ; + } + - ec_commit_date="$(cd ${EC_DIR} ; git log -1 HEAD --format=%cd --date=iso)" + - checkout_at_date "https://chromium.googlesource.com/chromiumos/third_party/zephyr" "main" "${ZEPHYR_BASE}" "${ec_commit_date}" + - checkout_at_date "https://chromium.googlesource.com/chromiumos/third_party/zephyr/cmsis" "chromeos-main" "${MODULES_DIR}/cmsis" "${ec_commit_date}" + - checkout_at_date "https://chromium.googlesource.com/chromiumos/third_party/zephyr/hal_stm32" "chromeos-main" "${MODULES_DIR}/hal_stm32" "${ec_commit_date}" + - checkout_at_date "https://chromium.googlesource.com/chromiumos/third_party/zephyr/nanopb" "main" "${MODULES_DIR}/nanopb" "${ec_commit_date}" + - checkout_at_date "https://chromium.googlesource.com/chromiumos/third_party/cryptoc" "main" "${MODULES_DIR}/cryptoc" "${ec_commit_date}" + # Add a symlink so the ec appears in modules directory (hack to make zmake work) + - ln -s "${EC_DIR}" "${MODULES_DIR}/ec" + # Install Python and packages - python3 -V # Print out python version for debugging - - python3 -m pip install 'zephyr/zmake[tests]' --user - - python3 -m pip install pyyaml packaging - - export BUILD_DIR=build + - python3 -m pip install "${EC_DIR}/zephyr/zmake[tests]" --user + - python3 -m pip install pyyaml packaging ply psutil 'pyelftools>=0.28' - export PATH="$PATH:$HOME/.local/bin" - export PYTHONIOENCODING=utf-8 - - export EC_DIR=/builds/zephyr-ec/ec - -seed_cache: - stage: build - needs: [] - cache: - key: ${CI_COMMIT_SHA} - paths: - - .cache/pip - - venv/ - - modules/ - - zephyr/ - policy: push - script: - - ls "${MODULES_DIR}" "${ZEPHYR_BASE}" + # Build directory + - export BUILD_DIR=${EC_DIR}/build + - export TWISTER_OUT_DIR=${EC_DIR}/twister-out + # Set up Twister to use the "host" toolchain, as defined in upstream Zephyr + # repo, ultimately including ${ZEPHYR_BASE}/cmake/toolchain/host/generic.cmake + - export TOOLCHAIN_ROOT=${ZEPHYR_BASE} + - export ZEPHYR_TOOLCHAIN_VARIANT=host # Users of this template must set: # $PROJECT to the project to build. E.g., "lazor" .build_template: &build_template stage: build - needs: ["seed_cache"] + needs: [] script: - zmake --zephyr-base "${ZEPHYR_BASE}" --modules-dir "${MODULES_DIR}" -l DEBUG build -B "${BUILD_DIR}" -t ${TOOLCHAIN:-zephyr} "${PROJECT}" - for b in "${BUILD_DIR}/${PROJECT}"/build-* ; do - bdir=$(basename ${b}) ; - ninja -C ${b} ram_report >"${BUILD_DIR}/${PROJECT}/output/${bdir}_ram_report.txt" ; - cp ${b}/ram.json "${BUILD_DIR}/${PROJECT}/output/${bdir}_ram.json" ; - ninja -C ${b} rom_report >"${BUILD_DIR}/${PROJECT}/output/${bdir}_rom_report.txt" ; - cp ${b}/rom.json "${BUILD_DIR}/${PROJECT}/output/${bdir}_rom.json" ; + bdir=$(basename ${b}) ; + if ninja -C ${b} ram_report >"${BUILD_DIR}/${PROJECT}/output/${bdir}_ram_report.txt" ; then + cp ${b}/ram.json "${BUILD_DIR}/${PROJECT}/output/${bdir}_ram.json" ; + else + echo "Ram report failed" ; + fi ; + if ninja -C ${b} rom_report >"${BUILD_DIR}/${PROJECT}/output/${bdir}_rom_report.txt" ; then + cp ${b}/rom.json "${BUILD_DIR}/${PROJECT}/output/${bdir}_rom.json" ; + else + echo "Rom report failed" ; + fi ; done - ls "${BUILD_DIR}/${PROJECT}" "${BUILD_DIR}/${PROJECT}/output" artifacts: paths: - - build/${PROJECT}/output/* + - ec_dir/build/${PROJECT}/output/* expire_in: 1 week # Users of this template must set: @@ -98,6 +129,7 @@ seed_cache: stage: test needs: ["merged_coverage", "zephyr_coverage", "twister_coverage"] script: + - cd ${EC_DIR} - zmake --zephyr-base "${ZEPHYR_BASE}" --modules-dir "${MODULES_DIR}" -l DEBUG build --coverage -B "${BUILD_DIR}" -t ${TOOLCHAIN:-zephyr} @@ -145,8 +177,8 @@ seed_cache: -s "${BUILD_DIR}/${PROJECT}/output/filtered_no_zephyr.info" artifacts: paths: - - build/${PROJECT}/output/*.info - - build/${PROJECT}/output/*_rpt + - ec_dir/build/${PROJECT}/output/*.info + - ec_dir/build/${PROJECT}/output/*_rpt expire_in: 1 week when: always coverage: '/lines\.*: \d+\.\d+%/' @@ -191,12 +223,6 @@ lazor: PROJECT: "lazor" <<: *build_template -native_posix: - variables: - PROJECT: "posix-ec" - TOOLCHAIN: "host" - <<: *build_template - npcx7_evb: variables: PROJECT: "npcx7" @@ -219,42 +245,48 @@ skyrim_coverage: ec_coverage: stage: test - needs: ["seed_cache"] + needs: [] script: - - make -j8 CRYPTOC_DIR="${MODULES_DIR}/cryptoc" - HOSTGCOV='gcov' + - cd ${EC_DIR} + - make -j$(nproc) CRYPTOC_DIR="${MODULES_DIR}/cryptoc" CROSS_COMPILE_arm=/opt/zephyr-sdk/arm-zephyr-eabi/bin/arm-zephyr-eabi- + CROSS_COMPILE_host=/usr/bin/ test-coverage artifacts: paths: - - build/coverage/coverage_rpt/* - - build/coverage/lcov.info + - ec_dir/build/coverage/coverage_rpt/* + - ec_dir/build/coverage/lcov.info expire_in: 1 week coverage: '/lines\.*: \d+\.\d+%/' zephyr_coverage: stage: test - needs: ["seed_cache"] + needs: [] script: - - zmake --zephyr-base "${ZEPHYR_BASE}" - --modules-dir "${MODULES_DIR}" -l DEBUG test - --coverage --host-tests-only + - python3 ${EC_DIR}/twister --coverage --outdir "${TWISTER_OUT_DIR}" + -v -i --gcov-tool gcov -x=ALLOW_WARNINGS=ON artifacts: paths: - - build/zephyr/all_tests.info + - ec_dir/twister-out/coverage.info expire_in: 1 week coverage: '/lines\.*: \d+\.\d+%/' zephyr_boards_coverage: stage: build - needs: ["seed_cache"] + needs: [] + parallel: 5 script: + - cd ${EC_DIR} + - projects=( $(zmake --zephyr-base "${ZEPHYR_BASE}" + --modules-dir "${MODULES_DIR}" list-projects | + split -n r/"${CI_NODE_INDEX}"/"${CI_NODE_TOTAL}") ) - zmake --zephyr-base "${ZEPHYR_BASE}" --modules-dir "${MODULES_DIR}" -l DEBUG build - --coverage -a + --coverage --delete-intermediates "${projects[@]}" + - mv build/zephyr/all_builds.info "build/zephyr/all_builds${CI_NODE_INDEX}.info" artifacts: paths: - - build/zephyr/all_builds.info + - ec_dir/build/zephyr/all_builds*.info expire_in: 1 week coverage: '/lines\.*: \d+\.\d+%/' @@ -264,10 +296,10 @@ merged_coverage: stage: test needs: ["ec_coverage", "zephyr_coverage", "zephyr_boards_coverage"] script: - - lcov --rc lcov_branch_coverage=1 -o build/merged.info - -a build/coverage/lcov.info -a build/zephyr/all_tests.info - - lcov --rc lcov_branch_coverage=1 -o build/merged_no_zephyr.info - -r build/merged.info + - lcov --rc lcov_branch_coverage=1 -o ${BUILD_DIR}/merged.info + -a ${BUILD_DIR}/coverage/lcov.info -a "${TWISTER_OUT_DIR}/coverage.info" + - lcov --rc lcov_branch_coverage=1 -o ${BUILD_DIR}/merged_no_zephyr.info + -r ${BUILD_DIR}/merged.info "${ZEPHYR_BASE}/**" "${MODULES_DIR}/**" "${EC_DIR}/zephyr/drivers/**" "${EC_DIR}/zephyr/include/drivers/**" "${EC_DIR}/zephyr/shim/chip/**" "${EC_DIR}/zephyr/shim/core/**" @@ -276,10 +308,10 @@ merged_coverage: "${EC_DIR}/zephyr/shim/chip/npcx/npcx_monitor/**" "${EC_DIR}/zephyr/emul/**" "${EC_DIR}/zephyr/test/**" "**/testsuite/**" "**/subsys/emul/**" - - lcov --rc lcov_branch_coverage=1 -o "build/all_builds_merged.info" - -a "build/zephyr/all_builds.info" -a build/merged.info - - lcov --rc lcov_branch_coverage=1 -o "build/all_builds_no_zephyr.info" - -r "build/all_builds_merged.info" "${ZEPHYR_BASE}/**" + - lcov --rc lcov_branch_coverage=1 -o "${BUILD_DIR}/all_builds_merged.info" + -a ${BUILD_DIR}/merged.info $(printf ' -a %s' ${BUILD_DIR}/zephyr/all_builds[0-9]*.info) + - lcov --rc lcov_branch_coverage=1 -o "${BUILD_DIR}/all_builds_no_zephyr.info" + -r "${BUILD_DIR}/all_builds_merged.info" "${ZEPHYR_BASE}/**" "${ZEPHYR_BASE}/**" "${MODULES_DIR}/**" "${EC_DIR}/zephyr/drivers/**" "${EC_DIR}/zephyr/include/drivers/**" "${EC_DIR}/zephyr/shim/chip/**" "${EC_DIR}/zephyr/shim/core/**" @@ -288,66 +320,66 @@ merged_coverage: "${EC_DIR}/zephyr/shim/chip/npcx/npcx_monitor/**" "${EC_DIR}/zephyr/emul/**" "${EC_DIR}/zephyr/test/**" "**/testsuite/**" "**/subsys/emul/**" - - grep "SF:" "build/zephyr/all_builds.info" | sort -u | + - grep -h "SF:" ${BUILD_DIR}/zephyr/all_builds[0-9]*.info | sort -u | sed -e 's|^SF:||' | xargs lcov --rc lcov_branch_coverage=1 - -o "build/all_builds_filtered.info" - -e "build/all_builds_no_zephyr.info" + -o "${BUILD_DIR}/all_builds_filtered.info" + -e "${BUILD_DIR}/all_builds_no_zephyr.info" - /usr/bin/genhtml --branch-coverage -q - -o "build/all_builds_filtered_rpt" + -o "${BUILD_DIR}/all_builds_filtered_rpt" -t "All boards coverage w/o zephyr" -p ${EC_DIR} - -s "build/all_builds_filtered.info" + -s "${BUILD_DIR}/all_builds_filtered.info" artifacts: paths: - - build/*.info - - build/*_rpt + - ec_dir/build/*.info + - ec_dir/build/*_rpt expire_in: 1 week coverage: '/lines\.*: \d+\.\d+%/' testall: stage: test - needs: ["seed_cache"] + needs: [] script: - - zmake --zephyr-base "${ZEPHYR_BASE}" - --modules-dir "${MODULES_DIR}" -l DEBUG test --host-tests-only + - python3 ${EC_DIR}/twister --outdir "${TWISTER_OUT_DIR}" -v -i + -x=ALLOW_WARNINGS=ON twister_coverage: stage: test - needs: ["seed_cache"] + needs: [] script: - - mkdir -p build/zephyr_codecov + - mkdir -p ${BUILD_DIR}/zephyr_codecov - for commitid in $(cd "${ZEPHYR_BASE}" ; git fetch --depth=100 ; git log | awk '/GitOrigin-RevId:/ {print $2}') ; do echo "COMMITID = ${commitid?}" ; if wget -O /tmp/coverage.html "https://codecov.io/gh/zephyrproject-rtos/zephyr/commit/${commitid?}/build" ; then downloadurl=$(sed -e '/download\/build/!d' -e 's|^.*href="|https://codecov.io|' -e 's|".*$||' /tmp/coverage.html | head -1) ; echo "DOWNLOADURL = ${downloadurl?}" ; - wget -O build/zephyr_codecov/merged.info.raw "${downloadurl?}" ; - sed <build/zephyr_codecov/merged.info.raw >build/zephyr_codecov/merged.info + wget -O ${BUILD_DIR}/zephyr_codecov/merged.info.raw "${downloadurl?}" ; + sed <${BUILD_DIR}/zephyr_codecov/merged.info.raw >${BUILD_DIR}/zephyr_codecov/merged.info -e '1,/<<<<<< network/d' -e '/<<<<<< EOF/,$d' ; - sed <build/zephyr_codecov/merged.info >build/zephyr_codecov/fixed.info + sed <${BUILD_DIR}/zephyr_codecov/merged.info >${BUILD_DIR}/zephyr_codecov/fixed.info -e "s|/__w/zephyr/zephyr|${ZEPHYR_BASE}|" -e "s|/__w/zephyr/modules/hal/cmsis|${MODULES_DIR}/cmsis|" -e "s|/__w/zephyr/modules|${MODULES_DIR}|" ; break ; fi ; done - - lcov --rc lcov_branch_coverage=1 --summary build/zephyr_codecov/fixed.info + - lcov --rc lcov_branch_coverage=1 --summary ${BUILD_DIR}/zephyr_codecov/fixed.info artifacts: paths: - - build/zephyr_codecov/fixed.info + - ec_dir/build/zephyr_codecov/fixed.info expire_in: 1 week coverage: '/lines\.*: \d+\.\d+%/' zmake_coverage: stage: test - needs: ["seed_cache"] + needs: [] script: - - cd zephyr/zmake + - cd ${EC_DIR}/zephyr/zmake - coverage run --source=zmake -m pytest . - coverage report - coverage html artifacts: paths: - - zephyr/zmake/htmlcov/* + - ec_dir/zephyr/zmake/htmlcov/* expire_in: 1 week coverage: '/^TOTAL.+?(\d+\%)$/' |