CI: Special case porting test in 'sanity check'

Add a special case for the porting test in the sanity check of the
CI run.

Before:
    When a porting test failed (such as t/porting/authors.t,
    t/porting/cmp_version.t, ...) then it would skip all the other
    test configurations.

Now:
    When a porting test fails:
    - result of sanity check is failure (i.e. it's red)
    - other test configurations are run and these will also run the
      porting tests. (These may or may not fail, depending on the test)

    When a non-porting test fails:
    - result of sanity check is failure (i.e. it's red)
    - sanity check still runs the porting tests
    - other test configurations are skipped

Note: in the `if`conditions of the other test configurations it was
      needed to include `always()` because otherwise GitHub uses an
      implicit `success() &&` (which check the result of the
      sanity check - which is unwanted if only the porting tests failed)

Fixes #19867

1 files changed, 18 insertions, 12 deletions

diff --git a/.github/workflows/testsuite.yml b/.github/workflows/testsuite.yml
index 5af908d9cf..3e0970019e 100644
--- a/.github/workflows/testsuite.yml
+++ b/.github/workflows/testsuite.yml
@@ -102,15 +102,16 @@ jobs:
         env:
           CONFIGURE_ARGS: "-Dusethreads"
       - name: Build
+        id: build
         run: |
           make -j2 test_prep
       - name: Show Config
         run: |
           ./perl -Ilib -V
           ./perl -Ilib -e 'use Config; print Config::config_sh'
-      - name: Run Tests
+      - name: Run Tests (excluding t/porting)
         run: |
-          TEST_JOBS=2 ./perl t/harness
+          TEST_JOBS=2 ./perl t/harness -nre='^porting/'
 
       # Set a variable for dependent jobs to signal if full testsuite is enabled
       - name: "Check if EXTENDED_TESTING is set"
@@ -127,6 +128,11 @@ jobs:
             echo "::set-output name=run_all_jobs::true"
           fi
 
+      - name: Run Porting Tests (t/porting)
+        if: always() && steps.build.outcome == 'success'
+        run: |
+          TEST_JOBS=2 ./perl t/harness -re='^porting/'
+
   #  _ _
   # | (_)_ _ _  ___ __
   # | | | ' \ || \ \ /
@@ -136,7 +142,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 120
     needs: sanity_check
-    if: needs.sanity_check.outputs.run_all_jobs == 'true'
+    if: always() && needs.sanity_check.outputs.run_all_jobs == 'true'
 
     strategy:
       fail-fast: false
@@ -193,7 +199,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 120
     needs: sanity_check
-    if: needs.sanity_check.outputs.run_all_jobs == 'true'
+    if: always() && needs.sanity_check.outputs.run_all_jobs == 'true'
 
     # https://hub.docker.com/r/i386/ubuntu/
     container:
@@ -247,7 +253,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 120
     needs: sanity_check
-    if: needs.sanity_check.outputs.run_all_jobs == 'true'
+    if: always() && needs.sanity_check.outputs.run_all_jobs == 'true'
 
     steps:
       - name: Install System dependencies
@@ -297,7 +303,7 @@ jobs:
     runs-on: macos-10.15
     timeout-minutes: 120
     needs: sanity_check
-    if: needs.sanity_check.outputs.run_all_jobs == 'true'
+    if: always() && needs.sanity_check.outputs.run_all_jobs == 'true'
 
     strategy:
       fail-fast: false
@@ -333,7 +339,7 @@ jobs:
     runs-on: windows-2019
     timeout-minutes: 120
     needs: sanity_check
-    if: needs.sanity_check.outputs.run_all_jobs == 'true'
+    if: always() && needs.sanity_check.outputs.run_all_jobs == 'true'
 
     steps:
       - run: git config --global core.autocrlf false
@@ -389,7 +395,7 @@ jobs:
     runs-on: windows-2019
     timeout-minutes: 120
     needs: sanity_check
-    if: needs.sanity_check.outputs.run_all_jobs == 'true'
+    if: always() && needs.sanity_check.outputs.run_all_jobs == 'true'
 
     steps:
       - run: git config --global core.autocrlf false
@@ -439,7 +445,7 @@ jobs:
     runs-on: windows-2019
     timeout-minutes: 120
     needs: sanity_check
-    if: needs.sanity_check.outputs.run_all_jobs == 'true'
+    if: always() && needs.sanity_check.outputs.run_all_jobs == 'true'
 
     steps:
       # we use Cygwin git, so no need to configure git here.
@@ -502,7 +508,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 120
     needs: sanity_check
-    if: needs.sanity_check.outputs.run_all_jobs == 'true'
+    if: always() && needs.sanity_check.outputs.run_all_jobs == 'true'
 
     outputs:
       run_all_jobs: ${{ steps.check_extended_testing.outputs.run_all_jobs }}
@@ -551,7 +557,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 120
     needs: sanity_check
-    if: needs.sanity_check.outputs.run_all_jobs == 'true'
+    if: always() && needs.sanity_check.outputs.run_all_jobs == 'true'
 
     strategy:
       matrix:
@@ -602,7 +608,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 120
     needs: sanity_check
-    if: needs.sanity_check.outputs.run_all_jobs == 'true'
+    if: always() && needs.sanity_check.outputs.run_all_jobs == 'true'
 
     strategy:
       matrix: