Control Travis apt retries on our own

because Travis team does not do it for us:
https://github.com/travis-ci/travis-build/pull/1712

The retried part has failed often even in one day:
https://travis-ci.org/ruby/ruby/jobs/567802384
https://travis-ci.org/ruby/ruby/jobs/567802388
https://travis-ci.org/ruby/ruby/jobs/567695879
https://travis-ci.org/ruby/ruby/jobs/567666931

For doing it easily, this also changes major aliases to
compiler-specific ones, because partially updating `before_install`
logic using `env` key was too hard and we needed to directly write
`before_install` for each package set. As a bonus of it, it may also
skip installing unnecessary packages when just `os: linux` is needed.

I'll revert this if this patch does not contribute to stabilize CI.

1 files changed, 120 insertions, 94 deletions

diff --git a/.travis.yml b/.travis.yml
index 1a0753990d..189aca20ae 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -24,31 +24,6 @@ dist: xenial
 git:
   quiet: true
 
-addons:
-  apt:
-    config:
-      retries: true
-    update: true
-    sources:
-      - ubuntu-toolchain-r-test
-    packages:
-      - gcc-8
-      - libffi-dev
-      - libgdbm-dev
-      - libgmp-dev
-      - libjemalloc-dev
-      - libncurses5-dev
-      - libncursesw5-dev
-      - libreadline6-dev
-      - libssl-dev
-      - libyaml-dev
-      - openssl
-      - valgrind
-      - zlib1g-dev
-  # # Travis homebrew addon is unstable for now. Use `before_install` instead.
-  # # https://travis-ci.community/t/osx-homebrew-addons-module-not-as-reliable-as-claimed/4054
-  # homebrew:
-
 cache:
   ccache: true
   directories:
@@ -72,6 +47,8 @@ env:
     - RUBY_PREFIX=/tmp/ruby-prefix
     - GEMS_FOR_TEST='timezone tzinfo'
     - UPDATE_UNICODE="UNICODE_FILES=. UNICODE_PROPERTY_FILES=. UNICODE_AUXILIARY_FILES=. UNICODE_EMOJI_FILES=."
+    # https://github.com/travis-ci/travis-build/blob/e411371dda21430a60f61b8f3f57943d2fe4d344/lib/travis/build/bash/travis_apt_get_options.bash#L7
+    - travis_apt_get_options='--allow-downgrades --allow-remove-essential --allow-change-held-packages'
 
 .org.ruby-lang.ci.matrix-definitions:
 
@@ -82,12 +59,7 @@ env:
     script:
       - $SETARCH make -s test TESTOPTS="${TESTOPTS=$JOBS -q --tty=no}"
 
-  - &linux
-    os: linux
-    compiler: gcc-8
-
-  - &osx
-    os: osx
+  - &osx-clang
     compiler: clang
     before_install:
       - gem uninstall ntlm-http plist -x --force
@@ -96,6 +68,8 @@ env:
         timeout 300 brew update ||
         timeout 300 brew update ||
         timeout 300 brew update
+      # Travis homebrew addon is unstable for now. Use `before_install` instead.
+      # https://travis-ci.community/t/osx-homebrew-addons-module-not-as-reliable-as-claimed/4054
       - |-
         brew install \
           ccache \
@@ -106,47 +80,90 @@ env:
           zlib
       - /usr/local/opt/openssl@1.1/bin/openssl version
 
+  - &gcc-8
+    compiler: gcc-8
+    addons:
+      apt:
+        config:
+          retries: true
+        sources:
+          - ubuntu-toolchain-r-test
+        # `packages` are installed in `before_install` for retries.
+    before_install:
+      - |-
+        for seconds in 1 25 100 -1; do
+          travis_apt_get_update
+
+          sudo -E apt-get -yq --no-install-suggests --no-install-recommends $travis_apt_get_options install \
+            gcc-8 \
+            libffi-dev \
+            libgdbm-dev \
+            libgmp-dev \
+            libjemalloc-dev \
+            libncurses5-dev \
+            libncursesw5-dev \
+            libreadline6-dev \
+            libssl-dev \
+            libyaml-dev \
+            openssl \
+            valgrind \
+            zlib1g-dev
+
+          if [[ $? = 0 ]]; then break; else sleep "$seconds" || exit 1; fi
+        done
+
   - &clang-8
     compiler: clang-8
     addons:
       apt:
         config:
           retries: true
-        update: true
         sources:
           - llvm-toolchain-xenial-8
-        packages:
-          - clang-8
-          - llvm-8-tools
-          - libffi-dev
-          - libgdbm-dev
-          - libgmp-dev
-          - libjemalloc-dev
-          - libncurses5-dev
-          - libncursesw5-dev
-          - libreadline6-dev
-          - libssl-dev
-          - libyaml-dev
-          - openssl
-          - valgrind
-          - zlib1g-dev
+        # `packages` are installed in `before_install` for retries.
+    before_install:
+      - |-
+        for seconds in 1 25 100 -1; do
+          travis_apt_get_update
+
+          sudo -E apt-get -yq --no-install-suggests --no-install-recommends $travis_apt_get_options install \
+            clang-8 \
+            llvm-8-tools \
+            libffi-dev \
+            libgdbm-dev \
+            libgmp-dev \
+            libjemalloc-dev \
+            libncurses5-dev \
+            libncursesw5-dev \
+            libreadline6-dev \
+            libssl-dev \
+            libyaml-dev \
+            openssl \
+            valgrind \
+            zlib1g-dev
+
+          if [[ $? = 0 ]]; then break; else sleep "$seconds" || exit 1; fi
+        done
 
   # --------
 
   - &x86_64-linux
     name: x86_64-linux
-    <<: *linux
+    os: linux
+    <<: *gcc-8
 
   - &jemalloc
     name: --with-jemalloc
-    <<: *linux
+    os: linux
+    <<: *gcc-8
     <<: *cron-only
     env:
       - CONFIG_FLAG='--with-gmp --with-jemalloc --with-valgrind'
 
   - &assertions
     name: RUBY_DEBUG=1
-    <<: *linux
+    os: linux
+    <<: *gcc-8
     #<<: *cron-only
     <<: *make-test-only
     env:
@@ -155,7 +172,8 @@ env:
 
   - &VM_CHECK_MODE
     name: VM_CHECK_MODE=3
-    <<: *linux
+    os: linux
+    <<: *gcc-8
     <<: *cron-only
     <<: *make-test-only
     env:
@@ -164,21 +182,24 @@ env:
 
   - &WITH_COROUTINE_UCONTEXT
     name: COROUTINE=ucontext
-    <<: *linux
+    os: linux
+    <<: *gcc-8
     <<: *cron-only
     env:
       - CONFIG_FLAG='--with-coroutine=ucontext'
 
   - &WITH_COROUTINE_COPY
     name: COROUTINE=copy
-    <<: *linux
+    os: linux
+    <<: *gcc-8
     <<: *cron-only
     env:
       - CONFIG_FLAG='--with-coroutine=copy'
 
   - &TOKEN_THREADED_CODE
     name: TOKEN_THREADED_CODE
-    <<: *linux
+    os: linux
+    <<: *gcc-8
     <<: *cron-only
     <<: *make-test-only
     env:
@@ -187,7 +208,8 @@ env:
 
   - &CALL_THREADED_CODE
     name: CALL_THREADED_CODE
-    <<: *linux
+    os: linux
+    <<: *gcc-8
     <<: *cron-only
     <<: *make-test-only
     env:
@@ -196,7 +218,8 @@ env:
 
   - &NO_THREADED_CODE
     name: NO_THREADED_CODE
-    <<: *linux
+    os: linux
+    <<: *gcc-8
     <<: *cron-only
     <<: *make-test-only
     env:
@@ -205,10 +228,10 @@ env:
 
   - &ASAN
     name: -fsanitize=address
-    <<: *linux
+    os: linux
+    <<: *clang-8
     #<<: *cron-only
     <<: *make-test-only
-    <<: *clang-8
     env:
       - GEMS_FOR_TEST=
       - ASAN_OPTIONS=detect_leaks=0
@@ -221,10 +244,10 @@ env:
 
   - &MSAN
     name: -fsanitize=memory
-    <<: *linux
+    os: linux
+    <<: *clang-8
     #<<: *cron-only
     <<: *make-test-only
-    <<: *clang-8
     env:
       - GEMS_FOR_TEST=
       - cflags='-U_FORTIFY_SOURCE -fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -fPIC'
@@ -235,10 +258,10 @@ env:
 
   - &UBSAN
     name: -fsanitize=undefined
-    <<: *linux
+    os: linux
+    <<: *clang-8
     #<<: *cron-only
     <<: *make-test-only
-    <<: *clang-8
     env:
       - GEMS_FOR_TEST=
       - cflags='-U_FORTIFY_SOURCE -fsanitize=undefined,integer,nullability -fno-sanitize=implicit-integer-sign-change,unsigned-integer-overflow'
@@ -250,7 +273,8 @@ env:
 
   - &i686-linux
     name: i686-linux
-    <<: *linux
+    os: linux
+    compiler: gcc-8
     env:
       - GCC_FLAGS=-m32
       - debugflags=-g0
@@ -259,31 +283,39 @@ env:
       apt:
         config:
           retries: true
-        update: true
         sources:
           - ubuntu-toolchain-r-test
-        packages:
-          - gcc-8-multilib
-          - libffi-dev:i386
-          - libffi6:i386
-          - libgdbm-dev:i386
-          - libgdbm3:i386
-          - libncurses5-dev:i386
-          - libncurses5:i386
-          - libncursesw5-dev:i386
-          - libreadline6-dev:i386
-          - libreadline6:i386
-          - libssl-dev:i386
-          - libssl1.0.0:i386
-          - linux-libc-dev:i386
-          - zlib1g-dev:i386
-          - zlib1g:i386
+        # `packages` are installed in `before_install` for retries.
+    before_install:
+      - |-
+        for seconds in 1 25 100 -1; do
+          travis_apt_get_update
+
+          sudo -E apt-get -yq --no-install-suggests --no-install-recommends $travis_apt_get_options install \
+            gcc-8-multilib \
+            libffi-dev:i386 \
+            libffi6:i386 \
+            libgdbm-dev:i386 \
+            libgdbm3:i386 \
+            libncurses5-dev:i386 \
+            libncurses5:i386 \
+            libncursesw5-dev:i386 \
+            libreadline6-dev:i386 \
+            libreadline6:i386 \
+            libssl-dev:i386 \
+            libssl1.0.0:i386 \
+            linux-libc-dev:i386 \
+            zlib1g-dev:i386 \
+            zlib1g:i386
+
+          if [[ $? = 0 ]]; then break; else sleep "$seconds" || exit 1; fi
+        done
 
   - &pedanticism
     name: -std=c99 -pedantic
-    <<: *linux
-    <<: *make-test-only
+    os: linux
     compiler: clang
+    <<: *make-test-only
     env:
       - GEMS_FOR_TEST=
       - GCC_FLAGS='-std=c99 -Werror=pedantic -pedantic-errors'
@@ -310,18 +342,14 @@ env:
         -Wno-tautological-compare
         -Wno-unused-local-typedef
         -Wno-unused-parameter
-        -Wunused-variable
-        '
+        -Wunused-variable'
       - LDFLAGS=-Wno-unused-command-line-argument
 
   - &rubyspec
     name: Check ruby/spec version guards on Ruby 2.4
-    <<: *linux
+    os: linux
     language: ruby
     rvm: 2.4.6
-    addons:
-      apt:
-        packages:
     before_install:
     install:
     before_script: chmod -R u+w spec/ruby
@@ -332,7 +360,7 @@ env:
 
   - &baseruby
     name: "BASERUBY: Ruby 1.9.3"
-    <<: *linux
+    os: linux
     <<: *make-test-only
     dist: trusty # xenial no longer has ruby-1.9.3
     language: ruby
@@ -341,7 +369,8 @@ env:
   - &x86_64-darwin18
     name: x86_64-darwin18
     osx_image: xcode11
-    <<: *osx
+    os: osx
+    <<: *osx-clang
     env:
       - CONFIG_FLAG=--with-opt-dir=/usr/local/opt/openssl@1.1:/usr/local/opt/zlib
       # Adding `-v` because we're not sure which test could hang forever: https://travis-ci.org/ruby/ruby/jobs/564804923
@@ -353,11 +382,8 @@ env:
 
   - &dependency
     name: Check dependencies in makefiles
-    <<: *linux
+    os: linux
     language: ruby
-    addons:
-      apt:
-        packages:
     before_install:
     install:
     before_script: