diff options
-rw-r--r-- | .gitlab-ci.yml | 239 | ||||
-rw-r--r-- | config/database.yml.postgresql | 5 | ||||
-rwxr-xr-x | scripts/prepare_build.sh | 50 | ||||
-rw-r--r-- | scripts/utils.sh | 14 |
4 files changed, 220 insertions, 88 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 66f8b6e6f9a..3ceb0b4ba79 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.3-git-2.7-phantomjs-2.1-node-7.1" +image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.3-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6" cache: key: "ruby-233" @@ -10,8 +10,6 @@ variables: RAILS_ENV: "test" NODE_ENV: "test" SIMPLECOV: "true" - SETUP_DB: "true" - USE_BUNDLE_INSTALL: "true" GIT_DEPTH: "20" PHANTOMJS_VERSION: "2.1.1" GET_SOURCES_ATTEMPTS: "3" @@ -19,12 +17,9 @@ variables: KNAPSACK_SPINACH_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/spinach_report-master.json before_script: - - source ./scripts/prepare_build.sh - - cp config/gitlab.yml.example config/gitlab.yml - bundle --version - - '[ "$USE_BUNDLE_INSTALL" != "true" ] || retry bundle install --without postgres production --jobs $(nproc) --clean $FLAGS' - - retry gem install knapsack fog-aws mime-types - - '[ "$SETUP_DB" != "true" ] || bundle exec rake db:drop db:create db:schema:load db:migrate add_limits_mysql' + - . scripts/utils.sh + - ./scripts/prepare_build.sh stages: - prepare @@ -52,20 +47,38 @@ stages: paths: - knapsack/ -.use-db: &use-db +.use-pg: &use-pg + services: + - postgres:latest + - redis:alpine + variables: + GITLAB_DATABASE: "postgresql" + +.use-mysql: &use-mysql services: - mysql:latest - redis:alpine + variables: + GITLAB_DATABASE: "mysql" + +.only-master-and-ee-or-mysql: &only-master-and-ee-or-mysql + only: + - /\-(?i)mysql$/ + - master@gitlab-org/gitlab-ce + - master@gitlab/gitlabhq + - tags@gitlab-org/gitlab-ce + - tags@gitlab/gitlabhq + - //@gitlab-org/gitlab-ee + - //@gitlab/gitlab-ee .rspec-knapsack: &rspec-knapsack stage: test <<: *dedicated-runner - <<: *use-db script: - JOB_NAME=( $CI_JOB_NAME ) - - export CI_NODE_INDEX=${JOB_NAME[1]} - - export CI_NODE_TOTAL=${JOB_NAME[2]} - - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json + - export CI_NODE_INDEX=${JOB_NAME[-2]} + - export CI_NODE_TOTAL=${JOB_NAME[-1]} + - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_${JOB_NAME[1]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - export KNAPSACK_GENERATE_REPORT=true - cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH} - knapsack rspec "--color --format documentation" @@ -77,15 +90,23 @@ stages: - knapsack/ - tmp/capybara/ +.rspec-knapsack-pg: &rspec-knapsack-pg + <<: *rspec-knapsack + <<: *use-pg + +.rspec-knapsack-mysql: &rspec-knapsack-mysql + <<: *rspec-knapsack + <<: *use-mysql + <<: *only-master-and-ee-or-mysql + .spinach-knapsack: &spinach-knapsack stage: test <<: *dedicated-runner - <<: *use-db script: - JOB_NAME=( $CI_JOB_NAME ) - - export CI_NODE_INDEX=${JOB_NAME[1]} - - export CI_NODE_TOTAL=${JOB_NAME[2]} - - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json + - export CI_NODE_INDEX=${JOB_NAME[-2]} + - export CI_NODE_TOTAL=${JOB_NAME[-1]} + - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_${JOB_NAME[1]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - export KNAPSACK_GENERATE_REPORT=true - cp ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH} - knapsack spinach "-r rerun" || retry '[[ -e tmp/spinach-rerun.txt ]] && bundle exec spinach -r rerun $(cat tmp/spinach-rerun.txt)' @@ -97,6 +118,15 @@ stages: - knapsack/ - tmp/capybara/ +.spinach-knapsack-pg: &spinach-knapsack-pg + <<: *spinach-knapsack + <<: *use-pg + +.spinach-knapsack-mysql: &spinach-knapsack-mysql + <<: *spinach-knapsack + <<: *use-mysql + <<: *only-master-and-ee-or-mysql + # Prepare and merge knapsack tests knapsack: <<: *knapsack-state @@ -114,8 +144,8 @@ update-knapsack: <<: *dedicated-runner stage: post-test script: - - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_node_*.json - - scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach_node_*.json + - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_pg_node_*.json + - scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach_pg_node_*.json - '[[ -z ${KNAPSACK_S3_BUCKET} ]] || scripts/sync-reports put $KNAPSACK_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH $KNAPSACK_SPINACH_SUITE_REPORT_PATH' - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json only: @@ -125,7 +155,7 @@ update-knapsack: - master@gitlab/gitlab-ee setup-test-env: - <<: *use-db + <<: *use-pg <<: *dedicated-runner stage: prepare script: @@ -140,37 +170,69 @@ setup-test-env: - public/assets - tmp/tests -rspec 0 20: *rspec-knapsack -rspec 1 20: *rspec-knapsack -rspec 2 20: *rspec-knapsack -rspec 3 20: *rspec-knapsack -rspec 4 20: *rspec-knapsack -rspec 5 20: *rspec-knapsack -rspec 6 20: *rspec-knapsack -rspec 7 20: *rspec-knapsack -rspec 8 20: *rspec-knapsack -rspec 9 20: *rspec-knapsack -rspec 10 20: *rspec-knapsack -rspec 11 20: *rspec-knapsack -rspec 12 20: *rspec-knapsack -rspec 13 20: *rspec-knapsack -rspec 14 20: *rspec-knapsack -rspec 15 20: *rspec-knapsack -rspec 16 20: *rspec-knapsack -rspec 17 20: *rspec-knapsack -rspec 18 20: *rspec-knapsack -rspec 19 20: *rspec-knapsack - -spinach 0 10: *spinach-knapsack -spinach 1 10: *spinach-knapsack -spinach 2 10: *spinach-knapsack -spinach 3 10: *spinach-knapsack -spinach 4 10: *spinach-knapsack -spinach 5 10: *spinach-knapsack -spinach 6 10: *spinach-knapsack -spinach 7 10: *spinach-knapsack -spinach 8 10: *spinach-knapsack -spinach 9 10: *spinach-knapsack +rspec pg 0 20: *rspec-knapsack-pg +rspec pg 1 20: *rspec-knapsack-pg +rspec pg 2 20: *rspec-knapsack-pg +rspec pg 3 20: *rspec-knapsack-pg +rspec pg 4 20: *rspec-knapsack-pg +rspec pg 5 20: *rspec-knapsack-pg +rspec pg 6 20: *rspec-knapsack-pg +rspec pg 7 20: *rspec-knapsack-pg +rspec pg 8 20: *rspec-knapsack-pg +rspec pg 9 20: *rspec-knapsack-pg +rspec pg 10 20: *rspec-knapsack-pg +rspec pg 11 20: *rspec-knapsack-pg +rspec pg 12 20: *rspec-knapsack-pg +rspec pg 13 20: *rspec-knapsack-pg +rspec pg 14 20: *rspec-knapsack-pg +rspec pg 15 20: *rspec-knapsack-pg +rspec pg 16 20: *rspec-knapsack-pg +rspec pg 17 20: *rspec-knapsack-pg +rspec pg 18 20: *rspec-knapsack-pg +rspec pg 19 20: *rspec-knapsack-pg + +rspec mysql 0 20: *rspec-knapsack-mysql +rspec mysql 1 20: *rspec-knapsack-mysql +rspec mysql 2 20: *rspec-knapsack-mysql +rspec mysql 3 20: *rspec-knapsack-mysql +rspec mysql 4 20: *rspec-knapsack-mysql +rspec mysql 5 20: *rspec-knapsack-mysql +rspec mysql 6 20: *rspec-knapsack-mysql +rspec mysql 7 20: *rspec-knapsack-mysql +rspec mysql 8 20: *rspec-knapsack-mysql +rspec mysql 9 20: *rspec-knapsack-mysql +rspec mysql 10 20: *rspec-knapsack-mysql +rspec mysql 11 20: *rspec-knapsack-mysql +rspec mysql 12 20: *rspec-knapsack-mysql +rspec mysql 13 20: *rspec-knapsack-mysql +rspec mysql 14 20: *rspec-knapsack-mysql +rspec mysql 15 20: *rspec-knapsack-mysql +rspec mysql 16 20: *rspec-knapsack-mysql +rspec mysql 17 20: *rspec-knapsack-mysql +rspec mysql 18 20: *rspec-knapsack-mysql +rspec mysql 19 20: *rspec-knapsack-mysql + +spinach pg 0 10: *spinach-knapsack-pg +spinach pg 1 10: *spinach-knapsack-pg +spinach pg 2 10: *spinach-knapsack-pg +spinach pg 3 10: *spinach-knapsack-pg +spinach pg 4 10: *spinach-knapsack-pg +spinach pg 5 10: *spinach-knapsack-pg +spinach pg 6 10: *spinach-knapsack-pg +spinach pg 7 10: *spinach-knapsack-pg +spinach pg 8 10: *spinach-knapsack-pg +spinach pg 9 10: *spinach-knapsack-pg + +spinach mysql 0 10: *spinach-knapsack-mysql +spinach mysql 1 10: *spinach-knapsack-mysql +spinach mysql 2 10: *spinach-knapsack-mysql +spinach mysql 3 10: *spinach-knapsack-mysql +spinach mysql 4 10: *spinach-knapsack-mysql +spinach mysql 5 10: *spinach-knapsack-mysql +spinach mysql 6 10: *spinach-knapsack-mysql +spinach mysql 7 10: *spinach-knapsack-mysql +spinach mysql 8 10: *spinach-knapsack-mysql +spinach mysql 9 10: *spinach-knapsack-mysql # Other generic tests .ruby-static-analysis: &ruby-static-analysis @@ -220,29 +282,43 @@ rake ee_compat_check: paths: - ee_compat_check/patches/*.patch -rake db:migrate:reset: +.db-migrate-reset: &db-migrate-reset stage: test - <<: *use-db <<: *dedicated-runner script: - bundle exec rake db:migrate:reset -rake db:rollback: +pg rake db:migrate:reset: + <<: *db-migrate-reset + <<: *use-pg + +mysql rake db:migrate:reset: + <<: *db-migrate-reset + <<: *use-mysql + +.db-rollback: &db-rollback stage: test - <<: *use-db <<: *dedicated-runner script: - bundle exec rake db:rollback STEP=120 - bundle exec rake db:migrate -rake db:seed_fu: +pg rake db:rollback: + <<: *db-rollback + <<: *use-pg + +mysql rake db:rollback: + <<: *db-rollback + <<: *use-mysql + +.db-seed_fu-variables: &db-seed_fu-variables + SIZE: "1" + SETUP_DB: "false" + RAILS_ENV: "development" + +.db-seed_fu: &db-seed_fu stage: test - <<: *use-db <<: *dedicated-runner - variables: - SIZE: "1" - SETUP_DB: "false" - RAILS_ENV: "development" script: - git clone https://gitlab.com/gitlab-org/gitlab-test.git /home/git/repositories/gitlab-org/gitlab-test.git @@ -253,6 +329,20 @@ rake db:seed_fu: paths: - log/development.log +pg rake db:seed_fu: + <<: *db-seed_fu + <<: *use-pg + variables: + <<: *db-seed_fu-variables + GITLAB_DATABASE: "postgresql" + +mysql rake db:seed_fu: + <<: *db-seed_fu + <<: *use-mysql + variables: + <<: *db-seed_fu-variables + GITLAB_DATABASE: "mysql" + rake gitlab:assets:compile: stage: test <<: *dedicated-runner @@ -278,7 +368,7 @@ rake karma: - vendor/ruby - node_modules stage: test - <<: *use-db + <<: *use-pg <<: *dedicated-runner variables: BABEL_ENV: "coverage" @@ -335,12 +425,9 @@ bundler:audit: script: - "bundle exec bundle-audit check --update --ignore CVE-2016-4658" -migration paths: +.migration-paths: &migration-paths stage: test - <<: *use-db <<: *dedicated-runner - variables: - SETUP_DB: "false" only: - master@gitlab-org/gitlab-ce - master@gitlab-org/gitlab-ee @@ -351,13 +438,27 @@ migration paths: - git checkout -f FETCH_HEAD - cp config/resque.yml.example config/resque.yml - sed -i 's/localhost/redis/g' config/resque.yml - - bundle install --without postgres production --jobs $(nproc) $FLAGS --retry=3 + - bundle install --without production --jobs $(nproc) $FLAGS --retry=3 - bundle exec rake db:drop db:create db:schema:load db:seed_fu - git checkout $CI_COMMIT_SHA - - bundle install --without postgres production --jobs $(nproc) $FLAGS --retry=3 + - bundle install --without production --jobs $(nproc) $FLAGS --retry=3 - source scripts/prepare_build.sh - bundle exec rake db:migrate +pg migration paths: + <<: *migration-paths + <<: *use-pg + variables: + SETUP_DB: "false" + GITLAB_DATABASE: "postgresql" + +mysql migration paths: + <<: *migration-paths + <<: *use-mysql + variables: + SETUP_DB: "false" + GITLAB_DATABASE: "mysql" + coverage: stage: post-test services: [] @@ -410,7 +511,7 @@ trigger_docs: before_script: - apk update && apk add curl variables: - GIT_STRATEGY: none + GIT_STRATEGY: "none" cache: {} artifacts: {} script: diff --git a/config/database.yml.postgresql b/config/database.yml.postgresql index 7067e0fe402..c517a4c0cb8 100644 --- a/config/database.yml.postgresql +++ b/config/database.yml.postgresql @@ -9,7 +9,7 @@ production: # username: git # password: # host: localhost - # port: 5432 + # port: 5432 # # Development specific @@ -21,6 +21,7 @@ development: pool: 5 username: postgres password: + # host: localhost # # Staging specific @@ -32,6 +33,7 @@ staging: pool: 5 username: postgres password: + # host: localhost # Warning: The database defined as "test" will be erased and # re-generated from your development database when you run "rake". @@ -43,3 +45,4 @@ test: &test pool: 5 username: postgres password: + # host: localhost diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh index 6e3f76b8399..f170743aea3 100755 --- a/scripts/prepare_build.sh +++ b/scripts/prepare_build.sh @@ -1,25 +1,21 @@ #!/bin/sh -retry() { - if eval "$@"; then - return 0 - fi +. scripts/utils.sh - for i in 2 1; do - sleep 3s - echo "Retrying $i..." - if eval "$@"; then - return 0 - fi - done - return 1 -} +export SETUP_DB=${SETUP_DB:-true} +export GITLAB_DATABASE=${GITLAB_DATABASE:-postgresql} +export USE_BUNDLE_INSTALL=${USE_BUNDLE_INSTALL:-true} if [ -f /.dockerenv ] || [ -f ./dockerinit ]; then - cp config/database.yml.mysql config/database.yml - sed -i 's/username:.*/username: root/g' config/database.yml - sed -i 's/password:.*/password:/g' config/database.yml - sed -i 's/# socket:.*/host: mysql/g' config/database.yml + cp config/database.yml.$GITLAB_DATABASE config/database.yml + + if [ "$GITLAB_DATABASE" = 'postgresql' ]; then + sed -i 's/# host:.*/host: postgres/g' config/database.yml + else # Assume it's mysql + sed -i 's/username:.*/username: root/g' config/database.yml + sed -i 's/password:.*/password:/g' config/database.yml + sed -i 's/# socket:.*/host: mysql/g' config/database.yml + fi cp config/resque.yml.example config/resque.yml sed -i 's/localhost/redis/g' config/resque.yml @@ -28,8 +24,26 @@ if [ -f /.dockerenv ] || [ -f ./dockerinit ]; then else rnd=$(awk 'BEGIN { srand() ; printf("%d\n",rand()*5) }') export PATH="$HOME/bin:/usr/local/bin:/usr/bin:/bin" - cp config/database.yml.mysql config/database.yml + cp config/database.yml.$GITLAB_DATABASE config/database.yml sed "s/username\:.*$/username\: runner/" -i config/database.yml sed "s/password\:.*$/password\: 'password'/" -i config/database.yml sed "s/gitlabhq_test/gitlabhq_test_$rnd/" -i config/database.yml fi + +cp config/gitlab.yml.example config/gitlab.yml + +if [ "$USE_BUNDLE_INSTALL" != "false" ]; then + retry bundle install --without production --jobs $(nproc) --clean $FLAGS +fi + +# Only install knapsack after bundle install! Otherwise oddly some native +# gems could not be found under some circumstance. No idea why, hours wasted. +retry gem install knapsack fog-aws mime-types + +if [ "$SETUP_DB" != "false" ]; then + bundle exec rake db:drop db:create db:schema:load db:migrate + + if [ "$GITLAB_DATABASE" = "mysql" ]; then + bundle exec rake add_limits_mysql + fi +fi diff --git a/scripts/utils.sh b/scripts/utils.sh new file mode 100644 index 00000000000..6faa701f0ce --- /dev/null +++ b/scripts/utils.sh @@ -0,0 +1,14 @@ +retry() { + if eval "$@"; then + return 0 + fi + + for i in 2 1; do + sleep 3s + echo "Retrying $i..." + if eval "$@"; then + return 0 + fi + done + return 1 +} |