diff options
| -rw-r--r-- | .gitlab-ci.yml | 67 | ||||
| -rwxr-xr-x | scripts/merge-reports | 1 | ||||
| -rwxr-xr-x | scripts/sync-reports | 95 |
3 files changed, 131 insertions, 32 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2da8207a3cf..96a9e6687df 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,13 +14,15 @@ variables: GIT_DEPTH: "20" PHANTOMJS_VERSION: "2.1.1" GET_SOURCES_ATTEMPTS: "3" + KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/rspec_report-${CI_COMMIT_REF_SLUG}.json + KNAPSACK_SPINACH_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/spinach_report-${CI_COMMIT_REF_SLUG}.json before_script: - source ./scripts/prepare_build.sh - cp config/gitlab.yml.example config/gitlab.yml - bundle --version - '[ "$USE_BUNDLE_INSTALL" != "true" ] || retry bundle install --without postgres production --jobs $(nproc) --clean $FLAGS' - - retry gem install knapsack + - retry gem install knapsack fog-aws mime-types - '[ "$SETUP_DB" != "true" ] || bundle exec rake db:drop db:create db:schema:load db:migrate add_limits_mysql' stages: @@ -39,14 +41,15 @@ stages: variables: SETUP_DB: "false" USE_BUNDLE_INSTALL: "false" + KNAPSACK_S3_BUCKET: "gitlab-ce-cache" cache: key: "knapsack" paths: - - knapsack/ + - knapsack/ artifacts: expire_in: 31d paths: - - knapsack/ + - knapsack/ .use-db: &use-db services: @@ -61,17 +64,17 @@ stages: - JOB_NAME=( $CI_JOB_NAME ) - export CI_NODE_INDEX=${JOB_NAME[1]} - export CI_NODE_TOTAL=${JOB_NAME[2]} - - export KNAPSACK_REPORT_PATH=knapsack/rspec_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json + - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - export KNAPSACK_GENERATE_REPORT=true - - cp knapsack/rspec_report.json ${KNAPSACK_REPORT_PATH} + - cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH} - knapsack rspec "--color --format documentation" artifacts: expire_in: 31d when: always paths: - - coverage/ - - knapsack/ - - tmp/capybara/ + - coverage/ + - knapsack/ + - tmp/capybara/ .spinach-knapsack: &spinach-knapsack stage: test @@ -81,28 +84,44 @@ stages: - JOB_NAME=( $CI_JOB_NAME ) - export CI_NODE_INDEX=${JOB_NAME[1]} - export CI_NODE_TOTAL=${JOB_NAME[2]} - - export KNAPSACK_REPORT_PATH=knapsack/spinach_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json + - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - export KNAPSACK_GENERATE_REPORT=true - - cp knapsack/spinach_report.json ${KNAPSACK_REPORT_PATH} + - cp ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH} - knapsack spinach "-r rerun" || retry '[[ -e tmp/spinach-rerun.txt ]] && bundle exec spinach -r rerun $(cat tmp/spinach-rerun.txt)' artifacts: expire_in: 31d when: always paths: - - coverage/ - - knapsack/ - - tmp/capybara/ + - coverage/ + - knapsack/ + - tmp/capybara/ # Prepare and merge knapsack tests - knapsack: <<: *knapsack-state <<: *dedicated-runner stage: prepare script: - - mkdir -p knapsack/ - - '[[ -f knapsack/rspec_report.json ]] || echo "{}" > knapsack/rspec_report.json' - - '[[ -f knapsack/spinach_report.json ]] || echo "{}" > knapsack/spinach_report.json' + - mkdir -p knapsack/${CI_PROJECT_NAME}/ + - wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || true + - wget -O $KNAPSACK_SPINACH_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_SPINACH_SUITE_REPORT_PATH || true + - '[[ -f $KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}' + - '[[ -f $KNAPSACK_SPINACH_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_SPINACH_SUITE_REPORT_PATH}' + +update-knapsack: + <<: *knapsack-state + <<: *dedicated-runner + stage: post-test + script: + - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_node_*.json + - scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach_node_*.json + - '[[ -z ${KNAPSACK_S3_BUCKET} ]] || scripts/sync-reports put $KNAPSACK_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH $KNAPSACK_SPINACH_SUITE_REPORT_PATH' + - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json + only: + - master@gitlab-org/gitlab-ce + - master@gitlab-org/gitlab-ee + - master@gitlab/gitlabhq + - master@gitlab/gitlab-ee setup-test-env: <<: *use-db @@ -122,20 +141,6 @@ setup-test-env: - public/assets - tmp/tests -update-knapsack: - <<: *knapsack-state - <<: *dedicated-runner - stage: post-test - script: - - scripts/merge-reports knapsack/rspec_report.json knapsack/rspec_node_*.json - - scripts/merge-reports knapsack/spinach_report.json knapsack/spinach_node_*.json - - rm -f knapsack/*_node_*.json - only: - - master@gitlab-org/gitlab-ce - - master@gitlab-org/gitlab-ee - - master@gitlab/gitlabhq - - master@gitlab/gitlab-ee - rspec 0 20: *rspec-knapsack rspec 1 20: *rspec-knapsack rspec 2 20: *rspec-knapsack diff --git a/scripts/merge-reports b/scripts/merge-reports index f7b574001ac..aad76bcc327 100755 --- a/scripts/merge-reports +++ b/scripts/merge-reports @@ -1,7 +1,6 @@ #!/usr/bin/env ruby require 'json' -require 'yaml' main_report_file = ARGV.shift unless main_report_file diff --git a/scripts/sync-reports b/scripts/sync-reports new file mode 100755 index 00000000000..5ed65e78005 --- /dev/null +++ b/scripts/sync-reports @@ -0,0 +1,95 @@ +#!/usr/bin/env ruby + +require 'rubygems' +require 'fog/aws' + +class SyncReports + ACTIONS = %w[get put].freeze + + attr_reader :options + + def initialize(options) + @options = options + + perform_sync! + end + + private + + def perform_sync! + case options[:action] + when 'get' + get_reports! + when 'put' + put_reports! + end + end + + def get_reports! + options[:report_paths].each { |report_path| get_report!(report_path) } + end + + def put_reports! + options[:report_paths].each { |report_path| put_report!(report_path) } + end + + def get_report!(report_path) + file = bucket.files.get(report_path) + + if file.respond_to?(:body) + File.write(report_path, file.body) + puts "#{report_path} was retrieved from S3." + else + puts "#{report_path} does not seem to exist on S3." + end + end + + def put_report!(report_path) + bucket.files.create( + key: report_path, + body: File.open(report_path), + public: true + ) + puts "#{report_path} was uploaded to S3." + end + + def bucket + @bucket ||= storage.directories.get(options[:bucket]) + end + + def storage + @storage ||= + Fog::Storage.new( + provider: 'AWS', + aws_access_key_id: ENV['AWS_ACCESS_KEY_ID'], + aws_secret_access_key: ENV['AWS_SECRET_ACCESS_KEY'] + ) + end +end + +def usage!(error: 'action') + print "\n[ERROR]: " + case error + when 'action' + puts "Please specify an action as first argument: #{SyncReports::ACTIONS.join(', ')}\n\n" + when 'bucket' + puts "Please specify a bucket as second argument!\n\n" + when 'files' + puts "Please specify one or more file paths as third argument!\n\n" + end + puts "Usage: #{__FILE__} [get|put] bucket report_path ...\n\n" + puts "Note: the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment "\ + "variables need to be set\n\n" + exit 1 +end + +if $0 == __FILE__ + action = ARGV.shift + usage!(error: 'action') unless SyncReports::ACTIONS.include?(action) + + bucket = ARGV.shift + usage!(error: 'bucket') unless bucket + usage!(error: 'files') unless ARGV.any? + + SyncReports.new(action: action, bucket: bucket, report_paths: ARGV) +end |