diff options
| -rw-r--r-- | .gitlab/ci/global.gitlab-ci.yml | 14 | ||||
| -rw-r--r-- | .gitlab/ci/reports.gitlab-ci.yml | 88 | ||||
| -rw-r--r-- | changelogs/unreleased/60879-fix-reports-timing-out.yml | 5 | ||||
| -rw-r--r-- | config/initializers/forbid_sidekiq_in_transactions.rb | 2 | ||||
| -rw-r--r-- | lib/after_commit_queue.rb | 14 | ||||
| -rw-r--r-- | lib/gitlab/database.rb | 18 |
6 files changed, 43 insertions, 98 deletions
diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index b7ef4b31743..eb50f08c1a7 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -28,11 +28,15 @@ policy: pull stage: test -.dedicated-no-docs-pull-cache-job: - extends: .dedicated-pull-cache-job +.no-docs: except: - /(^docs[\/-].*|.*-docs$)/ +.dedicated-no-docs-pull-cache-job: + extends: + - .dedicated-pull-cache-job + - .no-docs + .dedicated-no-docs-and-no-qa-pull-cache-job: extends: .dedicated-pull-cache-job except: @@ -45,6 +49,12 @@ variables: SETUP_DB: "false" +# Jobs that need a dedicated runner, with no cache +.dedicated-no-docs: + extends: + - .dedicated-runner + - .no-docs + .single-script-job-dedicated-runner: extends: .dedicated-runner image: ruby:2.6-alpine diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index d0e09dbf2f8..89b5ae38072 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -1,98 +1,26 @@ include: - template: Code-Quality.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml + - template: Security/Dependency-Scanning.gitlab-ci.yml code_quality: - extends: .dedicated-no-docs-no-db-pull-cache-job + extends: .dedicated-no-docs # gitlab-org runners set `privileged: false` but we need to have it set to true # since we're using Docker in Docker tags: [] before_script: [] cache: {} dependencies: [] - variables: - SETUP_DB: "false" sast: - extends: .dedicated-no-docs-no-db-pull-cache-job - image: docker:stable + extends: .dedicated-no-docs + before_script: [] + tags: [] variables: SAST_CONFIDENCE_LEVEL: 2 DOCKER_DRIVER: overlay2 - allow_failure: true - tags: [] - before_script: [] - cache: {} - dependencies: [] - services: - - docker:stable-dind - script: - - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage - function propagate_env_vars() { - CURRENT_ENV=$(printenv) - - for VAR_NAME; do - echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " - done - } - - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') - - | - docker run \ - $(propagate_env_vars \ - SAST_ANALYZER_IMAGES \ - SAST_ANALYZER_IMAGE_PREFIX \ - SAST_ANALYZER_IMAGE_TAG \ - SAST_DEFAULT_ANALYZERS \ - SAST_BRAKEMAN_LEVEL \ - SAST_GOSEC_LEVEL \ - SAST_FLAWFINDER_LEVEL \ - SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ - SAST_PULL_ANALYZER_IMAGE_TIMEOUT \ - SAST_RUN_ANALYZER_TIMEOUT \ - ) \ - --volume "$PWD:/code" \ - --volume /var/run/docker.sock:/var/run/docker.sock \ - "registry.gitlab.com/gitlab-org/security-products/sast:$SP_VERSION" /app/bin/run /code - artifacts: - reports: - sast: gl-sast-report.json dependency_scanning: - extends: .dedicated-no-docs-no-db-pull-cache-job - image: docker:stable - variables: - DOCKER_DRIVER: overlay2 - allow_failure: true - tags: [] before_script: [] - cache: {} - dependencies: [] - services: - - docker:stable-dind - script: - - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') - - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage - function propagate_env_vars() { - CURRENT_ENV=$(printenv) - - for VAR_NAME; do - echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " - done - } - - | - docker run \ - $(propagate_env_vars \ - DS_ANALYZER_IMAGES \ - DS_ANALYZER_IMAGE_PREFIX \ - DS_ANALYZER_IMAGE_TAG \ - DS_DEFAULT_ANALYZERS \ - DEP_SCAN_DISABLE_REMOTE_CHECKS \ - DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ - DS_PULL_ANALYZER_IMAGE_TIMEOUT \ - DS_RUN_ANALYZER_TIMEOUT \ - ) \ - --volume "$PWD:/code" \ - --volume /var/run/docker.sock:/var/run/docker.sock \ - "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$SP_VERSION" /code - artifacts: - reports: - dependency_scanning: gl-dependency-scanning-report.json + tags: [] + extends: .dedicated-no-docs diff --git a/changelogs/unreleased/60879-fix-reports-timing-out.yml b/changelogs/unreleased/60879-fix-reports-timing-out.yml new file mode 100644 index 00000000000..845162fe10f --- /dev/null +++ b/changelogs/unreleased/60879-fix-reports-timing-out.yml @@ -0,0 +1,5 @@ +--- +title: Fix reports jobs timing out because of cache +merge_request: 29780 +author: +type: fixed diff --git a/config/initializers/forbid_sidekiq_in_transactions.rb b/config/initializers/forbid_sidekiq_in_transactions.rb index deb94d7dbce..a69f1ba090e 100644 --- a/config/initializers/forbid_sidekiq_in_transactions.rb +++ b/config/initializers/forbid_sidekiq_in_transactions.rb @@ -17,7 +17,7 @@ module Sidekiq module NoEnqueueingFromTransactions %i(perform_async perform_at perform_in).each do |name| define_method(name) do |*args| - if !Sidekiq::Worker.skip_transaction_check && AfterCommitQueue.inside_transaction? + if !Sidekiq::Worker.skip_transaction_check && Gitlab::Database.inside_transaction? begin raise Sidekiq::Worker::EnqueueFromTransactionError, <<~MSG `#{self}.#{name}` cannot be called inside a transaction as this can lead to diff --git a/lib/after_commit_queue.rb b/lib/after_commit_queue.rb index 6fb7985f955..6a180fdf338 100644 --- a/lib/after_commit_queue.rb +++ b/lib/after_commit_queue.rb @@ -15,7 +15,7 @@ module AfterCommitQueue end def run_after_commit_or_now(&block) - if AfterCommitQueue.inside_transaction? + if Gitlab::Database.inside_transaction? if ActiveRecord::Base.connection.current_transaction.records.include?(self) run_after_commit(&block) else @@ -32,18 +32,6 @@ module AfterCommitQueue true end - def self.open_transactions_baseline - if ::Rails.env.test? - return DatabaseCleaner.connections.count { |conn| conn.strategy.is_a?(DatabaseCleaner::ActiveRecord::Transaction) } - end - - 0 - end - - def self.inside_transaction? - ActiveRecord::Base.connection.open_transactions > open_transactions_baseline - end - protected def _run_after_commit_queue diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 8da98cc3909..e4d4779ba9a 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -234,6 +234,7 @@ module Gitlab def self.connection ActiveRecord::Base.connection end + private_class_method :connection def self.cached_column_exists?(table_name, column_name) connection.schema_cache.columns_hash(table_name).has_key?(column_name.to_s) @@ -243,8 +244,6 @@ module Gitlab connection.schema_cache.data_source_exists?(table_name) end - private_class_method :connection - def self.database_version row = connection.execute("SELECT VERSION()").first @@ -272,5 +271,20 @@ module Gitlab end end end + + # inside_transaction? will return true if the caller is running within a transaction. Handles special cases + # when running inside a test environment, in which the entire test is running with a DatabaseCleaner transaction + def self.inside_transaction? + ActiveRecord::Base.connection.open_transactions > open_transactions_baseline + end + + def self.open_transactions_baseline + if ::Rails.env.test? + return DatabaseCleaner.connections.count { |conn| conn.strategy.is_a?(DatabaseCleaner::ActiveRecord::Transaction) } + end + + 0 + end + private_class_method :open_transactions_baseline end end |