diff options
29 files changed, 225 insertions, 59 deletions
diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index 4ff14b660b3..81cc3e7dd2f 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -20,7 +20,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/security-products/codequality:12-5-stable" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/security-products/codequality:0.85.6" script: - | if ! docker info &>/dev/null; then diff --git a/GITLAB_WORKHORSE_VERSION b/GITLAB_WORKHORSE_VERSION index 38b5f0deaee..401cc8dd3e7 100644 --- a/GITLAB_WORKHORSE_VERSION +++ b/GITLAB_WORKHORSE_VERSION @@ -1 +1 @@ -8.18.0 +8.19.0 diff --git a/app/controllers/projects/ci/lints_controller.rb b/app/controllers/projects/ci/lints_controller.rb index 812420e9708..b50afa12da0 100644 --- a/app/controllers/projects/ci/lints_controller.rb +++ b/app/controllers/projects/ci/lints_controller.rb @@ -10,8 +10,8 @@ class Projects::Ci::LintsController < Projects::ApplicationController @content = params[:content] result = Gitlab::Ci::YamlProcessor.new_with_validation_errors(@content, yaml_processor_options) - @error = result.errors.join(', ') - @status = result.valid? + @status = result.valid? + @errors = result.errors if result.valid? @config_processor = result.content diff --git a/app/models/concerns/sha256_attribute.rb b/app/models/concerns/sha256_attribute.rb index 1bd1ad177a2..9dfe1b77829 100644 --- a/app/models/concerns/sha256_attribute.rb +++ b/app/models/concerns/sha256_attribute.rb @@ -39,11 +39,7 @@ module Sha256Attribute end def database_exists? - ApplicationRecord.connection - - true - rescue - false + Gitlab::Database.exists? end end end diff --git a/app/models/concerns/sha_attribute.rb b/app/models/concerns/sha_attribute.rb index c5826f58966..c807dcbf418 100644 --- a/app/models/concerns/sha_attribute.rb +++ b/app/models/concerns/sha_attribute.rb @@ -39,11 +39,7 @@ module ShaAttribute end def database_exists? - ApplicationRecord.connection - - true - rescue - false + Gitlab::Database.exists? end end end diff --git a/app/views/layouts/nav/sidebar/_instance_statistics_links.html.haml b/app/views/layouts/nav/sidebar/_instance_statistics_links.html.haml index 580a25db171..ee2c83dc31e 100644 --- a/app/views/layouts/nav/sidebar/_instance_statistics_links.html.haml +++ b/app/views/layouts/nav/sidebar/_instance_statistics_links.html.haml @@ -1,4 +1,4 @@ -- return unless dashboard_nav_link?(:analytics) +- return unless can?(current_user, :read_instance_statistics) = nav_link(controller: :dev_ops_score) do = link_to instance_statistics_dev_ops_score_index_path do .nav-icon-container diff --git a/app/views/projects/ci/lints/_create.html.haml b/app/views/projects/ci/lints/_create.html.haml index 59b5b9f8a30..d65c06aa2a4 100644 --- a/app/views/projects/ci/lints/_create.html.haml +++ b/app/views/projects/ci/lints/_create.html.haml @@ -1,8 +1,8 @@ - if @status - %p - %b= _("Status:") - = _("syntax is correct") - %i.fa.fa-ok.correct-syntax + .bs-callout.bs-callout-success + %p + %b= _("Status:") + = _("syntax is correct") .table-holder %table.table.table-bordered @@ -40,9 +40,10 @@ %b= _("Allowed to fail") - else - %p - %b= _("Status:") - = _("syntax is incorrect") - %i.fa.fa-remove.incorrect-syntax - %b= _("Error:") - = @error + .bs-callout.bs-callout-danger + %p + %b= _("Status:") + = _("syntax is incorrect") + %pre + - @errors.each do |message| + %p= message diff --git a/changelogs/unreleased/better-errors-from-extends.yml b/changelogs/unreleased/better-errors-from-extends.yml new file mode 100644 index 00000000000..5eede9b2d6b --- /dev/null +++ b/changelogs/unreleased/better-errors-from-extends.yml @@ -0,0 +1,5 @@ +--- +title: Add JSON error context to extends error in CI lint +merge_request: 30066 +author: +type: changed diff --git a/changelogs/unreleased/enable-cross-project-artifacts.yml b/changelogs/unreleased/enable-cross-project-artifacts.yml new file mode 100644 index 00000000000..06db7cd034c --- /dev/null +++ b/changelogs/unreleased/enable-cross-project-artifacts.yml @@ -0,0 +1,5 @@ +--- +title: Download cross-project artifacts by using needs keyword in the CI file +merge_request: 22161 +author: +type: added diff --git a/changelogs/unreleased/update-codequality-to-0-85-6.yml b/changelogs/unreleased/update-codequality-to-0-85-6.yml new file mode 100644 index 00000000000..29cb3d34f53 --- /dev/null +++ b/changelogs/unreleased/update-codequality-to-0-85-6.yml @@ -0,0 +1,5 @@ +--- +title: Update GitLab's codeclimate to 0.85.6 +merge_request: 22659 +author: Takuya Noguchi +type: other diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index c2d6ed58118..5c5dd12969a 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -2385,6 +2385,51 @@ rspec: - build_job_3 ``` +#### Cross project artifact downloads with `needs` **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/14311) in GitLab v12.7. + +`needs` can be used to download artifacts from up to five jobs in pipelines on +[other refs in the same project](#artifact-downloads-between-pipelines-in-the-same-project), +or pipelines in different projects: + +```yaml +build_job: + stage: build + script: + - ls -lhR + needs: + - project: group/project-name + job: build-1 + ref: master + artifacts: true +``` + +`build_job` will download the artifacts from the latest successful `build-1` job +on the `master` branch in the `group/project-name` project. + +##### Artifact downloads between pipelines in the same project + +`needs` can be used to download artifacts from different pipelines in the current project +by setting the `project` keyword as the current project's name, and specifying a ref. +In the example below, `build_job` will download the artifacts for the latest successful +`build-1` job with the `other-ref` ref: + +```yaml +build_job: + stage: build + script: + - ls -lhR + needs: + - project: group/same-project-name + job: build-1 + ref: other-ref + artifacts: true +``` + +NOTE: **Note:** +Downloading artifacts from jobs that are run in [`parallel:`](#parallel) is not supported. + ### `coverage` > [Introduced][ce-7447] in GitLab 8.17. diff --git a/doc/user/admin_area/settings/continuous_integration.md b/doc/user/admin_area/settings/continuous_integration.md index 8f8af0d7481..43b0671d9e0 100644 --- a/doc/user/admin_area/settings/continuous_integration.md +++ b/doc/user/admin_area/settings/continuous_integration.md @@ -169,9 +169,10 @@ but commented out to help encourage others to add to it in the future. --> ## Required pipeline configuration **(PREMIUM ONLY)** CAUTION: **Caution:** -The Required Pipeline Configuration feature is deprecated and will be removed when an -[improved compliance solution](https://gitlab.com/gitlab-org/gitlab/issues/34830) -is added to GitLab. It is recommended to avoid using this feature. +This feature is being re-evaluated in favor of a different +[compliance solution](https://gitlab.com/gitlab-org/gitlab/issues/34830). +We recommend that users who haven't yet implemented this feature wait for +the new solution. GitLab administrators can force a pipeline configuration to run on every pipeline. diff --git a/lib/feature.rb b/lib/feature.rb index 88b0d871c3a..543512b1598 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -52,6 +52,10 @@ class Feature # use `default_enabled: true` to default the flag to being `enabled` # unless set explicitly. The default is `disabled` def enabled?(key, thing = nil, default_enabled: false) + # During setup the database does not exist yet. So we haven't stored a value + # for the feature yet and return the default. + return default_enabled unless Gitlab::Database.exists? + feature = Feature.get(key) # If we're not default enabling the flag or the feature has been set, always evaluate. diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 1708984c1cb..8bc60a36ebd 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,7 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/security-products/codequality:0.85.5" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/security-products/codequality:0.85.6" script: - | if ! docker info &>/dev/null; then diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index ceab9322857..82ec740ade1 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -241,6 +241,14 @@ module Gitlab row['version'] end + def self.exists? + connection + + true + rescue + false + end + private_class_method :database_version def self.add_post_migrate_path_to_rails(force: false) diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 186e395872c..66c95b378cd 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -7278,9 +7278,6 @@ msgstr "" msgid "Error with Akismet. Please check the logs for more info." msgstr "" -msgid "Error:" -msgstr "" - msgid "ErrorTracking|Active" msgstr "" diff --git a/qa/qa/page/base.rb b/qa/qa/page/base.rb index 3ac714940cd..29946791df7 100644 --- a/qa/qa/page/base.rb +++ b/qa/qa/page/base.rb @@ -132,20 +132,25 @@ module QA element.select value end + def has_active_element?(name, **kwargs) + has_element?(name, class: 'active', **kwargs) + end + def has_element?(name, **kwargs) wait_for_requests - wait = kwargs[:wait] ? kwargs[:wait] && kwargs.delete(:wait) : Capybara.default_max_wait_time - text = kwargs[:text] ? kwargs[:text] && kwargs.delete(:text) : nil + wait = kwargs.delete(:wait) || Capybara.default_max_wait_time + text = kwargs.delete(:text) + klass = kwargs.delete(:class) - has_css?(element_selector_css(name, kwargs), text: text, wait: wait) + has_css?(element_selector_css(name, kwargs), text: text, wait: wait, class: klass) end def has_no_element?(name, **kwargs) wait_for_requests - wait = kwargs[:wait] ? kwargs[:wait] && kwargs.delete(:wait) : Capybara.default_max_wait_time - text = kwargs[:text] ? kwargs[:text] && kwargs.delete(:text) : nil + wait = kwargs.delete(:wait) || Capybara.default_max_wait_time + text = kwargs.delete(:text) has_no_css?(element_selector_css(name, kwargs), wait: wait, text: text) end diff --git a/qa/qa/page/main/menu.rb b/qa/qa/page/main/menu.rb index 5f4b3946e6a..8ad30632fa1 100644 --- a/qa/qa/page/main/menu.rb +++ b/qa/qa/page/main/menu.rb @@ -76,8 +76,14 @@ module QA end def sign_out - within_user_menu do - click_element :sign_out_link + retry_until do + break true unless signed_in? + + within_user_menu do + click_element :sign_out_link + end + + has_no_element?(:user_avatar) end end diff --git a/qa/qa/page/search/results.rb b/qa/qa/page/search/results.rb index 2f99d8da784..85f1d224935 100644 --- a/qa/qa/page/search/results.rb +++ b/qa/qa/page/search/results.rb @@ -19,11 +19,11 @@ module QA::Page end def switch_to_code - click_element(:code_tab) + switch_to_tab(:code_tab) end def switch_to_projects - click_element(:projects_tab) + switch_to_tab(:projects_tab) end def has_file_in_project?(file_name, project_name) @@ -32,7 +32,7 @@ module QA::Page def has_file_with_content?(file_name, file_text) within_element_by_index(:result_item_content, 0) do - false unless has_element?(:file_title_content, text: file_name) + break false unless has_element?(:file_title_content, text: file_name) has_element?(:file_text_content, text: file_text) end @@ -41,6 +41,15 @@ module QA::Page def has_project?(project_name) has_element?(:project, project_name: project_name) end + + private + + def switch_to_tab(tab) + retry_until do + click_element(tab) + has_active_element?(tab) + end + end end end end diff --git a/qa/qa/page/validatable.rb b/qa/qa/page/validatable.rb index 8467d261285..20aad65688d 100644 --- a/qa/qa/page/validatable.rb +++ b/qa/qa/page/validatable.rb @@ -11,8 +11,7 @@ module QA elements.each do |element| next unless element.required? - # TODO: this wait needs to be replaced by the wait class - unless base_page.has_element?(element.name, wait: 60) + unless base_page.wait(reload: false) { base_page.has_element?(element.name, wait: 15) } raise Validatable::PageValidationError, "#{element.name} did not appear on #{self.name} as expected" end end diff --git a/qa/qa/resource/repository/commit.rb b/qa/qa/resource/repository/commit.rb index 4b5e8535ade..e3fb5bf486d 100644 --- a/qa/qa/resource/repository/commit.rb +++ b/qa/qa/resource/repository/commit.rb @@ -11,6 +11,8 @@ module QA :file_path, :sha + attribute :short_id + attribute :project do Project.fabricate! do |resource| resource.name = 'project-with-commit' diff --git a/qa/qa/runtime/search.rb b/qa/qa/runtime/search.rb index faa110c96e7..74402301098 100644 --- a/qa/qa/runtime/search.rb +++ b/qa/qa/runtime/search.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'securerandom' + module QA module Runtime module Search @@ -8,26 +10,83 @@ module QA ElasticSearchServerError = Class.new(RuntimeError) - def elasticsearch_responding? + def assert_elasticsearch_responding QA::Runtime::Logger.debug("Attempting to search via Elasticsearch...") - QA::Support::Retrier.retry_on_exception do - # We don't care about the results of the search, we just need - # any search that uses Elasticsearch, not the native search - # The Elasticsearch-only scopes are blobs, wiki_blobs, and commits. - request = Runtime::API::Request.new(api_client, "/search?scope=blobs&search=foo") - response = get(request.url) + QA::Support::Retrier.retry_on_exception(max_attempts: 3) do + search_term = SecureRandom.hex(8) + + QA::Runtime::Logger.debug("Creating commit and project including search term '#{search_term}'...") - unless response.code == singleton_class::HTTP_STATUS_OK - raise ElasticSearchServerError, "Search attempt failed. Request returned (#{response.code}): `#{response}`." + content = "Elasticsearch test commit #{search_term}" + project = Resource::Project.fabricate_via_api! do |project| + project.name = "project-to-search-#{search_term}" + end + commit = Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = project + commit.commit_message = content + commit.add_files( + [ + { + file_path: 'test.txt', + content: content + } + ] + ) end - true + find_commit(commit, "commit*#{search_term}") + find_project(project, "to-search*#{search_term}") + end + end + + def find_code(file_name, search_term) + find_target_in_scope('blobs', search_term) do |record| + record[:filename] == file_name && record[:data].include?(search_term) end + + QA::Runtime::Logger.debug("Found file '#{file_name} containing code '#{search_term}'") + end + + def find_commit(commit, search_term) + find_target_in_scope('commits', search_term) do |record| + record[:message] == commit.commit_message + end + + QA::Runtime::Logger.debug("Found commit '#{commit.commit_message} (#{commit.short_id})' via '#{search_term}'") + end + + def find_project(project, search_term) + find_target_in_scope('projects', search_term) do |record| + record[:name] == project.name + end + + QA::Runtime::Logger.debug("Found project '#{project.name}' via '#{search_term}'") end private + def find_target_in_scope(scope, search_term) + QA::Support::Retrier.retry_until(max_attempts: 10, sleep_interval: 10, raise_on_failure: true, retry_on_exception: true) do + result = search(scope, search_term) + result && result.any? { |record| yield record } + end + end + + def search(scope, term) + QA::Runtime::Logger.debug("Search scope '#{scope}' for '#{term}'...") + request = Runtime::API::Request.new(api_client, "/search?scope=#{scope}&search=#{term}") + response = get(request.url) + + unless response.code == singleton_class::HTTP_STATUS_OK + msg = "Search attempt failed. Request returned (#{response.code}): `#{response}`." + QA::Runtime::Logger.debug(msg) + raise ElasticSearchServerError, msg + end + + parse_body(response) + end + def api_client @api_client ||= Runtime::API::Client.new(:gitlab) end diff --git a/qa/qa/specs/features/browser_ui/non_devops/performance_bar_spec.rb b/qa/qa/specs/features/browser_ui/non_devops/performance_bar_spec.rb index 74e2f51cf28..d78ad6d86d3 100644 --- a/qa/qa/specs/features/browser_ui/non_devops/performance_bar_spec.rb +++ b/qa/qa/specs/features/browser_ui/non_devops/performance_bar_spec.rb @@ -2,7 +2,7 @@ module QA context 'Performance bar' do - context 'when logged in as an admin user', :requires_admin do + context 'when logged in as an admin user', :requires_admin, quarantine: 'https://gitlab.com/gitlab-org/gitlab/issues/196141' do before do Flow::Login.sign_in_as_admin Page::Main::Menu.perform(&:go_to_admin_area) diff --git a/qa/qa/support/page/logging.rb b/qa/qa/support/page/logging.rb index 5d73d9635f8..8420f4112e9 100644 --- a/qa/qa/support/page/logging.rb +++ b/qa/qa/support/page/logging.rb @@ -173,6 +173,7 @@ module QA def log_has_element_or_not(method, name, found, **kwargs) msg = ["#{method} :#{name}"] msg << %Q(with text "#{kwargs[:text]}") if kwargs[:text] + msg << "class: #{kwargs[:class]}" if kwargs[:class] msg << "(wait: #{kwargs[:wait] || Capybara.default_max_wait_time})" msg << "returned: #{found}" diff --git a/spec/controllers/projects/ci/lints_controller_spec.rb b/spec/controllers/projects/ci/lints_controller_spec.rb index 3d8f287f999..8fb39f734b6 100644 --- a/spec/controllers/projects/ci/lints_controller_spec.rb +++ b/spec/controllers/projects/ci/lints_controller_spec.rb @@ -103,7 +103,7 @@ describe Projects::Ci::LintsController do end it 'assigns errors' do - expect(assigns[:error]).to eq('root config contains unknown keys: rubocop') + expect(assigns[:errors]).to eq(['root config contains unknown keys: rubocop']) end end diff --git a/spec/lib/feature_spec.rb b/spec/lib/feature_spec.rb index 3d59b1f35a9..2525dd17b89 100644 --- a/spec/lib/feature_spec.rb +++ b/spec/lib/feature_spec.rb @@ -171,6 +171,13 @@ describe Feature do end end + it 'returns the default value when the database does not exist' do + fake_default = double('fake default') + expect(ActiveRecord::Base).to receive(:connection) { raise ActiveRecord::NoDatabaseError, "No database" } + + expect(described_class.enabled?(:a_feature, default_enabled: fake_default)).to eq(fake_default) + end + context 'cached feature flag', :request_store do let(:flag) { :some_feature_flag } diff --git a/spec/lib/gitlab/database_spec.rb b/spec/lib/gitlab/database_spec.rb index 3db8900ed8e..4a0eab3ea27 100644 --- a/spec/lib/gitlab/database_spec.rb +++ b/spec/lib/gitlab/database_spec.rb @@ -396,6 +396,20 @@ describe Gitlab::Database do end end + describe '.exists?' do + it 'returns true if `ActiveRecord::Base.connection` succeeds' do + expect(ActiveRecord::Base).to receive(:connection) + + expect(described_class.exists?).to be(true) + end + + it 'returns false if `ActiveRecord::Base.connection` fails' do + expect(ActiveRecord::Base).to receive(:connection) { raise ActiveRecord::NoDatabaseError, 'broken' } + + expect(described_class.exists?).to be(false) + end + end + describe '#true_value' do it 'returns correct value' do expect(described_class.true_value).to eq "'t'" diff --git a/spec/services/ci/create_pipeline_service_spec.rb b/spec/services/ci/create_pipeline_service_spec.rb index c4ce06d9da9..bdf4dcc3142 100644 --- a/spec/services/ci/create_pipeline_service_spec.rb +++ b/spec/services/ci/create_pipeline_service_spec.rb @@ -499,7 +499,7 @@ describe Ci::CreatePipelineService do it 'pull it from Auto-DevOps' do pipeline = execute_service expect(pipeline).to be_auto_devops_source - expect(pipeline.builds.map(&:name)).to eq %w[test code_quality build] + expect(pipeline.builds.map(&:name)).to match_array(%w[test code_quality build]) end end diff --git a/spec/views/projects/ci/lints/show.html.haml_spec.rb b/spec/views/projects/ci/lints/show.html.haml_spec.rb index ea67478ff98..8c3cf04bae6 100644 --- a/spec/views/projects/ci/lints/show.html.haml_spec.rb +++ b/spec/views/projects/ci/lints/show.html.haml_spec.rb @@ -75,6 +75,7 @@ describe 'projects/ci/lints/show' do it 'shows the correct values' do render + expect(rendered).to have_content('Status: syntax is correct') expect(rendered).to have_content('Tag list: dotnet') expect(rendered).to have_content('Only policy: refs, test@dude/repo') expect(rendered).to have_content('Except policy: refs, deploy') @@ -87,14 +88,14 @@ describe 'projects/ci/lints/show' do before do assign(:project, project) assign(:status, false) - assign(:error, 'Undefined error') + assign(:errors, ['Undefined error']) end it 'shows error message' do render expect(rendered).to have_content('Status: syntax is incorrect') - expect(rendered).to have_content('Error: Undefined error') + expect(rendered).to have_content('Undefined error') expect(rendered).not_to have_content('Tag list:') end end |