diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-08-19 09:08:42 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-08-19 09:08:42 +0000 |
| commit | b76ae638462ab0f673e5915986070518dd3f9ad3 (patch) | |
| tree | bdab0533383b52873be0ec0eb4d3c66598ff8b91 /spec/lib | |
| parent | 434373eabe7b4be9593d18a585fb763f1e5f1a6f (diff) | |
| download | gitlab-ce-b76ae638462ab0f673e5915986070518dd3f9ad3.tar.gz | |
Add latest changes from gitlab-org/gitlab@14-2-stable-eev14.2.0-rc42
Diffstat (limited to 'spec/lib')
248 files changed, 7812 insertions, 2611 deletions
diff --git a/spec/lib/api/helpers/runner_helpers_spec.rb b/spec/lib/api/ci/helpers/runner_helpers_spec.rb index 65b35845aab..c6638bea59e 100644 --- a/spec/lib/api/helpers/runner_helpers_spec.rb +++ b/spec/lib/api/ci/helpers/runner_helpers_spec.rb @@ -2,12 +2,12 @@ require 'spec_helper' -RSpec.describe API::Helpers::Runner do +RSpec.describe API::Ci::Helpers::Runner do let(:ip_address) { '1.2.3.4' } let(:runner_class) do Class.new do include API::Helpers - include API::Helpers::Runner + include API::Ci::Helpers::Runner attr_accessor :params diff --git a/spec/lib/api/helpers/runner_spec.rb b/spec/lib/api/ci/helpers/runner_spec.rb index e55c20b7ab6..99f2db544a5 100644 --- a/spec/lib/api/helpers/runner_spec.rb +++ b/spec/lib/api/ci/helpers/runner_spec.rb @@ -2,8 +2,8 @@ require 'spec_helper' -RSpec.describe API::Helpers::Runner do - let(:helper) { Class.new { include API::Helpers::Runner }.new } +RSpec.describe API::Ci::Helpers::Runner do + let(:helper) { Class.new { include API::Ci::Helpers::Runner }.new } before do allow(helper).to receive(:env).and_return({}) diff --git a/spec/lib/api/helpers/packages/dependency_proxy_helpers_spec.rb b/spec/lib/api/helpers/packages/dependency_proxy_helpers_spec.rb index 99b52236771..ae0c0f53acd 100644 --- a/spec/lib/api/helpers/packages/dependency_proxy_helpers_spec.rb +++ b/spec/lib/api/helpers/packages/dependency_proxy_helpers_spec.rb @@ -5,7 +5,7 @@ require 'spec_helper' RSpec.describe API::Helpers::Packages::DependencyProxyHelpers do let_it_be(:helper) { Class.new.include(described_class).new } - describe 'redirect_registry_request' do + describe '#redirect_registry_request' do using RSpec::Parameterized::TableSyntax let(:options) { {} } @@ -13,7 +13,7 @@ RSpec.describe API::Helpers::Packages::DependencyProxyHelpers do subject { helper.redirect_registry_request(forward_to_registry, package_type, options) { helper.fallback } } before do - allow(helper).to receive(:options).and_return(for: API::NpmInstancePackages) + allow(helper).to receive(:options).and_return(for: described_class) end shared_examples 'executing fallback' do @@ -34,38 +34,66 @@ RSpec.describe API::Helpers::Packages::DependencyProxyHelpers do subject - expect_snowplow_event(category: 'API::NpmInstancePackages', action: 'npm_request_forward') + expect_snowplow_event(category: described_class.to_s, action: "#{package_type}_request_forward") end end - context 'with npm packages' do - let(:package_type) { :npm } + %i[npm pypi].each do |forwardable_package_type| + context "with #{forwardable_package_type} packages" do + include_context 'dependency proxy helpers context' - where(:application_setting, :forward_to_registry, :example_name) do - true | true | 'executing redirect' - true | false | 'executing fallback' - false | true | 'executing fallback' - false | false | 'executing fallback' - end + let(:package_type) { forwardable_package_type } - with_them do - before do - stub_application_setting(npm_package_requests_forwarding: application_setting) + where(:application_setting, :forward_to_registry, :example_name) do + true | true | 'executing redirect' + true | false | 'executing fallback' + false | true | 'executing fallback' + false | false | 'executing fallback' end - it_behaves_like params[:example_name] + with_them do + before do + allow_fetch_application_setting(attribute: "#{forwardable_package_type}_package_requests_forwarding", return_value: application_setting) + end + + it_behaves_like params[:example_name] + end end end - context 'with non-forwardable packages' do + context 'with non-forwardable package type' do let(:forward_to_registry) { true } before do stub_application_setting(npm_package_requests_forwarding: true) + stub_application_setting(pypi_package_requests_forwarding: true) end - Packages::Package.package_types.keys.without('npm').each do |pkg_type| + Packages::Package.package_types.keys.without('npm', 'pypi').each do |pkg_type| context "#{pkg_type}" do + let(:package_type) { pkg_type.to_sym } + + it 'raises an error' do + expect { subject }.to raise_error(ArgumentError, "Can't find application setting for package_type #{package_type}") + end + end + end + end + + describe '#registry_url' do + subject { helper.registry_url(package_type, package_name: 'test') } + + where(:package_type, :expected_result) do + :npm | 'https://registry.npmjs.org/test' + :pypi | 'https://pypi.org/simple/test/' + end + + with_them do + it { is_expected.to eq(expected_result) } + end + + Packages::Package.package_types.keys.without('npm', 'pypi').each do |pkg_type| + context "with non-forwardable package type #{pkg_type}" do let(:package_type) { pkg_type } it 'raises an error' do diff --git a/spec/lib/api/helpers_spec.rb b/spec/lib/api/helpers_spec.rb index 6e48ee4c315..587fe60860a 100644 --- a/spec/lib/api/helpers_spec.rb +++ b/spec/lib/api/helpers_spec.rb @@ -581,4 +581,40 @@ RSpec.describe API::Helpers do end end end + + describe '#order_by_similarity?' do + where(:params, :allow_unauthorized, :current_user_set, :expected) do + {} | false | false | false + {} | true | false | false + {} | false | true | false + {} | true | true | false + { order_by: 'similarity' } | false | false | false + { order_by: 'similarity' } | true | false | false + { order_by: 'similarity' } | true | true | false + { order_by: 'similarity' } | false | true | false + { search: 'test' } | false | false | false + { search: 'test' } | true | false | false + { search: 'test' } | true | true | false + { search: 'test' } | false | true | false + { order_by: 'similarity', search: 'test' } | false | false | false + { order_by: 'similarity', search: 'test' } | true | false | true + { order_by: 'similarity', search: 'test' } | true | true | true + { order_by: 'similarity', search: 'test' } | false | true | true + end + + with_them do + let_it_be(:user) { create(:user) } + + before do + u = current_user_set ? user : nil + subject.instance_variable_set(:@current_user, u) + + allow(subject).to receive(:params).and_return(params) + end + + it 'returns the expected result' do + expect(subject.order_by_similarity?(allow_unauthorized: allow_unauthorized)).to eq(expected) + end + end + end end diff --git a/spec/lib/backup/database_backup_error_spec.rb b/spec/lib/backup/database_backup_error_spec.rb new file mode 100644 index 00000000000..ef627900050 --- /dev/null +++ b/spec/lib/backup/database_backup_error_spec.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Backup::DatabaseBackupError do + let(:config) do + { + host: 'localhost', + port: 5432, + database: 'gitlabhq_test' + } + end + + let(:db_file_name) { File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz') } + + subject { described_class.new(config, db_file_name) } + + it { is_expected.to respond_to :config } + it { is_expected.to respond_to :db_file_name } + + it 'expects exception message to include database file' do + expect(subject.message).to include("#{db_file_name}") + end + + it 'expects exception message to include database paths being back-up' do + expect(subject.message).to include("#{config[:host]}") + expect(subject.message).to include("#{config[:port]}") + expect(subject.message).to include("#{config[:database]}") + end +end diff --git a/spec/lib/backup/file_backup_error_spec.rb b/spec/lib/backup/file_backup_error_spec.rb new file mode 100644 index 00000000000..bb174bbe4a0 --- /dev/null +++ b/spec/lib/backup/file_backup_error_spec.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Backup::FileBackupError do + let_it_be(:lfs) { create(:lfs_object) } + let_it_be(:upload) { create(:upload) } + + let(:backup_tarball) { '/tmp/backup/uploads' } + + shared_examples 'includes backup path' do + it { is_expected.to respond_to :app_files_dir } + it { is_expected.to respond_to :backup_tarball } + + it 'expects exception message to include file backup path location' do + expect(subject.message).to include("#{subject.backup_tarball}") + end + + it 'expects exception message to include file being back-up' do + expect(subject.message).to include("#{subject.app_files_dir}") + end + end + + context 'with lfs file' do + subject { described_class.new(lfs, backup_tarball) } + + it_behaves_like 'includes backup path' + end + + context 'with uploads file' do + subject { described_class.new(upload, backup_tarball) } + + it_behaves_like 'includes backup path' + end +end diff --git a/spec/lib/backup/gitaly_backup_spec.rb b/spec/lib/backup/gitaly_backup_spec.rb index cdb35c0ce01..a48a1752eff 100644 --- a/spec/lib/backup/gitaly_backup_spec.rb +++ b/spec/lib/backup/gitaly_backup_spec.rb @@ -32,7 +32,7 @@ RSpec.describe Backup::GitalyBackup do project_snippet = create(:project_snippet, :repository, project: project) personal_snippet = create(:personal_snippet, :repository, author: project.owner) - expect(Process).to receive(:spawn).with(anything, 'create', '-path', anything, { in: anything, out: progress }).and_call_original + expect(Open3).to receive(:popen2).with(ENV, anything, 'create', '-path', anything).and_call_original subject.start(:create) subject.enqueue(project, Gitlab::GlRepository::PROJECT) @@ -53,7 +53,7 @@ RSpec.describe Backup::GitalyBackup do let(:parallel) { 3 } it 'passes parallel option through' do - expect(Process).to receive(:spawn).with(anything, 'create', '-path', anything, '-parallel', '3', { in: anything, out: progress }).and_call_original + expect(Open3).to receive(:popen2).with(ENV, anything, 'create', '-path', anything, '-parallel', '3').and_call_original subject.start(:create) subject.wait @@ -64,7 +64,7 @@ RSpec.describe Backup::GitalyBackup do let(:parallel_storage) { 3 } it 'passes parallel option through' do - expect(Process).to receive(:spawn).with(anything, 'create', '-path', anything, '-parallel-storage', '3', { in: anything, out: progress }).and_call_original + expect(Open3).to receive(:popen2).with(ENV, anything, 'create', '-path', anything, '-parallel-storage', '3').and_call_original subject.start(:create) subject.wait @@ -109,7 +109,7 @@ RSpec.describe Backup::GitalyBackup do copy_bundle_to_backup_path('personal_snippet_repo.bundle', personal_snippet.disk_path + '.bundle') copy_bundle_to_backup_path('project_snippet_repo.bundle', project_snippet.disk_path + '.bundle') - expect(Process).to receive(:spawn).with(anything, 'restore', '-path', anything, { in: anything, out: progress }).and_call_original + expect(Open3).to receive(:popen2).with(ENV, anything, 'restore', '-path', anything).and_call_original subject.start(:restore) subject.enqueue(project, Gitlab::GlRepository::PROJECT) @@ -132,7 +132,7 @@ RSpec.describe Backup::GitalyBackup do let(:parallel) { 3 } it 'does not pass parallel option through' do - expect(Process).to receive(:spawn).with(anything, 'restore', '-path', anything, { in: anything, out: progress }).and_call_original + expect(Open3).to receive(:popen2).with(ENV, anything, 'restore', '-path', anything).and_call_original subject.start(:restore) subject.wait diff --git a/spec/lib/backup/manager_spec.rb b/spec/lib/backup/manager_spec.rb index feaca6164eb..2cc1bf41d18 100644 --- a/spec/lib/backup/manager_spec.rb +++ b/spec/lib/backup/manager_spec.rb @@ -12,20 +12,13 @@ RSpec.describe Backup::Manager do before do allow(progress).to receive(:puts) allow(progress).to receive(:print) - - @old_progress = $progress # rubocop:disable Style/GlobalVars - $progress = progress # rubocop:disable Style/GlobalVars - end - - after do - $progress = @old_progress # rubocop:disable Style/GlobalVars end describe '#pack' do - let(:backup_contents) { ['backup_contents'] } + let(:expected_backup_contents) { %w(repositories db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz backup_information.yml) } + let(:tar_file) { '1546300800_2019_01_01_12.3_gitlab_backup.tar' } let(:tar_system_options) { { out: [tar_file, 'w', Gitlab.config.backup.archive_permissions] } } - let(:tar_cmdline) { ['tar', '-cf', '-', *backup_contents, tar_system_options] } - + let(:tar_cmdline) { ['tar', '-cf', '-', *expected_backup_contents, tar_system_options] } let(:backup_information) do { backup_created_at: Time.zone.parse('2019-01-01'), @@ -36,20 +29,20 @@ RSpec.describe Backup::Manager do before do allow(ActiveRecord::Base.connection).to receive(:reconnect!) allow(Kernel).to receive(:system).and_return(true) + allow(YAML).to receive(:load_file).and_return(backup_information) + + ::Backup::Manager::FOLDERS_TO_BACKUP.each do |folder| + allow(Dir).to receive(:exist?).with(File.join(Gitlab.config.backup.path, folder)).and_return(true) + end - allow(subject).to receive(:backup_contents).and_return(backup_contents) allow(subject).to receive(:backup_information).and_return(backup_information) allow(subject).to receive(:upload) end - context 'when BACKUP is not set' do - let(:tar_file) { '1546300800_2019_01_01_12.3_gitlab_backup.tar' } - - it 'uses the default tar file name' do - subject.pack + it 'executes tar' do + subject.pack - expect(Kernel).to have_received(:system).with(*tar_cmdline) - end + expect(Kernel).to have_received(:system).with(*tar_cmdline) end context 'when BACKUP is set' do @@ -62,6 +55,58 @@ RSpec.describe Backup::Manager do expect(Kernel).to have_received(:system).with(*tar_cmdline) end end + + context 'when skipped is set in backup_information.yml' do + let(:expected_backup_contents) { %w{db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz backup_information.yml} } + let(:backup_information) do + { + backup_created_at: Time.zone.parse('2019-01-01'), + gitlab_version: '12.3', + skipped: ['repositories'] + } + end + + it 'executes tar' do + subject.pack + + expect(Kernel).to have_received(:system).with(*tar_cmdline) + end + end + + context 'when a directory does not exist' do + let(:expected_backup_contents) { %w{db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz backup_information.yml} } + + before do + expect(Dir).to receive(:exist?).with(File.join(Gitlab.config.backup.path, 'repositories')).and_return(false) + end + + it 'executes tar' do + subject.pack + + expect(Kernel).to have_received(:system).with(*tar_cmdline) + end + end + end + + describe '#remove_tmp' do + let(:path) { File.join(Gitlab.config.backup.path, 'tmp') } + + before do + allow(FileUtils).to receive(:rm_rf).and_return(true) + end + + it 'removes backups/tmp dir' do + subject.remove_tmp + + expect(FileUtils).to have_received(:rm_rf).with(path) + end + + it 'prints running task with a done confirmation' do + subject.remove_tmp + + expect(progress).to have_received(:print).with('Deleting backups/tmp ... ') + expect(progress).to have_received(:puts).with('done') + end end describe '#remove_old' do diff --git a/spec/lib/backup/repository_backup_error_spec.rb b/spec/lib/backup/repository_backup_error_spec.rb new file mode 100644 index 00000000000..44c75c1cf77 --- /dev/null +++ b/spec/lib/backup/repository_backup_error_spec.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Backup::RepositoryBackupError do + let_it_be(:snippet) { create(:snippet, content: 'foo', file_name: 'foo') } + let_it_be(:project) { create(:project, :repository) } + let_it_be(:wiki) { ProjectWiki.new(project, nil ) } + + let(:backup_repos_path) { '/tmp/backup/repositories' } + + shared_examples 'includes backup path' do + it { is_expected.to respond_to :container } + it { is_expected.to respond_to :backup_repos_path } + + it 'expects exception message to include repo backup path location' do + expect(subject.message).to include("#{subject.backup_repos_path}") + end + + it 'expects exception message to include container being back-up' do + expect(subject.message).to include("#{subject.container.disk_path}") + end + end + + context 'with snippet repository' do + subject { described_class.new(snippet, backup_repos_path) } + + it_behaves_like 'includes backup path' + end + + context 'with project repository' do + subject { described_class.new(project, backup_repos_path) } + + it_behaves_like 'includes backup path' + end + + context 'with wiki repository' do + subject { described_class.new(wiki, backup_repos_path) } + + it_behaves_like 'includes backup path' + end +end diff --git a/spec/lib/banzai/filter/references/alert_reference_filter_spec.rb b/spec/lib/banzai/filter/references/alert_reference_filter_spec.rb index 7c6b0cac24b..cba41166be4 100644 --- a/spec/lib/banzai/filter/references/alert_reference_filter_spec.rb +++ b/spec/lib/banzai/filter/references/alert_reference_filter_spec.rb @@ -220,4 +220,33 @@ RSpec.describe Banzai::Filter::References::AlertReferenceFilter do expect(reference_filter(act, project: nil, group: group).to_html).to eq exp end end + + context 'checking N+1' do + let(:namespace) { create(:namespace) } + let(:project2) { create(:project, :public, namespace: namespace) } + let(:alert2) { create(:alert_management_alert, project: project2) } + let(:alert_reference) { alert.to_reference } + let(:alert2_reference) { alert2.to_reference(full: true) } + + it 'does not have N+1 per multiple references per project', :use_sql_query_cache do + markdown = "#{alert_reference}" + max_count = ActiveRecord::QueryRecorder.new(skip_cached: false) do + reference_filter(markdown) + end.count + + expect(max_count).to eq 1 + + markdown = "#{alert_reference} ^alert#2 ^alert#3 ^alert#4 #{alert2_reference}" + + # Since we're not batching alert queries across projects, + # we have to account for that. + # 1 for both projects, 1 for alerts in each project == 3 + # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/330359 + max_count += 2 + + expect do + reference_filter(markdown) + end.not_to exceed_all_query_limit(max_count) + end + end end diff --git a/spec/lib/banzai/filter/references/commit_reference_filter_spec.rb b/spec/lib/banzai/filter/references/commit_reference_filter_spec.rb index bee8e42d12e..6bcea41a603 100644 --- a/spec/lib/banzai/filter/references/commit_reference_filter_spec.rb +++ b/spec/lib/banzai/filter/references/commit_reference_filter_spec.rb @@ -269,4 +269,34 @@ RSpec.describe Banzai::Filter::References::CommitReferenceFilter do expect(reference_filter(act, context).css('a').first.text).to eql("#{project.full_path}@#{commit.short_id}") end end + + context 'checking N+1' do + let(:namespace2) { create(:namespace) } + let(:namespace3) { create(:namespace) } + let(:project2) { create(:project, :public, :repository, namespace: namespace2) } + let(:project3) { create(:project, :public, :repository, namespace: namespace3) } + let(:commit2) { project2.commit } + let(:commit3) { project3.commit } + let(:commit_reference) { commit.to_reference } + let(:commit2_reference) { commit2.to_reference(full: true) } + let(:commit3_reference) { commit3.to_reference(full: true) } + + it 'does not have N+1 per multiple references per project', :use_sql_query_cache do + markdown = "#{commit_reference}" + max_count = ActiveRecord::QueryRecorder.new(skip_cached: false) do + reference_filter(markdown) + end.count + + markdown = "#{commit_reference} 8b95f2f1 8b95f2f2 8b95f2f3 #{commit2_reference} #{commit3_reference}" + + # Commits are not DB entries, they are on the project itself. + # So adding commits from two more projects to the markdown should + # only increase by 1 query + max_count += 1 + + expect do + reference_filter(markdown) + end.not_to exceed_all_query_limit(max_count) + end + end end diff --git a/spec/lib/banzai/filter/references/milestone_reference_filter_spec.rb b/spec/lib/banzai/filter/references/milestone_reference_filter_spec.rb index f8a00716680..cdf6110dd6c 100644 --- a/spec/lib/banzai/filter/references/milestone_reference_filter_spec.rb +++ b/spec/lib/banzai/filter/references/milestone_reference_filter_spec.rb @@ -92,6 +92,11 @@ RSpec.describe Banzai::Filter::References::MilestoneReferenceFilter do expect(doc.to_html).to match(%r(\(<a.+>#{milestone.reference_link_text}</a>\.\))) end + it 'links with adjacent html tags' do + doc = reference_filter("Milestone <p>#{reference}</p>.") + expect(doc.to_html).to match(%r(<p><a.+>#{milestone.reference_link_text}</a></p>)) + end + it 'ignores invalid milestone names' do exp = act = "Milestone #{Milestone.reference_prefix}#{milestone.name.reverse}" diff --git a/spec/lib/banzai/filter/references/project_reference_filter_spec.rb b/spec/lib/banzai/filter/references/project_reference_filter_spec.rb index 63a5a9184c1..d88e262883f 100644 --- a/spec/lib/banzai/filter/references/project_reference_filter_spec.rb +++ b/spec/lib/banzai/filter/references/project_reference_filter_spec.rb @@ -97,4 +97,34 @@ RSpec.describe Banzai::Filter::References::ProjectReferenceFilter do expect(filter.send(:projects)).to eq([project.full_path]) end end + + context 'checking N+1' do + let_it_be(:normal_project) { create(:project, :public) } + let_it_be(:group) { create(:group) } + let_it_be(:group_project) { create(:project, group: group) } + let_it_be(:nested_group) { create(:group, :nested) } + let_it_be(:nested_project) { create(:project, group: nested_group) } + let_it_be(:normal_project_reference) { get_reference(normal_project) } + let_it_be(:group_project_reference) { get_reference(group_project) } + let_it_be(:nested_project_reference) { get_reference(nested_project) } + + it 'does not have N+1 per multiple project references', :use_sql_query_cache do + markdown = "#{normal_project_reference}" + + # warm up first + reference_filter(markdown) + + max_count = ActiveRecord::QueryRecorder.new(skip_cached: false) do + reference_filter(markdown) + end.count + + expect(max_count).to eq 1 + + markdown = "#{normal_project_reference} #{invalidate_reference(normal_project_reference)} #{group_project_reference} #{nested_project_reference}" + + expect do + reference_filter(markdown) + end.not_to exceed_all_query_limit(max_count) + end + end end diff --git a/spec/lib/banzai/filter/table_of_contents_tag_filter_spec.rb b/spec/lib/banzai/filter/table_of_contents_tag_filter_spec.rb index 56f36af5066..082e5c92e53 100644 --- a/spec/lib/banzai/filter/table_of_contents_tag_filter_spec.rb +++ b/spec/lib/banzai/filter/table_of_contents_tag_filter_spec.rb @@ -6,18 +6,42 @@ RSpec.describe Banzai::Filter::TableOfContentsTagFilter do include FilterSpecHelper context 'table of contents' do - let(:html) { '<p>[[<em>TOC</em>]]</p>' } + shared_examples 'table of contents tag' do + it 'replaces toc tag with ToC result' do + doc = filter(html, {}, { toc: "FOO" }) - it 'replaces [[<em>TOC</em>]] with ToC result' do - doc = filter(html, {}, { toc: "FOO" }) + expect(doc.to_html).to eq("FOO") + end - expect(doc.to_html).to eq("FOO") + it 'handles an empty ToC result' do + doc = filter(html) + + expect(doc.to_html).to eq '' + end + end + + context '[[_TOC_]] as tag' do + it_behaves_like 'table of contents tag' do + let(:html) { '<p>[[<em>TOC</em>]]</p>' } + end end - it 'handles an empty ToC result' do - doc = filter(html) + context '[[_toc_]] as tag' do + it_behaves_like 'table of contents tag' do + let(:html) { '<p>[[<em>toc</em>]]</p>' } + end + end + + context '[TOC] as tag' do + it_behaves_like 'table of contents tag' do + let(:html) { '<p>[TOC]</p>' } + end + end - expect(doc.to_html).to eq '' + context '[toc] as tag' do + it_behaves_like 'table of contents tag' do + let(:html) { '<p>[toc]</p>' } + end end end end diff --git a/spec/lib/banzai/pipeline/full_pipeline_spec.rb b/spec/lib/banzai/pipeline/full_pipeline_spec.rb index 989e06a992d..72661003361 100644 --- a/spec/lib/banzai/pipeline/full_pipeline_spec.rb +++ b/spec/lib/banzai/pipeline/full_pipeline_spec.rb @@ -102,33 +102,45 @@ RSpec.describe Banzai::Pipeline::FullPipeline do describe 'table of contents' do let(:project) { create(:project, :public) } - let(:markdown) do - <<-MARKDOWN.strip_heredoc - [[_TOC_]] + + shared_examples 'table of contents tag' do |tag, tag_html| + let(:markdown) do + <<-MARKDOWN.strip_heredoc + #{tag} # Header - MARKDOWN - end + MARKDOWN + end - let(:invalid_markdown) do - <<-MARKDOWN.strip_heredoc - test [[_TOC_]] + let(:invalid_markdown) do + <<-MARKDOWN.strip_heredoc + test #{tag} # Header - MARKDOWN - end + MARKDOWN + end - it 'inserts a table of contents' do - output = described_class.to_html(markdown, project: project) + it 'inserts a table of contents' do + output = described_class.to_html(markdown, project: project) - expect(output).to include("<ul class=\"section-nav\">") - expect(output).to include("<li><a href=\"#header\">Header</a></li>") + expect(output).to include("<ul class=\"section-nav\">") + expect(output).to include("<li><a href=\"#header\">Header</a></li>") + end + + it 'does not insert a table of contents' do + output = described_class.to_html(invalid_markdown, project: project) + + expect(output).to include("test #{tag_html}") + end end - it 'does not insert a table of contents' do - output = described_class.to_html(invalid_markdown, project: project) + context 'with [[_TOC_]] as tag' do + it_behaves_like 'table of contents tag', '[[_TOC_]]', '[[<em>TOC</em>]]' + end - expect(output).to include("test [[<em>TOC</em>]]") + context 'with [toc] as tag' do + it_behaves_like 'table of contents tag', '[toc]', '[toc]' + it_behaves_like 'table of contents tag', '[TOC]', '[TOC]' end end diff --git a/spec/lib/banzai/pipeline/wiki_pipeline_spec.rb b/spec/lib/banzai/pipeline/wiki_pipeline_spec.rb index 007d310247b..59f5e4a6900 100644 --- a/spec/lib/banzai/pipeline/wiki_pipeline_spec.rb +++ b/spec/lib/banzai/pipeline/wiki_pipeline_spec.rb @@ -27,7 +27,7 @@ RSpec.describe Banzai::Pipeline::WikiPipeline do end end - it 'is case-sensitive' do + it 'is not case-sensitive' do markdown = <<-MD.strip_heredoc [[_toc_]] @@ -36,9 +36,22 @@ RSpec.describe Banzai::Pipeline::WikiPipeline do Foo MD - output = described_class.to_html(markdown, project: project, wiki: wiki) + result = described_class.call(markdown, project: project, wiki: wiki) + + expect(result[:output].to_html).to include(result[:toc]) + end + + it 'works with alternative [toc] tag' do + markdown = <<-MD.strip_heredoc + [toc] - expect(output).to include('[[<em>toc</em>]]') + # Header 1 + + Foo + MD + + result = described_class.call(markdown, project: project, wiki: wiki) + expect(result[:output].to_html).to include(result[:toc]) end it 'handles an empty pipeline result' do diff --git a/spec/lib/error_tracking/collector/sentry_auth_parser_spec.rb b/spec/lib/error_tracking/collector/sentry_auth_parser_spec.rb new file mode 100644 index 00000000000..4f00b1ec654 --- /dev/null +++ b/spec/lib/error_tracking/collector/sentry_auth_parser_spec.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ErrorTracking::Collector::SentryAuthParser do + describe '.parse' do + let(:headers) { { 'X-Sentry-Auth' => "Sentry sentry_key=glet_1fedb514e17f4b958435093deb02048c" } } + let(:request) { double('request', headers: headers) } + + subject { described_class.parse(request) } + + context 'empty headers' do + let(:headers) { {} } + + it 'fails with exception' do + expect { subject }.to raise_error(StandardError) + end + end + + context 'missing sentry_key' do + let(:headers) { { 'X-Sentry-Auth' => "Sentry foo=bar" } } + + it 'returns empty value for public_key' do + expect(subject[:public_key]).to be_nil + end + end + + it 'returns correct value for public_key' do + expect(subject[:public_key]).to eq('glet_1fedb514e17f4b958435093deb02048c') + end + end +end diff --git a/spec/lib/extracts_path_spec.rb b/spec/lib/extracts_path_spec.rb index 05f3bb2f71a..9b2bb024fa6 100644 --- a/spec/lib/extracts_path_spec.rb +++ b/spec/lib/extracts_path_spec.rb @@ -213,20 +213,4 @@ RSpec.describe ExtractsPath do expect(extract_ref_without_atom('foo.atom')).to eq(nil) end end - - describe '#lfs_blob_ids' do - let(:tag) { @project.repository.add_tag(@project.owner, 'my-annotated-tag', 'master', 'test tag') } - let(:ref) { tag.target } - let(:params) { { ref: ref, path: 'README.md' } } - - before do - @project = create(:project, :repository) - end - - it 'handles annotated tags' do - assign_ref_vars - - expect(lfs_blob_ids).to eq([]) - end - end end diff --git a/spec/lib/feature/gitaly_spec.rb b/spec/lib/feature/gitaly_spec.rb index 696427bb8b6..311589c3253 100644 --- a/spec/lib/feature/gitaly_spec.rb +++ b/spec/lib/feature/gitaly_spec.rb @@ -78,7 +78,7 @@ RSpec.describe Feature::Gitaly do context 'when table does not exist' do before do - allow(::Gitlab::Database).to receive(:cached_table_exists?).and_return(false) + allow(::Gitlab::Database.main).to receive(:cached_table_exists?).and_return(false) end it 'returns an empty Hash' do diff --git a/spec/lib/feature_spec.rb b/spec/lib/feature_spec.rb index dc8fd0de313..9d4820f9a4c 100644 --- a/spec/lib/feature_spec.rb +++ b/spec/lib/feature_spec.rb @@ -310,7 +310,7 @@ RSpec.describe Feature, stub_feature_flags: false do context 'when database exists' do before do - allow(Gitlab::Database).to receive(:exists?).and_return(true) + allow(Gitlab::Database.main).to receive(:exists?).and_return(true) end it 'checks the persisted status and returns false' do @@ -322,7 +322,7 @@ RSpec.describe Feature, stub_feature_flags: false do context 'when database does not exist' do before do - allow(Gitlab::Database).to receive(:exists?).and_return(false) + allow(Gitlab::Database.main).to receive(:exists?).and_return(false) end it 'returns false without checking the status in the database' do diff --git a/spec/lib/generators/gitlab/usage_metric_generator_spec.rb b/spec/lib/generators/gitlab/usage_metric_generator_spec.rb index f38815acca6..207ecb88aad 100644 --- a/spec/lib/generators/gitlab/usage_metric_generator_spec.rb +++ b/spec/lib/generators/gitlab/usage_metric_generator_spec.rb @@ -8,7 +8,7 @@ RSpec.describe Gitlab::UsageMetricGenerator, :silence_stdout do let(:spec_ce_temp_dir) { Dir.mktmpdir } let(:spec_ee_temp_dir) { Dir.mktmpdir } let(:args) { ['CountFoo'] } - let(:options) { { 'type' => 'redis_hll' } } + let(:options) { { 'type' => 'generic' } } before do stub_const("#{described_class}::CE_DIR", ce_temp_dir) @@ -30,27 +30,39 @@ RSpec.describe Gitlab::UsageMetricGenerator, :silence_stdout do describe 'Creating metric instrumentation files' do let(:sample_metric_dir) { 'lib/generators/gitlab/usage_metric_generator' } - let(:sample_metric) { fixture_file(File.join(sample_metric_dir, 'sample_metric.rb')) } + let(:generic_sample_metric) { fixture_file(File.join(sample_metric_dir, 'sample_generic_metric.rb')) } + let(:database_sample_metric) { fixture_file(File.join(sample_metric_dir, 'sample_database_metric.rb')) } let(:sample_spec) { fixture_file(File.join(sample_metric_dir, 'sample_metric_test.rb')) } it 'creates CE metric instrumentation files using the template' do described_class.new(args, options).invoke_all - expect_generated_file(ce_temp_dir, 'count_foo_metric.rb', sample_metric) + expect_generated_file(ce_temp_dir, 'count_foo_metric.rb', generic_sample_metric) expect_generated_file(spec_ce_temp_dir, 'count_foo_metric_spec.rb', sample_spec) end context 'with EE flag true' do - let(:options) { { 'type' => 'redis_hll', 'ee' => true } } + let(:options) { { 'type' => 'generic', 'ee' => true } } it 'creates EE metric instrumentation files using the template' do described_class.new(args, options).invoke_all - expect_generated_file(ee_temp_dir, 'count_foo_metric.rb', sample_metric) + expect_generated_file(ee_temp_dir, 'count_foo_metric.rb', generic_sample_metric) expect_generated_file(spec_ee_temp_dir, 'count_foo_metric_spec.rb', sample_spec) end end + context 'for database type' do + let(:options) { { 'type' => 'database', 'operation' => 'count' } } + + it 'creates the metric instrumentation file using the template' do + described_class.new(args, options).invoke_all + + expect_generated_file(ce_temp_dir, 'count_foo_metric.rb', database_sample_metric) + expect_generated_file(spec_ce_temp_dir, 'count_foo_metric_spec.rb', sample_spec) + end + end + context 'with type option missing' do let(:options) { {} } @@ -66,5 +78,21 @@ RSpec.describe Gitlab::UsageMetricGenerator, :silence_stdout do expect { described_class.new(args, options).invoke_all }.to raise_error(ArgumentError, /Unknown type 'some_other_type'/) end end + + context 'without operation for database metric' do + let(:options) { { 'type' => 'database' } } + + it 'raises an ArgumentError' do + expect { described_class.new(args, options).invoke_all }.to raise_error(ArgumentError, /Unknown operation ''/) + end + end + + context 'with wrong operation for database metric' do + let(:options) { { 'type' => 'database', 'operation' => 'sleep' } } + + it 'raises an ArgumentError' do + expect { described_class.new(args, options).invoke_all }.to raise_error(ArgumentError, /Unknown operation 'sleep'/) + end + end end end diff --git a/spec/lib/gitlab/analytics/cycle_analytics/records_fetcher_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/records_fetcher_spec.rb index ebc5ae2a632..4fe55ba0c0c 100644 --- a/spec/lib/gitlab/analytics/cycle_analytics/records_fetcher_spec.rb +++ b/spec/lib/gitlab/analytics/cycle_analytics/records_fetcher_spec.rb @@ -79,56 +79,6 @@ RSpec.describe Gitlab::Analytics::CycleAnalytics::RecordsFetcher do include_context 'when records are loaded by maintainer' end - - describe 'special case' do - let(:mr1) { create(:merge_request, source_project: project, allow_broken: true, created_at: 20.days.ago) } - let(:mr2) { create(:merge_request, source_project: project, allow_broken: true, created_at: 19.days.ago) } - let(:ci_build1) { create(:ci_build) } - let(:ci_build2) { create(:ci_build) } - let(:default_stages) { Gitlab::Analytics::CycleAnalytics::DefaultStages } - let(:stage) { build(:cycle_analytics_project_stage, default_stages.params_for_test_stage.merge(project: project)) } - - before do - mr1.metrics.update!({ - merged_at: 5.days.ago, - first_deployed_to_production_at: 1.day.ago, - latest_build_started_at: 5.days.ago, - latest_build_finished_at: 1.day.ago, - pipeline: ci_build1.pipeline - }) - mr2.metrics.update!({ - merged_at: 10.days.ago, - first_deployed_to_production_at: 5.days.ago, - latest_build_started_at: 9.days.ago, - latest_build_finished_at: 7.days.ago, - pipeline: ci_build2.pipeline - }) - - project.add_user(user, Gitlab::Access::MAINTAINER) - end - - context 'returns build records' do - shared_examples 'orders build records by `latest_build_finished_at`' do - it 'orders by `latest_build_finished_at`' do - build_ids = subject.map { |item| item[:id] } - - expect(build_ids).to eq([ci_build1.id, ci_build2.id]) - end - end - - context 'when requesting records for default test stage' do - include_examples 'orders build records by `latest_build_finished_at`' - end - - context 'when requesting records for default staging stage' do - before do - stage.assign_attributes(default_stages.params_for_staging_stage) - end - - include_examples 'orders build records by `latest_build_finished_at`' - end - end - end end describe 'pagination' do diff --git a/spec/lib/gitlab/auth/auth_finders_spec.rb b/spec/lib/gitlab/auth/auth_finders_spec.rb index 2d4239eb761..b0522e269e0 100644 --- a/spec/lib/gitlab/auth/auth_finders_spec.rb +++ b/spec/lib/gitlab/auth/auth_finders_spec.rb @@ -496,18 +496,6 @@ RSpec.describe Gitlab::Auth::AuthFinders do expect(find_user_from_web_access_token(:archive)).to eq(user) end - context 'when allow_archive_as_web_access_format feature flag is disabled' do - before do - stub_feature_flags(allow_archive_as_web_access_format: false) - end - - it 'returns nil for ARCHIVE requests' do - set_header('SCRIPT_NAME', '/-/archive/main.zip') - - expect(find_user_from_web_access_token(:archive)).to be_nil - end - end - context 'for API requests' do it 'returns the user' do set_header('SCRIPT_NAME', '/api/endpoint') diff --git a/spec/lib/gitlab/auth/result_spec.rb b/spec/lib/gitlab/auth/result_spec.rb new file mode 100644 index 00000000000..f8de4b80db2 --- /dev/null +++ b/spec/lib/gitlab/auth/result_spec.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Auth::Result do + let_it_be(:actor) { create(:user) } + + subject { described_class.new(actor, nil, nil, []) } + + context 'when actor is User' do + let_it_be(:actor) { create(:user) } + + it 'returns auth_user' do + expect(subject.auth_user).to eq(actor) + end + + it 'does not return deploy token' do + expect(subject.deploy_token).to be_nil + end + end + + context 'when actor is Deploy token' do + let_it_be(:actor) { create(:deploy_token) } + + it 'returns deploy token' do + expect(subject.deploy_token).to eq(actor) + end + + it 'does not return auth_user' do + expect(subject.auth_user).to be_nil + end + end + + describe '#authentication_abilities_include?' do + context 'when authentication abilities are empty' do + it 'returns false' do + expect(subject.authentication_abilities_include?(:read_code)).to be_falsey + end + end + + context 'when authentication abilities are not empty' do + subject { described_class.new(actor, nil, nil, [:push_code]) } + + it 'returns false when ability is not allowed' do + expect(subject.authentication_abilities_include?(:read_code)).to be_falsey + end + + it 'returns true when ability is allowed' do + expect(subject.authentication_abilities_include?(:push_code)).to be_truthy + end + end + end + + describe '#can_perform_action_on_project?' do + let(:project) { double } + + it 'returns if actor can do perform given action on given project' do + expect(Ability).to receive(:allowed?).with(actor, :push_code, project).and_return(true) + expect(subject.can_perform_action_on_project?(:push_code, project)).to be_truthy + end + + it 'returns if actor cannot do perform given action on given project' do + expect(Ability).to receive(:allowed?).with(actor, :push_code, project).and_return(false) + expect(subject.can_perform_action_on_project?(:push_code, project)).to be_falsey + end + end + + describe '#can?' do + it 'returns if actor can do perform given action on given project' do + expect(actor).to receive(:can?).with(:push_code).and_return(true) + expect(subject.can?(:push_code)).to be_truthy + end + + it 'returns if actor cannot do perform given action on given project' do + expect(actor).to receive(:can?).with(:push_code).and_return(false) + expect(subject.can?(:push_code)).to be_falsey + end + end +end diff --git a/spec/lib/gitlab/auth_spec.rb b/spec/lib/gitlab/auth_spec.rb index 1d708b17076..cc592bb8f24 100644 --- a/spec/lib/gitlab/auth_spec.rb +++ b/spec/lib/gitlab/auth_spec.rb @@ -5,6 +5,7 @@ require 'spec_helper' RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do let_it_be(:project) { create(:project) } + let(:auth_failure) { { actor: nil, project: nil, type: nil, authentication_abilities: nil } } let(:gl_auth) { described_class } describe 'constants' do @@ -159,26 +160,26 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do let(:project) { build.project } it 'recognises user-less build' do - expect(subject).to eq(Gitlab::Auth::Result.new(nil, build.project, :ci, described_class.build_authentication_abilities)) + expect(subject).to have_attributes(actor: nil, project: build.project, type: :ci, authentication_abilities: described_class.build_authentication_abilities) end it 'recognises user token' do build.update(user: create(:user)) - expect(subject).to eq(Gitlab::Auth::Result.new(build.user, build.project, :build, described_class.build_authentication_abilities)) + expect(subject).to have_attributes(actor: build.user, project: build.project, type: :build, authentication_abilities: described_class.build_authentication_abilities) end it 'fails with blocked user token' do build.update(user: create(:user, :blocked)) - expect(subject).to eq(Gitlab::Auth::Result.new(nil, nil, nil, nil)) + expect(subject).to have_attributes(auth_failure) end context 'username is not gitlab-ci-token' do let(:username) { 'another_username' } it 'fails to authenticate' do - expect(subject).to eq(Gitlab::Auth::Result.new(nil, nil, nil, nil)) + expect(subject).to have_attributes(auth_failure) end end end @@ -189,7 +190,7 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do let(:project) { build.project } it 'denies authentication' do - expect(subject).to eq(Gitlab::Auth::Result.new) + expect(subject).to have_attributes(auth_failure) end end end @@ -199,20 +200,20 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do project.create_drone_ci_integration(active: true) project.drone_ci_integration.update(token: 'token') - expect(gl_auth.find_for_git_client('drone-ci-token', 'token', project: project, ip: 'ip')).to eq(Gitlab::Auth::Result.new(nil, project, :ci, described_class.build_authentication_abilities)) + expect(gl_auth.find_for_git_client('drone-ci-token', 'token', project: project, ip: 'ip')).to have_attributes(actor: nil, project: project, type: :ci, authentication_abilities: described_class.build_authentication_abilities) end it 'recognizes master passwords' do user = create(:user, password: 'password') - expect(gl_auth.find_for_git_client(user.username, 'password', project: nil, ip: 'ip')).to eq(Gitlab::Auth::Result.new(user, nil, :gitlab_or_ldap, described_class.full_authentication_abilities)) + expect(gl_auth.find_for_git_client(user.username, 'password', project: nil, ip: 'ip')).to have_attributes(actor: user, project: nil, type: :gitlab_or_ldap, authentication_abilities: described_class.full_authentication_abilities) end include_examples 'user login operation with unique ip limit' do let(:user) { create(:user, password: 'password') } def operation - expect(gl_auth.find_for_git_client(user.username, 'password', project: nil, ip: 'ip')).to eq(Gitlab::Auth::Result.new(user, nil, :gitlab_or_ldap, described_class.full_authentication_abilities)) + expect(gl_auth.find_for_git_client(user.username, 'password', project: nil, ip: 'ip')).to have_attributes(actor: user, project: nil, type: :gitlab_or_ldap, authentication_abilities: described_class.full_authentication_abilities) end end @@ -221,14 +222,14 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do user = create(:user) token = Gitlab::LfsToken.new(user).token - expect(gl_auth.find_for_git_client(user.username, token, project: nil, ip: 'ip')).to eq(Gitlab::Auth::Result.new(user, nil, :lfs_token, described_class.read_write_project_authentication_abilities)) + expect(gl_auth.find_for_git_client(user.username, token, project: nil, ip: 'ip')).to have_attributes(actor: user, project: nil, type: :lfs_token, authentication_abilities: described_class.read_write_project_authentication_abilities) end it 'recognizes deploy key lfs tokens' do key = create(:deploy_key) token = Gitlab::LfsToken.new(key).token - expect(gl_auth.find_for_git_client("lfs+deploy-key-#{key.id}", token, project: nil, ip: 'ip')).to eq(Gitlab::Auth::Result.new(key, nil, :lfs_deploy_token, described_class.read_only_authentication_abilities)) + expect(gl_auth.find_for_git_client("lfs+deploy-key-#{key.id}", token, project: nil, ip: 'ip')).to have_attributes(actor: key, project: nil, type: :lfs_deploy_token, authentication_abilities: described_class.read_only_authentication_abilities) end it 'does not try password auth before oauth' do @@ -245,14 +246,14 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do create(:deploy_keys_project, :write_access, deploy_key: key, project: project) token = Gitlab::LfsToken.new(key).token - expect(gl_auth.find_for_git_client("lfs+deploy-key-#{key.id}", token, project: project, ip: 'ip')).to eq(Gitlab::Auth::Result.new(key, nil, :lfs_deploy_token, described_class.read_write_authentication_abilities)) + expect(gl_auth.find_for_git_client("lfs+deploy-key-#{key.id}", token, project: project, ip: 'ip')).to have_attributes(actor: key, project: nil, type: :lfs_deploy_token, authentication_abilities: described_class.read_write_authentication_abilities) end it 'does not grant deploy key write permissions' do key = create(:deploy_key) token = Gitlab::LfsToken.new(key).token - expect(gl_auth.find_for_git_client("lfs+deploy-key-#{key.id}", token, project: project, ip: 'ip')).to eq(Gitlab::Auth::Result.new(key, nil, :lfs_deploy_token, described_class.read_only_authentication_abilities)) + expect(gl_auth.find_for_git_client("lfs+deploy-key-#{key.id}", token, project: project, ip: 'ip')).to have_attributes(actor: key, project: nil, type: :lfs_deploy_token, authentication_abilities: described_class.read_only_authentication_abilities) end end @@ -264,18 +265,18 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do shared_examples 'an oauth failure' do it 'fails' do expect(gl_auth.find_for_git_client("oauth2", token_w_api_scope.token, project: nil, ip: 'ip')) - .to eq(Gitlab::Auth::Result.new(nil, nil, nil, nil)) + .to have_attributes(auth_failure) end end it 'succeeds for OAuth tokens with the `api` scope' do - expect(gl_auth.find_for_git_client("oauth2", token_w_api_scope.token, project: nil, ip: 'ip')).to eq(Gitlab::Auth::Result.new(user, nil, :oauth, described_class.full_authentication_abilities)) + expect(gl_auth.find_for_git_client("oauth2", token_w_api_scope.token, project: nil, ip: 'ip')).to have_attributes(actor: user, project: nil, type: :oauth, authentication_abilities: described_class.full_authentication_abilities) end it 'fails for OAuth tokens with other scopes' do token = Doorkeeper::AccessToken.create!(application_id: application.id, resource_owner_id: user.id, scopes: 'read_user') - expect(gl_auth.find_for_git_client("oauth2", token.token, project: nil, ip: 'ip')).to eq(Gitlab::Auth::Result.new(nil, nil)) + expect(gl_auth.find_for_git_client("oauth2", token.token, project: nil, ip: 'ip')).to have_attributes(auth_failure) end it 'does not try password auth before oauth' do @@ -342,7 +343,7 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do impersonation_token = create(:personal_access_token, :impersonation, scopes: ['api']) expect(gl_auth.find_for_git_client('', impersonation_token.token, project: nil, ip: 'ip')) - .to eq(Gitlab::Auth::Result.new(nil, nil, nil, nil)) + .to have_attributes(auth_failure) end it 'limits abilities based on scope' do @@ -365,36 +366,27 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do it 'fails if user is blocked' do expect(gl_auth.find_for_git_client('', personal_access_token.token, project: nil, ip: 'ip')) - .to eq(Gitlab::Auth::Result.new(nil, nil, nil, nil)) + .to have_attributes(auth_failure) end end - context 'when using a project access token' do - let_it_be(:project_bot_user) { create(:user, :project_bot) } - let_it_be(:project_access_token) { create(:personal_access_token, user: project_bot_user) } - - context 'with valid project access token' do - before do - project.add_maintainer(project_bot_user) - end - + context 'when using a resource access token' do + shared_examples 'with a valid access token' do it 'successfully authenticates the project bot' do - expect(gl_auth.find_for_git_client(project_bot_user.username, project_access_token.token, project: project, ip: 'ip')) - .to eq(Gitlab::Auth::Result.new(project_bot_user, nil, :personal_access_token, described_class.full_authentication_abilities)) + expect(gl_auth.find_for_git_client(project_bot_user.username, access_token.token, project: project, ip: 'ip')) + .to have_attributes(actor: project_bot_user, project: nil, type: :personal_access_token, authentication_abilities: described_class.full_authentication_abilities) end it 'successfully authenticates the project bot with a nil project' do - expect(gl_auth.find_for_git_client(project_bot_user.username, project_access_token.token, project: nil, ip: 'ip')) - .to eq(Gitlab::Auth::Result.new(project_bot_user, nil, :personal_access_token, described_class.full_authentication_abilities)) + expect(gl_auth.find_for_git_client(project_bot_user.username, access_token.token, project: nil, ip: 'ip')) + .to have_attributes(actor: project_bot_user, project: nil, type: :personal_access_token, authentication_abilities: described_class.full_authentication_abilities) end end - context 'with invalid project access token' do - context 'when project bot is not a project member' do - it 'fails for a non-project member' do - expect(gl_auth.find_for_git_client(project_bot_user.username, project_access_token.token, project: project, ip: 'ip')) - .to eq(Gitlab::Auth::Result.new(nil, nil, nil, nil)) - end + shared_examples 'with an invalid access token' do + it 'fails for a non-member' do + expect(gl_auth.find_for_git_client(project_bot_user.username, access_token.token, project: project, ip: 'ip')) + .to have_attributes(auth_failure ) end context 'when project bot user is blocked' do @@ -403,9 +395,59 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do end it 'fails for a blocked project bot' do - expect(gl_auth.find_for_git_client(project_bot_user.username, project_access_token.token, project: project, ip: 'ip')) - .to eq(Gitlab::Auth::Result.new(nil, nil, nil, nil)) + expect(gl_auth.find_for_git_client(project_bot_user.username, access_token.token, project: project, ip: 'ip')) + .to have_attributes(auth_failure ) + end + end + end + + context 'when using a personal namespace project access token' do + let_it_be(:project_bot_user) { create(:user, :project_bot) } + let_it_be(:access_token) { create(:personal_access_token, user: project_bot_user) } + + context 'when the token belongs to the project' do + before do + project.add_maintainer(project_bot_user) + end + + it_behaves_like 'with a valid access token' + end + + it_behaves_like 'with an invalid access token' + end + + context 'when in a group namespace' do + let_it_be(:group) { create(:group) } + let_it_be(:project) { create(:project, group: group) } + + context 'when using a project access token' do + let_it_be(:project_bot_user) { create(:user, :project_bot) } + let_it_be(:access_token) { create(:personal_access_token, user: project_bot_user) } + + context 'when token user belongs to the project' do + before do + project.add_maintainer(project_bot_user) + end + + it_behaves_like 'with a valid access token' + end + + it_behaves_like 'with an invalid access token' + end + + context 'when using a group access token' do + let_it_be(:project_bot_user) { create(:user, name: 'Group token bot', email: "group_#{group.id}_bot@example.com", username: "group_#{group.id}_bot", user_type: :project_bot) } + let_it_be(:access_token) { create(:personal_access_token, user: project_bot_user) } + + context 'when the token belongs to the group' do + before do + group.add_maintainer(project_bot_user) + end + + it_behaves_like 'with a valid access token' end + + it_behaves_like 'with an invalid access token' end end end @@ -421,7 +463,7 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do ) expect(gl_auth.find_for_git_client(user.username, user.password, project: nil, ip: 'ip')) - .to eq(Gitlab::Auth::Result.new(nil, nil, nil, nil)) + .to have_attributes(auth_failure) end it 'goes through lfs authentication' do @@ -432,7 +474,7 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do ) expect(gl_auth.find_for_git_client(user.username, user.password, project: nil, ip: 'ip')) - .to eq(Gitlab::Auth::Result.new(user, nil, :gitlab_or_ldap, described_class.full_authentication_abilities)) + .to have_attributes(actor: user, project: nil, type: :gitlab_or_ldap, authentication_abilities: described_class.full_authentication_abilities) end it 'goes through oauth authentication when the username is oauth2' do @@ -443,14 +485,14 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do ) expect(gl_auth.find_for_git_client(user.username, user.password, project: nil, ip: 'ip')) - .to eq(Gitlab::Auth::Result.new(user, nil, :gitlab_or_ldap, described_class.full_authentication_abilities)) + .to have_attributes(actor: user, project: nil, type: :gitlab_or_ldap, authentication_abilities: described_class.full_authentication_abilities) end end it 'returns double nil for invalid credentials' do login = 'foo' - expect(gl_auth.find_for_git_client(login, 'bar', project: nil, ip: 'ip')).to eq(Gitlab::Auth::Result.new) + expect(gl_auth.find_for_git_client(login, 'bar', project: nil, ip: 'ip')).to have_attributes(auth_failure) end it 'throws an error suggesting user create a PAT when internal auth is disabled' do @@ -460,27 +502,25 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do end context 'while using deploy tokens' do - let(:auth_failure) { Gitlab::Auth::Result.new(nil, nil) } - shared_examples 'registry token scope' do it 'fails when login is not valid' do expect(gl_auth.find_for_git_client('random_login', deploy_token.token, project: project, ip: 'ip')) - .to eq(auth_failure) + .to have_attributes(auth_failure) end it 'fails when token is not valid' do expect(gl_auth.find_for_git_client(login, '123123', project: project, ip: 'ip')) - .to eq(auth_failure) + .to have_attributes(auth_failure) end it 'fails if token is nil' do expect(gl_auth.find_for_git_client(login, nil, project: nil, ip: 'ip')) - .to eq(auth_failure) + .to have_attributes(auth_failure) end it 'fails if token is not related to project' do expect(gl_auth.find_for_git_client(login, 'abcdef', project: nil, ip: 'ip')) - .to eq(auth_failure) + .to have_attributes(auth_failure) end it 'fails if token has been revoked' do @@ -488,7 +528,7 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do expect(deploy_token.revoked?).to be_truthy expect(gl_auth.find_for_git_client('deploy-token', deploy_token.token, project: nil, ip: 'ip')) - .to eq(auth_failure) + .to have_attributes(auth_failure) end end @@ -500,7 +540,7 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do it 'fails when login and token are valid' do expect(gl_auth.find_for_git_client(login, deploy_token.token, project: nil, ip: 'ip')) - .to eq(auth_failure) + .to have_attributes(auth_failure) end end @@ -509,7 +549,7 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do it 'fails when login and token are valid' do expect(gl_auth.find_for_git_client(login, deploy_token.token, project: project, ip: 'ip')) - .to eq(auth_failure) + .to have_attributes(auth_failure) end end end @@ -520,17 +560,17 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do let(:deploy_token) { create(:deploy_token, username: username, read_registry: false, projects: [project]) } it 'succeeds for the token' do - auth_success = Gitlab::Auth::Result.new(deploy_token, project, :deploy_token, [:download_code]) + auth_success = { actor: deploy_token, project: project, type: :deploy_token, authentication_abilities: [:download_code] } expect(gl_auth.find_for_git_client(username, deploy_token.token, project: project, ip: 'ip')) - .to eq(auth_success) + .to have_attributes(auth_success) end it 'succeeds for the user' do - auth_success = Gitlab::Auth::Result.new(user, nil, :gitlab_or_ldap, described_class.full_authentication_abilities) + auth_success = { actor: user, project: nil, type: :gitlab_or_ldap, authentication_abilities: described_class.full_authentication_abilities } expect(gl_auth.find_for_git_client(username, 'my-secret', project: project, ip: 'ip')) - .to eq(auth_success) + .to have_attributes(auth_success) end end @@ -538,16 +578,16 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do context 'and belong to the same project' do let!(:read_registry) { create(:deploy_token, username: 'deployer', read_repository: false, projects: [project]) } let!(:read_repository) { create(:deploy_token, username: read_registry.username, read_registry: false, projects: [project]) } - let(:auth_success) { Gitlab::Auth::Result.new(read_repository, project, :deploy_token, [:download_code]) } + let(:auth_success) { { actor: read_repository, project: project, type: :deploy_token, authentication_abilities: [:download_code] } } it 'succeeds for the right token' do expect(gl_auth.find_for_git_client('deployer', read_repository.token, project: project, ip: 'ip')) - .to eq(auth_success) + .to have_attributes(auth_success) end it 'fails for the wrong token' do expect(gl_auth.find_for_git_client('deployer', read_registry.token, project: project, ip: 'ip')) - .not_to eq(auth_success) + .not_to have_attributes(auth_success) end end @@ -556,16 +596,16 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do let!(:read_registry) { create(:deploy_token, username: 'deployer', read_repository: false, projects: [project]) } let!(:read_repository) { create(:deploy_token, username: read_registry.username, read_registry: false, projects: [other_project]) } - let(:auth_success) { Gitlab::Auth::Result.new(read_repository, other_project, :deploy_token, [:download_code]) } + let(:auth_success) { { actor: read_repository, project: other_project, type: :deploy_token, authentication_abilities: [:download_code] } } it 'succeeds for the right token' do expect(gl_auth.find_for_git_client('deployer', read_repository.token, project: other_project, ip: 'ip')) - .to eq(auth_success) + .to have_attributes(auth_success) end it 'fails for the wrong token' do expect(gl_auth.find_for_git_client('deployer', read_registry.token, project: other_project, ip: 'ip')) - .not_to eq(auth_success) + .not_to have_attributes(auth_success) end end end @@ -575,18 +615,18 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do let(:login) { deploy_token.username } it 'succeeds when login and token are valid' do - auth_success = Gitlab::Auth::Result.new(deploy_token, project, :deploy_token, [:download_code]) + auth_success = { actor: deploy_token, project: project, type: :deploy_token, authentication_abilities: [:download_code] } expect(gl_auth.find_for_git_client(login, deploy_token.token, project: project, ip: 'ip')) - .to eq(auth_success) + .to have_attributes(auth_success) end it 'succeeds when custom login and token are valid' do deploy_token = create(:deploy_token, username: 'deployer', read_registry: false, projects: [project]) - auth_success = Gitlab::Auth::Result.new(deploy_token, project, :deploy_token, [:download_code]) + auth_success = { actor: deploy_token, project: project, type: :deploy_token, authentication_abilities: [:download_code] } expect(gl_auth.find_for_git_client('deployer', deploy_token.token, project: project, ip: 'ip')) - .to eq(auth_success) + .to have_attributes(auth_success) end it 'does not attempt to rate limit unique IPs for a deploy token' do @@ -597,23 +637,23 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do it 'fails when login is not valid' do expect(gl_auth.find_for_git_client('random_login', deploy_token.token, project: project, ip: 'ip')) - .to eq(auth_failure) + .to have_attributes(auth_failure) end it 'fails when token is not valid' do expect(gl_auth.find_for_git_client(login, '123123', project: project, ip: 'ip')) - .to eq(auth_failure) + .to have_attributes(auth_failure) end it 'fails if token is nil' do expect(gl_auth.find_for_git_client(login, nil, project: project, ip: 'ip')) - .to eq(auth_failure) + .to have_attributes(auth_failure) end it 'fails if token is not related to project' do another_deploy_token = create(:deploy_token) expect(gl_auth.find_for_git_client(another_deploy_token.username, another_deploy_token.token, project: project, ip: 'ip')) - .to eq(auth_failure) + .to have_attributes(auth_failure) end it 'fails if token has been revoked' do @@ -621,7 +661,7 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do expect(deploy_token.revoked?).to be_truthy expect(gl_auth.find_for_git_client('deploy-token', deploy_token.token, project: project, ip: 'ip')) - .to eq(auth_failure) + .to have_attributes(auth_failure) end end @@ -633,16 +673,16 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do subject { gl_auth.find_for_git_client(login, deploy_token.token, project: project_with_group, ip: 'ip') } it 'succeeds when login and a group deploy token are valid' do - auth_success = Gitlab::Auth::Result.new(deploy_token, project_with_group, :deploy_token, [:download_code, :read_container_image]) + auth_success = { actor: deploy_token, project: project_with_group, type: :deploy_token, authentication_abilities: [:download_code, :read_container_image] } - expect(subject).to eq(auth_success) + expect(subject).to have_attributes(auth_success) end it 'fails if token is not related to group' do another_deploy_token = create(:deploy_token, :group, read_repository: true) expect(gl_auth.find_for_git_client(another_deploy_token.username, another_deploy_token.token, project: project_with_group, ip: 'ip')) - .to eq(auth_failure) + .to have_attributes(auth_failure) end end @@ -656,10 +696,10 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do end it 'succeeds when login and a project token are valid' do - auth_success = Gitlab::Auth::Result.new(deploy_token, project, :deploy_token, [:read_container_image]) + auth_success = { actor: deploy_token, project: project, type: :deploy_token, authentication_abilities: [:read_container_image] } expect(gl_auth.find_for_git_client(login, deploy_token.token, project: project, ip: 'ip')) - .to eq(auth_success) + .to have_attributes(auth_success) end it_behaves_like 'registry token scope' @@ -678,10 +718,10 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do end it 'succeeds when login and a project token are valid' do - auth_success = Gitlab::Auth::Result.new(deploy_token, project, :deploy_token, [:create_container_image]) + auth_success = { actor: deploy_token, project: project, type: :deploy_token, authentication_abilities: [:create_container_image] } expect(gl_auth.find_for_git_client(login, deploy_token.token, project: project, ip: 'ip')) - .to eq(auth_success) + .to have_attributes(auth_success) end it_behaves_like 'registry token scope' @@ -891,6 +931,6 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do def expect_results_with_abilities(personal_access_token, abilities, success = true) expect(gl_auth.find_for_git_client('', personal_access_token&.token, project: nil, ip: 'ip')) - .to eq(Gitlab::Auth::Result.new(personal_access_token&.user, nil, personal_access_token.nil? ? nil : :personal_access_token, abilities)) + .to have_attributes(actor: personal_access_token&.user, project: nil, type: personal_access_token.nil? ? nil : :personal_access_token, authentication_abilities: abilities) end end diff --git a/spec/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_spec.rb b/spec/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_spec.rb index f56cf899410..a7895623d6f 100644 --- a/spec/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_spec.rb @@ -23,6 +23,8 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillDraftStatusOnMergeRequests d end context "for MRs with #draft? == true titles but draft attribute false" do + let(:mr_ids) { merge_requests.all.collect(&:id) } + before do draft_prefixes.each do |prefix| (1..4).each do |n| @@ -37,11 +39,16 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillDraftStatusOnMergeRequests d it "updates all open draft merge request's draft field to true" do mr_count = merge_requests.all.count - mr_ids = merge_requests.all.collect(&:id) expect { subject.perform(mr_ids.first, mr_ids.last) } .to change { MergeRequest.where(draft: false).count } .from(mr_count).to(mr_count - draft_prefixes.length) end + + it "marks successful slices as completed" do + expect(subject).to receive(:mark_job_as_succeeded).with(mr_ids.first, mr_ids.last) + + subject.perform(mr_ids.first, mr_ids.last) + end end end diff --git a/spec/lib/gitlab/background_migration/backfill_integrations_type_new_spec.rb b/spec/lib/gitlab/background_migration/backfill_integrations_type_new_spec.rb new file mode 100644 index 00000000000..8f765a7a536 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_integrations_type_new_spec.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillIntegrationsTypeNew do + let(:migration) { described_class.new } + let(:integrations) { table(:integrations) } + let(:namespaced_integrations) { Gitlab::Integrations::StiType.namespaced_integrations } + + before do + integrations.connection.execute 'ALTER TABLE integrations DISABLE TRIGGER "trigger_type_new_on_insert"' + + namespaced_integrations.each_with_index do |type, i| + integrations.create!(id: i + 1, type: "#{type}Service") + end + + integrations.create!(id: namespaced_integrations.size + 1, type: 'LegacyService') + ensure + integrations.connection.execute 'ALTER TABLE integrations ENABLE TRIGGER "trigger_type_new_on_insert"' + end + + it 'backfills `type_new` for the selected records' do + # We don't want to mock `Kernel.sleep`, so instead we mock it on the migration + # class before it gets forwarded. + expect(migration).to receive(:sleep).with(0.05).exactly(5).times + + queries = ActiveRecord::QueryRecorder.new do + migration.perform(2, 10, :integrations, :id, 2, 50) + end + + expect(queries.count).to be(16) + expect(queries.log.grep(/^SELECT/).size).to be(11) + expect(queries.log.grep(/^UPDATE/).size).to be(5) + expect(queries.log.grep(/^UPDATE/).join.scan(/WHERE .*/)).to eq([ + 'WHERE integrations.id BETWEEN 2 AND 3', + 'WHERE integrations.id BETWEEN 4 AND 5', + 'WHERE integrations.id BETWEEN 6 AND 7', + 'WHERE integrations.id BETWEEN 8 AND 9', + 'WHERE integrations.id BETWEEN 10 AND 10' + ]) + + expect(integrations.where(id: 2..10).pluck(:type, :type_new)).to contain_exactly( + ['AssemblaService', 'Integrations::Assembla'], + ['BambooService', 'Integrations::Bamboo'], + ['BugzillaService', 'Integrations::Bugzilla'], + ['BuildkiteService', 'Integrations::Buildkite'], + ['CampfireService', 'Integrations::Campfire'], + ['ConfluenceService', 'Integrations::Confluence'], + ['CustomIssueTrackerService', 'Integrations::CustomIssueTracker'], + ['DatadogService', 'Integrations::Datadog'], + ['DiscordService', 'Integrations::Discord'] + ) + + expect(integrations.where.not(id: 2..10)).to all(have_attributes(type_new: nil)) + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb b/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb index dbf74bd9333..d22aa86dbe0 100644 --- a/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb @@ -304,7 +304,7 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillSnippetRepositories, :migrat end def blob_at(snippet, path) - raw_repository(snippet).blob_at('master', path) + raw_repository(snippet).blob_at('main', path) end def repository_exists?(snippet) diff --git a/spec/lib/gitlab/background_migration/copy_ci_builds_columns_to_security_scans_spec.rb b/spec/lib/gitlab/background_migration/copy_ci_builds_columns_to_security_scans_spec.rb new file mode 100644 index 00000000000..db822f36c21 --- /dev/null +++ b/spec/lib/gitlab/background_migration/copy_ci_builds_columns_to_security_scans_spec.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::CopyCiBuildsColumnsToSecurityScans, schema: 20210728174349 do + let(:migration) { described_class.new } + + let_it_be(:namespaces) { table(:namespaces) } + let_it_be(:projects) { table(:projects) } + let_it_be(:ci_pipelines) { table(:ci_pipelines) } + let_it_be(:ci_builds) { table(:ci_builds) } + let_it_be(:security_scans) { table(:security_scans) } + + let!(:namespace) { namespaces.create!(name: 'namespace', path: 'namespace') } + let!(:project1) { projects.create!(namespace_id: namespace.id) } + let!(:project2) { projects.create!(namespace_id: namespace.id) } + let!(:pipeline1) { ci_pipelines.create!(status: "success")} + let!(:pipeline2) { ci_pipelines.create!(status: "success")} + + let!(:build1) { ci_builds.create!(commit_id: pipeline1.id, type: 'Ci::Build', project_id: project1.id) } + let!(:build2) { ci_builds.create!(commit_id: pipeline2.id, type: 'Ci::Build', project_id: project2.id) } + let!(:build3) { ci_builds.create!(commit_id: pipeline1.id, type: 'Ci::Build', project_id: project1.id) } + + let!(:scan1) { security_scans.create!(build_id: build1.id, scan_type: 1) } + let!(:scan2) { security_scans.create!(build_id: build2.id, scan_type: 1) } + let!(:scan3) { security_scans.create!(build_id: build3.id, scan_type: 1) } + + subject { migration.perform(scan1.id, scan2.id) } + + before do + stub_const("#{described_class}::UPDATE_BATCH_SIZE", 2) + end + + it 'copies `project_id`, `commit_id` from `ci_builds` to `security_scans`', :aggregate_failures do + expect(migration).to receive(:mark_job_as_succeeded).with(scan1.id, scan2.id) + + subject + + scan1.reload + expect(scan1.project_id).to eq(project1.id) + expect(scan1.pipeline_id).to eq(pipeline1.id) + + scan2.reload + expect(scan2.project_id).to eq(project2.id) + expect(scan2.pipeline_id).to eq(pipeline2.id) + + scan3.reload + expect(scan3.project_id).to be_nil + expect(scan3.pipeline_id).to be_nil + end +end diff --git a/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb b/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb index 70906961641..30908145782 100644 --- a/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb +++ b/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb @@ -13,12 +13,13 @@ RSpec.describe Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrence let(:vulnerabilities) { table(:vulnerabilities) } let(:vulnerabilities_findings) { table(:vulnerability_occurrences) } let(:vulnerability_identifiers) { table(:vulnerability_identifiers) } + let(:vulnerability_identifier) do vulnerability_identifiers.create!( project_id: project.id, external_type: 'uuid-v5', external_id: 'uuid-v5', - fingerprint: '7e394d1b1eb461a7406d7b1e08f057a1cf11287a', + fingerprint: Gitlab::Database::ShaAttribute.serialize('7e394d1b1eb461a7406d7b1e08f057a1cf11287a'), name: 'Identifier for UUIDv5') end @@ -27,7 +28,7 @@ RSpec.describe Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrence project_id: project.id, external_type: 'uuid-v4', external_id: 'uuid-v4', - fingerprint: '772da93d34a1ba010bcb5efa9fb6f8e01bafcc89', + fingerprint: Gitlab::Database::ShaAttribute.serialize('772da93d34a1ba010bcb5efa9fb6f8e01bafcc89'), name: 'Identifier for UUIDv4') end @@ -59,7 +60,7 @@ RSpec.describe Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrence scanner_id: different_scanner.id, primary_identifier_id: different_vulnerability_identifier.id, report_type: 0, # "sast" - location_fingerprint: "fa18f432f1d56675f4098d318739c3cd5b14eb3e", + location_fingerprint: Gitlab::Database::ShaAttribute.serialize("fa18f432f1d56675f4098d318739c3cd5b14eb3e"), uuid: known_uuid_v4 ) end @@ -91,7 +92,7 @@ RSpec.describe Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrence scanner_id: scanner.id, primary_identifier_id: vulnerability_identifier.id, report_type: 0, # "sast" - location_fingerprint: "838574be0210968bf6b9f569df9c2576242cbf0a", + location_fingerprint: Gitlab::Database::ShaAttribute.serialize("838574be0210968bf6b9f569df9c2576242cbf0a"), uuid: known_uuid_v5 ) end @@ -115,7 +116,7 @@ RSpec.describe Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrence scanner_id: different_scanner.id, primary_identifier_id: different_vulnerability_identifier.id, report_type: 0, # "sast" - location_fingerprint: "fa18f432f1d56675f4098d318739c3cd5b14eb3e", + location_fingerprint: Gitlab::Database::ShaAttribute.serialize("fa18f432f1d56675f4098d318739c3cd5b14eb3e"), uuid: known_uuid_v4 ) diff --git a/spec/lib/gitlab/background_migration_spec.rb b/spec/lib/gitlab/background_migration_spec.rb index 5b20572578c..f32e6891716 100644 --- a/spec/lib/gitlab/background_migration_spec.rb +++ b/spec/lib/gitlab/background_migration_spec.rb @@ -13,7 +13,11 @@ RSpec.describe Gitlab::BackgroundMigration do describe '.steal' do context 'when there are enqueued jobs present' do let(:queue) do - [double(args: ['Foo', [10, 20]], queue: described_class.queue)] + [ + double(args: ['Foo', [10, 20]], klass: 'BackgroundMigrationWorker'), + double(args: ['Bar', [20, 30]], klass: 'BackgroundMigrationWorker'), + double(args: ['Foo', [20, 30]], klass: 'MergeWorker') + ] end before do @@ -45,7 +49,7 @@ RSpec.describe Gitlab::BackgroundMigration do expect(queue[0]).not_to receive(:delete) - described_class.steal('Bar') + described_class.steal('Baz') end context 'when a custom predicate is given' do @@ -72,8 +76,8 @@ RSpec.describe Gitlab::BackgroundMigration do let(:migration) { spy(:migration) } let(:queue) do - [double(args: ['Foo', [10, 20]], queue: described_class.queue), - double(args: ['Foo', [20, 30]], queue: described_class.queue)] + [double(args: ['Foo', [10, 20]], klass: 'BackgroundMigrationWorker'), + double(args: ['Foo', [20, 30]], klass: 'BackgroundMigrationWorker')] end before do @@ -128,11 +132,11 @@ RSpec.describe Gitlab::BackgroundMigration do context 'when retry_dead_jobs is true', :redis do let(:retry_queue) do - [double(args: ['Object', [3]], queue: described_class.queue, delete: true)] + [double(args: ['Object', [3]], klass: 'BackgroundMigrationWorker', delete: true)] end let(:dead_queue) do - [double(args: ['Object', [4]], queue: described_class.queue, delete: true)] + [double(args: ['Object', [4]], klass: 'BackgroundMigrationWorker', delete: true)] end before do @@ -187,20 +191,22 @@ RSpec.describe Gitlab::BackgroundMigration do describe '.remaining', :redis do context 'when there are jobs remaining' do - let(:queue) { Array.new(12) } - before do - allow(Sidekiq::Queue).to receive(:new) - .with(described_class.queue) - .and_return(Array.new(12)) - Sidekiq::Testing.disable! do - BackgroundMigrationWorker.perform_in(10.minutes, 'Foo') + MergeWorker.perform_async('Foo') + MergeWorker.perform_in(10.minutes, 'Foo') + + 5.times do + BackgroundMigrationWorker.perform_async('Foo') + end + 3.times do + BackgroundMigrationWorker.perform_in(10.minutes, 'Foo') + end end end it 'returns the enqueued jobs plus the scheduled jobs' do - expect(described_class.remaining).to eq(13) + expect(described_class.remaining).to eq(8) end end @@ -211,16 +217,13 @@ RSpec.describe Gitlab::BackgroundMigration do end end - describe '.exists?' do + describe '.exists?', :redis do context 'when there are enqueued jobs present' do - let(:queue) do - [double(args: ['Foo', [10, 20]], queue: described_class.queue)] - end - before do - allow(Sidekiq::Queue).to receive(:new) - .with(described_class.queue) - .and_return(queue) + Sidekiq::Testing.disable! do + MergeWorker.perform_async('Bar') + BackgroundMigrationWorker.perform_async('Foo') + end end it 'returns true if specific job exists' do @@ -232,19 +235,14 @@ RSpec.describe Gitlab::BackgroundMigration do end end - context 'when there are scheduled jobs present', :redis do + context 'when there are scheduled jobs present' do before do Sidekiq::Testing.disable! do + MergeWorker.perform_in(10.minutes, 'Bar') BackgroundMigrationWorker.perform_in(10.minutes, 'Foo') - - expect(Sidekiq::ScheduledSet.new).to be_one end end - after do - Sidekiq::ScheduledSet.new.clear - end - it 'returns true if specific job exists' do expect(described_class.exists?('Foo')).to eq(true) end @@ -257,7 +255,10 @@ RSpec.describe Gitlab::BackgroundMigration do describe '.dead_jobs?' do let(:queue) do - [double(args: ['Foo', [10, 20]], queue: described_class.queue)] + [ + double(args: ['Foo', [10, 20]], klass: 'BackgroundMigrationWorker'), + double(args: ['Bar'], klass: 'MergeWorker') + ] end context 'when there are dead jobs present' do @@ -277,7 +278,10 @@ RSpec.describe Gitlab::BackgroundMigration do describe '.retrying_jobs?' do let(:queue) do - [double(args: ['Foo', [10, 20]], queue: described_class.queue)] + [ + double(args: ['Foo', [10, 20]], klass: 'BackgroundMigrationWorker'), + double(args: ['Bar'], klass: 'MergeWorker') + ] end context 'when there are dead jobs present' do diff --git a/spec/lib/gitlab/bitbucket_server_import/importer_spec.rb b/spec/lib/gitlab/bitbucket_server_import/importer_spec.rb index c9ad78ec760..4e4d921d67f 100644 --- a/spec/lib/gitlab/bitbucket_server_import/importer_spec.rb +++ b/spec/lib/gitlab/bitbucket_server_import/importer_spec.rb @@ -32,8 +32,7 @@ RSpec.describe Gitlab::BitbucketServerImport::Importer do expect(subject).to receive(:delete_temp_branches) expect(project.repository).to receive(:fetch_as_mirror) .with('http://bitbucket:test@my-bitbucket', - refmap: [:heads, :tags, '+refs/pull-requests/*/to:refs/merge-requests/*/head'], - remote_name: 'bitbucket_server') + refmap: [:heads, :tags, '+refs/pull-requests/*/to:refs/merge-requests/*/head']) subject.execute end diff --git a/spec/lib/gitlab/checks/changes_access_spec.rb b/spec/lib/gitlab/checks/changes_access_spec.rb index a46732f8255..4a74dfcec34 100644 --- a/spec/lib/gitlab/checks/changes_access_spec.rb +++ b/spec/lib/gitlab/checks/changes_access_spec.rb @@ -3,40 +3,199 @@ require 'spec_helper' RSpec.describe Gitlab::Checks::ChangesAccess do + include_context 'changes access checks context' + + subject { changes_access } + describe '#validate!' do - include_context 'changes access checks context' + shared_examples '#validate!' do + before do + allow(project).to receive(:lfs_enabled?).and_return(true) + end - before do - allow(project).to receive(:lfs_enabled?).and_return(true) - end + context 'without failed checks' do + it "doesn't raise an error" do + expect { subject.validate! }.not_to raise_error + end - subject { changes_access } + it 'calls lfs checks' do + expect_next_instance_of(Gitlab::Checks::LfsCheck) do |instance| + expect(instance).to receive(:validate!) + end - context 'without failed checks' do - it "doesn't raise an error" do - expect { subject.validate! }.not_to raise_error + subject.validate! + end end - it 'calls lfs checks' do - expect_next_instance_of(Gitlab::Checks::LfsCheck) do |instance| - expect(instance).to receive(:validate!) + context 'when time limit was reached' do + it 'raises a TimeoutError' do + logger = Gitlab::Checks::TimedLogger.new(start_time: timeout.ago, timeout: timeout) + access = described_class.new(changes, + project: project, + user_access: user_access, + protocol: protocol, + logger: logger) + + expect { access.validate! }.to raise_error(Gitlab::Checks::TimedLogger::TimeoutError) end + end + end + + context 'with batched commits enabled' do + before do + stub_feature_flags(changes_batch_commits: true) + end + + it_behaves_like '#validate!' + end + + context 'with batched commits disabled' do + before do + stub_feature_flags(changes_batch_commits: false) + end + + it_behaves_like '#validate!' + end + end + + describe '#commits' do + it 'calls #new_commits' do + expect(project.repository).to receive(:new_commits).and_call_original + + expect(subject.commits).to eq([]) + end + + context 'when changes contain empty revisions' do + let(:changes) { [{ newrev: newrev }, { newrev: '' }, { newrev: Gitlab::Git::BLANK_SHA }] } + let(:expected_commit) { instance_double(Commit) } + + it 'returns only commits with non empty revisions' do + expect(project.repository).to receive(:new_commits).with([newrev], { allow_quarantine: true }) { [expected_commit] } + expect(subject.commits).to eq([expected_commit]) + end + end + end + + describe '#commits_for' do + let(:new_commits) { [] } + let(:expected_commits) { [] } + + shared_examples 'a listing of new commits' do + it 'returns expected commits' do + expect(subject).to receive(:commits).and_return(new_commits) + + expect(subject.commits_for(newrev)).to eq(expected_commits) + end + end + + context 'with no commits' do + it_behaves_like 'a listing of new commits' + end + + context 'with unrelated commits' do + let(:new_commits) { [create_commit('1234', %w[1111 2222])] } + + it_behaves_like 'a listing of new commits' + end + + context 'with single related commit' do + let(:new_commits) { [create_commit(newrev, %w[1111 2222])] } + let(:expected_commits) { new_commits } - subject.validate! + it_behaves_like 'a listing of new commits' + end + + context 'with single related and unrelated commit' do + let(:new_commits) do + [ + create_commit(newrev, %w[1111 2222]), + create_commit('abcd', %w[1111 2222]) + ] + end + + let(:expected_commits) do + [create_commit(newrev, %w[1111 2222])] end + + it_behaves_like 'a listing of new commits' end - context 'when time limit was reached' do - it 'raises a TimeoutError' do - logger = Gitlab::Checks::TimedLogger.new(start_time: timeout.ago, timeout: timeout) - access = described_class.new(changes, - project: project, - user_access: user_access, - protocol: protocol, - logger: logger) + context 'with multiple related commits' do + let(:new_commits) do + [ + create_commit(newrev, %w[1111]), + create_commit('1111', %w[2222]), + create_commit('abcd', []) + ] + end - expect { access.validate! }.to raise_error(Gitlab::Checks::TimedLogger::TimeoutError) + let(:expected_commits) do + [ + create_commit(newrev, %w[1111]), + create_commit('1111', %w[2222]) + ] end + + it_behaves_like 'a listing of new commits' end + + context 'with merge commits' do + let(:new_commits) do + [ + create_commit(newrev, %w[1111 2222 3333]), + create_commit('1111', []), + create_commit('3333', %w[4444]), + create_commit('4444', []) + ] + end + + let(:expected_commits) do + [ + create_commit(newrev, %w[1111 2222 3333]), + create_commit('1111', []), + create_commit('3333', %w[4444]), + create_commit('4444', []) + ] + end + + it_behaves_like 'a listing of new commits' + end + + context 'with criss-cross merges' do + let(:new_commits) do + [ + create_commit(newrev, %w[a1 b1]), + create_commit('a1', %w[a2 b2]), + create_commit('a2', %w[a3 b3]), + create_commit('a3', %w[c]), + create_commit('b1', %w[b2 a2]), + create_commit('b2', %w[b3 a3]), + create_commit('b3', %w[c]), + create_commit('c', []) + ] + end + + let(:expected_commits) do + [ + create_commit(newrev, %w[a1 b1]), + create_commit('a1', %w[a2 b2]), + create_commit('b1', %w[b2 a2]), + create_commit('a2', %w[a3 b3]), + create_commit('b2', %w[b3 a3]), + create_commit('a3', %w[c]), + create_commit('b3', %w[c]), + create_commit('c', []) + ] + end + + it_behaves_like 'a listing of new commits' + end + end + + def create_commit(id, parent_ids) + Gitlab::Git::Commit.new(project.repository, { + id: id, + parent_ids: parent_ids + }) end end diff --git a/spec/lib/gitlab/checks/matching_merge_request_spec.rb b/spec/lib/gitlab/checks/matching_merge_request_spec.rb index feda488a936..2e562a5a350 100644 --- a/spec/lib/gitlab/checks/matching_merge_request_spec.rb +++ b/spec/lib/gitlab/checks/matching_merge_request_spec.rb @@ -49,12 +49,11 @@ RSpec.describe Gitlab::Checks::MatchingMergeRequest do end end - context 'with load balancing enabled', :request_store, :redis do + context 'with load balancing enabled', :db_load_balancing do let(:session) { ::Gitlab::Database::LoadBalancing::Session.current } let(:all_caught_up) { true } before do - expect(::Gitlab::Database::LoadBalancing).to receive(:enable?).at_least(:once).and_return(true) allow(::Gitlab::Database::LoadBalancing::Sticking).to receive(:all_caught_up?).and_return(all_caught_up) expect(::Gitlab::Database::LoadBalancing::Sticking).to receive(:select_valid_host).with(:project, project.id).and_call_original diff --git a/spec/lib/gitlab/checks/single_change_access_spec.rb b/spec/lib/gitlab/checks/single_change_access_spec.rb index 8b235005b3e..e81e4951539 100644 --- a/spec/lib/gitlab/checks/single_change_access_spec.rb +++ b/spec/lib/gitlab/checks/single_change_access_spec.rb @@ -58,5 +58,52 @@ RSpec.describe Gitlab::Checks::SingleChangeAccess do expect { access.validate! }.to raise_error(Gitlab::Checks::TimedLogger::TimeoutError) end end + + describe '#commits' do + let(:expected_commits) { [Gitlab::Git::Commit.new(project.repository, { id: "1234" })] } + + let(:access) do + described_class.new(changes, + project: project, + user_access: user_access, + protocol: protocol, + logger: logger, + commits: provided_commits) + end + + shared_examples '#commits' do + it 'returns expected commits' do + expect(access.commits).to eq(expected_commits) + end + + it 'returns expected commits on repeated calls' do + expect(access.commits).to eq(expected_commits) + expect(access.commits).to eq(expected_commits) + end + end + + context 'with provided commits' do + let(:provided_commits) { expected_commits } + + before do + expect(project.repository).not_to receive(:new_commits) + end + + it_behaves_like '#commits' + end + + context 'without provided commits' do + let(:provided_commits) { nil } + + before do + expect(project.repository) + .to receive(:new_commits) + .once + .and_return(expected_commits) + end + + it_behaves_like '#commits' + end + end end end diff --git a/spec/lib/gitlab/ci/ansi2html_spec.rb b/spec/lib/gitlab/ci/ansi2html_spec.rb index bf1f2bae7da..27c2b005a93 100644 --- a/spec/lib/gitlab/ci/ansi2html_spec.rb +++ b/spec/lib/gitlab/ci/ansi2html_spec.rb @@ -150,6 +150,10 @@ RSpec.describe Gitlab::Ci::Ansi2html do expect(convert_html("\r\n")).to eq('<span><br/></span>') end + it 'replaces invalid UTF-8 data' do + expect(convert_html("UTF-8 dashes here: ───\n🐤🐤🐤🐤\xF0\x9F\x90\n")).to eq("<span>UTF-8 dashes here: ───<br/>🐤🐤🐤🐤�<br/></span>") + end + describe "incremental update" do shared_examples 'stateable converter' do let(:pass1_stream) { StringIO.new(pre_text) } diff --git a/spec/lib/gitlab/ci/build/auto_retry_spec.rb b/spec/lib/gitlab/ci/build/auto_retry_spec.rb index b107553bbce..e83e1326206 100644 --- a/spec/lib/gitlab/ci/build/auto_retry_spec.rb +++ b/spec/lib/gitlab/ci/build/auto_retry_spec.rb @@ -53,24 +53,8 @@ RSpec.describe Gitlab::Ci::Build::AutoRetry do context 'with retries max config option' do let(:build) { create(:ci_build, options: { retry: { max: 1 } }) } - context 'when build_metadata_config is set' do - before do - stub_feature_flags(ci_build_metadata_config: true) - end - - it 'returns the number of configured max retries' do - expect(result).to eq 1 - end - end - - context 'when build_metadata_config is not set' do - before do - stub_feature_flags(ci_build_metadata_config: false) - end - - it 'returns the number of configured max retries' do - expect(result).to eq 1 - end + it 'returns the number of configured max retries' do + expect(result).to eq 1 end end diff --git a/spec/lib/gitlab/ci/config/entry/bridge_spec.rb b/spec/lib/gitlab/ci/config/entry/bridge_spec.rb index d294eca7f15..6c9c8fa5df5 100644 --- a/spec/lib/gitlab/ci/config/entry/bridge_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/bridge_spec.rb @@ -106,7 +106,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Bridge do ignore: false, stage: 'test', only: { refs: %w[branches tags] }, - variables: {}, job_variables: {}, root_variables_inheritance: true, scheduling_type: :stage) @@ -131,7 +130,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Bridge do ignore: false, stage: 'test', only: { refs: %w[branches tags] }, - variables: {}, job_variables: {}, root_variables_inheritance: true, scheduling_type: :stage) @@ -287,7 +285,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Bridge do only: { refs: %w[branches tags] }, parallel: { matrix: [{ 'PROVIDER' => ['aws'], 'STACK' => %w(monitoring app1) }, { 'PROVIDER' => ['gcp'], 'STACK' => %w(data) }] }, - variables: {}, job_variables: {}, root_variables_inheritance: true, scheduling_type: :stage diff --git a/spec/lib/gitlab/ci/config/entry/include/rules/rule_spec.rb b/spec/lib/gitlab/ci/config/entry/include/rules/rule_spec.rb new file mode 100644 index 00000000000..b99048e2c18 --- /dev/null +++ b/spec/lib/gitlab/ci/config/entry/include/rules/rule_spec.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Ci::Config::Entry::Include::Rules::Rule do + let(:factory) do + Gitlab::Config::Entry::Factory.new(described_class) + .value(config) + end + + subject(:entry) { factory.create! } + + describe '.new' do + shared_examples 'an invalid config' do |error_message| + it { is_expected.not_to be_valid } + + it 'has errors' do + expect(entry.errors).to include(error_message) + end + end + + context 'when specifying an if: clause' do + let(:config) { { if: '$THIS || $THAT' } } + + it { is_expected.to be_valid } + end + + context 'using a list of multiple expressions' do + let(:config) { { if: ['$MY_VAR == "this"', '$YOUR_VAR == "that"'] } } + + it_behaves_like 'an invalid config', /invalid expression syntax/ + end + + context 'when specifying an invalid if: clause expression' do + let(:config) { { if: ['$MY_VAR =='] } } + + it_behaves_like 'an invalid config', /invalid expression syntax/ + end + + context 'when specifying an if: clause expression with an invalid token' do + let(:config) { { if: ['$MY_VAR == 123'] } } + + it_behaves_like 'an invalid config', /invalid expression syntax/ + end + + context 'when using invalid regex in an if: clause' do + let(:config) { { if: ['$MY_VAR =~ /some ( thing/'] } } + + it_behaves_like 'an invalid config', /invalid expression syntax/ + end + + context 'when using an if: clause with lookahead regex character "?"' do + let(:config) { { if: '$CI_COMMIT_REF =~ /^(?!master).+/' } } + + context 'when allow_unsafe_ruby_regexp is disabled' do + it_behaves_like 'an invalid config', /invalid expression syntax/ + end + end + + context 'when specifying unknown policy' do + let(:config) { { invalid: :something } } + + it_behaves_like 'an invalid config', /unknown keys: invalid/ + end + + context 'when clause is empty' do + let(:config) { {} } + + it_behaves_like 'an invalid config', /can't be blank/ + end + + context 'when policy strategy does not match' do + let(:config) { 'string strategy' } + + it_behaves_like 'an invalid config', /should be a hash/ + end + end + + describe '#value' do + subject(:value) { entry.value } + + context 'when specifying an if: clause' do + let(:config) { { if: '$THIS || $THAT' } } + + it 'returns the config' do + expect(subject).to eq(if: '$THIS || $THAT') + end + end + end +end diff --git a/spec/lib/gitlab/ci/config/entry/include/rules_spec.rb b/spec/lib/gitlab/ci/config/entry/include/rules_spec.rb new file mode 100644 index 00000000000..c255d6e9dd6 --- /dev/null +++ b/spec/lib/gitlab/ci/config/entry/include/rules_spec.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Ci::Config::Entry::Include::Rules do + let(:factory) do + Gitlab::Config::Entry::Factory.new(described_class) + .value(config) + end + + subject(:entry) { factory.create! } + + describe '.new' do + shared_examples 'a valid config' do + it { is_expected.to be_valid } + + context 'when composed' do + before do + entry.compose! + end + + it { is_expected.to be_valid } + end + end + + shared_examples 'an invalid config' do |error_message| + it { is_expected.not_to be_valid } + + it 'has errors' do + expect(entry.errors).to include(error_message) + end + end + + context 'with an "if"' do + let(:config) do + [{ if: '$THIS == "that"' }] + end + + it_behaves_like 'a valid config' + end + + context 'with a "changes"' do + let(:config) do + [{ changes: ['filename.txt'] }] + end + + context 'when composed' do + before do + entry.compose! + end + + it_behaves_like 'an invalid config', /contains unknown keys: changes/ + end + end + + context 'with a list of two rules' do + let(:config) do + [ + { if: '$THIS == "that"' }, + { if: '$SKIP' } + ] + end + + it_behaves_like 'a valid config' + end + + context 'without an array' do + let(:config) do + { if: '$SKIP' } + end + + it_behaves_like 'an invalid config', /should be a array/ + end + end + + describe '#value' do + subject(:value) { entry.value } + + context 'with an "if"' do + let(:config) do + [{ if: '$THIS == "that"' }] + end + + it { is_expected.to eq(config) } + end + + context 'with a list of two rules' do + let(:config) do + [ + { if: '$THIS == "that"' }, + { if: '$SKIP' } + ] + end + + it { is_expected.to eq(config) } + end + end +end diff --git a/spec/lib/gitlab/ci/config/entry/include_spec.rb b/spec/lib/gitlab/ci/config/entry/include_spec.rb index 59f0b0e7a48..275cdcddeb0 100644 --- a/spec/lib/gitlab/ci/config/entry/include_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/include_spec.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require 'spec_helper' +require 'fast_spec_helper' RSpec.describe ::Gitlab::Ci::Config::Entry::Include do subject(:include_entry) { described_class.new(config) } @@ -86,6 +86,22 @@ RSpec.describe ::Gitlab::Ci::Config::Entry::Include do end end end + + context 'when using with "rules"' do + let(:config) { { local: 'test.yml', rules: [{ if: '$VARIABLE' }] } } + + it { is_expected.to be_valid } + + context 'when rules is not an array of hashes' do + let(:config) { { local: 'test.yml', rules: ['$VARIABLE'] } } + + it { is_expected.not_to be_valid } + + it 'has specific error' do + expect(include_entry.errors).to include('include rules should be an array of hashes') + end + end + end end context 'when value is something else' do @@ -94,4 +110,26 @@ RSpec.describe ::Gitlab::Ci::Config::Entry::Include do it { is_expected.not_to be_valid } end end + + describe '#value' do + subject(:value) { include_entry.value } + + context 'when config is a string' do + let(:config) { 'test.yml' } + + it { is_expected.to eq('test.yml') } + end + + context 'when config is a hash' do + let(:config) { { local: 'test.yml' } } + + it { is_expected.to eq(local: 'test.yml') } + end + + context 'when config has "rules"' do + let(:config) { { local: 'test.yml', rules: [{ if: '$VARIABLE' }] } } + + it { is_expected.to eq(local: 'test.yml', rules: [{ if: '$VARIABLE' }]) } + end + end end diff --git a/spec/lib/gitlab/ci/config/entry/inherit/variables_spec.rb b/spec/lib/gitlab/ci/config/entry/inherit/variables_spec.rb index b1a8fbcdbe0..bdb4d25c142 100644 --- a/spec/lib/gitlab/ci/config/entry/inherit/variables_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/inherit/variables_spec.rb @@ -24,19 +24,4 @@ RSpec.describe ::Gitlab::Ci::Config::Entry::Inherit::Variables do end end end - - describe '#inherit?' do - where(:config, :inherit) do - true | true - false | false - %w[A] | true - %w[B] | false - end - - with_them do - it do - expect(subject.inherit?('A')).to eq(inherit) - end - end - end end diff --git a/spec/lib/gitlab/ci/config/entry/job_spec.rb b/spec/lib/gitlab/ci/config/entry/job_spec.rb index 1d23ab0c2c7..5b47d3a3922 100644 --- a/spec/lib/gitlab/ci/config/entry/job_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/job_spec.rb @@ -434,20 +434,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job do expect(entry.errors).to include 'job dependencies the another-job should be part of needs' end end - - context 'when stage: is missing' do - let(:config) do - { - script: 'echo', - needs: ['build-job'] - } - end - - it 'returns error about invalid data' do - expect(entry).not_to be_valid - expect(entry.errors).to include 'job config missing required keys: stage' - end - end end context 'when timeout value is not correct' do @@ -626,7 +612,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job do ignore: false, after_script: %w[cleanup], only: { refs: %w[branches tags] }, - variables: {}, job_variables: {}, root_variables_inheritance: true, scheduling_type: :stage) diff --git a/spec/lib/gitlab/ci/config/entry/jobs_spec.rb b/spec/lib/gitlab/ci/config/entry/jobs_spec.rb index cb73044b62b..9a2a67389fc 100644 --- a/spec/lib/gitlab/ci/config/entry/jobs_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/jobs_spec.rb @@ -99,7 +99,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Jobs do only: { refs: %w[branches tags] }, stage: 'test', trigger: { project: 'my/project' }, - variables: {}, job_variables: {}, root_variables_inheritance: true, scheduling_type: :stage @@ -110,7 +109,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Jobs do only: { refs: %w[branches tags] }, script: ['something'], stage: 'test', - variables: {}, job_variables: {}, root_variables_inheritance: true, scheduling_type: :stage diff --git a/spec/lib/gitlab/ci/config/entry/processable_spec.rb b/spec/lib/gitlab/ci/config/entry/processable_spec.rb index f98a6a869d6..b872f6644a2 100644 --- a/spec/lib/gitlab/ci/config/entry/processable_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/processable_spec.rb @@ -362,76 +362,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Processable do end context 'with inheritance' do - context 'of variables' do - let(:config) do - { variables: { A: 'job', B: 'job' } } - end - - before do - entry.compose!(deps) - end - - context 'with only job variables' do - it 'does return defined variables' do - expect(entry.value).to include( - variables: { 'A' => 'job', 'B' => 'job' }, - job_variables: { 'A' => 'job', 'B' => 'job' }, - root_variables_inheritance: true - ) - end - end - - context 'when root yaml variables are used' do - let(:variables) do - Gitlab::Ci::Config::Entry::Variables.new( - { A: 'root', C: 'root', D: 'root' } - ).value - end - - it 'does return job and root variables' do - expect(entry.value).to include( - variables: { 'A' => 'job', 'B' => 'job', 'C' => 'root', 'D' => 'root' }, - job_variables: { 'A' => 'job', 'B' => 'job' }, - root_variables_inheritance: true - ) - end - - context 'when inherit of defaults is disabled' do - let(:config) do - { - variables: { A: 'job', B: 'job' }, - inherit: { variables: false } - } - end - - it 'does return job and root variables' do - expect(entry.value).to include( - variables: { 'A' => 'job', 'B' => 'job' }, - job_variables: { 'A' => 'job', 'B' => 'job' }, - root_variables_inheritance: false - ) - end - end - - context 'when inherit of only specific variable is enabled' do - let(:config) do - { - variables: { A: 'job', B: 'job' }, - inherit: { variables: ['D'] } - } - end - - it 'does return job and root variables' do - expect(entry.value).to include( - variables: { 'A' => 'job', 'B' => 'job', 'D' => 'root' }, - job_variables: { 'A' => 'job', 'B' => 'job' }, - root_variables_inheritance: ['D'] - ) - end - end - end - end - context 'of default:tags' do using RSpec::Parameterized::TableSyntax @@ -493,7 +423,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Processable do name: :rspec, stage: 'test', only: { refs: %w[branches tags] }, - variables: {}, job_variables: {}, root_variables_inheritance: true ) diff --git a/spec/lib/gitlab/ci/config/entry/root_spec.rb b/spec/lib/gitlab/ci/config/entry/root_spec.rb index 31e3545e8d8..d862fbf5b78 100644 --- a/spec/lib/gitlab/ci/config/entry/root_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/root_spec.rb @@ -132,7 +132,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Root do services: [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }], stage: 'test', cache: [{ key: 'k', untracked: true, paths: ['public/'], policy: 'pull-push', when: 'on_success' }], - variables: { 'VAR' => 'root', 'VAR2' => 'val 2' }, job_variables: {}, root_variables_inheritance: true, ignore: false, @@ -148,7 +147,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Root do services: [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }], stage: 'test', cache: [{ key: 'k', untracked: true, paths: ['public/'], policy: 'pull-push', when: 'on_success' }], - variables: { 'VAR' => 'root', 'VAR2' => 'val 2' }, job_variables: {}, root_variables_inheritance: true, ignore: false, @@ -166,7 +164,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Root do services: [{ name: "postgres:9.1" }, { name: "mysql:5.5" }], cache: [{ key: "k", untracked: true, paths: ["public/"], policy: "pull-push", when: 'on_success' }], only: { refs: %w(branches tags) }, - variables: { 'VAR' => 'job', 'VAR2' => 'val 2' }, job_variables: { 'VAR' => 'job' }, root_variables_inheritance: true, after_script: [], @@ -214,7 +211,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Root do services: [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }], stage: 'test', cache: [{ key: 'k', untracked: true, paths: ['public/'], policy: 'pull-push', when: 'on_success' }], - variables: { 'VAR' => 'root' }, job_variables: {}, root_variables_inheritance: true, ignore: false, @@ -228,7 +224,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Root do services: [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }], stage: 'test', cache: [{ key: 'k', untracked: true, paths: ['public/'], policy: 'pull-push', when: 'on_success' }], - variables: { 'VAR' => 'job' }, job_variables: { 'VAR' => 'job' }, root_variables_inheritance: true, ignore: false, diff --git a/spec/lib/gitlab/ci/config/entry/rules_spec.rb b/spec/lib/gitlab/ci/config/entry/rules_spec.rb index 7d26365e7b3..91252378541 100644 --- a/spec/lib/gitlab/ci/config/entry/rules_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/rules_spec.rb @@ -17,6 +17,10 @@ RSpec.describe Gitlab::Ci::Config::Entry::Rules do describe '.new' do subject { entry } + before do + subject.compose! + end + context 'with a list of rule rule' do let(:config) do [{ if: '$THIS == "that"', when: 'never' }] @@ -24,14 +28,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Rules do it { is_expected.to be_a(described_class) } it { is_expected.to be_valid } - - context 'when composed' do - before do - subject.compose! - end - - it { is_expected.to be_valid } - end end context 'with a list of two rules' do @@ -42,21 +38,34 @@ RSpec.describe Gitlab::Ci::Config::Entry::Rules do ] end - it { is_expected.to be_a(described_class) } it { is_expected.to be_valid } + end - context 'when composed' do - before do - subject.compose! - end + context 'with a single rule object' do + let(:config) do + { if: '$SKIP', when: 'never' } + end - it { is_expected.to be_valid } + it { is_expected.not_to be_valid } + end + + context 'with nested rules' do + let(:config) do + [ + { if: '$THIS == "that"', when: 'always' }, + [{ if: '$SKIP', when: 'never' }] + ] end + + it { is_expected.to be_valid } end - context 'with a single rule object' do + context 'with rules nested more than one level' do let(:config) do - { if: '$SKIP', when: 'never' } + [ + { if: '$THIS == "that"', when: 'always' }, + [{ if: '$SKIP', when: 'never' }, [{ if: '$THIS == "other"', when: 'aways' }]] + ] end it { is_expected.not_to be_valid } @@ -90,7 +99,36 @@ RSpec.describe Gitlab::Ci::Config::Entry::Rules do { if: '$SKIP', when: 'never' } end - it { is_expected.to eq(config) } + it { is_expected.to eq([config]) } + end + + context 'with nested rules' do + let(:first_rule) { { if: '$THIS == "that"', when: 'always' } } + let(:second_rule) { { if: '$SKIP', when: 'never' } } + + let(:config) do + [ + first_rule, + [second_rule] + ] + end + + it { is_expected.to contain_exactly(first_rule, second_rule) } + end + + context 'with rules nested more than one level' do + let(:first_rule) { { if: '$THIS == "that"', when: 'always' } } + let(:second_rule) { { if: '$SKIP', when: 'never' } } + let(:third_rule) { { if: '$THIS == "other"', when: 'aways' } } + + let(:config) do + [ + first_rule, + [second_rule, [third_rule]] + ] + end + + it { is_expected.to contain_exactly(first_rule, second_rule, third_rule) } end end diff --git a/spec/lib/gitlab/ci/config/external/mapper_spec.rb b/spec/lib/gitlab/ci/config/external/mapper_spec.rb index 88097f3f56a..a471997e43a 100644 --- a/spec/lib/gitlab/ci/config/external/mapper_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper_spec.rb @@ -11,7 +11,7 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do let(:local_file) { '/lib/gitlab/ci/templates/non-existent-file.yml' } let(:remote_url) { 'https://gitlab.com/gitlab-org/gitlab-foss/blob/1234/.gitlab-ci-1.yml' } let(:template_file) { 'Auto-DevOps.gitlab-ci.yml' } - let(:context_params) { { project: project, sha: '123456', user: user, variables: project.predefined_variables.to_runner_variables } } + let(:context_params) { { project: project, sha: '123456', user: user, variables: project.predefined_variables } } let(:context) { Gitlab::Ci::Config::External::Context.new(**context_params) } let(:file_content) do @@ -347,15 +347,51 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do expect(subject.map(&:location)).to contain_exactly('myfolder/file1.yml', 'myfolder/file2.yml') end + end + + context "when 'include' has rules" do + let(:values) do + { include: [{ remote: remote_url }, + { local: local_file, rules: [{ if: "$CI_PROJECT_ID == '#{project_id}'" }] }], + image: 'ruby:2.7' } + end - context 'when the FF ci_wildcard_file_paths is disabled' do - before do - stub_feature_flags(ci_wildcard_file_paths: false) + context 'when the rules matches' do + let(:project_id) { project.id } + + it 'includes the file' do + expect(subject).to contain_exactly(an_instance_of(Gitlab::Ci::Config::External::File::Remote), + an_instance_of(Gitlab::Ci::Config::External::File::Local)) end - it 'cannot find any file returns an error message' do - expect(subject).to contain_exactly(an_instance_of(Gitlab::Ci::Config::External::File::Local)) - expect(subject[0].errors).to eq(['Local file `myfolder/*.yml` does not exist!']) + context 'when the FF ci_include_rules is disabled' do + before do + stub_feature_flags(ci_include_rules: false) + end + + it 'includes the file' do + expect(subject).to contain_exactly(an_instance_of(Gitlab::Ci::Config::External::File::Remote), + an_instance_of(Gitlab::Ci::Config::External::File::Local)) + end + end + end + + context 'when the rules does not match' do + let(:project_id) { non_existing_record_id } + + it 'does not include the file' do + expect(subject).to contain_exactly(an_instance_of(Gitlab::Ci::Config::External::File::Remote)) + end + + context 'when the FF ci_include_rules is disabled' do + before do + stub_feature_flags(ci_include_rules: false) + end + + it 'includes the file' do + expect(subject).to contain_exactly(an_instance_of(Gitlab::Ci::Config::External::File::Remote), + an_instance_of(Gitlab::Ci::Config::External::File::Local)) + end end end end diff --git a/spec/lib/gitlab/ci/config/external/rules_spec.rb b/spec/lib/gitlab/ci/config/external/rules_spec.rb new file mode 100644 index 00000000000..89ea13d710d --- /dev/null +++ b/spec/lib/gitlab/ci/config/external/rules_spec.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Ci::Config::External::Rules do + let(:rule_hashes) {} + + subject(:rules) { described_class.new(rule_hashes) } + + describe '#evaluate' do + let(:context) { double(variables: {}) } + + subject(:result) { rules.evaluate(context).pass? } + + context 'when there is no rule' do + it { is_expected.to eq(true) } + end + + context 'when there is a rule' do + let(:rule_hashes) { [{ if: '$MY_VAR == "hello"' }] } + + context 'when the rule matches' do + let(:context) { double(variables: { MY_VAR: 'hello' }) } + + it { is_expected.to eq(true) } + end + + context 'when the rule does not match' do + let(:context) { double(variables: { MY_VAR: 'invalid' }) } + + it { is_expected.to eq(false) } + end + end + end +end diff --git a/spec/lib/gitlab/ci/config/normalizer/matrix_strategy_spec.rb b/spec/lib/gitlab/ci/config/normalizer/matrix_strategy_spec.rb index e5f0341c5fe..a29471706cc 100644 --- a/spec/lib/gitlab/ci/config/normalizer/matrix_strategy_spec.rb +++ b/spec/lib/gitlab/ci/config/normalizer/matrix_strategy_spec.rb @@ -50,10 +50,6 @@ RSpec.describe Gitlab::Ci::Config::Normalizer::MatrixStrategy do name: 'test: [aws, app1]', instance: 1, parallel: { total: 4 }, - variables: { - 'PROVIDER' => 'aws', - 'STACK' => 'app1' - }, job_variables: { 'PROVIDER' => 'aws', 'STACK' => 'app1' @@ -63,10 +59,6 @@ RSpec.describe Gitlab::Ci::Config::Normalizer::MatrixStrategy do name: 'test: [aws, app2]', instance: 2, parallel: { total: 4 }, - variables: { - 'PROVIDER' => 'aws', - 'STACK' => 'app2' - }, job_variables: { 'PROVIDER' => 'aws', 'STACK' => 'app2' @@ -76,10 +68,6 @@ RSpec.describe Gitlab::Ci::Config::Normalizer::MatrixStrategy do name: 'test: [ovh, app]', instance: 3, parallel: { total: 4 }, - variables: { - 'PROVIDER' => 'ovh', - 'STACK' => 'app' - }, job_variables: { 'PROVIDER' => 'ovh', 'STACK' => 'app' @@ -89,10 +77,6 @@ RSpec.describe Gitlab::Ci::Config::Normalizer::MatrixStrategy do name: 'test: [gcp, app]', instance: 4, parallel: { total: 4 }, - variables: { - 'PROVIDER' => 'gcp', - 'STACK' => 'app' - }, job_variables: { 'PROVIDER' => 'gcp', 'STACK' => 'app' diff --git a/spec/lib/gitlab/ci/config/normalizer_spec.rb b/spec/lib/gitlab/ci/config/normalizer_spec.rb index 4c19657413c..354392eb42e 100644 --- a/spec/lib/gitlab/ci/config/normalizer_spec.rb +++ b/spec/lib/gitlab/ci/config/normalizer_spec.rb @@ -4,7 +4,7 @@ require 'fast_spec_helper' RSpec.describe Gitlab::Ci::Config::Normalizer do let(:job_name) { :rspec } - let(:job_config) { { script: 'rspec', parallel: parallel_config, name: 'rspec', variables: variables_config } } + let(:job_config) { { script: 'rspec', parallel: parallel_config, name: 'rspec', job_variables: variables_config } } let(:config) { { job_name => job_config } } describe '.normalize_jobs' do @@ -202,21 +202,21 @@ RSpec.describe Gitlab::Ci::Config::Normalizer do it 'sets job variables', :aggregate_failures do expect(subject.values[0]).to match( - a_hash_including(variables: { VAR_1: 'A', VAR_2: 'B', USER_VARIABLE: 'user value' }) + a_hash_including(job_variables: { VAR_1: 'A', VAR_2: 'B', USER_VARIABLE: 'user value' }) ) expect(subject.values[1]).to match( - a_hash_including(variables: { VAR_1: 'A', VAR_2: 'C', USER_VARIABLE: 'user value' }) + a_hash_including(job_variables: { VAR_1: 'A', VAR_2: 'C', USER_VARIABLE: 'user value' }) ) end it 'parallelizes jobs with original config' do configs = subject.values.map do |config| - config.except(:name, :instance, :variables) + config.except(:name, :instance, :job_variables) end original_config = config[job_name] - .except(:name, :variables) + .except(:name, :job_variables) .deep_merge(parallel: { total: 2 }) expect(configs).to all(match(a_hash_including(original_config))) diff --git a/spec/lib/gitlab/ci/config_spec.rb b/spec/lib/gitlab/ci/config_spec.rb index 45ce4cac6c4..3ec4519748f 100644 --- a/spec/lib/gitlab/ci/config_spec.rb +++ b/spec/lib/gitlab/ci/config_spec.rb @@ -286,7 +286,9 @@ RSpec.describe Gitlab::Ci::Config do end context "when using 'include' directive" do - let(:project) { create(:project, :repository) } + let(:group) { create(:group) } + let(:project) { create(:project, :repository, group: group) } + let(:main_project) { create(:project, :repository, :public, group: group) } let(:remote_location) { 'https://gitlab.com/gitlab-org/gitlab-foss/blob/1234/.gitlab-ci-1.yml' } let(:local_location) { 'spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml' } @@ -317,7 +319,9 @@ RSpec.describe Gitlab::Ci::Config do include: - #{local_location} - #{remote_location} - + - project: '$MAIN_PROJECT' + ref: '$REF' + file: '$FILENAME' image: ruby:2.7 HEREDOC end @@ -331,6 +335,26 @@ RSpec.describe Gitlab::Ci::Config do allow(project.repository) .to receive(:blob_data_at).and_return(local_file_content) + + main_project.repository.create_file( + main_project.creator, + '.gitlab-ci.yml', + local_file_content, + message: 'Add README.md', + branch_name: 'master' + ) + + main_project.repository.create_file( + main_project.creator, + '.another-ci-file.yml', + local_file_content, + message: 'Add README.md', + branch_name: 'master' + ) + + create(:ci_variable, project: project, key: "REF", value: "HEAD") + create(:ci_group_variable, group: group, key: "FILENAME", value: ".gitlab-ci.yml") + create(:ci_instance_variable, key: 'MAIN_PROJECT', value: main_project.full_path) end context "when gitlab_ci_yml has valid 'include' defined" do @@ -344,6 +368,38 @@ RSpec.describe Gitlab::Ci::Config do expect(config.to_hash).to eq(composed_hash) end + + context 'handling variables' do + it 'contains all project variables' do + ref = config.context.variables.find { |v| v[:key] == 'REF' } + + expect(ref[:value]).to eq("HEAD") + end + + it 'contains all group variables' do + filename = config.context.variables.find { |v| v[:key] == 'FILENAME' } + + expect(filename[:value]).to eq(".gitlab-ci.yml") + end + + it 'contains all instance variables' do + project = config.context.variables.find { |v| v[:key] == 'MAIN_PROJECT' } + + expect(project[:value]).to eq(main_project.full_path) + end + + context 'overriding a group variable at project level' do + before do + create(:ci_variable, project: project, key: "FILENAME", value: ".another-ci-file.yml") + end + + it 'successfully overrides' do + filename = config.context.variables.to_hash[:FILENAME] + + expect(filename).to eq('.another-ci-file.yml') + end + end + end end context "when gitlab_ci.yml has invalid 'include' defined" do @@ -667,5 +723,33 @@ RSpec.describe Gitlab::Ci::Config do expect(config.to_hash).to eq(composed_hash) end end + + context "when an 'include' has rules" do + let(:gitlab_ci_yml) do + <<~HEREDOC + include: + - local: #{local_location} + rules: + - if: $CI_PROJECT_ID == "#{project_id}" + image: ruby:2.7 + HEREDOC + end + + context 'when the rules condition is satisfied' do + let(:project_id) { project.id } + + it 'includes the file' do + expect(config.to_hash).to include(local_location_hash) + end + end + + context 'when the rules condition is satisfied' do + let(:project_id) { non_existing_record_id } + + it 'does not include the file' do + expect(config.to_hash).not_to include(local_location_hash) + end + end + end end end diff --git a/spec/lib/gitlab/ci/lint_spec.rb b/spec/lib/gitlab/ci/lint_spec.rb index 77f6608eb85..1e433d7854a 100644 --- a/spec/lib/gitlab/ci/lint_spec.rb +++ b/spec/lib/gitlab/ci/lint_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::Ci::Lint do - let(:project) { create(:project, :repository) } + let_it_be(:project) { create(:project, :repository) } let_it_be(:user) { create(:user) } let(:lint) { described_class.new(project: project, current_user: user) } @@ -89,6 +89,15 @@ RSpec.describe Gitlab::Ci::Lint do ) end + after do + project.repository.delete_file( + project.creator, + 'another-gitlab-ci.yml', + message: 'Remove another-gitlab-ci.yml', + branch_name: 'master' + ) + end + it 'sets merged_config' do root_config = YAML.safe_load(content, [Symbol]) included_config = YAML.safe_load(included_content, [Symbol]) diff --git a/spec/lib/gitlab/ci/parsers/security/common_spec.rb b/spec/lib/gitlab/ci/parsers/security/common_spec.rb new file mode 100644 index 00000000000..c6387bf615b --- /dev/null +++ b/spec/lib/gitlab/ci/parsers/security/common_spec.rb @@ -0,0 +1,350 @@ +# frozen_string_literal: true + +# TODO remove duplication from spec/lib/gitlab/ci/parsers/security/common_spec.rb and spec/lib/gitlab/ci/parsers/security/common_spec.rb +# See https://gitlab.com/gitlab-org/gitlab/-/issues/336589 +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Parsers::Security::Common do + describe '#parse!' do + where(vulnerability_finding_signatures_enabled: [true, false]) + with_them do + let_it_be(:pipeline) { create(:ci_pipeline) } + + let(:artifact) { build(:ci_job_artifact, :common_security_report) } + let(:report) { Gitlab::Ci::Reports::Security::Report.new(artifact.file_type, pipeline, 2.weeks.ago) } + # The path 'yarn.lock' was initially used by DependencyScanning, it is okay for SAST locations to use it, but this could be made better + let(:location) { ::Gitlab::Ci::Reports::Security::Locations::Sast.new(file_path: 'yarn.lock', start_line: 1, end_line: 1) } + let(:tracking_data) { nil } + + before do + allow_next_instance_of(described_class) do |parser| + allow(parser).to receive(:create_location).and_return(location) + allow(parser).to receive(:tracking_data).and_return(tracking_data) + end + + artifact.each_blob { |blob| described_class.parse!(blob, report, vulnerability_finding_signatures_enabled) } + end + + describe 'schema validation' do + let(:validator_class) { Gitlab::Ci::Parsers::Security::Validators::SchemaValidator } + let(:parser) { described_class.new('{}', report, vulnerability_finding_signatures_enabled, validate: validate) } + + subject(:parse_report) { parser.parse! } + + before do + allow(validator_class).to receive(:new).and_call_original + end + + context 'when the validate flag is set as `false`' do + let(:validate) { false } + + it 'does not run the validation logic' do + parse_report + + expect(validator_class).not_to have_received(:new) + end + end + + context 'when the validate flag is set as `true`' do + let(:validate) { true } + let(:valid?) { false } + + before do + allow_next_instance_of(validator_class) do |instance| + allow(instance).to receive(:valid?).and_return(valid?) + allow(instance).to receive(:errors).and_return(['foo']) + end + + allow(parser).to receive_messages(create_scanner: true, create_scan: true) + end + + it 'instantiates the validator with correct params' do + parse_report + + expect(validator_class).to have_received(:new).with(report.type, {}) + end + + context 'when the report data is not valid according to the schema' do + it 'adds errors to the report' do + expect { parse_report }.to change { report.errors }.from([]).to([{ message: 'foo', type: 'Schema' }]) + end + + it 'does not try to create report entities' do + parse_report + + expect(parser).not_to have_received(:create_scanner) + expect(parser).not_to have_received(:create_scan) + end + end + + context 'when the report data is valid according to the schema' do + let(:valid?) { true } + + it 'does not add errors to the report' do + expect { parse_report }.not_to change { report.errors }.from([]) + end + + it 'keeps the execution flow as normal' do + parse_report + + expect(parser).to have_received(:create_scanner) + expect(parser).to have_received(:create_scan) + end + end + end + end + + describe 'parsing finding.name' do + let(:artifact) { build(:ci_job_artifact, :common_security_report_with_blank_names) } + + context 'when message is provided' do + it 'sets message from the report as a finding name' do + finding = report.findings.find { |x| x.compare_key == 'CVE-1020' } + expected_name = Gitlab::Json.parse(finding.raw_metadata)['message'] + + expect(finding.name).to eq(expected_name) + end + end + + context 'when message is not provided' do + context 'and name is provided' do + it 'sets name from the report as a name' do + finding = report.findings.find { |x| x.compare_key == 'CVE-1030' } + expected_name = Gitlab::Json.parse(finding.raw_metadata)['name'] + + expect(finding.name).to eq(expected_name) + end + end + + context 'and name is not provided' do + context 'when CVE identifier exists' do + it 'combines identifier with location to create name' do + finding = report.findings.find { |x| x.compare_key == 'CVE-2017-11429' } + expect(finding.name).to eq("CVE-2017-11429 in yarn.lock") + end + end + + context 'when CWE identifier exists' do + it 'combines identifier with location to create name' do + finding = report.findings.find { |x| x.compare_key == 'CWE-2017-11429' } + expect(finding.name).to eq("CWE-2017-11429 in yarn.lock") + end + end + + context 'when neither CVE nor CWE identifier exist' do + it 'combines identifier with location to create name' do + finding = report.findings.find { |x| x.compare_key == 'OTHER-2017-11429' } + expect(finding.name).to eq("other-2017-11429 in yarn.lock") + end + end + end + end + end + + describe 'parsing finding.details' do + context 'when details are provided' do + it 'sets details from the report' do + finding = report.findings.find { |x| x.compare_key == 'CVE-1020' } + expected_details = Gitlab::Json.parse(finding.raw_metadata)['details'] + + expect(finding.details).to eq(expected_details) + end + end + + context 'when details are not provided' do + it 'sets empty hash' do + finding = report.findings.find { |x| x.compare_key == 'CVE-1030' } + expect(finding.details).to eq({}) + end + end + end + + describe 'top-level scanner' do + it 'is the primary scanner' do + expect(report.primary_scanner.external_id).to eq('gemnasium') + expect(report.primary_scanner.name).to eq('Gemnasium') + expect(report.primary_scanner.vendor).to eq('GitLab') + expect(report.primary_scanner.version).to eq('2.18.0') + end + + it 'returns nil report has no scanner' do + empty_report = Gitlab::Ci::Reports::Security::Report.new(artifact.file_type, pipeline, 2.weeks.ago) + described_class.parse!({}.to_json, empty_report) + + expect(empty_report.primary_scanner).to be_nil + end + end + + describe 'parsing scanners' do + subject(:scanner) { report.findings.first.scanner } + + context 'when vendor is not missing in scanner' do + it 'returns scanner with parsed vendor value' do + expect(scanner.vendor).to eq('GitLab') + end + end + end + + describe 'parsing scan' do + it 'returns scan object for each finding' do + scans = report.findings.map(&:scan) + + expect(scans.map(&:status).all?('success')).to be(true) + expect(scans.map(&:start_time).all?('placeholder-value')).to be(true) + expect(scans.map(&:end_time).all?('placeholder-value')).to be(true) + expect(scans.size).to eq(3) + expect(scans.first).to be_a(::Gitlab::Ci::Reports::Security::Scan) + end + + it 'returns nil when scan is not a hash' do + empty_report = Gitlab::Ci::Reports::Security::Report.new(artifact.file_type, pipeline, 2.weeks.ago) + described_class.parse!({}.to_json, empty_report) + + expect(empty_report.scan).to be(nil) + end + end + + describe 'parsing schema version' do + it 'parses the version' do + expect(report.version).to eq('14.0.2') + end + + it 'returns nil when there is no version' do + empty_report = Gitlab::Ci::Reports::Security::Report.new(artifact.file_type, pipeline, 2.weeks.ago) + described_class.parse!({}.to_json, empty_report) + + expect(empty_report.version).to be_nil + end + end + + describe 'parsing analyzer' do + it 'associates analyzer with report' do + expect(report.analyzer.id).to eq('common-analyzer') + expect(report.analyzer.name).to eq('Common Analyzer') + expect(report.analyzer.version).to eq('2.0.1') + expect(report.analyzer.vendor).to eq('Common') + end + + it 'returns nil when analyzer data is not available' do + empty_report = Gitlab::Ci::Reports::Security::Report.new(artifact.file_type, pipeline, 2.weeks.ago) + described_class.parse!({}.to_json, empty_report) + + expect(empty_report.analyzer).to be_nil + end + end + + describe 'parsing links' do + it 'returns links object for each finding', :aggregate_failures do + links = report.findings.flat_map(&:links) + + expect(links.map(&:url)).to match_array(['https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-1020', 'https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-1030']) + expect(links.map(&:name)).to match_array([nil, 'CVE-1030']) + expect(links.size).to eq(2) + expect(links.first).to be_a(::Gitlab::Ci::Reports::Security::Link) + end + end + + describe 'setting the uuid' do + let(:finding_uuids) { report.findings.map(&:uuid) } + let(:uuid_1) do + Security::VulnerabilityUUID.generate( + report_type: "sast", + primary_identifier_fingerprint: report.findings[0].identifiers.first.fingerprint, + location_fingerprint: location.fingerprint, + project_id: pipeline.project_id + ) + end + + let(:uuid_2) do + Security::VulnerabilityUUID.generate( + report_type: "sast", + primary_identifier_fingerprint: report.findings[1].identifiers.first.fingerprint, + location_fingerprint: location.fingerprint, + project_id: pipeline.project_id + ) + end + + let(:expected_uuids) { [uuid_1, uuid_2, nil] } + + it 'sets the UUIDv5 for findings', :aggregate_failures do + allow_next_instance_of(Gitlab::Ci::Reports::Security::Report) do |report| + allow(report).to receive(:type).and_return('sast') + + expect(finding_uuids).to match_array(expected_uuids) + end + end + end + + describe 'parsing tracking' do + let(:tracking_data) do + { + 'type' => 'source', + 'items' => [ + 'signatures' => [ + { 'algorithm' => 'hash', 'value' => 'hash_value' }, + { 'algorithm' => 'location', 'value' => 'location_value' }, + { 'algorithm' => 'scope_offset', 'value' => 'scope_offset_value' } + ] + ] + } + end + + context 'with valid tracking information' do + it 'creates signatures for each algorithm' do + finding = report.findings.first + expect(finding.signatures.size).to eq(3) + expect(finding.signatures.map(&:algorithm_type).to_set).to eq(Set['hash', 'location', 'scope_offset']) + end + end + + context 'with invalid tracking information' do + let(:tracking_data) do + { + 'type' => 'source', + 'items' => [ + 'signatures' => [ + { 'algorithm' => 'hash', 'value' => 'hash_value' }, + { 'algorithm' => 'location', 'value' => 'location_value' }, + { 'algorithm' => 'INVALID', 'value' => 'scope_offset_value' } + ] + ] + } + end + + it 'ignores invalid algorithm types' do + finding = report.findings.first + expect(finding.signatures.size).to eq(2) + expect(finding.signatures.map(&:algorithm_type).to_set).to eq(Set['hash', 'location']) + end + end + + context 'with valid tracking information' do + it 'creates signatures for each signature algorithm' do + finding = report.findings.first + expect(finding.signatures.size).to eq(3) + expect(finding.signatures.map(&:algorithm_type)).to eq(%w[hash location scope_offset]) + + signatures = finding.signatures.index_by(&:algorithm_type) + expected_values = tracking_data['items'][0]['signatures'].index_by { |x| x['algorithm'] } + expect(signatures['hash'].signature_value).to eq(expected_values['hash']['value']) + expect(signatures['location'].signature_value).to eq(expected_values['location']['value']) + expect(signatures['scope_offset'].signature_value).to eq(expected_values['scope_offset']['value']) + end + + it 'sets the uuid according to the higest priority signature' do + finding = report.findings.first + highest_signature = finding.signatures.max_by(&:priority) + + identifiers = if vulnerability_finding_signatures_enabled + "#{finding.report_type}-#{finding.primary_identifier.fingerprint}-#{highest_signature.signature_hex}-#{report.project_id}" + else + "#{finding.report_type}-#{finding.primary_identifier.fingerprint}-#{finding.location.fingerprint}-#{report.project_id}" + end + + expect(finding.uuid).to eq(Gitlab::UUID.v5(identifiers)) + end + end + end + end + end +end diff --git a/spec/lib/gitlab/ci/parsers/security/sast_spec.rb b/spec/lib/gitlab/ci/parsers/security/sast_spec.rb new file mode 100644 index 00000000000..4bc48f6611a --- /dev/null +++ b/spec/lib/gitlab/ci/parsers/security/sast_spec.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Parsers::Security::Sast do + using RSpec::Parameterized::TableSyntax + + describe '#parse!' do + let_it_be(:pipeline) { create(:ci_pipeline) } + + let(:created_at) { 2.weeks.ago } + + context "when parsing valid reports" do + where(:report_format, :report_version, :scanner_length, :finding_length, :identifier_length, :file_path, :line) do + :sast | '14.0.0' | 1 | 5 | 6 | 'groovy/src/main/java/com/gitlab/security_products/tests/App.groovy' | 47 + :sast_deprecated | '1.2' | 3 | 33 | 17 | 'python/hardcoded/hardcoded-tmp.py' | 1 + end + + with_them do + let(:report) { Gitlab::Ci::Reports::Security::Report.new(artifact.file_type, pipeline, created_at) } + let(:artifact) { create(:ci_job_artifact, report_format) } + + before do + artifact.each_blob { |blob| described_class.parse!(blob, report) } + end + + it "parses all identifiers and findings" do + expect(report.findings.length).to eq(finding_length) + expect(report.identifiers.length).to eq(identifier_length) + expect(report.scanners.length).to eq(scanner_length) + end + + it 'generates expected location' do + location = report.findings.first.location + + expect(location).to be_a(::Gitlab::Ci::Reports::Security::Locations::Sast) + expect(location).to have_attributes( + file_path: file_path, + end_line: line, + start_line: line + ) + end + + it "generates expected metadata_version" do + expect(report.findings.first.metadata_version).to eq(report_version) + end + end + end + + context "when parsing an empty report" do + let(:report) { Gitlab::Ci::Reports::Security::Report.new('sast', pipeline, created_at) } + let(:blob) { Gitlab::Json.generate({}) } + + it { expect(described_class.parse!(blob, report)).to be_empty } + end + end +end diff --git a/spec/lib/gitlab/ci/parsers/security/secret_detection_spec.rb b/spec/lib/gitlab/ci/parsers/security/secret_detection_spec.rb new file mode 100644 index 00000000000..1d361e16aad --- /dev/null +++ b/spec/lib/gitlab/ci/parsers/security/secret_detection_spec.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Parsers::Security::SecretDetection do + describe '#parse!' do + let_it_be(:pipeline) { create(:ci_pipeline) } + + let(:created_at) { 2.weeks.ago } + + context "when parsing valid reports" do + where(report_format: %i(secret_detection)) + + with_them do + let(:report) { Gitlab::Ci::Reports::Security::Report.new(artifact.file_type, pipeline, created_at) } + let(:artifact) { create(:ci_job_artifact, report_format) } + + before do + artifact.each_blob { |blob| described_class.parse!(blob, report) } + end + + it "parses all identifiers and findings" do + expect(report.findings.length).to eq(1) + expect(report.identifiers.length).to eq(1) + expect(report.scanners.length).to eq(1) + end + + it 'generates expected location' do + location = report.findings.first.location + + expect(location).to be_a(::Gitlab::Ci::Reports::Security::Locations::SecretDetection) + expect(location).to have_attributes( + file_path: 'aws-key.py', + start_line: nil, + end_line: nil, + class_name: nil, + method_name: nil + ) + end + + it "generates expected metadata_version" do + expect(report.findings.first.metadata_version).to eq('3.0') + end + end + end + + context "when parsing an empty report" do + let(:report) { Gitlab::Ci::Reports::Security::Report.new('secret_detection', pipeline, created_at) } + let(:blob) { Gitlab::Json.generate({}) } + + it { expect(described_class.parse!(blob, report)).to be_empty } + end + end +end diff --git a/spec/lib/gitlab/ci/parsers/security/validators/schema_validator_spec.rb b/spec/lib/gitlab/ci/parsers/security/validators/schema_validator_spec.rb new file mode 100644 index 00000000000..f434ffd12bf --- /dev/null +++ b/spec/lib/gitlab/ci/parsers/security/validators/schema_validator_spec.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator do + using RSpec::Parameterized::TableSyntax + + where(:report_type, :expected_errors, :valid_data) do + :sast | ['root is missing required keys: vulnerabilities'] | { 'version' => '10.0.0', 'vulnerabilities' => [] } + :secret_detection | ['root is missing required keys: vulnerabilities'] | { 'version' => '10.0.0', 'vulnerabilities' => [] } + end + + with_them do + let(:validator) { described_class.new(report_type, report_data) } + + describe '#valid?' do + subject { validator.valid? } + + context 'when given data is invalid according to the schema' do + let(:report_data) { {} } + + it { is_expected.to be_falsey } + end + + context 'when given data is valid according to the schema' do + let(:report_data) { valid_data } + + it { is_expected.to be_truthy } + end + end + + describe '#errors' do + let(:report_data) { { 'version' => '10.0.0' } } + + subject { validator.errors } + + it { is_expected.to eq(expected_errors) } + end + end +end diff --git a/spec/lib/gitlab/ci/pipeline/chain/command_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/command_spec.rb index 2e73043e309..c22a0e23794 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/command_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/command_spec.rb @@ -295,31 +295,6 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Command do end end - describe '#dangling_build?' do - let(:project) { create(:project, :repository) } - let(:command) { described_class.new(project: project, source: source) } - - subject { command.dangling_build? } - - context 'when source is :webide' do - let(:source) { :webide } - - it { is_expected.to eq(true) } - end - - context 'when source is :ondemand_dast_scan' do - let(:source) { :ondemand_dast_scan } - - it { is_expected.to eq(true) } - end - - context 'when source something else' do - let(:source) { :web } - - it { is_expected.to eq(false) } - end - end - describe '#creates_child_pipeline?' do let(:command) { described_class.new(bridge: bridge) } diff --git a/spec/lib/gitlab/ci/pipeline/chain/limit/deployments_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/limit/deployments_spec.rb index 499dc3554a3..1aa104310af 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/limit/deployments_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/limit/deployments_spec.rb @@ -85,7 +85,7 @@ RSpec.describe ::Gitlab::Ci::Pipeline::Chain::Limit::Deployments do end it 'logs the error' do - expect(Gitlab::ErrorTracking).to receive(:track_exception).with( + expect(Gitlab::ErrorTracking).to receive(:log_exception).with( instance_of(Gitlab::Ci::Limit::LimitExceededError), project_id: project.id, plan: namespace.actual_plan_name ) diff --git a/spec/lib/gitlab/ci/pipeline/chain/sequence_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/sequence_spec.rb index cc4aaffb0a4..83d47ae6819 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/sequence_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/sequence_spec.rb @@ -7,7 +7,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Sequence do let_it_be(:user) { create(:user) } let(:pipeline) { build_stubbed(:ci_pipeline) } - let(:command) { Gitlab::Ci::Pipeline::Chain::Command.new } + let(:command) { Gitlab::Ci::Pipeline::Chain::Command.new(project: project) } let(:first_step) { spy('first step') } let(:second_step) { spy('second step') } let(:sequence) { [first_step, second_step] } @@ -71,5 +71,20 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Sequence do expect(histogram).to have_received(:observe) .with({ source: 'push' }, 0) end + + it 'records active jobs by pipeline plan in a histogram' do + allow(command.metrics) + .to receive(:active_jobs_histogram) + .and_return(histogram) + + pipeline = create(:ci_pipeline, project: project, status: :running) + create(:ci_build, :finished, project: project, pipeline: pipeline) + create(:ci_build, :failed, project: project, pipeline: pipeline) + create(:ci_build, :running, project: project, pipeline: pipeline) + subject.build! + + expect(histogram).to have_received(:observe) + .with(hash_including(plan: project.actual_plan_name), 3) + end end end diff --git a/spec/lib/gitlab/ci/reports/security/aggregated_report_spec.rb b/spec/lib/gitlab/ci/reports/security/aggregated_report_spec.rb new file mode 100644 index 00000000000..c56177a6453 --- /dev/null +++ b/spec/lib/gitlab/ci/reports/security/aggregated_report_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Reports::Security::AggregatedReport do + subject { described_class.new(reports, findings) } + + let(:reports) { build_list(:ci_reports_security_report, 1) } + let(:findings) { build_list(:ci_reports_security_finding, 1) } + + describe '#created_at' do + context 'no reports' do + let(:reports) { [] } + + it 'has no created date' do + expect(subject.created_at).to be_nil + end + end + + context 'report with no created date' do + let(:reports) { build_list(:ci_reports_security_report, 1, created_at: nil) } + + it 'has no created date' do + expect(subject.created_at).to be_nil + end + end + + context 'has reports' do + let(:a_long_time_ago) { 2.months.ago } + let(:a_while_ago) { 2.weeks.ago } + let(:yesterday) { 1.day.ago } + + let(:reports) do + [build(:ci_reports_security_report, created_at: a_while_ago), + build(:ci_reports_security_report, created_at: a_long_time_ago), + build(:ci_reports_security_report, created_at: nil), + build(:ci_reports_security_report, created_at: yesterday)] + end + + it 'has oldest created date' do + expect(subject.created_at).to eq(a_long_time_ago) + end + end + end +end diff --git a/spec/lib/gitlab/ci/reports/security/finding_key_spec.rb b/spec/lib/gitlab/ci/reports/security/finding_key_spec.rb new file mode 100644 index 00000000000..784c1183320 --- /dev/null +++ b/spec/lib/gitlab/ci/reports/security/finding_key_spec.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Reports::Security::FindingKey do + using RSpec::Parameterized::TableSyntax + + describe '#==' do + where(:location_fp_1, :location_fp_2, :identifier_fp_1, :identifier_fp_2, :equals?) do + nil | 'different location fp' | 'identifier fp' | 'different identifier fp' | false + 'location fp' | nil | 'identifier fp' | 'different identifier fp' | false + 'location fp' | 'different location fp' | nil | 'different identifier fp' | false + 'location fp' | 'different location fp' | 'identifier fp' | nil | false + nil | nil | 'identifier fp' | 'identifier fp' | false + 'location fp' | 'location fp' | nil | nil | false + nil | nil | nil | nil | false + 'location fp' | 'different location fp' | 'identifier fp' | 'different identifier fp' | false + 'location fp' | 'different location fp' | 'identifier fp' | 'identifier fp' | false + 'location fp' | 'location fp' | 'identifier fp' | 'different identifier fp' | false + 'location fp' | 'location fp' | 'identifier fp' | 'identifier fp' | true + end + + with_them do + let(:finding_key_1) do + build(:ci_reports_security_finding_key, + location_fingerprint: location_fp_1, + identifier_fingerprint: identifier_fp_1) + end + + let(:finding_key_2) do + build(:ci_reports_security_finding_key, + location_fingerprint: location_fp_2, + identifier_fingerprint: identifier_fp_2) + end + + subject { finding_key_1 == finding_key_2 } + + it { is_expected.to be(equals?) } + end + end +end diff --git a/spec/lib/gitlab/ci/reports/security/finding_signature_spec.rb b/spec/lib/gitlab/ci/reports/security/finding_signature_spec.rb new file mode 100644 index 00000000000..23e6b40a039 --- /dev/null +++ b/spec/lib/gitlab/ci/reports/security/finding_signature_spec.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Reports::Security::FindingSignature do + subject { described_class.new(params.with_indifferent_access) } + + let(:params) do + { + algorithm_type: 'hash', + signature_value: 'SIGNATURE' + } + end + + describe '#initialize' do + context 'when a supported algorithm type is given' do + it 'allows itself to be created' do + expect(subject.algorithm_type).to eq(params[:algorithm_type]) + expect(subject.signature_value).to eq(params[:signature_value]) + end + + describe '#valid?' do + it 'returns true' do + expect(subject.valid?).to eq(true) + end + end + end + end + + describe '#valid?' do + context 'when supported algorithm_type is given' do + it 'is valid' do + expect(subject.valid?).to eq(true) + end + end + + context 'when an unsupported algorithm_type is given' do + let(:params) do + { + algorithm_type: 'INVALID', + signature_value: 'SIGNATURE' + } + end + + it 'is not valid' do + expect(subject.valid?).to eq(false) + end + end + end + + describe '#to_hash' do + it 'returns a hash representation of the signature' do + expect(subject.to_hash).to eq( + algorithm_type: params[:algorithm_type], + signature_sha: Digest::SHA1.digest(params[:signature_value]) + ) + end + end +end diff --git a/spec/lib/gitlab/ci/reports/security/locations/sast_spec.rb b/spec/lib/gitlab/ci/reports/security/locations/sast_spec.rb new file mode 100644 index 00000000000..effa7a60400 --- /dev/null +++ b/spec/lib/gitlab/ci/reports/security/locations/sast_spec.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Reports::Security::Locations::Sast do + let(:params) do + { + file_path: 'src/main/App.java', + start_line: 29, + end_line: 31, + class_name: 'com.gitlab.security_products.tests.App', + method_name: 'insecureCypher' + } + end + + let(:mandatory_params) { %i[file_path start_line] } + let(:expected_fingerprint) { Digest::SHA1.hexdigest('src/main/App.java:29:31') } + let(:expected_fingerprint_path) { 'App.java' } + + it_behaves_like 'vulnerability location' +end diff --git a/spec/lib/gitlab/ci/reports/security/locations/secret_detection_spec.rb b/spec/lib/gitlab/ci/reports/security/locations/secret_detection_spec.rb new file mode 100644 index 00000000000..3b84a548713 --- /dev/null +++ b/spec/lib/gitlab/ci/reports/security/locations/secret_detection_spec.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Reports::Security::Locations::SecretDetection do + let(:params) do + { + file_path: 'src/main/App.java', + start_line: 29, + end_line: 31, + class_name: 'com.gitlab.security_products.tests.App', + method_name: 'insecureCypher' + } + end + + let(:mandatory_params) { %i[file_path start_line] } + let(:expected_fingerprint) { Digest::SHA1.hexdigest('src/main/App.java:29:31') } + let(:expected_fingerprint_path) { 'App.java' } + + it_behaves_like 'vulnerability location' +end diff --git a/spec/lib/gitlab/ci/reports/security/report_spec.rb b/spec/lib/gitlab/ci/reports/security/report_spec.rb new file mode 100644 index 00000000000..5a85c3f19fc --- /dev/null +++ b/spec/lib/gitlab/ci/reports/security/report_spec.rb @@ -0,0 +1,224 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Reports::Security::Report do + let_it_be(:pipeline) { create(:ci_pipeline) } + + let(:created_at) { 2.weeks.ago } + + subject(:report) { described_class.new('sast', pipeline, created_at) } + + it { expect(report.type).to eq('sast') } + it { is_expected.to delegate_method(:project_id).to(:pipeline) } + + describe '#add_scanner' do + let(:scanner) { create(:ci_reports_security_scanner, external_id: 'find_sec_bugs') } + + subject { report.add_scanner(scanner) } + + it 'stores given scanner params in the map' do + subject + + expect(report.scanners).to eq({ 'find_sec_bugs' => scanner }) + end + + it 'returns the added scanner' do + expect(subject).to eq(scanner) + end + end + + describe '#add_identifier' do + let(:identifier) { create(:ci_reports_security_identifier) } + + subject { report.add_identifier(identifier) } + + it 'stores given identifier params in the map' do + subject + + expect(report.identifiers).to eq({ identifier.fingerprint => identifier }) + end + + it 'returns the added identifier' do + expect(subject).to eq(identifier) + end + end + + describe '#add_finding' do + let(:finding) { create(:ci_reports_security_finding) } + + it 'enriches given finding and stores it in the collection' do + report.add_finding(finding) + + expect(report.findings).to eq([finding]) + end + end + + describe '#clone_as_blank' do + let(:report) do + create( + :ci_reports_security_report, + findings: [create(:ci_reports_security_finding)], + scanners: [create(:ci_reports_security_scanner)], + identifiers: [create(:ci_reports_security_identifier)] + ) + end + + it 'creates a blank report with copied type and pipeline' do + clone = report.clone_as_blank + + expect(clone.type).to eq(report.type) + expect(clone.pipeline).to eq(report.pipeline) + expect(clone.created_at).to eq(report.created_at) + expect(clone.findings).to eq([]) + expect(clone.scanners).to eq({}) + expect(clone.identifiers).to eq({}) + end + end + + describe '#replace_with!' do + let(:report) do + create( + :ci_reports_security_report, + findings: [create(:ci_reports_security_finding)], + scanners: [create(:ci_reports_security_scanner)], + identifiers: [create(:ci_reports_security_identifier)] + ) + end + + let(:other_report) do + create( + :ci_reports_security_report, + findings: [create(:ci_reports_security_finding, compare_key: 'other_finding')], + scanners: [create(:ci_reports_security_scanner, external_id: 'other_scanner', name: 'Other Scanner')], + identifiers: [create(:ci_reports_security_identifier, external_id: 'other_id', name: 'other_scanner')] + ) + end + + before do + report.replace_with!(other_report) + end + + it 'replaces report contents with other reports contents' do + expect(report.findings).to eq(other_report.findings) + expect(report.scanners).to eq(other_report.scanners) + expect(report.identifiers).to eq(other_report.identifiers) + end + end + + describe '#merge!' do + let(:merged_report) { double('Report') } + + before do + merge_reports_service = double('MergeReportsService') + + allow(::Security::MergeReportsService).to receive(:new).and_return(merge_reports_service) + allow(merge_reports_service).to receive(:execute).and_return(merged_report) + allow(report).to receive(:replace_with!) + end + + subject { report.merge!(described_class.new('sast', pipeline, created_at)) } + + it 'invokes the merge with other report and then replaces this report contents by merge result' do + subject + + expect(report).to have_received(:replace_with!).with(merged_report) + end + end + + describe '#primary_scanner' do + let(:scanner_1) { create(:ci_reports_security_scanner, external_id: 'external_id_1') } + let(:scanner_2) { create(:ci_reports_security_scanner, external_id: 'external_id_2') } + + subject { report.primary_scanner } + + before do + report.add_scanner(scanner_1) + report.add_scanner(scanner_2) + end + + it { is_expected.to eq(scanner_1) } + end + + describe '#add_error' do + context 'when the message is not given' do + it 'adds a new error to report with the generic error message' do + expect { report.add_error('foo') }.to change { report.errors } + .from([]) + .to([{ type: 'foo', message: 'An unexpected error happened!' }]) + end + end + + context 'when the message is given' do + it 'adds a new error to report' do + expect { report.add_error('foo', 'bar') }.to change { report.errors } + .from([]) + .to([{ type: 'foo', message: 'bar' }]) + end + end + end + + describe 'errored?' do + subject { report.errored? } + + context 'when the report does not have any errors' do + it { is_expected.to be_falsey } + end + + context 'when the report has errors' do + before do + report.add_error('foo', 'bar') + end + + it { is_expected.to be_truthy } + end + end + + describe '#primary_scanner_order_to' do + let(:scanner_1) { build(:ci_reports_security_scanner) } + let(:scanner_2) { build(:ci_reports_security_scanner) } + let(:report_1) { described_class.new('sast', pipeline, created_at) } + let(:report_2) { described_class.new('sast', pipeline, created_at) } + + subject(:compare_based_on_primary_scanners) { report_1.primary_scanner_order_to(report_2) } + + context 'when the primary scanner of the receiver is nil' do + context 'when the primary scanner of the other is nil' do + it { is_expected.to be(1) } + end + + context 'when the primary scanner of the other is not nil' do + before do + report_2.add_scanner(scanner_2) + end + + it { is_expected.to be(1) } + end + end + + context 'when the primary scanner of the receiver is not nil' do + before do + report_1.add_scanner(scanner_1) + end + + context 'when the primary scanner of the other is nil' do + let(:scanner_2) { nil } + + it { is_expected.to be(-1) } + end + + context 'when the primary scanner of the other is not nil' do + before do + report_2.add_scanner(scanner_2) + + allow(scanner_1).to receive(:<=>).and_return(0) + end + + it 'compares two scanners' do + expect(compare_based_on_primary_scanners).to be(0) + expect(scanner_1).to have_received(:<=>).with(scanner_2) + end + end + end + end +end diff --git a/spec/lib/gitlab/ci/reports/security/reports_spec.rb b/spec/lib/gitlab/ci/reports/security/reports_spec.rb new file mode 100644 index 00000000000..9b1e02f1418 --- /dev/null +++ b/spec/lib/gitlab/ci/reports/security/reports_spec.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Reports::Security::Reports do + let_it_be(:pipeline) { create(:ci_pipeline) } + let_it_be(:artifact) { create(:ci_job_artifact, :sast) } + + let(:security_reports) { described_class.new(pipeline) } + + describe '#get_report' do + subject { security_reports.get_report(report_type, artifact) } + + context 'when report type is sast' do + let(:report_type) { 'sast' } + + it { expect(subject.type).to eq('sast') } + it { expect(subject.created_at).to eq(artifact.created_at) } + + it 'initializes a new report and returns it' do + expect(Gitlab::Ci::Reports::Security::Report).to receive(:new) + .with('sast', pipeline, artifact.created_at).and_call_original + + is_expected.to be_a(Gitlab::Ci::Reports::Security::Report) + end + + context 'when report type is already allocated' do + before do + subject + end + + it 'does not initialize a new report' do + expect(Gitlab::Ci::Reports::Security::Report).not_to receive(:new) + + is_expected.to be_a(Gitlab::Ci::Reports::Security::Report) + end + end + end + end + + describe '#findings' do + let(:finding_1) { build(:ci_reports_security_finding, severity: 'low') } + let(:finding_2) { build(:ci_reports_security_finding, severity: 'high') } + let!(:expected_findings) { [finding_1, finding_2] } + + subject { security_reports.findings } + + before do + security_reports.get_report('sast', artifact).add_finding(finding_1) + security_reports.get_report('dependency_scanning', artifact).add_finding(finding_2) + end + + it { is_expected.to match_array(expected_findings) } + end + + describe "#violates_default_policy_against?" do + let(:high_severity_dast) { build(:ci_reports_security_finding, severity: 'high', report_type: :dast) } + let(:vulnerabilities_allowed) { 0 } + let(:severity_levels) { %w(critical high) } + + subject { security_reports.violates_default_policy_against?(target_reports, vulnerabilities_allowed, severity_levels) } + + before do + security_reports.get_report('sast', artifact).add_finding(high_severity_dast) + end + + context 'when the target_reports is `nil`' do + let(:target_reports) { nil } + + context 'with severity levels matching the existing vulnerabilities' do + it { is_expected.to be(true) } + end + + context "without any severity levels matching the existing vulnerabilities" do + let(:severity_levels) { %w(critical) } + + it { is_expected.to be(false) } + end + end + + context 'when the target_reports is not `nil`' do + let(:target_reports) { described_class.new(pipeline) } + + context "when a report has a new unsafe vulnerability" do + context 'with severity levels matching the existing vulnerabilities' do + it { is_expected.to be(true) } + end + + it { is_expected.to be(true) } + + context 'with vulnerabilities_allowed higher than the number of new vulnerabilities' do + let(:vulnerabilities_allowed) { 10000 } + + it { is_expected.to be(false) } + end + + context "without any severity levels matching the existing vulnerabilities" do + let(:severity_levels) { %w(critical) } + + it { is_expected.to be(false) } + end + end + + context "when none of the reports have a new unsafe vulnerability" do + before do + target_reports.get_report('sast', artifact).add_finding(high_severity_dast) + end + + it { is_expected.to be(false) } + end + end + end +end diff --git a/spec/lib/gitlab/ci/reports/security/vulnerability_reports_comparer_spec.rb b/spec/lib/gitlab/ci/reports/security/vulnerability_reports_comparer_spec.rb new file mode 100644 index 00000000000..44e66fd9028 --- /dev/null +++ b/spec/lib/gitlab/ci/reports/security/vulnerability_reports_comparer_spec.rb @@ -0,0 +1,163 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Reports::Security::VulnerabilityReportsComparer do + let(:identifier) { build(:ci_reports_security_identifier) } + + let_it_be(:project) { create(:project, :repository) } + + let(:location_param) { build(:ci_reports_security_locations_sast, :dynamic) } + let(:vulnerability_params) { vuln_params(project.id, [identifier], confidence: :low, severity: :critical) } + let(:base_vulnerability) { build(:ci_reports_security_finding, location: location_param, **vulnerability_params) } + let(:base_report) { build(:ci_reports_security_aggregated_reports, findings: [base_vulnerability]) } + + let(:head_vulnerability) { build(:ci_reports_security_finding, location: location_param, uuid: base_vulnerability.uuid, **vulnerability_params) } + let(:head_report) { build(:ci_reports_security_aggregated_reports, findings: [head_vulnerability]) } + + shared_context 'comparing reports' do + let(:vul_params) { vuln_params(project.id, [identifier]) } + let(:base_vulnerability) { build(:ci_reports_security_finding, :dynamic, **vul_params) } + let(:head_vulnerability) { build(:ci_reports_security_finding, :dynamic, **vul_params) } + let(:head_vul_findings) { [head_vulnerability, vuln] } + end + + subject { described_class.new(project, base_report, head_report) } + + where(vulnerability_finding_signatures: [true, false]) + + with_them do + before do + stub_licensed_features(vulnerability_finding_signatures: vulnerability_finding_signatures) + end + + describe '#base_report_out_of_date' do + context 'no base report' do + let(:base_report) { build(:ci_reports_security_aggregated_reports, reports: [], findings: []) } + + it 'is not out of date' do + expect(subject.base_report_out_of_date).to be false + end + end + + context 'base report older than one week' do + let(:report) { build(:ci_reports_security_report, created_at: 1.week.ago - 60.seconds) } + let(:base_report) { build(:ci_reports_security_aggregated_reports, reports: [report]) } + + it 'is not out of date' do + expect(subject.base_report_out_of_date).to be true + end + end + + context 'base report less than one week old' do + let(:report) { build(:ci_reports_security_report, created_at: 1.week.ago + 60.seconds) } + let(:base_report) { build(:ci_reports_security_aggregated_reports, reports: [report]) } + + it 'is not out of date' do + expect(subject.base_report_out_of_date).to be false + end + end + end + + describe '#added' do + let(:new_location) {build(:ci_reports_security_locations_sast, :dynamic) } + let(:vul_params) { vuln_params(project.id, [identifier], confidence: :high) } + let(:vuln) { build(:ci_reports_security_finding, severity: Enums::Vulnerability.severity_levels[:critical], location: new_location, **vul_params) } + let(:low_vuln) { build(:ci_reports_security_finding, severity: Enums::Vulnerability.severity_levels[:low], location: new_location, **vul_params) } + + context 'with new vulnerability' do + let(:head_report) { build(:ci_reports_security_aggregated_reports, findings: [head_vulnerability, vuln]) } + + it 'points to source tree' do + expect(subject.added).to eq([vuln]) + end + end + + context 'when comparing reports with different fingerprints' do + include_context 'comparing reports' + + let(:head_report) { build(:ci_reports_security_aggregated_reports, findings: head_vul_findings) } + + it 'does not find any overlap' do + expect(subject.added).to eq(head_vul_findings) + end + end + + context 'order' do + let(:head_report) { build(:ci_reports_security_aggregated_reports, findings: [head_vulnerability, vuln, low_vuln]) } + + it 'does not change' do + expect(subject.added).to eq([vuln, low_vuln]) + end + end + end + + describe '#fixed' do + let(:vul_params) { vuln_params(project.id, [identifier]) } + let(:vuln) { build(:ci_reports_security_finding, :dynamic, **vul_params ) } + let(:medium_vuln) { build(:ci_reports_security_finding, confidence: ::Enums::Vulnerability.confidence_levels[:high], severity: Enums::Vulnerability.severity_levels[:medium], uuid: vuln.uuid, **vul_params) } + + context 'with fixed vulnerability' do + let(:base_report) { build(:ci_reports_security_aggregated_reports, findings: [base_vulnerability, vuln]) } + + it 'points to base tree' do + expect(subject.fixed).to eq([vuln]) + end + end + + context 'when comparing reports with different fingerprints' do + include_context 'comparing reports' + + let(:base_report) { build(:ci_reports_security_aggregated_reports, findings: [base_vulnerability, vuln]) } + + it 'does not find any overlap' do + expect(subject.fixed).to eq([base_vulnerability, vuln]) + end + end + + context 'order' do + let(:vul_findings) { [vuln, medium_vuln] } + let(:base_report) { build(:ci_reports_security_aggregated_reports, findings: [*vul_findings, base_vulnerability]) } + + it 'does not change' do + expect(subject.fixed).to eq(vul_findings) + end + end + end + + describe 'with empty vulnerabilities' do + let(:empty_report) { build(:ci_reports_security_aggregated_reports, reports: [], findings: []) } + + it 'returns empty array when reports are not present' do + comparer = described_class.new(project, empty_report, empty_report) + + expect(comparer.fixed).to eq([]) + expect(comparer.added).to eq([]) + end + + it 'returns added vulnerability when base is empty and head is not empty' do + comparer = described_class.new(project, empty_report, head_report) + + expect(comparer.fixed).to eq([]) + expect(comparer.added).to eq([head_vulnerability]) + end + + it 'returns fixed vulnerability when head is empty and base is not empty' do + comparer = described_class.new(project, base_report, empty_report) + + expect(comparer.fixed).to eq([base_vulnerability]) + expect(comparer.added).to eq([]) + end + end + end + + def vuln_params(project_id, identifiers, confidence: :high, severity: :critical) + { + project_id: project_id, + report_type: :sast, + identifiers: identifiers, + confidence: ::Enums::Vulnerability.confidence_levels[confidence], + severity: ::Enums::Vulnerability.severity_levels[severity] + } + end +end diff --git a/spec/lib/gitlab/ci/templates/5_minute_production_app_ci_yaml_spec.rb b/spec/lib/gitlab/ci/templates/5_minute_production_app_ci_yaml_spec.rb index 6bc8e261640..f8df2266689 100644 --- a/spec/lib/gitlab/ci/templates/5_minute_production_app_ci_yaml_spec.rb +++ b/spec/lib/gitlab/ci/templates/5_minute_production_app_ci_yaml_spec.rb @@ -12,7 +12,7 @@ RSpec.describe '5-Minute-Production-App.gitlab-ci.yml' do let(:default_branch) { 'master' } let(:pipeline_branch) { default_branch } let(:service) { Ci::CreatePipelineService.new(project, user, ref: pipeline_branch ) } - let(:pipeline) { service.execute!(:push) } + let(:pipeline) { service.execute!(:push).payload } let(:build_names) { pipeline.builds.pluck(:name) } before do diff --git a/spec/lib/gitlab/ci/templates/AWS/deploy_ecs_gitlab_ci_yaml_spec.rb b/spec/lib/gitlab/ci/templates/AWS/deploy_ecs_gitlab_ci_yaml_spec.rb index e8aeb93a2ba..ca6f6872f89 100644 --- a/spec/lib/gitlab/ci/templates/AWS/deploy_ecs_gitlab_ci_yaml_spec.rb +++ b/spec/lib/gitlab/ci/templates/AWS/deploy_ecs_gitlab_ci_yaml_spec.rb @@ -11,7 +11,7 @@ RSpec.describe 'Deploy-ECS.gitlab-ci.yml' do let(:project) { create(:project, :auto_devops, :custom_repo, files: { 'README.md' => '' }) } let(:user) { project.owner } let(:service) { Ci::CreatePipelineService.new(project, user, ref: pipeline_branch ) } - let(:pipeline) { service.execute!(:push) } + let(:pipeline) { service.execute!(:push).payload } let(:build_names) { pipeline.builds.pluck(:name) } let(:platform_target) { 'ECS' } diff --git a/spec/lib/gitlab/ci/templates/Jobs/build_gitlab_ci_yaml_spec.rb b/spec/lib/gitlab/ci/templates/Jobs/build_gitlab_ci_yaml_spec.rb index 053499344e1..bd701aec8fc 100644 --- a/spec/lib/gitlab/ci/templates/Jobs/build_gitlab_ci_yaml_spec.rb +++ b/spec/lib/gitlab/ci/templates/Jobs/build_gitlab_ci_yaml_spec.rb @@ -12,7 +12,7 @@ RSpec.describe 'Jobs/Build.gitlab-ci.yml' do let(:default_branch) { 'master' } let(:pipeline_ref) { default_branch } let(:service) { Ci::CreatePipelineService.new(project, user, ref: pipeline_ref) } - let(:pipeline) { service.execute!(:push) } + let(:pipeline) { service.execute!(:push).payload } let(:build_names) { pipeline.builds.pluck(:name) } before do @@ -47,7 +47,7 @@ RSpec.describe 'Jobs/Build.gitlab-ci.yml' do context 'on merge request' do let(:service) { MergeRequests::CreatePipelineService.new(project: project, current_user: user) } let(:merge_request) { create(:merge_request, :simple, source_project: project) } - let(:pipeline) { service.execute(merge_request) } + let(:pipeline) { service.execute(merge_request).payload } it 'has no jobs' do expect(pipeline).to be_merge_request_event diff --git a/spec/lib/gitlab/ci/templates/Jobs/code_quality_gitlab_ci_yaml_spec.rb b/spec/lib/gitlab/ci/templates/Jobs/code_quality_gitlab_ci_yaml_spec.rb index b23457315cc..64243f2d205 100644 --- a/spec/lib/gitlab/ci/templates/Jobs/code_quality_gitlab_ci_yaml_spec.rb +++ b/spec/lib/gitlab/ci/templates/Jobs/code_quality_gitlab_ci_yaml_spec.rb @@ -12,7 +12,7 @@ RSpec.describe 'Jobs/Code-Quality.gitlab-ci.yml' do let(:default_branch) { 'master' } let(:pipeline_ref) { default_branch } let(:service) { Ci::CreatePipelineService.new(project, user, ref: pipeline_ref) } - let(:pipeline) { service.execute!(:push) } + let(:pipeline) { service.execute!(:push).payload } let(:build_names) { pipeline.builds.pluck(:name) } before do @@ -47,7 +47,7 @@ RSpec.describe 'Jobs/Code-Quality.gitlab-ci.yml' do context 'on merge request' do let(:service) { MergeRequests::CreatePipelineService.new(project: project, current_user: user) } let(:merge_request) { create(:merge_request, :simple, source_project: project) } - let(:pipeline) { service.execute(merge_request) } + let(:pipeline) { service.execute(merge_request).payload } it 'has no jobs' do expect(pipeline).to be_merge_request_event diff --git a/spec/lib/gitlab/ci/templates/Jobs/deploy_gitlab_ci_yaml_spec.rb b/spec/lib/gitlab/ci/templates/Jobs/deploy_gitlab_ci_yaml_spec.rb index 1d137ef89e1..d377cf0c735 100644 --- a/spec/lib/gitlab/ci/templates/Jobs/deploy_gitlab_ci_yaml_spec.rb +++ b/spec/lib/gitlab/ci/templates/Jobs/deploy_gitlab_ci_yaml_spec.rb @@ -33,7 +33,7 @@ RSpec.describe 'Jobs/Deploy.gitlab-ci.yml' do let(:default_branch) { 'master' } let(:pipeline_ref) { default_branch } let(:service) { Ci::CreatePipelineService.new(project, user, ref: pipeline_ref) } - let(:pipeline) { service.execute!(:push) } + let(:pipeline) { service.execute!(:push).payload } let(:build_names) { pipeline.builds.pluck(:name) } before do @@ -210,7 +210,7 @@ RSpec.describe 'Jobs/Deploy.gitlab-ci.yml' do context 'on merge request' do let(:service) { MergeRequests::CreatePipelineService.new(project: project, current_user: user) } let(:merge_request) { create(:merge_request, :simple, source_project: project) } - let(:pipeline) { service.execute(merge_request) } + let(:pipeline) { service.execute(merge_request).payload } it 'has no jobs' do expect(pipeline).to be_merge_request_event diff --git a/spec/lib/gitlab/ci/templates/Jobs/test_gitlab_ci_yaml_spec.rb b/spec/lib/gitlab/ci/templates/Jobs/test_gitlab_ci_yaml_spec.rb index 7fa8d906d07..db9d7496251 100644 --- a/spec/lib/gitlab/ci/templates/Jobs/test_gitlab_ci_yaml_spec.rb +++ b/spec/lib/gitlab/ci/templates/Jobs/test_gitlab_ci_yaml_spec.rb @@ -12,7 +12,7 @@ RSpec.describe 'Jobs/Test.gitlab-ci.yml' do let(:default_branch) { 'master' } let(:pipeline_ref) { default_branch } let(:service) { Ci::CreatePipelineService.new(project, user, ref: pipeline_ref) } - let(:pipeline) { service.execute!(:push) } + let(:pipeline) { service.execute!(:push).payload } let(:build_names) { pipeline.builds.pluck(:name) } before do @@ -47,7 +47,7 @@ RSpec.describe 'Jobs/Test.gitlab-ci.yml' do context 'on merge request' do let(:service) { MergeRequests::CreatePipelineService.new(project: project, current_user: user) } let(:merge_request) { create(:merge_request, :simple, source_project: project) } - let(:pipeline) { service.execute(merge_request) } + let(:pipeline) { service.execute(merge_request).payload } it 'has no jobs' do expect(pipeline).to be_merge_request_event diff --git a/spec/lib/gitlab/ci/templates/Terraform/base_gitlab_ci_yaml_spec.rb b/spec/lib/gitlab/ci/templates/Terraform/base_gitlab_ci_yaml_spec.rb index 0811c07e896..4685d843ce0 100644 --- a/spec/lib/gitlab/ci/templates/Terraform/base_gitlab_ci_yaml_spec.rb +++ b/spec/lib/gitlab/ci/templates/Terraform/base_gitlab_ci_yaml_spec.rb @@ -2,8 +2,8 @@ require 'spec_helper' -RSpec.describe 'Terraform/Base.latest.gitlab-ci.yml' do - subject(:template) { Gitlab::Template::GitlabCiYmlTemplate.find('Terraform/Base.latest') } +RSpec.describe 'Terraform/Base.gitlab-ci.yml' do + subject(:template) { Gitlab::Template::GitlabCiYmlTemplate.find('Terraform/Base') } describe 'the created pipeline' do let(:default_branch) { 'master' } @@ -11,7 +11,7 @@ RSpec.describe 'Terraform/Base.latest.gitlab-ci.yml' do let(:project) { create(:project, :custom_repo, files: { 'README.md' => '' }) } let(:user) { project.owner } let(:service) { Ci::CreatePipelineService.new(project, user, ref: pipeline_branch ) } - let(:pipeline) { service.execute!(:push) } + let(:pipeline) { service.execute!(:push).payload } let(:build_names) { pipeline.builds.pluck(:name) } before do diff --git a/spec/lib/gitlab/ci/templates/Terraform/base_latest_gitlab_ci_yaml_spec.rb b/spec/lib/gitlab/ci/templates/Terraform/base_latest_gitlab_ci_yaml_spec.rb new file mode 100644 index 00000000000..e35f2eabe8e --- /dev/null +++ b/spec/lib/gitlab/ci/templates/Terraform/base_latest_gitlab_ci_yaml_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe 'Terraform/Base.latest.gitlab-ci.yml' do + subject(:template) { Gitlab::Template::GitlabCiYmlTemplate.find('Terraform/Base.latest') } + + describe 'the created pipeline' do + let(:default_branch) { 'master' } + let(:pipeline_branch) { default_branch } + let(:project) { create(:project, :custom_repo, files: { 'README.md' => '' }) } + let(:user) { project.owner } + let(:service) { Ci::CreatePipelineService.new(project, user, ref: pipeline_branch ) } + let(:pipeline) { service.execute!(:push).payload } + let(:build_names) { pipeline.builds.pluck(:name) } + + before do + stub_ci_pipeline_yaml_file(template.content) + allow(project).to receive(:default_branch).and_return(default_branch) + end + + it 'does not create any jobs' do + expect(build_names).to be_empty + end + end +end diff --git a/spec/lib/gitlab/ci/templates/Verify/load_performance_testing_gitlab_ci_yaml_spec.rb b/spec/lib/gitlab/ci/templates/Verify/load_performance_testing_gitlab_ci_yaml_spec.rb index e53d2f4f975..004261bc617 100644 --- a/spec/lib/gitlab/ci/templates/Verify/load_performance_testing_gitlab_ci_yaml_spec.rb +++ b/spec/lib/gitlab/ci/templates/Verify/load_performance_testing_gitlab_ci_yaml_spec.rb @@ -25,7 +25,7 @@ RSpec.describe 'Verify/Load-Performance-Testing.gitlab-ci.yml' do let(:default_branch) { 'master' } let(:pipeline_ref) { default_branch } let(:service) { Ci::CreatePipelineService.new(project, user, ref: pipeline_ref) } - let(:pipeline) { service.execute!(:push) } + let(:pipeline) { service.execute!(:push).payload } let(:build_names) { pipeline.builds.pluck(:name) } before do @@ -64,7 +64,7 @@ RSpec.describe 'Verify/Load-Performance-Testing.gitlab-ci.yml' do context 'on merge request' do let(:service) { MergeRequests::CreatePipelineService.new(project: project, current_user: user) } let(:merge_request) { create(:merge_request, :simple, source_project: project) } - let(:pipeline) { service.execute(merge_request) } + let(:pipeline) { service.execute(merge_request).payload } it 'has no jobs' do expect(pipeline).to be_merge_request_event diff --git a/spec/lib/gitlab/ci/templates/auto_devops_gitlab_ci_yaml_spec.rb b/spec/lib/gitlab/ci/templates/auto_devops_gitlab_ci_yaml_spec.rb index b40b4f5645f..7602309627b 100644 --- a/spec/lib/gitlab/ci/templates/auto_devops_gitlab_ci_yaml_spec.rb +++ b/spec/lib/gitlab/ci/templates/auto_devops_gitlab_ci_yaml_spec.rb @@ -17,7 +17,7 @@ RSpec.describe 'Auto-DevOps.gitlab-ci.yml' do let(:project) { create(:project, :auto_devops, :custom_repo, files: { 'README.md' => '' }) } let(:user) { project.owner } let(:service) { Ci::CreatePipelineService.new(project, user, ref: pipeline_branch ) } - let(:pipeline) { service.execute!(:push) } + let(:pipeline) { service.execute!(:push).payload } let(:build_names) { pipeline.builds.pluck(:name) } before do @@ -264,7 +264,7 @@ RSpec.describe 'Auto-DevOps.gitlab-ci.yml' do let(:project) { create(:project, :custom_repo, files: files) } let(:user) { project.owner } let(:service) { Ci::CreatePipelineService.new(project, user, ref: default_branch ) } - let(:pipeline) { service.execute(:push) } + let(:pipeline) { service.execute(:push).payload } let(:build_names) { pipeline.builds.pluck(:name) } before do diff --git a/spec/lib/gitlab/ci/templates/flutter_gitlab_ci_yaml_spec.rb b/spec/lib/gitlab/ci/templates/flutter_gitlab_ci_yaml_spec.rb index 4e5fe622648..3d97b47473d 100644 --- a/spec/lib/gitlab/ci/templates/flutter_gitlab_ci_yaml_spec.rb +++ b/spec/lib/gitlab/ci/templates/flutter_gitlab_ci_yaml_spec.rb @@ -10,7 +10,7 @@ RSpec.describe 'Flutter.gitlab-ci.yml' do let(:project) { create(:project, :custom_repo, files: { 'README.md' => '' }) } let(:user) { project.owner } let(:service) { Ci::CreatePipelineService.new(project, user, ref: pipeline_branch ) } - let(:pipeline) { service.execute!(:push) } + let(:pipeline) { service.execute!(:push).payload } let(:build_names) { pipeline.builds.pluck(:name) } before do diff --git a/spec/lib/gitlab/ci/templates/managed_cluster_applications_gitlab_ci_yaml_spec.rb b/spec/lib/gitlab/ci/templates/managed_cluster_applications_gitlab_ci_yaml_spec.rb index 151880e27a3..14aaf717453 100644 --- a/spec/lib/gitlab/ci/templates/managed_cluster_applications_gitlab_ci_yaml_spec.rb +++ b/spec/lib/gitlab/ci/templates/managed_cluster_applications_gitlab_ci_yaml_spec.rb @@ -10,7 +10,7 @@ RSpec.describe 'Managed-Cluster-Applications.gitlab-ci.yml' do let(:project) { create(:project, :custom_repo, namespace: user.namespace, files: { 'README.md' => '' }) } let(:service) { Ci::CreatePipelineService.new(project, user, ref: pipeline_branch ) } - let(:pipeline) { service.execute!(:push) } + let(:pipeline) { service.execute!(:push).payload } let(:build_names) { pipeline.builds.pluck(:name) } let(:default_branch) { project.default_branch_or_main } let(:pipeline_branch) { default_branch } diff --git a/spec/lib/gitlab/ci/templates/npm_spec.rb b/spec/lib/gitlab/ci/templates/npm_spec.rb index 2456c9ae545..ea954690133 100644 --- a/spec/lib/gitlab/ci/templates/npm_spec.rb +++ b/spec/lib/gitlab/ci/templates/npm_spec.rb @@ -14,7 +14,7 @@ RSpec.describe 'npm.gitlab-ci.yml' do let(:pipeline_tag) { 'v1.2.1' } let(:pipeline_ref) { pipeline_branch } let(:service) { Ci::CreatePipelineService.new(project, user, ref: pipeline_ref ) } - let(:pipeline) { service.execute!(:push) } + let(:pipeline) { service.execute!(:push).payload } let(:build_names) { pipeline.builds.pluck(:name) } def create_branch(name:) diff --git a/spec/lib/gitlab/ci/templates/terraform_gitlab_ci_yaml_spec.rb b/spec/lib/gitlab/ci/templates/terraform_gitlab_ci_yaml_spec.rb new file mode 100644 index 00000000000..936cd6ac8aa --- /dev/null +++ b/spec/lib/gitlab/ci/templates/terraform_gitlab_ci_yaml_spec.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe 'Terraform.gitlab-ci.yml' do + before do + allow(Gitlab::Template::GitlabCiYmlTemplate).to receive(:excluded_patterns).and_return([]) + end + + subject(:template) { Gitlab::Template::GitlabCiYmlTemplate.find('Terraform') } + + describe 'the created pipeline' do + let(:default_branch) { project.default_branch_or_main } + let(:pipeline_branch) { default_branch } + let(:project) { create(:project, :custom_repo, files: { 'README.md' => '' }) } + let(:user) { project.owner } + let(:service) { Ci::CreatePipelineService.new(project, user, ref: pipeline_branch ) } + let(:pipeline) { service.execute!(:push).payload } + let(:build_names) { pipeline.builds.pluck(:name) } + + before do + stub_ci_pipeline_yaml_file(template.content) + allow(project).to receive(:default_branch).and_return(default_branch) + end + + context 'on master branch' do + it 'creates init, validate and build jobs', :aggregate_failures do + expect(pipeline.errors).to be_empty + expect(build_names).to include('init', 'validate', 'build', 'deploy') + end + end + + context 'outside the master branch' do + let(:pipeline_branch) { 'patch-1' } + + before do + project.repository.create_branch(pipeline_branch, default_branch) + end + + it 'does not creates a deploy and a test job', :aggregate_failures do + expect(pipeline.errors).to be_empty + expect(build_names).not_to include('deploy') + end + end + end +end diff --git a/spec/lib/gitlab/ci/templates/terraform_latest_gitlab_ci_yaml_spec.rb b/spec/lib/gitlab/ci/templates/terraform_latest_gitlab_ci_yaml_spec.rb index 5ab3035486f..3d1306e82a5 100644 --- a/spec/lib/gitlab/ci/templates/terraform_latest_gitlab_ci_yaml_spec.rb +++ b/spec/lib/gitlab/ci/templates/terraform_latest_gitlab_ci_yaml_spec.rb @@ -15,7 +15,7 @@ RSpec.describe 'Terraform.latest.gitlab-ci.yml' do let(:project) { create(:project, :custom_repo, files: { 'README.md' => '' }) } let(:user) { project.owner } let(:service) { Ci::CreatePipelineService.new(project, user, ref: pipeline_branch ) } - let(:pipeline) { service.execute!(:push) } + let(:pipeline) { service.execute!(:push).payload } let(:build_names) { pipeline.builds.pluck(:name) } before do @@ -25,7 +25,8 @@ RSpec.describe 'Terraform.latest.gitlab-ci.yml' do end context 'on master branch' do - it 'creates init, validate and build jobs' do + it 'creates init, validate and build jobs', :aggregate_failures do + expect(pipeline.errors).to be_empty expect(build_names).to include('init', 'validate', 'build', 'deploy') end end @@ -37,7 +38,8 @@ RSpec.describe 'Terraform.latest.gitlab-ci.yml' do project.repository.create_branch(pipeline_branch, default_branch) end - it 'does not creates a deploy and a test job' do + it 'does not creates a deploy and a test job', :aggregate_failures do + expect(pipeline.errors).to be_empty expect(build_names).not_to include('deploy') end end diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb index 19c2e34a0f0..49a470f9e01 100644 --- a/spec/lib/gitlab/ci/yaml_processor_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb @@ -42,7 +42,6 @@ module Gitlab interruptible: true, allow_failure: false, when: "on_success", - yaml_variables: [], job_variables: [], root_variables_inheritance: true, scheduling_type: :stage @@ -75,7 +74,6 @@ module Gitlab ], allow_failure: false, when: 'on_success', - yaml_variables: [], job_variables: [], root_variables_inheritance: true, scheduling_type: :stage @@ -115,7 +113,6 @@ module Gitlab tag_list: %w[A B], allow_failure: false, when: "on_success", - yaml_variables: [], job_variables: [], root_variables_inheritance: true }) @@ -163,7 +160,6 @@ module Gitlab interruptible: true, allow_failure: false, when: "on_success", - yaml_variables: [], job_variables: [], root_variables_inheritance: true, scheduling_type: :stage @@ -354,7 +350,6 @@ module Gitlab name: "rspec", allow_failure: false, when: "on_success", - yaml_variables: [], job_variables: [], root_variables_inheritance: true, scheduling_type: :stage, @@ -368,7 +363,6 @@ module Gitlab name: "prod", allow_failure: false, when: "on_success", - yaml_variables: [], job_variables: [], root_variables_inheritance: true, scheduling_type: :stage, @@ -847,7 +841,6 @@ module Gitlab }, allow_failure: false, when: "on_success", - yaml_variables: [], job_variables: [], root_variables_inheritance: true, scheduling_type: :stage @@ -882,7 +875,6 @@ module Gitlab }, allow_failure: false, when: "on_success", - yaml_variables: [], job_variables: [], root_variables_inheritance: true, scheduling_type: :stage @@ -913,7 +905,6 @@ module Gitlab }, allow_failure: false, when: "on_success", - yaml_variables: [], job_variables: [], root_variables_inheritance: true, scheduling_type: :stage @@ -942,7 +933,6 @@ module Gitlab }, allow_failure: false, when: "on_success", - yaml_variables: [], job_variables: [], root_variables_inheritance: true, scheduling_type: :stage @@ -955,7 +945,6 @@ module Gitlab subject { Gitlab::Ci::YamlProcessor.new(YAML.dump(config)).execute } let(:build) { subject.builds.first } - let(:yaml_variables) { build[:yaml_variables] } let(:job_variables) { build[:job_variables] } let(:root_variables_inheritance) { build[:root_variables_inheritance] } @@ -973,84 +962,11 @@ module Gitlab end it 'returns global variables' do - expect(yaml_variables).to contain_exactly( - { key: 'VAR1', value: 'value1', public: true }, - { key: 'VAR2', value: 'value2', public: true } - ) expect(job_variables).to eq([]) expect(root_variables_inheritance).to eq(true) end end - context 'when job and global variables are defined' do - let(:global_variables) do - { 'VAR1' => 'global1', 'VAR3' => 'global3', 'VAR4' => 'global4' } - end - - let(:build_variables) do - { 'VAR1' => 'value1', 'VAR2' => 'value2' } - end - - let(:config) do - { - before_script: ['pwd'], - variables: global_variables, - rspec: { script: 'rspec', variables: build_variables, inherit: inherit } - } - end - - context 'when no inheritance is specified' do - let(:inherit) { } - - it 'returns all variables' do - expect(yaml_variables).to contain_exactly( - { key: 'VAR1', value: 'value1', public: true }, - { key: 'VAR2', value: 'value2', public: true }, - { key: 'VAR3', value: 'global3', public: true }, - { key: 'VAR4', value: 'global4', public: true } - ) - expect(job_variables).to contain_exactly( - { key: 'VAR1', value: 'value1', public: true }, - { key: 'VAR2', value: 'value2', public: true } - ) - expect(root_variables_inheritance).to eq(true) - end - end - - context 'when inheritance is disabled' do - let(:inherit) { { variables: false } } - - it 'does not inherit variables' do - expect(yaml_variables).to contain_exactly( - { key: 'VAR1', value: 'value1', public: true }, - { key: 'VAR2', value: 'value2', public: true } - ) - expect(job_variables).to contain_exactly( - { key: 'VAR1', value: 'value1', public: true }, - { key: 'VAR2', value: 'value2', public: true } - ) - expect(root_variables_inheritance).to eq(false) - end - end - - context 'when specific variables are to inherited' do - let(:inherit) { { variables: %w[VAR1 VAR4] } } - - it 'returns all variables and inherits only specified variables' do - expect(yaml_variables).to contain_exactly( - { key: 'VAR1', value: 'value1', public: true }, - { key: 'VAR2', value: 'value2', public: true }, - { key: 'VAR4', value: 'global4', public: true } - ) - expect(job_variables).to contain_exactly( - { key: 'VAR1', value: 'value1', public: true }, - { key: 'VAR2', value: 'value2', public: true } - ) - expect(root_variables_inheritance).to eq(%w[VAR1 VAR4]) - end - end - end - context 'when job variables are defined' do let(:config) do { @@ -1065,10 +981,6 @@ module Gitlab end it 'returns job variables' do - expect(yaml_variables).to contain_exactly( - { key: 'VAR1', value: 'value1', public: true }, - { key: 'VAR2', value: 'value2', public: true } - ) expect(job_variables).to contain_exactly( { key: 'VAR1', value: 'value1', public: true }, { key: 'VAR2', value: 'value2', public: true } @@ -1096,9 +1008,6 @@ module Gitlab # When variables config is empty, we assume this is a valid # configuration, see issue #18775 # - expect(yaml_variables).to be_an_instance_of(Array) - expect(yaml_variables).to be_empty - expect(job_variables).to eq([]) expect(root_variables_inheritance).to eq(true) end @@ -1115,9 +1024,6 @@ module Gitlab end it 'returns empty array' do - expect(yaml_variables).to be_an_instance_of(Array) - expect(yaml_variables).to be_empty - expect(job_variables).to eq([]) expect(root_variables_inheritance).to eq(true) end @@ -1246,6 +1152,10 @@ module Gitlab end it { is_expected.to be_valid } + + it 'adds the job from the included file' do + expect(subject.builds.map { |build| build[:name] }).to contain_exactly('job1', 'rspec') + end end context "when the included internal file is not present" do @@ -1349,7 +1259,7 @@ module Gitlab end it 'sets matrix variables' do - build_variables = builds.map { |build| build[:yaml_variables] } + build_variables = builds.map { |build| build[:job_variables] } expected_variables = [ [ { key: 'VAR1', value: '1' }, @@ -1601,7 +1511,6 @@ module Gitlab }, when: "on_success", allow_failure: false, - yaml_variables: [], job_variables: [], root_variables_inheritance: true, scheduling_type: :stage @@ -1972,7 +1881,6 @@ module Gitlab }, when: 'on_success', allow_failure: false, - yaml_variables: [], job_variables: [], root_variables_inheritance: true, scheduling_type: :stage @@ -1988,7 +1896,6 @@ module Gitlab ], when: 'on_success', allow_failure: false, - yaml_variables: [], job_variables: [], root_variables_inheritance: true, scheduling_type: :dag @@ -2011,7 +1918,6 @@ module Gitlab }, when: "on_success", allow_failure: false, - yaml_variables: [], job_variables: [], root_variables_inheritance: true, scheduling_type: :stage @@ -2028,7 +1934,6 @@ module Gitlab ], when: "on_success", allow_failure: false, - yaml_variables: [], job_variables: [], root_variables_inheritance: true, scheduling_type: :dag @@ -2057,7 +1962,6 @@ module Gitlab }, when: "on_success", allow_failure: false, - yaml_variables: [], job_variables: [], root_variables_inheritance: true, scheduling_type: :stage @@ -2076,7 +1980,6 @@ module Gitlab ], when: "on_success", allow_failure: false, - yaml_variables: [], job_variables: [], root_variables_inheritance: true, scheduling_type: :dag @@ -2101,7 +2004,6 @@ module Gitlab ], when: "on_success", allow_failure: false, - yaml_variables: [], job_variables: [], root_variables_inheritance: true, scheduling_type: :dag @@ -2134,7 +2036,6 @@ module Gitlab ], when: "on_success", allow_failure: false, - yaml_variables: [], job_variables: [], root_variables_inheritance: true, scheduling_type: :dag @@ -2342,7 +2243,6 @@ module Gitlab }, when: "on_success", allow_failure: false, - yaml_variables: [], job_variables: [], root_variables_inheritance: true, scheduling_type: :stage @@ -2391,7 +2291,6 @@ module Gitlab }, when: "on_success", allow_failure: false, - yaml_variables: [], job_variables: [], root_variables_inheritance: true, scheduling_type: :stage @@ -2406,7 +2305,6 @@ module Gitlab }, when: "on_success", allow_failure: false, - yaml_variables: [], job_variables: [], root_variables_inheritance: true, scheduling_type: :stage @@ -2851,7 +2749,7 @@ module Gitlab YAML end - it_behaves_like 'returns errors', 'The pipeline has circular dependencies.' + it_behaves_like 'returns errors', 'The pipeline has circular dependencies' end end @@ -2883,7 +2781,7 @@ module Gitlab expect(subject.valid?).to eq(false) expect(subject.errors).to contain_exactly( 'jobs:rspec config contains unknown keys: bad_tags', - 'jobs:rspec rules should be an array of hashes') + 'jobs:rspec rules should be an array containing hashes and arrays of hashes') end end diff --git a/spec/lib/gitlab/config_checker/external_database_checker_spec.rb b/spec/lib/gitlab/config_checker/external_database_checker_spec.rb index 85bafc77553..5a4e9001ac9 100644 --- a/spec/lib/gitlab/config_checker/external_database_checker_spec.rb +++ b/spec/lib/gitlab/config_checker/external_database_checker_spec.rb @@ -8,7 +8,7 @@ RSpec.describe Gitlab::ConfigChecker::ExternalDatabaseChecker do context 'when database meets minimum supported version' do before do - allow(Gitlab::Database).to receive(:postgresql_minimum_supported_version?).and_return(true) + allow(Gitlab::Database.main).to receive(:postgresql_minimum_supported_version?).and_return(true) end it { is_expected.to be_empty } @@ -16,7 +16,7 @@ RSpec.describe Gitlab::ConfigChecker::ExternalDatabaseChecker do context 'when database does not meet minimum supported version' do before do - allow(Gitlab::Database).to receive(:postgresql_minimum_supported_version?).and_return(false) + allow(Gitlab::Database.main).to receive(:postgresql_minimum_supported_version?).and_return(false) end let(:notice_deprecated_database) do @@ -26,7 +26,7 @@ RSpec.describe Gitlab::ConfigChecker::ExternalDatabaseChecker do '%{pg_version_minimum} is required for this version of GitLab. ' \ 'Please upgrade your environment to a supported PostgreSQL version, ' \ 'see %{pg_requirements_url} for details.') % { - pg_version_current: Gitlab::Database.version, + pg_version_current: Gitlab::Database.main.version, pg_version_minimum: Gitlab::Database::MINIMUM_POSTGRES_VERSION, pg_requirements_url: '<a href="https://docs.gitlab.com/ee/install/requirements.html#database">database requirements</a>' } diff --git a/spec/lib/gitlab/conflict/file_spec.rb b/spec/lib/gitlab/conflict/file_spec.rb index f8a007cdd75..aac4936b20e 100644 --- a/spec/lib/gitlab/conflict/file_spec.rb +++ b/spec/lib/gitlab/conflict/file_spec.rb @@ -17,6 +17,18 @@ RSpec.describe Gitlab::Conflict::File do let(:raw_conflict_file) { Gitlab::Git::Conflict::File.new(repository, our_commit.oid, rugged_conflict, raw_conflict_content) } let(:conflict_file) { described_class.new(raw_conflict_file, merge_request: merge_request) } + describe 'delegates' do + it { expect(conflict_file).to delegate_method(:type).to(:raw) } + it { expect(conflict_file).to delegate_method(:content).to(:raw) } + it { expect(conflict_file).to delegate_method(:path).to(:raw) } + it { expect(conflict_file).to delegate_method(:ancestor_path).to(:raw) } + it { expect(conflict_file).to delegate_method(:their_path).to(:raw) } + it { expect(conflict_file).to delegate_method(:our_path).to(:raw) } + it { expect(conflict_file).to delegate_method(:our_mode).to(:raw) } + it { expect(conflict_file).to delegate_method(:our_blob).to(:raw) } + it { expect(conflict_file).to delegate_method(:repository).to(:raw) } + end + describe '#resolve_lines' do let(:section_keys) { conflict_file.sections.map { |section| section[:id] }.compact } @@ -324,4 +336,27 @@ RSpec.describe Gitlab::Conflict::File do end end end + + describe '#conflict_type' do + using RSpec::Parameterized::TableSyntax + + let(:rugged_conflict) { { ancestor: { path: ancestor_path }, theirs: { path: their_path }, ours: { path: our_path } } } + let(:diff_file) { double(renamed_file?: renamed_file?) } + + subject(:conflict_type) { conflict_file.conflict_type(diff_file) } + + where(:ancestor_path, :their_path, :our_path, :renamed_file?, :result) do + '/ancestor/path' | '/their/path' | '/our/path' | false | :both_modified + '/ancestor/path' | '' | '/our/path' | false | :modified_source_removed_target + '/ancestor/path' | '/their/path' | '' | false | :modified_target_removed_source + '' | '/their/path' | '/our/path' | false | :both_added + '' | '' | '/our/path' | false | :removed_target_renamed_source + '' | '' | '/our/path' | true | :renamed_same_file + '' | '/their/path' | '' | false | :removed_source_renamed_target + end + + with_them do + it { expect(conflict_type).to eq(result) } + end + end end diff --git a/spec/lib/gitlab/content_security_policy/config_loader_spec.rb b/spec/lib/gitlab/content_security_policy/config_loader_spec.rb index 8e63e771caa..239eff11bf3 100644 --- a/spec/lib/gitlab/content_security_policy/config_loader_spec.rb +++ b/spec/lib/gitlab/content_security_policy/config_loader_spec.rb @@ -19,14 +19,28 @@ RSpec.describe Gitlab::ContentSecurityPolicy::ConfigLoader do } end - describe '.default_settings_hash' do - let(:settings) { described_class.default_settings_hash } + describe '.default_enabled' do + let(:enabled) { described_class.default_enabled } - it 'returns defaults for all keys' do - expect(settings['enabled']).to be_truthy - expect(settings['report_only']).to be_falsey + it 'is enabled' do + expect(enabled).to be_truthy + end + + context 'when in production' do + before do + allow(Rails).to receive(:env).and_return(ActiveSupport::StringInquirer.new('production')) + end + + it 'is disabled' do + expect(enabled).to be_falsey + end + end + end + + describe '.default_directives' do + let(:directives) { described_class.default_directives } - directives = settings['directives'] + it 'returns default directives' do directive_names = (described_class::DIRECTIVES - ['report_uri']) directive_names.each do |directive| expect(directives.has_key?(directive)).to be_truthy @@ -38,27 +52,25 @@ RSpec.describe Gitlab::ContentSecurityPolicy::ConfigLoader do expect(directives['child_src']).to eq(directives['frame_src']) end - context 'when in production' do + context 'when CDN host is defined' do before do - allow(Rails).to receive(:env).and_return(ActiveSupport::StringInquirer.new('production')) + stub_config_setting(cdn_host: 'https://example.com') end - it 'is disabled' do - expect(settings['enabled']).to be_falsey + it 'adds CDN host to CSP' do + expect(directives['script_src']).to eq("'strict-dynamic' 'self' 'unsafe-inline' 'unsafe-eval' https://www.google.com/recaptcha/ https://www.recaptcha.net https://apis.google.com https://example.com") + expect(directives['style_src']).to eq("'self' 'unsafe-inline' https://example.com") + expect(directives['font_src']).to eq("'self' https://example.com") end end - context 'when GITLAB_CDN_HOST is set' do + context 'when sentry is configured' do before do - stub_env('GITLAB_CDN_HOST', 'https://example.com') + stub_sentry_settings end - it 'adds GITLAB_CDN_HOST to CSP' do - directives = settings['directives'] - - expect(directives['script_src']).to eq("'strict-dynamic' 'self' 'unsafe-inline' 'unsafe-eval' https://www.google.com/recaptcha/ https://www.recaptcha.net https://apis.google.com https://example.com") - expect(directives['style_src']).to eq("'self' 'unsafe-inline' https://example.com") - expect(directives['font_src']).to eq("'self' https://example.com") + it 'adds sentry path to CSP without user' do + expect(directives['connect_src']).to eq("'self' dummy://example.com/43") end end @@ -73,8 +85,6 @@ RSpec.describe Gitlab::ContentSecurityPolicy::ConfigLoader do end it 'does not add CUSTOMER_PORTAL_URL to CSP' do - directives = settings['directives'] - expect(directives['frame_src']).to eq("'self' https://www.google.com/recaptcha/ https://www.recaptcha.net/ https://content.googleapis.com https://content-compute.googleapis.com https://content-cloudbilling.googleapis.com https://content-cloudresourcemanager.googleapis.com") end end @@ -85,8 +95,6 @@ RSpec.describe Gitlab::ContentSecurityPolicy::ConfigLoader do end it 'adds CUSTOMER_PORTAL_URL to CSP' do - directives = settings['directives'] - expect(directives['frame_src']).to eq("'self' https://www.google.com/recaptcha/ https://www.recaptcha.net/ https://content.googleapis.com https://content-compute.googleapis.com https://content-cloudbilling.googleapis.com https://content-cloudresourcemanager.googleapis.com https://customers.example.com") end end diff --git a/spec/lib/gitlab/data_builder/deployment_spec.rb b/spec/lib/gitlab/data_builder/deployment_spec.rb index d64dfc957ca..75741c52579 100644 --- a/spec/lib/gitlab/data_builder/deployment_spec.rb +++ b/spec/lib/gitlab/data_builder/deployment_spec.rb @@ -27,6 +27,7 @@ RSpec.describe Gitlab::DataBuilder::Deployment do expect(data[:status]).to eq('failed') expect(data[:status_changed_at]).to eq(status_changed_at) + expect(data[:deployment_id]).to eq(deployment.id) expect(data[:deployable_id]).to eq(deployable.id) expect(data[:deployable_url]).to eq(expected_deployable_url) expect(data[:environment]).to eq("somewhere") diff --git a/spec/lib/gitlab/data_builder/pipeline_spec.rb b/spec/lib/gitlab/data_builder/pipeline_spec.rb index c05a044f0de..0e574c7aa84 100644 --- a/spec/lib/gitlab/data_builder/pipeline_spec.rb +++ b/spec/lib/gitlab/data_builder/pipeline_spec.rb @@ -3,10 +3,10 @@ require 'spec_helper' RSpec.describe Gitlab::DataBuilder::Pipeline do - let(:user) { create(:user) } - let(:project) { create(:project, :repository) } + let_it_be(:user) { create(:user) } + let_it_be(:project) { create(:project, :repository) } - let(:pipeline) do + let_it_be_with_reload(:pipeline) do create(:ci_pipeline, project: project, status: 'success', @@ -20,7 +20,7 @@ RSpec.describe Gitlab::DataBuilder::Pipeline do describe '.build' do let(:data) { described_class.build(pipeline) } let(:attributes) { data[:object_attributes] } - let(:build_data) { data[:builds].first } + let(:build_data) { data[:builds].last } let(:runner_data) { build_data[:runner] } let(:project_data) { data[:project] } @@ -51,9 +51,9 @@ RSpec.describe Gitlab::DataBuilder::Pipeline do end context 'build with runner' do - let!(:build) { create(:ci_build, pipeline: pipeline, runner: ci_runner) } - let!(:tag_names) { %w(tag-1 tag-2) } - let(:ci_runner) { create(:ci_runner, tag_list: tag_names.map { |n| ActsAsTaggableOn::Tag.create!(name: n)}) } + let_it_be(:tag_names) { %w(tag-1 tag-2) } + let_it_be(:ci_runner) { create(:ci_runner, tag_list: tag_names.map { |n| ActsAsTaggableOn::Tag.create!(name: n)}) } + let_it_be(:build) { create(:ci_build, pipeline: pipeline, runner: ci_runner) } it 'has runner attributes', :aggregate_failures do expect(runner_data[:id]).to eq(ci_runner.id) @@ -73,18 +73,15 @@ RSpec.describe Gitlab::DataBuilder::Pipeline do end context 'pipeline with variables' do - let(:build) { create(:ci_build, pipeline: pipeline) } - let(:data) { described_class.build(pipeline) } - let(:attributes) { data[:object_attributes] } - let!(:pipeline_variable) { create(:ci_pipeline_variable, pipeline: pipeline, key: 'TRIGGER_KEY_1', value: 'TRIGGER_VALUE_1') } + let_it_be(:pipeline_variable) { create(:ci_pipeline_variable, pipeline: pipeline, key: 'TRIGGER_KEY_1', value: 'TRIGGER_VALUE_1') } it { expect(attributes[:variables]).to be_a(Array) } it { expect(attributes[:variables]).to contain_exactly({ key: 'TRIGGER_KEY_1', value: 'TRIGGER_VALUE_1' }) } end context 'when pipeline is a detached merge request pipeline' do - let(:merge_request) { create(:merge_request, :with_detached_merge_request_pipeline) } - let(:pipeline) { merge_request.all_pipelines.first } + let_it_be(:merge_request) { create(:merge_request, :with_detached_merge_request_pipeline) } + let_it_be(:pipeline) { merge_request.all_pipelines.first } it 'returns a source ref' do expect(attributes[:ref]).to eq(merge_request.source_branch) @@ -108,21 +105,67 @@ RSpec.describe Gitlab::DataBuilder::Pipeline do end context 'when pipeline has retried builds' do - before do - create(:ci_build, :retried, pipeline: pipeline) - end + let_it_be(:retried_build) { create(:ci_build, :retried, pipeline: pipeline) } it 'does not contain retried builds in payload' do - expect(data[:builds].count).to eq(1) - expect(build_data[:id]).to eq(build.id) + builds = data[:builds] + + expect(builds.pluck(:id)).to contain_exactly(build.id) + end + + it 'contains retried builds if requested' do + builds = data.with_retried_builds[:builds] + + expect(builds.pluck(:id)).to contain_exactly(build.id, retried_build.id) end end context 'build with environment' do - let!(:build) { create(:ci_build, :teardown_environment, pipeline: pipeline) } + let_it_be(:build) { create(:ci_build, :environment_with_deployment_tier, :with_deployment, pipeline: pipeline) } + + let(:build_environment_data) { build_data[:environment] } + + it 'has environment attributes', :aggregate_failures do + expect(build_environment_data[:name]).to eq(build.expanded_environment_name) + expect(build_environment_data[:action]).to eq(build.environment_action) + expect(build_environment_data[:deployment_tier]).to eq(build.persisted_environment.try(:tier)) + end + end - it { expect(build_data[:environment][:name]).to eq(build.expanded_environment_name) } - it { expect(build_data[:environment][:action]).to eq(build.environment_action) } + context 'avoids N+1 database queries' do + it "with multiple builds" do + # Preparing the pipeline with the minimal builds + pipeline = create(:ci_pipeline, user: user, project: project) + create(:ci_build, user: user, project: project, pipeline: pipeline) + create(:ci_build, :deploy_to_production, :with_deployment, user: user, project: project, pipeline: pipeline) + + # We need `.to_json` as the build hook data is wrapped within `Gitlab::Lazy` + control_count = ActiveRecord::QueryRecorder.new { described_class.build(pipeline.reload).to_json }.count + + # Adding more builds to the pipeline and serializing the data again + create_list(:ci_build, 3, user: user, project: project, pipeline: pipeline) + create(:ci_build, :start_review_app, :with_deployment, user: user, project: project, pipeline: pipeline) + create(:ci_build, :stop_review_app, :with_deployment, user: user, project: project, pipeline: pipeline) + + expect { described_class.build(pipeline.reload).to_json }.not_to exceed_query_limit(control_count) + end + + it "with multiple retried builds" do + # Preparing the pipeline with the minimal builds + pipeline = create(:ci_pipeline, user: user, project: project) + create(:ci_build, :retried, user: user, project: project, pipeline: pipeline) + create(:ci_build, :deploy_to_production, :retried, :with_deployment, user: user, project: project, pipeline: pipeline) + + # We need `.to_json` as the build hook data is wrapped within `Gitlab::Lazy` + control_count = ActiveRecord::QueryRecorder.new { described_class.build(pipeline.reload).with_retried_builds.to_json }.count + + # Adding more builds to the pipeline and serializing the data again + create_list(:ci_build, 3, :retried, user: user, project: project, pipeline: pipeline) + create(:ci_build, :start_review_app, :retried, :with_deployment, user: user, project: project, pipeline: pipeline) + create(:ci_build, :stop_review_app, :retried, :with_deployment, user: user, project: project, pipeline: pipeline) + + expect { described_class.build(pipeline.reload).with_retried_builds.to_json }.not_to exceed_query_limit(control_count) + end end end end diff --git a/spec/lib/gitlab/database/async_indexes/index_creator_spec.rb b/spec/lib/gitlab/database/async_indexes/index_creator_spec.rb new file mode 100644 index 00000000000..b4010d0fe8d --- /dev/null +++ b/spec/lib/gitlab/database/async_indexes/index_creator_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::AsyncIndexes::IndexCreator do + describe '#perform' do + subject { described_class.new(async_index) } + + let(:async_index) { create(:postgres_async_index) } + + let(:index_model) { Gitlab::Database::AsyncIndexes::PostgresAsyncIndex } + + let(:connection) { ApplicationRecord.connection } + + context 'when the index already exists' do + before do + connection.execute(async_index.definition) + end + + it 'skips index creation' do + expect(connection).not_to receive(:execute).with(/CREATE INDEX/) + + subject.perform + end + end + + it 'creates the index while controlling statement timeout' do + allow(connection).to receive(:execute).and_call_original + expect(connection).to receive(:execute).with("SET statement_timeout TO '32400s'").ordered.and_call_original + expect(connection).to receive(:execute).with(async_index.definition).ordered.and_call_original + expect(connection).to receive(:execute).with("RESET statement_timeout").ordered.and_call_original + + subject.perform + end + + it 'removes the index preparation record from postgres_async_indexes' do + expect(async_index).to receive(:destroy).and_call_original + + expect { subject.perform }.to change { index_model.count }.by(-1) + end + + it 'skips logic if not able to acquire exclusive lease' do + expect(subject).to receive(:try_obtain_lease).and_return(false) + expect(connection).not_to receive(:execute).with(/CREATE INDEX/) + expect(async_index).not_to receive(:destroy) + + expect { subject.perform }.not_to change { index_model.count } + end + end +end diff --git a/spec/lib/gitlab/database/async_indexes/migration_helpers_spec.rb b/spec/lib/gitlab/database/async_indexes/migration_helpers_spec.rb new file mode 100644 index 00000000000..ed15951dfb0 --- /dev/null +++ b/spec/lib/gitlab/database/async_indexes/migration_helpers_spec.rb @@ -0,0 +1,176 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::AsyncIndexes::MigrationHelpers do + let(:migration) { ActiveRecord::Migration.new.extend(described_class) } + let(:index_model) { Gitlab::Database::AsyncIndexes::PostgresAsyncIndex } + let(:connection) { ApplicationRecord.connection } + let(:table_name) { '_test_async_indexes' } + let(:index_name) { "index_#{table_name}_on_id" } + + before do + allow(migration).to receive(:puts) + end + + describe '#unprepare_async_index' do + let!(:async_index) { create(:postgres_async_index, name: index_name) } + + context 'when the flag is enabled' do + before do + stub_feature_flags(database_async_index_creation: true) + end + + it 'destroys the record' do + expect do + migration.unprepare_async_index(table_name, 'id') + end.to change { index_model.where(name: index_name).count }.by(-1) + end + + context 'when an explicit name is given' do + let(:index_name) { 'my_test_async_index' } + + it 'destroys the record' do + expect do + migration.unprepare_async_index(table_name, 'id', name: index_name) + end.to change { index_model.where(name: index_name).count }.by(-1) + end + end + + context 'when the async index table does not exist' do + it 'does not raise an error' do + connection.drop_table(:postgres_async_indexes) + + expect(index_model).not_to receive(:find_by) + + expect { migration.unprepare_async_index(table_name, 'id') }.not_to raise_error + end + end + end + + context 'when the feature flag is disabled' do + it 'does not destroy the record' do + stub_feature_flags(database_async_index_creation: false) + + expect do + migration.unprepare_async_index(table_name, 'id') + end.not_to change { index_model.where(name: index_name).count } + end + end + end + + describe '#unprepare_async_index_by_name' do + let(:index_name) { "index_#{table_name}_on_id" } + let!(:async_index) { create(:postgres_async_index, name: index_name) } + + context 'when the flag is enabled' do + before do + stub_feature_flags(database_async_index_creation: true) + end + + it 'destroys the record' do + expect do + migration.unprepare_async_index_by_name(table_name, index_name) + end.to change { index_model.where(name: index_name).count }.by(-1) + end + + context 'when the async index table does not exist' do + it 'does not raise an error' do + connection.drop_table(:postgres_async_indexes) + + expect(index_model).not_to receive(:find_by) + + expect { migration.unprepare_async_index_by_name(table_name, index_name) }.not_to raise_error + end + end + end + + context 'when the feature flag is disabled' do + it 'does not destroy the record' do + stub_feature_flags(database_async_index_creation: false) + + expect do + migration.unprepare_async_index_by_name(table_name, index_name) + end.not_to change { index_model.where(name: index_name).count } + end + end + end + + describe '#prepare_async_index' do + before do + connection.create_table(table_name) + end + + context 'when the feature flag is enabled' do + before do + stub_feature_flags(database_async_index_creation: true) + end + + it 'creates the record for the async index' do + expect do + migration.prepare_async_index(table_name, 'id') + end.to change { index_model.where(name: index_name).count }.by(1) + + record = index_model.find_by(name: index_name) + + expect(record.table_name).to eq(table_name) + expect(record.definition).to match(/CREATE INDEX CONCURRENTLY "#{index_name}"/) + end + + context 'when an explicit name is given' do + let(:index_name) { 'my_async_index_name' } + + it 'creates the record with the given name' do + expect do + migration.prepare_async_index(table_name, 'id', name: index_name) + end.to change { index_model.where(name: index_name).count }.by(1) + + record = index_model.find_by(name: index_name) + + expect(record.table_name).to eq(table_name) + expect(record.definition).to match(/CREATE INDEX CONCURRENTLY "#{index_name}"/) + end + end + + context 'when the index already exists' do + it 'does not create the record' do + connection.add_index(table_name, 'id', name: index_name) + + expect do + migration.prepare_async_index(table_name, 'id') + end.not_to change { index_model.where(name: index_name).count } + end + end + + context 'when the record already exists' do + it 'does attempt to create the record' do + create(:postgres_async_index, table_name: table_name, name: index_name) + + expect do + migration.prepare_async_index(table_name, 'id') + end.not_to change { index_model.where(name: index_name).count } + end + end + + context 'when the async index table does not exist' do + it 'does not raise an error' do + connection.drop_table(:postgres_async_indexes) + + expect(index_model).not_to receive(:safe_find_or_create_by!) + + expect { migration.prepare_async_index(table_name, 'id') }.not_to raise_error + end + end + end + + context 'when the feature flag is disabled' do + it 'does not create the record' do + stub_feature_flags(database_async_index_creation: false) + + expect do + migration.prepare_async_index(table_name, 'id') + end.not_to change { index_model.where(name: index_name).count } + end + end + end +end diff --git a/spec/lib/gitlab/database/async_indexes/postgres_async_index_spec.rb b/spec/lib/gitlab/database/async_indexes/postgres_async_index_spec.rb new file mode 100644 index 00000000000..434cba4edde --- /dev/null +++ b/spec/lib/gitlab/database/async_indexes/postgres_async_index_spec.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::AsyncIndexes::PostgresAsyncIndex, type: :model do + describe 'validations' do + let(:identifier_limit) { described_class::MAX_IDENTIFIER_LENGTH } + let(:definition_limit) { described_class::MAX_DEFINITION_LENGTH } + + it { is_expected.to validate_presence_of(:name) } + it { is_expected.to validate_length_of(:name).is_at_most(identifier_limit) } + it { is_expected.to validate_presence_of(:table_name) } + it { is_expected.to validate_length_of(:table_name).is_at_most(identifier_limit) } + it { is_expected.to validate_presence_of(:definition) } + it { is_expected.to validate_length_of(:definition).is_at_most(definition_limit) } + end +end diff --git a/spec/lib/gitlab/database/async_indexes_spec.rb b/spec/lib/gitlab/database/async_indexes_spec.rb new file mode 100644 index 00000000000..74e30ea2c4e --- /dev/null +++ b/spec/lib/gitlab/database/async_indexes_spec.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::AsyncIndexes do + describe '.create_pending_indexes!' do + subject { described_class.create_pending_indexes! } + + before do + create_list(:postgres_async_index, 4) + end + + it 'takes 2 pending indexes and creates those' do + Gitlab::Database::AsyncIndexes::PostgresAsyncIndex.order(:id).limit(2).each do |index| + creator = double('index creator') + expect(Gitlab::Database::AsyncIndexes::IndexCreator).to receive(:new).with(index).and_return(creator) + expect(creator).to receive(:perform) + end + + subject + end + end +end diff --git a/spec/lib/gitlab/database/connection_spec.rb b/spec/lib/gitlab/database/connection_spec.rb new file mode 100644 index 00000000000..5e0e6039afc --- /dev/null +++ b/spec/lib/gitlab/database/connection_spec.rb @@ -0,0 +1,467 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::Connection do + let(:connection) { described_class.new } + + describe '#default_pool_size' do + before do + allow(Gitlab::Runtime).to receive(:max_threads).and_return(7) + end + + it 'returns the max thread size plus a fixed headroom of 10' do + expect(connection.default_pool_size).to eq(17) + end + + it 'returns the max thread size plus a DB_POOL_HEADROOM if this env var is present' do + stub_env('DB_POOL_HEADROOM', '7') + + expect(connection.default_pool_size).to eq(14) + end + end + + describe '#config' do + it 'returns a HashWithIndifferentAccess' do + expect(connection.config).to be_an_instance_of(HashWithIndifferentAccess) + end + + it 'returns a default pool size' do + expect(connection.config).to include(pool: connection.default_pool_size) + end + + it 'does not cache its results' do + a = connection.config + b = connection.config + + expect(a).not_to equal(b) + end + end + + describe '#pool_size' do + context 'when no explicit size is configured' do + it 'returns the default pool size' do + expect(connection).to receive(:config).and_return({ pool: nil }) + + expect(connection.pool_size).to eq(connection.default_pool_size) + end + end + + context 'when an explicit pool size is set' do + it 'returns the pool size' do + expect(connection).to receive(:config).and_return({ pool: 4 }) + + expect(connection.pool_size).to eq(4) + end + end + end + + describe '#username' do + context 'when a username is set' do + it 'returns the username' do + allow(connection).to receive(:config).and_return(username: 'bob') + + expect(connection.username).to eq('bob') + end + end + + context 'when a username is not set' do + it 'returns the value of the USER environment variable' do + allow(connection).to receive(:config).and_return(username: nil) + allow(ENV).to receive(:[]).with('USER').and_return('bob') + + expect(connection.username).to eq('bob') + end + end + end + + describe '#database_name' do + it 'returns the name of the database' do + allow(connection).to receive(:config).and_return(database: 'test') + + expect(connection.database_name).to eq('test') + end + end + + describe '#adapter_name' do + it 'returns the database adapter name' do + allow(connection).to receive(:config).and_return(adapter: 'test') + + expect(connection.adapter_name).to eq('test') + end + end + + describe '#human_adapter_name' do + context 'when the adapter is PostgreSQL' do + it 'returns PostgreSQL' do + allow(connection).to receive(:config).and_return(adapter: 'postgresql') + + expect(connection.human_adapter_name).to eq('PostgreSQL') + end + end + + context 'when the adapter is not PostgreSQL' do + it 'returns Unknown' do + allow(connection).to receive(:config).and_return(adapter: 'kittens') + + expect(connection.human_adapter_name).to eq('Unknown') + end + end + end + + describe '#postgresql?' do + context 'when using PostgreSQL' do + it 'returns true' do + allow(connection).to receive(:adapter_name).and_return('PostgreSQL') + + expect(connection.postgresql?).to eq(true) + end + end + + context 'when not using PostgreSQL' do + it 'returns false' do + allow(connection).to receive(:adapter_name).and_return('MySQL') + + expect(connection.postgresql?).to eq(false) + end + end + end + + describe '#db_config_with_default_pool_size' do + it 'returns db_config with our default pool size' do + allow(connection).to receive(:default_pool_size).and_return(9) + + expect(connection.db_config_with_default_pool_size.pool).to eq(9) + end + + it 'returns db_config with the correct database name' do + db_name = connection.scope.connection.pool.db_config.name + + expect(connection.db_config_with_default_pool_size.name).to eq(db_name) + end + end + + describe '#disable_prepared_statements' do + around do |example| + original_config = ::Gitlab::Database.main.config + + example.run + + connection.scope.establish_connection(original_config) + end + + it 'disables prepared statements' do + connection.scope.establish_connection( + ::Gitlab::Database.main.config.merge(prepared_statements: true) + ) + + expect(connection.scope.connection.prepared_statements).to eq(true) + + connection.disable_prepared_statements + + expect(connection.scope.connection.prepared_statements).to eq(false) + end + + context 'with dynamic connection pool size' do + before do + connection.scope.establish_connection(connection.config.merge(pool: 7)) + end + + it 'retains the set pool size' do + connection.disable_prepared_statements + + expect(connection.scope.connection.prepared_statements).to eq(false) + expect(connection.scope.connection.pool.size).to eq(7) + end + end + end + + describe '#db_read_only?' do + it 'detects a read-only database' do + allow(connection.scope.connection) + .to receive(:execute) + .with('SELECT pg_is_in_recovery()') + .and_return([{ "pg_is_in_recovery" => "t" }]) + + expect(connection.db_read_only?).to be_truthy + end + + it 'detects a read-only database' do + allow(connection.scope.connection) + .to receive(:execute) + .with('SELECT pg_is_in_recovery()') + .and_return([{ "pg_is_in_recovery" => true }]) + + expect(connection.db_read_only?).to be_truthy + end + + it 'detects a read-write database' do + allow(connection.scope.connection) + .to receive(:execute) + .with('SELECT pg_is_in_recovery()') + .and_return([{ "pg_is_in_recovery" => "f" }]) + + expect(connection.db_read_only?).to be_falsey + end + + it 'detects a read-write database' do + allow(connection.scope.connection) + .to receive(:execute) + .with('SELECT pg_is_in_recovery()') + .and_return([{ "pg_is_in_recovery" => false }]) + + expect(connection.db_read_only?).to be_falsey + end + end + + describe '#db_read_write?' do + it 'detects a read-only database' do + allow(connection.scope.connection) + .to receive(:execute) + .with('SELECT pg_is_in_recovery()') + .and_return([{ "pg_is_in_recovery" => "t" }]) + + expect(connection.db_read_write?).to eq(false) + end + + it 'detects a read-only database' do + allow(connection.scope.connection) + .to receive(:execute) + .with('SELECT pg_is_in_recovery()') + .and_return([{ "pg_is_in_recovery" => true }]) + + expect(connection.db_read_write?).to eq(false) + end + + it 'detects a read-write database' do + allow(connection.scope.connection) + .to receive(:execute) + .with('SELECT pg_is_in_recovery()') + .and_return([{ "pg_is_in_recovery" => "f" }]) + + expect(connection.db_read_write?).to eq(true) + end + + it 'detects a read-write database' do + allow(connection.scope.connection) + .to receive(:execute) + .with('SELECT pg_is_in_recovery()') + .and_return([{ "pg_is_in_recovery" => false }]) + + expect(connection.db_read_write?).to eq(true) + end + end + + describe '#version' do + around do |example| + connection.instance_variable_set(:@version, nil) + example.run + connection.instance_variable_set(:@version, nil) + end + + context "on postgresql" do + it "extracts the version number" do + allow(connection) + .to receive(:database_version) + .and_return("PostgreSQL 9.4.4 on x86_64-apple-darwin14.3.0") + + expect(connection.version).to eq '9.4.4' + end + end + + it 'memoizes the result' do + count = ActiveRecord::QueryRecorder + .new { 2.times { connection.version } } + .count + + expect(count).to eq(1) + end + end + + describe '#postgresql_minimum_supported_version?' do + it 'returns false when using PostgreSQL 10' do + allow(connection).to receive(:version).and_return('10') + + expect(connection.postgresql_minimum_supported_version?).to eq(false) + end + + it 'returns false when using PostgreSQL 11' do + allow(connection).to receive(:version).and_return('11') + + expect(connection.postgresql_minimum_supported_version?).to eq(false) + end + + it 'returns true when using PostgreSQL 12' do + allow(connection).to receive(:version).and_return('12') + + expect(connection.postgresql_minimum_supported_version?).to eq(true) + end + end + + describe '#bulk_insert' do + before do + allow(connection).to receive(:connection).and_return(dummy_connection) + allow(dummy_connection).to receive(:quote_column_name, &:itself) + allow(dummy_connection).to receive(:quote, &:itself) + allow(dummy_connection).to receive(:execute) + end + + let(:dummy_connection) { double(:connection) } + + let(:rows) do + [ + { a: 1, b: 2, c: 3 }, + { c: 6, a: 4, b: 5 } + ] + end + + it 'does nothing with empty rows' do + expect(dummy_connection).not_to receive(:execute) + + connection.bulk_insert('test', []) + end + + it 'uses the ordering from the first row' do + expect(dummy_connection).to receive(:execute) do |sql| + expect(sql).to include('(1, 2, 3)') + expect(sql).to include('(4, 5, 6)') + end + + connection.bulk_insert('test', rows) + end + + it 'quotes column names' do + expect(dummy_connection).to receive(:quote_column_name).with(:a) + expect(dummy_connection).to receive(:quote_column_name).with(:b) + expect(dummy_connection).to receive(:quote_column_name).with(:c) + + connection.bulk_insert('test', rows) + end + + it 'quotes values' do + 1.upto(6) do |i| + expect(dummy_connection).to receive(:quote).with(i) + end + + connection.bulk_insert('test', rows) + end + + it 'does not quote values of a column in the disable_quote option' do + [1, 2, 4, 5].each do |i| + expect(dummy_connection).to receive(:quote).with(i) + end + + connection.bulk_insert('test', rows, disable_quote: :c) + end + + it 'does not quote values of columns in the disable_quote option' do + [2, 5].each do |i| + expect(dummy_connection).to receive(:quote).with(i) + end + + connection.bulk_insert('test', rows, disable_quote: [:a, :c]) + end + + it 'handles non-UTF-8 data' do + expect { connection.bulk_insert('test', [{ a: "\255" }]) }.not_to raise_error + end + + context 'when using PostgreSQL' do + it 'allows the returning of the IDs of the inserted rows' do + result = double(:result, values: [['10']]) + + expect(dummy_connection) + .to receive(:execute) + .with(/RETURNING id/) + .and_return(result) + + ids = connection + .bulk_insert('test', [{ number: 10 }], return_ids: true) + + expect(ids).to eq([10]) + end + + it 'allows setting the upsert to do nothing' do + expect(dummy_connection) + .to receive(:execute) + .with(/ON CONFLICT DO NOTHING/) + + connection + .bulk_insert('test', [{ number: 10 }], on_conflict: :do_nothing) + end + end + end + + describe '#cached_column_exists?' do + it 'only retrieves data once' do + expect(connection.scope.connection) + .to receive(:columns) + .once.and_call_original + + 2.times do + expect(connection.cached_column_exists?(:projects, :id)).to be_truthy + expect(connection.cached_column_exists?(:projects, :bogus_column)).to be_falsey + end + end + end + + describe '#cached_table_exists?' do + it 'only retrieves data once per table' do + expect(connection.scope.connection) + .to receive(:data_source_exists?) + .with(:projects) + .once.and_call_original + + expect(connection.scope.connection) + .to receive(:data_source_exists?) + .with(:bogus_table_name) + .once.and_call_original + + 2.times do + expect(connection.cached_table_exists?(:projects)).to be_truthy + expect(connection.cached_table_exists?(:bogus_table_name)).to be_falsey + end + end + + it 'returns false when database does not exist' do + expect(connection.scope).to receive(:connection) do + raise ActiveRecord::NoDatabaseError, 'broken' + end + + expect(connection.cached_table_exists?(:projects)).to be(false) + end + end + + describe '#exists?' do + it 'returns true if `ActiveRecord::Base.connection` succeeds' do + expect(connection.scope).to receive(:connection) + + expect(connection.exists?).to be(true) + end + + it 'returns false if `ActiveRecord::Base.connection` fails' do + expect(connection.scope).to receive(:connection) do + raise ActiveRecord::NoDatabaseError, 'broken' + end + + expect(connection.exists?).to be(false) + end + end + + describe '#system_id' do + it 'returns the PostgreSQL system identifier' do + expect(connection.system_id).to be_an_instance_of(Integer) + end + end + + describe '#get_write_location' do + it 'returns a string' do + expect(connection.get_write_location(connection.scope.connection)) + .to be_a(String) + end + + it 'returns nil if there are no results' do + expect(connection.get_write_location(double(select_all: []))).to be_nil + end + end +end diff --git a/spec/lib/gitlab/database/load_balancing/connection_proxy_spec.rb b/spec/lib/gitlab/database/load_balancing/connection_proxy_spec.rb index 015dd2ba8d2..0ca99ec9acf 100644 --- a/spec/lib/gitlab/database/load_balancing/connection_proxy_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/connection_proxy_spec.rb @@ -7,7 +7,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::ConnectionProxy do describe '#select' do it 'performs a read' do - expect(proxy).to receive(:read_using_load_balancer).with(:select, ['foo']) + expect(proxy).to receive(:read_using_load_balancer).with(:select, 'foo') proxy.select('foo') end @@ -26,7 +26,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::ConnectionProxy do arel = double(:arel) expect(proxy).to receive(:read_using_load_balancer) - .with(:select_all, [arel, 'foo', []]) + .with(:select_all, arel, 'foo', []) proxy.select_all(arel, 'foo') end @@ -37,7 +37,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::ConnectionProxy do arel = double(:arel, locked: true) expect(proxy).to receive(:write_using_load_balancer) - .with(:select_all, [arel, 'foo', []], sticky: true) + .with(:select_all, arel, 'foo', [], sticky: true) proxy.select_all(arel, 'foo') end @@ -48,7 +48,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::ConnectionProxy do describe "#{name}" do it 'runs the query on the replica' do expect(proxy).to receive(:read_using_load_balancer) - .with(name, ['foo']) + .with(name, 'foo') proxy.send(name, 'foo') end @@ -59,7 +59,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::ConnectionProxy do describe "#{name}" do it 'runs the query on the primary and sticks to it' do expect(proxy).to receive(:write_using_load_balancer) - .with(name, ['foo'], sticky: true) + .with(name, 'foo', sticky: true) proxy.send(name, 'foo') end @@ -187,7 +187,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::ConnectionProxy do describe '#method_missing' do it 'runs the query on the primary without sticking to it' do expect(proxy).to receive(:write_using_load_balancer) - .with(:foo, ['foo']) + .with(:foo, 'foo') proxy.foo('foo') end @@ -197,7 +197,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::ConnectionProxy do expect(proxy).to receive(:write_using_load_balancer).and_call_original - expect { proxy.case_sensitive_comparison(:table, :attribute, :column, { value: :value, format: :format }) } + expect { proxy.case_sensitive_comparison(:table, :attribute, :column, value: :value, format: :format) } .not_to raise_error end @@ -212,7 +212,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::ConnectionProxy do end it 'runs the query on the replica' do - expect(proxy).to receive(:read_using_load_balancer).with(:foo, ['foo']) + expect(proxy).to receive(:read_using_load_balancer).with(:foo, 'foo') proxy.foo('foo') end @@ -222,7 +222,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::ConnectionProxy do expect(proxy).to receive(:read_using_load_balancer).and_call_original - expect { proxy.case_sensitive_comparison(:table, :attribute, :column, { value: :value, format: :format }) } + expect { proxy.case_sensitive_comparison(:table, :attribute, :column, value: :value, format: :format) } .not_to raise_error end end @@ -245,7 +245,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::ConnectionProxy do expect(connection).to receive(:foo).with('foo') expect(proxy.load_balancer).to receive(:read).and_yield(connection) - proxy.read_using_load_balancer(:foo, ['foo']) + proxy.read_using_load_balancer(:foo, 'foo') end end @@ -257,7 +257,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::ConnectionProxy do expect(connection).to receive(:foo).with('foo') expect(proxy.load_balancer).to receive(:read).and_yield(connection) - proxy.read_using_load_balancer(:foo, ['foo']) + proxy.read_using_load_balancer(:foo, 'foo') end end @@ -269,7 +269,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::ConnectionProxy do expect(connection).to receive(:foo).with('foo') expect(proxy.load_balancer).to receive(:read).and_yield(connection) - proxy.read_using_load_balancer(:foo, ['foo']) + proxy.read_using_load_balancer(:foo, 'foo') end end @@ -283,7 +283,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::ConnectionProxy do expect(proxy.load_balancer).to receive(:read_write) .and_yield(connection) - proxy.read_using_load_balancer(:foo, ['foo']) + proxy.read_using_load_balancer(:foo, 'foo') end end end @@ -302,7 +302,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::ConnectionProxy do expect(connection).to receive(:foo).with('foo') expect(session).not_to receive(:write!) - proxy.write_using_load_balancer(:foo, ['foo']) + proxy.write_using_load_balancer(:foo, 'foo') end it 'sticks to the primary when sticking is enabled' do @@ -310,7 +310,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::ConnectionProxy do expect(connection).to receive(:foo).with('foo') expect(session).to receive(:write!) - proxy.write_using_load_balancer(:foo, ['foo'], sticky: true) + proxy.write_using_load_balancer(:foo, 'foo', sticky: true) end end end diff --git a/spec/lib/gitlab/database/load_balancing/host_list_spec.rb b/spec/lib/gitlab/database/load_balancing/host_list_spec.rb index 873b599f84d..ad4ca18d5e6 100644 --- a/spec/lib/gitlab/database/load_balancing/host_list_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/host_list_spec.rb @@ -3,25 +3,17 @@ require 'spec_helper' RSpec.describe Gitlab::Database::LoadBalancing::HostList do - def expect_metrics(hosts) - expect(Gitlab::Metrics.registry.get(:db_load_balancing_hosts).get({})).to eq(hosts) - end - - before do - allow(Gitlab::Database) - .to receive(:create_connection_pool) - .and_return(ActiveRecord::Base.connection_pool) - end - + let(:db_host) { ActiveRecord::Base.connection_pool.db_config.host } let(:load_balancer) { double(:load_balancer) } let(:host_count) { 2 } + let(:hosts) { Array.new(host_count) { Gitlab::Database::LoadBalancing::Host.new(db_host, load_balancer, port: 5432) } } + let(:host_list) { described_class.new(hosts) } - let(:host_list) do - hosts = Array.new(host_count) do - Gitlab::Database::LoadBalancing::Host.new('localhost', load_balancer, port: 5432) + before do + # each call generate a new replica pool + allow(load_balancer).to receive(:create_replica_connection_pool) do + double(:replica_connection_pool) end - - described_class.new(hosts) end describe '#initialize' do @@ -42,8 +34,8 @@ RSpec.describe Gitlab::Database::LoadBalancing::HostList do context 'with ports' do it 'returns the host names of all hosts' do hosts = [ - ['localhost', 5432], - ['localhost', 5432] + [db_host, 5432], + [db_host, 5432] ] expect(host_list.host_names_and_ports).to eq(hosts) @@ -51,18 +43,12 @@ RSpec.describe Gitlab::Database::LoadBalancing::HostList do end context 'without ports' do - let(:host_list) do - hosts = Array.new(2) do - Gitlab::Database::LoadBalancing::Host.new('localhost', load_balancer) - end - - described_class.new(hosts) - end + let(:hosts) { Array.new(2) { Gitlab::Database::LoadBalancing::Host.new(db_host, load_balancer) } } it 'returns the host names of all hosts' do hosts = [ - ['localhost', nil], - ['localhost', nil] + [db_host, nil], + [db_host, nil] ] expect(host_list.host_names_and_ports).to eq(hosts) @@ -70,48 +56,6 @@ RSpec.describe Gitlab::Database::LoadBalancing::HostList do end end - describe '#manage_pool?' do - before do - allow(Gitlab::Database).to receive(:create_connection_pool) { double(:connection) } - end - - context 'when the testing pool belongs to one host of the host list' do - it 'returns true' do - pool = host_list.hosts.first.pool - - expect(host_list.manage_pool?(pool)).to be(true) - end - end - - context 'when the testing pool belongs to a former host of the host list' do - it 'returns false' do - pool = host_list.hosts.first.pool - host_list.hosts = [ - Gitlab::Database::LoadBalancing::Host.new('foo', load_balancer) - ] - - expect(host_list.manage_pool?(pool)).to be(false) - end - end - - context 'when the testing pool belongs to a new host of the host list' do - it 'returns true' do - host = Gitlab::Database::LoadBalancing::Host.new('foo', load_balancer) - host_list.hosts = [host] - - expect(host_list.manage_pool?(host.pool)).to be(true) - end - end - - context 'when the testing pool does not have any relation with the host list' do - it 'returns false' do - host = Gitlab::Database::LoadBalancing::Host.new('foo', load_balancer) - - expect(host_list.manage_pool?(host.pool)).to be(false) - end - end - end - describe '#hosts' do it 'returns a copy of the host' do first = host_list.hosts @@ -185,4 +129,8 @@ RSpec.describe Gitlab::Database::LoadBalancing::HostList do end end end + + def expect_metrics(hosts) + expect(Gitlab::Metrics.registry.get(:db_load_balancing_hosts).get({})).to eq(hosts) + end end diff --git a/spec/lib/gitlab/database/load_balancing/host_spec.rb b/spec/lib/gitlab/database/load_balancing/host_spec.rb index 4dfddef68c8..f42ac8be1bb 100644 --- a/spec/lib/gitlab/database/load_balancing/host_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/host_spec.rb @@ -3,15 +3,16 @@ require 'spec_helper' RSpec.describe Gitlab::Database::LoadBalancing::Host do - let(:load_balancer) do - Gitlab::Database::LoadBalancing::LoadBalancer.new(%w[localhost]) - end + let(:load_balancer) { Gitlab::Database::LoadBalancing::LoadBalancer.new } - let(:host) { load_balancer.host_list.hosts.first } + let(:host) do + Gitlab::Database::LoadBalancing::Host.new('localhost', load_balancer) + end before do - allow(Gitlab::Database).to receive(:create_connection_pool) - .and_return(ActiveRecord::Base.connection_pool) + allow(load_balancer).to receive(:create_replica_connection_pool) do + ActiveRecord::Base.connection_pool + end end def raise_and_wrap(wrapper, original) @@ -63,7 +64,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::Host do expect(host.pool) .to receive(:disconnect!) - host.disconnect!(1) + host.disconnect!(timeout: 1) end end diff --git a/spec/lib/gitlab/database/load_balancing/load_balancer_spec.rb b/spec/lib/gitlab/database/load_balancing/load_balancer_spec.rb index b82b8d9a311..c647f5a8f5d 100644 --- a/spec/lib/gitlab/database/load_balancing/load_balancer_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/load_balancer_spec.rb @@ -3,20 +3,22 @@ require 'spec_helper' RSpec.describe Gitlab::Database::LoadBalancing::LoadBalancer, :request_store do - let(:pool) { Gitlab::Database.create_connection_pool(2) } let(:conflict_error) { Class.new(RuntimeError) } - - let(:lb) { described_class.new(%w(localhost localhost)) } + let(:db_host) { ActiveRecord::Base.connection_pool.db_config.host } + let(:lb) { described_class.new([db_host, db_host]) } + let(:request_cache) { lb.send(:request_cache) } before do - allow(Gitlab::Database).to receive(:create_connection_pool) - .and_return(pool) stub_const( 'Gitlab::Database::LoadBalancing::LoadBalancer::PG::TRSerializationFailure', conflict_error ) end + after do |example| + lb.disconnect!(timeout: 0) unless example.metadata[:skip_disconnect] + end + def raise_and_wrap(wrapper, original) raise original rescue original.class @@ -123,8 +125,9 @@ RSpec.describe Gitlab::Database::LoadBalancing::LoadBalancer, :request_store do describe '#read_write' do it 'yields a connection for a write' do - expect { |b| lb.read_write(&b) } - .to yield_with_args(ActiveRecord::Base.retrieve_connection) + connection = ActiveRecord::Base.connection_pool.connection + + expect { |b| lb.read_write(&b) }.to yield_with_args(connection) end it 'uses a retry with exponential backoffs' do @@ -134,140 +137,30 @@ RSpec.describe Gitlab::Database::LoadBalancing::LoadBalancer, :request_store do end end - describe '#db_role_for_connection' do - context 'when the load balancer creates the connection with #read' do - it 'returns :replica' do - role = nil - lb.read do |connection| - role = lb.db_role_for_connection(connection) - end - - expect(role).to be(:replica) - end - end - - context 'when the load balancer uses nested #read' do - it 'returns :replica' do - roles = [] - lb.read do |connection_1| - lb.read do |connection_2| - roles << lb.db_role_for_connection(connection_2) - end - roles << lb.db_role_for_connection(connection_1) - end - - expect(roles).to eq([:replica, :replica]) - end - end - - context 'when the load balancer creates the connection with #read_write' do - it 'returns :primary' do - role = nil - lb.read_write do |connection| - role = lb.db_role_for_connection(connection) - end - - expect(role).to be(:primary) - end - end - - context 'when the load balancer uses nested #read_write' do - it 'returns :primary' do - roles = [] - lb.read_write do |connection_1| - lb.read_write do |connection_2| - roles << lb.db_role_for_connection(connection_2) - end - roles << lb.db_role_for_connection(connection_1) - end - - expect(roles).to eq([:primary, :primary]) - end - end - - context 'when the load balancer falls back the connection creation to primary' do - it 'returns :primary' do - allow(lb).to receive(:serialization_failure?).and_return(true) - - role = nil - raised = 7 # 2 hosts = 6 retries - - lb.read do |connection| - if raised > 0 - raised -= 1 - raise - end - - role = lb.db_role_for_connection(connection) - end - - expect(role).to be(:primary) - end - end - - context 'when the load balancer uses replica after recovery from a failure' do - it 'returns :replica' do - allow(lb).to receive(:connection_error?).and_return(true) - - role = nil - raised = false - - lb.read do |connection| - unless raised - raised = true - raise - end - - role = lb.db_role_for_connection(connection) - end - - expect(role).to be(:replica) - end - end - - context 'when the connection comes from a pool managed by the host list' do - it 'returns :replica' do - connection = double(:connection) - allow(connection).to receive(:pool).and_return(lb.host_list.hosts.first.pool) - - expect(lb.db_role_for_connection(connection)).to be(:replica) - end - end - - context 'when the connection comes from the primary pool' do - it 'returns :primary' do - connection = double(:connection) - allow(connection).to receive(:pool).and_return(ActiveRecord::Base.connection_pool) - - expect(lb.db_role_for_connection(connection)).to be(:primary) - end - end - - context 'when the connection does not come from any known pool' do - it 'returns nil' do - connection = double(:connection) - pool = double(:connection_pool) - allow(connection).to receive(:pool).and_return(pool) - - expect(lb.db_role_for_connection(connection)).to be(nil) - end - end - end - describe '#host' do it 'returns the secondary host to use' do expect(lb.host).to be_an_instance_of(Gitlab::Database::LoadBalancing::Host) end it 'stores the host in a thread-local variable' do - RequestStore.delete(described_class::CACHE_KEY) - RequestStore.delete(described_class::VALID_HOSTS_CACHE_KEY) + request_cache.delete(described_class::CACHE_KEY) expect(lb.host_list).to receive(:next).once.and_call_original lb.host lb.host end + + it 'does not create conflicts with other load balancers when caching hosts' do + lb1 = described_class.new([db_host, db_host], ActiveRecord::Base) + lb2 = described_class.new([db_host, db_host], Ci::CiDatabaseRecord) + + host1 = lb1.host + host2 = lb2.host + + expect(lb1.send(:request_cache)[described_class::CACHE_KEY]).to eq(host1) + expect(lb2.send(:request_cache)[described_class::CACHE_KEY]).to eq(host2) + end end describe '#release_host' do @@ -278,8 +171,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::LoadBalancer, :request_store do lb.release_host - expect(RequestStore[described_class::CACHE_KEY]).to be_nil - expect(RequestStore[described_class::VALID_HOSTS_CACHE_KEY]).to be_nil + expect(request_cache[described_class::CACHE_KEY]).to be_nil end end @@ -414,89 +306,76 @@ RSpec.describe Gitlab::Database::LoadBalancing::LoadBalancer, :request_store do end end - describe '#select_caught_up_hosts' do + describe '#select_up_to_date_host' do let(:location) { 'AB/12345'} let(:hosts) { lb.host_list.hosts } - let(:valid_host_list) { RequestStore[described_class::VALID_HOSTS_CACHE_KEY] } - let(:valid_hosts) { valid_host_list.hosts } + let(:set_host) { request_cache[described_class::CACHE_KEY] } - subject { lb.select_caught_up_hosts(location) } - - context 'when all replicas are caught up' do - before do - expect(hosts).to all(receive(:caught_up?).with(location).and_return(true)) - end - - it 'returns true and sets all hosts to valid' do - expect(subject).to be true - expect(valid_host_list).to be_a(Gitlab::Database::LoadBalancing::HostList) - expect(valid_hosts).to contain_exactly(*hosts) - end - end + subject { lb.select_up_to_date_host(location) } context 'when none of the replicas are caught up' do before do expect(hosts).to all(receive(:caught_up?).with(location).and_return(false)) end - it 'returns false and does not set the valid hosts' do + it 'returns false and does not update the host thread-local variable' do expect(subject).to be false - expect(valid_host_list).to be_nil + expect(set_host).to be_nil end end - context 'when one of the replicas is caught up' do + context 'when any of the replicas is caught up' do before do - expect(hosts[0]).to receive(:caught_up?).with(location).and_return(false) + # `allow` for non-caught up host, because we may not even check it, if will find the caught up one earlier + allow(hosts[0]).to receive(:caught_up?).with(location).and_return(false) expect(hosts[1]).to receive(:caught_up?).with(location).and_return(true) end - it 'returns true and sets one host to valid' do + it 'returns true and sets host thread-local variable' do expect(subject).to be true - expect(valid_host_list).to be_a(Gitlab::Database::LoadBalancing::HostList) - expect(valid_hosts).to contain_exactly(hosts[1]) - end - - it 'host always returns the caught-up replica' do - subject - - 3.times do - expect(lb.host).to eq(hosts[1]) - RequestStore.delete(described_class::CACHE_KEY) - end + expect(set_host).to eq(hosts[1]) end end end - describe '#select_up_to_date_host' do - let(:location) { 'AB/12345'} - let(:hosts) { lb.host_list.hosts } - let(:set_host) { RequestStore[described_class::CACHE_KEY] } + describe '#create_replica_connection_pool' do + it 'creates a new connection pool with specific pool size and name' do + with_replica_pool(5, 'other_host') do |replica_pool| + expect(replica_pool) + .to be_kind_of(ActiveRecord::ConnectionAdapters::ConnectionPool) - subject { lb.select_up_to_date_host(location) } - - context 'when none of the replicas are caught up' do - before do - expect(hosts).to all(receive(:caught_up?).with(location).and_return(false)) + expect(replica_pool.db_config.host).to eq('other_host') + expect(replica_pool.db_config.pool).to eq(5) + expect(replica_pool.db_config.name).to end_with("_replica") end + end - it 'returns false and does not update the host thread-local variable' do - expect(subject).to be false - expect(set_host).to be_nil + it 'allows setting of a custom hostname and port' do + with_replica_pool(5, 'other_host', 5432) do |replica_pool| + expect(replica_pool.db_config.host).to eq('other_host') + expect(replica_pool.db_config.configuration_hash[:port]).to eq(5432) end end - context 'when any of the replicas is caught up' do - before do - # `allow` for non-caught up host, because we may not even check it, if will find the caught up one earlier - allow(hosts[0]).to receive(:caught_up?).with(location).and_return(false) - expect(hosts[1]).to receive(:caught_up?).with(location).and_return(true) - end + it 'does not modify connection class pool' do + expect { with_replica_pool(5) { } }.not_to change { ActiveRecord::Base.connection_pool } + end - it 'returns true and sets host thread-local variable' do - expect(subject).to be true - expect(set_host).to eq(hosts[1]) + def with_replica_pool(*args) + pool = lb.create_replica_connection_pool(*args) + yield pool + ensure + pool&.disconnect! + end + end + + describe '#disconnect!' do + it 'calls disconnect on all hosts with a timeout', :skip_disconnect do + expect_next_instances_of(Gitlab::Database::LoadBalancing::Host, 2) do |host| + expect(host).to receive(:disconnect!).with(timeout: 30) end + + lb.disconnect!(timeout: 30) end end end diff --git a/spec/lib/gitlab/database/load_balancing/rack_middleware_spec.rb b/spec/lib/gitlab/database/load_balancing/rack_middleware_spec.rb index 9381ffa59fe..ea0c7f781fd 100644 --- a/spec/lib/gitlab/database/load_balancing/rack_middleware_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/rack_middleware_spec.rb @@ -183,18 +183,17 @@ RSpec.describe Gitlab::Database::LoadBalancing::RackMiddleware, :redis do describe '#clear' do it 'clears the currently used host and session' do lb = double(:lb) - session = double(:session) + session = spy(:session) allow(middleware).to receive(:load_balancer).and_return(lb) expect(lb).to receive(:release_host) - stub_const('Gitlab::Database::LoadBalancing::RackMiddleware::Session', - session) - - expect(session).to receive(:clear_session) + stub_const('Gitlab::Database::LoadBalancing::Session', session) middleware.clear + + expect(session).to have_received(:clear_session) end end diff --git a/spec/lib/gitlab/database/load_balancing/service_discovery_spec.rb b/spec/lib/gitlab/database/load_balancing/service_discovery_spec.rb index 7fc7b5e8d11..a27341a3324 100644 --- a/spec/lib/gitlab/database/load_balancing/service_discovery_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/service_discovery_spec.rb @@ -3,8 +3,14 @@ require 'spec_helper' RSpec.describe Gitlab::Database::LoadBalancing::ServiceDiscovery do + let(:load_balancer) { Gitlab::Database::LoadBalancing::LoadBalancer.new([]) } let(:service) do - described_class.new(nameserver: 'localhost', port: 8600, record: 'foo') + described_class.new( + nameserver: 'localhost', + port: 8600, + record: 'foo', + load_balancer: load_balancer + ) end before do @@ -18,7 +24,15 @@ RSpec.describe Gitlab::Database::LoadBalancing::ServiceDiscovery do describe '#initialize' do describe ':record_type' do - subject { described_class.new(nameserver: 'localhost', port: 8600, record: 'foo', record_type: record_type) } + subject do + described_class.new( + nameserver: 'localhost', + port: 8600, + record: 'foo', + record_type: record_type, + load_balancer: load_balancer + ) + end context 'with a supported type' do let(:record_type) { 'SRV' } @@ -44,21 +58,17 @@ RSpec.describe Gitlab::Database::LoadBalancing::ServiceDiscovery do end it 'starts service discovery in a new thread' do - expect(service) - .to receive(:refresh_if_necessary) - .and_return(5) - - expect(service) - .to receive(:rand) - .and_return(2) + expect(Thread).to receive(:new).ordered.and_call_original # Thread starts - expect(service) - .to receive(:sleep) - .with(7) + expect(service).to receive(:perform_service_discovery).ordered.and_return(5) + expect(service).to receive(:rand).ordered.and_return(2) + expect(service).to receive(:sleep).ordered.with(7) # Sleep runs after thread starts service.start.join end + end + describe '#perform_service_discovery' do it 'reports exceptions to Sentry' do error = StandardError.new @@ -70,15 +80,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::ServiceDiscovery do .to receive(:track_exception) .with(error) - expect(service) - .to receive(:rand) - .and_return(2) - - expect(service) - .to receive(:sleep) - .with(62) - - service.start.join + service.perform_service_discovery end end @@ -155,14 +157,23 @@ RSpec.describe Gitlab::Database::LoadBalancing::ServiceDiscovery do expect(host) .to receive(:disconnect!) - .with(2) + .with(timeout: 2) service.replace_hosts([address_bar]) end end describe '#addresses_from_dns' do - let(:service) { described_class.new(nameserver: 'localhost', port: 8600, record: 'foo', record_type: record_type) } + let(:service) do + described_class.new( + nameserver: 'localhost', + port: 8600, + record: 'foo', + record_type: record_type, + load_balancer: load_balancer + ) + end + let(:packet) { double(:packet, answer: [res1, res2]) } before do @@ -234,13 +245,11 @@ RSpec.describe Gitlab::Database::LoadBalancing::ServiceDiscovery do end describe '#addresses_from_load_balancer' do - it 'returns the ordered host names of the load balancer' do - load_balancer = Gitlab::Database::LoadBalancing::LoadBalancer.new(%w[b a]) - - allow(service) - .to receive(:load_balancer) - .and_return(load_balancer) + let(:load_balancer) do + Gitlab::Database::LoadBalancing::LoadBalancer.new(%w[b a]) + end + it 'returns the ordered host names of the load balancer' do addresses = [ described_class::Address.new('a'), described_class::Address.new('b') diff --git a/spec/lib/gitlab/database/load_balancing/sticking_spec.rb b/spec/lib/gitlab/database/load_balancing/sticking_spec.rb index 53445d73756..cf52e59db3a 100644 --- a/spec/lib/gitlab/database/load_balancing/sticking_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/sticking_spec.rb @@ -237,7 +237,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::Sticking, :redis do context 'when write location is nil' do before do - allow(Gitlab::Database).to receive(:get_write_location).and_return(nil) + allow(Gitlab::Database.main).to receive(:get_write_location).and_return(nil) end it 'does not update the write location' do @@ -313,7 +313,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::Sticking, :redis do end it 'returns false and does not try to find caught up hosts' do - expect(described_class).not_to receive(:select_caught_up_hosts) + expect(lb).not_to receive(:select_up_to_date_host) expect(described_class.select_caught_up_replicas(:project, 42)).to be false end end @@ -329,18 +329,6 @@ RSpec.describe Gitlab::Database::LoadBalancing::Sticking, :redis do expect(described_class).to receive(:unstick).with(:project, 42) expect(described_class.select_caught_up_replicas(:project, 42)).to be true end - - context 'when :load_balancing_refine_load_balancer_methods FF is disabled' do - before do - stub_feature_flags(load_balancing_refine_load_balancer_methods: false) - end - - it 'returns true, selects hosts, and unsticks if any secondary has caught up' do - expect(lb).to receive(:select_caught_up_hosts).and_return(true) - expect(described_class).to receive(:unstick).with(:project, 42) - expect(described_class.select_caught_up_replicas(:project, 42)).to be true - end - end end end end diff --git a/spec/lib/gitlab/database/load_balancing_spec.rb b/spec/lib/gitlab/database/load_balancing_spec.rb index 94717a10492..6ec8e0516f6 100644 --- a/spec/lib/gitlab/database/load_balancing_spec.rb +++ b/spec/lib/gitlab/database/load_balancing_spec.rb @@ -3,25 +3,28 @@ require 'spec_helper' RSpec.describe Gitlab::Database::LoadBalancing do - include_context 'clear DB Load Balancing configuration' + describe '.proxy' do + before do + @previous_proxy = ActiveRecord::Base.load_balancing_proxy - before do - stub_env('ENABLE_LOAD_BALANCING_FOR_FOSS', 'true') - end + ActiveRecord::Base.load_balancing_proxy = connection_proxy + end + + after do + ActiveRecord::Base.load_balancing_proxy = @previous_proxy + end - describe '.proxy' do context 'when configured' do - before do - allow(ActiveRecord::Base.singleton_class).to receive(:prepend) - subject.configure_proxy - end + let(:connection_proxy) { double(:connection_proxy) } it 'returns the connection proxy' do - expect(subject.proxy).to be_an_instance_of(subject::ConnectionProxy) + expect(subject.proxy).to eq(connection_proxy) end end context 'when not configured' do + let(:connection_proxy) { nil } + it 'returns nil' do expect(subject.proxy).to be_nil end @@ -40,9 +43,9 @@ RSpec.describe Gitlab::Database::LoadBalancing do it 'returns a Hash' do lb_config = { 'hosts' => %w(foo) } - original_db_config = Gitlab::Database.config + original_db_config = Gitlab::Database.main.config modified_db_config = original_db_config.merge(load_balancing: lb_config) - expect(Gitlab::Database).to receive(:config).and_return(modified_db_config) + expect(Gitlab::Database.main).to receive(:config).and_return(modified_db_config) expect(described_class.configuration).to eq(lb_config) end @@ -132,7 +135,6 @@ RSpec.describe Gitlab::Database::LoadBalancing do describe '.enable?' do before do - clear_load_balancing_configuration allow(described_class).to receive(:hosts).and_return(%w(foo)) end @@ -173,10 +175,6 @@ RSpec.describe Gitlab::Database::LoadBalancing do end describe '.configured?' do - before do - clear_load_balancing_configuration - end - it 'returns true when Sidekiq is being used' do allow(described_class).to receive(:hosts).and_return(%w(foo)) allow(Gitlab::Runtime).to receive(:sidekiq?).and_return(true) @@ -207,12 +205,27 @@ RSpec.describe Gitlab::Database::LoadBalancing do describe '.configure_proxy' do it 'configures the connection proxy' do - allow(ActiveRecord::Base.singleton_class).to receive(:prepend) + allow(ActiveRecord::Base).to receive(:load_balancing_proxy=) described_class.configure_proxy - expect(ActiveRecord::Base.singleton_class).to have_received(:prepend) - .with(Gitlab::Database::LoadBalancing::ActiveRecordProxy) + expect(ActiveRecord::Base).to have_received(:load_balancing_proxy=) + .with(Gitlab::Database::LoadBalancing::ConnectionProxy) + end + + context 'when service discovery is enabled' do + let(:service_discovery) { double(Gitlab::Database::LoadBalancing::ServiceDiscovery) } + + it 'runs initial service discovery when configuring the connection proxy' do + allow(described_class) + .to receive(:configuration) + .and_return('discover' => { 'record' => 'foo' }) + + expect(Gitlab::Database::LoadBalancing::ServiceDiscovery).to receive(:new).and_return(service_discovery) + expect(service_discovery).to receive(:perform_service_discovery) + + described_class.configure_proxy + end end end @@ -298,59 +311,46 @@ RSpec.describe Gitlab::Database::LoadBalancing do end describe '.db_role_for_connection' do - let(:connection) { double(:conneciton) } - context 'when the load balancing is not configured' do - before do - allow(described_class).to receive(:enable?).and_return(false) - end + let(:connection) { ActiveRecord::Base.connection } it 'returns primary' do - expect(described_class.db_role_for_connection(connection)).to be(:primary) + expect(described_class.db_role_for_connection(connection)).to eq(:primary) end end - context 'when the load balancing is configured' do - let(:proxy) { described_class::ConnectionProxy.new(%w(foo)) } - let(:load_balancer) { described_class::LoadBalancer.new(%w(foo)) } - - before do - allow(ActiveRecord::Base.singleton_class).to receive(:prepend) + context 'when the NullPool is used for connection' do + let(:pool) { ActiveRecord::ConnectionAdapters::NullPool.new } + let(:connection) { double(:connection, pool: pool) } - allow(described_class).to receive(:enable?).and_return(true) - allow(described_class).to receive(:proxy).and_return(proxy) - allow(proxy).to receive(:load_balancer).and_return(load_balancer) - - subject.configure_proxy(proxy) + it 'returns unknown' do + expect(described_class.db_role_for_connection(connection)).to eq(:unknown) end + end - context 'when the load balancer returns :replica' do - it 'returns :replica' do - allow(load_balancer).to receive(:db_role_for_connection).and_return(:replica) - - expect(described_class.db_role_for_connection(connection)).to be(:replica) + context 'when the load balancing is configured' do + let(:db_host) { ActiveRecord::Base.connection_pool.db_config.host } + let(:proxy) { described_class::ConnectionProxy.new([db_host]) } - expect(load_balancer).to have_received(:db_role_for_connection).with(connection) + context 'when a proxy connection is used' do + it 'returns :unknown' do + expect(described_class.db_role_for_connection(proxy)).to eq(:unknown) end end - context 'when the load balancer returns :primary' do - it 'returns :primary' do - allow(load_balancer).to receive(:db_role_for_connection).and_return(:primary) - - expect(described_class.db_role_for_connection(connection)).to be(:primary) - - expect(load_balancer).to have_received(:db_role_for_connection).with(connection) + context 'when a read connection is used' do + it 'returns :replica' do + proxy.load_balancer.read do |connection| + expect(described_class.db_role_for_connection(connection)).to eq(:replica) + end end end - context 'when the load balancer returns nil' do - it 'returns nil' do - allow(load_balancer).to receive(:db_role_for_connection).and_return(nil) - - expect(described_class.db_role_for_connection(connection)).to be(nil) - - expect(load_balancer).to have_received(:db_role_for_connection).with(connection) + context 'when a read_write connection is used' do + it 'returns :primary' do + proxy.load_balancer.read_write do |connection| + expect(described_class.db_role_for_connection(connection)).to eq(:primary) + end end end end @@ -366,7 +366,7 @@ RSpec.describe Gitlab::Database::LoadBalancing do # - In each test, we listen to the SQL queries (via sql.active_record # instrumentation) while triggering real queries from the defined model. # - We assert the desinations (replica/primary) of the queries in order. - describe 'LoadBalancing integration tests', :delete do + describe 'LoadBalancing integration tests', :db_load_balancing, :delete do before(:all) do ActiveRecord::Schema.define do create_table :load_balancing_test, force: true do |t| @@ -381,30 +381,14 @@ RSpec.describe Gitlab::Database::LoadBalancing do end end - shared_context 'LoadBalancing setup' do - let(:development_db_config) { ActiveRecord::Base.configurations.configs_for(env_name: 'development').first.configuration_hash } - let(:hosts) { [development_db_config[:host]] } - let(:model) do - Class.new(ApplicationRecord) do - self.table_name = "load_balancing_test" - end + let(:model) do + Class.new(ApplicationRecord) do + self.table_name = "load_balancing_test" end + end - before do - # Preloading testing class - model.singleton_class.prepend ::Gitlab::Database::LoadBalancing::ActiveRecordProxy - - # Setup load balancing - clear_load_balancing_configuration - allow(ActiveRecord::Base.singleton_class).to receive(:prepend) - subject.configure_proxy(::Gitlab::Database::LoadBalancing::ConnectionProxy.new(hosts)) - - original_db_config = Gitlab::Database.config - modified_db_config = original_db_config.merge(load_balancing: { hosts: hosts }) - allow(Gitlab::Database).to receive(:config).and_return(modified_db_config) - - ::Gitlab::Database::LoadBalancing::Session.clear_session - end + before do + model.singleton_class.prepend ::Gitlab::Database::LoadBalancing::ActiveRecordProxy end where(:queries, :include_transaction, :expected_results) do @@ -715,8 +699,6 @@ RSpec.describe Gitlab::Database::LoadBalancing do end with_them do - include_context 'LoadBalancing setup' - it 'redirects queries to the right roles' do roles = [] @@ -785,8 +767,6 @@ RSpec.describe Gitlab::Database::LoadBalancing do end with_them do - include_context 'LoadBalancing setup' - it 'redirects queries to the right roles' do roles = [] @@ -805,8 +785,6 @@ RSpec.describe Gitlab::Database::LoadBalancing do end context 'a write inside a transaction inside fallback_to_replicas_for_ambiguous_queries block' do - include_context 'LoadBalancing setup' - it 'raises an exception' do expect do ::Gitlab::Database::LoadBalancing::Session.current.fallback_to_replicas_for_ambiguous_queries do diff --git a/spec/lib/gitlab/database/migration_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers_spec.rb index 8e25f9249fe..9f9aef77de7 100644 --- a/spec/lib/gitlab/database/migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers_spec.rb @@ -278,6 +278,16 @@ RSpec.describe Gitlab::Database::MigrationHelpers do model.add_concurrent_index(:users, :foo, unique: true) end + + it 'unprepares the async index creation' do + expect(model).to receive(:add_index) + .with(:users, :foo, algorithm: :concurrently) + + expect(model).to receive(:unprepare_async_index) + .with(:users, :foo, algorithm: :concurrently) + + model.add_concurrent_index(:users, :foo) + end end context 'inside a transaction' do @@ -314,6 +324,16 @@ RSpec.describe Gitlab::Database::MigrationHelpers do model.remove_concurrent_index(:users, :foo, unique: true) end + it 'unprepares the async index creation' do + expect(model).to receive(:remove_index) + .with(:users, { algorithm: :concurrently, column: :foo }) + + expect(model).to receive(:unprepare_async_index) + .with(:users, :foo, { algorithm: :concurrently }) + + model.remove_concurrent_index(:users, :foo) + end + describe 'by index name' do before do allow(model).to receive(:index_exists_by_name?).with(:users, "index_x_by_y").and_return(true) @@ -345,6 +365,16 @@ RSpec.describe Gitlab::Database::MigrationHelpers do model.remove_concurrent_index_by_name(:users, wrong_key: "index_x_by_y") end.to raise_error 'remove_concurrent_index_by_name must get an index name as the second argument' end + + it 'unprepares the async index creation' do + expect(model).to receive(:remove_index) + .with(:users, { algorithm: :concurrently, name: "index_x_by_y" }) + + expect(model).to receive(:unprepare_async_index_by_name) + .with(:users, "index_x_by_y", { algorithm: :concurrently }) + + model.remove_concurrent_index_by_name(:users, "index_x_by_y") + end end end end @@ -384,9 +414,9 @@ RSpec.describe Gitlab::Database::MigrationHelpers do expect(model).to receive(:with_lock_retries).and_call_original expect(model).to receive(:disable_statement_timeout).and_call_original expect(model).to receive(:statement_timeout_disabled?).and_return(false) - expect(model).to receive(:execute).with(/statement_timeout/) + expect(model).to receive(:execute).with(/SET statement_timeout TO/) expect(model).to receive(:execute).ordered.with(/VALIDATE CONSTRAINT/) - expect(model).to receive(:execute).ordered.with(/RESET ALL/) + expect(model).to receive(:execute).ordered.with(/RESET statement_timeout/) expect(model).to receive(:execute).with(/REFERENCES users \(id\)/) @@ -398,9 +428,9 @@ RSpec.describe Gitlab::Database::MigrationHelpers do expect(model).to receive(:with_lock_retries).and_call_original expect(model).to receive(:disable_statement_timeout).and_call_original expect(model).to receive(:statement_timeout_disabled?).and_return(false) - expect(model).to receive(:execute).with(/statement_timeout/) + expect(model).to receive(:execute).with(/SET statement_timeout TO/) expect(model).to receive(:execute).ordered.with(/VALIDATE CONSTRAINT/) - expect(model).to receive(:execute).ordered.with(/RESET ALL/) + expect(model).to receive(:execute).ordered.with(/RESET statement_timeout/) expect(model).to receive(:execute).with(/REFERENCES users \(id_convert_to_bigint\)/) @@ -416,9 +446,9 @@ RSpec.describe Gitlab::Database::MigrationHelpers do expect(model).to receive(:with_lock_retries).and_call_original expect(model).to receive(:disable_statement_timeout).and_call_original expect(model).to receive(:statement_timeout_disabled?).and_return(false) - expect(model).to receive(:execute).with(/statement_timeout/) + expect(model).to receive(:execute).with(/SET statement_timeout TO/) expect(model).to receive(:execute).ordered.with(/VALIDATE CONSTRAINT/) - expect(model).to receive(:execute).ordered.with(/RESET ALL/) + expect(model).to receive(:execute).ordered.with(/RESET statement_timeout/) expect(model).to receive(:execute).with(/ON DELETE SET NULL/) @@ -433,9 +463,9 @@ RSpec.describe Gitlab::Database::MigrationHelpers do expect(model).to receive(:with_lock_retries).and_call_original expect(model).to receive(:disable_statement_timeout).and_call_original expect(model).to receive(:statement_timeout_disabled?).and_return(false) - expect(model).to receive(:execute).with(/statement_timeout/) + expect(model).to receive(:execute).with(/SET statement_timeout TO/) expect(model).to receive(:execute).ordered.with(/VALIDATE CONSTRAINT/) - expect(model).to receive(:execute).ordered.with(/RESET ALL/) + expect(model).to receive(:execute).ordered.with(/RESET statement_timeout/) expect(model).to receive(:execute).with(/ON DELETE CASCADE/) @@ -450,9 +480,9 @@ RSpec.describe Gitlab::Database::MigrationHelpers do expect(model).to receive(:with_lock_retries).and_call_original expect(model).to receive(:disable_statement_timeout).and_call_original expect(model).to receive(:statement_timeout_disabled?).and_return(false) - expect(model).to receive(:execute).with(/statement_timeout/) + expect(model).to receive(:execute).with(/SET statement_timeout TO/) expect(model).to receive(:execute).ordered.with(/VALIDATE CONSTRAINT/) - expect(model).to receive(:execute).ordered.with(/RESET ALL/) + expect(model).to receive(:execute).ordered.with(/RESET statement_timeout/) expect(model).not_to receive(:execute).with(/ON DELETE/) @@ -468,10 +498,10 @@ RSpec.describe Gitlab::Database::MigrationHelpers do expect(model).to receive(:with_lock_retries).and_call_original expect(model).to receive(:disable_statement_timeout).and_call_original expect(model).to receive(:statement_timeout_disabled?).and_return(false) - expect(model).to receive(:execute).with(/statement_timeout/) + expect(model).to receive(:execute).with(/SET statement_timeout TO/) expect(model).to receive(:execute).ordered.with(/NOT VALID/) expect(model).to receive(:execute).ordered.with(/VALIDATE CONSTRAINT/) - expect(model).to receive(:execute).ordered.with(/RESET ALL/) + expect(model).to receive(:execute).ordered.with(/RESET statement_timeout/) model.add_concurrent_foreign_key(:projects, :users, column: :user_id) end @@ -497,10 +527,10 @@ RSpec.describe Gitlab::Database::MigrationHelpers do expect(model).to receive(:with_lock_retries).and_call_original expect(model).to receive(:disable_statement_timeout).and_call_original expect(model).to receive(:statement_timeout_disabled?).and_return(false) - expect(model).to receive(:execute).with(/statement_timeout/) + expect(model).to receive(:execute).with(/SET statement_timeout TO/) expect(model).to receive(:execute).ordered.with(/NOT VALID/) expect(model).to receive(:execute).ordered.with(/VALIDATE CONSTRAINT.+foo/) - expect(model).to receive(:execute).ordered.with(/RESET ALL/) + expect(model).to receive(:execute).ordered.with(/RESET statement_timeout/) model.add_concurrent_foreign_key(:projects, :users, column: :user_id, name: :foo) end @@ -527,10 +557,10 @@ RSpec.describe Gitlab::Database::MigrationHelpers do expect(model).to receive(:with_lock_retries).and_call_original expect(model).to receive(:disable_statement_timeout).and_call_original expect(model).to receive(:statement_timeout_disabled?).and_return(false) - expect(model).to receive(:execute).with(/statement_timeout/) + expect(model).to receive(:execute).with(/SET statement_timeout TO/) expect(model).to receive(:execute).ordered.with(/NOT VALID/) expect(model).to receive(:execute).ordered.with(/VALIDATE CONSTRAINT.+bar/) - expect(model).to receive(:execute).ordered.with(/RESET ALL/) + expect(model).to receive(:execute).ordered.with(/RESET statement_timeout/) model.add_concurrent_foreign_key(:projects, :users, column: :user_id, name: :bar) end @@ -556,6 +586,22 @@ RSpec.describe Gitlab::Database::MigrationHelpers do it_behaves_like 'performs validation', {} end end + + context 'when the reverse_lock_order flag is set' do + it 'explicitly locks the tables in target-source order', :aggregate_failures do + expect(model).to receive(:with_lock_retries).and_call_original + expect(model).to receive(:disable_statement_timeout).and_call_original + expect(model).to receive(:statement_timeout_disabled?).and_return(false) + expect(model).to receive(:execute).with(/SET statement_timeout TO/) + expect(model).to receive(:execute).ordered.with(/VALIDATE CONSTRAINT/) + expect(model).to receive(:execute).ordered.with(/RESET statement_timeout/) + + expect(model).to receive(:execute).with('LOCK TABLE users, projects IN SHARE ROW EXCLUSIVE MODE') + expect(model).to receive(:execute).with(/REFERENCES users \(id\)/) + + model.add_concurrent_foreign_key(:projects, :users, column: :user_id, reverse_lock_order: true) + end + end end end @@ -568,9 +614,9 @@ RSpec.describe Gitlab::Database::MigrationHelpers do expect(model).not_to receive(:concurrent_foreign_key_name) expect(model).to receive(:disable_statement_timeout).and_call_original expect(model).to receive(:statement_timeout_disabled?).and_return(false) - expect(model).to receive(:execute).with(/statement_timeout/) + expect(model).to receive(:execute).with(/SET statement_timeout TO/) expect(model).to receive(:execute).ordered.with(/ALTER TABLE projects VALIDATE CONSTRAINT/) - expect(model).to receive(:execute).ordered.with(/RESET ALL/) + expect(model).to receive(:execute).ordered.with(/RESET statement_timeout/) end model.validate_foreign_key(:projects, :user_id, name: :foo) @@ -585,9 +631,9 @@ RSpec.describe Gitlab::Database::MigrationHelpers do expect(model).to receive(:concurrent_foreign_key_name) expect(model).to receive(:disable_statement_timeout).and_call_original expect(model).to receive(:statement_timeout_disabled?).and_return(false) - expect(model).to receive(:execute).with(/statement_timeout/) + expect(model).to receive(:execute).with(/SET statement_timeout TO/) expect(model).to receive(:execute).ordered.with(/ALTER TABLE projects VALIDATE CONSTRAINT/) - expect(model).to receive(:execute).ordered.with(/RESET ALL/) + expect(model).to receive(:execute).ordered.with(/RESET statement_timeout/) end model.validate_foreign_key(:projects, :user_id) @@ -702,7 +748,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do end after do - model.execute('RESET ALL') + model.execute('RESET statement_timeout') end it 'defines statement to 0 only for current transaction' do @@ -719,7 +765,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do context 'when passing a blocks' do it 'disables statement timeouts on session level and executes the block' do expect(model).to receive(:execute).with('SET statement_timeout TO 0') - expect(model).to receive(:execute).with('RESET ALL').at_least(:once) + expect(model).to receive(:execute).with('RESET statement_timeout').at_least(:once) expect { |block| model.disable_statement_timeout(&block) }.to yield_control end @@ -731,7 +777,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do end after do - model.execute('RESET ALL') + model.execute('RESET statement_timeout') end it 'defines statement to 0 for any code run inside the block' do @@ -758,12 +804,12 @@ RSpec.describe Gitlab::Database::MigrationHelpers do after do # Use ActiveRecord::Base.connection instead of model.execute # so that this call is not counted below - ActiveRecord::Base.connection.execute('RESET ALL') + ActiveRecord::Base.connection.execute('RESET statement_timeout') end it 'yields control without disabling the timeout or resetting' do expect(model).not_to receive(:execute).with('SET statement_timeout TO 0') - expect(model).not_to receive(:execute).with('RESET ALL') + expect(model).not_to receive(:execute).with('RESET statement_timeout') expect { |block| model.disable_statement_timeout(&block) }.to yield_control end @@ -2486,7 +2532,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do expect(model).to receive(:disable_statement_timeout).and_call_original expect(model).to receive(:statement_timeout_disabled?).and_return(false) - expect(model).to receive(:execute).with(/statement_timeout/) + expect(model).to receive(:execute).with(/SET statement_timeout TO/) expect(model).to receive(:with_lock_retries).and_call_original expect(model).to receive(:execute).with(/ADD CONSTRAINT check_name_not_null/) @@ -2496,7 +2542,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do .and_return(true).exactly(1) expect(model).to receive(:execute).ordered.with(/VALIDATE CONSTRAINT/) - expect(model).to receive(:execute).ordered.with(/RESET ALL/) + expect(model).to receive(:execute).ordered.with(/RESET statement_timeout/) model.add_check_constraint( :test_table, @@ -2530,7 +2576,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do expect(model).to receive(:disable_statement_timeout).and_call_original expect(model).to receive(:statement_timeout_disabled?).and_return(false) - expect(model).to receive(:execute).with(/statement_timeout/) + expect(model).to receive(:execute).with(/SET statement_timeout TO/) expect(model).to receive(:with_lock_retries).and_call_original expect(model).to receive(:execute).with(/ADD CONSTRAINT check_name_not_null/) @@ -2539,7 +2585,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do .and_return(true).exactly(1) expect(model).to receive(:execute).ordered.with(/VALIDATE CONSTRAINT/) - expect(model).to receive(:execute).ordered.with(/RESET ALL/) + expect(model).to receive(:execute).ordered.with(/RESET statement_timeout/) model.add_check_constraint( :test_table, @@ -2572,9 +2618,9 @@ RSpec.describe Gitlab::Database::MigrationHelpers do expect(model).to receive(:check_constraint_exists?).and_return(true) expect(model).to receive(:disable_statement_timeout).and_call_original expect(model).to receive(:statement_timeout_disabled?).and_return(false) - expect(model).to receive(:execute).with(/statement_timeout/) + expect(model).to receive(:execute).with(/SET statement_timeout TO/) expect(model).to receive(:execute).ordered.with(validate_sql) - expect(model).to receive(:execute).ordered.with(/RESET ALL/) + expect(model).to receive(:execute).ordered.with(/RESET statement_timeout/) model.validate_check_constraint(:test_table, 'check_name') end diff --git a/spec/lib/gitlab/database/migrations/background_migration_helpers_spec.rb b/spec/lib/gitlab/database/migrations/background_migration_helpers_spec.rb index e096e7f6e91..1a7116e75e5 100644 --- a/spec/lib/gitlab/database/migrations/background_migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/migrations/background_migration_helpers_spec.rb @@ -581,4 +581,101 @@ RSpec.describe Gitlab::Database::Migrations::BackgroundMigrationHelpers do model.delete_queued_jobs('BackgroundMigrationClassName') end end + + describe '#finalized_background_migration' do + include_context 'background migration job class' + + let!(:tracked_pending_job) { create(:background_migration_job, class_name: job_class_name, status: :pending, arguments: [1]) } + let!(:tracked_successful_job) { create(:background_migration_job, class_name: job_class_name, status: :succeeded, arguments: [2]) } + + before do + Sidekiq::Testing.disable! do + BackgroundMigrationWorker.perform_async(job_class_name, [1, 2]) + BackgroundMigrationWorker.perform_async(job_class_name, [3, 4]) + BackgroundMigrationWorker.perform_in(10, job_class_name, [5, 6]) + BackgroundMigrationWorker.perform_in(20, job_class_name, [7, 8]) + end + end + + it_behaves_like 'finalized tracked background migration' do + before do + model.finalize_background_migration(job_class_name) + end + end + + context 'when removing all tracked job records' do + # Force pending jobs to remain pending. + let!(:job_perform_method) { ->(*arguments) { } } + + before do + model.finalize_background_migration(job_class_name, delete_tracking_jobs: %w[pending succeeded]) + end + + it_behaves_like 'finalized tracked background migration' + it_behaves_like 'removed tracked jobs', 'pending' + it_behaves_like 'removed tracked jobs', 'succeeded' + end + + context 'when retaining all tracked job records' do + before do + model.finalize_background_migration(job_class_name, delete_tracking_jobs: false) + end + + it_behaves_like 'finalized background migration' + include_examples 'retained tracked jobs', 'succeeded' + end + + context 'during retry race condition' do + let(:queue_items_added) { [] } + let!(:job_perform_method) do + ->(*arguments) do + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + RSpec.current_example.example_group_instance.job_class_name, + arguments + ) + + # Mock another process pushing queue jobs. + queue_items_added = RSpec.current_example.example_group_instance.queue_items_added + if queue_items_added.count < 10 + Sidekiq::Testing.disable! do + job_class_name = RSpec.current_example.example_group_instance.job_class_name + queue_items_added << BackgroundMigrationWorker.perform_async(job_class_name, [Time.current]) + queue_items_added << BackgroundMigrationWorker.perform_in(10, job_class_name, [Time.current]) + end + end + end + end + + it_behaves_like 'finalized tracked background migration' do + before do + model.finalize_background_migration(job_class_name, delete_tracking_jobs: ['succeeded']) + end + end + end + end + + describe '#delete_job_tracking' do + let!(:job_class_name) { 'TestJob' } + + let!(:tracked_pending_job) { create(:background_migration_job, class_name: job_class_name, status: :pending, arguments: [1]) } + let!(:tracked_successful_job) { create(:background_migration_job, class_name: job_class_name, status: :succeeded, arguments: [2]) } + + context 'with default status' do + before do + model.delete_job_tracking(job_class_name) + end + + include_examples 'retained tracked jobs', 'pending' + include_examples 'removed tracked jobs', 'succeeded' + end + + context 'with explicit status' do + before do + model.delete_job_tracking(job_class_name, status: %w[pending succeeded]) + end + + include_examples 'removed tracked jobs', 'pending' + include_examples 'removed tracked jobs', 'succeeded' + end + end end diff --git a/spec/lib/gitlab/database/migrations/instrumentation_spec.rb b/spec/lib/gitlab/database/migrations/instrumentation_spec.rb index 6d047eed3bb..5945e5a2039 100644 --- a/spec/lib/gitlab/database/migrations/instrumentation_spec.rb +++ b/spec/lib/gitlab/database/migrations/instrumentation_spec.rb @@ -5,24 +5,35 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do describe '#observe' do subject { described_class.new } - let(:migration) { 1234 } + let(:migration_name) { 'test' } + let(:migration_version) { '12345' } it 'executes the given block' do - expect { |b| subject.observe(migration, &b) }.to yield_control + expect { |b| subject.observe(version: migration_version, name: migration_name, &b) }.to yield_control end context 'behavior with observers' do - subject { described_class.new(observers).observe(migration) {} } + subject { described_class.new([Gitlab::Database::Migrations::Observers::MigrationObserver]).observe(version: migration_version, name: migration_name) {} } - let(:observers) { [observer] } let(:observer) { instance_double('Gitlab::Database::Migrations::Observers::MigrationObserver', before: nil, after: nil, record: nil) } + before do + allow(Gitlab::Database::Migrations::Observers::MigrationObserver).to receive(:new).and_return(observer) + end + + it 'instantiates observer with observation' do + expect(Gitlab::Database::Migrations::Observers::MigrationObserver) + .to receive(:new) + .with(instance_of(Gitlab::Database::Migrations::Observation)) { |observation| expect(observation.version).to eq(migration_version) } + .and_return(observer) + + subject + end + it 'calls #before, #after, #record on given observers' do expect(observer).to receive(:before).ordered expect(observer).to receive(:after).ordered - expect(observer).to receive(:record).ordered do |observation| - expect(observation.migration).to eq(migration) - end + expect(observer).to receive(:record).ordered subject end @@ -47,7 +58,7 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do end context 'on successful execution' do - subject { described_class.new.observe(migration) {} } + subject { described_class.new.observe(version: migration_version, name: migration_name) {} } it 'records walltime' do expect(subject.walltime).not_to be_nil @@ -58,12 +69,16 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do end it 'records the migration version' do - expect(subject.migration).to eq(migration) + expect(subject.version).to eq(migration_version) + end + + it 'records the migration name' do + expect(subject.name).to eq(migration_name) end end context 'upon failure' do - subject { described_class.new.observe(migration) { raise 'something went wrong' } } + subject { described_class.new.observe(version: migration_version, name: migration_name) { raise 'something went wrong' } } it 'raises the exception' do expect { subject }.to raise_error(/something went wrong/) @@ -73,7 +88,7 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do subject { instance.observations.first } before do - instance.observe(migration) { raise 'something went wrong' } + instance.observe(version: migration_version, name: migration_name) { raise 'something went wrong' } rescue StandardError # ignore end @@ -89,7 +104,11 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do end it 'records the migration version' do - expect(subject.migration).to eq(migration) + expect(subject.version).to eq(migration_version) + end + + it 'records the migration name' do + expect(subject.name).to eq(migration_name) end end end @@ -101,8 +120,8 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do let(:migration2) { double('migration2', call: nil) } it 'records observations for all migrations' do - subject.observe('migration1') {} - subject.observe('migration2') { raise 'something went wrong' } rescue nil + subject.observe(version: migration_version, name: migration_name) {} + subject.observe(version: migration_version, name: migration_name) { raise 'something went wrong' } rescue nil expect(subject.observations.size).to eq(2) end diff --git a/spec/lib/gitlab/database/migrations/observers/query_details_spec.rb b/spec/lib/gitlab/database/migrations/observers/query_details_spec.rb index 8aac3ed67c6..36885a1594f 100644 --- a/spec/lib/gitlab/database/migrations/observers/query_details_spec.rb +++ b/spec/lib/gitlab/database/migrations/observers/query_details_spec.rb @@ -2,16 +2,17 @@ require 'spec_helper' RSpec.describe Gitlab::Database::Migrations::Observers::QueryDetails do - subject { described_class.new } + subject { described_class.new(observation) } - let(:observation) { Gitlab::Database::Migrations::Observation.new(migration) } + let(:observation) { Gitlab::Database::Migrations::Observation.new(migration_version, migration_name) } let(:connection) { ActiveRecord::Base.connection } let(:query) { "select date_trunc('day', $1::timestamptz) + $2 * (interval '1 hour')" } let(:query_binds) { [Time.current, 3] } let(:directory_path) { Dir.mktmpdir } - let(:log_file) { "#{directory_path}/#{migration}-query-details.json" } + let(:log_file) { "#{directory_path}/#{migration_version}_#{migration_name}-query-details.json" } let(:query_details) { Gitlab::Json.parse(File.read(log_file)) } - let(:migration) { 20210422152437 } + let(:migration_version) { 20210422152437 } + let(:migration_name) { 'test' } before do stub_const('Gitlab::Database::Migrations::Instrumentation::RESULT_DIR', directory_path) @@ -49,7 +50,7 @@ RSpec.describe Gitlab::Database::Migrations::Observers::QueryDetails do subject.before run_query subject.after - subject.record(observation) + subject.record end def run_query diff --git a/spec/lib/gitlab/database/migrations/observers/query_log_spec.rb b/spec/lib/gitlab/database/migrations/observers/query_log_spec.rb index 195e7114582..2a49d8e8b73 100644 --- a/spec/lib/gitlab/database/migrations/observers/query_log_spec.rb +++ b/spec/lib/gitlab/database/migrations/observers/query_log_spec.rb @@ -2,14 +2,14 @@ require 'spec_helper' RSpec.describe Gitlab::Database::Migrations::Observers::QueryLog do - subject { described_class.new } + subject { described_class.new(observation) } - let(:observation) { Gitlab::Database::Migrations::Observation.new(migration) } + let(:observation) { Gitlab::Database::Migrations::Observation.new(migration_version, migration_name) } let(:connection) { ActiveRecord::Base.connection } let(:query) { 'select 1' } let(:directory_path) { Dir.mktmpdir } - let(:log_file) { "#{directory_path}/current.log" } - let(:migration) { 20210422152437 } + let(:migration_version) { 20210422152437 } + let(:migration_name) { 'test' } before do stub_const('Gitlab::Database::Migrations::Instrumentation::RESULT_DIR', directory_path) @@ -22,7 +22,7 @@ RSpec.describe Gitlab::Database::Migrations::Observers::QueryLog do it 'writes a file with the query log' do observe - expect(File.read("#{directory_path}/#{migration}.log")).to include(query) + expect(File.read("#{directory_path}/#{migration_version}_#{migration_name}.log")).to include(query) end it 'does not change the default logger' do @@ -33,6 +33,6 @@ RSpec.describe Gitlab::Database::Migrations::Observers::QueryLog do subject.before connection.execute(query) subject.after - subject.record(observation) + subject.record end end diff --git a/spec/lib/gitlab/database/migrations/observers/query_statistics_spec.rb b/spec/lib/gitlab/database/migrations/observers/query_statistics_spec.rb index a3b03050b33..32a25fdaa28 100644 --- a/spec/lib/gitlab/database/migrations/observers/query_statistics_spec.rb +++ b/spec/lib/gitlab/database/migrations/observers/query_statistics_spec.rb @@ -2,8 +2,9 @@ require 'spec_helper' RSpec.describe Gitlab::Database::Migrations::Observers::QueryStatistics do - subject { described_class.new } + subject { described_class.new(observation) } + let(:observation) { Gitlab::Database::Migrations::Observation.new } let(:connection) { ActiveRecord::Base.connection } def mock_pgss(enabled: true) @@ -37,7 +38,6 @@ RSpec.describe Gitlab::Database::Migrations::Observers::QueryStatistics do end describe '#record' do - let(:observation) { Gitlab::Database::Migrations::Observation.new } let(:result) { double } let(:pgss_query) do <<~SQL @@ -52,7 +52,7 @@ RSpec.describe Gitlab::Database::Migrations::Observers::QueryStatistics do mock_pgss(enabled: true) expect(connection).to receive(:execute).with(pgss_query).once.and_return(result) - expect { subject.record(observation) }.to change { observation.query_statistics }.from(nil).to(result) + expect { subject.record }.to change { observation.query_statistics }.from(nil).to(result) end end @@ -61,7 +61,7 @@ RSpec.describe Gitlab::Database::Migrations::Observers::QueryStatistics do mock_pgss(enabled: false) expect(connection).not_to receive(:execute) - expect { subject.record(observation) }.not_to change { observation.query_statistics } + expect { subject.record }.not_to change { observation.query_statistics } end end end diff --git a/spec/lib/gitlab/database/migrations/observers/total_database_size_change_spec.rb b/spec/lib/gitlab/database/migrations/observers/total_database_size_change_spec.rb index 73466471944..61e28003e66 100644 --- a/spec/lib/gitlab/database/migrations/observers/total_database_size_change_spec.rb +++ b/spec/lib/gitlab/database/migrations/observers/total_database_size_change_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' RSpec.describe Gitlab::Database::Migrations::Observers::TotalDatabaseSizeChange do - subject { described_class.new } + subject { described_class.new(observation) } let(:observation) { Gitlab::Database::Migrations::Observation.new } let(:connection) { ActiveRecord::Base.connection } @@ -14,7 +14,7 @@ RSpec.describe Gitlab::Database::Migrations::Observers::TotalDatabaseSizeChange subject.before subject.after - subject.record(observation) + subject.record expect(observation.total_database_size_change).to eq(256 - 1024) end @@ -27,13 +27,13 @@ RSpec.describe Gitlab::Database::Migrations::Observers::TotalDatabaseSizeChange it 'does not record anything if before size is unknown' do subject.after - expect { subject.record(observation) }.not_to change { observation.total_database_size_change } + expect { subject.record }.not_to change { observation.total_database_size_change } end it 'does not record anything if after size is unknown' do subject.before - expect { subject.record(observation) }.not_to change { observation.total_database_size_change } + expect { subject.record }.not_to change { observation.total_database_size_change } end end end diff --git a/spec/lib/gitlab/database/multi_threaded_migration_spec.rb b/spec/lib/gitlab/database/multi_threaded_migration_spec.rb deleted file mode 100644 index 78dd9e88064..00000000000 --- a/spec/lib/gitlab/database/multi_threaded_migration_spec.rb +++ /dev/null @@ -1,43 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Database::MultiThreadedMigration do - let(:migration) do - Class.new { include Gitlab::Database::MultiThreadedMigration }.new - end - - describe '#connection' do - after do - Thread.current[described_class::MULTI_THREAD_AR_CONNECTION] = nil - end - - it 'returns the thread-local connection if present' do - Thread.current[described_class::MULTI_THREAD_AR_CONNECTION] = 10 - - expect(migration.connection).to eq(10) - end - - it 'returns the global connection if no thread-local connection was set' do - expect(migration.connection).to eq(ActiveRecord::Base.connection) - end - end - - describe '#with_multiple_threads' do - it 'starts multiple threads and yields the supplied block in every thread' do - output = Queue.new - - migration.with_multiple_threads(2) do - output << migration.connection.execute('SELECT 1') - end - - expect(output.size).to eq(2) - end - - it 'joins the threads when the join parameter is set' do - expect_any_instance_of(Thread).to receive(:join).and_call_original - - migration.with_multiple_threads(1) { } - end - end -end diff --git a/spec/lib/gitlab/database/partitioning/detached_partition_dropper_spec.rb b/spec/lib/gitlab/database/partitioning/detached_partition_dropper_spec.rb new file mode 100644 index 00000000000..8523b7104f0 --- /dev/null +++ b/spec/lib/gitlab/database/partitioning/detached_partition_dropper_spec.rb @@ -0,0 +1,181 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::Partitioning::DetachedPartitionDropper do + include Database::TableSchemaHelpers + + let(:connection) { ActiveRecord::Base.connection } + + def expect_partition_present(name) + aggregate_failures do + expect(table_oid(name)).not_to be_nil + expect(Postgresql::DetachedPartition.find_by(table_name: name)).not_to be_nil + end + end + + def expect_partition_removed(name) + aggregate_failures do + expect(table_oid(name)).to be_nil + expect(Postgresql::DetachedPartition.find_by(table_name: name)).to be_nil + end + end + + before do + connection.execute(<<~SQL) + CREATE TABLE parent_table ( + id bigserial not null, + created_at timestamptz not null, + primary key (id, created_at) + ) PARTITION BY RANGE(created_at) + SQL + end + + def create_partition(name:, table: 'parent_table', from:, to:, attached:, drop_after:) + from = from.beginning_of_month + to = to.beginning_of_month + full_name = "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{name}" + connection.execute(<<~SQL) + CREATE TABLE #{full_name} + PARTITION OF #{table} + FOR VALUES FROM ('#{from.strftime('%Y-%m-%d')}') TO ('#{to.strftime('%Y-%m-%d')}') + SQL + + unless attached + connection.execute(<<~SQL) + ALTER TABLE #{table} DETACH PARTITION #{full_name} + SQL + end + + Postgresql::DetachedPartition.create!(table_name: name, + drop_after: drop_after) + end + + describe '#perform' do + context 'when the partition should not be dropped yet' do + it 'does not drop the partition' do + create_partition(name: 'test_partition', + from: 2.months.ago, to: 1.month.ago, + attached: false, + drop_after: 1.day.from_now) + + subject.perform + + expect_partition_present('test_partition') + end + end + + context 'with a partition to drop' do + before do + create_partition(name: 'test_partition', + from: 2.months.ago, + to: 1.month.ago.beginning_of_month, + attached: false, + drop_after: 1.second.ago) + end + + it 'drops the partition' do + subject.perform + + expect(table_oid('test_partition')).to be_nil + end + + context 'when the drop_detached_partitions feature flag is disabled' do + before do + stub_feature_flags(drop_detached_partitions: false) + end + it 'does not drop the partition' do + subject.perform + + expect(table_oid('test_partition')).not_to be_nil + end + end + + context 'when another process drops the table while the first waits for a lock' do + it 'skips the table' do + # Rspec's receive_method_chain does not support .and_wrap_original, so we need to nest here. + expect(Postgresql::DetachedPartition).to receive(:lock).and_wrap_original do |lock_meth| + locked = lock_meth.call + expect(locked).to receive(:find_by).and_wrap_original do |find_meth, *find_args| + # Another process drops the table then deletes this entry + Postgresql::DetachedPartition.where(*find_args).delete_all + find_meth.call(*find_args) + end + + locked + end + + expect(subject).not_to receive(:drop_one) + + subject.perform + end + end + end + + context 'when the partition to drop is still attached to its table' do + before do + create_partition(name: 'test_partition', + from: 2.months.ago, + to: 1.month.ago.beginning_of_month, + attached: true, + drop_after: 1.second.ago) + end + + it 'does not drop the partition, but does remove the DetachedPartition entry' do + subject.perform + aggregate_failures do + expect(table_oid('test_partition')).not_to be_nil + expect(Postgresql::DetachedPartition.find_by(table_name: 'test_partition')).to be_nil + end + end + + it 'removes the detached_partition entry' do + detached_partition = Postgresql::DetachedPartition.find_by!(table_name: 'test_partition') + + subject.perform + + expect(Postgresql::DetachedPartition.exists?(id: detached_partition.id)).to be_falsey + end + end + + context 'with multiple partitions to drop' do + before do + create_partition(name: 'partition_1', + from: 3.months.ago, + to: 2.months.ago, + attached: false, + drop_after: 1.second.ago) + + create_partition(name: 'partition_2', + from: 2.months.ago, + to: 1.month.ago, + attached: false, + drop_after: 1.second.ago) + end + + it 'drops both partitions' do + subject.perform + + expect_partition_removed('partition_1') + expect_partition_removed('partition_2') + end + + context 'when the first drop returns an error' do + it 'still drops the second partition' do + expect(subject).to receive(:drop_one).ordered.and_raise('injected error') + expect(subject).to receive(:drop_one).ordered.and_call_original + + subject.perform + + # We don't know which partition we tried to drop first, so the tests here have to work with either one + expect(Postgresql::DetachedPartition.count).to eq(1) + errored_partition_name = Postgresql::DetachedPartition.first!.table_name + + dropped_partition_name = (%w[partition_1 partition_2] - [errored_partition_name]).first + expect_partition_present(errored_partition_name) + expect_partition_removed(dropped_partition_name) + end + end + end + end +end diff --git a/spec/lib/gitlab/database/partitioning/monthly_strategy_spec.rb b/spec/lib/gitlab/database/partitioning/monthly_strategy_spec.rb index f9dca371398..c4fbf53d1c2 100644 --- a/spec/lib/gitlab/database/partitioning/monthly_strategy_spec.rb +++ b/spec/lib/gitlab/database/partitioning/monthly_strategy_spec.rb @@ -237,16 +237,6 @@ RSpec.describe Gitlab::Database::Partitioning::MonthlyStrategy do expect(subject).to contain_exactly(min_value_to_may) end - - context 'when the feature flag is toggled off' do - before do - stub_feature_flags(partition_pruning_dry_run: false) - end - - it 'is empty' do - expect(subject).to eq([]) - end - end end context 'with a time retention policy of 2 months' do @@ -258,16 +248,6 @@ RSpec.describe Gitlab::Database::Partitioning::MonthlyStrategy do Gitlab::Database::Partitioning::TimePartition.new(model.table_name, '2020-05-01', '2020-06-01', partition_name: 'partitioned_test_202005') ) end - - context 'when the feature flag is toggled off' do - before do - stub_feature_flags(partition_pruning_dry_run: false) - end - - it 'is empty' do - expect(subject).to eq([]) - end - end end end end diff --git a/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb b/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb index 903a41d6dd2..3d60457c3a9 100644 --- a/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb +++ b/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb @@ -4,9 +4,14 @@ require 'spec_helper' RSpec.describe Gitlab::Database::Partitioning::PartitionManager do include Database::PartitioningHelpers - include Database::TableSchemaHelpers include ExclusiveLeaseHelpers + def has_partition(model, month) + Gitlab::Database::PostgresPartition.for_parent_table(model.table_name).any? do |partition| + Gitlab::Database::Partitioning::TimePartition.from_sql(model.table_name, partition.name, partition.condition).from == month + end + end + describe '.register' do let(:model) { double(partitioning_strategy: nil) } @@ -111,14 +116,14 @@ RSpec.describe Gitlab::Database::Partitioning::PartitionManager do let(:extra_partitions) do [ - instance_double(Gitlab::Database::Partitioning::TimePartition, table: table, partition_name: 'foo1'), - instance_double(Gitlab::Database::Partitioning::TimePartition, table: table, partition_name: 'foo2') + instance_double(Gitlab::Database::Partitioning::TimePartition, table: table, partition_name: 'foo1', to_detach_sql: 'SELECT 1'), + instance_double(Gitlab::Database::Partitioning::TimePartition, table: table, partition_name: 'foo2', to_detach_sql: 'SELECT 2') ] end - context 'with the partition_pruning_dry_run feature flag enabled' do + context 'with the partition_pruning feature flag enabled' do before do - stub_feature_flags(partition_pruning_dry_run: true) + stub_feature_flags(partition_pruning: true) end it 'detaches each extra partition' do @@ -146,9 +151,9 @@ RSpec.describe Gitlab::Database::Partitioning::PartitionManager do end end - context 'with the partition_pruning_dry_run feature flag disabled' do + context 'with the partition_pruning feature flag disabled' do before do - stub_feature_flags(partition_pruning_dry_run: false) + stub_feature_flags(partition_pruning: false) end it 'returns immediately' do @@ -158,4 +163,128 @@ RSpec.describe Gitlab::Database::Partitioning::PartitionManager do end end end + + describe '#detach_partitions' do + around do |ex| + travel_to(Date.parse('2021-06-23')) do + ex.run + end + end + + subject { described_class.new([my_model]).sync_partitions } + + let(:connection) { ActiveRecord::Base.connection } + let(:my_model) do + Class.new(ApplicationRecord) do + include PartitionedTable + + self.table_name = 'my_model_example_table' + + partitioned_by :created_at, strategy: :monthly, retain_for: 1.month + end + end + + before do + connection.execute(<<~SQL) + CREATE TABLE my_model_example_table + (id serial not null, created_at timestamptz not null, primary key (id, created_at)) + PARTITION BY RANGE (created_at); + + CREATE TABLE #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.my_model_example_table_202104 + PARTITION OF my_model_example_table + FOR VALUES FROM ('2021-04-01') TO ('2021-05-01'); + + CREATE TABLE #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.my_model_example_table_202105 + PARTITION OF my_model_example_table + FOR VALUES FROM ('2021-05-01') TO ('2021-06-01'); + SQL + + # Also create all future partitions so that the sync is only trying to detach old partitions + my_model.partitioning_strategy.missing_partitions.each do |p| + connection.execute p.to_sql + end + end + + def num_tables + connection.select_value(<<~SQL) + SELECT COUNT(*) + FROM pg_class + where relkind IN ('r', 'p') + SQL + end + + it 'detaches exactly one partition' do + expect { subject }.to change { find_partitions(my_model.table_name, schema: Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA).size }.from(9).to(8) + end + + it 'detaches the old partition' do + expect { subject }.to change { has_partition(my_model, 2.months.ago.beginning_of_month) }.from(true).to(false) + end + + it 'deletes zero tables' do + expect { subject }.not_to change { num_tables } + end + + it 'creates the appropriate PendingPartitionDrop entry' do + subject + + pending_drop = Postgresql::DetachedPartition.find_by!(table_name: 'my_model_example_table_202104') + expect(pending_drop.drop_after).to eq(Time.current + described_class::RETAIN_DETACHED_PARTITIONS_FOR) + end + + # Postgres 11 does not support foreign keys to partitioned tables + if Gitlab::Database.main.version.to_f >= 12 + context 'when the model is the target of a foreign key' do + before do + connection.execute(<<~SQL) + create unique index idx_for_fk ON my_model_example_table(created_at); + + create table referencing_table ( + id bigserial primary key not null, + referencing_created_at timestamptz references my_model_example_table(created_at) + ); + SQL + end + + it 'does not detach partitions with a referenced foreign key' do + expect { subject }.not_to change { find_partitions(my_model.table_name).size } + end + end + end + end + + context 'creating and then detaching partitions for a table' do + let(:connection) { ActiveRecord::Base.connection } + let(:my_model) do + Class.new(ApplicationRecord) do + include PartitionedTable + + self.table_name = 'my_model_example_table' + + partitioned_by :created_at, strategy: :monthly, retain_for: 1.month + end + end + + before do + connection.execute(<<~SQL) + CREATE TABLE my_model_example_table + (id serial not null, created_at timestamptz not null, primary key (id, created_at)) + PARTITION BY RANGE (created_at); + SQL + end + + def num_partitions(model) + find_partitions(model.table_name, schema: Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA).size + end + + it 'creates partitions for the future then drops the oldest one after a month' do + # 1 month for the current month, 1 month for the old month that we're retaining data for, headroom + expected_num_partitions = (Gitlab::Database::Partitioning::MonthlyStrategy::HEADROOM + 2.months) / 1.month + expect { described_class.new([my_model]).sync_partitions }.to change { num_partitions(my_model) }.from(0).to(expected_num_partitions) + + travel 1.month + + expect { described_class.new([my_model]).sync_partitions }.to change { has_partition(my_model, 2.months.ago.beginning_of_month) }.from(true).to(false).and(change { num_partitions(my_model) }.by(0)) + end + end end diff --git a/spec/lib/gitlab/database/partitioning/partition_monitoring_spec.rb b/spec/lib/gitlab/database/partitioning/partition_monitoring_spec.rb index 67596211f71..7024cbd55ff 100644 --- a/spec/lib/gitlab/database/partitioning/partition_monitoring_spec.rb +++ b/spec/lib/gitlab/database/partitioning/partition_monitoring_spec.rb @@ -8,7 +8,7 @@ RSpec.describe Gitlab::Database::Partitioning::PartitionMonitoring do let(:models) { [model] } let(:model) { double(partitioning_strategy: partitioning_strategy, table_name: table) } - let(:partitioning_strategy) { double(missing_partitions: missing_partitions, current_partitions: current_partitions) } + let(:partitioning_strategy) { double(missing_partitions: missing_partitions, current_partitions: current_partitions, extra_partitions: extra_partitions) } let(:table) { "some_table" } let(:missing_partitions) do @@ -19,6 +19,10 @@ RSpec.describe Gitlab::Database::Partitioning::PartitionMonitoring do [double, double] end + let(:extra_partitions) do + [double, double, double] + end + it 'reports number of present partitions' do subject @@ -30,5 +34,11 @@ RSpec.describe Gitlab::Database::Partitioning::PartitionMonitoring do expect(Gitlab::Metrics.registry.get(:db_partitions_missing).get({ table: table })).to eq(missing_partitions.size) end + + it 'reports number of extra partitions' do + subject + + expect(Gitlab::Metrics.registry.get(:db_partitions_extra).get({ table: table })).to eq(extra_partitions.size) + end end end diff --git a/spec/lib/gitlab/database/postgres_foreign_key_spec.rb b/spec/lib/gitlab/database/postgres_foreign_key_spec.rb new file mode 100644 index 00000000000..ec39e5bfee7 --- /dev/null +++ b/spec/lib/gitlab/database/postgres_foreign_key_spec.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::PostgresForeignKey, type: :model do + # PostgresForeignKey does not `behaves_like 'a postgres model'` because it does not correspond 1-1 with a single entry + # in pg_class + + before do + ActiveRecord::Base.connection.execute(<<~SQL) + CREATE TABLE public.referenced_table ( + id bigserial primary key not null + ); + + CREATE TABLE public.other_referenced_table ( + id bigserial primary key not null + ); + + CREATE TABLE public.constrained_table ( + id bigserial primary key not null, + referenced_table_id bigint not null, + other_referenced_table_id bigint not null, + CONSTRAINT fk_constrained_to_referenced FOREIGN KEY(referenced_table_id) REFERENCES referenced_table(id), + CONSTRAINT fk_constrained_to_other_referenced FOREIGN KEY(other_referenced_table_id) + REFERENCES other_referenced_table(id) + ); + SQL + end + + describe '#by_referenced_table_identifier' do + it 'throws an error when the identifier name is not fully qualified' do + expect { described_class.by_referenced_table_identifier('referenced_table') }.to raise_error(ArgumentError, /not fully qualified/) + end + + it 'finds the foreign keys for the referenced table' do + expected = described_class.find_by!(name: 'fk_constrained_to_referenced') + + expect(described_class.by_referenced_table_identifier('public.referenced_table')).to contain_exactly(expected) + end + end +end diff --git a/spec/lib/gitlab/database/postgres_index_spec.rb b/spec/lib/gitlab/database/postgres_index_spec.rb index e1832219ebf..9088719d5a4 100644 --- a/spec/lib/gitlab/database/postgres_index_spec.rb +++ b/spec/lib/gitlab/database/postgres_index_spec.rb @@ -40,6 +40,37 @@ RSpec.describe Gitlab::Database::PostgresIndex do expect(types & %w(btree gist)).to eq(types) end + + context 'with leftover indexes' do + before do + ActiveRecord::Base.connection.execute(<<~SQL) + CREATE INDEX foobar_ccnew ON users (id); + CREATE INDEX foobar_ccnew1 ON users (id); + SQL + end + + subject { described_class.reindexing_support.map(&:name) } + + it 'excludes temporary indexes from reindexing' do + expect(subject).not_to include('foobar_ccnew') + expect(subject).not_to include('foobar_ccnew1') + end + end + end + + describe '.reindexing_leftovers' do + subject { described_class.reindexing_leftovers } + + before do + ActiveRecord::Base.connection.execute(<<~SQL) + CREATE INDEX foobar_ccnew ON users (id); + CREATE INDEX foobar_ccnew1 ON users (id); + SQL + end + + it 'retrieves leftover indexes matching the /_ccnew[0-9]*$/ pattern' do + expect(subject.map(&:name)).to eq(%w(foobar_ccnew foobar_ccnew1)) + end end describe '.not_match' do diff --git a/spec/lib/gitlab/database/reindexing_spec.rb b/spec/lib/gitlab/database/reindexing_spec.rb index 8aff99544ca..550f9db2b5b 100644 --- a/spec/lib/gitlab/database/reindexing_spec.rb +++ b/spec/lib/gitlab/database/reindexing_spec.rb @@ -26,14 +26,31 @@ RSpec.describe Gitlab::Database::Reindexing do end end - describe '.candidate_indexes' do - subject { described_class.candidate_indexes } + describe '.cleanup_leftovers!' do + subject { described_class.cleanup_leftovers! } + + before do + ApplicationRecord.connection.execute(<<~SQL) + CREATE INDEX foobar_ccnew ON users (id); + CREATE INDEX foobar_ccnew1 ON users (id); + SQL + end - it 'retrieves regular indexes that are no left-overs from previous runs' do - result = double - expect(Gitlab::Database::PostgresIndex).to receive_message_chain('not_match.reindexing_support').with('\_ccnew[0-9]*$').with(no_args).and_return(result) + it 'drops both leftover indexes' do + expect_query("SET lock_timeout TO '60000ms'") + expect_query("DROP INDEX CONCURRENTLY IF EXISTS \"public\".\"foobar_ccnew\"") + expect_query("RESET idle_in_transaction_session_timeout; RESET lock_timeout") + expect_query("SET lock_timeout TO '60000ms'") + expect_query("DROP INDEX CONCURRENTLY IF EXISTS \"public\".\"foobar_ccnew1\"") + expect_query("RESET idle_in_transaction_session_timeout; RESET lock_timeout") - expect(subject).to eq(result) + subject + end + + def expect_query(sql) + expect(ApplicationRecord.connection).to receive(:execute).ordered.with(sql).and_wrap_original do |method, sql| + method.call(sql.sub(/CONCURRENTLY/, '')) + end end end end diff --git a/spec/lib/gitlab/database/schema_migrations/context_spec.rb b/spec/lib/gitlab/database/schema_migrations/context_spec.rb index f3bed9b40d6..1f1943d00a3 100644 --- a/spec/lib/gitlab/database/schema_migrations/context_spec.rb +++ b/spec/lib/gitlab/database/schema_migrations/context_spec.rb @@ -3,7 +3,8 @@ require 'spec_helper' RSpec.describe Gitlab::Database::SchemaMigrations::Context do - let(:connection) { ActiveRecord::Base.connection } + let(:connection_class) { ActiveRecord::Base } + let(:connection) { connection_class.connection } let(:context) { described_class.new(connection) } @@ -12,13 +13,65 @@ RSpec.describe Gitlab::Database::SchemaMigrations::Context do expect(context.schema_directory).to eq(File.join(Rails.root, 'db/schema_migrations')) end - context 'multiple databases' do - let(:connection) { Ci::BaseModel.connection } + context 'CI database' do + let(:connection_class) { Ci::CiDatabaseRecord } it 'returns a directory path that is database specific' do skip_if_multiple_databases_not_setup - expect(context.schema_directory).to eq(File.join(Rails.root, 'db/ci_schema_migrations')) + expect(context.schema_directory).to eq(File.join(Rails.root, 'db/schema_migrations')) + end + end + + context 'multiple databases' do + let(:connection_class) do + Class.new(::ApplicationRecord) do + self.abstract_class = true + + def self.name + 'Gitlab::Database::SchemaMigrations::Context::TestConnection' + end + end + end + + let(:configuration_overrides) { {} } + + before do + connection_class.establish_connection( + ActiveRecord::Base + .connection_pool + .db_config + .configuration_hash + .merge(configuration_overrides) + ) + end + + after do + connection_class.remove_connection + end + + context 'when `schema_migrations_path` is configured as string' do + let(:configuration_overrides) do + { "schema_migrations_path" => "db/ci_schema_migrations" } + end + + it 'returns a configured directory path that' do + skip_if_multiple_databases_not_setup + + expect(context.schema_directory).to eq(File.join(Rails.root, 'db/ci_schema_migrations')) + end + end + + context 'when `schema_migrations_path` is configured as symbol' do + let(:configuration_overrides) do + { schema_migrations_path: "db/ci_schema_migrations" } + end + + it 'returns a configured directory path that' do + skip_if_multiple_databases_not_setup + + expect(context.schema_directory).to eq(File.join(Rails.root, 'db/ci_schema_migrations')) + end end end end diff --git a/spec/lib/gitlab/database/transaction/context_spec.rb b/spec/lib/gitlab/database/transaction/context_spec.rb new file mode 100644 index 00000000000..65d52b4d099 --- /dev/null +++ b/spec/lib/gitlab/database/transaction/context_spec.rb @@ -0,0 +1,144 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::Transaction::Context do + subject { described_class.new } + + let(:data) { subject.context } + + before do + stub_const("#{described_class}::LOG_THROTTLE", 100) + end + + describe '#set_start_time' do + before do + subject.set_start_time + end + + it 'sets start_time' do + expect(data).to have_key(:start_time) + end + end + + describe '#increment_savepoints' do + before do + 2.times { subject.increment_savepoints } + end + + it { expect(data[:savepoints]).to eq(2) } + end + + describe '#increment_rollbacks' do + before do + 3.times { subject.increment_rollbacks } + end + + it { expect(data[:rollbacks]).to eq(3) } + end + + describe '#increment_releases' do + before do + 4.times { subject.increment_releases } + end + + it { expect(data[:releases]).to eq(4) } + end + + describe '#set_depth' do + before do + subject.set_depth(2) + end + + it { expect(data[:depth]).to eq(2) } + end + + describe '#track_sql' do + before do + subject.track_sql('SELECT 1') + subject.track_sql('SELECT * FROM users') + end + + it { expect(data[:queries]).to eq(['SELECT 1', 'SELECT * FROM users']) } + end + + describe '#duration' do + before do + subject.set_start_time + end + + it { expect(subject.duration).to be >= 0 } + end + + context 'when depth is low' do + it 'does not log data upon COMMIT' do + expect(subject).not_to receive(:application_info) + + subject.commit + end + + it 'does not log data upon ROLLBACK' do + expect(subject).not_to receive(:application_info) + + subject.rollback + end + + it '#should_log? returns false' do + expect(subject.should_log?).to be false + end + end + + shared_examples 'logs transaction data' do + it 'logs once upon COMMIT' do + expect(subject).to receive(:application_info).and_call_original + + 2.times { subject.commit } + end + + it 'logs once upon ROLLBACK' do + expect(subject).to receive(:application_info).once + + 2.times { subject.rollback } + end + + it 'logs again when log throttle duration passes' do + expect(subject).to receive(:application_info).twice.and_call_original + + 2.times { subject.commit } + + data[:last_log_timestamp] -= (described_class::LOG_THROTTLE_DURATION + 1) + + subject.commit + end + + it '#should_log? returns true' do + expect(subject.should_log?).to be true + end + end + + context 'when depth exceeds threshold' do + before do + subject.set_depth(described_class::LOG_DEPTH_THRESHOLD + 1) + end + + it_behaves_like 'logs transaction data' + end + + context 'when savepoints count exceeds threshold' do + before do + data[:savepoints] = described_class::LOG_SAVEPOINTS_THRESHOLD + 1 + end + + it_behaves_like 'logs transaction data' + end + + context 'when duration exceeds threshold' do + before do + subject.set_start_time + + data[:start_time] -= (described_class::LOG_DURATION_S_THRESHOLD + 1) + end + + it_behaves_like 'logs transaction data' + end +end diff --git a/spec/lib/gitlab/database/transaction/observer_spec.rb b/spec/lib/gitlab/database/transaction/observer_spec.rb new file mode 100644 index 00000000000..7aa24217dc3 --- /dev/null +++ b/spec/lib/gitlab/database/transaction/observer_spec.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::Transaction::Observer do + # Use the delete DB strategy so that the test won't be wrapped in a transaction + describe '.instrument_transactions', :delete do + let(:transaction_context) { ActiveRecord::Base.connection.transaction_manager.transaction_context } + let(:context) { transaction_context.context } + + around do |example| + # Emulate production environment when SQL comments come first to avoid truncation + Marginalia::Comment.prepend_comment = true + subscriber = described_class.register! + + example.run + + ActiveSupport::Notifications.unsubscribe(subscriber) + Marginalia::Comment.prepend_comment = false + end + + it 'tracks transaction data', :aggregate_failures do + ActiveRecord::Base.transaction do + ActiveRecord::Base.transaction(requires_new: true) do + User.first + + expect(transaction_context).to be_a(::Gitlab::Database::Transaction::Context) + expect(context.keys).to match_array(%i(start_time depth savepoints queries)) + expect(context[:depth]).to eq(2) + expect(context[:savepoints]).to eq(1) + expect(context[:queries].length).to eq(1) + end + end + + expect(context[:depth]).to eq(2) + expect(context[:savepoints]).to eq(1) + expect(context[:releases]).to eq(1) + end + + describe '.extract_sql_command' do + using RSpec::Parameterized::TableSyntax + + where(:sql, :expected) do + 'SELECT 1' | 'SELECT 1' + '/* test comment */ SELECT 1' | 'SELECT 1' + '/* test comment */ ROLLBACK TO SAVEPOINT point1' | 'ROLLBACK TO SAVEPOINT ' + 'SELECT 1 /* trailing comment */' | 'SELECT 1 /* trailing comment */' + end + + with_them do + it do + expect(described_class.extract_sql_command(sql)).to eq(expected) + end + end + end + end +end diff --git a/spec/lib/gitlab/database/with_lock_retries_outside_transaction_spec.rb b/spec/lib/gitlab/database/with_lock_retries_outside_transaction_spec.rb index ff8e76311ae..0282a7af0df 100644 --- a/spec/lib/gitlab/database/with_lock_retries_outside_transaction_spec.rb +++ b/spec/lib/gitlab/database/with_lock_retries_outside_transaction_spec.rb @@ -37,22 +37,20 @@ RSpec.describe Gitlab::Database::WithLockRetriesOutsideTransaction do context 'when lock retry is enabled' do let(:lock_fiber) do Fiber.new do - configuration = ActiveRecordSecond.configurations.find_db_config(Rails.env).configuration_hash + # Initiating a separate DB connection for the lock + conn = ActiveRecord::Base.connection_pool.checkout - # Initiating a second DB connection for the lock - conn = ActiveRecordSecond.establish_connection(configuration).connection conn.transaction do conn.execute("LOCK TABLE #{Project.table_name} in exclusive mode") Fiber.yield end - ActiveRecordSecond.remove_connection # force disconnect + # Releasing the connection we requested + ActiveRecord::Base.connection_pool.checkin(conn) end end before do - stub_const('ActiveRecordSecond', Class.new(ActiveRecord::Base)) - lock_fiber.resume # start the transaction and lock the table end diff --git a/spec/lib/gitlab/database/with_lock_retries_spec.rb b/spec/lib/gitlab/database/with_lock_retries_spec.rb index 367f793b117..72074f06210 100644 --- a/spec/lib/gitlab/database/with_lock_retries_spec.rb +++ b/spec/lib/gitlab/database/with_lock_retries_spec.rb @@ -37,22 +37,19 @@ RSpec.describe Gitlab::Database::WithLockRetries do context 'when lock retry is enabled' do let(:lock_fiber) do Fiber.new do - configuration = ActiveRecordSecond.configurations.find_db_config(Rails.env).configuration_hash - - # Initiating a second DB connection for the lock - conn = ActiveRecordSecond.establish_connection(configuration).connection + # Initiating a separate DB connection for the lock + conn = ActiveRecord::Base.connection_pool.checkout conn.transaction do conn.execute("LOCK TABLE #{Project.table_name} in exclusive mode") Fiber.yield end - ActiveRecordSecond.remove_connection # force disconnect + # Releasing the connection we requested + ActiveRecord::Base.connection_pool.checkin(conn) end end before do - stub_const('ActiveRecordSecond', Class.new(ActiveRecord::Base)) - lock_fiber.resume # start the transaction and lock the table end diff --git a/spec/lib/gitlab/database_spec.rb b/spec/lib/gitlab/database_spec.rb index a834e41c019..c67b5af5e3c 100644 --- a/spec/lib/gitlab/database_spec.rb +++ b/spec/lib/gitlab/database_spec.rb @@ -15,32 +15,6 @@ RSpec.describe Gitlab::Database do end end - describe '.default_pool_size' do - before do - allow(Gitlab::Runtime).to receive(:max_threads).and_return(7) - end - - it 'returns the max thread size plus a fixed headroom of 10' do - expect(described_class.default_pool_size).to eq(17) - end - - it 'returns the max thread size plus a DB_POOL_HEADROOM if this env var is present' do - stub_env('DB_POOL_HEADROOM', '7') - - expect(described_class.default_pool_size).to eq(14) - end - end - - describe '.config' do - it 'returns a HashWithIndifferentAccess' do - expect(described_class.config).to be_an_instance_of(HashWithIndifferentAccess) - end - - it 'returns a default pool size' do - expect(described_class.config).to include(pool: described_class.default_pool_size) - end - end - describe '.has_config?' do context 'two tier database config' do before do @@ -114,108 +88,11 @@ RSpec.describe Gitlab::Database do end end - describe '.adapter_name' do - it 'returns the name of the adapter' do - expect(described_class.adapter_name).to be_an_instance_of(String) - end - - it 'returns Unknown when using anything else' do - allow(described_class).to receive(:postgresql?).and_return(false) - - expect(described_class.human_adapter_name).to eq('Unknown') - end - end - - describe '.human_adapter_name' do - it 'returns PostgreSQL when using PostgreSQL' do - expect(described_class.human_adapter_name).to eq('PostgreSQL') - end - end - - describe '.system_id' do - it 'returns the PostgreSQL system identifier' do - expect(described_class.system_id).to be_an_instance_of(Integer) - end - end - - describe '.disable_prepared_statements' do - around do |example| - original_config = ::Gitlab::Database.config - - example.run - - ActiveRecord::Base.establish_connection(original_config) - end - - it 'disables prepared statements' do - ActiveRecord::Base.establish_connection(::Gitlab::Database.config.merge(prepared_statements: true)) - expect(ActiveRecord::Base.connection.prepared_statements).to eq(true) - - expect(ActiveRecord::Base).to receive(:establish_connection) - .with(a_hash_including({ 'prepared_statements' => false })).and_call_original - - described_class.disable_prepared_statements - - expect(ActiveRecord::Base.connection.prepared_statements).to eq(false) - end - end - - describe '.postgresql?' do - subject { described_class.postgresql? } - - it { is_expected.to satisfy { |val| val == true || val == false } } - end - - describe '.version' do - around do |example| - described_class.instance_variable_set(:@version, nil) - example.run - described_class.instance_variable_set(:@version, nil) - end - - context "on postgresql" do - it "extracts the version number" do - allow(described_class).to receive(:database_version) - .and_return("PostgreSQL 9.4.4 on x86_64-apple-darwin14.3.0") - - expect(described_class.version).to eq '9.4.4' - end - end - - it 'memoizes the result' do - count = ActiveRecord::QueryRecorder - .new { 2.times { described_class.version } } - .count - - expect(count).to eq(1) - end - end - - describe '.postgresql_minimum_supported_version?' do - it 'returns false when using PostgreSQL 10' do - allow(described_class).to receive(:version).and_return('10') - - expect(described_class.postgresql_minimum_supported_version?).to eq(false) - end - - it 'returns false when using PostgreSQL 11' do - allow(described_class).to receive(:version).and_return('11') - - expect(described_class.postgresql_minimum_supported_version?).to eq(false) - end - - it 'returns true when using PostgreSQL 12' do - allow(described_class).to receive(:version).and_return('12') - - expect(described_class.postgresql_minimum_supported_version?).to eq(true) - end - end - describe '.check_postgres_version_and_print_warning' do subject { described_class.check_postgres_version_and_print_warning } it 'prints a warning if not compliant with minimum postgres version' do - allow(described_class).to receive(:postgresql_minimum_supported_version?).and_return(false) + allow(described_class.main).to receive(:postgresql_minimum_supported_version?).and_return(false) expect(Kernel).to receive(:warn).with(/You are using PostgreSQL/) @@ -223,7 +100,7 @@ RSpec.describe Gitlab::Database do end it 'doesnt print a warning if compliant with minimum postgres version' do - allow(described_class).to receive(:postgresql_minimum_supported_version?).and_return(true) + allow(described_class.main).to receive(:postgresql_minimum_supported_version?).and_return(true) expect(Kernel).not_to receive(:warn).with(/You are using PostgreSQL/) @@ -231,7 +108,7 @@ RSpec.describe Gitlab::Database do end it 'doesnt print a warning in Rails runner environment' do - allow(described_class).to receive(:postgresql_minimum_supported_version?).and_return(false) + allow(described_class.main).to receive(:postgresql_minimum_supported_version?).and_return(false) allow(Gitlab::Runtime).to receive(:rails_runner?).and_return(true) expect(Kernel).not_to receive(:warn).with(/You are using PostgreSQL/) @@ -240,13 +117,13 @@ RSpec.describe Gitlab::Database do end it 'ignores ActiveRecord errors' do - allow(described_class).to receive(:postgresql_minimum_supported_version?).and_raise(ActiveRecord::ActiveRecordError) + allow(described_class.main).to receive(:postgresql_minimum_supported_version?).and_raise(ActiveRecord::ActiveRecordError) expect { subject }.not_to raise_error end it 'ignores Postgres errors' do - allow(described_class).to receive(:postgresql_minimum_supported_version?).and_raise(PG::Error) + allow(described_class.main).to receive(:postgresql_minimum_supported_version?).and_raise(PG::Error) expect { subject }.not_to raise_error end @@ -262,244 +139,19 @@ RSpec.describe Gitlab::Database do it { expect(described_class.nulls_first_order('column', 'DESC')).to eq 'column DESC NULLS FIRST'} end - describe '.with_connection_pool' do - it 'creates a new connection pool and disconnect it after used' do - closed_pool = nil - - described_class.with_connection_pool(1) do |pool| - pool.with_connection do |connection| - connection.execute('SELECT 1 AS value') - end - - expect(pool).to be_connected - - closed_pool = pool - end - - expect(closed_pool).not_to be_connected - end - - it 'disconnects the pool even an exception was raised' do - error = Class.new(RuntimeError) - closed_pool = nil - - begin - described_class.with_connection_pool(1) do |pool| - pool.with_connection do |connection| - connection.execute('SELECT 1 AS value') - end - - closed_pool = pool - - raise error, 'boom' - end - rescue error - end - - expect(closed_pool).not_to be_connected - end - end - - describe '.bulk_insert' do - before do - allow(described_class).to receive(:connection).and_return(connection) - allow(connection).to receive(:quote_column_name, &:itself) - allow(connection).to receive(:quote, &:itself) - allow(connection).to receive(:execute) - end - - let(:connection) { double(:connection) } - - let(:rows) do - [ - { a: 1, b: 2, c: 3 }, - { c: 6, a: 4, b: 5 } - ] - end - - it 'does nothing with empty rows' do - expect(connection).not_to receive(:execute) - - described_class.bulk_insert('test', []) - end - - it 'uses the ordering from the first row' do - expect(connection).to receive(:execute) do |sql| - expect(sql).to include('(1, 2, 3)') - expect(sql).to include('(4, 5, 6)') - end - - described_class.bulk_insert('test', rows) - end - - it 'quotes column names' do - expect(connection).to receive(:quote_column_name).with(:a) - expect(connection).to receive(:quote_column_name).with(:b) - expect(connection).to receive(:quote_column_name).with(:c) - - described_class.bulk_insert('test', rows) - end - - it 'quotes values' do - 1.upto(6) do |i| - expect(connection).to receive(:quote).with(i) - end - - described_class.bulk_insert('test', rows) - end - - it 'does not quote values of a column in the disable_quote option' do - [1, 2, 4, 5].each do |i| - expect(connection).to receive(:quote).with(i) - end - - described_class.bulk_insert('test', rows, disable_quote: :c) - end - - it 'does not quote values of columns in the disable_quote option' do - [2, 5].each do |i| - expect(connection).to receive(:quote).with(i) - end - - described_class.bulk_insert('test', rows, disable_quote: [:a, :c]) - end - - it 'handles non-UTF-8 data' do - expect { described_class.bulk_insert('test', [{ a: "\255" }]) }.not_to raise_error - end - - context 'when using PostgreSQL' do - it 'allows the returning of the IDs of the inserted rows' do - result = double(:result, values: [['10']]) - - expect(connection) - .to receive(:execute) - .with(/RETURNING id/) - .and_return(result) - - ids = described_class - .bulk_insert('test', [{ number: 10 }], return_ids: true) - - expect(ids).to eq([10]) - end - - it 'allows setting the upsert to do nothing' do - expect(connection) - .to receive(:execute) - .with(/ON CONFLICT DO NOTHING/) - - described_class - .bulk_insert('test', [{ number: 10 }], on_conflict: :do_nothing) - end - end - end - - describe '.create_connection_pool' do - it 'creates a new connection pool with specific pool size' do - pool = described_class.create_connection_pool(5) - - begin - expect(pool) - .to be_kind_of(ActiveRecord::ConnectionAdapters::ConnectionPool) - - expect(pool.db_config.pool).to eq(5) - ensure - pool.disconnect! - end - end - - it 'allows setting of a custom hostname' do - pool = described_class.create_connection_pool(5, '127.0.0.1') - - begin - expect(pool.db_config.host).to eq('127.0.0.1') - ensure - pool.disconnect! - end - end - - it 'allows setting of a custom hostname and port' do - pool = described_class.create_connection_pool(5, '127.0.0.1', 5432) - - begin - expect(pool.db_config.host).to eq('127.0.0.1') - expect(pool.db_config.configuration_hash[:port]).to eq(5432) - ensure - pool.disconnect! - end - end - end - - describe '.cached_column_exists?' do - it 'only retrieves data once' do - expect(ActiveRecord::Base.connection).to receive(:columns).once.and_call_original - - 2.times do - expect(described_class.cached_column_exists?(:projects, :id)).to be_truthy - expect(described_class.cached_column_exists?(:projects, :bogus_column)).to be_falsey - end - end - end - - describe '.cached_table_exists?' do - it 'only retrieves data once per table' do - expect(ActiveRecord::Base.connection).to receive(:data_source_exists?).with(:projects).once.and_call_original - expect(ActiveRecord::Base.connection).to receive(:data_source_exists?).with(:bogus_table_name).once.and_call_original - - 2.times do - expect(described_class.cached_table_exists?(:projects)).to be_truthy - expect(described_class.cached_table_exists?(:bogus_table_name)).to be_falsey - end - end - - it 'returns false when database does not exist' do - expect(ActiveRecord::Base).to receive(:connection) { raise ActiveRecord::NoDatabaseError, 'broken' } - - expect(described_class.cached_table_exists?(:projects)).to be(false) - end - end - - describe '.exists?' do - it 'returns true if `ActiveRecord::Base.connection` succeeds' do - expect(ActiveRecord::Base).to receive(:connection) - - expect(described_class.exists?).to be(true) - end - - it 'returns false if `ActiveRecord::Base.connection` fails' do - expect(ActiveRecord::Base).to receive(:connection) { raise ActiveRecord::NoDatabaseError, 'broken' } - - expect(described_class.exists?).to be(false) - end - end - - describe '.get_write_location' do - it 'returns a string' do + describe '.db_config_name' do + it 'returns the db_config name for the connection' do connection = ActiveRecord::Base.connection - expect(described_class.get_write_location(connection)).to be_a(String) - end - - it 'returns nil if there are no results' do - connection = double(select_all: []) - - expect(described_class.get_write_location(connection)).to be_nil - end - end - - describe '.dbname' do - it 'returns the dbname for the connection' do - connection = ActiveRecord::Base.connection - - expect(described_class.dbname(connection)).to be_a(String) - expect(described_class.dbname(connection)).to eq(connection.pool.db_config.database) + expect(described_class.db_config_name(connection)).to be_a(String) + expect(described_class.db_config_name(connection)).to eq(connection.pool.db_config.name) end context 'when the pool is a NullPool' do it 'returns unknown' do connection = double(:active_record_connection, pool: ActiveRecord::ConnectionAdapters::NullPool.new) - expect(described_class.dbname(connection)).to eq('unknown') + expect(described_class.db_config_name(connection)).to eq('unknown') end end end @@ -516,42 +168,6 @@ RSpec.describe Gitlab::Database do end end - describe '.read_only?' do - it 'returns false' do - expect(described_class.read_only?).to be_falsey - end - end - - describe '.db_read_only?' do - before do - allow(ActiveRecord::Base.connection).to receive(:execute).and_call_original - end - - it 'detects a read-only database' do - allow(ActiveRecord::Base.connection).to receive(:execute).with('SELECT pg_is_in_recovery()').and_return([{ "pg_is_in_recovery" => "t" }]) - - expect(described_class.db_read_only?).to be_truthy - end - - it 'detects a read-only database' do - allow(ActiveRecord::Base.connection).to receive(:execute).with('SELECT pg_is_in_recovery()').and_return([{ "pg_is_in_recovery" => true }]) - - expect(described_class.db_read_only?).to be_truthy - end - - it 'detects a read-write database' do - allow(ActiveRecord::Base.connection).to receive(:execute).with('SELECT pg_is_in_recovery()').and_return([{ "pg_is_in_recovery" => "f" }]) - - expect(described_class.db_read_only?).to be_falsey - end - - it 'detects a read-write database' do - allow(ActiveRecord::Base.connection).to receive(:execute).with('SELECT pg_is_in_recovery()').and_return([{ "pg_is_in_recovery" => false }]) - - expect(described_class.db_read_only?).to be_falsey - end - end - describe '#sanitize_timestamp' do let(:max_timestamp) { Time.at((1 << 31) - 1) } @@ -574,6 +190,18 @@ RSpec.describe Gitlab::Database do end end + describe '.read_only?' do + it 'returns false' do + expect(described_class.read_only?).to eq(false) + end + end + + describe '.read_write' do + it 'returns true' do + expect(described_class.read_write?).to eq(true) + end + end + describe 'ActiveRecordBaseTransactionMetrics' do def subscribe_events events = [] diff --git a/spec/lib/gitlab/email/handler/create_note_handler_spec.rb b/spec/lib/gitlab/email/handler/create_note_handler_spec.rb index e76a5d3fe32..c0ac40e3249 100644 --- a/spec/lib/gitlab/email/handler/create_note_handler_spec.rb +++ b/spec/lib/gitlab/email/handler/create_note_handler_spec.rb @@ -110,6 +110,60 @@ RSpec.describe Gitlab::Email::Handler::CreateNoteHandler do end end + context 'when email contains reply' do + shared_examples 'no content message' do + context 'when email contains quoted text only' do + let(:email_raw) { fixture_file('emails/no_content_with_quote.eml') } + + it 'raises an EmptyEmailError' do + expect { receiver.execute }.to raise_error(Gitlab::Email::EmptyEmailError) + end + end + + context 'when email contains quoted text and quick commands only' do + let(:email_raw) { fixture_file('emails/commands_only_reply.eml') } + + it 'does not create a discussion' do + expect { receiver.execute }.not_to change { noteable.notes.count } + end + end + end + + context 'when noteable is not an issue' do + let_it_be(:note) { create(:note_on_merge_request, project: project) } + + it_behaves_like 'no content message' + + context 'when email contains text, quoted text and quick commands' do + let(:email_raw) { fixture_file('emails/commands_in_reply.eml') } + + it 'creates a discussion without appended reply' do + expect { receiver.execute }.to change { noteable.notes.count }.by(1) + new_note = noteable.notes.last + + expect(new_note.note).not_to include('<details><summary>...</summary>') + end + end + end + + context 'when noteable is an issue' do + let_it_be(:note) { create(:note_on_issue, project: project) } + + it_behaves_like 'no content message' + + context 'when email contains text, quoted text and quick commands' do + let(:email_raw) { fixture_file('emails/commands_in_reply.eml') } + + it 'creates a discussion with appended reply' do + expect { receiver.execute }.to change { noteable.notes.count }.by(1) + new_note = noteable.notes.last + + expect(new_note.note).to include('<details><summary>...</summary>') + end + end + end + end + context 'when note is not a discussion' do let(:note) { create(:note_on_merge_request, project: project) } diff --git a/spec/lib/gitlab/email/message/in_product_marketing/admin_verify_spec.rb b/spec/lib/gitlab/email/message/in_product_marketing/admin_verify_spec.rb new file mode 100644 index 00000000000..b5c3415fe12 --- /dev/null +++ b/spec/lib/gitlab/email/message/in_product_marketing/admin_verify_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Email::Message::InProductMarketing::AdminVerify do + let_it_be(:group) { build(:group) } + let_it_be(:user) { build(:user) } + + let(:series) { 0 } + + subject(:message) { described_class.new(group: group, user: user, series: series)} + + describe 'public methods' do + it 'returns value for series', :aggregate_failures do + expect(message.subject_line).to eq 'Create a custom CI runner with just a few clicks' + expect(message.tagline).to be_nil + expect(message.title).to eq 'Spin up an autoscaling runner in GitLab' + expect(message.subtitle).to eq 'Use our AWS cloudformation template to spin up your runners in just a few clicks!' + expect(message.body_line1).to be_empty + expect(message.body_line2).to be_empty + expect(message.cta_text).to eq 'Create a custom runner' + expect(message.logo_path).to eq 'mailers/in_product_marketing/admin_verify-0.png' + end + + describe '#progress' do + subject { message.progress } + + before do + allow(Gitlab).to receive(:com?).and_return(is_gitlab_com) + end + + context 'on gitlab.com' do + let(:is_gitlab_com) { true } + + it { is_expected.to eq('This is email 1 of 1 in the Admin series.') } + end + + context 'not on gitlab.com' do + let(:is_gitlab_com) { false } + + it { is_expected.to include('This is email 1 of 1 in the Admin series', Gitlab::Routing.url_helpers.profile_notifications_url) } + end + end + end +end diff --git a/spec/lib/gitlab/email/message/in_product_marketing/team_short_spec.rb b/spec/lib/gitlab/email/message/in_product_marketing/team_short_spec.rb new file mode 100644 index 00000000000..daeacef53f6 --- /dev/null +++ b/spec/lib/gitlab/email/message/in_product_marketing/team_short_spec.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Email::Message::InProductMarketing::TeamShort do + using RSpec::Parameterized::TableSyntax + + let_it_be(:group) { build(:group) } + let_it_be(:user) { build(:user) } + + let(:series) { 0 } + + subject(:message) { described_class.new(group: group, user: user, series: series)} + + describe 'public methods' do + it 'returns value for series', :aggregate_failures do + expect(message.subject_line).to eq 'Team up in GitLab for greater efficiency' + expect(message.tagline).to be_nil + expect(message.title).to eq 'Turn coworkers into collaborators' + expect(message.subtitle).to eq 'Invite your team today to build better code (and processes) together' + expect(message.body_line1).to be_empty + expect(message.body_line2).to be_empty + expect(message.cta_text).to eq 'Invite your colleagues today' + expect(message.logo_path).to eq 'mailers/in_product_marketing/team-0.png' + end + + describe '#progress' do + subject { message.progress } + + before do + allow(Gitlab).to receive(:com?).and_return(is_gitlab_com) + end + + context 'on gitlab.com' do + let(:is_gitlab_com) { true } + + it { is_expected.to include('This is email 1 of 4 in the Team series') } + end + + context 'not on gitlab.com' do + let(:is_gitlab_com) { false } + + it { is_expected.to include('This is email 1 of 4 in the Team series', Gitlab::Routing.url_helpers.profile_notifications_url) } + end + end + end +end diff --git a/spec/lib/gitlab/email/message/in_product_marketing/team_spec.rb b/spec/lib/gitlab/email/message/in_product_marketing/team_spec.rb index f72994fcce1..eca8ba1df00 100644 --- a/spec/lib/gitlab/email/message/in_product_marketing/team_spec.rb +++ b/spec/lib/gitlab/email/message/in_product_marketing/team_spec.rb @@ -23,6 +23,26 @@ RSpec.describe Gitlab::Email::Message::InProductMarketing::Team do expect(message.body_line2).to be_present expect(message.cta_text).to be_present end + + describe '#progress' do + subject { message.progress } + + before do + allow(Gitlab).to receive(:com?).and_return(is_gitlab_com) + end + + context 'on gitlab.com' do + let(:is_gitlab_com) { true } + + it { is_expected.to include("This is email #{series + 2} of 4 in the Team series") } + end + + context 'not on gitlab.com' do + let(:is_gitlab_com) { false } + + it { is_expected.to include("This is email #{series + 2} of 4 in the Team series", Gitlab::Routing.url_helpers.profile_notifications_url) } + end + end end context 'with series 2' do @@ -37,6 +57,26 @@ RSpec.describe Gitlab::Email::Message::InProductMarketing::Team do expect(message.body_line2).to be_present expect(message.cta_text).to be_present end + + describe '#progress' do + subject { message.progress } + + before do + allow(Gitlab).to receive(:com?).and_return(is_gitlab_com) + end + + context 'on gitlab.com' do + let(:is_gitlab_com) { true } + + it { is_expected.to include('This is email 4 of 4 in the Team series') } + end + + context 'not on gitlab.com' do + let(:is_gitlab_com) { false } + + it { is_expected.to include('This is email 4 of 4 in the Team series', Gitlab::Routing.url_helpers.profile_notifications_url) } + end + end end end end diff --git a/spec/lib/gitlab/email/message/in_product_marketing/trial_short_spec.rb b/spec/lib/gitlab/email/message/in_product_marketing/trial_short_spec.rb new file mode 100644 index 00000000000..ebad4672eb3 --- /dev/null +++ b/spec/lib/gitlab/email/message/in_product_marketing/trial_short_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Email::Message::InProductMarketing::TrialShort do + let_it_be(:group) { build(:group) } + let_it_be(:user) { build(:user) } + + let(:series) { 0 } + + subject(:message) { described_class.new(group: group, user: user, series: series)} + + describe 'public methods' do + it 'returns value for series', :aggregate_failures do + expect(message.subject_line).to eq 'Be a DevOps hero' + expect(message.tagline).to be_nil + expect(message.title).to eq 'Expand your DevOps journey with a free GitLab trial' + expect(message.subtitle).to eq 'Start your trial today to experience single application success and discover all the features of GitLab Ultimate for free!' + expect(message.body_line1).to be_empty + expect(message.body_line2).to be_empty + expect(message.cta_text).to eq 'Start a trial' + expect(message.logo_path).to eq 'mailers/in_product_marketing/trial-0.png' + end + + describe '#progress' do + subject { message.progress } + + before do + allow(Gitlab).to receive(:com?).and_return(is_gitlab_com) + end + + context 'on gitlab.com' do + let(:is_gitlab_com) { true } + + it { is_expected.to eq('This is email 1 of 4 in the Trial series.') } + end + + context 'not on gitlab.com' do + let(:is_gitlab_com) { false } + + it { is_expected.to include('This is email 1 of 4 in the Trial series', Gitlab::Routing.url_helpers.profile_notifications_url) } + end + end + end +end diff --git a/spec/lib/gitlab/email/message/in_product_marketing/trial_spec.rb b/spec/lib/gitlab/email/message/in_product_marketing/trial_spec.rb index 5f7639a9ed6..3e18b8e35b6 100644 --- a/spec/lib/gitlab/email/message/in_product_marketing/trial_spec.rb +++ b/spec/lib/gitlab/email/message/in_product_marketing/trial_spec.rb @@ -23,6 +23,26 @@ RSpec.describe Gitlab::Email::Message::InProductMarketing::Trial do expect(message.body_line2).to be_present expect(message.cta_text).to be_present end + + describe '#progress' do + subject { message.progress } + + before do + allow(Gitlab).to receive(:com?).and_return(is_gitlab_com) + end + + context 'on gitlab.com' do + let(:is_gitlab_com) { true } + + it { is_expected.to eq("This is email #{series + 2} of 4 in the Trial series.") } + end + + context 'not on gitlab.com' do + let(:is_gitlab_com) { false } + + it { is_expected.to include("This is email #{series + 2} of 4 in the Trial series", Gitlab::Routing.url_helpers.profile_notifications_url) } + end + end end end end diff --git a/spec/lib/gitlab/email/reply_parser_spec.rb b/spec/lib/gitlab/email/reply_parser_spec.rb index bc4c6cf007d..3b01b568fb4 100644 --- a/spec/lib/gitlab/email/reply_parser_spec.rb +++ b/spec/lib/gitlab/email/reply_parser_spec.rb @@ -228,5 +228,21 @@ RSpec.describe Gitlab::Email::ReplyParser do BODY ) end + + it "appends trimmed reply when when append_reply option is true" do + body = <<-BODY.strip_heredoc.chomp + The reply by email functionality should be extended to allow creating a new issue by email. + even when the email is forwarded to the project which may include lines that begin with ">" + + there should be a quote below this line: + BODY + + reply = <<-BODY.strip_heredoc.chomp + > this is a quote + BODY + + expect(test_parse_body(fixture_file("emails/valid_new_issue_with_quote.eml"), { append_reply: true })) + .to contain_exactly(body, reply) + end end end diff --git a/spec/lib/gitlab/encoding_helper_spec.rb b/spec/lib/gitlab/encoding_helper_spec.rb index cf0d1577314..268ac5dcc21 100644 --- a/spec/lib/gitlab/encoding_helper_spec.rb +++ b/spec/lib/gitlab/encoding_helper_spec.rb @@ -3,6 +3,8 @@ require "spec_helper" RSpec.describe Gitlab::EncodingHelper do + using RSpec::Parameterized::TableSyntax + let(:ext_class) { Class.new { extend Gitlab::EncodingHelper } } let(:binary_string) { File.read(Rails.root + "spec/fixtures/dk.png") } @@ -90,6 +92,22 @@ RSpec.describe Gitlab::EncodingHelper do end end + describe '#encode_utf8_no_detect' do + where(:input, :expected) do + "abcd" | "abcd" + "DzDzDz" | "DzDzDz" + "\xC7\xB2\xC7DzDzDz" | "Dz�DzDzDz" + "🐤🐤🐤🐤\xF0\x9F\x90" | "🐤🐤🐤🐤�" + end + + with_them do + it 'drops invalid UTF-8' do + expect(ext_class.encode_utf8_no_detect(input.dup.force_encoding(Encoding::ASCII_8BIT))).to eq(expected) + expect(ext_class.encode_utf8_no_detect(input)).to eq(expected) + end + end + end + describe '#encode_utf8' do [ ["nil", nil, nil], diff --git a/spec/lib/gitlab/error_tracking/stack_trace_highlight_decorator_spec.rb b/spec/lib/gitlab/error_tracking/stack_trace_highlight_decorator_spec.rb index 5c496d653b2..577d59798da 100644 --- a/spec/lib/gitlab/error_tracking/stack_trace_highlight_decorator_spec.rb +++ b/spec/lib/gitlab/error_tracking/stack_trace_highlight_decorator_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::ErrorTracking::StackTraceHighlightDecorator do - let(:error_event) { build(:error_tracking_error_event) } + let(:error_event) { build(:error_tracking_sentry_error_event) } describe '.decorate' do subject(:decorate) { described_class.decorate(error_event) } diff --git a/spec/lib/gitlab/etag_caching/router/restful_spec.rb b/spec/lib/gitlab/etag_caching/router/restful_spec.rb index 877789b320f..1f5cac09b6d 100644 --- a/spec/lib/gitlab/etag_caching/router/restful_spec.rb +++ b/spec/lib/gitlab/etag_caching/router/restful_spec.rb @@ -87,12 +87,18 @@ RSpec.describe Gitlab::EtagCaching::Router::Restful do end it 'matches the environments path' do - result = match_route('/my-group/my-project/environments.json') + result = match_route('/my-group/my-project/-/environments.json') expect(result).to be_present expect(result.name).to eq 'environments' end + it 'does not match the operations environments list path' do + result = match_route('/-/operations/environments.json') + + expect(result).not_to be_present + end + it 'matches pipeline#show endpoint' do result = match_route('/my-group/my-project/-/pipelines/2.json') diff --git a/spec/lib/gitlab/experimentation/controller_concern_spec.rb b/spec/lib/gitlab/experimentation/controller_concern_spec.rb index 7a619c9f155..8535d72a61f 100644 --- a/spec/lib/gitlab/experimentation/controller_concern_spec.rb +++ b/spec/lib/gitlab/experimentation/controller_concern_spec.rb @@ -3,6 +3,8 @@ require 'spec_helper' RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do + include TrackingHelpers + before do stub_const('Gitlab::Experimentation::EXPERIMENTS', { backwards_compatible_test_experiment: { @@ -43,7 +45,7 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do let(:cookie_value) { nil } before do - request.headers['DNT'] = do_not_track if do_not_track.present? + stub_do_not_track(do_not_track) if do_not_track.present? request.cookies[:experimentation_subject_id] = cookie_value if cookie_value get :index @@ -242,7 +244,7 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do context 'do not track is disabled' do before do - request.headers['DNT'] = '0' + stub_do_not_track('0') end it 'does track the event' do @@ -260,7 +262,7 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do context 'do not track enabled' do before do - request.headers['DNT'] = '1' + stub_do_not_track('1') end it 'does not track the event' do @@ -396,7 +398,7 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do context 'do not track disabled' do before do - request.headers['DNT'] = '0' + stub_do_not_track('0') end it 'pushes the right parameters to gon' do @@ -414,7 +416,7 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do context 'do not track enabled' do before do - request.headers['DNT'] = '1' + stub_do_not_track('1') end it 'does not push data to gon' do @@ -525,7 +527,7 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do context 'is disabled' do before do - request.headers['DNT'] = '0' + stub_do_not_track('0') stub_experiment_for_subject(test_experiment: false) end @@ -538,7 +540,7 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do context 'is enabled' do before do - request.headers['DNT'] = '1' + stub_do_not_track('1') end it 'does not call add_user on the Experiment model' do diff --git a/spec/lib/gitlab/experimentation_spec.rb b/spec/lib/gitlab/experimentation_spec.rb index 10bfa9e8d0e..c486538a260 100644 --- a/spec/lib/gitlab/experimentation_spec.rb +++ b/spec/lib/gitlab/experimentation_spec.rb @@ -2,22 +2,6 @@ require 'spec_helper' -# As each associated, backwards-compatible experiment gets cleaned up and removed from the EXPERIMENTS list, its key will also get removed from this list. Once the list here is empty, we can remove the backwards compatibility code altogether. -# Originally created as part of https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45733 for https://gitlab.com/gitlab-org/gitlab/-/issues/270858. -RSpec.describe Gitlab::Experimentation::EXPERIMENTS do - it 'temporarily ensures we know what experiments exist for backwards compatibility' do - expected_experiment_keys = [ - :invite_members_empty_group_version_a, - :contact_sales_btn_in_app - ] - - backwards_compatible_experiment_keys = described_class.filter { |_, v| v[:use_backwards_compatible_subject_index] }.keys - - expect(backwards_compatible_experiment_keys).not_to be_empty, "Oh, hey! Let's clean up that :use_backwards_compatible_subject_index stuff now :D" - expect(backwards_compatible_experiment_keys).to match(expected_experiment_keys) - end -end - RSpec.describe Gitlab::Experimentation do using RSpec::Parameterized::TableSyntax diff --git a/spec/lib/gitlab/fake_application_settings_spec.rb b/spec/lib/gitlab/fake_application_settings_spec.rb index ec32afcfb7b..b300498e898 100644 --- a/spec/lib/gitlab/fake_application_settings_spec.rb +++ b/spec/lib/gitlab/fake_application_settings_spec.rb @@ -6,27 +6,35 @@ RSpec.describe Gitlab::FakeApplicationSettings do let(:defaults) do described_class.defaults.merge( foobar: 'asdf', - 'test?' => 123 + 'test?'.to_sym => 123, + # these two settings have no default in ApplicationSettingImplementation, + # so we need to set one here + domain_denylist: [], + archive_builds_in_seconds: nil ) end let(:setting) { described_class.new(defaults) } - it 'wraps OpenStruct variables properly' do + it 'defines methods for default attributes' do expect(setting.password_authentication_enabled_for_web).to be_truthy expect(setting.signup_enabled).to be_truthy expect(setting.foobar).to eq('asdf') end - it 'defines predicate methods' do + it 'defines predicate methods for boolean properties' do expect(setting.password_authentication_enabled_for_web?).to be_truthy expect(setting.signup_enabled?).to be_truthy end - it 'does not define a predicate method' do + it 'does not define a predicate method for non-boolean properties' do expect(setting.foobar?).to be_nil end + it 'returns nil for undefined attributes' do + expect(setting.does_not_exist).to be_nil + end + it 'does not override an existing predicate method' do expect(setting.test?).to eq(123) end diff --git a/spec/lib/gitlab/form_builders/gitlab_ui_form_builder_spec.rb b/spec/lib/gitlab/form_builders/gitlab_ui_form_builder_spec.rb new file mode 100644 index 00000000000..a46846e9820 --- /dev/null +++ b/spec/lib/gitlab/form_builders/gitlab_ui_form_builder_spec.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::FormBuilders::GitlabUiFormBuilder do + let_it_be(:user) { build(:user) } + let_it_be(:fake_template) do + Object.new.tap do |template| + template.extend ActionView::Helpers::FormHelper + template.extend ActionView::Helpers::FormOptionsHelper + template.extend ActionView::Helpers::TagHelper + template.extend ActionView::Context + end + end + + let_it_be(:form_builder) { described_class.new(:user, user, fake_template, {}) } + + describe '#gitlab_ui_checkbox_component' do + let(:optional_args) { {} } + + subject(:checkbox_html) { form_builder.gitlab_ui_checkbox_component(:view_diffs_file_by_file, "Show one file at a time on merge request's Changes tab", **optional_args) } + + context 'without optional arguments' do + it 'renders correct html' do + expected_html = <<~EOS + <div class="gl-form-checkbox custom-control custom-checkbox"> + <input name="user[view_diffs_file_by_file]" type="hidden" value="0" /> + <input class="custom-control-input" type="checkbox" value="1" name="user[view_diffs_file_by_file]" id="user_view_diffs_file_by_file" /> + <label class="custom-control-label" for="user_view_diffs_file_by_file"> + Show one file at a time on merge request's Changes tab + </label> + </div> + EOS + + expect(checkbox_html).to eq(html_strip_whitespace(expected_html)) + end + end + + context 'with optional arguments' do + let(:optional_args) do + { + help_text: 'Instead of all the files changed, show only one file at a time.', + checkbox_options: { class: 'checkbox-foo-bar' }, + label_options: { class: 'label-foo-bar' }, + checked_value: '3', + unchecked_value: '1' + } + end + + it 'renders help text' do + expected_html = <<~EOS + <div class="gl-form-checkbox custom-control custom-checkbox"> + <input name="user[view_diffs_file_by_file]" type="hidden" value="1" /> + <input class="custom-control-input checkbox-foo-bar" type="checkbox" value="3" name="user[view_diffs_file_by_file]" id="user_view_diffs_file_by_file" /> + <label class="custom-control-label label-foo-bar" for="user_view_diffs_file_by_file"> + <span>Show one file at a time on merge request's Changes tab</span> + <p class="help-text">Instead of all the files changed, show only one file at a time.</p> + </label> + </div> + EOS + + expect(checkbox_html).to eq(html_strip_whitespace(expected_html)) + end + + it 'passes arguments to `check_box` method' do + allow(fake_template).to receive(:check_box).and_return('') + + checkbox_html + + expect(fake_template).to have_received(:check_box).with(:user, :view_diffs_file_by_file, { class: %w(custom-control-input checkbox-foo-bar), object: user }, '3', '1') + end + + it 'passes arguments to `label` method' do + allow(fake_template).to receive(:label).and_return('') + + checkbox_html + + expect(fake_template).to have_received(:label).with(:user, :view_diffs_file_by_file, { class: %w(custom-control-label label-foo-bar), object: user }) + end + end + end + + private + + def html_strip_whitespace(html) + html.lines.map(&:strip).join('') + end +end diff --git a/spec/lib/gitlab/git/commit_spec.rb b/spec/lib/gitlab/git/commit_spec.rb index 49f1e6e994f..f58bab52cfa 100644 --- a/spec/lib/gitlab/git/commit_spec.rb +++ b/spec/lib/gitlab/git/commit_spec.rb @@ -369,11 +369,15 @@ RSpec.describe Gitlab::Git::Commit, :seed_helper do commits.map { |c| c.id } end - it 'has 1 element' do - expect(subject.size).to eq(1) + it { is_expected.to contain_exactly(SeedRepo::Commit::ID) } + + context 'between_uses_list_commits FF disabled' do + before do + stub_feature_flags(between_uses_list_commits: false) + end + + it { is_expected.to contain_exactly(SeedRepo::Commit::ID) } end - it { is_expected.to include(SeedRepo::Commit::ID) } - it { is_expected.not_to include(SeedRepo::FirstCommit::ID) } end describe '.shas_with_signatures' do diff --git a/spec/lib/gitlab/git/commit_stats_spec.rb b/spec/lib/gitlab/git/commit_stats_spec.rb new file mode 100644 index 00000000000..29d3909efec --- /dev/null +++ b/spec/lib/gitlab/git/commit_stats_spec.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require "spec_helper" + +RSpec.describe Gitlab::Git::CommitStats, :seed_helper do + let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '', 'group/project') } + let(:commit) { Gitlab::Git::Commit.find(repository, SeedRepo::Commit::ID) } + + def verify_stats! + stats = described_class.new(repository, commit) + + expect(stats).to have_attributes( + additions: eq(11), + deletions: eq(6), + total: eq(17) + ) + end + + it 'returns commit stats and caches them', :use_clean_rails_redis_caching do + expect(repository.gitaly_commit_client).to receive(:commit_stats).with(commit.id).and_call_original + + verify_stats! + + expect(Rails.cache.fetch("commit_stats:group/project:#{commit.id}")).to eq([11, 6]) + + expect(repository.gitaly_commit_client).not_to receive(:commit_stats) + + verify_stats! + end +end diff --git a/spec/lib/gitlab/git/conflict/file_spec.rb b/spec/lib/gitlab/git/conflict/file_spec.rb index 454a48a1d3a..6eb7a7e394e 100644 --- a/spec/lib/gitlab/git/conflict/file_spec.rb +++ b/spec/lib/gitlab/git/conflict/file_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::Git::Conflict::File do - let(:conflict) { { theirs: { path: 'foo', mode: 33188 }, ours: { path: 'foo', mode: 33188 } } } + let(:conflict) { { ancestor: { path: 'ancestor' }, theirs: { path: 'foo', mode: 33188 }, ours: { path: 'foo', mode: 33188 } } } let(:invalid_content) { described_class.new(nil, nil, conflict, (+"a\xC4\xFC").force_encoding(Encoding::ASCII_8BIT)) } let(:valid_content) { described_class.new(nil, nil, conflict, (+"Espa\xC3\xB1a").force_encoding(Encoding::ASCII_8BIT)) } @@ -48,4 +48,18 @@ RSpec.describe Gitlab::Git::Conflict::File do end end end + + describe '#path' do + it 'returns our_path' do + expect(valid_content.path).to eq(conflict[:ours][:path]) + end + + context 'when our_path is not present' do + let(:conflict) { { ancestor: { path: 'ancestor' }, theirs: { path: 'theirs', mode: 33188 }, ours: { path: '', mode: 0 } } } + + it 'returns their_path' do + expect(valid_content.path).to eq(conflict[:theirs][:path]) + end + end + end end diff --git a/spec/lib/gitlab/git/remote_mirror_spec.rb b/spec/lib/gitlab/git/remote_mirror_spec.rb index 0954879f6bd..4b827e5d2d0 100644 --- a/spec/lib/gitlab/git/remote_mirror_spec.rb +++ b/spec/lib/gitlab/git/remote_mirror_spec.rb @@ -6,30 +6,17 @@ RSpec.describe Gitlab::Git::RemoteMirror do describe '#update' do let(:project) { create(:project, :repository) } let(:repository) { project.repository } - let(:ref_name) { 'foo' } let(:url) { 'https://example.com' } let(:options) { { only_branches_matching: ['master'], ssh_key: 'KEY', known_hosts: 'KNOWN HOSTS', keep_divergent_refs: true } } - subject(:remote_mirror) { described_class.new(repository, ref_name, url, **options) } + subject(:remote_mirror) { described_class.new(repository, url, **options) } - shared_examples 'an update' do - it 'delegates to the Gitaly client' do - expect(repository.gitaly_remote_client) - .to receive(:update_remote_mirror) - .with(ref_name, url, ['master'], ssh_key: 'KEY', known_hosts: 'KNOWN HOSTS', keep_divergent_refs: true) - - remote_mirror.update # rubocop:disable Rails/SaveBang - end - end - - context 'with url' do - it_behaves_like 'an update' - end - - context 'without url' do - let(:url) { nil } + it 'delegates to the Gitaly client' do + expect(repository.gitaly_remote_client) + .to receive(:update_remote_mirror) + .with(url, ['master'], ssh_key: 'KEY', known_hosts: 'KNOWN HOSTS', keep_divergent_refs: true) - it_behaves_like 'an update' + remote_mirror.update # rubocop:disable Rails/SaveBang end it 'wraps gitaly errors' do diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index 706bcdea291..29e7a1dce1d 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -491,6 +491,8 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do end describe '#fetch_remote' do + let(:url) { 'http://example.clom' } + it 'delegates to the gitaly RepositoryService' do ssh_auth = double(:ssh_auth) expected_opts = { @@ -500,17 +502,17 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do timeout: described_class::GITLAB_PROJECTS_TIMEOUT, prune: false, check_tags_changed: false, - url: nil, - refmap: nil + refmap: nil, + http_authorization_header: "" } - expect(repository.gitaly_repository_client).to receive(:fetch_remote).with('remote-name', expected_opts) + expect(repository.gitaly_repository_client).to receive(:fetch_remote).with(url, expected_opts) - repository.fetch_remote('remote-name', ssh_auth: ssh_auth, forced: true, no_tags: true, prune: false, check_tags_changed: false) + repository.fetch_remote(url, ssh_auth: ssh_auth, forced: true, no_tags: true, prune: false, check_tags_changed: false) end it_behaves_like 'wrapping gRPC errors', Gitlab::GitalyClient::RepositoryService, :fetch_remote do - subject { repository.fetch_remote('remote-name') } + subject { repository.fetch_remote(url) } end end @@ -584,29 +586,29 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do expect_any_instance_of(Gitlab::GitalyClient::RemoteService) .to receive(:find_remote_root_ref).and_call_original - expect(repository.find_remote_root_ref('origin', SeedHelper::GITLAB_GIT_TEST_REPO_URL)).to eq 'master' + expect(repository.find_remote_root_ref(SeedHelper::GITLAB_GIT_TEST_REPO_URL)).to eq 'master' end it 'returns UTF-8' do - expect(repository.find_remote_root_ref('origin', SeedHelper::GITLAB_GIT_TEST_REPO_URL)).to be_utf8 + expect(repository.find_remote_root_ref(SeedHelper::GITLAB_GIT_TEST_REPO_URL)).to be_utf8 end it 'returns nil when remote name is nil' do expect_any_instance_of(Gitlab::GitalyClient::RemoteService) .not_to receive(:find_remote_root_ref) - expect(repository.find_remote_root_ref(nil, nil)).to be_nil + expect(repository.find_remote_root_ref(nil)).to be_nil end it 'returns nil when remote name is empty' do expect_any_instance_of(Gitlab::GitalyClient::RemoteService) .not_to receive(:find_remote_root_ref) - expect(repository.find_remote_root_ref('', '')).to be_nil + expect(repository.find_remote_root_ref('')).to be_nil end it_behaves_like 'wrapping gRPC errors', Gitlab::GitalyClient::RemoteService, :find_remote_root_ref do - subject { repository.find_remote_root_ref('origin', SeedHelper::GITLAB_GIT_TEST_REPO_URL) } + subject { repository.find_remote_root_ref(SeedHelper::GITLAB_GIT_TEST_REPO_URL) } end end @@ -950,44 +952,23 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do let(:expected_commits) { 1 } let(:revisions) { [new_commit] } - shared_examples 'an enumeration of new commits' do - it 'enumerates commits' do - commits = repository.new_commits(revisions).to_a - - expect(commits.size).to eq(expected_commits) - commits.each do |commit| - expect(commit.id).to eq(new_commit) - expect(commit.message).to eq("Message") - end + before do + expect_next_instance_of(Gitlab::GitalyClient::CommitService) do |service| + expect(service) + .to receive(:list_commits) + .with([new_commit, '--not', '--all']) + .and_call_original end end - context 'with list_commits disabled' do - before do - stub_feature_flags(list_commits: false) + it 'enumerates commits' do + commits = repository.new_commits(revisions).to_a - expect_next_instance_of(Gitlab::GitalyClient::RefService) do |service| - expect(service) - .to receive(:list_new_commits) - .with(new_commit) - .and_call_original - end + expect(commits.size).to eq(expected_commits) + commits.each do |commit| + expect(commit.id).to eq(new_commit) + expect(commit.message).to eq("Message") end - - it_behaves_like 'an enumeration of new commits' - end - - context 'with list_commits enabled' do - before do - expect_next_instance_of(Gitlab::GitalyClient::CommitService) do |service| - expect(service) - .to receive(:list_commits) - .with([new_commit, '--not', '--all']) - .and_call_original - end - end - - it_behaves_like 'an enumeration of new commits' end end @@ -1750,43 +1731,61 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do end end - describe '#write_config' do - before do - repository_rugged.config["gitlab.fullpath"] = repository_path - end + describe '#set_full_path' do + shared_examples '#set_full_path' do + before do + repository_rugged.config["gitlab.fullpath"] = repository_path + end - context 'is given a path' do - it 'writes it to disk' do - repository.write_config(full_path: "not-the/real-path.git") + context 'is given a path' do + it 'writes it to disk' do + repository.set_full_path(full_path: "not-the/real-path.git") - config = File.read(File.join(repository_path, "config")) + config = File.read(File.join(repository_path, "config")) - expect(config).to include("[gitlab]") - expect(config).to include("fullpath = not-the/real-path.git") + expect(config).to include("[gitlab]") + expect(config).to include("fullpath = not-the/real-path.git") + end end - end - context 'it is given an empty path' do - it 'does not write it to disk' do - repository.write_config(full_path: "") + context 'it is given an empty path' do + it 'does not write it to disk' do + repository.set_full_path(full_path: "") - config = File.read(File.join(repository_path, "config")) + config = File.read(File.join(repository_path, "config")) - expect(config).to include("[gitlab]") - expect(config).to include("fullpath = #{repository_path}") + expect(config).to include("[gitlab]") + expect(config).to include("fullpath = #{repository_path}") + end + end + + context 'repository does not exist' do + it 'raises NoRepository and does not call Gitaly WriteConfig' do + repository = Gitlab::Git::Repository.new('default', 'does/not/exist.git', '', 'group/project') + + expect(repository.gitaly_repository_client).not_to receive(:set_full_path) + + expect do + repository.set_full_path(full_path: 'foo/bar.git') + end.to raise_error(Gitlab::Git::Repository::NoRepository) + end end end - context 'repository does not exist' do - it 'raises NoRepository and does not call Gitaly WriteConfig' do - repository = Gitlab::Git::Repository.new('default', 'does/not/exist.git', '', 'group/project') + context 'with :set_full_path enabled' do + before do + stub_feature_flags(set_full_path: true) + end - expect(repository.gitaly_repository_client).not_to receive(:write_config) + it_behaves_like '#set_full_path' + end - expect do - repository.write_config(full_path: 'foo/bar.git') - end.to raise_error(Gitlab::Git::Repository::NoRepository) + context 'with :set_full_path disabled' do + before do + stub_feature_flags(set_full_path: false) end + + it_behaves_like '#set_full_path' end end @@ -1813,34 +1812,6 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do end end - describe '#delete_config' do - let(:repository) { mutable_repository } - let(:entries) do - { - 'test.foo1' => 'bla bla', - 'test.foo2' => 1234, - 'test.foo3' => true - } - end - - it 'can delete config settings' do - entries.each do |key, value| - repository_rugged.config[key] = value - end - - expect(repository.delete_config(*%w[does.not.exist test.foo1 test.foo2])).to be_nil - - # Workaround for https://github.com/libgit2/rugged/issues/785: If - # Gitaly changes .gitconfig while Rugged has the file loaded - # Rugged::Repository#each_key will report stale values unless a - # lookup is done first. - expect(repository_rugged.config['test.foo1']).to be_nil - config_keys = repository_rugged.config.each_key.to_a - expect(config_keys).not_to include('test.foo1') - expect(config_keys).not_to include('test.foo2') - end - end - describe '#merge_to_ref' do let(:repository) { mutable_repository } let(:branch_head) { '6d394385cf567f80a8fd85055db1ab4c5295806f' } @@ -2001,47 +1972,6 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do end end - describe 'remotes' do - let(:repository) { mutable_repository } - let(:remote_name) { 'my-remote' } - let(:url) { 'http://my-repo.git' } - - after do - ensure_seeds - end - - describe '#add_remote' do - let(:mirror_refmap) { '+refs/*:refs/*' } - - it 'added the remote' do - begin - repository_rugged.remotes.delete(remote_name) - rescue Rugged::ConfigError - end - - repository.add_remote(remote_name, url, mirror_refmap: mirror_refmap) - - expect(repository_rugged.remotes[remote_name]).not_to be_nil - expect(repository_rugged.config["remote.#{remote_name}.mirror"]).to eq('true') - expect(repository_rugged.config["remote.#{remote_name}.prune"]).to eq('true') - expect(repository_rugged.config["remote.#{remote_name}.fetch"]).to eq(mirror_refmap) - end - end - - describe '#remove_remote' do - it 'removes the remote' do - repository_rugged.remotes.create(remote_name, url) - - expect(repository.remove_remote(remote_name)).to be true - - # Since we deleted the remote via Gitaly, Rugged doesn't know - # this changed underneath it. Let's refresh the Rugged repo. - repository_rugged = Rugged::Repository.new(repository_path) - expect(repository_rugged.remotes[remote_name]).to be_nil - end - end - end - describe '#bundle_to_disk' do let(:save_path) { File.join(Dir.tmpdir, "repo-#{SecureRandom.hex}.bundle") } diff --git a/spec/lib/gitlab/git/tag_spec.rb b/spec/lib/gitlab/git/tag_spec.rb index b6ff76c5e1c..79ae47f8a7b 100644 --- a/spec/lib/gitlab/git/tag_spec.rb +++ b/spec/lib/gitlab/git/tag_spec.rb @@ -38,7 +38,7 @@ RSpec.describe Gitlab::Git::Tag, :seed_helper do it { expect(tag.tagger.timezone).to eq("+0200") } end - describe 'signed tag' do + shared_examples 'signed tag' do let(:project) { create(:project, :repository) } let(:tag) { project.repository.find_tag('v1.1.1') } @@ -54,6 +54,18 @@ RSpec.describe Gitlab::Git::Tag, :seed_helper do it { expect(tag.tagger.timezone).to eq("+0100") } end + context 'with :get_tag_signatures enabled' do + it_behaves_like 'signed tag' + end + + context 'with :get_tag_signatures disabled' do + before do + stub_feature_flags(get_tag_signatures: false) + end + + it_behaves_like 'signed tag' + end + it { expect(repository.tags.size).to eq(SeedRepo::Repo::TAGS.size) } end @@ -77,6 +89,75 @@ RSpec.describe Gitlab::Git::Tag, :seed_helper do end end + describe '.extract_signature_lazily' do + let(:project) { create(:project, :repository) } + + subject { described_class.extract_signature_lazily(project.repository, tag_id).itself } + + context 'when the tag is signed' do + let(:tag_id) { project.repository.find_tag('v1.1.1').id } + + it 'returns signature and signed text' do + signature, signed_text = subject + + expect(signature).to eq(X509Helpers::User1.signed_tag_signature.chomp) + expect(signature).to be_a_binary_string + expect(signed_text).to eq(X509Helpers::User1.signed_tag_base_data) + expect(signed_text).to be_a_binary_string + end + end + + context 'when the tag has no signature' do + let(:tag_id) { project.repository.find_tag('v1.0.0').id } + + it 'returns empty signature and message as signed text' do + signature, signed_text = subject + + expect(signature).to be_empty + expect(signed_text).to eq(X509Helpers::User1.unsigned_tag_base_data) + expect(signed_text).to be_a_binary_string + end + end + + context 'when the tag cannot be found' do + let(:tag_id) { Gitlab::Git::BLANK_SHA } + + it 'raises GRPC::Internal' do + expect { subject }.to raise_error(GRPC::Internal) + end + end + + context 'when the tag ID is invalid' do + let(:tag_id) { '4b4918a572fa86f9771e5ba40fbd48e' } + + it 'raises GRPC::Internal' do + expect { subject }.to raise_error(GRPC::Internal) + end + end + + context 'when loading signatures in batch once' do + it 'fetches signatures in batch once' do + tag_ids = [project.repository.find_tag('v1.1.1').id, project.repository.find_tag('v1.0.0').id] + signatures = tag_ids.map do |tag_id| + described_class.extract_signature_lazily(repository, tag_id) + end + + other_repository = double(:repository) + described_class.extract_signature_lazily(other_repository, tag_ids.first) + + expect(described_class).to receive(:batch_signature_extraction) + .with(repository, tag_ids) + .once + .and_return({}) + + expect(described_class).not_to receive(:batch_signature_extraction) + .with(other_repository, tag_ids.first) + + 2.times { signatures.each(&:itself) } + end + end + end + describe 'tag into from Gitaly tag' do context 'message_size != message.size' do let(:gitaly_tag) { build(:gitaly_tag, message: ''.b, message_size: message_size) } diff --git a/spec/lib/gitlab/git/tree_spec.rb b/spec/lib/gitlab/git/tree_spec.rb index c44d7e44751..f11d84bd8d3 100644 --- a/spec/lib/gitlab/git/tree_spec.rb +++ b/spec/lib/gitlab/git/tree_spec.rb @@ -6,29 +6,44 @@ RSpec.describe Gitlab::Git::Tree, :seed_helper do let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '', 'group/project') } shared_examples :repo do - let(:tree) { Gitlab::Git::Tree.where(repository, SeedRepo::Commit::ID) } + subject(:tree) { Gitlab::Git::Tree.where(repository, sha, path, recursive, pagination_params) } - it { expect(tree).to be_kind_of Array } - it { expect(tree.empty?).to be_falsey } - it { expect(tree.count(&:dir?)).to eq(2) } - it { expect(tree.count(&:file?)).to eq(10) } - it { expect(tree.count(&:submodule?)).to eq(2) } + let(:sha) { SeedRepo::Commit::ID } + let(:path) { nil } + let(:recursive) { false } + let(:pagination_params) { nil } - it 'returns an empty array when called with an invalid ref' do - expect(described_class.where(repository, 'foobar-does-not-exist')).to eq([]) + let(:entries) { tree.first } + let(:cursor) { tree.second } + + it { expect(entries).to be_kind_of Array } + it { expect(entries.empty?).to be_falsey } + it { expect(entries.count(&:dir?)).to eq(2) } + it { expect(entries.count(&:file?)).to eq(10) } + it { expect(entries.count(&:submodule?)).to eq(2) } + it { expect(cursor&.next_cursor).to be_blank } + + context 'with an invalid ref' do + let(:sha) { 'foobar-does-not-exist' } + + it { expect(entries).to eq([]) } + it { expect(cursor).to be_nil } end - it 'returns a list of tree objects' do - entries = described_class.where(repository, SeedRepo::Commit::ID, 'files', true) + context 'when path is provided' do + let(:path) { 'files' } + let(:recursive) { true } - expect(entries.map(&:path)).to include('files/html', - 'files/markdown/ruby-style-guide.md') - expect(entries.count).to be >= 10 - expect(entries).to all(be_a(Gitlab::Git::Tree)) + it 'returns a list of tree objects' do + expect(entries.map(&:path)).to include('files/html', + 'files/markdown/ruby-style-guide.md') + expect(entries.count).to be >= 10 + expect(entries).to all(be_a(Gitlab::Git::Tree)) + end end describe '#dir?' do - let(:dir) { tree.select(&:dir?).first } + let(:dir) { entries.select(&:dir?).first } it { expect(dir).to be_kind_of Gitlab::Git::Tree } it { expect(dir.id).to eq('3c122d2b7830eca25235131070602575cf8b41a1') } @@ -41,7 +56,8 @@ RSpec.describe Gitlab::Git::Tree, :seed_helper do context :subdir do # rubocop: disable Rails/FindBy # This is not ActiveRecord where..first - let(:subdir) { Gitlab::Git::Tree.where(repository, SeedRepo::Commit::ID, 'files').first } + let(:path) { 'files' } + let(:subdir) { entries.first } # rubocop: enable Rails/FindBy it { expect(subdir).to be_kind_of Gitlab::Git::Tree } @@ -55,7 +71,8 @@ RSpec.describe Gitlab::Git::Tree, :seed_helper do context :subdir_file do # rubocop: disable Rails/FindBy # This is not ActiveRecord where..first - let(:subdir_file) { Gitlab::Git::Tree.where(repository, SeedRepo::Commit::ID, 'files/ruby').first } + let(:path) { 'files/ruby' } + let(:subdir_file) { entries.first } # rubocop: enable Rails/FindBy it { expect(subdir_file).to be_kind_of Gitlab::Git::Tree } @@ -68,10 +85,11 @@ RSpec.describe Gitlab::Git::Tree, :seed_helper do context :flat_path do let(:filename) { 'files/flat/path/correct/content.txt' } - let(:oid) { create_file(filename) } + let(:sha) { create_file(filename) } + let(:path) { 'files/flat' } # rubocop: disable Rails/FindBy # This is not ActiveRecord where..first - let(:subdir_file) { Gitlab::Git::Tree.where(repository, oid, 'files/flat').first } + let(:subdir_file) { entries.first } # rubocop: enable Rails/FindBy let(:repository_rugged) { Rugged::Repository.new(File.join(SEED_STORAGE_PATH, TEST_REPO_PATH)) } @@ -116,7 +134,7 @@ RSpec.describe Gitlab::Git::Tree, :seed_helper do end describe '#file?' do - let(:file) { tree.select(&:file?).first } + let(:file) { entries.select(&:file?).first } it { expect(file).to be_kind_of Gitlab::Git::Tree } it { expect(file.id).to eq('dfaa3f97ca337e20154a98ac9d0be76ddd1fcc82') } @@ -125,21 +143,21 @@ RSpec.describe Gitlab::Git::Tree, :seed_helper do end describe '#readme?' do - let(:file) { tree.select(&:readme?).first } + let(:file) { entries.select(&:readme?).first } it { expect(file).to be_kind_of Gitlab::Git::Tree } it { expect(file.name).to eq('README.md') } end describe '#contributing?' do - let(:file) { tree.select(&:contributing?).first } + let(:file) { entries.select(&:contributing?).first } it { expect(file).to be_kind_of Gitlab::Git::Tree } it { expect(file.name).to eq('CONTRIBUTING.md') } end describe '#submodule?' do - let(:submodule) { tree.select(&:submodule?).first } + let(:submodule) { entries.select(&:submodule?).first } it { expect(submodule).to be_kind_of Gitlab::Git::Tree } it { expect(submodule.id).to eq('79bceae69cb5750d6567b223597999bfa91cb3b9') } @@ -149,7 +167,16 @@ RSpec.describe Gitlab::Git::Tree, :seed_helper do end describe '.where with Gitaly enabled' do - it_behaves_like :repo + it_behaves_like :repo do + context 'with pagination parameters' do + let(:pagination_params) { { limit: 3, page_token: nil } } + + it 'returns paginated list of tree objects' do + expect(entries.count).to eq(3) + expect(cursor.next_cursor).to be_present + end + end + end end describe '.where with Rugged enabled', :enable_rugged do @@ -161,6 +188,15 @@ RSpec.describe Gitlab::Git::Tree, :seed_helper do described_class.where(repository, SeedRepo::Commit::ID, 'files', false) end - it_behaves_like :repo + it_behaves_like :repo do + context 'with pagination parameters' do + let(:pagination_params) { { limit: 3, page_token: nil } } + + it 'does not support pagination' do + expect(entries.count).to be >= 10 + expect(cursor).to be_nil + end + end + end end end diff --git a/spec/lib/gitlab/git_access_snippet_spec.rb b/spec/lib/gitlab/git_access_snippet_spec.rb index 3b85e3ddd1d..d690a4b2db4 100644 --- a/spec/lib/gitlab/git_access_snippet_spec.rb +++ b/spec/lib/gitlab/git_access_snippet_spec.rb @@ -382,12 +382,11 @@ RSpec.describe Gitlab::GitAccessSnippet do it_behaves_like 'a push to repository to make it over the limit' end - shared_examples_for 'a change with GIT_OBJECT_DIRECTORY_RELATIVE env var unset' do + context 'a change with GIT_OBJECT_DIRECTORY_RELATIVE env var unset' do let(:change_size) { 200 } before do - stub_feature_flags(git_access_batched_changes_size: batched) - allow(snippet.repository).to receive(expected_call).and_return( + allow(snippet.repository).to receive(:blobs).and_return( [double(:blob, size: change_size)] ) end @@ -396,20 +395,6 @@ RSpec.describe Gitlab::GitAccessSnippet do it_behaves_like 'a push to repository below the limit' it_behaves_like 'a push to repository to make it over the limit' end - - context 'when batched computation is enabled' do - let(:batched) { true } - let(:expected_call) { :blobs } - - it_behaves_like 'a change with GIT_OBJECT_DIRECTORY_RELATIVE env var unset' - end - - context 'when batched computation is disabled' do - let(:batched) { false } - let(:expected_call) { :new_blobs } - - it_behaves_like 'a change with GIT_OBJECT_DIRECTORY_RELATIVE env var unset' - end end describe 'HEAD realignment' do diff --git a/spec/lib/gitlab/gitaly_client/commit_service_spec.rb b/spec/lib/gitlab/gitaly_client/commit_service_spec.rb index 22c29403255..a0e2d43cf45 100644 --- a/spec/lib/gitlab/gitaly_client/commit_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/commit_service_spec.rb @@ -169,7 +169,11 @@ RSpec.describe Gitlab::GitalyClient::CommitService do end describe '#tree_entries' do + subject { client.tree_entries(repository, revision, path, recursive, pagination_params) } + let(:path) { '/' } + let(:recursive) { false } + let(:pagination_params) { nil } it 'sends a get_tree_entries message' do expect_any_instance_of(Gitaly::CommitService::Stub) @@ -177,7 +181,7 @@ RSpec.describe Gitlab::GitalyClient::CommitService do .with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash)) .and_return([]) - client.tree_entries(repository, revision, path, false) + is_expected.to eq([[], nil]) end context 'with UTF-8 params strings' do @@ -190,7 +194,26 @@ RSpec.describe Gitlab::GitalyClient::CommitService do .with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash)) .and_return([]) - client.tree_entries(repository, revision, path, false) + is_expected.to eq([[], nil]) + end + end + + context 'with pagination parameters' do + let(:pagination_params) { { limit: 3, page_token: nil } } + + it 'responds with a pagination cursor' do + pagination_cursor = Gitaly::PaginationCursor.new(next_cursor: 'aabbccdd') + response = Gitaly::GetTreeEntriesResponse.new( + entries: [], + pagination_cursor: pagination_cursor + ) + + expect_any_instance_of(Gitaly::CommitService::Stub) + .to receive(:get_tree_entries) + .with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash)) + .and_return([response]) + + is_expected.to eq([[], pagination_cursor]) end end end @@ -320,6 +343,92 @@ RSpec.describe Gitlab::GitalyClient::CommitService do end end + describe '#list_new_commits' do + let(:revisions) { [revision] } + let(:gitaly_commits) { create_list(:gitaly_commit, 3) } + let(:commits) { gitaly_commits.map { |c| Gitlab::Git::Commit.new(repository, c) }} + + subject { client.list_new_commits(revisions, allow_quarantine: allow_quarantine) } + + shared_examples 'a #list_all_commits message' do + it 'sends a list_all_commits message' do + expected_repository = repository.gitaly_repository.dup + expected_repository.git_alternate_object_directories = Google::Protobuf::RepeatedField.new(:string) + + expect_next_instance_of(Gitaly::CommitService::Stub) do |service| + expect(service).to receive(:list_all_commits) + .with(gitaly_request_with_params(repository: expected_repository), kind_of(Hash)) + .and_return([Gitaly::ListAllCommitsResponse.new(commits: gitaly_commits)]) + end + + expect(subject).to eq(commits) + end + end + + shared_examples 'a #list_commits message' do + it 'sends a list_commits message' do + expect_next_instance_of(Gitaly::CommitService::Stub) do |service| + expect(service).to receive(:list_commits) + .with(gitaly_request_with_params(revisions: revisions + %w[--not --all]), kind_of(Hash)) + .and_return([Gitaly::ListCommitsResponse.new(commits: gitaly_commits)]) + end + + expect(subject).to eq(commits) + end + end + + before do + ::Gitlab::GitalyClient.clear_stubs! + + allow(Gitlab::Git::HookEnv) + .to receive(:all) + .with(repository.gl_repository) + .and_return(git_env) + end + + context 'with hook environment' do + let(:git_env) do + { + 'GIT_OBJECT_DIRECTORY_RELATIVE' => '.git/objects', + 'GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE' => ['/dir/one', '/dir/two'] + } + end + + context 'with allowed quarantine' do + let(:allow_quarantine) { true } + + it_behaves_like 'a #list_all_commits message' + end + + context 'with disallowed quarantine' do + let(:allow_quarantine) { false } + + it_behaves_like 'a #list_commits message' + end + end + + context 'without hook environment' do + let(:git_env) do + { + 'GIT_OBJECT_DIRECTORY_RELATIVE' => '', + 'GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE' => [] + } + end + + context 'with allowed quarantine' do + let(:allow_quarantine) { true } + + it_behaves_like 'a #list_commits message' + end + + context 'with disallowed quarantine' do + let(:allow_quarantine) { false } + + it_behaves_like 'a #list_commits message' + end + end + end + describe '#commit_stats' do let(:request) do Gitaly::CommitStatsRequest.new( diff --git a/spec/lib/gitlab/gitaly_client/conflict_files_stitcher_spec.rb b/spec/lib/gitlab/gitaly_client/conflict_files_stitcher_spec.rb index 0bb8628af6c..0eecdfcb630 100644 --- a/spec/lib/gitlab/gitaly_client/conflict_files_stitcher_spec.rb +++ b/spec/lib/gitlab/gitaly_client/conflict_files_stitcher_spec.rb @@ -9,22 +9,37 @@ RSpec.describe Gitlab::GitalyClient::ConflictFilesStitcher do target_repository = target_project.repository.raw target_gitaly_repository = target_repository.gitaly_repository + ancestor_path_1 = 'ancestor/path/1' our_path_1 = 'our/path/1' their_path_1 = 'their/path/1' our_mode_1 = 0744 commit_oid_1 = 'f00' content_1 = 'content of the first file' + ancestor_path_2 = 'ancestor/path/2' our_path_2 = 'our/path/2' their_path_2 = 'their/path/2' our_mode_2 = 0600 commit_oid_2 = 'ba7' content_2 = 'content of the second file' - header_1 = double(repository: target_gitaly_repository, commit_oid: commit_oid_1, - our_path: our_path_1, their_path: their_path_1, our_mode: our_mode_1) - header_2 = double(repository: target_gitaly_repository, commit_oid: commit_oid_2, - our_path: our_path_2, their_path: their_path_2, our_mode: our_mode_2) + header_1 = double( + repository: target_gitaly_repository, + commit_oid: commit_oid_1, + ancestor_path: ancestor_path_1, + our_path: our_path_1, + their_path: their_path_1, + our_mode: our_mode_1 + ) + + header_2 = double( + repository: target_gitaly_repository, + commit_oid: commit_oid_2, + ancestor_path: ancestor_path_2, + our_path: our_path_2, + their_path: their_path_2, + our_mode: our_mode_2 + ) messages = [ double(files: [double(header: header_1), double(header: nil, content: content_1[0..5])]), @@ -39,6 +54,7 @@ RSpec.describe Gitlab::GitalyClient::ConflictFilesStitcher do expect(conflict_files.size).to be(2) expect(conflict_files[0].content).to eq(content_1) + expect(conflict_files[0].ancestor_path).to eq(ancestor_path_1) expect(conflict_files[0].their_path).to eq(their_path_1) expect(conflict_files[0].our_path).to eq(our_path_1) expect(conflict_files[0].our_mode).to be(our_mode_1) @@ -46,6 +62,7 @@ RSpec.describe Gitlab::GitalyClient::ConflictFilesStitcher do expect(conflict_files[0].commit_oid).to eq(commit_oid_1) expect(conflict_files[1].content).to eq(content_2) + expect(conflict_files[1].ancestor_path).to eq(ancestor_path_2) expect(conflict_files[1].their_path).to eq(their_path_2) expect(conflict_files[1].our_path).to eq(our_path_2) expect(conflict_files[1].our_mode).to be(our_mode_2) diff --git a/spec/lib/gitlab/gitaly_client/conflicts_service_spec.rb b/spec/lib/gitlab/gitaly_client/conflicts_service_spec.rb index e90cb966917..89a41ae71f3 100644 --- a/spec/lib/gitlab/gitaly_client/conflicts_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/conflicts_service_spec.rb @@ -15,18 +15,31 @@ RSpec.describe Gitlab::GitalyClient::ConflictsService do end describe '#list_conflict_files' do + let(:allow_tree_conflicts) { false } let(:request) do Gitaly::ListConflictFilesRequest.new( - repository: target_gitaly_repository, our_commit_oid: our_commit_oid, - their_commit_oid: their_commit_oid + repository: target_gitaly_repository, + our_commit_oid: our_commit_oid, + their_commit_oid: their_commit_oid, + allow_tree_conflicts: allow_tree_conflicts ) end - it 'sends an RPC request' do - expect_any_instance_of(Gitaly::ConflictsService::Stub).to receive(:list_conflict_files) - .with(request, kind_of(Hash)).and_return([].to_enum) + shared_examples_for 'listing conflicts' do + it 'sends an RPC request' do + expect_any_instance_of(Gitaly::ConflictsService::Stub).to receive(:list_conflict_files) + .with(request, kind_of(Hash)).and_return([].to_enum) + + client.list_conflict_files(allow_tree_conflicts: allow_tree_conflicts) + end + end + + it_behaves_like 'listing conflicts' + + context 'when allow_tree_conflicts is set to true' do + let(:allow_tree_conflicts) { true } - client.list_conflict_files + it_behaves_like 'listing conflicts' end end diff --git a/spec/lib/gitlab/gitaly_client/ref_service_spec.rb b/spec/lib/gitlab/gitaly_client/ref_service_spec.rb index a4c6e30bba8..e19be965e68 100644 --- a/spec/lib/gitlab/gitaly_client/ref_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/ref_service_spec.rb @@ -178,6 +178,17 @@ RSpec.describe Gitlab::GitalyClient::RefService do end end + describe '#get_tag_signatures' do + it 'sends a get_tag_signatures message' do + expect_any_instance_of(Gitaly::RefService::Stub) + .to receive(:get_tag_signatures) + .with(gitaly_request_with_params(tag_revisions: ['some_tag_id']), kind_of(Hash)) + .and_return([]) + + client.get_tag_signatures(['some_tag_id']) + end + end + describe '#find_ref_name', :seed_helper do subject { client.find_ref_name(SeedRepo::Commit::ID, 'refs/heads/master') } diff --git a/spec/lib/gitlab/gitaly_client/remote_service_spec.rb b/spec/lib/gitlab/gitaly_client/remote_service_spec.rb index 2ec5f70be76..3d0f8358406 100644 --- a/spec/lib/gitlab/gitaly_client/remote_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/remote_service_spec.rb @@ -6,36 +6,9 @@ RSpec.describe Gitlab::GitalyClient::RemoteService do let(:project) { create(:project) } let(:storage_name) { project.repository_storage } let(:relative_path) { project.disk_path + '.git' } - let(:remote_name) { 'my-remote' } let(:client) { described_class.new(project.repository) } - describe '#add_remote' do - let(:url) { 'http://my-repo.git' } - let(:mirror_refmap) { :all_refs } - - it 'sends an add_remote message' do - expect_any_instance_of(Gitaly::RemoteService::Stub) - .to receive(:add_remote) - .with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash)) - .and_return(double(:add_remote_response)) - - client.add_remote(remote_name, url, mirror_refmap) - end - end - - describe '#remove_remote' do - it 'sends an remove_remote message and returns the result value' do - expect_any_instance_of(Gitaly::RemoteService::Stub) - .to receive(:remove_remote) - .with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash)) - .and_return(double(result: true)) - - expect(client.remove_remote(remote_name)).to be(true) - end - end - describe '#find_remote_root_ref' do - let(:remote) { 'origin' } let(:url) { 'http://git.example.com/my-repo.git' } let(:auth) { 'Basic secret' } let(:expected_params) { { remote_url: url, http_authorization_header: auth } } @@ -47,7 +20,7 @@ RSpec.describe Gitlab::GitalyClient::RemoteService do .with(gitaly_request_with_params(expected_params), kind_of(Hash)) .and_return(double(ref: 'master')) - expect(client.find_remote_root_ref(remote, url, auth)).to eq 'master' + expect(client.find_remote_root_ref(url, auth)).to eq 'master' end it 'ensure ref is a valid UTF-8 string' do @@ -57,39 +30,24 @@ RSpec.describe Gitlab::GitalyClient::RemoteService do .with(gitaly_request_with_params(expected_params), kind_of(Hash)) .and_return(double(ref: "an_invalid_ref_\xE5")) - expect(client.find_remote_root_ref(remote, url, auth)).to eq "an_invalid_ref_å" + expect(client.find_remote_root_ref(url, auth)).to eq "an_invalid_ref_å" end end describe '#update_remote_mirror' do - let(:ref_name) { 'remote_mirror_1' } let(:only_branches_matching) { %w[my-branch master] } let(:ssh_key) { 'KEY' } let(:known_hosts) { 'KNOWN HOSTS' } + let(:url) { 'http:://git.example.com/my-repo.git' } + let(:expected_params) { { remote: Gitaly::UpdateRemoteMirrorRequest::Remote.new(url: url) } } - shared_examples 'an update' do - it 'sends an update_remote_mirror message' do - expect_any_instance_of(Gitaly::RemoteService::Stub) - .to receive(:update_remote_mirror) - .with(array_including(gitaly_request_with_params(expected_params)), kind_of(Hash)) - .and_return(double(:update_remote_mirror_response)) - - client.update_remote_mirror(ref_name, url, only_branches_matching, ssh_key: ssh_key, known_hosts: known_hosts, keep_divergent_refs: true) - end - end - - context 'with remote name' do - let(:url) { nil } - let(:expected_params) { { ref_name: ref_name } } - - it_behaves_like 'an update' - end - - context 'with remote URL' do - let(:url) { 'http:://git.example.com/my-repo.git' } - let(:expected_params) { { remote: Gitaly::UpdateRemoteMirrorRequest::Remote.new(url: url) } } + it 'sends an update_remote_mirror message' do + expect_any_instance_of(Gitaly::RemoteService::Stub) + .to receive(:update_remote_mirror) + .with(array_including(gitaly_request_with_params(expected_params)), kind_of(Hash)) + .and_return(double(:update_remote_mirror_response)) - it_behaves_like 'an update' + client.update_remote_mirror(url, only_branches_matching, ssh_key: ssh_key, known_hosts: known_hosts, keep_divergent_refs: true) end end diff --git a/spec/lib/gitlab/gitaly_client/repository_service_spec.rb b/spec/lib/gitlab/gitaly_client/repository_service_spec.rb index 53805d67f9f..4b037d3f836 100644 --- a/spec/lib/gitlab/gitaly_client/repository_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/repository_service_spec.rb @@ -122,89 +122,75 @@ RSpec.describe Gitlab::GitalyClient::RepositoryService do end describe '#fetch_remote' do - shared_examples 'a fetch' do - it 'sends a fetch_remote_request message' do - expected_remote_params = Gitaly::Remote.new( - url: url, http_authorization_header: "", mirror_refmaps: []) - - expected_request = gitaly_request_with_params( - remote: remote, - remote_params: url ? expected_remote_params : nil, - ssh_key: '', - known_hosts: '', - force: false, - no_tags: false, - no_prune: false, - check_tags_changed: false - ) - - expect_any_instance_of(Gitaly::RepositoryService::Stub) - .to receive(:fetch_remote) - .with(expected_request, kind_of(Hash)) - .and_return(double(value: true)) - - client.fetch_remote(remote, url: url, refmap: nil, ssh_auth: nil, forced: false, no_tags: false, timeout: 1, check_tags_changed: false) - end + let(:url) { 'https://example.com/git/repo.git' } + + it 'sends a fetch_remote_request message' do + expected_request = gitaly_request_with_params( + remote_params: Gitaly::Remote.new( + url: url, + http_authorization_header: "", + mirror_refmaps: [] + ), + ssh_key: '', + known_hosts: '', + force: false, + no_tags: false, + no_prune: false, + check_tags_changed: false + ) - context 'SSH auth' do - where(:ssh_mirror_url, :ssh_key_auth, :ssh_private_key, :ssh_known_hosts, :expected_params) do - false | false | 'key' | 'known_hosts' | {} - false | true | 'key' | 'known_hosts' | {} - true | false | 'key' | 'known_hosts' | { known_hosts: 'known_hosts' } - true | true | 'key' | 'known_hosts' | { ssh_key: 'key', known_hosts: 'known_hosts' } - true | true | 'key' | nil | { ssh_key: 'key' } - true | true | nil | 'known_hosts' | { known_hosts: 'known_hosts' } - true | true | nil | nil | {} - true | true | '' | '' | {} - end + expect_any_instance_of(Gitaly::RepositoryService::Stub) + .to receive(:fetch_remote) + .with(expected_request, kind_of(Hash)) + .and_return(double(value: true)) - with_them do - let(:ssh_auth) do - double( - :ssh_auth, - ssh_mirror_url?: ssh_mirror_url, - ssh_key_auth?: ssh_key_auth, - ssh_private_key: ssh_private_key, - ssh_known_hosts: ssh_known_hosts - ) - end - - it do - expected_remote_params = Gitaly::Remote.new( - url: url, http_authorization_header: "", mirror_refmaps: []) - - expected_request = gitaly_request_with_params({ - remote: remote, - remote_params: url ? expected_remote_params : nil, - ssh_key: '', - known_hosts: '', - force: false, - no_tags: false, - no_prune: false - }.update(expected_params)) - - expect_any_instance_of(Gitaly::RepositoryService::Stub) - .to receive(:fetch_remote) - .with(expected_request, kind_of(Hash)) - .and_return(double(value: true)) - - client.fetch_remote(remote, url: url, refmap: nil, ssh_auth: ssh_auth, forced: false, no_tags: false, timeout: 1) - end - end - end + client.fetch_remote(url, refmap: nil, ssh_auth: nil, forced: false, no_tags: false, timeout: 1, check_tags_changed: false) end - context 'with remote' do - it_behaves_like 'a fetch' do - let(:remote) { 'remote-name' } - let(:url) { nil } + context 'SSH auth' do + where(:ssh_mirror_url, :ssh_key_auth, :ssh_private_key, :ssh_known_hosts, :expected_params) do + false | false | 'key' | 'known_hosts' | {} + false | true | 'key' | 'known_hosts' | {} + true | false | 'key' | 'known_hosts' | { known_hosts: 'known_hosts' } + true | true | 'key' | 'known_hosts' | { ssh_key: 'key', known_hosts: 'known_hosts' } + true | true | 'key' | nil | { ssh_key: 'key' } + true | true | nil | 'known_hosts' | { known_hosts: 'known_hosts' } + true | true | nil | nil | {} + true | true | '' | '' | {} end - end - context 'with URL' do - it_behaves_like 'a fetch' do - let(:remote) { "" } - let(:url) { 'https://example.com/git/repo.git' } + with_them do + let(:ssh_auth) do + double( + :ssh_auth, + ssh_mirror_url?: ssh_mirror_url, + ssh_key_auth?: ssh_key_auth, + ssh_private_key: ssh_private_key, + ssh_known_hosts: ssh_known_hosts + ) + end + + it do + expected_request = gitaly_request_with_params({ + remote_params: Gitaly::Remote.new( + url: url, + http_authorization_header: "", + mirror_refmaps: [] + ), + ssh_key: '', + known_hosts: '', + force: false, + no_tags: false, + no_prune: false + }.update(expected_params)) + + expect_any_instance_of(Gitaly::RepositoryService::Stub) + .to receive(:fetch_remote) + .with(expected_request, kind_of(Hash)) + .and_return(double(value: true)) + + client.fetch_remote(url, refmap: nil, ssh_auth: ssh_auth, forced: false, no_tags: false, timeout: 1) + end end end end @@ -333,4 +319,17 @@ RSpec.describe Gitlab::GitalyClient::RepositoryService do client.replicate(source_repository) end end + + describe '#set_full_path' do + let(:path) { 'repo/path' } + + it 'sends a set_full_path message' do + expect_any_instance_of(Gitaly::RepositoryService::Stub) + .to receive(:set_full_path) + .with(gitaly_request_with_params(path: path), kind_of(Hash)) + .and_return(double) + + client.set_full_path(path) + end + end end diff --git a/spec/lib/gitlab/github_import/bulk_importing_spec.rb b/spec/lib/gitlab/github_import/bulk_importing_spec.rb index 63dce51c5da..6c94973b5a8 100644 --- a/spec/lib/gitlab/github_import/bulk_importing_spec.rb +++ b/spec/lib/gitlab/github_import/bulk_importing_spec.rb @@ -3,8 +3,20 @@ require 'spec_helper' RSpec.describe Gitlab::GithubImport::BulkImporting do - let(:importer) do - Class.new { include(Gitlab::GithubImport::BulkImporting) }.new + let(:project) { instance_double(Project, id: 1) } + let(:importer) { MyImporter.new(project, double) } + let(:importer_class) do + Class.new do + include Gitlab::GithubImport::BulkImporting + + def object_type + :object_type + end + end + end + + before do + stub_const 'MyImporter', importer_class end describe '#build_database_rows' do @@ -21,6 +33,24 @@ RSpec.describe Gitlab::GithubImport::BulkImporting do .with(object) .and_return(false) + expect(Gitlab::Import::Logger) + .to receive(:info) + .with( + import_type: :github, + project_id: 1, + importer: 'MyImporter', + message: '1 object_types fetched' + ) + + expect(Gitlab::GithubImport::ObjectCounter) + .to receive(:increment) + .with( + project, + :object_type, + :fetched, + value: 1 + ) + enum = [[object, 1]].to_enum expect(importer.build_database_rows(enum)).to eq([{ title: 'Foo' }]) @@ -37,6 +67,24 @@ RSpec.describe Gitlab::GithubImport::BulkImporting do .with(object) .and_return(true) + expect(Gitlab::Import::Logger) + .to receive(:info) + .with( + import_type: :github, + project_id: 1, + importer: 'MyImporter', + message: '0 object_types fetched' + ) + + expect(Gitlab::GithubImport::ObjectCounter) + .to receive(:increment) + .with( + project, + :object_type, + :fetched, + value: 0 + ) + enum = [[object, 1]].to_enum expect(importer.build_database_rows(enum)).to be_empty @@ -48,12 +96,32 @@ RSpec.describe Gitlab::GithubImport::BulkImporting do rows = [{ title: 'Foo' }] * 10 model = double(:model, table_name: 'kittens') - expect(Gitlab::Database) + expect(Gitlab::Import::Logger) + .to receive(:info) + .twice + .with( + import_type: :github, + project_id: 1, + importer: 'MyImporter', + message: '5 object_types imported' + ) + + expect(Gitlab::GithubImport::ObjectCounter) + .to receive(:increment) + .twice + .with( + project, + :object_type, + :imported, + value: 5 + ) + + expect(Gitlab::Database.main) .to receive(:bulk_insert) .ordered .with('kittens', rows.first(5)) - expect(Gitlab::Database) + expect(Gitlab::Database.main) .to receive(:bulk_insert) .ordered .with('kittens', rows.last(5)) diff --git a/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb b/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb index 9eea85526f5..0af840d2c10 100644 --- a/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb @@ -36,7 +36,7 @@ RSpec.describe Gitlab::GithubImport::Importer::DiffNoteImporter do describe '#execute' do context 'when the merge request no longer exists' do it 'does not import anything' do - expect(Gitlab::Database).not_to receive(:bulk_insert) + expect(Gitlab::Database.main).not_to receive(:bulk_insert) importer.execute end @@ -58,7 +58,7 @@ RSpec.describe Gitlab::GithubImport::Importer::DiffNoteImporter do .to receive(:author_id_for) .and_return([user.id, true]) - expect(Gitlab::Database) + expect(Gitlab::Database.main) .to receive(:bulk_insert) .with( LegacyDiffNote.table_name, @@ -89,7 +89,7 @@ RSpec.describe Gitlab::GithubImport::Importer::DiffNoteImporter do .to receive(:author_id_for) .and_return([project.creator_id, false]) - expect(Gitlab::Database) + expect(Gitlab::Database.main) .to receive(:bulk_insert) .with( LegacyDiffNote.table_name, @@ -133,7 +133,7 @@ RSpec.describe Gitlab::GithubImport::Importer::DiffNoteImporter do .to receive(:author_id_for) .and_return([project.creator_id, false]) - expect(Gitlab::Database) + expect(Gitlab::Database.main) .to receive(:bulk_insert) .and_raise(ActiveRecord::InvalidForeignKey, 'invalid foreign key') diff --git a/spec/lib/gitlab/github_import/importer/issue_importer_spec.rb b/spec/lib/gitlab/github_import/importer/issue_importer_spec.rb index fb826c987e1..0926000428c 100644 --- a/spec/lib/gitlab/github_import/importer/issue_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/issue_importer_spec.rb @@ -190,7 +190,7 @@ RSpec.describe Gitlab::GithubImport::Importer::IssueImporter, :clean_gitlab_redi .with(issue.assignees[1]) .and_return(5) - expect(Gitlab::Database) + expect(Gitlab::Database.main) .to receive(:bulk_insert) .with( IssueAssignee.table_name, diff --git a/spec/lib/gitlab/github_import/importer/label_links_importer_spec.rb b/spec/lib/gitlab/github_import/importer/label_links_importer_spec.rb index 6d143f78c66..241a0fef600 100644 --- a/spec/lib/gitlab/github_import/importer/label_links_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/label_links_importer_spec.rb @@ -39,7 +39,7 @@ RSpec.describe Gitlab::GithubImport::Importer::LabelLinksImporter do .and_return(1) freeze_time do - expect(Gitlab::Database) + expect(Gitlab::Database.main) .to receive(:bulk_insert) .with( LabelLink.table_name, @@ -64,7 +64,7 @@ RSpec.describe Gitlab::GithubImport::Importer::LabelLinksImporter do .with('bug') .and_return(nil) - expect(Gitlab::Database) + expect(Gitlab::Database.main) .to receive(:bulk_insert) .with(LabelLink.table_name, []) diff --git a/spec/lib/gitlab/github_import/importer/lfs_objects_importer_spec.rb b/spec/lib/gitlab/github_import/importer/lfs_objects_importer_spec.rb index 8ee534734f0..a2c7d51214a 100644 --- a/spec/lib/gitlab/github_import/importer/lfs_objects_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/lfs_objects_importer_spec.rb @@ -3,7 +3,8 @@ require 'spec_helper' RSpec.describe Gitlab::GithubImport::Importer::LfsObjectsImporter do - let(:project) { double(:project, id: 4, import_source: 'foo/bar') } + let_it_be(:project) { create(:project, :import_started) } + let(:client) { double(:client) } let(:download_link) { "http://www.gitlab.com/lfs_objects/oid" } @@ -61,27 +62,12 @@ RSpec.describe Gitlab::GithubImport::Importer::LfsObjectsImporter do .and_raise(exception) end - expect_next_instance_of(Gitlab::Import::Logger) do |logger| - expect(logger) - .to receive(:error) - .with( - message: 'importer failed', - import_source: :github, - project_id: project.id, - parallel: false, - importer: 'Gitlab::GithubImport::Importer::LfsObjectImporter', - 'error.message': 'Invalid Project URL' - ) - end - - expect(Gitlab::ErrorTracking) - .to receive(:track_exception) + expect(Gitlab::Import::ImportFailureService) + .to receive(:track) .with( - exception, - import_source: :github, - parallel: false, project_id: project.id, - importer: 'Gitlab::GithubImport::Importer::LfsObjectImporter' + exception: exception, + error_source: 'Gitlab::GithubImport::Importer::LfsObjectImporter' ).and_call_original importer.execute diff --git a/spec/lib/gitlab/github_import/importer/note_importer_spec.rb b/spec/lib/gitlab/github_import/importer/note_importer_spec.rb index ef0bb90db4a..820f46c7286 100644 --- a/spec/lib/gitlab/github_import/importer/note_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/note_importer_spec.rb @@ -41,7 +41,7 @@ RSpec.describe Gitlab::GithubImport::Importer::NoteImporter do .with(github_note) .and_return([user.id, true]) - expect(Gitlab::Database) + expect(Gitlab::Database.main) .to receive(:bulk_insert) .with( Note.table_name, @@ -71,7 +71,7 @@ RSpec.describe Gitlab::GithubImport::Importer::NoteImporter do .with(github_note) .and_return([project.creator_id, false]) - expect(Gitlab::Database) + expect(Gitlab::Database.main) .to receive(:bulk_insert) .with( Note.table_name, @@ -115,7 +115,7 @@ RSpec.describe Gitlab::GithubImport::Importer::NoteImporter do context 'when the noteable does not exist' do it 'does not import the note' do - expect(Gitlab::Database).not_to receive(:bulk_insert) + expect(Gitlab::Database.main).not_to receive(:bulk_insert) importer.execute end @@ -134,7 +134,7 @@ RSpec.describe Gitlab::GithubImport::Importer::NoteImporter do .with(github_note) .and_return([user.id, true]) - expect(Gitlab::Database) + expect(Gitlab::Database.main) .to receive(:bulk_insert) .and_raise(ActiveRecord::InvalidForeignKey, 'invalid foreign key') diff --git a/spec/lib/gitlab/github_import/importer/pull_requests_importer_spec.rb b/spec/lib/gitlab/github_import/importer/pull_requests_importer_spec.rb index 133d515246a..067b8b09516 100644 --- a/spec/lib/gitlab/github_import/importer/pull_requests_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/pull_requests_importer_spec.rb @@ -148,7 +148,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsImporter do end end - shared_examples '#update_repository' do + describe '#update_repository' do it 'updates the repository' do importer = described_class.new(project, client) @@ -162,6 +162,10 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsImporter do .to receive(:increment) .and_call_original + expect(project.repository) + .to receive(:fetch_remote) + .with(url, forced: false, refmap: Gitlab::GithubImport.refmap) + freeze_time do importer.update_repository @@ -170,28 +174,6 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsImporter do end end - describe '#update_repository with :fetch_remote_params enabled' do - before do - stub_feature_flags(fetch_remote_params: true) - expect(project.repository) - .to receive(:fetch_remote) - .with('github', forced: false, url: url, refmap: Gitlab::GithubImport.refmap) - end - - it_behaves_like '#update_repository' - end - - describe '#update_repository with :fetch_remote_params disabled' do - before do - stub_feature_flags(fetch_remote_params: false) - expect(project.repository) - .to receive(:fetch_remote) - .with('github', forced: false) - end - - it_behaves_like '#update_repository' - end - describe '#update_repository?' do let(:importer) { described_class.new(project, client) } diff --git a/spec/lib/gitlab/github_import/importer/pull_requests_reviews_importer_spec.rb b/spec/lib/gitlab/github_import/importer/pull_requests_reviews_importer_spec.rb index 08be350f0f9..c5fa67e50aa 100644 --- a/spec/lib/gitlab/github_import/importer/pull_requests_reviews_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/pull_requests_reviews_importer_spec.rb @@ -27,100 +27,62 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsReviewsImporter do end describe '#each_object_to_import', :clean_gitlab_redis_cache do - context 'when github_review_importer_query_only_unimported_merge_requests is enabled' do - before do - stub_feature_flags(github_review_importer_query_only_unimported_merge_requests: true) - end - - let(:merge_request) do - create( - :merged_merge_request, - iid: 999, - source_project: project, - target_project: project - ) - end - - let(:review) { double(id: 1) } - - it 'fetches the pull requests reviews data' do - page = double(objects: [review], number: 1) - - expect(review) - .to receive(:merge_request_id=) - .with(merge_request.id) - - expect(client) - .to receive(:each_page) - .exactly(:once) # ensure to be cached on the second call - .with(:pull_request_reviews, 'github/repo', merge_request.iid, page: 1) - .and_yield(page) + let(:merge_request) do + create( + :merged_merge_request, + iid: 999, + source_project: project, + target_project: project + ) + end - expect { |b| subject.each_object_to_import(&b) } - .to yield_with_args(review) + let(:review) { double(id: 1) } - subject.each_object_to_import {} - end + it 'fetches the pull requests reviews data' do + page = double(objects: [review], number: 1) - it 'skips cached pages' do - Gitlab::GithubImport::PageCounter - .new(project, "merge_request/#{merge_request.id}/pull_request_reviews") - .set(2) + expect(review) + .to receive(:merge_request_id=) + .with(merge_request.id) - expect(review).not_to receive(:merge_request_id=) + expect(client) + .to receive(:each_page) + .exactly(:once) # ensure to be cached on the second call + .with(:pull_request_reviews, 'github/repo', merge_request.iid, page: 1) + .and_yield(page) - expect(client) - .to receive(:each_page) - .exactly(:once) # ensure to be cached on the second call - .with(:pull_request_reviews, 'github/repo', merge_request.iid, page: 2) + expect { |b| subject.each_object_to_import(&b) } + .to yield_with_args(review) - subject.each_object_to_import {} - end + subject.each_object_to_import {} + end - it 'skips cached merge requests' do - Gitlab::Cache::Import::Caching.set_add( - "github-importer/merge_request/already-imported/#{project.id}", - merge_request.id - ) + it 'skips cached pages' do + Gitlab::GithubImport::PageCounter + .new(project, "merge_request/#{merge_request.id}/pull_request_reviews") + .set(2) - expect(review).not_to receive(:merge_request_id=) + expect(review).not_to receive(:merge_request_id=) - expect(client).not_to receive(:each_page) + expect(client) + .to receive(:each_page) + .exactly(:once) # ensure to be cached on the second call + .with(:pull_request_reviews, 'github/repo', merge_request.iid, page: 2) - subject.each_object_to_import {} - end + subject.each_object_to_import {} end - context 'when github_review_importer_query_only_unimported_merge_requests is disabled' do - before do - stub_feature_flags(github_review_importer_query_only_unimported_merge_requests: false) - end - - it 'fetchs the merged pull requests data' do - merge_request = create( - :merged_merge_request, - iid: 999, - source_project: project, - target_project: project - ) - - review = double - - expect(review) - .to receive(:merge_request_id=) - .with(merge_request.id) + it 'skips cached merge requests' do + Gitlab::Cache::Import::Caching.set_add( + "github-importer/merge_request/already-imported/#{project.id}", + merge_request.id + ) - allow(client) - .to receive(:pull_request_reviews) - .exactly(:once) # ensure to be cached on the second call - .with('github/repo', merge_request.iid) - .and_return([review]) + expect(review).not_to receive(:merge_request_id=) - expect { |b| subject.each_object_to_import(&b) } - .to yield_with_args(review) + expect(client).not_to receive(:each_page) - subject.each_object_to_import {} - end + subject.each_object_to_import {} end end end diff --git a/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb b/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb index 3839303b881..58a8fb1b7e4 100644 --- a/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb @@ -202,7 +202,7 @@ RSpec.describe Gitlab::GithubImport::Importer::RepositoryImporter do expect(repository) .to receive(:fetch_as_mirror) - .with(project.import_url, refmap: Gitlab::GithubImport.refmap, forced: true, remote_name: 'github') + .with(project.import_url, refmap: Gitlab::GithubImport.refmap, forced: true) service = double expect(Repositories::HousekeepingService) @@ -211,17 +211,6 @@ RSpec.describe Gitlab::GithubImport::Importer::RepositoryImporter do expect(importer.import_repository).to eq(true) end - - it 'marks the import as failed when an error was raised' do - expect(project).to receive(:ensure_repository) - .and_raise(Gitlab::Git::Repository::NoRepository) - - expect(importer) - .to receive(:fail_import) - .and_return(false) - - expect(importer.import_repository).to eq(false) - end end describe '#import_wiki_repository' do @@ -234,28 +223,40 @@ RSpec.describe Gitlab::GithubImport::Importer::RepositoryImporter do expect(importer.import_wiki_repository).to eq(true) end - it 'marks the import as failed and creates an empty repo if an error was raised' do - expect(wiki_repository) - .to receive(:import_repository) - .with(importer.wiki_url) - .and_raise(Gitlab::Git::CommandError) + context 'when it raises a Gitlab::Git::CommandError' do + context 'when the error is not a "repository not exported"' do + it 'creates the wiki and re-raise the exception' do + exception = Gitlab::Git::CommandError.new - expect(importer) - .to receive(:fail_import) - .and_return(false) + expect(wiki_repository) + .to receive(:import_repository) + .with(importer.wiki_url) + .and_raise(exception) - expect(project) - .to receive(:create_wiki) + expect(project) + .to receive(:create_wiki) - expect(importer.import_wiki_repository).to eq(false) - end - end + expect { importer.import_wiki_repository } + .to raise_error(exception) + end + end + + context 'when the error is a "repository not exported"' do + it 'returns true' do + exception = Gitlab::Git::CommandError.new('repository not exported') - describe '#fail_import' do - it 'marks the import as failed' do - expect(project.import_state).to receive(:mark_as_failed).with('foo') + expect(wiki_repository) + .to receive(:import_repository) + .with(importer.wiki_url) + .and_raise(exception) - expect(importer.fail_import('foo')).to eq(false) + expect(project) + .not_to receive(:create_wiki) + + expect(importer.import_wiki_repository) + .to eq(true) + end + end end end diff --git a/spec/lib/gitlab/github_import/logger_spec.rb b/spec/lib/gitlab/github_import/logger_spec.rb new file mode 100644 index 00000000000..6fd0f5db93e --- /dev/null +++ b/spec/lib/gitlab/github_import/logger_spec.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Logger do + subject(:logger) { described_class.new('/dev/null') } + + let(:now) { Time.zone.now } + + describe '#format_message' do + before do + allow(Labkit::Correlation::CorrelationId).to receive(:current_id).and_return('new-correlation-id') + end + + it 'formats strings' do + output = subject.format_message('INFO', now, 'test', 'Hello world') + + expect(Gitlab::Json.parse(output)).to eq({ + 'severity' => 'INFO', + 'time' => now.utc.iso8601(3), + 'message' => 'Hello world', + 'correlation_id' => 'new-correlation-id', + 'feature_category' => 'importers', + 'import_type' => 'github' + }) + end + + it 'formats hashes' do + output = subject.format_message('INFO', now, 'test', { hello: 1 }) + + expect(Gitlab::Json.parse(output)).to eq({ + 'severity' => 'INFO', + 'time' => now.utc.iso8601(3), + 'hello' => 1, + 'correlation_id' => 'new-correlation-id', + 'feature_category' => 'importers', + 'import_type' => 'github' + }) + end + end +end diff --git a/spec/lib/gitlab/github_import/object_counter_spec.rb b/spec/lib/gitlab/github_import/object_counter_spec.rb index 668c11667b5..c9e4ac67061 100644 --- a/spec/lib/gitlab/github_import/object_counter_spec.rb +++ b/spec/lib/gitlab/github_import/object_counter_spec.rb @@ -7,7 +7,7 @@ RSpec.describe Gitlab::GithubImport::ObjectCounter, :clean_gitlab_redis_cache do it 'validates the operation being incremented' do expect { described_class.increment(project, :issue, :unknown) } - .to raise_error(ArgumentError, 'Operation must be fetched or imported') + .to raise_error(ArgumentError, 'operation must be fetched or imported') end it 'increments the counter and saves the key to be listed in the summary later' do @@ -33,4 +33,20 @@ RSpec.describe Gitlab::GithubImport::ObjectCounter, :clean_gitlab_redis_cache do 'imported' => { 'issue' => 2 } }) end + + it 'does not increment the counter if the given value is <= 0' do + expect(Gitlab::Metrics) + .not_to receive(:counter) + + expect(Gitlab::Metrics) + .not_to receive(:counter) + + described_class.increment(project, :issue, :fetched, value: 0) + described_class.increment(project, :issue, :imported, value: nil) + + expect(described_class.summary(project)).to eq({ + 'fetched' => {}, + 'imported' => {} + }) + end end diff --git a/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb b/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb index d56d4708385..1fc7d3c887f 100644 --- a/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb +++ b/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb @@ -5,6 +5,10 @@ require 'spec_helper' RSpec.describe Gitlab::GithubImport::ParallelScheduling do let(:importer_class) do Class.new do + def self.name + 'MyImporter' + end + include(Gitlab::GithubImport::ParallelScheduling) def importer_class @@ -21,7 +25,8 @@ RSpec.describe Gitlab::GithubImport::ParallelScheduling do end end - let(:project) { double(:project, id: 4, import_source: 'foo/bar') } + let_it_be(:project) { create(:project, :import_started, import_source: 'foo/bar') } + let(:client) { double(:client) } describe '#parallel?' do @@ -79,73 +84,130 @@ RSpec.describe Gitlab::GithubImport::ParallelScheduling do .to receive(:sequential_import) .and_return([]) - expect_next_instance_of(Gitlab::Import::Logger) do |logger| - expect(logger) + expect(Gitlab::GithubImport::Logger) + .to receive(:info) + .with( + message: 'starting importer', + parallel: false, + project_id: project.id, + importer: 'Class' + ) + + expect(Gitlab::GithubImport::Logger) + .to receive(:info) + .with( + message: 'importer finished', + parallel: false, + project_id: project.id, + importer: 'Class' + ) + + importer.execute + end + + context 'when abort_on_failure is false' do + it 'logs the error when it fails' do + exception = StandardError.new('some error') + + importer = importer_class.new(project, client, parallel: false) + + expect(importer) + .to receive(:sequential_import) + .and_raise(exception) + + expect(Gitlab::GithubImport::Logger) .to receive(:info) .with( message: 'starting importer', - import_source: :github, parallel: false, project_id: project.id, importer: 'Class' ) - expect(logger) - .to receive(:info) + + expect(Gitlab::Import::ImportFailureService) + .to receive(:track) .with( - message: 'importer finished', - import_source: :github, - parallel: false, project_id: project.id, - importer: 'Class' - ) - end + exception: exception, + error_source: 'MyImporter', + fail_import: false + ).and_call_original - importer.execute + expect { importer.execute } + .to raise_error(exception) + + expect(project.import_state.reload.status).to eq('started') + + expect(project.import_failures).not_to be_empty + expect(project.import_failures.last.exception_class).to eq('StandardError') + expect(project.import_failures.last.exception_message).to eq('some error') + end end - it 'logs the error when it fails' do - exception = StandardError.new('some error') + context 'when abort_on_failure is true' do + let(:importer_class) do + Class.new do + def self.name + 'MyImporter' + end - importer = importer_class.new(project, client, parallel: false) + include(Gitlab::GithubImport::ParallelScheduling) - expect(importer) - .to receive(:sequential_import) - .and_raise(exception) + def importer_class + Class + end + + def object_type + :dummy + end + + def collection_method + :issues + end + + def abort_on_failure + true + end + end + end + + it 'logs the error when it fails and marks import as failed' do + exception = StandardError.new('some error') + + importer = importer_class.new(project, client, parallel: false) - expect_next_instance_of(Gitlab::Import::Logger) do |logger| - expect(logger) + expect(importer) + .to receive(:sequential_import) + .and_raise(exception) + + expect(Gitlab::GithubImport::Logger) .to receive(:info) .with( message: 'starting importer', - import_source: :github, parallel: false, project_id: project.id, importer: 'Class' ) - expect(logger) - .to receive(:error) + + expect(Gitlab::Import::ImportFailureService) + .to receive(:track) .with( - message: 'importer failed', - import_source: :github, project_id: project.id, - parallel: false, - importer: 'Class', - 'error.message': 'some error' - ) - end + exception: exception, + error_source: 'MyImporter', + fail_import: true + ).and_call_original - expect(Gitlab::ErrorTracking) - .to receive(:track_exception) - .with( - exception, - import_source: :github, - parallel: false, - project_id: project.id, - importer: 'Class' - ) - .and_call_original + expect { importer.execute } + .to raise_error(exception) + + expect(project.import_state.reload.status).to eq('failed') + expect(project.import_state.last_error).to eq('some error') - expect { importer.execute }.to raise_error(exception) + expect(project.import_failures).not_to be_empty + expect(project.import_failures.last.exception_class).to eq('StandardError') + expect(project.import_failures.last.exception_message).to eq('some error') + end end end diff --git a/spec/lib/gitlab/github_import/user_finder_spec.rb b/spec/lib/gitlab/github_import/user_finder_spec.rb index 20e67a784e1..f81fa3b1e2e 100644 --- a/spec/lib/gitlab/github_import/user_finder_spec.rb +++ b/spec/lib/gitlab/github_import/user_finder_spec.rb @@ -3,7 +3,14 @@ require 'spec_helper' RSpec.describe Gitlab::GithubImport::UserFinder, :clean_gitlab_redis_cache do - let(:project) { create(:project) } + let(:project) do + create( + :project, + import_type: 'github', + import_url: 'https://api.github.com/user/repo' + ) + end + let(:client) { double(:client) } let(:finder) { described_class.new(project, client) } @@ -263,6 +270,26 @@ RSpec.describe Gitlab::GithubImport::UserFinder, :clean_gitlab_redis_cache do finder.id_for_github_id(id) end + + context 'when importing from github enterprise' do + let(:project) do + create( + :project, + import_type: 'github', + import_url: 'https://othergithub.net/user/repo' + ) + end + + it 'does not look up the user by external id' do + expect(finder).not_to receive(:query_id_for_github_id) + + expect(Gitlab::Cache::Import::Caching) + .to receive(:write) + .with(described_class::ID_CACHE_KEY % id, nil) + + finder.id_for_github_id(id) + end + end end describe '#id_for_github_email' do diff --git a/spec/lib/gitlab/grape_logging/loggers/perf_logger_spec.rb b/spec/lib/gitlab/grape_logging/loggers/perf_logger_spec.rb index 95e3af34174..641fb27a071 100644 --- a/spec/lib/gitlab/grape_logging/loggers/perf_logger_spec.rb +++ b/spec/lib/gitlab/grape_logging/loggers/perf_logger_spec.rb @@ -3,26 +3,23 @@ require 'spec_helper' RSpec.describe Gitlab::GrapeLogging::Loggers::PerfLogger do - subject { described_class.new } + let(:mock_request) { OpenStruct.new(env: {}) } describe ".parameters" do - let(:mock_request) { OpenStruct.new(env: {}) } + subject { described_class.new.parameters(mock_request, nil) } - describe 'when no performance datais are present' do - it 'returns an empty Hash' do - expect(subject.parameters(mock_request, nil)).to eq({}) - end + let(:perf_data) { { redis_calls: 1 } } + + describe 'when no performance data present' do + it { is_expected.not_to include(perf_data) } end - describe 'when Redis calls are present', :request_store do - it 'returns a Hash with Redis information' do + describe 'when performance data present', :request_store do + before do Gitlab::Redis::SharedState.with { |redis| redis.get('perf-logger-test') } - - payload = subject.parameters(mock_request, nil) - - expect(payload[:redis_calls]).to eq(1) - expect(payload[:redis_duration_s]).to be >= 0 end + + it { is_expected.to include(perf_data) } end end end diff --git a/spec/lib/gitlab/graphql/calls_gitaly/field_extension_spec.rb b/spec/lib/gitlab/graphql/calls_gitaly/field_extension_spec.rb index 1d8849f7e38..33f49dbc8d4 100644 --- a/spec/lib/gitlab/graphql/calls_gitaly/field_extension_spec.rb +++ b/spec/lib/gitlab/graphql/calls_gitaly/field_extension_spec.rb @@ -7,7 +7,7 @@ RSpec.describe Gitlab::Graphql::CallsGitaly::FieldExtension, :request_store do let(:field_args) { {} } let(:owner) { fresh_object_type } let(:field) do - ::Types::BaseField.new(name: 'value', type: GraphQL::STRING_TYPE, null: true, owner: owner, **field_args) + ::Types::BaseField.new(name: 'value', type: GraphQL::Types::String, null: true, owner: owner, **field_args) end def resolve_value diff --git a/spec/lib/gitlab/graphql/copy_field_description_spec.rb b/spec/lib/gitlab/graphql/copy_field_description_spec.rb index 310b4046b56..84aa548f2cf 100644 --- a/spec/lib/gitlab/graphql/copy_field_description_spec.rb +++ b/spec/lib/gitlab/graphql/copy_field_description_spec.rb @@ -10,7 +10,7 @@ RSpec.describe Gitlab::Graphql::CopyFieldDescription do Class.new(Types::BaseObject) do graphql_name "TestType" - field :field_name, GraphQL::STRING_TYPE, null: true, description: 'Foo' + field :field_name, GraphQL::Types::String, null: true, description: 'Foo' end end diff --git a/spec/lib/gitlab/graphql/markdown_field_spec.rb b/spec/lib/gitlab/graphql/markdown_field_spec.rb index 44ca23f547c..a3fb0bbbed8 100644 --- a/spec/lib/gitlab/graphql/markdown_field_spec.rb +++ b/spec/lib/gitlab/graphql/markdown_field_spec.rb @@ -10,7 +10,7 @@ RSpec.describe Gitlab::Graphql::MarkdownField do expect(field.name).to eq('testHtml') expect(field.description).to eq('The GitLab Flavored Markdown rendering of `hello`') - expect(field.type).to eq(GraphQL::STRING_TYPE) + expect(field.type).to eq(GraphQL::Types::String) expect(field.to_graphql.complexity).to eq(5) end diff --git a/spec/lib/gitlab/graphql/mount_mutation_spec.rb b/spec/lib/gitlab/graphql/mount_mutation_spec.rb index d6b932e08d2..fe25e923506 100644 --- a/spec/lib/gitlab/graphql/mount_mutation_spec.rb +++ b/spec/lib/gitlab/graphql/mount_mutation_spec.rb @@ -6,8 +6,8 @@ RSpec.describe Gitlab::Graphql::MountMutation do Class.new(Mutations::BaseMutation) do graphql_name 'TestMutation' - argument :foo, GraphQL::STRING_TYPE, required: false - field :bar, GraphQL::STRING_TYPE, null: true + argument :foo, GraphQL::Types::String, required: false + field :bar, GraphQL::Types::String, null: true end end diff --git a/spec/lib/gitlab/graphql/negatable_arguments_spec.rb b/spec/lib/gitlab/graphql/negatable_arguments_spec.rb index bc6e25eb018..71ef75836c0 100644 --- a/spec/lib/gitlab/graphql/negatable_arguments_spec.rb +++ b/spec/lib/gitlab/graphql/negatable_arguments_spec.rb @@ -19,7 +19,7 @@ RSpec.describe Gitlab::Graphql::NegatableArguments do it 'defines any arguments passed as block' do test_resolver.negated do - argument :foo, GraphQL::STRING_TYPE, required: false + argument :foo, GraphQL::Types::String, required: false end expect(test_resolver.arguments['not'].type.arguments.keys).to match_array(['foo']) @@ -27,10 +27,10 @@ RSpec.describe Gitlab::Graphql::NegatableArguments do it 'defines all arguments passed as block even if called multiple times' do test_resolver.negated do - argument :foo, GraphQL::STRING_TYPE, required: false + argument :foo, GraphQL::Types::String, required: false end test_resolver.negated do - argument :bar, GraphQL::STRING_TYPE, required: false + argument :bar, GraphQL::Types::String, required: false end expect(test_resolver.arguments['not'].type.arguments.keys).to match_array(%w[foo bar]) diff --git a/spec/lib/gitlab/graphql/pagination/connections_spec.rb b/spec/lib/gitlab/graphql/pagination/connections_spec.rb index e89e5c17644..f3f59113c81 100644 --- a/spec/lib/gitlab/graphql/pagination/connections_spec.rb +++ b/spec/lib/gitlab/graphql/pagination/connections_spec.rb @@ -33,7 +33,7 @@ RSpec.describe ::Gitlab::Graphql::Pagination::Connections do let(:node_type) do Class.new(::GraphQL::Schema::Object) do graphql_name 'Node' - field :value, GraphQL::INT_TYPE, null: false + field :value, GraphQL::Types::Int, null: false end end diff --git a/spec/lib/gitlab/graphql/present/field_extension_spec.rb b/spec/lib/gitlab/graphql/present/field_extension_spec.rb index 6ea313d30b3..5f0f444e0bb 100644 --- a/spec/lib/gitlab/graphql/present/field_extension_spec.rb +++ b/spec/lib/gitlab/graphql/present/field_extension_spec.rb @@ -10,7 +10,7 @@ RSpec.describe Gitlab::Graphql::Present::FieldExtension do let(:owner) { fresh_object_type } let(:field_name) { 'value' } let(:field) do - ::Types::BaseField.new(name: field_name, type: GraphQL::STRING_TYPE, null: true, owner: owner) + ::Types::BaseField.new(name: field_name, type: GraphQL::Types::String, null: true, owner: owner) end let(:base_presenter) do @@ -38,7 +38,7 @@ RSpec.describe Gitlab::Graphql::Present::FieldExtension do Module.new do include ::Types::BaseInterface - field :interface_field, GraphQL::STRING_TYPE, null: true + field :interface_field, GraphQL::Types::String, null: true end end @@ -58,7 +58,7 @@ RSpec.describe Gitlab::Graphql::Present::FieldExtension do end it 'resolves the interface field using the implementation from the presenter' do - field = ::Types::BaseField.new(name: :interface_field, type: GraphQL::STRING_TYPE, null: true, owner: interface) + field = ::Types::BaseField.new(name: :interface_field, type: GraphQL::Types::String, null: true, owner: interface) value = resolve_field(field, object, object_type: implementation) expect(value).to eq 'made of concrete' @@ -67,7 +67,7 @@ RSpec.describe Gitlab::Graphql::Present::FieldExtension do context 'when the implementation is inherited' do it 'resolves the interface field using the implementation from the presenter' do subclass = Class.new(implementation) { graphql_name 'Subclass' } - field = ::Types::BaseField.new(name: :interface_field, type: GraphQL::STRING_TYPE, null: true, owner: interface) + field = ::Types::BaseField.new(name: :interface_field, type: GraphQL::Types::String, null: true, owner: interface) value = resolve_field(field, object, object_type: subclass) expect(value).to eq 'made of concrete' @@ -79,8 +79,8 @@ RSpec.describe Gitlab::Graphql::Present::FieldExtension do def parent type = fresh_object_type('Parent') type.present_using(provide_foo) - type.field :foo, ::GraphQL::INT_TYPE, null: true - type.field :value, ::GraphQL::STRING_TYPE, null: true + type.field :foo, ::GraphQL::Types::Int, null: true + type.field :value, ::GraphQL::Types::String, null: true type end @@ -88,7 +88,7 @@ RSpec.describe Gitlab::Graphql::Present::FieldExtension do type = Class.new(parent) type.graphql_name 'Child' type.present_using(provide_bar) - type.field :bar, ::GraphQL::INT_TYPE, null: true + type.field :bar, ::GraphQL::Types::Int, null: true type end @@ -150,7 +150,7 @@ RSpec.describe Gitlab::Graphql::Present::FieldExtension do let(:field) do ::Types::BaseField.new( name: field_name, - type: GraphQL::STRING_TYPE, + type: GraphQL::Types::String, null: true, owner: owner, resolve: ->(obj, args, ctx) { 'Hello from a proc' } diff --git a/spec/lib/gitlab/graphql/queries_spec.rb b/spec/lib/gitlab/graphql/queries_spec.rb index a1cd2cdb2de..8b7f4ca7933 100644 --- a/spec/lib/gitlab/graphql/queries_spec.rb +++ b/spec/lib/gitlab/graphql/queries_spec.rb @@ -21,30 +21,30 @@ RSpec.describe Gitlab::Graphql::Queries do let_it_be(:schema) do author = Class.new(GraphQL::Schema::Object) do graphql_name 'Author' - field :name, GraphQL::STRING_TYPE, null: true - field :handle, GraphQL::STRING_TYPE, null: false - field :verified, GraphQL::BOOLEAN_TYPE, null: false + field :name, GraphQL::Types::String, null: true + field :handle, GraphQL::Types::String, null: false + field :verified, GraphQL::Types::Boolean, null: false end post = Class.new(GraphQL::Schema::Object) do graphql_name 'Post' - field :name, GraphQL::STRING_TYPE, null: false - field :title, GraphQL::STRING_TYPE, null: false - field :content, GraphQL::STRING_TYPE, null: true + field :name, GraphQL::Types::String, null: false + field :title, GraphQL::Types::String, null: false + field :content, GraphQL::Types::String, null: true field :author, author, null: false end author.field :posts, [post], null: false do - argument :blog_title, GraphQL::STRING_TYPE, required: false + argument :blog_title, GraphQL::Types::String, required: false end blog = Class.new(GraphQL::Schema::Object) do graphql_name 'Blog' - field :title, GraphQL::STRING_TYPE, null: false - field :description, GraphQL::STRING_TYPE, null: false + field :title, GraphQL::Types::String, null: false + field :description, GraphQL::Types::String, null: false field :main_author, author, null: false field :posts, [post], null: false field :post, post, null: true do - argument :slug, GraphQL::STRING_TYPE, required: true + argument :slug, GraphQL::Types::String, required: true end end @@ -52,10 +52,10 @@ RSpec.describe Gitlab::Graphql::Queries do query(Class.new(GraphQL::Schema::Object) do graphql_name 'Query' field :blog, blog, null: true do - argument :title, GraphQL::STRING_TYPE, required: true + argument :title, GraphQL::Types::String, required: true end field :post, post, null: true do - argument :slug, GraphQL::STRING_TYPE, required: true + argument :slug, GraphQL::Types::String, required: true end end) end diff --git a/spec/lib/gitlab/highlight_spec.rb b/spec/lib/gitlab/highlight_spec.rb index ccb3ae1018a..1f06019c929 100644 --- a/spec/lib/gitlab/highlight_spec.rb +++ b/spec/lib/gitlab/highlight_spec.rb @@ -50,16 +50,9 @@ RSpec.describe Gitlab::Highlight do let(:result) { described_class.highlight(file_name, content) } # content is 44 bytes before do - stub_feature_flags(one_megabyte_file_size_limit: false) stub_config(extra: { 'maximum_text_highlight_size_kilobytes' => 0.0001 } ) # 1.024 bytes end - it 'confirm file size is 1MB when `one_megabyte_file_size_limit` is enabled' do - stub_feature_flags(one_megabyte_file_size_limit: true) - expect(described_class.too_large?(1024.kilobytes)).to eq(false) - expect(described_class.too_large?(1025.kilobytes)).to eq(true) - end - it 'increments the metric for oversized files' do expect { result }.to change { over_highlight_size_limit('file size: 0.0001') }.by(1) end diff --git a/spec/lib/gitlab/http_spec.rb b/spec/lib/gitlab/http_spec.rb index 71e80de9f89..d0aae2ac475 100644 --- a/spec/lib/gitlab/http_spec.rb +++ b/spec/lib/gitlab/http_spec.rb @@ -33,7 +33,7 @@ RSpec.describe Gitlab::HTTP do WebMock.stub_request(:post, /.*/).to_return do |request| sleep 0.002.seconds - { body: 'I\m slow', status: 200 } + { body: 'I\'m slow', status: 200 } end end @@ -41,25 +41,67 @@ RSpec.describe Gitlab::HTTP do subject(:request_slow_responder) { described_class.post('http://example.org', **options) } - specify do - expect { request_slow_responder }.not_to raise_error + shared_examples 'tracks the timeout but does not raise an error' do + specify :aggregate_failures do + expect(Gitlab::ErrorTracking).to receive(:track_exception).with( + an_instance_of(Gitlab::HTTP::ReadTotalTimeout) + ).once + + expect { request_slow_responder }.not_to raise_error + end + + it 'still calls the block' do + expect { |b| described_class.post('http://example.org', **options, &b) }.to yield_with_args + end end - context 'with use_read_total_timeout option' do + shared_examples 'does not track or raise timeout error' do + specify :aggregate_failures do + expect(Gitlab::ErrorTracking).not_to receive(:track_exception) + + expect { request_slow_responder }.not_to raise_error + end + end + + it_behaves_like 'tracks the timeout but does not raise an error' + + context 'and use_read_total_timeout option is truthy' do let(:options) { { use_read_total_timeout: true } } - it 'raises a timeout error' do + it 'raises an error' do expect { request_slow_responder }.to raise_error(Gitlab::HTTP::ReadTotalTimeout, /Request timed out after ?([0-9]*[.])?[0-9]+ seconds/) end + end - context 'and timeout option' do - let(:options) { { use_read_total_timeout: true, timeout: 10.seconds } } + context 'and timeout option is greater than DEFAULT_READ_TOTAL_TIMEOUT' do + let(:options) { { timeout: 10.seconds } } - it 'overrides the default timeout when timeout option is present' do - expect { request_slow_responder }.not_to raise_error - end + it_behaves_like 'does not track or raise timeout error' + end + + context 'and stream_body option is truthy' do + let(:options) { { stream_body: true } } + + it_behaves_like 'does not track or raise timeout error' + + context 'but skip_read_total_timeout option is falsey' do + let(:options) { { stream_body: true, skip_read_total_timeout: false } } + + it_behaves_like 'tracks the timeout but does not raise an error' end end + + context 'and skip_read_total_timeout option is truthy' do + let(:options) { { skip_read_total_timeout: true } } + + it_behaves_like 'does not track or raise timeout error' + end + + context 'and skip_read_total_timeout option is falsely' do + let(:options) { { skip_read_total_timeout: false } } + + it_behaves_like 'tracks the timeout but does not raise an error' + end end it 'calls a block' do diff --git a/spec/lib/gitlab/import/database_helpers_spec.rb b/spec/lib/gitlab/import/database_helpers_spec.rb index d56e05df5d7..079faed2518 100644 --- a/spec/lib/gitlab/import/database_helpers_spec.rb +++ b/spec/lib/gitlab/import/database_helpers_spec.rb @@ -16,7 +16,7 @@ RSpec.describe Gitlab::Import::DatabaseHelpers do let(:project) { create(:project) } it 'returns the ID returned by the query' do - expect(Gitlab::Database) + expect(Gitlab::Database.main) .to receive(:bulk_insert) .with(Issue.table_name, [attributes], return_ids: true) .and_return([10]) diff --git a/spec/lib/gitlab/import/import_failure_service_spec.rb b/spec/lib/gitlab/import/import_failure_service_spec.rb new file mode 100644 index 00000000000..50b32d634ad --- /dev/null +++ b/spec/lib/gitlab/import/import_failure_service_spec.rb @@ -0,0 +1,136 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Import::ImportFailureService do + let_it_be(:import_type) { 'import_type' } + + let_it_be(:project) do + create( + :project, + :import_started, + import_type: import_type + ) + end + + let(:import_state) { project.import_state } + let(:exception) { StandardError.new('some error') } + + shared_examples 'logs the exception and fails the import' do + it 'when the failure does not abort the import' do + expect(Gitlab::ErrorTracking) + .to receive(:track_exception) + .with( + exception, + project_id: project.id, + import_type: import_type, + source: 'SomeImporter' + ) + + expect(Gitlab::Import::Logger) + .to receive(:error) + .with( + message: 'importer failed', + 'error.message': 'some error', + project_id: project.id, + import_type: import_type, + source: 'SomeImporter' + ) + + described_class.track(**arguments) + + expect(project.import_state.reload.status).to eq('failed') + + expect(project.import_failures).not_to be_empty + expect(project.import_failures.last.exception_class).to eq('StandardError') + expect(project.import_failures.last.exception_message).to eq('some error') + end + end + + shared_examples 'logs the exception and does not fail the import' do + it 'when the failure does not abort the import' do + expect(Gitlab::ErrorTracking) + .to receive(:track_exception) + .with( + exception, + project_id: project.id, + import_type: import_type, + source: 'SomeImporter' + ) + + expect(Gitlab::Import::Logger) + .to receive(:error) + .with( + message: 'importer failed', + 'error.message': 'some error', + project_id: project.id, + import_type: import_type, + source: 'SomeImporter' + ) + + described_class.track(**arguments) + + expect(project.import_state.reload.status).to eq('started') + + expect(project.import_failures).not_to be_empty + expect(project.import_failures.last.exception_class).to eq('StandardError') + expect(project.import_failures.last.exception_message).to eq('some error') + end + end + + context 'when using the project as reference' do + context 'when it fails the import' do + let(:arguments) do + { + project_id: project.id, + exception: exception, + error_source: 'SomeImporter', + fail_import: true + } + end + + it_behaves_like 'logs the exception and fails the import' + end + + context 'when it does not fail the import' do + let(:arguments) do + { + project_id: project.id, + exception: exception, + error_source: 'SomeImporter', + fail_import: false + } + end + + it_behaves_like 'logs the exception and does not fail the import' + end + end + + context 'when using the import_state as reference' do + context 'when it fails the import' do + let(:arguments) do + { + import_state: import_state, + exception: exception, + error_source: 'SomeImporter', + fail_import: true + } + end + + it_behaves_like 'logs the exception and fails the import' + end + + context 'when it does not fail the import' do + let(:arguments) do + { + import_state: import_state, + exception: exception, + error_source: 'SomeImporter', + fail_import: false + } + end + + it_behaves_like 'logs the exception and does not fail the import' + end + end +end diff --git a/spec/lib/gitlab/import/logger_spec.rb b/spec/lib/gitlab/import/logger_spec.rb new file mode 100644 index 00000000000..60978aaa25c --- /dev/null +++ b/spec/lib/gitlab/import/logger_spec.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Import::Logger do + subject { described_class.new('/dev/null') } + + let(:now) { Time.zone.now } + + describe '#format_message' do + before do + allow(Labkit::Correlation::CorrelationId).to receive(:current_id).and_return('new-correlation-id') + end + + it 'formats strings' do + output = subject.format_message('INFO', now, 'test', 'Hello world') + + expect(Gitlab::Json.parse(output)).to eq({ + 'severity' => 'INFO', + 'time' => now.utc.iso8601(3), + 'message' => 'Hello world', + 'correlation_id' => 'new-correlation-id', + 'feature_category' => 'importers' + }) + end + + it 'formats hashes' do + output = subject.format_message('INFO', now, 'test', { hello: 1 }) + + expect(Gitlab::Json.parse(output)).to eq({ + 'severity' => 'INFO', + 'time' => now.utc.iso8601(3), + 'hello' => 1, + 'correlation_id' => 'new-correlation-id', + 'feature_category' => 'importers' + }) + end + end +end diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 78805cea66a..2b7138a7a10 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -7,6 +7,7 @@ issues: - updated_by - milestone - iteration +- work_item_type - notes - resource_label_events - resource_weight_events @@ -56,6 +57,9 @@ issues: - issue_email_participants - test_reports - requirement +- incident_management_issuable_escalation_status +work_item_type: +- issues events: - author - project @@ -461,7 +465,6 @@ project: - file_uploads - import_state - members_and_requesters -- build_trace_section_names - build_trace_chunks - job_artifacts - root_of_fork_network @@ -579,6 +582,7 @@ project: - security_orchestration_policy_configuration - timelogs - error_tracking_errors +- error_tracking_client_keys award_emoji: - awardable - user diff --git a/spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb b/spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb index 29b192de809..fc08a13a8bd 100644 --- a/spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb +++ b/spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb @@ -190,7 +190,7 @@ RSpec.describe Gitlab::ImportExport::FastHashSerializer do end it 'does not complain about non UTF-8 characters in MR diff files' do - ActiveRecord::Base.connection.execute("UPDATE merge_request_diff_files SET diff = '---\n- :diff: !binary |-\n LS0tIC9kZXYvbnVsbAorKysgYi9pbWFnZXMvbnVjb3IucGRmCkBAIC0wLDAg\n KzEsMTY3OSBAQAorJVBERi0xLjUNJeLjz9MNCisxIDAgb2JqDTw8L01ldGFk\n YXR'") + MergeRequest.connection.execute("UPDATE merge_request_diff_files SET diff = '---\n- :diff: !binary |-\n LS0tIC9kZXYvbnVsbAorKysgYi9pbWFnZXMvbnVjb3IucGRmCkBAIC0wLDAg\n KzEsMTY3OSBAQAorJVBERi0xLjUNJeLjz9MNCisxIDAgb2JqDTw8L01ldGFk\n YXR'") expect(subject['merge_requests'].first['merge_request_diff']).not_to be_empty end diff --git a/spec/lib/gitlab/import_export/group/legacy_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/group/legacy_tree_restorer_spec.rb index bfcd4994995..dbd6cb243f6 100644 --- a/spec/lib/gitlab/import_export/group/legacy_tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/group/legacy_tree_restorer_spec.rb @@ -77,7 +77,7 @@ RSpec.describe Gitlab::ImportExport::Group::LegacyTreeRestorer do let(:group) { create(:group) } let(:shared) { Gitlab::ImportExport::Shared.new(group) } let(:group_tree_restorer) { described_class.new(user: importer_user, shared: shared, group: group, group_hash: nil) } - let(:group_json) { ActiveSupport::JSON.decode(IO.read(File.join(shared.export_path, 'group.json'))) } + let(:group_json) { Gitlab::Json.parse(IO.read(File.join(shared.export_path, 'group.json'))) } shared_examples 'excluded attributes' do excluded_attributes = %w[ diff --git a/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb b/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb index d2153221e8f..b67d42d1b71 100644 --- a/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb @@ -111,7 +111,7 @@ RSpec.describe Gitlab::ImportExport::Group::TreeRestorer do let(:shared) { Gitlab::ImportExport::Shared.new(group) } let(:group_tree_restorer) { described_class.new(user: importer_user, shared: shared, group: group) } let(:exported_file) { File.join(shared.export_path, 'tree/groups/4352.json') } - let(:group_json) { ActiveSupport::JSON.decode(IO.read(exported_file)) } + let(:group_json) { Gitlab::Json.parse(IO.read(exported_file)) } shared_examples 'excluded attributes' do excluded_attributes = %w[ diff --git a/spec/lib/gitlab/import_export/import_test_coverage_spec.rb b/spec/lib/gitlab/import_export/import_test_coverage_spec.rb index 9c6d2708607..90966cb4915 100644 --- a/spec/lib/gitlab/import_export/import_test_coverage_spec.rb +++ b/spec/lib/gitlab/import_export/import_test_coverage_spec.rb @@ -86,7 +86,7 @@ RSpec.describe 'Test coverage of the Project Import' do end def relations_from_json(json_file) - json = ActiveSupport::JSON.decode(IO.read(json_file)) + json = Gitlab::Json.parse(IO.read(json_file)) [].tap {|res| gather_relations({ project: json }, res, [])} .map {|relation_names| relation_names.join('.')} diff --git a/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb b/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb index deb22de9160..9e30564b437 100644 --- a/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb +++ b/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb @@ -156,6 +156,41 @@ RSpec.describe Gitlab::ImportExport::Json::StreamingSerializer do subject.execute end end + + describe 'load balancing' do + context 'when feature flag load_balancing_for_export_workers is enabled' do + before do + stub_feature_flags(load_balancing_for_export_workers: true) + end + + context 'when enabled', :db_load_balancing do + it 'reads from replica' do + expect(Gitlab::Database::LoadBalancing::Session.current).to receive(:use_replicas_for_read_queries).and_call_original + + subject.execute + end + end + + context 'when disabled' do + it 'reads from primary' do + allow(Gitlab::Database::LoadBalancing).to receive(:enable?).and_return(false) + expect(Gitlab::Database::LoadBalancing::Session.current).not_to receive(:use_replicas_for_read_queries) + + subject.execute + end + end + end + + context 'when feature flag load_balancing_for_export_workers is disabled' do + it 'reads from primary' do + stub_feature_flags(load_balancing_for_export_workers: false) + + expect(Gitlab::Database::LoadBalancing::Session.current).not_to receive(:use_replicas_for_read_queries) + + subject.execute + end + end + end end describe '.batch_size' do diff --git a/spec/lib/gitlab/import_export/members_mapper_spec.rb b/spec/lib/gitlab/import_export/members_mapper_spec.rb index 9755e322221..04c27b6f8ad 100644 --- a/spec/lib/gitlab/import_export/members_mapper_spec.rb +++ b/spec/lib/gitlab/import_export/members_mapper_spec.rb @@ -165,11 +165,10 @@ RSpec.describe Gitlab::ImportExport::MembersMapper do let(:member_class) { ProjectMember } let(:importable) { create(:project, :public, name: 'searchable_project') } - it 'authorizes the users to the project' do + it 'adds users to project members' do members_mapper.map - expect(user.authorized_project?(importable)).to be true - expect(user2.authorized_project?(importable)).to be true + expect(importable.reload.members.map(&:user)).to include(user, user2) end it 'maps an owner as a maintainer' do diff --git a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb index fd6c66a10a7..bee7c59cab0 100644 --- a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb +++ b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb @@ -386,7 +386,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver do end it 'does not complain about non UTF-8 characters in MR diff files' do - ActiveRecord::Base.connection.execute("UPDATE merge_request_diff_files SET diff = '---\n- :diff: !binary |-\n LS0tIC9kZXYvbnVsbAorKysgYi9pbWFnZXMvbnVjb3IucGRmCkBAIC0wLDAg\n KzEsMTY3OSBAQAorJVBERi0xLjUNJeLjz9MNCisxIDAgb2JqDTw8L01ldGFk\n YXR'") + MergeRequestDiffFile.connection.execute("UPDATE merge_request_diff_files SET diff = '---\n- :diff: !binary |-\n LS0tIC9kZXYvbnVsbAorKysgYi9pbWFnZXMvbnVjb3IucGRmCkBAIC0wLDAg\n KzEsMTY3OSBAQAorJVBERi0xLjUNJeLjz9MNCisxIDAgb2JqDTw8L01ldGFk\n YXR'") expect(project_tree_saver.save).to be true end diff --git a/spec/lib/gitlab/instrumentation/redis_interceptor_spec.rb b/spec/lib/gitlab/instrumentation/redis_interceptor_spec.rb index 09280402e2b..cd1828791c3 100644 --- a/spec/lib/gitlab/instrumentation/redis_interceptor_spec.rb +++ b/spec/lib/gitlab/instrumentation/redis_interceptor_spec.rb @@ -111,4 +111,35 @@ RSpec.describe Gitlab::Instrumentation::RedisInterceptor, :clean_gitlab_redis_sh end end end + + context 'when a command takes longer than DURATION_ERROR_THRESHOLD' do + let(:threshold) { 0.5 } + + before do + stub_const("#{described_class}::DURATION_ERROR_THRESHOLD", threshold) + end + + context 'when report_on_long_redis_durations is disabled' do + it 'does nothing' do + stub_feature_flags(report_on_long_redis_durations: false) + + expect(Gitlab::ErrorTracking).not_to receive(:track_exception) + + Gitlab::Redis::SharedState.with { |r| r.mget('foo', 'foo') { sleep threshold + 0.1 } } + end + end + + context 'when report_on_long_redis_durations is enabled' do + it 'tracks an exception and continues' do + expect(Gitlab::ErrorTracking) + .to receive(:track_exception) + .with(an_instance_of(described_class::MysteryRedisDurationError), + command: 'mget', + duration: be > threshold, + timestamp: a_string_matching(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{5}/)) + + Gitlab::Redis::SharedState.with { |r| r.mget('foo', 'foo') { sleep threshold + 0.1 } } + end + end + end end diff --git a/spec/lib/gitlab/instrumentation_helper_spec.rb b/spec/lib/gitlab/instrumentation_helper_spec.rb index 48fcc9f93db..85daf50717c 100644 --- a/spec/lib/gitlab/instrumentation_helper_spec.rb +++ b/spec/lib/gitlab/instrumentation_helper_spec.rb @@ -83,6 +83,12 @@ RSpec.describe Gitlab::InstrumentationHelper do expect(payload).to include(:cpu_s) end + it 'logs the process ID' do + subject + + expect(payload).to include(:pid) + end + context 'when logging memory allocations' do include MemoryInstrumentationHelper @@ -102,8 +108,6 @@ RSpec.describe Gitlab::InstrumentationHelper do end context 'when load balancing is enabled' do - include_context 'clear DB Load Balancing configuration' - before do allow(Gitlab::Database::LoadBalancing).to receive(:enable?).and_return(true) end diff --git a/spec/lib/gitlab/jira_import/issue_serializer_spec.rb b/spec/lib/gitlab/jira_import/issue_serializer_spec.rb index e57a8457e7c..198d2db234c 100644 --- a/spec/lib/gitlab/jira_import/issue_serializer_spec.rb +++ b/spec/lib/gitlab/jira_import/issue_serializer_spec.rb @@ -192,6 +192,19 @@ RSpec.describe Gitlab::JiraImport::IssueSerializer do expect(subject[:assignee_ids]).to be_nil end end + + context 'with jira server response' do + let(:assignee) { double(attrs: { 'displayName' => 'Solver', 'key' => '1234' }) } + + context 'when assignee maps to a valid GitLab user' do + it 'sets the issue assignees to the mapped user' do + expect(Gitlab::JiraImport).to receive(:get_user_mapping).with(project.id, '1234') + .and_return(user.id) + + expect(subject[:assignee_ids]).to eq([user.id]) + end + end + end end end diff --git a/spec/lib/gitlab/json_cache_spec.rb b/spec/lib/gitlab/json_cache_spec.rb index 8265c3449bb..7899d01b475 100644 --- a/spec/lib/gitlab/json_cache_spec.rb +++ b/spec/lib/gitlab/json_cache_spec.rb @@ -130,7 +130,7 @@ RSpec.describe Gitlab::JsonCache do .with(expanded_key) .and_return(nil) - expect(ActiveSupport::JSON).not_to receive(:decode) + expect(Gitlab::Json).not_to receive(:parse) expect(cache.read(key)).to be_nil end @@ -140,7 +140,7 @@ RSpec.describe Gitlab::JsonCache do .with(expanded_key) .and_return(true) - expect(ActiveSupport::JSON).to receive(:decode).with("true").and_call_original + expect(Gitlab::Json).to receive(:parse).with("true").and_call_original expect(cache.read(key, BroadcastMessage)).to eq(true) end end @@ -151,7 +151,7 @@ RSpec.describe Gitlab::JsonCache do .with(expanded_key) .and_return(false) - expect(ActiveSupport::JSON).to receive(:decode).with("false").and_call_original + expect(Gitlab::Json).to receive(:parse).with("false").and_call_original expect(cache.read(key, BroadcastMessage)).to eq(false) end end diff --git a/spec/lib/gitlab/kas_spec.rb b/spec/lib/gitlab/kas_spec.rb index 24d2b03fe2a..17d038ed16c 100644 --- a/spec/lib/gitlab/kas_spec.rb +++ b/spec/lib/gitlab/kas_spec.rb @@ -65,6 +65,38 @@ RSpec.describe Gitlab::Kas do end end + describe '.tunnel_url' do + before do + stub_config(gitlab_kas: { external_url: external_url }) + end + + subject { described_class.tunnel_url } + + context 'external_url uses wss://' do + let(:external_url) { 'wss://kas.gitlab.example.com' } + + it { is_expected.to eq('https://kas.gitlab.example.com/k8s-proxy') } + end + + context 'external_url uses ws://' do + let(:external_url) { 'ws://kas.gitlab.example.com' } + + it { is_expected.to eq('http://kas.gitlab.example.com/k8s-proxy') } + end + + context 'external_url uses grpcs://' do + let(:external_url) { 'grpcs://kas.gitlab.example.com' } + + it { is_expected.to eq('https://kas.gitlab.example.com/k8s-proxy') } + end + + context 'external_url uses grpc://' do + let(:external_url) { 'grpc://kas.gitlab.example.com' } + + it { is_expected.to eq('http://kas.gitlab.example.com/k8s-proxy') } + end + end + describe '.internal_url' do it 'returns gitlab_kas internal_url config' do expect(described_class.internal_url).to eq(Gitlab.config.gitlab_kas.internal_url) diff --git a/spec/lib/gitlab/kubernetes/default_namespace_spec.rb b/spec/lib/gitlab/kubernetes/default_namespace_spec.rb index 976fe4a0a87..b6816a18baa 100644 --- a/spec/lib/gitlab/kubernetes/default_namespace_spec.rb +++ b/spec/lib/gitlab/kubernetes/default_namespace_spec.rb @@ -32,6 +32,14 @@ RSpec.describe Gitlab::Kubernetes::DefaultNamespace do subject { generator.from_environment_slug(environment.slug) } + shared_examples_for 'handles very long project paths' do + before do + allow(project).to receive(:path).and_return 'x' * 100 + end + + it { is_expected.to satisfy { |s| s.length <= 63 } } + end + context 'namespace per environment is enabled' do context 'platform namespace is specified' do let(:platform_namespace) { 'platform-namespace' } @@ -47,15 +55,12 @@ RSpec.describe Gitlab::Kubernetes::DefaultNamespace do context 'platform namespace is blank' do let(:platform_namespace) { nil } - let(:mock_namespace) { 'mock-namespace' } - it 'constructs a namespace from the project and environment' do - expect(Gitlab::NamespaceSanitizer).to receive(:sanitize) - .with("#{project.path}-#{project.id}-#{environment.slug}".downcase) - .and_return(mock_namespace) - - expect(subject).to eq mock_namespace + it 'constructs a namespace from the project and environment slug' do + expect(subject).to eq "path-with-capitals-#{project.id}-#{environment.slug}" end + + it_behaves_like 'handles very long project paths' end end @@ -70,15 +75,12 @@ RSpec.describe Gitlab::Kubernetes::DefaultNamespace do context 'platform namespace is blank' do let(:platform_namespace) { nil } - let(:mock_namespace) { 'mock-namespace' } - it 'constructs a namespace from the project and environment' do - expect(Gitlab::NamespaceSanitizer).to receive(:sanitize) - .with("#{project.path}-#{project.id}".downcase) - .and_return(mock_namespace) - - expect(subject).to eq mock_namespace + it 'constructs a namespace from just the project' do + expect(subject).to eq "path-with-capitals-#{project.id}" end + + it_behaves_like 'handles very long project paths' end end end diff --git a/spec/lib/gitlab/kubernetes/kubeconfig/entry/cluster_spec.rb b/spec/lib/gitlab/kubernetes/kubeconfig/entry/cluster_spec.rb new file mode 100644 index 00000000000..508808be1be --- /dev/null +++ b/spec/lib/gitlab/kubernetes/kubeconfig/entry/cluster_spec.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Kubernetes::Kubeconfig::Entry::Cluster do + describe '#to_h' do + let(:name) { 'name' } + let(:url) { 'url' } + + subject { described_class.new(name: name, url: url).to_h } + + it { is_expected.to eq({ name: name, cluster: { server: url } }) } + + context 'with a certificate' do + let(:cert) { 'certificate' } + let(:cert_encoded) { Base64.strict_encode64(cert) } + + subject { described_class.new(name: name, url: url, ca_pem: cert).to_h } + + it { is_expected.to eq({ name: name, cluster: { server: url, 'certificate-authority-data': cert_encoded } }) } + end + end +end diff --git a/spec/lib/gitlab/kubernetes/kubeconfig/entry/context_spec.rb b/spec/lib/gitlab/kubernetes/kubeconfig/entry/context_spec.rb new file mode 100644 index 00000000000..43d4c46fda1 --- /dev/null +++ b/spec/lib/gitlab/kubernetes/kubeconfig/entry/context_spec.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Kubernetes::Kubeconfig::Entry::Context do + describe '#to_h' do + let(:name) { 'name' } + let(:user) { 'user' } + let(:cluster) { 'cluster' } + + subject { described_class.new(name: name, user: user, cluster: cluster).to_h } + + it { is_expected.to eq({ name: name, context: { cluster: cluster, user: user } }) } + + context 'with a namespace' do + let(:namespace) { 'namespace' } + + subject { described_class.new(name: name, user: user, cluster: cluster, namespace: namespace).to_h } + + it { is_expected.to eq({ name: name, context: { cluster: cluster, user: user, namespace: namespace } }) } + end + end +end diff --git a/spec/lib/gitlab/kubernetes/kubeconfig/entry/user_spec.rb b/spec/lib/gitlab/kubernetes/kubeconfig/entry/user_spec.rb new file mode 100644 index 00000000000..3d6acc80823 --- /dev/null +++ b/spec/lib/gitlab/kubernetes/kubeconfig/entry/user_spec.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Kubernetes::Kubeconfig::Entry::User do + describe '#to_h' do + let(:name) { 'name' } + let(:token) { 'token' } + + subject { described_class.new(name: name, token: token).to_h } + + it { is_expected.to eq({ name: name, user: { token: token } }) } + end +end diff --git a/spec/lib/gitlab/kubernetes/kubeconfig/template_spec.rb b/spec/lib/gitlab/kubernetes/kubeconfig/template_spec.rb new file mode 100644 index 00000000000..057c4373329 --- /dev/null +++ b/spec/lib/gitlab/kubernetes/kubeconfig/template_spec.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Kubernetes::Kubeconfig::Template do + let(:template) { described_class.new } + + describe '#valid?' do + subject { template.valid? } + + it { is_expected.to be_falsey } + + context 'with configuration added' do + before do + template.add_context(name: 'name', cluster: 'cluster', user: 'user') + end + + it { is_expected.to be_truthy } + end + end + + describe '#to_h' do + subject { described_class.new.to_h } + + it do + is_expected.to eq( + apiVersion: 'v1', + kind: 'Config', + clusters: [], + users: [], + contexts: [] + ) + end + end + + describe '#to_yaml' do + subject { template.to_yaml } + + it { is_expected.to eq(YAML.dump(template.to_h.deep_stringify_keys)) } + end + + describe 'adding entries' do + let(:entry) { instance_double(entry_class, to_h: attributes) } + let(:attributes) do + { name: 'name', other: 'other' } + end + + subject { template.to_h } + + before do + expect(entry_class).to receive(:new).with(attributes).and_return(entry) + end + + describe '#add_cluster' do + let(:entry_class) { Gitlab::Kubernetes::Kubeconfig::Entry::Cluster } + + before do + template.add_cluster(**attributes) + end + + it { is_expected.to include(clusters: [attributes]) } + end + + describe '#add_user' do + let(:entry_class) { Gitlab::Kubernetes::Kubeconfig::Entry::User } + + before do + template.add_user(**attributes) + end + + it { is_expected.to include(users: [attributes]) } + end + + describe '#add_context' do + let(:entry_class) { Gitlab::Kubernetes::Kubeconfig::Entry::Context } + + before do + template.add_context(**attributes) + end + + it { is_expected.to include(contexts: [attributes]) } + end + end +end diff --git a/spec/lib/gitlab/markdown_cache/active_record/extension_spec.rb b/spec/lib/gitlab/markdown_cache/active_record/extension_spec.rb index 23dbd4a5bb3..98385cd80cc 100644 --- a/spec/lib/gitlab/markdown_cache/active_record/extension_spec.rb +++ b/spec/lib/gitlab/markdown_cache/active_record/extension_spec.rb @@ -178,4 +178,66 @@ RSpec.describe Gitlab::MarkdownCache::ActiveRecord::Extension do thing.refresh_markdown_cache! end end + + context 'with note' do + let(:klass) do + Class.new(ActiveRecord::Base) do + self.table_name = 'notes' + include CacheMarkdownField + include Importable + include Mentionable + + attr_mentionable :note, pipeline: :note + cache_markdown_field :note, pipeline: :note + end + end + + let(:thing) { klass.new(note: markdown) } + + before do + thing.note = "hello world" + end + + it 'calls store_mentions!' do + expect(thing).to receive(:store_mentions!).and_call_original + + thing.save! + end + + context 'during import' do + before do + thing.importing = true + end + + it 'does not call store_mentions!' do + expect(thing).not_to receive(:store_mentions!) + + thing.save! + end + end + end + + context 'when persisted cache is newer than current version' do + before do + thing.update_column(:cached_markdown_version, thing.cached_markdown_version + 1) + end + + it 'does not save the generated HTML' do + expect(thing).not_to receive(:update_columns) + + thing.refresh_markdown_cache! + end + end + + context 'when persisted cache is nil' do + before do + thing.update_column(:cached_markdown_version, nil) + end + + it 'does not save the generated HTML' do + expect(thing).to receive(:update_columns) + + thing.refresh_markdown_cache! + end + end end diff --git a/spec/lib/gitlab/metrics/samplers/database_sampler_spec.rb b/spec/lib/gitlab/metrics/samplers/database_sampler_spec.rb index 9572e9f50be..7dda10ab41d 100644 --- a/spec/lib/gitlab/metrics/samplers/database_sampler_spec.rb +++ b/spec/lib/gitlab/metrics/samplers/database_sampler_spec.rb @@ -18,8 +18,8 @@ RSpec.describe Gitlab::Metrics::Samplers::DatabaseSampler do let(:labels) do { class: 'ActiveRecord::Base', - host: Gitlab::Database.config['host'], - port: Gitlab::Database.config['port'] + host: Gitlab::Database.main.config['host'], + port: Gitlab::Database.main.config['port'] } end diff --git a/spec/lib/gitlab/metrics/subscribers/action_cable_spec.rb b/spec/lib/gitlab/metrics/subscribers/action_cable_spec.rb index 0516091a8ec..08437920e0c 100644 --- a/spec/lib/gitlab/metrics/subscribers/action_cable_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/action_cable_spec.rb @@ -40,7 +40,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActionCable, :request_store do allow(::Gitlab::Metrics).to receive(:histogram).with( :action_cable_transmitted_bytes, /transmit/ ).and_return(counter) - message_size = ::ActiveSupport::JSON.encode(data).bytesize + message_size = ::Gitlab::Json.generate(data).bytesize expect(counter).to receive(:observe).with({ channel: channel_class, operation: 'event' }, message_size) diff --git a/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb b/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb index 6fc8f090431..3ffbcbea03c 100644 --- a/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb @@ -8,6 +8,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActiveRecord do let(:env) { {} } let(:subscriber) { described_class.new } let(:connection) { ActiveRecord::Base.connection } + let(:db_config_name) { ::Gitlab::Database.db_config_name(connection) } describe '#transaction' do let(:web_transaction) { double('Gitlab::Metrics::WebTransaction') } @@ -36,7 +37,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActiveRecord do end it 'captures the metrics for web only' do - expect(web_transaction).to receive(:observe).with(:gitlab_database_transaction_seconds, 0.23) + expect(web_transaction).to receive(:observe).with(:gitlab_database_transaction_seconds, 0.23, db_config_name: db_config_name) expect(background_transaction).not_to receive(:observe) expect(background_transaction).not_to receive(:increment) @@ -56,7 +57,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActiveRecord do end it 'captures the metrics for web only' do - expect(web_transaction).to receive(:observe).with(:gitlab_database_transaction_seconds, 0.23) + expect(web_transaction).to receive(:observe).with(:gitlab_database_transaction_seconds, 0.23, { db_config_name: db_config_name }) expect(background_transaction).not_to receive(:observe) expect(background_transaction).not_to receive(:increment) @@ -76,7 +77,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActiveRecord do end it 'captures the metrics for web only' do - expect(background_transaction).to receive(:observe).with(:gitlab_database_transaction_seconds, 0.23) + expect(background_transaction).to receive(:observe).with(:gitlab_database_transaction_seconds, 0.23, db_config_name: db_config_name) expect(web_transaction).not_to receive(:observe) expect(web_transaction).not_to receive(:increment) diff --git a/spec/lib/gitlab/middleware/multipart_spec.rb b/spec/lib/gitlab/middleware/multipart_spec.rb index 65ec3535271..294a5ee82ed 100644 --- a/spec/lib/gitlab/middleware/multipart_spec.rb +++ b/spec/lib/gitlab/middleware/multipart_spec.rb @@ -77,7 +77,8 @@ RSpec.describe Gitlab::Middleware::Multipart do result = subject expect(result[0]).to eq(400) - expect(result[2]).to include('insecure path used') + expect(result[2]).to be_a(Array) + expect(result[2].first).to include('insecure path used') end end end diff --git a/spec/lib/gitlab/object_hierarchy_spec.rb b/spec/lib/gitlab/object_hierarchy_spec.rb index 64161fbafdd..86d09f4601c 100644 --- a/spec/lib/gitlab/object_hierarchy_spec.rb +++ b/spec/lib/gitlab/object_hierarchy_spec.rb @@ -23,7 +23,7 @@ RSpec.describe Gitlab::ObjectHierarchy do end it 'can find ancestors upto a certain level' do - relation = described_class.new(Group.where(id: child2), options: options).base_and_ancestors(upto: child1) + relation = described_class.new(Group.where(id: child2), options: options).base_and_ancestors(upto: child1.id) expect(relation).to contain_exactly(child2) end @@ -143,7 +143,7 @@ RSpec.describe Gitlab::ObjectHierarchy do end it 'can find ancestors upto a certain level' do - relation = described_class.new(Group.where(id: child2), options: options).ancestors(upto: child1) + relation = described_class.new(Group.where(id: child2), options: options).ancestors(upto: child1.id) expect(relation).to be_empty end diff --git a/spec/lib/gitlab/pagination/keyset/order_spec.rb b/spec/lib/gitlab/pagination/keyset/order_spec.rb index 562a9bf4460..b867dd533e0 100644 --- a/spec/lib/gitlab/pagination/keyset/order_spec.rb +++ b/spec/lib/gitlab/pagination/keyset/order_spec.rb @@ -6,32 +6,67 @@ RSpec.describe Gitlab::Pagination::Keyset::Order do describe 'paginate over items correctly' do let(:table) { Arel::Table.new(:my_table) } let(:order) { nil } + let(:default_limit) { 999 } + let(:query_building_method) { :build_query } def run_query(query) - ActiveRecord::Base.connection.execute(query).to_a + ApplicationRecord.connection.execute(query).to_a end - def build_query(order:, where_conditions: nil, limit: nil) + def where_conditions_as_sql(where_conditions) + "WHERE #{Array(where_conditions).map(&:to_sql).join(' OR ')}" + end + + def build_query(order:, where_conditions: [], limit: nil) + where_string = where_conditions_as_sql(where_conditions) + + <<-SQL + SELECT id, year, month + FROM (#{table_data}) my_table (id, year, month) + #{where_string if where_conditions.present?} + ORDER BY #{order} + LIMIT #{limit || default_limit}; + SQL + end + + def build_union_query(order:, where_conditions: [], limit: nil) + return build_query(order: order, where_conditions: where_conditions, limit: limit) if where_conditions.blank? + + union_queries = Array(where_conditions).map do |where_condition| + <<-SQL + (SELECT id, year, month + FROM (#{table_data}) my_table (id, year, month) + WHERE #{where_condition.to_sql} + ORDER BY #{order} + LIMIT #{limit || default_limit}) + SQL + end + + union_query = union_queries.join(" UNION ALL ") + <<-SQL - SELECT id, year, month - FROM (#{table_data}) my_table (id, year, month) - WHERE #{where_conditions || '1=1'} - ORDER BY #{order} - LIMIT #{limit || 999}; + SELECT id, year, month + FROM (#{union_query}) as my_table + ORDER BY #{order} + LIMIT #{limit || default_limit}; SQL end + def cursor_attributes_for_node(node) + order.cursor_attributes_for_node(node) + end + def iterate_and_collect(order:, page_size:, where_conditions: nil) all_items = [] loop do - paginated_items = run_query(build_query(order: order, where_conditions: where_conditions, limit: page_size)) + paginated_items = run_query(send(query_building_method, order: order, where_conditions: where_conditions, limit: page_size)) break if paginated_items.empty? all_items.concat(paginated_items) last_item = paginated_items.last - cursor_attributes = order.cursor_attributes_for_node(last_item) - where_conditions = order.where_values_with_or_query(cursor_attributes).to_sql + cursor_attributes = cursor_attributes_for_node(last_item) + where_conditions = order.build_where_values(cursor_attributes) end all_items @@ -54,15 +89,41 @@ RSpec.describe Gitlab::Pagination::Keyset::Order do it { expect(subject).to eq(expected) } end + + context 'when using the conditions in an UNION query' do + let(:query_building_method) { :build_union_query } + + it { expect(subject).to eq(expected) } + end + + context 'when the cursor attributes are SQL literals' do + def cursor_attributes_for_node(node) + # Simulate the scenario where the cursor attributes are SQL literals + order.cursor_attributes_for_node(node).transform_values.each_with_index do |value, i| + index = i + 1 + value_sql = value.nil? ? 'NULL::integer' : value + values = [value_sql] * index + Arel.sql("(ARRAY[#{values.join(',')}])[#{index}]") # example: ARRAY[cursor_value][1] will return cursor_value + end + end + + it { expect(subject).to eq(expected) } + + context 'when using the conditions in an UNION query' do + let(:query_building_method) { :build_union_query } + + it { expect(subject).to eq(expected) } + end + end end context 'when paginating backwards' do subject do last_item = expected.last cursor_attributes = order.cursor_attributes_for_node(last_item) - where_conditions = order.reversed_order.where_values_with_or_query(cursor_attributes) + where_conditions = order.reversed_order.build_where_values(cursor_attributes) - iterate_and_collect(order: order.reversed_order, page_size: 2, where_conditions: where_conditions.to_sql) + iterate_and_collect(order: order.reversed_order, page_size: 2, where_conditions: where_conditions) end it do @@ -371,7 +432,7 @@ RSpec.describe Gitlab::Pagination::Keyset::Order do reversed = order.reversed_order before_conditions = reversed.where_values_with_or_query(before_cursor) - query = build_query(order: order, where_conditions: "(#{after_conditions.to_sql}) AND (#{before_conditions.to_sql})", limit: 100) + query = build_query(order: order, where_conditions: [Arel::Nodes::And.new([after_conditions, before_conditions])], limit: 100) expect(run_query(query)).to eq([ { "id" => 2, "year" => 2011, "month" => 0 }, diff --git a/spec/lib/gitlab/query_limiting/active_support_subscriber_spec.rb b/spec/lib/gitlab/query_limiting/active_support_subscriber_spec.rb index 1ab8e22d6d1..5ccde789a2e 100644 --- a/spec/lib/gitlab/query_limiting/active_support_subscriber_spec.rb +++ b/spec/lib/gitlab/query_limiting/active_support_subscriber_spec.rb @@ -27,7 +27,7 @@ RSpec.describe Gitlab::QueryLimiting::ActiveSupportSubscriber do context 'when the query is actually a rails cache hit' do it 'does not increment the number of executed SQL queries' do - ActiveRecord::Base.connection.cache do + User.connection.cache do User.count User.count end diff --git a/spec/lib/gitlab/repository_set_cache_spec.rb b/spec/lib/gitlab/repository_set_cache_spec.rb index 4dcf9dc2c05..c93fd884347 100644 --- a/spec/lib/gitlab/repository_set_cache_spec.rb +++ b/spec/lib/gitlab/repository_set_cache_spec.rb @@ -94,12 +94,6 @@ RSpec.describe Gitlab::RepositorySetCache, :clean_gitlab_redis_cache do expect(cache.read(:foo)).to be_empty end - - it 'expires the old key format' do - expect_any_instance_of(Redis).to receive(:unlink).with(cache.cache_key(:foo), cache.old_cache_key(:foo)) # rubocop:disable RSpec/AnyInstanceOf - - subject - end end context 'multiple keys' do diff --git a/spec/lib/gitlab/search_results_spec.rb b/spec/lib/gitlab/search_results_spec.rb index 2974893ec4a..b8972f28889 100644 --- a/spec/lib/gitlab/search_results_spec.rb +++ b/spec/lib/gitlab/search_results_spec.rb @@ -33,6 +33,10 @@ RSpec.describe Gitlab::SearchResults do expect(results.objects('projects', page: 1, per_page: 1, without_count: false)).not_to be_kind_of(Kaminari::PaginatableWithoutCount) end + it 'returns without counts collection when requested' do + expect(results.objects('projects', page: 1, per_page: 1, without_count: true)).to be_kind_of(Kaminari::PaginatableWithoutCount) + end + it 'uses page and per_page to paginate results' do project2 = create(:project, name: 'foo') diff --git a/spec/lib/gitlab/setup_helper/workhorse_spec.rb b/spec/lib/gitlab/setup_helper/workhorse_spec.rb index aa9b4595799..18cb266bf4e 100644 --- a/spec/lib/gitlab/setup_helper/workhorse_spec.rb +++ b/spec/lib/gitlab/setup_helper/workhorse_spec.rb @@ -22,4 +22,28 @@ RSpec.describe Gitlab::SetupHelper::Workhorse do end end end + + describe '.redis_url' do + it 'matches the SharedState URL' do + expect(Gitlab::Redis::SharedState).to receive(:url).and_return('foo') + + expect(described_class.redis_url).to eq('foo') + end + end + + describe '.redis_db' do + subject { described_class.redis_db } + + it 'matches the SharedState DB' do + expect(Gitlab::Redis::SharedState).to receive(:params).and_return(db: 1) + + is_expected.to eq(1) + end + + it 'defaults to 0 if unspecified' do + expect(Gitlab::Redis::SharedState).to receive(:params).and_return({}) + + is_expected.to eq(0) + end + end end diff --git a/spec/lib/gitlab/sidekiq_cluster/cli_spec.rb b/spec/lib/gitlab/sidekiq_cluster/cli_spec.rb index 5347680b253..3dd5ac8ee6c 100644 --- a/spec/lib/gitlab/sidekiq_cluster/cli_spec.rb +++ b/spec/lib/gitlab/sidekiq_cluster/cli_spec.rb @@ -81,7 +81,7 @@ RSpec.describe Gitlab::SidekiqCluster::CLI do end end - context '-timeout flag' do + context 'with --timeout flag' do it 'when given', 'starts Sidekiq workers with given timeout' do expect(Gitlab::SidekiqCluster).to receive(:start) .with([['foo']], default_options.merge(timeout: 10)) @@ -97,6 +97,27 @@ RSpec.describe Gitlab::SidekiqCluster::CLI do end end + context 'with --list-queues flag' do + it 'errors when given --list-queues and --dryrun' do + expect { cli.run(%w(foo --list-queues --dryrun)) }.to raise_error(described_class::CommandError) + end + + it 'prints out a list of queues in alphabetical order' do + expected_queues = [ + 'epics:epics_update_epics_dates', + 'epics_new_epic_issue', + 'new_epic', + 'todos_destroyer:todos_destroyer_confidential_epic' + ] + + allow(Gitlab::SidekiqConfig::CliMethods).to receive(:query_queues).and_return(expected_queues.shuffle) + + expect(cli).to receive(:puts).with([expected_queues]) + + cli.run(%w(--queue-selector feature_category=epics --list-queues)) + end + end + context 'queue namespace expansion' do it 'starts Sidekiq workers for all queues in all_queues.yml with a namespace in argv' do expect(Gitlab::SidekiqConfig::CliMethods).to receive(:worker_queues).and_return(['cronjob:foo', 'cronjob:bar']) diff --git a/spec/lib/gitlab/sidekiq_config/worker_router_spec.rb b/spec/lib/gitlab/sidekiq_config/worker_router_spec.rb index 687e35813b1..4a8dbe69d36 100644 --- a/spec/lib/gitlab/sidekiq_config/worker_router_spec.rb +++ b/spec/lib/gitlab/sidekiq_config/worker_router_spec.rb @@ -114,6 +114,13 @@ RSpec.describe Gitlab::SidekiqConfig::WorkerRouter do ['resource_boundary=cpu', 'queue_b'], ['tags=expensive', 'queue_c'] ] | 'queue_foo' + # Match by generated queue name + [ + ['name=foo_bar', 'queue_foo'], + ['feature_category=feature_a|urgency=low', 'queue_a'], + ['resource_boundary=cpu', 'queue_b'], + ['tags=expensive', 'queue_c'] + ] | 'queue_foo' end end diff --git a/spec/lib/gitlab/sidekiq_config/worker_spec.rb b/spec/lib/gitlab/sidekiq_config/worker_spec.rb index 0c43c33ff8c..f4d7a4b3359 100644 --- a/spec/lib/gitlab/sidekiq_config/worker_spec.rb +++ b/spec/lib/gitlab/sidekiq_config/worker_spec.rb @@ -7,7 +7,7 @@ RSpec.describe Gitlab::SidekiqConfig::Worker do namespace = queue.include?(':') && queue.split(':').first inner_worker = double( name: attributes[:worker_name] || 'Foo::BarWorker', - queue: queue, + generated_queue_name: queue, queue_namespace: namespace, get_feature_category: attributes[:feature_category], get_weight: attributes[:weight], @@ -48,9 +48,9 @@ RSpec.describe Gitlab::SidekiqConfig::Worker do describe 'delegations' do [ - :feature_category_not_owned?, :get_feature_category, :get_weight, - :get_worker_resource_boundary, :get_urgency, :queue, - :queue_namespace, :worker_has_external_dependencies? + :feature_category_not_owned?, :generated_queue_name, + :get_feature_category, :get_weight, :get_worker_resource_boundary, + :get_urgency, :queue_namespace, :worker_has_external_dependencies? ].each do |meth| it "delegates #{meth} to the worker class" do worker = double diff --git a/spec/lib/gitlab/sidekiq_config_spec.rb b/spec/lib/gitlab/sidekiq_config_spec.rb index d2a53185acd..da135f202f6 100644 --- a/spec/lib/gitlab/sidekiq_config_spec.rb +++ b/spec/lib/gitlab/sidekiq_config_spec.rb @@ -28,7 +28,7 @@ RSpec.describe Gitlab::SidekiqConfig do describe '.workers_for_all_queues_yml' do it 'returns a tuple with FOSS workers first' do expect(described_class.workers_for_all_queues_yml.first) - .to include(an_object_having_attributes(queue: 'post_receive')) + .to include(an_object_having_attributes(generated_queue_name: 'post_receive')) end end diff --git a/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb b/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb index 4406b34e638..a98038cd3f8 100644 --- a/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb +++ b/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb @@ -228,8 +228,6 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do end context 'when the job performs database queries' do - include_context 'clear DB Load Balancing configuration' - before do allow(Time).to receive(:now).and_return(timestamp) allow(Process).to receive(:clock_gettime).and_call_original @@ -256,7 +254,7 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do expect(logger).to receive(:info).with(expected_end_payload_with_db).ordered call_subject(job, 'test_queue') do - ActiveRecord::Base.connection.execute('SELECT pg_sleep(0.1);') + ApplicationRecord.connection.execute('SELECT pg_sleep(0.1);') end end @@ -267,7 +265,7 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do expect(logger).to receive(:info).with(expected_end_payload).ordered call_subject(job.dup, 'test_queue') do - ActiveRecord::Base.connection.execute('SELECT pg_sleep(0.1);') + ApplicationRecord.connection.execute('SELECT pg_sleep(0.1);') end Gitlab::SafeRequestStore.clear! @@ -293,54 +291,41 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do include_examples 'performs database queries' end - context 'when load balancing is enabled' do - before do - allow(Gitlab::Database::LoadBalancing).to receive(:enable?).and_return(true) - end + context 'when load balancing is enabled', :db_load_balancing do + let(:db_config_name) { ::Gitlab::Database.db_config_name(ApplicationRecord.connection) } - let(:dbname) { ::Gitlab::Database.dbname(ActiveRecord::Base.connection) } + let(:expected_db_payload_defaults) do + metrics = + ::Gitlab::Metrics::Subscribers::ActiveRecord.load_balancing_metric_counter_keys + + ::Gitlab::Metrics::Subscribers::ActiveRecord.load_balancing_metric_duration_keys + + ::Gitlab::Metrics::Subscribers::ActiveRecord.db_counter_keys + + [:db_duration_s] + + metrics.each_with_object({}) do |key, result| + result[key.to_s] = 0 + end + end let(:expected_end_payload_with_db) do - expected_end_payload.merge( + expected_end_payload.merge(expected_db_payload_defaults).merge( 'db_duration_s' => a_value >= 0.1, 'db_count' => a_value >= 1, - 'db_cached_count' => 0, - 'db_write_count' => 0, - 'db_replica_count' => 0, - 'db_replica_cached_count' => 0, - 'db_replica_wal_count' => 0, + "db_replica_#{db_config_name}_count" => 0, 'db_replica_duration_s' => a_value >= 0, 'db_primary_count' => a_value >= 1, - 'db_primary_cached_count' => 0, - 'db_primary_wal_count' => 0, + "db_primary_#{db_config_name}_count" => a_value >= 1, 'db_primary_duration_s' => a_value > 0, - "db_primary_#{dbname}_duration_s" => a_value > 0, - 'db_primary_wal_cached_count' => 0, - 'db_replica_wal_cached_count' => 0 + "db_primary_#{db_config_name}_duration_s" => a_value > 0 ) end let(:end_payload) do - start_payload.merge( + start_payload.merge(expected_db_payload_defaults).merge( 'message' => 'TestWorker JID-da883554ee4fe414012f5f42: done: 0.0 sec', 'job_status' => 'done', 'duration_s' => 0.0, 'completed_at' => timestamp.to_f, - 'cpu_s' => 1.111112, - 'db_duration_s' => 0.0, - 'db_cached_count' => 0, - 'db_count' => 0, - 'db_write_count' => 0, - 'db_replica_count' => 0, - 'db_replica_cached_count' => 0, - 'db_replica_wal_count' => 0, - 'db_replica_duration_s' => 0, - 'db_primary_count' => 0, - 'db_primary_cached_count' => 0, - 'db_primary_wal_count' => 0, - 'db_primary_wal_cached_count' => 0, - 'db_replica_wal_cached_count' => 0, - 'db_primary_duration_s' => 0 + 'cpu_s' => 1.111112 ) end diff --git a/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb b/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb index 3ec8d404bf0..cae0bb6b167 100644 --- a/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb @@ -236,7 +236,6 @@ RSpec.describe Gitlab::SidekiqMiddleware::ServerMetrics do include_context 'server metrics with mocked prometheus' include_context 'server metrics call' - include_context 'clear DB Load Balancing configuration' shared_context 'worker declaring data consistency' do let(:worker_class) { LBTestWorker } diff --git a/spec/lib/gitlab/sql/cte_spec.rb b/spec/lib/gitlab/sql/cte_spec.rb index 4cf94f4dcab..18ae2cb065f 100644 --- a/spec/lib/gitlab/sql/cte_spec.rb +++ b/spec/lib/gitlab/sql/cte_spec.rb @@ -8,9 +8,9 @@ RSpec.describe Gitlab::SQL::CTE do relation = User.where(id: 1) cte = described_class.new(:cte_name, relation) sql = cte.to_arel.to_sql - name = ActiveRecord::Base.connection.quote_table_name(:cte_name) + name = ApplicationRecord.connection.quote_table_name(:cte_name) - sql1 = ActiveRecord::Base.connection.unprepared_statement do + sql1 = ApplicationRecord.connection.unprepared_statement do relation.except(:order).to_sql end @@ -30,8 +30,8 @@ RSpec.describe Gitlab::SQL::CTE do cte = described_class.new(:cte_name, nil) table = Arel::Table.new(:kittens) - source_name = ActiveRecord::Base.connection.quote_table_name(:cte_name) - alias_name = ActiveRecord::Base.connection.quote_table_name(:kittens) + source_name = ApplicationRecord.connection.quote_table_name(:cte_name) + alias_name = ApplicationRecord.connection.quote_table_name(:kittens) expect(cte.alias_to(table).to_sql).to eq("#{source_name} AS #{alias_name}") end diff --git a/spec/lib/gitlab/sql/glob_spec.rb b/spec/lib/gitlab/sql/glob_spec.rb index 8e2b842add6..bb3ca0d3f5b 100644 --- a/spec/lib/gitlab/sql/glob_spec.rb +++ b/spec/lib/gitlab/sql/glob_spec.rb @@ -46,10 +46,10 @@ RSpec.describe Gitlab::SQL::Glob do end def query(sql) - ActiveRecord::Base.connection.select_all(sql) + ApplicationRecord.connection.select_all(sql) end def quote(string) - ActiveRecord::Base.connection.quote(string) + ApplicationRecord.connection.quote(string) end end diff --git a/spec/lib/gitlab/sql/recursive_cte_spec.rb b/spec/lib/gitlab/sql/recursive_cte_spec.rb index edcacd404c2..f78c4a0cc02 100644 --- a/spec/lib/gitlab/sql/recursive_cte_spec.rb +++ b/spec/lib/gitlab/sql/recursive_cte_spec.rb @@ -14,9 +14,9 @@ RSpec.describe Gitlab::SQL::RecursiveCTE do cte << rel2 sql = cte.to_arel.to_sql - name = ActiveRecord::Base.connection.quote_table_name(:cte_name) + name = ApplicationRecord.connection.quote_table_name(:cte_name) - sql1, sql2 = ActiveRecord::Base.connection.unprepared_statement do + sql1, sql2 = ApplicationRecord.connection.unprepared_statement do [rel1.except(:order).to_sql, rel2.except(:order).to_sql] end @@ -28,8 +28,8 @@ RSpec.describe Gitlab::SQL::RecursiveCTE do it 'returns an alias for the CTE' do table = Arel::Table.new(:kittens) - source_name = ActiveRecord::Base.connection.quote_table_name(:cte_name) - alias_name = ActiveRecord::Base.connection.quote_table_name(:kittens) + source_name = ApplicationRecord.connection.quote_table_name(:cte_name) + alias_name = ApplicationRecord.connection.quote_table_name(:kittens) expect(cte.alias_to(table).to_sql).to eq("#{source_name} AS #{alias_name}") end @@ -37,8 +37,8 @@ RSpec.describe Gitlab::SQL::RecursiveCTE do it 'replaces dots with an underscore' do table = Arel::Table.new('gitlab.kittens') - source_name = ActiveRecord::Base.connection.quote_table_name(:cte_name) - alias_name = ActiveRecord::Base.connection.quote_table_name(:gitlab_kittens) + source_name = ApplicationRecord.connection.quote_table_name(:cte_name) + alias_name = ApplicationRecord.connection.quote_table_name(:gitlab_kittens) expect(cte.alias_to(table).to_sql).to eq("#{source_name} AS #{alias_name}") end diff --git a/spec/lib/gitlab/subscription_portal_spec.rb b/spec/lib/gitlab/subscription_portal_spec.rb index ed551521b1d..628eb380396 100644 --- a/spec/lib/gitlab/subscription_portal_spec.rb +++ b/spec/lib/gitlab/subscription_portal_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe ::Gitlab::SubscriptionPortal, skip: Gitlab.jh? do +RSpec.describe ::Gitlab::SubscriptionPortal do using RSpec::Parameterized::TableSyntax where(:method_name, :test, :development, :result) do diff --git a/spec/lib/gitlab/usage/docs/helper_spec.rb b/spec/lib/gitlab/usage/docs/helper_spec.rb deleted file mode 100644 index e2bb1d8d818..00000000000 --- a/spec/lib/gitlab/usage/docs/helper_spec.rb +++ /dev/null @@ -1,79 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Usage::Docs::Helper do - subject(:helper) { klass.new } - - let_it_be(:klass) do - Class.new do - include Gitlab::Usage::Docs::Helper - end - end - - let(:metric_definition) do - { - data_category: 'Standard', - name: 'test_metric', - description: description, - product_group: 'group::product intelligence', - status: 'data_available', - tier: %w(free premium) - } - end - - let(:description) { 'Metric description' } - - describe '#render_name' do - it { expect(helper.render_name(metric_definition[:name])).to eq('### `test_metric`') } - end - - describe '#render_description' do - context 'without description' do - let(:description) { nil } - - it { expect(helper.render_description(metric_definition)).to eq('Missing description') } - end - - context 'without description' do - it { expect(helper.render_description(metric_definition)).to eq('Metric description') } - end - end - - describe '#render_yaml_link' do - let(:yaml_link) { 'config/metrics/license/test_metric.yml' } - let(:expected) { "[YAML definition](#{yaml_link})" } - - it { expect(helper.render_yaml_link(yaml_link)).to eq(expected) } - end - - describe '#render_status' do - let(:expected) { "Status: `data_available`" } - - it { expect(helper.render_status(metric_definition)).to eq(expected) } - end - - describe '#render_owner' do - let(:expected) { "Group: `group::product intelligence`" } - - it { expect(helper.render_owner(metric_definition)).to eq(expected) } - end - - describe '#render_tiers' do - let(:expected) { "Tiers: `free`, `premium`" } - - it { expect(helper.render_tiers(metric_definition)).to eq(expected) } - end - - describe '#render_data_category' do - let(:expected) { 'Data Category: `Standard`' } - - it { expect(helper.render_data_category(metric_definition)).to eq(expected) } - end - - describe '#render_owner' do - let(:expected) { "Group: `group::product intelligence`" } - - it { expect(helper.render_owner(metric_definition)).to eq(expected) } - end -end diff --git a/spec/lib/gitlab/usage/docs/renderer_spec.rb b/spec/lib/gitlab/usage/docs/renderer_spec.rb deleted file mode 100644 index f3b83a4a4b3..00000000000 --- a/spec/lib/gitlab/usage/docs/renderer_spec.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -CODE_REGEX = %r{<code>(.*)</code>}.freeze - -RSpec.describe Gitlab::Usage::Docs::Renderer do - describe 'contents' do - let(:dictionary_path) { Gitlab::Usage::Docs::Renderer::DICTIONARY_PATH } - let(:items) { Gitlab::Usage::MetricDefinition.definitions.first(10).to_h } - - it 'generates dictionary for given items' do - generated_dictionary = described_class.new(items).contents - - generated_dictionary_keys = RDoc::Markdown - .parse(generated_dictionary) - .table_of_contents - .select { |metric_doc| metric_doc.level == 3 } - .map { |item| item.text.match(CODE_REGEX)&.captures&.first } - - expect(generated_dictionary_keys).to match_array(items.keys) - end - end -end diff --git a/spec/lib/gitlab/usage/docs/value_formatter_spec.rb b/spec/lib/gitlab/usage/docs/value_formatter_spec.rb deleted file mode 100644 index f21656df894..00000000000 --- a/spec/lib/gitlab/usage/docs/value_formatter_spec.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Usage::Docs::ValueFormatter do - describe '.format' do - using RSpec::Parameterized::TableSyntax - where(:key, :value, :expected_value) do - :product_group | 'growth::product intelligence' | '`growth::product intelligence`' - :data_source | 'redis' | 'Redis' - :data_source | 'ruby' | 'Ruby' - :introduced_by_url | 'http://test.com' | '[Introduced by](http://test.com)' - :tier | %w(gold premium) | ' `gold`, `premium`' - :distribution | %w(ce ee) | ' `ce`, `ee`' - :key_path | 'key.path' | '**`key.path`**' - :milestone | '13.4' | '13.4' - :status | 'data_available' | '`data_available`' - end - - with_them do - subject { described_class.format(key, value) } - - it { is_expected.to eq(expected_value) } - end - end -end diff --git a/spec/lib/gitlab/usage/metric_definition_spec.rb b/spec/lib/gitlab/usage/metric_definition_spec.rb index f3c3e5fc550..1ae8a0881ef 100644 --- a/spec/lib/gitlab/usage/metric_definition_spec.rb +++ b/spec/lib/gitlab/usage/metric_definition_spec.rb @@ -18,7 +18,7 @@ RSpec.describe Gitlab::Usage::MetricDefinition do distribution: %w(ee ce), tier: %w(free starter premium ultimate bronze silver gold), name: 'uuid', - data_category: 'Standard' + data_category: 'standard' } end @@ -87,14 +87,14 @@ RSpec.describe Gitlab::Usage::MetricDefinition do end it 'raise exception' do - expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).at_least(:once).with(instance_of(Gitlab::Usage::Metric::InvalidMetricError)) + expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).at_least(:once).with(instance_of(Gitlab::Usage::MetricDefinition::InvalidError)) described_class.new(path, attributes).validate! end context 'with skip_validation' do it 'raise exception if skip_validation: false' do - expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).at_least(:once).with(instance_of(Gitlab::Usage::Metric::InvalidMetricError)) + expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).at_least(:once).with(instance_of(Gitlab::Usage::MetricDefinition::InvalidError)) described_class.new(path, attributes.merge( { skip_validation: false } )).validate! end @@ -113,7 +113,7 @@ RSpec.describe Gitlab::Usage::MetricDefinition do attributes[:status] = 'broken' attributes.delete(:repair_issue_url) - expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).at_least(:once).with(instance_of(Gitlab::Usage::Metric::InvalidMetricError)) + expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).at_least(:once).with(instance_of(Gitlab::Usage::MetricDefinition::InvalidError)) described_class.new(path, attributes).validate! end @@ -173,7 +173,7 @@ RSpec.describe Gitlab::Usage::MetricDefinition do write_metric(metric1, path, yaml_content) write_metric(metric2, path, yaml_content) - expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).with(instance_of(Gitlab::Usage::Metric::InvalidMetricError)) + expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).with(instance_of(Gitlab::Usage::MetricDefinition::InvalidError)) subject end @@ -199,7 +199,7 @@ RSpec.describe Gitlab::Usage::MetricDefinition do data_source: 'database', distribution: %w(ee ce), tier: %w(free starter premium ultimate bronze silver gold), - data_category: 'Optional' + data_category: 'optional' } end diff --git a/spec/lib/gitlab/usage/metric_spec.rb b/spec/lib/gitlab/usage/metric_spec.rb index d4a789419a4..d83f59e4a7d 100644 --- a/spec/lib/gitlab/usage/metric_spec.rb +++ b/spec/lib/gitlab/usage/metric_spec.rb @@ -3,27 +3,46 @@ require 'spec_helper' RSpec.describe Gitlab::Usage::Metric do - describe '#definition' do - it 'returns key_path metric definiton' do - expect(described_class.new(key_path: 'uuid').definition).to be_an(Gitlab::Usage::MetricDefinition) - end + let!(:issue) { create(:issue) } + + let(:attributes) do + { + data_category: "Operational", + key_path: "counts.issues", + description: "Count of Issues created", + product_section: "dev", + product_stage: "plan", + product_group: "group::plan", + product_category: "issue_tracking", + value_type: "number", + status: "data_available", + time_frame: "all", + data_source: "database", + instrumentation_class: "CountIssuesMetric", + distribution: %w(ce ee), + tier: %w(free premium ultimate) + } end - describe '#unflatten_default_path' do - using RSpec::Parameterized::TableSyntax + let(:issue_count_metric_definiton) do + double(:issue_count_metric_definiton, + attributes.merge({ attributes: attributes }) + ) + end - where(:key_path, :value, :expected_hash) do - 'uuid' | nil | { uuid: nil } - 'uuid' | '1111' | { uuid: '1111' } - 'counts.issues' | nil | { counts: { issues: nil } } - 'counts.issues' | 100 | { counts: { issues: 100 } } - 'usage_activity_by_stage.verify.ci_builds' | 100 | { usage_activity_by_stage: { verify: { ci_builds: 100 } } } - end + before do + allow(ApplicationRecord.connection).to receive(:transaction_open?).and_return(false) + end - with_them do - subject { described_class.new(key_path: key_path, value: value).unflatten_key_path } + describe '#with_value' do + it 'returns key_path metric with the corresponding value' do + expect(described_class.new(issue_count_metric_definiton).with_value).to eq({ counts: { issues: 1 } }) + end + end - it { is_expected.to eq(expected_hash) } + describe '#with_instrumentation' do + it 'returns key_path metric with the corresponding generated query' do + expect(described_class.new(issue_count_metric_definiton).with_instrumentation).to eq({ counts: { issues: "SELECT COUNT(\"issues\".\"id\") FROM \"issues\"" } }) end end end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric_spec.rb index 8f52d550e5c..1b2170baf17 100644 --- a/spec/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric_spec.rb +++ b/spec/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric_spec.rb @@ -4,7 +4,7 @@ require 'spec_helper' RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CollectedDataCategoriesMetric do it_behaves_like 'a correct instrumented metric value', {} do - let(:expected_value) { %w[Standard Subscription Operational Optional] } + let(:expected_value) { %w[standard subscription operational optional] } before do allow_next_instance_of(ServicePing::PermitDataCategoriesService) do |instance| diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/database_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/database_metric_spec.rb index 5e36820df5e..0a32bdb95d3 100644 --- a/spec/lib/gitlab/usage/metrics/instrumentations/database_metric_spec.rb +++ b/spec/lib/gitlab/usage/metrics/instrumentations/database_metric_spec.rb @@ -4,11 +4,11 @@ require 'spec_helper' RSpec.describe Gitlab::Usage::Metrics::Instrumentations::DatabaseMetric do subject do - described_class.tap do |m| - m.relation { Issue } - m.operation :count - m.start { m.relation.minimum(:id) } - m.finish { m.relation.maximum(:id) } + described_class.tap do |metric_class| + metric_class.relation { Issue } + metric_class.operation :count + metric_class.start { metric_class.relation.minimum(:id) } + metric_class.finish { metric_class.relation.maximum(:id) } end.new(time_frame: 'all') end @@ -38,9 +38,9 @@ RSpec.describe Gitlab::Usage::Metrics::Instrumentations::DatabaseMetric do context 'with start and finish not called' do subject do - described_class.tap do |m| - m.relation { Issue } - m.operation :count + described_class.tap do |metric_class| + metric_class.relation { Issue } + metric_class.operation :count end.new(time_frame: 'all') end @@ -51,12 +51,12 @@ RSpec.describe Gitlab::Usage::Metrics::Instrumentations::DatabaseMetric do context 'with cache_start_and_finish_as called' do subject do - described_class.tap do |m| - m.relation { Issue } - m.operation :count - m.start { m.relation.minimum(:id) } - m.finish { m.relation.maximum(:id) } - m.cache_start_and_finish_as :special_issue_count + described_class.tap do |metric_class| + metric_class.relation { Issue } + metric_class.operation :count + metric_class.start { metric_class.relation.minimum(:id) } + metric_class.finish { metric_class.relation.maximum(:id) } + metric_class.cache_start_and_finish_as :special_issue_count end.new(time_frame: 'all') end @@ -71,5 +71,45 @@ RSpec.describe Gitlab::Usage::Metrics::Instrumentations::DatabaseMetric do expect(Rails.cache.read('metric_instrumentation/special_issue_count_maximum_id')).to eq(issues.max_by(&:id).id) end end + + context 'with estimate_batch_distinct_count' do + subject do + described_class.tap do |metric_class| + metric_class.relation { Issue } + metric_class.operation(:estimate_batch_distinct_count) + metric_class.start { metric_class.relation.minimum(:id) } + metric_class.finish { metric_class.relation.maximum(:id) } + end.new(time_frame: 'all') + end + + it 'calculates a correct result' do + expect(subject.value).to be_within(Gitlab::Database::PostgresHll::BatchDistinctCounter::ERROR_RATE).percent_of(3) + end + + context 'with block passed to operation' do + let(:buckets) { double('Buckets').as_null_object } + + subject do + described_class.tap do |metric_class| + metric_class.relation { Issue } + metric_class.operation(:estimate_batch_distinct_count) do |result| + result.foo + end + metric_class.start { metric_class.relation.minimum(:id) } + metric_class.finish { metric_class.relation.maximum(:id) } + end.new(time_frame: 'all') + end + + before do + allow(Gitlab::Database::PostgresHll::Buckets).to receive(:new).and_return(buckets) + end + + it 'calls the block passing HLL buckets as an argument' do + expect(buckets).to receive(:foo) + + subject.value + end + end + end end end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/generic_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/generic_metric_spec.rb new file mode 100644 index 00000000000..158be34d39c --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/generic_metric_spec.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::GenericMetric do + shared_examples 'custom fallback' do |custom_fallback| + subject do + Class.new(described_class) do + fallback(custom_fallback) + value { Gitlab::Database.main.version } + end.new(time_frame: 'none') + end + + describe '#value' do + it 'gives the correct value' do + expect(subject.value).to eq(Gitlab::Database.main.version) + end + + context 'when raising an exception' do + it 'return the custom fallback' do + expect(Gitlab::Database.main).to receive(:version).and_raise('Error') + expect(subject.value).to eq(custom_fallback) + end + end + end + end + + context 'with default fallback' do + subject do + Class.new(described_class) do + value { Gitlab::Database.main.version } + end.new(time_frame: 'none') + end + + describe '#value' do + it 'gives the correct value' do + expect(subject.value).to eq(Gitlab::Database.main.version ) + end + + context 'when raising an exception' do + it 'return the default fallback' do + expect(Gitlab::Database.main).to receive(:version).and_raise('Error') + expect(subject.value).to eq(described_class::FALLBACK) + end + end + end + end + + context 'with custom fallback -2' do + it_behaves_like 'custom fallback', -2 + end + + context 'with custom fallback nil' do + it_behaves_like 'custom fallback', nil + end + + context 'with custom fallback false' do + it_behaves_like 'custom fallback', false + end + + context 'with custom fallback true' do + it_behaves_like 'custom fallback', true + end + + context 'with custom fallback []' do + it_behaves_like 'custom fallback', [] + end + + context 'with custom fallback { major: -1 }' do + it_behaves_like 'custom fallback', { major: -1 } + end +end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/redis_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/redis_metric_spec.rb new file mode 100644 index 00000000000..fb3bd1ba834 --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/redis_metric_spec.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::RedisMetric, :clean_gitlab_redis_shared_state do + before do + 4.times do + Gitlab::UsageDataCounters::SourceCodeCounter.count(:pushes) + end + end + + let(:expected_value) { 4 } + + it_behaves_like 'a correct instrumented metric value', { options: { event: 'pushes', counter_class: 'SourceCodeCounter' } } + + it 'raises an exception if event option is not present' do + expect { described_class.new(counter_class: 'SourceCodeCounter') }.to raise_error(ArgumentError) + end + + it 'raises an exception if counter_class option is not present' do + expect { described_class.new(event: 'pushes') }.to raise_error(ArgumentError) + end +end diff --git a/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb b/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb index b4ab9d4861b..0f95da74ff9 100644 --- a/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb +++ b/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb @@ -16,6 +16,14 @@ RSpec.describe Gitlab::Usage::Metrics::NamesSuggestions::Generator do end end + describe '#add_metric' do + let(:metric) {'CountIssuesMetric' } + + it 'computes the suggested name for given metric' do + expect(described_class.add_metric(metric)).to eq('count_issues') + end + end + context 'for count with default column metrics' do it_behaves_like 'name suggestion' do # corresponding metric is collected with count(Board) diff --git a/spec/lib/gitlab/usage_data_counters/ci_template_unique_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/ci_template_unique_counter_spec.rb index b1d5d106082..d4148b57348 100644 --- a/spec/lib/gitlab/usage_data_counters/ci_template_unique_counter_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/ci_template_unique_counter_spec.rb @@ -60,7 +60,7 @@ RSpec.describe Gitlab::UsageDataCounters::CiTemplateUniqueCounter do Gitlab::Ci::Pipeline::Chain::Config::Content::AutoDevops.new(pipeline, command).content, project: project, user: double, - sha: double + sha: 'd310cc759caaa20cd05a9e0983d6017896d9c34c' ).execute config_source = :auto_devops_source diff --git a/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb index d89202ae7fe..887759014f5 100644 --- a/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb @@ -143,7 +143,7 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s context 'when usage_ping is disabled' do it 'does not track the event' do - stub_application_setting(usage_ping_enabled: false) + allow(::ServicePing::ServicePingSettings).to receive(:enabled?).and_return(false) described_class.track_event(weekly_event, values: entity1, time: Date.current) @@ -153,7 +153,7 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s context 'when usage_ping is enabled' do before do - stub_application_setting(usage_ping_enabled: true) + allow(::ServicePing::ServicePingSettings).to receive(:enabled?).and_return(true) end it 'tracks event when using symbol' do diff --git a/spec/lib/gitlab/usage_data_counters/redis_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/redis_counter_spec.rb index d4f6110b3df..753e09731bf 100644 --- a/spec/lib/gitlab/usage_data_counters/redis_counter_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/redis_counter_spec.rb @@ -8,12 +8,12 @@ RSpec.describe Gitlab::UsageDataCounters::RedisCounter, :clean_gitlab_redis_shar subject { Class.new.extend(described_class) } before do - stub_application_setting(usage_ping_enabled: setting_value) + allow(::ServicePing::ServicePingSettings).to receive(:enabled?).and_return(service_ping_enabled) end describe '.increment' do context 'when usage_ping is disabled' do - let(:setting_value) { false } + let(:service_ping_enabled) { false } it 'counter is not increased' do expect do @@ -23,7 +23,7 @@ RSpec.describe Gitlab::UsageDataCounters::RedisCounter, :clean_gitlab_redis_shar end context 'when usage_ping is enabled' do - let(:setting_value) { true } + let(:service_ping_enabled) { true } it 'counter is increased' do expect do @@ -35,7 +35,7 @@ RSpec.describe Gitlab::UsageDataCounters::RedisCounter, :clean_gitlab_redis_shar describe '.increment_by' do context 'when usage_ping is disabled' do - let(:setting_value) { false } + let(:service_ping_enabled) { false } it 'counter is not increased' do expect do @@ -45,7 +45,7 @@ RSpec.describe Gitlab::UsageDataCounters::RedisCounter, :clean_gitlab_redis_shar end context 'when usage_ping is enabled' do - let(:setting_value) { true } + let(:service_ping_enabled) { true } it 'counter is increased' do expect do diff --git a/spec/lib/gitlab/usage_data_metrics_spec.rb b/spec/lib/gitlab/usage_data_metrics_spec.rb index 18acd767c6d..e0063194f9b 100644 --- a/spec/lib/gitlab/usage_data_metrics_spec.rb +++ b/spec/lib/gitlab/usage_data_metrics_spec.rb @@ -16,7 +16,7 @@ RSpec.describe Gitlab::UsageDataMetrics do allow(ActiveRecord::Base.connection).to receive(:transaction_open?).and_return(false) end - context 'whith instrumentation_class' do + context 'with instrumentation_class' do it 'includes top level keys' do expect(subject).to include(:uuid) expect(subject).to include(:hostname) @@ -26,11 +26,6 @@ RSpec.describe Gitlab::UsageDataMetrics do expect(subject[:counts]).to include(:boards) end - it 'includes i_quickactions_approve monthly and weekly key' do - expect(subject[:redis_hll_counters][:quickactions]).to include(:i_quickactions_approve_monthly) - expect(subject[:redis_hll_counters][:quickactions]).to include(:i_quickactions_approve_weekly) - end - it 'includes counts keys' do expect(subject[:counts]).to include(:issues) end @@ -42,6 +37,43 @@ RSpec.describe Gitlab::UsageDataMetrics do it 'includes usage_activity_by_stage_monthly keys' do expect(subject[:usage_activity_by_stage_monthly][:plan]).to include(:issues) end + + it 'includes settings keys' do + expect(subject[:settings]).to include(:collected_data_categories) + end + + describe 'Redis_HLL_counters' do + let(:metric_files_key_paths) do + Gitlab::Usage::MetricDefinition + .definitions + .select { |k, v| v.attributes[:data_source] == 'redis_hll' && v.key_path.starts_with?('redis_hll_counters') } + .keys + .sort + end + + # Recursively traverse nested Hash of a generated Service Ping to return an Array of key paths + # in the dotted format used in metric definition YAML files, e.g.: 'count.category.metric_name' + def parse_service_ping_keys(object, key_path = []) + if object.is_a?(Hash) + object.each_with_object([]) do |(key, value), result| + result.append parse_service_ping_keys(value, key_path + [key]) + end + else + key_path.join('.') + end + end + + let(:service_ping_key_paths) do + parse_service_ping_keys(subject) + .flatten + .select { |k| k.starts_with?('redis_hll_counters') } + .sort + end + + it 'is included in the Service Ping hash structure' do + expect(metric_files_key_paths).to match_array(service_ping_key_paths) + end + end end end end diff --git a/spec/lib/gitlab/usage_data_non_sql_metrics_spec.rb b/spec/lib/gitlab/usage_data_non_sql_metrics_spec.rb index 32d1288c59c..49682acbc66 100644 --- a/spec/lib/gitlab/usage_data_non_sql_metrics_spec.rb +++ b/spec/lib/gitlab/usage_data_non_sql_metrics_spec.rb @@ -5,6 +5,14 @@ require 'spec_helper' RSpec.describe Gitlab::UsageDataNonSqlMetrics do let(:default_count) { Gitlab::UsageDataNonSqlMetrics::SQL_METRIC_DEFAULT } + describe '#add_metric' do + let(:metric) { 'UuidMetric' } + + it 'computes the metric value for given metric' do + expect(described_class.add_metric(metric)).to eq(Gitlab::CurrentSettings.uuid) + end + end + describe '.count' do it 'returns default value for count' do expect(described_class.count(User)).to eq(default_count) diff --git a/spec/lib/gitlab/usage_data_queries_spec.rb b/spec/lib/gitlab/usage_data_queries_spec.rb index 438ae3efd11..64eff76a9f2 100644 --- a/spec/lib/gitlab/usage_data_queries_spec.rb +++ b/spec/lib/gitlab/usage_data_queries_spec.rb @@ -7,6 +7,14 @@ RSpec.describe Gitlab::UsageDataQueries do allow(ActiveRecord::Base.connection).to receive(:transaction_open?).and_return(false) end + describe '#add_metric' do + let(:metric) { 'CountBoardsMetric' } + + it 'builds the query for given metric' do + expect(described_class.add_metric(metric)).to eq('SELECT COUNT("boards"."id") FROM "boards"') + end + end + describe '.count' do it 'returns the raw SQL' do expect(described_class.count(User)).to start_with('SELECT COUNT("users"."id") FROM "users"') diff --git a/spec/lib/gitlab/usage_data_spec.rb b/spec/lib/gitlab/usage_data_spec.rb index d84974e562a..5d85ad5ad01 100644 --- a/spec/lib/gitlab/usage_data_spec.rb +++ b/spec/lib/gitlab/usage_data_spec.rb @@ -568,7 +568,6 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do expect(count_data[:projects_custom_issue_tracker_active]).to eq(1) expect(count_data[:projects_mattermost_active]).to eq(1) expect(count_data[:groups_mattermost_active]).to eq(1) - expect(count_data[:templates_mattermost_active]).to eq(1) expect(count_data[:instances_mattermost_active]).to eq(1) expect(count_data[:projects_inheriting_mattermost_active]).to eq(1) expect(count_data[:groups_inheriting_slack_active]).to eq(1) @@ -623,6 +622,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do expect(count_data[:deployments]).to eq(4) expect(count_data[:successful_deployments]).to eq(2) expect(count_data[:failed_deployments]).to eq(2) + expect(count_data[:feature_flags]).to eq(1) expect(count_data[:snippets]).to eq(6) expect(count_data[:personal_snippets]).to eq(2) expect(count_data[:project_snippets]).to eq(4) @@ -892,9 +892,9 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do expect(subject[:gitlab_pages][:enabled]).to eq(Gitlab.config.pages.enabled) expect(subject[:gitlab_pages][:version]).to eq(Gitlab::Pages::VERSION) expect(subject[:git][:version]).to eq(Gitlab::Git.version) - expect(subject[:database][:adapter]).to eq(Gitlab::Database.adapter_name) - expect(subject[:database][:version]).to eq(Gitlab::Database.version) - expect(subject[:database][:pg_system_id]).to eq(Gitlab::Database.system_id) + expect(subject[:database][:adapter]).to eq(Gitlab::Database.main.adapter_name) + expect(subject[:database][:version]).to eq(Gitlab::Database.main.version) + expect(subject[:database][:pg_system_id]).to eq(Gitlab::Database.main.system_id) expect(subject[:mail][:smtp_server]).to eq(ActionMailer::Base.smtp_settings[:address]) expect(subject[:gitaly][:version]).to be_present expect(subject[:gitaly][:servers]).to be >= 1 @@ -1067,8 +1067,9 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do subject { described_class.system_usage_data_settings } - it 'gathers settings usage data', :aggregate_failures do + it 'gathers encrypted secrets usage data', :aggregate_failures do expect(subject[:settings][:ldap_encrypted_secrets_enabled]).to eq(Gitlab::Auth::Ldap::Config.encrypted_secrets.active?) + expect(subject[:settings][:smtp_encrypted_secrets_enabled]).to eq(Gitlab::Email::SmtpConfig.encrypted_secrets.active?) end it 'populates operating system information' do @@ -1080,7 +1081,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do end it 'reports collected data categories' do - expected_value = %w[Standard Subscription Operational Optional] + expected_value = %w[standard subscription operational optional] allow_next_instance_of(ServicePing::PermitDataCategoriesService) do |instance| expect(instance).to receive(:execute).and_return(expected_value) @@ -1360,6 +1361,12 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do "in_product_marketing_email_create_1_cta_clicked" => -1, "in_product_marketing_email_create_2_sent" => -1, "in_product_marketing_email_create_2_cta_clicked" => -1, + "in_product_marketing_email_team_short_0_sent" => -1, + "in_product_marketing_email_team_short_0_cta_clicked" => -1, + "in_product_marketing_email_trial_short_0_sent" => -1, + "in_product_marketing_email_trial_short_0_cta_clicked" => -1, + "in_product_marketing_email_admin_verify_0_sent" => -1, + "in_product_marketing_email_admin_verify_0_cta_clicked" => -1, "in_product_marketing_email_verify_0_sent" => -1, "in_product_marketing_email_verify_0_cta_clicked" => -1, "in_product_marketing_email_verify_1_sent" => -1, @@ -1399,6 +1406,12 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do "in_product_marketing_email_create_1_cta_clicked" => 0, "in_product_marketing_email_create_2_sent" => 0, "in_product_marketing_email_create_2_cta_clicked" => 0, + "in_product_marketing_email_team_short_0_sent" => 0, + "in_product_marketing_email_team_short_0_cta_clicked" => 0, + "in_product_marketing_email_trial_short_0_sent" => 0, + "in_product_marketing_email_trial_short_0_cta_clicked" => 0, + "in_product_marketing_email_admin_verify_0_sent" => 0, + "in_product_marketing_email_admin_verify_0_cta_clicked" => 0, "in_product_marketing_email_verify_0_sent" => 1, "in_product_marketing_email_verify_0_cta_clicked" => 0, "in_product_marketing_email_verify_1_sent" => 0, diff --git a/spec/lib/gitlab/utils/usage_data_spec.rb b/spec/lib/gitlab/utils/usage_data_spec.rb index 8f705d6a487..1d01d5c7e6a 100644 --- a/spec/lib/gitlab/utils/usage_data_spec.rb +++ b/spec/lib/gitlab/utils/usage_data_spec.rb @@ -5,6 +5,14 @@ require 'spec_helper' RSpec.describe Gitlab::Utils::UsageData do include Database::DatabaseHelpers + describe '#add_metric' do + let(:metric) { 'UuidMetric'} + + it 'computes the metric value for given metric' do + expect(described_class.add_metric(metric)).to eq(Gitlab::CurrentSettings.uuid) + end + end + describe '#count' do let(:relation) { double(:relation) } @@ -41,10 +49,10 @@ RSpec.describe Gitlab::Utils::UsageData do describe '#estimate_batch_distinct_count' do let(:error_rate) { Gitlab::Database::PostgresHll::BatchDistinctCounter::ERROR_RATE } # HyperLogLog is a probabilistic algorithm, which provides estimated data, with given error margin - let(:relation) { double(:relation) } + let(:relation) { double(:relation, connection: double(:connection)) } before do - allow(ActiveRecord::Base.connection).to receive(:transaction_open?).and_return(false) + allow(ActiveRecord::Base.connection).to receive(:transaction_open?).and_return(false) # rubocop: disable Database/MultipleDatabases end it 'delegates counting to counter class instance' do @@ -95,6 +103,10 @@ RSpec.describe Gitlab::Utils::UsageData do let(:build_needs_estimated_cardinality) { 5.217656147118495 } let(:ci_builds_estimated_cardinality) { 2.0809220082170614 } + before do + allow(ActiveRecord::Base.connection).to receive(:transaction_open?).and_return(false) # rubocop: disable Database/MultipleDatabases + end + context 'different counting parameters' do before_all do 1.upto(3) { |i| create(:ci_build_need, name: i, build: build) } diff --git a/spec/lib/gitlab/web_ide/config/entry/global_spec.rb b/spec/lib/gitlab/web_ide/config/entry/global_spec.rb index 8dbe64af1c7..9af21685c9e 100644 --- a/spec/lib/gitlab/web_ide/config/entry/global_spec.rb +++ b/spec/lib/gitlab/web_ide/config/entry/global_spec.rb @@ -82,7 +82,6 @@ RSpec.describe Gitlab::WebIde::Config::Entry::Global do it 'returns correct script' do expect(global.terminal_value).to eq({ tag_list: [], - yaml_variables: [], job_variables: [], options: { before_script: ['ls'], diff --git a/spec/lib/gitlab/web_ide/config/entry/terminal_spec.rb b/spec/lib/gitlab/web_ide/config/entry/terminal_spec.rb index d6d0fc4224d..f8c4a28ed45 100644 --- a/spec/lib/gitlab/web_ide/config/entry/terminal_spec.rb +++ b/spec/lib/gitlab/web_ide/config/entry/terminal_spec.rb @@ -141,7 +141,6 @@ RSpec.describe Gitlab::WebIde::Config::Entry::Terminal do expect(entry.value) .to eq( tag_list: ['webide'], - yaml_variables: [{ key: 'KEY', value: 'value', public: true }], job_variables: [{ key: 'KEY', value: 'value', public: true }], options: { image: { name: "ruby:3.0" }, diff --git a/spec/lib/gitlab/x509/tag_spec.rb b/spec/lib/gitlab/x509/tag_spec.rb index b011ea515de..be120aaf16a 100644 --- a/spec/lib/gitlab/x509/tag_spec.rb +++ b/spec/lib/gitlab/x509/tag_spec.rb @@ -2,13 +2,13 @@ require 'spec_helper' RSpec.describe Gitlab::X509::Tag do - subject(:signature) { described_class.new(tag).signature } + subject(:signature) { described_class.new(project.repository, tag).signature } describe '#signature' do let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '', 'group/project') } let(:project) { create(:project, :repository) } - describe 'signed tag' do + shared_examples 'signed tag' do let(:tag) { project.repository.find_tag('v1.1.1') } let(:certificate_attributes) do { @@ -33,10 +33,24 @@ RSpec.describe Gitlab::X509::Tag do it { expect(signature.x509_certificate.x509_issuer).to have_attributes(issuer_attributes) } end - context 'unsigned tag' do + shared_examples 'unsigned tag' do let(:tag) { project.repository.find_tag('v1.0.0') } it { expect(signature).to be_nil } end + + context 'with :get_tag_signatures enabled' do + it_behaves_like 'signed tag' + it_behaves_like 'unsigned tag' + end + + context 'with :get_tag_signatures disabled' do + before do + stub_feature_flags(get_tag_signatures: false) + end + + it_behaves_like 'signed tag' + it_behaves_like 'unsigned tag' + end end end diff --git a/spec/lib/peek/views/active_record_spec.rb b/spec/lib/peek/views/active_record_spec.rb index e5aae2822ed..6d50922904e 100644 --- a/spec/lib/peek/views/active_record_spec.rb +++ b/spec/lib/peek/views/active_record_spec.rb @@ -52,6 +52,7 @@ RSpec.describe Peek::Views::ActiveRecord, :request_store do allow(connection_primary_1).to receive(:transaction_open?).and_return(false) allow(connection_primary_2).to receive(:transaction_open?).and_return(true) allow(connection_unknown).to receive(:transaction_open?).and_return(false) + allow(::Gitlab::Database).to receive(:db_config_name).and_return('the_db_config_name') end context 'when database load balancing is not enabled' do @@ -77,32 +78,48 @@ RSpec.describe Peek::Views::ActiveRecord, :request_store do cached: '', transaction: '', duration: 1000.0, - sql: 'SELECT * FROM users WHERE id = 10' + sql: 'SELECT * FROM users WHERE id = 10', + db_config_name: "Config name: the_db_config_name" ), a_hash_including( start: be_a(Time), cached: 'Cached', transaction: '', duration: 2000.0, - sql: 'SELECT * FROM users WHERE id = 10' + sql: 'SELECT * FROM users WHERE id = 10', + db_config_name: "Config name: the_db_config_name" ), a_hash_including( start: be_a(Time), cached: '', transaction: 'In a transaction', duration: 3000.0, - sql: 'UPDATE users SET admin = true WHERE id = 10' + sql: 'UPDATE users SET admin = true WHERE id = 10', + db_config_name: "Config name: the_db_config_name" ), a_hash_including( start: be_a(Time), cached: '', transaction: '', duration: 4000.0, - sql: 'SELECT VERSION()' + sql: 'SELECT VERSION()', + db_config_name: "Config name: the_db_config_name" ) ) ) end + + context 'when the GITLAB_MULTIPLE_DATABASE_METRICS env var is disabled' do + before do + stub_env('GITLAB_MULTIPLE_DATABASE_METRICS', nil) + end + + it 'does not include db_config_name field' do + ActiveSupport::Notifications.publish('sql.active_record', Time.current, Time.current + 1.second, '1', event_1) + + expect(subject.results[:details][0][:db_config_name]).to be_nil + end + end end context 'when database load balancing is enabled' do @@ -114,7 +131,7 @@ RSpec.describe Peek::Views::ActiveRecord, :request_store do allow(Gitlab::Database::LoadBalancing).to receive(:db_role_for_connection).with(connection_unknown).and_return(nil) end - it 'includes db role data' do + it 'includes db role data and db_config_name name' do Timecop.freeze(2021, 2, 23, 10, 0) do ActiveSupport::Notifications.publish('sql.active_record', Time.current, Time.current + 1.second, '1', event_1) ActiveSupport::Notifications.publish('sql.active_record', Time.current, Time.current + 2.seconds, '2', event_2) @@ -127,9 +144,9 @@ RSpec.describe Peek::Views::ActiveRecord, :request_store do summary: { "Cached" => 1, "In a transaction" => 1, - "Primary" => 2, - "Replica" => 1, - "Unknown" => 1 + "Role: Primary" => 2, + "Role: Replica" => 1, + "Role: Unknown" => 1 }, duration: '10000.00ms', warnings: ["active-record duration: 10000.0 over 3000"], @@ -140,7 +157,8 @@ RSpec.describe Peek::Views::ActiveRecord, :request_store do transaction: '', duration: 1000.0, sql: 'SELECT * FROM users WHERE id = 10', - db_role: 'Primary' + db_role: 'Role: Primary', + db_config_name: "Config name: the_db_config_name" ), a_hash_including( start: be_a(Time), @@ -148,7 +166,8 @@ RSpec.describe Peek::Views::ActiveRecord, :request_store do transaction: '', duration: 2000.0, sql: 'SELECT * FROM users WHERE id = 10', - db_role: 'Replica' + db_role: 'Role: Replica', + db_config_name: "Config name: the_db_config_name" ), a_hash_including( start: be_a(Time), @@ -156,7 +175,8 @@ RSpec.describe Peek::Views::ActiveRecord, :request_store do transaction: 'In a transaction', duration: 3000.0, sql: 'UPDATE users SET admin = true WHERE id = 10', - db_role: 'Primary' + db_role: 'Role: Primary', + db_config_name: "Config name: the_db_config_name" ), a_hash_including( start: be_a(Time), @@ -164,10 +184,23 @@ RSpec.describe Peek::Views::ActiveRecord, :request_store do transaction: '', duration: 4000.0, sql: 'SELECT VERSION()', - db_role: 'Unknown' + db_role: 'Role: Unknown', + db_config_name: "Config name: the_db_config_name" ) ) ) end + + context 'when the GITLAB_MULTIPLE_DATABASE_METRICS env var is disabled' do + before do + stub_env('GITLAB_MULTIPLE_DATABASE_METRICS', nil) + end + + it 'does not include db_config_name field' do + ActiveSupport::Notifications.publish('sql.active_record', Time.current, Time.current + 1.second, '1', event_1) + + expect(subject.results[:details][0][:db_config_name]).to be_nil + end + end end end diff --git a/spec/lib/sidebars/groups/menus/ci_cd_menu_spec.rb b/spec/lib/sidebars/groups/menus/ci_cd_menu_spec.rb new file mode 100644 index 00000000000..1ba89af1b02 --- /dev/null +++ b/spec/lib/sidebars/groups/menus/ci_cd_menu_spec.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Sidebars::Groups::Menus::CiCdMenu do + let_it_be(:owner) { create(:user) } + let_it_be(:root_group) do + build(:group, :private).tap do |g| + g.add_owner(owner) + end + end + + let(:group) { root_group } + let(:user) { owner } + let(:context) { Sidebars::Groups::Context.new(current_user: user, container: group) } + + describe 'Menu Items' do + subject { described_class.new(context).renderable_items.index { |e| e.item_id == item_id } } + + describe 'Runners' do + let(:item_id) { :runners } + + specify { is_expected.not_to be_nil } + + describe 'when feature flag :runner_list_group_view_vue_ui is disabled' do + before do + stub_feature_flags(runner_list_group_view_vue_ui: false) + end + + specify { is_expected.to be_nil } + end + + describe 'when the user does not have access' do + let(:user) { nil } + + specify { is_expected.to be_nil } + end + end + end +end diff --git a/spec/lib/sidebars/groups/menus/group_information_menu_spec.rb b/spec/lib/sidebars/groups/menus/group_information_menu_spec.rb new file mode 100644 index 00000000000..b68af6fb8ab --- /dev/null +++ b/spec/lib/sidebars/groups/menus/group_information_menu_spec.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Sidebars::Groups::Menus::GroupInformationMenu do + let_it_be(:owner) { create(:user) } + let_it_be(:root_group) do + build(:group, :private).tap do |g| + g.add_owner(owner) + end + end + + let(:group) { root_group } + let(:user) { owner } + let(:context) { Sidebars::Groups::Context.new(current_user: user, container: group) } + + describe '#title' do + subject { described_class.new(context).title } + + context 'when group is a root group' do + specify { is_expected.to eq 'Group information'} + end + + context 'when group is a child group' do + let(:group) { build(:group, parent: root_group) } + + specify { is_expected.to eq 'Subgroup information'} + end + end + + describe 'Menu Items' do + subject { described_class.new(context).renderable_items.index { |e| e.item_id == item_id } } + + shared_examples 'menu access rights' do + specify { is_expected.not_to be_nil } + + describe 'when the user does not have access' do + let(:user) { nil } + + specify { is_expected.to be_nil } + end + end + + describe 'Activity' do + let(:item_id) { :activity } + + specify { is_expected.not_to be_nil } + + it_behaves_like 'menu access rights' + end + + describe 'Labels' do + let(:item_id) { :labels } + + it_behaves_like 'menu access rights' + end + + describe 'Members' do + let(:item_id) { :members } + + it_behaves_like 'menu access rights' + end + end +end diff --git a/spec/lib/sidebars/groups/menus/issues_menu_spec.rb b/spec/lib/sidebars/groups/menus/issues_menu_spec.rb new file mode 100644 index 00000000000..3d55eb3af40 --- /dev/null +++ b/spec/lib/sidebars/groups/menus/issues_menu_spec.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Sidebars::Groups::Menus::IssuesMenu do + let_it_be(:owner) { create(:user) } + let_it_be(:group) do + build(:group, :private).tap do |g| + g.add_owner(owner) + end + end + + let(:user) { owner } + let(:context) { Sidebars::Groups::Context.new(current_user: user, container: group) } + let(:menu) { described_class.new(context) } + + describe 'Menu Items' do + subject { menu.renderable_items.index { |e| e.item_id == item_id } } + + shared_examples 'menu access rights' do + specify { is_expected.not_to be_nil } + + describe 'when the user does not have access' do + let(:user) { nil } + + specify { is_expected.to be_nil } + end + end + + describe 'List' do + let(:item_id) { :issue_list } + + specify { is_expected.not_to be_nil } + + it_behaves_like 'menu access rights' + end + + describe 'Boards' do + let(:item_id) { :boards } + + it_behaves_like 'menu access rights' + end + + describe 'Milestones' do + let(:item_id) { :milestones } + + it_behaves_like 'menu access rights' + end + end + + it_behaves_like 'pill_count formatted results' do + let(:count_service) { ::Groups::OpenIssuesCountService } + end +end diff --git a/spec/lib/sidebars/groups/menus/kubernetes_menu_spec.rb b/spec/lib/sidebars/groups/menus/kubernetes_menu_spec.rb new file mode 100644 index 00000000000..76e58367c9d --- /dev/null +++ b/spec/lib/sidebars/groups/menus/kubernetes_menu_spec.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Sidebars::Groups::Menus::KubernetesMenu do + let_it_be(:owner) { create(:user) } + let_it_be(:group) do + build(:group, :private).tap do |g| + g.add_owner(owner) + end + end + + let(:user) { owner } + let(:context) { Sidebars::Groups::Context.new(current_user: user, container: group) } + let(:menu) { described_class.new(context) } + + describe '#render?' do + context 'when user can read clusters' do + it 'returns true' do + expect(menu.render?).to eq true + end + end + + context 'when user cannot read clusters rules' do + let(:user) { nil } + + it 'returns false' do + expect(menu.render?).to eq false + end + end + end +end diff --git a/spec/lib/sidebars/groups/menus/merge_requests_menu_spec.rb b/spec/lib/sidebars/groups/menus/merge_requests_menu_spec.rb new file mode 100644 index 00000000000..3aceff29d6d --- /dev/null +++ b/spec/lib/sidebars/groups/menus/merge_requests_menu_spec.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Sidebars::Groups::Menus::MergeRequestsMenu do + let_it_be(:owner) { create(:user) } + let_it_be(:group) do + build(:group, :private).tap do |g| + g.add_owner(owner) + end + end + + let(:user) { owner } + let(:context) { Sidebars::Groups::Context.new(current_user: user, container: group) } + let(:menu) { described_class.new(context) } + + describe '#render?' do + context 'when user can read merge requests' do + it 'returns true' do + expect(menu.render?).to eq true + end + end + + context 'when user cannot read merge requests' do + let(:user) { nil } + + it 'returns false' do + expect(menu.render?).to eq false + end + end + end + + it_behaves_like 'pill_count formatted results' do + let(:count_service) { ::Groups::MergeRequestsCountService } + end +end diff --git a/spec/lib/sidebars/groups/menus/packages_registries_menu_spec.rb b/spec/lib/sidebars/groups/menus/packages_registries_menu_spec.rb new file mode 100644 index 00000000000..5ebd67462f8 --- /dev/null +++ b/spec/lib/sidebars/groups/menus/packages_registries_menu_spec.rb @@ -0,0 +1,163 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Sidebars::Groups::Menus::PackagesRegistriesMenu do + let_it_be(:owner) { create(:user) } + let_it_be(:group) do + build(:group, :private).tap do |g| + g.add_owner(owner) + end + end + + let(:user) { owner } + let(:context) { Sidebars::Groups::Context.new(current_user: user, container: group) } + let(:menu) { described_class.new(context) } + + describe '#render?' do + context 'when menu has menu items to show' do + it 'returns true' do + expect(menu.render?).to eq true + end + end + + context 'when menu does not have any menu item to show' do + it 'returns false' do + stub_container_registry_config(enabled: false) + stub_config(packages: { enabled: false }) + stub_config(dependency_proxy: { enabled: false }) + + expect(menu.render?).to eq false + end + end + end + + describe '#link' do + let(:registry_enabled) { true } + let(:packages_enabled) { true } + + before do + stub_container_registry_config(enabled: registry_enabled) + stub_config(packages: { enabled: packages_enabled }) + stub_config(dependency_proxy: { enabled: true }) + end + + subject { menu.link } + + context 'when Packages Registry is visible' do + it 'menu link points to Packages Registry page' do + expect(subject).to eq find_menu(menu, :packages_registry).link + end + end + + context 'when Packages Registry is not visible' do + let(:packages_enabled) { false } + + it 'menu link points to Container Registry page' do + expect(subject).to eq find_menu(menu, :container_registry).link + end + + context 'when Container Registry is not visible' do + let(:registry_enabled) { false } + + it 'menu link points to Dependency Proxy page' do + expect(subject).to eq find_menu(menu, :dependency_proxy).link + end + end + end + end + + describe 'Menu items' do + subject { find_menu(menu, item_id) } + + describe 'Packages Registry' do + let(:item_id) { :packages_registry } + + context 'when user can read packages' do + before do + stub_config(packages: { enabled: packages_enabled }) + end + + context 'when config package setting is disabled' do + let(:packages_enabled) { false } + + it 'the menu item is not added to list of menu items' do + is_expected.to be_nil + end + end + + context 'when config package setting is enabled' do + let(:packages_enabled) { true } + + it 'the menu item is added to list of menu items' do + is_expected.not_to be_nil + end + end + end + end + + describe 'Container Registry' do + let(:item_id) { :container_registry } + + context 'when user can read container images' do + before do + stub_container_registry_config(enabled: container_enabled) + end + + context 'when config registry setting is disabled' do + let(:container_enabled) { false } + + it 'the menu item is not added to list of menu items' do + is_expected.to be_nil + end + end + + context 'when config registry setting is enabled' do + let(:container_enabled) { true } + + it 'the menu item is added to list of menu items' do + is_expected.not_to be_nil + end + + context 'when user cannot read container images' do + let(:user) { nil } + + it 'the menu item is not added to list of menu items' do + is_expected.to be_nil + end + end + end + end + end + + describe 'Dependency Proxy' do + let(:item_id) { :dependency_proxy } + + before do + stub_config(dependency_proxy: { enabled: dependency_enabled }) + end + + context 'when config dependency_proxy is enabled' do + let(:dependency_enabled) { true } + + it 'the menu item is added to list of menu items' do + is_expected.not_to be_nil + end + end + + context 'when config dependency_proxy is not enabled' do + let(:dependency_enabled) { false } + + it 'the menu item is not added to list of menu items' do + is_expected.to be_nil + end + end + end + end + + private + + def find_menu(menu, item) + menu.renderable_items.find { |i| i.item_id == item } + end +end diff --git a/spec/lib/sidebars/groups/menus/settings_menu_spec.rb b/spec/lib/sidebars/groups/menus/settings_menu_spec.rb new file mode 100644 index 00000000000..314c4cdc602 --- /dev/null +++ b/spec/lib/sidebars/groups/menus/settings_menu_spec.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Sidebars::Groups::Menus::SettingsMenu do + let_it_be(:owner) { create(:user) } + + let_it_be_with_refind(:group) do + build(:group, :private).tap do |g| + g.add_owner(owner) + end + end + + let(:user) { owner } + let(:context) { Sidebars::Groups::Context.new(current_user: user, container: group) } + let(:menu) { described_class.new(context) } + + describe '#render?' do + context 'when user cannot admin group' do + let(:user) { nil } + + it 'returns false' do + expect(menu.render?).to be false + end + end + end + + describe 'Menu items' do + subject { menu.renderable_items.find { |e| e.item_id == item_id } } + + shared_examples 'access rights checks' do + specify { is_expected.not_to be_nil } + + context 'when the user does not have access' do + let(:user) { nil } + + specify { is_expected.to be_nil } + end + end + + describe 'General menu' do + let(:item_id) { :general } + + it_behaves_like 'access rights checks' + end + + describe 'Integrations menu' do + let(:item_id) { :integrations } + + it_behaves_like 'access rights checks' + end + + describe 'Projects menu' do + let(:item_id) { :group_projects } + + it_behaves_like 'access rights checks' + end + + describe 'Repository menu' do + let(:item_id) { :repository } + + it_behaves_like 'access rights checks' + end + + describe 'CI/CD menu' do + let(:item_id) { :ci_cd } + + it_behaves_like 'access rights checks' + end + + describe 'Applications menu' do + let(:item_id) { :applications } + + it_behaves_like 'access rights checks' + end + + describe 'Packages & Registries' do + let(:item_id) { :packages_and_registries } + + before do + allow(group).to receive(:packages_feature_enabled?).and_return(packages_enabled) + end + + describe 'when packages feature is disabled' do + let(:packages_enabled) { false } + + specify { is_expected.to be_nil } + end + + describe 'when packages feature is enabled' do + let(:packages_enabled) { true } + + it_behaves_like 'access rights checks' + end + end + end +end diff --git a/spec/lib/sidebars/menu_spec.rb b/spec/lib/sidebars/menu_spec.rb index 95009aa063f..1db80351e45 100644 --- a/spec/lib/sidebars/menu_spec.rb +++ b/spec/lib/sidebars/menu_spec.rb @@ -26,6 +26,14 @@ RSpec.describe Sidebars::Menu do it 'returns false' do expect(menu.render?).to be false end + + context 'when menu has a partial' do + it 'returns true' do + allow(menu).to receive(:menu_partial).and_return('foo') + + expect(menu.render?).to be true + end + end end context 'when the menu has items' do diff --git a/spec/lib/sidebars/projects/menus/packages_registries_menu_spec.rb b/spec/lib/sidebars/projects/menus/packages_registries_menu_spec.rb index cc4760e69e5..d6807451a25 100644 --- a/spec/lib/sidebars/projects/menus/packages_registries_menu_spec.rb +++ b/spec/lib/sidebars/projects/menus/packages_registries_menu_spec.rb @@ -51,8 +51,8 @@ RSpec.describe Sidebars::Projects::Menus::PackagesRegistriesMenu do context 'when Container Registry is not visible' do let(:registry_enabled) { false } - it 'menu link points to Infrastructure Registry page' do - expect(subject.link).to eq described_class.new(context).renderable_items.find { |i| i.item_id == :infrastructure_registry }.link + it 'does not display menu link' do + expect(subject.render?).to eq false end end end @@ -124,18 +124,22 @@ RSpec.describe Sidebars::Projects::Menus::PackagesRegistriesMenu do describe 'Infrastructure Registry' do let(:item_id) { :infrastructure_registry } - context 'when feature flag :infrastructure_registry_page is enabled' do - it 'the menu item is added to list of menu items' do - stub_feature_flags(infrastructure_registry_page: true) + it 'the menu item is added to list of menu items' do + is_expected.not_to be_nil + end + + context 'when config package setting is disabled' do + it 'does not add the menu item to the list' do + stub_config(packages: { enabled: false }) - is_expected.not_to be_nil + is_expected.to be_nil end end - context 'when feature flag :infrastructure_registry_page is disabled' do - it 'the menu item is not added to list of menu items' do - stub_feature_flags(infrastructure_registry_page: false) + context 'when user cannot read packages' do + let(:user) { nil } + it 'does not add the menu item to the list' do is_expected.to be_nil end end |