diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-08-20 18:42:06 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-08-20 18:42:06 +0000 |
| commit | 6e4e1050d9dba2b7b2523fdd1768823ab85feef4 (patch) | |
| tree | 78be5963ec075d80116a932011d695dd33910b4e /spec/lib | |
| parent | 1ce776de4ae122aba3f349c02c17cebeaa8ecf07 (diff) | |
| download | gitlab-ce-6e4e1050d9dba2b7b2523fdd1768823ab85feef4.tar.gz | |
Add latest changes from gitlab-org/gitlab@13-3-stable-ee
Diffstat (limited to 'spec/lib')
292 files changed, 7345 insertions, 3499 deletions
diff --git a/spec/lib/api/entities/nuget/dependency_group_spec.rb b/spec/lib/api/entities/nuget/dependency_group_spec.rb index 5a649be846b..5e6de45adf2 100644 --- a/spec/lib/api/entities/nuget/dependency_group_spec.rb +++ b/spec/lib/api/entities/nuget/dependency_group_spec.rb @@ -34,6 +34,7 @@ RSpec.describe API::Entities::Nuget::DependencyGroup do ] } end + let(:entity) { described_class.new(dependency_group) } subject { entity.as_json } diff --git a/spec/lib/api/entities/nuget/dependency_spec.rb b/spec/lib/api/entities/nuget/dependency_spec.rb index 13897cc91f0..fb87b21bd1e 100644 --- a/spec/lib/api/entities/nuget/dependency_spec.rb +++ b/spec/lib/api/entities/nuget/dependency_spec.rb @@ -20,6 +20,7 @@ RSpec.describe API::Entities::Nuget::Dependency do 'range': '2.0.0' } end + let(:entity) { described_class.new(dependency) } subject { entity.as_json } diff --git a/spec/lib/api/entities/nuget/metadatum_spec.rb b/spec/lib/api/entities/nuget/metadatum_spec.rb index fe94ea3a69a..210ff0abdd3 100644 --- a/spec/lib/api/entities/nuget/metadatum_spec.rb +++ b/spec/lib/api/entities/nuget/metadatum_spec.rb @@ -10,6 +10,7 @@ RSpec.describe API::Entities::Nuget::Metadatum do icon_url: 'http://sandbox.com/icon' } end + let(:expected) do { 'projectUrl': 'http://sandbox.com/project', @@ -17,6 +18,7 @@ RSpec.describe API::Entities::Nuget::Metadatum do 'iconUrl': 'http://sandbox.com/icon' } end + let(:entity) { described_class.new(metadatum) } subject { entity.as_json } diff --git a/spec/lib/api/entities/nuget/search_result_spec.rb b/spec/lib/api/entities/nuget/search_result_spec.rb index 2a760c70224..a24cd44be9e 100644 --- a/spec/lib/api/entities/nuget/search_result_spec.rb +++ b/spec/lib/api/entities/nuget/search_result_spec.rb @@ -27,6 +27,7 @@ RSpec.describe API::Entities::Nuget::SearchResult do } } end + let(:expected) do { '@type': 'Package', diff --git a/spec/lib/api/entities/snippet_spec.rb b/spec/lib/api/entities/snippet_spec.rb index bcb8c364392..068851f7f6c 100644 --- a/spec/lib/api/entities/snippet_spec.rb +++ b/spec/lib/api/entities/snippet_spec.rb @@ -123,11 +123,11 @@ RSpec.describe ::API::Entities::Snippet do it_behaves_like 'common attributes' it 'returns snippet web_url attribute' do - expect(subject[:web_url]).to match("/snippets/#{snippet.id}") + expect(subject[:web_url]).to match("/-/snippets/#{snippet.id}") end it 'returns snippet raw_url attribute' do - expect(subject[:raw_url]).to match("/snippets/#{snippet.id}/raw") + expect(subject[:raw_url]).to match("/-/snippets/#{snippet.id}/raw") end end @@ -137,11 +137,11 @@ RSpec.describe ::API::Entities::Snippet do it_behaves_like 'common attributes' it 'returns snippet web_url attribute' do - expect(subject[:web_url]).to match("#{snippet.project.full_path}/snippets/#{snippet.id}") + expect(subject[:web_url]).to match("#{snippet.project.full_path}/-/snippets/#{snippet.id}") end it 'returns snippet raw_url attribute' do - expect(subject[:raw_url]).to match("#{snippet.project.full_path}/snippets/#{snippet.id}/raw") + expect(subject[:raw_url]).to match("#{snippet.project.full_path}/-/snippets/#{snippet.id}/raw") end end end diff --git a/spec/lib/api/helpers/merge_requests_helpers_spec.rb b/spec/lib/api/helpers/merge_requests_helpers_spec.rb new file mode 100644 index 00000000000..1d68b7985f1 --- /dev/null +++ b/spec/lib/api/helpers/merge_requests_helpers_spec.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe API::Helpers::MergeRequestsHelpers do + describe '#handle_merge_request_errors!' do + let(:helper) do + Class.new do + include API::Helpers::MergeRequestsHelpers + end.new + end + + let(:merge_request) { double } + + context 'when merge request is valid' do + it 'returns nil' do + allow(merge_request).to receive(:valid?).and_return(true) + + expect(merge_request).not_to receive(:errors) + + helper.handle_merge_request_errors!(merge_request) + end + end + + context 'when merge request is invalid' do + before do + allow(merge_request).to receive(:valid?).and_return(false) + allow(helper).to receive_messages([ + :unprocessable_entity!, :conflict!, :render_validation_error! + ]) + end + + API::Helpers::MergeRequestsHelpers::UNPROCESSABLE_ERROR_KEYS.each do |error_key| + it "responds to a #{error_key} error with unprocessable_entity" do + error = double + allow(merge_request).to receive(:errors).and_return({ error_key => error }) + + expect(helper).to receive(:unprocessable_entity!).with(error) + + helper.handle_merge_request_errors!(merge_request) + end + end + + it 'responds to a validate_branches error with conflict' do + error = double + allow(merge_request).to receive(:errors).and_return({ validate_branches: error }) + + expect(helper).to receive(:conflict!).with(error) + + helper.handle_merge_request_errors!(merge_request) + end + + it 'responds with bad request' do + error = double + allow(merge_request).to receive(:errors).and_return({ other_error: error }) + + expect(helper).to receive(:render_validation_error!).with(merge_request) + + helper.handle_merge_request_errors!(merge_request) + end + end + end +end diff --git a/spec/lib/api/helpers/packages_manager_clients_helpers_spec.rb b/spec/lib/api/helpers/packages_manager_clients_helpers_spec.rb index 80be5f7d10a..832f4abe545 100644 --- a/spec/lib/api/helpers/packages_manager_clients_helpers_spec.rb +++ b/spec/lib/api/helpers/packages_manager_clients_helpers_spec.rb @@ -8,40 +8,6 @@ RSpec.describe API::Helpers::PackagesManagerClientsHelpers do let_it_be(:helper) { Class.new.include(described_class).new } let(:password) { personal_access_token.token } - describe '#find_personal_access_token_from_http_basic_auth' do - let(:headers) { { Authorization: basic_http_auth(username, password) } } - - subject { helper.find_personal_access_token_from_http_basic_auth } - - before do - allow(helper).to receive(:headers).and_return(headers&.with_indifferent_access) - end - - context 'with a valid Authorization header' do - it { is_expected.to eq personal_access_token } - end - - context 'with an invalid Authorization header' do - where(:headers) do - [ - [{ Authorization: 'Invalid' }], - [{}], - [nil] - ] - end - - with_them do - it { is_expected.to be nil } - end - end - - context 'with an unknown Authorization header' do - let(:password) { 'Unknown' } - - it { is_expected.to be nil } - end - end - describe '#find_job_from_http_basic_auth' do let_it_be(:user) { personal_access_token.user } diff --git a/spec/lib/api/helpers_spec.rb b/spec/lib/api/helpers_spec.rb index 8cba1e0794a..d0fe9163c6e 100644 --- a/spec/lib/api/helpers_spec.rb +++ b/spec/lib/api/helpers_spec.rb @@ -230,4 +230,90 @@ RSpec.describe API::Helpers do end end end + + describe "#destroy_conditionally!" do + let!(:project) { create(:project) } + + context 'when unmodified check passes' do + before do + allow(subject).to receive(:check_unmodified_since!).with(project.updated_at).and_return(true) + end + + it 'destroys given project' do + allow(subject).to receive(:status).with(204) + allow(subject).to receive(:body).with(false) + expect(project).to receive(:destroy).and_call_original + + expect { subject.destroy_conditionally!(project) }.to change(Project, :count).by(-1) + end + end + + context 'when unmodified check fails' do + before do + allow(subject).to receive(:check_unmodified_since!).with(project.updated_at).and_throw(:error) + end + + # #destroy_conditionally! uses Grape errors which Ruby-throws a symbol, shifting execution to somewhere else. + # Since this spec isn't in the Grape context, we need to simulate this ourselves. + # Grape throws here: https://github.com/ruby-grape/grape/blob/470f80cd48933cdf11d4c1ee02cb43e0f51a7300/lib/grape/dsl/inside_route.rb#L168-L171 + # And catches here: https://github.com/ruby-grape/grape/blob/cf57d250c3d77a9a488d9f56918d62fd4ac745ff/lib/grape/middleware/error.rb#L38-L40 + it 'does not destroy given project' do + expect(project).not_to receive(:destroy) + + expect { subject.destroy_conditionally!(project) }.to throw_symbol(:error).and change { Project.count }.by(0) + end + end + end + + describe "#check_unmodified_since!" do + let(:unmodified_since_header) { Time.now.change(usec: 0) } + + before do + allow(subject).to receive(:headers).and_return('If-Unmodified-Since' => unmodified_since_header.to_s) + end + + context 'when last modified is later than header value' do + it 'renders error' do + expect(subject).to receive(:render_api_error!) + + subject.check_unmodified_since!(unmodified_since_header + 1.hour) + end + end + + context 'when last modified is earlier than header value' do + it 'does not render error' do + expect(subject).not_to receive(:render_api_error!) + + subject.check_unmodified_since!(unmodified_since_header - 1.hour) + end + end + + context 'when last modified is equal to header value' do + it 'does not render error' do + expect(subject).not_to receive(:render_api_error!) + + subject.check_unmodified_since!(unmodified_since_header) + end + end + + context 'when there is no header value present' do + let(:unmodified_since_header) { nil } + + it 'does not render error' do + expect(subject).not_to receive(:render_api_error!) + + subject.check_unmodified_since!(Time.now) + end + end + + context 'when header value is not a valid time value' do + let(:unmodified_since_header) { "abcd" } + + it 'does not render error' do + expect(subject).not_to receive(:render_api_error!) + + subject.check_unmodified_since!(Time.now) + end + end + end end diff --git a/spec/lib/api/support/git_access_actor_spec.rb b/spec/lib/api/support/git_access_actor_spec.rb index 70753856419..143cc6e56ee 100644 --- a/spec/lib/api/support/git_access_actor_spec.rb +++ b/spec/lib/api/support/git_access_actor_spec.rb @@ -36,7 +36,7 @@ RSpec.describe API::Support::GitAccessActor do describe 'attributes' do describe '#user' do context 'when initialized with a User' do - let(:user) { create(:user) } + let(:user) { build(:user) } it 'returns the User' do expect(subject.user).to eq(user) @@ -44,7 +44,7 @@ RSpec.describe API::Support::GitAccessActor do end context 'when initialized with a Key' do - let(:user_for_key) { create(:user) } + let(:user_for_key) { build(:user) } let(:key) { create(:key, user: user_for_key) } it 'returns the User associated to the Key' do @@ -85,7 +85,7 @@ RSpec.describe API::Support::GitAccessActor do describe '#username' do context 'when initialized with a User' do - let(:user) { create(:user) } + let(:user) { build(:user) } it 'returns the username' do expect(subject.username).to eq(user.username) @@ -104,7 +104,7 @@ RSpec.describe API::Support::GitAccessActor do end context 'that has a User associated' do - let(:user_for_key) { create(:user) } + let(:user_for_key) { build(:user) } it 'returns the username of the User associated to the Key' do expect(subject.username).to eq(user_for_key.username) @@ -113,9 +113,47 @@ RSpec.describe API::Support::GitAccessActor do end end + describe '#key_details' do + context 'when initialized with a User' do + let(:user) { build(:user) } + + it 'returns an empty Hash' do + expect(subject.key_details).to eq({}) + end + end + + context 'when initialized with a Key' do + let(:key) { create(:key, user: user_for_key) } + + context 'that has no User associated' do + let(:user_for_key) { nil } + + it 'returns a Hash' do + expect(subject.key_details).to eq({ gl_key_type: 'key', gl_key_id: key.id }) + end + end + + context 'that has a User associated' do + let(:user_for_key) { build(:user) } + + it 'returns a Hash' do + expect(subject.key_details).to eq({ gl_key_type: 'key', gl_key_id: key.id }) + end + end + end + + context 'when initialized with a DeployKey' do + let(:key) { create(:deploy_key) } + + it 'returns a Hash' do + expect(subject.key_details).to eq({ gl_key_type: 'deploy_key', gl_key_id: key.id }) + end + end + end + describe '#update_last_used_at!' do context 'when initialized with a User' do - let(:user) { create(:user) } + let(:user) { build(:user) } it 'does nothing' do expect(user).not_to receive(:update_last_used_at) diff --git a/spec/lib/api/validations/validators/file_path_spec.rb b/spec/lib/api/validations/validators/file_path_spec.rb index 2c79260b8d5..cbeada6faa1 100644 --- a/spec/lib/api/validations/validators/file_path_spec.rb +++ b/spec/lib/api/validations/validators/file_path_spec.rb @@ -6,31 +6,64 @@ RSpec.describe API::Validations::Validators::FilePath do include ApiValidatorsHelpers subject do - described_class.new(['test'], {}, false, scope.new) + described_class.new(['test'], params, false, scope.new) end - context 'valid file path' do - it 'does not raise a validation error' do - expect_no_validation_error('test' => './foo') - expect_no_validation_error('test' => './bar.rb') - expect_no_validation_error('test' => 'foo%2Fbar%2Fnew%2Ffile.rb') - expect_no_validation_error('test' => 'foo%2Fbar%2Fnew') - expect_no_validation_error('test' => 'foo%252Fbar%252Fnew%252Ffile.rb') + context 'when allowlist is not set' do + shared_examples 'file validation' do + context 'valid file path' do + it 'does not raise a validation error' do + expect_no_validation_error('test' => './foo') + expect_no_validation_error('test' => './bar.rb') + expect_no_validation_error('test' => 'foo%2Fbar%2Fnew%2Ffile.rb') + expect_no_validation_error('test' => 'foo%2Fbar%2Fnew') + expect_no_validation_error('test' => 'foo/bar') + end + end + + context 'invalid file path' do + it 'raise a validation error' do + expect_validation_error('test' => '../foo') + expect_validation_error('test' => '../') + expect_validation_error('test' => 'foo/../../bar') + expect_validation_error('test' => 'foo/../') + expect_validation_error('test' => 'foo/..') + expect_validation_error('test' => '../') + expect_validation_error('test' => '..\\') + expect_validation_error('test' => '..\/') + expect_validation_error('test' => '%2e%2e%2f') + expect_validation_error('test' => '/etc/passwd') + expect_validation_error('test' => 'test%0a/etc/passwd') + expect_validation_error('test' => '%2Ffoo%2Fbar%2Fnew%2Ffile.rb') + expect_validation_error('test' => '%252Ffoo%252Fbar%252Fnew%252Ffile.rb') + expect_validation_error('test' => 'foo%252Fbar%252Fnew%252Ffile.rb') + expect_validation_error('test' => 'foo%25252Fbar%25252Fnew%25252Ffile.rb') + end + end + end + + it_behaves_like 'file validation' do + let(:params) { {} } + end + + it_behaves_like 'file validation' do + let(:params) { true } end end - context 'invalid file path' do - it 'raise a validation error' do - expect_validation_error('test' => '../foo') - expect_validation_error('test' => '../') - expect_validation_error('test' => 'foo/../../bar') - expect_validation_error('test' => 'foo/../') - expect_validation_error('test' => 'foo/..') - expect_validation_error('test' => '../') - expect_validation_error('test' => '..\\') - expect_validation_error('test' => '..\/') - expect_validation_error('test' => '%2e%2e%2f') - expect_validation_error('test' => '/etc/passwd') + context 'when allowlist is set' do + let(:params) { { allowlist: ['/home/bar'] } } + + context 'when file path is included in the allowlist' do + it 'does not raise a validation error' do + expect_no_validation_error('test' => '/home/bar') + end + end + + context 'when file path is not included in the allowlist' do + it 'raises a validation error' do + expect_validation_error('test' => '/foo/xyz') + end end end end diff --git a/spec/lib/backup/repository_spec.rb b/spec/lib/backup/repository_spec.rb index c073a45bf68..fef5e018231 100644 --- a/spec/lib/backup/repository_spec.rb +++ b/spec/lib/backup/repository_spec.rb @@ -3,8 +3,9 @@ require 'spec_helper' RSpec.describe Backup::Repository do + let_it_be(:project) { create(:project, :wiki_repo) } + let(:progress) { StringIO.new } - let!(:project) { create(:project, :wiki_repo) } subject { described_class.new(progress) } @@ -19,13 +20,88 @@ RSpec.describe Backup::Repository do end describe '#dump' do - describe 'repo failure' do - before do - allow(Gitlab::Popen).to receive(:popen).and_return(['normal output', 0]) + before do + allow(Gitlab.config.repositories.storages).to receive(:keys).and_return(storage_keys) + end + + let_it_be(:projects) { create_list(:project, 5, :wiki_repo) + [project] } + + let(:storage_keys) { %w[default test_second_storage] } + + context 'no concurrency' do + it 'creates the expected number of threads' do + expect(Thread).not_to receive(:new) + + projects.each do |project| + expect(subject).to receive(:dump_project).with(project).and_call_original + end + + subject.dump(max_concurrency: 1, max_storage_concurrency: 1) end - it 'does not raise error' do - expect { subject.dump }.not_to raise_error + describe 'command failure' do + it 'dump_project raises an error' do + allow(subject).to receive(:dump_project).and_raise(IOError) + + expect { subject.dump(max_concurrency: 1, max_storage_concurrency: 1) }.to raise_error(IOError) + end + + it 'project query raises an error' do + allow(Project).to receive(:find_each).and_raise(ActiveRecord::StatementTimeout) + + expect { subject.dump(max_concurrency: 1, max_storage_concurrency: 1) }.to raise_error(ActiveRecord::StatementTimeout) + end + end + end + + [4, 10].each do |max_storage_concurrency| + context "max_storage_concurrency #{max_storage_concurrency}" do + it 'creates the expected number of threads' do + expect(Thread).to receive(:new) + .exactly(storage_keys.length * (max_storage_concurrency + 1)).times + .and_call_original + + projects.each do |project| + expect(subject).to receive(:dump_project).with(project).and_call_original + end + + subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency) + end + + it 'creates the expected number of threads with extra max concurrency' do + expect(Thread).to receive(:new) + .exactly(storage_keys.length * (max_storage_concurrency + 1)).times + .and_call_original + + projects.each do |project| + expect(subject).to receive(:dump_project).with(project).and_call_original + end + + subject.dump(max_concurrency: 3, max_storage_concurrency: max_storage_concurrency) + end + + describe 'command failure' do + it 'dump_project raises an error' do + allow(subject).to receive(:dump_project) + .and_raise(IOError) + + expect { subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency) }.to raise_error(IOError) + end + + it 'project query raises an error' do + allow(Project).to receive_message_chain('for_repository_storage.find_each').and_raise(ActiveRecord::StatementTimeout) + + expect { subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency) }.to raise_error(ActiveRecord::StatementTimeout) + end + + context 'misconfigured storages' do + let(:storage_keys) { %w[test_second_storage] } + + it 'raises an error' do + expect { subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency) }.to raise_error(Backup::Error, 'repositories.storages in gitlab.yml is misconfigured') + end + end + end end end end diff --git a/spec/lib/banzai/filter/absolute_link_filter_spec.rb b/spec/lib/banzai/filter/absolute_link_filter_spec.rb index 2cb70850dca..0c159e8bac8 100644 --- a/spec/lib/banzai/filter/absolute_link_filter_spec.rb +++ b/spec/lib/banzai/filter/absolute_link_filter_spec.rb @@ -12,6 +12,7 @@ RSpec.describe Banzai::Filter::AbsoluteLinkFilter do let(:only_path_context) do { only_path: false } end + let(:fake_url) { 'http://www.example.com' } before do diff --git a/spec/lib/banzai/filter/ascii_doc_post_processing_filter_spec.rb b/spec/lib/banzai/filter/ascii_doc_post_processing_filter_spec.rb index 334d5c59828..7af22ea7db1 100644 --- a/spec/lib/banzai/filter/ascii_doc_post_processing_filter_spec.rb +++ b/spec/lib/banzai/filter/ascii_doc_post_processing_filter_spec.rb @@ -10,6 +10,12 @@ RSpec.describe Banzai::Filter::AsciiDocPostProcessingFilter do expect(result).to eq('<pre data-math-style="inline" class="code math js-render-math">some code</pre><div data-math>and</div>') end + it "adds class for elements with data-mermaid-style" do + result = filter('<pre data-mermaid-style="display">some code</pre>').to_html + + expect(result).to eq('<pre data-mermaid-style="display" class="js-render-mermaid">some code</pre>') + end + it "keeps content when no data-math-style found" do result = filter('<pre>some code</pre><div data-math>and</div>').to_html expect(result).to eq('<pre>some code</pre><div data-math>and</div>') diff --git a/spec/lib/banzai/filter/gollum_tags_filter_spec.rb b/spec/lib/banzai/filter/gollum_tags_filter_spec.rb index 2576dd1bf07..f39b5280490 100644 --- a/spec/lib/banzai/filter/gollum_tags_filter_spec.rb +++ b/spec/lib/banzai/filter/gollum_tags_filter_spec.rb @@ -6,8 +6,7 @@ RSpec.describe Banzai::Filter::GollumTagsFilter do include FilterSpecHelper let(:project) { create(:project) } - let(:user) { double } - let(:wiki) { ProjectWiki.new(project, user) } + let(:wiki) { ProjectWiki.new(project, nil) } describe 'validation' do it 'ensure that a :wiki key exists in context' do diff --git a/spec/lib/banzai/filter/inline_alert_metrics_filter_spec.rb b/spec/lib/banzai/filter/inline_alert_metrics_filter_spec.rb new file mode 100644 index 00000000000..be40195f001 --- /dev/null +++ b/spec/lib/banzai/filter/inline_alert_metrics_filter_spec.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Banzai::Filter::InlineAlertMetricsFilter do + include FilterSpecHelper + + let(:params) { ['foo', 'bar', 12] } + let(:query_params) { {} } + + let(:trigger_url) { urls.metrics_dashboard_namespace_project_prometheus_alert_url(*params, query_params) } + let(:dashboard_url) { urls.metrics_dashboard_namespace_project_prometheus_alert_url(*params, **query_params, embedded: true, format: :json) } + + it_behaves_like 'a metrics embed filter' + + context 'with query params specified' do + let(:query_params) { { timestamp: 'yesterday' } } + + it_behaves_like 'a metrics embed filter' + end +end diff --git a/spec/lib/banzai/filter/inline_metrics_redactor_filter_spec.rb b/spec/lib/banzai/filter/inline_metrics_redactor_filter_spec.rb index cafcaef8ae2..5f66844f498 100644 --- a/spec/lib/banzai/filter/inline_metrics_redactor_filter_spec.rb +++ b/spec/lib/banzai/filter/inline_metrics_redactor_filter_spec.rb @@ -74,5 +74,20 @@ RSpec.describe Banzai::Filter::InlineMetricsRedactorFilter do end end end + + context 'for an alert embed' do + let_it_be(:alert) { create(:prometheus_alert, project: project) } + let(:url) do + urls.metrics_dashboard_project_prometheus_alert_url( + project, + alert.prometheus_metric_id, + environment_id: alert.environment_id, + embedded: true + ) + end + + it_behaves_like 'redacts the embed placeholder' + it_behaves_like 'retains the embed placeholder when applicable' + end end end diff --git a/spec/lib/banzai/filter/label_reference_filter_spec.rb b/spec/lib/banzai/filter/label_reference_filter_spec.rb index dadf98d9b76..726ef8c57ab 100644 --- a/spec/lib/banzai/filter/label_reference_filter_spec.rb +++ b/spec/lib/banzai/filter/label_reference_filter_spec.rb @@ -31,6 +31,19 @@ RSpec.describe Banzai::Filter::LabelReferenceFilter do expect(doc.css('a').first.attr('class')).to eq 'gfm gfm-label has-tooltip gl-link gl-label-link' end + it 'avoids N+1 cached queries', :use_sql_query_cache, :request_store do + # Run this once to establish a baseline + reference_filter("Label #{reference}") + + control_count = ActiveRecord::QueryRecorder.new(skip_cached: false) do + reference_filter("Label #{reference}") + end + + labels_markdown = Array.new(10, "Label #{reference}").join('\n') + + expect { reference_filter(labels_markdown) }.not_to exceed_all_query_limit(control_count.count) + end + it 'includes a data-project attribute' do doc = reference_filter("Label #{reference}") link = doc.css('a').first diff --git a/spec/lib/banzai/filter/merge_request_reference_filter_spec.rb b/spec/lib/banzai/filter/merge_request_reference_filter_spec.rb index f24fcf98b1f..df78a3321ba 100644 --- a/spec/lib/banzai/filter/merge_request_reference_filter_spec.rb +++ b/spec/lib/banzai/filter/merge_request_reference_filter_spec.rb @@ -220,6 +220,7 @@ RSpec.describe Banzai::Filter::MergeRequestReferenceFilter do let(:reference) do urls.project_merge_request_url(mr.project, mr) + "/diffs?commit_id=#{mr.diff_head_sha}" end + let(:commit) { mr.commits.find { |commit| commit.sha == mr.diff_head_sha } } it 'links to a valid reference' do diff --git a/spec/lib/banzai/filter/reference_filter_spec.rb b/spec/lib/banzai/filter/reference_filter_spec.rb index d5978db13c0..2888965dbc4 100644 --- a/spec/lib/banzai/filter/reference_filter_spec.rb +++ b/spec/lib/banzai/filter/reference_filter_spec.rb @@ -110,20 +110,6 @@ RSpec.describe Banzai::Filter::ReferenceFilter do expect(filter.instance_variable_get(:@new_nodes)).to eq({ index => [filter.each_node.to_a[index]] }) end - - context "with update_nodes_for_banzai_reference_filter feature flag disabled" do - before do - stub_feature_flags(update_nodes_for_banzai_reference_filter: false) - end - - it 'does not call replace_and_update_new_nodes' do - expect(filter).not_to receive(:replace_and_update_new_nodes).with(filter.nodes[index], index, html) - - filter.send(method_name, *args) do - html - end - end - end end end @@ -198,49 +184,20 @@ RSpec.describe Banzai::Filter::ReferenceFilter do end describe "#call_and_update_nodes" do - context "with update_nodes_for_banzai_reference_filter feature flag enabled" do - include_context 'new nodes' - let(:document) { Nokogiri::HTML.fragment('<a href="foo">foo</a>') } - let(:filter) { described_class.new(document, project: project) } - - before do - stub_feature_flags(update_nodes_for_banzai_reference_filter: true) - end - - it "updates all new nodes", :aggregate_failures do - filter.instance_variable_set('@nodes', nodes) - - expect(filter).to receive(:call) { filter.instance_variable_set('@new_nodes', new_nodes) } - expect(filter).to receive(:with_update_nodes).and_call_original - expect(filter).to receive(:update_nodes!).and_call_original - - filter.call_and_update_nodes - - expect(filter.result[:reference_filter_nodes]).to eq(expected_nodes) - end - end - - context "with update_nodes_for_banzai_reference_filter feature flag disabled" do - include_context 'new nodes' - - before do - stub_feature_flags(update_nodes_for_banzai_reference_filter: false) - end + include_context 'new nodes' + let(:document) { Nokogiri::HTML.fragment('<a href="foo">foo</a>') } + let(:filter) { described_class.new(document, project: project) } - it "does not change nodes", :aggregate_failures do - document = Nokogiri::HTML.fragment('<a href="foo">foo</a>') - filter = described_class.new(document, project: project) - filter.instance_variable_set('@nodes', nodes) + it "updates all new nodes", :aggregate_failures do + filter.instance_variable_set('@nodes', nodes) - expect(filter).to receive(:call) { filter.instance_variable_set('@new_nodes', new_nodes) } - expect(filter).not_to receive(:with_update_nodes) - expect(filter).not_to receive(:update_nodes!) + expect(filter).to receive(:call) { filter.instance_variable_set('@new_nodes', new_nodes) } + expect(filter).to receive(:with_update_nodes).and_call_original + expect(filter).to receive(:update_nodes!).and_call_original - filter.call_and_update_nodes + filter.call_and_update_nodes - expect(filter.nodes).to eq(nodes) - expect(filter.result[:reference_filter_nodes]).to be nil - end + expect(filter.result[:reference_filter_nodes]).to eq(expected_nodes) end end @@ -251,10 +208,6 @@ RSpec.describe Banzai::Filter::ReferenceFilter do let(:result) { { reference_filter_nodes: nodes } } - before do - stub_feature_flags(update_nodes_for_banzai_reference_filter: true) - end - it "updates all nodes", :aggregate_failures do expect_next_instance_of(described_class) do |filter| expect(filter).to receive(:call_and_update_nodes).and_call_original @@ -267,26 +220,5 @@ RSpec.describe Banzai::Filter::ReferenceFilter do expect(result[:reference_filter_nodes]).to eq(expected_nodes) end - - context "with update_nodes_for_banzai_reference_filter feature flag disabled" do - let(:result) { {} } - - before do - stub_feature_flags(update_nodes_for_banzai_reference_filter: false) - end - - it "updates all nodes", :aggregate_failures do - expect_next_instance_of(described_class) do |filter| - expect(filter).to receive(:call_and_update_nodes).and_call_original - expect(filter).not_to receive(:with_update_nodes) - expect(filter).to receive(:call) { filter.instance_variable_set('@new_nodes', new_nodes) } - expect(filter).not_to receive(:update_nodes!) - end - - described_class.call(document, { project: project }, result) - - expect(result[:reference_filter_nodes]).to be nil - end - end end end diff --git a/spec/lib/banzai/filter/syntax_highlight_filter_spec.rb b/spec/lib/banzai/filter/syntax_highlight_filter_spec.rb index a2875fad421..78f84ee44f7 100644 --- a/spec/lib/banzai/filter/syntax_highlight_filter_spec.rb +++ b/spec/lib/banzai/filter/syntax_highlight_filter_spec.rb @@ -26,6 +26,14 @@ RSpec.describe Banzai::Filter::SyntaxHighlightFilter do include_examples "XSS prevention", "" end + context "when contains mermaid diagrams" do + it "ignores mermaid blocks" do + result = filter('<pre data-mermaid-style="display"><code>mermaid code</code></pre>') + + expect(result.to_html).to eq('<pre data-mermaid-style="display"><code>mermaid code</code></pre>') + end + end + context "when a valid language is specified" do it "highlights as that language" do result = filter('<pre><code lang="ruby">def fun end</code></pre>') diff --git a/spec/lib/banzai/filter/wiki_link_filter_spec.rb b/spec/lib/banzai/filter/wiki_link_filter_spec.rb index 7a4464a2604..d1f6ee49260 100644 --- a/spec/lib/banzai/filter/wiki_link_filter_spec.rb +++ b/spec/lib/banzai/filter/wiki_link_filter_spec.rb @@ -7,8 +7,7 @@ RSpec.describe Banzai::Filter::WikiLinkFilter do let(:namespace) { build_stubbed(:namespace, name: "wiki_link_ns") } let(:project) { build_stubbed(:project, :public, name: "wiki_link_project", namespace: namespace) } - let(:user) { double } - let(:wiki) { ProjectWiki.new(project, user) } + let(:wiki) { ProjectWiki.new(project, nil) } let(:repository_upload_folder) { Wikis::CreateAttachmentService::ATTACHMENT_PATH } it "doesn't rewrite absolute links" do diff --git a/spec/lib/banzai/issuable_extractor_spec.rb b/spec/lib/banzai/issuable_extractor_spec.rb index c4ee7160e12..8fec9691d7f 100644 --- a/spec/lib/banzai/issuable_extractor_spec.rb +++ b/spec/lib/banzai/issuable_extractor_spec.rb @@ -13,6 +13,7 @@ RSpec.describe Banzai::IssuableExtractor do "<a href='' data-issue='#{issue.id}' data-reference-type='issue' class='gfm'>text</a>" ) end + let(:merge_request_link) do html_to_node( "<a href='' data-merge-request='#{merge_request.id}' data-reference-type='merge_request' class='gfm'>text</a>" diff --git a/spec/lib/banzai/object_renderer_spec.rb b/spec/lib/banzai/object_renderer_spec.rb index f8d7acd3148..e64ab5dfce3 100644 --- a/spec/lib/banzai/object_renderer_spec.rb +++ b/spec/lib/banzai/object_renderer_spec.rb @@ -73,6 +73,7 @@ RSpec.describe Banzai::ObjectRenderer do end end end + let(:cacheless_thing) do cacheless_class.new.tap do |thing| thing.title = "Merge branch 'branch-merged' into 'master'" diff --git a/spec/lib/banzai/pipeline/full_pipeline_spec.rb b/spec/lib/banzai/pipeline/full_pipeline_spec.rb index 0127ac11c81..9391ca386cf 100644 --- a/spec/lib/banzai/pipeline/full_pipeline_spec.rb +++ b/spec/lib/banzai/pipeline/full_pipeline_spec.rb @@ -109,6 +109,7 @@ RSpec.describe Banzai::Pipeline::FullPipeline do # Header MARKDOWN end + let(:invalid_markdown) do <<-MARKDOWN.strip_heredoc test [[_TOC_]] diff --git a/spec/lib/banzai/pipeline/gfm_pipeline_spec.rb b/spec/lib/banzai/pipeline/gfm_pipeline_spec.rb index beb760637b0..247f4591632 100644 --- a/spec/lib/banzai/pipeline/gfm_pipeline_spec.rb +++ b/spec/lib/banzai/pipeline/gfm_pipeline_spec.rb @@ -30,34 +30,6 @@ RSpec.describe Banzai::Pipeline::GfmPipeline do described_class.call(markdown, project: project) end - context "with update_nodes_for_banzai_reference_filter feature flag disabled" do - before do - stub_feature_flags(update_nodes_for_banzai_reference_filter: false) - end - - context 'when shorthand pattern #ISSUE_ID is used' do - it 'links an internal issues and doesnt store nodes in result[:reference_filter_nodes]', :aggregate_failures do - issue = create(:issue, project: project) - markdown = "text #{issue.to_reference(project, full: true)}" - result = described_class.call(markdown, project: project) - link = result[:output].css('a').first - - expect(link['href']).to eq(Gitlab::Routing.url_helpers.project_issue_path(project, issue)) - expect(result[:reference_filter_nodes]).to eq nil - end - end - - it 'execute :each_node for each reference_filter', :aggregate_failures do - issue = create(:issue, project: project) - markdown = "text #{issue.to_reference(project, full: true)}" - described_class.reference_filters do |reference_filter| - expect_any_instance_of(reference_filter).to receive(:each_node).once - end - - described_class.call(markdown, project: project) - end - end - context 'when shorthand pattern #ISSUE_ID is used' do it 'links an internal issue if it exists' do issue = create(:issue, project: project) diff --git a/spec/lib/banzai/pipeline/wiki_pipeline_spec.rb b/spec/lib/banzai/pipeline/wiki_pipeline_spec.rb index 4af782c7d73..b102de24041 100644 --- a/spec/lib/banzai/pipeline/wiki_pipeline_spec.rb +++ b/spec/lib/banzai/pipeline/wiki_pipeline_spec.rb @@ -5,7 +5,7 @@ require 'spec_helper' RSpec.describe Banzai::Pipeline::WikiPipeline do let_it_be(:namespace) { create(:namespace, name: "wiki_link_ns") } let_it_be(:project) { create(:project, :public, name: "wiki_link_project", namespace: namespace) } - let_it_be(:wiki) { ProjectWiki.new(project, double(:user)) } + let_it_be(:wiki) { ProjectWiki.new(project, nil) } let_it_be(:page) { build(:wiki_page, wiki: wiki, title: 'nested/twice/start-page') } describe 'TableOfContents' do diff --git a/spec/lib/banzai/reference_parser/base_parser_spec.rb b/spec/lib/banzai/reference_parser/base_parser_spec.rb index 0eea51262ba..5ab76b2c68b 100644 --- a/spec/lib/banzai/reference_parser/base_parser_spec.rb +++ b/spec/lib/banzai/reference_parser/base_parser_spec.rb @@ -8,13 +8,14 @@ RSpec.describe Banzai::ReferenceParser::BaseParser do let(:user) { create(:user) } let(:project) { create(:project, :public) } let(:context) { Banzai::RenderContext.new(project, user) } - - subject do - klass = Class.new(described_class) do + let(:parser_class) do + Class.new(described_class) do self.reference_type = :foo end + end - klass.new(context) + subject do + parser_class.new(context) end describe '.reference_type=' do @@ -43,12 +44,20 @@ RSpec.describe Banzai::ReferenceParser::BaseParser do let(:link) { empty_html_link } context 'when the link has a data-project attribute' do - it 'checks if user can read the resource' do + before do link['data-project'] = project.id.to_s + end - expect(subject).to receive(:can_read_reference?).with(user, project, link) + it 'includes the link if can_read_reference? returns true' do + expect(subject).to receive(:can_read_reference?).with(user, project, link).and_return(true) - subject.nodes_visible_to_user(user, [link]) + expect(subject.nodes_visible_to_user(user, [link])).to contain_exactly(link) + end + + it 'excludes the link if can_read_reference? returns false' do + expect(subject).to receive(:can_read_reference?).with(user, project, link).and_return(false) + + expect(subject.nodes_visible_to_user(user, [link])).to be_empty end end @@ -178,58 +187,56 @@ RSpec.describe Banzai::ReferenceParser::BaseParser do it 'gathers the references for every node matching the reference type' do dummy = Class.new(described_class) do self.reference_type = :test - end - - instance = dummy.new(Banzai::RenderContext.new(project, user)) - document = Nokogiri::HTML.fragment('<a class="gfm"></a><a class="gfm" data-reference-type="test"></a>') - expect(instance).to receive(:gather_references) - .with([document.children[1]]) - .and_return([user]) + def gather_references(nodes) + nodes + end + end - expect(instance.process([document])).to eq([user]) + instance = dummy.new(context) + document_a = Nokogiri::HTML.fragment(<<-FRAG) + <a class="gfm">one</a> + <a class="gfm" data-reference-type="test">two</a> + <a class="gfm" data-reference-type="other">three</a> + FRAG + document_b = Nokogiri::HTML.fragment(<<-FRAG) + <a class="gfm" data-reference-type="test">four</a> + FRAG + document_c = Nokogiri::HTML.fragment('') + + expect(instance.process([document_a, document_b, document_c])) + .to contain_exactly(document_a.css('a')[1], document_b.css('a')[0]) end end describe '#gather_references' do - let(:link) { double(:link) } + let(:nodes) { (1..10).map { |n| double(:link, id: n) } } - it 'does not process links a user can not reference' do - expect(subject).to receive(:nodes_user_can_reference) - .with(user, [link]) - .and_return([]) + let(:parser_class) do + Class.new(described_class) do + def nodes_user_can_reference(_user, nodes) + nodes.select { |n| n.id.even? } + end - expect(subject).to receive(:referenced_by).with([]) + def nodes_visible_to_user(_user, nodes) + nodes.select { |n| n.id > 5 } + end - subject.gather_references([link]) + def referenced_by(nodes) + nodes.map(&:id) + end + end end - it 'does not process links a user can not see' do - expect(subject).to receive(:nodes_user_can_reference) - .with(user, [link]) - .and_return([link]) - - expect(subject).to receive(:nodes_visible_to_user) - .with(user, [link]) - .and_return([]) - - expect(subject).to receive(:referenced_by).with([]) - - subject.gather_references([link]) + it 'returns referenceable and visible objects, alongside nodes that are referenceable but not visible' do + expect(subject.gather_references(nodes)).to match( + visible: contain_exactly(6, 8, 10), + not_visible: match_array(nodes.select { |n| n.id.even? && n.id <= 5 }) + ) end - it 'returns the references if a user can reference and see a link' do - expect(subject).to receive(:nodes_user_can_reference) - .with(user, [link]) - .and_return([link]) - - expect(subject).to receive(:nodes_visible_to_user) - .with(user, [link]) - .and_return([link]) - - expect(subject).to receive(:referenced_by).with([link]) - - subject.gather_references([link]) + it 'is always empty if the input is empty' do + expect(subject.gather_references([])) .to match(visible: be_empty, not_visible: be_empty) end end diff --git a/spec/lib/banzai/reference_parser/snippet_parser_spec.rb b/spec/lib/banzai/reference_parser/snippet_parser_spec.rb index cdc660b4f4a..3459784708f 100644 --- a/spec/lib/banzai/reference_parser/snippet_parser_spec.rb +++ b/spec/lib/banzai/reference_parser/snippet_parser_spec.rb @@ -33,6 +33,17 @@ RSpec.describe Banzai::ReferenceParser::SnippetParser do project.project_feature.update_attribute(:snippets_access_level, ProjectFeature::ENABLED) end + it 'avoids N+1 cached queries', :use_sql_query_cache do + # Run this once to establish a baseline + visible_references(:public) + + control_count = ActiveRecord::QueryRecorder.new(skip_cached: false) do + subject.nodes_visible_to_user(user, [link]) + end + + expect { subject.nodes_visible_to_user(user, Array.new(10, link)) }.not_to exceed_all_query_limit(control_count.count) + end + it 'creates a reference for guest for a public snippet' do expect(visible_references(:public)).to eq([link]) end diff --git a/spec/lib/container_registry/client_spec.rb b/spec/lib/container_registry/client_spec.rb index de92ca5eeec..aa947329c33 100644 --- a/spec/lib/container_registry/client_spec.rb +++ b/spec/lib/container_registry/client_spec.rb @@ -14,6 +14,7 @@ RSpec.describe ContainerRegistry::Client do 'User-Agent' => "GitLab/#{Gitlab::VERSION}" } end + let(:headers_with_accept_types) do { 'Accept' => 'application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json', diff --git a/spec/lib/gitlab/alert_management/alert_params_spec.rb b/spec/lib/gitlab/alert_management/alert_params_spec.rb index 393838ab042..1fe27365c83 100644 --- a/spec/lib/gitlab/alert_management/alert_params_spec.rb +++ b/spec/lib/gitlab/alert_management/alert_params_spec.rb @@ -18,6 +18,7 @@ RSpec.describe Gitlab::AlertManagement::AlertParams do 'some' => { 'extra' => { 'payload' => 'here' } } } end + let(:payload) { default_payload } subject { described_class.from_generic_alert(project: project, payload: payload) } @@ -75,6 +76,7 @@ RSpec.describe Gitlab::AlertManagement::AlertParams do 'fingerprint' => 'b6ac4d42057c43c1' } end + let(:parsed_alert) { Gitlab::Alerting::Alert.new(project: project, payload: payload) } subject { described_class.from_prometheus_alert(project: project, parsed_alert: parsed_alert) } diff --git a/spec/lib/gitlab/alert_management/alert_status_counts_spec.rb b/spec/lib/gitlab/alert_management/alert_status_counts_spec.rb index 4e471a8eac0..a2b8f0aa8d4 100644 --- a/spec/lib/gitlab/alert_management/alert_status_counts_spec.rb +++ b/spec/lib/gitlab/alert_management/alert_status_counts_spec.rb @@ -5,9 +5,9 @@ require 'spec_helper' RSpec.describe Gitlab::AlertManagement::AlertStatusCounts do let_it_be(:current_user) { create(:user) } let_it_be(:project) { create(:project) } - let_it_be(:alert_1) { create(:alert_management_alert, :resolved, project: project) } - let_it_be(:alert_2) { create(:alert_management_alert, :ignored, project: project) } - let_it_be(:alert_3) { create(:alert_management_alert) } + let_it_be(:alert_resolved) { create(:alert_management_alert, :resolved, project: project) } + let_it_be(:alert_ignored) { create(:alert_management_alert, :ignored, project: project) } + let_it_be(:alert_triggered) { create(:alert_management_alert) } let(:params) { {} } describe '#execute' do @@ -52,7 +52,7 @@ RSpec.describe Gitlab::AlertManagement::AlertStatusCounts do end context 'when search param is included' do - let(:params) { { search: alert_1.title } } + let(:params) { { search: alert_resolved.title } } it 'returns the correct countss' do expect(counts.open).to eq(0) diff --git a/spec/lib/gitlab/alerting/alert_spec.rb b/spec/lib/gitlab/alerting/alert_spec.rb index 9663e6af0d2..b53b71e3f3e 100644 --- a/spec/lib/gitlab/alerting/alert_spec.rb +++ b/spec/lib/gitlab/alerting/alert_spec.rb @@ -191,7 +191,7 @@ RSpec.describe Gitlab::Alerting::Alert do end context 'with payload' do - let(:time) { Time.now.change(usec: 0) } + let(:time) { Time.current.change(usec: 0) } before do payload['startsAt'] = time.rfc3339 @@ -274,7 +274,7 @@ RSpec.describe Gitlab::Alerting::Alert do before do payload.update( 'annotations' => { 'title' => 'some title' }, - 'startsAt' => Time.now.rfc3339 + 'startsAt' => Time.current.rfc3339 ) end diff --git a/spec/lib/gitlab/analytics/unique_visits_spec.rb b/spec/lib/gitlab/analytics/unique_visits_spec.rb index ff3623a3a71..1432c9ac58f 100644 --- a/spec/lib/gitlab/analytics/unique_visits_spec.rb +++ b/spec/lib/gitlab/analytics/unique_visits_spec.rb @@ -7,8 +7,11 @@ RSpec.describe Gitlab::Analytics::UniqueVisits, :clean_gitlab_redis_shared_state let(:target1_id) { 'g_analytics_contribution' } let(:target2_id) { 'g_analytics_insights' } let(:target3_id) { 'g_analytics_issues' } + let(:target4_id) { 'g_compliance_dashboard' } + let(:target5_id) { 'i_compliance_credential_inventory' } let(:visitor1_id) { 'dfb9d2d2-f56c-4c77-8aeb-6cddc4a1f857' } let(:visitor2_id) { '1dd9afb2-a3ee-4de1-8ae3-a405579c8584' } + let(:visitor3_id) { '34rfjuuy-ce56-sa35-ds34-dfer567dfrf2' } around do |example| # We need to freeze to a reference time @@ -29,24 +32,40 @@ RSpec.describe Gitlab::Analytics::UniqueVisits, :clean_gitlab_redis_shared_state unique_visits.track_visit(visitor1_id, target2_id, 8.days.ago) unique_visits.track_visit(visitor1_id, target2_id, 15.days.ago) - expect(unique_visits.weekly_unique_visits_for_target(target1_id)).to eq(2) - expect(unique_visits.weekly_unique_visits_for_target(target2_id)).to eq(1) + unique_visits.track_visit(visitor3_id, target4_id, 7.days.ago) - expect(unique_visits.weekly_unique_visits_for_target(target2_id, week_of: 15.days.ago)).to eq(1) + unique_visits.track_visit(visitor3_id, target5_id, 15.days.ago) + unique_visits.track_visit(visitor2_id, target5_id, 15.days.ago) - expect(unique_visits.weekly_unique_visits_for_target(target3_id)).to eq(0) + expect(unique_visits.unique_visits_for(targets: target1_id)).to eq(2) + expect(unique_visits.unique_visits_for(targets: target2_id)).to eq(1) + expect(unique_visits.unique_visits_for(targets: target4_id)).to eq(1) - expect(unique_visits.weekly_unique_visits_for_any_target).to eq(2) - expect(unique_visits.weekly_unique_visits_for_any_target(week_of: 15.days.ago)).to eq(1) - expect(unique_visits.weekly_unique_visits_for_any_target(week_of: 30.days.ago)).to eq(0) + expect(unique_visits.unique_visits_for(targets: target2_id, start_date: 15.days.ago)).to eq(1) + + expect(unique_visits.unique_visits_for(targets: target3_id)).to eq(0) + + expect(unique_visits.unique_visits_for(targets: target5_id, start_date: 15.days.ago)).to eq(2) + + expect(unique_visits.unique_visits_for(targets: :analytics)).to eq(2) + expect(unique_visits.unique_visits_for(targets: :analytics, start_date: 15.days.ago)).to eq(1) + expect(unique_visits.unique_visits_for(targets: :analytics, start_date: 30.days.ago)).to eq(0) + + expect(unique_visits.unique_visits_for(targets: :analytics, start_date: 4.weeks.ago, end_date: Date.current)).to eq(2) + + expect(unique_visits.unique_visits_for(targets: :compliance)).to eq(1) + expect(unique_visits.unique_visits_for(targets: :compliance, start_date: 15.days.ago)).to eq(2) + expect(unique_visits.unique_visits_for(targets: :compliance, start_date: 30.days.ago)).to eq(0) + + expect(unique_visits.unique_visits_for(targets: :compliance, start_date: 4.weeks.ago, end_date: Date.current)).to eq(2) end - it 'sets the keys in Redis to expire automatically after 28 days' do + it 'sets the keys in Redis to expire automatically after 12 weeks' do unique_visits.track_visit(visitor1_id, target1_id) Gitlab::Redis::SharedState.with do |redis| - redis.scan_each(match: "#{target1_id}-*").each do |key| - expect(redis.ttl(key)).to be_within(5.seconds).of(28.days) + redis.scan_each(match: "{#{target1_id}}-*").each do |key| + expect(redis.ttl(key)).to be_within(5.seconds).of(12.weeks) end end end @@ -56,7 +75,7 @@ RSpec.describe Gitlab::Analytics::UniqueVisits, :clean_gitlab_redis_shared_state expect do unique_visits.track_visit(visitor1_id, invalid_target_id) - end.to raise_error("Invalid target id #{invalid_target_id}") + end.to raise_error(Gitlab::UsageDataCounters::HLLRedisCounter::UnknownEvent) end end end diff --git a/spec/lib/gitlab/app_logger_spec.rb b/spec/lib/gitlab/app_logger_spec.rb index 166b1fda268..23bac444dbe 100644 --- a/spec/lib/gitlab/app_logger_spec.rb +++ b/spec/lib/gitlab/app_logger_spec.rb @@ -19,4 +19,12 @@ RSpec.describe Gitlab::AppLogger do subject.info('Hello World!') end + + it 'logs info to only the AppJsonLogger when unstructured logs are disabled' do + stub_env('UNSTRUCTURED_RAILS_LOG', 'false') + expect_any_instance_of(Gitlab::AppTextLogger).not_to receive(:info).and_call_original + expect_any_instance_of(Gitlab::AppJsonLogger).to receive(:info).and_call_original + + subject.info('Hello World!') + end end diff --git a/spec/lib/gitlab/application_rate_limiter_spec.rb b/spec/lib/gitlab/application_rate_limiter_spec.rb index 14a7e25a2e8..2525b1ce41e 100644 --- a/spec/lib/gitlab/application_rate_limiter_spec.rb +++ b/spec/lib/gitlab/application_rate_limiter_spec.rb @@ -14,6 +14,7 @@ RSpec.describe Gitlab::ApplicationRateLimiter, :clean_gitlab_redis_cache do } } end + let(:key) { rate_limits.keys[0] } subject { described_class } diff --git a/spec/lib/gitlab/asciidoc/include_processor_spec.rb b/spec/lib/gitlab/asciidoc/include_processor_spec.rb index 067dcefb525..5c225575965 100644 --- a/spec/lib/gitlab/asciidoc/include_processor_spec.rb +++ b/spec/lib/gitlab/asciidoc/include_processor_spec.rb @@ -13,6 +13,7 @@ RSpec.describe Gitlab::Asciidoc::IncludeProcessor do ref: ref } end + let(:ref) { project.repository.root_ref } let(:max_includes) { 10 } diff --git a/spec/lib/gitlab/asciidoc_spec.rb b/spec/lib/gitlab/asciidoc_spec.rb index 40a4ab3e173..1b669e691e7 100644 --- a/spec/lib/gitlab/asciidoc_spec.rb +++ b/spec/lib/gitlab/asciidoc_spec.rb @@ -418,6 +418,50 @@ module Gitlab expect(output).to include("a href=\"README.adoc\"") end end + + context 'with mermaid diagrams' do + it 'adds class js-render-mermaid to the output' do + input = <<~MD + [mermaid] + .... + graph LR + A[Square Rect] -- Link text --> B((Circle)) + A --> C(Round Rect) + B --> D{Rhombus} + C --> D + .... + MD + + output = <<~HTML + <pre data-mermaid-style="display" class="js-render-mermaid">graph LR + A[Square Rect] -- Link text --> B((Circle)) + A --> C(Round Rect) + B --> D{Rhombus} + C --> D</pre> + HTML + + expect(render(input, context)).to include(output.strip) + end + + it 'applies subs in diagram block' do + input = <<~MD + :class-name: AveryLongClass + + [mermaid,subs=+attributes] + .... + classDiagram + Class01 <|-- {class-name} : Cool + .... + MD + + output = <<~HTML + <pre data-mermaid-style="display" class="js-render-mermaid">classDiagram + Class01 <|-- AveryLongClass : Cool</pre> + HTML + + expect(render(input, context)).to include(output.strip) + end + end end context 'with project' do @@ -429,6 +473,7 @@ module Gitlab requested_path: requested_path } end + let(:commit) { project.commit(ref) } let(:project) { create(:project, :repository) } let(:ref) { 'asciidoc' } diff --git a/spec/lib/gitlab/audit/null_author_spec.rb b/spec/lib/gitlab/audit/null_author_spec.rb new file mode 100644 index 00000000000..eb80e5faa89 --- /dev/null +++ b/spec/lib/gitlab/audit/null_author_spec.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Audit::NullAuthor do + subject { described_class } + + describe '.for' do + it 'returns an DeletedAuthor' do + expect(subject.for(666, 'Old Hat')).to be_a(Gitlab::Audit::DeletedAuthor) + end + + it 'returns an UnauthenticatedAuthor when id equals -1', :aggregate_failures do + expect(subject.for(-1, 'Frank')).to be_a(Gitlab::Audit::UnauthenticatedAuthor) + expect(subject.for(-1, 'Frank')).to have_attributes(id: -1, name: 'Frank') + end + end + + describe '#current_sign_in_ip' do + it { expect(subject.new(id: 888, name: 'Guest').current_sign_in_ip).to be_nil } + end +end diff --git a/spec/lib/gitlab/audit/unauthenticated_author_spec.rb b/spec/lib/gitlab/audit/unauthenticated_author_spec.rb new file mode 100644 index 00000000000..4e5c477fc2a --- /dev/null +++ b/spec/lib/gitlab/audit/unauthenticated_author_spec.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Audit::UnauthenticatedAuthor do + describe '#initialize' do + it 'sets correct attributes' do + expect(described_class.new(name: 'Peppa Pig')) + .to have_attributes(id: -1, name: 'Peppa Pig') + end + + it 'sets default name when it is not provided' do + expect(described_class.new) + .to have_attributes(id: -1, name: 'An unauthenticated user') + end + end +end diff --git a/spec/lib/gitlab/auth/auth_finders_spec.rb b/spec/lib/gitlab/auth/auth_finders_spec.rb index d0f5d0a9b35..a73ac0b34af 100644 --- a/spec/lib/gitlab/auth/auth_finders_spec.rb +++ b/spec/lib/gitlab/auth/auth_finders_spec.rb @@ -12,6 +12,7 @@ RSpec.describe Gitlab::Auth::AuthFinders do 'rack.input' => '' } end + let(:request) { ActionDispatch::Request.new(env) } def set_param(key, value) @@ -554,7 +555,7 @@ RSpec.describe Gitlab::Auth::AuthFinders do end context 'with CI username' do - let(:username) { ::Ci::Build::CI_REGISTRY_USER } + let(:username) { ::Gitlab::Auth::CI_JOB_USER } let(:user) { create(:user) } let(:build) { create(:ci_build, user: user) } @@ -727,7 +728,7 @@ RSpec.describe Gitlab::Auth::AuthFinders do context 'when the job token is provided via basic auth' do let(:route_authentication_setting) { { job_token_allowed: :basic_auth } } - let(:username) { Ci::Build::CI_REGISTRY_USER } + let(:username) { ::Gitlab::Auth::CI_JOB_USER } let(:token) { job.token } before do @@ -744,6 +745,56 @@ RSpec.describe Gitlab::Auth::AuthFinders do end end + describe '#cluster_agent_token_from_authorization_token' do + let_it_be(:agent_token) { create(:cluster_agent_token) } + + context 'when route_setting is empty' do + it 'returns nil' do + expect(cluster_agent_token_from_authorization_token).to be_nil + end + end + + context 'when route_setting allows cluster agent token' do + let(:route_authentication_setting) { { cluster_agent_token_allowed: true } } + + context 'Authorization header is empty' do + it 'returns nil' do + expect(cluster_agent_token_from_authorization_token).to be_nil + end + end + + context 'Authorization header is incorrect' do + before do + request.headers['Authorization'] = 'Bearer ABCD' + end + + it 'returns nil' do + expect(cluster_agent_token_from_authorization_token).to be_nil + end + end + + context 'Authorization header is malformed' do + before do + request.headers['Authorization'] = 'Bearer' + end + + it 'returns nil' do + expect(cluster_agent_token_from_authorization_token).to be_nil + end + end + + context 'Authorization header matches agent token' do + before do + request.headers['Authorization'] = "Bearer #{agent_token.token}" + end + + it 'returns the agent token' do + expect(cluster_agent_token_from_authorization_token).to eq(agent_token) + end + end + end + end + describe '#find_runner_from_token' do let(:runner) { create(:ci_runner) } diff --git a/spec/lib/gitlab/auth/ldap/user_spec.rb b/spec/lib/gitlab/auth/ldap/user_spec.rb index 7ca2878e583..ccaed94b5c8 100644 --- a/spec/lib/gitlab/auth/ldap/user_spec.rb +++ b/spec/lib/gitlab/auth/ldap/user_spec.rb @@ -14,9 +14,11 @@ RSpec.describe Gitlab::Auth::Ldap::User do nickname: 'john' } end + let(:auth_hash) do OmniAuth::AuthHash.new(uid: 'uid=John Smith,ou=People,dc=example,dc=com', provider: 'ldapmain', info: info) end + let(:ldap_user_upper_case) { described_class.new(auth_hash_upper_case) } let(:info_upper_case) do { @@ -25,6 +27,7 @@ RSpec.describe Gitlab::Auth::Ldap::User do nickname: 'john' } end + let(:auth_hash_upper_case) do OmniAuth::AuthHash.new(uid: 'uid=John Smith,ou=People,dc=example,dc=com', provider: 'ldapmain', info: info_upper_case) end diff --git a/spec/lib/gitlab/auth/o_auth/auth_hash_spec.rb b/spec/lib/gitlab/auth/o_auth/auth_hash_spec.rb index 7a60acca95b..67ffdee0c4a 100644 --- a/spec/lib/gitlab/auth/o_auth/auth_hash_spec.rb +++ b/spec/lib/gitlab/auth/o_auth/auth_hash_spec.rb @@ -17,6 +17,7 @@ RSpec.describe Gitlab::Auth::OAuth::AuthHash do let(:uid_raw) do +"CN=Onur K\xC3\xBC\xC3\xA7\xC3\xBCk,OU=Test,DC=example,DC=net" end + let(:email_raw) { +"onur.k\xC3\xBC\xC3\xA7\xC3\xBCk_ABC-123@example.net" } let(:nickname_raw) { +"ok\xC3\xBC\xC3\xA7\xC3\xBCk" } let(:first_name_raw) { +'Onur' } diff --git a/spec/lib/gitlab/auth/o_auth/user_spec.rb b/spec/lib/gitlab/auth/o_auth/user_spec.rb index ad04fddc675..12e774ec1f8 100644 --- a/spec/lib/gitlab/auth/o_auth/user_spec.rb +++ b/spec/lib/gitlab/auth/o_auth/user_spec.rb @@ -22,6 +22,7 @@ RSpec.describe Gitlab::Auth::OAuth::User do } } end + let(:ldap_user) { Gitlab::Auth::Ldap::Person.new(Net::LDAP::Entry.new, 'ldapmain') } describe '#persisted?' do @@ -193,6 +194,43 @@ RSpec.describe Gitlab::Auth::OAuth::User do end end + context "with auto_link_user disabled (default)" do + before do + stub_omniauth_config(auto_link_user: false) + end + + include_examples "to verify compliance with allow_single_sign_on" + end + + context "with auto_link_user enabled" do + before do + stub_omniauth_config(auto_link_user: true) + end + + context "and a current GitLab user with a matching email" do + let!(:existing_user) { create(:user, email: 'john@mail.com', username: 'john') } + + it "adds the OmniAuth identity to the GitLab user account" do + oauth_user.save + + expect(gl_user).to be_valid + expect(gl_user.username).to eql 'john' + expect(gl_user.email).to eql 'john@mail.com' + expect(gl_user.identities.length).to be 1 + identities_as_hash = gl_user.identities.map { |id| { provider: id.provider, extern_uid: id.extern_uid } } + expect(identities_as_hash).to match_array( + [ + { provider: 'twitter', extern_uid: uid } + ] + ) + end + end + + context "and no current GitLab user with a matching email" do + include_examples "to verify compliance with allow_single_sign_on" + end + end + context "with auto_link_ldap_user disabled (default)" do before do stub_omniauth_config(auto_link_ldap_user: false) @@ -229,39 +267,56 @@ RSpec.describe Gitlab::Auth::OAuth::User do end context "and no account for the LDAP user" do - before do - allow(Gitlab::Auth::Ldap::Person).to receive(:find_by_uid).and_return(ldap_user) + context 'when the LDAP user is found by UID' do + before do + allow(Gitlab::Auth::Ldap::Person).to receive(:find_by_uid).and_return(ldap_user) - oauth_user.save - end + oauth_user.save + end - it "creates a user with dual LDAP and omniauth identities" do - expect(gl_user).to be_valid - expect(gl_user.username).to eql uid - expect(gl_user.name).to eql 'John Doe' - expect(gl_user.email).to eql 'johndoe@example.com' - expect(gl_user.identities.length).to be 2 - identities_as_hash = gl_user.identities.map { |id| { provider: id.provider, extern_uid: id.extern_uid } } - expect(identities_as_hash).to match_array( - [ - { provider: 'ldapmain', extern_uid: dn }, - { provider: 'twitter', extern_uid: uid } - ] - ) - end + it "creates a user with dual LDAP and omniauth identities" do + expect(gl_user).to be_valid + expect(gl_user.username).to eql uid + expect(gl_user.name).to eql 'John Doe' + expect(gl_user.email).to eql 'johndoe@example.com' + expect(gl_user.identities.length).to be 2 + identities_as_hash = gl_user.identities.map { |id| { provider: id.provider, extern_uid: id.extern_uid } } + expect(identities_as_hash).to match_array( + [ + { provider: 'ldapmain', extern_uid: dn }, + { provider: 'twitter', extern_uid: uid } + ] + ) + end - it "has name and email set as synced" do - expect(gl_user.user_synced_attributes_metadata.name_synced).to be_truthy - expect(gl_user.user_synced_attributes_metadata.email_synced).to be_truthy - end + it "has name and email set as synced" do + expect(gl_user.user_synced_attributes_metadata.name_synced).to be_truthy + expect(gl_user.user_synced_attributes_metadata.email_synced).to be_truthy + end - it "has name and email set as read-only" do - expect(gl_user.read_only_attribute?(:name)).to be_truthy - expect(gl_user.read_only_attribute?(:email)).to be_truthy + it "has name and email set as read-only" do + expect(gl_user.read_only_attribute?(:name)).to be_truthy + expect(gl_user.read_only_attribute?(:email)).to be_truthy + end + + it "has synced attributes provider set to ldapmain" do + expect(gl_user.user_synced_attributes_metadata.provider).to eql 'ldapmain' + end end - it "has synced attributes provider set to ldapmain" do - expect(gl_user.user_synced_attributes_metadata.provider).to eql 'ldapmain' + context 'when the LDAP user is found by email address' do + before do + allow(Gitlab::Auth::Ldap::Person).to receive(:find_by_uid).and_return(nil) + allow(Gitlab::Auth::Ldap::Person).to receive(:find_by_email).with(uid, any_args).and_return(nil) + allow(Gitlab::Auth::Ldap::Person).to receive(:find_by_email).with(info_hash[:email], any_args).and_return(ldap_user) + + oauth_user.save + end + + it 'creates the LDAP identity' do + identities_as_hash = gl_user.identities.map { |id| { provider: id.provider, extern_uid: id.extern_uid } } + expect(identities_as_hash).to include({ provider: 'ldapmain', extern_uid: dn }) + end end end @@ -363,6 +418,90 @@ RSpec.describe Gitlab::Auth::OAuth::User do end end end + + context "with both auto_link_user and auto_link_ldap_user enabled" do + before do + stub_omniauth_config(auto_link_user: true, auto_link_ldap_user: true) + end + + context "and at least one LDAP provider is defined" do + before do + stub_ldap_config(providers: %w(ldapmain)) + end + + context "and a corresponding LDAP person" do + before do + allow(ldap_user).to receive_messages( + uid: uid, + username: uid, + name: 'John Doe', + email: ['john@mail.com'], + dn: dn + ) + end + + context "and no account for the LDAP user" do + before do + allow(Gitlab::Auth::Ldap::Person).to receive(:find_by_uid).and_return(ldap_user) + + oauth_user.save + end + + it "creates a user with dual LDAP and omniauth identities" do + expect(gl_user).to be_valid + expect(gl_user.username).to eql uid + expect(gl_user.name).to eql 'John Doe' + expect(gl_user.email).to eql 'john@mail.com' + expect(gl_user.identities.length).to be 2 + identities_as_hash = gl_user.identities.map { |id| { provider: id.provider, extern_uid: id.extern_uid } } + expect(identities_as_hash).to match_array( + [ + { provider: 'ldapmain', extern_uid: dn }, + { provider: 'twitter', extern_uid: uid } + ] + ) + end + + it "has name and email set as synced" do + expect(gl_user.user_synced_attributes_metadata.name_synced).to be_truthy + expect(gl_user.user_synced_attributes_metadata.email_synced).to be_truthy + end + + it "has name and email set as read-only" do + expect(gl_user.read_only_attribute?(:name)).to be_truthy + expect(gl_user.read_only_attribute?(:email)).to be_truthy + end + + it "has synced attributes provider set to ldapmain" do + expect(gl_user.user_synced_attributes_metadata.provider).to eql 'ldapmain' + end + end + + context "and LDAP user has an account already" do + let!(:existing_user) { create(:omniauth_user, name: 'John Doe', email: 'john@mail.com', extern_uid: dn, provider: 'ldapmain', username: 'john') } + + it "adds the omniauth identity to the LDAP account" do + allow(Gitlab::Auth::Ldap::Person).to receive(:find_by_uid).and_return(ldap_user) + + oauth_user.save + + expect(gl_user).to be_valid + expect(gl_user.username).to eql 'john' + expect(gl_user.name).to eql 'John Doe' + expect(gl_user.email).to eql 'john@mail.com' + expect(gl_user.identities.length).to be 2 + identities_as_hash = gl_user.identities.map { |id| { provider: id.provider, extern_uid: id.extern_uid } } + expect(identities_as_hash).to match_array( + [ + { provider: 'ldapmain', extern_uid: dn }, + { provider: 'twitter', extern_uid: uid } + ] + ) + end + end + end + end + end end describe 'blocking' do @@ -790,7 +929,7 @@ RSpec.describe Gitlab::Auth::OAuth::User do end end - describe '.find_by_uid_and_provider' do + describe '._uid_and_provider' do let!(:existing_user) { create(:omniauth_user, extern_uid: 'my-uid', provider: 'my-provider') } it 'normalizes extern_uid' do diff --git a/spec/lib/gitlab/auth/request_authenticator_spec.rb b/spec/lib/gitlab/auth/request_authenticator_spec.rb index 32d64519e2c..ef83321cc0e 100644 --- a/spec/lib/gitlab/auth/request_authenticator_spec.rb +++ b/spec/lib/gitlab/auth/request_authenticator_spec.rb @@ -9,6 +9,7 @@ RSpec.describe Gitlab::Auth::RequestAuthenticator do 'REQUEST_METHOD' => 'GET' } end + let(:request) { ActionDispatch::Request.new(env) } subject { described_class.new(request) } diff --git a/spec/lib/gitlab/auth/saml/user_spec.rb b/spec/lib/gitlab/auth/saml/user_spec.rb index 7f8346f0486..fd48492f18d 100644 --- a/spec/lib/gitlab/auth/saml/user_spec.rb +++ b/spec/lib/gitlab/auth/saml/user_spec.rb @@ -19,6 +19,7 @@ RSpec.describe Gitlab::Auth::Saml::User do email: 'john@mail.com' } end + let(:ldap_user) { Gitlab::Auth::Ldap::Person.new(Net::LDAP::Entry.new, 'ldapmain') } describe '#save' do @@ -194,6 +195,7 @@ RSpec.describe Gitlab::Auth::Saml::User do } } end + let(:auth_hash) { OmniAuth::AuthHash.new(auth_hash_base_attributes) } let(:uid_types) { %w(uid dn email) } diff --git a/spec/lib/gitlab/auth_spec.rb b/spec/lib/gitlab/auth_spec.rb index b62f9b55b64..dcaaa8d4188 100644 --- a/spec/lib/gitlab/auth_spec.rb +++ b/spec/lib/gitlab/auth_spec.rb @@ -149,7 +149,9 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do end context 'build token' do - subject { gl_auth.find_for_git_client('gitlab-ci-token', build.token, project: project, ip: 'ip') } + subject { gl_auth.find_for_git_client(username, build.token, project: project, ip: 'ip') } + + let(:username) { 'gitlab-ci-token' } context 'for running build' do let!(:build) { create(:ci_build, :running) } @@ -170,6 +172,14 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do expect(subject).to eq(Gitlab::Auth::Result.new(nil, nil, nil, nil)) end + + context 'username is not gitlab-ci-token' do + let(:username) { 'another_username' } + + it 'fails to authenticate' do + expect(subject).to eq(Gitlab::Auth::Result.new(nil, nil, nil, nil)) + end + end end (Ci::HasStatus::AVAILABLE_STATUSES - ['running']).each do |build_status| @@ -628,6 +638,7 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do password: password, password_confirmation: password) end + let(:username) { 'John' } # username isn't lowercase, test this let(:password) { 'my-secret' } diff --git a/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb b/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb deleted file mode 100644 index 7991ad69007..00000000000 --- a/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::ArchiveLegacyTraces do - include TraceHelpers - - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:builds) { table(:ci_builds) } - let(:job_artifacts) { table(:ci_job_artifacts) } - - before do - namespaces.create!(id: 123, name: 'gitlab1', path: 'gitlab1') - projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1', namespace_id: 123) - @build = builds.create!(id: 1, project_id: 123, status: 'success', type: 'Ci::Build') - end - - context 'when trace file exsits at the right place' do - before do - create_legacy_trace(@build, 'trace in file') - end - - it 'correctly archive legacy traces' do - expect(job_artifacts.count).to eq(0) - expect(File.exist?(legacy_trace_path(@build))).to be_truthy - - described_class.new.perform(1, 1) - - expect(job_artifacts.count).to eq(1) - expect(File.exist?(legacy_trace_path(@build))).to be_falsy - expect(File.read(archived_trace_path(job_artifacts.first))).to eq('trace in file') - end - end - - context 'when trace file does not exsits at the right place' do - it 'does not raise errors nor create job artifact' do - expect { described_class.new.perform(1, 1) }.not_to raise_error - - expect(job_artifacts.count).to eq(0) - end - end - - context 'when trace data exsits in database' do - before do - create_legacy_trace_in_db(@build, 'trace in db') - end - - it 'correctly archive legacy traces' do - expect(job_artifacts.count).to eq(0) - expect(@build.read_attribute(:trace)).not_to be_empty - - described_class.new.perform(1, 1) - - @build.reload - expect(job_artifacts.count).to eq(1) - expect(@build.read_attribute(:trace)).to be_nil - expect(File.read(archived_trace_path(job_artifacts.first))).to eq('trace in db') - end - end -end diff --git a/spec/lib/gitlab/background_migration/backfill_hashed_project_repositories_spec.rb b/spec/lib/gitlab/background_migration/backfill_hashed_project_repositories_spec.rb deleted file mode 100644 index 79b344ea6fa..00000000000 --- a/spec/lib/gitlab/background_migration/backfill_hashed_project_repositories_spec.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillHashedProjectRepositories do - it_behaves_like 'backfill migration for project repositories', :hashed -end diff --git a/spec/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config_spec.rb deleted file mode 100644 index 1b2e1ed0c1a..00000000000 --- a/spec/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config_spec.rb +++ /dev/null @@ -1,88 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillProjectFullpathInRepoConfig do - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:group) { namespaces.create!(name: 'foo', path: 'foo') } - let(:subgroup) { namespaces.create!(name: 'bar', path: 'bar', parent_id: group.id) } - - describe described_class::Storage::Hashed do - let(:project) { double(id: 555) } - - subject(:project_storage) { described_class.new(project) } - - it 'has the correct disk_path' do - expect(project_storage.disk_path).to eq('@hashed/91/a7/91a73fd806ab2c005c13b4dc19130a884e909dea3f72d46e30266fe1a1f588d8') - end - end - - describe described_class::Storage::LegacyProject do - let(:project) { double(full_path: 'this/is/the/full/path') } - - subject(:project_storage) { described_class.new(project) } - - it 'has the correct disk_path' do - expect(project_storage.disk_path).to eq('this/is/the/full/path') - end - end - - describe described_class::Project do - let(:project_record) { projects.create!(namespace_id: subgroup.id, name: 'baz', path: 'baz') } - - subject(:project) { described_class.find(project_record.id) } - - describe '#full_path' do - it 'returns path containing all parent namespaces' do - expect(project.full_path).to eq('foo/bar/baz') - end - - it 'raises OrphanedNamespaceError when any parent namespace does not exist' do - subgroup.update_attribute(:parent_id, non_existing_record_id) - - expect { project.full_path }.to raise_error(Gitlab::BackgroundMigration::BackfillProjectFullpathInRepoConfig::OrphanedNamespaceError) - end - end - end - - describe described_class::Up do - describe '#perform' do - subject(:migrate) { described_class.new.perform(projects.minimum(:id), projects.maximum(:id)) } - - it 'asks the gitaly client to set config' do - projects.create!(namespace_id: subgroup.id, name: 'baz', path: 'baz') - projects.create!(namespace_id: subgroup.id, name: 'buzz', path: 'buzz', storage_version: 1) - - expect_next_instance_of(Gitlab::GitalyClient::RepositoryService) do |repository_service| - allow(repository_service).to receive(:cleanup) - expect(repository_service).to receive(:set_config).with('gitlab.fullpath' => 'foo/bar/baz') - end - - expect_next_instance_of(Gitlab::GitalyClient::RepositoryService) do |repository_service| - allow(repository_service).to receive(:cleanup) - expect(repository_service).to receive(:set_config).with('gitlab.fullpath' => 'foo/bar/buzz') - end - - migrate - end - end - end - - describe described_class::Down do - describe '#perform' do - subject(:migrate) { described_class.new.perform(projects.minimum(:id), projects.maximum(:id)) } - - it 'asks the gitaly client to set config' do - projects.create!(namespace_id: subgroup.id, name: 'baz', path: 'baz') - - expect_next_instance_of(Gitlab::GitalyClient::RepositoryService) do |repository_service| - allow(repository_service).to receive(:cleanup) - expect(repository_service).to receive(:delete_config).with(['gitlab.fullpath']) - end - - migrate - end - end - end -end diff --git a/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb b/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb index ec2fd3cc4e0..fad33265030 100644 --- a/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb @@ -264,6 +264,7 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillSnippetRepositories, :migrat user_type: user_type, confirmed_at: 1.day.ago) end + let!(:invalid_snippet) { snippets.create(id: 4, type: 'PersonalSnippet', author_id: user.id, file_name: '.', content: content) } let!(:snippet) { snippets.create(id: 5, type: 'PersonalSnippet', author_id: other_user.id, file_name: file_name, content: content) } let(:ids) { [4, 5] } diff --git a/spec/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics_spec.rb b/spec/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics_spec.rb new file mode 100644 index 00000000000..71bb794d539 --- /dev/null +++ b/spec/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics_spec.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::CopyMergeRequestTargetProjectToMergeRequestMetrics, :migration, schema: 20200723125205 do + let(:migration) { described_class.new } + + let_it_be(:namespaces) { table(:namespaces) } + let_it_be(:projects) { table(:projects) } + let_it_be(:merge_requests) { table(:merge_requests) } + let_it_be(:metrics) { table(:merge_request_metrics) } + + let!(:namespace) { namespaces.create!(name: 'namespace', path: 'namespace') } + let!(:project_1) { projects.create!(namespace_id: namespace.id) } + let!(:project_2) { projects.create!(namespace_id: namespace.id) } + let!(:merge_request_to_migrate_1) { merge_requests.create!(source_branch: 'a', target_branch: 'b', target_project_id: project_1.id) } + let!(:merge_request_to_migrate_2) { merge_requests.create!(source_branch: 'c', target_branch: 'd', target_project_id: project_2.id) } + let!(:merge_request_without_metrics) { merge_requests.create!(source_branch: 'e', target_branch: 'f', target_project_id: project_2.id) } + + let!(:metrics_1) { metrics.create!(merge_request_id: merge_request_to_migrate_1.id) } + let!(:metrics_2) { metrics.create!(merge_request_id: merge_request_to_migrate_2.id) } + + let(:merge_request_ids) { [merge_request_to_migrate_1.id, merge_request_to_migrate_2.id, merge_request_without_metrics.id] } + + subject { migration.perform(merge_request_ids.min, merge_request_ids.max) } + + it 'copies `target_project_id` to the associated `merge_request_metrics` record' do + subject + + expect(metrics_1.reload.target_project_id).to eq(project_1.id) + expect(metrics_2.reload.target_project_id).to eq(project_2.id) + end + + it 'does not create metrics record when it is missing' do + subject + + expect(metrics.find_by_merge_request_id(merge_request_without_metrics.id)).to be_nil + end +end diff --git a/spec/lib/gitlab/background_migration/fix_cross_project_label_links_spec.rb b/spec/lib/gitlab/background_migration/fix_cross_project_label_links_spec.rb deleted file mode 100644 index 8e3ace083fc..00000000000 --- a/spec/lib/gitlab/background_migration/fix_cross_project_label_links_spec.rb +++ /dev/null @@ -1,111 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::FixCrossProjectLabelLinks do - let(:namespaces_table) { table(:namespaces) } - let(:projects_table) { table(:projects) } - let(:issues_table) { table(:issues) } - let(:merge_requests_table) { table(:merge_requests) } - let(:labels_table) { table(:labels) } - let(:label_links_table) { table(:label_links) } - - let!(:group1) { namespaces_table.create(id: 10, type: 'Group', name: 'group1', path: 'group1') } - let!(:group2) { namespaces_table.create(id: 20, type: 'Group', name: 'group2', path: 'group2') } - - let!(:project1) { projects_table.create(id: 1, name: 'project1', path: 'group1/project1', namespace_id: 10) } - let!(:project2) { projects_table.create(id: 3, name: 'project2', path: 'group1/project2', namespace_id: 20) } - - let!(:label1) { labels_table.create(id: 1, title: 'bug', color: 'red', group_id: 10, type: 'GroupLabel') } - let!(:label2) { labels_table.create(id: 2, title: 'bug', color: 'red', group_id: 20, type: 'GroupLabel') } - - def create_merge_request(id, project_id) - merge_requests_table.create(id: id, - target_project_id: project_id, - target_branch: 'master', - source_project_id: project_id, - source_branch: 'mr name', - title: "mr name#{id}") - end - - def create_issue(id, project_id) - issues_table.create(id: id, title: "issue#{id}", project_id: project_id) - end - - def create_resource(target_type, id, project_id) - target_type == 'Issue' ? create_issue(id, project_id) : create_merge_request(id, project_id) - end - - shared_examples_for 'resource with cross-project labels' do - it 'updates only cross-project label links which exist in the local project or group' do - create_resource(target_type, 1, 1) - create_resource(target_type, 2, 3) - labels_table.create(id: 3, title: 'bug', color: 'red', project_id: 3, type: 'ProjectLabel') - link = label_links_table.create(label_id: 2, target_type: target_type, target_id: 1) - link2 = label_links_table.create(label_id: 3, target_type: target_type, target_id: 2) - - subject.perform(1, 100) - - expect(link.reload.label_id).to eq(1) - expect(link2.reload.label_id).to eq(3) - end - - it 'ignores cross-project label links if label color is different' do - labels_table.create(id: 3, title: 'bug', color: 'green', group_id: 20, type: 'GroupLabel') - create_resource(target_type, 1, 1) - link = label_links_table.create(label_id: 3, target_type: target_type, target_id: 1) - - subject.perform(1, 100) - - expect(link.reload.label_id).to eq(3) - end - - it 'ignores cross-project label links if label name is different' do - labels_table.create(id: 3, title: 'bug1', color: 'red', group_id: 20, type: 'GroupLabel') - create_resource(target_type, 1, 1) - link = label_links_table.create(label_id: 3, target_type: target_type, target_id: 1) - - subject.perform(1, 100) - - expect(link.reload.label_id).to eq(3) - end - - context 'with nested group' do - before do - namespaces_table.create(id: 11, type: 'Group', name: 'subgroup1', path: 'group1/subgroup1', parent_id: 10) - projects_table.create(id: 2, name: 'subproject1', path: 'group1/subgroup1/subproject1', namespace_id: 11) - create_resource(target_type, 1, 2) - end - - it 'ignores label links referencing ancestor group labels' do - labels_table.create(id: 4, title: 'bug', color: 'red', project_id: 2, type: 'ProjectLabel') - label_links_table.create(label_id: 4, target_type: target_type, target_id: 1) - link = label_links_table.create(label_id: 1, target_type: target_type, target_id: 1) - - subject.perform(1, 100) - - expect(link.reload.label_id).to eq(1) - end - - it 'checks also issues and MRs in subgroups' do - link = label_links_table.create(label_id: 2, target_type: target_type, target_id: 1) - - subject.perform(1, 100) - - expect(link.reload.label_id).to eq(1) - end - end - end - - context 'resource is Issue' do - it_behaves_like 'resource with cross-project labels' do - let(:target_type) { 'Issue' } - end - end - - context 'resource is Merge Request' do - it_behaves_like 'resource with cross-project labels' do - let(:target_type) { 'MergeRequest' } - end - end -end diff --git a/spec/lib/gitlab/background_migration/legacy_upload_mover_spec.rb b/spec/lib/gitlab/background_migration/legacy_upload_mover_spec.rb index bf793e7c537..1637589d272 100644 --- a/spec/lib/gitlab/background_migration/legacy_upload_mover_spec.rb +++ b/spec/lib/gitlab/background_migration/legacy_upload_mover_spec.rb @@ -264,6 +264,7 @@ RSpec.describe Gitlab::BackgroundMigration::LegacyUploadMover do let(:remote_file) do { key: "#{legacy_upload.path}" } end + let(:connection) { ::Fog::Storage.new(FileUploader.object_store_credentials) } let(:bucket) { connection.directories.create(key: 'uploads') } diff --git a/spec/lib/gitlab/background_migration/migrate_build_stage_spec.rb b/spec/lib/gitlab/background_migration/migrate_build_stage_spec.rb deleted file mode 100644 index 65d45ec694f..00000000000 --- a/spec/lib/gitlab/background_migration/migrate_build_stage_spec.rb +++ /dev/null @@ -1,84 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::MigrateBuildStage do - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:pipelines) { table(:ci_pipelines) } - let(:stages) { table(:ci_stages) } - let(:jobs) { table(:ci_builds) } - - let(:statuses) do - { - created: 0, - pending: 1, - running: 2, - success: 3, - failed: 4, - canceled: 5, - skipped: 6, - manual: 7 - } - end - - before do - namespace = namespaces.create!(name: 'gitlab-org', path: 'gitlab-org') - projects.create!(id: 123, name: 'gitlab', path: 'gitlab-ce', namespace_id: namespace.id) - pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a') - - jobs.create!(id: 1, commit_id: 1, project_id: 123, - stage_idx: 2, stage: 'build', status: :success) - jobs.create!(id: 2, commit_id: 1, project_id: 123, - stage_idx: 2, stage: 'build', status: :success) - jobs.create!(id: 3, commit_id: 1, project_id: 123, - stage_idx: 1, stage: 'test', status: :failed) - jobs.create!(id: 4, commit_id: 1, project_id: 123, - stage_idx: 1, stage: 'test', status: :success) - jobs.create!(id: 5, commit_id: 1, project_id: 123, - stage_idx: 3, stage: 'deploy', status: :pending) - jobs.create!(id: 6, commit_id: 1, project_id: 123, - stage_idx: 3, stage: nil, status: :pending) - end - - it 'correctly migrates builds stages' do - expect(stages.count).to be_zero - - described_class.new.perform(1, 6) - - expect(stages.count).to eq 3 - expect(stages.all.pluck(:name)).to match_array %w[test build deploy] - expect(jobs.where(stage_id: nil)).to be_one - expect(jobs.find_by(stage_id: nil).id).to eq 6 - expect(stages.all.pluck(:status)).to match_array [statuses[:success], - statuses[:failed], - statuses[:pending]] - end - - it 'recovers from unique constraint violation only twice', quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/28128' do - allow(described_class::Migratable::Stage) - .to receive(:find_by).and_return(nil) - - expect(described_class::Migratable::Stage) - .to receive(:find_by).exactly(3).times - - expect { described_class.new.perform(1, 6) } - .to raise_error ActiveRecord::RecordNotUnique - end - - context 'when invalid class can be loaded due to single table inheritance' do - let(:commit_status) do - jobs.create!(id: 7, commit_id: 1, project_id: 123, stage_idx: 4, - stage: 'post-deploy', status: :failed) - end - - before do - commit_status.update_column(:type, 'SomeClass') - end - - it 'does ignore single table inheritance type' do - expect { described_class.new.perform(1, 7) }.not_to raise_error - expect(jobs.find(7)).to have_attributes(stage_id: (a_value > 0)) - end - end -end diff --git a/spec/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data_spec.rb b/spec/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data_spec.rb index d53f79c61c1..d829fd5daf5 100644 --- a/spec/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data_spec.rb +++ b/spec/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data_spec.rb @@ -286,9 +286,11 @@ RSpec.describe Gitlab::BackgroundMigration::MigrateIssueTrackersSensitiveData, s let!(:jira_service_invalid) do services.create(id: 19, title: 'invalid - title', description: 'invalid - description', type: 'JiraService', properties: 'invalid data', category: 'issue_tracker') end + let!(:jira_service_valid) do services.create(id: 20, type: 'JiraService', properties: jira_properties.to_json, category: 'issue_tracker') end + let!(:bugzilla_service_valid) do services.create(id: 11, type: 'BugzillaService', title: nil, properties: tracker_properties.to_json, category: 'issue_tracker') end diff --git a/spec/lib/gitlab/background_migration/migrate_stage_index_spec.rb b/spec/lib/gitlab/background_migration/migrate_stage_index_spec.rb deleted file mode 100644 index 81874ff7982..00000000000 --- a/spec/lib/gitlab/background_migration/migrate_stage_index_spec.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::MigrateStageIndex do - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:pipelines) { table(:ci_pipelines) } - let(:stages) { table(:ci_stages) } - let(:jobs) { table(:ci_builds) } - let(:namespace) { namespaces.create(name: 'gitlab-org', path: 'gitlab-org') } - let(:project) { projects.create!(namespace_id: namespace.id, name: 'gitlab', path: 'gitlab') } - let(:pipeline) { pipelines.create!(project_id: project.id, ref: 'master', sha: 'adf43c3a') } - let(:stage1) { stages.create(project_id: project.id, pipeline_id: pipeline.id, name: 'build') } - let(:stage2) { stages.create(project_id: project.id, pipeline_id: pipeline.id, name: 'test') } - - before do - jobs.create!(commit_id: pipeline.id, project_id: project.id, - stage_idx: 2, stage_id: stage1.id) - jobs.create!(commit_id: pipeline.id, project_id: project.id, - stage_idx: 2, stage_id: stage1.id) - jobs.create!(commit_id: pipeline.id, project_id: project.id, - stage_idx: 10, stage_id: stage1.id) - jobs.create!(commit_id: pipeline.id, project_id: project.id, - stage_idx: 3, stage_id: stage2.id) - end - - it 'correctly migrates stages indices' do - expect(stages.all.pluck(:position)).to all(be_nil) - - described_class.new.perform(stage1.id, stage2.id) - - expect(stages.all.order(:id).pluck(:position)).to eq [2, 3] - end -end diff --git a/spec/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table_spec.rb b/spec/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table_spec.rb deleted file mode 100644 index 73faca54b52..00000000000 --- a/spec/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table_spec.rb +++ /dev/null @@ -1,94 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::PopulateClusterKubernetesNamespaceTable do - include MigrationHelpers::ClusterHelpers - - let(:migration) { described_class.new } - let(:clusters_table) { table(:clusters) } - let(:cluster_projects_table) { table(:cluster_projects) } - let(:cluster_kubernetes_namespaces_table) { table(:clusters_kubernetes_namespaces) } - let(:projects_table) { table(:projects) } - let(:namespaces_table) { table(:namespaces) } - let(:provider_gcp_table) { table(:cluster_providers_gcp) } - let(:platform_kubernetes_table) { table(:cluster_platforms_kubernetes) } - - before do - create_cluster_project_list(10) - end - - shared_examples 'consistent kubernetes namespace attributes' do - it 'populates namespace and service account information' do - migration.perform - - clusters_with_namespace.each do |cluster| - cluster_project = cluster_projects_table.find_by(cluster_id: cluster.id) - project = projects_table.find(cluster_project.project_id) - kubernetes_namespace = cluster_kubernetes_namespaces_table.find_by(cluster_id: cluster.id) - namespace = "#{project.path}-#{project.id}" - - expect(kubernetes_namespace).to be_present - expect(kubernetes_namespace.cluster_project_id).to eq(cluster_project.id) - expect(kubernetes_namespace.project_id).to eq(cluster_project.project_id) - expect(kubernetes_namespace.cluster_id).to eq(cluster_project.cluster_id) - expect(kubernetes_namespace.namespace).to eq(namespace) - expect(kubernetes_namespace.service_account_name).to eq("#{namespace}-service-account") - end - end - end - - context 'when no Clusters::Project has a Clusters::KubernetesNamespace' do - let(:cluster_projects) { cluster_projects_table.all } - - it 'creates a Clusters::KubernetesNamespace per Clusters::Project' do - expect do - migration.perform - end.to change(Clusters::KubernetesNamespace, :count).by(cluster_projects_table.count) - end - - it_behaves_like 'consistent kubernetes namespace attributes' do - let(:clusters_with_namespace) { clusters_table.all } - end - end - - context 'when every Clusters::Project has Clusters::KubernetesNamespace' do - before do - create_kubernetes_namespace(clusters_table.all) - end - - it 'does not create any Clusters::KubernetesNamespace' do - expect do - migration.perform - end.not_to change(Clusters::KubernetesNamespace, :count) - end - end - - context 'when only some Clusters::Project have Clusters::KubernetesNamespace related' do - let(:with_kubernetes_namespace) { clusters_table.first(6) } - let(:with_no_kubernetes_namespace) { clusters_table.last(4) } - - before do - create_kubernetes_namespace(with_kubernetes_namespace) - end - - it 'creates limited number of Clusters::KubernetesNamespace' do - expect do - migration.perform - end.to change(Clusters::KubernetesNamespace, :count).by(with_no_kubernetes_namespace.count) - end - - it 'does not modify clusters with Clusters::KubernetesNamespace' do - migration.perform - - with_kubernetes_namespace.each do |cluster| - kubernetes_namespace = cluster_kubernetes_namespaces_table.where(cluster_id: cluster.id) - expect(kubernetes_namespace.count).to eq(1) - end - end - - it_behaves_like 'consistent kubernetes namespace attributes' do - let(:clusters_with_namespace) { with_no_kubernetes_namespace } - end - end -end diff --git a/spec/lib/gitlab/background_migration/populate_personal_snippet_statistics_spec.rb b/spec/lib/gitlab/background_migration/populate_personal_snippet_statistics_spec.rb new file mode 100644 index 00000000000..e746451b1b9 --- /dev/null +++ b/spec/lib/gitlab/background_migration/populate_personal_snippet_statistics_spec.rb @@ -0,0 +1,141 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::PopulatePersonalSnippetStatistics do + let(:file_name) { 'file_name.rb' } + let(:content) { 'content' } + let(:snippets) { table(:snippets) } + let(:snippet_repositories) { table(:snippet_repositories) } + let(:users) { table(:users) } + let(:namespaces) { table(:namespaces) } + let(:snippet_statistics) { table(:snippet_statistics) } + let(:namespace_statistics) { table(:namespace_root_storage_statistics) } + let(:routes) { table(:routes) } + let(:repo_size) { 123456 } + let(:expected_repo_size) { repo_size.megabytes } + + let(:user1) { users.create!(id: 1, email: 'test@example.com', projects_limit: 100, username: 'test1') } + let(:user2) { users.create!(id: 2, email: 'test2@example.com', projects_limit: 100, username: 'test2') } + let!(:user1_namespace) { namespaces.create!(id: 1, name: 'user1', path: 'user1', owner_id: user1.id) } + let!(:user2_namespace) { namespaces.create!(id: 2, name: 'user2', path: 'user2', owner_id: user2.id) } + let(:user1_namespace_statistics) { namespace_statistics.find_by(namespace_id: user1_namespace.id) } + let(:user2_namespace_statistics) { namespace_statistics.find_by(namespace_id: user2_namespace.id) } + + let(:ids) { snippets.pluck(:id) } + let(:migration) { described_class.new } + + subject do + migration.perform(ids) + end + + before do + allow_any_instance_of(Repository).to receive(:size).and_return(repo_size) + end + + after do + snippets.all.each { |s| raw_repository(s).remove } + end + + context 'with existing personal snippets' do + let!(:snippet1) { create_snippet(1, user1) } + let!(:snippet2) { create_snippet(2, user1) } + let!(:snippet3) { create_snippet(3, user2) } + let!(:snippet4) { create_snippet(4, user2) } + + before do + create_snippet_statistics(2, 0) + create_snippet_statistics(4, 123) + end + + it 'creates/updates all snippet_statistics' do + expect { subject }.to change { snippet_statistics.count }.from(2).to(4) + + expect(snippet_statistics.pluck(:repository_size)).to be_all(expected_repo_size) + end + + it 'creates/updates the associated namespace statistics' do + expect(migration).to receive(:update_namespace_statistics).twice.and_call_original + + subject + + stats = snippet_statistics.where(snippet_id: [snippet1, snippet2]).sum(:repository_size) + expect(user1_namespace_statistics.snippets_size).to eq stats + + stats = snippet_statistics.where(snippet_id: [snippet3, snippet4]).sum(:repository_size) + expect(user2_namespace_statistics.snippets_size).to eq stats + end + + context 'when an error is raised when updating a namespace statistics' do + it 'logs the error and continue execution' do + expect_next_instance_of(Namespaces::StatisticsRefresherService) do |instance| + expect(instance).to receive(:execute).with(Namespace.find(user1_namespace.id)).and_raise('Error') + end + + expect_next_instance_of(Namespaces::StatisticsRefresherService) do |instance| + expect(instance).to receive(:execute).and_call_original + end + + expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |instance| + expect(instance).to receive(:error).with(message: /Error updating statistics for namespace/).once + end + + subject + + expect(user1_namespace_statistics).to be_nil + + stats = snippet_statistics.where(snippet_id: [snippet3, snippet4]).sum(:repository_size) + expect(user2_namespace_statistics.snippets_size).to eq stats + end + end + end + + context 'when a snippet repository is empty' do + let!(:snippet1) { create_snippet(1, user1, with_repo: false) } + let!(:snippet2) { create_snippet(2, user1) } + + it 'logs error and continues execution' do + expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |instance| + expect(instance).to receive(:error).with(message: /Invalid snippet repository/).once + end + + subject + + expect(snippet_statistics.find_by(snippet_id: snippet1.id)).to be_nil + expect(user1_namespace_statistics.snippets_size).to eq expected_repo_size + end + end + + def create_snippet(id, author, with_repo: true) + snippets.create!(id: id, type: 'PersonalSnippet', author_id: author.id, file_name: file_name, content: content).tap do |snippet| + if with_repo + allow(snippet).to receive(:disk_path).and_return(disk_path(snippet)) + + TestEnv.copy_repo(snippet, + bare_repo: TestEnv.factory_repo_path_bare, + refs: TestEnv::BRANCH_SHA) + + raw_repository(snippet).create_repository + end + end + end + + def create_snippet_statistics(snippet_id, repository_size = 0) + snippet_statistics.create!(snippet_id: snippet_id, repository_size: repository_size) + end + + def raw_repository(snippet) + Gitlab::Git::Repository.new('default', + "#{disk_path(snippet)}.git", + Gitlab::GlRepository::SNIPPET.identifier_for_container(snippet), + "@snippets/#{snippet.id}") + end + + def hashed_repository(snippet) + Storage::Hashed.new(snippet, prefix: '@snippets') + end + + def disk_path(snippet) + hashed_repository(snippet).disk_path + end +end diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_dependencies/untracked_file_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_dependencies/untracked_file_spec.rb deleted file mode 100644 index 6a25e8e2784..00000000000 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_dependencies/untracked_file_spec.rb +++ /dev/null @@ -1,263 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile do - include MigrationsHelpers::TrackUntrackedUploadsHelpers - - let!(:appearances) { table(:appearances) } - let!(:namespaces) { table(:namespaces) } - let!(:projects) { table(:projects) } - let!(:routes) { table(:routes) } - let!(:uploads) { table(:uploads) } - - before(:all) do - ensure_temporary_tracking_table_exists - end - - describe '#upload_path' do - def assert_upload_path(file_path, expected_upload_path) - untracked_file = create_untracked_file(file_path) - - expect(untracked_file.upload_path).to eq(expected_upload_path) - end - - context 'for an appearance logo file path' do - it 'returns the file path relative to the CarrierWave root' do - assert_upload_path('/-/system/appearance/logo/1/some_logo.jpg', 'uploads/-/system/appearance/logo/1/some_logo.jpg') - end - end - - context 'for an appearance header_logo file path' do - it 'returns the file path relative to the CarrierWave root' do - assert_upload_path('/-/system/appearance/header_logo/1/some_logo.jpg', 'uploads/-/system/appearance/header_logo/1/some_logo.jpg') - end - end - - context 'for a pre-Markdown Note attachment file path' do - it 'returns the file path relative to the CarrierWave root' do - assert_upload_path('/-/system/note/attachment/1234/some_attachment.pdf', 'uploads/-/system/note/attachment/1234/some_attachment.pdf') - end - end - - context 'for a user avatar file path' do - it 'returns the file path relative to the CarrierWave root' do - assert_upload_path('/-/system/user/avatar/1234/avatar.jpg', 'uploads/-/system/user/avatar/1234/avatar.jpg') - end - end - - context 'for a group avatar file path' do - it 'returns the file path relative to the CarrierWave root' do - assert_upload_path('/-/system/group/avatar/1234/avatar.jpg', 'uploads/-/system/group/avatar/1234/avatar.jpg') - end - end - - context 'for a project avatar file path' do - it 'returns the file path relative to the CarrierWave root' do - assert_upload_path('/-/system/project/avatar/1234/avatar.jpg', 'uploads/-/system/project/avatar/1234/avatar.jpg') - end - end - - context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do - it 'returns the file path relative to the project directory in uploads' do - project = create_project - random_hex = SecureRandom.hex - - assert_upload_path("/#{get_full_path(project)}/#{random_hex}/Some file.jpg", "#{random_hex}/Some file.jpg") - end - end - end - - describe '#uploader' do - def assert_uploader(file_path, expected_uploader) - untracked_file = create_untracked_file(file_path) - - expect(untracked_file.uploader).to eq(expected_uploader) - end - - context 'for an appearance logo file path' do - it 'returns AttachmentUploader as a string' do - assert_uploader('/-/system/appearance/logo/1/some_logo.jpg', 'AttachmentUploader') - end - end - - context 'for an appearance header_logo file path' do - it 'returns AttachmentUploader as a string' do - assert_uploader('/-/system/appearance/header_logo/1/some_logo.jpg', 'AttachmentUploader') - end - end - - context 'for a pre-Markdown Note attachment file path' do - it 'returns AttachmentUploader as a string' do - assert_uploader('/-/system/note/attachment/1234/some_attachment.pdf', 'AttachmentUploader') - end - end - - context 'for a user avatar file path' do - it 'returns AvatarUploader as a string' do - assert_uploader('/-/system/user/avatar/1234/avatar.jpg', 'AvatarUploader') - end - end - - context 'for a group avatar file path' do - it 'returns AvatarUploader as a string' do - assert_uploader('/-/system/group/avatar/1234/avatar.jpg', 'AvatarUploader') - end - end - - context 'for a project avatar file path' do - it 'returns AvatarUploader as a string' do - assert_uploader('/-/system/project/avatar/1234/avatar.jpg', 'AvatarUploader') - end - end - - context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do - it 'returns FileUploader as a string' do - project = create_project - - assert_uploader("/#{get_full_path(project)}/#{SecureRandom.hex}/Some file.jpg", 'FileUploader') - end - end - end - - describe '#model_type' do - def assert_model_type(file_path, expected_model_type) - untracked_file = create_untracked_file(file_path) - - expect(untracked_file.model_type).to eq(expected_model_type) - end - - context 'for an appearance logo file path' do - it 'returns Appearance as a string' do - assert_model_type('/-/system/appearance/logo/1/some_logo.jpg', 'Appearance') - end - end - - context 'for an appearance header_logo file path' do - it 'returns Appearance as a string' do - assert_model_type('/-/system/appearance/header_logo/1/some_logo.jpg', 'Appearance') - end - end - - context 'for a pre-Markdown Note attachment file path' do - it 'returns Note as a string' do - assert_model_type('/-/system/note/attachment/1234/some_attachment.pdf', 'Note') - end - end - - context 'for a user avatar file path' do - it 'returns User as a string' do - assert_model_type('/-/system/user/avatar/1234/avatar.jpg', 'User') - end - end - - context 'for a group avatar file path' do - it 'returns Namespace as a string' do - assert_model_type('/-/system/group/avatar/1234/avatar.jpg', 'Namespace') - end - end - - context 'for a project avatar file path' do - it 'returns Project as a string' do - assert_model_type('/-/system/project/avatar/1234/avatar.jpg', 'Project') - end - end - - context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do - it 'returns Project as a string' do - project = create_project - - assert_model_type("/#{get_full_path(project)}/#{SecureRandom.hex}/Some file.jpg", 'Project') - end - end - end - - describe '#model_id' do - def assert_model_id(file_path, expected_model_id) - untracked_file = create_untracked_file(file_path) - - expect(untracked_file.model_id).to eq(expected_model_id) - end - - context 'for an appearance logo file path' do - it 'returns the ID as a string' do - assert_model_id('/-/system/appearance/logo/1/some_logo.jpg', 1) - end - end - - context 'for an appearance header_logo file path' do - it 'returns the ID as a string' do - assert_model_id('/-/system/appearance/header_logo/1/some_logo.jpg', 1) - end - end - - context 'for a pre-Markdown Note attachment file path' do - it 'returns the ID as a string' do - assert_model_id('/-/system/note/attachment/1234/some_attachment.pdf', 1234) - end - end - - context 'for a user avatar file path' do - it 'returns the ID as a string' do - assert_model_id('/-/system/user/avatar/1234/avatar.jpg', 1234) - end - end - - context 'for a group avatar file path' do - it 'returns the ID as a string' do - assert_model_id('/-/system/group/avatar/1234/avatar.jpg', 1234) - end - end - - context 'for a project avatar file path' do - it 'returns the ID as a string' do - assert_model_id('/-/system/project/avatar/1234/avatar.jpg', 1234) - end - end - - context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do - it 'returns the ID as a string' do - project = create_project - - assert_model_id("/#{get_full_path(project)}/#{SecureRandom.hex}/Some file.jpg", project.id) - end - end - end - - describe '#file_size' do - context 'for an appearance logo file path' do - let(:appearance) { create_or_update_appearance(logo: true) } - let(:untracked_file) { described_class.create!(path: get_uploads(appearance, 'Appearance').first.path) } - - it 'returns the file size' do - expect(untracked_file.file_size).to eq(1062) - end - end - - context 'for a project avatar file path' do - let(:project) { create_project(avatar: true) } - let(:untracked_file) { described_class.create!(path: get_uploads(project, 'Project').first.path) } - - it 'returns the file size' do - expect(untracked_file.file_size).to eq(1062) - end - end - - context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do - let(:project) { create_project } - let(:untracked_file) { create_untracked_file("/#{get_full_path(project)}/#{get_uploads(project, 'Project').first.path}") } - - before do - add_markdown_attachment(project) - end - - it 'returns the file size' do - expect(untracked_file.file_size).to eq(1062) - end - end - end - - def create_untracked_file(path_relative_to_upload_dir) - described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}#{path_relative_to_upload_dir}") - end -end diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb deleted file mode 100644 index 787cc54e79a..00000000000 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ /dev/null @@ -1,254 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::PopulateUntrackedUploads do - include MigrationsHelpers::TrackUntrackedUploadsHelpers - - subject { described_class.new } - - let!(:appearances) { table(:appearances) } - let!(:namespaces) { table(:namespaces) } - let!(:notes) { table(:notes) } - let!(:projects) { table(:projects) } - let!(:routes) { table(:routes) } - let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } - let!(:uploads) { table(:uploads) } - let!(:users) { table(:users) } - - before do - ensure_temporary_tracking_table_exists - uploads.delete_all - end - - context 'with untracked files and tracked files in untracked_files_for_uploads' do - let!(:appearance) { create_or_update_appearance(logo: true, header_logo: true) } - let!(:user1) { create_user(avatar: true) } - let!(:user2) { create_user(avatar: true) } - let!(:project1) { create_project(avatar: true) } - let!(:project2) { create_project(avatar: true) } - - before do - add_markdown_attachment(project1) - add_markdown_attachment(project2) - - # File records created by PrepareUntrackedUploads - untracked_files_for_uploads.create!(path: get_uploads(appearance, 'Appearance').first.path) - untracked_files_for_uploads.create!(path: get_uploads(appearance, 'Appearance').last.path) - untracked_files_for_uploads.create!(path: get_uploads(user1, 'User').first.path) - untracked_files_for_uploads.create!(path: get_uploads(user2, 'User').first.path) - untracked_files_for_uploads.create!(path: get_uploads(project1, 'Project').first.path) - untracked_files_for_uploads.create!(path: get_uploads(project2, 'Project').first.path) - untracked_files_for_uploads.create!(path: "#{legacy_project_uploads_dir(project1).sub("#{MigrationsHelpers::TrackUntrackedUploadsHelpers::PUBLIC_DIR}/", '')}/#{get_uploads(project1, 'Project').last.path}") - untracked_files_for_uploads.create!(path: "#{legacy_project_uploads_dir(project2).sub("#{MigrationsHelpers::TrackUntrackedUploadsHelpers::PUBLIC_DIR}/", '')}/#{get_uploads(project2, 'Project').last.path}") - - # Untrack 4 files - get_uploads(user2, 'User').delete_all - get_uploads(project2, 'Project').delete_all # 2 files: avatar and a Markdown upload - get_uploads(appearance, 'Appearance').where("path like '%header_logo%'").delete_all - end - - it 'adds untracked files to the uploads table' do - expect do - subject.perform(1, untracked_files_for_uploads.reorder(:id).last.id) - end.to change { uploads.count }.from(4).to(8) - - expect(get_uploads(user2, 'User').count).to eq(1) - expect(get_uploads(project2, 'Project').count).to eq(2) - expect(get_uploads(appearance, 'Appearance').count).to eq(2) - end - - it 'deletes rows after processing them' do - expect(subject).to receive(:drop_temp_table_if_finished) # Don't drop the table so we can look at it - - expect do - subject.perform(1, untracked_files_for_uploads.last.id) - end.to change { untracked_files_for_uploads.count }.from(8).to(0) - end - - it 'does not create duplicate uploads of already tracked files' do - subject.perform(1, untracked_files_for_uploads.last.id) - - expect(get_uploads(user1, 'User').count).to eq(1) - expect(get_uploads(project1, 'Project').count).to eq(2) - expect(get_uploads(appearance, 'Appearance').count).to eq(2) - end - - it 'uses the start and end batch ids [only 1st half]' do - ids = untracked_files_for_uploads.all.order(:id).pluck(:id) - start_id = ids[0] - end_id = ids[3] - - expect do - subject.perform(start_id, end_id) - end.to change { uploads.count }.from(4).to(6) - - expect(get_uploads(user1, 'User').count).to eq(1) - expect(get_uploads(user2, 'User').count).to eq(1) - expect(get_uploads(appearance, 'Appearance').count).to eq(2) - expect(get_uploads(project1, 'Project').count).to eq(2) - expect(get_uploads(project2, 'Project').count).to eq(0) - - # Only 4 have been either confirmed or added to uploads - expect(untracked_files_for_uploads.count).to eq(4) - end - - it 'uses the start and end batch ids [only 2nd half]' do - ids = untracked_files_for_uploads.all.order(:id).pluck(:id) - start_id = ids[4] - end_id = ids[7] - - expect do - subject.perform(start_id, end_id) - end.to change { uploads.count }.from(4).to(6) - - expect(get_uploads(user1, 'User').count).to eq(1) - expect(get_uploads(user2, 'User').count).to eq(0) - expect(get_uploads(appearance, 'Appearance').count).to eq(1) - expect(get_uploads(project1, 'Project').count).to eq(2) - expect(get_uploads(project2, 'Project').count).to eq(2) - - # Only 4 have been either confirmed or added to uploads - expect(untracked_files_for_uploads.count).to eq(4) - end - - it 'does not drop the temporary tracking table after processing the batch, if there are still untracked rows' do - subject.perform(1, untracked_files_for_uploads.last.id - 1) - - expect(ActiveRecord::Base.connection.table_exists?(:untracked_files_for_uploads)).to be_truthy - end - - it 'drops the temporary tracking table after processing the batch, if there are no untracked rows left' do - expect(subject).to receive(:drop_temp_table_if_finished) - - subject.perform(1, untracked_files_for_uploads.last.id) - end - - it 'does not block a whole batch because of one bad path' do - untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{get_full_path(project2)}/._7d37bf4c747916390e596744117d5d1a") - expect(untracked_files_for_uploads.count).to eq(9) - expect(uploads.count).to eq(4) - - subject.perform(1, untracked_files_for_uploads.last.id) - - expect(untracked_files_for_uploads.count).to eq(1) - expect(uploads.count).to eq(8) - end - - it 'an unparseable path is shown in error output' do - bad_path = "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{get_full_path(project2)}/._7d37bf4c747916390e596744117d5d1a" - untracked_files_for_uploads.create!(path: bad_path) - - expect(Rails.logger).to receive(:error).with(/Error parsing path "#{bad_path}":/) - - subject.perform(1, untracked_files_for_uploads.last.id) - end - end - - context 'with no untracked files' do - it 'does not add to the uploads table (and does not raise error)' do - expect do - subject.perform(1, 1000) - end.not_to change { uploads.count }.from(0) - end - end - - describe 'upload outcomes for each path pattern' do - shared_examples_for 'non_markdown_file' do - let!(:expected_upload_attrs) { model_uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') } - let!(:untracked_file) { untracked_files_for_uploads.create!(path: expected_upload_attrs['path']) } - - before do - model_uploads.delete_all - end - - it 'creates an Upload record' do - expect do - subject.perform(1, untracked_files_for_uploads.last.id) - end.to change { model_uploads.count }.from(0).to(1) - - expect(model_uploads.first.attributes).to include(expected_upload_attrs) - end - end - - context 'for an appearance logo file path' do - let(:model) { create_or_update_appearance(logo: true) } - let(:model_uploads) { get_uploads(model, 'Appearance') } - - it_behaves_like 'non_markdown_file' - end - - context 'for an appearance header_logo file path' do - let(:model) { create_or_update_appearance(header_logo: true) } - let(:model_uploads) { get_uploads(model, 'Appearance') } - - it_behaves_like 'non_markdown_file' - end - - context 'for a pre-Markdown Note attachment file path' do - let(:model) { create_note(attachment: true) } - let!(:expected_upload_attrs) { get_uploads(model, 'Note').first.attributes.slice('path', 'uploader', 'size', 'checksum') } - let!(:untracked_file) { untracked_files_for_uploads.create!(path: expected_upload_attrs['path']) } - - before do - get_uploads(model, 'Note').delete_all - end - - # Can't use the shared example because Note doesn't have an `uploads` association - it 'creates an Upload record' do - expect do - subject.perform(1, untracked_files_for_uploads.last.id) - end.to change { get_uploads(model, 'Note').count }.from(0).to(1) - - expect(get_uploads(model, 'Note').first.attributes).to include(expected_upload_attrs) - end - end - - context 'for a user avatar file path' do - let(:model) { create_user(avatar: true) } - let(:model_uploads) { get_uploads(model, 'User') } - - it_behaves_like 'non_markdown_file' - end - - context 'for a group avatar file path' do - let(:model) { create_group(avatar: true) } - let(:model_uploads) { get_uploads(model, 'Namespace') } - - it_behaves_like 'non_markdown_file' - end - - context 'for a project avatar file path' do - let(:model) { create_project(avatar: true) } - let(:model_uploads) { get_uploads(model, 'Project') } - - it_behaves_like 'non_markdown_file' - end - - context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do - let(:model) { create_project } - - before do - # Upload the file - add_markdown_attachment(model) - - # Create the untracked_files_for_uploads record - untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{get_full_path(model)}/#{get_uploads(model, 'Project').first.path}") - - # Save the expected upload attributes - @expected_upload_attrs = get_uploads(model, 'Project').first.attributes.slice('path', 'uploader', 'size', 'checksum') - - # Untrack the file - get_uploads(model, 'Project').delete_all - end - - it 'creates an Upload record' do - expect do - subject.perform(1, untracked_files_for_uploads.last.id) - end.to change { get_uploads(model, 'Project').count }.from(0).to(1) - - expect(get_uploads(model, 'Project').first.attributes).to include(@expected_upload_attrs) - end - end - end -end diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb deleted file mode 100644 index 9b01407dc8b..00000000000 --- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb +++ /dev/null @@ -1,159 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -# Rollback DB to 10.5 (later than this was originally written for) because it still needs to work. -RSpec.describe Gitlab::BackgroundMigration::PrepareUntrackedUploads do - include MigrationsHelpers::TrackUntrackedUploadsHelpers - - let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } - let!(:appearances) { table(:appearances) } - let!(:namespaces) { table(:namespaces) } - let!(:projects) { table(:projects) } - let!(:routes) { table(:routes) } - let!(:uploads) { table(:uploads) } - let!(:users) { table(:users) } - - around do |example| - # Especially important so the follow-up migration does not get run - Sidekiq::Testing.fake! do - example.run - end - end - - shared_examples 'prepares the untracked_files_for_uploads table' do - context 'when files were uploaded before and after hashed storage was enabled' do - let!(:appearance) { create_or_update_appearance(logo: true, header_logo: true) } - let!(:user) { create_user(avatar: true) } - let!(:project1) { create_project(avatar: true) } - let(:project2) { create_project } # instantiate after enabling hashed_storage - - before do - # Markdown upload before enabling hashed_storage - add_markdown_attachment(project1) - - # Markdown upload after enabling hashed_storage - add_markdown_attachment(project2, hashed_storage: true) - end - - it 'has a path field long enough for really long paths' do - described_class.new.perform - - component = 'a' * 255 - - long_path = [ - 'uploads', - component, # project.full_path - component # filename - ].flatten.join('/') - - record = untracked_files_for_uploads.create!(path: long_path) - expect(record.reload.path.size).to eq(519) - end - - it 'adds unhashed files to the untracked_files_for_uploads table' do - described_class.new.perform - - expect(untracked_files_for_uploads.count).to eq(5) - end - - it 'adds files with paths relative to CarrierWave.root' do - described_class.new.perform - untracked_files_for_uploads.all.each do |file| - expect(file.path.start_with?('uploads/')).to be_truthy - end - end - - it 'does not add hashed files to the untracked_files_for_uploads table' do - described_class.new.perform - - hashed_file_path = get_uploads(project2, 'Project').find_by(uploader: 'FileUploader').path - expect(untracked_files_for_uploads.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey - end - - it 'correctly schedules the follow-up background migration jobs' do - described_class.new.perform - - ids = described_class::UntrackedFile.all.order(:id).pluck(:id) - expect(described_class::FOLLOW_UP_MIGRATION).to be_scheduled_migration(ids.first, ids.last) - expect(BackgroundMigrationWorker.jobs.size).to eq(1) - end - - # E.g. from a previous failed run of this background migration - context 'when there is existing data in untracked_files_for_uploads' do - before do - described_class.new.perform - end - - it 'does not error or produce duplicates of existing data' do - expect do - described_class.new.perform - end.not_to change { untracked_files_for_uploads.count }.from(5) - end - end - - # E.g. The installation is in use at the time of migration, and someone has - # just uploaded a file - context 'when there are files in /uploads/tmp' do - let(:tmp_file) { Rails.root.join(described_class::ABSOLUTE_UPLOAD_DIR, 'tmp', 'some_file.jpg') } - - before do - FileUtils.mkdir(File.dirname(tmp_file)) - FileUtils.touch(tmp_file) - end - - after do - FileUtils.rm(tmp_file) - end - - it 'does not add files from /uploads/tmp' do - described_class.new.perform - - expect(untracked_files_for_uploads.count).to eq(5) - end - end - - context 'when the last batch size exactly matches the max batch size' do - it 'does not raise error' do - stub_const("#{described_class}::FIND_BATCH_SIZE", 5) - - expect do - described_class.new.perform - end.not_to raise_error - - expect(untracked_files_for_uploads.count).to eq(5) - end - end - end - end - - # If running on Postgres 9.2 (like on CI), this whole context is skipped - # since we're unable to use ON CONFLICT DO NOTHING or IGNORE. - context "test bulk insert with ON CONFLICT DO NOTHING or IGNORE", if: described_class.new.send(:can_bulk_insert_and_ignore_duplicates?) do - it_behaves_like 'prepares the untracked_files_for_uploads table' - end - - # If running on Postgres 9.2 (like on CI), the stubbed method has no effect. - # - # If running on Postgres 9.5+ or MySQL, then this context effectively tests - # the bulk insert functionality without ON CONFLICT DO NOTHING or IGNORE. - context 'test bulk insert without ON CONFLICT DO NOTHING or IGNORE' do - before do - allow_any_instance_of(described_class).to receive(:postgresql_pre_9_5?).and_return(true) - end - - it_behaves_like 'prepares the untracked_files_for_uploads table' - end - - # Very new or lightly-used installations that are running this migration - # may not have an upload directory because they have no uploads. - context 'when no files were ever uploaded' do - it 'deletes the `untracked_files_for_uploads` table (and does not raise error)' do - background_migration = described_class.new - - expect(background_migration).to receive(:drop_temp_table) - - background_migration.perform - end - end -end diff --git a/spec/lib/gitlab/background_migration/remove_restricted_todos_spec.rb b/spec/lib/gitlab/background_migration/remove_restricted_todos_spec.rb deleted file mode 100644 index 7019d5d4212..00000000000 --- a/spec/lib/gitlab/background_migration/remove_restricted_todos_spec.rb +++ /dev/null @@ -1,126 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::RemoveRestrictedTodos do - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:users) { table(:users) } - let(:todos) { table(:todos) } - let(:issues) { table(:issues) } - let(:assignees) { table(:issue_assignees) } - let(:project_authorizations) { table(:project_authorizations) } - let(:project_features) { table(:project_features) } - - let(:todo_params) { { author_id: 1, target_type: 'Issue', action: 1, state: :pending } } - - before do - users.create(id: 1, email: 'user@example.com', projects_limit: 10) - users.create(id: 2, email: 'reporter@example.com', projects_limit: 10) - users.create(id: 3, email: 'guest@example.com', projects_limit: 10) - - namespace = namespaces.create(name: 'gitlab-org', path: 'gitlab-org') - projects.create!(id: 1, name: 'project-1', path: 'project-1', visibility_level: 0, namespace_id: namespace.id) - projects.create!(id: 2, name: 'project-2', path: 'project-2', visibility_level: 0, namespace_id: namespace.id) - - issues.create(id: 1, project_id: 1) - issues.create(id: 2, project_id: 2) - - project_authorizations.create(user_id: 2, project_id: 2, access_level: 20) # reporter - project_authorizations.create(user_id: 3, project_id: 2, access_level: 10) # guest - - todos.create(todo_params.merge(user_id: 1, project_id: 1, target_id: 1)) # out of project ids range - todos.create(todo_params.merge(user_id: 1, project_id: 2, target_id: 2)) # non member - todos.create(todo_params.merge(user_id: 2, project_id: 2, target_id: 2)) # reporter - todos.create(todo_params.merge(user_id: 3, project_id: 2, target_id: 2)) # guest - end - - subject { described_class.new.perform(2, 5) } - - context 'when a project is private' do - it 'removes todos of users without project access' do - expect { subject }.to change { Todo.count }.from(4).to(3) - end - - context 'with a confidential issue' do - it 'removes todos of users without project access and guests for confidential issues' do - issues.create(id: 3, project_id: 2, confidential: true) - issues.create(id: 4, project_id: 1, confidential: true) # not in the batch - todos.create(todo_params.merge(user_id: 3, project_id: 2, target_id: 3)) - todos.create(todo_params.merge(user_id: 2, project_id: 2, target_id: 3)) - todos.create(todo_params.merge(user_id: 1, project_id: 1, target_id: 4)) - - expect { subject }.to change { Todo.count }.from(7).to(5) - end - end - end - - context 'when a project is public' do - before do - projects.find(2).update_attribute(:visibility_level, 20) - end - - context 'when all features have the same visibility as the project, no confidential issues' do - it 'does not remove any todos' do - expect { subject }.not_to change { Todo.count } - end - end - - context 'with confidential issues' do - before do - users.create(id: 4, email: 'author@example.com', projects_limit: 10) - users.create(id: 5, email: 'assignee@example.com', projects_limit: 10) - issues.create(id: 3, project_id: 2, confidential: true, author_id: 4) - assignees.create(user_id: 5, issue_id: 3) - - todos.create(todo_params.merge(user_id: 1, project_id: 2, target_id: 3)) # to be deleted - todos.create(todo_params.merge(user_id: 2, project_id: 2, target_id: 3)) # authorized user - todos.create(todo_params.merge(user_id: 3, project_id: 2, target_id: 3)) # to be deleted guest - todos.create(todo_params.merge(user_id: 4, project_id: 2, target_id: 3)) # conf issue author - todos.create(todo_params.merge(user_id: 5, project_id: 2, target_id: 3)) # conf issue assignee - end - - it 'removes confidential issue todos for non authorized users' do - expect { subject }.to change { Todo.count }.from(9).to(7) - end - end - - context 'features visibility restrictions' do - before do - todo_params.merge!(project_id: 2, user_id: 1, target_id: 3) - todos.create(todo_params.merge(user_id: 1, target_id: 3, target_type: 'MergeRequest')) - todos.create(todo_params.merge(user_id: 1, target_id: 3, target_type: 'Commit')) - end - - context 'when issues are restricted to project members' do - before do - project_features.create(issues_access_level: 10, pages_access_level: 10, project_id: 2) - end - - it 'removes non members issue todos' do - expect { subject }.to change { Todo.count }.from(6).to(5) - end - end - - context 'when merge requests are restricted to project members' do - before do - project_features.create(merge_requests_access_level: 10, pages_access_level: 10, project_id: 2) - end - - it 'removes non members issue todos' do - expect { subject }.to change { Todo.count }.from(6).to(5) - end - end - - context 'when repository and merge requests are restricted to project members' do - before do - project_features.create(repository_access_level: 10, merge_requests_access_level: 10, pages_access_level: 10, project_id: 2) - end - - it 'removes non members commit and merge requests todos' do - expect { subject }.to change { Todo.count }.from(6).to(4) - end - end - end - end -end diff --git a/spec/lib/gitlab/background_migration/set_confidential_note_events_on_services_spec.rb b/spec/lib/gitlab/background_migration/set_confidential_note_events_on_services_spec.rb deleted file mode 100644 index 364edf3ed2a..00000000000 --- a/spec/lib/gitlab/background_migration/set_confidential_note_events_on_services_spec.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::SetConfidentialNoteEventsOnServices do - let(:services) { table(:services) } - - describe '#perform' do - it 'migrates services where note_events is true' do - service = services.create(confidential_note_events: nil, note_events: true) - - subject.perform(service.id, service.id) - - expect(service.reload.confidential_note_events).to eq(true) - end - - it 'ignores services where note_events is false' do - service = services.create(confidential_note_events: nil, note_events: false) - - subject.perform(service.id, service.id) - - expect(service.reload.confidential_note_events).to eq(nil) - end - - it 'ignores services where confidential_note_events has already been set' do - service = services.create(confidential_note_events: false, note_events: true) - - subject.perform(service.id, service.id) - - expect(service.reload.confidential_note_events).to eq(false) - end - end -end diff --git a/spec/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks_spec.rb b/spec/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks_spec.rb deleted file mode 100644 index 28b06ac3ba3..00000000000 --- a/spec/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks_spec.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::SetConfidentialNoteEventsOnWebhooks do - let(:web_hooks) { table(:web_hooks) } - - describe '#perform' do - it 'migrates hooks where note_events is true' do - hook = web_hooks.create(confidential_note_events: nil, note_events: true) - - subject.perform(hook.id, hook.id) - - expect(hook.reload.confidential_note_events).to eq(true) - end - - it 'ignores hooks where note_events is false' do - hook = web_hooks.create(confidential_note_events: nil, note_events: false) - - subject.perform(hook.id, hook.id) - - expect(hook.reload.confidential_note_events).to eq(nil) - end - - it 'ignores hooks where confidential_note_events has already been set' do - hook = web_hooks.create(confidential_note_events: false, note_events: true) - - subject.perform(hook.id, hook.id) - - expect(hook.reload.confidential_note_events).to eq(false) - end - end -end diff --git a/spec/lib/gitlab/background_migration/set_merge_request_diff_files_count_spec.rb b/spec/lib/gitlab/background_migration/set_merge_request_diff_files_count_spec.rb new file mode 100644 index 00000000000..6e9f51f510a --- /dev/null +++ b/spec/lib/gitlab/background_migration/set_merge_request_diff_files_count_spec.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::SetMergeRequestDiffFilesCount, schema: 20200807152315 do + let(:merge_request_diff_files) { table(:merge_request_diff_files) } + let(:merge_request_diffs) { table(:merge_request_diffs) } + let(:merge_requests) { table(:merge_requests) } + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + + let(:namespace) { namespaces.create!(name: 'foo', path: 'foo') } + let(:project) { projects.create!(namespace_id: namespace.id) } + let(:merge_request) { merge_requests.create!(source_branch: 'x', target_branch: 'master', target_project_id: project.id) } + + it 'fills the files_count column' do + empty_diff = merge_request_diffs.create!(merge_request_id: merge_request.id) + filled_diff = merge_request_diffs.create!(merge_request_id: merge_request.id) + + 3.times do |n| + merge_request_diff_files.create!( + merge_request_diff_id: filled_diff.id, + relative_order: n, + new_file: false, + renamed_file: false, + deleted_file: false, + too_large: false, + a_mode: '', + b_mode: '', + old_path: '', + new_path: '' + ) + end + + described_class.new.perform(empty_diff.id, filled_diff.id) + + expect(empty_diff.reload.files_count).to eq(0) + expect(filled_diff.reload.files_count).to eq(3) + end +end diff --git a/spec/lib/gitlab/background_migration/set_null_external_diff_store_to_local_value_spec.rb b/spec/lib/gitlab/background_migration/set_null_external_diff_store_to_local_value_spec.rb new file mode 100644 index 00000000000..6079ad2dd2a --- /dev/null +++ b/spec/lib/gitlab/background_migration/set_null_external_diff_store_to_local_value_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' + +# The test setup must begin before +# 20200804041930_add_not_null_constraint_on_external_diff_store_to_merge_request_diffs.rb +# has run, or else we cannot insert a row with `NULL` `external_diff_store` to +# test against. +RSpec.describe Gitlab::BackgroundMigration::SetNullExternalDiffStoreToLocalValue, schema: 20200804035230 do + let!(:merge_request_diffs) { table(:merge_request_diffs) } + let!(:merge_requests) { table(:merge_requests) } + let!(:namespaces) { table(:namespaces) } + let!(:projects) { table(:projects) } + let!(:namespace) { namespaces.create!(name: 'foo', path: 'foo') } + let!(:project) { projects.create!(namespace_id: namespace.id) } + let!(:merge_request) { merge_requests.create!(source_branch: 'x', target_branch: 'master', target_project_id: project.id) } + + it 'correctly migrates nil external_diff_store to 1' do + external_diff_store_1 = merge_request_diffs.create!(external_diff_store: 1, merge_request_id: merge_request.id) + external_diff_store_2 = merge_request_diffs.create!(external_diff_store: 2, merge_request_id: merge_request.id) + external_diff_store_nil = merge_request_diffs.create!(external_diff_store: nil, merge_request_id: merge_request.id) + + described_class.new.perform(external_diff_store_1.id, external_diff_store_nil.id) + + external_diff_store_1.reload + external_diff_store_2.reload + external_diff_store_nil.reload + + expect(external_diff_store_1.external_diff_store).to eq(1) # unchanged + expect(external_diff_store_2.external_diff_store).to eq(2) # unchanged + expect(external_diff_store_nil.external_diff_store).to eq(1) # nil => 1 + end +end diff --git a/spec/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value_spec.rb b/spec/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value_spec.rb new file mode 100644 index 00000000000..40d41262fc7 --- /dev/null +++ b/spec/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' + +# The test setup must begin before +# 20200806004742_add_not_null_constraint_on_file_store_to_package_files.rb +# has run, or else we cannot insert a row with `NULL` `file_store` to +# test against. +RSpec.describe Gitlab::BackgroundMigration::SetNullPackageFilesFileStoreToLocalValue, schema: 20200806004232 do + let!(:packages_package_files) { table(:packages_package_files) } + let!(:packages_packages) { table(:packages_packages) } + let!(:projects) { table(:projects) } + let!(:namespaces) { table(:namespaces) } + let!(:namespace) { namespaces.create!(name: 'foo', path: 'foo') } + let!(:project) { projects.create!(namespace_id: namespace.id) } + let!(:package) { packages_packages.create!(project_id: project.id, name: 'bar', package_type: 1) } + + it 'correctly migrates nil file_store to 1' do + file_store_1 = packages_package_files.create!(file_store: 1, file_name: 'foo_1', file: 'foo_1', package_id: package.id) + file_store_2 = packages_package_files.create!(file_store: 2, file_name: 'foo_2', file: 'foo_2', package_id: package.id) + file_store_nil = packages_package_files.create!(file_store: nil, file_name: 'foo_nil', file: 'foo_nil', package_id: package.id) + + described_class.new.perform(file_store_1.id, file_store_nil.id) + + file_store_1.reload + file_store_2.reload + file_store_nil.reload + + expect(file_store_1.file_store).to eq(1) # unchanged + expect(file_store_2.file_store).to eq(2) # unchanged + expect(file_store_nil.file_store).to eq(1) # nil => 1 + end +end diff --git a/spec/lib/gitlab/background_migration/user_mentions/create_resource_user_mention_spec.rb b/spec/lib/gitlab/background_migration/user_mentions/create_resource_user_mention_spec.rb index 08a4bbe38ac..392b44d1a1f 100644 --- a/spec/lib/gitlab/background_migration/user_mentions/create_resource_user_mention_spec.rb +++ b/spec/lib/gitlab/background_migration/user_mentions/create_resource_user_mention_spec.rb @@ -75,6 +75,14 @@ RSpec.describe Gitlab::BackgroundMigration::UserMentions::CreateResourceUserMent let(:resource) { merge_request } it_behaves_like 'resource mentions migration', MigrateMergeRequestMentionsToDb, MergeRequest + + context 'when FF disabled' do + before do + stub_feature_flags(migrate_user_mentions: false) + end + + it_behaves_like 'resource migration not run', MigrateMergeRequestMentionsToDb, MergeRequest + end end context 'migrate commit mentions' do @@ -96,6 +104,14 @@ RSpec.describe Gitlab::BackgroundMigration::UserMentions::CreateResourceUserMent let(:resource) { commit } it_behaves_like 'resource notes mentions migration', MigrateCommitNotesMentionsToDb, Commit + + context 'when FF disabled' do + before do + stub_feature_flags(migrate_user_mentions: false) + end + + it_behaves_like 'resource notes migration not run', MigrateCommitNotesMentionsToDb, Commit + end end end diff --git a/spec/lib/gitlab/background_migration_spec.rb b/spec/lib/gitlab/background_migration_spec.rb index b110fa484ff..052a01a8dd8 100644 --- a/spec/lib/gitlab/background_migration_spec.rb +++ b/spec/lib/gitlab/background_migration_spec.rb @@ -130,6 +130,7 @@ RSpec.describe Gitlab::BackgroundMigration do let(:retry_queue) do [double(args: ['Object', [3]], queue: described_class.queue, delete: true)] end + let(:dead_queue) do [double(args: ['Object', [4]], queue: described_class.queue, delete: true)] end diff --git a/spec/lib/gitlab/badge/coverage/report_spec.rb b/spec/lib/gitlab/badge/coverage/report_spec.rb index 9c4dfcbfd54..4a9508712a4 100644 --- a/spec/lib/gitlab/badge/coverage/report_spec.rb +++ b/spec/lib/gitlab/badge/coverage/report_spec.rb @@ -102,7 +102,7 @@ RSpec.describe Gitlab::Badge::Coverage::Report do create(:ci_pipeline, opts).tap do |pipeline| yield pipeline - pipeline.update_legacy_status + ::Ci::ProcessPipelineService.new(pipeline).execute end end end diff --git a/spec/lib/gitlab/bitbucket_import/importer_spec.rb b/spec/lib/gitlab/bitbucket_import/importer_spec.rb index 08b7bafddf0..d4483bf1754 100644 --- a/spec/lib/gitlab/bitbucket_import/importer_spec.rb +++ b/spec/lib/gitlab/bitbucket_import/importer_spec.rb @@ -87,6 +87,7 @@ RSpec.describe Gitlab::BitbucketImport::Importer do values: sample_issues_statuses } end + let(:counter) { double('counter', increment: true) } subject { described_class.new(project) } @@ -109,6 +110,7 @@ RSpec.describe Gitlab::BitbucketImport::Importer do created_at: Time.now, updated_at: Time.now) end + let(:author_line) { "*Created by: someuser*\n\n" } before do diff --git a/spec/lib/gitlab/build_access_spec.rb b/spec/lib/gitlab/build_access_spec.rb index c6248f94772..4a1c172a975 100644 --- a/spec/lib/gitlab/build_access_spec.rb +++ b/spec/lib/gitlab/build_access_spec.rb @@ -7,7 +7,7 @@ RSpec.describe Gitlab::BuildAccess do let(:project) { create(:project) } describe '#can_do_action' do - subject { described_class.new(user, project: project).can_do_action?(:download_code) } + subject { described_class.new(user, container: project).can_do_action?(:download_code) } context 'when the user can do an action on the project but cannot access git' do before do diff --git a/spec/lib/gitlab/checks/change_access_spec.rb b/spec/lib/gitlab/checks/change_access_spec.rb index 87936d19239..6f82dabb285 100644 --- a/spec/lib/gitlab/checks/change_access_spec.rb +++ b/spec/lib/gitlab/checks/change_access_spec.rb @@ -3,14 +3,14 @@ require 'spec_helper' RSpec.describe Gitlab::Checks::ChangeAccess do - describe '#exec' do + describe '#validate!' do include_context 'change access checks context' subject { change_access } context 'without failed checks' do it "doesn't raise an error" do - expect { subject.exec }.not_to raise_error + expect { subject.validate! }.not_to raise_error end it 'calls pushes checks' do @@ -18,7 +18,7 @@ RSpec.describe Gitlab::Checks::ChangeAccess do expect(instance).to receive(:validate!) end - subject.exec + subject.validate! end it 'calls branches checks' do @@ -26,7 +26,7 @@ RSpec.describe Gitlab::Checks::ChangeAccess do expect(instance).to receive(:validate!) end - subject.exec + subject.validate! end it 'calls tags checks' do @@ -34,7 +34,7 @@ RSpec.describe Gitlab::Checks::ChangeAccess do expect(instance).to receive(:validate!) end - subject.exec + subject.validate! end it 'calls lfs checks' do @@ -42,7 +42,7 @@ RSpec.describe Gitlab::Checks::ChangeAccess do expect(instance).to receive(:validate!) end - subject.exec + subject.validate! end it 'calls diff checks' do @@ -50,7 +50,7 @@ RSpec.describe Gitlab::Checks::ChangeAccess do expect(instance).to receive(:validate!) end - subject.exec + subject.validate! end end @@ -63,7 +63,7 @@ RSpec.describe Gitlab::Checks::ChangeAccess do protocol: protocol, logger: logger) - expect { access.exec }.to raise_error(Gitlab::Checks::TimedLogger::TimeoutError) + expect { access.validate! }.to raise_error(Gitlab::Checks::TimedLogger::TimeoutError) end end end diff --git a/spec/lib/gitlab/ci/ansi2html_spec.rb b/spec/lib/gitlab/ci/ansi2html_spec.rb index f29a39e4e66..bf1f2bae7da 100644 --- a/spec/lib/gitlab/ci/ansi2html_spec.rb +++ b/spec/lib/gitlab/ci/ansi2html_spec.rb @@ -213,6 +213,7 @@ RSpec.describe Gitlab::Ci::Ansi2html do " data-timestamp=\"#{section_start_time.to_i}\" data-section=\"#{class_name(section_name)}\"" \ ' role="button"></div>' end + let(:section_end_html) do "<div class=\"section-end\" data-section=\"#{class_name(section_name)}\"></div>" end diff --git a/spec/lib/gitlab/ci/build/artifacts/expire_in_parser_spec.rb b/spec/lib/gitlab/ci/build/artifacts/expire_in_parser_spec.rb new file mode 100644 index 00000000000..0e26a9fa571 --- /dev/null +++ b/spec/lib/gitlab/ci/build/artifacts/expire_in_parser_spec.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Build::Artifacts::ExpireInParser do + describe '.validate_duration' do + subject { described_class.validate_duration(value) } + + context 'with never' do + let(:value) { 'never' } + + it { is_expected.to be_truthy } + end + + context 'with never value camelized' do + let(:value) { 'Never' } + + it { is_expected.to be_truthy } + end + + context 'with a duration' do + let(:value) { '1 Day' } + + it { is_expected.to be_truthy } + end + + context 'without a duration' do + let(:value) { 'something' } + + it { is_expected.to be_falsy } + end + end + + describe '#seconds_from_now' do + subject { described_class.new(value).seconds_from_now } + + context 'with never' do + let(:value) { 'never' } + + it { is_expected.to be_nil } + end + + context 'with an empty string' do + let(:value) { '' } + + it { is_expected.to be_nil } + end + + context 'with a duration' do + let(:value) { '1 day' } + + it { is_expected.to be_like_time(1.day.from_now) } + end + end +end diff --git a/spec/lib/gitlab/ci/build/auto_retry_spec.rb b/spec/lib/gitlab/ci/build/auto_retry_spec.rb new file mode 100644 index 00000000000..cfa8c9cd938 --- /dev/null +++ b/spec/lib/gitlab/ci/build/auto_retry_spec.rb @@ -0,0 +1,127 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Build::AutoRetry do + let(:auto_retry) { described_class.new(build) } + + describe '#allowed?' do + using RSpec::Parameterized::TableSyntax + + let(:build) { create(:ci_build) } + + subject { auto_retry.allowed? } + + where(:description, :retry_count, :options, :failure_reason, :result) do + "retries are disabled" | 0 | { max: 0 } | nil | false + "max equals count" | 2 | { max: 2 } | nil | false + "max is higher than count" | 1 | { max: 2 } | nil | true + "max is a string" | 1 | { max: '2' } | nil | true + "matching failure reason" | 0 | { when: %w[api_failure], max: 2 } | :api_failure | true + "not matching with always" | 0 | { when: %w[always], max: 2 } | :api_failure | true + "not matching reason" | 0 | { when: %w[script_error], max: 2 } | :api_failure | false + "scheduler failure override" | 1 | { when: %w[scheduler_failure], max: 1 } | :scheduler_failure | false + "default for scheduler failure" | 1 | {} | :scheduler_failure | true + end + + with_them do + before do + allow(build).to receive(:retries_count) { retry_count } + + build.options[:retry] = options + build.failure_reason = failure_reason + allow(build).to receive(:retryable?).and_return(true) + end + + it { is_expected.to eq(result) } + end + + context 'when build is not retryable' do + before do + allow(build).to receive(:retryable?).and_return(false) + end + + specify { expect(subject).to eq(false) } + end + end + + describe '#options_retry_max' do + subject(:result) { auto_retry.send(:options_retry_max) } + + context 'with retries max config option' do + let(:build) { create(:ci_build, options: { retry: { max: 1 } }) } + + context 'when build_metadata_config is set' do + before do + stub_feature_flags(ci_build_metadata_config: true) + end + + it 'returns the number of configured max retries' do + expect(result).to eq 1 + end + end + + context 'when build_metadata_config is not set' do + before do + stub_feature_flags(ci_build_metadata_config: false) + end + + it 'returns the number of configured max retries' do + expect(result).to eq 1 + end + end + end + + context 'without retries max config option' do + let(:build) { create(:ci_build) } + + it 'returns nil' do + expect(result).to be_nil + end + end + + context 'when build is degenerated' do + let(:build) { create(:ci_build, :degenerated) } + + it 'returns nil' do + expect(result).to be_nil + end + end + + context 'with integer only config option' do + let(:build) { create(:ci_build, options: { retry: 1 }) } + + it 'returns the number of configured max retries' do + expect(result).to eq 1 + end + end + end + + describe '#options_retry_when' do + subject(:result) { auto_retry.send(:options_retry_when) } + + context 'with retries when config option' do + let(:build) { create(:ci_build, options: { retry: { when: ['some_reason'] } }) } + + it 'returns the configured when' do + expect(result).to eq ['some_reason'] + end + end + + context 'without retries when config option' do + let(:build) { create(:ci_build) } + + it 'returns always array' do + expect(result).to eq ['always'] + end + end + + context 'with integer only config option' do + let(:build) { create(:ci_build, options: { retry: 1 }) } + + it 'returns always array' do + expect(result).to eq ['always'] + end + end + end +end diff --git a/spec/lib/gitlab/ci/config/entry/job_spec.rb b/spec/lib/gitlab/ci/config/entry/job_spec.rb index 180c52ee1ab..ca02eaee0a0 100644 --- a/spec/lib/gitlab/ci/config/entry/job_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/job_spec.rb @@ -30,7 +30,7 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job do %i[before_script script stage type after_script cache image services only except rules needs variables artifacts environment coverage retry interruptible timeout release tags - inherit] + inherit parallel] end it { is_expected.to include(*result) } @@ -73,6 +73,45 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job do it { is_expected.to be_falsey } end + + context 'when config does not contain script' do + let(:name) { :build } + + let(:config) do + { before_script: "cd ${PROJ_DIR} " } + end + + it { is_expected.to be_truthy } + end + + context 'when using the default job without script' do + let(:name) { :default } + let(:config) do + { before_script: "cd ${PROJ_DIR} " } + end + + it { is_expected.to be_falsey } + end + + context 'when using the default job with script' do + let(:name) { :default } + let(:config) do + { + before_script: "cd ${PROJ_DIR} ", + script: "ls" + } + end + + it { is_expected.to be_truthy } + end + + context 'there are no shared keys between jobs and bridges' do + subject(:shared_values) do + described_class::ALLOWED_KEYS & Gitlab::Ci::Config::Entry::Bridge::ALLOWED_KEYS + end + + it { is_expected.to be_empty } + end end describe 'validations' do @@ -202,56 +241,47 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job do context 'when parallel value is not correct' do context 'when it is not a numeric value' do - let(:config) { { parallel: true } } + let(:config) { { script: 'echo', parallel: true } } it 'returns error about invalid type' do expect(entry).not_to be_valid - expect(entry.errors).to include 'job parallel is not a number' + expect(entry.errors).to include 'parallel should be an integer or a hash' end end context 'when it is lower than two' do - let(:config) { { parallel: 1 } } + let(:config) { { script: 'echo', parallel: 1 } } it 'returns error about value too low' do expect(entry).not_to be_valid expect(entry.errors) - .to include 'job parallel must be greater than or equal to 2' + .to include 'parallel config must be greater than or equal to 2' end end - context 'when it is bigger than 50' do - let(:config) { { parallel: 51 } } + context 'when it is an empty hash' do + let(:config) { { script: 'echo', parallel: {} } } - it 'returns error about value too high' do + it 'returns error about missing matrix' do expect(entry).not_to be_valid expect(entry.errors) - .to include 'job parallel must be less than or equal to 50' + .to include 'parallel config missing required keys: matrix' end end + end - context 'when it is not an integer' do - let(:config) { { parallel: 1.5 } } - - it 'returns error about wrong value' do - expect(entry).not_to be_valid - expect(entry.errors).to include 'job parallel must be an integer' - end + context 'when it uses both "when:" and "rules:"' do + let(:config) do + { + script: 'echo', + when: 'on_failure', + rules: [{ if: '$VARIABLE', when: 'on_success' }] + } end - context 'when it uses both "when:" and "rules:"' do - let(:config) do - { - script: 'echo', - when: 'on_failure', - rules: [{ if: '$VARIABLE', when: 'on_success' }] - } - end - - it 'returns an error about when: being combined with rules' do - expect(entry).not_to be_valid - expect(entry.errors).to include 'job config key may not be used with `rules`: when' - end + it 'returns an error about when: being combined with rules' do + expect(entry).not_to be_valid + expect(entry.errors).to include 'job config key may not be used with `rules`: when' end end diff --git a/spec/lib/gitlab/ci/config/entry/processable_spec.rb b/spec/lib/gitlab/ci/config/entry/processable_spec.rb index fdf6008f89f..ac8dd2a3267 100644 --- a/spec/lib/gitlab/ci/config/entry/processable_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/processable_spec.rb @@ -230,6 +230,12 @@ RSpec.describe Gitlab::Ci::Config::Entry::Processable do end end + shared_examples 'has no warnings' do + it 'does not raise the warning' do + expect(entry.warnings).to be_empty + end + end + context 'when workflow rules is used' do let(:workflow) { double('workflow', 'has_rules?' => true) } @@ -254,6 +260,86 @@ RSpec.describe Gitlab::Ci::Config::Entry::Processable do end end + context 'when workflow rules is not used' do + let(:workflow) { double('workflow', 'has_rules?' => false) } + let(:feature_flag_value) { true } + + before do + stub_feature_flags(ci_raise_job_rules_without_workflow_rules_warning: feature_flag_value) + entry.compose!(deps) + end + + context 'when rules are valid' do + let(:config) do + { + script: 'ls', + rules: [ + { if: '$CI_COMMIT_BRANCH', when: 'on_success' }, + last_rule + ] + } + end + + context 'when last rule contains only `when`' do + let(:last_rule) { { when: when_value } } + + context 'and its value is not `never`' do + let(:when_value) { 'on_success' } + + it 'raises a warning' do + expect(entry.warnings).to contain_exactly(/may allow multiple pipelines/) + end + + context 'when feature flag is disabled' do + let(:feature_flag_value) { false } + + it_behaves_like 'has no warnings' + end + end + + context 'and its value is `never`' do + let(:when_value) { 'never' } + + it_behaves_like 'has no warnings' + end + end + + context 'when last rule does not contain only `when`' do + let(:last_rule) { { if: '$CI_MERGE_REQUEST_ID', when: 'always' } } + + it_behaves_like 'has no warnings' + end + end + + context 'when rules are invalid' do + let(:config) { { script: 'ls', rules: { when: 'always' } } } + + it_behaves_like 'has no warnings' + end + end + + context 'when workflow rules is used' do + let(:workflow) { double('workflow', 'has_rules?' => true) } + + before do + entry.compose!(deps) + end + + context 'when last rule contains only `when' do + let(:config) do + { + script: 'ls', + rules: [ + { if: '$CI_COMMIT_BRANCH', when: 'on_success' }, + { when: 'always' } + ] + } + end + + it_behaves_like 'has no warnings' + end + end + context 'with inheritance' do context 'of variables' do let(:config) do diff --git a/spec/lib/gitlab/ci/config/entry/product/matrix_spec.rb b/spec/lib/gitlab/ci/config/entry/product/matrix_spec.rb new file mode 100644 index 00000000000..39697884e3b --- /dev/null +++ b/spec/lib/gitlab/ci/config/entry/product/matrix_spec.rb @@ -0,0 +1,188 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' +require_dependency 'active_model' + +RSpec.describe ::Gitlab::Ci::Config::Entry::Product::Matrix do + subject(:matrix) { described_class.new(config) } + + describe 'validations' do + before do + matrix.compose! + end + + context 'when entry config value is correct' do + let(:config) do + [ + { 'VAR_1' => [1, 2, 3], 'VAR_2' => [4, 5, 6] }, + { 'VAR_3' => %w[a b], 'VAR_4' => %w[c d] } + ] + end + + describe '#valid?' do + it { is_expected.to be_valid } + end + end + + context 'when entry config generates too many jobs' do + let(:config) do + [ + { + 'VAR_1' => (1..10).to_a, + 'VAR_2' => (11..20).to_a + } + ] + end + + describe '#valid?' do + it { is_expected.not_to be_valid } + end + + describe '#errors' do + it 'returns error about too many jobs' do + expect(matrix.errors) + .to include('matrix config generates too many jobs (maximum is 50)') + end + end + end + + context 'when entry config has only one variable' do + let(:config) do + [ + { + 'VAR_1' => %w[test] + } + ] + end + + describe '#valid?' do + it { is_expected.not_to be_valid } + end + + describe '#errors' do + it 'returns error about too many jobs' do + expect(matrix.errors) + .to include('variables config requires at least 2 items') + end + end + + describe '#value' do + before do + matrix.compose! + end + + it 'returns the value without raising an error' do + expect(matrix.value).to eq([{ 'VAR_1' => ['test'] }]) + end + end + end + + context 'when config value has wrong type' do + let(:config) { {} } + + describe '#valid?' do + it { is_expected.not_to be_valid } + end + + describe '#errors' do + it 'returns error about incorrect type' do + expect(matrix.errors) + .to include('matrix config should be an array of hashes') + end + end + end + end + + describe '.compose!' do + context 'when valid job entries composed' do + let(:config) do + [ + { PROVIDER: 'aws', STACK: %w[monitoring app1 app2] }, + { STACK: %w[monitoring backup app], PROVIDER: 'ovh' }, + { PROVIDER: 'gcp', STACK: %w[data processing], ARGS: 'normal' }, + { PROVIDER: 'vultr', STACK: 'data', ARGS: 'store' } + ] + end + + before do + matrix.compose! + end + + describe '#value' do + it 'returns key value' do + expect(matrix.value).to match( + [ + { 'PROVIDER' => %w[aws], 'STACK' => %w[monitoring app1 app2] }, + { 'PROVIDER' => %w[ovh], 'STACK' => %w[monitoring backup app] }, + { 'ARGS' => %w[normal], 'PROVIDER' => %w[gcp], 'STACK' => %w[data processing] }, + { 'ARGS' => %w[store], 'PROVIDER' => %w[vultr], 'STACK' => %w[data] } + ] + ) + end + end + + describe '#descendants' do + it 'creates valid descendant nodes' do + expect(matrix.descendants.count).to eq(config.size) + expect(matrix.descendants) + .to all(be_an_instance_of(::Gitlab::Ci::Config::Entry::Product::Variables)) + end + end + end + + context 'with empty config' do + let(:config) { [] } + + before do + matrix.compose! + end + + describe '#value' do + it 'returns empty value' do + expect(matrix.value).to eq([]) + end + end + end + end + + describe '#number_of_generated_jobs' do + before do + matrix.compose! + end + + subject { matrix.number_of_generated_jobs } + + context 'with empty config' do + let(:config) { [] } + + it { is_expected.to be_zero } + end + + context 'with only one variable' do + let(:config) do + [{ 'VAR_1' => (1..10).to_a }] + end + + it { is_expected.to eq(10) } + end + + context 'with two variables' do + let(:config) do + [{ 'VAR_1' => (1..10).to_a, 'VAR_2' => (1..5).to_a }] + end + + it { is_expected.to eq(50) } + end + + context 'with two sets of variables' do + let(:config) do + [ + { 'VAR_1' => (1..10).to_a, 'VAR_2' => (1..5).to_a }, + { 'VAR_3' => (1..2).to_a, 'VAR_4' => (1..3).to_a } + ] + end + + it { is_expected.to eq(56) } + end + end +end diff --git a/spec/lib/gitlab/ci/config/entry/product/parallel_spec.rb b/spec/lib/gitlab/ci/config/entry/product/parallel_spec.rb new file mode 100644 index 00000000000..bc09e20d748 --- /dev/null +++ b/spec/lib/gitlab/ci/config/entry/product/parallel_spec.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' +require_dependency 'active_model' + +RSpec.describe ::Gitlab::Ci::Config::Entry::Product::Parallel do + subject(:parallel) { described_class.new(config) } + + context 'with invalid config' do + shared_examples 'invalid config' do |error_message| + describe '#valid?' do + it { is_expected.not_to be_valid } + end + + describe '#errors' do + it 'returns error about invalid type' do + expect(parallel.errors).to match(a_collection_including(error_message)) + end + end + end + + context 'when it is not a numeric value' do + let(:config) { true } + + it_behaves_like 'invalid config', /should be an integer or a hash/ + end + + context 'when it is lower than two' do + let(:config) { 1 } + + it_behaves_like 'invalid config', /must be greater than or equal to 2/ + end + + context 'when it is bigger than 50' do + let(:config) { 51 } + + it_behaves_like 'invalid config', /must be less than or equal to 50/ + end + + context 'when it is not an integer' do + let(:config) { 1.5 } + + it_behaves_like 'invalid config', /must be an integer/ + end + + context 'with empty hash config' do + let(:config) { {} } + + it_behaves_like 'invalid config', /matrix builds config missing required keys: matrix/ + end + end + + context 'with numeric config' do + context 'when job is specified' do + let(:config) { 2 } + + describe '#valid?' do + it { is_expected.to be_valid } + end + + describe '#value' do + it 'returns job needs configuration' do + expect(parallel.value).to match(number: config) + end + end + end + end + + context 'with matrix builds config' do + context 'when matrix is specified' do + let(:config) do + { + matrix: [ + { PROVIDER: 'aws', STACK: %w[monitoring app1 app2] }, + { PROVIDER: 'gcp', STACK: %w[data processing] } + ] + } + end + + describe '#valid?' do + it { is_expected.to be_valid } + end + + describe '#value' do + it 'returns job needs configuration' do + expect(parallel.value).to match(matrix: [ + { PROVIDER: 'aws', STACK: %w[monitoring app1 app2] }, + { PROVIDER: 'gcp', STACK: %w[data processing] } + ]) + end + end + end + end +end diff --git a/spec/lib/gitlab/ci/config/entry/product/variables_spec.rb b/spec/lib/gitlab/ci/config/entry/product/variables_spec.rb new file mode 100644 index 00000000000..230b001d620 --- /dev/null +++ b/spec/lib/gitlab/ci/config/entry/product/variables_spec.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' +require_dependency 'active_model' + +RSpec.describe Gitlab::Ci::Config::Entry::Product::Variables do + let(:entry) { described_class.new(config) } + + describe 'validations' do + context 'when entry config value is correct' do + let(:config) do + { + 'VARIABLE_1' => 1, + 'VARIABLE_2' => 'value 2', + 'VARIABLE_3' => :value_3, + :VARIABLE_4 => 'value 4', + 5 => ['value 5'], + 'VARIABLE_6' => ['value 6'] + } + end + + describe '#value' do + it 'returns hash with key value strings' do + expect(entry.value).to match({ + 'VARIABLE_1' => ['1'], + 'VARIABLE_2' => ['value 2'], + 'VARIABLE_3' => ['value_3'], + 'VARIABLE_4' => ['value 4'], + '5' => ['value 5'], + 'VARIABLE_6' => ['value 6'] + }) + end + end + + describe '#errors' do + it 'does not append errors' do + expect(entry.errors).to be_empty + end + end + + describe '#valid?' do + it 'is valid' do + expect(entry).to be_valid + end + end + end + + context 'when entry value is not correct' do + shared_examples 'invalid variables' do |message| + describe '#errors' do + it 'saves errors' do + expect(entry.errors).to include(message) + end + end + + describe '#valid?' do + it 'is not valid' do + expect(entry).not_to be_valid + end + end + end + + context 'with array' do + let(:config) { [:VAR, 'test'] } + + it_behaves_like 'invalid variables', /should be a hash of key value pairs/ + end + + context 'with empty array' do + let(:config) { { VAR: 'test', VAR2: [] } } + + it_behaves_like 'invalid variables', /should be a hash of key value pairs/ + end + + context 'with nested array' do + let(:config) { { VAR: 'test', VAR2: [1, [2]] } } + + it_behaves_like 'invalid variables', /should be a hash of key value pairs/ + end + + context 'with only one variable' do + let(:config) { { VAR: 'test' } } + + it_behaves_like 'invalid variables', /variables config requires at least 2 items/ + end + end + end +end diff --git a/spec/lib/gitlab/ci/config/entry/service_spec.rb b/spec/lib/gitlab/ci/config/entry/service_spec.rb index 9fbc14c19b9..ec137ef2ae4 100644 --- a/spec/lib/gitlab/ci/config/entry/service_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/service_spec.rb @@ -95,6 +95,7 @@ RSpec.describe Gitlab::Ci::Config::Entry::Service do let(:config) do { name: 'postgresql:9.5', alias: 'db', command: %w(cmd run), entrypoint: %w(/bin/sh run), ports: ports } end + let(:entry) { described_class.new(config, { with_image_ports: image_ports }) } let(:image_ports) { false } diff --git a/spec/lib/gitlab/ci/config/external/file/local_spec.rb b/spec/lib/gitlab/ci/config/external/file/local_spec.rb index 993a07568de..fdd29afe2d6 100644 --- a/spec/lib/gitlab/ci/config/external/file/local_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/local_spec.rb @@ -92,6 +92,7 @@ RSpec.describe Gitlab::Ci::Config::External::File::Local do - bundle install --jobs $(nproc) "${FLAGS[@]}" HEREDOC end + let(:location) { '/lib/gitlab/ci/templates/existent-file.yml' } before do diff --git a/spec/lib/gitlab/ci/config/external/processor_spec.rb b/spec/lib/gitlab/ci/config/external/processor_spec.rb index b2cf36b2597..9786e050399 100644 --- a/spec/lib/gitlab/ci/config/external/processor_spec.rb +++ b/spec/lib/gitlab/ci/config/external/processor_spec.rb @@ -128,6 +128,7 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do remote_file ] end + let(:values) do { include: external_files, diff --git a/spec/lib/gitlab/ci/config/normalizer/factory_spec.rb b/spec/lib/gitlab/ci/config/normalizer/factory_spec.rb new file mode 100644 index 00000000000..e355740222f --- /dev/null +++ b/spec/lib/gitlab/ci/config/normalizer/factory_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Ci::Config::Normalizer::Factory do + describe '#create' do + context 'when no strategy applies' do + subject(:subject) { described_class.new(nil, nil).create } # rubocop:disable Rails/SaveBang + + it { is_expected.to be_empty } + end + end +end diff --git a/spec/lib/gitlab/ci/config/normalizer/matrix_strategy_spec.rb b/spec/lib/gitlab/ci/config/normalizer/matrix_strategy_spec.rb new file mode 100644 index 00000000000..bab604c4504 --- /dev/null +++ b/spec/lib/gitlab/ci/config/normalizer/matrix_strategy_spec.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Ci::Config::Normalizer::MatrixStrategy do + describe '.applies_to?' do + subject { described_class.applies_to?(config) } + + context 'with hash that has :matrix key' do + let(:config) { { matrix: [] } } + + it { is_expected.to be_truthy } + end + + context 'with hash that does not have :matrix key' do + let(:config) { { number: [] } } + + it { is_expected.to be_falsey } + end + + context 'with a number' do + let(:config) { 5 } + + it { is_expected.to be_falsey } + end + end + + describe '.build_from' do + subject { described_class.build_from('test', config) } + + let(:config) do + { + matrix: [ + { 'PROVIDER' => %w[aws], 'STACK' => %w[app1 app2] }, + { 'PROVIDER' => %w[ovh gcp], 'STACK' => %w[app] } + ] + } + end + + it { expect(subject.size).to eq(4) } + + it 'has attributes' do + expect(subject.map(&:attributes)).to match_array( + [ + { + name: 'test 1/4', + instance: 1, + parallel: { total: 4 }, + variables: { + 'PROVIDER' => 'aws', + 'STACK' => 'app1' + } + }, + { + name: 'test 2/4', + instance: 2, + parallel: { total: 4 }, + variables: { + 'PROVIDER' => 'aws', + 'STACK' => 'app2' + } + }, + { + name: 'test 3/4', + instance: 3, + parallel: { total: 4 }, + variables: { + 'PROVIDER' => 'ovh', + 'STACK' => 'app' + } + }, + { + name: 'test 4/4', + instance: 4, + parallel: { total: 4 }, + variables: { + 'PROVIDER' => 'gcp', + 'STACK' => 'app' + } + } + ] + ) + end + + it 'has parallelized name' do + expect(subject.map(&:name)).to match_array( + ['test 1/4', 'test 2/4', 'test 3/4', 'test 4/4'] + ) + end + + it 'has details' do + expect(subject.map(&:name_with_details)).to match_array( + [ + 'test (PROVIDER=aws; STACK=app1)', + 'test (PROVIDER=aws; STACK=app2)', + 'test (PROVIDER=gcp; STACK=app)', + 'test (PROVIDER=ovh; STACK=app)' + ] + ) + end + end +end diff --git a/spec/lib/gitlab/ci/config/normalizer/number_strategy_spec.rb b/spec/lib/gitlab/ci/config/normalizer/number_strategy_spec.rb new file mode 100644 index 00000000000..06f47fe11c6 --- /dev/null +++ b/spec/lib/gitlab/ci/config/normalizer/number_strategy_spec.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Ci::Config::Normalizer::NumberStrategy do + describe '.applies_to?' do + subject { described_class.applies_to?(config) } + + context 'with numbers' do + let(:config) { 5 } + + it { is_expected.to be_truthy } + end + + context 'with hash that has :number key' do + let(:config) { { number: 5 } } + + it { is_expected.to be_truthy } + end + + context 'with a float number' do + let(:config) { 5.5 } + + it { is_expected.to be_falsey } + end + + context 'with hash that does not have :number key' do + let(:config) { { matrix: 5 } } + + it { is_expected.to be_falsey } + end + end + + describe '.build_from' do + subject { described_class.build_from('test', config) } + + shared_examples 'parallelized job' do + it { expect(subject.size).to eq(3) } + + it 'has attributes' do + expect(subject.map(&:attributes)).to match_array( + [ + { name: 'test 1/3', instance: 1, parallel: { total: 3 } }, + { name: 'test 2/3', instance: 2, parallel: { total: 3 } }, + { name: 'test 3/3', instance: 3, parallel: { total: 3 } } + ] + ) + end + + it 'has parallelized name' do + expect(subject.map(&:name)).to match_array( + ['test 1/3', 'test 2/3', 'test 3/3']) + end + end + + context 'with numbers' do + let(:config) { 3 } + + it_behaves_like 'parallelized job' + end + + context 'with hash that has :number key' do + let(:config) { { number: 3 } } + + it_behaves_like 'parallelized job' + end + end +end diff --git a/spec/lib/gitlab/ci/config/normalizer_spec.rb b/spec/lib/gitlab/ci/config/normalizer_spec.rb index d3d165ba00f..949af8cdc4c 100644 --- a/spec/lib/gitlab/ci/config/normalizer_spec.rb +++ b/spec/lib/gitlab/ci/config/normalizer_spec.rb @@ -4,66 +4,13 @@ require 'fast_spec_helper' RSpec.describe Gitlab::Ci::Config::Normalizer do let(:job_name) { :rspec } - let(:job_config) { { script: 'rspec', parallel: 5, name: 'rspec' } } + let(:job_config) { { script: 'rspec', parallel: parallel_config, name: 'rspec', variables: variables_config } } let(:config) { { job_name => job_config } } - let(:expanded_job_names) do - [ - "rspec 1/5", - "rspec 2/5", - "rspec 3/5", - "rspec 4/5", - "rspec 5/5" - ] - end - describe '.normalize_jobs' do subject { described_class.new(config).normalize_jobs } - it 'does not have original job' do - is_expected.not_to include(job_name) - end - - it 'has parallelized jobs' do - is_expected.to include(*expanded_job_names.map(&:to_sym)) - end - - it 'sets job instance in options' do - expect(subject.values).to all(include(:instance)) - end - - it 'parallelizes jobs with original config' do - original_config = config[job_name].except(:name) - configs = subject.values.map { |config| config.except(:name, :instance) } - - expect(configs).to all(eq(original_config)) - end - - context 'when the job is not parallelized' do - let(:job_config) { { script: 'rspec', name: 'rspec' } } - - it 'returns the same hash' do - is_expected.to eq(config) - end - end - - context 'when there is a job with a slash in it' do - let(:job_name) { :"rspec 35/2" } - - it 'properly parallelizes job names' do - job_names = [ - :"rspec 35/2 1/5", - :"rspec 35/2 2/5", - :"rspec 35/2 3/5", - :"rspec 35/2 4/5", - :"rspec 35/2 5/5" - ] - - is_expected.to include(*job_names) - end - end - - context 'for dependencies' do + shared_examples 'parallel dependencies' do context "when job has dependencies on parallelized jobs" do let(:config) do { @@ -91,9 +38,7 @@ RSpec.describe Gitlab::Ci::Config::Normalizer do end it "parallelizes dependencies" do - job_names = ["rspec 1/5", "rspec 2/5", "rspec 3/5", "rspec 4/5", "rspec 5/5"] - - expect(subject[:final_job][:dependencies]).to include(*job_names) + expect(subject[:final_job][:dependencies]).to include(*expanded_job_names) end it "includes the regular job in dependencies" do @@ -102,14 +47,14 @@ RSpec.describe Gitlab::Ci::Config::Normalizer do end end - context 'for needs' do + shared_examples 'parallel needs' do let(:expanded_job_attributes) do expanded_job_names.map do |job_name| { name: job_name, extra: :key } end end - context "when job has needs on parallelized jobs" do + context 'when job has needs on parallelized jobs' do let(:config) do { job_name => job_config, @@ -124,12 +69,12 @@ RSpec.describe Gitlab::Ci::Config::Normalizer do } end - it "parallelizes needs" do + it 'parallelizes needs' do expect(subject.dig(:other_job, :needs, :job)).to eq(expanded_job_attributes) end end - context "when there are dependencies which are both parallelized and not" do + context 'when there are dependencies which are both parallelized and not' do let(:config) do { job_name => job_config, @@ -141,21 +86,157 @@ RSpec.describe Gitlab::Ci::Config::Normalizer do needs: { job: [ { name: job_name.to_s, extra: :key }, - { name: "other_job", extra: :key } + { name: 'other_job', extra: :key } ] } } } end - it "parallelizes dependencies" do + it 'parallelizes dependencies' do expect(subject.dig(:final_job, :needs, :job)).to include(*expanded_job_attributes) end - it "includes the regular job in dependencies" do + it 'includes the regular job in dependencies' do expect(subject.dig(:final_job, :needs, :job)).to include(name: 'other_job', extra: :key) end end end + + context 'with parallel config as integer' do + let(:variables_config) { {} } + let(:parallel_config) { 5 } + + let(:expanded_job_names) do + [ + 'rspec 1/5', + 'rspec 2/5', + 'rspec 3/5', + 'rspec 4/5', + 'rspec 5/5' + ] + end + + it 'does not have original job' do + is_expected.not_to include(job_name) + end + + it 'has parallelized jobs' do + is_expected.to include(*expanded_job_names.map(&:to_sym)) + end + + it 'sets job instance in options' do + expect(subject.values).to all(include(:instance)) + end + + it 'parallelizes jobs with original config' do + original_config = config[job_name] + .except(:name) + .deep_merge(parallel: { total: parallel_config }) + + configs = subject.values.map { |config| config.except(:name, :instance) } + + expect(configs).to all(eq(original_config)) + end + + context 'when the job is not parallelized' do + let(:job_config) { { script: 'rspec', name: 'rspec' } } + + it 'returns the same hash' do + is_expected.to eq(config) + end + end + + context 'when there is a job with a slash in it' do + let(:job_name) { :"rspec 35/2" } + + it 'properly parallelizes job names' do + job_names = [ + :"rspec 35/2 1/5", + :"rspec 35/2 2/5", + :"rspec 35/2 3/5", + :"rspec 35/2 4/5", + :"rspec 35/2 5/5" + ] + + is_expected.to include(*job_names) + end + end + + it_behaves_like 'parallel dependencies' + it_behaves_like 'parallel needs' + end + + context 'with parallel matrix config' do + let(:variables_config) do + { + USER_VARIABLE: 'user value' + } + end + + let(:parallel_config) do + { + matrix: [ + { + VAR_1: [1], + VAR_2: [2, 3] + } + ] + } + end + + let(:expanded_job_names) do + [ + 'rspec 1/2', + 'rspec 2/2' + ] + end + + it 'does not have original job' do + is_expected.not_to include(job_name) + end + + it 'has parallelized jobs' do + is_expected.to include(*expanded_job_names.map(&:to_sym)) + end + + it 'sets job instance in options' do + expect(subject.values).to all(include(:instance)) + end + + it 'sets job variables', :aggregate_failures do + expect(subject.values[0]).to match( + a_hash_including(variables: { VAR_1: 1, VAR_2: 2, USER_VARIABLE: 'user value' }) + ) + + expect(subject.values[1]).to match( + a_hash_including(variables: { VAR_1: 1, VAR_2: 3, USER_VARIABLE: 'user value' }) + ) + end + + it 'parallelizes jobs with original config' do + configs = subject.values.map do |config| + config.except(:name, :instance, :variables) + end + + original_config = config[job_name] + .except(:name, :variables) + .deep_merge(parallel: { total: 2 }) + + expect(configs).to all(match(a_hash_including(original_config))) + end + + it_behaves_like 'parallel dependencies' + it_behaves_like 'parallel needs' + end + + context 'when parallel config does not matches a factory' do + let(:variables_config) { {} } + let(:parallel_config) { } + + it 'does not alter the job config' do + is_expected.to match(config) + end + end end end diff --git a/spec/lib/gitlab/ci/parsers/coverage/cobertura_spec.rb b/spec/lib/gitlab/ci/parsers/coverage/cobertura_spec.rb index 08a3fbd7867..45e87466532 100644 --- a/spec/lib/gitlab/ci/parsers/coverage/cobertura_spec.rb +++ b/spec/lib/gitlab/ci/parsers/coverage/cobertura_spec.rb @@ -19,6 +19,41 @@ RSpec.describe Gitlab::Ci::Parsers::Coverage::Cobertura do end end + context 'when there is a <sources>' do + shared_examples_for 'ignoring sources' do + it 'parses XML without errors' do + expect { subject }.not_to raise_error + + expect(coverage_report.files).to eq({}) + end + end + + context 'and has a single source' do + let(:cobertura) do + <<-EOF.strip_heredoc + <sources> + <source>project/src</source> + </sources> + EOF + end + + it_behaves_like 'ignoring sources' + end + + context 'and has multiple sources' do + let(:cobertura) do + <<-EOF.strip_heredoc + <sources> + <source>project/src/foo</source> + <source>project/src/bar</source> + </sources> + EOF + end + + it_behaves_like 'ignoring sources' + end + end + context 'when there is a single <class>' do context 'with no lines' do let(:cobertura) do diff --git a/spec/lib/gitlab/ci/pipeline/chain/sequence_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/sequence_spec.rb index 5d20b1b8fda..cc4aaffb0a4 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/sequence_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/sequence_spec.rb @@ -23,9 +23,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Sequence do end it 'does not process the second step' do - subject.build! do |pipeline, sequence| - expect(sequence).not_to be_complete - end + subject.build! expect(second_step).not_to have_received(:perform!) end @@ -43,9 +41,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Sequence do end it 'iterates through entire sequence' do - subject.build! do |pipeline, sequence| - expect(sequence).to be_complete - end + subject.build! expect(first_step).to have_received(:perform!) expect(second_step).to have_received(:perform!) diff --git a/spec/lib/gitlab/ci/pipeline/chain/validate/external_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/validate/external_spec.rb index 931c62701ce..de580d2e148 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/validate/external_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/validate/external_spec.rb @@ -41,9 +41,10 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Validate::External do ) end + let(:save_incompleted) { true } let(:command) do Gitlab::Ci::Pipeline::Chain::Command.new( - project: project, current_user: user, config_processor: yaml_processor + project: project, current_user: user, config_processor: yaml_processor, save_incompleted: save_incompleted ) end @@ -84,6 +85,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Validate::External do perform! expect(pipeline.status).to eq('failed') + expect(pipeline).to be_persisted expect(pipeline.errors.to_a).to include('External validation failed') end @@ -98,6 +100,30 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Validate::External do perform! end + + context 'when save_incompleted is false' do + let(:save_incompleted) { false} + + it 'adds errors to the pipeline without dropping it' do + perform! + + expect(pipeline.status).to eq('pending') + expect(pipeline).not_to be_persisted + expect(pipeline.errors.to_a).to include('External validation failed') + end + + it 'breaks the chain' do + perform! + + expect(step.break?).to be true + end + + it 'logs the authorization' do + expect(Gitlab::AppLogger).to receive(:info).with(message: 'Pipeline not authorized', project_id: project.id, user_id: user.id) + + perform! + end + end end end diff --git a/spec/lib/gitlab/ci/pipeline/chain/validate/repository_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/validate/repository_spec.rb index 1dc2e0a1822..7eefb4d7876 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/validate/repository_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/validate/repository_spec.rb @@ -68,6 +68,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Validate::Repository do proj.repository.add_tag(user, 'master', 'master') end end + let(:command) do Gitlab::Ci::Pipeline::Chain::Command.new( project: project, current_user: user, origin_ref: 'master') diff --git a/spec/lib/gitlab/ci/pipeline/expression/lexeme/and_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/lexeme/and_spec.rb index 6601537a2d3..1448b045b18 100644 --- a/spec/lib/gitlab/ci/pipeline/expression/lexeme/and_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/expression/lexeme/and_spec.rb @@ -24,7 +24,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Expression::Lexeme::And do describe '.type' do it 'is an operator' do - expect(described_class.type).to eq :operator + expect(described_class.type).to eq :logical_operator end end diff --git a/spec/lib/gitlab/ci/pipeline/expression/lexeme/equals_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/lexeme/equals_spec.rb index 2bed47f0a87..ab223ae41fa 100644 --- a/spec/lib/gitlab/ci/pipeline/expression/lexeme/equals_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/expression/lexeme/equals_spec.rb @@ -27,7 +27,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Expression::Lexeme::Equals do describe '.type' do it 'is an operator' do - expect(described_class.type).to eq :operator + expect(described_class.type).to eq :logical_operator end end diff --git a/spec/lib/gitlab/ci/pipeline/expression/lexeme/matches_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/lexeme/matches_spec.rb index efcea0b0e09..0da04d8dcf7 100644 --- a/spec/lib/gitlab/ci/pipeline/expression/lexeme/matches_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/expression/lexeme/matches_spec.rb @@ -28,7 +28,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Expression::Lexeme::Matches do describe '.type' do it 'is an operator' do - expect(described_class.type).to eq :operator + expect(described_class.type).to eq :logical_operator end end diff --git a/spec/lib/gitlab/ci/pipeline/expression/lexeme/not_equals_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/lexeme/not_equals_spec.rb index a81e1713ef0..3cde4c5d9dc 100644 --- a/spec/lib/gitlab/ci/pipeline/expression/lexeme/not_equals_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/expression/lexeme/not_equals_spec.rb @@ -27,7 +27,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Expression::Lexeme::NotEquals do describe '.type' do it 'is an operator' do - expect(described_class.type).to eq :operator + expect(described_class.type).to eq :logical_operator end end diff --git a/spec/lib/gitlab/ci/pipeline/expression/lexeme/not_matches_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/lexeme/not_matches_spec.rb index f44fe19f86d..9bff2355d58 100644 --- a/spec/lib/gitlab/ci/pipeline/expression/lexeme/not_matches_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/expression/lexeme/not_matches_spec.rb @@ -28,7 +28,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Expression::Lexeme::NotMatches do describe '.type' do it 'is an operator' do - expect(described_class.type).to eq :operator + expect(described_class.type).to eq :logical_operator end end diff --git a/spec/lib/gitlab/ci/pipeline/expression/lexeme/or_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/lexeme/or_spec.rb index 7fe445975eb..c7d89c4e1e9 100644 --- a/spec/lib/gitlab/ci/pipeline/expression/lexeme/or_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/expression/lexeme/or_spec.rb @@ -24,7 +24,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Expression::Lexeme::Or do describe '.type' do it 'is an operator' do - expect(described_class.type).to eq :operator + expect(described_class.type).to eq :logical_operator end end diff --git a/spec/lib/gitlab/ci/pipeline/expression/lexeme/pattern_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/lexeme/pattern_spec.rb index 1a56a91c471..fa4f8a20984 100644 --- a/spec/lib/gitlab/ci/pipeline/expression/lexeme/pattern_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/expression/lexeme/pattern_spec.rb @@ -70,7 +70,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Expression::Lexeme::Pattern do .to eq Gitlab::UntrustedRegexp.new('pattern') end - it 'is a eager scanner for regexp boundaries' do + it 'is an eager scanner for regexp boundaries' do scanner = StringScanner.new('/some .* / pattern/') token = described_class.scan(scanner) diff --git a/spec/lib/gitlab/ci/pipeline/expression/lexer_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/lexer_spec.rb index 61c6ced4dac..6e242faa885 100644 --- a/spec/lib/gitlab/ci/pipeline/expression/lexer_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/expression/lexer_spec.rb @@ -81,6 +81,35 @@ RSpec.describe Gitlab::Ci::Pipeline::Expression::Lexer do with_them do it { is_expected.to eq(tokens) } end + + context 'with parentheses are used' do + where(:expression, :tokens) do + '($PRESENT_VARIABLE =~ /my var/) && $EMPTY_VARIABLE =~ /nope/' | ['(', '$PRESENT_VARIABLE', '=~', '/my var/', ')', '&&', '$EMPTY_VARIABLE', '=~', '/nope/'] + '$PRESENT_VARIABLE =~ /my var/ || ($EMPTY_VARIABLE =~ /nope/)' | ['$PRESENT_VARIABLE', '=~', '/my var/', '||', '(', '$EMPTY_VARIABLE', '=~', '/nope/', ')'] + '($PRESENT_VARIABLE && (null || $EMPTY_VARIABLE == ""))' | ['(', '$PRESENT_VARIABLE', '&&', '(', 'null', '||', '$EMPTY_VARIABLE', '==', '""', ')', ')'] + end + + with_them do + context 'when ci_if_parenthesis_enabled is enabled' do + before do + stub_feature_flags(ci_if_parenthesis_enabled: true) + end + + it { is_expected.to eq(tokens) } + end + + context 'when ci_if_parenthesis_enabled is disabled' do + before do + stub_feature_flags(ci_if_parenthesis_enabled: false) + end + + it do + expect { subject } + .to raise_error described_class::SyntaxError + end + end + end + end end end diff --git a/spec/lib/gitlab/ci/pipeline/expression/parser_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/parser_spec.rb index 1704cabfd2e..3394a75ac0a 100644 --- a/spec/lib/gitlab/ci/pipeline/expression/parser_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/expression/parser_spec.rb @@ -1,51 +1,79 @@ # frozen_string_literal: true -require 'fast_spec_helper' +require 'spec_helper' RSpec.describe Gitlab::Ci::Pipeline::Expression::Parser do + before do + stub_feature_flags(ci_if_parenthesis_enabled: true) + end + describe '#tree' do - context 'when using two operators' do - it 'returns a reverse descent parse tree' do - expect(described_class.seed('$VAR1 == "123"').tree) - .to be_a Gitlab::Ci::Pipeline::Expression::Lexeme::Equals + context 'validates simple operators' do + using RSpec::Parameterized::TableSyntax + + where(:expression, :result_tree) do + '$VAR1 == "123"' | 'equals($VAR1, "123")' + '$VAR1 == "123" == $VAR2' | 'equals(equals($VAR1, "123"), $VAR2)' + '$VAR' | '$VAR' + '"some value"' | '"some value"' + 'null' | 'null' + '$VAR1 || $VAR2 && $VAR3' | 'or($VAR1, and($VAR2, $VAR3))' + '$VAR1 && $VAR2 || $VAR3' | 'or(and($VAR1, $VAR2), $VAR3)' + '$VAR1 && $VAR2 || $VAR3 && $VAR4' | 'or(and($VAR1, $VAR2), and($VAR3, $VAR4))' + '$VAR1 && ($VAR2 || $VAR3) && $VAR4' | 'and(and($VAR1, or($VAR2, $VAR3)), $VAR4)' end - end - context 'when using three operators' do - it 'returns a reverse descent parse tree' do - expect(described_class.seed('$VAR1 == "123" == $VAR2').tree) - .to be_a Gitlab::Ci::Pipeline::Expression::Lexeme::Equals + with_them do + it { expect(described_class.seed(expression).tree.inspect).to eq(result_tree) } end end - context 'when using a single variable token' do - it 'returns a single token instance' do - expect(described_class.seed('$VAR').tree) - .to be_a Gitlab::Ci::Pipeline::Expression::Lexeme::Variable + context 'when combining && and OR operators' do + subject { described_class.seed('$VAR1 == "a" || $VAR2 == "b" && $VAR3 == "c" || $VAR4 == "d" && $VAR5 == "e"').tree } + + context 'when parenthesis engine is enabled' do + before do + stub_feature_flags(ci_if_parenthesis_enabled: true) + end + + it 'returns operations in a correct order' do + expect(subject.inspect) + .to eq('or(or(equals($VAR1, "a"), and(equals($VAR2, "b"), equals($VAR3, "c"))), and(equals($VAR4, "d"), equals($VAR5, "e")))') + end + end + + context 'when parenthesis engine is disabled (legacy)' do + before do + stub_feature_flags(ci_if_parenthesis_enabled: false) + end + + it 'returns operations in a invalid order' do + expect(subject.inspect) + .to eq('or(equals($VAR1, "a"), and(equals($VAR2, "b"), or(equals($VAR3, "c"), and(equals($VAR4, "d"), equals($VAR5, "e")))))') + end end end - context 'when using a single string token' do - it 'returns a single token instance' do - expect(described_class.seed('"some value"').tree) - .to be_a Gitlab::Ci::Pipeline::Expression::Lexeme::String + context 'when using parenthesis' do + subject { described_class.seed('(($VAR1 == "a" || $VAR2 == "b") && $VAR3 == "c" || $VAR4 == "d") && $VAR5 == "e"').tree } + + before do + stub_feature_flags(ci_if_parenthesis_enabled: true) + end + + it 'returns operations in a correct order' do + expect(subject.inspect) + .to eq('and(or(and(or(equals($VAR1, "a"), equals($VAR2, "b")), equals($VAR3, "c")), equals($VAR4, "d")), equals($VAR5, "e"))') end end context 'when expression is empty' do - it 'returns a null token' do + it 'raises a parsing error' do expect { described_class.seed('').tree } .to raise_error Gitlab::Ci::Pipeline::Expression::Parser::ParseError end end - context 'when expression is null' do - it 'returns a null token' do - expect(described_class.seed('null').tree) - .to be_a Gitlab::Ci::Pipeline::Expression::Lexeme::Null - end - end - context 'when two value tokens have no operator' do it 'raises a parsing error' do expect { described_class.seed('$VAR "text"').tree } @@ -66,5 +94,42 @@ RSpec.describe Gitlab::Ci::Pipeline::Expression::Parser do .to raise_error Gitlab::Ci::Pipeline::Expression::Lexeme::Operator::OperatorError end end + + context 'when parenthesis are unmatched' do + context 'when parenthesis engine is enabled' do + before do + stub_feature_flags(ci_if_parenthesis_enabled: true) + end + + where(:expression) do + [ + '$VAR == (', + '$VAR2 == ("aa"', + '$VAR2 == ("aa"))', + '$VAR2 == "aa")', + '(($VAR2 == "aa")', + '($VAR2 == "aa"))' + ] + end + + with_them do + it 'raises a ParseError' do + expect { described_class.seed(expression).tree } + .to raise_error Gitlab::Ci::Pipeline::Expression::Parser::ParseError + end + end + end + + context 'when parenthesis engine is disabled' do + before do + stub_feature_flags(ci_if_parenthesis_enabled: false) + end + + it 'raises an SyntaxError' do + expect { described_class.seed('$VAR == (').tree } + .to raise_error Gitlab::Ci::Pipeline::Expression::Lexer::SyntaxError + end + end + end end end diff --git a/spec/lib/gitlab/ci/pipeline/expression/statement_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/statement_spec.rb index 642d6816030..cf3644c9ad5 100644 --- a/spec/lib/gitlab/ci/pipeline/expression/statement_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/expression/statement_spec.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true -require 'fast_spec_helper' -require 'rspec-parameterized' +require 'spec_helper' RSpec.describe Gitlab::Ci::Pipeline::Expression::Statement do subject do @@ -109,6 +108,17 @@ RSpec.describe Gitlab::Ci::Pipeline::Expression::Statement do '$UNDEFINED_VARIABLE || $PRESENT_VARIABLE' | 'my variable' '$UNDEFINED_VARIABLE == null || $PRESENT_VARIABLE' | true '$PRESENT_VARIABLE || $UNDEFINED_VARIABLE == null' | 'my variable' + + '($PRESENT_VARIABLE)' | 'my variable' + '(($PRESENT_VARIABLE))' | 'my variable' + '(($PRESENT_VARIABLE && null) || $EMPTY_VARIABLE == "")' | true + '($PRESENT_VARIABLE) && (null || $EMPTY_VARIABLE == "")' | true + '("string" || "test") == "string"' | true + '(null || ("test" == "string"))' | false + '("string" == ("test" && "string"))' | true + '("string" == ("test" || "string"))' | false + '("string" == "test" || "string")' | "string" + '("string" == ("string" || (("1" == "1") && ("2" == "3"))))' | true end with_them do diff --git a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb index 2dea554fe56..733ab30132d 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb @@ -928,29 +928,51 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build do end end - context 'when lower limit of needs is reached' do - before do - stub_feature_flags(ci_dag_limit_needs: true) - end + context 'when using 101 needs' do + let(:needs_count) { 101 } - let(:needs_count) { described_class::LOW_NEEDS_LIMIT + 1 } + context 'when ci_plan_needs_size_limit is disabled' do + before do + stub_feature_flags(ci_plan_needs_size_limit: false) + end - it "returns an error" do - expect(subject.errors).to contain_exactly( - "rspec: one job can only need 10 others, but you have listed 11. See needs keyword documentation for more details") + it "returns an error" do + expect(subject.errors).to contain_exactly( + "rspec: one job can only need 10 others, but you have listed 101. See needs keyword documentation for more details") + end end - end - context 'when upper limit of needs is reached' do - before do - stub_feature_flags(ci_dag_limit_needs: false) - end + context 'when ci_plan_needs_size_limit is enabled' do + before do + stub_feature_flags(ci_plan_needs_size_limit: true) + end - let(:needs_count) { described_class::HARD_NEEDS_LIMIT + 1 } + it "returns an error" do + expect(subject.errors).to contain_exactly( + "rspec: one job can only need 50 others, but you have listed 101. See needs keyword documentation for more details") + end - it "returns an error" do - expect(subject.errors).to contain_exactly( - "rspec: one job can only need 50 others, but you have listed 51. See needs keyword documentation for more details") + context 'when ci_needs_size_limit is set to 100' do + before do + project.actual_limits.update!(ci_needs_size_limit: 100) + end + + it "returns an error" do + expect(subject.errors).to contain_exactly( + "rspec: one job can only need 100 others, but you have listed 101. See needs keyword documentation for more details") + end + end + + context 'when ci_needs_size_limit is set to 0' do + before do + project.actual_limits.update!(ci_needs_size_limit: 0) + end + + it "returns an error" do + expect(subject.errors).to contain_exactly( + "rspec: one job can only need 0 others, but you have listed 101. See needs keyword documentation for more details") + end + end end end end diff --git a/spec/lib/gitlab/ci/reports/accessibility_reports_comparer_spec.rb b/spec/lib/gitlab/ci/reports/accessibility_reports_comparer_spec.rb index 240ede790e0..650ae41320b 100644 --- a/spec/lib/gitlab/ci/reports/accessibility_reports_comparer_spec.rb +++ b/spec/lib/gitlab/ci/reports/accessibility_reports_comparer_spec.rb @@ -21,6 +21,7 @@ RSpec.describe Gitlab::Ci::Reports::AccessibilityReportsComparer do } ] end + let(:different_error) do [ { diff --git a/spec/lib/gitlab/ci/reports/test_report_summary_spec.rb b/spec/lib/gitlab/ci/reports/test_report_summary_spec.rb index 70d82851125..555682cc006 100644 --- a/spec/lib/gitlab/ci/reports/test_report_summary_spec.rb +++ b/spec/lib/gitlab/ci/reports/test_report_summary_spec.rb @@ -11,68 +11,8 @@ RSpec.describe Gitlab::Ci::Reports::TestReportSummary do subject { test_report_summary.total } context 'when test report summary has several build report results' do - it 'returns test suite summary object' do - expect(subject).to be_a_kind_of(Gitlab::Ci::Reports::TestSuiteSummary) - end - end - end - - describe '#total_time' do - subject { test_report_summary.total_time } - - context 'when test report summary has several build report results' do - it 'returns the total' do - expect(subject).to eq(0.84) - end - end - end - - describe '#total_count' do - subject { test_report_summary.total_count } - - context 'when test report summary has several build report results' do - it 'returns the total count' do - expect(subject).to eq(4) - end - end - end - - describe '#success_count' do - subject { test_report_summary.success_count } - - context 'when test suite summary has several build report results' do - it 'returns the total success' do - expect(subject).to eq(2) - end - end - end - - describe '#failed_count' do - subject { test_report_summary.failed_count } - - context 'when test suite summary has several build report results' do - it 'returns the total failed' do - expect(subject).to eq(0) - end - end - end - - describe '#error_count' do - subject { test_report_summary.error_count } - - context 'when test suite summary has several build report results' do - it 'returns the total errored' do - expect(subject).to eq(2) - end - end - end - - describe '#skipped_count' do - subject { test_report_summary.skipped_count } - - context 'when test suite summary has several build report results' do - it 'returns the total skipped' do - expect(subject).to eq(0) + it 'returns all the total count in a hash' do + expect(subject).to include(:time, :count, :success, :failed, :skipped, :error) end end end diff --git a/spec/lib/gitlab/ci/reports/test_suite_spec.rb b/spec/lib/gitlab/ci/reports/test_suite_spec.rb index c4c4d2c3704..fbe3473f6b0 100644 --- a/spec/lib/gitlab/ci/reports/test_suite_spec.rb +++ b/spec/lib/gitlab/ci/reports/test_suite_spec.rb @@ -50,9 +50,11 @@ RSpec.describe Gitlab::Ci::Reports::TestSuite do before do test_suite.add_test_case(test_case_success) test_suite.add_test_case(test_case_failed) + test_suite.add_test_case(test_case_skipped) + test_suite.add_test_case(test_case_error) end - it { is_expected.to eq(2) } + it { is_expected.to eq(4) } end describe '#total_status' do diff --git a/spec/lib/gitlab/ci/reports/test_suite_summary_spec.rb b/spec/lib/gitlab/ci/reports/test_suite_summary_spec.rb index 12c96acdcf3..a98d3db4e82 100644 --- a/spec/lib/gitlab/ci/reports/test_suite_summary_spec.rb +++ b/spec/lib/gitlab/ci/reports/test_suite_summary_spec.rb @@ -86,4 +86,14 @@ RSpec.describe Gitlab::Ci::Reports::TestSuiteSummary do end end end + + describe '#to_h' do + subject { test_suite_summary.to_h } + + context 'when test suite summary has several build report results' do + it 'returns the total as a hash' do + expect(subject).to include(:time, :count, :success, :failed, :skipped, :error) + end + end + end end diff --git a/spec/lib/gitlab/ci/runner_instructions_spec.rb b/spec/lib/gitlab/ci/runner_instructions_spec.rb new file mode 100644 index 00000000000..32ee2ceb040 --- /dev/null +++ b/spec/lib/gitlab/ci/runner_instructions_spec.rb @@ -0,0 +1,217 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::RunnerInstructions do + using RSpec::Parameterized::TableSyntax + + let(:params) { {} } + let(:user) { create(:user) } + + describe 'OS' do + Gitlab::Ci::RunnerInstructions::OS.each do |name, subject| + context name do + it 'has the required fields' do + expect(subject).to have_key(:human_readable_name) + expect(subject).to have_key(:download_locations) + expect(subject).to have_key(:install_script_template_path) + expect(subject).to have_key(:runner_executable) + end + + it 'has a valid script' do + expect(File.read(subject[:install_script_template_path]).length).not_to eq(0) + end + end + end + end + + describe 'OTHER_ENVIRONMENTS' do + Gitlab::Ci::RunnerInstructions::OTHER_ENVIRONMENTS.each do |name, subject| + context name do + it 'has the required fields' do + expect(subject).to have_key(:human_readable_name) + expect(subject).to have_key(:installation_instructions_url) + end + end + end + end + + describe '#install_script' do + subject { described_class.new(current_user: user, **params) } + + context 'invalid params' do + where(:current_params, :expected_error_message) do + { os: nil, arch: nil } | 'Missing OS' + { os: 'linux', arch: nil } | 'Missing arch' + { os: nil, arch: 'amd64' } | 'Missing OS' + { os: 'non_existing_os', arch: 'amd64' } | 'Invalid OS' + { os: 'linux', arch: 'non_existing_arch' } | 'Architecture not found for OS' + { os: 'windows', arch: 'non_existing_arch' } | 'Architecture not found for OS' + end + + with_them do + let(:params) { current_params } + + it 'raises argument error' do + result = subject.install_script + + expect(result).to be_nil + expect(subject.errors).to include(expected_error_message) + end + end + end + + context 'with valid params' do + where(:os, :arch) do + 'linux' | 'amd64' + 'linux' | '386' + 'linux' | 'arm' + 'linux' | 'arm64' + 'windows' | 'amd64' + 'windows' | '386' + 'osx' | 'amd64' + end + + with_them do + let(:params) { { os: os, arch: arch } } + + it 'returns string containing correct params' do + result = subject.install_script + + expect(result).to be_a(String) + + if os == 'osx' + expect(result).to include("darwin-#{arch}") + else + expect(result).to include("#{os}-#{arch}") + end + end + end + end + end + + describe '#register_command' do + let(:params) { { os: 'linux', arch: 'foo' } } + + where(:commands) do + Gitlab::Ci::RunnerInstructions::OS.map do |name, values| + { name => values[:runner_executable] } + end + end + + context 'group' do + let(:group) { create(:group) } + + subject { described_class.new(current_user: user, group: group, **params) } + + context 'user is owner' do + before do + group.add_owner(user) + end + + with_them do + let(:params) { { os: commands.each_key.first, arch: 'foo' } } + + it 'have correct configurations' do + result = subject.register_command + + expect(result).to include("#{commands[commands.each_key.first]} register") + expect(result).to include("--registration-token #{group.runners_token}") + expect(result).to include("--url #{Gitlab::Routing.url_helpers.root_url(only_path: false)}") + end + end + end + + context 'user is not owner' do + where(:user_permission) do + [:maintainer, :developer, :reporter, :guest] + end + + with_them do + before do + create(:group_member, user_permission, group: group, user: user) + end + + it 'raises error' do + result = subject.register_command + + expect(result).to be_nil + expect(subject.errors).to include("Gitlab::Access::AccessDeniedError") + end + end + end + end + + context 'project' do + let(:project) { create(:project) } + + subject { described_class.new(current_user: user, project: project, **params) } + + context 'user is maintainer' do + before do + project.add_maintainer(user) + end + + with_them do + let(:params) { { os: commands.each_key.first, arch: 'foo' } } + + it 'have correct configurations' do + result = subject.register_command + + expect(result).to include("#{commands[commands.each_key.first]} register") + expect(result).to include("--registration-token #{project.runners_token}") + expect(result).to include("--url #{Gitlab::Routing.url_helpers.root_url(only_path: false)}") + end + end + end + + context 'user is not maintainer' do + where(:user_permission) do + [:developer, :reporter, :guest] + end + + with_them do + before do + create(:project_member, user_permission, project: project, user: user) + end + + it 'raises error' do + result = subject.register_command + + expect(result).to be_nil + expect(subject.errors).to include("Gitlab::Access::AccessDeniedError") + end + end + end + end + + context 'instance' do + subject { described_class.new(current_user: user, **params) } + + context 'user is admin' do + let(:user) { create(:user, :admin) } + + with_them do + let(:params) { { os: commands.each_key.first, arch: 'foo' } } + + it 'have correct configurations' do + result = subject.register_command + + expect(result).to include("#{commands[commands.each_key.first]} register") + expect(result).to include("--registration-token #{Gitlab::CurrentSettings.runners_registration_token}") + expect(result).to include("--url #{Gitlab::Routing.url_helpers.root_url(only_path: false)}") + end + end + end + + context 'user is not admin' do + it 'raises error' do + result = subject.register_command + + expect(result).to be_nil + expect(subject.errors).to include("Gitlab::Access::AccessDeniedError") + end + end + end + end +end diff --git a/spec/lib/gitlab/ci/status/composite_spec.rb b/spec/lib/gitlab/ci/status/composite_spec.rb index 47bbc4169b6..e1dcd05373f 100644 --- a/spec/lib/gitlab/ci/status/composite_spec.rb +++ b/spec/lib/gitlab/ci/status/composite_spec.rb @@ -16,48 +16,61 @@ RSpec.describe Gitlab::Ci::Status::Composite do end describe '#status' do - shared_examples 'compares composite with SQL status' do - it 'returns exactly the same result' do - builds = Ci::Build.where(id: all_statuses) + using RSpec::Parameterized::TableSyntax - expect(composite_status.status).to eq(builds.legacy_status) - expect(composite_status.warnings?).to eq(builds.failed_but_allowed.any?) + shared_examples 'compares status and warnings' do + let(:composite_status) do + described_class.new(all_statuses) + end + + it 'returns status and warnings?' do + expect(composite_status.status).to eq(result) + expect(composite_status.warnings?).to eq(has_warnings) end end - shared_examples 'validate all combinations' do |perms| - Ci::HasStatus::STATUSES_ENUM.keys.combination(perms).each do |statuses| - context "with #{statuses.join(",")}" do - it_behaves_like 'compares composite with SQL status' do - let(:all_statuses) do - statuses.map { |status| @statuses[status] } - end - - let(:composite_status) do - described_class.new(all_statuses) - end - end - - Ci::HasStatus::STATUSES_ENUM.each do |allow_failure_status, _| - context "and allow_failure #{allow_failure_status}" do - it_behaves_like 'compares composite with SQL status' do - let(:all_statuses) do - statuses.map { |status| @statuses[status] } + - [@statuses_with_allow_failure[allow_failure_status]] - end - - let(:composite_status) do - described_class.new(all_statuses) - end - end - end - end + context 'allow_failure: false' do + where(:build_statuses, :result, :has_warnings) do + %i(skipped) | 'skipped' | false + %i(skipped success) | 'success' | false + %i(created) | 'created' | false + %i(preparing) | 'preparing' | false + %i(canceled success skipped) | 'canceled' | false + %i(pending created skipped) | 'pending' | false + %i(pending created skipped success) | 'running' | false + %i(running created skipped success) | 'running' | false + %i(success waiting_for_resource) | 'waiting_for_resource' | false + %i(success manual) | 'manual' | false + %i(success scheduled) | 'scheduled' | false + %i(created preparing) | 'preparing' | false + %i(created success pending) | 'running' | false + %i(skipped success failed) | 'failed' | false + end + + with_them do + let(:all_statuses) do + build_statuses.map { |status| @statuses[status] } end + + it_behaves_like 'compares status and warnings' end end - it_behaves_like 'validate all combinations', 0 - it_behaves_like 'validate all combinations', 1 - it_behaves_like 'validate all combinations', 2 + context 'allow_failure: true' do + where(:build_statuses, :result, :has_warnings) do + %i(manual) | 'skipped' | false + %i(skipped failed) | 'success' | true + %i(created failed) | 'created' | true + %i(preparing manual) | 'preparing' | false + end + + with_them do + let(:all_statuses) do + build_statuses.map { |status| @statuses_with_allow_failure[status] } + end + + it_behaves_like 'compares status and warnings' + end + end end end diff --git a/spec/lib/gitlab/ci/trace/stream_spec.rb b/spec/lib/gitlab/ci/trace/stream_spec.rb index 568c10bbac2..e28469c9404 100644 --- a/spec/lib/gitlab/ci/trace/stream_spec.rb +++ b/spec/lib/gitlab/ci/trace/stream_spec.rb @@ -101,7 +101,7 @@ RSpec.describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do describe '#append' do shared_examples_for 'appends' do - it "truncates and append content" do + it "truncates and appends content" do stream.append(+"89", 4) stream.seek(0) diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb index 5c6d748d66c..1c81cc83cd1 100644 --- a/spec/lib/gitlab/ci/yaml_processor_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb @@ -443,15 +443,15 @@ module Gitlab context 'when a warning is raised in a given entry' do let(:config) do <<-EOYML - rspec: - script: rspec - rules: - - if: '$VAR == "value"' + rspec: + script: echo + rules: + - when: always EOYML end it 'is propagated all the way up to the processor' do - expect(subject.warnings).to contain_exactly('jobs:rspec uses `rules` without defining `workflow:rules`') + expect(subject.warnings).to contain_exactly(/jobs:rspec may allow multiple pipelines to run/) end end @@ -461,7 +461,7 @@ module Gitlab rspec: script: rspec rules: - - if: '$VAR == "value"' + - when: always invalid: script: echo artifacts: @@ -473,7 +473,7 @@ module Gitlab expect { subject }.to raise_error do |error| expect(error).to be_a(described_class::ValidationError) expect(error.message).to eq('jobs:invalid:artifacts config should be a hash') - expect(error.warnings).to contain_exactly('jobs:rspec uses `rules` without defining `workflow:rules`') + expect(error.warnings).to contain_exactly(/jobs:rspec may allow multiple pipelines to run/) end end end @@ -485,7 +485,7 @@ module Gitlab rspec: script: rspec rules: - - if: '$VAR == "value"' + - when: always EOYML end @@ -516,7 +516,7 @@ module Gitlab stage: custom_stage script: rspec rules: - - if: '$VAR == "value"' + - when: always EOYML end @@ -530,7 +530,7 @@ module Gitlab stage: build script: echo rules: - - if: '$VAR == "value"' + - when: always test: stage: test script: echo @@ -549,7 +549,7 @@ module Gitlab script: echo needs: [test] rules: - - if: '$VAR == "value"' + - when: always test: stage: test script: echo @@ -571,7 +571,7 @@ module Gitlab rspec: script: rspec rules: - - if: '$VAR == "value"' + - when: always EOYML end @@ -942,6 +942,7 @@ module Gitlab let(:variables) do { 'VAR1' => 'value1', 'VAR2' => 'value2' } end + let(:config) do { variables: variables, @@ -962,9 +963,11 @@ module Gitlab let(:global_variables) do { 'VAR1' => 'global1', 'VAR3' => 'global3', 'VAR4' => 'global4' } end + let(:job_variables) do { 'VAR1' => 'value1', 'VAR2' => 'value2' } end + let(:config) do { before_script: ['pwd'], @@ -1269,27 +1272,104 @@ module Gitlab end describe 'Parallel' do + let(:config) do + YAML.dump(rspec: { script: 'rspec', + parallel: parallel, + variables: { 'VAR1' => 1 } }) + end + + let(:config_processor) { Gitlab::Ci::YamlProcessor.new(config) } + let(:builds) { config_processor.stage_builds_attributes('test') } + context 'when job is parallelized' do let(:parallel) { 5 } - let(:config) do - YAML.dump(rspec: { script: 'rspec', - parallel: parallel }) - end - it 'returns parallelized jobs' do - config_processor = Gitlab::Ci::YamlProcessor.new(config) - builds = config_processor.stage_builds_attributes('test') build_options = builds.map { |build| build[:options] } expect(builds.size).to eq(5) - expect(build_options).to all(include(:instance, parallel: parallel)) + expect(build_options).to all(include(:instance, parallel: { number: parallel, total: parallel })) end it 'does not have the original job' do - config_processor = Gitlab::Ci::YamlProcessor.new(config) - builds = config_processor.stage_builds_attributes('test') + expect(builds).not_to include(:rspec) + end + end + + context 'with build matrix' do + let(:parallel) do + { + matrix: [ + { 'PROVIDER' => 'aws', 'STACK' => %w[monitoring app1 app2] }, + { 'PROVIDER' => 'ovh', 'STACK' => %w[monitoring backup app] }, + { 'PROVIDER' => 'gcp', 'STACK' => %w[data processing] } + ] + } + end + + it 'returns the number of parallelized jobs' do + expect(builds.size).to eq(8) + end + + it 'returns the parallel config' do + build_options = builds.map { |build| build[:options] } + parallel_config = { + matrix: parallel[:matrix].map { |var| var.transform_values { |v| Array(v).flatten }}, + total: build_options.size + } + + expect(build_options).to all(include(:instance, parallel: parallel_config)) + end + it 'sets matrix variables' do + build_variables = builds.map { |build| build[:yaml_variables] } + expected_variables = [ + [ + { key: 'VAR1', value: '1' }, + { key: 'PROVIDER', value: 'aws' }, + { key: 'STACK', value: 'monitoring' } + ], + [ + { key: 'VAR1', value: '1' }, + { key: 'PROVIDER', value: 'aws' }, + { key: 'STACK', value: 'app1' } + ], + [ + { key: 'VAR1', value: '1' }, + { key: 'PROVIDER', value: 'aws' }, + { key: 'STACK', value: 'app2' } + ], + [ + { key: 'VAR1', value: '1' }, + { key: 'PROVIDER', value: 'ovh' }, + { key: 'STACK', value: 'monitoring' } + ], + [ + { key: 'VAR1', value: '1' }, + { key: 'PROVIDER', value: 'ovh' }, + { key: 'STACK', value: 'backup' } + ], + [ + { key: 'VAR1', value: '1' }, + { key: 'PROVIDER', value: 'ovh' }, + { key: 'STACK', value: 'app' } + ], + [ + { key: 'VAR1', value: '1' }, + { key: 'PROVIDER', value: 'gcp' }, + { key: 'STACK', value: 'data' } + ], + [ + { key: 'VAR1', value: '1' }, + { key: 'PROVIDER', value: 'gcp' }, + { key: 'STACK', value: 'processing' } + ] + ].map { |vars| vars.map { |var| a_hash_including(var) } } + + expect(build_variables).to match(expected_variables) + end + + it 'does not have the original job' do expect(builds).not_to include(:rspec) end end @@ -1482,6 +1562,21 @@ module Gitlab }) end + it "returns artifacts with expire_in never keyword" do + config = YAML.dump({ + rspec: { + script: "rspec", + artifacts: { paths: ["releases/"], expire_in: "never" } + } + }) + + config_processor = Gitlab::Ci::YamlProcessor.new(config) + builds = config_processor.stage_builds_attributes("test") + + expect(builds.size).to eq(1) + expect(builds.first[:options][:artifacts][:expire_in]).to eq('never') + end + %w[on_success on_failure always].each do |when_state| it "returns artifacts for when #{when_state} defined" do config = YAML.dump({ @@ -1564,26 +1659,9 @@ module Gitlab } end - context 'with feature flag active' do - before do - stub_feature_flags(ci_release_generation: true) - end - - it "returns release info" do - expect(processor.stage_builds_attributes('release').first[:options]) - .to eq(config[:release].except(:stage, :only)) - end - end - - context 'with feature flag inactive' do - before do - stub_feature_flags(ci_release_generation: false) - end - - it 'raises error' do - expect { processor }.to raise_error( - 'jobs:release config release features are not enabled: release') - end + it "returns release info" do + expect(processor.stage_builds_attributes('release').first[:options]) + .to eq(config[:release].except(:stage, :only)) end end @@ -1998,6 +2076,7 @@ module Gitlab { job: "build2" } ] end + let(:dependencies) { %w(build3) } it { expect { subject }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, 'jobs:test1 dependencies the build3 should be part of needs') } @@ -2407,6 +2486,14 @@ module Gitlab end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, "jobs config should contain at least one visible job") end + it "returns errors if the job script is not defined" do + config = YAML.dump({ rspec: { before_script: "test" } }) + + expect do + Gitlab::Ci::YamlProcessor.new(config) + end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, "jobs:rspec script can't be blank") + end + it "returns errors if there are no visible jobs defined" do config = YAML.dump({ before_script: ["bundle update"], '.hidden'.to_sym => { script: 'ls' } }) expect do @@ -2619,6 +2706,14 @@ module Gitlab .to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, 'rspec: unknown keys in `extends` (something)') end + + it 'returns errors if parallel is invalid' do + config = YAML.dump({ rspec: { parallel: 'test', script: 'test' } }) + + expect { Gitlab::Ci::YamlProcessor.new(config) } + .to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, + 'jobs:rspec:parallel should be an integer or a hash') + end end describe "#validation_message" do diff --git a/spec/lib/gitlab/cleanup/remote_uploads_spec.rb b/spec/lib/gitlab/cleanup/remote_uploads_spec.rb index 1752608f844..c59b7f004dd 100644 --- a/spec/lib/gitlab/cleanup/remote_uploads_spec.rb +++ b/spec/lib/gitlab/cleanup/remote_uploads_spec.rb @@ -11,6 +11,7 @@ RSpec.describe Gitlab::Cleanup::RemoteUploads do create(:upload, path: 'dir/file2', store: ObjectStorage::Store::LOCAL) ] end + let(:remote_files) do [ double(key: 'dir/file1'), diff --git a/spec/lib/gitlab/cross_project_access/class_methods_spec.rb b/spec/lib/gitlab/cross_project_access/class_methods_spec.rb index cc2c431fc07..afc45c86362 100644 --- a/spec/lib/gitlab/cross_project_access/class_methods_spec.rb +++ b/spec/lib/gitlab/cross_project_access/class_methods_spec.rb @@ -8,6 +8,7 @@ RSpec.describe Gitlab::CrossProjectAccess::ClassMethods do extend Gitlab::CrossProjectAccess::ClassMethods end end + let(:dummy_proc) { lambda { false } } describe '#requires_cross_project_access' do diff --git a/spec/lib/gitlab/danger/changelog_spec.rb b/spec/lib/gitlab/danger/changelog_spec.rb index f5954cd8c1e..3c67e9ca8ea 100644 --- a/spec/lib/gitlab/danger/changelog_spec.rb +++ b/spec/lib/gitlab/danger/changelog_spec.rb @@ -57,6 +57,24 @@ RSpec.describe Gitlab::Danger::Changelog do is_expected.to be_truthy end end + + context 'when MR contains a category that require changelog and a category that require no changelog with changelog label' do + let(:changes_by_category) { { category_with_changelog => nil, category_without_changelog => nil } } + let(:mr_labels) { ['feature'] } + + it 'is truthy' do + is_expected.to be_truthy + end + end + + context 'when MR contains a category that require changelog and a category that require no changelog with no changelog label' do + let(:changes_by_category) { { category_with_changelog => nil, category_without_changelog => nil } } + let(:mr_labels) { ['tooling'] } + + it 'is truthy' do + is_expected.to be_falsey + end + end end describe '#found' do diff --git a/spec/lib/gitlab/danger/commit_linter_spec.rb b/spec/lib/gitlab/danger/commit_linter_spec.rb index 06bec6f793d..c31522c538d 100644 --- a/spec/lib/gitlab/danger/commit_linter_spec.rb +++ b/spec/lib/gitlab/danger/commit_linter_spec.rb @@ -16,6 +16,7 @@ RSpec.describe Gitlab::Danger::CommitLinter do let(:commit_class) do Struct.new(:message, :sha, :diff_parent) end + let(:commit_message) { 'A commit message' } let(:commit_sha) { 'abcd1234' } let(:commit) { commit_class.new(commit_message, commit_sha, diff_parent) } diff --git a/spec/lib/gitlab/danger/helper_spec.rb b/spec/lib/gitlab/danger/helper_spec.rb index e73742b5911..e5018e46634 100644 --- a/spec/lib/gitlab/danger/helper_spec.rb +++ b/spec/lib/gitlab/danger/helper_spec.rb @@ -76,6 +76,14 @@ RSpec.describe Gitlab::Danger::Helper do end end + describe "changed_files" do + it 'returns list of changed files matching given regex' do + expect(helper).to receive(:all_changed_files).and_return(%w[migration.rb usage_data.rb]) + + expect(helper.changed_files(/usage_data/)).to contain_exactly('usage_data.rb') + end + end + describe '#all_ee_changes' do subject { helper.all_ee_changes } @@ -98,21 +106,21 @@ RSpec.describe Gitlab::Danger::Helper do it 'delegates to CHANGELOG-EE.md existence if CI_PROJECT_NAME is set to something else' do stub_env('CI_PROJECT_NAME', 'something else') - expect(Dir).to receive(:exist?).with('../../ee') { true } + expect(Dir).to receive(:exist?).with(File.expand_path('../../../../ee', __dir__)) { true } is_expected.to be_truthy end it 'returns true if ee exists' do stub_env('CI_PROJECT_NAME', nil) - expect(Dir).to receive(:exist?).with('../../ee') { true } + expect(Dir).to receive(:exist?).with(File.expand_path('../../../../ee', __dir__)) { true } is_expected.to be_truthy end it "returns false if ee doesn't exist" do stub_env('CI_PROJECT_NAME', nil) - expect(Dir).to receive(:exist?).with('../../ee') { false } + expect(Dir).to receive(:exist?).with(File.expand_path('../../../../ee', __dir__)) { false } is_expected.to be_falsy end @@ -217,9 +225,17 @@ RSpec.describe Gitlab::Danger::Helper do 'ee/spec/foo' | [:backend] 'ee/spec/foo/bar' | [:backend] + 'spec/features/foo' | [:test] + 'ee/spec/features/foo' | [:test] + 'spec/support/shared_examples/features/foo' | [:test] + 'ee/spec/support/shared_examples/features/foo' | [:test] + 'spec/support/shared_contexts/features/foo' | [:test] + 'ee/spec/support/shared_contexts/features/foo' | [:test] + 'spec/support/helpers/features/foo' | [:test] + 'ee/spec/support/helpers/features/foo' | [:test] + 'generator_templates/foo' | [:backend] 'vendor/languages.yml' | [:backend] - 'vendor/licenses.csv' | [:backend] 'file_hooks/examples/' | [:backend] 'Gemfile' | [:backend] @@ -242,6 +258,7 @@ RSpec.describe Gitlab::Danger::Helper do '.editorconfig' | [:engineering_productivity] 'tooling/overcommit/foo' | [:engineering_productivity] '.codeclimate.yml' | [:engineering_productivity] + '.gitlab/CODEOWNERS' | [:engineering_productivity] 'lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml' | [:backend] @@ -295,9 +312,13 @@ RSpec.describe Gitlab::Danger::Helper do context 'having specific changes' do it 'has database and backend categories' do - allow(fake_git).to receive(:diff_for_file).with('usage_data.rb') { double(:diff, patch: "+ count(User.active)") } + changed_files = ['usage_data.rb', 'lib/gitlab/usage_data.rb', 'ee/lib/ee/gitlab/usage_data.rb'] - expect(helper.categories_for_file('usage_data.rb')).to eq([:database, :backend]) + changed_files.each do |file| + allow(fake_git).to receive(:diff_for_file).with(file) { double(:diff, patch: "+ count(User.active)") } + + expect(helper.categories_for_file(file)).to eq([:database, :backend]) + end end it 'has backend category' do @@ -311,6 +332,13 @@ RSpec.describe Gitlab::Danger::Helper do expect(helper.categories_for_file('user.rb')).to eq([:backend]) end + + it 'has backend category for files that are not usage_data.rb' do + changed_file = 'usage_data/topology.rb' + allow(fake_git).to receive(:diff_for_file).with(changed_file) { double(:diff, patch: "+ count(User.active)") } + + expect(helper.categories_for_file(changed_file)).to eq([:backend]) + end end end diff --git a/spec/lib/gitlab/danger/roulette_spec.rb b/spec/lib/gitlab/danger/roulette_spec.rb index 676edca2459..b471e17e2e7 100644 --- a/spec/lib/gitlab/danger/roulette_spec.rb +++ b/spec/lib/gitlab/danger/roulette_spec.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true -require 'fast_spec_helper' require 'webmock/rspec' require 'timecop' @@ -11,102 +10,99 @@ RSpec.describe Gitlab::Danger::Roulette do Timecop.freeze(Time.utc(2020, 06, 22, 10)) { example.run } end + let(:backend_available) { true } + let(:backend_tz_offset_hours) { 2.0 } let(:backend_maintainer) do - { - username: 'backend-maintainer', - name: 'Backend maintainer', - role: 'Backend engineer', - projects: { 'gitlab' => 'maintainer backend' }, - available: true, - tz_offset_hours: 2.0 - } + Gitlab::Danger::Teammate.new( + 'username' => 'backend-maintainer', + 'name' => 'Backend maintainer', + 'role' => 'Backend engineer', + 'projects' => { 'gitlab' => 'maintainer backend' }, + 'available' => backend_available, + 'tz_offset_hours' => backend_tz_offset_hours + ) end + let(:frontend_reviewer) do - { - username: 'frontend-reviewer', - name: 'Frontend reviewer', - role: 'Frontend engineer', - projects: { 'gitlab' => 'reviewer frontend' }, - available: true, - tz_offset_hours: 2.0 - } + Gitlab::Danger::Teammate.new( + 'username' => 'frontend-reviewer', + 'name' => 'Frontend reviewer', + 'role' => 'Frontend engineer', + 'projects' => { 'gitlab' => 'reviewer frontend' }, + 'available' => true, + 'tz_offset_hours' => 2.0 + ) end + let(:frontend_maintainer) do - { - username: 'frontend-maintainer', - name: 'Frontend maintainer', - role: 'Frontend engineer', - projects: { 'gitlab' => "maintainer frontend" }, - available: true, - tz_offset_hours: 2.0 - } + Gitlab::Danger::Teammate.new( + 'username' => 'frontend-maintainer', + 'name' => 'Frontend maintainer', + 'role' => 'Frontend engineer', + 'projects' => { 'gitlab' => "maintainer frontend" }, + 'available' => true, + 'tz_offset_hours' => 2.0 + ) end + let(:software_engineer_in_test) do - { - username: 'software-engineer-in-test', - name: 'Software Engineer in Test', - role: 'Software Engineer in Test, Create:Source Code', - projects: { - 'gitlab' => 'reviewer qa', - 'gitlab-qa' => 'maintainer' - }, - available: true, - tz_offset_hours: 2.0 - } + Gitlab::Danger::Teammate.new( + 'username' => 'software-engineer-in-test', + 'name' => 'Software Engineer in Test', + 'role' => 'Software Engineer in Test, Create:Source Code', + 'projects' => { 'gitlab' => 'reviewer qa', 'gitlab-qa' => 'maintainer' }, + 'available' => true, + 'tz_offset_hours' => 2.0 + ) end + let(:engineering_productivity_reviewer) do - { - username: 'eng-prod-reviewer', - name: 'EP engineer', - role: 'Engineering Productivity', - projects: { 'gitlab' => 'reviewer backend' }, - available: true, - tz_offset_hours: 2.0 - } + Gitlab::Danger::Teammate.new( + 'username' => 'eng-prod-reviewer', + 'name' => 'EP engineer', + 'role' => 'Engineering Productivity', + 'projects' => { 'gitlab' => 'reviewer backend' }, + 'available' => true, + 'tz_offset_hours' => 2.0 + ) end let(:teammate_json) do [ - backend_maintainer, - frontend_maintainer, - frontend_reviewer, - software_engineer_in_test, - engineering_productivity_reviewer + backend_maintainer.to_h, + frontend_maintainer.to_h, + frontend_reviewer.to_h, + software_engineer_in_test.to_h, + engineering_productivity_reviewer.to_h ].to_json end subject(:roulette) { Object.new.extend(described_class) } - def matching_teammate(person) - satisfy do |teammate| - teammate.username == person[:username] && - teammate.name == person[:name] && - teammate.role == person[:role] && - teammate.projects == person[:projects] - end - end - - def matching_spin(category, reviewer: { username: nil }, maintainer: { username: nil }, optional: nil) - satisfy do |spin| - bool = spin.category == category - bool &&= spin.reviewer&.username == reviewer[:username] - - bool &&= - if maintainer - spin.maintainer&.username == maintainer[:username] - else - spin.maintainer.nil? - end - - bool && spin.optional_role == optional + describe 'Spin#==' do + it 'compares Spin attributes' do + spin1 = described_class::Spin.new(:backend, frontend_reviewer, frontend_maintainer, false, false) + spin2 = described_class::Spin.new(:backend, frontend_reviewer, frontend_maintainer, false, false) + spin3 = described_class::Spin.new(:backend, frontend_reviewer, frontend_maintainer, false, true) + spin4 = described_class::Spin.new(:backend, frontend_reviewer, frontend_maintainer, true, false) + spin5 = described_class::Spin.new(:backend, frontend_reviewer, backend_maintainer, false, false) + spin6 = described_class::Spin.new(:backend, backend_maintainer, frontend_maintainer, false, false) + spin7 = described_class::Spin.new(:frontend, frontend_reviewer, frontend_maintainer, false, false) + + expect(spin1).to eq(spin2) + expect(spin1).not_to eq(spin3) + expect(spin1).not_to eq(spin4) + expect(spin1).not_to eq(spin5) + expect(spin1).not_to eq(spin6) + expect(spin1).not_to eq(spin7) end end describe '#spin' do let!(:project) { 'gitlab' } - let!(:branch_name) { 'a-branch' } + let!(:mr_source_branch) { 'a-branch' } let!(:mr_labels) { ['backend', 'devops::create'] } - let!(:author) { Gitlab::Danger::Teammate.new('username' => 'filipa') } + let!(:author) { Gitlab::Danger::Teammate.new('username' => 'johndoe') } let(:timezone_experiment) { false } let(:spins) do # Stub the request at the latest time so that we can modify the raw data, e.g. available fields. @@ -114,12 +110,13 @@ RSpec.describe Gitlab::Danger::Roulette do .stub_request(:get, described_class::ROULETTE_DATA_URL) .to_return(body: teammate_json) - subject.spin(project, categories, branch_name, timezone_experiment: timezone_experiment) + subject.spin(project, categories, timezone_experiment: timezone_experiment) end before do - allow(subject).to receive_message_chain(:gitlab, :mr_author).and_return(author.username) - allow(subject).to receive_message_chain(:gitlab, :mr_labels).and_return(mr_labels) + allow(subject).to receive(:mr_author_username).and_return(author.username) + allow(subject).to receive(:mr_labels).and_return(mr_labels) + allow(subject).to receive(:mr_source_branch).and_return(mr_source_branch) end context 'when timezone_experiment == false' do @@ -127,16 +124,16 @@ RSpec.describe Gitlab::Danger::Roulette do let(:categories) { [:backend] } it 'assigns backend reviewer and maintainer' do - expect(spins).to contain_exactly(matching_spin(:backend, reviewer: engineering_productivity_reviewer, maintainer: backend_maintainer)) + expect(spins[0].reviewer).to eq(engineering_productivity_reviewer) + expect(spins[0].maintainer).to eq(backend_maintainer) + expect(spins).to eq([described_class::Spin.new(:backend, engineering_productivity_reviewer, backend_maintainer, false, false)]) end context 'when teammate is not available' do - before do - backend_maintainer[:available] = false - end + let(:backend_available) { false } it 'assigns backend reviewer and no maintainer' do - expect(spins).to contain_exactly(matching_spin(:backend, reviewer: engineering_productivity_reviewer, maintainer: nil)) + expect(spins).to eq([described_class::Spin.new(:backend, engineering_productivity_reviewer, nil, false, false)]) end end end @@ -145,7 +142,7 @@ RSpec.describe Gitlab::Danger::Roulette do let(:categories) { [:frontend] } it 'assigns frontend reviewer and maintainer' do - expect(spins).to contain_exactly(matching_spin(:frontend, reviewer: frontend_reviewer, maintainer: frontend_maintainer)) + expect(spins).to eq([described_class::Spin.new(:frontend, frontend_reviewer, frontend_maintainer, false, false)]) end end @@ -153,7 +150,7 @@ RSpec.describe Gitlab::Danger::Roulette do let(:categories) { [:qa] } it 'assigns QA reviewer' do - expect(spins).to contain_exactly(matching_spin(:qa, reviewer: software_engineer_in_test)) + expect(spins).to eq([described_class::Spin.new(:qa, software_engineer_in_test, nil, false, false)]) end end @@ -161,7 +158,7 @@ RSpec.describe Gitlab::Danger::Roulette do let(:categories) { [:engineering_productivity] } it 'assigns Engineering Productivity reviewer and fallback to backend maintainer' do - expect(spins).to contain_exactly(matching_spin(:engineering_productivity, reviewer: engineering_productivity_reviewer, maintainer: backend_maintainer)) + expect(spins).to eq([described_class::Spin.new(:engineering_productivity, engineering_productivity_reviewer, backend_maintainer, false, false)]) end end @@ -169,7 +166,7 @@ RSpec.describe Gitlab::Danger::Roulette do let(:categories) { [:test] } it 'assigns corresponding SET' do - expect(spins).to contain_exactly(matching_spin(:test, reviewer: software_engineer_in_test)) + expect(spins).to eq([described_class::Spin.new(:test, software_engineer_in_test, nil, :maintainer, false)]) end end end @@ -181,16 +178,14 @@ RSpec.describe Gitlab::Danger::Roulette do let(:categories) { [:backend] } it 'assigns backend reviewer and maintainer' do - expect(spins).to contain_exactly(matching_spin(:backend, reviewer: engineering_productivity_reviewer, maintainer: backend_maintainer)) + expect(spins).to eq([described_class::Spin.new(:backend, engineering_productivity_reviewer, backend_maintainer, false, true)]) end context 'when teammate is not in a good timezone' do - before do - backend_maintainer[:tz_offset_hours] = 5.0 - end + let(:backend_tz_offset_hours) { 5.0 } it 'assigns backend reviewer and no maintainer' do - expect(spins).to contain_exactly(matching_spin(:backend, reviewer: engineering_productivity_reviewer, maintainer: nil)) + expect(spins).to eq([described_class::Spin.new(:backend, engineering_productivity_reviewer, nil, false, true)]) end end end @@ -203,22 +198,33 @@ RSpec.describe Gitlab::Danger::Roulette do end it 'assigns backend reviewer and maintainer' do - expect(spins).to contain_exactly(matching_spin(:backend, reviewer: engineering_productivity_reviewer, maintainer: backend_maintainer)) + expect(spins).to eq([described_class::Spin.new(:backend, engineering_productivity_reviewer, backend_maintainer, false, false)]) end context 'when teammate is not in a good timezone' do - before do - backend_maintainer[:tz_offset_hours] = 5.0 - end + let(:backend_tz_offset_hours) { 5.0 } it 'assigns backend reviewer and maintainer' do - expect(spins).to contain_exactly(matching_spin(:backend, reviewer: engineering_productivity_reviewer, maintainer: backend_maintainer)) + expect(spins).to eq([described_class::Spin.new(:backend, engineering_productivity_reviewer, backend_maintainer, false, false)]) end end end end end + RSpec::Matchers.define :match_teammates do |expected| + match do |actual| + expected.each do |expected_person| + actual_person_found = actual.find { |actual_person| actual_person.name == expected_person.username } + + actual_person_found && + actual_person_found.name == expected_person.name && + actual_person_found.role == expected_person.role && + actual_person_found.projects == expected_person.projects + end + end + end + describe '#team' do subject(:team) { roulette.team } @@ -254,15 +260,13 @@ RSpec.describe Gitlab::Danger::Roulette do end it 'returns an array of teammates' do - expected_teammates = [ - matching_teammate(backend_maintainer), - matching_teammate(frontend_reviewer), - matching_teammate(frontend_maintainer), - matching_teammate(software_engineer_in_test), - matching_teammate(engineering_productivity_reviewer) - ] - - is_expected.to contain_exactly(*expected_teammates) + is_expected.to match_teammates([ + backend_maintainer, + frontend_reviewer, + frontend_maintainer, + software_engineer_in_test, + engineering_productivity_reviewer + ]) end it 'memoizes the result' do @@ -281,7 +285,9 @@ RSpec.describe Gitlab::Danger::Roulette do end it 'filters team by project_name' do - is_expected.to contain_exactly(matching_teammate(software_engineer_in_test)) + is_expected.to match_teammates([ + software_engineer_in_test + ]) end end @@ -289,32 +295,35 @@ RSpec.describe Gitlab::Danger::Roulette do let(:person_tz_offset_hours) { 0.0 } let(:person1) do Gitlab::Danger::Teammate.new( - 'username' => 'rymai', + 'username' => 'user1', 'available' => true, 'tz_offset_hours' => person_tz_offset_hours ) end + let(:person2) do Gitlab::Danger::Teammate.new( - 'username' => 'godfat', + 'username' => 'user2', 'available' => true, 'tz_offset_hours' => person_tz_offset_hours) end + let(:author) do Gitlab::Danger::Teammate.new( - 'username' => 'filipa', + 'username' => 'johndoe', 'available' => true, 'tz_offset_hours' => 0.0) end + let(:unavailable) do Gitlab::Danger::Teammate.new( - 'username' => 'jacopo-beschi', + 'username' => 'janedoe', 'available' => false, 'tz_offset_hours' => 0.0) end before do - allow(subject).to receive_message_chain(:gitlab, :mr_author).and_return(author.username) + allow(subject).to receive(:mr_author_username).and_return(author.username) end (-4..4).each do |utc_offset| @@ -328,7 +337,7 @@ RSpec.describe Gitlab::Danger::Roulette do selected = subject.spin_for_person(persons, random: Random.new, timezone_experiment: timezone_experiment) - expect(selected.username).to be_in(persons.map(&:username)) + expect(persons.map(&:username)).to include(selected.username) end end end @@ -349,7 +358,7 @@ RSpec.describe Gitlab::Danger::Roulette do if timezone_experiment expect(selected).to be_nil else - expect(selected.username).to be_in(persons.map(&:username)) + expect(persons.map(&:username)).to include(selected.username) end end end diff --git a/spec/lib/gitlab/danger/teammate_spec.rb b/spec/lib/gitlab/danger/teammate_spec.rb index a0540a9fbf5..12819614fab 100644 --- a/spec/lib/gitlab/danger/teammate_spec.rb +++ b/spec/lib/gitlab/danger/teammate_spec.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'fast_spec_helper' - require 'timecop' require 'rspec-parameterized' @@ -10,24 +8,45 @@ require 'gitlab/danger/teammate' RSpec.describe Gitlab::Danger::Teammate do using RSpec::Parameterized::TableSyntax - subject { described_class.new(options.stringify_keys) } + subject { described_class.new(options) } let(:tz_offset_hours) { 2.0 } let(:options) do { - username: 'luigi', - projects: projects, - role: role, - markdown_name: '[Luigi](https://gitlab.com/luigi) (`@luigi`)', - tz_offset_hours: tz_offset_hours + 'username' => 'luigi', + 'projects' => projects, + 'role' => role, + 'markdown_name' => '[Luigi](https://gitlab.com/luigi) (`@luigi`)', + 'tz_offset_hours' => tz_offset_hours } end + let(:capabilities) { ['reviewer backend'] } let(:projects) { { project => capabilities } } let(:role) { 'Engineer, Manage' } let(:labels) { [] } let(:project) { double } + describe '#==' do + it 'compares Teammate username' do + joe1 = described_class.new('username' => 'joe', 'projects' => projects) + joe2 = described_class.new('username' => 'joe', 'projects' => []) + jane1 = described_class.new('username' => 'jane', 'projects' => projects) + jane2 = described_class.new('username' => 'jane', 'projects' => []) + + expect(joe1).to eq(joe2) + expect(jane1).to eq(jane2) + expect(jane1).not_to eq(nil) + expect(described_class.new('username' => nil)).not_to eq(nil) + end + end + + describe '#to_h' do + it 'returns the given options' do + expect(subject.to_h).to eq(options) + end + end + context 'when having multiple capabilities' do let(:capabilities) { ['reviewer backend', 'maintainer frontend', 'trainee_maintainer qa'] } @@ -153,44 +172,44 @@ RSpec.describe Gitlab::Danger::Teammate do describe '#markdown_name' do context 'when timezone_experiment == false' do it 'returns markdown name as-is' do - expect(subject.markdown_name).to eq(options[:markdown_name]) - expect(subject.markdown_name(timezone_experiment: false)).to eq(options[:markdown_name]) + expect(subject.markdown_name).to eq(options['markdown_name']) + expect(subject.markdown_name(timezone_experiment: false)).to eq(options['markdown_name']) end end context 'when timezone_experiment == true' do it 'returns markdown name with timezone info' do - expect(subject.markdown_name(timezone_experiment: true)).to eq("#{options[:markdown_name]} (UTC+2)") + expect(subject.markdown_name(timezone_experiment: true)).to eq("#{options['markdown_name']} (UTC+2)") end context 'when offset is 1.5' do let(:tz_offset_hours) { 1.5 } it 'returns markdown name with timezone info, not truncated' do - expect(subject.markdown_name(timezone_experiment: true)).to eq("#{options[:markdown_name]} (UTC+1.5)") + expect(subject.markdown_name(timezone_experiment: true)).to eq("#{options['markdown_name']} (UTC+1.5)") end end context 'when author is given' do where(:tz_offset_hours, :author_offset, :diff_text) do -12 | -10 | "2 hours behind `@mario`" - -10 | -12 | "2 hours ahead `@mario`" + -10 | -12 | "2 hours ahead of `@mario`" -10 | 2 | "12 hours behind `@mario`" 2 | 4 | "2 hours behind `@mario`" - 4 | 2 | "2 hours ahead `@mario`" + 4 | 2 | "2 hours ahead of `@mario`" 2 | 3 | "1 hour behind `@mario`" - 3 | 2 | "1 hour ahead `@mario`" + 3 | 2 | "1 hour ahead of `@mario`" 2 | 2 | "same timezone as `@mario`" end with_them do it 'returns markdown name with timezone info' do - author = described_class.new(options.merge(username: 'mario', tz_offset_hours: author_offset).stringify_keys) + author = described_class.new(options.merge('username' => 'mario', 'tz_offset_hours' => author_offset)) floored_offset_hours = subject.__send__(:floored_offset_hours) utc_offset = floored_offset_hours >= 0 ? "+#{floored_offset_hours}" : floored_offset_hours - expect(subject.markdown_name(timezone_experiment: true, author: author)).to eq("#{options[:markdown_name]} (UTC#{utc_offset}, #{diff_text})") + expect(subject.markdown_name(timezone_experiment: true, author: author)).to eq("#{options['markdown_name']} (UTC#{utc_offset}, #{diff_text})") end end end diff --git a/spec/lib/gitlab/database/batch_count_spec.rb b/spec/lib/gitlab/database/batch_count_spec.rb index 656501dbf56..1f84a915cdc 100644 --- a/spec/lib/gitlab/database/batch_count_spec.rb +++ b/spec/lib/gitlab/database/batch_count_spec.rb @@ -13,11 +13,34 @@ RSpec.describe Gitlab::Database::BatchCount do let(:another_user) { create(:user) } before do - create_list(:issue, 3, author: user ) - create_list(:issue, 2, author: another_user ) + create_list(:issue, 3, author: user) + create_list(:issue, 2, author: another_user) allow(ActiveRecord::Base.connection).to receive(:transaction_open?).and_return(in_transaction) end + shared_examples 'disallowed configurations' do |method| + it 'returns fallback if start is bigger than finish' do + expect(described_class.public_send(method, *args, start: 1, finish: 0)).to eq(fallback) + end + + it 'returns fallback if loops more than allowed' do + large_finish = Gitlab::Database::BatchCounter::MAX_ALLOWED_LOOPS * default_batch_size + 1 + expect(described_class.public_send(method, *args, start: 1, finish: large_finish)).to eq(fallback) + end + + it 'returns fallback if batch size is less than min required' do + expect(described_class.public_send(method, *args, batch_size: small_batch_size)).to eq(fallback) + end + end + + shared_examples 'when a transaction is open' do + let(:in_transaction) { true } + + it 'raises an error' do + expect { subject }.to raise_error('BatchCount can not be run inside a transaction') + end + end + describe '#batch_count' do it 'counts table' do expect(described_class.batch_count(model)).to eq(5) @@ -53,38 +76,32 @@ RSpec.describe Gitlab::Database::BatchCount do [1, 2, 4, 5, 6].each { |i| expect(described_class.batch_count(model, batch_size: i)).to eq(5) } end - it 'will raise an error if distinct count is requested' do - expect do - described_class.batch_count(model.distinct(column)) - end.to raise_error 'Use distinct count for optimized distinct counting' + it 'counts with a start and finish' do + expect(described_class.batch_count(model, start: model.minimum(:id), finish: model.maximum(:id))).to eq(5) end - context 'in a transaction' do - let(:in_transaction) { true } + it "defaults the batch size to #{Gitlab::Database::BatchCounter::DEFAULT_BATCH_SIZE}" do + min_id = model.minimum(:id) - it 'cannot count' do - expect do - described_class.batch_count(model) - end.to raise_error 'BatchCount can not be run inside a transaction' + expect_next_instance_of(Gitlab::Database::BatchCounter) do |batch_counter| + expect(batch_counter).to receive(:batch_fetch).with(min_id, Gitlab::Database::BatchCounter::DEFAULT_BATCH_SIZE + min_id, :itself).once.and_call_original end - end - it 'counts with a start and finish' do - expect(described_class.batch_count(model, start: model.minimum(:id), finish: model.maximum(:id))).to eq(5) + described_class.batch_count(model) end - context 'disallowed configurations' do - it 'returns fallback if start is bigger than finish' do - expect(described_class.batch_count(model, start: 1, finish: 0)).to eq(fallback) - end + it_behaves_like 'when a transaction is open' do + subject { described_class.batch_count(model) } + end - it 'returns fallback if loops more than allowed' do - large_finish = Gitlab::Database::BatchCounter::MAX_ALLOWED_LOOPS * Gitlab::Database::BatchCounter::DEFAULT_BATCH_SIZE + 1 - expect(described_class.batch_count(model, start: 1, finish: large_finish)).to eq(fallback) + context 'disallowed_configurations' do + include_examples 'disallowed configurations', :batch_count do + let(:args) { [Issue] } + let(:default_batch_size) { Gitlab::Database::BatchCounter::DEFAULT_BATCH_SIZE } end - it 'returns fallback if batch size is less than min required' do - expect(described_class.batch_count(model, batch_size: small_batch_size)).to eq(fallback) + it 'raises an error if distinct count is requested' do + expect { described_class.batch_count(model.distinct(column)) }.to raise_error 'Use distinct count for optimized distinct counting' end end end @@ -128,18 +145,24 @@ RSpec.describe Gitlab::Database::BatchCount do expect(described_class.batch_distinct_count(model, column, start: User.minimum(:id), finish: User.maximum(:id))).to eq(2) end - context 'disallowed configurations' do - it 'returns fallback if start is bigger than finish' do - expect(described_class.batch_distinct_count(model, column, start: 1, finish: 0)).to eq(fallback) - end + it "defaults the batch size to #{Gitlab::Database::BatchCounter::DEFAULT_DISTINCT_BATCH_SIZE}" do + min_id = model.minimum(:id) - it 'returns fallback if loops more than allowed' do - large_finish = Gitlab::Database::BatchCounter::MAX_ALLOWED_LOOPS * Gitlab::Database::BatchCounter::DEFAULT_DISTINCT_BATCH_SIZE + 1 - expect(described_class.batch_distinct_count(model, column, start: 1, finish: large_finish)).to eq(fallback) + expect_next_instance_of(Gitlab::Database::BatchCounter) do |batch_counter| + expect(batch_counter).to receive(:batch_fetch).with(min_id, Gitlab::Database::BatchCounter::DEFAULT_DISTINCT_BATCH_SIZE + min_id, :distinct).once.and_call_original end - it 'returns fallback if batch size is less than min required' do - expect(described_class.batch_distinct_count(model, column, batch_size: small_batch_size)).to eq(fallback) + described_class.batch_distinct_count(model) + end + + it_behaves_like 'when a transaction is open' do + subject { described_class.batch_distinct_count(model, column) } + end + + context 'disallowed configurations' do + include_examples 'disallowed configurations', :batch_distinct_count do + let(:args) { [model, column] } + let(:default_batch_size) { Gitlab::Database::BatchCounter::DEFAULT_DISTINCT_BATCH_SIZE } end it 'will raise an error if distinct count with the :id column is requested' do @@ -149,4 +172,55 @@ RSpec.describe Gitlab::Database::BatchCount do end end end + + describe '#batch_sum' do + let(:column) { :weight } + + before do + Issue.first.update_attribute(column, 3) + Issue.last.update_attribute(column, 4) + end + + it 'returns the sum of values in the given column' do + expect(described_class.batch_sum(model, column)).to eq(7) + end + + it 'works when given an Arel column' do + expect(described_class.batch_sum(model, model.arel_table[column])).to eq(7) + end + + it 'works with a batch size of 50K' do + expect(described_class.batch_sum(model, column, batch_size: 50_000)).to eq(7) + end + + it 'works with start and finish provided' do + expect(described_class.batch_sum(model, column, start: model.minimum(:id), finish: model.maximum(:id))).to eq(7) + end + + it 'returns the same result regardless of batch size' do + stub_const('Gitlab::Database::BatchCounter::DEFAULT_SUM_BATCH_SIZE', 0) + + (1..(model.count + 1)).each { |i| expect(described_class.batch_sum(model, column, batch_size: i)).to eq(7) } + end + + it "defaults the batch size to #{Gitlab::Database::BatchCounter::DEFAULT_SUM_BATCH_SIZE}" do + min_id = model.minimum(:id) + + expect_next_instance_of(Gitlab::Database::BatchCounter) do |batch_counter| + expect(batch_counter).to receive(:batch_fetch).with(min_id, Gitlab::Database::BatchCounter::DEFAULT_SUM_BATCH_SIZE + min_id, :itself).once.and_call_original + end + + described_class.batch_sum(model, column) + end + + it_behaves_like 'when a transaction is open' do + subject { described_class.batch_sum(model, column) } + end + + it_behaves_like 'disallowed configurations', :batch_sum do + let(:args) { [model, column] } + let(:default_batch_size) { Gitlab::Database::BatchCounter::DEFAULT_SUM_BATCH_SIZE } + let(:small_batch_size) { Gitlab::Database::BatchCounter::DEFAULT_SUM_BATCH_SIZE - 1 } + end + end end diff --git a/spec/lib/gitlab/database/count/tablesample_count_strategy_spec.rb b/spec/lib/gitlab/database/count/tablesample_count_strategy_spec.rb index e488bf5ee4c..c2028f8c238 100644 --- a/spec/lib/gitlab/database/count/tablesample_count_strategy_spec.rb +++ b/spec/lib/gitlab/database/count/tablesample_count_strategy_spec.rb @@ -23,6 +23,7 @@ RSpec.describe Gitlab::Database::Count::TablesampleCountStrategy do Namespace => threshold + 1 } end + let(:threshold) { Gitlab::Database::Count::TablesampleCountStrategy::EXACT_COUNT_THRESHOLD } before do diff --git a/spec/lib/gitlab/database/custom_structure_spec.rb b/spec/lib/gitlab/database/custom_structure_spec.rb index beda9df3684..b3bdca0acdd 100644 --- a/spec/lib/gitlab/database/custom_structure_spec.rb +++ b/spec/lib/gitlab/database/custom_structure_spec.rb @@ -32,6 +32,7 @@ RSpec.describe Gitlab::Database::CustomStructure do Gitlab::Database::PartitioningMigrationHelpers::PartitionedForeignKey.create( cascade_delete: true, from_table: 'issues', from_column: 'project_id', to_table: 'projects', to_column: 'id') end + let!(:second_fk) do Gitlab::Database::PartitioningMigrationHelpers::PartitionedForeignKey.create( cascade_delete: false, from_table: 'issues', from_column: 'moved_to_id', to_table: 'issues', to_column: 'id') diff --git a/spec/lib/gitlab/database/migration_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers_spec.rb index 48e1c97e97f..4b7f371b25a 100644 --- a/spec/lib/gitlab/database/migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers_spec.rb @@ -712,7 +712,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do expect(model).to receive(:add_not_null_constraint).with(:users, :new) expect(model).to receive(:execute).with("UPDATE \"users\" SET \"new\" = cast_to_jsonb_with_default(\"users\".\"id\") WHERE \"users\".\"id\" >= #{user.id}") expect(model).to receive(:execute).with("DROP TRIGGER IF EXISTS #{trigger_name}\nON \"users\"\n") - expect(model).to receive(:execute).with("CREATE TRIGGER #{trigger_name}\nBEFORE INSERT OR UPDATE\nON \"users\"\nFOR EACH ROW\nEXECUTE PROCEDURE #{trigger_name}()\n") + expect(model).to receive(:execute).with("CREATE TRIGGER #{trigger_name}\nBEFORE INSERT OR UPDATE\nON \"users\"\nFOR EACH ROW\nEXECUTE FUNCTION #{trigger_name}()\n") expect(model).to receive(:execute).with("CREATE OR REPLACE FUNCTION #{trigger_name}()\nRETURNS trigger AS\n$BODY$\nBEGIN\n NEW.\"new\" := NEW.\"id\";\n RETURN NEW;\nEND;\n$BODY$\nLANGUAGE 'plpgsql'\nVOLATILE\n") model.rename_column_concurrently(:users, :id, :new, type_cast_function: 'cast_to_jsonb_with_default') diff --git a/spec/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers_spec.rb b/spec/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers_spec.rb index efa9c83b2d2..7d88c17c9b3 100644 --- a/spec/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers_spec.rb +++ b/spec/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers_spec.rb @@ -8,6 +8,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::ForeignKeyHelpers let(:model) do ActiveRecord::Migration.new.extend(described_class) end + let_it_be(:connection) { ActiveRecord::Base.connection } let(:referenced_table) { :issues } let(:function_name) { '_test_partitioned_foreign_keys_function' } diff --git a/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb b/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb index 9b24ab7cad4..86f79b213ae 100644 --- a/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb +++ b/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb @@ -315,42 +315,13 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe expect(model.find(second_todo.id).attributes).to eq(second_todo.attributes) end end - - describe 'copying historic data to the partitioned table' do - let(:source_table) { 'todos' } - let(:migration_class) { '::Gitlab::Database::PartitioningMigrationHelpers::BackfillPartitionedTable' } - let(:sub_batch_size) { described_class::SUB_BATCH_SIZE } - let(:pause_seconds) { described_class::PAUSE_SECONDS } - let!(:first_id) { create(:todo).id } - let!(:second_id) { create(:todo).id } - let!(:third_id) { create(:todo).id } - - before do - stub_const("#{described_class.name}::BATCH_SIZE", 2) - - expect(migration).to receive(:queue_background_migration_jobs_by_range_at_intervals).and_call_original - end - - it 'enqueues jobs to copy each batch of data' do - Sidekiq::Testing.fake! do - migration.partition_table_by_date source_table, partition_column, min_date: min_date, max_date: max_date - - expect(BackgroundMigrationWorker.jobs.size).to eq(2) - - first_job_arguments = [first_id, second_id, source_table, partitioned_table, 'id'] - expect(BackgroundMigrationWorker.jobs[0]['args']).to eq([migration_class, first_job_arguments]) - - second_job_arguments = [third_id, third_id, source_table, partitioned_table, 'id'] - expect(BackgroundMigrationWorker.jobs[1]['args']).to eq([migration_class, second_job_arguments]) - end - end - end end describe '#drop_partitioned_table_for' do let(:expected_tables) do %w[000000 201912 202001 202002].map { |suffix| "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{partitioned_table}_#{suffix}" }.unshift(partitioned_table) end + let(:migration_class) { 'Gitlab::Database::PartitioningMigrationHelpers::BackfillPartitionedTable' } context 'when the table is not allowed' do @@ -390,16 +361,85 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe expect(connection.table_exists?(table)).to be(false) end end + end + + describe '#enqueue_partitioning_data_migration' do + context 'when the table is not allowed' do + let(:source_table) { :this_table_is_not_allowed } + + it 'raises an error' do + expect(migration).to receive(:assert_table_is_allowed).with(source_table).and_call_original + + expect do + migration.enqueue_partitioning_data_migration source_table + end.to raise_error(/#{source_table} is not allowed for use/) + end + end - context 'cleaning up background migration tracking records' do + context 'when run inside a transaction block' do + it 'raises an error' do + expect(migration).to receive(:transaction_open?).and_return(true) + + expect do + migration.enqueue_partitioning_data_migration source_table + end.to raise_error(/can not be run inside a transaction/) + end + end + + context 'when records exist in the source table' do + let(:source_table) { 'todos' } + let(:migration_class) { '::Gitlab::Database::PartitioningMigrationHelpers::BackfillPartitionedTable' } + let(:sub_batch_size) { described_class::SUB_BATCH_SIZE } + let(:pause_seconds) { described_class::PAUSE_SECONDS } + let!(:first_id) { create(:todo).id } + let!(:second_id) { create(:todo).id } + let!(:third_id) { create(:todo).id } + + before do + stub_const("#{described_class.name}::BATCH_SIZE", 2) + + expect(migration).to receive(:queue_background_migration_jobs_by_range_at_intervals).and_call_original + end + + it 'enqueues jobs to copy each batch of data' do + migration.partition_table_by_date source_table, partition_column, min_date: min_date, max_date: max_date + + Sidekiq::Testing.fake! do + migration.enqueue_partitioning_data_migration source_table + + expect(BackgroundMigrationWorker.jobs.size).to eq(2) + + first_job_arguments = [first_id, second_id, source_table, partitioned_table, 'id'] + expect(BackgroundMigrationWorker.jobs[0]['args']).to eq([migration_class, first_job_arguments]) + + second_job_arguments = [third_id, third_id, source_table, partitioned_table, 'id'] + expect(BackgroundMigrationWorker.jobs[1]['args']).to eq([migration_class, second_job_arguments]) + end + end + end + end + + describe '#cleanup_partitioning_data_migration' do + context 'when the table is not allowed' do + let(:source_table) { :this_table_is_not_allowed } + + it 'raises an error' do + expect(migration).to receive(:assert_table_is_allowed).with(source_table).and_call_original + + expect do + migration.cleanup_partitioning_data_migration source_table + end.to raise_error(/#{source_table} is not allowed for use/) + end + end + + context 'when tracking records exist in the background_migration_jobs table' do + let(:migration_class) { 'Gitlab::Database::PartitioningMigrationHelpers::BackfillPartitionedTable' } let!(:job1) { create(:background_migration_job, class_name: migration_class, arguments: [1, 10, source_table]) } let!(:job2) { create(:background_migration_job, class_name: migration_class, arguments: [11, 20, source_table]) } let!(:job3) { create(:background_migration_job, class_name: migration_class, arguments: [1, 10, 'other_table']) } - it 'deletes any tracking records from the background_migration_jobs table' do - migration.partition_table_by_date source_table, partition_column, min_date: min_date, max_date: max_date - - expect { migration.drop_partitioned_table_for(source_table) } + it 'deletes those pertaining to the given table' do + expect { migration.cleanup_partitioning_data_migration(source_table) } .to change { ::Gitlab::Database::BackgroundMigrationJob.count }.from(3).to(1) remaining_record = ::Gitlab::Database::BackgroundMigrationJob.first diff --git a/spec/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin_spec.rb b/spec/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin_spec.rb new file mode 100644 index 00000000000..ca9f4af9187 --- /dev/null +++ b/spec/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin_spec.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::PostgresqlAdapter::DumpSchemaVersionsMixin do + let(:schema_migration) { double('schema_migration', all_versions: versions) } + + let(:instance) do + Object.new.extend(described_class) + end + + before do + allow(instance).to receive(:schema_migration).and_return(schema_migration) + end + + context 'when version files exist' do + let(:versions) { %w(5 2 1000 200 4 93 2) } + + it 'touches version files' do + expect(Gitlab::Database::SchemaVersionFiles).to receive(:touch_all).with(versions) + + instance.dump_schema_information + end + end + + context 'when version files do not exist' do + let(:versions) { [] } + + it 'does not touch version files' do + expect(Gitlab::Database::SchemaVersionFiles).not_to receive(:touch_all) + + instance.dump_schema_information + end + end +end diff --git a/spec/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin_spec.rb b/spec/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin_spec.rb index 8b3a0ceb804..ea8c9e2cfd7 100644 --- a/spec/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin_spec.rb +++ b/spec/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin_spec.rb @@ -13,6 +13,7 @@ RSpec.describe Gitlab::Database::PostgresqlAdapter::ForceDisconnectableMixin do end end end + let(:config) { Rails.application.config_for(:database).merge(pool: 1) } let(:pool) { model.establish_connection(config) } diff --git a/spec/lib/gitlab/database/postgresql_adapter/schema_versions_copy_mixin_spec.rb b/spec/lib/gitlab/database/postgresql_adapter/schema_versions_copy_mixin_spec.rb deleted file mode 100644 index c6333e4a4dc..00000000000 --- a/spec/lib/gitlab/database/postgresql_adapter/schema_versions_copy_mixin_spec.rb +++ /dev/null @@ -1,42 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Database::PostgresqlAdapter::SchemaVersionsCopyMixin do - let(:schema_migration) { double('schem_migration', table_name: table_name, all_versions: versions) } - let(:versions) { %w(5 2 1000 200 4 93 2) } - let(:table_name) { "schema_migrations" } - - let(:instance) do - Object.new.extend(described_class) - end - - before do - allow(instance).to receive(:schema_migration).and_return(schema_migration) - allow(instance).to receive(:quote_table_name).with(table_name).and_return("\"#{table_name}\"") - end - - subject { instance.dump_schema_information } - - it 'uses COPY FROM STDIN' do - expect(subject.split("\n").first).to match(/COPY "schema_migrations" \(version\) FROM STDIN;/) - end - - it 'contains a sorted list of versions by their numeric value' do - version_lines = subject.split("\n")[1..-2].map(&:to_i) - - expect(version_lines).to eq(versions.map(&:to_i).sort) - end - - it 'contains a end-of-data marker' do - expect(subject).to end_with("\\.\n") - end - - context 'with non-Integer versions' do - let(:versions) { %w(5 2 4 abc) } - - it 'raises an error' do - expect { subject }.to raise_error(/invalid value for Integer/) - end - end -end diff --git a/spec/lib/gitlab/database/schema_version_files_spec.rb b/spec/lib/gitlab/database/schema_version_files_spec.rb new file mode 100644 index 00000000000..c3b3ae0a07f --- /dev/null +++ b/spec/lib/gitlab/database/schema_version_files_spec.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaVersionFiles do + describe '.touch_all' do + let(:version1) { '20200123' } + let(:version2) { '20200410' } + let(:version3) { '20200602' } + let(:version4) { '20200809' } + let(:relative_schema_directory) { 'db/schema_migrations' } + let(:relative_migrate_directory) { 'db/migrate' } + let(:relative_post_migrate_directory) { 'db/post_migrate' } + + it 'creates a file containing a checksum for each version with a matching migration' do + Dir.mktmpdir do |tmpdir| + schema_directory = Pathname.new(tmpdir).join(relative_schema_directory) + migrate_directory = Pathname.new(tmpdir).join(relative_migrate_directory) + post_migrate_directory = Pathname.new(tmpdir).join(relative_post_migrate_directory) + + FileUtils.mkdir_p(migrate_directory) + FileUtils.mkdir_p(post_migrate_directory) + FileUtils.mkdir_p(schema_directory) + + migration1_filepath = migrate_directory.join("#{version1}_migration.rb") + FileUtils.touch(migration1_filepath) + + migration2_filepath = post_migrate_directory.join("#{version2}_post_migration.rb") + FileUtils.touch(migration2_filepath) + + old_version_filepath = schema_directory.join('20200101') + FileUtils.touch(old_version_filepath) + + expect(File.exist?(old_version_filepath)).to be(true) + + allow(described_class).to receive(:schema_directory).and_return(schema_directory) + allow(described_class).to receive(:migration_directories).and_return([migrate_directory, post_migrate_directory]) + + described_class.touch_all([version1, version2, version3, version4]) + + expect(File.exist?(old_version_filepath)).to be(false) + [version1, version2].each do |version| + version_filepath = schema_directory.join(version) + expect(File.exist?(version_filepath)).to be(true) + + hashed_value = Digest::SHA256.hexdigest(version) + expect(File.read(version_filepath)).to eq(hashed_value) + end + + [version3, version4].each do |version| + version_filepath = schema_directory.join(version) + expect(File.exist?(version_filepath)).to be(false) + end + end + end + end + + describe '.load_all' do + let(:connection) { double('connection') } + + before do + allow(described_class).to receive(:connection).and_return(connection) + allow(described_class).to receive(:find_version_filenames).and_return(filenames) + end + + context 'when there are no version files' do + let(:filenames) { [] } + + it 'does nothing' do + expect(connection).not_to receive(:quote_string) + expect(connection).not_to receive(:execute) + + described_class.load_all + end + end + + context 'when there are version files' do + let(:filenames) { %w[123 456 789] } + + it 'inserts the missing versions into schema_migrations' do + filenames.each do |filename| + expect(connection).to receive(:quote_string).with(filename).and_return(filename) + end + + expect(connection).to receive(:execute).with(<<~SQL) + INSERT INTO schema_migrations (version) + VALUES ('123'),('456'),('789') + ON CONFLICT DO NOTHING + SQL + + described_class.load_all + end + end + end +end diff --git a/spec/lib/gitlab/database/similarity_score_spec.rb b/spec/lib/gitlab/database/similarity_score_spec.rb new file mode 100644 index 00000000000..e36a4f610e1 --- /dev/null +++ b/spec/lib/gitlab/database/similarity_score_spec.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SimilarityScore do + let(:search) { '' } + let(:query_result) { ActiveRecord::Base.connection.execute(query).to_a } + + let(:query) do + # In memory query, with the id as the tie breaker. + <<-SQL + SELECT *, #{order_expression} AS similarity + FROM ( + VALUES (1, 'Git', 'git', 'git source code mirror. this is a publish-only repository.'), + (2, 'GitLab Runner', 'gitlab-runner', 'official helm chart for the gitlab runner'), + (3, 'gitaly', 'gitaly', 'gitaly is a git rpc service for handling all the git calls made by gitlab'), + (4, 'GitLab', 'gitlab', 'gitlab is an open source end-to-end software development platform with built-in version control'), + (5, 'Gitlab Danger', 'gitlab-danger', 'this gem provides common dangerfile and plugins for gitlab projects'), + (6, 'different', 'same', 'same'), + (7, 'same', 'different', 'same'), + (8, 'gitlab-styles', 'gitlab-styles', 'gitlab style guides and shared style configs.'), + (9, '🔒 gitaly', 'gitaly-sec', 'security mirror for gitaly') + ) tbl (id, name, path, descrption) ORDER BY #{order_expression} DESC, id DESC; + SQL + end + + let(:order_expression) do + Gitlab::Database::SimilarityScore.build_expression(search: search, rules: [{ column: Arel.sql('path') }]).to_sql + end + + subject { query_result.take(3).map { |row| row['path'] } } + + context 'when passing empty values' do + context 'when search is nil' do + let(:search) { nil } + + it 'orders by a constant 0 value' do + expect(query).to include('ORDER BY CAST(0 AS integer) DESC') + end + end + + context 'when rules are empty' do + let(:search) { 'text' } + + let(:order_expression) do + Gitlab::Database::SimilarityScore.build_expression(search: search, rules: []).to_sql + end + + it 'orders by a constant 0 value' do + expect(query).to include('ORDER BY CAST(0 AS integer) DESC') + end + end + end + + context 'when similarity scoring based on the path' do + let(:search) { 'git' } + + context 'when searching for `git`' do + let(:search) { 'git' } + + it { expect(subject).to eq(%w[git gitlab gitaly]) } + end + + context 'when searching for `gitlab`' do + let(:search) { 'gitlab' } + + it { expect(subject).to eq(%w[gitlab gitlab-styles gitlab-danger]) } + end + + context 'when searching for something unrelated' do + let(:search) { 'xyz' } + + it 'results have 0 similarity score' do + expect(query_result.map { |row| row['similarity'] }).to all(eq(0)) + end + end + end + + describe 'score multiplier' do + let(:order_expression) do + Gitlab::Database::SimilarityScore.build_expression(search: search, rules: [ + { column: Arel.sql('path'), multiplier: 1 }, + { column: Arel.sql('name'), multiplier: 0.8 } + ]).to_sql + end + + let(:search) { 'different' } + + it 'ranks `path` matches higher' do + expect(subject).to eq(%w[different same gitlab-danger]) + end + end +end diff --git a/spec/lib/gitlab/database/with_lock_retries_spec.rb b/spec/lib/gitlab/database/with_lock_retries_spec.rb index 70cbddbb7b7..2cc6e175500 100644 --- a/spec/lib/gitlab/database/with_lock_retries_spec.rb +++ b/spec/lib/gitlab/database/with_lock_retries_spec.rb @@ -72,9 +72,14 @@ RSpec.describe Gitlab::Database::WithLockRetries do lock_attempts = 0 lock_acquired = false - expect_any_instance_of(Gitlab::Database::WithLockRetries).to receive(:sleep).exactly(retry_count - 1).times # we don't sleep in the last iteration - - allow_any_instance_of(Gitlab::Database::WithLockRetries).to receive(:run_block_with_transaction).and_wrap_original do |method| + # the actual number of attempts to run_block_with_transaction can never exceed the number of + # timings_configurations, so here we limit the retry_count if it exceeds that value + # + # also, there is no call to sleep after the final attempt, which is why it will always be one less + expected_runs_with_timeout = [retry_count, timing_configuration.size].min + expect(subject).to receive(:sleep).exactly(expected_runs_with_timeout - 1).times + + expect(subject).to receive(:run_block_with_transaction).exactly(expected_runs_with_timeout).times.and_wrap_original do |method| lock_fiber.resume if lock_attempts == retry_count method.call @@ -114,6 +119,33 @@ RSpec.describe Gitlab::Database::WithLockRetries do end end + context 'after the retries, when requested to raise an error' do + let(:expected_attempts_with_timeout) { timing_configuration.size } + let(:retry_count) { timing_configuration.size + 1 } + + it 'raises an error instead of waiting indefinitely for the lock' do + lock_attempts = 0 + lock_acquired = false + + expect(subject).to receive(:sleep).exactly(expected_attempts_with_timeout - 1).times + expect(subject).to receive(:run_block_with_transaction).exactly(expected_attempts_with_timeout).times.and_call_original + + expect do + subject.run(raise_on_exhaustion: true) do + lock_attempts += 1 + + ActiveRecord::Base.transaction do + ActiveRecord::Base.connection.execute("LOCK TABLE #{Project.table_name} in exclusive mode") + lock_acquired = true + end + end + end.to raise_error(described_class::AttemptsExhaustedError) + + expect(lock_attempts).to eq(retry_count - 1) + expect(lock_acquired).to eq(false) + end + end + context 'when statement timeout is reached' do it 'raises QueryCanceled error' do lock_acquired = false diff --git a/spec/lib/gitlab/diff/file_collection/commit_spec.rb b/spec/lib/gitlab/diff/file_collection/commit_spec.rb index 6c109e96a53..7773604a638 100644 --- a/spec/lib/gitlab/diff/file_collection/commit_spec.rb +++ b/spec/lib/gitlab/diff/file_collection/commit_spec.rb @@ -9,6 +9,7 @@ RSpec.describe Gitlab::Diff::FileCollection::Commit do let(:collection_default_args) do { diff_options: {} } end + let(:diffable) { project.commit } let(:stub_path) { 'bar/branch-test.txt' } end diff --git a/spec/lib/gitlab/diff/file_collection/compare_spec.rb b/spec/lib/gitlab/diff/file_collection/compare_spec.rb index 168d58e584e..dda4513a3a1 100644 --- a/spec/lib/gitlab/diff/file_collection/compare_spec.rb +++ b/spec/lib/gitlab/diff/file_collection/compare_spec.rb @@ -23,6 +23,7 @@ RSpec.describe Gitlab::Diff::FileCollection::Compare do diff_refs: diffable.diff_refs } end + let(:diffable) { Compare.new(raw_compare, project) } let(:stub_path) { '.gitignore' } end diff --git a/spec/lib/gitlab/diff/file_collection/merge_request_diff_spec.rb b/spec/lib/gitlab/diff/file_collection/merge_request_diff_spec.rb index a5e714c90fc..429e552278d 100644 --- a/spec/lib/gitlab/diff/file_collection/merge_request_diff_spec.rb +++ b/spec/lib/gitlab/diff/file_collection/merge_request_diff_spec.rb @@ -42,6 +42,7 @@ RSpec.describe Gitlab::Diff::FileCollection::MergeRequestDiff do let(:collection_default_args) do { diff_options: {} } end + let(:diffable) { merge_request.merge_request_diff } let(:stub_path) { '.gitignore' } end diff --git a/spec/lib/gitlab/diff/file_spec.rb b/spec/lib/gitlab/diff/file_spec.rb index 94abfcf079a..78be89c449b 100644 --- a/spec/lib/gitlab/diff/file_spec.rb +++ b/spec/lib/gitlab/diff/file_spec.rb @@ -800,6 +800,7 @@ RSpec.describe Gitlab::Diff::File do let(:project) do create(:project, :custom_repo, files: {}) end + let(:branch_name) { 'master' } context 'when empty file is created' do @@ -842,6 +843,7 @@ RSpec.describe Gitlab::Diff::File do let(:project) do create(:project, :custom_repo, files: {}) end + let(:branch_name) { 'master' } context 'when empty file is created' do diff --git a/spec/lib/gitlab/diff/highlight_cache_spec.rb b/spec/lib/gitlab/diff/highlight_cache_spec.rb index 80cc10051c4..7e926f86096 100644 --- a/spec/lib/gitlab/diff/highlight_cache_spec.rb +++ b/spec/lib/gitlab/diff/highlight_cache_spec.rb @@ -173,57 +173,32 @@ RSpec.describe Gitlab::Diff::HighlightCache, :clean_gitlab_redis_cache do fallback_diff_refs: diffs.fallback_diff_refs) end - context "feature flag :gzip_diff_cache disabled" do - before do - stub_feature_flags(gzip_diff_cache: true) - end - - it "uses ActiveSupport::Gzip when reading from the cache" do - expect(ActiveSupport::Gzip).to receive(:decompress).at_least(:once).and_call_original - - cache.write_if_empty - cache.decorate(diff_file) - end + it "uses ActiveSupport::Gzip when reading from the cache" do + expect(ActiveSupport::Gzip).to receive(:decompress).at_least(:once).and_call_original - it "uses ActiveSupport::Gzip to compress data when writing to cache" do - expect(ActiveSupport::Gzip).to receive(:compress).and_call_original - - cache.send(:write_to_redis_hash, diff_hash) - end + cache.write_if_empty + cache.decorate(diff_file) end - context "feature flag :gzip_diff_cache disabled" do - before do - stub_feature_flags(gzip_diff_cache: false) - end - - it "doesn't use ActiveSupport::Gzip when reading from the cache" do - expect(ActiveSupport::Gzip).not_to receive(:decompress) - - cache.write_if_empty - cache.decorate(diff_file) - end - - it "doesn't use ActiveSupport::Gzip to compress data when writing to cache" do - expect(ActiveSupport::Gzip).not_to receive(:compress) + it "uses ActiveSupport::Gzip to compress data when writing to cache" do + expect(ActiveSupport::Gzip).to receive(:compress).and_call_original - expect { cache.send(:write_to_redis_hash, diff_hash) } - .to change { Gitlab::Redis::Cache.with { |r| r.hgetall(cache_key) } } - end + cache.send(:write_to_redis_hash, diff_hash) end end describe 'metrics' do - it 'defines :gitlab_redis_diff_caching_memory_usage_bytes histogram' do - expect(described_class).to respond_to(:gitlab_redis_diff_caching_memory_usage_bytes) - end + let(:transaction) { Gitlab::Metrics::WebTransaction.new({} ) } - it 'defines :gitlab_redis_diff_caching_hit' do - expect(described_class).to respond_to(:gitlab_redis_diff_caching_hit) + before do + allow(cache).to receive(:current_transaction).and_return(transaction) end - it 'defines :gitlab_redis_diff_caching_miss' do - expect(described_class).to respond_to(:gitlab_redis_diff_caching_miss) + it 'observes :gitlab_redis_diff_caching_memory_usage_bytes' do + expect(transaction) + .to receive(:observe).with(:gitlab_redis_diff_caching_memory_usage_bytes, a_kind_of(Numeric)) + + cache.write_if_empty end end end diff --git a/spec/lib/gitlab/diff/position_collection_spec.rb b/spec/lib/gitlab/diff/position_collection_spec.rb index b1478c774f1..d2bb82983c0 100644 --- a/spec/lib/gitlab/diff/position_collection_spec.rb +++ b/spec/lib/gitlab/diff/position_collection_spec.rb @@ -8,9 +8,11 @@ RSpec.describe Gitlab::Diff::PositionCollection do let(:text_position) do build(:text_diff_position, :added, diff_refs: diff_refs) end + let(:folded_text_position) do build(:text_diff_position, diff_refs: diff_refs, old_line: 1, new_line: 1) end + let(:image_position) do build(:image_diff_position, diff_refs: diff_refs) end diff --git a/spec/lib/gitlab/diff/stats_cache_spec.rb b/spec/lib/gitlab/diff/stats_cache_spec.rb index 8bf510c0bdd..5b01c1913bf 100644 --- a/spec/lib/gitlab/diff/stats_cache_spec.rb +++ b/spec/lib/gitlab/diff/stats_cache_spec.rb @@ -9,6 +9,7 @@ RSpec.describe Gitlab::Diff::StatsCache, :use_clean_rails_memory_store_caching d let(:cachable_key) { 'cachecachecache' } let(:stat) { Gitaly::DiffStats.new(path: 'temp', additions: 10, deletions: 15) } let(:stats) { Gitlab::Git::DiffStatsCollection.new([stat]) } + let(:serialized_stats) { stats.map(&:to_h).as_json } let(:cache) { Rails.cache } describe '#read' do @@ -38,7 +39,7 @@ RSpec.describe Gitlab::Diff::StatsCache, :use_clean_rails_memory_store_caching d it 'writes the stats' do expect(cache) .to receive(:write) - .with(key, stats.as_json, expires_in: described_class::EXPIRATION) + .with(key, serialized_stats, expires_in: described_class::EXPIRATION) .and_call_original stats_cache.write_if_empty(stats) @@ -53,7 +54,7 @@ RSpec.describe Gitlab::Diff::StatsCache, :use_clean_rails_memory_store_caching d it 'writes the stats' do expect(cache) .to receive(:write) - .with(key, stats.as_json, expires_in: described_class::EXPIRATION) + .with(key, serialized_stats, expires_in: described_class::EXPIRATION) .and_call_original stats_cache.write_if_empty(stats) @@ -81,4 +82,28 @@ RSpec.describe Gitlab::Diff::StatsCache, :use_clean_rails_memory_store_caching d stats_cache.clear end end + + it 'VERSION is set' do + expect(described_class::VERSION).to be_present + end + + context 'with multiple cache versions' do + before do + stats_cache.write_if_empty(stats) + end + + it 'does not read from a stale cache' do + expect(stats_cache.read.to_json).to eq(stats.to_json) + + stub_const('Gitlab::Diff::StatsCache::VERSION', '1.0.new-new-thing') + + stats_cache = described_class.new(cachable_key: cachable_key) + + expect(stats_cache.read).to be_nil + + stats_cache.write_if_empty(stats) + + expect(stats_cache.read.to_json).to eq(stats.to_json) + end + end end diff --git a/spec/lib/gitlab/diff/suggestion_spec.rb b/spec/lib/gitlab/diff/suggestion_spec.rb index 5a5c5555818..40779faf917 100644 --- a/spec/lib/gitlab/diff/suggestion_spec.rb +++ b/spec/lib/gitlab/diff/suggestion_spec.rb @@ -31,9 +31,11 @@ RSpec.describe Gitlab::Diff::Suggestion do new_line: 9, diff_refs: merge_request.diff_refs) end + let(:diff_file) do position.diff_file(project.repository) end + let(:text) { "# parsed suggestion content\n# with comments" } def blob_lines_data(from_line, to_line) diff --git a/spec/lib/gitlab/email/message/repository_push_spec.rb b/spec/lib/gitlab/email/message/repository_push_spec.rb index 10586527239..6b1f03e0385 100644 --- a/spec/lib/gitlab/email/message/repository_push_spec.rb +++ b/spec/lib/gitlab/email/message/repository_push_spec.rb @@ -18,10 +18,12 @@ RSpec.describe Gitlab::Email::Message::RepositoryPush do { author_id: author.id, ref: 'master', action: :push, compare: compare, send_from_committer_email: true } end + let(:raw_compare) do Gitlab::Git::Compare.new(project.repository.raw_repository, sample_image_commit.id, sample_commit.id) end + let(:compare) do Compare.decorate(raw_compare, project) end diff --git a/spec/lib/gitlab/encoding_helper_spec.rb b/spec/lib/gitlab/encoding_helper_spec.rb index 5394c04c6ba..0ea974921bc 100644 --- a/spec/lib/gitlab/encoding_helper_spec.rb +++ b/spec/lib/gitlab/encoding_helper_spec.rb @@ -138,6 +138,7 @@ RSpec.describe Gitlab::EncodingHelper do let(:test_string) do "refs/heads/FixSymbolsTitleDropdown".encode("ASCII-8BIT") end + let(:expected_string) do "refs/heads/FixSymbolsTitleDropdown".encode("UTF-8") end diff --git a/spec/lib/gitlab/experimentation_spec.rb b/spec/lib/gitlab/experimentation_spec.rb index a6408aeae8b..2de5e1e20d6 100644 --- a/spec/lib/gitlab/experimentation_spec.rb +++ b/spec/lib/gitlab/experimentation_spec.rb @@ -233,6 +233,68 @@ RSpec.describe Gitlab::Experimentation do end end end + + describe '#record_experiment_user' do + let(:user) { build(:user) } + + context 'when the experiment is enabled' do + before do + stub_experiment(test_experiment: true) + allow(controller).to receive(:current_user).and_return(user) + end + + context 'the user is part of the experimental group' do + before do + stub_experiment_for_user(test_experiment: true) + end + + it 'calls add_user on the Experiment model' do + expect(::Experiment).to receive(:add_user).with(:test_experiment, :experimental, user) + + controller.record_experiment_user(:test_experiment) + end + end + + context 'the user is part of the control group' do + before do + allow_next_instance_of(described_class) do |instance| + allow(instance).to receive(:experiment_enabled?).with(:test_experiment).and_return(false) + end + end + + it 'calls add_user on the Experiment model' do + expect(::Experiment).to receive(:add_user).with(:test_experiment, :control, user) + + controller.record_experiment_user(:test_experiment) + end + end + end + + context 'when the experiment is disabled' do + before do + stub_experiment(test_experiment: false) + allow(controller).to receive(:current_user).and_return(user) + end + + it 'does not call add_user on the Experiment model' do + expect(::Experiment).not_to receive(:add_user) + + controller.record_experiment_user(:test_experiment) + end + end + + context 'when there is no current_user' do + before do + stub_experiment(test_experiment: true) + end + + it 'does not call add_user on the Experiment model' do + expect(::Experiment).not_to receive(:add_user) + + controller.record_experiment_user(:test_experiment) + end + end + end end describe '.enabled?' do diff --git a/spec/lib/gitlab/external_authorization/client_spec.rb b/spec/lib/gitlab/external_authorization/client_spec.rb index 473b57441fa..c08da382486 100644 --- a/spec/lib/gitlab/external_authorization/client_spec.rb +++ b/spec/lib/gitlab/external_authorization/client_spec.rb @@ -14,7 +14,7 @@ RSpec.describe Gitlab::ExternalAuthorization::Client do describe '#request_access' do it 'performs requests to the configured endpoint' do - expect(Excon).to receive(:post).with(dummy_url, any_args) + expect(Gitlab::HTTP).to receive(:post).with(dummy_url, any_args) client.request_access end @@ -25,7 +25,7 @@ RSpec.describe Gitlab::ExternalAuthorization::Client do project_classification_label: 'dummy_label', identities: [] }.to_json - expect(Excon).to receive(:post) + expect(Gitlab::HTTP).to receive(:post) .with(dummy_url, hash_including(body: expected_body)) client.request_access @@ -36,7 +36,7 @@ RSpec.describe Gitlab::ExternalAuthorization::Client do external_authorization_service_timeout: 3 ) - expect(Excon).to receive(:post).with(dummy_url, + expect(Gitlab::HTTP).to receive(:post).with(dummy_url, hash_including( connect_timeout: 3, read_timeout: 3, @@ -58,25 +58,33 @@ RSpec.describe Gitlab::ExternalAuthorization::Client do client_key_pass: 'open sesame' } - expect(Excon).to receive(:post).with(dummy_url, hash_including(expected_params)) + expect(Gitlab::HTTP).to receive(:post).with(dummy_url, hash_including(expected_params)) client.request_access end it 'returns an expected response' do - expect(Excon).to receive(:post) + expect(Gitlab::HTTP).to receive(:post) expect(client.request_access) .to be_kind_of(::Gitlab::ExternalAuthorization::Response) end it 'wraps exceptions if the request fails' do - expect(Excon).to receive(:post) { raise Excon::Error.new('the request broke') } + expect(Gitlab::HTTP).to receive(:post) { raise Gitlab::HTTP::BlockedUrlError.new('the request broke') } expect { client.request_access } .to raise_error(::Gitlab::ExternalAuthorization::RequestFailed) end + it 'passes local request setting to Gitlab::HTTP' do + stub_application_setting(allow_local_requests_from_system_hooks: false) + + expect(Gitlab::HTTP).to receive(:post).with(dummy_url, hash_including(allow_local_requests: false)) + + client.request_access + end + describe 'for ldap users' do let(:user) do create(:omniauth_user, @@ -92,7 +100,7 @@ RSpec.describe Gitlab::ExternalAuthorization::Client do identities: [{ provider: 'ldapprovider', extern_uid: 'external id' }], user_ldap_dn: 'external id' }.to_json - expect(Excon).to receive(:post) + expect(Gitlab::HTTP).to receive(:post) .with(dummy_url, hash_including(body: expected_body)) client.request_access @@ -115,7 +123,7 @@ RSpec.describe Gitlab::ExternalAuthorization::Client do { provider: 'facebook', extern_uid: 'facebook_external_id' } ] }.to_json - expect(Excon).to receive(:post) + expect(Gitlab::HTTP).to receive(:post) .with(dummy_url, hash_including(body: expected_body)) client.request_access diff --git a/spec/lib/gitlab/external_authorization/response_spec.rb b/spec/lib/gitlab/external_authorization/response_spec.rb index 11f83feb76f..716196e0aa2 100644 --- a/spec/lib/gitlab/external_authorization/response_spec.rb +++ b/spec/lib/gitlab/external_authorization/response_spec.rb @@ -3,21 +3,21 @@ require 'spec_helper' RSpec.describe Gitlab::ExternalAuthorization::Response do - let(:excon_response) { double } + let(:http_response) { double } - subject(:response) { described_class.new(excon_response) } + subject(:response) { described_class.new(http_response) } describe '#valid?' do it 'is valid for 200, 401, and 403 responses' do - [200, 401, 403].each do |status| - allow(excon_response).to receive(:status).and_return(status) + [200, 401, 403].each do |code| + allow(http_response).to receive(:code).and_return(code) expect(response).to be_valid end end it "is invalid for other statuses" do - expect(excon_response).to receive(:status).and_return(500) + expect(http_response).to receive(:code).and_return(500) expect(response).not_to be_valid end @@ -25,13 +25,13 @@ RSpec.describe Gitlab::ExternalAuthorization::Response do describe '#reason' do it 'returns a reason if it was included in the response body' do - expect(excon_response).to receive(:body).and_return({ reason: 'Not authorized' }.to_json) + expect(http_response).to receive(:body).and_return({ reason: 'Not authorized' }.to_json) expect(response.reason).to eq('Not authorized') end it 'returns nil when there was no body' do - expect(excon_response).to receive(:body).and_return('') + expect(http_response).to receive(:body).and_return('') expect(response.reason).to eq(nil) end @@ -39,14 +39,14 @@ RSpec.describe Gitlab::ExternalAuthorization::Response do describe '#successful?' do it 'is `true` if the status is 200' do - allow(excon_response).to receive(:status).and_return(200) + allow(http_response).to receive(:code).and_return(200) expect(response).to be_successful end it 'is `false` if the status is 401 or 403' do - [401, 403].each do |status| - allow(excon_response).to receive(:status).and_return(status) + [401, 403].each do |code| + allow(http_response).to receive(:code).and_return(code) expect(response).not_to be_successful end diff --git a/spec/lib/gitlab/file_finder_spec.rb b/spec/lib/gitlab/file_finder_spec.rb index 36fb4c48fb2..8d6df62b3f6 100644 --- a/spec/lib/gitlab/file_finder_spec.rb +++ b/spec/lib/gitlab/file_finder_spec.rb @@ -13,22 +13,44 @@ RSpec.describe Gitlab::FileFinder do let(:expected_file_by_content) { 'CHANGELOG' } end - it 'filters by filename' do - results = subject.find('files filename:wm.svg') + context 'with inclusive filters' do + it 'filters by filename' do + results = subject.find('files filename:wm.svg') - expect(results.count).to eq(1) - end + expect(results.count).to eq(1) + end + + it 'filters by path' do + results = subject.find('white path:images') - it 'filters by path' do - results = subject.find('white path:images') + expect(results.count).to eq(1) + end - expect(results.count).to eq(1) + it 'filters by extension' do + results = subject.find('files extension:md') + + expect(results.count).to eq(4) + end end - it 'filters by extension' do - results = subject.find('files extension:svg') + context 'with exclusive filters' do + it 'filters by filename' do + results = subject.find('files -filename:wm.svg') + + expect(results.count).to eq(26) + end + + it 'filters by path' do + results = subject.find('white -path:images') + + expect(results.count).to eq(4) + end + + it 'filters by extension' do + results = subject.find('files -extension:md') - expect(results.count).to eq(1) + expect(results.count).to eq(23) + end end it 'does not cause N+1 query' do diff --git a/spec/lib/gitlab/fogbugz_import/importer_spec.rb b/spec/lib/gitlab/fogbugz_import/importer_spec.rb index d2be3e3f6b1..eb0c4da6ce3 100644 --- a/spec/lib/gitlab/fogbugz_import/importer_spec.rb +++ b/spec/lib/gitlab/fogbugz_import/importer_spec.rb @@ -11,6 +11,7 @@ RSpec.describe Gitlab::FogbugzImport::Importer do path: 'vim', raw_data: '') end + let(:import_data) { { 'repo' => repo } } let(:credentials) do { diff --git a/spec/lib/gitlab/gfm/reference_rewriter_spec.rb b/spec/lib/gitlab/gfm/reference_rewriter_spec.rb index 1c9004262c5..f4875aa0ebc 100644 --- a/spec/lib/gitlab/gfm/reference_rewriter_spec.rb +++ b/spec/lib/gitlab/gfm/reference_rewriter_spec.rb @@ -110,6 +110,20 @@ RSpec.describe Gitlab::Gfm::ReferenceRewriter do end end + context 'when description contains a local reference' do + let(:local_issue) { create(:issue, project: old_project) } + let(:text) { "See ##{local_issue.iid}" } + + it { is_expected.to eq("See #{old_project.path}##{local_issue.iid}") } + end + + context 'when description contains a cross reference' do + let(:merge_request) { create(:merge_request) } + let(:text) { "See #{merge_request.project.full_path}!#{merge_request.iid}" } + + it { is_expected.to eq(text) } + end + context 'with a commit' do let(:old_project) { create(:project, :repository, name: 'old-project', group: group) } let(:commit) { old_project.commit } diff --git a/spec/lib/gitlab/git/branch_spec.rb b/spec/lib/gitlab/git/branch_spec.rb index bac1b4c57f9..e1bcf4aeeb1 100644 --- a/spec/lib/gitlab/git/branch_spec.rb +++ b/spec/lib/gitlab/git/branch_spec.rb @@ -84,6 +84,7 @@ RSpec.describe Gitlab::Git::Branch, :seed_helper do parents: parents } end + let(:stale_sha) { Timecop.freeze(Gitlab::Git::Branch::STALE_BRANCH_THRESHOLD.ago - 5.days) { create_commit } } let(:active_sha) { Timecop.freeze(Gitlab::Git::Branch::STALE_BRANCH_THRESHOLD.ago + 5.days) { create_commit } } let(:future_sha) { Timecop.freeze(100.days.since) { create_commit } } diff --git a/spec/lib/gitlab/git/commit_spec.rb b/spec/lib/gitlab/git/commit_spec.rb index 666b49f27f7..491437856d4 100644 --- a/spec/lib/gitlab/git/commit_spec.rb +++ b/spec/lib/gitlab/git/commit_spec.rb @@ -9,6 +9,7 @@ RSpec.describe Gitlab::Git::Commit, :seed_helper do let(:rugged_repo) do Rugged::Repository.new(File.join(TestEnv.repos_path, TEST_REPO_PATH)) end + let(:commit) { described_class.find(repository, SeedRepo::Commit::ID) } let(:rugged_commit) { rugged_repo.lookup(SeedRepo::Commit::ID) } diff --git a/spec/lib/gitlab/git/conflict/parser_spec.rb b/spec/lib/gitlab/git/conflict/parser_spec.rb index 67bd48256ce..02b00f711b4 100644 --- a/spec/lib/gitlab/git/conflict/parser_spec.rb +++ b/spec/lib/gitlab/git/conflict/parser_spec.rb @@ -89,12 +89,15 @@ RSpec.describe Gitlab::Git::Conflict::Parser do let(:lines) do described_class.parse(text, our_path: 'files/ruby/regex.rb', their_path: 'files/ruby/regex.rb') end + let(:old_line_numbers) do lines.select { |line| line[:type] != 'new' }.map { |line| line[:line_old] } end + let(:new_line_numbers) do lines.select { |line| line[:type] != 'old' }.map { |line| line[:line_new] } end + let(:line_indexes) { lines.map { |line| line[:line_obj_index] } } it 'sets our lines as new lines' do diff --git a/spec/lib/gitlab/git/diff_collection_spec.rb b/spec/lib/gitlab/git/diff_collection_spec.rb index 6da07ce84a1..b202015464f 100644 --- a/spec/lib/gitlab/git/diff_collection_spec.rb +++ b/spec/lib/gitlab/git/diff_collection_spec.rb @@ -18,7 +18,7 @@ RSpec.describe Gitlab::Git::DiffCollection, :seed_helper do return enum_for(:each) unless block_given? loop do - break if @count.zero? + break if @count == 0 # It is critical to decrement before yielding. We may never reach the lines after 'yield'. @count -= 1 diff --git a/spec/lib/gitlab/git/diff_spec.rb b/spec/lib/gitlab/git/diff_spec.rb index 91688c31f5e..117c519e98d 100644 --- a/spec/lib/gitlab/git/diff_spec.rb +++ b/spec/lib/gitlab/git/diff_spec.rb @@ -78,6 +78,7 @@ EOT patch: raw_patch ) end + let(:diff) { described_class.new(gitaly_diff) } context 'with a small diff' do @@ -134,6 +135,7 @@ EOT to_id: '8e5177d718c561d36efde08bad36b43687ee6bf0' ) end + let(:diff) { described_class.new(commit_delta) } it 'initializes the diff' do diff --git a/spec/lib/gitlab/git/patches/collection_spec.rb b/spec/lib/gitlab/git/patches/collection_spec.rb index eb92f4663c8..67a502242ea 100644 --- a/spec/lib/gitlab/git/patches/collection_spec.rb +++ b/spec/lib/gitlab/git/patches/collection_spec.rb @@ -6,6 +6,7 @@ RSpec.describe Gitlab::Git::Patches::Collection do let(:patch_content1) do File.read(File.join(patches_folder, "0001-This-does-not-apply-to-the-feature-branch.patch")) end + let(:patch_content2) do File.read(File.join(patches_folder, "0001-A-commit-from-a-patch.patch")) end diff --git a/spec/lib/gitlab/git/patches/commit_patches_spec.rb b/spec/lib/gitlab/git/patches/commit_patches_spec.rb index cd1e03a6de0..9ab0893eb55 100644 --- a/spec/lib/gitlab/git/patches/commit_patches_spec.rb +++ b/spec/lib/gitlab/git/patches/commit_patches_spec.rb @@ -10,6 +10,7 @@ RSpec.describe Gitlab::Git::Patches::CommitPatches do Gitlab::Git::Patches::Collection.new([content_1, content_2]) end + let(:user) { build(:user) } let(:branch_name) { 'branch-with-patches' } let(:repository) { create(:project, :repository).repository } diff --git a/spec/lib/gitlab/git/patches/patch_spec.rb b/spec/lib/gitlab/git/patches/patch_spec.rb index 629f43d3636..6588b18d0ae 100644 --- a/spec/lib/gitlab/git/patches/patch_spec.rb +++ b/spec/lib/gitlab/git/patches/patch_spec.rb @@ -6,6 +6,7 @@ RSpec.describe Gitlab::Git::Patches::Patch do let(:patch_content) do File.read(File.join(patches_folder, "0001-This-does-not-apply-to-the-feature-branch.patch")) end + let(:patch) { described_class.new(patch_content) } describe '#size' do diff --git a/spec/lib/gitlab/git/pre_receive_error_spec.rb b/spec/lib/gitlab/git/pre_receive_error_spec.rb index bf4530c8945..2ad27361c80 100644 --- a/spec/lib/gitlab/git/pre_receive_error_spec.rb +++ b/spec/lib/gitlab/git/pre_receive_error_spec.rb @@ -6,15 +6,27 @@ RSpec.describe Gitlab::Git::PreReceiveError do Gitlab::Git::PreReceiveError::SAFE_MESSAGE_PREFIXES.each do |prefix| context "error messages prefixed with #{prefix}" do it 'accepts only errors lines with the prefix' do - ex = described_class.new("#{prefix} Hello,\nworld!") + raw_message = "#{prefix} Hello,\nworld!" + ex = described_class.new(raw_message) expect(ex.message).to eq('Hello,') + expect(ex.raw_message).to eq(raw_message) end it 'makes its message HTML-friendly' do - ex = described_class.new("#{prefix} Hello,\n#{prefix} world!\n") + raw_message = "#{prefix} Hello,\n#{prefix} world!\n" + ex = described_class.new(raw_message) expect(ex.message).to eq('Hello,<br>world!') + expect(ex.raw_message).to eq(raw_message) + end + + it 'sanitizes the user message' do + raw_message = 'Raw message' + ex = described_class.new(raw_message, "#{prefix} User message") + + expect(ex.raw_message).to eq(raw_message) + expect(ex.message).to eq('User message') end end end diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index e7f4573c95f..73eecd3401a 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -579,9 +579,11 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do let(:commit_with_old_name) do Gitlab::Git::Commit.find(repository, @commit_with_old_name_id) end + let(:commit_with_new_name) do Gitlab::Git::Commit.find(repository, @commit_with_new_name_id) end + let(:rename_commit) do Gitlab::Git::Commit.find(repository, @rename_commit_id) end @@ -2178,6 +2180,7 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do let(:new_repository) do Gitlab::Git::Repository.new('test_second_storage', TEST_REPO_PATH, '', 'group/project') end + let(:new_repository_path) { File.join(TestEnv::SECOND_STORAGE_PATH, new_repository.relative_path) } subject { new_repository.replicate(repository) } diff --git a/spec/lib/gitlab/git_access_project_spec.rb b/spec/lib/gitlab/git_access_project_spec.rb index 520300363c9..f80915b2be9 100644 --- a/spec/lib/gitlab/git_access_project_spec.rb +++ b/spec/lib/gitlab/git_access_project_spec.rb @@ -5,6 +5,7 @@ require 'spec_helper' RSpec.describe Gitlab::GitAccessProject do let_it_be(:user) { create(:user) } let_it_be(:project) { create(:project, :repository) } + let(:container) { project } let(:actor) { user } let(:project_path) { project.path } let(:namespace_path) { project&.namespace&.path } @@ -13,19 +14,32 @@ RSpec.describe Gitlab::GitAccessProject do let(:changes) { Gitlab::GitAccess::ANY } let(:push_access_check) { access.check('git-receive-pack', changes) } let(:pull_access_check) { access.check('git-upload-pack', changes) } + let(:access) do + described_class.new(actor, container, protocol, + authentication_abilities: authentication_abilities, + repository_path: project_path, namespace_path: namespace_path) + end + + describe '#check_namespace!' do + context 'when namespace is nil' do + let(:namespace_path) { nil } + + it 'does not allow push and pull access' do + aggregate_failures do + expect { push_access_check }.to raise_namespace_not_found + expect { pull_access_check }.to raise_namespace_not_found + end + end + end + end describe '#check_project_accessibility!' do context 'when the project is nil' do - let(:project) { nil } + let(:container) { nil } let(:project_path) { "new-project" } context 'when user is allowed to create project in namespace' do let(:namespace_path) { user.namespace.path } - let(:access) do - described_class.new(actor, nil, - protocol, authentication_abilities: authentication_abilities, - repository_path: project_path, namespace_path: namespace_path) - end it 'blocks pull access with "not found"' do expect { pull_access_check }.to raise_not_found @@ -39,11 +53,6 @@ RSpec.describe Gitlab::GitAccessProject do context 'when user is not allowed to create project in namespace' do let(:user2) { create(:user) } let(:namespace_path) { user2.namespace.path } - let(:access) do - described_class.new(actor, nil, - protocol, authentication_abilities: authentication_abilities, - repository_path: project_path, namespace_path: namespace_path) - end it 'blocks push and pull with "not found"' do aggregate_failures do @@ -56,22 +65,27 @@ RSpec.describe Gitlab::GitAccessProject do end describe '#ensure_project_on_push!' do - let(:access) do - described_class.new(actor, project, - protocol, authentication_abilities: authentication_abilities, - repository_path: project_path, namespace_path: namespace_path) - end - before do allow(access).to receive(:changes).and_return(changes) end + shared_examples 'no project is created' do + let(:raise_specific_error) { raise_not_found } + let(:action) { push_access_check } + + it 'does not create a new project' do + expect { action } + .to raise_specific_error + .and change { Project.count }.by(0) + end + end + context 'when push' do let(:cmd) { 'git-receive-pack' } context 'when project does not exist' do let(:project_path) { "nonexistent" } - let(:project) { nil } + let(:container) { nil } context 'when changes is _any' do let(:changes) { Gitlab::GitAccess::ANY } @@ -82,8 +96,8 @@ RSpec.describe Gitlab::GitAccessProject do context 'when user can create project in namespace' do let(:namespace_path) { user.namespace.path } - it 'creates a new project' do - expect { access.send(:ensure_project_on_push!, cmd) } + it 'creates a new project in the correct namespace' do + expect { push_access_check } .to change { Project.count }.by(1) .and change { Project.where(namespace: user.namespace, name: project_path).count }.by(1) end @@ -93,9 +107,7 @@ RSpec.describe Gitlab::GitAccessProject do let(:user2) { create(:user) } let(:namespace_path) { user2.namespace.path } - it 'does not create a new project' do - expect { access.send(:ensure_project_on_push!, cmd) }.not_to change { Project.count } - end + it_behaves_like 'no project is created' end end @@ -105,8 +117,8 @@ RSpec.describe Gitlab::GitAccessProject do context 'when user can create project in namespace' do let(:namespace_path) { user.namespace.path } - it 'does not create a new project' do - expect { access.send(:ensure_project_on_push!, cmd) }.not_to change { Project.count } + it_behaves_like 'no project is created' do + let(:raise_specific_error) { raise_forbidden } end end end @@ -115,32 +127,26 @@ RSpec.describe Gitlab::GitAccessProject do context 'when check contains actual changes' do let(:changes) { "#{Gitlab::Git::BLANK_SHA} 570e7b2abdd848b95f2f578043fc23bd6f6fd24d refs/heads/new_branch" } - it 'does not create a new project' do - expect { access.send(:ensure_project_on_push!, cmd) }.not_to change { Project.count } - end + it_behaves_like 'no project is created' end end context 'when project exists' do let(:changes) { Gitlab::GitAccess::ANY } - let!(:project) { create(:project) } + let!(:container) { project } - it 'does not create a new project' do - expect { access.send(:ensure_project_on_push!, cmd) }.not_to change { Project.count } - end + it_behaves_like 'no project is created' end context 'when deploy key is used' do let(:key) { create(:deploy_key, user: user) } let(:actor) { key } let(:project_path) { "nonexistent" } - let(:project) { nil } + let(:container) { nil } let(:namespace_path) { user.namespace.path } let(:changes) { Gitlab::GitAccess::ANY } - it 'does not create a new project' do - expect { access.send(:ensure_project_on_push!, cmd) }.not_to change { Project.count } - end + it_behaves_like 'no project is created' end end @@ -151,10 +157,10 @@ RSpec.describe Gitlab::GitAccessProject do context 'when project does not exist' do let(:project_path) { "new-project" } let(:namespace_path) { user.namespace.path } - let(:project) { nil } + let(:container) { nil } - it 'does not create a new project' do - expect { access.send(:ensure_project_on_push!, cmd) }.not_to change { Project.count } + it_behaves_like 'no project is created' do + let(:action) { pull_access_check } end end end @@ -163,4 +169,12 @@ RSpec.describe Gitlab::GitAccessProject do def raise_not_found raise_error(Gitlab::GitAccess::NotFoundError, Gitlab::GitAccess::ERROR_MESSAGES[:project_not_found]) end + + def raise_forbidden + raise_error(Gitlab::GitAccess::ForbiddenError) + end + + def raise_namespace_not_found + raise_error(Gitlab::GitAccess::NotFoundError, described_class::ERROR_MESSAGES[:namespace_not_found]) + end end diff --git a/spec/lib/gitlab/git_access_spec.rb b/spec/lib/gitlab/git_access_spec.rb index 01691f87092..8153886a2ab 100644 --- a/spec/lib/gitlab/git_access_spec.rb +++ b/spec/lib/gitlab/git_access_spec.rb @@ -20,6 +20,18 @@ RSpec.describe Gitlab::GitAccess do let(:push_access_check) { access.check('git-receive-pack', changes) } let(:pull_access_check) { access.check('git-upload-pack', changes) } + let(:access_class) do + Class.new(described_class) do + def push_ability + :push_code + end + + def download_ability + :download_code + end + end + end + describe '#check with single protocols allowed' do def disable_protocol(protocol) allow(Gitlab::ProtocolAccess).to receive(:allowed?).with(protocol).and_return(false) @@ -58,7 +70,7 @@ RSpec.describe Gitlab::GitAccess do it "doesn't block http pull" do aggregate_failures do - expect { pull_access_check }.not_to raise_forbidden('Git access over HTTP is not allowed') + expect { pull_access_check }.not_to raise_error end end @@ -67,7 +79,7 @@ RSpec.describe Gitlab::GitAccess do it "doesn't block http pull" do aggregate_failures do - expect { pull_access_check }.not_to raise_forbidden('Git access over HTTP is not allowed') + expect { pull_access_check }.not_to raise_error end end end @@ -75,33 +87,6 @@ RSpec.describe Gitlab::GitAccess do end end - describe '#check_namespace!' do - context 'when namespace exists' do - before do - project.add_maintainer(user) - end - - it 'allows push and pull access' do - aggregate_failures do - expect { push_access_check }.not_to raise_error - expect { pull_access_check }.not_to raise_error - end - end - end - - context 'when namespace and project are nil' do - let(:project) { nil } - let(:namespace_path) { nil } - - it 'does not allow push and pull access' do - aggregate_failures do - expect { push_access_check }.to raise_namespace_not_found - expect { pull_access_check }.to raise_namespace_not_found - end - end - end - end - describe '#check_project_accessibility!' do context 'when the project exists' do context 'when actor exists' do @@ -464,7 +449,7 @@ RSpec.describe Gitlab::GitAccess do let(:public_project) { create(:project, :public, :repository) } let(:project_path) { public_project.path } let(:namespace_path) { public_project.namespace.path } - let(:access) { described_class.new(nil, public_project, 'web', authentication_abilities: [:download_code], repository_path: project_path, namespace_path: namespace_path) } + let(:access) { access_class.new(nil, public_project, 'web', authentication_abilities: [:download_code], repository_path: project_path, namespace_path: namespace_path) } context 'when repository is enabled' do it 'give access to download code' do @@ -859,7 +844,7 @@ RSpec.describe Gitlab::GitAccess do message = "Push operation timed out\n\nTiming information for debugging purposes:\nRunning checks for ref: wow" expect_next_instance_of(Gitlab::Checks::ChangeAccess) do |check| - expect(check).to receive(:exec).and_raise(Gitlab::Checks::TimedLogger::TimeoutError) + expect(check).to receive(:validate!).and_raise(Gitlab::Checks::TimedLogger::TimeoutError) end expect { access.check('git-receive-pack', changes) }.to raise_error(described_class::TimeoutError, message) @@ -1067,7 +1052,7 @@ RSpec.describe Gitlab::GitAccess do private def access - described_class.new(actor, project, protocol, + access_class.new(actor, project, protocol, authentication_abilities: authentication_abilities, namespace_path: namespace_path, repository_path: project_path, redirected_path: redirected_path, auth_result_type: auth_result_type) @@ -1078,15 +1063,11 @@ RSpec.describe Gitlab::GitAccess do end def raise_forbidden(message) - raise_error(Gitlab::GitAccess::ForbiddenError, message) + raise_error(described_class::ForbiddenError, message) end def raise_not_found - raise_error(Gitlab::GitAccess::NotFoundError, Gitlab::GitAccess::ERROR_MESSAGES[:project_not_found]) - end - - def raise_namespace_not_found - raise_error(Gitlab::GitAccess::NotFoundError, Gitlab::GitAccess::ERROR_MESSAGES[:namespace_not_found]) + raise_error(described_class::NotFoundError, described_class::ERROR_MESSAGES[:project_not_found]) end def build_authentication_abilities diff --git a/spec/lib/gitlab/git_access_wiki_spec.rb b/spec/lib/gitlab/git_access_wiki_spec.rb index 738269e4a14..688089f4862 100644 --- a/spec/lib/gitlab/git_access_wiki_spec.rb +++ b/spec/lib/gitlab/git_access_wiki_spec.rb @@ -4,8 +4,8 @@ require 'spec_helper' RSpec.describe Gitlab::GitAccessWiki do let(:access) { described_class.new(user, project, 'web', authentication_abilities: authentication_abilities, redirected_path: redirected_path) } - let(:project) { create(:project, :wiki_repo) } - let(:user) { create(:user) } + let_it_be(:project) { create(:project, :wiki_repo) } + let_it_be(:user) { create(:user) } let(:changes) { ['6f6d7e7ed 570e7b2ab refs/heads/master'] } let(:redirected_path) { nil } let(:authentication_abilities) do @@ -17,56 +17,65 @@ RSpec.describe Gitlab::GitAccessWiki do end describe '#push_access_check' do + subject { access.check('git-receive-pack', changes) } + context 'when user can :create_wiki' do before do - create(:protected_branch, name: 'master', project: project) project.add_developer(user) end - subject { access.check('git-receive-pack', changes) } - it { expect { subject }.not_to raise_error } context 'when in a read-only GitLab instance' do + let(:message) { "You can't push code to a read-only GitLab instance." } + before do allow(Gitlab::Database).to receive(:read_only?) { true } end - it 'does not give access to upload wiki code' do - expect { subject }.to raise_error(Gitlab::GitAccess::ForbiddenError, "You can't push code to a read-only GitLab instance.") - end + it_behaves_like 'forbidden git access' + end + end + + context 'the user cannot :create_wiki' do + it_behaves_like 'not-found git access' do + let(:message) { 'The wiki you were looking for could not be found.' } end end end - describe '#access_check_download!' do + describe '#check_download_access!' do subject { access.check('git-upload-pack', Gitlab::GitAccess::ANY) } - before do - project.add_developer(user) - end - - context 'when wiki feature is enabled' do - it 'give access to download wiki code' do - expect { subject }.not_to raise_error + context 'the user can :download_wiki_code' do + before do + project.add_developer(user) end - context 'when the wiki repository does not exist' do - let(:project) { create(:project) } + context 'when wiki feature is disabled' do + before do + project.project_feature.update_attribute(:wiki_access_level, ProjectFeature::DISABLED) + end - it 'returns not found' do - expect(project.wiki_repository_exists?).to eq(false) + it_behaves_like 'forbidden git access' do + let(:message) { include('wiki') } + end + end - expect { subject }.to raise_error(Gitlab::GitAccess::NotFoundError, 'A repository for this project does not exist yet.') + context 'when the repository does not exist' do + before do + allow(project.wiki).to receive(:repository).and_return(double('Repository', exists?: false)) + end + + it_behaves_like 'not-found git access' do + let(:message) { include('for this wiki') } end end end - context 'when wiki feature is disabled' do - it 'does not give access to download wiki code' do - project.project_feature.update_attribute(:wiki_access_level, ProjectFeature::DISABLED) - - expect { subject }.to raise_error(Gitlab::GitAccess::ForbiddenError, 'You are not allowed to download code from this project.') + context 'the user cannot :download_wiki_code' do + it_behaves_like 'not-found git access' do + let(:message) { include('wiki') } end end end diff --git a/spec/lib/gitlab/gitaly_client/commit_service_spec.rb b/spec/lib/gitlab/gitaly_client/commit_service_spec.rb index 45a25ccfc88..9581b017839 100644 --- a/spec/lib/gitlab/gitaly_client/commit_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/commit_service_spec.rb @@ -266,6 +266,7 @@ RSpec.describe Gitlab::GitalyClient::CommitService do repository: repository_message, revision: revision ) end + let(:response) do Gitaly::CommitStatsResponse.new( oid: revision, diff --git a/spec/lib/gitlab/gitaly_client/conflicts_service_spec.rb b/spec/lib/gitlab/gitaly_client/conflicts_service_spec.rb index b016e8bdf5a..e90cb966917 100644 --- a/spec/lib/gitlab/gitaly_client/conflicts_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/conflicts_service_spec.rb @@ -35,6 +35,7 @@ RSpec.describe Gitlab::GitalyClient::ConflictsService do let(:files) do [{ old_path: 'some/path', new_path: 'some/path', content: '' }] end + let(:source_branch) { 'master' } let(:target_branch) { 'feature' } let(:commit_message) { 'Solving conflicts\n\nTést' } diff --git a/spec/lib/gitlab/gitaly_client/operation_service_spec.rb b/spec/lib/gitlab/gitaly_client/operation_service_spec.rb index 4e16f760235..b974f456914 100644 --- a/spec/lib/gitlab/gitaly_client/operation_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/operation_service_spec.rb @@ -20,11 +20,13 @@ RSpec.describe Gitlab::GitalyClient::OperationService do user: gitaly_user ) end + let(:gitaly_commit) { build(:gitaly_commit) } let(:commit_id) { gitaly_commit.id } let(:gitaly_branch) do Gitaly::Branch.new(name: branch_name, target_commit: gitaly_commit) end + let(:response) { Gitaly::UserCreateBranchResponse.new(branch: gitaly_branch) } let(:commit) { Gitlab::Git::Commit.new(repository, gitaly_commit) } @@ -68,6 +70,7 @@ RSpec.describe Gitlab::GitalyClient::OperationService do user: gitaly_user ) end + let(:response) { Gitaly::UserUpdateBranchResponse.new } subject { client.user_update_branch(branch_name, user, newrev, oldrev) } @@ -123,6 +126,7 @@ RSpec.describe Gitlab::GitalyClient::OperationService do user: gitaly_user ) end + let(:response) { Gitaly::UserDeleteBranchResponse.new } subject { client.user_delete_branch(branch_name, user) } @@ -162,6 +166,7 @@ RSpec.describe Gitlab::GitalyClient::OperationService do user: gitaly_user ) end + let(:branch_update) do Gitaly::OperationBranchUpdate.new( commit_id: source_sha, @@ -169,6 +174,7 @@ RSpec.describe Gitlab::GitalyClient::OperationService do branch_created: false ) end + let(:response) { Gitaly::UserFFBranchResponse.new(branch_update: branch_update) } before do @@ -303,6 +309,7 @@ RSpec.describe Gitlab::GitalyClient::OperationService do commit_message: commit_message ) end + let(:squash_sha) { 'f00' } let(:response) { Gitaly::UserSquashResponse.new(squash_sha: squash_sha) } @@ -375,6 +382,7 @@ RSpec.describe Gitlab::GitalyClient::OperationService do let(:patch_content) do patch_names.map { |name| File.read(File.join(patches_folder, name)) }.join("\n") end + let(:patch_names) { %w(0001-This-does-not-apply-to-the-feature-branch.patch) } let(:branch_name) { 'branch-with-patches' } diff --git a/spec/lib/gitlab/gitaly_client/wiki_service_spec.rb b/spec/lib/gitlab/gitaly_client/wiki_service_spec.rb index a06f8459963..8a169acb69c 100644 --- a/spec/lib/gitlab/gitaly_client/wiki_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/wiki_service_spec.rb @@ -18,6 +18,7 @@ RSpec.describe Gitlab::GitalyClient::WikiService do Gitaly::WikiFindPageResponse.new(page: Gitaly::WikiPage.new(raw_data: 'b')) ] end + let(:wiki_page) { subject.first } let(:wiki_page_version) { subject.last } @@ -60,6 +61,7 @@ RSpec.describe Gitlab::GitalyClient::WikiService do Gitaly::WikiGetAllPagesResponse.new(end_of_page: true) ] end + let(:wiki_page_1) { subject[0].first } let(:wiki_page_1_version) { subject[0].last } let(:wiki_page_2) { subject[1].first } diff --git a/spec/lib/gitlab/github_import/client_spec.rb b/spec/lib/gitlab/github_import/client_spec.rb index 7cff6ed1388..5f6ab42d0d2 100644 --- a/spec/lib/gitlab/github_import/client_spec.rb +++ b/spec/lib/gitlab/github_import/client_spec.rb @@ -169,7 +169,7 @@ RSpec.describe Gitlab::GithubImport::Client do expect(client).to receive(:raise_or_wait_for_rate_limit) client.with_rate_limit do - if retries.zero? + if retries == 0 retries += 1 raise(Octokit::TooManyRequests) end diff --git a/spec/lib/gitlab/gitlab_import/project_creator_spec.rb b/spec/lib/gitlab/gitlab_import/project_creator_spec.rb index 44bcfb93c51..53bf1db3438 100644 --- a/spec/lib/gitlab/gitlab_import/project_creator_spec.rb +++ b/spec/lib/gitlab/gitlab_import/project_creator_spec.rb @@ -14,6 +14,7 @@ RSpec.describe Gitlab::GitlabImport::ProjectCreator do owner: { name: "john" } }.with_indifferent_access end + let(:namespace) { create(:group) } let(:token) { "asdffg" } let(:access_params) { { gitlab_access_token: token } } diff --git a/spec/lib/gitlab/google_code_import/importer_spec.rb b/spec/lib/gitlab/google_code_import/importer_spec.rb index f681e3c9f31..a22e80ae1c0 100644 --- a/spec/lib/gitlab/google_code_import/importer_spec.rb +++ b/spec/lib/gitlab/google_code_import/importer_spec.rb @@ -12,6 +12,7 @@ RSpec.describe Gitlab::GoogleCodeImport::Importer do 'user_map' => { 'thilo...' => "@#{mapped_user.username}" } } end + let(:project) { create(:project) } subject { described_class.new(project) } diff --git a/spec/lib/gitlab/google_code_import/project_creator_spec.rb b/spec/lib/gitlab/google_code_import/project_creator_spec.rb index 4be2e16c116..cfebe57aed3 100644 --- a/spec/lib/gitlab/google_code_import/project_creator_spec.rb +++ b/spec/lib/gitlab/google_code_import/project_creator_spec.rb @@ -11,6 +11,7 @@ RSpec.describe Gitlab::GoogleCodeImport::ProjectCreator do "repositoryUrls" => ["https://vim.googlecode.com/git/"] ) end + let(:namespace) { create(:group) } before do diff --git a/spec/lib/gitlab/grape_logging/formatters/lograge_with_timestamp_spec.rb b/spec/lib/gitlab/grape_logging/formatters/lograge_with_timestamp_spec.rb index e5d0adcfd5f..91299de0751 100644 --- a/spec/lib/gitlab/grape_logging/formatters/lograge_with_timestamp_spec.rb +++ b/spec/lib/gitlab/grape_logging/formatters/lograge_with_timestamp_spec.rb @@ -29,6 +29,7 @@ RSpec.describe Gitlab::GrapeLogging::Formatters::LogrageWithTimestamp do correlation_id: 'WMefXn60429' } end + let(:time) { Time.now } let(:result) { Gitlab::Json.parse(subject) } diff --git a/spec/lib/gitlab/grape_logging/loggers/exception_logger_spec.rb b/spec/lib/gitlab/grape_logging/loggers/exception_logger_spec.rb index bcb1f6c5af7..3ce09740ec8 100644 --- a/spec/lib/gitlab/grape_logging/loggers/exception_logger_spec.rb +++ b/spec/lib/gitlab/grape_logging/loggers/exception_logger_spec.rb @@ -98,7 +98,7 @@ RSpec.describe Gitlab::GrapeLogging::Loggers::ExceptionLogger do before do current_backtrace = caller allow(exception).to receive(:backtrace).and_return(current_backtrace) - expected['exception.backtrace'] = Gitlab::BacktraceCleaner.clean_backtrace(current_backtrace) + expected['exception.backtrace'] = Rails.backtrace_cleaner.clean(current_backtrace) end it 'includes the backtrace' do diff --git a/spec/lib/gitlab/graphql/authorize/authorize_resource_spec.rb b/spec/lib/gitlab/graphql/authorize/authorize_resource_spec.rb index 83873081a98..c5d7665c3b2 100644 --- a/spec/lib/gitlab/graphql/authorize/authorize_resource_spec.rb +++ b/spec/lib/gitlab/graphql/authorize/authorize_resource_spec.rb @@ -116,6 +116,7 @@ RSpec.describe Gitlab::Graphql::Authorize::AuthorizeResource do end end end + let(:error) { /#{fake_class.name} has no authorizations/ } describe '#authorized_find!' do diff --git a/spec/lib/gitlab/graphql/find_argument_in_parent_spec.rb b/spec/lib/gitlab/graphql/find_argument_in_parent_spec.rb index 68b24a60a99..1b9301cd1aa 100644 --- a/spec/lib/gitlab/graphql/find_argument_in_parent_spec.rb +++ b/spec/lib/gitlab/graphql/find_argument_in_parent_spec.rb @@ -21,6 +21,7 @@ RSpec.describe Gitlab::Graphql::FindArgumentInParent do ) ) end + let(:arg_name) { :my_arg } it 'searches parents and returns the argument' do diff --git a/spec/lib/gitlab/graphql/pagination/keyset/connection_spec.rb b/spec/lib/gitlab/graphql/pagination/keyset/connection_spec.rb index 65698caac34..09d7e084172 100644 --- a/spec/lib/gitlab/graphql/pagination/keyset/connection_spec.rb +++ b/spec/lib/gitlab/graphql/pagination/keyset/connection_spec.rb @@ -185,6 +185,7 @@ RSpec.describe Gitlab::Graphql::Pagination::Keyset::Connection do let(:nodes) do Project.order(Arel.sql('projects.last_repository_check_at IS NULL')).order(last_repository_check_at: :asc).order(id: :asc) end + let(:ascending_nodes) { [project5, project1, project3, project2, project4] } it_behaves_like 'nodes are in ascending order' @@ -210,6 +211,7 @@ RSpec.describe Gitlab::Graphql::Pagination::Keyset::Connection do let(:nodes) do Project.order(Arel.sql('projects.last_repository_check_at IS NULL')).order(last_repository_check_at: :desc).order(id: :asc) end + let(:descending_nodes) { [project3, project1, project5, project2, project4] } it_behaves_like 'nodes are in descending order' @@ -243,6 +245,7 @@ RSpec.describe Gitlab::Graphql::Pagination::Keyset::Connection do let(:nodes) do Project.order(Arel::Table.new(:projects)['name'].lower.asc).order(id: :asc) end + let(:ascending_nodes) { [project1, project5, project3, project2, project4] } it_behaves_like 'nodes are in ascending order' @@ -252,6 +255,7 @@ RSpec.describe Gitlab::Graphql::Pagination::Keyset::Connection do let(:nodes) do Project.order(Arel::Table.new(:projects)['name'].lower.desc).order(id: :desc) end + let(:descending_nodes) { [project4, project2, project3, project5, project1] } it_behaves_like 'nodes are in descending order' diff --git a/spec/lib/gitlab/hashed_path_spec.rb b/spec/lib/gitlab/hashed_path_spec.rb new file mode 100644 index 00000000000..051c5196748 --- /dev/null +++ b/spec/lib/gitlab/hashed_path_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::HashedPath do + let(:root_hash) { 1 } + let(:hashed_path) { described_class.new(*path, root_hash: root_hash) } + + describe '#to_s' do + subject { hashed_path } + + context 'when path contains a single value' do + let(:path) { 'path' } + + it 'returns the disk path' do + expect(subject).to match(%r[\h{2}/\h{2}/\h{64}/path]) + end + end + + context 'when path contains multiple values' do + let(:path) { %w(path1 path2) } + + it 'returns the disk path' do + expect(subject).to match(%r[\h{2}/\h{2}/\h{64}/path1/path2]) + end + end + end +end diff --git a/spec/lib/gitlab/hook_data/issuable_builder_spec.rb b/spec/lib/gitlab/hook_data/issuable_builder_spec.rb index 50f3a4776be..f5ee8eba8bc 100644 --- a/spec/lib/gitlab/hook_data/issuable_builder_spec.rb +++ b/spec/lib/gitlab/hook_data/issuable_builder_spec.rb @@ -56,6 +56,7 @@ RSpec.describe Gitlab::HookData::IssuableBuilder do ] } end + let(:data) { builder.build(user: user, changes: changes) } it 'populates the :changes hash' do diff --git a/spec/lib/gitlab/http_spec.rb b/spec/lib/gitlab/http_spec.rb index 09da94e7559..5c990eb3248 100644 --- a/spec/lib/gitlab/http_spec.rb +++ b/spec/lib/gitlab/http_spec.rb @@ -5,6 +5,8 @@ require 'spec_helper' RSpec.describe Gitlab::HTTP do include StubRequests + let(:default_options) { described_class::DEFAULT_TIMEOUT_OPTIONS } + context 'when allow_local_requests' do it 'sends the request to the correct URI' do stub_full_request('https://example.org:8080', ip_address: '8.8.8.8').to_return(status: 200) @@ -101,6 +103,73 @@ RSpec.describe Gitlab::HTTP do end end + describe 'setting default timeouts' do + before do + stub_full_request('http://example.org', method: :any) + end + + context 'when no timeouts are set' do + it 'sets default open and read and write timeouts' do + expect(described_class).to receive(:httparty_perform_request).with( + Net::HTTP::Get, 'http://example.org', default_options + ).and_call_original + + described_class.get('http://example.org') + end + end + + context 'when :timeout is set' do + it 'does not set any default timeouts' do + expect(described_class).to receive(:httparty_perform_request).with( + Net::HTTP::Get, 'http://example.org', timeout: 1 + ).and_call_original + + described_class.get('http://example.org', timeout: 1) + end + end + + context 'when :open_timeout is set' do + it 'only sets default read and write timeout' do + expect(described_class).to receive(:httparty_perform_request).with( + Net::HTTP::Get, 'http://example.org', default_options.merge(open_timeout: 1) + ).and_call_original + + described_class.get('http://example.org', open_timeout: 1) + end + end + + context 'when :read_timeout is set' do + it 'only sets default open and write timeout' do + expect(described_class).to receive(:httparty_perform_request).with( + Net::HTTP::Get, 'http://example.org', default_options.merge(read_timeout: 1) + ).and_call_original + + described_class.get('http://example.org', read_timeout: 1) + end + end + + context 'when :write_timeout is set' do + it 'only sets default open and read timeout' do + expect(described_class).to receive(:httparty_perform_request).with( + Net::HTTP::Put, 'http://example.org', default_options.merge(write_timeout: 1) + ).and_call_original + + described_class.put('http://example.org', write_timeout: 1) + end + end + + context 'when default timeouts feature is disabled' do + it 'does not apply any defaults' do + stub_feature_flags(http_default_timeouts: false) + expect(described_class).to receive(:httparty_perform_request).with( + Net::HTTP::Get, 'http://example.org', open_timeout: 1 + ).and_call_original + + described_class.get('http://example.org', open_timeout: 1) + end + end + end + describe '.try_get' do let(:path) { 'http://example.org' } @@ -111,10 +180,10 @@ RSpec.describe Gitlab::HTTP do end let(:request_options) do - { + default_options.merge({ verify: false, basic_auth: { username: 'user', password: 'pass' } - } + }) end described_class::HTTP_ERRORS.each do |exception_class| @@ -123,8 +192,8 @@ RSpec.describe Gitlab::HTTP do context 'with path' do before do - expect(described_class).to receive(:get) - .with(path, {}) + expect(described_class).to receive(:httparty_perform_request) + .with(Net::HTTP::Get, path, default_options) .and_raise(klass) end @@ -155,8 +224,8 @@ RSpec.describe Gitlab::HTTP do context 'with path and options' do before do - expect(described_class).to receive(:get) - .with(path, request_options) + expect(described_class).to receive(:httparty_perform_request) + .with(Net::HTTP::Get, path, request_options) .and_raise(klass) end @@ -191,8 +260,8 @@ RSpec.describe Gitlab::HTTP do end before do - expect(described_class).to receive(:get) - .with(path, request_options, &block) + expect(described_class).to receive(:httparty_perform_request) + .with(Net::HTTP::Get, path, request_options, &block) .and_raise(klass) end diff --git a/spec/lib/gitlab/i18n/po_linter_spec.rb b/spec/lib/gitlab/i18n/po_linter_spec.rb index 5dfc9d3613c..cfa39d95ebd 100644 --- a/spec/lib/gitlab/i18n/po_linter_spec.rb +++ b/spec/lib/gitlab/i18n/po_linter_spec.rb @@ -6,7 +6,7 @@ require 'simple_po_parser' # Disabling this cop to allow for multi-language examples in comments # rubocop:disable Style/AsciiComments RSpec.describe Gitlab::I18n::PoLinter do - let(:linter) { described_class.new(po_path) } + let(:linter) { described_class.new(po_path: po_path, html_todolist: {}) } let(:po_path) { 'spec/fixtures/valid.po' } def fake_translation(msgid:, translation:, plural_id: nil, plurals: []) @@ -23,8 +23,9 @@ RSpec.describe Gitlab::I18n::PoLinter do end Gitlab::I18n::TranslationEntry.new( - data, - plurals.size + 1 + entry_data: data, + nplurals: plurals.size + 1, + html_allowed: nil ) end @@ -145,6 +146,67 @@ RSpec.describe Gitlab::I18n::PoLinter do expect(errors[message_id]).to include(expected_error) end end + + context 'when an entry contains html' do + let(:po_path) { 'spec/fixtures/potential_html.po' } + + it 'presents an error for each component containing angle brackets' do + message_id = 'String with some <strong>emphasis</strong>' + + expect(errors[message_id]).to match_array [ + a_string_starting_with('contains < or >.'), + a_string_starting_with('plural id contains < or >.'), + a_string_starting_with('translation contains < or >.') + ] + end + end + + context 'when an entry contains html on the todolist' do + subject(:linter) { described_class.new(po_path: po_path, html_todolist: todolist) } + + let(:po_path) { 'spec/fixtures/potential_html.po' } + let(:todolist) do + { + 'String with a legitimate < use' => { + 'plural_id' => 'String with lots of < > uses', + 'translations' => [ + 'Translated string with a legitimate < use', + 'Translated string with lots of < > uses' + ] + } + } + end + + it 'does not present an error' do + message_id = 'String with a legitimate < use' + + expect(errors[message_id]).to be_nil + end + end + + context 'when an entry on the html todolist has changed' do + subject(:linter) { described_class.new(po_path: po_path, html_todolist: todolist) } + + let(:po_path) { 'spec/fixtures/potential_html.po' } + let(:todolist) do + { + 'String with a legitimate < use' => { + 'plural_id' => 'String with lots of < > uses', + 'translations' => [ + 'Translated string with a different legitimate < use', + 'Translated string with lots of < > uses' + ] + } + } + end + + it 'presents an error for the changed component' do + message_id = 'String with a legitimate < use' + + expect(errors[message_id]) + .to include a_string_starting_with('translation contains < or >.') + end + end end describe '#parse_po' do @@ -200,6 +262,7 @@ RSpec.describe Gitlab::I18n::PoLinter do expect(linter).to receive(:validate_number_of_plurals).with([], fake_entry) expect(linter).to receive(:validate_unescaped_chars).with([], fake_entry) expect(linter).to receive(:validate_translation).with([], fake_entry) + expect(linter).to receive(:validate_html).with([], fake_entry) linter.validate_entry(fake_entry) end @@ -212,8 +275,9 @@ RSpec.describe Gitlab::I18n::PoLinter do allow(linter).to receive(:metadata_entry).and_return(fake_metadata) fake_entry = Gitlab::I18n::TranslationEntry.new( - { msgid: 'the singular', msgid_plural: 'the plural', 'msgstr[0]' => 'the singular' }, - 2 + entry_data: { msgid: 'the singular', msgid_plural: 'the plural', 'msgstr[0]' => 'the singular' }, + nplurals: 2, + html_allowed: nil ) errors = [] diff --git a/spec/lib/gitlab/i18n/translation_entry_spec.rb b/spec/lib/gitlab/i18n/translation_entry_spec.rb index 76879f75bec..2c95b0b0124 100644 --- a/spec/lib/gitlab/i18n/translation_entry_spec.rb +++ b/spec/lib/gitlab/i18n/translation_entry_spec.rb @@ -6,7 +6,7 @@ RSpec.describe Gitlab::I18n::TranslationEntry do describe '#singular_translation' do it 'returns the normal `msgstr` for translations without plural' do data = { msgid: 'Hello world', msgstr: 'Bonjour monde' } - entry = described_class.new(data, 2) + entry = described_class.new(entry_data: data, nplurals: 2, html_allowed: nil) expect(entry.singular_translation).to eq('Bonjour monde') end @@ -18,7 +18,7 @@ RSpec.describe Gitlab::I18n::TranslationEntry do 'msgstr[0]' => 'Bonjour monde', 'msgstr[1]' => 'Bonjour mondes' } - entry = described_class.new(data, 2) + entry = described_class.new(entry_data: data, nplurals: 2, html_allowed: nil) expect(entry.singular_translation).to eq('Bonjour monde') end @@ -27,7 +27,7 @@ RSpec.describe Gitlab::I18n::TranslationEntry do describe '#all_translations' do it 'returns all translations for singular translations' do data = { msgid: 'Hello world', msgstr: 'Bonjour monde' } - entry = described_class.new(data, 2) + entry = described_class.new(entry_data: data, nplurals: 2, html_allowed: nil) expect(entry.all_translations).to eq(['Bonjour monde']) end @@ -39,7 +39,7 @@ RSpec.describe Gitlab::I18n::TranslationEntry do 'msgstr[0]' => 'Bonjour monde', 'msgstr[1]' => 'Bonjour mondes' } - entry = described_class.new(data, 2) + entry = described_class.new(entry_data: data, nplurals: 2, html_allowed: nil) expect(entry.all_translations).to eq(['Bonjour monde', 'Bonjour mondes']) end @@ -52,7 +52,7 @@ RSpec.describe Gitlab::I18n::TranslationEntry do msgid_plural: 'Hello worlds', 'msgstr[0]' => 'Bonjour monde' } - entry = described_class.new(data, 1) + entry = described_class.new(entry_data: data, nplurals: 1, html_allowed: nil) expect(entry.plural_translations).to eq(['Bonjour monde']) end @@ -65,7 +65,7 @@ RSpec.describe Gitlab::I18n::TranslationEntry do 'msgstr[1]' => 'Bonjour mondes', 'msgstr[2]' => 'Bonjour tous les mondes' } - entry = described_class.new(data, 3) + entry = described_class.new(entry_data: data, nplurals: 3, html_allowed: nil) expect(entry.plural_translations).to eq(['Bonjour mondes', 'Bonjour tous les mondes']) end @@ -77,7 +77,7 @@ RSpec.describe Gitlab::I18n::TranslationEntry do msgid: 'hello world', msgstr: 'hello' } - entry = described_class.new(data, 2) + entry = described_class.new(entry_data: data, nplurals: 2, html_allowed: nil) expect(entry).to have_singular_translation end @@ -89,7 +89,7 @@ RSpec.describe Gitlab::I18n::TranslationEntry do "msgstr[0]" => 'hello world', "msgstr[1]" => 'hello worlds' } - entry = described_class.new(data, 2) + entry = described_class.new(entry_data: data, nplurals: 2, html_allowed: nil) expect(entry).to have_singular_translation end @@ -100,7 +100,7 @@ RSpec.describe Gitlab::I18n::TranslationEntry do msgid_plural: 'hello worlds', "msgstr[0]" => 'hello worlds' } - entry = described_class.new(data, 1) + entry = described_class.new(entry_data: data, nplurals: 1, html_allowed: nil) expect(entry).not_to have_singular_translation end @@ -109,7 +109,7 @@ RSpec.describe Gitlab::I18n::TranslationEntry do describe '#msgid_contains_newlines' do it 'is true when the msgid is an array' do data = { msgid: %w(hello world) } - entry = described_class.new(data, 2) + entry = described_class.new(entry_data: data, nplurals: 2, html_allowed: nil) expect(entry.msgid_has_multiple_lines?).to be_truthy end @@ -118,7 +118,7 @@ RSpec.describe Gitlab::I18n::TranslationEntry do describe '#plural_id_contains_newlines' do it 'is true when the msgid is an array' do data = { msgid_plural: %w(hello world) } - entry = described_class.new(data, 2) + entry = described_class.new(entry_data: data, nplurals: 2, html_allowed: nil) expect(entry.plural_id_has_multiple_lines?).to be_truthy end @@ -127,7 +127,7 @@ RSpec.describe Gitlab::I18n::TranslationEntry do describe '#translations_contain_newlines' do it 'is true when the msgid is an array' do data = { msgstr: %w(hello world) } - entry = described_class.new(data, 2) + entry = described_class.new(entry_data: data, nplurals: 2, html_allowed: nil) expect(entry.translations_have_multiple_lines?).to be_truthy end @@ -135,7 +135,7 @@ RSpec.describe Gitlab::I18n::TranslationEntry do describe '#contains_unescaped_chars' do let(:data) { { msgid: '' } } - let(:entry) { described_class.new(data, 2) } + let(:entry) { described_class.new(entry_data: data, nplurals: 2, html_allowed: nil) } it 'is true when the msgid is an array' do string = '「100%確定」' @@ -177,7 +177,7 @@ RSpec.describe Gitlab::I18n::TranslationEntry do describe '#msgid_contains_unescaped_chars' do it 'is true when the msgid contains a `%`' do data = { msgid: '「100%確定」' } - entry = described_class.new(data, 2) + entry = described_class.new(entry_data: data, nplurals: 2, html_allowed: nil) expect(entry).to receive(:contains_unescaped_chars?).and_call_original expect(entry.msgid_contains_unescaped_chars?).to be_truthy @@ -187,7 +187,7 @@ RSpec.describe Gitlab::I18n::TranslationEntry do describe '#plural_id_contains_unescaped_chars' do it 'is true when the plural msgid contains a `%`' do data = { msgid_plural: '「100%確定」' } - entry = described_class.new(data, 2) + entry = described_class.new(entry_data: data, nplurals: 2, html_allowed: nil) expect(entry).to receive(:contains_unescaped_chars?).and_call_original expect(entry.plural_id_contains_unescaped_chars?).to be_truthy @@ -197,10 +197,144 @@ RSpec.describe Gitlab::I18n::TranslationEntry do describe '#translations_contain_unescaped_chars' do it 'is true when the translation contains a `%`' do data = { msgstr: '「100%確定」' } - entry = described_class.new(data, 2) + entry = described_class.new(entry_data: data, nplurals: 2, html_allowed: nil) expect(entry).to receive(:contains_unescaped_chars?).and_call_original expect(entry.translations_contain_unescaped_chars?).to be_truthy end end + + describe '#msgid_contains_potential_html?' do + subject(:entry) { described_class.new(entry_data: data, nplurals: 2, html_allowed: nil) } + + context 'when there are no angle brackets in the msgid' do + let(:data) { { msgid: 'String with no brackets' } } + + it 'returns false' do + expect(entry.msgid_contains_potential_html?).to be_falsey + end + end + + context 'when there are angle brackets in the msgid' do + let(:data) { { msgid: 'String with <strong> tag' } } + + it 'returns true' do + expect(entry.msgid_contains_potential_html?).to be_truthy + end + end + end + + describe '#plural_id_contains_potential_html?' do + subject(:entry) { described_class.new(entry_data: data, nplurals: 2, html_allowed: nil) } + + context 'when there are no angle brackets in the plural_id' do + let(:data) { { msgid_plural: 'String with no brackets' } } + + it 'returns false' do + expect(entry.plural_id_contains_potential_html?).to be_falsey + end + end + + context 'when there are angle brackets in the plural_id' do + let(:data) { { msgid_plural: 'This string has a <strong>' } } + + it 'returns true' do + expect(entry.plural_id_contains_potential_html?).to be_truthy + end + end + end + + describe '#translations_contain_potential_html?' do + subject(:entry) { described_class.new(entry_data: data, nplurals: 2, html_allowed: nil) } + + context 'when there are no angle brackets in the translations' do + let(:data) { { msgstr: 'This string has no angle brackets' } } + + it 'returns false' do + expect(entry.translations_contain_potential_html?).to be_falsey + end + end + + context 'when there are angle brackets in the translations' do + let(:data) { { msgstr: 'This string has a <strong>' } } + + it 'returns true' do + expect(entry.translations_contain_potential_html?).to be_truthy + end + end + end + + describe '#msgid_html_allowed?' do + subject(:entry) do + described_class.new(entry_data: { msgid: 'String with a <strong>' }, nplurals: 2, html_allowed: html_todo) + end + + context 'when the html in the string is in the todolist' do + let(:html_todo) { { 'plural_id' => nil, 'translations' => [] } } + + it 'returns true' do + expect(entry.msgid_html_allowed?).to be true + end + end + + context 'when the html in the string is not in the todolist' do + let(:html_todo) { nil } + + it 'returns false' do + expect(entry.msgid_html_allowed?).to be false + end + end + end + + describe '#plural_id_html_allowed?' do + subject(:entry) do + described_class.new(entry_data: { msgid_plural: 'String with many <strong>' }, nplurals: 2, html_allowed: html_todo) + end + + context 'when the html in the string is in the todolist' do + let(:html_todo) { { 'plural_id' => 'String with many <strong>', 'translations' => [] } } + + it 'returns true' do + expect(entry.plural_id_html_allowed?).to be true + end + end + + context 'when the html in the string is not in the todolist' do + let(:html_todo) { { 'plural_id' => 'String with some <strong>', 'translations' => [] } } + + it 'returns false' do + expect(entry.plural_id_html_allowed?).to be false + end + end + end + + describe '#translations_html_allowed?' do + subject(:entry) do + described_class.new(entry_data: { msgstr: 'String with a <strong>' }, nplurals: 2, html_allowed: html_todo) + end + + context 'when the html in the string is in the todolist' do + let(:html_todo) { { 'plural_id' => nil, 'translations' => ['String with a <strong>'] } } + + it 'returns true' do + expect(entry.translations_html_allowed?).to be true + end + end + + context 'when the html in the string is not in the todolist' do + let(:html_todo) { { 'plural_id' => nil, 'translations' => ['String with a different <strong>'] } } + + it 'returns false' do + expect(entry.translations_html_allowed?).to be false + end + end + + context 'when the todolist only has the msgid' do + let(:html_todo) { { 'plural_id' => nil, 'translations' => nil } } + + it 'returns false' do + expect(entry.translations_html_allowed?).to be false + end + end + end end diff --git a/spec/lib/gitlab/i18n_spec.rb b/spec/lib/gitlab/i18n_spec.rb index 592adadc362..ee10739195a 100644 --- a/spec/lib/gitlab/i18n_spec.rb +++ b/spec/lib/gitlab/i18n_spec.rb @@ -5,6 +5,14 @@ require 'spec_helper' RSpec.describe Gitlab::I18n do let(:user) { create(:user, preferred_language: 'es') } + describe '.selectable_locales' do + it 'does not return languages that should not be available in the UI' do + Gitlab::I18n::NOT_AVAILABLE_IN_UI.each do |language| + expect(described_class.selectable_locales).not_to include(language) + end + end + end + describe '.locale=' do after do described_class.use_default_locale diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 02500778426..37b5d8a1021 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -12,6 +12,7 @@ issues: - resource_weight_events - resource_milestone_events - resource_state_events +- resource_iteration_events - sent_notifications - sentry_issue - label_links @@ -46,6 +47,8 @@ issues: - system_note_metadata - alert_management_alert - status_page_published_incident +- namespace +- note_authors events: - author - project @@ -166,6 +169,7 @@ merge_requests: - deployments - user_mentions - system_note_metadata +- note_authors external_pull_requests: - project merge_request_diff: @@ -229,6 +233,7 @@ ci_pipelines: - daily_report_results - latest_builds_report_results - messages +- pipeline_artifacts ci_refs: - project - ci_pipelines @@ -312,6 +317,7 @@ project: - chat_services - cluster - clusters +- cluster_agents - cluster_project - creator - cycle_analytics_stages @@ -351,7 +357,6 @@ project: - youtrack_service - custom_issue_tracker_service - bugzilla_service -- gitlab_issue_tracker_service - external_wiki_service - mock_ci_service - mock_deployment_service @@ -465,6 +470,7 @@ project: - vulnerability_identifiers - vulnerability_scanners - dast_site_profiles +- dast_scanner_profiles - dast_sites - operations_feature_flags - operations_feature_flags_client @@ -515,6 +521,9 @@ project: - webex_teams_service - build_report_results - vulnerability_statistic +- vulnerability_historical_statistics +- product_analytics_events +- pipeline_artifacts award_emoji: - awardable - user @@ -665,6 +674,7 @@ epic: - events - resource_label_events - user_mentions +- note_authors epic_issue: - epic - issue diff --git a/spec/lib/gitlab/import_export/base/object_builder_spec.rb b/spec/lib/gitlab/import_export/base/object_builder_spec.rb index d560c8ea5a7..38c3b23db36 100644 --- a/spec/lib/gitlab/import_export/base/object_builder_spec.rb +++ b/spec/lib/gitlab/import_export/base/object_builder_spec.rb @@ -10,6 +10,7 @@ RSpec.describe Gitlab::ImportExport::Base::ObjectBuilder do name: 'project', path: 'project') end + let(:klass) { Milestone } let(:attributes) { { 'title' => 'Test Base::ObjectBuilder Milestone', 'project' => project } } diff --git a/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb b/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb index 6cc16ee9cbb..2eb983cc050 100644 --- a/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb @@ -172,7 +172,7 @@ RSpec.describe Gitlab::ImportExport::Group::TreeRestorer do let(:filepath) { "group_exports/visibility_levels/#{visibility_level}" } it "imports all subgroups as #{visibility_level}" do - expect(group.children.map(&:visibility_level)).to eq(expected_visibilities) + expect(group.children.map(&:visibility_level)).to match_array(expected_visibilities) end end end diff --git a/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb b/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb index eb6b07ce02f..949cfb5a34d 100644 --- a/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb +++ b/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb @@ -81,6 +81,7 @@ RSpec.describe Gitlab::ImportExport::JSON::StreamingSerializer do let(:group_options) do { include: [], only: [:name, :path, :description] } end + let(:include) do [{ group: group_options }] end diff --git a/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb index 6d5604dc40f..f75494aa7c7 100644 --- a/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb @@ -502,6 +502,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer do let(:project_tree_restorer) do described_class.new(user: user, shared: shared, project: project) end + let(:restored_project_json) { project_tree_restorer.restore } it 'does not read a symlink' do @@ -919,6 +920,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer do } ] end + let(:tree_hash) { { 'project_members' => project_members } } before do diff --git a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb index 40c103eeda6..a2c5848f100 100644 --- a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb +++ b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb @@ -275,6 +275,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver do File.join(shared.export_path, Gitlab::ImportExport.project_filename) end end + let(:shared) { project.import_export_shared } let(:params) { {} } diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index 2d313b4dcad..a108bc94da5 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -32,6 +32,7 @@ Issue: - discussion_locked - health_status - external_key +- issue_type Event: - id - target_type @@ -41,6 +42,7 @@ Event: - updated_at - action - author_id +- fingerprint WikiPage::Meta: - id - title @@ -215,6 +217,7 @@ MergeRequestDiff: - head_commit_sha - start_commit_sha - commits_count +- files_count MergeRequestDiffCommit: - merge_request_diff_id - relative_order @@ -285,6 +288,7 @@ MergeRequest::Metrics: - first_approved_at - first_reassigned_at - added_lines +- target_project_id - removed_lines Ci::Pipeline: - id @@ -656,6 +660,7 @@ PrometheusMetric: - group - common - identifier +- dashboard_path PrometheusAlert: - threshold - operator @@ -735,6 +740,8 @@ Board: - milestone_id - weight - name +- hide_backlog_list +- hide_closed_list List: - id - board_id @@ -765,6 +772,7 @@ DesignManagement::Design: - id - project_id - filename +- relative_position DesignManagement::Action: - id - event diff --git a/spec/lib/gitlab/incident_management/pager_duty/incident_issue_description_spec.rb b/spec/lib/gitlab/incident_management/pager_duty/incident_issue_description_spec.rb index 9a55e21d031..6dc96217f09 100644 --- a/spec/lib/gitlab/incident_management/pager_duty/incident_issue_description_spec.rb +++ b/spec/lib/gitlab/incident_management/pager_duty/incident_issue_description_spec.rb @@ -10,9 +10,11 @@ RSpec.describe Gitlab::IncidentManagement::PagerDuty::IncidentIssueDescription d let(:assignees) do [{ 'summary' => 'Laura Haley', 'url' => 'https://webdemo.pagerduty.com/users/P553OPV' }] end + let(:impacted_services) do [{ 'summary' => 'Production XDB Cluster', 'url' => 'https://webdemo.pagerduty.com/services/PN49J75' }] end + let(:incident_payload) do { 'url' => 'https://webdemo.pagerduty.com/incidents/PRORDTY', diff --git a/spec/lib/gitlab/incoming_email_spec.rb b/spec/lib/gitlab/incoming_email_spec.rb index 19d608cf48e..72d201eed77 100644 --- a/spec/lib/gitlab/incoming_email_spec.rb +++ b/spec/lib/gitlab/incoming_email_spec.rb @@ -10,7 +10,7 @@ RSpec.describe Gitlab::IncomingEmail do end it 'returns true' do - expect(described_class.enabled?).to be_truthy + expect(described_class.enabled?).to be(true) end end @@ -20,7 +20,7 @@ RSpec.describe Gitlab::IncomingEmail do end it "returns false" do - expect(described_class.enabled?).to be_falsey + expect(described_class.enabled?).to be(false) end end end @@ -32,7 +32,7 @@ RSpec.describe Gitlab::IncomingEmail do end it 'confirms that wildcard is supported' do - expect(described_class.supports_wildcard?).to be_truthy + expect(described_class.supports_wildcard?).to be(true) end end @@ -42,7 +42,7 @@ RSpec.describe Gitlab::IncomingEmail do end it 'returns that wildcard is not supported' do - expect(described_class.supports_wildcard?).to be_falsey + expect(described_class.supports_wildcard?).to be(false) end end @@ -52,7 +52,7 @@ RSpec.describe Gitlab::IncomingEmail do end it 'returns that wildcard is not supported' do - expect(described_class.supports_wildcard?).to be_falsey + expect(described_class.supports_wildcard?).to be(false) end end end diff --git a/spec/lib/gitlab/instrumentation/redis_interceptor_spec.rb b/spec/lib/gitlab/instrumentation/redis_interceptor_spec.rb index 5b0ad63ee72..09280402e2b 100644 --- a/spec/lib/gitlab/instrumentation/redis_interceptor_spec.rb +++ b/spec/lib/gitlab/instrumentation/redis_interceptor_spec.rb @@ -25,8 +25,8 @@ RSpec.describe Gitlab::Instrumentation::RedisInterceptor, :clean_gitlab_redis_sh # Exercise counting of a bulk reply [[:set, 'foo', 'bar' * 100]] | [:get, 'foo'] | 3 + 3 | 3 * 100 - # Nested array response: ['123456-89', ['foo', 'bar']] - [[:xadd, 'mystream', '123456-89', 'foo', 'bar']] | [:xrange, 'mystream', '-', '+'] | 6 + 8 + 1 + 1 | 9 + 3 + 3 + # Nested array response: [['foo', 0], ['bar', 1]] + [[:zadd, 'myset', 0, 'foo'], [:zadd, 'myset', 1, 'bar']] | [:zrange, 'myset', 0, -1, 'withscores'] | 6 + 5 + 1 + 2 + 10 | 3 + 1 + 3 + 1 end with_them do diff --git a/spec/lib/gitlab/issuable_sorter_spec.rb b/spec/lib/gitlab/issuable_sorter_spec.rb index 60f62062f04..b8d0c7b0609 100644 --- a/spec/lib/gitlab/issuable_sorter_spec.rb +++ b/spec/lib/gitlab/issuable_sorter_spec.rb @@ -52,6 +52,7 @@ RSpec.describe Gitlab::IssuableSorter do build_stubbed(:issue, iid: 1, project: project5), build_stubbed(:issue, iid: 1, project: project6)] end + let(:unsorted) do [sorted[3], sorted[1], sorted[4], sorted[2], sorted[6], sorted[5], sorted[0], sorted[7]] diff --git a/spec/lib/gitlab/issuables_count_for_state_spec.rb b/spec/lib/gitlab/issuables_count_for_state_spec.rb index 1c186a8e6ca..d96152e47ea 100644 --- a/spec/lib/gitlab/issuables_count_for_state_spec.rb +++ b/spec/lib/gitlab/issuables_count_for_state_spec.rb @@ -9,6 +9,21 @@ RSpec.describe Gitlab::IssuablesCountForState do let(:counter) { described_class.new(finder) } + describe 'project given' do + let(:project) { build(:project) } + let(:counter) { described_class.new(finder, project) } + + it 'provides the project' do + expect(counter.project).to eq(project) + end + end + + describe '.declarative_policy_class' do + subject { described_class.declarative_policy_class } + + it { is_expected.to eq('IssuablePolicy') } + end + describe '#for_state_or_opened' do it 'returns the number of issuables for the given state' do expect(counter.for_state_or_opened(:closed)).to eq(1) diff --git a/spec/lib/gitlab/jira_import/issue_serializer_spec.rb b/spec/lib/gitlab/jira_import/issue_serializer_spec.rb index 4adc4e4d22a..e57a8457e7c 100644 --- a/spec/lib/gitlab/jira_import/issue_serializer_spec.rb +++ b/spec/lib/gitlab/jira_import/issue_serializer_spec.rb @@ -25,6 +25,7 @@ RSpec.describe Gitlab::JiraImport::IssueSerializer do let(:parent_field) do { 'key' => 'FOO-2', 'id' => '1050', 'fields' => { 'summary' => 'parent issue FOO' } } end + let(:priority_field) { { 'name' => 'Medium' } } let(:labels_field) { %w(bug dev backend frontend) } diff --git a/spec/lib/gitlab/jira_import/metadata_collector_spec.rb b/spec/lib/gitlab/jira_import/metadata_collector_spec.rb index 86863d67f25..51751c7b75f 100644 --- a/spec/lib/gitlab/jira_import/metadata_collector_spec.rb +++ b/spec/lib/gitlab/jira_import/metadata_collector_spec.rb @@ -14,6 +14,7 @@ RSpec.describe Gitlab::JiraImport::MetadataCollector do let(:parent_field) do { 'key' => 'FOO-2', 'id' => '1050', 'fields' => { 'summary' => 'parent issue FOO' } } end + let(:issue_type_field) { { 'name' => 'Task' } } let(:fix_versions_field) { [{ 'name' => '1.0' }, { 'name' => '1.1' }] } let(:priority_field) { { 'name' => 'Medium' } } @@ -30,6 +31,7 @@ RSpec.describe Gitlab::JiraImport::MetadataCollector do 'duedate' => duedate_field } end + let(:jira_issue) do double( id: '1234', diff --git a/spec/lib/gitlab/job_waiter_spec.rb b/spec/lib/gitlab/job_waiter_spec.rb index 4d7c838aa3b..7aa0a3485fb 100644 --- a/spec/lib/gitlab/job_waiter_spec.rb +++ b/spec/lib/gitlab/job_waiter_spec.rb @@ -60,16 +60,14 @@ RSpec.describe Gitlab::JobWaiter do described_class.notify(waiter.key, 'a') described_class.notify(waiter.key, 'b') - result = nil - expect { Timeout.timeout(1) { result = waiter.wait(2) } }.not_to raise_error + expect { Timeout.timeout(1) { waiter.wait(2) } }.not_to raise_error end it 'increments job_waiter_started_total and job_waiter_timeouts_total when it times out' do expect(started_total).to receive(:increment).with(worker: 'Foo') expect(timeouts_total).to receive(:increment).with(worker: 'Foo') - result = nil - expect { Timeout.timeout(2) { result = waiter.wait(1) } }.not_to raise_error + expect { Timeout.timeout(2) { waiter.wait(1) } }.not_to raise_error end end end diff --git a/spec/lib/gitlab/json_spec.rb b/spec/lib/gitlab/json_spec.rb index d7671dda323..0402296a3a8 100644 --- a/spec/lib/gitlab/json_spec.rb +++ b/spec/lib/gitlab/json_spec.rb @@ -407,4 +407,36 @@ RSpec.describe Gitlab::Json do end end end + + describe Gitlab::Json::LimitedEncoder do + subject { described_class.encode(obj, limit: 8.kilobytes) } + + context 'when object size is acceptable' do + let(:obj) { { test: true } } + + it 'returns json string' do + is_expected.to eq("{\"test\":true}") + end + end + + context 'when object is too big' do + let(:obj) { [{ test: true }] * 1000 } + + it 'raises LimitExceeded error' do + expect { subject }.to raise_error( + Gitlab::Json::LimitedEncoder::LimitExceeded + ) + end + end + + context 'when json_limited_encoder is disabled' do + let(:obj) { [{ test: true }] * 1000 } + + it 'does not raise an error' do + stub_feature_flags(json_limited_encoder: false) + + expect { subject }.not_to raise_error + end + end + end end diff --git a/spec/lib/gitlab/kubernetes/cilium_network_policy_spec.rb b/spec/lib/gitlab/kubernetes/cilium_network_policy_spec.rb new file mode 100644 index 00000000000..9600a70a95d --- /dev/null +++ b/spec/lib/gitlab/kubernetes/cilium_network_policy_spec.rb @@ -0,0 +1,217 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Kubernetes::CiliumNetworkPolicy do + let(:policy) do + described_class.new( + name: name, + namespace: namespace, + creation_timestamp: '2020-04-14T00:08:30Z', + endpoint_selector: endpoint_selector, + ingress: ingress, + egress: egress, + description: description + ) + end + + let(:resource) do + ::Kubeclient::Resource.new( + kind: partial_class_name, + apiVersion: "cilium.io/v2", + metadata: { name: name, namespace: namespace, resourceVersion: resource_version }, + spec: { endpointSelector: endpoint_selector, ingress: ingress, egress: nil } + ) + end + + let(:name) { 'example-name' } + let(:namespace) { 'example-namespace' } + let(:endpoint_selector) { { matchLabels: { role: 'db' } } } + let(:description) { 'example-description' } + let(:partial_class_name) { described_class.name.split('::').last } + let(:resource_version) { 101 } + let(:ingress) do + [ + { + fromEndpoints: [ + { matchLabels: { project: 'myproject' } } + ] + } + ] + end + + let(:egress) do + [ + { + ports: [{ port: 5978 }] + } + ] + end + + include_examples 'network policy common specs' do + let(:selector) { endpoint_selector} + let(:policy) do + described_class.new( + name: name, + namespace: namespace, + selector: selector, + ingress: ingress, + labels: labels, + resource_version: resource_version + ) + end + + let(:spec) { { endpointSelector: selector, ingress: ingress, egress: nil } } + let(:metadata) { { name: name, namespace: namespace, resourceVersion: resource_version } } + end + + describe '#generate' do + subject { policy.generate } + + it { is_expected.to eq(resource) } + end + + describe '.from_yaml' do + let(:manifest) do + <<~POLICY + apiVersion: cilium.io/v2 + kind: CiliumNetworkPolicy + metadata: + name: example-name + namespace: example-namespace + resourceVersion: 101 + spec: + endpointSelector: + matchLabels: + role: db + ingress: + - fromEndpoints: + - matchLabels: + project: myproject + POLICY + end + + subject { Gitlab::Kubernetes::CiliumNetworkPolicy.from_yaml(manifest)&.generate } + + it { is_expected.to eq(resource) } + + context 'with nil manifest' do + let(:manifest) { nil } + + it { is_expected.to be_nil } + end + + context 'with invalid manifest' do + let(:manifest) { "\tfoo: bar" } + + it { is_expected.to be_nil } + end + + context 'with manifest without metadata' do + let(:manifest) do + <<~POLICY + apiVersion: cilium.io/v2 + kind: CiliumNetworkPolicy + spec: + endpointSelector: + matchLabels: + role: db + ingress: + - fromEndpoints: + matchLabels: + project: myproject + POLICY + end + + it { is_expected.to be_nil } + end + + context 'with manifest without spec' do + let(:manifest) do + <<~POLICY + apiVersion: cilium.io/v2 + kind: CiliumNetworkPolicy + metadata: + name: example-name + namespace: example-namespace + POLICY + end + + it { is_expected.to be_nil } + end + + context 'with disallowed class' do + let(:manifest) do + <<~POLICY + apiVersion: cilium.io/v2 + kind: CiliumNetworkPolicy + metadata: + name: example-name + namespace: example-namespace + creationTimestamp: 2020-04-14T00:08:30Z + spec: + endpointSelector: + matchLabels: + role: db + ingress: + - fromEndpoints: + matchLabels: + project: myproject + POLICY + end + + it { is_expected.to be_nil } + end + end + + describe '.from_resource' do + let(:resource) do + ::Kubeclient::Resource.new( + metadata: { + name: name, namespace: namespace, creationTimestamp: '2020-04-14T00:08:30Z', + labels: { app: 'foo' }, resourceVersion: resource_version + }, + spec: { endpointSelector: endpoint_selector, ingress: ingress, egress: nil, labels: nil, description: nil } + ) + end + + let(:generated_resource) do + ::Kubeclient::Resource.new( + kind: partial_class_name, + apiVersion: "cilium.io/v2", + metadata: { name: name, namespace: namespace, resourceVersion: resource_version, labels: { app: 'foo' } }, + spec: { endpointSelector: endpoint_selector, ingress: ingress, egress: nil } + ) + end + + subject { Gitlab::Kubernetes::CiliumNetworkPolicy.from_resource(resource)&.generate } + + it { is_expected.to eq(generated_resource) } + + context 'with nil resource' do + let(:resource) { nil } + + it { is_expected.to be_nil } + end + + context 'with resource without metadata' do + let(:resource) do + ::Kubeclient::Resource.new( + spec: { endpointSelector: endpoint_selector, ingress: ingress, egress: nil, labels: nil, description: nil } + ) + end + + it { is_expected.to be_nil } + end + + context 'with resource without spec' do + let(:resource) do + ::Kubeclient::Resource.new( + metadata: { name: name, namespace: namespace, uid: '128cf288-7de4-11ea-aceb-42010a800089', resourceVersion: resource_version } + ) + end + + it { is_expected.to be_nil } + end + end +end diff --git a/spec/lib/gitlab/kubernetes/helm/api_spec.rb b/spec/lib/gitlab/kubernetes/helm/api_spec.rb index dabbab27b13..bcc95bdbf2b 100644 --- a/spec/lib/gitlab/kubernetes/helm/api_spec.rb +++ b/spec/lib/gitlab/kubernetes/helm/api_spec.rb @@ -17,8 +17,7 @@ RSpec.describe Gitlab::Kubernetes::Helm::API do name: application_name, chart: 'chart-name', rbac: rbac, - files: files, - local_tiller_enabled: true + files: files ) end @@ -143,7 +142,7 @@ RSpec.describe Gitlab::Kubernetes::Helm::API do end context 'with a service account' do - let(:command) { Gitlab::Kubernetes::Helm::InitCommand.new(name: application_name, files: files, rbac: rbac, local_tiller_enabled: true) } + let(:command) { Gitlab::Kubernetes::Helm::InitCommand.new(name: application_name, files: files, rbac: rbac) } context 'rbac-enabled cluster' do let(:rbac) { true } diff --git a/spec/lib/gitlab/kubernetes/helm/base_command_spec.rb b/spec/lib/gitlab/kubernetes/helm/base_command_spec.rb index 57fafaebf98..a7abd6ab1bf 100644 --- a/spec/lib/gitlab/kubernetes/helm/base_command_spec.rb +++ b/spec/lib/gitlab/kubernetes/helm/base_command_spec.rb @@ -16,8 +16,7 @@ RSpec.describe Gitlab::Kubernetes::Helm::BaseCommand do super( name: 'test-class-name', rbac: rbac, - files: { some: 'value' }, - local_tiller_enabled: false + files: { some: 'value' } ) end end diff --git a/spec/lib/gitlab/kubernetes/helm/delete_command_spec.rb b/spec/lib/gitlab/kubernetes/helm/delete_command_spec.rb index 7b182478cc3..ff2c2d76f22 100644 --- a/spec/lib/gitlab/kubernetes/helm/delete_command_spec.rb +++ b/spec/lib/gitlab/kubernetes/helm/delete_command_spec.rb @@ -3,12 +3,11 @@ require 'spec_helper' RSpec.describe Gitlab::Kubernetes::Helm::DeleteCommand do - subject(:delete_command) { described_class.new(name: app_name, rbac: rbac, files: files, local_tiller_enabled: local_tiller_enabled) } + subject(:delete_command) { described_class.new(name: app_name, rbac: rbac, files: files) } let(:app_name) { 'app-name' } let(:rbac) { true } let(:files) { {} } - let(:local_tiller_enabled) { true } it_behaves_like 'helm command generator' do let(:commands) do @@ -21,50 +20,6 @@ RSpec.describe Gitlab::Kubernetes::Helm::DeleteCommand do end end - context 'tillerless feature disabled' do - let(:local_tiller_enabled) { false } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - helm init --upgrade - for i in $(seq 1 30); do helm version && s=0 && break || s=$?; sleep 1s; echo \"Retrying ($i)...\"; done; (exit $s) - helm delete --purge app-name - EOS - end - end - - context 'when there is a ca.pem file' do - let(:files) { { 'ca.pem': 'some file content' } } - - let(:tls_flags) do - <<~EOS.squish - --tls - --tls-ca-cert /data/helm/app-name/config/ca.pem - --tls-cert /data/helm/app-name/config/cert.pem - --tls-key /data/helm/app-name/config/key.pem - EOS - end - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - helm init --upgrade - for i in $(seq 1 30); do helm version #{tls_flags} && s=0 && break || s=$?; sleep 1s; echo \"Retrying ($i)...\"; done; (exit $s) - #{helm_delete_command} - EOS - end - - let(:helm_delete_command) do - <<~EOS.squish - helm delete --purge app-name - #{tls_flags} - EOS - end - end - end - end - describe '#pod_name' do subject { delete_command.pod_name } diff --git a/spec/lib/gitlab/kubernetes/helm/init_command_spec.rb b/spec/lib/gitlab/kubernetes/helm/init_command_spec.rb index c982a417682..d538ed12a07 100644 --- a/spec/lib/gitlab/kubernetes/helm/init_command_spec.rb +++ b/spec/lib/gitlab/kubernetes/helm/init_command_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::Kubernetes::Helm::InitCommand do - subject(:init_command) { described_class.new(name: application.name, files: files, rbac: rbac, local_tiller_enabled: false) } + subject(:init_command) { described_class.new(name: application.name, files: files, rbac: rbac) } let(:application) { create(:clusters_applications_helm) } let(:rbac) { false } diff --git a/spec/lib/gitlab/kubernetes/helm/install_command_spec.rb b/spec/lib/gitlab/kubernetes/helm/install_command_spec.rb index aad350256ec..6ed7323c96f 100644 --- a/spec/lib/gitlab/kubernetes/helm/install_command_spec.rb +++ b/spec/lib/gitlab/kubernetes/helm/install_command_spec.rb @@ -12,8 +12,7 @@ RSpec.describe Gitlab::Kubernetes::Helm::InstallCommand do version: version, repository: repository, preinstall: preinstall, - postinstall: postinstall, - local_tiller_enabled: local_tiller_enabled + postinstall: postinstall ) end @@ -23,7 +22,6 @@ RSpec.describe Gitlab::Kubernetes::Helm::InstallCommand do let(:version) { '1.2.3' } let(:preinstall) { nil } let(:postinstall) { nil } - let(:local_tiller_enabled) { true } it_behaves_like 'helm command generator' do let(:commands) do @@ -52,46 +50,6 @@ RSpec.describe Gitlab::Kubernetes::Helm::InstallCommand do end end - context 'tillerless feature disabled' do - let(:local_tiller_enabled) { false } - - let(:tls_flags) do - <<~EOS.squish - --tls - --tls-ca-cert /data/helm/app-name/config/ca.pem - --tls-cert /data/helm/app-name/config/cert.pem - --tls-key /data/helm/app-name/config/key.pem - EOS - end - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - helm init --upgrade - for i in $(seq 1 30); do helm version #{tls_flags} && s=0 && break || s=$?; sleep 1s; echo \"Retrying ($i)...\"; done; (exit $s) - helm repo add app-name https://repository.example.com - helm repo update - #{helm_install_comand} - EOS - end - - let(:helm_install_comand) do - <<~EOS.squish - helm upgrade app-name chart-name - --install - --atomic - --cleanup-on-fail - --reset-values - #{tls_flags} - --version 1.2.3 - --set rbac.create\\=false,rbac.enabled\\=false - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - end - context 'when rbac is true' do let(:rbac) { true } diff --git a/spec/lib/gitlab/kubernetes/helm/patch_command_spec.rb b/spec/lib/gitlab/kubernetes/helm/patch_command_spec.rb index ea2ade18e37..487a38f286d 100644 --- a/spec/lib/gitlab/kubernetes/helm/patch_command_spec.rb +++ b/spec/lib/gitlab/kubernetes/helm/patch_command_spec.rb @@ -7,7 +7,6 @@ RSpec.describe Gitlab::Kubernetes::Helm::PatchCommand do let(:repository) { 'https://repository.example.com' } let(:rbac) { false } let(:version) { '1.2.3' } - let(:local_tiller_enabled) { true } subject(:patch_command) do described_class.new( @@ -16,47 +15,10 @@ RSpec.describe Gitlab::Kubernetes::Helm::PatchCommand do rbac: rbac, files: files, version: version, - repository: repository, - local_tiller_enabled: local_tiller_enabled + repository: repository ) end - context 'when local tiller feature is disabled' do - let(:local_tiller_enabled) { false } - - let(:tls_flags) do - <<~EOS.squish - --tls - --tls-ca-cert /data/helm/app-name/config/ca.pem - --tls-cert /data/helm/app-name/config/cert.pem - --tls-key /data/helm/app-name/config/key.pem - EOS - end - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - helm init --upgrade - for i in $(seq 1 30); do helm version #{tls_flags} && s=0 && break || s=$?; sleep 1s; echo \"Retrying ($i)...\"; done; (exit $s) - helm repo add app-name https://repository.example.com - helm repo update - #{helm_upgrade_comand} - EOS - end - - let(:helm_upgrade_comand) do - <<~EOS.squish - helm upgrade app-name chart-name - --reuse-values - #{tls_flags} - --version 1.2.3 - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - end - it_behaves_like 'helm command generator' do let(:commands) do <<~EOS diff --git a/spec/lib/gitlab/kubernetes/helm/reset_command_spec.rb b/spec/lib/gitlab/kubernetes/helm/reset_command_spec.rb index 8d386d41ad5..5a3ba59b8c0 100644 --- a/spec/lib/gitlab/kubernetes/helm/reset_command_spec.rb +++ b/spec/lib/gitlab/kubernetes/helm/reset_command_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::Kubernetes::Helm::ResetCommand do - subject(:reset_command) { described_class.new(name: name, rbac: rbac, files: files, local_tiller_enabled: false) } + subject(:reset_command) { described_class.new(name: name, rbac: rbac, files: files) } let(:rbac) { true } let(:name) { 'helm' } diff --git a/spec/lib/gitlab/kubernetes/kube_client_spec.rb b/spec/lib/gitlab/kubernetes/kube_client_spec.rb index a15be42f393..8211b096d3b 100644 --- a/spec/lib/gitlab/kubernetes/kube_client_spec.rb +++ b/spec/lib/gitlab/kubernetes/kube_client_spec.rb @@ -80,13 +80,13 @@ RSpec.describe Gitlab::Kubernetes::KubeClient do context 'errored' do using RSpec::Parameterized::TableSyntax - where(:error, :error_status) do - SocketError | :unreachable - OpenSSL::X509::CertificateError | :authentication_failure - StandardError | :unknown_failure - Kubeclient::HttpError.new(408, "timed out", nil) | :unreachable - Kubeclient::HttpError.new(408, "timeout", nil) | :unreachable - Kubeclient::HttpError.new(408, "", nil) | :authentication_failure + where(:error, :connection_status, :error_status) do + SocketError | :unreachable | :connection_error + OpenSSL::X509::CertificateError | :authentication_failure | :authentication_error + StandardError | :unknown_failure | :unknown_error + Kubeclient::HttpError.new(408, "timed out", nil) | :unreachable | :http_error + Kubeclient::HttpError.new(408, "timeout", nil) | :unreachable | :http_error + Kubeclient::HttpError.new(408, "", nil) | :authentication_failure | :http_error end with_them do @@ -97,7 +97,7 @@ RSpec.describe Gitlab::Kubernetes::KubeClient do it 'returns error status' do result = described_class.graceful_request(1) { client.foo } - expect(result).to eq({ status: error_status }) + expect(result).to eq({ status: connection_status, connection_error: error_status }) end end end @@ -227,6 +227,20 @@ RSpec.describe Gitlab::Kubernetes::KubeClient do end end + describe '#cilium_networking_client' do + subject { client.cilium_networking_client } + + it_behaves_like 'a Kubeclient' + + it 'has the cilium API group endpoint' do + expect(subject.api_endpoint.to_s).to match(%r{\/apis\/cilium.io\Z}) + end + + it 'has the api_version' do + expect(subject.instance_variable_get(:@api_version)).to eq('v2') + end + end + describe '#metrics_client' do subject { client.metrics_client } @@ -380,6 +394,30 @@ RSpec.describe Gitlab::Kubernetes::KubeClient do end end + describe 'cilium API group' do + let(:cilium_networking_client) { client.cilium_networking_client } + + [ + :create_cilium_network_policy, + :get_cilium_network_policies, + :update_cilium_network_policy, + :delete_cilium_network_policy + ].each do |method| + describe "##{method}" do + include_examples 'redirection not allowed', method + include_examples 'dns rebinding not allowed', method + + it 'delegates to the cilium client' do + expect(client).to delegate_method(method).to(:cilium_networking_client) + end + + it 'responds to the method' do + expect(client).to respond_to method + end + end + end + end + describe 'non-entity methods' do it 'does not proxy for non-entity methods' do expect(client).not_to respond_to :proxy_url diff --git a/spec/lib/gitlab/kubernetes/network_policy_spec.rb b/spec/lib/gitlab/kubernetes/network_policy_spec.rb index a8ca15f998b..5d1dd5dec59 100644 --- a/spec/lib/gitlab/kubernetes/network_policy_spec.rb +++ b/spec/lib/gitlab/kubernetes/network_policy_spec.rb @@ -8,13 +8,20 @@ RSpec.describe Gitlab::Kubernetes::NetworkPolicy do name: name, namespace: namespace, creation_timestamp: '2020-04-14T00:08:30Z', - pod_selector: pod_selector, + selector: pod_selector, policy_types: %w(Ingress Egress), ingress: ingress, egress: egress ) end + let(:resource) do + ::Kubeclient::Resource.new( + metadata: { name: name, namespace: namespace }, + spec: { podSelector: pod_selector, policyTypes: %w(Ingress), ingress: ingress, egress: nil } + ) + end + let(:name) { 'example-name' } let(:namespace) { 'example-namespace' } let(:pod_selector) { { matchLabels: { role: 'db' } } } @@ -37,6 +44,28 @@ RSpec.describe Gitlab::Kubernetes::NetworkPolicy do ] end + include_examples 'network policy common specs' do + let(:selector) { pod_selector } + let(:policy) do + described_class.new( + name: name, + namespace: namespace, + selector: selector, + ingress: ingress, + labels: labels + ) + end + + let(:spec) { { podSelector: selector, policyTypes: ["Ingress"], ingress: ingress, egress: nil } } + let(:metadata) { { name: name, namespace: namespace } } + end + + describe '#generate' do + subject { policy.generate } + + it { is_expected.to eq(resource) } + end + describe '.from_yaml' do let(:manifest) do <<~POLICY @@ -45,8 +74,6 @@ RSpec.describe Gitlab::Kubernetes::NetworkPolicy do metadata: name: example-name namespace: example-namespace - labels: - app: foo spec: podSelector: matchLabels: @@ -60,12 +87,6 @@ RSpec.describe Gitlab::Kubernetes::NetworkPolicy do project: myproject POLICY end - let(:resource) do - ::Kubeclient::Resource.new( - metadata: { name: name, namespace: namespace, labels: { app: 'foo' } }, - spec: { podSelector: pod_selector, policyTypes: %w(Ingress), ingress: ingress, egress: nil } - ) - end subject { Gitlab::Kubernetes::NetworkPolicy.from_yaml(manifest)&.generate } @@ -156,6 +177,7 @@ RSpec.describe Gitlab::Kubernetes::NetworkPolicy do spec: { podSelector: pod_selector, policyTypes: %w(Ingress), ingress: ingress, egress: nil } ) end + let(:generated_resource) do ::Kubeclient::Resource.new( metadata: { name: name, namespace: namespace, labels: { app: 'foo' } }, @@ -193,202 +215,4 @@ RSpec.describe Gitlab::Kubernetes::NetworkPolicy do it { is_expected.to be_nil } end end - - describe '#generate' do - let(:resource) do - ::Kubeclient::Resource.new( - metadata: { name: name, namespace: namespace }, - spec: { podSelector: pod_selector, policyTypes: %w(Ingress Egress), ingress: ingress, egress: egress } - ) - end - - subject { policy.generate } - - it { is_expected.to eq(resource) } - end - - describe '#as_json' do - let(:json_policy) do - { - name: name, - namespace: namespace, - creation_timestamp: '2020-04-14T00:08:30Z', - manifest: YAML.dump( - { - metadata: { name: name, namespace: namespace }, - spec: { podSelector: pod_selector, policyTypes: %w(Ingress Egress), ingress: ingress, egress: egress } - }.deep_stringify_keys - ), - is_autodevops: false, - is_enabled: true - } - end - - subject { policy.as_json } - - it { is_expected.to eq(json_policy) } - end - - describe '#autodevops?' do - subject { policy.autodevops? } - - let(:chart) { nil } - let(:policy) do - described_class.new( - name: name, - namespace: namespace, - labels: { chart: chart }, - pod_selector: pod_selector, - ingress: ingress - ) - end - - it { is_expected.to be false } - - context 'with non-autodevops chart' do - let(:chart) { 'foo' } - - it { is_expected.to be false } - end - - context 'with autodevops chart' do - let(:chart) { 'auto-deploy-app-0.6.0' } - - it { is_expected.to be true } - end - end - - describe '#enabled?' do - subject { policy.enabled? } - - let(:pod_selector) { nil } - let(:policy) do - described_class.new( - name: name, - namespace: namespace, - pod_selector: pod_selector, - ingress: ingress - ) - end - - it { is_expected.to be true } - - context 'with empty pod_selector' do - let(:pod_selector) { {} } - - it { is_expected.to be true } - end - - context 'with nil matchLabels in pod_selector' do - let(:pod_selector) { { matchLabels: nil } } - - it { is_expected.to be true } - end - - context 'with empty matchLabels in pod_selector' do - let(:pod_selector) { { matchLabels: {} } } - - it { is_expected.to be true } - end - - context 'with disabled_by label in matchLabels in pod_selector' do - let(:pod_selector) do - { matchLabels: { Gitlab::Kubernetes::NetworkPolicy::DISABLED_BY_LABEL => 'gitlab' } } - end - - it { is_expected.to be false } - end - end - - describe '#enable' do - subject { policy.enabled? } - - let(:pod_selector) { nil } - let(:policy) do - described_class.new( - name: name, - namespace: namespace, - pod_selector: pod_selector, - ingress: ingress - ) - end - - before do - policy.enable - end - - it { is_expected.to be true } - - context 'with empty pod_selector' do - let(:pod_selector) { {} } - - it { is_expected.to be true } - end - - context 'with nil matchLabels in pod_selector' do - let(:pod_selector) { { matchLabels: nil } } - - it { is_expected.to be true } - end - - context 'with empty matchLabels in pod_selector' do - let(:pod_selector) { { matchLabels: {} } } - - it { is_expected.to be true } - end - - context 'with disabled_by label in matchLabels in pod_selector' do - let(:pod_selector) do - { matchLabels: { Gitlab::Kubernetes::NetworkPolicy::DISABLED_BY_LABEL => 'gitlab' } } - end - - it { is_expected.to be true } - end - end - - describe '#disable' do - subject { policy.enabled? } - - let(:pod_selector) { nil } - let(:policy) do - described_class.new( - name: name, - namespace: namespace, - pod_selector: pod_selector, - ingress: ingress - ) - end - - before do - policy.disable - end - - it { is_expected.to be false } - - context 'with empty pod_selector' do - let(:pod_selector) { {} } - - it { is_expected.to be false } - end - - context 'with nil matchLabels in pod_selector' do - let(:pod_selector) { { matchLabels: nil } } - - it { is_expected.to be false } - end - - context 'with empty matchLabels in pod_selector' do - let(:pod_selector) { { matchLabels: {} } } - - it { is_expected.to be false } - end - - context 'with disabled_by label in matchLabels in pod_selector' do - let(:pod_selector) do - { matchLabels: { Gitlab::Kubernetes::NetworkPolicy::DISABLED_BY_LABEL => 'gitlab' } } - end - - it { is_expected.to be false } - end - end end diff --git a/spec/lib/gitlab/kubernetes/node_spec.rb b/spec/lib/gitlab/kubernetes/node_spec.rb index 732bf29bc44..fdc3433ff0f 100644 --- a/spec/lib/gitlab/kubernetes/node_spec.rb +++ b/spec/lib/gitlab/kubernetes/node_spec.rb @@ -7,45 +7,51 @@ RSpec.describe Gitlab::Kubernetes::Node do describe '#all' do let(:cluster) { create(:cluster, :provided_by_user, :group) } - let(:expected_nodes) { [] } + let(:expected_nodes) { nil } + let(:nodes) { [kube_node.merge(kube_node_metrics)] } + + subject { described_class.new(cluster).all } before do stub_kubeclient_nodes_and_nodes_metrics(cluster.platform.api_url) end - subject { described_class.new(cluster).all } - context 'when connection to the cluster is successful' do - let(:expected_nodes) { [kube_node.merge(kube_node_metrics)] } + let(:expected_nodes) { { nodes: nodes } } it { is_expected.to eq(expected_nodes) } end - context 'when cluster cannot be reached' do - before do - allow(cluster.kubeclient.core_client).to receive(:discover) - .and_raise(SocketError) + context 'when there is a connection error' do + using RSpec::Parameterized::TableSyntax + + where(:error, :error_status) do + SocketError | :kubernetes_connection_error + OpenSSL::X509::CertificateError | :kubernetes_authentication_error + StandardError | :unknown_error + Kubeclient::HttpError.new(408, "", nil) | :kubeclient_http_error end - it { is_expected.to eq(expected_nodes) } - end + context 'when there is an error while querying nodes' do + with_them do + before do + allow(cluster.kubeclient).to receive(:get_nodes).and_raise(error) + end - context 'when cluster cannot be authenticated to' do - before do - allow(cluster.kubeclient.core_client).to receive(:discover) - .and_raise(OpenSSL::X509::CertificateError.new('Certificate error')) + it { is_expected.to eq({ node_connection_error: error_status }) } + end end - it { is_expected.to eq(expected_nodes) } - end + context 'when there is an error while querying metrics' do + with_them do + before do + allow(cluster.kubeclient).to receive(:get_nodes).and_return({ response: nodes }) + allow(cluster.kubeclient).to receive(:metrics_client).and_raise(error) + end - context 'when Kubeclient::HttpError is raised' do - before do - allow(cluster.kubeclient.core_client).to receive(:discover) - .and_raise(Kubeclient::HttpError.new(403, 'Forbidden', nil)) + it { is_expected.to eq({ nodes: nodes, metrics_connection_error: error_status }) } + end end - - it { is_expected.to eq(expected_nodes) } end context 'when an uncategorised error is raised' do @@ -54,7 +60,7 @@ RSpec.describe Gitlab::Kubernetes::Node do .and_raise(StandardError) end - it { is_expected.to eq(expected_nodes) } + it { is_expected.to eq({ node_connection_error: :unknown_error }) } it 'notifies Sentry' do expect(Gitlab::ErrorTracking).to receive(:track_exception) diff --git a/spec/lib/gitlab/language_detection_spec.rb b/spec/lib/gitlab/language_detection_spec.rb index 04ad19a04ec..14523be8ec6 100644 --- a/spec/lib/gitlab/language_detection_spec.rb +++ b/spec/lib/gitlab/language_detection_spec.rb @@ -15,6 +15,7 @@ RSpec.describe Gitlab::LanguageDetection do { value: 1.51, label: "Go", color: "#2a4776", highlight: "#244776" }, { value: 1.1, label: "MepmepLang", color: "#2a4776", highlight: "#244776" }] end + let(:repository_languages) do [RepositoryLanguage.new(share: 10, programming_language: ruby)] end diff --git a/spec/lib/gitlab/legacy_github_import/importer_spec.rb b/spec/lib/gitlab/legacy_github_import/importer_spec.rb index c443944678f..56d708a1e11 100644 --- a/spec/lib/gitlab/legacy_github_import/importer_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/importer_spec.rb @@ -238,6 +238,7 @@ RSpec.describe Gitlab::LegacyGithubImport::Importer do labels: [double(name: 'Label #2')] ) end + let(:closed_pull_request) do double( number: 1347, diff --git a/spec/lib/gitlab/legacy_github_import/issuable_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/issuable_formatter_spec.rb index 6a51cb6f39d..a5d2e00890b 100644 --- a/spec/lib/gitlab/legacy_github_import/issuable_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/issuable_formatter_spec.rb @@ -6,6 +6,7 @@ RSpec.describe Gitlab::LegacyGithubImport::IssuableFormatter do let(:raw_data) do double(number: 42) end + let(:project) { double(import_type: 'github') } let(:issuable_formatter) { described_class.new(project, raw_data) } diff --git a/spec/lib/gitlab/legacy_github_import/milestone_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/milestone_formatter_spec.rb index 2ac79c4f5b8..148b59dedab 100644 --- a/spec/lib/gitlab/legacy_github_import/milestone_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/milestone_formatter_spec.rb @@ -17,6 +17,7 @@ RSpec.describe Gitlab::LegacyGithubImport::MilestoneFormatter do closed_at: nil } end + let(:iid_attr) { :number } subject(:formatter) { described_class.new(project, raw_data) } diff --git a/spec/lib/gitlab/lograge/custom_options_spec.rb b/spec/lib/gitlab/lograge/custom_options_spec.rb index 218007c6e2a..9daedfc37e4 100644 --- a/spec/lib/gitlab/lograge/custom_options_spec.rb +++ b/spec/lib/gitlab/lograge/custom_options_spec.rb @@ -22,6 +22,7 @@ RSpec.describe Gitlab::Lograge::CustomOptions do metadata: { 'meta.user' => 'jane.doe' } } end + let(:event) { ActiveSupport::Notifications::Event.new('test', 1, 2, 'transaction_id', event_payload) } subject { described_class.call(event) } diff --git a/spec/lib/gitlab/manifest_import/project_creator_spec.rb b/spec/lib/gitlab/manifest_import/project_creator_spec.rb index 354acf53b7a..0ab5b277552 100644 --- a/spec/lib/gitlab/manifest_import/project_creator_spec.rb +++ b/spec/lib/gitlab/manifest_import/project_creator_spec.rb @@ -23,13 +23,14 @@ RSpec.describe Gitlab::ManifestImport::ProjectCreator do it { expect { subject.execute }.to change { Project.count }.by(1) } it { expect { subject.execute }.to change { Group.count }.by(1) } - it 'creates project with valid full path and import url' do + it 'creates project with valid full path, import url and import source' do subject.execute project = Project.last expect(project.full_path).to eq(File.join(group.path, 'device/common')) expect(project.import_url).to eq('https://android-review.googlesource.com/device/common') + expect(project.import_source).to eq('https://android-review.googlesource.com/device/common') end end end diff --git a/spec/lib/gitlab/markdown_cache/redis/store_spec.rb b/spec/lib/gitlab/markdown_cache/redis/store_spec.rb index 40ff9a765a6..bf40af8e62e 100644 --- a/spec/lib/gitlab/markdown_cache/redis/store_spec.rb +++ b/spec/lib/gitlab/markdown_cache/redis/store_spec.rb @@ -18,6 +18,7 @@ RSpec.describe Gitlab::MarkdownCache::Redis::Store, :clean_gitlab_redis_cache do end end end + let(:storable) { storable_class.new } let(:cache_key) { "markdown_cache:#{storable.cache_key}" } diff --git a/spec/lib/gitlab/metrics/background_transaction_spec.rb b/spec/lib/gitlab/metrics/background_transaction_spec.rb index 640bbebf0da..b2a53fe1626 100644 --- a/spec/lib/gitlab/metrics/background_transaction_spec.rb +++ b/spec/lib/gitlab/metrics/background_transaction_spec.rb @@ -4,16 +4,30 @@ require 'spec_helper' RSpec.describe Gitlab::Metrics::BackgroundTransaction do let(:test_worker_class) { double(:class, name: 'TestWorker') } + let(:prometheus_metric) { instance_double(Prometheus::Client::Metric, base_labels: {}) } + + before do + allow(described_class).to receive(:prometheus_metric).and_return(prometheus_metric) + end subject { described_class.new(test_worker_class) } + RSpec.shared_examples 'metric with worker labels' do |metric_method| + it 'measures with correct labels and value' do + value = 1 + expect(prometheus_metric).to receive(metric_method).with({ controller: 'TestWorker', action: 'perform', feature_category: '' }, value) + + subject.send(metric_method, :bau, value) + end + end + describe '#label' do it 'returns labels based on class name' do expect(subject.labels).to eq(controller: 'TestWorker', action: 'perform', feature_category: '') end it 'contains only the labels defined for metrics' do - expect(subject.labels.keys).to contain_exactly(*described_class.superclass::BASE_LABELS.keys) + expect(subject.labels.keys).to contain_exactly(*described_class.superclass::BASE_LABEL_KEYS) end it 'includes the feature category if there is one' do @@ -21,4 +35,22 @@ RSpec.describe Gitlab::Metrics::BackgroundTransaction do expect(subject.labels).to include(feature_category: 'source_code_management') end end + + describe '#increment' do + let(:prometheus_metric) { instance_double(Prometheus::Client::Counter, :increment, base_labels: {}) } + + it_behaves_like 'metric with worker labels', :increment + end + + describe '#set' do + let(:prometheus_metric) { instance_double(Prometheus::Client::Gauge, :set, base_labels: {}) } + + it_behaves_like 'metric with worker labels', :set + end + + describe '#observe' do + let(:prometheus_metric) { instance_double(Prometheus::Client::Histogram, :observe, base_labels: {}) } + + it_behaves_like 'metric with worker labels', :observe + end end diff --git a/spec/lib/gitlab/metrics/dashboard/cache_spec.rb b/spec/lib/gitlab/metrics/dashboard/cache_spec.rb new file mode 100644 index 00000000000..9467d441ae1 --- /dev/null +++ b/spec/lib/gitlab/metrics/dashboard/cache_spec.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Metrics::Dashboard::Cache, :use_clean_rails_memory_store_caching do + let_it_be(:project1) { build_stubbed(:project) } + let_it_be(:project2) { build_stubbed(:project) } + + let(:project1_key1) { "#{project1.id}_key1" } + let(:project1_key2) { "#{project1.id}_key2" } + let(:project2_key1) { "#{project2.id}_key1" } + + let(:cache1) { described_class.for(project1) } + let(:cache2) { described_class.for(project2) } + + before do + cache1.fetch(project1_key1) { 'data1' } + cache1.fetch(project1_key2) { 'data2' } + cache2.fetch(project2_key1) { 'data3' } + end + + describe '.fetch' do + it 'stores data correctly' do + described_class.fetch('key1') { 'data1' } + described_class.fetch('key2') { 'data2' } + + expect(described_class.fetch('key1')).to eq('data1') + expect(described_class.fetch('key2')).to eq('data2') + end + end + + describe '.for' do + it 'returns a new instance' do + expect(described_class.for(project1)).to be_instance_of(described_class) + end + end + + describe '#fetch' do + it 'stores data correctly' do + expect(cache1.fetch(project1_key1)).to eq('data1') + expect(cache1.fetch(project1_key2)).to eq('data2') + expect(cache2.fetch(project2_key1)).to eq('data3') + end + end + + describe '#delete_all!' do + it 'deletes keys of the given project', :aggregate_failures do + cache1.delete_all! + + expect(Rails.cache.exist?(project1_key1)).to be(false) + expect(Rails.cache.exist?(project1_key2)).to be(false) + expect(cache2.fetch(project2_key1)).to eq('data3') + + cache2.delete_all! + + expect(Rails.cache.exist?(project2_key1)).to be(false) + end + + it 'does not fail when nothing to delete' do + project3 = build_stubbed(:project) + cache3 = described_class.for(project3) + + expect { cache3.delete_all! }.not_to raise_error + end + end + + context 'multiple fetches and deletes' do + specify :aggregate_failures do + cache1.delete_all! + + expect(Rails.cache.exist?(project1_key1)).to be(false) + expect(Rails.cache.exist?(project1_key2)).to be(false) + + cache1.fetch("#{project1.id}_key3") { 'data1' } + cache1.fetch("#{project1.id}_key4") { 'data2' } + + expect(cache1.fetch("#{project1.id}_key3")).to eq('data1') + expect(cache1.fetch("#{project1.id}_key4")).to eq('data2') + + cache1.delete_all! + + expect(Rails.cache.exist?("#{project1.id}_key3")).to be(false) + expect(Rails.cache.exist?("#{project1.id}_key4")).to be(false) + end + end +end diff --git a/spec/lib/gitlab/metrics/dashboard/defaults_spec.rb b/spec/lib/gitlab/metrics/dashboard/defaults_spec.rb index dd61f8ebc4d..1f306753c39 100644 --- a/spec/lib/gitlab/metrics/dashboard/defaults_spec.rb +++ b/spec/lib/gitlab/metrics/dashboard/defaults_spec.rb @@ -4,5 +4,4 @@ require 'spec_helper' RSpec.describe Gitlab::Metrics::Dashboard::Defaults do it { is_expected.to be_const_defined(:DEFAULT_PANEL_TYPE) } - it { is_expected.to be_const_defined(:DEFAULT_PANEL_WEIGHT) } end diff --git a/spec/lib/gitlab/metrics/dashboard/finder_spec.rb b/spec/lib/gitlab/metrics/dashboard/finder_spec.rb index 60e1e29d4c5..730a31346d7 100644 --- a/spec/lib/gitlab/metrics/dashboard/finder_spec.rb +++ b/spec/lib/gitlab/metrics/dashboard/finder_spec.rb @@ -142,20 +142,42 @@ RSpec.describe Gitlab::Metrics::Dashboard::Finder, :use_clean_rails_memory_store describe '.find_all_paths' do let(:all_dashboard_paths) { described_class.find_all_paths(project) } - let(:system_dashboard) { { path: system_dashboard_path, display_name: 'Default dashboard', default: true, system_dashboard: true, out_of_the_box_dashboard: true } } + let(:system_dashboard) { { path: system_dashboard_path, display_name: 'Overview', default: true, system_dashboard: true, out_of_the_box_dashboard: true } } + let(:k8s_pod_health_dashboard) { { path: pod_dashboard_path, display_name: 'K8s pod health', default: false, system_dashboard: false, out_of_the_box_dashboard: true } } - it 'includes only the system dashboard by default' do - expect(all_dashboard_paths).to eq([system_dashboard]) + it 'includes OOTB dashboards by default' do + expect(all_dashboard_paths).to eq([k8s_pod_health_dashboard, system_dashboard]) end context 'when the project contains dashboards' do - let(:dashboard_path) { '.gitlab/dashboards/test.yml' } - let(:project) { project_with_dashboard(dashboard_path) } + let(:dashboard_content) { fixture_file('lib/gitlab/metrics/dashboard/sample_dashboard.yml') } + let(:project) { project_with_dashboards(dashboards) } - it 'includes system and project dashboards' do - project_dashboard = { path: dashboard_path, display_name: 'test.yml', default: false, system_dashboard: false, out_of_the_box_dashboard: false } + let(:dashboards) do + { + '.gitlab/dashboards/metrics.yml' => dashboard_content, + '.gitlab/dashboards/better_metrics.yml' => dashboard_content + } + end - expect(all_dashboard_paths).to contain_exactly(system_dashboard, project_dashboard) + it 'includes OOTB and project dashboards' do + project_dashboard1 = { + path: '.gitlab/dashboards/metrics.yml', + display_name: 'metrics.yml', + default: false, + system_dashboard: false, + out_of_the_box_dashboard: false + } + + project_dashboard2 = { + path: '.gitlab/dashboards/better_metrics.yml', + display_name: 'better_metrics.yml', + default: false, + system_dashboard: false, + out_of_the_box_dashboard: false + } + + expect(all_dashboard_paths).to eq([project_dashboard2, k8s_pod_health_dashboard, project_dashboard1, system_dashboard]) end end @@ -163,12 +185,13 @@ RSpec.describe Gitlab::Metrics::Dashboard::Finder, :use_clean_rails_memory_store let(:self_monitoring_dashboard) do { path: self_monitoring_dashboard_path, - display_name: 'Default dashboard', + display_name: 'Overview', default: true, - system_dashboard: false, + system_dashboard: true, out_of_the_box_dashboard: true } end + let(:dashboard_path) { '.gitlab/dashboards/test.yml' } let(:project) { project_with_dashboard(dashboard_path) } @@ -185,7 +208,7 @@ RSpec.describe Gitlab::Metrics::Dashboard::Finder, :use_clean_rails_memory_store out_of_the_box_dashboard: false } - expect(all_dashboard_paths).to contain_exactly(self_monitoring_dashboard, project_dashboard) + expect(all_dashboard_paths).to eq([self_monitoring_dashboard, project_dashboard]) end end end diff --git a/spec/lib/gitlab/metrics/dashboard/processor_spec.rb b/spec/lib/gitlab/metrics/dashboard/processor_spec.rb index 7f7070dfafb..14a4c01fce3 100644 --- a/spec/lib/gitlab/metrics/dashboard/processor_spec.rb +++ b/spec/lib/gitlab/metrics/dashboard/processor_spec.rb @@ -16,7 +16,6 @@ RSpec.describe Gitlab::Metrics::Dashboard::Processor do Gitlab::Metrics::Dashboard::Stages::CustomMetricsInserter, Gitlab::Metrics::Dashboard::Stages::CustomMetricsDetailsInserter, Gitlab::Metrics::Dashboard::Stages::MetricEndpointInserter, - Gitlab::Metrics::Dashboard::Stages::Sorter, Gitlab::Metrics::Dashboard::Stages::AlertsInserter, Gitlab::Metrics::Dashboard::Stages::PanelIdsInserter, Gitlab::Metrics::Dashboard::Stages::UrlValidator @@ -26,12 +25,6 @@ RSpec.describe Gitlab::Metrics::Dashboard::Processor do let(:process_params) { [project, dashboard_yml, sequence, { environment: environment }] } let(:dashboard) { described_class.new(*process_params).process } - it 'includes a path for the prometheus endpoint with each metric' do - expect(all_metrics).to satisfy_all do |metric| - metric[:prometheus_endpoint_path] == prometheus_path(metric[:query_range]) - end - end - it 'includes an id for each dashboard panel' do expect(all_panels).to satisfy_all do |panel| panel[:id].present? @@ -72,14 +65,14 @@ RSpec.describe Gitlab::Metrics::Dashboard::Processor do expect(all_metrics).to include get_metric_details(project_business_metric) end - it 'orders groups by priority and panels by weight' do + it 'display groups and panels in the order they are defined' do expected_metrics_order = [ - 'metric_b', # group priority 10, panel weight 1 - 'metric_a2', # group priority 1, panel weight 2 - 'metric_a1', # group priority 1, panel weight 1 - project_business_metric.id, # group priority 0, panel weight nil (0) - project_response_metric.id, # group priority -5, panel weight nil (0) - project_system_metric.id # group priority -10, panel weight nil (0) + 'metric_b', + 'metric_a2', + 'metric_a1', + project_business_metric.id, + project_response_metric.id, + project_system_metric.id ] actual_metrics_order = all_metrics.map { |m| m[:id] || m[:metric_id] } @@ -100,10 +93,10 @@ RSpec.describe Gitlab::Metrics::Dashboard::Processor do let(:sequence) do [ Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter, - Gitlab::Metrics::Dashboard::Stages::MetricEndpointInserter, - Gitlab::Metrics::Dashboard::Stages::Sorter + Gitlab::Metrics::Dashboard::Stages::MetricEndpointInserter ] end + let(:dashboard) { described_class.new(*process_params).process } it 'includes only dashboard metrics' do diff --git a/spec/lib/gitlab/metrics/dashboard/repo_dashboard_finder_spec.rb b/spec/lib/gitlab/metrics/dashboard/repo_dashboard_finder_spec.rb new file mode 100644 index 00000000000..a2c9906c0e9 --- /dev/null +++ b/spec/lib/gitlab/metrics/dashboard/repo_dashboard_finder_spec.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Metrics::Dashboard::RepoDashboardFinder do + include MetricsDashboardHelpers + + let_it_be(:project) { create(:project) } + + describe '.list_dashboards' do + it 'deletes dashboard cache entries' do + cache = instance_double(Gitlab::Metrics::Dashboard::Cache) + allow(Gitlab::Metrics::Dashboard::Cache).to receive(:for).and_return(cache) + + expect(cache).to receive(:delete_all!) + + described_class.list_dashboards(project) + end + + it 'returns empty array when there are no dashboards' do + expect(described_class.list_dashboards(project)).to eq([]) + end + + context 'when there are project dashboards available' do + let_it_be(:dashboard_path) { '.gitlab/dashboards/test.yml' } + let_it_be(:project) { project_with_dashboard(dashboard_path) } + + it 'returns the dashboard list' do + expect(described_class.list_dashboards(project)).to contain_exactly(dashboard_path) + end + end + end + + describe '.read_dashboard' do + it 'raises error when dashboard does not exist' do + dashboard_path = '.gitlab/dashboards/test.yml' + + expect { described_class.read_dashboard(project, dashboard_path) }.to raise_error( + Gitlab::Metrics::Dashboard::Errors::NOT_FOUND_ERROR + ) + end + + context 'when there are project dashboards available' do + let_it_be(:dashboard_path) { '.gitlab/dashboards/test.yml' } + let_it_be(:project) { project_with_dashboard(dashboard_path) } + + it 'reads dashboard' do + expect(described_class.read_dashboard(project, dashboard_path)).to eq( + fixture_file('lib/gitlab/metrics/dashboard/sample_dashboard.yml') + ) + end + end + end +end diff --git a/spec/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter_spec.rb b/spec/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter_spec.rb new file mode 100644 index 00000000000..bb3c8626d32 --- /dev/null +++ b/spec/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter_spec.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Metrics::Dashboard::Stages::MetricEndpointInserter do + include MetricsDashboardHelpers + + let(:project) { build_stubbed(:project) } + let(:environment) { build_stubbed(:environment, project: project) } + + describe '#transform!' do + subject(:transform!) { described_class.new(project, dashboard, environment: environment).transform! } + + let(:dashboard) { load_sample_dashboard.deep_symbolize_keys } + + it 'generates prometheus_endpoint_path without newlines' do + query = 'avg( sum( container_memory_usage_bytes{ container_name!="POD", '\ + 'pod_name=~"^{{ci_environment_slug}}-(.*)", namespace="{{kube_namespace}}" } ) '\ + 'by (job) ) without (job) /1024/1024/1024' + + transform! + + expect(all_metrics[2][:prometheus_endpoint_path]).to eq(prometheus_path(query)) + end + + it 'includes a path for the prometheus endpoint with each metric' do + transform! + + expect(all_metrics).to satisfy_all do |metric| + metric[:prometheus_endpoint_path].present? && !metric[:prometheus_endpoint_path].include?("\n") + end + end + + it 'works when query/query_range is a number' do + query = 2000 + + transform! + + expect(all_metrics[1][:prometheus_endpoint_path]).to eq(prometheus_path(query)) + end + end + + private + + def all_metrics + dashboard[:panel_groups].flat_map do |group| + group[:panels].flat_map { |panel| panel[:metrics] } + end + end + + def prometheus_path(query) + Gitlab::Routing.url_helpers.prometheus_api_project_environment_path( + project, + environment, + proxy_path: :query_range, + query: query + ) + end +end diff --git a/spec/lib/gitlab/metrics/dashboard/stages/track_panel_type_spec.rb b/spec/lib/gitlab/metrics/dashboard/stages/track_panel_type_spec.rb new file mode 100644 index 00000000000..d9987b67127 --- /dev/null +++ b/spec/lib/gitlab/metrics/dashboard/stages/track_panel_type_spec.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Metrics::Dashboard::Stages::TrackPanelType do + include MetricsDashboardHelpers + + let(:project) { build_stubbed(:project) } + let(:environment) { build_stubbed(:environment, project: project) } + + describe '#transform!' do + subject { described_class.new(project, dashboard, environment: environment) } + + let(:dashboard) { load_sample_dashboard.deep_symbolize_keys } + + it 'creates tracking event' do + stub_application_setting(snowplow_enabled: true, snowplow_collector_hostname: 'localhost') + allow(Gitlab::Tracking).to receive(:event).and_call_original + + subject.transform! + + expect(Gitlab::Tracking).to have_received(:event) + .with('MetricsDashboard::Chart', 'chart_rendered', { label: 'area-chart' }) + .at_least(:once) + end + end +end diff --git a/spec/lib/gitlab/metrics/dashboard/url_spec.rb b/spec/lib/gitlab/metrics/dashboard/url_spec.rb index 56556423b05..205e1000376 100644 --- a/spec/lib/gitlab/metrics/dashboard/url_spec.rb +++ b/spec/lib/gitlab/metrics/dashboard/url_spec.rb @@ -102,6 +102,34 @@ RSpec.describe Gitlab::Metrics::Dashboard::Url do it_behaves_like 'regex which matches url when expected' end + describe '#alert_regex' do + let(:url) do + Gitlab::Routing.url_helpers.metrics_dashboard_namespace_project_prometheus_alert_url( + 'foo', + 'bar', + '1', + start: '2020-02-10T12:59:49.938Z', + end: '2020-02-10T20:59:49.938Z', + anchor: "anchor" + ) + end + + let(:expected_params) do + { + 'url' => url, + 'namespace' => 'foo', + 'project' => 'bar', + 'alert' => '1', + 'query' => "?end=2020-02-10T20%3A59%3A49.938Z&start=2020-02-10T12%3A59%3A49.938Z", + 'anchor' => '#anchor' + } + end + + subject { described_class.alert_regex } + + it_behaves_like 'regex which matches url when expected' + end + describe '#build_dashboard_url' do it 'builds the url for the dashboard endpoint' do url = described_class.build_dashboard_url('foo', 'bar', 1) diff --git a/spec/lib/gitlab/metrics/dashboard/validator/client_spec.rb b/spec/lib/gitlab/metrics/dashboard/validator/client_spec.rb new file mode 100644 index 00000000000..4b07f9dbbab --- /dev/null +++ b/spec/lib/gitlab/metrics/dashboard/validator/client_spec.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Metrics::Dashboard::Validator::Client do + include MetricsDashboardHelpers + + let_it_be(:schema_path) { 'lib/gitlab/metrics/dashboard/validator/schemas/dashboard.json' } + + subject { described_class.new(dashboard, schema_path) } + + describe '#execute' do + context 'with no validation errors' do + let(:dashboard) { load_sample_dashboard } + + it 'returns empty array' do + expect(subject.execute).to eq([]) + end + end + + context 'with validation errors' do + let(:dashboard) { load_dashboard_yaml(fixture_file('lib/gitlab/metrics/dashboard/invalid_dashboard.yml')) } + + it 'returns array of error objects' do + expect(subject.execute).to include(Gitlab::Metrics::Dashboard::Validator::Errors::SchemaValidationError) + end + end + end +end diff --git a/spec/lib/gitlab/metrics/dashboard/validator/custom_formats_spec.rb b/spec/lib/gitlab/metrics/dashboard/validator/custom_formats_spec.rb new file mode 100644 index 00000000000..129fb631f3e --- /dev/null +++ b/spec/lib/gitlab/metrics/dashboard/validator/custom_formats_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Metrics::Dashboard::Validator::CustomFormats do + describe '#format_handlers' do + describe 'add_to_metric_id_cache' do + it 'adds data to metric id cache' do + subject.format_handlers['add_to_metric_id_cache'].call('metric_id', '_schema') + + expect(subject.metric_ids_cache).to eq(["metric_id"]) + end + end + end +end diff --git a/spec/lib/gitlab/metrics/dashboard/validator/errors_spec.rb b/spec/lib/gitlab/metrics/dashboard/validator/errors_spec.rb new file mode 100644 index 00000000000..f0db1bd0d33 --- /dev/null +++ b/spec/lib/gitlab/metrics/dashboard/validator/errors_spec.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Metrics::Dashboard::Validator::Errors do + describe Gitlab::Metrics::Dashboard::Validator::Errors::SchemaValidationError do + context 'empty error hash' do + let(:error_hash) { {} } + + it 'uses default error message' do + expect(described_class.new(error_hash).message).to eq('Dashboard failed schema validation') + end + end + + context 'formatted message' do + subject { described_class.new(error_hash).message } + + let(:error_hash) do + { + 'data' => 'property_name', + 'data_pointer' => pointer, + 'type' => type, + 'schema' => 'schema', + 'details' => details + } + end + + context 'for root object' do + let(:pointer) { '' } + + context 'when required keys are missing' do + let(:type) { 'required' } + let(:details) { { 'missing_keys' => ['one'] } } + + it { is_expected.to eq 'root is missing required keys: one' } + end + end + + context 'for nested object' do + let(:pointer) { '/nested_objects/0' } + + context 'when required keys are missing' do + let(:type) { 'required' } + let(:details) { { 'missing_keys' => ['two'] } } + + it { is_expected.to eq '/nested_objects/0 is missing required keys: two' } + end + + context 'when there is type mismatch' do + %w(null string boolean integer number array object).each do |expected_type| + context "on type: #{expected_type}" do + let(:type) { expected_type } + let(:details) { nil } + + subject { described_class.new(error_hash).message } + + it { is_expected.to eq "'property_name' at /nested_objects/0 is not of type: #{expected_type}" } + end + end + end + + context 'when data does not match pattern' do + let(:type) { 'pattern' } + let(:error_hash) do + { + 'data' => 'property_name', + 'data_pointer' => pointer, + 'type' => type, + 'schema' => { 'pattern' => 'aa.*' } + } + end + + it { is_expected.to eq "'property_name' at /nested_objects/0 does not match pattern: aa.*" } + end + + context 'when data does not match format' do + let(:type) { 'format' } + let(:error_hash) do + { + 'data' => 'property_name', + 'data_pointer' => pointer, + 'type' => type, + 'schema' => { 'format' => 'date-time' } + } + end + + it { is_expected.to eq "'property_name' at /nested_objects/0 does not match format: date-time" } + end + + context 'when data is not const' do + let(:type) { 'const' } + let(:error_hash) do + { + 'data' => 'property_name', + 'data_pointer' => pointer, + 'type' => type, + 'schema' => { 'const' => 'one' } + } + end + + it { is_expected.to eq "'property_name' at /nested_objects/0 is not: \"one\"" } + end + + context 'when data is not included in enum' do + let(:type) { 'enum' } + let(:error_hash) do + { + 'data' => 'property_name', + 'data_pointer' => pointer, + 'type' => type, + 'schema' => { 'enum' => %w(one two) } + } + end + + it { is_expected.to eq "'property_name' at /nested_objects/0 is not one of: [\"one\", \"two\"]" } + end + + context 'when data is not included in enum' do + let(:type) { 'unknown' } + let(:error_hash) do + { + 'data' => 'property_name', + 'data_pointer' => pointer, + 'type' => type, + 'schema' => 'schema' + } + end + + it { is_expected.to eq "'property_name' at /nested_objects/0 is invalid: error_type=unknown" } + end + end + end + end + + describe Gitlab::Metrics::Dashboard::Validator::Errors::DuplicateMetricIds do + it 'has custom error message' do + expect(described_class.new.message).to eq('metric_id must be unique across a project') + end + end +end diff --git a/spec/lib/gitlab/metrics/dashboard/validator/post_schema_validator_spec.rb b/spec/lib/gitlab/metrics/dashboard/validator/post_schema_validator_spec.rb new file mode 100644 index 00000000000..e7cb1429ca9 --- /dev/null +++ b/spec/lib/gitlab/metrics/dashboard/validator/post_schema_validator_spec.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Metrics::Dashboard::Validator::PostSchemaValidator do + describe '#validate' do + context 'with no project and dashboard_path provided' do + context 'unique local metric_ids' do + it 'returns empty array' do + expect(described_class.new(metric_ids: [1, 2, 3]).validate).to eq([]) + end + end + + context 'duplicate local metrics_ids' do + it 'returns error' do + expect(described_class.new(metric_ids: [1, 1]).validate) + .to eq([Gitlab::Metrics::Dashboard::Validator::Errors::DuplicateMetricIds]) + end + end + end + + context 'with project and dashboard_path' do + let(:project) { create(:project) } + + subject do + described_class.new( + project: project, + metric_ids: ['some_identifier'], + dashboard_path: 'test/path.yml' + ).validate + end + + context 'with unique metric identifiers' do + before do + create(:prometheus_metric, + project: project, + identifier: 'some_other_identifier', + dashboard_path: 'test/path.yml' + ) + end + + it 'returns empty array' do + expect(subject).to eq([]) + end + end + + context 'duplicate metric identifiers in database' do + context 'with different dashboard_path' do + before do + create(:prometheus_metric, + project: project, + identifier: 'some_identifier', + dashboard_path: 'some/other/path.yml' + ) + end + + it 'returns error' do + expect(subject).to include(Gitlab::Metrics::Dashboard::Validator::Errors::DuplicateMetricIds) + end + end + + context 'with same dashboard_path' do + before do + create(:prometheus_metric, + project: project, + identifier: 'some_identifier', + dashboard_path: 'test/path.yml' + ) + end + + it 'returns empty array' do + expect(subject).to eq([]) + end + end + end + end + end +end diff --git a/spec/lib/gitlab/metrics/dashboard/validator_spec.rb b/spec/lib/gitlab/metrics/dashboard/validator_spec.rb new file mode 100644 index 00000000000..c4cda271408 --- /dev/null +++ b/spec/lib/gitlab/metrics/dashboard/validator_spec.rb @@ -0,0 +1,146 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Metrics::Dashboard::Validator do + include MetricsDashboardHelpers + + let_it_be(:valid_dashboard) { load_sample_dashboard } + let_it_be(:invalid_dashboard) { load_dashboard_yaml(fixture_file('lib/gitlab/metrics/dashboard/invalid_dashboard.yml')) } + let_it_be(:duplicate_id_dashboard) { load_dashboard_yaml(fixture_file('lib/gitlab/metrics/dashboard/duplicate_id_dashboard.yml')) } + + let_it_be(:project) { create(:project) } + + describe '#validate' do + context 'valid dashboard schema' do + it 'returns true' do + expect(described_class.validate(valid_dashboard)).to be true + end + + context 'with duplicate metric_ids' do + it 'returns false' do + expect(described_class.validate(duplicate_id_dashboard)).to be false + end + end + + context 'with dashboard_path and project' do + subject { described_class.validate(valid_dashboard, dashboard_path: 'test/path.yml', project: project) } + + context 'with no conflicting metric identifiers in db' do + it { is_expected.to be true } + end + + context 'with metric identifier present in current dashboard' do + before do + create(:prometheus_metric, + identifier: 'metric_a1', + dashboard_path: 'test/path.yml', + project: project + ) + end + + it { is_expected.to be true } + end + + context 'with metric identifier present in another dashboard' do + before do + create(:prometheus_metric, + identifier: 'metric_a1', + dashboard_path: 'some/other/dashboard/path.yml', + project: project + ) + end + + it { is_expected.to be false } + end + end + end + + context 'invalid dashboard schema' do + it 'returns false' do + expect(described_class.validate(invalid_dashboard)).to be false + end + end + end + + describe '#validate!' do + shared_examples 'validation failed' do |errors_message| + it 'raises error with corresponding messages', :aggregate_failures do + expect { subject }.to raise_error do |error| + expect(error).to be_kind_of(Gitlab::Metrics::Dashboard::Validator::Errors::InvalidDashboardError) + expect(error.message).to eq(errors_message) + end + end + end + + context 'valid dashboard schema' do + it 'returns true' do + expect(described_class.validate!(valid_dashboard)).to be true + end + + context 'with duplicate metric_ids' do + subject { described_class.validate!(duplicate_id_dashboard) } + + it_behaves_like 'validation failed', 'metric_id must be unique across a project' + end + + context 'with dashboard_path and project' do + subject { described_class.validate!(valid_dashboard, dashboard_path: 'test/path.yml', project: project) } + + context 'with no conflicting metric identifiers in db' do + it { is_expected.to be true } + end + + context 'with metric identifier present in current dashboard' do + before do + create(:prometheus_metric, + identifier: 'metric_a1', + dashboard_path: 'test/path.yml', + project: project + ) + end + + it { is_expected.to be true } + end + + context 'with metric identifier present in another dashboard' do + before do + create(:prometheus_metric, + identifier: 'metric_a1', + dashboard_path: 'some/other/dashboard/path.yml', + project: project + ) + end + + it_behaves_like 'validation failed', 'metric_id must be unique across a project' + end + end + end + + context 'invalid dashboard schema' do + subject { described_class.validate!(invalid_dashboard) } + + context 'wrong property type' do + it_behaves_like 'validation failed', "'this_should_be_a_int' at /panel_groups/0/panels/0/weight is not of type: number" + end + + context 'panel groups missing' do + let_it_be(:invalid_dashboard) { load_dashboard_yaml(fixture_file('lib/gitlab/metrics/dashboard/dashboard_missing_panel_groups.yml')) } + + it_behaves_like 'validation failed', 'root is missing required keys: panel_groups' + end + + context 'groups are missing panels and group keys' do + let_it_be(:invalid_dashboard) { load_dashboard_yaml(fixture_file('lib/gitlab/metrics/dashboard/dashboard_groups_missing_panels_and_group.yml')) } + + it_behaves_like 'validation failed', '/panel_groups/0 is missing required keys: group' + end + + context 'panel is missing metrics key' do + let_it_be(:invalid_dashboard) { load_dashboard_yaml(fixture_file('lib/gitlab/metrics/dashboard/dashboard_panel_is_missing_metrics.yml')) } + + it_behaves_like 'validation failed', '/panel_groups/0/panels/0 is missing required keys: metrics' + end + end + end +end diff --git a/spec/lib/gitlab/metrics/elasticsearch_rack_middleware_spec.rb b/spec/lib/gitlab/metrics/elasticsearch_rack_middleware_spec.rb index 1fbd41bcc88..78b73f148e4 100644 --- a/spec/lib/gitlab/metrics/elasticsearch_rack_middleware_spec.rb +++ b/spec/lib/gitlab/metrics/elasticsearch_rack_middleware_spec.rb @@ -9,8 +9,6 @@ RSpec.describe Gitlab::Metrics::ElasticsearchRackMiddleware do let(:transaction) { Gitlab::Metrics::WebTransaction.new(env) } describe '#call' do - let(:counter) { instance_double(Prometheus::Client::Counter, increment: nil) } - let(:histogram) { instance_double(Prometheus::Client::Histogram, observe: nil) } let(:elasticsearch_query_time) { 0.1 } let(:elasticsearch_requests_count) { 2 } @@ -18,19 +16,6 @@ RSpec.describe Gitlab::Metrics::ElasticsearchRackMiddleware do allow(Gitlab::Instrumentation::ElasticsearchTransport).to receive(:query_time) { elasticsearch_query_time } allow(Gitlab::Instrumentation::ElasticsearchTransport).to receive(:get_request_count) { elasticsearch_requests_count } - allow(Gitlab::Metrics).to receive(:counter) - .with(:http_elasticsearch_requests_total, - an_instance_of(String), - Gitlab::Metrics::Transaction::BASE_LABELS) - .and_return(counter) - - allow(Gitlab::Metrics).to receive(:histogram) - .with(:http_elasticsearch_requests_duration_seconds, - an_instance_of(String), - Gitlab::Metrics::Transaction::BASE_LABELS, - described_class::HISTOGRAM_BUCKETS) - .and_return(histogram) - allow(Gitlab::Metrics).to receive(:current_transaction).and_return(transaction) end @@ -39,19 +24,30 @@ RSpec.describe Gitlab::Metrics::ElasticsearchRackMiddleware do end it 'records elasticsearch metrics' do - expect(counter).to receive(:increment).with(transaction.labels, elasticsearch_requests_count) - expect(histogram).to receive(:observe).with(transaction.labels, elasticsearch_query_time) + expect(transaction).to receive(:increment).with(:http_elasticsearch_requests_total, elasticsearch_requests_count) + expect(transaction).to receive(:observe).with(:http_elasticsearch_requests_duration_seconds, elasticsearch_query_time) middleware.call(env) end it 'records elasticsearch metrics if an error is raised' do - expect(counter).to receive(:increment).with(transaction.labels, elasticsearch_requests_count) - expect(histogram).to receive(:observe).with(transaction.labels, elasticsearch_query_time) + expect(transaction).to receive(:increment).with(:http_elasticsearch_requests_total, elasticsearch_requests_count) + expect(transaction).to receive(:observe).with(:http_elasticsearch_requests_duration_seconds, elasticsearch_query_time) allow(app).to receive(:call).with(env).and_raise(StandardError) expect { middleware.call(env) }.to raise_error(StandardError) end + + context 'when there are no elasticsearch requests' do + let(:elasticsearch_requests_count) { 0 } + + it 'does not record any metrics' do + expect(transaction).not_to receive(:observe).with(:http_elasticsearch_requests_duration_seconds) + expect(transaction).not_to receive(:increment).with(:http_elasticsearch_requests_total, 0) + + middleware.call(env) + end + end end end diff --git a/spec/lib/gitlab/metrics/method_call_spec.rb b/spec/lib/gitlab/metrics/method_call_spec.rb index 42361cbc36a..825c91b6cb4 100644 --- a/spec/lib/gitlab/metrics/method_call_spec.rb +++ b/spec/lib/gitlab/metrics/method_call_spec.rb @@ -3,12 +3,12 @@ require 'spec_helper' RSpec.describe Gitlab::Metrics::MethodCall do - let(:transaction) { double(:transaction, labels: {}) } + let(:transaction) { Gitlab::Metrics::WebTransaction.new({}) } let(:method_call) { described_class.new('Foo#bar', :Foo, '#bar', transaction) } describe '#measure' do after do - described_class.reload_metric!(:gitlab_method_call_duration_seconds) + ::Gitlab::Metrics::Transaction.reload_metric!(:gitlab_method_call_duration_seconds) end it 'measures the performance of the supplied block' do @@ -36,13 +36,13 @@ RSpec.describe Gitlab::Metrics::MethodCall do end it 'metric is not a NullMetric' do - expect(described_class).not_to be_instance_of(Gitlab::Metrics::NullMetric) + method_call.measure { 'foo' } + expect(::Gitlab::Metrics::Transaction.prometheus_metric(:gitlab_method_call_duration_seconds, :histogram)).not_to be_instance_of(Gitlab::Metrics::NullMetric) end it 'observes the performance of the supplied block' do - expect(described_class.gitlab_method_call_duration_seconds) - .to receive(:observe) - .with({ module: :Foo, method: '#bar' }, be_a_kind_of(Numeric)) + expect(transaction) + .to receive(:observe).with(:gitlab_method_call_duration_seconds, be_a_kind_of(Numeric), { method: "#bar", module: :Foo }) method_call.measure { 'foo' } end @@ -53,11 +53,17 @@ RSpec.describe Gitlab::Metrics::MethodCall do stub_feature_flags(prometheus_metrics_method_instrumentation: false) end - it 'observes using NullMetric' do - expect(described_class.gitlab_method_call_duration_seconds).to be_instance_of(Gitlab::Metrics::NullMetric) - expect(described_class.gitlab_method_call_duration_seconds).to receive(:observe) + it 'observes the performance of the supplied block' do + expect(transaction) + .to receive(:observe).with(:gitlab_method_call_duration_seconds, be_a_kind_of(Numeric), { method: "#bar", module: :Foo }) + + method_call.measure { 'foo' } + end + it 'observes using NullMetric' do method_call.measure { 'foo' } + + expect(::Gitlab::Metrics::Transaction.prometheus_metric(:gitlab_method_call_duration_seconds, :histogram)).to be_instance_of(Gitlab::Metrics::NullMetric) end end end @@ -68,8 +74,9 @@ RSpec.describe Gitlab::Metrics::MethodCall do end it 'does not observe the performance' do - expect(described_class.gitlab_method_call_duration_seconds) + expect(transaction) .not_to receive(:observe) + .with(:gitlab_method_call_duration_seconds, be_a_kind_of(Numeric)) method_call.measure { 'foo' } end diff --git a/spec/lib/gitlab/metrics/methods_spec.rb b/spec/lib/gitlab/metrics/methods_spec.rb index 3c171680272..71135a6e9c5 100644 --- a/spec/lib/gitlab/metrics/methods_spec.rb +++ b/spec/lib/gitlab/metrics/methods_spec.rb @@ -9,9 +9,9 @@ RSpec.describe Gitlab::Metrics::Methods do let(:docstring) { 'description' } let(:metric_name) { :sample_metric } - describe "#define_#{metric_type}" do + describe "#define_metrics" do define_method(:call_define_metric_method) do |**args| - subject.__send__("define_#{metric_type}", metric_name, **args) + subject.__send__(:define_metric, metric_type, metric_name, **args) end context 'metrics access method not defined' do @@ -55,11 +55,11 @@ RSpec.describe Gitlab::Metrics::Methods do end end - describe "#fetch_#{metric_type}" do + describe "#fetch_metric" do let(:null_metric) { Gitlab::Metrics::NullMetric.instance } define_method(:call_fetch_metric_method) do |**args| - subject.__send__("fetch_#{metric_type}", metric_name, **args) + subject.__send__(:fetch_metric, metric_type, metric_name, **args) end context "when #{metric_type} is not cached" do @@ -135,5 +135,5 @@ RSpec.describe Gitlab::Metrics::Methods do include_examples 'metric', :counter, {} include_examples 'metric', :gauge, {}, :all - include_examples 'metric', :histogram, {}, [0.005, 0.01, 0.1, 1, 10] + include_examples 'metric', :histogram, {}, ::Prometheus::Client::Histogram::DEFAULT_BUCKETS end diff --git a/spec/lib/gitlab/metrics/rack_middleware_spec.rb b/spec/lib/gitlab/metrics/rack_middleware_spec.rb index 335e5a490a6..ab56f38f0c1 100644 --- a/spec/lib/gitlab/metrics/rack_middleware_spec.rb +++ b/spec/lib/gitlab/metrics/rack_middleware_spec.rb @@ -25,12 +25,4 @@ RSpec.describe Gitlab::Metrics::RackMiddleware do expect { middleware.call(env) }.to raise_error(RuntimeError) end end - - describe '#transaction_from_env' do - let(:transaction) { middleware.transaction_from_env(env) } - - it 'returns a Transaction' do - expect(transaction).to be_an_instance_of(Gitlab::Metrics::WebTransaction) - end - end end diff --git a/spec/lib/gitlab/metrics/redis_rack_middleware_spec.rb b/spec/lib/gitlab/metrics/redis_rack_middleware_spec.rb deleted file mode 100644 index a85968dbd43..00000000000 --- a/spec/lib/gitlab/metrics/redis_rack_middleware_spec.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Metrics::RedisRackMiddleware do - let(:app) { double(:app) } - let(:middleware) { described_class.new(app) } - let(:env) { {} } - let(:transaction) { Gitlab::Metrics::WebTransaction.new(env) } - - before do - allow(app).to receive(:call).with(env).and_return('wub wub') - end - - describe '#call' do - let(:counter) { double(Prometheus::Client::Counter, increment: nil) } - let(:histogram) { double(Prometheus::Client::Histogram, observe: nil) } - let(:redis_query_time) { 0.1 } - let(:redis_requests_count) { 2 } - - before do - allow(Gitlab::Instrumentation::Redis).to receive(:query_time) { redis_query_time } - allow(Gitlab::Instrumentation::Redis).to receive(:get_request_count) { redis_requests_count } - - allow(Gitlab::Metrics).to receive(:counter) - .with(:http_redis_requests_total, - an_instance_of(String), - Gitlab::Metrics::Transaction::BASE_LABELS) - .and_return(counter) - - allow(Gitlab::Metrics).to receive(:histogram) - .with(:http_redis_requests_duration_seconds, - an_instance_of(String), - Gitlab::Metrics::Transaction::BASE_LABELS, - Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS) - .and_return(histogram) - - allow(Gitlab::Metrics).to receive(:current_transaction).and_return(transaction) - end - - it 'calls the app' do - expect(middleware.call(env)).to eq('wub wub') - end - - it 'records redis metrics' do - expect(counter).to receive(:increment).with(transaction.labels, redis_requests_count) - expect(histogram).to receive(:observe).with(transaction.labels, redis_query_time) - - middleware.call(env) - end - - it 'records redis metrics if an error is raised' do - expect(counter).to receive(:increment).with(transaction.labels, redis_requests_count) - expect(histogram).to receive(:observe).with(transaction.labels, redis_query_time) - - allow(app).to receive(:call).with(env).and_raise(StandardError) - - expect { middleware.call(env) }.to raise_error(StandardError) - end - end -end diff --git a/spec/lib/gitlab/metrics/samplers/threads_sampler_spec.rb b/spec/lib/gitlab/metrics/samplers/threads_sampler_spec.rb new file mode 100644 index 00000000000..19477589289 --- /dev/null +++ b/spec/lib/gitlab/metrics/samplers/threads_sampler_spec.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Metrics::Samplers::ThreadsSampler do + subject { described_class.new } + + describe '#interval' do + it 'samples every five seconds by default' do + expect(subject.interval).to eq(5) + end + + it 'samples at other intervals if requested' do + expect(described_class.new(11).interval).to eq(11) + end + end + + describe '#sample' do + before do + described_class::METRIC_DESCRIPTIONS.each_key do |metric| + allow(subject.metrics[metric]).to receive(:set) + end + end + + it 'sets the gauge for the concurrency total' do + expect(Gitlab::Runtime).to receive(:max_threads).and_return(9000) + expect(subject.metrics[:max_expected_threads]).to receive(:set).with({}, 9000) + + subject.sample + end + + context 'thread counts' do + it 'reports if any of the threads per group uses the db' do + threads = [ + fake_thread(described_class::SIDEKIQ_WORKER_THREAD_NAME, true), fake_thread(described_class::SIDEKIQ_WORKER_THREAD_NAME, false), + fake_thread(described_class::SIDEKIQ_WORKER_THREAD_NAME, nil) + ] + allow(Thread).to receive(:list).and_return(threads) + + expect(subject.metrics[:running_threads]).to receive(:set) + .with({ uses_db_connection: 'yes', thread_name: described_class::SIDEKIQ_WORKER_THREAD_NAME }, 1) + expect(subject.metrics[:running_threads]).to receive(:set) + .with({ uses_db_connection: 'no', thread_name: described_class::SIDEKIQ_WORKER_THREAD_NAME }, 2) + + subject.sample + end + + context 'thread names', :aggregate_failures do + where(:thread_names, :expected_names) do + [ + [[nil], %w(unnamed)], + [['puma threadpool 1', 'puma threadpool 001', 'puma threadpool 002'], ['puma threadpool']], + [%w(sidekiq_worker_thread), %w(sidekiq_worker_thread)], + [%w(some_sampler some_exporter), %w(some_sampler some_exporter)], + [%w(unknown thing), %w(unrecognized)] + ] + end + + with_them do + it do + allow(Thread).to receive(:list).and_return(thread_names.map { |name| fake_thread(name) }) + + expected_names.each do |expected_name| + expect(subject.metrics[:running_threads]).to receive(:set) + .with({ uses_db_connection: 'yes', thread_name: expected_name }, instance_of(Integer)) + expect(subject.metrics[:running_threads]).to receive(:set) + .with({ uses_db_connection: 'no', thread_name: expected_name }, instance_of(Integer)) + end + + subject.sample + end + end + end + end + + def fake_thread(name = nil, db_connection = nil) + thready = { uses_db_connection: db_connection } + allow(thready).to receive(:name).and_return(name) + + thready + end + end +end diff --git a/spec/lib/gitlab/metrics/sidekiq_middleware_spec.rb b/spec/lib/gitlab/metrics/sidekiq_middleware_spec.rb index c66d8b1075c..047d1e5d205 100644 --- a/spec/lib/gitlab/metrics/sidekiq_middleware_spec.rb +++ b/spec/lib/gitlab/metrics/sidekiq_middleware_spec.rb @@ -11,8 +11,8 @@ RSpec.describe Gitlab::Metrics::SidekiqMiddleware do worker = double(:worker, class: double(:class, name: 'TestWorker')) expect_next_instance_of(Gitlab::Metrics::BackgroundTransaction) do |transaction| - expect(transaction).to receive(:set).with(:sidekiq_queue_duration, instance_of(Float)) - expect(transaction).to receive(:increment).with(:db_count, 1) + expect(transaction).to receive(:set).with(:gitlab_transaction_sidekiq_queue_duration_total, instance_of(Float)) + expect(transaction).to receive(:increment).with(:gitlab_transaction_db_count_total, 1) end middleware.call(worker, message, :test) do @@ -42,7 +42,7 @@ RSpec.describe Gitlab::Metrics::SidekiqMiddleware do .and_call_original expect_any_instance_of(Gitlab::Metrics::Transaction).to receive(:set) - .with(:sidekiq_queue_duration, instance_of(Float)) + .with(:gitlab_transaction_sidekiq_queue_duration_total, instance_of(Float)) middleware.call(worker, {}, :test) { nil } end diff --git a/spec/lib/gitlab/metrics/subscribers/action_view_spec.rb b/spec/lib/gitlab/metrics/subscribers/action_view_spec.rb index 161527c01aa..adbc474343f 100644 --- a/spec/lib/gitlab/metrics/subscribers/action_view_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/action_view_spec.rb @@ -22,15 +22,15 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActionView do describe '#render_template' do it 'tracks rendering of a template' do expect(transaction).to receive(:increment) - .with(:view_duration, 2.1) + .with(:gitlab_transaction_view_duration_total, 2.1) subscriber.render_template(event) end it 'observes view rendering time' do - expect(described_class.gitlab_view_rendering_duration_seconds) + expect(transaction) .to receive(:observe) - .with({ view: 'app/views/x.html.haml' }, 2.1) + .with(:gitlab_view_rendering_duration_seconds, 2.1, { view: "app/views/x.html.haml" }) subscriber.render_template(event) end diff --git a/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb b/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb index 2fd5dd1d83b..a31686b8061 100644 --- a/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb @@ -37,10 +37,11 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActiveRecord do it 'increments only db count value' do described_class::DB_COUNTERS.each do |counter| + prometheus_counter = "gitlab_transaction_#{counter}_total".to_sym if expected_counters[counter] > 0 - expect(transaction).to receive(:increment).with(counter, 1) + expect(transaction).to receive(:increment).with(prometheus_counter, 1) else - expect(transaction).not_to receive(:increment).with(counter, 1) + expect(transaction).not_to receive(:increment).with(prometheus_counter, 1) end end @@ -74,10 +75,18 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActiveRecord do expect(subscriber).to receive(:current_transaction) .at_least(:once) .and_return(transaction) - expect(described_class.send(:gitlab_sql_duration_seconds)).to receive(:observe).with({}, 0.002) + expect(transaction).to receive(:observe).with(:gitlab_sql_duration_seconds, 0.002) + subscriber.sql(event) end + it 'marks the current thread as using the database' do + # since it would already have been toggled by other specs + Thread.current[:uses_db_connection] = nil + + expect { subscriber.sql(event) }.to change { Thread.current[:uses_db_connection] }.from(nil).to(true) + end + context 'with read query' do let(:expected_counters) do { @@ -217,7 +226,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActiveRecord do end it 'skips schema/begin/commit sql commands' do - expect(subscriber).to receive(:current_transaction) + allow(subscriber).to receive(:current_transaction) .at_least(:once) .and_return(transaction) diff --git a/spec/lib/gitlab/metrics/subscribers/rails_cache_spec.rb b/spec/lib/gitlab/metrics/subscribers/rails_cache_spec.rb index f7ac719c16a..9aba6ac293c 100644 --- a/spec/lib/gitlab/metrics/subscribers/rails_cache_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/rails_cache_spec.rb @@ -26,21 +26,12 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do context 'with hit event' do let(:event) { double(:event, duration: 15.2, payload: { hit: true }) } - it 'increments the cache_read_hit count' do - expect(transaction).to receive(:increment) - .with(:cache_read_hit_count, 1, false) - expect(transaction).to receive(:increment) - .with(any_args).at_least(1) # Other calls - - subscriber.cache_read(event) - end - context 'when super operation is fetch' do let(:event) { double(:event, duration: 15.2, payload: { hit: true, super_operation: :fetch }) } - it 'does not increment cache read miss' do + it 'does not increment cache read miss total' do expect(transaction).not_to receive(:increment) - .with(:cache_read_hit_count, 1) + .with(:gitlab_cache_misses_total, 1) subscriber.cache_read(event) end @@ -50,33 +41,21 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do context 'with miss event' do let(:event) { double(:event, duration: 15.2, payload: { hit: false }) } - it 'increments the cache_read_miss count' do + it 'increments the cache_read_miss total' do expect(transaction).to receive(:increment) - .with(:cache_read_miss_count, 1, false) + .with(:gitlab_cache_misses_total, 1) expect(transaction).to receive(:increment) .with(any_args).at_least(1) # Other calls subscriber.cache_read(event) end - it 'increments the cache_read_miss total' do - expect(subscriber.send(:metric_cache_misses_total)).to receive(:increment).with({}) - - subscriber.cache_read(event) - end - context 'when super operation is fetch' do let(:event) { double(:event, duration: 15.2, payload: { hit: false, super_operation: :fetch }) } - it 'does not increment cache read miss' do + it 'does not increment cache read miss total' do expect(transaction).not_to receive(:increment) - .with(:cache_read_miss_count, 1) - - subscriber.cache_read(event) - end - - it 'does not increment cache_read_miss total' do - expect(subscriber.send(:metric_cache_misses_total)).not_to receive(:increment).with({}) + .with(:gitlab_cache_misses_total, 1) subscriber.cache_read(event) end @@ -129,7 +108,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do it 'increments the cache_read_hit count' do expect(transaction).to receive(:increment) - .with(:cache_read_hit_count, 1) + .with(:gitlab_transaction_cache_read_hit_count_total, 1) subscriber.cache_fetch_hit(event) end @@ -146,25 +125,17 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do end context 'with a transaction' do - let(:metric_cache_misses_total) { double('metric_cache_misses_total', increment: nil) } - before do - allow(subscriber).to receive(:metric_cache_misses_total).and_return(metric_cache_misses_total) allow(subscriber).to receive(:current_transaction) .and_return(transaction) end - it 'increments the cache_fetch_miss count' do + it 'increments the cache_fetch_miss count and cache_read_miss total' do + expect(transaction).to receive(:increment).with(:gitlab_cache_misses_total, 1) expect(transaction).to receive(:increment) - .with(:cache_read_miss_count, 1) - - subscriber.cache_generate(event) - end + .with(:gitlab_transaction_cache_read_miss_count_total, 1) - it 'increments the cache_read_miss total' do subscriber.cache_generate(event) - - expect(metric_cache_misses_total).to have_received(:increment).with({}) end end end @@ -184,22 +155,6 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do .and_return(transaction) end - it 'increments the total and specific cache duration' do - expect(transaction).to receive(:increment) - .with(:cache_duration, event.duration, false) - - expect(transaction).to receive(:increment) - .with(:cache_count, 1, false) - - expect(transaction).to receive(:increment) - .with(:cache_delete_duration, event.duration, false) - - expect(transaction).to receive(:increment) - .with(:cache_delete_count, 1, false) - - subscriber.observe(:delete, event.duration) - end - it 'observes cache metric' do expect(subscriber.send(:metric_cache_operation_duration_seconds)) .to receive(:observe) @@ -209,9 +164,9 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do end it 'increments the operations total' do - expect(subscriber.send(:metric_cache_operations_total)) + expect(transaction) .to receive(:increment) - .with(transaction.labels.merge(operation: :delete)) + .with(:gitlab_cache_operations_total, 1, { operation: :delete }) subscriber.observe(:delete, event.duration) end diff --git a/spec/lib/gitlab/metrics/transaction_spec.rb b/spec/lib/gitlab/metrics/transaction_spec.rb index e64179bd5c1..88293f11149 100644 --- a/spec/lib/gitlab/metrics/transaction_spec.rb +++ b/spec/lib/gitlab/metrics/transaction_spec.rb @@ -28,14 +28,6 @@ RSpec.describe Gitlab::Metrics::Transaction do end end - describe '#allocated_memory' do - it 'returns the allocated memory in bytes' do - transaction.run { 'a' * 32 } - - expect(transaction.allocated_memory).to be_a_kind_of(Numeric) - end - end - describe '#run' do it 'yields the supplied block' do expect { |b| transaction.run(&b) }.to yield_control @@ -63,7 +55,7 @@ RSpec.describe Gitlab::Metrics::Transaction do end describe '#add_event' do - let(:prometheus_metric) { instance_double(Prometheus::Client::Counter, increment: nil) } + let(:prometheus_metric) { instance_double(Prometheus::Client::Counter, increment: nil, base_labels: {}) } it 'adds a metric' do expect(prometheus_metric).to receive(:increment) @@ -82,7 +74,7 @@ RSpec.describe Gitlab::Metrics::Transaction do context 'with sensitive tags' do before do transaction.add_event(:baubau, **sensitive_tags.merge(sane: 'yes')) - allow(described_class).to receive(:transaction_metric).and_return(prometheus_metric) + allow(described_class).to receive(:prometheus_metric).and_return(prometheus_metric) end it 'filters tags' do @@ -94,24 +86,119 @@ RSpec.describe Gitlab::Metrics::Transaction do end describe '#increment' do - let(:prometheus_metric) { instance_double(Prometheus::Client::Counter, increment: nil) } + let(:prometheus_metric) { instance_double(Prometheus::Client::Counter, increment: nil, base_labels: {}) } it 'adds a metric' do - expect(prometheus_metric).to receive(:increment).with(hash_including(:action, :controller), 1) - expect(described_class).to receive(:fetch_metric).with(:counter, :gitlab_transaction_meow_total).and_return(prometheus_metric) + expect(prometheus_metric).to receive(:increment) + expect(::Gitlab::Metrics).to receive(:counter).with(:meow, 'Meow counter', hash_including(:controller, :action)).and_return(prometheus_metric) transaction.increment(:meow, 1) end + + context 'with block' do + it 'overrides docstring' do + expect(::Gitlab::Metrics).to receive(:counter).with(:block_docstring, 'test', hash_including(:controller, :action)).and_return(prometheus_metric) + + transaction.increment(:block_docstring, 1) do + docstring 'test' + end + end + + it 'overrides labels' do + expect(::Gitlab::Metrics).to receive(:counter).with(:block_labels, 'Block labels counter', hash_including(:controller, :action, :sane)).and_return(prometheus_metric) + + labels = { sane: 'yes' } + transaction.increment(:block_labels, 1, labels) do + label_keys %i(sane) + end + end + + it 'filters sensitive tags' do + expect(::Gitlab::Metrics).to receive(:counter).with(:metric_with_sensitive_block, 'Metric with sensitive block counter', hash_excluding(sensitive_tags)).and_return(prometheus_metric) + + labels_keys = sensitive_tags.keys + transaction.increment(:metric_with_sensitive_block, 1, sensitive_tags) do + label_keys labels_keys + end + end + end end describe '#set' do - let(:prometheus_metric) { instance_double(Prometheus::Client::Gauge, set: nil) } + let(:prometheus_metric) { instance_double(Prometheus::Client::Gauge, set: nil, base_labels: {}) } it 'adds a metric' do - expect(prometheus_metric).to receive(:set).with(hash_including(:action, :controller), 1) - expect(described_class).to receive(:fetch_metric).with(:gauge, :gitlab_transaction_meow_total).and_return(prometheus_metric) + expect(prometheus_metric).to receive(:set) + expect(::Gitlab::Metrics).to receive(:gauge).with(:meow_set, 'Meow set gauge', hash_including(:controller, :action), :all).and_return(prometheus_metric) + + transaction.set(:meow_set, 1) + end + + context 'with block' do + it 'overrides docstring' do + expect(::Gitlab::Metrics).to receive(:gauge).with(:block_docstring_set, 'test', hash_including(:controller, :action), :all).and_return(prometheus_metric) + + transaction.set(:block_docstring_set, 1) do + docstring 'test' + end + end + + it 'overrides labels' do + expect(::Gitlab::Metrics).to receive(:gauge).with(:block_labels_set, 'Block labels set gauge', hash_including(:controller, :action, :sane), :all).and_return(prometheus_metric) - transaction.set(:meow, 1) + labels = { sane: 'yes' } + transaction.set(:block_labels_set, 1, labels) do + label_keys %i(sane) + end + end + + it 'filters sensitive tags' do + expect(::Gitlab::Metrics).to receive(:gauge).with(:metric_set_with_sensitive_block, 'Metric set with sensitive block gauge', hash_excluding(sensitive_tags), :all).and_return(prometheus_metric) + + label_keys = sensitive_tags.keys + transaction.set(:metric_set_with_sensitive_block, 1, sensitive_tags) do + label_keys label_keys + end + end + end + end + + describe '#observe' do + let(:prometheus_metric) { instance_double(Prometheus::Client::Histogram, observe: nil, base_labels: {}) } + + it 'adds a metric' do + expect(prometheus_metric).to receive(:observe) + expect(::Gitlab::Metrics).to receive(:histogram).with(:meow_observe, 'Meow observe histogram', hash_including(:controller, :action), kind_of(Array)).and_return(prometheus_metric) + + transaction.observe(:meow_observe, 1) + end + + context 'with block' do + it 'overrides docstring' do + expect(::Gitlab::Metrics).to receive(:histogram).with(:block_docstring_observe, 'test', hash_including(:controller, :action), kind_of(Array)).and_return(prometheus_metric) + + transaction.observe(:block_docstring_observe, 1) do + docstring 'test' + end + end + + it 'overrides labels' do + expect(::Gitlab::Metrics).to receive(:histogram).with(:block_labels_observe, 'Block labels observe histogram', hash_including(:controller, :action, :sane), kind_of(Array)).and_return(prometheus_metric) + + labels = { sane: 'yes' } + transaction.observe(:block_labels_observe, 1, labels) do + label_keys %i(sane) + end + end + + it 'filters sensitive tags' do + expect(::Gitlab::Metrics).to receive(:histogram).with(:metric_observe_with_sensitive_block, 'Metric observe with sensitive block histogram', hash_excluding(sensitive_tags), kind_of(Array)).and_return(prometheus_metric) + + label_keys = sensitive_tags.keys + transaction.observe(:metric_observe_with_sensitive_block, 1, sensitive_tags) do + label_keys label_keys + end + end end end end diff --git a/spec/lib/gitlab/metrics/web_transaction_spec.rb b/spec/lib/gitlab/metrics/web_transaction_spec.rb index 12e98089066..6903ce53f65 100644 --- a/spec/lib/gitlab/metrics/web_transaction_spec.rb +++ b/spec/lib/gitlab/metrics/web_transaction_spec.rb @@ -5,29 +5,52 @@ require 'spec_helper' RSpec.describe Gitlab::Metrics::WebTransaction do let(:env) { {} } let(:transaction) { described_class.new(env) } - let(:prometheus_metric) { double("prometheus metric") } + let(:prometheus_metric) { instance_double(Prometheus::Client::Metric, base_labels: {}) } before do - allow(described_class).to receive(:transaction_metric).and_return(prometheus_metric) + allow(described_class).to receive(:prometheus_metric).and_return(prometheus_metric) end - describe '#duration' do - it 'returns the duration of a transaction in seconds' do - transaction.run { sleep(0.5) } + RSpec.shared_context 'ActionController request' do + let(:request) { double(:request, format: double(:format, ref: :html)) } + let(:controller_class) { double(:controller_class, name: 'TestController') } - expect(transaction.duration).to be >= 0.5 + before do + controller = double(:controller, class: controller_class, action_name: 'show', request: request) + env['action_controller.instance'] = controller + end + end + + RSpec.shared_context 'transaction observe metrics' do + before do + allow(transaction).to receive(:observe) + end + end + + RSpec.shared_examples 'metric with labels' do |metric_method| + include_context 'ActionController request' + + it 'measures with correct labels and value' do + value = 1 + expect(prometheus_metric).to receive(metric_method).with({ controller: 'TestController', action: 'show', feature_category: '' }, value) + + transaction.send(metric_method, :bau, value) end end - describe '#allocated_memory' do - it 'returns the allocated memory in bytes' do - transaction.run { 'a' * 32 } + describe '#duration' do + include_context 'transaction observe metrics' + + it 'returns the duration of a transaction in seconds' do + transaction.run { sleep(0.5) } - expect(transaction.allocated_memory).to be_a_kind_of(Numeric) + expect(transaction.duration).to be >= 0.5 end end describe '#run' do + include_context 'transaction observe metrics' + it 'yields the supplied block' do expect { |b| transaction.run(&b) }.to yield_control end @@ -53,26 +76,7 @@ RSpec.describe Gitlab::Metrics::WebTransaction do end end - describe '#increment' do - it 'increments a counter' do - expect(prometheus_metric).to receive(:increment).with({}, 1) - - transaction.increment(:time, 1) - end - end - - describe '#set' do - it 'sets a value' do - expect(prometheus_metric).to receive(:set).with({}, 10) - - transaction.set(:number, 10) - end - end - describe '#labels' do - let(:request) { double(:request, format: double(:format, ref: :html)) } - let(:controller_class) { double(:controller_class, name: 'TestController') } - context 'when request goes to Grape endpoint' do before do route = double(:route, request_method: 'GET', path: '/:version/projects/:id/archive(.:format)') @@ -86,7 +90,7 @@ RSpec.describe Gitlab::Metrics::WebTransaction do end it 'contains only the labels defined for transactions' do - expect(transaction.labels.keys).to contain_exactly(*described_class.superclass::BASE_LABELS.keys) + expect(transaction.labels.keys).to contain_exactly(*described_class.superclass::BASE_LABEL_KEYS) end it 'does not provide labels if route infos are missing' do @@ -100,18 +104,14 @@ RSpec.describe Gitlab::Metrics::WebTransaction do end context 'when request goes to ActionController' do - before do - controller = double(:controller, class: controller_class, action_name: 'show', request: request) - - env['action_controller.instance'] = controller - end + include_context 'ActionController request' it 'tags a transaction with the name and action of a controller' do expect(transaction.labels).to eq({ controller: 'TestController', action: 'show', feature_category: '' }) end it 'contains only the labels defined for transactions' do - expect(transaction.labels.keys).to contain_exactly(*described_class.superclass::BASE_LABELS.keys) + expect(transaction.labels.keys).to contain_exactly(*described_class.superclass::BASE_LABEL_KEYS) end context 'when the request content type is not :html' do @@ -144,6 +144,8 @@ RSpec.describe Gitlab::Metrics::WebTransaction do end describe '#add_event' do + let(:prometheus_metric) { instance_double(Prometheus::Client::Counter, :increment, base_labels: {}) } + it 'adds a metric' do expect(prometheus_metric).to receive(:increment) @@ -156,4 +158,22 @@ RSpec.describe Gitlab::Metrics::WebTransaction do transaction.add_event(:bau, animal: 'dog') end end + + describe '#increment' do + let(:prometheus_metric) { instance_double(Prometheus::Client::Counter, :increment, base_labels: {}) } + + it_behaves_like 'metric with labels', :increment + end + + describe '#set' do + let(:prometheus_metric) { instance_double(Prometheus::Client::Gauge, :set, base_labels: {}) } + + it_behaves_like 'metric with labels', :set + end + + describe '#observe' do + let(:prometheus_metric) { instance_double(Prometheus::Client::Histogram, :observe, base_labels: {}) } + + it_behaves_like 'metric with labels', :observe + end end diff --git a/spec/lib/gitlab/metrics_spec.rb b/spec/lib/gitlab/metrics_spec.rb index bdf72a3c288..db5a23e2328 100644 --- a/spec/lib/gitlab/metrics_spec.rb +++ b/spec/lib/gitlab/metrics_spec.rb @@ -71,14 +71,9 @@ RSpec.describe Gitlab::Metrics do end it 'adds a metric to the current transaction' do - expect(transaction).to receive(:increment) - .with('foo_real_time', a_kind_of(Numeric), false) + expect(transaction).to receive(:observe).with(:gitlab_foo_real_duration_seconds, a_kind_of(Numeric)) - expect(transaction).to receive(:increment) - .with('foo_cpu_time', a_kind_of(Numeric), false) - - expect(transaction).to receive(:increment) - .with('foo_call_count', 1, false) + expect(transaction).to receive(:observe).with(:gitlab_foo_cpu_duration_seconds, a_kind_of(Numeric)) described_class.measure(:foo) { 10 } end diff --git a/spec/lib/gitlab/middleware/rails_queue_duration_spec.rb b/spec/lib/gitlab/middleware/rails_queue_duration_spec.rb index 8f9b0aec9eb..cdb48024531 100644 --- a/spec/lib/gitlab/middleware/rails_queue_duration_spec.rb +++ b/spec/lib/gitlab/middleware/rails_queue_duration_spec.rb @@ -29,26 +29,19 @@ RSpec.describe Gitlab::Middleware::RailsQueueDuration do it 'sets proxy_flight_time and calls the app when the header is present' do env['HTTP_GITLAB_WORKHORSE_PROXY_START'] = '123' - expect(transaction).to receive(:set).with(:rails_queue_duration, an_instance_of(Float)) + expect(transaction).to receive(:set).with(:gitlab_transaction_rails_queue_duration_total, an_instance_of(Float)) expect(middleware.call(env)).to eq('yay') end it 'observes rails queue duration metrics and calls the app when the header is present' do env['HTTP_GITLAB_WORKHORSE_PROXY_START'] = '2000000000' - expect(middleware.send(:metric_rails_queue_duration_seconds)).to receive(:observe).with(transaction.labels, 1) + expect(transaction).to receive(:observe).with(:gitlab_rails_queue_duration_seconds, 1) Timecop.freeze(Time.at(3)) do expect(middleware.call(env)).to eq('yay') end end - - it 'creates a metric with a docstring' do - metric = middleware.send(:metric_rails_queue_duration_seconds) - - expect(metric).to be_instance_of(Prometheus::Client::Histogram) - expect(metric.docstring).to eq('Measures latency between GitLab Workhorse forwarding a request to Rails') - end end end end diff --git a/spec/lib/gitlab/middleware/read_only_spec.rb b/spec/lib/gitlab/middleware/read_only_spec.rb index 3bdf2a5077f..50dd38278b9 100644 --- a/spec/lib/gitlab/middleware/read_only_spec.rb +++ b/spec/lib/gitlab/middleware/read_only_spec.rb @@ -110,6 +110,19 @@ RSpec.describe Gitlab::Middleware::ReadOnly do expect(subject).not_to disallow_request end + context 'relative URL is configured' do + before do + stub_config_setting(relative_url_root: '/gitlab') + end + + it 'expects a graphql request to be allowed' do + response = request.post("/gitlab/api/graphql") + + expect(response).not_to be_redirect + expect(subject).not_to disallow_request + end + end + context 'sidekiq admin requests' do where(:mounted_at) do [ diff --git a/spec/lib/gitlab/pages/settings_spec.rb b/spec/lib/gitlab/pages/settings_spec.rb new file mode 100644 index 00000000000..7d4db073d73 --- /dev/null +++ b/spec/lib/gitlab/pages/settings_spec.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Pages::Settings do + describe '#path' do + subject { described_class.new(settings).path } + + let(:settings) { double(path: 'the path') } + + it { is_expected.to eq('the path') } + + it 'does not track calls' do + expect(::Gitlab::ErrorTracking).not_to receive(:track_exception) + + subject + end + + context 'when running under a web server' do + before do + allow(::Gitlab::Runtime).to receive(:web_server?).and_return(true) + end + + it { is_expected.to eq('the path') } + + it 'does not track calls' do + expect(::Gitlab::ErrorTracking).not_to receive(:track_exception) + + subject + end + + context 'with the env var' do + before do + stub_env('GITLAB_PAGES_DENY_DISK_ACCESS', '1') + end + + it { is_expected.to eq('the path') } + + it 'tracks a DiskAccessDenied exception' do + expect(::Gitlab::ErrorTracking).to receive(:track_exception) + .with(instance_of(described_class::DiskAccessDenied)).and_call_original + + subject + end + end + end + end +end diff --git a/spec/lib/gitlab/pagination/gitaly_keyset_pager_spec.rb b/spec/lib/gitlab/pagination/gitaly_keyset_pager_spec.rb new file mode 100644 index 00000000000..156a440833c --- /dev/null +++ b/spec/lib/gitlab/pagination/gitaly_keyset_pager_spec.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Pagination::GitalyKeysetPager do + let(:pager) { described_class.new(request_context, project) } + + let_it_be(:project) { create(:project, :repository) } + + let(:request_context) { double("request context") } + let(:finder) { double("branch finder") } + let(:custom_port) { 8080 } + let(:incoming_api_projects_url) { "#{Gitlab.config.gitlab.url}:#{custom_port}/api/v4/projects" } + + before do + stub_config_setting(port: custom_port) + end + + describe '.paginate' do + let(:base_query) { { per_page: 2 } } + let(:query) { base_query } + + before do + allow(request_context).to receive(:params).and_return(query) + allow(request_context).to receive(:header) + end + + shared_examples_for 'offset pagination' do + let(:paginated_array) { double 'paginated array' } + let(:branches) { [] } + + it 'uses offset pagination' do + expect(finder).to receive(:execute).and_return(branches) + expect(Kaminari).to receive(:paginate_array).with(branches).and_return(paginated_array) + expect_next_instance_of(Gitlab::Pagination::OffsetPagination) do |offset_pagination| + expect(offset_pagination).to receive(:paginate).with(paginated_array) + end + + pager.paginate(finder) + end + end + + context 'with branch_list_keyset_pagination feature off' do + before do + stub_feature_flags(branch_list_keyset_pagination: false) + end + + context 'without keyset pagination option' do + it_behaves_like 'offset pagination' + end + + context 'with keyset pagination option' do + let(:query) { base_query.merge(pagination: 'keyset') } + + it_behaves_like 'offset pagination' + end + end + + context 'with branch_list_keyset_pagination feature on' do + before do + stub_feature_flags(branch_list_keyset_pagination: project) + end + + context 'without keyset pagination option' do + it_behaves_like 'offset pagination' + end + + context 'with keyset pagination option' do + let(:query) { base_query.merge(pagination: 'keyset') } + let(:fake_request) { double(url: "#{incoming_api_projects_url}?#{query.to_query}") } + + before do + allow(request_context).to receive(:request).and_return(fake_request) + expect(finder).to receive(:execute).with(gitaly_pagination: true).and_return(branches) + end + + context 'when next page could be available' do + let(:branch1) { double 'branch', name: 'branch1' } + let(:branch2) { double 'branch', name: 'branch2' } + let(:branches) { [branch1, branch2] } + + let(:expected_next_page_link) { %Q(<#{incoming_api_projects_url}?#{query.merge(page_token: branch2.name).to_query}>; rel="next") } + + it 'uses keyset pagination and adds link headers' do + expect(request_context).to receive(:header).with('Links', expected_next_page_link) + expect(request_context).to receive(:header).with('Link', expected_next_page_link) + + pager.paginate(finder) + end + end + + context 'when the current page is the last page' do + let(:branch1) { double 'branch', name: 'branch1' } + let(:branches) { [branch1] } + + it 'uses keyset pagination without link headers' do + expect(request_context).not_to receive(:header).with('Links', anything) + expect(request_context).not_to receive(:header).with('Link', anything) + + pager.paginate(finder) + end + end + end + end + end +end diff --git a/spec/lib/gitlab/phabricator_import/user_finder_spec.rb b/spec/lib/gitlab/phabricator_import/user_finder_spec.rb index e63ad6e6626..2ec2571b7fe 100644 --- a/spec/lib/gitlab/phabricator_import/user_finder_spec.rb +++ b/spec/lib/gitlab/phabricator_import/user_finder_spec.rb @@ -58,6 +58,7 @@ RSpec.describe Gitlab::PhabricatorImport::UserFinder, :clean_gitlab_redis_cache ) ] end + let(:client) do client = instance_double(Gitlab::PhabricatorImport::Conduit::User) allow(client).to receive(:users).and_return(response) diff --git a/spec/lib/gitlab/popen/runner_spec.rb b/spec/lib/gitlab/popen/runner_spec.rb index 5f72852c63e..c7b64e8108b 100644 --- a/spec/lib/gitlab/popen/runner_spec.rb +++ b/spec/lib/gitlab/popen/runner_spec.rb @@ -118,7 +118,7 @@ RSpec.describe Gitlab::Popen::Runner do stdout: 'stdout', stderr: '', exitstatus: 0, - status: double(exitstatus: exitstatus, success?: exitstatus.zero?), + status: double(exitstatus: exitstatus, success?: exitstatus == 0), duration: 0.1) result = diff --git a/spec/lib/gitlab/project_search_results_spec.rb b/spec/lib/gitlab/project_search_results_spec.rb index 75a3fe06632..6e3c60b58dc 100644 --- a/spec/lib/gitlab/project_search_results_spec.rb +++ b/spec/lib/gitlab/project_search_results_spec.rb @@ -430,6 +430,7 @@ RSpec.describe Gitlab::ProjectSearchResults do private_project.add_maintainer(user) user end + let(:team_reporter) do user = create(:user, username: 'private-project-reporter') private_project.add_reporter(user) diff --git a/spec/lib/gitlab/prometheus/queries/matched_metric_query_spec.rb b/spec/lib/gitlab/prometheus/queries/matched_metric_query_spec.rb index 117ca798022..60449aeef7d 100644 --- a/spec/lib/gitlab/prometheus/queries/matched_metric_query_spec.rb +++ b/spec/lib/gitlab/prometheus/queries/matched_metric_query_spec.rb @@ -17,6 +17,7 @@ RSpec.describe Gitlab::Prometheus::Queries::MatchedMetricQuery do [{ '__name__' => 'metric_a' }, { '__name__' => 'metric_b' }] end + let(:partially_empty_series_info) { [{ '__name__' => 'metric_a', 'environment' => '' }] } let(:empty_series_info) { [] } diff --git a/spec/lib/gitlab/prometheus_client_spec.rb b/spec/lib/gitlab/prometheus_client_spec.rb index 242a0ced031..0774c2f3144 100644 --- a/spec/lib/gitlab/prometheus_client_spec.rb +++ b/spec/lib/gitlab/prometheus_client_spec.rb @@ -191,6 +191,7 @@ RSpec.describe Gitlab::PrometheusClient do } } end + let(:query_url) { prometheus_query_with_time_url(query, Time.now.utc) } around do |example| diff --git a/spec/lib/gitlab/redis/hll_spec.rb b/spec/lib/gitlab/redis/hll_spec.rb new file mode 100644 index 00000000000..cbf78f23036 --- /dev/null +++ b/spec/lib/gitlab/redis/hll_spec.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Redis::HLL, :clean_gitlab_redis_shared_state do + using RSpec::Parameterized::TableSyntax + + let(:expiry) { 1.day } + + describe '.add' do + context 'when checking key format' do + context 'for invalid keys' do + where(:metric_key, :value) do + 'test' | 1 + 'test-{metric' | 1 + 'test-{metric}}' | 1 + end + + with_them do + it 'raise an error when using an invalid key format' do + expect { described_class.add(key: metric_key, value: value, expiry: expiry) }.to raise_error(Gitlab::Redis::HLL::KeyFormatError) + end + end + end + + context 'for valid keys' do + where(:metric_key, :value) do + 'test-{metric}' | 1 + 'test-{metric}-1' | 1 + 'test:{metric}-1' | 1 + '2020-216-{project_action}' | 1 + 'i_{analytics}_dev_ops_score-2020-32' | 1 + end + + with_them do + it "doesn't raise error when having correct format" do + expect { described_class.add(key: metric_key, value: value, expiry: expiry) }.not_to raise_error + end + end + end + end + end + + describe '.count' do + let(:event_2020_32) { '2020-32-{expand_vulnerabilities}' } + let(:event_2020_33) { '2020-33-{expand_vulnerabilities}' } + let(:event_2020_34) { '2020-34-{expand_vulnerabilities}' } + + let(:entity1) { 'user_id_1'} + let(:entity2) { 'user_id_2'} + let(:entity3) { 'user_id_3'} + let(:entity4) { 'user_id_4'} + + before do + track_event(event_2020_32, entity1) + track_event(event_2020_32, entity1) + track_event(event_2020_32, entity2) + track_event(event_2020_32, entity3) + + track_event(event_2020_33, entity3) + track_event(event_2020_33, entity3) + + track_event(event_2020_34, entity3) + track_event(event_2020_34, entity2) + end + + it 'has 3 distinct users for weeks 32, 33, 34' do + unique_counts = count_unique_events([event_2020_32, event_2020_33, event_2020_34]) + + expect(unique_counts).to eq(3) + end + + it 'has 3 distinct users for weeks 32, 33' do + unique_counts = count_unique_events([event_2020_32, event_2020_33]) + + expect(unique_counts).to eq(3) + end + + it 'has 2 distinct users for weeks 33, 34' do + unique_counts = count_unique_events([event_2020_33, event_2020_34]) + + expect(unique_counts).to eq(2) + end + + it 'has one distinct user for week 33' do + unique_counts = count_unique_events([event_2020_33]) + + expect(unique_counts).to eq(1) + end + + it 'has 4 distinct users when one different user has an action on week 34' do + track_event(event_2020_34, entity4, 29.days) + unique_counts = count_unique_events([event_2020_32, event_2020_33, event_2020_34]) + + expect(unique_counts).to eq(4) + end + + def track_event(key, value, expiry = 1.day) + described_class.add(key: key, value: value, expiry: expiry) + end + + def count_unique_events(keys) + described_class.count(keys: keys) + end + end +end diff --git a/spec/lib/gitlab/regex_spec.rb b/spec/lib/gitlab/regex_spec.rb index 7aece6fe697..1a6858858a7 100644 --- a/spec/lib/gitlab/regex_spec.rb +++ b/spec/lib/gitlab/regex_spec.rb @@ -131,6 +131,22 @@ RSpec.describe Gitlab::Regex do it { is_expected.not_to match('9/9/2018') } end + describe '.cluster_agent_name_regex' do + subject { described_class.cluster_agent_name_regex } + + it { is_expected.to match('foo') } + it { is_expected.to match('foo-bar') } + it { is_expected.to match('1foo-bar') } + it { is_expected.to match('foo-bar2') } + it { is_expected.to match('foo-1bar') } + it { is_expected.not_to match('foo.bar') } + it { is_expected.not_to match('Foo') } + it { is_expected.not_to match('FoO') } + it { is_expected.not_to match('FoO-') } + it { is_expected.not_to match('-foo-') } + it { is_expected.not_to match('foo/bar') } + end + describe '.kubernetes_namespace_regex' do subject { described_class.kubernetes_namespace_regex } diff --git a/spec/lib/gitlab/repository_cache_adapter_spec.rb b/spec/lib/gitlab/repository_cache_adapter_spec.rb index 3727217203e..c9ad79234d3 100644 --- a/spec/lib/gitlab/repository_cache_adapter_spec.rb +++ b/spec/lib/gitlab/repository_cache_adapter_spec.rb @@ -9,6 +9,89 @@ RSpec.describe Gitlab::RepositoryCacheAdapter do let(:redis_set_cache) { repository.send(:redis_set_cache) } let(:redis_hash_cache) { repository.send(:redis_hash_cache) } + describe '.cache_method_output_as_redis_set', :clean_gitlab_redis_cache, :aggregate_failures do + let(:klass) do + Class.new do + include Gitlab::RepositoryCacheAdapter # can't use described_class here + + def letters + %w(b a c) + end + cache_method_as_redis_set(:letters) + + def redis_set_cache + @redis_set_cache ||= Gitlab::RepositorySetCache.new(self) + end + + def full_path + 'foo/bar' + end + + def project + end + end + end + + let(:fake_repository) { klass.new } + + context 'with an existing repository' do + it 'caches the output, sorting the results' do + expect(fake_repository).to receive(:_uncached_letters).once.and_call_original + + 2.times do + expect(fake_repository.letters).to eq(%w(a b c)) + end + + expect(fake_repository.redis_set_cache.exist?(:letters)).to eq(true) + expect(fake_repository.instance_variable_get(:@letters)).to eq(%w(a b c)) + end + + context 'membership checks' do + context 'when the cache key does not exist' do + it 'calls the original method and populates the cache' do + expect(fake_repository.redis_set_cache.exist?(:letters)).to eq(false) + expect(fake_repository).to receive(:_uncached_letters).once.and_call_original + + # This populates the cache and memoizes the full result + expect(fake_repository.letters_include?('a')).to eq(true) + expect(fake_repository.letters_include?('d')).to eq(false) + expect(fake_repository.redis_set_cache.exist?(:letters)).to eq(true) + end + end + + context 'when the cache key exists' do + before do + fake_repository.redis_set_cache.write(:letters, %w(b a c)) + end + + it 'calls #include? on the set cache' do + expect(fake_repository.redis_set_cache) + .to receive(:include?).with(:letters, 'a').and_call_original + expect(fake_repository.redis_set_cache) + .to receive(:include?).with(:letters, 'd').and_call_original + + expect(fake_repository.letters_include?('a')).to eq(true) + expect(fake_repository.letters_include?('d')).to eq(false) + end + + it 'memoizes the result' do + expect(fake_repository.redis_set_cache) + .to receive(:include?).once.and_call_original + + expect(fake_repository.letters_include?('a')).to eq(true) + expect(fake_repository.letters_include?('a')).to eq(true) + + expect(fake_repository.redis_set_cache) + .to receive(:include?).once.and_call_original + + expect(fake_repository.letters_include?('d')).to eq(false) + expect(fake_repository.letters_include?('d')).to eq(false) + end + end + end + end + end + describe '#cache_method_output', :use_clean_rails_memory_store_caching do let(:fallback) { 10 } @@ -212,8 +295,7 @@ RSpec.describe Gitlab::RepositoryCacheAdapter do expect(cache).to receive(:expire).with(:rendered_readme) expect(cache).to receive(:expire).with(:branch_names) expect(redis_set_cache).to receive(:expire).with(:rendered_readme, :branch_names) - expect(redis_hash_cache).to receive(:delete).with(:rendered_readme) - expect(redis_hash_cache).to receive(:delete).with(:branch_names) + expect(redis_hash_cache).to receive(:delete).with(:rendered_readme, :branch_names) repository.expire_method_caches(%i(rendered_readme branch_names)) end diff --git a/spec/lib/gitlab/repository_hash_cache_spec.rb b/spec/lib/gitlab/repository_hash_cache_spec.rb index ea856c14a77..9b4ca3f9dca 100644 --- a/spec/lib/gitlab/repository_hash_cache_spec.rb +++ b/spec/lib/gitlab/repository_hash_cache_spec.rb @@ -48,6 +48,24 @@ RSpec.describe Gitlab::RepositoryHashCache, :clean_gitlab_redis_cache do context "key doesn't exist" do it { is_expected.to eq(0) } end + + context "multiple keys" do + before do + cache.write(:test1, test_hash) + cache.write(:test2, test_hash) + end + + it "deletes multiple keys" do + cache.delete(:test1, :test2) + + expect(cache.read_members(:test1, ["test"])).to eq("test" => nil) + expect(cache.read_members(:test2, ["test"])).to eq("test" => nil) + end + + it "returns deleted key count" do + expect(cache.delete(:test1, :test2)).to eq(2) + end + end end describe "#key?" do diff --git a/spec/lib/gitlab/repository_set_cache_spec.rb b/spec/lib/gitlab/repository_set_cache_spec.rb index 024aae49b04..07f4d7c462d 100644 --- a/spec/lib/gitlab/repository_set_cache_spec.rb +++ b/spec/lib/gitlab/repository_set_cache_spec.rb @@ -93,23 +93,6 @@ RSpec.describe Gitlab::RepositorySetCache, :clean_gitlab_redis_cache do it { is_expected.to eq(0) } end - - context "unlink isn't supported" do - before do - allow_any_instance_of(Redis).to receive(:unlink) { raise ::Redis::CommandError } - end - - it 'still deletes the given key' do - expect(cache.expire(:foo)).to eq(1) - expect(cache.read(:foo)).to be_empty - end - - it 'logs the failure' do - expect(Gitlab::ErrorTracking).to receive(:log_exception) - - cache.expire(:foo) - end - end end describe '#exist?' do diff --git a/spec/lib/gitlab/search/query_spec.rb b/spec/lib/gitlab/search/query_spec.rb index e9601002922..dd2f23a7e47 100644 --- a/spec/lib/gitlab/search/query_spec.rb +++ b/spec/lib/gitlab/search/query_spec.rb @@ -38,4 +38,12 @@ RSpec.describe Gitlab::Search::Query do expect(subject.term).to eq(query) end end + + context 'with an exclusive filter' do + let(:query) { 'something -name:bingo -other:dingo' } + + it 'negates the filter' do + expect(subject.filters).to all(include(negated: true)) + end + end end diff --git a/spec/lib/gitlab/service_desk_email_spec.rb b/spec/lib/gitlab/service_desk_email_spec.rb index 23e2b2ff3cf..67a1f07eec6 100644 --- a/spec/lib/gitlab/service_desk_email_spec.rb +++ b/spec/lib/gitlab/service_desk_email_spec.rb @@ -56,4 +56,26 @@ RSpec.describe Gitlab::ServiceDeskEmail do end end end + + describe '.address_for_key' do + context 'when service desk address is set' do + before do + stub_service_desk_email_setting(address: 'address+%{key}@example.com') + end + + it 'returns address' do + expect(described_class.address_for_key('foo')).to eq('address+foo@example.com') + end + end + + context 'when service desk address is not set' do + before do + stub_service_desk_email_setting(address: nil) + end + + it 'returns nil' do + expect(described_class.key_from_address('foo')).to be_nil + end + end + end end diff --git a/spec/lib/gitlab/sidekiq_cluster_spec.rb b/spec/lib/gitlab/sidekiq_cluster_spec.rb index d625a2a2185..5dd913aebb0 100644 --- a/spec/lib/gitlab/sidekiq_cluster_spec.rb +++ b/spec/lib/gitlab/sidekiq_cluster_spec.rb @@ -91,6 +91,7 @@ RSpec.describe Gitlab::SidekiqCluster do let(:options) do { env: :production, directory: 'foo/bar', max_concurrency: 20, min_concurrency: 0, worker_id: first_worker_id, timeout: 10, dryrun: false } end + let(:env) { { "ENABLE_SIDEKIQ_CLUSTER" => "1", "SIDEKIQ_WORKER_ID" => first_worker_id.to_s } } let(:args) { ['bundle', 'exec', 'sidekiq', anything, '-eproduction', '-t10', *([anything] * 5)] } diff --git a/spec/lib/gitlab/sidekiq_logging/exception_handler_spec.rb b/spec/lib/gitlab/sidekiq_logging/exception_handler_spec.rb index 5a68b88c02d..94dcf6f9b9a 100644 --- a/spec/lib/gitlab/sidekiq_logging/exception_handler_spec.rb +++ b/spec/lib/gitlab/sidekiq_logging/exception_handler_spec.rb @@ -33,7 +33,7 @@ RSpec.describe Gitlab::SidekiqLogging::ExceptionHandler do error_class: 'RuntimeError', error_message: exception_message, context: 'Test', - error_backtrace: Gitlab::BacktraceCleaner.clean_backtrace(backtrace) + error_backtrace: Rails.backtrace_cleaner.clean(backtrace) ) expect(logger).to receive(:warn).with(expected_data) diff --git a/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb b/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb index 03ace9a01c7..ad106837c47 100644 --- a/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb +++ b/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb @@ -41,6 +41,7 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do 'scheduling_latency_s' => scheduling_latency_s ) end + let(:end_payload) do start_payload.merge( 'message' => 'TestWorker JID-da883554ee4fe414012f5f42: done: 0.0 sec', @@ -51,6 +52,7 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do 'db_duration_s' => 0.0 ) end + let(:exception_payload) do end_payload.merge( 'message' => 'TestWorker JID-da883554ee4fe414012f5f42: fail: 0.0 sec', diff --git a/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb b/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb index 1b3b108d9ea..44bfaf4cc3c 100644 --- a/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb @@ -128,6 +128,13 @@ RSpec.describe Gitlab::SidekiqMiddleware::ServerMetrics do subject.call(worker, job, :test) { nil } end + it 'sets the thread name if it was nil' do + allow(Thread.current).to receive(:name).and_return(nil) + expect(Thread.current).to receive(:name=).with(Gitlab::Metrics::Samplers::ThreadsSampler::SIDEKIQ_WORKER_THREAD_NAME) + + subject.call(worker, job, :test) { nil } + end + context 'when job_duration is not available' do let(:queue_duration_for_job) { nil } diff --git a/spec/lib/gitlab/sidekiq_middleware_spec.rb b/spec/lib/gitlab/sidekiq_middleware_spec.rb index 018821e6c5e..4ee9569a0cf 100644 --- a/spec/lib/gitlab/sidekiq_middleware_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware_spec.rb @@ -51,6 +51,7 @@ RSpec.describe Gitlab::SidekiqMiddleware do Gitlab::SidekiqMiddleware::BatchLoader, Labkit::Middleware::Sidekiq::Server, Gitlab::SidekiqMiddleware::InstrumentationLogger, + Gitlab::SidekiqVersioning::Middleware, Gitlab::SidekiqStatus::ServerMiddleware, Gitlab::SidekiqMiddleware::ServerMetrics, Gitlab::SidekiqMiddleware::ArgumentsLogger, @@ -62,6 +63,7 @@ RSpec.describe Gitlab::SidekiqMiddleware do Gitlab::SidekiqMiddleware::DuplicateJobs::Server ] end + let(:enabled_sidekiq_middlewares) { all_sidekiq_middlewares - disabled_sidekiq_middlewares } shared_examples "a server middleware chain" do @@ -78,6 +80,41 @@ RSpec.describe Gitlab::SidekiqMiddleware do end end + shared_examples "a server middleware chain for mailer" do + let(:worker_class) { ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper } + let(:job_args) do + [ + { + "job_class" => "ActionMailer::MailDeliveryJob", + "job_id" => "a180b47c-3fd6-41b8-81e9-34da61c3400e", + "provider_job_id" => nil, + "queue_name" => "mailers", + "priority" => nil, + "arguments" => [ + "Notify", + "test_email", + "deliver_now", + { + "args" => [ + "test@example.com", + "subject", + "body" + ], + "_aj_symbol_keys" => ["args"] + } + ], + "executions" => 0, + "exception_executions" => {}, + "locale" => "en", + "timezone" => "UTC", + "enqueued_at" => "2020-07-27T07:43:31Z" + } + ] + end + + it_behaves_like "a server middleware chain" + end + context "all optional middlewares off" do let(:metrics) { false } let(:arguments_logger) { false } @@ -91,6 +128,7 @@ RSpec.describe Gitlab::SidekiqMiddleware do end it_behaves_like "a server middleware chain" + it_behaves_like "a server middleware chain for mailer" end context "all optional middlewares on" do @@ -100,6 +138,7 @@ RSpec.describe Gitlab::SidekiqMiddleware do let(:disabled_sidekiq_middlewares) { [] } it_behaves_like "a server middleware chain" + it_behaves_like "a server middleware chain for mailer" context "server metrics" do let(:gitaly_histogram) { double(:gitaly_histogram) } diff --git a/spec/lib/gitlab/sidekiq_versioning/middleware_spec.rb b/spec/lib/gitlab/sidekiq_versioning/middleware_spec.rb new file mode 100644 index 00000000000..b372f16de5e --- /dev/null +++ b/spec/lib/gitlab/sidekiq_versioning/middleware_spec.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::SidekiqVersioning::Middleware do + let(:worker_class) do + Class.new do + def self.name + 'DummyWorker' + end + + include ApplicationWorker + + version 2 + end + end + + describe '#call' do + let(:worker) { worker_class.new } + let(:job) { { 'version' => 3, 'queue' => queue } } + let(:queue) { worker_class.queue } + + def call!(&block) + block ||= -> {} + subject.call(worker, job, queue, &block) + end + + it 'sets worker.job_version' do + call! + + expect(worker.job_version).to eq(job['version']) + end + + it 'yields' do + expect { |b| call!(&b) }.to yield_control + end + + context 'when worker is not ApplicationWorker' do + let(:worker_class) do + ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper + end + + it 'does not err' do + expect { call! }.not_to raise_error + end + end + end +end diff --git a/spec/lib/gitlab/sidekiq_versioning/worker_spec.rb b/spec/lib/gitlab/sidekiq_versioning/worker_spec.rb new file mode 100644 index 00000000000..0781c5100fd --- /dev/null +++ b/spec/lib/gitlab/sidekiq_versioning/worker_spec.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::SidekiqVersioning::Worker do + let(:worker) do + Class.new do + def self.name + 'DummyWorker' + end + + # ApplicationWorker includes Gitlab::SidekiqVersioning::Worker + include ApplicationWorker + + version 2 + end + end + + describe '.version' do + context 'when called with an argument' do + it 'sets the version option' do + worker.version 3 + + expect(worker.get_sidekiq_options['version']).to eq(3) + end + end + + context 'when called without an argument' do + it 'returns the version option' do + worker.sidekiq_options version: 3 + + expect(worker.version).to eq(3) + end + end + end + + describe '#job_version' do + let(:job) { worker.new } + + context 'when job_version is not set' do + it 'returns latest version' do + expect(job.job_version).to eq(2) + end + end + + context 'when job_version is set' do + it 'returns the set version' do + job.job_version = 0 + + expect(job.job_version).to eq(0) + end + end + end +end diff --git a/spec/lib/gitlab/static_site_editor/config_spec.rb b/spec/lib/gitlab/static_site_editor/config_spec.rb index b60a6a9b006..56cdb573785 100644 --- a/spec/lib/gitlab/static_site_editor/config_spec.rb +++ b/spec/lib/gitlab/static_site_editor/config_spec.rb @@ -46,8 +46,6 @@ RSpec.describe Gitlab::StaticSiteEditor::Config do end context 'when file has .md.erb extension' do - let(:file_path) { 'README.md.erb' } - before do repository.create_file( project.creator, @@ -58,7 +56,25 @@ RSpec.describe Gitlab::StaticSiteEditor::Config do ) end - it { is_expected.to include(is_supported_content: 'true') } + context 'when feature flag is enabled' do + let(:file_path) { 'FEATURE_ON.md.erb' } + + before do + stub_feature_flags(sse_erb_support: project) + end + + it { is_expected.to include(is_supported_content: 'true') } + end + + context 'when feature flag is disabled' do + let(:file_path) { 'FEATURE_OFF.md.erb' } + + before do + stub_feature_flags(sse_erb_support: false) + end + + it { is_expected.to include(is_supported_content: 'false') } + end end context 'when file path is nested' do diff --git a/spec/lib/gitlab/template/gitlab_ci_yml_template_spec.rb b/spec/lib/gitlab/template/gitlab_ci_yml_template_spec.rb index 3004de6fe08..55444114d39 100644 --- a/spec/lib/gitlab/template/gitlab_ci_yml_template_spec.rb +++ b/spec/lib/gitlab/template/gitlab_ci_yml_template_spec.rb @@ -6,10 +6,6 @@ RSpec.describe Gitlab::Template::GitlabCiYmlTemplate do subject { described_class } describe '.all' do - it 'strips the gitlab-ci suffix' do - expect(subject.all.first.name).not_to end_with('.gitlab-ci.yml') - end - it 'combines the globals and rest' do all = subject.all.map(&:name) @@ -17,34 +13,6 @@ RSpec.describe Gitlab::Template::GitlabCiYmlTemplate do expect(all).to include('Docker') expect(all).to include('Ruby') end - - it 'ensure that the template name is used exactly once' do - all = subject.all.group_by(&:name) - duplicates = all.select { |_, templates| templates.length > 1 } - - expect(duplicates).to be_empty - end - end - - describe '.find' do - it 'returns nil if the file does not exist' do - expect(subject.find('mepmep-yadida')).to be nil - end - - it 'returns the GitlabCiYml object of a valid file' do - ruby = subject.find('Ruby') - - expect(ruby).to be_a described_class - expect(ruby.name).to eq('Ruby') - end - end - - describe '.by_category' do - it 'returns sorted results' do - result = described_class.by_category('General') - - expect(result).to eq(result.sort) - end end describe '#content' do @@ -56,13 +24,5 @@ RSpec.describe Gitlab::Template::GitlabCiYmlTemplate do end end - describe '#<=>' do - it 'sorts lexicographically' do - one = described_class.new('a.gitlab-ci.yml') - other = described_class.new('z.gitlab-ci.yml') - - expect(one.<=>(other)).to be(-1) - expect([other, one].sort).to eq([one, other]) - end - end + it_behaves_like 'file template shared examples', 'Ruby', '.gitlab-ci.yml' end diff --git a/spec/lib/gitlab/template/metrics_dashboard_template_spec.rb b/spec/lib/gitlab/template/metrics_dashboard_template_spec.rb new file mode 100644 index 00000000000..4c2b3dea600 --- /dev/null +++ b/spec/lib/gitlab/template/metrics_dashboard_template_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Template::MetricsDashboardTemplate do + subject { described_class } + + describe '.all' do + it 'combines the globals and rest' do + all = subject.all.map(&:name) + + expect(all).to include('Default') + end + end + + describe '#content' do + it 'loads the full file' do + example_dashboard = subject.new(Rails.root.join('lib/gitlab/metrics/templates/Default.metrics-dashboard.yml')) + + expect(example_dashboard.name).to eq 'Default' + expect(example_dashboard.content).to start_with('#') + end + end + + it_behaves_like 'file template shared examples', 'Default', '.metrics-dashboard.yml' +end diff --git a/spec/lib/gitlab/tree_summary_spec.rb b/spec/lib/gitlab/tree_summary_spec.rb index 4bd08fab60a..303a4a80581 100644 --- a/spec/lib/gitlab/tree_summary_spec.rb +++ b/spec/lib/gitlab/tree_summary_spec.rb @@ -191,7 +191,7 @@ RSpec.describe Gitlab::TreeSummary do with_them do before do - create_file('dummy', path: 'other') if num_entries.zero? + create_file('dummy', path: 'other') if num_entries == 0 1.upto(num_entries) { |n| create_file(n, path: path) } end @@ -218,7 +218,7 @@ RSpec.describe Gitlab::TreeSummary do with_them do before do - create_file('dummy', path: 'other') if num_entries.zero? + create_file('dummy', path: 'other') if num_entries == 0 1.upto(num_entries) { |n| create_file(n, path: path) } end diff --git a/spec/lib/gitlab/url_blocker_spec.rb b/spec/lib/gitlab/url_blocker_spec.rb index 7edfde09864..b49efd6a092 100644 --- a/spec/lib/gitlab/url_blocker_spec.rb +++ b/spec/lib/gitlab/url_blocker_spec.rb @@ -280,6 +280,7 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do '[fc00:bf8b:e62c:abcd:abcd:aaaa:aaaa:aaaa]' ] end + let(:fake_domain) { 'www.fakedomain.fake' } shared_examples 'allows local requests' do |url_blocker_attributes| diff --git a/spec/lib/gitlab/url_builder_spec.rb b/spec/lib/gitlab/url_builder_spec.rb index a16ff252bc1..b58b5a84662 100644 --- a/spec/lib/gitlab/url_builder_spec.rb +++ b/spec/lib/gitlab/url_builder_spec.rb @@ -22,7 +22,7 @@ RSpec.describe Gitlab::UrlBuilder do :issue | ->(issue) { "/#{issue.project.full_path}/-/issues/#{issue.iid}" } :merge_request | ->(merge_request) { "/#{merge_request.project.full_path}/-/merge_requests/#{merge_request.iid}" } :project_milestone | ->(milestone) { "/#{milestone.project.full_path}/-/milestones/#{milestone.iid}" } - :project_snippet | ->(snippet) { "/#{snippet.project.full_path}/snippets/#{snippet.id}" } + :project_snippet | ->(snippet) { "/#{snippet.project.full_path}/-/snippets/#{snippet.id}" } :project_wiki | ->(wiki) { "/#{wiki.container.full_path}/-/wikis/home" } :ci_build | ->(build) { "/#{build.project.full_path}/-/jobs/#{build.id}" } :design | ->(design) { "/#{design.project.full_path}/-/design_management/designs/#{design.id}/raw_image" } @@ -31,7 +31,7 @@ RSpec.describe Gitlab::UrlBuilder do :group_milestone | ->(milestone) { "/groups/#{milestone.group.full_path}/-/milestones/#{milestone.iid}" } :user | ->(user) { "/#{user.full_path}" } - :personal_snippet | ->(snippet) { "/snippets/#{snippet.id}" } + :personal_snippet | ->(snippet) { "/-/snippets/#{snippet.id}" } :wiki_page | ->(wiki_page) { "#{wiki_page.wiki.wiki_base_path}/#{wiki_page.slug}" } :note_on_commit | ->(note) { "/#{note.project.full_path}/-/commit/#{note.commit_id}#note_#{note.id}" } @@ -47,10 +47,10 @@ RSpec.describe Gitlab::UrlBuilder do :discussion_note_on_merge_request | ->(note) { "/#{note.project.full_path}/-/merge_requests/#{note.noteable.iid}#note_#{note.id}" } :legacy_diff_note_on_merge_request | ->(note) { "/#{note.project.full_path}/-/merge_requests/#{note.noteable.iid}#note_#{note.id}" } - :note_on_project_snippet | ->(note) { "/#{note.project.full_path}/snippets/#{note.noteable_id}#note_#{note.id}" } - :discussion_note_on_project_snippet | ->(note) { "/#{note.project.full_path}/snippets/#{note.noteable_id}#note_#{note.id}" } - :discussion_note_on_personal_snippet | ->(note) { "/snippets/#{note.noteable_id}#note_#{note.id}" } - :note_on_personal_snippet | ->(note) { "/snippets/#{note.noteable_id}#note_#{note.id}" } + :note_on_project_snippet | ->(note) { "/#{note.project.full_path}/-/snippets/#{note.noteable_id}#note_#{note.id}" } + :discussion_note_on_project_snippet | ->(note) { "/#{note.project.full_path}/-/snippets/#{note.noteable_id}#note_#{note.id}" } + :discussion_note_on_personal_snippet | ->(note) { "/-/snippets/#{note.noteable_id}#note_#{note.id}" } + :note_on_personal_snippet | ->(note) { "/-/snippets/#{note.noteable_id}#note_#{note.id}" } end with_them do @@ -98,7 +98,7 @@ RSpec.describe Gitlab::UrlBuilder do it 'returns a raw snippet URL if requested' do url = subject.build(snippet, raw: true) - expect(url).to eq "#{Gitlab.config.gitlab.url}/snippets/#{snippet.id}/raw" + expect(url).to eq "#{Gitlab.config.gitlab.url}/-/snippets/#{snippet.id}/raw" end it 'returns a raw snippet blob URL if requested' do @@ -114,7 +114,7 @@ RSpec.describe Gitlab::UrlBuilder do it 'returns a raw snippet URL if requested' do url = subject.build(snippet, raw: true) - expect(url).to eq "#{Gitlab.config.gitlab.url}/#{snippet.project.full_path}/snippets/#{snippet.id}/raw" + expect(url).to eq "#{Gitlab.config.gitlab.url}/#{snippet.project.full_path}/-/snippets/#{snippet.id}/raw" end it 'returns a raw snippet blob URL if requested' do diff --git a/spec/lib/gitlab/usage_data/topology_spec.rb b/spec/lib/gitlab/usage_data/topology_spec.rb index 2a7adea261d..7f4a25297e6 100644 --- a/spec/lib/gitlab/usage_data/topology_spec.rb +++ b/spec/lib/gitlab/usage_data/topology_spec.rb @@ -24,7 +24,9 @@ RSpec.describe Gitlab::UsageData::Topology do expect_prometheus_api_to( receive_app_request_volume_query, receive_node_memory_query, + receive_node_memory_utilization_query, receive_node_cpu_count_query, + receive_node_cpu_utilization_query, receive_node_uname_info_query, receive_node_service_memory_rss_query, receive_node_service_memory_uss_query, @@ -40,7 +42,9 @@ RSpec.describe Gitlab::UsageData::Topology do nodes: [ { node_memory_total_bytes: 512, + node_memory_utilization: 0.45, node_cpus: 8, + node_cpu_utilization: 0.1, node_uname_info: { machine: 'x86_64', sysname: 'Linux', @@ -64,7 +68,9 @@ RSpec.describe Gitlab::UsageData::Topology do }, { node_memory_total_bytes: 1024, + node_memory_utilization: 0.25, node_cpus: 16, + node_cpu_utilization: 0.2, node_uname_info: { machine: 'x86_64', sysname: 'Linux', @@ -102,7 +108,9 @@ RSpec.describe Gitlab::UsageData::Topology do expect_prometheus_api_to( receive_app_request_volume_query(result: []), receive_node_memory_query(result: []), + receive_node_memory_utilization_query(result: []), receive_node_cpu_count_query, + receive_node_cpu_utilization_query, receive_node_uname_info_query, receive_node_service_memory_rss_query(result: []), receive_node_service_memory_uss_query(result: []), @@ -116,6 +124,7 @@ RSpec.describe Gitlab::UsageData::Topology do failures: [ { 'app_requests' => 'empty_result' }, { 'node_memory' => 'empty_result' }, + { 'node_memory_utilization' => 'empty_result' }, { 'service_rss' => 'empty_result' }, { 'service_uss' => 'empty_result' }, { 'service_workers' => 'empty_result' } @@ -123,6 +132,7 @@ RSpec.describe Gitlab::UsageData::Topology do nodes: [ { node_cpus: 16, + node_cpu_utilization: 0.2, node_uname_info: { machine: 'x86_64', release: '4.15.0-101-generic', @@ -146,6 +156,7 @@ RSpec.describe Gitlab::UsageData::Topology do }, { node_cpus: 8, + node_cpu_utilization: 0.1, node_uname_info: { machine: 'x86_64', release: '4.19.76-linuxkit', @@ -177,6 +188,16 @@ RSpec.describe Gitlab::UsageData::Topology do } ] end + + let(:node_memory_utilization_response) do + [ + { + 'metric' => { 'instance' => 'localhost:9100' }, + 'value' => [1000, '0.35'] + } + ] + end + let(:node_uname_info_response) do [ { @@ -195,6 +216,7 @@ RSpec.describe Gitlab::UsageData::Topology do ] end # The services in this response should all be mapped to localhost i.e. the same node + let(:service_memory_response) do [ { @@ -224,7 +246,9 @@ RSpec.describe Gitlab::UsageData::Topology do expect_prometheus_api_to( receive_app_request_volume_query(result: []), receive_node_memory_query(result: node_memory_response), + receive_node_memory_utilization_query(result: node_memory_utilization_response), receive_node_cpu_count_query(result: []), + receive_node_cpu_utilization_query(result: []), receive_node_uname_info_query(result: node_uname_info_response), receive_node_service_memory_rss_query(result: service_memory_response), receive_node_service_memory_uss_query(result: []), @@ -238,6 +262,7 @@ RSpec.describe Gitlab::UsageData::Topology do failures: [ { 'app_requests' => 'empty_result' }, { 'node_cpus' => 'empty_result' }, + { 'node_cpu_utilization' => 'empty_result' }, { 'service_uss' => 'empty_result' }, { 'service_pss' => 'empty_result' }, { 'service_process_count' => 'empty_result' }, @@ -246,6 +271,7 @@ RSpec.describe Gitlab::UsageData::Topology do nodes: [ { node_memory_total_bytes: 512, + node_memory_utilization: 0.35, node_uname_info: { machine: 'x86_64', sysname: 'Linux', @@ -284,7 +310,9 @@ RSpec.describe Gitlab::UsageData::Topology do expect_prometheus_api_to( receive_app_request_volume_query(result: []), receive_node_memory_query(result: []), + receive_node_memory_utilization_query(result: []), receive_node_cpu_count_query(result: []), + receive_node_cpu_utilization_query(result: []), receive_node_uname_info_query(result: []), receive_node_service_memory_rss_query, receive_node_service_memory_uss_query(result: []), @@ -298,7 +326,9 @@ RSpec.describe Gitlab::UsageData::Topology do failures: [ { 'app_requests' => 'empty_result' }, { 'node_memory' => 'empty_result' }, + { 'node_memory_utilization' => 'empty_result' }, { 'node_cpus' => 'empty_result' }, + { 'node_cpu_utilization' => 'empty_result' }, { 'node_uname_info' => 'empty_result' }, { 'service_uss' => 'empty_result' }, { 'service_pss' => 'empty_result' }, @@ -335,27 +365,98 @@ RSpec.describe Gitlab::UsageData::Topology do end end + context 'and unknown services are encountered' do + let(:unknown_service_process_count_response) do + [ + { + 'metric' => { 'instance' => 'instance2:9000', 'job' => 'unknown-service-A' }, + 'value' => [1000, '42'] + }, + { + 'metric' => { 'instance' => 'instance2:9001', 'job' => 'unknown-service-B' }, + 'value' => [1000, '42'] + } + ] + end + + it 'filters out unknown service data and reports the unknown services as a failure' do + expect_prometheus_api_to( + receive_app_request_volume_query(result: []), + receive_node_memory_query(result: []), + receive_node_memory_utilization_query(result: []), + receive_node_cpu_count_query(result: []), + receive_node_cpu_utilization_query(result: []), + receive_node_uname_info_query(result: []), + receive_node_service_memory_rss_query(result: []), + receive_node_service_memory_uss_query(result: []), + receive_node_service_memory_pss_query(result: []), + receive_node_service_process_count_query(result: unknown_service_process_count_response), + receive_node_service_app_server_workers_query(result: []) + ) + + expect(subject.dig(:topology, :failures)).to include( + { 'service_unknown' => 'unknown-service-A' }, + { 'service_unknown' => 'unknown-service-B' } + ) + end + end + context 'and an error is raised when querying Prometheus' do - it 'returns empty result with failures' do - expect_prometheus_api_to receive(:query) - .at_least(:once) - .and_raise(Gitlab::PrometheusClient::ConnectionError) + context 'without timeout failures' do + it 'returns empty result and executes subsequent queries as usual' do + expect_prometheus_api_to receive(:query) + .at_least(:once) + .and_raise(Gitlab::PrometheusClient::ConnectionError) - expect(subject[:topology]).to eq({ - duration_s: 0, - failures: [ - { 'app_requests' => 'Gitlab::PrometheusClient::ConnectionError' }, - { 'node_memory' => 'Gitlab::PrometheusClient::ConnectionError' }, - { 'node_cpus' => 'Gitlab::PrometheusClient::ConnectionError' }, - { 'node_uname_info' => 'Gitlab::PrometheusClient::ConnectionError' }, - { 'service_rss' => 'Gitlab::PrometheusClient::ConnectionError' }, - { 'service_uss' => 'Gitlab::PrometheusClient::ConnectionError' }, - { 'service_pss' => 'Gitlab::PrometheusClient::ConnectionError' }, - { 'service_process_count' => 'Gitlab::PrometheusClient::ConnectionError' }, - { 'service_workers' => 'Gitlab::PrometheusClient::ConnectionError' } - ], - nodes: [] - }) + expect(subject[:topology]).to eq({ + duration_s: 0, + failures: [ + { 'app_requests' => 'Gitlab::PrometheusClient::ConnectionError' }, + { 'node_memory' => 'Gitlab::PrometheusClient::ConnectionError' }, + { 'node_memory_utilization' => 'Gitlab::PrometheusClient::ConnectionError' }, + { 'node_cpus' => 'Gitlab::PrometheusClient::ConnectionError' }, + { 'node_cpu_utilization' => 'Gitlab::PrometheusClient::ConnectionError' }, + { 'node_uname_info' => 'Gitlab::PrometheusClient::ConnectionError' }, + { 'service_rss' => 'Gitlab::PrometheusClient::ConnectionError' }, + { 'service_uss' => 'Gitlab::PrometheusClient::ConnectionError' }, + { 'service_pss' => 'Gitlab::PrometheusClient::ConnectionError' }, + { 'service_process_count' => 'Gitlab::PrometheusClient::ConnectionError' }, + { 'service_workers' => 'Gitlab::PrometheusClient::ConnectionError' } + ], + nodes: [] + }) + end + end + + context 'with timeout failures' do + where(:exception) do + described_class::TIMEOUT_ERRORS + end + + with_them do + it 'returns empty result and cancelled subsequent queries' do + expect_prometheus_api_to receive(:query) + .and_raise(exception) + + expect(subject[:topology]).to eq({ + duration_s: 0, + failures: [ + { 'app_requests' => exception.to_s }, + { 'node_memory' => 'timeout_cancellation' }, + { 'node_memory_utilization' => 'timeout_cancellation' }, + { 'node_cpus' => 'timeout_cancellation' }, + { 'node_cpu_utilization' => 'timeout_cancellation' }, + { 'node_uname_info' => 'timeout_cancellation' }, + { 'service_rss' => 'timeout_cancellation' }, + { 'service_uss' => 'timeout_cancellation' }, + { 'service_pss' => 'timeout_cancellation' }, + { 'service_process_count' => 'timeout_cancellation' }, + { 'service_workers' => 'timeout_cancellation' } + ], + nodes: [] + }) + end + end end end end @@ -411,6 +512,21 @@ RSpec.describe Gitlab::UsageData::Topology do ]) end + def receive_node_memory_utilization_query(result: nil) + receive(:query) + .with(/node_memory_utilization/, an_instance_of(Hash)) + .and_return(result || [ + { + 'metric' => { 'instance' => 'instance1:8080' }, + 'value' => [1000, '0.45'] + }, + { + 'metric' => { 'instance' => 'instance2:8090' }, + 'value' => [1000, '0.25'] + } + ]) + end + def receive_node_cpu_count_query(result: nil) receive(:query) .with(/node_cpus/, an_instance_of(Hash)) @@ -426,6 +542,21 @@ RSpec.describe Gitlab::UsageData::Topology do ]) end + def receive_node_cpu_utilization_query(result: nil) + receive(:query) + .with(/node_cpu_utilization/, an_instance_of(Hash)) + .and_return(result || [ + { + 'metric' => { 'instance' => 'instance2:8090' }, + 'value' => [1000, '0.2'] + }, + { + 'metric' => { 'instance' => 'instance1:8080' }, + 'value' => [1000, '0.1'] + } + ]) + end + def receive_node_uname_info_query(result: nil) receive(:query) .with('node_uname_info') @@ -534,11 +665,6 @@ RSpec.describe Gitlab::UsageData::Topology do { 'metric' => { 'instance' => 'instance2:8080', 'job' => 'registry' }, 'value' => [1000, '1'] - }, - # unknown service => should be stripped out - { - 'metric' => { 'instance' => 'instance2:9000', 'job' => 'not-a-gitlab-service' }, - 'value' => [1000, '42'] } ]) end diff --git a/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb new file mode 100644 index 00000000000..2ab349a67d9 --- /dev/null +++ b/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_shared_state do + let(:entity1) { 'dfb9d2d2-f56c-4c77-8aeb-6cddc4a1f857' } + let(:entity2) { '1dd9afb2-a3ee-4de1-8ae3-a405579c8584' } + let(:entity3) { '34rfjuuy-ce56-sa35-ds34-dfer567dfrf2' } + let(:entity4) { '8b9a2671-2abf-4bec-a682-22f6a8f7bf31' } + + let(:weekly_event) { 'g_analytics_contribution' } + let(:daily_event) { 'g_search' } + let(:different_aggregation) { 'different_aggregation' } + + let(:known_events) do + [ + { name: "g_analytics_contribution", redis_slot: "analytics", category: "analytics", expiry: 84, aggregation: "weekly" }, + { name: "g_analytics_valuestream", redis_slot: "analytics", category: "analytics", expiry: 84, aggregation: "daily" }, + { name: "g_analytics_productivity", redis_slot: "analytics", category: "productivity", expiry: 84, aggregation: "weekly" }, + { name: "g_compliance_dashboard", redis_slot: "compliance", category: "compliance", aggregation: "weekly" }, + { name: "g_search", category: "global", aggregation: "daily" }, + { name: "different_aggregation", category: "global", aggregation: "monthly" } + ].map(&:with_indifferent_access) + end + + before do + allow(described_class).to receive(:known_events).and_return(known_events) + end + + around do |example| + # We need to freeze to a reference time + # because visits are grouped by the week number in the year + # Without freezing the time, the test may behave inconsistently + # depending on which day of the week test is run. + # Monday 6th of June + reference_time = Time.utc(2020, 6, 1) + Timecop.freeze(reference_time) { example.run } + end + + describe '.track_event' do + it "raise error if metrics don't have same aggregation" do + expect { described_class.track_event(entity1, different_aggregation, Date.current) } .to raise_error(Gitlab::UsageDataCounters::HLLRedisCounter::UnknownAggregation) + end + + it 'raise error if metrics of unknown aggregation' do + expect { described_class.track_event(entity1, 'unknown', Date.current) } .to raise_error(Gitlab::UsageDataCounters::HLLRedisCounter::UnknownEvent) + end + end + + describe '.unique_events' do + before do + # events in current week, should not be counted as week is not complete + described_class.track_event(entity1, weekly_event, Date.current) + described_class.track_event(entity2, weekly_event, Date.current) + + # Events last week + described_class.track_event(entity1, weekly_event, 2.days.ago) + described_class.track_event(entity1, weekly_event, 2.days.ago) + + # Events 2 weeks ago + described_class.track_event(entity1, weekly_event, 2.weeks.ago) + + # Events 4 weeks ago + described_class.track_event(entity3, weekly_event, 4.weeks.ago) + described_class.track_event(entity4, weekly_event, 29.days.ago) + + # events in current day should be counted in daily aggregation + described_class.track_event(entity1, daily_event, Date.current) + described_class.track_event(entity2, daily_event, Date.current) + + # Events last week + described_class.track_event(entity1, daily_event, 2.days.ago) + described_class.track_event(entity1, daily_event, 2.days.ago) + + # Events 2 weeks ago + described_class.track_event(entity1, daily_event, 14.days.ago) + + # Events 4 weeks ago + described_class.track_event(entity3, daily_event, 28.days.ago) + described_class.track_event(entity4, daily_event, 29.days.ago) + end + + it 'raise error if metrics are not in the same slot' do + expect { described_class.unique_events(event_names: %w(g_analytics_contribution g_compliance_dashboard), start_date: 4.weeks.ago, end_date: Date.current) }.to raise_error('Events should be in same slot') + end + + it 'raise error if metrics are not in the same category' do + expect { described_class.unique_events(event_names: %w(g_analytics_contribution g_analytics_productivity), start_date: 4.weeks.ago, end_date: Date.current) }.to raise_error('Events should be in same category') + end + + it "raise error if metrics don't have same aggregation" do + expect { described_class.unique_events(event_names: %w(g_analytics_contribution g_analytics_valuestream), start_date: 4.weeks.ago, end_date: Date.current) }.to raise_error('Events should have same aggregation level') + end + + context 'when data for the last complete week' do + it { expect(described_class.unique_events(event_names: weekly_event, start_date: 1.week.ago, end_date: Date.current)).to eq(1) } + end + + context 'when data for the last 4 complete weeks' do + it { expect(described_class.unique_events(event_names: weekly_event, start_date: 4.weeks.ago, end_date: Date.current)).to eq(2) } + end + + context 'when data for the week 4 weeks ago' do + it { expect(described_class.unique_events(event_names: weekly_event, start_date: 4.weeks.ago, end_date: 3.weeks.ago)).to eq(1) } + end + + context 'when using daily aggregation' do + it { expect(described_class.unique_events(event_names: daily_event, start_date: 7.days.ago, end_date: Date.current)).to eq(2) } + it { expect(described_class.unique_events(event_names: daily_event, start_date: 28.days.ago, end_date: Date.current)).to eq(3) } + it { expect(described_class.unique_events(event_names: daily_event, start_date: 28.days.ago, end_date: 21.days.ago)).to eq(1) } + end + end +end diff --git a/spec/lib/gitlab/usage_data_counters/track_unique_actions_spec.rb b/spec/lib/gitlab/usage_data_counters/track_unique_actions_spec.rb index 584d8407e79..bd348666729 100644 --- a/spec/lib/gitlab/usage_data_counters/track_unique_actions_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/track_unique_actions_spec.rb @@ -7,20 +7,19 @@ RSpec.describe Gitlab::UsageDataCounters::TrackUniqueActions, :clean_gitlab_redi let(:time) { Time.zone.now } - def track_action(params) - track_unique_events.track_action(params) + def track_event(params) + track_unique_events.track_event(params) end - def count_unique_events(params) - track_unique_events.count_unique_events(params) + def count_unique(params) + track_unique_events.count_unique(params) end context 'tracking an event' do context 'when tracking successfully' do - context 'when the feature flag and the application setting is enabled' do + context 'when the application setting is enabled' do context 'when the target and the action is valid' do before do - stub_feature_flags(described_class::FEATURE_FLAG => true) stub_application_setting(usage_ping_enabled: true) end @@ -29,28 +28,28 @@ RSpec.describe Gitlab::UsageDataCounters::TrackUniqueActions, :clean_gitlab_redi design = Event::TARGET_TYPES[:design] wiki = Event::TARGET_TYPES[:wiki] - expect(track_action(event_action: :pushed, event_target: project, author_id: 1)).to be_truthy - expect(track_action(event_action: :pushed, event_target: project, author_id: 1)).to be_truthy - expect(track_action(event_action: :pushed, event_target: project, author_id: 2)).to be_truthy - expect(track_action(event_action: :pushed, event_target: project, author_id: 3)).to be_truthy - expect(track_action(event_action: :pushed, event_target: project, author_id: 4, time: time - 3.days)).to be_truthy - expect(track_action(event_action: :created, event_target: project, author_id: 5, time: time - 3.days)).to be_truthy + expect(track_event(event_action: :pushed, event_target: project, author_id: 1)).to be_truthy + expect(track_event(event_action: :pushed, event_target: project, author_id: 1)).to be_truthy + expect(track_event(event_action: :pushed, event_target: project, author_id: 2)).to be_truthy + expect(track_event(event_action: :pushed, event_target: project, author_id: 3)).to be_truthy + expect(track_event(event_action: :pushed, event_target: project, author_id: 4, time: time - 3.days)).to be_truthy + expect(track_event(event_action: :created, event_target: project, author_id: 5, time: time - 3.days)).to be_truthy - expect(track_action(event_action: :destroyed, event_target: design, author_id: 3)).to be_truthy - expect(track_action(event_action: :created, event_target: design, author_id: 4)).to be_truthy - expect(track_action(event_action: :updated, event_target: design, author_id: 5)).to be_truthy - expect(track_action(event_action: :pushed, event_target: design, author_id: 6)).to be_truthy + expect(track_event(event_action: :destroyed, event_target: design, author_id: 3)).to be_truthy + expect(track_event(event_action: :created, event_target: design, author_id: 4)).to be_truthy + expect(track_event(event_action: :updated, event_target: design, author_id: 5)).to be_truthy + expect(track_event(event_action: :pushed, event_target: design, author_id: 6)).to be_truthy - expect(track_action(event_action: :destroyed, event_target: wiki, author_id: 5)).to be_truthy - expect(track_action(event_action: :created, event_target: wiki, author_id: 3)).to be_truthy - expect(track_action(event_action: :updated, event_target: wiki, author_id: 4)).to be_truthy - expect(track_action(event_action: :pushed, event_target: wiki, author_id: 6)).to be_truthy + expect(track_event(event_action: :destroyed, event_target: wiki, author_id: 5)).to be_truthy + expect(track_event(event_action: :created, event_target: wiki, author_id: 3)).to be_truthy + expect(track_event(event_action: :updated, event_target: wiki, author_id: 4)).to be_truthy + expect(track_event(event_action: :pushed, event_target: wiki, author_id: 6)).to be_truthy - expect(count_unique_events(event_action: described_class::PUSH_ACTION, date_from: time, date_to: Date.today)).to eq(3) - expect(count_unique_events(event_action: described_class::PUSH_ACTION, date_from: time - 5.days, date_to: Date.tomorrow)).to eq(4) - expect(count_unique_events(event_action: described_class::DESIGN_ACTION, date_from: time - 5.days, date_to: Date.today)).to eq(3) - expect(count_unique_events(event_action: described_class::WIKI_ACTION, date_from: time - 5.days, date_to: Date.today)).to eq(3) - expect(count_unique_events(event_action: described_class::PUSH_ACTION, date_from: time - 5.days, date_to: time - 2.days)).to eq(1) + expect(count_unique(event_action: described_class::PUSH_ACTION, date_from: time, date_to: Date.today)).to eq(3) + expect(count_unique(event_action: described_class::PUSH_ACTION, date_from: time - 5.days, date_to: Date.tomorrow)).to eq(4) + expect(count_unique(event_action: described_class::DESIGN_ACTION, date_from: time - 5.days, date_to: Date.today)).to eq(3) + expect(count_unique(event_action: described_class::WIKI_ACTION, date_from: time - 5.days, date_to: Date.today)).to eq(3) + expect(count_unique(event_action: described_class::PUSH_ACTION, date_from: time - 5.days, date_to: time - 2.days)).to eq(1) end end end @@ -59,22 +58,20 @@ RSpec.describe Gitlab::UsageDataCounters::TrackUniqueActions, :clean_gitlab_redi context 'when tracking unsuccessfully' do using RSpec::Parameterized::TableSyntax - where(:feature_flag, :application_setting, :target, :action) do - true | true | Project | :invalid_action - false | true | Project | :pushed - true | false | Project | :pushed - true | true | :invalid_target | :pushed + where(:application_setting, :target, :action) do + true | Project | :invalid_action + false | Project | :pushed + true | :invalid_target | :pushed end with_them do before do stub_application_setting(usage_ping_enabled: application_setting) - stub_feature_flags(described_class::FEATURE_FLAG => feature_flag) end it 'returns the expected values' do - expect(track_action(event_action: action, event_target: target, author_id: 2)).to be_nil - expect(count_unique_events(event_action: described_class::PUSH_ACTION, date_from: time, date_to: Date.today)).to eq(0) + expect(track_event(event_action: action, event_target: target, author_id: 2)).to be_nil + expect(count_unique(event_action: described_class::PUSH_ACTION, date_from: time, date_to: Date.today)).to eq(0) end end end diff --git a/spec/lib/gitlab/usage_data_counters/wiki_page_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/wiki_page_counter_spec.rb index 685f6ea0a85..0c8f89fdf07 100644 --- a/spec/lib/gitlab/usage_data_counters/wiki_page_counter_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/wiki_page_counter_spec.rb @@ -3,11 +3,13 @@ require 'spec_helper' RSpec.describe Gitlab::UsageDataCounters::WikiPageCounter do + it_behaves_like 'a redis usage counter', 'Wiki Page', :view it_behaves_like 'a redis usage counter', 'Wiki Page', :create it_behaves_like 'a redis usage counter', 'Wiki Page', :update it_behaves_like 'a redis usage counter', 'Wiki Page', :delete it_behaves_like 'a redis usage counter with totals', :wiki_pages, + view: 8, create: 5, update: 3, delete: 2 diff --git a/spec/lib/gitlab/usage_data_spec.rb b/spec/lib/gitlab/usage_data_spec.rb index bca2f49eb33..3be8a770b2b 100644 --- a/spec/lib/gitlab/usage_data_spec.rb +++ b/spec/lib/gitlab/usage_data_spec.rb @@ -13,8 +13,14 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do describe '.uncached_data' do describe '.usage_activity_by_stage' do it 'includes usage_activity_by_stage data' do - expect(described_class.uncached_data).to include(:usage_activity_by_stage) - expect(described_class.uncached_data).to include(:usage_activity_by_stage_monthly) + uncached_data = described_class.uncached_data + + expect(uncached_data).to include(:usage_activity_by_stage) + expect(uncached_data).to include(:usage_activity_by_stage_monthly) + expect(uncached_data[:usage_activity_by_stage]) + .to include(:configure, :create, :manage, :monitor, :plan, :release, :verify) + expect(uncached_data[:usage_activity_by_stage_monthly]) + .to include(:configure, :create, :manage, :monitor, :plan, :release, :verify) end it 'clears memoized values' do @@ -30,269 +36,269 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do described_class.uncached_data end - context 'for configure' do - it 'includes accurate usage_activity_by_stage data' do - for_defined_days_back do - user = create(:user) - cluster = create(:cluster, user: user) - create(:clusters_applications_cert_manager, :installed, cluster: cluster) - create(:clusters_applications_helm, :installed, cluster: cluster) - create(:clusters_applications_ingress, :installed, cluster: cluster) - create(:clusters_applications_knative, :installed, cluster: cluster) - create(:cluster, :disabled, user: user) - create(:cluster_provider_gcp, :created) - create(:cluster_provider_aws, :created) - create(:cluster_platform_kubernetes) - create(:cluster, :group, :disabled, user: user) - create(:cluster, :group, user: user) - create(:cluster, :instance, :disabled, :production_environment) - create(:cluster, :instance, :production_environment) - create(:cluster, :management_project) - end + it 'merge_requests_users is included only in montly counters' do + uncached_data = described_class.uncached_data - expect(described_class.uncached_data[:usage_activity_by_stage][:configure]).to include( - clusters_applications_cert_managers: 2, - clusters_applications_helm: 2, - clusters_applications_ingress: 2, - clusters_applications_knative: 2, - clusters_management_project: 2, - clusters_disabled: 4, - clusters_enabled: 12, - clusters_platforms_gke: 2, - clusters_platforms_eks: 2, - clusters_platforms_user: 2, - instance_clusters_disabled: 2, - instance_clusters_enabled: 2, - group_clusters_disabled: 2, - group_clusters_enabled: 2, - project_clusters_disabled: 2, - project_clusters_enabled: 10 - ) - expect(described_class.uncached_data[:usage_activity_by_stage_monthly][:configure]).to include( - clusters_applications_cert_managers: 1, - clusters_applications_helm: 1, - clusters_applications_ingress: 1, - clusters_applications_knative: 1, - clusters_management_project: 1, - clusters_disabled: 2, - clusters_enabled: 6, - clusters_platforms_gke: 1, - clusters_platforms_eks: 1, - clusters_platforms_user: 1, - instance_clusters_disabled: 1, - instance_clusters_enabled: 1, - group_clusters_disabled: 1, - group_clusters_enabled: 1, - project_clusters_disabled: 1, - project_clusters_enabled: 5 - ) - end + expect(uncached_data[:usage_activity_by_stage][:create]) + .not_to include(:merge_requests_users) + expect(uncached_data[:usage_activity_by_stage_monthly][:create]) + .to include(:merge_requests_users) end + end - context 'for create' do - it 'include usage_activity_by_stage data' do - expect(described_class.uncached_data[:usage_activity_by_stage][:create]) - .not_to include( - :merge_requests_users - ) - end - - it 'includes monthly usage_activity_by_stage data' do - expect(described_class.uncached_data[:usage_activity_by_stage_monthly][:create]) - .to include( - :merge_requests_users - ) - end - - it 'includes accurate usage_activity_by_stage data' do - for_defined_days_back do - user = create(:user) - project = create(:project, :repository_private, - :test_repo, :remote_mirror, creator: user) - create(:merge_request, source_project: project) - create(:deploy_key, user: user) - create(:key, user: user) - create(:project, creator: user, disable_overriding_approvers_per_merge_request: true) - create(:project, creator: user, disable_overriding_approvers_per_merge_request: false) - create(:remote_mirror, project: project) - create(:snippet, author: user) - end - - expect(described_class.uncached_data[:usage_activity_by_stage][:create]).to include( - deploy_keys: 2, - keys: 2, - merge_requests: 2, - projects_with_disable_overriding_approvers_per_merge_request: 2, - projects_without_disable_overriding_approvers_per_merge_request: 4, - remote_mirrors: 2, - snippets: 2 - ) - expect(described_class.uncached_data[:usage_activity_by_stage_monthly][:create]).to include( - deploy_keys: 1, - keys: 1, - merge_requests: 1, - projects_with_disable_overriding_approvers_per_merge_request: 1, - projects_without_disable_overriding_approvers_per_merge_request: 2, - remote_mirrors: 1, - snippets: 1 - ) - end + it 'ensures recorded_at is set before any other usage data calculation' do + %i(alt_usage_data redis_usage_data distinct_count count).each do |method| + expect(described_class).not_to receive(method) end + expect(described_class).to receive(:recorded_at).and_raise(Exception.new('Stopped calculating recorded_at')) - context 'for manage' do - it 'includes accurate usage_activity_by_stage data' do - stub_config( - omniauth: - { providers: omniauth_providers } - ) - - for_defined_days_back do - user = create(:user) - create(:event, author: user) - create(:group_member, user: user) - end + expect { described_class.uncached_data }.to raise_error('Stopped calculating recorded_at') + end + end - expect(described_class.uncached_data[:usage_activity_by_stage][:manage]).to include( - events: 2, - groups: 2, - users_created: Gitlab.ee? ? 6 : 5, - omniauth_providers: ['google_oauth2'] - ) - expect(described_class.uncached_data[:usage_activity_by_stage_monthly][:manage]).to include( - events: 1, - groups: 1, - users_created: Gitlab.ee? ? 4 : 3, - omniauth_providers: ['google_oauth2'] - ) - end + describe '.usage_activity_by_stage_configure' do + it 'includes accurate usage_activity_by_stage data' do + for_defined_days_back do + user = create(:user) + cluster = create(:cluster, user: user) + create(:clusters_applications_cert_manager, :installed, cluster: cluster) + create(:clusters_applications_helm, :installed, cluster: cluster) + create(:clusters_applications_ingress, :installed, cluster: cluster) + create(:clusters_applications_knative, :installed, cluster: cluster) + create(:cluster, :disabled, user: user) + create(:cluster_provider_gcp, :created) + create(:cluster_provider_aws, :created) + create(:cluster_platform_kubernetes) + create(:cluster, :group, :disabled, user: user) + create(:cluster, :group, user: user) + create(:cluster, :instance, :disabled, :production_environment) + create(:cluster, :instance, :production_environment) + create(:cluster, :management_project) + end + + expect(described_class.usage_activity_by_stage_configure({})).to include( + clusters_applications_cert_managers: 2, + clusters_applications_helm: 2, + clusters_applications_ingress: 2, + clusters_applications_knative: 2, + clusters_management_project: 2, + clusters_disabled: 4, + clusters_enabled: 12, + clusters_platforms_gke: 2, + clusters_platforms_eks: 2, + clusters_platforms_user: 2, + instance_clusters_disabled: 2, + instance_clusters_enabled: 2, + group_clusters_disabled: 2, + group_clusters_enabled: 2, + project_clusters_disabled: 2, + project_clusters_enabled: 10 + ) + expect(described_class.usage_activity_by_stage_configure(described_class.last_28_days_time_period)).to include( + clusters_applications_cert_managers: 1, + clusters_applications_helm: 1, + clusters_applications_ingress: 1, + clusters_applications_knative: 1, + clusters_management_project: 1, + clusters_disabled: 2, + clusters_enabled: 6, + clusters_platforms_gke: 1, + clusters_platforms_eks: 1, + clusters_platforms_user: 1, + instance_clusters_disabled: 1, + instance_clusters_enabled: 1, + group_clusters_disabled: 1, + group_clusters_enabled: 1, + project_clusters_disabled: 1, + project_clusters_enabled: 5 + ) + end + end - def omniauth_providers - [ - OpenStruct.new(name: 'google_oauth2'), - OpenStruct.new(name: 'ldapmain'), - OpenStruct.new(name: 'group_saml') - ] - end - end + describe 'usage_activity_by_stage_create' do + it 'includes accurate usage_activity_by_stage data' do + for_defined_days_back do + user = create(:user) + project = create(:project, :repository_private, + :test_repo, :remote_mirror, creator: user) + create(:merge_request, source_project: project) + create(:deploy_key, user: user) + create(:key, user: user) + create(:project, creator: user, disable_overriding_approvers_per_merge_request: true) + create(:project, creator: user, disable_overriding_approvers_per_merge_request: false) + create(:remote_mirror, project: project) + create(:snippet, author: user) + end + + expect(described_class.usage_activity_by_stage_create({})).to include( + deploy_keys: 2, + keys: 2, + merge_requests: 2, + projects_with_disable_overriding_approvers_per_merge_request: 2, + projects_without_disable_overriding_approvers_per_merge_request: 4, + remote_mirrors: 2, + snippets: 2 + ) + expect(described_class.usage_activity_by_stage_create(described_class.last_28_days_time_period)).to include( + deploy_keys: 1, + keys: 1, + merge_requests: 1, + projects_with_disable_overriding_approvers_per_merge_request: 1, + projects_without_disable_overriding_approvers_per_merge_request: 2, + remote_mirrors: 1, + snippets: 1 + ) + end + end - context 'for monitor' do - it 'includes accurate usage_activity_by_stage data' do - for_defined_days_back do - user = create(:user, dashboard: 'operations') - cluster = create(:cluster, user: user) - create(:project, creator: user) - create(:clusters_applications_prometheus, :installed, cluster: cluster) - end + describe 'usage_activity_by_stage_manage' do + it 'includes accurate usage_activity_by_stage data' do + stub_config( + omniauth: + { providers: omniauth_providers } + ) - expect(described_class.uncached_data[:usage_activity_by_stage][:monitor]).to include( - clusters: 2, - clusters_applications_prometheus: 2, - operations_dashboard_default_dashboard: 2 - ) - expect(described_class.uncached_data[:usage_activity_by_stage_monthly][:monitor]).to include( - clusters: 1, - clusters_applications_prometheus: 1, - operations_dashboard_default_dashboard: 1 - ) - end + for_defined_days_back do + user = create(:user) + create(:event, author: user) + create(:group_member, user: user) end - context 'for plan' do - it 'includes accurate usage_activity_by_stage data' do - for_defined_days_back do - user = create(:user) - project = create(:project, creator: user) - issue = create(:issue, project: project, author: user) - create(:note, project: project, noteable: issue, author: user) - create(:todo, project: project, target: issue, author: user) - end - - expect(described_class.uncached_data[:usage_activity_by_stage][:plan]).to include( - issues: 2, - notes: 2, - projects: 2, - todos: 2 - ) - expect(described_class.uncached_data[:usage_activity_by_stage_monthly][:plan]).to include( - issues: 1, - notes: 1, - projects: 1, - todos: 1 - ) - end - end + expect(described_class.usage_activity_by_stage_manage({})).to include( + events: 2, + groups: 2, + users_created: 4, + omniauth_providers: ['google_oauth2'] + ) + expect(described_class.usage_activity_by_stage_manage(described_class.last_28_days_time_period)).to include( + events: 1, + groups: 1, + users_created: 2, + omniauth_providers: ['google_oauth2'] + ) + end - context 'for release' do - it 'includes accurate usage_activity_by_stage data' do - for_defined_days_back do - user = create(:user) - create(:deployment, :failed, user: user) - create(:release, author: user) - create(:deployment, :success, user: user) - end + def omniauth_providers + [ + OpenStruct.new(name: 'google_oauth2'), + OpenStruct.new(name: 'ldapmain'), + OpenStruct.new(name: 'group_saml') + ] + end + end - expect(described_class.uncached_data[:usage_activity_by_stage][:release]).to include( - deployments: 2, - failed_deployments: 2, - releases: 2, - successful_deployments: 2 - ) - expect(described_class.uncached_data[:usage_activity_by_stage_monthly][:release]).to include( - deployments: 1, - failed_deployments: 1, - releases: 1, - successful_deployments: 1 - ) - end + describe 'usage_activity_by_stage_monitor' do + it 'includes accurate usage_activity_by_stage data' do + for_defined_days_back do + user = create(:user, dashboard: 'operations') + cluster = create(:cluster, user: user) + create(:project, creator: user) + create(:clusters_applications_prometheus, :installed, cluster: cluster) end - context 'for verify' do - it 'includes accurate usage_activity_by_stage data' do - for_defined_days_back do - user = create(:user) - create(:ci_build, user: user) - create(:ci_empty_pipeline, source: :external, user: user) - create(:ci_empty_pipeline, user: user) - create(:ci_pipeline, :auto_devops_source, user: user) - create(:ci_pipeline, :repository_source, user: user) - create(:ci_pipeline_schedule, owner: user) - create(:ci_trigger, owner: user) - create(:clusters_applications_runner, :installed) - end + expect(described_class.usage_activity_by_stage_monitor({})).to include( + clusters: 2, + clusters_applications_prometheus: 2, + operations_dashboard_default_dashboard: 2 + ) + expect(described_class.usage_activity_by_stage_monitor(described_class.last_28_days_time_period)).to include( + clusters: 1, + clusters_applications_prometheus: 1, + operations_dashboard_default_dashboard: 1 + ) + end + end - expect(described_class.uncached_data[:usage_activity_by_stage][:verify]).to include( - ci_builds: 2, - ci_external_pipelines: 2, - ci_internal_pipelines: 2, - ci_pipeline_config_auto_devops: 2, - ci_pipeline_config_repository: 2, - ci_pipeline_schedules: 2, - ci_pipelines: 2, - ci_triggers: 2, - clusters_applications_runner: 2 - ) - expect(described_class.uncached_data[:usage_activity_by_stage_monthly][:verify]).to include( - ci_builds: 1, - ci_external_pipelines: 1, - ci_internal_pipelines: 1, - ci_pipeline_config_auto_devops: 1, - ci_pipeline_config_repository: 1, - ci_pipeline_schedules: 1, - ci_pipelines: 1, - ci_triggers: 1, - clusters_applications_runner: 1 - ) - end - end + describe 'usage_activity_by_stage_plan' do + it 'includes accurate usage_activity_by_stage data' do + for_defined_days_back do + user = create(:user) + project = create(:project, creator: user) + issue = create(:issue, project: project, author: user) + create(:issue, project: project, author: User.support_bot) + create(:note, project: project, noteable: issue, author: user) + create(:todo, project: project, target: issue, author: user) + end + + expect(described_class.usage_activity_by_stage_plan({})).to include( + issues: 3, + notes: 2, + projects: 2, + todos: 2, + service_desk_enabled_projects: 2, + service_desk_issues: 2 + ) + expect(described_class.usage_activity_by_stage_plan(described_class.last_28_days_time_period)).to include( + issues: 2, + notes: 1, + projects: 1, + todos: 1, + service_desk_enabled_projects: 1, + service_desk_issues: 1 + ) end + end - it 'ensures recorded_at is set before any other usage data calculation' do - %i(alt_usage_data redis_usage_data distinct_count count).each do |method| - expect(described_class).not_to receive(method) + describe 'usage_activity_by_stage_release' do + it 'includes accurate usage_activity_by_stage data' do + for_defined_days_back do + user = create(:user) + create(:deployment, :failed, user: user) + create(:release, author: user) + create(:deployment, :success, user: user) end - expect(described_class).to receive(:recorded_at).and_raise(Exception.new('Stopped calculating recorded_at')) - expect { described_class.uncached_data }.to raise_error('Stopped calculating recorded_at') + expect(described_class.usage_activity_by_stage_release({})).to include( + deployments: 2, + failed_deployments: 2, + releases: 2, + successful_deployments: 2 + ) + expect(described_class.usage_activity_by_stage_release(described_class.last_28_days_time_period)).to include( + deployments: 1, + failed_deployments: 1, + releases: 1, + successful_deployments: 1 + ) + end + end + + describe 'usage_activity_by_stage_verify' do + it 'includes accurate usage_activity_by_stage data' do + for_defined_days_back do + user = create(:user) + create(:ci_build, user: user) + create(:ci_empty_pipeline, source: :external, user: user) + create(:ci_empty_pipeline, user: user) + create(:ci_pipeline, :auto_devops_source, user: user) + create(:ci_pipeline, :repository_source, user: user) + create(:ci_pipeline_schedule, owner: user) + create(:ci_trigger, owner: user) + create(:clusters_applications_runner, :installed) + end + + expect(described_class.usage_activity_by_stage_verify({})).to include( + ci_builds: 2, + ci_external_pipelines: 2, + ci_internal_pipelines: 2, + ci_pipeline_config_auto_devops: 2, + ci_pipeline_config_repository: 2, + ci_pipeline_schedules: 2, + ci_pipelines: 2, + ci_triggers: 2, + clusters_applications_runner: 2 + ) + expect(described_class.usage_activity_by_stage_verify(described_class.last_28_days_time_period)).to include( + ci_builds: 1, + ci_external_pipelines: 1, + ci_internal_pipelines: 1, + ci_pipeline_config_auto_devops: 1, + ci_pipeline_config_repository: 1, + ci_pipeline_schedules: 1, + ci_pipelines: 1, + ci_triggers: 1, + clusters_applications_runner: 1 + ) end end @@ -338,13 +344,18 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do expect(count_data[:projects_slack_active]).to eq(2) expect(count_data[:projects_slack_slash_commands_active]).to eq(1) expect(count_data[:projects_custom_issue_tracker_active]).to eq(1) - expect(count_data[:projects_mattermost_active]).to eq(0) + expect(count_data[:projects_mattermost_active]).to eq(1) + expect(count_data[:templates_mattermost_active]).to eq(1) + expect(count_data[:instances_mattermost_active]).to eq(1) + expect(count_data[:projects_inheriting_instance_mattermost_active]).to eq(1) expect(count_data[:projects_with_repositories_enabled]).to eq(3) expect(count_data[:projects_with_error_tracking_enabled]).to eq(1) expect(count_data[:projects_with_alerts_service_enabled]).to eq(1) expect(count_data[:projects_with_prometheus_alerts]).to eq(2) expect(count_data[:projects_with_terraform_reports]).to eq(2) expect(count_data[:projects_with_terraform_states]).to eq(2) + expect(count_data[:protected_branches]).to eq(2) + expect(count_data[:protected_branches_except_default]).to eq(1) expect(count_data[:terraform_reports]).to eq(6) expect(count_data[:terraform_states]).to eq(3) expect(count_data[:issues_created_from_gitlab_error_tracking_ui]).to eq(1) @@ -574,9 +585,8 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do subject { described_class.components_usage_data } it 'gathers basic components usage data' do - stub_runtime(:puma) + stub_application_setting(container_registry_vendor: 'gitlab', container_registry_version: 'x.y.z') - expect(subject[:app_server][:type]).to eq('puma') expect(subject[:gitlab_pages][:enabled]).to eq(Gitlab.config.pages.enabled) expect(subject[:gitlab_pages][:version]).to eq(Gitlab::Pages::VERSION) expect(subject[:git][:version]).to eq(Gitlab::Git.version) @@ -587,32 +597,8 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do expect(subject[:gitaly][:clusters]).to be >= 0 expect(subject[:gitaly][:filesystems]).to be_an(Array) expect(subject[:gitaly][:filesystems].first).to be_a(String) - end - - def stub_runtime(runtime) - allow(Gitlab::Runtime).to receive(:identify).and_return(runtime) - end - end - - describe '.app_server_type' do - subject { described_class.app_server_type } - - it 'successfully identifies runtime and returns the identifier' do - expect(Gitlab::Runtime).to receive(:identify).and_return(:runtime_identifier) - - is_expected.to eq('runtime_identifier') - end - - context 'when runtime is not identified' do - let(:exception) { Gitlab::Runtime::IdentificationError.new('exception message from runtime identify') } - - it 'logs the exception and returns unknown app server type' do - expect(Gitlab::Runtime).to receive(:identify).and_raise(exception) - - expect(Gitlab::AppLogger).to receive(:error).with(exception.message) - expect(Gitlab::ErrorTracking).to receive(:track_exception).with(exception) - expect(subject).to eq('unknown_app_server_type') - end + expect(subject[:container_registry_server][:vendor]).to eq('gitlab') + expect(subject[:container_registry_server][:version]).to eq('x.y.z') end end @@ -926,45 +912,29 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do let(:time) { Time.zone.now } before do - stub_feature_flags(Gitlab::UsageDataCounters::TrackUniqueActions::FEATURE_FLAG => feature_flag) - end - - context 'when the feature flag is enabled' do - let(:feature_flag) { true } - - before do - counter = Gitlab::UsageDataCounters::TrackUniqueActions - project = Event::TARGET_TYPES[:project] - wiki = Event::TARGET_TYPES[:wiki] - design = Event::TARGET_TYPES[:design] - - counter.track_action(event_action: :pushed, event_target: project, author_id: 1) - counter.track_action(event_action: :pushed, event_target: project, author_id: 1) - counter.track_action(event_action: :pushed, event_target: project, author_id: 2) - counter.track_action(event_action: :pushed, event_target: project, author_id: 3) - counter.track_action(event_action: :pushed, event_target: project, author_id: 4, time: time - 3.days) - counter.track_action(event_action: :created, event_target: project, author_id: 5, time: time - 3.days) - counter.track_action(event_action: :created, event_target: wiki, author_id: 3) - counter.track_action(event_action: :created, event_target: design, author_id: 3) - end - - it 'returns the distinct count of user actions within the specified time period' do - expect(described_class.action_monthly_active_users(time_period)).to eq( - { - action_monthly_active_users_design_management: 1, - action_monthly_active_users_project_repo: 3, - action_monthly_active_users_wiki_repo: 1 - } - ) - end - end - - context 'when the feature flag is disabled' do - let(:feature_flag) { false } - - it 'returns an empty hash' do - expect(described_class.action_monthly_active_users(time_period)).to eq({}) - end + counter = Gitlab::UsageDataCounters::TrackUniqueActions + project = Event::TARGET_TYPES[:project] + wiki = Event::TARGET_TYPES[:wiki] + design = Event::TARGET_TYPES[:design] + + counter.track_event(event_action: :pushed, event_target: project, author_id: 1) + counter.track_event(event_action: :pushed, event_target: project, author_id: 1) + counter.track_event(event_action: :pushed, event_target: project, author_id: 2) + counter.track_event(event_action: :pushed, event_target: project, author_id: 3) + counter.track_event(event_action: :pushed, event_target: project, author_id: 4, time: time - 3.days) + counter.track_event(event_action: :created, event_target: project, author_id: 5, time: time - 3.days) + counter.track_event(event_action: :created, event_target: wiki, author_id: 3) + counter.track_event(event_action: :created, event_target: design, author_id: 3) + end + + it 'returns the distinct count of user actions within the specified time period' do + expect(described_class.action_monthly_active_users(time_period)).to eq( + { + action_monthly_active_users_design_management: 1, + action_monthly_active_users_project_repo: 3, + action_monthly_active_users_wiki_repo: 1 + } + ) end end @@ -972,11 +942,12 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do subject { described_class.analytics_unique_visits_data } it 'returns the number of unique visits to pages with analytics features' do - ::Gitlab::Analytics::UniqueVisits::TARGET_IDS.each do |target_id| - expect_any_instance_of(::Gitlab::Analytics::UniqueVisits).to receive(:weekly_unique_visits_for_target).with(target_id).and_return(123) + ::Gitlab::Analytics::UniqueVisits.analytics_ids.each do |target_id| + expect_any_instance_of(::Gitlab::Analytics::UniqueVisits).to receive(:unique_visits_for).with(targets: target_id).and_return(123) end - expect_any_instance_of(::Gitlab::Analytics::UniqueVisits).to receive(:weekly_unique_visits_for_any_target).and_return(543) + expect_any_instance_of(::Gitlab::Analytics::UniqueVisits).to receive(:unique_visits_for).with(targets: :analytics).and_return(543) + expect_any_instance_of(::Gitlab::Analytics::UniqueVisits).to receive(:unique_visits_for).with(targets: :analytics, start_date: 4.weeks.ago.to_date, end_date: Date.current).and_return(987) expect(subject).to eq({ analytics_unique_visits: { @@ -991,12 +962,56 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do 'p_analytics_insights' => 123, 'p_analytics_issues' => 123, 'p_analytics_repo' => 123, - 'u_analytics_todos' => 123, 'i_analytics_cohorts' => 123, 'i_analytics_dev_ops_score' => 123, - 'analytics_unique_visits_for_any_target' => 543 + 'analytics_unique_visits_for_any_target' => 543, + 'analytics_unique_visits_for_any_target_monthly' => 987 + } + }) + end + end + + describe '.compliance_unique_visits_data' do + subject { described_class.compliance_unique_visits_data } + + before do + described_class.clear_memoization(:unique_visit_service) + + allow_next_instance_of(::Gitlab::Analytics::UniqueVisits) do |instance| + ::Gitlab::Analytics::UniqueVisits.compliance_ids.each do |target_id| + allow(instance).to receive(:unique_visits_for).with(targets: target_id).and_return(123) + end + + allow(instance).to receive(:unique_visits_for).with(targets: :compliance).and_return(543) + + allow(instance).to receive(:unique_visits_for).with(targets: :compliance, start_date: 4.weeks.ago.to_date, end_date: Date.current).and_return(987) + end + end + + it 'returns the number of unique visits to pages with compliance features' do + expect(subject).to eq({ + compliance_unique_visits: { + 'g_compliance_dashboard' => 123, + 'g_compliance_audit_events' => 123, + 'i_compliance_credential_inventory' => 123, + 'i_compliance_audit_events' => 123, + 'compliance_unique_visits_for_any_target' => 543, + 'compliance_unique_visits_for_any_target_monthly' => 987 } }) end end + + describe '.service_desk_counts' do + subject { described_class.send(:service_desk_counts) } + + let(:project) { create(:project, :service_desk_enabled) } + + it 'gathers Service Desk data' do + create_list(:issue, 2, :confidential, author: User.support_bot, project: project) + + expect(subject).to eq(service_desk_enabled_projects: 1, + service_desk_issues: 2) + end + end end diff --git a/spec/lib/gitlab/user_access_spec.rb b/spec/lib/gitlab/user_access_spec.rb index 1a81d0127dc..d6b1e3b2d4b 100644 --- a/spec/lib/gitlab/user_access_spec.rb +++ b/spec/lib/gitlab/user_access_spec.rb @@ -5,7 +5,7 @@ require 'spec_helper' RSpec.describe Gitlab::UserAccess do include ProjectForksHelper - let(:access) { described_class.new(user, project: project) } + let(:access) { described_class.new(user, container: project) } let(:project) { create(:project, :repository) } let(:user) { create(:user) } @@ -43,7 +43,7 @@ RSpec.describe Gitlab::UserAccess do describe 'push to empty project' do let(:empty_project) { create(:project_empty_repo) } - let(:project_access) { described_class.new(user, project: empty_project) } + let(:project_access) { described_class.new(user, container: empty_project) } it 'returns true for admins' do user.update!(admin: true) diff --git a/spec/lib/gitlab/utils/usage_data_spec.rb b/spec/lib/gitlab/utils/usage_data_spec.rb index 7940c9af6ff..4675cbd7fa1 100644 --- a/spec/lib/gitlab/utils/usage_data_spec.rb +++ b/spec/lib/gitlab/utils/usage_data_spec.rb @@ -122,7 +122,7 @@ RSpec.describe Gitlab::Utils::UsageData do freeze_time do result = described_class.with_finished_at(:current_time) { { a: 1 } } - expect(result).to eq(a: 1, current_time: Time.now) + expect(result).to eq(a: 1, current_time: Time.current) end end end diff --git a/spec/lib/gitlab/utils_spec.rb b/spec/lib/gitlab/utils_spec.rb index 7a0d40ff0d2..1eaceec1d8a 100644 --- a/spec/lib/gitlab/utils_spec.rb +++ b/spec/lib/gitlab/utils_spec.rb @@ -5,39 +5,93 @@ require 'spec_helper' RSpec.describe Gitlab::Utils do delegate :to_boolean, :boolean_to_yes_no, :slugify, :random_string, :which, :ensure_array_from_string, :to_exclusive_sentence, :bytes_to_megabytes, - :append_path, :check_path_traversal!, :ms_to_round_sec, to: :described_class + :append_path, :check_path_traversal!, :allowlisted?, :check_allowed_absolute_path!, :decode_path, :ms_to_round_sec, to: :described_class describe '.check_path_traversal!' do + it 'detects path traversal in string without any separators' do + expect { check_path_traversal!('.') }.to raise_error(/Invalid path/) + expect { check_path_traversal!('..') }.to raise_error(/Invalid path/) + end + it 'detects path traversal at the start of the string' do expect { check_path_traversal!('../foo') }.to raise_error(/Invalid path/) + expect { check_path_traversal!('..\\foo') }.to raise_error(/Invalid path/) end it 'detects path traversal at the start of the string, even to just the subdirectory' do expect { check_path_traversal!('../') }.to raise_error(/Invalid path/) + expect { check_path_traversal!('..\\') }.to raise_error(/Invalid path/) + expect { check_path_traversal!('/../') }.to raise_error(/Invalid path/) + expect { check_path_traversal!('\\..\\') }.to raise_error(/Invalid path/) end it 'detects path traversal in the middle of the string' do expect { check_path_traversal!('foo/../../bar') }.to raise_error(/Invalid path/) + expect { check_path_traversal!('foo\\..\\..\\bar') }.to raise_error(/Invalid path/) + expect { check_path_traversal!('foo/..\\bar') }.to raise_error(/Invalid path/) + expect { check_path_traversal!('foo\\../bar') }.to raise_error(/Invalid path/) + expect { check_path_traversal!('foo/..\\..\\..\\..\\../bar') }.to raise_error(/Invalid path/) end it 'detects path traversal at the end of the string when slash-terminates' do expect { check_path_traversal!('foo/../') }.to raise_error(/Invalid path/) + expect { check_path_traversal!('foo\\..\\') }.to raise_error(/Invalid path/) end it 'detects path traversal at the end of the string' do expect { check_path_traversal!('foo/..') }.to raise_error(/Invalid path/) + expect { check_path_traversal!('foo\\..') }.to raise_error(/Invalid path/) end it 'does nothing for a safe string' do expect(check_path_traversal!('./foo')).to eq('./foo') + expect(check_path_traversal!('.test/foo')).to eq('.test/foo') + expect(check_path_traversal!('..test/foo')).to eq('..test/foo') + expect(check_path_traversal!('dir/..foo.rb')).to eq('dir/..foo.rb') + expect(check_path_traversal!('dir/.foo.rb')).to eq('dir/.foo.rb') + end + end + + describe '.allowlisted?' do + let(:allowed_paths) { ['/home/foo', '/foo/bar', '/etc/passwd']} + + it 'returns true if path is allowed' do + expect(allowlisted?('/foo/bar', allowed_paths)).to be(true) + end + + it 'returns false if path is not allowed' do + expect(allowlisted?('/test/test', allowed_paths)).to be(false) + end + end + + describe '.check_allowed_absolute_path!' do + let(:allowed_paths) { ['/home/foo'] } + + it 'raises an exception if an absolute path is not allowed' do + expect { check_allowed_absolute_path!('/etc/passwd', allowed_paths) }.to raise_error(StandardError) end - it 'does nothing if an absolute path is allowed' do - expect(check_path_traversal!('/etc/folder/path', allowed_absolute: true)). to eq('/etc/folder/path') + it 'does nothing for an allowed absolute path' do + expect(check_allowed_absolute_path!('/home/foo', allowed_paths)).to be_nil end + end - it 'raises exception if an absolute path is not allowed' do - expect { check_path_traversal!('/etc/folder/path') }.to raise_error(/Invalid path/) + describe '.decode_path' do + it 'returns path unencoded for singled-encoded paths' do + expect(decode_path('%2Fhome%2Fbar%3Fasd%3Dqwe')).to eq('/home/bar?asd=qwe') + end + + it 'returns path when it is unencoded' do + expect(decode_path('/home/bar?asd=qwe')).to eq('/home/bar?asd=qwe') + end + + [ + '..%252F..%252F..%252Fetc%252Fpasswd', + '%25252Fresult%25252Fchosennickname%25253D%252522jj%252522' + ].each do |multiple_encoded_path| + it 'raises an exception when the path is multiple-encoded' do + expect { decode_path(multiple_encoded_path) }.to raise_error(/path #{multiple_encoded_path} is not allowed/) + end end end diff --git a/spec/lib/gitlab/view/presenter/base_spec.rb b/spec/lib/gitlab/view/presenter/base_spec.rb index 1ab6973e279..97d5e2b280d 100644 --- a/spec/lib/gitlab/view/presenter/base_spec.rb +++ b/spec/lib/gitlab/view/presenter/base_spec.rb @@ -57,4 +57,32 @@ RSpec.describe Gitlab::View::Presenter::Base do expect(presenter.present).to eq(presenter) end end + + describe '#url_builder' do + it 'returns the UrlBuilder instance' do + presenter = presenter_class.new(project) + + expect(presenter.url_builder).to eq(Gitlab::UrlBuilder.instance) + end + end + + describe '#web_url' do + it 'delegates to the UrlBuilder' do + presenter = presenter_class.new(project) + + expect(presenter.url_builder).to receive(:build).with(project) + + presenter.web_url + end + end + + describe '#web_path' do + it 'delegates to the UrlBuilder' do + presenter = presenter_class.new(project) + + expect(presenter.url_builder).to receive(:build).with(project, only_path: true) + + presenter.web_path + end + end end diff --git a/spec/lib/gitlab/workhorse_spec.rb b/spec/lib/gitlab/workhorse_spec.rb index 5cb08ac1e76..da327ce7706 100644 --- a/spec/lib/gitlab/workhorse_spec.rb +++ b/spec/lib/gitlab/workhorse_spec.rb @@ -421,6 +421,24 @@ RSpec.describe Gitlab::Workhorse do end end + describe '.send_scaled_image' do + let(:location) { 'http://example.com/avatar.png' } + let(:width) { '150' } + + subject { described_class.send_scaled_image(location, width) } + + it 'sets the header correctly' do + key, command, params = decode_workhorse_header(subject) + + expect(key).to eq("Gitlab-Workhorse-Send-Data") + expect(command).to eq("send-scaled-img") + expect(params).to eq({ + 'Location' => location, + 'Width' => width + }.deep_stringify_keys) + end + end + describe '.send_git_snapshot' do let(:url) { 'http://example.com' } diff --git a/spec/lib/json_web_token/rsa_token_spec.rb b/spec/lib/json_web_token/rsa_token_spec.rb index f19471917c2..8f0d62d8f0c 100644 --- a/spec/lib/json_web_token/rsa_token_spec.rb +++ b/spec/lib/json_web_token/rsa_token_spec.rb @@ -14,6 +14,7 @@ RSpec.describe JSONWebToken::RSAToken do -----END RSA PRIVATE KEY----- eos end + let(:rsa_token) { described_class.new(nil) } let(:rsa_encoded) { rsa_token.encoded } diff --git a/spec/lib/mattermost/session_spec.rb b/spec/lib/mattermost/session_spec.rb index 5110d3cdfa3..93422b01ca7 100644 --- a/spec/lib/mattermost/session_spec.rb +++ b/spec/lib/mattermost/session_spec.rb @@ -61,6 +61,7 @@ RSpec.describe Mattermost::Session, type: :request do redirect_uri: "#{mattermost_url}/signup/gitlab/complete", state: state } end + let(:location) do "#{gitlab_url}/oauth/authorize?#{URI.encode_www_form(params)}" end diff --git a/spec/lib/object_storage/config_spec.rb b/spec/lib/object_storage/config_spec.rb new file mode 100644 index 00000000000..a48b5100065 --- /dev/null +++ b/spec/lib/object_storage/config_spec.rb @@ -0,0 +1,179 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' +require 'rspec-parameterized' + +RSpec.describe ObjectStorage::Config do + using RSpec::Parameterized::TableSyntax + + let(:region) { 'us-east-1' } + let(:bucket_name) { 'test-bucket' } + let(:credentials) do + { + provider: 'AWS', + aws_access_key_id: 'AWS_ACCESS_KEY_ID', + aws_secret_access_key: 'AWS_SECRET_ACCESS_KEY', + region: region + } + end + + let(:storage_options) do + { + server_side_encryption: 'AES256', + server_side_encryption_kms_key_id: 'arn:aws:12345' + } + end + + let(:raw_config) do + { + enabled: true, + connection: credentials, + remote_directory: bucket_name, + storage_options: storage_options + } + end + + subject { described_class.new(raw_config.as_json) } + + describe '#credentials' do + it { expect(subject.credentials).to eq(credentials) } + end + + describe '#storage_options' do + it { expect(subject.storage_options).to eq(storage_options) } + end + + describe '#enabled?' do + it { expect(subject.enabled?).to eq(true) } + end + + describe '#bucket' do + it { expect(subject.bucket).to eq(bucket_name) } + end + + describe '#use_iam_profile' do + it { expect(subject.use_iam_profile?).to be false } + end + + describe '#use_path_style' do + it { expect(subject.use_path_style?).to be false } + end + + context 'with unconsolidated settings' do + describe 'consolidated_settings? returns false' do + it { expect(subject.consolidated_settings?).to be false } + end + end + + context 'with consolidated settings' do + before do + raw_config[:consolidated_settings] = true + end + + describe 'consolidated_settings? returns true' do + it { expect(subject.consolidated_settings?).to be true } + end + end + + context 'with IAM profile configured' do + where(:value, :expected) do + true | true + "true" | true + "yes" | true + false | false + "false" | false + "no" | false + nil | false + end + + with_them do + before do + credentials[:use_iam_profile] = value + end + + it 'coerces the value to a boolean' do + expect(subject.use_iam_profile?).to be expected + end + end + end + + context 'with path style configured' do + where(:value, :expected) do + true | true + "true" | true + "yes" | true + false | false + "false" | false + "no" | false + nil | false + end + + with_them do + before do + credentials[:path_style] = value + end + + it 'coerces the value to a boolean' do + expect(subject.use_path_style?).to be expected + end + end + end + + context 'with hostname style access' do + it '#use_path_style? returns false' do + expect(subject.use_path_style?).to be false + end + end + + context 'with AWS credentials' do + it { expect(subject.provider).to eq('AWS') } + it { expect(subject.aws?).to be true } + it { expect(subject.google?).to be false } + end + + context 'with Google credentials' do + let(:credentials) do + { + provider: 'Google', + google_client_email: 'foo@gcp-project.example.com', + google_json_key_location: '/path/to/gcp.json' + } + end + + it { expect(subject.provider).to eq('Google') } + it { expect(subject.aws?).to be false } + it { expect(subject.google?).to be true } + it { expect(subject.fog_attributes).to eq({}) } + end + + context 'with SSE-KMS enabled' do + it { expect(subject.server_side_encryption).to eq('AES256') } + it { expect(subject.server_side_encryption_kms_key_id).to eq('arn:aws:12345') } + it { expect(subject.fog_attributes.keys).to match_array(%w(x-amz-server-side-encryption x-amz-server-side-encryption-aws-kms-key-id)) } + end + + context 'with only server side encryption enabled' do + let(:storage_options) { { server_side_encryption: 'AES256' } } + + it { expect(subject.server_side_encryption).to eq('AES256') } + it { expect(subject.server_side_encryption_kms_key_id).to be_nil } + it { expect(subject.fog_attributes).to eq({ 'x-amz-server-side-encryption' => 'AES256' }) } + end + + context 'without encryption enabled' do + let(:storage_options) { {} } + + it { expect(subject.server_side_encryption).to be_nil } + it { expect(subject.server_side_encryption_kms_key_id).to be_nil } + it { expect(subject.fog_attributes).to eq({}) } + end + + context 'with object storage disabled' do + before do + raw_config['enabled'] = false + end + + it { expect(subject.enabled?).to be false } + it { expect(subject.fog_attributes).to eq({}) } + end +end diff --git a/spec/lib/object_storage/direct_upload_spec.rb b/spec/lib/object_storage/direct_upload_spec.rb index 1c1455e2456..b11926aeb49 100644 --- a/spec/lib/object_storage/direct_upload_spec.rb +++ b/spec/lib/object_storage/direct_upload_spec.rb @@ -18,13 +18,25 @@ RSpec.describe ObjectStorage::DirectUpload do } end + let(:storage_options) { {} } + let(:raw_config) do + { + enabled: true, + connection: credentials, + remote_directory: bucket_name, + storage_options: storage_options, + consolidated_settings: consolidated_settings + } + end + + let(:config) { ObjectStorage::Config.new(raw_config) } let(:storage_url) { 'https://uploads.s3.amazonaws.com/' } let(:bucket_name) { 'uploads' } let(:object_name) { 'tmp/uploads/my-file' } let(:maximum_size) { 1.gigabyte } - let(:direct_upload) { described_class.new(credentials, bucket_name, object_name, has_length: has_length, maximum_size: maximum_size, consolidated_settings: consolidated_settings) } + let(:direct_upload) { described_class.new(config, object_name, has_length: has_length, maximum_size: maximum_size) } before do Fog.unmock! @@ -62,7 +74,7 @@ RSpec.describe ObjectStorage::DirectUpload do end describe '#get_url' do - subject { described_class.new(credentials, bucket_name, object_name, has_length: true) } + subject { described_class.new(config, object_name, has_length: true) } context 'when AWS is used' do it 'calls the proper method' do @@ -93,7 +105,7 @@ RSpec.describe ObjectStorage::DirectUpload do end end - describe '#to_hash' do + describe '#to_hash', :aggregate_failures do subject { direct_upload.to_hash } shared_examples 'a valid S3 upload' do @@ -111,6 +123,7 @@ RSpec.describe ObjectStorage::DirectUpload do expect(s3_config[:Region]).to eq(region) expect(s3_config[:PathStyle]).to eq(path_style) expect(s3_config[:UseIamProfile]).to eq(use_iam_profile) + expect(s3_config.keys).not_to include(%i(ServerSideEncryption SSEKMSKeyID)) end context 'when feature flag is disabled' do @@ -150,6 +163,33 @@ RSpec.describe ObjectStorage::DirectUpload do expect(subject[:UseWorkhorseClient]).to be true end end + + context 'when only server side encryption is used' do + let(:storage_options) { { server_side_encryption: 'AES256' } } + + it 'sends server side encryption settings' do + s3_config = subject[:ObjectStorage][:S3Config] + + expect(s3_config[:ServerSideEncryption]).to eq('AES256') + expect(s3_config.keys).not_to include(:SSEKMSKeyID) + end + end + + context 'when SSE-KMS is used' do + let(:storage_options) do + { + server_side_encryption: 'AES256', + server_side_encryption_kms_key_id: 'arn:aws:12345' + } + end + + it 'sends server side encryption settings' do + s3_config = subject[:ObjectStorage][:S3Config] + + expect(s3_config[:ServerSideEncryption]).to eq('AES256') + expect(s3_config[:SSEKMSKeyID]).to eq('arn:aws:12345') + end + end end shared_examples 'a valid Google upload' do @@ -160,6 +200,21 @@ RSpec.describe ObjectStorage::DirectUpload do end end + shared_examples 'a valid AzureRM upload' do + before do + require 'fog/azurerm' + end + + it_behaves_like 'a valid upload' + + it 'enables the Workhorse client' do + expect(subject[:UseWorkhorseClient]).to be true + expect(subject[:RemoteTempObjectID]).to eq(object_name) + expect(subject[:ObjectStorage][:Provider]).to eq('AzureRM') + expect(subject[:ObjectStorage][:GoCloudConfig]).to eq({ URL: "azblob://#{bucket_name}" }) + end + end + shared_examples 'a valid upload' do it "returns valid structure" do expect(subject).to have_key(:Timeout) @@ -330,5 +385,31 @@ RSpec.describe ObjectStorage::DirectUpload do it_behaves_like 'a valid upload without multipart data' end end + + context 'when AzureRM is used' do + let(:credentials) do + { + provider: 'AzureRM', + azure_storage_account_name: 'azuretest', + azure_storage_access_key: 'ABCD1234' + } + end + + let(:storage_url) { 'https://azuretest.blob.core.windows.net' } + + context 'when length is known' do + let(:has_length) { true } + + it_behaves_like 'a valid AzureRM upload' + it_behaves_like 'a valid upload without multipart data' + end + + context 'when length is unknown' do + let(:has_length) { false } + + it_behaves_like 'a valid AzureRM upload' + it_behaves_like 'a valid upload without multipart data' + end + end end end diff --git a/spec/lib/omni_auth/strategies/jwt_spec.rb b/spec/lib/omni_auth/strategies/jwt_spec.rb index 0f4528d4fbe..b29e48b0de5 100644 --- a/spec/lib/omni_auth/strategies/jwt_spec.rb +++ b/spec/lib/omni_auth/strategies/jwt_spec.rb @@ -19,6 +19,7 @@ RSpec.describe OmniAuth::Strategies::Jwt do iat: timestamp } end + let(:algorithm) { 'HS256' } let(:secret) { jwt_config.strategy.secret } let(:private_key) { secret } @@ -61,6 +62,7 @@ RSpec.describe OmniAuth::Strategies::Jwt do private_key_class.new(jwt_config.strategy.secret) end end + let(:private_key) { private_key_class ? private_key_class.new(secret) : secret } it 'decodes the user information' do diff --git a/spec/lib/product_analytics/event_params_spec.rb b/spec/lib/product_analytics/event_params_spec.rb index d6c098599d6..e560fd10dfd 100644 --- a/spec/lib/product_analytics/event_params_spec.rb +++ b/spec/lib/product_analytics/event_params_spec.rb @@ -23,7 +23,12 @@ RSpec.describe ProductAnalytics::EventParams do br_lang: 'en-US', br_cookies: true, os_timezone: 'America/Los_Angeles', - doc_charset: 'UTF-8' + doc_charset: 'UTF-8', + se_category: 'category', + se_action: 'action', + se_label: 'label', + se_property: 'property', + se_value: 12.34 } expect(subject).to include(expected_params) diff --git a/spec/lib/product_analytics/tracker_spec.rb b/spec/lib/product_analytics/tracker_spec.rb new file mode 100644 index 00000000000..d5e85e6e1cd --- /dev/null +++ b/spec/lib/product_analytics/tracker_spec.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require "spec_helper" + +RSpec.describe ProductAnalytics::Tracker do + it { expect(described_class::URL).to eq('http://localhost/-/sp.js') } + it { expect(described_class::COLLECTOR_URL).to eq('localhost/-/collector') } +end diff --git a/spec/lib/rspec_flaky/example_spec.rb b/spec/lib/rspec_flaky/example_spec.rb index aaf5ddc6f74..4b45a15c463 100644 --- a/spec/lib/rspec_flaky/example_spec.rb +++ b/spec/lib/rspec_flaky/example_spec.rb @@ -15,6 +15,7 @@ RSpec.describe RspecFlaky::Example do attempts: 1 } end + let(:rspec_example) { double(example_attrs) } describe '#initialize' do diff --git a/spec/lib/rspec_flaky/flaky_example_spec.rb b/spec/lib/rspec_flaky/flaky_example_spec.rb index 8ac323475d6..b1647d5830a 100644 --- a/spec/lib/rspec_flaky/flaky_example_spec.rb +++ b/spec/lib/rspec_flaky/flaky_example_spec.rb @@ -16,6 +16,7 @@ RSpec.describe RspecFlaky::FlakyExample, :aggregate_failures do flaky_reports: 1 } end + let(:example_attrs) do { uid: 'abc123', @@ -28,6 +29,7 @@ RSpec.describe RspecFlaky::FlakyExample, :aggregate_failures do attempts: flaky_example_attrs[:last_attempts_count] } end + let(:example) { double(example_attrs) } before do diff --git a/spec/lib/rspec_flaky/flaky_examples_collection_spec.rb b/spec/lib/rspec_flaky/flaky_examples_collection_spec.rb index 5718d8211af..b2fd1d3733a 100644 --- a/spec/lib/rspec_flaky/flaky_examples_collection_spec.rb +++ b/spec/lib/rspec_flaky/flaky_examples_collection_spec.rb @@ -9,6 +9,7 @@ RSpec.describe RspecFlaky::FlakyExamplesCollection, :aggregate_failures do b: { example_id: 'spec/foo/baz_spec.rb:3' } } end + let(:collection_report) do { a: { diff --git a/spec/lib/rspec_flaky/listener_spec.rb b/spec/lib/rspec_flaky/listener_spec.rb index 2438ae171d3..10ed724d4de 100644 --- a/spec/lib/rspec_flaky/listener_spec.rb +++ b/spec/lib/rspec_flaky/listener_spec.rb @@ -19,6 +19,7 @@ RSpec.describe RspecFlaky::Listener, :aggregate_failures do } } end + let(:already_flaky_example_attrs) do { id: 'spec/foo/bar_spec.rb:2', @@ -30,6 +31,7 @@ RSpec.describe RspecFlaky::Listener, :aggregate_failures do execution_result: double(status: 'passed', exception: nil) } end + let(:already_flaky_example) { RspecFlaky::FlakyExample.new(suite_flaky_example_report[already_flaky_example_uid]) } let(:new_example_attrs) do { diff --git a/spec/lib/rspec_flaky/report_spec.rb b/spec/lib/rspec_flaky/report_spec.rb index e735329a8a3..5cacfdb82fb 100644 --- a/spec/lib/rspec_flaky/report_spec.rb +++ b/spec/lib/rspec_flaky/report_spec.rb @@ -10,6 +10,7 @@ RSpec.describe RspecFlaky::Report, :aggregate_failures do b: { example_id: 'spec/foo/baz_spec.rb:3', first_flaky_at: (Time.now - thirty_one_days).to_s, last_flaky_at: (Time.now - thirty_one_days).to_s } } end + let(:suite_flaky_example_report) do { '6e869794f4cfd2badd93eb68719371d1': { @@ -25,6 +26,7 @@ RSpec.describe RspecFlaky::Report, :aggregate_failures do } } end + let(:flaky_examples) { RspecFlaky::FlakyExamplesCollection.new(collection_hash) } let(:report) { described_class.new(flaky_examples) } diff --git a/spec/lib/sentry/client/event_spec.rb b/spec/lib/sentry/client/event_spec.rb index af1e28d09bb..07ed331c44c 100644 --- a/spec/lib/sentry/client/event_spec.rb +++ b/spec/lib/sentry/client/event_spec.rb @@ -13,6 +13,7 @@ RSpec.describe Sentry::Client do headers: { "Authorization" => "Bearer test-token" } } end + let(:client) { described_class.new(sentry_url, token) } describe '#issue_latest_event' do @@ -21,6 +22,7 @@ RSpec.describe Sentry::Client do Gitlab::Json.parse(fixture_file('sentry/issue_latest_event_sample_response.json')) ) end + let(:issue_id) { '1234' } let(:sentry_api_response) { sample_response } let(:sentry_url) { 'https://sentrytest.gitlab.com/api/0' } diff --git a/spec/lib/sentry/client/issue_spec.rb b/spec/lib/sentry/client/issue_spec.rb index a6279aeadd2..dedef905c95 100644 --- a/spec/lib/sentry/client/issue_spec.rb +++ b/spec/lib/sentry/client/issue_spec.rb @@ -65,6 +65,7 @@ RSpec.describe Sentry::Client::Issue do link: '<https://sentrytest.gitlab.com>; rel="previous"; results="true"; cursor="1573556671000:0:1", <https://sentrytest.gitlab.com>; rel="next"; results="true"; cursor="1572959139000:0:0"' } end + let!(:sentry_api_request) { stub_sentry_request(sentry_request_url, body: sentry_api_response, headers: headers) } it 'parses the pagination' do @@ -114,6 +115,7 @@ RSpec.describe Sentry::Client::Issue do 'https://sentrytest.gitlab.com/api/0/projects/sentry-org/sentry-project/' \ 'issues/?limit=20&query=is:unresolved&sort=freq' end + let!(:sentry_api_request) { stub_sentry_request(sentry_request_url, body: sentry_api_response) } subject { client.list_issues(issue_status: issue_status, limit: limit, sort: 'frequency') } |