diff options
Diffstat (limited to 'spec/lib/gitlab')
241 files changed, 6831 insertions, 4658 deletions
diff --git a/spec/lib/gitlab/analytics/usage_trends/workers_argument_builder_spec.rb b/spec/lib/gitlab/analytics/usage_trends/workers_argument_builder_spec.rb index 34c5bd6c6ae..06438f8497d 100644 --- a/spec/lib/gitlab/analytics/usage_trends/workers_argument_builder_spec.rb +++ b/spec/lib/gitlab/analytics/usage_trends/workers_argument_builder_spec.rb @@ -23,10 +23,11 @@ RSpec.describe Gitlab::Analytics::UsageTrends::WorkersArgumentBuilder do subject { described_class.new(measurement_identifiers: measurement_identifiers, recorded_at: recorded_at).execute } it 'returns worker arguments' do - expect(subject).to eq([ - [projects_measurement_identifier, project_1.id, project_3.id, recorded_at], - [users_measurement_identifier, user_1.id, user_1.id, recorded_at] - ]) + expect(subject).to eq( + [ + [projects_measurement_identifier, project_1.id, project_3.id, recorded_at], + [users_measurement_identifier, user_1.id, user_1.id, recorded_at] + ]) end context 'when bogus measurement identifiers are given' do @@ -36,10 +37,11 @@ RSpec.describe Gitlab::Analytics::UsageTrends::WorkersArgumentBuilder do end it 'skips bogus measurement identifiers' do - expect(subject).to eq([ - [projects_measurement_identifier, project_1.id, project_3.id, recorded_at], - [users_measurement_identifier, user_1.id, user_1.id, recorded_at] - ]) + expect(subject).to eq( + [ + [projects_measurement_identifier, project_1.id, project_3.id, recorded_at], + [users_measurement_identifier, user_1.id, user_1.id, recorded_at] + ]) end end diff --git a/spec/lib/gitlab/anonymous_session_spec.rb b/spec/lib/gitlab/anonymous_session_spec.rb index 64186e9003a..08087096d49 100644 --- a/spec/lib/gitlab/anonymous_session_spec.rb +++ b/spec/lib/gitlab/anonymous_session_spec.rb @@ -61,7 +61,7 @@ RSpec.describe Gitlab::AnonymousSession, :clean_gitlab_redis_sessions do subject.cleanup_session_per_ip_count Gitlab::Redis::Sessions.with do |redis| - expect(redis.exists("session:lookup:ip:gitlab2:127.0.0.1")).to eq(false) + expect(redis.exists?("session:lookup:ip:gitlab2:127.0.0.1")).to eq(false) end end end diff --git a/spec/lib/gitlab/auth/auth_finders_spec.rb b/spec/lib/gitlab/auth/auth_finders_spec.rb index d0b44135a2f..e2226952d15 100644 --- a/spec/lib/gitlab/auth/auth_finders_spec.rb +++ b/spec/lib/gitlab/auth/auth_finders_spec.rb @@ -188,7 +188,7 @@ RSpec.describe Gitlab::Auth::AuthFinders do end it 'returns nil if valid feed_token and disabled' do - stub_application_setting(disable_feed_token: true) + allow(Gitlab::CurrentSettings).to receive_messages(disable_feed_token: true) set_param(:feed_token, user.feed_token) expect(find_user_from_feed_token(:rss)).to be_nil diff --git a/spec/lib/gitlab/auth/o_auth/user_spec.rb b/spec/lib/gitlab/auth/o_auth/user_spec.rb index b160f322fb8..95a518afcf1 100644 --- a/spec/lib/gitlab/auth/o_auth/user_spec.rb +++ b/spec/lib/gitlab/auth/o_auth/user_spec.rb @@ -4,7 +4,6 @@ require 'spec_helper' RSpec.describe Gitlab::Auth::OAuth::User do include LdapHelpers - include TermsHelper let(:oauth_user) { described_class.new(auth_hash) } let(:oauth_user_2) { described_class.new(auth_hash_2) } @@ -145,49 +144,6 @@ RSpec.describe Gitlab::Auth::OAuth::User do expect(gl_user).to be_password_automatically_set end - context 'terms of service' do - context 'when terms are enforced' do - before do - enforce_terms - end - - context 'when feature flag update_oauth_registration_flow is enabled' do - before do - stub_feature_flags(update_oauth_registration_flow: true) - end - - it 'creates the user with accepted terms' do - oauth_user.save # rubocop:disable Rails/SaveBang - - expect(gl_user).to be_persisted - expect(gl_user.terms_accepted?).to be(true) - end - end - - context 'when feature flag update_oauth_registration_flow is disabled' do - before do - stub_feature_flags(update_oauth_registration_flow: false) - end - - it 'creates the user without accepted terms' do - oauth_user.save # rubocop:disable Rails/SaveBang - - expect(gl_user).to be_persisted - expect(gl_user.terms_accepted?).to be(false) - end - end - end - - context 'when terms are not enforced' do - it 'creates the user without accepted terms' do - oauth_user.save # rubocop:disable Rails/SaveBang - - expect(gl_user).to be_persisted - expect(gl_user.terms_accepted?).to be(false) - end - end - end - shared_examples 'to verify compliance with allow_single_sign_on' do context 'provider is marked as external' do it 'marks user as external' do diff --git a/spec/lib/gitlab/auth_spec.rb b/spec/lib/gitlab/auth_spec.rb index c2d64aa2fb3..5a6fa7c416b 100644 --- a/spec/lib/gitlab/auth_spec.rb +++ b/spec/lib/gitlab/auth_spec.rb @@ -495,6 +495,12 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do end end end + + it 'updates last_used_at column if token is valid' do + personal_access_token = create(:personal_access_token, scopes: ['write_repository']) + + expect { gl_auth.find_for_git_client('', personal_access_token.token, project: nil, ip: 'ip') }.to change { personal_access_token.reload.last_used_at } + end end context 'while using regular user and password' do diff --git a/spec/lib/gitlab/background_migration/backfill_integrations_type_new_spec.rb b/spec/lib/gitlab/background_migration/backfill_integrations_type_new_spec.rb index d8a7ec775dd..e6588644b4f 100644 --- a/spec/lib/gitlab/background_migration/backfill_integrations_type_new_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_integrations_type_new_spec.rb @@ -7,13 +7,14 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillIntegrationsTypeNew, :migrat let(:integrations) { table(:integrations) } let(:namespaced_integrations) do - Set.new(%w[ - Asana Assembla Bamboo Bugzilla Buildkite Campfire Confluence CustomIssueTracker Datadog - Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Harbor Irker Jenkins Jira Mattermost - MattermostSlashCommands MicrosoftTeams MockCi MockMonitoring Packagist PipelinesEmail Pivotaltracker - Prometheus Pushover Redmine Shimo Slack SlackSlashCommands Teamcity UnifyCircuit WebexTeams Youtrack Zentao - Github GitlabSlackApplication - ]).freeze + Set.new( + %w[ + Asana Assembla Bamboo Bugzilla Buildkite Campfire Confluence CustomIssueTracker Datadog + Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Harbor Irker Jenkins Jira Mattermost + MattermostSlashCommands MicrosoftTeams MockCi MockMonitoring Packagist PipelinesEmail Pivotaltracker + Prometheus Pushover Redmine Shimo Slack SlackSlashCommands Teamcity UnifyCircuit WebexTeams Youtrack Zentao + Github GitlabSlackApplication + ]).freeze end before do @@ -40,13 +41,14 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillIntegrationsTypeNew, :migrat expect(queries.count).to be(16) expect(queries.log.grep(/^SELECT/).size).to be(11) expect(queries.log.grep(/^UPDATE/).size).to be(5) - expect(queries.log.grep(/^UPDATE/).join.scan(/WHERE .*/)).to eq([ - 'WHERE integrations.id BETWEEN 2 AND 3', - 'WHERE integrations.id BETWEEN 4 AND 5', - 'WHERE integrations.id BETWEEN 6 AND 7', - 'WHERE integrations.id BETWEEN 8 AND 9', - 'WHERE integrations.id BETWEEN 10 AND 10' - ]) + expect(queries.log.grep(/^UPDATE/).join.scan(/WHERE .*/)).to eq( + [ + 'WHERE integrations.id BETWEEN 2 AND 3', + 'WHERE integrations.id BETWEEN 4 AND 5', + 'WHERE integrations.id BETWEEN 6 AND 7', + 'WHERE integrations.id BETWEEN 8 AND 9', + 'WHERE integrations.id BETWEEN 10 AND 10' + ]) expect(integrations.where(id: 2..10).pluck(:type, :type_new)).to contain_exactly( ['AssemblaService', 'Integrations::Assembla'], diff --git a/spec/lib/gitlab/background_migration/backfill_internal_on_notes_spec.rb b/spec/lib/gitlab/background_migration/backfill_internal_on_notes_spec.rb new file mode 100644 index 00000000000..40a4758ba5f --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_internal_on_notes_spec.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillInternalOnNotes, :migration, schema: 20220920124709 do + let(:notes_table) { table(:notes) } + + let!(:confidential_note) { notes_table.create!(id: 1, confidential: true, internal: false) } + let!(:non_confidential_note) { notes_table.create!(id: 2, confidential: false, internal: false) } + + describe '#perform' do + subject(:perform) do + described_class.new( + start_id: 1, + end_id: 2, + batch_table: :notes, + batch_column: :id, + sub_batch_size: 1, + pause_ms: 0, + connection: ApplicationRecord.connection + ).perform + end + + it 'backfills internal column on notes when confidential' do + expect { perform } + .to change { confidential_note.reload.internal }.from(false).to(true) + .and not_change { non_confidential_note.reload.internal } + end + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_details_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_details_spec.rb new file mode 100644 index 00000000000..b6282de0da6 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_namespace_details_spec.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceDetails, :migration do + let(:namespaces) { table(:namespaces) } + let(:namespace_details) { table(:namespace_details) } + + subject(:perform_migration) do + described_class.new(start_id: namespaces.minimum(:id), + end_id: namespaces.maximum(:id), + batch_table: :namespaces, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection) + .perform + end + + describe '#perform' do + it 'creates details for all namespaces in range' do + namespace1 = namespaces.create!(id: 5, name: 'test1', path: 'test1', description: "Some description1", + description_html: "Some description html1", cached_markdown_version: 4) + namespaces.create!(id: 6, name: 'test2', path: 'test2', type: 'Project', + description: "Some description2", description_html: "Some description html2", + cached_markdown_version: 4) + namespace3 = namespaces.create!(id: 7, name: 'test3', path: 'test3', description: "Some description3", + description_html: "Some description html3", cached_markdown_version: 4) + namespace4 = namespaces.create!(id: 8, name: 'test4', path: 'test4', description: "Some description3", + description_html: "Some description html4", cached_markdown_version: 4) + namespace_details.delete_all + + expect(namespace_details.pluck(:namespace_id)).to eql [] + + expect { perform_migration } + .to change { namespace_details.pluck(:namespace_id) }.from([]).to contain_exactly( + namespace1.id, + namespace3.id, + namespace4.id + ) + + expect(namespace_details.find_by_namespace_id(namespace1.id)).to have_attributes(migrated_attributes(namespace1)) + expect(namespace_details.find_by_namespace_id(namespace3.id)).to have_attributes(migrated_attributes(namespace3)) + expect(namespace_details.find_by_namespace_id(namespace4.id)).to have_attributes(migrated_attributes(namespace4)) + end + end + + def migrated_attributes(namespace) + { + description: namespace.description, + description_html: namespace.description_html, + cached_markdown_version: namespace.cached_markdown_version + } + end +end diff --git a/spec/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects_spec.rb b/spec/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects_spec.rb index 8a3671b2e53..dd202acc372 100644 --- a/spec/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects_spec.rb +++ b/spec/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects_spec.rb @@ -40,23 +40,26 @@ RSpec.describe Gitlab::BackgroundMigration::CleanupOrphanedLfsObjectsProjects, s it 'lfs_objects_projects without an existing lfs object or project are removed' do subject.perform(without_object1.id, without_object3.id) - expect(lfs_objects_projects.all).to match_array([ - with_project_and_object1, with_project_and_object2, with_project_and_object3, - without_project1, without_project2, without_project_and_object - ]) + expect(lfs_objects_projects.all).to match_array( + [ + with_project_and_object1, with_project_and_object2, with_project_and_object3, + without_project1, without_project2, without_project_and_object + ]) subject.perform(with_project_and_object1.id, with_project_and_object3.id) - expect(lfs_objects_projects.all).to match_array([ - with_project_and_object1, with_project_and_object2, with_project_and_object3, - without_project1, without_project2, without_project_and_object - ]) + expect(lfs_objects_projects.all).to match_array( + [ + with_project_and_object1, with_project_and_object2, with_project_and_object3, + without_project1, without_project2, without_project_and_object + ]) subject.perform(without_project1.id, without_project_and_object.id) - expect(lfs_objects_projects.all).to match_array([ - with_project_and_object1, with_project_and_object2, with_project_and_object3 - ]) + expect(lfs_objects_projects.all).to match_array( + [ + with_project_and_object1, with_project_and_object2, with_project_and_object3 + ]) expect(lfs_objects.ids).to contain_exactly(lfs_object.id, another_lfs_object.id) expect(projects.ids).to contain_exactly(project.id, another_project.id) diff --git a/spec/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities_spec.rb b/spec/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities_spec.rb new file mode 100644 index 00000000000..afa955a6056 --- /dev/null +++ b/spec/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities_spec.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::DeleteOrphanedOperationalVulnerabilities, :migration do + include MigrationHelpers::VulnerabilitiesHelper + + let_it_be(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } + let_it_be(:users) { table(:users) } + let_it_be(:user) do + users.create!( + name: "Example User", + email: "user@example.com", + username: "Example User", + projects_limit: 0, + confirmed_at: Time.current + ) + end + + let_it_be(:project) do + table(:projects).create!( + id: 123, + namespace_id: namespace.id, + project_namespace_id: namespace.id + ) + end + + let_it_be(:scanners) { table(:vulnerability_scanners) } + let_it_be(:scanner) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') } + let_it_be(:different_scanner) do + scanners.create!( + project_id: project.id, + external_id: 'test 2', + name: 'test scanner 2' + ) + end + + let_it_be(:vulnerabilities) { table(:vulnerabilities) } + let_it_be(:vulnerability_with_finding) do + create_vulnerability!( + project_id: project.id, + author_id: user.id + ) + end + + let_it_be(:vulnerability_without_finding) do + create_vulnerability!( + project_id: project.id, + author_id: user.id + ) + end + + let_it_be(:cis_vulnerability_without_finding) do + create_vulnerability!( + project_id: project.id, + author_id: user.id, + report_type: 7 + ) + end + + let_it_be(:custom_vulnerability_without_finding) do + create_vulnerability!( + project_id: project.id, + author_id: user.id, + report_type: 99 + ) + end + + let_it_be(:vulnerability_identifiers) { table(:vulnerability_identifiers) } + let_it_be(:primary_identifier) do + vulnerability_identifiers.create!( + project_id: project.id, + external_type: 'uuid-v5', + external_id: 'uuid-v5', + fingerprint: '7e394d1b1eb461a7406d7b1e08f057a1cf11287a', + name: 'Identifier for UUIDv5') + end + + let_it_be(:vulnerabilities_findings) { table(:vulnerability_occurrences) } + let_it_be(:finding) do + create_finding!( + vulnerability_id: vulnerability_with_finding.id, + project_id: project.id, + scanner_id: scanner.id, + primary_identifier_id: primary_identifier.id + ) + end + + subject(:background_migration) do + described_class.new(start_id: vulnerabilities.minimum(:id), + end_id: vulnerabilities.maximum(:id), + batch_table: :vulnerabilities, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection) + end + + it 'drops Cluster Image Scanning and Custom Vulnerabilities without any Findings' do + expect(vulnerabilities.pluck(:id)).to match_array([ + vulnerability_with_finding.id, + vulnerability_without_finding.id, + cis_vulnerability_without_finding.id, + custom_vulnerability_without_finding.id + ]) + + expect { background_migration.perform }.to change(vulnerabilities, :count).by(-2) + + expect(vulnerabilities.pluck(:id)).to match_array([vulnerability_with_finding.id, vulnerability_without_finding.id]) + end +end diff --git a/spec/lib/gitlab/background_migration/destroy_invalid_members_spec.rb b/spec/lib/gitlab/background_migration/destroy_invalid_members_spec.rb new file mode 100644 index 00000000000..9b0cb96b30b --- /dev/null +++ b/spec/lib/gitlab/background_migration/destroy_invalid_members_spec.rb @@ -0,0 +1,141 @@ +# frozen_string_literal: true + +require 'spec_helper' + +# rubocop: disable RSpec/MultipleMemoizedHelpers +RSpec.describe Gitlab::BackgroundMigration::DestroyInvalidMembers, :migration, schema: 20221004094814 do + let!(:migration_attrs) do + { + start_id: 1, + end_id: 1000, + batch_table: :members, + batch_column: :id, + sub_batch_size: 100, + pause_ms: 0, + connection: ApplicationRecord.connection + } + end + + let(:users_table) { table(:users) } + let(:namespaces_table) { table(:namespaces) } + let(:members_table) { table(:members) } + let(:projects_table) { table(:projects) } + let(:members_table_name) { 'members' } + let(:connection) { ApplicationRecord.connection } + let(:user1) { users_table.create!(name: 'user1', email: 'user1@example.com', projects_limit: 5) } + let(:user2) { users_table.create!(name: 'user2', email: 'user2@example.com', projects_limit: 5) } + let(:user3) { users_table.create!(name: 'user3', email: 'user3@example.com', projects_limit: 5) } + let(:user4) { users_table.create!(name: 'user4', email: 'user4@example.com', projects_limit: 5) } + let(:user5) { users_table.create!(name: 'user5', email: 'user5@example.com', projects_limit: 5) } + let(:user6) { users_table.create!(name: 'user6', email: 'user6@example.com', projects_limit: 5) } + let(:user7) { users_table.create!(name: 'user7', email: 'user7@example.com', projects_limit: 5) } + let(:user8) { users_table.create!(name: 'user8', email: 'user8@example.com', projects_limit: 5) } + let!(:group1) { namespaces_table.create!(name: 'marvellous group 1', path: 'group-path-1', type: 'Group') } + let!(:group2) { namespaces_table.create!(name: 'outstanding group 2', path: 'group-path-2', type: 'Group') } + let!(:project_namespace1) do + namespaces_table.create!(name: 'fabulous project', path: 'project-path-1', + type: 'ProjectNamespace', parent_id: group1.id) + end + + let!(:project1) do + projects_table.create!(name: 'fabulous project', path: 'project-path-1', + project_namespace_id: project_namespace1.id, namespace_id: group1.id) + end + + let!(:project_namespace2) do + namespaces_table.create!(name: 'splendiferous project', path: 'project-path-2', + type: 'ProjectNamespace', parent_id: group1.id) + end + + let!(:project2) do + projects_table.create!(name: 'splendiferous project', path: 'project-path-2', + project_namespace_id: project_namespace2.id, namespace_id: group1.id) + end + + # create valid project member records + let!(:project_member1) { create_valid_project_member(id: 1, user_id: user1.id, project: project1) } + let!(:project_member2) { create_valid_project_member(id: 2, user_id: user2.id, project: project2) } + # create valid group member records + let!(:group_member5) { create_valid_group_member(id: 5, user_id: user5.id, group_id: group1.id) } + let!(:group_member6) { create_valid_group_member(id: 6, user_id: user6.id, group_id: group2.id) } + + let!(:migration) { described_class.new(**migration_attrs) } + + subject(:perform_migration) { migration.perform } + + # create invalid project and group member records + def create_members + [ + create_invalid_project_member(id: 3, user_id: user3.id), + create_invalid_project_member(id: 4, user_id: user4.id), + create_invalid_group_member(id: 7, user_id: user7.id), + create_invalid_group_member(id: 8, user_id: user8.id) + ] + end + + it 'removes invalid memberships but keeps valid ones', :aggregate_failures do + without_check_constraint(members_table_name, 'check_508774aac0', connection: connection) do + create_members + + expect(members_table.count).to eq 8 + + queries = ActiveRecord::QueryRecorder.new do + perform_migration + end + + expect(queries.count).to eq(4) + expect(members_table.all).to match_array([project_member1, project_member2, group_member5, group_member6]) + end + end + + it 'tracks timings of queries' do + without_check_constraint(members_table_name, 'check_508774aac0', connection: connection) do + create_members + + expect(migration.batch_metrics.timings).to be_empty + + expect { perform_migration }.to change { migration.batch_metrics.timings } + end + end + + it 'logs IDs of deleted records' do + without_check_constraint(members_table_name, 'check_508774aac0', connection: connection) do + members = create_members + + member_data = members.map do |m| + { id: m.id, source_id: m.source_id, source_type: m.source_type } + end + + expect(Gitlab::AppLogger).to receive(:info).with({ message: 'Removing invalid member records', + deleted_count: 4, + deleted_member_data: member_data }) + + perform_migration + end + end + + def create_invalid_project_member(id:, user_id:) + members_table.create!(id: id, user_id: user_id, source_id: non_existing_record_id, + access_level: Gitlab::Access::MAINTAINER, type: "ProjectMember", + source_type: "Project", notification_level: 3, member_namespace_id: nil) + end + + def create_valid_project_member(id:, user_id:, project:) + members_table.create!(id: id, user_id: user_id, source_id: project.id, + access_level: Gitlab::Access::MAINTAINER, type: "ProjectMember", source_type: "Project", + member_namespace_id: project.project_namespace_id, notification_level: 3) + end + + def create_invalid_group_member(id:, user_id:) + members_table.create!(id: id, user_id: user_id, source_id: non_existing_record_id, + access_level: Gitlab::Access::MAINTAINER, type: "GroupMember", + source_type: "Namespace", notification_level: 3, member_namespace_id: nil) + end + + def create_valid_group_member(id:, user_id:, group_id:) + members_table.create!(id: id, user_id: user_id, source_id: group_id, + access_level: Gitlab::Access::MAINTAINER, type: "GroupMember", + source_type: "Namespace", member_namespace_id: group_id, notification_level: 3) + end +end +# rubocop: enable RSpec/MultipleMemoizedHelpers diff --git a/spec/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users_spec.rb b/spec/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users_spec.rb index 31b6ee0c7cd..c3ae2cc060c 100644 --- a/spec/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users_spec.rb +++ b/spec/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users_spec.rb @@ -79,10 +79,11 @@ RSpec.describe Gitlab::BackgroundMigration::MigrateMergeRequestDiffCommitUsers, it 'produces a union of the given queries' do alice = commit_users.create!(name: 'Alice', email: 'alice@example.com') bob = commit_users.create!(name: 'Bob', email: 'bob@example.com') - users = commit_users.union([ - commit_users.where(name: 'Alice').to_sql, - commit_users.where(name: 'Bob').to_sql - ]) + users = commit_users.union( + [ + commit_users.where(name: 'Alice').to_sql, + commit_users.where(name: 'Bob').to_sql + ]) expect(users).to include(alice) expect(users).to include(bob) diff --git a/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb b/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb index a609227be05..29cc4f34f6d 100644 --- a/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb +++ b/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb @@ -246,9 +246,15 @@ RSpec.describe Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrence end it 'drops duplicates and related records', :aggregate_failures do - expect(vulnerability_findings.pluck(:id)).to match_array([ - finding_with_correct_uuid.id, finding_with_incorrect_uuid.id, finding_with_correct_uuid2.id, finding_with_incorrect_uuid2.id, finding_with_incorrect_uuid3.id, duplicate_not_in_the_same_batch.id - ]) + expect(vulnerability_findings.pluck(:id)).to match_array( + [ + finding_with_correct_uuid.id, + finding_with_incorrect_uuid.id, + finding_with_correct_uuid2.id, + finding_with_incorrect_uuid2.id, + finding_with_incorrect_uuid3.id, + duplicate_not_in_the_same_batch.id + ]) expect { subject }.to change(vulnerability_finding_pipelines, :count).from(16).to(8) .and change(vulnerability_findings, :count).from(6).to(3) @@ -306,7 +312,8 @@ RSpec.describe Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrence it 'retries the recalculation' do subject - expect(Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid::VulnerabilitiesFinding).to have_received(:find_by).with(uuid: uuid).once + expect(Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid::VulnerabilitiesFinding) + .to have_received(:find_by).with(uuid: uuid).once end it 'logs the conflict' do diff --git a/spec/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_spec.rb b/spec/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_spec.rb new file mode 100644 index 00000000000..b6da8f7fc2d --- /dev/null +++ b/spec/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_spec.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::ResetDuplicateCiRunnersTokenEncryptedValues, + :migration, + schema: 20220922143634 do + it { expect(described_class).to be < Gitlab::BackgroundMigration::BatchedMigrationJob } + + describe '#perform' do + let(:ci_runners) { table(:ci_runners, database: :ci) } + + let(:test_worker) do + described_class.new( + start_id: 1, + end_id: 4, + batch_table: :ci_runners, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: Ci::ApplicationRecord.connection + ) + end + + subject(:perform) { test_worker.perform } + + before do + ci_runners.create!(id: 1, runner_type: 1, token_encrypted: 'duplicate') + ci_runners.create!(id: 2, runner_type: 1, token_encrypted: 'a-token') + ci_runners.create!(id: 3, runner_type: 1, token_encrypted: 'duplicate-2') + ci_runners.create!(id: 4, runner_type: 1, token_encrypted: nil) + ci_runners.create!(id: 5, runner_type: 1, token_encrypted: 'duplicate-2') + ci_runners.create!(id: 6, runner_type: 1, token_encrypted: 'duplicate') + ci_runners.create!(id: 7, runner_type: 1, token_encrypted: 'another-token') + ci_runners.create!(id: 8, runner_type: 1, token_encrypted: 'another-token') + end + + it 'nullifies duplicate encrypted tokens', :aggregate_failures do + expect { perform }.to change { ci_runners.all.order(:id).pluck(:id, :token_encrypted).to_h } + .from( + { + 1 => 'duplicate', + 2 => 'a-token', + 3 => 'duplicate-2', + 4 => nil, + 5 => 'duplicate-2', + 6 => 'duplicate', + 7 => 'another-token', + 8 => 'another-token' + } + ) + .to( + { + 1 => nil, + 2 => 'a-token', + 3 => nil, + 4 => nil, + 5 => nil, + 6 => nil, + 7 => 'another-token', + 8 => 'another-token' + } + ) + expect(ci_runners.count).to eq(8) + expect(ci_runners.pluck(:token_encrypted).uniq).to match_array [ + nil, 'a-token', 'another-token' + ] + end + end +end diff --git a/spec/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_spec.rb b/spec/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_spec.rb new file mode 100644 index 00000000000..423b1815e75 --- /dev/null +++ b/spec/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_spec.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::ResetDuplicateCiRunnersTokenValues, + :migration, + schema: 20220922143143 do + it { expect(described_class).to be < Gitlab::BackgroundMigration::BatchedMigrationJob } + + describe '#perform' do + let(:ci_runners) { table(:ci_runners, database: :ci) } + + let(:test_worker) do + described_class.new( + start_id: 1, + end_id: 4, + batch_table: :ci_runners, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: Ci::ApplicationRecord.connection + ) + end + + subject(:perform) { test_worker.perform } + + before do + ci_runners.create!(id: 1, runner_type: 1, token: 'duplicate') + ci_runners.create!(id: 2, runner_type: 1, token: 'a-token') + ci_runners.create!(id: 3, runner_type: 1, token: 'duplicate-2') + ci_runners.create!(id: 4, runner_type: 1, token: nil) + ci_runners.create!(id: 5, runner_type: 1, token: 'duplicate-2') + ci_runners.create!(id: 6, runner_type: 1, token: 'duplicate') + ci_runners.create!(id: 7, runner_type: 1, token: 'another-token') + ci_runners.create!(id: 8, runner_type: 1, token: 'another-token') + end + + it 'nullifies duplicate tokens', :aggregate_failures do + expect { perform }.to change { ci_runners.all.order(:id).pluck(:id, :token).to_h } + .from( + { + 1 => 'duplicate', + 2 => 'a-token', + 3 => 'duplicate-2', + 4 => nil, + 5 => 'duplicate-2', + 6 => 'duplicate', + 7 => 'another-token', + 8 => 'another-token' + } + ) + .to( + { + 1 => nil, + 2 => 'a-token', + 3 => nil, + 4 => nil, + 5 => nil, + 6 => nil, + 7 => 'another-token', + 8 => 'another-token' + } + ) + expect(ci_runners.count).to eq(8) + expect(ci_runners.pluck(:token).uniq).to match_array [ + nil, 'a-token', 'another-token' + ] + end + end +end diff --git a/spec/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status_spec.rb b/spec/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status_spec.rb new file mode 100644 index 00000000000..98939e15952 --- /dev/null +++ b/spec/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status_spec.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::UpdateCiPipelineArtifactsUnknownLockedStatus do + describe '#perform' do + let(:batch_table) { :ci_pipeline_artifacts } + let(:batch_column) { :id } + + let(:sub_batch_size) { 1 } + let(:pause_ms) { 0 } + let(:connection) { Ci::ApplicationRecord.connection } + + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:pipelines) { table(:ci_pipelines, database: :ci) } + let(:pipeline_artifacts) { table(:ci_pipeline_artifacts, database: :ci) } + + let(:namespace) { namespaces.create!(name: 'name', path: 'path') } + let(:project) do + projects + .create!(name: "project", path: "project", namespace_id: namespace.id, project_namespace_id: namespace.id) + end + + let(:unlocked) { 0 } + let(:locked) { 1 } + let(:unknown) { 2 } + + let(:unlocked_pipeline) { pipelines.create!(locked: unlocked) } + let(:locked_pipeline) { pipelines.create!(locked: locked) } + + # rubocop:disable Layout/LineLength + let!(:locked_artifact) { pipeline_artifacts.create!(project_id: project.id, pipeline_id: locked_pipeline.id, size: 1024, file_type: 0, file_format: 'gzip', file: 'a.gz', locked: unknown) } + let!(:unlocked_artifact_1) { pipeline_artifacts.create!(project_id: project.id, pipeline_id: unlocked_pipeline.id, size: 2048, file_type: 1, file_format: 'raw', file: 'b', locked: unknown) } + let!(:unlocked_artifact_2) { pipeline_artifacts.create!(project_id: project.id, pipeline_id: unlocked_pipeline.id, size: 4096, file_type: 2, file_format: 'gzip', file: 'c.gz', locked: unknown) } + let!(:already_unlocked_artifact) { pipeline_artifacts.create!(project_id: project.id, pipeline_id: unlocked_pipeline.id, size: 8192, file_type: 3, file_format: 'raw', file: 'd', locked: unlocked) } + let!(:already_locked_artifact) { pipeline_artifacts.create!(project_id: project.id, pipeline_id: locked_pipeline.id, size: 8192, file_type: 3, file_format: 'raw', file: 'd', locked: locked) } + # rubocop:enable Layout/LineLength + + subject do + described_class.new( + start_id: locked_artifact.id, + end_id: already_locked_artifact.id, + batch_table: batch_table, + batch_column: batch_column, + sub_batch_size: sub_batch_size, + pause_ms: pause_ms, + connection: connection + ).perform + end + + it 'updates ci_pipeline_artifacts with unknown lock status' do + subject + + expect(locked_artifact.reload.locked).to eq(locked) + expect(unlocked_artifact_1.reload.locked).to eq(unlocked) + expect(unlocked_artifact_2.reload.locked).to eq(unlocked) + expect(already_unlocked_artifact.reload.locked).to eq(unlocked) + expect(already_locked_artifact.reload.locked).to eq(locked) + end + end +end diff --git a/spec/lib/gitlab/bare_repository_import/importer_spec.rb b/spec/lib/gitlab/bare_repository_import/importer_spec.rb index 8fb903154f3..3a885d70eb4 100644 --- a/spec/lib/gitlab/bare_repository_import/importer_spec.rb +++ b/spec/lib/gitlab/bare_repository_import/importer_spec.rb @@ -192,6 +192,6 @@ RSpec.describe Gitlab::BareRepositoryImport::Importer do cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{source_project} #{repo_path}) - system(git_env, *cmd, chdir: SEED_STORAGE_PATH, out: '/dev/null', err: '/dev/null') + system(git_env, *cmd, chdir: base_dir, out: '/dev/null', err: '/dev/null') end end diff --git a/spec/lib/gitlab/bare_repository_import/repository_spec.rb b/spec/lib/gitlab/bare_repository_import/repository_spec.rb index becfdced5fb..a9778e0e8a7 100644 --- a/spec/lib/gitlab/bare_repository_import/repository_spec.rb +++ b/spec/lib/gitlab/bare_repository_import/repository_spec.rb @@ -55,7 +55,7 @@ RSpec.describe ::Gitlab::BareRepositoryImport::Repository do context 'hashed storage' do let(:hashed_path) { "@hashed/6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b" } - let(:root_path) { TestEnv.repos_path } + let(:root_path) { Gitlab::GitalyClient::StorageSettings.allow_disk_access { TestEnv.repos_path } } let(:repo_path) { File.join(root_path, "#{hashed_path}.git") } let(:wiki_path) { File.join(root_path, "#{hashed_path}.wiki.git") } let(:raw_repository) { Gitlab::Git::Repository.new('default', "#{hashed_path}.git", nil, nil) } diff --git a/spec/lib/gitlab/batch_pop_queueing_spec.rb b/spec/lib/gitlab/batch_pop_queueing_spec.rb deleted file mode 100644 index 5af78ddabe7..00000000000 --- a/spec/lib/gitlab/batch_pop_queueing_spec.rb +++ /dev/null @@ -1,147 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BatchPopQueueing do - include ExclusiveLeaseHelpers - using RSpec::Parameterized::TableSyntax - - describe '#initialize' do - where(:namespace, :queue_id, :expect_error, :error_type) do - 'feature' | '1' | false | nil - :feature | '1' | false | nil - nil | '1' | true | NoMethodError - 'feature' | nil | true | NoMethodError - '' | '1' | true | ArgumentError - 'feature' | '' | true | ArgumentError - 'feature' | 1 | true | NoMethodError - end - - with_them do - it do - if expect_error - expect { described_class.new(namespace, queue_id) }.to raise_error(error_type) - else - expect { described_class.new(namespace, queue_id) }.not_to raise_error - end - end - end - end - - describe '#safe_execute', :clean_gitlab_redis_queues do - subject { queue.safe_execute(new_items, lock_timeout: lock_timeout) } - - let(:queue) { described_class.new(namespace, queue_id) } - let(:namespace) { 'feature' } - let(:queue_id) { '1' } - let(:lock_timeout) { 10.minutes } - let(:new_items) { %w[A B] } - let(:lock_key) { queue.send(:lock_key) } - let(:queue_key) { queue.send(:queue_key) } - - it 'enqueues new items always' do - Gitlab::Redis::Queues.with do |redis| - expect(redis).to receive(:sadd).with(queue_key, new_items) - expect(redis).to receive(:expire).with(queue_key, (lock_timeout + described_class::EXTRA_QUEUE_EXPIRE_WINDOW).to_i) - end - - subject - end - - it 'yields the new items with exclusive lease' do - uuid = 'test' - expect_to_obtain_exclusive_lease(lock_key, uuid, timeout: lock_timeout) - expect_to_cancel_exclusive_lease(lock_key, uuid) - - expect { |b| queue.safe_execute(new_items, lock_timeout: lock_timeout, &b) } - .to yield_with_args(match_array(new_items)) - end - - it 'returns the result and no items in the queue' do - expect(subject[:status]).to eq(:finished) - expect(subject[:new_items]).to be_empty - - Gitlab::Redis::Queues.with do |redis| - expect(redis.llen(queue_key)).to be(0) - end - end - - context 'when new items are enqueued during the process' do - it 'returns the result with newly added items' do - result = queue.safe_execute(new_items) do - queue.safe_execute(['C']) - end - - expect(result[:status]).to eq(:finished) - expect(result[:new_items]).to eq(['C']) - - Gitlab::Redis::Queues.with do |redis| - expect(redis.scard(queue_key)).to be(1) - end - end - end - - context 'when interger items are enqueued' do - let(:new_items) { [1, 2, 3] } - - it 'yields as String values' do - expect { |b| queue.safe_execute(new_items, lock_timeout: lock_timeout, &b) } - .to yield_with_args(%w[1 2 3]) - end - end - - context 'when the queue key does not exist in Redis' do - before do - allow(queue).to receive(:enqueue) {} - end - - it 'yields empty array' do - expect { |b| queue.safe_execute(new_items, lock_timeout: lock_timeout, &b) } - .to yield_with_args([]) - end - end - - context 'when the other process has already been working on the queue' do - before do - stub_exclusive_lease_taken(lock_key, timeout: lock_timeout) - end - - it 'does not yield the block' do - expect { |b| queue.safe_execute(new_items, lock_timeout: lock_timeout, &b) } - .not_to yield_control - end - - it 'returns the result' do - expect(subject[:status]).to eq(:enqueued) - end - end - - context 'when a duplicate item is enqueued' do - it 'returns the poped items to the queue and raise an error' do - expect { |b| queue.safe_execute(%w[1 1 2 2], &b) } - .to yield_with_args(match_array(%w[1 2])) - end - end - - context 'when there are two queues' do - it 'enqueues items to each queue' do - queue_1 = described_class.new(namespace, '1') - queue_2 = described_class.new(namespace, '2') - - result_2 = nil - - result_1 = queue_1.safe_execute(['A']) do |_| - result_2 = queue_2.safe_execute(['B']) do |_| - queue_1.safe_execute(['C']) - queue_2.safe_execute(['D']) - end - end - - expect(result_1[:status]).to eq(:finished) - expect(result_1[:new_items]).to eq(['C']) - expect(result_2[:status]).to eq(:finished) - expect(result_2[:new_items]).to eq(['D']) - end - end - end -end diff --git a/spec/lib/gitlab/bitbucket_import/importer_spec.rb b/spec/lib/gitlab/bitbucket_import/importer_spec.rb index e0a7044e5f9..186d4e1fb42 100644 --- a/spec/lib/gitlab/bitbucket_import/importer_spec.rb +++ b/spec/lib/gitlab/bitbucket_import/importer_spec.rb @@ -58,24 +58,15 @@ RSpec.describe Gitlab::BitbucketImport::Importer do issues end - let(:project_identifier) { 'namespace/repo' } + let_it_be(:project_identifier) { 'namespace/repo' } - let(:data) do - { - 'bb_session' => { - 'bitbucket_token' => "123456", - 'bitbucket_refresh_token' => "secret" - } - } - end - - let(:project) do + let_it_be_with_reload(:project) do create( :project, :repository, import_source: project_identifier, import_url: "https://bitbucket.org/#{project_identifier}.git", - import_data_attributes: { credentials: data } + import_data_attributes: { credentials: { 'token' => 'token' } } ) end @@ -88,6 +79,14 @@ RSpec.describe Gitlab::BitbucketImport::Importer do } end + let(:last_issue_data) do + { + page: 1, + pagelen: 1, + values: [sample_issues_statuses.last] + } + end + let(:counter) { double('counter', increment: true) } subject { described_class.new(project) } @@ -253,6 +252,13 @@ RSpec.describe Gitlab::BitbucketImport::Importer do stub_request( :get, + "https://api.bitbucket.org/2.0/repositories/#{project_identifier}/issues?pagelen=1&sort=-created_on&state=ALL" + ).to_return(status: 200, + headers: { "Content-Type" => "application/json" }, + body: last_issue_data.to_json) + + stub_request( + :get, "https://api.bitbucket.org/2.0/repositories/#{project_identifier}/issues?pagelen=50&sort=created_on" ).to_return(status: 200, headers: { "Content-Type" => "application/json" }, @@ -352,6 +358,12 @@ RSpec.describe Gitlab::BitbucketImport::Importer do end describe 'issue import' do + it 'allocates internal ids' do + expect(Issue).to receive(:track_project_iid!).with(project, 6) + + importer.execute + end + it 'maps reporters to anonymous if bitbucket reporter is nil' do allow(importer).to receive(:import_wiki) importer.execute @@ -371,6 +383,29 @@ RSpec.describe Gitlab::BitbucketImport::Importer do expect(project.issues.map(&:work_item_type_id).uniq).to contain_exactly(WorkItems::Type.default_issue_type.id) end + + context 'with issue comments' do + let(:inline_note) do + instance_double(Bitbucket::Representation::Comment, note: 'Hello world', author: 'someuser', created_at: Time.now, updated_at: Time.now) + end + + before do + allow_next_instance_of(Bitbucket::Client) do |instance| + allow(instance).to receive(:issue_comments).and_return([inline_note]) + end + end + + it 'imports issue comments' do + allow(importer).to receive(:import_wiki) + importer.execute + + comment = project.notes.first + expect(project.notes.size).to eq(7) + expect(comment.note).to include(inline_note.note) + expect(comment.note).to include(inline_note.author) + expect(importer.errors).to be_empty + end + end end context 'metrics' do diff --git a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb index c0e4d1b5355..c78140a70b3 100644 --- a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb +++ b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb @@ -288,7 +288,7 @@ RSpec.describe Gitlab::Cache::Ci::ProjectPipelineStatus, :clean_gitlab_redis_cac it 'deletes values from redis_cache' do pipeline_status.delete_from_cache - key_exists = Gitlab::Redis::Cache.with { |redis| redis.exists(cache_key) } + key_exists = Gitlab::Redis::Cache.with { |redis| redis.exists?(cache_key) } expect(key_exists).to be_falsy end diff --git a/spec/lib/gitlab/ci/ansi2json_spec.rb b/spec/lib/gitlab/ci/ansi2json_spec.rb index 4b3b049176f..0f8f3759834 100644 --- a/spec/lib/gitlab/ci/ansi2json_spec.rb +++ b/spec/lib/gitlab/ci/ansi2json_spec.rb @@ -7,70 +7,74 @@ RSpec.describe Gitlab::Ci::Ansi2json do describe 'lines' do it 'prints non-ansi as-is' do - expect(convert_json('Hello')).to eq([ - { offset: 0, content: [{ text: 'Hello' }] } - ]) + expect(convert_json('Hello')).to eq([{ offset: 0, content: [{ text: 'Hello' }] }]) end context 'new lines' do it 'adds new line when encountering \n' do - expect(convert_json("Hello\nworld")).to eq([ - { offset: 0, content: [{ text: 'Hello' }] }, - { offset: 6, content: [{ text: 'world' }] } - ]) + expect(convert_json("Hello\nworld")).to eq( + [ + { offset: 0, content: [{ text: 'Hello' }] }, + { offset: 6, content: [{ text: 'world' }] } + ]) end it 'adds new line when encountering \r\n' do - expect(convert_json("Hello\r\nworld")).to eq([ - { offset: 0, content: [{ text: 'Hello' }] }, - { offset: 7, content: [{ text: 'world' }] } - ]) + expect(convert_json("Hello\r\nworld")).to eq( + [ + { offset: 0, content: [{ text: 'Hello' }] }, + { offset: 7, content: [{ text: 'world' }] } + ]) end it 'ignores empty newlines' do - expect(convert_json("Hello\n\nworld")).to eq([ - { offset: 0, content: [{ text: 'Hello' }] }, - { offset: 7, content: [{ text: 'world' }] } - ]) - expect(convert_json("Hello\r\n\r\nworld")).to eq([ - { offset: 0, content: [{ text: 'Hello' }] }, - { offset: 9, content: [{ text: 'world' }] } - ]) + expect(convert_json("Hello\n\nworld")).to eq( + [ + { offset: 0, content: [{ text: 'Hello' }] }, + { offset: 7, content: [{ text: 'world' }] } + ]) + expect(convert_json("Hello\r\n\r\nworld")).to eq( + [ + { offset: 0, content: [{ text: 'Hello' }] }, + { offset: 9, content: [{ text: 'world' }] } + ]) end it 'replace the current line when encountering \r' do - expect(convert_json("Hello\rworld")).to eq([ - { offset: 0, content: [{ text: 'world' }] } - ]) + expect(convert_json("Hello\rworld")).to eq([{ offset: 0, content: [{ text: 'world' }] }]) end end it 'recognizes color changing ANSI sequences' do - expect(convert_json("\e[31mHello\e[0m")).to eq([ - { offset: 0, content: [{ text: 'Hello', style: 'term-fg-red' }] } - ]) + expect(convert_json("\e[31mHello\e[0m")).to eq( + [ + { offset: 0, content: [{ text: 'Hello', style: 'term-fg-red' }] } + ]) end it 'recognizes color changing ANSI sequences across multiple lines' do - expect(convert_json("\e[31mHello\nWorld\e[0m")).to eq([ - { offset: 0, content: [{ text: 'Hello', style: 'term-fg-red' }] }, - { offset: 11, content: [{ text: 'World', style: 'term-fg-red' }] } - ]) + expect(convert_json("\e[31mHello\nWorld\e[0m")).to eq( + [ + { offset: 0, content: [{ text: 'Hello', style: 'term-fg-red' }] }, + { offset: 11, content: [{ text: 'World', style: 'term-fg-red' }] } + ]) end it 'recognizes background and foreground colors' do - expect(convert_json("\e[31;44mHello")).to eq([ - { offset: 0, content: [{ text: 'Hello', style: 'term-fg-red term-bg-blue' }] } - ]) + expect(convert_json("\e[31;44mHello")).to eq( + [ + { offset: 0, content: [{ text: 'Hello', style: 'term-fg-red term-bg-blue' }] } + ]) end it 'recognizes style changes within the same line' do - expect(convert_json("\e[31;44mHello\e[0m world")).to eq([ - { offset: 0, content: [ - { text: 'Hello', style: 'term-fg-red term-bg-blue' }, - { text: ' world' } - ] } - ]) + expect(convert_json("\e[31;44mHello\e[0m world")).to eq( + [ + { offset: 0, content: [ + { text: 'Hello', style: 'term-fg-red term-bg-blue' }, + { text: ' world' } + ] } + ]) end context 'with section markers' do @@ -82,130 +86,137 @@ RSpec.describe Gitlab::Ci::Ansi2json do let(:section_end) { "section_end:#{section_end_time.to_i}:#{section_name}\r\033[0K" } it 'marks the first line of the section as header' do - expect(convert_json("Hello#{section_start}world!")).to eq([ - { - offset: 0, - content: [{ text: 'Hello' }] - }, - { - offset: 5, - content: [{ text: 'world!' }], - section: 'prepare-script', - section_header: true - } - ]) + expect(convert_json("Hello#{section_start}world!")).to eq( + [ + { + offset: 0, + content: [{ text: 'Hello' }] + }, + { + offset: 5, + content: [{ text: 'world!' }], + section: 'prepare-script', + section_header: true + } + ]) end it 'does not marks the other lines of the section as header' do - expect(convert_json("outside section#{section_start}Hello\nworld!")).to eq([ - { - offset: 0, - content: [{ text: 'outside section' }] - }, - { - offset: 15, - content: [{ text: 'Hello' }], - section: 'prepare-script', - section_header: true - }, - { - offset: 65, - content: [{ text: 'world!' }], - section: 'prepare-script' - } - ]) + expect(convert_json("outside section#{section_start}Hello\nworld!")).to eq( + [ + { + offset: 0, + content: [{ text: 'outside section' }] + }, + { + offset: 15, + content: [{ text: 'Hello' }], + section: 'prepare-script', + section_header: true + }, + { + offset: 65, + content: [{ text: 'world!' }], + section: 'prepare-script' + } + ]) end it 'marks the last line of the section as footer' do - expect(convert_json("#{section_start}Good\nmorning\nworld!#{section_end}")).to eq([ - { - offset: 0, - content: [{ text: 'Good' }], - section: 'prepare-script', - section_header: true - }, - { - offset: 49, - content: [{ text: 'morning' }], - section: 'prepare-script' - }, - { - offset: 57, - content: [{ text: 'world!' }], - section: 'prepare-script' - }, - { - offset: 63, - content: [], - section_duration: '01:03', - section: 'prepare-script' - } - ]) + expect(convert_json("#{section_start}Good\nmorning\nworld!#{section_end}")).to eq( + [ + { + offset: 0, + content: [{ text: 'Good' }], + section: 'prepare-script', + section_header: true + }, + { + offset: 49, + content: [{ text: 'morning' }], + section: 'prepare-script' + }, + { + offset: 57, + content: [{ text: 'world!' }], + section: 'prepare-script' + }, + { + offset: 63, + content: [], + section_duration: '01:03', + section: 'prepare-script' + } + ]) end it 'marks the first line as header and footer if is the only line in the section' do - expect(convert_json("#{section_start}Hello world!#{section_end}")).to eq([ - { - offset: 0, - content: [{ text: 'Hello world!' }], - section: 'prepare-script', - section_header: true - }, - { - offset: 56, - content: [], - section: 'prepare-script', - section_duration: '01:03' - } - ]) + expect(convert_json("#{section_start}Hello world!#{section_end}")).to eq( + [ + { + offset: 0, + content: [{ text: 'Hello world!' }], + section: 'prepare-script', + section_header: true + }, + { + offset: 56, + content: [], + section: 'prepare-script', + section_duration: '01:03' + } + ]) end it 'does not add sections attribute to lines after the section is closed' do - expect(convert_json("#{section_start}Hello#{section_end}world")).to eq([ - { - offset: 0, - content: [{ text: 'Hello' }], - section: 'prepare-script', - section_header: true - }, - { - offset: 49, - content: [], - section: 'prepare-script', - section_duration: '01:03' - }, - { - offset: 91, - content: [{ text: 'world' }] - } - ]) + expect(convert_json("#{section_start}Hello#{section_end}world")).to eq( + [ + { + offset: 0, + content: [{ text: 'Hello' }], + section: 'prepare-script', + section_header: true + }, + { + offset: 49, + content: [], + section: 'prepare-script', + section_duration: '01:03' + }, + { + offset: 91, + content: [{ text: 'world' }] + } + ]) end it 'ignores section_end marker if no section_start exists' do - expect(convert_json("Hello #{section_end}world")).to eq([ - { - offset: 0, - content: [{ text: 'Hello world' }] - } - ]) + expect(convert_json("Hello #{section_end}world")).to eq( + [ + { + offset: 0, + content: [{ text: 'Hello world' }] + } + ]) end context 'when section name contains .-_ and capital letters' do let(:section_name) { 'a.Legit-SeCtIoN_namE' } it 'sanitizes the section name' do - expect(convert_json("Hello#{section_start}world!")).to eq([ - { - offset: 0, - content: [{ text: 'Hello' }] - }, - { - offset: 5, - content: [{ text: 'world!' }], - section: 'a-legit-section-name', - section_header: true - } - ]) + expect(convert_json("Hello#{section_start}world!")).to eq( + [ + { + offset: 0, + content: [{ text: 'Hello' }] + }, + { + offset: 5, + content: [{ text: 'world!' }], + section: 'a-legit-section-name', + section_header: true + } + ]) end end @@ -213,12 +224,13 @@ RSpec.describe Gitlab::Ci::Ansi2json do let(:section_name) { 'my_$ection' } it 'ignores the section' do - expect(convert_json("#{section_start}hello")).to eq([ - { - offset: 0, - content: [{ text: 'hello' }] - } - ]) + expect(convert_json("#{section_start}hello")).to eq( + [ + { + offset: 0, + content: [{ text: 'hello' }] + } + ]) end end @@ -226,31 +238,33 @@ RSpec.describe Gitlab::Ci::Ansi2json do let(:section_name) { '<a_tag>' } it 'ignores the section' do - expect(convert_json("#{section_start}hello")).to eq([ - { - offset: 0, - content: [{ text: 'hello' }] - } - ]) + expect(convert_json("#{section_start}hello")).to eq( + [ + { + offset: 0, + content: [{ text: 'hello' }] + } + ]) end end it 'prints HTML tags as is' do trace = "#{section_start}section_end:1:2<div>hello</div>#{section_end}" - expect(convert_json(trace)).to eq([ - { - offset: 0, - content: [{ text: 'section_end:1:2<div>hello</div>' }], - section: 'prepare-script', - section_header: true - }, - { - offset: 75, - content: [], - section: 'prepare-script', - section_duration: '01:03' - } - ]) + expect(convert_json(trace)).to eq( + [ + { + offset: 0, + content: [{ text: 'section_end:1:2<div>hello</div>' }], + section: 'prepare-script', + section_header: true + }, + { + offset: 75, + content: [], + section: 'prepare-script', + section_duration: '01:03' + } + ]) end context 'with nested section' do @@ -264,7 +278,8 @@ RSpec.describe Gitlab::Ci::Ansi2json do it 'adds multiple sections to the lines inside the nested section' do trace = "Hello#{section_start}foo#{nested_section_start}bar#{nested_section_end}baz#{section_end}world" - expect(convert_json(trace)).to eq([ + expect(convert_json(trace)).to eq( + [ { offset: 0, content: [{ text: 'Hello' }] @@ -308,7 +323,8 @@ RSpec.describe Gitlab::Ci::Ansi2json do it 'adds multiple sections to the lines inside the nested section and closes all sections together' do trace = "Hello#{section_start}\e[91mfoo\e[0m#{nested_section_start}bar#{nested_section_end}#{section_end}" - expect(convert_json(trace)).to eq([ + expect(convert_json(trace)).to eq( + [ { offset: 0, content: [{ text: 'Hello' }] @@ -346,24 +362,25 @@ RSpec.describe Gitlab::Ci::Ansi2json do it 'provides section options when set' do trace = "#{option_section_start}hello#{section_end}" - expect(convert_json(trace)).to eq([ - { - offset: 0, - content: [{ text: 'hello' }], - section: 'prepare-script', - section_header: true, - section_options: { - 'collapsed' => 'true', - 'unused_option' => '123' + expect(convert_json(trace)).to eq( + [ + { + offset: 0, + content: [{ text: 'hello' }], + section: 'prepare-script', + section_header: true, + section_options: { + 'collapsed' => 'true', + 'unused_option' => '123' + } + }, + { + offset: 83, + content: [], + section: 'prepare-script', + section_duration: '01:03' } - }, - { - offset: 83, - content: [], - section: 'prepare-script', - section_duration: '01:03' - } - ]) + ]) end end end diff --git a/spec/lib/gitlab/ci/build/rules/rule/clause/changes_spec.rb b/spec/lib/gitlab/ci/build/rules/rule/clause/changes_spec.rb index 234ba68d627..a22aa30304b 100644 --- a/spec/lib/gitlab/ci/build/rules/rule/clause/changes_spec.rb +++ b/spec/lib/gitlab/ci/build/rules/rule/clause/changes_spec.rb @@ -122,19 +122,17 @@ RSpec.describe Gitlab::Ci::Build::Rules::Rule::Clause::Changes do context 'when compare_to is branch or tag' do using RSpec::Parameterized::TableSyntax - where(:pipeline_ref, :compare_to, :paths, :ff, :result) do - 'feature_1' | 'master' | ['file1.txt'] | true | true - 'feature_1' | 'master' | ['README.md'] | true | false - 'feature_1' | 'master' | ['xyz.md'] | true | false - 'feature_2' | 'master' | ['file1.txt'] | true | true - 'feature_2' | 'master' | ['file2.txt'] | true | true - 'feature_2' | 'feature_1' | ['file1.txt'] | true | false - 'feature_2' | 'feature_1' | ['file1.txt'] | false | true - 'feature_2' | 'feature_1' | ['file2.txt'] | true | true - 'feature_1' | 'tag_1' | ['file1.txt'] | true | false - 'feature_1' | 'tag_1' | ['file1.txt'] | false | true - 'feature_1' | 'tag_1' | ['file2.txt'] | true | true - 'feature_2' | 'tag_1' | ['file2.txt'] | true | true + where(:pipeline_ref, :compare_to, :paths, :result) do + 'feature_1' | 'master' | ['file1.txt'] | true + 'feature_1' | 'master' | ['README.md'] | false + 'feature_1' | 'master' | ['xyz.md'] | false + 'feature_2' | 'master' | ['file1.txt'] | true + 'feature_2' | 'master' | ['file2.txt'] | true + 'feature_2' | 'feature_1' | ['file1.txt'] | false + 'feature_2' | 'feature_1' | ['file2.txt'] | true + 'feature_1' | 'tag_1' | ['file1.txt'] | false + 'feature_1' | 'tag_1' | ['file2.txt'] | true + 'feature_2' | 'tag_1' | ['file2.txt'] | true end with_them do @@ -144,10 +142,6 @@ RSpec.describe Gitlab::Ci::Build::Rules::Rule::Clause::Changes do build(:ci_pipeline, project: project, ref: pipeline_ref, sha: project.commit(pipeline_ref).sha) end - before do - stub_feature_flags(ci_rules_changes_compare: ff) - end - it { is_expected.to eq(result) } end end @@ -174,14 +168,6 @@ RSpec.describe Gitlab::Ci::Build::Rules::Rule::Clause::Changes do ::Gitlab::Ci::Build::Rules::Rule::Clause::ParseError, 'rules:changes:compare_to is not a valid ref' ) end - - context 'when the FF ci_rules_changes_compare is disabled' do - before do - stub_feature_flags(ci_rules_changes_compare: false) - end - - it { is_expected.to be_truthy } - end end end end diff --git a/spec/lib/gitlab/ci/config/entry/legacy_variables_spec.rb b/spec/lib/gitlab/ci/config/entry/legacy_variables_spec.rb deleted file mode 100644 index e9edec9a0a4..00000000000 --- a/spec/lib/gitlab/ci/config/entry/legacy_variables_spec.rb +++ /dev/null @@ -1,173 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Ci::Config::Entry::LegacyVariables do - let(:config) { {} } - let(:metadata) { {} } - - subject(:entry) { described_class.new(config, **metadata) } - - before do - entry.compose! - end - - shared_examples 'valid config' do - describe '#value' do - it 'returns hash with key value strings' do - expect(entry.value).to eq result - end - end - - describe '#errors' do - it 'does not append errors' do - expect(entry.errors).to be_empty - end - end - - describe '#valid?' do - it 'is valid' do - expect(entry).to be_valid - end - end - end - - shared_examples 'invalid config' do |error_message| - describe '#valid?' do - it 'is not valid' do - expect(entry).not_to be_valid - end - end - - describe '#errors' do - it 'saves errors' do - expect(entry.errors) - .to include(error_message) - end - end - end - - context 'when entry config value has key-value pairs' do - let(:config) do - { 'VARIABLE_1' => 'value 1', 'VARIABLE_2' => 'value 2' } - end - - let(:result) do - { 'VARIABLE_1' => 'value 1', 'VARIABLE_2' => 'value 2' } - end - - it_behaves_like 'valid config' - - describe '#value_with_data' do - it 'returns variable with data' do - expect(entry.value_with_data).to eq( - 'VARIABLE_1' => { value: 'value 1' }, - 'VARIABLE_2' => { value: 'value 2' } - ) - end - end - end - - context 'with numeric keys and values in the config' do - let(:config) { { 10 => 20 } } - let(:result) do - { '10' => '20' } - end - - it_behaves_like 'valid config' - end - - context 'when key is an array' do - let(:config) { { ['VAR1'] => 'val1' } } - let(:result) do - { 'VAR1' => 'val1' } - end - - it_behaves_like 'invalid config', /should be a hash of key value pairs/ - end - - context 'when value is a symbol' do - let(:config) { { 'VAR1' => :val1 } } - let(:result) do - { 'VAR1' => 'val1' } - end - - it_behaves_like 'valid config' - end - - context 'when value is a boolean' do - let(:config) { { 'VAR1' => true } } - let(:result) do - { 'VAR1' => 'val1' } - end - - it_behaves_like 'invalid config', /should be a hash of key value pairs/ - end - - context 'when entry config value has key-value pair and hash' do - let(:config) do - { 'VARIABLE_1' => { value: 'value 1', description: 'variable 1' }, - 'VARIABLE_2' => 'value 2' } - end - - it_behaves_like 'invalid config', /should be a hash of key value pairs/ - - context 'when metadata has use_value_data: true' do - let(:metadata) { { use_value_data: true } } - - let(:result) do - { 'VARIABLE_1' => 'value 1', 'VARIABLE_2' => 'value 2' } - end - - it_behaves_like 'valid config' - - describe '#value_with_data' do - it 'returns variable with data' do - expect(entry.value_with_data).to eq( - 'VARIABLE_1' => { value: 'value 1', description: 'variable 1' }, - 'VARIABLE_2' => { value: 'value 2' } - ) - end - end - end - end - - context 'when entry value is an array' do - let(:config) { [:VAR, 'test'] } - - it_behaves_like 'invalid config', /should be a hash of key value pairs/ - end - - context 'when metadata has use_value_data: true' do - let(:metadata) { { use_value_data: true } } - - context 'when entry value has hash with other key-pairs' do - let(:config) do - { 'VARIABLE_1' => { value: 'value 1', hello: 'variable 1' }, - 'VARIABLE_2' => 'value 2' } - end - - it_behaves_like 'invalid config', /should be a hash of key value pairs, value can be a hash/ - end - - context 'when entry config value has hash with nil description' do - let(:config) do - { 'VARIABLE_1' => { value: 'value 1', description: nil } } - end - - it_behaves_like 'invalid config', /should be a hash of key value pairs, value can be a hash/ - end - - context 'when entry config value has hash without description' do - let(:config) do - { 'VARIABLE_1' => { value: 'value 1' } } - end - - let(:result) do - { 'VARIABLE_1' => 'value 1' } - end - - it_behaves_like 'valid config' - end - end -end diff --git a/spec/lib/gitlab/ci/config/entry/processable_spec.rb b/spec/lib/gitlab/ci/config/entry/processable_spec.rb index 5f42a8c49a7..ad90dd59585 100644 --- a/spec/lib/gitlab/ci/config/entry/processable_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/processable_spec.rb @@ -210,20 +210,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Processable do expect(entry.errors) .to include 'variables:var2 config must be a string' end - - context 'when the FF ci_variables_refactoring_to_variable is disabled' do - let(:entry_without_ff) { node_class.new(config, name: :rspec) } - - before do - stub_feature_flags(ci_variables_refactoring_to_variable: false) - entry_without_ff.compose! - end - - it 'reports error about variable' do - expect(entry_without_ff.errors) - .to include /config should be a hash of key value pairs/ - end - end end end end diff --git a/spec/lib/gitlab/ci/config/entry/product/parallel_spec.rb b/spec/lib/gitlab/ci/config/entry/product/parallel_spec.rb index 937642f07e7..a16f1cf9e43 100644 --- a/spec/lib/gitlab/ci/config/entry/product/parallel_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/product/parallel_spec.rb @@ -91,10 +91,11 @@ RSpec.describe ::Gitlab::Ci::Config::Entry::Product::Parallel do describe '#value' do it 'returns job needs configuration' do - expect(parallel.value).to match(matrix: [ - { PROVIDER: 'aws', STACK: %w[monitoring app1 app2] }, - { PROVIDER: 'gcp', STACK: %w[data processing] } - ]) + expect(parallel.value).to match(matrix: + [ + { PROVIDER: 'aws', STACK: %w[monitoring app1 app2] }, + { PROVIDER: 'gcp', STACK: %w[data processing] } + ]) end end diff --git a/spec/lib/gitlab/ci/config/entry/root_spec.rb b/spec/lib/gitlab/ci/config/entry/root_spec.rb index 3d19987a0be..a55e13e7c2d 100644 --- a/spec/lib/gitlab/ci/config/entry/root_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/root_spec.rb @@ -34,7 +34,11 @@ RSpec.describe Gitlab::Ci::Config::Entry::Root do image: 'image:1.0', default: {}, services: ['postgres:9.1', 'mysql:5.5'], - variables: { VAR: 'root', VAR2: { value: 'val 2', description: 'this is var 2' } }, + variables: { + VAR: 'root', + VAR2: { value: 'val 2', description: 'this is var 2' }, + VAR3: { value: %w[val3 val3b], description: 'this is var 3' } + }, after_script: ['make clean'], stages: %w(build pages release), cache: { key: 'k', untracked: true, paths: ['public/'] }, @@ -83,7 +87,7 @@ RSpec.describe Gitlab::Ci::Config::Entry::Root do end it 'sets correct variables value' do - expect(root.variables_value).to eq('VAR' => 'root', 'VAR2' => 'val 2') + expect(root.variables_value).to eq('VAR' => 'root', 'VAR2' => 'val 2', 'VAR3' => 'val3') end describe '#leaf?' do @@ -361,20 +365,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Root do expect(root.errors) .to include /var1 config uses invalid data keys: invalid/ end - - context 'when the FF ci_variables_refactoring_to_variable is disabled' do - let(:root_without_ff) { described_class.new(hash, user: user, project: project) } - - before do - stub_feature_flags(ci_variables_refactoring_to_variable: false) - root_without_ff.compose! - end - - it 'reports errors about the invalid variable' do - expect(root_without_ff.errors) - .to include /variables config should be a hash of key value pairs, value can be a hash/ - end - end end end end diff --git a/spec/lib/gitlab/ci/config/entry/rules/rule_spec.rb b/spec/lib/gitlab/ci/config/entry/rules/rule_spec.rb index 303d825c591..3531d6e9f1a 100644 --- a/spec/lib/gitlab/ci/config/entry/rules/rule_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/rules/rule_spec.rb @@ -364,19 +364,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Rules::Rule do it 'returns an error about invalid variables:' do expect(subject.errors).to include(/variables config should be a hash/) end - - context 'when the FF ci_variables_refactoring_to_variable is disabled' do - let(:entry_without_ff) { factory.create! } - - before do - stub_feature_flags(ci_variables_refactoring_to_variable: false) - entry_without_ff.compose! - end - - it 'returns an error about invalid variables:' do - expect(subject.errors).to include(/variables config should be a hash/) - end - end end end diff --git a/spec/lib/gitlab/ci/config/entry/variable_spec.rb b/spec/lib/gitlab/ci/config/entry/variable_spec.rb index 744a89d4509..076a5b32e92 100644 --- a/spec/lib/gitlab/ci/config/entry/variable_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/variable_spec.rb @@ -127,20 +127,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Variable do end end - context 'when config value is an array' do - let(:config) { { value: ['value'], description: 'description' } } - - describe '#valid?' do - it { is_expected.not_to be_valid } - end - - describe '#errors' do - subject(:errors) { entry.errors } - - it { is_expected.to include 'var1 config value must be an alphanumeric string' } - end - end - context 'when config description is a symbol' do let(:config) { { value: 'value', description: :description } } @@ -209,4 +195,42 @@ RSpec.describe Gitlab::Ci::Config::Entry::Variable do end end end + + describe 'ComplexArrayVariable' do + context 'when allow_array_value metadata is false' do + let(:config) { { value: %w[value value2], description: 'description' } } + let(:metadata) { { allow_array_value: false } } + + describe '#valid?' do + it { is_expected.not_to be_valid } + end + + describe '#errors' do + subject(:errors) { entry.errors } + + it { is_expected.to include 'var1 config value must be an alphanumeric string' } + end + end + + context 'when allow_array_value metadata is true' do + let(:config) { { value: %w[value value2], description: 'description' } } + let(:metadata) { { allowed_value_data: %i[value description], allow_array_value: true } } + + describe '#valid?' do + it { is_expected.to be_valid } + end + + describe '#value' do + subject(:value) { entry.value } + + it { is_expected.to eq('value') } + end + + describe '#value_with_data' do + subject(:value_with_data) { entry.value_with_data } + + it { is_expected.to eq(value: 'value', description: 'description', value_options: %w[value value2]) } + end + end + end end diff --git a/spec/lib/gitlab/ci/config/entry/variables_spec.rb b/spec/lib/gitlab/ci/config/entry/variables_spec.rb index ad7290d0589..085f304094e 100644 --- a/spec/lib/gitlab/ci/config/entry/variables_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/variables_spec.rb @@ -98,6 +98,62 @@ RSpec.describe Gitlab::Ci::Config::Entry::Variables do it_behaves_like 'invalid config', /must be either a string or a hash/ end + context 'when entry config value has unallowed value key-value pair and value is a string' do + let(:config) do + { 'VARIABLE_1' => { value: 'value', description: 'variable 1' } } + end + + context 'when there is no allowed_value_data metadata' do + it_behaves_like 'invalid config', /variable_1 config must be a string/ + end + + context 'when metadata has allow_array_value and allowed_value_data' do + let(:metadata) { { allowed_value_data: %i[value description], allow_array_value: true } } + + let(:result) do + { 'VARIABLE_1' => 'value' } + end + + it_behaves_like 'valid config' + + describe '#value_with_data' do + it 'returns variable with data' do + expect(entry.value_with_data).to eq( + 'VARIABLE_1' => { value: 'value', description: 'variable 1' } + ) + end + end + end + end + + context 'when entry config value has key-value pair and value is an array' do + let(:config) do + { 'VARIABLE_1' => { value: %w[value1 value2], description: 'variable 1' } } + end + + context 'when there is no allowed_value_data metadata' do + it_behaves_like 'invalid config', /variable_1 config value must be an alphanumeric string/ + end + + context 'when metadata has allow_array_value and allowed_value_data' do + let(:metadata) { { allowed_value_data: %i[value description], allow_array_value: true } } + + let(:result) do + { 'VARIABLE_1' => 'value1' } + end + + it_behaves_like 'valid config' + + describe '#value_with_data' do + it 'returns variable with data' do + expect(entry.value_with_data).to eq( + 'VARIABLE_1' => { value: 'value1', value_options: %w[value1 value2], description: 'variable 1' } + ) + end + end + end + end + context 'when entry config value has key-value pair and hash' do let(:config) do { 'VARIABLE_1' => { value: 'value 1', description: 'variable 1' }, diff --git a/spec/lib/gitlab/ci/config/entry/workflow_spec.rb b/spec/lib/gitlab/ci/config/entry/workflow_spec.rb index 3d19832e13d..97ac199f47d 100644 --- a/spec/lib/gitlab/ci/config/entry/workflow_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/workflow_spec.rb @@ -65,6 +65,54 @@ RSpec.describe Gitlab::Ci::Config::Entry::Workflow do end end end + + context 'with workflow name' do + let(:factory) { Gitlab::Config::Entry::Factory.new(described_class).value(workflow_hash) } + + context 'with a blank name' do + let(:workflow_hash) do + { name: '' } + end + + it 'is invalid' do + expect(config).not_to be_valid + end + + it 'returns error about invalid name' do + expect(config.errors).to include('workflow name is too short (minimum is 1 character)') + end + end + + context 'with too long name' do + let(:workflow_hash) do + { name: 'a' * 256 } + end + + it 'is invalid' do + expect(config).not_to be_valid + end + + it 'returns error about invalid name' do + expect(config.errors).to include('workflow name is too long (maximum is 255 characters)') + end + end + + context 'when name is nil' do + let(:workflow_hash) { { name: nil } } + + it 'is valid' do + expect(config).to be_valid + end + end + + context 'when name is not provided' do + let(:workflow_hash) { { rules: [{ if: '$VAR' }] } } + + it 'is valid' do + expect(config).to be_valid + end + end + end end end diff --git a/spec/lib/gitlab/ci/config/external/file/artifact_spec.rb b/spec/lib/gitlab/ci/config/external/file/artifact_spec.rb index 9da8d106862..a8dc7897082 100644 --- a/spec/lib/gitlab/ci/config/external/file/artifact_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/artifact_spec.rb @@ -174,9 +174,10 @@ RSpec.describe Gitlab::Ci::Config::External::File::Artifact do context 'when job is provided as a variable' do let(:variables) do - Gitlab::Ci::Variables::Collection.new([ - { key: 'VAR1', value: 'a_secret_variable_value', masked: true } - ]) + Gitlab::Ci::Variables::Collection.new( + [ + { key: 'VAR1', value: 'a_secret_variable_value', masked: true } + ]) end let(:params) { { artifact: 'generated.yml', job: 'a_secret_variable_value' } } diff --git a/spec/lib/gitlab/ci/config/external/file/project_spec.rb b/spec/lib/gitlab/ci/config/external/file/project_spec.rb index 72a85c9b03d..0ba92d1e92d 100644 --- a/spec/lib/gitlab/ci/config/external/file/project_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/project_spec.rb @@ -163,9 +163,7 @@ RSpec.describe Gitlab::Ci::Config::External::File::Project do context 'when non-existing project is used with a masked variable' do let(:variables) do - Gitlab::Ci::Variables::Collection.new([ - { key: 'VAR1', value: 'a_secret_variable_value', masked: true } - ]) + Gitlab::Ci::Variables::Collection.new([{ key: 'VAR1', value: 'a_secret_variable_value', masked: true }]) end let(:params) do @@ -180,9 +178,7 @@ RSpec.describe Gitlab::Ci::Config::External::File::Project do context 'when a project contained in an array is used with a masked variable' do let(:variables) do - Gitlab::Ci::Variables::Collection.new([ - { key: 'VAR1', value: 'a_secret_variable_value', masked: true } - ]) + Gitlab::Ci::Variables::Collection.new([{ key: 'VAR1', value: 'a_secret_variable_value', masked: true }]) end let(:params) do @@ -231,10 +227,11 @@ RSpec.describe Gitlab::Ci::Config::External::File::Project do context 'when project name and ref include masked variables' do let(:variables) do - Gitlab::Ci::Variables::Collection.new([ - { key: 'VAR1', value: 'a_secret_variable_value1', masked: true }, - { key: 'VAR2', value: 'a_secret_variable_value2', masked: true } - ]) + Gitlab::Ci::Variables::Collection.new( + [ + { key: 'VAR1', value: 'a_secret_variable_value1', masked: true }, + { key: 'VAR2', value: 'a_secret_variable_value2', masked: true } + ]) end let(:params) { { project: 'a_secret_variable_value1', ref: 'a_secret_variable_value2', file: '/file.yml' } } diff --git a/spec/lib/gitlab/ci/config/external/mapper_spec.rb b/spec/lib/gitlab/ci/config/external/mapper_spec.rb index 9eaba12f388..e12f5dcee0a 100644 --- a/spec/lib/gitlab/ci/config/external/mapper_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper_spec.rb @@ -207,9 +207,9 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do context "when duplicate 'include's are defined" do let(:values) do { include: [ - { 'local' => local_file }, - { 'local' => local_file } - ], + { 'local' => local_file }, + { 'local' => local_file } + ], image: 'image:1.0' } end @@ -416,17 +416,18 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do context "when locations are same after masking variables" do let(:variables) do - Gitlab::Ci::Variables::Collection.new([ - { 'key' => 'GITLAB_TOKEN', 'value' => 'secret-file1', 'masked' => true }, - { 'key' => 'GITLAB_TOKEN', 'value' => 'secret-file2', 'masked' => true } - ]) + Gitlab::Ci::Variables::Collection.new( + [ + { 'key' => 'GITLAB_TOKEN', 'value' => 'secret-file1', 'masked' => true }, + { 'key' => 'GITLAB_TOKEN', 'value' => 'secret-file2', 'masked' => true } + ]) end let(:values) do { include: [ - { 'local' => 'hello/secret-file1.yml' }, - { 'local' => 'hello/secret-file2.yml' } - ], + { 'local' => 'hello/secret-file1.yml' }, + { 'local' => 'hello/secret-file2.yml' } + ], image: 'ruby:2.7' } end diff --git a/spec/lib/gitlab/ci/config_spec.rb b/spec/lib/gitlab/ci/config_spec.rb index 055114769ea..475503de7da 100644 --- a/spec/lib/gitlab/ci/config_spec.rb +++ b/spec/lib/gitlab/ci/config_spec.rb @@ -889,4 +889,31 @@ RSpec.describe Gitlab::Ci::Config do it { is_expected.to eq([{ if: '$CI_COMMIT_REF_NAME == "master"' }]) } end + + describe '#workflow_name' do + subject(:workflow_name) { config.workflow_name } + + let(:yml) do + <<-EOS + workflow: + name: 'Pipeline name' + + rspec: + script: exit 0 + EOS + end + + it { is_expected.to eq('Pipeline name') } + + context 'with no name' do + let(:yml) do + <<-EOS + rspec: + script: exit 0 + EOS + end + + it { is_expected.to be_nil } + end + end end diff --git a/spec/lib/gitlab/ci/jwt_v2_spec.rb b/spec/lib/gitlab/ci/jwt_v2_spec.rb index 33aaa145a39..5eeab658a8e 100644 --- a/spec/lib/gitlab/ci/jwt_v2_spec.rb +++ b/spec/lib/gitlab/ci/jwt_v2_spec.rb @@ -7,6 +7,8 @@ RSpec.describe Gitlab::Ci::JwtV2 do let(:project) { build_stubbed(:project, namespace: namespace) } let(:user) { build_stubbed(:user) } let(:pipeline) { build_stubbed(:ci_pipeline, ref: 'auto-deploy-2020-03-19') } + let(:aud) { described_class::DEFAULT_AUD } + let(:build) do build_stubbed( :ci_build, @@ -16,7 +18,7 @@ RSpec.describe Gitlab::Ci::JwtV2 do ) end - subject(:ci_job_jwt_v2) { described_class.new(build, ttl: 30) } + subject(:ci_job_jwt_v2) { described_class.new(build, ttl: 30, aud: aud) } it { is_expected.to be_a Gitlab::Ci::Jwt } @@ -30,5 +32,13 @@ RSpec.describe Gitlab::Ci::JwtV2 do expect(payload[:sub]).to eq("project_path:#{project.full_path}:ref_type:branch:ref:#{pipeline.source_ref}") end end + + context 'when given an aud' do + let(:aud) { 'AWS' } + + it 'uses that aud in the payload' do + expect(payload[:aud]).to eq('AWS') + end + end end end diff --git a/spec/lib/gitlab/ci/lint_spec.rb b/spec/lib/gitlab/ci/lint_spec.rb index 3d46d266c13..cf07e952f26 100644 --- a/spec/lib/gitlab/ci/lint_spec.rb +++ b/spec/lib/gitlab/ci/lint_spec.rb @@ -342,6 +342,7 @@ RSpec.describe Gitlab::Ci::Lint do { 'count' => a_kind_of(Numeric), 'avg' => a_kind_of(Numeric), + 'sum' => a_kind_of(Numeric), 'max' => a_kind_of(Numeric), 'min' => a_kind_of(Numeric) } diff --git a/spec/lib/gitlab/ci/parsers/sbom/source/dependency_scanning_spec.rb b/spec/lib/gitlab/ci/parsers/sbom/source/dependency_scanning_spec.rb index 7222ebc3cb8..e12fa380209 100644 --- a/spec/lib/gitlab/ci/parsers/sbom/source/dependency_scanning_spec.rb +++ b/spec/lib/gitlab/ci/parsers/sbom/source/dependency_scanning_spec.rb @@ -19,8 +19,7 @@ RSpec.describe Gitlab::Ci::Parsers::Sbom::Source::DependencyScanning do it 'returns expected source data' do is_expected.to have_attributes( source_type: :dependency_scanning, - data: property_data, - fingerprint: '4dbcb747e6f0fb3ed4f48d96b777f1d64acdf43e459fdfefad404e55c004a188' + data: property_data ) end end diff --git a/spec/lib/gitlab/ci/parsers/sbom/validators/cyclonedx_schema_validator_spec.rb b/spec/lib/gitlab/ci/parsers/sbom/validators/cyclonedx_schema_validator_spec.rb index c54a3268bbe..f58a463f047 100644 --- a/spec/lib/gitlab/ci/parsers/sbom/validators/cyclonedx_schema_validator_spec.rb +++ b/spec/lib/gitlab/ci/parsers/sbom/validators/cyclonedx_schema_validator_spec.rb @@ -72,12 +72,13 @@ RSpec.describe Gitlab::Ci::Parsers::Sbom::Validators::CyclonedxSchemaValidator d it { is_expected.not_to be_valid } it "outputs errors for each validation failure" do - expect(validator.errors).to match_array([ - "property '/components/0' is missing required keys: name", - "property '/components/0/type' is not one of: [\"application\", \"framework\"," \ - " \"library\", \"container\", \"operating-system\", \"device\", \"firmware\", \"file\"]", - "property '/components/1' is missing required keys: type" - ]) + expect(validator.errors).to match_array( + [ + "property '/components/0' is missing required keys: name", + "property '/components/0/type' is not one of: [\"application\", \"framework\"," \ + " \"library\", \"container\", \"operating-system\", \"device\", \"firmware\", \"file\"]", + "property '/components/1' is missing required keys: type" + ]) end end end @@ -121,10 +122,11 @@ RSpec.describe Gitlab::Ci::Parsers::Sbom::Validators::CyclonedxSchemaValidator d it { is_expected.not_to be_valid } it "outputs errors for each validation failure" do - expect(validator.errors).to match_array([ - "property '/metadata/properties/0/name' is not of type: string", - "property '/metadata/properties/0/value' is not of type: string" - ]) + expect(validator.errors).to match_array( + [ + "property '/metadata/properties/0/name' is not of type: string", + "property '/metadata/properties/0/value' is not of type: string" + ]) end end end diff --git a/spec/lib/gitlab/ci/parsers/security/common_spec.rb b/spec/lib/gitlab/ci/parsers/security/common_spec.rb index 297ef1f5bb9..7dbad354e4c 100644 --- a/spec/lib/gitlab/ci/parsers/security/common_spec.rb +++ b/spec/lib/gitlab/ci/parsers/security/common_spec.rb @@ -54,24 +54,15 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Common do context 'when the validate flag is set to `false`' do let(:validate) { false } - let(:valid?) { false } - let(:errors) { ['foo'] } - let(:warnings) { ['bar'] } before do - allow_next_instance_of(validator_class) do |instance| - allow(instance).to receive(:valid?).and_return(valid?) - allow(instance).to receive(:errors).and_return(errors) - allow(instance).to receive(:warnings).and_return(warnings) - end - allow(parser).to receive_messages(create_scanner: true, create_scan: true) end - it 'instantiates the validator with correct params' do + it 'does not instantiate the validator' do parse_report - expect(validator_class).to have_received(:new).with( + expect(validator_class).not_to have_received(:new).with( report.type, data.deep_stringify_keys, report.version, @@ -80,43 +71,17 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Common do ) end - context 'when the report data is not valid according to the schema' do - it 'adds warnings to the report' do - expect { parse_report }.to change { report.warnings }.from([]).to( - [ - { message: 'foo', type: 'Schema' }, - { message: 'bar', type: 'Schema' } - ] - ) - end - - it 'keeps the execution flow as normal' do - parse_report + it 'marks the report as valid' do + parse_report - expect(parser).to have_received(:create_scanner) - expect(parser).to have_received(:create_scan) - end + expect(report).not_to be_errored end - context 'when the report data is valid according to the schema' do - let(:valid?) { true } - let(:errors) { [] } - let(:warnings) { [] } - - it 'does not add errors to the report' do - expect { parse_report }.not_to change { report.errors } - end - - it 'does not add warnings to the report' do - expect { parse_report }.not_to change { report.warnings } - end - - it 'keeps the execution flow as normal' do - parse_report + it 'keeps the execution flow as normal' do + parse_report - expect(parser).to have_received(:create_scanner) - expect(parser).to have_received(:create_scan) - end + expect(parser).to have_received(:create_scanner) + expect(parser).to have_received(:create_scan) end end @@ -152,12 +117,17 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Common do it 'adds errors to the report' do expect { parse_report }.to change { report.errors }.from([]).to( [ - { message: 'foo', type: 'Schema' }, - { message: 'bar', type: 'Schema' } + { message: 'foo', type: 'Schema' } ] ) end + it 'marks the report as invalid' do + parse_report + + expect(report).to be_errored + end + it 'does not try to create report entities' do parse_report @@ -175,8 +145,24 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Common do expect { parse_report }.not_to change { report.errors }.from([]) end - it 'does not add warnings to the report' do - expect { parse_report }.not_to change { report.warnings }.from([]) + context 'and no warnings are present' do + let(:warnings) { [] } + + it 'does not add warnings to the report' do + expect { parse_report }.not_to change { report.warnings }.from([]) + end + end + + context 'and some warnings are present' do + let(:warnings) { ['bar'] } + + it 'does add warnings to the report' do + expect { parse_report }.to change { report.warnings }.from([]).to( + [ + { message: 'bar', type: 'Schema' } + ] + ) + end end it 'keeps the execution flow as normal' do @@ -298,8 +284,8 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Common do scans = report.findings.map(&:scan) expect(scans.map(&:status).all?('success')).to be(true) - expect(scans.map(&:start_time).all?('placeholder-value')).to be(true) - expect(scans.map(&:end_time).all?('placeholder-value')).to be(true) + expect(scans.map(&:start_time).all?('2022-08-10T21:37:00')).to be(true) + expect(scans.map(&:end_time).all?('2022-08-10T21:38:00')).to be(true) expect(scans.size).to eq(7) expect(scans.first).to be_a(::Gitlab::Ci::Reports::Security::Scan) end @@ -418,11 +404,11 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Common do { 'type' => 'source', 'items' => [ - 'signatures' => [ - { 'algorithm' => 'hash', 'value' => 'hash_value' }, - { 'algorithm' => 'location', 'value' => 'location_value' }, - { 'algorithm' => 'scope_offset', 'value' => 'scope_offset_value' } - ] + 'signatures' => [ + { 'algorithm' => 'hash', 'value' => 'hash_value' }, + { 'algorithm' => 'location', 'value' => 'location_value' }, + { 'algorithm' => 'scope_offset', 'value' => 'scope_offset_value' } + ] ] } end @@ -440,11 +426,11 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Common do { 'type' => 'source', 'items' => [ - 'signatures' => [ - { 'algorithm' => 'hash', 'value' => 'hash_value' }, - { 'algorithm' => 'location', 'value' => 'location_value' }, - { 'algorithm' => 'INVALID', 'value' => 'scope_offset_value' } - ] + 'signatures' => [ + { 'algorithm' => 'hash', 'value' => 'hash_value' }, + { 'algorithm' => 'location', 'value' => 'location_value' }, + { 'algorithm' => 'INVALID', 'value' => 'scope_offset_value' } + ] ] } end diff --git a/spec/lib/gitlab/ci/parsers/security/sast_spec.rb b/spec/lib/gitlab/ci/parsers/security/sast_spec.rb index 4bc48f6611a..f6113308201 100644 --- a/spec/lib/gitlab/ci/parsers/security/sast_spec.rb +++ b/spec/lib/gitlab/ci/parsers/security/sast_spec.rb @@ -10,24 +10,39 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Sast do let(:created_at) { 2.weeks.ago } - context "when parsing valid reports" do - where(:report_format, :report_version, :scanner_length, :finding_length, :identifier_length, :file_path, :line) do - :sast | '14.0.0' | 1 | 5 | 6 | 'groovy/src/main/java/com/gitlab/security_products/tests/App.groovy' | 47 - :sast_deprecated | '1.2' | 3 | 33 | 17 | 'python/hardcoded/hardcoded-tmp.py' | 1 + context "when passing valid report" do + # rubocop: disable Layout/LineLength + where(:report_format, :report_version, :scanner_length, :finding_length, :identifier_length, :file_path, :start_line, :end_line, :primary_identifiers_length) do + :sast | '14.0.0' | 1 | 5 | 6 | 'groovy/src/main/java/com/gitlab/security_products/tests/App.groovy' | 47 | 47 | nil + :sast_semgrep_for_multiple_findings | '14.0.4' | 1 | 2 | 6 | 'app/app.py' | 39 | nil | 2 end + # rubocop: enable Layout/LineLength with_them do - let(:report) { Gitlab::Ci::Reports::Security::Report.new(artifact.file_type, pipeline, created_at) } + let(:report) do + Gitlab::Ci::Reports::Security::Report.new( + artifact.file_type, + pipeline, + created_at + ) + end + let(:artifact) { create(:ci_job_artifact, report_format) } before do - artifact.each_blob { |blob| described_class.parse!(blob, report) } + artifact.each_blob { |blob| described_class.parse!(blob, report, validate: true) } end it "parses all identifiers and findings" do expect(report.findings.length).to eq(finding_length) expect(report.identifiers.length).to eq(identifier_length) expect(report.scanners.length).to eq(scanner_length) + + if primary_identifiers_length + expect( + report.scanners.each_value.first.primary_identifiers.length + ).to eq(primary_identifiers_length) + end end it 'generates expected location' do @@ -36,8 +51,8 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Sast do expect(location).to be_a(::Gitlab::Ci::Reports::Security::Locations::Sast) expect(location).to have_attributes( file_path: file_path, - end_line: line, - start_line: line + end_line: end_line, + start_line: start_line ) end diff --git a/spec/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines_spec.rb index b570f2a7f75..fc3de2a14cd 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines_spec.rb @@ -44,6 +44,20 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::CancelPendingPipelines do expect(build_statuses(pipeline)).to contain_exactly('pending') end + it 'logs canceled pipelines' do + allow(Gitlab::AppLogger).to receive(:info) + + perform + + expect(Gitlab::AppLogger).to have_received(:info).with( + class: described_class.name, + message: "Pipeline #{pipeline.id} auto-canceling pipeline #{prev_pipeline.id}", + canceled_pipeline_id: prev_pipeline.id, + canceled_by_pipeline_id: pipeline.id, + canceled_by_pipeline_source: pipeline.source + ) + end + it 'cancels the builds with 2 queries to avoid query timeout' do second_query_regex = /WHERE "ci_pipelines"\."id" = \d+ AND \(NOT EXISTS/ recorder = ActiveRecord::QueryRecorder.new { perform } @@ -141,7 +155,42 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::CancelPendingPipelines do end end - context 'when the prev pipeline source is webide' do + context 'when the pipeline is a child pipeline' do + let!(:parent_pipeline) { create(:ci_pipeline, project: project, sha: new_commit.sha) } + let(:pipeline) { create(:ci_pipeline, child_of: parent_pipeline) } + + before do + create(:ci_build, :interruptible, :running, pipeline: parent_pipeline) + create(:ci_build, :interruptible, :running, pipeline: parent_pipeline) + end + + it 'does not cancel any builds' do + expect(build_statuses(prev_pipeline)).to contain_exactly('running', 'success', 'created') + expect(build_statuses(parent_pipeline)).to contain_exactly('running', 'running') + + perform + + expect(build_statuses(prev_pipeline)).to contain_exactly('running', 'success', 'created') + expect(build_statuses(parent_pipeline)).to contain_exactly('running', 'running') + end + + context 'when feature flag ci_skip_auto_cancelation_on_child_pipelines is disabled' do + before do + stub_feature_flags(ci_skip_auto_cancelation_on_child_pipelines: false) + end + + it 'does not cancel the parent pipeline' do + expect(build_statuses(parent_pipeline)).to contain_exactly('running', 'running') + + perform + + expect(build_statuses(prev_pipeline)).to contain_exactly('success', 'canceled', 'canceled') + expect(build_statuses(parent_pipeline)).to contain_exactly('running', 'running') + end + end + end + + context 'when the previous pipeline source is webide' do let(:prev_pipeline) { create(:ci_pipeline, :webide, project: project) } it 'does not cancel builds of the previous pipeline' do diff --git a/spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb index f451bd6bfef..e0d656f456e 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb @@ -11,9 +11,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content do subject { described_class.new(pipeline, command) } - # TODO: change this to `describe` and remove rubocop-disable - # when removing the FF ci_project_pipeline_config_refactoring - shared_context '#perform!' do # rubocop:disable RSpec/ContextWording + describe '#perform!' do context 'when bridge job is passed in as parameter' do let(:ci_config_path) { nil } let(:bridge) { create(:ci_bridge) } @@ -203,14 +201,4 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content do end end end - - it_behaves_like '#perform!' - - context 'when the FF ci_project_pipeline_config_refactoring is disabled' do - before do - stub_feature_flags(ci_project_pipeline_config_refactoring: false) - end - - it_behaves_like '#perform!' - end end diff --git a/spec/lib/gitlab/ci/pipeline/chain/limit/active_jobs_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/limit/active_jobs_spec.rb new file mode 100644 index 00000000000..bc453f1502b --- /dev/null +++ b/spec/lib/gitlab/ci/pipeline/chain/limit/active_jobs_spec.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ::Gitlab::Ci::Pipeline::Chain::Limit::ActiveJobs do + let_it_be(:namespace) { create(:namespace) } + let_it_be(:project) { create(:project, namespace: namespace) } + let_it_be(:user) { create(:user) } + let_it_be(:default_plan) { create(:default_plan) } + + let(:command) do + instance_double( + ::Gitlab::Ci::Pipeline::Chain::Command, + project: project, + current_user: user, + save_incompleted: true, + pipeline_seed: pipeline_seed_double + ) + end + + let(:pipeline_seed_double) do + instance_double(::Gitlab::Ci::Pipeline::Seed::Pipeline, size: 5) + end + + let(:pipeline) do + create(:ci_pipeline, project: project) + end + + let(:existing_pipeline) { create(:ci_pipeline, project: project) } + let(:step) { described_class.new(pipeline, command) } + let(:limit) { 10 } + + subject { step.perform! } + + before do + create(:plan_limits, plan: default_plan, ci_active_jobs: limit) + namespace.clear_memoization(:actual_plan) + end + + shared_examples 'successful step' do + it 'doest not fail the pipeline and does not interrupt the chain' do + subject + + expect(pipeline).not_to be_failed + expect(step).not_to be_break + end + end + + context 'when active jobs limit is exceeded' do + before do + create_list(:ci_build, 3, pipeline: existing_pipeline) + create_list(:ci_bridge, 3, pipeline: existing_pipeline) + end + + it 'fails the pipeline with an error', :aggregate_failures do + subject + + expect(pipeline).to be_failed + expect(pipeline).to be_job_activity_limit_exceeded + expect(pipeline.errors.full_messages).to include(described_class::MESSAGE) + end + + it 'logs the failure' do + allow(Gitlab::AppLogger).to receive(:info) + + subject + + expect(Gitlab::AppLogger).to have_received(:info).with( + class: described_class.name, + message: described_class::MESSAGE, + project_id: project.id, + plan: default_plan.name + ) + end + + it 'breaks the chain' do + subject + + expect(step).to be_break + end + + context 'when active jobs limit not enabled' do + let(:limit) { 0 } + + it_behaves_like 'successful step' + end + end + + context 'when active jobs limit is not exceeded' do + before do + create_list(:ci_build, 3, pipeline: existing_pipeline) + create_list(:ci_bridge, 1, pipeline: existing_pipeline) + end + + it_behaves_like 'successful step' + end +end diff --git a/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb index 62de4d2e96d..51d1661b586 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb @@ -236,4 +236,47 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Populate do end end end + + context 'with pipeline name' do + let(:config) do + { workflow: { name: ' Pipeline name ' }, rspec: { script: 'rspec' } } + end + + context 'with feature flag disabled' do + before do + stub_feature_flags(pipeline_name: false) + end + + it 'does not build pipeline_metadata' do + run_chain + + expect(pipeline.pipeline_metadata).to be_nil + end + end + + context 'with feature flag enabled' do + before do + stub_feature_flags(pipeline_name: true) + end + + it 'builds pipeline_metadata' do + run_chain + + expect(pipeline.pipeline_metadata.title).to eq('Pipeline name') + expect(pipeline.pipeline_metadata.project).to eq(pipeline.project) + end + + context 'with empty name' do + let(:config) do + { workflow: { name: ' ' }, rspec: { script: 'rspec' } } + end + + it 'strips whitespace from name' do + run_chain + + expect(pipeline.pipeline_metadata).to be_nil + end + end + end + end end diff --git a/spec/lib/gitlab/ci/pipeline/chain/seed_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/seed_spec.rb index 8c4f7af0ef4..323bab89e6a 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/seed_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/seed_spec.rb @@ -68,8 +68,10 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Seed do end context 'when refs policy is specified' do + let(:tag_name) { project.repository.tags.first.name } + let(:pipeline) do - build(:ci_pipeline, project: project, ref: 'feature', tag: true) + build(:ci_pipeline, project: project, ref: tag_name, tag: true) end let(:config) do diff --git a/spec/lib/gitlab/ci/pipeline/chain/sequence_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/sequence_spec.rb index ee32661f267..c69aa661b05 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/sequence_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/sequence_spec.rb @@ -100,19 +100,6 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Sequence do expect(histogram).to have_received(:observe) .with(hash_including(plan: project.actual_plan_name), 4) end - - context 'when feature flag ci_limit_active_jobs_early is disabled' do - before do - stub_feature_flags(ci_limit_active_jobs_early: false) - end - - it 'counts all the active builds' do - subject.build! - - expect(histogram).to have_received(:observe) - .with(hash_including(plan: project.actual_plan_name), 3) - end - end end end end diff --git a/spec/lib/gitlab/ci/pipeline/duration_spec.rb b/spec/lib/gitlab/ci/pipeline/duration_spec.rb index 46c7072ad8e..36714413da6 100644 --- a/spec/lib/gitlab/ci/pipeline/duration_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/duration_spec.rb @@ -1,117 +1,187 @@ # frozen_string_literal: true -require 'fast_spec_helper' +require 'spec_helper' RSpec.describe Gitlab::Ci::Pipeline::Duration do - let(:calculated_duration) { calculate(data) } + describe '.from_periods' do + let(:calculated_duration) { calculate(data) } - shared_examples 'calculating duration' do - it do - expect(calculated_duration).to eq(duration) + shared_examples 'calculating duration' do + it do + expect(calculated_duration).to eq(duration) + end end - end - context 'test sample A' do - let(:data) do - [[0, 1], - [1, 2], - [3, 4], - [5, 6]] + context 'test sample A' do + let(:data) do + [[0, 1], + [1, 2], + [3, 4], + [5, 6]] + end + + let(:duration) { 4 } + + it_behaves_like 'calculating duration' end - let(:duration) { 4 } + context 'test sample B' do + let(:data) do + [[0, 1], + [1, 2], + [2, 3], + [3, 4], + [0, 4]] + end - it_behaves_like 'calculating duration' - end + let(:duration) { 4 } - context 'test sample B' do - let(:data) do - [[0, 1], - [1, 2], - [2, 3], - [3, 4], - [0, 4]] + it_behaves_like 'calculating duration' end - let(:duration) { 4 } + context 'test sample C' do + let(:data) do + [[0, 4], + [2, 6], + [5, 7], + [8, 9]] + end - it_behaves_like 'calculating duration' - end + let(:duration) { 8 } - context 'test sample C' do - let(:data) do - [[0, 4], - [2, 6], - [5, 7], - [8, 9]] + it_behaves_like 'calculating duration' end - let(:duration) { 8 } + context 'test sample D' do + let(:data) do + [[0, 1], + [2, 3], + [4, 5], + [6, 7]] + end - it_behaves_like 'calculating duration' - end + let(:duration) { 4 } + + it_behaves_like 'calculating duration' + end - context 'test sample D' do - let(:data) do - [[0, 1], - [2, 3], - [4, 5], - [6, 7]] + context 'test sample E' do + let(:data) do + [[0, 1], + [3, 9], + [3, 4], + [3, 5], + [3, 8], + [4, 5], + [4, 7], + [5, 8]] + end + + let(:duration) { 7 } + + it_behaves_like 'calculating duration' end - let(:duration) { 4 } + context 'test sample F' do + let(:data) do + [[1, 3], + [2, 4], + [2, 4], + [2, 4], + [5, 8]] + end - it_behaves_like 'calculating duration' - end + let(:duration) { 6 } - context 'test sample E' do - let(:data) do - [[0, 1], - [3, 9], - [3, 4], - [3, 5], - [3, 8], - [4, 5], - [4, 7], - [5, 8]] + it_behaves_like 'calculating duration' end - let(:duration) { 7 } + context 'test sample G' do + let(:data) do + [[1, 3], + [2, 4], + [6, 7]] + end - it_behaves_like 'calculating duration' - end + let(:duration) { 4 } - context 'test sample F' do - let(:data) do - [[1, 3], - [2, 4], - [2, 4], - [2, 4], - [5, 8]] + it_behaves_like 'calculating duration' end - let(:duration) { 6 } + def calculate(data) + periods = data.shuffle.map do |(first, last)| + described_class::Period.new(first, last) + end - it_behaves_like 'calculating duration' + described_class.from_periods(periods.sort_by(&:first)) + end end - context 'test sample G' do - let(:data) do - [[1, 3], - [2, 4], - [6, 7]] + describe '.from_pipeline' do + let_it_be(:start_time) { Time.current.change(usec: 0) } + let_it_be(:current) { start_time + 1000 } + let_it_be(:pipeline) { create(:ci_pipeline) } + let_it_be(:success_build) { create_build(:success, started_at: start_time, finished_at: start_time + 60) } + let_it_be(:failed_build) { create_build(:failed, started_at: start_time + 60, finished_at: start_time + 120) } + let_it_be(:canceled_build) { create_build(:canceled, started_at: start_time + 120, finished_at: start_time + 180) } + let_it_be(:skipped_build) { create_build(:skipped, started_at: start_time) } + let_it_be(:pending_build) { create_build(:pending) } + let_it_be(:created_build) { create_build(:created) } + let_it_be(:preparing_build) { create_build(:preparing) } + let_it_be(:scheduled_build) { create_build(:scheduled) } + let_it_be(:expired_scheduled_build) { create_build(:expired_scheduled) } + let_it_be(:manual_build) { create_build(:manual) } + + let!(:running_build) { create_build(:running, started_at: start_time) } + + it 'returns the duration of the running build' do + travel_to(current) do + expect(described_class.from_pipeline(pipeline)).to eq 1000.seconds + end end - let(:duration) { 4 } + context 'when there is no running build' do + let(:running_build) { nil } - it_behaves_like 'calculating duration' - end + it 'returns the duration for all the builds' do + travel_to(current) do + expect(described_class.from_pipeline(pipeline)).to eq 180.seconds + end + end + end - def calculate(data) - periods = data.shuffle.map do |(first, last)| - described_class::Period.new(first, last) + context 'when there are bridge jobs' do + let!(:success_bridge) { create_bridge(:success, started_at: start_time + 220, finished_at: start_time + 280) } + let!(:failed_bridge) { create_bridge(:failed, started_at: start_time + 180, finished_at: start_time + 240) } + let!(:skipped_bridge) { create_bridge(:skipped, started_at: start_time) } + let!(:created_bridge) { create_bridge(:created) } + let!(:manual_bridge) { create_bridge(:manual) } + + it 'returns the duration of the running build' do + travel_to(current) do + expect(described_class.from_pipeline(pipeline)).to eq 1000.seconds + end + end + + context 'when there is no running build' do + let!(:running_build) { nil } + + it 'returns the duration for all the builds and bridge jobs' do + travel_to(current) do + expect(described_class.from_pipeline(pipeline)).to eq 280.seconds + end + end + end end - described_class.from_periods(periods.sort_by(&:first)) + private + + def create_build(trait, **opts) + create(:ci_build, trait, pipeline: pipeline, **opts) + end + + def create_bridge(trait, **opts) + create(:ci_bridge, trait, pipeline: pipeline, **opts) + end end end diff --git a/spec/lib/gitlab/ci/pipeline/logger_spec.rb b/spec/lib/gitlab/ci/pipeline/logger_spec.rb index f31361431f2..3af0ebe7484 100644 --- a/spec/lib/gitlab/ci/pipeline/logger_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/logger_spec.rb @@ -25,6 +25,7 @@ RSpec.describe ::Gitlab::Ci::Pipeline::Logger do loggable_data = { 'expensive_operation_duration_s' => { 'count' => 1, + 'sum' => a_kind_of(Numeric), 'avg' => a_kind_of(Numeric), 'max' => a_kind_of(Numeric), 'min' => a_kind_of(Numeric) @@ -62,6 +63,7 @@ RSpec.describe ::Gitlab::Ci::Pipeline::Logger do accumulator[key] = { 'count' => count, 'avg' => a_kind_of(Numeric), + 'sum' => a_kind_of(Numeric), 'max' => a_kind_of(Numeric), 'min' => a_kind_of(Numeric) } @@ -71,6 +73,7 @@ RSpec.describe ::Gitlab::Ci::Pipeline::Logger do data['expensive_operation_db_count']['max'] = db_count data['expensive_operation_db_count']['min'] = db_count data['expensive_operation_db_count']['avg'] = db_count + data['expensive_operation_db_count']['sum'] = count * db_count end data @@ -131,7 +134,7 @@ RSpec.describe ::Gitlab::Ci::Pipeline::Logger do it 'records durations of observed operations' do loggable_data = { 'pipeline_creation_duration_s' => { - 'avg' => 30, 'count' => 1, 'max' => 30, 'min' => 30 + 'avg' => 30, 'sum' => 30, 'count' => 1, 'max' => 30, 'min' => 30 } } @@ -165,10 +168,10 @@ RSpec.describe ::Gitlab::Ci::Pipeline::Logger do 'pipeline_creation_caller' => 'source', 'pipeline_source' => pipeline.source, 'pipeline_save_duration_s' => { - 'avg' => 60, 'count' => 1, 'max' => 60, 'min' => 60 + 'avg' => 60, 'sum' => 60, 'count' => 1, 'max' => 60, 'min' => 60 }, 'pipeline_creation_duration_s' => { - 'avg' => 20, 'count' => 2, 'max' => 30, 'min' => 10 + 'avg' => 20, 'sum' => 40, 'count' => 2, 'max' => 30, 'min' => 10 } } end @@ -215,10 +218,10 @@ RSpec.describe ::Gitlab::Ci::Pipeline::Logger do 'pipeline_creation_service_duration_s' => a_kind_of(Numeric), 'pipeline_creation_caller' => 'source', 'pipeline_save_duration_s' => { - 'avg' => 60, 'count' => 1, 'max' => 60, 'min' => 60 + 'avg' => 60, 'sum' => 60, 'count' => 1, 'max' => 60, 'min' => 60 }, 'pipeline_creation_duration_s' => { - 'avg' => 20, 'count' => 2, 'max' => 30, 'min' => 10 + 'avg' => 20, 'sum' => 40, 'count' => 2, 'max' => 30, 'min' => 10 } } end diff --git a/spec/lib/gitlab/ci/reports/codequality_reports_comparer_spec.rb b/spec/lib/gitlab/ci/reports/codequality_reports_comparer_spec.rb index e289e59b281..effa2c43418 100644 --- a/spec/lib/gitlab/ci/reports/codequality_reports_comparer_spec.rb +++ b/spec/lib/gitlab/ci/reports/codequality_reports_comparer_spec.rb @@ -191,11 +191,12 @@ RSpec.describe Gitlab::Ci::Reports::CodequalityReportsComparer do end it 'includes the base report errors sorted by severity' do - expect(existing_errors).to eq([ - blocker_degradation, - critical_degradation, - major_degradation - ]) + expect(existing_errors).to eq( + [ + blocker_degradation, + critical_degradation, + major_degradation + ]) end end @@ -242,11 +243,12 @@ RSpec.describe Gitlab::Ci::Reports::CodequalityReportsComparer do end it 'includes errors not found in the base report sorted by severity' do - expect(new_errors).to eq([ - blocker_degradation, - critical_degradation, - minor_degradation - ]) + expect(new_errors).to eq( + [ + blocker_degradation, + critical_degradation, + minor_degradation + ]) end end @@ -304,11 +306,12 @@ RSpec.describe Gitlab::Ci::Reports::CodequalityReportsComparer do end it 'returns the base report errors not found in the head report, sorted by severity' do - expect(resolved_errors).to eq([ - blocker_degradation, - critical_degradation, - minor_degradation - ]) + expect(resolved_errors).to eq( + [ + blocker_degradation, + critical_degradation, + minor_degradation + ]) end end diff --git a/spec/lib/gitlab/ci/reports/codequality_reports_spec.rb b/spec/lib/gitlab/ci/reports/codequality_reports_spec.rb index f4b47893805..68e70525c55 100644 --- a/spec/lib/gitlab/ci/reports/codequality_reports_spec.rb +++ b/spec/lib/gitlab/ci/reports/codequality_reports_spec.rb @@ -103,15 +103,16 @@ RSpec.describe Gitlab::Ci::Reports::CodequalityReports do end it 'sorts degradations based on severity' do - expect(codequality_report.degradations.values).to eq([ - blocker, - critical, - major, - major_2, - minor, - info, - unknown - ]) + expect(codequality_report.degradations.values).to eq( + [ + blocker, + critical, + major, + major_2, + minor, + info, + unknown + ]) end context 'with non-existence and uppercase severities' do @@ -126,12 +127,13 @@ RSpec.describe Gitlab::Ci::Reports::CodequalityReports do end it 'sorts unknown last' do - expect(other_report.degradations.values).to eq([ - blocker, - uppercase_major, - minor, - non_existent - ]) + expect(other_report.degradations.values).to eq( + [ + blocker, + uppercase_major, + minor, + non_existent + ]) end end end diff --git a/spec/lib/gitlab/ci/reports/sbom/source_spec.rb b/spec/lib/gitlab/ci/reports/sbom/source_spec.rb index cb30bd721dd..343c0d8c15c 100644 --- a/spec/lib/gitlab/ci/reports/sbom/source_spec.rb +++ b/spec/lib/gitlab/ci/reports/sbom/source_spec.rb @@ -12,8 +12,7 @@ RSpec.describe Gitlab::Ci::Reports::Sbom::Source do 'source_file' => { 'path' => 'package.json' }, 'package_manager' => { 'name' => 'npm' }, 'language' => { 'name' => 'JavaScript' } - }, - fingerprint: '4dbcb747e6f0fb3ed4f48d96b777f1d64acdf43e459fdfefad404e55c004a188' + } } end @@ -22,8 +21,7 @@ RSpec.describe Gitlab::Ci::Reports::Sbom::Source do it 'has correct attributes' do expect(subject).to have_attributes( source_type: attributes[:type], - data: attributes[:data], - fingerprint: attributes[:fingerprint] + data: attributes[:data] ) end end diff --git a/spec/lib/gitlab/ci/reports/security/report_spec.rb b/spec/lib/gitlab/ci/reports/security/report_spec.rb index ab0efb90901..d7f967f1c55 100644 --- a/spec/lib/gitlab/ci/reports/security/report_spec.rb +++ b/spec/lib/gitlab/ci/reports/security/report_spec.rb @@ -140,6 +140,24 @@ RSpec.describe Gitlab::Ci::Reports::Security::Report do it { is_expected.to eq(scanner_1) } end + describe '#primary_identifiers' do + it 'returns matching identifiers' do + scanner_with_identifiers = create( + :ci_reports_security_scanner, + external_id: 'external_id_1', + primary_identifiers: [create(:ci_reports_security_identifier, external_id: 'other_id', name: 'other_scanner')] + ) + scanner_without_identifiers = create( + :ci_reports_security_scanner, + external_id: 'external_id_2') + + report.add_scanner(scanner_with_identifiers) + report.add_scanner(scanner_without_identifiers) + + expect(report.primary_identifiers).to eq(scanner_with_identifiers.primary_identifiers) + end + end + describe '#add_error' do context 'when the message is not given' do it 'adds a new error to report with the generic error message' do diff --git a/spec/lib/gitlab/ci/secure_files/cer_spec.rb b/spec/lib/gitlab/ci/secure_files/cer_spec.rb new file mode 100644 index 00000000000..6b9cd0e3bfc --- /dev/null +++ b/spec/lib/gitlab/ci/secure_files/cer_spec.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::SecureFiles::Cer do + context 'when the supplied certificate cannot be parsed' do + let(:invalid_certificate) { described_class.new('xyzabc') } + + describe '#certificate_data' do + it 'assigns the error message and returns nil' do + expect(invalid_certificate.certificate_data).to be nil + expect(invalid_certificate.error).to eq('not enough data') + end + end + + describe '#metadata' do + it 'returns an empty hash' do + expect(invalid_certificate.metadata).to eq({}) + end + end + + describe '#expires_at' do + it 'returns nil' do + expect(invalid_certificate.metadata[:expires_at]).to be_nil + end + end + end + + context 'when the supplied certificate can be parsed' do + let(:sample_file) { fixture_file('ci_secure_files/sample.cer') } + let(:subject) { described_class.new(sample_file) } + + describe '#certificate_data' do + it 'returns an OpenSSL::X509::Certificate object' do + expect(subject.certificate_data.class).to be(OpenSSL::X509::Certificate) + end + end + + describe '#metadata' do + it 'returns a hash with the expected keys' do + expect(subject.metadata.keys).to match_array([:issuer, :subject, :id, :expires_at]) + end + end + + describe '#id' do + it 'returns the certificate serial number' do + expect(subject.metadata[:id]).to eq('33669367788748363528491290218354043267') + end + end + + describe '#expires_at' do + it 'returns the certificate expiration timestamp' do + expect(subject.metadata[:expires_at]).to eq('2022-04-26 19:20:40 UTC') + end + end + + describe '#issuer' do + it 'calls parse on X509Name' do + expect(subject.metadata[:issuer]["O"]).to eq('Apple Inc.') + end + end + + describe '#subject' do + it 'calls parse on X509Name' do + expect(subject.metadata[:subject]["OU"]).to eq('N7SYAN8PX8') + end + end + end +end diff --git a/spec/lib/gitlab/ci/secure_files/mobile_provision_spec.rb b/spec/lib/gitlab/ci/secure_files/mobile_provision_spec.rb new file mode 100644 index 00000000000..fb382174c64 --- /dev/null +++ b/spec/lib/gitlab/ci/secure_files/mobile_provision_spec.rb @@ -0,0 +1,149 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::SecureFiles::MobileProvision do + context 'when the supplied profile cannot be parsed' do + context 'when the supplied certificate cannot be parsed' do + let(:invalid_profile) { described_class.new('xyzabc') } + + describe '#decoded_plist' do + it 'assigns the error message and returns nil' do + expect(invalid_profile.decoded_plist).to be nil + expect(invalid_profile.error).to eq('Could not parse the PKCS7: not enough data') + end + end + + describe '#properties' do + it 'returns nil' do + expect(invalid_profile.properties).to be_nil + end + end + + describe '#metadata' do + it 'returns an empty hash' do + expect(invalid_profile.metadata).to eq({}) + end + end + + describe '#expires_at' do + it 'returns nil' do + expect(invalid_profile.metadata[:expires_at]).to be_nil + end + end + end + end + + context 'when the supplied profile can be parsed' do + let(:sample_file) { fixture_file('ci_secure_files/sample.mobileprovision') } + let(:subject) { described_class.new(sample_file) } + + describe '#decoded_plist' do + it 'returns an XML string' do + expect(subject.decoded_plist.class).to be(String) + expect(subject.decoded_plist.starts_with?('<?xml version="1.0"')).to be true + end + end + + describe '#properties' do + it 'returns the property list of the decoded plist provided' do + expect(subject.properties.class).to be(Hash) + expect(subject.properties.keys).to match_array(%w[AppIDName ApplicationIdentifierPrefix CreationDate + Platform IsXcodeManaged DeveloperCertificates + DER-Encoded-Profile PPQCheck Entitlements ExpirationDate + Name ProvisionedDevices TeamIdentifier TeamName + TimeToLive UUID Version]) + end + + it 'returns nil if the property list fails to be parsed from the decoded plist' do + allow(subject).to receive(:decoded_plist).and_return('foo/bar') + expect(subject.properties).to be nil + expect(subject.error).to start_with('invalid XML') + end + end + + describe '#metadata' do + it 'returns a hash with the expected keys' do + expect(subject.metadata.keys).to match_array([:id, :expires_at, :app_id, :app_id_prefix, :app_name, + :certificate_ids, :devices, :entitlements, :platforms, + :team_id, :team_name, :xcode_managed]) + end + end + + describe '#id' do + it 'returns the profile UUID' do + expect(subject.metadata[:id]).to eq('6b9fcce1-b9a9-4b37-b2ce-ec4da2044abf') + end + end + + describe '#expires_at' do + it 'returns the expiration timestamp of the profile' do + expect(subject.metadata[:expires_at].utc).to eq('2023-08-01 23:15:13 UTC') + end + end + + describe '#platforms' do + it 'returns the platforms assigned to the profile' do + expect(subject.metadata[:platforms]).to match_array(['iOS']) + end + end + + describe '#team_name' do + it 'returns the team name in the profile' do + expect(subject.metadata[:team_name]).to eq('Darby Frey') + end + end + + describe '#team_id' do + it 'returns the team ids in the profile' do + expect(subject.metadata[:team_id]).to match_array(['N7SYAN8PX8']) + end + end + + describe '#app_name' do + it 'returns the app name in the profile' do + expect(subject.metadata[:app_name]).to eq('iOS Demo') + end + end + + describe '#app_id' do + it 'returns the app id in the profile' do + expect(subject.metadata[:app_id]).to eq('match Development com.gitlab.ios-demo') + end + end + + describe '#app_id_prefix' do + it 'returns the app id prefixes in the profile' do + expect(subject.metadata[:app_id_prefix]).to match_array(['N7SYAN8PX8']) + end + end + + describe '#xcode_managed' do + it 'returns the xcode_managed property in the profile' do + expect(subject.metadata[:xcode_managed]).to be false + end + end + + describe '#entitlements' do + it 'returns the entitlements in the profile' do + expect(subject.metadata[:entitlements].keys).to match_array(['application-identifier', + 'com.apple.developer.game-center', + 'com.apple.developer.team-identifier', + 'get-task-allow', + 'keychain-access-groups']) + end + end + + describe '#devices' do + it 'returns the devices attached to the profile' do + expect(subject.metadata[:devices]).to match_array(["00008101-001454860C10001E"]) + end + end + + describe '#certificate_ids' do + it 'returns the certificate ids attached to the profile' do + expect(subject.metadata[:certificate_ids]).to match_array(["23380136242930206312716563638445789376"]) + end + end + end +end diff --git a/spec/lib/gitlab/ci/secure_files/p12_spec.rb b/spec/lib/gitlab/ci/secure_files/p12_spec.rb new file mode 100644 index 00000000000..beabf4b4856 --- /dev/null +++ b/spec/lib/gitlab/ci/secure_files/p12_spec.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::SecureFiles::P12 do + context 'when the supplied certificate cannot be parsed' do + let(:invalid_certificate) { described_class.new('xyzabc') } + + describe '#certificate_data' do + it 'assigns the error message and returns nil' do + expect(invalid_certificate.certificate_data).to be nil + expect(invalid_certificate.error).to eq('PKCS12_parse: mac verify failure') + end + end + + describe '#metadata' do + it 'returns an empty hash' do + expect(invalid_certificate.metadata).to eq({}) + end + end + + describe '#expires_at' do + it 'returns nil' do + expect(invalid_certificate.metadata[:expires_at]).to be_nil + end + end + end + + context 'when the supplied certificate can be parsed, but the password is invalid' do + let(:sample_file) { fixture_file('ci_secure_files/sample.p12') } + let(:subject) { described_class.new(sample_file, 'foo') } + + describe '#certificate_data' do + it 'assigns the error message and returns nil' do + expect(subject.certificate_data).to be nil + expect(subject.error).to eq('PKCS12_parse: mac verify failure') + end + end + end + + context 'when the supplied certificate can be parsed' do + let(:sample_file) { fixture_file('ci_secure_files/sample.p12') } + let(:subject) { described_class.new(sample_file) } + + describe '#certificate_data' do + it 'returns an OpenSSL::X509::Certificate object' do + expect(subject.certificate_data.class).to be(OpenSSL::X509::Certificate) + end + end + + describe '#metadata' do + it 'returns a hash with the expected keys' do + expect(subject.metadata.keys).to match_array([:issuer, :subject, :id, :expires_at]) + end + end + + describe '#id' do + it 'returns the certificate serial number' do + expect(subject.metadata[:id]).to eq('75949910542696343243264405377658443914') + end + end + + describe '#expires_at' do + it 'returns the certificate expiration timestamp' do + expect(subject.metadata[:expires_at]).to eq('2022-09-21 14:56:00 UTC') + end + end + + describe '#issuer' do + it 'calls parse on X509Name' do + expect(subject.metadata[:issuer]["O"]).to eq('Apple Inc.') + end + end + + describe '#subject' do + it 'calls parse on X509Name' do + expect(subject.metadata[:subject]["OU"]).to eq('N7SYAN8PX8') + end + end + end +end diff --git a/spec/lib/gitlab/ci/secure_files/x509_name_spec.rb b/spec/lib/gitlab/ci/secure_files/x509_name_spec.rb new file mode 100644 index 00000000000..3a523924c5b --- /dev/null +++ b/spec/lib/gitlab/ci/secure_files/x509_name_spec.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::SecureFiles::X509Name do + describe '.parse' do + it 'parses an X509Name object into a hash format' do + sample = OpenSSL::X509::Name.new([ + ['C', 'Test Country'], + ['O', 'Test Org Name'], + ['OU', 'Test Org Unit'], + ['CN', 'Test Common Name'], + ['UID', 'Test UID'] + ]) + + parsed_sample = described_class.parse(sample) + + expect(parsed_sample["C"]).to eq('Test Country') + expect(parsed_sample["O"]).to eq('Test Org Name') + expect(parsed_sample["OU"]).to eq('Test Org Unit') + expect(parsed_sample["CN"]).to eq('Test Common Name') + expect(parsed_sample["UID"]).to eq('Test UID') + end + + it 'returns an empty hash when an error occurs' do + parsed_sample = described_class.parse('unexpectedinput') + expect(parsed_sample).to eq({}) + end + end +end diff --git a/spec/lib/gitlab/ci/trace_spec.rb b/spec/lib/gitlab/ci/trace_spec.rb index 3043c8c5467..321a47c0634 100644 --- a/spec/lib/gitlab/ci/trace_spec.rb +++ b/spec/lib/gitlab/ci/trace_spec.rb @@ -74,7 +74,7 @@ RSpec.describe Gitlab::Ci::Trace, :clean_gitlab_redis_shared_state, factory_defa trace.being_watched! result = Gitlab::Redis::SharedState.with do |redis| - redis.exists(cache_key) + redis.exists?(cache_key) end expect(result).to eq(true) diff --git a/spec/lib/gitlab/ci/variables/builder/group_spec.rb b/spec/lib/gitlab/ci/variables/builder/group_spec.rb index 72487588cde..c3743ebd2d7 100644 --- a/spec/lib/gitlab/ci/variables/builder/group_spec.rb +++ b/spec/lib/gitlab/ci/variables/builder/group_spec.rb @@ -132,11 +132,12 @@ RSpec.describe Gitlab::Ci::Variables::Builder::Group do end it 'orders the variables from least to most matched' do - variables_collection = Gitlab::Ci::Variables::Collection.new([ - variable, - partially_matched_variable, - perfectly_matched_variable - ]).to_runner_variables + variables_collection = Gitlab::Ci::Variables::Collection.new( + [ + variable, + partially_matched_variable, + perfectly_matched_variable + ]).to_runner_variables expect(subject.to_runner_variables).to eq(variables_collection) end diff --git a/spec/lib/gitlab/ci/variables/builder/project_spec.rb b/spec/lib/gitlab/ci/variables/builder/project_spec.rb index b64b6ea98e2..c1cefc425f5 100644 --- a/spec/lib/gitlab/ci/variables/builder/project_spec.rb +++ b/spec/lib/gitlab/ci/variables/builder/project_spec.rb @@ -132,11 +132,12 @@ RSpec.describe Gitlab::Ci::Variables::Builder::Project do end it 'puts variables matching environment scope more in the end' do - variables_collection = Gitlab::Ci::Variables::Collection.new([ - variable, - partially_matched_variable, - perfectly_matched_variable - ]).to_runner_variables + variables_collection = Gitlab::Ci::Variables::Collection.new( + [ + variable, + partially_matched_variable, + perfectly_matched_variable + ]).to_runner_variables expect(subject.to_runner_variables).to eq(variables_collection) end diff --git a/spec/lib/gitlab/ci/variables/builder/release_spec.rb b/spec/lib/gitlab/ci/variables/builder/release_spec.rb new file mode 100644 index 00000000000..85b1659d07b --- /dev/null +++ b/spec/lib/gitlab/ci/variables/builder/release_spec.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Variables::Builder::Release do + let_it_be(:project) { create(:project, :public, :repository) } + let_it_be(:release) { create(:release, project: project) } + + let(:builder) { described_class.new(release) } + + describe '#variables' do + let(:description_variable) do + { + key: 'CI_RELEASE_DESCRIPTION', + value: release.description, + public: true, + masked: false, + raw: true + } + end + + subject do + builder.variables + end + + context 'when the release is present' do + let(:description_item) { item(description_variable) } + + it 'contains all the variables' do + is_expected.to contain_exactly(description_item) + end + + context 'for large description' do + before do + release.update_attribute(:description, "Test Description ..." * 5000) + end + + it 'truncates' do + expect(subject['CI_RELEASE_DESCRIPTION'].value.length).to eq(1024) + end + end + + context 'when description is nil' do + before do + release.update_attribute(:description, nil) + end + + it 'returns without error' do + builder = subject + + expect(builder).to match_array([]) + expect(builder.errors).to be_nil + end + end + end + + context 'when the release is not present' do + let(:release) { nil } + + it 'contains no variables' do + is_expected.to match_array([]) + end + end + end + + def item(variable) + ::Gitlab::Ci::Variables::Collection::Item.fabricate(variable) + end +end diff --git a/spec/lib/gitlab/ci/variables/builder_spec.rb b/spec/lib/gitlab/ci/variables/builder_spec.rb index 4833ccf9093..52ba85d2df1 100644 --- a/spec/lib/gitlab/ci/variables/builder_spec.rb +++ b/spec/lib/gitlab/ci/variables/builder_spec.rb @@ -10,6 +10,7 @@ RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache do let_it_be(:user) { create(:user) } let_it_be_with_reload(:job) do create(:ci_build, + name: 'rspec:test 1', pipeline: pipeline, user: user, yaml_variables: [{ key: 'YAML_VARIABLE', value: 'value' }] @@ -24,13 +25,15 @@ RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache do let(:predefined_variables) do [ { key: 'CI_JOB_NAME', - value: job.name }, + value: 'rspec:test 1' }, + { key: 'CI_JOB_NAME_SLUG', + value: 'rspec-test-1' }, { key: 'CI_JOB_STAGE', value: job.stage_name }, { key: 'CI_NODE_TOTAL', value: '1' }, { key: 'CI_BUILD_NAME', - value: job.name }, + value: 'rspec:test 1' }, { key: 'CI_BUILD_STAGE', value: job.stage_name }, { key: 'CI', @@ -171,6 +174,7 @@ RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache do allow(builder).to receive(:secret_project_variables) { [var('L', 12), var('M', 12)] } allow(pipeline).to receive(:variables) { [var('M', 13), var('N', 13)] } allow(pipeline).to receive(:pipeline_schedule) { double(job_variables: [var('N', 14), var('O', 14)]) } + allow(builder).to receive(:release_variables) { [var('P', 15), var('Q', 15)] } end it 'returns variables in order depending on resource hierarchy' do @@ -187,7 +191,8 @@ RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache do var('K', 11), var('L', 11), var('L', 12), var('M', 12), var('M', 13), var('N', 13), - var('N', 14), var('O', 14)]) + var('N', 14), var('O', 14), + var('P', 15), var('Q', 15)]) end it 'overrides duplicate keys depending on resource hierarchy' do @@ -199,7 +204,8 @@ RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache do 'I' => '9', 'J' => '10', 'K' => '11', 'L' => '12', 'M' => '13', 'N' => '14', - 'O' => '14') + 'O' => '14', 'P' => '15', + 'Q' => '15') end end @@ -216,6 +222,27 @@ RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache do .to include(a_hash_including(key: schedule_variable.key, value: schedule_variable.value)) end end + + context 'with release variables' do + let(:release_description_key) { 'CI_RELEASE_DESCRIPTION' } + + let_it_be(:tag) { project.repository.tags.first } + let_it_be(:pipeline) { create(:ci_pipeline, project: project, tag: true, ref: tag.name) } + let_it_be(:release) { create(:release, tag: tag.name, project: project) } + + it 'includes release variables' do + expect(subject.to_hash).to include(release_description_key => release.description) + end + + context 'when there is no release' do + let_it_be(:pipeline) { create(:ci_pipeline, project: project, tag: false, ref: 'master') } + let(:release) { nil } + + it 'does not include release variables' do + expect(subject.to_hash).not_to have_key(release_description_key) + end + end + end end describe '#user_variables' do @@ -261,10 +288,11 @@ RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache do end it 'includes #deployment_variables and merges the KUBECONFIG values', :aggregate_failures do - expect(builder).to receive(:deployment_variables).and_return([ - { key: 'KUBECONFIG', value: 'deployment-kubeconfig' }, - { key: 'OTHER', value: 'some value' } - ]) + expect(builder).to receive(:deployment_variables).and_return( + [ + { key: 'KUBECONFIG', value: 'deployment-kubeconfig' }, + { key: 'OTHER', value: 'some value' } + ]) expect(template).to receive(:merge_yaml).with('deployment-kubeconfig') expect(subject['KUBECONFIG'].value).to eq('example-kubeconfig') expect(subject['OTHER'].value).to eq('some value') diff --git a/spec/lib/gitlab/ci/variables/collection/sort_spec.rb b/spec/lib/gitlab/ci/variables/collection/sort_spec.rb index 57171e5be69..432225c53f0 100644 --- a/spec/lib/gitlab/ci/variables/collection/sort_spec.rb +++ b/spec/lib/gitlab/ci/variables/collection/sort_spec.rb @@ -192,13 +192,14 @@ RSpec.describe Gitlab::Ci::Variables::Collection::Sort do end it 'preserves relative order of overridden variables' do - is_expected.to eq([ - { 'TOP_LEVEL_GROUP_NAME' => 'top-level-group' }, - { 'SUBGROUP_VAR' => '$TOP_LEVEL_GROUP_NAME' }, - { 'SUB_GROUP_NAME' => 'vars-in-vars-subgroup' }, - { 'SUBGROUP_VAR' => '$SUB_GROUP_NAME' }, - { 'PROJECT_VAR' => '$SUBGROUP_VAR' } - ]) + is_expected.to eq( + [ + { 'TOP_LEVEL_GROUP_NAME' => 'top-level-group' }, + { 'SUBGROUP_VAR' => '$TOP_LEVEL_GROUP_NAME' }, + { 'SUB_GROUP_NAME' => 'vars-in-vars-subgroup' }, + { 'SUBGROUP_VAR' => '$SUB_GROUP_NAME' }, + { 'PROJECT_VAR' => '$SUBGROUP_VAR' } + ]) end end end diff --git a/spec/lib/gitlab/ci/variables/collection_spec.rb b/spec/lib/gitlab/ci/variables/collection_spec.rb index 8ac03301322..7d4a1eef70b 100644 --- a/spec/lib/gitlab/ci/variables/collection_spec.rb +++ b/spec/lib/gitlab/ci/variables/collection_spec.rb @@ -571,5 +571,42 @@ RSpec.describe Gitlab::Ci::Variables::Collection do end end end + + context 'with the file_variable_is_referenced_in_another_variable logging' do + let(:collection) do + Gitlab::Ci::Variables::Collection.new + .append(key: 'VAR1', value: 'test-1') + .append(key: 'VAR2', value: '$VAR1') + .append(key: 'VAR3', value: '$VAR1', raw: true) + .append(key: 'FILEVAR4', value: 'file-test-4', file: true) + .append(key: 'VAR5', value: '$FILEVAR4') + .append(key: 'VAR6', value: '$FILEVAR4', raw: true) + end + + subject(:sort_and_expand_all) { collection.sort_and_expand_all(project: project) } + + context 'when a project is not passed' do + let(:project) {} + + it 'does not log anything' do + expect(Gitlab::AppJsonLogger).not_to receive(:info) + + sort_and_expand_all + end + end + + context 'when a project is passed' do + let(:project) { create(:project) } + + it 'logs file_variable_is_referenced_in_another_variable once for VAR5' do + expect(Gitlab::AppJsonLogger).to receive(:info).with( + event: 'file_variable_is_referenced_in_another_variable', + project_id: project.id + ).once + + sort_and_expand_all + end + end + end end end diff --git a/spec/lib/gitlab/ci/yaml_processor/result_spec.rb b/spec/lib/gitlab/ci/yaml_processor/result_spec.rb index f7a0905d9da..7f203168706 100644 --- a/spec/lib/gitlab/ci/yaml_processor/result_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor/result_spec.rb @@ -71,10 +71,11 @@ module Gitlab subject(:yaml_variables_for) { result.yaml_variables_for(job_name) } it 'returns calculated variables with root and job variables' do - is_expected.to match_array([ - { key: 'VAR1', value: 'value 11' }, - { key: 'VAR2', value: 'value 2' } - ]) + is_expected.to match_array( + [ + { key: 'VAR1', value: 'value 11' }, + { key: 'VAR2', value: 'value 2' } + ]) end context 'when an absent job is sent' do diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb index cc327f5b5f1..ebf8422489e 100644 --- a/spec/lib/gitlab/ci/yaml_processor_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb @@ -15,8 +15,10 @@ module Gitlab end end - describe '#build_attributes' do - subject { described_class.new(config, user: nil).execute.build_attributes(:rspec) } + describe '#builds' do + subject(:builds) { described_class.new(config, user: nil).execute.builds } + + let(:rspec_build) { builds.find { |build| build[:name] == 'rspec' } } describe 'attributes list' do let(:config) do @@ -30,7 +32,7 @@ module Gitlab end it 'returns valid build attributes' do - expect(subject).to eq({ + expect(builds).to eq([{ stage: "test", stage_idx: 2, name: "rspec", @@ -45,7 +47,7 @@ module Gitlab job_variables: [], root_variables_inheritance: true, scheduling_type: :stage - }) + }]) end end @@ -63,7 +65,7 @@ module Gitlab end it 'returns valid build attributes' do - expect(subject).to eq({ + expect(builds).to eq([{ stage: 'test', stage_idx: 2, name: 'rspec', @@ -77,7 +79,7 @@ module Gitlab job_variables: [], root_variables_inheritance: true, scheduling_type: :stage - }) + }]) end end @@ -89,21 +91,22 @@ module Gitlab end it 'includes coverage regexp in build attributes' do - expect(subject) + expect(rspec_build) .to include(coverage_regex: 'Code coverage: \d+\.\d+') end end end describe 'tags entry with default values' do - it 'applies default values' do - config = YAML.dump({ default: { tags: %w[A B] }, - rspec: { script: "rspec" } }) - - config_processor = Gitlab::Ci::YamlProcessor.new(config).execute + let(:config) do + YAML.dump( + default: { tags: %w[A B] }, + rspec: { script: "rspec" } + ) + end - expect(config_processor.stage_builds_attributes("test").size).to eq(1) - expect(config_processor.stage_builds_attributes("test").first).to eq({ + it 'applies default values' do + expect(rspec_build).to eq({ stage: "test", stage_idx: 2, name: "rspec", @@ -125,7 +128,7 @@ module Gitlab YAML.dump(rspec: { script: 'rspec', interruptible: true }) end - it { expect(subject[:interruptible]).to be_truthy } + it { expect(rspec_build[:interruptible]).to be_truthy } end describe 'interruptible job with default value' do @@ -133,7 +136,7 @@ module Gitlab YAML.dump(rspec: { script: 'rspec' }) end - it { expect(subject).not_to have_key(:interruptible) } + it { expect(rspec_build).not_to have_key(:interruptible) } end describe 'uninterruptible job' do @@ -141,7 +144,7 @@ module Gitlab YAML.dump(rspec: { script: 'rspec', interruptible: false }) end - it { expect(subject[:interruptible]).to be_falsy } + it { expect(rspec_build[:interruptible]).to be_falsy } end it "returns interruptible when overridden for job" do @@ -149,9 +152,10 @@ module Gitlab rspec: { script: "rspec" } }) config_processor = Gitlab::Ci::YamlProcessor.new(config).execute + builds = config_processor.builds.select { |b| b[:stage] == "test" } - expect(config_processor.stage_builds_attributes("test").size).to eq(1) - expect(config_processor.stage_builds_attributes("test").first).to eq({ + expect(builds.size).to eq(1) + expect(builds.first).to eq({ stage: "test", stage_idx: 2, name: "rspec", @@ -174,7 +178,7 @@ module Gitlab end it 'includes retry count in build options attribute' do - expect(subject[:options]).to include(retry: { max: 1 }) + expect(rspec_build[:options]).to include(retry: { max: 1 }) end end @@ -184,7 +188,7 @@ module Gitlab end it 'does not persist retry count in the database' do - expect(subject[:options]).not_to have_key(:retry) + expect(rspec_build[:options]).not_to have_key(:retry) end end @@ -195,7 +199,7 @@ module Gitlab end it 'does use the default value' do - expect(subject[:options]).to include(retry: { max: 1 }) + expect(rspec_build[:options]).to include(retry: { max: 1 }) end end @@ -206,7 +210,7 @@ module Gitlab end it 'does use the job value' do - expect(subject[:options]).to include(retry: { max: 2 }) + expect(rspec_build[:options]).to include(retry: { max: 2 }) end end end @@ -221,7 +225,7 @@ module Gitlab end it 'is not allowed to fail' do - expect(subject[:allow_failure]).to be false + expect(rspec_build[:allow_failure]).to be false end end @@ -232,7 +236,7 @@ module Gitlab end it 'is allowed to fail' do - expect(subject[:allow_failure]).to be true + expect(rspec_build[:allow_failure]).to be true end end @@ -244,11 +248,11 @@ module Gitlab end it 'is not allowed to fail' do - expect(subject[:allow_failure]).to be false + expect(rspec_build[:allow_failure]).to be false end it 'saves allow_failure_criteria into options' do - expect(subject[:options]).to match( + expect(rspec_build[:options]).to match( a_hash_including(allow_failure_criteria: { exit_codes: [1] })) end end @@ -262,7 +266,7 @@ module Gitlab end it 'is not allowed to fail' do - expect(subject[:allow_failure]).to be false + expect(rspec_build[:allow_failure]).to be false end end @@ -272,7 +276,7 @@ module Gitlab end it 'is not allowed to fail' do - expect(subject[:allow_failure]).to be false + expect(rspec_build[:allow_failure]).to be false end end @@ -283,11 +287,11 @@ module Gitlab end it 'is not allowed to fail' do - expect(subject[:allow_failure]).to be false + expect(rspec_build[:allow_failure]).to be false end it 'saves allow_failure_criteria into options' do - expect(subject[:options]).to match( + expect(rspec_build[:options]).to match( a_hash_including(allow_failure_criteria: { exit_codes: [1] })) end end @@ -305,8 +309,8 @@ module Gitlab end it 'has the attributes' do - expect(subject[:when]).to eq 'delayed' - expect(subject[:options][:start_in]).to eq '1 day' + expect(rspec_build[:when]).to eq 'delayed' + expect(rspec_build[:options][:start_in]).to eq '1 day' end end end @@ -321,7 +325,7 @@ module Gitlab end it 'has the attributes' do - expect(subject[:resource_group_key]).to eq 'iOS' + expect(rspec_build[:resource_group_key]).to eq 'iOS' end end end @@ -337,7 +341,7 @@ module Gitlab end it 'has the attributes' do - expect(subject[:options]).to eq( + expect(rspec_build[:options]).to eq( trigger: { project: 'namespace/project', branch: 'main' } ) end @@ -353,7 +357,7 @@ module Gitlab end it 'has the attributes' do - expect(subject[:options]).to eq( + expect(rspec_build[:options]).to eq( trigger: { project: 'namespace/project', forward: { pipeline_variables: true } } ) end @@ -510,6 +514,35 @@ module Gitlab expect(subject.root_variables).to eq([]) end end + + context 'with name' do + let(:config) do + <<-EOYML + workflow: + name: 'Pipeline name' + + hello: + script: echo world + EOYML + end + + it 'parses the workflow:name as workflow_name' do + expect(subject.workflow_name).to eq('Pipeline name') + end + end + + context 'with no name' do + let(:config) do + <<-EOYML + hello: + script: echo world + EOYML + end + + it 'parses the workflow:name' do + expect(subject.workflow_name).to be_nil + end + end end describe '#warnings' do @@ -682,7 +715,7 @@ module Gitlab let(:config_data) { YAML.dump(config) } let(:config_processor) { Gitlab::Ci::YamlProcessor.new(config_data).execute } - subject { config_processor.stage_builds_attributes('test').first } + subject(:test_build) { config_processor.builds.find { |build| build[:name] == 'test' } } describe "before_script" do context "in global context" do @@ -850,9 +883,9 @@ module Gitlab rspec: { script: "rspec" } }) config_processor = Gitlab::Ci::YamlProcessor.new(config).execute + rspec_build = config_processor.builds.find { |build| build[:name] == 'rspec' } - expect(config_processor.stage_builds_attributes("test").size).to eq(1) - expect(config_processor.stage_builds_attributes("test").first).to eq({ + expect(rspec_build).to eq({ stage: "test", stage_idx: 2, name: "rspec", @@ -884,9 +917,9 @@ module Gitlab script: "rspec" } }) config_processor = Gitlab::Ci::YamlProcessor.new(config).execute + rspec_build = config_processor.builds.find { |build| build[:name] == 'rspec' } - expect(config_processor.stage_builds_attributes("test").size).to eq(1) - expect(config_processor.stage_builds_attributes("test").first).to eq({ + expect(rspec_build).to eq({ stage: "test", stage_idx: 2, name: "rspec", @@ -916,9 +949,9 @@ module Gitlab rspec: { script: "rspec" } }) config_processor = Gitlab::Ci::YamlProcessor.new(config).execute + rspec_build = config_processor.builds.find { |build| build[:name] == 'rspec' } - expect(config_processor.stage_builds_attributes("test").size).to eq(1) - expect(config_processor.stage_builds_attributes("test").first).to eq({ + expect(rspec_build).to eq({ stage: "test", stage_idx: 2, name: "rspec", @@ -944,9 +977,9 @@ module Gitlab rspec: { image: "image:1.0", services: ["postgresql", "docker:dind"], script: "rspec" } }) config_processor = Gitlab::Ci::YamlProcessor.new(config).execute + rspec_build = config_processor.builds.find { |build| build[:name] == 'rspec' } - expect(config_processor.stage_builds_attributes("test").size).to eq(1) - expect(config_processor.stage_builds_attributes("test").first).to eq({ + expect(rspec_build).to eq({ stage: "test", stage_idx: 2, name: "rspec", @@ -981,7 +1014,7 @@ module Gitlab it { is_expected.to be_valid } it "returns with image" do - expect(processor.stage_builds_attributes("test")).to contain_exactly({ + expect(processor.builds).to contain_exactly({ stage: "test", stage_idx: 2, name: "test", @@ -1014,7 +1047,7 @@ module Gitlab it { is_expected.to be_valid } it "returns with service" do - expect(processor.stage_builds_attributes("test")).to contain_exactly({ + expect(processor.builds).to contain_exactly({ stage: "test", stage_idx: 2, name: "test", @@ -1033,8 +1066,7 @@ module Gitlab end end - # Change this to a `describe` block when removing the FF ci_variables_refactoring_to_variable - shared_examples 'Variables' do + describe 'Variables' do subject(:execute) { described_class.new(config).execute } let(:build) { execute.builds.first } @@ -1163,18 +1195,6 @@ module Gitlab end end - context 'when ci_variables_refactoring_to_variable is enabled' do - it_behaves_like 'Variables' - end - - context 'when ci_variables_refactoring_to_variable is disabled' do - before do - stub_feature_flags(ci_variables_refactoring_to_variable: false) - end - - it_behaves_like 'Variables' - end - context 'when using `extends`' do let(:config_processor) { Gitlab::Ci::YamlProcessor.new(config).execute } @@ -1375,7 +1395,7 @@ module Gitlab }) config_processor = Gitlab::Ci::YamlProcessor.new(config).execute - builds = config_processor.stage_builds_attributes("test") + builds = config_processor.builds expect(builds.size).to eq(1) expect(builds.first[:when]).to eq(when_state) @@ -1391,7 +1411,7 @@ module Gitlab end it 'creates one build and sets when:' do - builds = subject.stage_builds_attributes("test") + builds = processor.builds expect(builds.size).to eq(1) expect(builds.first[:when]).to eq('delayed') @@ -1419,7 +1439,7 @@ module Gitlab end let(:config_processor) { Gitlab::Ci::YamlProcessor.new(config).execute } - let(:builds) { config_processor.stage_builds_attributes('test') } + let(:builds) { config_processor.builds } context 'when job is parallelized' do let(:parallel) { 5 } @@ -1535,15 +1555,16 @@ module Gitlab }) config_processor = Gitlab::Ci::YamlProcessor.new(config).execute + rspec_build = config_processor.builds.find { |build| build[:name] == 'rspec' } - expect(config_processor.stage_builds_attributes("test").size).to eq(1) - expect(config_processor.stage_builds_attributes("test").first[:cache]).to eq([ - paths: ["logs/", "binaries/"], - untracked: true, - key: 'key', - policy: 'pull-push', - when: 'on_success' - ]) + expect(rspec_build[:cache]).to eq( + [ + paths: ["logs/", "binaries/"], + untracked: true, + key: 'key', + policy: 'pull-push', + when: 'on_success' + ]) end it "returns cache when defined in default context" do @@ -1558,32 +1579,34 @@ module Gitlab }) config_processor = Gitlab::Ci::YamlProcessor.new(config).execute + rspec_build = config_processor.builds.find { |build| build[:name] == 'rspec' } - expect(config_processor.stage_builds_attributes("test").size).to eq(1) - expect(config_processor.stage_builds_attributes("test").first[:cache]).to eq([ - paths: ["logs/", "binaries/"], - untracked: true, - key: { files: ['file'] }, - policy: 'pull-push', - when: 'on_success' - ]) + expect(rspec_build[:cache]).to eq( + [ + paths: ["logs/", "binaries/"], + untracked: true, + key: { files: ['file'] }, + policy: 'pull-push', + when: 'on_success' + ]) end it 'returns cache key/s when defined in a job' do - config = YAML.dump({ - rspec: { - cache: [ - { paths: ['binaries/'], untracked: true, key: 'keya' }, - { paths: ['logs/', 'binaries/'], untracked: true, key: 'key' } - ], - script: 'rspec' - } - }) + config = YAML.dump( + { + rspec: { + cache: [ + { paths: ['binaries/'], untracked: true, key: 'keya' }, + { paths: ['logs/', 'binaries/'], untracked: true, key: 'key' } + ], + script: 'rspec' + } + }) config_processor = Gitlab::Ci::YamlProcessor.new(config).execute + rspec_build = config_processor.builds.find { |build| build[:name] == 'rspec' } - expect(config_processor.stage_builds_attributes('test').size).to eq(1) - expect(config_processor.stage_builds_attributes('test').first[:cache]).to eq( + expect(rspec_build[:cache]).to eq( [ { paths: ['binaries/'], @@ -1616,15 +1639,16 @@ module Gitlab ) config_processor = Gitlab::Ci::YamlProcessor.new(config).execute + rspec_build = config_processor.builds.find { |build| build[:name] == 'rspec' } - expect(config_processor.stage_builds_attributes('test').size).to eq(1) - expect(config_processor.stage_builds_attributes('test').first[:cache]).to eq([ - paths: ['binaries/'], - untracked: true, - key: { files: ['file'] }, - policy: 'pull-push', - when: 'on_success' - ]) + expect(rspec_build[:cache]).to eq( + [ + paths: ['binaries/'], + untracked: true, + key: { files: ['file'] }, + policy: 'pull-push', + when: 'on_success' + ]) end it 'returns cache files with prefix' do @@ -1640,61 +1664,65 @@ module Gitlab ) config_processor = Gitlab::Ci::YamlProcessor.new(config).execute + rspec_build = config_processor.builds.find { |build| build[:name] == 'rspec' } - expect(config_processor.stage_builds_attributes('test').size).to eq(1) - expect(config_processor.stage_builds_attributes('test').first[:cache]).to eq([ - paths: ['logs/', 'binaries/'], - untracked: true, - key: { files: ['file'], prefix: 'prefix' }, - policy: 'pull-push', - when: 'on_success' - ]) + expect(rspec_build[:cache]).to eq( + [ + paths: ['logs/', 'binaries/'], + untracked: true, + key: { files: ['file'], prefix: 'prefix' }, + policy: 'pull-push', + when: 'on_success' + ]) end it "overwrite cache when defined for a job and globally" do - config = YAML.dump({ - cache: { paths: ["logs/", "binaries/"], untracked: true, key: 'global' }, - rspec: { - script: "rspec", - cache: { paths: ["test/"], untracked: false, key: 'local' } - } - }) + config = YAML.dump( + { + cache: { paths: ["logs/", "binaries/"], untracked: true, key: 'global' }, + rspec: { + script: "rspec", + cache: { paths: ["test/"], untracked: false, key: 'local' } + } + }) config_processor = Gitlab::Ci::YamlProcessor.new(config).execute + rspec_build = config_processor.builds.find { |build| build[:name] == 'rspec' } - expect(config_processor.stage_builds_attributes("test").size).to eq(1) - expect(config_processor.stage_builds_attributes("test").first[:cache]).to eq([ - paths: ["test/"], - untracked: false, - key: 'local', - policy: 'pull-push', - when: 'on_success' - ]) + expect(rspec_build[:cache]).to eq( + [ + paths: ["test/"], + untracked: false, + key: 'local', + policy: 'pull-push', + when: 'on_success' + ]) end end describe "Artifacts" do it "returns artifacts when defined" do - config = YAML.dump({ - image: "image:1.0", - services: ["mysql"], - before_script: ["pwd"], - rspec: { - artifacts: { - paths: ["logs/", "binaries/"], - expose_as: "Exposed artifacts", - untracked: true, - name: "custom_name", - expire_in: "7d" - }, - script: "rspec" - } - }) + config = YAML.dump( + { + image: "image:1.0", + services: ["mysql"], + before_script: ["pwd"], + rspec: { + artifacts: { + paths: ["logs/", "binaries/"], + expose_as: "Exposed artifacts", + untracked: true, + name: "custom_name", + expire_in: "7d" + }, + script: "rspec" + } + }) config_processor = Gitlab::Ci::YamlProcessor.new(config).execute + rspec_build = config_processor.builds.find { |build| build[:name] == 'rspec' } - expect(config_processor.stage_builds_attributes("test").size).to eq(1) - expect(config_processor.stage_builds_attributes("test").first).to eq({ + expect(rspec_build).to eq({ stage: "test", stage_idx: 2, name: "rspec", @@ -1729,7 +1757,7 @@ module Gitlab }) config_processor = Gitlab::Ci::YamlProcessor.new(config).execute - builds = config_processor.stage_builds_attributes("test") + builds = config_processor.builds expect(builds.size).to eq(1) expect(builds.first[:options][:artifacts][:expire_in]).to eq('never') @@ -1745,7 +1773,7 @@ module Gitlab }) config_processor = Gitlab::Ci::YamlProcessor.new(config).execute - builds = config_processor.stage_builds_attributes("test") + builds = config_processor.builds expect(builds.size).to eq(1) expect(builds.first[:options][:artifacts][:when]).to eq(when_state) @@ -1778,7 +1806,7 @@ module Gitlab - my/test/something YAML - attributes = Gitlab::Ci::YamlProcessor.new(config).execute.build_attributes('test') + attributes = Gitlab::Ci::YamlProcessor.new(config).execute.builds.find { |build| build[:name] == 'test' } expect(attributes.dig(*%i[options artifacts exclude])).to eq(%w[my/test/something]) end @@ -1819,7 +1847,7 @@ module Gitlab end it "returns release info" do - expect(processor.stage_builds_attributes('release').first[:options]) + expect(processor.builds.first[:options]) .to eq(config[:release].except(:stage, :only)) end end @@ -1833,7 +1861,7 @@ module Gitlab subject { Gitlab::Ci::YamlProcessor.new(YAML.dump(config)).execute } - let(:builds) { subject.stage_builds_attributes('deploy') } + let(:builds) { subject.builds } context 'when a production environment is specified' do let(:environment) { 'production' } @@ -1943,7 +1971,7 @@ module Gitlab subject { Gitlab::Ci::YamlProcessor.new(YAML.dump(config)).execute } - let(:builds) { subject.stage_builds_attributes('deploy') } + let(:builds) { subject.builds } context 'when no timeout was provided' do it 'does not include job_timeout' do @@ -2370,8 +2398,8 @@ module Gitlab it 'returns a valid configuration and sets artifacts: true by default' do expect(subject).to be_valid - rspec = subject.build_attributes(:rspec) - expect(rspec.dig(:options, :cross_dependencies)).to eq( + rspec_build = subject.builds.find { |build| build[:name] == 'rspec' } + expect(rspec_build.dig(:options, :cross_dependencies)).to eq( [{ pipeline: '$THE_PIPELINE_ID', job: 'dependency-job', artifacts: true }] ) end @@ -2391,8 +2419,8 @@ module Gitlab it 'returns a valid configuration and sets artifacts: true by default' do expect(subject).to be_valid - rspec = subject.build_attributes(:rspec) - expect(rspec.dig(:options, :cross_dependencies)).to eq( + rspec_build = subject.builds.find { |build| build[:name] == 'rspec' } + expect(rspec_build.dig(:options, :cross_dependencies)).to eq( [{ pipeline: '123', job: 'dependency-job', artifacts: true }] ) end @@ -2422,7 +2450,7 @@ module Gitlab describe "Hidden jobs" do let(:config_processor) { Gitlab::Ci::YamlProcessor.new(config).execute } - subject { config_processor.stage_builds_attributes("test") } + subject { config_processor.builds } shared_examples 'hidden_job_handling' do it "doesn't create jobs that start with dot" do @@ -2470,7 +2498,7 @@ module Gitlab describe "YAML Alias/Anchor" do let(:config_processor) { Gitlab::Ci::YamlProcessor.new(config).execute } - subject { config_processor.stage_builds_attributes("build") } + subject { config_processor.builds } shared_examples 'job_templates_handling' do it "is correctly supported for jobs" do diff --git a/spec/lib/gitlab/config/entry/validators_spec.rb b/spec/lib/gitlab/config/entry/validators_spec.rb index cbc09aac586..0458bcd6354 100644 --- a/spec/lib/gitlab/config/entry/validators_spec.rb +++ b/spec/lib/gitlab/config/entry/validators_spec.rb @@ -35,7 +35,7 @@ RSpec.describe Gitlab::Config::Entry::Validators do expect(instance.valid?).to be(valid_result) unless valid_result - expect(instance.errors.messages_for(:config)).to include /please use only one the following keys: foo, bar/ + expect(instance.errors.messages_for(:config)).to include /please use only one of the following keys: foo, bar/ end end end diff --git a/spec/lib/gitlab/config_checker/external_database_checker_spec.rb b/spec/lib/gitlab/config_checker/external_database_checker_spec.rb index 933b6d6be9e..9af6aed2b02 100644 --- a/spec/lib/gitlab/config_checker/external_database_checker_spec.rb +++ b/spec/lib/gitlab/config_checker/external_database_checker_spec.rb @@ -6,36 +6,97 @@ RSpec.describe Gitlab::ConfigChecker::ExternalDatabaseChecker do describe '#check' do subject { described_class.check } - context 'when database meets minimum supported version' do + let(:old_database_version) { 8 } + let(:old_database) { instance_double(Gitlab::Database::Reflection) } + let(:new_database) { instance_double(Gitlab::Database::Reflection) } + + before do + allow(Gitlab::Database::Reflection).to receive(:new).and_return(new_database) + allow(old_database).to receive(:postgresql_minimum_supported_version?).and_return(false) + allow(old_database).to receive(:version).and_return(old_database_version) + allow(new_database).to receive(:postgresql_minimum_supported_version?).and_return(true) + end + + context 'with a single database' do before do - allow(ApplicationRecord.database).to receive(:postgresql_minimum_supported_version?).and_return(true) + skip_if_multiple_databases_are_setup + end + + context 'when database meets minimum supported version' do + before do + allow(Gitlab::Database::Reflection).to receive(:new).with(ActiveRecord::Base).and_return(new_database) + end + + it { is_expected.to be_empty } end - it { is_expected.to be_empty } + context 'when database does not meet minimum supported version' do + before do + allow(Gitlab::Database::Reflection).to receive(:new).with(ActiveRecord::Base).and_return(old_database) + end + + it 'reports deprecated database notice' do + is_expected.to contain_exactly(notice_deprecated_database(old_database_version)) + end + end end - context 'when database does not meet minimum supported version' do + context 'with a multiple database' do before do - allow(ApplicationRecord.database).to receive(:postgresql_minimum_supported_version?).and_return(false) + skip_if_multiple_databases_not_setup end - let(:notice_deprecated_database) do - { - type: 'warning', - message: _('You are using PostgreSQL %{pg_version_current}, but PostgreSQL ' \ - '%{pg_version_minimum} is required for this version of GitLab. ' \ - 'Please upgrade your environment to a supported PostgreSQL version, ' \ - 'see %{pg_requirements_url} for details.') % { - pg_version_current: ApplicationRecord.database.version, - pg_version_minimum: Gitlab::Database::MINIMUM_POSTGRES_VERSION, - pg_requirements_url: '<a href="https://docs.gitlab.com/ee/install/requirements.html#database">database requirements</a>' - } - } + context 'when both databases meets minimum supported version' do + before do + allow(Gitlab::Database::Reflection).to receive(:new).with(ActiveRecord::Base).and_return(new_database) + allow(Gitlab::Database::Reflection).to receive(:new).with(Ci::ApplicationRecord).and_return(new_database) + end + + it { is_expected.to be_empty } + end + + context 'when the one of the databases does not meet minimum supported version' do + it 'reports deprecated database notice if the main database is using an old version' do + allow(Gitlab::Database::Reflection).to receive(:new).with(ActiveRecord::Base).and_return(old_database) + allow(Gitlab::Database::Reflection).to receive(:new).with(Ci::ApplicationRecord).and_return(new_database) + is_expected.to contain_exactly(notice_deprecated_database(old_database_version)) + end + + it 'reports deprecated database notice if the ci database is using an old version' do + allow(Gitlab::Database::Reflection).to receive(:new).with(ActiveRecord::Base).and_return(new_database) + allow(Gitlab::Database::Reflection).to receive(:new).with(Ci::ApplicationRecord).and_return(old_database) + is_expected.to contain_exactly(notice_deprecated_database(old_database_version)) + end end - it 'reports deprecated database notice' do - is_expected.to contain_exactly(notice_deprecated_database) + context 'when both databases do not meet minimum supported version' do + before do + allow(Gitlab::Database::Reflection).to receive(:new).with(ActiveRecord::Base).and_return(old_database) + allow(Gitlab::Database::Reflection).to receive(:new).with(Ci::ApplicationRecord).and_return(old_database) + end + + it 'reports deprecated database notice' do + is_expected.to match_array [ + notice_deprecated_database(old_database_version), + notice_deprecated_database(old_database_version) + ] + end end end end + + def notice_deprecated_database(database_version) + { + type: 'warning', + message: _('You are using PostgreSQL %{pg_version_current}, but PostgreSQL ' \ + '%{pg_version_minimum} is required for this version of GitLab. ' \ + 'Please upgrade your environment to a supported PostgreSQL version, ' \ + 'see %{pg_requirements_url} for details.') % \ + { + pg_version_current: database_version, + pg_version_minimum: Gitlab::Database::MINIMUM_POSTGRES_VERSION, + pg_requirements_url: Gitlab::ConfigChecker::ExternalDatabaseChecker::PG_REQUIREMENTS_LINK + } + } + end end diff --git a/spec/lib/gitlab/conflict/file_spec.rb b/spec/lib/gitlab/conflict/file_spec.rb index aac4936b20e..1fa6eee9813 100644 --- a/spec/lib/gitlab/conflict/file_spec.rb +++ b/spec/lib/gitlab/conflict/file_spec.rb @@ -141,19 +141,20 @@ RSpec.describe Gitlab::Conflict::File do let(:raw_conflict_content) { index.merge_file('files/ruby/popen.rb')[:data] } it 'assign conflict types and adds match line to the end of the section' do - expect(diff_line_types).to eq([ - 'match', - nil, nil, nil, - "conflict_marker_our", - "conflict_our", - "conflict_marker", - "conflict_their", - "conflict_their", - "conflict_their", - "conflict_marker_their", - nil, nil, nil, - "match" - ]) + expect(diff_line_types).to eq( + [ + 'match', + nil, nil, nil, + "conflict_marker_our", + "conflict_our", + "conflict_marker", + "conflict_their", + "conflict_their", + "conflict_their", + "conflict_marker_their", + nil, nil, nil, + "match" + ]) end end end diff --git a/spec/lib/gitlab/data_builder/pipeline_spec.rb b/spec/lib/gitlab/data_builder/pipeline_spec.rb index 86a1539a836..46a12d8c6f6 100644 --- a/spec/lib/gitlab/data_builder/pipeline_spec.rb +++ b/spec/lib/gitlab/data_builder/pipeline_spec.rb @@ -30,6 +30,7 @@ RSpec.describe Gitlab::DataBuilder::Pipeline do expect(attributes[:sha]).to eq(pipeline.sha) expect(attributes[:tag]).to eq(pipeline.tag) expect(attributes[:id]).to eq(pipeline.id) + expect(attributes[:iid]).to eq(pipeline.iid) expect(attributes[:source]).to eq(pipeline.source) expect(attributes[:status]).to eq(pipeline.status) expect(attributes[:detailed_status]).to eq('passed') diff --git a/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb b/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb index 3daed2508a2..1ac9cbae036 100644 --- a/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb +++ b/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb @@ -83,7 +83,7 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m describe '#execute!' do context 'when an invalid transition is applied' do - %i[finished finalizing].each do |state| + %i[finalizing finished].each do |state| it 'raises an exception' do batched_migration = create(:batched_background_migration, state) @@ -103,6 +103,48 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m end end + describe '#finish!' do + context 'when an invalid transition is applied' do + it 'raises an exception' do + batched_migration = create(:batched_background_migration, :failed) + + expect { batched_migration.finish! }.to raise_error(StateMachines::InvalidTransition, /Cannot transition status/) + end + end + + context 'when a valid transition is applied' do + %i[active paused finished finalizing].each do |state| + it 'moves to active' do + batched_migration = create(:batched_background_migration, state) + + expect(batched_migration.finish!).to be_truthy + end + end + end + end + + describe '#failure!' do + context 'when an invalid transition is applied' do + %i[paused finished].each do |state| + it 'raises an exception' do + batched_migration = create(:batched_background_migration, state) + + expect { batched_migration.failure! }.to raise_error(StateMachines::InvalidTransition, /Cannot transition status/) + end + end + end + + context 'when a valid transition is applied' do + %i[failed finalizing active].each do |state| + it 'moves to active' do + batched_migration = create(:batched_background_migration, state) + + expect(batched_migration.failure!).to be_truthy + end + end + end + end + describe '.valid_status' do valid_status = [:paused, :active, :finished, :failed, :finalizing] diff --git a/spec/lib/gitlab/database/each_database_spec.rb b/spec/lib/gitlab/database/each_database_spec.rb index 2a6eb8f779d..75b543bee85 100644 --- a/spec/lib/gitlab/database/each_database_spec.rb +++ b/spec/lib/gitlab/database/each_database_spec.rb @@ -93,12 +93,13 @@ RSpec.describe Gitlab::Database::EachDatabase do end it 'yields each model with SharedModel connected to each database connection' do - expect_yielded_models([model1, model2], [ - { model: model1, connection: ActiveRecord::Base.connection, name: 'main' }, - { model: model1, connection: Ci::ApplicationRecord.connection, name: 'ci' }, - { model: model2, connection: ActiveRecord::Base.connection, name: 'main' }, - { model: model2, connection: Ci::ApplicationRecord.connection, name: 'ci' } - ]) + expect_yielded_models([model1, model2], + [ + { model: model1, connection: ActiveRecord::Base.connection, name: 'main' }, + { model: model1, connection: Ci::ApplicationRecord.connection, name: 'ci' }, + { model: model2, connection: ActiveRecord::Base.connection, name: 'main' }, + { model: model2, connection: Ci::ApplicationRecord.connection, name: 'ci' } + ]) end context 'when the model limits connection names' do @@ -108,10 +109,11 @@ RSpec.describe Gitlab::Database::EachDatabase do end it 'only yields the model with SharedModel connected to the limited connections' do - expect_yielded_models([model1, model2], [ - { model: model1, connection: ActiveRecord::Base.connection, name: 'main' }, - { model: model2, connection: Ci::ApplicationRecord.connection, name: 'ci' } - ]) + expect_yielded_models([model1, model2], + [ + { model: model1, connection: ActiveRecord::Base.connection, name: 'main' }, + { model: model2, connection: Ci::ApplicationRecord.connection, name: 'ci' } + ]) end end end @@ -132,10 +134,11 @@ RSpec.describe Gitlab::Database::EachDatabase do end it 'yields each model after connecting SharedModel' do - expect_yielded_models([main_model, ci_model], [ - { model: main_model, connection: main_connection, name: 'main' }, - { model: ci_model, connection: ci_connection, name: 'ci' } - ]) + expect_yielded_models([main_model, ci_model], + [ + { model: main_model, connection: main_connection, name: 'main' }, + { model: ci_model, connection: ci_connection, name: 'ci' } + ]) end end @@ -154,21 +157,23 @@ RSpec.describe Gitlab::Database::EachDatabase do context 'when a single name is passed in' do it 'yields models only connected to the given database' do - expect_yielded_models([main_model, ci_model, shared_model], [ - { model: ci_model, connection: Ci::ApplicationRecord.connection, name: 'ci' }, - { model: shared_model, connection: Ci::ApplicationRecord.connection, name: 'ci' } - ], only_on: 'ci') + expect_yielded_models([main_model, ci_model, shared_model], + [ + { model: ci_model, connection: Ci::ApplicationRecord.connection, name: 'ci' }, + { model: shared_model, connection: Ci::ApplicationRecord.connection, name: 'ci' } + ], only_on: 'ci') end end context 'when a list of names are passed in' do it 'yields models only connected to the given databases' do - expect_yielded_models([main_model, ci_model, shared_model], [ - { model: main_model, connection: ActiveRecord::Base.connection, name: 'main' }, - { model: ci_model, connection: Ci::ApplicationRecord.connection, name: 'ci' }, - { model: shared_model, connection: ActiveRecord::Base.connection, name: 'main' }, - { model: shared_model, connection: Ci::ApplicationRecord.connection, name: 'ci' } - ], only_on: %i[main ci]) + expect_yielded_models([main_model, ci_model, shared_model], + [ + { model: main_model, connection: ActiveRecord::Base.connection, name: 'main' }, + { model: ci_model, connection: Ci::ApplicationRecord.connection, name: 'ci' }, + { model: shared_model, connection: ActiveRecord::Base.connection, name: 'main' }, + { model: shared_model, connection: Ci::ApplicationRecord.connection, name: 'ci' } + ], only_on: %i[main ci]) end end end diff --git a/spec/lib/gitlab/database/load_balancing/load_balancer_spec.rb b/spec/lib/gitlab/database/load_balancing/load_balancer_spec.rb index 9c09253b24c..997c7a31cba 100644 --- a/spec/lib/gitlab/database/load_balancing/load_balancer_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/load_balancer_spec.rb @@ -210,10 +210,25 @@ RSpec.describe Gitlab::Database::LoadBalancing::LoadBalancer, :request_store do end it 'uses a retry with exponential backoffs' do - expect(lb).to receive(:retry_with_backoff).and_yield + expect(lb).to receive(:retry_with_backoff).and_yield(0) lb.read_write { 10 } end + + it 'does not raise NoMethodError error when primary_only?' do + connection = ActiveRecord::Base.connection_pool.connection + expected_error = Gitlab::Database::LoadBalancing::CONNECTION_ERRORS.first + + allow(lb).to receive(:primary_only?).and_return(true) + + expect do + lb.read_write do + connection.transaction do + raise expected_error + end + end + end.to raise_error(expected_error) + end end describe '#host' do @@ -330,6 +345,19 @@ RSpec.describe Gitlab::Database::LoadBalancing::LoadBalancer, :request_store do expect { lb.retry_with_backoff { raise } }.to raise_error(RuntimeError) end + + it 'yields the current retry iteration' do + allow(lb).to receive(:connection_error?).and_return(true) + expect(lb).to receive(:release_primary_connection).exactly(3).times + iterations = [] + + # time: 0 so that we don't sleep and slow down the test + # rubocop: disable Style/Semicolon + expect { lb.retry_with_backoff(attempts: 3, time: 0) { |i| iterations << i; raise } }.to raise_error(RuntimeError) + # rubocop: enable Style/Semicolon + + expect(iterations).to eq([1, 2, 3]) + end end describe '#connection_error?' do diff --git a/spec/lib/gitlab/database/load_balancing/rack_middleware_spec.rb b/spec/lib/gitlab/database/load_balancing/rack_middleware_spec.rb index a1c141af537..713bff5feea 100644 --- a/spec/lib/gitlab/database/load_balancing/rack_middleware_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/rack_middleware_spec.rb @@ -9,10 +9,10 @@ RSpec.describe Gitlab::Database::LoadBalancing::RackMiddleware, :redis do let(:single_sticking_object) { Set.new([[ActiveRecord::Base.sticking, :user, 42]]) } let(:multiple_sticking_objects) do Set.new([ - [ActiveRecord::Base.sticking, :user, 42], - [ActiveRecord::Base.sticking, :runner, '123456789'], - [ActiveRecord::Base.sticking, :runner, '1234'] - ]) + [ActiveRecord::Base.sticking, :user, 42], + [ActiveRecord::Base.sticking, :runner, '123456789'], + [ActiveRecord::Base.sticking, :runner, '1234'] + ]) end after do @@ -182,11 +182,12 @@ RSpec.describe Gitlab::Database::LoadBalancing::RackMiddleware, :redis do it 'returns the sticking object' do env = { described_class::STICK_OBJECT => multiple_sticking_objects } - expect(middleware.sticking_namespaces(env)).to eq([ - [ActiveRecord::Base.sticking, :user, 42], - [ActiveRecord::Base.sticking, :runner, '123456789'], - [ActiveRecord::Base.sticking, :runner, '1234'] - ]) + expect(middleware.sticking_namespaces(env)).to eq( + [ + [ActiveRecord::Base.sticking, :user, 42], + [ActiveRecord::Base.sticking, :runner, '123456789'], + [ActiveRecord::Base.sticking, :runner, '1234'] + ]) end end diff --git a/spec/lib/gitlab/database/load_balancing/sidekiq_server_middleware_spec.rb b/spec/lib/gitlab/database/load_balancing/sidekiq_server_middleware_spec.rb index 8053bd57bba..88007de53d3 100644 --- a/spec/lib/gitlab/database/load_balancing/sidekiq_server_middleware_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/sidekiq_server_middleware_spec.rb @@ -6,7 +6,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ let(:middleware) { described_class.new } let(:worker) { worker_class.new } let(:location) { '0/D525E3A8' } - let(:wal_locations) { { Gitlab::Database::MAIN_DATABASE_NAME.to_sym => location } } + let(:wal_locations) { { Gitlab::Database::MAIN_DATABASE_NAME.to_s => location } } let(:job) { { "retry" => 3, "job_id" => "a180b47c-3fd6-41b8-81e9-34da61c3400e", 'wal_locations' => wal_locations } } before do @@ -315,6 +315,46 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ expect(middleware.send(:databases_in_sync?, locations)) .to eq(false) end + + context 'when locations have string keys' do + it 'returns false when the load balancers are not in sync' do + locations = {} + + Gitlab::Database::LoadBalancing.each_load_balancer do |lb| + locations[lb.name.to_s] = 'foo' + + allow(lb) + .to receive(:select_up_to_date_host) + .with('foo') + .and_return(false) + end + + expect(middleware.send(:databases_in_sync?, locations)) + .to eq(false) + end + + context 'when "indifferent_wal_location_keys" FF is off' do + before do + stub_feature_flags(indifferent_wal_location_keys: false) + end + + it 'returns true when the load balancers are not in sync' do + locations = {} + + Gitlab::Database::LoadBalancing.each_load_balancer do |lb| + locations[lb.name.to_s] = 'foo' + + allow(lb) + .to receive(:select_up_to_date_host) + .with('foo') + .and_return(false) + end + + expect(middleware.send(:databases_in_sync?, locations)) + .to eq(true) + end + end + end end def process_job(job) diff --git a/spec/lib/gitlab/database/load_balancing/sticking_spec.rb b/spec/lib/gitlab/database/load_balancing/sticking_spec.rb index 2ffb2c32c32..1e316c55786 100644 --- a/spec/lib/gitlab/database/load_balancing/sticking_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/sticking_spec.rb @@ -41,10 +41,12 @@ RSpec.describe Gitlab::Database::LoadBalancing::Sticking, :redis do sticking.stick_or_unstick_request(env, :user, 42) sticking.stick_or_unstick_request(env, :runner, '123456789') - expect(env[Gitlab::Database::LoadBalancing::RackMiddleware::STICK_OBJECT].to_a).to eq([ - [sticking, :user, 42], - [sticking, :runner, '123456789'] - ]) + expect(env[Gitlab::Database::LoadBalancing::RackMiddleware::STICK_OBJECT].to_a).to eq( + [ + [sticking, :user, 42], + [sticking, :runner, + '123456789'] + ]) end end diff --git a/spec/lib/gitlab/database/load_balancing/transaction_leaking_spec.rb b/spec/lib/gitlab/database/load_balancing/transaction_leaking_spec.rb index 30e5fbbd803..6026d979bcf 100644 --- a/spec/lib/gitlab/database/load_balancing/transaction_leaking_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/transaction_leaking_spec.rb @@ -3,6 +3,7 @@ require 'spec_helper' RSpec.describe 'Load balancer behavior with errors inside a transaction', :redis, :delete do + include StubENV let(:model) { ApplicationRecord } let(:db_host) { model.connection_pool.db_config.host } @@ -19,6 +20,10 @@ RSpec.describe 'Load balancer behavior with errors inside a transaction', :redis model.connection.execute(<<~SQL) CREATE TABLE IF NOT EXISTS #{test_table_name} (id SERIAL PRIMARY KEY, value INTEGER) SQL + + # The load balancer sleeps between attempts to retry a query. + # Mocking the sleep call significantly reduces the runtime of this spec file. + allow(model.connection.load_balancer).to receive(:sleep) end after do @@ -46,36 +51,62 @@ RSpec.describe 'Load balancer behavior with errors inside a transaction', :redis conn.execute("INSERT INTO #{test_table_name} (value) VALUES (2)") end - it 'logs a warning when violating transaction semantics with writes' do - conn = model.connection + context 'with the PREVENT_LOAD_BALANCER_RETRIES_IN_TRANSACTION environment variable not set' do + it 'logs a warning when violating transaction semantics with writes' do + conn = model.connection + + expect(::Gitlab::Database::LoadBalancing::Logger).to receive(:warn).with(hash_including(event: :transaction_leak)) + + conn.transaction do + expect(conn).to be_transaction_open + + execute(conn) - expect(::Gitlab::Database::LoadBalancing::Logger).to receive(:warn).with(hash_including(event: :transaction_leak)) + expect(conn).not_to be_transaction_open + end - conn.transaction do - expect(conn).to be_transaction_open + values = conn.execute("SELECT value FROM #{test_table_name}").to_a.map { |row| row['value'] } + expect(values).to contain_exactly(2) # Does not include 1 because the transaction was aborted and leaked + end + + it 'does not log a warning when no transaction is open to be leaked' do + conn = model.connection + + expect(::Gitlab::Database::LoadBalancing::Logger) + .not_to receive(:warn).with(hash_including(event: :transaction_leak)) + + expect(conn).not_to be_transaction_open execute(conn) expect(conn).not_to be_transaction_open - end - values = conn.execute("SELECT value FROM #{test_table_name}").to_a.map { |row| row['value'] } - expect(values).to contain_exactly(2) # Does not include 1 because the transaction was aborted and leaked + values = conn.execute("SELECT value FROM #{test_table_name}").to_a.map { |row| row['value'] } + expect(values).to contain_exactly(1, 2) # Includes both rows because there was no transaction to roll back + end end - it 'does not log a warning when no transaction is open to be leaked' do - conn = model.connection - - expect(::Gitlab::Database::LoadBalancing::Logger) - .not_to receive(:warn).with(hash_including(event: :transaction_leak)) + context 'with the PREVENT_LOAD_BALANCER_RETRIES_IN_TRANSACTION environment variable set' do + before do + stub_env('PREVENT_LOAD_BALANCER_RETRIES_IN_TRANSACTION' => '1') + end - expect(conn).not_to be_transaction_open + it 'raises an exception when a retry would occur during a transaction' do + expect(::Gitlab::Database::LoadBalancing::Logger) + .not_to receive(:warn).with(hash_including(event: :transaction_leak)) - execute(conn) + expect do + model.transaction do + execute(model.connection) + end + end.to raise_error(ActiveRecord::StatementInvalid) { |e| expect(e.cause).to be_a(PG::ConnectionBad) } + end - expect(conn).not_to be_transaction_open + it 'retries when not in a transaction' do + expect(::Gitlab::Database::LoadBalancing::Logger) + .not_to receive(:warn).with(hash_including(event: :transaction_leak)) - values = conn.execute("SELECT value FROM #{test_table_name}").to_a.map { |row| row['value'] } - expect(values).to contain_exactly(1, 2) # Includes both rows because there was no transaction to roll back + expect { execute(model.connection) }.not_to raise_error + end end end diff --git a/spec/lib/gitlab/database/migration_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers_spec.rb index d73b478ee7c..bcdd5646994 100644 --- a/spec/lib/gitlab/database/migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers_spec.rb @@ -757,6 +757,58 @@ RSpec.describe Gitlab::Database::MigrationHelpers do model.add_concurrent_foreign_key(:projects, :users, column: :user_id, reverse_lock_order: true) end end + + context 'when creating foreign key for a group of columns' do + it 'references the custom target columns when provided', :aggregate_failures do + expect(model).to receive(:with_lock_retries).and_yield + expect(model).to receive(:execute).with( + "ALTER TABLE projects\n" \ + "ADD CONSTRAINT fk_multiple_columns\n" \ + "FOREIGN KEY \(partition_number, user_id\)\n" \ + "REFERENCES users \(partition_number, id\)\n" \ + "ON DELETE CASCADE\n" \ + "NOT VALID;\n" + ) + + model.add_concurrent_foreign_key( + :projects, + :users, + column: [:partition_number, :user_id], + target_column: [:partition_number, :id], + validate: false, + name: :fk_multiple_columns + ) + end + + context 'when foreign key is already defined' do + before do + expect(model).to receive(:foreign_key_exists?).with( + :projects, + :users, + { + column: [:partition_number, :user_id], + name: :fk_multiple_columns, + on_delete: :cascade, + primary_key: [:partition_number, :id] + } + ).and_return(true) + end + + it 'does not create foreign key', :aggregate_failures do + expect(model).not_to receive(:with_lock_retries).and_yield + expect(model).not_to receive(:execute).with(/FOREIGN KEY/) + + model.add_concurrent_foreign_key( + :projects, + :users, + column: [:partition_number, :user_id], + target_column: [:partition_number, :id], + validate: false, + name: :fk_multiple_columns + ) + end + end + end end end @@ -813,6 +865,15 @@ RSpec.describe Gitlab::Database::MigrationHelpers do expect(name).to be_an_instance_of(String) expect(name.length).to eq(13) end + + context 'when using multiple columns' do + it 'returns the name of the foreign key', :aggregate_failures do + result = model.concurrent_foreign_key_name(:table_name, [:partition_number, :id]) + + expect(result).to be_an_instance_of(String) + expect(result.length).to eq(13) + end + end end describe '#foreign_key_exists?' do @@ -887,6 +948,62 @@ RSpec.describe Gitlab::Database::MigrationHelpers do it 'compares by target table if no column given' do expect(model.foreign_key_exists?(:projects, :other_table)).to be_falsey end + + context 'with foreign key using multiple columns' do + before do + key = ActiveRecord::ConnectionAdapters::ForeignKeyDefinition.new( + :projects, :users, + { + column: [:partition_number, :id], + name: :fk_projects_users_partition_number_id, + on_delete: :cascade, + primary_key: [:partition_number, :id] + } + ) + allow(model).to receive(:foreign_keys).with(:projects).and_return([key]) + end + + it 'finds existing foreign keys by columns' do + expect(model.foreign_key_exists?(:projects, :users, column: [:partition_number, :id])).to be_truthy + end + + it 'finds existing foreign keys by name' do + expect(model.foreign_key_exists?(:projects, :users, name: :fk_projects_users_partition_number_id)).to be_truthy + end + + it 'finds existing foreign_keys by name and column' do + expect(model.foreign_key_exists?(:projects, :users, name: :fk_projects_users_partition_number_id, column: [:partition_number, :id])).to be_truthy + end + + it 'finds existing foreign_keys by name, column and on_delete' do + expect(model.foreign_key_exists?(:projects, :users, name: :fk_projects_users_partition_number_id, column: [:partition_number, :id], on_delete: :cascade)).to be_truthy + end + + it 'finds existing foreign keys by target table only' do + expect(model.foreign_key_exists?(:projects, :users)).to be_truthy + end + + it 'compares by column name if given' do + expect(model.foreign_key_exists?(:projects, :users, column: :id)).to be_falsey + end + + it 'compares by target column name if given' do + expect(model.foreign_key_exists?(:projects, :users, primary_key: :user_id)).to be_falsey + expect(model.foreign_key_exists?(:projects, :users, primary_key: [:partition_number, :id])).to be_truthy + end + + it 'compares by foreign key name if given' do + expect(model.foreign_key_exists?(:projects, :users, name: :non_existent_foreign_key_name)).to be_falsey + end + + it 'compares by foreign key name and column if given' do + expect(model.foreign_key_exists?(:projects, :users, name: :non_existent_foreign_key_name, column: [:partition_number, :id])).to be_falsey + end + + it 'compares by foreign key name, column and on_delete if given' do + expect(model.foreign_key_exists?(:projects, :users, name: :fk_projects_users_partition_number_id, column: [:partition_number, :id], on_delete: :nullify)).to be_falsey + end + end end describe '#disable_statement_timeout' do @@ -3359,6 +3476,73 @@ RSpec.describe Gitlab::Database::MigrationHelpers do end end + describe '#drop_constraint' do + it "executes the statement to drop the constraint" do + expect(model).to receive(:execute).with("ALTER TABLE \"test_table\" DROP CONSTRAINT \"constraint_name\" CASCADE\n") + + model.drop_constraint(:test_table, :constraint_name, cascade: true) + end + + context 'when cascade option is false' do + it "executes the statement to drop the constraint without cascade" do + expect(model).to receive(:execute).with("ALTER TABLE \"test_table\" DROP CONSTRAINT \"constraint_name\" \n") + + model.drop_constraint(:test_table, :constraint_name, cascade: false) + end + end + end + + describe '#add_primary_key_using_index' do + it "executes the statement to add the primary key" do + expect(model).to receive(:execute).with /ALTER TABLE "test_table" ADD CONSTRAINT "old_name" PRIMARY KEY USING INDEX "new_name"/ + + model.add_primary_key_using_index(:test_table, :old_name, :new_name) + end + end + + context 'when changing the primary key of a given table' do + before do + model.create_table(:test_table, primary_key: :id) do |t| + t.integer :partition_number, default: 1 + end + + model.add_index(:test_table, :id, unique: true, name: :old_index_name) + model.add_index(:test_table, [:id, :partition_number], unique: true, name: :new_index_name) + end + + describe '#swap_primary_key' do + it 'executes statements to swap primary key', :aggregate_failures do + expect(model).to receive(:with_lock_retries).with(raise_on_exhaustion: true).ordered.and_yield + expect(model).to receive(:execute).with(/ALTER TABLE "test_table" DROP CONSTRAINT "test_table_pkey" CASCADE/).and_call_original + expect(model).to receive(:execute).with(/ALTER TABLE "test_table" ADD CONSTRAINT "test_table_pkey" PRIMARY KEY USING INDEX "new_index_name"/).and_call_original + + model.swap_primary_key(:test_table, :test_table_pkey, :new_index_name) + end + + context 'when new index does not exist' do + before do + model.remove_index(:test_table, column: [:id, :partition_number]) + end + + it 'raises ActiveRecord::StatementInvalid' do + expect do + model.swap_primary_key(:test_table, :test_table_pkey, :new_index_name) + end.to raise_error(ActiveRecord::StatementInvalid) + end + end + end + + describe '#unswap_primary_key' do + it 'executes statements to unswap primary key' do + expect(model).to receive(:with_lock_retries).with(raise_on_exhaustion: true).ordered.and_yield + expect(model).to receive(:execute).with(/ALTER TABLE "test_table" DROP CONSTRAINT "test_table_pkey" CASCADE/).ordered.and_call_original + expect(model).to receive(:execute).with(/ALTER TABLE "test_table" ADD CONSTRAINT "test_table_pkey" PRIMARY KEY USING INDEX "old_index_name"/).ordered.and_call_original + + model.unswap_primary_key(:test_table, :test_table_pkey, :old_index_name) + end + end + end + describe '#drop_sequence' do it "executes the statement to drop the sequence" do expect(model).to receive(:execute).with /ALTER TABLE "test_table" ALTER COLUMN "test_column" DROP DEFAULT;\nDROP SEQUENCE IF EXISTS "test_table_id_seq"/ diff --git a/spec/lib/gitlab/database/migrations/base_background_runner_spec.rb b/spec/lib/gitlab/database/migrations/base_background_runner_spec.rb index 34c83c42056..c2dc260b2ff 100644 --- a/spec/lib/gitlab/database/migrations/base_background_runner_spec.rb +++ b/spec/lib/gitlab/database/migrations/base_background_runner_spec.rb @@ -3,6 +3,8 @@ require 'spec_helper' RSpec.describe Gitlab::Database::Migrations::BaseBackgroundRunner, :freeze_time do + let(:connection) { ApplicationRecord.connection } + let(:result_dir) { Dir.mktmpdir } after do @@ -10,7 +12,7 @@ RSpec.describe Gitlab::Database::Migrations::BaseBackgroundRunner, :freeze_time end context 'subclassing' do - subject { described_class.new(result_dir: result_dir) } + subject { described_class.new(result_dir: result_dir, connection: connection) } it 'requires that jobs_by_migration_name be implemented' do expect { subject.jobs_by_migration_name }.to raise_error(NotImplementedError) diff --git a/spec/lib/gitlab/database/migrations/runner_spec.rb b/spec/lib/gitlab/database/migrations/runner_spec.rb index a37247ba0c6..f364ebfa522 100644 --- a/spec/lib/gitlab/database/migrations/runner_spec.rb +++ b/spec/lib/gitlab/database/migrations/runner_spec.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true require 'spec_helper' -RSpec.describe Gitlab::Database::Migrations::Runner do +RSpec.describe Gitlab::Database::Migrations::Runner, :reestablished_active_record_base do include Database::MultipleDatabases - let(:result_dir) { Pathname.new(Dir.mktmpdir) } + let(:base_result_dir) { Pathname.new(Dir.mktmpdir) } let(:migration_runs) { [] } # This list gets populated as the runner tries to run migrations @@ -26,11 +26,14 @@ RSpec.describe Gitlab::Database::Migrations::Runner do end before do - stub_const('Gitlab::Database::Migrations::Runner::BASE_RESULT_DIR', result_dir) + skip_if_multiple_databases_not_setup unless database == :main + + stub_const('Gitlab::Database::Migrations::Runner::BASE_RESULT_DIR', base_result_dir) allow(ActiveRecord::Migrator).to receive(:new) do |dir, _all_migrations, _schema_migration_class, version_to_migrate| migrator = double(ActiveRecord::Migrator) expect(migrator).to receive(:run) do - migration_runs << double('migrator', dir: dir, version_to_migrate: version_to_migrate) + config_for_migration_run = ActiveRecord::Base.connection_db_config + migration_runs << double('migrator', dir: dir, version_to_migrate: version_to_migrate, database: config_for_migration_run.name) end migrator end @@ -39,133 +42,153 @@ RSpec.describe Gitlab::Database::Migrations::Runner do migrations = applied_migrations_other_branches + applied_migrations_this_branch + pending_migrations ctx = double(ActiveRecord::MigrationContext, get_all_versions: all_versions, migrations: migrations, schema_migration: ActiveRecord::SchemaMigration) - allow(described_class).to receive(:migration_context).and_return(ctx) + allow(ActiveRecord::Base.connection).to receive(:migration_context).and_return(ctx) names_this_branch = (applied_migrations_this_branch + pending_migrations).map { |m| "db/migrate/#{m.version}_#{m.name}.rb" } allow(described_class).to receive(:migration_file_names_this_branch).and_return(names_this_branch) end after do - FileUtils.rm_rf(result_dir) + FileUtils.rm_rf(base_result_dir) end - it 'creates the results dir when one does not exist' do - FileUtils.rm_rf(result_dir) - - expect do - described_class.new(direction: :up, migrations: [], result_dir: result_dir).run - end.to change { Dir.exist?(result_dir) }.from(false).to(true) + where(:case_name, :database, :result_dir, :legacy_mode, :expected_schema_version) do + [ + ['main database', :main, lazy { base_result_dir.join('main') }, false, described_class::SCHEMA_VERSION], + ['main database (legacy mode)', :main, lazy { base_result_dir }, true, 3], + ['ci database', :ci, lazy { base_result_dir.join('ci') }, false, described_class::SCHEMA_VERSION] + ] end - describe '.up' do - context 'result directory' do - it 'uses the /up subdirectory' do - expect(described_class.up.result_dir).to eq(result_dir.join('up')) - end + with_them do + it 'creates the results dir when one does not exist' do + FileUtils.rm_rf(result_dir) + + expect do + described_class.new(direction: :up, migrations: [], database: database).run + end.to change { Dir.exist?(result_dir) }.from(false).to(true) end - context 'migrations to run' do - subject(:up) { described_class.up } + describe '.up' do + context 'result directory' do + it 'uses the /up subdirectory' do + expect(described_class.up(database: database, legacy_mode: legacy_mode).result_dir).to eq(result_dir.join('up')) + end + end + + context 'migrations to run' do + subject(:up) { described_class.up(database: database, legacy_mode: legacy_mode) } - it 'is the list of pending migrations' do - expect(up.migrations).to eq(pending_migrations) + it 'is the list of pending migrations' do + expect(up.migrations).to eq(pending_migrations) + end end - end - context 'running migrations' do - subject(:up) { described_class.up } + context 'running migrations' do + subject(:up) { described_class.up(database: database, legacy_mode: legacy_mode) } - it 'runs the unapplied migrations in version order', :aggregate_failures do - up.run + it 'runs the unapplied migrations in version order', :aggregate_failures do + up.run - expect(migration_runs.map(&:dir)).to match_array([:up, :up]) - expect(migration_runs.map(&:version_to_migrate)).to eq(pending_migrations.map(&:version)) - end + expect(migration_runs.map(&:dir)).to match_array([:up, :up]) + expect(migration_runs.map(&:version_to_migrate)).to eq(pending_migrations.map(&:version)) + end - it 'writes a metadata file with the current schema version' do - up.run + it 'writes a metadata file with the current schema version and database name' do + up.run - metadata_file = result_dir.join('up', described_class::METADATA_FILENAME) - expect(metadata_file.exist?).to be_truthy - metadata = Gitlab::Json.parse(File.read(metadata_file)) - expect(metadata).to match('version' => described_class::SCHEMA_VERSION) + metadata_file = result_dir.join('up', described_class::METADATA_FILENAME) + expect(metadata_file.exist?).to be_truthy + metadata = Gitlab::Json.parse(File.read(metadata_file)) + expect(metadata).to match('version' => expected_schema_version, 'database' => database.to_s) + end + + it 'runs the unapplied migrations on the correct database' do + up.run + + expect(migration_runs.map(&:database).uniq).to contain_exactly(database.to_s) + end end end - end - describe '.down' do - subject(:down) { described_class.down } + describe '.down' do + subject(:down) { described_class.down(database: database, legacy_mode: legacy_mode) } - context 'result directory' do - it 'is the /down subdirectory' do - expect(down.result_dir).to eq(result_dir.join('down')) + context 'result directory' do + it 'is the /down subdirectory' do + expect(down.result_dir).to eq(result_dir.join('down')) + end end - end - context 'migrations to run' do - it 'is the list of migrations that are up and on this branch' do - expect(down.migrations).to eq(applied_migrations_this_branch) + context 'migrations to run' do + it 'is the list of migrations that are up and on this branch' do + expect(down.migrations).to eq(applied_migrations_this_branch) + end end - end - context 'running migrations' do - it 'runs the applied migrations for the current branch in reverse order', :aggregate_failures do - down.run + context 'running migrations' do + it 'runs the applied migrations for the current branch in reverse order', :aggregate_failures do + down.run - expect(migration_runs.map(&:dir)).to match_array([:down, :down]) - expect(migration_runs.map(&:version_to_migrate)).to eq(applied_migrations_this_branch.reverse.map(&:version)) + expect(migration_runs.map(&:dir)).to match_array([:down, :down]) + expect(migration_runs.map(&:version_to_migrate)).to eq(applied_migrations_this_branch.reverse.map(&:version)) + end end - end - - it 'writes a metadata file with the current schema version' do - down.run - metadata_file = result_dir.join('down', described_class::METADATA_FILENAME) - expect(metadata_file.exist?).to be_truthy - metadata = Gitlab::Json.parse(File.read(metadata_file)) - expect(metadata).to match('version' => described_class::SCHEMA_VERSION) - end - end + it 'writes a metadata file with the current schema version' do + down.run - describe '.background_migrations' do - it 'is a TestBackgroundRunner' do - expect(described_class.background_migrations).to be_a(Gitlab::Database::Migrations::TestBackgroundRunner) + metadata_file = result_dir.join('down', described_class::METADATA_FILENAME) + expect(metadata_file.exist?).to be_truthy + metadata = Gitlab::Json.parse(File.read(metadata_file)) + expect(metadata).to match('version' => expected_schema_version, 'database' => database.to_s) + end end - it 'is configured with a result dir of /background_migrations' do - runner = described_class.background_migrations + describe '.background_migrations' do + it 'is a TestBackgroundRunner' do + expect(described_class.background_migrations).to be_a(Gitlab::Database::Migrations::TestBackgroundRunner) + end - expect(runner.result_dir).to eq(described_class::BASE_RESULT_DIR.join( 'background_migrations')) - end - end + it 'is configured with a result dir of /background_migrations' do + runner = described_class.background_migrations - describe '.batched_background_migrations' do - it 'is a TestBatchedBackgroundRunner' do - expect(described_class.batched_background_migrations(for_database: 'main')).to be_a(Gitlab::Database::Migrations::TestBatchedBackgroundRunner) + expect(runner.result_dir).to eq(described_class::BASE_RESULT_DIR.join( 'background_migrations')) + end end - context 'choosing the database to test against' do - it 'chooses the main database' do - runner = described_class.batched_background_migrations(for_database: 'main') + describe '.batched_background_migrations' do + it 'is a TestBatchedBackgroundRunner' do + expect(described_class.batched_background_migrations(for_database: database)).to be_a(Gitlab::Database::Migrations::TestBatchedBackgroundRunner) + end - chosen_connection_name = Gitlab::Database.db_config_name(runner.connection) + context 'choosing the database to test against' do + it 'chooses the provided database' do + runner = described_class.batched_background_migrations(for_database: database) - expect(chosen_connection_name).to eq('main') - end + chosen_connection_name = Gitlab::Database.db_config_name(runner.connection) - it 'chooses the ci database' do - skip_if_multiple_databases_not_setup + expect(chosen_connection_name).to eq(database.to_s) + end - runner = described_class.batched_background_migrations(for_database: 'ci') + it 'throws an error with an invalid name' do + expect { described_class.batched_background_migrations(for_database: 'not_a_database') } + .to raise_error(/not a valid database name/) + end - chosen_connection_name = Gitlab::Database.db_config_name(runner.connection) + it 'includes the database name in the result dir' do + runner = described_class.batched_background_migrations(for_database: database) - expect(chosen_connection_name).to eq('ci') + expect(runner.result_dir).to eq(base_result_dir.join(database.to_s, 'background_migrations')) + end end - it 'throws an error with an invalid name' do - expect { described_class.batched_background_migrations(for_database: 'not_a_database') } - .to raise_error(/not a valid database name/) + context 'legacy mode' do + it 'does not include the database name in the path' do + runner = described_class.batched_background_migrations(for_database: database, legacy_mode: true) + + expect(runner.result_dir).to eq(base_result_dir.join('background_migrations')) + end end end end diff --git a/spec/lib/gitlab/database/migrations/test_batched_background_runner_spec.rb b/spec/lib/gitlab/database/migrations/test_batched_background_runner_spec.rb index 3ac483c8ab7..07226f3d025 100644 --- a/spec/lib/gitlab/database/migrations/test_batched_background_runner_spec.rb +++ b/spec/lib/gitlab/database/migrations/test_batched_background_runner_spec.rb @@ -6,106 +6,156 @@ RSpec.describe Gitlab::Database::Migrations::TestBatchedBackgroundRunner, :freez include Gitlab::Database::MigrationHelpers include Database::MigrationTestingHelpers - let(:result_dir) { Dir.mktmpdir } - - after do - FileUtils.rm_rf(result_dir) + def queue_migration( + job_class_name, + batch_table_name, + batch_column_name, + *job_arguments, + job_interval:, + batch_size: Gitlab::Database::Migrations::BatchedBackgroundMigrationHelpers::BATCH_SIZE, + sub_batch_size: Gitlab::Database::Migrations::BatchedBackgroundMigrationHelpers::SUB_BATCH_SIZE + ) + + batch_max_value = define_batchable_model(batch_table_name, connection: connection).maximum(batch_column_name) + + Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::BackgroundMigration::BatchedMigration.create!( + job_class_name: job_class_name, + table_name: batch_table_name, + column_name: batch_column_name, + job_arguments: job_arguments, + interval: job_interval, + min_value: Gitlab::Database::Migrations::BatchedBackgroundMigrationHelpers::BATCH_MIN_VALUE, + max_value: batch_max_value, + batch_class_name: Gitlab::Database::Migrations::BatchedBackgroundMigrationHelpers::BATCH_CLASS_NAME, + batch_size: batch_size, + sub_batch_size: sub_batch_size, + status_event: :execute, + max_batch_size: nil, + gitlab_schema: gitlab_schema + ) + end end - let(:migration) do - ActiveRecord::Migration.new.extend(Gitlab::Database::Migrations::BatchedBackgroundMigrationHelpers) + where(:case_name, :base_model, :gitlab_schema) do + [ + ['main database', ApplicationRecord, :gitlab_main], + ['ci database', Ci::ApplicationRecord, :gitlab_ci] + ] end - let(:connection) { ApplicationRecord.connection } + with_them do + let(:result_dir) { Dir.mktmpdir } - let(:table_name) { "_test_column_copying" } + after do + FileUtils.rm_rf(result_dir) + end - before do - connection.execute(<<~SQL) - CREATE TABLE #{table_name} ( - id bigint primary key not null, - data bigint default 0 - ); + let(:connection) { base_model.connection } - insert into #{table_name} (id) select i from generate_series(1, 1000) g(i); - SQL + let(:table_name) { "_test_column_copying" } - allow(migration).to receive(:transaction_open?).and_return(false) - end + before do + connection.execute(<<~SQL) + CREATE TABLE #{table_name} ( + id bigint primary key not null, + data bigint default 0 + ); - context 'running a real background migration' do - it 'runs sampled jobs from the batched background migration' do - migration.queue_batched_background_migration('CopyColumnUsingBackgroundMigrationJob', - table_name, :id, - :id, :data, - batch_size: 100, - job_interval: 5.minutes) # job_interval is skipped when testing - - # Expect that running sampling for this migration processes some of the rows. Sampling doesn't run - # over every row in the table, so this does not completely migrate the table. - expect { described_class.new(result_dir: result_dir, connection: connection).run_jobs(for_duration: 1.minute) } - .to change { define_batchable_model(table_name).where('id IS DISTINCT FROM data').count } - .by_at_most(-1) + insert into #{table_name} (id) select i from generate_series(1, 1000) g(i); + SQL end - end - context 'with jobs to run' do - let(:migration_name) { 'TestBackgroundMigration' } + context 'running a real background migration' do + before do + queue_migration('CopyColumnUsingBackgroundMigrationJob', + table_name, :id, + :id, :data, + batch_size: 100, + job_interval: 5.minutes) # job_interval is skipped when testing + end - it 'samples jobs' do - calls = [] - define_background_migration(migration_name) do |*args| - calls << args + subject(:sample_migration) do + described_class.new(result_dir: result_dir, connection: connection).run_jobs(for_duration: 1.minute) end - migration.queue_batched_background_migration(migration_name, table_name, :id, - job_interval: 5.minutes, - batch_size: 100) + it 'runs sampled jobs from the batched background migration' do + # Expect that running sampling for this migration processes some of the rows. Sampling doesn't run + # over every row in the table, so this does not completely migrate the table. + expect { subject }.to change { + define_batchable_model(table_name, connection: connection) + .where('id IS DISTINCT FROM data').count + }.by_at_most(-1) + end - described_class.new(result_dir: result_dir, connection: connection).run_jobs(for_duration: 3.minutes) + it 'uses the correct connection to instrument the background migration' do + expect_next_instance_of(Gitlab::Database::Migrations::Instrumentation) do |instrumentation| + expect(instrumentation).to receive(:observe).with(hash_including(connection: connection)) + .at_least(:once).and_call_original + end - expect(calls).not_to be_empty + subject + end end - context 'with multiple jobs to run' do - it 'runs all jobs created within the last 3 hours' do - old_migration = define_background_migration(migration_name) - migration.queue_batched_background_migration(migration_name, table_name, :id, - job_interval: 5.minutes, - batch_size: 100) - - travel 4.hours - - new_migration = define_background_migration('NewMigration') { travel 1.second } - migration.queue_batched_background_migration('NewMigration', table_name, :id, - job_interval: 5.minutes, - batch_size: 10, - sub_batch_size: 5) - - other_new_migration = define_background_migration('NewMigration2') { travel 2.seconds } - migration.queue_batched_background_migration('NewMigration2', table_name, :id, - job_interval: 5.minutes, - batch_size: 10, - sub_batch_size: 5) - - expect_migration_runs(new_migration => 3, other_new_migration => 2, old_migration => 0) do - described_class.new(result_dir: result_dir, connection: connection).run_jobs(for_duration: 5.seconds) + context 'with jobs to run' do + let(:migration_name) { 'TestBackgroundMigration' } + + it 'samples jobs' do + calls = [] + define_background_migration(migration_name) do |*args| + calls << args + end + + queue_migration(migration_name, table_name, :id, + job_interval: 5.minutes, + batch_size: 100) + + described_class.new(result_dir: result_dir, connection: connection).run_jobs(for_duration: 3.minutes) + + expect(calls).not_to be_empty + end + + context 'with multiple jobs to run' do + it 'runs all jobs created within the last 3 hours' do + old_migration = define_background_migration(migration_name) + queue_migration(migration_name, table_name, :id, + job_interval: 5.minutes, + batch_size: 100) + + travel 4.hours + + new_migration = define_background_migration('NewMigration') { travel 1.second } + queue_migration('NewMigration', table_name, :id, + job_interval: 5.minutes, + batch_size: 10, + sub_batch_size: 5) + + other_new_migration = define_background_migration('NewMigration2') { travel 2.seconds } + queue_migration('NewMigration2', table_name, :id, + job_interval: 5.minutes, + batch_size: 10, + sub_batch_size: 5) + + expect_migration_runs(new_migration => 3, other_new_migration => 2, old_migration => 0) do + described_class.new(result_dir: result_dir, connection: connection).run_jobs(for_duration: 5.seconds) + end end end end - end - context 'choosing uniform batches to run' do - subject { described_class.new(result_dir: result_dir, connection: connection) } + context 'choosing uniform batches to run' do + subject { described_class.new(result_dir: result_dir, connection: connection) } - describe '#uniform_fractions' do - it 'generates evenly distributed sequences of fractions' do - received = subject.uniform_fractions.take(9) - expected = [0, 1, 1.0 / 2, 1.0 / 4, 3.0 / 4, 1.0 / 8, 3.0 / 8, 5.0 / 8, 7.0 / 8] + describe '#uniform_fractions' do + it 'generates evenly distributed sequences of fractions' do + received = subject.uniform_fractions.take(9) + expected = [0, 1, 1.0 / 2, 1.0 / 4, 3.0 / 4, 1.0 / 8, 3.0 / 8, 5.0 / 8, 7.0 / 8] - # All the fraction numerators are small integers, and all denominators are powers of 2, so these - # fit perfectly into floating point numbers with zero loss of precision - expect(received).to eq(expected) + # All the fraction numerators are small integers, and all denominators are powers of 2, so these + # fit perfectly into floating point numbers with zero loss of precision + expect(received).to eq(expected) + end end end end diff --git a/spec/lib/gitlab/database/obsolete_ignored_columns_spec.rb b/spec/lib/gitlab/database/obsolete_ignored_columns_spec.rb index 8a35d8149ad..b39b273bba9 100644 --- a/spec/lib/gitlab/database/obsolete_ignored_columns_spec.rb +++ b/spec/lib/gitlab/database/obsolete_ignored_columns_spec.rb @@ -53,15 +53,16 @@ RSpec.describe Gitlab::Database::ObsoleteIgnoredColumns do describe '#execute' do it 'returns a list of class names and columns pairs' do travel_to(REMOVE_DATE) do - expect(subject.execute).to eq([ - ['Testing::A', { - 'unused' => IgnorableColumns::ColumnIgnore.new(Date.parse('2019-01-01'), '12.0'), - 'also_unused' => IgnorableColumns::ColumnIgnore.new(Date.parse('2019-02-01'), '12.1') - }], - ['Testing::B', { - 'other' => IgnorableColumns::ColumnIgnore.new(Date.parse('2019-01-01'), '12.0') - }] - ]) + expect(subject.execute).to eq( + [ + ['Testing::A', { + 'unused' => IgnorableColumns::ColumnIgnore.new(Date.parse('2019-01-01'), '12.0'), + 'also_unused' => IgnorableColumns::ColumnIgnore.new(Date.parse('2019-02-01'), '12.1') + }], + ['Testing::B', { + 'other' => IgnorableColumns::ColumnIgnore.new(Date.parse('2019-01-01'), '12.0') + }] + ]) end end end diff --git a/spec/lib/gitlab/database/partitioning/convert_table_to_first_list_partition_spec.rb b/spec/lib/gitlab/database/partitioning/convert_table_to_first_list_partition_spec.rb index af7d751a404..0e804b4feac 100644 --- a/spec/lib/gitlab/database/partitioning/convert_table_to_first_list_partition_spec.rb +++ b/spec/lib/gitlab/database/partitioning/convert_table_to_first_list_partition_spec.rb @@ -153,6 +153,21 @@ RSpec.describe Gitlab::Database::Partitioning::ConvertTableToFirstListPartition expect(parent_model.pluck(:id)).to match_array([1, 2, 3]) end + context 'when the existing table is owned by a different user' do + before do + connection.execute(<<~SQL) + CREATE USER other_user SUPERUSER; + ALTER TABLE #{table_name} OWNER TO other_user; + SQL + end + + let(:current_user) { model.connection.select_value('select current_user') } + + it 'partitions without error' do + expect { partition }.not_to raise_error + end + end + context 'when an error occurs during the conversion' do def fail_first_time # We can't directly use a boolean here, as we need something that will be passed by-reference to the proc diff --git a/spec/lib/gitlab/database/partitioning/monthly_strategy_spec.rb b/spec/lib/gitlab/database/partitioning/monthly_strategy_spec.rb index 67d80d71e2a..50115a6f3dd 100644 --- a/spec/lib/gitlab/database/partitioning/monthly_strategy_spec.rb +++ b/spec/lib/gitlab/database/partitioning/monthly_strategy_spec.rb @@ -29,10 +29,11 @@ RSpec.describe Gitlab::Database::Partitioning::MonthlyStrategy do end it 'detects both partitions' do - expect(subject).to eq([ - Gitlab::Database::Partitioning::TimePartition.new(table_name, nil, '2020-05-01', partition_name: '_test_partitioned_test_000000'), - Gitlab::Database::Partitioning::TimePartition.new(table_name, '2020-05-01', '2020-06-01', partition_name: '_test_partitioned_test_202005') - ]) + expect(subject).to eq( + [ + Gitlab::Database::Partitioning::TimePartition.new(table_name, nil, '2020-05-01', partition_name: '_test_partitioned_test_000000'), + Gitlab::Database::Partitioning::TimePartition.new(table_name, '2020-05-01', '2020-06-01', partition_name: '_test_partitioned_test_202005') + ]) end end diff --git a/spec/lib/gitlab/database/partitioning/sliding_list_strategy_spec.rb b/spec/lib/gitlab/database/partitioning/sliding_list_strategy_spec.rb index 07c2c6606d8..550f254c4da 100644 --- a/spec/lib/gitlab/database/partitioning/sliding_list_strategy_spec.rb +++ b/spec/lib/gitlab/database/partitioning/sliding_list_strategy_spec.rb @@ -36,14 +36,15 @@ RSpec.describe Gitlab::Database::Partitioning::SlidingListStrategy do describe '#current_partitions' do it 'detects both partitions' do - expect(strategy.current_partitions).to eq([ - Gitlab::Database::Partitioning::SingleNumericListPartition.new( - table_name, 1, partition_name: '_test_partitioned_test_1' - ), - Gitlab::Database::Partitioning::SingleNumericListPartition.new( - table_name, 2, partition_name: '_test_partitioned_test_2' - ) - ]) + expect(strategy.current_partitions).to eq( + [ + Gitlab::Database::Partitioning::SingleNumericListPartition.new( + table_name, 1, partition_name: '_test_partitioned_test_1' + ), + Gitlab::Database::Partitioning::SingleNumericListPartition.new( + table_name, 2, partition_name: '_test_partitioned_test_2' + ) + ]) end end diff --git a/spec/lib/gitlab/database/partitioning/time_partition_spec.rb b/spec/lib/gitlab/database/partitioning/time_partition_spec.rb index 700202d81c5..5a17e8d20cf 100644 --- a/spec/lib/gitlab/database/partitioning/time_partition_spec.rb +++ b/spec/lib/gitlab/database/partitioning/time_partition_spec.rb @@ -156,12 +156,13 @@ RSpec.describe Gitlab::Database::Partitioning::TimePartition do described_class.new(table, '2020-03-01', '2020-04-01') ] - expect(partitions.sort).to eq([ - described_class.new(table, nil, '2020-02-01'), - described_class.new(table, '2020-02-01', '2020-03-01'), - described_class.new(table, '2020-03-01', '2020-04-01'), - described_class.new(table, '2020-04-01', '2020-05-01') - ]) + expect(partitions.sort).to eq( + [ + described_class.new(table, nil, '2020-02-01'), + described_class.new(table, '2020-02-01', '2020-03-01'), + described_class.new(table, '2020-03-01', '2020-04-01'), + described_class.new(table, '2020-04-01', '2020-05-01') + ]) end it 'returns nil for partitions of different tables' do diff --git a/spec/lib/gitlab/database/partitioning_spec.rb b/spec/lib/gitlab/database/partitioning_spec.rb index 94cdbfb2328..db5ca890155 100644 --- a/spec/lib/gitlab/database/partitioning_spec.rb +++ b/spec/lib/gitlab/database/partitioning_spec.rb @@ -130,12 +130,14 @@ RSpec.describe Gitlab::Database::Partitioning do context 'when no partitioned models are given' do it 'manages partitions for each registered model' do described_class.register_models([models.first]) - described_class.register_tables([ - { - table_name: table_names.last, - partitioned_column: :created_at, strategy: :monthly - } - ]) + described_class.register_tables( + [ + { + table_name: table_names.last, + partitioned_column: :created_at, + strategy: :monthly + } + ]) expect { described_class.sync_partitions } .to change { find_partitions(table_names.first).size }.from(0) diff --git a/spec/lib/gitlab/database/reflection_spec.rb b/spec/lib/gitlab/database/reflection_spec.rb index efc5bd1c1e1..389e93364c8 100644 --- a/spec/lib/gitlab/database/reflection_spec.rb +++ b/spec/lib/gitlab/database/reflection_spec.rb @@ -314,6 +314,12 @@ RSpec.describe Gitlab::Database::Reflection do expect(database.flavor).to eq('Azure Database for PostgreSQL - Single Server') end + it 'recognizes AlloyDB for PostgreSQL' do + stub_statements("SELECT name FROM pg_settings WHERE name LIKE 'alloydb%'") + + expect(database.flavor).to eq('AlloyDB for PostgreSQL') + end + it 'returns nil if can not recognize the flavor' do expect(database.flavor).to be_nil end diff --git a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces_spec.rb b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces_spec.rb index e222a29c6a1..ac2de43b7c6 100644 --- a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces_spec.rb +++ b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces_spec.rb @@ -98,7 +98,9 @@ RSpec.describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespa it 'moves a project for a namespace' do create(:project, :repository, :legacy_storage, namespace: namespace, path: 'hello-project') - expected_path = File.join(TestEnv.repos_path, 'bye-group', 'hello-project.git') + expected_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + File.join(TestEnv.repos_path, 'bye-group', 'hello-project.git') + end subject.move_repositories(namespace, 'hello-group', 'bye-group') @@ -109,7 +111,9 @@ RSpec.describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespa child_namespace = create(:group, name: 'sub-group', parent: namespace) create(:project, :repository, :legacy_storage, namespace: child_namespace, path: 'hello-project') - expected_path = File.join(TestEnv.repos_path, 'hello-group', 'renamed-sub-group', 'hello-project.git') + expected_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + File.join(TestEnv.repos_path, 'hello-group', 'renamed-sub-group', 'hello-project.git') + end subject.move_repositories(child_namespace, 'hello-group/sub-group', 'hello-group/renamed-sub-group') @@ -119,7 +123,9 @@ RSpec.describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespa it 'moves a parent namespace with subdirectories' do child_namespace = create(:group, name: 'sub-group', parent: namespace) create(:project, :repository, :legacy_storage, namespace: child_namespace, path: 'hello-project') - expected_path = File.join(TestEnv.repos_path, 'renamed-group', 'sub-group', 'hello-project.git') + expected_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + File.join(TestEnv.repos_path, 'renamed-group', 'sub-group', 'hello-project.git') + end subject.move_repositories(child_namespace, 'hello-group', 'renamed-group') @@ -170,7 +176,9 @@ RSpec.describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespa describe '#rename_namespace_dependencies' do it "moves the repository for a project in the namespace" do create(:project, :repository, :legacy_storage, namespace: namespace, path: "the-path-project") - expected_repo = File.join(TestEnv.repos_path, "the-path0", "the-path-project.git") + expected_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + File.join(TestEnv.repos_path, "the-path0", "the-path-project.git") + end subject.rename_namespace_dependencies(namespace, 'the-path', 'the-path0') @@ -268,7 +276,9 @@ RSpec.describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespa project.create_repository subject.rename_namespace(namespace) - expected_path = File.join(TestEnv.repos_path, 'the-path', 'a-project.git') + expected_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + File.join(TestEnv.repos_path, 'the-path', 'a-project.git') + end expect(subject).to receive(:rename_namespace_dependencies) .with( diff --git a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb index 50071e3e22b..6292f0246f7 100644 --- a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb +++ b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb @@ -126,7 +126,9 @@ RSpec.describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameProject let(:project) { create(:project, :repository, :legacy_storage, path: 'the-path', namespace: known_parent) } it 'moves the repository for a project' do - expected_path = File.join(TestEnv.repos_path, 'known-parent', 'new-repo.git') + expected_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + File.join(TestEnv.repos_path, 'known-parent', 'new-repo.git') + end subject.move_repository(project, 'known-parent/the-path', 'known-parent/new-repo') @@ -155,7 +157,9 @@ RSpec.describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameProject project.create_repository subject.rename_project(project) - expected_path = File.join(TestEnv.repos_path, 'known-parent', 'the-path.git') + expected_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + File.join(TestEnv.repos_path, 'known-parent', 'the-path.git') + end expect(subject).to receive(:move_project_folders) .with( diff --git a/spec/lib/gitlab/database/similarity_score_spec.rb b/spec/lib/gitlab/database/similarity_score_spec.rb index b7b66494390..cfee70ed208 100644 --- a/spec/lib/gitlab/database/similarity_score_spec.rb +++ b/spec/lib/gitlab/database/similarity_score_spec.rb @@ -78,10 +78,11 @@ RSpec.describe Gitlab::Database::SimilarityScore do describe 'score multiplier' do let(:order_expression) do - Gitlab::Database::SimilarityScore.build_expression(search: search, rules: [ - { column: Arel.sql('path'), multiplier: 1 }, - { column: Arel.sql('name'), multiplier: 0.8 } - ]).to_sql + Gitlab::Database::SimilarityScore.build_expression(search: search, rules: + [ + { column: Arel.sql('path'), multiplier: 1 }, + { column: Arel.sql('name'), multiplier: 0.8 } + ]).to_sql end let(:search) { 'different' } @@ -93,10 +94,11 @@ RSpec.describe Gitlab::Database::SimilarityScore do describe 'annotation' do it 'annotates the generated SQL expression' do - expression = Gitlab::Database::SimilarityScore.build_expression(search: 'test', rules: [ - { column: Arel.sql('path'), multiplier: 1 }, - { column: Arel.sql('name'), multiplier: 0.8 } - ]) + expression = Gitlab::Database::SimilarityScore.build_expression(search: 'test', rules: + [ + { column: Arel.sql('path'), multiplier: 1 }, + { column: Arel.sql('name'), multiplier: 0.8 } + ]) expect(Gitlab::Database::SimilarityScore).to be_order_by_similarity(expression) end diff --git a/spec/lib/gitlab/database_spec.rb b/spec/lib/gitlab/database_spec.rb index c893bca9e62..eb42734d044 100644 --- a/spec/lib/gitlab/database_spec.rb +++ b/spec/lib/gitlab/database_spec.rb @@ -32,21 +32,6 @@ RSpec.describe Gitlab::Database do end describe '.has_config?' do - context 'two tier database config' do - before do - allow(Gitlab::Application).to receive_message_chain(:config, :database_configuration, :[]).with(Rails.env) - .and_return({ "adapter" => "postgresql", "database" => "gitlabhq_test" }) - end - - it 'returns false for primary' do - expect(described_class.has_config?(:primary)).to eq(false) - end - - it 'returns false for ci' do - expect(described_class.has_config?(:ci)).to eq(false) - end - end - context 'three tier database config' do before do allow(Gitlab::Application).to receive_message_chain(:config, :database_configuration, :[]).with(Rails.env) diff --git a/spec/lib/gitlab/diff/char_diff_spec.rb b/spec/lib/gitlab/diff/char_diff_spec.rb index d38008c16f2..ca0ed6e840d 100644 --- a/spec/lib/gitlab/diff/char_diff_spec.rb +++ b/spec/lib/gitlab/diff/char_diff_spec.rb @@ -20,22 +20,24 @@ RSpec.describe Gitlab::Diff::CharDiff do it 'treats nil values as blank strings' do changes = subject.generate_diff - expect(changes).to eq([ - [:insert, "Hello \n World"] - ]) + expect(changes).to eq( + [ + [:insert, "Hello \n World"] + ]) end end it 'generates an array of changes' do changes = subject.generate_diff - expect(changes).to eq([ - [:equal, "Hel"], - [:insert, "l"], - [:equal, "o \n Worl"], - [:delete, "l"], - [:equal, "d"] - ]) + expect(changes).to eq( + [ + [:equal, "Hel"], + [:insert, "l"], + [:equal, "o \n Worl"], + [:delete, "l"], + [:equal, "d"] + ]) end end diff --git a/spec/lib/gitlab/diff/file_collection_sorter_spec.rb b/spec/lib/gitlab/diff/file_collection_sorter_spec.rb index ca9c156c1ad..3f0b0ad5775 100644 --- a/spec/lib/gitlab/diff/file_collection_sorter_spec.rb +++ b/spec/lib/gitlab/diff/file_collection_sorter_spec.rb @@ -33,27 +33,28 @@ RSpec.describe Gitlab::Diff::FileCollectionSorter do let(:sorted_files_paths) { subject.sort.map { |file| file.new_path.presence || file.old_path } } it 'returns list sorted directory first' do - expect(sorted_files_paths).to eq([ - '.dir/test', - '1-folder/nested/A-file.ext', - '1-folder/nested/M-file.ext', - '1-folder/nested/Z-file.ext', - '1-folder/A-file.ext', - '1-folder/M-file.ext', - '1-folder/README', - '1-folder/README', - '1-folder/Z-file.ext', - '2-folder/nested/A-file.ext', - '2-folder/A-file.ext', - '2-folder/M-file.ext', - '2-folder/Z-file.ext', - '.file', - 'A-file.ext', - 'M-file.ext', - 'README', - 'README', - 'Z-file.ext' - ]) + expect(sorted_files_paths).to eq( + [ + '.dir/test', + '1-folder/nested/A-file.ext', + '1-folder/nested/M-file.ext', + '1-folder/nested/Z-file.ext', + '1-folder/A-file.ext', + '1-folder/M-file.ext', + '1-folder/README', + '1-folder/README', + '1-folder/Z-file.ext', + '2-folder/nested/A-file.ext', + '2-folder/A-file.ext', + '2-folder/M-file.ext', + '2-folder/Z-file.ext', + '.file', + 'A-file.ext', + 'M-file.ext', + 'README', + 'README', + 'Z-file.ext' + ]) end end end diff --git a/spec/lib/gitlab/diff/file_spec.rb b/spec/lib/gitlab/diff/file_spec.rb index 28557aab830..d623a390dc8 100644 --- a/spec/lib/gitlab/diff/file_spec.rb +++ b/spec/lib/gitlab/diff/file_spec.rb @@ -309,12 +309,13 @@ RSpec.describe Gitlab::Diff::File do let(:diffs) { commit.diffs } before do - info_dir_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do - File.join(project.repository.path_to_repo, 'info') - end - - FileUtils.mkdir(info_dir_path) unless File.exist?(info_dir_path) - File.write(File.join(info_dir_path, 'attributes'), "*.md -diff\n") + project.repository.commit_files( + project.creator, + branch_name: 'master', + message: 'Add attributes', + actions: [{ action: :update, file_path: '.gitattributes', content: "*.md -diff\n" }] + ) + project.repository.copy_gitattributes('master') end it "returns true for files that do not have attributes" do diff --git a/spec/lib/gitlab/diff/highlight_cache_spec.rb b/spec/lib/gitlab/diff/highlight_cache_spec.rb index 53e74748234..33e9360ee01 100644 --- a/spec/lib/gitlab/diff/highlight_cache_spec.rb +++ b/spec/lib/gitlab/diff/highlight_cache_spec.rb @@ -109,58 +109,36 @@ RSpec.describe Gitlab::Diff::HighlightCache, :clean_gitlab_redis_cache do end shared_examples 'caches missing entries' do - where(:expiration_period, :renewable_expiration_ff, :short_renewable_expiration_ff) do - [ - [1.day, false, true], - [1.day, false, false], - [1.hour, true, true], - [8.hours, true, false] - ] - end - - with_them do - before do - stub_feature_flags( - highlight_diffs_renewable_expiration: renewable_expiration_ff, - highlight_diffs_short_renewable_expiration: short_renewable_expiration_ff - ) - end + it 'filters the key/value list of entries to be caches for each invocation' do + expect(cache).to receive(:write_to_redis_hash) + .with(hash_including(*paths)) + .once + .and_call_original - it 'filters the key/value list of entries to be caches for each invocation' do - expect(cache).to receive(:write_to_redis_hash) - .with(hash_including(*paths)) - .once - .and_call_original - - 2.times { cache.write_if_empty } - end + 2.times { cache.write_if_empty } + end - it 'reads from cache once' do - expect(cache).to receive(:read_cache).once.and_call_original + it 'reads from cache once' do + expect(cache).to receive(:read_cache).once.and_call_original - cache.write_if_empty - end + cache.write_if_empty + end - it 'refreshes TTL of the key on read' do - cache.write_if_empty + it 'refreshes TTL of the key on read' do + cache.write_if_empty - time_until_expire = 30.minutes + time_until_expire = 30.minutes - Gitlab::Redis::Cache.with do |redis| - # Emulate that a key is going to expire soon - redis.expire(cache.key, time_until_expire) + Gitlab::Redis::Cache.with do |redis| + # Emulate that a key is going to expire soon + redis.expire(cache.key, time_until_expire) - expect(redis.ttl(cache.key)).to be <= time_until_expire + expect(redis.ttl(cache.key)).to be <= time_until_expire - cache.send(:read_cache) + cache.send(:read_cache) - if renewable_expiration_ff - expect(redis.ttl(cache.key)).to be > time_until_expire - expect(redis.ttl(cache.key)).to be_within(1.minute).of(expiration_period) - else - expect(redis.ttl(cache.key)).to be <= time_until_expire - end - end + expect(redis.ttl(cache.key)).to be > time_until_expire + expect(redis.ttl(cache.key)).to be_within(1.minute).of(described_class::EXPIRATION) end end end diff --git a/spec/lib/gitlab/encoding_helper_spec.rb b/spec/lib/gitlab/encoding_helper_spec.rb index 690396d4dbc..c62e3071fc1 100644 --- a/spec/lib/gitlab/encoding_helper_spec.rb +++ b/spec/lib/gitlab/encoding_helper_spec.rb @@ -114,18 +114,6 @@ RSpec.describe Gitlab::EncodingHelper do expect(ext_class.encode_utf8_with_escaping!(input)).to eq(expected) end end - - context 'when feature flag is disabled' do - before do - stub_feature_flags(escape_gitaly_refs: false) - end - - it 'uses #encode! method' do - expect(ext_class).to receive(:encode!).with('String') - - ext_class.encode_utf8_with_escaping!('String') - end - end end describe '#encode_utf8' do diff --git a/spec/lib/gitlab/error_tracking/stack_trace_highlight_decorator_spec.rb b/spec/lib/gitlab/error_tracking/stack_trace_highlight_decorator_spec.rb index 3d23249d00d..73ebee49169 100644 --- a/spec/lib/gitlab/error_tracking/stack_trace_highlight_decorator_spec.rb +++ b/spec/lib/gitlab/error_tracking/stack_trace_highlight_decorator_spec.rb @@ -53,9 +53,9 @@ RSpec.describe Gitlab::ErrorTracking::StackTraceHighlightDecorator do 'lineNo' => 3, 'filename' => 'hello_world.php', 'context' => [ - [1, '<span id="LC1" class="line" lang="hack"><span class="c1">// PHP/Hack example</span></span>'], - [2, '<span id="LC1" class="line" lang="hack"><span class="cp"><?php</span></span>'], - [3, '<span id="LC1" class="line" lang="hack"><span class="k">echo</span> <span class="s1">\'Hello, World!\'</span><span class="p">;</span></span>'] + [1, '<span id="LC1" class="line" lang="hack"><span class="c1">// PHP/Hack example</span></span>'], + [2, '<span id="LC1" class="line" lang="hack"><span class="cp"><?php</span></span>'], + [3, '<span id="LC1" class="line" lang="hack"><span class="k">echo</span> <span class="s1">\'Hello, World!\'</span><span class="p">;</span></span>'] ] }, { diff --git a/spec/lib/gitlab/experimentation/controller_concern_spec.rb b/spec/lib/gitlab/experimentation/controller_concern_spec.rb deleted file mode 100644 index 799884d7a74..00000000000 --- a/spec/lib/gitlab/experimentation/controller_concern_spec.rb +++ /dev/null @@ -1,675 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do - include TrackingHelpers - - before do - stub_const('Gitlab::Experimentation::EXPERIMENTS', { - test_experiment: { - tracking_category: 'Team', - rollout_strategy: rollout_strategy - }, - my_experiment: { - tracking_category: 'Team' - } - } - ) - - allow(Gitlab).to receive(:com?).and_return(is_gitlab_com) - - Feature.enable_percentage_of_time(:test_experiment_experiment_percentage, enabled_percentage) - end - - let(:enabled_percentage) { 10 } - let(:rollout_strategy) { nil } - let(:is_gitlab_com) { true } - - controller(ApplicationController) do - include Gitlab::Experimentation::ControllerConcern - - def index - head :ok - end - end - - describe '#set_experimentation_subject_id_cookie' do - let(:do_not_track) { nil } - let(:cookie) { cookies.permanent.signed[:experimentation_subject_id] } - let(:cookie_value) { nil } - - before do - stub_do_not_track(do_not_track) if do_not_track.present? - request.cookies[:experimentation_subject_id] = cookie_value if cookie_value - - get :index - end - - context 'cookie is present' do - let(:cookie_value) { 'test' } - - it 'does not change the cookie' do - expect(cookies[:experimentation_subject_id]).to eq 'test' - end - end - - context 'cookie is not present' do - it 'sets a permanent signed cookie' do - expect(cookie).to be_present - end - - context 'DNT: 0' do - let(:do_not_track) { '0' } - - it 'sets a permanent signed cookie' do - expect(cookie).to be_present - end - end - - context 'DNT: 1' do - let(:do_not_track) { '1' } - - it 'does nothing' do - expect(cookie).not_to be_present - end - end - end - - context 'when not on gitlab.com' do - let(:is_gitlab_com) { false } - - context 'when cookie was set' do - let(:cookie_value) { 'test' } - - it 'cookie gets deleted' do - expect(cookie).not_to be_present - end - end - - context 'when no cookie was set before' do - it 'does nothing' do - expect(cookie).not_to be_present - end - end - end - end - - describe '#push_frontend_experiment' do - it 'pushes an experiment to the frontend' do - gon = class_double('Gon') - stub_experiment_for_subject(my_experiment: true) - allow(controller).to receive(:gon).and_return(gon) - - expect(gon).to receive(:push).with({ experiments: { 'myExperiment' => true } }, true) - - controller.push_frontend_experiment(:my_experiment) - end - end - - describe '#experiment_enabled?' do - def check_experiment(exp_key = :test_experiment, subject = nil) - controller.experiment_enabled?(exp_key, subject: subject) - end - - subject { check_experiment } - - context 'cookie is not present' do - it { is_expected.to eq(false) } - end - - context 'cookie is present' do - before do - cookies.permanent.signed[:experimentation_subject_id] = 'abcd-1234' - get :index - end - - it 'calls Gitlab::Experimentation.in_experiment_group? with the name of the experiment and the calculated experimentation_subject_index based on the uuid' do - expect(Gitlab::Experimentation).to receive(:in_experiment_group?).with(:test_experiment, subject: 'abcd-1234') - - check_experiment(:test_experiment) - end - - context 'when subject is given' do - let(:rollout_strategy) { :user } - let(:user) { build(:user) } - - it 'uses the subject' do - expect(Gitlab::Experimentation).to receive(:in_experiment_group?).with(:test_experiment, subject: user) - - check_experiment(:test_experiment, user) - end - end - end - - context 'do not track' do - before do - allow(Gitlab::Experimentation).to receive(:in_experiment_group?) { true } - end - - context 'when do not track is disabled' do - before do - controller.request.headers['DNT'] = '0' - end - - it { is_expected.to eq(true) } - end - - context 'when do not track is enabled' do - before do - controller.request.headers['DNT'] = '1' - end - - it { is_expected.to eq(false) } - end - end - - context 'URL parameter to force enable experiment' do - it 'returns true unconditionally' do - get :index, params: { force_experiment: :test_experiment } - - is_expected.to eq(true) - end - end - - context 'Cookie parameter to force enable experiment' do - it 'returns true unconditionally' do - cookies[:force_experiment] = 'test_experiment,another_experiment' - get :index - - expect(check_experiment(:test_experiment)).to eq(true) - expect(check_experiment(:another_experiment)).to eq(true) - end - end - end - - describe '#track_experiment_event', :snowplow do - let(:user) { build(:user) } - - context 'when the experiment is enabled' do - before do - stub_experiment(test_experiment: true) - allow(controller).to receive(:current_user).and_return(user) - end - - context 'the user is part of the experimental group' do - before do - stub_experiment_for_subject(test_experiment: true) - end - - it 'tracks the event with the right parameters' do - controller.track_experiment_event(:test_experiment, 'start', 1) - - expect_snowplow_event( - category: 'Team', - action: 'start', - property: 'experimental_group', - value: 1, - user: user - ) - end - end - - context 'the user is part of the control group' do - before do - stub_experiment_for_subject(test_experiment: false) - end - - it 'tracks the event with the right parameters' do - controller.track_experiment_event(:test_experiment, 'start', 1) - - expect_snowplow_event( - category: 'Team', - action: 'start', - property: 'control_group', - value: 1, - user: user - ) - end - end - - context 'do not track is disabled' do - before do - stub_do_not_track('0') - end - - it 'does track the event' do - controller.track_experiment_event(:test_experiment, 'start', 1) - - expect_snowplow_event( - category: 'Team', - action: 'start', - property: 'control_group', - value: 1, - user: user - ) - end - end - - context 'do not track enabled' do - before do - stub_do_not_track('1') - end - - it 'does not track the event' do - controller.track_experiment_event(:test_experiment, 'start', 1) - - expect_no_snowplow_event - end - end - - context 'subject is provided' do - before do - stub_experiment_for_subject(test_experiment: false) - end - - it "provides the subject's hashed global_id as label" do - experiment_subject = double(:subject, to_global_id: 'abc') - allow(Gitlab::Experimentation).to receive(:valid_subject_for_rollout_strategy?).and_return(true) - - controller.track_experiment_event(:test_experiment, 'start', 1, subject: experiment_subject) - - expect_snowplow_event( - category: 'Team', - action: 'start', - property: 'control_group', - value: 1, - label: Digest::SHA256.hexdigest('abc'), - user: user - ) - end - - it "provides the subject's hashed string representation as label" do - experiment_subject = 'somestring' - - controller.track_experiment_event(:test_experiment, 'start', 1, subject: experiment_subject) - - expect_snowplow_event( - category: 'Team', - action: 'start', - property: 'control_group', - value: 1, - label: Digest::SHA256.hexdigest('somestring'), - user: user - ) - end - end - - context 'no subject is provided but cookie is set' do - before do - get :index - stub_experiment_for_subject(test_experiment: false) - end - - it 'uses the experimentation_subject_id as fallback' do - controller.track_experiment_event(:test_experiment, 'start', 1) - - expect_snowplow_event( - category: 'Team', - action: 'start', - property: 'control_group', - value: 1, - label: cookies.permanent.signed[:experimentation_subject_id], - user: user - ) - end - end - end - - context 'when the experiment is disabled' do - before do - stub_experiment(test_experiment: false) - end - - it 'does not track the event' do - controller.track_experiment_event(:test_experiment, 'start') - - expect_no_snowplow_event - end - end - end - - describe '#frontend_experimentation_tracking_data' do - context 'when the experiment is enabled' do - before do - stub_experiment(test_experiment: true) - end - - context 'the user is part of the experimental group' do - before do - stub_experiment_for_subject(test_experiment: true) - end - - it 'pushes the right parameters to gon' do - controller.frontend_experimentation_tracking_data(:test_experiment, 'start', 'team_id') - expect(Gon.tracking_data).to eq( - { - category: 'Team', - action: 'start', - property: 'experimental_group', - value: 'team_id' - } - ) - end - end - - context 'the user is part of the control group' do - before do - stub_experiment_for_subject(test_experiment: false) - end - - it 'pushes the right parameters to gon' do - controller.frontend_experimentation_tracking_data(:test_experiment, 'start', 'team_id') - expect(Gon.tracking_data).to eq( - { - category: 'Team', - action: 'start', - property: 'control_group', - value: 'team_id' - } - ) - end - - it 'does not send nil value to gon' do - controller.frontend_experimentation_tracking_data(:test_experiment, 'start') - expect(Gon.tracking_data).to eq( - { - category: 'Team', - action: 'start', - property: 'control_group' - } - ) - end - end - - context 'do not track disabled' do - before do - stub_do_not_track('0') - end - - it 'pushes the right parameters to gon' do - controller.frontend_experimentation_tracking_data(:test_experiment, 'start') - - expect(Gon.tracking_data).to eq( - { - category: 'Team', - action: 'start', - property: 'control_group' - } - ) - end - end - - context 'do not track enabled' do - before do - stub_do_not_track('1') - end - - it 'does not push data to gon' do - controller.frontend_experimentation_tracking_data(:test_experiment, 'start') - - expect(Gon.method_defined?(:tracking_data)).to eq(false) - end - end - end - - context 'when the experiment is disabled' do - before do - stub_experiment(test_experiment: false) - end - - it 'does not push data to gon' do - expect(Gon.method_defined?(:tracking_data)).to eq(false) - controller.track_experiment_event(:test_experiment, 'start') - end - end - end - - describe '#record_experiment_user' do - let(:user) { build(:user) } - let(:context) { { a: 42 } } - - context 'when the experiment is enabled' do - before do - stub_experiment(test_experiment: true) - allow(controller).to receive(:current_user).and_return(user) - end - - context 'the user is part of the experimental group' do - before do - stub_experiment_for_subject(test_experiment: true) - end - - it 'calls add_user on the Experiment model' do - expect(::Experiment).to receive(:add_user).with(:test_experiment, :experimental, user, context) - - controller.record_experiment_user(:test_experiment, context) - end - - context 'with a cookie based rollout strategy' do - it 'calls tracking_group with a nil subject' do - expect(controller).to receive(:tracking_group).with(:test_experiment, nil, subject: nil).and_return(:experimental) - allow(::Experiment).to receive(:add_user).with(:test_experiment, :experimental, user, context) - - controller.record_experiment_user(:test_experiment, context) - end - end - - context 'with a user based rollout strategy' do - let(:rollout_strategy) { :user } - - it 'calls tracking_group with a user subject' do - expect(controller).to receive(:tracking_group).with(:test_experiment, nil, subject: user).and_return(:experimental) - allow(::Experiment).to receive(:add_user).with(:test_experiment, :experimental, user, context) - - controller.record_experiment_user(:test_experiment, context) - end - end - end - - context 'the user is part of the control group' do - before do - stub_experiment_for_subject(test_experiment: false) - end - - it 'calls add_user on the Experiment model' do - expect(::Experiment).to receive(:add_user).with(:test_experiment, :control, user, context) - - controller.record_experiment_user(:test_experiment, context) - end - end - end - - context 'when the experiment is disabled' do - before do - stub_experiment(test_experiment: false) - allow(controller).to receive(:current_user).and_return(user) - end - - it 'does not call add_user on the Experiment model' do - expect(::Experiment).not_to receive(:add_user) - - controller.record_experiment_user(:test_experiment, context) - end - end - - context 'when there is no current_user' do - before do - stub_experiment(test_experiment: true) - end - - it 'does not call add_user on the Experiment model' do - expect(::Experiment).not_to receive(:add_user) - - controller.record_experiment_user(:test_experiment, context) - end - end - - context 'do not track' do - before do - stub_experiment(test_experiment: true) - allow(controller).to receive(:current_user).and_return(user) - end - - context 'is disabled' do - before do - stub_do_not_track('0') - stub_experiment_for_subject(test_experiment: false) - end - - it 'calls add_user on the Experiment model' do - expect(::Experiment).to receive(:add_user).with(:test_experiment, :control, user, context) - - controller.record_experiment_user(:test_experiment, context) - end - end - - context 'is enabled' do - before do - stub_do_not_track('1') - end - - it 'does not call add_user on the Experiment model' do - expect(::Experiment).not_to receive(:add_user) - - controller.record_experiment_user(:test_experiment, context) - end - end - end - end - - describe '#record_experiment_group' do - let(:group) { 'a group object' } - let(:experiment_key) { :some_experiment_key } - let(:dnt_enabled) { false } - let(:experiment_active) { true } - let(:rollout_strategy) { :whatever } - let(:variant) { 'variant' } - - before do - allow(controller).to receive(:dnt_enabled?).and_return(dnt_enabled) - allow(::Gitlab::Experimentation).to receive(:active?).and_return(experiment_active) - allow(::Gitlab::Experimentation).to receive(:rollout_strategy).and_return(rollout_strategy) - allow(controller).to receive(:tracking_group).and_return(variant) - allow(::Experiment).to receive(:add_group) - end - - subject(:record_experiment_group) { controller.record_experiment_group(experiment_key, group) } - - shared_examples 'exits early without recording' do - it 'returns early without recording the group as an ExperimentSubject' do - expect(::Experiment).not_to receive(:add_group) - record_experiment_group - end - end - - shared_examples 'calls tracking_group' do |using_cookie_rollout| - it "calls tracking_group with #{using_cookie_rollout ? 'a nil' : 'the group as the'} subject" do - expect(controller).to receive(:tracking_group).with(experiment_key, nil, subject: using_cookie_rollout ? nil : group).and_return(variant) - record_experiment_group - end - end - - shared_examples 'records the group' do - it 'records the group' do - expect(::Experiment).to receive(:add_group).with(experiment_key, group: group, variant: variant) - record_experiment_group - end - end - - context 'when DNT is enabled' do - let(:dnt_enabled) { true } - - include_examples 'exits early without recording' - end - - context 'when the experiment is not active' do - let(:experiment_active) { false } - - include_examples 'exits early without recording' - end - - context 'when a nil group is given' do - let(:group) { nil } - - include_examples 'exits early without recording' - end - - context 'when the experiment uses a cookie-based rollout strategy' do - let(:rollout_strategy) { :cookie } - - include_examples 'calls tracking_group', true - include_examples 'records the group' - end - - context 'when the experiment uses a non-cookie-based rollout strategy' do - let(:rollout_strategy) { :group } - - include_examples 'calls tracking_group', false - include_examples 'records the group' - end - end - - describe '#record_experiment_conversion_event' do - let(:user) { build(:user) } - - before do - allow(controller).to receive(:dnt_enabled?).and_return(false) - allow(controller).to receive(:current_user).and_return(user) - stub_experiment(test_experiment: true) - end - - subject(:record_conversion_event) do - controller.record_experiment_conversion_event(:test_experiment) - end - - it 'records the conversion event for the experiment & user' do - expect(::Experiment).to receive(:record_conversion_event).with(:test_experiment, user, {}) - record_conversion_event - end - - shared_examples 'does not record the conversion event' do - it 'does not record the conversion event' do - expect(::Experiment).not_to receive(:record_conversion_event) - record_conversion_event - end - end - - context 'when DNT is enabled' do - before do - allow(controller).to receive(:dnt_enabled?).and_return(true) - end - - include_examples 'does not record the conversion event' - end - - context 'when there is no current user' do - before do - allow(controller).to receive(:current_user).and_return(nil) - end - - include_examples 'does not record the conversion event' - end - - context 'when the experiment is not enabled' do - before do - stub_experiment(test_experiment: false) - end - - include_examples 'does not record the conversion event' - end - end - - describe '#experiment_tracking_category_and_group' do - let_it_be(:experiment_key) { :test_something } - - subject { controller.experiment_tracking_category_and_group(experiment_key) } - - it 'returns a string with the experiment tracking category & group joined with a ":"' do - expect(controller).to receive(:tracking_category).with(experiment_key).and_return('Experiment::Category') - expect(controller).to receive(:tracking_group).with(experiment_key, '_group', subject: nil).and_return('experimental_group') - - expect(subject).to eq('Experiment::Category:experimental_group') - end - end -end diff --git a/spec/lib/gitlab/experimentation/experiment_spec.rb b/spec/lib/gitlab/experimentation/experiment_spec.rb deleted file mode 100644 index a5cc69b9538..00000000000 --- a/spec/lib/gitlab/experimentation/experiment_spec.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Experimentation::Experiment do - using RSpec::Parameterized::TableSyntax - - let(:percentage) { 50 } - let(:params) do - { - tracking_category: 'Category1', - rollout_strategy: nil - } - end - - before do - skip_feature_flags_yaml_validation - skip_default_enabled_yaml_check - allow(Feature).to receive(:log_feature_flag_states?).and_return(false) - feature = double('FeatureFlag', percentage_of_time_value: percentage, enabled?: true) - allow(Feature).to receive(:get).with(:experiment_key_experiment_percentage).and_return(feature) - end - - subject(:experiment) { described_class.new(:experiment_key, **params) } - - describe '#active?' do - before do - allow(Gitlab).to receive(:com?).and_return(on_gitlab_com) - end - - subject { experiment.active? } - - where(:on_gitlab_com, :percentage, :is_active) do - true | 0 | false - true | 10 | true - false | 0 | false - false | 10 | false - end - - with_them do - it { is_expected.to eq(is_active) } - end - end - - describe '#enabled_for_index?' do - subject { experiment.enabled_for_index?(index) } - - where(:index, :percentage, :is_enabled) do - 50 | 40 | false - 40 | 50 | true - nil | 50 | false - end - - with_them do - it { is_expected.to eq(is_enabled) } - end - end -end diff --git a/spec/lib/gitlab/experimentation_spec.rb b/spec/lib/gitlab/experimentation_spec.rb deleted file mode 100644 index c482874b725..00000000000 --- a/spec/lib/gitlab/experimentation_spec.rb +++ /dev/null @@ -1,161 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Experimentation do - using RSpec::Parameterized::TableSyntax - - before do - stub_const('Gitlab::Experimentation::EXPERIMENTS', { - test_experiment: { - tracking_category: 'Team' - }, - tabular_experiment: { - tracking_category: 'Team', - rollout_strategy: rollout_strategy - } - }) - - skip_feature_flags_yaml_validation - skip_default_enabled_yaml_check - Feature.enable_percentage_of_time(:test_experiment_experiment_percentage, enabled_percentage) - allow(Gitlab).to receive(:com?).and_return(true) - end - - let(:enabled_percentage) { 10 } - let(:rollout_strategy) { nil } - - describe '.get_experiment' do - subject { described_class.get_experiment(:test_experiment) } - - context 'returns experiment' do - it { is_expected.to be_instance_of(Gitlab::Experimentation::Experiment) } - end - - context 'experiment is not defined' do - subject { described_class.get_experiment(:missing_experiment) } - - it { is_expected.to be_nil } - end - end - - describe '.active?' do - subject { described_class.active?(:test_experiment) } - - context 'feature toggle is enabled' do - it { is_expected.to eq(true) } - end - - describe 'experiment is not defined' do - it 'returns false' do - expect(described_class.active?(:missing_experiment)).to eq(false) - end - end - - describe 'experiment is disabled' do - let(:enabled_percentage) { 0 } - - it { is_expected.to eq(false) } - end - end - - describe '.in_experiment_group?' do - let(:enabled_percentage) { 50 } - let(:experiment_subject) { 'z' } # Zlib.crc32('test_experimentz') % 100 = 33 - - subject { described_class.in_experiment_group?(:test_experiment, subject: experiment_subject) } - - context 'when experiment is active' do - context 'when subject is part of the experiment' do - it { is_expected.to eq(true) } - end - - context 'when subject is not part of the experiment' do - let(:experiment_subject) { 'a' } # Zlib.crc32('test_experimenta') % 100 = 61 - - it { is_expected.to eq(false) } - end - - context 'when subject has a global_id' do - let(:experiment_subject) { double(:subject, to_global_id: 'z') } - - it { is_expected.to eq(true) } - end - - context 'when subject is nil' do - let(:experiment_subject) { nil } - - it { is_expected.to eq(false) } - end - - context 'when subject is an empty string' do - let(:experiment_subject) { '' } - - it { is_expected.to eq(false) } - end - end - - context 'when experiment is not active' do - before do - allow(described_class).to receive(:active?).and_return(false) - end - - it { is_expected.to eq(false) } - end - end - - describe '.log_invalid_rollout' do - subject { described_class.log_invalid_rollout(:test_experiment, 1) } - - before do - allow(described_class).to receive(:valid_subject_for_rollout_strategy?).and_return(valid) - end - - context 'subject is not valid for experiment' do - let(:valid) { false } - - it 'logs a warning message' do - expect_next_instance_of(Gitlab::ExperimentationLogger) do |logger| - expect(logger) - .to receive(:warn) - .with( - message: 'Subject must conform to the rollout strategy', - experiment_key: :test_experiment, - subject: 'Integer', - rollout_strategy: :cookie - ) - end - - subject - end - end - - context 'subject is valid for experiment' do - let(:valid) { true } - - it 'does not log a warning message' do - expect(Gitlab::ExperimentationLogger).not_to receive(:build) - - subject - end - end - end - - describe '.valid_subject_for_rollout_strategy?' do - subject { described_class.valid_subject_for_rollout_strategy?(:tabular_experiment, experiment_subject) } - - where(:rollout_strategy, :experiment_subject, :result) do - :cookie | nil | true - nil | nil | true - :cookie | 'string' | true - nil | User.new | false - :user | User.new | true - :group | User.new | false - :group | Group.new | true - end - - with_them do - it { is_expected.to be(result) } - end - end -end diff --git a/spec/lib/gitlab/git/keep_around_spec.rb b/spec/lib/gitlab/git/keep_around_spec.rb index 44c3caf3f8d..d6359d55646 100644 --- a/spec/lib/gitlab/git/keep_around_spec.rb +++ b/spec/lib/gitlab/git/keep_around_spec.rb @@ -18,23 +18,14 @@ RSpec.describe Gitlab::Git::KeepAround do expect(service.kept_around?(sample_commit.id)).to be_truthy end - it "attempting to call keep around on truncated ref does not fail" do - service.execute([sample_commit.id]) - ref = service.send(:keep_around_ref_name, sample_commit.id) - - path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do - File.join(repository.path, ref) - end - # Corrupt the reference - File.truncate(path, 0) + it "does not fail if writting the ref fails" do + expect(repository.raw).to receive(:write_ref).and_raise(Gitlab::Git::CommandError) expect(service.kept_around?(sample_commit.id)).to be_falsey service.execute([sample_commit.id]) expect(service.kept_around?(sample_commit.id)).to be_falsey - - File.delete(path) end context 'for multiple SHAs' do diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index 9a87911b6e8..f3d3fd2034c 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -461,7 +461,11 @@ RSpec.describe Gitlab::Git::Repository do end it 'raises an error if it failed' do - expect { repository.delete_refs('refs\heads\fix') }.to raise_error(Gitlab::Git::Repository::GitError) + # TODO: Once https://gitlab.com/gitlab-org/gitaly/-/merge_requests/4921 + # is merged, remove the assertion for Gitlab::Git::Repository::GitError + expect { repository.delete_refs('refs\heads\fix') }.to raise_error do |e| + expect(e).to be_a(Gitlab::Git::Repository::GitError).or be_a(Gitlab::Git::InvalidRefFormatError) + end end end @@ -483,6 +487,12 @@ RSpec.describe Gitlab::Git::Repository do it 'displays that branch' do expect(repository.branch_names_contains_sha(head_id)).to include('master', new_branch, utf8_branch) end + + context 'when limit is provided' do + it 'displays limited number of branches' do + expect(repository.branch_names_contains_sha(head_id, limit: 1)).to match_array(['2-mb-file']) + end + end end describe "#refs_hash" do @@ -668,11 +678,11 @@ RSpec.describe Gitlab::Git::Repository do expect_any_instance_of(Gitlab::GitalyClient::RemoteService) .to receive(:find_remote_root_ref).and_call_original - expect(repository.find_remote_root_ref(SeedHelper::GITLAB_GIT_TEST_REPO_URL)).to eq 'master' + expect(repository.find_remote_root_ref(TestEnv.factory_repo_path.to_s)).to eq 'master' end it 'returns UTF-8' do - expect(repository.find_remote_root_ref(SeedHelper::GITLAB_GIT_TEST_REPO_URL)).to be_utf8 + expect(repository.find_remote_root_ref(TestEnv.factory_repo_path.to_s)).to be_utf8 end it 'returns nil when remote name is nil' do @@ -690,7 +700,7 @@ RSpec.describe Gitlab::Git::Repository do end it_behaves_like 'wrapping gRPC errors', Gitlab::GitalyClient::RemoteService, :find_remote_root_ref do - subject { repository.find_remote_root_ref(SeedHelper::GITLAB_GIT_TEST_REPO_URL) } + subject { repository.find_remote_root_ref(TestEnv.factory_repo_path.to_s) } end end @@ -1769,12 +1779,13 @@ RSpec.describe Gitlab::Git::Repository do it 'returns exactly the expected results' do languages = repository.languages(TestEnv::BRANCH_SHA['master']) - expect(languages).to match_array([ - { value: a_value_within(0.1).of(66.7), label: "Ruby", color: "#701516", highlight: "#701516" }, - { value: a_value_within(0.1).of(22.96), label: "JavaScript", color: "#f1e05a", highlight: "#f1e05a" }, - { value: a_value_within(0.1).of(7.9), label: "HTML", color: "#e34c26", highlight: "#e34c26" }, - { value: a_value_within(0.1).of(2.51), label: "CoffeeScript", color: "#244776", highlight: "#244776" } - ]) + expect(languages).to match_array( + [ + { value: a_value_within(0.1).of(66.7), label: "Ruby", color: "#701516", highlight: "#701516" }, + { value: a_value_within(0.1).of(22.96), label: "JavaScript", color: "#f1e05a", highlight: "#f1e05a" }, + { value: a_value_within(0.1).of(7.9), label: "HTML", color: "#e34c26", highlight: "#e34c26" }, + { value: a_value_within(0.1).of(2.51), label: "CoffeeScript", color: "#244776", highlight: "#244776" } + ]) end it "uses the repository's HEAD when no ref is passed" do @@ -1784,22 +1795,48 @@ RSpec.describe Gitlab::Git::Repository do end end - describe '#license_short_name' do - subject { repository.license_short_name } + describe '#license' do + where(from_gitaly: [true, false]) + with_them do + subject(:license) { repository.license(from_gitaly) } - context 'when no license file can be found' do - let(:project) { create(:project, :repository) } - let(:repository) { project.repository.raw_repository } + context 'when no license file can be found' do + let_it_be(:project) { create(:project, :repository) } + let(:repository) { project.repository.raw_repository } - before do - project.repository.delete_file(project.owner, 'LICENSE', message: 'remove license', branch_name: 'master') + before do + project.repository.delete_file(project.owner, 'LICENSE', message: 'remove license', branch_name: 'master') + end + + it { is_expected.to be_nil } + end + + context 'when an mit license is found' do + it { is_expected.to have_attributes(key: 'mit') } end - it { is_expected.to be_nil } + context 'when license is not recognized ' do + let_it_be(:project) { create(:project, :repository) } + let(:repository) { project.repository.raw_repository } + + before do + project.repository.update_file( + project.owner, + 'LICENSE', + 'This software is licensed under the Dummy license.', + message: 'Update license', + branch_name: 'master') + end + + it { is_expected.to have_attributes(key: 'other', nickname: 'LICENSE') } + end end - context 'when an mit license is found' do - it { is_expected.to eq('mit') } + it 'does not crash when license is invalid' do + expect(Licensee::License).to receive(:new) + .and_raise(Licensee::InvalidLicense) + + expect(repository.license(false)).to be_nil end end diff --git a/spec/lib/gitlab/git/rugged_impl/use_rugged_spec.rb b/spec/lib/gitlab/git/rugged_impl/use_rugged_spec.rb index 747611a59e6..524b373a5b7 100644 --- a/spec/lib/gitlab/git/rugged_impl/use_rugged_spec.rb +++ b/spec/lib/gitlab/git/rugged_impl/use_rugged_spec.rb @@ -213,7 +213,8 @@ RSpec.describe Gitlab::Git::RuggedImpl::UseRugged do end def create_gitaly_metadata_file - File.open(File.join(SEED_STORAGE_PATH, '.gitaly-metadata'), 'w+') do |f| + metadata_filename = File.join(TestEnv.repos_path, '.gitaly-metadata') + File.open(metadata_filename, 'w+') do |f| gitaly_metadata = { "gitaly_filesystem_id" => SecureRandom.uuid } diff --git a/spec/lib/gitlab/git/wiki_spec.rb b/spec/lib/gitlab/git/wiki_spec.rb deleted file mode 100644 index 05c7ac149e4..00000000000 --- a/spec/lib/gitlab/git/wiki_spec.rb +++ /dev/null @@ -1,134 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Git::Wiki do - using RSpec::Parameterized::TableSyntax - - let(:project) { create(:project) } - let(:user) { project.first_owner } - let(:project_wiki) { ProjectWiki.new(project, user) } - let(:repository) { project_wiki.repository } - let(:default_branch) { described_class.default_ref(project) } - - subject(:wiki) { project_wiki.wiki } - - before do - repository.create_if_not_exists(project_wiki.default_branch) - end - - describe '#pages' do - before do - create_page('page1', 'content') - create_page('page2', 'content2') - end - - after do - destroy_page('page1') - destroy_page('page2') - end - - it 'returns all the pages' do - expect(subject.list_pages.count).to eq(2) - expect(subject.list_pages.first.title).to eq 'page1' - expect(subject.list_pages.last.title).to eq 'page2' - end - - it 'returns only one page' do - pages = subject.list_pages(limit: 1) - - expect(pages.count).to eq(1) - expect(pages.first.title).to eq 'page1' - end - end - - describe '#page' do - before do - create_page('page1', 'content') - create_page('foo/page1', 'content foo/page1') - end - - after do - destroy_page('page1') - destroy_page('foo/page1') - end - - it 'returns the right page' do - page = subject.page(title: 'page1', dir: '') - expect(page.url_path).to eq 'page1' - expect(page.raw_data).to eq 'content' - - page = subject.page(title: 'page1', dir: 'foo') - expect(page.url_path).to eq 'foo/page1' - expect(page.raw_data).to eq 'content foo/page1' - end - - it 'returns nil for invalid arguments' do - expect(subject.page(title: '')).to be_nil - expect(subject.page(title: 'foo', version: ':')).to be_nil - end - - it 'does not return content if load_content param is set to false' do - page = subject.page(title: 'page1', dir: '', load_content: false) - - expect(page.url_path).to eq 'page1' - expect(page.raw_data).to be_empty - end - end - - describe '#preview_slug' do - where(:title, :file_extension, :format, :expected_slug) do - 'The Best Thing' | :md | :markdown | 'The-Best-Thing' - 'The Best Thing' | :md | :md | 'The-Best-Thing' - 'The Best Thing' | :txt | :txt | 'The-Best-Thing' - 'A Subject/Title Here' | :txt | :txt | 'A-Subject/Title-Here' - 'A subject' | :txt | :txt | 'A-subject' - 'A 1/B 2/C 3' | :txt | :txt | 'A-1/B-2/C-3' - 'subject/title' | :txt | :txt | 'subject/title' - 'subject/title.md' | :txt | :txt | 'subject/title.md' - 'foo<bar>+baz' | :txt | :txt | 'foo-bar--baz' - 'foo%2Fbar' | :txt | :txt | 'foo%2Fbar' - '' | :md | :markdown | '.md' - '' | :md | :md | '.md' - '' | :txt | :txt | '.txt' - end - - with_them do - subject { wiki.preview_slug(title, format) } - - let(:gitaly_slug) { wiki.list_pages.first } - - it { is_expected.to eq(expected_slug) } - - it 'matches the slug generated by gitaly' do - skip('Gitaly cannot generate a slug for an empty title') unless title.present? - - create_page(title, 'content', file_extension) - - gitaly_slug = wiki.list_pages.first.url_path - - is_expected.to eq(gitaly_slug) - end - end - end - - def create_page(name, content, extension = :md) - repository.create_file( - user, ::Wiki.sluggified_full_path(name, extension.to_s), content, - branch_name: default_branch, - message: "created page #{name}", - author_email: user.email, - author_name: user.name - ) - end - - def destroy_page(name, extension = :md) - repository.delete_file( - user, ::Wiki.sluggified_full_path(name, extension.to_s), - branch_name: described_class.default_ref(project), - message: "delete page #{name}", - author_email: user.email, - author_name: user.name - ) - end -end diff --git a/spec/lib/gitlab/git_access_snippet_spec.rb b/spec/lib/gitlab/git_access_snippet_spec.rb index a7036a4f20a..0d069d36e48 100644 --- a/spec/lib/gitlab/git_access_snippet_spec.rb +++ b/spec/lib/gitlab/git_access_snippet_spec.rb @@ -4,6 +4,7 @@ require 'spec_helper' RSpec.describe Gitlab::GitAccessSnippet do include ProjectHelpers + include UserHelpers include TermsHelper include AdminModeHelper include_context 'ProjectPolicyTable context' diff --git a/spec/lib/gitlab/git_access_spec.rb b/spec/lib/gitlab/git_access_spec.rb index 8577cad1011..7e3a1bf61bc 100644 --- a/spec/lib/gitlab/git_access_spec.rb +++ b/spec/lib/gitlab/git_access_spec.rb @@ -4,11 +4,9 @@ require 'spec_helper' RSpec.describe Gitlab::GitAccess, :aggregate_failures do include TermsHelper - include GitHelpers include AdminModeHelper let(:user) { create(:user) } - let(:actor) { user } let(:project) { create(:project, :repository) } let(:repository_path) { "#{project.full_path}.git" } @@ -139,27 +137,18 @@ RSpec.describe Gitlab::GitAccess, :aggregate_failures do end end - # For backwards compatibility + # legacy behavior that is blocked/deprecated context 'when actor is :ci' do let(:actor) { :ci } let(:authentication_abilities) { build_authentication_abilities } - it 'allows pull access' do - expect { pull_access_check }.not_to raise_error + it 'disallows pull access' do + expect { pull_access_check }.to raise_error(Gitlab::GitAccess::NotFoundError) end it 'does not block pushes with "not found"' do expect { push_access_check }.to raise_forbidden(described_class::ERROR_MESSAGES[:auth_upload]) end - - it 'logs' do - expect(Gitlab::AppJsonLogger).to receive(:info).with( - message: 'Actor was :ci', - project_id: project.id - ).once - - pull_access_check - end end context 'when actor is DeployToken' do @@ -741,18 +730,7 @@ RSpec.describe Gitlab::GitAccess, :aggregate_failures do describe 'generic CI (build without a user)' do let(:actor) { :ci } - context 'pull code' do - it { expect { pull_access_check }.not_to raise_error } - - it 'logs' do - expect(Gitlab::AppJsonLogger).to receive(:info).with( - message: 'Actor was :ci', - project_id: project.id - ).once - - pull_access_check - end - end + specify { expect { pull_access_check }.to raise_error Gitlab::GitAccess::NotFoundError } end end end @@ -810,18 +788,29 @@ RSpec.describe Gitlab::GitAccess, :aggregate_failures do def merge_into_protected_branch @protected_branch_merge_commit ||= begin project.repository.add_branch(user, unprotected_branch, 'feature') - rugged = rugged_repo(project.repository) - target_branch = rugged.rev_parse('feature') + target_branch = TestEnv::BRANCH_SHA['feature'] source_branch = project.repository.create_file( user, 'filename', 'This is the file content', message: 'This is a good commit message', branch_name: unprotected_branch) - author = { email: "email@example.com", time: Time.now, name: "Example Git User" } - - merge_index = rugged.merge_commits(target_branch, source_branch) - Rugged::Commit.create(rugged, author: author, committer: author, message: "commit message", parents: [target_branch, source_branch], tree: merge_index.write_tree(rugged)) + merge_id = project.repository.raw.merge_to_ref( + user, + branch: target_branch, + first_parent_ref: target_branch, + source_sha: source_branch, + target_ref: 'refs/merge-requests/test', + message: 'commit message' + ) + + # We are trying to simulate what the repository would look like + # during the pre-receive hook, before the actual ref is + # written/created. Repository#new_commits relies on there being no + # ref pointing to the merge commit. + project.repository.delete_refs('refs/merge-requests/test') + + merge_id end end diff --git a/spec/lib/gitlab/gitaly_client/blob_service_spec.rb b/spec/lib/gitlab/gitaly_client/blob_service_spec.rb index f869c66337e..d02b4492216 100644 --- a/spec/lib/gitlab/gitaly_client/blob_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/blob_service_spec.rb @@ -174,20 +174,22 @@ RSpec.describe Gitlab::GitalyClient::BlobService do expect(service) .to receive(:list_blobs) .with(gitaly_request_with_params(expected_params), kind_of(Hash)) - .and_return([ - Gitaly::ListBlobsResponse.new(blobs: [ - Gitaly::ListBlobsResponse::Blob.new(oid: "012345", size: 8, data: "0x01"), - Gitaly::ListBlobsResponse::Blob.new(data: "23") - ]), - Gitaly::ListBlobsResponse.new(blobs: [ - Gitaly::ListBlobsResponse::Blob.new(data: "45"), - Gitaly::ListBlobsResponse::Blob.new(oid: "56", size: 4, data: "0x5"), - Gitaly::ListBlobsResponse::Blob.new(data: "6") - ]), - Gitaly::ListBlobsResponse.new(blobs: [ - Gitaly::ListBlobsResponse::Blob.new(oid: "78", size: 4, data: "0x78") + .and_return( + [ + Gitaly::ListBlobsResponse.new( + blobs: [ + Gitaly::ListBlobsResponse::Blob.new(oid: "012345", size: 8, data: "0x01"), + Gitaly::ListBlobsResponse::Blob.new(data: "23") + ]), + Gitaly::ListBlobsResponse.new( + blobs: [ + Gitaly::ListBlobsResponse::Blob.new(data: "45"), + Gitaly::ListBlobsResponse::Blob.new(oid: "56", size: 4, data: "0x5"), + Gitaly::ListBlobsResponse::Blob.new(data: "6") + ]), + Gitaly::ListBlobsResponse.new( + blobs: [Gitaly::ListBlobsResponse::Blob.new(oid: "78", size: 4, data: "0x78")]) ]) - ]) end blobs = subject.to_a diff --git a/spec/lib/gitlab/gitaly_client/ref_service_spec.rb b/spec/lib/gitlab/gitaly_client/ref_service_spec.rb index b7c21516c77..5ce88b06241 100644 --- a/spec/lib/gitlab/gitaly_client/ref_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/ref_service_spec.rb @@ -3,7 +3,8 @@ require 'spec_helper' RSpec.describe Gitlab::GitalyClient::RefService do - let(:project) { create(:project, :repository) } + let_it_be(:project) { create(:project, :repository) } + let(:storage_name) { project.repository_storage } let(:relative_path) { project.disk_path + '.git' } let(:repository) { project.repository } @@ -179,13 +180,22 @@ RSpec.describe Gitlab::GitalyClient::RefService do ) ) end + local_branches = target_commits.each_with_index.map do |gitaly_commit, i| Gitaly::Branch.new(name: "#{remote_name}/#{i}", target_commit: gitaly_commit) end - response = [ - Gitaly::FindLocalBranchesResponse.new(branches: branches[0, 2], local_branches: local_branches[0, 2]), - Gitaly::FindLocalBranchesResponse.new(branches: branches[2, 2], local_branches: local_branches[2, 2]) - ] + + response = if set_local_branches + [ + Gitaly::FindLocalBranchesResponse.new(local_branches: local_branches[0, 2]), + Gitaly::FindLocalBranchesResponse.new(local_branches: local_branches[2, 2]) + ] + else + [ + Gitaly::FindLocalBranchesResponse.new(branches: branches[0, 2]), + Gitaly::FindLocalBranchesResponse.new(branches: branches[2, 2]) + ] + end expect_any_instance_of(Gitaly::RefService::Stub) .to receive(:find_local_branches) @@ -220,18 +230,14 @@ RSpec.describe Gitlab::GitalyClient::RefService do end end - context 'when feature flag :gitaly_simplify_find_local_branches_response is enabled' do - before do - stub_feature_flags(gitaly_simplify_find_local_branches_response: true) - end + context 'when local_branches variable is not set' do + let(:set_local_branches) { false } it_behaves_like 'common examples' end - context 'when feature flag :gitaly_simplify_find_local_branches_response is disabled' do - before do - stub_feature_flags(gitaly_simplify_find_local_branches_response: false) - end + context 'when local_branches variable is set' do + let(:set_local_branches) { true } it_behaves_like 'common examples' end diff --git a/spec/lib/gitlab/gitaly_client/repository_service_spec.rb b/spec/lib/gitlab/gitaly_client/repository_service_spec.rb index 63d32cb906f..58ace05b0d3 100644 --- a/spec/lib/gitlab/gitaly_client/repository_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/repository_service_spec.rb @@ -308,7 +308,7 @@ RSpec.describe Gitlab::GitalyClient::RepositoryService do end describe '#replicate' do - let(:source_repository) { Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '', 'group/project') } + let(:source_repository) { Gitlab::Git::Repository.new('default', 'repo/path', '', 'group/project') } it 'sends a replicate_repository message' do expect_any_instance_of(Gitaly::RepositoryService::Stub) @@ -343,4 +343,18 @@ RSpec.describe Gitlab::GitalyClient::RepositoryService do expect(client.full_path).to eq(path) end end + + describe "#find_license" do + it 'sends a find_license request with medium timeout' do + expect_any_instance_of(Gitaly::RepositoryService::Stub) + .to receive(:find_license) do |_service, _request, headers| + expect(headers[:deadline]).to be_between( + Gitlab::GitalyClient.fast_timeout.seconds.from_now.to_f, + Gitlab::GitalyClient.medium_timeout.seconds.from_now.to_f + ) + end + + client.find_license + end + end end diff --git a/spec/lib/gitlab/gitaly_client/wiki_service_spec.rb b/spec/lib/gitlab/gitaly_client/wiki_service_spec.rb deleted file mode 100644 index 8a169acb69c..00000000000 --- a/spec/lib/gitlab/gitaly_client/wiki_service_spec.rb +++ /dev/null @@ -1,118 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::GitalyClient::WikiService do - let(:project) { create(:project) } - let(:storage_name) { project.repository_storage } - let(:relative_path) { project.disk_path + '.git' } - let(:client) { described_class.new(project.repository) } - let(:commit) { create(:gitaly_commit) } - let(:page_version) { Gitaly::WikiPageVersion.new(format: 'markdown', commit: commit) } - let(:page_info) { { title: 'My Page', raw_data: 'a', version: page_version } } - - describe '#find_page' do - let(:response) do - [ - Gitaly::WikiFindPageResponse.new(page: Gitaly::WikiPage.new(page_info)), - Gitaly::WikiFindPageResponse.new(page: Gitaly::WikiPage.new(raw_data: 'b')) - ] - end - - let(:wiki_page) { subject.first } - let(:wiki_page_version) { subject.last } - - subject { client.find_page(title: 'My Page', version: 'master', dir: '') } - - it 'sends a wiki_find_page message' do - expect_any_instance_of(Gitaly::WikiService::Stub) - .to receive(:wiki_find_page) - .with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash)) - .and_return([].each) - - subject - end - - it 'concatenates the raw data and returns a pair of WikiPage and WikiPageVersion' do - expect_any_instance_of(Gitaly::WikiService::Stub) - .to receive(:wiki_find_page) - .with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash)) - .and_return(response.each) - - expect(wiki_page.title).to eq('My Page') - expect(wiki_page.raw_data).to eq('ab') - expect(wiki_page_version.format).to eq('markdown') - - expect(wiki_page.title).to be_utf8 - expect(wiki_page.path).to be_utf8 - expect(wiki_page.name).to be_utf8 - end - end - - describe '#load_all_pages' do - let(:page_2_info) { { title: 'My Page 2', raw_data: 'c', version: page_version } } - let(:response) do - [ - Gitaly::WikiGetAllPagesResponse.new(page: Gitaly::WikiPage.new(page_info)), - Gitaly::WikiGetAllPagesResponse.new(page: Gitaly::WikiPage.new(raw_data: 'b')), - Gitaly::WikiGetAllPagesResponse.new(end_of_page: true), - Gitaly::WikiGetAllPagesResponse.new(page: Gitaly::WikiPage.new(page_2_info)), - Gitaly::WikiGetAllPagesResponse.new(page: Gitaly::WikiPage.new(raw_data: 'd')), - Gitaly::WikiGetAllPagesResponse.new(end_of_page: true) - ] - end - - let(:wiki_page_1) { subject[0].first } - let(:wiki_page_1_version) { subject[0].last } - let(:wiki_page_2) { subject[1].first } - let(:wiki_page_2_version) { subject[1].last } - - subject { client.load_all_pages } - - it 'sends a wiki_get_all_pages message' do - expect_any_instance_of(Gitaly::WikiService::Stub) - .to receive(:wiki_get_all_pages) - .with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash)) - .and_return([].each) - - subject - end - - it 'sends a limit of 0 to wiki_get_all_pages' do - expect_any_instance_of(Gitaly::WikiService::Stub) - .to receive(:wiki_get_all_pages) - .with(gitaly_request_with_params(limit: 0), kind_of(Hash)) - .and_return([].each) - - subject - end - - it 'concatenates the raw data and returns a pair of WikiPage and WikiPageVersion for each page' do - expect_any_instance_of(Gitaly::WikiService::Stub) - .to receive(:wiki_get_all_pages) - .with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash)) - .and_return(response.each) - - expect(subject.size).to be(2) - expect(wiki_page_1.title).to eq('My Page') - expect(wiki_page_1.raw_data).to eq('ab') - expect(wiki_page_1_version.format).to eq('markdown') - expect(wiki_page_2.title).to eq('My Page 2') - expect(wiki_page_2.raw_data).to eq('cd') - expect(wiki_page_2_version.format).to eq('markdown') - end - - context 'with limits' do - subject { client.load_all_pages(limit: 1) } - - it 'sends a request with the limit' do - expect_any_instance_of(Gitaly::WikiService::Stub) - .to receive(:wiki_get_all_pages) - .with(gitaly_request_with_params(limit: 1), kind_of(Hash)) - .and_return([].each) - - subject - end - end - end -end diff --git a/spec/lib/gitlab/github_import/client_spec.rb b/spec/lib/gitlab/github_import/client_spec.rb index c88bb6de859..3361b039a27 100644 --- a/spec/lib/gitlab/github_import/client_spec.rb +++ b/spec/lib/gitlab/github_import/client_spec.rb @@ -148,7 +148,25 @@ RSpec.describe Gitlab::GithubImport::Client do .to receive(:branch_protection).with('org/repo', 'bar') expect(client).to receive(:with_rate_limit).and_yield - client.branch_protection('org/repo', 'bar') + branch_protection = client.branch_protection('org/repo', 'bar') + + expect(branch_protection).to be_a(Hash) + end + end + + describe '#each_object' do + it 'converts each object into a hash' do + client = described_class.new('foo') + + stub_request(:get, 'https://api.github.com/rate_limit') + .to_return(status: 200, headers: { 'X-RateLimit-Limit' => 5000, 'X-RateLimit-Remaining' => 5000 }) + + stub_request(:get, 'https://api.github.com/repos/foo/bar/releases?per_page=100') + .to_return(status: 200, body: [{ id: 1 }].to_json, headers: { 'Content-Type' => 'application/json' }) + + client.each_object(:releases, 'foo/bar') do |release| + expect(release).to eq({ id: 1 }) + end end end @@ -575,11 +593,11 @@ RSpec.describe Gitlab::GithubImport::Client do describe 'search' do let(:client) { described_class.new('foo') } - let(:user) { double(:user, login: 'user') } - let(:org1) { double(:org, login: 'org1') } - let(:org2) { double(:org, login: 'org2') } - let(:repo1) { double(:repo, full_name: 'repo1') } - let(:repo2) { double(:repo, full_name: 'repo2') } + let(:user) { { login: 'user' } } + let(:org1) { { login: 'org1' } } + let(:org2) { { login: 'org2' } } + let(:repo1) { { full_name: 'repo1' } } + let(:repo2) { { full_name: 'repo2' } } before do allow(client) diff --git a/spec/lib/gitlab/github_import/importer/attachments/base_importer_spec.rb b/spec/lib/gitlab/github_import/importer/attachments/base_importer_spec.rb new file mode 100644 index 00000000000..5e60be44621 --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/attachments/base_importer_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Importer::Attachments::BaseImporter do + subject(:importer) { importer_class.new(project, client) } + + let(:project) { instance_double(Project, id: 1) } + let(:client) { instance_double(Gitlab::GithubImport::Client) } + let(:importer_class) do + Class.new(described_class) do + private + + def collection_method + 'test' + end + end + end + + describe '#each_object_to_import' do + context 'with not implemented #collection interface' do + it 'raises NotImplementedError' do + expect { importer.each_object_to_import } + .to raise_error(Gitlab::GithubImport::Exceptions::NotImplementedError, '#collection') + end + end + end +end diff --git a/spec/lib/gitlab/github_import/importer/attachments/issues_importer_spec.rb b/spec/lib/gitlab/github_import/importer/attachments/issues_importer_spec.rb new file mode 100644 index 00000000000..85bc67376d3 --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/attachments/issues_importer_spec.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Importer::Attachments::IssuesImporter do + subject(:importer) { described_class.new(project, client) } + + let_it_be(:project) { create(:project) } + + let(:client) { instance_double(Gitlab::GithubImport::Client) } + + describe '#sequential_import', :clean_gitlab_redis_cache do + let_it_be(:issue_1) { create(:issue, project: project) } + let_it_be(:issue_2) { create(:issue, project: project) } + + let(:importer_stub) { instance_double('Gitlab::GithubImport::Importer::NoteAttachmentsImporter') } + let(:importer_attrs) { [instance_of(Gitlab::GithubImport::Representation::NoteText), project, client] } + + it 'imports each project issue attachments' do + expect_next_instances_of( + Gitlab::GithubImport::Importer::NoteAttachmentsImporter, 2, false, *importer_attrs + ) do |note_attachments_importer| + expect(note_attachments_importer).to receive(:execute) + end + + importer.sequential_import + end + + context 'when issue is already processed' do + it "doesn't import this issue attachments" do + importer.mark_as_imported(issue_1) + + expect_next_instance_of( + Gitlab::GithubImport::Importer::NoteAttachmentsImporter, *importer_attrs + ) do |note_attachments_importer| + expect(note_attachments_importer).to receive(:execute) + end + + importer.sequential_import + end + end + end + + describe '#sidekiq_worker_class' do + it { expect(importer.sidekiq_worker_class).to eq(Gitlab::GithubImport::Attachments::ImportIssueWorker) } + end + + describe '#collection_method' do + it { expect(importer.collection_method).to eq(:issue_attachments) } + end + + describe '#object_type' do + it { expect(importer.object_type).to eq(:issue_attachment) } + end + + describe '#id_for_already_imported_cache' do + let(:issue) { build_stubbed(:issue) } + + it { expect(importer.id_for_already_imported_cache(issue)).to eq(issue.id) } + end +end diff --git a/spec/lib/gitlab/github_import/importer/attachments/merge_requests_importer_spec.rb b/spec/lib/gitlab/github_import/importer/attachments/merge_requests_importer_spec.rb new file mode 100644 index 00000000000..e4718c2d17c --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/attachments/merge_requests_importer_spec.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Importer::Attachments::MergeRequestsImporter do + subject(:importer) { described_class.new(project, client) } + + let_it_be(:project) { create(:project) } + + let(:client) { instance_double(Gitlab::GithubImport::Client) } + + describe '#sequential_import', :clean_gitlab_redis_cache do + let_it_be(:merge_request_1) { create(:merge_request, source_project: project, target_branch: 'feature1') } + let_it_be(:merge_request_2) { create(:merge_request, source_project: project, target_branch: 'feature2') } + + let(:importer_stub) { instance_double('Gitlab::GithubImport::Importer::NoteAttachmentsImporter') } + let(:importer_attrs) { [instance_of(Gitlab::GithubImport::Representation::NoteText), project, client] } + + it 'imports each project merge request attachments' do + expect_next_instances_of( + Gitlab::GithubImport::Importer::NoteAttachmentsImporter, 2, false, *importer_attrs + ) do |note_attachments_importer| + expect(note_attachments_importer).to receive(:execute) + end + + importer.sequential_import + end + + context 'when merge request is already processed' do + it "doesn't import this merge request attachments" do + importer.mark_as_imported(merge_request_1) + + expect_next_instance_of( + Gitlab::GithubImport::Importer::NoteAttachmentsImporter, *importer_attrs + ) do |note_attachments_importer| + expect(note_attachments_importer).to receive(:execute) + end + + importer.sequential_import + end + end + end + + describe '#sidekiq_worker_class' do + it { expect(importer.sidekiq_worker_class).to eq(Gitlab::GithubImport::Attachments::ImportMergeRequestWorker) } + end + + describe '#collection_method' do + it { expect(importer.collection_method).to eq(:merge_request_attachments) } + end + + describe '#object_type' do + it { expect(importer.object_type).to eq(:merge_request_attachment) } + end + + describe '#id_for_already_imported_cache' do + let(:merge_request) { build_stubbed(:merge_request) } + + it { expect(importer.id_for_already_imported_cache(merge_request)).to eq(merge_request.id) } + end +end diff --git a/spec/lib/gitlab/github_import/importer/attachments/notes_importer_spec.rb b/spec/lib/gitlab/github_import/importer/attachments/notes_importer_spec.rb new file mode 100644 index 00000000000..7ed353e1b71 --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/attachments/notes_importer_spec.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Importer::Attachments::NotesImporter do + subject(:importer) { described_class.new(project, client) } + + let_it_be(:project) { create(:project) } + + let(:client) { instance_double(Gitlab::GithubImport::Client) } + + describe '#sequential_import', :clean_gitlab_redis_cache do + let_it_be(:note_1) { create(:note, project: project) } + let_it_be(:note_2) { create(:note, project: project) } + let_it_be(:system_note) { create(:note, :system, project: project) } + + let(:importer_stub) { instance_double('Gitlab::GithubImport::Importer::NoteAttachmentsImporter') } + let(:importer_attrs) { [instance_of(Gitlab::GithubImport::Representation::NoteText), project, client] } + + it 'imports each project user note' do + expect(Gitlab::GithubImport::Importer::NoteAttachmentsImporter).to receive(:new) + .with(*importer_attrs).twice.and_return(importer_stub) + expect(importer_stub).to receive(:execute).twice + + importer.sequential_import + end + + context 'when note is already processed' do + it "doesn't import this note" do + importer.mark_as_imported(note_1) + + expect(Gitlab::GithubImport::Importer::NoteAttachmentsImporter).to receive(:new) + .with(*importer_attrs).once.and_return(importer_stub) + expect(importer_stub).to receive(:execute).once + + importer.sequential_import + end + end + end + + describe '#sidekiq_worker_class' do + it { expect(importer.sidekiq_worker_class).to eq(Gitlab::GithubImport::Attachments::ImportNoteWorker) } + end + + describe '#collection_method' do + it { expect(importer.collection_method).to eq(:note_attachments) } + end + + describe '#object_type' do + it { expect(importer.object_type).to eq(:note_attachment) } + end + + describe '#id_for_already_imported_cache' do + let(:note) { build_stubbed(:note) } + + it { expect(importer.id_for_already_imported_cache(note)).to eq(note.id) } + end +end diff --git a/spec/lib/gitlab/github_import/importer/attachments/releases_importer_spec.rb b/spec/lib/gitlab/github_import/importer/attachments/releases_importer_spec.rb new file mode 100644 index 00000000000..b989345ae09 --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/attachments/releases_importer_spec.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Importer::Attachments::ReleasesImporter do + subject(:importer) { described_class.new(project, client) } + + let_it_be(:project) { create(:project) } + + let(:client) { instance_double(Gitlab::GithubImport::Client) } + + describe '#sequential_import', :clean_gitlab_redis_cache do + let_it_be(:release_1) { create(:release, project: project) } + let_it_be(:release_2) { create(:release, project: project) } + + let(:importer_stub) { instance_double('Gitlab::GithubImport::Importer::NoteAttachmentsImporter') } + let(:importer_attrs) { [instance_of(Gitlab::GithubImport::Representation::NoteText), project, client] } + + it 'imports each project release' do + expect(Gitlab::GithubImport::Importer::NoteAttachmentsImporter).to receive(:new) + .with(*importer_attrs).twice.and_return(importer_stub) + expect(importer_stub).to receive(:execute).twice + + importer.sequential_import + end + + context 'when note is already processed' do + it "doesn't import this release" do + importer.mark_as_imported(release_1) + + expect(Gitlab::GithubImport::Importer::NoteAttachmentsImporter).to receive(:new) + .with(*importer_attrs).once.and_return(importer_stub) + expect(importer_stub).to receive(:execute).once + + importer.sequential_import + end + end + end + + describe '#sidekiq_worker_class' do + it { expect(importer.sidekiq_worker_class).to eq(Gitlab::GithubImport::Attachments::ImportReleaseWorker) } + end + + describe '#collection_method' do + it { expect(importer.collection_method).to eq(:release_attachments) } + end + + describe '#object_type' do + it { expect(importer.object_type).to eq(:release_attachment) } + end + + describe '#id_for_already_imported_cache' do + let(:release) { build_stubbed(:release) } + + it { expect(importer.id_for_already_imported_cache(release)).to eq(release.id) } + end +end diff --git a/spec/lib/gitlab/github_import/importer/diff_notes_importer_spec.rb b/spec/lib/gitlab/github_import/importer/diff_notes_importer_spec.rb index 6eb92cdeab9..a8dd6b4725d 100644 --- a/spec/lib/gitlab/github_import/importer/diff_notes_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/diff_notes_importer_spec.rb @@ -7,14 +7,13 @@ RSpec.describe Gitlab::GithubImport::Importer::DiffNotesImporter do let(:client) { double(:client) } let(:github_comment) do - double( - :response, + { html_url: 'https://github.com/foo/bar/pull/42', path: 'README.md', commit_id: '123abc', original_commit_id: 'original123abc', diff_hunk: "@@ -1 +1 @@\n-Hello\n+Hello world", - user: double(:user, id: 4, login: 'alice'), + user: { id: 4, login: 'alice' }, created_at: Time.zone.now, updated_at: Time.zone.now, line: 23, @@ -29,7 +28,7 @@ RSpec.describe Gitlab::GithubImport::Importer::DiffNotesImporter do sug1 ``` BODY - ) + } end describe '#parallel?' do @@ -98,9 +97,10 @@ RSpec.describe Gitlab::GithubImport::Importer::DiffNotesImporter do .to receive(:each_object_to_import) .and_yield(github_comment) - expect(Gitlab::GithubImport::ImportDiffNoteWorker).to receive(:bulk_perform_in).with(1.second, [ - [project.id, an_instance_of(Hash), an_instance_of(String)] - ], batch_size: 1000, batch_delay: 1.minute) + expect(Gitlab::GithubImport::ImportDiffNoteWorker).to receive(:bulk_perform_in) + .with(1.second, [ + [project.id, an_instance_of(Hash), an_instance_of(String)] + ], batch_size: 1000, batch_delay: 1.minute) waiter = importer.parallel_import diff --git a/spec/lib/gitlab/github_import/importer/issues_importer_spec.rb b/spec/lib/gitlab/github_import/importer/issues_importer_spec.rb index 6b807bdf098..308b8185589 100644 --- a/spec/lib/gitlab/github_import/importer/issues_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/issues_importer_spec.rb @@ -9,20 +9,19 @@ RSpec.describe Gitlab::GithubImport::Importer::IssuesImporter do let(:updated_at) { Time.new(2017, 1, 1, 12, 15) } let(:github_issue) do - double( - :response, + { number: 42, title: 'My Issue', body: 'This is my issue', - milestone: double(:milestone, number: 4), + milestone: { number: 4 }, state: 'open', - assignees: [double(:user, id: 4, login: 'alice')], - labels: [double(:label, name: 'bug')], - user: double(:user, id: 4, login: 'alice'), + assignees: [{ id: 4, login: 'alice' }], + labels: [{ name: 'bug' }], + user: { id: 4, login: 'alice' }, created_at: created_at, updated_at: updated_at, pull_request: false - ) + } end describe '#parallel?' do @@ -110,4 +109,24 @@ RSpec.describe Gitlab::GithubImport::Importer::IssuesImporter do .to eq(42) end end + + describe '#increment_object_counter?' do + let(:importer) { described_class.new(project, client) } + + context 'when issue is a pull request' do + let(:github_issue) { { pull_request: { url: 'some_url' } } } + + it 'returns false' do + expect(importer).not_to be_increment_object_counter(github_issue) + end + end + + context 'when issue is a regular issue' do + let(:github_issue) { {} } + + it 'returns true' do + expect(importer).to be_increment_object_counter(github_issue) + end + end + end end diff --git a/spec/lib/gitlab/github_import/importer/labels_importer_spec.rb b/spec/lib/gitlab/github_import/importer/labels_importer_spec.rb index ca9d3e1e21c..81d534c566f 100644 --- a/spec/lib/gitlab/github_import/importer/labels_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/labels_importer_spec.rb @@ -28,7 +28,7 @@ RSpec.describe Gitlab::GithubImport::Importer::LabelsImporter, :clean_gitlab_red describe '#build_labels' do it 'returns an Array containnig label rows' do - label = double(:label, name: 'bug', color: 'ffffff') + label = { name: 'bug', color: 'ffffff' } expect(importer).to receive(:each_label).and_return([label]) @@ -41,7 +41,7 @@ RSpec.describe Gitlab::GithubImport::Importer::LabelsImporter, :clean_gitlab_red it 'does not create labels that already exist' do create(:label, project: project, title: 'bug') - label = double(:label, name: 'bug', color: 'ffffff') + label = { name: 'bug', color: 'ffffff' } expect(importer).to receive(:each_label).and_return([label]) expect(importer.build_labels).to be_empty @@ -60,7 +60,7 @@ RSpec.describe Gitlab::GithubImport::Importer::LabelsImporter, :clean_gitlab_red describe '#build' do let(:label_hash) do - importer.build(double(:label, name: 'bug', color: 'ffffff')) + importer.build({ name: 'bug', color: 'ffffff' }) end it 'returns the attributes of the label as a Hash' do diff --git a/spec/lib/gitlab/github_import/importer/lfs_objects_importer_spec.rb b/spec/lib/gitlab/github_import/importer/lfs_objects_importer_spec.rb index 251829b83a0..99536588718 100644 --- a/spec/lib/gitlab/github_import/importer/lfs_objects_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/lfs_objects_importer_spec.rb @@ -118,9 +118,10 @@ RSpec.describe Gitlab::GithubImport::Importer::LfsObjectsImporter do expect(service).to receive(:execute).and_return([lfs_download_object]) end - expect(Gitlab::GithubImport::ImportLfsObjectWorker).to receive(:bulk_perform_in).with(1.second, [ - [project.id, an_instance_of(Hash), an_instance_of(String)] - ], batch_size: 1000, batch_delay: 1.minute) + expect(Gitlab::GithubImport::ImportLfsObjectWorker).to receive(:bulk_perform_in) + .with(1.second, [ + [project.id, an_instance_of(Hash), an_instance_of(String)] + ], batch_size: 1000, batch_delay: 1.minute) waiter = importer.parallel_import diff --git a/spec/lib/gitlab/github_import/importer/milestones_importer_spec.rb b/spec/lib/gitlab/github_import/importer/milestones_importer_spec.rb index dad1efc5a8d..04d76bd1f06 100644 --- a/spec/lib/gitlab/github_import/importer/milestones_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/milestones_importer_spec.rb @@ -11,8 +11,7 @@ RSpec.describe Gitlab::GithubImport::Importer::MilestonesImporter, :clean_gitlab let(:updated_at) { Time.new(2017, 1, 1, 12, 15) } let(:milestone) do - double( - :milestone, + { number: 1, title: '1.0', description: 'The first release', @@ -20,12 +19,11 @@ RSpec.describe Gitlab::GithubImport::Importer::MilestonesImporter, :clean_gitlab due_on: due_on, created_at: created_at, updated_at: updated_at - ) + } end let(:milestone2) do - double( - :milestone, + { number: 1, title: '1.0', description: 'The first release', @@ -33,7 +31,7 @@ RSpec.describe Gitlab::GithubImport::Importer::MilestonesImporter, :clean_gitlab due_on: nil, created_at: created_at, updated_at: updated_at - ) + } end describe '#execute' do diff --git a/spec/lib/gitlab/github_import/importer/note_attachments_importer_spec.rb b/spec/lib/gitlab/github_import/importer/note_attachments_importer_spec.rb new file mode 100644 index 00000000000..7d4e3c3bcce --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/note_attachments_importer_spec.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Importer::NoteAttachmentsImporter do + subject(:importer) { described_class.new(note_text, project, client) } + + let_it_be(:project) { create(:project) } + + let(:note_text) { Gitlab::GithubImport::Representation::NoteText.from_db_record(record) } + let(:client) { instance_double('Gitlab::GithubImport::Client') } + + let(:doc_url) { 'https://github.com/nickname/public-test-repo/files/9020437/git-cheat-sheet.txt' } + let(:image_url) { 'https://user-images.githubusercontent.com/6833842/0cf366b61ef2.jpeg' } + let(:image_tag_url) { 'https://user-images.githubusercontent.com/6833842/0cf366b61ea5.jpeg' } + let(:text) do + <<-TEXT.split("\n").map(&:strip).join("\n") + Some text... + + [special-doc](#{doc_url}) +  + <img width=\"248\" alt=\"tag-image\" src="#{image_tag_url}"> + TEXT + end + + shared_examples 'updates record description' do + it do + importer.execute + + record.reload + expect(record.description).to start_with("Some text...\n\n[special-doc](/uploads/") + expect(record.description).to include(' + expect(record.description).to include('<img width="248" alt="tag-image" src="/uploads') + end + end + + describe '#execute' do + let(:downloader_stub) { instance_double(Gitlab::GithubImport::AttachmentsDownloader) } + let(:tmp_stub_doc) { Tempfile.create('attachment_download_test.txt') } + let(:tmp_stub_image) { Tempfile.create('image.jpeg') } + let(:tmp_stub_image_tag) { Tempfile.create('image-tag.jpeg') } + + before do + allow(Gitlab::GithubImport::AttachmentsDownloader).to receive(:new).with(doc_url) + .and_return(downloader_stub) + allow(Gitlab::GithubImport::AttachmentsDownloader).to receive(:new).with(image_url) + .and_return(downloader_stub) + allow(Gitlab::GithubImport::AttachmentsDownloader).to receive(:new).with(image_tag_url) + .and_return(downloader_stub) + allow(downloader_stub).to receive(:perform).and_return(tmp_stub_doc, tmp_stub_image, tmp_stub_image_tag) + allow(downloader_stub).to receive(:delete).exactly(3).times + end + + context 'when importing release attachments' do + let(:record) { create(:release, project: project, description: text) } + + it_behaves_like 'updates record description' + end + + context 'when importing issue attachments' do + let(:record) { create(:issue, project: project, description: text) } + + it_behaves_like 'updates record description' + end + + context 'when importing merge request attachments' do + let(:record) { create(:merge_request, source_project: project, description: text) } + + it_behaves_like 'updates record description' + end + + context 'when importing note attachments' do + let(:record) { create(:note, project: project, note: text) } + + it 'updates note text with new attachment urls' do + importer.execute + + record.reload + expect(record.note).to start_with("Some text...\n\n[special-doc](/uploads/") + expect(record.note).to include(' + expect(record.note).to include('<img width="248" alt="tag-image" src="/uploads') + end + end + end +end diff --git a/spec/lib/gitlab/github_import/importer/note_importer_spec.rb b/spec/lib/gitlab/github_import/importer/note_importer_spec.rb index 165f543525d..c60ecd85e92 100644 --- a/spec/lib/gitlab/github_import/importer/note_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/note_importer_spec.rb @@ -160,6 +160,13 @@ RSpec.describe Gitlab::GithubImport::Importer::NoteImporter do expect(project.notes.take).to be_valid end + + # rubocop:disable RSpec/AnyInstanceOf + it 'skips markdown field cache callback' do + expect_any_instance_of(Note).not_to receive(:refresh_markdown_cache) + importer.execute + end + # rubocop:enable RSpec/AnyInstanceOf end describe '#find_noteable_id' do diff --git a/spec/lib/gitlab/github_import/importer/notes_importer_spec.rb b/spec/lib/gitlab/github_import/importer/notes_importer_spec.rb index 3b4fe652da8..ca4560b6a1a 100644 --- a/spec/lib/gitlab/github_import/importer/notes_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/notes_importer_spec.rb @@ -7,15 +7,14 @@ RSpec.describe Gitlab::GithubImport::Importer::NotesImporter do let(:client) { double(:client) } let(:github_comment) do - double( - :response, + { html_url: 'https://github.com/foo/bar/issues/42', - user: double(:user, id: 4, login: 'alice'), + user: { id: 4, login: 'alice' }, body: 'Hello world', created_at: Time.zone.now, updated_at: Time.zone.now, id: 1 - ) + } end describe '#parallel?' do @@ -84,9 +83,10 @@ RSpec.describe Gitlab::GithubImport::Importer::NotesImporter do .to receive(:each_object_to_import) .and_yield(github_comment) - expect(Gitlab::GithubImport::ImportNoteWorker).to receive(:bulk_perform_in).with(1.second, [ - [project.id, an_instance_of(Hash), an_instance_of(String)] - ], batch_size: 1000, batch_delay: 1.minute) + expect(Gitlab::GithubImport::ImportNoteWorker).to receive(:bulk_perform_in) + .with(1.second, [ + [project.id, an_instance_of(Hash), an_instance_of(String)] + ], batch_size: 1000, batch_delay: 1.minute) waiter = importer.parallel_import diff --git a/spec/lib/gitlab/github_import/importer/protected_branch_importer_spec.rb b/spec/lib/gitlab/github_import/importer/protected_branch_importer_spec.rb index 6dc6db739f4..027b2ac422e 100644 --- a/spec/lib/gitlab/github_import/importer/protected_branch_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/protected_branch_importer_spec.rb @@ -5,11 +5,21 @@ require 'spec_helper' RSpec.describe Gitlab::GithubImport::Importer::ProtectedBranchImporter do subject(:importer) { described_class.new(github_protected_branch, project, client) } + let(:branch_name) { 'protection' } let(:allow_force_pushes_on_github) { true } + let(:required_conversation_resolution) { false } + let(:required_signatures) { false } + let(:required_pull_request_reviews) { false } + let(:expected_push_access_level) { Gitlab::Access::MAINTAINER } + let(:expected_merge_access_level) { Gitlab::Access::MAINTAINER } + let(:expected_allow_force_push) { true } let(:github_protected_branch) do Gitlab::GithubImport::Representation::ProtectedBranch.new( - id: 'protection', - allow_force_pushes: allow_force_pushes_on_github + id: branch_name, + allow_force_pushes: allow_force_pushes_on_github, + required_conversation_resolution: required_conversation_resolution, + required_signatures: required_signatures, + required_pull_request_reviews: required_pull_request_reviews ) end @@ -23,8 +33,8 @@ RSpec.describe Gitlab::GithubImport::Importer::ProtectedBranchImporter do let(:expected_ruleset) do { name: 'protection', - push_access_levels_attributes: [{ access_level: Gitlab::Access::MAINTAINER }], - merge_access_levels_attributes: [{ access_level: Gitlab::Access::MAINTAINER }], + push_access_levels_attributes: [{ access_level: expected_push_access_level }], + merge_access_levels_attributes: [{ access_level: expected_merge_access_level }], allow_force_push: expected_allow_force_push } end @@ -47,6 +57,18 @@ RSpec.describe Gitlab::GithubImport::Importer::ProtectedBranchImporter do end end + shared_examples 'does not change project attributes' do + it 'does not change only_allow_merge_if_all_discussions_are_resolved' do + expect { importer.execute }.not_to change(project, :only_allow_merge_if_all_discussions_are_resolved) + end + + it 'does not change push_rule for the project' do + expect(project).not_to receive(:push_rule) + + importer.execute + end + end + context 'when branch is protected on GitLab' do before do create( @@ -87,5 +109,193 @@ RSpec.describe Gitlab::GithubImport::Importer::ProtectedBranchImporter do it_behaves_like 'create branch protection by the strictest ruleset' end + + context "when branch is default" do + before do + allow(project).to receive(:default_branch).and_return(branch_name) + end + + context 'when required_conversation_resolution rule is enabled' do + let(:required_conversation_resolution) { true } + + it 'changes project settings' do + expect { importer.execute }.to change(project, :only_allow_merge_if_all_discussions_are_resolved).to(true) + end + end + + context 'when required_conversation_resolution rule is disabled' do + let(:required_conversation_resolution) { false } + + it_behaves_like 'does not change project attributes' + end + + context 'when required_signatures rule is enabled' do + let(:required_signatures) { true } + let(:push_rules_feature_available?) { true } + + before do + stub_licensed_features(push_rules: push_rules_feature_available?) + end + + context 'when the push_rules feature is available', if: Gitlab.ee? do + context 'when project push_rules did previously exist' do + before do + create(:push_rule, project: project) + end + + it 'updates push_rule reject_unsigned_commits attribute' do + expect { importer.execute }.to change { project.reload.push_rule.reject_unsigned_commits }.to(true) + end + end + + context 'when project push_rules did not previously exist' do + it 'creates project push_rule with the enabled reject_unsigned_commits attribute' do + expect { importer.execute }.to change(project, :push_rule).from(nil) + expect(project.push_rule.reject_unsigned_commits).to be_truthy + end + end + end + + context 'when the push_rules feature is not available' do + let(:push_rules_feature_available?) { false } + + it_behaves_like 'does not change project attributes' + end + end + + context 'when required_signatures rule is disabled' do + let(:required_signatures) { false } + + it_behaves_like 'does not change project attributes' + end + end + + context 'when branch is not default' do + context 'when required_conversation_resolution rule is enabled' do + let(:required_conversation_resolution) { true } + + it_behaves_like 'does not change project attributes' + end + + context 'when required_conversation_resolution rule is disabled' do + let(:required_conversation_resolution) { false } + + it_behaves_like 'does not change project attributes' + end + + context 'when required_signatures rule is enabled' do + let(:required_signatures) { true } + + it_behaves_like 'does not change project attributes' + end + + context 'when required_signatures rule is disabled' do + let(:required_signatures) { false } + + it_behaves_like 'does not change project attributes' + end + end + + context 'when required_pull_request_reviews rule is enabled on GitHub' do + let(:required_pull_request_reviews) { true } + let(:expected_push_access_level) { Gitlab::Access::NO_ACCESS } + let(:expected_merge_access_level) { Gitlab::Access::MAINTAINER } + + it_behaves_like 'create branch protection by the strictest ruleset' + end + + context 'when required_pull_request_reviews rule is disabled on GitHub' do + let(:required_pull_request_reviews) { false } + + context 'when branch is default' do + before do + allow(project).to receive(:default_branch).and_return(branch_name) + end + + context 'when default branch protection = Gitlab::Access::PROTECTION_DEV_CAN_PUSH' do + before do + stub_application_setting(default_branch_protection: Gitlab::Access::PROTECTION_DEV_CAN_PUSH) + end + + let(:expected_push_access_level) { Gitlab::Access::DEVELOPER } + let(:expected_merge_access_level) { Gitlab::Access::MAINTAINER } + + it_behaves_like 'create branch protection by the strictest ruleset' + end + + context 'when default branch protection = Gitlab::Access::PROTECTION_DEV_CAN_MERGE' do + before do + stub_application_setting(default_branch_protection: Gitlab::Access::PROTECTION_DEV_CAN_MERGE) + end + + let(:expected_push_access_level) { Gitlab::Access::MAINTAINER } + let(:expected_merge_access_level) { Gitlab::Access::DEVELOPER } + + it_behaves_like 'create branch protection by the strictest ruleset' + end + end + + context 'when branch is protected on GitLab' do + let(:protected_branch) do + create( + :protected_branch, + project: project, + name: 'protect*', + allow_force_push: true + ) + end + + let(:push_access_level) { protected_branch.push_access_levels.first } + let(:merge_access_level) { protected_branch.merge_access_levels.first } + + context 'when there is branch protection rule for the role' do + context 'when No one can merge' do + before do + merge_access_level.update_column(:access_level, Gitlab::Access::NO_ACCESS) + end + + let(:expected_push_access_level) { push_access_level.access_level } + let(:expected_merge_access_level) { Gitlab::Access::NO_ACCESS } + + it_behaves_like 'create branch protection by the strictest ruleset' + end + + context 'when Maintainers and Developers can merge' do + before do + merge_access_level.update_column(:access_level, Gitlab::Access::DEVELOPER) + end + + let(:gitlab_push_access_level) { push_access_level.access_level } + let(:gitlab_merge_access_level) { merge_access_level.access_level } + let(:expected_push_access_level) { gitlab_push_access_level } + let(:expected_merge_access_level) { [gitlab_merge_access_level, github_default_merge_access_level].max } + let(:github_default_merge_access_level) do + Gitlab::GithubImport::Importer::ProtectedBranchImporter::GITHUB_DEFAULT_MERGE_ACCESS_LEVEL + end + + it_behaves_like 'create branch protection by the strictest ruleset' + end + end + + context 'when there is no branch protection rule for the role' do + before do + push_access_level.update_column(:user_id, project.owner.id) + merge_access_level.update_column(:user_id, project.owner.id) + end + + let(:expected_push_access_level) { ProtectedBranch::PushAccessLevel::GITLAB_DEFAULT_ACCESS_LEVEL } + let(:expected_merge_access_level) { Gitlab::Access::MAINTAINER } + + it_behaves_like 'create branch protection by the strictest ruleset' + end + end + + context 'when branch is neither default nor protected on GitLab' do + let(:expected_push_access_level) { ProtectedBranch::PushAccessLevel::GITLAB_DEFAULT_ACCESS_LEVEL } + let(:expected_merge_access_level) { ProtectedBranch::MergeAccessLevel::GITLAB_DEFAULT_ACCESS_LEVEL } + + it_behaves_like 'create branch protection by the strictest ruleset' + end + end end end diff --git a/spec/lib/gitlab/github_import/importer/protected_branches_importer_spec.rb b/spec/lib/gitlab/github_import/importer/protected_branches_importer_spec.rb index 4e9208be985..a0ced456391 100644 --- a/spec/lib/gitlab/github_import/importer/protected_branches_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/protected_branches_importer_spec.rb @@ -23,11 +23,13 @@ RSpec.describe Gitlab::GithubImport::Importer::ProtectedBranchesImporter do let(:github_protection_rule) do response = Struct.new(:name, :url, :required_signatures, :enforce_admins, :required_linear_history, :allow_force_pushes, :allow_deletion, :block_creations, :required_conversation_resolution, + :required_pull_request_reviews, keyword_init: true ) required_signatures = Struct.new(:url, :enabled, keyword_init: true) enforce_admins = Struct.new(:url, :enabled, keyword_init: true) allow_option = Struct.new(:enabled, keyword_init: true) + required_pull_request_reviews = Struct.new(:url, :dismissal_restrictions, keyword_init: true) response.new( name: 'main', url: 'https://example.com/branches/main/protection', @@ -53,6 +55,10 @@ RSpec.describe Gitlab::GithubImport::Importer::ProtectedBranchesImporter do ), required_conversation_resolution: allow_option.new( enabled: false + ), + required_pull_request_reviews: required_pull_request_reviews.new( + url: 'https://example.com/branches/main/protection/required_pull_request_reviews', + dismissal_restrictions: {} ) ) end diff --git a/spec/lib/gitlab/github_import/importer/pull_request_merged_by_importer_spec.rb b/spec/lib/gitlab/github_import/importer/pull_request_merged_by_importer_spec.rb index 016f6e5377b..f3a9bbac785 100644 --- a/spec/lib/gitlab/github_import/importer/pull_request_merged_by_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/pull_request_merged_by_importer_spec.rb @@ -7,15 +7,16 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestMergedByImporter, :cle let(:project) { merge_request.project } let(:merged_at) { Time.new(2017, 1, 1, 12, 00).utc } - let(:client_double) { double(user: double(id: 999, login: 'merger', email: 'merger@email.com')) } - let(:merger_user) { double(id: 999, login: 'merger') } + let(:client_double) { double(user: { id: 999, login: 'merger', email: 'merger@email.com' } ) } + let(:merger_user) { { id: 999, login: 'merger' } } let(:pull_request) do - instance_double( - Gitlab::GithubImport::Representation::PullRequest, - iid: merge_request.iid, - merged_at: merged_at, - merged_by: merger_user + Gitlab::GithubImport::Representation::PullRequest.from_api_response( + { + number: merge_request.iid, + merged_at: merged_at, + merged_by: merger_user + } ) end diff --git a/spec/lib/gitlab/github_import/importer/pull_request_review_importer_spec.rb b/spec/lib/gitlab/github_import/importer/pull_request_review_importer_spec.rb index a6da40f47f1..fb6024d0952 100644 --- a/spec/lib/gitlab/github_import/importer/pull_request_review_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/pull_request_review_importer_spec.rb @@ -8,7 +8,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestReviewImporter, :clean let_it_be(:merge_request) { create(:merge_request) } let(:project) { merge_request.project } - let(:client_double) { double(user: double(id: 999, login: 'author', email: 'author@email.com')) } + let(:client_double) { double(user: { id: 999, login: 'author', email: 'author@email.com' }) } let(:submitted_at) { Time.new(2017, 1, 1, 12, 00).utc } subject { described_class.new(review, project, client_double) } diff --git a/spec/lib/gitlab/github_import/importer/pull_requests_importer_spec.rb b/spec/lib/gitlab/github_import/importer/pull_requests_importer_spec.rb index c5846fa7a87..aa92abdb110 100644 --- a/spec/lib/gitlab/github_import/importer/pull_requests_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/pull_requests_importer_spec.rb @@ -8,33 +8,30 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsImporter do let(:client) { double(:client) } let(:pull_request) do - double( - :response, + { number: 42, title: 'My Pull Request', body: 'This is my pull request', state: 'closed', - head: double( - :head, + head: { sha: '123abc', ref: 'my-feature', - repo: double(:repo, id: 400), - user: double(:user, id: 4, login: 'alice') - ), - base: double( - :base, + repo: { id: 400 }, + user: { id: 4, login: 'alice' } + }, + base: { sha: '456def', ref: 'master', - repo: double(:repo, id: 200) - ), - milestone: double(:milestone, number: 4), - user: double(:user, id: 4, login: 'alice'), - assignee: double(:user, id: 4, login: 'alice'), - merged_by: double(:user, id: 4, login: 'alice'), + repo: { id: 200 } + }, + milestone: { number: 4 }, + user: { id: 4, login: 'alice' }, + assignee: { id: 4, login: 'alice' }, + merged_by: { id: 4, login: 'alice' }, created_at: 1.second.ago, updated_at: 1.second.ago, merged_at: 1.second.ago - ) + } end describe '#parallel?' do @@ -184,12 +181,11 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsImporter do context 'when the pull request was updated after the last update' do let(:pr) do - double( - :pr, + { updated_at: Time.zone.now, - head: double(:head, sha: '123'), - base: double(:base, sha: '456') - ) + head: { sha: '123' }, + base: { sha: '456' } + } end before do @@ -201,7 +197,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsImporter do it 'returns true when the head SHA is not present' do expect(importer) .to receive(:commit_exists?) - .with(pr.head.sha) + .with('123') .and_return(false) expect(importer.update_repository?(pr)).to eq(true) @@ -210,12 +206,12 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsImporter do it 'returns true when the base SHA is not present' do expect(importer) .to receive(:commit_exists?) - .with(pr.head.sha) + .with('123') .and_return(true) expect(importer) .to receive(:commit_exists?) - .with(pr.base.sha) + .with('456') .and_return(false) expect(importer.update_repository?(pr)).to eq(true) @@ -224,12 +220,12 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsImporter do it 'returns false if both the head and base SHAs are present' do expect(importer) .to receive(:commit_exists?) - .with(pr.head.sha) + .with('123') .and_return(true) expect(importer) .to receive(:commit_exists?) - .with(pr.base.sha) + .with('456') .and_return(true) expect(importer.update_repository?(pr)).to eq(false) @@ -238,7 +234,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsImporter do context 'when the pull request was updated before the last update' do it 'returns false' do - pr = double(:pr, updated_at: 1.year.ago) + pr = { updated_at: 1.year.ago } allow(project) .to receive(:last_repository_updated_at) diff --git a/spec/lib/gitlab/github_import/importer/pull_requests_reviews_importer_spec.rb b/spec/lib/gitlab/github_import/importer/pull_requests_reviews_importer_spec.rb index 0eb86feb040..5f9c73cbfff 100644 --- a/spec/lib/gitlab/github_import/importer/pull_requests_reviews_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/pull_requests_reviews_importer_spec.rb @@ -23,7 +23,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsReviewsImporter do end describe '#id_for_already_imported_cache' do - it { expect(subject.id_for_already_imported_cache(double(id: 1))).to eq(1) } + it { expect(subject.id_for_already_imported_cache({ id: 1 })).to eq(1) } end describe '#each_object_to_import', :clean_gitlab_redis_cache do @@ -36,15 +36,11 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsReviewsImporter do ) end - let(:review) { double(id: 1) } + let(:review) { { id: 1 } } it 'fetches the pull requests reviews data' do page = double(objects: [review], number: 1) - expect(review) - .to receive(:merge_request_id=) - .with(merge_request.id) - expect(client) .to receive(:each_page) .exactly(:once) # ensure to be cached on the second call @@ -55,6 +51,8 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsReviewsImporter do .to yield_with_args(review) subject.each_object_to_import {} + + expect(review[:merge_request_id]).to eq(merge_request.id) end it 'skips cached pages' do diff --git a/spec/lib/gitlab/github_import/importer/release_attachments_importer_spec.rb b/spec/lib/gitlab/github_import/importer/release_attachments_importer_spec.rb deleted file mode 100644 index 4779f9c8982..00000000000 --- a/spec/lib/gitlab/github_import/importer/release_attachments_importer_spec.rb +++ /dev/null @@ -1,57 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::GithubImport::Importer::ReleaseAttachmentsImporter do - subject(:importer) { described_class.new(release_attachments, project, client) } - - let_it_be(:project) { create(:project) } - - let(:client) { instance_double('Gitlab::GithubImport::Client') } - let(:release) { create(:release, project: project, description: description) } - let(:release_attachments) do - Gitlab::GithubImport::Representation::ReleaseAttachments - .from_json_hash(release_db_id: release.id, description: release.description) - end - - let(:doc_url) { 'https://github.com/nickname/public-test-repo/files/9020437/git-cheat-sheet.txt' } - let(:image_url) { 'https://user-images.githubusercontent.com/6833842/0cf366b61ef2.jpeg' } - let(:description) do - <<-TEXT.strip - Some text... - - [special-doc](#{doc_url}) -  - TEXT - end - - describe '#execute' do - let(:downloader_stub) { instance_double(Gitlab::GithubImport::AttachmentsDownloader) } - let(:tmp_stub_doc) { Tempfile.create('attachment_download_test.txt') } - let(:tmp_stub_image) { Tempfile.create('image.jpeg') } - - context 'when importing doc attachment' do - before do - allow(Gitlab::GithubImport::AttachmentsDownloader).to receive(:new).with(doc_url) - .and_return(downloader_stub) - allow(Gitlab::GithubImport::AttachmentsDownloader).to receive(:new).with(image_url) - .and_return(downloader_stub) - allow(downloader_stub).to receive(:perform).and_return(tmp_stub_doc, tmp_stub_image) - allow(downloader_stub).to receive(:delete).twice - - allow(UploadService).to receive(:new) - .with(project, tmp_stub_doc, FileUploader).and_call_original - allow(UploadService).to receive(:new) - .with(project, tmp_stub_image, FileUploader).and_call_original - end - - it 'updates release description with new attachment url' do - importer.execute - - release.reload - expect(release.description).to start_with("Some text...\n\n [special-doc](/uploads/") - expect(release.description).to include(' - end - end - end -end diff --git a/spec/lib/gitlab/github_import/importer/releases_attachments_importer_spec.rb b/spec/lib/gitlab/github_import/importer/releases_attachments_importer_spec.rb deleted file mode 100644 index 1aeb3462cd5..00000000000 --- a/spec/lib/gitlab/github_import/importer/releases_attachments_importer_spec.rb +++ /dev/null @@ -1,74 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::GithubImport::Importer::ReleasesAttachmentsImporter do - subject { described_class.new(project, client) } - - let_it_be(:project) { create(:project) } - - let(:client) { instance_double(Gitlab::GithubImport::Client) } - - describe '#each_object_to_import', :clean_gitlab_redis_cache do - let!(:release_1) { create(:release, project: project) } - let!(:release_2) { create(:release, project: project) } - - it 'iterates each project release' do - list = [] - subject.each_object_to_import do |object| - list << object - end - expect(list).to contain_exactly(release_1, release_2) - end - - context 'when release is already processed' do - it "doesn't process this release" do - subject.mark_as_imported(release_1) - - list = [] - subject.each_object_to_import do |object| - list << object - end - expect(list).to contain_exactly(release_2) - end - end - end - - describe '#representation_class' do - it { expect(subject.representation_class).to eq(Gitlab::GithubImport::Representation::ReleaseAttachments) } - end - - describe '#importer_class' do - it { expect(subject.importer_class).to eq(Gitlab::GithubImport::Importer::ReleaseAttachmentsImporter) } - end - - describe '#sidekiq_worker_class' do - it { expect(subject.sidekiq_worker_class).to eq(Gitlab::GithubImport::ImportReleaseAttachmentsWorker) } - end - - describe '#collection_method' do - it { expect(subject.collection_method).to eq(:release_attachments) } - end - - describe '#object_type' do - it { expect(subject.object_type).to eq(:release_attachment) } - end - - describe '#id_for_already_imported_cache' do - let(:release) { build_stubbed(:release) } - - it { expect(subject.id_for_already_imported_cache(release)).to eq(release.id) } - end - - describe '#object_representation' do - let(:release) { build_stubbed(:release) } - - it 'returns release attachments representation' do - representation = subject.object_representation(release) - - expect(representation.class).to eq subject.representation_class - expect(representation.release_db_id).to eq release.id - expect(representation.description).to eq release.description - end - end -end diff --git a/spec/lib/gitlab/github_import/importer/releases_importer_spec.rb b/spec/lib/gitlab/github_import/importer/releases_importer_spec.rb index b0f553dbef7..84d639a09ef 100644 --- a/spec/lib/gitlab/github_import/importer/releases_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/releases_importer_spec.rb @@ -10,22 +10,21 @@ RSpec.describe Gitlab::GithubImport::Importer::ReleasesImporter do let(:created_at) { Time.new(2017, 1, 1, 12, 00) } let(:released_at) { Time.new(2017, 1, 1, 12, 00) } let(:author) do - double( + { login: 'User A', id: 1 - ) + } end let(:github_release) do - double( - :github_release, + { tag_name: '1.0', name: github_release_name, body: 'This is my release', created_at: created_at, published_at: released_at, author: author - ) + } end def stub_email_for_github_username(user_name = 'User A', user_email = 'user@example.com') @@ -56,7 +55,7 @@ RSpec.describe Gitlab::GithubImport::Importer::ReleasesImporter do end it 'imports draft releases' do - release_double = double( + release_double = { name: 'Test', body: 'This is description', tag_name: '1.0', @@ -65,7 +64,7 @@ RSpec.describe Gitlab::GithubImport::Importer::ReleasesImporter do updated_at: created_at, published_at: nil, author: author - ) + } expect(importer).to receive(:each_release).and_return([release_double]) @@ -101,7 +100,7 @@ RSpec.describe Gitlab::GithubImport::Importer::ReleasesImporter do end it 'uses a default release description if none is provided' do - expect(github_release).to receive(:body).and_return('') + github_release[:body] = nil expect(importer).to receive(:each_release).and_return([github_release]) release = importer.build_releases.first @@ -110,10 +109,10 @@ RSpec.describe Gitlab::GithubImport::Importer::ReleasesImporter do end it 'does not create releases that have a NULL tag' do - null_tag_release = double( + null_tag_release = { name: 'NULL Test', tag_name: nil - ) + } expect(importer).to receive(:each_release).and_return([null_tag_release]) expect(importer.build_releases).to be_empty @@ -179,13 +178,13 @@ RSpec.describe Gitlab::GithubImport::Importer::ReleasesImporter do end it 'returns ghost user when author is empty in Github release' do - allow(github_release).to receive(:author).and_return(nil) + github_release[:author] = nil expect(release_hash[:author_id]).to eq(Gitlab::GithubImport.ghost_user_id) end context 'when Github author is not found in Gitlab' do - let(:author) { double(login: 'octocat', id: 1 ) } + let(:author) { { login: 'octocat', id: 1 } } before do # Stub user email which does not match a Gitlab user. @@ -222,11 +221,11 @@ RSpec.describe Gitlab::GithubImport::Importer::ReleasesImporter do describe '#description_for' do it 'returns the description when present' do - expect(importer.description_for(github_release)).to eq(github_release.body) + expect(importer.description_for(github_release)).to eq(github_release[:body]) end it 'returns a generated description when one is not present' do - allow(github_release).to receive(:body).and_return('') + github_release[:body] = nil expect(importer.description_for(github_release)).to eq('Release for tag 1.0') end diff --git a/spec/lib/gitlab/github_import/importer/single_endpoint_diff_notes_importer_spec.rb b/spec/lib/gitlab/github_import/importer/single_endpoint_diff_notes_importer_spec.rb index 471302cb31b..081d08edfb3 100644 --- a/spec/lib/gitlab/github_import/importer/single_endpoint_diff_notes_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/single_endpoint_diff_notes_importer_spec.rb @@ -14,7 +14,7 @@ RSpec.describe Gitlab::GithubImport::Importer::SingleEndpointDiffNotesImporter d it { expect(subject.importer_class).to eq(Gitlab::GithubImport::Importer::DiffNoteImporter) } it { expect(subject.collection_method).to eq(:pull_request_comments) } it { expect(subject.object_type).to eq(:diff_note) } - it { expect(subject.id_for_already_imported_cache(double(id: 1))).to eq(1) } + it { expect(subject.id_for_already_imported_cache({ id: 1 })).to eq(1) } describe '#each_object_to_import', :clean_gitlab_redis_cache do let(:merge_request) do @@ -26,7 +26,7 @@ RSpec.describe Gitlab::GithubImport::Importer::SingleEndpointDiffNotesImporter d ) end - let(:note) { double(id: 1) } + let(:note) { { id: 1 } } let(:page) { double(objects: [note], number: 1) } it 'fetches data' do diff --git a/spec/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer_spec.rb b/spec/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer_spec.rb index 4ed01fd7e0b..dde730d46d2 100644 --- a/spec/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer_spec.rb @@ -40,7 +40,7 @@ RSpec.describe Gitlab::GithubImport::Importer::SingleEndpointIssueEventsImporter end describe '#id_for_already_imported_cache' do - let(:event) { instance_double('Event', id: 1) } + let(:event) { { id: 1 } } it { expect(subject.id_for_already_imported_cache(event)).to eq(1) } end @@ -88,7 +88,7 @@ RSpec.describe Gitlab::GithubImport::Importer::SingleEndpointIssueEventsImporter describe '#each_object_to_import', :clean_gitlab_redis_cache do let(:issue_event) do struct = Struct.new(:id, :event, :created_at, :issue, keyword_init: true) - struct.new(id: rand(10), event: 'closed', created_at: '2022-04-26 18:30:53 UTC') + struct.new(id: 1, event: 'closed', created_at: '2022-04-26 18:30:53 UTC') end let(:page) do @@ -115,9 +115,17 @@ RSpec.describe Gitlab::GithubImport::Importer::SingleEndpointIssueEventsImporter it 'imports each issue event page by page' do counter = 0 subject.each_object_to_import do |object| - expect(object).to eq issue_event - expect(issue_event.issue['number']).to eq issuable.iid - expect(issue_event.issue['pull_request']).to eq false + expect(object).to eq( + { + id: 1, + event: 'closed', + created_at: '2022-04-26 18:30:53 UTC', + issue: { + number: issuable.iid, + pull_request: false + } + } + ) counter += 1 end expect(counter).to eq 1 @@ -130,9 +138,17 @@ RSpec.describe Gitlab::GithubImport::Importer::SingleEndpointIssueEventsImporter it 'imports each merge request event page by page' do counter = 0 subject.each_object_to_import do |object| - expect(object).to eq issue_event - expect(issue_event.issue['number']).to eq issuable.iid - expect(issue_event.issue['pull_request']).to eq true + expect(object).to eq( + { + id: 1, + event: 'closed', + created_at: '2022-04-26 18:30:53 UTC', + issue: { + number: issuable.iid, + pull_request: true + } + } + ) counter += 1 end expect(counter).to eq 1 diff --git a/spec/lib/gitlab/github_import/importer/single_endpoint_issue_notes_importer_spec.rb b/spec/lib/gitlab/github_import/importer/single_endpoint_issue_notes_importer_spec.rb index d769f4fdcf5..e1f65546e1d 100644 --- a/spec/lib/gitlab/github_import/importer/single_endpoint_issue_notes_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/single_endpoint_issue_notes_importer_spec.rb @@ -14,7 +14,7 @@ RSpec.describe Gitlab::GithubImport::Importer::SingleEndpointIssueNotesImporter it { expect(subject.importer_class).to eq(Gitlab::GithubImport::Importer::NoteImporter) } it { expect(subject.collection_method).to eq(:issue_comments) } it { expect(subject.object_type).to eq(:note) } - it { expect(subject.id_for_already_imported_cache(double(id: 1))).to eq(1) } + it { expect(subject.id_for_already_imported_cache({ id: 1 })).to eq(1) } describe '#each_object_to_import', :clean_gitlab_redis_cache do let(:issue) do @@ -25,7 +25,7 @@ RSpec.describe Gitlab::GithubImport::Importer::SingleEndpointIssueNotesImporter ) end - let(:note) { double(id: 1) } + let(:note) { { id: 1 } } let(:page) { double(objects: [note], number: 1) } it 'fetches data' do diff --git a/spec/lib/gitlab/github_import/importer/single_endpoint_merge_request_notes_importer_spec.rb b/spec/lib/gitlab/github_import/importer/single_endpoint_merge_request_notes_importer_spec.rb index 1dcc466d34c..5523b97acc3 100644 --- a/spec/lib/gitlab/github_import/importer/single_endpoint_merge_request_notes_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/single_endpoint_merge_request_notes_importer_spec.rb @@ -14,7 +14,7 @@ RSpec.describe Gitlab::GithubImport::Importer::SingleEndpointMergeRequestNotesIm it { expect(subject.importer_class).to eq(Gitlab::GithubImport::Importer::NoteImporter) } it { expect(subject.collection_method).to eq(:issue_comments) } it { expect(subject.object_type).to eq(:note) } - it { expect(subject.id_for_already_imported_cache(double(id: 1))).to eq(1) } + it { expect(subject.id_for_already_imported_cache({ id: 1 })).to eq(1) } describe '#each_object_to_import', :clean_gitlab_redis_cache do let(:merge_request) do @@ -26,7 +26,7 @@ RSpec.describe Gitlab::GithubImport::Importer::SingleEndpointMergeRequestNotesIm ) end - let(:note) { double(id: 1) } + let(:note) { { id: 1 } } let(:page) { double(objects: [note], number: 1) } it 'fetches data' do diff --git a/spec/lib/gitlab/github_import/issuable_finder_spec.rb b/spec/lib/gitlab/github_import/issuable_finder_spec.rb index d550f15e8c5..d3236994cef 100644 --- a/spec/lib/gitlab/github_import/issuable_finder_spec.rb +++ b/spec/lib/gitlab/github_import/issuable_finder_spec.rb @@ -3,11 +3,20 @@ require 'spec_helper' RSpec.describe Gitlab::GithubImport::IssuableFinder, :clean_gitlab_redis_cache do - let(:project) { double(:project, id: 4, group: nil) } - let(:issue) do - double(:issue, issuable_type: MergeRequest, issuable_id: 1) + let(:project) { double(:project, id: 4, import_data: import_data) } + let(:single_endpoint_optional_stage) { false } + let(:import_data) do + instance_double( + ProjectImportData, + data: { + optional_stages: { + single_endpoint_notes_import: single_endpoint_optional_stage + } + }.deep_stringify_keys + ) end + let(:issue) { double(:issue, issuable_type: MergeRequest, issuable_id: 1) } let(:finder) { described_class.new(project, issue) } describe '#database_id' do @@ -28,13 +37,10 @@ RSpec.describe Gitlab::GithubImport::IssuableFinder, :clean_gitlab_redis_cache d end context 'when group is present' do - context 'when github_importer_single_endpoint_notes_import feature flag is enabled' do - it 'reads cache value with longer timeout' do - project = create(:project, import_url: 'http://t0ken@github.com/user/repo.git') - group = create(:group, projects: [project]) - - stub_feature_flags(github_importer_single_endpoint_notes_import: group) + context 'when settings single_endpoint_notes_import is enabled' do + let(:single_endpoint_optional_stage) { true } + it 'reads cache value with longer timeout' do expect(Gitlab::Cache::Import::Caching) .to receive(:read) .with(anything, timeout: Gitlab::Cache::Import::Caching::LONGER_TIMEOUT) @@ -43,12 +49,8 @@ RSpec.describe Gitlab::GithubImport::IssuableFinder, :clean_gitlab_redis_cache d end end - context 'when github_importer_single_endpoint_notes_import feature flag is disabled' do + context 'when settings single_endpoint_notes_import is disabled' do it 'reads cache value with default timeout' do - project = double(:project, id: 4, group: create(:group)) - - stub_feature_flags(github_importer_single_endpoint_notes_import: false) - expect(Gitlab::Cache::Import::Caching) .to receive(:read) .with(anything, timeout: Gitlab::Cache::Import::Caching::TIMEOUT) @@ -68,34 +70,25 @@ RSpec.describe Gitlab::GithubImport::IssuableFinder, :clean_gitlab_redis_cache d finder.cache_database_id(10) end - context 'when group is present' do - context 'when github_importer_single_endpoint_notes_import feature flag is enabled' do - it 'caches value with longer timeout' do - project = create(:project, import_url: 'http://t0ken@github.com/user/repo.git') - group = create(:group, projects: [project]) - - stub_feature_flags(github_importer_single_endpoint_notes_import: group) + context 'when settings single_endpoint_notes_import is enabled' do + let(:single_endpoint_optional_stage) { true } - expect(Gitlab::Cache::Import::Caching) - .to receive(:write) - .with(anything, anything, timeout: Gitlab::Cache::Import::Caching::LONGER_TIMEOUT) + it 'caches value with longer timeout' do + expect(Gitlab::Cache::Import::Caching) + .to receive(:write) + .with(anything, anything, timeout: Gitlab::Cache::Import::Caching::LONGER_TIMEOUT) - described_class.new(project, issue).cache_database_id(10) - end + described_class.new(project, issue).cache_database_id(10) end + end - context 'when github_importer_single_endpoint_notes_import feature flag is disabled' do - it 'caches value with default timeout' do - project = double(:project, id: 4, group: create(:group)) - - stub_feature_flags(github_importer_single_endpoint_notes_import: false) - - expect(Gitlab::Cache::Import::Caching) - .to receive(:write) - .with(anything, anything, timeout: Gitlab::Cache::Import::Caching::TIMEOUT) + context 'when settings single_endpoint_notes_import is disabled' do + it 'caches value with default timeout' do + expect(Gitlab::Cache::Import::Caching) + .to receive(:write) + .with(anything, anything, timeout: Gitlab::Cache::Import::Caching::TIMEOUT) - described_class.new(project, issue).cache_database_id(10) - end + described_class.new(project, issue).cache_database_id(10) end end end diff --git a/spec/lib/gitlab/github_import/markdown/attachment_spec.rb b/spec/lib/gitlab/github_import/markdown/attachment_spec.rb new file mode 100644 index 00000000000..5d29de34141 --- /dev/null +++ b/spec/lib/gitlab/github_import/markdown/attachment_spec.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Markdown::Attachment do + let(:name) { FFaker::Lorem.word } + let(:url) { FFaker::Internet.uri('https') } + + describe '.from_markdown' do + context "when it's a doc attachment" do + let(:doc_extension) { Gitlab::GithubImport::Markdown::Attachment::DOC_TYPES.sample } + let(:url) { "https://github.com/nickname/public-test-repo/files/3/git-cheat-sheet.#{doc_extension}" } + let(:name) { FFaker::Lorem.word } + let(:markdown_node) do + instance_double('CommonMarker::Node', url: url, to_plaintext: name, type: :link) + end + + it 'returns instance with attachment info' do + attachment = described_class.from_markdown(markdown_node) + + expect(attachment.name).to eq name + expect(attachment.url).to eq url + end + + context "when type is not in whitelist" do + let(:doc_extension) { 'exe' } + + it { expect(described_class.from_markdown(markdown_node)).to eq nil } + end + + context 'when domain name is unknown' do + let(:url) do + "https://bitbucket.com/nickname/public-test-repo/files/3/git-cheat-sheet.#{doc_extension}" + end + + it { expect(described_class.from_markdown(markdown_node)).to eq nil } + end + end + + context "when it's an image attachment" do + let(:image_extension) { Gitlab::GithubImport::Markdown::Attachment::MEDIA_TYPES.sample } + let(:url) { "https://user-images.githubusercontent.com/1/uuid-1.#{image_extension}" } + let(:name) { FFaker::Lorem.word } + let(:markdown_node) do + instance_double('CommonMarker::Node', url: url, to_plaintext: name, type: :image) + end + + it 'returns instance with attachment info' do + attachment = described_class.from_markdown(markdown_node) + + expect(attachment.name).to eq name + expect(attachment.url).to eq url + end + + context "when type is not in whitelist" do + let(:image_extension) { 'mkv' } + + it { expect(described_class.from_markdown(markdown_node)).to eq nil } + end + + context 'when domain name is unknown' do + let(:url) { "https://user-images.github.com/1/uuid-1.#{image_extension}" } + + it { expect(described_class.from_markdown(markdown_node)).to eq nil } + end + end + + context "when it's an inline html node" do + let(:name) { FFaker::Lorem.word } + let(:image_extension) { Gitlab::GithubImport::Markdown::Attachment::MEDIA_TYPES.sample } + let(:url) { "https://user-images.githubusercontent.com/1/uuid-1.#{image_extension}" } + let(:img) { "<img width=\"248\" alt=\"#{name}\" src=\"#{url}\">" } + let(:markdown_node) do + instance_double('CommonMarker::Node', string_content: img, type: :inline_html) + end + + it 'returns instance with attachment info' do + attachment = described_class.from_markdown(markdown_node) + + expect(attachment.name).to eq name + expect(attachment.url).to eq url + end + end + end + + describe '#inspect' do + it 'returns attachment basic info' do + attachment = described_class.new(name, url) + + expect(attachment.inspect).to eq "<Gitlab::GithubImport::Markdown::Attachment: { name: #{name}, url: #{url} }>" + end + end +end diff --git a/spec/lib/gitlab/github_import/markdown_text_spec.rb b/spec/lib/gitlab/github_import/markdown_text_spec.rb index 1da6bb06403..3f771970588 100644 --- a/spec/lib/gitlab/github_import/markdown_text_spec.rb +++ b/spec/lib/gitlab/github_import/markdown_text_spec.rb @@ -60,31 +60,48 @@ RSpec.describe Gitlab::GithubImport::MarkdownText do end end - describe '.fetch_attachment_urls' do - let(:image_extension) { described_class::MEDIA_TYPES.sample } + describe '.fetch_attachments' do + let(:image_extension) { Gitlab::GithubImport::Markdown::Attachment::MEDIA_TYPES.sample } let(:image_attachment) do - "![special-image](https://user-images.githubusercontent.com/6833862/"\ - "176685788-e7a93168-7ded-406a-82b5-eb1c56685a93.#{image_extension})" + "" end - let(:doc_extension) { described_class::DOC_TYPES.sample } + let(:img_tag_attachment) do + "<img width=\"248\" alt=\"tag-image\" src=\"https://user-images.githubusercontent.com/2/"\ + "uuid-2.#{image_extension}\">" + end + + let(:damaged_img_tag) do + "<img width=\"248\" alt=\"tag-image\" src=\"https://user-images.githubusercontent.com" + end + + let(:doc_extension) { Gitlab::GithubImport::Markdown::Attachment::DOC_TYPES.sample } let(:doc_attachment) do "[some-doc](https://github.com/nickname/public-test-repo/"\ - "files/9020437/git-cheat-sheet.#{doc_extension})" + "files/3/git-cheat-sheet.#{doc_extension})" end let(:text) do - <<-TEXT + <<-TEXT.split("\n").map(&:strip).join("\n") Comment with an attachment #{image_attachment} #{FFaker::Lorem.sentence} #{doc_attachment} + #{damaged_img_tag} + #{FFaker::Lorem.paragraph} + #{img_tag_attachment} TEXT end - it 'fetches attachment urls' do - expect(described_class.fetch_attachment_urls(text)) - .to contain_exactly(image_attachment, doc_attachment) + it 'fetches attachments' do + attachments = described_class.fetch_attachments(text) + + expect(attachments.map(&:name)).to contain_exactly('special-image', 'tag-image', 'some-doc') + expect(attachments.map(&:url)).to contain_exactly( + "https://user-images.githubusercontent.com/1/uuid-1.#{image_extension}", + "https://user-images.githubusercontent.com/2/uuid-2.#{image_extension}", + "https://github.com/nickname/public-test-repo/files/3/git-cheat-sheet.#{doc_extension}" + ) end end diff --git a/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb b/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb index 860bb60f3ed..cefad3baa31 100644 --- a/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb +++ b/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb @@ -295,11 +295,12 @@ RSpec.describe Gitlab::GithubImport::ParallelScheduling do end it 'imports data in parallel batches with delays' do - expect(worker_class).to receive(:bulk_perform_in).with(1.second, [ - [project.id, { title: 'Foo' }, an_instance_of(String)], - [project.id, { title: 'Foo' }, an_instance_of(String)], - [project.id, { title: 'Foo' }, an_instance_of(String)] - ], batch_size: batch_size, batch_delay: batch_delay) + expect(worker_class).to receive(:bulk_perform_in) + .with(1.second, [ + [project.id, { title: 'Foo' }, an_instance_of(String)], + [project.id, { title: 'Foo' }, an_instance_of(String)], + [project.id, { title: 'Foo' }, an_instance_of(String)] + ], batch_size: batch_size, batch_delay: batch_delay) importer.parallel_import end @@ -308,7 +309,8 @@ RSpec.describe Gitlab::GithubImport::ParallelScheduling do describe '#each_object_to_import' do let(:importer) { importer_class.new(project, client) } - let(:object) { double(:object) } + let(:object) { {} } + let(:object_counter_class) { Gitlab::GithubImport::ObjectCounter } before do expect(importer) @@ -334,6 +336,9 @@ RSpec.describe Gitlab::GithubImport::ParallelScheduling do .with(object) .and_return(false) + expect(object_counter_class) + .to receive(:increment) + expect(importer) .to receive(:mark_as_imported) .with(object) @@ -364,6 +369,9 @@ RSpec.describe Gitlab::GithubImport::ParallelScheduling do .with(object) .and_return(false) + expect(object_counter_class) + .to receive(:increment) + expect(importer) .to receive(:mark_as_imported) .with(object) @@ -407,6 +415,9 @@ RSpec.describe Gitlab::GithubImport::ParallelScheduling do .with(object) .and_return(true) + expect(object_counter_class) + .not_to receive(:increment) + expect(importer) .not_to receive(:mark_as_imported) @@ -463,4 +474,13 @@ RSpec.describe Gitlab::GithubImport::ParallelScheduling do importer.mark_as_imported(object) end end + + describe '#increment_object_counter?' do + let(:github_issue) { {} } + let(:importer) { importer_class.new(project, client) } + + it 'returns true' do + expect(importer).to be_increment_object_counter(github_issue) + end + end end diff --git a/spec/lib/gitlab/github_import/representation/diff_note_spec.rb b/spec/lib/gitlab/github_import/representation/diff_note_spec.rb index fe3040c102b..a656cd0d056 100644 --- a/spec/lib/gitlab/github_import/representation/diff_note_spec.rb +++ b/spec/lib/gitlab/github_import/representation/diff_note_spec.rb @@ -28,7 +28,7 @@ RSpec.describe Gitlab::GithubImport::Representation::DiffNote, :clean_gitlab_red let(:start_line) { nil } let(:end_line) { 23 } let(:note_body) { 'Hello world' } - let(:user_data) { { 'id' => 4, 'login' => 'alice' } } + let(:user_data) { { id: 4, login: 'alice' } } let(:side) { 'RIGHT' } let(:created_at) { Time.new(2017, 1, 1, 12, 00) } let(:updated_at) { Time.new(2017, 1, 1, 12, 15) } @@ -275,15 +275,14 @@ RSpec.describe Gitlab::GithubImport::Representation::DiffNote, :clean_gitlab_red describe '.from_api_response' do it_behaves_like 'a DiffNote representation' do let(:response) do - double( - :response, + { id: note_id, html_url: 'https://github.com/foo/bar/pull/42', path: 'README.md', commit_id: '123abc', original_commit_id: 'original123abc', side: side, - user: user_data && double(:user, user_data), + user: user_data, diff_hunk: hunk, body: note_body, created_at: created_at, @@ -291,7 +290,7 @@ RSpec.describe Gitlab::GithubImport::Representation::DiffNote, :clean_gitlab_red line: end_line, start_line: start_line, in_reply_to_id: in_reply_to_id - ) + } end subject(:note) { described_class.from_api_response(response) } diff --git a/spec/lib/gitlab/github_import/representation/issue_event_spec.rb b/spec/lib/gitlab/github_import/representation/issue_event_spec.rb index 0256858ecf1..0dd281cb3b0 100644 --- a/spec/lib/gitlab/github_import/representation/issue_event_spec.rb +++ b/spec/lib/gitlab/github_import/representation/issue_event_spec.rb @@ -186,7 +186,7 @@ RSpec.describe Gitlab::GithubImport::Representation::IssueEvent do assignee: with_assignee ? user_resource.new(id: 5, login: 'tom') : nil, requested_reviewer: with_reviewer ? user_resource.new(id: 6, login: 'mickey') : nil, review_requester: with_reviewer ? user_resource.new(id: 7, login: 'minnie') : nil, - issue: { 'number' => 2, 'pull_request' => pull_request }, + issue: { number: 2, pull_request: pull_request }, created_at: '2022-04-26 18:30:53 UTC', performed_via_github_app: nil ) diff --git a/spec/lib/gitlab/github_import/representation/issue_spec.rb b/spec/lib/gitlab/github_import/representation/issue_spec.rb index 5898518343a..263ef8b1708 100644 --- a/spec/lib/gitlab/github_import/representation/issue_spec.rb +++ b/spec/lib/gitlab/github_import/representation/issue_spec.rb @@ -74,20 +74,19 @@ RSpec.describe Gitlab::GithubImport::Representation::Issue do describe '.from_api_response' do let(:response) do - double( - :response, + { number: 42, title: 'My Issue', body: 'This is my issue', - milestone: double(:milestone, number: 4), + milestone: { number: 4 }, state: 'open', - assignees: [double(:user, id: 4, login: 'alice')], - labels: [double(:label, name: 'bug')], - user: double(:user, id: 4, login: 'alice'), + assignees: [{ id: 4, login: 'alice' }], + labels: [{ name: 'bug' }], + user: { id: 4, login: 'alice' }, created_at: created_at, updated_at: updated_at, pull_request: false - ) + } end let(:additional_data) { { work_item_type_id: work_item_type_id } } @@ -97,9 +96,7 @@ RSpec.describe Gitlab::GithubImport::Representation::Issue do end it 'does not set the user if the response did not include a user' do - allow(response) - .to receive(:user) - .and_return(nil) + response[:user] = nil issue = described_class.from_api_response(response, additional_data) diff --git a/spec/lib/gitlab/github_import/representation/note_spec.rb b/spec/lib/gitlab/github_import/representation/note_spec.rb index 9f416eb3c02..49126dbe9c5 100644 --- a/spec/lib/gitlab/github_import/representation/note_spec.rb +++ b/spec/lib/gitlab/github_import/representation/note_spec.rb @@ -48,15 +48,14 @@ RSpec.describe Gitlab::GithubImport::Representation::Note do describe '.from_api_response' do let(:response) do - double( - :response, + { html_url: 'https://github.com/foo/bar/issues/42', - user: double(:user, id: 4, login: 'alice'), + user: { id: 4, login: 'alice' }, body: 'Hello world', created_at: created_at, updated_at: updated_at, id: 1 - ) + } end it_behaves_like 'a Note' do @@ -64,9 +63,7 @@ RSpec.describe Gitlab::GithubImport::Representation::Note do end it 'does not set the user if the response did not include a user' do - allow(response) - .to receive(:user) - .and_return(nil) + response[:user] = nil note = described_class.from_api_response(response) diff --git a/spec/lib/gitlab/github_import/representation/note_text_spec.rb b/spec/lib/gitlab/github_import/representation/note_text_spec.rb new file mode 100644 index 00000000000..8b57c9a0373 --- /dev/null +++ b/spec/lib/gitlab/github_import/representation/note_text_spec.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Representation::NoteText do + shared_examples 'a Note text data' do |match_record_type| + it 'returns an instance of NoteText' do + expect(representation).to be_an_instance_of(described_class) + end + + it 'includes record DB id' do + expect(representation.record_db_id).to eq 42 + end + + it 'includes record type' do + expect(representation.record_type).to eq match_record_type + end + + it 'includes note text' do + expect(representation.text).to eq 'Some text here..' + end + end + + describe '.from_db_record' do + context 'with Release' do + let(:record) { build_stubbed(:release, id: 42, description: 'Some text here..') } + + it_behaves_like 'a Note text data', 'Release' do + let(:representation) { described_class.from_db_record(record) } + end + end + + context 'with Issue' do + let(:record) { build_stubbed(:issue, id: 42, description: 'Some text here..') } + + it_behaves_like 'a Note text data', 'Issue' do + let(:representation) { described_class.from_db_record(record) } + end + end + + context 'with MergeRequest' do + let(:record) { build_stubbed(:merge_request, id: 42, description: 'Some text here..') } + + it_behaves_like 'a Note text data', 'MergeRequest' do + let(:representation) { described_class.from_db_record(record) } + end + end + + context 'with Note' do + let(:record) { build_stubbed(:note, id: 42, note: 'Some text here..') } + + it_behaves_like 'a Note text data', 'Note' do + let(:representation) { described_class.from_db_record(record) } + end + end + end + + describe '.from_json_hash' do + it_behaves_like 'a Note text data', 'Release' do + let(:hash) do + { + 'record_db_id' => 42, + 'record_type' => 'Release', + 'text' => 'Some text here..' + } + end + + let(:representation) { described_class.from_json_hash(hash) } + end + end + + describe '#github_identifiers' do + it 'returns a hash with needed identifiers' do + record_id = rand(100) + representation = described_class.new(record_db_id: record_id, text: 'text') + + expect(representation.github_identifiers).to eq({ db_id: record_id }) + end + end +end diff --git a/spec/lib/gitlab/github_import/representation/protected_branch_spec.rb b/spec/lib/gitlab/github_import/representation/protected_branch_spec.rb index e762dc469c1..30b29659eee 100644 --- a/spec/lib/gitlab/github_import/representation/protected_branch_spec.rb +++ b/spec/lib/gitlab/github_import/representation/protected_branch_spec.rb @@ -9,24 +9,47 @@ RSpec.describe Gitlab::GithubImport::Representation::ProtectedBranch do end context 'with ProtectedBranch' do - it 'includes the protected branch ID (name)' do + it 'includes the protected branch ID (name) attribute' do expect(protected_branch.id).to eq 'main' end - it 'includes the protected branch allow_force_pushes' do + it 'includes the protected branch allow_force_pushes attribute' do expect(protected_branch.allow_force_pushes).to eq true end + + it 'includes the protected branch required_conversation_resolution attribute' do + expect(protected_branch.required_conversation_resolution).to eq true + end + + it 'includes the protected branch required_pull_request_reviews' do + expect(protected_branch.required_pull_request_reviews).to eq true + end end end describe '.from_api_response' do let(:response) do - response = Struct.new(:url, :allow_force_pushes, keyword_init: true) - allow_force_pushes = Struct.new(:enabled, keyword_init: true) + response = Struct.new( + :url, :allow_force_pushes, :required_conversation_resolution, :required_signatures, + :required_pull_request_reviews, + keyword_init: true + ) + enabled_setting = Struct.new(:enabled, keyword_init: true) + required_pull_request_reviews = Struct.new(:url, :dismissal_restrictions, keyword_init: true) response.new( url: 'https://example.com/branches/main/protection', - allow_force_pushes: allow_force_pushes.new( + allow_force_pushes: enabled_setting.new( + enabled: true + ), + required_conversation_resolution: enabled_setting.new( + enabled: true + ), + required_signatures: enabled_setting.new( enabled: true + ), + required_pull_request_reviews: required_pull_request_reviews.new( + url: 'https://example.com/branches/main/protection/required_pull_request_reviews', + dismissal_restrictions: {} ) ) end @@ -41,7 +64,10 @@ RSpec.describe Gitlab::GithubImport::Representation::ProtectedBranch do let(:hash) do { 'id' => 'main', - 'allow_force_pushes' => true + 'allow_force_pushes' => true, + 'required_conversation_resolution' => true, + 'required_signatures' => true, + 'required_pull_request_reviews' => true } end diff --git a/spec/lib/gitlab/github_import/representation/pull_request_review_spec.rb b/spec/lib/gitlab/github_import/representation/pull_request_review_spec.rb index d6e7a8172f7..0203da9f4fb 100644 --- a/spec/lib/gitlab/github_import/representation/pull_request_review_spec.rb +++ b/spec/lib/gitlab/github_import/representation/pull_request_review_spec.rb @@ -21,15 +21,14 @@ RSpec.describe Gitlab::GithubImport::Representation::PullRequestReview do describe '.from_api_response' do let(:response) do - double( - :response, + { id: 999, merge_request_id: 42, body: 'note', state: 'APPROVED', - user: double(:user, id: 4, login: 'alice'), + user: { id: 4, login: 'alice' }, submitted_at: submitted_at - ) + } end it_behaves_like 'a PullRequest review' do @@ -37,9 +36,7 @@ RSpec.describe Gitlab::GithubImport::Representation::PullRequestReview do end it 'does not set the user if the response did not include a user' do - allow(response) - .to receive(:user) - .and_return(nil) + response[:user] = nil review = described_class.from_api_response(response) diff --git a/spec/lib/gitlab/github_import/representation/pull_request_spec.rb b/spec/lib/gitlab/github_import/representation/pull_request_spec.rb index deb9535a845..b8c1c67e07c 100644 --- a/spec/lib/gitlab/github_import/representation/pull_request_spec.rb +++ b/spec/lib/gitlab/github_import/representation/pull_request_spec.rb @@ -93,33 +93,30 @@ RSpec.describe Gitlab::GithubImport::Representation::PullRequest do describe '.from_api_response' do let(:response) do - double( - :response, + { number: 42, title: 'My Pull Request', body: 'This is my pull request', state: 'closed', - head: double( - :head, + head: { sha: '123abc', ref: 'my-feature', - repo: double(:repo, id: 400), - user: double(:user, id: 4, login: 'alice') - ), - base: double( - :base, + repo: { id: 400 }, + user: { id: 4, login: 'alice' } + }, + base: { sha: '456def', ref: 'master', - repo: double(:repo, id: 200) - ), - milestone: double(:milestone, number: 4), - user: double(:user, id: 4, login: 'alice'), - assignee: double(:user, id: 4, login: 'alice'), - merged_by: double(:user, id: 4, login: 'alice'), + repo: { id: 200 } + }, + milestone: { number: 4 }, + user: { id: 4, login: 'alice' }, + assignee: { id: 4, login: 'alice' }, + merged_by: { id: 4, login: 'alice' }, created_at: created_at, updated_at: updated_at, merged_at: merged_at - ) + } end it_behaves_like 'a PullRequest' do @@ -127,9 +124,7 @@ RSpec.describe Gitlab::GithubImport::Representation::PullRequest do end it 'does not set the user if the response did not include a user' do - allow(response) - .to receive(:user) - .and_return(nil) + response[:user] = nil pr = described_class.from_api_response(response) diff --git a/spec/lib/gitlab/github_import/representation/release_attachments_spec.rb b/spec/lib/gitlab/github_import/representation/release_attachments_spec.rb deleted file mode 100644 index 0ef9dad6a13..00000000000 --- a/spec/lib/gitlab/github_import/representation/release_attachments_spec.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::GithubImport::Representation::ReleaseAttachments do - shared_examples 'a Release attachments data' do - it 'returns an instance of ReleaseAttachments' do - expect(representation).to be_an_instance_of(described_class) - end - - it 'includes release DB id' do - expect(representation.release_db_id).to eq 42 - end - - it 'includes release description' do - expect(representation.description).to eq 'Some text here..' - end - end - - describe '.from_db_record' do - let(:release) { build_stubbed(:release, id: 42, description: 'Some text here..') } - - it_behaves_like 'a Release attachments data' do - let(:representation) { described_class.from_db_record(release) } - end - end - - describe '.from_json_hash' do - it_behaves_like 'a Release attachments data' do - let(:hash) do - { - 'release_db_id' => 42, - 'description' => 'Some text here..' - } - end - - let(:representation) { described_class.from_json_hash(hash) } - end - end - - describe '#github_identifiers' do - it 'returns a hash with needed identifiers' do - release_id = rand(100) - representation = described_class.new(release_db_id: release_id, description: 'text') - - expect(representation.github_identifiers).to eq({ db_id: release_id }) - end - end -end diff --git a/spec/lib/gitlab/github_import/representation/user_spec.rb b/spec/lib/gitlab/github_import/representation/user_spec.rb index d7219556ada..ccada558f8b 100644 --- a/spec/lib/gitlab/github_import/representation/user_spec.rb +++ b/spec/lib/gitlab/github_import/representation/user_spec.rb @@ -21,7 +21,7 @@ RSpec.describe Gitlab::GithubImport::Representation::User do describe '.from_api_response' do it_behaves_like 'a User' do - let(:response) { double(:response, id: 42, login: 'alice') } + let(:response) { { id: 42, login: 'alice' } } let(:user) { described_class.from_api_response(response) } end end diff --git a/spec/lib/gitlab/github_import/settings_spec.rb b/spec/lib/gitlab/github_import/settings_spec.rb new file mode 100644 index 00000000000..ad0c47e8e8a --- /dev/null +++ b/spec/lib/gitlab/github_import/settings_spec.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Settings do + subject(:settings) { described_class.new(project) } + + let_it_be(:project) { create(:project) } + + let(:optional_stages) do + { + single_endpoint_issue_events_import: true, + single_endpoint_notes_import: false, + attachments_import: false + } + end + + describe '.stages_array' do + let(:expected_list) do + stages = described_class::OPTIONAL_STAGES + [ + { + name: 'single_endpoint_issue_events_import', + label: stages[:single_endpoint_issue_events_import][:label], + details: stages[:single_endpoint_issue_events_import][:details] + }, + { + name: 'single_endpoint_notes_import', + label: stages[:single_endpoint_notes_import][:label], + details: stages[:single_endpoint_notes_import][:details] + }, + { + name: 'attachments_import', + label: stages[:attachments_import][:label].strip, + details: stages[:attachments_import][:details] + } + ] + end + + it 'returns stages list as array' do + expect(described_class.stages_array).to match_array(expected_list) + end + end + + describe '#write' do + let(:data_input) do + { + single_endpoint_issue_events_import: true, + single_endpoint_notes_import: 'false', + attachments_import: nil, + foo: :bar + }.stringify_keys + end + + it 'puts optional steps flags into projects import_data' do + settings.write(data_input) + + expect(project.import_data.data['optional_stages']) + .to eq optional_stages.stringify_keys + end + end + + describe '#enabled?' do + it 'returns is enabled or not specific optional stage' do + project.create_or_update_import_data(data: { optional_stages: optional_stages }) + + expect(settings.enabled?(:single_endpoint_issue_events_import)).to eq true + expect(settings.enabled?(:single_endpoint_notes_import)).to eq false + expect(settings.enabled?(:attachments_import)).to eq false + end + end + + describe '#disabled?' do + it 'returns is disabled or not specific optional stage' do + project.create_or_update_import_data(data: { optional_stages: optional_stages }) + + expect(settings.disabled?(:single_endpoint_issue_events_import)).to eq false + expect(settings.disabled?(:single_endpoint_notes_import)).to eq true + expect(settings.disabled?(:attachments_import)).to eq true + end + end +end diff --git a/spec/lib/gitlab/github_import/user_finder_spec.rb b/spec/lib/gitlab/github_import/user_finder_spec.rb index 8ebbff31f64..d77aaa0e846 100644 --- a/spec/lib/gitlab/github_import/user_finder_spec.rb +++ b/spec/lib/gitlab/github_import/user_finder_spec.rb @@ -17,8 +17,8 @@ RSpec.describe Gitlab::GithubImport::UserFinder, :clean_gitlab_redis_cache do describe '#author_id_for' do context 'with default author_key' do it 'returns the user ID for the author of an object' do - user = double(:user, id: 4, login: 'kittens') - note = double(:note, author: user) + user = { id: 4, login: 'kittens' } + note = { author: user } expect(finder).to receive(:user_id_for).with(user).and_return(42) @@ -26,8 +26,8 @@ RSpec.describe Gitlab::GithubImport::UserFinder, :clean_gitlab_redis_cache do end it 'returns the ID of the project creator if no user ID could be found' do - user = double(:user, id: 4, login: 'kittens') - note = double(:note, author: user) + user = { id: 4, login: 'kittens' } + note = { author: user } expect(finder).to receive(:user_id_for).with(user).and_return(nil) @@ -35,7 +35,7 @@ RSpec.describe Gitlab::GithubImport::UserFinder, :clean_gitlab_redis_cache do end it 'returns the ID of the ghost user when the object has no user' do - note = double(:note, author: nil) + note = { author: nil } expect(finder.author_id_for(note)).to eq([User.ghost.id, true]) end @@ -46,7 +46,7 @@ RSpec.describe Gitlab::GithubImport::UserFinder, :clean_gitlab_redis_cache do end context 'with a non-default author_key' do - let(:user) { double(:user, id: 4, login: 'kittens') } + let(:user) { { id: 4, login: 'kittens' } } shared_examples 'user ID finder' do |author_key| it 'returns the user ID for an object' do @@ -57,25 +57,25 @@ RSpec.describe Gitlab::GithubImport::UserFinder, :clean_gitlab_redis_cache do end context 'when the author_key parameter is :actor' do - let(:issue_event) { double('Gitlab::GithubImport::Representation::IssueEvent', actor: user) } + let(:issue_event) { { actor: user } } it_behaves_like 'user ID finder', :actor end context 'when the author_key parameter is :assignee' do - let(:issue_event) { double('Gitlab::GithubImport::Representation::IssueEvent', assignee: user) } + let(:issue_event) { { assignee: user } } it_behaves_like 'user ID finder', :assignee end context 'when the author_key parameter is :requested_reviewer' do - let(:issue_event) { double('Gitlab::GithubImport::Representation::IssueEvent', requested_reviewer: user) } + let(:issue_event) { { requested_reviewer: user } } it_behaves_like 'user ID finder', :requested_reviewer end context 'when the author_key parameter is :review_requester' do - let(:issue_event) { double('Gitlab::GithubImport::Representation::IssueEvent', review_requester: user) } + let(:issue_event) { { review_requester: user } } it_behaves_like 'user ID finder', :review_requester end @@ -84,15 +84,15 @@ RSpec.describe Gitlab::GithubImport::UserFinder, :clean_gitlab_redis_cache do describe '#assignee_id_for' do it 'returns the user ID for the assignee of an issuable' do - user = double(:user, id: 4, login: 'kittens') - issue = double(:issue, assignee: user) + user = { id: 4, login: 'kittens' } + issue = { assignee: user } expect(finder).to receive(:user_id_for).with(user).and_return(42) expect(finder.assignee_id_for(issue)).to eq(42) end it 'returns nil if the issuable does not have an assignee' do - issue = double(:issue, assignee: nil) + issue = { assignee: nil } expect(finder).not_to receive(:user_id_for) expect(finder.assignee_id_for(issue)).to be_nil @@ -101,9 +101,9 @@ RSpec.describe Gitlab::GithubImport::UserFinder, :clean_gitlab_redis_cache do describe '#user_id_for' do it 'returns the user ID for the given user' do - user = double(:user, id: 4, login: 'kittens') + user = { id: 4, login: 'kittens' } - expect(finder).to receive(:find).with(user.id, user.login).and_return(42) + expect(finder).to receive(:find).with(user[:id], user[:login]).and_return(42) expect(finder.user_id_for(user)).to eq(42) end @@ -221,7 +221,7 @@ RSpec.describe Gitlab::GithubImport::UserFinder, :clean_gitlab_redis_cache do end context 'when an Email address is not cached' do - let(:user) { double(:user, email: email) } + let(:user) { { email: email } } it 'retrieves the Email address from the GitHub API' do expect(client).to receive(:user).with('kittens').and_return(user) @@ -251,7 +251,7 @@ RSpec.describe Gitlab::GithubImport::UserFinder, :clean_gitlab_redis_cache do end it 'shortens the timeout for Email address in cache when an Email address is private/nil from GitHub' do - user = double(:user, email: nil) + user = { email: nil } expect(client).to receive(:user).with('kittens').and_return(user) expect(Gitlab::Cache::Import::Caching) diff --git a/spec/lib/gitlab/grape_logging/formatters/lograge_with_timestamp_spec.rb b/spec/lib/gitlab/grape_logging/formatters/lograge_with_timestamp_spec.rb index 5006d27c356..f14f0098a1f 100644 --- a/spec/lib/gitlab/grape_logging/formatters/lograge_with_timestamp_spec.rb +++ b/spec/lib/gitlab/grape_logging/formatters/lograge_with_timestamp_spec.rb @@ -43,10 +43,11 @@ RSpec.describe Gitlab::GrapeLogging::Formatters::LogrageWithTimestamp do it 're-formats the params hash' do params = result['params'] - expect(params).to eq([ - { 'key' => 'description', 'value' => '[FILTERED]' }, - { 'key' => 'name', 'value' => 'gitlab test' }, - { 'key' => 'int', 'value' => 42 } - ]) + expect(params).to eq( + [ + { 'key' => 'description', 'value' => '[FILTERED]' }, + { 'key' => 'name', 'value' => 'gitlab test' }, + { 'key' => 'int', 'value' => 42 } + ]) end end diff --git a/spec/lib/gitlab/graphql/pagination/keyset/connection_spec.rb b/spec/lib/gitlab/graphql/pagination/keyset/connection_spec.rb index bf09e98331f..1124868bdae 100644 --- a/spec/lib/gitlab/graphql/pagination/keyset/connection_spec.rb +++ b/spec/lib/gitlab/graphql/pagination/keyset/connection_spec.rb @@ -51,6 +51,7 @@ RSpec.describe Gitlab::Graphql::Pagination::Keyset::Connection do before do stub_feature_flags(graphql_keyset_pagination_without_next_page_query: false) + allow(GitlabSchema).to receive(:default_max_page_size).and_return(2) end it 'invokes an extra query for the next page check' do diff --git a/spec/lib/gitlab/health_checks/master_check_spec.rb b/spec/lib/gitlab/health_checks/master_check_spec.rb index 8a87b01c560..5cd26f6302a 100644 --- a/spec/lib/gitlab/health_checks/master_check_spec.rb +++ b/spec/lib/gitlab/health_checks/master_check_spec.rb @@ -4,16 +4,14 @@ require 'fast_spec_helper' require_relative './simple_check_shared' RSpec.describe Gitlab::HealthChecks::MasterCheck do - before do - stub_const('SUCCESS_CODE', 100) - stub_const('FAILURE_CODE', 101) - end - context 'when Puma runs in Clustered mode' do before do allow(Gitlab::Runtime).to receive(:puma_in_clustered_mode?).and_return(true) - described_class.register_master + # We need to capture the read pipe here to stub out the non-blocking read. + # The original implementation actually forked the test suite for a more + # end-to-end test but that caused knock-on effects on other tests. + @pipe_read, _ = described_class.register_master end after do @@ -25,34 +23,40 @@ RSpec.describe Gitlab::HealthChecks::MasterCheck do end describe '.readiness' do - context 'when master is running' do - it 'worker does return success' do - _, child_status = run_worker - - expect(child_status.exitstatus).to eq(SUCCESS_CODE) + context 'when no worker registered' do + it 'succeeds' do + expect(described_class.readiness.success).to be(true) end end - context 'when master finishes early' do - before do - described_class.send(:close_write) + context 'when worker registers itself' do + context 'when reading from pipe succeeds' do + it 'succeeds' do + expect(@pipe_read).to receive(:read_nonblock) # rubocop: disable RSpec/InstanceVariable + + described_class.register_worker + + expect(described_class.readiness.success).to be(true) + end end - it 'worker does return failure' do - _, child_status = run_worker + context 'when read pipe is open but not ready for reading' do + it 'succeeds' do + expect(@pipe_read).to receive(:read_nonblock).and_raise(IO::EAGAINWaitReadable) # rubocop: disable RSpec/InstanceVariable + + described_class.register_worker - expect(child_status.exitstatus).to eq(FAILURE_CODE) + expect(described_class.readiness.success).to be(true) + end end end - def run_worker - pid = fork do - described_class.register_worker + context 'when master finishes early' do + it 'fails' do + described_class.finish_master - exit(described_class.readiness.success ? SUCCESS_CODE : FAILURE_CODE) + expect(described_class.readiness.success).to be(false) end - - Process.wait2(pid) end end end diff --git a/spec/lib/gitlab/hook_data/release_builder_spec.rb b/spec/lib/gitlab/hook_data/release_builder_spec.rb index 449965f5df1..08f9de4a2ed 100644 --- a/spec/lib/gitlab/hook_data/release_builder_spec.rb +++ b/spec/lib/gitlab/hook_data/release_builder_spec.rb @@ -13,12 +13,12 @@ RSpec.describe Gitlab::HookData::ReleaseBuilder do it 'includes safe attribute' do %w[ - id - created_at - description - name - released_at - tag + id + created_at + description + name + released_at + tag ].each do |key| expect(data).to include(key) end diff --git a/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb b/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb index 42cf9c54798..297fe3ade07 100644 --- a/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb +++ b/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb @@ -9,8 +9,6 @@ RSpec.describe Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy do allow_next_instance_of(ProjectExportWorker) do |job| allow(job).to receive(:jid).and_return(SecureRandom.hex(8)) end - - stub_feature_flags(import_export_web_upload_stream: false) stub_uploads_object_storage(FileUploader, enabled: false) end @@ -109,108 +107,68 @@ RSpec.describe Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy do end context 'when object store is enabled' do + let(:object_store_url) { 'http://object-storage/project.tar.gz' } + before do - object_store_url = 'http://object-storage/project.tar.gz' stub_uploads_object_storage(FileUploader) - stub_request(:get, object_store_url) - stub_request(:post, example_url) + allow(import_export_upload.export_file).to receive(:url).and_return(object_store_url) allow(import_export_upload.export_file).to receive(:file_storage?).and_return(false) end - it 'reads file using Gitlab::HttpIO and uploads to external url' do - expect_next_instance_of(Gitlab::HttpIO) do |http_io| - expect(http_io).to receive(:read).and_call_original + it 'uploads file as a remote stream' do + arguments = { + download_url: object_store_url, + upload_url: example_url, + options: { + upload_method: :post, + upload_content_type: 'application/gzip' + } + } + + expect_next_instance_of(Gitlab::ImportExport::RemoteStreamUpload, arguments) do |remote_stream_upload| + expect(remote_stream_upload).to receive(:execute) end - expect(Gitlab::ImportExport::RemoteStreamUpload).not_to receive(:new) + expect(Gitlab::HttpIO).not_to receive(:new) strategy.execute(user, project) - - expect(a_request(:post, example_url)).to have_been_made end - end - - context 'when `import_export_web_upload_stream` feature is enabled' do - before do - stub_feature_flags(import_export_web_upload_stream: true) - end - - context 'when remote object store is disabled' do - it 'reads file from disk and uploads to external url' do - stub_request(:post, example_url).to_return(status: 200) - expect(Gitlab::ImportExport::RemoteStreamUpload).not_to receive(:new) - expect(Gitlab::HttpIO).not_to receive(:new) - - strategy.execute(user, project) - - expect(a_request(:post, example_url)).to have_been_made - end - end - - context 'when object store is enabled' do - let(:object_store_url) { 'http://object-storage/project.tar.gz' } + context 'when upload as remote stream raises an exception' do before do - stub_uploads_object_storage(FileUploader) - - allow(import_export_upload.export_file).to receive(:url).and_return(object_store_url) - allow(import_export_upload.export_file).to receive(:file_storage?).and_return(false) + allow_next_instance_of(Gitlab::ImportExport::RemoteStreamUpload) do |remote_stream_upload| + allow(remote_stream_upload).to receive(:execute).and_raise( + Gitlab::ImportExport::RemoteStreamUpload::StreamError.new('Exception error message', 'Response body') + ) + end end - it 'uploads file as a remote stream' do - arguments = { - download_url: object_store_url, - upload_url: example_url, - options: { - upload_method: :post, - upload_content_type: 'application/gzip' - } - } - - expect_next_instance_of(Gitlab::ImportExport::RemoteStreamUpload, arguments) do |remote_stream_upload| - expect(remote_stream_upload).to receive(:execute) + it 'logs the exception and stores the error message' do + expect_next_instance_of(Gitlab::Export::Logger) do |logger| + expect(logger).to receive(:error).ordered.with( + { + project_id: project.id, + project_name: project.name, + message: 'Exception error message', + response_body: 'Response body' + } + ) + + expect(logger).to receive(:error).ordered.with( + { + project_id: project.id, + project_name: project.name, + message: 'After export strategy failed', + 'exception.class' => 'Gitlab::ImportExport::RemoteStreamUpload::StreamError', + 'exception.message' => 'Exception error message', + 'exception.backtrace' => anything + } + ) end - expect(Gitlab::HttpIO).not_to receive(:new) strategy.execute(user, project) - end - context 'when upload as remote stream raises an exception' do - before do - allow_next_instance_of(Gitlab::ImportExport::RemoteStreamUpload) do |remote_stream_upload| - allow(remote_stream_upload).to receive(:execute).and_raise( - Gitlab::ImportExport::RemoteStreamUpload::StreamError.new('Exception error message', 'Response body') - ) - end - end - - it 'logs the exception and stores the error message' do - expect_next_instance_of(Gitlab::Export::Logger) do |logger| - expect(logger).to receive(:error).ordered.with( - { - project_id: project.id, - project_name: project.name, - message: 'Exception error message', - response_body: 'Response body' - } - ) - - expect(logger).to receive(:error).ordered.with( - { - project_id: project.id, - project_name: project.name, - message: 'After export strategy failed', - 'exception.class' => 'Gitlab::ImportExport::RemoteStreamUpload::StreamError', - 'exception.message' => 'Exception error message', - 'exception.backtrace' => anything - } - ) - end - - strategy.execute(user, project) - - expect(project.import_export_shared.errors.first).to eq('Exception error message') - end + expect(project.import_export_shared.errors.first).to eq('Exception error message') end end end diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index e270ca9ec6a..ccc4f1f7149 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -61,6 +61,8 @@ issues: - requirement - incident_management_issuable_escalation_status - incident_management_timeline_events +- incident_management_timeline_event_tags +- incident_management_timeline_event_links - pending_escalations - customer_relations_contacts - issue_customer_relations_contacts @@ -95,6 +97,7 @@ label_links: label: - subscriptions - project +- parent_container - lists - label_links - issues @@ -296,6 +299,10 @@ ci_pipelines: - package_build_infos - package_file_build_infos - build_trace_chunks +- pipeline_metadata +pipeline_metadata: +- project +- pipeline ci_refs: - project - ci_pipelines @@ -541,6 +548,7 @@ project: - path_locks - approver_groups - repository_state +- wiki_repository_state - source_pipelines - sourced_pipelines - prometheus_metrics @@ -567,6 +575,7 @@ project: - project_registry - packages - package_files +- repository_files - packages_cleanup_policy - alerting_setting - project_setting @@ -615,6 +624,7 @@ project: - incident_management_oncall_rotations - incident_management_escalation_policies - incident_management_issuable_escalation_statuses +- incident_management_timeline_event_tags - debian_distributions - merge_request_metrics - security_orchestration_policy_configuration @@ -632,6 +642,7 @@ project: - vulnerability_reads - build_artifacts_size_refresh - project_callouts +- pipeline_metadata award_emoji: - awardable - user diff --git a/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb b/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb index 89ae869ae86..1444897e136 100644 --- a/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb @@ -116,15 +116,15 @@ RSpec.describe Gitlab::ImportExport::Group::TreeRestorer do shared_examples 'excluded attributes' do excluded_attributes = %w[ - id - parent_id - owner_id - created_at - updated_at - runners_token - runners_token_encrypted - saml_discovery_token - ] + id + parent_id + owner_id + created_at + updated_at + runners_token + runners_token_encrypted + saml_discovery_token + ] before do group.add_owner(importer_user) diff --git a/spec/lib/gitlab/import_export/group/tree_saver_spec.rb b/spec/lib/gitlab/import_export/group/tree_saver_spec.rb index de4d193a21c..85d07e3fe63 100644 --- a/spec/lib/gitlab/import_export/group/tree_saver_spec.rb +++ b/spec/lib/gitlab/import_export/group/tree_saver_spec.rb @@ -51,11 +51,12 @@ RSpec.describe Gitlab::ImportExport::Group::TreeSaver do .map { |line| Integer(line) } expect(groups_catalog.size).to eq(3) - expect(groups_catalog).to eq([ - group.id, - group.descendants.first.id, - group.descendants.first.descendants.first.id - ]) + expect(groups_catalog).to eq( + [ + group.id, + group.descendants.first.id, + group.descendants.first.descendants.first.id + ]) end it 'has a file per group' do diff --git a/spec/lib/gitlab/import_export/project/relation_factory_spec.rb b/spec/lib/gitlab/import_export/project/relation_factory_spec.rb index 52b33e22089..936c63fd6cd 100644 --- a/spec/lib/gitlab/import_export/project/relation_factory_spec.rb +++ b/spec/lib/gitlab/import_export/project/relation_factory_spec.rb @@ -41,7 +41,7 @@ RSpec.describe Gitlab::ImportExport::Project::RelationFactory, :use_clean_rails_ context 'hook object' do let(:relation_sym) { :hooks } let(:id) { 999 } - let(:service_id) { 99 } + let(:integration_id) { 99 } let(:original_project_id) { 8 } let(:token) { 'secret' } @@ -52,7 +52,7 @@ RSpec.describe Gitlab::ImportExport::Project::RelationFactory, :use_clean_rails_ 'project_id' => original_project_id, 'created_at' => '2016-08-12T09:41:03.462Z', 'updated_at' => '2016-08-12T09:41:03.462Z', - 'service_id' => service_id, + 'integration_id' => integration_id, 'push_events' => true, 'issues_events' => false, 'confidential_issues_events' => false, @@ -71,8 +71,8 @@ RSpec.describe Gitlab::ImportExport::Project::RelationFactory, :use_clean_rails_ expect(created_object.id).not_to eq(id) end - it 'does not have the original service_id' do - expect(created_object.service_id).not_to eq(service_id) + it 'does not have the original integration_id' do + expect(created_object.integration_id).not_to eq(integration_id) end it 'does not have the original project_id' do @@ -88,10 +88,10 @@ RSpec.describe Gitlab::ImportExport::Project::RelationFactory, :use_clean_rails_ end context 'original service exists' do - let(:service_id) { create(:integration, project: project).id } + let(:integration_id) { create(:integration, project: project).id } - it 'does not have the original service_id' do - expect(created_object.service_id).not_to eq(service_id) + it 'does not have the original integration_id' do + expect(created_object.integration_id).not_to eq(integration_id) end end @@ -302,7 +302,7 @@ RSpec.describe Gitlab::ImportExport::Project::RelationFactory, :use_clean_rails_ let(:relation_sym) { :hazardous_foo_model } let(:relation_hash) do { - 'service_id' => 99, + 'integration_id' => 99, 'moved_to_id' => 99, 'namespace_id' => 99, 'ci_id' => 99, @@ -317,7 +317,7 @@ RSpec.describe Gitlab::ImportExport::Project::RelationFactory, :use_clean_rails_ before do stub_const('HazardousFooModel', Class.new(FooModel)) HazardousFooModel.class_eval do - attr_accessor :service_id, :moved_to_id, :namespace_id, :ci_id, :random_project_id, :random_id, :milestone_id, :project_id + attr_accessor :integration_id, :moved_to_id, :namespace_id, :ci_id, :random_project_id, :random_id, :milestone_id, :project_id end allow(HazardousFooModel).to receive(:reflect_on_association).and_return(nil) diff --git a/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb index 299e107c881..fae94a3b544 100644 --- a/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb @@ -140,13 +140,13 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer do it 'restores pipelines based on ascending id order' do expected_ordered_shas = %w[ - 2ea1f3dec713d940208fb5ce4a38765ecb5d3f73 - ce84140e8b878ce6e7c4d298c7202ff38170e3ac - 048721d90c449b244b7b4c53a9186b04330174ec - sha-notes - 5f923865dde3436854e9ceb9cdb7815618d4e849 - d2d430676773caa88cdaf7c55944073b2fd5561a - 2ea1f3dec713d940208fb5ce4a38765ecb5d3f73 + 2ea1f3dec713d940208fb5ce4a38765ecb5d3f73 + ce84140e8b878ce6e7c4d298c7202ff38170e3ac + 048721d90c449b244b7b4c53a9186b04330174ec + sha-notes + 5f923865dde3436854e9ceb9cdb7815618d4e849 + d2d430676773caa88cdaf7c55944073b2fd5561a + 2ea1f3dec713d940208fb5ce4a38765ecb5d3f73 ] project = Project.find_by_path('project') @@ -156,6 +156,15 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer do end end + it 'restores pipeline metadata' do + pipeline = Ci::Pipeline.find_by_sha('sha-notes') + pipeline_metadata = pipeline.pipeline_metadata + + expect(pipeline_metadata.title).to eq('Build pipeline') + expect(pipeline_metadata.pipeline_id).to eq(pipeline.id) + expect(pipeline_metadata.project_id).to eq(pipeline.project_id) + end + it 'preserves updated_at on issues' do issue = Issue.find_by(description: 'Aliquam enim illo et possimus.') diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index e591cbd05a0..23eb93a1bce 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -332,6 +332,11 @@ Ci::Pipeline: - iid - merge_request_id - external_pull_request_id +Ci::PipelineMetadata: +- id +- project_id +- pipeline_id +- title Ci::Stage: - id - name @@ -697,6 +702,7 @@ ProjectCiCdSetting: - runner_token_expiration_interval ProjectSetting: - allow_merge_on_skipped_pipeline +- only_allow_merge_if_all_status_checks_passed - has_confluence - has_shimo - has_vulnerabilities diff --git a/spec/lib/gitlab/import_export/uploads_manager_spec.rb b/spec/lib/gitlab/import_export/uploads_manager_spec.rb index 0cfe3a69a09..5fc3a70169a 100644 --- a/spec/lib/gitlab/import_export/uploads_manager_spec.rb +++ b/spec/lib/gitlab/import_export/uploads_manager_spec.rb @@ -78,16 +78,30 @@ RSpec.describe Gitlab::ImportExport::UploadsManager do context 'when upload is in object storage' do before do stub_uploads_object_storage(FileUploader) - allow(manager).to receive(:download_or_copy_upload).and_raise(Errno::ENAMETOOLONG) end - it 'ignores problematic upload and logs exception' do - expect(Gitlab::ErrorTracking).to receive(:log_exception).with(instance_of(Errno::ENAMETOOLONG), project_id: project.id) + shared_examples 'export with invalid upload' do + it 'ignores problematic upload and logs exception' do + allow(manager).to receive(:download_or_copy_upload).and_raise(exception) + expect(Gitlab::ErrorTracking).to receive(:log_exception).with(instance_of(exception), project_id: project.id) - manager.save # rubocop:disable Rails/SaveBang + manager.save # rubocop:disable Rails/SaveBang - expect(shared.errors).to be_empty - expect(File).not_to exist(exported_file_path) + expect(shared.errors).to be_empty + expect(File).not_to exist(exported_file_path) + end + end + + context 'when filename is too long' do + let(:exception) { Errno::ENAMETOOLONG } + + include_examples 'export with invalid upload' + end + + context 'when network exception occurs' do + let(:exception) { Net::OpenTimeout } + + include_examples 'export with invalid upload' end end end diff --git a/spec/lib/gitlab/import_export/wiki_repo_saver_spec.rb b/spec/lib/gitlab/import_export/wiki_repo_saver_spec.rb index c936d2bc27d..0e6173b611f 100644 --- a/spec/lib/gitlab/import_export/wiki_repo_saver_spec.rb +++ b/spec/lib/gitlab/import_export/wiki_repo_saver_spec.rb @@ -17,7 +17,7 @@ RSpec.describe Gitlab::ImportExport::WikiRepoSaver do allow_next_instance_of(Gitlab::ImportExport) do |instance| allow(instance).to receive(:storage_path).and_return(export_path) end - project_wiki.wiki + project_wiki.create_wiki_repository project_wiki.create_page("index", "test content") end diff --git a/spec/lib/gitlab/jira_import/issues_importer_spec.rb b/spec/lib/gitlab/jira_import/issues_importer_spec.rb index a2a482dde7c..9f654bbcd15 100644 --- a/spec/lib/gitlab/jira_import/issues_importer_spec.rb +++ b/spec/lib/gitlab/jira_import/issues_importer_spec.rb @@ -40,11 +40,11 @@ RSpec.describe Gitlab::JiraImport::IssuesImporter do context 'with results returned' do jira_issue = Struct.new(:id) - let_it_be(:jira_issues) { [jira_issue.new(1), jira_issue.new(2), jira_issue.new(3)] } + let_it_be(:jira_issues) { [jira_issue.new(1), jira_issue.new(2)] } def mock_issue_serializer(count, raise_exception_on_even_mocks: false) serializer = instance_double(Gitlab::JiraImport::IssueSerializer, execute: { key: 'data' }) - next_iid = project.issues.maximum(:iid).to_i + allow(Issue).to receive(:with_project_iid_supply).and_return('issue_iid') count.times do |i| if raise_exception_on_even_mocks && i.even? @@ -53,16 +53,15 @@ RSpec.describe Gitlab::JiraImport::IssuesImporter do jira_issues[i], current_user.id, default_issue_type_id, - { iid: next_iid + 1 } + { iid: 'issue_iid' } ).and_raise('Some error') else - next_iid += 1 expect(Gitlab::JiraImport::IssueSerializer).to receive(:new).with( project, jira_issues[i], current_user.id, default_issue_type_id, - { iid: next_iid } + { iid: 'issue_iid' } ).and_return(serializer) end end @@ -125,47 +124,6 @@ RSpec.describe Gitlab::JiraImport::IssuesImporter do expect(Gitlab::JiraImport.get_issues_next_start_at(project.id)).to eq(2) end end - - context 'when number of issues is above the threshold' do - before do - stub_const("#{described_class.name}::JIRA_IMPORT_THRESHOLD", 2) - stub_const("#{described_class.name}::JIRA_IMPORT_PAUSE_LIMIT", 1) - allow(Gitlab::ErrorTracking).to receive(:track_exception) - allow_next_instance_of(Gitlab::JobWaiter) do |job_waiter| - allow(job_waiter).to receive(:wait).with(5).and_return(job_waiter.wait(0.1)) - end - end - - it 'schedules 2 import jobs with two pause points' do - expect(subject).to receive(:fetch_issues).with(0).and_return([jira_issues[0], jira_issues[1], jira_issues[2]]) - expect(Gitlab::JiraImport::ImportIssueWorker).to receive(:perform_async).exactly(3).times - expect(Gitlab::JiraImport::ImportIssueWorker) - .to receive(:queue_size) - .exactly(6).times - .and_return(1, 2, 3, 2, 1, 0) - - mock_issue_serializer(3) - - expect(subject.execute).to have_received(:wait).with(5).twice - end - - it 'tracks the exception if the queue size does not reduce' do - expect(subject).to receive(:fetch_issues).with(0).and_return([jira_issues[0]]) - expect(Gitlab::JiraImport::ImportIssueWorker).not_to receive(:perform_async) - expect(Gitlab::JiraImport::ImportIssueWorker) - .to receive(:queue_size) - .exactly(11).times - .and_return(3) - - mock_issue_serializer(1) - - expect(subject.execute).to have_received(:wait).with(5).exactly(10).times - expect(Gitlab::ErrorTracking) - .to have_received(:track_exception) - .with(described_class::RetriesExceededError, { project_id: project.id }) - .once - end - end end end end diff --git a/spec/lib/gitlab/json_spec.rb b/spec/lib/gitlab/json_spec.rb index 7c093049e18..73276288765 100644 --- a/spec/lib/gitlab/json_spec.rb +++ b/spec/lib/gitlab/json_spec.rb @@ -8,6 +8,12 @@ RSpec.describe Gitlab::Json do end describe ".parse" do + it "is aliased" do + [:parse!, :load, :decode].each do |method| + expect(described_class.method(method)).to eq(described_class.method(:parse)) + end + end + context "legacy_mode is disabled by default" do it "parses an object" do expect(subject.parse('{ "foo": "bar" }')).to eq({ "foo" => "bar" }) @@ -178,6 +184,10 @@ RSpec.describe Gitlab::Json do { test: true, "foo.bar" => "baz", is_json: 1, some: [1, 2, 3] } end + it "is aliased" do + expect(described_class.method(:encode)).to eq(described_class.method(:generate)) + end + it "generates JSON" do expected_string = <<~STR.chomp {"test":true,"foo.bar":"baz","is_json":1,"some":[1,2,3]} diff --git a/spec/lib/gitlab/kubernetes/rollout_instances_spec.rb b/spec/lib/gitlab/kubernetes/rollout_instances_spec.rb index 3ac97ddc75d..a7b2352f496 100644 --- a/spec/lib/gitlab/kubernetes/rollout_instances_spec.rb +++ b/spec/lib/gitlab/kubernetes/rollout_instances_spec.rb @@ -51,13 +51,14 @@ RSpec.describe Gitlab::Kubernetes::RolloutInstances do end it 'returns instances when there are two stable deployments' do - deployments, pods = setup([ - kube_deployment(name: 'one', track: 'stable', replicas: 1), - kube_deployment(name: 'two', track: 'stable', replicas: 1) - ], [ - kube_pod(name: 'one', status: 'Running', track: 'stable'), - kube_pod(name: 'two', status: 'Running', track: 'stable') - ]) + deployments, pods = setup( + [ + kube_deployment(name: 'one', track: 'stable', replicas: 1), + kube_deployment(name: 'two', track: 'stable', replicas: 1) + ], [ + kube_pod(name: 'one', status: 'Running', track: 'stable'), + kube_pod(name: 'two', status: 'Running', track: 'stable') + ]) rollout_instances = described_class.new(deployments, pods) expect(rollout_instances.pod_instances).to eq([{ @@ -76,13 +77,14 @@ RSpec.describe Gitlab::Kubernetes::RolloutInstances do end it 'returns instances for two deployments with different tracks' do - deployments, pods = setup([ - kube_deployment(name: 'one', track: 'mytrack', replicas: 1), - kube_deployment(name: 'two', track: 'othertrack', replicas: 1) - ], [ - kube_pod(name: 'one', status: 'Running', track: 'mytrack'), - kube_pod(name: 'two', status: 'Running', track: 'othertrack') - ]) + deployments, pods = setup( + [ + kube_deployment(name: 'one', track: 'mytrack', replicas: 1), + kube_deployment(name: 'two', track: 'othertrack', replicas: 1) + ], [ + kube_pod(name: 'one', status: 'Running', track: 'mytrack'), + kube_pod(name: 'two', status: 'Running', track: 'othertrack') + ]) rollout_instances = described_class.new(deployments, pods) expect(rollout_instances.pod_instances).to eq([{ @@ -101,13 +103,14 @@ RSpec.describe Gitlab::Kubernetes::RolloutInstances do end it 'sorts stable tracks after canary tracks' do - deployments, pods = setup([ - kube_deployment(name: 'one', track: 'stable', replicas: 1), - kube_deployment(name: 'two', track: 'canary', replicas: 1) - ], [ - kube_pod(name: 'one', status: 'Running', track: 'stable'), - kube_pod(name: 'two', status: 'Running', track: 'canary') - ]) + deployments, pods = setup( + [ + kube_deployment(name: 'one', track: 'stable', replicas: 1), + kube_deployment(name: 'two', track: 'canary', replicas: 1) + ], [ + kube_pod(name: 'one', status: 'Running', track: 'stable'), + kube_pod(name: 'two', status: 'Running', track: 'canary') + ]) rollout_instances = described_class.new(deployments, pods) expect(rollout_instances.pod_instances).to eq([{ diff --git a/spec/lib/gitlab/legacy_github_import/branch_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/branch_formatter_spec.rb index 1a21ed29ab7..09dd04c76c9 100644 --- a/spec/lib/gitlab/legacy_github_import/branch_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/branch_formatter_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::LegacyGithubImport::BranchFormatter do - let(:project) { create(:project, :repository) } + let_it_be(:project) { create(:project, :repository) } let(:commit) { create(:commit, project: project) } let(:repo) { double } let(:raw) do @@ -16,19 +16,19 @@ RSpec.describe Gitlab::LegacyGithubImport::BranchFormatter do describe '#exists?' do it 'returns true when branch exists and commit is part of the branch' do - branch = described_class.new(project, double(raw)) + branch = described_class.new(project, raw) expect(branch.exists?).to eq true end it 'returns false when branch exists and commit is not part of the branch' do - branch = described_class.new(project, double(raw.merge(ref: 'feature'))) + branch = described_class.new(project, raw.merge(ref: 'feature')) expect(branch.exists?).to eq false end it 'returns false when branch does not exist' do - branch = described_class.new(project, double(raw.merge(ref: 'removed-branch'))) + branch = described_class.new(project, raw.merge(ref: 'removed-branch')) expect(branch.exists?).to eq false end @@ -36,7 +36,7 @@ RSpec.describe Gitlab::LegacyGithubImport::BranchFormatter do describe '#repo' do it 'returns raw repo' do - branch = described_class.new(project, double(raw)) + branch = described_class.new(project, raw) expect(branch.repo).to eq repo end @@ -44,7 +44,7 @@ RSpec.describe Gitlab::LegacyGithubImport::BranchFormatter do describe '#sha' do it 'returns raw sha' do - branch = described_class.new(project, double(raw)) + branch = described_class.new(project, raw) expect(branch.sha).to eq commit.id end @@ -52,19 +52,19 @@ RSpec.describe Gitlab::LegacyGithubImport::BranchFormatter do describe '#valid?' do it 'returns true when raw sha and ref are present' do - branch = described_class.new(project, double(raw)) + branch = described_class.new(project, raw) expect(branch.valid?).to eq true end it 'returns false when raw sha is blank' do - branch = described_class.new(project, double(raw.merge(sha: nil))) + branch = described_class.new(project, raw.merge(sha: nil)) expect(branch.valid?).to eq false end it 'returns false when raw ref is blank' do - branch = described_class.new(project, double(raw.merge(ref: nil))) + branch = described_class.new(project, raw.merge(ref: nil)) expect(branch.valid?).to eq false end diff --git a/spec/lib/gitlab/legacy_github_import/comment_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/comment_formatter_spec.rb index 85f7666fe85..8d6415b8179 100644 --- a/spec/lib/gitlab/legacy_github_import/comment_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/comment_formatter_spec.rb @@ -3,9 +3,9 @@ require 'spec_helper' RSpec.describe Gitlab::LegacyGithubImport::CommentFormatter do + let_it_be(:project) { create(:project) } let(:client) { double } - let(:project) { create(:project) } - let(:octocat) { double(id: 123456, login: 'octocat', email: 'octocat@example.com') } + let(:octocat) { { id: 123456, login: 'octocat', email: 'octocat@example.com' } } let(:created_at) { DateTime.strptime('2013-04-10T20:09:31Z') } let(:updated_at) { DateTime.strptime('2014-03-03T18:58:10Z') } let(:base) do @@ -27,7 +27,7 @@ RSpec.describe Gitlab::LegacyGithubImport::CommentFormatter do describe '#attributes' do context 'when do not reference a portion of the diff' do - let(:raw) { double(base) } + let(:raw) { base } it 'returns formatted attributes' do expected = { @@ -55,7 +55,7 @@ RSpec.describe Gitlab::LegacyGithubImport::CommentFormatter do } end - let(:raw) { double(base.merge(diff)) } + let(:raw) { base.merge(diff) } it 'returns formatted attributes' do expected = { @@ -74,22 +74,22 @@ RSpec.describe Gitlab::LegacyGithubImport::CommentFormatter do end context 'when author is a GitLab user' do - let(:raw) { double(base.merge(user: octocat)) } + let(:raw) { base.merge(user: octocat) } it 'returns GitLab user id associated with GitHub id as author_id' do - gl_user = create(:omniauth_user, extern_uid: octocat.id, provider: 'github') + gl_user = create(:omniauth_user, extern_uid: octocat[:id], provider: 'github') expect(comment.attributes.fetch(:author_id)).to eq gl_user.id end it 'returns GitLab user id associated with GitHub email as author_id' do - gl_user = create(:user, email: octocat.email) + gl_user = create(:user, email: octocat[:email]) expect(comment.attributes.fetch(:author_id)).to eq gl_user.id end it 'returns note without created at tag line' do - create(:omniauth_user, extern_uid: octocat.id, provider: 'github') + create(:omniauth_user, extern_uid: octocat[:id], provider: 'github') expect(comment.attributes.fetch(:note)).to eq("I'm having a problem with this.") end diff --git a/spec/lib/gitlab/legacy_github_import/importer_spec.rb b/spec/lib/gitlab/legacy_github_import/importer_spec.rb index 1800b42160d..cd66b93eb8b 100644 --- a/spec/lib/gitlab/legacy_github_import/importer_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/importer_spec.rb @@ -59,23 +59,23 @@ RSpec.describe Gitlab::LegacyGithubImport::Importer do end let(:label1) do - double( + { name: 'Bug', color: 'ff0000', url: "#{api_root}/repos/octocat/Hello-World/labels/bug" - ) + } end let(:label2) do - double( + { name: nil, color: 'ff0000', url: "#{api_root}/repos/octocat/Hello-World/labels/bug" - ) + } end let(:milestone) do - double( + { id: 1347, # For Gitea number: 1347, state: 'open', @@ -86,11 +86,11 @@ RSpec.describe Gitlab::LegacyGithubImport::Importer do updated_at: updated_at, closed_at: nil, url: "#{api_root}/repos/octocat/Hello-World/milestones/1" - ) + } end let(:issue1) do - double( + { number: 1347, milestone: nil, state: 'open', @@ -104,12 +104,12 @@ RSpec.describe Gitlab::LegacyGithubImport::Importer do updated_at: updated_at, closed_at: nil, url: "#{api_root}/repos/octocat/Hello-World/issues/1347", - labels: [double(name: 'Label #1')] - ) + labels: [{ name: 'Label #1' }] + } end let(:issue2) do - double( + { number: 1348, milestone: nil, state: 'open', @@ -123,12 +123,12 @@ RSpec.describe Gitlab::LegacyGithubImport::Importer do updated_at: updated_at, closed_at: nil, url: "#{api_root}/repos/octocat/Hello-World/issues/1348", - labels: [double(name: 'Label #2')] - ) + labels: [{ name: 'Label #2' }] + } end let(:release1) do - double( + { tag_name: 'v1.0.0', name: 'First release', body: 'Release v1.0.0', @@ -137,11 +137,11 @@ RSpec.describe Gitlab::LegacyGithubImport::Importer do published_at: created_at, updated_at: updated_at, url: "#{api_root}/repos/octocat/Hello-World/releases/1" - ) + } end let(:release2) do - double( + { tag_name: 'v1.1.0', name: 'Second release', body: nil, @@ -150,7 +150,7 @@ RSpec.describe Gitlab::LegacyGithubImport::Importer do published_at: created_at, updated_at: updated_at, url: "#{api_root}/repos/octocat/Hello-World/releases/2" - ) + } end subject { described_class.new(project) } @@ -210,18 +210,18 @@ RSpec.describe Gitlab::LegacyGithubImport::Importer do end let(:project) { create(:project, :repository, :wiki_disabled, import_url: "#{repo_root}/octocat/Hello-World.git") } - let(:octocat) { double(id: 123456, login: 'octocat', email: 'octocat@example.com') } + let(:octocat) { { id: 123456, login: 'octocat', email: 'octocat@example.com' } } let(:credentials) { { user: 'joe' } } let(:created_at) { DateTime.strptime('2011-01-26T19:01:12Z') } let(:updated_at) { DateTime.strptime('2011-01-27T19:01:12Z') } - let(:repository) { double(id: 1, fork: false) } + let(:repository) { { id: 1, fork: false } } let(:source_sha) { create(:commit, project: project).id } - let(:source_branch) { double(ref: 'branch-merged', repo: repository, sha: source_sha, user: octocat) } + let(:source_branch) { { ref: 'branch-merged', repo: repository, sha: source_sha, user: octocat } } let(:target_sha) { create(:commit, project: project, git_commit: RepoHelpers.another_sample_commit).id } - let(:target_branch) { double(ref: 'master', repo: repository, sha: target_sha, user: octocat) } + let(:target_branch) { { ref: 'master', repo: repository, sha: target_sha, user: octocat } } let(:pull_request) do - double( + { number: 1347, milestone: nil, state: 'open', @@ -236,12 +236,12 @@ RSpec.describe Gitlab::LegacyGithubImport::Importer do closed_at: nil, merged_at: nil, url: "#{api_root}/repos/octocat/Hello-World/pulls/1347", - labels: [double(name: 'Label #2')] - ) + labels: [{ name: 'Label #2' }] + } end let(:closed_pull_request) do - double( + { number: 1347, milestone: nil, state: 'closed', @@ -256,8 +256,8 @@ RSpec.describe Gitlab::LegacyGithubImport::Importer do closed_at: updated_at, merged_at: nil, url: "#{api_root}/repos/octocat/Hello-World/pulls/1347", - labels: [double(name: 'Label #2')] - ) + labels: [{ name: 'Label #2' }] + } end context 'when importing a Gitea project' do diff --git a/spec/lib/gitlab/legacy_github_import/issuable_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/issuable_formatter_spec.rb index a285a5820a2..56a51c6bddd 100644 --- a/spec/lib/gitlab/legacy_github_import/issuable_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/issuable_formatter_spec.rb @@ -4,7 +4,7 @@ require 'fast_spec_helper' RSpec.describe Gitlab::LegacyGithubImport::IssuableFormatter do let(:raw_data) do - double(number: 42) + { number: 42 } end let(:project) { double(import_type: 'github') } diff --git a/spec/lib/gitlab/legacy_github_import/issue_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/issue_formatter_spec.rb index 454bab8846c..d3548fecbcd 100644 --- a/spec/lib/gitlab/legacy_github_import/issue_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/issue_formatter_spec.rb @@ -3,9 +3,9 @@ require 'spec_helper' RSpec.describe Gitlab::LegacyGithubImport::IssueFormatter do + let_it_be(:project) { create(:project, namespace: create(:namespace, path: 'octocat')) } let(:client) { double } - let!(:project) { create(:project, namespace: create(:namespace, path: 'octocat')) } - let(:octocat) { double(id: 123456, login: 'octocat', email: 'octocat@example.com') } + let(:octocat) { { id: 123456, login: 'octocat', email: 'octocat@example.com' } } let(:created_at) { DateTime.strptime('2011-01-26T19:01:12Z') } let(:updated_at) { DateTime.strptime('2011-01-27T19:01:12Z') } @@ -34,7 +34,7 @@ RSpec.describe Gitlab::LegacyGithubImport::IssueFormatter do shared_examples 'Gitlab::LegacyGithubImport::IssueFormatter#attributes' do context 'when issue is open' do - let(:raw_data) { double(base_data.merge(state: 'open')) } + let(:raw_data) { base_data.merge(state: 'open') } it 'returns formatted attributes' do expected = { @@ -55,7 +55,7 @@ RSpec.describe Gitlab::LegacyGithubImport::IssueFormatter do end context 'when issue is closed' do - let(:raw_data) { double(base_data.merge(state: 'closed')) } + let(:raw_data) { base_data.merge(state: 'closed') } it 'returns formatted attributes' do expected = { @@ -76,28 +76,28 @@ RSpec.describe Gitlab::LegacyGithubImport::IssueFormatter do end context 'when it is assigned to someone' do - let(:raw_data) { double(base_data.merge(assignee: octocat)) } + let(:raw_data) { base_data.merge(assignee: octocat) } it 'returns nil as assignee_id when is not a GitLab user' do expect(issue.attributes.fetch(:assignee_ids)).to be_empty end it 'returns GitLab user id associated with GitHub id as assignee_id' do - gl_user = create(:omniauth_user, extern_uid: octocat.id, provider: 'github') + gl_user = create(:omniauth_user, extern_uid: octocat[:id], provider: 'github') expect(issue.attributes.fetch(:assignee_ids)).to eq [gl_user.id] end it 'returns GitLab user id associated with GitHub email as assignee_id' do - gl_user = create(:user, email: octocat.email) + gl_user = create(:user, email: octocat[:email]) expect(issue.attributes.fetch(:assignee_ids)).to eq [gl_user.id] end end context 'when it has a milestone' do - let(:milestone) { double(id: 42, number: 42) } - let(:raw_data) { double(base_data.merge(milestone: milestone)) } + let(:milestone) { { id: 42, number: 42 } } + let(:raw_data) { base_data.merge(milestone: milestone) } it 'returns nil when milestone does not exist' do expect(issue.attributes.fetch(:milestone)).to be_nil @@ -111,26 +111,26 @@ RSpec.describe Gitlab::LegacyGithubImport::IssueFormatter do end context 'when author is a GitLab user' do - let(:raw_data) { double(base_data.merge(user: octocat)) } + let(:raw_data) { base_data.merge(user: octocat) } it 'returns project creator_id as author_id when is not a GitLab user' do expect(issue.attributes.fetch(:author_id)).to eq project.creator_id end it 'returns GitLab user id associated with GitHub id as author_id' do - gl_user = create(:omniauth_user, extern_uid: octocat.id, provider: 'github') + gl_user = create(:omniauth_user, extern_uid: octocat[:id], provider: 'github') expect(issue.attributes.fetch(:author_id)).to eq gl_user.id end it 'returns GitLab user id associated with GitHub email as author_id' do - gl_user = create(:user, email: octocat.email) + gl_user = create(:user, email: octocat[:email]) expect(issue.attributes.fetch(:author_id)).to eq gl_user.id end it 'returns description without created at tag line' do - create(:omniauth_user, extern_uid: octocat.id, provider: 'github') + create(:omniauth_user, extern_uid: octocat[:id], provider: 'github') expect(issue.attributes.fetch(:description)).to eq("I'm having a problem with this.") end @@ -138,7 +138,7 @@ RSpec.describe Gitlab::LegacyGithubImport::IssueFormatter do end shared_examples 'Gitlab::LegacyGithubImport::IssueFormatter#number' do - let(:raw_data) { double(base_data.merge(number: 1347)) } + let(:raw_data) { base_data.merge(number: 1347) } it 'returns issue number' do expect(issue.number).to eq 1347 @@ -161,7 +161,7 @@ RSpec.describe Gitlab::LegacyGithubImport::IssueFormatter do describe '#has_comments?' do context 'when number of comments is greater than zero' do - let(:raw_data) { double(base_data.merge(comments: 1)) } + let(:raw_data) { base_data.merge(comments: 1) } it 'returns true' do expect(issue.has_comments?).to eq true @@ -169,7 +169,7 @@ RSpec.describe Gitlab::LegacyGithubImport::IssueFormatter do end context 'when number of comments is equal to zero' do - let(:raw_data) { double(base_data.merge(comments: 0)) } + let(:raw_data) { base_data.merge(comments: 0) } it 'returns false' do expect(issue.has_comments?).to eq false @@ -179,7 +179,7 @@ RSpec.describe Gitlab::LegacyGithubImport::IssueFormatter do describe '#pull_request?' do context 'when mention a pull request' do - let(:raw_data) { double(base_data.merge(pull_request: double)) } + let(:raw_data) { base_data.merge(pull_request: double) } it 'returns true' do expect(issue.pull_request?).to eq true @@ -187,7 +187,7 @@ RSpec.describe Gitlab::LegacyGithubImport::IssueFormatter do end context 'when does not mention a pull request' do - let(:raw_data) { double(base_data.merge(pull_request: nil)) } + let(:raw_data) { base_data.merge(pull_request: nil) } it 'returns false' do expect(issue.pull_request?).to eq false diff --git a/spec/lib/gitlab/legacy_github_import/label_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/label_formatter_spec.rb index ab7c8ea4a58..8e2c8031a6f 100644 --- a/spec/lib/gitlab/legacy_github_import/label_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/label_formatter_spec.rb @@ -3,8 +3,8 @@ require 'spec_helper' RSpec.describe Gitlab::LegacyGithubImport::LabelFormatter do - let(:project) { create(:project) } - let(:raw) { double(name: 'improvements', color: 'e6e6e6') } + let_it_be(:project) { create(:project) } + let(:raw) { { name: 'improvements', color: 'e6e6e6' } } subject { described_class.new(project, raw) } @@ -27,7 +27,7 @@ RSpec.describe Gitlab::LegacyGithubImport::LabelFormatter do context 'when label exists' do it 'does not create a new label' do - Labels::CreateService.new(name: raw.name).execute(project: project) + Labels::CreateService.new(name: raw[:name]).execute(project: project) expect { subject.create! }.not_to change(Label, :count) end diff --git a/spec/lib/gitlab/legacy_github_import/milestone_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/milestone_formatter_spec.rb index 64fcc46d304..7c57bf9c707 100644 --- a/spec/lib/gitlab/legacy_github_import/milestone_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/milestone_formatter_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::LegacyGithubImport::MilestoneFormatter do - let(:project) { create(:project) } + let_it_be(:project) { create(:project) } let(:created_at) { DateTime.strptime('2011-01-26T19:01:12Z') } let(:updated_at) { DateTime.strptime('2011-01-27T19:01:12Z') } let(:base_data) do @@ -26,7 +26,7 @@ RSpec.describe Gitlab::LegacyGithubImport::MilestoneFormatter do let(:data) { base_data.merge(iid_attr => 1347) } context 'when milestone is open' do - let(:raw_data) { double(data.merge(state: 'open')) } + let(:raw_data) { data.merge(state: 'open') } it 'returns formatted attributes' do expected = { @@ -45,7 +45,7 @@ RSpec.describe Gitlab::LegacyGithubImport::MilestoneFormatter do end context 'when milestone is closed' do - let(:raw_data) { double(data.merge(state: 'closed')) } + let(:raw_data) { data.merge(state: 'closed') } it 'returns formatted attributes' do expected = { @@ -65,7 +65,7 @@ RSpec.describe Gitlab::LegacyGithubImport::MilestoneFormatter do context 'when milestone has a due date' do let(:due_date) { DateTime.strptime('2011-01-28T19:01:12Z') } - let(:raw_data) { double(data.merge(due_on: due_date)) } + let(:raw_data) { data.merge(due_on: due_date) } it 'returns formatted attributes' do expected = { diff --git a/spec/lib/gitlab/legacy_github_import/pull_request_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/pull_request_formatter_spec.rb index 7d8875e36c3..90469693820 100644 --- a/spec/lib/gitlab/legacy_github_import/pull_request_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/pull_request_formatter_spec.rb @@ -3,22 +3,22 @@ require 'spec_helper' RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter do + let_it_be(:project) { create(:project, :repository) } let(:client) { double } - let(:project) { create(:project, :repository) } let(:source_sha) { create(:commit, project: project).id } let(:target_commit) { create(:commit, project: project, git_commit: RepoHelpers.another_sample_commit) } let(:target_sha) { target_commit.id } let(:target_short_sha) { target_commit.id.to_s[0..7] } - let(:repository) { double(id: 1, fork: false) } + let(:repository) { { id: 1, fork: false } } let(:source_repo) { repository } - let(:source_branch) { double(ref: 'branch-merged', repo: source_repo, sha: source_sha) } - let(:forked_source_repo) { double(id: 2, fork: true, name: 'otherproject', full_name: 'company/otherproject') } + let(:source_branch) { { ref: 'branch-merged', repo: source_repo, sha: source_sha } } + let(:forked_source_repo) { { id: 2, fork: true, name: 'otherproject', full_name: 'company/otherproject' } } let(:target_repo) { repository } - let(:target_branch) { double(ref: 'master', repo: target_repo, sha: target_sha, user: octocat) } - let(:removed_branch) { double(ref: 'removed-branch', repo: source_repo, sha: '2e5d3239642f9161dcbbc4b70a211a68e5e45e2b', user: octocat) } - let(:forked_branch) { double(ref: 'master', repo: forked_source_repo, sha: '2e5d3239642f9161dcbbc4b70a211a68e5e45e2b', user: octocat) } - let(:branch_deleted_repo) { double(ref: 'master', repo: nil, sha: '2e5d3239642f9161dcbbc4b70a211a68e5e45e2b', user: octocat) } - let(:octocat) { double(id: 123456, login: 'octocat', email: 'octocat@example.com') } + let(:target_branch) { { ref: 'master', repo: target_repo, sha: target_sha, user: octocat } } + let(:removed_branch) { { ref: 'removed-branch', repo: source_repo, sha: '2e5d3239642f9161dcbbc4b70a211a68e5e45e2b', user: octocat } } + let(:forked_branch) { { ref: 'master', repo: forked_source_repo, sha: '2e5d3239642f9161dcbbc4b70a211a68e5e45e2b', user: octocat } } + let(:branch_deleted_repo) { { ref: 'master', repo: nil, sha: '2e5d3239642f9161dcbbc4b70a211a68e5e45e2b', user: octocat } } + let(:octocat) { { id: 123456, login: 'octocat', email: 'octocat@example.com' } } let(:created_at) { DateTime.strptime('2011-01-26T19:01:12Z') } let(:updated_at) { DateTime.strptime('2011-01-27T19:01:12Z') } let(:base_data) do @@ -48,7 +48,7 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter do shared_examples 'Gitlab::LegacyGithubImport::PullRequestFormatter#attributes' do context 'when pull request is open' do - let(:raw_data) { double(base_data.merge(state: 'open')) } + let(:raw_data) { base_data.merge(state: 'open') } it 'returns formatted attributes' do expected = { @@ -75,7 +75,7 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter do end context 'when pull request is closed' do - let(:raw_data) { double(base_data.merge(state: 'closed')) } + let(:raw_data) { base_data.merge(state: 'closed') } it 'returns formatted attributes' do expected = { @@ -103,7 +103,7 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter do context 'when pull request is merged' do let(:merged_at) { DateTime.strptime('2011-01-28T13:01:12Z') } - let(:raw_data) { double(base_data.merge(state: 'closed', merged_at: merged_at)) } + let(:raw_data) { base_data.merge(state: 'closed', merged_at: merged_at) } it 'returns formatted attributes' do expected = { @@ -130,54 +130,54 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter do end context 'when it is assigned to someone' do - let(:raw_data) { double(base_data.merge(assignee: octocat)) } + let(:raw_data) { base_data.merge(assignee: octocat) } it 'returns nil as assignee_id when is not a GitLab user' do expect(pull_request.attributes.fetch(:assignee_id)).to be_nil end it 'returns GitLab user id associated with GitHub id as assignee_id' do - gl_user = create(:omniauth_user, extern_uid: octocat.id, provider: 'github') + gl_user = create(:omniauth_user, extern_uid: octocat[:id], provider: 'github') expect(pull_request.attributes.fetch(:assignee_id)).to eq gl_user.id end it 'returns GitLab user id associated with GitHub email as assignee_id' do - gl_user = create(:user, email: octocat.email) + gl_user = create(:user, email: octocat[:email]) expect(pull_request.attributes.fetch(:assignee_id)).to eq gl_user.id end end context 'when author is a GitLab user' do - let(:raw_data) { double(base_data.merge(user: octocat)) } + let(:raw_data) { base_data.merge(user: octocat) } it 'returns project creator_id as author_id when is not a GitLab user' do expect(pull_request.attributes.fetch(:author_id)).to eq project.creator_id end it 'returns GitLab user id associated with GitHub id as author_id' do - gl_user = create(:omniauth_user, extern_uid: octocat.id, provider: 'github') + gl_user = create(:omniauth_user, extern_uid: octocat[:id], provider: 'github') expect(pull_request.attributes.fetch(:author_id)).to eq gl_user.id end it 'returns GitLab user id associated with GitHub email as author_id' do - gl_user = create(:user, email: octocat.email) + gl_user = create(:user, email: octocat[:email]) expect(pull_request.attributes.fetch(:author_id)).to eq gl_user.id end it 'returns description without created at tag line' do - create(:omniauth_user, extern_uid: octocat.id, provider: 'github') + create(:omniauth_user, extern_uid: octocat[:id], provider: 'github') expect(pull_request.attributes.fetch(:description)).to eq('Please pull these awesome changes') end end context 'when it has a milestone' do - let(:milestone) { double(id: 42, number: 42) } - let(:raw_data) { double(base_data.merge(milestone: milestone)) } + let(:milestone) { { id: 42, number: 42 } } + let(:raw_data) { base_data.merge(milestone: milestone) } it 'returns nil when milestone does not exist' do expect(pull_request.attributes.fetch(:milestone)).to be_nil @@ -192,7 +192,7 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter do end shared_examples 'Gitlab::LegacyGithubImport::PullRequestFormatter#number' do - let(:raw_data) { double(base_data) } + let(:raw_data) { base_data } it 'returns pull request number' do expect(pull_request.number).to eq 1347 @@ -201,7 +201,7 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter do shared_examples 'Gitlab::LegacyGithubImport::PullRequestFormatter#source_branch_name' do context 'when source branch exists' do - let(:raw_data) { double(base_data) } + let(:raw_data) { base_data } it 'returns branch ref' do expect(pull_request.source_branch_name).to eq 'branch-merged' @@ -209,7 +209,7 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter do end context 'when source branch does not exist' do - let(:raw_data) { double(base_data.merge(head: removed_branch)) } + let(:raw_data) { base_data.merge(head: removed_branch) } it 'prefixes branch name with gh-:short_sha/:number/:user pattern to avoid collision' do expect(pull_request.source_branch_name).to eq "gh-#{target_short_sha}/1347/octocat/removed-branch" @@ -217,7 +217,7 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter do end context 'when source branch is from a fork' do - let(:raw_data) { double(base_data.merge(head: forked_branch)) } + let(:raw_data) { base_data.merge(head: forked_branch) } it 'prefixes branch name with gh-:short_sha/:number/:user pattern to avoid collision' do expect(pull_request.source_branch_name).to eq "gh-#{target_short_sha}/1347/octocat/master" @@ -225,7 +225,7 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter do end context 'when source branch is from a deleted fork' do - let(:raw_data) { double(base_data.merge(head: branch_deleted_repo)) } + let(:raw_data) { base_data.merge(head: branch_deleted_repo) } it 'prefixes branch name with gh-:short_sha/:number/:user pattern to avoid collision' do expect(pull_request.source_branch_name).to eq "gh-#{target_short_sha}/1347/octocat/master" @@ -235,7 +235,7 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter do shared_examples 'Gitlab::LegacyGithubImport::PullRequestFormatter#target_branch_name' do context 'when target branch exists' do - let(:raw_data) { double(base_data) } + let(:raw_data) { base_data } it 'returns branch ref' do expect(pull_request.target_branch_name).to eq 'master' @@ -243,7 +243,7 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter do end context 'when target branch does not exist' do - let(:raw_data) { double(base_data.merge(base: removed_branch)) } + let(:raw_data) { base_data.merge(base: removed_branch) } it 'prefixes branch name with gh-:short_sha/:number/:user pattern to avoid collision' do expect(pull_request.target_branch_name).to eq 'gl-2e5d3239/1347/octocat/removed-branch' @@ -271,7 +271,7 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter do describe '#valid?' do context 'when source, and target repos are not a fork' do - let(:raw_data) { double(base_data) } + let(:raw_data) { base_data } it 'returns true' do expect(pull_request.valid?).to eq true @@ -279,8 +279,8 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter do end context 'when source repo is a fork' do - let(:source_repo) { double(id: 2) } - let(:raw_data) { double(base_data) } + let(:source_repo) { { id: 2 } } + let(:raw_data) { base_data } it 'returns true' do expect(pull_request.valid?).to eq true @@ -288,8 +288,8 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter do end context 'when target repo is a fork' do - let(:target_repo) { double(id: 2) } - let(:raw_data) { double(base_data) } + let(:target_repo) { { id: 2 } } + let(:raw_data) { base_data } it 'returns true' do expect(pull_request.valid?).to eq true @@ -299,7 +299,7 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter do describe '#cross_project?' do context 'when source and target repositories are different' do - let(:raw_data) { double(base_data.merge(head: forked_branch)) } + let(:raw_data) { base_data.merge(head: forked_branch) } it 'returns true' do expect(pull_request.cross_project?).to eq true @@ -307,7 +307,7 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter do end context 'when source repository does not exist anymore' do - let(:raw_data) { double(base_data.merge(head: branch_deleted_repo)) } + let(:raw_data) { base_data.merge(head: branch_deleted_repo) } it 'returns true' do expect(pull_request.cross_project?).to eq true @@ -315,7 +315,7 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter do end context 'when source and target repositories are the same' do - let(:raw_data) { double(base_data.merge(head: source_branch)) } + let(:raw_data) { base_data.merge(head: source_branch) } it 'returns false' do expect(pull_request.cross_project?).to eq false @@ -324,7 +324,7 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter do end describe '#source_branch_exists?' do - let(:raw_data) { double(base_data.merge(head: forked_branch)) } + let(:raw_data) { base_data.merge(head: forked_branch) } it 'returns false when is a cross_project' do expect(pull_request.source_branch_exists?).to eq false @@ -332,7 +332,7 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter do end describe '#url' do - let(:raw_data) { double(base_data) } + let(:raw_data) { base_data } it 'return raw url' do expect(pull_request.url).to eq 'https://api.github.com/repos/octocat/Hello-World/pulls/1347' @@ -340,7 +340,7 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter do end describe '#opened?' do - let(:raw_data) { double(base_data.merge(state: 'open')) } + let(:raw_data) { base_data.merge(state: 'open') } it 'returns true when state is "open"' do expect(pull_request.opened?).to be_truthy diff --git a/spec/lib/gitlab/legacy_github_import/release_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/release_formatter_spec.rb index cbd1a30c417..237646f81dc 100644 --- a/spec/lib/gitlab/legacy_github_import/release_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/release_formatter_spec.rb @@ -3,8 +3,8 @@ require 'spec_helper' RSpec.describe Gitlab::LegacyGithubImport::ReleaseFormatter do - let!(:project) { create(:project, namespace: create(:namespace, path: 'octocat')) } - let(:octocat) { double(id: 123456, login: 'octocat') } + let_it_be(:project) { create(:project, namespace: create(:namespace, path: 'octocat')) } + let(:octocat) { { id: 123456, login: 'octocat' } } let(:created_at) { DateTime.strptime('2011-01-26T19:01:12Z') } let(:published_at) { DateTime.strptime('2011-01-26T20:00:00Z') } @@ -22,7 +22,7 @@ RSpec.describe Gitlab::LegacyGithubImport::ReleaseFormatter do subject(:release) { described_class.new(project, raw_data) } describe '#attributes' do - let(:raw_data) { double(base_data) } + let(:raw_data) { base_data } it 'returns formatted attributes' do expected = { @@ -49,7 +49,7 @@ RSpec.describe Gitlab::LegacyGithubImport::ReleaseFormatter do describe '#valid' do context 'when release is not a draft' do - let(:raw_data) { double(base_data) } + let(:raw_data) { base_data } it 'returns true' do expect(release.valid?).to eq true @@ -57,7 +57,7 @@ RSpec.describe Gitlab::LegacyGithubImport::ReleaseFormatter do end context 'when release is draft' do - let(:raw_data) { double(base_data.merge(draft: true)) } + let(:raw_data) { base_data.merge(draft: true) } it 'returns false' do expect(release.valid?).to eq false @@ -65,7 +65,7 @@ RSpec.describe Gitlab::LegacyGithubImport::ReleaseFormatter do end context 'when release has NULL tag' do - let(:raw_data) { double(base_data.merge(tag_name: '')) } + let(:raw_data) { base_data.merge(tag_name: '') } it 'returns false' do expect(release.valid?).to eq false diff --git a/spec/lib/gitlab/legacy_github_import/user_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/user_formatter_spec.rb index ab3ffddc042..bc127f74e84 100644 --- a/spec/lib/gitlab/legacy_github_import/user_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/user_formatter_spec.rb @@ -4,7 +4,7 @@ require 'spec_helper' RSpec.describe Gitlab::LegacyGithubImport::UserFormatter do let(:client) { double } - let(:octocat) { double(id: 123456, login: 'octocat', email: 'octocat@example.com') } + let(:octocat) { { id: 123456, login: 'octocat', email: 'octocat@example.com' } } subject(:user) { described_class.new(client, octocat) } @@ -15,33 +15,33 @@ RSpec.describe Gitlab::LegacyGithubImport::UserFormatter do describe '#gitlab_id' do context 'when GitHub user is a GitLab user' do it 'return GitLab user id when user associated their account with GitHub' do - gl_user = create(:omniauth_user, extern_uid: octocat.id, provider: 'github') + gl_user = create(:omniauth_user, extern_uid: octocat[:id], provider: 'github') expect(user.gitlab_id).to eq gl_user.id end it 'returns GitLab user id when user confirmed primary email matches GitHub email' do - gl_user = create(:user, email: octocat.email) + gl_user = create(:user, email: octocat[:email]) expect(user.gitlab_id).to eq gl_user.id end it 'returns GitLab user id when user unconfirmed primary email matches GitHub email' do - gl_user = create(:user, :unconfirmed, email: octocat.email) + gl_user = create(:user, :unconfirmed, email: octocat[:email]) expect(user.gitlab_id).to eq gl_user.id end it 'returns GitLab user id when user confirmed secondary email matches GitHub email' do gl_user = create(:user, email: 'johndoe@example.com') - create(:email, :confirmed, user: gl_user, email: octocat.email) + create(:email, :confirmed, user: gl_user, email: octocat[:email]) expect(user.gitlab_id).to eq gl_user.id end it 'returns nil when user unconfirmed secondary email matches GitHub email' do gl_user = create(:user, email: 'johndoe@example.com') - create(:email, user: gl_user, email: octocat.email) + create(:email, user: gl_user, email: octocat[:email]) expect(user.gitlab_id).to be_nil end diff --git a/spec/lib/gitlab/memory/diagnostic_reports_logger_spec.rb b/spec/lib/gitlab/memory/diagnostic_reports_logger_spec.rb new file mode 100644 index 00000000000..6be528e34b6 --- /dev/null +++ b/spec/lib/gitlab/memory/diagnostic_reports_logger_spec.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Memory::DiagnosticReportsLogger do + subject { described_class.new('/dev/null') } + + let(:now) { Time.current } + + describe '#format_message' do + it 'formats incoming hash properly' do + output = subject.format_message('INFO', now, 'test', { hello: 1 }) + # Disabling the cop because it is not relevant, we encode with `JSON.generate`. Allows `fast_spec_helper`. + data = JSON.parse(output) # rubocop: disable Gitlab/Json + + expect(data['severity']).to eq('INFO') + expect(data['time']).to eq(now.utc.iso8601(3)) + expect(data['hello']).to eq(1) + expect(data['message']).to be_nil + end + end +end diff --git a/spec/lib/gitlab/memory/reports_daemon_spec.rb b/spec/lib/gitlab/memory/reports_daemon_spec.rb index c9562470971..0473e170502 100644 --- a/spec/lib/gitlab/memory/reports_daemon_spec.rb +++ b/spec/lib/gitlab/memory/reports_daemon_spec.rb @@ -2,9 +2,15 @@ require 'spec_helper' -RSpec.describe Gitlab::Memory::ReportsDaemon do +RSpec.describe Gitlab::Memory::ReportsDaemon, :aggregate_failures do let(:daemon) { described_class.new } + let_it_be(:tmp_dir) { Dir.mktmpdir } + + after(:all) do + FileUtils.remove_entry(tmp_dir) + end + describe '#run_thread' do let(:report_duration_counter) { instance_double(::Prometheus::Client::Counter) } let(:file_size) { 1_000_000 } @@ -22,13 +28,10 @@ RSpec.describe Gitlab::Memory::ReportsDaemon do allow(File).to receive(:size).with(/#{daemon.reports_path}.*\.json/).and_return(file_size) end - it 'runs reports' do - expect(daemon.send(:reports)).to all(receive(:run).twice.and_call_original) - - daemon.send(:run_thread) - end + it 'runs reports, logs and sets gauge' do + expect(daemon.send(:reports)) + .to all(receive(:run).twice { Tempfile.new("report.json", tmp_dir).path }) - it 'logs report execution' do expect(::Prometheus::PidProvider).to receive(:worker_id).at_least(:once).and_return('worker_1') expect(Gitlab::AppLogger).to receive(:info).with( @@ -42,6 +45,8 @@ RSpec.describe Gitlab::Memory::ReportsDaemon do perf_report: 'jemalloc_stats' )).twice + expect(report_duration_counter).to receive(:increment).with({ report: 'jemalloc_stats' }, an_instance_of(Float)) + daemon.send(:run_thread) end @@ -51,18 +56,15 @@ RSpec.describe Gitlab::Memory::ReportsDaemon do end it 'logs `0` as `perf_report_size_bytes`' do + expect(daemon.send(:reports)) + .to all(receive(:run).twice { Tempfile.new("report.json", tmp_dir).path }) + expect(Gitlab::AppLogger).to receive(:info).with(hash_including(perf_report_size_bytes: 0)).twice daemon.send(:run_thread) end end - it 'sets real time duration gauge' do - expect(report_duration_counter).to receive(:increment).with({ report: 'jemalloc_stats' }, an_instance_of(Float)) - - daemon.send(:run_thread) - end - it 'allows configure and run multiple reports' do # rubocop: disable RSpec/VerifiedDoubles # We test how ReportsDaemon could be extended in the future @@ -74,8 +76,8 @@ RSpec.describe Gitlab::Memory::ReportsDaemon do allow(daemon).to receive(:reports).and_return([active_report_1, inactive_report, active_report_2]) - expect(active_report_1).to receive(:run).and_return('/tmp/report_1.json').twice - expect(active_report_2).to receive(:run).and_return('/tmp/report_2.json').twice + expect(active_report_1).to receive(:run).and_return(File.join(tmp_dir, 'report_1.json')).twice + expect(active_report_2).to receive(:run).and_return(File.join(tmp_dir, 'report_2.json')).twice expect(inactive_report).not_to receive(:run) daemon.send(:run_thread) @@ -87,6 +89,9 @@ RSpec.describe Gitlab::Memory::ReportsDaemon do daemon = described_class.new allow(daemon).to receive(:alive).and_return(true, true, false) + expect(daemon.send(:reports)) + .to all(receive(:run).twice { Tempfile.new("report.json", tmp_dir).path }) + expect(daemon).to receive(:sleep).with(described_class::DEFAULT_SLEEP_S).ordered expect(daemon).to receive(:sleep).with(described_class::DEFAULT_SLEEP_BETWEEN_REPORTS_S).ordered expect(daemon).to receive(:sleep).with(described_class::DEFAULT_SLEEP_S).ordered @@ -120,7 +125,7 @@ RSpec.describe Gitlab::Memory::ReportsDaemon do stub_env('GITLAB_DIAGNOSTIC_REPORTS_SLEEP_S', 100) stub_env('GITLAB_DIAGNOSTIC_REPORTS_SLEEP_MAX_DELTA_S', 50) stub_env('GITLAB_DIAGNOSTIC_REPORTS_SLEEP_BETWEEN_REPORTS_S', 2) - stub_env('GITLAB_DIAGNOSTIC_REPORTS_PATH', '/empty-dir') + stub_env('GITLAB_DIAGNOSTIC_REPORTS_PATH', tmp_dir) end it 'uses provided values' do @@ -129,7 +134,7 @@ RSpec.describe Gitlab::Memory::ReportsDaemon do expect(daemon.sleep_s).to eq(100) expect(daemon.sleep_max_delta_s).to eq(50) expect(daemon.sleep_between_reports_s).to eq(2) - expect(daemon.reports_path).to eq('/empty-dir') + expect(daemon.reports_path).to eq(tmp_dir) end end end diff --git a/spec/lib/gitlab/memory/reports_uploader_spec.rb b/spec/lib/gitlab/memory/reports_uploader_spec.rb new file mode 100644 index 00000000000..9ff830716f2 --- /dev/null +++ b/spec/lib/gitlab/memory/reports_uploader_spec.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Memory::ReportsUploader, :aggregate_failures do + let(:gcs_key) { 'test_gcs_key' } + let(:gcs_project) { 'test_gcs_project' } + let(:gcs_bucket) { 'test_gcs_bucket' } + let(:logger) { instance_double(Gitlab::Memory::DiagnosticReportsLogger) } + + let(:uploader) do + described_class.new(gcs_key: gcs_key, gcs_project: gcs_project, gcs_bucket: gcs_bucket, logger: logger) + end + + # rubocop: disable RSpec/VerifiedDoubles + # `Fog::Storage::Google` does not implement `put_object` itself, so it is tricky to pinpoint particular method + # with instance_double without revealing `Fog::Storage::Google` internals. For simplicity, we use a simple double. + let(:fog) { double("Fog::Storage::Google") } + # rubocop: enable RSpec/VerifiedDoubles + + let(:report) { Tempfile.new("report.1.worker_1.#{Time.current.to_i}.json") } + + after do + FileUtils.remove_entry(report) + end + + describe '#upload' do + before do + allow(Fog::Storage::Google) + .to receive(:new) + .with(google_project: gcs_project, google_json_key_location: gcs_key) + .and_return(fog) + end + + it 'calls fog, logs upload requested and success with duration' do + expect(logger) + .to receive(:info) + .with(hash_including(:pid, message: "Diagnostic reports", perf_report_status: "upload requested", + class: 'Gitlab::Memory::ReportsUploader', perf_report_path: report.path)) + .ordered + + expect(fog).to receive(:put_object).with(gcs_bucket, File.basename(report), instance_of(File)) + + expect(logger) + .to receive(:info) + .with(hash_including(:pid, :duration_s, + message: "Diagnostic reports", perf_report_status: "upload success", + class: 'Gitlab::Memory::ReportsUploader', perf_report_path: report.path)) + .ordered + + uploader.upload(report.path) + end + + context 'when Google API responds with an error' do + let(:invalid_bucket) { 'WRONG BUCKET' } + + let(:uploader) do + described_class.new(gcs_key: gcs_key, gcs_project: gcs_project, gcs_bucket: invalid_bucket, logger: logger) + end + + it 'logs error raised by Fog and do not re-raise' do + expect(logger) + .to receive(:info) + .with(hash_including(:pid, message: "Diagnostic reports", perf_report_status: "upload requested", + class: 'Gitlab::Memory::ReportsUploader', perf_report_path: report.path)) + + expect(fog).to receive(:put_object).with(invalid_bucket, File.basename(report), instance_of(File)) + .and_raise(Google::Apis::ClientError.new("invalid: Invalid bucket name: #{invalid_bucket}")) + + expect(logger) + .to receive(:error) + .with(hash_including(:pid, + message: "Diagnostic reports", class: 'Gitlab::Memory::ReportsUploader', + perf_report_status: 'error', error: "invalid: Invalid bucket name: #{invalid_bucket}")) + + expect { uploader.upload(report.path) }.not_to raise_error + end + end + end +end diff --git a/spec/lib/gitlab/memory/upload_and_cleanup_reports_spec.rb b/spec/lib/gitlab/memory/upload_and_cleanup_reports_spec.rb new file mode 100644 index 00000000000..f3351b276cc --- /dev/null +++ b/spec/lib/gitlab/memory/upload_and_cleanup_reports_spec.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Memory::UploadAndCleanupReports, :aggregate_failures do + let(:uploader) { instance_double(Gitlab::Memory::ReportsUploader) } + let(:logger) { instance_double(Gitlab::Memory::DiagnosticReportsLogger) } + + describe '#initalize' do + let(:reports_path) { '/path/to/reports' } + + context 'when sleep_time_seconds is passed through the environment' do + before do + stub_env('GITLAB_DIAGNOSTIC_REPORTS_UPLOADER_SLEEP_S', '600') + end + + it 'initializes with these settings' do + upload_and_cleanup = described_class.new(uploader: uploader, reports_path: reports_path, logger: logger) + + expect(upload_and_cleanup.sleep_time_seconds).to eq(600) + end + end + + context 'when sleep_time_seconds is passed through the initializer' do + it 'initializes with these settings' do + upload_and_cleanup = described_class.new(uploader: uploader, reports_path: reports_path, sleep_time_seconds: 60, + logger: logger) + + expect(upload_and_cleanup.sleep_time_seconds).to eq(60) + end + end + + context 'when `sleep_time_seconds` is not passed' do + it 'initialized with the default' do + upload_and_cleanup = described_class.new(uploader: uploader, reports_path: reports_path, logger: logger) + + expect(upload_and_cleanup.sleep_time_seconds).to eq(described_class::DEFAULT_SLEEP_TIME_SECONDS) + end + end + end + + describe '#call' do + let(:upload_and_cleanup) do + described_class.new(sleep_time_seconds: 600, reports_path: dir, uploader: uploader, + logger: logger).tap do |instance| + allow(instance).to receive(:loop).and_yield + allow(instance).to receive(:sleep) + end + end + + let(:dir) { Dir.mktmpdir } + + let(:reports_count) { 3 } + + let(:reports) do + (1..reports_count).map do |i| + Tempfile.new("report.1.worker_#{i}.#{Time.current.to_i}.json", dir) + end + end + + after do + FileUtils.remove_entry(dir) + end + + it 'invokes the uploader and cleans the files' do + expect(logger) + .to receive(:info) + .with(hash_including(:pid, + message: "Diagnostic reports", + class: 'Gitlab::Memory::UploadAndCleanupReports', + perf_report_status: 'started')) + + reports.each do |report| + expect(upload_and_cleanup.uploader).to receive(:upload).with(report.path) + end + + expect { upload_and_cleanup.call } + .to change { Dir.entries(dir).count { |e| e.match(/report.*/) } } + .from(reports_count).to(0) + end + + context 'when there is an exception' do + let(:report) { Tempfile.new("report.1.worker_1.#{Time.current.to_i}.json", dir) } + + it 'logs it and does not crash the loop' do + expect(logger) + .to receive(:info) + .with(hash_including(:pid, + message: "Diagnostic reports", + class: 'Gitlab::Memory::UploadAndCleanupReports', + perf_report_status: 'started')) + .ordered + + expect(upload_and_cleanup.uploader) + .to receive(:upload) + .with(report.path) + .and_raise(StandardError, 'Error Message') + + expect(logger) + .to receive(:error) + .with(hash_including(:pid, message: "Diagnostic reports", class: 'Gitlab::Memory::UploadAndCleanupReports', + perf_report_status: 'error', error: 'Error Message')) + .ordered + + expect { upload_and_cleanup.call }.not_to raise_error + end + end + end +end diff --git a/spec/lib/gitlab/memory/watchdog/configuration_spec.rb b/spec/lib/gitlab/memory/watchdog/configuration_spec.rb new file mode 100644 index 00000000000..892a4b06ad0 --- /dev/null +++ b/spec/lib/gitlab/memory/watchdog/configuration_spec.rb @@ -0,0 +1,121 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' +require_dependency 'gitlab/cluster/lifecycle_events' + +RSpec.describe Gitlab::Memory::Watchdog::Configuration do + subject(:configuration) { described_class.new } + + describe '#initialize' do + it 'initialize monitors' do + expect(configuration.monitors).to be_an_instance_of(described_class::MonitorStack) + end + end + + describe '#handler' do + context 'when handler is not set' do + it 'defaults to NullHandler' do + expect(configuration.handler).to be(Gitlab::Memory::Watchdog::NullHandler.instance) + end + end + end + + describe '#logger' do + context 'when logger is not set, defaults to stdout logger' do + it 'defaults to Logger' do + expect(configuration.logger).to be_an_instance_of(::Gitlab::Logger) + end + end + end + + describe '#sleep_time_seconds' do + context 'when sleep_time_seconds is not set' do + it 'defaults to SLEEP_TIME_SECONDS' do + expect(configuration.sleep_time_seconds).to eq(described_class::DEFAULT_SLEEP_TIME_SECONDS) + end + end + end + + describe '#monitors' do + context 'when monitors are configured to be used' do + let(:payload1) do + { + message: 'monitor_1_text', + memwd_max_strikes: 5, + memwd_cur_strikes: 0 + } + end + + let(:payload2) do + { + message: 'monitor_2_text', + memwd_max_strikes: 0, + memwd_cur_strikes: 1 + } + end + + let(:monitor_class_1) do + Struct.new(:threshold_violated, :payload) do + def call + { threshold_violated: !!threshold_violated, payload: payload || {} } + end + + def self.name + 'Monitor1' + end + end + end + + let(:monitor_class_2) do + Struct.new(:threshold_violated, :payload) do + def call + { threshold_violated: !!threshold_violated, payload: payload || {} } + end + + def self.name + 'Monitor2' + end + end + end + + context 'when two monitors are configured to be used' do + before do + configuration.monitors.use monitor_class_1, false, { message: 'monitor_1_text' }, max_strikes: 5 + configuration.monitors.use monitor_class_2, true, { message: 'monitor_2_text' }, max_strikes: 0 + end + + it 'calls each monitor and returns correct results', :aggregate_failures do + payloads = [] + thresholds = [] + strikes = [] + monitor_names = [] + + configuration.monitors.call_each do |result| + payloads << result.payload + thresholds << result.threshold_violated? + strikes << result.strikes_exceeded? + monitor_names << result.monitor_name + end + + expect(payloads).to eq([payload1, payload2]) + expect(thresholds).to eq([false, true]) + expect(strikes).to eq([false, true]) + expect(monitor_names).to eq([:monitor1, :monitor2]) + end + end + + context 'when same monitor class is configured to be used twice' do + before do + configuration.monitors.use monitor_class_1, max_strikes: 1 + configuration.monitors.use monitor_class_1, max_strikes: 1 + end + + it 'calls same monitor only once' do + expect do |b| + configuration.monitors.call_each(&b) + end.to yield_control.once + end + end + end + end +end diff --git a/spec/lib/gitlab/memory/watchdog/monitor/heap_fragmentation_spec.rb b/spec/lib/gitlab/memory/watchdog/monitor/heap_fragmentation_spec.rb new file mode 100644 index 00000000000..dad19cfd588 --- /dev/null +++ b/spec/lib/gitlab/memory/watchdog/monitor/heap_fragmentation_spec.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' +require 'support/shared_examples/lib/gitlab/memory/watchdog/monitor_result_shared_examples' +require 'prometheus/client' + +RSpec.describe Gitlab::Memory::Watchdog::Monitor::HeapFragmentation do + let(:heap_frag_limit_gauge) { instance_double(::Prometheus::Client::Gauge) } + let(:max_heap_fragmentation) { 0.2 } + let(:fragmentation) { 0.3 } + + subject(:monitor) do + described_class.new(max_heap_fragmentation: max_heap_fragmentation) + end + + before do + allow(Gitlab::Metrics).to receive(:gauge) + .with(:gitlab_memwd_heap_frag_limit, anything) + .and_return(heap_frag_limit_gauge) + allow(heap_frag_limit_gauge).to receive(:set) + + allow(Gitlab::Metrics::Memory).to receive(:gc_heap_fragmentation).and_return(fragmentation) + end + + describe '#initialize' do + it 'sets the heap fragmentation limit gauge' do + expect(heap_frag_limit_gauge).to receive(:set).with({}, max_heap_fragmentation) + + monitor + end + end + + describe '#call' do + it 'gets gc_heap_fragmentation' do + expect(Gitlab::Metrics::Memory).to receive(:gc_heap_fragmentation) + + monitor.call + end + + context 'when process exceeds threshold' do + let(:fragmentation) { max_heap_fragmentation + 0.1 } + let(:payload) do + { + message: 'heap fragmentation limit exceeded', + memwd_cur_heap_frag: fragmentation, + memwd_max_heap_frag: max_heap_fragmentation + } + end + + include_examples 'returns Watchdog Monitor result', threshold_violated: true + end + + context 'when process does not exceed threshold' do + let(:fragmentation) { max_heap_fragmentation - 0.1 } + let(:payload) { {} } + + include_examples 'returns Watchdog Monitor result', threshold_violated: false + end + end +end diff --git a/spec/lib/gitlab/memory/watchdog/monitor/unique_memory_growth_spec.rb b/spec/lib/gitlab/memory/watchdog/monitor/unique_memory_growth_spec.rb new file mode 100644 index 00000000000..22494af4425 --- /dev/null +++ b/spec/lib/gitlab/memory/watchdog/monitor/unique_memory_growth_spec.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' +require 'support/shared_examples/lib/gitlab/memory/watchdog/monitor_result_shared_examples' +require_dependency 'gitlab/cluster/lifecycle_events' + +RSpec.describe Gitlab::Memory::Watchdog::Monitor::UniqueMemoryGrowth do + let(:primary_memory) { 2048 } + let(:worker_memory) { 0 } + let(:max_mem_growth) { 2 } + + subject(:monitor) do + described_class.new(max_mem_growth: max_mem_growth) + end + + before do + allow(Gitlab::Metrics::System).to receive(:memory_usage_uss_pss).and_return({ uss: worker_memory }) + allow(Gitlab::Metrics::System).to receive(:memory_usage_uss_pss).with( + pid: Gitlab::Cluster::PRIMARY_PID + ).and_return({ uss: primary_memory }) + end + + describe '#call' do + it 'gets memory_usage_uss_pss' do + expect(Gitlab::Metrics::System).to receive(:memory_usage_uss_pss).with(no_args) + expect(Gitlab::Metrics::System).to receive(:memory_usage_uss_pss).with(pid: Gitlab::Cluster::PRIMARY_PID) + + monitor.call + end + + context 'when monitor is called twice' do + it 'reference memory is calculated only once' do + expect(Gitlab::Metrics::System).to receive(:memory_usage_uss_pss).with(no_args).twice + expect(Gitlab::Metrics::System).to receive(:memory_usage_uss_pss).with(pid: Gitlab::Cluster::PRIMARY_PID).once + + monitor.call + monitor.call + end + end + + context 'when process exceeds threshold' do + let(:worker_memory) { max_mem_growth * primary_memory + 1 } + let(:payload) do + { + message: 'memory limit exceeded', + memwd_max_uss_bytes: max_mem_growth * primary_memory, + memwd_ref_uss_bytes: primary_memory, + memwd_uss_bytes: worker_memory + } + end + + include_examples 'returns Watchdog Monitor result', threshold_violated: true + end + + context 'when process does not exceed threshold' do + let(:worker_memory) { max_mem_growth * primary_memory - 1 } + let(:payload) { {} } + + include_examples 'returns Watchdog Monitor result', threshold_violated: false + end + end +end diff --git a/spec/lib/gitlab/memory/watchdog/monitor_state_spec.rb b/spec/lib/gitlab/memory/watchdog/monitor_state_spec.rb new file mode 100644 index 00000000000..ace1353c6e3 --- /dev/null +++ b/spec/lib/gitlab/memory/watchdog/monitor_state_spec.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Memory::Watchdog::MonitorState do + let(:max_strikes) { 2 } + let(:payload) { { message: 'DummyMessage' } } + let(:threshold_violated) { true } + let(:monitor) { monitor_class.new(threshold_violated, payload) } + let(:monitor_class) do + Struct.new(:threshold_violated, :payload) do + def call + { threshold_violated: threshold_violated, payload: payload } + end + + def self.name + 'MonitorName' + end + end + end + + subject(:monitor_state) { described_class.new(monitor, max_strikes: max_strikes) } + + shared_examples 'returns correct result' do + it 'returns correct result', :aggregate_failures do + result = monitor_state.call + + expect(result).to be_an_instance_of(described_class::Result) + expect(result.strikes_exceeded?).to eq(strikes_exceeded) + expect(result.threshold_violated?).to eq(threshold_violated) + expect(result.payload).to eq(expected_payload) + expect(result.monitor_name).to eq(:monitor_name) + end + end + + describe '#call' do + let(:strikes_exceeded) { false } + let(:curr_strikes) { 0 } + let(:expected_payload) do + { + memwd_max_strikes: max_strikes, + memwd_cur_strikes: curr_strikes + }.merge(payload) + end + + context 'when threshold is not violated' do + let(:threshold_violated) { false } + + include_examples 'returns correct result' + end + + context 'when threshold is violated' do + let(:curr_strikes) { 1 } + let(:threshold_violated) { true } + + include_examples 'returns correct result' + + context 'when strikes_exceeded' do + let(:max_strikes) { 0 } + let(:strikes_exceeded) { true } + + include_examples 'returns correct result' + end + end + end + + describe '#monitor_class' do + subject { monitor_state.monitor_class } + + it { is_expected.to eq(monitor_class) } + end +end diff --git a/spec/lib/gitlab/memory/watchdog_spec.rb b/spec/lib/gitlab/memory/watchdog_spec.rb index beb49660022..84e9a577afb 100644 --- a/spec/lib/gitlab/memory/watchdog_spec.rb +++ b/spec/lib/gitlab/memory/watchdog_spec.rb @@ -1,35 +1,35 @@ # frozen_string_literal: true require 'spec_helper' -require_relative '../../../../lib/gitlab/cluster/lifecycle_events' -RSpec.describe Gitlab::Memory::Watchdog, :aggregate_failures, :prometheus do +RSpec.describe Gitlab::Memory::Watchdog, :aggregate_failures do context 'watchdog' do - let(:logger) { instance_double(::Logger) } + let(:configuration) { instance_double(described_class::Configuration) } let(:handler) { instance_double(described_class::NullHandler) } - - let(:heap_frag_limit_gauge) { instance_double(::Prometheus::Client::Gauge) } + let(:logger) { instance_double(::Logger) } + let(:sleep_time_seconds) { 60 } + let(:threshold_violated) { false } let(:violations_counter) { instance_double(::Prometheus::Client::Counter) } let(:violations_handled_counter) { instance_double(::Prometheus::Client::Counter) } - - let(:sleep_time) { 0.1 } - let(:max_heap_fragmentation) { 0.2 } - let(:max_mem_growth) { 2 } - - # Defaults that will not trigger any events. - let(:fragmentation) { 0 } - let(:worker_memory) { 0 } - let(:primary_memory) { 0 } - let(:max_strikes) { 0 } - - # Tests should set this to control the number of loop iterations in `call`. let(:watchdog_iterations) { 1 } + let(:name) { :monitor_name } + let(:payload) { { message: 'dummy_text' } } + let(:max_strikes) { 2 } + let(:monitor_class) do + Struct.new(:threshold_violated, :payload) do + def call + { threshold_violated: threshold_violated, payload: payload } + end + + def self.name + 'MonitorName' + end + end + end subject(:watchdog) do - described_class.new(handler: handler, logger: logger, sleep_time_seconds: sleep_time, - max_strikes: max_strikes, max_mem_growth: max_mem_growth, - max_heap_fragmentation: max_heap_fragmentation).tap do |instance| - # We need to defuse `sleep` and stop the internal loop after N iterations. + described_class.new.tap do |instance| + # We need to defuse `sleep` and stop the internal loop after 1 iteration iterations = 0 allow(instance).to receive(:sleep) do instance.stop if (iterations += 1) > watchdog_iterations @@ -38,9 +38,6 @@ RSpec.describe Gitlab::Memory::Watchdog, :aggregate_failures, :prometheus do end def stub_prometheus_metrics - allow(Gitlab::Metrics).to receive(:gauge) - .with(:gitlab_memwd_heap_frag_limit, anything) - .and_return(heap_frag_limit_gauge) allow(Gitlab::Metrics).to receive(:counter) .with(:gitlab_memwd_violations_total, anything, anything) .and_return(violations_counter) @@ -48,318 +45,195 @@ RSpec.describe Gitlab::Memory::Watchdog, :aggregate_failures, :prometheus do .with(:gitlab_memwd_violations_handled_total, anything, anything) .and_return(violations_handled_counter) - allow(heap_frag_limit_gauge).to receive(:set) allow(violations_counter).to receive(:increment) allow(violations_handled_counter).to receive(:increment) end - before do - stub_prometheus_metrics - - allow(handler).to receive(:call).and_return(true) - - allow(logger).to receive(:warn) - allow(logger).to receive(:info) - - allow(Gitlab::Metrics::Memory).to receive(:gc_heap_fragmentation).and_return(fragmentation) - allow(Gitlab::Metrics::System).to receive(:memory_usage_uss_pss).and_return({ uss: worker_memory }) - allow(Gitlab::Metrics::System).to receive(:memory_usage_uss_pss).with( - pid: Gitlab::Cluster::PRIMARY_PID - ).and_return({ uss: primary_memory }) - - allow(::Prometheus::PidProvider).to receive(:worker_id).and_return('worker_1') - end - - context 'when created' do - it 'sets the heap fragmentation limit gauge' do - expect(heap_frag_limit_gauge).to receive(:set).with({}, max_heap_fragmentation) + describe '#initialize' do + it 'initialize new configuration' do + expect(described_class::Configuration).to receive(:new) watchdog end - - context 'when no settings are set in the environment' do - it 'initializes with defaults' do - watchdog = described_class.new(handler: handler, logger: logger) - - expect(watchdog.max_heap_fragmentation).to eq(described_class::DEFAULT_MAX_HEAP_FRAG) - expect(watchdog.max_mem_growth).to eq(described_class::DEFAULT_MAX_MEM_GROWTH) - expect(watchdog.max_strikes).to eq(described_class::DEFAULT_MAX_STRIKES) - expect(watchdog.sleep_time_seconds).to eq(described_class::DEFAULT_SLEEP_TIME_SECONDS) - end - end - - context 'when settings are passed through the environment' do - before do - stub_env('GITLAB_MEMWD_MAX_HEAP_FRAG', 1) - stub_env('GITLAB_MEMWD_MAX_STRIKES', 2) - stub_env('GITLAB_MEMWD_SLEEP_TIME_SEC', 3) - stub_env('GITLAB_MEMWD_MAX_MEM_GROWTH', 4) - end - - it 'initializes with these settings' do - watchdog = described_class.new(handler: handler, logger: logger) - - expect(watchdog.max_heap_fragmentation).to eq(1) - expect(watchdog.max_strikes).to eq(2) - expect(watchdog.sleep_time_seconds).to eq(3) - expect(watchdog.max_mem_growth).to eq(4) - end - end end - shared_examples 'has strikes left' do |stat| - context 'when process has not exceeded allowed number of strikes' do - let(:watchdog_iterations) { max_strikes } - - it 'does not signal the handler' do - expect(handler).not_to receive(:call) - - watchdog.call - end - - it 'does not log any events' do - expect(logger).not_to receive(:warn) - - watchdog.call - end - - it 'increments the violations counter' do - expect(violations_counter).to receive(:increment).with(reason: stat).exactly(watchdog_iterations) - - watchdog.call + describe '#call' do + before do + stub_prometheus_metrics + allow(Gitlab::Metrics::System).to receive(:memory_usage_rss).at_least(:once).and_return(1024) + allow(::Prometheus::PidProvider).to receive(:worker_id).and_return('worker_1') + + watchdog.configure do |config| + config.handler = handler + config.logger = logger + config.sleep_time_seconds = sleep_time_seconds + config.monitors.use monitor_class, threshold_violated, payload, max_strikes: max_strikes end - it 'does not increment violations handled counter' do - expect(violations_handled_counter).not_to receive(:increment) - - watchdog.call - end + allow(handler).to receive(:call).and_return(true) + allow(logger).to receive(:info) + allow(logger).to receive(:warn) end - end - shared_examples 'no strikes left' do |stat| - it 'signals the handler and resets strike counter' do - expect(handler).to receive(:call).and_return(true) + it 'logs start message once' do + expect(logger).to receive(:info).once + .with( + pid: Process.pid, + worker_id: 'worker_1', + memwd_handler_class: handler.class.name, + memwd_sleep_time_s: sleep_time_seconds, + memwd_rss_bytes: 1024, + message: 'started') watchdog.call - - expect(watchdog.strikes(stat.to_sym)).to eq(0) end - it 'increments both the violations and violations handled counters' do - expect(violations_counter).to receive(:increment).with(reason: stat).exactly(watchdog_iterations) - expect(violations_handled_counter).to receive(:increment).with(reason: stat) + it 'waits for check interval seconds' do + expect(watchdog).to receive(:sleep).with(sleep_time_seconds) watchdog.call end - context 'when enforce_memory_watchdog ops toggle is off' do + context 'when gitlab_memory_watchdog ops toggle is off' do before do - stub_feature_flags(enforce_memory_watchdog: false) + stub_feature_flags(gitlab_memory_watchdog: false) end - it 'always uses the NullHandler' do - expect(handler).not_to receive(:call) - expect(described_class::NullHandler.instance).to receive(:call).and_return(true) - - watchdog.call + it 'does not trigger any monitor' do + expect(configuration).not_to receive(:monitors) end end - context 'when handler result is true' do - it 'considers the event handled and stops itself' do - expect(handler).to receive(:call).once.and_return(true) - expect(logger).to receive(:info).with(hash_including(message: 'stopped')) + context 'when process does not exceed threshold' do + it 'does not increment violations counters' do + expect(violations_counter).not_to receive(:increment) + expect(violations_handled_counter).not_to receive(:increment) watchdog.call end - end - - context 'when handler result is false' do - let(:max_strikes) { 0 } # to make sure the handler fires each iteration - let(:watchdog_iterations) { 3 } - it 'keeps running' do - expect(violations_counter).to receive(:increment).exactly(watchdog_iterations) - expect(violations_handled_counter).to receive(:increment).exactly(watchdog_iterations) - # Return true the third time to terminate the daemon. - expect(handler).to receive(:call).and_return(false, false, true) + it 'does not log violation' do + expect(logger).not_to receive(:warn) watchdog.call end - end - end - - context 'when monitoring memory growth' do - let(:primary_memory) { 2048 } - - context 'when process does not exceed threshold' do - let(:worker_memory) { max_mem_growth * primary_memory - 1 } - it 'does not signal the handler' do + it 'does not execute handler' do expect(handler).not_to receive(:call) watchdog.call end end - context 'when process exceeds threshold permanently' do - let(:worker_memory) { max_mem_growth * primary_memory + 1 } - let(:max_strikes) { 3 } - - it_behaves_like 'has strikes left', 'mem_growth' + context 'when process exceeds threshold' do + let(:threshold_violated) { true } - context 'when process exceeds the allowed number of strikes' do - let(:watchdog_iterations) { max_strikes + 1 } + it 'increments violations counter' do + expect(violations_counter).to receive(:increment).with(reason: name) - it_behaves_like 'no strikes left', 'mem_growth' + watchdog.call + end - it 'only reads reference memory once' do - expect(Gitlab::Metrics::System).to receive(:memory_usage_uss_pss) - .with(pid: Gitlab::Cluster::PRIMARY_PID) - .once + context 'when process does not exceed the allowed number of strikes' do + it 'does not increment handled violations counter' do + expect(violations_handled_counter).not_to receive(:increment) watchdog.call end - it 'logs the event' do - expect(Gitlab::Metrics::System).to receive(:memory_usage_rss).at_least(:once).and_return(1024) - expect(logger).to receive(:warn).with({ - message: 'memory limit exceeded', - pid: Process.pid, - worker_id: 'worker_1', - memwd_handler_class: 'RSpec::Mocks::InstanceVerifyingDouble', - memwd_sleep_time_s: sleep_time, - memwd_max_uss_bytes: max_mem_growth * primary_memory, - memwd_ref_uss_bytes: primary_memory, - memwd_uss_bytes: worker_memory, - memwd_rss_bytes: 1024, - memwd_max_strikes: max_strikes, - memwd_cur_strikes: max_strikes + 1 - }) + it 'does not log violation' do + expect(logger).not_to receive(:warn) watchdog.call end - end - end - context 'when process exceeds threshold temporarily' do - let(:worker_memory) { max_mem_growth * primary_memory } - let(:max_strikes) { 1 } - let(:watchdog_iterations) { 4 } + it 'does not execute handler' do + expect(handler).not_to receive(:call) - before do - allow(Gitlab::Metrics::System).to receive(:memory_usage_uss_pss).and_return( - { uss: worker_memory - 0.1 }, - { uss: worker_memory + 0.2 }, - { uss: worker_memory - 0.1 }, - { uss: worker_memory + 0.1 } - ) - allow(Gitlab::Metrics::System).to receive(:memory_usage_uss_pss).with( - pid: Gitlab::Cluster::PRIMARY_PID - ).and_return({ uss: primary_memory }) + watchdog.call + end end - it 'does not signal the handler' do - expect(handler).not_to receive(:call) + context 'when monitor exceeds the allowed number of strikes' do + let(:max_strikes) { 0 } - watchdog.call - end - end - end + it 'increments handled violations counter' do + expect(violations_handled_counter).to receive(:increment).with(reason: name) - context 'when monitoring heap fragmentation' do - context 'when process does not exceed threshold' do - let(:fragmentation) { max_heap_fragmentation - 0.1 } - - it 'does not signal the handler' do - expect(handler).not_to receive(:call) - - watchdog.call - end - end - - context 'when process exceeds threshold permanently' do - let(:fragmentation) { max_heap_fragmentation + 0.1 } - let(:max_strikes) { 3 } - - it_behaves_like 'has strikes left', 'heap_frag' + watchdog.call + end - context 'when process exceeds the allowed number of strikes' do - let(:watchdog_iterations) { max_strikes + 1 } + it 'logs violation' do + expect(logger).to receive(:warn) + .with( + pid: Process.pid, + worker_id: 'worker_1', + memwd_handler_class: handler.class.name, + memwd_sleep_time_s: sleep_time_seconds, + memwd_rss_bytes: 1024, + memwd_cur_strikes: 1, + memwd_max_strikes: max_strikes, + message: 'dummy_text') - it_behaves_like 'no strikes left', 'heap_frag' + watchdog.call + end - it 'logs the event' do - expect(Gitlab::Metrics::System).to receive(:memory_usage_rss).at_least(:once).and_return(1024) - expect(logger).to receive(:warn).with({ - message: 'heap fragmentation limit exceeded', - pid: Process.pid, - worker_id: 'worker_1', - memwd_handler_class: 'RSpec::Mocks::InstanceVerifyingDouble', - memwd_sleep_time_s: sleep_time, - memwd_max_heap_frag: max_heap_fragmentation, - memwd_cur_heap_frag: fragmentation, - memwd_max_strikes: max_strikes, - memwd_cur_strikes: max_strikes + 1, - memwd_rss_bytes: 1024 - }) + it 'executes handler' do + expect(handler).to receive(:call) watchdog.call end - end - end - context 'when process exceeds threshold temporarily' do - let(:fragmentation) { max_heap_fragmentation } - let(:max_strikes) { 1 } - let(:watchdog_iterations) { 4 } + context 'when enforce_memory_watchdog ops toggle is off' do + before do + stub_feature_flags(enforce_memory_watchdog: false) + end - before do - allow(Gitlab::Metrics::Memory).to receive(:gc_heap_fragmentation).and_return( - fragmentation - 0.1, - fragmentation + 0.2, - fragmentation - 0.1, - fragmentation + 0.1 - ) - end + it 'always uses the NullHandler' do + expect(handler).not_to receive(:call) + expect(described_class::NullHandler.instance).to receive(:call).and_return(true) - it 'does not signal the handler' do - expect(handler).not_to receive(:call) + watchdog.call + end + end - watchdog.call + context 'when multiple monitors exceeds allowed number of strikes' do + before do + watchdog.configure do |config| + config.handler = handler + config.logger = logger + config.sleep_time_seconds = sleep_time_seconds + config.monitors.use monitor_class, threshold_violated, payload, max_strikes: max_strikes + config.monitors.use monitor_class, threshold_violated, payload, max_strikes: max_strikes + end + end + + it 'only calls the handler once' do + expect(handler).to receive(:call).once.and_return(true) + + watchdog.call + end + end end end - end - - context 'when both memory fragmentation and growth exceed thresholds' do - let(:fragmentation) { max_heap_fragmentation + 0.1 } - let(:primary_memory) { 2048 } - let(:worker_memory) { max_mem_growth * primary_memory + 1 } - let(:watchdog_iterations) { max_strikes + 1 } - it 'only calls the handler once' do - expect(handler).to receive(:call).once.and_return(true) + it 'logs stop message once' do + expect(logger).to receive(:info).once + .with( + pid: Process.pid, + worker_id: 'worker_1', + memwd_handler_class: handler.class.name, + memwd_sleep_time_s: sleep_time_seconds, + memwd_rss_bytes: 1024, + message: 'stopped') watchdog.call end end - context 'when gitlab_memory_watchdog ops toggle is off' do - before do - stub_feature_flags(gitlab_memory_watchdog: false) - end - - it 'does not monitor heap fragmentation' do - expect(Gitlab::Metrics::Memory).not_to receive(:gc_heap_fragmentation) - - watchdog.call - end - - it 'does not monitor memory growth' do - expect(Gitlab::Metrics::System).not_to receive(:memory_usage_uss_pss) - - watchdog.call + describe '#configure' do + it 'yields block' do + expect { |b| watchdog.configure(&b) }.to yield_control end end end diff --git a/spec/lib/gitlab/metrics/global_search_slis_spec.rb b/spec/lib/gitlab/metrics/global_search_slis_spec.rb index 28496eff2fc..0c09cf6dd71 100644 --- a/spec/lib/gitlab/metrics/global_search_slis_spec.rb +++ b/spec/lib/gitlab/metrics/global_search_slis_spec.rb @@ -5,26 +5,20 @@ require 'spec_helper' RSpec.describe Gitlab::Metrics::GlobalSearchSlis do using RSpec::Parameterized::TableSyntax - let(:apdex_feature_flag_enabled) { true } let(:error_rate_feature_flag_enabled) { true } before do - stub_feature_flags(global_search_custom_slis: apdex_feature_flag_enabled) stub_feature_flags(global_search_error_rate_sli: error_rate_feature_flag_enabled) end describe '#initialize_slis!' do - context 'when global_search_custom_slis feature flag is enabled' do - let(:apdex_feature_flag_enabled) { true } + it 'initializes Apdex SLIs for global_search' do + expect(Gitlab::Metrics::Sli::Apdex).to receive(:initialize_sli).with( + :global_search, + a_kind_of(Array) + ) - it 'initializes Apdex SLIs for global_search' do - expect(Gitlab::Metrics::Sli::Apdex).to receive(:initialize_sli).with( - :global_search, - a_kind_of(Array) - ) - - described_class.initialize_slis! - end + described_class.initialize_slis! end context 'when global_search_error_rate_sli feature flag is enabled' do @@ -40,16 +34,6 @@ RSpec.describe Gitlab::Metrics::GlobalSearchSlis do end end - context 'when global_search_custom_slis feature flag is disabled' do - let(:apdex_feature_flag_enabled) { false } - - it 'does not initialize the Apdex SLIs for global_search' do - expect(Gitlab::Metrics::Sli::Apdex).not_to receive(:initialize_sli) - - described_class.initialize_slis! - end - end - context 'when global_search_error_rate_sli feature flag is disabled' do let(:error_rate_feature_flag_enabled) { false } @@ -62,78 +46,59 @@ RSpec.describe Gitlab::Metrics::GlobalSearchSlis do end describe '#record_apdex' do - context 'when global_search_custom_slis feature flag is enabled' do - let(:apdex_feature_flag_enabled) { true } - - where(:search_type, :code_search, :duration_target) do - 'basic' | false | 7.031 - 'basic' | true | 21.903 - 'advanced' | false | 4.865 - 'advanced' | true | 13.546 - end - - with_them do - before do - allow(::Gitlab::ApplicationContext).to receive(:current_context_attribute).with(:caller_id).and_return('end') - end + where(:search_type, :code_search, :duration_target) do + 'basic' | false | 7.031 + 'basic' | true | 21.903 + 'advanced' | false | 4.865 + 'advanced' | true | 13.546 + end - let(:search_scope) { code_search ? 'blobs' : 'issues' } + with_them do + before do + allow(::Gitlab::ApplicationContext).to receive(:current_context_attribute).with(:caller_id).and_return('end') + end - it 'increments the global_search SLI as a success if the elapsed time is within the target' do - duration = duration_target - 0.1 + let(:search_scope) { code_search ? 'blobs' : 'issues' } - expect(Gitlab::Metrics::Sli::Apdex[:global_search]).to receive(:increment).with( - labels: { - search_type: search_type, - search_level: 'global', - search_scope: search_scope, - endpoint_id: 'end' - }, - success: true - ) + it 'increments the global_search SLI as a success if the elapsed time is within the target' do + duration = duration_target - 0.1 - described_class.record_apdex( - elapsed: duration, - search_type: search_type, - search_level: 'global', - search_scope: search_scope - ) - end - - it 'increments the global_search SLI as a failure if the elapsed time is not within the target' do - duration = duration_target + 0.1 - - expect(Gitlab::Metrics::Sli::Apdex[:global_search]).to receive(:increment).with( - labels: { - search_type: search_type, - search_level: 'global', - search_scope: search_scope, - endpoint_id: 'end' - }, - success: false - ) - - described_class.record_apdex( - elapsed: duration, + expect(Gitlab::Metrics::Sli::Apdex[:global_search]).to receive(:increment).with( + labels: { search_type: search_type, search_level: 'global', - search_scope: search_scope - ) - end + search_scope: search_scope, + endpoint_id: 'end' + }, + success: true + ) + + described_class.record_apdex( + elapsed: duration, + search_type: search_type, + search_level: 'global', + search_scope: search_scope + ) end - end - context 'when global_search_custom_slis feature flag is disabled' do - let(:apdex_feature_flag_enabled) { false } + it 'increments the global_search SLI as a failure if the elapsed time is not within the target' do + duration = duration_target + 0.1 - it 'does not call increment on the apdex SLI' do - expect(Gitlab::Metrics::Sli::Apdex[:global_search]).not_to receive(:increment) + expect(Gitlab::Metrics::Sli::Apdex[:global_search]).to receive(:increment).with( + labels: { + search_type: search_type, + search_level: 'global', + search_scope: search_scope, + endpoint_id: 'end' + }, + success: false + ) described_class.record_apdex( - elapsed: 1, - search_type: 'basic', + elapsed: duration, + search_type: search_type, search_level: 'global', - search_scope: 'issues' + search_scope: search_scope ) end end diff --git a/spec/lib/gitlab/metrics/system_spec.rb b/spec/lib/gitlab/metrics/system_spec.rb index 7739501dd95..b86469eacd1 100644 --- a/spec/lib/gitlab/metrics/system_spec.rb +++ b/spec/lib/gitlab/metrics/system_spec.rb @@ -71,6 +71,65 @@ RSpec.describe Gitlab::Metrics::System do SNIP end + let(:mem_info) do + # full snapshot + <<~SNIP + MemTotal: 15362536 kB + MemFree: 3403136 kB + MemAvailable: 13044528 kB + Buffers: 272188 kB + Cached: 8171312 kB + SwapCached: 0 kB + Active: 3332084 kB + Inactive: 6981076 kB + Active(anon): 1603868 kB + Inactive(anon): 9044 kB + Active(file): 1728216 kB + Inactive(file): 6972032 kB + Unevictable: 18676 kB + Mlocked: 18676 kB + SwapTotal: 0 kB + SwapFree: 0 kB + Dirty: 6808 kB + Writeback: 0 kB + AnonPages: 1888300 kB + Mapped: 166164 kB + Shmem: 12932 kB + KReclaimable: 1275120 kB + Slab: 1495480 kB + SReclaimable: 1275120 kB + SUnreclaim: 220360 kB + KernelStack: 7072 kB + PageTables: 11936 kB + NFS_Unstable: 0 kB + Bounce: 0 kB + WritebackTmp: 0 kB + CommitLimit: 7681268 kB + Committed_AS: 4976100 kB + VmallocTotal: 34359738367 kB + VmallocUsed: 25532 kB + VmallocChunk: 0 kB + Percpu: 23200 kB + HardwareCorrupted: 0 kB + AnonHugePages: 202752 kB + ShmemHugePages: 0 kB + ShmemPmdMapped: 0 kB + FileHugePages: 0 kB + FilePmdMapped: 0 kB + CmaTotal: 0 kB + CmaFree: 0 kB + HugePages_Total: 0 + HugePages_Free: 0 + HugePages_Rsvd: 0 + HugePages_Surp: 0 + Hugepagesize: 2048 kB + Hugetlb: 0 kB + DirectMap4k: 4637504 kB + DirectMap2M: 11087872 kB + DirectMap1G: 2097152 kB + SNIP + end + describe '.memory_usage_rss' do context 'without PID' do it "returns the current process' resident set size (RSS) in bytes" do @@ -125,6 +184,14 @@ RSpec.describe Gitlab::Metrics::System do end end + describe '.memory_total' do + it "returns the current process' resident set size (RSS) in bytes" do + mock_existing_proc_file('/proc/meminfo', mem_info) + + expect(described_class.memory_total).to eq(15731236864) + end + end + describe '.process_runtime_elapsed_seconds' do it 'returns the seconds elapsed since the process was started' do # sets process starttime ticks to 1000 diff --git a/spec/lib/gitlab/middleware/handle_malformed_strings_spec.rb b/spec/lib/gitlab/middleware/handle_malformed_strings_spec.rb index cf7b0dbb5fd..ed1440f23b6 100644 --- a/spec/lib/gitlab/middleware/handle_malformed_strings_spec.rb +++ b/spec/lib/gitlab/middleware/handle_malformed_strings_spec.rb @@ -132,11 +132,12 @@ RSpec.describe Gitlab::Middleware::HandleMalformedStrings do end it "rejects bad params for arrays containing hashes with string values" do - env = env_for(name: [ - { - inner_key: "I am #{problematic_input} bad" - } - ]) + env = env_for( + name: [ + { + inner_key: "I am #{problematic_input} bad" + } + ]) expect(subject.call(env)).to eq error_400 end @@ -148,11 +149,12 @@ RSpec.describe Gitlab::Middleware::HandleMalformedStrings do it_behaves_like 'checks params' it "gives up and does not reject too deeply nested params" do - env = env_for(name: [ - { - inner_key: { deeper_key: [{ hash_inside_array_key: "I am #{problematic_input} bad" }] } - } - ]) + env = env_for( + name: [ + { + inner_key: { deeper_key: [{ hash_inside_array_key: "I am #{problematic_input} bad" }] } + } + ]) expect(subject.call(env)).not_to eq error_400 end diff --git a/spec/lib/gitlab/pages/cache_control_spec.rb b/spec/lib/gitlab/pages/cache_control_spec.rb index 6ed823427fb..431c989e874 100644 --- a/spec/lib/gitlab/pages/cache_control_spec.rb +++ b/spec/lib/gitlab/pages/cache_control_spec.rb @@ -3,21 +3,16 @@ require 'spec_helper' RSpec.describe Gitlab::Pages::CacheControl do - it 'fails with invalid type' do - expect { described_class.new(type: :unknown, id: nil) } - .to raise_error(ArgumentError, "type must be :namespace or :project") - end - describe '.for_namespace' do - let(:subject) { described_class.for_namespace(1) } + subject(:cache_control) { described_class.for_namespace(1) } - it { expect(subject.cache_key).to eq('pages_domain_for_namespace_1') } + it { expect(subject.cache_key).to match(/pages_domain_for_namespace_1_*/) } describe '#clear_cache' do it 'clears the cache' do expect(Rails.cache) .to receive(:delete) - .with('pages_domain_for_namespace_1') + .with(/pages_domain_for_namespace_1_*/) subject.clear_cache end @@ -25,18 +20,48 @@ RSpec.describe Gitlab::Pages::CacheControl do end describe '.for_project' do - let(:subject) { described_class.for_project(1) } + subject(:cache_control) { described_class.for_project(1) } - it { expect(subject.cache_key).to eq('pages_domain_for_project_1') } + it { expect(subject.cache_key).to match(/pages_domain_for_project_1_*/) } describe '#clear_cache' do it 'clears the cache' do expect(Rails.cache) .to receive(:delete) - .with('pages_domain_for_project_1') + .with(/pages_domain_for_project_1_*/) subject.clear_cache end end end + + describe '#cache_key' do + it 'does not change the pages config' do + expect { described_class.new(type: :project, id: 1).cache_key } + .not_to change(Gitlab.config, :pages) + end + + it 'is based on pages settings' do + access_control = Gitlab.config.pages.access_control + cache_key = described_class.new(type: :project, id: 1).cache_key + + stub_config(pages: { access_control: !access_control }) + + expect(described_class.new(type: :project, id: 1).cache_key).not_to eq(cache_key) + end + + it 'is based on the force_pages_access_control settings' do + force_pages_access_control = ::Gitlab::CurrentSettings.force_pages_access_control + cache_key = described_class.new(type: :project, id: 1).cache_key + + ::Gitlab::CurrentSettings.force_pages_access_control = !force_pages_access_control + + expect(described_class.new(type: :project, id: 1).cache_key).not_to eq(cache_key) + end + end + + it 'fails with invalid type' do + expect { described_class.new(type: :unknown, id: nil) } + .to raise_error(ArgumentError, "type must be :namespace or :project") + end end diff --git a/spec/lib/gitlab/pagination/keyset/column_order_definition_spec.rb b/spec/lib/gitlab/pagination/keyset/column_order_definition_spec.rb index 100574cc75f..64bc4555bcc 100644 --- a/spec/lib/gitlab/pagination/keyset/column_order_definition_spec.rb +++ b/spec/lib/gitlab/pagination/keyset/column_order_definition_spec.rb @@ -23,10 +23,11 @@ RSpec.describe Gitlab::Pagination::Keyset::ColumnOrderDefinition do let_it_be(:project_calculated_column_expression) do # COALESCE("projects"."description", 'No Description') - Arel::Nodes::NamedFunction.new('COALESCE', [ - Project.arel_table[:description], - Arel.sql("'No Description'") - ]) + Arel::Nodes::NamedFunction.new('COALESCE', + [ + Project.arel_table[:description], + Arel.sql("'No Description'") + ]) end let_it_be(:project_calculated_column) do diff --git a/spec/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder_spec.rb b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder_spec.rb index 9f2ac9a953d..cc85c897019 100644 --- a/spec/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder_spec.rb +++ b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder_spec.rb @@ -117,23 +117,24 @@ RSpec.describe Gitlab::Pagination::Keyset::InOperatorOptimization::QueryBuilder let(:order) do # NULLS LAST ordering requires custom Order object for keyset pagination: # https://docs.gitlab.com/ee/development/database/keyset_pagination.html#complex-order-configuration - Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: :relative_position, - column_expression: Issue.arel_table[:relative_position], - order_expression: Issue.arel_table[:relative_position].desc.nulls_last, - reversed_order_expression: Issue.arel_table[:relative_position].asc.nulls_first, - order_direction: :desc, - nullable: :nulls_last, - distinct: false - ), - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: :id, - order_expression: Issue.arel_table[:id].desc, - nullable: :not_nullable, - distinct: true - ) - ]) + Gitlab::Pagination::Keyset::Order.build( + [ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: :relative_position, + column_expression: Issue.arel_table[:relative_position], + order_expression: Issue.arel_table[:relative_position].desc.nulls_last, + reversed_order_expression: Issue.arel_table[:relative_position].asc.nulls_first, + order_direction: :desc, + nullable: :nulls_last, + distinct: false + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: :id, + order_expression: Issue.arel_table[:id].desc, + nullable: :not_nullable, + distinct: true + ) + ]) end let(:in_operator_optimization_options) do @@ -279,17 +280,18 @@ RSpec.describe Gitlab::Pagination::Keyset::InOperatorOptimization::QueryBuilder context 'when ordering by SQL expression' do let(:order) do # ORDER BY (id * 10), id - Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'id_multiplied_by_ten', - order_expression: Arel.sql('(id * 10)').asc, - sql_type: 'integer' - ), - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: :id, - order_expression: Issue.arel_table[:id].asc - ) - ]) + Gitlab::Pagination::Keyset::Order.build( + [ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'id_multiplied_by_ten', + order_expression: Arel.sql('(id * 10)').asc, + sql_type: 'integer' + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: :id, + order_expression: Issue.arel_table[:id].asc + ) + ]) end let(:scope) { Issue.reorder(order) } @@ -328,4 +330,148 @@ RSpec.describe Gitlab::Pagination::Keyset::InOperatorOptimization::QueryBuilder end end end + + context 'when ordering by JOIN-ed columns' do + let(:scope) { cte_with_issues_and_projects.apply_to(Issue.where({})).reorder(order) } + + let(:cte_with_issues_and_projects) do + cte_query = Issue.select('issues.id AS id', 'project_id', 'projects.id AS projects_id', 'projects.name AS projects_name').joins(:project) + Gitlab::SQL::CTE.new(:issue_with_project, cte_query, materialized: false) + end + + let(:in_operator_optimization_options) do + { + array_scope: Project.where(namespace_id: top_level_group.self_and_descendants.select(:id)).select(:id), + array_mapping_scope: -> (id_expression) { Issue.where(Issue.arel_table[:project_id].eq(id_expression)) } + } + end + + context 'when directions are project.id DESC, issues.id ASC' do + let(:order) do + Gitlab::Pagination::Keyset::Order.build([ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'projects_id', + order_expression: Issue.arel_table[:projects_id].asc, + sql_type: 'integer', + nullable: :not_nullable, + distinct: false + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: :id, + order_expression: Issue.arel_table[:id].asc + ) + ]) + end + + let(:expected_order) { issues.sort_by { |issue| [issue.project_id, issue.id] } } + + context 'when iterating records one by one' do + let(:batch_size) { 1 } + + it_behaves_like 'correct ordering examples', skip_finder_query_test: true + end + + context 'when iterating records with LIMIT 2' do + let(:batch_size) { 2 } + + it_behaves_like 'correct ordering examples', skip_finder_query_test: true + end + end + + context 'when directions are projects.id DESC, issues.id ASC' do + let(:order) do + Gitlab::Pagination::Keyset::Order.build([ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'projects_id', + order_expression: Issue.arel_table[:projects_id].desc, + sql_type: 'integer', + nullable: :not_nullable, + distinct: false + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: :id, + order_expression: Issue.arel_table[:id].asc + ) + ]) + end + + let(:expected_order) { issues.sort_by { |issue| [issue.project_id * -1, issue.id] } } + + context 'when iterating records one by one' do + let(:batch_size) { 1 } + + it_behaves_like 'correct ordering examples', skip_finder_query_test: true + end + + context 'when iterating records with LIMIT 2' do + let(:batch_size) { 2 } + + it_behaves_like 'correct ordering examples', skip_finder_query_test: true + end + end + + context 'when directions are projects.name ASC, projects.id ASC, issues.id ASC' do + let(:order) do + Gitlab::Pagination::Keyset::Order.build([ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'projects_name', + order_expression: Issue.arel_table[:projects_name].asc, + sql_type: 'character varying', + nullable: :not_nullable, + distinct: false + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'projects_id', + order_expression: Issue.arel_table[:projects_id].asc, + sql_type: 'integer', + nullable: :not_nullable, + distinct: false + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: :id, + order_expression: Issue.arel_table[:id].asc + ) + ]) + end + + let(:expected_order) { issues.sort_by { |issue| [issue.project.name, issue.project.id, issue.id] } } + + context 'when iterating records with LIMIT 2' do + let(:batch_size) { 2 } + + it_behaves_like 'correct ordering examples', skip_finder_query_test: true + end + end + + context 'when directions are projects.name ASC (nullable), issues.id ASC' do + let(:cte_with_issues_and_projects) do + cte_query = Issue.select('issues.id AS id', 'project_id', 'projects.id AS projects_id', 'NULL AS projects_name').joins(:project) + Gitlab::SQL::CTE.new(:issue_with_project, cte_query, materialized: false) + end + + let(:order) do + Gitlab::Pagination::Keyset::Order.build([ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'projects_name', + order_expression: Issue.arel_table[:projects_name].asc, + sql_type: 'character varying', + nullable: :nulls_last, + distinct: false + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: :id, + order_expression: Issue.arel_table[:id].asc + ) + ]) + end + + let(:expected_order) { issues.sort_by { |issue| [issue.id] } } + + context 'when iterating records with LIMIT 2' do + let(:batch_size) { 2 } + + it_behaves_like 'correct ordering examples', skip_finder_query_test: true + end + end + end end diff --git a/spec/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy_spec.rb b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy_spec.rb index ab1037b318b..2073142f077 100644 --- a/spec/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy_spec.rb +++ b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy_spec.rb @@ -25,22 +25,24 @@ RSpec.describe Gitlab::Pagination::Keyset::InOperatorOptimization::Strategies::O describe '#initializer_columns' do it 'returns NULLs for each ORDER BY columns' do - expect(strategy.initializer_columns).to eq([ - 'NULL::timestamp without time zone AS created_at', - 'NULL::integer AS id' - ]) + expect(strategy.initializer_columns).to eq( + [ + 'NULL::timestamp without time zone AS created_at', + 'NULL::integer AS id' + ]) end end context 'when an SQL expression is given' do context 'when the sql_type attribute is missing' do let(:order) do - Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'id_times_ten', - order_expression: Arel.sql('id * 10').asc - ) - ]) + Gitlab::Pagination::Keyset::Order.build( + [ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'id_times_ten', + order_expression: Arel.sql('id * 10').asc + ) + ]) end let(:keyset_scope) { Project.order(order) } @@ -52,13 +54,14 @@ RSpec.describe Gitlab::Pagination::Keyset::InOperatorOptimization::Strategies::O context 'when the sql_type_attribute is present' do let(:order) do - Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'id_times_ten', - order_expression: Arel.sql('id * 10').asc, - sql_type: 'integer' - ) - ]) + Gitlab::Pagination::Keyset::Order.build( + [ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'id_times_ten', + order_expression: Arel.sql('id * 10').asc, + sql_type: 'integer' + ) + ]) end let(:keyset_scope) { Project.order(order) } diff --git a/spec/lib/gitlab/pagination/keyset/iterator_spec.rb b/spec/lib/gitlab/pagination/keyset/iterator_spec.rb index d62d20d2d2c..eee743c5e48 100644 --- a/spec/lib/gitlab/pagination/keyset/iterator_spec.rb +++ b/spec/lib/gitlab/pagination/keyset/iterator_spec.rb @@ -15,21 +15,22 @@ RSpec.describe Gitlab::Pagination::Keyset::Iterator do let(:nulls_position) { :nulls_last } let(:reverse_nulls_position) { ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::REVERSED_NULL_POSITIONS[nulls_position] } let(:custom_reorder) do - Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: column, - column_expression: klass.arel_table[column], - order_expression: klass.arel_table[column].public_send(direction).public_send(nulls_position), # rubocop:disable GitlabSecurity/PublicSend - reversed_order_expression: klass.arel_table[column].public_send(reverse_direction).public_send(reverse_nulls_position), # rubocop:disable GitlabSecurity/PublicSend - order_direction: direction, - nullable: nulls_position, - distinct: false - ), - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'id', - order_expression: klass.arel_table[:id].send(direction) - ) - ]) + Gitlab::Pagination::Keyset::Order.build( + [ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: column, + column_expression: klass.arel_table[column], + order_expression: klass.arel_table[column].public_send(direction).public_send(nulls_position), # rubocop:disable GitlabSecurity/PublicSend + reversed_order_expression: klass.arel_table[column].public_send(reverse_direction).public_send(reverse_nulls_position), # rubocop:disable GitlabSecurity/PublicSend + order_direction: direction, + nullable: nulls_position, + distinct: false + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'id', + order_expression: klass.arel_table[:id].send(direction) + ) + ]) end let(:iterator_params) { nil } diff --git a/spec/lib/gitlab/pagination/keyset/order_spec.rb b/spec/lib/gitlab/pagination/keyset/order_spec.rb index c1fc73603d6..e99846ad424 100644 --- a/spec/lib/gitlab/pagination/keyset/order_spec.rb +++ b/spec/lib/gitlab/pagination/keyset/order_spec.rb @@ -148,15 +148,16 @@ RSpec.describe Gitlab::Pagination::Keyset::Order do end let(:order) do - Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'id', - column_expression: table['id'], - order_expression: table['id'].desc, - nullable: :not_nullable, - distinct: true - ) - ]) + Gitlab::Pagination::Keyset::Order.build( + [ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'id', + column_expression: table['id'], + order_expression: table['id'].desc, + nullable: :not_nullable, + distinct: true + ) + ]) end let(:expected) do @@ -192,29 +193,30 @@ RSpec.describe Gitlab::Pagination::Keyset::Order do end let(:order) do - Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'year', - column_expression: table['year'], - order_expression: table['year'].asc, - nullable: :not_nullable, - distinct: false - ), - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'month', - column_expression: table['month'], - order_expression: table['month'].asc, - nullable: :not_nullable, - distinct: false - ), - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'id', - column_expression: table['id'], - order_expression: table['id'].asc, - nullable: :not_nullable, - distinct: true - ) - ]) + Gitlab::Pagination::Keyset::Order.build( + [ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'year', + column_expression: table['year'], + order_expression: table['year'].asc, + nullable: :not_nullable, + distinct: false + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'month', + column_expression: table['month'], + order_expression: table['month'].asc, + nullable: :not_nullable, + distinct: false + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'id', + column_expression: table['id'], + order_expression: table['id'].asc, + nullable: :not_nullable, + distinct: true + ) + ]) end let(:expected) do @@ -258,33 +260,34 @@ RSpec.describe Gitlab::Pagination::Keyset::Order do end let(:order) do - Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'year', - column_expression: table['year'], - order_expression: table[:year].asc.nulls_last, - reversed_order_expression: table[:year].desc.nulls_first, - order_direction: :asc, - nullable: :nulls_last, - distinct: false - ), - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'month', - column_expression: table['month'], - order_expression: table[:month].asc.nulls_last, - reversed_order_expression: table[:month].desc.nulls_first, - order_direction: :asc, - nullable: :nulls_last, - distinct: false - ), - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'id', - column_expression: table['id'], - order_expression: table['id'].asc, - nullable: :not_nullable, - distinct: true - ) - ]) + Gitlab::Pagination::Keyset::Order.build( + [ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'year', + column_expression: table['year'], + order_expression: table[:year].asc.nulls_last, + reversed_order_expression: table[:year].desc.nulls_first, + order_direction: :asc, + nullable: :nulls_last, + distinct: false + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'month', + column_expression: table['month'], + order_expression: table[:month].asc.nulls_last, + reversed_order_expression: table[:month].desc.nulls_first, + order_direction: :asc, + nullable: :nulls_last, + distinct: false + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'id', + column_expression: table['id'], + order_expression: table['id'].asc, + nullable: :not_nullable, + distinct: true + ) + ]) end let(:expected) do @@ -324,33 +327,34 @@ RSpec.describe Gitlab::Pagination::Keyset::Order do end let(:order) do - Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'year', - column_expression: table['year'], - order_expression: table[:year].asc.nulls_first, - reversed_order_expression: table[:year].desc.nulls_last, - order_direction: :asc, - nullable: :nulls_first, - distinct: false - ), - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'month', - column_expression: table['month'], - order_expression: table[:month].asc.nulls_first, - order_direction: :asc, - reversed_order_expression: table[:month].desc.nulls_last, - nullable: :nulls_first, - distinct: false - ), - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'id', - column_expression: table['id'], - order_expression: table['id'].asc, - nullable: :not_nullable, - distinct: true - ) - ]) + Gitlab::Pagination::Keyset::Order.build( + [ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'year', + column_expression: table['year'], + order_expression: table[:year].asc.nulls_first, + reversed_order_expression: table[:year].desc.nulls_last, + order_direction: :asc, + nullable: :nulls_first, + distinct: false + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'month', + column_expression: table['month'], + order_expression: table[:month].asc.nulls_first, + order_direction: :asc, + reversed_order_expression: table[:month].desc.nulls_last, + nullable: :nulls_first, + distinct: false + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'id', + column_expression: table['id'], + order_expression: table['id'].asc, + nullable: :not_nullable, + distinct: true + ) + ]) end let(:expected) do @@ -390,22 +394,23 @@ RSpec.describe Gitlab::Pagination::Keyset::Order do end let(:order) do - Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'year', - column_expression: table['year'], - order_expression: table['year'].asc, - nullable: :not_nullable, - distinct: false - ), - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'id', - column_expression: table['id'], - order_expression: table['id'].desc, - nullable: :not_nullable, - distinct: true - ) - ]) + Gitlab::Pagination::Keyset::Order.build( + [ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'year', + column_expression: table['year'], + order_expression: table['year'].asc, + nullable: :not_nullable, + distinct: false + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'id', + column_expression: table['id'], + order_expression: table['id'].desc, + nullable: :not_nullable, + distinct: true + ) + ]) end let(:expected) do @@ -432,33 +437,38 @@ RSpec.describe Gitlab::Pagination::Keyset::Order do reversed = order.reversed_order before_conditions = reversed.where_values_with_or_query(before_cursor) - query = build_query(order: order, where_conditions: [Arel::Nodes::And.new([after_conditions, before_conditions])], limit: 100) + query = build_query( + order: order, + where_conditions: [Arel::Nodes::And.new([after_conditions, before_conditions])], + limit: 100) - expect(run_query(query)).to eq([ - { "id" => 2, "year" => 2011, "month" => 0 }, - { "id" => 6, "year" => 2012, "month" => 0 } - ]) + expect(run_query(query)).to eq( + [ + { "id" => 2, "year" => 2011, "month" => 0 }, + { "id" => 6, "year" => 2012, "month" => 0 } + ]) end end context 'when ordering by the named function LOWER' do let(:order) do - Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'title', - column_expression: Arel::Nodes::NamedFunction.new("LOWER", [table['title'].desc]), - order_expression: table['title'].lower.desc, - nullable: :not_nullable, - distinct: false - ), - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'id', - column_expression: table['id'], - order_expression: table['id'].desc, - nullable: :not_nullable, - distinct: true - ) - ]) + Gitlab::Pagination::Keyset::Order.build( + [ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'title', + column_expression: Arel::Nodes::NamedFunction.new("LOWER", [table['title'].desc]), + order_expression: table['title'].lower.desc, + nullable: :not_nullable, + distinct: false + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'id', + column_expression: table['id'], + order_expression: table['id'].desc, + nullable: :not_nullable, + distinct: true + ) + ]) end let(:table_data) do @@ -484,22 +494,23 @@ RSpec.describe Gitlab::Pagination::Keyset::Order do context 'when the passed cursor values do not match with the order definition' do let(:order) do - Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'year', - column_expression: table['year'], - order_expression: table['year'].asc, - nullable: :not_nullable, - distinct: false - ), - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'id', - column_expression: table['id'], - order_expression: table['id'].desc, - nullable: :not_nullable, - distinct: true - ) - ]) + Gitlab::Pagination::Keyset::Order.build( + [ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'year', + column_expression: table['year'], + order_expression: table['year'].asc, + nullable: :not_nullable, + distinct: false + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'id', + column_expression: table['id'], + order_expression: table['id'].desc, + nullable: :not_nullable, + distinct: true + ) + ]) end context 'when values are missing' do @@ -553,14 +564,15 @@ RSpec.describe Gitlab::Pagination::Keyset::Order do context 'when string attribute name is given' do let(:order) do - Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'id', - order_expression: Project.arel_table['id'].desc, - nullable: :not_nullable, - distinct: true - ) - ]) + Gitlab::Pagination::Keyset::Order.build( + [ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'id', + order_expression: Project.arel_table['id'].desc, + nullable: :not_nullable, + distinct: true + ) + ]) end it_behaves_like 'cursor attribute examples' @@ -568,14 +580,15 @@ RSpec.describe Gitlab::Pagination::Keyset::Order do context 'when symbol attribute name is given' do let(:order) do - Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: :id, - order_expression: Project.arel_table['id'].desc, - nullable: :not_nullable, - distinct: true - ) - ]) + Gitlab::Pagination::Keyset::Order.build( + [ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: :id, + order_expression: Project.arel_table['id'].desc, + nullable: :not_nullable, + distinct: true + ) + ]) end it_behaves_like 'cursor attribute examples' @@ -593,20 +606,21 @@ RSpec.describe Gitlab::Pagination::Keyset::Order do context 'when there are additional_projections' do let(:order) do - order = Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'created_at_field', - column_expression: Project.arel_table[:created_at], - order_expression: Project.arel_table[:created_at].desc, - order_direction: :desc, - distinct: false, - add_to_projections: true - ), - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'id', - order_expression: Project.arel_table[:id].desc - ) - ]) + order = Gitlab::Pagination::Keyset::Order.build( + [ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'created_at_field', + column_expression: Project.arel_table[:created_at], + order_expression: Project.arel_table[:created_at].desc, + order_direction: :desc, + distinct: false, + add_to_projections: true + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'id', + order_expression: Project.arel_table[:id].desc + ) + ]) order end @@ -684,20 +698,21 @@ RSpec.describe Gitlab::Pagination::Keyset::Order do describe '#attribute_names' do let(:expected_attribute_names) { %w(id name) } let(:order) do - Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'id', - order_expression: Project.arel_table['id'].desc, - nullable: :not_nullable, - distinct: true - ), - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'name', - order_expression: Project.arel_table['name'].desc, - nullable: :not_nullable, - distinct: true - ) - ]) + Gitlab::Pagination::Keyset::Order.build( + [ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'id', + order_expression: Project.arel_table['id'].desc, + nullable: :not_nullable, + distinct: true + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'name', + order_expression: Project.arel_table['name'].desc, + nullable: :not_nullable, + distinct: true + ) + ]) end subject { order.attribute_names } diff --git a/spec/lib/gitlab/profiler_spec.rb b/spec/lib/gitlab/profiler_spec.rb index bfe1a588489..7c365990627 100644 --- a/spec/lib/gitlab/profiler_spec.rb +++ b/spec/lib/gitlab/profiler_spec.rb @@ -15,7 +15,7 @@ RSpec.describe Gitlab::Profiler do end it 'returns a profile result' do - expect(described_class.profile('/')).to be_an_instance_of(RubyProf::Profile) + expect(described_class.profile('/')).to be_an_instance_of(File) end it 'uses the custom logger given' do @@ -59,28 +59,26 @@ RSpec.describe Gitlab::Profiler do described_class.profile('/', user: user, private_token: private_token) end - context 'with sampling profiler' do - it 'generates sampling data' do - user = double(:user) - temp_data = Tempfile.new + it 'generates sampling data' do + user = double(:user) + temp_data = Tempfile.new - expect(described_class).to receive(:with_user).with(user).and_call_original - described_class.profile('/', user: user, sampling_mode: true, profiler_options: { out: temp_data.path }) + expect(described_class).to receive(:with_user).with(user).and_call_original + described_class.profile('/', user: user, profiler_options: { out: temp_data.path }) - expect(File.stat(temp_data).size).to be > 0 - File.unlink(temp_data) - end + expect(File.stat(temp_data).size).to be > 0 + File.unlink(temp_data) + end - it 'saves sampling data with a randomly-generated filename' do - user = double(:user) + it 'saves sampling data with a randomly-generated filename' do + user = double(:user) - expect(described_class).to receive(:with_user).with(user).and_call_original - result = described_class.profile('/', user: user, sampling_mode: true) + expect(described_class).to receive(:with_user).with(user).and_call_original + result = described_class.profile('/', user: user) - expect(result).to be_a(File) - expect(File.stat(result.path).size).to be > 0 - File.unlink(result.path) - end + expect(result).to be_a(File) + expect(File.stat(result.path).size).to be > 0 + File.unlink(result.path) end end @@ -211,54 +209,4 @@ RSpec.describe Gitlab::Profiler do expect(described_class.log_load_times_by_model(null_logger)).to be_nil end end - - describe '.print_by_total_time' do - let(:stdout) { StringIO.new } - let(:regexp) { /^\s+\d+\.\d+\s+(\d+\.\d+)/ } - - let(:output) do - stdout.rewind - stdout.read - end - - let_it_be(:result) do - Thread.new { sleep 1 } - - RubyProf.profile do - sleep 0.1 - 1.to_s - end - end - - around do |example| - original_stdout = $stdout - - $stdout = stdout # rubocop: disable RSpec/ExpectOutput - example.run - $stdout = original_stdout # rubocop: disable RSpec/ExpectOutput - end - - it 'prints a profile result sorted by total time' do - described_class.print_by_total_time(result) - - expect(output).to include('Kernel#sleep') - - thread_profiles = output.split('Sort by: total_time').select { |x| x =~ regexp } - - thread_profiles.each do |profile| - total_times = - profile - .scan(regexp) - .map { |(total)| total.to_f } - - expect(total_times).to eq(total_times.sort.reverse) - end - end - - it 'accepts a max_percent option' do - described_class.print_by_total_time(result, max_percent: 50) - - expect(output).not_to include('Kernel#sleep') - end - end end diff --git a/spec/lib/gitlab/project_search_results_spec.rb b/spec/lib/gitlab/project_search_results_spec.rb index a9c0262fdb2..a762fdbde6b 100644 --- a/spec/lib/gitlab/project_search_results_spec.rb +++ b/spec/lib/gitlab/project_search_results_spec.rb @@ -118,7 +118,7 @@ RSpec.describe Gitlab::ProjectSearchResults do shared_examples 'blob search repository ref' do |entity_type, blob_type| let(:query) { 'files' } let(:file_finder) { double } - let(:project_branch) { 'project_branch' } + let(:project_branch) { blob_type == 'wiki_blobs' ? entity.default_branch : 'project_branch' } subject(:objects) { results.objects(blob_type) } @@ -209,8 +209,11 @@ RSpec.describe Gitlab::ProjectSearchResults do describe 'wiki search' do let(:project) { create(:project, :public, :wiki_repo) } + let(:project_branch) { 'project_branch' } before do + allow(project.wiki).to receive(:root_ref).and_return(project_branch) + project.wiki.create_page('Files/Title', 'Content') project.wiki.create_page('CHANGELOG', 'Files example') end diff --git a/spec/lib/gitlab/project_transfer_spec.rb b/spec/lib/gitlab/project_transfer_spec.rb index 87c4014264f..3d6aa80c51f 100644 --- a/spec/lib/gitlab/project_transfer_spec.rb +++ b/spec/lib/gitlab/project_transfer_spec.rb @@ -15,10 +15,11 @@ RSpec.describe Gitlab::ProjectTransfer do end after do - FileUtils.rm_rf([ - File.join(@root_dir, @namespace_path), - File.join(@root_dir, @namespace_path_was) - ]) + FileUtils.rm_rf( + [ + File.join(@root_dir, @namespace_path), + File.join(@root_dir, @namespace_path_was) + ]) end describe '#move_project' do diff --git a/spec/lib/gitlab/prometheus_client_spec.rb b/spec/lib/gitlab/prometheus_client_spec.rb index 9083c5625d4..d0bfc6e5610 100644 --- a/spec/lib/gitlab/prometheus_client_spec.rb +++ b/spec/lib/gitlab/prometheus_client_spec.rb @@ -300,12 +300,13 @@ RSpec.describe Gitlab::PrometheusClient do it 'returns data from the API call' do req_stub = stub_prometheus_request(query_url, body: prometheus_values_body('matrix')) - expect(subject.query_range(prometheus_query)).to eq([ - { - "metric" => {}, - "values" => [[1488758662.506, "0.00002996364761904785"], [1488758722.506, "0.00003090239047619091"]] - } - ]) + expect(subject.query_range(prometheus_query)).to eq( + [ + { + "metric" => {}, + "values" => [[1488758662.506, "0.00002996364761904785"], [1488758722.506, "0.00003090239047619091"]] + } + ]) expect(req_stub).to have_been_requested end end diff --git a/spec/lib/gitlab/push_options_spec.rb b/spec/lib/gitlab/push_options_spec.rb index 3ff1c8e9012..054beaf7012 100644 --- a/spec/lib/gitlab/push_options_spec.rb +++ b/spec/lib/gitlab/push_options_spec.rb @@ -52,10 +52,11 @@ RSpec.describe Gitlab::PushOptions do end it 'can parse multiple push options' do - options = described_class.new([ - 'merge_request.create', - 'merge_request.target=value' - ]) + options = described_class.new( + [ + 'merge_request.create', + 'merge_request.target=value' + ]) expect(options.get(:merge_request)).to include({ create: true, @@ -66,19 +67,21 @@ RSpec.describe Gitlab::PushOptions do end it 'stores options internally as a HashWithIndifferentAccess' do - options = described_class.new([ - 'merge_request.create' - ]) + options = described_class.new( + [ + 'merge_request.create' + ]) expect(options.get('merge_request', 'create')).to eq(true) expect(options.get(:merge_request, :create)).to eq(true) end it 'selects the last option when options contain duplicate namespace and key pairs' do - options = described_class.new([ - 'merge_request.target=value1', - 'merge_request.target=value2' - ]) + options = described_class.new( + [ + 'merge_request.target=value1', + 'merge_request.target=value2' + ]) expect(options.get(:merge_request, :target)).to eq('value2') end diff --git a/spec/lib/gitlab/query_limiting/transaction_spec.rb b/spec/lib/gitlab/query_limiting/transaction_spec.rb index 27da1f23556..d8eb2040ccc 100644 --- a/spec/lib/gitlab/query_limiting/transaction_spec.rb +++ b/spec/lib/gitlab/query_limiting/transaction_spec.rb @@ -52,7 +52,7 @@ RSpec.describe Gitlab::QueryLimiting::Transaction do context 'when the query threshold is exceeded' do let(:transaction) do trans = described_class.new - trans.count = described_class::THRESHOLD + 1 + trans.count = described_class.threshold + 1 trans end @@ -120,7 +120,7 @@ RSpec.describe Gitlab::QueryLimiting::Transaction do it 'returns true when the threshold is exceeded' do transaction = described_class.new - transaction.count = described_class::THRESHOLD + 1 + transaction.count = described_class.threshold + 1 expect(transaction.threshold_exceeded?).to eq(true) end @@ -129,7 +129,7 @@ RSpec.describe Gitlab::QueryLimiting::Transaction do describe '#error_message' do it 'returns the error message to display when the threshold is exceeded' do transaction = described_class.new - transaction.count = max = described_class::THRESHOLD + transaction.count = max = described_class.threshold expect(transaction.error_message).to eq( "Too many SQL queries were executed: a maximum of #{max} " \ @@ -139,7 +139,7 @@ RSpec.describe Gitlab::QueryLimiting::Transaction do it 'includes a list of executed queries' do transaction = described_class.new - transaction.count = max = described_class::THRESHOLD + transaction.count = max = described_class.threshold %w[foo bar baz].each { |sql| transaction.executed_sql(sql) } message = transaction.error_message @@ -154,7 +154,7 @@ RSpec.describe Gitlab::QueryLimiting::Transaction do it 'indicates if the log is truncated' do transaction = described_class.new - transaction.count = described_class::THRESHOLD * 2 + transaction.count = described_class.threshold * 2 message = transaction.error_message @@ -163,7 +163,7 @@ RSpec.describe Gitlab::QueryLimiting::Transaction do it 'includes the action name in the error message when present' do transaction = described_class.new - transaction.count = max = described_class::THRESHOLD + transaction.count = max = described_class.threshold transaction.action = 'UsersController#show' expect(transaction.error_message).to eq( diff --git a/spec/lib/gitlab/rack_attack/request_spec.rb b/spec/lib/gitlab/rack_attack/request_spec.rb index b8a26a64e5b..5345205e15b 100644 --- a/spec/lib/gitlab/rack_attack/request_spec.rb +++ b/spec/lib/gitlab/rack_attack/request_spec.rb @@ -217,10 +217,11 @@ RSpec.describe Gitlab::RackAttack::Request do subject { request.protected_path? } before do - stub_application_setting(protected_paths: [ - '/protected', - '/secure' - ]) + stub_application_setting( + protected_paths: [ + '/protected', + '/secure' + ]) end where(:path, :expected) do diff --git a/spec/lib/gitlab/reference_extractor_spec.rb b/spec/lib/gitlab/reference_extractor_spec.rb index 177e9d346b6..0ee8c35ae81 100644 --- a/spec/lib/gitlab/reference_extractor_spec.rb +++ b/spec/lib/gitlab/reference_extractor_spec.rb @@ -193,7 +193,7 @@ RSpec.describe Gitlab::ReferenceExtractor do end context 'with an external issue tracker' do - let(:project) { create(:jira_project) } + let(:project) { create(:project, :with_jira_integration) } let(:issue) { create(:issue, project: project) } context 'when GitLab issues are enabled' do diff --git a/spec/lib/gitlab/regex_requires_app_spec.rb b/spec/lib/gitlab/regex_requires_app_spec.rb index 5808033dc4c..780184cdfd2 100644 --- a/spec/lib/gitlab/regex_requires_app_spec.rb +++ b/spec/lib/gitlab/regex_requires_app_spec.rb @@ -30,6 +30,8 @@ RSpec.describe Gitlab::Regex do it { is_expected.not_to match('AMD64') } it { is_expected.not_to match('Amd64') } it { is_expected.not_to match('aMD64') } + + it_behaves_like 'regex rejecting path traversal' end describe '.npm_package_name_regex' do @@ -73,6 +75,8 @@ RSpec.describe Gitlab::Regex do # Do not allow Unicode it { is_expected.not_to match('hé') } + + it_behaves_like 'regex rejecting path traversal' end describe '.debian_component_regex' do @@ -86,5 +90,7 @@ RSpec.describe Gitlab::Regex do # Do not allow Unicode it { is_expected.not_to match('hé') } + + it_behaves_like 'regex rejecting path traversal' end end diff --git a/spec/lib/gitlab/regex_spec.rb b/spec/lib/gitlab/regex_spec.rb index d8f182d903d..89ef76d246e 100644 --- a/spec/lib/gitlab/regex_spec.rb +++ b/spec/lib/gitlab/regex_spec.rb @@ -3,6 +3,7 @@ require 'fast_spec_helper' require_relative '../../../lib/gitlab/regex' +require_relative '../../support/shared_examples/lib/gitlab/regex_shared_examples' # All specs that can be run with fast_spec_helper only # See regex_requires_app_spec for tests that require the full spec_helper @@ -543,6 +544,8 @@ RSpec.describe Gitlab::Regex do it { is_expected.not_to match('aA') } # No underscore it { is_expected.not_to match('a_b') } + + it_behaves_like 'regex rejecting path traversal' end describe '.debian_version_regex' do @@ -596,6 +599,13 @@ RSpec.describe Gitlab::Regex do it { is_expected.to match('1-2-3-4-5-6-7-8-9-10-11-12-13-14-15') } it { is_expected.not_to match('1-2-3-4-5-6-7-8-9-10-11-12-13-14-15-16') } end + + context 'path traversals' do + it { is_expected.not_to match('1../0') } + it { is_expected.not_to match('1..%2f0') } + it { is_expected.not_to match('1%2e%2e%2f0') } + it { is_expected.not_to match('1%2e%2e/0') } + end end describe '.helm_channel_regex' do diff --git a/spec/lib/gitlab/search/abuse_detection_spec.rb b/spec/lib/gitlab/search/abuse_detection_spec.rb index 2a8d74a62ab..7fb9621141c 100644 --- a/spec/lib/gitlab/search/abuse_detection_spec.rb +++ b/spec/lib/gitlab/search/abuse_detection_spec.rb @@ -21,16 +21,16 @@ RSpec.describe Gitlab::Search::AbuseDetection do describe 'abusive character matching' do refs = %w( - main - тест - maiñ - main123 - main-v123 - main-v12.3 - feature/it_works - really_important! - 测试 - ) + main + тест + maiñ + main123 + main-v123 + main-v12.3 + feature/it_works + really_important! + 测试 + ) refs.each do |ref| it "does match refs permitted by git refname: #{ref}" do diff --git a/spec/lib/gitlab/search/query_spec.rb b/spec/lib/gitlab/search/query_spec.rb index 234b683ba1f..cdab7f1c04b 100644 --- a/spec/lib/gitlab/search/query_spec.rb +++ b/spec/lib/gitlab/search/query_spec.rb @@ -64,4 +64,38 @@ RSpec.describe Gitlab::Search::Query do expect(subject.filters[0]).to include(name: :name, negated: false, value: "MY TEST.TXT") end end + + context 'with mutliple filename filters' do + let(:query) { 'something filename:myfile.txt -filename:ANOTHERFILE.yml filename:somethingelse.txt' } + let(:subject) do + described_class.new(query) do + filter :filename + end + end + + it 'creates a filter for each filename in query' do + expect(subject.filters.count).to eq(3) + expect(subject.filters[0]).to include(name: :filename, negated: false, value: 'myfile.txt') + expect(subject.filters[1]).to include(name: :filename, negated: true, value: 'anotherfile.yml') + expect(subject.filters[2]).to include(name: :filename, negated: false, value: 'somethingelse.txt') + end + + context 'when multiple extension filters are added' do + let(:query) { 'something filename:myfile.txt -extension:yml -filename:ANOTHERFILE.yml extension:txt' } + let(:subject) do + described_class.new(query) do + filter :filename + filter :extension + end + end + + it 'creates a filter for each filename and extension in query' do + expect(subject.filters.count).to eq(4) + expect(subject.filters[0]).to include(name: :filename, negated: false, value: 'myfile.txt') + expect(subject.filters[1]).to include(name: :filename, negated: true, value: 'anotherfile.yml') + expect(subject.filters[2]).to include(name: :extension, negated: true, value: 'yml') + expect(subject.filters[3]).to include(name: :extension, negated: false, value: 'txt') + end + end + end end diff --git a/spec/lib/gitlab/serializer/ci/variables_spec.rb b/spec/lib/gitlab/serializer/ci/variables_spec.rb index 9b0475259fe..02f1d543e4b 100644 --- a/spec/lib/gitlab/serializer/ci/variables_spec.rb +++ b/spec/lib/gitlab/serializer/ci/variables_spec.rb @@ -13,9 +13,10 @@ RSpec.describe Gitlab::Serializer::Ci::Variables do end it 'converts keys into strings and symbolizes hash' do - is_expected.to eq([ - { key: 'key', value: 'value', public: true }, - { key: 'wee', value: 1, public: false } - ]) + is_expected.to eq( + [ + { key: 'key', value: 'value', public: true }, + { key: 'wee', value: 1, public: false } + ]) end end diff --git a/spec/lib/gitlab/sidekiq_config_spec.rb b/spec/lib/gitlab/sidekiq_config_spec.rb index c62302d8bba..c5b00afe672 100644 --- a/spec/lib/gitlab/sidekiq_config_spec.rb +++ b/spec/lib/gitlab/sidekiq_config_spec.rb @@ -193,9 +193,7 @@ RSpec.describe Gitlab::SidekiqConfig do it 'returns worker queue mappings that have queues in the current Sidekiq options' do queues = described_class.routing_queues - expect(queues).to match_array(%w[ - default mailers high_urgency gitaly - ]) + expect(queues).to match_array(%w[default mailers high_urgency gitaly]) expect(queues).not_to include('not_exist') end end diff --git a/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb b/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb index dff04a2e509..62681b21756 100644 --- a/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb +++ b/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb @@ -130,9 +130,10 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do end it 'return true when everything is within limit', :aggregate_failures do - expect(memory_killer).to receive(:get_rss).and_return(100) - expect(memory_killer).to receive(:get_soft_limit_rss).and_return(200) - expect(memory_killer).to receive(:get_hard_limit_rss).and_return(300) + expect(memory_killer).to receive(:get_rss_kb).and_return(100) + expect(memory_killer).to receive(:get_soft_limit_rss_kb).and_return(200) + expect(memory_killer).to receive(:get_hard_limit_rss_kb).and_return(300) + expect(memory_killer).to receive(:get_memory_total_kb).and_return(3072) expect(memory_killer).to receive(:refresh_state) .with(:running) @@ -145,9 +146,10 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do end it 'return false when rss exceeds hard_limit_rss', :aggregate_failures do - expect(memory_killer).to receive(:get_rss).at_least(:once).and_return(400) - expect(memory_killer).to receive(:get_soft_limit_rss).at_least(:once).and_return(200) - expect(memory_killer).to receive(:get_hard_limit_rss).at_least(:once).and_return(300) + expect(memory_killer).to receive(:get_rss_kb).at_least(:once).and_return(400) + expect(memory_killer).to receive(:get_soft_limit_rss_kb).at_least(:once).and_return(200) + expect(memory_killer).to receive(:get_hard_limit_rss_kb).at_least(:once).and_return(300) + expect(memory_killer).to receive(:get_memory_total_kb).at_least(:once).and_return(3072) expect(memory_killer).to receive(:refresh_state) .with(:running) @@ -165,9 +167,10 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do end it 'return false when rss exceed hard_limit_rss after a while', :aggregate_failures do - expect(memory_killer).to receive(:get_rss).and_return(250, 400, 400) - expect(memory_killer).to receive(:get_soft_limit_rss).at_least(:once).and_return(200) - expect(memory_killer).to receive(:get_hard_limit_rss).at_least(:once).and_return(300) + expect(memory_killer).to receive(:get_rss_kb).and_return(250, 400, 400) + expect(memory_killer).to receive(:get_soft_limit_rss_kb).at_least(:once).and_return(200) + expect(memory_killer).to receive(:get_hard_limit_rss_kb).at_least(:once).and_return(300) + expect(memory_killer).to receive(:get_memory_total_kb).at_least(:once).and_return(3072) expect(memory_killer).to receive(:refresh_state) .with(:running) @@ -187,9 +190,10 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do end it 'return true when rss below soft_limit_rss after a while within GRACE_BALLOON_SECONDS', :aggregate_failures do - expect(memory_killer).to receive(:get_rss).and_return(250, 100) - expect(memory_killer).to receive(:get_soft_limit_rss).and_return(200, 200) - expect(memory_killer).to receive(:get_hard_limit_rss).and_return(300, 300) + expect(memory_killer).to receive(:get_rss_kb).and_return(250, 100) + expect(memory_killer).to receive(:get_soft_limit_rss_kb).and_return(200, 200) + expect(memory_killer).to receive(:get_hard_limit_rss_kb).and_return(300, 300) + expect(memory_killer).to receive(:get_memory_total_kb).and_return(3072, 3072) expect(memory_killer).to receive(:refresh_state) .with(:running) @@ -211,9 +215,10 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do let(:grace_balloon_seconds) { 0 } it 'return false when rss exceed soft_limit_rss', :aggregate_failures do - allow(memory_killer).to receive(:get_rss).and_return(250) - allow(memory_killer).to receive(:get_soft_limit_rss).and_return(200) - allow(memory_killer).to receive(:get_hard_limit_rss).and_return(300) + allow(memory_killer).to receive(:get_rss_kb).and_return(250) + allow(memory_killer).to receive(:get_soft_limit_rss_kb).and_return(200) + allow(memory_killer).to receive(:get_hard_limit_rss_kb).and_return(300) + allow(memory_killer).to receive(:get_memory_total_kb).and_return(3072) expect(memory_killer).to receive(:refresh_state) .with(:running) @@ -235,40 +240,57 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do subject { memory_killer.send(:restart_sidekiq) } - before do - stub_const("#{described_class}::SHUTDOWN_TIMEOUT_SECONDS", shutdown_timeout_seconds) - allow(Sidekiq).to receive(:options).and_return(timeout: 9) - allow(memory_killer).to receive(:get_rss).and_return(100) - allow(memory_killer).to receive(:get_soft_limit_rss).and_return(200) - allow(memory_killer).to receive(:get_hard_limit_rss).and_return(300) + context 'when sidekiq_memory_killer_read_only_mode is enabled' do + before do + stub_feature_flags(sidekiq_memory_killer_read_only_mode: true) + end + + it 'does not send signal' do + expect(memory_killer).not_to receive(:refresh_state) + expect(memory_killer).not_to receive(:signal_and_wait) + + subject + end end - it 'send signal' do - expect(memory_killer).to receive(:refresh_state) - .with(:stop_fetching_new_jobs) - .ordered - .and_call_original - expect(memory_killer).to receive(:signal_and_wait) - .with(shutdown_timeout_seconds, 'SIGTSTP', 'stop fetching new jobs') - .ordered + context 'when sidekiq_memory_killer_read_only_mode is disabled' do + before do + stub_const("#{described_class}::SHUTDOWN_TIMEOUT_SECONDS", shutdown_timeout_seconds) + stub_feature_flags(sidekiq_memory_killer_read_only_mode: false) + allow(Sidekiq).to receive(:options).and_return(timeout: 9) + allow(memory_killer).to receive(:get_rss_kb).and_return(100) + allow(memory_killer).to receive(:get_soft_limit_rss_kb).and_return(200) + allow(memory_killer).to receive(:get_hard_limit_rss_kb).and_return(300) + allow(memory_killer).to receive(:get_memory_total_kb).and_return(3072) + end - expect(memory_killer).to receive(:refresh_state) - .with(:shutting_down) - .ordered - .and_call_original - expect(memory_killer).to receive(:signal_and_wait) - .with(11, 'SIGTERM', 'gracefully shut down') - .ordered + it 'send signal' do + expect(memory_killer).to receive(:refresh_state) + .with(:stop_fetching_new_jobs) + .ordered + .and_call_original + expect(memory_killer).to receive(:signal_and_wait) + .with(shutdown_timeout_seconds, 'SIGTSTP', 'stop fetching new jobs') + .ordered - expect(memory_killer).to receive(:refresh_state) - .with(:killing_sidekiq) - .ordered - .and_call_original - expect(memory_killer).to receive(:signal_pgroup) - .with('SIGKILL', 'die') - .ordered + expect(memory_killer).to receive(:refresh_state) + .with(:shutting_down) + .ordered + .and_call_original + expect(memory_killer).to receive(:signal_and_wait) + .with(11, 'SIGTERM', 'gracefully shut down') + .ordered - subject + expect(memory_killer).to receive(:refresh_state) + .with(:killing_sidekiq) + .ordered + .and_call_original + expect(memory_killer).to receive(:signal_pgroup) + .with('SIGKILL', 'die') + .ordered + + subject + end end end @@ -351,6 +373,7 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do let(:current_rss) { 100 } let(:soft_limit_rss) { 200 } let(:hard_limit_rss) { 300 } + let(:memory_total) { 3072 } let(:jid) { 1 } let(:reason) { 'rss out of range reason description' } let(:queue) { 'default' } @@ -369,9 +392,10 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do before do stub_const("DummyWorker", worker) - allow(memory_killer).to receive(:get_rss).and_return(*current_rss) - allow(memory_killer).to receive(:get_soft_limit_rss).and_return(soft_limit_rss) - allow(memory_killer).to receive(:get_hard_limit_rss).and_return(hard_limit_rss) + allow(memory_killer).to receive(:get_rss_kb).and_return(*current_rss) + allow(memory_killer).to receive(:get_soft_limit_rss_kb).and_return(soft_limit_rss) + allow(memory_killer).to receive(:get_hard_limit_rss_kb).and_return(hard_limit_rss) + allow(memory_killer).to receive(:get_memory_total_kb).and_return(memory_total) memory_killer.send(:refresh_state, :running) end @@ -389,7 +413,8 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do hard_limit_rss: hard_limit_rss, soft_limit_rss: soft_limit_rss, reason: reason, - running_jobs: running_jobs) + running_jobs: running_jobs, + memory_total_kb: memory_total) expect(metrics[:sidekiq_memory_killer_running_jobs]).to receive(:increment) .with({ worker_class: "DummyWorker", deadline_exceeded: true }) @@ -525,9 +550,10 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do subject { memory_killer.send(:refresh_state, :shutting_down) } it 'calls gitlab metrics gauge set methods' do - expect(memory_killer).to receive(:get_rss) { 1010 } - expect(memory_killer).to receive(:get_soft_limit_rss) { 1020 } - expect(memory_killer).to receive(:get_hard_limit_rss) { 1040 } + expect(memory_killer).to receive(:get_rss_kb) { 1010 } + expect(memory_killer).to receive(:get_soft_limit_rss_kb) { 1020 } + expect(memory_killer).to receive(:get_hard_limit_rss_kb) { 1040 } + expect(memory_killer).to receive(:get_memory_total_kb) { 3072 } expect(metrics[:sidekiq_memory_killer_phase]).to receive(:set) .with({}, described_class::PHASE[:shutting_down]) diff --git a/spec/lib/gitlab/sidekiq_middleware/memory_killer_spec.rb b/spec/lib/gitlab/sidekiq_middleware/memory_killer_spec.rb deleted file mode 100644 index 1667622ad8e..00000000000 --- a/spec/lib/gitlab/sidekiq_middleware/memory_killer_spec.rb +++ /dev/null @@ -1,83 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::SidekiqMiddleware::MemoryKiller do - subject { described_class.new } - - let(:pid) { 999 } - - let(:worker) { double(:worker, class: ProjectCacheWorker) } - let(:job) { { 'jid' => 123 } } - let(:queue) { 'test_queue' } - - def run - thread = subject.call(worker, job, queue) { nil } - thread&.join - end - - before do - allow(subject).to receive(:get_rss).and_return(10.kilobytes) - allow(subject).to receive(:pid).and_return(pid) - end - - context 'when MAX_RSS is set to 0' do - before do - stub_const("#{described_class}::MAX_RSS", 0) - end - - it 'does nothing' do - expect(subject).not_to receive(:sleep) - - run - end - end - - context 'when MAX_RSS is exceeded' do - before do - stub_const("#{described_class}::MAX_RSS", 5.kilobytes) - end - - it 'sends the TSTP, TERM and KILL signals at expected times' do - expect(subject).to receive(:sleep).with(15 * 60).ordered - expect(Process).to receive(:kill).with('SIGTSTP', pid).ordered - - expect(subject).to receive(:sleep).with(30).ordered - expect(Process).to receive(:kill).with('SIGTERM', pid).ordered - - expect(subject).to receive(:sleep).with(Sidekiq.options[:timeout] + 2).ordered - expect(Process).to receive(:kill).with('SIGKILL', pid).ordered - - expect(Sidekiq.logger) - .to receive(:warn).with(class: 'ProjectCacheWorker', - message: anything, - pid: pid, - signal: anything).at_least(:once) - - run - end - - it 'sends TSTP and TERM to the pid, but KILL to the pgroup, when running as process leader' do - allow(Process).to receive(:getpgrp) { pid } - allow(subject).to receive(:sleep) - - expect(Process).to receive(:kill).with('SIGTSTP', pid).ordered - expect(Process).to receive(:kill).with('SIGTERM', pid).ordered - expect(Process).to receive(:kill).with('SIGKILL', 0).ordered - - run - end - end - - context 'when MAX_RSS is not exceeded' do - before do - stub_const("#{described_class}::MAX_RSS", 15.kilobytes) - end - - it 'does nothing' do - expect(subject).not_to receive(:sleep) - - run - end - end -end diff --git a/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb b/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb index 52b50a143fc..54a1723afbc 100644 --- a/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb @@ -322,8 +322,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::ServerMetrics do with_sidekiq_server_middleware do |chain| Gitlab::SidekiqMiddleware.server_configurator( metrics: true, - arguments_logger: false, - memory_killer: false + arguments_logger: false ).call(chain) Sidekiq::Testing.inline! { example.run } diff --git a/spec/lib/gitlab/sidekiq_middleware_spec.rb b/spec/lib/gitlab/sidekiq_middleware_spec.rb index e687c8e8cf7..14dbeac37e8 100644 --- a/spec/lib/gitlab/sidekiq_middleware_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware_spec.rb @@ -60,7 +60,6 @@ RSpec.describe Gitlab::SidekiqMiddleware do ::Labkit::Middleware::Sidekiq::Server, ::Gitlab::SidekiqMiddleware::ServerMetrics, ::Gitlab::SidekiqMiddleware::ArgumentsLogger, - ::Gitlab::SidekiqMiddleware::MemoryKiller, ::Gitlab::SidekiqMiddleware::RequestStoreMiddleware, ::Gitlab::SidekiqMiddleware::ExtraDoneLogMetadata, ::Gitlab::SidekiqMiddleware::BatchLoader, @@ -79,8 +78,7 @@ RSpec.describe Gitlab::SidekiqMiddleware do with_sidekiq_server_middleware do |chain| described_class.server_configurator( metrics: true, - arguments_logger: true, - memory_killer: true + arguments_logger: true ).call(chain) Sidekiq::Testing.inline! { example.run } @@ -112,16 +110,14 @@ RSpec.describe Gitlab::SidekiqMiddleware do let(:configurator) do described_class.server_configurator( metrics: false, - arguments_logger: false, - memory_killer: false + arguments_logger: false ) end let(:disabled_sidekiq_middlewares) do [ Gitlab::SidekiqMiddleware::ServerMetrics, - Gitlab::SidekiqMiddleware::ArgumentsLogger, - Gitlab::SidekiqMiddleware::MemoryKiller + Gitlab::SidekiqMiddleware::ArgumentsLogger ] end diff --git a/spec/lib/gitlab/sidekiq_status_spec.rb b/spec/lib/gitlab/sidekiq_status_spec.rb index 027697db7e1..7f1504a8df9 100644 --- a/spec/lib/gitlab/sidekiq_status_spec.rb +++ b/spec/lib/gitlab/sidekiq_status_spec.rb @@ -11,7 +11,7 @@ RSpec.describe Gitlab::SidekiqStatus, :clean_gitlab_redis_queues, :clean_gitlab_ key = described_class.key_for('123') with_redis do |redis| - expect(redis.exists(key)).to eq(true) + expect(redis.exists?(key)).to eq(true) expect(redis.ttl(key) > 0).to eq(true) expect(redis.get(key)).to eq('1') end @@ -23,7 +23,7 @@ RSpec.describe Gitlab::SidekiqStatus, :clean_gitlab_redis_queues, :clean_gitlab_ key = described_class.key_for('123') with_redis do |redis| - expect(redis.exists(key)).to eq(true) + expect(redis.exists?(key)).to eq(true) expect(redis.ttl(key) > described_class::DEFAULT_EXPIRATION).to eq(true) expect(redis.get(key)).to eq('1') end @@ -35,7 +35,7 @@ RSpec.describe Gitlab::SidekiqStatus, :clean_gitlab_redis_queues, :clean_gitlab_ key = described_class.key_for('123') with_redis do |redis| - expect(redis.exists(key)).to eq(false) + expect(redis.exists?(key)).to eq(false) end end end @@ -48,7 +48,7 @@ RSpec.describe Gitlab::SidekiqStatus, :clean_gitlab_redis_queues, :clean_gitlab_ key = described_class.key_for('123') with_redis do |redis| - expect(redis.exists(key)).to eq(false) + expect(redis.exists?(key)).to eq(false) end end end diff --git a/spec/lib/gitlab/slash_commands/issue_new_spec.rb b/spec/lib/gitlab/slash_commands/issue_new_spec.rb index c17cee887ee..29a941f3691 100644 --- a/spec/lib/gitlab/slash_commands/issue_new_spec.rb +++ b/spec/lib/gitlab/slash_commands/issue_new_spec.rb @@ -53,6 +53,21 @@ RSpec.describe Gitlab::SlashCommands::IssueNew do expect(subject[:response_type]).to be(:ephemeral) expect(subject[:text]).to match("- Title is too long") end + + context 'when create issue service return an unrecoverable error' do + let(:regex_match) { described_class.match("issue create title}") } + + before do + allow_next_instance_of(Issues::CreateService) do |create_service| + allow(create_service).to receive(:execute).and_return(ServiceResponse.error(message: 'unauthorized')) + end + end + + it 'displays the errors' do + expect(subject[:response_type]).to be(:ephemeral) + expect(subject[:text]).to eq('unauthorized') + end + end end end diff --git a/spec/lib/gitlab/ssh_public_key_spec.rb b/spec/lib/gitlab/ssh_public_key_spec.rb index 114a18cf99a..a2524314458 100644 --- a/spec/lib/gitlab/ssh_public_key_spec.rb +++ b/spec/lib/gitlab/ssh_public_key_spec.rb @@ -88,12 +88,12 @@ RSpec.describe Gitlab::SSHPublicKey, lib: true, fips_mode: false do it 'returns all supported algorithms' do expect(described_class.supported_algorithms).to eq( %w( - ssh-rsa - ssh-dss - ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521 - ssh-ed25519 - sk-ecdsa-sha2-nistp256@openssh.com - sk-ssh-ed25519@openssh.com + ssh-rsa + ssh-dss + ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521 + ssh-ed25519 + sk-ecdsa-sha2-nistp256@openssh.com + sk-ssh-ed25519@openssh.com ) ) end @@ -102,12 +102,12 @@ RSpec.describe Gitlab::SSHPublicKey, lib: true, fips_mode: false do it 'returns all supported algorithms' do expect(described_class.supported_algorithms).to eq( %w( - ssh-rsa - ssh-dss - ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521 - ssh-ed25519 - sk-ecdsa-sha2-nistp256@openssh.com - sk-ssh-ed25519@openssh.com + ssh-rsa + ssh-dss + ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521 + ssh-ed25519 + sk-ecdsa-sha2-nistp256@openssh.com + sk-ssh-ed25519@openssh.com ) ) end diff --git a/spec/lib/gitlab/tracking/service_ping_context_spec.rb b/spec/lib/gitlab/tracking/service_ping_context_spec.rb new file mode 100644 index 00000000000..d70dfaa4e0b --- /dev/null +++ b/spec/lib/gitlab/tracking/service_ping_context_spec.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Tracking::ServicePingContext do + describe '#init' do + it 'does not accept unsupported data sources' do + expect { described_class.new(data_source: :random, event: 'event a') }.to raise_error(ArgumentError) + end + end + + describe '#to_context' do + let(:subject) { described_class.new(data_source: :redis_hll, event: 'sample_event') } + + it 'contains event_name' do + expect(subject.to_context.to_json.dig(:data, :event_name)).to eq('sample_event') + end + end +end diff --git a/spec/lib/gitlab/usage/metrics/aggregates/aggregate_spec.rb b/spec/lib/gitlab/usage/metrics/aggregates/aggregate_spec.rb index 8e02f4f562c..76eec2755df 100644 --- a/spec/lib/gitlab/usage/metrics/aggregates/aggregate_spec.rb +++ b/spec/lib/gitlab/usage/metrics/aggregates/aggregate_spec.rb @@ -235,10 +235,27 @@ RSpec.describe Gitlab::Usage::Metrics::Aggregates::Aggregate, :clean_gitlab_redi end end - it 'allows for YAML aliases in aggregated metrics configs' do - expect(YAML).to receive(:safe_load).with(kind_of(String), aliases: true).at_least(:once) + context 'legacy aggregated metrics configuration' do + let(:temp_dir) { Dir.mktmpdir } + let(:temp_file) { Tempfile.new(%w[common .yml], temp_dir) } + + before do + stub_const("#{namespace}::AGGREGATED_METRICS_PATH", File.expand_path('*.yml', temp_dir)) + File.open(temp_file.path, "w+b") do |file| + file.write [aggregated_metric(name: "gmau_1", time_frame: '7d')].to_yaml + end + end + + after do + temp_file.unlink + FileUtils.remove_entry(temp_dir) if Dir.exist?(temp_dir) + end - described_class.new(recorded_at) + it 'allows for YAML aliases in aggregated metrics configs' do + expect(YAML).to receive(:safe_load).with(kind_of(String), aliases: true).at_least(:once) + + described_class.new(recorded_at) + end end describe '.aggregated_metrics_weekly_data' do @@ -260,5 +277,132 @@ RSpec.describe Gitlab::Usage::Metrics::Aggregates::Aggregate, :clean_gitlab_redi it_behaves_like 'database_sourced_aggregated_metrics' it_behaves_like 'redis_sourced_aggregated_metrics' end + + describe '.calculate_count_for_aggregation' do + using RSpec::Parameterized::TableSyntax + + context 'with valid configuration' do + where(:number_of_days, :operator, :datasource, :expected_method) do + 28 | 'AND' | 'redis' | :calculate_metrics_intersections + 7 | 'AND' | 'redis' | :calculate_metrics_intersections + 28 | 'AND' | 'database' | :calculate_metrics_intersections + 7 | 'AND' | 'database' | :calculate_metrics_intersections + 28 | 'OR' | 'redis' | :calculate_metrics_union + 7 | 'OR' | 'redis' | :calculate_metrics_union + 28 | 'OR' | 'database' | :calculate_metrics_union + 7 | 'OR' | 'database' | :calculate_metrics_union + end + + with_them do + let(:time_frame) { "#{number_of_days}d" } + let(:start_date) { number_of_days.days.ago.to_date } + let(:params) { { start_date: start_date, end_date: end_date, recorded_at: recorded_at } } + let(:aggregate) do + { + source: datasource, + operator: operator, + events: %w[event1 event2] + } + end + + subject(:calculate_count_for_aggregation) do + described_class + .new(recorded_at) + .calculate_count_for_aggregation(aggregation: aggregate, time_frame: time_frame) + end + + it 'returns the number of unique events for aggregation', :aggregate_failures do + expect(namespace::SOURCES[datasource]) + .to receive(expected_method) + .with(params.merge(metric_names: %w[event1 event2])) + .and_return(5) + expect(calculate_count_for_aggregation).to eq(5) + end + end + end + + context 'with invalid configuration' do + where(:time_frame, :operator, :datasource, :expected_error) do + '28d' | 'SUM' | 'redis' | namespace::UnknownAggregationOperator + '7d' | 'AND' | 'mongodb' | namespace::UnknownAggregationSource + 'all' | 'AND' | 'redis' | namespace::DisallowedAggregationTimeFrame + end + + with_them do + let(:aggregate) do + { + source: datasource, + operator: operator, + events: %w[event1 event2] + } + end + + subject(:calculate_count_for_aggregation) do + described_class + .new(recorded_at) + .calculate_count_for_aggregation(aggregation: aggregate, time_frame: time_frame) + end + + context 'with non prod environment' do + it 'raises error' do + expect { calculate_count_for_aggregation }.to raise_error expected_error + end + end + + context 'with prod environment' do + before do + stub_rails_env('production') + end + + it 'returns fallback value' do + expect(calculate_count_for_aggregation).to be(-1) + end + end + end + end + + context 'when union data is not available' do + subject(:calculate_count_for_aggregation) do + described_class + .new(recorded_at) + .calculate_count_for_aggregation(aggregation: aggregate, time_frame: time_frame) + end + + where(:time_frame, :operator, :datasource) do + '28d' | 'OR' | 'redis' + '7d' | 'OR' | 'database' + end + + with_them do + before do + allow(namespace::SOURCES[datasource]).to receive(:calculate_metrics_union).and_raise(sources::UnionNotAvailable) + end + + let(:aggregate) do + { + source: datasource, + operator: operator, + events: %w[event1 event2] + } + end + + context 'with non prod environment' do + it 'raises error' do + expect { calculate_count_for_aggregation }.to raise_error sources::UnionNotAvailable + end + end + + context 'with prod environment' do + before do + stub_rails_env('production') + end + + it 'returns fallback value' do + expect(calculate_count_for_aggregation).to be(-1) + end + end + end + end + end end end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/aggregated_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/aggregated_metric_spec.rb new file mode 100644 index 00000000000..3e7b13e21c1 --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/aggregated_metric_spec.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::AggregatedMetric, :clean_gitlab_redis_shared_state do + using RSpec::Parameterized::TableSyntax + before do + # weekly AND 1 weekly OR 2 + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(:i_quickactions_approve, values: 1, time: 1.week.ago) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(:i_quickactions_unapprove, values: 1, time: 1.week.ago) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(:i_quickactions_unapprove, values: 2, time: 1.week.ago) + + # monthly AND 2 weekly OR 3 + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(:i_quickactions_approve, values: 2, time: 2.weeks.ago) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(:i_quickactions_unapprove, values: 3, time: 2.weeks.ago) + + # out of date range + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(:i_quickactions_approve, values: 3, time: 2.months.ago) + + # database events + Gitlab::Usage::Metrics::Aggregates::Sources::PostgresHll + .save_aggregated_metrics( + metric_name: :i_quickactions_approve, + time_period: { created_at: (1.week.ago..Date.current) }, + recorded_at_timestamp: Time.current, + data: ::Gitlab::Database::PostgresHll::Buckets.new(141 => 1, 56 => 1) + ) + Gitlab::Usage::Metrics::Aggregates::Sources::PostgresHll + .save_aggregated_metrics( + metric_name: :i_quickactions_unapprove, + time_period: { created_at: (1.week.ago..Date.current) }, + recorded_at_timestamp: Time.current, + data: ::Gitlab::Database::PostgresHll::Buckets.new(10 => 1, 56 => 1) + ) + end + + where(:data_source, :time_frame, :operator, :expected_value) do + 'redis_hll' | '28d' | 'AND' | 2 + 'redis_hll' | '28d' | 'OR' | 3 + 'redis_hll' | '7d' | 'AND' | 1 + 'redis_hll' | '7d' | 'OR' | 2 + 'database' | '7d' | 'OR' | 3.0 + 'database' | '7d' | 'AND' | 1.0 + end + + with_them do + let(:error_rate) { Gitlab::Database::PostgresHll::BatchDistinctCounter::ERROR_RATE } + let(:metric_definition) do + { + data_source: data_source, + time_frame: time_frame, + options: { + aggregate: { + operator: operator + }, + events: %w[ + i_quickactions_approve + i_quickactions_unapprove + ] + } + } + end + + around do |example| + freeze_time { example.run } + end + + it 'has correct value' do + expect(described_class.new(metric_definition).value).to be_within(error_rate).percent_of(expected_value) + end + end +end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/distinct_count_projects_with_expiration_policy_disabled_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/distinct_count_projects_with_expiration_policy_disabled_metric_spec.rb new file mode 100644 index 00000000000..757adee6117 --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/distinct_count_projects_with_expiration_policy_disabled_metric_spec.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::DistinctCountProjectsWithExpirationPolicyDisabledMetric do + before_all do + create(:container_expiration_policy, enabled: false) + create(:container_expiration_policy, enabled: false, created_at: 29.days.ago) + create(:container_expiration_policy, enabled: true) + end + + it_behaves_like 'a correct instrumented metric value', { time_frame: '28d' } do + let(:expected_value) { 1 } + end + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'all' } do + let(:expected_value) { 2 } + end +end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/gitlab_for_jira_app_direct_installations_count_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/gitlab_for_jira_app_direct_installations_count_metric_spec.rb new file mode 100644 index 00000000000..061558085a1 --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/gitlab_for_jira_app_direct_installations_count_metric_spec.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::GitlabForJiraAppDirectInstallationsCountMetric do + before do + create(:jira_connect_subscription) + end + + let(:expected_value) { 1 } + let(:expected_query) do + 'SELECT COUNT("jira_connect_installations"."id") FROM "jira_connect_installations"'\ + ' INNER JOIN "jira_connect_subscriptions" ON "jira_connect_subscriptions"."jira_connect_installation_id"'\ + ' = "jira_connect_installations"."id"' + end + + it_behaves_like 'a correct instrumented metric value and query', { time_frame: 'all' } +end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/gitlab_for_jira_app_proxy_installations_count_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/gitlab_for_jira_app_proxy_installations_count_metric_spec.rb new file mode 100644 index 00000000000..4535bab7702 --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/gitlab_for_jira_app_proxy_installations_count_metric_spec.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::GitlabForJiraAppProxyInstallationsCountMetric do + let_it_be(:installation) { create(:jira_connect_installation, instance_url: 'http://self-managed-gitlab.com') } + + before do + create(:jira_connect_subscription, installation: installation) + end + + let(:expected_value) { 1 } + let(:expected_query) do + 'SELECT COUNT("jira_connect_installations"."id") FROM "jira_connect_installations"'\ + ' WHERE "jira_connect_installations"."instance_url" IS NOT NULL' + end + + it_behaves_like 'a correct instrumented metric value and query', { time_frame: 'all' } +end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/merge_request_widget_extension_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/merge_request_widget_extension_metric_spec.rb new file mode 100644 index 00000000000..c0ac00c9cdd --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/merge_request_widget_extension_metric_spec.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::MergeRequestWidgetExtensionMetric, + :clean_gitlab_redis_shared_state do + before do + 4.times do + Gitlab::UsageDataCounters::MergeRequestWidgetExtensionCounter.count(:terraform_count_expand) + end + end + + let(:expected_value) { 4 } + + it_behaves_like 'a correct instrumented metric value', { + options: { event: 'expand', widget: 'terraform' }, + time_frame: 'all' + } + + it 'raises an exception if widget option is not present' do + expect do + described_class.new(options: { event: 'expand' }, time_frame: 'all') + end.to raise_error(ArgumentError, /'widget' option is required/) + end +end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/redis_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/redis_metric_spec.rb index 80ae5c6fd21..c4d6edd43e1 100644 --- a/spec/lib/gitlab/usage/metrics/instrumentations/redis_metric_spec.rb +++ b/spec/lib/gitlab/usage/metrics/instrumentations/redis_metric_spec.rb @@ -11,14 +11,21 @@ RSpec.describe Gitlab::Usage::Metrics::Instrumentations::RedisMetric, :clean_git let(:expected_value) { 4 } - it_behaves_like 'a correct instrumented metric value', { options: { event: 'pushes', prefix: 'source_code' } } + it_behaves_like 'a correct instrumented metric value', { + options: { event: 'pushes', prefix: 'source_code' }, + time_frame: 'all' + } it 'raises an exception if event option is not present' do - expect { described_class.new(prefix: 'source_code') }.to raise_error(ArgumentError) + expect do + described_class.new(options: { prefix: 'source_code' }, time_frame: 'all') + end.to raise_error(ArgumentError, /'event' option is required/) end it 'raises an exception if prefix option is not present' do - expect { described_class.new(event: 'pushes') }.to raise_error(ArgumentError) + expect do + described_class.new(options: { event: 'pushes' }, time_frame: 'all') + end.to raise_error(ArgumentError, /'prefix' option is required/) end describe 'children classes' do @@ -55,7 +62,22 @@ RSpec.describe Gitlab::Usage::Metrics::Instrumentations::RedisMetric, :clean_git end it_behaves_like 'a correct instrumented metric value', { - options: { event: 'merge_requests_count', prefix: 'web_ide', include_usage_prefix: false } + options: { event: 'merge_requests_count', prefix: 'web_ide', include_usage_prefix: false }, + time_frame: 'all' + } + end + + context "with prefix disabled" do + let(:expected_value) { 3 } + + before do + 3.times do + Gitlab::UsageDataCounters::SearchCounter.count(:all_searches) + end + end + + it_behaves_like 'a correct instrumented metric value', { + options: { event: 'all_searches_count', prefix: nil, include_usage_prefix: false }, time_frame: 'all' } end end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/work_items_activity_aggregated_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/work_items_activity_aggregated_metric_spec.rb new file mode 100644 index 00000000000..3e315692d0a --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/work_items_activity_aggregated_metric_spec.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::WorkItemsActivityAggregatedMetric do + let(:metric_definition) do + { + data_source: 'redis_hll', + time_frame: time_frame, + options: { + aggregate: { + operator: 'OR' + }, + events: %w[ + users_creating_work_items + users_updating_work_item_title + users_updating_work_item_dates + users_updating_work_item_iteration + ] + } + } + end + + around do |example| + freeze_time { example.run } + end + + where(:time_frame) { [['28d'], ['7d']] } + + with_them do + describe '#available?' do + it 'returns false without track_work_items_activity feature' do + stub_feature_flags(track_work_items_activity: false) + + expect(described_class.new(metric_definition).available?).to eq(false) + end + + it 'returns true with track_work_items_activity feature' do + stub_feature_flags(track_work_items_activity: true) + + expect(described_class.new(metric_definition).available?).to eq(true) + end + end + + describe '#value', :clean_gitlab_redis_shared_state do + let(:counter) { Gitlab::UsageDataCounters::HLLRedisCounter } + + before do + counter.track_event(:users_creating_work_items, values: 1, time: 1.week.ago) + counter.track_event(:users_updating_work_item_title, values: 1, time: 1.week.ago) + counter.track_event(:users_updating_work_item_dates, values: 2, time: 1.week.ago) + counter.track_event(:users_updating_work_item_iteration, values: 2, time: 1.week.ago) + end + + it 'has correct value' do + expect(described_class.new(metric_definition).value).to eq 2 + end + end + end +end diff --git a/spec/lib/gitlab/usage_data/topology_spec.rb b/spec/lib/gitlab/usage_data/topology_spec.rb index dfdf8eaabe8..3fb87e77457 100644 --- a/spec/lib/gitlab/usage_data/topology_spec.rb +++ b/spec/lib/gitlab/usage_data/topology_spec.rb @@ -523,210 +523,210 @@ RSpec.describe Gitlab::UsageData::Topology do receive(:query) .with(/gitlab_usage_ping:ops:rate/) .and_return(result || [ - { - 'metric' => { 'component' => 'http_requests', 'service' => 'workhorse' }, - 'value' => [1000, '0.01'] - } - ]) + { + 'metric' => { 'component' => 'http_requests', 'service' => 'workhorse' }, + 'value' => [1000, '0.01'] + } + ]) end def receive_query_apdex_ratio_query(result: nil) receive(:query) .with(/gitlab_usage_ping:sql_duration_apdex:ratio_rate5m/) .and_return(result || [ - { - 'metric' => {}, - 'value' => [1000, '0.996'] - } - ]) + { + 'metric' => {}, + 'value' => [1000, '0.996'] + } + ]) end def receive_node_memory_query(result: nil) receive(:query) .with(/node_memory_total_bytes/, an_instance_of(Hash)) .and_return(result || [ - { - 'metric' => { 'instance' => 'instance1:8080' }, - 'value' => [1000, '512'] - }, - { - 'metric' => { 'instance' => 'instance2:8090' }, - 'value' => [1000, '1024'] - } - ]) + { + 'metric' => { 'instance' => 'instance1:8080' }, + 'value' => [1000, '512'] + }, + { + 'metric' => { 'instance' => 'instance2:8090' }, + 'value' => [1000, '1024'] + } + ]) end def receive_node_memory_utilization_query(result: nil) receive(:query) .with(/node_memory_utilization/, an_instance_of(Hash)) .and_return(result || [ - { - 'metric' => { 'instance' => 'instance1:8080' }, - 'value' => [1000, '0.45'] - }, - { - 'metric' => { 'instance' => 'instance2:8090' }, - 'value' => [1000, '0.25'] - } - ]) + { + 'metric' => { 'instance' => 'instance1:8080' }, + 'value' => [1000, '0.45'] + }, + { + 'metric' => { 'instance' => 'instance2:8090' }, + 'value' => [1000, '0.25'] + } + ]) end def receive_node_cpu_count_query(result: nil) receive(:query) .with(/node_cpus/, an_instance_of(Hash)) .and_return(result || [ - { - 'metric' => { 'instance' => 'instance2:8090' }, - 'value' => [1000, '16'] - }, - { - 'metric' => { 'instance' => 'instance1:8080' }, - 'value' => [1000, '8'] - } - ]) + { + 'metric' => { 'instance' => 'instance2:8090' }, + 'value' => [1000, '16'] + }, + { + 'metric' => { 'instance' => 'instance1:8080' }, + 'value' => [1000, '8'] + } + ]) end def receive_node_cpu_utilization_query(result: nil) receive(:query) .with(/node_cpu_utilization/, an_instance_of(Hash)) .and_return(result || [ - { - 'metric' => { 'instance' => 'instance2:8090' }, - 'value' => [1000, '0.2'] - }, - { - 'metric' => { 'instance' => 'instance1:8080' }, - 'value' => [1000, '0.1'] - } - ]) + { + 'metric' => { 'instance' => 'instance2:8090' }, + 'value' => [1000, '0.2'] + }, + { + 'metric' => { 'instance' => 'instance1:8080' }, + 'value' => [1000, '0.1'] + } + ]) end def receive_node_uname_info_query(result: nil) receive(:query) .with('node_uname_info') .and_return(result || [ - { - "metric" => { - "__name__" => "node_uname_info", - "domainname" => "(none)", - "instance" => "instance1:9100", - "job" => "node_exporter", - "machine" => "x86_64", - "nodename" => "instance1", - "release" => "4.19.76-linuxkit", - "sysname" => "Linux" - }, - "value" => [1592463033.359, "1"] - }, - { - "metric" => { - "__name__" => "node_uname_info", - "domainname" => "(none)", - "instance" => "instance2:9100", - "job" => "node_exporter", - "machine" => "x86_64", - "nodename" => "instance2", - "release" => "4.15.0-101-generic", - "sysname" => "Linux" - }, - "value" => [1592463033.359, "1"] - } - ]) + { + "metric" => { + "__name__" => "node_uname_info", + "domainname" => "(none)", + "instance" => "instance1:9100", + "job" => "node_exporter", + "machine" => "x86_64", + "nodename" => "instance1", + "release" => "4.19.76-linuxkit", + "sysname" => "Linux" + }, + "value" => [1592463033.359, "1"] + }, + { + "metric" => { + "__name__" => "node_uname_info", + "domainname" => "(none)", + "instance" => "instance2:9100", + "job" => "node_exporter", + "machine" => "x86_64", + "nodename" => "instance2", + "release" => "4.15.0-101-generic", + "sysname" => "Linux" + }, + "value" => [1592463033.359, "1"] + } + ]) end def receive_node_service_memory_rss_query(result: nil) receive(:query) .with(/process_resident_memory_bytes/, an_instance_of(Hash)) .and_return(result || [ - { - 'metric' => { 'instance' => 'instance1:8080', 'job' => 'gitlab-rails' }, - 'value' => [1000, '300'] - }, - { - 'metric' => { 'instance' => 'instance1:8090', 'job' => 'gitlab-sidekiq' }, - 'value' => [1000, '303'] - }, - # instance 2: runs a dedicated Sidekiq + Redis (which uses a different metric name) - { - 'metric' => { 'instance' => 'instance2:8090', 'job' => 'gitlab-sidekiq' }, - 'value' => [1000, '400'] - }, - { - 'metric' => { 'instance' => 'instance2:9121', 'job' => 'redis' }, - 'value' => [1000, '402'] - } - ]) + { + 'metric' => { 'instance' => 'instance1:8080', 'job' => 'gitlab-rails' }, + 'value' => [1000, '300'] + }, + { + 'metric' => { 'instance' => 'instance1:8090', 'job' => 'gitlab-sidekiq' }, + 'value' => [1000, '303'] + }, + # instance 2: runs a dedicated Sidekiq + Redis (which uses a different metric name) + { + 'metric' => { 'instance' => 'instance2:8090', 'job' => 'gitlab-sidekiq' }, + 'value' => [1000, '400'] + }, + { + 'metric' => { 'instance' => 'instance2:9121', 'job' => 'redis' }, + 'value' => [1000, '402'] + } + ]) end def receive_node_service_memory_uss_query(result: nil) receive(:query) .with(/process_unique_memory_bytes/, an_instance_of(Hash)) .and_return(result || [ - { - 'metric' => { 'instance' => 'instance1:8080', 'job' => 'gitlab-rails' }, - 'value' => [1000, '301'] - } - ]) + { + 'metric' => { 'instance' => 'instance1:8080', 'job' => 'gitlab-rails' }, + 'value' => [1000, '301'] + } + ]) end def receive_node_service_memory_pss_query(result: nil) receive(:query) .with(/process_proportional_memory_bytes/, an_instance_of(Hash)) .and_return(result || [ - { - 'metric' => { 'instance' => 'instance1:8080', 'job' => 'gitlab-rails' }, - 'value' => [1000, '302'] - }, - { - 'metric' => { 'instance' => 'instance2:8090', 'job' => 'gitlab-sidekiq' }, - 'value' => [1000, '401'] - } - ]) + { + 'metric' => { 'instance' => 'instance1:8080', 'job' => 'gitlab-rails' }, + 'value' => [1000, '302'] + }, + { + 'metric' => { 'instance' => 'instance2:8090', 'job' => 'gitlab-sidekiq' }, + 'value' => [1000, '401'] + } + ]) end def receive_node_service_process_count_query(result: nil) receive(:query) .with(/service_process:count/, an_instance_of(Hash)) .and_return(result || [ - # instance 1 - { - 'metric' => { 'instance' => 'instance1:8080', 'job' => 'gitlab-rails' }, - 'value' => [1000, '10'] - }, - { - 'metric' => { 'instance' => 'instance1:8090', 'job' => 'gitlab-sidekiq' }, - 'value' => [1000, '5'] - }, - # instance 2 - { - 'metric' => { 'instance' => 'instance2:8090', 'job' => 'gitlab-sidekiq' }, - 'value' => [1000, '15'] - }, - { - 'metric' => { 'instance' => 'instance2:9121', 'job' => 'redis' }, - 'value' => [1000, '1'] - }, - { - 'metric' => { 'instance' => 'instance2:8080', 'job' => 'registry' }, - 'value' => [1000, '1'] - } - ]) + # instance 1 + { + 'metric' => { 'instance' => 'instance1:8080', 'job' => 'gitlab-rails' }, + 'value' => [1000, '10'] + }, + { + 'metric' => { 'instance' => 'instance1:8090', 'job' => 'gitlab-sidekiq' }, + 'value' => [1000, '5'] + }, + # instance 2 + { + 'metric' => { 'instance' => 'instance2:8090', 'job' => 'gitlab-sidekiq' }, + 'value' => [1000, '15'] + }, + { + 'metric' => { 'instance' => 'instance2:9121', 'job' => 'redis' }, + 'value' => [1000, '1'] + }, + { + 'metric' => { 'instance' => 'instance2:8080', 'job' => 'registry' }, + 'value' => [1000, '1'] + } + ]) end def receive_node_service_app_server_workers_query(result: nil) receive(:query) .with(/app_server_workers/, an_instance_of(Hash)) .and_return(result || [ - # instance 1 - { - 'metric' => { 'instance' => 'instance1:8080', 'job' => 'gitlab-rails', 'server' => 'puma' }, - 'value' => [1000, '2'] - }, - # instance 2 - { - 'metric' => { 'instance' => 'instance2:8080', 'job' => 'gitlab-rails', 'server' => 'puma' }, - 'value' => [1000, '1'] - } - ]) + # instance 1 + { + 'metric' => { 'instance' => 'instance1:8080', 'job' => 'gitlab-rails', 'server' => 'puma' }, + 'value' => [1000, '2'] + }, + # instance 2 + { + 'metric' => { 'instance' => 'instance2:8080', 'job' => 'gitlab-rails', 'server' => 'puma' }, + 'value' => [1000, '1'] + } + ]) end end diff --git a/spec/lib/gitlab/usage_data_counters/ci_template_unique_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/ci_template_unique_counter_spec.rb index 6a37bfd106d..1ca0bb0e9ea 100644 --- a/spec/lib/gitlab/usage_data_counters/ci_template_unique_counter_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/ci_template_unique_counter_spec.rb @@ -65,17 +65,11 @@ RSpec.describe Gitlab::UsageDataCounters::CiTemplateUniqueCounter do context 'with implicit includes', :snowplow do let(:config_source) { :auto_devops_source } - [ - ['', ['Auto-DevOps.gitlab-ci.yml']], - ['Jobs', described_class.ci_templates('lib/gitlab/ci/templates/Jobs')], - ['Security', described_class.ci_templates('lib/gitlab/ci/templates/Security')] - ].each do |directory, templates| - templates.each do |template| - context "for #{template}" do - let(:template_path) { File.join(directory, template) } - - include_examples 'tracks template' - end + described_class.all_included_templates('Auto-DevOps.gitlab-ci.yml').each do |template_name| + context "for #{template_name}" do + let(:template_path) { Gitlab::Template::GitlabCiYmlTemplate.find(template_name.delete_suffix('.gitlab-ci.yml')).full_name } + + include_examples 'tracks template' end end end diff --git a/spec/lib/gitlab/usage_data_counters/code_review_events_spec.rb b/spec/lib/gitlab/usage_data_counters/code_review_events_spec.rb index 01396602f29..e122d9a3026 100644 --- a/spec/lib/gitlab/usage_data_counters/code_review_events_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/code_review_events_spec.rb @@ -6,19 +6,22 @@ require 'spec_helper' # NOTE: ONLY user related metrics to be added to the aggregates - otherwise add it to the exception list RSpec.describe 'Code review events' do it 'the aggregated metrics contain all the code review metrics' do - path = Rails.root.join('config/metrics/aggregates/code_review.yml') - aggregated_events = YAML.safe_load(File.read(path), aliases: true)&.map(&:with_indifferent_access) - - code_review_aggregated_events = aggregated_events - .map { |event| event['events'] } - .flatten - .uniq - code_review_events = Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category("code_review") + code_review_aggregated_events = Gitlab::Usage::MetricDefinition.all.flat_map do |definition| + next [] unless code_review_aggregated_metric?(definition.attributes) + + definition.attributes.dig(:options, :events) + end.uniq exceptions = %w[i_code_review_mr_diffs i_code_review_mr_with_invalid_approvers i_code_review_mr_single_file_diffs i_code_review_total_suggestions_applied i_code_review_total_suggestions_added i_code_review_create_note_in_ipynb_diff i_code_review_create_note_in_ipynb_diff_mr i_code_review_create_note_in_ipynb_diff_commit] code_review_aggregated_events += exceptions expect(code_review_events - code_review_aggregated_events).to be_empty end + + def code_review_aggregated_metric?(attributes) + return false unless attributes[:product_group] == 'code_review' && attributes[:status] == 'active' + + attributes[:instrumentation_class] == 'AggregatedMetric' + end end diff --git a/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb index 3fb2532521a..d0b935d59dd 100644 --- a/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb @@ -24,8 +24,10 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s context 'migration to instrumentation classes data collection' do let_it_be(:instrumented_events) do + instrumentation_classes = %w[AggregatedMetric RedisHLLMetric] ::Gitlab::Usage::MetricDefinition.all.map do |definition| - next unless definition.attributes[:instrumentation_class] == 'RedisHLLMetric' && definition.available? + next unless definition.available? + next unless instrumentation_classes.include?(definition.attributes[:instrumentation_class]) definition.attributes.dig(:options, :events)&.sort end.compact.to_set @@ -96,21 +98,17 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s 'source_code', 'incident_management', 'incident_management_alerts', - 'incident_management_oncall', 'testing', 'issues_edit', - 'ci_secrets_management', 'snippets', 'code_review', 'terraform', 'ci_templates', 'quickactions', 'pipeline_authoring', - 'epics_usage', 'secure', 'importer', 'geo', - 'growth', 'work_items', 'ci_users', 'error_tracking', diff --git a/spec/lib/gitlab/usage_data_counters/work_item_activity_unique_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/work_item_activity_unique_counter_spec.rb index 0bcdbe82a7a..2d251017c87 100644 --- a/spec/lib/gitlab/usage_data_counters/work_item_activity_unique_counter_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/work_item_activity_unique_counter_spec.rb @@ -28,4 +28,12 @@ RSpec.describe Gitlab::UsageDataCounters::WorkItemActivityUniqueCounter, :clean_ it_behaves_like 'work item unique counter' end + + describe '.track_work_item_labels_changed_action' do + subject(:track_event) { described_class.track_work_item_labels_changed_action(author: user) } + + let(:event_name) { described_class::WORK_ITEM_LABELS_CHANGED } + + it_behaves_like 'work item unique counter' + end end diff --git a/spec/lib/gitlab/usage_data_metrics_spec.rb b/spec/lib/gitlab/usage_data_metrics_spec.rb index ed0eabf1b4d..5d58933f1fd 100644 --- a/spec/lib/gitlab/usage_data_metrics_spec.rb +++ b/spec/lib/gitlab/usage_data_metrics_spec.rb @@ -31,6 +31,8 @@ RSpec.describe Gitlab::UsageDataMetrics do it 'includes counts keys', :aggregate_failures do expect(subject[:counts]).to include(:boards) expect(subject[:counts]).to include(:issues) + expect(subject[:counts]).to include(:gitlab_for_jira_app_direct_installations) + expect(subject[:counts]).to include(:gitlab_for_jira_app_proxy_installations) end it 'includes usage_activity_by_stage keys' do diff --git a/spec/lib/gitlab/usage_data_spec.rb b/spec/lib/gitlab/usage_data_spec.rb index 46ed4b57d3a..cb645ae3e53 100644 --- a/spec/lib/gitlab/usage_data_spec.rb +++ b/spec/lib/gitlab/usage_data_spec.rb @@ -624,7 +624,6 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do it 'gathers usage data' do expect(subject[:projects_with_expiration_policy_enabled]).to eq 19 - expect(subject[:projects_with_expiration_policy_disabled]).to eq 5 expect(subject[:projects_with_expiration_policy_enabled_with_keep_n_unset]).to eq 1 expect(subject[:projects_with_expiration_policy_enabled_with_keep_n_set_to_1]).to eq 1 @@ -758,13 +757,6 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do end end - describe '.usage_counters' do - subject { described_class.usage_counters } - - it { is_expected.to include(:kubernetes_agent_gitops_sync) } - it { is_expected.to include(:kubernetes_agent_k8s_api_proxy_request) } - end - describe '.usage_data_counters' do subject { described_class.usage_data_counters } @@ -1057,12 +1049,13 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do before do allow(described_class).to receive(:operating_system).and_return('ubuntu-20.04') - expect(prometheus_client).to receive(:query).with(/gitlab_usage_ping:gitaly_apdex:ratio_avg_over_time_5m/).and_return([ - { - 'metric' => {}, - 'value' => [1616016381.473, '0.95'] - } - ]) + expect(prometheus_client).to receive(:query) + .with(/gitlab_usage_ping:gitaly_apdex:ratio_avg_over_time_5m/) + .and_return( + [ + { 'metric' => {}, + 'value' => [1616016381.473, '0.95'] } + ]) expect(described_class).to receive(:with_prometheus_client).and_yield(prometheus_client) end diff --git a/spec/lib/gitlab/user_access_snippet_spec.rb b/spec/lib/gitlab/user_access_snippet_spec.rb index 4143a3017e8..916e920e2ac 100644 --- a/spec/lib/gitlab/user_access_snippet_spec.rb +++ b/spec/lib/gitlab/user_access_snippet_spec.rb @@ -49,7 +49,7 @@ RSpec.describe Gitlab::UserAccessSnippet do end describe '#can_push_to_branch?' do - include ProjectHelpers + include UserHelpers [:anonymous, :non_member, :guest, :reporter, :maintainer, :admin, :author].each do |membership| context membership.to_s do diff --git a/spec/lib/gitlab/utils_spec.rb b/spec/lib/gitlab/utils_spec.rb index 61323f0646b..d1fdaf7a9db 100644 --- a/spec/lib/gitlab/utils_spec.rb +++ b/spec/lib/gitlab/utils_spec.rb @@ -582,11 +582,12 @@ RSpec.describe Gitlab::Utils do end it 'sorts items like the regular sort_by' do - expect(sorted_list).to eq([ - { name: 'obj 2', priority: 1 }, - { name: 'obj 1', priority: 2 }, - { name: 'obj 3', priority: 3 } - ]) + expect(sorted_list).to eq( + [ + { name: 'obj 2', priority: 1 }, + { name: 'obj 1', priority: 2 }, + { name: 'obj 3', priority: 3 } + ]) end end end diff --git a/spec/lib/gitlab/web_ide/config/entry/terminal_spec.rb b/spec/lib/gitlab/web_ide/config/entry/terminal_spec.rb index 8d4629bf48b..7d96adf95e8 100644 --- a/spec/lib/gitlab/web_ide/config/entry/terminal_spec.rb +++ b/spec/lib/gitlab/web_ide/config/entry/terminal_spec.rb @@ -150,29 +150,6 @@ RSpec.describe Gitlab::WebIde::Config::Entry::Terminal do } ) end - - context 'when the FF ci_variables_refactoring_to_variable is disabled' do - let(:entry_without_ff) { described_class.new(config, with_image_ports: true) } - - before do - stub_feature_flags(ci_variables_refactoring_to_variable: false) - entry_without_ff.compose! - end - - it 'returns correct value' do - expect(entry_without_ff.value) - .to eq( - tag_list: ['webide'], - job_variables: [{ key: 'KEY', value: 'value', public: true }], - options: { - image: { name: "image:1.0" }, - services: [{ name: "mysql" }], - before_script: %w[ls pwd], - script: ['sleep 100'] - } - ) - end - end end end end diff --git a/spec/lib/gitlab/webpack/manifest_spec.rb b/spec/lib/gitlab/webpack/manifest_spec.rb index 08b4774dd67..24a36258379 100644 --- a/spec/lib/gitlab/webpack/manifest_spec.rb +++ b/spec/lib/gitlab/webpack/manifest_spec.rb @@ -66,10 +66,11 @@ RSpec.describe Gitlab::Webpack::Manifest do describe "webpack errors" do context "when webpack has 'Module build failed' errors in its manifest" do it "errors" do - error_manifest = Gitlab::Json.parse(manifest).merge("errors" => [ - "somethingModule build failed something", - "I am an error" - ]).to_json + error_manifest = Gitlab::Json.parse(manifest).merge("errors" => + [ + "somethingModule build failed something", + "I am an error" + ]).to_json stub_request(:get, "http://hostname:2000/public_path/my_manifest.json").to_return(body: error_manifest, status: 200) expect { Gitlab::Webpack::Manifest.asset_paths("entry1") }.to raise_error(Gitlab::Webpack::Manifest::WebpackError) diff --git a/spec/lib/gitlab/x509/signature_spec.rb b/spec/lib/gitlab/x509/signature_spec.rb index 5626e49bfe1..31f66232f38 100644 --- a/spec/lib/gitlab/x509/signature_spec.rb +++ b/spec/lib/gitlab/x509/signature_spec.rb @@ -30,6 +30,20 @@ RSpec.describe Gitlab::X509::Signature do expect(signature.verification_status).to eq(:verified) end + it 'returns a verified signature if email does match, case-insensitively' do + signature = described_class.new( + X509Helpers::User1.signed_commit_signature, + X509Helpers::User1.signed_commit_base_data, + X509Helpers::User1.certificate_email.upcase, + X509Helpers::User1.signed_commit_time + ) + + expect(signature.x509_certificate).to have_attributes(certificate_attributes) + expect(signature.x509_certificate.x509_issuer).to have_attributes(issuer_attributes) + expect(signature.verified_signature).to be_truthy + expect(signature.verification_status).to eq(:verified) + end + context "if the email matches but isn't confirmed" do let!(:user) { create(:user, :unconfirmed, email: X509Helpers::User1.certificate_email) } diff --git a/spec/lib/gitlab/x509/tag_spec.rb b/spec/lib/gitlab/x509/tag_spec.rb index f52880cfc52..e20ef688db5 100644 --- a/spec/lib/gitlab/x509/tag_spec.rb +++ b/spec/lib/gitlab/x509/tag_spec.rb @@ -5,8 +5,8 @@ RSpec.describe Gitlab::X509::Tag do subject(:signature) { described_class.new(project.repository, tag).signature } describe '#signature' do - let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '', 'group/project') } - let(:project) { create(:project, :repository) } + let_it_be(:project) { create(:project, :repository) } + let_it_be(:repository) { project.repository.raw } describe 'signed tag' do let(:tag) { project.repository.find_tag('v1.1.1') } |