diff options
Diffstat (limited to 'spec/lib/gitlab')
19 files changed, 1433 insertions, 8 deletions
diff --git a/spec/lib/gitlab/background_migration/backfill_ci_queuing_tables_spec.rb b/spec/lib/gitlab/background_migration/backfill_ci_queuing_tables_spec.rb new file mode 100644 index 00000000000..aaf8c124a83 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_ci_queuing_tables_spec.rb @@ -0,0 +1,245 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillCiQueuingTables, :migration, + :suppress_gitlab_schemas_validate_connection, schema: 20220208115439 do + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:ci_cd_settings) { table(:project_ci_cd_settings) } + let(:builds) { table(:ci_builds) } + let(:queuing_entries) { table(:ci_pending_builds) } + let(:tags) { table(:tags) } + let(:taggings) { table(:taggings) } + + subject { described_class.new } + + describe '#perform' do + let!(:namespace) do + namespaces.create!( + id: 10, + name: 'namespace10', + path: 'namespace10', + traversal_ids: [10]) + end + + let!(:other_namespace) do + namespaces.create!( + id: 11, + name: 'namespace11', + path: 'namespace11', + traversal_ids: [11]) + end + + let!(:project) do + projects.create!(id: 5, namespace_id: 10, name: 'test1', path: 'test1') + end + + let!(:ci_cd_setting) do + ci_cd_settings.create!(id: 5, project_id: 5, group_runners_enabled: true) + end + + let!(:other_project) do + projects.create!(id: 7, namespace_id: 11, name: 'test2', path: 'test2') + end + + let!(:other_ci_cd_setting) do + ci_cd_settings.create!(id: 7, project_id: 7, group_runners_enabled: false) + end + + let!(:another_project) do + projects.create!(id: 9, namespace_id: 10, name: 'test3', path: 'test3', shared_runners_enabled: false) + end + + let!(:ruby_tag) do + tags.create!(id: 22, name: 'ruby') + end + + let!(:postgres_tag) do + tags.create!(id: 23, name: 'postgres') + end + + it 'creates ci_pending_builds for all pending builds in range' do + builds.create!(id: 50, status: :pending, name: 'test1', project_id: 5, type: 'Ci::Build') + builds.create!(id: 51, status: :created, name: 'test2', project_id: 5, type: 'Ci::Build') + builds.create!(id: 52, status: :pending, name: 'test3', project_id: 5, protected: true, type: 'Ci::Build') + + taggings.create!(taggable_id: 52, taggable_type: 'CommitStatus', tag_id: 22) + taggings.create!(taggable_id: 52, taggable_type: 'CommitStatus', tag_id: 23) + + builds.create!(id: 60, status: :pending, name: 'test1', project_id: 7, type: 'Ci::Build') + builds.create!(id: 61, status: :running, name: 'test2', project_id: 7, protected: true, type: 'Ci::Build') + builds.create!(id: 62, status: :pending, name: 'test3', project_id: 7, type: 'Ci::Build') + + taggings.create!(taggable_id: 60, taggable_type: 'CommitStatus', tag_id: 23) + taggings.create!(taggable_id: 62, taggable_type: 'CommitStatus', tag_id: 22) + + builds.create!(id: 70, status: :pending, name: 'test1', project_id: 9, protected: true, type: 'Ci::Build') + builds.create!(id: 71, status: :failed, name: 'test2', project_id: 9, type: 'Ci::Build') + builds.create!(id: 72, status: :pending, name: 'test3', project_id: 9, type: 'Ci::Build') + + taggings.create!(taggable_id: 71, taggable_type: 'CommitStatus', tag_id: 22) + + subject.perform(1, 100) + + expect(queuing_entries.all).to contain_exactly( + an_object_having_attributes( + build_id: 50, + project_id: 5, + namespace_id: 10, + protected: false, + instance_runners_enabled: true, + minutes_exceeded: false, + tag_ids: [], + namespace_traversal_ids: [10]), + an_object_having_attributes( + build_id: 52, + project_id: 5, + namespace_id: 10, + protected: true, + instance_runners_enabled: true, + minutes_exceeded: false, + tag_ids: match_array([22, 23]), + namespace_traversal_ids: [10]), + an_object_having_attributes( + build_id: 60, + project_id: 7, + namespace_id: 11, + protected: false, + instance_runners_enabled: true, + minutes_exceeded: false, + tag_ids: [23], + namespace_traversal_ids: []), + an_object_having_attributes( + build_id: 62, + project_id: 7, + namespace_id: 11, + protected: false, + instance_runners_enabled: true, + minutes_exceeded: false, + tag_ids: [22], + namespace_traversal_ids: []), + an_object_having_attributes( + build_id: 70, + project_id: 9, + namespace_id: 10, + protected: true, + instance_runners_enabled: false, + minutes_exceeded: false, + tag_ids: [], + namespace_traversal_ids: []), + an_object_having_attributes( + build_id: 72, + project_id: 9, + namespace_id: 10, + protected: false, + instance_runners_enabled: false, + minutes_exceeded: false, + tag_ids: [], + namespace_traversal_ids: []) + ) + end + + it 'skips builds that already have ci_pending_builds' do + builds.create!(id: 50, status: :pending, name: 'test1', project_id: 5, type: 'Ci::Build') + builds.create!(id: 51, status: :created, name: 'test2', project_id: 5, type: 'Ci::Build') + builds.create!(id: 52, status: :pending, name: 'test3', project_id: 5, protected: true, type: 'Ci::Build') + + taggings.create!(taggable_id: 50, taggable_type: 'CommitStatus', tag_id: 22) + taggings.create!(taggable_id: 52, taggable_type: 'CommitStatus', tag_id: 23) + + queuing_entries.create!(build_id: 50, project_id: 5, namespace_id: 10) + + subject.perform(1, 100) + + expect(queuing_entries.all).to contain_exactly( + an_object_having_attributes( + build_id: 50, + project_id: 5, + namespace_id: 10, + protected: false, + instance_runners_enabled: false, + minutes_exceeded: false, + tag_ids: [], + namespace_traversal_ids: []), + an_object_having_attributes( + build_id: 52, + project_id: 5, + namespace_id: 10, + protected: true, + instance_runners_enabled: true, + minutes_exceeded: false, + tag_ids: [23], + namespace_traversal_ids: [10]) + ) + end + + it 'upserts values in case of conflicts' do + builds.create!(id: 50, status: :pending, name: 'test1', project_id: 5, type: 'Ci::Build') + queuing_entries.create!(build_id: 50, project_id: 5, namespace_id: 10) + + build = described_class::Ci::Build.find(50) + described_class::Ci::PendingBuild.upsert_from_build!(build) + + expect(queuing_entries.all).to contain_exactly( + an_object_having_attributes( + build_id: 50, + project_id: 5, + namespace_id: 10, + protected: false, + instance_runners_enabled: true, + minutes_exceeded: false, + tag_ids: [], + namespace_traversal_ids: [10]) + ) + end + end + + context 'Ci::Build' do + describe '.each_batch' do + let(:model) { described_class::Ci::Build } + + before do + builds.create!(id: 1, status: :pending, name: 'test1', project_id: 5, type: 'Ci::Build') + builds.create!(id: 2, status: :pending, name: 'test2', project_id: 5, type: 'Ci::Build') + builds.create!(id: 3, status: :pending, name: 'test3', project_id: 5, type: 'Ci::Build') + builds.create!(id: 4, status: :pending, name: 'test4', project_id: 5, type: 'Ci::Build') + builds.create!(id: 5, status: :pending, name: 'test5', project_id: 5, type: 'Ci::Build') + end + + it 'yields an ActiveRecord::Relation when a block is given' do + model.each_batch do |relation| + expect(relation).to be_a_kind_of(ActiveRecord::Relation) + end + end + + it 'yields a batch index as the second argument' do + model.each_batch do |_, index| + expect(index).to eq(1) + end + end + + it 'accepts a custom batch size' do + amount = 0 + + model.each_batch(of: 1) { amount += 1 } + + expect(amount).to eq(5) + end + + it 'does not include ORDER BYs in the yielded relations' do + model.each_batch do |relation| + expect(relation.to_sql).not_to include('ORDER BY') + end + end + + it 'orders ascending' do + ids = [] + + model.each_batch(of: 1) { |rel| ids.concat(rel.ids) } + + expect(ids).to eq(ids.sort) + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_group_features_spec.rb b/spec/lib/gitlab/background_migration/backfill_group_features_spec.rb index 2c2740434de..e0be5a785b8 100644 --- a/spec/lib/gitlab/background_migration/backfill_group_features_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_group_features_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::BackfillGroupFeatures, :migration, schema: 20220314184009 do +RSpec.describe Gitlab::BackgroundMigration::BackfillGroupFeatures, :migration, schema: 20220302114046 do let(:group_features) { table(:group_features) } let(:namespaces) { table(:namespaces) } diff --git a/spec/lib/gitlab/background_migration/backfill_integrations_type_new_spec.rb b/spec/lib/gitlab/background_migration/backfill_integrations_type_new_spec.rb new file mode 100644 index 00000000000..e6588644b4f --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_integrations_type_new_spec.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillIntegrationsTypeNew, :migration, schema: 20220212120735 do + let(:migration) { described_class.new } + let(:integrations) { table(:integrations) } + + let(:namespaced_integrations) do + Set.new( + %w[ + Asana Assembla Bamboo Bugzilla Buildkite Campfire Confluence CustomIssueTracker Datadog + Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Harbor Irker Jenkins Jira Mattermost + MattermostSlashCommands MicrosoftTeams MockCi MockMonitoring Packagist PipelinesEmail Pivotaltracker + Prometheus Pushover Redmine Shimo Slack SlackSlashCommands Teamcity UnifyCircuit WebexTeams Youtrack Zentao + Github GitlabSlackApplication + ]).freeze + end + + before do + integrations.connection.execute 'ALTER TABLE integrations DISABLE TRIGGER "trigger_type_new_on_insert"' + + namespaced_integrations.each_with_index do |type, i| + integrations.create!(id: i + 1, type: "#{type}Service") + end + + integrations.create!(id: namespaced_integrations.size + 1, type: 'LegacyService') + ensure + integrations.connection.execute 'ALTER TABLE integrations ENABLE TRIGGER "trigger_type_new_on_insert"' + end + + it 'backfills `type_new` for the selected records' do + # We don't want to mock `Kernel.sleep`, so instead we mock it on the migration + # class before it gets forwarded. + expect(migration).to receive(:sleep).with(0.05).exactly(5).times + + queries = ActiveRecord::QueryRecorder.new do + migration.perform(2, 10, :integrations, :id, 2, 50) + end + + expect(queries.count).to be(16) + expect(queries.log.grep(/^SELECT/).size).to be(11) + expect(queries.log.grep(/^UPDATE/).size).to be(5) + expect(queries.log.grep(/^UPDATE/).join.scan(/WHERE .*/)).to eq( + [ + 'WHERE integrations.id BETWEEN 2 AND 3', + 'WHERE integrations.id BETWEEN 4 AND 5', + 'WHERE integrations.id BETWEEN 6 AND 7', + 'WHERE integrations.id BETWEEN 8 AND 9', + 'WHERE integrations.id BETWEEN 10 AND 10' + ]) + + expect(integrations.where(id: 2..10).pluck(:type, :type_new)).to contain_exactly( + ['AssemblaService', 'Integrations::Assembla'], + ['BambooService', 'Integrations::Bamboo'], + ['BugzillaService', 'Integrations::Bugzilla'], + ['BuildkiteService', 'Integrations::Buildkite'], + ['CampfireService', 'Integrations::Campfire'], + ['ConfluenceService', 'Integrations::Confluence'], + ['CustomIssueTrackerService', 'Integrations::CustomIssueTracker'], + ['DatadogService', 'Integrations::Datadog'], + ['DiscordService', 'Integrations::Discord'] + ) + + expect(integrations.where.not(id: 2..10)).to all(have_attributes(type_new: nil)) + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_member_namespace_for_group_members_spec.rb b/spec/lib/gitlab/background_migration/backfill_member_namespace_for_group_members_spec.rb index ea07079f9ee..e1ef12a1479 100644 --- a/spec/lib/gitlab/background_migration/backfill_member_namespace_for_group_members_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_member_namespace_for_group_members_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::BackfillMemberNamespaceForGroupMembers, :migration, schema: 20220314184009 do +RSpec.describe Gitlab::BackgroundMigration::BackfillMemberNamespaceForGroupMembers, :migration, schema: 20220120211832 do let(:migration) { described_class.new } let(:members_table) { table(:members) } let(:namespaces_table) { table(:namespaces) } diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route_spec.rb index f4e8fa1bbac..b821efcadb0 100644 --- a/spec/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceIdForNamespaceRoute, :migration, schema: 20220314184009 do +RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceIdForNamespaceRoute, :migration, schema: 20220120123800 do let(:migration) { described_class.new } let(:namespaces_table) { table(:namespaces) } let(:projects_table) { table(:projects) } diff --git a/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb b/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb index 6f6ff9232e0..4a50d08b2aa 100644 --- a/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::BackfillSnippetRepositories, :migration, schema: 20220314184009, +RSpec.describe Gitlab::BackgroundMigration::BackfillSnippetRepositories, :migration, schema: 20211202041233, feature_category: :source_code_management do let(:gitlab_shell) { Gitlab::Shell.new } let(:users) { table(:users) } diff --git a/spec/lib/gitlab/background_migration/encrypt_integration_properties_spec.rb b/spec/lib/gitlab/background_migration/encrypt_integration_properties_spec.rb new file mode 100644 index 00000000000..c788b701d79 --- /dev/null +++ b/spec/lib/gitlab/background_migration/encrypt_integration_properties_spec.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::EncryptIntegrationProperties, schema: 20220415124804 do + let(:integrations) do + table(:integrations) do |integrations| + integrations.send :attr_encrypted, :encrypted_properties_tmp, + attribute: :encrypted_properties, + mode: :per_attribute_iv, + key: ::Settings.attr_encrypted_db_key_base_32, + algorithm: 'aes-256-gcm', + marshal: true, + marshaler: ::Gitlab::Json, + encode: false, + encode_iv: false + end + end + + let!(:no_properties) { integrations.create! } + let!(:with_plaintext_1) { integrations.create!(properties: json_props(1)) } + let!(:with_plaintext_2) { integrations.create!(properties: json_props(2)) } + let!(:with_encrypted) do + x = integrations.new + x.properties = nil + x.encrypted_properties_tmp = some_props(3) + x.save! + x + end + + let(:start_id) { integrations.minimum(:id) } + let(:end_id) { integrations.maximum(:id) } + + it 'ensures all properties are encrypted', :aggregate_failures do + described_class.new.perform(start_id, end_id) + + props = integrations.all.to_h do |record| + [record.id, [Gitlab::Json.parse(record.properties), record.encrypted_properties_tmp]] + end + + expect(integrations.count).to eq(4) + + expect(props).to match( + no_properties.id => both(be_nil), + with_plaintext_1.id => both(eq some_props(1)), + with_plaintext_2.id => both(eq some_props(2)), + with_encrypted.id => match([be_nil, eq(some_props(3))]) + ) + end + + private + + def both(obj) + match [obj, obj] + end + + def some_props(id) + HashWithIndifferentAccess.new({ id: id, foo: 1, bar: true, baz: %w[a string array] }) + end + + def json_props(id) + some_props(id).to_json + end +end diff --git a/spec/lib/gitlab/background_migration/encrypt_static_object_token_spec.rb b/spec/lib/gitlab/background_migration/encrypt_static_object_token_spec.rb new file mode 100644 index 00000000000..4e7b97d33f6 --- /dev/null +++ b/spec/lib/gitlab/background_migration/encrypt_static_object_token_spec.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::EncryptStaticObjectToken do + let(:users) { table(:users) } + let!(:user_without_tokens) { create_user!(name: 'notoken') } + let!(:user_with_plaintext_token_1) { create_user!(name: 'plaintext_1', token: 'token') } + let!(:user_with_plaintext_token_2) { create_user!(name: 'plaintext_2', token: 'TOKEN') } + let!(:user_with_plaintext_empty_token) { create_user!(name: 'plaintext_3', token: '') } + let!(:user_with_encrypted_token) { create_user!(name: 'encrypted', encrypted_token: 'encrypted') } + let!(:user_with_both_tokens) { create_user!(name: 'both', token: 'token2', encrypted_token: 'encrypted2') } + + before do + allow(Gitlab::CryptoHelper).to receive(:aes256_gcm_encrypt).and_call_original + allow(Gitlab::CryptoHelper).to receive(:aes256_gcm_encrypt).with('token') { 'secure_token' } + allow(Gitlab::CryptoHelper).to receive(:aes256_gcm_encrypt).with('TOKEN') { 'SECURE_TOKEN' } + end + + subject { described_class.new.perform(start_id, end_id) } + + let(:start_id) { users.minimum(:id) } + let(:end_id) { users.maximum(:id) } + + it 'backfills encrypted tokens to users with plaintext token only', :aggregate_failures do + subject + + new_state = users.pluck(:id, :static_object_token, :static_object_token_encrypted).to_h do |row| + [row[0], [row[1], row[2]]] + end + + expect(new_state.count).to eq(6) + + expect(new_state[user_with_plaintext_token_1.id]).to match_array(%w[token secure_token]) + expect(new_state[user_with_plaintext_token_2.id]).to match_array(%w[TOKEN SECURE_TOKEN]) + + expect(new_state[user_with_plaintext_empty_token.id]).to match_array(['', nil]) + expect(new_state[user_without_tokens.id]).to match_array([nil, nil]) + expect(new_state[user_with_both_tokens.id]).to match_array(%w[token2 encrypted2]) + expect(new_state[user_with_encrypted_token.id]).to match_array([nil, 'encrypted']) + end + + context 'when id range does not include existing user ids' do + let(:arguments) { [non_existing_record_id, non_existing_record_id.succ] } + + it_behaves_like 'marks background migration job records' do + subject { described_class.new } + end + end + + private + + def create_user!(name:, token: nil, encrypted_token: nil) + email = "#{name}@example.com" + + table(:users).create!( + name: name, + email: email, + username: name, + projects_limit: 0, + static_object_token: token, + static_object_token_encrypted: encrypted_token + ) + end +end diff --git a/spec/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata_spec.rb b/spec/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata_spec.rb index 3cbc05b762a..af551861d47 100644 --- a/spec/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata_spec.rb +++ b/spec/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::FixVulnerabilityOccurrencesWithHashesAsRawMetadata, schema: 20220314184009 do +RSpec.describe Gitlab::BackgroundMigration::FixVulnerabilityOccurrencesWithHashesAsRawMetadata, schema: 20211209203821 do let(:users) { table(:users) } let(:namespaces) { table(:namespaces) } let(:projects) { table(:projects) } diff --git a/spec/lib/gitlab/background_migration/merge_topics_with_same_name_spec.rb b/spec/lib/gitlab/background_migration/merge_topics_with_same_name_spec.rb new file mode 100644 index 00000000000..2c2c048992f --- /dev/null +++ b/spec/lib/gitlab/background_migration/merge_topics_with_same_name_spec.rb @@ -0,0 +1,148 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::MergeTopicsWithSameName, schema: 20220331133802 do + def set_avatar(topic_id, avatar) + topic = ::Projects::Topic.find(topic_id) + topic.avatar = avatar + topic.save! + topic.avatar.absolute_path + end + + it 'merges project topics with same case insensitive name' do + namespaces = table(:namespaces) + projects = table(:projects) + topics = table(:topics) + project_topics = table(:project_topics) + + group_1 = namespaces.create!(name: 'space1', type: 'Group', path: 'space1') + group_2 = namespaces.create!(name: 'space2', type: 'Group', path: 'space2') + group_3 = namespaces.create!(name: 'space3', type: 'Group', path: 'space3') + proj_space_1 = namespaces.create!(name: 'proj1', path: 'proj1', type: 'Project', parent_id: group_1.id) + proj_space_2 = namespaces.create!(name: 'proj2', path: 'proj2', type: 'Project', parent_id: group_2.id) + proj_space_3 = namespaces.create!(name: 'proj3', path: 'proj3', type: 'Project', parent_id: group_3.id) + project_1 = projects.create!(namespace_id: group_1.id, project_namespace_id: proj_space_1.id, visibility_level: 20) + project_2 = projects.create!(namespace_id: group_2.id, project_namespace_id: proj_space_2.id, visibility_level: 10) + project_3 = projects.create!(namespace_id: group_3.id, project_namespace_id: proj_space_3.id, visibility_level: 0) + topic_1_keep = topics.create!( + name: 'topic1', + title: 'Topic 1', + description: 'description 1 to keep', + total_projects_count: 2, + non_private_projects_count: 2 + ) + topic_1_remove = topics.create!( + name: 'TOPIC1', + title: 'Topic 1', + description: 'description 1 to remove', + total_projects_count: 2, + non_private_projects_count: 1 + ) + topic_2_remove = topics.create!( + name: 'topic2', + title: 'Topic 2', + total_projects_count: 0 + ) + topic_2_keep = topics.create!( + name: 'TOPIC2', + title: 'Topic 2', + description: 'description 2 to keep', + total_projects_count: 1 + ) + topic_3_remove_1 = topics.create!( + name: 'topic3', + title: 'Topic 3', + total_projects_count: 2, + non_private_projects_count: 1 + ) + topic_3_keep = topics.create!( + name: 'Topic3', + title: 'Topic 3', + total_projects_count: 2, + non_private_projects_count: 2 + ) + topic_3_remove_2 = topics.create!( + name: 'TOPIC3', + title: 'Topic 3', + description: 'description 3 to keep', + total_projects_count: 2, + non_private_projects_count: 1 + ) + topic_4_keep = topics.create!( + name: 'topic4', + title: 'Topic 4' + ) + + project_topics_1 = [] + project_topics_3 = [] + project_topics_removed = [] + + project_topics_1 << project_topics.create!(topic_id: topic_1_keep.id, project_id: project_1.id) + project_topics_1 << project_topics.create!(topic_id: topic_1_keep.id, project_id: project_2.id) + project_topics_removed << project_topics.create!(topic_id: topic_1_remove.id, project_id: project_2.id) + project_topics_1 << project_topics.create!(topic_id: topic_1_remove.id, project_id: project_3.id) + + project_topics_3 << project_topics.create!(topic_id: topic_3_keep.id, project_id: project_1.id) + project_topics_3 << project_topics.create!(topic_id: topic_3_keep.id, project_id: project_2.id) + project_topics_removed << project_topics.create!(topic_id: topic_3_remove_1.id, project_id: project_1.id) + project_topics_3 << project_topics.create!(topic_id: topic_3_remove_1.id, project_id: project_3.id) + project_topics_removed << project_topics.create!(topic_id: topic_3_remove_2.id, project_id: project_1.id) + project_topics_removed << project_topics.create!(topic_id: topic_3_remove_2.id, project_id: project_3.id) + + avatar_paths = { + topic_1_keep: set_avatar(topic_1_keep.id, fixture_file_upload('spec/fixtures/avatars/avatar1.png')), + topic_1_remove: set_avatar(topic_1_remove.id, fixture_file_upload('spec/fixtures/avatars/avatar2.png')), + topic_2_remove: set_avatar(topic_2_remove.id, fixture_file_upload('spec/fixtures/avatars/avatar3.png')), + topic_3_remove_1: set_avatar(topic_3_remove_1.id, fixture_file_upload('spec/fixtures/avatars/avatar4.png')), + topic_3_remove_2: set_avatar(topic_3_remove_2.id, fixture_file_upload('spec/fixtures/avatars/avatar5.png')) + } + + subject.perform(%w[topic1 topic2 topic3 topic4]) + + # Topics + [topic_1_keep, topic_2_keep, topic_3_keep, topic_4_keep].each(&:reload) + expect(topic_1_keep.name).to eq('topic1') + expect(topic_1_keep.description).to eq('description 1 to keep') + expect(topic_1_keep.total_projects_count).to eq(3) + expect(topic_1_keep.non_private_projects_count).to eq(2) + expect(topic_2_keep.name).to eq('TOPIC2') + expect(topic_2_keep.description).to eq('description 2 to keep') + expect(topic_2_keep.total_projects_count).to eq(0) + expect(topic_2_keep.non_private_projects_count).to eq(0) + expect(topic_3_keep.name).to eq('Topic3') + expect(topic_3_keep.description).to eq('description 3 to keep') + expect(topic_3_keep.total_projects_count).to eq(3) + expect(topic_3_keep.non_private_projects_count).to eq(2) + expect(topic_4_keep.reload.name).to eq('topic4') + + [topic_1_remove, topic_2_remove, topic_3_remove_1, topic_3_remove_2].each do |topic| + expect { topic.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + + # Topic avatars + expect(topic_1_keep.avatar).to eq('avatar1.png') + expect(File.exist?(::Projects::Topic.find(topic_1_keep.id).avatar.absolute_path)).to be_truthy + expect(topic_2_keep.avatar).to eq('avatar3.png') + expect(File.exist?(::Projects::Topic.find(topic_2_keep.id).avatar.absolute_path)).to be_truthy + expect(topic_3_keep.avatar).to eq('avatar4.png') + expect(File.exist?(::Projects::Topic.find(topic_3_keep.id).avatar.absolute_path)).to be_truthy + + [:topic_1_remove, :topic_2_remove, :topic_3_remove_1, :topic_3_remove_2].each do |topic| + expect(File.exist?(avatar_paths[topic])).to be_falsey + end + + # Project Topic assignments + project_topics_1.each do |project_topic| + expect(project_topic.reload.topic_id).to eq(topic_1_keep.id) + end + + project_topics_3.each do |project_topic| + expect(project_topic.reload.topic_id).to eq(topic_3_keep.id) + end + + project_topics_removed.each do |project_topic| + expect { project_topic.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end +end diff --git a/spec/lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner_spec.rb b/spec/lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner_spec.rb index 90d05ccbe1a..07e77bdbc13 100644 --- a/spec/lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner_spec.rb +++ b/spec/lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::MigratePersonalNamespaceProjectMaintainerToOwner, :migration, schema: 20220314184009 do +RSpec.describe Gitlab::BackgroundMigration::MigratePersonalNamespaceProjectMaintainerToOwner, :migration, schema: 20220208080921 do let(:migration) { described_class.new } let(:users_table) { table(:users) } let(:members_table) { table(:members) } diff --git a/spec/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds_spec.rb b/spec/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds_spec.rb index 7c78350e697..2f0eef3c399 100644 --- a/spec/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds_spec.rb +++ b/spec/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::NullifyOrphanRunnerIdOnCiBuilds, - :suppress_gitlab_schemas_validate_connection, migration: :gitlab_ci, schema: 20220314184009 do + :suppress_gitlab_schemas_validate_connection, migration: :gitlab_ci, schema: 20220223112304 do let(:namespaces) { table(:namespaces) } let(:projects) { table(:projects) } let(:ci_runners) { table(:ci_runners) } diff --git a/spec/lib/gitlab/background_migration/populate_namespace_statistics_spec.rb b/spec/lib/gitlab/background_migration/populate_namespace_statistics_spec.rb new file mode 100644 index 00000000000..4a7d52ee784 --- /dev/null +++ b/spec/lib/gitlab/background_migration/populate_namespace_statistics_spec.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::PopulateNamespaceStatistics do + let!(:namespaces) { table(:namespaces) } + let!(:namespace_statistics) { table(:namespace_statistics) } + let!(:dependency_proxy_manifests) { table(:dependency_proxy_manifests) } + let!(:dependency_proxy_blobs) { table(:dependency_proxy_blobs) } + + let!(:group1) { namespaces.create!(id: 10, type: 'Group', name: 'group1', path: 'group1') } + let!(:group2) { namespaces.create!(id: 20, type: 'Group', name: 'group2', path: 'group2') } + + let!(:group1_manifest) do + dependency_proxy_manifests.create!(group_id: 10, size: 20, file_name: 'test-file', file: 'test', digest: 'abc123') + end + + let!(:group2_manifest) do + dependency_proxy_manifests.create!(group_id: 20, size: 20, file_name: 'test-file', file: 'test', digest: 'abc123') + end + + let!(:group1_stats) { namespace_statistics.create!(id: 10, namespace_id: 10) } + + let(:ids) { namespaces.pluck(:id) } + let(:statistics) { [] } + + subject(:perform) { described_class.new.perform(ids, statistics) } + + it 'creates/updates all namespace_statistics and updates root storage statistics', :aggregate_failures do + expect(Namespaces::ScheduleAggregationWorker).to receive(:perform_async).with(group1.id) + expect(Namespaces::ScheduleAggregationWorker).to receive(:perform_async).with(group2.id) + + expect { perform }.to change(namespace_statistics, :count).from(1).to(2) + + namespace_statistics.all.each do |stat| + expect(stat.dependency_proxy_size).to eq 20 + expect(stat.storage_size).to eq 20 + end + end + + context 'when just a stat is passed' do + let(:statistics) { [:dependency_proxy_size] } + + it 'calls the statistics update service with just that stat' do + expect(Groups::UpdateStatisticsService) + .to receive(:new) + .with(anything, statistics: [:dependency_proxy_size]) + .twice.and_call_original + + perform + end + end + + context 'when a statistics update fails' do + before do + error_response = instance_double(ServiceResponse, message: 'an error', error?: true) + + allow_next_instance_of(Groups::UpdateStatisticsService) do |instance| + allow(instance).to receive(:execute).and_return(error_response) + end + end + + it 'logs an error' do + expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |instance| + expect(instance).to receive(:error).twice + end + + perform + end + end +end diff --git a/spec/lib/gitlab/background_migration/populate_topics_non_private_projects_count_spec.rb b/spec/lib/gitlab/background_migration/populate_topics_non_private_projects_count_spec.rb new file mode 100644 index 00000000000..e72e3392210 --- /dev/null +++ b/spec/lib/gitlab/background_migration/populate_topics_non_private_projects_count_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::PopulateTopicsNonPrivateProjectsCount, schema: 20220125122640 do + it 'correctly populates the non private projects counters' do + namespaces = table(:namespaces) + projects = table(:projects) + topics = table(:topics) + project_topics = table(:project_topics) + + group = namespaces.create!(name: 'group', path: 'group') + project_public = projects.create!(namespace_id: group.id, visibility_level: Gitlab::VisibilityLevel::PUBLIC) + project_internal = projects.create!(namespace_id: group.id, visibility_level: Gitlab::VisibilityLevel::INTERNAL) + project_private = projects.create!(namespace_id: group.id, visibility_level: Gitlab::VisibilityLevel::PRIVATE) + topic_1 = topics.create!(name: 'Topic1') + topic_2 = topics.create!(name: 'Topic2') + topic_3 = topics.create!(name: 'Topic3') + topic_4 = topics.create!(name: 'Topic4') + topic_5 = topics.create!(name: 'Topic5') + topic_6 = topics.create!(name: 'Topic6') + topic_7 = topics.create!(name: 'Topic7') + topic_8 = topics.create!(name: 'Topic8') + + project_topics.create!(topic_id: topic_1.id, project_id: project_public.id) + project_topics.create!(topic_id: topic_2.id, project_id: project_internal.id) + project_topics.create!(topic_id: topic_3.id, project_id: project_private.id) + project_topics.create!(topic_id: topic_4.id, project_id: project_public.id) + project_topics.create!(topic_id: topic_4.id, project_id: project_internal.id) + project_topics.create!(topic_id: topic_5.id, project_id: project_public.id) + project_topics.create!(topic_id: topic_5.id, project_id: project_private.id) + project_topics.create!(topic_id: topic_6.id, project_id: project_internal.id) + project_topics.create!(topic_id: topic_6.id, project_id: project_private.id) + project_topics.create!(topic_id: topic_7.id, project_id: project_public.id) + project_topics.create!(topic_id: topic_7.id, project_id: project_internal.id) + project_topics.create!(topic_id: topic_7.id, project_id: project_private.id) + project_topics.create!(topic_id: topic_8.id, project_id: project_public.id) + + subject.perform(topic_1.id, topic_7.id) + + expect(topic_1.reload.non_private_projects_count).to eq(1) + expect(topic_2.reload.non_private_projects_count).to eq(1) + expect(topic_3.reload.non_private_projects_count).to eq(0) + expect(topic_4.reload.non_private_projects_count).to eq(2) + expect(topic_5.reload.non_private_projects_count).to eq(1) + expect(topic_6.reload.non_private_projects_count).to eq(1) + expect(topic_7.reload.non_private_projects_count).to eq(2) + expect(topic_8.reload.non_private_projects_count).to eq(0) + end +end diff --git a/spec/lib/gitlab/background_migration/populate_vulnerability_reads_spec.rb b/spec/lib/gitlab/background_migration/populate_vulnerability_reads_spec.rb new file mode 100644 index 00000000000..c0470f26d9e --- /dev/null +++ b/spec/lib/gitlab/background_migration/populate_vulnerability_reads_spec.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::PopulateVulnerabilityReads, :migration, schema: 20220326161803 do + let(:vulnerabilities) { table(:vulnerabilities) } + let(:vulnerability_reads) { table(:vulnerability_reads) } + let(:vulnerabilities_findings) { table(:vulnerability_occurrences) } + let(:vulnerability_issue_links) { table(:vulnerability_issue_links) } + let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } + let(:user) { table(:users).create!(email: 'author@example.com', username: 'author', projects_limit: 10) } + let(:project) { table(:projects).create!(namespace_id: namespace.id) } + let(:scanner) { table(:vulnerability_scanners).create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') } + let(:sub_batch_size) { 1000 } + + before do + vulnerabilities_findings.connection.execute 'ALTER TABLE vulnerability_occurrences DISABLE TRIGGER "trigger_insert_or_update_vulnerability_reads_from_occurrences"' + vulnerabilities.connection.execute 'ALTER TABLE vulnerabilities DISABLE TRIGGER "trigger_update_vulnerability_reads_on_vulnerability_update"' + vulnerability_issue_links.connection.execute 'ALTER TABLE vulnerability_issue_links DISABLE TRIGGER "trigger_update_has_issues_on_vulnerability_issue_links_update"' + + 10.times.each do |x| + vulnerability = create_vulnerability!( + project_id: project.id, + report_type: 7, + author_id: user.id + ) + identifier = table(:vulnerability_identifiers).create!( + project_id: project.id, + external_type: 'uuid-v5', + external_id: 'uuid-v5', + fingerprint: Digest::SHA1.hexdigest(vulnerability.id.to_s), + name: 'Identifier for UUIDv5') + + create_finding!( + vulnerability_id: vulnerability.id, + project_id: project.id, + scanner_id: scanner.id, + primary_identifier_id: identifier.id + ) + end + end + + it 'creates vulnerability_reads for the given records' do + described_class.new.perform(vulnerabilities.first.id, vulnerabilities.last.id, sub_batch_size) + + expect(vulnerability_reads.count).to eq(10) + end + + it 'does not create new records when records already exists' do + described_class.new.perform(vulnerabilities.first.id, vulnerabilities.last.id, sub_batch_size) + described_class.new.perform(vulnerabilities.first.id, vulnerabilities.last.id, sub_batch_size) + + expect(vulnerability_reads.count).to eq(10) + end + + private + + def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0) + vulnerabilities.create!( + project_id: project_id, + author_id: author_id, + title: title, + severity: severity, + confidence: confidence, + report_type: report_type + ) + end + + # rubocop:disable Metrics/ParameterLists + def create_finding!( + project_id:, scanner_id:, primary_identifier_id:, vulnerability_id: nil, + name: "test", severity: 7, confidence: 7, report_type: 0, + project_fingerprint: '123qweasdzxc', location: { "image" => "alpine:3.4" }, location_fingerprint: 'test', + metadata_version: 'test', raw_metadata: 'test', uuid: SecureRandom.uuid) + vulnerabilities_findings.create!( + vulnerability_id: vulnerability_id, + project_id: project_id, + name: name, + severity: severity, + confidence: confidence, + report_type: report_type, + project_fingerprint: project_fingerprint, + scanner_id: scanner_id, + primary_identifier_id: primary_identifier_id, + location: location, + location_fingerprint: location_fingerprint, + metadata_version: metadata_version, + raw_metadata: raw_metadata, + uuid: uuid + ) + end + # rubocop:enable Metrics/ParameterLists +end diff --git a/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb b/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb new file mode 100644 index 00000000000..543dd204f89 --- /dev/null +++ b/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb @@ -0,0 +1,530 @@ +# frozen_string_literal: true + +require 'spec_helper' + +def create_background_migration_job(ids, status) + proper_status = case status + when :pending + Gitlab::Database::BackgroundMigrationJob.statuses['pending'] + when :succeeded + Gitlab::Database::BackgroundMigrationJob.statuses['succeeded'] + else + raise ArgumentError + end + + background_migration_jobs.create!( + class_name: 'RecalculateVulnerabilitiesOccurrencesUuid', + arguments: Array(ids), + status: proper_status, + created_at: Time.now.utc + ) +end + +RSpec.describe Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid, :suppress_gitlab_schemas_validate_connection, schema: 20211202041233 do + let(:background_migration_jobs) { table(:background_migration_jobs) } + let(:pending_jobs) { background_migration_jobs.where(status: Gitlab::Database::BackgroundMigrationJob.statuses['pending']) } + let(:succeeded_jobs) { background_migration_jobs.where(status: Gitlab::Database::BackgroundMigrationJob.statuses['succeeded']) } + let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } + let(:users) { table(:users) } + let(:user) { create_user! } + let(:project) { table(:projects).create!(id: 123, namespace_id: namespace.id) } + let(:scanners) { table(:vulnerability_scanners) } + let(:scanner) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') } + let(:scanner2) { scanners.create!(project_id: project.id, external_id: 'test 2', name: 'test scanner 2') } + let(:vulnerabilities) { table(:vulnerabilities) } + let(:vulnerability_findings) { table(:vulnerability_occurrences) } + let(:vulnerability_finding_pipelines) { table(:vulnerability_occurrence_pipelines) } + let(:vulnerability_finding_signatures) { table(:vulnerability_finding_signatures) } + let(:vulnerability_identifiers) { table(:vulnerability_identifiers) } + + let(:identifier_1) { 'identifier-1' } + let!(:vulnerability_identifier) do + vulnerability_identifiers.create!( + project_id: project.id, + external_type: identifier_1, + external_id: identifier_1, + fingerprint: Gitlab::Database::ShaAttribute.serialize('ff9ef548a6e30a0462795d916f3f00d1e2b082ca'), + name: 'Identifier 1') + end + + let(:identifier_2) { 'identifier-2' } + let!(:vulnerability_identfier2) do + vulnerability_identifiers.create!( + project_id: project.id, + external_type: identifier_2, + external_id: identifier_2, + fingerprint: Gitlab::Database::ShaAttribute.serialize('4299e8ddd819f9bde9cfacf45716724c17b5ddf7'), + name: 'Identifier 2') + end + + let(:identifier_3) { 'identifier-3' } + let!(:vulnerability_identifier3) do + vulnerability_identifiers.create!( + project_id: project.id, + external_type: identifier_3, + external_id: identifier_3, + fingerprint: Gitlab::Database::ShaAttribute.serialize('8e91632f9c6671e951834a723ee221c44cc0d844'), + name: 'Identifier 3') + end + + let(:known_uuid_v4) { "b3cc2518-5446-4dea-871c-89d5e999c1ac" } + let(:known_uuid_v5) { "05377088-dc26-5161-920e-52a7159fdaa1" } + let(:desired_uuid_v5) { "f3e9a23f-9181-54bf-a5ab-c5bc7a9b881a" } + + subject { described_class.new.perform(start_id, end_id) } + + context "when finding has a UUIDv4" do + before do + @uuid_v4 = create_finding!( + vulnerability_id: nil, + project_id: project.id, + scanner_id: scanner2.id, + primary_identifier_id: vulnerability_identfier2.id, + report_type: 0, # "sast" + location_fingerprint: Gitlab::Database::ShaAttribute.serialize("fa18f432f1d56675f4098d318739c3cd5b14eb3e"), + uuid: known_uuid_v4 + ) + end + + let(:start_id) { @uuid_v4.id } + let(:end_id) { @uuid_v4.id } + + it "replaces it with UUIDv5" do + expect(vulnerability_findings.pluck(:uuid)).to match_array([known_uuid_v4]) + + subject + + expect(vulnerability_findings.pluck(:uuid)).to match_array([desired_uuid_v5]) + end + + it 'logs recalculation' do + expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |instance| + expect(instance).to receive(:info).twice + end + + subject + end + end + + context "when finding has a UUIDv5" do + before do + @uuid_v5 = create_finding!( + vulnerability_id: nil, + project_id: project.id, + scanner_id: scanner.id, + primary_identifier_id: vulnerability_identifier.id, + report_type: 0, # "sast" + location_fingerprint: Gitlab::Database::ShaAttribute.serialize("838574be0210968bf6b9f569df9c2576242cbf0a"), + uuid: known_uuid_v5 + ) + end + + let(:start_id) { @uuid_v5.id } + let(:end_id) { @uuid_v5.id } + + it "stays the same" do + expect(vulnerability_findings.pluck(:uuid)).to match_array([known_uuid_v5]) + + subject + + expect(vulnerability_findings.pluck(:uuid)).to match_array([known_uuid_v5]) + end + end + + context 'if a duplicate UUID would be generated' do # rubocop: disable RSpec/MultipleMemoizedHelpers + let(:v1) do + create_vulnerability!( + project_id: project.id, + author_id: user.id + ) + end + + let!(:finding_with_incorrect_uuid) do + create_finding!( + vulnerability_id: v1.id, + project_id: project.id, + scanner_id: scanner.id, + primary_identifier_id: vulnerability_identifier.id, + report_type: 0, # "sast" + location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') + uuid: 'bd95c085-71aa-51d7-9bb6-08ae669c262e' + ) + end + + let(:v2) do + create_vulnerability!( + project_id: project.id, + author_id: user.id + ) + end + + let!(:finding_with_correct_uuid) do + create_finding!( + vulnerability_id: v2.id, + project_id: project.id, + primary_identifier_id: vulnerability_identifier.id, + scanner_id: scanner2.id, + report_type: 0, # "sast" + location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') + uuid: '91984483-5efe-5215-b471-d524ac5792b1' + ) + end + + let(:v3) do + create_vulnerability!( + project_id: project.id, + author_id: user.id + ) + end + + let!(:finding_with_incorrect_uuid2) do + create_finding!( + vulnerability_id: v3.id, + project_id: project.id, + scanner_id: scanner.id, + primary_identifier_id: vulnerability_identfier2.id, + report_type: 0, # "sast" + location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') + uuid: '00000000-1111-2222-3333-444444444444' + ) + end + + let(:v4) do + create_vulnerability!( + project_id: project.id, + author_id: user.id + ) + end + + let!(:finding_with_correct_uuid2) do + create_finding!( + vulnerability_id: v4.id, + project_id: project.id, + scanner_id: scanner2.id, + primary_identifier_id: vulnerability_identfier2.id, + report_type: 0, # "sast" + location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') + uuid: '1edd751e-ef9a-5391-94db-a832c8635bfc' + ) + end + + let!(:finding_with_incorrect_uuid3) do + create_finding!( + vulnerability_id: nil, + project_id: project.id, + scanner_id: scanner.id, + primary_identifier_id: vulnerability_identifier3.id, + report_type: 0, # "sast" + location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') + uuid: '22222222-3333-4444-5555-666666666666' + ) + end + + let!(:duplicate_not_in_the_same_batch) do + create_finding!( + id: 99999, + vulnerability_id: nil, + project_id: project.id, + scanner_id: scanner2.id, + primary_identifier_id: vulnerability_identifier3.id, + report_type: 0, # "sast" + location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') + uuid: '4564f9d5-3c6b-5cc3-af8c-7c25285362a7' + ) + end + + let(:start_id) { finding_with_incorrect_uuid.id } + let(:end_id) { finding_with_incorrect_uuid3.id } + + before do + 4.times do + create_finding_pipeline!(project_id: project.id, finding_id: finding_with_incorrect_uuid.id) + create_finding_pipeline!(project_id: project.id, finding_id: finding_with_correct_uuid.id) + create_finding_pipeline!(project_id: project.id, finding_id: finding_with_incorrect_uuid2.id) + create_finding_pipeline!(project_id: project.id, finding_id: finding_with_correct_uuid2.id) + end + end + + it 'drops duplicates and related records', :aggregate_failures do + expect(vulnerability_findings.pluck(:id)).to match_array( + [ + finding_with_correct_uuid.id, + finding_with_incorrect_uuid.id, + finding_with_correct_uuid2.id, + finding_with_incorrect_uuid2.id, + finding_with_incorrect_uuid3.id, + duplicate_not_in_the_same_batch.id + ]) + + expect { subject }.to change(vulnerability_finding_pipelines, :count).from(16).to(8) + .and change(vulnerability_findings, :count).from(6).to(3) + .and change(vulnerabilities, :count).from(4).to(2) + + expect(vulnerability_findings.pluck(:id)).to match_array([finding_with_incorrect_uuid.id, finding_with_incorrect_uuid2.id, finding_with_incorrect_uuid3.id]) + end + + context 'if there are conflicting UUID values within the batch' do # rubocop: disable RSpec/MultipleMemoizedHelpers + let(:end_id) { finding_with_broken_data_integrity.id } + let(:vulnerability_5) { create_vulnerability!(project_id: project.id, author_id: user.id) } + let(:different_project) { table(:projects).create!(namespace_id: namespace.id) } + let!(:identifier_with_broken_data_integrity) do + vulnerability_identifiers.create!( + project_id: different_project.id, + external_type: identifier_2, + external_id: identifier_2, + fingerprint: Gitlab::Database::ShaAttribute.serialize('4299e8ddd819f9bde9cfacf45716724c17b5ddf7'), + name: 'Identifier 2') + end + + let(:finding_with_broken_data_integrity) do + create_finding!( + vulnerability_id: vulnerability_5, + project_id: project.id, + scanner_id: scanner.id, + primary_identifier_id: identifier_with_broken_data_integrity.id, + report_type: 0, # "sast" + location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') + uuid: SecureRandom.uuid + ) + end + + it 'deletes the conflicting record' do + expect { subject }.to change { vulnerability_findings.find_by_id(finding_with_broken_data_integrity.id) }.to(nil) + end + end + + context 'if a conflicting UUID is found during the migration' do # rubocop:disable RSpec/MultipleMemoizedHelpers + let(:finding_class) { Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid::VulnerabilitiesFinding } + let(:uuid) { '4564f9d5-3c6b-5cc3-af8c-7c25285362a7' } + + before do + exception = ActiveRecord::RecordNotUnique.new("(uuid)=(#{uuid})") + + call_count = 0 + allow(::Gitlab::Database::BulkUpdate).to receive(:execute) do + call_count += 1 + call_count.eql?(1) ? raise(exception) : {} + end + + allow(finding_class).to receive(:find_by).with(uuid: uuid).and_return(duplicate_not_in_the_same_batch) + end + + it 'retries the recalculation' do + subject + + expect(Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid::VulnerabilitiesFinding) + .to have_received(:find_by).with(uuid: uuid).once + end + + it 'logs the conflict' do + expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |instance| + expect(instance).to receive(:info).exactly(6).times + end + + subject + end + + it 'marks the job as done' do + create_background_migration_job([start_id, end_id], :pending) + + subject + + expect(pending_jobs.count).to eq(0) + expect(succeeded_jobs.count).to eq(1) + end + end + + it 'logs an exception if a different uniquness problem was found' do + exception = ActiveRecord::RecordNotUnique.new("Totally not an UUID uniqueness problem") + allow(::Gitlab::Database::BulkUpdate).to receive(:execute).and_raise(exception) + allow(Gitlab::ErrorTracking).to receive(:track_and_raise_exception) + + subject + + expect(Gitlab::ErrorTracking).to have_received(:track_and_raise_exception).with(exception).once + end + + it 'logs a duplicate found message' do + expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |instance| + expect(instance).to receive(:info).exactly(3).times + end + + subject + end + end + + context 'when finding has a signature' do + before do + @f1 = create_finding!( + vulnerability_id: nil, + project_id: project.id, + scanner_id: scanner.id, + primary_identifier_id: vulnerability_identifier.id, + report_type: 0, # "sast" + location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') + uuid: 'd15d774d-e4b1-5a1b-929b-19f2a53e35ec' + ) + + vulnerability_finding_signatures.create!( + finding_id: @f1.id, + algorithm_type: 2, # location + signature_sha: Gitlab::Database::ShaAttribute.serialize('57d4e05205f6462a73f039a5b2751aa1ab344e6e') # sha1('youshouldusethis') + ) + + vulnerability_finding_signatures.create!( + finding_id: @f1.id, + algorithm_type: 1, # hash + signature_sha: Gitlab::Database::ShaAttribute.serialize('c554d8d8df1a7a14319eafdaae24af421bf5b587') # sha1('andnotthis') + ) + + @f2 = create_finding!( + vulnerability_id: nil, + project_id: project.id, + scanner_id: scanner.id, + primary_identifier_id: vulnerability_identfier2.id, + report_type: 0, # "sast" + location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') + uuid: '4be029b5-75e5-5ac0-81a2-50ab41726135' + ) + + vulnerability_finding_signatures.create!( + finding_id: @f2.id, + algorithm_type: 2, # location + signature_sha: Gitlab::Database::ShaAttribute.serialize('57d4e05205f6462a73f039a5b2751aa1ab344e6e') # sha1('youshouldusethis') + ) + + vulnerability_finding_signatures.create!( + finding_id: @f2.id, + algorithm_type: 1, # hash + signature_sha: Gitlab::Database::ShaAttribute.serialize('c554d8d8df1a7a14319eafdaae24af421bf5b587') # sha1('andnotthis') + ) + end + + let(:start_id) { @f1.id } + let(:end_id) { @f2.id } + + let(:uuids_before) { [@f1.uuid, @f2.uuid] } + let(:uuids_after) { %w[d3b60ddd-d312-5606-b4d3-ad058eebeacb 349d9bec-c677-5530-a8ac-5e58889c3b1a] } + + it 'is recalculated using signature' do + expect(vulnerability_findings.pluck(:uuid)).to match_array(uuids_before) + + subject + + expect(vulnerability_findings.pluck(:uuid)).to match_array(uuids_after) + end + end + + context 'if all records are removed before the job ran' do + let(:start_id) { 1 } + let(:end_id) { 9 } + + before do + create_background_migration_job([start_id, end_id], :pending) + end + + it 'does not error out' do + expect { subject }.not_to raise_error + end + + it 'marks the job as done' do + subject + + expect(pending_jobs.count).to eq(0) + expect(succeeded_jobs.count).to eq(1) + end + end + + context 'when recalculation fails' do + before do + @uuid_v4 = create_finding!( + vulnerability_id: nil, + project_id: project.id, + scanner_id: scanner2.id, + primary_identifier_id: vulnerability_identfier2.id, + report_type: 0, # "sast" + location_fingerprint: Gitlab::Database::ShaAttribute.serialize("fa18f432f1d56675f4098d318739c3cd5b14eb3e"), + uuid: known_uuid_v4 + ) + + allow(Gitlab::ErrorTracking).to receive(:track_and_raise_exception) + allow(::Gitlab::Database::BulkUpdate).to receive(:execute).and_raise(expected_error) + end + + let(:start_id) { @uuid_v4.id } + let(:end_id) { @uuid_v4.id } + let(:expected_error) { RuntimeError.new } + + it 'captures the errors and does not crash entirely' do + expect { subject }.not_to raise_error + + allow(Gitlab::ErrorTracking).to receive(:track_and_raise_exception) + expect(Gitlab::ErrorTracking).to have_received(:track_and_raise_exception).with(expected_error).once + end + + it_behaves_like 'marks background migration job records' do + let(:arguments) { [1, 4] } + subject { described_class.new } + end + end + + it_behaves_like 'marks background migration job records' do + let(:arguments) { [1, 4] } + subject { described_class.new } + end + + private + + def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0) + vulnerabilities.create!( + project_id: project_id, + author_id: author_id, + title: title, + severity: severity, + confidence: confidence, + report_type: report_type + ) + end + + # rubocop:disable Metrics/ParameterLists + def create_finding!( + vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:, id: nil, + name: "test", severity: 7, confidence: 7, report_type: 0, + project_fingerprint: '123qweasdzxc', location_fingerprint: 'test', + metadata_version: 'test', raw_metadata: 'test', uuid: SecureRandom.uuid) + vulnerability_findings.create!({ + id: id, + vulnerability_id: vulnerability_id, + project_id: project_id, + name: name, + severity: severity, + confidence: confidence, + report_type: report_type, + project_fingerprint: project_fingerprint, + scanner_id: scanner_id, + primary_identifier_id: primary_identifier_id, + location_fingerprint: location_fingerprint, + metadata_version: metadata_version, + raw_metadata: raw_metadata, + uuid: uuid + }.compact + ) + end + # rubocop:enable Metrics/ParameterLists + + def create_user!(name: "Example User", email: "user@example.com", user_type: nil, created_at: Time.zone.now, confirmed_at: Time.zone.now) + users.create!( + name: name, + email: email, + username: name, + projects_limit: 0, + user_type: user_type, + confirmed_at: confirmed_at + ) + end + + def create_finding_pipeline!(project_id:, finding_id:) + pipeline = table(:ci_pipelines).create!(project_id: project_id) + vulnerability_finding_pipelines.create!(pipeline_id: pipeline.id, occurrence_id: finding_id) + end +end diff --git a/spec/lib/gitlab/background_migration/remove_all_trace_expiration_dates_spec.rb b/spec/lib/gitlab/background_migration/remove_all_trace_expiration_dates_spec.rb new file mode 100644 index 00000000000..eabc012f98b --- /dev/null +++ b/spec/lib/gitlab/background_migration/remove_all_trace_expiration_dates_spec.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::RemoveAllTraceExpirationDates, :migration, + :suppress_gitlab_schemas_validate_connection, schema: 20220131000001 do + subject(:perform) { migration.perform(1, 99) } + + let(:migration) { described_class.new } + + let(:trace_in_range) { create_trace!(id: 10, created_at: Date.new(2020, 06, 20), expire_at: Date.new(2021, 01, 22)) } + let(:trace_outside_range) { create_trace!(id: 40, created_at: Date.new(2020, 06, 22), expire_at: Date.new(2021, 01, 22)) } + let(:trace_without_expiry) { create_trace!(id: 30, created_at: Date.new(2020, 06, 21), expire_at: nil) } + let(:archive_in_range) { create_archive!(id: 10, created_at: Date.new(2020, 06, 20), expire_at: Date.new(2021, 01, 22)) } + let(:trace_outside_id_range) { create_trace!(id: 100, created_at: Date.new(2020, 06, 20), expire_at: Date.new(2021, 01, 22)) } + + before do + table(:namespaces).create!(id: 1, name: 'the-namespace', path: 'the-path') + table(:projects).create!(id: 1, name: 'the-project', namespace_id: 1) + table(:ci_builds).create!(id: 1, allow_failure: false) + end + + context 'for self-hosted instances' do + it 'sets expire_at for artifacts in range to nil' do + expect { perform }.not_to change { trace_in_range.reload.expire_at } + end + + it 'does not change expire_at timestamps that are not set to midnight' do + expect { perform }.not_to change { trace_outside_range.reload.expire_at } + end + + it 'does not change expire_at timestamps that are set to midnight on a day other than the 22nd' do + expect { perform }.not_to change { trace_without_expiry.reload.expire_at } + end + + it 'does not touch artifacts outside id range' do + expect { perform }.not_to change { archive_in_range.reload.expire_at } + end + + it 'does not touch artifacts outside date range' do + expect { perform }.not_to change { trace_outside_id_range.reload.expire_at } + end + end + + private + + def create_trace!(**args) + table(:ci_job_artifacts).create!(**args, project_id: 1, job_id: 1, file_type: 3) + end + + def create_archive!(**args) + table(:ci_job_artifacts).create!(**args, project_id: 1, job_id: 1, file_type: 1) + end +end diff --git a/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb b/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb index da14381aae2..32134b99e37 100644 --- a/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb +++ b/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::RemoveVulnerabilityFindingLinks, :migration, schema: 20220314184009 do +RSpec.describe Gitlab::BackgroundMigration::RemoveVulnerabilityFindingLinks, :migration, schema: 20211202041233 do let(:vulnerability_findings) { table(:vulnerability_occurrences) } let(:finding_links) { table(:vulnerability_finding_links) } diff --git a/spec/lib/gitlab/background_migration/update_timelogs_null_spent_at_spec.rb b/spec/lib/gitlab/background_migration/update_timelogs_null_spent_at_spec.rb new file mode 100644 index 00000000000..908f11aabc3 --- /dev/null +++ b/spec/lib/gitlab/background_migration/update_timelogs_null_spent_at_spec.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::UpdateTimelogsNullSpentAt, schema: 20211215090620 do + let!(:previous_time) { 10.days.ago } + let!(:namespace) { table(:namespaces).create!(name: 'namespace', path: 'namespace') } + let!(:project) { table(:projects).create!(namespace_id: namespace.id) } + let!(:issue) { table(:issues).create!(project_id: project.id) } + let!(:merge_request) { table(:merge_requests).create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature') } + let!(:timelog1) { create_timelog!(issue_id: issue.id) } + let!(:timelog2) { create_timelog!(merge_request_id: merge_request.id) } + let!(:timelog3) { create_timelog!(issue_id: issue.id, spent_at: previous_time) } + let!(:timelog4) { create_timelog!(merge_request_id: merge_request.id, spent_at: previous_time) } + + subject(:background_migration) { described_class.new } + + before do + table(:timelogs).where.not(id: [timelog3.id, timelog4.id]).update_all(spent_at: nil) + end + + describe '#perform' do + it 'sets correct spent_at' do + background_migration.perform(timelog1.id, timelog4.id) + + expect(timelog1.reload.spent_at).to be_like_time(timelog1.created_at) + expect(timelog2.reload.spent_at).to be_like_time(timelog2.created_at) + expect(timelog3.reload.spent_at).to be_like_time(previous_time) + expect(timelog4.reload.spent_at).to be_like_time(previous_time) + expect(timelog3.reload.spent_at).not_to be_like_time(timelog3.created_at) + expect(timelog4.reload.spent_at).not_to be_like_time(timelog4.created_at) + end + end + + private + + def create_timelog!(**args) + table(:timelogs).create!(**args, time_spent: 1) + end +end |