diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-05-17 16:05:49 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-05-17 16:05:49 +0000 |
commit | 43a25d93ebdabea52f99b05e15b06250cd8f07d7 (patch) | |
tree | dceebdc68925362117480a5d672bcff122fb625b /spec/lib/gitlab/background_migration | |
parent | 20c84b99005abd1c82101dfeff264ac50d2df211 (diff) | |
download | gitlab-ce-0f94cf6ca9d272d8e0fda4a7a597866cf3dc1fc0.tar.gz |
Add latest changes from gitlab-org/gitlab@16-0-stable-eev16.0.0-rc4216-0-stable
Diffstat (limited to 'spec/lib/gitlab/background_migration')
65 files changed, 1791 insertions, 2613 deletions
diff --git a/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb b/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb index 7075d4694ae..92fec48454c 100644 --- a/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::BackfillAdminModeScopeForPersonalAccessTokens, - :migration, schema: 20221228103133, feature_category: :authentication_and_authorization do + :migration, schema: 20221228103133, feature_category: :system_access do let(:users) { table(:users) } let(:personal_access_tokens) { table(:personal_access_tokens) } @@ -24,8 +24,12 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillAdminModeScopeForPersonalAcc personal_access_tokens.create!(name: 'admin 4', user_id: admin.id, scopes: "---\n- admin_mode\n") end - let!(:pat_admin_2) { personal_access_tokens.create!(name: 'admin 5', user_id: admin.id, scopes: "---\n- read_api\n") } - let!(:pat_not_in_range) { personal_access_tokens.create!(name: 'admin 6', user_id: admin.id, scopes: "---\n- api\n") } + let!(:pat_with_symbol_in_scopes) do + personal_access_tokens.create!(name: 'admin 5', user_id: admin.id, scopes: "---\n- :api\n") + end + + let!(:pat_admin_2) { personal_access_tokens.create!(name: 'admin 6', user_id: admin.id, scopes: "---\n- read_api\n") } + let!(:pat_not_in_range) { personal_access_tokens.create!(name: 'admin 7', user_id: admin.id, scopes: "---\n- api\n") } subject do described_class.new( @@ -47,6 +51,7 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillAdminModeScopeForPersonalAcc expect(pat_revoked.reload.scopes).to eq("---\n- api\n") expect(pat_expired.reload.scopes).to eq("---\n- api\n") expect(pat_admin_mode.reload.scopes).to eq("---\n- admin_mode\n") + expect(pat_with_symbol_in_scopes.reload.scopes).to eq("---\n- api\n- admin_mode\n") expect(pat_admin_2.reload.scopes).to eq("---\n- read_api\n- admin_mode\n") expect(pat_not_in_range.reload.scopes).to eq("---\n- api\n") end diff --git a/spec/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities_spec.rb b/spec/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities_spec.rb index 3aab0cdf54b..edb6ff59340 100644 --- a/spec/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities_spec.rb @@ -4,10 +4,12 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::BackfillClusterAgentsHasVulnerabilities, :migration do # rubocop:disable Layout/LineLength let(:migration) do - described_class.new(start_id: 1, end_id: 10, - batch_table: table_name, batch_column: batch_column, - sub_batch_size: sub_batch_size, pause_ms: pause_ms, - connection: ApplicationRecord.connection) + described_class.new( + start_id: 1, end_id: 10, + batch_table: table_name, batch_column: batch_column, + sub_batch_size: sub_batch_size, pause_ms: pause_ms, + connection: ApplicationRecord.connection + ) end let(:users_table) { table(:users) } diff --git a/spec/lib/gitlab/background_migration/backfill_design_management_repositories_spec.rb b/spec/lib/gitlab/background_migration/backfill_design_management_repositories_spec.rb new file mode 100644 index 00000000000..0cabdc78db8 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_design_management_repositories_spec.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe( + Gitlab::BackgroundMigration::BackfillDesignManagementRepositories, + schema: 20230406121544, + feature_category: :geo_replication +) do + let!(:namespaces) { table(:namespaces) } + let!(:projects) { table(:projects) } + let!(:design_management_repositories) { table(:design_management_repositories) } + + subject(:migration) do + described_class.new( + start_id: projects.minimum(:id), + end_id: projects.maximum(:id), + batch_table: :projects, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ) + end + + describe '#perform' do + it 'creates design_management_repositories entries for all projects in range' do + namespace1 = create_namespace('test1') + namespace2 = create_namespace('test2') + project1 = create_project(namespace1, 'test1') + project2 = create_project(namespace2, 'test2') + design_management_repositories.create!(project_id: project2.id) + + expect { migration.perform } + .to change { design_management_repositories.pluck(:project_id) } + .from([project2.id]) + .to match_array([project1.id, project2.id]) + end + + context 'when project_id already exists in design_management_repositories' do + it "doesn't duplicate project_id" do + namespace = create_namespace('test1') + project = create_project(namespace, 'test1') + design_management_repositories.create!(project_id: project.id) + + expect { migration.perform } + .not_to change { design_management_repositories.pluck(:project_id) } + end + end + + def create_namespace(name) + namespaces.create!( + name: name, + path: name, + type: 'Project' + ) + end + + def create_project(namespace, name) + projects.create!( + namespace_id: namespace.id, + project_namespace_id: namespace.id, + name: name, + path: name + ) + end + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_environment_tiers_spec.rb b/spec/lib/gitlab/background_migration/backfill_environment_tiers_spec.rb index 788ed40b61e..9026c327e3c 100644 --- a/spec/lib/gitlab/background_migration/backfill_environment_tiers_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_environment_tiers_spec.rb @@ -8,10 +8,12 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillEnvironmentTiers, let!(:project) { table(:projects).create!(namespace_id: namespace.id, project_namespace_id: namespace.id) } let(:migration) do - described_class.new(start_id: 1, end_id: 1000, - batch_table: :environments, batch_column: :id, - sub_batch_size: 10, pause_ms: 0, - connection: ApplicationRecord.connection) + described_class.new( + start_id: 1, end_id: 1000, + batch_table: :environments, batch_column: :id, + sub_batch_size: 10, pause_ms: 0, + connection: ApplicationRecord.connection + ) end describe '#perform' do diff --git a/spec/lib/gitlab/background_migration/backfill_group_features_spec.rb b/spec/lib/gitlab/background_migration/backfill_group_features_spec.rb index e0be5a785b8..023d4b04e63 100644 --- a/spec/lib/gitlab/background_migration/backfill_group_features_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_group_features_spec.rb @@ -7,14 +7,16 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillGroupFeatures, :migration, s let(:namespaces) { table(:namespaces) } subject do - described_class.new(start_id: 1, - end_id: 4, - batch_table: :namespaces, - batch_column: :id, - sub_batch_size: 10, - pause_ms: 0, - job_arguments: [4], - connection: ActiveRecord::Base.connection) + described_class.new( + start_id: 1, + end_id: 4, + batch_table: :namespaces, + batch_column: :id, + sub_batch_size: 10, + pause_ms: 0, + job_arguments: [4], + connection: ActiveRecord::Base.connection + ) end describe '#perform' do diff --git a/spec/lib/gitlab/background_migration/backfill_imported_issue_search_data_spec.rb b/spec/lib/gitlab/background_migration/backfill_imported_issue_search_data_spec.rb index 479afb56210..b3f04055e0a 100644 --- a/spec/lib/gitlab/background_migration/backfill_imported_issue_search_data_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_imported_issue_search_data_spec.rb @@ -30,13 +30,15 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillImportedIssueSearchData, end let(:migration) do - described_class.new(start_id: issue.id, - end_id: issue.id + 30, - batch_table: :issues, - batch_column: :id, - sub_batch_size: 2, - pause_ms: 0, - connection: ApplicationRecord.connection) + described_class.new( + start_id: issue.id, + end_id: issue.id + 30, + batch_table: :issues, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ApplicationRecord.connection + ) end let(:perform_migration) { migration.perform } diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_details_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_details_spec.rb index b6282de0da6..39ad60fb13b 100644 --- a/spec/lib/gitlab/background_migration/backfill_namespace_details_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_namespace_details_spec.rb @@ -7,27 +7,36 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceDetails, :migration let(:namespace_details) { table(:namespace_details) } subject(:perform_migration) do - described_class.new(start_id: namespaces.minimum(:id), - end_id: namespaces.maximum(:id), - batch_table: :namespaces, - batch_column: :id, - sub_batch_size: 2, - pause_ms: 0, - connection: ActiveRecord::Base.connection) - .perform + described_class.new( + start_id: namespaces.minimum(:id), + end_id: namespaces.maximum(:id), + batch_table: :namespaces, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform end describe '#perform' do it 'creates details for all namespaces in range' do - namespace1 = namespaces.create!(id: 5, name: 'test1', path: 'test1', description: "Some description1", - description_html: "Some description html1", cached_markdown_version: 4) - namespaces.create!(id: 6, name: 'test2', path: 'test2', type: 'Project', - description: "Some description2", description_html: "Some description html2", - cached_markdown_version: 4) - namespace3 = namespaces.create!(id: 7, name: 'test3', path: 'test3', description: "Some description3", - description_html: "Some description html3", cached_markdown_version: 4) - namespace4 = namespaces.create!(id: 8, name: 'test4', path: 'test4', description: "Some description3", - description_html: "Some description html4", cached_markdown_version: 4) + namespace1 = namespaces.create!( + id: 5, name: 'test1', path: 'test1', description: "Some description1", + description_html: "Some description html1", cached_markdown_version: 4 + ) + namespaces.create!( + id: 6, name: 'test2', path: 'test2', type: 'Project', + description: "Some description2", description_html: "Some description html2", + cached_markdown_version: 4 + ) + namespace3 = namespaces.create!( + id: 7, name: 'test3', path: 'test3', description: "Some description3", + description_html: "Some description html3", cached_markdown_version: 4 + ) + namespace4 = namespaces.create!( + id: 8, name: 'test4', path: 'test4', description: "Some description3", + description_html: "Some description html4", cached_markdown_version: 4 + ) namespace_details.delete_all expect(namespace_details.pluck(:namespace_id)).to eql [] diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route_spec.rb index b821efcadb0..3a8a327550b 100644 --- a/spec/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route_spec.rb @@ -22,18 +22,29 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceIdForNamespaceRoute subject(:perform_migration) { migration.perform(1, 10, table_name, batch_column, sub_batch_size, pause_ms) } before do - routes_table.create!(id: 1, name: 'test1', path: 'test1', source_id: namespace1.id, - source_type: namespace1.class.sti_name) - routes_table.create!(id: 2, name: 'test2', path: 'test2', source_id: namespace2.id, - source_type: namespace2.class.sti_name) - routes_table.create!(id: 5, name: 'test3', path: 'test3', source_id: project1.id, - source_type: project1.class.sti_name) # should be ignored - project route - routes_table.create!(id: 6, name: 'test4', path: 'test4', source_id: non_existing_record_id, - source_type: namespace3.class.sti_name) # should be ignored - invalid source_id - routes_table.create!(id: 10, name: 'test5', path: 'test5', source_id: namespace3.id, - source_type: namespace3.class.sti_name) - routes_table.create!(id: 11, name: 'test6', path: 'test6', source_id: namespace4.id, - source_type: namespace4.class.sti_name) # should be ignored - outside the scope + routes_table.create!( + id: 1, name: 'test1', path: 'test1', source_id: namespace1.id, source_type: namespace1.class.sti_name + ) + + routes_table.create!( + id: 2, name: 'test2', path: 'test2', source_id: namespace2.id, source_type: namespace2.class.sti_name + ) + + routes_table.create!( + id: 5, name: 'test3', path: 'test3', source_id: project1.id, source_type: project1.class.sti_name + ) # should be ignored - project route + + routes_table.create!( + id: 6, name: 'test4', path: 'test4', source_id: non_existing_record_id, source_type: namespace3.class.sti_name + ) # should be ignored - invalid source_id + + routes_table.create!( + id: 10, name: 'test5', path: 'test5', source_id: namespace3.id, source_type: namespace3.class.sti_name + ) + + routes_table.create!( + id: 11, name: 'test6', path: 'test6', source_id: namespace4.id, source_type: namespace4.class.sti_name + ) # should be ignored - outside the scope end it 'backfills `type` for the selected records', :aggregate_failures do diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads_spec.rb index 564aa3b8c01..6a55c6951d5 100644 --- a/spec/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads_spec.rb @@ -38,14 +38,15 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceIdOfVulnerabilityRe end subject(:perform_migration) do - described_class.new(start_id: vulnerability_read.vulnerability_id, - end_id: vulnerability_read.vulnerability_id, - batch_table: :vulnerability_reads, - batch_column: :vulnerability_id, - sub_batch_size: 1, - pause_ms: 0, - connection: ActiveRecord::Base.connection) - .perform + described_class.new( + start_id: vulnerability_read.vulnerability_id, + end_id: vulnerability_read.vulnerability_id, + batch_table: :vulnerability_reads, + batch_column: :vulnerability_id, + sub_batch_size: 1, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform end it 'sets the namespace_id of existing record' do diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children_spec.rb deleted file mode 100644 index 876eb070745..00000000000 --- a/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children_spec.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceTraversalIdsChildren, :migration, schema: 20210826171758 do - let(:namespaces_table) { table(:namespaces) } - - let!(:user_namespace) { namespaces_table.create!(id: 1, name: 'user', path: 'user', type: nil) } - let!(:root_group) { namespaces_table.create!(id: 2, name: 'group', path: 'group', type: 'Group', parent_id: nil) } - let!(:sub_group) { namespaces_table.create!(id: 3, name: 'subgroup', path: 'subgroup', type: 'Group', parent_id: 2) } - - describe '#perform' do - it 'backfills traversal_ids for child namespaces' do - described_class.new.perform(1, 3, 5) - - expect(user_namespace.reload.traversal_ids).to eq([]) - expect(root_group.reload.traversal_ids).to eq([]) - expect(sub_group.reload.traversal_ids).to eq([root_group.id, sub_group.id]) - end - end -end diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots_spec.rb deleted file mode 100644 index ad9b54608c6..00000000000 --- a/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots_spec.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceTraversalIdsRoots, :migration, schema: 20210826171758 do - let(:namespaces_table) { table(:namespaces) } - - let!(:user_namespace) { namespaces_table.create!(id: 1, name: 'user', path: 'user', type: nil) } - let!(:root_group) { namespaces_table.create!(id: 2, name: 'group', path: 'group', type: 'Group', parent_id: nil) } - let!(:sub_group) { namespaces_table.create!(id: 3, name: 'subgroup', path: 'subgroup', type: 'Group', parent_id: 2) } - - describe '#perform' do - it 'backfills traversal_ids for root namespaces' do - described_class.new.perform(1, 3, 5) - - expect(user_namespace.reload.traversal_ids).to eq([user_namespace.id]) - expect(root_group.reload.traversal_ids).to eq([root_group.id]) - expect(sub_group.reload.traversal_ids).to eq([]) - end - end -end diff --git a/spec/lib/gitlab/background_migration/backfill_partitioned_table_spec.rb b/spec/lib/gitlab/background_migration/backfill_partitioned_table_spec.rb new file mode 100644 index 00000000000..53216cc780b --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_partitioned_table_spec.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillPartitionedTable, feature_category: :database do + subject(:backfill_job) do + described_class.new( + start_id: 1, + end_id: 3, + batch_table: source_table, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + job_arguments: [destination_table], + connection: connection + ) + end + + let(:connection) { ApplicationRecord.connection } + let(:source_table) { '_test_source_table' } + let(:destination_table) { "#{source_table}_partitioned" } + let(:source_model) { Class.new(ApplicationRecord) } + let(:destination_model) { Class.new(ApplicationRecord) } + + describe '#perform' do + context 'without the destination table' do + let(:expected_error_message) do + "exiting backfill migration because partitioned table #{destination_table} does not exist. " \ + "This could be due to rollback of the migration which created the partitioned table." + end + + it 'raises an exception' do + expect { backfill_job.perform }.to raise_error(expected_error_message) + end + end + + context 'with destination table being not partitioned' do + before do + connection.execute(<<~SQL) + CREATE TABLE #{destination_table} ( + id serial NOT NULL, + col1 int NOT NULL, + col2 text NOT NULL, + created_at timestamptz NOT NULL, + PRIMARY KEY (id, created_at) + ) + SQL + end + + after do + connection.drop_table destination_table + end + + let(:expected_error_message) do + "exiting backfill migration because the given destination table is not partitioned." + end + + it 'raises an exception' do + expect { backfill_job.perform }.to raise_error(expected_error_message) + end + end + + context 'when the destination table exists' do + before do + connection.execute(<<~SQL) + CREATE TABLE #{source_table} ( + id serial NOT NULL PRIMARY KEY, + col1 int NOT NULL, + col2 text NOT NULL, + created_at timestamptz NOT NULL + ) + SQL + + connection.execute(<<~SQL) + CREATE TABLE #{destination_table} ( + id serial NOT NULL, + col1 int NOT NULL, + col2 text NOT NULL, + created_at timestamptz NOT NULL, + PRIMARY KEY (id, created_at) + ) PARTITION BY RANGE (created_at) + SQL + + connection.execute(<<~SQL) + CREATE TABLE #{destination_table}_202001 PARTITION OF #{destination_table} + FOR VALUES FROM ('2020-01-01') TO ('2020-02-01') + SQL + + connection.execute(<<~SQL) + CREATE TABLE #{destination_table}_202002 PARTITION OF #{destination_table} + FOR VALUES FROM ('2020-02-01') TO ('2020-03-01') + SQL + + source_model.table_name = source_table + destination_model.table_name = destination_table + end + + after do + connection.drop_table source_table + connection.drop_table destination_table + end + + let(:timestamp) { Time.utc(2020, 1, 2).round } + let!(:source1) { create_source_record(timestamp) } + let!(:source2) { create_source_record(timestamp + 1.day) } + let!(:source3) { create_source_record(timestamp + 1.month) } + + it 'copies data into the destination table idempotently' do + expect(destination_model.count).to eq(0) + + backfill_job.perform + + expect(destination_model.count).to eq(3) + + source_model.find_each do |source_record| + destination_record = destination_model.find_by_id(source_record.id) + + expect(destination_record.attributes).to eq(source_record.attributes) + end + + backfill_job.perform + + expect(destination_model.count).to eq(3) + end + + it 'breaks the assigned batch into smaller sub batches' do + expect_next_instance_of(Gitlab::Database::PartitioningMigrationHelpers::BulkCopy) do |bulk_copy| + expect(bulk_copy).to receive(:copy_between).with(source1.id, source2.id) + expect(bulk_copy).to receive(:copy_between).with(source3.id, source3.id) + end + + backfill_job.perform + end + end + end + + def create_source_record(timestamp) + source_model.create!(col1: 123, col2: 'original value', created_at: timestamp) + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_prepared_at_merge_requests_spec.rb b/spec/lib/gitlab/background_migration/backfill_prepared_at_merge_requests_spec.rb new file mode 100644 index 00000000000..28ecfae1bd4 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_prepared_at_merge_requests_spec.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillPreparedAtMergeRequests, :migration, + feature_category: :code_review_workflow, schema: 20230202135758 do + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:mr_table) { table(:merge_requests) } + + let(:namespace) { namespaces.create!(name: 'batchtest1', type: 'Group', path: 'space1') } + let(:proj_namespace) { namespaces.create!(name: 'proj1', path: 'proj1', type: 'Project', parent_id: namespace.id) } + let(:project) do + projects.create!(name: 'proj1', path: 'proj1', namespace_id: namespace.id, project_namespace_id: proj_namespace.id) + end + + it 'updates merge requests with prepared_at nil' do + time = Time.current + + mr_1 = mr_table.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature', + prepared_at: nil, merge_status: 'checking') + mr_2 = mr_table.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature', + prepared_at: nil, merge_status: 'preparing') + mr_3 = mr_table.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature', + prepared_at: time) + mr_4 = mr_table.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature', + prepared_at: time, merge_status: 'checking') + mr_5 = mr_table.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature', + prepared_at: time, merge_status: 'preparing') + + test_worker = described_class.new( + start_id: mr_1.id, + end_id: [(mr_5.id + 1), 100].max, + batch_table: :merge_requests, + batch_column: :id, + sub_batch_size: 10, + pause_ms: 0, + connection: ApplicationRecord.connection + ) + + expect(mr_1.prepared_at).to be_nil + expect(mr_2.prepared_at).to be_nil + expect(mr_3.prepared_at.to_i).to eq(time.to_i) + expect(mr_4.prepared_at.to_i).to eq(time.to_i) + expect(mr_5.prepared_at.to_i).to eq(time.to_i) + + test_worker.perform + + expect(mr_1.reload.prepared_at.to_i).to eq(mr_1.created_at.to_i) + expect(mr_2.reload.prepared_at).to be_nil + expect(mr_3.reload.prepared_at.to_i).to eq(time.to_i) + expect(mr_4.reload.prepared_at.to_i).to eq(time.to_i) + expect(mr_5.reload.prepared_at.to_i).to eq(time.to_i) + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level_spec.rb index fd6c055b9f6..47ff2883fb2 100644 --- a/spec/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level_spec.rb @@ -101,14 +101,15 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillProjectFeaturePackageRegistr end subject(:perform_migration) do - described_class.new(start_id: project1.id, - end_id: project5.id, - batch_table: :projects, - batch_column: :id, - sub_batch_size: 2, - pause_ms: 0, - connection: ActiveRecord::Base.connection) - .perform + described_class.new( + start_id: project1.id, + end_id: project5.id, + batch_table: :projects, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform end it 'backfills project_features.package_registry_access_level', :aggregate_failures do diff --git a/spec/lib/gitlab/background_migration/backfill_project_member_namespace_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_member_namespace_id_spec.rb index ca7ca41a33e..96f49624d22 100644 --- a/spec/lib/gitlab/background_migration/backfill_project_member_namespace_id_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_project_member_namespace_id_spec.rb @@ -4,10 +4,12 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::BackfillProjectMemberNamespaceId, :migration, schema: 20220516054011 do let(:migration) do - described_class.new(start_id: 1, end_id: 10, - batch_table: table_name, batch_column: batch_column, - sub_batch_size: sub_batch_size, pause_ms: pause_ms, - connection: ApplicationRecord.connection) + described_class.new( + start_id: 1, end_id: 10, + batch_table: table_name, batch_column: batch_column, + sub_batch_size: sub_batch_size, pause_ms: pause_ms, + connection: ApplicationRecord.connection + ) end let(:members_table) { table(:members) } @@ -35,37 +37,55 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillProjectMemberNamespaceId, :m projects_table.create!(id: 102, name: 'project3', path: 'project3', namespace_id: 202, project_namespace_id: 302) # project1, no member namespace (fill in) - members_table.create!(id: 1, source_id: 100, - source_type: 'Project', type: 'ProjectMember', - member_namespace_id: nil, access_level: 10, notification_level: 3) + members_table.create!( + id: 1, source_id: 100, + source_type: 'Project', type: 'ProjectMember', + member_namespace_id: nil, access_level: 10, notification_level: 3 + ) + # bogus source id, no member namespace id (do nothing) - members_table.create!(id: 2, source_id: non_existing_record_id, - source_type: 'Project', type: 'ProjectMember', - member_namespace_id: nil, access_level: 10, notification_level: 3) + members_table.create!( + id: 2, source_id: non_existing_record_id, + source_type: 'Project', type: 'ProjectMember', + member_namespace_id: nil, access_level: 10, notification_level: 3 + ) + # project3, existing member namespace id (do nothing) - members_table.create!(id: 3, source_id: 102, - source_type: 'Project', type: 'ProjectMember', - member_namespace_id: 300, access_level: 10, notification_level: 3) + members_table.create!( + id: 3, source_id: 102, + source_type: 'Project', type: 'ProjectMember', + member_namespace_id: 300, access_level: 10, notification_level: 3 + ) # Group memberships (do not change) # group1, no member namespace (do nothing) - members_table.create!(id: 4, source_id: 201, - source_type: 'Namespace', type: 'GroupMember', - member_namespace_id: nil, access_level: 10, notification_level: 3) + members_table.create!( + id: 4, source_id: 201, + source_type: 'Namespace', type: 'GroupMember', + member_namespace_id: nil, access_level: 10, notification_level: 3 + ) + # group2, existing member namespace (do nothing) - members_table.create!(id: 5, source_id: 202, - source_type: 'Namespace', type: 'GroupMember', - member_namespace_id: 201, access_level: 10, notification_level: 3) + members_table.create!( + id: 5, source_id: 202, + source_type: 'Namespace', type: 'GroupMember', + member_namespace_id: 201, access_level: 10, notification_level: 3 + ) # Project Namespace memberships (do not change) # project namespace, existing member namespace (do nothing) - members_table.create!(id: 6, source_id: 300, - source_type: 'Namespace', type: 'ProjectNamespaceMember', - member_namespace_id: 201, access_level: 10, notification_level: 3) + members_table.create!( + id: 6, source_id: 300, + source_type: 'Namespace', type: 'ProjectNamespaceMember', + member_namespace_id: 201, access_level: 10, notification_level: 3 + ) + # project namespace, not member namespace (do nothing) - members_table.create!(id: 7, source_id: 301, - source_type: 'Namespace', type: 'ProjectNamespaceMember', - member_namespace_id: 201, access_level: 10, notification_level: 3) + members_table.create!( + id: 7, source_id: 301, + source_type: 'Namespace', type: 'ProjectNamespaceMember', + member_namespace_id: 201, access_level: 10, notification_level: 3 + ) end it 'backfills `member_namespace_id` for the selected records', :aggregate_failures do diff --git a/spec/lib/gitlab/background_migration/backfill_project_namespace_details_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_namespace_details_spec.rb index 01daf16d10c..aac17a426b5 100644 --- a/spec/lib/gitlab/background_migration/backfill_project_namespace_details_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_project_namespace_details_spec.rb @@ -8,32 +8,41 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillProjectNamespaceDetails, :mi let!(:projects) { table(:projects) } subject(:perform_migration) do - described_class.new(start_id: projects.minimum(:id), - end_id: projects.maximum(:id), - batch_table: :projects, - batch_column: :id, - sub_batch_size: 2, - pause_ms: 0, - connection: ActiveRecord::Base.connection) - .perform + described_class.new( + start_id: projects.minimum(:id), + end_id: projects.maximum(:id), + batch_table: :projects, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform end describe '#perform' do it 'creates details for all project namespaces in range' do - namespaces.create!(id: 5, name: 'test1', path: 'test1', description: "Some description1", - description_html: "Some description html1", cached_markdown_version: 4) + namespaces.create!( + id: 5, name: 'test1', path: 'test1', description: "Some description1", + description_html: "Some description html1", cached_markdown_version: 4 + ) project_namespace1 = namespaces.create!(id: 6, name: 'test2', path: 'test2', type: 'Project') - namespaces.create!(id: 7, name: 'test3', path: 'test3', description: "Some description3", - description_html: "Some description html3", cached_markdown_version: 4) + namespaces.create!( + id: 7, name: 'test3', path: 'test3', description: "Some description3", + description_html: "Some description html3", cached_markdown_version: 4 + ) project_namespace2 = namespaces.create!(id: 8, name: 'test4', path: 'test4', type: 'Project') - project1 = projects.create!(namespace_id: project_namespace1.id, name: 'gitlab1', path: 'gitlab1', - project_namespace_id: project_namespace1.id, description: "Some description2", - description_html: "Some description html2", cached_markdown_version: 4) - project2 = projects.create!(namespace_id: project_namespace2.id, name: 'gitlab2', path: 'gitlab2', - project_namespace_id: project_namespace2.id, - description: "Some description3", - description_html: "Some description html4", cached_markdown_version: 4) + project1 = projects.create!( + namespace_id: project_namespace1.id, name: 'gitlab1', path: 'gitlab1', + project_namespace_id: project_namespace1.id, description: "Some description2", + description_html: "Some description html2", cached_markdown_version: 4 + ) + project2 = projects.create!( + namespace_id: project_namespace2.id, name: 'gitlab2', path: 'gitlab2', + project_namespace_id: project_namespace2.id, + description: "Some description3", + description_html: "Some description html4", cached_markdown_version: 4 + ) namespace_details.delete_all diff --git a/spec/lib/gitlab/background_migration/backfill_project_wiki_repositories_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_wiki_repositories_spec.rb new file mode 100644 index 00000000000..e81bd0604e6 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_project_wiki_repositories_spec.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe( + Gitlab::BackgroundMigration::BackfillProjectWikiRepositories, + schema: 20230306195007, + feature_category: :geo_replication) do + let!(:namespaces) { table(:namespaces) } + let!(:projects) { table(:projects) } + let!(:project_wiki_repositories) { table(:project_wiki_repositories) } + + subject(:migration) do + described_class.new( + start_id: projects.minimum(:id), + end_id: projects.maximum(:id), + batch_table: :projects, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ) + end + + describe '#perform' do + it 'creates project_wiki_repositories entries for all projects in range' do + namespace1 = create_namespace('test1') + namespace2 = create_namespace('test2') + project1 = create_project(namespace1, 'test1') + project2 = create_project(namespace2, 'test2') + project_wiki_repositories.create!(project_id: project2.id) + + expect { migration.perform } + .to change { project_wiki_repositories.pluck(:project_id) } + .from([project2.id]) + .to match_array([project1.id, project2.id]) + end + + it 'does nothing if project_id already exist in project_wiki_repositories' do + namespace = create_namespace('test1') + project = create_project(namespace, 'test1') + project_wiki_repositories.create!(project_id: project.id) + + expect { migration.perform } + .not_to change { project_wiki_repositories.pluck(:project_id) } + end + + def create_namespace(name) + namespaces.create!( + name: name, + path: name, + type: 'Project' + ) + end + + def create_project(namespace, name) + projects.create!( + namespace_id: namespace.id, + project_namespace_id: namespace.id, + name: name, + path: name + ) + end + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_releases_author_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_releases_author_id_spec.rb index d8ad10849f2..898f241a930 100644 --- a/spec/lib/gitlab/background_migration/backfill_releases_author_id_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_releases_author_id_spec.rb @@ -10,35 +10,52 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillReleasesAuthorId, let!(:test_user) { user_table.create!(name: 'test', email: 'test@example.com', username: 'test', projects_limit: 10) } let!(:ghost_user) do - user_table.create!(name: 'ghost', email: 'ghost@example.com', - username: 'ghost', user_type: User::USER_TYPES['ghost'], projects_limit: 100000) + user_table.create!( + name: 'ghost', email: 'ghost@example.com', + username: 'ghost', user_type: User::USER_TYPES['ghost'], projects_limit: 100000 + ) end let(:migration) do - described_class.new(start_id: 1, end_id: 100, - batch_table: :releases, batch_column: :id, - sub_batch_size: 10, pause_ms: 0, - job_arguments: [ghost_user.id], - connection: ApplicationRecord.connection) + described_class.new( + start_id: 1, end_id: 100, + batch_table: :releases, batch_column: :id, + sub_batch_size: 10, pause_ms: 0, + job_arguments: [ghost_user.id], + connection: ApplicationRecord.connection + ) end subject(:perform_migration) { migration.perform } before do - releases_table.create!(tag: 'tag1', name: 'tag1', - released_at: (date_time - 1.minute), author_id: test_user.id) - releases_table.create!(tag: 'tag2', name: 'tag2', - released_at: (date_time - 2.minutes), author_id: test_user.id) - releases_table.new(tag: 'tag3', name: 'tag3', - released_at: (date_time - 3.minutes), author_id: nil).save!(validate: false) - releases_table.new(tag: 'tag4', name: 'tag4', - released_at: (date_time - 4.minutes), author_id: nil).save!(validate: false) - releases_table.new(tag: 'tag5', name: 'tag5', - released_at: (date_time - 5.minutes), author_id: nil).save!(validate: false) - releases_table.create!(tag: 'tag6', name: 'tag6', - released_at: (date_time - 6.minutes), author_id: test_user.id) - releases_table.new(tag: 'tag7', name: 'tag7', - released_at: (date_time - 7.minutes), author_id: nil).save!(validate: false) + releases_table.create!( + tag: 'tag1', name: 'tag1', released_at: (date_time - 1.minute), author_id: test_user.id + ) + + releases_table.create!( + tag: 'tag2', name: 'tag2', released_at: (date_time - 2.minutes), author_id: test_user.id + ) + + releases_table.new( + tag: 'tag3', name: 'tag3', released_at: (date_time - 3.minutes), author_id: nil + ).save!(validate: false) + + releases_table.new( + tag: 'tag4', name: 'tag4', released_at: (date_time - 4.minutes), author_id: nil + ).save!(validate: false) + + releases_table.new( + tag: 'tag5', name: 'tag5', released_at: (date_time - 5.minutes), author_id: nil + ).save!(validate: false) + + releases_table.create!( + tag: 'tag6', name: 'tag6', released_at: (date_time - 6.minutes), author_id: test_user.id + ) + + releases_table.new( + tag: 'tag7', name: 'tag7', released_at: (date_time - 7.minutes), author_id: nil + ).save!(validate: false) end it 'backfills `author_id` for the selected records', :aggregate_failures do diff --git a/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb b/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb index 80fd86e90bb..d8874cb811b 100644 --- a/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb @@ -2,8 +2,8 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::BackfillSnippetRepositories, :migration, schema: 20210826171758, -feature_category: :source_code_management do +RSpec.describe Gitlab::BackgroundMigration::BackfillSnippetRepositories, :migration, schema: 20211202041233, + feature_category: :source_code_management do let(:gitlab_shell) { Gitlab::Shell.new } let(:users) { table(:users) } let(:snippets) { table(:snippets) } @@ -14,24 +14,28 @@ feature_category: :source_code_management do let(:user_name) { 'Test' } let!(:user) do - users.create!(id: 1, - email: 'user@example.com', - projects_limit: 10, - username: 'test', - name: user_name, - state: user_state, - last_activity_on: 1.minute.ago, - user_type: user_type, - confirmed_at: 1.day.ago) + users.create!( + id: 1, + email: 'user@example.com', + projects_limit: 10, + username: 'test', + name: user_name, + state: user_state, + last_activity_on: 1.minute.ago, + user_type: user_type, + confirmed_at: 1.day.ago + ) end let!(:migration_bot) do - users.create!(id: 100, - email: "noreply+gitlab-migration-bot%s@#{Settings.gitlab.host}", - user_type: HasUserType::USER_TYPES[:migration_bot], - name: 'GitLab Migration Bot', - projects_limit: 10, - username: 'bot') + users.create!( + id: 100, + email: "noreply+gitlab-migration-bot%s@#{Settings.gitlab.host}", + user_type: HasUserType::USER_TYPES[:migration_bot], + name: 'GitLab Migration Bot', + projects_limit: 10, + username: 'bot' + ) end let!(:snippet_with_repo) { snippets.create!(id: 1, type: 'PersonalSnippet', author_id: user.id, file_name: file_name, content: content) } @@ -260,15 +264,17 @@ feature_category: :source_code_management do context 'when both user name and snippet file_name are invalid' do let(:user_name) { '.' } let!(:other_user) do - users.create!(id: 2, - email: 'user2@example.com', - projects_limit: 10, - username: 'test2', - name: 'Test2', - state: user_state, - last_activity_on: 1.minute.ago, - user_type: user_type, - confirmed_at: 1.day.ago) + users.create!( + id: 2, + email: 'user2@example.com', + projects_limit: 10, + username: 'test2', + name: 'Test2', + state: user_state, + last_activity_on: 1.minute.ago, + user_type: user_type, + confirmed_at: 1.day.ago + ) end let!(:invalid_snippet) { snippets.create!(id: 4, type: 'PersonalSnippet', author_id: user.id, file_name: '.', content: content) } @@ -322,10 +328,12 @@ feature_category: :source_code_management do end def raw_repository(snippet) - Gitlab::Git::Repository.new('default', - "#{disk_path(snippet)}.git", - Gitlab::GlRepository::SNIPPET.identifier_for_container(snippet), - "@snippets/#{snippet.id}") + Gitlab::Git::Repository.new( + 'default', + "#{disk_path(snippet)}.git", + Gitlab::GlRepository::SNIPPET.identifier_for_container(snippet), + "@snippets/#{snippet.id}" + ) end def hashed_repository(snippet) diff --git a/spec/lib/gitlab/background_migration/backfill_upvotes_count_on_issues_spec.rb b/spec/lib/gitlab/background_migration/backfill_upvotes_count_on_issues_spec.rb deleted file mode 100644 index 7142aea3ab2..00000000000 --- a/spec/lib/gitlab/background_migration/backfill_upvotes_count_on_issues_spec.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillUpvotesCountOnIssues, schema: 20210826171758 do - let(:award_emoji) { table(:award_emoji) } - - let!(:namespace) { table(:namespaces).create!(name: 'namespace', path: 'namespace') } - let!(:project1) { table(:projects).create!(namespace_id: namespace.id) } - let!(:project2) { table(:projects).create!(namespace_id: namespace.id) } - let!(:issue1) { table(:issues).create!(project_id: project1.id) } - let!(:issue2) { table(:issues).create!(project_id: project2.id) } - let!(:issue3) { table(:issues).create!(project_id: project2.id) } - let!(:issue4) { table(:issues).create!(project_id: project2.id) } - - describe '#perform' do - before do - add_upvotes(issue1, :thumbsdown, 1) - add_upvotes(issue2, :thumbsup, 2) - add_upvotes(issue2, :thumbsdown, 1) - add_upvotes(issue3, :thumbsup, 3) - add_upvotes(issue4, :thumbsup, 4) - end - - it 'updates upvotes_count' do - subject.perform(issue1.id, issue4.id) - - expect(issue1.reload.upvotes_count).to eq(0) - expect(issue2.reload.upvotes_count).to eq(2) - expect(issue3.reload.upvotes_count).to eq(3) - expect(issue4.reload.upvotes_count).to eq(4) - end - end - - private - - def add_upvotes(issue, name, count) - count.times do - award_emoji.create!( - name: name.to_s, - awardable_type: 'Issue', - awardable_id: issue.id - ) - end - end -end diff --git a/spec/lib/gitlab/background_migration/backfill_user_namespace_spec.rb b/spec/lib/gitlab/background_migration/backfill_user_namespace_spec.rb deleted file mode 100644 index 395248b786d..00000000000 --- a/spec/lib/gitlab/background_migration/backfill_user_namespace_spec.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillUserNamespace, :migration, schema: 20210930211936 do - let(:migration) { described_class.new } - let(:namespaces_table) { table(:namespaces) } - - let(:table_name) { 'namespaces' } - let(:batch_column) { :id } - let(:sub_batch_size) { 100 } - let(:pause_ms) { 0 } - - subject(:perform_migration) { migration.perform(1, 10, table_name, batch_column, sub_batch_size, pause_ms) } - - before do - namespaces_table.create!(id: 1, name: 'test1', path: 'test1', type: nil) - namespaces_table.create!(id: 2, name: 'test2', path: 'test2', type: 'User') - namespaces_table.create!(id: 3, name: 'test3', path: 'test3', type: 'Group') - namespaces_table.create!(id: 4, name: 'test4', path: 'test4', type: nil) - namespaces_table.create!(id: 11, name: 'test11', path: 'test11', type: nil) - end - - it 'backfills `type` for the selected records', :aggregate_failures do - queries = ActiveRecord::QueryRecorder.new do - perform_migration - end - - expect(queries.count).to eq(3) - expect(namespaces_table.where(type: 'User').count).to eq 3 - expect(namespaces_table.where(type: 'User').pluck(:id)).to match_array([1, 2, 4]) - end - - it 'tracks timings of queries' do - expect(migration.batch_metrics.timings).to be_empty - - expect { perform_migration }.to change { migration.batch_metrics.timings } - end -end diff --git a/spec/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent_spec.rb b/spec/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent_spec.rb index f642ec8c20d..3f1a57434a7 100644 --- a/spec/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent_spec.rb @@ -4,10 +4,12 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::BackfillVulnerabilityReadsClusterAgent, :migration, schema: 20220525221133 do # rubocop:disable Layout/LineLength let(:migration) do - described_class.new(start_id: 1, end_id: 10, - batch_table: table_name, batch_column: batch_column, - sub_batch_size: sub_batch_size, pause_ms: pause_ms, - connection: ApplicationRecord.connection) + described_class.new( + start_id: 1, end_id: 10, + batch_table: table_name, batch_column: batch_column, + sub_batch_size: sub_batch_size, pause_ms: pause_ms, + connection: ApplicationRecord.connection + ) end let(:users_table) { table(:users) } diff --git a/spec/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues_spec.rb b/spec/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues_spec.rb index 5f93424faf6..c7e4095a488 100644 --- a/spec/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues_spec.rb @@ -2,7 +2,10 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::BackfillWorkItemTypeIdForIssues, :migration, schema: 20220825142324 do +RSpec.describe Gitlab::BackgroundMigration::BackfillWorkItemTypeIdForIssues, + :migration, + schema: 20220825142324, + feature_category: :team_planning do let(:batch_column) { 'id' } let(:sub_batch_size) { 2 } let(:pause_ms) { 0 } @@ -13,6 +16,7 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillWorkItemTypeIdForIssues, :mi let(:project) { table(:projects).create!(namespace_id: namespace.id, project_namespace_id: namespace.id) } let(:issues_table) { table(:issues) } let(:issue_type) { table(:work_item_types).find_by!(namespace_id: nil, base_type: issue_type_enum[:issue]) } + let(:task_type) { table(:work_item_types).find_by!(namespace_id: nil, base_type: issue_type_enum[:task]) } let(:issue1) { issues_table.create!(project_id: project.id, issue_type: issue_type_enum[:issue]) } let(:issue2) { issues_table.create!(project_id: project.id, issue_type: issue_type_enum[:issue]) } @@ -25,7 +29,7 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillWorkItemTypeIdForIssues, :mi let(:start_id) { issue1.id } let(:end_id) { requirement1.id } - let(:all_issues) { [issue1, issue2, issue3, incident1, test_case1, requirement1] } + let!(:all_issues) { [issue1, issue2, issue3, incident1, test_case1, requirement1] } let(:migration) do described_class.new( @@ -52,6 +56,27 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillWorkItemTypeIdForIssues, :mi expect(all_issues - [issue1, issue2, issue3]).to all(have_attributes(work_item_type_id: nil)) end + context 'when a record already had a work_item_type_id assigned' do + let!(:issue4) do + issues_table.create!( + project_id: project.id, + issue_type: issue_type_enum[:issue], + work_item_type_id: task_type.id + ) + end + + let(:end_id) { issue4.id } + + it 'ovewrites the work_item_type_id' do + # creating with the wrong issue_type/work_item_type_id on purpose so we can test + # that the migration is capable of fixing such inconsistencies + expect do + migrate + issue4.reload + end.to change { issue4.work_item_type_id }.from(task_type.id).to(issue_type.id) + end + end + it 'tracks timings of queries' do expect(migration.batch_metrics.timings).to be_empty diff --git a/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb b/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb index faaaccfdfaf..781bf93dd85 100644 --- a/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb +++ b/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb @@ -301,6 +301,28 @@ RSpec.describe Gitlab::BackgroundMigration::BatchedMigrationJob do perform_job end + context 'when using a sub batch exception for timeouts' do + let(:job_class) do + Class.new(described_class) do + operation_name :update + + def perform(*_) + each_sub_batch { raise ActiveRecord::StatementTimeout } # rubocop:disable Lint/UnreachableLoop + end + end + end + + let(:job_instance) do + job_class.new(start_id: 1, end_id: 10, batch_table: '_test_table', batch_column: 'id', + sub_batch_size: 2, pause_ms: 1000, connection: connection, + sub_batch_exception: StandardError) + end + + it 'raises the expected error type' do + expect { job_instance.perform }.to raise_error(StandardError) + end + end + context 'when batching_arguments are given' do it 'forwards them for batching' do expect(job_instance).to receive(:base_relation).and_return(test_table) diff --git a/spec/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects_spec.rb b/spec/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects_spec.rb deleted file mode 100644 index 5ffe665f0ad..00000000000 --- a/spec/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects_spec.rb +++ /dev/null @@ -1,85 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::CleanupOrphanedLfsObjectsProjects, schema: 20210826171758 do - let(:lfs_objects_projects) { table(:lfs_objects_projects) } - let(:lfs_objects) { table(:lfs_objects) } - let(:projects) { table(:projects) } - let(:namespaces) { table(:namespaces) } - - let(:namespace) { namespaces.create!(name: 'namespace', path: 'namespace') } - let(:project) { projects.create!(namespace_id: namespace.id) } - let(:another_project) { projects.create!(namespace_id: namespace.id) } - let(:lfs_object) { lfs_objects.create!(oid: 'abcdef', size: 1) } - let(:another_lfs_object) { lfs_objects.create!(oid: '1abcde', size: 2) } - - let!(:without_object1) { create_object(project_id: project.id) } - let!(:without_object2) { create_object(project_id: another_project.id) } - let!(:without_object3) { create_object(project_id: another_project.id) } - let!(:with_project_and_object1) { create_object(project_id: project.id, lfs_object_id: lfs_object.id) } - let!(:with_project_and_object2) { create_object(project_id: project.id, lfs_object_id: another_lfs_object.id) } - let!(:with_project_and_object3) { create_object(project_id: another_project.id, lfs_object_id: another_lfs_object.id) } - let!(:without_project1) { create_object(lfs_object_id: lfs_object.id) } - let!(:without_project2) { create_object(lfs_object_id: another_lfs_object.id) } - let!(:without_project_and_object) { create_object } - - def create_object(project_id: non_existing_record_id, lfs_object_id: non_existing_record_id) - lfs_objects_project = nil - - ActiveRecord::Base.connection.disable_referential_integrity do - lfs_objects_project = lfs_objects_projects.create!(project_id: project_id, lfs_object_id: lfs_object_id) - end - - lfs_objects_project - end - - subject { described_class.new } - - describe '#perform' do - it 'lfs_objects_projects without an existing lfs object or project are removed' do - subject.perform(without_object1.id, without_object3.id) - - expect(lfs_objects_projects.all).to match_array( - [ - with_project_and_object1, with_project_and_object2, with_project_and_object3, - without_project1, without_project2, without_project_and_object - ]) - - subject.perform(with_project_and_object1.id, with_project_and_object3.id) - - expect(lfs_objects_projects.all).to match_array( - [ - with_project_and_object1, with_project_and_object2, with_project_and_object3, - without_project1, without_project2, without_project_and_object - ]) - - subject.perform(without_project1.id, without_project_and_object.id) - - expect(lfs_objects_projects.all).to match_array( - [ - with_project_and_object1, with_project_and_object2, with_project_and_object3 - ]) - - expect(lfs_objects.ids).to contain_exactly(lfs_object.id, another_lfs_object.id) - expect(projects.ids).to contain_exactly(project.id, another_project.id) - end - - it 'cache for affected projects is being reset' do - expect(ProjectCacheWorker).to receive(:bulk_perform_in) do |delay, args| - expect(delay).to eq(1.minute) - expect(args).to match_array([[project.id, [], [:lfs_objects_size]], [another_project.id, [], [:lfs_objects_size]]]) - end - - subject.perform(without_object1.id, with_project_and_object1.id) - - expect(ProjectCacheWorker).not_to receive(:bulk_perform_in) - - subject.perform(with_project_and_object1.id, with_project_and_object3.id) - - expect(ProjectCacheWorker).not_to receive(:bulk_perform_in) - - subject.perform(without_project1.id, without_project_and_object.id) - end - end -end diff --git a/spec/lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at_spec.rb b/spec/lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at_spec.rb new file mode 100644 index 00000000000..ade16c0a780 --- /dev/null +++ b/spec/lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at_spec.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::CleanupPersonalAccessTokensWithNilExpiresAt, schema: 20230510062503, feature_category: :system_access do # rubocop:disable Layout/LineLength + let(:personal_access_tokens_table) { table(:personal_access_tokens) } + let(:users_table) { table(:users) } + let(:expires_at_default) { described_class::EXPIRES_AT_DEFAULT } + + subject(:perform_migration) do + described_class.new( + start_id: 1, + end_id: 30, + batch_table: :personal_access_tokens, + batch_column: :id, + sub_batch_size: 3, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform + end + + before do + user = users_table.create!(name: 'PAT_USER', email: 'pat_user@gmail.com', username: "pat_user1", projects_limit: 0) + personal_access_tokens_table.create!(user_id: user.id, name: "PAT#1", expires_at: expires_at_default + 1.day) + personal_access_tokens_table.create!(user_id: user.id, name: "PAT#2", expires_at: nil) + personal_access_tokens_table.create!(user_id: user.id, name: "PAT#3", expires_at: Time.zone.now + 2.days) + end + + it 'adds expiry to personal access tokens', :aggregate_failures do + freeze_time do + expect(ActiveRecord::QueryRecorder.new { perform_migration }.count).to eq(3) + + expect(personal_access_tokens_table.find_by_name("PAT#1").expires_at).to eq(expires_at_default.to_date + 1.day) + expect(personal_access_tokens_table.find_by_name("PAT#2").expires_at).to eq(expires_at_default.to_date) + expect(personal_access_tokens_table.find_by_name("PAT#3").expires_at).to eq(Time.zone.now.to_date + 2.days) + end + end +end diff --git a/spec/lib/gitlab/background_migration/delete_orphaned_deployments_spec.rb b/spec/lib/gitlab/background_migration/delete_orphaned_deployments_spec.rb deleted file mode 100644 index 8f058c875a2..00000000000 --- a/spec/lib/gitlab/background_migration/delete_orphaned_deployments_spec.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::DeleteOrphanedDeployments, :migration, schema: 20210826171758 do - let!(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } - let!(:project) { table(:projects).create!(namespace_id: namespace.id) } - let!(:environment) { table(:environments).create!(name: 'production', slug: 'production', project_id: project.id) } - let(:background_migration_jobs) { table(:background_migration_jobs) } - - before do - create_deployment!(environment.id, project.id) - end - - it 'deletes only orphaned deployments' do - expect(valid_deployments.pluck(:id)).not_to be_empty - - subject.perform(table(:deployments).minimum(:id), table(:deployments).maximum(:id)) - - expect(valid_deployments.pluck(:id)).not_to be_empty - end - - it 'marks jobs as done' do - first_job = background_migration_jobs.create!( - class_name: 'DeleteOrphanedDeployments', - arguments: [table(:deployments).minimum(:id), table(:deployments).minimum(:id)] - ) - - subject.perform(table(:deployments).minimum(:id), table(:deployments).minimum(:id)) - - expect(first_job.reload.status).to eq(Gitlab::Database::BackgroundMigrationJob.statuses[:succeeded]) - end - - private - - def valid_deployments - table(:deployments).where('EXISTS (SELECT 1 FROM environments WHERE deployments.environment_id = environments.id)') - end - - def orphaned_deployments - table(:deployments).where('NOT EXISTS (SELECT 1 FROM environments WHERE deployments.environment_id = environments.id)') - end - - def create_deployment!(environment_id, project_id) - table(:deployments).create!( - environment_id: environment_id, - project_id: project_id, - ref: 'master', - tag: false, - sha: 'x', - status: 1, - iid: table(:deployments).count + 1) - end -end diff --git a/spec/lib/gitlab/background_migration/delete_orphaned_packages_dependencies_spec.rb b/spec/lib/gitlab/background_migration/delete_orphaned_packages_dependencies_spec.rb new file mode 100644 index 00000000000..0d82717c7de --- /dev/null +++ b/spec/lib/gitlab/background_migration/delete_orphaned_packages_dependencies_spec.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::DeleteOrphanedPackagesDependencies, schema: 20230303105806, + feature_category: :package_registry do + let!(:migration_attrs) do + { + start_id: 1, + end_id: 1000, + batch_table: :packages_dependencies, + batch_column: :id, + sub_batch_size: 500, + pause_ms: 0, + connection: ApplicationRecord.connection + } + end + + let!(:migration) { described_class.new(**migration_attrs) } + + let(:packages_dependencies) { table(:packages_dependencies) } + + let!(:namespace) { table(:namespaces).create!(name: 'project', path: 'project', type: 'Project') } + let!(:project) do + table(:projects).create!(name: 'project', path: 'project', project_namespace_id: namespace.id, + namespace_id: namespace.id) + end + + let!(:package) do + table(:packages_packages).create!(name: 'test', version: '1.2.3', package_type: 2, project_id: project.id) + end + + let!(:orphan_dependency_1) { packages_dependencies.create!(name: 'dependency 1', version_pattern: '~0.0.1') } + let!(:orphan_dependency_2) { packages_dependencies.create!(name: 'dependency 2', version_pattern: '~0.0.2') } + let!(:orphan_dependency_3) { packages_dependencies.create!(name: 'dependency 3', version_pattern: '~0.0.3') } + let!(:linked_dependency) do + packages_dependencies.create!(name: 'dependency 4', version_pattern: '~0.0.4').tap do |dependency| + table(:packages_dependency_links).create!(package_id: package.id, dependency_id: dependency.id, + dependency_type: 'dependencies') + end + end + + subject(:perform_migration) { migration.perform } + + it 'executes 3 queries' do + queries = ActiveRecord::QueryRecorder.new do + perform_migration + end + + expect(queries.count).to eq(3) + end + + it 'deletes only orphaned dependencies' do + expect { perform_migration }.to change { packages_dependencies.count }.by(-3) + expect(packages_dependencies.all).to eq([linked_dependency]) + end +end diff --git a/spec/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images_spec.rb b/spec/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images_spec.rb deleted file mode 100644 index e7b0471810d..00000000000 --- a/spec/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images_spec.rb +++ /dev/null @@ -1,142 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::DisableExpirationPoliciesLinkedToNoContainerImages, :migration, schema: 20220326161803 do # rubocop:disable Layout/LineLength - let!(:projects) { table(:projects) } - let!(:container_expiration_policies) { table(:container_expiration_policies) } - let!(:container_repositories) { table(:container_repositories) } - let!(:namespaces) { table(:namespaces) } - - let!(:namespace) { namespaces.create!(name: 'test', path: 'test') } - - let!(:policy1) { create_expiration_policy(project_id: 1, enabled: true) } - let!(:policy2) { create_expiration_policy(project_id: 2, enabled: false) } - let!(:policy3) { create_expiration_policy(project_id: 3, enabled: false) } - let!(:policy4) { create_expiration_policy(project_id: 4, enabled: true, with_images: true) } - let!(:policy5) { create_expiration_policy(project_id: 5, enabled: false, with_images: true) } - let!(:policy6) { create_expiration_policy(project_id: 6, enabled: false) } - let!(:policy7) { create_expiration_policy(project_id: 7, enabled: true) } - let!(:policy8) { create_expiration_policy(project_id: 8, enabled: true, with_images: true) } - let!(:policy9) { create_expiration_policy(project_id: 9, enabled: true) } - - describe '#perform' do - subject { described_class.new.perform(from_id, to_id) } - - shared_examples 'disabling policies with no images' do - it 'disables the proper policies' do - subject - - rows = container_expiration_policies.order(:project_id).to_h do |row| - [row.project_id, row.enabled] - end - expect(rows).to eq(expected_rows) - end - end - - context 'the whole range' do - let(:from_id) { 1 } - let(:to_id) { 9 } - - it_behaves_like 'disabling policies with no images' do - let(:expected_rows) do - { - 1 => false, - 2 => false, - 3 => false, - 4 => true, - 5 => false, - 6 => false, - 7 => false, - 8 => true, - 9 => false - } - end - end - end - - context 'a range with no policies to disable' do - let(:from_id) { 2 } - let(:to_id) { 6 } - - it_behaves_like 'disabling policies with no images' do - let(:expected_rows) do - { - 1 => true, - 2 => false, - 3 => false, - 4 => true, - 5 => false, - 6 => false, - 7 => true, - 8 => true, - 9 => true - } - end - end - end - - context 'a range with only images' do - let(:from_id) { 4 } - let(:to_id) { 5 } - - it_behaves_like 'disabling policies with no images' do - let(:expected_rows) do - { - 1 => true, - 2 => false, - 3 => false, - 4 => true, - 5 => false, - 6 => false, - 7 => true, - 8 => true, - 9 => true - } - end - end - end - - context 'a range with a single element' do - let(:from_id) { 9 } - let(:to_id) { 9 } - - it_behaves_like 'disabling policies with no images' do - let(:expected_rows) do - { - 1 => true, - 2 => false, - 3 => false, - 4 => true, - 5 => false, - 6 => false, - 7 => true, - 8 => true, - 9 => false - } - end - end - end - end - - def create_expiration_policy(project_id:, enabled:, with_images: false) - projects.create!(id: project_id, namespace_id: namespace.id, name: "gitlab-#{project_id}") - - if with_images - container_repositories.create!(project_id: project_id, name: "image-#{project_id}") - end - - container_expiration_policies.create!( - enabled: enabled, - project_id: project_id - ) - end - - def enabled_policies - container_expiration_policies.where(enabled: true) - end - - def disabled_policies - container_expiration_policies.where(enabled: false) - end -end diff --git a/spec/lib/gitlab/background_migration/drop_invalid_security_findings_spec.rb b/spec/lib/gitlab/background_migration/drop_invalid_security_findings_spec.rb deleted file mode 100644 index 5fdd8683d06..00000000000 --- a/spec/lib/gitlab/background_migration/drop_invalid_security_findings_spec.rb +++ /dev/null @@ -1,57 +0,0 @@ -# frozen_string_literal: true -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::DropInvalidSecurityFindings, :suppress_gitlab_schemas_validate_connection, - schema: 20211108211434 do - let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user', type: Namespaces::UserNamespace.sti_name) } - let(:project) { table(:projects).create!(namespace_id: namespace.id) } - - let(:pipelines) { table(:ci_pipelines) } - let!(:pipeline) { pipelines.create!(project_id: project.id) } - - let(:ci_builds) { table(:ci_builds) } - let!(:ci_build) { ci_builds.create! } - - let(:security_scans) { table(:security_scans) } - let!(:security_scan) do - security_scans.create!( - scan_type: 1, - status: 1, - build_id: ci_build.id, - project_id: project.id, - pipeline_id: pipeline.id - ) - end - - let(:vulnerability_scanners) { table(:vulnerability_scanners) } - let!(:vulnerability_scanner) { vulnerability_scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') } - - let(:security_findings) { table(:security_findings) } - let!(:security_finding_without_uuid) do - security_findings.create!( - severity: 1, - confidence: 1, - scan_id: security_scan.id, - scanner_id: vulnerability_scanner.id, - uuid: nil - ) - end - - let!(:security_finding_with_uuid) do - security_findings.create!( - severity: 1, - confidence: 1, - scan_id: security_scan.id, - scanner_id: vulnerability_scanner.id, - uuid: 'bd95c085-71aa-51d7-9bb6-08ae669c262e' - ) - end - - let(:sub_batch_size) { 10_000 } - - subject { described_class.new.perform(security_finding_without_uuid.id, security_finding_with_uuid.id, sub_batch_size) } - - it 'drops Security::Finding objects with no UUID' do - expect { subject }.to change(security_findings, :count).from(2).to(1) - end -end diff --git a/spec/lib/gitlab/background_migration/drop_invalid_vulnerabilities_spec.rb b/spec/lib/gitlab/background_migration/drop_invalid_vulnerabilities_spec.rb deleted file mode 100644 index 8f3ef44e00c..00000000000 --- a/spec/lib/gitlab/background_migration/drop_invalid_vulnerabilities_spec.rb +++ /dev/null @@ -1,126 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::DropInvalidVulnerabilities, schema: 20210826171758 do - let!(:background_migration_jobs) { table(:background_migration_jobs) } - let!(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } - let!(:users) { table(:users) } - let!(:user) { create_user! } - let!(:project) { table(:projects).create!(id: 123, namespace_id: namespace.id) } - - let!(:scanners) { table(:vulnerability_scanners) } - let!(:scanner) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') } - let!(:different_scanner) { scanners.create!(project_id: project.id, external_id: 'test 2', name: 'test scanner 2') } - - let!(:vulnerabilities) { table(:vulnerabilities) } - let!(:vulnerability_with_finding) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:vulnerability_without_finding) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:vulnerability_identifiers) { table(:vulnerability_identifiers) } - let!(:primary_identifier) do - vulnerability_identifiers.create!( - project_id: project.id, - external_type: 'uuid-v5', - external_id: 'uuid-v5', - fingerprint: '7e394d1b1eb461a7406d7b1e08f057a1cf11287a', - name: 'Identifier for UUIDv5') - end - - let!(:vulnerabilities_findings) { table(:vulnerability_occurrences) } - let!(:finding) do - create_finding!( - vulnerability_id: vulnerability_with_finding.id, - project_id: project.id, - scanner_id: scanner.id, - primary_identifier_id: primary_identifier.id - ) - end - - let(:succeeded_status) { 1 } - let(:pending_status) { 0 } - - it 'drops Vulnerabilities without any Findings' do - expect(vulnerabilities.pluck(:id)).to eq([vulnerability_with_finding.id, vulnerability_without_finding.id]) - - expect { subject.perform(vulnerability_with_finding.id, vulnerability_without_finding.id) }.to change(vulnerabilities, :count).by(-1) - - expect(vulnerabilities.pluck(:id)).to eq([vulnerability_with_finding.id]) - end - - it 'marks jobs as done' do - background_migration_jobs.create!( - class_name: 'DropInvalidVulnerabilities', - arguments: [vulnerability_with_finding.id, vulnerability_with_finding.id] - ) - - background_migration_jobs.create!( - class_name: 'DropInvalidVulnerabilities', - arguments: [vulnerability_without_finding.id, vulnerability_without_finding.id] - ) - - subject.perform(vulnerability_with_finding.id, vulnerability_with_finding.id) - - expect(background_migration_jobs.first.status).to eq(succeeded_status) - expect(background_migration_jobs.second.status).to eq(pending_status) - end - - private - - def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0) - vulnerabilities.create!( - project_id: project_id, - author_id: author_id, - title: title, - severity: severity, - confidence: confidence, - report_type: report_type - ) - end - - # rubocop:disable Metrics/ParameterLists - def create_finding!( - vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:, - name: "test", severity: 7, confidence: 7, report_type: 0, - project_fingerprint: '123qweasdzxc', location_fingerprint: 'test', - metadata_version: 'test', raw_metadata: 'test', uuid: SecureRandom.uuid) - vulnerabilities_findings.create!( - vulnerability_id: vulnerability_id, - project_id: project_id, - name: name, - severity: severity, - confidence: confidence, - report_type: report_type, - project_fingerprint: project_fingerprint, - scanner_id: scanner_id, - primary_identifier_id: primary_identifier_id, - location_fingerprint: location_fingerprint, - metadata_version: metadata_version, - raw_metadata: raw_metadata, - uuid: uuid - ) - end - # rubocop:enable Metrics/ParameterLists - - def create_user!(name: "Example User", email: "user@example.com", user_type: nil) - users.create!( - name: name, - email: email, - username: name, - projects_limit: 0, - user_type: user_type, - confirmed_at: Time.current - ) - end -end diff --git a/spec/lib/gitlab/background_migration/encrypt_ci_trigger_token_spec.rb b/spec/lib/gitlab/background_migration/encrypt_ci_trigger_token_spec.rb index b52f30a5e21..dd3e7877f8a 100644 --- a/spec/lib/gitlab/background_migration/encrypt_ci_trigger_token_spec.rb +++ b/spec/lib/gitlab/background_migration/encrypt_ci_trigger_token_spec.rb @@ -10,8 +10,7 @@ RSpec.describe Gitlab::BackgroundMigration::EncryptCiTriggerToken, feature_categ mode: :per_attribute_iv, key: ::Settings.attr_encrypted_db_key_base_32, algorithm: 'aes-256-gcm', - encode: false, - encode_iv: false + encode: false end end @@ -52,6 +51,7 @@ RSpec.describe Gitlab::BackgroundMigration::EncryptCiTriggerToken, feature_categ already_encrypted_token = Ci::Trigger.find(with_encryption.id) expect(already_encrypted_token.encrypted_token).to eq(with_encryption.encrypted_token) expect(already_encrypted_token.encrypted_token_iv).to eq(with_encryption.encrypted_token_iv) + expect(already_encrypted_token.token).to eq(already_encrypted_token.encrypted_token_tmp) expect(with_encryption.token).to eq(with_encryption.encrypted_token_tmp) end end diff --git a/spec/lib/gitlab/background_migration/extract_project_topics_into_separate_table_spec.rb b/spec/lib/gitlab/background_migration/extract_project_topics_into_separate_table_spec.rb deleted file mode 100644 index 586e75ffb37..00000000000 --- a/spec/lib/gitlab/background_migration/extract_project_topics_into_separate_table_spec.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::ExtractProjectTopicsIntoSeparateTable, - :suppress_gitlab_schemas_validate_connection, schema: 20210826171758 do - it 'correctly extracts project topics into separate table' do - namespaces = table(:namespaces) - projects = table(:projects) - taggings = table(:taggings) - tags = table(:tags) - project_topics = table(:project_topics) - topics = table(:topics) - - namespace = namespaces.create!(name: 'foo', path: 'foo') - project = projects.create!(namespace_id: namespace.id) - tag_1 = tags.create!(name: 'Topic1') - tag_2 = tags.create!(name: 'Topic2') - tag_3 = tags.create!(name: 'Topic3') - topic_3 = topics.create!(name: 'Topic3') - tagging_1 = taggings.create!(taggable_type: 'Project', taggable_id: project.id, context: 'topics', tag_id: tag_1.id) - tagging_2 = taggings.create!(taggable_type: 'Project', taggable_id: project.id, context: 'topics', tag_id: tag_2.id) - other_tagging = taggings.create!(taggable_type: 'Other', taggable_id: project.id, context: 'topics', tag_id: tag_1.id) - tagging_3 = taggings.create!(taggable_type: 'Project', taggable_id: project.id, context: 'topics', tag_id: tag_3.id) - tagging_4 = taggings.create!(taggable_type: 'Project', taggable_id: -1, context: 'topics', tag_id: tag_1.id) - tagging_5 = taggings.create!(taggable_type: 'Project', taggable_id: project.id, context: 'topics', tag_id: -1) - - subject.perform(tagging_1.id, tagging_5.id) - - # Tagging records - expect { tagging_1.reload }.to raise_error(ActiveRecord::RecordNotFound) - expect { tagging_2.reload }.to raise_error(ActiveRecord::RecordNotFound) - expect { other_tagging.reload }.not_to raise_error - expect { tagging_3.reload }.to raise_error(ActiveRecord::RecordNotFound) - expect { tagging_4.reload }.to raise_error(ActiveRecord::RecordNotFound) - expect { tagging_5.reload }.to raise_error(ActiveRecord::RecordNotFound) - - # Topic records - topic_1 = topics.find_by(name: 'Topic1') - topic_2 = topics.find_by(name: 'Topic2') - expect(topics.all).to contain_exactly(topic_1, topic_2, topic_3) - - # ProjectTopic records - expect(project_topics.all.map(&:topic_id)).to contain_exactly(topic_1.id, topic_2.id, topic_3.id) - end -end diff --git a/spec/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at_spec.rb b/spec/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at_spec.rb deleted file mode 100644 index 7f15aceca42..00000000000 --- a/spec/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at_spec.rb +++ /dev/null @@ -1,166 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require Rails.root.join('db', 'post_migrate', '20211004110500_add_temporary_index_to_issue_metrics.rb') - -RSpec.describe Gitlab::BackgroundMigration::FixFirstMentionedInCommitAt, :migration, schema: 20211004110500 do - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:users) { table(:users) } - let(:merge_requests) { table(:merge_requests) } - let(:issues) { table(:issues) } - let(:issue_metrics) { table(:issue_metrics) } - let(:merge_requests_closing_issues) { table(:merge_requests_closing_issues) } - let(:diffs) { table(:merge_request_diffs) } - let(:ten_days_ago) { 10.days.ago } - let(:commits) do - table(:merge_request_diff_commits).tap do |t| - t.extend(SuppressCompositePrimaryKeyWarning) - end - end - - let(:namespace) { namespaces.create!(name: 'ns', path: 'ns') } - let(:project) { projects.create!(namespace_id: namespace.id) } - - let!(:issue1) do - issues.create!( - title: 'issue', - description: 'description', - project_id: project.id - ) - end - - let!(:issue2) do - issues.create!( - title: 'issue', - description: 'description', - project_id: project.id - ) - end - - let!(:merge_request1) do - merge_requests.create!( - source_branch: 'a', - target_branch: 'master', - target_project_id: project.id - ) - end - - let!(:merge_request2) do - merge_requests.create!( - source_branch: 'b', - target_branch: 'master', - target_project_id: project.id - ) - end - - let!(:merge_request_closing_issue1) do - merge_requests_closing_issues.create!(issue_id: issue1.id, merge_request_id: merge_request1.id) - end - - let!(:merge_request_closing_issue2) do - merge_requests_closing_issues.create!(issue_id: issue2.id, merge_request_id: merge_request2.id) - end - - let!(:diff1) { diffs.create!(merge_request_id: merge_request1.id) } - let!(:diff2) { diffs.create!(merge_request_id: merge_request1.id) } - - let!(:other_diff) { diffs.create!(merge_request_id: merge_request2.id) } - - let!(:commit1) do - commits.create!( - merge_request_diff_id: diff2.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('aaa'), - authored_date: 5.days.ago - ) - end - - let!(:commit2) do - commits.create!( - merge_request_diff_id: diff2.id, - relative_order: 1, - sha: Gitlab::Database::ShaAttribute.serialize('aaa'), - authored_date: 10.days.ago - ) - end - - let!(:commit3) do - commits.create!( - merge_request_diff_id: other_diff.id, - relative_order: 1, - sha: Gitlab::Database::ShaAttribute.serialize('aaa'), - authored_date: 5.days.ago - ) - end - - def run_migration - described_class - .new - .perform(issue_metrics.minimum(:issue_id), issue_metrics.maximum(:issue_id)) - end - - shared_examples 'fixes first_mentioned_in_commit_at' do - it "marks successful slices as completed" do - min_issue_id = issue_metrics.minimum(:issue_id) - max_issue_id = issue_metrics.maximum(:issue_id) - - expect(subject).to receive(:mark_job_as_succeeded).with(min_issue_id, max_issue_id) - - subject.perform(min_issue_id, max_issue_id) - end - - context 'when the persisted first_mentioned_in_commit_at is later than the first commit authored_date' do - it 'updates the issue_metrics record' do - record1 = issue_metrics.create!(issue_id: issue1.id, first_mentioned_in_commit_at: Time.current) - record2 = issue_metrics.create!(issue_id: issue2.id, first_mentioned_in_commit_at: Time.current) - - run_migration - record1.reload - record2.reload - - expect(record1.first_mentioned_in_commit_at).to be_within(2.seconds).of(commit2.authored_date) - expect(record2.first_mentioned_in_commit_at).to be_within(2.seconds).of(commit3.authored_date) - end - end - - context 'when the persisted first_mentioned_in_commit_at is earlier than the first commit authored_date' do - it 'does not update the issue_metrics record' do - record = issue_metrics.create!(issue_id: issue1.id, first_mentioned_in_commit_at: 20.days.ago) - - expect { run_migration }.not_to change { record.reload.first_mentioned_in_commit_at } - end - end - - context 'when the first_mentioned_in_commit_at is null' do - it 'does nothing' do - record = issue_metrics.create!(issue_id: issue1.id, first_mentioned_in_commit_at: nil) - - expect { run_migration }.not_to change { record.reload.first_mentioned_in_commit_at } - end - end - end - - describe 'running the migration when first_mentioned_in_commit_at is timestamp without time zone' do - it_behaves_like 'fixes first_mentioned_in_commit_at' - end - - describe 'running the migration when first_mentioned_in_commit_at is timestamp with time zone' do - around do |example| - AddTemporaryIndexToIssueMetrics.new.down - - ActiveRecord::Base.connection.execute "ALTER TABLE issue_metrics ALTER first_mentioned_in_commit_at type timestamp with time zone" - Gitlab::BackgroundMigration::FixFirstMentionedInCommitAt::TmpIssueMetrics.reset_column_information - AddTemporaryIndexToIssueMetrics.new.up - - example.run - - AddTemporaryIndexToIssueMetrics.new.down - ActiveRecord::Base.connection.execute "ALTER TABLE issue_metrics ALTER first_mentioned_in_commit_at type timestamp without time zone" - Gitlab::BackgroundMigration::FixFirstMentionedInCommitAt::TmpIssueMetrics.reset_column_information - AddTemporaryIndexToIssueMetrics.new.up - end - - it_behaves_like 'fixes first_mentioned_in_commit_at' - end -end diff --git a/spec/lib/gitlab/background_migration/fix_merge_request_diff_commit_users_spec.rb b/spec/lib/gitlab/background_migration/fix_merge_request_diff_commit_users_spec.rb deleted file mode 100644 index 99df21562b0..00000000000 --- a/spec/lib/gitlab/background_migration/fix_merge_request_diff_commit_users_spec.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -# rubocop: disable RSpec/FactoriesInMigrationSpecs -RSpec.describe Gitlab::BackgroundMigration::FixMergeRequestDiffCommitUsers do - let(:migration) { described_class.new } - - describe '#perform' do - context 'when the project exists' do - it 'does nothing' do - project = create(:project) - - expect { migration.perform(project.id) }.not_to raise_error - end - end - - context 'when the project does not exist' do - it 'does nothing' do - expect { migration.perform(-1) }.not_to raise_error - end - end - end -end -# rubocop: enable RSpec/FactoriesInMigrationSpecs diff --git a/spec/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues_spec.rb b/spec/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues_spec.rb new file mode 100644 index 00000000000..9f431c43f39 --- /dev/null +++ b/spec/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues_spec.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::FixVulnerabilityReadsHasIssues, schema: 20230302185739, feature_category: :vulnerability_management do # rubocop:disable Layout/LineLength + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:users) { table(:users) } + let(:scanners) { table(:vulnerability_scanners) } + let(:vulnerabilities) { table(:vulnerabilities) } + let(:vulnerability_reads) { table(:vulnerability_reads) } + let(:work_item_types) { table(:work_item_types) } + let(:issues) { table(:issues) } + let(:vulnerability_issue_links) { table(:vulnerability_issue_links) } + + let(:namespace) { namespaces.create!(name: 'user', path: 'user') } + let(:project) { projects.create!(namespace_id: namespace.id, project_namespace_id: namespace.id) } + let(:user) { users.create!(username: 'john_doe', email: 'johndoe@gitlab.com', projects_limit: 10) } + let(:scanner) { scanners.create!(project_id: project.id, external_id: 'external_id', name: 'Test Scanner') } + let(:work_item_type) { work_item_types.create!(name: 'test') } + + let(:vulnerability_records) do + Array.new(4).map do |_, n| + vulnerabilities.create!( + project_id: project.id, + author_id: user.id, + title: "vulnerability #{n}", + severity: 1, + confidence: 1, + report_type: 1 + ) + end + end + + let(:vulnerabilities_with_issues) { [vulnerability_records.first, vulnerability_records.third] } + let(:vulnerabilities_without_issues) { vulnerability_records - vulnerabilities_with_issues } + + let(:vulnerability_read_records) do + vulnerability_records.map do |vulnerability| + vulnerability_reads.create!( + project_id: project.id, + vulnerability_id: vulnerability.id, + scanner_id: scanner.id, + has_issues: false, + severity: 1, + report_type: 1, + state: 1, + uuid: SecureRandom.uuid + ) + end + end + + let!(:issue_links) do + vulnerabilities_with_issues.map do |vulnerability| + issue = issues.create!( + title: vulnerability.title, + author_id: user.id, + project_id: project.id, + confidential: true, + work_item_type_id: work_item_type.id, + namespace_id: namespace.id + ) + + vulnerability_issue_links.create!( + vulnerability_id: vulnerability.id, + issue_id: issue.id + ) + end + end + + def vulnerability_read_for(vulnerability) + vulnerability_read_records.find { |read| read.vulnerability_id == vulnerability.id } + end + + subject(:perform_migration) do + described_class.new( + start_id: issue_links.first.vulnerability_id, + end_id: issue_links.last.vulnerability_id, + batch_table: :vulnerability_issue_links, + batch_column: :vulnerability_id, + sub_batch_size: issue_links.size, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform + end + + it 'only changes records with issue links' do + expect(vulnerability_read_records).to all(have_attributes(has_issues: false)) + + perform_migration + + vulnerabilities_with_issues.each do |vulnerability| + expect(vulnerability_read_for(vulnerability).reload.has_issues).to eq(true) + end + + vulnerabilities_without_issues.each do |vulnerability| + expect(vulnerability_read_for(vulnerability).reload.has_issues).to eq(false) + end + end +end diff --git a/spec/lib/gitlab/background_migration/issues_internal_id_scope_updater_spec.rb b/spec/lib/gitlab/background_migration/issues_internal_id_scope_updater_spec.rb new file mode 100644 index 00000000000..1adff322b41 --- /dev/null +++ b/spec/lib/gitlab/background_migration/issues_internal_id_scope_updater_spec.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +require 'spec_helper' +# this needs the schema to be before we introduce the not null constraint on routes#namespace_id +# rubocop:disable RSpec/MultipleMemoizedHelpers +RSpec.describe Gitlab::BackgroundMigration::IssuesInternalIdScopeUpdater, feature_category: :team_planning do + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:internal_ids) { table(:internal_ids) } + + let(:gr1) { namespaces.create!(name: 'batchtest1', type: 'Group', path: 'space1') } + let(:gr2) { namespaces.create!(name: 'batchtest2', type: 'Group', parent_id: gr1.id, path: 'space2') } + + let(:pr_nmsp1) { namespaces.create!(name: 'proj1', path: 'proj1', type: 'Project', parent_id: gr1.id) } + let(:pr_nmsp2) { namespaces.create!(name: 'proj2', path: 'proj2', type: 'Project', parent_id: gr1.id) } + let(:pr_nmsp3) { namespaces.create!(name: 'proj3', path: 'proj3', type: 'Project', parent_id: gr2.id) } + let(:pr_nmsp4) { namespaces.create!(name: 'proj4', path: 'proj4', type: 'Project', parent_id: gr2.id) } + let(:pr_nmsp5) { namespaces.create!(name: 'proj5', path: 'proj5', type: 'Project', parent_id: gr2.id) } + let(:pr_nmsp6) { namespaces.create!(name: 'proj6', path: 'proj6', type: 'Project', parent_id: gr2.id) } + + # rubocop:disable Layout/LineLength + let(:p1) { projects.create!(name: 'proj1', path: 'proj1', namespace_id: gr1.id, project_namespace_id: pr_nmsp1.id) } + let(:p2) { projects.create!(name: 'proj2', path: 'proj2', namespace_id: gr1.id, project_namespace_id: pr_nmsp2.id) } + let(:p3) { projects.create!(name: 'proj3', path: 'proj3', namespace_id: gr2.id, project_namespace_id: pr_nmsp3.id) } + let(:p4) { projects.create!(name: 'proj4', path: 'proj4', namespace_id: gr2.id, project_namespace_id: pr_nmsp4.id) } + let(:p5) { projects.create!(name: 'proj5', path: 'proj5', namespace_id: gr2.id, project_namespace_id: pr_nmsp5.id) } + let(:p6) { projects.create!(name: 'proj6', path: 'proj6', namespace_id: gr2.id, project_namespace_id: pr_nmsp6.id) } + # rubocop:enable Layout/LineLength + + # a project that already is covered by a record for its namespace. This should result in no new record added and + # project related record deleted + let!(:issues_internal_ids_p1) { internal_ids.create!(project_id: p1.id, usage: 0, last_value: 100) } + let!(:issues_internal_ids_pr_nmsp1) { internal_ids.create!(namespace_id: pr_nmsp1.id, usage: 0, last_value: 111) } + + # project records that do not have a corresponding namespace record. This should result 2 new records + # scoped to corresponding project namespaces being added and the project related records being deleted. + let!(:issues_internal_ids_p2) { internal_ids.create!(project_id: p2.id, usage: 0, last_value: 200) } + let!(:issues_internal_ids_p3) { internal_ids.create!(project_id: p3.id, usage: 0, last_value: 300) } + + # a project record on a different usage, should not be affected by the migration and + # no new record should be created for this case + let!(:issues_internal_ids_p4) { internal_ids.create!(project_id: p4.id, usage: 4, last_value: 400) } + + # a project namespace scoped record without a corresponding project record, should not affect anything. + let!(:issues_internal_ids_pr_nmsp5) { internal_ids.create!(namespace_id: pr_nmsp5.id, usage: 0, last_value: 500) } + + # a record scoped to a group, should not affect anything. + let!(:issues_internal_ids_gr1) { internal_ids.create!(namespace_id: gr1.id, usage: 0, last_value: 600) } + + # a project that is covered by a record for its namespace, but has a higher last_value, due to updates during rolling + # deploy for instance, see https://gitlab.com/gitlab-com/gl-infra/production/-/issues/8548 + let!(:issues_internal_ids_p6) { internal_ids.create!(project_id: p6.id, usage: 0, last_value: 111) } + let!(:issues_internal_ids_pr_nmsp6) { internal_ids.create!(namespace_id: pr_nmsp6.id, usage: 0, last_value: 100) } + + subject(:perform_migration) do + described_class.new( + start_id: internal_ids.minimum(:id), + end_id: internal_ids.maximum(:id), + batch_table: :internal_ids, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform + end + + it 'backfills internal_ids records and removes related project records', :aggregate_failures do + perform_migration + + expected_recs = [pr_nmsp1.id, pr_nmsp2.id, pr_nmsp3.id, pr_nmsp5.id, gr1.id, pr_nmsp6.id] + + # all namespace scoped records for issues(0) usage + expect(internal_ids.where.not(namespace_id: nil).where(usage: 0).count).to eq(6) + # all namespace_ids for issues(0) usage + expect(internal_ids.where.not(namespace_id: nil).where(usage: 0).pluck(:namespace_id)).to match_array(expected_recs) + # this is the record with usage: 4 + expect(internal_ids.where.not(project_id: nil).count).to eq(1) + # no project scoped records for issues usage left + expect(internal_ids.where.not(project_id: nil).where(usage: 0).count).to eq(0) + + # the case when the project_id scoped record had the higher last_value, + # see `issues_internal_ids_p6` and issues_internal_ids_pr_nmsp6 definitions above + expect(internal_ids.where(namespace_id: pr_nmsp6.id).first.last_value).to eq(111) + + # the case when the namespace_id scoped record had the higher last_value, + # see `issues_internal_ids_p1` and issues_internal_ids_pr_nmsp1 definitions above. + expect(internal_ids.where(namespace_id: pr_nmsp1.id).first.last_value).to eq(111) + end +end +# rubocop:enable RSpec/MultipleMemoizedHelpers diff --git a/spec/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings_spec.rb b/spec/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings_spec.rb new file mode 100644 index 00000000000..ba2f571f5aa --- /dev/null +++ b/spec/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings_spec.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::MigrateEvidencesForVulnerabilityFindings, + feature_category: :vulnerability_management do + let(:vulnerability_occurrences) { table(:vulnerability_occurrences) } + let(:vulnerability_finding_evidences) { table(:vulnerability_finding_evidences) } + let(:evidence_hash) { { url: 'http://test.com' } } + let(:namespace1) { table(:namespaces).create!(name: 'namespace 1', path: 'namespace1') } + let(:project1) { table(:projects).create!(namespace_id: namespace1.id, project_namespace_id: namespace1.id) } + let(:user) { table(:users).create!(email: 'test1@example.com', projects_limit: 5) } + + let(:scanner1) do + table(:vulnerability_scanners).create!(project_id: project1.id, external_id: 'test 1', name: 'test scanner 1') + end + + let(:stating_id) { vulnerability_occurrences.pluck(:id).min } + let(:end_id) { vulnerability_occurrences.pluck(:id).max } + + let(:migration) do + described_class.new( + start_id: stating_id, + end_id: end_id, + batch_table: :vulnerability_occurrences, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 2, + connection: ApplicationRecord.connection + ) + end + + subject(:perform_migration) { migration.perform } + + context 'without the presence of evidence key' do + before do + create_finding!(project1.id, scanner1.id, { other_keys: 'test' }) + end + + it 'does not create any evidence' do + expect { perform_migration }.not_to change { vulnerability_finding_evidences.count } + end + end + + context 'with evidence equals to nil' do + before do + create_finding!(project1.id, scanner1.id, { evidence: nil }) + end + + it 'does not create any evidence' do + expect { perform_migration }.not_to change { vulnerability_finding_evidences.count } + end + end + + context 'with existing evidence within raw_metadata' do + let!(:finding1) { create_finding!(project1.id, scanner1.id, { evidence: evidence_hash }) } + let!(:finding2) { create_finding!(project1.id, scanner1.id, { evidence: evidence_hash }) } + + it 'creates new evidence for each finding' do + expect { perform_migration }.to change { vulnerability_finding_evidences.count }.by(2) + end + + context 'when parse throws exception JSON::ParserError' do + before do + allow(Gitlab::Json).to receive(:parse).and_raise(JSON::ParserError) + end + + it 'does not create new records' do + expect { perform_migration }.not_to change { vulnerability_finding_evidences.count } + end + end + end + + context 'with unsupported Unicode escape sequence' do + let!(:finding1) { create_finding!(project1.id, scanner1.id, { evidence: { 'summary' => "\u0000" } }) } + + it 'does not create new evidence' do + expect { perform_migration }.not_to change { vulnerability_finding_evidences.count } + end + end + + context 'with existing evidence records' do + let!(:finding) { create_finding!(project1.id, scanner1.id, { evidence: evidence_hash }) } + + before do + vulnerability_finding_evidences.create!(vulnerability_occurrence_id: finding.id, data: evidence_hash) + end + + it 'does not create new evidence' do + expect { perform_migration }.not_to change { vulnerability_finding_evidences.count } + end + + context 'with non-existing evidence' do + let!(:finding3) { create_finding!(project1.id, scanner1.id, { evidence: { url: 'http://secondary.com' } }) } + + it 'creates a new evidence only to the non-existing evidence' do + expect { perform_migration }.to change { vulnerability_finding_evidences.count }.by(1) + end + end + end + + private + + def create_finding!(project_id, scanner_id, raw_metadata) + vulnerability = table(:vulnerabilities).create!(project_id: project_id, author_id: user.id, title: 'test', + severity: 4, confidence: 4, report_type: 0) + + identifier = table(:vulnerability_identifiers).create!(project_id: project_id, external_type: 'uuid-v5', + external_id: 'uuid-v5', fingerprint: OpenSSL::Digest::SHA256.hexdigest(vulnerability.id.to_s), + name: 'Identifier for UUIDv5 2 2') + + table(:vulnerability_occurrences).create!( + vulnerability_id: vulnerability.id, project_id: project_id, scanner_id: scanner_id, + primary_identifier_id: identifier.id, name: 'test', severity: 4, confidence: 4, report_type: 0, + uuid: SecureRandom.uuid, project_fingerprint: '123qweasdzxc', location: { "image" => "alpine:3.4" }, + location_fingerprint: 'test', metadata_version: 'test', + raw_metadata: raw_metadata.to_json) + end +end diff --git a/spec/lib/gitlab/background_migration/migrate_human_user_type_spec.rb b/spec/lib/gitlab/background_migration/migrate_human_user_type_spec.rb new file mode 100644 index 00000000000..7edeaed5794 --- /dev/null +++ b/spec/lib/gitlab/background_migration/migrate_human_user_type_spec.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::MigrateHumanUserType, schema: 20230327103401, feature_category: :user_management do # rubocop:disable Layout/LineLength + let!(:valid_users) do + # 13 is the max value we have at the moment. + (0..13).map do |type| + table(:users).create!(username: "user#{type}", email: "user#{type}@test.com", user_type: type, projects_limit: 0) + end + end + + let!(:user_to_update) do + table(:users).create!(username: "user_nil", email: "user_nil@test.com", user_type: nil, projects_limit: 0) + end + + let(:starting_id) { table(:users).pluck(:id).min } + let(:end_id) { table(:users).pluck(:id).max } + + let(:migration) do + described_class.new( + start_id: starting_id, + end_id: end_id, + batch_table: :users, + batch_column: :id, + sub_batch_size: 100, + pause_ms: 2, + connection: ::ApplicationRecord.connection + ) + end + + describe 'perform' do + it 'updates user with `nil` user type only' do + expect do + migration.perform + valid_users.map(&:reload) + user_to_update.reload + end.not_to change { valid_users.map(&:user_type) } + + expect(user_to_update.user_type).to eq(0) + end + end +end diff --git a/spec/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings_spec.rb b/spec/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings_spec.rb new file mode 100644 index 00000000000..b973f9d4350 --- /dev/null +++ b/spec/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings_spec.rb @@ -0,0 +1,192 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::MigrateLinksForVulnerabilityFindings, + feature_category: :vulnerability_management do + let(:vulnerability_occurrences) { table(:vulnerability_occurrences) } + let(:vulnerability_finding_links) { table(:vulnerability_finding_links) } + let(:link_hash) { { url: 'http://test.com' } } + let(:namespace1) { table(:namespaces).create!(name: 'namespace 1', path: 'namespace1') } + let(:project1) { table(:projects).create!(namespace_id: namespace1.id, project_namespace_id: namespace1.id) } + let(:user) { table(:users).create!(email: 'test1@example.com', projects_limit: 5) } + + let(:scanner1) do + table(:vulnerability_scanners).create!(project_id: project1.id, external_id: 'test 1', name: 'test scanner 1') + end + + let(:stating_id) { vulnerability_occurrences.pluck(:id).min } + let(:end_id) { vulnerability_occurrences.pluck(:id).max } + + let(:migration) do + described_class.new( + start_id: stating_id, + end_id: end_id, + batch_table: :vulnerability_occurrences, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 2, + connection: ApplicationRecord.connection + ) + end + + subject(:perform_migration) { migration.perform } + + context 'without the presence of links key' do + before do + create_finding!(project1.id, scanner1.id, { other_keys: 'test' }) + end + + it 'does not create any link' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + end + + context 'with links equals to an array of nil element' do + before do + create_finding!(project1.id, scanner1.id, { links: [nil] }) + end + + it 'does not create any link' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + end + + context 'with links equals to a string' do + before do + create_finding!(project1.id, scanner1.id, { links: "wrong format" }) + end + + it 'does not create any link' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + end + + context 'with some elements which do not contain the key url' do + let!(:finding) do + create_finding!(project1.id, scanner1.id, { links: [link_hash, "wrong format", {}] }) + end + + it 'creates links only to valid elements' do + expect(Gitlab::AppLogger).not_to receive(:error) + + perform_migration + + expect(vulnerability_finding_links.all).to contain_exactly(have_attributes( + url: link_hash[:url], + vulnerability_occurrence_id: finding.id)) + end + end + + context 'when link name is too long' do + let!(:finding) do + create_finding!(project1.id, scanner1.id, { links: [{ name: 'A' * 300, url: 'https://foo' }] }) + end + + it 'skips creation of link and logs error' do + expect(Gitlab::AppLogger).to receive(:error).with({ + class: described_class.name, + message: /check_55f0a95439/, + model_id: finding.id + }) + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + end + + context 'when link url is too long' do + let!(:finding) do + create_finding!(project1.id, scanner1.id, { links: [{ url: "https://f#{'o' * 2050}" }] }) + end + + it 'skips creation of link and logs error' do + expect(Gitlab::AppLogger).to receive(:error).with({ + class: described_class.name, + message: /check_b7fe886df6/, + model_id: finding.id + }) + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + end + + context 'with links equals to an array of duplicated elements' do + let!(:finding) do + create_finding!(project1.id, scanner1.id, { links: [link_hash, link_hash] }) + end + + it 'creates one new link' do + expect(Gitlab::AppLogger).not_to receive(:error) + + perform_migration + + expect(vulnerability_finding_links.all).to contain_exactly(have_attributes( + url: link_hash[:url], + vulnerability_occurrence_id: finding.id)) + end + end + + context 'with existing links within raw_metadata' do + let!(:finding1) { create_finding!(project1.id, scanner1.id, { links: [link_hash] }) } + let!(:finding2) { create_finding!(project1.id, scanner1.id, { links: [link_hash] }) } + + it 'creates new link for each finding' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.to change { vulnerability_finding_links.count }.by(2) + end + end + + context 'when Gitlab::Json throws exception JSON::ParserError' do + before do + create_finding!(project1.id, scanner1.id, { links: [link_hash] }) + allow(Gitlab::Json).to receive(:parse).and_raise(JSON::ParserError) + end + + it 'does not log this error nor create new records' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + end + + context 'with existing link records' do + let!(:finding) { create_finding!(project1.id, scanner1.id, { links: [link_hash] }) } + + before do + vulnerability_finding_links.create!(vulnerability_occurrence_id: finding.id, url: link_hash[:url]) + end + + it 'does not create new link' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + + it 'does not raise ActiveRecord::RecordNotUnique' do + expect { perform_migration }.not_to raise_error + end + end + + private + + def create_finding!(project_id, scanner_id, raw_metadata) + vulnerability = table(:vulnerabilities).create!(project_id: project_id, author_id: user.id, title: 'test', + severity: 4, confidence: 4, report_type: 0) + + identifier = table(:vulnerability_identifiers).create!(project_id: project_id, external_type: 'uuid-v5', + external_id: 'uuid-v5', fingerprint: OpenSSL::Digest::SHA256.hexdigest(vulnerability.id.to_s), + name: 'Identifier for UUIDv5 2 2') + + table(:vulnerability_occurrences).create!( + vulnerability_id: vulnerability.id, project_id: project_id, scanner_id: scanner_id, + primary_identifier_id: identifier.id, name: 'test', severity: 4, confidence: 4, report_type: 0, + uuid: SecureRandom.uuid, project_fingerprint: '123qweasdzxc', location: { "image" => "alpine:3.4" }, + location_fingerprint: 'test', metadata_version: 'test', + raw_metadata: raw_metadata.to_json) + end +end diff --git a/spec/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users_spec.rb b/spec/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users_spec.rb deleted file mode 100644 index c3ae2cc060c..00000000000 --- a/spec/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users_spec.rb +++ /dev/null @@ -1,413 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::MigrateMergeRequestDiffCommitUsers, schema: 20211012134316 do - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:users) { table(:users) } - let(:merge_requests) { table(:merge_requests) } - let(:diffs) { table(:merge_request_diffs) } - let(:commits) do - table(:merge_request_diff_commits).tap do |t| - t.extend(SuppressCompositePrimaryKeyWarning) - end - end - - let(:commit_users) { described_class::MergeRequestDiffCommitUser } - - let(:namespace) { namespaces.create!(name: 'foo', path: 'foo') } - let(:project) { projects.create!(namespace_id: namespace.id) } - let(:merge_request) do - merge_requests.create!( - source_branch: 'x', - target_branch: 'master', - target_project_id: project.id - ) - end - - let(:diff) { diffs.create!(merge_request_id: merge_request.id) } - let(:migration) { described_class.new } - - describe 'MergeRequestDiffCommit' do - describe '.each_row_to_migrate' do - it 'yields the rows to migrate for a given range' do - commit1 = commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc'), - author_name: 'bob', - author_email: 'bob@example.com', - committer_name: 'bob', - committer_email: 'bob@example.com' - ) - - commit2 = commits.create!( - merge_request_diff_id: diff.id, - relative_order: 1, - sha: Gitlab::Database::ShaAttribute.serialize('123abc'), - author_name: 'Alice', - author_email: 'alice@example.com', - committer_name: 'Alice', - committer_email: 'alice@example.com' - ) - - # We stub this constant to make sure we run at least two pagination - # queries for getting the data. This way we can test if the pagination - # is actually working properly. - stub_const( - 'Gitlab::BackgroundMigration::MigrateMergeRequestDiffCommitUsers::COMMIT_ROWS_PER_QUERY', - 1 - ) - - rows = [] - - described_class::MergeRequestDiffCommit.each_row_to_migrate(diff.id, diff.id + 1) do |row| - rows << row - end - - expect(rows.length).to eq(2) - - expect(rows[0].author_name).to eq(commit1.author_name) - expect(rows[1].author_name).to eq(commit2.author_name) - end - end - end - - describe 'MergeRequestDiffCommitUser' do - describe '.union' do - it 'produces a union of the given queries' do - alice = commit_users.create!(name: 'Alice', email: 'alice@example.com') - bob = commit_users.create!(name: 'Bob', email: 'bob@example.com') - users = commit_users.union( - [ - commit_users.where(name: 'Alice').to_sql, - commit_users.where(name: 'Bob').to_sql - ]) - - expect(users).to include(alice) - expect(users).to include(bob) - end - end - end - - describe '#perform' do - it 'skips jobs that have already been completed' do - Gitlab::Database::BackgroundMigrationJob.create!( - class_name: 'MigrateMergeRequestDiffCommitUsers', - arguments: [1, 10], - status: :succeeded - ) - - expect(migration).not_to receive(:get_data_to_update) - - migration.perform(1, 10) - end - - it 'migrates the data in the range' do - commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc'), - author_name: 'bob', - author_email: 'bob@example.com', - committer_name: 'bob', - committer_email: 'bob@example.com' - ) - - migration.perform(diff.id, diff.id + 1) - - bob = commit_users.find_by(name: 'bob') - commit = commits.first - - expect(commit.commit_author_id).to eq(bob.id) - expect(commit.committer_id).to eq(bob.id) - end - - it 'treats empty names and Emails the same as NULL values' do - commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc'), - author_name: 'bob', - author_email: 'bob@example.com', - committer_name: '', - committer_email: '' - ) - - migration.perform(diff.id, diff.id + 1) - - bob = commit_users.find_by(name: 'bob') - commit = commits.first - - expect(commit.commit_author_id).to eq(bob.id) - expect(commit.committer_id).to be_nil - end - - it 'does not update rows without a committer and author' do - commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc') - ) - - migration.perform(diff.id, diff.id + 1) - - commit = commits.first - - expect(commit_users.count).to eq(0) - expect(commit.commit_author_id).to be_nil - expect(commit.committer_id).to be_nil - end - - it 'marks the background job as done' do - Gitlab::Database::BackgroundMigrationJob.create!( - class_name: 'MigrateMergeRequestDiffCommitUsers', - arguments: [diff.id, diff.id + 1] - ) - - migration.perform(diff.id, diff.id + 1) - - job = Gitlab::Database::BackgroundMigrationJob.first - - expect(job.status).to eq('succeeded') - end - end - - describe '#get_data_to_update' do - it 'returns the users and commit rows to update' do - commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc'), - author_name: 'bob' + ('a' * 510), - author_email: 'bob@example.com', - committer_name: 'bob' + ('a' * 510), - committer_email: 'bob@example.com' - ) - - commits.create!( - merge_request_diff_id: diff.id, - relative_order: 1, - sha: Gitlab::Database::ShaAttribute.serialize('456abc'), - author_name: 'alice', - author_email: 'alice@example.com', - committer_name: 'alice', - committer_email: 'alice@example.com' - ) - - users, to_update = migration.get_data_to_update(diff.id, diff.id + 1) - - bob_name = 'bob' + ('a' * 509) - - expect(users).to include(%w[alice alice@example.com]) - expect(users).to include([bob_name, 'bob@example.com']) - - expect(to_update[[diff.id, 0]]) - .to eq([[bob_name, 'bob@example.com'], [bob_name, 'bob@example.com']]) - - expect(to_update[[diff.id, 1]]) - .to eq([%w[alice alice@example.com], %w[alice alice@example.com]]) - end - - it 'does not include a user if both the name and Email are missing' do - commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc'), - author_name: nil, - author_email: nil, - committer_name: 'bob', - committer_email: 'bob@example.com' - ) - - users, _ = migration.get_data_to_update(diff.id, diff.id + 1) - - expect(users).to eq([%w[bob bob@example.com]].to_set) - end - end - - describe '#get_user_rows_in_batches' do - it 'retrieves all existing users' do - alice = commit_users.create!(name: 'alice', email: 'alice@example.com') - bob = commit_users.create!(name: 'bob', email: 'bob@example.com') - - users = [[alice.name, alice.email], [bob.name, bob.email]] - mapping = {} - - migration.get_user_rows_in_batches(users, mapping) - - expect(mapping[%w[alice alice@example.com]]).to eq(alice) - expect(mapping[%w[bob bob@example.com]]).to eq(bob) - end - end - - describe '#create_missing_users' do - it 'creates merge request diff commit users that are missing' do - alice = commit_users.create!(name: 'alice', email: 'alice@example.com') - users = [%w[alice alice@example.com], %w[bob bob@example.com]] - mapping = { %w[alice alice@example.com] => alice } - - migration.create_missing_users(users, mapping) - - expect(mapping[%w[alice alice@example.com]]).to eq(alice) - expect(mapping[%w[bob bob@example.com]].name).to eq('bob') - expect(mapping[%w[bob bob@example.com]].email).to eq('bob@example.com') - end - end - - describe '#update_commit_rows' do - it 'updates the merge request diff commit rows' do - to_update = { [42, 0] => [%w[alice alice@example.com], []] } - user_mapping = { %w[alice alice@example.com] => double(:user, id: 1) } - - expect(migration) - .to receive(:bulk_update_commit_rows) - .with({ [42, 0] => [1, nil] }) - - migration.update_commit_rows(to_update, user_mapping) - end - end - - describe '#bulk_update_commit_rows' do - context 'when there are no authors and committers' do - it 'does not update any rows' do - migration.bulk_update_commit_rows({ [1, 0] => [] }) - - expect(described_class::MergeRequestDiffCommit.connection) - .not_to receive(:execute) - end - end - - context 'when there are only authors' do - it 'only updates the author IDs' do - author = commit_users.create!(name: 'Alice', email: 'alice@example.com') - commit = commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc') - ) - - mapping = { - [commit.merge_request_diff_id, commit.relative_order] => - [author.id, nil] - } - - migration.bulk_update_commit_rows(mapping) - - commit = commits.first - - expect(commit.commit_author_id).to eq(author.id) - expect(commit.committer_id).to be_nil - end - end - - context 'when there are only committers' do - it 'only updates the committer IDs' do - committer = - commit_users.create!(name: 'Alice', email: 'alice@example.com') - - commit = commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc') - ) - - mapping = { - [commit.merge_request_diff_id, commit.relative_order] => - [nil, committer.id] - } - - migration.bulk_update_commit_rows(mapping) - - commit = commits.first - - expect(commit.committer_id).to eq(committer.id) - expect(commit.commit_author_id).to be_nil - end - end - - context 'when there are both authors and committers' do - it 'updates both the author and committer IDs' do - author = commit_users.create!(name: 'Bob', email: 'bob@example.com') - committer = - commit_users.create!(name: 'Alice', email: 'alice@example.com') - - commit = commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc') - ) - - mapping = { - [commit.merge_request_diff_id, commit.relative_order] => - [author.id, committer.id] - } - - migration.bulk_update_commit_rows(mapping) - - commit = commits.first - - expect(commit.commit_author_id).to eq(author.id) - expect(commit.committer_id).to eq(committer.id) - end - end - - context 'when there are multiple commit rows to update' do - it 'updates all the rows' do - author = commit_users.create!(name: 'Bob', email: 'bob@example.com') - committer = - commit_users.create!(name: 'Alice', email: 'alice@example.com') - - commit1 = commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc') - ) - - commit2 = commits.create!( - merge_request_diff_id: diff.id, - relative_order: 1, - sha: Gitlab::Database::ShaAttribute.serialize('456abc') - ) - - mapping = { - [commit1.merge_request_diff_id, commit1.relative_order] => - [author.id, committer.id], - - [commit2.merge_request_diff_id, commit2.relative_order] => - [author.id, nil] - } - - migration.bulk_update_commit_rows(mapping) - - commit1 = commits.find_by(relative_order: 0) - commit2 = commits.find_by(relative_order: 1) - - expect(commit1.commit_author_id).to eq(author.id) - expect(commit1.committer_id).to eq(committer.id) - - expect(commit2.commit_author_id).to eq(author.id) - expect(commit2.committer_id).to be_nil - end - end - end - - describe '#primary_key' do - it 'returns the primary key for the commits table' do - key = migration.primary_key - - expect(key.to_sql).to eq('("merge_request_diff_commits"."merge_request_diff_id", "merge_request_diff_commits"."relative_order")') - end - end - - describe '#prepare' do - it 'trims a value to at most 512 characters' do - expect(migration.prepare('€' * 1_000)).to eq('€' * 512) - end - - it 'returns nil if the value is an empty string' do - expect(migration.prepare('')).to be_nil - end - end -end diff --git a/spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb b/spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb deleted file mode 100644 index b252df4ecff..00000000000 --- a/spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::MigrateProjectTaggingsContextFromTagsToTopics, - :suppress_gitlab_schemas_validate_connection, schema: 20210826171758 do - it 'correctly migrates project taggings context from tags to topics' do - taggings = table(:taggings) - - project_old_tagging_1 = taggings.create!(taggable_type: 'Project', context: 'tags') - project_new_tagging_1 = taggings.create!(taggable_type: 'Project', context: 'topics') - project_other_context_tagging_1 = taggings.create!(taggable_type: 'Project', context: 'other') - project_old_tagging_2 = taggings.create!(taggable_type: 'Project', context: 'tags') - project_old_tagging_3 = taggings.create!(taggable_type: 'Project', context: 'tags') - - subject.perform(project_old_tagging_1.id, project_old_tagging_2.id) - - project_old_tagging_1.reload - project_new_tagging_1.reload - project_other_context_tagging_1.reload - project_old_tagging_2.reload - project_old_tagging_3.reload - - expect(project_old_tagging_1.context).to eq('topics') - expect(project_new_tagging_1.context).to eq('topics') - expect(project_other_context_tagging_1.context).to eq('other') - expect(project_old_tagging_2.context).to eq('topics') - expect(project_old_tagging_3.context).to eq('tags') - end -end diff --git a/spec/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings_spec.rb b/spec/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings_spec.rb new file mode 100644 index 00000000000..b75c0e61b19 --- /dev/null +++ b/spec/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings_spec.rb @@ -0,0 +1,173 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::MigrateRemediationsForVulnerabilityFindings, + feature_category: :vulnerability_management do + let(:vulnerability_occurrences) { table(:vulnerability_occurrences) } + let(:vulnerability_findings_remediations) { table(:vulnerability_findings_remediations) } + let(:vulnerability_remediations) { table(:vulnerability_remediations) } + let(:remediation_hash) { { summary: 'summary', diff: "ZGlmZiAtLWdp" } } + let(:namespace1) { table(:namespaces).create!(name: 'namespace 1', path: 'namespace1') } + let(:project1) { table(:projects).create!(namespace_id: namespace1.id, project_namespace_id: namespace1.id) } + let(:user) { table(:users).create!(email: 'test1@example.com', projects_limit: 5) } + + let(:scanner1) do + table(:vulnerability_scanners).create!(project_id: project1.id, external_id: 'test 1', name: 'test scanner 1') + end + + let(:stating_id) { vulnerability_occurrences.pluck(:id).min } + let(:end_id) { vulnerability_occurrences.pluck(:id).max } + + let(:migration) do + described_class.new( + start_id: stating_id, + end_id: end_id, + batch_table: :vulnerability_occurrences, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 2, + connection: ApplicationRecord.connection + ) + end + + subject(:perform_migration) { migration.perform } + + context 'without the presence of remediation key' do + before do + create_finding!(project1.id, scanner1.id, { other_keys: 'test' }) + end + + it 'does not create any remediation' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_remediations.count } + end + end + + context 'with remediation equals to an array of nil element' do + before do + create_finding!(project1.id, scanner1.id, { remediations: [nil] }) + end + + it 'does not create any remediation' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_remediations.count } + end + end + + context 'with remediation with empty string as the diff key' do + let!(:finding) do + create_finding!(project1.id, scanner1.id, { remediations: [{ summary: 'summary', diff: '' }] }) + end + + it 'does not create any remediation' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_remediations.count } + end + end + + context 'with remediation equals to an array of duplicated elements' do + let!(:finding) do + create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash, remediation_hash] }) + end + + it 'creates new remediation' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.to change { vulnerability_remediations.count }.by(1) + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding.id).length).to eq(1) + end + end + + context 'with existing remediations within raw_metadata' do + let!(:finding1) { create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash] }) } + let!(:finding2) { create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash] }) } + + it 'creates new remediation' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.to change { vulnerability_remediations.count }.by(1) + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding1.id).length).to eq(1) + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding2.id).length).to eq(1) + end + + context 'when create throws exception other than ActiveRecord::RecordNotUnique' do + before do + allow(migration).to receive(:create_finding_remediations).and_raise(StandardError) + end + + it 'rolls back all related transactions' do + expect(Gitlab::AppLogger).to receive(:error).with({ + class: described_class.name, message: StandardError.to_s, model_id: finding1.id + }) + expect(Gitlab::AppLogger).to receive(:error).with({ + class: described_class.name, message: StandardError.to_s, model_id: finding2.id + }) + expect { perform_migration }.not_to change { vulnerability_remediations.count } + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding1.id).length).to eq(0) + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding2.id).length).to eq(0) + end + end + end + + context 'with existing remediation records' do + let!(:finding) { create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash] }) } + + before do + vulnerability_remediations.create!(project_id: project1.id, summary: remediation_hash[:summary], + checksum: checksum(remediation_hash[:diff]), file: Tempfile.new.path) + end + + it 'does not create new remediation' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_remediations.count } + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding.id).length).to eq(1) + end + end + + context 'with same raw_metadata for different projects' do + let(:namespace2) { table(:namespaces).create!(name: 'namespace 2', path: 'namespace2') } + let(:project2) { table(:projects).create!(namespace_id: namespace2.id, project_namespace_id: namespace2.id) } + let(:scanner2) do + table(:vulnerability_scanners).create!(project_id: project2.id, external_id: 'test 2', name: 'test scanner 2') + end + + let!(:finding1) { create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash] }) } + let!(:finding2) { create_finding!(project2.id, scanner2.id, { remediations: [remediation_hash] }) } + + it 'creates new remediation for each project' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.to change { vulnerability_remediations.count }.by(2) + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding1.id).length).to eq(1) + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding2.id).length).to eq(1) + end + end + + private + + def create_finding!(project_id, scanner_id, raw_metadata) + vulnerability = table(:vulnerabilities).create!(project_id: project_id, author_id: user.id, title: 'test', + severity: 4, confidence: 4, report_type: 0) + + identifier = table(:vulnerability_identifiers).create!(project_id: project_id, external_type: 'uuid-v5', + external_id: 'uuid-v5', fingerprint: OpenSSL::Digest::SHA256.hexdigest(vulnerability.id.to_s), + name: 'Identifier for UUIDv5 2 2') + + table(:vulnerability_occurrences).create!( + vulnerability_id: vulnerability.id, project_id: project_id, scanner_id: scanner_id, + primary_identifier_id: identifier.id, name: 'test', severity: 4, confidence: 4, report_type: 0, + uuid: SecureRandom.uuid, project_fingerprint: '123qweasdzxc', location: { "image" => "alpine:3.4" }, + location_fingerprint: 'test', metadata_version: 'test', + raw_metadata: raw_metadata.to_json) + end + + def checksum(value) + sha = Digest::SHA256.hexdigest(value) + Gitlab::Database::ShaAttribute.new.serialize(sha) + end +end diff --git a/spec/lib/gitlab/background_migration/migrate_u2f_webauthn_spec.rb b/spec/lib/gitlab/background_migration/migrate_u2f_webauthn_spec.rb deleted file mode 100644 index 08fde0d0ff4..00000000000 --- a/spec/lib/gitlab/background_migration/migrate_u2f_webauthn_spec.rb +++ /dev/null @@ -1,67 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -require 'webauthn/u2f_migrator' - -RSpec.describe Gitlab::BackgroundMigration::MigrateU2fWebauthn, :migration, schema: 20210826171758 do - let(:users) { table(:users) } - - let(:user) { users.create!(email: 'email@email.com', name: 'foo', username: 'foo', projects_limit: 0) } - - let(:u2f_registrations) { table(:u2f_registrations) } - let(:webauthn_registrations) { table(:webauthn_registrations) } - - let!(:u2f_registration_not_migrated) { create_u2f_registration(1, 'reg1') } - let!(:u2f_registration_not_migrated_no_name) { create_u2f_registration(2, nil, 2) } - let!(:u2f_registration_migrated) { create_u2f_registration(3, 'reg3') } - - subject { described_class.new.perform(1, 3) } - - before do - converted_credential = convert_credential_for(u2f_registration_migrated) - webauthn_registrations.create!(converted_credential) - end - - it 'migrates all records' do - expect { subject }.to change { webauthn_registrations.count }.from(1).to(3) - - all_webauthn_registrations = webauthn_registrations.all.map(&:attributes) - - [u2f_registration_not_migrated, u2f_registration_not_migrated_no_name].each do |u2f_registration| - expected_credential = convert_credential_for(u2f_registration).except(:created_at).stringify_keys - expect(all_webauthn_registrations).to include(a_hash_including(expected_credential)) - end - end - - def create_u2f_registration(id, name, counter = 5) - device = U2F::FakeU2F.new(FFaker::BaconIpsum.characters(5)) - u2f_registrations.create!({ id: id, - certificate: Base64.strict_encode64(device.cert_raw), - key_handle: U2F.urlsafe_encode64(device.key_handle_raw), - public_key: Base64.strict_encode64(device.origin_public_key_raw), - counter: counter, - name: name, - user_id: user.id }) - end - - def convert_credential_for(u2f_registration) - converted_credential = WebAuthn::U2fMigrator.new( - app_id: Gitlab.config.gitlab.url, - certificate: u2f_registration.certificate, - key_handle: u2f_registration.key_handle, - public_key: u2f_registration.public_key, - counter: u2f_registration.counter - ).credential - - { - credential_xid: Base64.strict_encode64(converted_credential.id), - public_key: Base64.strict_encode64(converted_credential.public_key), - counter: u2f_registration.counter, - name: u2f_registration.name || '', - user_id: u2f_registration.user_id, - u2f_registration_id: u2f_registration.id, - created_at: u2f_registration.created_at - } - end -end diff --git a/spec/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature_spec.rb b/spec/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature_spec.rb deleted file mode 100644 index 71cf58a933f..00000000000 --- a/spec/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature_spec.rb +++ /dev/null @@ -1,98 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::MoveContainerRegistryEnabledToProjectFeature, :migration, schema: 20210826171758 do - let(:enabled) { 20 } - let(:disabled) { 0 } - - let(:namespaces) { table(:namespaces) } - let(:project_features) { table(:project_features) } - let(:projects) { table(:projects) } - - let(:namespace) { namespaces.create!(name: 'user', path: 'user') } - let!(:project1) { projects.create!(namespace_id: namespace.id) } - let!(:project2) { projects.create!(namespace_id: namespace.id) } - let!(:project3) { projects.create!(namespace_id: namespace.id) } - let!(:project4) { projects.create!(namespace_id: namespace.id) } - - # pages_access_level cannot be null. - let(:non_null_project_features) { { pages_access_level: enabled } } - let!(:project_feature1) { project_features.create!(project_id: project1.id, **non_null_project_features) } - let!(:project_feature2) { project_features.create!(project_id: project2.id, **non_null_project_features) } - let!(:project_feature3) { project_features.create!(project_id: project3.id, **non_null_project_features) } - - describe '#perform' do - before do - project1.update!(container_registry_enabled: true) - project2.update!(container_registry_enabled: false) - project3.update!(container_registry_enabled: nil) - project4.update!(container_registry_enabled: true) - end - - it 'copies values to project_features' do - table(:background_migration_jobs).create!( - class_name: 'MoveContainerRegistryEnabledToProjectFeature', - arguments: [project1.id, project4.id] - ) - table(:background_migration_jobs).create!( - class_name: 'MoveContainerRegistryEnabledToProjectFeature', - arguments: [-1, -3] - ) - - expect(project1.container_registry_enabled).to eq(true) - expect(project2.container_registry_enabled).to eq(false) - expect(project3.container_registry_enabled).to eq(nil) - expect(project4.container_registry_enabled).to eq(true) - - expect(project_feature1.container_registry_access_level).to eq(disabled) - expect(project_feature2.container_registry_access_level).to eq(disabled) - expect(project_feature3.container_registry_access_level).to eq(disabled) - - expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |logger| - expect(logger).to receive(:info) - .with(message: "#{described_class}: Copied container_registry_enabled values for projects with IDs between #{project1.id}..#{project4.id}") - - expect(logger).not_to receive(:info) - end - - subject.perform(project1.id, project4.id) - - expect(project1.reload.container_registry_enabled).to eq(true) - expect(project2.reload.container_registry_enabled).to eq(false) - expect(project3.reload.container_registry_enabled).to eq(nil) - expect(project4.container_registry_enabled).to eq(true) - - expect(project_feature1.reload.container_registry_access_level).to eq(enabled) - expect(project_feature2.reload.container_registry_access_level).to eq(disabled) - expect(project_feature3.reload.container_registry_access_level).to eq(disabled) - - expect(table(:background_migration_jobs).first.status).to eq(1) # succeeded - expect(table(:background_migration_jobs).second.status).to eq(0) # pending - end - - context 'when no projects exist in range' do - it 'does not fail' do - expect(project1.container_registry_enabled).to eq(true) - expect(project_feature1.container_registry_access_level).to eq(disabled) - - expect { subject.perform(-1, -2) }.not_to raise_error - - expect(project1.container_registry_enabled).to eq(true) - expect(project_feature1.container_registry_access_level).to eq(disabled) - end - end - - context 'when projects in range all have nil container_registry_enabled' do - it 'does not fail' do - expect(project3.container_registry_enabled).to eq(nil) - expect(project_feature3.container_registry_access_level).to eq(disabled) - - expect { subject.perform(project3.id, project3.id) }.not_to raise_error - - expect(project3.container_registry_enabled).to eq(nil) - expect(project_feature3.container_registry_access_level).to eq(disabled) - end - end - end -end diff --git a/spec/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects_spec.rb b/spec/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects_spec.rb index a8574411957..f671a673a08 100644 --- a/spec/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects_spec.rb +++ b/spec/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects_spec.rb @@ -2,7 +2,8 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::NullifyCreatorIdColumnOfOrphanedProjects, feature_category: :projects do +RSpec.describe Gitlab::BackgroundMigration::NullifyCreatorIdColumnOfOrphanedProjects, feature_category: :projects, + schema: 20230130073109 do let(:users) { table(:users) } let(:projects) { table(:projects) } let(:namespaces) { table(:namespaces) } diff --git a/spec/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds_spec.rb b/spec/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds_spec.rb index 2f0eef3c399..5b234679e22 100644 --- a/spec/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds_spec.rb +++ b/spec/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::NullifyOrphanRunnerIdOnCiBuilds, - :suppress_gitlab_schemas_validate_connection, migration: :gitlab_ci, schema: 20220223112304 do + :suppress_gitlab_schemas_validate_connection, migration: :gitlab_ci, schema: 20220223112304 do let(:namespaces) { table(:namespaces) } let(:projects) { table(:projects) } let(:ci_runners) { table(:ci_runners) } diff --git a/spec/lib/gitlab/background_migration/populate_topics_total_projects_count_cache_spec.rb b/spec/lib/gitlab/background_migration/populate_topics_total_projects_count_cache_spec.rb deleted file mode 100644 index 8e07b43f5b9..00000000000 --- a/spec/lib/gitlab/background_migration/populate_topics_total_projects_count_cache_spec.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::PopulateTopicsTotalProjectsCountCache, schema: 20211006060436 do - it 'correctly populates total projects count cache' do - namespaces = table(:namespaces) - projects = table(:projects) - topics = table(:topics) - project_topics = table(:project_topics) - - group = namespaces.create!(name: 'group', path: 'group') - project_1 = projects.create!(namespace_id: group.id) - project_2 = projects.create!(namespace_id: group.id) - project_3 = projects.create!(namespace_id: group.id) - topic_1 = topics.create!(name: 'Topic1') - topic_2 = topics.create!(name: 'Topic2') - topic_3 = topics.create!(name: 'Topic3') - topic_4 = topics.create!(name: 'Topic4') - - project_topics.create!(project_id: project_1.id, topic_id: topic_1.id) - project_topics.create!(project_id: project_1.id, topic_id: topic_3.id) - project_topics.create!(project_id: project_2.id, topic_id: topic_3.id) - project_topics.create!(project_id: project_1.id, topic_id: topic_4.id) - project_topics.create!(project_id: project_2.id, topic_id: topic_4.id) - project_topics.create!(project_id: project_3.id, topic_id: topic_4.id) - - subject.perform(topic_1.id, topic_4.id) - - expect(topic_1.reload.total_projects_count).to eq(1) - expect(topic_2.reload.total_projects_count).to eq(0) - expect(topic_3.reload.total_projects_count).to eq(2) - expect(topic_4.reload.total_projects_count).to eq(3) - end -end diff --git a/spec/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields_spec.rb b/spec/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields_spec.rb new file mode 100644 index 00000000000..50380247c9f --- /dev/null +++ b/spec/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields_spec.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::PopulateVulnerabilityDismissalFields, schema: 20230412185837, feature_category: :vulnerability_management do # rubocop:disable Layout/LineLength + let(:users) { table(:users) } + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:vulnerabilities) { table(:vulnerabilities) } + let(:findings) { table(:vulnerability_occurrences) } + let(:scanners) { table(:vulnerability_scanners) } + let(:identifiers) { table(:vulnerability_identifiers) } + let(:feedback) { table(:vulnerability_feedback) } + + let(:user) { users.create!(name: 'test', email: 'test@example.com', projects_limit: 5) } + let(:namespace) { namespaces.create!(name: 'gitlab', path: 'gitlab-org') } + let(:project) { projects.create!(namespace_id: namespace.id, name: 'foo', project_namespace_id: namespace.id) } + let(:vulnerability_1) do + vulnerabilities.create!(title: 'title', state: 2, severity: 0, + confidence: 5, report_type: 2, project_id: project.id, author_id: user.id + ) + end + + let(:vulnerability_2) do + vulnerabilities.create!(title: 'title', state: 2, severity: 0, + confidence: 5, report_type: 2, project_id: project.id, author_id: user.id + ) + end + + let(:scanner) { scanners.create!(project_id: project.id, external_id: 'foo', name: 'bar') } + let(:identifier) do + identifiers.create!(project_id: project.id, fingerprint: 'foo', + external_type: 'bar', external_id: 'zoo', name: 'identifier' + ) + end + + let(:uuid) { SecureRandom.uuid } + + before do + feedback.create!(feedback_type: 0, + category: 'sast', + project_fingerprint: '418291a26024a1445b23fe64de9380cdcdfd1fa8', + project_id: project.id, + author_id: user.id, + created_at: Time.current, + finding_uuid: uuid + ) + + findings.create!(name: 'Finding', + report_type: 'sast', + project_fingerprint: '418291a26024a1445b23fe64de9380cdcdfd1f98', + location_fingerprint: 'bar', + severity: 1, + confidence: 1, + metadata_version: 1, + raw_metadata: '', + details: {}, + uuid: uuid, + project_id: project.id, + vulnerability_id: vulnerability_1.id, + scanner_id: scanner.id, + primary_identifier_id: identifier.id + ) + + allow(::Gitlab::BackgroundMigration::Logger).to receive_messages(info: true, warn: true, error: true) + end + + subject do + described_class.new( + start_id: vulnerability_1.id, + end_id: vulnerability_2.id, + batch_table: :vulnerabilities, + batch_column: :id, + sub_batch_size: 200, + pause_ms: 2.minutes, + connection: ApplicationRecord.connection + ) + end + + describe '#perform' do + it 'updates the missing dismissal information of the vulnerability' do + expect { subject.perform }.to change { vulnerability_1.reload.dismissed_at } + .from(nil) + .and change { vulnerability_1.reload.dismissed_by_id }.from(nil).to(user.id) + end + + it 'writes log messages', :aggregate_failures do + subject.perform + + expect(::Gitlab::BackgroundMigration::Logger).to have_received(:info).with(migrator: described_class.name, + message: 'Dismissal information has been copied', + count: 2 + ) + expect(::Gitlab::BackgroundMigration::Logger).to have_received(:warn).with(migrator: described_class.name, + message: 'Could not update vulnerability!', + vulnerability_id: vulnerability_2.id + ) + end + + context 'when logger throws exception StandardError' do + before do + allow(::Gitlab::BackgroundMigration::Logger).to receive(:warn).and_raise(StandardError) + end + + it 'logs StandardError' do + expect(::Gitlab::BackgroundMigration::Logger).to receive(:error).with({ + migrator: described_class.name, message: StandardError.to_s, vulnerability_id: vulnerability_2.id + }) + + subject.perform + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/prune_stale_project_export_jobs_spec.rb b/spec/lib/gitlab/background_migration/prune_stale_project_export_jobs_spec.rb index 5150d0ea4b0..3446b9f0676 100644 --- a/spec/lib/gitlab/background_migration/prune_stale_project_export_jobs_spec.rb +++ b/spec/lib/gitlab/background_migration/prune_stale_project_export_jobs_spec.rb @@ -10,14 +10,15 @@ RSpec.describe Gitlab::BackgroundMigration::PruneStaleProjectExportJobs, feature let(:uploads) { table(:project_relation_export_uploads) } subject(:perform_migration) do - described_class.new(start_id: 1, - end_id: 300, - batch_table: :project_export_jobs, - batch_column: :id, - sub_batch_size: 2, - pause_ms: 0, - connection: ActiveRecord::Base.connection) - .perform + described_class.new( + start_id: 1, + end_id: 300, + batch_table: :project_export_jobs, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform end it 'removes export jobs and associated relations older than 7 days' do diff --git a/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb b/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb deleted file mode 100644 index 2271bbfb2f3..00000000000 --- a/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb +++ /dev/null @@ -1,530 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -def create_background_migration_job(ids, status) - proper_status = case status - when :pending - Gitlab::Database::BackgroundMigrationJob.statuses['pending'] - when :succeeded - Gitlab::Database::BackgroundMigrationJob.statuses['succeeded'] - else - raise ArgumentError - end - - background_migration_jobs.create!( - class_name: 'RecalculateVulnerabilitiesOccurrencesUuid', - arguments: Array(ids), - status: proper_status, - created_at: Time.now.utc - ) -end - -RSpec.describe Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid, :suppress_gitlab_schemas_validate_connection, schema: 20211124132705 do - let(:background_migration_jobs) { table(:background_migration_jobs) } - let(:pending_jobs) { background_migration_jobs.where(status: Gitlab::Database::BackgroundMigrationJob.statuses['pending']) } - let(:succeeded_jobs) { background_migration_jobs.where(status: Gitlab::Database::BackgroundMigrationJob.statuses['succeeded']) } - let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } - let(:users) { table(:users) } - let(:user) { create_user! } - let(:project) { table(:projects).create!(id: 123, namespace_id: namespace.id) } - let(:scanners) { table(:vulnerability_scanners) } - let(:scanner) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') } - let(:scanner2) { scanners.create!(project_id: project.id, external_id: 'test 2', name: 'test scanner 2') } - let(:vulnerabilities) { table(:vulnerabilities) } - let(:vulnerability_findings) { table(:vulnerability_occurrences) } - let(:vulnerability_finding_pipelines) { table(:vulnerability_occurrence_pipelines) } - let(:vulnerability_finding_signatures) { table(:vulnerability_finding_signatures) } - let(:vulnerability_identifiers) { table(:vulnerability_identifiers) } - - let(:identifier_1) { 'identifier-1' } - let!(:vulnerability_identifier) do - vulnerability_identifiers.create!( - project_id: project.id, - external_type: identifier_1, - external_id: identifier_1, - fingerprint: Gitlab::Database::ShaAttribute.serialize('ff9ef548a6e30a0462795d916f3f00d1e2b082ca'), - name: 'Identifier 1') - end - - let(:identifier_2) { 'identifier-2' } - let!(:vulnerability_identfier2) do - vulnerability_identifiers.create!( - project_id: project.id, - external_type: identifier_2, - external_id: identifier_2, - fingerprint: Gitlab::Database::ShaAttribute.serialize('4299e8ddd819f9bde9cfacf45716724c17b5ddf7'), - name: 'Identifier 2') - end - - let(:identifier_3) { 'identifier-3' } - let!(:vulnerability_identifier3) do - vulnerability_identifiers.create!( - project_id: project.id, - external_type: identifier_3, - external_id: identifier_3, - fingerprint: Gitlab::Database::ShaAttribute.serialize('8e91632f9c6671e951834a723ee221c44cc0d844'), - name: 'Identifier 3') - end - - let(:known_uuid_v4) { "b3cc2518-5446-4dea-871c-89d5e999c1ac" } - let(:known_uuid_v5) { "05377088-dc26-5161-920e-52a7159fdaa1" } - let(:desired_uuid_v5) { "f3e9a23f-9181-54bf-a5ab-c5bc7a9b881a" } - - subject { described_class.new.perform(start_id, end_id) } - - context "when finding has a UUIDv4" do - before do - @uuid_v4 = create_finding!( - vulnerability_id: nil, - project_id: project.id, - scanner_id: scanner2.id, - primary_identifier_id: vulnerability_identfier2.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize("fa18f432f1d56675f4098d318739c3cd5b14eb3e"), - uuid: known_uuid_v4 - ) - end - - let(:start_id) { @uuid_v4.id } - let(:end_id) { @uuid_v4.id } - - it "replaces it with UUIDv5" do - expect(vulnerability_findings.pluck(:uuid)).to match_array([known_uuid_v4]) - - subject - - expect(vulnerability_findings.pluck(:uuid)).to match_array([desired_uuid_v5]) - end - - it 'logs recalculation' do - expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |instance| - expect(instance).to receive(:info).twice - end - - subject - end - end - - context "when finding has a UUIDv5" do - before do - @uuid_v5 = create_finding!( - vulnerability_id: nil, - project_id: project.id, - scanner_id: scanner.id, - primary_identifier_id: vulnerability_identifier.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize("838574be0210968bf6b9f569df9c2576242cbf0a"), - uuid: known_uuid_v5 - ) - end - - let(:start_id) { @uuid_v5.id } - let(:end_id) { @uuid_v5.id } - - it "stays the same" do - expect(vulnerability_findings.pluck(:uuid)).to match_array([known_uuid_v5]) - - subject - - expect(vulnerability_findings.pluck(:uuid)).to match_array([known_uuid_v5]) - end - end - - context 'if a duplicate UUID would be generated' do # rubocop: disable RSpec/MultipleMemoizedHelpers - let(:v1) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:finding_with_incorrect_uuid) do - create_finding!( - vulnerability_id: v1.id, - project_id: project.id, - scanner_id: scanner.id, - primary_identifier_id: vulnerability_identifier.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: 'bd95c085-71aa-51d7-9bb6-08ae669c262e' - ) - end - - let(:v2) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:finding_with_correct_uuid) do - create_finding!( - vulnerability_id: v2.id, - project_id: project.id, - primary_identifier_id: vulnerability_identifier.id, - scanner_id: scanner2.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: '91984483-5efe-5215-b471-d524ac5792b1' - ) - end - - let(:v3) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:finding_with_incorrect_uuid2) do - create_finding!( - vulnerability_id: v3.id, - project_id: project.id, - scanner_id: scanner.id, - primary_identifier_id: vulnerability_identfier2.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: '00000000-1111-2222-3333-444444444444' - ) - end - - let(:v4) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:finding_with_correct_uuid2) do - create_finding!( - vulnerability_id: v4.id, - project_id: project.id, - scanner_id: scanner2.id, - primary_identifier_id: vulnerability_identfier2.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: '1edd751e-ef9a-5391-94db-a832c8635bfc' - ) - end - - let!(:finding_with_incorrect_uuid3) do - create_finding!( - vulnerability_id: nil, - project_id: project.id, - scanner_id: scanner.id, - primary_identifier_id: vulnerability_identifier3.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: '22222222-3333-4444-5555-666666666666' - ) - end - - let!(:duplicate_not_in_the_same_batch) do - create_finding!( - id: 99999, - vulnerability_id: nil, - project_id: project.id, - scanner_id: scanner2.id, - primary_identifier_id: vulnerability_identifier3.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: '4564f9d5-3c6b-5cc3-af8c-7c25285362a7' - ) - end - - let(:start_id) { finding_with_incorrect_uuid.id } - let(:end_id) { finding_with_incorrect_uuid3.id } - - before do - 4.times do - create_finding_pipeline!(project_id: project.id, finding_id: finding_with_incorrect_uuid.id) - create_finding_pipeline!(project_id: project.id, finding_id: finding_with_correct_uuid.id) - create_finding_pipeline!(project_id: project.id, finding_id: finding_with_incorrect_uuid2.id) - create_finding_pipeline!(project_id: project.id, finding_id: finding_with_correct_uuid2.id) - end - end - - it 'drops duplicates and related records', :aggregate_failures do - expect(vulnerability_findings.pluck(:id)).to match_array( - [ - finding_with_correct_uuid.id, - finding_with_incorrect_uuid.id, - finding_with_correct_uuid2.id, - finding_with_incorrect_uuid2.id, - finding_with_incorrect_uuid3.id, - duplicate_not_in_the_same_batch.id - ]) - - expect { subject }.to change(vulnerability_finding_pipelines, :count).from(16).to(8) - .and change(vulnerability_findings, :count).from(6).to(3) - .and change(vulnerabilities, :count).from(4).to(2) - - expect(vulnerability_findings.pluck(:id)).to match_array([finding_with_incorrect_uuid.id, finding_with_incorrect_uuid2.id, finding_with_incorrect_uuid3.id]) - end - - context 'if there are conflicting UUID values within the batch' do # rubocop: disable RSpec/MultipleMemoizedHelpers - let(:end_id) { finding_with_broken_data_integrity.id } - let(:vulnerability_5) { create_vulnerability!(project_id: project.id, author_id: user.id) } - let(:different_project) { table(:projects).create!(namespace_id: namespace.id) } - let!(:identifier_with_broken_data_integrity) do - vulnerability_identifiers.create!( - project_id: different_project.id, - external_type: identifier_2, - external_id: identifier_2, - fingerprint: Gitlab::Database::ShaAttribute.serialize('4299e8ddd819f9bde9cfacf45716724c17b5ddf7'), - name: 'Identifier 2') - end - - let(:finding_with_broken_data_integrity) do - create_finding!( - vulnerability_id: vulnerability_5, - project_id: project.id, - scanner_id: scanner.id, - primary_identifier_id: identifier_with_broken_data_integrity.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: SecureRandom.uuid - ) - end - - it 'deletes the conflicting record' do - expect { subject }.to change { vulnerability_findings.find_by_id(finding_with_broken_data_integrity.id) }.to(nil) - end - end - - context 'if a conflicting UUID is found during the migration' do # rubocop:disable RSpec/MultipleMemoizedHelpers - let(:finding_class) { Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid::VulnerabilitiesFinding } - let(:uuid) { '4564f9d5-3c6b-5cc3-af8c-7c25285362a7' } - - before do - exception = ActiveRecord::RecordNotUnique.new("(uuid)=(#{uuid})") - - call_count = 0 - allow(::Gitlab::Database::BulkUpdate).to receive(:execute) do - call_count += 1 - call_count.eql?(1) ? raise(exception) : {} - end - - allow(finding_class).to receive(:find_by).with(uuid: uuid).and_return(duplicate_not_in_the_same_batch) - end - - it 'retries the recalculation' do - subject - - expect(Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid::VulnerabilitiesFinding) - .to have_received(:find_by).with(uuid: uuid).once - end - - it 'logs the conflict' do - expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |instance| - expect(instance).to receive(:info).exactly(6).times - end - - subject - end - - it 'marks the job as done' do - create_background_migration_job([start_id, end_id], :pending) - - subject - - expect(pending_jobs.count).to eq(0) - expect(succeeded_jobs.count).to eq(1) - end - end - - it 'logs an exception if a different uniquness problem was found' do - exception = ActiveRecord::RecordNotUnique.new("Totally not an UUID uniqueness problem") - allow(::Gitlab::Database::BulkUpdate).to receive(:execute).and_raise(exception) - allow(Gitlab::ErrorTracking).to receive(:track_and_raise_exception) - - subject - - expect(Gitlab::ErrorTracking).to have_received(:track_and_raise_exception).with(exception).once - end - - it 'logs a duplicate found message' do - expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |instance| - expect(instance).to receive(:info).exactly(3).times - end - - subject - end - end - - context 'when finding has a signature' do - before do - @f1 = create_finding!( - vulnerability_id: nil, - project_id: project.id, - scanner_id: scanner.id, - primary_identifier_id: vulnerability_identifier.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: 'd15d774d-e4b1-5a1b-929b-19f2a53e35ec' - ) - - vulnerability_finding_signatures.create!( - finding_id: @f1.id, - algorithm_type: 2, # location - signature_sha: Gitlab::Database::ShaAttribute.serialize('57d4e05205f6462a73f039a5b2751aa1ab344e6e') # sha1('youshouldusethis') - ) - - vulnerability_finding_signatures.create!( - finding_id: @f1.id, - algorithm_type: 1, # hash - signature_sha: Gitlab::Database::ShaAttribute.serialize('c554d8d8df1a7a14319eafdaae24af421bf5b587') # sha1('andnotthis') - ) - - @f2 = create_finding!( - vulnerability_id: nil, - project_id: project.id, - scanner_id: scanner.id, - primary_identifier_id: vulnerability_identfier2.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: '4be029b5-75e5-5ac0-81a2-50ab41726135' - ) - - vulnerability_finding_signatures.create!( - finding_id: @f2.id, - algorithm_type: 2, # location - signature_sha: Gitlab::Database::ShaAttribute.serialize('57d4e05205f6462a73f039a5b2751aa1ab344e6e') # sha1('youshouldusethis') - ) - - vulnerability_finding_signatures.create!( - finding_id: @f2.id, - algorithm_type: 1, # hash - signature_sha: Gitlab::Database::ShaAttribute.serialize('c554d8d8df1a7a14319eafdaae24af421bf5b587') # sha1('andnotthis') - ) - end - - let(:start_id) { @f1.id } - let(:end_id) { @f2.id } - - let(:uuids_before) { [@f1.uuid, @f2.uuid] } - let(:uuids_after) { %w[d3b60ddd-d312-5606-b4d3-ad058eebeacb 349d9bec-c677-5530-a8ac-5e58889c3b1a] } - - it 'is recalculated using signature' do - expect(vulnerability_findings.pluck(:uuid)).to match_array(uuids_before) - - subject - - expect(vulnerability_findings.pluck(:uuid)).to match_array(uuids_after) - end - end - - context 'if all records are removed before the job ran' do - let(:start_id) { 1 } - let(:end_id) { 9 } - - before do - create_background_migration_job([start_id, end_id], :pending) - end - - it 'does not error out' do - expect { subject }.not_to raise_error - end - - it 'marks the job as done' do - subject - - expect(pending_jobs.count).to eq(0) - expect(succeeded_jobs.count).to eq(1) - end - end - - context 'when recalculation fails' do - before do - @uuid_v4 = create_finding!( - vulnerability_id: nil, - project_id: project.id, - scanner_id: scanner2.id, - primary_identifier_id: vulnerability_identfier2.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize("fa18f432f1d56675f4098d318739c3cd5b14eb3e"), - uuid: known_uuid_v4 - ) - - allow(Gitlab::ErrorTracking).to receive(:track_and_raise_exception) - allow(::Gitlab::Database::BulkUpdate).to receive(:execute).and_raise(expected_error) - end - - let(:start_id) { @uuid_v4.id } - let(:end_id) { @uuid_v4.id } - let(:expected_error) { RuntimeError.new } - - it 'captures the errors and does not crash entirely' do - expect { subject }.not_to raise_error - - allow(Gitlab::ErrorTracking).to receive(:track_and_raise_exception) - expect(Gitlab::ErrorTracking).to have_received(:track_and_raise_exception).with(expected_error).once - end - - it_behaves_like 'marks background migration job records' do - let(:arguments) { [1, 4] } - subject { described_class.new } - end - end - - it_behaves_like 'marks background migration job records' do - let(:arguments) { [1, 4] } - subject { described_class.new } - end - - private - - def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0) - vulnerabilities.create!( - project_id: project_id, - author_id: author_id, - title: title, - severity: severity, - confidence: confidence, - report_type: report_type - ) - end - - # rubocop:disable Metrics/ParameterLists - def create_finding!( - vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:, id: nil, - name: "test", severity: 7, confidence: 7, report_type: 0, - project_fingerprint: '123qweasdzxc', location_fingerprint: 'test', - metadata_version: 'test', raw_metadata: 'test', uuid: SecureRandom.uuid) - vulnerability_findings.create!({ - id: id, - vulnerability_id: vulnerability_id, - project_id: project_id, - name: name, - severity: severity, - confidence: confidence, - report_type: report_type, - project_fingerprint: project_fingerprint, - scanner_id: scanner_id, - primary_identifier_id: primary_identifier_id, - location_fingerprint: location_fingerprint, - metadata_version: metadata_version, - raw_metadata: raw_metadata, - uuid: uuid - }.compact - ) - end - # rubocop:enable Metrics/ParameterLists - - def create_user!(name: "Example User", email: "user@example.com", user_type: nil, created_at: Time.zone.now, confirmed_at: Time.zone.now) - users.create!( - name: name, - email: email, - username: name, - projects_limit: 0, - user_type: user_type, - confirmed_at: confirmed_at - ) - end - - def create_finding_pipeline!(project_id:, finding_id:) - pipeline = table(:ci_pipelines).create!(project_id: project_id) - vulnerability_finding_pipelines.create!(pipeline_id: pipeline.id, occurrence_id: finding_id) - end -end diff --git a/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb b/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb index 5fede892463..582c0fe1b1b 100644 --- a/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb +++ b/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb @@ -86,8 +86,10 @@ RSpec.describe Gitlab::BackgroundMigration::RemoveBackfilledJobArtifactsExpireAt def create_job_artifact(id:, file_type:, expire_at:) job = table(:ci_builds, database: :ci).create!(id: id, partition_id: 100) - job_artifact.create!(id: id, job_id: job.id, expire_at: expire_at, project_id: project.id, - file_type: file_type, partition_id: 100) + job_artifact.create!( + id: id, job_id: job.id, expire_at: expire_at, project_id: project.id, + file_type: file_type, partition_id: 100 + ) end end end diff --git a/spec/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings_spec.rb b/spec/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings_spec.rb deleted file mode 100644 index ed08ae22245..00000000000 --- a/spec/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings_spec.rb +++ /dev/null @@ -1,171 +0,0 @@ -# frozen_string_literal: true -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::RemoveDuplicateVulnerabilitiesFindings, :migration, schema: 20220326161803 do - let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } - let(:users) { table(:users) } - let(:user) { create_user! } - let(:project) { table(:projects).create!(id: 14219619, namespace_id: namespace.id) } - let(:scanners) { table(:vulnerability_scanners) } - let!(:scanner1) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') } - let!(:scanner2) { scanners.create!(project_id: project.id, external_id: 'test 2', name: 'test scanner 2') } - let!(:scanner3) { scanners.create!(project_id: project.id, external_id: 'test 3', name: 'test scanner 3') } - let!(:unrelated_scanner) { scanners.create!(project_id: project.id, external_id: 'unreleated_scanner', name: 'unrelated scanner') } - let(:vulnerabilities) { table(:vulnerabilities) } - let(:vulnerability_findings) { table(:vulnerability_occurrences) } - let(:vulnerability_identifiers) { table(:vulnerability_identifiers) } - let(:vulnerability_identifier) do - vulnerability_identifiers.create!( - id: 1244459, - project_id: project.id, - external_type: 'vulnerability-identifier', - external_id: 'vulnerability-identifier', - fingerprint: '0a203e8cd5260a1948edbedc76c7cb91ad6a2e45', - name: 'vulnerability identifier') - end - - let!(:vulnerability_for_first_duplicate) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:first_finding_duplicate) do - create_finding!( - id: 5606961, - uuid: "bd95c085-71aa-51d7-9bb6-08ae669c262e", - vulnerability_id: vulnerability_for_first_duplicate.id, - report_type: 0, - location_fingerprint: '00049d5119c2cb3bfb3d1ee1f6e031fe925aed75', - primary_identifier_id: vulnerability_identifier.id, - scanner_id: scanner1.id, - project_id: project.id - ) - end - - let!(:vulnerability_for_second_duplicate) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:second_finding_duplicate) do - create_finding!( - id: 8765432, - uuid: "5b714f58-1176-5b26-8fd5-e11dfcb031b5", - vulnerability_id: vulnerability_for_second_duplicate.id, - report_type: 0, - location_fingerprint: '00049d5119c2cb3bfb3d1ee1f6e031fe925aed75', - primary_identifier_id: vulnerability_identifier.id, - scanner_id: scanner2.id, - project_id: project.id - ) - end - - let!(:vulnerability_for_third_duplicate) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:third_finding_duplicate) do - create_finding!( - id: 8832995, - uuid: "cfe435fa-b25b-5199-a56d-7b007cc9e2d4", - vulnerability_id: vulnerability_for_third_duplicate.id, - report_type: 0, - location_fingerprint: '00049d5119c2cb3bfb3d1ee1f6e031fe925aed75', - primary_identifier_id: vulnerability_identifier.id, - scanner_id: scanner3.id, - project_id: project.id - ) - end - - let!(:unrelated_finding) do - create_finding!( - id: 9999999, - uuid: Gitlab::UUID.v5(SecureRandom.hex), - vulnerability_id: nil, - report_type: 1, - location_fingerprint: 'random_location_fingerprint', - primary_identifier_id: vulnerability_identifier.id, - scanner_id: unrelated_scanner.id, - project_id: project.id - ) - end - - subject { described_class.new.perform(first_finding_duplicate.id, unrelated_finding.id) } - - before do - stub_const("#{described_class}::DELETE_BATCH_SIZE", 1) - end - - it "removes entries which would result in duplicate UUIDv5" do - expect(vulnerability_findings.count).to eq(4) - - expect { subject }.to change { vulnerability_findings.count }.from(4).to(2) - - expect(vulnerability_findings.pluck(:id)).to match_array([third_finding_duplicate.id, unrelated_finding.id]) - end - - it "removes vulnerabilites without findings" do - expect(vulnerabilities.count).to eq(3) - - expect { subject }.to change { vulnerabilities.count }.from(3).to(1) - - expect(vulnerabilities.pluck(:id)).to match_array([vulnerability_for_third_duplicate.id]) - end - - private - - def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0) - vulnerabilities.create!( - project_id: project_id, - author_id: author_id, - title: title, - severity: severity, - confidence: confidence, - report_type: report_type - ) - end - - # rubocop:disable Metrics/ParameterLists - def create_finding!( - vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:, id: nil, - name: "test", severity: 7, confidence: 7, report_type: 0, - project_fingerprint: '123qweasdzxc', location_fingerprint: 'test', - metadata_version: 'test', raw_metadata: 'test', uuid: SecureRandom.uuid) - params = { - vulnerability_id: vulnerability_id, - project_id: project_id, - name: name, - severity: severity, - confidence: confidence, - report_type: report_type, - project_fingerprint: project_fingerprint, - scanner_id: scanner_id, - primary_identifier_id: vulnerability_identifier.id, - location_fingerprint: location_fingerprint, - metadata_version: metadata_version, - raw_metadata: raw_metadata, - uuid: uuid - } - params[:id] = id unless id.nil? - vulnerability_findings.create!(params) - end - # rubocop:enable Metrics/ParameterLists - - def create_user!(name: "Example User", email: "user@example.com", user_type: nil, created_at: Time.zone.now, confirmed_at: Time.zone.now) - users.create!( - name: name, - email: email, - username: name, - projects_limit: 0, - user_type: user_type, - confirmed_at: confirmed_at - ) - end -end diff --git a/spec/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings_spec.rb b/spec/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings_spec.rb index 1844347f4a9..60ee61cf50a 100644 --- a/spec/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings_spec.rb +++ b/spec/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::RemoveOccurrencePipelinesAndDuplicateVulnerabilitiesFindings, :migration, - :suppress_gitlab_schemas_validate_connection, schema: 20220326161803 do + :suppress_gitlab_schemas_validate_connection, schema: 20220326161803 do let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } let(:users) { table(:users) } let(:user) { create_user! } diff --git a/spec/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups_spec.rb b/spec/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups_spec.rb new file mode 100644 index 00000000000..c45c402ab9d --- /dev/null +++ b/spec/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups_spec.rb @@ -0,0 +1,124 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::RemoveProjectGroupLinkWithMissingGroups, :migration, + feature_category: :subgroups, schema: 20230206172702 do + let(:projects) { table(:projects) } + let(:namespaces) { table(:namespaces) } + let(:project_group_links) { table(:project_group_links) } + + let!(:group) do + namespaces.create!( + name: 'Group0', type: 'Group', path: 'space0' + ) + end + + let!(:group_1) do + namespaces.create!( + name: 'Group1', type: 'Group', path: 'space1' + ) + end + + let!(:group_2) do + namespaces.create!( + name: 'Group2', type: 'Group', path: 'space2' + ) + end + + let!(:group_3) do + namespaces.create!( + name: 'Group3', type: 'Group', path: 'space3' + ) + end + + let!(:project_namespace_1) do + namespaces.create!( + name: 'project_1', path: 'project_1', type: 'Project' + ) + end + + let!(:project_namespace_2) do + namespaces.create!( + name: 'project_2', path: 'project_2', type: 'Project' + ) + end + + let!(:project_namespace_3) do + namespaces.create!( + name: 'project_3', path: 'project_3', type: 'Project' + ) + end + + let!(:project_1) do + projects.create!( + name: 'project_1', path: 'project_1', namespace_id: group.id, project_namespace_id: project_namespace_1.id + ) + end + + let!(:project_2) do + projects.create!( + name: 'project_2', path: 'project_2', namespace_id: group.id, project_namespace_id: project_namespace_2.id + ) + end + + let!(:project_3) do + projects.create!( + name: 'project_3', path: 'project_3', namespace_id: group.id, project_namespace_id: project_namespace_3.id + ) + end + + let!(:project_group_link_1) do + project_group_links.create!( + project_id: project_1.id, group_id: group_1.id, group_access: Gitlab::Access::DEVELOPER + ) + end + + let!(:project_group_link_2) do + project_group_links.create!( + project_id: project_2.id, group_id: group_2.id, group_access: Gitlab::Access::DEVELOPER + ) + end + + let!(:project_group_link_3) do + project_group_links.create!( + project_id: project_3.id, group_id: group_3.id, group_access: Gitlab::Access::DEVELOPER + ) + end + + let!(:project_group_link_4) do + project_group_links.create!( + project_id: project_3.id, group_id: group_2.id, group_access: Gitlab::Access::DEVELOPER + ) + end + + subject do + described_class.new( + start_id: project_group_link_1.id, + end_id: project_group_link_4.id, + batch_table: :project_group_links, + batch_column: :id, + sub_batch_size: 1, + pause_ms: 0, + connection: ApplicationRecord.connection + ).perform + end + + it 'removes the `project_group_links` records whose associated group does not exist anymore' do + group_2.delete + + # Schema is fixed to `20230206172702` on this spec. + # This expectation is needed to make sure that the orphaned records are indeed deleted via the migration + # and not via the foreign_key relationship introduced after `20230206172702`, in `20230207002330` + expect(project_group_links.count).to eq(4) + + expect { subject } + .to change { project_group_links.count }.from(4).to(2) + .and change { + project_group_links.where(project_id: project_2.id, group_id: group_2.id).present? + }.from(true).to(false) + .and change { + project_group_links.where(project_id: project_3.id, group_id: group_2.id).present? + }.from(true).to(false) + end +end diff --git a/spec/lib/gitlab/background_migration/remove_self_managed_wiki_notes_spec.rb b/spec/lib/gitlab/background_migration/remove_self_managed_wiki_notes_spec.rb index 81927100562..59d5d56ebe8 100644 --- a/spec/lib/gitlab/background_migration/remove_self_managed_wiki_notes_spec.rb +++ b/spec/lib/gitlab/background_migration/remove_self_managed_wiki_notes_spec.rb @@ -6,14 +6,15 @@ RSpec.describe Gitlab::BackgroundMigration::RemoveSelfManagedWikiNotes, :migrati let(:notes) { table(:notes) } subject(:perform_migration) do - described_class.new(start_id: 1, - end_id: 30, - batch_table: :notes, - batch_column: :id, - sub_batch_size: 2, - pause_ms: 0, - connection: ActiveRecord::Base.connection) - .perform + described_class.new( + start_id: 1, + end_id: 30, + batch_table: :notes, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform end it 'removes all wiki notes' do diff --git a/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb b/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb index 918df8f4442..32134b99e37 100644 --- a/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb +++ b/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::RemoveVulnerabilityFindingLinks, :migration, schema: 20211104165220 do +RSpec.describe Gitlab::BackgroundMigration::RemoveVulnerabilityFindingLinks, :migration, schema: 20211202041233 do let(:vulnerability_findings) { table(:vulnerability_occurrences) } let(:finding_links) { table(:vulnerability_finding_links) } diff --git a/spec/lib/gitlab/background_migration/reset_too_many_tags_skipped_registry_imports_spec.rb b/spec/lib/gitlab/background_migration/reset_too_many_tags_skipped_registry_imports_spec.rb index 3f59b0a24a3..afdd855c5a8 100644 --- a/spec/lib/gitlab/background_migration/reset_too_many_tags_skipped_registry_imports_spec.rb +++ b/spec/lib/gitlab/background_migration/reset_too_many_tags_skipped_registry_imports_spec.rb @@ -3,8 +3,8 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::ResetTooManyTagsSkippedRegistryImports, :migration, - :aggregate_failures, - schema: 20220502173045 do + :aggregate_failures, + schema: 20220502173045 do let(:namespaces) { table(:namespaces) } let(:projects) { table(:projects) } let(:container_repositories) { table(:container_repositories) } @@ -15,46 +15,54 @@ RSpec.describe Gitlab::BackgroundMigration::ResetTooManyTagsSkippedRegistryImpor let!(:project) { projects.create!(id: 1, project_namespace_id: 1, namespace_id: 1, path: 'bar', name: 'bar') } let!(:container_repository1) do - container_repositories.create!(id: 1, - project_id: 1, - name: 'a', - migration_state: 'import_skipped', - migration_skipped_at: Time.zone.now, - migration_skipped_reason: 2, - migration_pre_import_started_at: Time.zone.now, - migration_pre_import_done_at: Time.zone.now, - migration_import_started_at: Time.zone.now, - migration_import_done_at: Time.zone.now, - migration_aborted_at: Time.zone.now, - migration_retries_count: 2, - migration_aborted_in_state: 'importing') + container_repositories.create!( + id: 1, + project_id: 1, + name: 'a', + migration_state: 'import_skipped', + migration_skipped_at: Time.zone.now, + migration_skipped_reason: 2, + migration_pre_import_started_at: Time.zone.now, + migration_pre_import_done_at: Time.zone.now, + migration_import_started_at: Time.zone.now, + migration_import_done_at: Time.zone.now, + migration_aborted_at: Time.zone.now, + migration_retries_count: 2, + migration_aborted_in_state: 'importing' + ) end let!(:container_repository2) do - container_repositories.create!(id: 2, - project_id: 1, - name: 'b', - migration_state: 'import_skipped', - migration_skipped_at: Time.zone.now, - migration_skipped_reason: 2) + container_repositories.create!( + id: 2, + project_id: 1, + name: 'b', + migration_state: 'import_skipped', + migration_skipped_at: Time.zone.now, + migration_skipped_reason: 2 + ) end let!(:container_repository3) do - container_repositories.create!(id: 3, - project_id: 1, - name: 'c', - migration_state: 'import_skipped', - migration_skipped_at: Time.zone.now, - migration_skipped_reason: 1) + container_repositories.create!( + id: 3, + project_id: 1, + name: 'c', + migration_state: 'import_skipped', + migration_skipped_at: Time.zone.now, + migration_skipped_reason: 1 + ) end # This is an unlikely state, but included here to test the edge case let!(:container_repository4) do - container_repositories.create!(id: 4, - project_id: 1, - name: 'd', - migration_state: 'default', - migration_skipped_reason: 2) + container_repositories.create!( + id: 4, + project_id: 1, + name: 'd', + migration_state: 'default', + migration_skipped_reason: 2 + ) end describe '#up' do diff --git a/spec/lib/gitlab/background_migration/set_correct_vulnerability_state_spec.rb b/spec/lib/gitlab/background_migration/set_correct_vulnerability_state_spec.rb index 2372ce21c4c..df1ee494987 100644 --- a/spec/lib/gitlab/background_migration/set_correct_vulnerability_state_spec.rb +++ b/spec/lib/gitlab/background_migration/set_correct_vulnerability_state_spec.rb @@ -35,13 +35,15 @@ RSpec.describe Gitlab::BackgroundMigration::SetCorrectVulnerabilityState do let(:dismissed_state) { 2 } let(:migration_job) do - described_class.new(start_id: vulnerability_with_dismissed_at.id, - end_id: vulnerability_without_dismissed_at.id, - batch_table: :vulnerabilities, - batch_column: :id, - sub_batch_size: 1, - pause_ms: 0, - connection: ActiveRecord::Base.connection) + described_class.new( + start_id: vulnerability_with_dismissed_at.id, + end_id: vulnerability_without_dismissed_at.id, + batch_table: :vulnerabilities, + batch_column: :id, + sub_batch_size: 1, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ) end describe '#filter_batch' do diff --git a/spec/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects_spec.rb b/spec/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects_spec.rb index e9f73672144..5109c3ec0c2 100644 --- a/spec/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects_spec.rb +++ b/spec/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects_spec.rb @@ -3,21 +3,22 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::SetLegacyOpenSourceLicenseAvailableForNonPublicProjects, - :migration, - schema: 20220722110026 do + :migration, + schema: 20220722110026 do let(:namespaces_table) { table(:namespaces) } let(:projects_table) { table(:projects) } let(:project_settings_table) { table(:project_settings) } subject(:perform_migration) do - described_class.new(start_id: projects_table.minimum(:id), - end_id: projects_table.maximum(:id), - batch_table: :projects, - batch_column: :id, - sub_batch_size: 2, - pause_ms: 0, - connection: ActiveRecord::Base.connection) - .perform + described_class.new( + start_id: projects_table.minimum(:id), + end_id: projects_table.maximum(:id), + batch_table: :projects, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform end it 'sets `legacy_open_source_license_available` attribute to false for non-public projects', :aggregate_failures do @@ -37,11 +38,13 @@ RSpec.describe Gitlab::BackgroundMigration::SetLegacyOpenSourceLicenseAvailableF def create_legacy_license_project(path, visibility_level:) namespace = namespaces_table.create!(name: "namespace-#{path}", path: "namespace-#{path}") project_namespace = namespaces_table.create!(name: "project-namespace-#{path}", path: path, type: 'Project') - project = projects_table.create!(name: path, - path: path, - namespace_id: namespace.id, - project_namespace_id: project_namespace.id, - visibility_level: visibility_level) + project = projects_table.create!( + name: path, + path: path, + namespace_id: namespace.id, + project_namespace_id: project_namespace.id, + visibility_level: visibility_level + ) project_settings_table.create!(project_id: project.id, legacy_open_source_license_available: true) project diff --git a/spec/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users_spec.rb b/spec/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users_spec.rb deleted file mode 100644 index 841a7f306d7..00000000000 --- a/spec/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users_spec.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::StealMigrateMergeRequestDiffCommitUsers, schema: 20211012134316 do - let(:migration) { described_class.new } - - describe '#perform' do - it 'processes the background migration' do - spy = instance_spy( - Gitlab::BackgroundMigration::MigrateMergeRequestDiffCommitUsers - ) - - allow(Gitlab::BackgroundMigration::MigrateMergeRequestDiffCommitUsers) - .to receive(:new) - .and_return(spy) - - expect(spy).to receive(:perform).with(1, 4) - expect(migration).to receive(:schedule_next_job) - - migration.perform(1, 4) - end - end - - describe '#schedule_next_job' do - it 'schedules the next job in ascending order' do - Gitlab::Database::BackgroundMigrationJob.create!( - class_name: 'MigrateMergeRequestDiffCommitUsers', - arguments: [10, 20] - ) - - Gitlab::Database::BackgroundMigrationJob.create!( - class_name: 'MigrateMergeRequestDiffCommitUsers', - arguments: [40, 50] - ) - - expect(BackgroundMigrationWorker) - .to receive(:perform_in) - .with(5.minutes, 'StealMigrateMergeRequestDiffCommitUsers', [10, 20]) - - migration.schedule_next_job - end - - it 'does not schedule any new jobs when there are none' do - expect(BackgroundMigrationWorker).not_to receive(:perform_in) - - migration.schedule_next_job - end - end -end diff --git a/spec/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces_spec.rb b/spec/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces_spec.rb index 980a7771f4c..0579a299c74 100644 --- a/spec/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces_spec.rb +++ b/spec/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::UpdateDelayedProjectRemovalToNullForUserNamespaces, - :migration do + :migration do let(:namespaces_table) { table(:namespaces) } let(:namespace_settings_table) { table(:namespace_settings) } diff --git a/spec/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url_spec.rb b/spec/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url_spec.rb index c090c1df424..75fe5699986 100644 --- a/spec/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url_spec.rb +++ b/spec/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url_spec.rb @@ -13,10 +13,12 @@ RSpec.describe Gitlab::BackgroundMigration::UpdateJiraTrackerDataDeploymentTypeB let(:sub_batch_size) { 1 } let(:pause_ms) { 0 } let(:migration) do - described_class.new(start_id: 1, end_id: 10, - batch_table: table_name, batch_column: batch_column, - sub_batch_size: sub_batch_size, pause_ms: pause_ms, - connection: ApplicationRecord.connection) + described_class.new( + start_id: 1, end_id: 10, + batch_table: table_name, batch_column: batch_column, + sub_batch_size: sub_batch_size, pause_ms: pause_ms, + connection: ApplicationRecord.connection + ) end subject(:perform_migration) do diff --git a/spec/lib/gitlab/background_migration/update_timelogs_project_id_spec.rb b/spec/lib/gitlab/background_migration/update_timelogs_project_id_spec.rb deleted file mode 100644 index b8c3bf8f3ac..00000000000 --- a/spec/lib/gitlab/background_migration/update_timelogs_project_id_spec.rb +++ /dev/null @@ -1,52 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::UpdateTimelogsProjectId, schema: 20210826171758 do - let!(:namespace) { table(:namespaces).create!(name: 'namespace', path: 'namespace') } - let!(:project1) { table(:projects).create!(namespace_id: namespace.id) } - let!(:project2) { table(:projects).create!(namespace_id: namespace.id) } - let!(:issue1) { table(:issues).create!(project_id: project1.id) } - let!(:issue2) { table(:issues).create!(project_id: project2.id) } - let!(:merge_request1) { table(:merge_requests).create!(target_project_id: project1.id, source_branch: 'master', target_branch: 'feature') } - let!(:merge_request2) { table(:merge_requests).create!(target_project_id: project2.id, source_branch: 'master', target_branch: 'feature') } - let!(:timelog1) { table(:timelogs).create!(issue_id: issue1.id, time_spent: 60) } - let!(:timelog2) { table(:timelogs).create!(issue_id: issue1.id, time_spent: 60) } - let!(:timelog3) { table(:timelogs).create!(issue_id: issue2.id, time_spent: 60) } - let!(:timelog4) { table(:timelogs).create!(merge_request_id: merge_request1.id, time_spent: 600) } - let!(:timelog5) { table(:timelogs).create!(merge_request_id: merge_request1.id, time_spent: 600) } - let!(:timelog6) { table(:timelogs).create!(merge_request_id: merge_request2.id, time_spent: 600) } - let!(:timelog7) { table(:timelogs).create!(issue_id: issue2.id, time_spent: 60, project_id: project1.id) } - let!(:timelog8) { table(:timelogs).create!(merge_request_id: merge_request2.id, time_spent: 600, project_id: project1.id) } - - describe '#perform' do - context 'when timelogs belong to issues' do - it 'sets correct project_id' do - subject.perform(timelog1.id, timelog3.id) - - expect(timelog1.reload.project_id).to eq(issue1.project_id) - expect(timelog2.reload.project_id).to eq(issue1.project_id) - expect(timelog3.reload.project_id).to eq(issue2.project_id) - end - end - - context 'when timelogs belong to merge requests' do - it 'sets correct project ids' do - subject.perform(timelog4.id, timelog6.id) - - expect(timelog4.reload.project_id).to eq(merge_request1.target_project_id) - expect(timelog5.reload.project_id).to eq(merge_request1.target_project_id) - expect(timelog6.reload.project_id).to eq(merge_request2.target_project_id) - end - end - - context 'when timelogs already belong to projects' do - it 'does not update the project id' do - subject.perform(timelog7.id, timelog8.id) - - expect(timelog7.reload.project_id).to eq(project1.id) - expect(timelog8.reload.project_id).to eq(project1.id) - end - end - end -end diff --git a/spec/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group_spec.rb b/spec/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group_spec.rb deleted file mode 100644 index f16ae489b78..00000000000 --- a/spec/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group_spec.rb +++ /dev/null @@ -1,84 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::UpdateUsersWhereTwoFactorAuthRequiredFromGroup, :migration, schema: 20210826171758 do - include MigrationHelpers::NamespacesHelpers - - let(:group_with_2fa_parent) { create_namespace('parent', Gitlab::VisibilityLevel::PRIVATE, require_two_factor_authentication: true) } - let(:group_with_2fa_child) { create_namespace('child', Gitlab::VisibilityLevel::PRIVATE, parent_id: group_with_2fa_parent.id) } - let(:members_table) { table(:members) } - let(:users_table) { table(:users) } - - subject { described_class.new } - - describe '#perform' do - context 'with group members' do - let(:user_1) { create_user('user@example.com') } - let!(:member) { create_group_member(user_1, group_with_2fa_parent) } - let!(:user_without_group) { create_user('user_without@example.com') } - let(:user_other) { create_user('user_other@example.com') } - let!(:member_other) { create_group_member(user_other, group_with_2fa_parent) } - - it 'updates user when user should be required to establish two factor authentication' do - subject.perform(user_1.id, user_without_group.id) - - expect(user_1.reload.require_two_factor_authentication_from_group).to eq(true) - end - - it 'does not update user who is not in current batch' do - subject.perform(user_1.id, user_without_group.id) - - expect(user_other.reload.require_two_factor_authentication_from_group).to eq(false) - end - - it 'updates all users in current batch' do - subject.perform(user_1.id, user_other.id) - - expect(user_other.reload.require_two_factor_authentication_from_group).to eq(true) - end - - it 'updates user when user is member of group in which parent group requires two factor authentication' do - member.destroy! - - subgroup = create_namespace('subgroup', Gitlab::VisibilityLevel::PRIVATE, require_two_factor_authentication: false, parent_id: group_with_2fa_child.id) - create_group_member(user_1, subgroup) - - subject.perform(user_1.id, user_other.id) - - expect(user_1.reload.require_two_factor_authentication_from_group).to eq(true) - end - - it 'updates user when user is member of a group and the subgroup requires two factor authentication' do - member.destroy! - - parent = create_namespace('other_parent', Gitlab::VisibilityLevel::PRIVATE, require_two_factor_authentication: false) - create_namespace('other_subgroup', Gitlab::VisibilityLevel::PRIVATE, require_two_factor_authentication: true, parent_id: parent.id) - create_group_member(user_1, parent) - - subject.perform(user_1.id, user_other.id) - - expect(user_1.reload.require_two_factor_authentication_from_group).to eq(true) - end - - it 'does not update user when not a member of a group that requires two factor authentication' do - member_other.destroy! - - other_group = create_namespace('other_group', Gitlab::VisibilityLevel::PRIVATE, require_two_factor_authentication: false) - create_group_member(user_other, other_group) - - subject.perform(user_1.id, user_other.id) - - expect(user_other.reload.require_two_factor_authentication_from_group).to eq(false) - end - end - end - - def create_user(email, require_2fa: false) - users_table.create!(email: email, projects_limit: 10, require_two_factor_authentication_from_group: require_2fa) - end - - def create_group_member(user, group) - members_table.create!(user_id: user.id, source_id: group.id, access_level: GroupMember::MAINTAINER, source_type: "Namespace", type: "GroupMember", notification_level: 3) - end -end |