summaryrefslogtreecommitdiff
path: root/spec/lib/gitlab/background_migration
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2023-05-17 16:05:49 +0000
committerGitLab Bot <gitlab-bot@gitlab.com>2023-05-17 16:05:49 +0000
commit43a25d93ebdabea52f99b05e15b06250cd8f07d7 (patch)
treedceebdc68925362117480a5d672bcff122fb625b /spec/lib/gitlab/background_migration
parent20c84b99005abd1c82101dfeff264ac50d2df211 (diff)
downloadgitlab-ce-0f94cf6ca9d272d8e0fda4a7a597866cf3dc1fc0.tar.gz
Add latest changes from gitlab-org/gitlab@16-0-stable-eev16.0.0-rc4216-0-stable
Diffstat (limited to 'spec/lib/gitlab/background_migration')
-rw-r--r--spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb11
-rw-r--r--spec/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities_spec.rb10
-rw-r--r--spec/lib/gitlab/background_migration/backfill_design_management_repositories_spec.rb68
-rw-r--r--spec/lib/gitlab/background_migration/backfill_environment_tiers_spec.rb10
-rw-r--r--spec/lib/gitlab/background_migration/backfill_group_features_spec.rb18
-rw-r--r--spec/lib/gitlab/background_migration/backfill_imported_issue_search_data_spec.rb16
-rw-r--r--spec/lib/gitlab/background_migration/backfill_namespace_details_spec.rb43
-rw-r--r--spec/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route_spec.rb35
-rw-r--r--spec/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads_spec.rb17
-rw-r--r--spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children_spec.rb21
-rw-r--r--spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots_spec.rb21
-rw-r--r--spec/lib/gitlab/background_migration/backfill_partitioned_table_spec.rb140
-rw-r--r--spec/lib/gitlab/background_migration/backfill_prepared_at_merge_requests_spec.rb55
-rw-r--r--spec/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level_spec.rb17
-rw-r--r--spec/lib/gitlab/background_migration/backfill_project_member_namespace_id_spec.rb70
-rw-r--r--spec/lib/gitlab/background_migration/backfill_project_namespace_details_spec.rb47
-rw-r--r--spec/lib/gitlab/background_migration/backfill_project_wiki_repositories_spec.rb65
-rw-r--r--spec/lib/gitlab/background_migration/backfill_releases_author_id_spec.rb59
-rw-r--r--spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb68
-rw-r--r--spec/lib/gitlab/background_migration/backfill_upvotes_count_on_issues_spec.rb46
-rw-r--r--spec/lib/gitlab/background_migration/backfill_user_namespace_spec.rb39
-rw-r--r--spec/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent_spec.rb10
-rw-r--r--spec/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues_spec.rb29
-rw-r--r--spec/lib/gitlab/background_migration/batched_migration_job_spec.rb22
-rw-r--r--spec/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects_spec.rb85
-rw-r--r--spec/lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at_spec.rb38
-rw-r--r--spec/lib/gitlab/background_migration/delete_orphaned_deployments_spec.rb54
-rw-r--r--spec/lib/gitlab/background_migration/delete_orphaned_packages_dependencies_spec.rb57
-rw-r--r--spec/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images_spec.rb142
-rw-r--r--spec/lib/gitlab/background_migration/drop_invalid_security_findings_spec.rb57
-rw-r--r--spec/lib/gitlab/background_migration/drop_invalid_vulnerabilities_spec.rb126
-rw-r--r--spec/lib/gitlab/background_migration/encrypt_ci_trigger_token_spec.rb4
-rw-r--r--spec/lib/gitlab/background_migration/extract_project_topics_into_separate_table_spec.rb46
-rw-r--r--spec/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at_spec.rb166
-rw-r--r--spec/lib/gitlab/background_migration/fix_merge_request_diff_commit_users_spec.rb25
-rw-r--r--spec/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues_spec.rb100
-rw-r--r--spec/lib/gitlab/background_migration/issues_internal_id_scope_updater_spec.rb90
-rw-r--r--spec/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings_spec.rb119
-rw-r--r--spec/lib/gitlab/background_migration/migrate_human_user_type_spec.rb43
-rw-r--r--spec/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings_spec.rb192
-rw-r--r--spec/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users_spec.rb413
-rw-r--r--spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb30
-rw-r--r--spec/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings_spec.rb173
-rw-r--r--spec/lib/gitlab/background_migration/migrate_u2f_webauthn_spec.rb67
-rw-r--r--spec/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature_spec.rb98
-rw-r--r--spec/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects_spec.rb3
-rw-r--r--spec/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds_spec.rb2
-rw-r--r--spec/lib/gitlab/background_migration/populate_topics_total_projects_count_cache_spec.rb35
-rw-r--r--spec/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields_spec.rb114
-rw-r--r--spec/lib/gitlab/background_migration/prune_stale_project_export_jobs_spec.rb17
-rw-r--r--spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb530
-rw-r--r--spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb6
-rw-r--r--spec/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings_spec.rb171
-rw-r--r--spec/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings_spec.rb2
-rw-r--r--spec/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups_spec.rb124
-rw-r--r--spec/lib/gitlab/background_migration/remove_self_managed_wiki_notes_spec.rb17
-rw-r--r--spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb2
-rw-r--r--spec/lib/gitlab/background_migration/reset_too_many_tags_skipped_registry_imports_spec.rb72
-rw-r--r--spec/lib/gitlab/background_migration/set_correct_vulnerability_state_spec.rb16
-rw-r--r--spec/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects_spec.rb33
-rw-r--r--spec/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users_spec.rb50
-rw-r--r--spec/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces_spec.rb2
-rw-r--r--spec/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url_spec.rb10
-rw-r--r--spec/lib/gitlab/background_migration/update_timelogs_project_id_spec.rb52
-rw-r--r--spec/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group_spec.rb84
65 files changed, 1791 insertions, 2613 deletions
diff --git a/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb b/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb
index 7075d4694ae..92fec48454c 100644
--- a/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb
+++ b/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb
@@ -3,7 +3,7 @@
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::BackfillAdminModeScopeForPersonalAccessTokens,
- :migration, schema: 20221228103133, feature_category: :authentication_and_authorization do
+ :migration, schema: 20221228103133, feature_category: :system_access do
let(:users) { table(:users) }
let(:personal_access_tokens) { table(:personal_access_tokens) }
@@ -24,8 +24,12 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillAdminModeScopeForPersonalAcc
personal_access_tokens.create!(name: 'admin 4', user_id: admin.id, scopes: "---\n- admin_mode\n")
end
- let!(:pat_admin_2) { personal_access_tokens.create!(name: 'admin 5', user_id: admin.id, scopes: "---\n- read_api\n") }
- let!(:pat_not_in_range) { personal_access_tokens.create!(name: 'admin 6', user_id: admin.id, scopes: "---\n- api\n") }
+ let!(:pat_with_symbol_in_scopes) do
+ personal_access_tokens.create!(name: 'admin 5', user_id: admin.id, scopes: "---\n- :api\n")
+ end
+
+ let!(:pat_admin_2) { personal_access_tokens.create!(name: 'admin 6', user_id: admin.id, scopes: "---\n- read_api\n") }
+ let!(:pat_not_in_range) { personal_access_tokens.create!(name: 'admin 7', user_id: admin.id, scopes: "---\n- api\n") }
subject do
described_class.new(
@@ -47,6 +51,7 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillAdminModeScopeForPersonalAcc
expect(pat_revoked.reload.scopes).to eq("---\n- api\n")
expect(pat_expired.reload.scopes).to eq("---\n- api\n")
expect(pat_admin_mode.reload.scopes).to eq("---\n- admin_mode\n")
+ expect(pat_with_symbol_in_scopes.reload.scopes).to eq("---\n- api\n- admin_mode\n")
expect(pat_admin_2.reload.scopes).to eq("---\n- read_api\n- admin_mode\n")
expect(pat_not_in_range.reload.scopes).to eq("---\n- api\n")
end
diff --git a/spec/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities_spec.rb b/spec/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities_spec.rb
index 3aab0cdf54b..edb6ff59340 100644
--- a/spec/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities_spec.rb
+++ b/spec/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities_spec.rb
@@ -4,10 +4,12 @@ require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::BackfillClusterAgentsHasVulnerabilities, :migration do # rubocop:disable Layout/LineLength
let(:migration) do
- described_class.new(start_id: 1, end_id: 10,
- batch_table: table_name, batch_column: batch_column,
- sub_batch_size: sub_batch_size, pause_ms: pause_ms,
- connection: ApplicationRecord.connection)
+ described_class.new(
+ start_id: 1, end_id: 10,
+ batch_table: table_name, batch_column: batch_column,
+ sub_batch_size: sub_batch_size, pause_ms: pause_ms,
+ connection: ApplicationRecord.connection
+ )
end
let(:users_table) { table(:users) }
diff --git a/spec/lib/gitlab/background_migration/backfill_design_management_repositories_spec.rb b/spec/lib/gitlab/background_migration/backfill_design_management_repositories_spec.rb
new file mode 100644
index 00000000000..0cabdc78db8
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/backfill_design_management_repositories_spec.rb
@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe(
+ Gitlab::BackgroundMigration::BackfillDesignManagementRepositories,
+ schema: 20230406121544,
+ feature_category: :geo_replication
+) do
+ let!(:namespaces) { table(:namespaces) }
+ let!(:projects) { table(:projects) }
+ let!(:design_management_repositories) { table(:design_management_repositories) }
+
+ subject(:migration) do
+ described_class.new(
+ start_id: projects.minimum(:id),
+ end_id: projects.maximum(:id),
+ batch_table: :projects,
+ batch_column: :id,
+ sub_batch_size: 2,
+ pause_ms: 0,
+ connection: ActiveRecord::Base.connection
+ )
+ end
+
+ describe '#perform' do
+ it 'creates design_management_repositories entries for all projects in range' do
+ namespace1 = create_namespace('test1')
+ namespace2 = create_namespace('test2')
+ project1 = create_project(namespace1, 'test1')
+ project2 = create_project(namespace2, 'test2')
+ design_management_repositories.create!(project_id: project2.id)
+
+ expect { migration.perform }
+ .to change { design_management_repositories.pluck(:project_id) }
+ .from([project2.id])
+ .to match_array([project1.id, project2.id])
+ end
+
+ context 'when project_id already exists in design_management_repositories' do
+ it "doesn't duplicate project_id" do
+ namespace = create_namespace('test1')
+ project = create_project(namespace, 'test1')
+ design_management_repositories.create!(project_id: project.id)
+
+ expect { migration.perform }
+ .not_to change { design_management_repositories.pluck(:project_id) }
+ end
+ end
+
+ def create_namespace(name)
+ namespaces.create!(
+ name: name,
+ path: name,
+ type: 'Project'
+ )
+ end
+
+ def create_project(namespace, name)
+ projects.create!(
+ namespace_id: namespace.id,
+ project_namespace_id: namespace.id,
+ name: name,
+ path: name
+ )
+ end
+ end
+end
diff --git a/spec/lib/gitlab/background_migration/backfill_environment_tiers_spec.rb b/spec/lib/gitlab/background_migration/backfill_environment_tiers_spec.rb
index 788ed40b61e..9026c327e3c 100644
--- a/spec/lib/gitlab/background_migration/backfill_environment_tiers_spec.rb
+++ b/spec/lib/gitlab/background_migration/backfill_environment_tiers_spec.rb
@@ -8,10 +8,12 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillEnvironmentTiers,
let!(:project) { table(:projects).create!(namespace_id: namespace.id, project_namespace_id: namespace.id) }
let(:migration) do
- described_class.new(start_id: 1, end_id: 1000,
- batch_table: :environments, batch_column: :id,
- sub_batch_size: 10, pause_ms: 0,
- connection: ApplicationRecord.connection)
+ described_class.new(
+ start_id: 1, end_id: 1000,
+ batch_table: :environments, batch_column: :id,
+ sub_batch_size: 10, pause_ms: 0,
+ connection: ApplicationRecord.connection
+ )
end
describe '#perform' do
diff --git a/spec/lib/gitlab/background_migration/backfill_group_features_spec.rb b/spec/lib/gitlab/background_migration/backfill_group_features_spec.rb
index e0be5a785b8..023d4b04e63 100644
--- a/spec/lib/gitlab/background_migration/backfill_group_features_spec.rb
+++ b/spec/lib/gitlab/background_migration/backfill_group_features_spec.rb
@@ -7,14 +7,16 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillGroupFeatures, :migration, s
let(:namespaces) { table(:namespaces) }
subject do
- described_class.new(start_id: 1,
- end_id: 4,
- batch_table: :namespaces,
- batch_column: :id,
- sub_batch_size: 10,
- pause_ms: 0,
- job_arguments: [4],
- connection: ActiveRecord::Base.connection)
+ described_class.new(
+ start_id: 1,
+ end_id: 4,
+ batch_table: :namespaces,
+ batch_column: :id,
+ sub_batch_size: 10,
+ pause_ms: 0,
+ job_arguments: [4],
+ connection: ActiveRecord::Base.connection
+ )
end
describe '#perform' do
diff --git a/spec/lib/gitlab/background_migration/backfill_imported_issue_search_data_spec.rb b/spec/lib/gitlab/background_migration/backfill_imported_issue_search_data_spec.rb
index 479afb56210..b3f04055e0a 100644
--- a/spec/lib/gitlab/background_migration/backfill_imported_issue_search_data_spec.rb
+++ b/spec/lib/gitlab/background_migration/backfill_imported_issue_search_data_spec.rb
@@ -30,13 +30,15 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillImportedIssueSearchData,
end
let(:migration) do
- described_class.new(start_id: issue.id,
- end_id: issue.id + 30,
- batch_table: :issues,
- batch_column: :id,
- sub_batch_size: 2,
- pause_ms: 0,
- connection: ApplicationRecord.connection)
+ described_class.new(
+ start_id: issue.id,
+ end_id: issue.id + 30,
+ batch_table: :issues,
+ batch_column: :id,
+ sub_batch_size: 2,
+ pause_ms: 0,
+ connection: ApplicationRecord.connection
+ )
end
let(:perform_migration) { migration.perform }
diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_details_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_details_spec.rb
index b6282de0da6..39ad60fb13b 100644
--- a/spec/lib/gitlab/background_migration/backfill_namespace_details_spec.rb
+++ b/spec/lib/gitlab/background_migration/backfill_namespace_details_spec.rb
@@ -7,27 +7,36 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceDetails, :migration
let(:namespace_details) { table(:namespace_details) }
subject(:perform_migration) do
- described_class.new(start_id: namespaces.minimum(:id),
- end_id: namespaces.maximum(:id),
- batch_table: :namespaces,
- batch_column: :id,
- sub_batch_size: 2,
- pause_ms: 0,
- connection: ActiveRecord::Base.connection)
- .perform
+ described_class.new(
+ start_id: namespaces.minimum(:id),
+ end_id: namespaces.maximum(:id),
+ batch_table: :namespaces,
+ batch_column: :id,
+ sub_batch_size: 2,
+ pause_ms: 0,
+ connection: ActiveRecord::Base.connection
+ ).perform
end
describe '#perform' do
it 'creates details for all namespaces in range' do
- namespace1 = namespaces.create!(id: 5, name: 'test1', path: 'test1', description: "Some description1",
- description_html: "Some description html1", cached_markdown_version: 4)
- namespaces.create!(id: 6, name: 'test2', path: 'test2', type: 'Project',
- description: "Some description2", description_html: "Some description html2",
- cached_markdown_version: 4)
- namespace3 = namespaces.create!(id: 7, name: 'test3', path: 'test3', description: "Some description3",
- description_html: "Some description html3", cached_markdown_version: 4)
- namespace4 = namespaces.create!(id: 8, name: 'test4', path: 'test4', description: "Some description3",
- description_html: "Some description html4", cached_markdown_version: 4)
+ namespace1 = namespaces.create!(
+ id: 5, name: 'test1', path: 'test1', description: "Some description1",
+ description_html: "Some description html1", cached_markdown_version: 4
+ )
+ namespaces.create!(
+ id: 6, name: 'test2', path: 'test2', type: 'Project',
+ description: "Some description2", description_html: "Some description html2",
+ cached_markdown_version: 4
+ )
+ namespace3 = namespaces.create!(
+ id: 7, name: 'test3', path: 'test3', description: "Some description3",
+ description_html: "Some description html3", cached_markdown_version: 4
+ )
+ namespace4 = namespaces.create!(
+ id: 8, name: 'test4', path: 'test4', description: "Some description3",
+ description_html: "Some description html4", cached_markdown_version: 4
+ )
namespace_details.delete_all
expect(namespace_details.pluck(:namespace_id)).to eql []
diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route_spec.rb
index b821efcadb0..3a8a327550b 100644
--- a/spec/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route_spec.rb
+++ b/spec/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route_spec.rb
@@ -22,18 +22,29 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceIdForNamespaceRoute
subject(:perform_migration) { migration.perform(1, 10, table_name, batch_column, sub_batch_size, pause_ms) }
before do
- routes_table.create!(id: 1, name: 'test1', path: 'test1', source_id: namespace1.id,
- source_type: namespace1.class.sti_name)
- routes_table.create!(id: 2, name: 'test2', path: 'test2', source_id: namespace2.id,
- source_type: namespace2.class.sti_name)
- routes_table.create!(id: 5, name: 'test3', path: 'test3', source_id: project1.id,
- source_type: project1.class.sti_name) # should be ignored - project route
- routes_table.create!(id: 6, name: 'test4', path: 'test4', source_id: non_existing_record_id,
- source_type: namespace3.class.sti_name) # should be ignored - invalid source_id
- routes_table.create!(id: 10, name: 'test5', path: 'test5', source_id: namespace3.id,
- source_type: namespace3.class.sti_name)
- routes_table.create!(id: 11, name: 'test6', path: 'test6', source_id: namespace4.id,
- source_type: namespace4.class.sti_name) # should be ignored - outside the scope
+ routes_table.create!(
+ id: 1, name: 'test1', path: 'test1', source_id: namespace1.id, source_type: namespace1.class.sti_name
+ )
+
+ routes_table.create!(
+ id: 2, name: 'test2', path: 'test2', source_id: namespace2.id, source_type: namespace2.class.sti_name
+ )
+
+ routes_table.create!(
+ id: 5, name: 'test3', path: 'test3', source_id: project1.id, source_type: project1.class.sti_name
+ ) # should be ignored - project route
+
+ routes_table.create!(
+ id: 6, name: 'test4', path: 'test4', source_id: non_existing_record_id, source_type: namespace3.class.sti_name
+ ) # should be ignored - invalid source_id
+
+ routes_table.create!(
+ id: 10, name: 'test5', path: 'test5', source_id: namespace3.id, source_type: namespace3.class.sti_name
+ )
+
+ routes_table.create!(
+ id: 11, name: 'test6', path: 'test6', source_id: namespace4.id, source_type: namespace4.class.sti_name
+ ) # should be ignored - outside the scope
end
it 'backfills `type` for the selected records', :aggregate_failures do
diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads_spec.rb
index 564aa3b8c01..6a55c6951d5 100644
--- a/spec/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads_spec.rb
+++ b/spec/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads_spec.rb
@@ -38,14 +38,15 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceIdOfVulnerabilityRe
end
subject(:perform_migration) do
- described_class.new(start_id: vulnerability_read.vulnerability_id,
- end_id: vulnerability_read.vulnerability_id,
- batch_table: :vulnerability_reads,
- batch_column: :vulnerability_id,
- sub_batch_size: 1,
- pause_ms: 0,
- connection: ActiveRecord::Base.connection)
- .perform
+ described_class.new(
+ start_id: vulnerability_read.vulnerability_id,
+ end_id: vulnerability_read.vulnerability_id,
+ batch_table: :vulnerability_reads,
+ batch_column: :vulnerability_id,
+ sub_batch_size: 1,
+ pause_ms: 0,
+ connection: ActiveRecord::Base.connection
+ ).perform
end
it 'sets the namespace_id of existing record' do
diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children_spec.rb
deleted file mode 100644
index 876eb070745..00000000000
--- a/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children_spec.rb
+++ /dev/null
@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceTraversalIdsChildren, :migration, schema: 20210826171758 do
- let(:namespaces_table) { table(:namespaces) }
-
- let!(:user_namespace) { namespaces_table.create!(id: 1, name: 'user', path: 'user', type: nil) }
- let!(:root_group) { namespaces_table.create!(id: 2, name: 'group', path: 'group', type: 'Group', parent_id: nil) }
- let!(:sub_group) { namespaces_table.create!(id: 3, name: 'subgroup', path: 'subgroup', type: 'Group', parent_id: 2) }
-
- describe '#perform' do
- it 'backfills traversal_ids for child namespaces' do
- described_class.new.perform(1, 3, 5)
-
- expect(user_namespace.reload.traversal_ids).to eq([])
- expect(root_group.reload.traversal_ids).to eq([])
- expect(sub_group.reload.traversal_ids).to eq([root_group.id, sub_group.id])
- end
- end
-end
diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots_spec.rb
deleted file mode 100644
index ad9b54608c6..00000000000
--- a/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots_spec.rb
+++ /dev/null
@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceTraversalIdsRoots, :migration, schema: 20210826171758 do
- let(:namespaces_table) { table(:namespaces) }
-
- let!(:user_namespace) { namespaces_table.create!(id: 1, name: 'user', path: 'user', type: nil) }
- let!(:root_group) { namespaces_table.create!(id: 2, name: 'group', path: 'group', type: 'Group', parent_id: nil) }
- let!(:sub_group) { namespaces_table.create!(id: 3, name: 'subgroup', path: 'subgroup', type: 'Group', parent_id: 2) }
-
- describe '#perform' do
- it 'backfills traversal_ids for root namespaces' do
- described_class.new.perform(1, 3, 5)
-
- expect(user_namespace.reload.traversal_ids).to eq([user_namespace.id])
- expect(root_group.reload.traversal_ids).to eq([root_group.id])
- expect(sub_group.reload.traversal_ids).to eq([])
- end
- end
-end
diff --git a/spec/lib/gitlab/background_migration/backfill_partitioned_table_spec.rb b/spec/lib/gitlab/background_migration/backfill_partitioned_table_spec.rb
new file mode 100644
index 00000000000..53216cc780b
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/backfill_partitioned_table_spec.rb
@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::BackfillPartitionedTable, feature_category: :database do
+ subject(:backfill_job) do
+ described_class.new(
+ start_id: 1,
+ end_id: 3,
+ batch_table: source_table,
+ batch_column: :id,
+ sub_batch_size: 2,
+ pause_ms: 0,
+ job_arguments: [destination_table],
+ connection: connection
+ )
+ end
+
+ let(:connection) { ApplicationRecord.connection }
+ let(:source_table) { '_test_source_table' }
+ let(:destination_table) { "#{source_table}_partitioned" }
+ let(:source_model) { Class.new(ApplicationRecord) }
+ let(:destination_model) { Class.new(ApplicationRecord) }
+
+ describe '#perform' do
+ context 'without the destination table' do
+ let(:expected_error_message) do
+ "exiting backfill migration because partitioned table #{destination_table} does not exist. " \
+ "This could be due to rollback of the migration which created the partitioned table."
+ end
+
+ it 'raises an exception' do
+ expect { backfill_job.perform }.to raise_error(expected_error_message)
+ end
+ end
+
+ context 'with destination table being not partitioned' do
+ before do
+ connection.execute(<<~SQL)
+ CREATE TABLE #{destination_table} (
+ id serial NOT NULL,
+ col1 int NOT NULL,
+ col2 text NOT NULL,
+ created_at timestamptz NOT NULL,
+ PRIMARY KEY (id, created_at)
+ )
+ SQL
+ end
+
+ after do
+ connection.drop_table destination_table
+ end
+
+ let(:expected_error_message) do
+ "exiting backfill migration because the given destination table is not partitioned."
+ end
+
+ it 'raises an exception' do
+ expect { backfill_job.perform }.to raise_error(expected_error_message)
+ end
+ end
+
+ context 'when the destination table exists' do
+ before do
+ connection.execute(<<~SQL)
+ CREATE TABLE #{source_table} (
+ id serial NOT NULL PRIMARY KEY,
+ col1 int NOT NULL,
+ col2 text NOT NULL,
+ created_at timestamptz NOT NULL
+ )
+ SQL
+
+ connection.execute(<<~SQL)
+ CREATE TABLE #{destination_table} (
+ id serial NOT NULL,
+ col1 int NOT NULL,
+ col2 text NOT NULL,
+ created_at timestamptz NOT NULL,
+ PRIMARY KEY (id, created_at)
+ ) PARTITION BY RANGE (created_at)
+ SQL
+
+ connection.execute(<<~SQL)
+ CREATE TABLE #{destination_table}_202001 PARTITION OF #{destination_table}
+ FOR VALUES FROM ('2020-01-01') TO ('2020-02-01')
+ SQL
+
+ connection.execute(<<~SQL)
+ CREATE TABLE #{destination_table}_202002 PARTITION OF #{destination_table}
+ FOR VALUES FROM ('2020-02-01') TO ('2020-03-01')
+ SQL
+
+ source_model.table_name = source_table
+ destination_model.table_name = destination_table
+ end
+
+ after do
+ connection.drop_table source_table
+ connection.drop_table destination_table
+ end
+
+ let(:timestamp) { Time.utc(2020, 1, 2).round }
+ let!(:source1) { create_source_record(timestamp) }
+ let!(:source2) { create_source_record(timestamp + 1.day) }
+ let!(:source3) { create_source_record(timestamp + 1.month) }
+
+ it 'copies data into the destination table idempotently' do
+ expect(destination_model.count).to eq(0)
+
+ backfill_job.perform
+
+ expect(destination_model.count).to eq(3)
+
+ source_model.find_each do |source_record|
+ destination_record = destination_model.find_by_id(source_record.id)
+
+ expect(destination_record.attributes).to eq(source_record.attributes)
+ end
+
+ backfill_job.perform
+
+ expect(destination_model.count).to eq(3)
+ end
+
+ it 'breaks the assigned batch into smaller sub batches' do
+ expect_next_instance_of(Gitlab::Database::PartitioningMigrationHelpers::BulkCopy) do |bulk_copy|
+ expect(bulk_copy).to receive(:copy_between).with(source1.id, source2.id)
+ expect(bulk_copy).to receive(:copy_between).with(source3.id, source3.id)
+ end
+
+ backfill_job.perform
+ end
+ end
+ end
+
+ def create_source_record(timestamp)
+ source_model.create!(col1: 123, col2: 'original value', created_at: timestamp)
+ end
+end
diff --git a/spec/lib/gitlab/background_migration/backfill_prepared_at_merge_requests_spec.rb b/spec/lib/gitlab/background_migration/backfill_prepared_at_merge_requests_spec.rb
new file mode 100644
index 00000000000..28ecfae1bd4
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/backfill_prepared_at_merge_requests_spec.rb
@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::BackfillPreparedAtMergeRequests, :migration,
+ feature_category: :code_review_workflow, schema: 20230202135758 do
+ let(:namespaces) { table(:namespaces) }
+ let(:projects) { table(:projects) }
+ let(:mr_table) { table(:merge_requests) }
+
+ let(:namespace) { namespaces.create!(name: 'batchtest1', type: 'Group', path: 'space1') }
+ let(:proj_namespace) { namespaces.create!(name: 'proj1', path: 'proj1', type: 'Project', parent_id: namespace.id) }
+ let(:project) do
+ projects.create!(name: 'proj1', path: 'proj1', namespace_id: namespace.id, project_namespace_id: proj_namespace.id)
+ end
+
+ it 'updates merge requests with prepared_at nil' do
+ time = Time.current
+
+ mr_1 = mr_table.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature',
+ prepared_at: nil, merge_status: 'checking')
+ mr_2 = mr_table.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature',
+ prepared_at: nil, merge_status: 'preparing')
+ mr_3 = mr_table.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature',
+ prepared_at: time)
+ mr_4 = mr_table.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature',
+ prepared_at: time, merge_status: 'checking')
+ mr_5 = mr_table.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature',
+ prepared_at: time, merge_status: 'preparing')
+
+ test_worker = described_class.new(
+ start_id: mr_1.id,
+ end_id: [(mr_5.id + 1), 100].max,
+ batch_table: :merge_requests,
+ batch_column: :id,
+ sub_batch_size: 10,
+ pause_ms: 0,
+ connection: ApplicationRecord.connection
+ )
+
+ expect(mr_1.prepared_at).to be_nil
+ expect(mr_2.prepared_at).to be_nil
+ expect(mr_3.prepared_at.to_i).to eq(time.to_i)
+ expect(mr_4.prepared_at.to_i).to eq(time.to_i)
+ expect(mr_5.prepared_at.to_i).to eq(time.to_i)
+
+ test_worker.perform
+
+ expect(mr_1.reload.prepared_at.to_i).to eq(mr_1.created_at.to_i)
+ expect(mr_2.reload.prepared_at).to be_nil
+ expect(mr_3.reload.prepared_at.to_i).to eq(time.to_i)
+ expect(mr_4.reload.prepared_at.to_i).to eq(time.to_i)
+ expect(mr_5.reload.prepared_at.to_i).to eq(time.to_i)
+ end
+end
diff --git a/spec/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level_spec.rb
index fd6c055b9f6..47ff2883fb2 100644
--- a/spec/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level_spec.rb
+++ b/spec/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level_spec.rb
@@ -101,14 +101,15 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillProjectFeaturePackageRegistr
end
subject(:perform_migration) do
- described_class.new(start_id: project1.id,
- end_id: project5.id,
- batch_table: :projects,
- batch_column: :id,
- sub_batch_size: 2,
- pause_ms: 0,
- connection: ActiveRecord::Base.connection)
- .perform
+ described_class.new(
+ start_id: project1.id,
+ end_id: project5.id,
+ batch_table: :projects,
+ batch_column: :id,
+ sub_batch_size: 2,
+ pause_ms: 0,
+ connection: ActiveRecord::Base.connection
+ ).perform
end
it 'backfills project_features.package_registry_access_level', :aggregate_failures do
diff --git a/spec/lib/gitlab/background_migration/backfill_project_member_namespace_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_member_namespace_id_spec.rb
index ca7ca41a33e..96f49624d22 100644
--- a/spec/lib/gitlab/background_migration/backfill_project_member_namespace_id_spec.rb
+++ b/spec/lib/gitlab/background_migration/backfill_project_member_namespace_id_spec.rb
@@ -4,10 +4,12 @@ require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::BackfillProjectMemberNamespaceId, :migration, schema: 20220516054011 do
let(:migration) do
- described_class.new(start_id: 1, end_id: 10,
- batch_table: table_name, batch_column: batch_column,
- sub_batch_size: sub_batch_size, pause_ms: pause_ms,
- connection: ApplicationRecord.connection)
+ described_class.new(
+ start_id: 1, end_id: 10,
+ batch_table: table_name, batch_column: batch_column,
+ sub_batch_size: sub_batch_size, pause_ms: pause_ms,
+ connection: ApplicationRecord.connection
+ )
end
let(:members_table) { table(:members) }
@@ -35,37 +37,55 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillProjectMemberNamespaceId, :m
projects_table.create!(id: 102, name: 'project3', path: 'project3', namespace_id: 202, project_namespace_id: 302)
# project1, no member namespace (fill in)
- members_table.create!(id: 1, source_id: 100,
- source_type: 'Project', type: 'ProjectMember',
- member_namespace_id: nil, access_level: 10, notification_level: 3)
+ members_table.create!(
+ id: 1, source_id: 100,
+ source_type: 'Project', type: 'ProjectMember',
+ member_namespace_id: nil, access_level: 10, notification_level: 3
+ )
+
# bogus source id, no member namespace id (do nothing)
- members_table.create!(id: 2, source_id: non_existing_record_id,
- source_type: 'Project', type: 'ProjectMember',
- member_namespace_id: nil, access_level: 10, notification_level: 3)
+ members_table.create!(
+ id: 2, source_id: non_existing_record_id,
+ source_type: 'Project', type: 'ProjectMember',
+ member_namespace_id: nil, access_level: 10, notification_level: 3
+ )
+
# project3, existing member namespace id (do nothing)
- members_table.create!(id: 3, source_id: 102,
- source_type: 'Project', type: 'ProjectMember',
- member_namespace_id: 300, access_level: 10, notification_level: 3)
+ members_table.create!(
+ id: 3, source_id: 102,
+ source_type: 'Project', type: 'ProjectMember',
+ member_namespace_id: 300, access_level: 10, notification_level: 3
+ )
# Group memberships (do not change)
# group1, no member namespace (do nothing)
- members_table.create!(id: 4, source_id: 201,
- source_type: 'Namespace', type: 'GroupMember',
- member_namespace_id: nil, access_level: 10, notification_level: 3)
+ members_table.create!(
+ id: 4, source_id: 201,
+ source_type: 'Namespace', type: 'GroupMember',
+ member_namespace_id: nil, access_level: 10, notification_level: 3
+ )
+
# group2, existing member namespace (do nothing)
- members_table.create!(id: 5, source_id: 202,
- source_type: 'Namespace', type: 'GroupMember',
- member_namespace_id: 201, access_level: 10, notification_level: 3)
+ members_table.create!(
+ id: 5, source_id: 202,
+ source_type: 'Namespace', type: 'GroupMember',
+ member_namespace_id: 201, access_level: 10, notification_level: 3
+ )
# Project Namespace memberships (do not change)
# project namespace, existing member namespace (do nothing)
- members_table.create!(id: 6, source_id: 300,
- source_type: 'Namespace', type: 'ProjectNamespaceMember',
- member_namespace_id: 201, access_level: 10, notification_level: 3)
+ members_table.create!(
+ id: 6, source_id: 300,
+ source_type: 'Namespace', type: 'ProjectNamespaceMember',
+ member_namespace_id: 201, access_level: 10, notification_level: 3
+ )
+
# project namespace, not member namespace (do nothing)
- members_table.create!(id: 7, source_id: 301,
- source_type: 'Namespace', type: 'ProjectNamespaceMember',
- member_namespace_id: 201, access_level: 10, notification_level: 3)
+ members_table.create!(
+ id: 7, source_id: 301,
+ source_type: 'Namespace', type: 'ProjectNamespaceMember',
+ member_namespace_id: 201, access_level: 10, notification_level: 3
+ )
end
it 'backfills `member_namespace_id` for the selected records', :aggregate_failures do
diff --git a/spec/lib/gitlab/background_migration/backfill_project_namespace_details_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_namespace_details_spec.rb
index 01daf16d10c..aac17a426b5 100644
--- a/spec/lib/gitlab/background_migration/backfill_project_namespace_details_spec.rb
+++ b/spec/lib/gitlab/background_migration/backfill_project_namespace_details_spec.rb
@@ -8,32 +8,41 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillProjectNamespaceDetails, :mi
let!(:projects) { table(:projects) }
subject(:perform_migration) do
- described_class.new(start_id: projects.minimum(:id),
- end_id: projects.maximum(:id),
- batch_table: :projects,
- batch_column: :id,
- sub_batch_size: 2,
- pause_ms: 0,
- connection: ActiveRecord::Base.connection)
- .perform
+ described_class.new(
+ start_id: projects.minimum(:id),
+ end_id: projects.maximum(:id),
+ batch_table: :projects,
+ batch_column: :id,
+ sub_batch_size: 2,
+ pause_ms: 0,
+ connection: ActiveRecord::Base.connection
+ ).perform
end
describe '#perform' do
it 'creates details for all project namespaces in range' do
- namespaces.create!(id: 5, name: 'test1', path: 'test1', description: "Some description1",
- description_html: "Some description html1", cached_markdown_version: 4)
+ namespaces.create!(
+ id: 5, name: 'test1', path: 'test1', description: "Some description1",
+ description_html: "Some description html1", cached_markdown_version: 4
+ )
project_namespace1 = namespaces.create!(id: 6, name: 'test2', path: 'test2', type: 'Project')
- namespaces.create!(id: 7, name: 'test3', path: 'test3', description: "Some description3",
- description_html: "Some description html3", cached_markdown_version: 4)
+ namespaces.create!(
+ id: 7, name: 'test3', path: 'test3', description: "Some description3",
+ description_html: "Some description html3", cached_markdown_version: 4
+ )
project_namespace2 = namespaces.create!(id: 8, name: 'test4', path: 'test4', type: 'Project')
- project1 = projects.create!(namespace_id: project_namespace1.id, name: 'gitlab1', path: 'gitlab1',
- project_namespace_id: project_namespace1.id, description: "Some description2",
- description_html: "Some description html2", cached_markdown_version: 4)
- project2 = projects.create!(namespace_id: project_namespace2.id, name: 'gitlab2', path: 'gitlab2',
- project_namespace_id: project_namespace2.id,
- description: "Some description3",
- description_html: "Some description html4", cached_markdown_version: 4)
+ project1 = projects.create!(
+ namespace_id: project_namespace1.id, name: 'gitlab1', path: 'gitlab1',
+ project_namespace_id: project_namespace1.id, description: "Some description2",
+ description_html: "Some description html2", cached_markdown_version: 4
+ )
+ project2 = projects.create!(
+ namespace_id: project_namespace2.id, name: 'gitlab2', path: 'gitlab2',
+ project_namespace_id: project_namespace2.id,
+ description: "Some description3",
+ description_html: "Some description html4", cached_markdown_version: 4
+ )
namespace_details.delete_all
diff --git a/spec/lib/gitlab/background_migration/backfill_project_wiki_repositories_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_wiki_repositories_spec.rb
new file mode 100644
index 00000000000..e81bd0604e6
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/backfill_project_wiki_repositories_spec.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe(
+ Gitlab::BackgroundMigration::BackfillProjectWikiRepositories,
+ schema: 20230306195007,
+ feature_category: :geo_replication) do
+ let!(:namespaces) { table(:namespaces) }
+ let!(:projects) { table(:projects) }
+ let!(:project_wiki_repositories) { table(:project_wiki_repositories) }
+
+ subject(:migration) do
+ described_class.new(
+ start_id: projects.minimum(:id),
+ end_id: projects.maximum(:id),
+ batch_table: :projects,
+ batch_column: :id,
+ sub_batch_size: 2,
+ pause_ms: 0,
+ connection: ActiveRecord::Base.connection
+ )
+ end
+
+ describe '#perform' do
+ it 'creates project_wiki_repositories entries for all projects in range' do
+ namespace1 = create_namespace('test1')
+ namespace2 = create_namespace('test2')
+ project1 = create_project(namespace1, 'test1')
+ project2 = create_project(namespace2, 'test2')
+ project_wiki_repositories.create!(project_id: project2.id)
+
+ expect { migration.perform }
+ .to change { project_wiki_repositories.pluck(:project_id) }
+ .from([project2.id])
+ .to match_array([project1.id, project2.id])
+ end
+
+ it 'does nothing if project_id already exist in project_wiki_repositories' do
+ namespace = create_namespace('test1')
+ project = create_project(namespace, 'test1')
+ project_wiki_repositories.create!(project_id: project.id)
+
+ expect { migration.perform }
+ .not_to change { project_wiki_repositories.pluck(:project_id) }
+ end
+
+ def create_namespace(name)
+ namespaces.create!(
+ name: name,
+ path: name,
+ type: 'Project'
+ )
+ end
+
+ def create_project(namespace, name)
+ projects.create!(
+ namespace_id: namespace.id,
+ project_namespace_id: namespace.id,
+ name: name,
+ path: name
+ )
+ end
+ end
+end
diff --git a/spec/lib/gitlab/background_migration/backfill_releases_author_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_releases_author_id_spec.rb
index d8ad10849f2..898f241a930 100644
--- a/spec/lib/gitlab/background_migration/backfill_releases_author_id_spec.rb
+++ b/spec/lib/gitlab/background_migration/backfill_releases_author_id_spec.rb
@@ -10,35 +10,52 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillReleasesAuthorId,
let!(:test_user) { user_table.create!(name: 'test', email: 'test@example.com', username: 'test', projects_limit: 10) }
let!(:ghost_user) do
- user_table.create!(name: 'ghost', email: 'ghost@example.com',
- username: 'ghost', user_type: User::USER_TYPES['ghost'], projects_limit: 100000)
+ user_table.create!(
+ name: 'ghost', email: 'ghost@example.com',
+ username: 'ghost', user_type: User::USER_TYPES['ghost'], projects_limit: 100000
+ )
end
let(:migration) do
- described_class.new(start_id: 1, end_id: 100,
- batch_table: :releases, batch_column: :id,
- sub_batch_size: 10, pause_ms: 0,
- job_arguments: [ghost_user.id],
- connection: ApplicationRecord.connection)
+ described_class.new(
+ start_id: 1, end_id: 100,
+ batch_table: :releases, batch_column: :id,
+ sub_batch_size: 10, pause_ms: 0,
+ job_arguments: [ghost_user.id],
+ connection: ApplicationRecord.connection
+ )
end
subject(:perform_migration) { migration.perform }
before do
- releases_table.create!(tag: 'tag1', name: 'tag1',
- released_at: (date_time - 1.minute), author_id: test_user.id)
- releases_table.create!(tag: 'tag2', name: 'tag2',
- released_at: (date_time - 2.minutes), author_id: test_user.id)
- releases_table.new(tag: 'tag3', name: 'tag3',
- released_at: (date_time - 3.minutes), author_id: nil).save!(validate: false)
- releases_table.new(tag: 'tag4', name: 'tag4',
- released_at: (date_time - 4.minutes), author_id: nil).save!(validate: false)
- releases_table.new(tag: 'tag5', name: 'tag5',
- released_at: (date_time - 5.minutes), author_id: nil).save!(validate: false)
- releases_table.create!(tag: 'tag6', name: 'tag6',
- released_at: (date_time - 6.minutes), author_id: test_user.id)
- releases_table.new(tag: 'tag7', name: 'tag7',
- released_at: (date_time - 7.minutes), author_id: nil).save!(validate: false)
+ releases_table.create!(
+ tag: 'tag1', name: 'tag1', released_at: (date_time - 1.minute), author_id: test_user.id
+ )
+
+ releases_table.create!(
+ tag: 'tag2', name: 'tag2', released_at: (date_time - 2.minutes), author_id: test_user.id
+ )
+
+ releases_table.new(
+ tag: 'tag3', name: 'tag3', released_at: (date_time - 3.minutes), author_id: nil
+ ).save!(validate: false)
+
+ releases_table.new(
+ tag: 'tag4', name: 'tag4', released_at: (date_time - 4.minutes), author_id: nil
+ ).save!(validate: false)
+
+ releases_table.new(
+ tag: 'tag5', name: 'tag5', released_at: (date_time - 5.minutes), author_id: nil
+ ).save!(validate: false)
+
+ releases_table.create!(
+ tag: 'tag6', name: 'tag6', released_at: (date_time - 6.minutes), author_id: test_user.id
+ )
+
+ releases_table.new(
+ tag: 'tag7', name: 'tag7', released_at: (date_time - 7.minutes), author_id: nil
+ ).save!(validate: false)
end
it 'backfills `author_id` for the selected records', :aggregate_failures do
diff --git a/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb b/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb
index 80fd86e90bb..d8874cb811b 100644
--- a/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb
+++ b/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb
@@ -2,8 +2,8 @@
require 'spec_helper'
-RSpec.describe Gitlab::BackgroundMigration::BackfillSnippetRepositories, :migration, schema: 20210826171758,
-feature_category: :source_code_management do
+RSpec.describe Gitlab::BackgroundMigration::BackfillSnippetRepositories, :migration, schema: 20211202041233,
+ feature_category: :source_code_management do
let(:gitlab_shell) { Gitlab::Shell.new }
let(:users) { table(:users) }
let(:snippets) { table(:snippets) }
@@ -14,24 +14,28 @@ feature_category: :source_code_management do
let(:user_name) { 'Test' }
let!(:user) do
- users.create!(id: 1,
- email: 'user@example.com',
- projects_limit: 10,
- username: 'test',
- name: user_name,
- state: user_state,
- last_activity_on: 1.minute.ago,
- user_type: user_type,
- confirmed_at: 1.day.ago)
+ users.create!(
+ id: 1,
+ email: 'user@example.com',
+ projects_limit: 10,
+ username: 'test',
+ name: user_name,
+ state: user_state,
+ last_activity_on: 1.minute.ago,
+ user_type: user_type,
+ confirmed_at: 1.day.ago
+ )
end
let!(:migration_bot) do
- users.create!(id: 100,
- email: "noreply+gitlab-migration-bot%s@#{Settings.gitlab.host}",
- user_type: HasUserType::USER_TYPES[:migration_bot],
- name: 'GitLab Migration Bot',
- projects_limit: 10,
- username: 'bot')
+ users.create!(
+ id: 100,
+ email: "noreply+gitlab-migration-bot%s@#{Settings.gitlab.host}",
+ user_type: HasUserType::USER_TYPES[:migration_bot],
+ name: 'GitLab Migration Bot',
+ projects_limit: 10,
+ username: 'bot'
+ )
end
let!(:snippet_with_repo) { snippets.create!(id: 1, type: 'PersonalSnippet', author_id: user.id, file_name: file_name, content: content) }
@@ -260,15 +264,17 @@ feature_category: :source_code_management do
context 'when both user name and snippet file_name are invalid' do
let(:user_name) { '.' }
let!(:other_user) do
- users.create!(id: 2,
- email: 'user2@example.com',
- projects_limit: 10,
- username: 'test2',
- name: 'Test2',
- state: user_state,
- last_activity_on: 1.minute.ago,
- user_type: user_type,
- confirmed_at: 1.day.ago)
+ users.create!(
+ id: 2,
+ email: 'user2@example.com',
+ projects_limit: 10,
+ username: 'test2',
+ name: 'Test2',
+ state: user_state,
+ last_activity_on: 1.minute.ago,
+ user_type: user_type,
+ confirmed_at: 1.day.ago
+ )
end
let!(:invalid_snippet) { snippets.create!(id: 4, type: 'PersonalSnippet', author_id: user.id, file_name: '.', content: content) }
@@ -322,10 +328,12 @@ feature_category: :source_code_management do
end
def raw_repository(snippet)
- Gitlab::Git::Repository.new('default',
- "#{disk_path(snippet)}.git",
- Gitlab::GlRepository::SNIPPET.identifier_for_container(snippet),
- "@snippets/#{snippet.id}")
+ Gitlab::Git::Repository.new(
+ 'default',
+ "#{disk_path(snippet)}.git",
+ Gitlab::GlRepository::SNIPPET.identifier_for_container(snippet),
+ "@snippets/#{snippet.id}"
+ )
end
def hashed_repository(snippet)
diff --git a/spec/lib/gitlab/background_migration/backfill_upvotes_count_on_issues_spec.rb b/spec/lib/gitlab/background_migration/backfill_upvotes_count_on_issues_spec.rb
deleted file mode 100644
index 7142aea3ab2..00000000000
--- a/spec/lib/gitlab/background_migration/backfill_upvotes_count_on_issues_spec.rb
+++ /dev/null
@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-RSpec.describe Gitlab::BackgroundMigration::BackfillUpvotesCountOnIssues, schema: 20210826171758 do
- let(:award_emoji) { table(:award_emoji) }
-
- let!(:namespace) { table(:namespaces).create!(name: 'namespace', path: 'namespace') }
- let!(:project1) { table(:projects).create!(namespace_id: namespace.id) }
- let!(:project2) { table(:projects).create!(namespace_id: namespace.id) }
- let!(:issue1) { table(:issues).create!(project_id: project1.id) }
- let!(:issue2) { table(:issues).create!(project_id: project2.id) }
- let!(:issue3) { table(:issues).create!(project_id: project2.id) }
- let!(:issue4) { table(:issues).create!(project_id: project2.id) }
-
- describe '#perform' do
- before do
- add_upvotes(issue1, :thumbsdown, 1)
- add_upvotes(issue2, :thumbsup, 2)
- add_upvotes(issue2, :thumbsdown, 1)
- add_upvotes(issue3, :thumbsup, 3)
- add_upvotes(issue4, :thumbsup, 4)
- end
-
- it 'updates upvotes_count' do
- subject.perform(issue1.id, issue4.id)
-
- expect(issue1.reload.upvotes_count).to eq(0)
- expect(issue2.reload.upvotes_count).to eq(2)
- expect(issue3.reload.upvotes_count).to eq(3)
- expect(issue4.reload.upvotes_count).to eq(4)
- end
- end
-
- private
-
- def add_upvotes(issue, name, count)
- count.times do
- award_emoji.create!(
- name: name.to_s,
- awardable_type: 'Issue',
- awardable_id: issue.id
- )
- end
- end
-end
diff --git a/spec/lib/gitlab/background_migration/backfill_user_namespace_spec.rb b/spec/lib/gitlab/background_migration/backfill_user_namespace_spec.rb
deleted file mode 100644
index 395248b786d..00000000000
--- a/spec/lib/gitlab/background_migration/backfill_user_namespace_spec.rb
+++ /dev/null
@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-RSpec.describe Gitlab::BackgroundMigration::BackfillUserNamespace, :migration, schema: 20210930211936 do
- let(:migration) { described_class.new }
- let(:namespaces_table) { table(:namespaces) }
-
- let(:table_name) { 'namespaces' }
- let(:batch_column) { :id }
- let(:sub_batch_size) { 100 }
- let(:pause_ms) { 0 }
-
- subject(:perform_migration) { migration.perform(1, 10, table_name, batch_column, sub_batch_size, pause_ms) }
-
- before do
- namespaces_table.create!(id: 1, name: 'test1', path: 'test1', type: nil)
- namespaces_table.create!(id: 2, name: 'test2', path: 'test2', type: 'User')
- namespaces_table.create!(id: 3, name: 'test3', path: 'test3', type: 'Group')
- namespaces_table.create!(id: 4, name: 'test4', path: 'test4', type: nil)
- namespaces_table.create!(id: 11, name: 'test11', path: 'test11', type: nil)
- end
-
- it 'backfills `type` for the selected records', :aggregate_failures do
- queries = ActiveRecord::QueryRecorder.new do
- perform_migration
- end
-
- expect(queries.count).to eq(3)
- expect(namespaces_table.where(type: 'User').count).to eq 3
- expect(namespaces_table.where(type: 'User').pluck(:id)).to match_array([1, 2, 4])
- end
-
- it 'tracks timings of queries' do
- expect(migration.batch_metrics.timings).to be_empty
-
- expect { perform_migration }.to change { migration.batch_metrics.timings }
- end
-end
diff --git a/spec/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent_spec.rb b/spec/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent_spec.rb
index f642ec8c20d..3f1a57434a7 100644
--- a/spec/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent_spec.rb
+++ b/spec/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent_spec.rb
@@ -4,10 +4,12 @@ require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::BackfillVulnerabilityReadsClusterAgent, :migration, schema: 20220525221133 do # rubocop:disable Layout/LineLength
let(:migration) do
- described_class.new(start_id: 1, end_id: 10,
- batch_table: table_name, batch_column: batch_column,
- sub_batch_size: sub_batch_size, pause_ms: pause_ms,
- connection: ApplicationRecord.connection)
+ described_class.new(
+ start_id: 1, end_id: 10,
+ batch_table: table_name, batch_column: batch_column,
+ sub_batch_size: sub_batch_size, pause_ms: pause_ms,
+ connection: ApplicationRecord.connection
+ )
end
let(:users_table) { table(:users) }
diff --git a/spec/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues_spec.rb b/spec/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues_spec.rb
index 5f93424faf6..c7e4095a488 100644
--- a/spec/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues_spec.rb
+++ b/spec/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues_spec.rb
@@ -2,7 +2,10 @@
require 'spec_helper'
-RSpec.describe Gitlab::BackgroundMigration::BackfillWorkItemTypeIdForIssues, :migration, schema: 20220825142324 do
+RSpec.describe Gitlab::BackgroundMigration::BackfillWorkItemTypeIdForIssues,
+ :migration,
+ schema: 20220825142324,
+ feature_category: :team_planning do
let(:batch_column) { 'id' }
let(:sub_batch_size) { 2 }
let(:pause_ms) { 0 }
@@ -13,6 +16,7 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillWorkItemTypeIdForIssues, :mi
let(:project) { table(:projects).create!(namespace_id: namespace.id, project_namespace_id: namespace.id) }
let(:issues_table) { table(:issues) }
let(:issue_type) { table(:work_item_types).find_by!(namespace_id: nil, base_type: issue_type_enum[:issue]) }
+ let(:task_type) { table(:work_item_types).find_by!(namespace_id: nil, base_type: issue_type_enum[:task]) }
let(:issue1) { issues_table.create!(project_id: project.id, issue_type: issue_type_enum[:issue]) }
let(:issue2) { issues_table.create!(project_id: project.id, issue_type: issue_type_enum[:issue]) }
@@ -25,7 +29,7 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillWorkItemTypeIdForIssues, :mi
let(:start_id) { issue1.id }
let(:end_id) { requirement1.id }
- let(:all_issues) { [issue1, issue2, issue3, incident1, test_case1, requirement1] }
+ let!(:all_issues) { [issue1, issue2, issue3, incident1, test_case1, requirement1] }
let(:migration) do
described_class.new(
@@ -52,6 +56,27 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillWorkItemTypeIdForIssues, :mi
expect(all_issues - [issue1, issue2, issue3]).to all(have_attributes(work_item_type_id: nil))
end
+ context 'when a record already had a work_item_type_id assigned' do
+ let!(:issue4) do
+ issues_table.create!(
+ project_id: project.id,
+ issue_type: issue_type_enum[:issue],
+ work_item_type_id: task_type.id
+ )
+ end
+
+ let(:end_id) { issue4.id }
+
+ it 'ovewrites the work_item_type_id' do
+ # creating with the wrong issue_type/work_item_type_id on purpose so we can test
+ # that the migration is capable of fixing such inconsistencies
+ expect do
+ migrate
+ issue4.reload
+ end.to change { issue4.work_item_type_id }.from(task_type.id).to(issue_type.id)
+ end
+ end
+
it 'tracks timings of queries' do
expect(migration.batch_metrics.timings).to be_empty
diff --git a/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb b/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb
index faaaccfdfaf..781bf93dd85 100644
--- a/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb
+++ b/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb
@@ -301,6 +301,28 @@ RSpec.describe Gitlab::BackgroundMigration::BatchedMigrationJob do
perform_job
end
+ context 'when using a sub batch exception for timeouts' do
+ let(:job_class) do
+ Class.new(described_class) do
+ operation_name :update
+
+ def perform(*_)
+ each_sub_batch { raise ActiveRecord::StatementTimeout } # rubocop:disable Lint/UnreachableLoop
+ end
+ end
+ end
+
+ let(:job_instance) do
+ job_class.new(start_id: 1, end_id: 10, batch_table: '_test_table', batch_column: 'id',
+ sub_batch_size: 2, pause_ms: 1000, connection: connection,
+ sub_batch_exception: StandardError)
+ end
+
+ it 'raises the expected error type' do
+ expect { job_instance.perform }.to raise_error(StandardError)
+ end
+ end
+
context 'when batching_arguments are given' do
it 'forwards them for batching' do
expect(job_instance).to receive(:base_relation).and_return(test_table)
diff --git a/spec/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects_spec.rb b/spec/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects_spec.rb
deleted file mode 100644
index 5ffe665f0ad..00000000000
--- a/spec/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects_spec.rb
+++ /dev/null
@@ -1,85 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-RSpec.describe Gitlab::BackgroundMigration::CleanupOrphanedLfsObjectsProjects, schema: 20210826171758 do
- let(:lfs_objects_projects) { table(:lfs_objects_projects) }
- let(:lfs_objects) { table(:lfs_objects) }
- let(:projects) { table(:projects) }
- let(:namespaces) { table(:namespaces) }
-
- let(:namespace) { namespaces.create!(name: 'namespace', path: 'namespace') }
- let(:project) { projects.create!(namespace_id: namespace.id) }
- let(:another_project) { projects.create!(namespace_id: namespace.id) }
- let(:lfs_object) { lfs_objects.create!(oid: 'abcdef', size: 1) }
- let(:another_lfs_object) { lfs_objects.create!(oid: '1abcde', size: 2) }
-
- let!(:without_object1) { create_object(project_id: project.id) }
- let!(:without_object2) { create_object(project_id: another_project.id) }
- let!(:without_object3) { create_object(project_id: another_project.id) }
- let!(:with_project_and_object1) { create_object(project_id: project.id, lfs_object_id: lfs_object.id) }
- let!(:with_project_and_object2) { create_object(project_id: project.id, lfs_object_id: another_lfs_object.id) }
- let!(:with_project_and_object3) { create_object(project_id: another_project.id, lfs_object_id: another_lfs_object.id) }
- let!(:without_project1) { create_object(lfs_object_id: lfs_object.id) }
- let!(:without_project2) { create_object(lfs_object_id: another_lfs_object.id) }
- let!(:without_project_and_object) { create_object }
-
- def create_object(project_id: non_existing_record_id, lfs_object_id: non_existing_record_id)
- lfs_objects_project = nil
-
- ActiveRecord::Base.connection.disable_referential_integrity do
- lfs_objects_project = lfs_objects_projects.create!(project_id: project_id, lfs_object_id: lfs_object_id)
- end
-
- lfs_objects_project
- end
-
- subject { described_class.new }
-
- describe '#perform' do
- it 'lfs_objects_projects without an existing lfs object or project are removed' do
- subject.perform(without_object1.id, without_object3.id)
-
- expect(lfs_objects_projects.all).to match_array(
- [
- with_project_and_object1, with_project_and_object2, with_project_and_object3,
- without_project1, without_project2, without_project_and_object
- ])
-
- subject.perform(with_project_and_object1.id, with_project_and_object3.id)
-
- expect(lfs_objects_projects.all).to match_array(
- [
- with_project_and_object1, with_project_and_object2, with_project_and_object3,
- without_project1, without_project2, without_project_and_object
- ])
-
- subject.perform(without_project1.id, without_project_and_object.id)
-
- expect(lfs_objects_projects.all).to match_array(
- [
- with_project_and_object1, with_project_and_object2, with_project_and_object3
- ])
-
- expect(lfs_objects.ids).to contain_exactly(lfs_object.id, another_lfs_object.id)
- expect(projects.ids).to contain_exactly(project.id, another_project.id)
- end
-
- it 'cache for affected projects is being reset' do
- expect(ProjectCacheWorker).to receive(:bulk_perform_in) do |delay, args|
- expect(delay).to eq(1.minute)
- expect(args).to match_array([[project.id, [], [:lfs_objects_size]], [another_project.id, [], [:lfs_objects_size]]])
- end
-
- subject.perform(without_object1.id, with_project_and_object1.id)
-
- expect(ProjectCacheWorker).not_to receive(:bulk_perform_in)
-
- subject.perform(with_project_and_object1.id, with_project_and_object3.id)
-
- expect(ProjectCacheWorker).not_to receive(:bulk_perform_in)
-
- subject.perform(without_project1.id, without_project_and_object.id)
- end
- end
-end
diff --git a/spec/lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at_spec.rb b/spec/lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at_spec.rb
new file mode 100644
index 00000000000..ade16c0a780
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at_spec.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::CleanupPersonalAccessTokensWithNilExpiresAt, schema: 20230510062503, feature_category: :system_access do # rubocop:disable Layout/LineLength
+ let(:personal_access_tokens_table) { table(:personal_access_tokens) }
+ let(:users_table) { table(:users) }
+ let(:expires_at_default) { described_class::EXPIRES_AT_DEFAULT }
+
+ subject(:perform_migration) do
+ described_class.new(
+ start_id: 1,
+ end_id: 30,
+ batch_table: :personal_access_tokens,
+ batch_column: :id,
+ sub_batch_size: 3,
+ pause_ms: 0,
+ connection: ActiveRecord::Base.connection
+ ).perform
+ end
+
+ before do
+ user = users_table.create!(name: 'PAT_USER', email: 'pat_user@gmail.com', username: "pat_user1", projects_limit: 0)
+ personal_access_tokens_table.create!(user_id: user.id, name: "PAT#1", expires_at: expires_at_default + 1.day)
+ personal_access_tokens_table.create!(user_id: user.id, name: "PAT#2", expires_at: nil)
+ personal_access_tokens_table.create!(user_id: user.id, name: "PAT#3", expires_at: Time.zone.now + 2.days)
+ end
+
+ it 'adds expiry to personal access tokens', :aggregate_failures do
+ freeze_time do
+ expect(ActiveRecord::QueryRecorder.new { perform_migration }.count).to eq(3)
+
+ expect(personal_access_tokens_table.find_by_name("PAT#1").expires_at).to eq(expires_at_default.to_date + 1.day)
+ expect(personal_access_tokens_table.find_by_name("PAT#2").expires_at).to eq(expires_at_default.to_date)
+ expect(personal_access_tokens_table.find_by_name("PAT#3").expires_at).to eq(Time.zone.now.to_date + 2.days)
+ end
+ end
+end
diff --git a/spec/lib/gitlab/background_migration/delete_orphaned_deployments_spec.rb b/spec/lib/gitlab/background_migration/delete_orphaned_deployments_spec.rb
deleted file mode 100644
index 8f058c875a2..00000000000
--- a/spec/lib/gitlab/background_migration/delete_orphaned_deployments_spec.rb
+++ /dev/null
@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-RSpec.describe Gitlab::BackgroundMigration::DeleteOrphanedDeployments, :migration, schema: 20210826171758 do
- let!(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') }
- let!(:project) { table(:projects).create!(namespace_id: namespace.id) }
- let!(:environment) { table(:environments).create!(name: 'production', slug: 'production', project_id: project.id) }
- let(:background_migration_jobs) { table(:background_migration_jobs) }
-
- before do
- create_deployment!(environment.id, project.id)
- end
-
- it 'deletes only orphaned deployments' do
- expect(valid_deployments.pluck(:id)).not_to be_empty
-
- subject.perform(table(:deployments).minimum(:id), table(:deployments).maximum(:id))
-
- expect(valid_deployments.pluck(:id)).not_to be_empty
- end
-
- it 'marks jobs as done' do
- first_job = background_migration_jobs.create!(
- class_name: 'DeleteOrphanedDeployments',
- arguments: [table(:deployments).minimum(:id), table(:deployments).minimum(:id)]
- )
-
- subject.perform(table(:deployments).minimum(:id), table(:deployments).minimum(:id))
-
- expect(first_job.reload.status).to eq(Gitlab::Database::BackgroundMigrationJob.statuses[:succeeded])
- end
-
- private
-
- def valid_deployments
- table(:deployments).where('EXISTS (SELECT 1 FROM environments WHERE deployments.environment_id = environments.id)')
- end
-
- def orphaned_deployments
- table(:deployments).where('NOT EXISTS (SELECT 1 FROM environments WHERE deployments.environment_id = environments.id)')
- end
-
- def create_deployment!(environment_id, project_id)
- table(:deployments).create!(
- environment_id: environment_id,
- project_id: project_id,
- ref: 'master',
- tag: false,
- sha: 'x',
- status: 1,
- iid: table(:deployments).count + 1)
- end
-end
diff --git a/spec/lib/gitlab/background_migration/delete_orphaned_packages_dependencies_spec.rb b/spec/lib/gitlab/background_migration/delete_orphaned_packages_dependencies_spec.rb
new file mode 100644
index 00000000000..0d82717c7de
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/delete_orphaned_packages_dependencies_spec.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::DeleteOrphanedPackagesDependencies, schema: 20230303105806,
+ feature_category: :package_registry do
+ let!(:migration_attrs) do
+ {
+ start_id: 1,
+ end_id: 1000,
+ batch_table: :packages_dependencies,
+ batch_column: :id,
+ sub_batch_size: 500,
+ pause_ms: 0,
+ connection: ApplicationRecord.connection
+ }
+ end
+
+ let!(:migration) { described_class.new(**migration_attrs) }
+
+ let(:packages_dependencies) { table(:packages_dependencies) }
+
+ let!(:namespace) { table(:namespaces).create!(name: 'project', path: 'project', type: 'Project') }
+ let!(:project) do
+ table(:projects).create!(name: 'project', path: 'project', project_namespace_id: namespace.id,
+ namespace_id: namespace.id)
+ end
+
+ let!(:package) do
+ table(:packages_packages).create!(name: 'test', version: '1.2.3', package_type: 2, project_id: project.id)
+ end
+
+ let!(:orphan_dependency_1) { packages_dependencies.create!(name: 'dependency 1', version_pattern: '~0.0.1') }
+ let!(:orphan_dependency_2) { packages_dependencies.create!(name: 'dependency 2', version_pattern: '~0.0.2') }
+ let!(:orphan_dependency_3) { packages_dependencies.create!(name: 'dependency 3', version_pattern: '~0.0.3') }
+ let!(:linked_dependency) do
+ packages_dependencies.create!(name: 'dependency 4', version_pattern: '~0.0.4').tap do |dependency|
+ table(:packages_dependency_links).create!(package_id: package.id, dependency_id: dependency.id,
+ dependency_type: 'dependencies')
+ end
+ end
+
+ subject(:perform_migration) { migration.perform }
+
+ it 'executes 3 queries' do
+ queries = ActiveRecord::QueryRecorder.new do
+ perform_migration
+ end
+
+ expect(queries.count).to eq(3)
+ end
+
+ it 'deletes only orphaned dependencies' do
+ expect { perform_migration }.to change { packages_dependencies.count }.by(-3)
+ expect(packages_dependencies.all).to eq([linked_dependency])
+ end
+end
diff --git a/spec/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images_spec.rb b/spec/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images_spec.rb
deleted file mode 100644
index e7b0471810d..00000000000
--- a/spec/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images_spec.rb
+++ /dev/null
@@ -1,142 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-RSpec.describe Gitlab::BackgroundMigration::DisableExpirationPoliciesLinkedToNoContainerImages, :migration, schema: 20220326161803 do # rubocop:disable Layout/LineLength
- let!(:projects) { table(:projects) }
- let!(:container_expiration_policies) { table(:container_expiration_policies) }
- let!(:container_repositories) { table(:container_repositories) }
- let!(:namespaces) { table(:namespaces) }
-
- let!(:namespace) { namespaces.create!(name: 'test', path: 'test') }
-
- let!(:policy1) { create_expiration_policy(project_id: 1, enabled: true) }
- let!(:policy2) { create_expiration_policy(project_id: 2, enabled: false) }
- let!(:policy3) { create_expiration_policy(project_id: 3, enabled: false) }
- let!(:policy4) { create_expiration_policy(project_id: 4, enabled: true, with_images: true) }
- let!(:policy5) { create_expiration_policy(project_id: 5, enabled: false, with_images: true) }
- let!(:policy6) { create_expiration_policy(project_id: 6, enabled: false) }
- let!(:policy7) { create_expiration_policy(project_id: 7, enabled: true) }
- let!(:policy8) { create_expiration_policy(project_id: 8, enabled: true, with_images: true) }
- let!(:policy9) { create_expiration_policy(project_id: 9, enabled: true) }
-
- describe '#perform' do
- subject { described_class.new.perform(from_id, to_id) }
-
- shared_examples 'disabling policies with no images' do
- it 'disables the proper policies' do
- subject
-
- rows = container_expiration_policies.order(:project_id).to_h do |row|
- [row.project_id, row.enabled]
- end
- expect(rows).to eq(expected_rows)
- end
- end
-
- context 'the whole range' do
- let(:from_id) { 1 }
- let(:to_id) { 9 }
-
- it_behaves_like 'disabling policies with no images' do
- let(:expected_rows) do
- {
- 1 => false,
- 2 => false,
- 3 => false,
- 4 => true,
- 5 => false,
- 6 => false,
- 7 => false,
- 8 => true,
- 9 => false
- }
- end
- end
- end
-
- context 'a range with no policies to disable' do
- let(:from_id) { 2 }
- let(:to_id) { 6 }
-
- it_behaves_like 'disabling policies with no images' do
- let(:expected_rows) do
- {
- 1 => true,
- 2 => false,
- 3 => false,
- 4 => true,
- 5 => false,
- 6 => false,
- 7 => true,
- 8 => true,
- 9 => true
- }
- end
- end
- end
-
- context 'a range with only images' do
- let(:from_id) { 4 }
- let(:to_id) { 5 }
-
- it_behaves_like 'disabling policies with no images' do
- let(:expected_rows) do
- {
- 1 => true,
- 2 => false,
- 3 => false,
- 4 => true,
- 5 => false,
- 6 => false,
- 7 => true,
- 8 => true,
- 9 => true
- }
- end
- end
- end
-
- context 'a range with a single element' do
- let(:from_id) { 9 }
- let(:to_id) { 9 }
-
- it_behaves_like 'disabling policies with no images' do
- let(:expected_rows) do
- {
- 1 => true,
- 2 => false,
- 3 => false,
- 4 => true,
- 5 => false,
- 6 => false,
- 7 => true,
- 8 => true,
- 9 => false
- }
- end
- end
- end
- end
-
- def create_expiration_policy(project_id:, enabled:, with_images: false)
- projects.create!(id: project_id, namespace_id: namespace.id, name: "gitlab-#{project_id}")
-
- if with_images
- container_repositories.create!(project_id: project_id, name: "image-#{project_id}")
- end
-
- container_expiration_policies.create!(
- enabled: enabled,
- project_id: project_id
- )
- end
-
- def enabled_policies
- container_expiration_policies.where(enabled: true)
- end
-
- def disabled_policies
- container_expiration_policies.where(enabled: false)
- end
-end
diff --git a/spec/lib/gitlab/background_migration/drop_invalid_security_findings_spec.rb b/spec/lib/gitlab/background_migration/drop_invalid_security_findings_spec.rb
deleted file mode 100644
index 5fdd8683d06..00000000000
--- a/spec/lib/gitlab/background_migration/drop_invalid_security_findings_spec.rb
+++ /dev/null
@@ -1,57 +0,0 @@
-# frozen_string_literal: true
-require 'spec_helper'
-
-RSpec.describe Gitlab::BackgroundMigration::DropInvalidSecurityFindings, :suppress_gitlab_schemas_validate_connection,
- schema: 20211108211434 do
- let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user', type: Namespaces::UserNamespace.sti_name) }
- let(:project) { table(:projects).create!(namespace_id: namespace.id) }
-
- let(:pipelines) { table(:ci_pipelines) }
- let!(:pipeline) { pipelines.create!(project_id: project.id) }
-
- let(:ci_builds) { table(:ci_builds) }
- let!(:ci_build) { ci_builds.create! }
-
- let(:security_scans) { table(:security_scans) }
- let!(:security_scan) do
- security_scans.create!(
- scan_type: 1,
- status: 1,
- build_id: ci_build.id,
- project_id: project.id,
- pipeline_id: pipeline.id
- )
- end
-
- let(:vulnerability_scanners) { table(:vulnerability_scanners) }
- let!(:vulnerability_scanner) { vulnerability_scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') }
-
- let(:security_findings) { table(:security_findings) }
- let!(:security_finding_without_uuid) do
- security_findings.create!(
- severity: 1,
- confidence: 1,
- scan_id: security_scan.id,
- scanner_id: vulnerability_scanner.id,
- uuid: nil
- )
- end
-
- let!(:security_finding_with_uuid) do
- security_findings.create!(
- severity: 1,
- confidence: 1,
- scan_id: security_scan.id,
- scanner_id: vulnerability_scanner.id,
- uuid: 'bd95c085-71aa-51d7-9bb6-08ae669c262e'
- )
- end
-
- let(:sub_batch_size) { 10_000 }
-
- subject { described_class.new.perform(security_finding_without_uuid.id, security_finding_with_uuid.id, sub_batch_size) }
-
- it 'drops Security::Finding objects with no UUID' do
- expect { subject }.to change(security_findings, :count).from(2).to(1)
- end
-end
diff --git a/spec/lib/gitlab/background_migration/drop_invalid_vulnerabilities_spec.rb b/spec/lib/gitlab/background_migration/drop_invalid_vulnerabilities_spec.rb
deleted file mode 100644
index 8f3ef44e00c..00000000000
--- a/spec/lib/gitlab/background_migration/drop_invalid_vulnerabilities_spec.rb
+++ /dev/null
@@ -1,126 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-RSpec.describe Gitlab::BackgroundMigration::DropInvalidVulnerabilities, schema: 20210826171758 do
- let!(:background_migration_jobs) { table(:background_migration_jobs) }
- let!(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') }
- let!(:users) { table(:users) }
- let!(:user) { create_user! }
- let!(:project) { table(:projects).create!(id: 123, namespace_id: namespace.id) }
-
- let!(:scanners) { table(:vulnerability_scanners) }
- let!(:scanner) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') }
- let!(:different_scanner) { scanners.create!(project_id: project.id, external_id: 'test 2', name: 'test scanner 2') }
-
- let!(:vulnerabilities) { table(:vulnerabilities) }
- let!(:vulnerability_with_finding) do
- create_vulnerability!(
- project_id: project.id,
- author_id: user.id
- )
- end
-
- let!(:vulnerability_without_finding) do
- create_vulnerability!(
- project_id: project.id,
- author_id: user.id
- )
- end
-
- let!(:vulnerability_identifiers) { table(:vulnerability_identifiers) }
- let!(:primary_identifier) do
- vulnerability_identifiers.create!(
- project_id: project.id,
- external_type: 'uuid-v5',
- external_id: 'uuid-v5',
- fingerprint: '7e394d1b1eb461a7406d7b1e08f057a1cf11287a',
- name: 'Identifier for UUIDv5')
- end
-
- let!(:vulnerabilities_findings) { table(:vulnerability_occurrences) }
- let!(:finding) do
- create_finding!(
- vulnerability_id: vulnerability_with_finding.id,
- project_id: project.id,
- scanner_id: scanner.id,
- primary_identifier_id: primary_identifier.id
- )
- end
-
- let(:succeeded_status) { 1 }
- let(:pending_status) { 0 }
-
- it 'drops Vulnerabilities without any Findings' do
- expect(vulnerabilities.pluck(:id)).to eq([vulnerability_with_finding.id, vulnerability_without_finding.id])
-
- expect { subject.perform(vulnerability_with_finding.id, vulnerability_without_finding.id) }.to change(vulnerabilities, :count).by(-1)
-
- expect(vulnerabilities.pluck(:id)).to eq([vulnerability_with_finding.id])
- end
-
- it 'marks jobs as done' do
- background_migration_jobs.create!(
- class_name: 'DropInvalidVulnerabilities',
- arguments: [vulnerability_with_finding.id, vulnerability_with_finding.id]
- )
-
- background_migration_jobs.create!(
- class_name: 'DropInvalidVulnerabilities',
- arguments: [vulnerability_without_finding.id, vulnerability_without_finding.id]
- )
-
- subject.perform(vulnerability_with_finding.id, vulnerability_with_finding.id)
-
- expect(background_migration_jobs.first.status).to eq(succeeded_status)
- expect(background_migration_jobs.second.status).to eq(pending_status)
- end
-
- private
-
- def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0)
- vulnerabilities.create!(
- project_id: project_id,
- author_id: author_id,
- title: title,
- severity: severity,
- confidence: confidence,
- report_type: report_type
- )
- end
-
- # rubocop:disable Metrics/ParameterLists
- def create_finding!(
- vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:,
- name: "test", severity: 7, confidence: 7, report_type: 0,
- project_fingerprint: '123qweasdzxc', location_fingerprint: 'test',
- metadata_version: 'test', raw_metadata: 'test', uuid: SecureRandom.uuid)
- vulnerabilities_findings.create!(
- vulnerability_id: vulnerability_id,
- project_id: project_id,
- name: name,
- severity: severity,
- confidence: confidence,
- report_type: report_type,
- project_fingerprint: project_fingerprint,
- scanner_id: scanner_id,
- primary_identifier_id: primary_identifier_id,
- location_fingerprint: location_fingerprint,
- metadata_version: metadata_version,
- raw_metadata: raw_metadata,
- uuid: uuid
- )
- end
- # rubocop:enable Metrics/ParameterLists
-
- def create_user!(name: "Example User", email: "user@example.com", user_type: nil)
- users.create!(
- name: name,
- email: email,
- username: name,
- projects_limit: 0,
- user_type: user_type,
- confirmed_at: Time.current
- )
- end
-end
diff --git a/spec/lib/gitlab/background_migration/encrypt_ci_trigger_token_spec.rb b/spec/lib/gitlab/background_migration/encrypt_ci_trigger_token_spec.rb
index b52f30a5e21..dd3e7877f8a 100644
--- a/spec/lib/gitlab/background_migration/encrypt_ci_trigger_token_spec.rb
+++ b/spec/lib/gitlab/background_migration/encrypt_ci_trigger_token_spec.rb
@@ -10,8 +10,7 @@ RSpec.describe Gitlab::BackgroundMigration::EncryptCiTriggerToken, feature_categ
mode: :per_attribute_iv,
key: ::Settings.attr_encrypted_db_key_base_32,
algorithm: 'aes-256-gcm',
- encode: false,
- encode_iv: false
+ encode: false
end
end
@@ -52,6 +51,7 @@ RSpec.describe Gitlab::BackgroundMigration::EncryptCiTriggerToken, feature_categ
already_encrypted_token = Ci::Trigger.find(with_encryption.id)
expect(already_encrypted_token.encrypted_token).to eq(with_encryption.encrypted_token)
expect(already_encrypted_token.encrypted_token_iv).to eq(with_encryption.encrypted_token_iv)
+ expect(already_encrypted_token.token).to eq(already_encrypted_token.encrypted_token_tmp)
expect(with_encryption.token).to eq(with_encryption.encrypted_token_tmp)
end
end
diff --git a/spec/lib/gitlab/background_migration/extract_project_topics_into_separate_table_spec.rb b/spec/lib/gitlab/background_migration/extract_project_topics_into_separate_table_spec.rb
deleted file mode 100644
index 586e75ffb37..00000000000
--- a/spec/lib/gitlab/background_migration/extract_project_topics_into_separate_table_spec.rb
+++ /dev/null
@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-RSpec.describe Gitlab::BackgroundMigration::ExtractProjectTopicsIntoSeparateTable,
- :suppress_gitlab_schemas_validate_connection, schema: 20210826171758 do
- it 'correctly extracts project topics into separate table' do
- namespaces = table(:namespaces)
- projects = table(:projects)
- taggings = table(:taggings)
- tags = table(:tags)
- project_topics = table(:project_topics)
- topics = table(:topics)
-
- namespace = namespaces.create!(name: 'foo', path: 'foo')
- project = projects.create!(namespace_id: namespace.id)
- tag_1 = tags.create!(name: 'Topic1')
- tag_2 = tags.create!(name: 'Topic2')
- tag_3 = tags.create!(name: 'Topic3')
- topic_3 = topics.create!(name: 'Topic3')
- tagging_1 = taggings.create!(taggable_type: 'Project', taggable_id: project.id, context: 'topics', tag_id: tag_1.id)
- tagging_2 = taggings.create!(taggable_type: 'Project', taggable_id: project.id, context: 'topics', tag_id: tag_2.id)
- other_tagging = taggings.create!(taggable_type: 'Other', taggable_id: project.id, context: 'topics', tag_id: tag_1.id)
- tagging_3 = taggings.create!(taggable_type: 'Project', taggable_id: project.id, context: 'topics', tag_id: tag_3.id)
- tagging_4 = taggings.create!(taggable_type: 'Project', taggable_id: -1, context: 'topics', tag_id: tag_1.id)
- tagging_5 = taggings.create!(taggable_type: 'Project', taggable_id: project.id, context: 'topics', tag_id: -1)
-
- subject.perform(tagging_1.id, tagging_5.id)
-
- # Tagging records
- expect { tagging_1.reload }.to raise_error(ActiveRecord::RecordNotFound)
- expect { tagging_2.reload }.to raise_error(ActiveRecord::RecordNotFound)
- expect { other_tagging.reload }.not_to raise_error
- expect { tagging_3.reload }.to raise_error(ActiveRecord::RecordNotFound)
- expect { tagging_4.reload }.to raise_error(ActiveRecord::RecordNotFound)
- expect { tagging_5.reload }.to raise_error(ActiveRecord::RecordNotFound)
-
- # Topic records
- topic_1 = topics.find_by(name: 'Topic1')
- topic_2 = topics.find_by(name: 'Topic2')
- expect(topics.all).to contain_exactly(topic_1, topic_2, topic_3)
-
- # ProjectTopic records
- expect(project_topics.all.map(&:topic_id)).to contain_exactly(topic_1.id, topic_2.id, topic_3.id)
- end
-end
diff --git a/spec/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at_spec.rb b/spec/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at_spec.rb
deleted file mode 100644
index 7f15aceca42..00000000000
--- a/spec/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at_spec.rb
+++ /dev/null
@@ -1,166 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-require Rails.root.join('db', 'post_migrate', '20211004110500_add_temporary_index_to_issue_metrics.rb')
-
-RSpec.describe Gitlab::BackgroundMigration::FixFirstMentionedInCommitAt, :migration, schema: 20211004110500 do
- let(:namespaces) { table(:namespaces) }
- let(:projects) { table(:projects) }
- let(:users) { table(:users) }
- let(:merge_requests) { table(:merge_requests) }
- let(:issues) { table(:issues) }
- let(:issue_metrics) { table(:issue_metrics) }
- let(:merge_requests_closing_issues) { table(:merge_requests_closing_issues) }
- let(:diffs) { table(:merge_request_diffs) }
- let(:ten_days_ago) { 10.days.ago }
- let(:commits) do
- table(:merge_request_diff_commits).tap do |t|
- t.extend(SuppressCompositePrimaryKeyWarning)
- end
- end
-
- let(:namespace) { namespaces.create!(name: 'ns', path: 'ns') }
- let(:project) { projects.create!(namespace_id: namespace.id) }
-
- let!(:issue1) do
- issues.create!(
- title: 'issue',
- description: 'description',
- project_id: project.id
- )
- end
-
- let!(:issue2) do
- issues.create!(
- title: 'issue',
- description: 'description',
- project_id: project.id
- )
- end
-
- let!(:merge_request1) do
- merge_requests.create!(
- source_branch: 'a',
- target_branch: 'master',
- target_project_id: project.id
- )
- end
-
- let!(:merge_request2) do
- merge_requests.create!(
- source_branch: 'b',
- target_branch: 'master',
- target_project_id: project.id
- )
- end
-
- let!(:merge_request_closing_issue1) do
- merge_requests_closing_issues.create!(issue_id: issue1.id, merge_request_id: merge_request1.id)
- end
-
- let!(:merge_request_closing_issue2) do
- merge_requests_closing_issues.create!(issue_id: issue2.id, merge_request_id: merge_request2.id)
- end
-
- let!(:diff1) { diffs.create!(merge_request_id: merge_request1.id) }
- let!(:diff2) { diffs.create!(merge_request_id: merge_request1.id) }
-
- let!(:other_diff) { diffs.create!(merge_request_id: merge_request2.id) }
-
- let!(:commit1) do
- commits.create!(
- merge_request_diff_id: diff2.id,
- relative_order: 0,
- sha: Gitlab::Database::ShaAttribute.serialize('aaa'),
- authored_date: 5.days.ago
- )
- end
-
- let!(:commit2) do
- commits.create!(
- merge_request_diff_id: diff2.id,
- relative_order: 1,
- sha: Gitlab::Database::ShaAttribute.serialize('aaa'),
- authored_date: 10.days.ago
- )
- end
-
- let!(:commit3) do
- commits.create!(
- merge_request_diff_id: other_diff.id,
- relative_order: 1,
- sha: Gitlab::Database::ShaAttribute.serialize('aaa'),
- authored_date: 5.days.ago
- )
- end
-
- def run_migration
- described_class
- .new
- .perform(issue_metrics.minimum(:issue_id), issue_metrics.maximum(:issue_id))
- end
-
- shared_examples 'fixes first_mentioned_in_commit_at' do
- it "marks successful slices as completed" do
- min_issue_id = issue_metrics.minimum(:issue_id)
- max_issue_id = issue_metrics.maximum(:issue_id)
-
- expect(subject).to receive(:mark_job_as_succeeded).with(min_issue_id, max_issue_id)
-
- subject.perform(min_issue_id, max_issue_id)
- end
-
- context 'when the persisted first_mentioned_in_commit_at is later than the first commit authored_date' do
- it 'updates the issue_metrics record' do
- record1 = issue_metrics.create!(issue_id: issue1.id, first_mentioned_in_commit_at: Time.current)
- record2 = issue_metrics.create!(issue_id: issue2.id, first_mentioned_in_commit_at: Time.current)
-
- run_migration
- record1.reload
- record2.reload
-
- expect(record1.first_mentioned_in_commit_at).to be_within(2.seconds).of(commit2.authored_date)
- expect(record2.first_mentioned_in_commit_at).to be_within(2.seconds).of(commit3.authored_date)
- end
- end
-
- context 'when the persisted first_mentioned_in_commit_at is earlier than the first commit authored_date' do
- it 'does not update the issue_metrics record' do
- record = issue_metrics.create!(issue_id: issue1.id, first_mentioned_in_commit_at: 20.days.ago)
-
- expect { run_migration }.not_to change { record.reload.first_mentioned_in_commit_at }
- end
- end
-
- context 'when the first_mentioned_in_commit_at is null' do
- it 'does nothing' do
- record = issue_metrics.create!(issue_id: issue1.id, first_mentioned_in_commit_at: nil)
-
- expect { run_migration }.not_to change { record.reload.first_mentioned_in_commit_at }
- end
- end
- end
-
- describe 'running the migration when first_mentioned_in_commit_at is timestamp without time zone' do
- it_behaves_like 'fixes first_mentioned_in_commit_at'
- end
-
- describe 'running the migration when first_mentioned_in_commit_at is timestamp with time zone' do
- around do |example|
- AddTemporaryIndexToIssueMetrics.new.down
-
- ActiveRecord::Base.connection.execute "ALTER TABLE issue_metrics ALTER first_mentioned_in_commit_at type timestamp with time zone"
- Gitlab::BackgroundMigration::FixFirstMentionedInCommitAt::TmpIssueMetrics.reset_column_information
- AddTemporaryIndexToIssueMetrics.new.up
-
- example.run
-
- AddTemporaryIndexToIssueMetrics.new.down
- ActiveRecord::Base.connection.execute "ALTER TABLE issue_metrics ALTER first_mentioned_in_commit_at type timestamp without time zone"
- Gitlab::BackgroundMigration::FixFirstMentionedInCommitAt::TmpIssueMetrics.reset_column_information
- AddTemporaryIndexToIssueMetrics.new.up
- end
-
- it_behaves_like 'fixes first_mentioned_in_commit_at'
- end
-end
diff --git a/spec/lib/gitlab/background_migration/fix_merge_request_diff_commit_users_spec.rb b/spec/lib/gitlab/background_migration/fix_merge_request_diff_commit_users_spec.rb
deleted file mode 100644
index 99df21562b0..00000000000
--- a/spec/lib/gitlab/background_migration/fix_merge_request_diff_commit_users_spec.rb
+++ /dev/null
@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-# rubocop: disable RSpec/FactoriesInMigrationSpecs
-RSpec.describe Gitlab::BackgroundMigration::FixMergeRequestDiffCommitUsers do
- let(:migration) { described_class.new }
-
- describe '#perform' do
- context 'when the project exists' do
- it 'does nothing' do
- project = create(:project)
-
- expect { migration.perform(project.id) }.not_to raise_error
- end
- end
-
- context 'when the project does not exist' do
- it 'does nothing' do
- expect { migration.perform(-1) }.not_to raise_error
- end
- end
- end
-end
-# rubocop: enable RSpec/FactoriesInMigrationSpecs
diff --git a/spec/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues_spec.rb b/spec/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues_spec.rb
new file mode 100644
index 00000000000..9f431c43f39
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues_spec.rb
@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::FixVulnerabilityReadsHasIssues, schema: 20230302185739, feature_category: :vulnerability_management do # rubocop:disable Layout/LineLength
+ let(:namespaces) { table(:namespaces) }
+ let(:projects) { table(:projects) }
+ let(:users) { table(:users) }
+ let(:scanners) { table(:vulnerability_scanners) }
+ let(:vulnerabilities) { table(:vulnerabilities) }
+ let(:vulnerability_reads) { table(:vulnerability_reads) }
+ let(:work_item_types) { table(:work_item_types) }
+ let(:issues) { table(:issues) }
+ let(:vulnerability_issue_links) { table(:vulnerability_issue_links) }
+
+ let(:namespace) { namespaces.create!(name: 'user', path: 'user') }
+ let(:project) { projects.create!(namespace_id: namespace.id, project_namespace_id: namespace.id) }
+ let(:user) { users.create!(username: 'john_doe', email: 'johndoe@gitlab.com', projects_limit: 10) }
+ let(:scanner) { scanners.create!(project_id: project.id, external_id: 'external_id', name: 'Test Scanner') }
+ let(:work_item_type) { work_item_types.create!(name: 'test') }
+
+ let(:vulnerability_records) do
+ Array.new(4).map do |_, n|
+ vulnerabilities.create!(
+ project_id: project.id,
+ author_id: user.id,
+ title: "vulnerability #{n}",
+ severity: 1,
+ confidence: 1,
+ report_type: 1
+ )
+ end
+ end
+
+ let(:vulnerabilities_with_issues) { [vulnerability_records.first, vulnerability_records.third] }
+ let(:vulnerabilities_without_issues) { vulnerability_records - vulnerabilities_with_issues }
+
+ let(:vulnerability_read_records) do
+ vulnerability_records.map do |vulnerability|
+ vulnerability_reads.create!(
+ project_id: project.id,
+ vulnerability_id: vulnerability.id,
+ scanner_id: scanner.id,
+ has_issues: false,
+ severity: 1,
+ report_type: 1,
+ state: 1,
+ uuid: SecureRandom.uuid
+ )
+ end
+ end
+
+ let!(:issue_links) do
+ vulnerabilities_with_issues.map do |vulnerability|
+ issue = issues.create!(
+ title: vulnerability.title,
+ author_id: user.id,
+ project_id: project.id,
+ confidential: true,
+ work_item_type_id: work_item_type.id,
+ namespace_id: namespace.id
+ )
+
+ vulnerability_issue_links.create!(
+ vulnerability_id: vulnerability.id,
+ issue_id: issue.id
+ )
+ end
+ end
+
+ def vulnerability_read_for(vulnerability)
+ vulnerability_read_records.find { |read| read.vulnerability_id == vulnerability.id }
+ end
+
+ subject(:perform_migration) do
+ described_class.new(
+ start_id: issue_links.first.vulnerability_id,
+ end_id: issue_links.last.vulnerability_id,
+ batch_table: :vulnerability_issue_links,
+ batch_column: :vulnerability_id,
+ sub_batch_size: issue_links.size,
+ pause_ms: 0,
+ connection: ActiveRecord::Base.connection
+ ).perform
+ end
+
+ it 'only changes records with issue links' do
+ expect(vulnerability_read_records).to all(have_attributes(has_issues: false))
+
+ perform_migration
+
+ vulnerabilities_with_issues.each do |vulnerability|
+ expect(vulnerability_read_for(vulnerability).reload.has_issues).to eq(true)
+ end
+
+ vulnerabilities_without_issues.each do |vulnerability|
+ expect(vulnerability_read_for(vulnerability).reload.has_issues).to eq(false)
+ end
+ end
+end
diff --git a/spec/lib/gitlab/background_migration/issues_internal_id_scope_updater_spec.rb b/spec/lib/gitlab/background_migration/issues_internal_id_scope_updater_spec.rb
new file mode 100644
index 00000000000..1adff322b41
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/issues_internal_id_scope_updater_spec.rb
@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+# this needs the schema to be before we introduce the not null constraint on routes#namespace_id
+# rubocop:disable RSpec/MultipleMemoizedHelpers
+RSpec.describe Gitlab::BackgroundMigration::IssuesInternalIdScopeUpdater, feature_category: :team_planning do
+ let(:namespaces) { table(:namespaces) }
+ let(:projects) { table(:projects) }
+ let(:internal_ids) { table(:internal_ids) }
+
+ let(:gr1) { namespaces.create!(name: 'batchtest1', type: 'Group', path: 'space1') }
+ let(:gr2) { namespaces.create!(name: 'batchtest2', type: 'Group', parent_id: gr1.id, path: 'space2') }
+
+ let(:pr_nmsp1) { namespaces.create!(name: 'proj1', path: 'proj1', type: 'Project', parent_id: gr1.id) }
+ let(:pr_nmsp2) { namespaces.create!(name: 'proj2', path: 'proj2', type: 'Project', parent_id: gr1.id) }
+ let(:pr_nmsp3) { namespaces.create!(name: 'proj3', path: 'proj3', type: 'Project', parent_id: gr2.id) }
+ let(:pr_nmsp4) { namespaces.create!(name: 'proj4', path: 'proj4', type: 'Project', parent_id: gr2.id) }
+ let(:pr_nmsp5) { namespaces.create!(name: 'proj5', path: 'proj5', type: 'Project', parent_id: gr2.id) }
+ let(:pr_nmsp6) { namespaces.create!(name: 'proj6', path: 'proj6', type: 'Project', parent_id: gr2.id) }
+
+ # rubocop:disable Layout/LineLength
+ let(:p1) { projects.create!(name: 'proj1', path: 'proj1', namespace_id: gr1.id, project_namespace_id: pr_nmsp1.id) }
+ let(:p2) { projects.create!(name: 'proj2', path: 'proj2', namespace_id: gr1.id, project_namespace_id: pr_nmsp2.id) }
+ let(:p3) { projects.create!(name: 'proj3', path: 'proj3', namespace_id: gr2.id, project_namespace_id: pr_nmsp3.id) }
+ let(:p4) { projects.create!(name: 'proj4', path: 'proj4', namespace_id: gr2.id, project_namespace_id: pr_nmsp4.id) }
+ let(:p5) { projects.create!(name: 'proj5', path: 'proj5', namespace_id: gr2.id, project_namespace_id: pr_nmsp5.id) }
+ let(:p6) { projects.create!(name: 'proj6', path: 'proj6', namespace_id: gr2.id, project_namespace_id: pr_nmsp6.id) }
+ # rubocop:enable Layout/LineLength
+
+ # a project that already is covered by a record for its namespace. This should result in no new record added and
+ # project related record deleted
+ let!(:issues_internal_ids_p1) { internal_ids.create!(project_id: p1.id, usage: 0, last_value: 100) }
+ let!(:issues_internal_ids_pr_nmsp1) { internal_ids.create!(namespace_id: pr_nmsp1.id, usage: 0, last_value: 111) }
+
+ # project records that do not have a corresponding namespace record. This should result 2 new records
+ # scoped to corresponding project namespaces being added and the project related records being deleted.
+ let!(:issues_internal_ids_p2) { internal_ids.create!(project_id: p2.id, usage: 0, last_value: 200) }
+ let!(:issues_internal_ids_p3) { internal_ids.create!(project_id: p3.id, usage: 0, last_value: 300) }
+
+ # a project record on a different usage, should not be affected by the migration and
+ # no new record should be created for this case
+ let!(:issues_internal_ids_p4) { internal_ids.create!(project_id: p4.id, usage: 4, last_value: 400) }
+
+ # a project namespace scoped record without a corresponding project record, should not affect anything.
+ let!(:issues_internal_ids_pr_nmsp5) { internal_ids.create!(namespace_id: pr_nmsp5.id, usage: 0, last_value: 500) }
+
+ # a record scoped to a group, should not affect anything.
+ let!(:issues_internal_ids_gr1) { internal_ids.create!(namespace_id: gr1.id, usage: 0, last_value: 600) }
+
+ # a project that is covered by a record for its namespace, but has a higher last_value, due to updates during rolling
+ # deploy for instance, see https://gitlab.com/gitlab-com/gl-infra/production/-/issues/8548
+ let!(:issues_internal_ids_p6) { internal_ids.create!(project_id: p6.id, usage: 0, last_value: 111) }
+ let!(:issues_internal_ids_pr_nmsp6) { internal_ids.create!(namespace_id: pr_nmsp6.id, usage: 0, last_value: 100) }
+
+ subject(:perform_migration) do
+ described_class.new(
+ start_id: internal_ids.minimum(:id),
+ end_id: internal_ids.maximum(:id),
+ batch_table: :internal_ids,
+ batch_column: :id,
+ sub_batch_size: 2,
+ pause_ms: 0,
+ connection: ActiveRecord::Base.connection
+ ).perform
+ end
+
+ it 'backfills internal_ids records and removes related project records', :aggregate_failures do
+ perform_migration
+
+ expected_recs = [pr_nmsp1.id, pr_nmsp2.id, pr_nmsp3.id, pr_nmsp5.id, gr1.id, pr_nmsp6.id]
+
+ # all namespace scoped records for issues(0) usage
+ expect(internal_ids.where.not(namespace_id: nil).where(usage: 0).count).to eq(6)
+ # all namespace_ids for issues(0) usage
+ expect(internal_ids.where.not(namespace_id: nil).where(usage: 0).pluck(:namespace_id)).to match_array(expected_recs)
+ # this is the record with usage: 4
+ expect(internal_ids.where.not(project_id: nil).count).to eq(1)
+ # no project scoped records for issues usage left
+ expect(internal_ids.where.not(project_id: nil).where(usage: 0).count).to eq(0)
+
+ # the case when the project_id scoped record had the higher last_value,
+ # see `issues_internal_ids_p6` and issues_internal_ids_pr_nmsp6 definitions above
+ expect(internal_ids.where(namespace_id: pr_nmsp6.id).first.last_value).to eq(111)
+
+ # the case when the namespace_id scoped record had the higher last_value,
+ # see `issues_internal_ids_p1` and issues_internal_ids_pr_nmsp1 definitions above.
+ expect(internal_ids.where(namespace_id: pr_nmsp1.id).first.last_value).to eq(111)
+ end
+end
+# rubocop:enable RSpec/MultipleMemoizedHelpers
diff --git a/spec/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings_spec.rb b/spec/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings_spec.rb
new file mode 100644
index 00000000000..ba2f571f5aa
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings_spec.rb
@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::MigrateEvidencesForVulnerabilityFindings,
+ feature_category: :vulnerability_management do
+ let(:vulnerability_occurrences) { table(:vulnerability_occurrences) }
+ let(:vulnerability_finding_evidences) { table(:vulnerability_finding_evidences) }
+ let(:evidence_hash) { { url: 'http://test.com' } }
+ let(:namespace1) { table(:namespaces).create!(name: 'namespace 1', path: 'namespace1') }
+ let(:project1) { table(:projects).create!(namespace_id: namespace1.id, project_namespace_id: namespace1.id) }
+ let(:user) { table(:users).create!(email: 'test1@example.com', projects_limit: 5) }
+
+ let(:scanner1) do
+ table(:vulnerability_scanners).create!(project_id: project1.id, external_id: 'test 1', name: 'test scanner 1')
+ end
+
+ let(:stating_id) { vulnerability_occurrences.pluck(:id).min }
+ let(:end_id) { vulnerability_occurrences.pluck(:id).max }
+
+ let(:migration) do
+ described_class.new(
+ start_id: stating_id,
+ end_id: end_id,
+ batch_table: :vulnerability_occurrences,
+ batch_column: :id,
+ sub_batch_size: 2,
+ pause_ms: 2,
+ connection: ApplicationRecord.connection
+ )
+ end
+
+ subject(:perform_migration) { migration.perform }
+
+ context 'without the presence of evidence key' do
+ before do
+ create_finding!(project1.id, scanner1.id, { other_keys: 'test' })
+ end
+
+ it 'does not create any evidence' do
+ expect { perform_migration }.not_to change { vulnerability_finding_evidences.count }
+ end
+ end
+
+ context 'with evidence equals to nil' do
+ before do
+ create_finding!(project1.id, scanner1.id, { evidence: nil })
+ end
+
+ it 'does not create any evidence' do
+ expect { perform_migration }.not_to change { vulnerability_finding_evidences.count }
+ end
+ end
+
+ context 'with existing evidence within raw_metadata' do
+ let!(:finding1) { create_finding!(project1.id, scanner1.id, { evidence: evidence_hash }) }
+ let!(:finding2) { create_finding!(project1.id, scanner1.id, { evidence: evidence_hash }) }
+
+ it 'creates new evidence for each finding' do
+ expect { perform_migration }.to change { vulnerability_finding_evidences.count }.by(2)
+ end
+
+ context 'when parse throws exception JSON::ParserError' do
+ before do
+ allow(Gitlab::Json).to receive(:parse).and_raise(JSON::ParserError)
+ end
+
+ it 'does not create new records' do
+ expect { perform_migration }.not_to change { vulnerability_finding_evidences.count }
+ end
+ end
+ end
+
+ context 'with unsupported Unicode escape sequence' do
+ let!(:finding1) { create_finding!(project1.id, scanner1.id, { evidence: { 'summary' => "\u0000" } }) }
+
+ it 'does not create new evidence' do
+ expect { perform_migration }.not_to change { vulnerability_finding_evidences.count }
+ end
+ end
+
+ context 'with existing evidence records' do
+ let!(:finding) { create_finding!(project1.id, scanner1.id, { evidence: evidence_hash }) }
+
+ before do
+ vulnerability_finding_evidences.create!(vulnerability_occurrence_id: finding.id, data: evidence_hash)
+ end
+
+ it 'does not create new evidence' do
+ expect { perform_migration }.not_to change { vulnerability_finding_evidences.count }
+ end
+
+ context 'with non-existing evidence' do
+ let!(:finding3) { create_finding!(project1.id, scanner1.id, { evidence: { url: 'http://secondary.com' } }) }
+
+ it 'creates a new evidence only to the non-existing evidence' do
+ expect { perform_migration }.to change { vulnerability_finding_evidences.count }.by(1)
+ end
+ end
+ end
+
+ private
+
+ def create_finding!(project_id, scanner_id, raw_metadata)
+ vulnerability = table(:vulnerabilities).create!(project_id: project_id, author_id: user.id, title: 'test',
+ severity: 4, confidence: 4, report_type: 0)
+
+ identifier = table(:vulnerability_identifiers).create!(project_id: project_id, external_type: 'uuid-v5',
+ external_id: 'uuid-v5', fingerprint: OpenSSL::Digest::SHA256.hexdigest(vulnerability.id.to_s),
+ name: 'Identifier for UUIDv5 2 2')
+
+ table(:vulnerability_occurrences).create!(
+ vulnerability_id: vulnerability.id, project_id: project_id, scanner_id: scanner_id,
+ primary_identifier_id: identifier.id, name: 'test', severity: 4, confidence: 4, report_type: 0,
+ uuid: SecureRandom.uuid, project_fingerprint: '123qweasdzxc', location: { "image" => "alpine:3.4" },
+ location_fingerprint: 'test', metadata_version: 'test',
+ raw_metadata: raw_metadata.to_json)
+ end
+end
diff --git a/spec/lib/gitlab/background_migration/migrate_human_user_type_spec.rb b/spec/lib/gitlab/background_migration/migrate_human_user_type_spec.rb
new file mode 100644
index 00000000000..7edeaed5794
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/migrate_human_user_type_spec.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::MigrateHumanUserType, schema: 20230327103401, feature_category: :user_management do # rubocop:disable Layout/LineLength
+ let!(:valid_users) do
+ # 13 is the max value we have at the moment.
+ (0..13).map do |type|
+ table(:users).create!(username: "user#{type}", email: "user#{type}@test.com", user_type: type, projects_limit: 0)
+ end
+ end
+
+ let!(:user_to_update) do
+ table(:users).create!(username: "user_nil", email: "user_nil@test.com", user_type: nil, projects_limit: 0)
+ end
+
+ let(:starting_id) { table(:users).pluck(:id).min }
+ let(:end_id) { table(:users).pluck(:id).max }
+
+ let(:migration) do
+ described_class.new(
+ start_id: starting_id,
+ end_id: end_id,
+ batch_table: :users,
+ batch_column: :id,
+ sub_batch_size: 100,
+ pause_ms: 2,
+ connection: ::ApplicationRecord.connection
+ )
+ end
+
+ describe 'perform' do
+ it 'updates user with `nil` user type only' do
+ expect do
+ migration.perform
+ valid_users.map(&:reload)
+ user_to_update.reload
+ end.not_to change { valid_users.map(&:user_type) }
+
+ expect(user_to_update.user_type).to eq(0)
+ end
+ end
+end
diff --git a/spec/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings_spec.rb b/spec/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings_spec.rb
new file mode 100644
index 00000000000..b973f9d4350
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings_spec.rb
@@ -0,0 +1,192 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::MigrateLinksForVulnerabilityFindings,
+ feature_category: :vulnerability_management do
+ let(:vulnerability_occurrences) { table(:vulnerability_occurrences) }
+ let(:vulnerability_finding_links) { table(:vulnerability_finding_links) }
+ let(:link_hash) { { url: 'http://test.com' } }
+ let(:namespace1) { table(:namespaces).create!(name: 'namespace 1', path: 'namespace1') }
+ let(:project1) { table(:projects).create!(namespace_id: namespace1.id, project_namespace_id: namespace1.id) }
+ let(:user) { table(:users).create!(email: 'test1@example.com', projects_limit: 5) }
+
+ let(:scanner1) do
+ table(:vulnerability_scanners).create!(project_id: project1.id, external_id: 'test 1', name: 'test scanner 1')
+ end
+
+ let(:stating_id) { vulnerability_occurrences.pluck(:id).min }
+ let(:end_id) { vulnerability_occurrences.pluck(:id).max }
+
+ let(:migration) do
+ described_class.new(
+ start_id: stating_id,
+ end_id: end_id,
+ batch_table: :vulnerability_occurrences,
+ batch_column: :id,
+ sub_batch_size: 2,
+ pause_ms: 2,
+ connection: ApplicationRecord.connection
+ )
+ end
+
+ subject(:perform_migration) { migration.perform }
+
+ context 'without the presence of links key' do
+ before do
+ create_finding!(project1.id, scanner1.id, { other_keys: 'test' })
+ end
+
+ it 'does not create any link' do
+ expect(Gitlab::AppLogger).not_to receive(:error)
+
+ expect { perform_migration }.not_to change { vulnerability_finding_links.count }
+ end
+ end
+
+ context 'with links equals to an array of nil element' do
+ before do
+ create_finding!(project1.id, scanner1.id, { links: [nil] })
+ end
+
+ it 'does not create any link' do
+ expect(Gitlab::AppLogger).not_to receive(:error)
+
+ expect { perform_migration }.not_to change { vulnerability_finding_links.count }
+ end
+ end
+
+ context 'with links equals to a string' do
+ before do
+ create_finding!(project1.id, scanner1.id, { links: "wrong format" })
+ end
+
+ it 'does not create any link' do
+ expect(Gitlab::AppLogger).not_to receive(:error)
+
+ expect { perform_migration }.not_to change { vulnerability_finding_links.count }
+ end
+ end
+
+ context 'with some elements which do not contain the key url' do
+ let!(:finding) do
+ create_finding!(project1.id, scanner1.id, { links: [link_hash, "wrong format", {}] })
+ end
+
+ it 'creates links only to valid elements' do
+ expect(Gitlab::AppLogger).not_to receive(:error)
+
+ perform_migration
+
+ expect(vulnerability_finding_links.all).to contain_exactly(have_attributes(
+ url: link_hash[:url],
+ vulnerability_occurrence_id: finding.id))
+ end
+ end
+
+ context 'when link name is too long' do
+ let!(:finding) do
+ create_finding!(project1.id, scanner1.id, { links: [{ name: 'A' * 300, url: 'https://foo' }] })
+ end
+
+ it 'skips creation of link and logs error' do
+ expect(Gitlab::AppLogger).to receive(:error).with({
+ class: described_class.name,
+ message: /check_55f0a95439/,
+ model_id: finding.id
+ })
+ expect { perform_migration }.not_to change { vulnerability_finding_links.count }
+ end
+ end
+
+ context 'when link url is too long' do
+ let!(:finding) do
+ create_finding!(project1.id, scanner1.id, { links: [{ url: "https://f#{'o' * 2050}" }] })
+ end
+
+ it 'skips creation of link and logs error' do
+ expect(Gitlab::AppLogger).to receive(:error).with({
+ class: described_class.name,
+ message: /check_b7fe886df6/,
+ model_id: finding.id
+ })
+ expect { perform_migration }.not_to change { vulnerability_finding_links.count }
+ end
+ end
+
+ context 'with links equals to an array of duplicated elements' do
+ let!(:finding) do
+ create_finding!(project1.id, scanner1.id, { links: [link_hash, link_hash] })
+ end
+
+ it 'creates one new link' do
+ expect(Gitlab::AppLogger).not_to receive(:error)
+
+ perform_migration
+
+ expect(vulnerability_finding_links.all).to contain_exactly(have_attributes(
+ url: link_hash[:url],
+ vulnerability_occurrence_id: finding.id))
+ end
+ end
+
+ context 'with existing links within raw_metadata' do
+ let!(:finding1) { create_finding!(project1.id, scanner1.id, { links: [link_hash] }) }
+ let!(:finding2) { create_finding!(project1.id, scanner1.id, { links: [link_hash] }) }
+
+ it 'creates new link for each finding' do
+ expect(Gitlab::AppLogger).not_to receive(:error)
+
+ expect { perform_migration }.to change { vulnerability_finding_links.count }.by(2)
+ end
+ end
+
+ context 'when Gitlab::Json throws exception JSON::ParserError' do
+ before do
+ create_finding!(project1.id, scanner1.id, { links: [link_hash] })
+ allow(Gitlab::Json).to receive(:parse).and_raise(JSON::ParserError)
+ end
+
+ it 'does not log this error nor create new records' do
+ expect(Gitlab::AppLogger).not_to receive(:error)
+
+ expect { perform_migration }.not_to change { vulnerability_finding_links.count }
+ end
+ end
+
+ context 'with existing link records' do
+ let!(:finding) { create_finding!(project1.id, scanner1.id, { links: [link_hash] }) }
+
+ before do
+ vulnerability_finding_links.create!(vulnerability_occurrence_id: finding.id, url: link_hash[:url])
+ end
+
+ it 'does not create new link' do
+ expect(Gitlab::AppLogger).not_to receive(:error)
+
+ expect { perform_migration }.not_to change { vulnerability_finding_links.count }
+ end
+
+ it 'does not raise ActiveRecord::RecordNotUnique' do
+ expect { perform_migration }.not_to raise_error
+ end
+ end
+
+ private
+
+ def create_finding!(project_id, scanner_id, raw_metadata)
+ vulnerability = table(:vulnerabilities).create!(project_id: project_id, author_id: user.id, title: 'test',
+ severity: 4, confidence: 4, report_type: 0)
+
+ identifier = table(:vulnerability_identifiers).create!(project_id: project_id, external_type: 'uuid-v5',
+ external_id: 'uuid-v5', fingerprint: OpenSSL::Digest::SHA256.hexdigest(vulnerability.id.to_s),
+ name: 'Identifier for UUIDv5 2 2')
+
+ table(:vulnerability_occurrences).create!(
+ vulnerability_id: vulnerability.id, project_id: project_id, scanner_id: scanner_id,
+ primary_identifier_id: identifier.id, name: 'test', severity: 4, confidence: 4, report_type: 0,
+ uuid: SecureRandom.uuid, project_fingerprint: '123qweasdzxc', location: { "image" => "alpine:3.4" },
+ location_fingerprint: 'test', metadata_version: 'test',
+ raw_metadata: raw_metadata.to_json)
+ end
+end
diff --git a/spec/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users_spec.rb b/spec/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users_spec.rb
deleted file mode 100644
index c3ae2cc060c..00000000000
--- a/spec/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users_spec.rb
+++ /dev/null
@@ -1,413 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-RSpec.describe Gitlab::BackgroundMigration::MigrateMergeRequestDiffCommitUsers, schema: 20211012134316 do
- let(:namespaces) { table(:namespaces) }
- let(:projects) { table(:projects) }
- let(:users) { table(:users) }
- let(:merge_requests) { table(:merge_requests) }
- let(:diffs) { table(:merge_request_diffs) }
- let(:commits) do
- table(:merge_request_diff_commits).tap do |t|
- t.extend(SuppressCompositePrimaryKeyWarning)
- end
- end
-
- let(:commit_users) { described_class::MergeRequestDiffCommitUser }
-
- let(:namespace) { namespaces.create!(name: 'foo', path: 'foo') }
- let(:project) { projects.create!(namespace_id: namespace.id) }
- let(:merge_request) do
- merge_requests.create!(
- source_branch: 'x',
- target_branch: 'master',
- target_project_id: project.id
- )
- end
-
- let(:diff) { diffs.create!(merge_request_id: merge_request.id) }
- let(:migration) { described_class.new }
-
- describe 'MergeRequestDiffCommit' do
- describe '.each_row_to_migrate' do
- it 'yields the rows to migrate for a given range' do
- commit1 = commits.create!(
- merge_request_diff_id: diff.id,
- relative_order: 0,
- sha: Gitlab::Database::ShaAttribute.serialize('123abc'),
- author_name: 'bob',
- author_email: 'bob@example.com',
- committer_name: 'bob',
- committer_email: 'bob@example.com'
- )
-
- commit2 = commits.create!(
- merge_request_diff_id: diff.id,
- relative_order: 1,
- sha: Gitlab::Database::ShaAttribute.serialize('123abc'),
- author_name: 'Alice',
- author_email: 'alice@example.com',
- committer_name: 'Alice',
- committer_email: 'alice@example.com'
- )
-
- # We stub this constant to make sure we run at least two pagination
- # queries for getting the data. This way we can test if the pagination
- # is actually working properly.
- stub_const(
- 'Gitlab::BackgroundMigration::MigrateMergeRequestDiffCommitUsers::COMMIT_ROWS_PER_QUERY',
- 1
- )
-
- rows = []
-
- described_class::MergeRequestDiffCommit.each_row_to_migrate(diff.id, diff.id + 1) do |row|
- rows << row
- end
-
- expect(rows.length).to eq(2)
-
- expect(rows[0].author_name).to eq(commit1.author_name)
- expect(rows[1].author_name).to eq(commit2.author_name)
- end
- end
- end
-
- describe 'MergeRequestDiffCommitUser' do
- describe '.union' do
- it 'produces a union of the given queries' do
- alice = commit_users.create!(name: 'Alice', email: 'alice@example.com')
- bob = commit_users.create!(name: 'Bob', email: 'bob@example.com')
- users = commit_users.union(
- [
- commit_users.where(name: 'Alice').to_sql,
- commit_users.where(name: 'Bob').to_sql
- ])
-
- expect(users).to include(alice)
- expect(users).to include(bob)
- end
- end
- end
-
- describe '#perform' do
- it 'skips jobs that have already been completed' do
- Gitlab::Database::BackgroundMigrationJob.create!(
- class_name: 'MigrateMergeRequestDiffCommitUsers',
- arguments: [1, 10],
- status: :succeeded
- )
-
- expect(migration).not_to receive(:get_data_to_update)
-
- migration.perform(1, 10)
- end
-
- it 'migrates the data in the range' do
- commits.create!(
- merge_request_diff_id: diff.id,
- relative_order: 0,
- sha: Gitlab::Database::ShaAttribute.serialize('123abc'),
- author_name: 'bob',
- author_email: 'bob@example.com',
- committer_name: 'bob',
- committer_email: 'bob@example.com'
- )
-
- migration.perform(diff.id, diff.id + 1)
-
- bob = commit_users.find_by(name: 'bob')
- commit = commits.first
-
- expect(commit.commit_author_id).to eq(bob.id)
- expect(commit.committer_id).to eq(bob.id)
- end
-
- it 'treats empty names and Emails the same as NULL values' do
- commits.create!(
- merge_request_diff_id: diff.id,
- relative_order: 0,
- sha: Gitlab::Database::ShaAttribute.serialize('123abc'),
- author_name: 'bob',
- author_email: 'bob@example.com',
- committer_name: '',
- committer_email: ''
- )
-
- migration.perform(diff.id, diff.id + 1)
-
- bob = commit_users.find_by(name: 'bob')
- commit = commits.first
-
- expect(commit.commit_author_id).to eq(bob.id)
- expect(commit.committer_id).to be_nil
- end
-
- it 'does not update rows without a committer and author' do
- commits.create!(
- merge_request_diff_id: diff.id,
- relative_order: 0,
- sha: Gitlab::Database::ShaAttribute.serialize('123abc')
- )
-
- migration.perform(diff.id, diff.id + 1)
-
- commit = commits.first
-
- expect(commit_users.count).to eq(0)
- expect(commit.commit_author_id).to be_nil
- expect(commit.committer_id).to be_nil
- end
-
- it 'marks the background job as done' do
- Gitlab::Database::BackgroundMigrationJob.create!(
- class_name: 'MigrateMergeRequestDiffCommitUsers',
- arguments: [diff.id, diff.id + 1]
- )
-
- migration.perform(diff.id, diff.id + 1)
-
- job = Gitlab::Database::BackgroundMigrationJob.first
-
- expect(job.status).to eq('succeeded')
- end
- end
-
- describe '#get_data_to_update' do
- it 'returns the users and commit rows to update' do
- commits.create!(
- merge_request_diff_id: diff.id,
- relative_order: 0,
- sha: Gitlab::Database::ShaAttribute.serialize('123abc'),
- author_name: 'bob' + ('a' * 510),
- author_email: 'bob@example.com',
- committer_name: 'bob' + ('a' * 510),
- committer_email: 'bob@example.com'
- )
-
- commits.create!(
- merge_request_diff_id: diff.id,
- relative_order: 1,
- sha: Gitlab::Database::ShaAttribute.serialize('456abc'),
- author_name: 'alice',
- author_email: 'alice@example.com',
- committer_name: 'alice',
- committer_email: 'alice@example.com'
- )
-
- users, to_update = migration.get_data_to_update(diff.id, diff.id + 1)
-
- bob_name = 'bob' + ('a' * 509)
-
- expect(users).to include(%w[alice alice@example.com])
- expect(users).to include([bob_name, 'bob@example.com'])
-
- expect(to_update[[diff.id, 0]])
- .to eq([[bob_name, 'bob@example.com'], [bob_name, 'bob@example.com']])
-
- expect(to_update[[diff.id, 1]])
- .to eq([%w[alice alice@example.com], %w[alice alice@example.com]])
- end
-
- it 'does not include a user if both the name and Email are missing' do
- commits.create!(
- merge_request_diff_id: diff.id,
- relative_order: 0,
- sha: Gitlab::Database::ShaAttribute.serialize('123abc'),
- author_name: nil,
- author_email: nil,
- committer_name: 'bob',
- committer_email: 'bob@example.com'
- )
-
- users, _ = migration.get_data_to_update(diff.id, diff.id + 1)
-
- expect(users).to eq([%w[bob bob@example.com]].to_set)
- end
- end
-
- describe '#get_user_rows_in_batches' do
- it 'retrieves all existing users' do
- alice = commit_users.create!(name: 'alice', email: 'alice@example.com')
- bob = commit_users.create!(name: 'bob', email: 'bob@example.com')
-
- users = [[alice.name, alice.email], [bob.name, bob.email]]
- mapping = {}
-
- migration.get_user_rows_in_batches(users, mapping)
-
- expect(mapping[%w[alice alice@example.com]]).to eq(alice)
- expect(mapping[%w[bob bob@example.com]]).to eq(bob)
- end
- end
-
- describe '#create_missing_users' do
- it 'creates merge request diff commit users that are missing' do
- alice = commit_users.create!(name: 'alice', email: 'alice@example.com')
- users = [%w[alice alice@example.com], %w[bob bob@example.com]]
- mapping = { %w[alice alice@example.com] => alice }
-
- migration.create_missing_users(users, mapping)
-
- expect(mapping[%w[alice alice@example.com]]).to eq(alice)
- expect(mapping[%w[bob bob@example.com]].name).to eq('bob')
- expect(mapping[%w[bob bob@example.com]].email).to eq('bob@example.com')
- end
- end
-
- describe '#update_commit_rows' do
- it 'updates the merge request diff commit rows' do
- to_update = { [42, 0] => [%w[alice alice@example.com], []] }
- user_mapping = { %w[alice alice@example.com] => double(:user, id: 1) }
-
- expect(migration)
- .to receive(:bulk_update_commit_rows)
- .with({ [42, 0] => [1, nil] })
-
- migration.update_commit_rows(to_update, user_mapping)
- end
- end
-
- describe '#bulk_update_commit_rows' do
- context 'when there are no authors and committers' do
- it 'does not update any rows' do
- migration.bulk_update_commit_rows({ [1, 0] => [] })
-
- expect(described_class::MergeRequestDiffCommit.connection)
- .not_to receive(:execute)
- end
- end
-
- context 'when there are only authors' do
- it 'only updates the author IDs' do
- author = commit_users.create!(name: 'Alice', email: 'alice@example.com')
- commit = commits.create!(
- merge_request_diff_id: diff.id,
- relative_order: 0,
- sha: Gitlab::Database::ShaAttribute.serialize('123abc')
- )
-
- mapping = {
- [commit.merge_request_diff_id, commit.relative_order] =>
- [author.id, nil]
- }
-
- migration.bulk_update_commit_rows(mapping)
-
- commit = commits.first
-
- expect(commit.commit_author_id).to eq(author.id)
- expect(commit.committer_id).to be_nil
- end
- end
-
- context 'when there are only committers' do
- it 'only updates the committer IDs' do
- committer =
- commit_users.create!(name: 'Alice', email: 'alice@example.com')
-
- commit = commits.create!(
- merge_request_diff_id: diff.id,
- relative_order: 0,
- sha: Gitlab::Database::ShaAttribute.serialize('123abc')
- )
-
- mapping = {
- [commit.merge_request_diff_id, commit.relative_order] =>
- [nil, committer.id]
- }
-
- migration.bulk_update_commit_rows(mapping)
-
- commit = commits.first
-
- expect(commit.committer_id).to eq(committer.id)
- expect(commit.commit_author_id).to be_nil
- end
- end
-
- context 'when there are both authors and committers' do
- it 'updates both the author and committer IDs' do
- author = commit_users.create!(name: 'Bob', email: 'bob@example.com')
- committer =
- commit_users.create!(name: 'Alice', email: 'alice@example.com')
-
- commit = commits.create!(
- merge_request_diff_id: diff.id,
- relative_order: 0,
- sha: Gitlab::Database::ShaAttribute.serialize('123abc')
- )
-
- mapping = {
- [commit.merge_request_diff_id, commit.relative_order] =>
- [author.id, committer.id]
- }
-
- migration.bulk_update_commit_rows(mapping)
-
- commit = commits.first
-
- expect(commit.commit_author_id).to eq(author.id)
- expect(commit.committer_id).to eq(committer.id)
- end
- end
-
- context 'when there are multiple commit rows to update' do
- it 'updates all the rows' do
- author = commit_users.create!(name: 'Bob', email: 'bob@example.com')
- committer =
- commit_users.create!(name: 'Alice', email: 'alice@example.com')
-
- commit1 = commits.create!(
- merge_request_diff_id: diff.id,
- relative_order: 0,
- sha: Gitlab::Database::ShaAttribute.serialize('123abc')
- )
-
- commit2 = commits.create!(
- merge_request_diff_id: diff.id,
- relative_order: 1,
- sha: Gitlab::Database::ShaAttribute.serialize('456abc')
- )
-
- mapping = {
- [commit1.merge_request_diff_id, commit1.relative_order] =>
- [author.id, committer.id],
-
- [commit2.merge_request_diff_id, commit2.relative_order] =>
- [author.id, nil]
- }
-
- migration.bulk_update_commit_rows(mapping)
-
- commit1 = commits.find_by(relative_order: 0)
- commit2 = commits.find_by(relative_order: 1)
-
- expect(commit1.commit_author_id).to eq(author.id)
- expect(commit1.committer_id).to eq(committer.id)
-
- expect(commit2.commit_author_id).to eq(author.id)
- expect(commit2.committer_id).to be_nil
- end
- end
- end
-
- describe '#primary_key' do
- it 'returns the primary key for the commits table' do
- key = migration.primary_key
-
- expect(key.to_sql).to eq('("merge_request_diff_commits"."merge_request_diff_id", "merge_request_diff_commits"."relative_order")')
- end
- end
-
- describe '#prepare' do
- it 'trims a value to at most 512 characters' do
- expect(migration.prepare('€' * 1_000)).to eq('€' * 512)
- end
-
- it 'returns nil if the value is an empty string' do
- expect(migration.prepare('')).to be_nil
- end
- end
-end
diff --git a/spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb b/spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb
deleted file mode 100644
index b252df4ecff..00000000000
--- a/spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb
+++ /dev/null
@@ -1,30 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-RSpec.describe Gitlab::BackgroundMigration::MigrateProjectTaggingsContextFromTagsToTopics,
- :suppress_gitlab_schemas_validate_connection, schema: 20210826171758 do
- it 'correctly migrates project taggings context from tags to topics' do
- taggings = table(:taggings)
-
- project_old_tagging_1 = taggings.create!(taggable_type: 'Project', context: 'tags')
- project_new_tagging_1 = taggings.create!(taggable_type: 'Project', context: 'topics')
- project_other_context_tagging_1 = taggings.create!(taggable_type: 'Project', context: 'other')
- project_old_tagging_2 = taggings.create!(taggable_type: 'Project', context: 'tags')
- project_old_tagging_3 = taggings.create!(taggable_type: 'Project', context: 'tags')
-
- subject.perform(project_old_tagging_1.id, project_old_tagging_2.id)
-
- project_old_tagging_1.reload
- project_new_tagging_1.reload
- project_other_context_tagging_1.reload
- project_old_tagging_2.reload
- project_old_tagging_3.reload
-
- expect(project_old_tagging_1.context).to eq('topics')
- expect(project_new_tagging_1.context).to eq('topics')
- expect(project_other_context_tagging_1.context).to eq('other')
- expect(project_old_tagging_2.context).to eq('topics')
- expect(project_old_tagging_3.context).to eq('tags')
- end
-end
diff --git a/spec/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings_spec.rb b/spec/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings_spec.rb
new file mode 100644
index 00000000000..b75c0e61b19
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings_spec.rb
@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::MigrateRemediationsForVulnerabilityFindings,
+ feature_category: :vulnerability_management do
+ let(:vulnerability_occurrences) { table(:vulnerability_occurrences) }
+ let(:vulnerability_findings_remediations) { table(:vulnerability_findings_remediations) }
+ let(:vulnerability_remediations) { table(:vulnerability_remediations) }
+ let(:remediation_hash) { { summary: 'summary', diff: "ZGlmZiAtLWdp" } }
+ let(:namespace1) { table(:namespaces).create!(name: 'namespace 1', path: 'namespace1') }
+ let(:project1) { table(:projects).create!(namespace_id: namespace1.id, project_namespace_id: namespace1.id) }
+ let(:user) { table(:users).create!(email: 'test1@example.com', projects_limit: 5) }
+
+ let(:scanner1) do
+ table(:vulnerability_scanners).create!(project_id: project1.id, external_id: 'test 1', name: 'test scanner 1')
+ end
+
+ let(:stating_id) { vulnerability_occurrences.pluck(:id).min }
+ let(:end_id) { vulnerability_occurrences.pluck(:id).max }
+
+ let(:migration) do
+ described_class.new(
+ start_id: stating_id,
+ end_id: end_id,
+ batch_table: :vulnerability_occurrences,
+ batch_column: :id,
+ sub_batch_size: 2,
+ pause_ms: 2,
+ connection: ApplicationRecord.connection
+ )
+ end
+
+ subject(:perform_migration) { migration.perform }
+
+ context 'without the presence of remediation key' do
+ before do
+ create_finding!(project1.id, scanner1.id, { other_keys: 'test' })
+ end
+
+ it 'does not create any remediation' do
+ expect(Gitlab::AppLogger).not_to receive(:error)
+
+ expect { perform_migration }.not_to change { vulnerability_remediations.count }
+ end
+ end
+
+ context 'with remediation equals to an array of nil element' do
+ before do
+ create_finding!(project1.id, scanner1.id, { remediations: [nil] })
+ end
+
+ it 'does not create any remediation' do
+ expect(Gitlab::AppLogger).not_to receive(:error)
+
+ expect { perform_migration }.not_to change { vulnerability_remediations.count }
+ end
+ end
+
+ context 'with remediation with empty string as the diff key' do
+ let!(:finding) do
+ create_finding!(project1.id, scanner1.id, { remediations: [{ summary: 'summary', diff: '' }] })
+ end
+
+ it 'does not create any remediation' do
+ expect(Gitlab::AppLogger).not_to receive(:error)
+
+ expect { perform_migration }.not_to change { vulnerability_remediations.count }
+ end
+ end
+
+ context 'with remediation equals to an array of duplicated elements' do
+ let!(:finding) do
+ create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash, remediation_hash] })
+ end
+
+ it 'creates new remediation' do
+ expect(Gitlab::AppLogger).not_to receive(:error)
+
+ expect { perform_migration }.to change { vulnerability_remediations.count }.by(1)
+ expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding.id).length).to eq(1)
+ end
+ end
+
+ context 'with existing remediations within raw_metadata' do
+ let!(:finding1) { create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash] }) }
+ let!(:finding2) { create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash] }) }
+
+ it 'creates new remediation' do
+ expect(Gitlab::AppLogger).not_to receive(:error)
+
+ expect { perform_migration }.to change { vulnerability_remediations.count }.by(1)
+ expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding1.id).length).to eq(1)
+ expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding2.id).length).to eq(1)
+ end
+
+ context 'when create throws exception other than ActiveRecord::RecordNotUnique' do
+ before do
+ allow(migration).to receive(:create_finding_remediations).and_raise(StandardError)
+ end
+
+ it 'rolls back all related transactions' do
+ expect(Gitlab::AppLogger).to receive(:error).with({
+ class: described_class.name, message: StandardError.to_s, model_id: finding1.id
+ })
+ expect(Gitlab::AppLogger).to receive(:error).with({
+ class: described_class.name, message: StandardError.to_s, model_id: finding2.id
+ })
+ expect { perform_migration }.not_to change { vulnerability_remediations.count }
+ expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding1.id).length).to eq(0)
+ expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding2.id).length).to eq(0)
+ end
+ end
+ end
+
+ context 'with existing remediation records' do
+ let!(:finding) { create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash] }) }
+
+ before do
+ vulnerability_remediations.create!(project_id: project1.id, summary: remediation_hash[:summary],
+ checksum: checksum(remediation_hash[:diff]), file: Tempfile.new.path)
+ end
+
+ it 'does not create new remediation' do
+ expect(Gitlab::AppLogger).not_to receive(:error)
+
+ expect { perform_migration }.not_to change { vulnerability_remediations.count }
+ expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding.id).length).to eq(1)
+ end
+ end
+
+ context 'with same raw_metadata for different projects' do
+ let(:namespace2) { table(:namespaces).create!(name: 'namespace 2', path: 'namespace2') }
+ let(:project2) { table(:projects).create!(namespace_id: namespace2.id, project_namespace_id: namespace2.id) }
+ let(:scanner2) do
+ table(:vulnerability_scanners).create!(project_id: project2.id, external_id: 'test 2', name: 'test scanner 2')
+ end
+
+ let!(:finding1) { create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash] }) }
+ let!(:finding2) { create_finding!(project2.id, scanner2.id, { remediations: [remediation_hash] }) }
+
+ it 'creates new remediation for each project' do
+ expect(Gitlab::AppLogger).not_to receive(:error)
+
+ expect { perform_migration }.to change { vulnerability_remediations.count }.by(2)
+ expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding1.id).length).to eq(1)
+ expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding2.id).length).to eq(1)
+ end
+ end
+
+ private
+
+ def create_finding!(project_id, scanner_id, raw_metadata)
+ vulnerability = table(:vulnerabilities).create!(project_id: project_id, author_id: user.id, title: 'test',
+ severity: 4, confidence: 4, report_type: 0)
+
+ identifier = table(:vulnerability_identifiers).create!(project_id: project_id, external_type: 'uuid-v5',
+ external_id: 'uuid-v5', fingerprint: OpenSSL::Digest::SHA256.hexdigest(vulnerability.id.to_s),
+ name: 'Identifier for UUIDv5 2 2')
+
+ table(:vulnerability_occurrences).create!(
+ vulnerability_id: vulnerability.id, project_id: project_id, scanner_id: scanner_id,
+ primary_identifier_id: identifier.id, name: 'test', severity: 4, confidence: 4, report_type: 0,
+ uuid: SecureRandom.uuid, project_fingerprint: '123qweasdzxc', location: { "image" => "alpine:3.4" },
+ location_fingerprint: 'test', metadata_version: 'test',
+ raw_metadata: raw_metadata.to_json)
+ end
+
+ def checksum(value)
+ sha = Digest::SHA256.hexdigest(value)
+ Gitlab::Database::ShaAttribute.new.serialize(sha)
+ end
+end
diff --git a/spec/lib/gitlab/background_migration/migrate_u2f_webauthn_spec.rb b/spec/lib/gitlab/background_migration/migrate_u2f_webauthn_spec.rb
deleted file mode 100644
index 08fde0d0ff4..00000000000
--- a/spec/lib/gitlab/background_migration/migrate_u2f_webauthn_spec.rb
+++ /dev/null
@@ -1,67 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-require 'webauthn/u2f_migrator'
-
-RSpec.describe Gitlab::BackgroundMigration::MigrateU2fWebauthn, :migration, schema: 20210826171758 do
- let(:users) { table(:users) }
-
- let(:user) { users.create!(email: 'email@email.com', name: 'foo', username: 'foo', projects_limit: 0) }
-
- let(:u2f_registrations) { table(:u2f_registrations) }
- let(:webauthn_registrations) { table(:webauthn_registrations) }
-
- let!(:u2f_registration_not_migrated) { create_u2f_registration(1, 'reg1') }
- let!(:u2f_registration_not_migrated_no_name) { create_u2f_registration(2, nil, 2) }
- let!(:u2f_registration_migrated) { create_u2f_registration(3, 'reg3') }
-
- subject { described_class.new.perform(1, 3) }
-
- before do
- converted_credential = convert_credential_for(u2f_registration_migrated)
- webauthn_registrations.create!(converted_credential)
- end
-
- it 'migrates all records' do
- expect { subject }.to change { webauthn_registrations.count }.from(1).to(3)
-
- all_webauthn_registrations = webauthn_registrations.all.map(&:attributes)
-
- [u2f_registration_not_migrated, u2f_registration_not_migrated_no_name].each do |u2f_registration|
- expected_credential = convert_credential_for(u2f_registration).except(:created_at).stringify_keys
- expect(all_webauthn_registrations).to include(a_hash_including(expected_credential))
- end
- end
-
- def create_u2f_registration(id, name, counter = 5)
- device = U2F::FakeU2F.new(FFaker::BaconIpsum.characters(5))
- u2f_registrations.create!({ id: id,
- certificate: Base64.strict_encode64(device.cert_raw),
- key_handle: U2F.urlsafe_encode64(device.key_handle_raw),
- public_key: Base64.strict_encode64(device.origin_public_key_raw),
- counter: counter,
- name: name,
- user_id: user.id })
- end
-
- def convert_credential_for(u2f_registration)
- converted_credential = WebAuthn::U2fMigrator.new(
- app_id: Gitlab.config.gitlab.url,
- certificate: u2f_registration.certificate,
- key_handle: u2f_registration.key_handle,
- public_key: u2f_registration.public_key,
- counter: u2f_registration.counter
- ).credential
-
- {
- credential_xid: Base64.strict_encode64(converted_credential.id),
- public_key: Base64.strict_encode64(converted_credential.public_key),
- counter: u2f_registration.counter,
- name: u2f_registration.name || '',
- user_id: u2f_registration.user_id,
- u2f_registration_id: u2f_registration.id,
- created_at: u2f_registration.created_at
- }
- end
-end
diff --git a/spec/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature_spec.rb b/spec/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature_spec.rb
deleted file mode 100644
index 71cf58a933f..00000000000
--- a/spec/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature_spec.rb
+++ /dev/null
@@ -1,98 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-RSpec.describe Gitlab::BackgroundMigration::MoveContainerRegistryEnabledToProjectFeature, :migration, schema: 20210826171758 do
- let(:enabled) { 20 }
- let(:disabled) { 0 }
-
- let(:namespaces) { table(:namespaces) }
- let(:project_features) { table(:project_features) }
- let(:projects) { table(:projects) }
-
- let(:namespace) { namespaces.create!(name: 'user', path: 'user') }
- let!(:project1) { projects.create!(namespace_id: namespace.id) }
- let!(:project2) { projects.create!(namespace_id: namespace.id) }
- let!(:project3) { projects.create!(namespace_id: namespace.id) }
- let!(:project4) { projects.create!(namespace_id: namespace.id) }
-
- # pages_access_level cannot be null.
- let(:non_null_project_features) { { pages_access_level: enabled } }
- let!(:project_feature1) { project_features.create!(project_id: project1.id, **non_null_project_features) }
- let!(:project_feature2) { project_features.create!(project_id: project2.id, **non_null_project_features) }
- let!(:project_feature3) { project_features.create!(project_id: project3.id, **non_null_project_features) }
-
- describe '#perform' do
- before do
- project1.update!(container_registry_enabled: true)
- project2.update!(container_registry_enabled: false)
- project3.update!(container_registry_enabled: nil)
- project4.update!(container_registry_enabled: true)
- end
-
- it 'copies values to project_features' do
- table(:background_migration_jobs).create!(
- class_name: 'MoveContainerRegistryEnabledToProjectFeature',
- arguments: [project1.id, project4.id]
- )
- table(:background_migration_jobs).create!(
- class_name: 'MoveContainerRegistryEnabledToProjectFeature',
- arguments: [-1, -3]
- )
-
- expect(project1.container_registry_enabled).to eq(true)
- expect(project2.container_registry_enabled).to eq(false)
- expect(project3.container_registry_enabled).to eq(nil)
- expect(project4.container_registry_enabled).to eq(true)
-
- expect(project_feature1.container_registry_access_level).to eq(disabled)
- expect(project_feature2.container_registry_access_level).to eq(disabled)
- expect(project_feature3.container_registry_access_level).to eq(disabled)
-
- expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |logger|
- expect(logger).to receive(:info)
- .with(message: "#{described_class}: Copied container_registry_enabled values for projects with IDs between #{project1.id}..#{project4.id}")
-
- expect(logger).not_to receive(:info)
- end
-
- subject.perform(project1.id, project4.id)
-
- expect(project1.reload.container_registry_enabled).to eq(true)
- expect(project2.reload.container_registry_enabled).to eq(false)
- expect(project3.reload.container_registry_enabled).to eq(nil)
- expect(project4.container_registry_enabled).to eq(true)
-
- expect(project_feature1.reload.container_registry_access_level).to eq(enabled)
- expect(project_feature2.reload.container_registry_access_level).to eq(disabled)
- expect(project_feature3.reload.container_registry_access_level).to eq(disabled)
-
- expect(table(:background_migration_jobs).first.status).to eq(1) # succeeded
- expect(table(:background_migration_jobs).second.status).to eq(0) # pending
- end
-
- context 'when no projects exist in range' do
- it 'does not fail' do
- expect(project1.container_registry_enabled).to eq(true)
- expect(project_feature1.container_registry_access_level).to eq(disabled)
-
- expect { subject.perform(-1, -2) }.not_to raise_error
-
- expect(project1.container_registry_enabled).to eq(true)
- expect(project_feature1.container_registry_access_level).to eq(disabled)
- end
- end
-
- context 'when projects in range all have nil container_registry_enabled' do
- it 'does not fail' do
- expect(project3.container_registry_enabled).to eq(nil)
- expect(project_feature3.container_registry_access_level).to eq(disabled)
-
- expect { subject.perform(project3.id, project3.id) }.not_to raise_error
-
- expect(project3.container_registry_enabled).to eq(nil)
- expect(project_feature3.container_registry_access_level).to eq(disabled)
- end
- end
- end
-end
diff --git a/spec/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects_spec.rb b/spec/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects_spec.rb
index a8574411957..f671a673a08 100644
--- a/spec/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects_spec.rb
+++ b/spec/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects_spec.rb
@@ -2,7 +2,8 @@
require 'spec_helper'
-RSpec.describe Gitlab::BackgroundMigration::NullifyCreatorIdColumnOfOrphanedProjects, feature_category: :projects do
+RSpec.describe Gitlab::BackgroundMigration::NullifyCreatorIdColumnOfOrphanedProjects, feature_category: :projects,
+ schema: 20230130073109 do
let(:users) { table(:users) }
let(:projects) { table(:projects) }
let(:namespaces) { table(:namespaces) }
diff --git a/spec/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds_spec.rb b/spec/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds_spec.rb
index 2f0eef3c399..5b234679e22 100644
--- a/spec/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds_spec.rb
+++ b/spec/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds_spec.rb
@@ -3,7 +3,7 @@
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::NullifyOrphanRunnerIdOnCiBuilds,
- :suppress_gitlab_schemas_validate_connection, migration: :gitlab_ci, schema: 20220223112304 do
+ :suppress_gitlab_schemas_validate_connection, migration: :gitlab_ci, schema: 20220223112304 do
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:ci_runners) { table(:ci_runners) }
diff --git a/spec/lib/gitlab/background_migration/populate_topics_total_projects_count_cache_spec.rb b/spec/lib/gitlab/background_migration/populate_topics_total_projects_count_cache_spec.rb
deleted file mode 100644
index 8e07b43f5b9..00000000000
--- a/spec/lib/gitlab/background_migration/populate_topics_total_projects_count_cache_spec.rb
+++ /dev/null
@@ -1,35 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-RSpec.describe Gitlab::BackgroundMigration::PopulateTopicsTotalProjectsCountCache, schema: 20211006060436 do
- it 'correctly populates total projects count cache' do
- namespaces = table(:namespaces)
- projects = table(:projects)
- topics = table(:topics)
- project_topics = table(:project_topics)
-
- group = namespaces.create!(name: 'group', path: 'group')
- project_1 = projects.create!(namespace_id: group.id)
- project_2 = projects.create!(namespace_id: group.id)
- project_3 = projects.create!(namespace_id: group.id)
- topic_1 = topics.create!(name: 'Topic1')
- topic_2 = topics.create!(name: 'Topic2')
- topic_3 = topics.create!(name: 'Topic3')
- topic_4 = topics.create!(name: 'Topic4')
-
- project_topics.create!(project_id: project_1.id, topic_id: topic_1.id)
- project_topics.create!(project_id: project_1.id, topic_id: topic_3.id)
- project_topics.create!(project_id: project_2.id, topic_id: topic_3.id)
- project_topics.create!(project_id: project_1.id, topic_id: topic_4.id)
- project_topics.create!(project_id: project_2.id, topic_id: topic_4.id)
- project_topics.create!(project_id: project_3.id, topic_id: topic_4.id)
-
- subject.perform(topic_1.id, topic_4.id)
-
- expect(topic_1.reload.total_projects_count).to eq(1)
- expect(topic_2.reload.total_projects_count).to eq(0)
- expect(topic_3.reload.total_projects_count).to eq(2)
- expect(topic_4.reload.total_projects_count).to eq(3)
- end
-end
diff --git a/spec/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields_spec.rb b/spec/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields_spec.rb
new file mode 100644
index 00000000000..50380247c9f
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields_spec.rb
@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::PopulateVulnerabilityDismissalFields, schema: 20230412185837, feature_category: :vulnerability_management do # rubocop:disable Layout/LineLength
+ let(:users) { table(:users) }
+ let(:namespaces) { table(:namespaces) }
+ let(:projects) { table(:projects) }
+ let(:vulnerabilities) { table(:vulnerabilities) }
+ let(:findings) { table(:vulnerability_occurrences) }
+ let(:scanners) { table(:vulnerability_scanners) }
+ let(:identifiers) { table(:vulnerability_identifiers) }
+ let(:feedback) { table(:vulnerability_feedback) }
+
+ let(:user) { users.create!(name: 'test', email: 'test@example.com', projects_limit: 5) }
+ let(:namespace) { namespaces.create!(name: 'gitlab', path: 'gitlab-org') }
+ let(:project) { projects.create!(namespace_id: namespace.id, name: 'foo', project_namespace_id: namespace.id) }
+ let(:vulnerability_1) do
+ vulnerabilities.create!(title: 'title', state: 2, severity: 0,
+ confidence: 5, report_type: 2, project_id: project.id, author_id: user.id
+ )
+ end
+
+ let(:vulnerability_2) do
+ vulnerabilities.create!(title: 'title', state: 2, severity: 0,
+ confidence: 5, report_type: 2, project_id: project.id, author_id: user.id
+ )
+ end
+
+ let(:scanner) { scanners.create!(project_id: project.id, external_id: 'foo', name: 'bar') }
+ let(:identifier) do
+ identifiers.create!(project_id: project.id, fingerprint: 'foo',
+ external_type: 'bar', external_id: 'zoo', name: 'identifier'
+ )
+ end
+
+ let(:uuid) { SecureRandom.uuid }
+
+ before do
+ feedback.create!(feedback_type: 0,
+ category: 'sast',
+ project_fingerprint: '418291a26024a1445b23fe64de9380cdcdfd1fa8',
+ project_id: project.id,
+ author_id: user.id,
+ created_at: Time.current,
+ finding_uuid: uuid
+ )
+
+ findings.create!(name: 'Finding',
+ report_type: 'sast',
+ project_fingerprint: '418291a26024a1445b23fe64de9380cdcdfd1f98',
+ location_fingerprint: 'bar',
+ severity: 1,
+ confidence: 1,
+ metadata_version: 1,
+ raw_metadata: '',
+ details: {},
+ uuid: uuid,
+ project_id: project.id,
+ vulnerability_id: vulnerability_1.id,
+ scanner_id: scanner.id,
+ primary_identifier_id: identifier.id
+ )
+
+ allow(::Gitlab::BackgroundMigration::Logger).to receive_messages(info: true, warn: true, error: true)
+ end
+
+ subject do
+ described_class.new(
+ start_id: vulnerability_1.id,
+ end_id: vulnerability_2.id,
+ batch_table: :vulnerabilities,
+ batch_column: :id,
+ sub_batch_size: 200,
+ pause_ms: 2.minutes,
+ connection: ApplicationRecord.connection
+ )
+ end
+
+ describe '#perform' do
+ it 'updates the missing dismissal information of the vulnerability' do
+ expect { subject.perform }.to change { vulnerability_1.reload.dismissed_at }
+ .from(nil)
+ .and change { vulnerability_1.reload.dismissed_by_id }.from(nil).to(user.id)
+ end
+
+ it 'writes log messages', :aggregate_failures do
+ subject.perform
+
+ expect(::Gitlab::BackgroundMigration::Logger).to have_received(:info).with(migrator: described_class.name,
+ message: 'Dismissal information has been copied',
+ count: 2
+ )
+ expect(::Gitlab::BackgroundMigration::Logger).to have_received(:warn).with(migrator: described_class.name,
+ message: 'Could not update vulnerability!',
+ vulnerability_id: vulnerability_2.id
+ )
+ end
+
+ context 'when logger throws exception StandardError' do
+ before do
+ allow(::Gitlab::BackgroundMigration::Logger).to receive(:warn).and_raise(StandardError)
+ end
+
+ it 'logs StandardError' do
+ expect(::Gitlab::BackgroundMigration::Logger).to receive(:error).with({
+ migrator: described_class.name, message: StandardError.to_s, vulnerability_id: vulnerability_2.id
+ })
+
+ subject.perform
+ end
+ end
+ end
+end
diff --git a/spec/lib/gitlab/background_migration/prune_stale_project_export_jobs_spec.rb b/spec/lib/gitlab/background_migration/prune_stale_project_export_jobs_spec.rb
index 5150d0ea4b0..3446b9f0676 100644
--- a/spec/lib/gitlab/background_migration/prune_stale_project_export_jobs_spec.rb
+++ b/spec/lib/gitlab/background_migration/prune_stale_project_export_jobs_spec.rb
@@ -10,14 +10,15 @@ RSpec.describe Gitlab::BackgroundMigration::PruneStaleProjectExportJobs, feature
let(:uploads) { table(:project_relation_export_uploads) }
subject(:perform_migration) do
- described_class.new(start_id: 1,
- end_id: 300,
- batch_table: :project_export_jobs,
- batch_column: :id,
- sub_batch_size: 2,
- pause_ms: 0,
- connection: ActiveRecord::Base.connection)
- .perform
+ described_class.new(
+ start_id: 1,
+ end_id: 300,
+ batch_table: :project_export_jobs,
+ batch_column: :id,
+ sub_batch_size: 2,
+ pause_ms: 0,
+ connection: ActiveRecord::Base.connection
+ ).perform
end
it 'removes export jobs and associated relations older than 7 days' do
diff --git a/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb b/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb
deleted file mode 100644
index 2271bbfb2f3..00000000000
--- a/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb
+++ /dev/null
@@ -1,530 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-def create_background_migration_job(ids, status)
- proper_status = case status
- when :pending
- Gitlab::Database::BackgroundMigrationJob.statuses['pending']
- when :succeeded
- Gitlab::Database::BackgroundMigrationJob.statuses['succeeded']
- else
- raise ArgumentError
- end
-
- background_migration_jobs.create!(
- class_name: 'RecalculateVulnerabilitiesOccurrencesUuid',
- arguments: Array(ids),
- status: proper_status,
- created_at: Time.now.utc
- )
-end
-
-RSpec.describe Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid, :suppress_gitlab_schemas_validate_connection, schema: 20211124132705 do
- let(:background_migration_jobs) { table(:background_migration_jobs) }
- let(:pending_jobs) { background_migration_jobs.where(status: Gitlab::Database::BackgroundMigrationJob.statuses['pending']) }
- let(:succeeded_jobs) { background_migration_jobs.where(status: Gitlab::Database::BackgroundMigrationJob.statuses['succeeded']) }
- let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') }
- let(:users) { table(:users) }
- let(:user) { create_user! }
- let(:project) { table(:projects).create!(id: 123, namespace_id: namespace.id) }
- let(:scanners) { table(:vulnerability_scanners) }
- let(:scanner) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') }
- let(:scanner2) { scanners.create!(project_id: project.id, external_id: 'test 2', name: 'test scanner 2') }
- let(:vulnerabilities) { table(:vulnerabilities) }
- let(:vulnerability_findings) { table(:vulnerability_occurrences) }
- let(:vulnerability_finding_pipelines) { table(:vulnerability_occurrence_pipelines) }
- let(:vulnerability_finding_signatures) { table(:vulnerability_finding_signatures) }
- let(:vulnerability_identifiers) { table(:vulnerability_identifiers) }
-
- let(:identifier_1) { 'identifier-1' }
- let!(:vulnerability_identifier) do
- vulnerability_identifiers.create!(
- project_id: project.id,
- external_type: identifier_1,
- external_id: identifier_1,
- fingerprint: Gitlab::Database::ShaAttribute.serialize('ff9ef548a6e30a0462795d916f3f00d1e2b082ca'),
- name: 'Identifier 1')
- end
-
- let(:identifier_2) { 'identifier-2' }
- let!(:vulnerability_identfier2) do
- vulnerability_identifiers.create!(
- project_id: project.id,
- external_type: identifier_2,
- external_id: identifier_2,
- fingerprint: Gitlab::Database::ShaAttribute.serialize('4299e8ddd819f9bde9cfacf45716724c17b5ddf7'),
- name: 'Identifier 2')
- end
-
- let(:identifier_3) { 'identifier-3' }
- let!(:vulnerability_identifier3) do
- vulnerability_identifiers.create!(
- project_id: project.id,
- external_type: identifier_3,
- external_id: identifier_3,
- fingerprint: Gitlab::Database::ShaAttribute.serialize('8e91632f9c6671e951834a723ee221c44cc0d844'),
- name: 'Identifier 3')
- end
-
- let(:known_uuid_v4) { "b3cc2518-5446-4dea-871c-89d5e999c1ac" }
- let(:known_uuid_v5) { "05377088-dc26-5161-920e-52a7159fdaa1" }
- let(:desired_uuid_v5) { "f3e9a23f-9181-54bf-a5ab-c5bc7a9b881a" }
-
- subject { described_class.new.perform(start_id, end_id) }
-
- context "when finding has a UUIDv4" do
- before do
- @uuid_v4 = create_finding!(
- vulnerability_id: nil,
- project_id: project.id,
- scanner_id: scanner2.id,
- primary_identifier_id: vulnerability_identfier2.id,
- report_type: 0, # "sast"
- location_fingerprint: Gitlab::Database::ShaAttribute.serialize("fa18f432f1d56675f4098d318739c3cd5b14eb3e"),
- uuid: known_uuid_v4
- )
- end
-
- let(:start_id) { @uuid_v4.id }
- let(:end_id) { @uuid_v4.id }
-
- it "replaces it with UUIDv5" do
- expect(vulnerability_findings.pluck(:uuid)).to match_array([known_uuid_v4])
-
- subject
-
- expect(vulnerability_findings.pluck(:uuid)).to match_array([desired_uuid_v5])
- end
-
- it 'logs recalculation' do
- expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |instance|
- expect(instance).to receive(:info).twice
- end
-
- subject
- end
- end
-
- context "when finding has a UUIDv5" do
- before do
- @uuid_v5 = create_finding!(
- vulnerability_id: nil,
- project_id: project.id,
- scanner_id: scanner.id,
- primary_identifier_id: vulnerability_identifier.id,
- report_type: 0, # "sast"
- location_fingerprint: Gitlab::Database::ShaAttribute.serialize("838574be0210968bf6b9f569df9c2576242cbf0a"),
- uuid: known_uuid_v5
- )
- end
-
- let(:start_id) { @uuid_v5.id }
- let(:end_id) { @uuid_v5.id }
-
- it "stays the same" do
- expect(vulnerability_findings.pluck(:uuid)).to match_array([known_uuid_v5])
-
- subject
-
- expect(vulnerability_findings.pluck(:uuid)).to match_array([known_uuid_v5])
- end
- end
-
- context 'if a duplicate UUID would be generated' do # rubocop: disable RSpec/MultipleMemoizedHelpers
- let(:v1) do
- create_vulnerability!(
- project_id: project.id,
- author_id: user.id
- )
- end
-
- let!(:finding_with_incorrect_uuid) do
- create_finding!(
- vulnerability_id: v1.id,
- project_id: project.id,
- scanner_id: scanner.id,
- primary_identifier_id: vulnerability_identifier.id,
- report_type: 0, # "sast"
- location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis')
- uuid: 'bd95c085-71aa-51d7-9bb6-08ae669c262e'
- )
- end
-
- let(:v2) do
- create_vulnerability!(
- project_id: project.id,
- author_id: user.id
- )
- end
-
- let!(:finding_with_correct_uuid) do
- create_finding!(
- vulnerability_id: v2.id,
- project_id: project.id,
- primary_identifier_id: vulnerability_identifier.id,
- scanner_id: scanner2.id,
- report_type: 0, # "sast"
- location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis')
- uuid: '91984483-5efe-5215-b471-d524ac5792b1'
- )
- end
-
- let(:v3) do
- create_vulnerability!(
- project_id: project.id,
- author_id: user.id
- )
- end
-
- let!(:finding_with_incorrect_uuid2) do
- create_finding!(
- vulnerability_id: v3.id,
- project_id: project.id,
- scanner_id: scanner.id,
- primary_identifier_id: vulnerability_identfier2.id,
- report_type: 0, # "sast"
- location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis')
- uuid: '00000000-1111-2222-3333-444444444444'
- )
- end
-
- let(:v4) do
- create_vulnerability!(
- project_id: project.id,
- author_id: user.id
- )
- end
-
- let!(:finding_with_correct_uuid2) do
- create_finding!(
- vulnerability_id: v4.id,
- project_id: project.id,
- scanner_id: scanner2.id,
- primary_identifier_id: vulnerability_identfier2.id,
- report_type: 0, # "sast"
- location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis')
- uuid: '1edd751e-ef9a-5391-94db-a832c8635bfc'
- )
- end
-
- let!(:finding_with_incorrect_uuid3) do
- create_finding!(
- vulnerability_id: nil,
- project_id: project.id,
- scanner_id: scanner.id,
- primary_identifier_id: vulnerability_identifier3.id,
- report_type: 0, # "sast"
- location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis')
- uuid: '22222222-3333-4444-5555-666666666666'
- )
- end
-
- let!(:duplicate_not_in_the_same_batch) do
- create_finding!(
- id: 99999,
- vulnerability_id: nil,
- project_id: project.id,
- scanner_id: scanner2.id,
- primary_identifier_id: vulnerability_identifier3.id,
- report_type: 0, # "sast"
- location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis')
- uuid: '4564f9d5-3c6b-5cc3-af8c-7c25285362a7'
- )
- end
-
- let(:start_id) { finding_with_incorrect_uuid.id }
- let(:end_id) { finding_with_incorrect_uuid3.id }
-
- before do
- 4.times do
- create_finding_pipeline!(project_id: project.id, finding_id: finding_with_incorrect_uuid.id)
- create_finding_pipeline!(project_id: project.id, finding_id: finding_with_correct_uuid.id)
- create_finding_pipeline!(project_id: project.id, finding_id: finding_with_incorrect_uuid2.id)
- create_finding_pipeline!(project_id: project.id, finding_id: finding_with_correct_uuid2.id)
- end
- end
-
- it 'drops duplicates and related records', :aggregate_failures do
- expect(vulnerability_findings.pluck(:id)).to match_array(
- [
- finding_with_correct_uuid.id,
- finding_with_incorrect_uuid.id,
- finding_with_correct_uuid2.id,
- finding_with_incorrect_uuid2.id,
- finding_with_incorrect_uuid3.id,
- duplicate_not_in_the_same_batch.id
- ])
-
- expect { subject }.to change(vulnerability_finding_pipelines, :count).from(16).to(8)
- .and change(vulnerability_findings, :count).from(6).to(3)
- .and change(vulnerabilities, :count).from(4).to(2)
-
- expect(vulnerability_findings.pluck(:id)).to match_array([finding_with_incorrect_uuid.id, finding_with_incorrect_uuid2.id, finding_with_incorrect_uuid3.id])
- end
-
- context 'if there are conflicting UUID values within the batch' do # rubocop: disable RSpec/MultipleMemoizedHelpers
- let(:end_id) { finding_with_broken_data_integrity.id }
- let(:vulnerability_5) { create_vulnerability!(project_id: project.id, author_id: user.id) }
- let(:different_project) { table(:projects).create!(namespace_id: namespace.id) }
- let!(:identifier_with_broken_data_integrity) do
- vulnerability_identifiers.create!(
- project_id: different_project.id,
- external_type: identifier_2,
- external_id: identifier_2,
- fingerprint: Gitlab::Database::ShaAttribute.serialize('4299e8ddd819f9bde9cfacf45716724c17b5ddf7'),
- name: 'Identifier 2')
- end
-
- let(:finding_with_broken_data_integrity) do
- create_finding!(
- vulnerability_id: vulnerability_5,
- project_id: project.id,
- scanner_id: scanner.id,
- primary_identifier_id: identifier_with_broken_data_integrity.id,
- report_type: 0, # "sast"
- location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis')
- uuid: SecureRandom.uuid
- )
- end
-
- it 'deletes the conflicting record' do
- expect { subject }.to change { vulnerability_findings.find_by_id(finding_with_broken_data_integrity.id) }.to(nil)
- end
- end
-
- context 'if a conflicting UUID is found during the migration' do # rubocop:disable RSpec/MultipleMemoizedHelpers
- let(:finding_class) { Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid::VulnerabilitiesFinding }
- let(:uuid) { '4564f9d5-3c6b-5cc3-af8c-7c25285362a7' }
-
- before do
- exception = ActiveRecord::RecordNotUnique.new("(uuid)=(#{uuid})")
-
- call_count = 0
- allow(::Gitlab::Database::BulkUpdate).to receive(:execute) do
- call_count += 1
- call_count.eql?(1) ? raise(exception) : {}
- end
-
- allow(finding_class).to receive(:find_by).with(uuid: uuid).and_return(duplicate_not_in_the_same_batch)
- end
-
- it 'retries the recalculation' do
- subject
-
- expect(Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid::VulnerabilitiesFinding)
- .to have_received(:find_by).with(uuid: uuid).once
- end
-
- it 'logs the conflict' do
- expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |instance|
- expect(instance).to receive(:info).exactly(6).times
- end
-
- subject
- end
-
- it 'marks the job as done' do
- create_background_migration_job([start_id, end_id], :pending)
-
- subject
-
- expect(pending_jobs.count).to eq(0)
- expect(succeeded_jobs.count).to eq(1)
- end
- end
-
- it 'logs an exception if a different uniquness problem was found' do
- exception = ActiveRecord::RecordNotUnique.new("Totally not an UUID uniqueness problem")
- allow(::Gitlab::Database::BulkUpdate).to receive(:execute).and_raise(exception)
- allow(Gitlab::ErrorTracking).to receive(:track_and_raise_exception)
-
- subject
-
- expect(Gitlab::ErrorTracking).to have_received(:track_and_raise_exception).with(exception).once
- end
-
- it 'logs a duplicate found message' do
- expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |instance|
- expect(instance).to receive(:info).exactly(3).times
- end
-
- subject
- end
- end
-
- context 'when finding has a signature' do
- before do
- @f1 = create_finding!(
- vulnerability_id: nil,
- project_id: project.id,
- scanner_id: scanner.id,
- primary_identifier_id: vulnerability_identifier.id,
- report_type: 0, # "sast"
- location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis')
- uuid: 'd15d774d-e4b1-5a1b-929b-19f2a53e35ec'
- )
-
- vulnerability_finding_signatures.create!(
- finding_id: @f1.id,
- algorithm_type: 2, # location
- signature_sha: Gitlab::Database::ShaAttribute.serialize('57d4e05205f6462a73f039a5b2751aa1ab344e6e') # sha1('youshouldusethis')
- )
-
- vulnerability_finding_signatures.create!(
- finding_id: @f1.id,
- algorithm_type: 1, # hash
- signature_sha: Gitlab::Database::ShaAttribute.serialize('c554d8d8df1a7a14319eafdaae24af421bf5b587') # sha1('andnotthis')
- )
-
- @f2 = create_finding!(
- vulnerability_id: nil,
- project_id: project.id,
- scanner_id: scanner.id,
- primary_identifier_id: vulnerability_identfier2.id,
- report_type: 0, # "sast"
- location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis')
- uuid: '4be029b5-75e5-5ac0-81a2-50ab41726135'
- )
-
- vulnerability_finding_signatures.create!(
- finding_id: @f2.id,
- algorithm_type: 2, # location
- signature_sha: Gitlab::Database::ShaAttribute.serialize('57d4e05205f6462a73f039a5b2751aa1ab344e6e') # sha1('youshouldusethis')
- )
-
- vulnerability_finding_signatures.create!(
- finding_id: @f2.id,
- algorithm_type: 1, # hash
- signature_sha: Gitlab::Database::ShaAttribute.serialize('c554d8d8df1a7a14319eafdaae24af421bf5b587') # sha1('andnotthis')
- )
- end
-
- let(:start_id) { @f1.id }
- let(:end_id) { @f2.id }
-
- let(:uuids_before) { [@f1.uuid, @f2.uuid] }
- let(:uuids_after) { %w[d3b60ddd-d312-5606-b4d3-ad058eebeacb 349d9bec-c677-5530-a8ac-5e58889c3b1a] }
-
- it 'is recalculated using signature' do
- expect(vulnerability_findings.pluck(:uuid)).to match_array(uuids_before)
-
- subject
-
- expect(vulnerability_findings.pluck(:uuid)).to match_array(uuids_after)
- end
- end
-
- context 'if all records are removed before the job ran' do
- let(:start_id) { 1 }
- let(:end_id) { 9 }
-
- before do
- create_background_migration_job([start_id, end_id], :pending)
- end
-
- it 'does not error out' do
- expect { subject }.not_to raise_error
- end
-
- it 'marks the job as done' do
- subject
-
- expect(pending_jobs.count).to eq(0)
- expect(succeeded_jobs.count).to eq(1)
- end
- end
-
- context 'when recalculation fails' do
- before do
- @uuid_v4 = create_finding!(
- vulnerability_id: nil,
- project_id: project.id,
- scanner_id: scanner2.id,
- primary_identifier_id: vulnerability_identfier2.id,
- report_type: 0, # "sast"
- location_fingerprint: Gitlab::Database::ShaAttribute.serialize("fa18f432f1d56675f4098d318739c3cd5b14eb3e"),
- uuid: known_uuid_v4
- )
-
- allow(Gitlab::ErrorTracking).to receive(:track_and_raise_exception)
- allow(::Gitlab::Database::BulkUpdate).to receive(:execute).and_raise(expected_error)
- end
-
- let(:start_id) { @uuid_v4.id }
- let(:end_id) { @uuid_v4.id }
- let(:expected_error) { RuntimeError.new }
-
- it 'captures the errors and does not crash entirely' do
- expect { subject }.not_to raise_error
-
- allow(Gitlab::ErrorTracking).to receive(:track_and_raise_exception)
- expect(Gitlab::ErrorTracking).to have_received(:track_and_raise_exception).with(expected_error).once
- end
-
- it_behaves_like 'marks background migration job records' do
- let(:arguments) { [1, 4] }
- subject { described_class.new }
- end
- end
-
- it_behaves_like 'marks background migration job records' do
- let(:arguments) { [1, 4] }
- subject { described_class.new }
- end
-
- private
-
- def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0)
- vulnerabilities.create!(
- project_id: project_id,
- author_id: author_id,
- title: title,
- severity: severity,
- confidence: confidence,
- report_type: report_type
- )
- end
-
- # rubocop:disable Metrics/ParameterLists
- def create_finding!(
- vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:, id: nil,
- name: "test", severity: 7, confidence: 7, report_type: 0,
- project_fingerprint: '123qweasdzxc', location_fingerprint: 'test',
- metadata_version: 'test', raw_metadata: 'test', uuid: SecureRandom.uuid)
- vulnerability_findings.create!({
- id: id,
- vulnerability_id: vulnerability_id,
- project_id: project_id,
- name: name,
- severity: severity,
- confidence: confidence,
- report_type: report_type,
- project_fingerprint: project_fingerprint,
- scanner_id: scanner_id,
- primary_identifier_id: primary_identifier_id,
- location_fingerprint: location_fingerprint,
- metadata_version: metadata_version,
- raw_metadata: raw_metadata,
- uuid: uuid
- }.compact
- )
- end
- # rubocop:enable Metrics/ParameterLists
-
- def create_user!(name: "Example User", email: "user@example.com", user_type: nil, created_at: Time.zone.now, confirmed_at: Time.zone.now)
- users.create!(
- name: name,
- email: email,
- username: name,
- projects_limit: 0,
- user_type: user_type,
- confirmed_at: confirmed_at
- )
- end
-
- def create_finding_pipeline!(project_id:, finding_id:)
- pipeline = table(:ci_pipelines).create!(project_id: project_id)
- vulnerability_finding_pipelines.create!(pipeline_id: pipeline.id, occurrence_id: finding_id)
- end
-end
diff --git a/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb b/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb
index 5fede892463..582c0fe1b1b 100644
--- a/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb
+++ b/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb
@@ -86,8 +86,10 @@ RSpec.describe Gitlab::BackgroundMigration::RemoveBackfilledJobArtifactsExpireAt
def create_job_artifact(id:, file_type:, expire_at:)
job = table(:ci_builds, database: :ci).create!(id: id, partition_id: 100)
- job_artifact.create!(id: id, job_id: job.id, expire_at: expire_at, project_id: project.id,
- file_type: file_type, partition_id: 100)
+ job_artifact.create!(
+ id: id, job_id: job.id, expire_at: expire_at, project_id: project.id,
+ file_type: file_type, partition_id: 100
+ )
end
end
end
diff --git a/spec/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings_spec.rb b/spec/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings_spec.rb
deleted file mode 100644
index ed08ae22245..00000000000
--- a/spec/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings_spec.rb
+++ /dev/null
@@ -1,171 +0,0 @@
-# frozen_string_literal: true
-require 'spec_helper'
-
-RSpec.describe Gitlab::BackgroundMigration::RemoveDuplicateVulnerabilitiesFindings, :migration, schema: 20220326161803 do
- let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') }
- let(:users) { table(:users) }
- let(:user) { create_user! }
- let(:project) { table(:projects).create!(id: 14219619, namespace_id: namespace.id) }
- let(:scanners) { table(:vulnerability_scanners) }
- let!(:scanner1) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') }
- let!(:scanner2) { scanners.create!(project_id: project.id, external_id: 'test 2', name: 'test scanner 2') }
- let!(:scanner3) { scanners.create!(project_id: project.id, external_id: 'test 3', name: 'test scanner 3') }
- let!(:unrelated_scanner) { scanners.create!(project_id: project.id, external_id: 'unreleated_scanner', name: 'unrelated scanner') }
- let(:vulnerabilities) { table(:vulnerabilities) }
- let(:vulnerability_findings) { table(:vulnerability_occurrences) }
- let(:vulnerability_identifiers) { table(:vulnerability_identifiers) }
- let(:vulnerability_identifier) do
- vulnerability_identifiers.create!(
- id: 1244459,
- project_id: project.id,
- external_type: 'vulnerability-identifier',
- external_id: 'vulnerability-identifier',
- fingerprint: '0a203e8cd5260a1948edbedc76c7cb91ad6a2e45',
- name: 'vulnerability identifier')
- end
-
- let!(:vulnerability_for_first_duplicate) do
- create_vulnerability!(
- project_id: project.id,
- author_id: user.id
- )
- end
-
- let!(:first_finding_duplicate) do
- create_finding!(
- id: 5606961,
- uuid: "bd95c085-71aa-51d7-9bb6-08ae669c262e",
- vulnerability_id: vulnerability_for_first_duplicate.id,
- report_type: 0,
- location_fingerprint: '00049d5119c2cb3bfb3d1ee1f6e031fe925aed75',
- primary_identifier_id: vulnerability_identifier.id,
- scanner_id: scanner1.id,
- project_id: project.id
- )
- end
-
- let!(:vulnerability_for_second_duplicate) do
- create_vulnerability!(
- project_id: project.id,
- author_id: user.id
- )
- end
-
- let!(:second_finding_duplicate) do
- create_finding!(
- id: 8765432,
- uuid: "5b714f58-1176-5b26-8fd5-e11dfcb031b5",
- vulnerability_id: vulnerability_for_second_duplicate.id,
- report_type: 0,
- location_fingerprint: '00049d5119c2cb3bfb3d1ee1f6e031fe925aed75',
- primary_identifier_id: vulnerability_identifier.id,
- scanner_id: scanner2.id,
- project_id: project.id
- )
- end
-
- let!(:vulnerability_for_third_duplicate) do
- create_vulnerability!(
- project_id: project.id,
- author_id: user.id
- )
- end
-
- let!(:third_finding_duplicate) do
- create_finding!(
- id: 8832995,
- uuid: "cfe435fa-b25b-5199-a56d-7b007cc9e2d4",
- vulnerability_id: vulnerability_for_third_duplicate.id,
- report_type: 0,
- location_fingerprint: '00049d5119c2cb3bfb3d1ee1f6e031fe925aed75',
- primary_identifier_id: vulnerability_identifier.id,
- scanner_id: scanner3.id,
- project_id: project.id
- )
- end
-
- let!(:unrelated_finding) do
- create_finding!(
- id: 9999999,
- uuid: Gitlab::UUID.v5(SecureRandom.hex),
- vulnerability_id: nil,
- report_type: 1,
- location_fingerprint: 'random_location_fingerprint',
- primary_identifier_id: vulnerability_identifier.id,
- scanner_id: unrelated_scanner.id,
- project_id: project.id
- )
- end
-
- subject { described_class.new.perform(first_finding_duplicate.id, unrelated_finding.id) }
-
- before do
- stub_const("#{described_class}::DELETE_BATCH_SIZE", 1)
- end
-
- it "removes entries which would result in duplicate UUIDv5" do
- expect(vulnerability_findings.count).to eq(4)
-
- expect { subject }.to change { vulnerability_findings.count }.from(4).to(2)
-
- expect(vulnerability_findings.pluck(:id)).to match_array([third_finding_duplicate.id, unrelated_finding.id])
- end
-
- it "removes vulnerabilites without findings" do
- expect(vulnerabilities.count).to eq(3)
-
- expect { subject }.to change { vulnerabilities.count }.from(3).to(1)
-
- expect(vulnerabilities.pluck(:id)).to match_array([vulnerability_for_third_duplicate.id])
- end
-
- private
-
- def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0)
- vulnerabilities.create!(
- project_id: project_id,
- author_id: author_id,
- title: title,
- severity: severity,
- confidence: confidence,
- report_type: report_type
- )
- end
-
- # rubocop:disable Metrics/ParameterLists
- def create_finding!(
- vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:, id: nil,
- name: "test", severity: 7, confidence: 7, report_type: 0,
- project_fingerprint: '123qweasdzxc', location_fingerprint: 'test',
- metadata_version: 'test', raw_metadata: 'test', uuid: SecureRandom.uuid)
- params = {
- vulnerability_id: vulnerability_id,
- project_id: project_id,
- name: name,
- severity: severity,
- confidence: confidence,
- report_type: report_type,
- project_fingerprint: project_fingerprint,
- scanner_id: scanner_id,
- primary_identifier_id: vulnerability_identifier.id,
- location_fingerprint: location_fingerprint,
- metadata_version: metadata_version,
- raw_metadata: raw_metadata,
- uuid: uuid
- }
- params[:id] = id unless id.nil?
- vulnerability_findings.create!(params)
- end
- # rubocop:enable Metrics/ParameterLists
-
- def create_user!(name: "Example User", email: "user@example.com", user_type: nil, created_at: Time.zone.now, confirmed_at: Time.zone.now)
- users.create!(
- name: name,
- email: email,
- username: name,
- projects_limit: 0,
- user_type: user_type,
- confirmed_at: confirmed_at
- )
- end
-end
diff --git a/spec/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings_spec.rb b/spec/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings_spec.rb
index 1844347f4a9..60ee61cf50a 100644
--- a/spec/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings_spec.rb
+++ b/spec/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings_spec.rb
@@ -2,7 +2,7 @@
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::RemoveOccurrencePipelinesAndDuplicateVulnerabilitiesFindings, :migration,
- :suppress_gitlab_schemas_validate_connection, schema: 20220326161803 do
+ :suppress_gitlab_schemas_validate_connection, schema: 20220326161803 do
let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') }
let(:users) { table(:users) }
let(:user) { create_user! }
diff --git a/spec/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups_spec.rb b/spec/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups_spec.rb
new file mode 100644
index 00000000000..c45c402ab9d
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups_spec.rb
@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::RemoveProjectGroupLinkWithMissingGroups, :migration,
+ feature_category: :subgroups, schema: 20230206172702 do
+ let(:projects) { table(:projects) }
+ let(:namespaces) { table(:namespaces) }
+ let(:project_group_links) { table(:project_group_links) }
+
+ let!(:group) do
+ namespaces.create!(
+ name: 'Group0', type: 'Group', path: 'space0'
+ )
+ end
+
+ let!(:group_1) do
+ namespaces.create!(
+ name: 'Group1', type: 'Group', path: 'space1'
+ )
+ end
+
+ let!(:group_2) do
+ namespaces.create!(
+ name: 'Group2', type: 'Group', path: 'space2'
+ )
+ end
+
+ let!(:group_3) do
+ namespaces.create!(
+ name: 'Group3', type: 'Group', path: 'space3'
+ )
+ end
+
+ let!(:project_namespace_1) do
+ namespaces.create!(
+ name: 'project_1', path: 'project_1', type: 'Project'
+ )
+ end
+
+ let!(:project_namespace_2) do
+ namespaces.create!(
+ name: 'project_2', path: 'project_2', type: 'Project'
+ )
+ end
+
+ let!(:project_namespace_3) do
+ namespaces.create!(
+ name: 'project_3', path: 'project_3', type: 'Project'
+ )
+ end
+
+ let!(:project_1) do
+ projects.create!(
+ name: 'project_1', path: 'project_1', namespace_id: group.id, project_namespace_id: project_namespace_1.id
+ )
+ end
+
+ let!(:project_2) do
+ projects.create!(
+ name: 'project_2', path: 'project_2', namespace_id: group.id, project_namespace_id: project_namespace_2.id
+ )
+ end
+
+ let!(:project_3) do
+ projects.create!(
+ name: 'project_3', path: 'project_3', namespace_id: group.id, project_namespace_id: project_namespace_3.id
+ )
+ end
+
+ let!(:project_group_link_1) do
+ project_group_links.create!(
+ project_id: project_1.id, group_id: group_1.id, group_access: Gitlab::Access::DEVELOPER
+ )
+ end
+
+ let!(:project_group_link_2) do
+ project_group_links.create!(
+ project_id: project_2.id, group_id: group_2.id, group_access: Gitlab::Access::DEVELOPER
+ )
+ end
+
+ let!(:project_group_link_3) do
+ project_group_links.create!(
+ project_id: project_3.id, group_id: group_3.id, group_access: Gitlab::Access::DEVELOPER
+ )
+ end
+
+ let!(:project_group_link_4) do
+ project_group_links.create!(
+ project_id: project_3.id, group_id: group_2.id, group_access: Gitlab::Access::DEVELOPER
+ )
+ end
+
+ subject do
+ described_class.new(
+ start_id: project_group_link_1.id,
+ end_id: project_group_link_4.id,
+ batch_table: :project_group_links,
+ batch_column: :id,
+ sub_batch_size: 1,
+ pause_ms: 0,
+ connection: ApplicationRecord.connection
+ ).perform
+ end
+
+ it 'removes the `project_group_links` records whose associated group does not exist anymore' do
+ group_2.delete
+
+ # Schema is fixed to `20230206172702` on this spec.
+ # This expectation is needed to make sure that the orphaned records are indeed deleted via the migration
+ # and not via the foreign_key relationship introduced after `20230206172702`, in `20230207002330`
+ expect(project_group_links.count).to eq(4)
+
+ expect { subject }
+ .to change { project_group_links.count }.from(4).to(2)
+ .and change {
+ project_group_links.where(project_id: project_2.id, group_id: group_2.id).present?
+ }.from(true).to(false)
+ .and change {
+ project_group_links.where(project_id: project_3.id, group_id: group_2.id).present?
+ }.from(true).to(false)
+ end
+end
diff --git a/spec/lib/gitlab/background_migration/remove_self_managed_wiki_notes_spec.rb b/spec/lib/gitlab/background_migration/remove_self_managed_wiki_notes_spec.rb
index 81927100562..59d5d56ebe8 100644
--- a/spec/lib/gitlab/background_migration/remove_self_managed_wiki_notes_spec.rb
+++ b/spec/lib/gitlab/background_migration/remove_self_managed_wiki_notes_spec.rb
@@ -6,14 +6,15 @@ RSpec.describe Gitlab::BackgroundMigration::RemoveSelfManagedWikiNotes, :migrati
let(:notes) { table(:notes) }
subject(:perform_migration) do
- described_class.new(start_id: 1,
- end_id: 30,
- batch_table: :notes,
- batch_column: :id,
- sub_batch_size: 2,
- pause_ms: 0,
- connection: ActiveRecord::Base.connection)
- .perform
+ described_class.new(
+ start_id: 1,
+ end_id: 30,
+ batch_table: :notes,
+ batch_column: :id,
+ sub_batch_size: 2,
+ pause_ms: 0,
+ connection: ActiveRecord::Base.connection
+ ).perform
end
it 'removes all wiki notes' do
diff --git a/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb b/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb
index 918df8f4442..32134b99e37 100644
--- a/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb
+++ b/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb
@@ -2,7 +2,7 @@
require 'spec_helper'
-RSpec.describe Gitlab::BackgroundMigration::RemoveVulnerabilityFindingLinks, :migration, schema: 20211104165220 do
+RSpec.describe Gitlab::BackgroundMigration::RemoveVulnerabilityFindingLinks, :migration, schema: 20211202041233 do
let(:vulnerability_findings) { table(:vulnerability_occurrences) }
let(:finding_links) { table(:vulnerability_finding_links) }
diff --git a/spec/lib/gitlab/background_migration/reset_too_many_tags_skipped_registry_imports_spec.rb b/spec/lib/gitlab/background_migration/reset_too_many_tags_skipped_registry_imports_spec.rb
index 3f59b0a24a3..afdd855c5a8 100644
--- a/spec/lib/gitlab/background_migration/reset_too_many_tags_skipped_registry_imports_spec.rb
+++ b/spec/lib/gitlab/background_migration/reset_too_many_tags_skipped_registry_imports_spec.rb
@@ -3,8 +3,8 @@
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::ResetTooManyTagsSkippedRegistryImports, :migration,
- :aggregate_failures,
- schema: 20220502173045 do
+ :aggregate_failures,
+ schema: 20220502173045 do
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:container_repositories) { table(:container_repositories) }
@@ -15,46 +15,54 @@ RSpec.describe Gitlab::BackgroundMigration::ResetTooManyTagsSkippedRegistryImpor
let!(:project) { projects.create!(id: 1, project_namespace_id: 1, namespace_id: 1, path: 'bar', name: 'bar') }
let!(:container_repository1) do
- container_repositories.create!(id: 1,
- project_id: 1,
- name: 'a',
- migration_state: 'import_skipped',
- migration_skipped_at: Time.zone.now,
- migration_skipped_reason: 2,
- migration_pre_import_started_at: Time.zone.now,
- migration_pre_import_done_at: Time.zone.now,
- migration_import_started_at: Time.zone.now,
- migration_import_done_at: Time.zone.now,
- migration_aborted_at: Time.zone.now,
- migration_retries_count: 2,
- migration_aborted_in_state: 'importing')
+ container_repositories.create!(
+ id: 1,
+ project_id: 1,
+ name: 'a',
+ migration_state: 'import_skipped',
+ migration_skipped_at: Time.zone.now,
+ migration_skipped_reason: 2,
+ migration_pre_import_started_at: Time.zone.now,
+ migration_pre_import_done_at: Time.zone.now,
+ migration_import_started_at: Time.zone.now,
+ migration_import_done_at: Time.zone.now,
+ migration_aborted_at: Time.zone.now,
+ migration_retries_count: 2,
+ migration_aborted_in_state: 'importing'
+ )
end
let!(:container_repository2) do
- container_repositories.create!(id: 2,
- project_id: 1,
- name: 'b',
- migration_state: 'import_skipped',
- migration_skipped_at: Time.zone.now,
- migration_skipped_reason: 2)
+ container_repositories.create!(
+ id: 2,
+ project_id: 1,
+ name: 'b',
+ migration_state: 'import_skipped',
+ migration_skipped_at: Time.zone.now,
+ migration_skipped_reason: 2
+ )
end
let!(:container_repository3) do
- container_repositories.create!(id: 3,
- project_id: 1,
- name: 'c',
- migration_state: 'import_skipped',
- migration_skipped_at: Time.zone.now,
- migration_skipped_reason: 1)
+ container_repositories.create!(
+ id: 3,
+ project_id: 1,
+ name: 'c',
+ migration_state: 'import_skipped',
+ migration_skipped_at: Time.zone.now,
+ migration_skipped_reason: 1
+ )
end
# This is an unlikely state, but included here to test the edge case
let!(:container_repository4) do
- container_repositories.create!(id: 4,
- project_id: 1,
- name: 'd',
- migration_state: 'default',
- migration_skipped_reason: 2)
+ container_repositories.create!(
+ id: 4,
+ project_id: 1,
+ name: 'd',
+ migration_state: 'default',
+ migration_skipped_reason: 2
+ )
end
describe '#up' do
diff --git a/spec/lib/gitlab/background_migration/set_correct_vulnerability_state_spec.rb b/spec/lib/gitlab/background_migration/set_correct_vulnerability_state_spec.rb
index 2372ce21c4c..df1ee494987 100644
--- a/spec/lib/gitlab/background_migration/set_correct_vulnerability_state_spec.rb
+++ b/spec/lib/gitlab/background_migration/set_correct_vulnerability_state_spec.rb
@@ -35,13 +35,15 @@ RSpec.describe Gitlab::BackgroundMigration::SetCorrectVulnerabilityState do
let(:dismissed_state) { 2 }
let(:migration_job) do
- described_class.new(start_id: vulnerability_with_dismissed_at.id,
- end_id: vulnerability_without_dismissed_at.id,
- batch_table: :vulnerabilities,
- batch_column: :id,
- sub_batch_size: 1,
- pause_ms: 0,
- connection: ActiveRecord::Base.connection)
+ described_class.new(
+ start_id: vulnerability_with_dismissed_at.id,
+ end_id: vulnerability_without_dismissed_at.id,
+ batch_table: :vulnerabilities,
+ batch_column: :id,
+ sub_batch_size: 1,
+ pause_ms: 0,
+ connection: ActiveRecord::Base.connection
+ )
end
describe '#filter_batch' do
diff --git a/spec/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects_spec.rb b/spec/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects_spec.rb
index e9f73672144..5109c3ec0c2 100644
--- a/spec/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects_spec.rb
+++ b/spec/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects_spec.rb
@@ -3,21 +3,22 @@
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::SetLegacyOpenSourceLicenseAvailableForNonPublicProjects,
- :migration,
- schema: 20220722110026 do
+ :migration,
+ schema: 20220722110026 do
let(:namespaces_table) { table(:namespaces) }
let(:projects_table) { table(:projects) }
let(:project_settings_table) { table(:project_settings) }
subject(:perform_migration) do
- described_class.new(start_id: projects_table.minimum(:id),
- end_id: projects_table.maximum(:id),
- batch_table: :projects,
- batch_column: :id,
- sub_batch_size: 2,
- pause_ms: 0,
- connection: ActiveRecord::Base.connection)
- .perform
+ described_class.new(
+ start_id: projects_table.minimum(:id),
+ end_id: projects_table.maximum(:id),
+ batch_table: :projects,
+ batch_column: :id,
+ sub_batch_size: 2,
+ pause_ms: 0,
+ connection: ActiveRecord::Base.connection
+ ).perform
end
it 'sets `legacy_open_source_license_available` attribute to false for non-public projects', :aggregate_failures do
@@ -37,11 +38,13 @@ RSpec.describe Gitlab::BackgroundMigration::SetLegacyOpenSourceLicenseAvailableF
def create_legacy_license_project(path, visibility_level:)
namespace = namespaces_table.create!(name: "namespace-#{path}", path: "namespace-#{path}")
project_namespace = namespaces_table.create!(name: "project-namespace-#{path}", path: path, type: 'Project')
- project = projects_table.create!(name: path,
- path: path,
- namespace_id: namespace.id,
- project_namespace_id: project_namespace.id,
- visibility_level: visibility_level)
+ project = projects_table.create!(
+ name: path,
+ path: path,
+ namespace_id: namespace.id,
+ project_namespace_id: project_namespace.id,
+ visibility_level: visibility_level
+ )
project_settings_table.create!(project_id: project.id, legacy_open_source_license_available: true)
project
diff --git a/spec/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users_spec.rb b/spec/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users_spec.rb
deleted file mode 100644
index 841a7f306d7..00000000000
--- a/spec/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users_spec.rb
+++ /dev/null
@@ -1,50 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-RSpec.describe Gitlab::BackgroundMigration::StealMigrateMergeRequestDiffCommitUsers, schema: 20211012134316 do
- let(:migration) { described_class.new }
-
- describe '#perform' do
- it 'processes the background migration' do
- spy = instance_spy(
- Gitlab::BackgroundMigration::MigrateMergeRequestDiffCommitUsers
- )
-
- allow(Gitlab::BackgroundMigration::MigrateMergeRequestDiffCommitUsers)
- .to receive(:new)
- .and_return(spy)
-
- expect(spy).to receive(:perform).with(1, 4)
- expect(migration).to receive(:schedule_next_job)
-
- migration.perform(1, 4)
- end
- end
-
- describe '#schedule_next_job' do
- it 'schedules the next job in ascending order' do
- Gitlab::Database::BackgroundMigrationJob.create!(
- class_name: 'MigrateMergeRequestDiffCommitUsers',
- arguments: [10, 20]
- )
-
- Gitlab::Database::BackgroundMigrationJob.create!(
- class_name: 'MigrateMergeRequestDiffCommitUsers',
- arguments: [40, 50]
- )
-
- expect(BackgroundMigrationWorker)
- .to receive(:perform_in)
- .with(5.minutes, 'StealMigrateMergeRequestDiffCommitUsers', [10, 20])
-
- migration.schedule_next_job
- end
-
- it 'does not schedule any new jobs when there are none' do
- expect(BackgroundMigrationWorker).not_to receive(:perform_in)
-
- migration.schedule_next_job
- end
- end
-end
diff --git a/spec/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces_spec.rb b/spec/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces_spec.rb
index 980a7771f4c..0579a299c74 100644
--- a/spec/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces_spec.rb
+++ b/spec/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces_spec.rb
@@ -3,7 +3,7 @@
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::UpdateDelayedProjectRemovalToNullForUserNamespaces,
- :migration do
+ :migration do
let(:namespaces_table) { table(:namespaces) }
let(:namespace_settings_table) { table(:namespace_settings) }
diff --git a/spec/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url_spec.rb b/spec/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url_spec.rb
index c090c1df424..75fe5699986 100644
--- a/spec/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url_spec.rb
+++ b/spec/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url_spec.rb
@@ -13,10 +13,12 @@ RSpec.describe Gitlab::BackgroundMigration::UpdateJiraTrackerDataDeploymentTypeB
let(:sub_batch_size) { 1 }
let(:pause_ms) { 0 }
let(:migration) do
- described_class.new(start_id: 1, end_id: 10,
- batch_table: table_name, batch_column: batch_column,
- sub_batch_size: sub_batch_size, pause_ms: pause_ms,
- connection: ApplicationRecord.connection)
+ described_class.new(
+ start_id: 1, end_id: 10,
+ batch_table: table_name, batch_column: batch_column,
+ sub_batch_size: sub_batch_size, pause_ms: pause_ms,
+ connection: ApplicationRecord.connection
+ )
end
subject(:perform_migration) do
diff --git a/spec/lib/gitlab/background_migration/update_timelogs_project_id_spec.rb b/spec/lib/gitlab/background_migration/update_timelogs_project_id_spec.rb
deleted file mode 100644
index b8c3bf8f3ac..00000000000
--- a/spec/lib/gitlab/background_migration/update_timelogs_project_id_spec.rb
+++ /dev/null
@@ -1,52 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-RSpec.describe Gitlab::BackgroundMigration::UpdateTimelogsProjectId, schema: 20210826171758 do
- let!(:namespace) { table(:namespaces).create!(name: 'namespace', path: 'namespace') }
- let!(:project1) { table(:projects).create!(namespace_id: namespace.id) }
- let!(:project2) { table(:projects).create!(namespace_id: namespace.id) }
- let!(:issue1) { table(:issues).create!(project_id: project1.id) }
- let!(:issue2) { table(:issues).create!(project_id: project2.id) }
- let!(:merge_request1) { table(:merge_requests).create!(target_project_id: project1.id, source_branch: 'master', target_branch: 'feature') }
- let!(:merge_request2) { table(:merge_requests).create!(target_project_id: project2.id, source_branch: 'master', target_branch: 'feature') }
- let!(:timelog1) { table(:timelogs).create!(issue_id: issue1.id, time_spent: 60) }
- let!(:timelog2) { table(:timelogs).create!(issue_id: issue1.id, time_spent: 60) }
- let!(:timelog3) { table(:timelogs).create!(issue_id: issue2.id, time_spent: 60) }
- let!(:timelog4) { table(:timelogs).create!(merge_request_id: merge_request1.id, time_spent: 600) }
- let!(:timelog5) { table(:timelogs).create!(merge_request_id: merge_request1.id, time_spent: 600) }
- let!(:timelog6) { table(:timelogs).create!(merge_request_id: merge_request2.id, time_spent: 600) }
- let!(:timelog7) { table(:timelogs).create!(issue_id: issue2.id, time_spent: 60, project_id: project1.id) }
- let!(:timelog8) { table(:timelogs).create!(merge_request_id: merge_request2.id, time_spent: 600, project_id: project1.id) }
-
- describe '#perform' do
- context 'when timelogs belong to issues' do
- it 'sets correct project_id' do
- subject.perform(timelog1.id, timelog3.id)
-
- expect(timelog1.reload.project_id).to eq(issue1.project_id)
- expect(timelog2.reload.project_id).to eq(issue1.project_id)
- expect(timelog3.reload.project_id).to eq(issue2.project_id)
- end
- end
-
- context 'when timelogs belong to merge requests' do
- it 'sets correct project ids' do
- subject.perform(timelog4.id, timelog6.id)
-
- expect(timelog4.reload.project_id).to eq(merge_request1.target_project_id)
- expect(timelog5.reload.project_id).to eq(merge_request1.target_project_id)
- expect(timelog6.reload.project_id).to eq(merge_request2.target_project_id)
- end
- end
-
- context 'when timelogs already belong to projects' do
- it 'does not update the project id' do
- subject.perform(timelog7.id, timelog8.id)
-
- expect(timelog7.reload.project_id).to eq(project1.id)
- expect(timelog8.reload.project_id).to eq(project1.id)
- end
- end
- end
-end
diff --git a/spec/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group_spec.rb b/spec/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group_spec.rb
deleted file mode 100644
index f16ae489b78..00000000000
--- a/spec/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group_spec.rb
+++ /dev/null
@@ -1,84 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-RSpec.describe Gitlab::BackgroundMigration::UpdateUsersWhereTwoFactorAuthRequiredFromGroup, :migration, schema: 20210826171758 do
- include MigrationHelpers::NamespacesHelpers
-
- let(:group_with_2fa_parent) { create_namespace('parent', Gitlab::VisibilityLevel::PRIVATE, require_two_factor_authentication: true) }
- let(:group_with_2fa_child) { create_namespace('child', Gitlab::VisibilityLevel::PRIVATE, parent_id: group_with_2fa_parent.id) }
- let(:members_table) { table(:members) }
- let(:users_table) { table(:users) }
-
- subject { described_class.new }
-
- describe '#perform' do
- context 'with group members' do
- let(:user_1) { create_user('user@example.com') }
- let!(:member) { create_group_member(user_1, group_with_2fa_parent) }
- let!(:user_without_group) { create_user('user_without@example.com') }
- let(:user_other) { create_user('user_other@example.com') }
- let!(:member_other) { create_group_member(user_other, group_with_2fa_parent) }
-
- it 'updates user when user should be required to establish two factor authentication' do
- subject.perform(user_1.id, user_without_group.id)
-
- expect(user_1.reload.require_two_factor_authentication_from_group).to eq(true)
- end
-
- it 'does not update user who is not in current batch' do
- subject.perform(user_1.id, user_without_group.id)
-
- expect(user_other.reload.require_two_factor_authentication_from_group).to eq(false)
- end
-
- it 'updates all users in current batch' do
- subject.perform(user_1.id, user_other.id)
-
- expect(user_other.reload.require_two_factor_authentication_from_group).to eq(true)
- end
-
- it 'updates user when user is member of group in which parent group requires two factor authentication' do
- member.destroy!
-
- subgroup = create_namespace('subgroup', Gitlab::VisibilityLevel::PRIVATE, require_two_factor_authentication: false, parent_id: group_with_2fa_child.id)
- create_group_member(user_1, subgroup)
-
- subject.perform(user_1.id, user_other.id)
-
- expect(user_1.reload.require_two_factor_authentication_from_group).to eq(true)
- end
-
- it 'updates user when user is member of a group and the subgroup requires two factor authentication' do
- member.destroy!
-
- parent = create_namespace('other_parent', Gitlab::VisibilityLevel::PRIVATE, require_two_factor_authentication: false)
- create_namespace('other_subgroup', Gitlab::VisibilityLevel::PRIVATE, require_two_factor_authentication: true, parent_id: parent.id)
- create_group_member(user_1, parent)
-
- subject.perform(user_1.id, user_other.id)
-
- expect(user_1.reload.require_two_factor_authentication_from_group).to eq(true)
- end
-
- it 'does not update user when not a member of a group that requires two factor authentication' do
- member_other.destroy!
-
- other_group = create_namespace('other_group', Gitlab::VisibilityLevel::PRIVATE, require_two_factor_authentication: false)
- create_group_member(user_other, other_group)
-
- subject.perform(user_1.id, user_other.id)
-
- expect(user_other.reload.require_two_factor_authentication_from_group).to eq(false)
- end
- end
- end
-
- def create_user(email, require_2fa: false)
- users_table.create!(email: email, projects_limit: 10, require_two_factor_authentication_from_group: require_2fa)
- end
-
- def create_group_member(user, group)
- members_table.create!(user_id: user.id, source_id: group.id, access_level: GroupMember::MAINTAINER, source_type: "Namespace", type: "GroupMember", notification_level: 3)
- end
-end