From 7021455bd1ed7b125c55eb1b33c5a01f2bc55ee0 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Thu, 17 Nov 2022 11:33:21 +0000 Subject: Add latest changes from gitlab-org/gitlab@15-6-stable-ee --- .../backfill_project_namespace_details_spec.rb | 62 ++++++ .../backfill_project_namespace_on_issues_spec.rb | 17 ++ .../backfill_projects_with_coverage_spec.rb | 95 --------- .../backfill_user_details_fields_spec.rb | 222 +++++++++++++++++++++ .../batched_migration_job_spec.rb | 77 ++++++- .../legacy_upload_mover_spec.rb | 2 +- .../populate_projects_star_count_spec.rb | 72 +++++++ .../populate_vulnerability_reads_spec.rb | 2 +- ...move_backfilled_job_artifacts_expire_at_spec.rb | 5 +- ...icate_ci_runners_token_encrypted_values_spec.rb | 70 ------- ...reset_duplicate_ci_runners_token_values_spec.rb | 70 ------- .../sanitize_confidential_todos_spec.rb | 89 +++++++++ ...ipeline_artifacts_unknown_locked_status_spec.rb | 4 +- 13 files changed, 542 insertions(+), 245 deletions(-) create mode 100644 spec/lib/gitlab/background_migration/backfill_project_namespace_details_spec.rb delete mode 100644 spec/lib/gitlab/background_migration/backfill_projects_with_coverage_spec.rb create mode 100644 spec/lib/gitlab/background_migration/backfill_user_details_fields_spec.rb create mode 100644 spec/lib/gitlab/background_migration/populate_projects_star_count_spec.rb delete mode 100644 spec/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_spec.rb delete mode 100644 spec/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_spec.rb create mode 100644 spec/lib/gitlab/background_migration/sanitize_confidential_todos_spec.rb (limited to 'spec/lib/gitlab/background_migration') diff --git a/spec/lib/gitlab/background_migration/backfill_project_namespace_details_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_namespace_details_spec.rb new file mode 100644 index 00000000000..77d6cc43114 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_project_namespace_details_spec.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillProjectNamespaceDetails, :migration do + let_it_be(:namespace_details) { table(:namespace_details) } + let_it_be(:namespaces) { table(:namespaces) } + let_it_be(:projects) { table(:projects) } + + subject(:perform_migration) do + described_class.new(start_id: projects.minimum(:id), + end_id: projects.maximum(:id), + batch_table: :projects, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection) + .perform + end + + describe '#perform' do + it 'creates details for all project namespaces in range' do + namespaces.create!(id: 5, name: 'test1', path: 'test1', description: "Some description1", + description_html: "Some description html1", cached_markdown_version: 4) + project_namespace1 = namespaces.create!(id: 6, name: 'test2', path: 'test2', type: 'Project') + namespaces.create!(id: 7, name: 'test3', path: 'test3', description: "Some description3", + description_html: "Some description html3", cached_markdown_version: 4) + project_namespace2 = namespaces.create!(id: 8, name: 'test4', path: 'test4', type: 'Project') + + project1 = projects.create!(namespace_id: project_namespace1.id, name: 'gitlab1', path: 'gitlab1', + project_namespace_id: project_namespace1.id, description: "Some description2", + description_html: "Some description html2", cached_markdown_version: 4) + project2 = projects.create!(namespace_id: project_namespace2.id, name: 'gitlab2', path: 'gitlab2', + project_namespace_id: project_namespace2.id, + description: "Some description3", + description_html: "Some description html4", cached_markdown_version: 4) + + namespace_details.delete_all + + expect(namespace_details.pluck(:namespace_id)).to eql [] + + expect { perform_migration } + .to change { namespace_details.pluck(:namespace_id) }.from([]).to contain_exactly( + project_namespace1.id, + project_namespace2.id + ) + + expect(namespace_details.find_by_namespace_id(project_namespace1.id)) + .to have_attributes(migrated_attributes(project1)) + expect(namespace_details.find_by_namespace_id(project_namespace2.id)) + .to have_attributes(migrated_attributes(project2)) + end + end + + def migrated_attributes(project) + { + description: project.description, + description_html: project.description_html, + cached_markdown_version: project.cached_markdown_version + } + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_project_namespace_on_issues_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_namespace_on_issues_spec.rb index 29833074109..3ca7d28f09d 100644 --- a/spec/lib/gitlab/background_migration/backfill_project_namespace_on_issues_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_project_namespace_on_issues_spec.rb @@ -54,4 +54,21 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillProjectNamespaceOnIssues do expect { perform_migration }.to change { migration.batch_metrics.timings } end + + context 'when database timeouts' do + using RSpec::Parameterized::TableSyntax + + where(error_class: [ActiveRecord::StatementTimeout, ActiveRecord::QueryCanceled]) + + with_them do + it 'retries on timeout error' do + expect(migration).to receive(:update_batch).exactly(3).times.and_raise(error_class) + expect(migration).to receive(:sleep).with(5).twice + + expect do + perform_migration + end.to raise_error(error_class) + end + end + end end diff --git a/spec/lib/gitlab/background_migration/backfill_projects_with_coverage_spec.rb b/spec/lib/gitlab/background_migration/backfill_projects_with_coverage_spec.rb deleted file mode 100644 index 4a65ecf8c75..00000000000 --- a/spec/lib/gitlab/background_migration/backfill_projects_with_coverage_spec.rb +++ /dev/null @@ -1,95 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillProjectsWithCoverage, - :suppress_gitlab_schemas_validate_connection, schema: 20210818185845 do - let(:projects) { table(:projects) } - let(:project_ci_feature_usages) { table(:project_ci_feature_usages) } - let(:ci_pipelines) { table(:ci_pipelines) } - let(:ci_daily_build_group_report_results) { table(:ci_daily_build_group_report_results) } - let(:group) { table(:namespaces).create!(name: 'user', path: 'user') } - let(:project_1) { projects.create!(namespace_id: group.id) } - let(:project_2) { projects.create!(namespace_id: group.id) } - let(:pipeline_1) { ci_pipelines.create!(project_id: project_1.id, source: 13) } - let(:pipeline_2) { ci_pipelines.create!(project_id: project_1.id, source: 13) } - let(:pipeline_3) { ci_pipelines.create!(project_id: project_2.id, source: 13) } - let(:pipeline_4) { ci_pipelines.create!(project_id: project_2.id, source: 13) } - - subject { described_class.new } - - describe '#perform' do - before do - ci_daily_build_group_report_results.create!( - id: 1, - project_id: project_1.id, - date: 4.days.ago, - last_pipeline_id: pipeline_1.id, - ref_path: 'main', - group_name: 'rspec', - data: { coverage: 95.0 }, - default_branch: true, - group_id: group.id - ) - - ci_daily_build_group_report_results.create!( - id: 2, - project_id: project_1.id, - date: 3.days.ago, - last_pipeline_id: pipeline_2.id, - ref_path: 'main', - group_name: 'rspec', - data: { coverage: 95.0 }, - default_branch: true, - group_id: group.id - ) - - ci_daily_build_group_report_results.create!( - id: 3, - project_id: project_2.id, - date: 2.days.ago, - last_pipeline_id: pipeline_3.id, - ref_path: 'main', - group_name: 'rspec', - data: { coverage: 95.0 }, - default_branch: true, - group_id: group.id - ) - - ci_daily_build_group_report_results.create!( - id: 4, - project_id: project_2.id, - date: 1.day.ago, - last_pipeline_id: pipeline_4.id, - ref_path: 'test_branch', - group_name: 'rspec', - data: { coverage: 95.0 }, - default_branch: false, - group_id: group.id - ) - - stub_const("#{described_class}::INSERT_DELAY_SECONDS", 0) - end - - it 'creates entries per project and default_branch combination in the given range', :aggregate_failures do - subject.perform(1, 4, 2) - - entries = project_ci_feature_usages.order('project_id ASC, default_branch DESC') - - expect(entries.count).to eq(3) - expect(entries[0]).to have_attributes(project_id: project_1.id, feature: 1, default_branch: true) - expect(entries[1]).to have_attributes(project_id: project_2.id, feature: 1, default_branch: true) - expect(entries[2]).to have_attributes(project_id: project_2.id, feature: 1, default_branch: false) - end - - context 'when an entry for the project and default branch combination already exists' do - before do - subject.perform(1, 4, 2) - end - - it 'does not create a new entry' do - expect { subject.perform(1, 4, 2) }.not_to change { project_ci_feature_usages.count } - end - end - end -end diff --git a/spec/lib/gitlab/background_migration/backfill_user_details_fields_spec.rb b/spec/lib/gitlab/background_migration/backfill_user_details_fields_spec.rb new file mode 100644 index 00000000000..04ada1703bc --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_user_details_fields_spec.rb @@ -0,0 +1,222 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillUserDetailsFields, :migration, schema: 20221018232820 do + let(:users) { table(:users) } + let(:user_details) { table(:user_details) } + + let!(:user_all_fields_backfill) do + users.create!( + name: generate(:name), + email: generate(:email), + projects_limit: 1, + linkedin: 'linked-in', + twitter: '@twitter', + skype: 'skype', + website_url: 'https://example.com', + location: 'Antarctica', + organization: 'Gitlab' + ) + end + + let!(:user_long_details_fields) do + length = UserDetail::DEFAULT_FIELD_LENGTH + 1 + users.create!( + name: generate(:name), + email: generate(:email), + projects_limit: 1, + linkedin: 'l' * length, + twitter: 't' * length, + skype: 's' * length, + website_url: "https://#{'a' * (length - 12)}.com", + location: 'l' * length, + organization: 'o' * length + ) + end + + let!(:user_nil_details_fields) do + users.create!( + name: generate(:name), + email: generate(:email), + projects_limit: 1 + ) + end + + let!(:user_empty_details_fields) do + users.create!( + name: generate(:name), + email: generate(:email), + projects_limit: 1, + linkedin: '', + twitter: '', + skype: '', + website_url: '', + location: '', + organization: '' + ) + end + + let!(:user_with_bio) do + users.create!( + name: generate(:name), + email: generate(:email), + projects_limit: 1, + linkedin: 'linked-in', + twitter: '@twitter', + skype: 'skype', + website_url: 'https://example.com', + location: 'Antarctica', + organization: 'Gitlab' + ) + end + + let!(:bio_user_details) do + user_details + .find_or_create_by!(user_id: user_with_bio.id) + .update!(bio: 'bio') + end + + let!(:user_with_details) do + users.create!( + name: generate(:name), + email: generate(:email), + projects_limit: 1, + linkedin: 'linked-in', + twitter: '@twitter', + skype: 'skype', + website_url: 'https://example.com', + location: 'Antarctica', + organization: 'Gitlab' + ) + end + + let!(:existing_user_details) do + user_details + .find_or_create_by!(user_id: user_with_details.id) + .update!( + linkedin: 'linked-in', + twitter: '@twitter', + skype: 'skype', + website_url: 'https://example.com', + location: 'Antarctica', + organization: 'Gitlab' + ) + end + + let!(:user_different_details) do + users.create!( + name: generate(:name), + email: generate(:email), + projects_limit: 1, + linkedin: 'linked-in', + twitter: '@twitter', + skype: 'skype', + website_url: 'https://example.com', + location: 'Antarctica', + organization: 'Gitlab' + ) + end + + let!(:differing_details) do + user_details + .find_or_create_by!(user_id: user_different_details.id) + .update!( + linkedin: 'details-in', + twitter: '@details', + skype: 'details_skype', + website_url: 'https://details.site', + location: 'Details Location', + organization: 'Details Organization' + ) + end + + let(:user_ids) do + [ + user_all_fields_backfill, + user_long_details_fields, + user_nil_details_fields, + user_empty_details_fields, + user_with_bio, + user_with_details, + user_different_details + ].map(&:id) + end + + subject do + described_class.new( + start_id: user_ids.min, + end_id: user_ids.max, + batch_table: 'users', + batch_column: 'id', + sub_batch_size: 1_000, + pause_ms: 0, + connection: ApplicationRecord.connection + ) + end + + it 'processes all relevant records' do + expect { subject.perform }.to change { user_details.all.size }.to(5) + end + + it 'backfills new user_details fields' do + subject.perform + + user_detail = user_details.find_by!(user_id: user_all_fields_backfill.id) + expect(user_detail.linkedin).to eq('linked-in') + expect(user_detail.twitter).to eq('@twitter') + expect(user_detail.skype).to eq('skype') + expect(user_detail.website_url).to eq('https://example.com') + expect(user_detail.location).to eq('Antarctica') + expect(user_detail.organization).to eq('Gitlab') + end + + it 'does not migrate nil fields' do + subject.perform + + expect(user_details.find_by(user_id: user_nil_details_fields)).to be_nil + end + + it 'does not migrate empty fields' do + subject.perform + + expect(user_details.find_by(user_id: user_empty_details_fields)).to be_nil + end + + it 'backfills new fields without overwriting existing `bio` field' do + subject.perform + + user_detail = user_details.find_by!(user_id: user_with_bio.id) + expect(user_detail.bio).to eq('bio') + expect(user_detail.linkedin).to eq('linked-in') + expect(user_detail.twitter).to eq('@twitter') + expect(user_detail.skype).to eq('skype') + expect(user_detail.website_url).to eq('https://example.com') + expect(user_detail.location).to eq('Antarctica') + expect(user_detail.organization).to eq('Gitlab') + end + + context 'when user details are unchanged' do + it 'does not change existing details' do + expect { subject.perform }.not_to change { + user_details.find_by!(user_id: user_with_details.id).attributes + } + end + end + + context 'when user details are changed' do + it 'updates existing user details' do + expect { subject.perform }.to change { + user_details.find_by!(user_id: user_different_details.id).attributes + } + + user_detail = user_details.find_by!(user_id: user_different_details.id) + expect(user_detail.linkedin).to eq('linked-in') + expect(user_detail.twitter).to eq('@twitter') + expect(user_detail.skype).to eq('skype') + expect(user_detail.website_url).to eq('https://example.com') + expect(user_detail.location).to eq('Antarctica') + expect(user_detail.organization).to eq('Gitlab') + end + end +end diff --git a/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb b/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb index f03f90ddbbb..95be14cefb1 100644 --- a/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb +++ b/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb @@ -57,6 +57,71 @@ RSpec.describe Gitlab::BackgroundMigration::BatchedMigrationJob do end end + describe '.operation_name' do + subject(:perform_job) { job_instance.perform } + + let(:job_instance) do + job_class.new(start_id: 1, end_id: 10, + batch_table: '_test_table', + batch_column: 'id', + sub_batch_size: 2, + pause_ms: 1000, + job_arguments: %w(a b), + connection: connection) + end + + let(:job_class) do + Class.new(described_class) do + operation_name :update_all + end + end + + it 'defines method' do + expect(job_instance.operation_name).to eq(:update_all) + end + + context 'when `operation_name` is not defined' do + let(:job_class) do + Class.new(described_class) do + def perform + each_sub_batch do |sub_batch| + sub_batch.update_all('to_column = from_column') + end + end + end + end + + let(:test_table) { table(:_test_table) } + let(:test_insert_table) { table(:_test_insert_table) } + + before do + allow(job_instance).to receive(:sleep) + + connection.create_table :_test_table do |t| + t.timestamps_with_timezone null: false + t.integer :from_column, null: false + end + + connection.create_table :_test_insert_table, id: false do |t| + t.integer :to_column + t.index :to_column, unique: true + end + + test_table.create!(id: 1, from_column: 5) + test_table.create!(id: 2, from_column: 10) + end + + after do + connection.drop_table(:_test_table) + connection.drop_table(:_test_insert_table) + end + + it 'raises an exception' do + expect { perform_job }.to raise_error(RuntimeError, /Operation name is required/) + end + end + end + describe '.scope_to' do subject(:job_instance) do job_class.new(start_id: 1, end_id: 10, @@ -133,9 +198,10 @@ RSpec.describe Gitlab::BackgroundMigration::BatchedMigrationJob do context 'when the subclass uses sub-batching' do let(:job_class) do Class.new(described_class) do + operation_name :update + def perform(*job_arguments) each_sub_batch( - operation_name: :update, batching_arguments: { order_hint: :updated_at }, batching_scope: -> (relation) { relation.where.not(bar: nil) } ) do |sub_batch| @@ -177,10 +243,10 @@ RSpec.describe Gitlab::BackgroundMigration::BatchedMigrationJob do let(:job_class) do Class.new(described_class) do scope_to ->(r) { r.where('mod(id, 2) = 0') } + operation_name :update def perform(*job_arguments) each_sub_batch( - operation_name: :update, batching_arguments: { order_hint: :updated_at }, batching_scope: -> (relation) { relation.where.not(bar: nil) } ) do |sub_batch| @@ -237,8 +303,10 @@ RSpec.describe Gitlab::BackgroundMigration::BatchedMigrationJob do let(:job_class) do Class.new(described_class) do + operation_name :insert + def perform(*job_arguments) - distinct_each_batch(operation_name: :insert) do |sub_batch| + distinct_each_batch do |sub_batch| sub_batch.pluck(:from_column).each do |value| connection.execute("INSERT INTO _test_insert_table VALUES (#{value})") end @@ -291,9 +359,10 @@ RSpec.describe Gitlab::BackgroundMigration::BatchedMigrationJob do let(:job_class) do Class.new(described_class) do scope_to ->(r) { r.where.not(from_column: 10) } + operation_name :insert def perform(*job_arguments) - distinct_each_batch(operation_name: :insert) do |sub_batch| + distinct_each_batch do |sub_batch| end end end diff --git a/spec/lib/gitlab/background_migration/legacy_upload_mover_spec.rb b/spec/lib/gitlab/background_migration/legacy_upload_mover_spec.rb index 264faa4de3b..c522c8b307f 100644 --- a/spec/lib/gitlab/background_migration/legacy_upload_mover_spec.rb +++ b/spec/lib/gitlab/background_migration/legacy_upload_mover_spec.rb @@ -241,7 +241,7 @@ RSpec.describe Gitlab::BackgroundMigration::LegacyUploadMover, :aggregate_failur context 'when legacy uploads are stored in object storage' do let(:legacy_upload) { create_remote_upload(note, filename) } let(:remote_file) do - { key: "#{legacy_upload.path}" } + { key: legacy_upload.path.to_s } end let(:connection) { ::Fog::Storage.new(FileUploader.object_store_credentials) } diff --git a/spec/lib/gitlab/background_migration/populate_projects_star_count_spec.rb b/spec/lib/gitlab/background_migration/populate_projects_star_count_spec.rb new file mode 100644 index 00000000000..74f674e052d --- /dev/null +++ b/spec/lib/gitlab/background_migration/populate_projects_star_count_spec.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::PopulateProjectsStarCount, schema: 20221019105041 do + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:users) { table(:users) } + let(:users_star_projects) { table(:users_star_projects) } + + let(:namespace1) { namespaces.create!(name: 'namespace 1', path: 'namespace1') } + let(:namespace2) { namespaces.create!(name: 'namespace 2', path: 'namespace2') } + let(:namespace3) { namespaces.create!(name: 'namespace 3', path: 'namespace3') } + let(:namespace4) { namespaces.create!(name: 'namespace 4', path: 'namespace4') } + let(:namespace5) { namespaces.create!(name: 'namespace 5', path: 'namespace5') } + + let(:project1) { projects.create!(namespace_id: namespace1.id, project_namespace_id: namespace1.id) } + let(:project2) { projects.create!(namespace_id: namespace2.id, project_namespace_id: namespace2.id) } + let(:project3) { projects.create!(namespace_id: namespace3.id, project_namespace_id: namespace3.id) } + let(:project4) { projects.create!(namespace_id: namespace4.id, project_namespace_id: namespace4.id) } + let(:project5) { projects.create!(namespace_id: namespace5.id, project_namespace_id: namespace5.id) } + + let(:user_active) { users.create!(state: 'active', email: 'test1@example.com', projects_limit: 5) } + let(:user_blocked) { users.create!(state: 'blocked', email: 'test2@example.com', projects_limit: 5) } + + let(:migration) do + described_class.new( + start_id: project1.id, + end_id: project4.id, + batch_table: :projects, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 2, + connection: ApplicationRecord.connection + ) + end + + subject(:perform_migration) { migration.perform } + + it 'correctly populates the star counters' do + users_star_projects.create!(project_id: project1.id, user_id: user_active.id) + users_star_projects.create!(project_id: project2.id, user_id: user_blocked.id) + users_star_projects.create!(project_id: project4.id, user_id: user_active.id) + users_star_projects.create!(project_id: project4.id, user_id: user_blocked.id) + users_star_projects.create!(project_id: project5.id, user_id: user_active.id) + + perform_migration + + expect(project1.reload.star_count).to eq(1) + expect(project2.reload.star_count).to eq(0) + expect(project3.reload.star_count).to eq(0) + expect(project4.reload.star_count).to eq(1) + expect(project5.reload.star_count).to eq(0) + end + + context 'when database timeouts' do + using RSpec::Parameterized::TableSyntax + + where(error_class: [ActiveRecord::StatementTimeout, ActiveRecord::QueryCanceled]) + + with_them do + it 'retries on timeout error' do + expect(migration).to receive(:update_batch).exactly(3).times.and_raise(error_class) + expect(migration).to receive(:sleep).with(5).twice + + expect do + perform_migration + end.to raise_error(error_class) + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/populate_vulnerability_reads_spec.rb b/spec/lib/gitlab/background_migration/populate_vulnerability_reads_spec.rb index 3de84a4e880..fc06012ed20 100644 --- a/spec/lib/gitlab/background_migration/populate_vulnerability_reads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_vulnerability_reads_spec.rb @@ -28,7 +28,7 @@ RSpec.describe Gitlab::BackgroundMigration::PopulateVulnerabilityReads, :migrati project_id: project.id, external_type: 'uuid-v5', external_id: 'uuid-v5', - fingerprint: Digest::SHA1.hexdigest("#{vulnerability.id}"), + fingerprint: Digest::SHA1.hexdigest(vulnerability.id.to_s), name: 'Identifier for UUIDv5') create_finding!( diff --git a/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb b/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb index 41266cb24da..10597e65910 100644 --- a/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb +++ b/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb @@ -85,8 +85,9 @@ RSpec.describe Gitlab::BackgroundMigration::RemoveBackfilledJobArtifactsExpireAt private def create_job_artifact(id:, file_type:, expire_at:) - job = table(:ci_builds, database: :ci).create!(id: id) - job_artifact.create!(id: id, job_id: job.id, expire_at: expire_at, project_id: project.id, file_type: file_type) + job = table(:ci_builds, database: :ci).create!(id: id, partition_id: 100) + job_artifact.create!(id: id, job_id: job.id, expire_at: expire_at, project_id: project.id, + file_type: file_type, partition_id: 100) end end end diff --git a/spec/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_spec.rb b/spec/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_spec.rb deleted file mode 100644 index b6da8f7fc2d..00000000000 --- a/spec/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_spec.rb +++ /dev/null @@ -1,70 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::ResetDuplicateCiRunnersTokenEncryptedValues, - :migration, - schema: 20220922143634 do - it { expect(described_class).to be < Gitlab::BackgroundMigration::BatchedMigrationJob } - - describe '#perform' do - let(:ci_runners) { table(:ci_runners, database: :ci) } - - let(:test_worker) do - described_class.new( - start_id: 1, - end_id: 4, - batch_table: :ci_runners, - batch_column: :id, - sub_batch_size: 2, - pause_ms: 0, - connection: Ci::ApplicationRecord.connection - ) - end - - subject(:perform) { test_worker.perform } - - before do - ci_runners.create!(id: 1, runner_type: 1, token_encrypted: 'duplicate') - ci_runners.create!(id: 2, runner_type: 1, token_encrypted: 'a-token') - ci_runners.create!(id: 3, runner_type: 1, token_encrypted: 'duplicate-2') - ci_runners.create!(id: 4, runner_type: 1, token_encrypted: nil) - ci_runners.create!(id: 5, runner_type: 1, token_encrypted: 'duplicate-2') - ci_runners.create!(id: 6, runner_type: 1, token_encrypted: 'duplicate') - ci_runners.create!(id: 7, runner_type: 1, token_encrypted: 'another-token') - ci_runners.create!(id: 8, runner_type: 1, token_encrypted: 'another-token') - end - - it 'nullifies duplicate encrypted tokens', :aggregate_failures do - expect { perform }.to change { ci_runners.all.order(:id).pluck(:id, :token_encrypted).to_h } - .from( - { - 1 => 'duplicate', - 2 => 'a-token', - 3 => 'duplicate-2', - 4 => nil, - 5 => 'duplicate-2', - 6 => 'duplicate', - 7 => 'another-token', - 8 => 'another-token' - } - ) - .to( - { - 1 => nil, - 2 => 'a-token', - 3 => nil, - 4 => nil, - 5 => nil, - 6 => nil, - 7 => 'another-token', - 8 => 'another-token' - } - ) - expect(ci_runners.count).to eq(8) - expect(ci_runners.pluck(:token_encrypted).uniq).to match_array [ - nil, 'a-token', 'another-token' - ] - end - end -end diff --git a/spec/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_spec.rb b/spec/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_spec.rb deleted file mode 100644 index 423b1815e75..00000000000 --- a/spec/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_spec.rb +++ /dev/null @@ -1,70 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::ResetDuplicateCiRunnersTokenValues, - :migration, - schema: 20220922143143 do - it { expect(described_class).to be < Gitlab::BackgroundMigration::BatchedMigrationJob } - - describe '#perform' do - let(:ci_runners) { table(:ci_runners, database: :ci) } - - let(:test_worker) do - described_class.new( - start_id: 1, - end_id: 4, - batch_table: :ci_runners, - batch_column: :id, - sub_batch_size: 2, - pause_ms: 0, - connection: Ci::ApplicationRecord.connection - ) - end - - subject(:perform) { test_worker.perform } - - before do - ci_runners.create!(id: 1, runner_type: 1, token: 'duplicate') - ci_runners.create!(id: 2, runner_type: 1, token: 'a-token') - ci_runners.create!(id: 3, runner_type: 1, token: 'duplicate-2') - ci_runners.create!(id: 4, runner_type: 1, token: nil) - ci_runners.create!(id: 5, runner_type: 1, token: 'duplicate-2') - ci_runners.create!(id: 6, runner_type: 1, token: 'duplicate') - ci_runners.create!(id: 7, runner_type: 1, token: 'another-token') - ci_runners.create!(id: 8, runner_type: 1, token: 'another-token') - end - - it 'nullifies duplicate tokens', :aggregate_failures do - expect { perform }.to change { ci_runners.all.order(:id).pluck(:id, :token).to_h } - .from( - { - 1 => 'duplicate', - 2 => 'a-token', - 3 => 'duplicate-2', - 4 => nil, - 5 => 'duplicate-2', - 6 => 'duplicate', - 7 => 'another-token', - 8 => 'another-token' - } - ) - .to( - { - 1 => nil, - 2 => 'a-token', - 3 => nil, - 4 => nil, - 5 => nil, - 6 => nil, - 7 => 'another-token', - 8 => 'another-token' - } - ) - expect(ci_runners.count).to eq(8) - expect(ci_runners.pluck(:token).uniq).to match_array [ - nil, 'a-token', 'another-token' - ] - end - end -end diff --git a/spec/lib/gitlab/background_migration/sanitize_confidential_todos_spec.rb b/spec/lib/gitlab/background_migration/sanitize_confidential_todos_spec.rb new file mode 100644 index 00000000000..2c5c47e39c9 --- /dev/null +++ b/spec/lib/gitlab/background_migration/sanitize_confidential_todos_spec.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::SanitizeConfidentialTodos, :migration, schema: 20221110045406 do + let(:todos) { table(:todos) } + let(:notes) { table(:notes) } + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:project_features) { table(:project_features) } + let(:users) { table(:users) } + let(:issues) { table(:issues) } + let(:members) { table(:members) } + let(:project_authorizations) { table(:project_authorizations) } + + let(:user) { users.create!(first_name: 'Test', last_name: 'User', email: 'test@user.com', projects_limit: 1) } + let(:project_namespace1) { namespaces.create!(path: 'pns1', name: 'pns1') } + let(:project_namespace2) { namespaces.create!(path: 'pns2', name: 'pns2') } + + let(:project1) do + projects.create!(namespace_id: project_namespace1.id, + project_namespace_id: project_namespace1.id, visibility_level: 20) + end + + let(:project2) do + projects.create!(namespace_id: project_namespace2.id, + project_namespace_id: project_namespace2.id) + end + + let(:issue1) { issues.create!(project_id: project1.id, issue_type: 1, title: 'issue1', author_id: user.id) } + let(:issue2) { issues.create!(project_id: project2.id, issue_type: 1, title: 'issue2') } + + let(:public_note) { notes.create!(note: 'text', project_id: project1.id) } + + let(:confidential_note) do + notes.create!(note: 'text', project_id: project1.id, confidential: true, + noteable_id: issue1.id, noteable_type: 'Issue') + end + + let(:other_confidential_note) do + notes.create!(note: 'text', project_id: project2.id, confidential: true, + noteable_id: issue2.id, noteable_type: 'Issue') + end + + let(:common_params) { { user_id: user.id, author_id: user.id, action: 1, state: 'pending', target_type: 'Note' } } + let!(:ignored_todo1) { todos.create!(**common_params) } + let!(:ignored_todo2) { todos.create!(**common_params, target_id: public_note.id, note_id: public_note.id) } + let!(:valid_todo) { todos.create!(**common_params, target_id: confidential_note.id, note_id: confidential_note.id) } + let!(:invalid_todo) do + todos.create!(**common_params, target_id: other_confidential_note.id, note_id: other_confidential_note.id) + end + + describe '#perform' do + before do + project_features.create!(project_id: project1.id, issues_access_level: 20, pages_access_level: 20) + members.create!(state: 0, source_id: project1.id, source_type: 'Project', + type: 'ProjectMember', user_id: user.id, access_level: 50, notification_level: 0, + member_namespace_id: project_namespace1.id) + project_authorizations.create!(project_id: project1.id, user_id: user.id, access_level: 50) + end + + subject(:perform) do + described_class.new( + start_id: notes.minimum(:id), + end_id: notes.maximum(:id), + batch_table: :notes, + batch_column: :id, + sub_batch_size: 1, + pause_ms: 0, + connection: ApplicationRecord.connection + ).perform + end + + it 'deletes todos where user can not read its note and logs deletion', :aggregate_failures do + expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |logger| + expect(logger).to receive(:info).with( + hash_including( + message: "#{described_class.name} deleting invalid todo", + attributes: hash_including(invalid_todo.attributes.slice(:id, :user_id, :target_id, :target_type)) + ) + ).once + end + + expect { perform }.to change(todos, :count).by(-1) + + expect(todos.all).to match_array([ignored_todo1, ignored_todo2, valid_todo]) + end + end +end diff --git a/spec/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status_spec.rb b/spec/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status_spec.rb index 98939e15952..fad10aba882 100644 --- a/spec/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status_spec.rb +++ b/spec/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status_spec.rb @@ -26,8 +26,8 @@ RSpec.describe Gitlab::BackgroundMigration::UpdateCiPipelineArtifactsUnknownLock let(:locked) { 1 } let(:unknown) { 2 } - let(:unlocked_pipeline) { pipelines.create!(locked: unlocked) } - let(:locked_pipeline) { pipelines.create!(locked: locked) } + let(:unlocked_pipeline) { pipelines.create!(locked: unlocked, partition_id: 100) } + let(:locked_pipeline) { pipelines.create!(locked: locked, partition_id: 100) } # rubocop:disable Layout/LineLength let!(:locked_artifact) { pipeline_artifacts.create!(project_id: project.id, pipeline_id: locked_pipeline.id, size: 1024, file_type: 0, file_format: 'gzip', file: 'a.gz', locked: unknown) } -- cgit v1.2.1