Add latest changes from gitlab-org/gitlab@14-7-stable-ee

7 files changed, 61 insertions, 32 deletions

diff --git a/db/migrate/20210811193033_add_unique_index_to_vulnerability_finding_links.rb b/db/migrate/20210811193033_add_unique_index_to_vulnerability_finding_links.rb
index 1bcee89ae57..cf51bca630a 100644
--- a/db/migrate/20210811193033_add_unique_index_to_vulnerability_finding_links.rb
+++ b/db/migrate/20210811193033_add_unique_index_to_vulnerability_finding_links.rb
@@ -1,18 +1,14 @@
 # frozen_string_literal: true
 
 class AddUniqueIndexToVulnerabilityFindingLinks < Gitlab::Database::Migration[1.0]
-  disable_ddl_transaction!
-
-  NAME_URL_INDEX_NAME = 'finding_link_name_url_idx'
-  URL_INDEX_NAME = 'finding_link_url_idx'
+  # This migration has been moved to db/post_migrate/20220201193033_add_unique_index_to_vulnerability_finding_links_with_truncate.rb
+  # Previously, this was causing an bug where there was a conflict between the table cleanup and the index creation.
 
   def up
-    add_concurrent_index :vulnerability_finding_links, [:vulnerability_occurrence_id, :name, :url], unique: true, name: NAME_URL_INDEX_NAME
-    add_concurrent_index :vulnerability_finding_links, [:vulnerability_occurrence_id, :url], unique: true, where: 'name is null', name: URL_INDEX_NAME
+    # no op
   end
 
   def down
-    remove_concurrent_index :vulnerability_finding_links, [:vulnerability_occurrence_id, :name, :url], name: NAME_URL_INDEX_NAME
-    remove_concurrent_index :vulnerability_finding_links, [:vulnerability_occurrence_id, :url], name: URL_INDEX_NAME
+    # no op
   end
 end
diff --git a/db/post_migrate/20211104165220_remove_vulnerability_finding_links.rb b/db/post_migrate/20211104165220_remove_vulnerability_finding_links.rb
index fc50aa812a7..1625d24ef92 100644
--- a/db/post_migrate/20211104165220_remove_vulnerability_finding_links.rb
+++ b/db/post_migrate/20211104165220_remove_vulnerability_finding_links.rb
@@ -1,21 +1,14 @@
 # frozen_string_literal: true
 
 class RemoveVulnerabilityFindingLinks < Gitlab::Database::Migration[1.0]
-  BATCH_SIZE = 50_000
-  MIGRATION = 'RemoveVulnerabilityFindingLinks'
-
-  disable_ddl_transaction!
+  # This migration has been moved to a TRUNCATE in db/post_migrate/20220201193033_add_unique_index_to_vulnerability_finding_links_with_truncate.rb
+  # Previously, this was causing an bug where there was a conflict between the table cleanup and the index creation.
 
   def up
-    queue_background_migration_jobs_by_range_at_intervals(
-      define_batchable_model('vulnerability_finding_links'),
-      MIGRATION,
-      2.minutes,
-      batch_size: BATCH_SIZE
-    )
+    # no op
   end
 
   def down
-    # no ops
+    # no op
   end
 end
diff --git a/db/post_migrate/20211210173137_remove_vulnerability_finding_links_again.rb b/db/post_migrate/20211210173137_remove_vulnerability_finding_links_again.rb
index 98ac4433193..ffe8bf0d794 100644
--- a/db/post_migrate/20211210173137_remove_vulnerability_finding_links_again.rb
+++ b/db/post_migrate/20211210173137_remove_vulnerability_finding_links_again.rb
@@ -1,21 +1,14 @@
 # frozen_string_literal: true
 
 class RemoveVulnerabilityFindingLinksAgain < Gitlab::Database::Migration[1.0]
-  BATCH_SIZE = 50_000
-  MIGRATION = 'RemoveVulnerabilityFindingLinks'
-
-  disable_ddl_transaction!
+  # This migration has been moved to a TRUNCATE in db/post_migrate/20220201193033_add_unique_index_to_vulnerability_finding_links_with_truncate.rb
+  # Previously, this was causing an bug where there was a conflict between the table cleanup and the index creation.
 
   def up
-    queue_background_migration_jobs_by_range_at_intervals(
-      define_batchable_model('vulnerability_finding_links'),
-      MIGRATION,
-      2.minutes,
-      batch_size: BATCH_SIZE
-    )
+    # no op
   end
 
   def down
-    # no ops
+    # no op
   end
 end
diff --git a/db/post_migrate/20220201193033_add_unique_index_to_vulnerability_finding_links_with_truncate.rb b/db/post_migrate/20220201193033_add_unique_index_to_vulnerability_finding_links_with_truncate.rb
new file mode 100644
index 00000000000..cc9dabdf624
--- /dev/null
+++ b/db/post_migrate/20220201193033_add_unique_index_to_vulnerability_finding_links_with_truncate.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class AddUniqueIndexToVulnerabilityFindingLinksWithTruncate < Gitlab::Database::Migration[1.0]
+  disable_ddl_transaction!
+
+  NAME_URL_INDEX_NAME = 'finding_link_name_url_idx'
+  URL_INDEX_NAME = 'finding_link_url_idx'
+
+  def up
+    execute('TRUNCATE TABLE vulnerability_finding_links')
+
+    add_concurrent_index :vulnerability_finding_links, [:vulnerability_occurrence_id, :name, :url], unique: true, name: NAME_URL_INDEX_NAME
+    add_concurrent_index :vulnerability_finding_links, [:vulnerability_occurrence_id, :url], unique: true, where: 'name is null', name: URL_INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index :vulnerability_finding_links, [:vulnerability_occurrence_id, :name, :url], name: NAME_URL_INDEX_NAME
+    remove_concurrent_index :vulnerability_finding_links, [:vulnerability_occurrence_id, :url], name: URL_INDEX_NAME
+  end
+end
diff --git a/db/schema_migrations/20220201193033 b/db/schema_migrations/20220201193033
new file mode 100644
index 00000000000..e40840c66fa
--- /dev/null
+++ b/db/schema_migrations/20220201193033
@@ -0,0 +1 @@
+92bbe74c6c3627dd26f709acd2a20f442212eab933f719be815701a3bc429539
\ No newline at end of file
diff --git a/lib/gitlab/cleanup/orphan_job_artifact_files.rb b/lib/gitlab/cleanup/orphan_job_artifact_files.rb
index 05dfdcd4486..90123b9d000 100644
--- a/lib/gitlab/cleanup/orphan_job_artifact_files.rb
+++ b/lib/gitlab/cleanup/orphan_job_artifact_files.rb
@@ -99,6 +99,9 @@ module Gitlab
           #                  ^--+--+- components of hashed storage project path
           cmd += %w[-mindepth 6 -maxdepth 6]
 
+          # Intentionally exclude pipeline artifacts which match the same path
+          cmd += %w[-not -path */pipelines/*]
+
           # Artifact directories are named on their ID
           cmd += %w[-type d]
 
diff --git a/spec/lib/gitlab/cleanup/orphan_job_artifact_files_spec.rb b/spec/lib/gitlab/cleanup/orphan_job_artifact_files_spec.rb
index b5adb603dab..e6ef2d8a541 100644
--- a/spec/lib/gitlab/cleanup/orphan_job_artifact_files_spec.rb
+++ b/spec/lib/gitlab/cleanup/orphan_job_artifact_files_spec.rb
@@ -34,10 +34,33 @@ RSpec.describe Gitlab::Cleanup::OrphanJobArtifactFiles do
     cleanup.run!
   end
 
-  it 'finds artifacts on disk' do
+  it 'finds job artifacts on disk' do
     artifact = create(:ci_job_artifact, :archive)
+    artifact_directory = artifact.file.relative_path.to_s.split('/')[0...6].join('/')
+
+    cleaned = []
+
+    expect(cleanup).to receive(:find_artifacts).and_wrap_original do |original_method, *args, &block|
+      original_method.call(*args) { |dir| cleaned << dir }
+    end
+
+    cleanup.run!
+
+    expect(cleaned).to include(/#{artifact_directory}/)
+  end
+
+  it 'does not find pipeline artifacts on disk' do
+    artifact = create(:ci_pipeline_artifact, :with_coverage_report)
+    # using 0...6 to match the -min/maxdepth 6 strictly, since this is one directory
+    # deeper than job artifacts, and .dirname would not match
+    artifact_directory = artifact.file.relative_path.to_s.split('/')[0...6].join('/')
+
+    expect(cleanup).to receive(:find_artifacts).and_wrap_original do |original_method, *args, &block|
+      # this can either _not_ yield at all, or yield with any other file
+      # except the one that we're explicitly excluding
+      original_method.call(*args) { |path| expect(path).not_to match(artifact_directory) }
+    end
 
-    expect(cleanup).to receive(:find_artifacts).and_yield(artifact.file.path)
     cleanup.run!
   end