Add latest changes from gitlab-org/gitlab@13-4-stable-ee

11 files changed, 136 insertions, 16 deletions

diff --git a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb
index c912628d0fc..5b9ee8a0ee2 100644
--- a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb
+++ b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb
@@ -9,7 +9,7 @@ module Gitlab
       end
 
       def perform(start_id, stop_id)
-        Rails.logger.info("Setting commits_count for merge request diffs: #{start_id} - #{stop_id}") # rubocop:disable Gitlab/RailsLogger
+        Gitlab::AppLogger.info("Setting commits_count for merge request diffs: #{start_id} - #{stop_id}")
 
         update = '
           commits_count = (
diff --git a/lib/gitlab/background_migration/calculate_wiki_sizes.rb b/lib/gitlab/background_migration/calculate_wiki_sizes.rb
index e62f5edd0e7..76598f6e2a6 100644
--- a/lib/gitlab/background_migration/calculate_wiki_sizes.rb
+++ b/lib/gitlab/background_migration/calculate_wiki_sizes.rb
@@ -10,7 +10,7 @@ module Gitlab
           .includes(project: [:route, :group, namespace: [:owner]]).find_each do |statistics|
           statistics.refresh!(only: [:wiki_size])
         rescue => e
-          Rails.logger.error "Failed to update wiki statistics. id: #{statistics.id} message: #{e.message}" # rubocop:disable Gitlab/RailsLogger
+          Gitlab::AppLogger.error "Failed to update wiki statistics. id: #{statistics.id} message: #{e.message}"
         end
       end
     end
diff --git a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb
index 4016b807f21..c0099d44b5a 100644
--- a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb
+++ b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb
@@ -25,7 +25,7 @@ module Gitlab
             certificate_valid_not_after: domain.x509&.not_after&.iso8601
           )
         rescue => e
-          Rails.logger.error "Failed to update pages domain certificate valid time. id: #{domain.id}, message: #{e.message}" # rubocop:disable Gitlab/RailsLogger
+          Gitlab::AppLogger.error "Failed to update pages domain certificate valid time. id: #{domain.id}, message: #{e.message}"
         end
       end
     end
diff --git a/lib/gitlab/background_migration/fix_pages_access_level.rb b/lib/gitlab/background_migration/fix_pages_access_level.rb
index 31d2e78b2d2..8e46021bd93 100644
--- a/lib/gitlab/background_migration/fix_pages_access_level.rb
+++ b/lib/gitlab/background_migration/fix_pages_access_level.rb
@@ -103,8 +103,8 @@ module Gitlab
       end
 
       # Private projects are not allowed to have enabled access level, only `private` and `public`
-      # If access control is enabled, these projects currently behave as if the have `private` pages_access_level
-      # if access control is disabled, these projects currently behave as if the have `public` pages_access_level
+      # If access control is enabled, these projects currently behave as if they have `private` pages_access_level
+      # if access control is disabled, these projects currently behave as if they have `public` pages_access_level
       # so we preserve this behaviour for projects with pages already deployed
       # for project without pages we always set `private` access_level
       def fix_private_access_level(start_id, stop_id)
diff --git a/lib/gitlab/background_migration/migrate_to_hashed_storage.rb b/lib/gitlab/background_migration/migrate_to_hashed_storage.rb
new file mode 100644
index 00000000000..4054db4fb87
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_to_hashed_storage.rb
@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Background migration to move any legacy project to Hashed Storage
+    class MigrateToHashedStorage
+      def perform
+        batch_size = helper.batch_size
+        legacy_projects_count = Project.with_unmigrated_storage.count
+
+        if storage_migrator.rollback_pending?
+          logger.warn(
+            migrator: 'MigrateToHashedStorage',
+            message: 'Aborting an storage rollback operation currently in progress'
+          )
+
+          storage_migrator.abort_rollback!
+        end
+
+        if legacy_projects_count == 0
+          logger.info(
+            migrator: 'MigrateToHashedStorage',
+            message: 'There are no projects requiring migration to Hashed Storage'
+          )
+
+          return
+        end
+
+        logger.info(
+          migrator: 'MigrateToHashedStorage',
+          message: "Enqueuing migration of #{legacy_projects_count} projects in batches of #{batch_size}"
+        )
+
+        helper.project_id_batches_migration do |start, finish|
+          storage_migrator.bulk_schedule_migration(start: start, finish: finish)
+
+          logger.info(
+            migrator: 'MigrateToHashedStorage',
+            message: "Enqueuing migration of projects in batches of #{batch_size} from ID=#{start} to ID=#{finish}",
+            batch_from: start,
+            batch_to: finish
+          )
+        end
+      end
+
+      private
+
+      def helper
+        Gitlab::HashedStorage::RakeHelper
+      end
+
+      def storage_migrator
+        @storage_migrator ||= Gitlab::HashedStorage::Migrator.new
+      end
+
+      def logger
+        @logger ||= ::Gitlab::BackgroundMigration::Logger.build
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb b/lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb
new file mode 100644
index 00000000000..eb72ef1de33
--- /dev/null
+++ b/lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # rubocop:disable Style/Documentation
+    class PopulateResolvedOnDefaultBranchColumn
+      def perform(*); end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn')
diff --git a/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb b/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb
new file mode 100644
index 00000000000..a0c89cc4664
--- /dev/null
+++ b/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # This class creates/updates those project historical vulnerability statistics
+    # that haven't been created nor initialized. It should only be executed in EE.
+    class PopulateVulnerabilityHistoricalStatistics
+      def perform(project_ids)
+      end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::PopulateVulnerabilityHistoricalStatistics.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateVulnerabilityHistoricalStatistics')
diff --git a/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb b/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb
new file mode 100644
index 00000000000..cc9b0329556
--- /dev/null
+++ b/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class RemoveDuplicateCsFindings
+      def perform(start_id, stop_id)
+      end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::RemoveDuplicateCsFindings.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveDuplicateCsFindings')
diff --git a/lib/gitlab/background_migration/set_merge_request_diff_files_count.rb b/lib/gitlab/background_migration/set_merge_request_diff_files_count.rb
index 9f765d03d62..527dd2a0a83 100644
--- a/lib/gitlab/background_migration/set_merge_request_diff_files_count.rb
+++ b/lib/gitlab/background_migration/set_merge_request_diff_files_count.rb
@@ -4,13 +4,18 @@ module Gitlab
   module BackgroundMigration
     # Sets the MergeRequestDiff#files_count value for old rows
     class SetMergeRequestDiffFilesCount
-      COUNT_SUBQUERY = <<~SQL
-        files_count = (
-          SELECT count(*)
-          FROM merge_request_diff_files
-          WHERE merge_request_diff_files.merge_request_diff_id = merge_request_diffs.id
-        )
-      SQL
+      # Some historic data has a *lot* of files. Apply a sentinel to these cases
+      FILES_COUNT_SENTINEL = 2**15 - 1
+
+      def self.count_subquery
+        <<~SQL
+          files_count = (
+            SELECT LEAST(#{FILES_COUNT_SENTINEL}, count(*))
+            FROM merge_request_diff_files
+            WHERE merge_request_diff_files.merge_request_diff_id = merge_request_diffs.id
+          )
+        SQL
+      end
 
       class MergeRequestDiff < ActiveRecord::Base # rubocop:disable Style/Documentation
         include EachBatch
@@ -20,7 +25,7 @@ module Gitlab
 
       def perform(start_id, end_id)
         MergeRequestDiff.where(id: start_id..end_id).each_batch do |relation|
-          relation.update_all(COUNT_SUBQUERY)
+          relation.update_all(self.class.count_subquery)
         end
       end
     end
diff --git a/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb b/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb
index 9ac92aab637..c485c23f3be 100644
--- a/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb
+++ b/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb
@@ -11,9 +11,11 @@ module Gitlab
     class SetNullPackageFilesFileStoreToLocalValue
       LOCAL_STORE = 1 # equal to ObjectStorage::Store::LOCAL
 
-      # Temporary AR class for package files
-      class PackageFile < ActiveRecord::Base
-        self.table_name = 'packages_package_files'
+      module Packages
+        # Temporary AR class for package files
+        class PackageFile < ActiveRecord::Base
+          self.table_name = 'packages_package_files'
+        end
       end
 
       def perform(start_id, stop_id)
diff --git a/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb b/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb
new file mode 100644
index 00000000000..651df36fcfd
--- /dev/null
+++ b/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class UpdateLocationFingerprintForContainerScanningFindings
+      def perform(start_id, stop_id)
+      end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings')