Add latest changes from gitlab-org/gitlab@13-6-stable-eev13.6.0-rc42

7 files changed, 429 insertions, 7 deletions

diff --git a/lib/gitlab/background_migration/backfill_design_internal_ids.rb b/lib/gitlab/background_migration/backfill_design_internal_ids.rb
new file mode 100644
index 00000000000..553571d5d00
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_design_internal_ids.rb
@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Backfill design.iid for a range of projects
+    class BackfillDesignInternalIds
+      # See app/models/internal_id
+      # This is a direct copy of the application code with the following changes:
+      # - usage enum is hard-coded to the value for design_management_designs
+      # - init is not passed around, but ignored
+      class InternalId < ActiveRecord::Base
+        def self.track_greatest(subject, scope, new_value)
+          InternalIdGenerator.new(subject, scope).track_greatest(new_value)
+        end
+
+        # Increments #last_value with new_value if it is greater than the current,
+        # and saves the record
+        #
+        # The operation locks the record and gathers a `ROW SHARE` lock (in PostgreSQL).
+        # As such, the increment is atomic and safe to be called concurrently.
+        def track_greatest_and_save!(new_value)
+          update_and_save { self.last_value = [last_value || 0, new_value].max }
+        end
+
+        private
+
+        def update_and_save(&block)
+          lock!
+          yield
+          # update_and_save_counter.increment(usage: usage, changed: last_value_changed?)
+          save!
+          last_value
+        end
+      end
+
+      # See app/models/internal_id
+      class InternalIdGenerator
+        attr_reader :subject, :scope, :scope_attrs
+
+        def initialize(subject, scope)
+          @subject = subject
+          @scope = scope
+
+          raise ArgumentError, 'Scope is not well-defined, need at least one column for scope (given: 0)' if scope.empty?
+        end
+
+        # Create a record in internal_ids if one does not yet exist
+        # and set its new_value if it is higher than the current last_value
+        #
+        # Note this will acquire a ROW SHARE lock on the InternalId record
+        def track_greatest(new_value)
+          subject.transaction do
+            record.track_greatest_and_save!(new_value)
+          end
+        end
+
+        def record
+          @record ||= (lookup || create_record)
+        end
+
+        def lookup
+          InternalId.find_by(**scope, usage: usage_value)
+        end
+
+        def usage_value
+          10 # see Enums::InternalId - this is the value for design_management_designs
+        end
+
+        # Create InternalId record for (scope, usage) combination, if it doesn't exist
+        #
+        # We blindly insert without synchronization. If another process
+        # was faster in doing this, we'll realize once we hit the unique key constraint
+        # violation. We can safely roll-back the nested transaction and perform
+        # a lookup instead to retrieve the record.
+        def create_record
+          subject.transaction(requires_new: true) do
+            InternalId.create!(
+              **scope,
+              usage: usage_value,
+              last_value: 0
+            )
+          end
+        rescue ActiveRecord::RecordNotUnique
+          lookup
+        end
+      end
+
+      attr_reader :design_class
+
+      def initialize(design_class)
+        @design_class = design_class
+      end
+
+      def perform(relation)
+        start_id, end_id = relation.pluck("min(project_id), max(project_id)").flatten
+        table = 'design_management_designs'
+
+        ActiveRecord::Base.connection.execute <<~SQL
+          WITH
+          starting_iids(project_id, iid) as (
+            SELECT project_id, MAX(COALESCE(iid, 0))
+            FROM #{table}
+            WHERE project_id BETWEEN #{start_id} AND #{end_id}
+            GROUP BY project_id
+          ),
+          with_calculated_iid(id, iid) as (
+            SELECT design.id,
+                   init.iid + ROW_NUMBER() OVER (PARTITION BY design.project_id ORDER BY design.id ASC)
+              FROM #{table} as design, starting_iids as init
+              WHERE design.project_id BETWEEN #{start_id} AND #{end_id}
+                AND design.iid IS NULL
+                AND init.project_id = design.project_id
+          )
+
+          UPDATE #{table}
+             SET iid = with_calculated_iid.iid
+            FROM with_calculated_iid
+           WHERE #{table}.id = with_calculated_iid.id
+        SQL
+
+        # track the new greatest IID value
+        relation.each do |design|
+          current_max = design_class.where(project_id: design.project_id).maximum(:iid)
+          scope = { project_id: design.project_id }
+          InternalId.track_greatest(design, scope, current_max)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb b/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb
new file mode 100644
index 00000000000..61145f6a445
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb
@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+# Based on https://community.developer.atlassian.com/t/get-rest-api-3-filter-search/29459/2,
+# it's enough at the moment to simply notice if the url is from `atlassian.net`
+module Gitlab
+  module BackgroundMigration
+    # Backfill the deployment_type in jira_tracker_data table
+    class BackfillJiraTrackerDeploymentType2
+      # Migration only version of jira_tracker_data table
+      class JiraTrackerDataTemp < ApplicationRecord
+        self.table_name = 'jira_tracker_data'
+
+        def self.encryption_options
+          {
+            key: Settings.attr_encrypted_db_key_base_32,
+            encode: true,
+            mode: :per_attribute_iv,
+            algorithm: 'aes-256-gcm'
+          }
+        end
+
+        attr_encrypted :url, encryption_options
+        attr_encrypted :api_url, encryption_options
+
+        enum deployment_type: { unknown: 0, server: 1, cloud: 2 }, _prefix: :deployment
+      end
+
+      # Migration only version of services table
+      class JiraServiceTemp < ApplicationRecord
+        self.table_name = 'services'
+        self.inheritance_column = :_type_disabled
+      end
+
+      def perform(start_id, stop_id)
+        @server_ids = []
+        @cloud_ids  = []
+
+        JiraTrackerDataTemp
+          .where(id: start_id..stop_id, deployment_type: 0)
+          .each do |jira_tracker_data|
+            collect_deployment_type(jira_tracker_data)
+          end
+
+        unless cloud_ids.empty?
+          JiraTrackerDataTemp.where(id: cloud_ids)
+            .update_all(deployment_type: JiraTrackerDataTemp.deployment_types[:cloud])
+        end
+
+        unless server_ids.empty?
+          JiraTrackerDataTemp.where(id: server_ids)
+            .update_all(deployment_type: JiraTrackerDataTemp.deployment_types[:server])
+        end
+
+        mark_jobs_as_succeeded(start_id, stop_id)
+      end
+
+      private
+
+      attr_reader :server_ids, :cloud_ids
+
+      def client_url(jira_tracker_data)
+        jira_tracker_data.api_url.presence || jira_tracker_data.url.presence
+      end
+
+      def server_type(url)
+        url.downcase.include?('.atlassian.net') ? :cloud : :server
+      end
+
+      def collect_deployment_type(jira_tracker_data)
+        url = client_url(jira_tracker_data)
+        return unless url
+
+        case server_type(url)
+        when :cloud
+          cloud_ids << jira_tracker_data.id
+        else
+          server_ids << jira_tracker_data.id
+        end
+      end
+
+      def mark_jobs_as_succeeded(*arguments)
+        Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(self.class.name, arguments)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/backfill_merge_request_cleanup_schedules.rb b/lib/gitlab/background_migration/backfill_merge_request_cleanup_schedules.rb
new file mode 100644
index 00000000000..8a58cf9b302
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_merge_request_cleanup_schedules.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Backfill merge request cleanup schedules of closed/merged merge requests
+    # without any corresponding records.
+    class BackfillMergeRequestCleanupSchedules
+      # Model used for migration added in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/46782.
+      class MergeRequest < ActiveRecord::Base
+        include EachBatch
+
+        self.table_name = 'merge_requests'
+
+        def self.eligible
+          where('merge_requests.state_id IN (2, 3)')
+        end
+      end
+
+      def perform(start_id, end_id)
+        eligible_mrs = MergeRequest.eligible.where(id: start_id..end_id)
+        scheduled_at_column = "COALESCE(metrics.merged_at, COALESCE(metrics.latest_closed_at, merge_requests.updated_at)) + interval '14 days'"
+        query =
+          eligible_mrs
+            .select("merge_requests.id, #{scheduled_at_column}, NOW(), NOW()")
+            .joins('LEFT JOIN merge_request_metrics metrics ON metrics.merge_request_id = merge_requests.id')
+
+        result = ActiveRecord::Base.connection.execute <<~SQL
+          INSERT INTO merge_request_cleanup_schedules (merge_request_id, scheduled_at, created_at, updated_at)
+          #{query.to_sql}
+          ON CONFLICT (merge_request_id) DO NOTHING;
+        SQL
+
+        ::Gitlab::BackgroundMigration::Logger.info(
+          message: 'Backfilled merge_request_cleanup_schedules records',
+          count: result.cmd_tuples
+        )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/populate_has_vulnerabilities.rb b/lib/gitlab/background_migration/populate_has_vulnerabilities.rb
new file mode 100644
index 00000000000..78140b768fc
--- /dev/null
+++ b/lib/gitlab/background_migration/populate_has_vulnerabilities.rb
@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # This class populates missing dismissal information for
+    # vulnerability entries.
+    class PopulateHasVulnerabilities
+      class ProjectSetting < ActiveRecord::Base # rubocop:disable Style/Documentation
+        self.table_name = 'project_settings'
+
+        UPSERT_SQL = <<~SQL
+          WITH upsert_data (project_id, has_vulnerabilities, created_at, updated_at) AS (
+            SELECT projects.id, true, current_timestamp, current_timestamp FROM projects WHERE projects.id IN (%{project_ids})
+          )
+          INSERT INTO project_settings
+          (project_id, has_vulnerabilities, created_at, updated_at)
+          (SELECT * FROM upsert_data)
+          ON CONFLICT (project_id)
+          DO UPDATE SET
+            has_vulnerabilities = true,
+            updated_at = EXCLUDED.updated_at
+        SQL
+
+        def self.upsert_for(project_ids)
+          connection.execute(UPSERT_SQL % { project_ids: project_ids.join(', ') })
+        end
+      end
+
+      class Vulnerability < ActiveRecord::Base # rubocop:disable Style/Documentation
+        include EachBatch
+
+        self.table_name = 'vulnerabilities'
+      end
+
+      def perform(*project_ids)
+        ProjectSetting.upsert_for(project_ids)
+      rescue StandardError => e
+        log_error(e, project_ids)
+      ensure
+        log_info(project_ids)
+      end
+
+      private
+
+      def log_error(error, project_ids)
+        ::Gitlab::BackgroundMigration::Logger.error(
+          migrator: self.class.name,
+          message: error.message,
+          project_ids: project_ids
+        )
+      end
+
+      def log_info(project_ids)
+        ::Gitlab::BackgroundMigration::Logger.info(
+          migrator: self.class.name,
+          message: 'Projects has been processed to populate `has_vulnerabilities` information',
+          count: project_ids.length
+        )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information.rb b/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information.rb
new file mode 100644
index 00000000000..bc0a181a06c
--- /dev/null
+++ b/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information.rb
@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # This class populates missing dismissal information for
+    # vulnerability entries.
+    class PopulateMissingVulnerabilityDismissalInformation
+      class Vulnerability < ActiveRecord::Base # rubocop:disable Style/Documentation
+        include EachBatch
+
+        self.table_name = 'vulnerabilities'
+
+        has_one :finding, class_name: '::Gitlab::BackgroundMigration::PopulateMissingVulnerabilityDismissalInformation::Finding'
+
+        scope :broken, -> { where('state = 2 AND (dismissed_at IS NULL OR dismissed_by_id IS NULL)') }
+
+        def copy_dismissal_information
+          return unless finding&.dismissal_feedback
+
+          update_columns(
+            dismissed_at: finding.dismissal_feedback.created_at,
+            dismissed_by_id: finding.dismissal_feedback.author_id
+          )
+        end
+      end
+
+      class Finding < ActiveRecord::Base # rubocop:disable Style/Documentation
+        include ShaAttribute
+
+        self.table_name = 'vulnerability_occurrences'
+
+        sha_attribute :project_fingerprint
+
+        def dismissal_feedback
+          Feedback.dismissal.where(category: report_type, project_fingerprint: project_fingerprint, project_id: project_id).first
+        end
+      end
+
+      class Feedback < ActiveRecord::Base # rubocop:disable Style/Documentation
+        DISMISSAL_TYPE = 0
+
+        self.table_name = 'vulnerability_feedback'
+
+        scope :dismissal, -> { where(feedback_type: DISMISSAL_TYPE) }
+      end
+
+      def perform(*vulnerability_ids)
+        Vulnerability.includes(:finding).where(id: vulnerability_ids).each { |vulnerability| populate_for(vulnerability) }
+
+        log_info(vulnerability_ids)
+      end
+
+      private
+
+      def populate_for(vulnerability)
+        log_warning(vulnerability) unless vulnerability.copy_dismissal_information
+      rescue StandardError => error
+        log_error(error, vulnerability)
+      end
+
+      def log_info(vulnerability_ids)
+        ::Gitlab::BackgroundMigration::Logger.info(
+          migrator: self.class.name,
+          message: 'Dismissal information has been copied',
+          count: vulnerability_ids.length
+        )
+      end
+
+      def log_warning(vulnerability)
+        ::Gitlab::BackgroundMigration::Logger.warn(
+          migrator: self.class.name,
+          message: 'Could not update vulnerability!',
+          vulnerability_id: vulnerability.id
+        )
+      end
+
+      def log_error(error, vulnerability)
+        ::Gitlab::BackgroundMigration::Logger.error(
+          migrator: self.class.name,
+          message: error.message,
+          vulnerability_id: vulnerability.id
+        )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb b/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb
new file mode 100644
index 00000000000..fc79f7125e3
--- /dev/null
+++ b/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # This class updates vulnerability feedback entities with no pipeline id assigned.
+    class PopulateVulnerabilityFeedbackPipelineId
+      def perform(project_ids)
+      end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::PopulateVulnerabilityFeedbackPipelineId.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateVulnerabilityFeedbackPipelineId')
diff --git a/lib/gitlab/background_migration/replace_blocked_by_links.rb b/lib/gitlab/background_migration/replace_blocked_by_links.rb
index 26626aaef79..0c29887bb00 100644
--- a/lib/gitlab/background_migration/replace_blocked_by_links.rb
+++ b/lib/gitlab/background_migration/replace_blocked_by_links.rb
@@ -12,14 +12,19 @@ module Gitlab
         blocked_by_links = IssueLink.where(id: start_id..stop_id).where(link_type: 2)
 
         ActiveRecord::Base.transaction do
-          # if there is duplicit bi-directional relation (issue2 is blocked by issue1
-          # and issue1 already links issue2), then we can just delete 'blocked by'.
-          # This should be rare as we have a pre-create check which checks if issues are
-          # already linked
-          blocked_by_links
+          # There could be two edge cases:
+          # 1) issue1 is blocked by issue2 AND issue2 blocks issue1 (type 1)
+          # 2) issue1 is blocked by issue2 AND issue2 is related to issue1 (type 0)
+          # In both cases cases we couldn't convert blocked by relation to
+          # `issue2 blocks issue` because there is already a link with the same
+          # source/target id. To avoid these conflicts, we first delete any
+          # "opposite" links before we update `blocked by` relation.  This
+          # should be rare as we have a pre-create check which checks if issues
+          # are already linked
+          opposite_ids = blocked_by_links
+            .select('opposite_links.id')
             .joins('INNER JOIN issue_links as opposite_links ON issue_links.source_id = opposite_links.target_id AND issue_links.target_id = opposite_links.source_id')
-            .where('opposite_links.link_type': 1)
-            .delete_all
+          IssueLink.where(id: opposite_ids).delete_all
 
           blocked_by_links.update_all('source_id=target_id,target_id=source_id,link_type=1')
         end