Add latest changes from gitlab-org/gitlab@master

14 files changed, 177 insertions, 35 deletions

diff --git a/lib/api/group_export.rb b/lib/api/group_export.rb
index 6fe72458da2..8ca5dfa082e 100644
--- a/lib/api/group_export.rb
+++ b/lib/api/group_export.rb
@@ -27,9 +27,13 @@ module API
         detail 'This feature was introduced in GitLab 12.5.'
       end
       post ':id/export' do
-        GroupExportWorker.perform_async(current_user.id, user_group.id, params) # rubocop:disable CodeReuse/Worker
+        export_service = ::Groups::ImportExport::ExportService.new(group: user_group, user: current_user)
 
-        accepted!
+        if export_service.async_execute
+          accepted!
+        else
+          render_api_error!(message: 'Group export could not be started.')
+        end
       end
     end
   end
diff --git a/lib/banzai/filter/epic_reference_filter.rb b/lib/banzai/filter/epic_reference_filter.rb
index d1e1a56424d..70a6cb0a6dc 100644
--- a/lib/banzai/filter/epic_reference_filter.rb
+++ b/lib/banzai/filter/epic_reference_filter.rb
@@ -4,8 +4,6 @@ module Banzai
   module Filter
     # The actual filter is implemented in the EE mixin
     class EpicReferenceFilter < IssuableReferenceFilter
-      prepend_if_ee('EE::Banzai::Filter::EpicReferenceFilter') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
       self.reference_type = :epic
 
       def self.object_class
@@ -20,3 +18,5 @@ module Banzai
     end
   end
 end
+
+Banzai::Filter::EpicReferenceFilter.prepend_if_ee('EE::Banzai::Filter::EpicReferenceFilter')
diff --git a/lib/banzai/issuable_extractor.rb b/lib/banzai/issuable_extractor.rb
index d1e8587314c..8994cdbed60 100644
--- a/lib/banzai/issuable_extractor.rb
+++ b/lib/banzai/issuable_extractor.rb
@@ -9,8 +9,6 @@ module Banzai
   # so we can avoid N+1 queries problem
 
   class IssuableExtractor
-    prepend_if_ee('EE::Banzai::IssuableExtractor') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
     attr_reader :context
 
     ISSUE_REFERENCE_TYPE = '@data-reference-type="issue"'
@@ -59,3 +57,5 @@ module Banzai
     end
   end
 end
+
+Banzai::IssuableExtractor.prepend_if_ee('EE::Banzai::IssuableExtractor')
diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb
index b6238dfe7f0..329bbb270bd 100644
--- a/lib/banzai/pipeline/gfm_pipeline.rb
+++ b/lib/banzai/pipeline/gfm_pipeline.rb
@@ -3,8 +3,6 @@
 module Banzai
   module Pipeline
     class GfmPipeline < BasePipeline
-      prepend_if_ee('EE::Banzai::Pipeline::GfmPipeline') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
       # These filters transform GitLab Flavored Markdown (GFM) to HTML.
       # The nodes and marks referenced in app/assets/javascripts/behaviors/markdown/editor_extensions.js
       # consequently transform that same HTML to GFM to be copied to the clipboard.
@@ -77,3 +75,5 @@ module Banzai
     end
   end
 end
+
+Banzai::Pipeline::GfmPipeline.prepend_if_ee('EE::Banzai::Pipeline::GfmPipeline')
diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb
index 8236b702147..32d7126c97d 100644
--- a/lib/banzai/pipeline/post_process_pipeline.rb
+++ b/lib/banzai/pipeline/post_process_pipeline.rb
@@ -3,8 +3,6 @@
 module Banzai
   module Pipeline
     class PostProcessPipeline < BasePipeline
-      prepend_if_ee('EE::Banzai::Pipeline::PostProcessPipeline') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
       def self.filters
         @filters ||= FilterArray[
           *internal_link_filters,
@@ -34,3 +32,5 @@ module Banzai
     end
   end
 end
+
+Banzai::Pipeline::PostProcessPipeline.prepend_if_ee('EE::Banzai::Pipeline::PostProcessPipeline')
diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb
index a16ab04b792..7fe13100ec2 100644
--- a/lib/banzai/pipeline/single_line_pipeline.rb
+++ b/lib/banzai/pipeline/single_line_pipeline.rb
@@ -3,8 +3,6 @@
 module Banzai
   module Pipeline
     class SingleLinePipeline < GfmPipeline
-      prepend_if_ee('EE::Banzai::Pipeline::SingleLinePipeline') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
       def self.filters
         @filters ||= FilterArray[
           Filter::HtmlEntityFilter,
@@ -41,3 +39,5 @@ module Banzai
     end
   end
 end
+
+Banzai::Pipeline::SingleLinePipeline.prepend_if_ee('EE::Banzai::Pipeline::SingleLinePipeline')
diff --git a/lib/banzai/reference_parser/epic_parser.rb b/lib/banzai/reference_parser/epic_parser.rb
index b5fbf7accc4..7e72a260839 100644
--- a/lib/banzai/reference_parser/epic_parser.rb
+++ b/lib/banzai/reference_parser/epic_parser.rb
@@ -4,8 +4,6 @@ module Banzai
   module ReferenceParser
     # The actual parser is implemented in the EE mixin
     class EpicParser < IssuableParser
-      prepend_if_ee('::EE::Banzai::ReferenceParser::EpicParser') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
       self.reference_type = :epic
 
       def records_for_nodes(_nodes)
@@ -14,3 +12,5 @@ module Banzai
     end
   end
 end
+
+Banzai::ReferenceParser::EpicParser.prepend_if_ee('::EE::Banzai::ReferenceParser::EpicParser')
diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb
index 211c59fe841..c1066d8fa62 100644
--- a/lib/gitlab/application_rate_limiter.rb
+++ b/lib/gitlab/application_rate_limiter.rb
@@ -25,7 +25,9 @@ module Gitlab
           project_generate_new_export: { threshold: 1, interval: 5.minutes },
           project_import:              { threshold: 30, interval: 5.minutes },
           play_pipeline_schedule:      { threshold: 1, interval: 1.minute },
-          show_raw_controller:         { threshold: -> { Gitlab::CurrentSettings.current_application_settings.raw_blob_request_limit }, interval: 1.minute }
+          show_raw_controller:         { threshold: -> { Gitlab::CurrentSettings.current_application_settings.raw_blob_request_limit }, interval: 1.minute },
+          group_export:                { threshold: 1, interval: 5.minutes },
+          group_download_export:       { threshold: 10, interval: 10.minutes }
         }.freeze
       end
 
diff --git a/lib/gitlab/background_migration/populate_canonical_emails.rb b/lib/gitlab/background_migration/populate_canonical_emails.rb
new file mode 100644
index 00000000000..052e75c5655
--- /dev/null
+++ b/lib/gitlab/background_migration/populate_canonical_emails.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Class to populate new rows of UserCanonicalEmail based on existing email addresses
+    class PopulateCanonicalEmails
+      def perform(start_id, stop_id)
+        ActiveRecord::Base.connection.execute <<~SQL
+          INSERT INTO
+            user_canonical_emails (
+                user_id,
+                canonical_email,
+                created_at,
+                updated_at
+            )
+            SELECT users.id AS user_id,
+                   concat(translate(split_part(split_part(users.email, '@', 1), '+', 1), '.', ''), '@gmail.com') AS canonical_email,
+                   NOW() AS created_at,
+                   NOW() AS updated_at
+             FROM users
+            WHERE users.email ILIKE '%@gmail.com'
+              AND users.id BETWEEN #{start_id} AND #{stop_id}
+            ON CONFLICT DO NOTHING;
+        SQL
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/grafana_embed_usage_data.rb b/lib/gitlab/grafana_embed_usage_data.rb
deleted file mode 100644
index 78a87623e1f..00000000000
--- a/lib/gitlab/grafana_embed_usage_data.rb
+++ /dev/null
@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  class GrafanaEmbedUsageData
-    class << self
-      def issue_count
-        # rubocop:disable CodeReuse/ActiveRecord
-        Issue.joins('JOIN grafana_integrations USING (project_id)')
-          .where("issues.description LIKE '%' || grafana_integrations.grafana_url || '%'")
-          .where(grafana_integrations: { enabled: true })
-          .count
-        # rubocop:enable CodeReuse/ActiveRecord
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/jira_import/base_importer.rb b/lib/gitlab/jira_import/base_importer.rb
index 24158b8e734..afb443020b7 100644
--- a/lib/gitlab/jira_import/base_importer.rb
+++ b/lib/gitlab/jira_import/base_importer.rb
@@ -9,7 +9,7 @@ module Gitlab
         raise Projects::ImportService::Error, _('Jira import feature is disabled.') unless Feature.enabled?(:jira_issue_import, project)
         raise Projects::ImportService::Error, _('Jira integration not configured.') unless project.jira_service&.active?
 
-        @jira_project_key = project&.import_data&.becomes(JiraImportData)&.current_project&.key
+        @jira_project_key = project.latest_jira_import&.jira_project_key
         raise Projects::ImportService::Error, _('Unable to find Jira project to import data from.') unless @jira_project_key
 
         @project = project
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index 6333e7923c6..6c98f8f5585 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -90,7 +90,7 @@ module Gitlab
             issues_created_from_gitlab_error_tracking_ui: count(SentryIssue),
             issues_with_associated_zoom_link: count(ZoomMeeting.added_to_issue),
             issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id),
-            issues_with_embedded_grafana_charts_approx: ::Gitlab::GrafanaEmbedUsageData.issue_count,
+            issues_with_embedded_grafana_charts_approx: grafana_embed_usage_data,
             incident_issues: count(::Issue.authored(::User.alert_bot)),
             keys: count(Key),
             label_lists: count(List.label),
@@ -133,6 +133,14 @@ module Gitlab
         { avg_cycle_analytics: {} }
       end
 
+      # rubocop:disable CodeReuse/ActiveRecord
+      def grafana_embed_usage_data
+        count(Issue.joins('JOIN grafana_integrations USING (project_id)')
+          .where("issues.description LIKE '%' || grafana_integrations.grafana_url || '%'")
+          .where(grafana_integrations: { enabled: true }))
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
       def features_usage_data
         features_usage_data_ce
       end
diff --git a/lib/quality/helm3_client.rb b/lib/quality/helm3_client.rb
new file mode 100644
index 00000000000..f5eb0834386
--- /dev/null
+++ b/lib/quality/helm3_client.rb
@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require 'time'
+require_relative '../gitlab/popen' unless defined?(Gitlab::Popen)
+
+module Quality
+  class Helm3Client
+    CommandFailedError = Class.new(StandardError)
+
+    attr_reader :namespace
+
+    RELEASE_JSON_ATTRIBUTES = %w[name revision updated status chart app_version namespace].freeze
+    PAGINATION_SIZE = 256 # Default helm list pagination size
+
+    Release = Struct.new(:name, :revision, :last_update, :status, :chart, :app_version, :namespace) do
+      def revision
+        @revision ||= self[:revision].to_i
+      end
+
+      def status
+        @status ||= self[:status].downcase
+      end
+
+      def last_update
+        @last_update ||= Time.parse(self[:last_update])
+      end
+    end
+
+    # A single page of data and the corresponding page number.
+    Page = Struct.new(:releases, :number)
+
+    def initialize(namespace:, tiller_namespace: nil)
+      @namespace = namespace
+    end
+
+    def releases(args: [])
+      each_release(args)
+    end
+
+    def delete(release_name:)
+      run_command([
+        'uninstall',
+        %(--namespace "#{namespace}"),
+        release_name
+      ])
+    end
+
+    private
+
+    def run_command(command)
+      final_command = ['helm', *command].join(' ')
+      puts "Running command: `#{final_command}`" # rubocop:disable Rails/Output
+
+      result = Gitlab::Popen.popen_with_detail([final_command])
+
+      if result.status.success?
+        result.stdout.chomp.freeze
+      else
+        raise CommandFailedError, "The `#{final_command}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}"
+      end
+    end
+
+    def raw_releases(page, args = [])
+      command = [
+        'list',
+        %(--namespace "#{namespace}"),
+        %(--max #{PAGINATION_SIZE}),
+        %(--offset #{PAGINATION_SIZE * page}),
+        %(--output json),
+        *args
+      ]
+      releases = JSON.parse(run_command(command))
+
+      releases.map do |release|
+        Release.new(*release.values_at(*RELEASE_JSON_ATTRIBUTES))
+      end
+    rescue JSON::ParserError => ex
+      puts "Ignoring this JSON parsing error: #{ex}" # rubocop:disable Rails/Output
+      []
+    end
+
+    # Fetches data from Helm and yields a Page object for every page
+    # of data, without loading all of them into memory.
+    #
+    # method - The Octokit method to use for getting the data.
+    # args - Arguments to pass to the `helm list` command.
+    def each_releases_page(args, &block)
+      return to_enum(__method__, args) unless block_given?
+
+      page = 0
+      final_args = args.dup
+
+      begin
+        collection = raw_releases(page, final_args)
+
+        yield Page.new(collection, page += 1)
+      end while collection.any?
+    end
+
+    # Iterates over all of the releases.
+    #
+    # args - Any arguments to pass to the `helm list` command.
+    def each_release(args, &block)
+      return to_enum(__method__, args) unless block_given?
+
+      each_releases_page(args) do |page|
+        page.releases.each do |release|
+          yield release
+        end
+      end
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index 69d542a4f02..506027aa866 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -80,16 +80,19 @@ namespace :gitlab do
     end
 
     desc 'This adjusts and cleans db/structure.sql - it runs after db:structure:dump'
-    task :clean_structure_sql do
+    task :clean_structure_sql do |task_name|
       structure_file = 'db/structure.sql'
       schema = File.read(structure_file)
 
       File.open(structure_file, 'wb+') do |io|
         Gitlab::Database::SchemaCleaner.new(schema).clean(io)
       end
+
+      # Allow this task to be called multiple times, as happens when running db:migrate:redo
+      Rake::Task[task_name].reenable
     end
 
-    # Inform Rake that gitlab:schema:fix_structure_sql should be run every time rake db:structure:dump is run
+    # Inform Rake that gitlab:schema:clean_structure_sql should be run every time rake db:structure:dump is run
     Rake::Task['db:structure:dump'].enhance do
       Rake::Task['gitlab:db:clean_structure_sql'].invoke
     end