Merge remote-tracking branch 'origin/active-record-each-batch' into fix/gb/stage-id-reference-background-migration

* origin/active-record-each-batch: (59 commits)
  Added EachBatch for iterating tables in batches
  Extend MR tabs a bit to cover up the avatar holder and collapse icon on scroll
  Update VERSION to 9.4.0-pre.
  Add CHANGELOG
  Fix some N+1 queries in the GET /projects API
  Don't show auxiliary blob viewer for README when there is no wiki
  Improve & fix the performance bar UI and behavior
  Remove orphaned haml files
  Fixed CHANGELOG.md for 9.3.4 release
  Add table for merge request commits
  34727 Remove two columned layout from project member settings
  Just draw :legacy_builds
  Re-enable polling for environments
  Cleanup minor UX issues in the performance dashboard
  Upgrade GitLab Workhorse to v2.3.0
  Added test for the chart legend
  Use correct field for label name, fix default for unit to be blank
  Fix shorter route helpers in production environment
  Encode certificate-authority-data in base64
  Revert "Merge branch 'winh-mr-widget-no-pipeline' into 'master'"
  ...

16 files changed, 223 insertions, 105 deletions

diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 99eda3b0c4b..94168fa4ebc 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -503,12 +503,20 @@ module API
     class ProjectWithAccess < Project
       expose :permissions do
         expose :project_access, using: Entities::ProjectAccess do |project, options|
-          project.project_members.find_by(user_id: options[:current_user].id)
+          if options.key?(:project_members)
+            (options[:project_members] || []).find { |member| member.source_id == project.id }
+          else
+            project.project_members.find_by(user_id: options[:current_user].id)
+          end
         end
 
         expose :group_access, using: Entities::GroupAccess do |project, options|
           if project.group
-            project.group.group_members.find_by(user_id: options[:current_user].id)
+            if options.key?(:group_members)
+              (options[:group_members] || []).find { |member| member.source_id == project.namespace_id }
+            else
+              project.group.group_members.find_by(user_id: options[:current_user].id)
+            end
           end
         end
       end
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index a2a661b205c..0f4791841d2 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -268,6 +268,7 @@ module API
       finder_params[:visibility_level] = Gitlab::VisibilityLevel.level_value(params[:visibility]) if params[:visibility]
       finder_params[:archived] = params[:archived]
       finder_params[:search] = params[:search] if params[:search]
+      finder_params[:user] = params.delete(:user) if params[:user]
       finder_params
     end
 
@@ -313,7 +314,7 @@ module API
 
     def present_artifacts!(artifacts_file)
       return not_found! unless artifacts_file.exists?
-  
+
       if artifacts_file.file_storage?
         present_file!(artifacts_file.path, artifacts_file.filename)
       else
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 35733ac7711..c459257158d 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -36,61 +36,86 @@ module API
       params :statistics_params do
         optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'
       end
-    end
 
-    resource :projects do
-      helpers do
-        params :collection_params do
-          use :sort_params
-          use :filter_params
-          use :pagination
-
-          optional :simple, type: Boolean, default: false,
-                            desc: 'Return only the ID, URL, name, and path of each project'
-        end
+      params :collection_params do
+        use :sort_params
+        use :filter_params
+        use :pagination
 
-        params :sort_params do
-          optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at],
-                              default: 'created_at', desc: 'Return projects ordered by field'
-          optional :sort, type: String, values: %w[asc desc], default: 'desc',
-                          desc: 'Return projects sorted in ascending and descending order'
-        end
+        optional :simple, type: Boolean, default: false,
+                          desc: 'Return only the ID, URL, name, and path of each project'
+      end
 
-        params :filter_params do
-          optional :archived, type: Boolean, default: false, desc: 'Limit by archived status'
-          optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values,
-                                desc: 'Limit by visibility'
-          optional :search, type: String, desc: 'Return list of projects matching the search criteria'
-          optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user'
-          optional :starred, type: Boolean, default: false, desc: 'Limit by starred status'
-          optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of'
-          optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature'
-          optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature'
-        end
+      params :sort_params do
+        optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at],
+                            default: 'created_at', desc: 'Return projects ordered by field'
+        optional :sort, type: String, values: %w[asc desc], default: 'desc',
+                        desc: 'Return projects sorted in ascending and descending order'
+      end
 
-        params :create_params do
-          optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.'
-          optional :import_url, type: String, desc: 'URL from which the project is imported'
-        end
+      params :filter_params do
+        optional :archived, type: Boolean, default: false, desc: 'Limit by archived status'
+        optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values,
+                              desc: 'Limit by visibility'
+        optional :search, type: String, desc: 'Return list of projects matching the search criteria'
+        optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user'
+        optional :starred, type: Boolean, default: false, desc: 'Limit by starred status'
+        optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of'
+        optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature'
+        optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature'
+      end
+
+      params :create_params do
+        optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.'
+        optional :import_url, type: String, desc: 'URL from which the project is imported'
+      end
 
-        def present_projects(options = {})
-          projects = ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute
-          projects = reorder_projects(projects)
-          projects = projects.with_statistics if params[:statistics]
-          projects = projects.with_issues_enabled if params[:with_issues_enabled]
-          projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled]
-
-          options = options.reverse_merge(
-            with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails,
-            statistics: params[:statistics],
-            current_user: current_user
-          )
-          options[:with] = Entities::BasicProjectDetails if params[:simple]
-
-          present paginate(projects), options
+      def present_projects(options = {})
+        projects = ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute
+        projects = reorder_projects(projects)
+        projects = projects.with_statistics if params[:statistics]
+        projects = projects.with_issues_enabled if params[:with_issues_enabled]
+        projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled]
+
+        if current_user
+          projects = projects.includes(:route, :taggings, namespace: :route)
+          project_members = current_user.project_members
+          group_members = current_user.group_members
         end
+
+        options = options.reverse_merge(
+          with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails,
+          statistics: params[:statistics],
+          project_members: project_members,
+          group_members: group_members,
+          current_user: current_user
+        )
+        options[:with] = Entities::BasicProjectDetails if params[:simple]
+
+        present paginate(projects), options
       end
+    end
 
+    resource :users, requirements: { user_id: %r{[^/]+} } do
+      desc 'Get a user projects' do
+        success Entities::BasicProjectDetails
+      end
+      params do
+        requires :user_id, type: String, desc: 'The ID or username of the user'
+        use :collection_params
+        use :statistics_params
+      end
+      get ":user_id/projects" do
+        user = find_user(params[:user_id])
+        not_found!('User') unless user
+
+        params[:user] = user
+
+        present_projects
+      end
+    end
+
+    resource :projects do
       desc 'Get a list of visible projects for authenticated user' do
         success Entities::BasicProjectDetails
       end
diff --git a/lib/api/variables.rb b/lib/api/variables.rb
index 10374995497..7fa528fb2d3 100644
--- a/lib/api/variables.rb
+++ b/lib/api/variables.rb
@@ -45,7 +45,9 @@ module API
         optional :protected, type: String, desc: 'Whether the variable is protected'
       end
       post ':id/variables' do
-        variable = user_project.variables.create(declared_params(include_missing: false))
+        variable_params = declared_params(include_missing: false)
+
+        variable = user_project.variables.create(variable_params)
 
         if variable.valid?
           present variable, with: Entities::Variable
@@ -67,7 +69,9 @@ module API
 
         return not_found!('Variable') unless variable
 
-        if variable.update(declared_params(include_missing: false).except(:key))
+        variable_params = declared_params(include_missing: false).except(:key)
+
+        if variable.update(variable_params)
           present variable, with: Entities::Variable
         else
           render_validation_error!(variable)
diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb
index 6640168bfa2..a6f8650ed3d 100644
--- a/lib/banzai/filter/reference_filter.rb
+++ b/lib/banzai/filter/reference_filter.rb
@@ -30,6 +30,8 @@ module Banzai
         attributes = attributes.reject { |_, v| v.nil? }
 
         attributes[:reference_type] ||= self.class.reference_type
+        attributes[:container] ||= 'body'
+        attributes[:placement] ||= 'bottom'
         attributes.delete(:original) if context[:no_original_data]
         attributes.map do |key, value|
           %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}")
diff --git a/lib/gitlab/cycle_analytics/metrics_tables.rb b/lib/gitlab/cycle_analytics/metrics_tables.rb
index 9d25ef078e8..f5d08c0b658 100644
--- a/lib/gitlab/cycle_analytics/metrics_tables.rb
+++ b/lib/gitlab/cycle_analytics/metrics_tables.rb
@@ -13,6 +13,10 @@ module Gitlab
         MergeRequestDiff.arel_table
       end
 
+      def mr_diff_commits_table
+        MergeRequestDiffCommit.arel_table
+      end
+
       def mr_closing_issues_table
         MergeRequestsClosingIssues.arel_table
       end
diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb
index 7d342a2d2cb..b260822788d 100644
--- a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb
+++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb
@@ -2,40 +2,59 @@ module Gitlab
   module CycleAnalytics
     class PlanEventFetcher < BaseEventFetcher
       def initialize(*args)
-        @projections = [mr_diff_table[:st_commits].as('commits'),
+        @projections = [mr_diff_table[:id],
+                        mr_diff_table[:st_commits],
                         issue_metrics_table[:first_mentioned_in_commit_at]]
 
         super(*args)
       end
 
       def events_query
-        base_query.join(mr_diff_table).on(mr_diff_table[:merge_request_id].eq(mr_table[:id]))
+        base_query
+          .join(mr_diff_table)
+          .on(mr_diff_table[:merge_request_id].eq(mr_table[:id]))
 
         super
       end
 
       private
 
+      def merge_request_diff_commits
+        @merge_request_diff_commits ||=
+          MergeRequestDiffCommit
+            .where(merge_request_diff_id: event_result.map { |event| event['id'] })
+            .group_by(&:merge_request_diff_id)
+      end
+
       def serialize(event)
-        st_commit = first_time_reference_commit(event.delete('commits'), event)
+        commit = first_time_reference_commit(event)
 
-        return unless st_commit
+        return unless commit
 
-        serialize_commit(event, st_commit, query)
+        serialize_commit(event, commit, query)
       end
 
-      def first_time_reference_commit(commits, event)
+      def first_time_reference_commit(event)
+        return nil unless event && merge_request_diff_commits
+
+        commits =
+          if event['st_commits'].present?
+            YAML.load(event['st_commits'])
+          else
+            merge_request_diff_commits[event['id'].to_i]
+          end
+
         return nil if commits.blank?
 
-        YAML.load(commits).find do |commit|
+        commits.find do |commit|
           next unless commit[:committed_date] && event['first_mentioned_in_commit_at']
 
           commit[:committed_date].to_i == DateTime.parse(event['first_mentioned_in_commit_at'].to_s).to_i
         end
       end
 
-      def serialize_commit(event, st_commit, query)
-        commit = Commit.new(Gitlab::Git::Commit.new(st_commit), @project)
+      def serialize_commit(event, commit, query)
+        commit = Commit.new(Gitlab::Git::Commit.new(commit.to_hash), @project)
 
         AnalyticsCommitSerializer.new(project: @project, total_time: event['total_time']).represent(commit)
       end
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index 1860352c96d..c8ad3a7a5e0 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -27,6 +27,7 @@ project_tree:
       - :author
       - :events
     - merge_request_diff:
+      - :merge_request_diff_commits
       - :merge_request_diff_files
     - :events
     - :timelogs
diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb
index c56c1a4322f..cdbdfa10d0e 100644
--- a/lib/gitlab/kubernetes.rb
+++ b/lib/gitlab/kubernetes.rb
@@ -76,5 +76,44 @@ module Gitlab
 
       url.to_s
     end
+
+    def to_kubeconfig(url:, namespace:, token:, ca_pem: nil)
+      config = {
+        apiVersion: 'v1',
+        clusters: [
+          name: 'gitlab-deploy',
+          cluster: {
+            server: url
+          }
+        ],
+        contexts: [
+          name: 'gitlab-deploy',
+          context: {
+            cluster: 'gitlab-deploy',
+            namespace: namespace,
+            user: 'gitlab-deploy'
+          }
+        ],
+        'current-context': 'gitlab-deploy',
+        kind: 'Config',
+        users: [
+          {
+            name: 'gitlab-deploy',
+            user: { token: token }
+          }
+        ]
+      }
+
+      kubeconfig_embed_ca_pem(config, ca_pem) if ca_pem
+
+      config.deep_stringify_keys
+    end
+
+    private
+
+    def kubeconfig_embed_ca_pem(config, ca_pem)
+      cluster = config.dig(:clusters, 0, :cluster)
+      cluster[:'certificate-authority-data'] = Base64.encode64(ca_pem)
+    end
   end
 end
diff --git a/lib/gitlab/otp_key_rotator.rb b/lib/gitlab/otp_key_rotator.rb
index 0d541935bc6..22332474945 100644
--- a/lib/gitlab/otp_key_rotator.rb
+++ b/lib/gitlab/otp_key_rotator.rb
@@ -34,7 +34,7 @@ module Gitlab
 
       write_csv do |csv|
         ActiveRecord::Base.transaction do
-          User.with_two_factor.in_batches do |relation|
+          User.with_two_factor.in_batches do |relation| # rubocop: disable Cop/InBatches
             rows = relation.pluck(:id, :encrypted_otp_secret, :encrypted_otp_secret_iv, :encrypted_otp_secret_salt)
             rows.each do |row|
               user = %i[id ciphertext iv salt].zip(row).to_h
diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb
index 163a40ad306..a4c8a77129e 100644
--- a/lib/gitlab/performance_bar.rb
+++ b/lib/gitlab/performance_bar.rb
@@ -1,7 +1,7 @@
 module Gitlab
   module PerformanceBar
     def self.enabled?
-      Feature.enabled?('gitlab_performance_bar')
+      Rails.env.development? || Feature.enabled?('gitlab_performance_bar')
     end
   end
 end
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index 057f32eaef7..c1ee20b6977 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -30,8 +30,12 @@ module Gitlab
       @container_repository_regex ||= %r{\A[a-z0-9]+(?:[-._/][a-z0-9]+)*\Z}
     end
 
+    def environment_name_regex_chars
+      'a-zA-Z0-9_/\\$\\{\\}\\. -'
+    end
+
     def environment_name_regex
-      @environment_name_regex ||= /\A[a-zA-Z0-9_\\\/\${}. -]+\z/.freeze
+      @environment_name_regex ||= /\A[#{environment_name_regex_chars}]+\z/.freeze
     end
 
     def environment_name_regex_message
diff --git a/lib/gitlab/routes/legacy_builds.rb b/lib/gitlab/routes/legacy_builds.rb
deleted file mode 100644
index 36d1a8a6f64..00000000000
--- a/lib/gitlab/routes/legacy_builds.rb
+++ /dev/null
@@ -1,36 +0,0 @@
-module Gitlab
-  module Routes
-    class LegacyBuilds
-      def initialize(map)
-        @map = map
-      end
-
-      def draw
-        @map.instance_eval do
-          resources :builds, only: [:index, :show], constraints: { id: /\d+/ } do
-            collection do
-              resources :artifacts, only: [], controller: 'build_artifacts' do
-                collection do
-                  get :latest_succeeded,
-                    path: '*ref_name_and_path',
-                    format: false
-                end
-              end
-            end
-
-            member do
-              get :raw
-            end
-
-            resource :artifacts, only: [], controller: 'build_artifacts' do
-              get :download
-              get :browse, path: 'browse(/*path)', format: false
-              get :file, path: 'file/*path', format: false
-              get :raw, path: 'raw/*path', format: false
-            end
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/sql/glob.rb b/lib/gitlab/sql/glob.rb
new file mode 100644
index 00000000000..5e89e12b2b1
--- /dev/null
+++ b/lib/gitlab/sql/glob.rb
@@ -0,0 +1,22 @@
+module Gitlab
+  module SQL
+    module Glob
+      extend self
+
+      # Convert a simple glob pattern with wildcard (*) to SQL LIKE pattern
+      # with SQL expression
+      def to_like(pattern)
+        <<~SQL
+          REPLACE(REPLACE(REPLACE(#{pattern},
+                                  #{q('%')}, #{q('\\%')}),
+                          #{q('_')}, #{q('\\_')}),
+                  #{q('*')}, #{q('%')})
+        SQL
+      end
+
+      def q(string)
+        ActiveRecord::Base.connection.quote(string)
+      end
+    end
+  end
+end
diff --git a/lib/peek/rblineprof/custom_controller_helpers.rb b/lib/peek/rblineprof/custom_controller_helpers.rb
index 99f9c2c9b04..7cfe76b7b71 100644
--- a/lib/peek/rblineprof/custom_controller_helpers.rb
+++ b/lib/peek/rblineprof/custom_controller_helpers.rb
@@ -41,9 +41,14 @@ module Peek
             ]
           end.sort_by{ |a,b,c,d,e,f| -f }
 
-          output = ''
-          per_file.each do |file_name, lines, file_wall, file_cpu, file_idle, file_sort|
+          output = "<div class='modal-dialog modal-full'><div class='modal-content'>"
+          output << "<div class='modal-header'>"
+          output << "<button class='close btn btn-link btn-sm' type='button' data-dismiss='modal'>X</button>"
+          output << "<h4>Line profiling: #{human_description(params[:lineprofiler])}</h4>"
+          output << "</div>"
+          output << "<div class='modal-body'>"
 
+          per_file.each do |file_name, lines, file_wall, file_cpu, file_idle, file_sort|
             output << "<div class='peek-rblineprof-file'><div class='heading'>"
 
             show_src = file_sort > min
@@ -86,11 +91,32 @@ module Peek
             output << "</div></div>" # .data then .peek-rblineprof-file
           end
 
-          response.body += "<div class='peek-rblineprof-modal' id='line-profile'>#{output}</div>".html_safe
+          output << "</div></div></div>"
+
+          response.body += "<div class='modal' id='modal-peek-line-profile' tabindex=-1>#{output}</div>".html_safe
         end
 
         ret
       end
+
+      private
+
+      def human_description(lineprofiler_param)
+        case lineprofiler_param
+        when 'app'
+          'app/ & lib/'
+        when 'views'
+          'app/view/'
+        when 'gems'
+          'vendor/gems'
+        when 'all'
+          'everything in Rails.root'
+        when 'stdlib'
+          'everything in the Ruby standard library'
+        else
+          'app/, config/, lib/, vendor/ & plugin/'
+        end
+      end
     end
   end
 end
diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake
index e3883278886..e9fb6a008b0 100644
--- a/lib/tasks/gitlab/info.rake
+++ b/lib/tasks/gitlab/info.rake
@@ -42,8 +42,7 @@ namespace :gitlab do
       http_clone_url = project.http_url_to_repo
       ssh_clone_url  = project.ssh_url_to_repo
 
-      omniauth_providers = Gitlab.config.omniauth.providers
-      omniauth_providers.map! { |provider| provider['name'] }
+      omniauth_providers = Gitlab.config.omniauth.providers.map { |provider| provider['name'] }
 
       puts ""
       puts "GitLab information".color(:yellow)