Merge branch 'master' into issue-discussions-refactor

* master: (162 commits)
  Since mysql is not a priority anymore, test it less
  Add container registry and spam logs icons
  Fix different Markdown styles
  Backport to CE for:
  Make new dropdown dividers full width
  Bump GITLAB_SHELL_VERSION and GITALY_VERSION to support unhiding refs
  Install yarn via apt in update guides
  Use long curl options
  Remove monkey-patched Array.prototype.first() and last() methods
  Openshift Getting Started
  35659 Rename Pipelines tab to CI / CD in new navigation
  Don't bother going through an entire Banzai pipeline for empty text
  Add active state for pipelines settings on old nav
  Bump rspec to 3.6.0
  Resolve "Specific Async Script Loading by using a Page Variable"
  Revert "Merge branch 'rs-warm-capybara-only-in-ci' into 'master'"
  another rubocop style fix
  Use mixin for new dropdown style
  Migrate Repository#last_commit_for_path to Gitaly
  Migrate blame loading to Gitaly
  ...

10 files changed, 243 insertions, 47 deletions

diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb
new file mode 100644
index 00000000000..0fbc6b70989
--- /dev/null
+++ b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb
@@ -0,0 +1,107 @@
+module Gitlab
+  module BackgroundMigration
+    class DeserializeMergeRequestDiffsAndCommits
+      attr_reader :diff_ids, :commit_rows, :file_rows
+
+      class MergeRequestDiff < ActiveRecord::Base
+        self.table_name = 'merge_request_diffs'
+      end
+
+      BUFFER_ROWS = 1000
+
+      def perform(start_id, stop_id)
+        merge_request_diffs = MergeRequestDiff
+                               .select(:id, :st_commits, :st_diffs)
+                               .where('st_commits IS NOT NULL OR st_diffs IS NOT NULL')
+                               .where(id: start_id..stop_id)
+
+        reset_buffers!
+
+        merge_request_diffs.each do |merge_request_diff|
+          commits, files = single_diff_rows(merge_request_diff)
+
+          diff_ids << merge_request_diff.id
+          commit_rows.concat(commits)
+          file_rows.concat(files)
+
+          if diff_ids.length > BUFFER_ROWS ||
+              commit_rows.length > BUFFER_ROWS ||
+              file_rows.length > BUFFER_ROWS
+
+            flush_buffers!
+          end
+        end
+
+        flush_buffers!
+      end
+
+      private
+
+      def reset_buffers!
+        @diff_ids = []
+        @commit_rows = []
+        @file_rows = []
+      end
+
+      def flush_buffers!
+        if diff_ids.any?
+          MergeRequestDiff.transaction do
+            Gitlab::Database.bulk_insert('merge_request_diff_commits', commit_rows)
+            Gitlab::Database.bulk_insert('merge_request_diff_files', file_rows)
+
+            MergeRequestDiff.where(id: diff_ids).update_all(st_commits: nil, st_diffs: nil)
+          end
+        end
+
+        reset_buffers!
+      end
+
+      def single_diff_rows(merge_request_diff)
+        sha_attribute = Gitlab::Database::ShaAttribute.new
+        commits = YAML.load(merge_request_diff.st_commits) rescue []
+
+        commit_rows = commits.map.with_index do |commit, index|
+          commit_hash = commit.to_hash.with_indifferent_access.except(:parent_ids)
+          sha = commit_hash.delete(:id)
+
+          commit_hash.merge(
+            merge_request_diff_id: merge_request_diff.id,
+            relative_order: index,
+            sha: sha_attribute.type_cast_for_database(sha)
+          )
+        end
+
+        diffs = YAML.load(merge_request_diff.st_diffs) rescue []
+        diffs = [] unless valid_raw_diffs?(diffs)
+
+        file_rows = diffs.map.with_index do |diff, index|
+          diff_hash = diff.to_hash.with_indifferent_access.merge(
+            binary: false,
+            merge_request_diff_id: merge_request_diff.id,
+            relative_order: index
+          )
+
+          # Compatibility with old diffs created with Psych.
+          diff_hash.tap do |hash|
+            diff_text = hash[:diff]
+
+            if diff_text.encoding == Encoding::BINARY && !diff_text.ascii_only?
+              hash[:binary] = true
+              hash[:diff] = [diff_text].pack('m0')
+            end
+          end
+        end
+
+        [commit_rows, file_rows]
+      end
+
+      # Unlike MergeRequestDiff#valid_raw_diff?, don't count Rugged objects as
+      # valid, because we don't render them usefully anyway.
+      def valid_raw_diffs?(diffs)
+        return false unless diffs.respond_to?(:each)
+
+        diffs.all? { |diff| diff.is_a?(Hash) }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb
index 31579e94a87..8eea33b9ab5 100644
--- a/lib/gitlab/email/handler/create_note_handler.rb
+++ b/lib/gitlab/email/handler/create_note_handler.rb
@@ -15,7 +15,6 @@ module Gitlab
 
         def execute
           raise SentNotificationNotFoundError unless sent_notification
-          raise AutoGeneratedEmailError if mail.header.to_s =~ /auto-(generated|replied)/
 
           validate_permission!(:create_note)
 
diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb
index 0d6b08b5d29..c8f4591d060 100644
--- a/lib/gitlab/email/receiver.rb
+++ b/lib/gitlab/email/receiver.rb
@@ -26,6 +26,9 @@ module Gitlab
         raise EmptyEmailError if @raw.blank?
 
         mail = build_mail
+
+        ignore_auto_submitted!(mail)
+
         mail_key = extract_mail_key(mail)
         handler = Handler.for(mail, mail_key)
 
@@ -87,6 +90,16 @@ module Gitlab
           break key if key
         end
       end
+
+      def ignore_auto_submitted!(mail)
+        # Mail::Header#[] is case-insensitive
+        auto_submitted = mail.header['Auto-Submitted']&.value
+
+        # Mail::Field#value would strip leading and trailing whitespace
+        raise AutoGeneratedEmailError if
+          # See also https://tools.ietf.org/html/rfc3834
+          auto_submitted && auto_submitted != 'no'
+      end
     end
   end
 end
diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb
index 0deaab01b5b..8dbe25e55f6 100644
--- a/lib/gitlab/git/blame.rb
+++ b/lib/gitlab/git/blame.rb
@@ -1,5 +1,3 @@
-# Gitaly note: JV: needs 1 RPC for #load_blame.
-
 module Gitlab
   module Git
     class Blame
@@ -26,15 +24,29 @@ module Gitlab
 
       private
 
-      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/376
       def load_blame
-        cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{@repo.path} blame -p #{@sha} -- #{@path})
-        # Read in binary mode to ensure ASCII-8BIT
-        raw_output = IO.popen(cmd, 'rb') {|io| io.read }
+        raw_output = @repo.gitaly_migrate(:blame) do |is_enabled|
+          if is_enabled
+            load_blame_by_gitaly
+          else
+            load_blame_by_shelling_out
+          end
+        end
+
         output = encode_utf8(raw_output)
         process_raw_blame output
       end
 
+      def load_blame_by_gitaly
+        @repo.gitaly_commit_client.raw_blame(@sha, @path)
+      end
+
+      def load_blame_by_shelling_out
+        cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{@repo.path} blame -p #{@sha} -- #{@path})
+        # Read in binary mode to ensure ASCII-8BIT
+        IO.popen(cmd, 'rb') {|io| io.read }
+      end
+
       def process_raw_blame(output)
         lines, final = [], []
         info, commits = {}, {}
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 88529ba2c47..734aed8fbc1 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -300,17 +300,14 @@ module Gitlab
         raw_log(options).map { |c| Commit.decorate(c) }
       end
 
-      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/382
       def count_commits(options)
-        cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list]
-        cmd << "--after=#{options[:after].iso8601}" if options[:after]
-        cmd << "--before=#{options[:before].iso8601}" if options[:before]
-        cmd += %W[--count #{options[:ref]}]
-        cmd += %W[-- #{options[:path]}] if options[:path].present?
-
-        raw_output = IO.popen(cmd) { |io| io.read }
-
-        raw_output.to_i
+        gitaly_migrate(:count_commits) do |is_enabled|
+          if is_enabled
+            count_commits_by_gitaly(options)
+          else
+            count_commits_by_shelling_out(options)
+          end
+        end
       end
 
       def sha_from_ref(ref)
@@ -353,6 +350,13 @@ module Gitlab
         rugged.merge_base(from, to)
       end
 
+      # Gitaly note: JV: check gitlab-ee before removing this method.
+      def rugged_is_ancestor?(ancestor_id, descendant_id)
+        return false if ancestor_id.nil? || descendant_id.nil?
+
+        merge_base_commit(ancestor_id, descendant_id) == ancestor_id
+      end
+
       # Returns true is +from+ is direct ancestor to +to+, otherwise false
       def is_ancestor?(from, to)
         gitaly_commit_client.is_ancestor(from, to)
@@ -679,6 +683,14 @@ module Gitlab
         @gitaly_repository_client ||= Gitlab::GitalyClient::RepositoryService.new(self)
       end
 
+      def gitaly_migrate(method, &block)
+        Gitlab::GitalyClient.migrate(method, &block)
+      rescue GRPC::NotFound => e
+        raise NoRepository.new(e)
+      rescue GRPC::BadStatus => e
+        raise CommandError.new(e)
+      end
+
       private
 
       # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'.
@@ -994,16 +1006,29 @@ module Gitlab
         end.sort_by(&:name)
       end
 
+      def last_commit_for_path_by_rugged(sha, path)
+        sha = last_commit_id_for_path(sha, path)
+        commit(sha)
+      end
+
       def tags_from_gitaly
         gitaly_ref_client.tags
       end
 
-      def gitaly_migrate(method, &block)
-        Gitlab::GitalyClient.migrate(method, &block)
-      rescue GRPC::NotFound => e
-        raise NoRepository.new(e)
-      rescue GRPC::BadStatus => e
-        raise CommandError.new(e)
+      def count_commits_by_gitaly(options)
+        gitaly_commit_client.commit_count(options[:ref], options)
+      end
+
+      def count_commits_by_shelling_out(options)
+        cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list]
+        cmd << "--after=#{options[:after].iso8601}" if options[:after]
+        cmd << "--before=#{options[:before].iso8601}" if options[:before]
+        cmd += %W[--count #{options[:ref]}]
+        cmd += %W[-- #{options[:path]}] if options[:path].present?
+
+        raw_output = IO.popen(cmd) { |io| io.read }
+
+        raw_output.to_i
       end
     end
   end
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index a834781b1f1..ac6817e6d0e 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -85,15 +85,32 @@ module Gitlab
         end
       end
 
-      def commit_count(ref)
+      def commit_count(ref, options = {})
         request = Gitaly::CountCommitsRequest.new(
           repository: @gitaly_repo,
           revision: ref
         )
+        request.after = Google::Protobuf::Timestamp.new(seconds: options[:after].to_i) if options[:after].present?
+        request.before = Google::Protobuf::Timestamp.new(seconds: options[:before].to_i) if options[:before].present?
+        request.path = options[:path] if options[:path].present?
 
         GitalyClient.call(@repository.storage, :commit_service, :count_commits, request).count
       end
 
+      def last_commit_for_path(revision, path)
+        request = Gitaly::LastCommitForPathRequest.new(
+          repository: @gitaly_repo,
+          revision: revision.force_encoding(Encoding::ASCII_8BIT),
+          path: path.to_s.force_encoding(Encoding::ASCII_8BIT)
+        )
+
+        gitaly_commit = GitalyClient.call(@repository.storage, :commit_service, :last_commit_for_path, request).commit
+        return unless gitaly_commit
+
+        commit = GitalyClient::Commit.new(@repository, gitaly_commit)
+        Gitlab::Git::Commit.new(commit)
+      end
+
       def between(from, to)
         request = Gitaly::CommitsBetweenRequest.new(
           repository: @gitaly_repo,
@@ -125,6 +142,17 @@ module Gitlab
         response.languages.map { |l| { value: l.share.round(2), label: l.name, color: l.color, highlight: l.color } }
       end
 
+      def raw_blame(revision, path)
+        request = Gitaly::RawBlameRequest.new(
+          repository: @gitaly_repo,
+          revision: revision,
+          path: path
+        )
+
+        response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request)
+        response.reduce("") { |memo, msg| memo << msg.data }
+      end
+
       private
 
       def commit_diff_request_params(commit, options = {})
diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb
index 3f2bbd9f6a6..e8330917e91 100644
--- a/lib/gitlab/o_auth/user.rb
+++ b/lib/gitlab/o_auth/user.rb
@@ -166,12 +166,17 @@ module Gitlab
         username ||= auth_hash.username
         email ||= auth_hash.email
 
+        valid_username = ::Namespace.clean_path(username)
+
+        uniquify = Uniquify.new
+        valid_username = uniquify.string(valid_username) { |s| !DynamicPathValidator.valid_user_path?(s) }
+
         name = auth_hash.name
-        name = ::Namespace.clean_path(username) if name.strip.empty?
+        name = valid_username if name.strip.empty?
 
         {
           name:                       name,
-          username:                   ::Namespace.clean_path(username),
+          username:                   valid_username,
           email:                      email,
           password:                   auth_hash.password,
           password_confirmation:      auth_hash.password,
diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb
index 67c69d9ccf3..69d055c901c 100644
--- a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb
+++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb
@@ -6,14 +6,13 @@ module Gitlab
 
         def query(deployment_id)
           Deployment.find_by(id: deployment_id).try do |deployment|
-            query_context = {
-              environment_slug: deployment.environment.slug,
-              environment_filter: %{container_name!="POD",environment="#{deployment.environment.slug}"},
-              timeframe_start: (deployment.created_at - 30.minutes).to_f,
-              timeframe_end: (deployment.created_at + 30.minutes).to_f
-            }
-
-            query_metrics(query_context)
+            query_metrics(
+              common_query_context(
+                deployment.environment,
+                timeframe_start: (deployment.created_at - 30.minutes).to_f,
+                timeframe_end: (deployment.created_at + 30.minutes).to_f
+              )
+            )
           end
         end
       end
diff --git a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb
index b5a679ddd79..db4708b22e4 100644
--- a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb
+++ b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb
@@ -6,14 +6,9 @@ module Gitlab
 
         def query(environment_id)
           Environment.find_by(id: environment_id).try do |environment|
-            query_context = {
-              environment_slug: environment.slug,
-              environment_filter: %{container_name!="POD",environment="#{environment.slug}"},
-              timeframe_start: 8.hours.ago.to_f,
-              timeframe_end: Time.now.to_f
-            }
-
-            query_metrics(query_context)
+            query_metrics(
+              common_query_context(environment, timeframe_start: 8.hours.ago.to_f, timeframe_end: Time.now.to_f)
+            )
           end
         end
       end
diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb
index e44be770544..7ac6162b54d 100644
--- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb
+++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb
@@ -42,15 +42,18 @@ module Gitlab
         end
 
         def process_query(context, query)
-          query_with_result = query.dup
+          query = query.dup
           result =
             if query.key?(:query_range)
-              client_query_range(query[:query_range] % context, start: context[:timeframe_start], stop: context[:timeframe_end])
+              query[:query_range] %= context
+              client_query_range(query[:query_range], start: context[:timeframe_start], stop: context[:timeframe_end])
             else
-              client_query(query[:query] % context, time: context[:timeframe_end])
+              query[:query] %= context
+              client_query(query[:query], time: context[:timeframe_end])
             end
-          query_with_result[:result] = result&.map(&:deep_symbolize_keys)
-          query_with_result
+
+          query[:result] = result&.map(&:deep_symbolize_keys)
+          query
         end
 
         def available_metrics
@@ -67,6 +70,16 @@ module Gitlab
 
           result.select { |group| group.metrics.any? }
         end
+
+        def common_query_context(environment, timeframe_start:, timeframe_end:)
+          {
+            timeframe_start: timeframe_start,
+            timeframe_end: timeframe_end,
+            ci_environment_slug: environment.slug,
+            kube_namespace: environment.project.kubernetes_service&.actual_namespace || '',
+            environment_filter: %{container_name!="POD",environment="#{environment.slug}"}
+          }
+        end
       end
     end
   end