Merge branch 'master' into refactor-clusters

12 files changed, 233 insertions, 36 deletions

diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb
index c4c60d1dfee..0ea534a5fd0 100644
--- a/lib/gitlab/ee_compat_check.rb
+++ b/lib/gitlab/ee_compat_check.rb
@@ -2,8 +2,8 @@
 module Gitlab
   # Checks if a set of migrations requires downtime or not.
   class EeCompatCheck
-    DEFAULT_CE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ce.git'.freeze
-    EE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze
+    DEFAULT_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-ce'.freeze
+    EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze
     CHECK_DIR = Rails.root.join('ee_compat_check')
     IGNORED_FILES_REGEX = /(VERSION|CHANGELOG\.md:\d+)/.freeze
     PLEASE_READ_THIS_BANNER = %Q{
@@ -17,14 +17,16 @@ module Gitlab
       ============================================================\n
     }.freeze
 
-    attr_reader :ee_repo_dir, :patches_dir, :ce_repo, :ce_branch, :ee_branch_found
-    attr_reader :failed_files
+    attr_reader :ee_repo_dir, :patches_dir, :ce_project_url, :ce_repo_url, :ce_branch, :ee_branch_found
+    attr_reader :job_id, :failed_files
 
-    def initialize(branch:, ce_repo: DEFAULT_CE_REPO)
+    def initialize(branch:, ce_project_url: DEFAULT_CE_PROJECT_URL, job_id: nil)
       @ee_repo_dir = CHECK_DIR.join('ee-repo')
       @patches_dir = CHECK_DIR.join('patches')
       @ce_branch = branch
-      @ce_repo = ce_repo
+      @ce_project_url = ce_project_url
+      @ce_repo_url = "#{ce_project_url}.git"
+      @job_id = job_id
     end
 
     def check
@@ -59,8 +61,8 @@ module Gitlab
         step("#{ee_repo_dir} already exists")
       else
         step(
-          "Cloning #{EE_REPO} into #{ee_repo_dir}",
-          %W[git clone --branch master --single-branch --depth=200 #{EE_REPO} #{ee_repo_dir}]
+          "Cloning #{EE_REPO_URL} into #{ee_repo_dir}",
+          %W[git clone --branch master --single-branch --depth=200 #{EE_REPO_URL} #{ee_repo_dir}]
         )
       end
     end
@@ -132,7 +134,7 @@ module Gitlab
     def check_patch(patch_path)
       step("Checking out master", %w[git checkout master])
       step("Resetting to latest master", %w[git reset --hard origin/master])
-      step("Fetching CE/#{ce_branch}", %W[git fetch #{ce_repo} #{ce_branch}])
+      step("Fetching CE/#{ce_branch}", %W[git fetch #{ce_repo_url} #{ce_branch}])
       step(
         "Checking if #{patch_path} applies cleanly to EE/master",
         # Don't use --check here because it can result in a 0-exit status even
@@ -237,7 +239,7 @@ module Gitlab
     end
 
     def patch_url
-      "https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/#{ENV['CI_JOB_ID']}/artifacts/raw/ee_compat_check/patches/#{ce_patch_name}"
+      "#{ce_project_url}/-/jobs/#{job_id}/artifacts/raw/ee_compat_check/patches/#{ce_patch_name}"
     end
 
     def step(desc, cmd = nil)
@@ -304,7 +306,7 @@ module Gitlab
           # In the EE repo
           $ git fetch origin
           $ git checkout -b #{ee_branch_prefix} origin/master
-          $ git fetch #{ce_repo} #{ce_branch}
+          $ git fetch #{ce_repo_url} #{ce_branch}
           $ git cherry-pick SHA # Repeat for all the commits you want to pick
 
           You can squash the `#{ce_branch}` commits into a single "Port of #{ce_branch} to EE" commit.
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index a4336facee5..cc6c7609ec7 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -12,6 +12,12 @@ module Gitlab
       # blob data should use load_all_data!.
       MAX_DATA_DISPLAY_SIZE = 10.megabytes
 
+      # These limits are used as a heuristic to ignore files which can't be LFS
+      # pointers. The format of these is described in
+      # https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md#the-pointer
+      LFS_POINTER_MIN_SIZE = 120.bytes
+      LFS_POINTER_MAX_SIZE = 200.bytes
+
       attr_accessor :name, :path, :size, :data, :mode, :id, :commit_id, :loaded_size, :binary
 
       class << self
@@ -30,16 +36,7 @@ module Gitlab
             if is_enabled
               Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE)
             else
-              blob = repository.lookup(sha)
-
-              next unless blob.is_a?(Rugged::Blob)
-
-              new(
-                id: blob.oid,
-                size: blob.size,
-                data: blob.content(MAX_DATA_DISPLAY_SIZE),
-                binary: blob.binary?
-              )
+              rugged_raw(repository, sha, limit: MAX_DATA_DISPLAY_SIZE)
             end
           end
         end
@@ -59,10 +56,25 @@ module Gitlab
           end
         end
 
+        # Find LFS blobs given an array of sha ids
+        # Returns array of Gitlab::Git::Blob
+        # Does not guarantee blob data will be set
+        def batch_lfs_pointers(repository, blob_ids)
+          blob_ids.lazy
+                  .select { |sha| possible_lfs_blob?(repository, sha) }
+                  .map { |sha| rugged_raw(repository, sha, limit: LFS_POINTER_MAX_SIZE) }
+                  .select(&:lfs_pointer?)
+                  .force
+        end
+
         def binary?(data)
           EncodingHelper.detect_libgit2_binary?(data)
         end
 
+        def size_could_be_lfs?(size)
+          size.between?(LFS_POINTER_MIN_SIZE, LFS_POINTER_MAX_SIZE)
+        end
+
         private
 
         # Recursive search of blob id by path
@@ -167,6 +179,29 @@ module Gitlab
             end
           end
         end
+
+        def rugged_raw(repository, sha, limit:)
+          blob = repository.lookup(sha)
+
+          return unless blob.is_a?(Rugged::Blob)
+
+          new(
+            id: blob.oid,
+            size: blob.size,
+            data: blob.content(limit),
+            binary: blob.binary?
+          )
+        end
+
+        # Efficient lookup to determine if object size
+        # and type make it a possible LFS blob without loading
+        # blob content into memory with repository.lookup(sha)
+        def possible_lfs_blob?(repository, sha)
+          object_header = repository.rugged.read_header(sha)
+
+          object_header[:type] == :blob &&
+            size_could_be_lfs?(object_header[:len])
+        end
       end
 
       def initialize(options)
@@ -226,7 +261,7 @@ module Gitlab
       # size
       # see https://github.com/github/git-lfs/blob/v1.1.0/docs/spec.md#the-pointer
       def lfs_pointer?
-        has_lfs_version_key? && lfs_oid.present? && lfs_size.present?
+        self.class.size_could_be_lfs?(size) && has_lfs_version_key? && lfs_oid.present? && lfs_size.present?
       end
 
       def lfs_oid
diff --git a/lib/gitlab/git/lfs_changes.rb b/lib/gitlab/git/lfs_changes.rb
new file mode 100644
index 00000000000..2749e2e69e2
--- /dev/null
+++ b/lib/gitlab/git/lfs_changes.rb
@@ -0,0 +1,36 @@
+module Gitlab
+  module Git
+    class LfsChanges
+      def initialize(repository, newrev)
+        @repository = repository
+        @newrev = newrev
+      end
+
+      def new_pointers(object_limit: nil, not_in: nil)
+        @new_pointers ||= begin
+          object_ids = new_objects(not_in: not_in)
+          object_ids = object_ids.take(object_limit) if object_limit
+
+          Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids)
+        end
+      end
+
+      def all_pointers
+        object_ids = rev_list.all_objects(require_path: true)
+
+        Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids)
+      end
+
+      private
+
+      def new_objects(not_in:)
+        rev_list.new_objects(require_path: true, lazy: true, not_in: not_in)
+      end
+
+      def rev_list
+        ::Gitlab::Git::RevList.new(path_to_repo: @repository.path_to_repo,
+                                   newrev: @newrev)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index fc8af38d4d9..a9e4e1130c3 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -290,6 +290,14 @@ module Gitlab
         end
       end
 
+      def batch_existence(object_ids, existing: true)
+        filter_method = existing ? :select : :reject
+
+        object_ids.public_send(filter_method) do |oid| # rubocop:disable GitlabSecurity/PublicSend
+          rugged.exists?(oid)
+        end
+      end
+
       # Returns an Array of branch and tag names
       def ref_names
         branch_names + tag_names
diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb
index 60b2a4ec411..e0c884aceaa 100644
--- a/lib/gitlab/git/rev_list.rb
+++ b/lib/gitlab/git/rev_list.rb
@@ -13,11 +13,31 @@ module Gitlab
         @path_to_repo = path_to_repo
       end
 
-      # This method returns an array of new references
+      # This method returns an array of new commit references
       def new_refs
         execute([*base_args, newrev, '--not', '--all'])
       end
 
+      # Finds newly added objects
+      # Returns an array of shas
+      #
+      # Can skip objects which do not have a path using required_path: true
+      # This skips commit objects and root trees, which might not be needed when
+      # looking for blobs
+      #
+      # Can return a lazy enumerator to limit work done on megabytes of data
+      def new_objects(require_path: nil, lazy: false, not_in: nil)
+        object_output = execute([*base_args, newrev, *not_in_refs(not_in), '--objects'])
+
+        objects_from_output(object_output, require_path: require_path, lazy: lazy)
+      end
+
+      def all_objects(require_path: nil)
+        object_output = execute([*base_args, '--all', '--objects'])
+
+        objects_from_output(object_output, require_path: require_path, lazy: true)
+      end
+
       # This methods returns an array of missed references
       #
       # Should become obsolete after https://gitlab.com/gitlab-org/gitaly/issues/348.
@@ -27,6 +47,13 @@ module Gitlab
 
       private
 
+      def not_in_refs(references)
+        return ['--not', '--all'] unless references
+        return [] if references.empty?
+
+        references.prepend('--not')
+      end
+
       def execute(args)
         output, status = popen(args, nil, Gitlab::Git::Env.to_env_hash)
 
@@ -44,6 +71,22 @@ module Gitlab
           'rev-list'
         ]
       end
+
+      def objects_from_output(object_output, require_path: nil, lazy: nil)
+        objects = object_output.lazy.map do |output_line|
+          sha, path = output_line.split(' ', 2)
+
+          next if require_path && path.blank?
+
+          sha
+        end.reject(&:nil?)
+
+        if lazy
+          objects
+        else
+          objects.force
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb
index 549d22adde5..45362ac438b 100644
--- a/lib/gitlab/git/wiki.rb
+++ b/lib/gitlab/git/wiki.rb
@@ -68,11 +68,13 @@ module Gitlab
       end
 
       def file(name, version)
-        version ||= self.class.default_ref
-        gollum_file = gollum_wiki.file(name, version)
-        return unless gollum_file
-
-        Gitlab::Git::WikiFile.new(gollum_file)
+        @repository.gitaly_migrate(:wiki_find_file) do |is_enabled|
+          if is_enabled
+            gitaly_find_file(name, version)
+          else
+            gollum_find_file(name, version)
+          end
+        end
       end
 
       def page_versions(page_path)
@@ -156,6 +158,14 @@ module Gitlab
         new_page(gollum_page)
       end
 
+      def gollum_find_file(name, version)
+        version ||= self.class.default_ref
+        gollum_file = gollum_wiki.file(name, version)
+        return unless gollum_file
+
+        Gitlab::Git::WikiFile.new(gollum_file)
+      end
+
       def gitaly_write_page(name, format, content, commit_details)
         gitaly_wiki_client.write_page(name, format, content, commit_details)
       end
@@ -170,6 +180,13 @@ module Gitlab
 
         Gitlab::Git::WikiPage.new(wiki_page, version)
       end
+
+      def gitaly_find_file(name, version)
+        wiki_file = gitaly_wiki_client.find_file(name, version)
+        return unless wiki_file
+
+        Gitlab::Git::WikiFile.new(wiki_file)
+      end
     end
   end
 end
diff --git a/lib/gitlab/gitaly_client/wiki_file.rb b/lib/gitlab/gitaly_client/wiki_file.rb
new file mode 100644
index 00000000000..a2e415864e6
--- /dev/null
+++ b/lib/gitlab/gitaly_client/wiki_file.rb
@@ -0,0 +1,17 @@
+module Gitlab
+  module GitalyClient
+    class WikiFile
+      FIELDS = %i(name mime_type path raw_data).freeze
+
+      attr_accessor(*FIELDS)
+
+      def initialize(params)
+        params = params.with_indifferent_access
+
+        FIELDS.each do |field|
+          instance_variable_set("@#{field}", params[field])
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb
index 5d7930f5ff9..15f0f30d303 100644
--- a/lib/gitlab/gitaly_client/wiki_service.rb
+++ b/lib/gitlab/gitaly_client/wiki_service.rb
@@ -80,6 +80,32 @@ module Gitlab
         [wiki_page, version]
       end
 
+      def find_file(name, revision)
+        request = Gitaly::WikiFindFileRequest.new(
+          repository: @gitaly_repo,
+          name: GitalyClient.encode(name),
+          revision: GitalyClient.encode(revision)
+        )
+
+        response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_file, request)
+        wiki_file = nil
+
+        response.each do |message|
+          next unless message.name.present?
+
+          if wiki_file
+            wiki_file.raw_data << message.raw_data
+          else
+            wiki_file = GitalyClient::WikiFile.new(message.to_h)
+            # All gRPC strings in a response are frozen, so we get
+            # an unfrozen version here so appending in the else clause below doesn't blow up.
+            wiki_file.raw_data = wiki_file.raw_data.dup
+          end
+        end
+
+        wiki_file
+      end
+
       private
 
       def gitaly_commit_details(commit_details)
diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb
index f9dd8e41912..b983a40611f 100644
--- a/lib/gitlab/metrics/sidekiq_middleware.rb
+++ b/lib/gitlab/metrics/sidekiq_middleware.rb
@@ -11,6 +11,8 @@ module Gitlab
           # Old gitlad-shell messages don't provide enqueued_at/created_at attributes
           trans.set(:sidekiq_queue_duration, Time.now.to_f - (message['enqueued_at'] || message['created_at'] || 0))
           trans.run { yield }
+
+          worker.metrics_tags.each { |tag, value| trans.add_tag(tag, value) } if worker.respond_to?(:metrics_tags)
         rescue Exception => error # rubocop: disable Lint/RescueException
           trans.add_event(:sidekiq_exception)
 
diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb
index f42168c720e..cfc6b2a2029 100644
--- a/lib/gitlab/middleware/go.rb
+++ b/lib/gitlab/middleware/go.rb
@@ -4,6 +4,7 @@ module Gitlab
   module Middleware
     class Go
       include ActionView::Helpers::TagHelper
+      include Gitlab::CurrentSettings
 
       PROJECT_PATH_REGEX = %r{\A(#{Gitlab::PathRegex.full_namespace_route_regex}/#{Gitlab::PathRegex.project_route_regex})/}.freeze
 
@@ -37,10 +38,20 @@ module Gitlab
       end
 
       def go_body(path)
-        project_url = URI.join(Gitlab.config.gitlab.url, path)
+        config = Gitlab.config
+        project_url = URI.join(config.gitlab.url, path)
         import_prefix = strip_url(project_url.to_s)
 
-        meta_tag = tag :meta, name: 'go-import', content: "#{import_prefix} git #{project_url}.git"
+        repository_url = case current_application_settings.enabled_git_access_protocol
+                         when 'ssh'
+                           shell = config.gitlab_shell
+                           port = ":#{shell.ssh_port}" unless shell.ssh_port == 22
+                           "ssh://#{shell.ssh_user}@#{shell.ssh_host}#{port}/#{path}.git"
+                         when 'http', nil
+                           "#{project_url}.git"
+                         end
+
+        meta_tag = tag :meta, name: 'go-import', content: "#{import_prefix} git #{repository_url}"
         head_tag = content_tag :head, meta_tag
         content_tag :html, head_tag
       end
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index 58d5b0da1c4..e1219df1b25 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -16,14 +16,15 @@ module Gitlab
     SECRET_LENGTH = 32
 
     class << self
-      def git_http_ok(repository, is_wiki, user, action)
+      def git_http_ok(repository, is_wiki, user, action, show_all_refs: false)
         project = repository.project
         repo_path = repository.path_to_repo
         params = {
           GL_ID: Gitlab::GlId.gl_id(user),
           GL_REPOSITORY: Gitlab::GlRepository.gl_repository(project, is_wiki),
           GL_USERNAME: user&.username,
-          RepoPath: repo_path
+          RepoPath: repo_path,
+          ShowAllRefs: show_all_refs
         }
         server = {
           address: Gitlab::GitalyClient.address(project.repository_storage),
diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake
index 930b4bc13e2..ba221e44e5d 100644
--- a/lib/tasks/gitlab/dev.rake
+++ b/lib/tasks/gitlab/dev.rake
@@ -5,10 +5,9 @@ namespace :gitlab do
       opts =
         if ENV['CI']
           {
-            # We don't use CI_REPOSITORY_URL since it includes `gitlab-ci-token:xxxxxxxxxxxxxxxxxxxx@`
-            # which is confusing in the steps suggested in the job's output.
-            ce_repo: "#{ENV['CI_PROJECT_URL']}.git",
-            branch: ENV['CI_COMMIT_REF_NAME']
+            ce_project_url: ENV['CI_PROJECT_URL'],
+            branch: ENV['CI_COMMIT_REF_NAME'],
+            job_id: ENV['CI_JOB_ID']
           }
         else
           unless args[:branch]