Merge branch 'master' into 33329-tech-article-deploying-maven-artifacts

22 files changed, 744 insertions, 737 deletions

diff --git a/lib/gitlab/git/attributes.rb b/lib/gitlab/git/attributes.rb
index 42140ecc993..2d20cd473a7 100644
--- a/lib/gitlab/git/attributes.rb
+++ b/lib/gitlab/git/attributes.rb
@@ -1,3 +1,8 @@
+# Gitaly note: JV: not sure what to make of this class. Why does it use
+# the full disk path of the repository to look up attributes This is
+# problematic in Gitaly, because Gitaly hides the full disk path to the
+# repository from gitlab-ce.
+
 module Gitlab
   module Git
     # Class for parsing Git attribute files and extracting the attributes for
diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb
index 66829a03c2e..0deaab01b5b 100644
--- a/lib/gitlab/git/blame.rb
+++ b/lib/gitlab/git/blame.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: needs 1 RPC for #load_blame.
+
 module Gitlab
   module Git
     class Blame
@@ -24,6 +26,7 @@ module Gitlab
 
       private
 
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/376
       def load_blame
         cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{@repo.path} blame -p #{@sha} -- #{@path})
         # Read in binary mode to ensure ASCII-8BIT
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index d60e607b02b..db6cfc9671f 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: seems to be completely migrated (behind feature flags).
+
 module Gitlab
   module Git
     class Blob
@@ -14,6 +16,47 @@ module Gitlab
 
       class << self
         def find(repository, sha, path)
+          Gitlab::GitalyClient.migrate(:project_raw_show) do |is_enabled|
+            if is_enabled
+              find_by_gitaly(repository, sha, path)
+            else
+              find_by_rugged(repository, sha, path)
+            end
+          end
+        end
+
+        def find_by_gitaly(repository, sha, path)
+          path = path.sub(/\A\/*/, '')
+          path = '/' if path.empty?
+          name = File.basename(path)
+          entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE)
+          return unless entry
+
+          case entry.type
+          when :COMMIT
+            new(
+              id: entry.oid,
+              name: name,
+              size: 0,
+              data: '',
+              path: path,
+              commit_id: sha
+            )
+          when :BLOB
+            new(
+              id: entry.oid,
+              name: name,
+              size: entry.size,
+              data: entry.data.dup,
+              mode: entry.mode.to_s(8),
+              path: path,
+              commit_id: sha,
+              binary: binary?(entry.data)
+            )
+          end
+        end
+
+        def find_by_rugged(repository, sha, path)
           commit = repository.lookup(sha)
           root_tree = commit.tree
 
@@ -42,16 +85,32 @@ module Gitlab
         end
 
         def raw(repository, sha)
-          blob = repository.lookup(sha)
+          Gitlab::GitalyClient.migrate(:git_blob_raw) do |is_enabled|
+            if is_enabled
+              Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE)
+            else
+              blob = repository.lookup(sha)
 
-          new(
-            id: blob.oid,
-            size: blob.size,
-            data: blob.content(MAX_DATA_DISPLAY_SIZE),
-            binary: blob.binary?
-          )
+              new(
+                id: blob.oid,
+                size: blob.size,
+                data: blob.content(MAX_DATA_DISPLAY_SIZE),
+                binary: blob.binary?
+              )
+            end
+          end
         end
 
+        def binary?(data)
+          # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks
+          # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15),
+          # which is what we use below to keep a consistent behavior.
+          detect = CharlockHolmes::EncodingDetector.new(8000).detect(data)
+          detect && detect[:type] == :binary
+        end
+
+        private
+
         # Recursive search of blob id by path
         #
         # Ex.
@@ -120,15 +179,29 @@ module Gitlab
         return if @data == '' # don't mess with submodule blobs
         return @data if @loaded_all_data
 
+        Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled|
+          @data = begin
+            if is_enabled
+              Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: id, limit: -1).data
+            else
+              repository.lookup(id).content
+            end
+          end
+        end
+
         @loaded_all_data = true
-        @data = repository.lookup(id).content
         @loaded_size = @data.bytesize
+        @binary = nil
       end
 
       def name
         encode! @name
       end
 
+      def path
+        encode! @path
+      end
+
       def truncated?
         size && (size > loaded_size)
       end
diff --git a/lib/gitlab/git/blob_snippet.rb b/lib/gitlab/git/blob_snippet.rb
index d7975f88aaa..68116e775c6 100644
--- a/lib/gitlab/git/blob_snippet.rb
+++ b/lib/gitlab/git/blob_snippet.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: no RPC's here.
+
 module Gitlab
   module Git
     class BlobSnippet
diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb
index 124526e4b59..c53882787f1 100644
--- a/lib/gitlab/git/branch.rb
+++ b/lib/gitlab/git/branch.rb
@@ -1,39 +1,10 @@
+# Gitaly note: JV: no RPC's here.
+
 module Gitlab
   module Git
     class Branch < Ref
-      def initialize(repository, name, target)
-        if target.is_a?(Gitaly::FindLocalBranchResponse)
-          target = target_from_gitaly_local_branches_response(target)
-        end
-
-        super(repository, name, target)
-      end
-
-      def target_from_gitaly_local_branches_response(response)
-        # Git messages have no encoding enforcements. However, in the UI we only
-        # handle UTF-8, so basically we cross our fingers that the message force
-        # encoded to UTF-8 is readable.
-        message = response.commit_subject.dup.force_encoding('UTF-8')
-
-        # NOTE: For ease of parsing in Gitaly, we have only the subject of
-        # the commit and not the full message. This is ok, since all the
-        # code that uses `local_branches` only cares at most about the
-        # commit message.
-        # TODO: Once gitaly "takes over" Rugged consider separating the
-        # subject from the message to make it clearer when there's one
-        # available but not the other.
-        hash = {
-          id: response.commit_id,
-          message: message,
-          authored_date: Time.at(response.commit_author.date.seconds),
-          author_name: response.commit_author.name,
-          author_email: response.commit_author.email,
-          committed_date: Time.at(response.commit_committer.date.seconds),
-          committer_name: response.commit_committer.name,
-          committer_email: response.commit_committer.email
-        }
-
-        Gitlab::Git::Commit.decorate(hash)
+      def initialize(repository, name, target, target_commit)
+        super(repository, name, target, target_commit)
       end
     end
   end
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index bb04731f08c..ca7e3a7c4be 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -4,7 +4,7 @@ module Gitlab
     class Commit
       include Gitlab::EncodingHelper
 
-      attr_accessor :raw_commit, :head, :refs
+      attr_accessor :raw_commit, :head
 
       SERIALIZE_KEYS = [
         :id, :message, :parent_ids,
@@ -38,7 +38,7 @@ module Gitlab
           repo = options.delete(:repo)
           raise 'Gitlab::Git::Repository is required' unless repo.respond_to?(:log)
 
-          repo.log(options).map { |c| decorate(c) }
+          repo.log(options)
         end
 
         # Get single commit
@@ -48,6 +48,7 @@ module Gitlab
         #
         #   Commit.find(repo, 'master')
         #
+        # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/321
         def find(repo, commit_id = "HEAD")
           return commit_id if commit_id.is_a?(Gitlab::Git::Commit)
           return decorate(commit_id) if commit_id.is_a?(Rugged::Commit)
@@ -97,16 +98,89 @@ module Gitlab
         #   Commit.between(repo, '29eda46b', 'master')
         #
         def between(repo, base, head)
-          repo.commits_between(base, head).map do |commit|
-            decorate(commit)
+          Gitlab::GitalyClient.migrate(:commits_between) do |is_enabled|
+            if is_enabled
+              repo.gitaly_commit_client.between(base, head)
+            else
+              repo.commits_between(base, head).map { |c| decorate(c) }
+            end
           end
         rescue Rugged::ReferenceError
           []
         end
 
-        # Delegate Repository#find_commits
+        # Returns commits collection
+        #
+        # Ex.
+        #   Commit.find_all(
+        #     repo,
+        #     ref: 'master',
+        #     max_count: 10,
+        #     skip: 5,
+        #     order: :date
+        #   )
+        #
+        #   +options+ is a Hash of optional arguments to git
+        #     :ref is the ref from which to begin (SHA1 or name)
+        #     :max_count is the maximum number of commits to fetch
+        #     :skip is the number of commits to skip
+        #     :order is the commits order and allowed value is :none (default), :date,
+        #        :topo, or any combination of them (in an array). Commit ordering types
+        #        are documented here:
+        #        http://www.rubydoc.info/github/libgit2/rugged/Rugged#SORT_NONE-constant)
+        #
+        # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/326
         def find_all(repo, options = {})
-          repo.find_commits(options)
+          Gitlab::GitalyClient.migrate(:find_all_commits) do |is_enabled|
+            if is_enabled
+              find_all_by_gitaly(repo, options)
+            else
+              find_all_by_rugged(repo, options)
+            end
+          end
+        end
+
+        def find_all_by_rugged(repo, options = {})
+          actual_options = options.dup
+
+          allowed_options = [:ref, :max_count, :skip, :order]
+
+          actual_options.keep_if do |key|
+            allowed_options.include?(key)
+          end
+
+          default_options = { skip: 0 }
+          actual_options = default_options.merge(actual_options)
+
+          rugged = repo.rugged
+          walker = Rugged::Walker.new(rugged)
+
+          if actual_options[:ref]
+            walker.push(rugged.rev_parse_oid(actual_options[:ref]))
+          else
+            rugged.references.each("refs/heads/*") do |ref|
+              walker.push(ref.target_id)
+            end
+          end
+
+          walker.sorting(rugged_sort_type(actual_options[:order]))
+
+          commits = []
+          offset = actual_options[:skip]
+          limit = actual_options[:max_count]
+          walker.each(offset: offset, limit: limit) do |commit|
+            commits.push(decorate(commit))
+          end
+
+          walker.reset
+
+          commits
+        rescue Rugged::OdbError
+          []
+        end
+
+        def find_all_by_gitaly(repo, options = {})
+          Gitlab::GitalyClient::CommitService.new(repo).find_all_commits(options)
         end
 
         def decorate(commit, ref = nil)
@@ -131,15 +205,32 @@ module Gitlab
           diff.find_similar!(break_rewrites: break_rewrites)
           diff
         end
+
+        # Returns the `Rugged` sorting type constant for one or more given
+        # sort types. Valid keys are `:none`, `:topo`, and `:date`, or an array
+        # containing more than one of them. `:date` uses a combination of date and
+        # topological sorting to closer mimic git's native ordering.
+        def rugged_sort_type(sort_type)
+          @rugged_sort_types ||= {
+            none: Rugged::SORT_NONE,
+            topo: Rugged::SORT_TOPO,
+            date: Rugged::SORT_DATE | Rugged::SORT_TOPO
+          }
+
+          @rugged_sort_types.fetch(sort_type, Rugged::SORT_NONE)
+        end
       end
 
       def initialize(raw_commit, head = nil)
         raise "Nil as raw commit passed" unless raw_commit
 
-        if raw_commit.is_a?(Hash)
+        case raw_commit
+        when Hash
           init_from_hash(raw_commit)
-        elsif raw_commit.is_a?(Rugged::Commit)
+        when Rugged::Commit
           init_from_rugged(raw_commit)
+        when Gitlab::GitalyClient::Commit
+          init_from_gitaly(raw_commit)
         else
           raise "Invalid raw commit type: #{raw_commit.class}"
         end
@@ -175,8 +266,10 @@ module Gitlab
       # Shows the diff between the commit's parent and the commit.
       #
       # Cuts out the header and stats from #to_patch and returns only the diff.
-      def to_diff(options = {})
-        diff_from_parent(options).patch
+      #
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/324
+      def to_diff
+        diff_from_parent.patch
       end
 
       # Returns a diff object for the changes from this commit's first parent.
@@ -216,7 +309,23 @@ module Gitlab
       end
 
       def parents
-        raw_commit.parents.map { |c| Gitlab::Git::Commit.new(c) }
+        case raw_commit
+        when Rugged::Commit
+          raw_commit.parents.map { |c| Gitlab::Git::Commit.new(c) }
+        when Gitlab::GitalyClient::Commit
+          parent_ids.map { |oid| self.class.find(raw_commit.repository, oid) }.compact
+        else
+          raise NotImplementedError, "commit source doesn't support #parents"
+        end
+      end
+
+      # Get the gpg signature of this commit.
+      #
+      # Ex.
+      #   commit.signature(repo)
+      #
+      def signature(repo)
+        Rugged::Commit.extract_signature(repo.rugged, sha)
       end
 
       def stats
@@ -227,7 +336,7 @@ module Gitlab
         begin
           raw_commit.to_mbox(options)
         rescue Rugged::InvalidError => ex
-          if ex.message =~ /Commit \w+ is a merge commit/
+          if ex.message =~ /commit \w+ is a merge commit/i
             'Patch format is not currently supported for merge commits.'
           end
         end
@@ -299,6 +408,22 @@ module Gitlab
         @parent_ids = commit.parents.map(&:oid)
       end
 
+      def init_from_gitaly(commit)
+        @raw_commit = commit
+        @id = commit.id
+        # TODO: Once gitaly "takes over" Rugged consider separating the
+        # subject from the message to make it clearer when there's one
+        # available but not the other.
+        @message = (commit.body.presence || commit.subject).dup
+        @authored_date = Time.at(commit.author.date.seconds)
+        @author_name = commit.author.name.dup
+        @author_email = commit.author.email.dup
+        @committed_date = Time.at(commit.committer.date.seconds)
+        @committer_name = commit.committer.name.dup
+        @committer_email = commit.committer.email.dup
+        @parent_ids = commit.parent_ids
+      end
+
       def serialize_keys
         SERIALIZE_KEYS
       end
diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb
index e9118bbed0e..57c29ad112c 100644
--- a/lib/gitlab/git/commit_stats.rb
+++ b/lib/gitlab/git/commit_stats.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: 1 RPC, migration in progress.
+
 # Gitlab::Git::CommitStats counts the additions, deletions, and total changes
 # in a commit.
 module Gitlab
@@ -6,6 +8,8 @@ module Gitlab
       attr_reader :id, :additions, :deletions, :total
 
       # Instantiate a CommitStats object
+      #
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/323
       def initialize(commit)
         @id = commit.id
         @additions = 0
diff --git a/lib/gitlab/git/compare.rb b/lib/gitlab/git/compare.rb
index 696a2acd5e3..7cb842256d0 100644
--- a/lib/gitlab/git/compare.rb
+++ b/lib/gitlab/git/compare.rb
@@ -1,9 +1,11 @@
+# Gitaly note: JV: no RPC's here.
+
 module Gitlab
   module Git
     class Compare
       attr_reader :head, :base, :straight
 
-      def initialize(repository, base, head, straight = false)
+      def initialize(repository, base, head, straight: false)
         @repository = repository
         @straight = straight
 
diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb
index 8926aa19925..9e00abefd02 100644
--- a/lib/gitlab/git/diff.rb
+++ b/lib/gitlab/git/diff.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: needs RPC for Gitlab::Git::Diff.between.
+
 # Gitlab::Git::Diff is a wrapper around native Rugged::Diff object
 module Gitlab
   module Git
@@ -16,13 +18,32 @@ module Gitlab
       alias_method :renamed_file?, :renamed_file
 
       attr_accessor :expanded
+      attr_writer :too_large
+
+      alias_method :expanded?, :expanded
 
-      # We need this accessor because of `to_hash` and `init_from_hash`
-      attr_accessor :too_large
+      SERIALIZE_KEYS = %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large).freeze
 
       class << self
         # The maximum size of a diff to display.
         def size_limit
+          if RequestStore.active?
+            RequestStore['gitlab_git_diff_size_limit'] ||= find_size_limit
+          else
+            find_size_limit
+          end
+        end
+
+        # The maximum size before a diff is collapsed.
+        def collapse_limit
+          if RequestStore.active?
+            RequestStore['gitlab_git_diff_collapse_limit'] ||= find_collapse_limit
+          else
+            find_collapse_limit
+          end
+        end
+
+        def find_size_limit
           if Feature.enabled?('gitlab_git_diff_size_limit_increase')
             200.kilobytes
           else
@@ -30,8 +51,7 @@ module Gitlab
           end
         end
 
-        # The maximum size before a diff is collapsed.
-        def collapse_limit
+        def find_collapse_limit
           if Feature.enabled?('gitlab_git_diff_size_limit_increase')
             100.kilobytes
           else
@@ -63,110 +83,16 @@ module Gitlab
         # Return a copy of the +options+ hash containing only keys that can be
         # passed to Rugged.  Allowed options are:
         #
-        #  :max_size ::
-        #    An integer specifying the maximum byte size of a file before a it
-        #    will be treated as binary. The default value is 512MB.
-        #
-        #  :context_lines ::
-        #    The number of unchanged lines that define the boundary of a hunk
-        #    (and to display before and after the actual changes). The default is
-        #    3.
-        #
-        #  :interhunk_lines ::
-        #    The maximum number of unchanged lines between hunk boundaries before
-        #    the hunks will be merged into a one. The default is 0.
-        #
-        #  :old_prefix ::
-        #    The virtual "directory" to prefix to old filenames in hunk headers.
-        #    The default is "a".
-        #
-        #  :new_prefix ::
-        #    The virtual "directory" to prefix to new filenames in hunk headers.
-        #    The default is "b".
-        #
-        #  :reverse ::
-        #    If true, the sides of the diff will be reversed.
-        #
-        #  :force_text ::
-        #    If true, all files will be treated as text, disabling binary
-        #    attributes & detection.
-        #
-        #  :ignore_whitespace ::
-        #    If true, all whitespace will be ignored.
-        #
         #  :ignore_whitespace_change ::
         #    If true, changes in amount of whitespace will be ignored.
         #
-        #  :ignore_whitespace_eol ::
-        #    If true, whitespace at end of line will be ignored.
-        #
-        #  :ignore_submodules ::
-        #    if true, submodules will be excluded from the diff completely.
-        #
-        #  :patience ::
-        #    If true, the "patience diff" algorithm will be used (currenlty
-        #    unimplemented).
-        #
-        #  :include_ignored ::
-        #    If true, ignored files will be included in the diff.
-        #
-        #  :include_untracked ::
-        #   If true, untracked files will be included in the diff.
-        #
-        #  :include_unmodified ::
-        #    If true, unmodified files will be included in the diff.
-        #
-        #  :recurse_untracked_dirs ::
-        #    Even if +:include_untracked+ is true, untracked directories will
-        #    only be marked with a single entry in the diff. If this flag is set
-        #    to true, all files under ignored directories will be included in the
-        #    diff, too.
-        #
         #  :disable_pathspec_match ::
         #    If true, the given +*paths+ will be applied as exact matches,
         #    instead of as fnmatch patterns.
         #
-        #  :deltas_are_icase ::
-        #    If true, filename comparisons will be made with case-insensitivity.
-        #
-        #  :include_untracked_content ::
-        #    if true, untracked content will be contained in the the diff patch
-        #    text.
-        #
-        #  :skip_binary_check ::
-        #    If true, diff deltas will be generated without spending time on
-        #    binary detection. This is useful to improve performance in cases
-        #    where the actual file content difference is not needed.
-        #
-        #  :include_typechange ::
-        #    If true, type changes for files will not be interpreted as deletion
-        #    of the "old file" and addition of the "new file", but will generate
-        #    typechange records.
-        #
-        #  :include_typechange_trees ::
-        #    Even if +:include_typechange+ is true, blob -> tree changes will
-        #    still usually be handled as a deletion of the blob. If this flag is
-        #    set to true, blob -> tree changes will be marked as typechanges.
-        #
-        #  :ignore_filemode ::
-        #    If true, file mode changes will be ignored.
-        #
-        #  :recurse_ignored_dirs ::
-        #    Even if +:include_ignored+ is true, ignored directories will only be
-        #    marked with a single entry in the diff. If this flag is set to true,
-        #    all files under ignored directories will be included in the diff,
-        #    too.
         def filter_diff_options(options, default_options = {})
-          allowed_options = [:max_size, :context_lines, :interhunk_lines,
-                             :old_prefix, :new_prefix, :reverse, :force_text,
-                             :ignore_whitespace, :ignore_whitespace_change,
-                             :ignore_whitespace_eol, :ignore_submodules,
-                             :patience, :include_ignored, :include_untracked,
-                             :include_unmodified, :recurse_untracked_dirs,
-                             :disable_pathspec_match, :deltas_are_icase,
-                             :include_untracked_content, :skip_binary_check,
-                             :include_typechange, :include_typechange_trees,
-                             :ignore_filemode, :recurse_ignored_dirs, :paths,
+          allowed_options = [:ignore_whitespace_change,
+                             :disable_pathspec_match, :paths,
                              :max_files, :max_lines, :limits, :expanded]
 
           if default_options
@@ -213,16 +139,10 @@ module Gitlab
         end
       end
 
-      def serialize_keys
-        @serialize_keys ||= %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large)
-      end
-
       def to_hash
         hash = {}
 
-        keys = serialize_keys
-
-        keys.each do |key|
+        SERIALIZE_KEYS.each do |key|
           hash[key] = send(key)
         end
 
@@ -249,6 +169,9 @@ module Gitlab
         end
       end
 
+      # This is used by `to_hash` and `init_from_hash`.
+      alias_method :too_large, :too_large?
+
       def too_large!
         @diff = ''
         @line_count = 0
@@ -297,13 +220,13 @@ module Gitlab
       def init_from_hash(hash)
         raw_diff = hash.symbolize_keys
 
-        serialize_keys.each do |key|
+        SERIALIZE_KEYS.each do |key|
           send(:"#{key}=", raw_diff[key.to_sym])
         end
       end
 
       def init_from_gitaly(diff)
-        @diff = diff.patch if diff.respond_to?(:patch)
+        @diff = encode!(diff.patch) if diff.respond_to?(:patch)
         @new_path = encode!(diff.to_path.dup)
         @old_path = encode!(diff.from_path.dup)
         @a_mode = diff.old_mode.to_s(8)
@@ -311,6 +234,8 @@ module Gitlab
         @new_file = diff.from_id == BLANK_SHA
         @renamed_file = diff.from_path != diff.to_path
         @deleted_file = diff.to_id == BLANK_SHA
+
+        collapse! if diff.respond_to?(:collapsed) && diff.collapsed
       end
 
       def prune_diff_if_eligible
diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb
index 334e06a6eca..87ed9c3ea26 100644
--- a/lib/gitlab/git/diff_collection.rb
+++ b/lib/gitlab/git/diff_collection.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: no RPC's here.
+
 module Gitlab
   module Git
     class DiffCollection
@@ -5,16 +7,28 @@ module Gitlab
 
       DEFAULT_LIMITS = { max_files: 100, max_lines: 5000 }.freeze
 
+      attr_reader :limits
+
+      delegate :max_files, :max_lines, :max_bytes, :safe_max_files, :safe_max_lines, :safe_max_bytes, to: :limits
+
+      def self.collection_limits(options = {})
+        limits = {}
+        limits[:max_files] = options.fetch(:max_files, DEFAULT_LIMITS[:max_files])
+        limits[:max_lines] = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines])
+        limits[:max_bytes] = limits[:max_files] * 5.kilobytes # Average 5 KB per file
+        limits[:safe_max_files] = [limits[:max_files], DEFAULT_LIMITS[:max_files]].min
+        limits[:safe_max_lines] = [limits[:max_lines], DEFAULT_LIMITS[:max_lines]].min
+        limits[:safe_max_bytes] = limits[:safe_max_files] * 5.kilobytes # Average 5 KB per file
+
+        OpenStruct.new(limits)
+      end
+
       def initialize(iterator, options = {})
         @iterator = iterator
-        @max_files = options.fetch(:max_files, DEFAULT_LIMITS[:max_files])
-        @max_lines = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines])
-        @max_bytes = @max_files * 5.kilobytes # Average 5 KB per file
-        @safe_max_files = [@max_files, DEFAULT_LIMITS[:max_files]].min
-        @safe_max_lines = [@max_lines, DEFAULT_LIMITS[:max_lines]].min
-        @safe_max_bytes = @safe_max_files * 5.kilobytes # Average 5 KB per file
+        @limits = self.class.collection_limits(options)
         @enforce_limits = !!options.fetch(:limits, true)
         @expanded = !!options.fetch(:expanded, true)
+        @from_gitaly = options.fetch(:from_gitaly, false)
 
         @line_count = 0
         @byte_count = 0
@@ -24,9 +38,23 @@ module Gitlab
       end
 
       def each(&block)
-        Gitlab::GitalyClient.migrate(:commit_raw_diffs) do
-          each_patch(&block)
+        @array.each(&block)
+
+        return if @overflow
+        return if @iterator.nil?
+
+        Gitlab::GitalyClient.migrate(:commit_raw_diffs) do |is_enabled|
+          if is_enabled && @from_gitaly
+            each_gitaly_patch(&block)
+          else
+            each_rugged_patch(&block)
+          end
         end
+
+        @populated = true
+
+        # Allow iterator to be garbage-collected. It cannot be reused anyway.
+        @iterator = nil
       end
 
       def empty?
@@ -72,23 +100,32 @@ module Gitlab
       end
 
       def over_safe_limits?(files)
-        files >= @safe_max_files || @line_count > @safe_max_lines || @byte_count >= @safe_max_bytes
+        files >= safe_max_files || @line_count > safe_max_lines || @byte_count >= safe_max_bytes
       end
 
-      def each_patch
-        i = 0
-        @array.each do |diff|
-          yield diff
+      def each_gitaly_patch
+        i = @array.length
+
+        @iterator.each do |raw|
+          diff = Gitlab::Git::Diff.new(raw, expanded: !@enforce_limits || @expanded)
+
+          if raw.overflow_marker
+            @overflow = true
+            break
+          end
+
+          yield @array[i] = diff
           i += 1
         end
+      end
 
-        return if @overflow
-        return if @iterator.nil?
+      def each_rugged_patch
+        i = @array.length
 
         @iterator.each do |raw|
           @empty = false
 
-          if @enforce_limits && i >= @max_files
+          if @enforce_limits && i >= max_files
             @overflow = true
             break
           end
@@ -97,14 +134,14 @@ module Gitlab
 
           diff = Gitlab::Git::Diff.new(raw, expanded: expanded)
 
-          if !expanded && over_safe_limits?(i)
+          if !expanded && over_safe_limits?(i) && diff.line_count > 0
             diff.collapse!
           end
 
           @line_count += diff.line_count
           @byte_count += diff.diff.bytesize
 
-          if @enforce_limits && (@line_count >= @max_lines || @byte_count >= @max_bytes)
+          if @enforce_limits && (@line_count >= max_lines || @byte_count >= max_bytes)
             # This last Diff instance pushes us over the lines limit. We stop and
             # discard it.
             @overflow = true
@@ -114,11 +151,6 @@ module Gitlab
           yield @array[i] = diff
           i += 1
         end
-
-        @populated = true
-
-        # Allow iterator to be garbage-collected. It cannot be reused anyway.
-        @iterator = nil
       end
     end
   end
diff --git a/lib/gitlab/git/env.rb b/lib/gitlab/git/env.rb
index 0fdc57ec954..f80193ac553 100644
--- a/lib/gitlab/git/env.rb
+++ b/lib/gitlab/git/env.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: no RPC's here.
+
 module Gitlab
   module Git
     # Ephemeral (per request) storage for environment variables that some Git
diff --git a/lib/gitlab/git/gitmodules_parser.rb b/lib/gitlab/git/gitmodules_parser.rb
new file mode 100644
index 00000000000..4a43b9b444d
--- /dev/null
+++ b/lib/gitlab/git/gitmodules_parser.rb
@@ -0,0 +1,79 @@
+# Gitaly note: JV: no RPC's here.
+
+module Gitlab
+  module Git
+    class GitmodulesParser
+      def initialize(content)
+        @content = content
+      end
+
+      # Parses the contents of a .gitmodules file and returns a hash of
+      # submodule information, indexed by path.
+      def parse
+        reindex_by_path(get_submodules_by_name)
+      end
+
+      private
+
+      class State
+        def initialize
+          @result = {}
+          @current_submodule = nil
+        end
+
+        def start_section(section)
+          # In some .gitmodules files (e.g. nodegit's), a header
+          # with the same name appears multiple times; we want to
+          # accumulate the configs across these
+          @current_submodule = @result[section] || { 'name' => section }
+          @result[section] = @current_submodule
+        end
+
+        def set_attribute(attr, value)
+          @current_submodule[attr] = value
+        end
+
+        def section_started?
+          !@current_submodule.nil?
+        end
+
+        def submodules_by_name
+          @result
+        end
+      end
+
+      def get_submodules_by_name
+        iterator = State.new
+
+        @content.split("\n").each_with_object(iterator) do |text, iterator|
+          next if text =~ /^\s*#/
+
+          if text =~ /\A\[submodule "(?<name>[^"]+)"\]\z/
+            iterator.start_section($~[:name])
+          else
+            next unless iterator.section_started?
+
+            next unless text =~ /\A\s*(?<key>\w+)\s*=\s*(?<value>.*)\z/
+
+            value = $~[:value].chomp
+            iterator.set_attribute($~[:key], value)
+          end
+        end
+
+        iterator.submodules_by_name
+      end
+
+      def reindex_by_path(submodules_by_name)
+        # Convert from an indexed by name to an array indexed by path
+        # If a submodule doesn't have a path, it is considered bogus
+        # and is ignored
+        submodules_by_name.each_with_object({}) do |(name, data), results|
+          path = data.delete 'path'
+          next unless path
+
+          results[path] = data
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb
index bd90d24a2ec..8f0c377ef4f 100644
--- a/lib/gitlab/git/hook.rb
+++ b/lib/gitlab/git/hook.rb
@@ -1,12 +1,17 @@
+# Gitaly note: JV: looks like this is only used by GitHooksService in
+# app/services. We shouldn't bother migrating this until we know how
+# GitHooksService will be migrated.
+
 module Gitlab
   module Git
     class Hook
       GL_PROTOCOL = 'web'.freeze
       attr_reader :name, :repo_path, :path
 
-      def initialize(name, repo_path)
+      def initialize(name, project)
         @name = name
-        @repo_path = repo_path
+        @project = project
+        @repo_path = project.repository.path
         @path = File.join(repo_path.strip, 'hooks', name)
       end
 
@@ -38,7 +43,8 @@ module Gitlab
         vars = {
           'GL_ID' => gl_id,
           'PWD' => repo_path,
-          'GL_PROTOCOL' => GL_PROTOCOL
+          'GL_PROTOCOL' => GL_PROTOCOL,
+          'GL_REPOSITORY' => Gitlab::GlRepository.gl_repository(@project, false)
         }
 
         options = {
diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb
index 1add037fa5f..db532600d1b 100644
--- a/lib/gitlab/git/index.rb
+++ b/lib/gitlab/git/index.rb
@@ -1,3 +1,7 @@
+# Gitaly note: JV: When the time comes I think we will want to copy this
+# class into Gitaly. None of its methods look like they should be RPC's.
+# The RPC's will be at a higher level.
+
 module Gitlab
   module Git
     class Index
@@ -110,10 +114,6 @@ module Gitlab
           if segment == '..'
             raise IndexError, 'Path cannot include directory traversal'
           end
-
-          unless segment =~ Gitlab::Regex.file_name_regex
-            raise IndexError, "Path #{Gitlab::Regex.file_name_regex_message}"
-          end
         end
 
         pathname.to_s
diff --git a/lib/gitlab/git/path_helper.rb b/lib/gitlab/git/path_helper.rb
index 0148cd8df05..42c80aabd0a 100644
--- a/lib/gitlab/git/path_helper.rb
+++ b/lib/gitlab/git/path_helper.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: no RPC's here.
+
 module Gitlab
   module Git
     class PathHelper
diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb
index df9ca3ee5ac..25fa62ce4bd 100644
--- a/lib/gitlab/git/popen.rb
+++ b/lib/gitlab/git/popen.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: no RPC's here.
+
 require 'open3'
 
 module Gitlab
diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb
index ebf7393dc61..372ce005b94 100644
--- a/lib/gitlab/git/ref.rb
+++ b/lib/gitlab/git/ref.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: probably no RPC's here (just one interaction with Rugged).
+
 module Gitlab
   module Git
     class Ref
@@ -24,16 +26,16 @@ module Gitlab
         str.gsub(/\Arefs\/heads\//, '')
       end
 
+      # Gitaly: this method will probably be migrated indirectly via its call sites.
       def self.dereference_object(object)
         object = object.target while object.is_a?(Rugged::Tag::Annotation)
 
         object
       end
 
-      def initialize(repository, name, target)
-        encode! name
-        @name = name.gsub(/\Arefs\/(tags|heads)\//, '')
-        @dereferenced_target = Gitlab::Git::Commit.find(repository, target)
+      def initialize(repository, name, target, derefenced_target)
+        @name = Gitlab::Git.ref_name(name)
+        @dereferenced_target = derefenced_target
         @target = if target.respond_to?(:oid)
                     target.oid
                   elsif target.respond_to?(:name)
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 9d6adbdb4ac..70d793f8e4a 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -45,6 +45,8 @@ module Gitlab
                 :bare?,
                 to: :rugged
 
+      delegate :exists?, to: :gitaly_repository_client
+
       # Default branch in the repository
       def root_ref
         @root_ref ||= gitaly_migrate(:root_ref) do |is_enabled|
@@ -80,16 +82,14 @@ module Gitlab
       end
 
       # Returns an Array of Branches
-      def branches(filter: nil, sort_by: nil)
-        branches = rugged.branches.each(filter).map do |rugged_ref|
-          begin
-            Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target)
-          rescue Rugged::ReferenceError
-            # Omit invalid branch
+      def branches
+        gitaly_migrate(:branches) do |is_enabled|
+          if is_enabled
+            gitaly_ref_client.branches
+          else
+            branches_filter
           end
-        end.compact
-
-        sort_branches(branches, sort_by)
+        end
       end
 
       def reload_rugged
@@ -107,17 +107,18 @@ module Gitlab
         reload_rugged if force_reload
 
         rugged_ref = rugged.branches[name]
-        Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) if rugged_ref
+        if rugged_ref
+          target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
+          Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
+        end
       end
 
       def local_branches(sort_by: nil)
         gitaly_migrate(:local_branches) do |is_enabled|
           if is_enabled
-            gitaly_ref_client.local_branches(sort_by: sort_by).map do |gitaly_branch|
-              Gitlab::Git::Branch.new(self, gitaly_branch.name, gitaly_branch)
-            end
+            gitaly_ref_client.local_branches(sort_by: sort_by)
           else
-            branches(filter: :local, sort_by: sort_by)
+            branches_filter(filter: :local, sort_by: sort_by)
           end
         end
       end
@@ -164,20 +165,16 @@ module Gitlab
       end
 
       # Returns an Array of Tags
+      #
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/390
       def tags
-        rugged.references.each("refs/tags/*").map do |ref|
-          message = nil
-
-          if ref.target.is_a?(Rugged::Tag::Annotation)
-            tag_message = ref.target.message
-
-            if tag_message.respond_to?(:chomp)
-              message = tag_message.chomp
-            end
+        gitaly_migrate(:tags) do |is_enabled|
+          if is_enabled
+            tags_from_gitaly
+          else
+            tags_from_rugged
           end
-
-          Gitlab::Git::Tag.new(self, ref.name, ref.target, message)
-        end.sort_by(&:name)
+        end
       end
 
       # Returns true if the given tag exists
@@ -206,21 +203,10 @@ module Gitlab
         branch_names + tag_names
       end
 
-      # Deprecated. Will be removed in 5.2
-      def heads
-        rugged.references.each("refs/heads/*").map do |head|
-          Gitlab::Git::Ref.new(self, head.name, head.target)
-        end.sort_by(&:name)
-      end
-
       def has_commits?
         !empty?
       end
 
-      def repo_exists?
-        !!rugged
-      end
-
       # Discovers the default branch based on the repository's available branches
       #
       # - If no branches are present, returns nil
@@ -299,28 +285,6 @@ module Gitlab
         (size.to_f / 1024).round(2)
       end
 
-      # Returns an array of BlobSnippets for files at the specified +ref+ that
-      # contain the +query+ string.
-      def search_files(query, ref = nil)
-        greps = []
-        ref ||= root_ref
-
-        populated_index(ref).each do |entry|
-          # Discard submodules
-          next if submodule?(entry)
-
-          blob = Gitlab::Git::Blob.raw(self, entry[:oid])
-
-          # Skip binary files
-          next if blob.data.encoding == Encoding::ASCII_8BIT
-
-          blob.load_all_data!(self)
-          greps += build_greps(blob.data, query, ref, entry[:path])
-        end
-
-        greps
-      end
-
       # Use the Rugged Walker API to build an array of commits.
       #
       # Usage.
@@ -333,85 +297,10 @@ module Gitlab
       #   )
       #
       def log(options)
-        default_options = {
-          limit: 10,
-          offset: 0,
-          path: nil,
-          follow: false,
-          skip_merges: false,
-          disable_walk: false,
-          after: nil,
-          before: nil
-        }
-
-        options = default_options.merge(options)
-        options[:limit] ||= 0
-        options[:offset] ||= 0
-        actual_ref = options[:ref] || root_ref
-        begin
-          sha = sha_from_ref(actual_ref)
-        rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError
-          # Return an empty array if the ref wasn't found
-          return []
-        end
-
-        if log_using_shell?(options)
-          log_by_shell(sha, options)
-        else
-          log_by_walk(sha, options)
-        end
-      end
-
-      def log_using_shell?(options)
-        options[:path].present? ||
-          options[:disable_walk] ||
-          options[:skip_merges] ||
-          options[:after] ||
-          options[:before]
-      end
-
-      def log_by_walk(sha, options)
-        walk_options = {
-          show: sha,
-          sort: Rugged::SORT_NONE,
-          limit: options[:limit],
-          offset: options[:offset]
-        }
-        Rugged::Walker.walk(rugged, walk_options).to_a
-      end
-
-      def log_by_shell(sha, options)
-        limit = options[:limit].to_i
-        offset = options[:offset].to_i
-        use_follow_flag = options[:follow] && options[:path].present?
-
-        # We will perform the offset in Ruby because --follow doesn't play well with --skip.
-        # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520
-        offset_in_ruby = use_follow_flag && options[:offset].present?
-        limit += offset if offset_in_ruby
-
-        cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log]
-        cmd << "--max-count=#{limit}"
-        cmd << '--format=%H'
-        cmd << "--skip=#{offset}" unless offset_in_ruby
-        cmd << '--follow' if use_follow_flag
-        cmd << '--no-merges' if options[:skip_merges]
-        cmd << "--after=#{options[:after].iso8601}" if options[:after]
-        cmd << "--before=#{options[:before].iso8601}" if options[:before]
-        cmd << sha
-
-        # :path can be a string or an array of strings
-        if options[:path].present?
-          cmd << '--'
-          cmd += Array(options[:path])
-        end
-
-        raw_output = IO.popen(cmd) { |io| io.read }
-        lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines
-
-        lines.map! { |c| Rugged::Commit.new(rugged, c.strip) }
+        raw_log(options).map { |c| Commit.decorate(c) }
       end
 
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/382
       def count_commits(options)
         cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list]
         cmd << "--after=#{options[:after].iso8601}" if options[:after]
@@ -456,7 +345,7 @@ module Gitlab
 
       # Counts the amount of commits between `from` and `to`.
       def count_commits_between(from, to)
-        commits_between(from, to).size
+        Commit.between(self, from, to).size
       end
 
       # Returns the SHA of the most recent common ancestor of +from+ and +to+
@@ -464,6 +353,13 @@ module Gitlab
         rugged.merge_base(from, to)
       end
 
+      # Gitaly note: JV: check gitlab-ee before removing this method.
+      def rugged_is_ancestor?(ancestor_id, descendant_id)
+        return false if ancestor_id.nil? || descendant_id.nil?
+
+        merge_base_commit(ancestor_id, descendant_id) == ancestor_id
+      end
+
       # Returns true is +from+ is direct ancestor to +to+, otherwise false
       def is_ancestor?(from, to)
         gitaly_commit_client.is_ancestor(from, to)
@@ -494,70 +390,6 @@ module Gitlab
         end
       end
 
-      # Returns commits collection
-      #
-      # Ex.
-      #   repo.find_commits(
-      #     ref: 'master',
-      #     max_count: 10,
-      #     skip: 5,
-      #     order: :date
-      #   )
-      #
-      #   +options+ is a Hash of optional arguments to git
-      #     :ref is the ref from which to begin (SHA1 or name)
-      #     :contains is the commit contained by the refs from which to begin (SHA1 or name)
-      #     :max_count is the maximum number of commits to fetch
-      #     :skip is the number of commits to skip
-      #     :order is the commits order and allowed value is :none (default), :date,
-      #        :topo, or any combination of them (in an array). Commit ordering types
-      #        are documented here:
-      #        http://www.rubydoc.info/github/libgit2/rugged/Rugged#SORT_NONE-constant)
-      #
-      def find_commits(options = {})
-        actual_options = options.dup
-
-        allowed_options = [:ref, :max_count, :skip, :contains, :order]
-
-        actual_options.keep_if do |key|
-          allowed_options.include?(key)
-        end
-
-        default_options = { skip: 0 }
-        actual_options = default_options.merge(actual_options)
-
-        walker = Rugged::Walker.new(rugged)
-
-        if actual_options[:ref]
-          walker.push(rugged.rev_parse_oid(actual_options[:ref]))
-        elsif actual_options[:contains]
-          branches_contains(actual_options[:contains]).each do |branch|
-            walker.push(branch.target_id)
-          end
-        else
-          rugged.references.each("refs/heads/*") do |ref|
-            walker.push(ref.target_id)
-          end
-        end
-
-        sort_type = rugged_sort_type(actual_options[:order])
-        walker.sorting(sort_type)
-
-        commits = []
-        offset = actual_options[:skip]
-        limit = actual_options[:max_count]
-        walker.each(offset: offset, limit: limit) do |commit|
-          gitlab_commit = Gitlab::Git::Commit.decorate(commit)
-          commits.push(gitlab_commit)
-        end
-
-        walker.reset
-
-        commits
-      rescue Rugged::OdbError
-        []
-      end
-
       # Returns branch names collection that contains the special commit(SHA1
       # or name)
       #
@@ -613,56 +445,43 @@ module Gitlab
         rugged.rev_parse(oid_or_ref_name)
       end
 
-      # Return hash with submodules info for this repository
+      # Returns url for submodule
       #
       # Ex.
-      #   {
-      #     "rack"  => {
-      #       "id" => "c67be4624545b4263184c4a0e8f887efd0a66320",
-      #       "path" => "rack",
-      #       "url" => "git://github.com/chneukirchen/rack.git"
-      #     },
-      #     "encoding" => {
-      #       "id" => ....
-      #     }
-      #   }
+      #   @repository.submodule_url_for('master', 'rack')
+      #   # => git@localhost:rack.git
       #
-      def submodules(ref)
-        commit = rev_parse_target(ref)
-        return {} unless commit
-
-        begin
-          content = blob_content(commit, ".gitmodules")
-        rescue InvalidBlobName
-          return {}
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/329
+      def submodule_url_for(ref, path)
+        Gitlab::GitalyClient.migrate(:submodule_url_for) do |is_enabled|
+          if is_enabled
+            gitaly_submodule_url_for(ref, path)
+          else
+            if submodules(ref).any?
+              submodule = submodules(ref)[path]
+              submodule['url'] if submodule
+            end
+          end
         end
-
-        parse_gitmodules(commit, content)
       end
 
       # Return total commits count accessible from passed ref
+      #
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/330
       def commit_count(ref)
-        walker = Rugged::Walker.new(rugged)
-        walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE)
-        oid = rugged.rev_parse_oid(ref)
-        walker.push(oid)
-        walker.count
+        gitaly_migrate(:commit_count) do |is_enabled|
+          if is_enabled
+            gitaly_commit_client.commit_count(ref)
+          else
+            walker = Rugged::Walker.new(rugged)
+            walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE)
+            oid = rugged.rev_parse_oid(ref)
+            walker.push(oid)
+            walker.count
+          end
+        end
       end
 
-      # Sets HEAD to the commit specified by +ref+; +ref+ can be a branch or
-      # tag name or a commit SHA.  Valid +reset_type+ values are:
-      #
-      #  [:soft]
-      #    the head will be moved to the commit.
-      #  [:mixed]
-      #    will trigger a +:soft+ reset, plus the index will be replaced
-      #    with the content of the commit tree.
-      #  [:hard]
-      #    will trigger a +:mixed+ reset and the working directory will be
-      #    replaced with the content of the index. (Untracked and ignored files
-      #    will be left alone)
-      delegate :reset, to: :rugged
-
       # Mimic the `git clean` command and recursively delete untracked files.
       # Valid keys that can be passed in the +options+ hash are:
       #
@@ -687,154 +506,6 @@ module Gitlab
         # TODO: implement this method
       end
 
-      # Check out the specified ref. Valid options are:
-      #
-      #  :b - Create a new branch at +start_point+ and set HEAD to the new
-      #       branch.
-      #
-      #  * These options are passed to the Rugged::Repository#checkout method:
-      #
-      #  :progress ::
-      #    A callback that will be executed for checkout progress notifications.
-      #    Up to 3 parameters are passed on each execution:
-      #
-      #    - The path to the last updated file (or +nil+ on the very first
-      #      invocation).
-      #    - The number of completed checkout steps.
-      #    - The number of total checkout steps to be performed.
-      #
-      #  :notify ::
-      #    A callback that will be executed for each checkout notification
-      #    types specified with +:notify_flags+. Up to 5 parameters are passed
-      #    on each execution:
-      #
-      #    - An array containing the +:notify_flags+ that caused the callback
-      #      execution.
-      #    - The path of the current file.
-      #    - A hash describing the baseline blob (or +nil+ if it does not
-      #      exist).
-      #    - A hash describing the target blob (or +nil+ if it does not exist).
-      #    - A hash describing the workdir blob (or +nil+ if it does not
-      #      exist).
-      #
-      #  :strategy ::
-      #    A single symbol or an array of symbols representing the strategies
-      #    to use when performing the checkout. Possible values are:
-      #
-      #    :none ::
-      #      Perform a dry run (default).
-      #
-      #    :safe ::
-      #      Allow safe updates that cannot overwrite uncommitted data.
-      #
-      #    :safe_create ::
-      #      Allow safe updates plus creation of missing files.
-      #
-      #    :force ::
-      #      Allow all updates to force working directory to look like index.
-      #
-      #    :allow_conflicts ::
-      #      Allow checkout to make safe updates even if conflicts are found.
-      #
-      #    :remove_untracked ::
-      #      Remove untracked files not in index (that are not ignored).
-      #
-      #    :remove_ignored ::
-      #      Remove ignored files not in index.
-      #
-      #    :update_only ::
-      #      Only update existing files, don't create new ones.
-      #
-      #    :dont_update_index ::
-      #      Normally checkout updates index entries as it goes; this stops
-      #      that.
-      #
-      #    :no_refresh ::
-      #      Don't refresh index/config/etc before doing checkout.
-      #
-      #    :disable_pathspec_match ::
-      #      Treat pathspec as simple list of exact match file paths.
-      #
-      #    :skip_locked_directories ::
-      #      Ignore directories in use, they will be left empty.
-      #
-      #    :skip_unmerged ::
-      #      Allow checkout to skip unmerged files (NOT IMPLEMENTED).
-      #
-      #    :use_ours ::
-      #      For unmerged files, checkout stage 2 from index (NOT IMPLEMENTED).
-      #
-      #    :use_theirs ::
-      #      For unmerged files, checkout stage 3 from index (NOT IMPLEMENTED).
-      #
-      #    :update_submodules ::
-      #      Recursively checkout submodules with same options (NOT
-      #      IMPLEMENTED).
-      #
-      #    :update_submodules_if_changed ::
-      #      Recursively checkout submodules if HEAD moved in super repo (NOT
-      #      IMPLEMENTED).
-      #
-      #  :disable_filters ::
-      #    If +true+, filters like CRLF line conversion will be disabled.
-      #
-      #  :dir_mode ::
-      #    Mode for newly created directories. Default: +0755+.
-      #
-      #  :file_mode ::
-      #    Mode for newly created files. Default: +0755+ or +0644+.
-      #
-      #  :file_open_flags ::
-      #    Mode for opening files. Default:
-      #    <code>IO::CREAT | IO::TRUNC | IO::WRONLY</code>.
-      #
-      #  :notify_flags ::
-      #    A single symbol or an array of symbols representing the cases in
-      #    which the +:notify+ callback should be invoked. Possible values are:
-      #
-      #    :none ::
-      #      Do not invoke the +:notify+ callback (default).
-      #
-      #    :conflict ::
-      #      Invoke the callback for conflicting paths.
-      #
-      #    :dirty ::
-      #      Invoke the callback for "dirty" files, i.e. those that do not need
-      #      an update but no longer match the baseline.
-      #
-      #    :updated ::
-      #      Invoke the callback for any file that was changed.
-      #
-      #    :untracked ::
-      #      Invoke the callback for untracked files.
-      #
-      #    :ignored ::
-      #      Invoke the callback for ignored files.
-      #
-      #    :all ::
-      #      Invoke the callback for all these cases.
-      #
-      #  :paths ::
-      #    A glob string or an array of glob strings specifying which paths
-      #    should be taken into account for the checkout operation. +nil+ will
-      #    match all files.  Default: +nil+.
-      #
-      #  :baseline ::
-      #    A Rugged::Tree that represents the current, expected contents of the
-      #    workdir.  Default: +HEAD+.
-      #
-      #  :target_directory ::
-      #    A path to an alternative workdir directory in which the checkout
-      #    should be performed.
-      def checkout(ref, options = {}, start_point = "HEAD")
-        if options[:b]
-          rugged.branches.create(ref, start_point)
-          options.delete(:b)
-        end
-        default_options = { strategy: [:recreate_missing, :safe] }
-        rugged.checkout(ref, default_options.merge(options))
-      end
-
       # Delete the specified branch from the repository
       def delete_branch(branch_name)
         rugged.branches.delete(branch_name)
@@ -847,7 +518,8 @@ module Gitlab
       #   create_branch("other-feature", "master")
       def create_branch(ref, start_point = "HEAD")
         rugged_ref = rugged.branches.create(ref, start_point)
-        Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target)
+        target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
+        Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
       rescue Rugged::ReferenceError => e
         raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/
         raise InvalidRef.new("Invalid reference #{start_point}")
@@ -906,6 +578,7 @@ module Gitlab
       # Ex.
       #   repo.ls_files('master')
       #
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/327
       def ls_files(ref)
         actual_ref = ref || root_ref
 
@@ -932,6 +605,7 @@ module Gitlab
         raw_output.compact
       end
 
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/328
       def copy_gitattributes(ref)
         begin
           commit = lookup(ref)
@@ -962,11 +636,6 @@ module Gitlab
         end
       end
 
-      # Checks if the blob should be diffable according to its attributes
-      def diffable?(blob)
-        attributes(blob.path).fetch('diff') { blob.text? }
-      end
-
       # Returns the Git attributes for the given file path.
       #
       # See `Gitlab::Git::Attributes` for more information.
@@ -974,12 +643,177 @@ module Gitlab
         @attributes.attributes(path)
       end
 
+      def languages(ref = nil)
+        Gitlab::GitalyClient.migrate(:commit_languages) do |is_enabled|
+          if is_enabled
+            gitaly_commit_client.languages(ref)
+          else
+            ref ||= rugged.head.target_id
+            languages = Linguist::Repository.new(rugged, ref).languages
+            total = languages.map(&:last).sum
+
+            languages = languages.map do |language|
+              name, share = language
+              color = Linguist::Language[name].color || "##{Digest::SHA256.hexdigest(name)[0...6]}"
+              {
+                value: (share.to_f * 100 / total).round(2),
+                label: name,
+                color: color,
+                highlight: color
+              }
+            end
+
+            languages.sort do |x, y|
+              y[:value] <=> x[:value]
+            end
+          end
+        end
+      end
+
       def gitaly_repository
         Gitlab::GitalyClient::Util.repository(@storage, @relative_path)
       end
 
+      def gitaly_ref_client
+        @gitaly_ref_client ||= Gitlab::GitalyClient::RefService.new(self)
+      end
+
+      def gitaly_commit_client
+        @gitaly_commit_client ||= Gitlab::GitalyClient::CommitService.new(self)
+      end
+
+      def gitaly_repository_client
+        @gitaly_repository_client ||= Gitlab::GitalyClient::RepositoryService.new(self)
+      end
+
       private
 
+      # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'.
+      def branches_filter(filter: nil, sort_by: nil)
+        branches = rugged.branches.each(filter).map do |rugged_ref|
+          begin
+            target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
+            Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
+          rescue Rugged::ReferenceError
+            # Omit invalid branch
+          end
+        end.compact
+
+        sort_branches(branches, sort_by)
+      end
+
+      def raw_log(options)
+        default_options = {
+          limit: 10,
+          offset: 0,
+          path: nil,
+          follow: false,
+          skip_merges: false,
+          disable_walk: false,
+          after: nil,
+          before: nil
+        }
+
+        options = default_options.merge(options)
+        options[:limit] ||= 0
+        options[:offset] ||= 0
+        actual_ref = options[:ref] || root_ref
+        begin
+          sha = sha_from_ref(actual_ref)
+        rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError
+          # Return an empty array if the ref wasn't found
+          return []
+        end
+
+        if log_using_shell?(options)
+          log_by_shell(sha, options)
+        else
+          log_by_walk(sha, options)
+        end
+      end
+
+      def log_using_shell?(options)
+        options[:path].present? ||
+          options[:disable_walk] ||
+          options[:skip_merges] ||
+          options[:after] ||
+          options[:before]
+      end
+
+      def log_by_walk(sha, options)
+        walk_options = {
+          show: sha,
+          sort: Rugged::SORT_NONE,
+          limit: options[:limit],
+          offset: options[:offset]
+        }
+        Rugged::Walker.walk(rugged, walk_options).to_a
+      end
+
+      # Gitaly note: JV: although #log_by_shell shells out to Git I think the
+      # complexity is such that we should migrate it as Ruby before trying to
+      # do it in Go.
+      def log_by_shell(sha, options)
+        limit = options[:limit].to_i
+        offset = options[:offset].to_i
+        use_follow_flag = options[:follow] && options[:path].present?
+
+        # We will perform the offset in Ruby because --follow doesn't play well with --skip.
+        # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520
+        offset_in_ruby = use_follow_flag && options[:offset].present?
+        limit += offset if offset_in_ruby
+
+        cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log]
+        cmd << "--max-count=#{limit}"
+        cmd << '--format=%H'
+        cmd << "--skip=#{offset}" unless offset_in_ruby
+        cmd << '--follow' if use_follow_flag
+        cmd << '--no-merges' if options[:skip_merges]
+        cmd << "--after=#{options[:after].iso8601}" if options[:after]
+        cmd << "--before=#{options[:before].iso8601}" if options[:before]
+        cmd << sha
+
+        # :path can be a string or an array of strings
+        if options[:path].present?
+          cmd << '--'
+          cmd += Array(options[:path])
+        end
+
+        raw_output = IO.popen(cmd) { |io| io.read }
+        lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines
+
+        lines.map! { |c| Rugged::Commit.new(rugged, c.strip) }
+      end
+
+      # We are trying to deprecate this method because it does a lot of work
+      # but it seems to be used only to look up submodule URL's.
+      # https://gitlab.com/gitlab-org/gitaly/issues/329
+      def submodules(ref)
+        commit = rev_parse_target(ref)
+        return {} unless commit
+
+        begin
+          content = blob_content(commit, ".gitmodules")
+        rescue InvalidBlobName
+          return {}
+        end
+
+        parser = GitmodulesParser.new(content)
+        fill_submodule_ids(commit, parser.parse)
+      end
+
+      def gitaly_submodule_url_for(ref, path)
+        # We don't care about the contents so 1 byte is enough. Can't request 0 bytes, 0 means unlimited.
+        commit_object = gitaly_commit_client.tree_entry(ref, path, 1)
+
+        return unless commit_object && commit_object.type == :COMMIT
+
+        gitmodules = gitaly_commit_client.tree_entry(ref, '.gitmodules', Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE)
+        found_module = GitmodulesParser.new(gitmodules.data).parse[path]
+
+        found_module && found_module['url']
+      end
+
       def alternate_object_directories
         Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_DIRECTORIES_VARIABLES).compact
       end
@@ -1003,42 +837,19 @@ module Gitlab
         end
       end
 
-      # Parses the contents of a .gitmodules file and returns a hash of
-      # submodule information.
-      def parse_gitmodules(commit, content)
-        modules = {}
-
-        name = nil
-        content.each_line do |line|
-          case line.strip
-          when /\A\[submodule "(?<name>[^"]+)"\]\z/ # Submodule header
-            name = $~[:name]
-            modules[name] = {}
-          when /\A(?<key>\w+)\s*=\s*(?<value>.*)\z/ # Key/value pair
-            key = $~[:key]
-            value = $~[:value].chomp
-
-            next unless name && modules[name]
-
-            modules[name][key] = value
-
-            if key == 'path'
-              begin
-                modules[name]['id'] = blob_content(commit, value)
-              rescue InvalidBlobName
-                # The current entry is invalid
-                modules.delete(name)
-                name = nil
-              end
-            end
-          when /\A#/ # Comment
-            next
-          else # Invalid line
-            name = nil
+      # Fill in the 'id' field of a submodule hash from its values
+      # as-of +commit+. Return a Hash consisting only of entries
+      # from the submodule hash for which the 'id' field is filled.
+      def fill_submodule_ids(commit, submodule_data)
+        submodule_data.each do |path, data|
+          id = begin
+            blob_content(commit, path)
+          rescue InvalidBlobName
+            nil
           end
+          data['id'] = id
         end
-
-        modules
+        submodule_data.select { |path, data| data['id'] }
       end
 
       # Returns true if +commit+ introduced changes to +path+, using commit
@@ -1145,73 +956,6 @@ module Gitlab
         index
       end
 
-      # Return an array of BlobSnippets for lines in +file_contents+ that match
-      # +query+
-      def build_greps(file_contents, query, ref, filename)
-        # The file_contents string is potentially huge so we make sure to loop
-        # through it one line at a time. This gives Ruby the chance to GC lines
-        # we are not interested in.
-        #
-        # We need to do a little extra work because we are not looking for just
-        # the lines that matches the query, but also for the context
-        # (surrounding lines). We will use Enumerable#each_cons to efficiently
-        # loop through the lines while keeping surrounding lines on hand.
-        #
-        # First, we turn "foo\nbar\nbaz" into
-        # [
-        #  [nil, -3], [nil, -2], [nil, -1],
-        #  ['foo', 0], ['bar', 1], ['baz', 3],
-        #  [nil, 4], [nil, 5], [nil, 6]
-        # ]
-        lines_with_index = Enumerator.new do |yielder|
-          # Yield fake 'before' lines for the first line of file_contents
-          (-SEARCH_CONTEXT_LINES..-1).each do |i|
-            yielder.yield [nil, i]
-          end
-
-          # Yield the actual file contents
-          count = 0
-          file_contents.each_line do |line|
-            line.chomp!
-            yielder.yield [line, count]
-            count += 1
-          end
-
-          # Yield fake 'after' lines for the last line of file_contents
-          (count + 1..count + SEARCH_CONTEXT_LINES).each do |i|
-            yielder.yield [nil, i]
-          end
-        end
-
-        greps = []
-
-        # Loop through consecutive blocks of lines with indexes
-        lines_with_index.each_cons(2 * SEARCH_CONTEXT_LINES + 1) do |line_block|
-          # Get the 'middle' line and index from the block
-          line, _ = line_block[SEARCH_CONTEXT_LINES]
-
-          next unless line && line.match(/#{Regexp.escape(query)}/i)
-
-          # Yay, 'line' contains a match!
-          # Get an array with just the context lines (no indexes)
-          match_with_context = line_block.map(&:first)
-          # Remove 'nil' lines in case we are close to the first or last line
-          match_with_context.compact!
-
-          # Get the line number (1-indexed) of the first context line
-          first_context_line_number = line_block[0][1] + 1
-
-          greps << Gitlab::Git::BlobSnippet.new(
-            ref,
-            match_with_context,
-            first_context_line_number,
-            filename
-          )
-        end
-
-        greps
-      end
-
       # Return the Rugged patches for the diff between +from+ and +to+.
       def diff_patches(from, to, options = {}, *paths)
         options ||= {}
@@ -1240,12 +984,25 @@ module Gitlab
         end
       end
 
-      def gitaly_ref_client
-        @gitaly_ref_client ||= Gitlab::GitalyClient::Ref.new(self)
+      def tags_from_rugged
+        rugged.references.each("refs/tags/*").map do |ref|
+          message = nil
+
+          if ref.target.is_a?(Rugged::Tag::Annotation)
+            tag_message = ref.target.message
+
+            if tag_message.respond_to?(:chomp)
+              message = tag_message.chomp
+            end
+          end
+
+          target_commit = Gitlab::Git::Commit.find(self, ref.target)
+          Gitlab::Git::Tag.new(self, ref.name, ref.target, target_commit, message)
+        end.sort_by(&:name)
       end
 
-      def gitaly_commit_client
-        @gitaly_commit_client ||= Gitlab::GitalyClient::Commit.new(self)
+      def tags_from_gitaly
+        gitaly_ref_client.tags
       end
 
       def gitaly_migrate(method, &block)
@@ -1255,20 +1012,6 @@ module Gitlab
       rescue GRPC::BadStatus => e
         raise CommandError.new(e)
       end
-
-      # Returns the `Rugged` sorting type constant for one or more given
-      # sort types. Valid keys are `:none`, `:topo`, and `:date`, or an array
-      # containing more than one of them. `:date` uses a combination of date and
-      # topological sorting to closer mimic git's native ordering.
-      def rugged_sort_type(sort_type)
-        @rugged_sort_types ||= {
-          none: Rugged::SORT_NONE,
-          topo: Rugged::SORT_TOPO,
-          date: Rugged::SORT_DATE | Rugged::SORT_TOPO
-        }
-
-        @rugged_sort_types.fetch(sort_type, Rugged::SORT_NONE)
-      end
     end
   end
 end
diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb
index a16b0ed76f4..2b5785a1f08 100644
--- a/lib/gitlab/git/rev_list.rb
+++ b/lib/gitlab/git/rev_list.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: will probably be migrated indirectly by migrating the call sites.
+
 module Gitlab
   module Git
     class RevList
@@ -15,6 +17,8 @@ module Gitlab
       end
 
       # This methods returns an array of missed references
+      #
+      # Should become obsolete after https://gitlab.com/gitlab-org/gitaly/issues/348.
       def missed_ref
         execute([*base_args, '--max-count=1', oldrev, "^#{newrev}"])
       end
diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb
index b5342c3d310..bc4e160dce9 100644
--- a/lib/gitlab/git/tag.rb
+++ b/lib/gitlab/git/tag.rb
@@ -1,10 +1,12 @@
+# Gitaly note: JV: no RPC's here.
+#
 module Gitlab
   module Git
     class Tag < Ref
       attr_reader :object_sha
 
-      def initialize(repository, name, target, message = nil)
-        super(repository, name, target)
+      def initialize(repository, name, target, target_commit, message = nil)
+        super(repository, name, target, target_commit)
 
         @message = message
       end
diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb
index b9afa05c819..8e959c57c7c 100644
--- a/lib/gitlab/git/tree.rb
+++ b/lib/gitlab/git/tree.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: needs 1 RPC, migration is in progress.
+
 module Gitlab
   module Git
     class Tree
@@ -10,36 +12,23 @@ module Gitlab
         # Get list of tree objects
         # for repository based on commit sha and path
         # Uses rugged for raw objects
+        #
+        # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/320
         def where(repository, sha, path = nil)
           path = nil if path == '' || path == '/'
 
-          commit = repository.lookup(sha)
-          root_tree = commit.tree
-
-          tree = if path
-                   id = find_id_by_path(repository, root_tree.oid, path)
-                   if id
-                     repository.lookup(id)
-                   else
-                     []
-                   end
-                 else
-                   root_tree
-                 end
-
-          tree.map do |entry|
-            new(
-              id: entry[:oid],
-              root_id: root_tree.oid,
-              name: entry[:name],
-              type: entry[:type],
-              mode: entry[:filemode].to_s(8),
-              path: path ? File.join(path, entry[:name]) : entry[:name],
-              commit_id: sha
-            )
+          Gitlab::GitalyClient.migrate(:tree_entries) do |is_enabled|
+            if is_enabled
+              client = Gitlab::GitalyClient::CommitService.new(repository)
+              client.tree_entries(repository, sha, path)
+            else
+              tree_entries_from_rugged(repository, sha, path)
+            end
           end
         end
 
+        private
+
         # Recursive search of tree id for path
         #
         # Ex.
@@ -68,6 +57,34 @@ module Gitlab
             entry[:oid]
           end
         end
+
+        def tree_entries_from_rugged(repository, sha, path)
+          commit = repository.lookup(sha)
+          root_tree = commit.tree
+
+          tree = if path
+                   id = find_id_by_path(repository, root_tree.oid, path)
+                   if id
+                     repository.lookup(id)
+                   else
+                     []
+                   end
+                 else
+                   root_tree
+                 end
+
+          tree.map do |entry|
+            new(
+              id: entry[:oid],
+              root_id: root_tree.oid,
+              name: entry[:name],
+              type: entry[:type],
+              mode: entry[:filemode].to_s(8),
+              path: path ? File.join(path, entry[:name]) : entry[:name],
+              commit_id: sha
+            )
+          end
+        end
       end
 
       def initialize(options)
@@ -80,6 +97,10 @@ module Gitlab
         encode! @name
       end
 
+      def path
+        encode! @path
+      end
+
       def dir?
         type == :tree
       end
diff --git a/lib/gitlab/git/util.rb b/lib/gitlab/git/util.rb
index 7973da2e8f8..4708f22dcb3 100644
--- a/lib/gitlab/git/util.rb
+++ b/lib/gitlab/git/util.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: no RPC's here.
+
 module Gitlab
   module Git
     module Util