1 files changed, 225 insertions, 151 deletions

diff --git a/app/models/repository.rb b/app/models/repository.rb
index 91bdafdac99..063dc74021d 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -11,6 +11,20 @@ class Repository
 
   attr_accessor :path_with_namespace, :project
 
+  def self.storages
+    Gitlab.config.repositories.storages
+  end
+
+  def self.remove_storage_from_path(repo_path)
+    storages.find do |_, storage_path|
+      if repo_path.start_with?(storage_path)
+        return repo_path.sub(storage_path, '')
+      end
+    end
+
+    repo_path
+  end
+
   def initialize(path_with_namespace, project)
     @path_with_namespace = path_with_namespace
     @project = project
@@ -70,15 +84,17 @@ class Repository
 
   def commit(ref = 'HEAD')
     return nil unless exists?
+
     commit =
       if ref.is_a?(Gitlab::Git::Commit)
         ref
       else
         Gitlab::Git::Commit.find(raw_repository, ref)
       end
+
     commit = ::Commit.new(commit, @project) if commit
     commit
-  rescue Rugged::OdbError
+  rescue Rugged::OdbError, Rugged::TreeError
     nil
   end
 
@@ -109,19 +125,37 @@ class Repository
   end
 
   def find_commits_by_message(query, ref = nil, path = nil, limit = 1000, offset = 0)
+    unless exists? && has_visible_content? && query.present?
+      return []
+    end
+
     ref ||= root_ref
 
-    # Limited to 1000 commits for now, could be parameterized?
-    args = %W(#{Gitlab.config.git.bin_path} log #{ref} --pretty=%H --skip #{offset} --max-count #{limit} --grep=#{query})
+    args = %W(
+      #{Gitlab.config.git.bin_path} log #{ref} --pretty=%H --skip #{offset}
+      --max-count #{limit} --grep=#{query} --regexp-ignore-case
+    )
     args = args.concat(%W(-- #{path})) if path.present?
 
-    git_log_results = Gitlab::Popen.popen(args, path_to_repo).first.lines.map(&:chomp)
-    commits = git_log_results.map { |c| commit(c) }
-    commits
+    git_log_results = Gitlab::Popen.popen(args, path_to_repo).first.lines
+    git_log_results.map { |c| commit(c.chomp) }.compact
   end
 
-  def find_branch(name)
-    raw_repository.branches.find { |branch| branch.name == name }
+  def find_branch(name, fresh_repo: true)
+    # Since the Repository object may have in-memory index changes, invalidating the memoized Repository object may
+    # cause unintended side effects. Because finding a branch is a read-only operation, we can safely instantiate
+    # a new repo here to ensure a consistent state to avoid a libgit2 bug where concurrent access (e.g. via git gc)
+    # may cause the branch to "disappear" erroneously or have the wrong SHA.
+    #
+    # See: https://github.com/libgit2/libgit2/issues/1534 and https://gitlab.com/gitlab-org/gitlab-ce/issues/15392
+    raw_repo =
+      if fresh_repo
+        Gitlab::Git::Repository.new(path_to_repo)
+      else
+        raw_repository
+      end
+
+    raw_repo.find_branch(name)
   end
 
   def find_tag(name)
@@ -136,7 +170,7 @@ class Repository
     return false unless target
 
     GitHooksService.new.execute(user, path_to_repo, oldrev, target, ref) do
-      rugged.branches.create(branch_name, target)
+      update_ref!(ref, target, oldrev)
     end
 
     after_create_branch
@@ -163,12 +197,12 @@ class Repository
     before_remove_branch
 
     branch = find_branch(branch_name)
-    oldrev = branch.try(:target).try(:id)
+    oldrev = branch.try(:dereferenced_target).try(:id)
     newrev = Gitlab::Git::BLANK_SHA
     ref    = Gitlab::Git::BRANCH_REF_PREFIX + branch_name
 
     GitHooksService.new.execute(user, path_to_repo, oldrev, newrev, ref) do
-      rugged.branches.delete(branch_name)
+      update_ref!(ref, newrev, oldrev)
     end
 
     after_remove_branch
@@ -200,6 +234,23 @@ class Repository
 
   def ref_exists?(ref)
     rugged.references.exist?(ref)
+  rescue Rugged::ReferenceError
+    false
+  end
+
+  def update_ref!(name, newrev, oldrev)
+    # We use 'git update-ref' because libgit2/rugged currently does not
+    # offer 'compare and swap' ref updates. Without compare-and-swap we can
+    # (and have!) accidentally reset the ref to an earlier state, clobbering
+    # commits. See also https://github.com/libgit2/libgit2/issues/1534.
+    command = %W(#{Gitlab.config.git.bin_path} update-ref --stdin -z)
+    _, status = Gitlab::Popen.popen(command, path_to_repo) do |stdin|
+      stdin.write("update #{name}\x00#{newrev}\x00#{oldrev}\x00")
+    end
+
+    return if status.zero?
+
+    raise CommitError.new("Could not update branch #{name.sub('refs/heads/', '')}. Please refresh and try again.")
   end
 
   # Makes sure a commit is kept around when Git garbage collection runs.
@@ -223,11 +274,7 @@ class Repository
   end
 
   def kept_around?(sha)
-    begin
-      ref_exists?(keep_around_ref_name(sha))
-    rescue Rugged::ReferenceError
-      false
-    end
+    ref_exists?(keep_around_ref_name(sha))
   end
 
   def tag_names
@@ -264,10 +311,10 @@ class Repository
       # Rugged seems to throw a `ReferenceError` when given branch_names rather
       # than SHA-1 hashes
       number_commits_behind = raw_repository.
-        count_commits_between(branch.target.sha, root_ref_hash)
+        count_commits_between(branch.dereferenced_target.sha, root_ref_hash)
 
       number_commits_ahead = raw_repository.
-        count_commits_between(root_ref_hash, branch.target.sha)
+        count_commits_between(root_ref_hash, branch.dereferenced_target.sha)
 
       { behind: number_commits_behind, ahead: number_commits_ahead }
     end
@@ -386,6 +433,17 @@ class Repository
     @exists = nil
   end
 
+  # expire cache that doesn't depend on repository data (when expiring)
+  def expire_content_cache
+    expire_tags_cache
+    expire_tag_count_cache
+    expire_branches_cache
+    expire_branch_count_cache
+    expire_root_ref_cache
+    expire_emptiness_caches
+    expire_exists_cache
+  end
+
   # Runs code after a repository has been created.
   def after_create
     expire_exists_cache
@@ -401,14 +459,7 @@ class Repository
 
     expire_cache if exists?
 
-    # expire cache that don't depend on repository data (when expiring)
-    expire_tags_cache
-    expire_tag_count_cache
-    expire_branches_cache
-    expire_branch_count_cache
-    expire_root_ref_cache
-    expire_emptiness_caches
-    expire_exists_cache
+    expire_content_cache
 
     repository_event(:remove_repository)
   end
@@ -440,14 +491,13 @@ class Repository
   end
 
   def before_import
-    expire_emptiness_caches
-    expire_exists_cache
+    expire_content_cache
   end
 
   # Runs code after a repository has been forked/imported.
   def after_import
-    expire_emptiness_caches
-    expire_exists_cache
+    expire_content_cache
+    build_cache
   end
 
   # Runs code after a new commit has been pushed.
@@ -646,11 +696,11 @@ class Repository
       branches.sort_by(&:name)
     when 'updated_desc'
       branches.sort do |a, b|
-        commit(b.target).committed_date <=> commit(a.target).committed_date
+        commit(b.dereferenced_target).committed_date <=> commit(a.dereferenced_target).committed_date
       end
     when 'updated_asc'
       branches.sort do |a, b|
-        commit(a.target).committed_date <=> commit(b.target).committed_date
+        commit(a.dereferenced_target).committed_date <=> commit(b.dereferenced_target).committed_date
       end
     else
       branches
@@ -689,6 +739,14 @@ class Repository
     end
   end
 
+  def ref_name_for_sha(ref_path, sha)
+    args = %W(#{Gitlab.config.git.bin_path} for-each-ref --count=1 #{ref_path} --contains #{sha})
+
+    # Not found -> ["", 0]
+    # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0]
+    Gitlab::Popen.popen(args, path_to_repo).first.split.last
+  end
+
   def refs_contains_sha(ref_type, sha)
     args = %W(#{Gitlab.config.git.bin_path} #{ref_type} --contains #{sha})
     names = Gitlab::Popen.popen(args, path_to_repo).first
@@ -728,64 +786,61 @@ class Repository
     @root_ref ||= cache.fetch(:root_ref) { raw_repository.root_ref }
   end
 
-  def commit_dir(user, path, message, branch)
-    commit_with_hooks(user, branch) do |ref|
-      committer = user_to_committer(user)
-      options = {}
-      options[:committer] = committer
-      options[:author] = committer
-
-      options[:commit] = {
-        message: message,
-        branch: ref,
-        update_ref: false,
+  def commit_dir(user, path, message, branch, author_email: nil, author_name: nil)
+    update_branch_with_hooks(user, branch) do |ref|
+      options = {
+        commit: {
+          branch: ref,
+          message: message,
+          update_ref: false
+        }
       }
 
+      options.merge!(get_committer_and_author(user, email: author_email, name: author_name))
+
       raw_repository.mkdir(path, options)
     end
   end
 
-  def commit_file(user, path, content, message, branch, update)
-    commit_with_hooks(user, branch) do |ref|
-      committer = user_to_committer(user)
-      options = {}
-      options[:committer] = committer
-      options[:author] = committer
-      options[:commit] = {
-        message: message,
-        branch: ref,
-        update_ref: false,
+  def commit_file(user, path, content, message, branch, update, author_email: nil, author_name: nil)
+    update_branch_with_hooks(user, branch) do |ref|
+      options = {
+        commit: {
+          branch: ref,
+          message: message,
+          update_ref: false
+        },
+        file: {
+          content: content,
+          path: path,
+          update: update
+        }
       }
 
-      options[:file] = {
-        content: content,
-        path: path,
-        update: update
-      }
+      options.merge!(get_committer_and_author(user, email: author_email, name: author_name))
 
       Gitlab::Git::Blob.commit(raw_repository, options)
     end
   end
 
-  def update_file(user, path, content, branch:, previous_path:, message:)
-    commit_with_hooks(user, branch) do |ref|
-      committer = user_to_committer(user)
-      options = {}
-      options[:committer] = committer
-      options[:author] = committer
-      options[:commit] = {
-        message: message,
-        branch: ref,
-        update_ref: false
+  def update_file(user, path, content, branch:, previous_path:, message:, author_email: nil, author_name: nil)
+    update_branch_with_hooks(user, branch) do |ref|
+      options = {
+        commit: {
+          branch: ref,
+          message: message,
+          update_ref: false
+        },
+        file: {
+          content: content,
+          path: path,
+          update: true
+        }
       }
 
-      options[:file] = {
-        content: content,
-        path: path,
-        update: true
-      }
+      options.merge!(get_committer_and_author(user, email: author_email, name: author_name))
 
-      if previous_path
+      if previous_path && previous_path != path
         options[:file][:previous_path] = previous_path
         Gitlab::Git::Blob.rename(raw_repository, options)
       else
@@ -794,34 +849,85 @@ class Repository
     end
   end
 
-  def remove_file(user, path, message, branch)
-    commit_with_hooks(user, branch) do |ref|
-      committer = user_to_committer(user)
-      options = {}
-      options[:committer] = committer
-      options[:author] = committer
-      options[:commit] = {
-        message: message,
-        branch: ref,
-        update_ref: false,
+  def remove_file(user, path, message, branch, author_email: nil, author_name: nil)
+    update_branch_with_hooks(user, branch) do |ref|
+      options = {
+        commit: {
+          branch: ref,
+          message: message,
+          update_ref: false
+        },
+        file: {
+          path: path
+        }
       }
 
-      options[:file] = {
-        path: path
-      }
+      options.merge!(get_committer_and_author(user, email: author_email, name: author_name))
 
       Gitlab::Git::Blob.remove(raw_repository, options)
     end
   end
 
-  def user_to_committer(user)
+  def multi_action(user:, branch:, message:, actions:, author_email: nil, author_name: nil)
+    update_branch_with_hooks(user, branch) do |ref|
+      index = rugged.index
+      parents = []
+      branch = find_branch(ref)
+
+      if branch
+        last_commit = branch.dereferenced_target
+        index.read_tree(last_commit.raw_commit.tree)
+        parents = [last_commit.sha]
+      end
+
+      actions.each do |action|
+        case action[:action]
+        when :create, :update, :move
+          mode =
+            case action[:action]
+            when :update
+              index.get(action[:file_path])[:mode]
+            when :move
+              index.get(action[:previous_path])[:mode]
+            end
+          mode ||= 0o100644
+
+          index.remove(action[:previous_path]) if action[:action] == :move
+
+          content = action[:encoding] == 'base64' ? Base64.decode64(action[:content]) : action[:content]
+          oid = rugged.write(content, :blob)
+
+          index.add(path: action[:file_path], oid: oid, mode: mode)
+        when :delete
+          index.remove(action[:file_path])
+        end
+      end
+
+      options = {
+        tree: index.write_tree(rugged),
+        message: message,
+        parents: parents
+      }
+      options.merge!(get_committer_and_author(user, email: author_email, name: author_name))
+
+      Rugged::Commit.create(rugged, options)
+    end
+  end
+
+  def get_committer_and_author(user, email: nil, name: nil)
+    committer = user_to_committer(user)
+    author = Gitlab::Git::committer_hash(email: email, name: name) || committer
+
     {
-      email: user.email,
-      name: user.name,
-      time: Time.now
+      author: author,
+      committer: committer
     }
   end
 
+  def user_to_committer(user)
+    Gitlab::Git::committer_hash(email: user.email, name: user.name)
+  end
+
   def can_be_merged?(source_sha, target_branch)
     our_commit = rugged.branches[target_branch].target
     their_commit = rugged.lookup(source_sha)
@@ -843,7 +949,7 @@ class Repository
     merge_index = rugged.merge_commits(our_commit, their_commit)
     return false if merge_index.conflicts?
 
-    commit_with_hooks(user, merge_request.target_branch) do
+    update_branch_with_hooks(user, merge_request.target_branch) do
       actual_options = options.merge(
         parents: [our_commit, their_commit],
         tree: merge_index.write_tree(rugged),
@@ -856,12 +962,12 @@ class Repository
   end
 
   def revert(user, commit, base_branch, revert_tree_id = nil)
-    source_sha = find_branch(base_branch).target.sha
+    source_sha = find_branch(base_branch).dereferenced_target.sha
     revert_tree_id ||= check_revert_content(commit, base_branch)
 
     return false unless revert_tree_id
 
-    commit_with_hooks(user, base_branch) do
+    update_branch_with_hooks(user, base_branch) do
       committer = user_to_committer(user)
       source_sha = Rugged::Commit.create(rugged,
         message: commit.revert_message,
@@ -873,12 +979,12 @@ class Repository
   end
 
   def cherry_pick(user, commit, base_branch, cherry_pick_tree_id = nil)
-    source_sha = find_branch(base_branch).target.sha
+    source_sha = find_branch(base_branch).dereferenced_target.sha
     cherry_pick_tree_id ||= check_cherry_pick_content(commit, base_branch)
 
     return false unless cherry_pick_tree_id
 
-    commit_with_hooks(user, base_branch) do
+    update_branch_with_hooks(user, base_branch) do
       committer = user_to_committer(user)
       source_sha = Rugged::Commit.create(rugged,
         message: commit.message,
@@ -894,7 +1000,7 @@ class Repository
   end
 
   def resolve_conflicts(user, branch, params)
-    commit_with_hooks(user, branch) do
+    update_branch_with_hooks(user, branch) do
       committer = user_to_committer(user)
 
       Rugged::Commit.create(rugged, params.merge(author: committer, committer: committer))
@@ -902,7 +1008,7 @@ class Repository
   end
 
   def check_revert_content(commit, base_branch)
-    source_sha = find_branch(base_branch).target.sha
+    source_sha = find_branch(base_branch).dereferenced_target.sha
     args       = [commit.id, source_sha]
     args << { mainline: 1 } if commit.merge_commit?
 
@@ -916,7 +1022,7 @@ class Repository
   end
 
   def check_cherry_pick_content(commit, base_branch)
-    source_sha = find_branch(base_branch).target.sha
+    source_sha = find_branch(base_branch).dereferenced_target.sha
     args       = [commit.id, source_sha]
     args << 1 if commit.merge_commit?
 
@@ -938,7 +1044,8 @@ class Repository
     root_ref_commit = commit(root_ref)
 
     if branch_commit
-      is_ancestor?(branch_commit.id, root_ref_commit.id)
+      same_head = branch_commit.id == root_ref_commit.id
+      !same_head && is_ancestor?(branch_commit.id, root_ref_commit.id)
     else
       nil
     end
@@ -957,59 +1064,31 @@ class Repository
   end
 
   def search_files(query, ref)
+    unless exists? && has_visible_content? && query.present?
+      return []
+    end
+
     offset = 2
     args = %W(#{Gitlab.config.git.bin_path} grep -i -I -n --before-context #{offset} --after-context #{offset} -E -e #{Regexp.escape(query)} #{ref || root_ref})
     Gitlab::Popen.popen(args, path_to_repo).first.scrub.split(/^--$/)
   end
 
-  def parse_search_result(result)
-    ref = nil
-    filename = nil
-    basename = nil
-    startline = 0
-
-    result.each_line.each_with_index do |line, index|
-      if line =~ /^.*:.*:\d+:/
-        ref, filename, startline = line.split(':')
-        startline = startline.to_i - index
-        extname = Regexp.escape(File.extname(filename))
-        basename = filename.sub(/#{extname}$/, '')
-        break
-      end
-    end
-
-    data = ""
-
-    result.each_line do |line|
-      data << line.sub(ref, '').sub(filename, '').sub(/^:-\d+-/, '').sub(/^::\d+:/, '')
-    end
-
-    OpenStruct.new(
-      filename: filename,
-      basename: basename,
-      ref: ref,
-      startline: startline,
-      data: data
-    )
-  end
-
   def fetch_ref(source_path, source_ref, target_ref)
     args = %W(#{Gitlab.config.git.bin_path} fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref})
     Gitlab::Popen.popen(args, path_to_repo)
   end
 
-  def commit_with_hooks(current_user, branch)
+  def create_ref(ref, ref_path)
+    fetch_ref(path_to_repo, ref, ref_path)
+  end
+
+  def update_branch_with_hooks(current_user, branch)
     update_autocrlf_option
 
-    oldrev = Gitlab::Git::BLANK_SHA
     ref = Gitlab::Git::BRANCH_REF_PREFIX + branch
     target_branch = find_branch(branch)
     was_empty = empty?
 
-    if !was_empty && target_branch
-      oldrev = target_branch.target.id
-    end
-
     # Make commit
     newrev = yield(ref)
 
@@ -1017,24 +1096,19 @@ class Repository
       raise CommitError.new('Failed to create commit')
     end
 
+    if rugged.lookup(newrev).parent_ids.empty? || target_branch.nil?
+      oldrev = Gitlab::Git::BLANK_SHA
+    else
+      oldrev = rugged.merge_base(newrev, target_branch.dereferenced_target.sha)
+    end
+
     GitHooksService.new.execute(current_user, path_to_repo, oldrev, newrev, ref) do
-      if was_empty || !target_branch
-        # Create branch
-        rugged.references.create(ref, newrev)
+      update_ref!(ref, newrev, oldrev)
 
+      if was_empty || !target_branch
         # If repo was empty expire cache
         after_create if was_empty
         after_create_branch
-      else
-        # Update head
-        current_head = find_branch(branch).target.id
-
-        # Make sure target branch was not changed during pre-receive hook
-        if current_head == oldrev
-          rugged.references.update(ref, newrev)
-        else
-          raise CommitError.new('Commit was rejected because branch received new push')
-        end
       end
     end
 
@@ -1085,7 +1159,7 @@ class Repository
   end
 
   def tags_sorted_by_committed_date
-    tags.sort_by { |tag| tag.target.committed_date }
+    tags.sort_by { |tag| tag.dereferenced_target.committed_date }
   end
 
   def keep_around_ref_name(sha)