Revert "Merge branch 'drop-satellites'"

This reverts commit 957e849f41d96fa9778fcdd06792d2f0274b29ab, reversing
changes made to 6b9dbe9f5a175a8162abf296367f561bab3eea1a.

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>

10 files changed, 520 insertions, 5 deletions

diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index 7412274b045..ce21c699e8f 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -198,11 +198,7 @@ module API
 
           if merge_request.open? && !merge_request.work_in_progress?
             if merge_request.can_be_merged?
-              commit_message = params[:merge_commit_message] || merge_request.merge_commit_message
-
-              ::MergeRequests::MergeService.new(merge_request.target_project, current_user).
-                execute(merge_request, commit_message)
-
+              merge_request.automerge!(current_user, params[:merge_commit_message] || merge_request.merge_commit_message)
               present merge_request, with: Entities::MergeRequest
             else
               render_api_error!('Branch cannot be merged', 405)
diff --git a/lib/gitlab.rb b/lib/gitlab.rb
index 6108697bc20..5fc1862c3e9 100644
--- a/lib/gitlab.rb
+++ b/lib/gitlab.rb
@@ -1,4 +1,5 @@
 require 'gitlab/git'
 
 module Gitlab
+  autoload :Satellite, 'gitlab/satellite/satellite'
 end
diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb
index 14ee4701e7b..172d4902add 100644
--- a/lib/gitlab/backend/shell.rb
+++ b/lib/gitlab/backend/shell.rb
@@ -217,6 +217,20 @@ module Gitlab
       FileUtils.mv(full_path(old_name), full_path(new_name))
     end
 
+    # Remove GitLab Satellites for provided path (namespace or repo dir)
+    #
+    # Ex.
+    #   rm_satellites("gitlab")
+    #
+    #   rm_satellites("gitlab/gitlab-ci.git")
+    #
+    def rm_satellites(path)
+      raise ArgumentError.new("Path can't be blank") if path.blank?
+
+      satellites_path = File.join(Gitlab.config.satellites.path, path)
+      FileUtils.rm_r(satellites_path, force: true)
+    end
+
     def url_to_repo(path)
       Gitlab.config.gitlab_shell.ssh_path_prefix + "#{path}.git"
     end
diff --git a/lib/gitlab/satellite/action.rb b/lib/gitlab/satellite/action.rb
new file mode 100644
index 00000000000..489070f1a3f
--- /dev/null
+++ b/lib/gitlab/satellite/action.rb
@@ -0,0 +1,58 @@
+module Gitlab
+  module Satellite
+    class Action
+      DEFAULT_OPTIONS = { git_timeout: Gitlab.config.satellites.timeout.seconds }
+
+      attr_accessor :options, :project, :user
+
+      def initialize(user, project, options = {})
+        @options = DEFAULT_OPTIONS.merge(options)
+        @project = project
+        @user = user
+      end
+
+      protected
+
+      # * Sets a 30s timeout for Git
+      # * Locks the satellite repo
+      # * Yields the prepared satellite repo
+      def in_locked_and_timed_satellite
+        Gitlab::ShellEnv.set_env(user)
+
+        Grit::Git.with_timeout(options[:git_timeout]) do
+          project.satellite.lock do
+            return yield project.satellite.repo
+          end
+        end
+      rescue Errno::ENOMEM => ex
+        return handle_exception(ex)
+      rescue Grit::Git::GitTimeout => ex
+        return handle_exception(ex)
+      ensure
+        Gitlab::ShellEnv.reset_env
+      end
+
+      # * Recreates the satellite
+      # * Sets up Git variables for the user
+      #
+      # Note: use this within #in_locked_and_timed_satellite
+      def prepare_satellite!(repo)
+        project.satellite.clear_and_update!
+
+        if user
+          repo.config['user.name'] = user.name
+          repo.config['user.email'] = user.email
+        end
+      end
+
+      def default_options(options = {})
+        { raise: true, timeout: true }.merge(options)
+      end
+
+      def handle_exception(exception)
+        Gitlab::GitLogger.error(exception.message)
+        false
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/satellite/compare_action.rb b/lib/gitlab/satellite/compare_action.rb
new file mode 100644
index 00000000000..46c98a8f4ca
--- /dev/null
+++ b/lib/gitlab/satellite/compare_action.rb
@@ -0,0 +1,44 @@
+module Gitlab
+  module Satellite
+    class BranchesWithoutParent < StandardError; end
+
+    class CompareAction < Action
+      def initialize(user, target_project, target_branch, source_project, source_branch)
+        super user, target_project
+
+        @target_project, @target_branch = target_project, target_branch
+        @source_project, @source_branch = source_project, source_branch
+      end
+
+      # Compare 2 repositories and return Gitlab::CompareResult object
+      def result
+        in_locked_and_timed_satellite do |target_repo|
+          prepare_satellite!(target_repo)
+          update_satellite_source_and_target!(target_repo)
+
+          Gitlab::CompareResult.new(compare(target_repo))
+        end
+      rescue Grit::Git::CommandFailed => ex
+        raise BranchesWithoutParent
+      end
+
+      private
+
+      # Assumes a satellite exists that is a fresh clone of the projects repo, prepares satellite for diffs
+      def update_satellite_source_and_target!(target_repo)
+        target_repo.remote_add('source', @source_project.repository.path_to_repo)
+        target_repo.remote_fetch('source')
+      rescue Grit::Git::CommandFailed => ex
+        handle_exception(ex)
+      end
+
+      def compare(repo)
+        @compare ||= Gitlab::Git::Compare.new(
+          Gitlab::Git::Repository.new(repo.path),
+          "origin/#{@target_branch}",
+          "source/#{@source_branch}"
+        )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/satellite/logger.rb b/lib/gitlab/satellite/logger.rb
new file mode 100644
index 00000000000..6f3f8255aca
--- /dev/null
+++ b/lib/gitlab/satellite/logger.rb
@@ -0,0 +1,13 @@
+module Gitlab
+  module Satellite
+    class Logger < Gitlab::Logger
+      def self.file_name
+        'satellites.log'
+      end
+
+      def format_message(severity, timestamp, progname, msg)
+        "#{timestamp.to_s(:long)}: #{msg}\n"
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/satellite/merge_action.rb b/lib/gitlab/satellite/merge_action.rb
new file mode 100644
index 00000000000..f9bf286697e
--- /dev/null
+++ b/lib/gitlab/satellite/merge_action.rb
@@ -0,0 +1,146 @@
+module Gitlab
+  module Satellite
+    # GitLab server-side merge
+    class MergeAction < Action
+      attr_accessor :merge_request
+
+      def initialize(user, merge_request)
+        super user, merge_request.target_project
+        @merge_request = merge_request
+      end
+
+      # Checks if a merge request can be executed without user interaction
+      def can_be_merged?
+        in_locked_and_timed_satellite do |merge_repo|
+          prepare_satellite!(merge_repo)
+          merge_in_satellite!(merge_repo)
+        end
+      end
+
+      # Merges the source branch into the target branch in the satellite and
+      # pushes it back to the repository.
+      # It also removes the source branch if requested in the merge request (and this is permitted by the merge request).
+      #
+      # Returns false if the merge produced conflicts
+      # Returns false if pushing from the satellite to the repository failed or was rejected
+      # Returns true otherwise
+      def merge!(merge_commit_message = nil)
+        in_locked_and_timed_satellite do |merge_repo|
+          prepare_satellite!(merge_repo)
+          if merge_in_satellite!(merge_repo, merge_commit_message)
+            # push merge back to bare repo
+            # will raise CommandFailed when push fails
+            merge_repo.git.push(default_options, :origin, merge_request.target_branch)
+
+            # remove source branch
+            if merge_request.remove_source_branch?
+              # will raise CommandFailed when push fails
+              merge_repo.git.push(default_options, :origin, ":#{merge_request.source_branch}")
+            end
+            # merge, push and branch removal successful
+            true
+          end
+        end
+      rescue Grit::Git::CommandFailed => ex
+        handle_exception(ex)
+      end
+
+      def diff_in_satellite
+        in_locked_and_timed_satellite do |merge_repo|
+          prepare_satellite!(merge_repo)
+          update_satellite_source_and_target!(merge_repo)
+
+          # Only show what is new in the source branch compared to the target branch, not the other way around.
+          # The line below with merge_base is equivalent to diff with three dots (git diff branch1...branch2)
+          # From the git documentation: "git diff A...B" is equivalent to "git diff $(git-merge-base A B) B"
+          common_commit = merge_repo.git.native(:merge_base, default_options, ["origin/#{merge_request.target_branch}", "source/#{merge_request.source_branch}"]).strip
+          merge_repo.git.native(:diff, default_options, common_commit, "source/#{merge_request.source_branch}")
+        end
+      rescue Grit::Git::CommandFailed => ex
+        handle_exception(ex)
+      end
+
+      def diffs_between_satellite
+        in_locked_and_timed_satellite do |merge_repo|
+          prepare_satellite!(merge_repo)
+          update_satellite_source_and_target!(merge_repo)
+          if merge_request.for_fork?
+            repository = Gitlab::Git::Repository.new(merge_repo.path)
+            diffs = Gitlab::Git::Diff.between(
+              repository,
+              "source/#{merge_request.source_branch}",
+              "origin/#{merge_request.target_branch}"
+            )
+          else
+            raise "Attempt to determine diffs between for a non forked merge request in satellite MergeRequest.id:[#{merge_request.id}]"
+          end
+
+          return diffs
+        end
+      rescue Grit::Git::CommandFailed => ex
+        handle_exception(ex)
+      end
+
+      # Get commit as an email patch
+      def format_patch
+        in_locked_and_timed_satellite do |merge_repo|
+          prepare_satellite!(merge_repo)
+          update_satellite_source_and_target!(merge_repo)
+          patch = merge_repo.git.format_patch(default_options({ stdout: true }), "origin/#{merge_request.target_branch}..source/#{merge_request.source_branch}")
+        end
+      rescue Grit::Git::CommandFailed => ex
+        handle_exception(ex)
+      end
+
+      # Retrieve an array of commits between the source and the target
+      def commits_between
+        in_locked_and_timed_satellite do |merge_repo|
+          prepare_satellite!(merge_repo)
+          update_satellite_source_and_target!(merge_repo)
+          if merge_request.for_fork?
+            repository = Gitlab::Git::Repository.new(merge_repo.path)
+            commits = Gitlab::Git::Commit.between(
+              repository,
+              "origin/#{merge_request.target_branch}",
+              "source/#{merge_request.source_branch}"
+            )
+          else
+            raise "Attempt to determine commits between for a non forked merge request in satellite MergeRequest.id:[#{merge_request.id}]"
+          end
+
+          return commits
+        end
+      rescue Grit::Git::CommandFailed => ex
+        handle_exception(ex)
+      end
+
+      private
+      # Merges the source_branch into the target_branch in the satellite.
+      #
+      # Note: it will clear out the satellite before doing anything
+      #
+      # Returns false if the merge produced conflicts
+      # Returns true otherwise
+      def merge_in_satellite!(repo, message = nil)
+        update_satellite_source_and_target!(repo)
+
+        message ||= "Merge branch '#{merge_request.source_branch}' into '#{merge_request.target_branch}'"
+
+        # merge the source branch into the satellite
+        # will raise CommandFailed when merge fails
+        repo.git.merge(default_options({ no_ff: true }), "-m#{message}", "source/#{merge_request.source_branch}")
+      rescue Grit::Git::CommandFailed => ex
+        handle_exception(ex)
+      end
+
+      # Assumes a satellite exists that is a fresh clone of the projects repo, prepares satellite for merges, diffs etc
+      def update_satellite_source_and_target!(repo)
+        repo.remote_add('source', merge_request.source_project.repository.path_to_repo)
+        repo.remote_fetch('source')
+        repo.git.checkout(default_options({ b: true }), merge_request.target_branch, "origin/#{merge_request.target_branch}")
+      rescue Grit::Git::CommandFailed => ex
+        handle_exception(ex)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/satellite/satellite.rb b/lib/gitlab/satellite/satellite.rb
new file mode 100644
index 00000000000..398643d68de
--- /dev/null
+++ b/lib/gitlab/satellite/satellite.rb
@@ -0,0 +1,148 @@
+module Gitlab
+  module Satellite
+    autoload :DeleteFileAction, 'gitlab/satellite/files/delete_file_action'
+    autoload :EditFileAction,   'gitlab/satellite/files/edit_file_action'
+    autoload :FileAction,       'gitlab/satellite/files/file_action'
+    autoload :NewFileAction,    'gitlab/satellite/files/new_file_action'
+
+    class CheckoutFailed < StandardError; end
+    class CommitFailed < StandardError; end
+    class PushFailed < StandardError; end
+
+    class Satellite
+      include Gitlab::Popen
+
+      PARKING_BRANCH = "__parking_branch"
+
+      attr_accessor :project
+
+      def initialize(project)
+        @project = project
+      end
+
+      def log(message)
+        Gitlab::Satellite::Logger.error(message)
+      end
+
+      def clear_and_update!
+        project.ensure_satellite_exists
+
+        @repo = nil
+        clear_working_dir!
+        delete_heads!
+        remove_remotes!
+        update_from_source!
+      end
+
+      def create
+        output, status = popen(%W(git clone -- #{project.repository.path_to_repo} #{path}),
+                               Gitlab.config.satellites.path)
+
+        log("PID: #{project.id}: git clone #{project.repository.path_to_repo} #{path}")
+        log("PID: #{project.id}: -> #{output}")
+
+        if status.zero?
+          true
+        else
+          log("Failed to create satellite for #{project.name_with_namespace}")
+          false
+        end
+      end
+
+      def exists?
+        File.exists? path
+      end
+
+      # * Locks the satellite
+      # * Changes the current directory to the satellite's working dir
+      # * Yields
+      def lock
+        project.ensure_satellite_exists
+
+        File.open(lock_file, "w+") do |f|
+          begin
+            f.flock File::LOCK_EX
+            yield
+          ensure
+            f.flock File::LOCK_UN
+          end
+        end
+      end
+
+      def lock_file
+        create_locks_dir unless File.exists?(lock_files_dir)
+        File.join(lock_files_dir, "satellite_#{project.id}.lock")
+      end
+
+      def path
+        File.join(Gitlab.config.satellites.path, project.path_with_namespace)
+      end
+
+      def repo
+        project.ensure_satellite_exists
+
+        @repo ||= Grit::Repo.new(path)
+      end
+
+      def destroy
+        FileUtils.rm_rf(path)
+      end
+
+      private
+
+      # Clear the working directory
+      def clear_working_dir!
+        repo.git.reset(hard: true)
+        repo.git.clean(f: true, d: true, x: true)
+      end
+
+      # Deletes all branches except the parking branch
+      #
+      # This ensures we have no name clashes or issues updating branches when
+      # working with the satellite.
+      def delete_heads!
+        heads = repo.heads.map(&:name)
+
+        # update or create the parking branch
+        repo.git.checkout(default_options({ B: true }), PARKING_BRANCH)
+
+        # remove the parking branch from the list of heads ...
+        heads.delete(PARKING_BRANCH)
+        # ... and delete all others
+        heads.each { |head| repo.git.branch(default_options({ D: true }), head) }
+      end
+
+      # Deletes all remotes except origin
+      #
+      # This ensures we have no remote name clashes or issues updating branches when
+      # working with the satellite.
+      def remove_remotes!
+        remotes = repo.git.remote.split(' ')
+        remotes.delete('origin')
+        remotes.each { |name| repo.git.remote(default_options,'rm', name)}
+      end
+
+      # Updates the satellite from bare repo
+      #
+      # Note: this will only update remote branches (i.e. origin/*)
+      def update_from_source!
+        repo.git.remote(default_options, 'set-url', :origin, project.repository.path_to_repo)
+        repo.git.fetch(default_options, :origin)
+      end
+
+      def default_options(options = {})
+        { raise: true, timeout: true }.merge(options)
+      end
+
+      # Create directory for storing
+      # satellites lock files
+      def create_locks_dir
+        FileUtils.mkdir_p(lock_files_dir)
+      end
+
+      def lock_files_dir
+        @lock_files_dir ||= File.join(Gitlab.config.satellites.path, "tmp")
+      end
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index 8acb6a7fd19..badb47c6779 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -25,6 +25,7 @@ namespace :gitlab do
       check_init_script_exists
       check_init_script_up_to_date
       check_projects_have_namespace
+      check_satellites_exist
       check_redis_version
       check_ruby_version
       check_git_version
@@ -237,6 +238,37 @@ namespace :gitlab do
       end
     end
 
+    def check_satellites_exist
+      print "Projects have satellites? ... "
+
+      unless Project.count > 0
+        puts "can't check, you have no projects".magenta
+        return
+      end
+      puts ""
+
+      Project.find_each(batch_size: 100) do |project|
+        print sanitized_message(project)
+
+        if project.satellite.exists?
+          puts "yes".green
+        elsif project.empty_repo?
+          puts "can't create, repository is empty".magenta
+        else
+          puts "no".red
+          try_fixing_it(
+            sudo_gitlab("bundle exec rake gitlab:satellites:create RAILS_ENV=production"),
+            "If necessary, remove the tmp/repo_satellites directory ...",
+            "... and rerun the above command"
+          )
+          for_more_information(
+            "doc/raketasks/maintenance.md "
+          )
+          fix_and_rerun
+        end
+      end
+    end
+
     def check_log_writable
       print "Log directory writable? ... "
 
@@ -307,6 +339,7 @@ namespace :gitlab do
       check_repo_base_is_not_symlink
       check_repo_base_user_and_group
       check_repo_base_permissions
+      check_satellites_permissions
       check_repos_hooks_directory_is_link
       check_gitlab_shell_self_test
 
@@ -384,6 +417,29 @@ namespace :gitlab do
       end
     end
 
+    def check_satellites_permissions
+      print "Satellites access is drwxr-x---? ... "
+
+      satellites_path = Gitlab.config.satellites.path
+      unless File.exists?(satellites_path)
+        puts "can't check because of previous errors".magenta
+        return
+      end
+
+      if File.stat(satellites_path).mode.to_s(8).ends_with?("0750")
+        puts "yes".green
+      else
+        puts "no".red
+        try_fixing_it(
+          "sudo chmod u+rwx,g=rx,o-rwx #{satellites_path}",
+        )
+        for_more_information(
+          see_installation_guide_section "GitLab"
+        )
+        fix_and_rerun
+      end
+    end
+
     def check_repo_base_user_and_group
       gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user
       gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group
diff --git a/lib/tasks/gitlab/enable_automerge.rake b/lib/tasks/gitlab/enable_automerge.rake
new file mode 100644
index 00000000000..3dade9d75b8
--- /dev/null
+++ b/lib/tasks/gitlab/enable_automerge.rake
@@ -0,0 +1,39 @@
+namespace :gitlab do
+  namespace :satellites do
+    desc "GitLab | Create satellite repos"
+    task create: :environment do
+      create_satellites
+    end
+  end
+
+  def create_satellites
+    warn_user_is_not_gitlab
+
+    print "Creating satellites for ..."
+    unless Project.count > 0
+      puts "skipping, because you have no projects".magenta
+      return
+    end
+    puts ""
+
+    Project.find_each(batch_size: 100) do |project|
+      print "#{project.name_with_namespace.yellow} ... "
+
+      unless project.repo_exists?
+        puts "skipping, because the repo is empty".magenta
+        next
+      end
+
+      if project.satellite.exists?
+        puts "exists already".green
+      else
+        print "\n... "
+        if project.satellite.create
+          puts "created".green
+        else
+          puts "error".red
+        end
+      end
+    end
+  end
+end