Merge branch 'master' into feature/multi-level-container-registry-images

* master: (192 commits)
  Implement new service for creating user
  Update sentry-raven 2.0.2 -> 2.4.0
  Update webmock 1.21.0 -> 1.24.6
  Update spring 1.7.2 -> 2.0.1
  Update simplecov 0.12.0 -> 0.14.1
  Update pry-rails 0.3.4 -> 0.3.5
  Update pry-byebug 3.4.1 -> 3.4.2
  Update flay 2.6.1 -> 2.8.1
  Remove Tags filter from Projects Explore dropdown
  Update capybara-screenshot 1.0.11 -> 1.0.14
  Update bullet 5.2.0 -> 5.5.1
  Update brakeman 3.4.1 -> 3.6.1
  Remove web-console gem
  Update better_errors 1.0.1 -> 2.1.1
  Display flash message to unauthenticated user when creating new issue
  Fix up emoji tests that should have failed :/
  Fix RSpec/DescribeSymbol cop violations
  Add event limit warning all tabs Cycle Analytics
  Adding non_archived scope for counting projects
  Resolve "Gitlab administrator cannot create projects in every group"
  ...

Conflicts:
	db/schema.rb

15 files changed, 449 insertions, 155 deletions

diff --git a/lib/api/users.rb b/lib/api/users.rb
index 2d4d5a25221..a4201fe6fed 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -27,7 +27,7 @@ module API
           optional :location, type: String, desc: 'The location of the user'
           optional :admin, type: Boolean, desc: 'Flag indicating the user is an administrator'
           optional :can_create_group, type: Boolean, desc: 'Flag indicating the user can create groups'
-          optional :confirm, type: Boolean, desc: 'Flag indicating the account needs to be confirmed'
+          optional :skip_confirmation, type: Boolean, default: false, desc: 'Flag indicating the account is confirmed'
           optional :external, type: Boolean, desc: 'Flag indicating the user is an external user'
           all_or_none_of :extern_uid, :provider
         end
@@ -97,29 +97,10 @@ module API
       post do
         authenticated_as_admin!
 
-        # Filter out params which are used later
-        user_params = declared_params(include_missing: false)
-        identity_attrs = user_params.slice(:provider, :extern_uid)
-        confirm = user_params.delete(:confirm)
-        user = User.new(user_params.except(:extern_uid, :provider, :reset_password))
-
-        if user_params.delete(:reset_password)
-          user.attributes = {
-            force_random_password: true,
-            password_expires_at: nil,
-            created_by_id: current_user.id
-          }
-          user.generate_password
-          user.generate_reset_token
-        end
-
-        user.skip_confirmation! unless confirm
-
-        if identity_attrs.any?
-          user.identities.build(identity_attrs)
-        end
+        params = declared_params(include_missing: false)
+        user = ::Users::CreateService.new(current_user, params).execute
 
-        if user.save
+        if user.persisted?
           present user, with: Entities::UserPublic
         else
           conflict!('Email has already been taken') if User.
diff --git a/lib/api/v3/users.rb b/lib/api/v3/users.rb
index 14f54731730..5e18cecc431 100644
--- a/lib/api/v3/users.rb
+++ b/lib/api/v3/users.rb
@@ -9,6 +9,59 @@ module API
       end
 
       resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do
+        helpers do
+          params :optional_attributes do
+            optional :skype, type: String, desc: 'The Skype username'
+            optional :linkedin, type: String, desc: 'The LinkedIn username'
+            optional :twitter, type: String, desc: 'The Twitter username'
+            optional :website_url, type: String, desc: 'The website of the user'
+            optional :organization, type: String, desc: 'The organization of the user'
+            optional :projects_limit, type: Integer, desc: 'The number of projects a user can create'
+            optional :extern_uid, type: String, desc: 'The external authentication provider UID'
+            optional :provider, type: String, desc: 'The external provider'
+            optional :bio, type: String, desc: 'The biography of the user'
+            optional :location, type: String, desc: 'The location of the user'
+            optional :admin, type: Boolean, desc: 'Flag indicating the user is an administrator'
+            optional :can_create_group, type: Boolean, desc: 'Flag indicating the user can create groups'
+            optional :confirm, type: Boolean, default: true, desc: 'Flag indicating the account needs to be confirmed'
+            optional :external, type: Boolean, desc: 'Flag indicating the user is an external user'
+            all_or_none_of :extern_uid, :provider
+          end
+        end
+
+        desc 'Create a user. Available only for admins.' do
+          success ::API::Entities::UserPublic
+        end
+        params do
+          requires :email, type: String, desc: 'The email of the user'
+          optional :password, type: String, desc: 'The password of the new user'
+          optional :reset_password, type: Boolean, desc: 'Flag indicating the user will be sent a password reset token'
+          at_least_one_of :password, :reset_password
+          requires :name, type: String, desc: 'The name of the user'
+          requires :username, type: String, desc: 'The username of the user'
+          use :optional_attributes
+        end
+        post do
+          authenticated_as_admin!
+
+          params = declared_params(include_missing: false)
+          user = ::Users::CreateService.new(current_user, params.merge!(skip_confirmation: !params[:confirm])).execute
+
+          if user.persisted?
+            present user, with: ::API::Entities::UserPublic
+          else
+            conflict!('Email has already been taken') if User.
+                where(email: user.email).
+                count > 0
+
+            conflict!('Username has already been taken') if User.
+                where(username: user.username).
+                count > 0
+
+            render_validation_error!(user)
+          end
+        end
+
         desc 'Get the SSH keys of a specified user. Available only for admins.' do
           success ::API::Entities::SSHKey
         end
diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb
index 1877429ac46..ee034d9cc56 100644
--- a/lib/gitlab/ci/build/step.rb
+++ b/lib/gitlab/ci/build/step.rb
@@ -7,13 +7,12 @@ module Gitlab
         WHEN_ALWAYS = 'always'.freeze
 
         attr_reader :name
-        attr_writer :script
-        attr_accessor :timeout, :when, :allow_failure
+        attr_accessor :script, :timeout, :when, :allow_failure
 
         class << self
           def from_commands(job)
             self.new(:script).tap do |step|
-              step.script = job.commands
+              step.script = job.commands.split("\n")
               step.timeout = job.timeout
               step.when = WHEN_ON_SUCCESS
             end
@@ -36,10 +35,6 @@ module Gitlab
           @name = name
           @allow_failure = false
         end
-
-        def script
-          @script.split("\n")
-        end
       end
     end
   end
diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb
index e0fdf3f3d64..0829c1c318e 100644
--- a/lib/gitlab/ee_compat_check.rb
+++ b/lib/gitlab/ee_compat_check.rb
@@ -5,35 +5,44 @@ module Gitlab
     CE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ce.git'.freeze
     EE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze
     CHECK_DIR = Rails.root.join('ee_compat_check')
-    MAX_FETCH_DEPTH = 500
     IGNORED_FILES_REGEX = /(VERSION|CHANGELOG\.md:\d+)/.freeze
-
-    attr_reader :repo_dir, :patches_dir, :ce_repo, :ce_branch
+    PLEASE_READ_THIS_BANNER = %Q{
+      ============================================================
+      ===================== PLEASE READ THIS =====================
+      ============================================================
+    }.freeze
+    THANKS_FOR_READING_BANNER = %Q{
+      ============================================================
+      ==================== THANKS FOR READING ====================
+      ============================================================\n
+    }.freeze
+
+    attr_reader :ee_repo_dir, :patches_dir, :ce_repo, :ce_branch, :ee_branch_found
+    attr_reader :failed_files
 
     def initialize(branch:, ce_repo: CE_REPO)
-      @repo_dir = CHECK_DIR.join('repo')
+      @ee_repo_dir = CHECK_DIR.join('ee-repo')
       @patches_dir = CHECK_DIR.join('patches')
       @ce_branch = branch
       @ce_repo = ce_repo
     end
 
     def check
-      ensure_ee_repo
       ensure_patches_dir
-
       generate_patch(ce_branch, ce_patch_full_path)
 
-      Dir.chdir(repo_dir) do
-        step("In the #{repo_dir} directory")
+      ensure_ee_repo
+      Dir.chdir(ee_repo_dir) do
+        step("In the #{ee_repo_dir} directory")
 
         status = catch(:halt_check) do
           ce_branch_compat_check!
-          delete_ee_branch_locally!
+          delete_ee_branches_locally!
           ee_branch_presence_check!
           ee_branch_compat_check!
         end
 
-        delete_ee_branch_locally!
+        delete_ee_branches_locally!
 
         if status.nil?
           true
@@ -46,11 +55,13 @@ module Gitlab
     private
 
     def ensure_ee_repo
-      if Dir.exist?(repo_dir)
-        step("#{repo_dir} already exists")
+      if Dir.exist?(ee_repo_dir)
+        step("#{ee_repo_dir} already exists")
       else
-        cmd = %W[git clone --branch master --single-branch --depth 200 #{EE_REPO} #{repo_dir}]
-        step("Cloning #{EE_REPO} into #{repo_dir}", cmd)
+        step(
+          "Cloning #{EE_REPO} into #{ee_repo_dir}",
+          %W[git clone --branch master --single-branch --depth=200 #{EE_REPO} #{ee_repo_dir}]
+        )
       end
     end
 
@@ -61,23 +72,18 @@ module Gitlab
     def generate_patch(branch, patch_path)
       FileUtils.rm(patch_path, force: true)
 
-      depth = 0
-      loop do
-        depth += 50
-        cmd = %W[git fetch --depth #{depth} origin --prune +refs/heads/master:refs/remotes/origin/master]
-        Gitlab::Popen.popen(cmd)
-        _, status = Gitlab::Popen.popen(%w[git merge-base FETCH_HEAD HEAD])
+      find_merge_base_with_master(branch: branch)
 
-        raise "#{branch} is too far behind master, please rebase it!" if depth >= MAX_FETCH_DEPTH
-        break if status.zero?
-      end
+      step(
+        "Generating the patch against origin/master in #{patch_path}",
+        %w[git format-patch origin/master --stdout]
+      ) do |output, status|
+        throw(:halt_check, :ko) unless status.zero?
 
-      step("Generating the patch against master in #{patch_path}")
-      output, status = Gitlab::Popen.popen(%w[git format-patch FETCH_HEAD --stdout])
-      throw(:halt_check, :ko) unless status.zero?
+        File.write(patch_path, output)
 
-      File.write(patch_path, output)
-      throw(:halt_check, :ko) unless File.exist?(patch_path)
+        throw(:halt_check, :ko) unless File.exist?(patch_path)
+      end
     end
 
     def ce_branch_compat_check!
@@ -88,9 +94,17 @@ module Gitlab
     end
 
     def ee_branch_presence_check!
-      status = step("Fetching origin/#{ee_branch}", %W[git fetch origin #{ee_branch}])
+      _, status = step("Fetching origin/#{ee_branch_prefix}", %W[git fetch origin #{ee_branch_prefix}])
 
-      unless status.zero?
+      if status.zero?
+        @ee_branch_found = ee_branch_prefix
+      else
+        _, status = step("Fetching origin/#{ee_branch_suffix}", %W[git fetch origin #{ee_branch_suffix}])
+      end
+
+      if status.zero?
+        @ee_branch_found = ee_branch_suffix
+      else
         puts
         puts ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg
 
@@ -99,9 +113,9 @@ module Gitlab
     end
 
     def ee_branch_compat_check!
-      step("Checking out origin/#{ee_branch}", %W[git checkout -b #{ee_branch} FETCH_HEAD])
+      step("Checking out origin/#{ee_branch_found}", %W[git checkout -b #{ee_branch_found} FETCH_HEAD])
 
-      generate_patch(ee_branch, ee_patch_full_path)
+      generate_patch(ee_branch_found, ee_patch_full_path)
 
       unless check_patch(ee_patch_full_path).zero?
         puts
@@ -116,36 +130,72 @@ module Gitlab
 
     def check_patch(patch_path)
       step("Checking out master", %w[git checkout master])
-      step("Reseting to latest master", %w[git reset --hard origin/master])
-
-      step("Checking if #{patch_path} applies cleanly to EE/master")
-      output, status = Gitlab::Popen.popen(%W[git apply --check --3way #{patch_path}])
-
-      unless status.zero?
-        failed_files = output.lines.reduce([]) do |memo, line|
-          if line.start_with?('error: patch failed:')
-            file = line.sub(/\Aerror: patch failed: /, '')
-            memo << file unless file =~ IGNORED_FILES_REGEX
+      step("Resetting to latest master", %w[git reset --hard origin/master])
+      step(
+        "Checking if #{patch_path} applies cleanly to EE/master",
+        %W[git apply --check --3way #{patch_path}]
+      ) do |output, status|
+        unless status.zero?
+          @failed_files = output.lines.reduce([]) do |memo, line|
+            if line.start_with?('error: patch failed:')
+              file = line.sub(/\Aerror: patch failed: /, '')
+              memo << file unless file =~ IGNORED_FILES_REGEX
+            end
+            memo
           end
-          memo
+
+          status = 0 if failed_files.empty?
         end
 
-        if failed_files.empty?
-          status = 0
-        else
-          puts "\nConflicting files:"
-          failed_files.each do |file|
-            puts "  - #{file}"
-          end
+        status
+      end
+    end
+
+    def delete_ee_branches_locally!
+      command(%w[git checkout master])
+      command(%W[git branch --delete --force #{ee_branch_prefix}])
+      command(%W[git branch --delete --force #{ee_branch_suffix}])
+    end
+
+    def merge_base_found?
+      step(
+        "Finding merge base with master",
+        %w[git merge-base origin/master HEAD]
+      ) do |output, status|
+        if status.zero?
+          puts "Merge base was found: #{output}"
+          true
         end
       end
+    end
+
+    def find_merge_base_with_master(branch:)
+      return if merge_base_found?
+
+      # Start with (Math.exp(3).to_i = 20) until (Math.exp(6).to_i = 403)
+      # In total we go (20 + 54 + 148 + 403 = 625) commits deeper
+      depth = 20
+      success =
+        (3..6).any? do |factor|
+          depth += Math.exp(factor).to_i
+          # Repository is initially cloned with a depth of 20 so we need to fetch
+          # deeper in the case the branch has more than 20 commits on top of master
+          fetch(branch: branch, depth: depth)
+          fetch(branch: 'master', depth: depth)
+
+          merge_base_found?
+        end
 
-      status
+      raise "\n#{branch} is too far behind master, please rebase it!\n" unless success
     end
 
-    def delete_ee_branch_locally!
-      command(%w[git checkout master])
-      step("Deleting the local #{ee_branch} branch", %W[git branch -D #{ee_branch}])
+    def fetch(branch:, depth:)
+      step(
+        "Fetching deeper...",
+        %W[git fetch --depth=#{depth} --prune origin +refs/heads/#{branch}:refs/remotes/origin/#{branch}]
+      ) do |output, status|
+        raise "Fetch failed: #{output}" unless status.zero?
+      end
     end
 
     def ce_patch_name
@@ -156,8 +206,12 @@ module Gitlab
       @ce_patch_full_path ||= patches_dir.join(ce_patch_name)
     end
 
-    def ee_branch
-      @ee_branch ||= "#{ce_branch}-ee"
+    def ee_branch_suffix
+      @ee_branch_suffix ||= "#{ce_branch}-ee"
+    end
+
+    def ee_branch_prefix
+      @ee_branch_prefix ||= "ee-#{ce_branch}"
     end
 
     def ee_patch_name
@@ -178,98 +232,125 @@ module Gitlab
       if cmd
         start = Time.now
         puts "\n$ #{cmd.join(' ')}"
-        status = command(cmd)
-        puts "\nFinished in #{Time.now - start} seconds"
-        status
+
+        output, status = command(cmd)
+        puts "\n==> Finished in #{Time.now - start} seconds"
+
+        if block_given?
+          yield(output, status)
+        else
+          [output, status]
+        end
       end
     end
 
     def command(cmd)
-      output, status = Gitlab::Popen.popen(cmd)
-      puts output
-
-      status
+      Gitlab::Popen.popen(cmd)
     end
 
     def applies_cleanly_msg(branch)
-      <<-MSG.strip_heredoc
-        =================================================================
+      %Q{
+        #{PLEASE_READ_THIS_BANNER}
         🎉 Congratulations!! 🎉
 
-        The #{branch} branch applies cleanly to EE/master!
+        The `#{branch}` branch applies cleanly to EE/master!
 
-        Much ❤️!!
-        =================================================================\n
-      MSG
+        Much ❤️! For more information, see
+        https://docs.gitlab.com/ce/development/limit_ee_conflicts.html#check-the-rake-ee_compat_check-in-your-merge-requests
+        #{THANKS_FOR_READING_BANNER}
+      }
     end
 
     def ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg
-      <<-MSG.strip_heredoc
-        =================================================================
+      %Q{
+        #{PLEASE_READ_THIS_BANNER}
         💥 Oh no! 💥
 
-        The #{ce_branch} branch does not apply cleanly to the current
-        EE/master, and no #{ee_branch} branch was found in the EE repository.
+        The `#{ce_branch}` branch does not apply cleanly to the current
+        EE/master, and no `#{ee_branch_prefix}` or `#{ee_branch_suffix}` branch
+        was found in the EE repository.
 
-        Please create a #{ee_branch} branch that includes changes from
-        #{ce_branch} but also specific changes than can be applied cleanly
-        to EE/master.
+        #{conflicting_files_msg}
+
+        We advise you to create a `#{ee_branch_prefix}` or `#{ee_branch_suffix}`
+        branch that includes changes from `#{ce_branch}` but also specific changes
+        than can be applied cleanly to EE/master. In some cases, the conflicts
+        are trivial and you can ignore the warning from this job. As always,
+        use your best judgment!
 
         There are different ways to create such branch:
 
-        1. Create a new branch based on the CE branch and rebase it on top of EE/master
+        1. Create a new branch from master and cherry-pick your CE commits
 
           # In the EE repo
-          $ git fetch #{ce_repo} #{ce_branch}
-          $ git checkout -b #{ee_branch} FETCH_HEAD
-
-          # You can squash the #{ce_branch} commits into a single "Port of #{ce_branch} to EE" commit
-          # before rebasing to limit the conflicts-resolving steps during the rebase
           $ git fetch origin
-          $ git rebase origin/master
+          $ git checkout -b #{ee_branch_prefix} origin/master
+          $ git fetch #{ce_repo} #{ce_branch}
+          $ git cherry-pick SHA # Repeat for all the commits you want to pick
 
-          At this point you will likely have conflicts.
-          Solve them, and continue/finish the rebase.
+          You can squash the `#{ce_branch}` commits into a single "Port of #{ce_branch} to EE" commit.
 
-          You can squash the #{ce_branch} commits into a single "Port of #{ce_branch} to EE".
+        2. Apply your branch's patch to EE
 
-        2. Create a new branch from master and cherry-pick your CE commits
+          # In the CE repo
+          $ git fetch origin master
+          $ git format-patch origin/master --stdout > #{ce_branch}.patch
 
           # In the EE repo
-          $ git fetch origin
-          $ git checkout -b #{ee_branch} origin/master
-          $ git fetch #{ce_repo} #{ce_branch}
-          $ git cherry-pick SHA # Repeat for all the commits you want to pick
+          $ git fetch origin master
+          $ git checkout -b #{ee_branch_prefix} origin/master
+          $ git apply --3way path/to/#{ce_branch}.patch
+
+          At this point you might have conflicts such as:
 
-          You can squash the #{ce_branch} commits into a single "Port of #{ce_branch} to EE" commit.
+            error: patch failed: lib/gitlab/ee_compat_check.rb:5
+            Falling back to three-way merge...
+            Applied patch to 'lib/gitlab/ee_compat_check.rb' with conflicts.
+            U lib/gitlab/ee_compat_check.rb
 
-        Don't forget to push your branch to #{EE_REPO}:
+          Resolve them, stage the changes and commit them.
+
+        ⚠️ Don't forget to push your branch to gitlab-ee:
 
           # In the EE repo
-          $ git push origin #{ee_branch}
+          $ git push origin #{ee_branch_prefix}
+
+        ⚠️ Also, don't forget to create a new merge request on gitlab-ce and
+        cross-link it with the CE merge request.
 
-        You can then retry this failed build, and hopefully it should pass.
+        Once this is done, you can retry this failed build, and it should pass.
 
-        Stay 💪 !
-        =================================================================\n
-      MSG
+        Stay 💪 ! For more information, see
+        https://docs.gitlab.com/ce/development/limit_ee_conflicts.html#check-the-rake-ee_compat_check-in-your-merge-requests
+        #{THANKS_FOR_READING_BANNER}
+      }
     end
 
     def ee_branch_doesnt_apply_cleanly_msg
-      <<-MSG.strip_heredoc
-        =================================================================
+      %Q{
+        #{PLEASE_READ_THIS_BANNER}
         💥 Oh no! 💥
 
-        The #{ce_branch} does not apply cleanly to the current
-        EE/master, and even though a #{ee_branch} branch exists in the EE
-        repository, it does not apply cleanly either to EE/master!
+        The `#{ce_branch}` does not apply cleanly to the current EE/master, and
+        even though a `#{ee_branch_found}` branch
+        exists in the EE repository, it does not apply cleanly either to
+        EE/master!
+
+        #{conflicting_files_msg}
 
-        Please update the #{ee_branch}, push it again to #{EE_REPO}, and
+        Please update the `#{ee_branch_found}`, push it again to gitlab-ee, and
         retry this build.
 
-        Stay 💪 !
-        =================================================================\n
-      MSG
+        Stay 💪 ! For more information, see
+        https://docs.gitlab.com/ce/development/limit_ee_conflicts.html#check-the-rake-ee_compat_check-in-your-merge-requests
+        #{THANKS_FOR_READING_BANNER}
+      }
+    end
+
+    def conflicting_files_msg
+      failed_files.reduce("The conflicts detected were as follows:\n") do |memo, file|
+        memo << "\n        - #{file}"
+      end
     end
   end
 end
diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb
index 9532e432f78..0039fc01c8f 100644
--- a/lib/gitlab/etag_caching/store.rb
+++ b/lib/gitlab/etag_caching/store.rb
@@ -1,7 +1,7 @@
 module Gitlab
   module EtagCaching
     class Store
-      EXPIRY_TIME = 10.minutes
+      EXPIRY_TIME = 20.minutes
       REDIS_NAMESPACE = 'etag:'.freeze
 
       def get(key)
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index 5534d4af439..1ce47ef2b05 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -2,6 +2,8 @@ require 'gitaly'
 
 module Gitlab
   module GitalyClient
+    SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze
+
     def self.gitaly_address
       if Gitlab.config.gitaly.socket_path
         "unix://#{Gitlab.config.gitaly.socket_path}"
@@ -39,5 +41,10 @@ module Gitlab
         yield is_enabled
       end
     end
+
+    def self.expected_server_version
+      path = Rails.root.join(SERVER_VERSION_FILE)
+      path.read.chomp
+    end
   end
 end
diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb
index fcf51b7fc5b..f98481c6d3a 100644
--- a/lib/gitlab/o_auth/user.rb
+++ b/lib/gitlab/o_auth/user.rb
@@ -147,10 +147,8 @@ module Gitlab
       end
 
       def build_new_user
-        user = ::User.new(user_attributes)
-        user.skip_confirmation!
-        user.identities.new(extern_uid: auth_hash.uid, provider: auth_hash.provider)
-        user
+        user_params = user_attributes.merge(extern_uid: auth_hash.uid, provider: auth_hash.provider, skip_confirmation: true)
+        Users::CreateService.new(nil, user_params).build
       end
 
       def user_attributes
diff --git a/lib/gitlab/testing/request_blocker_middleware.rb b/lib/gitlab/testing/request_blocker_middleware.rb
new file mode 100644
index 00000000000..aa67fa08577
--- /dev/null
+++ b/lib/gitlab/testing/request_blocker_middleware.rb
@@ -0,0 +1,61 @@
+# rubocop:disable Style/ClassVars
+
+# This is inspired by http://www.salsify.com/blog/engineering/tearing-capybara-ajax-tests
+# Rack middleware that keeps track of the number of active requests and can block new requests.
+module Gitlab
+  module Testing
+    class RequestBlockerMiddleware
+      @@num_active_requests = Concurrent::AtomicFixnum.new(0)
+      @@block_requests = Concurrent::AtomicBoolean.new(false)
+
+      # Returns the number of requests the server is currently processing.
+      def self.num_active_requests
+        @@num_active_requests.value
+      end
+
+      # Prevents the server from accepting new requests. Any new requests will return an HTTP
+      # 503 status.
+      def self.block_requests!
+        @@block_requests.value = true
+      end
+
+      # Allows the server to accept requests again.
+      def self.allow_requests!
+        @@block_requests.value = false
+      end
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        increment_active_requests
+        if block_requests?
+          block_request(env)
+        else
+          @app.call(env)
+        end
+      ensure
+        decrement_active_requests
+      end
+
+      private
+
+      def block_requests?
+        @@block_requests.true?
+      end
+
+      def block_request(env)
+        [503, {}, []]
+      end
+
+      def increment_active_requests
+        @@num_active_requests.increment
+      end
+
+      def decrement_active_requests
+        @@num_active_requests.decrement
+      end
+    end
+  end
+end
diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab
index 5fd7f0f98bd..09e121e5120 100755
--- a/lib/support/init.d/gitlab
+++ b/lib/support/init.d/gitlab
@@ -48,6 +48,10 @@ gitlab_pages_pid_path="$pid_path/gitlab-pages.pid"
 gitlab_pages_options="-pages-domain example.com -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090"
 gitlab_pages_log="$app_root/log/gitlab-pages.log"
 shell_path="/bin/bash"
+gitaly_enabled=false
+gitaly_dir=$(cd $app_root/../gitaly 2> /dev/null && pwd)
+gitaly_pid_path="$pid_path/gitaly.pid"
+gitaly_log="$app_root/log/gitaly.log"
 
 # Read configuration variable file if it is present
 test -f /etc/default/gitlab && . /etc/default/gitlab
@@ -101,13 +105,20 @@ check_pids(){
       gppid=0
     fi
   fi
+  if [ "$gitaly_enabled" = true ]; then
+    if [ -f "$gitaly_pid_path" ]; then
+      gapid=$(cat "$gitaly_pid_path")
+    else
+      gapid=0
+    fi
+  fi
 }
 
 ## Called when we have started the two processes and are waiting for their pid files.
 wait_for_pids(){
   # We are sleeping a bit here mostly because sidekiq is slow at writing its pid
   i=0;
-  while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || [ ! -f $gitlab_workhorse_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; } || { [ "$gitlab_pages_enabled" = true ] && [ ! -f $gitlab_pages_pid_path ]; }; do
+  while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || [ ! -f $gitlab_workhorse_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; } || { [ "$gitlab_pages_enabled" = true ] && [ ! -f $gitlab_pages_pid_path ]; } || { [ "$gitaly_enabled" = true ] && [ ! -f $gitaly_pid_path ]; }; do
     sleep 0.1;
     i=$((i+1))
     if [ $((i%10)) = 0 ]; then
@@ -164,7 +175,15 @@ check_status(){
       gitlab_pages_status="-1"
     fi
   fi
-  if [ $web_status = 0 ] && [ $sidekiq_status = 0 ] && [ $gitlab_workhorse_status = 0 ] && { [ "$mail_room_enabled" != true ] || [ $mail_room_status = 0 ]; } && { [ "$gitlab_pages_enabled" != true ] || [ $gitlab_pages_status = 0 ]; }; then
+  if [ "$gitaly_enabled" = true ]; then
+    if [ $gapid -ne 0 ]; then
+      kill -0 "$gapid" 2>/dev/null
+      gitaly_status="$?"
+    else
+      gitaly_status="-1"
+    fi
+  fi
+  if [ $web_status = 0 ] && [ $sidekiq_status = 0 ] && [ $gitlab_workhorse_status = 0 ] && { [ "$mail_room_enabled" != true ] || [ $mail_room_status = 0 ]; } && { [ "$gitlab_pages_enabled" != true ] || [ $gitlab_pages_status = 0 ]; } && { [ "$gitaly_enabled" != true ] || [ $gitaly_status = 0 ]; }; then
     gitlab_status=0
   else
     # http://refspecs.linuxbase.org/LSB_4.1.0/LSB-Core-generic/LSB-Core-generic/iniscrptact.html
@@ -213,12 +232,19 @@ check_stale_pids(){
       exit 1
     fi
   fi
+  if [ "$gitaly_enabled" = true ] && [ "$gapid" != "0" ] && [ "$gitaly_status" != "0" ]; then
+    echo "Removing stale Gitaly pid. This is most likely caused by Gitaly crashing the last time it ran."
+    if ! rm "$gitaly_pid_path"; then
+      echo "Unable to remove stale pid, exiting"
+      exit 1
+    fi
+  fi
 }
 
 ## If no parts of the service is running, bail out.
 exit_if_not_running(){
   check_stale_pids
-  if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; }; then
+  if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; } && { [ "$gitaly_enabled" != true ] || [ "$gitaly_status" != "0" ]; }; then
     echo "GitLab is not running."
     exit
   fi
@@ -243,6 +269,9 @@ start_gitlab() {
   if [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" != "0" ]; then
     echo "Starting GitLab Pages"
   fi
+  if [ "$gitaly_enabled" = true ] && [ "$gitaly_status" != "0" ]; then
+    echo "Starting Gitaly"
+  fi
 
   # Then check if the service is running. If it is: don't start again.
   if [ "$web_status" = "0" ]; then
@@ -292,6 +321,16 @@ start_gitlab() {
     fi
   fi
 
+  if [ "$gitaly_enabled" = true ]; then
+    if [ "$gitaly_status" = "0" ]; then
+      echo "Gitaly is already running with pid $gapid, not restarting"
+    else
+      $app_root/bin/daemon_with_pidfile $gitaly_pid_path \
+        $app_root/bin/with_env $gitaly_dir/env \
+          $gitaly_dir/gitaly >> $gitaly_log 2>&1 &
+    fi
+  fi
+
   # Wait for the pids to be planted
   wait_for_pids
   # Finally check the status to tell wether or not GitLab is running
@@ -322,13 +361,17 @@ stop_gitlab() {
     echo "Shutting down gitlab-pages"
     kill -- $(cat $gitlab_pages_pid_path)
   fi
+  if [ "$gitaly_status" = "0" ]; then
+    echo "Shutting down Gitaly"
+    kill -- $(cat $gitaly_pid_path)
+  fi
 
   # If something needs to be stopped, lets wait for it to stop. Never use SIGKILL in a script.
-  while [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; } || { [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" = "0" ]; }; do
+  while [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; } || { [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" = "0" ]; } || { [ "$gitaly_enabled" = true ] && [ "$gitaly_status" = "0" ]; }; do
     sleep 1
     check_status
     printf "."
-    if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; }; then
+    if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; } && { [ "$gitaly_enabled" != true ] || [ "$gitaly_status" != "0" ]; }; then
       printf "\n"
       break
     fi
@@ -343,6 +386,7 @@ stop_gitlab() {
     rm "$mail_room_pid_path" 2>/dev/null
   fi
   rm -f "$gitlab_pages_pid_path"
+  rm -f "$gitaly_pid_path"
 
   print_status
 }
@@ -350,7 +394,7 @@ stop_gitlab() {
 ## Prints the status of GitLab and its components.
 print_status() {
   check_status
-  if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; }; then
+  if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; } && { [ "$gitaly_enabled" != true ] || [ "$gitaly_status" != "0" ]; }; then
     echo "GitLab is not running."
     return
   fi
@@ -383,7 +427,14 @@ print_status() {
         printf "The GitLab Pages is \033[31mnot running\033[0m.\n"
     fi
   fi
-  if [ "$web_status" = "0" ] && [ "$sidekiq_status" = "0" ] && [ "$gitlab_workhorse_status" = "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" = "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" = "0" ]; }; then
+  if [ "$gitaly_enabled" = true ]; then
+    if [ "$gitaly_status" = "0" ]; then
+        echo "Gitaly with pid $gapid is running."
+    else
+        printf "Gitaly is \033[31mnot running\033[0m.\n"
+    fi
+  fi
+  if [ "$web_status" = "0" ] && [ "$sidekiq_status" = "0" ] && [ "$gitlab_workhorse_status" = "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" = "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" = "0" ]; } && { [ "$gitaly_enabled" != true ] || [ "$gitaly_status" = "0" ]; }; then
     printf "GitLab and all its components are \033[32mup and running\033[0m.\n"
   fi
 }
@@ -414,7 +465,7 @@ reload_gitlab(){
 ## Restarts Sidekiq and Unicorn.
 restart_gitlab(){
   check_status
-  if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; } || { [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" = "0" ]; }; then
+  if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; } || { [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" = "0" ]; } || { [ "$gitaly_enabled" = true ] && [ "$gitaly_status" = "0" ]; }; then
     stop_gitlab
   fi
   start_gitlab
diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example
index f6642527639..9472c3c992f 100644
--- a/lib/support/init.d/gitlab.default.example
+++ b/lib/support/init.d/gitlab.default.example
@@ -84,3 +84,7 @@ mail_room_pid_path="$pid_path/mail_room.pid"
 # shell other than "bash"
 # The default is "/bin/bash"
 shell_path="/bin/bash"
+
+# This variable controls whether the init script starts/stops Gitaly
+gitaly_enabled=false
+gitaly_log="$app_root/log/gitaly.log"
diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake
index 098f9851b45..003d57adbbd 100644
--- a/lib/tasks/gitlab/assets.rake
+++ b/lib/tasks/gitlab/assets.rake
@@ -3,16 +3,16 @@ namespace :gitlab do
     desc 'GitLab | Assets | Compile all frontend assets'
     task compile: [
       'yarn:check',
-      'assets:precompile',
+      'rake:assets:precompile',
       'webpack:compile',
-      'gitlab:assets:fix_urls'
+      'fix_urls'
     ]
 
     desc 'GitLab | Assets | Clean up old compiled frontend assets'
-    task clean: ['assets:clean']
+    task clean: ['rake:assets:clean']
 
     desc 'GitLab | Assets | Remove all compiled frontend assets'
-    task purge: ['assets:clobber']
+    task purge: ['rake:assets:clobber']
 
     desc 'GitLab | Assets | Uninstall frontend dependencies'
     task purge_modules: ['yarn:clobber']
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
new file mode 100644
index 00000000000..c288e17ac8d
--- /dev/null
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -0,0 +1,23 @@
+namespace :gitlab do
+  namespace :gitaly do
+    desc "GitLab | Install or upgrade gitaly"
+    task :install, [:dir] => :environment do |t, args|
+      warn_user_is_not_gitlab
+      unless args.dir.present?
+        abort %(Please specify the directory where you want to install gitaly:\n  rake "gitlab:gitaly:install[/home/git/gitaly]")
+      end
+
+      tag = "v#{Gitlab::GitalyClient.expected_server_version}"
+      repo = 'https://gitlab.com/gitlab-org/gitaly.git'
+
+      checkout_or_clone_tag(tag: tag, repo: repo, target_dir: args.dir)
+
+      _, status = Gitlab::Popen.popen(%w[which gmake])
+      command = status.zero? ? 'gmake' : 'make'
+
+      Dir.chdir(args.dir) do
+        run_command!([command])
+      end
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb
index bb755ae689b..cdba2262bc2 100644
--- a/lib/tasks/gitlab/task_helpers.rb
+++ b/lib/tasks/gitlab/task_helpers.rb
@@ -81,7 +81,7 @@ module Gitlab
     def run_command!(command)
       output, status = Gitlab::Popen.popen(command)
 
-      raise Gitlab::TaskFailedError unless status.zero?
+      raise Gitlab::TaskFailedError.new(output) unless status.zero?
 
       output
     end
diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake
index f5caca3ddbf..8938bc515f5 100644
--- a/lib/tasks/migrate/setup_postgresql.rake
+++ b/lib/tasks/migrate/setup_postgresql.rake
@@ -3,10 +3,12 @@ require Rails.root.join('lib/gitlab/database/migration_helpers')
 require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lower_indexes')
 require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes')
 require Rails.root.join('db/migrate/20161212142807_add_lower_path_index_to_routes')
+require Rails.root.join('db/migrate/20170317203554_index_routes_path_for_like')
 
 desc 'GitLab | Sets up PostgreSQL'
 task setup_postgresql: :environment do
   NamespacesProjectsPathLowerIndexes.new.up
   AddUsersLowerUsernameEmailIndexes.new.up
   AddLowerPathIndexToRoutes.new.up
+  IndexRoutesPathForLike.new.up
 end
diff --git a/lib/tasks/tokens.rake b/lib/tasks/tokens.rake
new file mode 100644
index 00000000000..95735f43802
--- /dev/null
+++ b/lib/tasks/tokens.rake
@@ -0,0 +1,38 @@
+require_relative '../../app/models/concerns/token_authenticatable.rb'
+
+namespace :tokens do
+  desc "Reset all GitLab user auth tokens"
+  task reset_all_auth: :environment do
+    reset_all_users_token(:reset_authentication_token!)
+  end
+
+  desc "Reset all GitLab email tokens"
+  task reset_all_email: :environment do
+    reset_all_users_token(:reset_incoming_email_token!)
+  end
+
+  def reset_all_users_token(reset_token_method)
+    TmpUser.find_in_batches do |batch|
+      puts "Processing batch starting with user ID: #{batch.first.id}"
+      STDOUT.flush
+
+      batch.each(&reset_token_method)
+    end
+  end
+end
+
+class TmpUser < ActiveRecord::Base
+  include TokenAuthenticatable
+
+  self.table_name = 'users'
+
+  def reset_authentication_token!
+    write_new_token(:authentication_token)
+    save!(validate: false)
+  end
+
+  def reset_incoming_email_token!
+    write_new_token(:incoming_email_token)
+    save!(validate: false)
+  end
+end