Merge branch 'master' into expiration-date-on-memberships

114 files changed, 2318 insertions, 1485 deletions

diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb
new file mode 100644
index 00000000000..d02b469dac8
--- /dev/null
+++ b/lib/api/access_requests.rb
@@ -0,0 +1,90 @@
+module API
+  class AccessRequests < Grape::API
+    before { authenticate! }
+
+    helpers ::API::Helpers::MembersHelpers
+
+    %w[group project].each do |source_type|
+      resource source_type.pluralize do
+        # Get a list of group/project access requests viewable by the authenticated user.
+        #
+        # Parameters:
+        #   id (required) - The group/project ID
+        #
+        # Example Request:
+        #  GET /groups/:id/access_requests
+        #  GET /projects/:id/access_requests
+        get ":id/access_requests" do
+          source = find_source(source_type, params[:id])
+          authorize_admin_source!(source_type, source)
+
+          access_requesters = paginate(source.requesters.includes(:user))
+
+          present access_requesters.map(&:user), with: Entities::AccessRequester, access_requesters: access_requesters
+        end
+
+        # Request access to the group/project
+        #
+        # Parameters:
+        #   id (required) - The group/project ID
+        #
+        # Example Request:
+        #  POST /groups/:id/access_requests
+        #  POST /projects/:id/access_requests
+        post ":id/access_requests" do
+          source = find_source(source_type, params[:id])
+          access_requester = source.request_access(current_user)
+
+          if access_requester.persisted?
+            present access_requester.user, with: Entities::AccessRequester, access_requester: access_requester
+          else
+            render_validation_error!(access_requester)
+          end
+        end
+
+        # Approve a group/project access request
+        #
+        # Parameters:
+        #   id (required) - The group/project ID
+        #   user_id (required) - The user ID of the access requester
+        #   access_level (optional) - Access level
+        #
+        # Example Request:
+        #   PUT /groups/:id/access_requests/:user_id/approve
+        #   PUT /projects/:id/access_requests/:user_id/approve
+        put ':id/access_requests/:user_id/approve' do
+          required_attributes! [:user_id]
+          source = find_source(source_type, params[:id])
+          authorize_admin_source!(source_type, source)
+
+          member = source.requesters.find_by!(user_id: params[:user_id])
+          if params[:access_level]
+            member.update(access_level: params[:access_level])
+          end
+          member.accept_request
+
+          status :created
+          present member.user, with: Entities::Member, member: member
+        end
+
+        # Deny a group/project access request
+        #
+        # Parameters:
+        #   id (required) - The group/project ID
+        #   user_id (required) - The user ID of the access requester
+        #
+        # Example Request:
+        #   DELETE /groups/:id/access_requests/:user_id
+        #   DELETE /projects/:id/access_requests/:user_id
+        delete ":id/access_requests/:user_id" do
+          required_attributes! [:user_id]
+          source = find_source(source_type, params[:id])
+
+          access_requester = source.requesters.find_by!(user_id: params[:user_id])
+
+          ::Members::DestroyService.new(access_requester, current_user).execute
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/api.rb b/lib/api/api.rb
index 3d7d67510a8..d43af3f24e9 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -3,10 +3,20 @@ module API
     include APIGuard
     version 'v3', using: :path
 
+    rescue_from Gitlab::Access::AccessDeniedError do
+      rack_response({ 'message' => '403 Forbidden' }.to_json, 403)
+    end
+
     rescue_from ActiveRecord::RecordNotFound do
       rack_response({ 'message' => '404 Not found' }.to_json, 404)
     end
 
+    # Retain 405 error rather than a 500 error for Grape 0.15.0+.
+    # See: https://github.com/ruby-grape/grape/commit/252bfd27c320466ec3c0751812cf44245e97e5de
+    rescue_from Grape::Exceptions::Base do |e|
+      error! e.message, e.status, e.headers
+    end
+
     rescue_from :all do |exception|
       # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60
       # why is this not wrapped in something reusable?
@@ -26,26 +36,27 @@ module API
     # Ensure the namespace is right, otherwise we might load Grape::API::Helpers
     helpers ::API::Helpers
 
+    mount ::API::AccessRequests
     mount ::API::AwardEmoji
     mount ::API::Branches
     mount ::API::Builds
     mount ::API::CommitStatuses
     mount ::API::Commits
     mount ::API::DeployKeys
+    mount ::API::Environments
     mount ::API::Files
-    mount ::API::GroupMembers
     mount ::API::Groups
     mount ::API::Internal
     mount ::API::Issues
     mount ::API::Keys
     mount ::API::Labels
     mount ::API::LicenseTemplates
+    mount ::API::Members
     mount ::API::MergeRequests
     mount ::API::Milestones
     mount ::API::Namespaces
     mount ::API::Notes
     mount ::API::ProjectHooks
-    mount ::API::ProjectMembers
     mount ::API::ProjectSnippets
     mount ::API::Projects
     mount ::API::Repositories
diff --git a/lib/api/branches.rb b/lib/api/branches.rb
index a77afe634f6..b615703df93 100644
--- a/lib/api/branches.rb
+++ b/lib/api/branches.rb
@@ -61,22 +61,27 @@ module API
           name: @branch.name
         }
 
-        unless developers_can_merge.nil?
-          protected_branch_params.merge!({
-            merge_access_level_attributes: {
-              access_level: developers_can_merge ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER
-            }
-          })
+        # If `developers_can_merge` is switched off, _all_ `DEVELOPER`
+        # merge_access_levels need to be deleted.
+        if developers_can_merge == false
+          protected_branch.merge_access_levels.where(access_level: Gitlab::Access::DEVELOPER).destroy_all
         end
 
-        unless developers_can_push.nil?
-          protected_branch_params.merge!({
-            push_access_level_attributes: {
-              access_level: developers_can_push ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER
-            }
-          })
+        # If `developers_can_push` is switched off, _all_ `DEVELOPER`
+        # push_access_levels need to be deleted.
+        if developers_can_push == false
+          protected_branch.push_access_levels.where(access_level: Gitlab::Access::DEVELOPER).destroy_all
         end
 
+        protected_branch_params.merge!(
+          merge_access_levels_attributes: [{
+            access_level: developers_can_merge ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER
+          }],
+          push_access_levels_attributes: [{
+            access_level: developers_can_push ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER
+          }]
+        )
+
         if protected_branch
           service = ProtectedBranches::UpdateService.new(user_project, current_user, protected_branch_params)
           service.execute(protected_branch)
diff --git a/lib/api/commits.rb b/lib/api/commits.rb
index 4a11c8e3620..b4eaf1813d4 100644
--- a/lib/api/commits.rb
+++ b/lib/api/commits.rb
@@ -54,7 +54,7 @@ module API
         sha = params[:sha]
         commit = user_project.commit(sha)
         not_found! "Commit" unless commit
-        commit.diffs.to_a
+        commit.raw_diffs.to_a
       end
 
       # Get a commit's comments
@@ -96,7 +96,7 @@ module API
         }
 
         if params[:path] && params[:line] && params[:line_type]
-          commit.diffs(all_diffs: true).each do |diff|
+          commit.raw_diffs(all_diffs: true).each do |diff|
             next unless diff.new_path == params[:path]
             lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line)
 
diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb
index 5c570b5e5ca..825e05fbae3 100644
--- a/lib/api/deploy_keys.rb
+++ b/lib/api/deploy_keys.rb
@@ -10,6 +10,9 @@ module API
       present keys, with: Entities::SSHKey
     end
 
+    params do
+      requires :id, type: String, desc: 'The ID of the project'
+    end
     resource :projects do
       before { authorize_admin_project }
 
@@ -17,52 +20,43 @@ module API
       # Use "projects/:id/deploy_keys/..." instead.
       #
       %w(keys deploy_keys).each do |path|
-        # Get a specific project's deploy keys
-        #
-        # Example Request:
-        #   GET /projects/:id/deploy_keys
+        desc "Get a specific project's deploy keys" do
+          success Entities::SSHKey
+        end
         get ":id/#{path}" do
           present user_project.deploy_keys, with: Entities::SSHKey
         end
 
-        # Get single deploy key owned by currently authenticated user
-        #
-        # Example Request:
-        #   GET /projects/:id/deploy_keys/:key_id
+        desc 'Get single deploy key' do
+          success Entities::SSHKey
+        end
+        params do
+          requires :key_id, type: Integer, desc: 'The ID of the deploy key'
+        end
         get ":id/#{path}/:key_id" do
           key = user_project.deploy_keys.find params[:key_id]
           present key, with: Entities::SSHKey
         end
 
-        # Add new deploy key to currently authenticated user
-        # If deploy key already exists - it will be joined to project
-        # but only if original one was accessible by same user
-        #
-        # Parameters:
-        #   key (required) - New deploy Key
-        #   title (required) - New deploy Key's title
-        # Example Request:
-        #   POST /projects/:id/deploy_keys
+        # TODO: for 9.0 we should check if params are there with the params block
+        # grape provides, at this point we'd change behaviour so we can't
+        # Behaviour now if you don't provide all required params: it renders a
+        # validation error or two.
+        desc 'Add new deploy key to currently authenticated user' do
+          success Entities::SSHKey
+        end
         post ":id/#{path}" do
           attrs = attributes_for_keys [:title, :key]
+          attrs[:key].strip! if attrs[:key]
 
-          if attrs[:key].present?
-            attrs[:key].strip!
-
-            # check if key already exist in project
-            key = user_project.deploy_keys.find_by(key: attrs[:key])
-            if key
-              present key, with: Entities::SSHKey
-              next
-            end
+          key = user_project.deploy_keys.find_by(key: attrs[:key])
+          present key, with: Entities::SSHKey if key
 
-            # Check for available deploy keys in other projects
-            key = current_user.accessible_deploy_keys.find_by(key: attrs[:key])
-            if key
-              user_project.deploy_keys << key
-              present key, with: Entities::SSHKey
-              next
-            end
+          # Check for available deploy keys in other projects
+          key = current_user.accessible_deploy_keys.find_by(key: attrs[:key])
+          if key
+            user_project.deploy_keys << key
+            present key, with: Entities::SSHKey
           end
 
           key = DeployKey.new attrs
@@ -74,12 +68,46 @@ module API
           end
         end
 
-        # Delete existing deploy key of currently authenticated user
-        #
-        # Example Request:
-        #   DELETE /projects/:id/deploy_keys/:key_id
+        desc 'Enable a deploy key for a project' do
+          detail 'This feature was added in GitLab 8.11'
+          success Entities::SSHKey
+        end
+        params do
+          requires :key_id, type: Integer, desc: 'The ID of the deploy key'
+        end
+        post ":id/#{path}/:key_id/enable" do
+          key = ::Projects::EnableDeployKeyService.new(user_project,
+                                                        current_user, declared(params)).execute
+
+          if key
+            present key, with: Entities::SSHKey
+          else
+            not_found!('Deploy Key')
+          end
+        end
+
+        desc 'Disable a deploy key for a project' do
+          detail 'This feature was added in GitLab 8.11'
+          success Entities::SSHKey
+        end
+        params do
+          requires :key_id, type: Integer, desc: 'The ID of the deploy key'
+        end
+        delete ":id/#{path}/:key_id/disable" do
+          key = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id])
+          key.destroy
+
+          present key.deploy_key, with: Entities::SSHKey
+        end
+
+        desc 'Delete existing deploy key of currently authenticated user' do
+          success Key
+        end
+        params do
+          requires :key_id, type: Integer, desc: 'The ID of the deploy key'
+        end
         delete ":id/#{path}/:key_id" do
-          key = user_project.deploy_keys.find params[:key_id]
+          key = user_project.deploy_keys.find(params[:key_id])
           key.destroy
         end
       end
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 595e634b7cb..dd9d8515853 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -48,7 +48,8 @@ module API
 
     class ProjectHook < Hook
       expose :project_id, :push_events
-      expose :issues_events, :merge_requests_events, :tag_push_events, :note_events, :build_events
+      expose :issues_events, :merge_requests_events, :tag_push_events
+      expose :note_events, :build_events, :pipeline_events
       expose :enable_ssl_verification
     end
 
@@ -91,9 +92,17 @@ module API
       end
     end
 
-    class ProjectMember < UserBasic
+    class Member < UserBasic
       expose :access_level do |user, options|
-        options[:project].project_members.find_by(user_id: user.id).access_level
+        member = options[:member] || options[:members].find { |m| m.user_id == user.id }
+        member.access_level
+      end
+    end
+
+    class AccessRequester < UserBasic
+      expose :requested_at do |user, options|
+        access_requester = options[:access_requester] || options[:access_requesters].find { |m| m.user_id == user.id }
+        access_requester.requested_at
       end
       expose :expires_at do |user, options|
         options[:project].project_members.find_by(user_id: user.id).expires_at
@@ -111,12 +120,6 @@ module API
       expose :shared_projects, using: Entities::Project
     end
 
-    class GroupMember < UserBasic
-      expose :access_level do |user, options|
-        options[:group].group_members.find_by(user_id: user.id).access_level
-      end
-    end
-
     class RepoBranch < Grape::Entity
       expose :name
 
@@ -130,12 +133,14 @@ module API
 
       expose :developers_can_push do |repo_branch, options|
         project = options[:project]
-        project.protected_branches.matching(repo_branch.name).any? { |protected_branch| protected_branch.push_access_level.access_level == Gitlab::Access::DEVELOPER }
+        access_levels = project.protected_branches.matching(repo_branch.name).map(&:push_access_levels).flatten
+        access_levels.any? { |access_level| access_level.access_level == Gitlab::Access::DEVELOPER }
       end
 
       expose :developers_can_merge do |repo_branch, options|
         project = options[:project]
-        project.protected_branches.matching(repo_branch.name).any? { |protected_branch| protected_branch.merge_access_level.access_level == Gitlab::Access::DEVELOPER }
+        access_levels = project.protected_branches.matching(repo_branch.name).map(&:merge_access_levels).flatten
+        access_levels.any? { |access_level| access_level.access_level == Gitlab::Access::DEVELOPER }
       end
     end
 
@@ -227,7 +232,7 @@ module API
 
     class MergeRequestChanges < MergeRequest
       expose :diffs, as: :changes, using: Entities::RepoDiff do |compare, _|
-        compare.diffs(all_diffs: true).to_a
+        compare.raw_diffs(all_diffs: true).to_a
       end
     end
 
@@ -328,7 +333,7 @@ module API
       expose :id, :path, :kind
     end
 
-    class Member < Grape::Entity
+    class MemberAccess < Grape::Entity
       expose :access_level
       expose :notification_level do |member, options|
         if member.notification_setting
@@ -337,15 +342,16 @@ module API
       end
     end
 
-    class ProjectAccess < Member
+    class ProjectAccess < MemberAccess
     end
 
-    class GroupAccess < Member
+    class GroupAccess < MemberAccess
     end
 
     class ProjectService < Grape::Entity
       expose :id, :title, :created_at, :updated_at, :active
-      expose :push_events, :issues_events, :merge_requests_events, :tag_push_events, :note_events, :build_events
+      expose :push_events, :issues_events, :merge_requests_events
+      expose :tag_push_events, :note_events, :build_events, :pipeline_events
       # Expose serialized properties
       expose :properties do |service, options|
         field_names = service.fields.
@@ -499,6 +505,10 @@ module API
       expose :key, :value
     end
 
+    class Environment < Grape::Entity
+      expose :id, :name, :external_url
+    end
+
     class RepoLicense < Grape::Entity
       expose :key, :name, :nickname
       expose :featured, as: :popular
diff --git a/lib/api/environments.rb b/lib/api/environments.rb
new file mode 100644
index 00000000000..819f80d8365
--- /dev/null
+++ b/lib/api/environments.rb
@@ -0,0 +1,83 @@
+module API
+  # Environments RESTfull API endpoints
+  class Environments < Grape::API
+    before { authenticate! }
+
+    params do
+      requires :id, type: String, desc: 'The project ID'
+    end
+    resource :projects do
+      desc 'Get all environments of the project' do
+        detail 'This feature was introduced in GitLab 8.11.'
+        success Entities::Environment
+      end
+      params do
+        optional :page,     type: Integer, desc: 'Page number of the current request'
+        optional :per_page, type: Integer, desc: 'Number of items per page'
+      end
+      get ':id/environments' do
+        authorize! :read_environment, user_project
+
+        present paginate(user_project.environments), with: Entities::Environment
+      end
+
+      desc 'Creates a new environment' do
+        detail 'This feature was introduced in GitLab 8.11.'
+        success Entities::Environment
+      end
+      params do
+        requires :name,           type: String,   desc: 'The name of the environment to be created'
+        optional :external_url,   type: String,   desc: 'URL on which this deployment is viewable'
+      end
+      post ':id/environments' do
+        authorize! :create_environment, user_project
+
+        create_params = declared(params, include_parent_namespaces: false).to_h
+        environment = user_project.environments.create(create_params)
+
+        if environment.persisted?
+          present environment, with: Entities::Environment
+        else
+          render_validation_error!(environment)
+        end
+      end
+
+      desc 'Updates an existing environment' do
+        detail 'This feature was introduced in GitLab 8.11.'
+        success Entities::Environment
+      end
+      params do
+        requires :environment_id, type: Integer,  desc: 'The environment ID'
+        optional :name,           type: String,   desc: 'The new environment name'
+        optional :external_url,   type: String,   desc: 'The new URL on which this deployment is viewable'
+      end
+      put ':id/environments/:environment_id' do
+        authorize! :update_environment, user_project
+
+        environment = user_project.environments.find(params[:environment_id])
+        
+        update_params = declared(params, include_missing: false).extract!(:name, :external_url).to_h
+        if environment.update(update_params)
+          present environment, with: Entities::Environment
+        else
+          render_validation_error!(environment)
+        end
+      end
+
+      desc 'Deletes an existing environment' do
+        detail 'This feature was introduced in GitLab 8.11.'
+        success Entities::Environment
+      end
+      params do
+        requires :environment_id, type: Integer,  desc: 'The environment ID'
+      end
+      delete ':id/environments/:environment_id' do
+        authorize! :update_environment, user_project
+
+        environment = user_project.environments.find(params[:environment_id])
+
+        present environment.destroy, with: Entities::Environment
+      end
+    end
+  end
+end
diff --git a/lib/api/group_members.rb b/lib/api/group_members.rb
deleted file mode 100644
index dbe5bb08d3f..00000000000
--- a/lib/api/group_members.rb
+++ /dev/null
@@ -1,87 +0,0 @@
-module API
-  class GroupMembers < Grape::API
-    before { authenticate! }
-
-    resource :groups do
-      # Get a list of group members viewable by the authenticated user.
-      #
-      # Example Request:
-      #  GET /groups/:id/members
-      get ":id/members" do
-        group = find_group(params[:id])
-        users = group.users
-        present users, with: Entities::GroupMember, group: group
-      end
-
-      # Add a user to the list of group members
-      #
-      # Parameters:
-      #   id (required) - group id
-      #   user_id (required) - the users id
-      #   access_level (required) - Project access level
-      # Example Request:
-      #  POST /groups/:id/members
-      post ":id/members" do
-        group = find_group(params[:id])
-        authorize! :admin_group, group
-        required_attributes! [:user_id, :access_level]
-
-        unless validate_access_level?(params[:access_level])
-          render_api_error!("Wrong access level", 422)
-        end
-
-        if group.group_members.find_by(user_id: params[:user_id])
-          render_api_error!("Already exists", 409)
-        end
-
-        group.add_users([params[:user_id]], params[:access_level], current_user)
-        member = group.group_members.find_by(user_id: params[:user_id])
-        present member.user, with: Entities::GroupMember, group: group
-      end
-
-      # Update group member
-      #
-      # Parameters:
-      #   id (required) - The ID of a group
-      #   user_id (required) - The ID of a group member
-      #   access_level (required) - Project access level
-      # Example Request:
-      #   PUT /groups/:id/members/:user_id
-      put ':id/members/:user_id' do
-        group = find_group(params[:id])
-        authorize! :admin_group, group
-        required_attributes! [:access_level]
-
-        group_member = group.group_members.find_by(user_id: params[:user_id])
-        not_found!('User can not be found') if group_member.nil?
-
-        if group_member.update_attributes(access_level: params[:access_level])
-          @member = group_member.user
-          present @member, with: Entities::GroupMember, group: group
-        else
-          handle_member_errors group_member.errors
-        end
-      end
-
-      # Remove member.
-      #
-      # Parameters:
-      #   id (required) - group id
-      #   user_id (required) - the users id
-      #
-      # Example Request:
-      #   DELETE /groups/:id/members/:user_id
-      delete ":id/members/:user_id" do
-        group = find_group(params[:id])
-        authorize! :admin_group, group
-        member = group.group_members.find_by(user_id: params[:user_id])
-
-        if member.nil?
-          render_api_error!("404 Not Found - user_id:#{params[:user_id]} not a member of group #{group.name}", 404)
-        else
-          member.destroy
-        end
-      end
-    end
-  end
-end
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 130509cdad6..d0469d6602d 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -28,7 +28,7 @@ module API
 
       # If the sudo is the current user do nothing
       if identifier && !(@current_user.id == identifier || @current_user.username == identifier)
-        render_api_error!('403 Forbidden: Must be admin to use sudo', 403) unless @current_user.is_admin?
+        forbidden!('Must be admin to use sudo') unless @current_user.is_admin?
         @current_user = User.by_username_or_id(identifier)
         not_found!("No user id or username for: #{identifier}") if @current_user.nil?
       end
@@ -49,16 +49,15 @@ module API
 
     def user_project
       @project ||= find_project(params[:id])
-      @project || not_found!("Project")
     end
 
     def find_project(id)
       project = Project.find_with_namespace(id) || Project.find_by(id: id)
 
-      if project && can?(current_user, :read_project, project)
+      if can?(current_user, :read_project, project)
         project
       else
-        nil
+        not_found!('Project')
       end
     end
 
@@ -89,11 +88,7 @@ module API
     end
 
     def find_group(id)
-      begin
-        group = Group.find(id)
-      rescue ActiveRecord::RecordNotFound
-        group = Group.find_by!(path: id)
-      end
+      group = Group.find_by(path: id) || Group.find_by(id: id)
 
       if can?(current_user, :read_group, group)
         group
@@ -135,7 +130,7 @@ module API
     end
 
     def authorize!(action, subject)
-      forbidden! unless abilities.allowed?(current_user, action, subject)
+      forbidden! unless can?(current_user, action, subject)
     end
 
     def authorize_push_project
@@ -197,10 +192,6 @@ module API
       errors
     end
 
-    def validate_access_level?(level)
-      Gitlab::Access.options_with_owner.values.include? level.to_i
-    end
-
     # Checks the occurrences of datetime attributes, each attribute if present in the params hash must be in ISO 8601
     # format (YYYY-MM-DDTHH:MM:SSZ) or a Bad Request error is invoked.
     #
@@ -411,11 +402,6 @@ module API
       File.read(Gitlab.config.gitlab_shell.secret_file).chomp
     end
 
-    def handle_member_errors(errors)
-      error!(errors[:access_level], 422) if errors[:access_level].any?
-      not_found!(errors)
-    end
-
     def send_git_blob(repository, blob)
       env['api.format'] = :txt
       content_type 'text/plain'
diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb
new file mode 100644
index 00000000000..90114f6f667
--- /dev/null
+++ b/lib/api/helpers/members_helpers.rb
@@ -0,0 +1,13 @@
+module API
+  module Helpers
+    module MembersHelpers
+      def find_source(source_type, id)
+        public_send("find_#{source_type}", id)
+      end
+
+      def authorize_admin_source!(source_type, source)
+        authorize! :"admin_#{source_type}", source
+      end
+    end
+  end
+end
diff --git a/lib/api/internal.rb b/lib/api/internal.rb
index 959b700de78..d8e9ac406c4 100644
--- a/lib/api/internal.rb
+++ b/lib/api/internal.rb
@@ -74,6 +74,10 @@ module API
         response
       end
 
+      get "/merge_request_urls" do
+        ::MergeRequests::GetUrlsService.new(project).execute(params[:changes])
+      end
+
       #
       # Discover user by ssh key
       #
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index c4d3134da6c..077258faee1 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -3,8 +3,6 @@ module API
   class Issues < Grape::API
     before { authenticate! }
 
-    helpers ::Gitlab::AkismetHelper
-
     helpers do
       def filter_issues_state(issues, state)
         case state
diff --git a/lib/api/members.rb b/lib/api/members.rb
new file mode 100644
index 00000000000..a45285b9c3d
--- /dev/null
+++ b/lib/api/members.rb
@@ -0,0 +1,156 @@
+module API
+  class Members < Grape::API
+    before { authenticate! }
+
+    helpers ::API::Helpers::MembersHelpers
+
+    %w[group project].each do |source_type|
+      resource source_type.pluralize do
+        # Get a list of group/project members viewable by the authenticated user.
+        #
+        # Parameters:
+        #   id (required) - The group/project ID
+        #   query         - Query string
+        #
+        # Example Request:
+        #   GET /groups/:id/members
+        #   GET /projects/:id/members
+        get ":id/members" do
+          source = find_source(source_type, params[:id])
+
+          members = source.members.includes(:user)
+          members = members.joins(:user).merge(User.search(params[:query])) if params[:query]
+          members = paginate(members)
+
+          present members.map(&:user), with: Entities::Member, members: members
+        end
+
+        # Get a group/project member
+        #
+        # Parameters:
+        #   id (required) - The group/project ID
+        #   user_id (required) - The user ID of the member
+        #
+        # Example Request:
+        #   GET /groups/:id/members/:user_id
+        #   GET /projects/:id/members/:user_id
+        get ":id/members/:user_id" do
+          source = find_source(source_type, params[:id])
+
+          members = source.members
+          member = members.find_by!(user_id: params[:user_id])
+
+          present member.user, with: Entities::Member, member: member
+        end
+
+        # Add a new group/project member
+        #
+        # Parameters:
+        #   id (required) - The group/project ID
+        #   user_id (required) - The user ID of the new member
+        #   access_level (required) - A valid access level
+        #
+        # Example Request:
+        #   POST /groups/:id/members
+        #   POST /projects/:id/members
+        post ":id/members" do
+          source = find_source(source_type, params[:id])
+          authorize_admin_source!(source_type, source)
+          required_attributes! [:user_id, :access_level]
+
+          access_requester = source.requesters.find_by(user_id: params[:user_id])
+          if access_requester
+            # We pass current_user = access_requester so that the requester doesn't
+            # receive a "access denied" email
+            ::Members::DestroyService.new(access_requester, access_requester.user).execute
+          end
+
+          member = source.members.find_by(user_id: params[:user_id])
+
+          # This is to ensure back-compatibility but 409 behavior should be used
+          # for both project and group members in 9.0!
+          conflict!('Member already exists') if source_type == 'group' && member
+
+          unless member
+            source.add_user(params[:user_id], params[:access_level], current_user)
+            member = source.members.find_by(user_id: params[:user_id])
+          end
+
+          if member
+            present member.user, with: Entities::Member, member: member
+          else
+            # Since `source.add_user` doesn't return a member object, we have to
+            # build a new one and populate its errors in order to render them.
+            member = source.members.build(attributes_for_keys([:user_id, :access_level]))
+            member.valid? # populate the errors
+
+            # This is to ensure back-compatibility but 400 behavior should be used
+            # for all validation errors in 9.0!
+            render_api_error!('Access level is not known', 422) if member.errors.key?(:access_level)
+            render_validation_error!(member)
+          end
+        end
+
+        # Update a group/project member
+        #
+        # Parameters:
+        #   id (required) - The group/project ID
+        #   user_id (required) - The user ID of the member
+        #   access_level (required) - A valid access level
+        #
+        # Example Request:
+        #   PUT /groups/:id/members/:user_id
+        #   PUT /projects/:id/members/:user_id
+        put ":id/members/:user_id" do
+          source = find_source(source_type, params[:id])
+          authorize_admin_source!(source_type, source)
+          required_attributes! [:user_id, :access_level]
+
+          member = source.members.find_by!(user_id: params[:user_id])
+          attrs = attributes_for_keys [:access_level]
+
+          if member.update_attributes(attrs)
+            present member.user, with: Entities::Member, member: member
+          else
+            # This is to ensure back-compatibility but 400 behavior should be used
+            # for all validation errors in 9.0!
+            render_api_error!('Access level is not known', 422) if member.errors.key?(:access_level)
+            render_validation_error!(member)
+          end
+        end
+
+        # Remove a group/project member
+        #
+        # Parameters:
+        #   id (required) - The group/project ID
+        #   user_id (required) - The user ID of the member
+        #
+        # Example Request:
+        #   DELETE /groups/:id/members/:user_id
+        #   DELETE /projects/:id/members/:user_id
+        delete ":id/members/:user_id" do
+          source = find_source(source_type, params[:id])
+          required_attributes! [:user_id]
+
+          # This is to ensure back-compatibility but find_by! should be used
+          # in that casse in 9.0!
+          member = source.members.find_by(user_id: params[:user_id])
+
+          # This is to ensure back-compatibility but this should be removed in
+          # favor of find_by! in 9.0!
+          not_found!("Member: user_id:#{params[:user_id]}") if source_type == 'group' && member.nil?
+
+          # This is to ensure back-compatibility but 204 behavior should be used
+          # for all DELETE endpoints in 9.0!
+          if member.nil?
+            { message: "Access revoked", id: params[:user_id].to_i }
+          else
+            ::Members::DestroyService.new(member, current_user).execute
+
+            present member.user, with: Entities::Member, member: member
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb
index 6bb70bc8bc3..3f63cd678e8 100644
--- a/lib/api/project_hooks.rb
+++ b/lib/api/project_hooks.rb
@@ -45,6 +45,7 @@ module API
           :tag_push_events,
           :note_events,
           :build_events,
+          :pipeline_events,
           :enable_ssl_verification
         ]
         @hook = user_project.hooks.new(attrs)
@@ -78,6 +79,7 @@ module API
           :tag_push_events,
           :note_events,
           :build_events,
+          :pipeline_events,
           :enable_ssl_verification
         ]
 
diff --git a/lib/api/project_members.rb b/lib/api/project_members.rb
deleted file mode 100644
index 590658c5023..00000000000
--- a/lib/api/project_members.rb
+++ /dev/null
@@ -1,113 +0,0 @@
-module API
-  # Projects members API
-  class ProjectMembers < Grape::API
-    before { authenticate! }
-
-    resource :projects do
-      # Get a project team members
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   query         - Query string
-      # Example Request:
-      #   GET /projects/:id/members
-      get ":id/members" do
-        if params[:query].present?
-          @members = paginate user_project.users.where("username LIKE ?", "%#{params[:query]}%")
-        else
-          @members = paginate user_project.users
-        end
-        present @members, with: Entities::ProjectMember, project: user_project
-      end
-
-      # Get a project team members
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   user_id (required) - The ID of a user
-      # Example Request:
-      #   GET /projects/:id/members/:user_id
-      get ":id/members/:user_id" do
-        @member = user_project.users.find params[:user_id]
-        present @member, with: Entities::ProjectMember, project: user_project
-      end
-
-      # Add a new project team member
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   user_id (required) - The ID of a user
-      #   access_level (required) - Project access level
-      #   expires_at (optional) - Date string in the format YEAR-MONTH-DAY
-      # Example Request:
-      #   POST /projects/:id/members
-      post ":id/members" do
-        authorize! :admin_project, user_project
-        required_attributes! [:user_id, :access_level]
-
-        # either the user is already a team member or a new one
-        project_member = user_project.project_member(params[:user_id])
-        if project_member.nil?
-          project_member = user_project.project_members.new(
-            user_id: params[:user_id],
-            access_level: params[:access_level],
-            expires_at: params[:expires_at]
-          )
-        end
-
-        if project_member.save
-          @member = project_member.user
-          present @member, with: Entities::ProjectMember, project: user_project
-        else
-          handle_member_errors project_member.errors
-        end
-      end
-
-      # Update project team member
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   user_id (required) - The ID of a team member
-      #   access_level (required) - Project access level
-      #   expires_at (optional) - Date string in the format YEAR-MONTH-DAY
-      # Example Request:
-      #   PUT /projects/:id/members/:user_id
-      put ":id/members/:user_id" do
-        authorize! :admin_project, user_project
-        required_attributes! [:access_level]
-        attrs = attributes_for_keys [:access_level, :expires_at]
-        project_member = user_project.project_members.find_by(user_id: params[:user_id])
-        not_found!("User can not be found") if project_member.nil?
-
-        if project_member.update_attributes(attrs)
-          @member = project_member.user
-          present @member, with: Entities::ProjectMember, project: user_project
-        else
-          handle_member_errors project_member.errors
-        end
-      end
-
-      # Remove a team member from project
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   user_id (required) - The ID of a team member
-      # Example Request:
-      #   DELETE /projects/:id/members/:user_id
-      delete ":id/members/:user_id" do
-        project_member = user_project.project_members.find_by(user_id: params[:user_id])
-
-        unless current_user.can?(:admin_project, user_project) ||
-                current_user.can?(:destroy_project_member, project_member)
-          forbidden!
-        end
-
-        if project_member.nil?
-          { message: "Access revoked", id: params[:user_id].to_i }
-        else
-          project_member.destroy
-        end
-      end
-    end
-  end
-end
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 8fed7db8803..60cfc103afd 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -323,7 +323,7 @@ module API
       #   DELETE /projects/:id
       delete ":id" do
         authorize! :remove_project, user_project
-        ::Projects::DestroyService.new(user_project, current_user, {}).pending_delete!
+        ::Projects::DestroyService.new(user_project, current_user, {}).async_execute
       end
 
       # Mark this project as forked from another
diff --git a/lib/api/templates.rb b/lib/api/templates.rb
index 18408797756..b9e718147e1 100644
--- a/lib/api/templates.rb
+++ b/lib/api/templates.rb
@@ -1,21 +1,28 @@
 module API
   class Templates < Grape::API
-    TEMPLATE_TYPES = {
-      gitignores:     Gitlab::Template::Gitignore,
-      gitlab_ci_ymls: Gitlab::Template::GitlabCiYml
+    GLOBAL_TEMPLATE_TYPES = {
+      gitignores:     Gitlab::Template::GitignoreTemplate,
+      gitlab_ci_ymls: Gitlab::Template::GitlabCiYmlTemplate
     }.freeze
 
-    TEMPLATE_TYPES.each do |template, klass|
+    helpers do
+      def render_response(template_type, template)
+        not_found!(template_type.to_s.singularize) unless template
+        present template, with: Entities::Template
+      end
+    end
+
+    GLOBAL_TEMPLATE_TYPES.each do |template_type, klass|
       # Get the list of the available template
       #
       # Example Request:
       #   GET /gitignores
       #   GET /gitlab_ci_ymls
-      get template.to_s do
+      get template_type.to_s do
         present klass.all, with: Entities::TemplatesList
       end
 
-      # Get the text for a specific template
+      # Get the text for a specific template present in local filesystem
       #
       # Parameters:
       #   name (required) - The name of a template
@@ -23,13 +30,10 @@ module API
       # Example Request:
       #   GET /gitignores/Elixir
       #   GET /gitlab_ci_ymls/Ruby
-      get "#{template}/:name" do
+      get "#{template_type}/:name" do
         required_attributes! [:name]
-
         new_template = klass.find(params[:name])
-        not_found!(template.to_s.singularize) unless new_template
-
-        present new_template, with: Entities::Template
+        render_response(template_type, new_template)
       end
     end
   end
diff --git a/lib/api/todos.rb b/lib/api/todos.rb
index 26c24c3baff..19df13d8aac 100644
--- a/lib/api/todos.rb
+++ b/lib/api/todos.rb
@@ -61,9 +61,9 @@ module API
       #
       delete ':id' do
         todo = current_user.todos.find(params[:id])
-        todo.done
+        TodoService.new.mark_todos_as_done([todo], current_user)
 
-        present todo, with: Entities::Todo, current_user: current_user
+        present todo.reload, with: Entities::Todo, current_user: current_user
       end
 
       # Mark all todos as done
@@ -73,9 +73,7 @@ module API
       #
       delete do
         todos = find_todos
-        todos.each(&:done)
-
-        todos.length
+        TodoService.new.mark_todos_as_done(todos, current_user)
       end
     end
   end
diff --git a/lib/backup/files.rb b/lib/backup/files.rb
index 654b4d1c896..cedbb289f6a 100644
--- a/lib/backup/files.rb
+++ b/lib/backup/files.rb
@@ -27,7 +27,7 @@ module Backup
 
     def backup_existing_files_dir
       timestamped_files_path = File.join(files_parent_dir, "#{name}.#{Time.now.to_i}")
-      if File.exists?(app_files_dir)
+      if File.exist?(app_files_dir)
         FileUtils.mv(app_files_dir, File.expand_path(timestamped_files_path))
       end
     end
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index 2ff3e3bdfb0..0dfffaf0bc6 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -114,7 +114,7 @@ module Backup
 
       tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_backup.tar")
 
-      unless File.exists?(tar_file)
+      unless File.exist?(tar_file)
         puts "The specified backup doesn't exist!"
         exit 1
       end
diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb
index 1f5917b8127..f117fc3d37d 100644
--- a/lib/backup/repository.rb
+++ b/lib/backup/repository.rb
@@ -28,7 +28,7 @@ module Backup
 
         wiki = ProjectWiki.new(project)
 
-        if File.exists?(path_to_repo(wiki))
+        if File.exist?(path_to_repo(wiki))
           $progress.print " * #{wiki.path_with_namespace} ... "
           if wiki.repository.empty?
             $progress.puts " [SKIPPED]".color(:cyan)
@@ -49,7 +49,7 @@ module Backup
 
     def restore
       Gitlab.config.repositories.storages.each do |name, path|
-        next unless File.exists?(path)
+        next unless File.exist?(path)
 
         # Move repos dir to 'repositories.old' dir
         bk_repos_path = File.join(path, '..', 'repositories.old.' + Time.now.to_i.to_s)
@@ -63,7 +63,7 @@ module Backup
 
         project.ensure_dir_exist
 
-        if File.exists?(path_to_bundle(project))
+        if File.exist?(path_to_bundle(project))
           FileUtils.mkdir_p(path_to_repo(project))
           cmd = %W(tar -xf #{path_to_bundle(project)} -C #{path_to_repo(project)})
         else
@@ -80,7 +80,7 @@ module Backup
 
         wiki = ProjectWiki.new(project)
 
-        if File.exists?(path_to_bundle(wiki))
+        if File.exist?(path_to_bundle(wiki))
           $progress.print " * #{wiki.path_with_namespace} ... "
 
           # If a wiki bundle exists, first remove the empty repo
diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb
index 9ed45707515..799b83b1069 100644
--- a/lib/banzai/filter/autolink_filter.rb
+++ b/lib/banzai/filter/autolink_filter.rb
@@ -31,6 +31,14 @@ module Banzai
       # Text matching LINK_PATTERN inside these elements will not be linked
       IGNORE_PARENTS = %w(a code kbd pre script style).to_set
 
+      # The XPath query to use for finding text nodes to parse.
+      TEXT_QUERY = %Q(descendant-or-self::text()[
+        not(#{IGNORE_PARENTS.map { |p| "ancestor::#{p}" }.join(' or ')})
+        and contains(., '://')
+        and not(starts-with(., 'http'))
+        and not(starts-with(., 'ftp'))
+      ])
+
       def call
         return doc if context[:autolink] == false
 
@@ -66,16 +74,11 @@ module Banzai
       # Autolinks any text matching LINK_PATTERN that Rinku didn't already
       # replace
       def text_parse
-        search_text_nodes(doc).each do |node|
+        doc.xpath(TEXT_QUERY).each do |node|
           content = node.to_html
 
-          next if has_ancestor?(node, IGNORE_PARENTS)
           next unless content.match(LINK_PATTERN)
 
-          # If Rinku didn't link this, there's probably a good reason, so we'll
-          # skip it too
-          next if content.start_with?(*%w(http https ftp))
-
           html = autolink_filter(content)
 
           next if html == content
diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb
index 337fb50317d..4fa8d05481f 100644
--- a/lib/banzai/filter/relative_link_filter.rb
+++ b/lib/banzai/filter/relative_link_filter.rb
@@ -35,6 +35,7 @@ module Banzai
 
       def process_link_attr(html_attr)
         return if html_attr.blank?
+        return if html_attr.value.start_with?('//')
 
         uri = URI(html_attr.value)
         if uri.relative? && uri.path.present?
@@ -51,7 +52,7 @@ module Banzai
           relative_url_root,
           context[:project].path_with_namespace,
           uri_type(file_path),
-          ref || context[:project].default_branch,  # if no ref exists, point to the default branch
+          ref,
           file_path
         ].compact.join('/').squeeze('/').chomp('/')
 
@@ -87,10 +88,13 @@ module Banzai
       def build_relative_path(path, request_path)
         return request_path if path.empty?
         return path unless request_path
+        return path[1..-1] if path.start_with?('/')
 
         parts = request_path.split('/')
         parts.pop if uri_type(request_path) != :tree
 
+        path.sub!(%r{\A\./}, '')
+
         while path.start_with?('../')
           parts.pop
           path.sub!('../', '')
@@ -112,7 +116,7 @@ module Banzai
       end
 
       def current_commit
-        @current_commit ||= context[:commit] || ref ? repository.commit(ref) : repository.head_commit
+        @current_commit ||= context[:commit] || repository.commit(ref)
       end
 
       def relative_url_root
@@ -120,7 +124,7 @@ module Banzai
       end
 
       def ref
-        context[:ref]
+        context[:ref] || context[:project].default_branch
       end
 
       def repository
diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb
index ca80aac5a08..6e13282d5f4 100644
--- a/lib/banzai/filter/sanitization_filter.rb
+++ b/lib/banzai/filter/sanitization_filter.rb
@@ -7,7 +7,7 @@ module Banzai
       UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze
 
       def whitelist
-        whitelist = super
+        whitelist = super.dup
 
         customize_whitelist(whitelist)
 
@@ -42,6 +42,8 @@ module Banzai
         # Allow any protocol in `a` elements...
         whitelist[:protocols].delete('a')
 
+        whitelist[:transformers] = whitelist[:transformers].dup
+
         # ...but then remove links with unsafe protocols
         whitelist[:transformers].push(remove_unsafe_links)
 
diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb
index 91f0159f9a1..fcdb496aed2 100644
--- a/lib/banzai/filter/syntax_highlight_filter.rb
+++ b/lib/banzai/filter/syntax_highlight_filter.rb
@@ -17,15 +17,12 @@ module Banzai
 
       def highlight_node(node)
         language = node.attr('class')
-        code     = node.text
-
+        code = node.text
         css_classes = "code highlight"
-
-        lexer = Rouge::Lexer.find_fancy(language) || Rouge::Lexers::PlainText
-        formatter = Rouge::Formatters::HTML.new
+        lexer = lexer_for(language)
 
         begin
-          code = formatter.format(lexer.lex(code))
+          code = format(lex(lexer, code))
 
           css_classes << " js-syntax-highlight #{lexer.tag}"
         rescue
@@ -41,14 +38,27 @@ module Banzai
 
       private
 
+      # Separate method so it can be instrumented.
+      def lex(lexer, code)
+        lexer.lex(code)
+      end
+
+      def format(tokens)
+        rouge_formatter.format(tokens)
+      end
+
+      def lexer_for(language)
+        (Rouge::Lexer.find(language) || Rouge::Lexers::PlainText).new
+      end
+
       def replace_parent_pre_element(node, highlighted)
         # Replace the parent `pre` element with the entire highlighted block
         node.parent.replace(highlighted)
       end
 
       # Override Rouge::Plugins::Redcarpet#rouge_formatter
-      def rouge_formatter(lexer)
-        Rouge::Formatters::HTML.new
+      def rouge_formatter(lexer = nil)
+        @rouge_formatter ||= Rouge::Formatters::HTML.new
       end
     end
   end
diff --git a/lib/banzai/filter/video_link_filter.rb b/lib/banzai/filter/video_link_filter.rb
index fd8b9a6f0cc..ac7bbcb0d10 100644
--- a/lib/banzai/filter/video_link_filter.rb
+++ b/lib/banzai/filter/video_link_filter.rb
@@ -1,11 +1,9 @@
 module Banzai
   module Filter
-
     # Find every image that isn't already wrapped in an `a` tag, and that has
     # a `src` attribute ending with a video extension, add a new video node and
     # a "Download" link in the case the video cannot be played.
     class VideoLinkFilter < HTML::Pipeline::Filter
-
       def call
         doc.xpath(query).each do |el|
           el.replace(video_node(doc, el))
@@ -54,6 +52,5 @@ module Banzai
         container
       end
     end
-
   end
 end
diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index 260ac81f5fa..9f3b582a263 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -20,8 +20,13 @@ module Ci
           build = Ci::RegisterBuildService.new.execute(current_runner)
 
           if build
+            Gitlab::Metrics.add_event(:build_found,
+                                      project: build.project.path_with_namespace)
+
             present build, with: Entities::BuildDetails
           else
+            Gitlab::Metrics.add_event(:build_not_found)
+
             not_found!
           end
         end
@@ -42,6 +47,9 @@ module Ci
 
           build.update_attributes(trace: params[:trace]) if params[:trace]
 
+          Gitlab::Metrics.add_event(:update_build,
+                                    project: build.project.path_with_namespace)
+
           case params[:state].to_s
           when 'success'
             build.success
diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb
index 1d7126a432d..3decc3b1a26 100644
--- a/lib/ci/charts.rb
+++ b/lib/ci/charts.rb
@@ -1,5 +1,37 @@
 module Ci
   module Charts
+    module DailyInterval
+      def grouped_count(query)
+        query.
+          group("DATE(#{Ci::Build.table_name}.created_at)").
+          count(:created_at).
+          transform_keys { |date| date.strftime(@format) }
+      end
+
+      def interval_step
+        @interval_step ||= 1.day
+      end
+    end
+
+    module MonthlyInterval
+      def grouped_count(query)
+        if Gitlab::Database.postgresql?
+          query.
+            group("to_char(#{Ci::Build.table_name}.created_at, '01 Month YYYY')").
+            count(:created_at).
+            transform_keys(&:squish)
+        else
+          query.
+            group("DATE_FORMAT(#{Ci::Build.table_name}.created_at, '01 %M %Y')").
+            count(:created_at)
+        end
+      end
+
+      def interval_step
+        @interval_step ||= 1.month
+      end
+    end
+
     class Chart
       attr_reader :labels, :total, :success, :project, :build_times
 
@@ -13,47 +45,59 @@ module Ci
         collect
       end
 
-      def push(from, to, format)
-        @labels << from.strftime(format)
-        @total << project.builds.
-          where("? > #{Ci::Build.table_name}.created_at AND #{Ci::Build.table_name}.created_at > ?", to, from).
-          count(:all)
-        @success << project.builds.
-          where("? > #{Ci::Build.table_name}.created_at AND #{Ci::Build.table_name}.created_at > ?", to, from).
-          success.count(:all)
+      def collect
+        query = project.builds.
+          where("? > #{Ci::Build.table_name}.created_at AND #{Ci::Build.table_name}.created_at > ?", @to, @from)
+
+        totals_count  = grouped_count(query)
+        success_count = grouped_count(query.success)
+
+        current = @from
+        while current < @to
+          label = current.strftime(@format)
+
+          @labels  << label
+          @total   << (totals_count[label] || 0)
+          @success << (success_count[label] || 0)
+
+          current += interval_step
+        end
       end
     end
 
     class YearChart < Chart
-      def collect
-        13.times do |i|
-          start_month = (Date.today.years_ago(1) + i.month).beginning_of_month
-          end_month = start_month.end_of_month
+      include MonthlyInterval
 
-          push(start_month, end_month, "%d %B %Y")
-        end
+      def initialize(*)
+        @to     = Date.today.end_of_month
+        @from   = @to.years_ago(1).beginning_of_month
+        @format = '%d %B %Y'
+
+        super
       end
     end
 
     class MonthChart < Chart
-      def collect
-        30.times do |i|
-          start_day = Date.today - 30.days + i.days
-          end_day = Date.today - 30.days + i.day + 1.day
+      include DailyInterval
 
-          push(start_day, end_day, "%d %B")
-        end
+      def initialize(*)
+        @to     = Date.today
+        @from   = @to - 30.days
+        @format = '%d %B'
+
+        super
       end
     end
 
     class WeekChart < Chart
-      def collect
-        7.times do |i|
-          start_day = Date.today - 7.days + i.days
-          end_day = Date.today - 7.days + i.day + 1.day
+      include DailyInterval
 
-          push(start_day, end_day, "%d %B")
-        end
+      def initialize(*)
+        @to     = Date.today
+        @from   = @to - 7.days
+        @format = '%d %B'
+
+        super
       end
     end
 
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index a2e8bd22a52..47efd5bd9f2 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -62,7 +62,7 @@ module Ci
         #  - before script should be a concatenated command
         commands: [job[:before_script] || @before_script, job[:script]].flatten.compact.join("\n"),
         tag_list: job[:tags] || [],
-        name: job[:name],
+        name: job[:name].to_s,
         allow_failure: job[:allow_failure] || false,
         when: job[:when] || 'on_success',
         environment: job[:environment],
diff --git a/lib/ci/static_model.rb b/lib/ci/static_model.rb
deleted file mode 100644
index bb2bdbed495..00000000000
--- a/lib/ci/static_model.rb
+++ /dev/null
@@ -1,49 +0,0 @@
-# Provides an ActiveRecord-like interface to a model whose data is not persisted to a database.
-module Ci
-  module StaticModel
-    extend ActiveSupport::Concern
-
-    module ClassMethods
-      # Used by ActiveRecord's polymorphic association to set object_id
-      def primary_key
-        'id'
-      end
-
-      # Used by ActiveRecord's polymorphic association to set object_type
-      def base_class
-        self
-      end
-    end
-
-    # Used by AR for fetching attributes
-    #
-    # Pass it along if we respond to it.
-    def [](key)
-      send(key) if respond_to?(key)
-    end
-
-    def to_param
-      id
-    end
-
-    def new_record?
-      false
-    end
-
-    def persisted?
-      false
-    end
-
-    def destroyed?
-      false
-    end
-
-    def ==(other)
-      if other.is_a? ::Ci::StaticModel
-        id == other.id
-      else
-        super
-      end
-    end
-  end
-end
diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb
index 51e46da82cc..84688f6646e 100644
--- a/lib/extracts_path.rb
+++ b/lib/extracts_path.rb
@@ -94,7 +94,7 @@ module ExtractsPath
     @options = params.select {|key, value| allowed_options.include?(key) && !value.blank? }
     @options = HashWithIndifferentAccess.new(@options)
 
-    @id = Addressable::URI.unescape(get_id)
+    @id = Addressable::URI.normalize_component(get_id)
     @ref, @path = extract_ref(@id)
     @repo = @project.repository
     if @options[:extended_sha1].blank?
diff --git a/lib/gitlab/akismet_helper.rb b/lib/gitlab/akismet_helper.rb
deleted file mode 100644
index 207736b59db..00000000000
--- a/lib/gitlab/akismet_helper.rb
+++ /dev/null
@@ -1,47 +0,0 @@
-module Gitlab
-  module AkismetHelper
-    def akismet_enabled?
-      current_application_settings.akismet_enabled
-    end
-
-    def akismet_client
-      @akismet_client ||= ::Akismet::Client.new(current_application_settings.akismet_api_key,
-        Gitlab.config.gitlab.url)
-    end
-
-    def client_ip(env)
-      env['action_dispatch.remote_ip'].to_s
-    end
-
-    def user_agent(env)
-      env['HTTP_USER_AGENT']
-    end
-
-    def check_for_spam?(project)
-      akismet_enabled? && project.public?
-    end
-
-    def is_spam?(environment, user, text)
-      client = akismet_client
-      ip_address = client_ip(environment)
-      user_agent = user_agent(environment)
-
-      params = {
-        type: 'comment',
-        text: text,
-        created_at: DateTime.now,
-        author: user.name,
-        author_email: user.email,
-        referrer: environment['HTTP_REFERER'],
-      }
-
-      begin
-        is_spam, is_blatant = client.check(ip_address, user_agent, params)
-        is_spam || is_blatant
-      rescue => e
-        Rails.logger.error("Unable to connect to Akismet: #{e}, skipping check")
-        false
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb
deleted file mode 100644
index ab94abeda77..00000000000
--- a/lib/gitlab/backend/grack_auth.rb
+++ /dev/null
@@ -1,163 +0,0 @@
-module Grack
-  class AuthSpawner
-    def self.call(env)
-      # Avoid issues with instance variables in Grack::Auth persisting across
-      # requests by creating a new instance for each request.
-      Auth.new({}).call(env)
-    end
-  end
-
-  class Auth < Rack::Auth::Basic
-    attr_accessor :user, :project, :env
-
-    def call(env)
-      @env = env
-      @request = Rack::Request.new(env)
-      @auth = Request.new(env)
-
-      @ci = false
-
-      # Need this patch due to the rails mount
-      # Need this if under RELATIVE_URL_ROOT
-      unless Gitlab.config.gitlab.relative_url_root.empty?
-        # If website is mounted using relative_url_root need to remove it first
-        @env['PATH_INFO'] = @request.path.sub(Gitlab.config.gitlab.relative_url_root, '')
-      else
-        @env['PATH_INFO'] = @request.path
-      end
-
-      @env['SCRIPT_NAME'] = ""
-
-      auth!
-
-      lfs_response = Gitlab::Lfs::Router.new(project, @user, @ci, @request).try_call
-      return lfs_response unless lfs_response.nil?
-
-      if @user.nil? && !@ci
-        unauthorized
-      else
-        render_not_found
-      end
-    end
-
-    private
-
-    def auth!
-      return unless @auth.provided?
-
-      return bad_request unless @auth.basic?
-
-      # Authentication with username and password
-      login, password = @auth.credentials
-
-      # Allow authentication for GitLab CI service
-      # if valid token passed
-      if ci_request?(login, password)
-        @ci = true
-        return
-      end
-
-      @user = authenticate_user(login, password)
-    end
-
-    def ci_request?(login, password)
-      matched_login = /(?<s>^[a-zA-Z]*-ci)-token$/.match(login)
-
-      if project && matched_login.present?
-        underscored_service = matched_login['s'].underscore
-
-        if underscored_service == 'gitlab_ci'
-          return project && project.valid_build_token?(password)
-        elsif Service.available_services_names.include?(underscored_service)
-          service_method = "#{underscored_service}_service"
-          service = project.send(service_method)
-
-          return service && service.activated? && service.valid_token?(password)
-        end
-      end
-
-      false
-    end
-
-    def oauth_access_token_check(login, password)
-      if login == "oauth2" && git_cmd == 'git-upload-pack' && password.present?
-        token = Doorkeeper::AccessToken.by_token(password)
-        token && token.accessible? && User.find_by(id: token.resource_owner_id)
-      end
-    end
-
-    def authenticate_user(login, password)
-      user = Gitlab::Auth.find_with_user_password(login, password)
-
-      unless user
-        user = oauth_access_token_check(login, password)
-      end
-
-      # If the user authenticated successfully, we reset the auth failure count
-      # from Rack::Attack for that IP. A client may attempt to authenticate
-      # with a username and blank password first, and only after it receives
-      # a 401 error does it present a password. Resetting the count prevents
-      # false positives from occurring.
-      #
-      # Otherwise, we let Rack::Attack know there was a failed authentication
-      # attempt from this IP. This information is stored in the Rails cache
-      # (Redis) and will be used by the Rack::Attack middleware to decide
-      # whether to block requests from this IP.
-      config = Gitlab.config.rack_attack.git_basic_auth
-
-      if config.enabled
-        if user
-          # A successful login will reset the auth failure count from this IP
-          Rack::Attack::Allow2Ban.reset(@request.ip, config)
-        else
-          banned = Rack::Attack::Allow2Ban.filter(@request.ip, config) do
-            # Unless the IP is whitelisted, return true so that Allow2Ban
-            # increments the counter (stored in Rails.cache) for the IP
-            if config.ip_whitelist.include?(@request.ip)
-              false
-            else
-              true
-            end
-          end
-
-          if banned
-            Rails.logger.info "IP #{@request.ip} failed to login " \
-              "as #{login} but has been temporarily banned from Git auth"
-          end
-        end
-      end
-
-      user
-    end
-
-    def git_cmd
-      if @request.get?
-        @request.params['service']
-      elsif @request.post?
-        File.basename(@request.path)
-      else
-        nil
-      end
-    end
-
-    def project
-      return @project if defined?(@project)
-
-      @project = project_by_path(@request.path_info)
-    end
-
-    def project_by_path(path)
-      if m = /^([\w\.\/-]+)\.git/.match(path).to_a
-        path_with_namespace = m.last
-        path_with_namespace.gsub!(/\.wiki$/, '')
-
-        path_with_namespace[0] = '' if path_with_namespace.start_with?('/')
-        Project.find_with_namespace(path_with_namespace)
-      end
-    end
-
-    def render_not_found
-      [404, { "Content-Type" => "text/plain" }, ["Not Found"]]
-    end
-  end
-end
diff --git a/lib/gitlab/badge/base.rb b/lib/gitlab/badge/base.rb
new file mode 100644
index 00000000000..909fa24fa90
--- /dev/null
+++ b/lib/gitlab/badge/base.rb
@@ -0,0 +1,21 @@
+module Gitlab
+  module Badge
+    class Base
+      def entity
+        raise NotImplementedError
+      end
+
+      def status
+        raise NotImplementedError
+      end
+
+      def metadata
+        raise NotImplementedError
+      end
+
+      def template
+        raise NotImplementedError
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/badge/build.rb b/lib/gitlab/badge/build.rb
deleted file mode 100644
index e5e9fab3f5c..00000000000
--- a/lib/gitlab/badge/build.rb
+++ /dev/null
@@ -1,46 +0,0 @@
-module Gitlab
-  module Badge
-    ##
-    # Build badge
-    #
-    class Build
-      include Gitlab::Application.routes.url_helpers
-      include ActionView::Helpers::AssetTagHelper
-      include ActionView::Helpers::UrlHelper
-
-      def initialize(project, ref)
-        @project, @ref = project, ref
-        @image = ::Ci::ImageForBuildService.new.execute(project, ref: ref)
-      end
-
-      def type
-        'image/svg+xml'
-      end
-
-      def data
-        File.read(@image[:path])
-      end
-
-      def to_s
-        @image[:name].sub(/\.svg$/, '')
-      end
-
-      def to_html
-        link_to(image_tag(image_url, alt: 'build status'), link_url)
-      end
-
-      def to_markdown
-        "[![build status](#{image_url})](#{link_url})"
-      end
-
-      def image_url
-        build_namespace_project_badges_url(@project.namespace,
-                                           @project, @ref, format: :svg)
-      end
-
-      def link_url
-        namespace_project_commits_url(@project.namespace, @project, id: @ref)
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/badge/build/metadata.rb b/lib/gitlab/badge/build/metadata.rb
new file mode 100644
index 00000000000..f87a7b7942e
--- /dev/null
+++ b/lib/gitlab/badge/build/metadata.rb
@@ -0,0 +1,28 @@
+module Gitlab
+  module Badge
+    module Build
+      ##
+      # Class that describes build badge metadata
+      #
+      class Metadata < Badge::Metadata
+        def initialize(badge)
+          @project = badge.project
+          @ref = badge.ref
+        end
+
+        def title
+          'build status'
+        end
+
+        def image_url
+          build_namespace_project_badges_url(@project.namespace,
+                                             @project, @ref, format: :svg)
+        end
+
+        def link_url
+          namespace_project_commits_url(@project.namespace, @project, id: @ref)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/badge/build/status.rb b/lib/gitlab/badge/build/status.rb
new file mode 100644
index 00000000000..50aa45e5406
--- /dev/null
+++ b/lib/gitlab/badge/build/status.rb
@@ -0,0 +1,37 @@
+module Gitlab
+  module Badge
+    module Build
+      ##
+      # Build status badge
+      #
+      class Status < Badge::Base
+        attr_reader :project, :ref
+
+        def initialize(project, ref)
+          @project = project
+          @ref = ref
+
+          @sha = @project.commit(@ref).try(:sha)
+        end
+
+        def entity
+          'build'
+        end
+
+        def status
+          @project.pipelines
+            .where(sha: @sha, ref: @ref)
+            .status || 'unknown'
+        end
+
+        def metadata
+          @metadata ||= Build::Metadata.new(self)
+        end
+
+        def template
+          @template ||= Build::Template.new(self)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/badge/build/template.rb b/lib/gitlab/badge/build/template.rb
new file mode 100644
index 00000000000..2b95ddfcb53
--- /dev/null
+++ b/lib/gitlab/badge/build/template.rb
@@ -0,0 +1,47 @@
+module Gitlab
+  module Badge
+    module Build
+      ##
+      # Class that represents a build badge template.
+      #
+      # Template object will be passed to badge.svg.erb template.
+      #
+      class Template < Badge::Template
+        STATUS_COLOR = {
+          success: '#4c1',
+          failed: '#e05d44',
+          running: '#dfb317',
+          pending: '#dfb317',
+          canceled: '#9f9f9f',
+          skipped: '#9f9f9f',
+          unknown: '#9f9f9f'
+        }
+
+        def initialize(badge)
+          @entity = badge.entity
+          @status = badge.status
+        end
+
+        def key_text
+          @entity.to_s
+        end
+
+        def value_text
+          @status.to_s
+        end
+
+        def key_width
+          38
+        end
+
+        def value_width
+          54
+        end
+
+        def value_color
+          STATUS_COLOR[@status.to_sym] || STATUS_COLOR[:unknown]
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/badge/coverage/metadata.rb b/lib/gitlab/badge/coverage/metadata.rb
new file mode 100644
index 00000000000..53588185622
--- /dev/null
+++ b/lib/gitlab/badge/coverage/metadata.rb
@@ -0,0 +1,30 @@
+module Gitlab
+  module Badge
+    module Coverage
+      ##
+      # Class that describes coverage badge metadata
+      #
+      class Metadata < Badge::Metadata
+        def initialize(badge)
+          @project = badge.project
+          @ref = badge.ref
+          @job = badge.job
+        end
+
+        def title
+          'coverage report'
+        end
+
+        def image_url
+          coverage_namespace_project_badges_url(@project.namespace,
+                                                @project, @ref,
+                                                format: :svg)
+        end
+
+        def link_url
+          namespace_project_commits_url(@project.namespace, @project, id: @ref)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb
new file mode 100644
index 00000000000..3d56ea3e47a
--- /dev/null
+++ b/lib/gitlab/badge/coverage/report.rb
@@ -0,0 +1,56 @@
+module Gitlab
+  module Badge
+    module Coverage
+      ##
+      # Test coverage report badge
+      #
+      class Report < Badge::Base
+        attr_reader :project, :ref, :job
+
+        def initialize(project, ref, job = nil)
+          @project = project
+          @ref = ref
+          @job = job
+
+          @pipeline = @project.pipelines
+            .where(ref: @ref)
+            .where(sha: @project.commit(@ref).try(:sha))
+            .first
+        end
+
+        def entity
+          'coverage'
+        end
+
+        def status
+          @coverage ||= raw_coverage
+          return unless @coverage
+
+          @coverage.to_i
+        end
+
+        def metadata
+          @metadata ||= Coverage::Metadata.new(self)
+        end
+
+        def template
+          @template ||= Coverage::Template.new(self)
+        end
+
+        private
+
+        def raw_coverage
+          return unless @pipeline
+
+          if @job.blank?
+            @pipeline.coverage
+          else
+            @pipeline.builds
+              .find_by(name: @job)
+              .try(:coverage)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/badge/coverage/template.rb b/lib/gitlab/badge/coverage/template.rb
new file mode 100644
index 00000000000..06e0d084e9f
--- /dev/null
+++ b/lib/gitlab/badge/coverage/template.rb
@@ -0,0 +1,52 @@
+module Gitlab
+  module Badge
+    module Coverage
+      ##
+      # Class that represents a coverage badge template.
+      #
+      # Template object will be passed to badge.svg.erb template.
+      #
+      class Template < Badge::Template
+        STATUS_COLOR = {
+          good: '#4c1',
+          acceptable: '#a3c51c',
+          medium: '#dfb317',
+          low: '#e05d44',
+          unknown: '#9f9f9f'
+        }
+
+        def initialize(badge)
+          @entity = badge.entity
+          @status = badge.status
+        end
+
+        def key_text
+          @entity.to_s
+        end
+
+        def value_text
+          @status ? "#{@status}%" : 'unknown'
+        end
+
+        def key_width
+          62
+        end
+
+        def value_width
+          @status ? 36 : 58
+        end
+
+        def value_color
+          case @status
+          when 95..100 then STATUS_COLOR[:good]
+          when 90..95 then STATUS_COLOR[:acceptable]
+          when 75..90 then STATUS_COLOR[:medium]
+          when 0..75 then STATUS_COLOR[:low]
+          else
+            STATUS_COLOR[:unknown]
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/badge/metadata.rb b/lib/gitlab/badge/metadata.rb
new file mode 100644
index 00000000000..548f85b78bb
--- /dev/null
+++ b/lib/gitlab/badge/metadata.rb
@@ -0,0 +1,36 @@
+module Gitlab
+  module Badge
+    ##
+    # Abstract class for badge metadata
+    #
+    class Metadata
+      include Gitlab::Application.routes.url_helpers
+      include ActionView::Helpers::AssetTagHelper
+      include ActionView::Helpers::UrlHelper
+
+      def initialize(badge)
+        @badge = badge
+      end
+
+      def to_html
+        link_to(image_tag(image_url, alt: title), link_url)
+      end
+
+      def to_markdown
+        "[![#{title}](#{image_url})](#{link_url})"
+      end
+
+      def title
+        raise NotImplementedError
+      end
+
+      def image_url
+        raise NotImplementedError
+      end
+
+      def link_url
+        raise NotImplementedError
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/badge/template.rb b/lib/gitlab/badge/template.rb
new file mode 100644
index 00000000000..bfeb0052642
--- /dev/null
+++ b/lib/gitlab/badge/template.rb
@@ -0,0 +1,49 @@
+module Gitlab
+  module Badge
+    ##
+    # Abstract template class for badges
+    #
+    class Template
+      def initialize(badge)
+        @entity = badge.entity
+        @status = badge.status
+      end
+
+      def key_text
+        raise NotImplementedError
+      end
+
+      def value_text
+        raise NotImplementedError
+      end
+
+      def key_width
+        raise NotImplementedError
+      end
+
+      def value_width
+        raise NotImplementedError
+      end
+
+      def value_color
+        raise NotImplementedError
+      end
+
+      def key_color
+        '#555'
+      end
+
+      def key_text_anchor
+        key_width / 2
+      end
+
+      def value_text_anchor
+        key_width + (value_width / 2)
+      end
+
+      def width
+        key_width + value_width
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/changes_list.rb b/lib/gitlab/changes_list.rb
new file mode 100644
index 00000000000..95308aca95f
--- /dev/null
+++ b/lib/gitlab/changes_list.rb
@@ -0,0 +1,25 @@
+module Gitlab
+  class ChangesList
+    include Enumerable
+
+    attr_reader :raw_changes
+
+    def initialize(changes)
+      @raw_changes = changes.kind_of?(String) ? changes.lines : changes
+    end
+
+    def each(&block)
+      changes.each(&block)
+    end
+
+    def changes
+      @changes ||= begin
+        @raw_changes.map do |change|
+          next if change.blank?
+          oldrev, newrev, ref = change.strip.split(' ')
+          { oldrev: oldrev, newrev: newrev, ref: ref }
+        end.compact
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb
index 5551fac4b8b..4b32eb966aa 100644
--- a/lib/gitlab/checks/change_access.rb
+++ b/lib/gitlab/checks/change_access.rb
@@ -4,14 +4,14 @@ module Gitlab
       attr_reader :user_access, :project
 
       def initialize(change, user_access:, project:)
-        @oldrev, @newrev, @ref = change.split(' ')
-        @branch_name = branch_name(@ref)
+        @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref)
+        @branch_name = Gitlab::Git.branch_name(@ref)
         @user_access = user_access
         @project = project
       end
 
       def exec
-        error = protected_branch_checks || tag_checks || push_checks
+        error = push_checks || tag_checks || protected_branch_checks
 
         if error
           GitAccessStatus.new(false, error)
@@ -47,7 +47,7 @@ module Gitlab
       end
 
       def tag_checks
-        tag_ref = tag_name(@ref)
+        tag_ref = Gitlab::Git.tag_name(@ref)
 
         if tag_ref && protected_tag?(tag_ref) && user_access.cannot_do_action?(:admin_project)
           "You are not allowed to change existing tags on this project."
@@ -73,24 +73,6 @@ module Gitlab
       def matching_merge_request?
         Checks::MatchingMergeRequest.new(@newrev, @branch_name, @project).match?
       end
-
-      def branch_name(ref)
-        ref = @ref.to_s
-        if Gitlab::Git.branch_ref?(ref)
-          Gitlab::Git.ref_name(ref)
-        else
-          nil
-        end
-      end
-
-      def tag_name(ref)
-        ref = @ref.to_s
-        if Gitlab::Git.tag_ref?(ref)
-          Gitlab::Git.ref_name(ref)
-        else
-          nil
-        end
-      end
     end
   end
 end
diff --git a/lib/gitlab/ci/config/node/validatable.rb b/lib/gitlab/ci/config/node/validatable.rb
index f6e2896dfb2..085e6e988d1 100644
--- a/lib/gitlab/ci/config/node/validatable.rb
+++ b/lib/gitlab/ci/config/node/validatable.rb
@@ -7,13 +7,11 @@ module Gitlab
 
           class_methods do
             def validator
-              validator = Class.new(Node::Validator)
-
-              if defined?(@validations)
-                @validations.each { |rules| validator.class_eval(&rules) }
+              @validator ||= Class.new(Node::Validator).tap do |validator|
+                if defined?(@validations)
+                  @validations.each { |rules| validator.class_eval(&rules) }
+                end
               end
-
-              validator
             end
 
             private
diff --git a/lib/gitlab/closing_issue_extractor.rb b/lib/gitlab/closing_issue_extractor.rb
index 9bef9037ad6..58f86abc5c4 100644
--- a/lib/gitlab/closing_issue_extractor.rb
+++ b/lib/gitlab/closing_issue_extractor.rb
@@ -22,7 +22,9 @@ module Gitlab
 
       @extractor.analyze(closing_statements.join(" "))
 
-      @extractor.issues
+      @extractor.issues.reject do |issue|
+        @extractor.project.forked_from?(issue.project) # Don't extract issues on original project
+      end
     end
   end
 end
diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb
new file mode 100644
index 00000000000..0a1fd27ced5
--- /dev/null
+++ b/lib/gitlab/conflict/file.rb
@@ -0,0 +1,186 @@
+module Gitlab
+  module Conflict
+    class File
+      include Gitlab::Routing.url_helpers
+      include IconsHelper
+
+      class MissingResolution < StandardError
+      end
+
+      CONTEXT_LINES = 3
+
+      attr_reader :merge_file_result, :their_path, :our_path, :our_mode, :merge_request, :repository
+
+      def initialize(merge_file_result, conflict, merge_request:)
+        @merge_file_result = merge_file_result
+        @their_path = conflict[:theirs][:path]
+        @our_path = conflict[:ours][:path]
+        @our_mode = conflict[:ours][:mode]
+        @merge_request = merge_request
+        @repository = merge_request.project.repository
+        @match_line_headers = {}
+      end
+
+      # Array of Gitlab::Diff::Line objects
+      def lines
+        @lines ||= Gitlab::Conflict::Parser.new.parse(merge_file_result[:data],
+                                                      our_path: our_path,
+                                                      their_path: their_path,
+                                                      parent_file: self)
+      end
+
+      def resolve_lines(resolution)
+        section_id = nil
+
+        lines.map do |line|
+          unless line.type
+            section_id = nil
+            next line
+          end
+
+          section_id ||= line_code(line)
+
+          case resolution[section_id]
+          when 'head'
+            next unless line.type == 'new'
+          when 'origin'
+            next unless line.type == 'old'
+          else
+            raise MissingResolution, "Missing resolution for section ID: #{section_id}"
+          end
+
+          line
+        end.compact
+      end
+
+      def highlight_lines!
+        their_file = lines.reject { |line| line.type == 'new' }.map(&:text).join("\n")
+        our_file = lines.reject { |line| line.type == 'old' }.map(&:text).join("\n")
+
+        their_highlight = Gitlab::Highlight.highlight(their_path, their_file, repository: repository).lines
+        our_highlight = Gitlab::Highlight.highlight(our_path, our_file, repository: repository).lines
+
+        lines.each do |line|
+          if line.type == 'old'
+            line.rich_text = their_highlight[line.old_line - 1].try(:html_safe)
+          else
+            line.rich_text = our_highlight[line.new_line - 1].try(:html_safe)
+          end
+        end
+      end
+
+      def sections
+        return @sections if @sections
+
+        chunked_lines = lines.chunk { |line| line.type.nil? }.to_a
+        match_line = nil
+
+        sections_count = chunked_lines.size
+
+        @sections = chunked_lines.flat_map.with_index do |(no_conflict, lines), i|
+          section = nil
+
+          # We need to reduce context sections to CONTEXT_LINES. Conflict sections are
+          # always shown in full.
+          if no_conflict
+            conflict_before = i > 0
+            conflict_after = (sections_count - i) > 1
+
+            if conflict_before && conflict_after
+              # Create a gap in a long context section.
+              if lines.length > CONTEXT_LINES * 2
+                head_lines = lines.first(CONTEXT_LINES)
+                tail_lines = lines.last(CONTEXT_LINES)
+
+                # Ensure any existing match line has text for all lines up to the last
+                # line of its context.
+                update_match_line_text(match_line, head_lines.last)
+
+                # Insert a new match line after the created gap.
+                match_line = create_match_line(tail_lines.first)
+
+                section = [
+                  { conflict: false, lines: head_lines },
+                  { conflict: false, lines: tail_lines.unshift(match_line) }
+                ]
+              end
+            elsif conflict_after
+              tail_lines = lines.last(CONTEXT_LINES)
+
+              # Create a gap and insert a match line at the start.
+              if lines.length > tail_lines.length
+                match_line = create_match_line(tail_lines.first)
+
+                tail_lines.unshift(match_line)
+              end
+
+              lines = tail_lines
+            elsif conflict_before
+              # We're at the end of the file (no conflicts after), so just remove extra
+              # trailing lines.
+              lines = lines.first(CONTEXT_LINES)
+            end
+          end
+
+          # We want to update the match line's text every time unless we've already
+          # created a gap and its corresponding match line.
+          update_match_line_text(match_line, lines.last) unless section
+
+          section ||= { conflict: !no_conflict, lines: lines }
+          section[:id] = line_code(lines.first) unless no_conflict
+          section
+        end
+      end
+
+      def line_code(line)
+        Gitlab::Diff::LineCode.generate(our_path, line.new_pos, line.old_pos)
+      end
+
+      def create_match_line(line)
+        Gitlab::Diff::Line.new('', 'match', line.index, line.old_pos, line.new_pos)
+      end
+
+      # Any line beginning with a letter, an underscore, or a dollar can be used in a
+      # match line header. Only context sections can contain match lines, as match lines
+      # have to exist in both versions of the file.
+      def find_match_line_header(index)
+        return @match_line_headers[index] if @match_line_headers.key?(index)
+
+        @match_line_headers[index] = begin
+          if index >= 0
+            line = lines[index]
+
+            if line.type.nil? && line.text.match(/\A[A-Za-z$_]/)
+              " #{line.text}"
+            else
+              find_match_line_header(index - 1)
+            end
+          end
+        end
+      end
+
+      # Set the match line's text for the current line. A match line takes its start
+      # position and context header (where present) from itself, and its end position from
+      # the line passed in.
+      def update_match_line_text(match_line, line)
+        return unless match_line
+
+        header = find_match_line_header(match_line.index - 1)
+
+        match_line.text = "@@ -#{match_line.old_pos},#{line.old_pos} +#{match_line.new_pos},#{line.new_pos} @@#{header}"
+      end
+
+      def as_json(opts = nil)
+        {
+          old_path: their_path,
+          new_path: our_path,
+          blob_icon: file_type_icon_class('file', our_mode, our_path),
+          blob_path: namespace_project_blob_path(merge_request.project.namespace,
+                                                 merge_request.project,
+                                                 ::File.join(merge_request.diff_refs.head_sha, our_path)),
+          sections: sections
+        }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb
new file mode 100644
index 00000000000..bbd0427a2c8
--- /dev/null
+++ b/lib/gitlab/conflict/file_collection.rb
@@ -0,0 +1,57 @@
+module Gitlab
+  module Conflict
+    class FileCollection
+      class ConflictSideMissing < StandardError
+      end
+
+      attr_reader :merge_request, :our_commit, :their_commit
+
+      def initialize(merge_request)
+        @merge_request = merge_request
+        @our_commit = merge_request.source_branch_head.raw.raw_commit
+        @their_commit = merge_request.target_branch_head.raw.raw_commit
+      end
+
+      def repository
+        merge_request.project.repository
+      end
+
+      def merge_index
+        @merge_index ||= repository.rugged.merge_commits(our_commit, their_commit)
+      end
+
+      def files
+        @files ||= merge_index.conflicts.map do |conflict|
+          raise ConflictSideMissing unless conflict[:theirs] && conflict[:ours]
+
+          Gitlab::Conflict::File.new(merge_index.merge_file(conflict[:ours][:path]),
+                                     conflict,
+                                     merge_request: merge_request)
+        end
+      end
+
+      def as_json(opts = nil)
+        {
+          target_branch: merge_request.target_branch,
+          source_branch: merge_request.source_branch,
+          commit_sha: merge_request.diff_head_sha,
+          commit_message: default_commit_message,
+          files: files
+        }
+      end
+
+      def default_commit_message
+        conflict_filenames = merge_index.conflicts.map do |conflict|
+          "#   #{conflict[:ours][:path]}"
+        end
+
+        <<EOM.chomp
+Merge branch '#{merge_request.target_branch}' into '#{merge_request.source_branch}'
+
+# Conflicts:
+#{conflict_filenames.join("\n")}
+EOM
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/conflict/parser.rb b/lib/gitlab/conflict/parser.rb
new file mode 100644
index 00000000000..6eccded7872
--- /dev/null
+++ b/lib/gitlab/conflict/parser.rb
@@ -0,0 +1,62 @@
+module Gitlab
+  module Conflict
+    class Parser
+      class ParserError < StandardError
+      end
+
+      class UnexpectedDelimiter < ParserError
+      end
+
+      class MissingEndDelimiter < ParserError
+      end
+
+      class UnmergeableFile < ParserError
+      end
+
+      def parse(text, our_path:, their_path:, parent_file: nil)
+        raise UnmergeableFile if text.blank? # Typically a binary file
+        raise UnmergeableFile if text.length > 102400
+
+        line_obj_index = 0
+        line_old = 1
+        line_new = 1
+        type = nil
+        lines = []
+        conflict_start = "<<<<<<< #{our_path}"
+        conflict_middle = '======='
+        conflict_end = ">>>>>>> #{their_path}"
+
+        text.each_line.map do |line|
+          full_line = line.delete("\n")
+
+          if full_line == conflict_start
+            raise UnexpectedDelimiter unless type.nil?
+
+            type = 'new'
+          elsif full_line == conflict_middle
+            raise UnexpectedDelimiter unless type == 'new'
+
+            type = 'old'
+          elsif full_line == conflict_end
+            raise UnexpectedDelimiter unless type == 'old'
+
+            type = nil
+          elsif line[0] == '\\'
+            type = 'nonewline'
+            lines << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: parent_file)
+          else
+            lines << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: parent_file)
+            line_old += 1 if type != 'new'
+            line_new += 1 if type != 'old'
+
+            line_obj_index += 1
+          end
+        end
+
+        raise MissingEndDelimiter unless type.nil?
+
+        lines
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/build_data_builder.rb b/lib/gitlab/data_builder/build.rb
index 9f45aefda0f..6548e6475c6 100644
--- a/lib/gitlab/build_data_builder.rb
+++ b/lib/gitlab/data_builder/build.rb
@@ -1,6 +1,8 @@
 module Gitlab
-  class BuildDataBuilder
-    class << self
+  module DataBuilder
+    module Build
+      extend self
+
       def build(build)
         project = build.project
         commit = build.pipeline
diff --git a/lib/gitlab/note_data_builder.rb b/lib/gitlab/data_builder/note.rb
index 8bdc89a7751..50fea1232af 100644
--- a/lib/gitlab/note_data_builder.rb
+++ b/lib/gitlab/data_builder/note.rb
@@ -1,6 +1,8 @@
 module Gitlab
-  class NoteDataBuilder
-    class << self
+  module DataBuilder
+    module Note
+      extend self
+
       # Produce a hash of post-receive data
       #
       # For all notes:
diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb
new file mode 100644
index 00000000000..06a783ebc1c
--- /dev/null
+++ b/lib/gitlab/data_builder/pipeline.rb
@@ -0,0 +1,62 @@
+module Gitlab
+  module DataBuilder
+    module Pipeline
+      extend self
+
+      def build(pipeline)
+        {
+          object_kind: 'pipeline',
+          object_attributes: hook_attrs(pipeline),
+          user: pipeline.user.try(:hook_attrs),
+          project: pipeline.project.hook_attrs(backward: false),
+          commit: pipeline.commit.try(:hook_attrs),
+          builds: pipeline.builds.map(&method(:build_hook_attrs))
+        }
+      end
+
+      def hook_attrs(pipeline)
+        {
+          id: pipeline.id,
+          ref: pipeline.ref,
+          tag: pipeline.tag,
+          sha: pipeline.sha,
+          before_sha: pipeline.before_sha,
+          status: pipeline.status,
+          stages: pipeline.stages,
+          created_at: pipeline.created_at,
+          finished_at: pipeline.finished_at,
+          duration: pipeline.duration
+        }
+      end
+
+      def build_hook_attrs(build)
+        {
+          id: build.id,
+          stage: build.stage,
+          name: build.name,
+          status: build.status,
+          created_at: build.created_at,
+          started_at: build.started_at,
+          finished_at: build.finished_at,
+          when: build.when,
+          manual: build.manual?,
+          user: build.user.try(:hook_attrs),
+          runner: build.runner && runner_hook_attrs(build.runner),
+          artifacts_file: {
+            filename: build.artifacts_file.filename,
+            size: build.artifacts_size
+          }
+        }
+      end
+
+      def runner_hook_attrs(runner)
+        {
+          id: runner.id,
+          description: runner.description,
+          active: runner.active?,
+          is_shared: runner.is_shared?
+        }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/push_data_builder.rb b/lib/gitlab/data_builder/push.rb
index c8f12577112..4f81863da35 100644
--- a/lib/gitlab/push_data_builder.rb
+++ b/lib/gitlab/data_builder/push.rb
@@ -1,6 +1,8 @@
 module Gitlab
-  class PushDataBuilder
-    class << self
+  module DataBuilder
+    module Push
+      extend self
+
       # Produce a hash of post-receive data
       #
       # data = {
diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb
index b09ca1fb8b0..e47df508ca2 100644
--- a/lib/gitlab/diff/file.rb
+++ b/lib/gitlab/diff/file.rb
@@ -63,15 +63,18 @@ module Gitlab
         diff_refs.try(:head_sha)
       end
 
+      attr_writer :highlighted_diff_lines
+
       # Array of Gitlab::Diff::Line objects
       def diff_lines
-        @lines ||= Gitlab::Diff::Parser.new.parse(raw_diff.each_line).to_a
+        @diff_lines ||= Gitlab::Diff::Parser.new.parse(raw_diff.each_line).to_a
       end
 
       def highlighted_diff_lines
         @highlighted_diff_lines ||= Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight
       end
 
+      # Array[<Hash>] with right/left keys that contains Gitlab::Diff::Line objects which text is hightlighted
       def parallel_diff_lines
         @parallel_diff_lines ||= Gitlab::Diff::ParallelDiff.new(self).parallelize
       end
diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb
new file mode 100644
index 00000000000..2b9fc65b985
--- /dev/null
+++ b/lib/gitlab/diff/file_collection/base.rb
@@ -0,0 +1,35 @@
+module Gitlab
+  module Diff
+    module FileCollection
+      class Base
+        attr_reader :project, :diff_options, :diff_view, :diff_refs
+
+        delegate :count, :size, :real_size, to: :diff_files
+
+        def self.default_options
+          ::Commit.max_diff_options.merge(ignore_whitespace_change: false, no_collapse: false)
+        end
+
+        def initialize(diffable, project:, diff_options: nil, diff_refs: nil)
+          diff_options = self.class.default_options.merge(diff_options || {})
+
+          @diffable     = diffable
+          @diffs        = diffable.raw_diffs(diff_options)
+          @project      = project
+          @diff_options = diff_options
+          @diff_refs    = diff_refs
+        end
+
+        def diff_files
+          @diff_files ||= @diffs.decorate! { |diff| decorate_diff!(diff) }
+        end
+
+        private
+
+        def decorate_diff!(diff)
+          Gitlab::Diff::File.new(diff, repository: project.repository, diff_refs: diff_refs)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/file_collection/commit.rb b/lib/gitlab/diff/file_collection/commit.rb
new file mode 100644
index 00000000000..4dc297ec036
--- /dev/null
+++ b/lib/gitlab/diff/file_collection/commit.rb
@@ -0,0 +1,14 @@
+module Gitlab
+  module Diff
+    module FileCollection
+      class Commit < Base
+        def initialize(commit, diff_options:)
+          super(commit,
+            project: commit.project,
+            diff_options: diff_options,
+            diff_refs: commit.diff_refs)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/file_collection/compare.rb b/lib/gitlab/diff/file_collection/compare.rb
new file mode 100644
index 00000000000..20d8f891cc3
--- /dev/null
+++ b/lib/gitlab/diff/file_collection/compare.rb
@@ -0,0 +1,14 @@
+module Gitlab
+  module Diff
+    module FileCollection
+      class Compare < Base
+        def initialize(compare, project:, diff_options:, diff_refs: nil)
+          super(compare,
+            project:      project,
+            diff_options: diff_options,
+            diff_refs:    diff_refs)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/file_collection/merge_request.rb b/lib/gitlab/diff/file_collection/merge_request.rb
new file mode 100644
index 00000000000..4f946908e2f
--- /dev/null
+++ b/lib/gitlab/diff/file_collection/merge_request.rb
@@ -0,0 +1,73 @@
+module Gitlab
+  module Diff
+    module FileCollection
+      class MergeRequest < Base
+        def initialize(merge_request, diff_options:)
+          @merge_request = merge_request
+
+          super(merge_request,
+            project: merge_request.project,
+            diff_options: diff_options,
+            diff_refs: merge_request.diff_refs)
+        end
+
+        def diff_files
+          super.tap { |_| store_highlight_cache }
+        end
+
+        private
+
+        # Extracted method to highlight in the same iteration to the diff_collection.
+        def decorate_diff!(diff)
+          diff_file = super
+          cache_highlight!(diff_file) if cacheable?
+          diff_file
+        end
+
+        def highlight_diff_file_from_cache!(diff_file, cache_diff_lines)
+          diff_file.highlighted_diff_lines = cache_diff_lines.map do |line|
+            Gitlab::Diff::Line.init_from_hash(line)
+          end
+        end
+
+        #
+        # If we find the highlighted diff files lines on the cache we replace existing diff_files lines (no highlighted)
+        # for the highlighted ones, so we just skip their execution.
+        # If the highlighted diff files lines are not cached we calculate and cache them.
+        #
+        # The content of the cache is a Hash where the key correspond to the file_path and the values are Arrays of
+        # hashes that represent serialized diff lines.
+        #
+        def cache_highlight!(diff_file)
+          file_path = diff_file.file_path
+
+          if highlight_cache[file_path]
+            highlight_diff_file_from_cache!(diff_file, highlight_cache[file_path])
+          else
+            highlight_cache[file_path] = diff_file.highlighted_diff_lines.map(&:to_hash)
+          end
+        end
+
+        def highlight_cache
+          return @highlight_cache if defined?(@highlight_cache)
+
+          @highlight_cache = Rails.cache.read(cache_key) || {}
+          @highlight_cache_was_empty = @highlight_cache.empty?
+          @highlight_cache
+        end
+
+        def store_highlight_cache
+          Rails.cache.write(cache_key, highlight_cache) if @highlight_cache_was_empty
+        end
+
+        def cacheable?
+          @merge_request.merge_request_diff.present?
+        end
+
+        def cache_key
+          [@merge_request.merge_request_diff, 'highlighted-diff-files', diff_options]
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb
index 649a265a02c..9ea976e18fa 100644
--- a/lib/gitlab/diff/highlight.rb
+++ b/lib/gitlab/diff/highlight.rb
@@ -40,8 +40,6 @@ module Gitlab
       def highlight_line(diff_line)
         return unless diff_file && diff_file.diff_refs
 
-        line_prefix = diff_line.text.match(/\A(.)/) ? $1 : ' '
-
         rich_line =
           if diff_line.unchanged? || diff_line.added?
             new_lines[diff_line.new_pos - 1]
@@ -51,7 +49,10 @@ module Gitlab
 
         # Only update text if line is found. This will prevent
         # issues with submodules given the line only exists in diff content.
-        "#{line_prefix}#{rich_line}".html_safe if rich_line
+        if rich_line
+          line_prefix = diff_line.text.match(/\A(.)/) ? $1 : ' '
+          "#{line_prefix}#{rich_line}".html_safe
+        end
       end
 
       def inline_diffs
diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb
index c6189d660c2..80a146b4a5a 100644
--- a/lib/gitlab/diff/line.rb
+++ b/lib/gitlab/diff/line.rb
@@ -2,11 +2,27 @@ module Gitlab
   module Diff
     class Line
       attr_reader :type, :index, :old_pos, :new_pos
+      attr_writer :rich_text
       attr_accessor :text
 
-      def initialize(text, type, index, old_pos, new_pos)
+      def initialize(text, type, index, old_pos, new_pos, parent_file: nil)
         @text, @type, @index = text, type, index
         @old_pos, @new_pos = old_pos, new_pos
+        @parent_file = parent_file
+      end
+
+      def self.init_from_hash(hash)
+        new(hash[:text], hash[:type], hash[:index], hash[:old_pos], hash[:new_pos])
+      end
+
+      def serialize_keys
+        @serialize_keys ||= %i(text type index old_pos new_pos)
+      end
+
+      def to_hash
+        hash = {}
+        serialize_keys.each { |key| hash[key] = send(key) }
+        hash
       end
 
       def old_line
@@ -29,9 +45,25 @@ module Gitlab
         type == 'old'
       end
 
+      def rich_text
+        @parent_file.highlight_lines! if @parent_file && !@rich_text
+
+        @rich_text
+      end
+
       def meta?
         type == 'match' || type == 'nonewline'
       end
+
+      def as_json(opts = nil)
+        {
+          type: type,
+          old_line: old_line,
+          new_line: new_line,
+          text: text,
+          rich_text: rich_text || text
+        }
+      end
     end
   end
 end
diff --git a/lib/gitlab/downtime_check/message.rb b/lib/gitlab/downtime_check/message.rb
index 4446e921e0d..40a4815a9a0 100644
--- a/lib/gitlab/downtime_check/message.rb
+++ b/lib/gitlab/downtime_check/message.rb
@@ -1,10 +1,10 @@
 module Gitlab
   class DowntimeCheck
     class Message
-      attr_reader :path, :offline, :reason
+      attr_reader :path, :offline
 
-      OFFLINE = "\e[32moffline\e[0m"
-      ONLINE = "\e[31monline\e[0m"
+      OFFLINE = "\e[31moffline\e[0m"
+      ONLINE = "\e[32monline\e[0m"
 
       # path - The file path of the migration.
       # offline - When set to `true` the migration will require downtime.
@@ -19,10 +19,21 @@ module Gitlab
         label = offline ? OFFLINE : ONLINE
 
         message = "[#{label}]: #{path}"
-        message += ": #{reason}" if reason
+
+        if reason?
+          message += ":\n\n#{reason}\n\n"
+        end
 
         message
       end
+
+      def reason?
+        @reason.present?
+      end
+
+      def reason
+        @reason.strip.lines.map(&:strip).join("\n")
+      end
     end
   end
 end
diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb
index 97701b0cd42..0e3b65fceb4 100644
--- a/lib/gitlab/email/message/repository_push.rb
+++ b/lib/gitlab/email/message/repository_push.rb
@@ -35,21 +35,22 @@ module Gitlab
         def commits
           return unless compare
 
-          @commits ||= Commit.decorate(compare.commits, project)
+          @commits ||= compare.commits
         end
 
         def diffs
           return unless compare
-          
-          @diffs ||= safe_diff_files(compare.diffs(max_files: 30), diff_refs: diff_refs, repository: project.repository)
+
+          # This diff is more moderated in number of files and lines
+          @diffs ||= compare.diffs(max_files: 30, max_lines: 5000, no_collapse: true).diff_files
         end
 
         def diffs_count
-          diffs.count if diffs
+          diffs.size if diffs
         end
 
         def compare
-          @opts[:compare]
+          @opts[:compare] if @opts[:compare]
         end
 
         def diff_refs
@@ -97,16 +98,18 @@ module Gitlab
             if commits.length > 1
               namespace_project_compare_url(project_namespace,
                                             project,
-                                            from: Commit.new(compare.base, project),
-                                            to:   Commit.new(compare.head, project))
+                                            from: compare.start_commit,
+                                            to:   compare.head_commit)
             else
               namespace_project_commit_url(project_namespace,
-                                           project, commits.first)
+                                           project,
+                                           commits.first)
             end
           else
             unless @action == :delete
               namespace_project_tree_url(project_namespace,
-                                         project, ref_name)
+                                         project,
+                                         ref_name)
             end
           end
         end
diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb
index 9213cfb51e8..a40c44eb1bc 100644
--- a/lib/gitlab/email/receiver.rb
+++ b/lib/gitlab/email/receiver.rb
@@ -1,5 +1,5 @@
 
-require 'gitlab/email/handler'
+require_dependency 'gitlab/email/handler'
 
 # Inspired in great part by Discourse's Email::Receiver
 module Gitlab
diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb
index 191bea86ac3..7584efe4fa8 100644
--- a/lib/gitlab/git.rb
+++ b/lib/gitlab/git.rb
@@ -9,6 +9,24 @@ module Gitlab
         ref.gsub(/\Arefs\/(tags|heads)\//, '')
       end
 
+      def branch_name(ref)
+        ref = ref.to_s
+        if self.branch_ref?(ref)
+          self.ref_name(ref)
+        else
+          nil
+        end
+      end
+
+      def tag_name(ref)
+        ref = ref.to_s
+        if self.tag_ref?(ref)
+          self.ref_name(ref)
+        else
+          nil
+        end
+      end
+
       def tag_ref?(ref)
         ref.start_with?(TAG_REF_PREFIX)
       end
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index 8e8f39d9cb2..1882eb8d050 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -14,7 +14,7 @@ module Gitlab
       @user_access = UserAccess.new(user, project: project)
     end
 
-    def check(cmd, changes = nil)
+    def check(cmd, changes)
       return build_status_object(false, "Git access over #{protocol.upcase} is not allowed") unless protocol_allowed?
 
       unless actor
@@ -76,10 +76,10 @@ module Gitlab
         return build_status_object(false, "A repository for this project does not exist yet.")
       end
 
-      changes = changes.lines if changes.kind_of?(String)
+      changes_list = Gitlab::ChangesList.new(changes)
 
       # Iterate over all changes to find if user allowed all of them to be applied
-      changes.map(&:strip).reject(&:blank?).each do |change|
+      changes_list.each do |change|
         status = change_access_check(change)
         unless status.allowed?
           # If user does not have access to make at least one change - cancel all push
@@ -134,7 +134,7 @@ module Gitlab
     end
 
     def build_status_object(status, message = '')
-      GitAccessStatus.new(status, message)
+      Gitlab::GitAccessStatus.new(status, message)
     end
   end
 end
diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb
index a088e19d1e7..d32bdd86427 100644
--- a/lib/gitlab/git_post_receive.rb
+++ b/lib/gitlab/git_post_receive.rb
@@ -39,7 +39,6 @@ module Gitlab
     end
 
     def deserialize_changes(changes)
-      changes = Base64.decode64(changes) unless changes.include?(' ')
       changes = utf8_encode_changes(changes)
       changes.lines
     end
diff --git a/lib/gitlab/github_import/branch_formatter.rb b/lib/gitlab/github_import/branch_formatter.rb
index 7d2d545b84e..4750675ae9d 100644
--- a/lib/gitlab/github_import/branch_formatter.rb
+++ b/lib/gitlab/github_import/branch_formatter.rb
@@ -7,10 +7,6 @@ module Gitlab
         branch_exists? && commit_exists?
       end
 
-      def name
-        @name ||= exists? ? ref : "#{ref}-#{short_id}"
-      end
-
       def valid?
         repo.present?
       end
diff --git a/lib/gitlab/github_import/hook_formatter.rb b/lib/gitlab/github_import/hook_formatter.rb
deleted file mode 100644
index db1fabaa18a..00000000000
--- a/lib/gitlab/github_import/hook_formatter.rb
+++ /dev/null
@@ -1,23 +0,0 @@
-module Gitlab
-  module GithubImport
-    class HookFormatter
-      EVENTS = %w[* create delete pull_request push].freeze
-
-      attr_reader :raw
-
-      delegate :id, :name, :active, to: :raw
-
-      def initialize(raw)
-        @raw = raw
-      end
-
-      def config
-        raw.config.attrs
-      end
-
-      def valid?
-        (EVENTS & raw.events).any? && active
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index 3932fcb1eda..9ddc8905bd6 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -12,7 +12,6 @@ module Gitlab
 
         if credentials
           @client = Client.new(credentials[:user])
-          @formatter = Gitlab::ImportFormatter.new
         else
           raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}"
         end
@@ -66,73 +65,45 @@ module Gitlab
       end
 
       def import_pull_requests
-        disable_webhooks
-
         pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100)
         pull_requests = pull_requests.map { |raw| PullRequestFormatter.new(project, raw) }.select(&:valid?)
 
-        source_branches_removed = pull_requests.reject(&:source_branch_exists?).map { |pr| [pr.source_branch_name, pr.source_branch_sha] }
-        target_branches_removed = pull_requests.reject(&:target_branch_exists?).map { |pr| [pr.target_branch_name, pr.target_branch_sha] }
-        branches_removed = source_branches_removed | target_branches_removed
-
-        restore_branches(branches_removed)
-
         pull_requests.each do |pull_request|
-          merge_request = pull_request.create!
-          apply_labels(merge_request)
-          import_comments(merge_request)
-          import_comments_on_diff(merge_request)
+          begin
+            restore_source_branch(pull_request) unless pull_request.source_branch_exists?
+            restore_target_branch(pull_request) unless pull_request.target_branch_exists?
+
+            merge_request = pull_request.create!
+            apply_labels(merge_request)
+            import_comments(merge_request)
+            import_comments_on_diff(merge_request)
+          rescue ActiveRecord::RecordInvalid => e
+            raise Projects::ImportService::Error, e.message
+          ensure
+            clean_up_restored_branches(pull_request)
+          end
         end
 
         true
-      rescue ActiveRecord::RecordInvalid => e
-        raise Projects::ImportService::Error, e.message
-      ensure
-        clean_up_restored_branches(branches_removed)
-        clean_up_disabled_webhooks
       end
 
-      def disable_webhooks
-        update_webhooks(hooks, active: false)
+      def restore_source_branch(pull_request)
+        project.repository.fetch_ref(repo_url, "pull/#{pull_request.number}/head", pull_request.source_branch_name)
       end
 
-      def clean_up_disabled_webhooks
-        update_webhooks(hooks, active: true)
+      def restore_target_branch(pull_request)
+        project.repository.create_branch(pull_request.target_branch_name, pull_request.target_branch_sha)
       end
 
-      def update_webhooks(hooks, options)
-        hooks.each do |hook|
-          client.edit_hook(repo, hook.id, hook.name, hook.config, options)
-        end
+      def remove_branch(name)
+        project.repository.delete_branch(name)
+      rescue Rugged::ReferenceError
+        nil
       end
 
-      def hooks
-        @hooks ||=
-          begin
-            client.hooks(repo).map { |raw| HookFormatter.new(raw) }.select(&:valid?)
-
-          # The GitHub Repository Webhooks API returns 404 for users
-          # without admin access to the repository when listing hooks.
-          # In this case we just want to return gracefully instead of
-          # spitting out an error and stop the import process.
-          rescue Octokit::NotFound
-            []
-          end
-      end
-
-      def restore_branches(branches)
-        branches.each do |name, sha|
-          client.create_ref(repo, "refs/heads/#{name}", sha)
-        end
-
-        project.repository.fetch_ref(repo_url, '+refs/heads/*', 'refs/heads/*')
-      end
-
-      def clean_up_restored_branches(branches)
-        branches.each do |name, _|
-          client.delete_ref(repo, "heads/#{name}")
-          project.repository.delete_branch(name) rescue Rugged::ReferenceError
-        end
+      def clean_up_restored_branches(pull_request)
+        remove_branch(pull_request.source_branch_name) unless pull_request.source_branch_exists?
+        remove_branch(pull_request.target_branch_name) unless pull_request.target_branch_exists?
 
         project.repository.after_remove_branch
       end
diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb
index a4ea2210abd..b84538a090a 100644
--- a/lib/gitlab/github_import/pull_request_formatter.rb
+++ b/lib/gitlab/github_import/pull_request_formatter.rb
@@ -1,8 +1,8 @@
 module Gitlab
   module GithubImport
     class PullRequestFormatter < BaseFormatter
-      delegate :exists?, :name, :project, :repo, :sha, to: :source_branch, prefix: true
-      delegate :exists?, :name, :project, :repo, :sha, to: :target_branch, prefix: true
+      delegate :exists?, :project, :ref, :repo, :sha, to: :source_branch, prefix: true
+      delegate :exists?, :project, :ref, :repo, :sha, to: :target_branch, prefix: true
 
       def attributes
         {
@@ -33,17 +33,29 @@ module Gitlab
       end
 
       def valid?
-        source_branch.valid? && target_branch.valid? && !cross_project?
+        source_branch.valid? && target_branch.valid?
       end
 
       def source_branch
         @source_branch ||= BranchFormatter.new(project, raw_data.head)
       end
 
+      def source_branch_name
+        @source_branch_name ||= begin
+          source_branch_exists? ? source_branch_ref : "pull/#{number}/#{source_branch_ref}"
+        end
+      end
+
       def target_branch
         @target_branch ||= BranchFormatter.new(project, raw_data.base)
       end
 
+      def target_branch_name
+        @target_branch_name ||= begin
+          target_branch_exists? ? target_branch_ref : "pull/#{number}/#{target_branch_ref}"
+        end
+      end
+
       private
 
       def assigned?
@@ -68,10 +80,6 @@ module Gitlab
         raw_data.body || ""
       end
 
-      def cross_project?
-        source_branch_repo.id != target_branch_repo.id
-      end
-
       def description
         formatter.author_line(author) + body
       end
diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb
index d6d14bd98a0..bb562bdcd2c 100644
--- a/lib/gitlab/import_export.rb
+++ b/lib/gitlab/import_export.rb
@@ -2,7 +2,7 @@ module Gitlab
   module ImportExport
     extend self
 
-    VERSION = '0.1.2'
+    VERSION = '0.1.3'
     FILENAME_LIMIT = 50
 
     def export_path(relative_path:)
@@ -13,6 +13,10 @@ module Gitlab
       File.join(Settings.shared['path'], 'tmp/project_exports')
     end
 
+    def import_upload_path(filename:)
+      File.join(storage_path, 'uploads', filename)
+    end
+
     def project_filename
       "project.json"
     end
diff --git a/lib/gitlab/import_export/avatar_restorer.rb b/lib/gitlab/import_export/avatar_restorer.rb
index 352539eb594..cfa595629f4 100644
--- a/lib/gitlab/import_export/avatar_restorer.rb
+++ b/lib/gitlab/import_export/avatar_restorer.rb
@@ -1,7 +1,6 @@
 module Gitlab
   module ImportExport
     class AvatarRestorer
-
       def initialize(project:, shared:)
         @project = project
         @shared = shared
diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
index 82d1e1805c5..eca6e5b6d51 100644
--- a/lib/gitlab/import_export/file_importer.rb
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -3,6 +3,8 @@ module Gitlab
     class FileImporter
       include Gitlab::ImportExport::CommandLineUtil
 
+      MAX_RETRIES = 8
+
       def self.import(*args)
         new(*args).import
       end
@@ -14,7 +16,10 @@ module Gitlab
 
       def import
         FileUtils.mkdir_p(@shared.export_path)
-        decompress_archive
+
+        wait_for_archived_file do
+          decompress_archive
+        end
       rescue => e
         @shared.error(e)
         false
@@ -22,6 +27,17 @@ module Gitlab
 
       private
 
+      # Exponentially sleep until I/O finishes copying the file
+      def wait_for_archived_file
+        MAX_RETRIES.times do |retry_number|
+          break if File.exist?(@archive_file)
+
+          sleep(2**retry_number)
+        end
+
+        yield
+      end
+
       def decompress_archive
         result = untar_zxf(archive: @archive_file, dir: @shared.export_path)
 
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index 15afe8174a4..1da51043611 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -3,11 +3,12 @@ project_tree:
   - issues:
     - :events
     - notes:
-        - :author
-        - :events
-  - :labels
-  - milestones:
-    - :events
+      - :author
+      - :events
+    - label_links:
+      - :label
+    - milestone:
+      - :events
   - snippets:
     - notes:
         :author
@@ -20,6 +21,10 @@ project_tree:
       - :events
     - :merge_request_diff
     - :events
+    - label_links:
+      - :label
+    - milestone:
+      - :events
   - pipelines:
     - notes:
       - :author
@@ -31,6 +36,9 @@ project_tree:
   - :services
   - :hooks
   - :protected_branches
+  - :labels
+  - milestones:
+    - :events
 
 # Only include the following attributes for the models specified.
 included_attributes:
@@ -55,6 +63,10 @@ excluded_attributes:
     - :expired_at
   merge_request_diff:
     - :st_diffs
+  issues:
+    - :milestone_id
+  merge_requests:
+    - :milestone_id
 
 methods:
   statuses:
diff --git a/lib/gitlab/import_export/json_hash_builder.rb b/lib/gitlab/import_export/json_hash_builder.rb
new file mode 100644
index 00000000000..0cc10f40087
--- /dev/null
+++ b/lib/gitlab/import_export/json_hash_builder.rb
@@ -0,0 +1,107 @@
+module Gitlab
+  module ImportExport
+    # Generates a hash that conforms with http://apidock.com/rails/Hash/to_json
+    # and its peculiar options.
+    class JsonHashBuilder
+      def self.build(model_objects, attributes_finder)
+        new(model_objects, attributes_finder).build
+      end
+
+      def initialize(model_objects, attributes_finder)
+        @model_objects = model_objects
+        @attributes_finder = attributes_finder
+      end
+
+      def build
+        process_model_objects(@model_objects)
+      end
+
+      private
+
+      # Called when the model is actually a hash containing other relations (more models)
+      # Returns the config in the right format for calling +to_json+
+      #
+      # +model_object_hash+ - A model relationship such as:
+      #   {:merge_requests=>[:merge_request_diff, :notes]}
+      def process_model_objects(model_object_hash)
+        json_config_hash = {}
+        current_key = model_object_hash.keys.first
+
+        model_object_hash.values.flatten.each do |model_object|
+          @attributes_finder.parse(current_key) { |hash| json_config_hash[current_key] ||= hash }
+          handle_model_object(current_key, model_object, json_config_hash)
+        end
+
+        json_config_hash
+      end
+
+      # Creates or adds to an existing hash an individual model or list
+      #
+      # +current_key+ main model that will be a key in the hash
+      # +model_object+ model or list of models to include in the hash
+      # +json_config_hash+ the original hash containing the root model
+      def handle_model_object(current_key, model_object, json_config_hash)
+        model_or_sub_model = model_object.is_a?(Hash) ? process_model_objects(model_object) : model_object
+
+        if json_config_hash[current_key]
+          add_model_value(current_key, model_or_sub_model, json_config_hash)
+        else
+          create_model_value(current_key, model_or_sub_model, json_config_hash)
+        end
+      end
+
+      # Constructs a new hash that will hold the configuration for that particular object
+      # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+
+      #
+      # +current_key+ main model that will be a key in the hash
+      # +value+ existing model to be included in the hash
+      # +json_config_hash+ the original hash containing the root model
+      def create_model_value(current_key, value, json_config_hash)
+        json_config_hash[current_key] = parse_hash(value) || { include: value }
+      end
+
+      # Calls attributes finder to parse the hash and add any attributes to it
+      #
+      # +value+ existing model to be included in the hash
+      # +parsed_hash+ the original hash
+      def parse_hash(value)
+        @attributes_finder.parse(value) do |hash|
+          { include: hash_or_merge(value, hash) }
+        end
+      end
+
+      # Adds new model configuration to an existing hash with key +current_key+
+      # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+
+      #
+      # +current_key+ main model that will be a key in the hash
+      # +value+ existing model to be included in the hash
+      # +json_config_hash+ the original hash containing the root model
+      def add_model_value(current_key, value, json_config_hash)
+        @attributes_finder.parse(value) { |hash| value = { value => hash } }
+
+        add_to_array(current_key, json_config_hash, value)
+      end
+
+      # Adds new model configuration to an existing hash with key +current_key+
+      # it creates a new array if it was previously a single value
+      #
+      # +current_key+ main model that will be a key in the hash
+      # +value+ existing model to be included in the hash
+      # +json_config_hash+ the original hash containing the root model
+      def add_to_array(current_key, json_config_hash, value)
+        old_values = json_config_hash[current_key][:include]
+
+        json_config_hash[current_key][:include] = ([old_values] + [value]).compact.flatten
+      end
+
+      # Construct a new hash or merge with an existing one a model configuration
+      # This is to fulfil +to_json+ requirements.
+      #
+      # +hash+ hash containing configuration generated mainly from +@attributes_finder+
+      # +value+ existing model to be included in the hash
+      def hash_or_merge(value, hash)
+        value.is_a?(Hash) ? value.merge(hash) : { value => hash }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb
index b459054c198..36c4cf6efa0 100644
--- a/lib/gitlab/import_export/members_mapper.rb
+++ b/lib/gitlab/import_export/members_mapper.rb
@@ -18,11 +18,14 @@ module Gitlab
         @map ||=
           begin
             @exported_members.inject(missing_keys_tracking_hash) do |hash, member|
-              existing_user = User.where(find_project_user_query(member)).first
-              old_user_id = member['user']['id']
-              if existing_user && add_user_as_team_member(existing_user, member)
-                hash[old_user_id] = existing_user.id
+              if member['user']
+                old_user_id = member['user']['id']
+                existing_user = User.where(find_project_user_query(member)).first
+                hash[old_user_id] = existing_user.id if existing_user && add_team_member(member, existing_user)
+              else
+                add_team_member(member)
               end
+
               hash
             end
           end
@@ -45,7 +48,7 @@ module Gitlab
         ProjectMember.create!(user: @user, access_level: ProjectMember::MASTER, source_id: @project.id, importing: true)
       end
 
-      def add_user_as_team_member(existing_user, member)
+      def add_team_member(member, existing_user = nil)
         member['user'] = existing_user
 
         ProjectMember.create(member_hash(member)).persisted?
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index 051110c23cf..c7b3551b84c 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -47,7 +47,7 @@ module Gitlab
 
           relation_key = relation.is_a?(Hash) ? relation.keys.first : relation
           relation_hash = create_relation(relation_key, @tree_hash[relation_key.to_s])
-          saved << restored_project.update_attribute(relation_key, relation_hash)
+          saved << restored_project.append_or_update_attribute(relation_key, relation_hash)
         end
         saved.all?
       end
@@ -78,7 +78,7 @@ module Gitlab
         relation_key = relation.keys.first.to_s
         return if tree_hash[relation_key].blank?
 
-        tree_hash[relation_key].each do |relation_item|
+        [tree_hash[relation_key]].flatten.each do |relation_item|
           relation.values.flatten.each do |sub_relation|
             # We just use author to get the user ID, do not attempt to create an instance.
             next if sub_relation == :author
diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb
index 15f5dd31035..5021a1a14ce 100644
--- a/lib/gitlab/import_export/reader.rb
+++ b/lib/gitlab/import_export/reader.rb
@@ -29,87 +29,12 @@ module Gitlab
       def build_hash(model_list)
         model_list.map do |model_objects|
           if model_objects.is_a?(Hash)
-            build_json_config_hash(model_objects)
+            Gitlab::ImportExport::JsonHashBuilder.build(model_objects, @attributes_finder)
           else
             @attributes_finder.find(model_objects)
           end
         end
       end
-
-      # Called when the model is actually a hash containing other relations (more models)
-      # Returns the config in the right format for calling +to_json+
-      # +model_object_hash+ - A model relationship such as:
-      #   {:merge_requests=>[:merge_request_diff, :notes]}
-      def build_json_config_hash(model_object_hash)
-        @json_config_hash = {}
-
-        model_object_hash.values.flatten.each do |model_object|
-          current_key = model_object_hash.keys.first
-
-          @attributes_finder.parse(current_key) { |hash| @json_config_hash[current_key] ||= hash }
-
-          handle_model_object(current_key, model_object)
-          process_sub_model(current_key, model_object) if model_object.is_a?(Hash)
-        end
-        @json_config_hash
-      end
-
-      # If the model is a hash, process the sub_models, which could also be hashes
-      # If there is a list, add to an existing array, otherwise use hash syntax
-      # +current_key+ main model that will be a key in the hash
-      # +model_object+ model or list of models to include in the hash
-      def process_sub_model(current_key, model_object)
-        sub_model_json = build_json_config_hash(model_object).dup
-        @json_config_hash.slice!(current_key)
-
-        if @json_config_hash[current_key] && @json_config_hash[current_key][:include]
-          @json_config_hash[current_key][:include] << sub_model_json
-        else
-          @json_config_hash[current_key] = { include: sub_model_json }
-        end
-      end
-
-      # Creates or adds to an existing hash an individual model or list
-      # +current_key+ main model that will be a key in the hash
-      # +model_object+ model or list of models to include in the hash
-      def handle_model_object(current_key, model_object)
-        if @json_config_hash[current_key]
-          add_model_value(current_key, model_object)
-        else
-          create_model_value(current_key, model_object)
-        end
-      end
-
-      # Constructs a new hash that will hold the configuration for that particular object
-      # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+
-      # +current_key+ main model that will be a key in the hash
-      # +value+ existing model to be included in the hash
-      def create_model_value(current_key, value)
-        parsed_hash = { include: value }
-
-        @attributes_finder.parse(value) do |hash|
-          parsed_hash = { include: hash_or_merge(value, hash) }
-        end
-        @json_config_hash[current_key] = parsed_hash
-      end
-
-      # Adds new model configuration to an existing hash with key +current_key+
-      # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+
-      # +current_key+ main model that will be a key in the hash
-      # +value+ existing model to be included in the hash
-      def add_model_value(current_key, value)
-        @attributes_finder.parse(value) { |hash| value = { value => hash } }
-        old_values = @json_config_hash[current_key][:include]
-        @json_config_hash[current_key][:include] = ([old_values] + [value]).compact.flatten
-      end
-
-      # Construct a new hash or merge with an existing one a model configuration
-      # This is to fulfil +to_json+ requirements.
-      # +value+ existing model to be included in the hash
-      # +hash+ hash containing configuration generated mainly from +@attributes_finder+
-      def hash_or_merge(value, hash)
-        value.is_a?(Hash) ? value.merge(hash) : { value => hash }
-      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index e41c7e6bf4f..b0726268ca6 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -13,6 +13,10 @@ module Gitlab
 
       BUILD_MODELS = %w[Ci::Build commit_status].freeze
 
+      IMPORTED_OBJECT_MAX_RETRIES = 5.freeze
+
+      EXISTING_OBJECT_CHECK = %i[milestone milestones label labels].freeze
+
       def self.create(*args)
         new(*args).create
       end
@@ -22,24 +26,35 @@ module Gitlab
         @relation_hash = relation_hash.except('id', 'noteable_id')
         @members_mapper = members_mapper
         @user = user
+        @imported_object_retries = 0
       end
 
       # Creates an object from an actual model with name "relation_sym" with params from
       # the relation_hash, updating references with new object IDs, mapping users using
       # the "members_mapper" object, also updating notes if required.
       def create
-        set_note_author if @relation_name == :notes
+        setup_models
+
+        generate_imported_object
+      end
+
+      private
+
+      def setup_models
+        if @relation_name == :notes
+          set_note_author
+
+          # attachment is deprecated and note uploads are handled by Markdown uploader
+          @relation_hash['attachment'] = nil
+        end
+
         update_user_references
         update_project_references
         reset_ci_tokens if @relation_name == 'Ci::Trigger'
         @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data']
         set_st_diffs if @relation_name == :merge_request_diff
-
-        generate_imported_object
       end
 
-      private
-
       def update_user_references
         USER_REFERENCES.each do |reference|
           if @relation_hash[reference]
@@ -87,17 +102,19 @@ module Gitlab
       def update_project_references
         project_id = @relation_hash.delete('project_id')
 
+        # If source and target are the same, populate them with the new project ID.
+        if @relation_hash['source_project_id']
+          @relation_hash['source_project_id'] = same_source_and_target? ? project_id : -1
+        end
+
         # project_id may not be part of the export, but we always need to populate it if required.
         @relation_hash['project_id'] = project_id
         @relation_hash['gl_project_id'] = project_id if @relation_hash['gl_project_id']
         @relation_hash['target_project_id'] = project_id if @relation_hash['target_project_id']
-        @relation_hash['source_project_id'] = -1 if @relation_hash['source_project_id']
+      end
 
-        # If source and target are the same, populate them with the new project ID.
-        if @relation_hash['source_project_id'] && @relation_hash['target_project_id'] &&
-          @relation_hash['target_project_id'] == @relation_hash['source_project_id']
-          @relation_hash['source_project_id'] = project_id
-        end
+      def same_source_and_target?
+        @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id']
       end
 
       def reset_ci_tokens
@@ -112,10 +129,14 @@ module Gitlab
       end
 
       def imported_object
-        imported_object = relation_class.new(parsed_relation_hash)
-        yield(imported_object) if block_given?
-        imported_object.importing = true if imported_object.respond_to?(:importing)
-        imported_object
+        yield(existing_or_new_object) if block_given?
+        existing_or_new_object.importing = true if existing_or_new_object.respond_to?(:importing)
+        existing_or_new_object
+      rescue ActiveRecord::RecordNotUnique
+        # as the operation is not atomic, retry in the unlikely scenario an INSERT is
+        # performed on the same object between the SELECT and the INSERT
+        @imported_object_retries += 1
+        retry if @imported_object_retries < IMPORTED_OBJECT_MAX_RETRIES
       end
 
       def update_note_for_missing_author(author_name)
@@ -134,6 +155,20 @@ module Gitlab
       def set_st_diffs
         @relation_hash['st_diffs'] = @relation_hash.delete('utf8_st_diffs')
       end
+
+      def existing_or_new_object
+        # Only find existing records to avoid mapping tables such as milestones
+        # Otherwise always create the record, skipping the extra SELECT clause.
+        @existing_or_new_object ||= begin
+          if EXISTING_OBJECT_CHECK.include?(@relation_name)
+            existing_object = relation_class.find_or_initialize_by(parsed_relation_hash.slice('title', 'project_id'))
+            existing_object.assign_attributes(parsed_relation_hash)
+            existing_object
+          else
+            relation_class.new(parsed_relation_hash)
+          end
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb
index abfc694b879..de3fe6d822e 100644
--- a/lib/gitlab/import_export/version_checker.rb
+++ b/lib/gitlab/import_export/version_checker.rb
@@ -25,7 +25,7 @@ module Gitlab
 
       def verify_version!(version)
         if Gem::Version.new(version) > Gem::Version.new(Gitlab::ImportExport.version)
-          raise Gitlab::ImportExport::Error("Import version mismatch: Required <= #{Gitlab::ImportExport.version} but was #{version}")
+          raise Gitlab::ImportExport::Error.new("Import version mismatch: Required <= #{Gitlab::ImportExport.version} but was #{version}")
         else
           true
         end
diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb
index f2b649e50a2..2f326d00a2f 100644
--- a/lib/gitlab/ldap/access.rb
+++ b/lib/gitlab/ldap/access.rb
@@ -25,7 +25,7 @@ module Gitlab
         end
       end
 
-      def initialize(user, adapter=nil)
+      def initialize(user, adapter = nil)
         @adapter = adapter
         @user = user
         @provider = user.ldap_identity.provider
diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb
index df65179bfea..9a5bcfb5c9b 100644
--- a/lib/gitlab/ldap/adapter.rb
+++ b/lib/gitlab/ldap/adapter.rb
@@ -13,7 +13,7 @@ module Gitlab
         Gitlab::LDAP::Config.new(provider)
       end
 
-      def initialize(provider, ldap=nil)
+      def initialize(provider, ldap = nil)
         @provider = provider
         @ldap = ldap || Net::LDAP.new(config.adapter_options)
       end
diff --git a/lib/gitlab/lfs/response.rb b/lib/gitlab/lfs/response.rb
deleted file mode 100644
index a1ee1aa81ff..00000000000
--- a/lib/gitlab/lfs/response.rb
+++ /dev/null
@@ -1,329 +0,0 @@
-module Gitlab
-  module Lfs
-    class Response
-      def initialize(project, user, ci, request)
-        @origin_project = project
-        @project = storage_project(project)
-        @user = user
-        @ci = ci
-        @env = request.env
-        @request = request
-      end
-
-      def render_download_object_response(oid)
-        render_response_to_download do
-          if check_download_sendfile_header?
-            render_lfs_sendfile(oid)
-          else
-            render_not_found
-          end
-        end
-      end
-
-      def render_batch_operation_response
-        request_body = JSON.parse(@request.body.read)
-        case request_body["operation"]
-        when "download"
-          render_batch_download(request_body)
-        when "upload"
-          render_batch_upload(request_body)
-        else
-          render_not_found
-        end
-      end
-
-      def render_storage_upload_authorize_response(oid, size)
-        render_response_to_push do
-          [
-            200,
-            { "Content-Type" => "application/json; charset=utf-8" },
-            [JSON.dump({
-              'StoreLFSPath' => "#{Gitlab.config.lfs.storage_path}/tmp/upload",
-              'LfsOid' => oid,
-              'LfsSize' => size
-            })]
-          ]
-        end
-      end
-
-      def render_storage_upload_store_response(oid, size, tmp_file_name)
-        return render_forbidden unless tmp_file_name
-
-        render_response_to_push do
-          render_lfs_upload_ok(oid, size, tmp_file_name)
-        end
-      end
-
-      def render_unsupported_deprecated_api
-        [
-          501,
-          { "Content-Type" => "application/json; charset=utf-8" },
-          [JSON.dump({
-            'message' => 'Server supports batch API only, please update your Git LFS client to version 1.0.1 and up.',
-            'documentation_url' => "#{Gitlab.config.gitlab.url}/help",
-          })]
-        ]
-      end
-
-      private
-
-      def render_not_enabled
-        [
-          501,
-          {
-            "Content-Type" => "application/json; charset=utf-8",
-          },
-          [JSON.dump({
-            'message' => 'Git LFS is not enabled on this GitLab server, contact your admin.',
-            'documentation_url' => "#{Gitlab.config.gitlab.url}/help",
-          })]
-        ]
-      end
-
-      def render_unauthorized
-        [
-          401,
-          {
-            'Content-Type' => 'text/plain'
-          },
-          ['Unauthorized']
-        ]
-      end
-
-      def render_not_found
-        [
-          404,
-          {
-            "Content-Type" => "application/vnd.git-lfs+json"
-          },
-          [JSON.dump({
-            'message' => 'Not found.',
-            'documentation_url' => "#{Gitlab.config.gitlab.url}/help",
-          })]
-        ]
-      end
-
-      def render_forbidden
-        [
-          403,
-          {
-            "Content-Type" => "application/vnd.git-lfs+json"
-          },
-          [JSON.dump({
-            'message' => 'Access forbidden. Check your access level.',
-            'documentation_url' => "#{Gitlab.config.gitlab.url}/help",
-          })]
-        ]
-      end
-
-      def render_lfs_sendfile(oid)
-        return render_not_found unless oid.present?
-
-        lfs_object = object_for_download(oid)
-
-        if lfs_object && lfs_object.file.exists?
-          [
-            200,
-            {
-              # GitLab-workhorse will forward Content-Type header
-              "Content-Type" => "application/octet-stream",
-              "X-Sendfile" => lfs_object.file.path
-            },
-            []
-          ]
-        else
-          render_not_found
-        end
-      end
-
-      def render_batch_upload(body)
-        return render_not_found if body.empty? || body['objects'].nil?
-
-        render_response_to_push do
-          response = build_upload_batch_response(body['objects'])
-          [
-            200,
-            {
-              "Content-Type" => "application/json; charset=utf-8",
-              "Cache-Control" => "private",
-            },
-            [JSON.dump(response)]
-          ]
-        end
-      end
-
-      def render_batch_download(body)
-        return render_not_found if body.empty? || body['objects'].nil?
-
-        render_response_to_download do
-          response = build_download_batch_response(body['objects'])
-          [
-            200,
-            {
-              "Content-Type" => "application/json; charset=utf-8",
-              "Cache-Control" => "private",
-            },
-            [JSON.dump(response)]
-          ]
-        end
-      end
-
-      def render_lfs_upload_ok(oid, size, tmp_file)
-        if store_file(oid, size, tmp_file)
-          [
-            200,
-            {
-              'Content-Type' => 'text/plain',
-              'Content-Length' => 0
-            },
-            []
-          ]
-        else
-          [
-            422,
-            { 'Content-Type' => 'text/plain' },
-            ["Unprocessable entity"]
-          ]
-        end
-      end
-
-      def render_response_to_download
-        return render_not_enabled unless Gitlab.config.lfs.enabled
-
-        unless @project.public?
-          return render_unauthorized unless @user || @ci
-          return render_forbidden unless user_can_fetch?
-        end
-
-        yield
-      end
-
-      def render_response_to_push
-        return render_not_enabled unless Gitlab.config.lfs.enabled
-        return render_unauthorized unless @user
-        return render_forbidden unless user_can_push?
-
-        yield
-      end
-
-      def check_download_sendfile_header?
-        @env['HTTP_X_SENDFILE_TYPE'].to_s == "X-Sendfile"
-      end
-
-      def user_can_fetch?
-        # Check user access against the project they used to initiate the pull
-        @ci || @user.can?(:download_code, @origin_project)
-      end
-
-      def user_can_push?
-        # Check user access against the project they used to initiate the push
-        @user.can?(:push_code, @origin_project)
-      end
-
-      def storage_project(project)
-        if project.forked?
-          storage_project(project.forked_from_project)
-        else
-          project
-        end
-      end
-
-      def store_file(oid, size, tmp_file)
-        tmp_file_path = File.join("#{Gitlab.config.lfs.storage_path}/tmp/upload", tmp_file)
-
-        object = LfsObject.find_or_create_by(oid: oid, size: size)
-        if object.file.exists?
-          success = true
-        else
-          success = move_tmp_file_to_storage(object, tmp_file_path)
-        end
-
-        if success
-          success = link_to_project(object)
-        end
-
-        success
-      ensure
-        # Ensure that the tmp file is removed
-        FileUtils.rm_f(tmp_file_path)
-      end
-
-      def object_for_download(oid)
-        @project.lfs_objects.find_by(oid: oid)
-      end
-
-      def move_tmp_file_to_storage(object, path)
-        File.open(path) do |f|
-          object.file = f
-        end
-
-        object.file.store!
-        object.save
-      end
-
-      def link_to_project(object)
-        if object && !object.projects.exists?(@project.id)
-          object.projects << @project
-          object.save
-        end
-      end
-
-      def select_existing_objects(objects)
-        objects_oids = objects.map { |o| o['oid'] }
-        @project.lfs_objects.where(oid: objects_oids).pluck(:oid).to_set
-      end
-
-      def build_upload_batch_response(objects)
-        selected_objects = select_existing_objects(objects)
-
-        upload_hypermedia_links(objects, selected_objects)
-      end
-
-      def build_download_batch_response(objects)
-        selected_objects = select_existing_objects(objects)
-
-        download_hypermedia_links(objects, selected_objects)
-      end
-
-      def download_hypermedia_links(all_objects, existing_objects)
-        all_objects.each do |object|
-          if existing_objects.include?(object['oid'])
-            object['actions'] = {
-              'download' => {
-                'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}",
-                'header' => {
-                  'Authorization' => @env['HTTP_AUTHORIZATION']
-                }.compact
-              }
-            }
-          else
-            object['error'] = {
-              'code' => 404,
-              'message' => "Object does not exist on the server or you don't have permissions to access it",
-            }
-          end
-        end
-
-        { 'objects' => all_objects }
-      end
-
-      def upload_hypermedia_links(all_objects, existing_objects)
-        all_objects.each do |object|
-          # generate actions only for non-existing objects
-          next if existing_objects.include?(object['oid'])
-
-          object['actions'] = {
-            'upload' => {
-              'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}/#{object['size']}",
-              'header' => {
-                'Authorization' => @env['HTTP_AUTHORIZATION']
-              }.compact
-            }
-          }
-        end
-
-        { 'objects' => all_objects }
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/lfs/router.rb b/lib/gitlab/lfs/router.rb
deleted file mode 100644
index f2a76a56b8f..00000000000
--- a/lib/gitlab/lfs/router.rb
+++ /dev/null
@@ -1,98 +0,0 @@
-module Gitlab
-  module Lfs
-    class Router
-      attr_reader :project, :user, :ci, :request
-
-      def initialize(project, user, ci, request)
-        @project = project
-        @user = user
-        @ci = ci
-        @env = request.env
-        @request = request
-      end
-
-      def try_call
-        return unless @request && @request.path.present?
-
-        case @request.request_method
-        when 'GET'
-          get_response
-        when 'POST'
-          post_response
-        when 'PUT'
-          put_response
-        else
-          nil
-        end
-      end
-
-      private
-
-      def get_response
-        path_match = @request.path.match(/\/(info\/lfs|gitlab-lfs)\/objects\/([0-9a-f]{64})$/)
-        return nil unless path_match
-
-        oid = path_match[2]
-        return nil unless oid
-
-        case path_match[1]
-        when "info/lfs"
-          lfs.render_unsupported_deprecated_api
-        when "gitlab-lfs"
-          lfs.render_download_object_response(oid)
-        else
-          nil
-        end
-      end
-
-      def post_response
-        post_path = @request.path.match(/\/info\/lfs\/objects(\/batch)?$/)
-        return nil unless post_path
-
-        # Check for Batch API
-        if post_path[0].ends_with?("/info/lfs/objects/batch")
-          lfs.render_batch_operation_response
-        elsif post_path[0].ends_with?("/info/lfs/objects")
-          lfs.render_unsupported_deprecated_api
-        else
-          nil
-        end
-      end
-
-      def put_response
-        object_match = @request.path.match(/\/gitlab-lfs\/objects\/([0-9a-f]{64})\/([0-9]+)(|\/authorize){1}$/)
-        return nil if object_match.nil?
-
-        oid = object_match[1]
-        size = object_match[2].try(:to_i)
-        return nil if oid.nil? || size.nil?
-
-        # GitLab-workhorse requests
-        # 1. Try to authorize the request
-        # 2. send a request with a header containing the name of the temporary file
-        if object_match[3] && object_match[3] == '/authorize'
-          lfs.render_storage_upload_authorize_response(oid, size)
-        else
-          tmp_file_name = sanitize_tmp_filename(@request.env['HTTP_X_GITLAB_LFS_TMP'])
-          lfs.render_storage_upload_store_response(oid, size, tmp_file_name)
-        end
-      end
-
-      def lfs
-        return unless @project
-
-        Gitlab::Lfs::Response.new(@project, @user, @ci, @request)
-      end
-
-      def sanitize_tmp_filename(name)
-        if name.present?
-          name.gsub!(/^.*(\\|\/)/, '')
-          name = name.match(/[0-9a-f]{73}/)
-          name[0] if name
-        else
-          nil
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb
new file mode 100644
index 00000000000..12999a90a29
--- /dev/null
+++ b/lib/gitlab/mail_room.rb
@@ -0,0 +1,47 @@
+require 'yaml'
+require 'json'
+require_relative 'redis' unless defined?(Gitlab::Redis)
+
+module Gitlab
+  module MailRoom
+    class << self
+      def enabled?
+        config[:enabled] && config[:address]
+      end
+
+      def config
+        @config ||= fetch_config
+      end
+
+      def reset_config!
+        @config = nil
+      end
+
+      private
+
+      def fetch_config
+        return {} unless File.exist?(config_file)
+
+        rails_env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+        all_config = YAML.load_file(config_file)[rails_env].deep_symbolize_keys
+
+        config = all_config[:incoming_email] || {}
+        config[:enabled] = false if config[:enabled].nil?
+        config[:port] = 143 if config[:port].nil?
+        config[:ssl] = false if config[:ssl].nil?
+        config[:start_tls] = false if config[:start_tls].nil?
+        config[:mailbox] = 'inbox' if config[:mailbox].nil?
+
+        if config[:enabled] && config[:address]
+          config[:redis_url] = Gitlab::Redis.new(rails_env).url
+        end
+
+        config
+      end
+
+      def config_file
+        ENV['MAIL_ROOM_GITLAB_CONFIG_FILE'] || File.expand_path('../../../config/gitlab.yml', __FILE__)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index 081576a440c..3d1ba33ec68 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -124,6 +124,15 @@ module Gitlab
       trans.action = action if trans
     end
 
+    # Tracks an event.
+    #
+    # See `Gitlab::Metrics::Transaction#add_event` for more details.
+    def self.add_event(*args)
+      trans = current_transaction
+
+      trans.add_event(*args) if trans
+    end
+
     # Returns the prefix to use for the name of a series.
     def self.series_prefix
       @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_'
@@ -141,10 +150,9 @@ module Gitlab
       end
     end
 
+    # Allow access from other metrics related middlewares
     def self.current_transaction
       Transaction.current
     end
-
-    private_class_method :current_transaction
   end
 end
diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb
index f23d67e1e38..bd0afe53c51 100644
--- a/lib/gitlab/metrics/metric.rb
+++ b/lib/gitlab/metrics/metric.rb
@@ -4,15 +4,20 @@ module Gitlab
     class Metric
       JITTER_RANGE = 0.000001..0.001
 
-      attr_reader :series, :values, :tags
+      attr_reader :series, :values, :tags, :type
 
       # series - The name of the series (as a String) to store the metric in.
       # values - A Hash containing the values to store.
       # tags   - A Hash containing extra tags to add to the metrics.
-      def initialize(series, values, tags = {})
+      def initialize(series, values, tags = {}, type = :metric)
         @values = values
         @series = series
         @tags   = tags
+        @type   = type
+      end
+
+      def event?
+        type == :event
       end
 
       # Returns a Hash in a format that can be directly written to InfluxDB.
diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb
index e61670f491c..b4493bf44d2 100644
--- a/lib/gitlab/metrics/rack_middleware.rb
+++ b/lib/gitlab/metrics/rack_middleware.rb
@@ -17,6 +17,10 @@ module Gitlab
         begin
           retval = trans.run { @app.call(env) }
 
+        rescue Exception => error # rubocop: disable Lint/RescueException
+          trans.add_event(:rails_exception)
+
+          raise error
         # Even in the event of an error we want to submit any metrics we
         # might've gathered up to this point.
         ensure
diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb
index a1240fd33ee..f9dd8e41912 100644
--- a/lib/gitlab/metrics/sidekiq_middleware.rb
+++ b/lib/gitlab/metrics/sidekiq_middleware.rb
@@ -11,6 +11,10 @@ module Gitlab
           # Old gitlad-shell messages don't provide enqueued_at/created_at attributes
           trans.set(:sidekiq_queue_duration, Time.now.to_f - (message['enqueued_at'] || message['created_at'] || 0))
           trans.run { yield }
+        rescue Exception => error # rubocop: disable Lint/RescueException
+          trans.add_event(:sidekiq_exception)
+
+          raise error
         ensure
           trans.finish
         end
diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb
index 968f3218950..7bc16181be6 100644
--- a/lib/gitlab/metrics/transaction.rb
+++ b/lib/gitlab/metrics/transaction.rb
@@ -4,7 +4,10 @@ module Gitlab
     class Transaction
       THREAD_KEY = :_gitlab_metrics_transaction
 
-      attr_reader :tags, :values, :methods
+      # The series to store events (e.g. Git pushes) in.
+      EVENT_SERIES = 'events'
+
+      attr_reader :tags, :values, :method, :metrics
 
       attr_accessor :action
 
@@ -55,6 +58,20 @@ module Gitlab
         @metrics << Metric.new("#{Metrics.series_prefix}#{series}", values, tags)
       end
 
+      # Tracks a business level event
+      #
+      # Business level events including events such as Git pushes, Emails being
+      # sent, etc.
+      #
+      # event_name - The name of the event (e.g. "git_push").
+      # tags - A set of tags to attach to the event.
+      def add_event(event_name, tags = {})
+        @metrics << Metric.new(EVENT_SERIES,
+                               { count: 1 },
+                               { event: event_name }.merge(tags),
+                               :event)
+      end
+
       # Returns a MethodCall object for the given name.
       def method_call_for(name)
         unless method = @methods[name]
@@ -101,7 +118,7 @@ module Gitlab
         submit_hashes = submit.map do |metric|
           hash = metric.to_hash
 
-          hash[:tags][:action] ||= @action if @action
+          hash[:tags][:action] ||= @action if @action && !metric.event?
 
           hash
         end
diff --git a/lib/gitlab/popen.rb b/lib/gitlab/popen.rb
index 43e07e09160..ca23ccef25b 100644
--- a/lib/gitlab/popen.rb
+++ b/lib/gitlab/popen.rb
@@ -5,7 +5,7 @@ module Gitlab
   module Popen
     extend self
 
-    def popen(cmd, path=nil)
+    def popen(cmd, path = nil)
       unless cmd.is_a?(Array)
         raise "System commands must be given as an array of strings"
       end
diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb
index 40766f35f77..9376b54f43b 100644
--- a/lib/gitlab/redis.rb
+++ b/lib/gitlab/redis.rb
@@ -1,50 +1,94 @@
+# This file should not have any direct dependency on Rails environment
+# please require all dependencies below:
+require 'active_support/core_ext/hash/keys'
+
 module Gitlab
   class Redis
     CACHE_NAMESPACE = 'cache:gitlab'
     SESSION_NAMESPACE = 'session:gitlab'
     SIDEKIQ_NAMESPACE = 'resque:gitlab'
-
-    attr_reader :url
+    MAILROOM_NAMESPACE = 'mail_room:gitlab'
+    DEFAULT_REDIS_URL = 'redis://localhost:6379'
 
     # To be thread-safe we must be careful when writing the class instance
     # variables @url and @pool. Because @pool depends on @url we need two
     # mutexes to prevent deadlock.
-    URL_MUTEX = Mutex.new
+    PARAMS_MUTEX = Mutex.new
     POOL_MUTEX = Mutex.new
-    private_constant :URL_MUTEX, :POOL_MUTEX
+    private_constant :PARAMS_MUTEX, :POOL_MUTEX
 
-    def self.url
-      @url || URL_MUTEX.synchronize { @url = new.url }
-    end
+    class << self
+      def params
+        @params || PARAMS_MUTEX.synchronize { @params = new.params }
+      end
+
+      # @deprecated Use .params instead to get sentinel support
+      def url
+        new.url
+      end
 
-    def self.with
-      if @pool.nil?
-        POOL_MUTEX.synchronize do
-          @pool = ConnectionPool.new { ::Redis.new(url: url) }
+      def with
+        if @pool.nil?
+          POOL_MUTEX.synchronize do
+            @pool = ConnectionPool.new { ::Redis.new(params) }
+          end
         end
+        @pool.with { |redis| yield redis }
+      end
+
+      def reset_params!
+        @params = nil
       end
-      @pool.with { |redis| yield redis }
     end
 
-    def self.redis_store_options
-      url = new.url
-      redis_config_hash = ::Redis::Store::Factory.extract_host_options_from_uri(url)
-      # Redis::Store does not handle Unix sockets well, so let's do it for them
-      redis_uri = URI.parse(url)
+    def initialize(rails_env = nil)
+      @rails_env = rails_env || ::Rails.env
+    end
+
+    def params
+      redis_store_options
+    end
+
+    def url
+      raw_config_hash[:url]
+    end
+
+    private
+
+    def redis_store_options
+      config = raw_config_hash
+      redis_url = config.delete(:url)
+      redis_uri = URI.parse(redis_url)
+
       if redis_uri.scheme == 'unix'
-        redis_config_hash[:path] = redis_uri.path
+        # Redis::Store does not handle Unix sockets well, so let's do it for them
+        config[:path] = redis_uri.path
+        config
+      else
+        redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url)
+        # order is important here, sentinels must be after the connection keys.
+        # {url: ..., port: ..., sentinels: [...]}
+        redis_hash.merge(config)
       end
-      redis_config_hash
     end
 
-    def initialize(rails_env=nil)
-      rails_env ||= Rails.env
-      config_file = File.expand_path('../../../config/resque.yml', __FILE__)
+    def raw_config_hash
+      config_data = fetch_config
 
-      @url = "redis://localhost:6379"
-      if File.exist?(config_file)
-        @url = YAML.load_file(config_file)[rails_env]
+      if config_data
+        config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys
+      else
+        { url: DEFAULT_REDIS_URL }
       end
     end
+
+    def fetch_config
+      file = config_file
+      File.exist?(file) ? YAML.load_file(file)[@rails_env] : false
+    end
+
+    def config_file
+      File.expand_path('../../../config/resque.yml', __FILE__)
+    end
   end
 end
diff --git a/lib/gitlab/request_profiler/middleware.rb b/lib/gitlab/request_profiler/middleware.rb
index 8da8b754975..786e1d49f5e 100644
--- a/lib/gitlab/request_profiler/middleware.rb
+++ b/lib/gitlab/request_profiler/middleware.rb
@@ -1,4 +1,5 @@
 require 'ruby-prof'
+require_dependency 'gitlab/request_profiler'
 
 module Gitlab
   module RequestProfiler
@@ -28,7 +29,9 @@ module Gitlab
       def call_with_profiling(env)
         ret = nil
         result = RubyProf::Profile.profile do
-          ret = @app.call(env)
+          ret = catch(:warden) do
+            @app.call(env)
+          end
         end
 
         printer   = RubyProf::CallStackPrinter.new(result)
@@ -40,7 +43,11 @@ module Gitlab
           printer.print(file)
         end
 
-        ret
+        if ret.is_a?(Array)
+          ret
+        else
+          throw(:warden, ret)
+        end
       end
     end
   end
diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb
index 760ff3e614a..7ebec8e2cff 100644
--- a/lib/gitlab/template/base_template.rb
+++ b/lib/gitlab/template/base_template.rb
@@ -1,8 +1,9 @@
 module Gitlab
   module Template
     class BaseTemplate
-      def initialize(path)
+      def initialize(path, project = nil)
         @path = path
+        @finder = self.class.finder(project)
       end
 
       def name
@@ -10,23 +11,32 @@ module Gitlab
       end
 
       def content
-        File.read(@path)
+        @finder.read(@path)
+      end
+
+      def to_json
+        { name: name, content: content }
       end
 
       class << self
-        def all
-          self.categories.keys.flat_map { |cat| by_category(cat) }
+        def all(project = nil)
+          if categories.any?
+            categories.keys.flat_map { |cat| by_category(cat, project) }
+          else
+            by_category("", project)
+          end
         end
 
-        def find(key)
-          file_name = "#{key}#{self.extension}"
-
-          directory = select_directory(file_name)
-          directory ? new(File.join(category_directory(directory), file_name)) : nil
+        def find(key, project = nil)
+          path = self.finder(project).find(key)
+          path.present? ? new(path, project) : nil
         end
 
+        # Set categories as sub directories
+        # Example: { "category_name_1" => "directory_path_1", "category_name_2" => "directory_name_2" }
+        # Default is no category with all files in base dir of each class
         def categories
-          raise NotImplementedError
+          {}
         end
 
         def extension
@@ -37,29 +47,40 @@ module Gitlab
           raise NotImplementedError
         end
 
-        def by_category(category)
-          templates_for_directory(category_directory(category))
+        # Defines which strategy will be used to get templates files
+        # RepoTemplateFinder - Finds templates on project repository, templates are filtered perproject
+        # GlobalTemplateFinder - Finds templates on gitlab installation source, templates can be used in all projects
+        def finder(project = nil)
+          raise NotImplementedError
         end
 
-        def category_directory(category)
-          File.join(base_dir, categories[category])
+        def by_category(category, project = nil)
+          directory = category_directory(category)
+          files = finder(project).list_files_for(directory)
+
+          files.map { |f| new(f, project) }
         end
 
-        private
+        def category_directory(category)
+          return base_dir unless category.present?
 
-        def select_directory(file_name)
-          categories.keys.find do |category|
-            File.exist?(File.join(category_directory(category), file_name))
-          end
+          File.join(base_dir, categories[category])
         end
 
-        def templates_for_directory(dir)
-          dir << '/' unless dir.end_with?('/')
-          Dir.glob(File.join(dir, "*#{self.extension}")).select { |f| f =~ filter_regex }.map { |f| new(f) }
-        end
+        # If template is organized by category it returns { category_name: [{ name: template_name }, { name: template2_name }] }
+        # If no category is present returns [{ name: template_name }, { name: template2_name}]
+        def dropdown_names(project = nil)
+          return [] if project && !project.repository.exists?
 
-        def filter_regex
-          @filter_reges ||= /#{Regexp.escape(extension)}\z/
+          if categories.any?
+            categories.keys.map do |category|
+              files = self.by_category(category, project)
+              [category, files.map { |t| { name: t.name } }]
+            end.to_h
+          else
+            files = self.all(project)
+            files.map { |t| { name: t.name } }
+          end
         end
       end
     end
diff --git a/lib/gitlab/template/finders/base_template_finder.rb b/lib/gitlab/template/finders/base_template_finder.rb
new file mode 100644
index 00000000000..473b05257c6
--- /dev/null
+++ b/lib/gitlab/template/finders/base_template_finder.rb
@@ -0,0 +1,35 @@
+module Gitlab
+  module Template
+    module Finders
+      class BaseTemplateFinder
+        def initialize(base_dir)
+          @base_dir = base_dir
+        end
+
+        def list_files_for
+          raise NotImplementedError
+        end
+
+        def read
+          raise NotImplementedError
+        end
+
+        def find
+          raise NotImplementedError
+        end
+
+        def category_directory(category)
+          return @base_dir unless category.present?
+
+          @base_dir + @categories[category]
+        end
+
+        class << self
+          def filter_regex(extension)
+            /#{Regexp.escape(extension)}\z/
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/template/finders/global_template_finder.rb b/lib/gitlab/template/finders/global_template_finder.rb
new file mode 100644
index 00000000000..831da45191f
--- /dev/null
+++ b/lib/gitlab/template/finders/global_template_finder.rb
@@ -0,0 +1,38 @@
+# Searches and reads file present on Gitlab installation directory
+module Gitlab
+  module Template
+    module Finders
+      class GlobalTemplateFinder < BaseTemplateFinder
+        def initialize(base_dir, extension, categories = {})
+          @categories = categories
+          @extension  = extension
+          super(base_dir)
+        end
+
+        def read(path)
+          File.read(path)
+        end
+
+        def find(key)
+          file_name = "#{key}#{@extension}"
+
+          directory = select_directory(file_name)
+          directory ? File.join(category_directory(directory), file_name) : nil
+        end
+
+        def list_files_for(dir)
+          dir << '/' unless dir.end_with?('/')
+          Dir.glob(File.join(dir, "*#{@extension}")).select { |f| f =~ self.class.filter_regex(@extension) }
+        end
+
+        private
+
+        def select_directory(file_name)
+          @categories.keys.find do |category|
+            File.exist?(File.join(category_directory(category), file_name))
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/template/finders/repo_template_finder.rb b/lib/gitlab/template/finders/repo_template_finder.rb
new file mode 100644
index 00000000000..22c39436cb2
--- /dev/null
+++ b/lib/gitlab/template/finders/repo_template_finder.rb
@@ -0,0 +1,59 @@
+# Searches and reads files present on each Gitlab project repository
+module Gitlab
+  module Template
+    module Finders
+      class RepoTemplateFinder < BaseTemplateFinder
+        # Raised when file is not found
+        class FileNotFoundError < StandardError; end
+
+        def initialize(project, base_dir, extension, categories = {})
+          @categories     = categories
+          @extension      = extension
+          @repository     = project.repository
+          @commit         = @repository.head_commit if @repository.exists?
+
+          super(base_dir)
+        end
+
+        def read(path)
+          blob = @repository.blob_at(@commit.id, path) if @commit
+          raise FileNotFoundError if blob.nil?
+          blob.data
+        end
+
+        def find(key)
+          file_name = "#{key}#{@extension}"
+          directory = select_directory(file_name)
+          raise FileNotFoundError if directory.nil?
+
+          category_directory(directory) + file_name
+        end
+
+        def list_files_for(dir)
+          return [] unless @commit
+
+          dir << '/' unless dir.end_with?('/')
+
+          entries = @repository.tree(:head, dir).entries
+
+          names = entries.map(&:name)
+          names.select { |f| f =~ self.class.filter_regex(@extension) }
+        end
+
+        private
+
+        def select_directory(file_name)
+          return [] unless @commit
+
+          # Insert root as directory
+          directories = ["", @categories.keys]
+
+          directories.find do |category|
+            path = category_directory(category) + file_name
+            @repository.blob_at(@commit.id, path)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/template/gitignore.rb b/lib/gitlab/template/gitignore_template.rb
index 964fbfd4de3..8d2a9d2305c 100644
--- a/lib/gitlab/template/gitignore.rb
+++ b/lib/gitlab/template/gitignore_template.rb
@@ -1,6 +1,6 @@
 module Gitlab
   module Template
-    class Gitignore < BaseTemplate
+    class GitignoreTemplate < BaseTemplate
       class << self
         def extension
           '.gitignore'
@@ -16,6 +16,10 @@ module Gitlab
         def base_dir
           Rails.root.join('vendor/gitignore')
         end
+
+        def finder(project = nil)
+          Gitlab::Template::Finders::GlobalTemplateFinder.new(self.base_dir, self.extension, self.categories)
+        end
       end
     end
   end
diff --git a/lib/gitlab/template/gitlab_ci_yml.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb
index 7f480fe33c0..8d1a1ed54c9 100644
--- a/lib/gitlab/template/gitlab_ci_yml.rb
+++ b/lib/gitlab/template/gitlab_ci_yml_template.rb
@@ -1,6 +1,6 @@
 module Gitlab
   module Template
-    class GitlabCiYml < BaseTemplate
+    class GitlabCiYmlTemplate < BaseTemplate
       def content
         explanation = "# This file is a template, and might need editing before it works on your project."
         [explanation, super].join("\n")
@@ -21,6 +21,10 @@ module Gitlab
         def base_dir
           Rails.root.join('vendor/gitlab-ci-yml')
         end
+
+        def finder(project = nil)
+          Gitlab::Template::Finders::GlobalTemplateFinder.new(self.base_dir, self.extension, self.categories)
+        end
       end
     end
   end
diff --git a/lib/gitlab/template/issue_template.rb b/lib/gitlab/template/issue_template.rb
new file mode 100644
index 00000000000..c6fa8d3eafc
--- /dev/null
+++ b/lib/gitlab/template/issue_template.rb
@@ -0,0 +1,19 @@
+module Gitlab
+  module Template
+    class IssueTemplate < BaseTemplate
+      class << self
+        def extension
+          '.md'
+        end
+
+        def base_dir
+          '.gitlab/issue_templates/'
+        end
+
+        def finder(project)
+          Gitlab::Template::Finders::RepoTemplateFinder.new(project, self.base_dir, self.extension, self.categories)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/template/merge_request_template.rb b/lib/gitlab/template/merge_request_template.rb
new file mode 100644
index 00000000000..f826c02f3b5
--- /dev/null
+++ b/lib/gitlab/template/merge_request_template.rb
@@ -0,0 +1,19 @@
+module Gitlab
+  module Template
+    class MergeRequestTemplate < BaseTemplate
+      class << self
+        def extension
+          '.md'
+        end
+
+        def base_dir
+          '.gitlab/merge_request_templates/'
+        end
+
+        def finder(project)
+          Gitlab::Template::Finders::RepoTemplateFinder.new(project, self.base_dir, self.extension, self.categories)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb
index 3a69027368f..9858d2e7d83 100644
--- a/lib/gitlab/user_access.rb
+++ b/lib/gitlab/user_access.rb
@@ -30,7 +30,9 @@ module Gitlab
       return false unless user
 
       if project.protected_branch?(ref)
-        access_levels = project.protected_branches.matching(ref).map(&:push_access_level)
+        return true if project.empty_repo? && project.user_can_push_to_empty_repo?(user)
+
+        access_levels = project.protected_branches.matching(ref).map(&:push_access_levels).flatten
         access_levels.any? { |access_level| access_level.check_access(user) }
       else
         user.can?(:push_code, project)
@@ -41,7 +43,7 @@ module Gitlab
       return false unless user
 
       if project.protected_branch?(ref)
-        access_levels = project.protected_branches.matching(ref).map(&:merge_access_level)
+        access_levels = project.protected_branches.matching(ref).map(&:merge_access_levels).flatten
         access_levels.any? { |access_level| access_level.check_access(user) }
       else
         user.can?(:push_code, project)
diff --git a/lib/rouge/formatters/html_gitlab.rb b/lib/rouge/formatters/html_gitlab.rb
index f818dc78d34..4edfd015074 100644
--- a/lib/rouge/formatters/html_gitlab.rb
+++ b/lib/rouge/formatters/html_gitlab.rb
@@ -18,7 +18,7 @@ module Rouge
           is_first = false
 
           yield %(<span id="LC#{@line_number}" class="line">)
-          line.each { |token, value| yield span(token, value) }
+          line.each { |token, value| yield span(token, value.chomp) }
           yield %(</span>)
 
           @line_number += 1
diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab
index 4a4892a2e07..d521de28e8a 100644
--- a/lib/support/nginx/gitlab
+++ b/lib/support/nginx/gitlab
@@ -49,12 +49,7 @@ server {
 
     proxy_http_version 1.1;
 
-    ## By overwriting Host and clearing X-Forwarded-Host we ensure that
-    ## internal HTTP redirects generated by GitLab always send users to
-    ## YOUR_SERVER_FQDN.
-    proxy_set_header    Host                YOUR_SERVER_FQDN;
-    proxy_set_header    X-Forwarded-Host    "";
-
+    proxy_set_header    Host                $http_host;
     proxy_set_header    X-Real-IP           $remote_addr;
     proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
     proxy_set_header    X-Forwarded-Proto   $scheme;
diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl
index 0b93d7f292f..bf014b56cf6 100644
--- a/lib/support/nginx/gitlab-ssl
+++ b/lib/support/nginx/gitlab-ssl
@@ -93,12 +93,7 @@ server {
 
     proxy_http_version 1.1;
 
-    ## By overwriting Host and clearing X-Forwarded-Host we ensure that
-    ## internal HTTP redirects generated by GitLab always send users to
-    ## YOUR_SERVER_FQDN.
-    proxy_set_header    Host                YOUR_SERVER_FQDN;
-    proxy_set_header    X-Forwarded-Host    "";
-
+    proxy_set_header    Host                $http_host;
     proxy_set_header    X-Real-IP           $remote_addr;
     proxy_set_header    X-Forwarded-Ssl     on;
     proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
diff --git a/lib/tasks/downtime_check.rake b/lib/tasks/downtime_check.rake
index 30a2e9be5ce..afe5d42910c 100644
--- a/lib/tasks/downtime_check.rake
+++ b/lib/tasks/downtime_check.rake
@@ -1,26 +1,12 @@
 desc 'Checks if migrations in a branch require downtime'
 task downtime_check: :environment do
-  # First we'll want to make sure we're comparing with the right upstream
-  # repository/branch.
-  current_branch = `git rev-parse --abbrev-ref HEAD`.strip
-
-  # Either the developer ran this task directly on the master branch, or they're
-  # making changes directly on the master branch.
-  if current_branch == 'master'
-    if defined?(Gitlab::License)
-      repo = 'gitlab-ee'
-    else
-      repo = 'gitlab-ce'
-    end
-
-    `git fetch https://gitlab.com/gitlab-org/#{repo}.git --depth 1`
-
-    compare_with = 'FETCH_HEAD'
-  # The developer is working on a different branch, in this case we can just
-  # compare with the master branch.
+  if defined?(Gitlab::License)
+    repo = 'gitlab-ee'
   else
-    compare_with = 'master'
+    repo = 'gitlab-ce'
   end
 
-  Rake::Task['gitlab:db:downtime_check'].invoke(compare_with)
+  `git fetch https://gitlab.com/gitlab-org/#{repo}.git --depth 1`
+
+  Rake::Task['gitlab:db:downtime_check'].invoke('FETCH_HEAD')
 end
diff --git a/lib/tasks/gitlab/bulk_add_permission.rake b/lib/tasks/gitlab/bulk_add_permission.rake
index 5dbf7d61e06..83dd870fa31 100644
--- a/lib/tasks/gitlab/bulk_add_permission.rake
+++ b/lib/tasks/gitlab/bulk_add_permission.rake
@@ -4,13 +4,13 @@ namespace :gitlab do
     task all_users_to_all_projects: :environment  do |t, args|
       user_ids = User.where(admin: false).pluck(:id)
       admin_ids = User.where(admin: true).pluck(:id)
-      projects_ids = Project.pluck(:id)
+      project_ids = Project.pluck(:id)
 
-      puts "Importing #{user_ids.size} users into #{projects_ids.size} projects"
-      ProjectMember.add_users_into_projects(projects_ids, user_ids, ProjectMember::DEVELOPER)
+      puts "Importing #{user_ids.size} users into #{project_ids.size} projects"
+      ProjectMember.add_users_to_projects(project_ids, user_ids, ProjectMember::DEVELOPER)
 
-      puts "Importing #{admin_ids.size} admins into #{projects_ids.size} projects"
-      ProjectMember.add_users_into_projects(projects_ids, admin_ids, ProjectMember::MASTER)
+      puts "Importing #{admin_ids.size} admins into #{project_ids.size} projects"
+      ProjectMember.add_users_to_projects(project_ids, admin_ids, ProjectMember::MASTER)
     end
 
     desc "GitLab | Add a specific user to all projects (as a developer)"
@@ -18,7 +18,7 @@ namespace :gitlab do
       user = User.find_by(email: args.email)
       project_ids = Project.pluck(:id)
       puts "Importing #{user.email} users into #{project_ids.size} projects"
-      ProjectMember.add_users_into_projects(project_ids, Array.wrap(user.id), ProjectMember::DEVELOPER)
+      ProjectMember.add_users_to_projects(project_ids, Array.wrap(user.id), ProjectMember::DEVELOPER)
     end
 
     desc "GitLab | Add all users to all groups (admin users are added as owners)"
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index 60f4636e737..5f4a6bbfa35 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -46,7 +46,7 @@ namespace :gitlab do
       }
 
       correct_options = options.map do |name, value|
-        run(%W(#{Gitlab.config.git.bin_path} config --global --get #{name})).try(:squish) == value
+        run_command(%W(#{Gitlab.config.git.bin_path} config --global --get #{name})).try(:squish) == value
       end
 
       if correct_options.all?
@@ -64,7 +64,7 @@ namespace :gitlab do
           for_more_information(
             see_installation_guide_section "GitLab"
           )
-       end
+        end
       end
     end
 
@@ -73,7 +73,7 @@ namespace :gitlab do
 
       database_config_file = Rails.root.join("config", "database.yml")
 
-      if File.exists?(database_config_file)
+      if File.exist?(database_config_file)
         puts "yes".color(:green)
       else
         puts "no".color(:red)
@@ -94,7 +94,7 @@ namespace :gitlab do
 
       gitlab_config_file = Rails.root.join("config", "gitlab.yml")
 
-      if File.exists?(gitlab_config_file)
+      if File.exist?(gitlab_config_file)
         puts "yes".color(:green)
       else
         puts "no".color(:red)
@@ -113,7 +113,7 @@ namespace :gitlab do
       print "GitLab config outdated? ... "
 
       gitlab_config_file = Rails.root.join("config", "gitlab.yml")
-      unless File.exists?(gitlab_config_file)
+      unless File.exist?(gitlab_config_file)
         puts "can't check because of previous errors".color(:magenta)
       end
 
@@ -144,7 +144,7 @@ namespace :gitlab do
 
       script_path = "/etc/init.d/gitlab"
 
-      if File.exists?(script_path)
+      if File.exist?(script_path)
         puts "yes".color(:green)
       else
         puts "no".color(:red)
@@ -169,7 +169,7 @@ namespace :gitlab do
       recipe_path = Rails.root.join("lib/support/init.d/", "gitlab")
       script_path = "/etc/init.d/gitlab"
 
-      unless File.exists?(script_path)
+      unless File.exist?(script_path)
         puts "can't check because of previous errors".color(:magenta)
         return
       end
@@ -316,7 +316,7 @@ namespace :gitlab do
       min_redis_version = "2.8.0"
       print "Redis version >= #{min_redis_version}? ... "
 
-      redis_version = run(%W(redis-cli --version))
+      redis_version = run_command(%W(redis-cli --version))
       redis_version = redis_version.try(:match, /redis-cli (\d+\.\d+\.\d+)/)
       if redis_version &&
           (Gem::Version.new(redis_version[1]) > Gem::Version.new(min_redis_version))
@@ -361,7 +361,7 @@ namespace :gitlab do
       Gitlab.config.repositories.storages.each do |name, repo_base_path|
         print "#{name}... "
 
-        if File.exists?(repo_base_path)
+        if File.exist?(repo_base_path)
           puts "yes".color(:green)
         else
           puts "no".color(:red)
@@ -385,7 +385,7 @@ namespace :gitlab do
       Gitlab.config.repositories.storages.each do |name, repo_base_path|
         print "#{name}... "
 
-        unless File.exists?(repo_base_path)
+        unless File.exist?(repo_base_path)
           puts "can't check because of previous errors".color(:magenta)
           return
         end
@@ -408,7 +408,7 @@ namespace :gitlab do
       Gitlab.config.repositories.storages.each do |name, repo_base_path|
         print "#{name}... "
 
-        unless File.exists?(repo_base_path)
+        unless File.exist?(repo_base_path)
           puts "can't check because of previous errors".color(:magenta)
           return
         end
@@ -438,7 +438,7 @@ namespace :gitlab do
       Gitlab.config.repositories.storages.each do |name, repo_base_path|
         print "#{name}... "
 
-        unless File.exists?(repo_base_path)
+        unless File.exist?(repo_base_path)
           puts "can't check because of previous errors".color(:magenta)
           return
         end
@@ -893,7 +893,7 @@ namespace :gitlab do
 
   def check_ruby_version
     required_version = Gitlab::VersionInfo.new(2, 1, 0)
-    current_version = Gitlab::VersionInfo.parse(run(%W(ruby --version)))
+    current_version = Gitlab::VersionInfo.parse(run_command(%W(ruby --version)))
 
     print "Ruby version >= #{required_version} ? ... "
 
@@ -910,7 +910,7 @@ namespace :gitlab do
 
   def check_git_version
     required_version = Gitlab::VersionInfo.new(2, 7, 3)
-    current_version = Gitlab::VersionInfo.parse(run(%W(#{Gitlab.config.git.bin_path} --version)))
+    current_version = Gitlab::VersionInfo.parse(run_command(%W(#{Gitlab.config.git.bin_path} --version)))
 
     puts "Your git bin path is \"#{Gitlab.config.git.bin_path}\""
     print "Git version >= #{required_version} ? ... "
diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake
index fe43d40e6d2..dffea8ed155 100644
--- a/lib/tasks/gitlab/info.rake
+++ b/lib/tasks/gitlab/info.rake
@@ -8,7 +8,7 @@ namespace :gitlab do
       # check Ruby version
       ruby_version = run_and_match(%W(ruby --version), /[\d\.p]+/).try(:to_s)
       # check Gem version
-      gem_version = run(%W(gem --version))
+      gem_version = run_command(%W(gem --version))
       # check Bundler version
       bunder_version = run_and_match(%W(bundle --version), /[\d\.]+/).try(:to_s)
       # check Bundler version
@@ -17,7 +17,7 @@ namespace :gitlab do
       puts ""
       puts "System information".color(:yellow)
       puts "System:\t\t#{os_name || "unknown".color(:red)}"
-      puts "Current User:\t#{run(%W(whoami))}"
+      puts "Current User:\t#{run_command(%W(whoami))}"
       puts "Using RVM:\t#{rvm_version.present? ? "yes".color(:green) : "no"}"
       puts "RVM Version:\t#{rvm_version}" if rvm_version.present?
       puts "Ruby Version:\t#{ruby_version || "unknown".color(:red)}"
diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake
index c85ebdf8619..bb7eb852f1b 100644
--- a/lib/tasks/gitlab/shell.rake
+++ b/lib/tasks/gitlab/shell.rake
@@ -5,7 +5,8 @@ namespace :gitlab do
       warn_user_is_not_gitlab
 
       default_version = Gitlab::Shell.version_required
-      args.with_defaults(tag: 'v' + default_version, repo: "https://gitlab.com/gitlab-org/gitlab-shell.git")
+      default_version_tag = 'v' + default_version
+      args.with_defaults(tag: default_version_tag, repo: "https://gitlab.com/gitlab-org/gitlab-shell.git")
 
       user = Gitlab.config.gitlab.user
       home_dir = Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home
@@ -15,7 +16,12 @@ namespace :gitlab do
       target_dir = Gitlab.config.gitlab_shell.path
 
       # Clone if needed
-      unless File.directory?(target_dir)
+      if File.directory?(target_dir)
+        Dir.chdir(target_dir) do
+          system(*%W(Gitlab.config.git.bin_path} fetch --tags --quiet))
+          system(*%W(Gitlab.config.git.bin_path} checkout --quiet #{default_version_tag}))
+        end
+      else
         system(*%W(#{Gitlab.config.git.bin_path} clone -- #{args.repo} #{target_dir}))
       end
 
@@ -84,7 +90,7 @@ namespace :gitlab do
     task build_missing_projects: :environment do
       Project.find_each(batch_size: 1000) do |project|
         path_to_repo = project.repository.path_to_repo
-        if File.exists?(path_to_repo)
+        if File.exist?(path_to_repo)
           print '-'
         else
           if Gitlab::Shell.new.add_repository(project.repository_storage_path,
diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake
index ab96b1d3593..74be413423a 100644
--- a/lib/tasks/gitlab/task_helpers.rake
+++ b/lib/tasks/gitlab/task_helpers.rake
@@ -23,7 +23,7 @@ namespace :gitlab do
   # It will primarily use lsb_relase to determine the OS.
   # It has fallbacks to Debian, SuSE, OS X and systems running systemd.
   def os_name
-    os_name = run(%W(lsb_release -irs))
+    os_name = run_command(%W(lsb_release -irs))
     os_name ||= if File.readable?('/etc/system-release')
                   File.read('/etc/system-release')
                 end
@@ -34,7 +34,7 @@ namespace :gitlab do
     os_name ||= if File.readable?('/etc/SuSE-release')
                   File.read('/etc/SuSE-release')
                 end
-    os_name ||= if os_x_version = run(%W(sw_vers -productVersion))
+    os_name ||= if os_x_version = run_command(%W(sw_vers -productVersion))
                   "Mac OS X #{os_x_version}"
                 end
     os_name ||= if File.readable?('/etc/os-release')
@@ -62,10 +62,10 @@ namespace :gitlab do
   # Returns nil if nothing matched
   # Returns the MatchData if the pattern matched
   #
-  # see also #run
+  # see also #run_command
   # see also String#match
   def run_and_match(command, regexp)
-    run(command).try(:match, regexp)
+    run_command(command).try(:match, regexp)
   end
 
   # Runs the given command
@@ -74,7 +74,7 @@ namespace :gitlab do
   # Returns the output of the command otherwise
   #
   # see also #run_and_match
-  def run(command)
+  def run_command(command)
     output, _ = Gitlab::Popen.popen(command)
     output
   rescue Errno::ENOENT
@@ -82,7 +82,7 @@ namespace :gitlab do
   end
 
   def uid_for(user_name)
-    run(%W(id -u #{user_name})).chomp.to_i
+    run_command(%W(id -u #{user_name})).chomp.to_i
   end
 
   def gid_for(group_name)
@@ -96,7 +96,7 @@ namespace :gitlab do
   def warn_user_is_not_gitlab
     unless @warned_user_not_gitlab
       gitlab_user = Gitlab.config.gitlab.user
-      current_user = run(%W(whoami)).chomp
+      current_user = run_command(%W(whoami)).chomp
       unless current_user == gitlab_user
         puts " Warning ".color(:black).background(:yellow)
         puts "  You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing."
diff --git a/lib/tasks/gitlab/web_hook.rake b/lib/tasks/gitlab/web_hook.rake
index f467cc0ee29..49530e7a372 100644
--- a/lib/tasks/gitlab/web_hook.rake
+++ b/lib/tasks/gitlab/web_hook.rake
@@ -26,10 +26,10 @@ namespace :gitlab do
       namespace_path = ENV['NAMESPACE']
 
       projects = find_projects(namespace_path)
-      projects_ids = projects.pluck(:id)
+      project_ids = projects.pluck(:id)
 
       puts "Removing webhooks with the url '#{web_hook_url}' ... "
-      count = WebHook.where(url: web_hook_url, project_id: projects_ids, type: 'ProjectHook').delete_all
+      count = WebHook.where(url: web_hook_url, project_id: project_ids, type: 'ProjectHook').delete_all
       puts "#{count} webhooks were removed."
     end
 
diff --git a/lib/tasks/spinach.rake b/lib/tasks/spinach.rake
index da255f5464b..8dbfa7751dc 100644
--- a/lib/tasks/spinach.rake
+++ b/lib/tasks/spinach.rake
@@ -34,21 +34,19 @@ task :spinach do
   run_spinach_tests(nil)
 end
 
-def run_command(cmd)
+def run_system_command(cmd)
   system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd)
 end
 
 def run_spinach_command(args)
-  run_command(%w(spinach -r rerun) + args)
+  run_system_command(%w(spinach -r rerun) + args)
 end
 
 def run_spinach_tests(tags)
-  #run_command(%w(rake gitlab:setup)) or raise('gitlab:setup failed!')
-
   success = run_spinach_command(%W(--tags #{tags}))
   3.times do |_|
     break if success
-    break unless File.exists?('tmp/spinach-rerun.txt')
+    break unless File.exist?('tmp/spinach-rerun.txt')
 
     tests = File.foreach('tmp/spinach-rerun.txt').map(&:chomp)
     puts ''