Merge remote-tracking branch 'upstream/master' into feature/1376-allow-write-access-deploy-keys

* upstream/master: (538 commits)
  Reject blank environment vcariables in Gitlab::Git::RevList
  Add online terminal documentation
  Add changelog entry
  Add terminal UI and controller actions
  Fix specs
  Even out padding on plus button in breadcrumb menu
  Update font size of detail page header to 14px
  Update CHANGELOG.md for 8.13.10
  Update CHANGELOG.md for 8.14.5
  Fix Route#rename_children behavior
  Remove inline-block styling from status
  Add terminals to the Kubernetes deployment service
  Add a ReactiveCaching concern for use in the KubernetesService
  Add xterm.js 2.1.0 and a wrapper class to the asset pipeline
  Remove unnecessary hidden svg elements for icons.
  Fix consistent typo in environment.js
  Use a block to insert extra check for authenticate_build!
  Align milestone column header with count number
  Add Wiki import to BB importer
  Make CI badge hitboxes better match container
  ...

98 files changed, 2756 insertions, 646 deletions

diff --git a/lib/api/api.rb b/lib/api/api.rb
index cec2702e44d..9d5adffd8f4 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -3,6 +3,8 @@ module API
     include APIGuard
     version 'v3', using: :path
 
+    before { allow_access_with_scope :api }
+
     rescue_from Gitlab::Access::AccessDeniedError do
       rack_response({ 'message' => '403 Forbidden' }.to_json, 403)
     end
diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb
index 8cc7a26f1fa..df6db140d0e 100644
--- a/lib/api/api_guard.rb
+++ b/lib/api/api_guard.rb
@@ -6,6 +6,9 @@ module API
   module APIGuard
     extend ActiveSupport::Concern
 
+    PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN"
+    PRIVATE_TOKEN_PARAM = :private_token
+
     included do |base|
       # OAuth2 Resource Server Authentication
       use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request|
@@ -44,27 +47,60 @@ module API
         access_token = find_access_token
         return nil unless access_token
 
-        case validate_access_token(access_token, scopes)
-        when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE
+        case AccessTokenValidationService.new(access_token).validate(scopes: scopes)
+        when AccessTokenValidationService::INSUFFICIENT_SCOPE
           raise InsufficientScopeError.new(scopes)
 
-        when Oauth2::AccessTokenValidationService::EXPIRED
+        when AccessTokenValidationService::EXPIRED
           raise ExpiredError
 
-        when Oauth2::AccessTokenValidationService::REVOKED
+        when AccessTokenValidationService::REVOKED
           raise RevokedError
 
-        when Oauth2::AccessTokenValidationService::VALID
+        when AccessTokenValidationService::VALID
           @current_user = User.find(access_token.resource_owner_id)
         end
       end
 
+      def find_user_by_private_token(scopes: [])
+        token_string = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s
+
+        return nil unless token_string.present?
+
+        find_user_by_authentication_token(token_string) || find_user_by_personal_access_token(token_string, scopes)
+      end
+
       def current_user
         @current_user
       end
 
+      # Set the authorization scope(s) allowed for the current request.
+      #
+      # Note: A call to this method adds to any previous scopes in place. This is done because
+      # `Grape` callbacks run from the outside-in: the top-level callback (API::API) runs first, then
+      # the next-level callback (API::API::Users, for example) runs. All these scopes are valid for the
+      # given endpoint (GET `/api/users` is accessible by the `api` and `read_user` scopes), and so they
+      # need to be stored.
+      def allow_access_with_scope(*scopes)
+        @scopes ||= []
+        @scopes.concat(scopes.map(&:to_s))
+      end
+
       private
 
+      def find_user_by_authentication_token(token_string)
+        User.find_by_authentication_token(token_string)
+      end
+
+      def find_user_by_personal_access_token(token_string, scopes)
+        access_token = PersonalAccessToken.active.find_by_token(token_string)
+        return unless access_token
+
+        if AccessTokenValidationService.new(access_token).include_any_scope?(scopes)
+          User.find(access_token.user_id)
+        end
+      end
+
       def find_access_token
         @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods)
       end
@@ -72,10 +108,6 @@ module API
       def doorkeeper_request
         @doorkeeper_request ||= ActionDispatch::Request.new(env)
       end
-
-      def validate_access_token(access_token, scopes)
-        Oauth2::AccessTokenValidationService.validate(access_token, scopes: scopes)
-      end
     end
 
     module ClassMethods
diff --git a/lib/api/commits.rb b/lib/api/commits.rb
index 2670a2d413a..cf2489dbb67 100644
--- a/lib/api/commits.rb
+++ b/lib/api/commits.rb
@@ -1,7 +1,6 @@
 require 'mime/types'
 
 module API
-  # Projects commits API
   class Commits < Grape::API
     include PaginationParams
 
@@ -121,6 +120,41 @@ module API
         present paginate(notes), with: Entities::CommitNote
       end
 
+      desc 'Cherry pick commit into a branch' do
+        detail 'This feature was introduced in GitLab 8.15'
+        success Entities::RepoCommit
+      end
+      params do
+        requires :sha, type: String, desc: 'A commit sha to be cherry picked'
+        requires :branch, type: String, desc: 'The name of the branch'
+      end
+      post ':id/repository/commits/:sha/cherry_pick' do
+        authorize! :push_code, user_project
+
+        commit = user_project.commit(params[:sha])
+        not_found!('Commit') unless commit
+
+        branch = user_project.repository.find_branch(params[:branch])
+        not_found!('Branch') unless branch
+
+        commit_params = {
+          commit: commit,
+          create_merge_request: false,
+          source_project: user_project,
+          source_branch: commit.cherry_pick_branch_name,
+          target_branch: params[:branch]
+        }
+
+        result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute
+
+        if result[:status] == :success
+          branch = user_project.repository.find_branch(params[:branch])
+          present user_project.repository.commit(branch.dereferenced_target), with: Entities::RepoCommit
+        else
+          render_api_error!(result[:message], 400)
+        end
+      end
+
       desc 'Post comment to commit' do
         success Entities::CommitNote
       end
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index bb8d740532d..cd5c4c4b3ce 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -629,7 +629,7 @@ module API
     end
 
     class EnvironmentBasic < Grape::Entity
-      expose :id, :name, :external_url
+      expose :id, :name, :slug, :external_url
     end
 
     class Environment < EnvironmentBasic
diff --git a/lib/api/environments.rb b/lib/api/environments.rb
index 80bbd9bb6e4..1a7e68f0528 100644
--- a/lib/api/environments.rb
+++ b/lib/api/environments.rb
@@ -1,6 +1,7 @@
 module API
   # Environments RESTfull API endpoints
   class Environments < Grape::API
+    include ::API::Helpers::CustomValidators
     include PaginationParams
 
     before { authenticate! }
@@ -29,6 +30,7 @@ module API
       params do
         requires :name,           type: String,   desc: 'The name of the environment to be created'
         optional :external_url,   type: String,   desc: 'URL on which this deployment is viewable'
+        optional :slug, absence: { message: "is automatically generated and cannot be changed" }
       end
       post ':id/environments' do
         authorize! :create_environment, user_project
@@ -50,6 +52,7 @@ module API
         requires :environment_id, type: Integer,  desc: 'The environment ID'
         optional :name,           type: String,   desc: 'The new environment name'
         optional :external_url,   type: String,   desc: 'The new URL on which this deployment is viewable'
+        optional :slug, absence: { message: "is automatically generated and cannot be changed" }
       end
       put ':id/environments/:environment_id' do
         authorize! :update_environment, user_project
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index 105d3ee342e..9b9d3df7435 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -125,13 +125,16 @@ module API
                             default: 'created_at', desc: 'Return projects ordered by field'
         optional :sort, type: String, values: %w[asc desc], default: 'desc',
                         desc: 'Return projects sorted in ascending and descending order'
+        optional :simple, type: Boolean, default: false,
+                          desc: 'Return only the ID, URL, name, and path of each project'
         use :pagination
       end
       get ":id/projects" do
         group = find_group!(params[:id])
         projects = GroupProjectsFinder.new(group).execute(current_user)
         projects = filter_projects(projects)
-        present paginate(projects), with: Entities::Project, user: current_user
+        entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project
+        present paginate(projects), with: entity, user: current_user
       end
 
       desc 'Transfer a project to the group namespace. Available only for admin.' do
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 8b0f8deadfa..4be659fc20b 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -2,72 +2,26 @@ module API
   module Helpers
     include Gitlab::Utils
 
-    PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN"
-    PRIVATE_TOKEN_PARAM = :private_token
     SUDO_HEADER = "HTTP_SUDO"
     SUDO_PARAM = :sudo
 
-    def private_token
-      params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]
-    end
-
-    def warden
-      env['warden']
-    end
-
-    # Check the Rails session for valid authentication details
-    #
-    # Until CSRF protection is added to the API, disallow this method for
-    # state-changing endpoints
-    def find_user_from_warden
-      warden.try(:authenticate) if %w[GET HEAD].include?(env['REQUEST_METHOD'])
-    end
-
     def declared_params(options = {})
       options = { include_parent_namespaces: false }.merge(options)
       declared(params, options).to_h.symbolize_keys
     end
 
-    def find_user_by_private_token
-      token = private_token
-      return nil unless token.present?
-
-      User.find_by_authentication_token(token) || User.find_by_personal_access_token(token)
-    end
-
     def current_user
-      @current_user ||= find_user_by_private_token
-      @current_user ||= doorkeeper_guard
-      @current_user ||= find_user_from_warden
+      return @current_user if defined?(@current_user)
 
-      unless @current_user && Gitlab::UserAccess.new(@current_user).allowed?
-        return nil
-      end
-
-      identifier = sudo_identifier
+      @current_user = initial_current_user
 
-      if identifier
-        # We check for private_token because we cannot allow PAT to be used
-        forbidden!('Must be admin to use sudo') unless @current_user.is_admin?
-        forbidden!('Private token must be specified in order to use sudo') unless private_token_used?
-
-        @impersonator = @current_user
-        @current_user = User.by_username_or_id(identifier)
-        not_found!("No user id or username for: #{identifier}") if @current_user.nil?
-      end
+      sudo!
 
       @current_user
     end
 
-    def sudo_identifier
-      identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER]
-
-      # Regex for integers
-      if !!(identifier =~ /\A[0-9]+\z/)
-        identifier.to_i
-      else
-        identifier
-      end
+    def sudo?
+      initial_current_user != current_user
     end
 
     def user_project
@@ -78,6 +32,14 @@ module API
       @available_labels ||= LabelsFinder.new(current_user, project_id: user_project.id).execute
     end
 
+    def find_user(id)
+      if id =~ /^\d+$/
+        User.find_by(id: id)
+      else
+        User.find_by(username: id)
+      end
+    end
+
     def find_project(id)
       if id =~ /^\d+$/
         Project.find_by(id: id)
@@ -96,17 +58,6 @@ module API
       end
     end
 
-    def project_service(project = user_project)
-      @project_service ||= project.find_or_initialize_service(params[:service_slug].underscore)
-      @project_service || not_found!("Service")
-    end
-
-    def service_attributes
-      @service_attributes ||= project_service.fields.inject([]) do |arr, hash|
-        arr << hash[:name].to_sym
-      end
-    end
-
     def find_group(id)
       if id =~ /^\d+$/
         Group.find_by(id: id)
@@ -354,6 +305,62 @@ module API
 
     private
 
+    def private_token
+      params[APIGuard::PRIVATE_TOKEN_PARAM] || env[APIGuard::PRIVATE_TOKEN_HEADER]
+    end
+
+    def warden
+      env['warden']
+    end
+
+    # Check the Rails session for valid authentication details
+    #
+    # Until CSRF protection is added to the API, disallow this method for
+    # state-changing endpoints
+    def find_user_from_warden
+      warden.try(:authenticate) if %w[GET HEAD].include?(env['REQUEST_METHOD'])
+    end
+
+    def initial_current_user
+      return @initial_current_user if defined?(@initial_current_user)
+
+      @initial_current_user ||= find_user_by_private_token(scopes: @scopes)
+      @initial_current_user ||= doorkeeper_guard(scopes: @scopes)
+      @initial_current_user ||= find_user_from_warden
+
+      unless @initial_current_user && Gitlab::UserAccess.new(@initial_current_user).allowed?
+        @initial_current_user = nil
+      end
+
+      @initial_current_user
+    end
+
+    def sudo!
+      return unless sudo_identifier
+      return unless initial_current_user
+
+      unless initial_current_user.is_admin?
+        forbidden!('Must be admin to use sudo')
+      end
+
+      # Only private tokens should be used for the SUDO feature
+      unless private_token == initial_current_user.private_token
+        forbidden!('Private token must be specified in order to use sudo')
+      end
+
+      sudoed_user = find_user(sudo_identifier)
+
+      if sudoed_user
+        @current_user = sudoed_user
+      else
+        not_found!("No user id or username for: #{sudo_identifier}")
+      end
+    end
+
+    def sudo_identifier
+      @sudo_identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER]
+    end
+
     def add_pagination_headers(paginated_data)
       header 'X-Total',       paginated_data.total_count.to_s
       header 'X-Total-Pages', paginated_data.total_pages.to_s
@@ -386,10 +393,6 @@ module API
       links.join(', ')
     end
 
-    def private_token_used?
-      private_token == @current_user.private_token
-    end
-
     def secret_token
       Gitlab::Shell.secret_token
     end
diff --git a/lib/api/helpers/custom_validators.rb b/lib/api/helpers/custom_validators.rb
new file mode 100644
index 00000000000..0a8f3073a50
--- /dev/null
+++ b/lib/api/helpers/custom_validators.rb
@@ -0,0 +1,14 @@
+module API
+  module Helpers
+    module CustomValidators
+      class Absence < Grape::Validations::Base
+        def validate_param!(attr_name, params)
+          return if params.respond_to?(:key?) && !params.key?(attr_name)
+          raise Grape::Exceptions::Validation, params: [@scope.full_name(attr_name)], message: message(:absence)
+        end
+      end
+    end
+  end
+end
+
+Grape::Validations.register_validator(:absence, ::API::Helpers::CustomValidators::Absence)
diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb
index eb223c1101d..e8975eb57e0 100644
--- a/lib/api/helpers/internal_helpers.rb
+++ b/lib/api/helpers/internal_helpers.rb
@@ -52,6 +52,14 @@ module API
           :push_code
         ]
       end
+
+      def parse_allowed_environment_variables
+        return if params[:env].blank?
+
+        JSON.parse(params[:env])
+
+      rescue JSON::ParserError
+      end
     end
   end
 end
diff --git a/lib/api/internal.rb b/lib/api/internal.rb
index 7087ce11401..db2d18f935d 100644
--- a/lib/api/internal.rb
+++ b/lib/api/internal.rb
@@ -32,7 +32,11 @@ module API
           if wiki?
             Gitlab::GitAccessWiki.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities)
           else
-            Gitlab::GitAccess.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities)
+            Gitlab::GitAccess.new(actor,
+                                  project,
+                                  protocol,
+                                  authentication_abilities: ssh_authentication_abilities,
+                                  env: parse_allowed_environment_variables)
           end
 
         access_status = access.check(params[:action], params[:changes])
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index 55bdbc6a47c..5d1fe22f2df 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -143,8 +143,8 @@ module API
           success Entities::MergeRequest
         end
         params do
-          optional :title, type: String, desc: 'The title of the merge request'
-          optional :target_branch, type: String, desc: 'The target branch'
+          optional :title, type: String, allow_blank: false, desc: 'The title of the merge request'
+          optional :target_branch, type: String, allow_blank: false, desc: 'The target branch'
           optional :state_event, type: String, values: %w[close reopen merge],
                                  desc: 'Status of the merge request'
           use :optional_params
diff --git a/lib/api/services.rb b/lib/api/services.rb
index bc427705777..aa97f6af0b2 100644
--- a/lib/api/services.rb
+++ b/lib/api/services.rb
@@ -1,84 +1,645 @@
 module API
-  # Projects API
   class Services < Grape::API
+    services = {
+      'asana' => [
+        {
+          required: true,
+          name: :api_key,
+          type: String,
+          desc: 'User API token'
+        },
+        {
+          required: false,
+          name: :restrict_to_branch,
+          type: String,
+          desc: 'Comma-separated list of branches which will be automatically inspected. Leave blank to include all branches'
+        }
+      ],
+      'assembla' => [
+        {
+          required: true,
+          name: :token,
+          type: String,
+          desc: 'The authentication token'
+        },
+        {
+          required: false,
+          name: :subdomain,
+          type: String,
+          desc: 'Subdomain setting'
+        }
+      ],
+      'bamboo' => [
+        {
+          required: true,
+          name: :bamboo_url,
+          type: String,
+          desc: 'Bamboo root URL like https://bamboo.example.com'
+        },
+        {
+          required: true,
+          name: :build_key,
+          type: String,
+          desc: 'Bamboo build plan key like'
+        },
+        {
+          required: true,
+          name: :username,
+          type: String,
+          desc: 'A user with API access, if applicable'
+        },
+        {
+          required: true,
+          name: :password,
+          type: String,
+          desc: 'Passord of the user'
+        }
+      ],
+      'bugzilla' => [
+        {
+          required: true,
+          name: :new_issue_url,
+          type: String,
+          desc: 'New issue URL'
+        },
+        {
+          required: true,
+          name: :issues_url,
+          type: String,
+          desc: 'Issues URL'
+        },
+        {
+          required: true,
+          name: :project_url,
+          type: String,
+          desc: 'Project URL'
+        },
+        {
+          required: false,
+          name: :description,
+          type: String,
+          desc: 'Description'
+        },
+        {
+          required: false,
+          name: :title,
+          type: String,
+          desc: 'Title'
+        }
+      ],
+      'buildkite' => [
+        {
+          required: true,
+          name: :token,
+          type: String,
+          desc: 'Buildkite project GitLab token'
+        },
+        {
+          required: true,
+          name: :project_url,
+          type: String,
+          desc: 'The buildkite project URL'
+        },
+        {
+          required: false,
+          name: :enable_ssl_verification,
+          type: Boolean,
+          desc: 'Enable SSL verification for communication'
+        }
+      ],
+      'builds-email' => [
+        {
+          required: true,
+          name: :recipients,
+          type: String,
+          desc: 'Comma-separated list of recipient email addresses'
+        },
+        {
+          required: false,
+          name: :add_pusher,
+          type: Boolean,
+          desc: 'Add pusher to recipients list'
+        },
+        {
+          required: false,
+          name: :notify_only_broken_builds,
+          type: Boolean,
+          desc: 'Notify only broken builds'
+        }
+      ],
+      'campfire' => [
+        {
+          required: true,
+          name: :token,
+          type: String,
+          desc: 'Campfire token'
+        },
+        {
+          required: false,
+          name: :subdomain,
+          type: String,
+          desc: 'Campfire subdomain'
+        },
+        {
+          required: false,
+          name: :room,
+          type: String,
+          desc: 'Campfire room'
+        },
+      ],
+      'custom-issue-tracker' => [
+        {
+          required: true,
+          name: :new_issue_url,
+          type: String,
+          desc: 'New issue URL'
+        },
+        {
+          required: true,
+          name: :issues_url,
+          type: String,
+          desc: 'Issues URL'
+        },
+        {
+          required: true,
+          name: :project_url,
+          type: String,
+          desc: 'Project URL'
+        },
+        {
+          required: false,
+          name: :description,
+          type: String,
+          desc: 'Description'
+        },
+        {
+          required: false,
+          name: :title,
+          type: String,
+          desc: 'Title'
+        }
+      ],
+      'drone-ci' => [
+        {
+          required: true,
+          name: :token,
+          type: String,
+          desc: 'Drone CI token'
+        },
+        {
+          required: true,
+          name: :drone_url,
+          type: String,
+          desc: 'Drone CI URL'
+        },
+        {
+          required: false,
+          name: :enable_ssl_verification,
+          type: Boolean,
+          desc: 'Enable SSL verification for communication'
+        }
+      ],
+      'emails-on-push' => [
+        {
+          required: true,
+          name: :recipients,
+          type: String,
+          desc: 'Comma-separated list of recipient email addresses'
+        },
+        {
+          required: false,
+          name: :disable_diffs,
+          type: Boolean,
+          desc: 'Disable code diffs'
+        },
+        {
+          required: false,
+          name: :send_from_committer_email,
+          type: Boolean,
+          desc: 'Send from committer'
+        }
+      ],
+      'external-wiki' => [
+        {
+          required: true,
+          name: :external_wiki_url,
+          type: String,
+          desc: 'The URL of the external Wiki'
+        }
+      ],
+      'flowdock' => [
+        {
+          required: true,
+          name: :token,
+          type: String,
+          desc: 'Flowdock token'
+        }
+      ],
+      'gemnasium' => [
+        {
+          required: true,
+          name: :api_key,
+          type: String,
+          desc: 'Your personal API key on gemnasium.com'
+        },
+        {
+          required: true,
+          name: :token,
+          type: String,
+          desc: "The project's slug on gemnasium.com"
+        }
+      ],
+      'hipchat' => [
+        {
+          required: true,
+          name: :token,
+          type: String,
+          desc: 'The room token'
+        },
+        {
+          required: false,
+          name: :room,
+          type: String,
+          desc: 'The room name or ID'
+        },
+        {
+          required: false,
+          name: :color,
+          type: String,
+          desc: 'The room color'
+        },
+        {
+          required: false,
+          name: :notify,
+          type: Boolean,
+          desc: 'Enable notifications'
+        },
+        {
+          required: false,
+          name: :api_version,
+          type: String,
+          desc: 'Leave blank for default (v2)'
+        },
+        {
+          required: false,
+          name: :server,
+          type: String,
+          desc: 'Leave blank for default. https://hipchat.example.com'
+        }
+      ],
+      'irker' => [
+        {
+          required: true,
+          name: :recipients,
+          type: String,
+          desc: 'Recipients/channels separated by whitespaces'
+        },
+        {
+          required: false,
+          name: :default_irc_uri,
+          type: String,
+          desc: 'Default: irc://irc.network.net:6697'
+        },
+        {
+          required: false,
+          name: :server_host,
+          type: String,
+          desc: 'Server host. Default localhost'
+        },
+        {
+          required: false,
+          name: :server_port,
+          type: Integer,
+          desc: 'Server port. Default 6659'
+        },
+        {
+          required: false,
+          name: :colorize_messages,
+          type: Boolean,
+          desc: 'Colorize messages'
+        }
+      ],
+      'jira' => [
+        {
+          required: true,
+          name: :url,
+          type: String,
+          desc: 'The URL to the JIRA project which is being linked to this GitLab project, e.g., https://jira.example.com'
+        },
+        {
+          required: true,
+          name: :project_key,
+          type: String,
+          desc: 'The short identifier for your JIRA project, all uppercase, e.g., PROJ'
+        },
+        {
+          required: false,
+          name: :username,
+          type: String,
+          desc: 'The username of the user created to be used with GitLab/JIRA'
+        },
+        {
+          required: false,
+          name: :password,
+          type: String,
+          desc: 'The password of the user created to be used with GitLab/JIRA'
+        },
+        {
+          required: false,
+          name: :jira_issue_transition_id,
+          type: Integer,
+          desc: 'The ID of a transition that moves issues to a closed state. You can find this number under the JIRA workflow administration (**Administration > Issues > Workflows**) by selecting **View** under **Operations** of the desired workflow of your project. The ID of each state can be found inside the parenthesis of each transition name under the **Transitions (id)** column ([see screenshot][trans]). By default, this ID is set to `2`'
+        }
+      ],
+
+      'kubernetes' => [
+        {
+          required: true,
+          name: :namespace,
+          type: String,
+          desc: 'The Kubernetes namespace to use'
+        },
+        {
+          required: true,
+          name: :api_url,
+          type: String,
+          desc: 'The URL to the Kubernetes cluster API, e.g., https://kubernetes.example.com'
+        },
+        {
+          required: true,
+          name: :token,
+          type: String,
+          desc: 'The service token to authenticate against the Kubernetes cluster with'
+        },
+        {
+          required: false,
+          name: :ca_pem,
+          type: String,
+          desc: 'A custom certificate authority bundle to verify the Kubernetes cluster with (PEM format)'
+        },
+      ],
+      'mattermost-slash-commands' => [
+        {
+          required: true,
+          name: :token,
+          type: String,
+          desc: 'The Mattermost token'
+        }
+      ],
+      'slack-slash-commands' => [
+        {
+          required: true,
+          name: :token,
+          type: String,
+          desc: 'The Slack token'
+        }
+      ],
+      'pipelines-email' => [
+        {
+          required: true,
+          name: :recipients,
+          type: String,
+          desc: 'Comma-separated list of recipient email addresses'
+        },
+        {
+          required: false,
+          name: :notify_only_broken_builds,
+          type: Boolean,
+          desc: 'Notify only broken builds'
+        }
+      ],
+      'pivotaltracker' => [
+        {
+          required: true,
+          name: :token,
+          type: String,
+          desc: 'The Pivotaltracker token'
+        },
+        {
+          required: false,
+          name: :restrict_to_branch,
+          type: String,
+          desc: 'Comma-separated list of branches which will be automatically inspected. Leave blank to include all branches.'
+        }
+      ],
+      'pushover' => [
+        {
+          required: true,
+          name: :api_key,
+          type: String,
+          desc: 'The application key'
+        },
+        {
+          required: true,
+          name: :user_key,
+          type: String,
+          desc: 'The user key'
+        },
+        {
+          required: true,
+          name: :priority,
+          type: String,
+          desc: 'The priority'
+        },
+        {
+          required: true,
+          name: :device,
+          type: String,
+          desc: 'Leave blank for all active devices'
+        },
+        {
+          required: true,
+          name: :sound,
+          type: String,
+          desc: 'The sound of the notification'
+        }
+      ],
+      'redmine' => [
+        {
+          required: true,
+          name: :new_issue_url,
+          type: String,
+          desc: 'The new issue URL'
+        },
+        {
+          required: true,
+          name: :project_url,
+          type: String,
+          desc: 'The project URL'
+        },
+        {
+          required: true,
+          name: :issues_url,
+          type: String,
+          desc: 'The issues URL'
+        },
+        {
+          required: false,
+          name: :description,
+          type: String,
+          desc: 'The description of the tracker'
+        }
+      ],
+      'slack-notification' => [
+        {
+          required: true,
+          name: :webhook,
+          type: String,
+          desc: 'The Slack webhook. e.g. https://hooks.slack.com/services/...'
+        },
+        {
+          required: false,
+          name: :new_issue_url,
+          type: String,
+          desc: 'The user name'
+        },
+        {
+          required: false,
+          name: :channel,
+          type: String,
+          desc: 'The channel name'
+        }
+      ],
+      'mattermost-notification' => [
+        {
+          required: true,
+          name: :webhook,
+          type: String,
+          desc: 'The Mattermost webhook. e.g. http://mattermost_host/hooks/...'
+        }
+      ],
+      'teamcity' => [
+        {
+          required: true,
+          name: :teamcity_url,
+          type: String,
+          desc: 'TeamCity root URL like https://teamcity.example.com'
+        },
+        {
+          required: true,
+          name: :build_type,
+          type: String,
+          desc: 'Build configuration ID'
+        },
+        {
+          required: true,
+          name: :username,
+          type: String,
+          desc: 'A user with permissions to trigger a manual build'
+        },
+        {
+          required: true,
+          name: :password,
+          type: String,
+          desc: 'The password of the user'
+        }
+      ]
+    }.freeze
+
+    trigger_services = {
+      'mattermost-slash-commands' => [
+        {
+          name: :token,
+          type: String,
+          desc: 'The Mattermost token'
+        }
+      ]
+    }.freeze
+
     resource :projects do
       before { authenticate! }
       before { authorize_admin_project }
 
-      # Set <service_slug> service for project
-      #
-      # Example Request:
-      #
-      #   PUT /projects/:id/services/gitlab-ci
-      #
-      put ':id/services/:service_slug' do
-        if project_service
-          validators = project_service.class.validators.select do |s|
-            s.class == ActiveRecord::Validations::PresenceValidator &&
-              s.attributes != [:project_id]
+      helpers do
+        def service_attributes(service)
+          service.fields.inject([]) do |arr, hash|
+            arr << hash[:name].to_sym
           end
+        end
+      end
 
-          required_attributes! validators.map(&:attributes).flatten.uniq
-          attrs = attributes_for_keys service_attributes
+      services.each do |service_slug, settings|
+        desc "Set #{service_slug} service for project"
+        params do
+          settings.each do |setting|
+            if setting[:required]
+              requires setting[:name], type: setting[:type], desc: setting[:desc]
+            else
+              optional setting[:name], type: setting[:type], desc: setting[:desc]
+            end
+          end
+        end
+        put ":id/services/#{service_slug}" do
+          service = user_project.find_or_initialize_service(service_slug.underscore)
+          service_params = declared_params(include_missing: false).merge(active: true)
 
-          if project_service.update_attributes(attrs.merge(active: true))
+          if service.update_attributes(service_params)
             true
           else
-            not_found!
+            render_api_error!('400 Bad Request', 400)
           end
         end
       end
 
-      # Delete <service_slug> service for project
-      #
-      # Example Request:
-      #
-      #   DELETE /project/:id/services/gitlab-ci
-      #
-      delete ':id/services/:service_slug' do
-        if project_service
-          attrs = service_attributes.inject({}) do |hash, key|
-            hash.merge!(key => nil)
-          end
+      desc "Delete a service for project"
+      params do
+        requires :service_slug, type: String, values: services.keys, desc: 'The name of the service'
+      end
+      delete ":id/services/:service_slug" do
+        service = user_project.find_or_initialize_service(params[:service_slug].underscore)
 
-          if project_service.update_attributes(attrs.merge(active: false))
-            true
-          else
-            not_found!
-          end
+        attrs = service_attributes(service).inject({}) do |hash, key|
+          hash.merge!(key => nil)
+        end
+
+        if service.update_attributes(attrs.merge(active: false))
+          true
+        else
+          render_api_error!('400 Bad Request', 400)
         end
       end
 
-      # Get <service_slug> service settings for project
-      #
-      # Example Request:
-      #
-      #   GET /project/:id/services/gitlab-ci
-      #
-      get ':id/services/:service_slug' do
-        present project_service, with: Entities::ProjectService, include_passwords: current_user.is_admin?
+      desc 'Get the service settings for project' do
+        success Entities::ProjectService
+      end
+      params do
+        requires :service_slug, type: String, values: services.keys, desc: 'The name of the service'
+      end
+      get ":id/services/:service_slug" do
+        service = user_project.find_or_initialize_service(params[:service_slug].underscore)
+        present service, with: Entities::ProjectService, include_passwords: current_user.is_admin?
       end
     end
 
-    resource :projects do
-      desc 'Trigger a slash command' do
-        detail 'Added in GitLab 8.13'
+    trigger_services.each do |service_slug, settings|
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
       end
-      post ':id/services/:service_slug/trigger' do
-        project = find_project(params[:id])
+      resource :projects do
+        desc "Trigger a slash command for #{service_slug}" do
+          detail 'Added in GitLab 8.13'
+        end
+        params do
+          settings.each do |setting|
+            requires setting[:name], type: setting[:type], desc: setting[:desc]
+          end
+        end
+        post ":id/services/#{service_slug.underscore}/trigger" do
+          project = find_project(params[:id])
 
-        # This is not accurate, but done to prevent leakage of the project names
-        not_found!('Service') unless project
+          # This is not accurate, but done to prevent leakage of the project names
+          not_found!('Service') unless project
 
-        service = project_service(project)
+          service = project.find_or_initialize_service(service_slug.underscore)
 
-        result = service.try(:active?) && service.try(:trigger, params)
+          result = service.try(:active?) && service.try(:trigger, params)
 
-        if result
-          status result[:status] || 200
-          present result
-        else
-          not_found!('Service')
+          if result
+            status result[:status] || 200
+            present result
+          else
+            not_found!('Service')
+          end
         end
       end
     end
diff --git a/lib/api/templates.rb b/lib/api/templates.rb
index 8a53d9c0095..e23f99256a5 100644
--- a/lib/api/templates.rb
+++ b/lib/api/templates.rb
@@ -8,6 +8,10 @@ module API
       gitlab_ci_ymls: {
         klass: Gitlab::Template::GitlabCiYmlTemplate,
         gitlab_version: 8.9
+      },
+      dockerfiles: {
+        klass: Gitlab::Template::DockerfileTemplate,
+        gitlab_version: 8.15
       }
     }.freeze
     PROJECT_TEMPLATE_REGEX =
@@ -51,7 +55,7 @@ module API
       end
       params do
         optional :popular, type: Boolean, desc: 'If passed, returns only popular licenses'
-      end 
+      end
       get route do
         options = {
           featured: declared(params).popular.present? ? true : nil
@@ -69,7 +73,7 @@ module API
       end
       params do
         requires :name, type: String, desc: 'The name of the template'
-      end 
+      end
       get route, requirements: { name: /[\w\.-]+/ } do
         not_found!('License') unless Licensee::License.find(declared(params).name)
 
@@ -78,7 +82,7 @@ module API
         present template, with: Entities::RepoLicense
       end
     end
-      
+
     GLOBAL_TEMPLATE_TYPES.each do |template_type, properties|
       klass = properties[:klass]
       gitlab_version = properties[:gitlab_version]
@@ -104,7 +108,7 @@ module API
         end
         params do
           requires :name, type: String, desc: 'The name of the template'
-        end 
+        end
         get route do
           new_template = klass.find(declared(params).name)
 
diff --git a/lib/api/users.rb b/lib/api/users.rb
index 1dab799dd61..0842c3874c5 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -2,7 +2,10 @@ module API
   class Users < Grape::API
     include PaginationParams
 
-    before { authenticate! }
+    before do
+      allow_access_with_scope :read_user if request.get?
+      authenticate!
+    end
 
     resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do
       helpers do
@@ -353,7 +356,7 @@ module API
         success Entities::UserPublic
       end
       get do
-        present current_user, with: @impersonator ? Entities::UserWithPrivateToken : Entities::UserPublic
+        present current_user, with: sudo? ? Entities::UserWithPrivateToken : Entities::UserPublic
       end
 
       desc "Get the currently authenticated user's SSH keys" do
diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb
index d904a8bd4ae..fd74eeaebe7 100644
--- a/lib/banzai/filter/abstract_reference_filter.rb
+++ b/lib/banzai/filter/abstract_reference_filter.rb
@@ -248,7 +248,7 @@ module Banzai
       end
 
       def projects_relation_for_paths(paths)
-        Project.where_paths_in(paths).includes(:namespace)
+        Project.where_full_path_in(paths).includes(:namespace)
       end
 
       # Returns projects for the given paths.
diff --git a/lib/banzai/filter/math_filter.rb b/lib/banzai/filter/math_filter.rb
new file mode 100644
index 00000000000..cb037f89337
--- /dev/null
+++ b/lib/banzai/filter/math_filter.rb
@@ -0,0 +1,51 @@
+require 'uri'
+
+module Banzai
+  module Filter
+    # HTML filter that adds class="code math" and removes the dollar sign in $`2+2`$.
+    #
+    class MathFilter < HTML::Pipeline::Filter
+      # This picks out <code>...</code>.
+      INLINE_MATH = 'descendant-or-self::code'.freeze
+
+      # Pick out a code block which is declared math
+      DISPLAY_MATH = "descendant-or-self::pre[contains(@class, 'math') and contains(@class, 'code')]".freeze
+
+      # Attribute indicating inline or display math.
+      STYLE_ATTRIBUTE = 'data-math-style'.freeze
+
+      # Class used for tagging elements that should be rendered
+      TAG_CLASS = 'js-render-math'.freeze
+
+      INLINE_CLASSES = "code math #{TAG_CLASS}".freeze
+
+      DOLLAR_SIGN = '$'.freeze
+
+      def call
+        doc.xpath(INLINE_MATH).each do |code|
+          closing = code.next
+          opening = code.previous
+
+          # We need a sibling before and after.
+          # They should end and start with $ respectively.
+          if closing && opening &&
+              closing.content.first == DOLLAR_SIGN &&
+              opening.content.last == DOLLAR_SIGN
+
+            code[:class] = INLINE_CLASSES
+            code[STYLE_ATTRIBUTE] = 'inline'
+            closing.content = closing.content[1..-1]
+            opening.content = opening.content[0..-2]
+          end
+        end
+
+        doc.xpath(DISPLAY_MATH).each do |el|
+          el[STYLE_ATTRIBUTE] = 'display'
+          el[:class] += " #{TAG_CLASS}"
+        end
+
+        doc
+      end
+    end
+  end
+end
diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb
index 5da2d0b008c..5a1f873496c 100644
--- a/lib/banzai/pipeline/gfm_pipeline.rb
+++ b/lib/banzai/pipeline/gfm_pipeline.rb
@@ -6,6 +6,7 @@ module Banzai
           Filter::SyntaxHighlightFilter,
           Filter::SanitizationFilter,
 
+          Filter::MathFilter,
           Filter::UploadLinkFilter,
           Filter::VideoLinkFilter,
           Filter::ImageLinkFilter,
diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb
new file mode 100644
index 00000000000..f8ee7e0f9ae
--- /dev/null
+++ b/lib/bitbucket/client.rb
@@ -0,0 +1,58 @@
+module Bitbucket
+  class Client
+    attr_reader :connection
+
+    def initialize(options = {})
+      @connection = Connection.new(options)
+    end
+
+    def issues(repo)
+      path = "/repositories/#{repo}/issues"
+      get_collection(path, :issue)
+    end
+
+    def issue_comments(repo, issue_id)
+      path = "/repositories/#{repo}/issues/#{issue_id}/comments"
+      get_collection(path, :comment)
+    end
+
+    def pull_requests(repo)
+      path = "/repositories/#{repo}/pullrequests?state=ALL"
+      get_collection(path, :pull_request)
+    end
+
+    def pull_request_comments(repo, pull_request)
+      path = "/repositories/#{repo}/pullrequests/#{pull_request}/comments"
+      get_collection(path, :pull_request_comment)
+    end
+
+    def pull_request_diff(repo, pull_request)
+      path = "/repositories/#{repo}/pullrequests/#{pull_request}/diff"
+      connection.get(path)
+    end
+
+    def repo(name)
+      parsed_response = connection.get("/repositories/#{name}")
+      Representation::Repo.new(parsed_response)
+    end
+
+    def repos
+      path = "/repositories?role=member"
+      get_collection(path, :repo)
+    end
+
+    def user
+      @user ||= begin
+        parsed_response = connection.get('/user')
+        Representation::User.new(parsed_response)
+      end
+    end
+
+    private
+
+    def get_collection(path, type)
+      paginator = Paginator.new(connection, path, type)
+      Collection.new(paginator)
+    end
+  end
+end
diff --git a/lib/bitbucket/collection.rb b/lib/bitbucket/collection.rb
new file mode 100644
index 00000000000..3a9379ff680
--- /dev/null
+++ b/lib/bitbucket/collection.rb
@@ -0,0 +1,21 @@
+module Bitbucket
+  class Collection < Enumerator
+    def initialize(paginator)
+      super() do |yielder|
+        loop do
+          paginator.items.each { |item| yielder << item }
+        end
+      end
+
+      lazy
+    end
+
+    def method_missing(method, *args)
+      return super unless self.respond_to?(method)
+
+      self.send(method, *args) do |item|
+        block_given? ? yield(item) : item
+      end
+    end
+  end
+end
diff --git a/lib/bitbucket/connection.rb b/lib/bitbucket/connection.rb
new file mode 100644
index 00000000000..7e55cf4deab
--- /dev/null
+++ b/lib/bitbucket/connection.rb
@@ -0,0 +1,69 @@
+module Bitbucket
+  class Connection
+    DEFAULT_API_VERSION = '2.0'
+    DEFAULT_BASE_URI    = 'https://api.bitbucket.org/'
+    DEFAULT_QUERY       = {}
+
+    attr_reader :expires_at, :expires_in, :refresh_token, :token
+
+    def initialize(options = {})
+      @api_version   = options.fetch(:api_version, DEFAULT_API_VERSION)
+      @base_uri      = options.fetch(:base_uri, DEFAULT_BASE_URI)
+      @default_query = options.fetch(:query, DEFAULT_QUERY)
+
+      @token         = options[:token]
+      @expires_at    = options[:expires_at]
+      @expires_in    = options[:expires_in]
+      @refresh_token = options[:refresh_token]
+    end
+
+    def get(path, extra_query = {})
+      refresh! if expired?
+
+      response = connection.get(build_url(path), params: @default_query.merge(extra_query))
+      response.parsed
+    end
+
+    def expired?
+      connection.expired?
+    end
+
+    def refresh!
+      response = connection.refresh!
+
+      @token         = response.token
+      @expires_at    = response.expires_at
+      @expires_in    = response.expires_in
+      @refresh_token = response.refresh_token
+      @connection    = nil
+    end
+
+    private
+
+    def client
+      @client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options)
+    end
+
+    def connection
+      @connection ||= OAuth2::AccessToken.new(client, @token, refresh_token: @refresh_token, expires_at: @expires_at, expires_in: @expires_in)
+    end
+
+    def build_url(path)
+      return path if path.starts_with?(root_url)
+
+      "#{root_url}#{path}"
+    end
+
+    def root_url
+      @root_url ||= "#{@base_uri}#{@api_version}"
+    end
+
+    def provider
+      Gitlab::OAuth::Provider.config_for('bitbucket')
+    end
+
+    def options
+      OmniAuth::Strategies::Bitbucket.default_options[:client_options].deep_symbolize_keys
+    end
+  end
+end
diff --git a/lib/bitbucket/error/unauthorized.rb b/lib/bitbucket/error/unauthorized.rb
new file mode 100644
index 00000000000..5e2eb57bb0e
--- /dev/null
+++ b/lib/bitbucket/error/unauthorized.rb
@@ -0,0 +1,6 @@
+module Bitbucket
+  module Error
+    class Unauthorized < StandardError
+    end
+  end
+end
diff --git a/lib/bitbucket/page.rb b/lib/bitbucket/page.rb
new file mode 100644
index 00000000000..2b0a3fe7b1a
--- /dev/null
+++ b/lib/bitbucket/page.rb
@@ -0,0 +1,34 @@
+module Bitbucket
+  class Page
+    attr_reader :attrs, :items
+
+    def initialize(raw, type)
+      @attrs = parse_attrs(raw)
+      @items = parse_values(raw, representation_class(type))
+    end
+
+    def next?
+      attrs.fetch(:next, false)
+    end
+
+    def next
+      attrs.fetch(:next)
+    end
+
+    private
+
+    def parse_attrs(raw)
+      raw.slice(*%w(size page pagelen next previous)).symbolize_keys
+    end
+
+    def parse_values(raw, bitbucket_rep_class)
+      return [] unless raw['values'] && raw['values'].is_a?(Array)
+
+      bitbucket_rep_class.decorate(raw['values'])
+    end
+
+    def representation_class(type)
+      Bitbucket::Representation.const_get(type.to_s.camelize)
+    end
+  end
+end
diff --git a/lib/bitbucket/paginator.rb b/lib/bitbucket/paginator.rb
new file mode 100644
index 00000000000..135d0d55674
--- /dev/null
+++ b/lib/bitbucket/paginator.rb
@@ -0,0 +1,36 @@
+module Bitbucket
+  class Paginator
+    PAGE_LENGTH = 50 # The minimum length is 10 and the maximum is 100.
+
+    def initialize(connection, url, type)
+      @connection = connection
+      @type = type
+      @url = url
+      @page = nil
+    end
+
+    def items
+      raise StopIteration unless has_next_page?
+
+      @page = fetch_next_page
+      @page.items
+    end
+
+    private
+
+    attr_reader :connection, :page, :url, :type
+
+    def has_next_page?
+      page.nil? || page.next?
+    end
+
+    def next_url
+      page.nil? ? url : page.next
+    end
+
+    def fetch_next_page
+      parsed_response = connection.get(next_url, pagelen: PAGE_LENGTH, sort: :created_on)
+      Page.new(parsed_response, type)
+    end
+  end
+end
diff --git a/lib/bitbucket/representation/base.rb b/lib/bitbucket/representation/base.rb
new file mode 100644
index 00000000000..94adaacc9b5
--- /dev/null
+++ b/lib/bitbucket/representation/base.rb
@@ -0,0 +1,17 @@
+module Bitbucket
+  module Representation
+    class Base
+      def initialize(raw)
+        @raw = raw
+      end
+
+      def self.decorate(entries)
+        entries.map { |entry| new(entry)}
+      end
+
+      private
+
+      attr_reader :raw
+    end
+  end
+end
diff --git a/lib/bitbucket/representation/comment.rb b/lib/bitbucket/representation/comment.rb
new file mode 100644
index 00000000000..4937aa9728f
--- /dev/null
+++ b/lib/bitbucket/representation/comment.rb
@@ -0,0 +1,27 @@
+module Bitbucket
+  module Representation
+    class Comment < Representation::Base
+      def author
+        user['username']
+      end
+
+      def note
+        raw.fetch('content', {}).fetch('raw', nil)
+      end
+
+      def created_at
+        raw['created_on']
+      end
+
+      def updated_at
+        raw['updated_on'] || raw['created_on']
+      end
+
+      private
+
+      def user
+        raw.fetch('user', {})
+      end
+    end
+  end
+end
diff --git a/lib/bitbucket/representation/issue.rb b/lib/bitbucket/representation/issue.rb
new file mode 100644
index 00000000000..054064395c3
--- /dev/null
+++ b/lib/bitbucket/representation/issue.rb
@@ -0,0 +1,53 @@
+module Bitbucket
+  module Representation
+    class Issue < Representation::Base
+      CLOSED_STATUS = %w(resolved invalid duplicate wontfix closed).freeze
+
+      def iid
+        raw['id']
+      end
+
+      def kind
+        raw['kind']
+      end
+
+      def author
+        raw.fetch('reporter', {}).fetch('username', nil)
+      end
+
+      def description
+        raw.fetch('content', {}).fetch('raw', nil)
+      end
+
+      def state
+        closed? ? 'closed' : 'opened'
+      end
+
+      def title
+        raw['title']
+      end
+
+      def milestone
+        raw['milestone']['name'] if raw['milestone'].present?
+      end
+
+      def created_at
+        raw['created_on']
+      end
+
+      def updated_at
+        raw['edited_on']
+      end
+
+      def to_s
+        iid
+      end
+
+      private
+
+      def closed?
+        CLOSED_STATUS.include?(raw['state'])
+      end
+    end
+  end
+end
diff --git a/lib/bitbucket/representation/pull_request.rb b/lib/bitbucket/representation/pull_request.rb
new file mode 100644
index 00000000000..eebf8093380
--- /dev/null
+++ b/lib/bitbucket/representation/pull_request.rb
@@ -0,0 +1,65 @@
+module Bitbucket
+  module Representation
+    class PullRequest < Representation::Base
+      def author
+        raw.fetch('author', {}).fetch('username', nil)
+      end
+
+      def description
+        raw['description']
+      end
+
+      def iid
+        raw['id']
+      end
+
+      def state
+        if raw['state'] == 'MERGED'
+          'merged'
+        elsif raw['state'] == 'DECLINED'
+          'closed'
+        else
+          'opened'
+        end
+      end
+
+      def created_at
+        raw['created_on']
+      end
+
+      def updated_at
+        raw['updated_on']
+      end
+
+      def title
+        raw['title']
+      end
+
+      def source_branch_name
+        source_branch.fetch('branch', {}).fetch('name', nil)
+      end
+
+      def source_branch_sha
+        source_branch.fetch('commit', {}).fetch('hash', nil)
+      end
+
+      def target_branch_name
+        target_branch.fetch('branch', {}).fetch('name', nil)
+      end
+
+      def target_branch_sha
+        target_branch.fetch('commit', {}).fetch('hash', nil)
+      end
+
+      private
+
+      def source_branch
+        raw['source']
+      end
+
+      def target_branch
+        raw['destination']
+      end
+    end
+  end
+end
diff --git a/lib/bitbucket/representation/pull_request_comment.rb b/lib/bitbucket/representation/pull_request_comment.rb
new file mode 100644
index 00000000000..4f8efe03bae
--- /dev/null
+++ b/lib/bitbucket/representation/pull_request_comment.rb
@@ -0,0 +1,39 @@
+module Bitbucket
+  module Representation
+    class PullRequestComment < Comment
+      def iid
+        raw['id']
+      end
+
+      def file_path
+        inline.fetch('path')
+      end
+
+      def old_pos
+        inline.fetch('from')
+      end
+
+      def new_pos
+        inline.fetch('to')
+      end
+
+      def parent_id
+        raw.fetch('parent', {}).fetch('id', nil)
+      end
+
+      def inline?
+        raw.has_key?('inline')
+      end
+
+      def has_parent?
+        raw.has_key?('parent')
+      end
+
+      private
+
+      def inline
+        raw.fetch('inline', {})
+      end
+    end
+  end
+end
diff --git a/lib/bitbucket/representation/repo.rb b/lib/bitbucket/representation/repo.rb
new file mode 100644
index 00000000000..423eff8f2a5
--- /dev/null
+++ b/lib/bitbucket/representation/repo.rb
@@ -0,0 +1,71 @@
+module Bitbucket
+  module Representation
+    class Repo < Representation::Base
+      attr_reader :owner, :slug
+
+      def initialize(raw)
+        super(raw)
+      end
+
+      def owner_and_slug
+        @owner_and_slug ||= full_name.split('/', 2)
+      end
+
+      def owner
+        owner_and_slug.first
+      end
+
+      def slug
+        owner_and_slug.last
+      end
+
+      def clone_url(token = nil)
+        url = raw['links']['clone'].find { |link| link['name'] == 'https' }.fetch('href')
+
+        if token.present?
+          clone_url = URI::parse(url)
+          clone_url.user = "x-token-auth:#{token}"
+          clone_url.to_s
+        else
+          url
+        end
+      end
+
+      def description
+        raw['description']
+      end
+
+      def full_name
+        raw['full_name']
+      end
+
+      def issues_enabled?
+        raw['has_issues']
+      end
+
+      def name
+        raw['name']
+      end
+
+      def valid?
+        raw['scm'] == 'git'
+      end
+
+      def has_wiki?
+        raw['has_wiki']
+      end
+
+      def visibility_level
+        if raw['is_private']
+          Gitlab::VisibilityLevel::PRIVATE
+        else
+          Gitlab::VisibilityLevel::PUBLIC
+        end
+      end
+
+      def to_s
+        full_name
+      end
+    end
+  end
+end
diff --git a/lib/bitbucket/representation/user.rb b/lib/bitbucket/representation/user.rb
new file mode 100644
index 00000000000..ba6b7667b49
--- /dev/null
+++ b/lib/bitbucket/representation/user.rb
@@ -0,0 +1,9 @@
+module Bitbucket
+  module Representation
+    class User < Representation::Base
+      def username
+        raw['username']
+      end
+    end
+  end
+end
diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index ed87a2603e8..142bce82286 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -41,7 +41,7 @@ module Ci
         put ":id" do
           authenticate_runner!
           build = Ci::Build.where(runner_id: current_runner.id).running.find(params[:id])
-          forbidden!('Build has been erased!') if build.erased?
+          validate_build!(build)
 
           update_runner_info
 
@@ -71,9 +71,7 @@ module Ci
         #   PATCH /builds/:id/trace.txt
         patch ":id/trace.txt" do
           build = Ci::Build.find_by_id(params[:id])
-          not_found! unless build
-          authenticate_build_token!(build)
-          forbidden!('Build has been erased!') if build.erased?
+          authenticate_build!(build)
 
           error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range')
           content_range = request.headers['Content-Range']
@@ -104,8 +102,7 @@ module Ci
           Gitlab::Workhorse.verify_api_request!(headers)
           not_allowed! unless Gitlab.config.artifacts.enabled
           build = Ci::Build.find_by_id(params[:id])
-          not_found! unless build
-          authenticate_build_token!(build)
+          authenticate_build!(build)
           forbidden!('build is not running') unless build.running?
 
           if params[:filesize]
@@ -142,10 +139,8 @@ module Ci
           require_gitlab_workhorse!
           not_allowed! unless Gitlab.config.artifacts.enabled
           build = Ci::Build.find_by_id(params[:id])
-          not_found! unless build
-          authenticate_build_token!(build)
+          authenticate_build!(build)
           forbidden!('Build is not running!') unless build.running?
-          forbidden!('Build has been erased!') if build.erased?
 
           artifacts_upload_path = ArtifactUploader.artifacts_upload_path
           artifacts = uploaded_file(:file, artifacts_upload_path)
@@ -176,8 +171,7 @@ module Ci
         #   GET /builds/:id/artifacts
         get ":id/artifacts" do
           build = Ci::Build.find_by_id(params[:id])
-          not_found! unless build
-          authenticate_build_token!(build)
+          authenticate_build!(build)
           artifacts_file = build.artifacts_file
 
           unless artifacts_file.file_storage?
@@ -202,8 +196,7 @@ module Ci
         #   DELETE /builds/:id/artifacts
         delete ":id/artifacts" do
           build = Ci::Build.find_by_id(params[:id])
-          not_found! unless build
-          authenticate_build_token!(build)
+          authenticate_build!(build)
 
           build.erase_artifacts!
         end
diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb
index e608f5f6cad..31fbd1da108 100644
--- a/lib/ci/api/helpers.rb
+++ b/lib/ci/api/helpers.rb
@@ -13,8 +13,19 @@ module Ci
         forbidden! unless current_runner
       end
 
-      def authenticate_build_token!(build)
-        forbidden! unless build_token_valid?(build)
+      def authenticate_build!(build)
+        validate_build!(build) do
+          forbidden! unless build_token_valid?(build)
+        end
+      end
+
+      def validate_build!(build)
+        not_found! unless build
+
+        yield if block_given?
+
+        forbidden!('Project has been deleted!') unless build.project
+        forbidden!('Build has been erased!') if build.erased?
       end
 
       def runner_registration_token_valid?
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index fef652cb975..7463bd719d5 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -118,7 +118,7 @@ module Ci
         .merge(job_variables(name))
 
       variables.map do |key, value|
-        { key: key, value: value, public: true }
+        { key: key.to_s, value: value, public: true }
       end
     end
 
diff --git a/lib/gitlab/allowable.rb b/lib/gitlab/allowable.rb
new file mode 100644
index 00000000000..f48abcc86d5
--- /dev/null
+++ b/lib/gitlab/allowable.rb
@@ -0,0 +1,7 @@
+module Gitlab
+  module Allowable
+    def can?(user, action, subject)
+      Ability.allowed?(user, action, subject)
+    end
+  end
+end
diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb
index 9667df4ffb8..fa234284361 100644
--- a/lib/gitlab/asciidoc.rb
+++ b/lib/gitlab/asciidoc.rb
@@ -1,4 +1,5 @@
 require 'asciidoctor'
+require 'asciidoctor/converter/html5'
 
 module Gitlab
   # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters
@@ -23,7 +24,7 @@ module Gitlab
     def self.render(input, context, asciidoc_opts = {})
       asciidoc_opts.reverse_merge!(
         safe: :secure,
-        backend: :html5,
+        backend: :gitlab_html5,
         attributes: []
       )
       asciidoc_opts[:attributes].unshift(*DEFAULT_ADOC_ATTRS)
@@ -34,5 +35,29 @@ module Gitlab
 
       html.html_safe
     end
+
+    class Html5Converter < Asciidoctor::Converter::Html5Converter
+      extend Asciidoctor::Converter::Config
+
+      register_for 'gitlab_html5'
+
+      def stem(node)
+        return super unless node.style.to_sym == :latexmath
+
+        %(<pre#{id_attribute(node)} class="code math js-render-math #{node.role}" data-math-style="display"><code>#{node.content}</code></pre>)
+      end
+
+      def inline_quoted(node)
+        return super unless node.type.to_sym == :latexmath
+
+        %(<code#{id_attribute(node)} class="code math js-render-math #{node.role}" data-math-style="inline">#{node.text}</code>)
+      end
+
+      private
+
+      def id_attribute(node)
+        node.id ? %( id="#{node.id}") : nil
+      end
+    end
   end
 end
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index aca5d0020cf..8dda65c71ef 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -2,6 +2,10 @@ module Gitlab
   module Auth
     class MissingPersonalTokenError < StandardError; end
 
+    SCOPES = [:api, :read_user]
+    DEFAULT_SCOPES = [:api]
+    OPTIONAL_SCOPES = SCOPES - DEFAULT_SCOPES
+
     class << self
       def find_for_git_client(login, password, project:, ip:)
         raise "Must provide an IP for rate limiting" if ip.nil?
@@ -88,7 +92,7 @@ module Gitlab
       def oauth_access_token_check(login, password)
         if login == "oauth2" && password.present?
           token = Doorkeeper::AccessToken.by_token(password)
-          if token && token.accessible?
+          if valid_oauth_token?(token)
             user = User.find_by(id: token.resource_owner_id)
             Gitlab::Auth::Result.new(user, nil, :oauth, read_authentication_abilities)
           end
@@ -97,12 +101,27 @@ module Gitlab
 
       def personal_access_token_check(login, password)
         if login && password
-          user = User.find_by_personal_access_token(password)
+          token = PersonalAccessToken.active.find_by_token(password)
           validation = User.by_login(login)
-          Gitlab::Auth::Result.new(user, nil, :personal_token, full_authentication_abilities) if user.present? && user == validation
+
+          if valid_personal_access_token?(token, validation)
+            Gitlab::Auth::Result.new(validation, nil, :personal_token, full_authentication_abilities)
+          end
         end
       end
 
+      def valid_oauth_token?(token)
+        token && token.accessible? && valid_api_token?(token)
+      end
+
+      def valid_personal_access_token?(token, user)
+        token && token.user == user && valid_api_token?(token)
+      end
+
+      def valid_api_token?(token)
+        AccessTokenValidationService.new(token).include_any_scope?(['api'])
+      end
+
       def lfs_token_check(login, password)
         deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/)
 
diff --git a/lib/gitlab/badge/build/status.rb b/lib/gitlab/badge/build/status.rb
index 50aa45e5406..b762d85b6e5 100644
--- a/lib/gitlab/badge/build/status.rb
+++ b/lib/gitlab/badge/build/status.rb
@@ -20,8 +20,8 @@ module Gitlab
 
         def status
           @project.pipelines
-            .where(sha: @sha, ref: @ref)
-            .status || 'unknown'
+            .where(sha: @sha)
+            .latest_status(@ref) || 'unknown'
         end
 
         def metadata
diff --git a/lib/gitlab/bitbucket_import.rb b/lib/gitlab/bitbucket_import.rb
deleted file mode 100644
index 7298152e7e9..00000000000
--- a/lib/gitlab/bitbucket_import.rb
+++ /dev/null
@@ -1,6 +0,0 @@
-module Gitlab
-  module BitbucketImport
-    mattr_accessor :public_key
-    @public_key = nil
-  end
-end
diff --git a/lib/gitlab/bitbucket_import/client.rb b/lib/gitlab/bitbucket_import/client.rb
deleted file mode 100644
index 8d1ad62fae0..00000000000
--- a/lib/gitlab/bitbucket_import/client.rb
+++ /dev/null
@@ -1,142 +0,0 @@
-module Gitlab
-  module BitbucketImport
-    class Client
-      class Unauthorized < StandardError; end
-
-      attr_reader :consumer, :api
-
-      def self.from_project(project)
-        import_data_credentials = project.import_data.credentials if project.import_data
-        if import_data_credentials && import_data_credentials[:bb_session]
-          token = import_data_credentials[:bb_session][:bitbucket_access_token]
-          token_secret = import_data_credentials[:bb_session][:bitbucket_access_token_secret]
-          new(token, token_secret)
-        else
-          raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{project.id}"
-        end
-      end
-
-      def initialize(access_token = nil, access_token_secret = nil)
-        @consumer = ::OAuth::Consumer.new(
-          config.app_id,
-          config.app_secret,
-          bitbucket_options
-        )
-
-        if access_token && access_token_secret
-          @api = ::OAuth::AccessToken.new(@consumer, access_token, access_token_secret)
-        end
-      end
-
-      def request_token(redirect_uri)
-        request_token = consumer.get_request_token(oauth_callback: redirect_uri)
-
-        {
-          oauth_token:              request_token.token,
-          oauth_token_secret:       request_token.secret,
-          oauth_callback_confirmed: request_token.callback_confirmed?.to_s
-        }
-      end
-
-      def authorize_url(request_token, redirect_uri)
-        request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash)
-
-        if request_token.callback_confirmed?
-          request_token.authorize_url
-        else
-          request_token.authorize_url(oauth_callback: redirect_uri)
-        end
-      end
-
-      def get_token(request_token, oauth_verifier, redirect_uri)
-        request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash)
-
-        if request_token.callback_confirmed?
-          request_token.get_access_token(oauth_verifier: oauth_verifier)
-        else
-          request_token.get_access_token(oauth_callback: redirect_uri)
-        end
-      end
-
-      def user
-        JSON.parse(get("/api/1.0/user").body)
-      end
-
-      def issues(project_identifier)
-        all_issues = []
-        offset = 0
-        per_page = 50 # Maximum number allowed by Bitbucket
-        index = 0
-
-        begin
-          issues = JSON.parse(get(issue_api_endpoint(project_identifier, per_page, offset)).body)
-          # Find out how many total issues are present
-          total = issues["count"] if index == 0
-          all_issues.concat(issues["issues"])
-          offset += issues["issues"].count
-          index += 1
-        end while all_issues.count < total
-
-        all_issues
-      end
-
-      def issue_comments(project_identifier, issue_id)
-        comments = JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body)
-        comments.sort_by { |comment| comment["utc_created_on"] }
-      end
-
-      def project(project_identifier)
-        JSON.parse(get("/api/1.0/repositories/#{project_identifier}").body)
-      end
-
-      def find_deploy_key(project_identifier, key)
-        JSON.parse(get("/api/1.0/repositories/#{project_identifier}/deploy-keys").body).find do |deploy_key|
-          deploy_key["key"].chomp == key.chomp
-        end
-      end
-
-      def add_deploy_key(project_identifier, key)
-        deploy_key = find_deploy_key(project_identifier, key)
-        return if deploy_key
-
-        JSON.parse(api.post("/api/1.0/repositories/#{project_identifier}/deploy-keys", key: key, label: "GitLab import key").body)
-      end
-
-      def delete_deploy_key(project_identifier, key)
-        deploy_key = find_deploy_key(project_identifier, key)
-        return unless deploy_key
-
-        api.delete("/api/1.0/repositories/#{project_identifier}/deploy-keys/#{deploy_key["pk"]}").code == "204"
-      end
-
-      def projects
-        JSON.parse(get("/api/1.0/user/repositories").body).select { |repo| repo["scm"] == "git" }
-      end
-
-      def incompatible_projects
-        JSON.parse(get("/api/1.0/user/repositories").body).reject { |repo| repo["scm"] == "git" }
-      end
-
-      private
-
-      def get(url)
-        response = api.get(url)
-        raise Unauthorized if (400..499).cover?(response.code.to_i)
-
-        response
-      end
-
-      def issue_api_endpoint(project_identifier, per_page, offset)
-        "/api/1.0/repositories/#{project_identifier}/issues?sort=utc_created_on&limit=#{per_page}&start=#{offset}"
-      end
-
-      def config
-        Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket" }
-      end
-
-      def bitbucket_options
-        OmniAuth::Strategies::Bitbucket.default_options[:client_options].symbolize_keys
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb
index f4b5097adb1..44323b47dca 100644
--- a/lib/gitlab/bitbucket_import/importer.rb
+++ b/lib/gitlab/bitbucket_import/importer.rb
@@ -1,84 +1,247 @@
 module Gitlab
   module BitbucketImport
     class Importer
-      attr_reader :project, :client
+      include Gitlab::ShellAdapter
+
+      LABELS = [{ title: 'bug', color: '#FF0000' },
+                { title: 'enhancement', color: '#428BCA' },
+                { title: 'proposal', color: '#69D100' },
+                { title: 'task', color: '#7F8C8D' }].freeze
+
+      attr_reader :project, :client, :errors, :users
 
       def initialize(project)
         @project = project
-        @client = Client.from_project(@project)
+        @client = Bitbucket::Client.new(project.import_data.credentials)
         @formatter = Gitlab::ImportFormatter.new
+        @labels = {}
+        @errors = []
+        @users = {}
       end
 
       def execute
-        import_issues if has_issues?
+        import_wiki
+        import_issues
+        import_pull_requests
+        handle_errors
 
         true
-      rescue ActiveRecord::RecordInvalid => e
-        raise Projects::ImportService::Error.new, e.message
-      ensure
-        Gitlab::BitbucketImport::KeyDeleter.new(project).execute
       end
 
       private
 
-      def gitlab_user_id(project, bitbucket_id)
-        if bitbucket_id
-          user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s)
-          (user && user.id) || project.creator_id
-        else
-          project.creator_id
-        end
+      def handle_errors
+        return unless errors.any?
+
+        project.update_column(:import_error, {
+          message: 'The remote data could not be fully imported.',
+          errors: errors
+        }.to_json)
+      end
+
+      def gitlab_user_id(project, username)
+        find_user_id(username) || project.creator_id
       end
 
-      def identifier
-        project.import_source
+      def find_user_id(username)
+        return nil unless username
+
+        return users[username] if users.key?(username)
+
+        users[username] = User.select(:id)
+                              .joins(:identities)
+                              .find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", username)
+                              .try(:id)
       end
 
-      def has_issues?
-        client.project(identifier)["has_issues"]
+      def repo
+        @repo ||= client.repo(project.import_source)
       end
 
-      def import_issues
-        issues = client.issues(identifier)
+      def import_wiki
+        return if project.wiki.repository_exists?
 
-        issues.each do |issue|
-          body = ''
-          reporter = nil
-          author = 'Anonymous'
+        path_with_namespace = "#{project.path_with_namespace}.wiki"
+        import_url = project.import_url.sub(/\.git\z/, ".git/wiki")
+        gitlab_shell.import_repository(project.repository_storage_path, path_with_namespace, import_url)
+      rescue StandardError => e
+        errors << { type: :wiki, errors: e.message }
+      end
 
-          if issue["reported_by"] && issue["reported_by"]["username"]
-            reporter = issue["reported_by"]["username"]
-            author = reporter
+      def import_issues
+        return unless repo.issues_enabled?
+
+        create_labels
+
+        client.issues(repo).each do |issue|
+          begin
+            description = ''
+            description += @formatter.author_line(issue.author) unless find_user_id(issue.author)
+            description += issue.description
+
+            label_name = issue.kind
+            milestone = issue.milestone ? project.milestones.find_or_create_by(title: issue.milestone) : nil
+
+            gitlab_issue = project.issues.create!(
+              iid: issue.iid,
+              title: issue.title,
+              description: description,
+              state: issue.state,
+              author_id: gitlab_user_id(project, issue.author),
+              milestone: milestone,
+              created_at: issue.created_at,
+              updated_at: issue.updated_at
+            )
+
+            gitlab_issue.labels << @labels[label_name]
+
+            import_issue_comments(issue, gitlab_issue) if gitlab_issue.persisted?
+          rescue StandardError => e
+            errors << { type: :issue, iid: issue.iid, errors: e.message }
           end
+        end
+      end
 
-          body = @formatter.author_line(author)
-          body += issue["content"]
+      def import_issue_comments(issue, gitlab_issue)
+        client.issue_comments(repo, issue.iid).each do |comment|
+          # The note can be blank for issue service messages like "Changed title: ..."
+          # We would like to import those comments as well but there is no any
+          # specific parameter that would allow to process them, it's just an empty comment.
+          # To prevent our importer from just crashing or from creating useless empty comments
+          # we do this check.
+          next unless comment.note.present?
+
+          note = ''
+          note += @formatter.author_line(comment.author) unless find_user_id(comment.author)
+          note += comment.note
+
+          begin
+            gitlab_issue.notes.create!(
+              project: project,
+              note: note,
+              author_id: gitlab_user_id(project, comment.author),
+              created_at: comment.created_at,
+              updated_at: comment.updated_at
+            )
+          rescue StandardError => e
+            errors << { type: :issue_comment, iid: issue.iid, errors: e.message }
+          end
+        end
+      end
 
-          comments = client.issue_comments(identifier, issue["local_id"])
+      def create_labels
+        LABELS.each do |label|
+          @labels[label[:title]] = project.labels.create!(label)
+        end
+      end
 
-          if comments.any?
-            body += @formatter.comments_header
+      def import_pull_requests
+        pull_requests = client.pull_requests(repo)
+
+        pull_requests.each do |pull_request|
+          begin
+            description = ''
+            description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author)
+            description += pull_request.description
+
+            merge_request = project.merge_requests.create(
+              iid: pull_request.iid,
+              title: pull_request.title,
+              description: description,
+              source_project: project,
+              source_branch: pull_request.source_branch_name,
+              source_branch_sha: pull_request.source_branch_sha,
+              target_project: project,
+              target_branch: pull_request.target_branch_name,
+              target_branch_sha: pull_request.target_branch_sha,
+              state: pull_request.state,
+              author_id: gitlab_user_id(project, pull_request.author),
+              assignee_id: nil,
+              created_at: pull_request.created_at,
+              updated_at: pull_request.updated_at
+            )
+
+            import_pull_request_comments(pull_request, merge_request) if merge_request.persisted?
+          rescue StandardError => e
+            errors << { type: :pull_request, iid: pull_request.iid, errors: e.message }
           end
+        end
+      end
+
+      def import_pull_request_comments(pull_request, merge_request)
+        comments = client.pull_request_comments(repo, pull_request.iid)
+
+        inline_comments, pr_comments = comments.partition(&:inline?)
+
+        import_inline_comments(inline_comments, pull_request, merge_request)
+        import_standalone_pr_comments(pr_comments, merge_request)
+      end
 
-          comments.each do |comment|
-            author = 'Anonymous'
+      def import_inline_comments(inline_comments, pull_request, merge_request)
+        line_code_map = {}
 
-            if comment["author_info"] && comment["author_info"]["username"]
-              author = comment["author_info"]["username"]
-            end
+        children, parents = inline_comments.partition(&:has_parent?)
 
-            body += @formatter.comment(author, comment["utc_created_on"], comment["content"])
+        # The Bitbucket API returns threaded replies as parent-child
+        # relationships. We assume that the child can appear in any order in
+        # the JSON.
+        parents.each do |comment|
+          line_code_map[comment.iid] = generate_line_code(comment)
+        end
+
+        children.each do |comment|
+          line_code_map[comment.iid] = line_code_map.fetch(comment.parent_id, nil)
+        end
+
+        inline_comments.each do |comment|
+          begin
+            attributes = pull_request_comment_attributes(comment)
+            attributes.merge!(
+              position: build_position(merge_request, comment),
+              line_code: line_code_map.fetch(comment.iid),
+              type: 'DiffNote')
+
+            merge_request.notes.create!(attributes)
+          rescue StandardError => e
+            errors << { type: :pull_request, iid: comment.iid, errors: e.message }
           end
+        end
+      end
+
+      def build_position(merge_request, pr_comment)
+        params = {
+          diff_refs: merge_request.diff_refs,
+          old_path: pr_comment.file_path,
+          new_path: pr_comment.file_path,
+          old_line: pr_comment.old_pos,
+          new_line: pr_comment.new_pos
+        }
 
-          project.issues.create!(
-            description: body,
-            title: issue["title"],
-            state: %w(resolved invalid duplicate wontfix closed).include?(issue["status"]) ? 'closed' : 'opened',
-            author_id: gitlab_user_id(project, reporter)
-          )
+        Gitlab::Diff::Position.new(params)
+      end
+
+      def import_standalone_pr_comments(pr_comments, merge_request)
+        pr_comments.each do |comment|
+          begin
+            merge_request.notes.create!(pull_request_comment_attributes(comment))
+          rescue StandardError => e
+            errors << { type: :pull_request, iid: comment.iid, errors: e.message }
+          end
         end
-      rescue ActiveRecord::RecordInvalid => e
-        raise Projects::ImportService::Error, e.message
+      end
+
+      def generate_line_code(pr_comment)
+        Gitlab::Diff::LineCode.generate(pr_comment.file_path, pr_comment.new_pos, pr_comment.old_pos)
+      end
+
+      def pull_request_comment_attributes(comment)
+        {
+          project: project,
+          note: comment.note,
+          author_id: gitlab_user_id(project, comment.author),
+          created_at: comment.created_at,
+          updated_at: comment.updated_at
+        }
       end
     end
   end
diff --git a/lib/gitlab/bitbucket_import/key_adder.rb b/lib/gitlab/bitbucket_import/key_adder.rb
deleted file mode 100644
index 0b63f025d0a..00000000000
--- a/lib/gitlab/bitbucket_import/key_adder.rb
+++ /dev/null
@@ -1,24 +0,0 @@
-module Gitlab
-  module BitbucketImport
-    class KeyAdder
-      attr_reader :repo, :current_user, :client
-
-      def initialize(repo, current_user, access_params)
-        @repo, @current_user = repo, current_user
-        @client = Client.new(access_params[:bitbucket_access_token],
-                             access_params[:bitbucket_access_token_secret])
-      end
-
-      def execute
-        return false unless BitbucketImport.public_key.present?
-
-        project_identifier = "#{repo["owner"]}/#{repo["slug"]}"
-        client.add_deploy_key(project_identifier, BitbucketImport.public_key)
-
-        true
-      rescue
-        false
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/bitbucket_import/key_deleter.rb b/lib/gitlab/bitbucket_import/key_deleter.rb
deleted file mode 100644
index e03c3155b3e..00000000000
--- a/lib/gitlab/bitbucket_import/key_deleter.rb
+++ /dev/null
@@ -1,23 +0,0 @@
-module Gitlab
-  module BitbucketImport
-    class KeyDeleter
-      attr_reader :project, :current_user, :client
-
-      def initialize(project)
-        @project = project
-        @current_user = project.creator
-        @client = Client.from_project(@project)
-      end
-
-      def execute
-        return false unless BitbucketImport.public_key.present?
-
-        client.delete_deploy_key(project.import_source, BitbucketImport.public_key)
-
-        true
-      rescue
-        false
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb
index b90ef0b0fba..d94f70fd1fb 100644
--- a/lib/gitlab/bitbucket_import/project_creator.rb
+++ b/lib/gitlab/bitbucket_import/project_creator.rb
@@ -1,10 +1,11 @@
 module Gitlab
   module BitbucketImport
     class ProjectCreator
-      attr_reader :repo, :namespace, :current_user, :session_data
+      attr_reader :repo, :name, :namespace, :current_user, :session_data
 
-      def initialize(repo, namespace, current_user, session_data)
+      def initialize(repo, name, namespace, current_user, session_data)
         @repo = repo
+        @name = name
         @namespace = namespace
         @current_user = current_user
         @session_data = session_data
@@ -13,17 +14,24 @@ module Gitlab
       def execute
         ::Projects::CreateService.new(
           current_user,
-          name: repo["name"],
-          path: repo["slug"],
-          description: repo["description"],
+          name: name,
+          path: name,
+          description: repo.description,
           namespace_id: namespace.id,
-          visibility_level: repo["is_private"] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC,
-          import_type: "bitbucket",
-          import_source: "#{repo["owner"]}/#{repo["slug"]}",
-          import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git",
-          import_data: { credentials: { bb_session: session_data } }
+          visibility_level: repo.visibility_level,
+          import_type: 'bitbucket',
+          import_source: repo.full_name,
+          import_url: repo.clone_url(session_data[:token]),
+          import_data: { credentials: session_data },
+          skip_wiki: skip_wiki
         ).execute
       end
+
+      private
+
+      def skip_wiki
+        repo.has_wiki?
+      end
     end
   end
 end
diff --git a/lib/gitlab/chat_commands/base_command.rb b/lib/gitlab/chat_commands/base_command.rb
index 25da8474e95..4fe53ce93a9 100644
--- a/lib/gitlab/chat_commands/base_command.rb
+++ b/lib/gitlab/chat_commands/base_command.rb
@@ -42,6 +42,10 @@ module Gitlab
       def find_by_iid(iid)
         collection.find_by(iid: iid)
       end
+
+      def presenter
+        Gitlab::ChatCommands::Presenter.new
+      end
     end
   end
 end
diff --git a/lib/gitlab/chat_commands/command.rb b/lib/gitlab/chat_commands/command.rb
index b0d3fdbc48a..145086755e4 100644
--- a/lib/gitlab/chat_commands/command.rb
+++ b/lib/gitlab/chat_commands/command.rb
@@ -22,8 +22,6 @@ module Gitlab
         end
       end
 
-      private
-
       def match_command
         match = nil
         service = available_commands.find do |klass|
@@ -33,6 +31,8 @@ module Gitlab
         [service, match]
       end
 
+      private
+
       def help_messages
         available_commands.map(&:help_message)
       end
@@ -48,15 +48,15 @@ module Gitlab
       end
 
       def help(messages)
-        Mattermost::Presenter.help(messages, params[:command])
+        presenter.help(messages, params[:command])
       end
 
       def access_denied
-        Mattermost::Presenter.access_denied
+        presenter.access_denied
       end
 
       def present(resource)
-        Mattermost::Presenter.present(resource)
+        presenter.present(resource)
       end
     end
   end
diff --git a/lib/gitlab/chat_commands/deploy.rb b/lib/gitlab/chat_commands/deploy.rb
index 0eed1fce0dc..6bb854dc080 100644
--- a/lib/gitlab/chat_commands/deploy.rb
+++ b/lib/gitlab/chat_commands/deploy.rb
@@ -4,7 +4,7 @@ module Gitlab
       include Gitlab::Routing.url_helpers
 
       def self.match(text)
-        /\Adeploy\s+(?<from>.*)\s+to+\s+(?<to>.*)\z/.match(text)
+        /\Adeploy\s+(?<from>\S+.*)\s+to+\s+(?<to>\S+.*)\z/.match(text)
       end
 
       def self.help_message
diff --git a/lib/gitlab/chat_commands/issue_create.rb b/lib/gitlab/chat_commands/issue_create.rb
index 1dba85c1b51..cefb6775db8 100644
--- a/lib/gitlab/chat_commands/issue_create.rb
+++ b/lib/gitlab/chat_commands/issue_create.rb
@@ -8,7 +8,7 @@ module Gitlab
       end
 
       def self.help_message
-        'issue new <title>\n<description>'
+        'issue new <title> *`⇧ Shift`*+*`↵ Enter`* <description>'
       end
 
       def self.allowed?(project, user)
diff --git a/lib/mattermost/presenter.rb b/lib/gitlab/chat_commands/presenter.rb
index 67eda983a74..caceaa25391 100644
--- a/lib/mattermost/presenter.rb
+++ b/lib/gitlab/chat_commands/presenter.rb
@@ -1,7 +1,7 @@
-module Mattermost
-  class Presenter
-    class << self
-      include Gitlab::Routing.url_helpers
+module Gitlab
+  module ChatCommands
+    class Presenter
+      include Gitlab::Routing
 
       def authorize_chat_name(url)
         message = if url
@@ -64,7 +64,7 @@ module Mattermost
       def single_resource(resource)
         return error(resource) if resource.errors.any? || !resource.persisted?
 
-        message = "### #{title(resource)}"
+        message = "#{title(resource)}:"
         message << "\n\n#{resource.description}" if resource.try(:description)
 
         in_channel_response(message)
diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb
index 6b6a86ffde9..9c391fa92a3 100644
--- a/lib/gitlab/checks/change_access.rb
+++ b/lib/gitlab/checks/change_access.rb
@@ -4,11 +4,12 @@ module Gitlab
       attr_reader :user_access, :project, :skip_authorization
 
       def initialize(
-        change, user_access:, project:, skip_authorization: false)
+        change, user_access:, project:, env: {}, skip_authorization: false)
         @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref)
         @branch_name = Gitlab::Git.branch_name(@ref)
         @user_access = user_access
         @project = project
+        @env = env
         @skip_authorization = skip_authorization
       end
 
@@ -75,7 +76,7 @@ module Gitlab
       end
 
       def forced_push?
-        Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev)
+        Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev, env: @env)
       end
 
       def matching_merge_request?
diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb
index 5fe86553bd0..de0c9049ebf 100644
--- a/lib/gitlab/checks/force_push.rb
+++ b/lib/gitlab/checks/force_push.rb
@@ -1,15 +1,20 @@
 module Gitlab
   module Checks
     class ForcePush
-      def self.force_push?(project, oldrev, newrev)
+      def self.force_push?(project, oldrev, newrev, env: {})
         return false if project.empty_repo?
 
         # Created or deleted branch
         if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev)
           false
         else
-          missed_ref, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --git-dir=#{project.repository.path_to_repo} rev-list --max-count=1 #{oldrev} ^#{newrev}))
-          missed_ref.present?
+          missed_ref, exit_status = Gitlab::Git::RevList.new(oldrev, newrev, project: project, env: env).execute
+
+          if exit_status == 0
+            missed_ref.present?
+          else
+            raise "Got a non-zero exit code while calling out to `git rev-list` in the force-push check."
+          end
         end
       end
     end
diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb
new file mode 100644
index 00000000000..a979fe7d573
--- /dev/null
+++ b/lib/gitlab/ci/status/build/cancelable.rb
@@ -0,0 +1,37 @@
+module Gitlab
+  module Ci
+    module Status
+      module Build
+        class Cancelable < SimpleDelegator
+          include Status::Extended
+
+          def has_action?
+            can?(user, :update_build, subject)
+          end
+
+          def action_icon
+            'ban'
+          end
+
+          def action_path
+            cancel_namespace_project_build_path(subject.project.namespace,
+                                                subject.project,
+                                                subject)
+          end
+
+          def action_method
+            :post
+          end
+
+          def action_title
+            'Cancel'
+          end
+
+          def self.matches?(build, user)
+            build.cancelable?
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/build/common.rb b/lib/gitlab/ci/status/build/common.rb
new file mode 100644
index 00000000000..3fec2c5d4db
--- /dev/null
+++ b/lib/gitlab/ci/status/build/common.rb
@@ -0,0 +1,19 @@
+module Gitlab
+  module Ci
+    module Status
+      module Build
+        module Common
+          def has_details?
+            can?(user, :read_build, subject)
+          end
+
+          def details_path
+            namespace_project_build_path(subject.project.namespace,
+                                         subject.project,
+                                         subject)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb
new file mode 100644
index 00000000000..eee9a64120b
--- /dev/null
+++ b/lib/gitlab/ci/status/build/factory.rb
@@ -0,0 +1,18 @@
+module Gitlab
+  module Ci
+    module Status
+      module Build
+        class Factory < Status::Factory
+          def self.extended_statuses
+            [Status::Build::Stop, Status::Build::Play,
+             Status::Build::Cancelable, Status::Build::Retryable]
+          end
+
+          def self.common_helpers
+            Status::Build::Common
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb
new file mode 100644
index 00000000000..1bf949c96dd
--- /dev/null
+++ b/lib/gitlab/ci/status/build/play.rb
@@ -0,0 +1,57 @@
+module Gitlab
+  module Ci
+    module Status
+      module Build
+        class Play < SimpleDelegator
+          include Status::Extended
+
+          def text
+            'manual'
+          end
+
+          def label
+            'manual play action'
+          end
+
+          def icon
+            'icon_status_manual'
+          end
+
+          def group
+            'manual'
+          end
+
+          def has_action?
+            can?(user, :update_build, subject)
+          end
+
+          def action_icon
+            'play'
+          end
+
+          def action_title
+            'Play'
+          end
+
+          def action_class
+            'ci-play-icon'
+          end
+
+          def action_path
+            play_namespace_project_build_path(subject.project.namespace,
+                                              subject.project,
+                                              subject)
+          end
+
+          def action_method
+            :post
+          end
+
+          def self.matches?(build, user)
+            build.playable? && !build.stops_environment?
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb
new file mode 100644
index 00000000000..8e38d6a8523
--- /dev/null
+++ b/lib/gitlab/ci/status/build/retryable.rb
@@ -0,0 +1,37 @@
+module Gitlab
+  module Ci
+    module Status
+      module Build
+        class Retryable < SimpleDelegator
+          include Status::Extended
+
+          def has_action?
+            can?(user, :update_build, subject)
+          end
+
+          def action_icon
+            'refresh'
+          end
+
+          def action_title
+            'Retry'
+          end
+
+          def action_path
+            retry_namespace_project_build_path(subject.project.namespace,
+                                               subject.project,
+                                               subject)
+          end
+
+          def action_method
+            :post
+          end
+
+          def self.matches?(build, user)
+            build.retryable?
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb
new file mode 100644
index 00000000000..e1dfdb76d41
--- /dev/null
+++ b/lib/gitlab/ci/status/build/stop.rb
@@ -0,0 +1,53 @@
+module Gitlab
+  module Ci
+    module Status
+      module Build
+        class Stop < SimpleDelegator
+          include Status::Extended
+
+          def text
+            'manual'
+          end
+
+          def label
+            'manual stop action'
+          end
+
+          def icon
+            'icon_status_manual'
+          end
+
+          def group
+            'manual'
+          end
+
+          def has_action?
+            can?(user, :update_build, subject)
+          end
+
+          def action_icon
+            'stop'
+          end
+
+          def action_title
+            'Stop'
+          end
+
+          def action_path
+            play_namespace_project_build_path(subject.project.namespace,
+                                              subject.project,
+                                              subject)
+          end
+
+          def action_method
+            :post
+          end
+
+          def self.matches?(build, user)
+            build.playable? && build.stops_environment?
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb
index ce4108fdcf2..73b6ab5a635 100644
--- a/lib/gitlab/ci/status/core.rb
+++ b/lib/gitlab/ci/status/core.rb
@@ -4,10 +4,14 @@ module Gitlab
       # Base abstract class fore core status
       #
       class Core
-        include Gitlab::Routing.url_helpers
+        include Gitlab::Routing
+        include Gitlab::Allowable
 
-        def initialize(subject)
+        attr_reader :subject, :user
+
+        def initialize(subject, user)
           @subject = subject
+          @user = user
         end
 
         def icon
@@ -18,23 +22,12 @@ module Gitlab
           raise NotImplementedError
         end
 
-        def title
-          "#{@subject.class.name.demodulize}: #{label}"
-        end
-
-        # Deprecation warning: this method is here because we need to maintain
-        # backwards compatibility with legacy statuses. We often do something
-        # like "ci-status ci-status-#{status}" to set CSS class.
-        #
-        # `to_s` method should be renamed to `group` at some point, after
-        # phasing legacy satuses out.
-        #
-        def to_s
-          self.class.name.demodulize.downcase.underscore
+        def group
+          self.class.name.demodulize.underscore
         end
 
         def has_details?
-          raise NotImplementedError
+          false
         end
 
         def details_path
@@ -42,16 +35,27 @@ module Gitlab
         end
 
         def has_action?
-          raise NotImplementedError
+          false
         end
 
         def action_icon
           raise NotImplementedError
         end
 
+        def action_class
+        end
+
         def action_path
           raise NotImplementedError
         end
+
+        def action_method
+          raise NotImplementedError
+        end
+
+        def action_title
+          raise NotImplementedError
+        end
       end
     end
   end
diff --git a/lib/gitlab/ci/status/extended.rb b/lib/gitlab/ci/status/extended.rb
index 6bfb5d38c1f..d367c9bda69 100644
--- a/lib/gitlab/ci/status/extended.rb
+++ b/lib/gitlab/ci/status/extended.rb
@@ -2,8 +2,12 @@ module Gitlab
   module Ci
     module Status
       module Extended
-        def matches?(_subject)
-          raise NotImplementedError
+        extend ActiveSupport::Concern
+
+        class_methods do
+          def matches?(_subject, _user)
+            raise NotImplementedError
+          end
         end
       end
     end
diff --git a/lib/gitlab/ci/status/factory.rb b/lib/gitlab/ci/status/factory.rb
index b2f896f2211..ae9ef895df4 100644
--- a/lib/gitlab/ci/status/factory.rb
+++ b/lib/gitlab/ci/status/factory.rb
@@ -2,10 +2,9 @@ module Gitlab
   module Ci
     module Status
       class Factory
-        attr_reader :subject
-
-        def initialize(subject)
+        def initialize(subject, user)
           @subject = subject
+          @user = user
         end
 
         def fabricate!
@@ -16,27 +15,32 @@ module Gitlab
           end
         end
 
+        def self.extended_statuses
+          []
+        end
+
+        def self.common_helpers
+          Module.new
+        end
+
         private
 
-        def subject_status
-          @subject_status ||= subject.status
+        def simple_status
+          @simple_status ||= @subject.status || :created
         end
 
         def core_status
           Gitlab::Ci::Status
-            .const_get(subject_status.capitalize)
-            .new(subject)
+            .const_get(simple_status.capitalize)
+            .new(@subject, @user)
+            .extend(self.class.common_helpers)
         end
 
         def extended_status
-          @extended ||= extended_statuses.find do |status|
-            status.matches?(subject)
+          @extended ||= self.class.extended_statuses.find do |status|
+            status.matches?(@subject, @user)
           end
         end
-
-        def extended_statuses
-          []
-        end
       end
     end
   end
diff --git a/lib/gitlab/ci/status/pipeline/common.rb b/lib/gitlab/ci/status/pipeline/common.rb
index 25e52bec3da..76bfd18bf40 100644
--- a/lib/gitlab/ci/status/pipeline/common.rb
+++ b/lib/gitlab/ci/status/pipeline/common.rb
@@ -4,13 +4,13 @@ module Gitlab
       module Pipeline
         module Common
           def has_details?
-            true
+            can?(user, :read_pipeline, subject)
           end
 
           def details_path
-            namespace_project_pipeline_path(@subject.project.namespace,
-                                            @subject.project,
-                                            @subject)
+            namespace_project_pipeline_path(subject.project.namespace,
+                                            subject.project,
+                                            subject)
           end
 
           def has_action?
diff --git a/lib/gitlab/ci/status/pipeline/factory.rb b/lib/gitlab/ci/status/pipeline/factory.rb
index 4ac4ec671d0..16dcb326be9 100644
--- a/lib/gitlab/ci/status/pipeline/factory.rb
+++ b/lib/gitlab/ci/status/pipeline/factory.rb
@@ -3,14 +3,12 @@ module Gitlab
     module Status
       module Pipeline
         class Factory < Status::Factory
-          private
-
-          def extended_statuses
+          def self.extended_statuses
             [Pipeline::SuccessWithWarnings]
           end
 
-          def core_status
-            super.extend(Status::Pipeline::Common)
+          def self.common_helpers
+            Status::Pipeline::Common
           end
         end
       end
diff --git a/lib/gitlab/ci/status/pipeline/success_with_warnings.rb b/lib/gitlab/ci/status/pipeline/success_with_warnings.rb
index 4b040d60df8..24bf8b869e0 100644
--- a/lib/gitlab/ci/status/pipeline/success_with_warnings.rb
+++ b/lib/gitlab/ci/status/pipeline/success_with_warnings.rb
@@ -3,7 +3,7 @@ module Gitlab
     module Status
       module Pipeline
         class SuccessWithWarnings < SimpleDelegator
-          extend Status::Extended
+          include Status::Extended
 
           def text
             'passed'
@@ -17,11 +17,11 @@ module Gitlab
             'icon_status_warning'
           end
 
-          def to_s
+          def group
             'success_with_warnings'
           end
 
-          def self.matches?(pipeline)
+          def self.matches?(pipeline, user)
             pipeline.success? && pipeline.has_warnings?
           end
         end
diff --git a/lib/gitlab/ci/status/stage/common.rb b/lib/gitlab/ci/status/stage/common.rb
index 14c437d2b98..7852f492e1d 100644
--- a/lib/gitlab/ci/status/stage/common.rb
+++ b/lib/gitlab/ci/status/stage/common.rb
@@ -4,14 +4,14 @@ module Gitlab
       module Stage
         module Common
           def has_details?
-            true
+            can?(user, :read_pipeline, subject.pipeline)
           end
 
           def details_path
-            namespace_project_pipeline_path(@subject.project.namespace,
-                                            @subject.project,
-                                            @subject.pipeline,
-                                            anchor: @subject.name)
+            namespace_project_pipeline_path(subject.project.namespace,
+                                            subject.project,
+                                            subject.pipeline,
+                                            anchor: subject.name)
           end
 
           def has_action?
diff --git a/lib/gitlab/ci/status/stage/factory.rb b/lib/gitlab/ci/status/stage/factory.rb
index c6522d5ada1..689a5dd45bc 100644
--- a/lib/gitlab/ci/status/stage/factory.rb
+++ b/lib/gitlab/ci/status/stage/factory.rb
@@ -3,10 +3,8 @@ module Gitlab
     module Status
       module Stage
         class Factory < Status::Factory
-          private
-
-          def core_status
-            super.extend(Status::Stage::Common)
+          def self.common_helpers
+            Status::Stage::Common
           end
         end
       end
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index c6bb8f9c8ed..9d142f1b82e 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -45,7 +45,7 @@ module Gitlab
         default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'],
         default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'],
         domain_whitelist: Settings.gitlab['domain_whitelist'],
-        import_sources: %w[github bitbucket gitlab google_code fogbugz git gitlab_project],
+        import_sources: %w[gitea github bitbucket gitlab google_code fogbugz git gitlab_project],
         shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
         max_artifacts_size: Settings.artifacts['max_size'],
         require_two_factor_authentication: false,
diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb
index 85402c2a278..f586c5ab062 100644
--- a/lib/gitlab/email/reply_parser.rb
+++ b/lib/gitlab/email/reply_parser.rb
@@ -69,7 +69,7 @@ module Gitlab
           # This one might be controversial but so many reply lines have years, times and end with a colon.
           # Let's try it and see how well it works.
           break if (l =~ /\d{4}/ && l =~ /\d:\d\d/ && l =~ /\:$/) ||
-                   (l =~ /On \w+ \d+,? \d+,?.*wrote:/)
+              (l =~ /On \w+ \d+,? \d+,?.*wrote:/)
 
           # Headers on subsequent lines
           break if (0..2).all? { |off| lines[idx + off] =~ REPLYING_HEADER_REGEX }
diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb
index abc8c8c55e6..8fab5489616 100644
--- a/lib/gitlab/gfm/uploads_rewriter.rb
+++ b/lib/gitlab/gfm/uploads_rewriter.rb
@@ -1,3 +1,5 @@
+require 'fileutils'
+
 module Gitlab
   module Gfm
     ##
@@ -22,7 +24,9 @@ module Gitlab
           return markdown unless file.try(:exists?)
 
           new_uploader = FileUploader.new(target_project)
-          new_uploader.store!(file)
+          with_link_in_tmp_dir(file.file) do |open_tmp_file|
+            new_uploader.store!(open_tmp_file)
+          end
           new_uploader.to_markdown
         end
       end
@@ -46,6 +50,19 @@ module Gitlab
         uploader.retrieve_from_store!(file)
         uploader.file
       end
+
+      # Because the uploaders use 'move_to_store' we must have a temporary
+      # file that is allowed to be (re)moved.
+      def with_link_in_tmp_dir(file)
+        dir = Dir.mktmpdir('UploadsRewriter', File.dirname(file))
+        # The filename matters to Carrierwave so we make sure to preserve it
+        tmp_file = File.join(dir, File.basename(file))
+        File.link(file, tmp_file)
+        # Open the file to placate Carrierwave
+        File.open(tmp_file) { |open_file| yield open_file }
+      ensure
+        FileUtils.rm_rf(dir)
+      end
     end
   end
 end
diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb
new file mode 100644
index 00000000000..79dd0cf7df2
--- /dev/null
+++ b/lib/gitlab/git/rev_list.rb
@@ -0,0 +1,42 @@
+module Gitlab
+  module Git
+    class RevList
+      attr_reader :project, :env
+
+      ALLOWED_VARIABLES = %w[GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES].freeze
+
+      def initialize(oldrev, newrev, project:, env: nil)
+        @project = project
+        @env = env.presence || {}
+        @args = [Gitlab.config.git.bin_path,
+                 "--git-dir=#{project.repository.path_to_repo}",
+                 "rev-list",
+                 "--max-count=1",
+                 oldrev,
+                 "^#{newrev}"]
+      end
+
+      def execute
+        Gitlab::Popen.popen(@args, nil, parse_environment_variables)
+      end
+
+      def valid?
+        environment_variables.all? do |(name, value)|
+          value.to_s.start_with?(project.repository.path_to_repo)
+        end
+      end
+
+      private
+
+      def parse_environment_variables
+        return {} unless valid?
+
+        environment_variables
+      end
+
+      def environment_variables
+        @environment_variables ||= env.slice(*ALLOWED_VARIABLES).compact
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index 13efc1ed73d..545506f3dfd 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -18,12 +18,13 @@ module Gitlab
 
     attr_reader :actor, :project, :protocol, :user_access, :authentication_abilities
 
-    def initialize(actor, project, protocol, authentication_abilities:)
+    def initialize(actor, project, protocol, authentication_abilities:, env: {})
       @actor    = actor
       @project  = project
       @protocol = protocol
       @authentication_abilities = authentication_abilities
       @user_access = UserAccess.new(user, project: project)
+      @env = env
     end
 
     def check(cmd, changes)
@@ -147,6 +148,7 @@ module Gitlab
         change,
         user_access: user_access,
         project: project,
+        env: @env,
         skip_authorization: deploy_key?).exec
     end
 
diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb
index 6dbae64a9fe..95dba9a327b 100644
--- a/lib/gitlab/github_import/base_formatter.rb
+++ b/lib/gitlab/github_import/base_formatter.rb
@@ -15,6 +15,10 @@ module Gitlab
         end
       end
 
+      def url
+        raw_data.url || ''
+      end
+
       private
 
       def gitlab_user_id(github_id)
diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb
index 85df6547a67..ba869faa92e 100644
--- a/lib/gitlab/github_import/client.rb
+++ b/lib/gitlab/github_import/client.rb
@@ -4,10 +4,12 @@ module Gitlab
       GITHUB_SAFE_REMAINING_REQUESTS = 100
       GITHUB_SAFE_SLEEP_TIME = 500
 
-      attr_reader :access_token
+      attr_reader :access_token, :host, :api_version
 
-      def initialize(access_token)
+      def initialize(access_token, host: nil, api_version: 'v3')
         @access_token = access_token
+        @host = host.to_s.sub(%r{/+\z}, '')
+        @api_version = api_version
 
         if access_token
           ::Octokit.auto_paginate = false
@@ -17,7 +19,7 @@ module Gitlab
       def api
         @api ||= ::Octokit::Client.new(
           access_token: access_token,
-          api_endpoint: github_options[:site],
+          api_endpoint: api_endpoint,
           # If there is no config, we're connecting to github.com and we
           # should verify ssl.
           connection_options: {
@@ -64,6 +66,14 @@ module Gitlab
 
       private
 
+      def api_endpoint
+        if host.present? && api_version.present?
+          "#{host}/api/#{api_version}"
+        else
+          github_options[:site]
+        end
+      end
+
       def config
         Gitlab.config.omniauth.providers.find { |provider| provider.name == "github" }
       end
diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index 281b65bdeba..ec1318ab33c 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -3,7 +3,7 @@ module Gitlab
     class Importer
       include Gitlab::ShellAdapter
 
-      attr_reader :client, :errors, :project, :repo, :repo_url
+      attr_reader :errors, :project, :repo, :repo_url
 
       def initialize(project)
         @project  = project
@@ -11,12 +11,27 @@ module Gitlab
         @repo_url = project.import_url
         @errors   = []
         @labels   = {}
+      end
+
+      def client
+        return @client if defined?(@client)
+        unless credentials
+          raise Projects::ImportService::Error,
+                "Unable to find project import data credentials for project ID: #{@project.id}"
+        end
 
-        if credentials
-          @client = Client.new(credentials[:user])
-        else
-          raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}"
+        opts = {}
+        # Gitea plan to be GitHub compliant
+        if project.gitea_import?
+          uri = URI.parse(project.import_url)
+          host = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}".sub(%r{/?[\w-]+/[\w-]+\.git\z}, '')
+          opts = {
+            host: host,
+            api_version: 'v1'
+          }
         end
+
+        @client = Client.new(credentials[:user], opts)
       end
 
       def execute
@@ -35,7 +50,13 @@ module Gitlab
         import_comments(:issues)
         import_comments(:pull_requests)
         import_wiki
-        import_releases
+
+        # Gitea doesn't have a Release API yet
+        # See https://github.com/go-gitea/gitea/issues/330
+        unless project.gitea_import?
+          import_releases
+        end
+
         handle_errors
 
         true
@@ -44,7 +65,9 @@ module Gitlab
       private
 
       def credentials
-        @credentials ||= project.import_data.credentials if project.import_data
+        return @credentials if defined?(@credentials)
+
+        @credentials = project.import_data ? project.import_data.credentials : nil
       end
 
       def handle_errors
@@ -60,9 +83,10 @@ module Gitlab
         fetch_resources(:labels, repo, per_page: 100) do |labels|
           labels.each do |raw|
             begin
-              LabelFormatter.new(project, raw).create!
+              gh_label = LabelFormatter.new(project, raw)
+              gh_label.create!
             rescue => e
-              errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
+              errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(gh_label.url), errors: e.message }
             end
           end
         end
@@ -74,9 +98,10 @@ module Gitlab
         fetch_resources(:milestones, repo, state: :all, per_page: 100) do |milestones|
           milestones.each do |raw|
             begin
-              MilestoneFormatter.new(project, raw).create!
+              gh_milestone = MilestoneFormatter.new(project, raw)
+              gh_milestone.create!
             rescue => e
-              errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
+              errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(gh_milestone.url), errors: e.message }
             end
           end
         end
@@ -97,7 +122,7 @@ module Gitlab
 
               apply_labels(issuable, raw)
             rescue => e
-              errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
+              errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(gh_issue.url), errors: e.message }
             end
           end
         end
@@ -106,18 +131,23 @@ module Gitlab
       def import_pull_requests
         fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests|
           pull_requests.each do |raw|
-            pull_request = PullRequestFormatter.new(project, raw)
-            next unless pull_request.valid?
+            gh_pull_request = PullRequestFormatter.new(project, raw)
+            next unless gh_pull_request.valid?
 
             begin
-              restore_source_branch(pull_request) unless pull_request.source_branch_exists?
-              restore_target_branch(pull_request) unless pull_request.target_branch_exists?
+              restore_source_branch(gh_pull_request) unless gh_pull_request.source_branch_exists?
+              restore_target_branch(gh_pull_request) unless gh_pull_request.target_branch_exists?
+
+              merge_request = gh_pull_request.create!
 
-              pull_request.create!
+              # Gitea doesn't return PR in the Issue API endpoint, so labels must be assigned at this stage
+              if project.gitea_import?
+                apply_labels(merge_request, raw)
+              end
             rescue => e
-              errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(pull_request.url), errors: e.message }
+              errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(gh_pull_request.url), errors: e.message }
             ensure
-              clean_up_restored_branches(pull_request)
+              clean_up_restored_branches(gh_pull_request)
             end
           end
         end
@@ -233,7 +263,7 @@ module Gitlab
               gh_release = ReleaseFormatter.new(project, raw)
               gh_release.create! if gh_release.valid?
             rescue => e
-              errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
+              errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(gh_release.url), errors: e.message }
             end
           end
         end
diff --git a/lib/gitlab/github_import/issuable_formatter.rb b/lib/gitlab/github_import/issuable_formatter.rb
new file mode 100644
index 00000000000..256f360efc7
--- /dev/null
+++ b/lib/gitlab/github_import/issuable_formatter.rb
@@ -0,0 +1,60 @@
+module Gitlab
+  module GithubImport
+    class IssuableFormatter < BaseFormatter
+      def project_association
+        raise NotImplementedError
+      end
+
+      def number
+        raw_data.number
+      end
+
+      def find_condition
+        { iid: number }
+      end
+
+      private
+
+      def state
+        raw_data.state == 'closed' ? 'closed' : 'opened'
+      end
+
+      def assigned?
+        raw_data.assignee.present?
+      end
+
+      def assignee_id
+        if assigned?
+          gitlab_user_id(raw_data.assignee.id)
+        end
+      end
+
+      def author
+        raw_data.user.login
+      end
+
+      def author_id
+        gitlab_author_id || project.creator_id
+      end
+
+      def body
+        raw_data.body || ""
+      end
+
+      def description
+        if gitlab_author_id
+          body
+        else
+          formatter.author_line(author) + body
+        end
+      end
+
+      def milestone
+        if raw_data.milestone.present?
+          milestone = MilestoneFormatter.new(project, raw_data.milestone)
+          project.milestones.find_by(milestone.find_condition)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb
index 887690bcc7c..6f5ac4dac0d 100644
--- a/lib/gitlab/github_import/issue_formatter.rb
+++ b/lib/gitlab/github_import/issue_formatter.rb
@@ -1,6 +1,6 @@
 module Gitlab
   module GithubImport
-    class IssueFormatter < BaseFormatter
+    class IssueFormatter < IssuableFormatter
       def attributes
         {
           iid: number,
@@ -24,59 +24,9 @@ module Gitlab
         :issues
       end
 
-      def find_condition
-        { iid: number }
-      end
-
-      def number
-        raw_data.number
-      end
-
       def pull_request?
         raw_data.pull_request.present?
       end
-
-      private
-
-      def assigned?
-        raw_data.assignee.present?
-      end
-
-      def assignee_id
-        if assigned?
-          gitlab_user_id(raw_data.assignee.id)
-        end
-      end
-
-      def author
-        raw_data.user.login
-      end
-
-      def author_id
-        gitlab_author_id || project.creator_id
-      end
-
-      def body
-        raw_data.body || ""
-      end
-
-      def description
-        if gitlab_author_id
-          body
-        else
-          formatter.author_line(author) + body
-        end
-      end
-
-      def milestone
-        if raw_data.milestone.present?
-          project.milestones.find_by(iid: raw_data.milestone.number)
-        end
-      end
-
-      def state
-        raw_data.state == 'closed' ? 'closed' : 'opened'
-      end
     end
   end
 end
diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/github_import/milestone_formatter.rb
index 401dd962521..dd782eff059 100644
--- a/lib/gitlab/github_import/milestone_formatter.rb
+++ b/lib/gitlab/github_import/milestone_formatter.rb
@@ -3,7 +3,7 @@ module Gitlab
     class MilestoneFormatter < BaseFormatter
       def attributes
         {
-          iid: raw_data.number,
+          iid: number,
           project: project,
           title: raw_data.title,
           description: raw_data.description,
@@ -19,7 +19,15 @@ module Gitlab
       end
 
       def find_condition
-        { iid: raw_data.number }
+        { iid: number }
+      end
+
+      def number
+        if project.gitea_import?
+          raw_data.id
+        else
+          raw_data.number
+        end
       end
 
       private
diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb
index a2410068845..3f635be22ba 100644
--- a/lib/gitlab/github_import/project_creator.rb
+++ b/lib/gitlab/github_import/project_creator.rb
@@ -1,14 +1,15 @@
 module Gitlab
   module GithubImport
     class ProjectCreator
-      attr_reader :repo, :name, :namespace, :current_user, :session_data
+      attr_reader :repo, :name, :namespace, :current_user, :session_data, :type
 
-      def initialize(repo, name, namespace, current_user, session_data)
+      def initialize(repo, name, namespace, current_user, session_data, type: 'github')
         @repo = repo
         @name = name
         @namespace = namespace
         @current_user = current_user
         @session_data = session_data
+        @type = type
       end
 
       def execute
@@ -19,7 +20,7 @@ module Gitlab
           description: repo.description,
           namespace_id: namespace.id,
           visibility_level: visibility_level,
-          import_type: "github",
+          import_type: type,
           import_source: repo.full_name,
           import_url: import_url,
           skip_wiki: skip_wiki
@@ -29,7 +30,7 @@ module Gitlab
       private
 
       def import_url
-        repo.clone_url.sub('https://', "https://#{session_data[:github_access_token]}@")
+        repo.clone_url.sub('://', "://#{session_data[:github_access_token]}@")
       end
 
       def visibility_level
diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb
index b9a227fb11a..4ea0200e89b 100644
--- a/lib/gitlab/github_import/pull_request_formatter.rb
+++ b/lib/gitlab/github_import/pull_request_formatter.rb
@@ -1,6 +1,6 @@
 module Gitlab
   module GithubImport
-    class PullRequestFormatter < BaseFormatter
+    class PullRequestFormatter < IssuableFormatter
       delegate :exists?, :project, :ref, :repo, :sha, to: :source_branch, prefix: true
       delegate :exists?, :project, :ref, :repo, :sha, to: :target_branch, prefix: true
 
@@ -28,14 +28,6 @@ module Gitlab
         :merge_requests
       end
 
-      def find_condition
-        { iid: number }
-      end
-
-      def number
-        raw_data.number
-      end
-
       def valid?
         source_branch.valid? && target_branch.valid?
       end
@@ -60,57 +52,15 @@ module Gitlab
         end
       end
 
-      def url
-        raw_data.url
-      end
-
       private
 
-      def assigned?
-        raw_data.assignee.present?
-      end
-
-      def assignee_id
-        if assigned?
-          gitlab_user_id(raw_data.assignee.id)
-        end
-      end
-
-      def author
-        raw_data.user.login
-      end
-
-      def author_id
-        gitlab_author_id || project.creator_id
-      end
-
-      def body
-        raw_data.body || ""
-      end
-
-      def description
-        if gitlab_author_id
-          body
+      def state
+        if raw_data.state == 'closed' && raw_data.merged_at.present?
+          'merged'
         else
-          formatter.author_line(author) + body
+          super
         end
       end
-
-      def milestone
-        if raw_data.milestone.present?
-          project.milestones.find_by(iid: raw_data.milestone.number)
-        end
-      end
-
-      def state
-        @state ||= if raw_data.state == 'closed' && raw_data.merged_at.present?
-                     'merged'
-                   elsif raw_data.state == 'closed'
-                     'closed'
-                   else
-                     'opened'
-                   end
-      end
     end
   end
 end
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index 2c21804fe7a..4d4e04e9e35 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -8,6 +8,8 @@ module Gitlab
       gon.shortcuts_path         = help_page_path('shortcuts')
       gon.user_color_scheme      = Gitlab::ColorSchemes.for_user(current_user).css_class
       gon.award_menu_url         = emojis_path
+      gon.katex_css_url          = ActionController::Base.helpers.asset_path('katex.css')
+      gon.katex_js_url           = ActionController::Base.helpers.asset_path('katex.js')
 
       if current_user
         gon.current_user_id = current_user.id
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index c551321c18d..cda6ddf0443 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -120,7 +120,7 @@ module Gitlab
                                                        members_mapper: members_mapper,
                                                        user: @user,
                                                        project_id: restored_project.id)
-        end
+        end.compact
 
         relation_hash_list.is_a?(Array) ? relation_array : relation_array.first
       end
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index a0e80fccad9..65b229ca8ff 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -14,7 +14,7 @@ module Gitlab
                     priorities: :label_priorities,
                     label: :project_label }.freeze
 
-      USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id].freeze
+      USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id merge_user_id].freeze
 
       PROJECT_REFERENCES = %w[project_id source_project_id gl_project_id target_project_id].freeze
 
@@ -40,6 +40,8 @@ module Gitlab
       # the relation_hash, updating references with new object IDs, mapping users using
       # the "members_mapper" object, also updating notes if required.
       def create
+        return nil if unknown_service?
+
         setup_models
 
         generate_imported_object
@@ -99,6 +101,8 @@ module Gitlab
       def generate_imported_object
         if BUILD_MODELS.include?(@relation_name) # call #trace= method after assigning the other attributes
           trace = @relation_hash.delete('trace')
+          @relation_hash.delete('token')
+
           imported_object do |object|
             object.trace = trace
             object.commit_id = nil
@@ -215,6 +219,11 @@ module Gitlab
             existing_object
           end
       end
+
+      def unknown_service?
+        @relation_name == :services && parsed_relation_hash['type'] &&
+          !Object.const_defined?(parsed_relation_hash['type'])
+      end
     end
   end
 end
diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb
index 94261b7eeed..45958710c13 100644
--- a/lib/gitlab/import_sources.rb
+++ b/lib/gitlab/import_sources.rb
@@ -7,21 +7,38 @@ module Gitlab
   module ImportSources
     extend CurrentSettings
 
+    ImportSource = Struct.new(:name, :title, :importer)
+
+    ImportTable = [
+      ImportSource.new('github',         'GitHub',        Gitlab::GithubImport::Importer),
+      ImportSource.new('bitbucket',      'Bitbucket',     Gitlab::BitbucketImport::Importer),
+      ImportSource.new('gitlab',         'GitLab.com',    Gitlab::GitlabImport::Importer),
+      ImportSource.new('google_code',    'Google Code',   Gitlab::GoogleCodeImport::Importer),
+      ImportSource.new('fogbugz',        'FogBugz',       Gitlab::FogbugzImport::Importer),
+      ImportSource.new('git',            'Repo by URL',   nil),
+      ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer),
+      ImportSource.new('gitea',          'Gitea',         Gitlab::GithubImport::Importer)
+    ].freeze
+
     class << self
+      def options
+        @options ||= Hash[ImportTable.map { |importer| [importer.title, importer.name] }]
+      end
+
       def values
-        options.values
+        @values ||= ImportTable.map(&:name)
       end
 
-      def options
-        {
-          'GitHub'        => 'github',
-          'Bitbucket'     => 'bitbucket',
-          'GitLab.com'    => 'gitlab',
-          'Google Code'   => 'google_code',
-          'FogBugz'       => 'fogbugz',
-          'Repo by URL'   => 'git',
-          'GitLab export' => 'gitlab_project'
-        }
+      def importer_names
+        @importer_names ||= ImportTable.select(&:importer).map(&:name)
+      end
+
+      def importer(name)
+        ImportTable.find { |import_source| import_source.name == name }.importer
+      end
+
+      def title(name)
+        options.key(name)
       end
     end
   end
diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb
new file mode 100644
index 00000000000..288771c1c12
--- /dev/null
+++ b/lib/gitlab/kubernetes.rb
@@ -0,0 +1,80 @@
+module Gitlab
+  # Helper methods to do with Kubernetes network services & resources
+  module Kubernetes
+    # This is the comand that is run to start a terminal session. Kubernetes
+    # expects `command=foo&command=bar, not `command[]=foo&command[]=bar`
+    EXEC_COMMAND = URI.encode_www_form(
+      ['sh', '-c', 'bash || sh'].map { |value| ['command', value] }
+    )
+
+    # Filters an array of pods (as returned by the kubernetes API) by their labels
+    def filter_pods(pods, labels = {})
+      pods.select do |pod|
+        metadata = pod.fetch("metadata", {})
+        pod_labels = metadata.fetch("labels", nil)
+        next unless pod_labels
+
+        labels.all? { |k, v| pod_labels[k.to_s] == v }
+      end
+    end
+
+    # Converts a pod (as returned by the kubernetes API) into a terminal
+    def terminals_for_pod(api_url, namespace, pod)
+      metadata = pod.fetch("metadata", {})
+      status   = pod.fetch("status", {})
+      spec     = pod.fetch("spec", {})
+
+      containers = spec["containers"]
+      pod_name   = metadata["name"]
+      phase      = status["phase"]
+
+      return unless containers.present? && pod_name.present? && phase == "Running"
+
+      created_at = DateTime.parse(metadata["creationTimestamp"]) rescue nil
+
+      containers.map do |container|
+        {
+          selectors:    { pod: pod_name, container: container["name"] },
+          url:          container_exec_url(api_url, namespace, pod_name, container["name"]),
+          subprotocols: ['channel.k8s.io'],
+          headers:      Hash.new { |h, k| h[k] = [] },
+          created_at:   created_at,
+        }
+      end
+    end
+
+    def add_terminal_auth(terminal, token, ca_pem = nil)
+      terminal[:headers]['Authorization'] << "Bearer #{token}"
+      terminal[:ca_pem] = ca_pem if ca_pem.present?
+      terminal
+    end
+
+    def container_exec_url(api_url, namespace, pod_name, container_name)
+      url = URI.parse(api_url)
+      url.path = [
+        url.path.sub(%r{/+\z}, ''),
+        'api', 'v1',
+        'namespaces', ERB::Util.url_encode(namespace),
+        'pods', ERB::Util.url_encode(pod_name),
+        'exec'
+      ].join('/')
+
+      url.query = {
+        container: container_name,
+        tty: true,
+        stdin: true,
+        stdout: true,
+        stderr: true,
+      }.to_query + '&' + EXEC_COMMAND
+
+      case url.scheme
+      when 'http'
+        url.scheme = 'ws'
+      when 'https'
+        url.scheme = 'wss'
+      end
+
+      url.to_s
+    end
+  end
+end
diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb
new file mode 100644
index 00000000000..65713e73a59
--- /dev/null
+++ b/lib/gitlab/middleware/multipart.rb
@@ -0,0 +1,99 @@
+# Gitlab::Middleware::Multipart - a Rack::Multipart replacement
+#
+# Rack::Multipart leaves behind tempfiles in /tmp and uses valuable Ruby
+# process time to copy files around. This alternative solution uses
+# gitlab-workhorse to clean up the tempfiles and puts the tempfiles in a
+# location where copying should not be needed.
+#
+# When gitlab-workhorse finds files in a multipart MIME body it sends
+# a signed message via a request header. This message lists the names of
+# the multipart entries that gitlab-workhorse filtered out of the
+# multipart structure and saved to tempfiles. Workhorse adds new entries
+# in the multipart structure with paths to the tempfiles.
+#
+# The job of this Rack middleware is to detect and decode the message
+# from workhorse. If present, it walks the Rack 'params' hash for the
+# current request, opens the respective tempfiles, and inserts the open
+# Ruby File objects in the params hash where Rack::Multipart would have
+# put them. The goal is that application code deeper down can keep
+# working the way it did with Rack::Multipart without changes.
+#
+# CAVEAT: the code that modifies the params hash is a bit complex. It is
+# conceivable that certain Rack params structures will not be modified
+# correctly. We are not aware of such bugs at this time though.
+#
+
+module Gitlab
+  module Middleware
+    class Multipart
+      RACK_ENV_KEY = 'HTTP_GITLAB_WORKHORSE_MULTIPART_FIELDS'
+
+      class Handler
+        def initialize(env, message)
+          @request = Rack::Request.new(env)
+          @rewritten_fields = message['rewritten_fields']
+          @open_files = []
+        end
+
+        def with_open_files
+          @rewritten_fields.each do |field, tmp_path|
+            parsed_field = Rack::Utils.parse_nested_query(field)
+            raise "unexpected field: #{field.inspect}" unless parsed_field.count == 1
+
+            key, value = parsed_field.first
+            if value.nil?
+              value = File.open(tmp_path)
+              @open_files << value
+            else
+              value = decorate_params_value(value, @request.params[key], tmp_path)
+            end
+            @request.update_param(key, value)
+          end
+
+          yield
+        ensure
+          @open_files.each(&:close)
+        end
+
+        # This function calls itself recursively
+        def decorate_params_value(path_hash, value_hash, tmp_path)
+          unless path_hash.is_a?(Hash) && path_hash.count == 1
+            raise "invalid path: #{path_hash.inspect}"
+          end
+          path_key, path_value = path_hash.first
+
+          unless value_hash.is_a?(Hash) && value_hash[path_key]
+            raise "invalid value hash: #{value_hash.inspect}"
+          end
+
+          case path_value
+          when nil
+            value_hash[path_key] = File.open(tmp_path)
+            @open_files << value_hash[path_key]
+            value_hash
+          when Hash
+            decorate_params_value(path_value, value_hash[path_key], tmp_path)
+            value_hash
+          else
+            raise "unexpected path value: #{path_value.inspect}"
+          end
+        end
+      end
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        encoded_message = env.delete(RACK_ENV_KEY)
+        return @app.call(env) if encoded_message.blank?
+
+        message = Gitlab::Workhorse.decode_jwt(encoded_message)[0]
+
+        Handler.new(env, message).with_open_files do
+          @app.call(env)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/popen.rb b/lib/gitlab/popen.rb
index cc74bb29087..4bc5cda8cb5 100644
--- a/lib/gitlab/popen.rb
+++ b/lib/gitlab/popen.rb
@@ -5,13 +5,13 @@ module Gitlab
   module Popen
     extend self
 
-    def popen(cmd, path = nil)
+    def popen(cmd, path = nil, vars = {})
       unless cmd.is_a?(Array)
         raise "System commands must be given as an array of strings"
       end
 
       path ||= Dir.pwd
-      vars = { "PWD" => path }
+      vars['PWD'] = path
       options = { chdir: path }
 
       unless File.directory?(path)
diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb
index 66e6b29e798..6bdf3db9cb8 100644
--- a/lib/gitlab/project_search_results.rb
+++ b/lib/gitlab/project_search_results.rb
@@ -110,7 +110,7 @@ module Gitlab
     end
 
     def notes
-      @notes ||= project.notes.user.search(query, as_user: @current_user).order('updated_at DESC')
+      @notes ||= NotesFinder.new(project, @current_user, search: query).execute.user.order('updated_at DESC')
     end
 
     def commits
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index a06cf6a989c..9e0b0e5ea98 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -61,7 +61,7 @@ module Gitlab
     end
 
     def file_name_regex
-      @file_name_regex ||= /\A[a-zA-Z0-9_\-\.\@]*\z/.freeze
+      @file_name_regex ||= /\A[[[:alnum:]]_\-\.\@]*\z/.freeze
     end
 
     def file_name_regex_message
@@ -69,7 +69,7 @@ module Gitlab
     end
 
     def file_path_regex
-      @file_path_regex ||= /\A[a-zA-Z0-9_\-\.\/\@]*\z/.freeze
+      @file_path_regex ||= /\A[[[:alnum:]]_\-\.\/\@]*\z/.freeze
     end
 
     def file_path_regex_message
@@ -123,5 +123,22 @@ module Gitlab
     def environment_name_regex_message
       "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.' and spaces"
     end
+
+    def kubernetes_namespace_regex
+      /\A[a-z0-9]([-a-z0-9]*[a-z0-9])?\z/
+    end
+
+    def kubernetes_namespace_regex_message
+      "can contain only letters, digits or '-', and cannot start or end with '-'"
+    end
+
+    def environment_slug_regex
+      @environment_slug_regex ||= /\A[a-z]([a-z0-9-]*[a-z0-9])?\z/.freeze
+    end
+
+    def environment_slug_regex_message
+      "can contain only lowercase letters, digits, and '-'. " \
+      "Must start with a letter, and cannot end with '-'"
+    end
   end
 end
diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb
index 5132177de51..632e2d87500 100644
--- a/lib/gitlab/routing.rb
+++ b/lib/gitlab/routing.rb
@@ -1,5 +1,11 @@
 module Gitlab
   module Routing
+    extend ActiveSupport::Concern
+
+    included do
+      include Gitlab::Routing.url_helpers
+    end
+
     # Returns the URL helpers Module.
     #
     # This method caches the output as Rails' "url_helpers" method creates an
diff --git a/lib/gitlab/serialize/ci/variables.rb b/lib/gitlab/serialize/ci/variables.rb
new file mode 100644
index 00000000000..3a9443bfcd9
--- /dev/null
+++ b/lib/gitlab/serialize/ci/variables.rb
@@ -0,0 +1,27 @@
+module Gitlab
+  module Serialize
+    module Ci
+      # This serializer could make sure our YAML variables' keys and values
+      # are always strings. This is more for legacy build data because
+      # from now on we convert them into strings before saving to database.
+      module Variables
+        extend self
+
+        def load(string)
+          return unless string
+
+          object = YAML.safe_load(string, [Symbol])
+
+          object.map do |variable|
+            variable[:key] = variable[:key].to_s
+            variable
+          end
+        end
+
+        def dump(object)
+          YAML.dump(object)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb
index 1cd89b3a9c4..222021e8802 100644
--- a/lib/gitlab/sql/union.rb
+++ b/lib/gitlab/sql/union.rb
@@ -22,9 +22,7 @@ module Gitlab
         # By using "unprepared_statements" we remove the usage of placeholders
         # (thus fixing this problem), at a slight performance cost.
         fragments = ActiveRecord::Base.connection.unprepared_statement do
-          @relations.map do |rel|
-            rel.reorder(nil).to_sql
-          end
+          @relations.map { |rel| rel.reorder(nil).to_sql }.reject(&:blank?)
         end
 
         fragments.join("\nUNION\n")
diff --git a/lib/gitlab/template/dockerfile_template.rb b/lib/gitlab/template/dockerfile_template.rb
new file mode 100644
index 00000000000..d5d3e045a42
--- /dev/null
+++ b/lib/gitlab/template/dockerfile_template.rb
@@ -0,0 +1,30 @@
+module Gitlab
+  module Template
+    class DockerfileTemplate < BaseTemplate
+      def content
+        explanation = "# This file is a template, and might need editing before it works on your project."
+        [explanation, super].join("\n")
+      end
+
+      class << self
+        def extension
+          'Dockerfile'
+        end
+
+        def categories
+          {
+            "General" => ''
+          }
+        end
+
+        def base_dir
+          Rails.root.join('vendor/dockerfile')
+        end
+
+        def finder(project = nil)
+          Gitlab::Template::Finders::GlobalTemplateFinder.new(self.base_dir, self.extension, self.categories)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index 594439a5d4b..d28bb583fe7 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -95,6 +95,19 @@ module Gitlab
         ]
       end
 
+      def terminal_websocket(terminal)
+        details = {
+          'Terminal' => {
+            'Subprotocols' => terminal[:subprotocols],
+            'Url' => terminal[:url],
+            'Header' => terminal[:headers]
+          }
+        }
+        details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.has_key?(:ca_pem)
+
+        details
+      end
+
       def version
         path = Rails.root.join(VERSION_FILE)
         path.readable? ? path.read.chomp : 'unknown'
@@ -117,8 +130,12 @@ module Gitlab
       end
 
       def verify_api_request!(request_headers)
+        decode_jwt(request_headers[INTERNAL_API_REQUEST_HEADER])
+      end
+
+      def decode_jwt(encoded_message)
         JWT.decode(
-          request_headers[INTERNAL_API_REQUEST_HEADER],
+          encoded_message,
           secret,
           true,
           { iss: 'gitlab-workhorse', verify_iss: true, algorithm: 'HS256' },
diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb
new file mode 100644
index 00000000000..fb8d7d97f8a
--- /dev/null
+++ b/lib/mattermost/session.rb
@@ -0,0 +1,115 @@
+module Mattermost
+  class NoSessionError < StandardError; end
+  # This class' prime objective is to obtain a session token on a Mattermost
+  # instance with SSO configured where this GitLab instance is the provider.
+  #
+  # The process depends on OAuth, but skips a step in the authentication cycle.
+  # For example, usually a user would click the 'login in GitLab' button on
+  # Mattermost, which would yield a 302 status code and redirects you to GitLab
+  # to approve the use of your account on Mattermost. Which would trigger a
+  # callback so Mattermost knows this request is approved and gets the required
+  # data to create the user account etc.
+  #
+  # This class however skips the button click, and also the approval phase to
+  # speed up the process and keep it without manual action and get a session
+  # going.
+  class Session
+    include Doorkeeper::Helpers::Controller
+    include HTTParty
+
+    base_uri Settings.mattermost.host
+
+    attr_accessor :current_resource_owner, :token
+
+    def initialize(current_user)
+      @current_resource_owner = current_user
+    end
+
+    def with_session
+      raise NoSessionError unless create
+
+      begin
+        yield self
+      ensure
+        destroy
+      end
+    end
+
+    # Next methods are needed for Doorkeeper
+    def pre_auth
+      @pre_auth ||= Doorkeeper::OAuth::PreAuthorization.new(
+        Doorkeeper.configuration, server.client_via_uid, params)
+    end
+
+    def authorization
+      @authorization ||= strategy.request
+    end
+
+    def strategy
+      @strategy ||= server.authorization_request(pre_auth.response_type)
+    end
+
+    def request
+      @request ||= OpenStruct.new(parameters: params)
+    end
+
+    def params
+      Rack::Utils.parse_query(oauth_uri.query).symbolize_keys
+    end
+
+    def get(path, options = {})
+      self.class.get(path, options.merge(headers: @headers))
+    end
+
+    def post(path, options = {})
+      self.class.post(path, options.merge(headers: @headers))
+    end
+
+    private
+
+    def create
+      return unless oauth_uri
+      return unless token_uri
+
+      @token = request_token
+      @headers = {
+        Authorization: "Bearer #{@token}"
+      }
+
+      @token
+    end
+
+    def destroy
+      post('/api/v3/users/logout')
+    end
+
+    def oauth_uri
+      return @oauth_uri if defined?(@oauth_uri)
+
+      @oauth_uri = nil
+
+      response = get("/api/v3/oauth/gitlab/login", follow_redirects: false)
+      return unless 300 <= response.code && response.code < 400
+
+      redirect_uri = response.headers['location']
+      return unless redirect_uri
+
+      @oauth_uri = URI.parse(redirect_uri)
+    end
+
+    def token_uri
+      @token_uri ||=
+        if oauth_uri
+          authorization.authorize.redirect_uri if pre_auth.authorizable?
+        end
+    end
+
+    def request_token
+      response = get(token_uri, follow_redirects: false)
+
+      if 200 <= response.code && response.code < 400
+        response.headers['token']
+      end
+    end
+  end
+end
diff --git a/lib/omniauth/strategies/bitbucket.rb b/lib/omniauth/strategies/bitbucket.rb
new file mode 100644
index 00000000000..5a7d67c2390
--- /dev/null
+++ b/lib/omniauth/strategies/bitbucket.rb
@@ -0,0 +1,41 @@
+require 'omniauth-oauth2'
+
+module OmniAuth
+  module Strategies
+    class Bitbucket < OmniAuth::Strategies::OAuth2
+      option :name, 'bitbucket'
+
+      option :client_options, {
+        site: 'https://bitbucket.org',
+        authorize_url: 'https://bitbucket.org/site/oauth2/authorize',
+        token_url: 'https://bitbucket.org/site/oauth2/access_token'
+      }
+
+      uid do
+        raw_info['username']
+      end
+
+      info do
+        {
+          name: raw_info['display_name'],
+          avatar: raw_info['links']['avatar']['href'],
+          email: primary_email
+        }
+      end
+
+      def raw_info
+        @raw_info ||= access_token.get('api/2.0/user').parsed
+      end
+
+      def primary_email
+        primary = emails.find { |i| i['is_primary'] && i['is_confirmed'] }
+        primary && primary['email'] || nil
+      end
+
+      def emails
+        email_response = access_token.get('api/2.0/user/emails').parsed
+        @emails ||= email_response && email_response['values'] || nil
+      end
+    end
+  end
+end
diff --git a/lib/rouge/lexers/math.rb b/lib/rouge/lexers/math.rb
new file mode 100644
index 00000000000..80784adfd76
--- /dev/null
+++ b/lib/rouge/lexers/math.rb
@@ -0,0 +1,21 @@
+module Rouge
+  module Lexers
+    class Math < Lexer
+      title "A passthrough lexer used for LaTeX input"
+      desc "A boring lexer that doesn't highlight anything"
+
+      tag 'math'
+      mimetypes 'text/plain'
+
+      default_options token: 'Text'
+
+      def token
+        @token ||= Token[option :token]
+      end
+
+      def stream_tokens(string, &b)
+        yield self.token, string
+      end
+    end
+  end
+end
diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab
index d521de28e8a..2f7c34a3f31 100644
--- a/lib/support/nginx/gitlab
+++ b/lib/support/nginx/gitlab
@@ -20,6 +20,11 @@ upstream gitlab-workhorse {
   server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0;
 }
 
+map $http_upgrade $connection_upgrade_gitlab {
+    default upgrade;
+    ''      close;
+}
+
 ## Normal HTTP host
 server {
   ## Either remove "default_server" from the listen line below,
@@ -53,6 +58,8 @@ server {
     proxy_set_header    X-Real-IP           $remote_addr;
     proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
     proxy_set_header    X-Forwarded-Proto   $scheme;
+    proxy_set_header    Upgrade             $http_upgrade;
+    proxy_set_header    Connection          $connection_upgrade_gitlab;
 
     proxy_pass http://gitlab-workhorse;
   }
diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl
index bf014b56cf6..5661394058d 100644
--- a/lib/support/nginx/gitlab-ssl
+++ b/lib/support/nginx/gitlab-ssl
@@ -24,6 +24,11 @@ upstream gitlab-workhorse {
   server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0;
 }
 
+map $http_upgrade $connection_upgrade_gitlab_ssl {
+    default upgrade;
+    ''      close;
+}
+
 ## Redirects all HTTP traffic to the HTTPS host
 server {
   ## Either remove "default_server" from the listen line below,
@@ -98,6 +103,9 @@ server {
     proxy_set_header    X-Forwarded-Ssl     on;
     proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
     proxy_set_header    X-Forwarded-Proto   $scheme;
+    proxy_set_header    Upgrade             $http_upgrade;
+    proxy_set_header    Connection          $connection_upgrade_gitlab_ssl;
+
     proxy_pass http://gitlab-workhorse;
   }
 
diff --git a/lib/tasks/gitlab/ldap.rake b/lib/tasks/gitlab/ldap.rake
new file mode 100644
index 00000000000..c66a2a263dc
--- /dev/null
+++ b/lib/tasks/gitlab/ldap.rake
@@ -0,0 +1,40 @@
+namespace :gitlab do
+  namespace :ldap do
+    desc 'GitLab | LDAP | Rename provider'
+    task :rename_provider, [:old_provider, :new_provider] => :environment  do |_, args|
+      old_provider = args[:old_provider] ||
+        prompt('What is the old provider? Ex. \'ldapmain\': '.color(:blue))
+      new_provider = args[:new_provider] ||
+        prompt('What is the new provider ID? Ex. \'ldapcustom\': '.color(:blue))
+      puts '' # Add some separation in the output
+
+      identities = Identity.where(provider: old_provider)
+      identity_count = identities.count
+
+      if identities.empty?
+        puts "Found no user identities with '#{old_provider}' provider."
+        puts 'Please check the provider name and try again.'
+        exit 1
+      end
+
+      plural_id_count = ActionController::Base.helpers.pluralize(identity_count, 'user')
+
+      unless ENV['force'] == 'yes'
+        puts "#{plural_id_count} with provider '#{old_provider}' will be updated to '#{new_provider}'"
+        puts 'If the new provider is incorrect, users will be unable to sign in'
+        ask_to_continue
+        puts ''
+      end
+
+      updated_count = identities.update_all(provider: new_provider)
+
+      if updated_count == identity_count
+        puts 'User identities were successfully updated'.color(:green)
+      else
+        plural_updated_count = ActionController::Base.helpers.pluralize(updated_count, 'user')
+        puts 'Some user identities could not be updated'.color(:red)
+        puts "Successfully updated #{plural_updated_count} out of #{plural_id_count} total"
+      end
+    end
+  end
+end
diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake
index 141a0b74ec0..f5caca3ddbf 100644
--- a/lib/tasks/migrate/setup_postgresql.rake
+++ b/lib/tasks/migrate/setup_postgresql.rake
@@ -1,8 +1,12 @@
+require Rails.root.join('lib/gitlab/database')
+require Rails.root.join('lib/gitlab/database/migration_helpers')
 require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lower_indexes')
 require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes')
+require Rails.root.join('db/migrate/20161212142807_add_lower_path_index_to_routes')
 
 desc 'GitLab | Sets up PostgreSQL'
 task setup_postgresql: :environment do
   NamespacesProjectsPathLowerIndexes.new.up
   AddUsersLowerUsernameEmailIndexes.new.up
+  AddLowerPathIndexToRoutes.new.up
 end