Merge remote-tracking branch 'upstream/master' into artifacts-from-ref-and-build-name

* upstream/master: (195 commits)
  Fix expansion of discussions in diff
  Improve performance of MR show page
  Fix jumping between discussions on changes tab
  Update doorkeeper to 4.2.0
  Fix MR note discussion ID
  Handle legacy sort order values
  Refactor `find_for_git_client` and its related methods.
  Remove right margin on Jump button icon
  Fix bug causing “Jump to discussion” button not to show
  Small refactor and syntax fixes.
  Removed unnecessary service for user retrieval and improved API error message.
  Added documentation and CHANGELOG item
  Added checks for 2FA to the API `/sessions` endpoint and the Resource Owner Password Credentials flow.
  Fix behavior around commands with optional arguments
  Fix behavior of label_ids and add/remove_label_ids
  Remove unneeded aliases
  Do not expose projects on deployments
  Incorporate feedback
  Docs for API endpoints
  Expose project for environments
  ...

11 files changed, 474 insertions, 6 deletions

diff --git a/lib/api/api.rb b/lib/api/api.rb
index d43af3f24e9..6b8bfbbdae6 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -43,6 +43,7 @@ module API
     mount ::API::CommitStatuses
     mount ::API::Commits
     mount ::API::DeployKeys
+    mount ::API::Deployments
     mount ::API::Environments
     mount ::API::Files
     mount ::API::Groups
@@ -56,6 +57,7 @@ module API
     mount ::API::Milestones
     mount ::API::Namespaces
     mount ::API::Notes
+    mount ::API::Pipelines
     mount ::API::ProjectHooks
     mount ::API::ProjectSnippets
     mount ::API::Projects
diff --git a/lib/api/builds.rb b/lib/api/builds.rb
index be5a3484ec8..52bdbcae5a8 100644
--- a/lib/api/builds.rb
+++ b/lib/api/builds.rb
@@ -189,6 +189,27 @@ module API
         present build, with: Entities::Build,
                        user_can_download_artifacts: can?(current_user, :read_build, user_project)
       end
+
+      desc 'Trigger a manual build' do
+        success Entities::Build
+        detail 'This feature was added in GitLab 8.11'
+      end
+      params do
+        requires :build_id, type: Integer, desc: 'The ID of a Build'
+      end
+      post ":id/builds/:build_id/play" do
+        authorize_read_builds!
+
+        build = get_build!(params[:build_id])
+
+        bad_request!("Unplayable Build") unless build.playable?
+
+        build.play(current_user)
+
+        status 200
+        present build, with: Entities::Build,
+                       user_can_download_artifacts: can?(current_user, :read_build, user_project)
+      end
     end
 
     helpers do
diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb
new file mode 100644
index 00000000000..f782bcaf7e9
--- /dev/null
+++ b/lib/api/deployments.rb
@@ -0,0 +1,40 @@
+module API
+  # Deployments RESTfull API endpoints
+  class Deployments < Grape::API
+    before { authenticate! }
+
+    params do
+      requires :id, type: String, desc: 'The project ID'
+    end
+    resource :projects do
+      desc 'Get all deployments of the project' do
+        detail 'This feature was introduced in GitLab 8.11.'
+        success Entities::Deployment
+      end
+      params do
+        optional :page,     type: Integer, desc: 'Page number of the current request'
+        optional :per_page, type: Integer, desc: 'Number of items per page'
+      end
+      get ':id/deployments' do
+        authorize! :read_deployment, user_project
+
+        present paginate(user_project.deployments), with: Entities::Deployment
+      end
+
+      desc 'Gets a specific deployment' do
+        detail 'This feature was introduced in GitLab 8.11.'
+        success Entities::Deployment
+      end
+      params do
+        requires :deployment_id, type: Integer,  desc: 'The deployment ID'
+      end
+      get ':id/deployments/:deployment_id' do
+        authorize! :read_deployment, user_project
+
+        deployment = user_project.deployments.find(params[:deployment_id])
+
+        present deployment, with: Entities::Deployment
+      end
+    end
+  end
+end
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 055716ab1e3..67420772335 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -502,8 +502,28 @@ module API
       expose :key, :value
     end
 
+    class Pipeline < Grape::Entity
+      expose :id, :status, :ref, :sha, :before_sha, :tag, :yaml_errors
+
+      expose :user, with: Entities::UserBasic
+      expose :created_at, :updated_at, :started_at, :finished_at, :committed_at
+      expose :duration
+    end
+
     class Environment < Grape::Entity
       expose :id, :name, :external_url
+      expose :project, using: Entities::Project
+    end
+
+    class EnvironmentBasic < Grape::Entity
+      expose :id, :name, :external_url
+    end
+
+    class Deployment < Grape::Entity
+      expose :id, :iid, :ref, :sha, :created_at
+      expose :user,        using: Entities::UserBasic
+      expose :environment, using: Entities::EnvironmentBasic
+      expose :deployable,  using: Entities::Build
     end
 
     class RepoLicense < Grape::Entity
diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb
new file mode 100644
index 00000000000..2aae75c471d
--- /dev/null
+++ b/lib/api/pipelines.rb
@@ -0,0 +1,74 @@
+module API
+  class Pipelines < Grape::API
+    before { authenticate! }
+
+    params do
+      requires :id, type: String, desc: 'The project ID'
+    end
+    resource :projects do
+      desc 'Get all Pipelines of the project' do
+        detail 'This feature was introduced in GitLab 8.11.'
+        success Entities::Pipeline
+      end
+      params do
+        optional :page,     type: Integer, desc: 'Page number of the current request'
+        optional :per_page, type: Integer, desc: 'Number of items per page'
+      end
+      get ':id/pipelines' do
+        authorize! :read_pipeline, user_project
+
+        present paginate(user_project.pipelines), with: Entities::Pipeline
+      end
+
+      desc 'Gets a specific pipeline for the project' do
+        detail 'This feature was introduced in GitLab 8.11'
+        success Entities::Pipeline
+      end
+      params do
+        requires :pipeline_id, type: Integer, desc: 'The pipeline ID'
+      end
+      get ':id/pipelines/:pipeline_id' do
+        authorize! :read_pipeline, user_project
+
+        present pipeline, with: Entities::Pipeline
+      end
+
+      desc 'Retry failed builds in the pipeline' do
+        detail 'This feature was introduced in GitLab 8.11.'
+        success Entities::Pipeline
+      end
+      params do
+        requires :pipeline_id, type: Integer,  desc: 'The pipeline ID'
+      end
+      post ':id/pipelines/:pipeline_id/retry' do
+        authorize! :update_pipeline, user_project
+
+        pipeline.retry_failed(current_user)
+
+        present pipeline, with: Entities::Pipeline
+      end
+
+      desc 'Cancel all builds in the pipeline' do
+        detail 'This feature was introduced in GitLab 8.11.'
+        success Entities::Pipeline
+      end
+      params do
+        requires :pipeline_id, type: Integer,  desc: 'The pipeline ID'
+      end
+      post ':id/pipelines/:pipeline_id/cancel' do
+        authorize! :update_pipeline, user_project
+
+        pipeline.cancel_running
+
+        status 200
+        present pipeline.reload, with: Entities::Pipeline
+      end
+    end
+
+    helpers do
+      def pipeline
+        @pipeline ||= user_project.pipelines.find(params[:pipeline_id])
+      end
+    end
+  end
+end
diff --git a/lib/api/session.rb b/lib/api/session.rb
index 56c202f1294..55ec66a6d67 100644
--- a/lib/api/session.rb
+++ b/lib/api/session.rb
@@ -14,6 +14,7 @@ module API
       user = Gitlab::Auth.find_with_user_password(params[:email] || params[:login], params[:password])
 
       return unauthorized! unless user
+      return render_api_error!('401 Unauthorized. You have 2FA enabled. Please use a personal access token to access the API', 401) if user.two_factor_enabled?
       present user, with: Entities::UserLogin
     end
   end
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index db1704af75e..91f0270818a 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -10,13 +10,12 @@ module Gitlab
 
         if valid_ci_request?(login, password, project)
           result.type = :ci
-        elsif result.user = find_with_user_password(login, password)
-          result.type = :gitlab_or_ldap
-        elsif result.user = oauth_access_token_check(login, password)
-          result.type = :oauth
+        else
+          result = populate_result(login, password)
         end
 
-        rate_limit!(ip, success: !!result.user || (result.type == :ci), login: login)
+        success = result.user.present? || [:ci, :missing_personal_token].include?(result.type)
+        rate_limit!(ip, success: success, login: login)
         result
       end
 
@@ -76,10 +75,43 @@ module Gitlab
         end
       end
 
+      def populate_result(login, password)
+        result =
+          user_with_password_for_git(login, password) ||
+          oauth_access_token_check(login, password) ||
+          personal_access_token_check(login, password)
+
+        if result
+          result.type = nil unless result.user
+
+          if result.user && result.user.two_factor_enabled? && result.type == :gitlab_or_ldap
+            result.type = :missing_personal_token
+          end
+        end
+
+        result || Result.new
+      end
+
+      def user_with_password_for_git(login, password)
+        user = find_with_user_password(login, password)
+        Result.new(user, :gitlab_or_ldap) if user
+      end
+
       def oauth_access_token_check(login, password)
         if login == "oauth2" && password.present?
           token = Doorkeeper::AccessToken.by_token(password)
-          token && token.accessible? && User.find_by(id: token.resource_owner_id)
+          if token && token.accessible?
+            user = User.find_by(id: token.resource_owner_id)
+            Result.new(user, :oauth)
+          end
+        end
+      end
+
+      def personal_access_token_check(login, password)
+        if login && password
+          user = User.find_by_personal_access_token(password)
+          validation = User.by_login(login)
+          Result.new(user, :personal_token) if user == validation
         end
       end
     end
diff --git a/lib/gitlab/email/handler/base_handler.rb b/lib/gitlab/email/handler/base_handler.rb
index b7ed11cb638..7cccf465334 100644
--- a/lib/gitlab/email/handler/base_handler.rb
+++ b/lib/gitlab/email/handler/base_handler.rb
@@ -45,6 +45,7 @@ module Gitlab
 
         def verify_record!(record:, invalid_exception:, record_name:)
           return if record.persisted?
+          return if record.errors.key?(:commands_only)
 
           error_title = "The #{record_name} could not be created for the following reasons:"
 
diff --git a/lib/gitlab/slash_commands/command_definition.rb b/lib/gitlab/slash_commands/command_definition.rb
new file mode 100644
index 00000000000..60d35be2599
--- /dev/null
+++ b/lib/gitlab/slash_commands/command_definition.rb
@@ -0,0 +1,57 @@
+module Gitlab
+  module SlashCommands
+    class CommandDefinition
+      attr_accessor :name, :aliases, :description, :params, :condition_block, :action_block
+
+      def initialize(name, attributes = {})
+        @name = name
+
+        @aliases         = attributes[:aliases] || []
+        @description     = attributes[:description] || ''
+        @params          = attributes[:params] || []
+        @condition_block = attributes[:condition_block]
+        @action_block    = attributes[:action_block]
+      end
+
+      def all_names
+        [name, *aliases]
+      end
+
+      def noop?
+        action_block.nil?
+      end
+
+      def available?(opts)
+        return true unless condition_block
+
+        context = OpenStruct.new(opts)
+        context.instance_exec(&condition_block)
+      end
+
+      def execute(context, opts, arg)
+        return if noop? || !available?(opts)
+
+        if arg.present?
+          context.instance_exec(arg, &action_block)
+        elsif action_block.arity == 0
+          context.instance_exec(&action_block)
+        end
+      end
+
+      def to_h(opts)
+        desc = description
+        if desc.respond_to?(:call)
+          context = OpenStruct.new(opts)
+          desc = context.instance_exec(&desc) rescue ''
+        end
+
+        {
+          name: name,
+          aliases: aliases,
+          description: desc,
+          params: params
+        }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/slash_commands/dsl.rb b/lib/gitlab/slash_commands/dsl.rb
new file mode 100644
index 00000000000..50b0937d267
--- /dev/null
+++ b/lib/gitlab/slash_commands/dsl.rb
@@ -0,0 +1,98 @@
+module Gitlab
+  module SlashCommands
+    module Dsl
+      extend ActiveSupport::Concern
+
+      included do
+        cattr_accessor :command_definitions, instance_accessor: false do
+          []
+        end
+
+        cattr_accessor :command_definitions_by_name, instance_accessor: false do
+          {}
+        end
+      end
+
+      class_methods do
+        # Allows to give a description to the next slash command.
+        # This description is shown in the autocomplete menu.
+        # It accepts a block that will be evaluated with the context given to
+        # `CommandDefintion#to_h`.
+        #
+        # Example:
+        #
+        #   desc do
+        #     "This is a dynamic description for #{noteable.to_ability_name}"
+        #   end
+        #   command :command_key do |arguments|
+        #     # Awesome code block
+        #   end
+        def desc(text = '', &block)
+          @description = block_given? ? block : text
+        end
+
+        # Allows to define params for the next slash command.
+        # These params are shown in the autocomplete menu.
+        #
+        # Example:
+        #
+        #   params "~label ~label2"
+        #   command :command_key do |arguments|
+        #     # Awesome code block
+        #   end
+        def params(*params)
+          @params = params
+        end
+
+        # Allows to define conditions that must be met in order for the command
+        # to be returned by `.command_names` & `.command_definitions`.
+        # It accepts a block that will be evaluated with the context given to
+        # `CommandDefintion#to_h`.
+        #
+        # Example:
+        #
+        #   condition do
+        #     project.public?
+        #   end
+        #   command :command_key do |arguments|
+        #     # Awesome code block
+        #   end
+        def condition(&block)
+          @condition_block = block
+        end
+
+        # Registers a new command which is recognizeable from body of email or
+        # comment.
+        # It accepts aliases and takes a block.
+        #
+        # Example:
+        #
+        #   command :my_command, :alias_for_my_command do |arguments|
+        #     # Awesome code block
+        #   end
+        def command(*command_names, &block)
+          name, *aliases = command_names
+
+          definition = CommandDefinition.new(
+            name,
+            aliases:          aliases,
+            description:      @description,
+            params:           @params,
+            condition_block:  @condition_block,
+            action_block:     block
+          )
+
+          self.command_definitions << definition
+
+          definition.all_names.each do |name|
+            self.command_definitions_by_name[name] = definition
+          end
+
+          @description = nil
+          @params = nil
+          @condition_block = nil
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/slash_commands/extractor.rb b/lib/gitlab/slash_commands/extractor.rb
new file mode 100644
index 00000000000..a672e5e4855
--- /dev/null
+++ b/lib/gitlab/slash_commands/extractor.rb
@@ -0,0 +1,122 @@
+module Gitlab
+  module SlashCommands
+    # This class takes an array of commands that should be extracted from a
+    # given text.
+    #
+    # ```
+    # extractor = Gitlab::SlashCommands::Extractor.new([:open, :assign, :labels])
+    # ```
+    class Extractor
+      attr_reader :command_definitions
+
+      def initialize(command_definitions)
+        @command_definitions = command_definitions
+      end
+
+      # Extracts commands from content and return an array of commands.
+      # The array looks like the following:
+      # [
+      #   ['command1'],
+      #   ['command3', 'arg1 arg2'],
+      # ]
+      # The command and the arguments are stripped.
+      # The original command text is removed from the given `content`.
+      #
+      # Usage:
+      # ```
+      # extractor = Gitlab::SlashCommands::Extractor.new([:open, :assign, :labels])
+      # msg = %(hello\n/labels ~foo ~"bar baz"\nworld)
+      # commands = extractor.extract_commands(msg) #=> [['labels', '~foo ~"bar baz"']]
+      # msg #=> "hello\nworld"
+      # ```
+      def extract_commands(content, opts = {})
+        return [content, []] unless content
+
+        content = content.dup
+
+        commands = []
+
+        content.delete!("\r")
+        content.gsub!(commands_regex(opts)) do
+          if $~[:cmd]
+            commands << [$~[:cmd], $~[:arg]].reject(&:blank?)
+            ''
+          else
+            $~[0]
+          end
+        end
+
+        [content.strip, commands]
+      end
+
+      private
+
+      # Builds a regular expression to match known commands.
+      # First match group captures the command name and
+      # second match group captures its arguments.
+      #
+      # It looks something like:
+      #
+      #   /^\/(?<cmd>close|reopen|...)(?:( |$))(?<arg>[^\/\n]*)(?:\n|$)/
+      def commands_regex(opts)
+        names = command_names(opts).map(&:to_s)
+
+        @commands_regex ||= %r{
+            (?<code>
+              # Code blocks:
+              # ```
+              # Anything, including `/cmd arg` which are ignored by this filter
+              # ```
+
+              ^```
+              .+?
+              \n```$
+            )
+          |
+            (?<html>
+              # HTML block:
+              # <tag>
+              # Anything, including `/cmd arg` which are ignored by this filter
+              # </tag>
+
+              ^<[^>]+?>\n
+              .+?
+              \n<\/[^>]+?>$
+            )
+          |
+            (?<html>
+              # Quote block:
+              # >>>
+              # Anything, including `/cmd arg` which are ignored by this filter
+              # >>>
+
+              ^>>>
+              .+?
+              \n>>>$
+            )
+          |
+            (?:
+              # Command not in a blockquote, blockcode, or HTML tag:
+              # /close
+
+              ^\/
+              (?<cmd>#{Regexp.union(names)})
+              (?:
+                [ ]
+                (?<arg>[^\/\n]*)
+              )?
+              (?:\n|$)
+            )
+        }mx
+      end
+
+      def command_names(opts)
+        command_definitions.flat_map do |command|
+          next if command.noop?
+
+          command.all_names
+        end.compact
+      end
+    end
+  end
+end