Merge remote-tracking branch 'dev_gitlab_org/master' into git-http-blacklist

Conflicts:
	CHANGELOG

17 files changed, 315 insertions, 43 deletions

diff --git a/lib/api/api.rb b/lib/api/api.rb
index d26667ba3f7..cb46f477ff9 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -2,6 +2,7 @@ Dir["#{Rails.root}/lib/api/*.rb"].each {|file| require file}
 
 module API
   class API < Grape::API
+    include APIGuard
     version 'v3', using: :path
 
     rescue_from ActiveRecord::RecordNotFound do
diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb
new file mode 100644
index 00000000000..23975518181
--- /dev/null
+++ b/lib/api/api_guard.rb
@@ -0,0 +1,175 @@
+# Guard API with OAuth 2.0 Access Token
+
+require 'rack/oauth2'
+
+module APIGuard
+  extend ActiveSupport::Concern
+
+  included do |base|
+    # OAuth2 Resource Server Authentication
+    use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request|
+      # The authenticator only fetches the raw token string
+
+      # Must yield access token to store it in the env
+      request.access_token
+    end
+
+    helpers HelperMethods
+
+    install_error_responders(base)
+  end
+
+  # Helper Methods for Grape Endpoint
+  module HelperMethods
+    # Invokes the doorkeeper guard.
+    #
+    # If token is presented and valid, then it sets @current_user.
+    #
+    # If the token does not have sufficient scopes to cover the requred scopes,
+    # then it raises InsufficientScopeError.
+    #
+    # If the token is expired, then it raises ExpiredError.
+    #
+    # If the token is revoked, then it raises RevokedError.
+    #
+    # If the token is not found (nil), then it raises TokenNotFoundError.
+    #
+    # Arguments:
+    #
+    #   scopes: (optional) scopes required for this guard.
+    #           Defaults to empty array.
+    #
+    def doorkeeper_guard!(scopes: [])
+      if (access_token = find_access_token).nil?
+        raise TokenNotFoundError
+
+      else
+        case validate_access_token(access_token, scopes)
+        when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE
+          raise InsufficientScopeError.new(scopes)
+
+        when Oauth2::AccessTokenValidationService::EXPIRED
+          raise ExpiredError
+
+        when Oauth2::AccessTokenValidationService::REVOKED
+          raise RevokedError
+
+        when Oauth2::AccessTokenValidationService::VALID
+          @current_user = User.find(access_token.resource_owner_id)
+
+        end
+      end
+    end
+
+    def doorkeeper_guard(scopes: [])
+      if access_token = find_access_token
+        case validate_access_token(access_token, scopes)
+        when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE
+          raise InsufficientScopeError.new(scopes)
+
+        when Oauth2::AccessTokenValidationService::EXPIRED
+          raise ExpiredError
+
+        when Oauth2::AccessTokenValidationService::REVOKED
+          raise RevokedError
+
+        when Oauth2::AccessTokenValidationService::VALID
+          @current_user = User.find(access_token.resource_owner_id)
+        end
+      end
+    end
+
+    def current_user
+      @current_user
+    end
+
+    private
+    def find_access_token
+      @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods)
+    end
+
+    def doorkeeper_request
+      @doorkeeper_request ||= ActionDispatch::Request.new(env)
+    end
+
+    def validate_access_token(access_token, scopes)
+      Oauth2::AccessTokenValidationService.validate(access_token, scopes: scopes)
+    end
+  end
+
+  module ClassMethods
+    # Installs the doorkeeper guard on the whole Grape API endpoint.
+    #
+    # Arguments:
+    #
+    #   scopes: (optional) scopes required for this guard.
+    #           Defaults to empty array.
+    #
+    def guard_all!(scopes: [])
+      before do
+        guard! scopes: scopes
+      end
+    end
+
+    private
+    def install_error_responders(base)
+      error_classes = [ MissingTokenError, TokenNotFoundError,
+                        ExpiredError, RevokedError, InsufficientScopeError]
+
+      base.send :rescue_from, *error_classes, oauth2_bearer_token_error_handler
+    end
+
+    def oauth2_bearer_token_error_handler
+      Proc.new {|e|
+        response = case e
+          when MissingTokenError
+            Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new
+
+          when TokenNotFoundError
+            Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new(
+              :invalid_token,
+              "Bad Access Token.")
+
+          when ExpiredError
+            Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new(
+              :invalid_token,
+              "Token is expired. You can either do re-authorization or token refresh.")
+
+          when RevokedError
+            Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new(
+              :invalid_token,
+              "Token was revoked. You have to re-authorize from the user.")
+
+          when InsufficientScopeError
+            # FIXME: ForbiddenError (inherited from Bearer::Forbidden of Rack::Oauth2)
+            # does not include WWW-Authenticate header, which breaks the standard.
+            Rack::OAuth2::Server::Resource::Bearer::Forbidden.new(
+              :insufficient_scope,
+              Rack::OAuth2::Server::Resource::ErrorMethods::DEFAULT_DESCRIPTION[:insufficient_scope],
+              { :scope => e.scopes})
+          end
+
+        response.finish
+      }
+    end
+  end
+
+  #
+  # Exceptions
+  #
+
+  class MissingTokenError < StandardError; end
+
+  class TokenNotFoundError < StandardError; end
+
+  class ExpiredError < StandardError; end
+
+  class RevokedError < StandardError; end
+
+  class InsufficientScopeError < StandardError
+    attr_reader :scopes
+    def initialize(scopes)
+      @scopes = scopes
+    end
+  end
+end
\ No newline at end of file
diff --git a/lib/api/branches.rb b/lib/api/branches.rb
index 6ec1a753a69..b52d786e020 100644
--- a/lib/api/branches.rb
+++ b/lib/api/branches.rb
@@ -14,7 +14,8 @@ module API
       # Example Request:
       #   GET /projects/:id/repository/branches
       get ":id/repository/branches" do
-        present user_project.repository.branches.sort_by(&:name), with: Entities::RepoObject, project: user_project
+        branches = user_project.repository.branches.sort_by(&:name)
+        present branches, with: Entities::RepoObject, project: user_project
       end
 
       # Get a single branch
@@ -26,7 +27,7 @@ module API
       #   GET /projects/:id/repository/branches/:branch
       get ':id/repository/branches/:branch', requirements: { branch: /.*/ } do
         @branch = user_project.repository.branches.find { |item| item.name == params[:branch] }
-        not_found!("Branch does not exist") if @branch.nil?
+        not_found!("Branch") unless @branch
         present @branch, with: Entities::RepoObject, project: user_project
       end
 
@@ -43,7 +44,7 @@ module API
         authorize_admin_project
 
         @branch = user_project.repository.find_branch(params[:branch])
-        not_found! unless @branch
+        not_found!("Branch") unless @branch
         protected_branch = user_project.protected_branches.find_by(name: @branch.name)
         user_project.protected_branches.create(name: @branch.name) unless protected_branch
 
@@ -63,7 +64,7 @@ module API
         authorize_admin_project
 
         @branch = user_project.repository.find_branch(params[:branch])
-        not_found! unless @branch
+        not_found!("Branch does not exist") unless @branch
         protected_branch = user_project.protected_branches.find_by(name: @branch.name)
         protected_branch.destroy if protected_branch
 
diff --git a/lib/api/commits.rb b/lib/api/commits.rb
index 6c5391b98c8..0de4e720ffe 100644
--- a/lib/api/commits.rb
+++ b/lib/api/commits.rb
@@ -108,7 +108,7 @@ module API
         if note.save
           present note, with: Entities::CommitNote
         else
-          not_found!
+          render_api_error!("Failed to save note #{note.errors.messages}", 400)
         end
       end
     end
diff --git a/lib/api/files.rb b/lib/api/files.rb
index 84e1d311781..e6e71bac367 100644
--- a/lib/api/files.rb
+++ b/lib/api/files.rb
@@ -35,7 +35,7 @@ module API
         file_path = attrs.delete(:file_path)
 
         commit = user_project.repository.commit(ref)
-        not_found! "Commit" unless commit
+        not_found! 'Commit' unless commit
 
         blob = user_project.repository.blob_at(commit.sha, file_path)
 
@@ -53,7 +53,7 @@ module API
             commit_id: commit.id,
           }
         else
-          render_api_error!('File not found', 404)
+          not_found! 'File'
         end
       end
 
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index f0ab6938b1c..bda60b3b7d5 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -25,11 +25,14 @@ module API
       # Example Request:
       #  GET /groups
       get do
-        if current_user.admin
-          @groups = paginate Group
-        else
-          @groups = paginate current_user.groups
-        end
+        @groups = if current_user.admin
+                    Group.all
+                  else
+                    current_user.groups
+                  end
+
+        @groups = @groups.search(params[:search]) if params[:search].present?
+        @groups = paginate @groups
         present @groups, with: Entities::Group
       end
 
@@ -51,7 +54,7 @@ module API
         if @group.save
           present @group, with: Entities::Group
         else
-          not_found!
+          render_api_error!("Failed to save group #{@group.errors.messages}", 400)
         end
       end
 
@@ -94,7 +97,7 @@ module API
         if result
           present group
         else
-          not_found!
+          render_api_error!("Failed to transfer project #{project.errors.messages}", 400)
         end
       end
     end
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 027fb20ec46..62c26ef76ce 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -11,7 +11,7 @@ module API
 
     def current_user
       private_token = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s
-      @current_user ||= User.find_by(authentication_token: private_token)
+      @current_user ||= (User.find_by(authentication_token: private_token) || doorkeeper_guard)
 
       unless @current_user && Gitlab::UserAccess.allowed?(@current_user)
         return nil
@@ -42,7 +42,7 @@ module API
 
     def user_project
       @project ||= find_project(params[:id])
-      @project || not_found!
+      @project || not_found!("Project")
     end
 
     def find_project(id)
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index a365f1db00f..81038d05f12 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -233,7 +233,7 @@ module API
         if note.save
           present note, with: Entities::MRNote
         else
-          render_validation_error!(note)
+          render_api_error!("Failed to save note #{note.errors.messages}", 400)
         end
       end
     end
diff --git a/lib/api/milestones.rb b/lib/api/milestones.rb
index a4fdb752d69..2ea49359df0 100644
--- a/lib/api/milestones.rb
+++ b/lib/api/milestones.rb
@@ -48,7 +48,7 @@ module API
         if milestone.valid?
           present milestone, with: Entities::Milestone
         else
-          not_found!
+          render_api_error!("Failed to create milestone #{milestone.errors.messages}", 400)
         end
       end
 
@@ -72,7 +72,7 @@ module API
         if milestone.valid?
           present milestone, with: Entities::Milestone
         else
-          not_found!
+          render_api_error!("Failed to update milestone #{milestone.errors.messages}", 400)
         end
       end
     end
diff --git a/lib/api/notes.rb b/lib/api/notes.rb
index 0ef9a3c4beb..3726be7c537 100644
--- a/lib/api/notes.rb
+++ b/lib/api/notes.rb
@@ -61,9 +61,42 @@ module API
           if @note.valid?
             present @note, with: Entities::Note
           else
-            not_found!
+            not_found!("Note #{@note.errors.messages}")
           end
         end
+
+        # Modify existing +noteable+ note
+        #
+        # Parameters:
+        #   id (required) - The ID of a project
+        #   noteable_id (required) - The ID of an issue or snippet
+        #   node_id (required) - The ID of a note
+        #   body (required) - New content of a note
+        # Example Request:
+        #   PUT /projects/:id/issues/:noteable_id/notes/:note_id
+        #   PUT /projects/:id/snippets/:noteable_id/notes/:node_id
+        put ":id/#{noteables_str}/:#{noteable_id_str}/notes/:note_id" do
+          required_attributes! [:body]
+
+          authorize! :admin_note, user_project.notes.find(params[:note_id])
+
+          opts = {
+            note: params[:body],
+            note_id: params[:note_id],
+            noteable_type: noteables_str.classify,
+            noteable_id: params[noteable_id_str]
+          }
+
+          @note = ::Notes::UpdateService.new(user_project, current_user,
+                                             opts).execute
+
+          if @note.valid?
+            present @note, with: Entities::Note
+          else
+            render_api_error!("Failed to save note #{note.errors.messages}", 400)
+          end
+        end
+
       end
     end
   end
diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb
index 7d056b9bf58..be9850367b9 100644
--- a/lib/api/project_hooks.rb
+++ b/lib/api/project_hooks.rb
@@ -53,7 +53,7 @@ module API
           if @hook.errors[:url].present?
             error!("Invalid url given", 422)
           end
-          not_found!
+          not_found!("Project hook #{@hook.errors.messages}")
         end
       end
 
@@ -82,7 +82,7 @@ module API
           if @hook.errors[:url].present?
             error!("Invalid url given", 422)
           end
-          not_found!
+          not_found!("Project hook #{@hook.errors.messages}")
         end
       end
 
diff --git a/lib/api/project_members.rb b/lib/api/project_members.rb
index 1595ed0bc36..8e32f124ea5 100644
--- a/lib/api/project_members.rb
+++ b/lib/api/project_members.rb
@@ -9,7 +9,7 @@ module API
           if errors[:access_level].any?
             error!(errors[:access_level], 422)
           end
-          not_found!
+          not_found!(errors)
         end
       end
 
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 7fcf97d1ad6..e1cc2348865 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -22,6 +22,15 @@ module API
       #   GET /projects
       get do
         @projects = current_user.authorized_projects
+        sort = params[:sort] == 'desc' ? 'desc' : 'asc'
+
+        @projects = case params["order_by"]
+                    when 'id' then @projects.reorder("id #{sort}")
+                    when 'name' then @projects.reorder("name #{sort}")
+                    when 'created_at' then @projects.reorder("created_at #{sort}")
+                    when 'last_activity_at' then @projects.reorder("last_activity_at #{sort}")
+                    else @projects
+                    end
 
         # If the archived parameter is passed, limit results accordingly
         if params[:archived].present?
@@ -37,7 +46,17 @@ module API
       # Example Request:
       #   GET /projects/owned
       get '/owned' do
-        @projects = paginate current_user.owned_projects
+        sort = params[:sort] == 'desc' ? 'desc' : 'asc'
+        @projects = current_user.owned_projects
+        @projects = case params["order_by"]
+                    when 'id' then @projects.reorder("id #{sort}")
+                    when 'name' then @projects.reorder("name #{sort}")
+                    when 'created_at' then @projects.reorder("created_at #{sort}")
+                    when 'last_activity_at' then @projects.reorder("last_activity_at #{sort}")
+                    else @projects
+                    end
+
+        @projects = paginate @projects
         present @projects, with: Entities::Project
       end
 
@@ -47,7 +66,17 @@ module API
       #   GET /projects/all
       get '/all' do
         authenticated_as_admin!
-        @projects = paginate Project
+        sort = params[:sort] == 'desc' ? 'desc' : 'asc'
+
+        @projects = case params["order_by"]
+                    when 'id' then Project.order("id #{sort}")
+                    when 'name' then Project.order("name #{sort}")
+                    when 'created_at' then Project.order("created_at #{sort}")
+                    when 'last_activity_at' then Project.order("last_activity_at #{sort}")
+                    else Project
+                    end
+
+        @projects = paginate @projects
         present @projects, with: Entities::Project
       end
 
@@ -198,7 +227,7 @@ module API
             render_api_error!("Project already forked", 409)
           end
         else
-          not_found!
+          not_found!("Source Project")
         end
 
       end
@@ -227,6 +256,16 @@ module API
         ids = current_user.authorized_projects.map(&:id)
         visibility_levels = [ Gitlab::VisibilityLevel::INTERNAL, Gitlab::VisibilityLevel::PUBLIC ]
         projects = Project.where("(id in (?) OR visibility_level in (?)) AND (name LIKE (?))", ids, visibility_levels, "%#{params[:query]}%")
+        sort = params[:sort] == 'desc' ? 'desc' : 'asc'
+
+        projects = case params["order_by"]
+                   when 'id' then projects.order("id #{sort}")
+                   when 'name' then projects.order("name #{sort}")
+                   when 'created_at' then projects.order("created_at #{sort}")
+                   when 'last_activity_at' then projects.order("last_activity_at #{sort}")
+                   else projects
+                   end
+
         present paginate(projects), with: Entities::Project
       end
 
diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb
index a1a7721b288..03a556a2c55 100644
--- a/lib/api/repositories.rb
+++ b/lib/api/repositories.rb
@@ -133,7 +133,7 @@ module API
           env['api.format'] = :binary
           present data
         else
-          not_found!
+          not_found!('File')
         end
       end
 
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index 875f8d8b3a3..d47ef61fd11 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -79,16 +79,8 @@ module Gitlab
       oldrev, newrev, ref = change.split(' ')
 
       action = if project.protected_branch?(branch_name(ref))
-                 # we dont allow force push to protected branch
-                 if forced_push?(project, oldrev, newrev)
-                   :force_push_code_to_protected_branches
-                   # and we dont allow remove of protected branch
-                 elsif newrev == Gitlab::Git::BLANK_SHA
-                   :remove_protected_branches
-                 else
-                   :push_code_to_protected_branches
-                 end
-               elsif project.repository.tag_names.include?(tag_name(ref))
+                 protected_branch_action(project, oldrev, newrev, branch_name(ref))
+               elsif protected_tag?(project, tag_name(ref))
                  # Prevent any changes to existing git tag unless user has permissions
                  :admin_project
                else
@@ -108,6 +100,24 @@ module Gitlab
 
     private
 
+    def protected_branch_action(project, oldrev, newrev, branch_name)
+      # we dont allow force push to protected branch
+      if forced_push?(project, oldrev, newrev)
+       :force_push_code_to_protected_branches
+       # and we dont allow remove of protected branch
+      elsif newrev == Gitlab::Git::BLANK_SHA
+       :remove_protected_branches
+      elsif project.developers_can_push_to_protected_branch?(branch_name)
+       :push_code
+      else
+       :push_code_to_protected_branches
+      end
+    end
+
+    def protected_tag?(project, tag_name)
+      project.repository.tag_names.include?(tag_name)
+    end
+
     def user_allowed?(user)
       Gitlab::UserAccess.allowed?(user)
     end
diff --git a/lib/gitlab/theme.rb b/lib/gitlab/theme.rb
index b7c50cb734d..a7c83a880f6 100644
--- a/lib/gitlab/theme.rb
+++ b/lib/gitlab/theme.rb
@@ -19,5 +19,19 @@ module Gitlab
 
       return themes[id]
     end
+
+    def self.type_css_class_by_id(id)
+      types = {
+        BASIC  => 'light_theme',
+        MARS   => 'dark_theme',
+        MODERN => 'dark_theme',
+        GRAY   => 'dark_theme',
+        COLOR  => 'dark_theme'
+      }
+
+      id ||= Gitlab.config.gitlab.default_theme
+
+      types[id]
+    end
   end
 end
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index 7ff23a7600a..43115915de1 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -585,10 +585,6 @@ namespace :gitlab do
     def gitlab_shell_patch_version
       Gitlab::Shell.version_required.split('.')[2].to_i
     end
-
-    def has_gitlab_shell3?
-      gitlab_shell_version.try(:start_with?, "v3.")
-    end
   end
 
 
@@ -790,14 +786,14 @@ namespace :gitlab do
   end
 
   def sanitized_message(project)
-    if sanitize
+    if should_sanitize?
       "#{project.namespace_id.to_s.yellow}/#{project.id.to_s.yellow} ... "
     else
       "#{project.name_with_namespace.yellow} ... "
     end
   end
 
-  def sanitize
+  def should_sanitize?
     if ENV['SANITIZE'] == "true"
       true
     else