Merge remote-tracking branch 'upstream/master' into artifacts-from-ref-and-build-name

* upstream/master: (123 commits)
  Limit the size of SVGs when viewing them as blobs
  Add a spec for ProjectsFinder project_ids_relation option
  Fix ProjectsFinder spec
  Pass project IDs relation to ProjectsFinder instead of using a block
  Speed up todos queries by limiting the projects set we join with
  Used phantomjs variable
  Ability to specify branches for pivotal tracker integration
  Fix a memory leak caused by Banzai::Filter::SanitizationFilter
  Update phantomjs link
  Remove sleeping and replace escaped text.
  Filters test fix
  Fixed filtering tests
  Removed screenshot command :poop:
  Updated failing tests
  Used mirrored version on GitLab
  Updated tests
  Fix `U2fSpec` for PhantomJS versions > 2.
  Fix file downloading
  Install latest stable phantomjs
  Use new PhantomJS version
  ...

16 files changed, 319 insertions, 122 deletions

diff --git a/app/controllers/admin/groups_controller.rb b/app/controllers/admin/groups_controller.rb
index f3a88a8e6c8..4ce18321649 100644
--- a/app/controllers/admin/groups_controller.rb
+++ b/app/controllers/admin/groups_controller.rb
@@ -48,9 +48,9 @@ class Admin::GroupsController < Admin::ApplicationController
   end
 
   def destroy
-    DestroyGroupService.new(@group, current_user).execute
+    DestroyGroupService.new(@group, current_user).async_execute
 
-    redirect_to admin_groups_path, notice: 'Group was successfully deleted.'
+    redirect_to admin_groups_path, alert: "Group '#{@group.name}' was scheduled for deletion."
   end
 
   private
diff --git a/app/controllers/dashboard/todos_controller.rb b/app/controllers/dashboard/todos_controller.rb
index 19a76a5b5d8..1243bb96d4d 100644
--- a/app/controllers/dashboard/todos_controller.rb
+++ b/app/controllers/dashboard/todos_controller.rb
@@ -37,8 +37,8 @@ class Dashboard::TodosController < Dashboard::ApplicationController
 
   def todos_counts
     {
-      count: TodosFinder.new(current_user, state: :pending).execute.count,
-      done_count: TodosFinder.new(current_user, state: :done).execute.count
+      count: current_user.todos_pending_count,
+      done_count: current_user.todos_done_count
     }
   end
 end
diff --git a/app/controllers/groups_controller.rb b/app/controllers/groups_controller.rb
index 6780a6d4d87..cb82d62616c 100644
--- a/app/controllers/groups_controller.rb
+++ b/app/controllers/groups_controller.rb
@@ -87,9 +87,9 @@ class GroupsController < Groups::ApplicationController
   end
 
   def destroy
-    DestroyGroupService.new(@group, current_user).execute
+    DestroyGroupService.new(@group, current_user).async_execute
 
-    redirect_to root_path, alert: "Group '#{@group.name}' was successfully deleted."
+    redirect_to root_path, alert: "Group '#{@group.name}' was scheduled for deletion."
   end
 
   protected
diff --git a/app/controllers/projects/badges_controller.rb b/app/controllers/projects/badges_controller.rb
index a9f482c8787..d0f5071d2cc 100644
--- a/app/controllers/projects/badges_controller.rb
+++ b/app/controllers/projects/badges_controller.rb
@@ -8,8 +8,9 @@ class Projects::BadgesController < Projects::ApplicationController
 
     respond_to do |format|
       format.html { render_404 }
+
       format.svg do
-        send_data(badge.data, type: badge.type, disposition: 'inline')
+        render 'badge', locals: { badge: badge.template }
       end
     end
   end
diff --git a/app/controllers/projects/branches_controller.rb b/app/controllers/projects/branches_controller.rb
index e926043f3eb..48fe81b0d74 100644
--- a/app/controllers/projects/branches_controller.rb
+++ b/app/controllers/projects/branches_controller.rb
@@ -1,12 +1,13 @@
 class Projects::BranchesController < Projects::ApplicationController
   include ActionView::Helpers::SanitizeHelper
+  include SortingHelper
   # Authorize
   before_action :require_non_empty_project
   before_action :authorize_download_code!
   before_action :authorize_push_code!, only: [:new, :create, :destroy]
 
   def index
-    @sort = params[:sort].presence || 'name'
+    @sort = params[:sort].presence || sort_value_name
     @branches = BranchesFinder.new(@repository, params).execute
     @branches = Kaminari.paginate_array(@branches).page(params[:page])
 
diff --git a/app/controllers/projects/builds_controller.rb b/app/controllers/projects/builds_controller.rb
index 553b62741a5..12195c3cbb8 100644
--- a/app/controllers/projects/builds_controller.rb
+++ b/app/controllers/projects/builds_controller.rb
@@ -6,7 +6,7 @@ class Projects::BuildsController < Projects::ApplicationController
 
   def index
     @scope = params[:scope]
-    @all_builds = project.builds
+    @all_builds = project.builds.relevant
     @builds = @all_builds.order('created_at DESC')
     @builds =
       case @scope
diff --git a/app/controllers/projects/commit_controller.rb b/app/controllers/projects/commit_controller.rb
index fdfe7c65b7b..f44e9bb3fd7 100644
--- a/app/controllers/projects/commit_controller.rb
+++ b/app/controllers/projects/commit_controller.rb
@@ -134,8 +134,8 @@ class Projects::CommitController < Projects::ApplicationController
   end
 
   def define_status_vars
-    @statuses = CommitStatus.where(pipeline: pipelines)
-    @builds = Ci::Build.where(pipeline: pipelines)
+    @statuses = CommitStatus.where(pipeline: pipelines).relevant
+    @builds = Ci::Build.where(pipeline: pipelines).relevant
   end
 
   def assign_change_commit_vars(mr_source_branch)
diff --git a/app/controllers/projects/git_http_client_controller.rb b/app/controllers/projects/git_http_client_controller.rb
new file mode 100644
index 00000000000..7c21bd181dc
--- /dev/null
+++ b/app/controllers/projects/git_http_client_controller.rb
@@ -0,0 +1,110 @@
+# This file should be identical in GitLab Community Edition and Enterprise Edition
+
+class Projects::GitHttpClientController < Projects::ApplicationController
+  include ActionController::HttpAuthentication::Basic
+  include KerberosSpnegoHelper
+
+  attr_reader :user
+
+  # Git clients will not know what authenticity token to send along
+  skip_before_action :verify_authenticity_token
+  skip_before_action :repository
+  before_action :authenticate_user
+  before_action :ensure_project_found!
+
+  private
+
+  def authenticate_user
+    if project && project.public? && download_request?
+      return # Allow access
+    end
+
+    if allow_basic_auth? && basic_auth_provided?
+      login, password = user_name_and_password(request)
+      auth_result = Gitlab::Auth.find_for_git_client(login, password, project: project, ip: request.ip)
+
+      if auth_result.type == :ci && download_request?
+        @ci = true
+      elsif auth_result.type == :oauth && !download_request?
+        # Not allowed
+      else
+        @user = auth_result.user
+      end
+
+      if ci? || user
+        return # Allow access
+      end
+    elsif allow_kerberos_spnego_auth? && spnego_provided?
+      @user = find_kerberos_user
+
+      if user
+        send_final_spnego_response
+        return # Allow access
+      end
+    end
+
+    send_challenges
+    render plain: "HTTP Basic: Access denied\n", status: 401
+  end
+
+  def basic_auth_provided?
+    has_basic_credentials?(request)
+  end
+
+  def send_challenges
+    challenges = []
+    challenges << 'Basic realm="GitLab"' if allow_basic_auth?
+    challenges << spnego_challenge if allow_kerberos_spnego_auth?
+    headers['Www-Authenticate'] = challenges.join("\n") if challenges.any?
+  end
+
+  def ensure_project_found!
+    render_not_found if project.blank?
+  end
+
+  def project
+    return @project if defined?(@project)
+
+    project_id, _ = project_id_with_suffix
+    if project_id.blank?
+      @project = nil
+    else
+      @project = Project.find_with_namespace("#{params[:namespace_id]}/#{project_id}")
+    end
+  end
+
+  # This method returns two values so that we can parse
+  # params[:project_id] (untrusted input!) in exactly one place.
+  def project_id_with_suffix
+    id = params[:project_id] || ''
+
+    %w[.wiki.git .git].each do |suffix|
+      if id.end_with?(suffix)
+        # Be careful to only remove the suffix from the end of 'id'.
+        # Accidentally removing it from the middle is how security
+        # vulnerabilities happen!
+        return [id.slice(0, id.length - suffix.length), suffix]
+      end
+    end
+
+    # Something is wrong with params[:project_id]; do not pass it on.
+    [nil, nil]
+  end
+
+  def repository
+    _, suffix = project_id_with_suffix
+    if suffix == '.wiki.git'
+      project.wiki.repository
+    else
+      project.repository
+    end
+  end
+
+  def render_not_found
+    render plain: 'Not Found', status: :not_found
+  end
+
+  def ci?
+    @ci.present?
+  end
+end
diff --git a/app/controllers/projects/git_http_controller.rb b/app/controllers/projects/git_http_controller.rb
index e2f93e239bd..b4373ef89ef 100644
--- a/app/controllers/projects/git_http_controller.rb
+++ b/app/controllers/projects/git_http_controller.rb
@@ -1,17 +1,6 @@
 # This file should be identical in GitLab Community Edition and Enterprise Edition
 
-class Projects::GitHttpController < Projects::ApplicationController
-  include ActionController::HttpAuthentication::Basic
-  include KerberosSpnegoHelper
-
-  attr_reader :user
-
-  # Git clients will not know what authenticity token to send along
-  skip_before_action :verify_authenticity_token
-  skip_before_action :repository
-  before_action :authenticate_user
-  before_action :ensure_project_found!
-
+class Projects::GitHttpController < Projects::GitHttpClientController
   # GET /foo/bar.git/info/refs?service=git-upload-pack (git pull)
   # GET /foo/bar.git/info/refs?service=git-receive-pack (git push)
   def info_refs
@@ -46,81 +35,8 @@ class Projects::GitHttpController < Projects::ApplicationController
 
   private
 
-  def authenticate_user
-    if project && project.public? && upload_pack?
-      return # Allow access
-    end
-
-    if allow_basic_auth? && basic_auth_provided?
-      login, password = user_name_and_password(request)
-      auth_result = Gitlab::Auth.find_for_git_client(login, password, project: project, ip: request.ip)
-
-      if auth_result.type == :ci && upload_pack?
-        @ci = true
-      elsif auth_result.type == :oauth && !upload_pack?
-        # Not allowed
-      else
-        @user = auth_result.user
-      end
-
-      if ci? || user
-        return # Allow access
-      end
-    elsif allow_kerberos_spnego_auth? && spnego_provided?
-      @user = find_kerberos_user
-
-      if user
-        send_final_spnego_response
-        return # Allow access
-      end
-    end
-
-    send_challenges
-    render plain: "HTTP Basic: Access denied\n", status: 401
-  end
-
-  def basic_auth_provided?
-    has_basic_credentials?(request)
-  end
-
-  def send_challenges
-    challenges = []
-    challenges << 'Basic realm="GitLab"' if allow_basic_auth?
-    challenges << spnego_challenge if allow_kerberos_spnego_auth?
-    headers['Www-Authenticate'] = challenges.join("\n") if challenges.any?
-  end
-
-  def ensure_project_found!
-    render_not_found if project.blank?
-  end
-
-  def project
-    return @project if defined?(@project)
-
-    project_id, _ = project_id_with_suffix
-    if project_id.blank?
-      @project = nil
-    else
-      @project = Project.find_with_namespace("#{params[:namespace_id]}/#{project_id}")
-    end
-  end
-
-  # This method returns two values so that we can parse
-  # params[:project_id] (untrusted input!) in exactly one place.
-  def project_id_with_suffix
-    id = params[:project_id] || ''
-
-    %w[.wiki.git .git].each do |suffix|
-      if id.end_with?(suffix)
-        # Be careful to only remove the suffix from the end of 'id'.
-        # Accidentally removing it from the middle is how security
-        # vulnerabilities happen!
-        return [id.slice(0, id.length - suffix.length), suffix]
-      end
-    end
-
-    # Something is wrong with params[:project_id]; do not pass it on.
-    [nil, nil]
+  def download_request?
+    upload_pack?
   end
 
   def upload_pack?
@@ -143,19 +59,6 @@ class Projects::GitHttpController < Projects::ApplicationController
     render json: Gitlab::Workhorse.git_http_ok(repository, user)
   end
 
-  def repository
-    _, suffix = project_id_with_suffix
-    if suffix == '.wiki.git'
-      project.wiki.repository
-    else
-      project.repository
-    end
-  end
-
-  def render_not_found
-    render plain: 'Not Found', status: :not_found
-  end
-
   def render_http_not_allowed
     render plain: access_check.message, status: :forbidden
   end
@@ -169,10 +72,6 @@ class Projects::GitHttpController < Projects::ApplicationController
     end
   end
 
-  def ci?
-    @ci.present?
-  end
-
   def upload_pack_allowed?
     return false unless Gitlab.config.gitlab_shell.upload_pack
 
diff --git a/app/controllers/projects/lfs_api_controller.rb b/app/controllers/projects/lfs_api_controller.rb
new file mode 100644
index 00000000000..ece49dcd922
--- /dev/null
+++ b/app/controllers/projects/lfs_api_controller.rb
@@ -0,0 +1,94 @@
+class Projects::LfsApiController < Projects::GitHttpClientController
+  include LfsHelper
+
+  before_action :require_lfs_enabled!
+  before_action :lfs_check_access!, except: [:deprecated]
+
+  def batch
+    unless objects.present?
+      render_lfs_not_found
+      return
+    end
+
+    if download_request?
+      render json: { objects: download_objects! }
+    elsif upload_request?
+      render json: { objects: upload_objects! }
+    else
+      raise "Never reached"
+    end
+  end
+
+  def deprecated
+    render(
+      json: {
+        message: 'Server supports batch API only, please update your Git LFS client to version 1.0.1 and up.',
+        documentation_url: "#{Gitlab.config.gitlab.url}/help",
+      },
+      status: 501
+    )
+  end
+
+  private
+
+  def objects
+    @objects ||= (params[:objects] || []).to_a
+  end
+
+  def existing_oids
+    @existing_oids ||= begin
+      storage_project.lfs_objects.where(oid: objects.map { |o| o['oid'].to_s }).pluck(:oid)
+    end
+  end
+
+  def download_objects!
+    objects.each do |object|
+      if existing_oids.include?(object[:oid])
+        object[:actions] = download_actions(object)
+      else
+        object[:error] = {
+          code: 404,
+          message: "Object does not exist on the server or you don't have permissions to access it",
+        }
+      end
+    end
+    objects
+  end
+
+  def upload_objects!
+    objects.each do |object|
+      object[:actions] = upload_actions(object) unless existing_oids.include?(object[:oid])
+    end
+    objects
+  end
+
+  def download_actions(object)
+    {
+      download: {
+        href: "#{project.http_url_to_repo}/gitlab-lfs/objects/#{object[:oid]}",
+        header: {
+          Authorization: request.headers['Authorization']
+        }.compact
+      }
+    }
+  end
+
+  def upload_actions(object)
+    {
+      upload: {
+        href: "#{project.http_url_to_repo}/gitlab-lfs/objects/#{object[:oid]}/#{object[:size]}",
+        header: {
+          Authorization: request.headers['Authorization']
+        }.compact
+      }
+    }
+  end
+
+  def download_request?
+    params[:operation] == 'download'
+  end
+
+  def upload_request?
+    params[:operation] == 'upload'
+  end
+end
diff --git a/app/controllers/projects/lfs_storage_controller.rb b/app/controllers/projects/lfs_storage_controller.rb
new file mode 100644
index 00000000000..69066cb40e6
--- /dev/null
+++ b/app/controllers/projects/lfs_storage_controller.rb
@@ -0,0 +1,92 @@
+class Projects::LfsStorageController < Projects::GitHttpClientController
+  include LfsHelper
+
+  before_action :require_lfs_enabled!
+  before_action :lfs_check_access!
+
+  def download
+    lfs_object = LfsObject.find_by_oid(oid)
+    unless lfs_object && lfs_object.file.exists?
+      render_lfs_not_found
+      return
+    end
+
+    send_file lfs_object.file.path, content_type: "application/octet-stream"
+  end
+
+  def upload_authorize
+    render(
+      json: {
+        StoreLFSPath: "#{Gitlab.config.lfs.storage_path}/tmp/upload",
+        LfsOid: oid,
+        LfsSize: size,
+      },
+      content_type: 'application/json; charset=utf-8'
+    )
+  end
+
+  def upload_finalize
+    unless tmp_filename
+      render_lfs_forbidden
+      return
+    end
+
+    if store_file(oid, size, tmp_filename)
+      head 200
+    else
+      render plain: 'Unprocessable entity', status: 422
+    end
+  end
+
+  private
+
+  def download_request?
+    action_name == 'download'
+  end
+
+  def upload_request?
+    %w[upload_authorize upload_finalize].include? action_name
+  end
+
+  def oid
+    params[:oid].to_s
+  end
+
+  def size
+    params[:size].to_i
+  end
+
+  def tmp_filename
+    name = request.headers['X-Gitlab-Lfs-Tmp']
+    return if name.include?('/')
+    return unless oid.present? && name.start_with?(oid)
+    name
+  end
+
+  def store_file(oid, size, tmp_file)
+    # Define tmp_file_path early because we use it in "ensure"
+    tmp_file_path = File.join("#{Gitlab.config.lfs.storage_path}/tmp/upload", tmp_file)
+
+    object = LfsObject.find_or_create_by(oid: oid, size: size)
+    file_exists = object.file.exists? || move_tmp_file_to_storage(object, tmp_file_path)
+    file_exists && link_to_project(object)
+  ensure
+    FileUtils.rm_f(tmp_file_path)
+  end
+
+  def move_tmp_file_to_storage(object, path)
+    File.open(path) do |f|
+      object.file = f
+    end
+
+    object.file.store!
+    object.save
+  end
+
+  def link_to_project(object)
+    if object && !object.projects.exists?(storage_project.id)
+      object.projects << storage_project
+      object.save
+    end
+  end
+end
diff --git a/app/controllers/projects/merge_requests_controller.rb b/app/controllers/projects/merge_requests_controller.rb
index 2cf6a2dd1b3..139680d2df9 100644
--- a/app/controllers/projects/merge_requests_controller.rb
+++ b/app/controllers/projects/merge_requests_controller.rb
@@ -160,7 +160,7 @@ class Projects::MergeRequestsController < Projects::ApplicationController
     @diff_notes_disabled = true
 
     @pipeline = @merge_request.pipeline
-    @statuses = @pipeline.statuses if @pipeline
+    @statuses = @pipeline.statuses.relevant if @pipeline
 
     @note_counts = Note.where(commit_id: @commits.map(&:id)).
       group(:commit_id).count
@@ -362,7 +362,7 @@ class Projects::MergeRequestsController < Projects::ApplicationController
     @commits_count = @merge_request.commits.count
 
     @pipeline = @merge_request.pipeline
-    @statuses = @pipeline.statuses if @pipeline
+    @statuses = @pipeline.statuses.relevant if @pipeline
 
     if @merge_request.locked_long_ago?
       @merge_request.unlock_mr
diff --git a/app/controllers/projects/pipelines_controller.rb b/app/controllers/projects/pipelines_controller.rb
index 487963fdcd7..b0c72cfe4b4 100644
--- a/app/controllers/projects/pipelines_controller.rb
+++ b/app/controllers/projects/pipelines_controller.rb
@@ -19,7 +19,7 @@ class Projects::PipelinesController < Projects::ApplicationController
   end
 
   def create
-    @pipeline = Ci::CreatePipelineService.new(project, current_user, create_params).execute
+    @pipeline = Ci::CreatePipelineService.new(project, current_user, create_params).execute(ignore_skip_ci: true, save_on_errors: false)
     unless @pipeline.persisted?
       render 'new'
       return
diff --git a/app/controllers/projects/pipelines_settings_controller.rb b/app/controllers/projects/pipelines_settings_controller.rb
index 85ba706e5cd..75dd3648e45 100644
--- a/app/controllers/projects/pipelines_settings_controller.rb
+++ b/app/controllers/projects/pipelines_settings_controller.rb
@@ -3,7 +3,7 @@ class Projects::PipelinesSettingsController < Projects::ApplicationController
 
   def show
     @ref = params[:ref] || @project.default_branch || 'master'
-    @build_badge = Gitlab::Badge::Build.new(@project, @ref)
+    @build_badge = Gitlab::Badge::Build.new(@project, @ref).metadata
   end
 
   def update
diff --git a/app/controllers/projects/wikis_controller.rb b/app/controllers/projects/wikis_controller.rb
index 607fe9c7fed..177ccf5eec9 100644
--- a/app/controllers/projects/wikis_controller.rb
+++ b/app/controllers/projects/wikis_controller.rb
@@ -91,7 +91,7 @@ class Projects::WikisController < Projects::ApplicationController
     )
   end
 
-  def markdown_preview
+  def preview_markdown
     text = params[:text]
 
     ext = Gitlab::ReferenceExtractor.new(@project, current_user)
diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb
index a6e1aa5ccc1..47efbd4a939 100644
--- a/app/controllers/projects_controller.rb
+++ b/app/controllers/projects_controller.rb
@@ -125,7 +125,7 @@ class ProjectsController < Projects::ApplicationController
   def destroy
     return access_denied! unless can?(current_user, :remove_project, @project)
 
-    ::Projects::DestroyService.new(@project, current_user, {}).pending_delete!
+    ::Projects::DestroyService.new(@project, current_user, {}).async_execute
     flash[:alert] = "Project '#{@project.name}' will be deleted."
 
     redirect_to dashboard_projects_path
@@ -238,7 +238,7 @@ class ProjectsController < Projects::ApplicationController
     }
   end
 
-  def markdown_preview
+  def preview_markdown
     text = params[:text]
 
     ext = Gitlab::ReferenceExtractor.new(@project, current_user)