Merge branch 'master' into 'focus-mode-board'focus-mode-board

# Conflicts:
#   app/views/shared/issuable/_filter.html.haml

18 files changed, 307 insertions, 49 deletions

diff --git a/app/models/ability.rb b/app/models/ability.rb
index ad6c588202e..f3692a5a067 100644
--- a/app/models/ability.rb
+++ b/app/models/ability.rb
@@ -56,15 +56,16 @@ class Ability
       end
     end
 
-    def allowed?(user, action, subject)
+    def allowed?(user, action, subject = :global)
       allowed(user, subject).include?(action)
     end
 
-    def allowed(user, subject)
+    def allowed(user, subject = :global)
+      return BasePolicy::RuleSet.none if subject.nil?
       return uncached_allowed(user, subject) unless RequestStore.active?
 
       user_key = user ? user.id : 'anonymous'
-      subject_key = subject ? "#{subject.class.name}/#{subject.id}" : 'global'
+      subject_key = subject == :global ? 'global' : "#{subject.class.name}/#{subject.id}"
       key = "/ability/#{user_key}/#{subject_key}"
       RequestStore[key] ||= uncached_allowed(user, subject).freeze
     end
diff --git a/app/models/blob.rb b/app/models/blob.rb
index ab92e820335..1376b86fdad 100644
--- a/app/models/blob.rb
+++ b/app/models/blob.rb
@@ -54,9 +54,13 @@ class Blob < SimpleDelegator
     UploaderHelper::VIDEO_EXT.include?(extname.downcase.delete('.'))
   end
 
-  def to_partial_path
+  def to_partial_path(project)
     if lfs_pointer?
-      'download'
+      if project.lfs_enabled?
+        'download'
+      else
+        'text'
+      end
     elsif image? || svg?
       'image'
     elsif text?
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index 8a5a9aa4adb..d1009f88549 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -22,6 +22,7 @@ module Ci
     validate :valid_commit_sha, unless: :importing?
 
     after_create :keep_around_commits, unless: :importing?
+    after_create :refresh_build_status_cache
 
     state_machine :status, initial: :created do
       event :enqueue do
@@ -114,6 +115,12 @@ module Ci
       success.latest(ref).order(id: :desc).first
     end
 
+    def self.latest_successful_for_refs(refs)
+      success.latest(refs).order(id: :desc).each_with_object({}) do |pipeline, hash|
+        hash[pipeline.ref] ||= pipeline
+      end
+    end
+
     def self.truncate_sha(sha)
       sha[0...8]
     end
@@ -328,6 +335,7 @@ module Ci
         when 'manual' then block
         end
       end
+      refresh_build_status_cache
     end
 
     def predefined_variables
@@ -369,6 +377,10 @@ module Ci
         .fabricate!
     end
 
+    def refresh_build_status_cache
+      Ci::PipelineStatus.new(project, sha: sha, status: status).store_in_cache_if_needed
+    end
+
     private
 
     def pipeline_data
diff --git a/app/models/ci/pipeline_status.rb b/app/models/ci/pipeline_status.rb
new file mode 100644
index 00000000000..048047d0e34
--- /dev/null
+++ b/app/models/ci/pipeline_status.rb
@@ -0,0 +1,86 @@
+# This class is not backed by a table in the main database.
+# It loads the latest Pipeline for the HEAD of a repository, and caches that
+# in Redis.
+module Ci
+  class PipelineStatus
+    attr_accessor :sha, :status, :project, :loaded
+
+    delegate :commit, to: :project
+
+    def self.load_for_project(project)
+      new(project).tap do |status|
+        status.load_status
+      end
+    end
+
+    def initialize(project, sha: nil, status: nil)
+      @project = project
+      @sha = sha
+      @status = status
+    end
+
+    def has_status?
+      loaded? && sha.present? && status.present?
+    end
+
+    def load_status
+      return if loaded?
+
+      if has_cache?
+        load_from_cache
+      else
+        load_from_commit
+        store_in_cache
+      end
+
+      self.loaded = true
+    end
+
+    def load_from_commit
+      return unless commit
+
+      self.sha = commit.sha
+      self.status = commit.status
+    end
+
+    # We only cache the status for the HEAD commit of a project
+    # This status is rendered in project lists
+    def store_in_cache_if_needed
+      return unless sha
+      return delete_from_cache unless commit
+      store_in_cache if commit.sha == self.sha
+    end
+
+    def load_from_cache
+      Gitlab::Redis.with do |redis|
+        self.sha, self.status = redis.hmget(cache_key, :sha, :status)
+      end
+    end
+
+    def store_in_cache
+      Gitlab::Redis.with do |redis|
+        redis.mapped_hmset(cache_key, { sha: sha, status: status })
+      end
+    end
+
+    def delete_from_cache
+      Gitlab::Redis.with do |redis|
+        redis.del(cache_key)
+      end
+    end
+
+    def has_cache?
+      Gitlab::Redis.with do |redis|
+        redis.exists(cache_key)
+      end
+    end
+
+    def loaded?
+      self.loaded
+    end
+
+    def cache_key
+      "projects/#{project.id}/build_status"
+    end
+  end
+end
diff --git a/app/models/commit.rb b/app/models/commit.rb
index 0a18986ef26..6ea5b1ae51f 100644
--- a/app/models/commit.rb
+++ b/app/models/commit.rb
@@ -231,6 +231,10 @@ class Commit
     project.pipelines.where(sha: sha)
   end
 
+  def latest_pipeline
+    pipelines.last
+  end
+
   def status(ref = nil)
     @statuses ||= {}
 
diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb
index 3cf4c67d7e7..91f4eb13ecc 100644
--- a/app/models/concerns/issuable.rb
+++ b/app/models/concerns/issuable.rb
@@ -144,7 +144,8 @@ module Issuable
                when 'milestone_due_desc' then order_milestone_due_desc
                when 'downvotes_desc' then order_downvotes_desc
                when 'upvotes_desc' then order_upvotes_desc
-               when 'priority' then order_labels_priority(excluded_labels: excluded_labels)
+               when 'label_priority' then order_labels_priority(excluded_labels: excluded_labels)
+               when 'priority' then order_due_date_and_labels_priority(excluded_labels: excluded_labels)
                when 'position_asc' then  order_position_asc
                else
                  order_by(method)
@@ -154,7 +155,28 @@ module Issuable
       sorted.order(id: :desc)
     end
 
-    def order_labels_priority(excluded_labels: [])
+    def order_due_date_and_labels_priority(excluded_labels: [])
+      # The order_ methods also modify the query in other ways:
+      #
+      # - For milestones, we add a JOIN.
+      # - For label priority, we change the SELECT, and add a GROUP BY.#
+      #
+      # After doing those, we need to reorder to the order we want. The existing
+      # ORDER BYs won't work because:
+      #
+      # 1. We need milestone due date first.
+      # 2. We can't ORDER BY a column that isn't in the GROUP BY and doesn't
+      #    have an aggregate function applied, so we do a useless MIN() instead.
+      #
+      milestones_due_date = 'MIN(milestones.due_date)'
+
+      order_milestone_due_asc.
+        order_labels_priority(excluded_labels: excluded_labels, extra_select_columns: [milestones_due_date]).
+        reorder(Gitlab::Database.nulls_last_order(milestones_due_date, 'ASC'),
+                Gitlab::Database.nulls_last_order('highest_priority', 'ASC'))
+    end
+
+    def order_labels_priority(excluded_labels: [], extra_select_columns: [])
       params = {
         target_type: name,
         target_column: "#{table_name}.id",
@@ -164,7 +186,12 @@ module Issuable
 
       highest_priority = highest_label_priority(params).to_sql
 
-      select("#{table_name}.*, (#{highest_priority}) AS highest_priority").
+      select_columns = [
+        "#{table_name}.*",
+        "(#{highest_priority}) AS highest_priority"
+      ] + extra_select_columns
+
+      select(select_columns.join(', ')).
         group(arel_table[:id]).
         reorder(Gitlab::Database.nulls_last_order('highest_priority', 'ASC'))
     end
@@ -234,6 +261,7 @@ module Issuable
       user: user.hook_attrs,
       project: project.hook_attrs,
       object_attributes: hook_attrs,
+      labels: labels.map(&:hook_attrs),
       # DEPRECATED
       repository: project.hook_attrs.slice(:name, :url, :description, :homepage)
     }
diff --git a/app/models/concerns/relative_positioning.rb b/app/models/concerns/relative_positioning.rb
index 603f2dd7e5d..f1d8532a6d6 100644
--- a/app/models/concerns/relative_positioning.rb
+++ b/app/models/concerns/relative_positioning.rb
@@ -2,16 +2,14 @@ module RelativePositioning
   extend ActiveSupport::Concern
 
   MIN_POSITION = 0
+  START_POSITION = Gitlab::Database::MAX_INT_VALUE / 2
   MAX_POSITION = Gitlab::Database::MAX_INT_VALUE
+  IDEAL_DISTANCE = 500
 
   included do
     after_save :save_positionable_neighbours
   end
 
-  def min_relative_position
-    self.class.in_projects(project.id).minimum(:relative_position)
-  end
-
   def max_relative_position
     self.class.in_projects(project.id).maximum(:relative_position)
   end
@@ -26,7 +24,7 @@ module RelativePositioning
         maximum(:relative_position)
     end
 
-    prev_pos || MIN_POSITION
+    prev_pos
   end
 
   def next_relative_position
@@ -39,55 +37,95 @@ module RelativePositioning
         minimum(:relative_position)
     end
 
-    next_pos || MAX_POSITION
+    next_pos
   end
 
   def move_between(before, after)
     return move_after(before) unless after
     return move_before(after) unless before
 
+    # If there is no place to insert an issue we need to create one by moving the before issue closer
+    # to its predecessor. This process will recursively move all the predecessors until we have a place
+    if (after.relative_position - before.relative_position) < 2
+      before.move_before
+      @positionable_neighbours = [before]
+    end
+
+    self.relative_position = position_between(before.relative_position, after.relative_position)
+  end
+
+  def move_after(before = self)
     pos_before = before.relative_position
+    pos_after = before.next_relative_position
+
+    if before.shift_after?
+      issue_to_move = self.class.in_projects(project.id).find_by!(relative_position: pos_after)
+      issue_to_move.move_after
+      @positionable_neighbours = [issue_to_move]
+
+      pos_after = issue_to_move.relative_position
+    end
+
+    self.relative_position = position_between(pos_before, pos_after)
+  end
+
+  def move_before(after = self)
     pos_after = after.relative_position
+    pos_before = after.prev_relative_position
 
-    if pos_after && (pos_before == pos_after)
-      self.relative_position = pos_before
-      before.move_before(self)
-      after.move_after(self)
+    if after.shift_before?
+      issue_to_move = self.class.in_projects(project.id).find_by!(relative_position: pos_before)
+      issue_to_move.move_before
+      @positionable_neighbours = [issue_to_move]
 
-      @positionable_neighbours = [before, after]
-    else
-      self.relative_position = position_between(pos_before, pos_after)
+      pos_before = issue_to_move.relative_position
     end
+
+    self.relative_position = position_between(pos_before, pos_after)
   end
 
-  def move_before(after)
-    self.relative_position = position_between(after.prev_relative_position, after.relative_position)
+  def move_to_end
+    self.relative_position = position_between(max_relative_position || START_POSITION, MAX_POSITION)
   end
 
-  def move_after(before)
-    self.relative_position = position_between(before.relative_position, before.next_relative_position)
+  # Indicates if there is an issue that should be shifted to free the place
+  def shift_after?
+    next_pos = next_relative_position
+    next_pos && (next_pos - relative_position) == 1
   end
 
-  def move_to_end
-    self.relative_position = position_between(max_relative_position, MAX_POSITION)
+  # Indicates if there is an issue that should be shifted to free the place
+  def shift_before?
+    prev_pos = prev_relative_position
+    prev_pos && (relative_position - prev_pos) == 1
   end
 
   private
 
   # This method takes two integer values (positions) and
-  # calculates some random position between them. The range is huge as
-  # the maximum integer value is 2147483647. Ideally, the calculated value would be
-  # exactly between those terminating values, but this will introduce possibility of a race condition
-  # so two or more issues can get the same value, we want to avoid that and we also want to avoid
-  # using a lock here. If we have two issues with distance more than one thousand, we are OK.
-  # Given the huge range of possible values that integer can fit we shoud never face a problem.
+  # calculates the position between them. The range is huge as
+  # the maximum integer value is 2147483647. We are incrementing position by IDEAL_DISTANCE * 2 every time
+  # when we have enough space. If distance is less then IDEAL_DISTANCE we are calculating an average number
   def position_between(pos_before, pos_after)
     pos_before ||= MIN_POSITION
     pos_after ||= MAX_POSITION
 
     pos_before, pos_after = [pos_before, pos_after].sort
 
-    rand(pos_before.next..pos_after.pred)
+    halfway = (pos_after + pos_before) / 2
+    distance_to_halfway = pos_after - halfway
+
+    if distance_to_halfway < IDEAL_DISTANCE
+      halfway
+    else
+      if pos_before == MIN_POSITION
+        pos_after - IDEAL_DISTANCE
+      elsif pos_after == MAX_POSITION
+        pos_before + IDEAL_DISTANCE
+      else
+        halfway
+      end
+    end
   end
 
   def save_positionable_neighbours
diff --git a/app/models/global_milestone.rb b/app/models/global_milestone.rb
index b991d78e27f..0afbca2cb32 100644
--- a/app/models/global_milestone.rb
+++ b/app/models/global_milestone.rb
@@ -28,6 +28,28 @@ class GlobalMilestone
     new(title, child_milestones)
   end
 
+  def self.states_count(projects)
+    relation = MilestonesFinder.new.execute(projects, state: 'all')
+    milestones_by_state_and_title = relation.reorder(nil).group(:state, :title).count
+
+    opened = count_by_state(milestones_by_state_and_title, 'active')
+    closed = count_by_state(milestones_by_state_and_title, 'closed')
+    all = milestones_by_state_and_title.map { |(_, title), _| title }.uniq.count
+
+    { 
+      opened: opened,
+      closed: closed,
+      all: all
+    }
+  end
+
+  def self.count_by_state(milestones_by_state_and_title, state)
+    milestones_by_state_and_title.count do |(milestone_state, _), _|
+      milestone_state == state
+    end
+  end
+  private_class_method :count_by_state
+
   def initialize(title, milestones)
     @title = title
     @name = title
diff --git a/app/models/guest.rb b/app/models/guest.rb
index 01285ca1264..df287c277a7 100644
--- a/app/models/guest.rb
+++ b/app/models/guest.rb
@@ -1,6 +1,6 @@
 class Guest
   class << self
-    def can?(action, subject)
+    def can?(action, subject = :global)
       Ability.allowed?(nil, action, subject)
     end
   end
diff --git a/app/models/issue.rb b/app/models/issue.rb
index 0f7a26ee3e1..1427fdc31a4 100644
--- a/app/models/issue.rb
+++ b/app/models/issue.rb
@@ -58,7 +58,13 @@ class Issue < ActiveRecord::Base
   end
 
   def hook_attrs
-    attributes
+    attrs = {
+      total_time_spent: total_time_spent,
+      human_total_time_spent: human_total_time_spent,
+      human_time_estimate: human_time_estimate
+    }
+
+    attributes.merge!(attrs)
   end
 
   def self.reference_prefix
@@ -96,6 +102,13 @@ class Issue < ActiveRecord::Base
     end
   end
 
+  def self.order_by_position_and_priority
+    order_labels_priority.
+      reorder(Gitlab::Database.nulls_last_order('relative_position', 'ASC'),
+              Gitlab::Database.nulls_last_order('highest_priority', 'ASC'),
+              "id DESC")
+  end
+
   # `from` argument can be a Namespace or Project.
   def to_reference(from = nil, full: false)
     reference = "#{self.class.reference_prefix}#{iid}"
diff --git a/app/models/label.rb b/app/models/label.rb
index f68a8c9cff2..568fa6d44f5 100644
--- a/app/models/label.rb
+++ b/app/models/label.rb
@@ -169,6 +169,10 @@ class Label < ActiveRecord::Base
     end
   end
 
+  def hook_attrs
+    attributes
+  end
+
   private
 
   def issues_count(user, params = {})
diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb
index 0f7b8311588..4759829a15c 100644
--- a/app/models/merge_request.rb
+++ b/app/models/merge_request.rb
@@ -523,7 +523,10 @@ class MergeRequest < ActiveRecord::Base
       source: source_project.try(:hook_attrs),
       target: target_project.hook_attrs,
       last_commit: nil,
-      work_in_progress: work_in_progress?
+      work_in_progress: work_in_progress?,
+      total_time_spent: total_time_spent,
+      human_total_time_spent: human_total_time_spent,
+      human_time_estimate: human_time_estimate
     }
 
     if diff_head_commit
diff --git a/app/models/milestone.rb b/app/models/milestone.rb
index 7331000a9f2..c0deb59ec4c 100644
--- a/app/models/milestone.rb
+++ b/app/models/milestone.rb
@@ -5,6 +5,7 @@ class Milestone < ActiveRecord::Base
   None = MilestoneStruct.new('No Milestone', 'No Milestone', 0)
   Any = MilestoneStruct.new('Any Milestone', '', -1)
   Upcoming = MilestoneStruct.new('Upcoming', '#upcoming', -2)
+  Started = MilestoneStruct.new('Started', '#started', -3)
 
   include CacheMarkdownField
   include InternalId
diff --git a/app/models/project.rb b/app/models/project.rb
index 8c2dadf4659..2ffaaac93f3 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -1209,6 +1209,10 @@ class Project < ActiveRecord::Base
     end
   end
 
+  def pipeline_status
+    @pipeline_status ||= Ci::PipelineStatus.load_for_project(self)
+  end
+
   def mark_import_as_failed(error_message)
     original_errors = errors.dup
     sanitized_message = Gitlab::UrlSanitizer.sanitize(error_message)
diff --git a/app/models/project_services/issue_tracker_service.rb b/app/models/project_services/issue_tracker_service.rb
index 9e65fdbf9d6..50435b67eda 100644
--- a/app/models/project_services/issue_tracker_service.rb
+++ b/app/models/project_services/issue_tracker_service.rb
@@ -1,4 +1,6 @@
 class IssueTrackerService < Service
+  validate :one_issue_tracker, if: :activated?, on: :manual_change
+
   default_value_for :category, 'issue_tracker'
 
   # Pattern used to extract links from comments
@@ -92,4 +94,13 @@ class IssueTrackerService < Service
   def issues_tracker
     Gitlab.config.issues_tracker[to_param]
   end
+
+  def one_issue_tracker
+    return if template?
+    return if project.blank?
+
+    if project.services.external_issue_trackers.where.not(id: id).any?
+      errors.add(:base, 'Another issue tracker is already in use. Only one issue tracker service can be active at a time')
+    end
+  end
 end
diff --git a/app/models/todo.rb b/app/models/todo.rb
index 47789a21133..da3fa7277c2 100644
--- a/app/models/todo.rb
+++ b/app/models/todo.rb
@@ -48,8 +48,14 @@ class Todo < ActiveRecord::Base
   after_save :keep_around_commit
 
   class << self
+    # Priority sorting isn't displayed in the dropdown, because we don't show
+    # milestones, but still show something if the user has a URL with that
+    # selected.
     def sort(method)
-      method == "priority" ? order_by_labels_priority : order_by(method)
+      case method.to_s
+      when 'priority', 'label_priority' then order_by_labels_priority
+      else order_by(method)
+      end
     end
 
     # Order by priority depending on which issue/merge request the Todo belongs to
diff --git a/app/models/user.rb b/app/models/user.rb
index 76fb4cd470e..39c1281179b 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -126,7 +126,6 @@ class User < ActiveRecord::Base
   validate :unique_email, if: ->(user) { user.email_changed? }
   validate :owns_notification_email, if: ->(user) { user.notification_email_changed? }
   validate :owns_public_email, if: ->(user) { user.public_email_changed? }
-  validate :ghost_users_must_be_blocked
   validates :avatar, file_size: { maximum: 200.kilobytes.to_i }
 
   before_validation :generate_password, on: :create
@@ -350,12 +349,27 @@ class User < ActiveRecord::Base
     def ghost
       unique_internal(where(ghost: true), 'ghost', 'ghost%s@example.com') do |u|
         u.bio = 'This is a "Ghost User", created to hold all issues authored by users that have since been deleted. This user cannot be removed.'
-        u.state = :blocked
         u.name = 'Ghost User'
       end
     end
   end
 
+  def self.internal_attributes
+    [:ghost]
+  end
+
+  def internal?
+    self.class.internal_attributes.any? { |a| self[a] }
+  end
+
+  def self.internal
+    where(Hash[internal_attributes.zip([true] * internal_attributes.size)])
+  end
+
+  def self.non_internal
+    where(Hash[internal_attributes.zip([[false, nil]] * internal_attributes.size)])
+  end
+
   #
   # Instance methods
   #
@@ -452,12 +466,6 @@ class User < ActiveRecord::Base
     errors.add(:public_email, "is not an email you own") unless all_emails.include?(public_email)
   end
 
-  def ghost_users_must_be_blocked
-    if ghost? && !blocked?
-      errors.add(:ghost, 'cannot be enabled for a user who is not blocked.')
-    end
-  end
-
   def update_emails_with_primary_email
     primary_email_record = emails.find_by(email: email)
     if primary_email_record
@@ -563,14 +571,14 @@ class User < ActiveRecord::Base
   end
 
   def can_create_group?
-    can?(:create_group, nil)
+    can?(:create_group)
   end
 
   def can_select_namespace?
     several_namespaces? || admin
   end
 
-  def can?(action, subject)
+  def can?(action, subject = :global)
     Ability.allowed?(self, action, subject)
   end
 
@@ -955,6 +963,14 @@ class User < ActiveRecord::Base
     self.admin = (new_level == 'admin')
   end
 
+  protected
+
+  # override, from Devise::Validatable
+  def password_required?
+    return false if internal?
+    super
+  end
+
   private
 
   def ci_projects_union
@@ -1055,7 +1071,6 @@ class User < ActiveRecord::Base
 
     scope.create(
       username: username,
-      password: Devise.friendly_token,
       email: email,
       &creation_block
     )
diff --git a/app/models/wiki_page.rb b/app/models/wiki_page.rb
index 2caebb496db..465c4d903ac 100644
--- a/app/models/wiki_page.rb
+++ b/app/models/wiki_page.rb
@@ -149,6 +149,12 @@ class WikiPage
   end
 
   # Returns boolean True or False if this instance
+  # is the latest commit version of the page.
+  def latest?
+    !historical?
+  end
+
+  # Returns boolean True or False if this instance
   # has been fully saved to disk or not.
   def persisted?
     @persisted == true