Merge branch 'master' into dz-merge-request-version

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>

25 files changed, 406 insertions, 172 deletions

diff --git a/app/models/ability.rb b/app/models/ability.rb
index d95a2507199..d9113ffd99a 100644
--- a/app/models/ability.rb
+++ b/app/models/ability.rb
@@ -6,6 +6,10 @@ class Ability
       return [] unless user.is_a?(User)
       return [] if user.blocked?
 
+      abilities_by_subject_class(user: user, subject: subject)
+    end
+
+    def abilities_by_subject_class(user:, subject:)
       case subject
       when CommitStatus then commit_status_abilities(user, subject)
       when Project then project_abilities(user, subject)
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 08f396210c9..3d6c6ea3209 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -16,7 +16,7 @@ module Ci
     scope :with_artifacts_not_expired, ->() { with_artifacts.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) }
     scope :with_expired_artifacts, ->() { with_artifacts.where('artifacts_expire_at < ?', Time.now) }
     scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) }
-    scope :manual_actions, ->() { where(when: :manual) }
+    scope :manual_actions, ->() { where(when: :manual).relevant }
 
     mount_uploader :artifacts_file, ArtifactUploader
     mount_uploader :artifacts_metadata, ArtifactUploader
@@ -42,40 +42,35 @@ module Ci
       end
 
       def retry(build, user = nil)
-        new_build = Ci::Build.new(status: 'pending')
-        new_build.ref = build.ref
-        new_build.tag = build.tag
-        new_build.options = build.options
-        new_build.commands = build.commands
-        new_build.tag_list = build.tag_list
-        new_build.project = build.project
-        new_build.pipeline = build.pipeline
-        new_build.name = build.name
-        new_build.allow_failure = build.allow_failure
-        new_build.stage = build.stage
-        new_build.stage_idx = build.stage_idx
-        new_build.trigger_request = build.trigger_request
-        new_build.yaml_variables = build.yaml_variables
-        new_build.when = build.when
-        new_build.user = user
-        new_build.environment = build.environment
-        new_build.save
+        new_build = Ci::Build.create(
+          ref: build.ref,
+          tag: build.tag,
+          options: build.options,
+          commands: build.commands,
+          tag_list: build.tag_list,
+          project: build.project,
+          pipeline: build.pipeline,
+          name: build.name,
+          allow_failure: build.allow_failure,
+          stage: build.stage,
+          stage_idx: build.stage_idx,
+          trigger_request: build.trigger_request,
+          yaml_variables: build.yaml_variables,
+          when: build.when,
+          user: user,
+          environment: build.environment,
+          status_event: 'enqueue'
+        )
         MergeRequests::AddTodoWhenBuildFailsService.new(build.project, nil).close(new_build)
         new_build
       end
     end
 
-    state_machine :status, initial: :pending do
+    state_machine :status do
       after_transition pending: :running do |build|
         build.execute_hooks
       end
 
-      # We use around_transition to create builds for next stage as soon as possible, before the `after_*` is executed
-      around_transition any => [:success, :failed, :canceled] do |build, block|
-        block.call
-        build.pipeline.create_next_builds(build) if build.pipeline
-      end
-
       after_transition any => [:success, :failed, :canceled] do |build|
         build.update_coverage
         build.execute_hooks
@@ -107,7 +102,7 @@ module Ci
 
     def play(current_user = nil)
       # Try to queue a current build
-      if self.queue
+      if self.enqueue
         self.update(user: current_user)
         self
       else
@@ -461,7 +456,7 @@ module Ci
 
     def build_attributes_from_config
       return {} unless pipeline.config_processor
-      
+
       pipeline.config_processor.build_attributes(name)
     end
   end
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index bce6a992af6..8cfba92ae9b 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -13,13 +13,51 @@ module Ci
     has_many :trigger_requests, dependent: :destroy, class_name: 'Ci::TriggerRequest', foreign_key: :commit_id
 
     validates_presence_of :sha
+    validates_presence_of :ref
     validates_presence_of :status
     validate :valid_commit_sha
 
-    # Invalidate object and save if when touched
-    after_touch :update_state
     after_save :keep_around_commits
 
+    state_machine :status, initial: :created do
+      event :enqueue do
+        transition created: :pending
+        transition [:success, :failed, :canceled, :skipped] => :running
+      end
+
+      event :run do
+        transition any => :running
+      end
+
+      event :skip do
+        transition any => :skipped
+      end
+
+      event :drop do
+        transition any => :failed
+      end
+
+      event :succeed do
+        transition any => :success
+      end
+
+      event :cancel do
+        transition any => :canceled
+      end
+
+      before_transition [:created, :pending] => :running do |pipeline|
+        pipeline.started_at = Time.now
+      end
+
+      before_transition any => [:success, :failed, :canceled] do |pipeline|
+        pipeline.finished_at = Time.now
+      end
+
+      before_transition do |pipeline|
+        pipeline.update_duration
+      end
+    end
+
     # ref can't be HEAD or SHA, can only be branch/tag name
     scope :latest_successful_for, ->(ref = default_branch) do
       where(ref: ref).success.order(id: :desc).limit(1)
@@ -109,37 +147,6 @@ module Ci
       trigger_requests.any?
     end
 
-    def create_builds(user, trigger_request = nil)
-      ##
-      # We persist pipeline only if there are builds available
-      #
-      return unless config_processor
-
-      build_builds_for_stages(config_processor.stages, user,
-                              'success', trigger_request) && save
-    end
-
-    def create_next_builds(build)
-      return unless config_processor
-
-      # don't create other builds if this one is retried
-      latest_builds = builds.latest
-      return unless latest_builds.exists?(build.id)
-
-      # get list of stages after this build
-      next_stages = config_processor.stages.drop_while { |stage| stage != build.stage }
-      next_stages.delete(build.stage)
-
-      # get status for all prior builds
-      prior_builds = latest_builds.where.not(stage: next_stages)
-      prior_status = prior_builds.status
-
-      # build builds for next stage that has builds available
-      # and save pipeline if we have builds
-      build_builds_for_stages(next_stages, build.user, prior_status,
-                              build.trigger_request) && save
-    end
-
     def retried
       @retried ||= (statuses.order(id: :desc) - statuses.latest)
     end
@@ -151,6 +158,14 @@ module Ci
       end
     end
 
+    def config_builds_attributes
+      return [] unless config_processor
+
+      config_processor.
+        builds_for_ref(ref, tag?, trigger_requests.first).
+        sort_by { |build| build[:stage_idx] }
+    end
+
     def has_warnings?
       builds.latest.ignored.any?
     end
@@ -182,10 +197,6 @@ module Ci
       end
     end
 
-    def skip_ci?
-      git_commit_message =~ /\[(ci skip|skip ci)\]/i if git_commit_message
-    end
-
     def environments
       builds.where.not(environment: nil).success.pluck(:environment).uniq
     end
@@ -207,37 +218,37 @@ module Ci
       Note.for_commit_id(sha)
     end
 
+    def process!
+      Ci::ProcessPipelineService.new(project, user).execute(self)
+    end
+
+    def build_updated
+      case latest_builds_status
+      when 'pending' then enqueue
+      when 'running' then run
+      when 'success' then succeed
+      when 'failed' then drop
+      when 'canceled' then cancel
+      when 'skipped' then skip
+      end
+    end
+
     def predefined_variables
       [
         { key: 'CI_PIPELINE_ID', value: id.to_s, public: true }
       ]
     end
 
+    def update_duration
+      self.duration = statuses.latest.duration
+    end
+
     private
 
-    def build_builds_for_stages(stages, user, status, trigger_request)
-      ##
-      # Note that `Array#any?` implements a short circuit evaluation, so we
-      # build builds only for the first stage that has builds available.
-      #
-      stages.any? do |stage|
-        CreateBuildsService.new(self).
-          execute(stage, user, status, trigger_request).
-          any?(&:active?)
-      end
-    end
-
-    def update_state
-      statuses.reload
-      self.status = if yaml_errors.blank?
-                      statuses.latest.status || 'skipped'
-                    else
-                      'failed'
-                    end
-      self.started_at = statuses.started_at
-      self.finished_at = statuses.finished_at
-      self.duration = statuses.latest.duration
-      save
+    def latest_builds_status
+      return 'failed' unless yaml_errors.blank?
+
+      statuses.latest.status || 'skipped'
     end
 
     def keep_around_commits
diff --git a/app/models/commit.rb b/app/models/commit.rb
index 486ad6714d9..cc413448ce8 100644
--- a/app/models/commit.rb
+++ b/app/models/commit.rb
@@ -104,7 +104,7 @@ class Commit
   end
 
   def diff_line_count
-    @diff_line_count ||= Commit::diff_line_count(self.diffs)
+    @diff_line_count ||= Commit::diff_line_count(raw_diffs)
     @diff_line_count
   end
 
@@ -123,15 +123,17 @@ class Commit
   # In case this first line is longer than 100 characters, it is cut off
   # after 80 characters and ellipses (`&hellp;`) are appended.
   def title
-    title = safe_message
+    full_title.length > 100 ? full_title[0..79] << "…" : full_title
+  end
 
-    return no_commit_message if title.blank?
+  # Returns the full commits title
+  def full_title
+    return @full_title if @full_title
 
-    title_end = title.index("\n")
-    if (!title_end && title.length > 100) || (title_end && title_end > 100)
-      title[0..79] << "…"
+    if safe_message.blank?
+      @full_title = no_commit_message
     else
-      title.split("\n", 2).first
+      @full_title = safe_message.split("\n", 2).first
     end
   end
 
@@ -315,6 +317,14 @@ class Commit
     nil
   end
 
+  def raw_diffs(*args)
+    raw.diffs(*args)
+  end
+
+  def diffs(diff_options = nil)
+    Gitlab::Diff::FileCollection::Commit.new(self, diff_options: diff_options)
+  end
+
   private
 
   def find_author_by_any_email
@@ -324,7 +334,7 @@ class Commit
   def repo_changes
     changes = { added: [], modified: [], removed: [] }
 
-    diffs.each do |diff|
+    raw_diffs(deltas_only: true).each do |diff|
       if diff.deleted_file
         changes[:removed] << diff.old_path
       elsif diff.renamed_file || diff.new_file
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb
index 2d185c28809..703ca90edb6 100644
--- a/app/models/commit_status.rb
+++ b/app/models/commit_status.rb
@@ -5,7 +5,7 @@ class CommitStatus < ActiveRecord::Base
   self.table_name = 'ci_builds'
 
   belongs_to :project, class_name: '::Project', foreign_key: :gl_project_id
-  belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :commit_id, touch: true
+  belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :commit_id
   belongs_to :user
 
   delegate :commit, to: :pipeline
@@ -25,28 +25,36 @@ class CommitStatus < ActiveRecord::Base
   scope :ordered, -> { order(:name) }
   scope :ignored, -> { where(allow_failure: true, status: [:failed, :canceled]) }
 
-  state_machine :status, initial: :pending do
-    event :queue do
-      transition skipped: :pending
+  state_machine :status do
+    event :enqueue do
+      transition [:created, :skipped] => :pending
     end
 
     event :run do
       transition pending: :running
     end
 
+    event :skip do
+      transition [:created, :pending] => :skipped
+    end
+
     event :drop do
-      transition [:pending, :running] => :failed
+      transition [:created, :pending, :running] => :failed
     end
 
     event :success do
-      transition [:pending, :running] => :success
+      transition [:created, :pending, :running] => :success
     end
 
     event :cancel do
-      transition [:pending, :running] => :canceled
+      transition [:created, :pending, :running] => :canceled
     end
 
-    after_transition pending: :running do |commit_status|
+    after_transition created: [:pending, :running] do |commit_status|
+      commit_status.update_attributes queued_at: Time.now
+    end
+
+    after_transition [:created, :pending] => :running do |commit_status|
       commit_status.update_attributes started_at: Time.now
     end
 
@@ -54,7 +62,18 @@ class CommitStatus < ActiveRecord::Base
       commit_status.update_attributes finished_at: Time.now
     end
 
-    after_transition [:pending, :running] => :success do |commit_status|
+    # We use around_transition to process pipeline on next stages as soon as possible, before the `after_*` is executed
+    around_transition any => [:success, :failed, :canceled] do |commit_status, block|
+      block.call
+
+      commit_status.pipeline.try(:process!)
+    end
+
+    after_transition do |commit_status, transition|
+      commit_status.pipeline.try(:build_updated) unless transition.loopback?
+    end
+
+    after_transition [:created, :pending, :running] => :success do |commit_status|
       MergeRequests::MergeWhenBuildSucceedsService.new(commit_status.pipeline.project, nil).trigger(commit_status)
     end
 
diff --git a/app/models/compare.rb b/app/models/compare.rb
new file mode 100644
index 00000000000..4856510f526
--- /dev/null
+++ b/app/models/compare.rb
@@ -0,0 +1,66 @@
+class Compare
+  delegate :same, :head, :base, to: :@compare
+
+  attr_reader :project
+
+  def self.decorate(compare, project)
+    if compare.is_a?(Compare)
+      compare
+    else
+      self.new(compare, project)
+    end
+  end
+
+  def initialize(compare, project)
+    @compare = compare
+    @project = project
+  end
+
+  def commits
+    @commits ||= Commit.decorate(@compare.commits, project)
+  end
+
+  def start_commit
+    return @start_commit if defined?(@start_commit)
+
+    commit = @compare.base
+    @start_commit = commit ? ::Commit.new(commit, project) : nil
+  end
+
+  def head_commit
+    return @head_commit if defined?(@head_commit)
+
+    commit = @compare.head
+    @head_commit = commit ? ::Commit.new(commit, project) : nil
+  end
+  alias_method :commit, :head_commit
+
+  def base_commit
+    return @base_commit if defined?(@base_commit)
+
+    @base_commit = if start_commit && head_commit
+                     project.merge_base_commit(start_commit.id, head_commit.id)
+                   else
+                     nil
+                   end
+  end
+
+  def raw_diffs(*args)
+    @compare.diffs(*args)
+  end
+
+  def diffs(diff_options = nil)
+    Gitlab::Diff::FileCollection::Compare.new(self,
+      project: project,
+      diff_options: diff_options,
+      diff_refs: diff_refs)
+  end
+
+  def diff_refs
+    Gitlab::Diff::DiffRefs.new(
+      base_sha:  base_commit.try(:sha),
+      start_sha: start_commit.try(:sha),
+      head_sha: commit.try(:sha)
+    )
+  end
+end
diff --git a/app/models/concerns/faster_cache_keys.rb b/app/models/concerns/faster_cache_keys.rb
new file mode 100644
index 00000000000..5b14723fa2d
--- /dev/null
+++ b/app/models/concerns/faster_cache_keys.rb
@@ -0,0 +1,16 @@
+module FasterCacheKeys
+  # A faster version of Rails' "cache_key" method.
+  #
+  # Rails' default "cache_key" method uses all kind of complex logic to figure
+  # out the cache key. In many cases this complexity and overhead may not be
+  # needed.
+  #
+  # This method does not do any timestamp parsing as this process is quite
+  # expensive and not needed when generating cache keys. This method also relies
+  # on the table name instead of the cache namespace name as the latter uses
+  # complex logic to generate the exact same value (as when using the table
+  # name) in 99% of the cases.
+  def cache_key
+    "#{self.class.table_name}/#{id}-#{read_attribute_before_type_cast(:updated_at)}"
+  end
+end
diff --git a/app/models/concerns/statuseable.rb b/app/models/concerns/statuseable.rb
index 44c6b30f278..5d4b0a86899 100644
--- a/app/models/concerns/statuseable.rb
+++ b/app/models/concerns/statuseable.rb
@@ -1,18 +1,22 @@
 module Statuseable
   extend ActiveSupport::Concern
 
-  AVAILABLE_STATUSES = %w(pending running success failed canceled skipped)
+  AVAILABLE_STATUSES = %w[created pending running success failed canceled skipped]
+  STARTED_STATUSES = %w[running success failed skipped]
+  ACTIVE_STATUSES = %w[pending running]
+  COMPLETED_STATUSES = %w[success failed canceled]
 
   class_methods do
     def status_sql
-      builds = all.select('count(*)').to_sql
-      success = all.success.select('count(*)').to_sql
-      ignored = all.ignored.select('count(*)').to_sql if all.respond_to?(:ignored)
+      scope = all.relevant
+      builds = scope.select('count(*)').to_sql
+      success = scope.success.select('count(*)').to_sql
+      ignored = scope.ignored.select('count(*)').to_sql if scope.respond_to?(:ignored)
       ignored ||= '0'
-      pending = all.pending.select('count(*)').to_sql
-      running = all.running.select('count(*)').to_sql
-      canceled = all.canceled.select('count(*)').to_sql
-      skipped = all.skipped.select('count(*)').to_sql
+      pending = scope.pending.select('count(*)').to_sql
+      running = scope.running.select('count(*)').to_sql
+      canceled = scope.canceled.select('count(*)').to_sql
+      skipped = scope.skipped.select('count(*)').to_sql
 
       deduce_status = "(CASE
         WHEN (#{builds})=0 THEN NULL
@@ -48,7 +52,8 @@ module Statuseable
   included do
     validates :status, inclusion: { in: AVAILABLE_STATUSES }
 
-    state_machine :status, initial: :pending do
+    state_machine :status, initial: :created do
+      state :created, value: 'created'
       state :pending, value: 'pending'
       state :running, value: 'running'
       state :failed, value: 'failed'
@@ -57,6 +62,8 @@ module Statuseable
       state :skipped, value: 'skipped'
     end
 
+    scope :created, -> { where(status: 'created') }
+    scope :relevant, -> { where.not(status: 'created') }
     scope :running, -> { where(status: 'running') }
     scope :pending, -> { where(status: 'pending') }
     scope :success, -> { where(status: 'success') }
@@ -68,14 +75,14 @@ module Statuseable
   end
 
   def started?
-    !pending? && !canceled? && started_at
+    STARTED_STATUSES.include?(status) && started_at
   end
 
   def active?
-    running? || pending?
+    ACTIVE_STATUSES.include?(status)
   end
 
   def complete?
-    canceled? || success? || failed?
+    COMPLETED_STATUSES.include?(status)
   end
 end
diff --git a/app/models/diff_note.rb b/app/models/diff_note.rb
index 9671955db36..c816deb4e0c 100644
--- a/app/models/diff_note.rb
+++ b/app/models/diff_note.rb
@@ -67,7 +67,7 @@ class DiffNote < Note
     return false unless supported?
     return true if for_commit?
 
-    diff_refs ||= self.noteable.diff_refs
+    diff_refs ||= noteable_diff_refs
 
     self.position.diff_refs == diff_refs
   end
@@ -78,6 +78,14 @@ class DiffNote < Note
     !self.for_merge_request? || self.noteable.support_new_diff_notes?
   end
 
+  def noteable_diff_refs
+    if noteable.respond_to?(:diff_sha_refs)
+      noteable.diff_sha_refs
+    else
+      noteable.diff_refs
+    end
+  end
+
   def set_original_position
     self.original_position = self.position.dup
   end
@@ -96,7 +104,7 @@ class DiffNote < Note
       self.project,
       nil,
       old_diff_refs: self.position.diff_refs,
-      new_diff_refs: self.noteable.diff_refs,
+      new_diff_refs: noteable_diff_refs,
       paths: self.position.paths
     ).execute(self)
   end
diff --git a/app/models/discussion.rb b/app/models/discussion.rb
index 74facfd1c9c..e2218a5f02b 100644
--- a/app/models/discussion.rb
+++ b/app/models/discussion.rb
@@ -49,6 +49,12 @@ class Discussion
     self.noteable == target && !diff_discussion?
   end
 
+  def active?
+    return @active if defined?(@active)
+
+    @active = first_note.active?
+  end
+
   def expanded?
     !diff_discussion? || active?
   end
diff --git a/app/models/issue.rb b/app/models/issue.rb
index 11f734cfc6d..d62ffb21467 100644
--- a/app/models/issue.rb
+++ b/app/models/issue.rb
@@ -7,6 +7,7 @@ class Issue < ActiveRecord::Base
   include Sortable
   include Taskable
   include Spammable
+  include FasterCacheKeys
 
   DueDateStruct = Struct.new(:title, :name).freeze
   NoDueDate     = DueDateStruct.new('No Due Date', '0').freeze
diff --git a/app/models/key.rb b/app/models/key.rb
index b9bc38a0436..568a60b8af3 100644
--- a/app/models/key.rb
+++ b/app/models/key.rb
@@ -26,8 +26,9 @@ class Key < ActiveRecord::Base
   end
 
   def publishable_key
-    # Removes anything beyond the keytype and key itself
-    self.key.split[0..1].join(' ')
+    # Strip out the keys comment so we don't leak email addresses
+    # Replace with simple ident of user_name (hostname)
+    self.key.split[0..1].push("#{self.user_name} (#{Gitlab.config.gitlab.host})").join(' ')
   end
 
   # projects that has this key
diff --git a/app/models/legacy_diff_note.rb b/app/models/legacy_diff_note.rb
index 865712268a0..6ed66001513 100644
--- a/app/models/legacy_diff_note.rb
+++ b/app/models/legacy_diff_note.rb
@@ -85,7 +85,7 @@ class LegacyDiffNote < Note
     return nil unless noteable
     return @diff if defined?(@diff)
 
-    @diff = noteable.diffs(Commit.max_diff_options).find do |d|
+    @diff = noteable.raw_diffs(Commit.max_diff_options).find do |d|
       d.new_path && Digest::SHA1.hexdigest(d.new_path) == diff_file_hash
     end
   end
@@ -116,7 +116,7 @@ class LegacyDiffNote < Note
 
   # Find the diff on noteable that matches our own
   def find_noteable_diff
-    diffs = noteable.diffs(Commit.max_diff_options)
+    diffs = noteable.raw_diffs(Commit.max_diff_options)
     diffs.find { |d| d.new_path == self.diff.new_path }
   end
 end
diff --git a/app/models/members/project_member.rb b/app/models/members/project_member.rb
index f39afc61ce9..18e97c969d7 100644
--- a/app/models/members/project_member.rb
+++ b/app/models/members/project_member.rb
@@ -8,6 +8,7 @@ class ProjectMember < Member
   # Make sure project member points only to project as it source
   default_value_for :source_type, SOURCE_TYPE
   validates_format_of :source_type, with: /\AProject\z/
+  validates :access_level, inclusion: { in: Gitlab::Access.values }
   default_scope { where(source_type: SOURCE_TYPE) }
 
   scope :in_project, ->(project) { where(source_id: project.id) }
@@ -21,19 +22,19 @@ class ProjectMember < Member
     # or symbol like :master representing role
     #
     # Ex.
-    #   add_users_into_projects(
+    #   add_users_to_projects(
     #     project_ids,
     #     user_ids,
     #     ProjectMember::MASTER
     #   )
     #
-    #   add_users_into_projects(
+    #   add_users_to_projects(
     #     project_ids,
     #     user_ids,
     #     :master
     #   )
     #
-    def add_users_into_projects(project_ids, user_ids, access, current_user = nil)
+    def add_users_to_projects(project_ids, user_ids, access, current_user = nil)
       access_level = if roles_hash.has_key?(access)
                        roles_hash[access]
                      elsif roles_hash.values.include?(access.to_i)
diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb
index b7c2afb0920..7c3845ef538 100644
--- a/app/models/merge_request.rb
+++ b/app/models/merge_request.rb
@@ -106,6 +106,7 @@ class MergeRequest < ActiveRecord::Base
   scope :from_project, ->(project) { where(source_project_id: project.id) }
   scope :merged, -> { with_state(:merged) }
   scope :closed_and_merged, -> { with_states(:closed, :merged) }
+  scope :from_source_branches, ->(branches) { where(source_branch: branches) }
 
   scope :join_project, -> { joins(:target_project) }
   scope :references_project, -> { references(:target_project) }
@@ -166,8 +167,16 @@ class MergeRequest < ActiveRecord::Base
     merge_request_diff ? merge_request_diff.first_commit : compare_commits.first
   end
 
-  def diffs(*args)
-    merge_request_diff ? merge_request_diff.diffs(*args) : compare.diffs(*args)
+  def raw_diffs(*args)
+    merge_request_diff ? merge_request_diff.raw_diffs(*args) : compare.raw_diffs(*args)
+  end
+
+  def diffs(diff_options = nil)
+    if self.compare
+      self.compare.diffs(diff_options)
+    else
+      Gitlab::Diff::FileCollection::MergeRequest.new(self, diff_options: diff_options)
+    end
   end
 
   def diff_size
@@ -278,6 +287,19 @@ class MergeRequest < ActiveRecord::Base
     end
   end
 
+  # Return diff_refs instance trying to not touch the git repository
+  def diff_sha_refs
+    if merge_request_diff && merge_request_diff.diff_refs_by_sha?
+      return Gitlab::Diff::DiffRefs.new(
+        base_sha:  merge_request_diff.base_commit_sha,
+        start_sha: merge_request_diff.start_commit_sha,
+        head_sha:  merge_request_diff.head_commit_sha
+      )
+    else
+      diff_refs
+    end
+  end
+
   def validate_branches
     if target_project == source_project && target_branch == source_branch
       errors.add :branch_conflict, "You can not use same project/branch for source and target"
@@ -334,6 +356,7 @@ class MergeRequest < ActiveRecord::Base
 
     old_diff_refs = self.diff_refs
     create_merge_request_diff
+    MergeRequests::MergeRequestDiffCacheService.new.execute(self)
     new_diff_refs = self.diff_refs
 
     update_diff_notes_positions(
@@ -693,7 +716,7 @@ class MergeRequest < ActiveRecord::Base
   end
 
   def support_new_diff_notes?
-    diff_refs && diff_refs.complete?
+    diff_sha_refs && diff_sha_refs.complete?
   end
 
   def update_diff_notes_positions(old_diff_refs:, new_diff_refs:)
diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb
index 950b00f1001..8920641cfec 100644
--- a/app/models/merge_request_diff.rb
+++ b/app/models/merge_request_diff.rb
@@ -72,10 +72,10 @@ class MergeRequestDiff < ActiveRecord::Base
   end
 
   def size
-    real_size.presence || diffs.size
+    real_size.presence || raw_diffs.size
   end
 
-  def diffs(options={})
+  def raw_diffs(options = {})
     if options[:ignore_whitespace_change]
       @diffs_no_whitespace ||=
         begin
@@ -87,8 +87,8 @@ class MergeRequestDiff < ActiveRecord::Base
           compare.diffs(options)
         end
     else
-      @diffs ||= {}
-      @diffs[options] ||= load_diffs(st_diffs, options)
+      @raw_diffs ||= {}
+      @raw_diffs[options] ||= load_diffs(st_diffs, options)
     end
   end
 
@@ -137,6 +137,10 @@ class MergeRequestDiff < ActiveRecord::Base
     )
   end
 
+  def diff_refs_by_sha?
+    base_commit_sha? && head_commit_sha? && start_commit_sha?
+  end
+
   private
 
   def compare
diff --git a/app/models/namespace.rb b/app/models/namespace.rb
index 8b52cc824cd..7c29d27ce97 100644
--- a/app/models/namespace.rb
+++ b/app/models/namespace.rb
@@ -1,4 +1,6 @@
 class Namespace < ActiveRecord::Base
+  acts_as_paranoid
+
   include Sortable
   include Gitlab::ShellAdapter
 
diff --git a/app/models/note.rb b/app/models/note.rb
index b6b2ac6aa42..ddcd7f9d034 100644
--- a/app/models/note.rb
+++ b/app/models/note.rb
@@ -5,6 +5,7 @@ class Note < ActiveRecord::Base
   include Mentionable
   include Awardable
   include Importable
+  include FasterCacheKeys
 
   # Attribute containing rendered and redacted Markdown as generated by
   # Banzai::ObjectRenderer.
diff --git a/app/models/project.rb b/app/models/project.rb
index 83b848ded8b..e0b28160937 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -379,9 +379,10 @@ class Project < ActiveRecord::Base
       joins(join_body).reorder('join_note_counts.amount DESC')
     end
 
-    # Deletes gitlab project export files older than 24 hours
-    def remove_gitlab_exports!
-      Gitlab::Popen.popen(%W(find #{Gitlab::ImportExport.storage_path} -not -path #{Gitlab::ImportExport.storage_path} -mmin +1440 -delete))
+    def cached_count
+      Rails.cache.fetch('total_project_count', expires_in: 5.minutes) do
+        Project.count
+      end
     end
   end
 
@@ -870,10 +871,16 @@ class Project < ActiveRecord::Base
 
   # Check if current branch name is marked as protected in the system
   def protected_branch?(branch_name)
+    return true if empty_repo? && default_branch_protected?
+
     @protected_branches ||= self.protected_branches.to_a
     ProtectedBranch.matching(branch_name, protected_branches: @protected_branches).present?
   end
 
+  def user_can_push_to_empty_repo?(user)
+    !default_branch_protected? || team.max_member_access(user.id) > Gitlab::Access::DEVELOPER
+  end
+
   def forked?
     !(forked_project_link.nil? || forked_project_link.forked_from_project.nil?)
   end
@@ -992,6 +999,10 @@ class Project < ActiveRecord::Base
     project_members.find_by(user_id: user)
   end
 
+  def add_user(user, access_level, current_user = nil)
+    team.add_user(user, access_level, current_user)
+  end
+
   def default_branch
     @default_branch ||= repository.root_ref if repository.exists?
   end
@@ -1154,16 +1165,6 @@ class Project < ActiveRecord::Base
     @wiki ||= ProjectWiki.new(self, self.owner)
   end
 
-  def schedule_delete!(user_id, params)
-    # Queue this task for after the commit, so once we mark pending_delete it will run
-    run_after_commit do
-      job_id = ProjectDestroyWorker.perform_async(id, user_id, params)
-      Rails.logger.info("User #{user_id} scheduled destruction of project #{path_with_namespace} with job ID #{job_id}")
-    end
-
-    update_attribute(:pending_delete, true)
-  end
-
   def running_or_pending_build_count(force: false)
     Rails.cache.fetch(['projects', id, 'running_or_pending_build_count'], force: force) do
       builds.running_or_pending.count(:all)
@@ -1265,6 +1266,11 @@ class Project < ActiveRecord::Base
 
   private
 
+  def default_branch_protected?
+    current_application_settings.default_branch_protection == Gitlab::Access::PROTECTION_FULL ||
+      current_application_settings.default_branch_protection == Gitlab::Access::PROTECTION_DEV_CAN_MERGE
+  end
+
   def authorized_for_user_by_group?(user, min_access_level)
     member = user.group_members.find_by(source_id: group)
 
diff --git a/app/models/project_services/campfire_service.rb b/app/models/project_services/campfire_service.rb
index 511b2eac792..5af93860d09 100644
--- a/app/models/project_services/campfire_service.rb
+++ b/app/models/project_services/campfire_service.rb
@@ -1,4 +1,6 @@
 class CampfireService < Service
+  include HTTParty
+
   prop_accessor :token, :subdomain, :room
   validates :token, presence: true, if: :activated?
 
@@ -29,18 +31,53 @@ class CampfireService < Service
   def execute(data)
     return unless supported_events.include?(data[:object_kind])
 
-    room = gate.find_room_by_name(self.room)
-    return true unless room
-
+    self.class.base_uri base_uri
     message = build_message(data)
-
-    room.speak(message)
+    speak(self.room, message, auth)
   end
 
   private
 
-  def gate
-    @gate ||= Tinder::Campfire.new(subdomain, token: token)
+  def base_uri
+    @base_uri ||= "https://#{subdomain}.campfirenow.com"
+  end
+
+  def auth
+    # use a dummy password, as explained in the Campfire API doc:
+    # https://github.com/basecamp/campfire-api#authentication
+    @auth ||= {
+      basic_auth: {
+        username: token,
+        password: 'X'
+      }
+    }
+  end
+
+  # Post a message into a room, returns the message Hash in case of success.
+  # Returns nil otherwise.
+  # https://github.com/basecamp/campfire-api/blob/master/sections/messages.md#create-message
+  def speak(room_name, message, auth)
+    room = rooms(auth).find { |r| r["name"] == room_name }
+    return nil unless room
+
+    path = "/room/#{room["id"]}/speak.json"
+    body = {
+      body: {
+        message: {
+          type: 'TextMessage',
+          body: message
+        }
+      }
+    }
+    res = self.class.post(path, auth.merge(body))
+    res.code == 201 ? res : nil
+  end
+
+  # Returns a list of rooms, or [].
+  # https://github.com/basecamp/campfire-api/blob/master/sections/rooms.md#get-rooms
+  def rooms(auth)
+    res = self.class.get("/rooms.json", auth) 
+    res.code == 200 ? res["rooms"] : []
   end
 
   def build_message(push)
diff --git a/app/models/project_services/pivotaltracker_service.rb b/app/models/project_services/pivotaltracker_service.rb
index ad19b7795da..5301f9fa0ff 100644
--- a/app/models/project_services/pivotaltracker_service.rb
+++ b/app/models/project_services/pivotaltracker_service.rb
@@ -1,7 +1,9 @@
 class PivotaltrackerService < Service
   include HTTParty
 
-  prop_accessor :token
+  API_ENDPOINT = 'https://www.pivotaltracker.com/services/v5/source_commits'
+
+  prop_accessor :token, :restrict_to_branch
   validates :token, presence: true, if: :activated?
 
   def title
@@ -18,7 +20,17 @@ class PivotaltrackerService < Service
 
   def fields
     [
-      { type: 'text', name: 'token', placeholder: '' }
+      {
+        type: 'text',
+        name: 'token',
+        placeholder: 'Pivotal Tracker API token.'
+      },
+      {
+        type: 'text',
+        name: 'restrict_to_branch',
+        placeholder: 'Comma-separated list of branches which will be ' \
+          'automatically inspected. Leave blank to include all branches.'
+      }
     ]
   end
 
@@ -28,8 +40,8 @@ class PivotaltrackerService < Service
 
   def execute(data)
     return unless supported_events.include?(data[:object_kind])
+    return unless allowed_branch?(data[:ref])
 
-    url = 'https://www.pivotaltracker.com/services/v5/source_commits'
     data[:commits].each do |commit|
       message = {
         'source_commit' => {
@@ -40,7 +52,7 @@ class PivotaltrackerService < Service
         }
       }
       PivotaltrackerService.post(
-        url,
+        API_ENDPOINT,
         body: message.to_json,
         headers: {
           'Content-Type' => 'application/json',
@@ -49,4 +61,15 @@ class PivotaltrackerService < Service
       )
     end
   end
+
+  private
+
+  def allowed_branch?(ref)
+    return true unless ref.present? && restrict_to_branch.present?
+
+    branch = Gitlab::Git.ref_name(ref)
+    allowed_branches = restrict_to_branch.split(',').map(&:strip)
+
+    branch.present? && allowed_branches.include?(branch)
+  end
 end
diff --git a/app/models/project_team.rb b/app/models/project_team.rb
index 19fd082534c..d0a714cd6fc 100644
--- a/app/models/project_team.rb
+++ b/app/models/project_team.rb
@@ -34,7 +34,7 @@ class ProjectTeam
   end
 
   def add_users(users, access, current_user = nil)
-    ProjectMember.add_users_into_projects(
+    ProjectMember.add_users_to_projects(
       [project.id],
       users,
       access,
diff --git a/app/models/repository.rb b/app/models/repository.rb
index bac37483c47..e56bac509a4 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -372,7 +372,7 @@ class Repository
     # We don't want to flush the cache if the commit didn't actually make any
     # changes to any of the possible avatar files.
     if revision && commit = self.commit(revision)
-      return unless commit.diffs.
+      return unless commit.raw_diffs(deltas_only: true).
         any? { |diff| AVATAR_FILES.include?(diff.new_path) }
     end
 
@@ -601,7 +601,7 @@ class Repository
     commit(sha)
   end
 
-  def next_branch(name, opts={})
+  def next_branch(name, opts = {})
     branch_ids = self.branch_names.map do |n|
       next 1 if n == name
       result = n.match(/\A#{name}-([0-9]+)\z/)
@@ -636,9 +636,7 @@ class Repository
   def tags_sorted_by(value)
     case value
     when 'name'
-      # Would be better to use `sort_by` but `version_sorter` only exposes
-      # `sort` and `rsort`
-      VersionSorter.rsort(tag_names).map { |tag_name| find_tag(tag_name) }
+      VersionSorter.rsort(tags) { |tag| tag.name }
     when 'updated_desc'
       tags_sorted_by_committed_date.reverse
     when 'updated_asc'
diff --git a/app/models/spam_report.rb b/app/models/spam_report.rb
deleted file mode 100644
index cdc7321b08e..00000000000
--- a/app/models/spam_report.rb
+++ /dev/null
@@ -1,5 +0,0 @@
-class SpamReport < ActiveRecord::Base
-  belongs_to :user
-
-  validates :user, presence: true
-end
diff --git a/app/models/user.rb b/app/models/user.rb
index db747434959..87a2d999843 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -23,13 +23,13 @@ class User < ActiveRecord::Base
   default_value_for :theme_id, gitlab_config.default_theme
 
   attr_encrypted :otp_secret,
-    key:       Gitlab::Application.config.secret_key_base,
+    key:       Gitlab::Application.secrets.otp_key_base,
     mode:      :per_attribute_iv_and_salt,
     insecure_mode: true,
     algorithm: 'aes-256-cbc'
 
   devise :two_factor_authenticatable,
-         otp_secret_encryption_key: Gitlab::Application.config.secret_key_base
+         otp_secret_encryption_key: Gitlab::Application.secrets.otp_key_base
 
   devise :two_factor_backupable, otp_number_of_backup_codes: 10
   serialize :otp_backup_codes, JSON
@@ -809,13 +809,13 @@ class User < ActiveRecord::Base
 
   def todos_done_count(force: false)
     Rails.cache.fetch(['users', id, 'todos_done_count'], force: force) do
-      todos.done.count
+      TodosFinder.new(self, state: :done).execute.count
     end
   end
 
   def todos_pending_count(force: false)
     Rails.cache.fetch(['users', id, 'todos_pending_count'], force: force) do
-      todos.pending.count
+      TodosFinder.new(self, state: :pending).execute.count
     end
   end