Merge branch 'master' into 'email-in-slash-commands'email-in-slash-commands

# Conflicts:
#   doc/user/project/slash_commands.md
#   doc/workflow/README.md

30 files changed, 490 insertions, 790 deletions

diff --git a/app/models/ability.rb b/app/models/ability.rb
index 07f703f205d..fa8f8bc3a5f 100644
--- a/app/models/ability.rb
+++ b/app/models/ability.rb
@@ -1,34 +1,5 @@
 class Ability
   class << self
-    # rubocop: disable Metrics/CyclomaticComplexity
-    def allowed(user, subject)
-      return anonymous_abilities(user, subject) if user.nil?
-      return [] unless user.is_a?(User)
-      return [] if user.blocked?
-
-      abilities_by_subject_class(user: user, subject: subject)
-    end
-
-    def abilities_by_subject_class(user:, subject:)
-      case subject
-      when CommitStatus then commit_status_abilities(user, subject)
-      when Project then project_abilities(user, subject)
-      when Issue then issue_abilities(user, subject)
-      when Note then note_abilities(user, subject)
-      when ProjectSnippet then project_snippet_abilities(user, subject)
-      when PersonalSnippet then personal_snippet_abilities(user, subject)
-      when MergeRequest then merge_request_abilities(user, subject)
-      when Group then group_abilities(user, subject)
-      when Namespace then namespace_abilities(user, subject)
-      when GroupMember then group_member_abilities(user, subject)
-      when ProjectMember then project_member_abilities(user, subject)
-      when User then user_abilities
-      when ExternalIssue, Deployment, Environment then project_abilities(user, subject.project)
-      when Ci::Runner then runner_abilities(user, subject)
-      else []
-      end.concat(global_abilities(user))
-    end
-
     # Given a list of users and a project this method returns the users that can
     # read the given project.
     def users_that_can_read_project(users, project)
@@ -61,353 +32,7 @@ class Ability
       issues.select { |issue| issue.visible_to_user?(user) }
     end
 
-    # List of possible abilities for anonymous user
-    def anonymous_abilities(user, subject)
-      if subject.is_a?(PersonalSnippet)
-        anonymous_personal_snippet_abilities(subject)
-      elsif subject.is_a?(ProjectSnippet)
-        anonymous_project_snippet_abilities(subject)
-      elsif subject.is_a?(CommitStatus)
-        anonymous_commit_status_abilities(subject)
-      elsif subject.is_a?(Project) || subject.respond_to?(:project)
-        anonymous_project_abilities(subject)
-      elsif subject.is_a?(Group) || subject.respond_to?(:group)
-        anonymous_group_abilities(subject)
-      elsif subject.is_a?(User)
-        anonymous_user_abilities
-      else
-        []
-      end
-    end
-
-    def anonymous_project_abilities(subject)
-      project = if subject.is_a?(Project)
-                  subject
-                else
-                  subject.project
-                end
-
-      if project && project.public?
-        rules = [
-          :read_project,
-          :read_board,
-          :read_list,
-          :read_wiki,
-          :read_label,
-          :read_milestone,
-          :read_project_snippet,
-          :read_project_member,
-          :read_merge_request,
-          :read_note,
-          :read_pipeline,
-          :read_commit_status,
-          :read_container_image,
-          :download_code
-        ]
-
-        # Allow to read builds by anonymous user if guests are allowed
-        rules << :read_build if project.public_builds?
-
-        # Allow to read issues by anonymous user if issue is not confidential
-        rules << :read_issue unless subject.is_a?(Issue) && subject.confidential?
-
-        rules - project_disabled_features_rules(project)
-      else
-        []
-      end
-    end
-
-    def anonymous_commit_status_abilities(subject)
-      rules = anonymous_project_abilities(subject.project)
-      # If subject is Ci::Build which inherits from CommitStatus filter the abilities
-      rules = filter_build_abilities(rules) if subject.is_a?(Ci::Build)
-      rules
-    end
-
-    def anonymous_group_abilities(subject)
-      rules = []
-
-      group = if subject.is_a?(Group)
-                subject
-              else
-                subject.group
-              end
-
-      rules << :read_group if group.public?
-
-      rules
-    end
-
-    def anonymous_personal_snippet_abilities(snippet)
-      if snippet.public?
-        [:read_personal_snippet]
-      else
-        []
-      end
-    end
-
-    def anonymous_project_snippet_abilities(snippet)
-      if snippet.public?
-        [:read_project_snippet]
-      else
-        []
-      end
-    end
-
-    def anonymous_user_abilities
-      [:read_user] unless restricted_public_level?
-    end
-
-    def global_abilities(user)
-      rules = []
-      rules << :create_group if user.can_create_group
-      rules << :read_users_list
-      rules
-    end
-
-    def project_abilities(user, project)
-      rules = []
-      key = "/user/#{user.id}/project/#{project.id}"
-
-      RequestStore.store[key] ||= begin
-        # Push abilities on the users team role
-        rules.push(*project_team_rules(project.team, user))
-
-        owner = user.admin? ||
-                project.owner == user ||
-                (project.group && project.group.has_owner?(user))
-
-        if owner
-          rules.push(*project_owner_rules)
-        end
-
-        if project.public? || (project.internal? && !user.external?)
-          rules.push(*public_project_rules)
-
-          # Allow to read builds for internal projects
-          rules << :read_build if project.public_builds?
-
-          unless owner || project.team.member?(user) || project_group_member?(project, user)
-            rules << :request_access if project.request_access_enabled
-          end
-        end
-
-        if project.archived?
-          rules -= project_archived_rules
-        end
-
-        rules - project_disabled_features_rules(project)
-      end
-    end
-
-    def project_team_rules(team, user)
-      # Rules based on role in project
-      if team.master?(user)
-        project_master_rules
-      elsif team.developer?(user)
-        project_dev_rules
-      elsif team.reporter?(user)
-        project_report_rules
-      elsif team.guest?(user)
-        project_guest_rules
-      else
-        []
-      end
-    end
-
-    def public_project_rules
-      @public_project_rules ||= project_guest_rules + [
-        :download_code,
-        :fork_project,
-        :read_commit_status,
-        :read_pipeline,
-        :read_container_image
-      ]
-    end
-
-    def project_guest_rules
-      @project_guest_rules ||= [
-        :read_project,
-        :read_wiki,
-        :read_issue,
-        :read_board,
-        :read_list,
-        :read_label,
-        :read_milestone,
-        :read_project_snippet,
-        :read_project_member,
-        :read_merge_request,
-        :read_note,
-        :create_project,
-        :create_issue,
-        :create_note,
-        :upload_file
-      ]
-    end
-
-    def project_report_rules
-      @project_report_rules ||= project_guest_rules + [
-        :download_code,
-        :fork_project,
-        :create_project_snippet,
-        :update_issue,
-        :admin_issue,
-        :admin_label,
-        :admin_list,
-        :read_commit_status,
-        :read_build,
-        :read_container_image,
-        :read_pipeline,
-        :read_environment,
-        :read_deployment
-      ]
-    end
-
-    def project_dev_rules
-      @project_dev_rules ||= project_report_rules + [
-        :admin_merge_request,
-        :update_merge_request,
-        :create_commit_status,
-        :update_commit_status,
-        :create_build,
-        :update_build,
-        :create_pipeline,
-        :update_pipeline,
-        :create_merge_request,
-        :create_wiki,
-        :push_code,
-        :resolve_note,
-        :create_container_image,
-        :update_container_image,
-        :create_environment,
-        :create_deployment
-      ]
-    end
-
-    def project_archived_rules
-      @project_archived_rules ||= [
-        :create_merge_request,
-        :push_code,
-        :push_code_to_protected_branches,
-        :update_merge_request,
-        :admin_merge_request
-      ]
-    end
-
-    def project_master_rules
-      @project_master_rules ||= project_dev_rules + [
-        :push_code_to_protected_branches,
-        :update_project_snippet,
-        :update_environment,
-        :update_deployment,
-        :admin_milestone,
-        :admin_project_snippet,
-        :admin_project_member,
-        :admin_merge_request,
-        :admin_note,
-        :admin_wiki,
-        :admin_project,
-        :admin_commit_status,
-        :admin_build,
-        :admin_container_image,
-        :admin_pipeline,
-        :admin_environment,
-        :admin_deployment
-      ]
-    end
-
-    def project_owner_rules
-      @project_owner_rules ||= project_master_rules + [
-        :change_namespace,
-        :change_visibility_level,
-        :rename_project,
-        :remove_project,
-        :archive_project,
-        :remove_fork_project,
-        :destroy_merge_request,
-        :destroy_issue
-      ]
-    end
-
-    def project_disabled_features_rules(project)
-      rules = []
-
-      unless project.issues_enabled
-        rules += named_abilities('issue')
-      end
-
-      unless project.merge_requests_enabled
-        rules += named_abilities('merge_request')
-      end
-
-      unless project.issues_enabled or project.merge_requests_enabled
-        rules += named_abilities('label')
-        rules += named_abilities('milestone')
-      end
-
-      unless project.snippets_enabled
-        rules += named_abilities('project_snippet')
-      end
-
-      unless project.wiki_enabled
-        rules += named_abilities('wiki')
-      end
-
-      unless project.builds_enabled
-        rules += named_abilities('build')
-        rules += named_abilities('pipeline')
-        rules += named_abilities('environment')
-        rules += named_abilities('deployment')
-      end
-
-      unless project.container_registry_enabled
-        rules += named_abilities('container_image')
-      end
-
-      rules
-    end
-
-    def group_abilities(user, group)
-      rules = []
-      rules << :read_group if can_read_group?(user, group)
-
-      owner = user.admin? || group.has_owner?(user)
-      master = owner || group.has_master?(user)
-
-      # Only group masters and group owners can create new projects
-      if master
-        rules += [
-          :create_projects,
-          :admin_milestones
-        ]
-      end
-
-      # Only group owner and administrators can admin group
-      if owner
-        rules += [
-          :admin_group,
-          :admin_namespace,
-          :admin_group_member,
-          :change_visibility_level
-        ]
-      end
-
-      if group.public? || (group.internal? && !user.external?)
-        rules << :request_access if group.request_access_enabled && group.users.exclude?(user)
-      end
-
-      rules.flatten
-    end
-
-    def can_read_group?(user, group)
-      return true if user.admin?
-      return true if group.public?
-      return true if group.internal? && !user.external?
-      return true if group.users.include?(user)
-
-      GroupProjectsFinder.new(group).execute(user).any?
-    end
-
+    # TODO: make this private and use the actual abilities stuff for this
     def can_edit_note?(user, note)
       return false if !note.editable? || !user.present?
       return true if note.author == user || user.admin?
@@ -420,207 +45,23 @@ class Ability
       end
     end
 
-    def namespace_abilities(user, namespace)
-      rules = []
-
-      # Only namespace owner and administrators can admin it
-      if namespace.owner == user || user.admin?
-        rules += [
-          :create_projects,
-          :admin_namespace
-        ]
-      end
-
-      rules.flatten
-    end
-
-    [:issue, :merge_request].each do |name|
-      define_method "#{name}_abilities" do |user, subject|
-        rules = []
-
-        if subject.author == user || (subject.respond_to?(:assignee) && subject.assignee == user)
-          rules += [
-            :"read_#{name}",
-            :"update_#{name}",
-          ]
-        end
-
-        rules += project_abilities(user, subject.project)
-        rules = filter_confidential_issues_abilities(user, subject, rules) if subject.is_a?(Issue)
-        rules
-      end
-    end
-
-    def note_abilities(user, note)
-      rules = []
-
-      if note.author == user
-        rules += [
-          :read_note,
-          :update_note,
-          :admin_note,
-          :resolve_note
-        ]
-      end
-
-      if note.respond_to?(:project) && note.project
-        rules += project_abilities(user, note.project)
-      end
-
-      if note.for_merge_request? && note.noteable.author == user
-        rules << :resolve_note
-      end
-
-      rules
-    end
-
-    def personal_snippet_abilities(user, snippet)
-      rules = []
-
-      if snippet.author == user
-        rules += [
-          :read_personal_snippet,
-          :update_personal_snippet,
-          :admin_personal_snippet
-        ]
-      end
-
-      if snippet.public? || (snippet.internal? && !user.external?)
-        rules << :read_personal_snippet
-      end
-
-      rules
+    def allowed?(user, action, subject)
+      allowed(user, subject).include?(action)
     end
 
-    def project_snippet_abilities(user, snippet)
-      rules = []
-
-      if snippet.author == user || user.admin?
-        rules += [
-          :read_project_snippet,
-          :update_project_snippet,
-          :admin_project_snippet
-        ]
-      end
-
-      if snippet.public? || (snippet.internal? && !user.external?) || (snippet.private? && snippet.project.team.member?(user))
-        rules << :read_project_snippet
-      end
-
-      rules
-    end
-
-    def group_member_abilities(user, subject)
-      rules = []
-      target_user = subject.user
-      group = subject.group
-
-      unless group.last_owner?(target_user)
-        can_manage = group_abilities(user, group).include?(:admin_group_member)
-
-        if can_manage
-          rules << :update_group_member
-          rules << :destroy_group_member
-        elsif user == target_user
-          rules << :destroy_group_member
-        end
-      end
-
-      rules
-    end
-
-    def project_member_abilities(user, subject)
-      rules = []
-      target_user = subject.user
-      project = subject.project
-
-      unless target_user == project.owner
-        can_manage = project_abilities(user, project).include?(:admin_project_member)
-
-        if can_manage
-          rules << :update_project_member
-          rules << :destroy_project_member
-        elsif user == target_user
-          rules << :destroy_project_member
-        end
-      end
-
-      rules
-    end
-
-    def commit_status_abilities(user, subject)
-      rules = project_abilities(user, subject.project)
-      # If subject is Ci::Build which inherits from CommitStatus filter the abilities
-      rules = filter_build_abilities(rules) if subject.is_a?(Ci::Build)
-      rules
-    end
-
-    def filter_build_abilities(rules)
-      # If we can't read build we should also not have that
-      # ability when looking at this in context of commit_status
-      %w(read create update admin).each do |rule|
-        rules.delete(:"#{rule}_commit_status") unless rules.include?(:"#{rule}_build")
-      end
-      rules
-    end
-
-    def runner_abilities(user, runner)
-      if user.is_admin?
-        [:assign_runner]
-      elsif runner.is_shared? || runner.locked?
-        []
-      elsif user.ci_authorized_runners.include?(runner)
-        [:assign_runner]
-      else
-        []
-      end
-    end
-
-    def user_abilities
-      [:read_user]
-    end
+    def allowed(user, subject)
+      return uncached_allowed(user, subject) unless RequestStore.active?
 
-    def abilities
-      @abilities ||= begin
-        abilities = Six.new
-        abilities << self
-        abilities
-      end
+      user_key = user ? user.id : 'anonymous'
+      subject_key = subject ? "#{subject.class.name}/#{subject.id}" : 'global'
+      key = "/ability/#{user_key}/#{subject_key}"
+      RequestStore[key] ||= uncached_allowed(user, subject).freeze
     end
 
     private
 
-    def restricted_public_level?
-      current_application_settings.restricted_visibility_levels.include?(Gitlab::VisibilityLevel::PUBLIC)
-    end
-
-    def named_abilities(name)
-      [
-        :"read_#{name}",
-        :"create_#{name}",
-        :"update_#{name}",
-        :"admin_#{name}"
-      ]
-    end
-
-    def filter_confidential_issues_abilities(user, issue, rules)
-      return rules if user.admin? || !issue.confidential?
-
-      unless issue.author == user || issue.assignee == user || issue.project.team.member?(user, Gitlab::Access::REPORTER)
-        rules.delete(:admin_issue)
-        rules.delete(:read_issue)
-        rules.delete(:update_issue)
-      end
-
-      rules
-    end
-
-    def project_group_member?(project, user)
-      project.group &&
-      (
-        project.group.members.exists?(user_id: user.id) ||
-        project.group.requesters.exists?(user_id: user.id)
-      )
+    def uncached_allowed(user, subject)
+      BasePolicy.class_for(subject).abilities(user, subject)
     end
   end
 end
diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb
index 8c19d9dc9c8..246477ffe88 100644
--- a/app/models/application_setting.rb
+++ b/app/models/application_setting.rb
@@ -55,6 +55,10 @@ class ApplicationSetting < ActiveRecord::Base
             presence: true,
             if: :akismet_enabled
 
+  validates :koding_url,
+            presence: true,
+            if: :koding_enabled
+
   validates :max_attachment_size,
             presence: true,
             numericality: { only_integer: true, greater_than: 0 }
@@ -142,13 +146,15 @@ class ApplicationSetting < ActiveRecord::Base
       default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'],
       default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'],
       domain_whitelist: Settings.gitlab['domain_whitelist'],
-      import_sources: %w[github bitbucket gitlab gitorious google_code fogbugz git gitlab_project],
+      import_sources: %w[github bitbucket gitlab google_code fogbugz git gitlab_project],
       shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
       max_artifacts_size: Settings.artifacts['max_size'],
       require_two_factor_authentication: false,
       two_factor_grace_period: 48,
       recaptcha_enabled: false,
       akismet_enabled: false,
+      koding_enabled: false,
+      koding_url: nil,
       repository_checks_enabled: true,
       disabled_oauth_sign_in_sources: [],
       send_user_confirmation_email: false,
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 096b3b801af..61052437318 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -208,22 +208,31 @@ module Ci
       end
     end
 
+    def has_trace_file?
+      File.exist?(path_to_trace) || has_old_trace_file?
+    end
+
     def has_trace?
       raw_trace.present?
     end
 
     def raw_trace
-      if File.file?(path_to_trace)
-        File.read(path_to_trace)
-      elsif project.ci_id && File.file?(old_path_to_trace)
-        # Temporary fix for build trace data integrity
-        File.read(old_path_to_trace)
+      if File.exist?(trace_file_path)
+        File.read(trace_file_path)
       else
         # backward compatibility
         read_attribute :trace
       end
     end
 
+    ##
+    # Deprecated
+    #
+    # This is a hotfix for CI build data integrity, see #4246
+    def has_old_trace_file?
+      project.ci_id && File.exist?(old_path_to_trace)
+    end
+
     def trace
       trace = raw_trace
       if project && trace.present? && project.runners_token.present?
@@ -262,6 +271,14 @@ module Ci
       end
     end
 
+    def trace_file_path
+      if has_old_trace_file?
+        old_path_to_trace
+      else
+        path_to_trace
+      end
+    end
+
     def dir_to_trace
       File.join(
         Settings.gitlab_ci.builds_path,
@@ -352,7 +369,7 @@ module Ci
     end
 
     def artifacts?
-      !artifacts_expired? && artifacts_file.exists?
+      !artifacts_expired? && self[:artifacts_file].present?
     end
 
     def artifacts_metadata?
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index 087abe4cbb1..bd1737c7587 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -1,7 +1,7 @@
 module Ci
   class Pipeline < ActiveRecord::Base
     extend Ci::Model
-    include Statuseable
+    include HasStatus
 
     self.table_name = 'ci_commits'
 
@@ -65,8 +65,8 @@ module Ci
     end
 
     # ref can't be HEAD or SHA, can only be branch/tag name
-    scope :latest_successful_for, ->(ref = default_branch) do
-      where(ref: ref).success.order(id: :desc).limit(1)
+    def self.latest_successful_for(ref)
+      where(ref: ref).order(id: :desc).success.first
     end
 
     def self.truncate_sha(sha)
@@ -83,7 +83,7 @@ module Ci
     end
 
     def stages_with_latest_statuses
-      statuses.latest.order(:stage_idx).group_by(&:stage)
+      statuses.latest.includes(project: :namespace).order(:stage_idx).group_by(&:stage)
     end
 
     def project_id
diff --git a/app/models/commit.rb b/app/models/commit.rb
index 817d063e4a2..e64fd1e0c1b 100644
--- a/app/models/commit.rb
+++ b/app/models/commit.rb
@@ -108,15 +108,6 @@ class Commit
     @diff_line_count
   end
 
-  # Returns a string describing the commit for use in a link title
-  #
-  # Example
-  #
-  #   "Commit: Alex Denisov - Project git clone panel"
-  def link_title
-    "Commit: #{author_name} - #{title}"
-  end
-
   # Returns the commits title.
   #
   # Usually, the commit title is the first line of the commit message.
diff --git a/app/models/commit_range.rb b/app/models/commit_range.rb
index 630ee9601e0..656a242c265 100644
--- a/app/models/commit_range.rb
+++ b/app/models/commit_range.rb
@@ -4,12 +4,10 @@
 #
 #   range = CommitRange.new('f3f85602...e86e1013', project)
 #   range.exclude_start?  # => false
-#   range.reference_title # => "Commits f3f85602 through e86e1013"
 #   range.to_s            # => "f3f85602...e86e1013"
 #
 #   range = CommitRange.new('f3f856029bc5f966c5a7ee24cf7efefdd20e6019..e86e1013709735be5bb767e2b228930c543f25ae', project)
 #   range.exclude_start?  # => true
-#   range.reference_title # => "Commits f3f85602^ through e86e1013"
 #   range.to_param        # => {from: "f3f856029bc5f966c5a7ee24cf7efefdd20e6019^", to: "e86e1013709735be5bb767e2b228930c543f25ae"}
 #   range.to_s            # => "f3f85602..e86e1013"
 #
@@ -109,11 +107,6 @@ class CommitRange
     reference
   end
 
-  # Returns a String for use in a link's title attribute
-  def reference_title
-    "Commits #{sha_start} through #{sha_to}"
-  end
-
   # Return a Hash of parameters for passing to a URL helper
   #
   # See `namespace_project_compare_url`
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb
index 84ceeac7d3e..4a628924499 100644
--- a/app/models/commit_status.rb
+++ b/app/models/commit_status.rb
@@ -1,5 +1,5 @@
 class CommitStatus < ActiveRecord::Base
-  include Statuseable
+  include HasStatus
   include Importable
 
   self.table_name = 'ci_builds'
@@ -25,6 +25,8 @@ class CommitStatus < ActiveRecord::Base
   scope :retried, -> { where.not(id: latest) }
   scope :ordered, -> { order(:name) }
   scope :ignored, -> { where(allow_failure: true, status: [:failed, :canceled]) }
+  scope :latest_ci_stages, -> { latest.ordered.includes(project: :namespace) }
+  scope :retried_ci_stages, -> { retried.ordered.includes(project: :namespace) }
 
   state_machine :status do
     event :enqueue do
diff --git a/app/models/concerns/awardable.rb b/app/models/concerns/awardable.rb
index 800a16ab246..d8d4575bb4d 100644
--- a/app/models/concerns/awardable.rb
+++ b/app/models/concerns/awardable.rb
@@ -2,7 +2,7 @@ module Awardable
   extend ActiveSupport::Concern
 
   included do
-    has_many :award_emoji, -> { includes(:user) }, as: :awardable, dependent: :destroy
+    has_many :award_emoji, -> { includes(:user).order(:id) }, as: :awardable, dependent: :destroy
 
     if self < Participable
       # By default we always load award_emoji user association
@@ -59,6 +59,18 @@ module Awardable
     true
   end
 
+  def awardable_votes?(name)
+    AwardEmoji::UPVOTE_NAME == name || AwardEmoji::DOWNVOTE_NAME == name
+  end
+
+  def user_can_award?(current_user, name)
+    if user_authored?(current_user)
+      !awardable_votes?(normalize_name(name))
+    else
+      true
+    end
+  end
+
   def awarded_emoji?(emoji_name, current_user)
     award_emoji.where(name: emoji_name, user: current_user).exists?
   end
diff --git a/app/models/concerns/statuseable.rb b/app/models/concerns/has_status.rb
index 750f937b724..f7b8352405c 100644
--- a/app/models/concerns/statuseable.rb
+++ b/app/models/concerns/has_status.rb
@@ -1,4 +1,4 @@
-module Statuseable
+module HasStatus
   extend ActiveSupport::Concern
 
   AVAILABLE_STATUSES = %w[created pending running success failed canceled skipped]
diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb
index cbae1cd439b..22231b2e0f0 100644
--- a/app/models/concerns/issuable.rb
+++ b/app/models/concerns/issuable.rb
@@ -87,6 +87,12 @@ module Issuable
       User.find(assignee_id_was).update_cache_counts if assignee_id_was
       assignee.update_cache_counts if assignee
     end
+
+    # We want to use optimistic lock for cases when only title or description are involved
+    # http://api.rubyonrails.org/classes/ActiveRecord/Locking/Optimistic.html
+    def locking_enabled?
+      title_changed? || description_changed?
+    end
   end
 
   module ClassMethods
@@ -131,7 +137,10 @@ module Issuable
     end
 
     def order_labels_priority(excluded_labels: [])
-      select("#{table_name}.*, (#{highest_label_priority(excluded_labels).to_sql}) AS highest_priority").
+      condition_field = "#{table_name}.id"
+      highest_priority = highest_label_priority(name, condition_field, excluded_labels: excluded_labels).to_sql
+
+      select("#{table_name}.*, (#{highest_priority}) AS highest_priority").
         group(arel_table[:id]).
         reorder(Gitlab::Database.nulls_last_order('highest_priority', 'ASC'))
     end
@@ -159,20 +168,6 @@ module Issuable
 
       grouping_columns
     end
-
-    private
-
-    def highest_label_priority(excluded_labels)
-      query = Label.select(Label.arel_table[:priority].minimum).
-        joins(:label_links).
-        where(label_links: { target_type: name }).
-        where("label_links.target_id = #{table_name}.id").
-        reorder(nil)
-
-      query.where.not(title: excluded_labels) if excluded_labels.present?
-
-      query
-    end
   end
 
   def today?
@@ -201,6 +196,10 @@ module Issuable
     end
   end
 
+  def user_authored?(user)
+    user == author
+  end
+
   def subscribed_without_subscriptions?(user)
     participants(user).include?(user)
   end
diff --git a/app/models/concerns/note_on_diff.rb b/app/models/concerns/note_on_diff.rb
index a881fb83b7f..b8dd27a7afe 100644
--- a/app/models/concerns/note_on_diff.rb
+++ b/app/models/concerns/note_on_diff.rb
@@ -28,4 +28,8 @@ module NoteOnDiff
   def can_be_award_emoji?
     false
   end
+
+  def to_discussion
+    Discussion.new([self])
+  end
 end
diff --git a/app/models/concerns/project_features_compatibility.rb b/app/models/concerns/project_features_compatibility.rb
new file mode 100644
index 00000000000..9216122923e
--- /dev/null
+++ b/app/models/concerns/project_features_compatibility.rb
@@ -0,0 +1,37 @@
+# Makes api V3 compatible with old project features permissions methods
+#
+# After migrating issues_enabled merge_requests_enabled builds_enabled snippets_enabled and wiki_enabled
+# fields to a new table "project_features", support for the old fields is still needed in the API.
+
+module ProjectFeaturesCompatibility
+  extend ActiveSupport::Concern
+
+  def wiki_enabled=(value)
+    write_feature_attribute(:wiki_access_level, value)
+  end
+
+  def builds_enabled=(value)
+    write_feature_attribute(:builds_access_level, value)
+  end
+
+  def merge_requests_enabled=(value)
+    write_feature_attribute(:merge_requests_access_level, value)
+  end
+
+  def issues_enabled=(value)
+    write_feature_attribute(:issues_access_level, value)
+  end
+
+  def snippets_enabled=(value)
+    write_feature_attribute(:snippets_access_level, value)
+  end
+
+  private
+
+  def write_feature_attribute(field, value)
+    build_project_feature unless project_feature
+
+    access_level = value == "true" ? ProjectFeature::ENABLED : ProjectFeature::DISABLED
+    project_feature.update_attribute(field, access_level)
+  end
+end
diff --git a/app/models/concerns/sortable.rb b/app/models/concerns/sortable.rb
index 8b47b9e0abd..1ebecd86af9 100644
--- a/app/models/concerns/sortable.rb
+++ b/app/models/concerns/sortable.rb
@@ -35,5 +35,19 @@ module Sortable
         all
       end
     end
+
+    private
+
+    def highest_label_priority(object_types, condition_field, excluded_labels: [])
+      query = Label.select(Label.arel_table[:priority].minimum).
+        joins(:label_links).
+        where(label_links: { target_type: object_types }).
+        where("label_links.target_id = #{condition_field}").
+        reorder(nil)
+
+      query.where.not(title: excluded_labels) if excluded_labels.present?
+
+      query
+    end
   end
 end
diff --git a/app/models/concerns/spammable.rb b/app/models/concerns/spammable.rb
index ce54fe5d3bf..1aa97debe42 100644
--- a/app/models/concerns/spammable.rb
+++ b/app/models/concerns/spammable.rb
@@ -23,7 +23,7 @@ module Spammable
 
   def submittable_as_spam?
     if user_agent_detail
-      user_agent_detail.submittable?
+      user_agent_detail.submittable? && current_application_settings.akismet_enabled
     else
       false
     end
diff --git a/app/models/concerns/taskable.rb b/app/models/concerns/taskable.rb
index df2a9e3e84b..a3ac577cf3e 100644
--- a/app/models/concerns/taskable.rb
+++ b/app/models/concerns/taskable.rb
@@ -52,11 +52,11 @@ module Taskable
   end
 
   # Return a string that describes the current state of this Taskable's task
-  # list items, e.g. "20 tasks (12 completed, 8 remaining)"
+  # list items, e.g. "12 of 20 tasks completed"
   def task_status
     return '' if description.blank?
 
     sum = tasks.summary
-    "#{sum.item_count} tasks (#{sum.complete_count} completed, #{sum.incomplete_count} remaining)"
+    "#{sum.complete_count} of #{sum.item_count} #{'task'.pluralize(sum.item_count)} completed"
   end
 end
diff --git a/app/models/diff_note.rb b/app/models/diff_note.rb
index aa54189fea9..4442cefc7e9 100644
--- a/app/models/diff_note.rb
+++ b/app/models/diff_note.rb
@@ -16,6 +16,9 @@ class DiffNote < Note
   after_initialize :ensure_original_discussion_id
   before_validation :set_original_position, :update_position, on: :create
   before_validation :set_line_code, :set_original_discussion_id
+  # We need to do this again, because it's already in `Note`, but is affected by
+  # `update_position` and needs to run after that.
+  before_validation :set_discussion_id
   after_save :keep_around_commits
 
   class << self
@@ -104,10 +107,6 @@ class DiffNote < Note
     self.noteable.find_diff_discussion(self.discussion_id)
   end
 
-  def to_discussion
-    Discussion.new([self])
-  end
-
   private
 
   def supported?
diff --git a/app/models/event.rb b/app/models/event.rb
index fd736d12359..a0b7b0dc2b5 100644
--- a/app/models/event.rb
+++ b/app/models/event.rb
@@ -65,7 +65,7 @@ class Event < ActiveRecord::Base
     elsif created_project?
       true
     elsif issue? || issue_note?
-      Ability.abilities.allowed?(user, :read_issue, note? ? note_target : target)
+      Ability.allowed?(user, :read_issue, note? ? note_target : target)
     else
       ((merge_request? || note?) && target.present?) || milestone?
     end
diff --git a/app/models/hooks/project_hook.rb b/app/models/hooks/project_hook.rb
index 836a75b0608..c631e7a7df5 100644
--- a/app/models/hooks/project_hook.rb
+++ b/app/models/hooks/project_hook.rb
@@ -2,6 +2,7 @@ class ProjectHook < WebHook
   belongs_to :project
 
   scope :issue_hooks, -> { where(issues_events: true) }
+  scope :confidential_issue_hooks, -> { where(confidential_issues_events: true) }
   scope :note_hooks, -> { where(note_events: true) }
   scope :merge_request_hooks, -> { where(merge_requests_events: true) }
   scope :build_hooks, -> { where(build_events: true) }
diff --git a/app/models/hooks/web_hook.rb b/app/models/hooks/web_hook.rb
index f365dee3141..595602e80fe 100644
--- a/app/models/hooks/web_hook.rb
+++ b/app/models/hooks/web_hook.rb
@@ -4,6 +4,7 @@ class WebHook < ActiveRecord::Base
 
   default_value_for :push_events, true
   default_value_for :issues_events, false
+  default_value_for :confidential_issues_events, false
   default_value_for :note_events, false
   default_value_for :merge_requests_events, false
   default_value_for :tag_push_events, false
diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb
index 5330a07ee35..b0b1313f94a 100644
--- a/app/models/merge_request.rb
+++ b/app/models/merge_request.rb
@@ -10,14 +10,16 @@ class MergeRequest < ActiveRecord::Base
   belongs_to :source_project, foreign_key: :source_project_id, class_name: "Project"
   belongs_to :merge_user, class_name: "User"
 
-  has_one :merge_request_diff, dependent: :destroy
+  has_many :merge_request_diffs, dependent: :destroy
+  has_one :merge_request_diff,
+    -> { order('merge_request_diffs.id DESC') }
 
   has_many :events, as: :target, dependent: :destroy
 
   serialize :merge_params, Hash
 
-  after_create :create_merge_request_diff, unless: :importing?
-  after_update :update_merge_request_diff
+  after_create :ensure_merge_request_diff, unless: :importing?
+  after_update :reload_diff_if_branch_changed
 
   delegate :commits, :real_size, to: :merge_request_diff, prefix: nil
 
@@ -89,13 +91,13 @@ class MergeRequest < ActiveRecord::Base
     end
   end
 
-  validates :source_project, presence: true, unless: [:allow_broken, :importing?]
+  validates :source_project, presence: true, unless: [:allow_broken, :importing?, :closed_without_fork?]
   validates :source_branch, presence: true
   validates :target_project, presence: true
   validates :target_branch, presence: true
   validates :merge_user, presence: true, if: :merge_when_build_succeeds?
-  validate :validate_branches, unless: [:allow_broken, :importing?]
-  validate :validate_fork
+  validate :validate_branches, unless: [:allow_broken, :importing?, :closed_without_fork?]
+  validate :validate_fork, unless: :closed_without_fork?
 
   scope :by_branch, ->(branch_name) { where("(source_branch LIKE :branch) OR (target_branch LIKE :branch)", branch: branch_name) }
   scope :cared, ->(user) { where('assignee_id = :user OR author_id = :user', user: user.id) }
@@ -170,10 +172,10 @@ class MergeRequest < ActiveRecord::Base
   end
 
   def diffs(diff_options = nil)
-    if self.compare
-      self.compare.diffs(diff_options)
+    if compare
+      compare.diffs(diff_options)
     else
-      Gitlab::Diff::FileCollection::MergeRequest.new(self, diff_options: diff_options)
+      merge_request_diff.diffs(diff_options)
     end
   end
 
@@ -184,8 +186,8 @@ class MergeRequest < ActiveRecord::Base
   def diff_base_commit
     if persisted?
       merge_request_diff.base_commit
-    elsif diff_start_commit && diff_head_commit
-      self.target_project.merge_base_commit(diff_start_sha, diff_head_sha)
+    else
+      branch_merge_base_commit
     end
   end
 
@@ -238,12 +240,21 @@ class MergeRequest < ActiveRecord::Base
 
   def source_branch_head
     source_branch_ref = @source_branch_sha || source_branch
-    source_project.repository.commit(source_branch) if source_branch_ref
+    source_project.repository.commit(source_branch_ref) if source_branch_ref
   end
 
   def target_branch_head
     target_branch_ref = @target_branch_sha || target_branch
-    target_project.repository.commit(target_branch) if target_branch_ref
+    target_project.repository.commit(target_branch_ref) if target_branch_ref
+  end
+
+  def branch_merge_base_commit
+    start_sha = target_branch_sha
+    head_sha  = source_branch_sha
+
+    if start_sha && head_sha
+      target_project.merge_base_commit(start_sha, head_sha)
+    end
   end
 
   def target_branch_sha
@@ -267,16 +278,16 @@ class MergeRequest < ActiveRecord::Base
   # Return diff_refs instance trying to not touch the git repository
   def diff_sha_refs
     if merge_request_diff && merge_request_diff.diff_refs_by_sha?
-      return Gitlab::Diff::DiffRefs.new(
-        base_sha:  merge_request_diff.base_commit_sha,
-        start_sha: merge_request_diff.start_commit_sha,
-        head_sha:  merge_request_diff.head_commit_sha
-      )
+      merge_request_diff.diff_refs
     else
       diff_refs
     end
   end
 
+  def branch_merge_base_sha
+    branch_merge_base_commit.try(:sha)
+  end
+
   def validate_branches
     if target_project == source_project && target_branch == source_branch
       errors.add :branch_conflict, "You can not use same project/branch for source and target"
@@ -294,36 +305,49 @@ class MergeRequest < ActiveRecord::Base
 
   def validate_fork
     return true unless target_project && source_project
+    return true if target_project == source_project
+    return true unless forked_source_project_missing?
 
-    if target_project == source_project
-      true
-    else
-      # If source and target projects are different
-      # we should check if source project is actually a fork of target project
-      if source_project.forked_from?(target_project)
-        true
-      else
-        errors.add :validate_fork,
-                   'Source project is not a fork of target project'
-      end
-    end
+    errors.add :validate_fork,
+               'Source project is not a fork of the target project'
   end
 
-  def update_merge_request_diff
+  def closed_without_fork?
+    closed? && forked_source_project_missing?
+  end
+
+  def forked_source_project_missing?
+    return false unless for_fork?
+    return true unless source_project
+
+    !source_project.forked_from?(target_project)
+  end
+
+  def ensure_merge_request_diff
+    merge_request_diff || create_merge_request_diff
+  end
+
+  def create_merge_request_diff
+    merge_request_diffs.create
+    reload_merge_request_diff
+  end
+
+  def reload_merge_request_diff
+    merge_request_diff(true)
+  end
+
+  def reload_diff_if_branch_changed
     if source_branch_changed? || target_branch_changed?
       reload_diff
     end
   end
 
   def reload_diff
-    return unless merge_request_diff && open?
+    return unless open?
 
     old_diff_refs = self.diff_refs
-
-    merge_request_diff.reload_content
-
+    create_merge_request_diff
     MergeRequests::MergeRequestDiffCacheService.new.execute(self)
-
     new_diff_refs = self.diff_refs
 
     update_diff_notes_positions(
@@ -387,7 +411,7 @@ class MergeRequest < ActiveRecord::Base
   def can_remove_source_branch?(current_user)
     !source_project.protected_branch?(source_branch) &&
       !source_project.root_ref?(source_branch) &&
-      Ability.abilities.allowed?(current_user, :push_code, source_project) &&
+      Ability.allowed?(current_user, :push_code, source_project) &&
       diff_head_commit == source_branch_head
   end
 
@@ -705,7 +729,9 @@ class MergeRequest < ActiveRecord::Base
   end
 
   def pipeline
-    @pipeline ||= source_project.pipeline(diff_head_sha, source_branch) if diff_head_sha && source_project
+    return unless diff_head_sha && source_project
+
+    @pipeline ||= source_project.pipeline_for(source_branch, diff_head_sha)
   end
 
   def all_pipelines
@@ -777,8 +803,12 @@ class MergeRequest < ActiveRecord::Base
     return @conflicts_can_be_resolved_in_ui = false unless has_complete_diff_refs?
 
     begin
-      @conflicts_can_be_resolved_in_ui = conflicts.files.each(&:lines)
-    rescue Gitlab::Conflict::Parser::ParserError, Gitlab::Conflict::FileCollection::ConflictSideMissing
+      # Try to parse each conflict. If the MR's mergeable status hasn't been updated,
+      # ensure that we don't say there are conflicts to resolve when there are no conflict
+      # files.
+      conflicts.files.each(&:lines)
+      @conflicts_can_be_resolved_in_ui = conflicts.files.length > 0
+    rescue Rugged::OdbError, Gitlab::Conflict::Parser::ParserError, Gitlab::Conflict::FileCollection::ConflictSideMissing
       @conflicts_can_be_resolved_in_ui = false
     end
   end
diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb
index 32cc6a3bfea..445179a4487 100644
--- a/app/models/merge_request_diff.rb
+++ b/app/models/merge_request_diff.rb
@@ -8,8 +8,6 @@ class MergeRequestDiff < ActiveRecord::Base
 
   belongs_to :merge_request
 
-  delegate :source_branch_sha, :target_branch_sha, :target_branch, :source_branch, to: :merge_request, prefix: nil
-
   state_machine :state, initial: :empty do
     state :collected
     state :overflow
@@ -24,12 +22,47 @@ class MergeRequestDiff < ActiveRecord::Base
   serialize :st_commits
   serialize :st_diffs
 
-  after_create :reload_content, unless: :importing?
-  after_save :keep_around_commits, unless: :importing?
+  # All diff information is collected from repository after object is created.
+  # It allows you to override variables like head_commit_sha before getting diff.
+  after_create :save_git_content, unless: :importing?
+
+  def self.select_without_diff
+    select(column_names - ['st_diffs'])
+  end
 
-  def reload_content
+  # Collect information about commits and diff from repository
+  # and save it to the database as serialized data
+  def save_git_content
+    ensure_commits_sha
+    save_commits
     reload_commits
-    reload_diffs
+    save_diffs
+    keep_around_commits
+  end
+
+  def ensure_commits_sha
+    merge_request.fetch_ref
+    self.start_commit_sha ||= merge_request.target_branch_sha
+    self.head_commit_sha  ||= merge_request.source_branch_sha
+    self.base_commit_sha  ||= find_base_sha
+    save
+  end
+
+  # Override head_commit_sha to keep compatibility with merge request diff
+  # created before version 8.4 that does not store head_commit_sha in separate db field.
+  def head_commit_sha
+    if persisted? && super.nil?
+      last_commit.try(:sha)
+    else
+      super
+    end
+  end
+
+  # This method will rely on repository branch sha
+  # in case start_commit_sha is nil. Its necesarry for old merge request diff
+  # created before version 8.4 to work
+  def safe_start_commit_sha
+    start_commit_sha || merge_request.target_branch_sha
   end
 
   def size
@@ -38,14 +71,11 @@ class MergeRequestDiff < ActiveRecord::Base
 
   def raw_diffs(options = {})
     if options[:ignore_whitespace_change]
-      @raw_diffs_no_whitespace ||= begin
-        compare = Gitlab::Git::Compare.new(
+      @diffs_no_whitespace ||=
+        Gitlab::Git::Compare.new(
           repository.raw_repository,
-          self.start_commit_sha || self.target_branch_sha,
-          self.head_commit_sha || self.source_branch_sha,
-        )
-        compare.diffs(options)
-      end
+          safe_start_commit_sha,
+          head_commit_sha).diffs(options)
     else
       @raw_diffs ||= {}
       @raw_diffs[options] ||= load_diffs(st_diffs, options)
@@ -56,6 +86,11 @@ class MergeRequestDiff < ActiveRecord::Base
     @commits ||= load_commits(st_commits || [])
   end
 
+  def reload_commits
+    @commits = nil
+    commits
+  end
+
   def last_commit
     commits.first
   end
@@ -65,55 +100,60 @@ class MergeRequestDiff < ActiveRecord::Base
   end
 
   def base_commit
-    return unless self.base_commit_sha
+    return unless base_commit_sha
 
-    project.commit(self.base_commit_sha)
+    project.commit(base_commit_sha)
   end
 
   def start_commit
-    return unless self.start_commit_sha
+    return unless start_commit_sha
 
-    project.commit(self.start_commit_sha)
+    project.commit(start_commit_sha)
   end
 
   def head_commit
-    return last_commit unless self.head_commit_sha
+    return unless head_commit_sha
+
+    project.commit(head_commit_sha)
+  end
+
+  def diff_refs
+    return unless start_commit_sha || base_commit_sha
 
-    project.commit(self.head_commit_sha)
+    Gitlab::Diff::DiffRefs.new(
+      base_sha:  base_commit_sha,
+      start_sha: start_commit_sha,
+      head_sha:  head_commit_sha
+    )
   end
 
   def diff_refs_by_sha?
     base_commit_sha? && head_commit_sha? && start_commit_sha?
   end
 
+  def diffs(diff_options = nil)
+    Gitlab::Diff::FileCollection::MergeRequestDiff.new(self, diff_options: diff_options)
+  end
+
+  def project
+    merge_request.target_project
+  end
+
   def compare
     @compare ||=
-      begin
-        # Update ref for merge request
-        merge_request.fetch_ref
+      Gitlab::Git::Compare.new(
+        repository.raw_repository,
+        safe_start_commit_sha,
+        head_commit_sha
+      )
+  end
 
-        Gitlab::Git::Compare.new(
-          repository.raw_repository,
-          self.target_branch_sha,
-          self.source_branch_sha
-        )
-      end
+  def latest?
+    self == merge_request.merge_request_diff
   end
 
   private
 
-  # Collect array of Git::Commit objects
-  # between target and source branches
-  def unmerged_commits
-    commits = compare.commits
-
-    if commits.present?
-      commits = Commit.decorate(commits, merge_request.source_project).reverse
-    end
-
-    commits
-  end
-
   def dump_commits(commits)
     commits.map(&:to_hash)
   end
@@ -122,26 +162,21 @@ class MergeRequestDiff < ActiveRecord::Base
     array.map { |hash| Commit.new(Gitlab::Git::Commit.new(hash), merge_request.source_project) }
   end
 
-  # Reload all commits related to current merge request from repo
+  # Load all commits related to current merge request diff from repo
   # and save it as array of hashes in st_commits db field
-  def reload_commits
+  def save_commits
     new_attributes = {}
 
-    commit_objects = unmerged_commits
+    commits = compare.commits
 
-    if commit_objects.present?
-      new_attributes[:st_commits] = dump_commits(commit_objects)
+    if commits.present?
+      commits = Commit.decorate(commits, merge_request.source_project).reverse
+      new_attributes[:st_commits] = dump_commits(commits)
     end
 
     update_columns_serialized(new_attributes)
   end
 
-  # Collect array of Git::Diff objects
-  # between target and source branches
-  def unmerged_diffs
-    compare.diffs(Commit.max_diff_options)
-  end
-
   def dump_diffs(diffs)
     if diffs.respond_to?(:map)
       diffs.map(&:to_hash)
@@ -162,16 +197,16 @@ class MergeRequestDiff < ActiveRecord::Base
     end
   end
 
-  # Reload diffs between branches related to current merge request from repo
+  # Load diffs between branches related to current merge request diff from repo
   # and save it as array of hashes in st_diffs db field
-  def reload_diffs
+  def save_diffs
     new_attributes = {}
     new_diffs = []
 
     if commits.size.zero?
       new_attributes[:state] = :empty
     else
-      diff_collection = unmerged_diffs
+      diff_collection = compare.diffs(Commit.max_diff_options)
 
       if diff_collection.overflow?
         # Set our state to 'overflow' to make the #empty? and #collected?
@@ -188,32 +223,17 @@ class MergeRequestDiff < ActiveRecord::Base
     end
 
     new_attributes[:st_diffs] = new_diffs
-
-    new_attributes[:start_commit_sha] = self.target_branch_sha
-    new_attributes[:head_commit_sha] = self.source_branch_sha
-    new_attributes[:base_commit_sha] = branch_base_sha
-
     update_columns_serialized(new_attributes)
-
-    keep_around_commits
-  end
-
-  def project
-    merge_request.target_project
   end
 
   def repository
     project.repository
   end
 
-  def branch_base_commit
-    return unless self.source_branch_sha && self.target_branch_sha
-
-    project.merge_base_commit(self.source_branch_sha, self.target_branch_sha)
-  end
+  def find_base_sha
+    return unless head_commit_sha && start_commit_sha
 
-  def branch_base_sha
-    branch_base_commit.try(:sha)
+    project.merge_base_commit(head_commit_sha, start_commit_sha).try(:sha)
   end
 
   def utf8_st_diffs
@@ -248,8 +268,8 @@ class MergeRequestDiff < ActiveRecord::Base
   end
 
   def keep_around_commits
-    repository.keep_around(target_branch_sha)
-    repository.keep_around(source_branch_sha)
-    repository.keep_around(branch_base_sha)
+    repository.keep_around(start_commit_sha)
+    repository.keep_around(head_commit_sha)
+    repository.keep_around(base_commit_sha)
   end
 end
diff --git a/app/models/note.rb b/app/models/note.rb
index 3bbf5db0b70..b94e3cff2ce 100644
--- a/app/models/note.rb
+++ b/app/models/note.rb
@@ -223,6 +223,10 @@ class Note < ActiveRecord::Base
     end
   end
 
+  def user_authored?(user)
+    user == author
+  end
+
   def award_emoji?
     can_be_award_emoji? && contains_emoji_only?
   end
@@ -259,6 +263,8 @@ class Note < ActiveRecord::Base
 
   def ensure_discussion_id
     return unless self.persisted?
+    # Needed in case the SELECT statement doesn't ask for `discussion_id`
+    return unless self.has_attribute?(:discussion_id)
     return if self.discussion_id
 
     set_discussion_id
diff --git a/app/models/project.rb b/app/models/project.rb
index f9c48a546e6..a6de2c48071 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -11,24 +11,23 @@ class Project < ActiveRecord::Base
   include AfterCommitQueue
   include CaseSensitivity
   include TokenAuthenticatable
+  include ProjectFeaturesCompatibility
 
   extend Gitlab::ConfigHelper
 
   UNKNOWN_IMPORT_URL = 'http://unknown.git'
 
+  delegate :feature_available?, :builds_enabled?, :wiki_enabled?, :merge_requests_enabled?, to: :project_feature, allow_nil: true
+
   default_value_for :archived, false
   default_value_for :visibility_level, gitlab_config_features.visibility_level
-  default_value_for :issues_enabled, gitlab_config_features.issues
-  default_value_for :merge_requests_enabled, gitlab_config_features.merge_requests
-  default_value_for :builds_enabled, gitlab_config_features.builds
-  default_value_for :wiki_enabled, gitlab_config_features.wiki
-  default_value_for :snippets_enabled, gitlab_config_features.snippets
   default_value_for :container_registry_enabled, gitlab_config_features.container_registry
   default_value_for(:repository_storage) { current_application_settings.repository_storage }
   default_value_for(:shared_runners_enabled) { current_application_settings.shared_runners_enabled }
 
   after_create :ensure_dir_exist
   after_save :ensure_dir_exist, if: :namespace_id_changed?
+  after_initialize :setup_project_feature
 
   # set last_activity_at to the same as created_at
   after_create :set_last_activity_at
@@ -62,10 +61,10 @@ class Project < ActiveRecord::Base
   belongs_to :group, -> { where(type: Group) }, foreign_key: 'namespace_id'
   belongs_to :namespace
 
-  has_one :board, dependent: :destroy
-
   has_one :last_event, -> {order 'events.created_at DESC'}, class_name: 'Event', foreign_key: 'project_id'
 
+  has_one :board, dependent: :destroy
+
   # Project services
   has_many :services
   has_one :campfire_service, dependent: :destroy
@@ -130,6 +129,7 @@ class Project < ActiveRecord::Base
   has_many :notification_settings, dependent: :destroy, as: :source
 
   has_one :import_data, dependent: :destroy, class_name: "ProjectImportData"
+  has_one :project_feature, dependent: :destroy
 
   has_many :commit_statuses, dependent: :destroy, class_name: 'CommitStatus', foreign_key: :gl_project_id
   has_many :pipelines, dependent: :destroy, class_name: 'Ci::Pipeline', foreign_key: :gl_project_id
@@ -142,6 +142,7 @@ class Project < ActiveRecord::Base
   has_many :deployments, dependent: :destroy
 
   accepts_nested_attributes_for :variables, allow_destroy: true
+  accepts_nested_attributes_for :project_feature
 
   delegate :name, to: :owner, allow_nil: true, prefix: true
   delegate :members, to: :team, prefix: true
@@ -159,8 +160,6 @@ class Project < ActiveRecord::Base
     length: { within: 0..255 },
     format: { with: Gitlab::Regex.project_path_regex,
               message: Gitlab::Regex.project_path_regex_message }
-  validates :issues_enabled, :merge_requests_enabled,
-            :wiki_enabled, inclusion: { in: [true, false] }
   validates :namespace, presence: true
   validates_uniqueness_of :name, scope: :namespace_id
   validates_uniqueness_of :path, scope: :namespace_id
@@ -196,6 +195,9 @@ class Project < ActiveRecord::Base
   scope :for_milestones, ->(ids) { joins(:milestones).where('milestones.id' => ids).distinct }
   scope :with_push, -> { joins(:events).where('events.action = ?', Event::PUSHED) }
 
+  scope :with_builds_enabled, -> { joins('LEFT JOIN project_features ON projects.id = project_features.project_id').where('project_features.builds_access_level IS NULL or project_features.builds_access_level > 0') }
+  scope :with_issues_enabled, -> { joins('LEFT JOIN project_features ON projects.id = project_features.project_id').where('project_features.issues_access_level IS NULL or project_features.issues_access_level > 0') }
+
   scope :active, -> { joins(:issues, :notes, :merge_requests).order('issues.created_at, notes.created_at, merge_requests.created_at DESC') }
   scope :abandoned, -> { where('projects.last_activity_at < ?', 6.months.ago) }
 
@@ -390,6 +392,13 @@ class Project < ActiveRecord::Base
     end
   end
 
+  def lfs_enabled?
+    return false unless Gitlab.config.lfs.enabled
+    return Gitlab.config.lfs.enabled if self[:lfs_enabled].nil?
+
+    self[:lfs_enabled]
+  end
+
   def repository_storage_path
     Gitlab.config.repositories.storages[repository_storage]
   end
@@ -436,7 +445,7 @@ class Project < ActiveRecord::Base
 
   # ref can't be HEAD, can only be branch/tag name or SHA
   def latest_successful_builds_for(ref = default_branch)
-    latest_pipeline = pipelines.latest_successful_for(ref).first
+    latest_pipeline = pipelines.latest_successful_for(ref)
 
     if latest_pipeline
       latest_pipeline.builds.latest.with_artifacts
@@ -471,8 +480,6 @@ class Project < ActiveRecord::Base
   end
 
   def reset_cache_and_import_attrs
-    update(import_error: nil)
-
     ProjectCacheWorker.perform_async(self.id)
 
     self.import_data.destroy if self.import_data
@@ -611,7 +618,10 @@ class Project < ActiveRecord::Base
   end
 
   def new_issue_address(author)
-    if Gitlab::IncomingEmail.enabled? && author
+    # This feature is disabled for the time being.
+    return nil
+
+    if Gitlab::IncomingEmail.enabled? && author # rubocop:disable Lint/UnreachableCode
       Gitlab::IncomingEmail.reply_address(
         "#{path_with_namespace}+#{author.authentication_token}")
     end
@@ -679,6 +689,10 @@ class Project < ActiveRecord::Base
     update_column(:has_external_issue_tracker, services.external_issue_trackers.any?)
   end
 
+  def has_wiki?
+    wiki_enabled? || has_external_wiki?
+  end
+
   def external_wiki
     if has_external_wiki.nil?
       cache_has_external_wiki # Populate
@@ -1034,6 +1048,7 @@ class Project < ActiveRecord::Base
                                         "refs/heads/#{branch}",
                                         force: true)
     repository.copy_gitattributes(branch)
+    repository.expire_avatar_cache(branch)
     reload_default_branch
   end
 
@@ -1094,16 +1109,21 @@ class Project < ActiveRecord::Base
     !namespace.share_with_group_lock
   end
 
-  def pipeline(sha, ref)
+  def pipeline_for(ref, sha = nil)
+    sha ||= commit(ref).try(:sha)
+
+    return unless sha
+
     pipelines.order(id: :desc).find_by(sha: sha, ref: ref)
   end
 
-  def ensure_pipeline(sha, ref, current_user = nil)
-    pipeline(sha, ref) || pipelines.create(sha: sha, ref: ref, user: current_user)
+  def ensure_pipeline(ref, sha, current_user = nil)
+    pipeline_for(ref, sha) ||
+      pipelines.create(sha: sha, ref: ref, user: current_user)
   end
 
   def enable_ci
-    self.builds_enabled = true
+    project_feature.update_attribute(:builds_access_level, ProjectFeature::ENABLED)
   end
 
   def any_runners?(&block)
@@ -1270,6 +1290,11 @@ class Project < ActiveRecord::Base
 
   private
 
+  # Prevents the creation of project_feature record for every project
+  def setup_project_feature
+    build_project_feature unless project_feature
+  end
+
   def default_branch_protected?
     current_application_settings.default_branch_protection == Gitlab::Access::PROTECTION_FULL ||
       current_application_settings.default_branch_protection == Gitlab::Access::PROTECTION_DEV_CAN_MERGE
diff --git a/app/models/project_feature.rb b/app/models/project_feature.rb
new file mode 100644
index 00000000000..9c602c582bd
--- /dev/null
+++ b/app/models/project_feature.rb
@@ -0,0 +1,63 @@
+class ProjectFeature < ActiveRecord::Base
+  # == Project features permissions
+  #
+  # Grants access level to project tools
+  #
+  # Tools can be enabled only for users, everyone or disabled
+  # Access control is made only for non private projects
+  #
+  # levels:
+  #
+  # Disabled: not enabled for anyone
+  # Private:  enabled only for team members
+  # Enabled:  enabled for everyone able to access the project
+  #
+
+  # Permision levels
+  DISABLED = 0
+  PRIVATE  = 10
+  ENABLED  = 20
+
+  FEATURES = %i(issues merge_requests wiki snippets builds)
+
+  belongs_to :project
+
+  def feature_available?(feature, user)
+    raise ArgumentError, 'invalid project feature' unless FEATURES.include?(feature)
+
+    get_permission(user, public_send("#{feature}_access_level"))
+  end
+
+  def builds_enabled?
+    return true unless builds_access_level
+
+    builds_access_level > DISABLED
+  end
+
+  def wiki_enabled?
+    return true unless wiki_access_level
+
+    wiki_access_level > DISABLED
+  end
+
+  def merge_requests_enabled?
+    return true unless merge_requests_access_level
+
+    merge_requests_access_level > DISABLED
+  end
+
+  private
+
+  def get_permission(user, level)
+    case level
+    when DISABLED
+      false
+    when PRIVATE
+      user && (project.team.member?(user) || user.admin?)
+    when ENABLED
+      true
+    else
+      true
+    end
+  end
+end
diff --git a/app/models/project_services/hipchat_service.rb b/app/models/project_services/hipchat_service.rb
index d7c986c1a91..afebd3b6a12 100644
--- a/app/models/project_services/hipchat_service.rb
+++ b/app/models/project_services/hipchat_service.rb
@@ -39,7 +39,7 @@ class HipchatService < Service
   end
 
   def supported_events
-    %w(push issue merge_request note tag_push build)
+    %w(push issue confidential_issue merge_request note tag_push build)
   end
 
   def execute(data)
diff --git a/app/models/project_services/slack_service.rb b/app/models/project_services/slack_service.rb
index abbc780dc1a..e6c943db2bf 100644
--- a/app/models/project_services/slack_service.rb
+++ b/app/models/project_services/slack_service.rb
@@ -44,7 +44,7 @@ class SlackService < Service
   end
 
   def supported_events
-    %w(push issue merge_request note tag_push build wiki_page)
+    %w(push issue confidential_issue merge_request note tag_push build wiki_page)
   end
 
   def execute(data)
diff --git a/app/models/repository.rb b/app/models/repository.rb
index 2494c266cd2..414b82516bc 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -120,8 +120,21 @@ class Repository
     commits
   end
 
-  def find_branch(name)
-    raw_repository.branches.find { |branch| branch.name == name }
+  def find_branch(name, fresh_repo: true)
+    # Since the Repository object may have in-memory index changes, invalidating the memoized Repository object may
+    # cause unintended side effects. Because finding a branch is a read-only operation, we can safely instantiate
+    # a new repo here to ensure a consistent state to avoid a libgit2 bug where concurrent access (e.g. via git gc)
+    # may cause the branch to "disappear" erroneously or have the wrong SHA.
+    #
+    # See: https://github.com/libgit2/libgit2/issues/1534 and https://gitlab.com/gitlab-org/gitlab-ce/issues/15392
+    raw_repo =
+      if fresh_repo
+        Gitlab::Git::Repository.new(path_to_repo)
+      else
+        raw_repository
+      end
+
+    raw_repo.find_branch(name)
   end
 
   def find_tag(name)
@@ -136,7 +149,7 @@ class Repository
     return false unless target
 
     GitHooksService.new.execute(user, path_to_repo, oldrev, target, ref) do
-      rugged.branches.create(branch_name, target)
+      update_ref!(ref, target, oldrev)
     end
 
     after_create_branch
@@ -168,7 +181,7 @@ class Repository
     ref    = Gitlab::Git::BRANCH_REF_PREFIX + branch_name
 
     GitHooksService.new.execute(user, path_to_repo, oldrev, newrev, ref) do
-      rugged.branches.delete(branch_name)
+      update_ref!(ref, newrev, oldrev)
     end
 
     after_remove_branch
@@ -202,6 +215,21 @@ class Repository
     rugged.references.exist?(ref)
   end
 
+  def update_ref!(name, newrev, oldrev)
+    # We use 'git update-ref' because libgit2/rugged currently does not
+    # offer 'compare and swap' ref updates. Without compare-and-swap we can
+    # (and have!) accidentally reset the ref to an earlier state, clobbering
+    # commits. See also https://github.com/libgit2/libgit2/issues/1534.
+    command = %w[git update-ref --stdin -z]
+    _, status = Gitlab::Popen.popen(command, path_to_repo) do |stdin|
+      stdin.write("update #{name}\x00#{newrev}\x00#{oldrev}\x00")
+    end
+
+    return if status.zero?
+
+    raise CommitError.new("Could not update branch #{name.sub('refs/heads/', '')}. Please refresh and try again.")
+  end
+
   # Makes sure a commit is kept around when Git garbage collection runs.
   # Git GC will delete commits from the repository that are no longer in any
   # branches or tags, but we want to keep some of these commits around, for
@@ -277,7 +305,7 @@ class Repository
   def cache_keys
     %i(size commit_count
        readme version contribution_guide changelog
-       license_blob license_key gitignore)
+       license_blob license_key gitignore koding_yml)
   end
 
   # Keys for data on branch/tag operations.
@@ -553,6 +581,14 @@ class Repository
     end
   end
 
+  def koding_yml
+    return nil unless head_exists?
+
+    cache.fetch(:koding_yml) do
+      file_on_head(/\A\.koding\.yml\z/)
+    end
+  end
+
   def gitlab_ci_yml
     return nil unless head_exists?
 
@@ -993,15 +1029,10 @@ class Repository
   def commit_with_hooks(current_user, branch)
     update_autocrlf_option
 
-    oldrev = Gitlab::Git::BLANK_SHA
     ref = Gitlab::Git::BRANCH_REF_PREFIX + branch
     target_branch = find_branch(branch)
     was_empty = empty?
 
-    if !was_empty && target_branch
-      oldrev = target_branch.target.id
-    end
-
     # Make commit
     newrev = yield(ref)
 
@@ -1009,24 +1040,15 @@ class Repository
       raise CommitError.new('Failed to create commit')
     end
 
+    oldrev = rugged.lookup(newrev).parent_ids.first || Gitlab::Git::BLANK_SHA
+
     GitHooksService.new.execute(current_user, path_to_repo, oldrev, newrev, ref) do
+      update_ref!(ref, newrev, oldrev)
+      
       if was_empty || !target_branch
-        # Create branch
-        rugged.references.create(ref, newrev)
-
         # If repo was empty expire cache
         after_create if was_empty
         after_create_branch
-      else
-        # Update head
-        current_head = find_branch(branch).target.id
-
-        # Make sure target branch was not changed during pre-receive hook
-        if current_head == oldrev
-          rugged.references.update(ref, newrev)
-        else
-          raise CommitError.new('Commit was rejected because branch received new push')
-        end
       end
     end
 
@@ -1057,7 +1079,7 @@ class Repository
 
     @avatar ||= cache.fetch(:avatar) do
       AVATAR_FILES.find do |file|
-        blob_at_branch('master', file)
+        blob_at_branch(root_ref, file)
       end
     end
   end
diff --git a/app/models/service.rb b/app/models/service.rb
index 09b4717a523..198e7247838 100644
--- a/app/models/service.rb
+++ b/app/models/service.rb
@@ -7,6 +7,7 @@ class Service < ActiveRecord::Base
   default_value_for :active, false
   default_value_for :push_events, true
   default_value_for :issues_events, true
+  default_value_for :confidential_issues_events, true
   default_value_for :merge_requests_events, true
   default_value_for :tag_push_events, true
   default_value_for :note_events, true
@@ -33,6 +34,7 @@ class Service < ActiveRecord::Base
   scope :push_hooks, -> { where(push_events: true, active: true) }
   scope :tag_push_hooks, -> { where(tag_push_events: true, active: true) }
   scope :issue_hooks, -> { where(issues_events: true, active: true) }
+  scope :confidential_issue_hooks, -> { where(confidential_issues_events: true, active: true) }
   scope :merge_request_hooks, -> { where(merge_requests_events: true, active: true) }
   scope :note_hooks, -> { where(note_events: true, active: true) }
   scope :build_hooks, -> { where(build_events: true, active: true) }
@@ -100,7 +102,7 @@ class Service < ActiveRecord::Base
   end
 
   def supported_events
-    %w(push tag_push issue merge_request wiki_page)
+    %w(push tag_push issue confidential_issue merge_request wiki_page)
   end
 
   def execute(data)
diff --git a/app/models/todo.rb b/app/models/todo.rb
index 8d7a5965aa1..6ae9956ade5 100644
--- a/app/models/todo.rb
+++ b/app/models/todo.rb
@@ -1,4 +1,6 @@
 class Todo < ActiveRecord::Base
+  include Sortable
+
   ASSIGNED          = 1
   MENTIONED         = 2
   BUILD_FAILED      = 3
@@ -41,6 +43,23 @@ class Todo < ActiveRecord::Base
 
   after_save :keep_around_commit
 
+  class << self
+    def sort(method)
+      method == "priority" ? order_by_labels_priority : order_by(method)
+    end
+
+    # Order by priority depending on which issue/merge request the Todo belongs to
+    # Todos with highest priority first then oldest todos
+    # Need to order by created_at last because of differences on Mysql and Postgres when joining by type "Merge_request/Issue"
+    def order_by_labels_priority
+      highest_priority = highest_label_priority(["Issue", "MergeRequest"], "todos.target_id").to_sql
+
+      select("#{table_name}.*, (#{highest_priority}) AS highest_priority").
+        order(Gitlab::Database.nulls_last_order('highest_priority', 'ASC')).
+        order('todos.created_at')
+    end
+  end
+
   def build_failed?
     action == BUILD_FAILED
   end
diff --git a/app/models/user.rb b/app/models/user.rb
index 48e83ab7e56..6996740eebd 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -433,7 +433,7 @@ class User < ActiveRecord::Base
   #
   # This logic is duplicated from `Ability#project_abilities` into a SQL form.
   def projects_where_can_admin_issues
-    authorized_projects(Gitlab::Access::REPORTER).non_archived.where.not(issues_enabled: false)
+    authorized_projects(Gitlab::Access::REPORTER).non_archived.with_issues_enabled
   end
 
   def is_admin?
@@ -460,16 +460,12 @@ class User < ActiveRecord::Base
     can?(:create_group, nil)
   end
 
-  def abilities
-    Ability.abilities
-  end
-
   def can_select_namespace?
     several_namespaces? || admin
   end
 
   def can?(action, subject)
-    abilities.allowed?(self, action, subject)
+    Ability.allowed?(self, action, subject)
   end
 
   def first_name
@@ -489,10 +485,10 @@ class User < ActiveRecord::Base
     (personal_projects.count.to_f / projects_limit) * 100
   end
 
-  def recent_push(project_id = nil)
+  def recent_push(project_ids = nil)
     # Get push events not earlier than 2 hours ago
     events = recent_events.code_push.where("created_at > ?", Time.now - 2.hours)
-    events = events.where(project_id: project_id) if project_id
+    events = events.where(project_id: project_ids) if project_ids
 
     # Use the latest event that has not been pushed or merged recently
     events.recent.find do |event|