Merge branch 'master' into build-chunks-on-object-storage

12 files changed, 78 insertions, 45 deletions

diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb
index 3d58a14882f..bddeb8b0352 100644
--- a/app/models/application_setting.rb
+++ b/app/models/application_setting.rb
@@ -212,14 +212,6 @@ class ApplicationSetting < ActiveRecord::Base
     end
   end
 
-  validates_each :disabled_oauth_sign_in_sources do |record, attr, value|
-    value&.each do |source|
-      unless Devise.omniauth_providers.include?(source.to_sym)
-        record.errors.add(attr, "'#{source}' is not an OAuth sign-in source")
-      end
-    end
-  end
-
   validate :terms_exist, if: :enforce_terms?
 
   before_validation :ensure_uuid!
@@ -330,6 +322,11 @@ class ApplicationSetting < ActiveRecord::Base
     ::Gitlab::Database.cached_column_exists?(:application_settings, :sidekiq_throttling_enabled)
   end
 
+  def disabled_oauth_sign_in_sources=(sources)
+    sources = (sources || []).map(&:to_s) & Devise.omniauth_providers.map(&:to_s)
+    super(sources)
+  end
+
   def domain_whitelist_raw
     self.domain_whitelist&.join("\n")
   end
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index f430f18ca9a..e5caa3ffa41 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -561,9 +561,9 @@ module Ci
         .append(key: 'CI_PIPELINE_IID', value: iid.to_s)
         .append(key: 'CI_CONFIG_PATH', value: ci_yaml_file_path)
         .append(key: 'CI_PIPELINE_SOURCE', value: source.to_s)
-        .append(key: 'CI_COMMIT_MESSAGE', value: git_commit_message)
-        .append(key: 'CI_COMMIT_TITLE', value: git_commit_full_title)
-        .append(key: 'CI_COMMIT_DESCRIPTION', value: git_commit_description)
+        .append(key: 'CI_COMMIT_MESSAGE', value: git_commit_message.to_s)
+        .append(key: 'CI_COMMIT_TITLE', value: git_commit_full_title.to_s)
+        .append(key: 'CI_COMMIT_DESCRIPTION', value: git_commit_description.to_s)
     end
 
     def queued_duration
diff --git a/app/models/concerns/sortable.rb b/app/models/concerns/sortable.rb
index db7254c27e0..cb76ae971d4 100644
--- a/app/models/concerns/sortable.rb
+++ b/app/models/concerns/sortable.rb
@@ -12,8 +12,8 @@ module Sortable
     scope :order_created_asc, -> { reorder(created_at: :asc) }
     scope :order_updated_desc, -> { reorder(updated_at: :desc) }
     scope :order_updated_asc, -> { reorder(updated_at: :asc) }
-    scope :order_name_asc, -> { reorder("lower(name) asc") }
-    scope :order_name_desc, -> { reorder("lower(name) desc") }
+    scope :order_name_asc, -> { reorder(Arel::Nodes::Ascending.new(arel_table[:name].lower)) }
+    scope :order_name_desc, -> { reorder(Arel::Nodes::Descending.new(arel_table[:name].lower)) }
   end
 
   module ClassMethods
diff --git a/app/models/issue.rb b/app/models/issue.rb
index d3df2da14e2..4715d942c8d 100644
--- a/app/models/issue.rb
+++ b/app/models/issue.rb
@@ -48,7 +48,7 @@ class Issue < ActiveRecord::Base
   scope :unassigned, -> { where('NOT EXISTS (SELECT TRUE FROM issue_assignees WHERE issue_id = issues.id)') }
   scope :assigned_to, ->(u) { where('EXISTS (SELECT TRUE FROM issue_assignees WHERE user_id = ? AND issue_id = issues.id)', u.id)}
 
-  scope :with_due_date, -> { where('due_date IS NOT NULL') }
+  scope :with_due_date, -> { where.not(due_date: nil) }
   scope :without_due_date, -> { where(due_date: nil) }
   scope :due_before, ->(date) { where('issues.due_date < ?', date) }
   scope :due_between, ->(from_date, to_date) { where('issues.due_date >= ?', from_date).where('issues.due_date <= ?', to_date) }
@@ -56,7 +56,7 @@ class Issue < ActiveRecord::Base
 
   scope :order_due_date_asc, -> { reorder('issues.due_date IS NULL, issues.due_date ASC') }
   scope :order_due_date_desc, -> { reorder('issues.due_date IS NULL, issues.due_date DESC') }
-  scope :order_closest_future_date, -> { reorder('CASE WHEN due_date >= CURRENT_DATE THEN 0 ELSE 1 END ASC, ABS(CURRENT_DATE - due_date) ASC') }
+  scope :order_closest_future_date, -> { reorder('CASE WHEN issues.due_date >= CURRENT_DATE THEN 0 ELSE 1 END ASC, ABS(CURRENT_DATE - issues.due_date) ASC') }
 
   scope :preload_associations, -> { preload(:labels, project: :namespace) }
 
diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb
index 3df1130a6e2..b4090fd8baf 100644
--- a/app/models/merge_request.rb
+++ b/app/models/merge_request.rb
@@ -128,16 +128,9 @@ class MergeRequest < ActiveRecord::Base
     end
 
     after_transition unchecked: :cannot_be_merged do |merge_request, transition|
-      begin
-        # Merge request can become unmergeable due to many reasons.
-        # We only notify if it is due to conflict.
-        unless merge_request.project.repository.can_be_merged?(merge_request.diff_head_sha, merge_request.target_branch)
-          NotificationService.new.merge_request_unmergeable(merge_request)
-          TodoService.new.merge_request_became_unmergeable(merge_request)
-        end
-      rescue Gitlab::Git::CommandError
-        # Checking mergeability can trigger exception, e.g. non-utf8
-        # We ignore this type of errors.
+      if merge_request.notify_conflict?
+        NotificationService.new.merge_request_unmergeable(merge_request)
+        TodoService.new.merge_request_became_unmergeable(merge_request)
       end
     end
 
@@ -378,6 +371,10 @@ class MergeRequest < ActiveRecord::Base
     end
   end
 
+  def non_latest_diffs
+    merge_request_diffs.where.not(id: merge_request_diff.id)
+  end
+
   def diff_size
     # Calling `merge_request_diff.diffs.real_size` will also perform
     # highlighting, which we don't need here.
@@ -619,18 +616,7 @@ class MergeRequest < ActiveRecord::Base
   def reload_diff(current_user = nil)
     return unless open?
 
-    old_diff_refs = self.diff_refs
-    new_diff = create_merge_request_diff
-
-    MergeRequests::MergeRequestDiffCacheService.new.execute(self, new_diff)
-
-    new_diff_refs = self.diff_refs
-
-    update_diff_discussion_positions(
-      old_diff_refs: old_diff_refs,
-      new_diff_refs: new_diff_refs,
-      current_user: current_user
-    )
+    MergeRequests::ReloadDiffsService.new(self, current_user).execute
   end
 
   def check_if_can_be_merged
@@ -715,6 +701,17 @@ class MergeRequest < ActiveRecord::Base
     should_remove_source_branch? || force_remove_source_branch?
   end
 
+  def notify_conflict?
+    (opened? || locked?) &&
+      has_commits? &&
+      !branch_missing? &&
+      !project.repository.can_be_merged?(diff_head_sha, target_branch)
+  rescue Gitlab::Git::CommandError
+    # Checking mergeability can trigger exception, e.g. non-utf8
+    # We ignore this type of errors.
+    false
+  end
+
   def related_notes
     # Fetch comments only from last 100 commits
     commits_for_notes_limit = 100
diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb
index 06aa67c600f..3d72c447b4b 100644
--- a/app/models/merge_request_diff.rb
+++ b/app/models/merge_request_diff.rb
@@ -3,6 +3,7 @@ class MergeRequestDiff < ActiveRecord::Base
   include Importable
   include ManualInverseAssociation
   include IgnorableColumn
+  include EachBatch
 
   # Don't display more than 100 commits at once
   COMMITS_SAFE_SIZE = 100
@@ -17,8 +18,14 @@ class MergeRequestDiff < ActiveRecord::Base
   has_many :merge_request_diff_commits, -> { order(:merge_request_diff_id, :relative_order) }
 
   state_machine :state, initial: :empty do
+    event :clean do
+      transition any => :without_files
+    end
+
     state :collected
     state :overflow
+    # Diff files have been deleted by the system
+    state :without_files
     # Deprecated states: these are no longer used but these values may still occur
     # in the database.
     state :timeout
@@ -27,6 +34,7 @@ class MergeRequestDiff < ActiveRecord::Base
     state :overflow_diff_lines_limit
   end
 
+  scope :with_files, -> { without_states(:without_files, :empty) }
   scope :viewable, -> { without_state(:empty) }
   scope :by_commit_sha, ->(sha) do
     joins(:merge_request_diff_commits).where(merge_request_diff_commits: { sha: sha }).reorder(nil)
@@ -42,6 +50,10 @@ class MergeRequestDiff < ActiveRecord::Base
     find_by(start_commit_sha: diff_refs.start_sha, head_commit_sha: diff_refs.head_sha, base_commit_sha: diff_refs.base_sha)
   end
 
+  def viewable?
+    collected? || without_files? || overflow?
+  end
+
   # Collect information about commits and diff from repository
   # and save it to the database as serialized data
   def save_git_content
@@ -170,6 +182,21 @@ class MergeRequestDiff < ActiveRecord::Base
   end
 
   def diffs(diff_options = nil)
+    if without_files? && comparison = diff_refs.compare_in(project)
+      # It should fetch the repository when diffs are cleaned by the system.
+      # We don't keep these for storage overload purposes.
+      # See https://gitlab.com/gitlab-org/gitlab-ce/issues/37639
+      comparison.diffs(diff_options)
+    else
+      diffs_collection(diff_options)
+    end
+  end
+
+  # Should always return the DB persisted diffs collection
+  # (e.g. Gitlab::Diff::FileCollection::MergeRequestDiff.
+  # It's useful when trying to invalidate old caches through
+  # FileCollection::MergeRequestDiff#clear_cache!
+  def diffs_collection(diff_options = nil)
     Gitlab::Diff::FileCollection::MergeRequestDiff.new(self, diff_options: diff_options)
   end
 
diff --git a/app/models/namespace.rb b/app/models/namespace.rb
index 52fe529c016..7034c633268 100644
--- a/app/models/namespace.rb
+++ b/app/models/namespace.rb
@@ -228,6 +228,10 @@ class Namespace < ActiveRecord::Base
     parent.present?
   end
 
+  def root_ancestor
+    ancestors.reorder(nil).find_by(parent_id: nil)
+  end
+
   def subgroup?
     has_parent?
   end
diff --git a/app/models/project.rb b/app/models/project.rb
index 0d777515536..d91d7dcfe9a 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -2019,6 +2019,10 @@ class Project < ActiveRecord::Base
   end
   request_cache(:any_lfs_file_locks?) { self.id }
 
+  def auto_cancel_pending_pipelines?
+    auto_cancel_pending_pipelines == 'enabled'
+  end
+
   private
 
   def storage
diff --git a/app/models/project_auto_devops.rb b/app/models/project_auto_devops.rb
index d7d6aaceb27..faa831b1949 100644
--- a/app/models/project_auto_devops.rb
+++ b/app/models/project_auto_devops.rb
@@ -29,8 +29,8 @@ class ProjectAutoDevops < ActiveRecord::Base
       end
 
       if manual?
-        variables.append(key: 'STAGING_ENABLED', value: 1)
-        variables.append(key: 'INCREMENTAL_ROLLOUT_ENABLED', value: 1)
+        variables.append(key: 'STAGING_ENABLED', value: '1')
+        variables.append(key: 'INCREMENTAL_ROLLOUT_ENABLED', value: '1')
       end
     end
   end
diff --git a/app/models/project_team.rb b/app/models/project_team.rb
index 33280eda0b9..9a38806baab 100644
--- a/app/models/project_team.rb
+++ b/app/models/project_team.rb
@@ -24,7 +24,7 @@ class ProjectTeam
   end
 
   def add_role(user, role, current_user: nil)
-    send(:"add_#{role}", user, current_user: current_user) # rubocop:disable GitlabSecurity/PublicSend
+    public_send(:"add_#{role}", user, current_user: current_user) # rubocop:disable GitlabSecurity/PublicSend
   end
 
   def find_member(user_id)
diff --git a/app/models/repository.rb b/app/models/repository.rb
index 3089d0162ee..5f9894f1168 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -21,7 +21,7 @@ class Repository
   attr_accessor :full_path, :disk_path, :project, :is_wiki
 
   delegate :ref_name_for_sha, to: :raw_repository
-  delegate :bundle_to_disk, :create_from_bundle, to: :raw_repository
+  delegate :bundle_to_disk, to: :raw_repository
 
   CreateTreeError = Class.new(StandardError)
 
@@ -99,11 +99,11 @@ class Repository
     "#<#{self.class.name}:#{@disk_path}>"
   end
 
-  def commit(ref = 'HEAD')
+  def commit(ref = nil)
     return nil unless exists?
     return ref if ref.is_a?(::Commit)
 
-    find_commit(ref)
+    find_commit(ref || root_ref)
   end
 
   # Finding a commit by the passed SHA
@@ -283,6 +283,10 @@ class Repository
     )
   end
 
+  def cached_methods
+    CACHED_METHODS
+  end
+
   def expire_tags_cache
     expire_method_caches(%i(tag_names tag_count))
     @tags = nil
@@ -423,7 +427,7 @@ class Repository
 
   # Runs code after the HEAD of a repository is changed.
   def after_change_head
-    expire_method_caches(METHOD_CACHES_FOR_FILE_TYPES.keys)
+    expire_all_method_caches
   end
 
   # Runs code after a repository has been forked/imported.
diff --git a/app/models/user.rb b/app/models/user.rb
index 8e0dc91b2a7..48629c58490 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -244,7 +244,7 @@ class User < ActiveRecord::Base
   scope :blocked, -> { with_states(:blocked, :ldap_blocked) }
   scope :external, -> { where(external: true) }
   scope :active, -> { with_state(:active).non_internal }
-  scope :without_projects, -> { where('id NOT IN (SELECT DISTINCT(user_id) FROM members WHERE user_id IS NOT NULL AND requested_at IS NULL)') }
+  scope :without_projects, -> { joins('LEFT JOIN project_authorizations ON users.id = project_authorizations.user_id').where(project_authorizations: { user_id: nil }) }
   scope :todo_authors, ->(user_id, state) { where(id: Todo.where(user_id: user_id, state: state).select(:author_id)) }
   scope :order_recent_sign_in, -> { reorder(Gitlab::Database.nulls_last_order('current_sign_in_at', 'DESC')) }
   scope :order_oldest_sign_in, -> { reorder(Gitlab::Database.nulls_last_order('current_sign_in_at', 'ASC')) }