Merge remote-tracking branch 'upstream/master' into 14995-custom_wiki_sidebar

* upstream/master: (4180 commits)
  Enable frozen string literals for app/workers/*.rb
  Resolve "Search dropdown hides & shows when typing"
  Revert merge request widget button max height
  Update CHANGELOG.md for 11.0.2
  Update external link icon in header user dropdown
  Added Diff Viewer to new VUE based MR page
  Fixed eslint failure in IDE spec helpers
  Use refs instead of querySelector.
  Show file in tree on WebIDE open
  Resolve "Remove unused bootstrap component CSS"
  Resolve "Explain what Groups are in the New Group page"
  [QA] Make sure we wait for the deploy key list to load
  Update _scopes_form.html.haml to remove duplicate information
  Use the branch instead of the tag to install
  port the EE changes
  Add index on deployable_type/id for deployments
  Add a helper to rename a column using a background migration
  Fix performance bottleneck when rendering large wiki pages
  Port Namespace#root_ancestor to CE
  Remove duplicate spec
  ...

1 files changed, 27 insertions, 4 deletions

diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb
index c1c27ccf3e5..3d72c447b4b 100644
--- a/app/models/merge_request_diff.rb
+++ b/app/models/merge_request_diff.rb
@@ -3,6 +3,7 @@ class MergeRequestDiff < ActiveRecord::Base
   include Importable
   include ManualInverseAssociation
   include IgnorableColumn
+  include EachBatch
 
   # Don't display more than 100 commits at once
   COMMITS_SAFE_SIZE = 100
@@ -17,8 +18,14 @@ class MergeRequestDiff < ActiveRecord::Base
   has_many :merge_request_diff_commits, -> { order(:merge_request_diff_id, :relative_order) }
 
   state_machine :state, initial: :empty do
+    event :clean do
+      transition any => :without_files
+    end
+
     state :collected
     state :overflow
+    # Diff files have been deleted by the system
+    state :without_files
     # Deprecated states: these are no longer used but these values may still occur
     # in the database.
     state :timeout
@@ -27,6 +34,7 @@ class MergeRequestDiff < ActiveRecord::Base
     state :overflow_diff_lines_limit
   end
 
+  scope :with_files, -> { without_states(:without_files, :empty) }
   scope :viewable, -> { without_state(:empty) }
   scope :by_commit_sha, ->(sha) do
     joins(:merge_request_diff_commits).where(merge_request_diff_commits: { sha: sha }).reorder(nil)
@@ -42,6 +50,10 @@ class MergeRequestDiff < ActiveRecord::Base
     find_by(start_commit_sha: diff_refs.start_sha, head_commit_sha: diff_refs.head_sha, base_commit_sha: diff_refs.base_sha)
   end
 
+  def viewable?
+    collected? || without_files? || overflow?
+  end
+
   # Collect information about commits and diff from repository
   # and save it to the database as serialized data
   def save_git_content
@@ -170,6 +182,21 @@ class MergeRequestDiff < ActiveRecord::Base
   end
 
   def diffs(diff_options = nil)
+    if without_files? && comparison = diff_refs.compare_in(project)
+      # It should fetch the repository when diffs are cleaned by the system.
+      # We don't keep these for storage overload purposes.
+      # See https://gitlab.com/gitlab-org/gitlab-ce/issues/37639
+      comparison.diffs(diff_options)
+    else
+      diffs_collection(diff_options)
+    end
+  end
+
+  # Should always return the DB persisted diffs collection
+  # (e.g. Gitlab::Diff::FileCollection::MergeRequestDiff.
+  # It's useful when trying to invalidate old caches through
+  # FileCollection::MergeRequestDiff#clear_cache!
+  def diffs_collection(diff_options = nil)
     Gitlab::Diff::FileCollection::MergeRequestDiff.new(self, diff_options: diff_options)
   end
 
@@ -197,10 +224,6 @@ class MergeRequestDiff < ActiveRecord::Base
     CompareService.new(project, head_commit_sha).execute(project, sha, straight: true)
   end
 
-  def commits_count
-    super || merge_request_diff_commits.size
-  end
-
   private
 
   def create_merge_request_diff_files(diffs)