Add table for merge request commits

This is an ID-less table with just three columns: an association to the merge
request diff the commit belongs to, the relative order of the commit within the
merge request diff, and the commit SHA itself.

Previously we stored much more information about the commits, so that we could
display them even when they were deleted from the repo. Since 8.0, we ensure
that those commits are kept around for as long as the target repo itself is, so
we don't need to duplicate that data in the database.

5 files changed, 79 insertions, 72 deletions

diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 48586ba8910..ba2ecbf82a2 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -218,7 +218,7 @@ module Ci
             .reorder(iid: :desc)
 
           merge_requests.find do |merge_request|
-            merge_request.commits_sha.include?(pipeline.sha)
+            merge_request.commit_shas.include?(pipeline.sha)
           end
         end
     end
diff --git a/app/models/commit.rb b/app/models/commit.rb
index 20206d57c4c..c7f62617c4c 100644
--- a/app/models/commit.rb
+++ b/app/models/commit.rb
@@ -138,7 +138,7 @@ class Commit
 
     safe_message.split("\n", 2)[1].try(:chomp)
   end
-  
+
   def description?
     description.present?
   end
diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb
index 2752181ed79..2fc6191e785 100644
--- a/app/models/merge_request.rb
+++ b/app/models/merge_request.rb
@@ -31,7 +31,7 @@ class MergeRequest < ActiveRecord::Base
   after_create :ensure_merge_request_diff, unless: :importing?
   after_update :reload_diff_if_branch_changed
 
-  delegate :commits, :real_size, :commits_sha, :commits_count,
+  delegate :commits, :real_size, :commit_shas, :commits_count,
     to: :merge_request_diff, prefix: nil
 
   # When this attribute is true some MR validation is ignored
@@ -518,7 +518,7 @@ class MergeRequest < ActiveRecord::Base
   def related_notes
     # Fetch comments only from last 100 commits
     commits_for_notes_limit = 100
-    commit_ids = commits.last(commits_for_notes_limit).map(&:id)
+    commit_ids = commit_shas.take(commits_for_notes_limit)
 
     Note.where(
       "(project_id = :target_project_id AND noteable_type = 'MergeRequest' AND noteable_id = :mr_id) OR" +
@@ -841,15 +841,15 @@ class MergeRequest < ActiveRecord::Base
     return Ci::Pipeline.none unless source_project
 
     @all_pipelines ||= source_project.pipelines
-      .where(sha: all_commits_sha, ref: source_branch)
+      .where(sha: all_commit_shas, ref: source_branch)
       .order(id: :desc)
   end
 
   # Note that this could also return SHA from now dangling commits
   #
-  def all_commits_sha
+  def all_commit_shas
     if persisted?
-      merge_request_diffs.flat_map(&:commits_sha).uniq
+      merge_request_diffs.preload(:merge_request_diff_commits).flat_map(&:commit_shas).uniq
     elsif compare_commits
       compare_commits.to_a.reverse.map(&:id)
     else
diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb
index 0cbf58a2325..4b141945ab4 100644
--- a/app/models/merge_request_diff.rb
+++ b/app/models/merge_request_diff.rb
@@ -11,6 +11,7 @@ class MergeRequestDiff < ActiveRecord::Base
 
   belongs_to :merge_request
   has_many :merge_request_diff_files, -> { order(:merge_request_diff_id, :relative_order) }
+  has_many :merge_request_diff_commits, -> { order(:merge_request_diff_id, :relative_order) }
 
   serialize :st_commits # rubocop:disable Cop/ActiveRecordSerialize
   serialize :st_diffs # rubocop:disable Cop/ActiveRecordSerialize
@@ -47,14 +48,13 @@ class MergeRequestDiff < ActiveRecord::Base
   # Collect information about commits and diff from repository
   # and save it to the database as serialized data
   def save_git_content
-    ensure_commits_sha
+    ensure_commit_shas
     save_commits
-    reload_commits
     save_diffs
     keep_around_commits
   end
 
-  def ensure_commits_sha
+  def ensure_commit_shas
     merge_request.fetch_ref
     self.start_commit_sha ||= merge_request.target_branch_sha
     self.head_commit_sha  ||= merge_request.source_branch_sha
@@ -66,7 +66,7 @@ class MergeRequestDiff < ActiveRecord::Base
   # created before version 8.4 that does not store head_commit_sha in separate db field.
   def head_commit_sha
     if persisted? && super.nil?
-      last_commit.try(:sha)
+      last_commit_sha
     else
       super
     end
@@ -97,16 +97,11 @@ class MergeRequestDiff < ActiveRecord::Base
   end
 
   def commits
-    @commits ||= load_commits(st_commits)
+    @commits ||= load_commits
   end
 
-  def reload_commits
-    @commits = nil
-    commits
-  end
-
-  def last_commit
-    commits.first
+  def last_commit_sha
+    commit_shas.first
   end
 
   def first_commit
@@ -131,8 +126,12 @@ class MergeRequestDiff < ActiveRecord::Base
     project.commit(head_commit_sha)
   end
 
-  def commits_sha
-    st_commits.map { |commit| commit[:id] }
+  def commit_shas
+    if st_commits.present?
+      st_commits.map { |commit| commit[:id] }
+    else
+      merge_request_diff_commits.map(&:sha)
+    end
   end
 
   def diff_refs=(new_diff_refs)
@@ -207,7 +206,11 @@ class MergeRequestDiff < ActiveRecord::Base
   end
 
   def commits_count
-    st_commits.count
+    if st_commits.present?
+      st_commits.size
+    else
+      merge_request_diff_commits.size
+    end
   end
 
   def utf8_st_diffs
@@ -231,29 +234,6 @@ class MergeRequestDiff < ActiveRecord::Base
     raw.any? { |element| VALID_CLASSES.include?(element.class) }
   end
 
-  def dump_commits(commits)
-    commits.map(&:to_hash)
-  end
-
-  def load_commits(array)
-    array.map { |hash| Commit.new(Gitlab::Git::Commit.new(hash), merge_request.source_project) }
-  end
-
-  # Load all commits related to current merge request diff from repo
-  # and save it as array of hashes in st_commits db field
-  def save_commits
-    new_attributes = {}
-
-    commits = compare.commits
-
-    if commits.present?
-      commits = Commit.decorate(commits, merge_request.source_project).reverse
-      new_attributes[:st_commits] = dump_commits(commits)
-    end
-
-    update_columns_serialized(new_attributes)
-  end
-
   def create_merge_request_diff_files(diffs)
     rows = diffs.map.with_index do |diff, index|
       diff.to_hash.merge(
@@ -294,12 +274,18 @@ class MergeRequestDiff < ActiveRecord::Base
       end
   end
 
-  # Load diffs between branches related to current merge request diff from repo
-  # and save it as array of hashes in st_diffs db field
+  def load_commits
+    commits = st_commits.presence || merge_request_diff_commits
+
+    commits.map do |commit|
+      Commit.new(Gitlab::Git::Commit.new(commit.to_hash), merge_request.source_project)
+    end
+  end
+
   def save_diffs
     new_attributes = {}
 
-    if commits.size.zero?
+    if compare.commits.size.zero?
       new_attributes[:state] = :empty
     else
       diff_collection = compare.diffs(Commit.max_diff_options)
@@ -319,7 +305,13 @@ class MergeRequestDiff < ActiveRecord::Base
       new_attributes[:state] = :overflow if diff_collection.overflow?
     end
 
-    update_columns_serialized(new_attributes)
+    update(new_attributes)
+  end
+
+  def save_commits
+    MergeRequestDiffCommit.create_bulk(self.id, compare.commits.reverse)
+
+    merge_request_diff_commits.reload
   end
 
   def repository
@@ -332,29 +324,6 @@ class MergeRequestDiff < ActiveRecord::Base
     project.merge_base_commit(head_commit_sha, start_commit_sha).try(:sha)
   end
 
-  #
-  # #save or #update_attributes providing changes on serialized attributes do a lot of
-  # serialization and deserialization calls resulting in bad performance.
-  # Using #update_columns solves the problem with just one YAML.dump per serialized attribute that we provide.
-  # As a tradeoff we need to reload the current instance to properly manage time objects on those serialized
-  # attributes. So to keep the same behaviour as the attribute assignment we reload the instance.
-  # The difference is in the usage of
-  # #write_attribute= (#update_attributes) and #raw_write_attribute= (#update_columns)
-  #
-  # Ex:
-  #
-  #   new_attributes[:st_commits].first.slice(:committed_date)
-  #   => {:committed_date=>2014-02-27 11:01:38 +0200}
-  #   YAML.load(YAML.dump(new_attributes[:st_commits].first.slice(:committed_date)))
-  #   => {:committed_date=>2014-02-27 10:01:38 +0100}
-  #
-  def update_columns_serialized(new_attributes)
-    return unless new_attributes.any?
-
-    update_columns(new_attributes.merge(updated_at: current_time_from_proper_timezone))
-    reload
-  end
-
   def keep_around_commits
     [repository, merge_request.source_project.repository].each do |repo|
       repo.keep_around(start_commit_sha)
diff --git a/app/models/merge_request_diff_commit.rb b/app/models/merge_request_diff_commit.rb
new file mode 100644
index 00000000000..cafdbe11849
--- /dev/null
+++ b/app/models/merge_request_diff_commit.rb
@@ -0,0 +1,38 @@
+class MergeRequestDiffCommit < ActiveRecord::Base
+  include ShaAttribute
+
+  belongs_to :merge_request_diff
+
+  sha_attribute :sha
+  alias_attribute :id, :sha
+
+  def self.create_bulk(merge_request_diff_id, commits)
+    sha_attribute = Gitlab::Database::ShaAttribute.new
+
+    rows = commits.map.with_index do |commit, index|
+      # See #parent_ids.
+      commit_hash = commit.to_hash.except(:parent_ids)
+      sha = commit_hash.delete(:id)
+
+      commit_hash.merge(
+        merge_request_diff_id: merge_request_diff_id,
+        relative_order: index,
+        sha: sha_attribute.type_cast_for_database(sha)
+      )
+    end
+
+    Gitlab::Database.bulk_insert(self.table_name, rows)
+  end
+
+  def to_hash
+    Gitlab::Git::Commit::SERIALIZE_KEYS.each_with_object({}) do |key, hash|
+      hash[key] = public_send(key)
+    end
+  end
+
+  # We don't save these, because they would need a table or a serialised
+  # field. They aren't used anywhere, so just pretend the commit has no parents.
+  def parent_ids
+    []
+  end
+end