Allow external diffs to be used conditionally

Since external diffs are likely to be a bit slower than in-database
ones, add a mode that makes diffs external after they've been obsoleted
by events. This should strike a balance between performance and disk
space.

A background cron drives the majority of migrations, since diffs become
outdated through user actions.

2 files changed, 45 insertions, 0 deletions

diff --git a/db/migrate/20190222051615_add_indexes_for_merge_request_diffs_query.rb b/db/migrate/20190222051615_add_indexes_for_merge_request_diffs_query.rb
new file mode 100644
index 00000000000..0048268ca6f
--- /dev/null
+++ b/db/migrate/20190222051615_add_indexes_for_merge_request_diffs_query.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+class AddIndexesForMergeRequestDiffsQuery < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  INDEX_SPECS = [
+    [
+      :merge_request_metrics,
+      :latest_closed_at,
+      { where: 'latest_closed_at IS NOT NULL' }
+    ],
+    [
+      :merge_request_metrics,
+      [:merge_request_id, :merged_at],
+      { where: 'merged_at IS NOT NULL' }
+    ],
+    [
+      :merge_request_diffs,
+      [:merge_request_id, :id],
+      {
+        name: 'index_merge_request_diffs_on_merge_request_id_and_id_partial',
+        where: 'NOT stored_externally OR stored_externally IS NULL'
+      }
+    ]
+  ].freeze
+
+  disable_ddl_transaction!
+
+  def up
+    INDEX_SPECS.each do |spec|
+      add_concurrent_index(*spec)
+    end
+  end
+
+  def down
+    INDEX_SPECS.reverse.each do |spec|
+      remove_concurrent_index(*spec)
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 0c997f3b8a2..eed9bb266fe 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -1256,6 +1256,7 @@ ActiveRecord::Schema.define(version: 20190322132835) do
     t.integer "external_diff_store"
     t.boolean "stored_externally"
     t.index ["merge_request_id", "id"], name: "index_merge_request_diffs_on_merge_request_id_and_id", using: :btree
+    t.index ["merge_request_id", "id"], name: "index_merge_request_diffs_on_merge_request_id_and_id_partial", where: "((NOT stored_externally) OR (stored_externally IS NULL))", using: :btree
   end
 
   create_table "merge_request_metrics", force: :cascade do |t|
@@ -1271,7 +1272,9 @@ ActiveRecord::Schema.define(version: 20190322132835) do
     t.integer "latest_closed_by_id"
     t.datetime_with_timezone "latest_closed_at"
     t.index ["first_deployed_to_production_at"], name: "index_merge_request_metrics_on_first_deployed_to_production_at", using: :btree
+    t.index ["latest_closed_at"], name: "index_merge_request_metrics_on_latest_closed_at", where: "(latest_closed_at IS NOT NULL)", using: :btree
     t.index ["latest_closed_by_id"], name: "index_merge_request_metrics_on_latest_closed_by_id", using: :btree
+    t.index ["merge_request_id", "merged_at"], name: "index_merge_request_metrics_on_merge_request_id_and_merged_at", where: "(merged_at IS NOT NULL)", using: :btree
     t.index ["merge_request_id"], name: "index_merge_request_metrics", using: :btree
     t.index ["merged_by_id"], name: "index_merge_request_metrics_on_merged_by_id", using: :btree
     t.index ["pipeline_id"], name: "index_merge_request_metrics_on_pipeline_id", using: :btree