Merge branch 'master' into 'backstage/gb/build-stages-catch-up-migration'

 Conflicts:
   db/schema.rb

5 files changed, 73 insertions, 2 deletions

diff --git a/db/fixtures/development/01_admin.rb b/db/fixtures/development/01_admin.rb
index 6f241f6fa4a..dfb50c195c1 100644
--- a/db/fixtures/development/01_admin.rb
+++ b/db/fixtures/development/01_admin.rb
@@ -9,7 +9,6 @@ Gitlab::Seeder.quiet do
     s.username = 'root'
     s.password = '5iveL!fe'
     s.admin = true
-    s.projects_limit = 100
     s.confirmed_at = DateTime.now
   end
 end
diff --git a/db/migrate/20180119160751_optimize_ci_job_artifacts.rb b/db/migrate/20180119160751_optimize_ci_job_artifacts.rb
new file mode 100644
index 00000000000..9b4340ed7b7
--- /dev/null
+++ b/db/migrate/20180119160751_optimize_ci_job_artifacts.rb
@@ -0,0 +1,23 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class OptimizeCiJobArtifacts < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    # job_id is just here to be a covering index for index only scans
+    # since we'll almost always be joining against ci_builds on job_id
+    add_concurrent_index(:ci_job_artifacts, [:expire_at, :job_id])
+    add_concurrent_index(:ci_builds, [:artifacts_expire_at], where: "artifacts_file <> ''")
+  end
+
+  def down
+    remove_concurrent_index(:ci_job_artifacts, [:expire_at, :job_id])
+    remove_concurrent_index(:ci_builds, [:artifacts_expire_at], where: "artifacts_file <> ''")
+  end
+end
diff --git a/db/migrate/20180208183958_schedule_populate_untracked_uploads_if_needed.rb b/db/migrate/20180208183958_schedule_populate_untracked_uploads_if_needed.rb
new file mode 100644
index 00000000000..e46e793d9d2
--- /dev/null
+++ b/db/migrate/20180208183958_schedule_populate_untracked_uploads_if_needed.rb
@@ -0,0 +1,47 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class SchedulePopulateUntrackedUploadsIfNeeded < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze
+
+  class UntrackedFile < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'untracked_files_for_uploads'
+  end
+
+  def up
+    if table_exists?(:untracked_files_for_uploads)
+      process_or_remove_table
+    end
+  end
+
+  def down
+    # nothing
+  end
+
+  private
+
+  def process_or_remove_table
+    if UntrackedFile.all.empty?
+      drop_temp_table
+    else
+      schedule_populate_untracked_uploads_jobs
+    end
+  end
+
+  def drop_temp_table
+    drop_table(:untracked_files_for_uploads, if_exists: true)
+  end
+
+  def schedule_populate_untracked_uploads_jobs
+    say "Scheduling #{FOLLOW_UP_MIGRATION} background migration jobs since there are rows in untracked_files_for_uploads."
+
+    bulk_queue_background_migration_jobs_by_range(
+      UntrackedFile, FOLLOW_UP_MIGRATION)
+  end
+end
diff --git a/db/post_migrate/20170717111152_cleanup_move_system_upload_folder_symlink.rb b/db/post_migrate/20170717111152_cleanup_move_system_upload_folder_symlink.rb
index 26b99b61424..c48f1c938d0 100644
--- a/db/post_migrate/20170717111152_cleanup_move_system_upload_folder_symlink.rb
+++ b/db/post_migrate/20170717111152_cleanup_move_system_upload_folder_symlink.rb
@@ -20,7 +20,7 @@ class CleanupMoveSystemUploadFolderSymlink < ActiveRecord::Migration
   def down
     if File.directory?(new_directory)
       say "Symlinking #{old_directory} -> #{new_directory}"
-      FileUtils.ln_s(new_directory, old_directory)
+      FileUtils.ln_s(new_directory, old_directory) unless File.exist?(old_directory)
     else
       say "#{new_directory} doesn't exist, skipping."
     end
diff --git a/db/schema.rb b/db/schema.rb
index e3474a4995f..b57094818e2 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -293,6 +293,7 @@ ActiveRecord::Schema.define(version: 20180212101928) do
     t.integer "failure_reason"
   end
 
+  add_index "ci_builds", ["artifacts_expire_at"], name: "index_ci_builds_on_artifacts_expire_at", where: "(artifacts_file <> ''::text)", using: :btree
   add_index "ci_builds", ["auto_canceled_by_id"], name: "index_ci_builds_on_auto_canceled_by_id", using: :btree
   add_index "ci_builds", ["commit_id", "stage_idx", "created_at"], name: "index_ci_builds_on_commit_id_and_stage_idx_and_created_at", using: :btree
   add_index "ci_builds", ["commit_id", "status", "type"], name: "index_ci_builds_on_commit_id_and_status_and_type", using: :btree
@@ -333,6 +334,7 @@ ActiveRecord::Schema.define(version: 20180212101928) do
     t.string "file"
   end
 
+  add_index "ci_job_artifacts", ["expire_at", "job_id"], name: "index_ci_job_artifacts_on_expire_at_and_job_id", using: :btree
   add_index "ci_job_artifacts", ["job_id", "file_type"], name: "index_ci_job_artifacts_on_job_id_and_file_type", unique: true, using: :btree
   add_index "ci_job_artifacts", ["project_id"], name: "index_ci_job_artifacts_on_project_id", using: :btree