summaryrefslogtreecommitdiff
path: root/db
diff options
context:
space:
mode:
authorKamil Trzciński <ayufan@ayufan.eu>2018-02-28 21:28:43 +0100
committerKamil Trzciński <ayufan@ayufan.eu>2018-02-28 21:28:43 +0100
commita2f375e8f74870dcdcfa1c7886bd1c14c80a684e (patch)
tree6b6e3a4f7554f4671edc17d87869dd6916984404 /db
parenta22f6fa6e50bb31921415b01fd345d6802581390 (diff)
parent81852d1f902c2923c239e9c33cab77f5fd6ca8d8 (diff)
downloadgitlab-ce-a2f375e8f74870dcdcfa1c7886bd1c14c80a684e.tar.gz
Merge remote-tracking branch 'origin/master' into object-storage-ee-to-ce-backportobject-storage-ee-to-ce-backport
Diffstat (limited to 'db')
-rw-r--r--db/fixtures/development/01_admin.rb1
-rw-r--r--db/migrate/20170929131201_populate_fork_networks.rb16
-rw-r--r--db/migrate/20180119160751_optimize_ci_job_artifacts.rb23
-rw-r--r--db/migrate/20180208183958_schedule_populate_untracked_uploads_if_needed.rb47
-rw-r--r--db/migrate/20180213131630_add_partial_index_to_projects_for_index_only_scans.rb21
-rw-r--r--db/migrate/20180215181245_users_name_lower_index.rb29
-rw-r--r--db/migrate/20180216120000_add_pages_domain_verification.rb8
-rw-r--r--db/migrate/20180216120010_add_pages_domain_verified_at_index.rb15
-rw-r--r--db/migrate/20180216120020_allow_domain_verification_to_be_disabled.rb7
-rw-r--r--db/migrate/20180216120030_add_pages_domain_enabled_until.rb7
-rw-r--r--db/migrate/20180216120040_add_pages_domain_enabled_until_index.rb17
-rw-r--r--db/migrate/20180216120050_pages_domains_verification_grace_period.rb26
-rw-r--r--db/post_migrate/20170717111152_cleanup_move_system_upload_folder_symlink.rb2
-rw-r--r--db/post_migrate/20171124150326_reschedule_fork_network_creation.rb16
-rw-r--r--db/post_migrate/20180216121020_fill_pages_domain_verification_code.rb41
-rw-r--r--db/post_migrate/20180216121030_enqueue_verify_pages_domain_workers.rb16
-rw-r--r--db/schema.rb12
17 files changed, 271 insertions, 33 deletions
diff --git a/db/fixtures/development/01_admin.rb b/db/fixtures/development/01_admin.rb
index 6f241f6fa4a..dfb50c195c1 100644
--- a/db/fixtures/development/01_admin.rb
+++ b/db/fixtures/development/01_admin.rb
@@ -9,7 +9,6 @@ Gitlab::Seeder.quiet do
s.username = 'root'
s.password = '5iveL!fe'
s.admin = true
- s.projects_limit = 100
s.confirmed_at = DateTime.now
end
end
diff --git a/db/migrate/20170929131201_populate_fork_networks.rb b/db/migrate/20170929131201_populate_fork_networks.rb
index 1214962770f..ddbf27e1852 100644
--- a/db/migrate/20170929131201_populate_fork_networks.rb
+++ b/db/migrate/20170929131201_populate_fork_networks.rb
@@ -6,22 +6,8 @@ class PopulateForkNetworks < ActiveRecord::Migration
DOWNTIME = false
- MIGRATION = 'PopulateForkNetworksRange'.freeze
- BATCH_SIZE = 100
- DELAY_INTERVAL = 15.seconds
-
- disable_ddl_transaction!
-
- class ForkedProjectLink < ActiveRecord::Base
- include EachBatch
-
- self.table_name = 'forked_project_links'
- end
-
def up
- say 'Populating the `fork_networks` based on existing `forked_project_links`'
-
- queue_background_migration_jobs_by_range_at_intervals(ForkedProjectLink, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+ say 'Fork networks will be populated in 20171205190711 - RescheduleForkNetworkCreationCaller'
end
def down
diff --git a/db/migrate/20180119160751_optimize_ci_job_artifacts.rb b/db/migrate/20180119160751_optimize_ci_job_artifacts.rb
new file mode 100644
index 00000000000..9b4340ed7b7
--- /dev/null
+++ b/db/migrate/20180119160751_optimize_ci_job_artifacts.rb
@@ -0,0 +1,23 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class OptimizeCiJobArtifacts < ActiveRecord::Migration
+ include Gitlab::Database::MigrationHelpers
+
+ # Set this constant to true if this migration requires downtime.
+ DOWNTIME = false
+
+ disable_ddl_transaction!
+
+ def up
+ # job_id is just here to be a covering index for index only scans
+ # since we'll almost always be joining against ci_builds on job_id
+ add_concurrent_index(:ci_job_artifacts, [:expire_at, :job_id])
+ add_concurrent_index(:ci_builds, [:artifacts_expire_at], where: "artifacts_file <> ''")
+ end
+
+ def down
+ remove_concurrent_index(:ci_job_artifacts, [:expire_at, :job_id])
+ remove_concurrent_index(:ci_builds, [:artifacts_expire_at], where: "artifacts_file <> ''")
+ end
+end
diff --git a/db/migrate/20180208183958_schedule_populate_untracked_uploads_if_needed.rb b/db/migrate/20180208183958_schedule_populate_untracked_uploads_if_needed.rb
new file mode 100644
index 00000000000..e46e793d9d2
--- /dev/null
+++ b/db/migrate/20180208183958_schedule_populate_untracked_uploads_if_needed.rb
@@ -0,0 +1,47 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class SchedulePopulateUntrackedUploadsIfNeeded < ActiveRecord::Migration
+ include Gitlab::Database::MigrationHelpers
+
+ DOWNTIME = false
+
+ FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze
+
+ class UntrackedFile < ActiveRecord::Base
+ include EachBatch
+
+ self.table_name = 'untracked_files_for_uploads'
+ end
+
+ def up
+ if table_exists?(:untracked_files_for_uploads)
+ process_or_remove_table
+ end
+ end
+
+ def down
+ # nothing
+ end
+
+ private
+
+ def process_or_remove_table
+ if UntrackedFile.all.empty?
+ drop_temp_table
+ else
+ schedule_populate_untracked_uploads_jobs
+ end
+ end
+
+ def drop_temp_table
+ drop_table(:untracked_files_for_uploads, if_exists: true)
+ end
+
+ def schedule_populate_untracked_uploads_jobs
+ say "Scheduling #{FOLLOW_UP_MIGRATION} background migration jobs since there are rows in untracked_files_for_uploads."
+
+ bulk_queue_background_migration_jobs_by_range(
+ UntrackedFile, FOLLOW_UP_MIGRATION)
+ end
+end
diff --git a/db/migrate/20180213131630_add_partial_index_to_projects_for_index_only_scans.rb b/db/migrate/20180213131630_add_partial_index_to_projects_for_index_only_scans.rb
new file mode 100644
index 00000000000..cedf2510dda
--- /dev/null
+++ b/db/migrate/20180213131630_add_partial_index_to_projects_for_index_only_scans.rb
@@ -0,0 +1,21 @@
+class AddPartialIndexToProjectsForIndexOnlyScans < ActiveRecord::Migration
+ include Gitlab::Database::MigrationHelpers
+
+ DOWNTIME = false
+ INDEX_NAME = 'index_projects_on_id_partial_for_visibility'
+
+ disable_ddl_transaction!
+
+ # Adds a partial index to leverage index-only scans when looking up project ids
+ def up
+ unless index_exists?(:projects, :id, name: INDEX_NAME)
+ add_concurrent_index :projects, :id, name: INDEX_NAME, unique: true, where: 'visibility_level IN (10,20)'
+ end
+ end
+
+ def down
+ if index_exists?(:projects, :id, name: INDEX_NAME)
+ remove_concurrent_index_by_name :projects, INDEX_NAME
+ end
+ end
+end
diff --git a/db/migrate/20180215181245_users_name_lower_index.rb b/db/migrate/20180215181245_users_name_lower_index.rb
new file mode 100644
index 00000000000..d3f68cb7d45
--- /dev/null
+++ b/db/migrate/20180215181245_users_name_lower_index.rb
@@ -0,0 +1,29 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class UsersNameLowerIndex < ActiveRecord::Migration
+ include Gitlab::Database::MigrationHelpers
+
+ # Set this constant to true if this migration requires downtime.
+ DOWNTIME = false
+ INDEX_NAME = 'index_on_users_name_lower'
+
+ disable_ddl_transaction!
+
+ def up
+ return unless Gitlab::Database.postgresql?
+
+ # On GitLab.com this produces an index with a size of roughly 60 MB.
+ execute "CREATE INDEX CONCURRENTLY #{INDEX_NAME} ON users (LOWER(name))"
+ end
+
+ def down
+ return unless Gitlab::Database.postgresql?
+
+ if supports_drop_index_concurrently?
+ execute "DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME}"
+ else
+ execute "DROP INDEX IF EXISTS #{INDEX_NAME}"
+ end
+ end
+end
diff --git a/db/migrate/20180216120000_add_pages_domain_verification.rb b/db/migrate/20180216120000_add_pages_domain_verification.rb
new file mode 100644
index 00000000000..8b7cae92285
--- /dev/null
+++ b/db/migrate/20180216120000_add_pages_domain_verification.rb
@@ -0,0 +1,8 @@
+class AddPagesDomainVerification < ActiveRecord::Migration
+ DOWNTIME = false
+
+ def change
+ add_column :pages_domains, :verified_at, :datetime_with_timezone
+ add_column :pages_domains, :verification_code, :string
+ end
+end
diff --git a/db/migrate/20180216120010_add_pages_domain_verified_at_index.rb b/db/migrate/20180216120010_add_pages_domain_verified_at_index.rb
new file mode 100644
index 00000000000..825dfb52dce
--- /dev/null
+++ b/db/migrate/20180216120010_add_pages_domain_verified_at_index.rb
@@ -0,0 +1,15 @@
+class AddPagesDomainVerifiedAtIndex < ActiveRecord::Migration
+ include Gitlab::Database::MigrationHelpers
+
+ DOWNTIME = false
+
+ disable_ddl_transaction!
+
+ def up
+ add_concurrent_index :pages_domains, :verified_at
+ end
+
+ def down
+ remove_concurrent_index :pages_domains, :verified_at
+ end
+end
diff --git a/db/migrate/20180216120020_allow_domain_verification_to_be_disabled.rb b/db/migrate/20180216120020_allow_domain_verification_to_be_disabled.rb
new file mode 100644
index 00000000000..06d458028b3
--- /dev/null
+++ b/db/migrate/20180216120020_allow_domain_verification_to_be_disabled.rb
@@ -0,0 +1,7 @@
+class AllowDomainVerificationToBeDisabled < ActiveRecord::Migration
+ DOWNTIME = false
+
+ def change
+ add_column :application_settings, :pages_domain_verification_enabled, :boolean, default: true, null: false
+ end
+end
diff --git a/db/migrate/20180216120030_add_pages_domain_enabled_until.rb b/db/migrate/20180216120030_add_pages_domain_enabled_until.rb
new file mode 100644
index 00000000000..b40653044dd
--- /dev/null
+++ b/db/migrate/20180216120030_add_pages_domain_enabled_until.rb
@@ -0,0 +1,7 @@
+class AddPagesDomainEnabledUntil < ActiveRecord::Migration
+ DOWNTIME = false
+
+ def change
+ add_column :pages_domains, :enabled_until, :datetime_with_timezone
+ end
+end
diff --git a/db/migrate/20180216120040_add_pages_domain_enabled_until_index.rb b/db/migrate/20180216120040_add_pages_domain_enabled_until_index.rb
new file mode 100644
index 00000000000..00f6e4979da
--- /dev/null
+++ b/db/migrate/20180216120040_add_pages_domain_enabled_until_index.rb
@@ -0,0 +1,17 @@
+class AddPagesDomainEnabledUntilIndex < ActiveRecord::Migration
+ include Gitlab::Database::MigrationHelpers
+
+ DOWNTIME = false
+
+ disable_ddl_transaction!
+
+ def up
+ add_concurrent_index :pages_domains, [:project_id, :enabled_until]
+ add_concurrent_index :pages_domains, [:verified_at, :enabled_until]
+ end
+
+ def down
+ remove_concurrent_index :pages_domains, [:verified_at, :enabled_until]
+ remove_concurrent_index :pages_domains, [:project_id, :enabled_until]
+ end
+end
diff --git a/db/migrate/20180216120050_pages_domains_verification_grace_period.rb b/db/migrate/20180216120050_pages_domains_verification_grace_period.rb
new file mode 100644
index 00000000000..d7f8634b536
--- /dev/null
+++ b/db/migrate/20180216120050_pages_domains_verification_grace_period.rb
@@ -0,0 +1,26 @@
+class PagesDomainsVerificationGracePeriod < ActiveRecord::Migration
+ DOWNTIME = false
+
+ class PagesDomain < ActiveRecord::Base
+ include EachBatch
+ end
+
+ # Allow this migration to resume if it fails partway through
+ disable_ddl_transaction!
+
+ def up
+ now = Time.now
+ grace = now + 30.days
+
+ PagesDomain.each_batch do |relation|
+ relation.update_all(verified_at: now, enabled_until: grace)
+
+ # Sleep 2 minutes between batches to not overload the DB with dead tuples
+ sleep(2.minutes) unless relation.reorder(:id).last == PagesDomain.reorder(:id).last
+ end
+ end
+
+ def down
+ # no-op
+ end
+end
diff --git a/db/post_migrate/20170717111152_cleanup_move_system_upload_folder_symlink.rb b/db/post_migrate/20170717111152_cleanup_move_system_upload_folder_symlink.rb
index 26b99b61424..c48f1c938d0 100644
--- a/db/post_migrate/20170717111152_cleanup_move_system_upload_folder_symlink.rb
+++ b/db/post_migrate/20170717111152_cleanup_move_system_upload_folder_symlink.rb
@@ -20,7 +20,7 @@ class CleanupMoveSystemUploadFolderSymlink < ActiveRecord::Migration
def down
if File.directory?(new_directory)
say "Symlinking #{old_directory} -> #{new_directory}"
- FileUtils.ln_s(new_directory, old_directory)
+ FileUtils.ln_s(new_directory, old_directory) unless File.exist?(old_directory)
else
say "#{new_directory} doesn't exist, skipping."
end
diff --git a/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb b/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb
index 05430efe1f6..26f917d5a1e 100644
--- a/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb
+++ b/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb
@@ -3,22 +3,8 @@ class RescheduleForkNetworkCreation < ActiveRecord::Migration
DOWNTIME = false
- MIGRATION = 'PopulateForkNetworksRange'.freeze
- BATCH_SIZE = 100
- DELAY_INTERVAL = 15.seconds
-
- disable_ddl_transaction!
-
- class ForkedProjectLink < ActiveRecord::Base
- include EachBatch
-
- self.table_name = 'forked_project_links'
- end
-
def up
- say 'Populating the `fork_networks` based on existing `forked_project_links`'
-
- queue_background_migration_jobs_by_range_at_intervals(ForkedProjectLink, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+ say 'Fork networks will be populated in 20171205190711 - RescheduleForkNetworkCreationCaller'
end
def down
diff --git a/db/post_migrate/20180216121020_fill_pages_domain_verification_code.rb b/db/post_migrate/20180216121020_fill_pages_domain_verification_code.rb
new file mode 100644
index 00000000000..d423673d2a5
--- /dev/null
+++ b/db/post_migrate/20180216121020_fill_pages_domain_verification_code.rb
@@ -0,0 +1,41 @@
+class FillPagesDomainVerificationCode < ActiveRecord::Migration
+ DOWNTIME = false
+
+ class PagesDomain < ActiveRecord::Base
+ include EachBatch
+ end
+
+ # Allow this migration to resume if it fails partway through
+ disable_ddl_transaction!
+
+ def up
+ PagesDomain.where(verification_code: [nil, '']).each_batch do |relation|
+ connection.execute(set_codes_sql(relation))
+
+ # Sleep 2 minutes between batches to not overload the DB with dead tuples
+ sleep(2.minutes) unless relation.reorder(:id).last == PagesDomain.reorder(:id).last
+ end
+
+ change_column_null(:pages_domains, :verification_code, false)
+ end
+
+ def down
+ change_column_null(:pages_domains, :verification_code, true)
+ end
+
+ private
+
+ def set_codes_sql(relation)
+ ids = relation.pluck(:id)
+ whens = ids.map { |id| "WHEN #{id} THEN '#{SecureRandom.hex(16)}'" }
+
+ <<~SQL
+ UPDATE pages_domains
+ SET verification_code =
+ CASE id
+ #{whens.join("\n")}
+ END
+ WHERE id IN(#{ids.join(',')})
+ SQL
+ end
+end
diff --git a/db/post_migrate/20180216121030_enqueue_verify_pages_domain_workers.rb b/db/post_migrate/20180216121030_enqueue_verify_pages_domain_workers.rb
new file mode 100644
index 00000000000..bf9bf4e660f
--- /dev/null
+++ b/db/post_migrate/20180216121030_enqueue_verify_pages_domain_workers.rb
@@ -0,0 +1,16 @@
+class EnqueueVerifyPagesDomainWorkers < ActiveRecord::Migration
+ class PagesDomain < ActiveRecord::Base
+ include EachBatch
+ end
+
+ def up
+ PagesDomain.each_batch do |relation|
+ ids = relation.pluck(:id).map { |id| [id] }
+ PagesDomainVerificationWorker.bulk_perform_async(ids)
+ end
+ end
+
+ def down
+ # no-op
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index e381fc9fb8a..fba0c710705 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 20180206200543) do
+ActiveRecord::Schema.define(version: 20180216121030) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
@@ -156,6 +156,7 @@ ActiveRecord::Schema.define(version: 20180206200543) do
t.integer "gitaly_timeout_fast", default: 10, null: false
t.boolean "authorized_keys_enabled", default: true, null: false
t.string "auto_devops_domain"
+ t.boolean "pages_domain_verification_enabled", default: true, null: false
end
create_table "audit_events", force: :cascade do |t|
@@ -295,6 +296,7 @@ ActiveRecord::Schema.define(version: 20180206200543) do
t.integer "failure_reason"
end
+ add_index "ci_builds", ["artifacts_expire_at"], name: "index_ci_builds_on_artifacts_expire_at", where: "(artifacts_file <> ''::text)", using: :btree
add_index "ci_builds", ["auto_canceled_by_id"], name: "index_ci_builds_on_auto_canceled_by_id", using: :btree
add_index "ci_builds", ["commit_id", "stage_idx", "created_at"], name: "index_ci_builds_on_commit_id_and_stage_idx_and_created_at", using: :btree
add_index "ci_builds", ["commit_id", "status", "type"], name: "index_ci_builds_on_commit_id_and_status_and_type", using: :btree
@@ -336,6 +338,7 @@ ActiveRecord::Schema.define(version: 20180206200543) do
t.string "file"
end
+ add_index "ci_job_artifacts", ["expire_at", "job_id"], name: "index_ci_job_artifacts_on_expire_at_and_job_id", using: :btree
add_index "ci_job_artifacts", ["job_id", "file_type"], name: "index_ci_job_artifacts_on_job_id_and_file_type", unique: true, using: :btree
add_index "ci_job_artifacts", ["project_id"], name: "index_ci_job_artifacts_on_project_id", using: :btree
@@ -1315,10 +1318,16 @@ ActiveRecord::Schema.define(version: 20180206200543) do
t.string "encrypted_key_iv"
t.string "encrypted_key_salt"
t.string "domain"
+ t.datetime_with_timezone "verified_at"
+ t.string "verification_code", null: false
+ t.datetime_with_timezone "enabled_until"
end
add_index "pages_domains", ["domain"], name: "index_pages_domains_on_domain", unique: true, using: :btree
+ add_index "pages_domains", ["project_id", "enabled_until"], name: "index_pages_domains_on_project_id_and_enabled_until", using: :btree
add_index "pages_domains", ["project_id"], name: "index_pages_domains_on_project_id", using: :btree
+ add_index "pages_domains", ["verified_at", "enabled_until"], name: "index_pages_domains_on_verified_at_and_enabled_until", using: :btree
+ add_index "pages_domains", ["verified_at"], name: "index_pages_domains_on_verified_at", using: :btree
create_table "personal_access_tokens", force: :cascade do |t|
t.integer "user_id", null: false
@@ -1470,6 +1479,7 @@ ActiveRecord::Schema.define(version: 20180206200543) do
add_index "projects", ["created_at"], name: "index_projects_on_created_at", using: :btree
add_index "projects", ["creator_id"], name: "index_projects_on_creator_id", using: :btree
add_index "projects", ["description"], name: "index_projects_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"}
+ add_index "projects", ["id"], name: "index_projects_on_id_partial_for_visibility", unique: true, where: "(visibility_level = ANY (ARRAY[10, 20]))", using: :btree
add_index "projects", ["last_activity_at"], name: "index_projects_on_last_activity_at", using: :btree
add_index "projects", ["last_repository_check_failed"], name: "index_projects_on_last_repository_check_failed", using: :btree
add_index "projects", ["last_repository_updated_at"], name: "index_projects_on_last_repository_updated_at", using: :btree