diff options
Diffstat (limited to 'db')
12 files changed, 302 insertions, 35 deletions
diff --git a/db/fixtures/development/06_teams.rb b/db/fixtures/development/06_teams.rb index 8afb0dde077..b218f4e71fd 100644 --- a/db/fixtures/development/06_teams.rb +++ b/db/fixtures/development/06_teams.rb @@ -14,7 +14,7 @@ Sidekiq::Testing.inline! do Project.all.each do |project| User.all.sample(4).each do |user| - if project.add_role(user, Gitlab::Access.values.sample) + if project.add_role(user, Gitlab::Access.sym_options.keys.sample) print '.' else print 'F' diff --git a/db/migrate/20171019141859_fix_dev_timezone_schema.rb b/db/migrate/20171019141859_fix_dev_timezone_schema.rb new file mode 100644 index 00000000000..fb7c17dd747 --- /dev/null +++ b/db/migrate/20171019141859_fix_dev_timezone_schema.rb @@ -0,0 +1,25 @@ +class FixDevTimezoneSchema < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # The this migrations tries to help solve unwanted changes to `schema.rb` + # while developing GitLab. Installations created before we started using + # `datetime_with_timezone` are likely to face this problem. Updating those + # columns to the new type should help fix this. + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + TIMEZONE_TABLES = %i(appearances ci_group_variables ci_pipeline_schedule_variables events gpg_keys gpg_signatures project_auto_devops) + + def up + return unless Rails.env.development? || Rails.env.test? + + TIMEZONE_TABLES.each do |table| + change_column table, :created_at, :datetime_with_timezone + change_column table, :updated_at, :datetime_with_timezone + end + end + + def down + end +end diff --git a/db/migrate/20171106151218_issues_moved_to_id_foreign_key.rb b/db/migrate/20171106151218_issues_moved_to_id_foreign_key.rb index 8d2ceb8cc18..6395462384b 100644 --- a/db/migrate/20171106151218_issues_moved_to_id_foreign_key.rb +++ b/db/migrate/20171106151218_issues_moved_to_id_foreign_key.rb @@ -15,8 +15,20 @@ class IssuesMovedToIdForeignKey < ActiveRecord::Migration self.table_name = 'issues' def self.with_orphaned_moved_to_issues - where('NOT EXISTS (SELECT true FROM issues WHERE issues.id = issues.moved_to_id)') - .where('moved_to_id IS NOT NULL') + if Gitlab::Database.postgresql? + # Be careful to use a second table here for comparison otherwise we'll null + # out all rows that don't have id == moved.to_id! + where('NOT EXISTS (SELECT true FROM issues B WHERE issues.moved_to_id = B.id)') + .where('moved_to_id IS NOT NULL') + else + # MySQL doesn't allow modification of the same table in a subquery, + # and using a temporary table isn't automatically guaranteed to work + # due to the MySQL query optimizer. See + # https://dev.mysql.com/doc/refman/5.7/en/update.html for more + # details. + joins('LEFT JOIN issues AS b ON issues.moved_to_id = b.id') + .where('issues.moved_to_id IS NOT NULL AND b.id IS NULL') + end end end diff --git a/db/migrate/20171127151038_add_events_related_columns_to_merge_request_metrics.rb b/db/migrate/20171127151038_add_events_related_columns_to_merge_request_metrics.rb new file mode 100644 index 00000000000..18af697cf88 --- /dev/null +++ b/db/migrate/20171127151038_add_events_related_columns_to_merge_request_metrics.rb @@ -0,0 +1,37 @@ +class AddEventsRelatedColumnsToMergeRequestMetrics < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + change_table :merge_request_metrics do |t| + t.references :merged_by, references: :users + t.references :latest_closed_by, references: :users + end + + add_column :merge_request_metrics, :latest_closed_at, :datetime_with_timezone + + add_concurrent_foreign_key :merge_request_metrics, :users, + column: :merged_by_id, + on_delete: :nullify + + add_concurrent_foreign_key :merge_request_metrics, :users, + column: :latest_closed_by_id, + on_delete: :nullify + end + + def down + if foreign_keys_for(:merge_request_metrics, :merged_by_id).any? + remove_foreign_key :merge_request_metrics, column: :merged_by_id + end + + if foreign_keys_for(:merge_request_metrics, :latest_closed_by_id).any? + remove_foreign_key :merge_request_metrics, column: :latest_closed_by_id + end + + remove_columns :merge_request_metrics, + :merged_by_id, :latest_closed_by_id, :latest_closed_at + end +end diff --git a/db/migrate/20171216111734_clean_up_for_members.rb b/db/migrate/20171216111734_clean_up_for_members.rb new file mode 100644 index 00000000000..22e0997dce6 --- /dev/null +++ b/db/migrate/20171216111734_clean_up_for_members.rb @@ -0,0 +1,31 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class CleanUpForMembers < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + disable_ddl_transaction! + + class Member < ActiveRecord::Base + include EachBatch + + self.table_name = 'members' + end + + def up + condition = <<~EOF.squish + invite_token IS NULL AND + NOT EXISTS (SELECT 1 FROM users WHERE users.id = members.user_id) + EOF + + Member.each_batch(of: 10_000) do |batch| + batch.where(condition).delete_all + end + end + + def down + end +end diff --git a/db/migrate/20171216112339_add_foreign_key_for_members.rb b/db/migrate/20171216112339_add_foreign_key_for_members.rb new file mode 100644 index 00000000000..be17769be6a --- /dev/null +++ b/db/migrate/20171216112339_add_foreign_key_for_members.rb @@ -0,0 +1,21 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddForeignKeyForMembers < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_foreign_key(:members, + :users, + column: :user_id) + end + + def down + remove_foreign_key(:members, column: :user_id) + end +end diff --git a/db/migrate/20171229225929_change_user_project_limit_not_null_and_remove_default.rb b/db/migrate/20171229225929_change_user_project_limit_not_null_and_remove_default.rb new file mode 100644 index 00000000000..54fbbcf1a0d --- /dev/null +++ b/db/migrate/20171229225929_change_user_project_limit_not_null_and_remove_default.rb @@ -0,0 +1,38 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class ChangeUserProjectLimitNotNullAndRemoveDefault < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + # When a migration requires downtime you **must** uncomment the following + # constant and define a short and easy to understand explanation as to why the + # migration requires downtime. + # DOWNTIME_REASON = '' + + # When using the methods "add_concurrent_index", "remove_concurrent_index" or + # "add_column_with_default" you must disable the use of transactions + # as these methods can not run in an existing transaction. + # When using "add_concurrent_index" or "remove_concurrent_index" methods make sure + # that either of them is the _only_ method called in the migration, + # any other changes should go in a separate migration. + # This ensures that upon failure _only_ the index creation or removing fails + # and can be retried or reverted easily. + # + # To disable transactions uncomment the following line and remove these + # comments: + # disable_ddl_transaction! + + def up + # Set Users#projects_limit to NOT NULL and remove the default value + change_column_null :users, :projects_limit, false + change_column_default :users, :projects_limit, nil + end + + def down + change_column_null :users, :projects_limit, true + change_column_default :users, :projects_limit, 10 + end +end diff --git a/db/migrate/20171230123729_add_rebase_commit_sha_to_merge_requests.rb b/db/migrate/20171230123729_add_rebase_commit_sha_to_merge_requests.rb new file mode 100644 index 00000000000..2ce156fa92e --- /dev/null +++ b/db/migrate/20171230123729_add_rebase_commit_sha_to_merge_requests.rb @@ -0,0 +1,7 @@ +class AddRebaseCommitShaToMergeRequests < ActiveRecord::Migration + DOWNTIME = false + + def change + add_column :merge_requests, :rebase_commit_sha, :string + end +end diff --git a/db/post_migrate/20170907170235_delete_conflicting_redirect_routes.rb b/db/post_migrate/20170907170235_delete_conflicting_redirect_routes.rb index 3e84b295be4..033019c398e 100644 --- a/db/post_migrate/20170907170235_delete_conflicting_redirect_routes.rb +++ b/db/post_migrate/20170907170235_delete_conflicting_redirect_routes.rb @@ -2,36 +2,12 @@ # for more information on how to write migrations for GitLab. class DeleteConflictingRedirectRoutes < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - DOWNTIME = false - MIGRATION = 'DeleteConflictingRedirectRoutesRange'.freeze - BATCH_SIZE = 200 # At 200, I expect under 20s per batch, which is under our query timeout of 60s. - DELAY_INTERVAL = 12.seconds - - disable_ddl_transaction! - - class Route < ActiveRecord::Base - include EachBatch - - self.table_name = 'routes' - end - def up - say opening_message - - queue_background_migration_jobs_by_range_at_intervals(Route, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE) + # No-op. + # See https://gitlab.com/gitlab-com/infrastructure/issues/3460#note_53223252 end def down # nothing end - - def opening_message - <<~MSG - Clean up redirect routes that conflict with regular routes. - See initial bug fix: - https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/13357 - MSG - end end diff --git a/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb b/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb new file mode 100644 index 00000000000..547cc68e10e --- /dev/null +++ b/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true +# rubocop:disable GitlabSecurity/SqlInjection + +class SchedulePopulateMergeRequestMetricsWithEventsData < ActiveRecord::Migration + DOWNTIME = false + BATCH_SIZE = 10_000 + MIGRATION = 'PopulateMergeRequestMetricsWithEventsData' + + disable_ddl_transaction! + + class MergeRequest < ActiveRecord::Base + self.table_name = 'merge_requests' + + include ::EachBatch + end + + def up + merge_requests = MergeRequest.where("id IN (#{updatable_merge_requests_union_sql})").reorder(:id) + + say 'Scheduling `PopulateMergeRequestMetricsWithEventsData` jobs' + # It will update around 4_000_000 records in batches of 10_000 merge + # requests (running between 10 minutes) and should take around 66 hours to complete. + # Apparently, production PostgreSQL is able to vacuum 10k-20k dead_tuples by + # minute, and at maximum, each of these jobs should UPDATE 20k records. + # + # More information about the updates in `PopulateMergeRequestMetricsWithEventsData` class. + # + merge_requests.each_batch(of: BATCH_SIZE) do |relation, index| + range = relation.pluck('MIN(id)', 'MAX(id)').first + + BackgroundMigrationWorker.perform_in(index * 10.minutes, MIGRATION, range) + end + end + + def down + execute "update merge_request_metrics set latest_closed_at = null" + execute "update merge_request_metrics set latest_closed_by_id = null" + execute "update merge_request_metrics set merged_by_id = null" + end + + private + + # On staging: + # Planning time: 0.682 ms + # Execution time: 22033.158 ms + # + def updatable_merge_requests_union_sql + metrics_not_exists_clause = + 'NOT EXISTS (SELECT 1 FROM merge_request_metrics WHERE merge_request_metrics.merge_request_id = merge_requests.id)' + + without_metrics_data = <<-SQL.strip_heredoc + merge_request_metrics.merged_by_id IS NULL OR + merge_request_metrics.latest_closed_by_id IS NULL OR + merge_request_metrics.latest_closed_at IS NULL + SQL + + mrs_without_metrics_record = MergeRequest + .where(metrics_not_exists_clause) + .select(:id) + + mrs_without_events_data = MergeRequest + .joins('INNER JOIN merge_request_metrics ON merge_requests.id = merge_request_metrics.merge_request_id') + .where(without_metrics_data) + .select(:id) + + Gitlab::SQL::Union.new([mrs_without_metrics_record, mrs_without_events_data]).to_sql + end +end diff --git a/db/post_migrate/20171221140220_schedule_issues_closed_at_type_change.rb b/db/post_migrate/20171221140220_schedule_issues_closed_at_type_change.rb new file mode 100644 index 00000000000..eeecc7b1de0 --- /dev/null +++ b/db/post_migrate/20171221140220_schedule_issues_closed_at_type_change.rb @@ -0,0 +1,45 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. +# rubocop:disable Migration/Datetime +class ScheduleIssuesClosedAtTypeChange < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + class Issue < ActiveRecord::Base + self.table_name = 'issues' + include EachBatch + + def self.to_migrate + where('closed_at IS NOT NULL') + end + end + + def up + return unless migrate_column_type? + + change_column_type_using_background_migration( + Issue.to_migrate, + :closed_at, + :datetime_with_timezone + ) + end + + def down + return if migrate_column_type? + + change_column_type_using_background_migration( + Issue.to_migrate, + :closed_at, + :datetime + ) + end + + def migrate_column_type? + # Some environments may have already executed the previous version of this + # migration, thus we don't need to migrate those environments again. + column_for('issues', 'closed_at').type == :datetime + end +end diff --git a/db/schema.rb b/db/schema.rb index aa5db5da4f0..740e80ccfd4 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20171220191323) do +ActiveRecord::Schema.define(version: 20171230123729) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -657,8 +657,8 @@ ActiveRecord::Schema.define(version: 20171220191323) do t.datetime "created_at" t.datetime "updated_at" t.string "confirmation_token" - t.datetime "confirmed_at" - t.datetime "confirmation_sent_at" + t.datetime_with_timezone "confirmed_at" + t.datetime_with_timezone "confirmation_sent_at" end add_index "emails", ["confirmation_token"], name: "index_emails_on_confirmation_token", unique: true, using: :btree @@ -1056,6 +1056,9 @@ ActiveRecord::Schema.define(version: 20171220191323) do t.datetime "created_at", null: false t.datetime "updated_at", null: false t.integer "pipeline_id" + t.integer "merged_by_id" + t.integer "latest_closed_by_id" + t.datetime_with_timezone "latest_closed_at" end add_index "merge_request_metrics", ["first_deployed_to_production_at"], name: "index_merge_request_metrics_on_first_deployed_to_production_at", using: :btree @@ -1096,6 +1099,7 @@ ActiveRecord::Schema.define(version: 20171220191323) do t.string "merge_jid" t.boolean "discussion_locked" t.integer "latest_merge_request_diff_id" + t.string "rebase_commit_sha" end add_index "merge_requests", ["assignee_id"], name: "index_merge_requests_on_assignee_id", using: :btree @@ -1770,8 +1774,8 @@ ActiveRecord::Schema.define(version: 20171220191323) do add_index "user_agent_details", ["subject_id", "subject_type"], name: "index_user_agent_details_on_subject_id_and_subject_type", using: :btree create_table "user_custom_attributes", force: :cascade do |t| - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime_with_timezone "created_at", null: false + t.datetime_with_timezone "updated_at", null: false t.integer "user_id", null: false t.string "key", null: false t.string "value", null: false @@ -1805,7 +1809,7 @@ ActiveRecord::Schema.define(version: 20171220191323) do t.datetime "updated_at" t.string "name" t.boolean "admin", default: false, null: false - t.integer "projects_limit", default: 10 + t.integer "projects_limit", null: false t.string "skype", default: "", null: false t.string "linkedin", default: "", null: false t.string "twitter", default: "", null: false @@ -1989,11 +1993,14 @@ ActiveRecord::Schema.define(version: 20171220191323) do add_foreign_key "labels", "projects", name: "fk_7de4989a69", on_delete: :cascade add_foreign_key "lists", "boards", name: "fk_0d3f677137", on_delete: :cascade add_foreign_key "lists", "labels", name: "fk_7a5553d60f", on_delete: :cascade + add_foreign_key "members", "users", name: "fk_2e88fb7ce9", on_delete: :cascade add_foreign_key "merge_request_diff_commits", "merge_request_diffs", on_delete: :cascade add_foreign_key "merge_request_diff_files", "merge_request_diffs", on_delete: :cascade add_foreign_key "merge_request_diffs", "merge_requests", name: "fk_8483f3258f", on_delete: :cascade add_foreign_key "merge_request_metrics", "ci_pipelines", column: "pipeline_id", on_delete: :cascade add_foreign_key "merge_request_metrics", "merge_requests", on_delete: :cascade + add_foreign_key "merge_request_metrics", "users", column: "latest_closed_by_id", name: "fk_ae440388cc", on_delete: :nullify + add_foreign_key "merge_request_metrics", "users", column: "merged_by_id", name: "fk_7f28d925f3", on_delete: :nullify add_foreign_key "merge_requests", "ci_pipelines", column: "head_pipeline_id", name: "fk_fd82eae0b9", on_delete: :nullify add_foreign_key "merge_requests", "merge_request_diffs", column: "latest_merge_request_diff_id", name: "fk_06067f5644", on_delete: :nullify add_foreign_key "merge_requests", "milestones", name: "fk_6a5165a692", on_delete: :nullify |