diff options
Diffstat (limited to 'db')
25 files changed, 780 insertions, 44 deletions
diff --git a/db/fixtures/development/04_project.rb b/db/fixtures/development/04_project.rb index 1f8f5cfc82b..213c8bca639 100644 --- a/db/fixtures/development/04_project.rb +++ b/db/fixtures/development/04_project.rb @@ -63,7 +63,8 @@ Sidekiq::Testing.inline! do namespace_id: group.id, name: project_path.titleize, description: FFaker::Lorem.sentence, - visibility_level: Gitlab::VisibilityLevel.values.sample + visibility_level: Gitlab::VisibilityLevel.values.sample, + skip_disk_validation: true } project = Projects::CreateService.new(User.first, params).execute diff --git a/db/fixtures/development/06_teams.rb b/db/fixtures/development/06_teams.rb index 8afb0dde077..b218f4e71fd 100644 --- a/db/fixtures/development/06_teams.rb +++ b/db/fixtures/development/06_teams.rb @@ -14,7 +14,7 @@ Sidekiq::Testing.inline! do Project.all.each do |project| User.all.sample(4).each do |user| - if project.add_role(user, Gitlab::Access.values.sample) + if project.add_role(user, Gitlab::Access.sym_options.keys.sample) print '.' else print 'F' diff --git a/db/migrate/20160301174731_add_fingerprint_index.rb b/db/migrate/20160301174731_add_fingerprint_index.rb new file mode 100644 index 00000000000..f2c3d1ba1ea --- /dev/null +++ b/db/migrate/20160301174731_add_fingerprint_index.rb @@ -0,0 +1,17 @@ +# rubocop:disable all +class AddFingerprintIndex < ActiveRecord::Migration + disable_ddl_transaction! + + DOWNTIME = false + + # https://gitlab.com/gitlab-org/gitlab-ee/issues/764 + def change + args = [:keys, :fingerprint] + + if Gitlab::Database.postgresql? + args << { algorithm: :concurrently } + end + + add_index(*args) unless index_exists?(:keys, :fingerprint) + end +end diff --git a/db/migrate/20160621123729_add_rebase_commit_sha_to_merge_requests.rb b/db/migrate/20160621123729_add_rebase_commit_sha_to_merge_requests.rb new file mode 100644 index 00000000000..1222dc640a8 --- /dev/null +++ b/db/migrate/20160621123729_add_rebase_commit_sha_to_merge_requests.rb @@ -0,0 +1,22 @@ +# This migration is a duplicate of 20171230123729_add_rebase_commit_sha_to_merge_requests_ce.rb +# +# We backported this feature from EE using the same migration, but with a new +# timestamp, which caused an error when the backport was then to be merged back +# into EE. +# +# See discussion at https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/3932 +class AddRebaseCommitShaToMergeRequests < ActiveRecord::Migration + DOWNTIME = false + + def up + unless column_exists?(:merge_requests, :rebase_commit_sha) + add_column :merge_requests, :rebase_commit_sha, :string + end + end + + def down + if column_exists?(:merge_requests, :rebase_commit_sha) + remove_column :merge_requests, :rebase_commit_sha + end + end +end diff --git a/db/migrate/20170531180233_add_authorized_keys_enabled_to_application_settings.rb b/db/migrate/20170531180233_add_authorized_keys_enabled_to_application_settings.rb new file mode 100644 index 00000000000..1d86a531eb3 --- /dev/null +++ b/db/migrate/20170531180233_add_authorized_keys_enabled_to_application_settings.rb @@ -0,0 +1,19 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddAuthorizedKeysEnabledToApplicationSettings < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_column_with_default :application_settings, :authorized_keys_enabled, :boolean, default: true, allow_null: false + end + + def down + remove_column :application_settings, :authorized_keys_enabled + end +end diff --git a/db/migrate/20170928124105_create_fork_networks.rb b/db/migrate/20170928124105_create_fork_networks.rb index ca906b953a3..89e5b871967 100644 --- a/db/migrate/20170928124105_create_fork_networks.rb +++ b/db/migrate/20170928124105_create_fork_networks.rb @@ -23,6 +23,7 @@ class CreateForkNetworks < ActiveRecord::Migration if foreign_keys_for(:fork_networks, :root_project_id).any? remove_foreign_key :fork_networks, column: :root_project_id end + drop_table :fork_networks end end diff --git a/db/migrate/20170928133643_create_fork_network_members.rb b/db/migrate/20170928133643_create_fork_network_members.rb index 836f023efdc..8c7d9ba859a 100644 --- a/db/migrate/20170928133643_create_fork_network_members.rb +++ b/db/migrate/20170928133643_create_fork_network_members.rb @@ -21,6 +21,7 @@ class CreateForkNetworkMembers < ActiveRecord::Migration if foreign_keys_for(:fork_network_members, :forked_from_project_id).any? remove_foreign_key :fork_network_members, column: :forked_from_project_id end + drop_table :fork_network_members end end diff --git a/db/migrate/20171019141859_fix_dev_timezone_schema.rb b/db/migrate/20171019141859_fix_dev_timezone_schema.rb new file mode 100644 index 00000000000..fb7c17dd747 --- /dev/null +++ b/db/migrate/20171019141859_fix_dev_timezone_schema.rb @@ -0,0 +1,25 @@ +class FixDevTimezoneSchema < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # The this migrations tries to help solve unwanted changes to `schema.rb` + # while developing GitLab. Installations created before we started using + # `datetime_with_timezone` are likely to face this problem. Updating those + # columns to the new type should help fix this. + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + TIMEZONE_TABLES = %i(appearances ci_group_variables ci_pipeline_schedule_variables events gpg_keys gpg_signatures project_auto_devops) + + def up + return unless Rails.env.development? || Rails.env.test? + + TIMEZONE_TABLES.each do |table| + change_column table, :created_at, :datetime_with_timezone + change_column table, :updated_at, :datetime_with_timezone + end + end + + def down + end +end diff --git a/db/migrate/20171106151218_issues_moved_to_id_foreign_key.rb b/db/migrate/20171106151218_issues_moved_to_id_foreign_key.rb index 8d2ceb8cc18..6395462384b 100644 --- a/db/migrate/20171106151218_issues_moved_to_id_foreign_key.rb +++ b/db/migrate/20171106151218_issues_moved_to_id_foreign_key.rb @@ -15,8 +15,20 @@ class IssuesMovedToIdForeignKey < ActiveRecord::Migration self.table_name = 'issues' def self.with_orphaned_moved_to_issues - where('NOT EXISTS (SELECT true FROM issues WHERE issues.id = issues.moved_to_id)') - .where('moved_to_id IS NOT NULL') + if Gitlab::Database.postgresql? + # Be careful to use a second table here for comparison otherwise we'll null + # out all rows that don't have id == moved.to_id! + where('NOT EXISTS (SELECT true FROM issues B WHERE issues.moved_to_id = B.id)') + .where('moved_to_id IS NOT NULL') + else + # MySQL doesn't allow modification of the same table in a subquery, + # and using a temporary table isn't automatically guaranteed to work + # due to the MySQL query optimizer. See + # https://dev.mysql.com/doc/refman/5.7/en/update.html for more + # details. + joins('LEFT JOIN issues AS b ON issues.moved_to_id = b.id') + .where('issues.moved_to_id IS NOT NULL AND b.id IS NULL') + end end end diff --git a/db/migrate/20171127151038_add_events_related_columns_to_merge_request_metrics.rb b/db/migrate/20171127151038_add_events_related_columns_to_merge_request_metrics.rb new file mode 100644 index 00000000000..18af697cf88 --- /dev/null +++ b/db/migrate/20171127151038_add_events_related_columns_to_merge_request_metrics.rb @@ -0,0 +1,37 @@ +class AddEventsRelatedColumnsToMergeRequestMetrics < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + change_table :merge_request_metrics do |t| + t.references :merged_by, references: :users + t.references :latest_closed_by, references: :users + end + + add_column :merge_request_metrics, :latest_closed_at, :datetime_with_timezone + + add_concurrent_foreign_key :merge_request_metrics, :users, + column: :merged_by_id, + on_delete: :nullify + + add_concurrent_foreign_key :merge_request_metrics, :users, + column: :latest_closed_by_id, + on_delete: :nullify + end + + def down + if foreign_keys_for(:merge_request_metrics, :merged_by_id).any? + remove_foreign_key :merge_request_metrics, column: :merged_by_id + end + + if foreign_keys_for(:merge_request_metrics, :latest_closed_by_id).any? + remove_foreign_key :merge_request_metrics, column: :latest_closed_by_id + end + + remove_columns :merge_request_metrics, + :merged_by_id, :latest_closed_by_id, :latest_closed_at + end +end diff --git a/db/migrate/20171216111734_clean_up_for_members.rb b/db/migrate/20171216111734_clean_up_for_members.rb new file mode 100644 index 00000000000..22e0997dce6 --- /dev/null +++ b/db/migrate/20171216111734_clean_up_for_members.rb @@ -0,0 +1,31 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class CleanUpForMembers < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + disable_ddl_transaction! + + class Member < ActiveRecord::Base + include EachBatch + + self.table_name = 'members' + end + + def up + condition = <<~EOF.squish + invite_token IS NULL AND + NOT EXISTS (SELECT 1 FROM users WHERE users.id = members.user_id) + EOF + + Member.each_batch(of: 10_000) do |batch| + batch.where(condition).delete_all + end + end + + def down + end +end diff --git a/db/migrate/20171216112339_add_foreign_key_for_members.rb b/db/migrate/20171216112339_add_foreign_key_for_members.rb new file mode 100644 index 00000000000..be17769be6a --- /dev/null +++ b/db/migrate/20171216112339_add_foreign_key_for_members.rb @@ -0,0 +1,21 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddForeignKeyForMembers < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_foreign_key(:members, + :users, + column: :user_id) + end + + def down + remove_foreign_key(:members, column: :user_id) + end +end diff --git a/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb b/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb index 7cf1d0cec68..d1a039ed551 100644 --- a/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb +++ b/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb @@ -9,6 +9,7 @@ class AddIndexOnNamespacesLowerName < ActiveRecord::Migration return unless Gitlab::Database.postgresql? disable_statement_timeout + if Gitlab::Database.version.to_f >= 9.5 # Allow us to hot-patch the index manually ahead of the migration execute "CREATE INDEX CONCURRENTLY IF NOT EXISTS #{INDEX_NAME} ON namespaces (lower(name));" @@ -21,6 +22,7 @@ class AddIndexOnNamespacesLowerName < ActiveRecord::Migration return unless Gitlab::Database.postgresql? disable_statement_timeout + if Gitlab::Database.version.to_f >= 9.2 execute "DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME};" else diff --git a/db/migrate/20171222183504_add_jobs_cache_index_to_project.rb b/db/migrate/20171222183504_add_jobs_cache_index_to_project.rb new file mode 100644 index 00000000000..607e9d027d7 --- /dev/null +++ b/db/migrate/20171222183504_add_jobs_cache_index_to_project.rb @@ -0,0 +1,13 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddJobsCacheIndexToProject < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + def change + add_column :projects, :jobs_cache_index, :integer + end +end diff --git a/db/migrate/20171229225929_change_user_project_limit_not_null_and_remove_default.rb b/db/migrate/20171229225929_change_user_project_limit_not_null_and_remove_default.rb new file mode 100644 index 00000000000..54fbbcf1a0d --- /dev/null +++ b/db/migrate/20171229225929_change_user_project_limit_not_null_and_remove_default.rb @@ -0,0 +1,38 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class ChangeUserProjectLimitNotNullAndRemoveDefault < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + # When a migration requires downtime you **must** uncomment the following + # constant and define a short and easy to understand explanation as to why the + # migration requires downtime. + # DOWNTIME_REASON = '' + + # When using the methods "add_concurrent_index", "remove_concurrent_index" or + # "add_column_with_default" you must disable the use of transactions + # as these methods can not run in an existing transaction. + # When using "add_concurrent_index" or "remove_concurrent_index" methods make sure + # that either of them is the _only_ method called in the migration, + # any other changes should go in a separate migration. + # This ensures that upon failure _only_ the index creation or removing fails + # and can be retried or reverted easily. + # + # To disable transactions uncomment the following line and remove these + # comments: + # disable_ddl_transaction! + + def up + # Set Users#projects_limit to NOT NULL and remove the default value + change_column_null :users, :projects_limit, false + change_column_default :users, :projects_limit, nil + end + + def down + change_column_null :users, :projects_limit, true + change_column_default :users, :projects_limit, 10 + end +end diff --git a/db/migrate/20171230123729_add_rebase_commit_sha_to_merge_requests_ce.rb b/db/migrate/20171230123729_add_rebase_commit_sha_to_merge_requests_ce.rb new file mode 100644 index 00000000000..94a7c1019d8 --- /dev/null +++ b/db/migrate/20171230123729_add_rebase_commit_sha_to_merge_requests_ce.rb @@ -0,0 +1,15 @@ +class AddRebaseCommitShaToMergeRequestsCe < ActiveRecord::Migration + DOWNTIME = false + + def up + unless column_exists?(:merge_requests, :rebase_commit_sha) + add_column :merge_requests, :rebase_commit_sha, :string + end + end + + def down + if column_exists?(:merge_requests, :rebase_commit_sha) + remove_column :merge_requests, :rebase_commit_sha + end + end +end diff --git a/db/migrate/20180105212544_add_commits_count_to_merge_request_diff.rb b/db/migrate/20180105212544_add_commits_count_to_merge_request_diff.rb new file mode 100644 index 00000000000..f942b4c062e --- /dev/null +++ b/db/migrate/20180105212544_add_commits_count_to_merge_request_diff.rb @@ -0,0 +1,29 @@ +class AddCommitsCountToMergeRequestDiff < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + MIGRATION = 'AddMergeRequestDiffCommitsCount'.freeze + BATCH_SIZE = 5000 + DELAY_INTERVAL = 5.minutes.to_i + + class MergeRequestDiff < ActiveRecord::Base + self.table_name = 'merge_request_diffs' + + include ::EachBatch + end + + disable_ddl_transaction! + + def up + add_column :merge_request_diffs, :commits_count, :integer + + say 'Populating the MergeRequestDiff `commits_count`' + + queue_background_migration_jobs_by_range_at_intervals(MergeRequestDiff, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE) + end + + def down + remove_column :merge_request_diffs, :commits_count + end +end diff --git a/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb b/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb index da0fcda87a6..17ad7de065d 100644 --- a/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb +++ b/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb @@ -31,6 +31,7 @@ class RenameUsersWithRenamedNamespace < ActiveRecord::Migration predicate = namespaces[:owner_id].eq(users[:id]) .and(namespaces[:type].eq(nil)) .and(users[:username].matches(path)) + update_sql = if Gitlab::Database.postgresql? "UPDATE users SET username = namespaces.path "\ "FROM namespaces WHERE #{predicate.to_sql}" diff --git a/db/post_migrate/20170907170235_delete_conflicting_redirect_routes.rb b/db/post_migrate/20170907170235_delete_conflicting_redirect_routes.rb index 3e84b295be4..033019c398e 100644 --- a/db/post_migrate/20170907170235_delete_conflicting_redirect_routes.rb +++ b/db/post_migrate/20170907170235_delete_conflicting_redirect_routes.rb @@ -2,36 +2,12 @@ # for more information on how to write migrations for GitLab. class DeleteConflictingRedirectRoutes < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - DOWNTIME = false - MIGRATION = 'DeleteConflictingRedirectRoutesRange'.freeze - BATCH_SIZE = 200 # At 200, I expect under 20s per batch, which is under our query timeout of 60s. - DELAY_INTERVAL = 12.seconds - - disable_ddl_transaction! - - class Route < ActiveRecord::Base - include EachBatch - - self.table_name = 'routes' - end - def up - say opening_message - - queue_background_migration_jobs_by_range_at_intervals(Route, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE) + # No-op. + # See https://gitlab.com/gitlab-com/infrastructure/issues/3460#note_53223252 end def down # nothing end - - def opening_message - <<~MSG - Clean up redirect routes that conflict with regular routes. - See initial bug fix: - https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/13357 - MSG - end end diff --git a/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb b/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb new file mode 100644 index 00000000000..11b581e4b57 --- /dev/null +++ b/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb @@ -0,0 +1,151 @@ +class MigrateKubernetesServiceToNewClustersArchitectures < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + DEFAULT_KUBERNETES_SERVICE_CLUSTER_NAME = 'KubernetesService'.freeze + + disable_ddl_transaction! + + class Project < ActiveRecord::Base + self.table_name = 'projects' + + has_many :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::ClustersProject' + has_many :clusters, through: :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Cluster' + has_many :services, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Service' + has_one :kubernetes_service, -> { where(category: 'deployment', type: 'KubernetesService') }, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Service', inverse_of: :project, foreign_key: :project_id + end + + class Cluster < ActiveRecord::Base + self.table_name = 'clusters' + + has_many :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::ClustersProject' + has_many :projects, through: :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Project' + has_one :platform_kubernetes, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::PlatformsKubernetes' + + accepts_nested_attributes_for :platform_kubernetes + + enum platform_type: { + kubernetes: 1 + } + + enum provider_type: { + user: 0, + gcp: 1 + } + end + + class ClustersProject < ActiveRecord::Base + self.table_name = 'cluster_projects' + + belongs_to :cluster, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Cluster' + belongs_to :project, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Project' + end + + class PlatformsKubernetes < ActiveRecord::Base + self.table_name = 'cluster_platforms_kubernetes' + + belongs_to :cluster, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Cluster' + + attr_encrypted :token, + mode: :per_attribute_iv, + key: Gitlab::Application.secrets.db_key_base, + algorithm: 'aes-256-cbc' + end + + class Service < ActiveRecord::Base + include EachBatch + + self.table_name = 'services' + self.inheritance_column = :_type_disabled # Disable STI, otherwise KubernetesModel will be looked up + + belongs_to :project, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Project', foreign_key: :project_id + + scope :unmanaged_kubernetes_service, -> do + joins('LEFT JOIN projects ON projects.id = services.project_id') + .joins('LEFT JOIN cluster_projects ON cluster_projects.project_id = projects.id') + .joins('LEFT JOIN cluster_platforms_kubernetes ON cluster_platforms_kubernetes.cluster_id = cluster_projects.cluster_id') + .where(category: 'deployment', type: 'KubernetesService', template: false) + .where("services.properties LIKE '%api_url%'") + .where("(services.properties NOT LIKE CONCAT('%', cluster_platforms_kubernetes.api_url, '%')) OR cluster_platforms_kubernetes.api_url IS NULL") + .group(:id) + .order(id: :asc) + end + + scope :kubernetes_service_without_template, -> do + where(category: 'deployment', type: 'KubernetesService', template: false) + end + + def api_url + parsed_properties['api_url'] + end + + def ca_pem + parsed_properties['ca_pem'] + end + + def namespace + parsed_properties['namespace'] + end + + def token + parsed_properties['token'] + end + + private + + def parsed_properties + @parsed_properties ||= JSON.parse(self.properties) + end + end + + def find_dedicated_environement_scope(project) + environment_scopes = project.clusters.map(&:environment_scope) + + return '*' if environment_scopes.exclude?('*') # KubernetesService should be added as a default cluster (environment_scope: '*') at first place + return 'migrated/*' if environment_scopes.exclude?('migrated/*') # If it's conflicted, the KubernetesService added as a migrated cluster + + unique_iid = 0 + + # If it's still conflicted, finding an unique environment scope incrementaly + loop do + candidate = "migrated#{unique_iid}/*" + return candidate if environment_scopes.exclude?(candidate) + + unique_iid += 1 + end + end + + def up + ActiveRecord::Base.transaction do + MigrateKubernetesServiceToNewClustersArchitectures::Service + .unmanaged_kubernetes_service.find_each(batch_size: 1) do |kubernetes_service| + MigrateKubernetesServiceToNewClustersArchitectures::Cluster.create( + enabled: kubernetes_service.active, + user_id: nil, # KubernetesService doesn't have + name: DEFAULT_KUBERNETES_SERVICE_CLUSTER_NAME, + provider_type: MigrateKubernetesServiceToNewClustersArchitectures::Cluster.provider_types[:user], + platform_type: MigrateKubernetesServiceToNewClustersArchitectures::Cluster.platform_types[:kubernetes], + projects: [kubernetes_service.project], + environment_scope: find_dedicated_environement_scope(kubernetes_service.project), + platform_kubernetes_attributes: { + api_url: kubernetes_service.api_url, + ca_cert: kubernetes_service.ca_pem, + namespace: kubernetes_service.namespace, + username: nil, # KubernetesService doesn't have + encrypted_password: nil, # KubernetesService doesn't have + encrypted_password_iv: nil, # KubernetesService doesn't have + token: kubernetes_service.token # encrypted_token and encrypted_token_iv + } ) + end + end + + MigrateKubernetesServiceToNewClustersArchitectures::Service + .kubernetes_service_without_template.each_batch(of: 100) do |kubernetes_service| + kubernetes_service.update_all(active: false) + end + end + + def down + # noop + end +end diff --git a/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb b/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb new file mode 100644 index 00000000000..fce1829c982 --- /dev/null +++ b/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true +# rubocop:disable GitlabSecurity/SqlInjection + +class SchedulePopulateMergeRequestMetricsWithEventsData < ActiveRecord::Migration + DOWNTIME = false + BATCH_SIZE = 10_000 + MIGRATION = 'PopulateMergeRequestMetricsWithEventsData' + + disable_ddl_transaction! + + class MergeRequest < ActiveRecord::Base + self.table_name = 'merge_requests' + + include ::EachBatch + end + + def up + say 'Scheduling `PopulateMergeRequestMetricsWithEventsData` jobs' + # It will update around 4_000_000 records in batches of 10_000 merge + # requests (running between 10 minutes) and should take around 66 hours to complete. + # Apparently, production PostgreSQL is able to vacuum 10k-20k dead_tuples by + # minute, and at maximum, each of these jobs should UPDATE 20k records. + # + # More information about the updates in `PopulateMergeRequestMetricsWithEventsData` class. + # + MergeRequest.all.each_batch(of: BATCH_SIZE) do |relation, index| + range = relation.pluck('MIN(id)', 'MAX(id)').first + + BackgroundMigrationWorker.perform_in(index * 10.minutes, MIGRATION, range) + end + end + + def down + execute "update merge_request_metrics set latest_closed_at = null" + execute "update merge_request_metrics set latest_closed_by_id = null" + execute "update merge_request_metrics set merged_by_id = null" + end +end diff --git a/db/post_migrate/20171207150343_remove_soft_removed_objects.rb b/db/post_migrate/20171207150343_remove_soft_removed_objects.rb new file mode 100644 index 00000000000..3e2dedfdd6a --- /dev/null +++ b/db/post_migrate/20171207150343_remove_soft_removed_objects.rb @@ -0,0 +1,208 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class RemoveSoftRemovedObjects < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + disable_ddl_transaction! + + module SoftRemoved + extend ActiveSupport::Concern + + included do + scope :soft_removed, -> { where('deleted_at IS NOT NULL') } + end + end + + class User < ActiveRecord::Base + self.table_name = 'users' + + include EachBatch + end + + class Issue < ActiveRecord::Base + self.table_name = 'issues' + + include EachBatch + include SoftRemoved + end + + class MergeRequest < ActiveRecord::Base + self.table_name = 'merge_requests' + + include EachBatch + include SoftRemoved + end + + class Namespace < ActiveRecord::Base + self.table_name = 'namespaces' + + include EachBatch + include SoftRemoved + + scope :soft_removed_personal, -> { soft_removed.where(type: nil) } + scope :soft_removed_group, -> { soft_removed.where(type: 'Group') } + end + + class Route < ActiveRecord::Base + self.table_name = 'routes' + + include EachBatch + include SoftRemoved + end + + class Project < ActiveRecord::Base + self.table_name = 'projects' + + include EachBatch + include SoftRemoved + end + + class CiPipelineSchedule < ActiveRecord::Base + self.table_name = 'ci_pipeline_schedules' + + include EachBatch + include SoftRemoved + end + + class CiTrigger < ActiveRecord::Base + self.table_name = 'ci_triggers' + + include EachBatch + include SoftRemoved + end + + MODELS = [Issue, MergeRequest, CiPipelineSchedule, CiTrigger].freeze + + def up + disable_statement_timeout + + remove_personal_routes + remove_personal_namespaces + remove_group_namespaces + remove_simple_soft_removed_rows + end + + def down + # The data removed by this migration can't be restored in an automated way. + end + + def remove_simple_soft_removed_rows + create_temporary_indexes + + MODELS.each do |model| + say_with_time("Removing soft removed rows from #{model.table_name}") do + model.soft_removed.each_batch do |batch, index| + batch.delete_all + end + end + end + ensure + remove_temporary_indexes + end + + def create_temporary_indexes + MODELS.each do |model| + index_name = temporary_index_name_for(model) + + # Without this index the removal process can take a very long time. For + # example, getting the next ID of a batch for the `issues` table in + # staging would take between 15 and 20 seconds. + next if temporary_index_exists?(model) + + say_with_time("Creating temporary index #{index_name}") do + add_concurrent_index( + model.table_name, + [:deleted_at, :id], + name: index_name, + where: 'deleted_at IS NOT NULL' + ) + end + end + end + + def remove_temporary_indexes + MODELS.each do |model| + index_name = temporary_index_name_for(model) + + next unless temporary_index_exists?(model) + + say_with_time("Removing temporary index #{index_name}") do + remove_concurrent_index_by_name(model.table_name, index_name) + end + end + end + + def temporary_index_name_for(model) + "index_on_#{model.table_name}_tmp" + end + + def temporary_index_exists?(model) + index_name = temporary_index_name_for(model) + + index_exists?(model.table_name, [:deleted_at, :id], name: index_name) + end + + def remove_personal_namespaces + # Some personal namespaces are left behind in case of GitLab.com. In these + # cases the associated data such as the projects and users has already been + # removed. + Namespace.soft_removed_personal.each_batch do |batch| + batch.delete_all + end + end + + def remove_group_namespaces + admin_id = id_for_admin_user + + unless admin_id + say 'Not scheduling soft removed groups for removal as no admin user ' \ + 'could be found. You will need to remove any such groups manually.' + + return + end + + # Left over groups can't be easily removed because we may also need to + # remove memberships, repositories, and other associated data. As a result + # we'll just schedule a Sidekiq job to remove these. + # + # As of January 5th, 2018 there are 36 groups that will be removed using + # this code. + Namespace.select(:id).soft_removed_group.each_batch(of: 10) do |batch, index| + batch.each do |ns| + schedule_group_removal(index * 5.minutes, ns.id, admin_id) + end + end + end + + def schedule_group_removal(delay, group_id, user_id) + if migrate_inline? + GroupDestroyWorker.new.perform(group_id, user_id) + else + GroupDestroyWorker.perform_in(delay, group_id, user_id) + end + end + + def remove_personal_routes + namespaces = Namespace.select(1) + .soft_removed + .where('namespaces.type IS NULL') + .where('routes.source_type = ?', 'Namespace') + .where('routes.source_id = namespaces.id') + + Route.where('EXISTS (?)', namespaces).each_batch do |batch| + batch.delete_all + end + end + + def id_for_admin_user + User.where(admin: true).limit(1).pluck(:id).first + end + + def migrate_inline? + Rails.env.test? || Rails.env.development? + end +end diff --git a/db/post_migrate/20171207150344_remove_deleted_at_columns.rb b/db/post_migrate/20171207150344_remove_deleted_at_columns.rb new file mode 100644 index 00000000000..154d7a1b926 --- /dev/null +++ b/db/post_migrate/20171207150344_remove_deleted_at_columns.rb @@ -0,0 +1,31 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class RemoveDeletedAtColumns < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + TABLES = %i[issues merge_requests namespaces ci_pipeline_schedules ci_triggers].freeze + COLUMN = :deleted_at + + def up + TABLES.each do |table| + remove_column(table, COLUMN) if column_exists?(table, COLUMN) + end + end + + def down + TABLES.each do |table| + unless column_exists?(table, COLUMN) + add_column(table, COLUMN, :datetime_with_timezone) + end + + unless index_exists?(table, COLUMN) + add_concurrent_index(table, COLUMN) + end + end + end +end diff --git a/db/post_migrate/20171221140220_schedule_issues_closed_at_type_change.rb b/db/post_migrate/20171221140220_schedule_issues_closed_at_type_change.rb new file mode 100644 index 00000000000..eeecc7b1de0 --- /dev/null +++ b/db/post_migrate/20171221140220_schedule_issues_closed_at_type_change.rb @@ -0,0 +1,45 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. +# rubocop:disable Migration/Datetime +class ScheduleIssuesClosedAtTypeChange < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + class Issue < ActiveRecord::Base + self.table_name = 'issues' + include EachBatch + + def self.to_migrate + where('closed_at IS NOT NULL') + end + end + + def up + return unless migrate_column_type? + + change_column_type_using_background_migration( + Issue.to_migrate, + :closed_at, + :datetime_with_timezone + ) + end + + def down + return if migrate_column_type? + + change_column_type_using_background_migration( + Issue.to_migrate, + :closed_at, + :datetime + ) + end + + def migrate_column_type? + # Some environments may have already executed the previous version of this + # migration, thus we don't need to migrate those environments again. + column_for('issues', 'closed_at').type == :datetime + end +end diff --git a/db/schema.rb b/db/schema.rb index aa5db5da4f0..8a6db61250b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20171220191323) do +ActiveRecord::Schema.define(version: 20180105212544) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -154,6 +154,7 @@ ActiveRecord::Schema.define(version: 20171220191323) do t.integer "gitaly_timeout_default", default: 55, null: false t.integer "gitaly_timeout_medium", default: 30, null: false t.integer "gitaly_timeout_fast", default: 10, null: false + t.boolean "authorized_keys_enabled", default: true, null: false end create_table "audit_events", force: :cascade do |t| @@ -356,7 +357,6 @@ ActiveRecord::Schema.define(version: 20171220191323) do t.integer "project_id" t.integer "owner_id" t.boolean "active", default: true - t.datetime "deleted_at" t.datetime "created_at" t.datetime "updated_at" end @@ -466,7 +466,6 @@ ActiveRecord::Schema.define(version: 20171220191323) do create_table "ci_triggers", force: :cascade do |t| t.string "token" - t.datetime "deleted_at" t.datetime "created_at" t.datetime "updated_at" t.integer "project_id" @@ -657,8 +656,8 @@ ActiveRecord::Schema.define(version: 20171220191323) do t.datetime "created_at" t.datetime "updated_at" t.string "confirmation_token" - t.datetime "confirmed_at" - t.datetime "confirmation_sent_at" + t.datetime_with_timezone "confirmed_at" + t.datetime_with_timezone "confirmation_sent_at" end add_index "emails", ["confirmation_token"], name: "index_emails_on_confirmation_token", unique: true, using: :btree @@ -860,7 +859,6 @@ ActiveRecord::Schema.define(version: 20171220191323) do t.integer "iid" t.integer "updated_by_id" t.boolean "confidential", default: false, null: false - t.datetime "deleted_at" t.date "due_date" t.integer "moved_to_id" t.integer "lock_version" @@ -877,7 +875,6 @@ ActiveRecord::Schema.define(version: 20171220191323) do add_index "issues", ["author_id"], name: "index_issues_on_author_id", using: :btree add_index "issues", ["confidential"], name: "index_issues_on_confidential", using: :btree - add_index "issues", ["deleted_at"], name: "index_issues_on_deleted_at", using: :btree add_index "issues", ["description"], name: "index_issues_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"} add_index "issues", ["milestone_id"], name: "index_issues_on_milestone_id", using: :btree add_index "issues", ["moved_to_id"], name: "index_issues_on_moved_to_id", where: "(moved_to_id IS NOT NULL)", using: :btree @@ -1043,6 +1040,7 @@ ActiveRecord::Schema.define(version: 20171220191323) do t.string "real_size" t.string "head_commit_sha" t.string "start_commit_sha" + t.integer "commits_count" end add_index "merge_request_diffs", ["merge_request_id", "id"], name: "index_merge_request_diffs_on_merge_request_id_and_id", using: :btree @@ -1056,6 +1054,9 @@ ActiveRecord::Schema.define(version: 20171220191323) do t.datetime "created_at", null: false t.datetime "updated_at", null: false t.integer "pipeline_id" + t.integer "merged_by_id" + t.integer "latest_closed_by_id" + t.datetime_with_timezone "latest_closed_at" end add_index "merge_request_metrics", ["first_deployed_to_production_at"], name: "index_merge_request_metrics_on_first_deployed_to_production_at", using: :btree @@ -1083,7 +1084,6 @@ ActiveRecord::Schema.define(version: 20171220191323) do t.boolean "merge_when_pipeline_succeeds", default: false, null: false t.integer "merge_user_id" t.string "merge_commit_sha" - t.datetime "deleted_at" t.string "in_progress_merge_commit_sha" t.integer "lock_version" t.text "title_html" @@ -1096,12 +1096,12 @@ ActiveRecord::Schema.define(version: 20171220191323) do t.string "merge_jid" t.boolean "discussion_locked" t.integer "latest_merge_request_diff_id" + t.string "rebase_commit_sha" end add_index "merge_requests", ["assignee_id"], name: "index_merge_requests_on_assignee_id", using: :btree add_index "merge_requests", ["author_id"], name: "index_merge_requests_on_author_id", using: :btree add_index "merge_requests", ["created_at"], name: "index_merge_requests_on_created_at", using: :btree - add_index "merge_requests", ["deleted_at"], name: "index_merge_requests_on_deleted_at", using: :btree add_index "merge_requests", ["description"], name: "index_merge_requests_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"} add_index "merge_requests", ["head_pipeline_id"], name: "index_merge_requests_on_head_pipeline_id", using: :btree add_index "merge_requests", ["latest_merge_request_diff_id"], name: "index_merge_requests_on_latest_merge_request_diff_id", using: :btree @@ -1161,7 +1161,6 @@ ActiveRecord::Schema.define(version: 20171220191323) do t.boolean "share_with_group_lock", default: false t.integer "visibility_level", default: 20, null: false t.boolean "request_access_enabled", default: false, null: false - t.datetime "deleted_at" t.text "description_html" t.boolean "lfs_enabled" t.integer "parent_id" @@ -1171,7 +1170,6 @@ ActiveRecord::Schema.define(version: 20171220191323) do end add_index "namespaces", ["created_at"], name: "index_namespaces_on_created_at", using: :btree - add_index "namespaces", ["deleted_at"], name: "index_namespaces_on_deleted_at", using: :btree add_index "namespaces", ["name", "parent_id"], name: "index_namespaces_on_name_and_parent_id", unique: true, using: :btree add_index "namespaces", ["name"], name: "index_namespaces_on_name_trigram", using: :gin, opclasses: {"name"=>"gin_trgm_ops"} add_index "namespaces", ["owner_id"], name: "index_namespaces_on_owner_id", using: :btree @@ -1447,6 +1445,7 @@ ActiveRecord::Schema.define(version: 20171220191323) do t.boolean "repository_read_only" t.boolean "merge_requests_ff_only_enabled", default: false t.boolean "merge_requests_rebase_enabled", default: false, null: false + t.integer "jobs_cache_index" end add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree @@ -1770,8 +1769,8 @@ ActiveRecord::Schema.define(version: 20171220191323) do add_index "user_agent_details", ["subject_id", "subject_type"], name: "index_user_agent_details_on_subject_id_and_subject_type", using: :btree create_table "user_custom_attributes", force: :cascade do |t| - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime_with_timezone "created_at", null: false + t.datetime_with_timezone "updated_at", null: false t.integer "user_id", null: false t.string "key", null: false t.string "value", null: false @@ -1805,7 +1804,7 @@ ActiveRecord::Schema.define(version: 20171220191323) do t.datetime "updated_at" t.string "name" t.boolean "admin", default: false, null: false - t.integer "projects_limit", default: 10 + t.integer "projects_limit", null: false t.string "skype", default: "", null: false t.string "linkedin", default: "", null: false t.string "twitter", default: "", null: false @@ -1989,11 +1988,14 @@ ActiveRecord::Schema.define(version: 20171220191323) do add_foreign_key "labels", "projects", name: "fk_7de4989a69", on_delete: :cascade add_foreign_key "lists", "boards", name: "fk_0d3f677137", on_delete: :cascade add_foreign_key "lists", "labels", name: "fk_7a5553d60f", on_delete: :cascade + add_foreign_key "members", "users", name: "fk_2e88fb7ce9", on_delete: :cascade add_foreign_key "merge_request_diff_commits", "merge_request_diffs", on_delete: :cascade add_foreign_key "merge_request_diff_files", "merge_request_diffs", on_delete: :cascade add_foreign_key "merge_request_diffs", "merge_requests", name: "fk_8483f3258f", on_delete: :cascade add_foreign_key "merge_request_metrics", "ci_pipelines", column: "pipeline_id", on_delete: :cascade add_foreign_key "merge_request_metrics", "merge_requests", on_delete: :cascade + add_foreign_key "merge_request_metrics", "users", column: "latest_closed_by_id", name: "fk_ae440388cc", on_delete: :nullify + add_foreign_key "merge_request_metrics", "users", column: "merged_by_id", name: "fk_7f28d925f3", on_delete: :nullify add_foreign_key "merge_requests", "ci_pipelines", column: "head_pipeline_id", name: "fk_fd82eae0b9", on_delete: :nullify add_foreign_key "merge_requests", "merge_request_diffs", column: "latest_merge_request_diff_id", name: "fk_06067f5644", on_delete: :nullify add_foreign_key "merge_requests", "milestones", name: "fk_6a5165a692", on_delete: :nullify |