diff options
Diffstat (limited to 'db/migrate')
9 files changed, 413 insertions, 0 deletions
diff --git a/db/migrate/20160804142904_add_ci_config_file_to_project.rb b/db/migrate/20160804142904_add_ci_config_file_to_project.rb new file mode 100644 index 00000000000..341ae555c1b --- /dev/null +++ b/db/migrate/20160804142904_add_ci_config_file_to_project.rb @@ -0,0 +1,11 @@ +class AddCiConfigFileToProject < ActiveRecord::Migration + DOWNTIME = false + + def change + add_column :projects, :ci_config_path, :string + end + + def down + remove_column :projects, :ci_config_path + end +end diff --git a/db/migrate/20170530130129_project_foreign_keys_with_cascading_deletes.rb b/db/migrate/20170530130129_project_foreign_keys_with_cascading_deletes.rb new file mode 100644 index 00000000000..3eaafac321d --- /dev/null +++ b/db/migrate/20170530130129_project_foreign_keys_with_cascading_deletes.rb @@ -0,0 +1,187 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class ProjectForeignKeysWithCascadingDeletes < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + CONCURRENCY = 4 + + disable_ddl_transaction! + + # The tables/columns for which to remove orphans and add foreign keys. Order + # matters as some tables/columns should be processed before others. + TABLES = [ + [:boards, :projects, :project_id], + [:lists, :labels, :label_id], + [:lists, :boards, :board_id], + [:services, :projects, :project_id], + [:forked_project_links, :projects, :forked_to_project_id], + [:merge_requests, :projects, :target_project_id], + [:labels, :projects, :project_id], + [:issues, :projects, :project_id], + [:events, :projects, :project_id], + [:milestones, :projects, :project_id], + [:notes, :projects, :project_id], + [:snippets, :projects, :project_id], + [:web_hooks, :projects, :project_id], + [:protected_branch_merge_access_levels, :protected_branches, :protected_branch_id], + [:protected_branch_push_access_levels, :protected_branches, :protected_branch_id], + [:protected_branches, :projects, :project_id], + [:protected_tags, :projects, :project_id], + [:deploy_keys_projects, :projects, :project_id], + [:users_star_projects, :projects, :project_id], + [:releases, :projects, :project_id], + [:project_group_links, :projects, :project_id], + [:pages_domains, :projects, :project_id], + [:todos, :projects, :project_id], + [:project_import_data, :projects, :project_id], + [:project_features, :projects, :project_id], + [:ci_builds, :projects, :project_id], + [:ci_pipelines, :projects, :project_id], + [:ci_runner_projects, :projects, :project_id], + [:ci_triggers, :projects, :project_id], + [:environments, :projects, :project_id], + [:deployments, :projects, :project_id] + ] + + def up + # These existing foreign keys don't have an "ON DELETE CASCADE" clause. + remove_foreign_key_without_error(:boards, :project_id) + remove_foreign_key_without_error(:lists, :label_id) + remove_foreign_key_without_error(:lists, :board_id) + remove_foreign_key_without_error(:protected_branch_merge_access_levels, + :protected_branch_id) + + remove_foreign_key_without_error(:protected_branch_push_access_levels, + :protected_branch_id) + + remove_orphaned_rows + add_foreign_keys + + # These columns are not indexed yet, meaning a cascading delete would take + # forever. + add_concurrent_index(:project_group_links, :project_id) + add_concurrent_index(:pages_domains, :project_id) + end + + def down + TABLES.each do |(source, _, column)| + remove_foreign_key_without_error(source, column) + end + + add_concurrent_foreign_key(:boards, :projects, column: :project_id) + add_concurrent_foreign_key(:lists, :labels, column: :label_id) + add_concurrent_foreign_key(:lists, :boards, column: :board_id) + + add_concurrent_foreign_key(:protected_branch_merge_access_levels, + :protected_branches, + column: :protected_branch_id) + + add_concurrent_foreign_key(:protected_branch_push_access_levels, + :protected_branches, + column: :protected_branch_id) + + remove_index_without_error(:project_group_links, :project_id) + remove_index_without_error(:pages_domains, :project_id) + end + + def add_foreign_keys + TABLES.each do |(source, target, column)| + add_concurrent_foreign_key(source, target, column: column) + end + end + + # Removes orphans from various tables concurrently. + def remove_orphaned_rows + Gitlab::Database.with_connection_pool(CONCURRENCY) do |pool| + queues = queues_for_rows(TABLES) + + threads = queues.map do |queue| + Thread.new do + pool.with_connection do |connection| + Thread.current[:foreign_key_connection] = connection + + # Disables statement timeouts for the current connection. This is + # necessary as removing of orphaned data might otherwise exceed the + # statement timeout. + disable_statement_timeout + + remove_orphans(*queue.pop) until queue.empty? + + steal_from_queues(queues - [queue]) + end + end + end + + threads.each(&:join) + end + end + + def steal_from_queues(queues) + loop do + stolen = false + + queues.each do |queue| + # Stealing is racy so it's possible a pop might be called on an + # already-empty queue. + begin + remove_orphans(*queue.pop(true)) + stolen = true + rescue ThreadError + end + end + + break unless stolen + end + end + + def remove_orphans(source, target, column) + quoted_source = quote_table_name(source) + quoted_target = quote_table_name(target) + quoted_column = quote_column_name(column) + + execute <<-EOF.strip_heredoc + DELETE FROM #{quoted_source} + WHERE NOT EXISTS ( + SELECT true + FROM #{quoted_target} + WHERE #{quoted_target}.id = #{quoted_source}.#{quoted_column} + ) + AND #{quoted_source}.#{quoted_column} IS NOT NULL + EOF + end + + def remove_foreign_key_without_error(table, column) + remove_foreign_key(table, column: column) + rescue ArgumentError + end + + def remove_index_without_error(table, column) + remove_concurrent_index(table, column) + rescue ArgumentError + end + + def connection + # Rails memoizes connection objects, but this causes them to be shared + # amongst threads; we don't want that. + Thread.current[:foreign_key_connection] || ActiveRecord::Base.connection + end + + def queues_for_rows(rows) + queues = Array.new(CONCURRENCY) { Queue.new } + slice_size = rows.length / CONCURRENCY + + # Divide all the tuples as evenly as possible amongst the queues. + rows.each_slice(slice_size).each_with_index do |slice, index| + bucket = index % CONCURRENCY + + slice.each do |row| + queues[bucket] << row + end + end + + queues + end +end diff --git a/db/migrate/20170622130029_correct_protected_branches_foreign_keys.rb b/db/migrate/20170622130029_correct_protected_branches_foreign_keys.rb new file mode 100644 index 00000000000..46497775527 --- /dev/null +++ b/db/migrate/20170622130029_correct_protected_branches_foreign_keys.rb @@ -0,0 +1,40 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class CorrectProtectedBranchesForeignKeys < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + disable_ddl_transaction! + + def up + remove_foreign_key_without_error(:protected_branch_push_access_levels, + column: :protected_branch_id) + + execute <<-EOF + DELETE FROM protected_branch_push_access_levels + WHERE NOT EXISTS ( + SELECT true + FROM protected_branches + WHERE protected_branch_push_access_levels.protected_branch_id = protected_branches.id + ) + AND protected_branch_id IS NOT NULL + EOF + + add_concurrent_foreign_key(:protected_branch_push_access_levels, + :protected_branches, + column: :protected_branch_id) + end + + def down + # Previously there was a foreign key without a CASCADING DELETE, so we'll + # just leave the foreign key in place. + end + + def remove_foreign_key_without_error(*args) + remove_foreign_key(*args) + rescue ArgumentError + end +end diff --git a/db/migrate/20170622132212_add_foreign_key_for_merge_request_diffs.rb b/db/migrate/20170622132212_add_foreign_key_for_merge_request_diffs.rb new file mode 100644 index 00000000000..9f524fac8a7 --- /dev/null +++ b/db/migrate/20170622132212_add_foreign_key_for_merge_request_diffs.rb @@ -0,0 +1,30 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddForeignKeyForMergeRequestDiffs < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + disable_ddl_transaction! + + def up + execute <<-EOF + DELETE FROM merge_request_diffs + WHERE NOT EXISTS ( + SELECT true + FROM merge_requests + WHERE merge_requests.id = merge_request_diffs.merge_request_id + ) + EOF + + add_concurrent_foreign_key(:merge_request_diffs, + :merge_requests, + column: :merge_request_id) + end + + def down + remove_foreign_key(:merge_request_diffs, column: :merge_request_id) + end +end diff --git a/db/migrate/20170622135451_rename_duplicated_variable_key.rb b/db/migrate/20170622135451_rename_duplicated_variable_key.rb new file mode 100644 index 00000000000..368718ab0ce --- /dev/null +++ b/db/migrate/20170622135451_rename_duplicated_variable_key.rb @@ -0,0 +1,38 @@ +class RenameDuplicatedVariableKey < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + execute(<<~SQL) + UPDATE ci_variables + SET #{key} = CONCAT(#{key}, #{underscore}, id) + WHERE id IN ( + SELECT * + FROM ( -- MySQL requires an extra layer + SELECT dup.id + FROM ci_variables dup + INNER JOIN (SELECT max(id) AS id, #{key}, project_id + FROM ci_variables tmp + GROUP BY #{key}, project_id) var + USING (#{key}, project_id) where dup.id <> var.id + ) dummy + ) + SQL + end + + def down + # noop + end + + def key + # key needs to be quoted in MySQL + quote_column_name('key') + end + + def underscore + quote('_') + end +end diff --git a/db/migrate/20170622135628_add_environment_scope_to_ci_variables.rb b/db/migrate/20170622135628_add_environment_scope_to_ci_variables.rb new file mode 100644 index 00000000000..17fe062d8d5 --- /dev/null +++ b/db/migrate/20170622135628_add_environment_scope_to_ci_variables.rb @@ -0,0 +1,15 @@ +class AddEnvironmentScopeToCiVariables < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_column_with_default(:ci_variables, :environment_scope, :string, default: '*') + end + + def down + remove_column(:ci_variables, :environment_scope) + end +end diff --git a/db/migrate/20170622135728_add_unique_constraint_to_ci_variables.rb b/db/migrate/20170622135728_add_unique_constraint_to_ci_variables.rb new file mode 100644 index 00000000000..8b2cc40ee59 --- /dev/null +++ b/db/migrate/20170622135728_add_unique_constraint_to_ci_variables.rb @@ -0,0 +1,38 @@ +class AddUniqueConstraintToCiVariables < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + unless this_index_exists? + add_concurrent_index(:ci_variables, columns, name: index_name, unique: true) + end + end + + def down + if this_index_exists? + if Gitlab::Database.mysql? && !index_exists?(:ci_variables, :project_id) + # Need to add this index for MySQL project_id foreign key constraint + add_concurrent_index(:ci_variables, :project_id) + end + + remove_concurrent_index(:ci_variables, columns, name: index_name) + end + end + + private + + def this_index_exists? + index_exists?(:ci_variables, columns, name: index_name) + end + + def columns + @columns ||= [:project_id, :key, :environment_scope] + end + + def index_name + 'index_ci_variables_on_project_id_and_key_and_environment_scope' + end +end diff --git a/db/migrate/20170623080805_remove_ci_variables_project_id_index.rb b/db/migrate/20170623080805_remove_ci_variables_project_id_index.rb new file mode 100644 index 00000000000..ddcc0292b9d --- /dev/null +++ b/db/migrate/20170623080805_remove_ci_variables_project_id_index.rb @@ -0,0 +1,19 @@ +class RemoveCiVariablesProjectIdIndex < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + if index_exists?(:ci_variables, :project_id) + remove_concurrent_index(:ci_variables, :project_id) + end + end + + def down + unless index_exists?(:ci_variables, :project_id) + add_concurrent_index(:ci_variables, :project_id) + end + end +end diff --git a/db/migrate/20170703102400_add_stage_id_foreign_key_to_builds.rb b/db/migrate/20170703102400_add_stage_id_foreign_key_to_builds.rb new file mode 100644 index 00000000000..68b947583d3 --- /dev/null +++ b/db/migrate/20170703102400_add_stage_id_foreign_key_to_builds.rb @@ -0,0 +1,35 @@ +class AddStageIdForeignKeyToBuilds < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + unless index_exists?(:ci_builds, :stage_id) + add_concurrent_index(:ci_builds, :stage_id) + end + + unless foreign_key_exists?(:ci_builds, :stage_id) + add_concurrent_foreign_key(:ci_builds, :ci_stages, column: :stage_id, on_delete: :cascade) + end + end + + def down + if foreign_key_exists?(:ci_builds, :stage_id) + remove_foreign_key(:ci_builds, column: :stage_id) + end + + if index_exists?(:ci_builds, :stage_id) + remove_concurrent_index(:ci_builds, :stage_id) + end + end + + private + + def foreign_key_exists?(table, column) + foreign_keys(:ci_builds).any? do |key| + key.options[:column] == column.to_s + end + end +end |