diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-06-18 11:18:50 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-06-18 11:18:50 +0000 |
commit | 8c7f4e9d5f36cff46365a7f8c4b9c21578c1e781 (patch) | |
tree | a77e7fe7a93de11213032ed4ab1f33a3db51b738 /spec/lib/gitlab/database/partitioning_migration_helpers | |
parent | 00b35af3db1abfe813a778f643dad221aad51fca (diff) | |
download | gitlab-ce-8c7f4e9d5f36cff46365a7f8c4b9c21578c1e781.tar.gz |
Add latest changes from gitlab-org/gitlab@13-1-stable-ee
Diffstat (limited to 'spec/lib/gitlab/database/partitioning_migration_helpers')
-rw-r--r-- | spec/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers_spec.rb | 193 | ||||
-rw-r--r-- | spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb | 289 |
2 files changed, 482 insertions, 0 deletions
diff --git a/spec/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers_spec.rb b/spec/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers_spec.rb new file mode 100644 index 00000000000..9cec77b434d --- /dev/null +++ b/spec/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers_spec.rb @@ -0,0 +1,193 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Database::PartitioningMigrationHelpers::ForeignKeyHelpers do + include TriggerHelpers + + let(:model) do + ActiveRecord::Migration.new.extend(described_class) + end + let_it_be(:connection) { ActiveRecord::Base.connection } + let(:referenced_table) { :issues } + let(:function_name) { '_test_partitioned_foreign_keys_function' } + let(:trigger_name) { '_test_partitioned_foreign_keys_trigger' } + + before do + allow(model).to receive(:puts) + allow(model).to receive(:fk_function_name).and_return(function_name) + allow(model).to receive(:fk_trigger_name).and_return(trigger_name) + end + + describe 'adding a foreign key' do + before do + allow(model).to receive(:transaction_open?).and_return(false) + end + + context 'when the table has no foreign keys' do + it 'creates a trigger function to handle the single cascade' do + model.add_partitioned_foreign_key :issue_assignees, referenced_table + + expect_function_to_contain(function_name, 'delete from issue_assignees where issue_id = old.id') + expect_valid_function_trigger(referenced_table, trigger_name, function_name, after: 'delete') + end + end + + context 'when the table already has foreign keys' do + context 'when the foreign key is from a different table' do + before do + model.add_partitioned_foreign_key :issue_assignees, referenced_table + end + + it 'creates a trigger function to handle the multiple cascades' do + model.add_partitioned_foreign_key :epic_issues, referenced_table + + expect_function_to_contain(function_name, + 'delete from issue_assignees where issue_id = old.id', + 'delete from epic_issues where issue_id = old.id') + expect_valid_function_trigger(referenced_table, trigger_name, function_name, after: 'delete') + end + end + + context 'when the foreign key is from the same table' do + before do + model.add_partitioned_foreign_key :issues, referenced_table, column: :moved_to_id + end + + context 'when the foreign key is from a different column' do + it 'creates a trigger function to handle the multiple cascades' do + model.add_partitioned_foreign_key :issues, referenced_table, column: :duplicated_to_id + + expect_function_to_contain(function_name, + 'delete from issues where moved_to_id = old.id', + 'delete from issues where duplicated_to_id = old.id') + expect_valid_function_trigger(referenced_table, trigger_name, function_name, after: 'delete') + end + end + + context 'when the foreign key is from the same column' do + it 'ignores the duplicate and properly recreates the trigger function' do + model.add_partitioned_foreign_key :issues, referenced_table, column: :moved_to_id + + expect_function_to_contain(function_name, 'delete from issues where moved_to_id = old.id') + expect_valid_function_trigger(referenced_table, trigger_name, function_name, after: 'delete') + end + end + end + end + + context 'when the foreign key is set to nullify' do + it 'creates a trigger function that nullifies the foreign key' do + model.add_partitioned_foreign_key :issue_assignees, referenced_table, on_delete: :nullify + + expect_function_to_contain(function_name, 'update issue_assignees set issue_id = null where issue_id = old.id') + expect_valid_function_trigger(referenced_table, trigger_name, function_name, after: 'delete') + end + end + + context 'when the referencing column is a custom value' do + it 'creates a trigger function with the correct column name' do + model.add_partitioned_foreign_key :issues, referenced_table, column: :duplicated_to_id + + expect_function_to_contain(function_name, 'delete from issues where duplicated_to_id = old.id') + expect_valid_function_trigger(referenced_table, trigger_name, function_name, after: 'delete') + end + end + + context 'when the referenced column is a custom value' do + let(:referenced_table) { :user_details } + + it 'creates a trigger function with the correct column name' do + model.add_partitioned_foreign_key :user_preferences, referenced_table, column: :user_id, primary_key: :user_id + + expect_function_to_contain(function_name, 'delete from user_preferences where user_id = old.user_id') + expect_valid_function_trigger(referenced_table, trigger_name, function_name, after: 'delete') + end + end + + context 'when the given key definition is invalid' do + it 'raises an error with the appropriate message' do + expect do + model.add_partitioned_foreign_key :issue_assignees, referenced_table, column: :not_a_real_issue_id + end.to raise_error(/From column must be a valid column/) + end + end + + context 'when run inside a transaction' do + it 'raises an error' do + expect(model).to receive(:transaction_open?).and_return(true) + + expect do + model.add_partitioned_foreign_key :issue_assignees, referenced_table + end.to raise_error(/can not be run inside a transaction/) + end + end + end + + context 'removing a foreign key' do + before do + allow(model).to receive(:transaction_open?).and_return(false) + end + + context 'when the table has multiple foreign keys' do + before do + model.add_partitioned_foreign_key :issue_assignees, referenced_table + model.add_partitioned_foreign_key :epic_issues, referenced_table + end + + it 'creates a trigger function without the removed cascade' do + expect_function_to_contain(function_name, + 'delete from issue_assignees where issue_id = old.id', + 'delete from epic_issues where issue_id = old.id') + expect_valid_function_trigger(referenced_table, trigger_name, function_name, after: 'delete') + + model.remove_partitioned_foreign_key :issue_assignees, referenced_table + + expect_function_to_contain(function_name, 'delete from epic_issues where issue_id = old.id') + expect_valid_function_trigger(referenced_table, trigger_name, function_name, after: 'delete') + end + end + + context 'when the table has only one remaining foreign key' do + before do + model.add_partitioned_foreign_key :issue_assignees, referenced_table + end + + it 'removes the trigger function altogether' do + expect_function_to_contain(function_name, 'delete from issue_assignees where issue_id = old.id') + expect_valid_function_trigger(referenced_table, trigger_name, function_name, after: 'delete') + + model.remove_partitioned_foreign_key :issue_assignees, referenced_table + + expect_function_not_to_exist(function_name) + expect_trigger_not_to_exist(referenced_table, trigger_name) + end + end + + context 'when the foreign key does not exist' do + before do + model.add_partitioned_foreign_key :issue_assignees, referenced_table + end + + it 'ignores the invalid key and properly recreates the trigger function' do + expect_function_to_contain(function_name, 'delete from issue_assignees where issue_id = old.id') + expect_valid_function_trigger(referenced_table, trigger_name, function_name, after: 'delete') + + model.remove_partitioned_foreign_key :issues, referenced_table, column: :moved_to_id + + expect_function_to_contain(function_name, 'delete from issue_assignees where issue_id = old.id') + expect_valid_function_trigger(referenced_table, trigger_name, function_name, after: 'delete') + end + end + + context 'when run outside a transaction' do + it 'raises an error' do + expect(model).to receive(:transaction_open?).and_return(true) + + expect do + model.remove_partitioned_foreign_key :issue_assignees, referenced_table + end.to raise_error(/can not be run inside a transaction/) + end + end + end +end diff --git a/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb b/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb new file mode 100644 index 00000000000..586b57d2002 --- /dev/null +++ b/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb @@ -0,0 +1,289 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHelpers do + include PartitioningHelpers + include TriggerHelpers + + let(:migration) do + ActiveRecord::Migration.new.extend(described_class) + end + + let_it_be(:connection) { ActiveRecord::Base.connection } + let(:template_table) { :audit_events } + let(:partitioned_table) { '_test_migration_partitioned_table' } + let(:function_name) { '_test_migration_function_name' } + let(:trigger_name) { '_test_migration_trigger_name' } + let(:partition_column) { 'created_at' } + let(:min_date) { Date.new(2019, 12) } + let(:max_date) { Date.new(2020, 3) } + + before do + allow(migration).to receive(:puts) + allow(migration).to receive(:transaction_open?).and_return(false) + allow(migration).to receive(:partitioned_table_name).and_return(partitioned_table) + allow(migration).to receive(:sync_function_name).and_return(function_name) + allow(migration).to receive(:sync_trigger_name).and_return(trigger_name) + allow(migration).to receive(:assert_table_is_whitelisted) + end + + describe '#partition_table_by_date' do + let(:partition_column) { 'created_at' } + let(:old_primary_key) { 'id' } + let(:new_primary_key) { [old_primary_key, partition_column] } + + context 'when the table is not whitelisted' do + let(:template_table) { :this_table_is_not_whitelisted } + + it 'raises an error' do + expect(migration).to receive(:assert_table_is_whitelisted).with(template_table).and_call_original + + expect do + migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date + end.to raise_error(/#{template_table} is not whitelisted for use/) + end + end + + context 'when run inside a transaction block' do + it 'raises an error' do + expect(migration).to receive(:transaction_open?).and_return(true) + + expect do + migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date + end.to raise_error(/can not be run inside a transaction/) + end + end + + context 'when the the max_date is less than the min_date' do + let(:max_date) { Time.utc(2019, 6) } + + it 'raises an error' do + expect do + migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date + end.to raise_error(/max_date #{max_date} must be greater than min_date #{min_date}/) + end + end + + context 'when the max_date is equal to the min_date' do + let(:max_date) { min_date } + + it 'raises an error' do + expect do + migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date + end.to raise_error(/max_date #{max_date} must be greater than min_date #{min_date}/) + end + end + + context 'when the given table does not have a primary key' do + let(:template_table) { :_partitioning_migration_helper_test_table } + let(:partition_column) { :some_field } + + it 'raises an error' do + migration.create_table template_table, id: false do |t| + t.integer :id + t.datetime partition_column + end + + expect do + migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date + end.to raise_error(/primary key not defined for #{template_table}/) + end + end + + context 'when an invalid partition column is given' do + let(:partition_column) { :_this_is_not_real } + + it 'raises an error' do + expect do + migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date + end.to raise_error(/partition column #{partition_column} does not exist/) + end + end + + describe 'constructing the partitioned table' do + it 'creates a table partitioned by the proper column' do + migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date + + expect(connection.table_exists?(partitioned_table)).to be(true) + expect(connection.primary_key(partitioned_table)).to eq(new_primary_key) + + expect_table_partitioned_by(partitioned_table, [partition_column]) + end + + it 'changes the primary key datatype to bigint' do + migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date + + pk_column = connection.columns(partitioned_table).find { |c| c.name == old_primary_key } + + expect(pk_column.sql_type).to eq('bigint') + end + + context 'with a non-integer primary key datatype' do + before do + connection.create_table :another_example, id: false do |t| + t.string :identifier, primary_key: true + t.timestamp :created_at + end + end + + let(:template_table) { :another_example } + let(:old_primary_key) { 'identifier' } + + it 'does not change the primary key datatype' do + migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date + + original_pk_column = connection.columns(template_table).find { |c| c.name == old_primary_key } + pk_column = connection.columns(partitioned_table).find { |c| c.name == old_primary_key } + + expect(pk_column).not_to be_nil + expect(pk_column).to eq(original_pk_column) + end + end + + it 'removes the default from the primary key column' do + migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date + + pk_column = connection.columns(partitioned_table).find { |c| c.name == old_primary_key } + + expect(pk_column.default_function).to be_nil + end + + it 'creates the partitioned table with the same non-key columns' do + migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date + + copied_columns = filter_columns_by_name(connection.columns(partitioned_table), new_primary_key) + original_columns = filter_columns_by_name(connection.columns(template_table), new_primary_key) + + expect(copied_columns).to match_array(original_columns) + end + + it 'creates a partition spanning over each month in the range given' do + migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date + + expect_range_partition_of("#{partitioned_table}_000000", partitioned_table, 'MINVALUE', "'2019-12-01 00:00:00'") + expect_range_partition_of("#{partitioned_table}_201912", partitioned_table, "'2019-12-01 00:00:00'", "'2020-01-01 00:00:00'") + expect_range_partition_of("#{partitioned_table}_202001", partitioned_table, "'2020-01-01 00:00:00'", "'2020-02-01 00:00:00'") + expect_range_partition_of("#{partitioned_table}_202002", partitioned_table, "'2020-02-01 00:00:00'", "'2020-03-01 00:00:00'") + end + end + + describe 'keeping data in sync with the partitioned table' do + let(:template_table) { :todos } + let(:model) { Class.new(ActiveRecord::Base) } + let(:timestamp) { Time.utc(2019, 12, 1, 12).round } + + before do + model.primary_key = :id + model.table_name = partitioned_table + end + + it 'creates a trigger function on the original table' do + expect_function_not_to_exist(function_name) + expect_trigger_not_to_exist(template_table, trigger_name) + + migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date + + expect_function_to_exist(function_name) + expect_valid_function_trigger(template_table, trigger_name, function_name, after: %w[delete insert update]) + end + + it 'syncs inserts to the partitioned tables' do + migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date + + expect(model.count).to eq(0) + + first_todo = create(:todo, created_at: timestamp, updated_at: timestamp) + second_todo = create(:todo, created_at: timestamp, updated_at: timestamp) + + expect(model.count).to eq(2) + expect(model.find(first_todo.id).attributes).to eq(first_todo.attributes) + expect(model.find(second_todo.id).attributes).to eq(second_todo.attributes) + end + + it 'syncs updates to the partitioned tables' do + migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date + + first_todo = create(:todo, :pending, commit_id: nil, created_at: timestamp, updated_at: timestamp) + second_todo = create(:todo, created_at: timestamp, updated_at: timestamp) + + expect(model.count).to eq(2) + + first_copy = model.find(first_todo.id) + second_copy = model.find(second_todo.id) + + expect(first_copy.attributes).to eq(first_todo.attributes) + expect(second_copy.attributes).to eq(second_todo.attributes) + + first_todo.update(state_event: 'done', commit_id: 'abc123', updated_at: timestamp + 1.second) + + expect(model.count).to eq(2) + expect(first_copy.reload.attributes).to eq(first_todo.attributes) + expect(second_copy.reload.attributes).to eq(second_todo.attributes) + end + + it 'syncs deletes to the partitioned tables' do + migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date + + first_todo = create(:todo, created_at: timestamp, updated_at: timestamp) + second_todo = create(:todo, created_at: timestamp, updated_at: timestamp) + + expect(model.count).to eq(2) + + first_todo.destroy + + expect(model.count).to eq(1) + expect(model.find_by_id(first_todo.id)).to be_nil + expect(model.find(second_todo.id).attributes).to eq(second_todo.attributes) + end + end + end + + describe '#drop_partitioned_table_for' do + let(:expected_tables) do + %w[000000 201912 202001 202002].map { |suffix| "#{partitioned_table}_#{suffix}" }.unshift(partitioned_table) + end + + context 'when the table is not whitelisted' do + let(:template_table) { :this_table_is_not_whitelisted } + + it 'raises an error' do + expect(migration).to receive(:assert_table_is_whitelisted).with(template_table).and_call_original + + expect do + migration.drop_partitioned_table_for template_table + end.to raise_error(/#{template_table} is not whitelisted for use/) + end + end + + it 'drops the trigger syncing to the partitioned table' do + migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date + + expect_function_to_exist(function_name) + expect_valid_function_trigger(template_table, trigger_name, function_name, after: %w[delete insert update]) + + migration.drop_partitioned_table_for template_table + + expect_function_not_to_exist(function_name) + expect_trigger_not_to_exist(template_table, trigger_name) + end + + it 'drops the partitioned copy and all partitions' do + migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date + + expected_tables.each do |table| + expect(connection.table_exists?(table)).to be(true) + end + + migration.drop_partitioned_table_for template_table + + expected_tables.each do |table| + expect(connection.table_exists?(table)).to be(false) + end + end + end + + def filter_columns_by_name(columns, names) + columns.reject { |c| names.include?(c.name) } + end +end |