diff options
Diffstat (limited to 'spec/lib/bulk_imports')
15 files changed, 512 insertions, 224 deletions
diff --git a/spec/lib/bulk_imports/common/loaders/entity_loader_spec.rb b/spec/lib/bulk_imports/common/loaders/entity_loader_spec.rb deleted file mode 100644 index 57ffdfa9aee..00000000000 --- a/spec/lib/bulk_imports/common/loaders/entity_loader_spec.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe BulkImports::Common::Loaders::EntityLoader do - describe '#load' do - it "creates entities for the given data" do - group = create(:group, path: "imported-group") - parent_entity = create(:bulk_import_entity, group: group, bulk_import: create(:bulk_import)) - context = BulkImports::Pipeline::Context.new(parent_entity) - - data = { - source_type: :group_entity, - source_full_path: "parent/subgroup", - destination_name: "subgroup", - destination_namespace: parent_entity.group.full_path, - parent_id: parent_entity.id - } - - expect { subject.load(context, data) }.to change(BulkImports::Entity, :count).by(1) - - subgroup_entity = BulkImports::Entity.last - - expect(subgroup_entity.source_full_path).to eq 'parent/subgroup' - expect(subgroup_entity.destination_namespace).to eq 'imported-group' - expect(subgroup_entity.destination_name).to eq 'subgroup' - expect(subgroup_entity.parent_id).to eq parent_entity.id - end - end -end diff --git a/spec/lib/bulk_imports/common/transformers/prohibited_attributes_transformer_spec.rb b/spec/lib/bulk_imports/common/transformers/prohibited_attributes_transformer_spec.rb index 03d138b227c..08a82bc84ed 100644 --- a/spec/lib/bulk_imports/common/transformers/prohibited_attributes_transformer_spec.rb +++ b/spec/lib/bulk_imports/common/transformers/prohibited_attributes_transformer_spec.rb @@ -68,5 +68,11 @@ RSpec.describe BulkImports::Common::Transformers::ProhibitedAttributesTransforme expect(transformed_hash).to eq(expected_hash) end + + context 'when there is no data to transform' do + it 'returns' do + expect(subject.transform(nil, nil)).to be_nil + end + end end end diff --git a/spec/lib/bulk_imports/common/transformers/award_emoji_transformer_spec.rb b/spec/lib/bulk_imports/common/transformers/user_reference_transformer_spec.rb index 5b560a30bf5..ff11a10bfe9 100644 --- a/spec/lib/bulk_imports/common/transformers/award_emoji_transformer_spec.rb +++ b/spec/lib/bulk_imports/common/transformers/user_reference_transformer_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe BulkImports::Common::Transformers::AwardEmojiTransformer do +RSpec.describe BulkImports::Common::Transformers::UserReferenceTransformer do describe '#transform' do let_it_be(:user) { create(:user) } let_it_be(:group) { create(:group) } @@ -12,7 +12,6 @@ RSpec.describe BulkImports::Common::Transformers::AwardEmojiTransformer do let(:hash) do { - 'name' => 'thumbs up', 'user' => { 'public_email' => email } @@ -44,5 +43,27 @@ RSpec.describe BulkImports::Common::Transformers::AwardEmojiTransformer do include_examples 'sets user_id and removes user key' end + + context 'when there is no data to transform' do + it 'returns' do + expect(subject.transform(nil, nil)).to be_nil + end + end + + context 'when custom reference is provided' do + it 'updates provided reference' do + hash = { + 'author' => { + 'public_email' => user.email + } + } + + transformer = described_class.new(reference: 'author') + result = transformer.transform(context, hash) + + expect(result['author']).to be_nil + expect(result['author_id']).to eq(user.id) + end + end end end diff --git a/spec/lib/bulk_imports/groups/graphql/get_labels_query_spec.rb b/spec/lib/bulk_imports/groups/graphql/get_labels_query_spec.rb index 247da200d68..85f82be7d18 100644 --- a/spec/lib/bulk_imports/groups/graphql/get_labels_query_spec.rb +++ b/spec/lib/bulk_imports/groups/graphql/get_labels_query_spec.rb @@ -3,15 +3,18 @@ require 'spec_helper' RSpec.describe BulkImports::Groups::Graphql::GetLabelsQuery do - describe '#variables' do - let(:entity) { double(source_full_path: 'test', next_page_for: 'next_page', bulk_import: nil) } - let(:context) { BulkImports::Pipeline::Context.new(entity) } - - it 'returns query variables based on entity information' do - expected = { full_path: entity.source_full_path, cursor: entity.next_page_for } - - expect(described_class.variables(context)).to eq(expected) - end + it 'has a valid query' do + entity = create(:bulk_import_entity) + context = BulkImports::Pipeline::Context.new(entity) + + query = GraphQL::Query.new( + GitlabSchema, + described_class.to_s, + variables: described_class.variables(context) + ) + result = GitlabSchema.static_validator.validate(query) + + expect(result[:errors]).to be_empty end describe '#data_path' do diff --git a/spec/lib/bulk_imports/groups/graphql/get_milestones_query_spec.rb b/spec/lib/bulk_imports/groups/graphql/get_milestones_query_spec.rb new file mode 100644 index 00000000000..a38505fbf85 --- /dev/null +++ b/spec/lib/bulk_imports/groups/graphql/get_milestones_query_spec.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::Groups::Graphql::GetMilestonesQuery do + it 'has a valid query' do + entity = create(:bulk_import_entity) + context = BulkImports::Pipeline::Context.new(entity) + + query = GraphQL::Query.new( + GitlabSchema, + described_class.to_s, + variables: described_class.variables(context) + ) + result = GitlabSchema.static_validator.validate(query) + + expect(result[:errors]).to be_empty + end + + describe '#data_path' do + it 'returns data path' do + expected = %w[data group milestones nodes] + + expect(described_class.data_path).to eq(expected) + end + end + + describe '#page_info_path' do + it 'returns pagination information path' do + expected = %w[data group milestones page_info] + + expect(described_class.page_info_path).to eq(expected) + end + end +end diff --git a/spec/lib/bulk_imports/groups/loaders/labels_loader_spec.rb b/spec/lib/bulk_imports/groups/loaders/labels_loader_spec.rb deleted file mode 100644 index ac2f9c8cb1d..00000000000 --- a/spec/lib/bulk_imports/groups/loaders/labels_loader_spec.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe BulkImports::Groups::Loaders::LabelsLoader do - describe '#load' do - let(:user) { create(:user) } - let(:group) { create(:group) } - let(:entity) { create(:bulk_import_entity, group: group) } - let(:context) { BulkImports::Pipeline::Context.new(entity) } - - let(:data) do - { - 'title' => 'label', - 'description' => 'description', - 'color' => '#FFFFFF' - } - end - - it 'creates the label' do - expect { subject.load(context, data) }.to change(Label, :count).by(1) - - label = group.labels.first - - expect(label.title).to eq(data['title']) - expect(label.description).to eq(data['description']) - expect(label.color).to eq(data['color']) - end - end -end diff --git a/spec/lib/bulk_imports/groups/loaders/members_loader_spec.rb b/spec/lib/bulk_imports/groups/loaders/members_loader_spec.rb deleted file mode 100644 index d552578e7be..00000000000 --- a/spec/lib/bulk_imports/groups/loaders/members_loader_spec.rb +++ /dev/null @@ -1,42 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe BulkImports::Groups::Loaders::MembersLoader do - describe '#load' do - let_it_be(:user_importer) { create(:user) } - let_it_be(:user_member) { create(:user) } - let_it_be(:group) { create(:group) } - let_it_be(:bulk_import) { create(:bulk_import, user: user_importer) } - let_it_be(:entity) { create(:bulk_import_entity, bulk_import: bulk_import, group: group) } - let_it_be(:context) { BulkImports::Pipeline::Context.new(entity) } - - let_it_be(:data) do - { - 'user_id' => user_member.id, - 'created_by_id' => user_importer.id, - 'access_level' => 30, - 'created_at' => '2020-01-01T00:00:00Z', - 'updated_at' => '2020-01-01T00:00:00Z', - 'expires_at' => nil - } - end - - it 'does nothing when there is no data' do - expect { subject.load(context, nil) }.not_to change(GroupMember, :count) - end - - it 'creates the member' do - expect { subject.load(context, data) }.to change(GroupMember, :count).by(1) - - member = group.members.last - - expect(member.user).to eq(user_member) - expect(member.created_by).to eq(user_importer) - expect(member.access_level).to eq(30) - expect(member.created_at).to eq('2020-01-01T00:00:00Z') - expect(member.updated_at).to eq('2020-01-01T00:00:00Z') - expect(member.expires_at).to eq(nil) - end - end -end diff --git a/spec/lib/bulk_imports/groups/pipelines/labels_pipeline_spec.rb b/spec/lib/bulk_imports/groups/pipelines/labels_pipeline_spec.rb index 63f28916d9a..3327a30f1d5 100644 --- a/spec/lib/bulk_imports/groups/pipelines/labels_pipeline_spec.rb +++ b/spec/lib/bulk_imports/groups/pipelines/labels_pipeline_spec.rb @@ -6,6 +6,7 @@ RSpec.describe BulkImports::Groups::Pipelines::LabelsPipeline do let(:user) { create(:user) } let(:group) { create(:group) } let(:cursor) { 'cursor' } + let(:timestamp) { Time.new(2020, 01, 01).utc } let(:entity) do create( :bulk_import_entity, @@ -20,21 +21,23 @@ RSpec.describe BulkImports::Groups::Pipelines::LabelsPipeline do subject { described_class.new(context) } - def extractor_data(title:, has_next_page:, cursor: nil) - data = [ - { - 'title' => title, - 'description' => 'desc', - 'color' => '#428BCA' - } - ] + def label_data(title) + { + 'title' => title, + 'description' => 'desc', + 'color' => '#428BCA', + 'created_at' => timestamp.to_s, + 'updated_at' => timestamp.to_s + } + end + def extractor_data(title:, has_next_page:, cursor: nil) page_info = { 'end_cursor' => cursor, 'has_next_page' => has_next_page } - BulkImports::Pipeline::ExtractedData.new(data: data, page_info: page_info) + BulkImports::Pipeline::ExtractedData.new(data: [label_data(title)], page_info: page_info) end describe '#run' do @@ -55,6 +58,8 @@ RSpec.describe BulkImports::Groups::Pipelines::LabelsPipeline do expect(label.title).to eq('label2') expect(label.description).to eq('desc') expect(label.color).to eq('#428BCA') + expect(label.created_at).to eq(timestamp) + expect(label.updated_at).to eq(timestamp) end end @@ -90,6 +95,20 @@ RSpec.describe BulkImports::Groups::Pipelines::LabelsPipeline do end end + describe '#load' do + it 'creates the label' do + data = label_data('label') + + expect { subject.load(context, data) }.to change(Label, :count).by(1) + + label = group.labels.first + + data.each do |key, value| + expect(label[key]).to eq(value) + end + end + end + describe 'pipeline parts' do it { expect(described_class).to include_module(BulkImports::Pipeline) } it { expect(described_class).to include_module(BulkImports::Pipeline::Runner) } @@ -110,9 +129,5 @@ RSpec.describe BulkImports::Groups::Pipelines::LabelsPipeline do { klass: BulkImports::Common::Transformers::ProhibitedAttributesTransformer, options: nil } ) end - - it 'has loaders' do - expect(described_class.get_loader).to eq(klass: BulkImports::Groups::Loaders::LabelsLoader, options: nil) - end end end diff --git a/spec/lib/bulk_imports/groups/pipelines/members_pipeline_spec.rb b/spec/lib/bulk_imports/groups/pipelines/members_pipeline_spec.rb index 9f498f8154f..74d3e09d263 100644 --- a/spec/lib/bulk_imports/groups/pipelines/members_pipeline_spec.rb +++ b/spec/lib/bulk_imports/groups/pipelines/members_pipeline_spec.rb @@ -37,6 +37,34 @@ RSpec.describe BulkImports::Groups::Pipelines::MembersPipeline do end end + describe '#load' do + it 'does nothing when there is no data' do + expect { subject.load(context, nil) }.not_to change(GroupMember, :count) + end + + it 'creates the member' do + data = { + 'user_id' => member_user1.id, + 'created_by_id' => member_user2.id, + 'access_level' => 30, + 'created_at' => '2020-01-01T00:00:00Z', + 'updated_at' => '2020-01-01T00:00:00Z', + 'expires_at' => nil + } + + expect { subject.load(context, data) }.to change(GroupMember, :count).by(1) + + member = group.members.last + + expect(member.user).to eq(member_user1) + expect(member.created_by).to eq(member_user2) + expect(member.access_level).to eq(30) + expect(member.created_at).to eq('2020-01-01T00:00:00Z') + expect(member.updated_at).to eq('2020-01-01T00:00:00Z') + expect(member.expires_at).to eq(nil) + end + end + describe 'pipeline parts' do it { expect(described_class).to include_module(BulkImports::Pipeline) } it { expect(described_class).to include_module(BulkImports::Pipeline::Runner) } @@ -58,10 +86,6 @@ RSpec.describe BulkImports::Groups::Pipelines::MembersPipeline do { klass: BulkImports::Groups::Transformers::MemberAttributesTransformer, options: nil } ) end - - it 'has loaders' do - expect(described_class.get_loader).to eq(klass: BulkImports::Groups::Loaders::MembersLoader, options: nil) - end end def member_data(email:, has_next_page:, cursor: nil) diff --git a/spec/lib/bulk_imports/groups/pipelines/milestones_pipeline_spec.rb b/spec/lib/bulk_imports/groups/pipelines/milestones_pipeline_spec.rb new file mode 100644 index 00000000000..f0c34c65257 --- /dev/null +++ b/spec/lib/bulk_imports/groups/pipelines/milestones_pipeline_spec.rb @@ -0,0 +1,151 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::Groups::Pipelines::MilestonesPipeline do + let_it_be(:user) { create(:user) } + let_it_be(:group) { create(:group) } + let_it_be(:cursor) { 'cursor' } + let_it_be(:timestamp) { Time.new(2020, 01, 01).utc } + let_it_be(:bulk_import) { create(:bulk_import, user: user) } + + let(:entity) do + create( + :bulk_import_entity, + bulk_import: bulk_import, + source_full_path: 'source/full/path', + destination_name: 'My Destination Group', + destination_namespace: group.full_path, + group: group + ) + end + + let(:context) { BulkImports::Pipeline::Context.new(entity) } + + subject { described_class.new(context) } + + def milestone_data(title) + { + 'title' => title, + 'description' => 'desc', + 'state' => 'closed', + 'start_date' => '2020-10-21', + 'due_date' => '2020-10-22', + 'created_at' => timestamp.to_s, + 'updated_at' => timestamp.to_s + } + end + + def extracted_data(title:, has_next_page:, cursor: nil) + page_info = { + 'end_cursor' => cursor, + 'has_next_page' => has_next_page + } + + BulkImports::Pipeline::ExtractedData.new(data: [milestone_data(title)], page_info: page_info) + end + + before do + group.add_owner(user) + end + + describe '#run' do + it 'imports group milestones' do + first_page = extracted_data(title: 'milestone1', has_next_page: true, cursor: cursor) + last_page = extracted_data(title: 'milestone2', has_next_page: false) + + allow_next_instance_of(BulkImports::Common::Extractors::GraphqlExtractor) do |extractor| + allow(extractor) + .to receive(:extract) + .and_return(first_page, last_page) + end + + expect { subject.run }.to change(Milestone, :count).by(2) + + expect(group.milestones.pluck(:title)).to contain_exactly('milestone1', 'milestone2') + + milestone = group.milestones.last + + expect(milestone.description).to eq('desc') + expect(milestone.state).to eq('closed') + expect(milestone.start_date.to_s).to eq('2020-10-21') + expect(milestone.due_date.to_s).to eq('2020-10-22') + expect(milestone.created_at).to eq(timestamp) + expect(milestone.updated_at).to eq(timestamp) + end + end + + describe '#after_run' do + context 'when extracted data has next page' do + it 'updates tracker information and runs pipeline again' do + data = extracted_data(title: 'milestone', has_next_page: true, cursor: cursor) + + expect(subject).to receive(:run) + + subject.after_run(data) + + tracker = entity.trackers.find_by(relation: :milestones) + + expect(tracker.has_next_page).to eq(true) + expect(tracker.next_page).to eq(cursor) + end + end + + context 'when extracted data has no next page' do + it 'updates tracker information and does not run pipeline' do + data = extracted_data(title: 'milestone', has_next_page: false) + + expect(subject).not_to receive(:run) + + subject.after_run(data) + + tracker = entity.trackers.find_by(relation: :milestones) + + expect(tracker.has_next_page).to eq(false) + expect(tracker.next_page).to be_nil + end + end + end + + describe '#load' do + it 'creates the milestone' do + data = milestone_data('milestone') + + expect { subject.load(context, data) }.to change(Milestone, :count).by(1) + end + + context 'when user is not authorized to create the milestone' do + before do + allow(user).to receive(:can?).with(:admin_milestone, group).and_return(false) + end + + it 'raises NotAllowedError' do + data = extracted_data(title: 'milestone', has_next_page: false) + + expect { subject.load(context, data) }.to raise_error(::BulkImports::Pipeline::NotAllowedError) + end + end + end + + describe 'pipeline parts' do + it { expect(described_class).to include_module(BulkImports::Pipeline) } + it { expect(described_class).to include_module(BulkImports::Pipeline::Runner) } + + it 'has extractors' do + expect(described_class.get_extractor) + .to eq( + klass: BulkImports::Common::Extractors::GraphqlExtractor, + options: { + query: BulkImports::Groups::Graphql::GetMilestonesQuery + } + ) + end + + it 'has transformers' do + expect(described_class.transformers) + .to contain_exactly( + { klass: BulkImports::Common::Transformers::ProhibitedAttributesTransformer, options: nil } + ) + end + end +end diff --git a/spec/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline_spec.rb b/spec/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline_spec.rb index 0404c52b895..2a99646bb4a 100644 --- a/spec/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline_spec.rb +++ b/spec/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline_spec.rb @@ -3,9 +3,14 @@ require 'spec_helper' RSpec.describe BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline do + let_it_be(:user) { create(:user) } + let_it_be(:group) { create(:group, path: 'group') } + let_it_be(:parent) { create(:group, name: 'imported-group', path: 'imported-group') } + let(:context) { BulkImports::Pipeline::Context.new(parent_entity) } + + subject { described_class.new(context) } + describe '#run' do - let_it_be(:user) { create(:user) } - let(:parent) { create(:group, name: 'imported-group', path: 'imported-group') } let!(:parent_entity) do create( :bulk_import_entity, @@ -14,8 +19,6 @@ RSpec.describe BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline do ) end - let(:context) { BulkImports::Pipeline::Context.new(parent_entity) } - let(:subgroup_data) do [ { @@ -25,8 +28,6 @@ RSpec.describe BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline do ] end - subject { described_class.new(context) } - before do allow_next_instance_of(BulkImports::Groups::Extractors::SubgroupsExtractor) do |extractor| allow(extractor).to receive(:extract).and_return(subgroup_data) @@ -47,6 +48,29 @@ RSpec.describe BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline do end end + describe '#load' do + let(:parent_entity) { create(:bulk_import_entity, group: group, bulk_import: create(:bulk_import)) } + + it 'creates entities for the given data' do + data = { + source_type: :group_entity, + source_full_path: 'parent/subgroup', + destination_name: 'subgroup', + destination_namespace: parent_entity.group.full_path, + parent_id: parent_entity.id + } + + expect { subject.load(context, data) }.to change(BulkImports::Entity, :count).by(1) + + subgroup_entity = BulkImports::Entity.last + + expect(subgroup_entity.source_full_path).to eq 'parent/subgroup' + expect(subgroup_entity.destination_namespace).to eq 'group' + expect(subgroup_entity.destination_name).to eq 'subgroup' + expect(subgroup_entity.parent_id).to eq parent_entity.id + end + end + describe 'pipeline parts' do it { expect(described_class).to include_module(BulkImports::Pipeline) } it { expect(described_class).to include_module(BulkImports::Pipeline::Runner) } @@ -61,9 +85,5 @@ RSpec.describe BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline do { klass: BulkImports::Groups::Transformers::SubgroupToEntityTransformer, options: nil } ) end - - it 'has loaders' do - expect(described_class.get_loader).to eq(klass: BulkImports::Common::Loaders::EntityLoader, options: nil) - end end end diff --git a/spec/lib/bulk_imports/groups/transformers/group_attributes_transformer_spec.rb b/spec/lib/bulk_imports/groups/transformers/group_attributes_transformer_spec.rb index 5a7a51675d6..b3fe8a2ba25 100644 --- a/spec/lib/bulk_imports/groups/transformers/group_attributes_transformer_spec.rb +++ b/spec/lib/bulk_imports/groups/transformers/group_attributes_transformer_spec.rb @@ -80,14 +80,14 @@ RSpec.describe BulkImports::Groups::Transformers::GroupAttributesTransformer do expect(transformed_data['parent_id']).to eq(parent.id) end - context 'when destination namespace is user namespace' do + context 'when destination namespace is empty' do it 'does not set parent id' do entity = create( :bulk_import_entity, bulk_import: bulk_import, source_full_path: 'source/full/path', destination_name: group.name, - destination_namespace: user.namespace.full_path + destination_namespace: '' ) context = BulkImports::Pipeline::Context.new(entity) diff --git a/spec/lib/bulk_imports/importers/group_importer_spec.rb b/spec/lib/bulk_imports/importers/group_importer_spec.rb index b4fdb7b5e5b..5d501b49e41 100644 --- a/spec/lib/bulk_imports/importers/group_importer_spec.rb +++ b/spec/lib/bulk_imports/importers/group_importer_spec.rb @@ -22,10 +22,13 @@ RSpec.describe BulkImports::Importers::GroupImporter do expect_to_run_pipeline BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline, context: context expect_to_run_pipeline BulkImports::Groups::Pipelines::MembersPipeline, context: context expect_to_run_pipeline BulkImports::Groups::Pipelines::LabelsPipeline, context: context + expect_to_run_pipeline BulkImports::Groups::Pipelines::MilestonesPipeline, context: context if Gitlab.ee? expect_to_run_pipeline('EE::BulkImports::Groups::Pipelines::EpicsPipeline'.constantize, context: context) expect_to_run_pipeline('EE::BulkImports::Groups::Pipelines::EpicAwardEmojiPipeline'.constantize, context: context) + expect_to_run_pipeline('EE::BulkImports::Groups::Pipelines::EpicEventsPipeline'.constantize, context: context) + expect_to_run_pipeline('EE::BulkImports::Groups::Pipelines::IterationsPipeline'.constantize, context: context) end subject.execute diff --git a/spec/lib/bulk_imports/pipeline/runner_spec.rb b/spec/lib/bulk_imports/pipeline/runner_spec.rb index 76e4e64a7d6..59f01c9caaa 100644 --- a/spec/lib/bulk_imports/pipeline/runner_spec.rb +++ b/spec/lib/bulk_imports/pipeline/runner_spec.rb @@ -27,29 +27,31 @@ RSpec.describe BulkImports::Pipeline::Runner do end end - describe 'pipeline runner' do - before do - stub_const('BulkImports::Extractor', extractor) - stub_const('BulkImports::Transformer', transformer) - stub_const('BulkImports::Loader', loader) - - pipeline = Class.new do - include BulkImports::Pipeline + before do + stub_const('BulkImports::Extractor', extractor) + stub_const('BulkImports::Transformer', transformer) + stub_const('BulkImports::Loader', loader) - extractor BulkImports::Extractor - transformer BulkImports::Transformer - loader BulkImports::Loader + pipeline = Class.new do + include BulkImports::Pipeline - def after_run(_); end - end + extractor BulkImports::Extractor + transformer BulkImports::Transformer + loader BulkImports::Loader - stub_const('BulkImports::MyPipeline', pipeline) + def after_run(_); end end - context 'when entity is not marked as failed' do - let(:entity) { create(:bulk_import_entity) } - let(:context) { BulkImports::Pipeline::Context.new(entity) } + stub_const('BulkImports::MyPipeline', pipeline) + end + let_it_be_with_refind(:entity) { create(:bulk_import_entity) } + let(:context) { BulkImports::Pipeline::Context.new(entity, extra: :data) } + + subject { BulkImports::MyPipeline.new(context) } + + describe 'pipeline runner' do + context 'when entity is not marked as failed' do it 'runs pipeline extractor, transformer, loader' do extracted_data = BulkImports::Pipeline::ExtractedData.new(data: { foo: :bar }) @@ -76,58 +78,61 @@ RSpec.describe BulkImports::Pipeline::Runner do expect_next_instance_of(Gitlab::Import::Logger) do |logger| expect(logger).to receive(:info) .with( - bulk_import_entity_id: entity.id, - bulk_import_entity_type: 'group_entity', - message: 'Pipeline started', - pipeline_class: 'BulkImports::MyPipeline' + log_params( + context, + message: 'Pipeline started', + pipeline_class: 'BulkImports::MyPipeline' + ) ) expect(logger).to receive(:info) .with( - bulk_import_entity_id: entity.id, - bulk_import_entity_type: 'group_entity', - pipeline_class: 'BulkImports::MyPipeline', - pipeline_step: :extractor, - step_class: 'BulkImports::Extractor' + log_params( + context, + pipeline_class: 'BulkImports::MyPipeline', + pipeline_step: :extractor, + step_class: 'BulkImports::Extractor' + ) ) expect(logger).to receive(:info) .with( - bulk_import_entity_id: entity.id, - bulk_import_entity_type: 'group_entity', - pipeline_class: 'BulkImports::MyPipeline', - pipeline_step: :transformer, - step_class: 'BulkImports::Transformer' + log_params( + context, + pipeline_class: 'BulkImports::MyPipeline', + pipeline_step: :transformer, + step_class: 'BulkImports::Transformer' + ) ) expect(logger).to receive(:info) .with( - bulk_import_entity_id: entity.id, - bulk_import_entity_type: 'group_entity', - pipeline_class: 'BulkImports::MyPipeline', - pipeline_step: :loader, - step_class: 'BulkImports::Loader' + log_params( + context, + pipeline_class: 'BulkImports::MyPipeline', + pipeline_step: :loader, + step_class: 'BulkImports::Loader' + ) ) expect(logger).to receive(:info) .with( - bulk_import_entity_id: entity.id, - bulk_import_entity_type: 'group_entity', - pipeline_class: 'BulkImports::MyPipeline', - pipeline_step: :after_run + log_params( + context, + pipeline_class: 'BulkImports::MyPipeline', + pipeline_step: :after_run + ) ) expect(logger).to receive(:info) .with( - bulk_import_entity_id: entity.id, - bulk_import_entity_type: 'group_entity', - message: 'Pipeline finished', - pipeline_class: 'BulkImports::MyPipeline' + log_params( + context, + message: 'Pipeline finished', + pipeline_class: 'BulkImports::MyPipeline' + ) ) end - BulkImports::MyPipeline.new(context).run + subject.run end context 'when exception is raised' do - let(:entity) { create(:bulk_import_entity, :created) } - let(:context) { BulkImports::Pipeline::Context.new(entity) } - before do allow_next_instance_of(BulkImports::Extractor) do |extractor| allow(extractor).to receive(:extract).with(context).and_raise(StandardError, 'Error!') @@ -135,7 +140,21 @@ RSpec.describe BulkImports::Pipeline::Runner do end it 'logs import failure' do - BulkImports::MyPipeline.new(context).run + expect_next_instance_of(Gitlab::Import::Logger) do |logger| + expect(logger).to receive(:error) + .with( + log_params( + context, + pipeline_step: :extractor, + pipeline_class: 'BulkImports::MyPipeline', + exception_class: 'StandardError', + exception_message: 'Error!' + ) + ) + end + + expect { subject.run } + .to change(entity.failures, :count).by(1) failure = entity.failures.first @@ -152,29 +171,29 @@ RSpec.describe BulkImports::Pipeline::Runner do end it 'marks entity as failed' do - BulkImports::MyPipeline.new(context).run - - expect(entity.failed?).to eq(true) + expect { subject.run } + .to change(entity, :status_name).to(:failed) end it 'logs warn message' do expect_next_instance_of(Gitlab::Import::Logger) do |logger| expect(logger).to receive(:warn) .with( - message: 'Pipeline failed', - pipeline_class: 'BulkImports::MyPipeline', - bulk_import_entity_id: entity.id, - bulk_import_entity_type: entity.source_type + log_params( + context, + message: 'Pipeline failed', + pipeline_class: 'BulkImports::MyPipeline' + ) ) end - BulkImports::MyPipeline.new(context).run + subject.run end end context 'when pipeline is not marked to abort on failure' do - it 'marks entity as failed' do - BulkImports::MyPipeline.new(context).run + it 'does not mark entity as failed' do + subject.run expect(entity.failed?).to eq(false) end @@ -183,24 +202,31 @@ RSpec.describe BulkImports::Pipeline::Runner do end context 'when entity is marked as failed' do - let(:entity) { create(:bulk_import_entity) } - let(:context) { BulkImports::Pipeline::Context.new(entity) } - it 'logs and returns without execution' do - allow(entity).to receive(:failed?).and_return(true) + entity.fail_op! expect_next_instance_of(Gitlab::Import::Logger) do |logger| expect(logger).to receive(:info) .with( - message: 'Skipping due to failed pipeline status', - pipeline_class: 'BulkImports::MyPipeline', - bulk_import_entity_id: entity.id, - bulk_import_entity_type: 'group_entity' + log_params( + context, + message: 'Skipping due to failed pipeline status', + pipeline_class: 'BulkImports::MyPipeline' + ) ) end - BulkImports::MyPipeline.new(context).run + subject.run end end end + + def log_params(context, extra = {}) + { + bulk_import_id: context.bulk_import.id, + bulk_import_entity_id: context.entity.id, + bulk_import_entity_type: context.entity.source_type, + context_extra: context.extra + }.merge(extra) + end end diff --git a/spec/lib/bulk_imports/pipeline_spec.rb b/spec/lib/bulk_imports/pipeline_spec.rb index 3811a02a7fd..c882e3d26ea 100644 --- a/spec/lib/bulk_imports/pipeline_spec.rb +++ b/spec/lib/bulk_imports/pipeline_spec.rb @@ -3,25 +3,25 @@ require 'spec_helper' RSpec.describe BulkImports::Pipeline do - describe 'pipeline attributes' do - before do - stub_const('BulkImports::Extractor', Class.new) - stub_const('BulkImports::Transformer', Class.new) - stub_const('BulkImports::Loader', Class.new) - - klass = Class.new do - include BulkImports::Pipeline + before do + stub_const('BulkImports::Extractor', Class.new) + stub_const('BulkImports::Transformer', Class.new) + stub_const('BulkImports::Loader', Class.new) - abort_on_failure! + klass = Class.new do + include BulkImports::Pipeline - extractor BulkImports::Extractor, { foo: :bar } - transformer BulkImports::Transformer, { foo: :bar } - loader BulkImports::Loader, { foo: :bar } - end + abort_on_failure! - stub_const('BulkImports::MyPipeline', klass) + extractor BulkImports::Extractor, foo: :bar + transformer BulkImports::Transformer, foo: :bar + loader BulkImports::Loader, foo: :bar end + stub_const('BulkImports::MyPipeline', klass) + end + + describe 'pipeline attributes' do describe 'getters' do it 'retrieves class attributes' do expect(BulkImports::MyPipeline.get_extractor).to eq({ klass: BulkImports::Extractor, options: { foo: :bar } }) @@ -29,6 +29,27 @@ RSpec.describe BulkImports::Pipeline do expect(BulkImports::MyPipeline.get_loader).to eq({ klass: BulkImports::Loader, options: { foo: :bar } }) expect(BulkImports::MyPipeline.abort_on_failure?).to eq(true) end + + context 'when extractor and loader are defined within the pipeline' do + before do + klass = Class.new do + include BulkImports::Pipeline + + def extract; end + + def load; end + end + + stub_const('BulkImports::AnotherPipeline', klass) + end + + it 'returns itself when retrieving extractor & loader' do + pipeline = BulkImports::AnotherPipeline.new(nil) + + expect(pipeline.send(:extractor)).to eq(pipeline) + expect(pipeline.send(:loader)).to eq(pipeline) + end + end end describe 'setters' do @@ -54,4 +75,69 @@ RSpec.describe BulkImports::Pipeline do end end end + + describe '#instantiate' do + context 'when options are present' do + it 'instantiates new object with options' do + expect(BulkImports::Extractor).to receive(:new).with(foo: :bar) + expect(BulkImports::Transformer).to receive(:new).with(foo: :bar) + expect(BulkImports::Loader).to receive(:new).with(foo: :bar) + + pipeline = BulkImports::MyPipeline.new(nil) + + pipeline.send(:extractor) + pipeline.send(:transformers) + pipeline.send(:loader) + end + end + + context 'when options are missing' do + before do + klass = Class.new do + include BulkImports::Pipeline + + extractor BulkImports::Extractor + transformer BulkImports::Transformer + loader BulkImports::Loader + end + + stub_const('BulkImports::NoOptionsPipeline', klass) + end + + it 'instantiates new object without options' do + expect(BulkImports::Extractor).to receive(:new).with(no_args) + expect(BulkImports::Transformer).to receive(:new).with(no_args) + expect(BulkImports::Loader).to receive(:new).with(no_args) + + pipeline = BulkImports::NoOptionsPipeline.new(nil) + + pipeline.send(:extractor) + pipeline.send(:transformers) + pipeline.send(:loader) + end + end + end + + describe '#transformers' do + before do + klass = Class.new do + include BulkImports::Pipeline + + transformer BulkImports::Transformer + + def transform; end + end + + stub_const('BulkImports::TransformersPipeline', klass) + end + + it 'has instance transform method first to run' do + transformer = double + allow(BulkImports::Transformer).to receive(:new).and_return(transformer) + + pipeline = BulkImports::TransformersPipeline.new(nil) + + expect(pipeline.send(:transformers)).to eq([pipeline, transformer]) + end + end end |