summaryrefslogtreecommitdiff
path: root/lib/bulk_imports
diff options
context:
space:
mode:
Diffstat (limited to 'lib/bulk_imports')
-rw-r--r--lib/bulk_imports/common/loaders/entity_loader.rb15
-rw-r--r--lib/bulk_imports/common/transformers/award_emoji_transformer.rb2
-rw-r--r--lib/bulk_imports/common/transformers/prohibited_attributes_transformer.rb4
-rw-r--r--lib/bulk_imports/groups/extractors/subgroups_extractor.rb2
-rw-r--r--lib/bulk_imports/groups/graphql/get_labels_query.rb4
-rw-r--r--lib/bulk_imports/groups/loaders/group_loader.rb4
-rw-r--r--lib/bulk_imports/groups/loaders/labels_loader.rb15
-rw-r--r--lib/bulk_imports/groups/loaders/members_loader.rb17
-rw-r--r--lib/bulk_imports/groups/pipelines/labels_pipeline.rb4
-rw-r--r--lib/bulk_imports/groups/pipelines/members_pipeline.rb6
-rw-r--r--lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline.rb5
-rw-r--r--lib/bulk_imports/groups/transformers/group_attributes_transformer.rb13
-rw-r--r--lib/bulk_imports/groups/transformers/member_attributes_transformer.rb2
-rw-r--r--lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb2
-rw-r--r--lib/bulk_imports/pipeline.rb74
15 files changed, 90 insertions, 79 deletions
diff --git a/lib/bulk_imports/common/loaders/entity_loader.rb b/lib/bulk_imports/common/loaders/entity_loader.rb
deleted file mode 100644
index 8644f3c9dcb..00000000000
--- a/lib/bulk_imports/common/loaders/entity_loader.rb
+++ /dev/null
@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module BulkImports
- module Common
- module Loaders
- class EntityLoader
- def initialize(*args); end
-
- def load(context, entity)
- context.bulk_import.entities.create!(entity)
- end
- end
- end
- end
-end
diff --git a/lib/bulk_imports/common/transformers/award_emoji_transformer.rb b/lib/bulk_imports/common/transformers/award_emoji_transformer.rb
index 260b47ab917..676c71f1316 100644
--- a/lib/bulk_imports/common/transformers/award_emoji_transformer.rb
+++ b/lib/bulk_imports/common/transformers/award_emoji_transformer.rb
@@ -4,8 +4,6 @@ module BulkImports
module Common
module Transformers
class AwardEmojiTransformer
- def initialize(*args); end
-
def transform(context, data)
user = find_user(context, data&.dig('user', 'public_email')) || context.current_user
diff --git a/lib/bulk_imports/common/transformers/prohibited_attributes_transformer.rb b/lib/bulk_imports/common/transformers/prohibited_attributes_transformer.rb
index 858c4c8976b..37b64923d56 100644
--- a/lib/bulk_imports/common/transformers/prohibited_attributes_transformer.rb
+++ b/lib/bulk_imports/common/transformers/prohibited_attributes_transformer.rb
@@ -14,10 +14,6 @@ module BulkImports
/\Aremote_\w+_(url|urls|request_header)\Z/ # carrierwave automatically creates these attribute methods for uploads
).freeze
- def initialize(options = {})
- @options = options
- end
-
def transform(context, data)
data.each_with_object({}) do |(key, value), result|
prohibited = prohibited_key?(key)
diff --git a/lib/bulk_imports/groups/extractors/subgroups_extractor.rb b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb
index b01fb6f68ac..e5e2b9fdbd4 100644
--- a/lib/bulk_imports/groups/extractors/subgroups_extractor.rb
+++ b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb
@@ -4,8 +4,6 @@ module BulkImports
module Groups
module Extractors
class SubgroupsExtractor
- def initialize(*args); end
-
def extract(context)
encoded_parent_path = ERB::Util.url_encode(context.entity.source_full_path)
diff --git a/lib/bulk_imports/groups/graphql/get_labels_query.rb b/lib/bulk_imports/groups/graphql/get_labels_query.rb
index d1fe791c2ce..23efbc33581 100644
--- a/lib/bulk_imports/groups/graphql/get_labels_query.rb
+++ b/lib/bulk_imports/groups/graphql/get_labels_query.rb
@@ -10,7 +10,7 @@ module BulkImports
<<-'GRAPHQL'
query ($full_path: ID!, $cursor: String) {
group(fullPath: $full_path) {
- labels(first: 100, after: $cursor) {
+ labels(first: 100, after: $cursor, onlyGroupLabels: true) {
page_info: pageInfo {
end_cursor: endCursor
has_next_page: hasNextPage
@@ -19,6 +19,8 @@ module BulkImports
title
description
color
+ created_at: createdAt
+ updated_at: updatedAt
}
}
}
diff --git a/lib/bulk_imports/groups/loaders/group_loader.rb b/lib/bulk_imports/groups/loaders/group_loader.rb
index 386fc695182..a631685c2ad 100644
--- a/lib/bulk_imports/groups/loaders/group_loader.rb
+++ b/lib/bulk_imports/groups/loaders/group_loader.rb
@@ -4,10 +4,6 @@ module BulkImports
module Groups
module Loaders
class GroupLoader
- def initialize(options = {})
- @options = options
- end
-
def load(context, data)
return unless user_can_create_group?(context.current_user, data)
diff --git a/lib/bulk_imports/groups/loaders/labels_loader.rb b/lib/bulk_imports/groups/loaders/labels_loader.rb
deleted file mode 100644
index b8c9ba9609c..00000000000
--- a/lib/bulk_imports/groups/loaders/labels_loader.rb
+++ /dev/null
@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module BulkImports
- module Groups
- module Loaders
- class LabelsLoader
- def initialize(*); end
-
- def load(context, data)
- Labels::CreateService.new(data).execute(group: context.group)
- end
- end
- end
- end
-end
diff --git a/lib/bulk_imports/groups/loaders/members_loader.rb b/lib/bulk_imports/groups/loaders/members_loader.rb
deleted file mode 100644
index ccf44b31aee..00000000000
--- a/lib/bulk_imports/groups/loaders/members_loader.rb
+++ /dev/null
@@ -1,17 +0,0 @@
-# frozen_string_literal: true
-
-module BulkImports
- module Groups
- module Loaders
- class MembersLoader
- def initialize(*); end
-
- def load(context, data)
- return unless data
-
- context.group.members.create!(data)
- end
- end
- end
- end
-end
diff --git a/lib/bulk_imports/groups/pipelines/labels_pipeline.rb b/lib/bulk_imports/groups/pipelines/labels_pipeline.rb
index 40dab9b444c..9f8b8682751 100644
--- a/lib/bulk_imports/groups/pipelines/labels_pipeline.rb
+++ b/lib/bulk_imports/groups/pipelines/labels_pipeline.rb
@@ -11,7 +11,9 @@ module BulkImports
transformer Common::Transformers::ProhibitedAttributesTransformer
- loader BulkImports::Groups::Loaders::LabelsLoader
+ def load(context, data)
+ Labels::CreateService.new(data).execute(group: context.group)
+ end
def after_run(extracted_data)
context.entity.update_tracker_for(
diff --git a/lib/bulk_imports/groups/pipelines/members_pipeline.rb b/lib/bulk_imports/groups/pipelines/members_pipeline.rb
index b00c4c1a659..32fc931e8c3 100644
--- a/lib/bulk_imports/groups/pipelines/members_pipeline.rb
+++ b/lib/bulk_imports/groups/pipelines/members_pipeline.rb
@@ -12,7 +12,11 @@ module BulkImports
transformer Common::Transformers::ProhibitedAttributesTransformer
transformer BulkImports::Groups::Transformers::MemberAttributesTransformer
- loader BulkImports::Groups::Loaders::MembersLoader
+ def load(context, data)
+ return unless data
+
+ context.group.members.create!(data)
+ end
def after_run(extracted_data)
context.entity.update_tracker_for(
diff --git a/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline.rb b/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline.rb
index d7e1a118d0b..c47a8bd1daa 100644
--- a/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline.rb
+++ b/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline.rb
@@ -9,7 +9,10 @@ module BulkImports
extractor BulkImports::Groups::Extractors::SubgroupsExtractor
transformer Common::Transformers::ProhibitedAttributesTransformer
transformer BulkImports::Groups::Transformers::SubgroupToEntityTransformer
- loader BulkImports::Common::Loaders::EntityLoader
+
+ def load(context, data)
+ context.bulk_import.entities.create!(data)
+ end
end
end
end
diff --git a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb
index 7de9a430421..23e898a7bb2 100644
--- a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb
+++ b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb
@@ -4,10 +4,6 @@ module BulkImports
module Groups
module Transformers
class GroupAttributesTransformer
- def initialize(options = {})
- @options = options
- end
-
def transform(context, data)
import_entity = context.entity
@@ -39,12 +35,11 @@ module BulkImports
end
def transform_parent(context, import_entity, data)
- current_user = context.current_user
- namespace = Namespace.find_by_full_path(import_entity.destination_namespace)
-
- return data if namespace == current_user.namespace
+ unless import_entity.destination_namespace.blank?
+ namespace = Namespace.find_by_full_path(import_entity.destination_namespace)
+ data['parent_id'] = namespace.id
+ end
- data['parent_id'] = namespace.id
data
end
diff --git a/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb b/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb
index 622f5b60ffe..e92c898171a 100644
--- a/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb
+++ b/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb
@@ -4,8 +4,6 @@ module BulkImports
module Groups
module Transformers
class MemberAttributesTransformer
- def initialize(*); end
-
def transform(context, data)
data
.then { |data| add_user(data) }
diff --git a/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb b/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb
index 6c3c299c2d2..676a6ca8d2a 100644
--- a/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb
+++ b/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb
@@ -4,8 +4,6 @@ module BulkImports
module Groups
module Transformers
class SubgroupToEntityTransformer
- def initialize(*args); end
-
def transform(context, entry)
{
source_type: :group_entity,
diff --git a/lib/bulk_imports/pipeline.rb b/lib/bulk_imports/pipeline.rb
index 1d55ad95887..254ae0e792e 100644
--- a/lib/bulk_imports/pipeline.rb
+++ b/lib/bulk_imports/pipeline.rb
@@ -15,16 +15,78 @@ module BulkImports
attr_reader :context
+ # Fetch pipeline extractor.
+ # An extractor is defined either by instance `#extract(context)` method
+ # or by using `extractor` DSL.
+ #
+ # @example
+ # class MyPipeline
+ # extractor MyExtractor, foo: :bar
+ # end
+ #
+ # class MyPipeline
+ # def extract(context)
+ # puts 'Fetch some data'
+ # end
+ # end
+ #
+ # If pipeline implements instance method `extract` - use it
+ # and ignore class `extractor` method implementation.
def extractor
- @extractor ||= instantiate(self.class.get_extractor)
+ @extractor ||= self.respond_to?(:extract) ? self : instantiate(self.class.get_extractor)
end
+ # Fetch pipeline transformers.
+ #
+ # A transformer can be defined using:
+ # - `transformer` class method
+ # - `transform` instance method
+ #
+ # Multiple transformers can be defined within a single
+ # pipeline and run sequentially for each record in the
+ # following order:
+ # - Transformers defined using `transformer` class method
+ # - Instance method `transform`
+ #
+ # Instance method `transform` is always the last to run.
+ #
+ # @example
+ # class MyPipeline
+ # transformer MyTransformerOne, foo: :bar
+ # transformer MyTransformerTwo, foo: :bar
+ #
+ # def transform(context, data)
+ # # perform transformation here
+ # end
+ # end
+ #
+ # In the example above `MyTransformerOne` is the first to run and
+ # the instance `#transform` method is the last.
def transformers
@transformers ||= self.class.transformers.map(&method(:instantiate))
+ @transformers << self if respond_to?(:transform) && @transformers.exclude?(self)
+ @transformers
end
+ # Fetch pipeline loader.
+ # A loader is defined either by instance method `#load(context, data)`
+ # or by using `loader` DSL.
+ #
+ # @example
+ # class MyPipeline
+ # loader MyLoader, foo: :bar
+ # end
+ #
+ # class MyPipeline
+ # def load(context, data)
+ # puts 'Load some data'
+ # end
+ # end
+ #
+ # If pipeline implements instance method `load` - use it
+ # and ignore class `loader` method implementation.
def loader
- @loaders ||= instantiate(self.class.get_loader)
+ @loader ||= self.respond_to?(:load) ? self : instantiate(self.class.get_loader)
end
def pipeline
@@ -32,7 +94,13 @@ module BulkImports
end
def instantiate(class_config)
- class_config[:klass].new(class_config[:options])
+ options = class_config[:options]
+
+ if options
+ class_config[:klass].new(class_config[:options])
+ else
+ class_config[:klass].new
+ end
end
def abort_on_failure?