diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-02-18 09:45:46 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-02-18 09:45:46 +0000 |
commit | a7b3560714b4d9cc4ab32dffcd1f74a284b93580 (patch) | |
tree | 7452bd5c3545c2fa67a28aa013835fb4fa071baf /lib/bulk_imports | |
parent | ee9173579ae56a3dbfe5afe9f9410c65bb327ca7 (diff) | |
download | gitlab-ce-a7b3560714b4d9cc4ab32dffcd1f74a284b93580.tar.gz |
Add latest changes from gitlab-org/gitlab@14-8-stable-eev14.8.0-rc42
Diffstat (limited to 'lib/bulk_imports')
-rw-r--r-- | lib/bulk_imports/common/extractors/graphql_extractor.rb | 7 | ||||
-rw-r--r-- | lib/bulk_imports/common/graphql/get_members_query.rb (renamed from lib/bulk_imports/groups/graphql/get_members_query.rb) | 37 | ||||
-rw-r--r-- | lib/bulk_imports/common/pipelines/lfs_objects_pipeline.rb | 134 | ||||
-rw-r--r-- | lib/bulk_imports/common/pipelines/members_pipeline.rb | 54 | ||||
-rw-r--r-- | lib/bulk_imports/groups/graphql/get_group_query.rb | 10 | ||||
-rw-r--r-- | lib/bulk_imports/groups/graphql/get_projects_query.rb | 10 | ||||
-rw-r--r-- | lib/bulk_imports/groups/pipelines/members_pipeline.rb | 26 | ||||
-rw-r--r-- | lib/bulk_imports/groups/stage.rb | 2 | ||||
-rw-r--r-- | lib/bulk_imports/groups/transformers/member_attributes_transformer.rb | 43 | ||||
-rw-r--r-- | lib/bulk_imports/projects/graphql/get_project_query.rb | 5 | ||||
-rw-r--r-- | lib/bulk_imports/projects/graphql/get_repository_query.rb | 5 | ||||
-rw-r--r-- | lib/bulk_imports/projects/graphql/get_snippet_repository_query.rb | 7 | ||||
-rw-r--r-- | lib/bulk_imports/projects/graphql/queryable.rb | 8 | ||||
-rw-r--r-- | lib/bulk_imports/projects/stage.rb | 8 |
14 files changed, 269 insertions, 87 deletions
diff --git a/lib/bulk_imports/common/extractors/graphql_extractor.rb b/lib/bulk_imports/common/extractors/graphql_extractor.rb index cde3d1cad5b..bfdc0b13603 100644 --- a/lib/bulk_imports/common/extractors/graphql_extractor.rb +++ b/lib/bulk_imports/common/extractors/graphql_extractor.rb @@ -5,15 +5,16 @@ module BulkImports module Extractors class GraphqlExtractor def initialize(options = {}) - @query = options[:query] + @query_klass = options[:query] end def extract(context) client = graphql_client(context) + query = query_klass.new(context: context) response = client.execute( client.parse(query.to_s), - query.variables(context) + query.variables ).original_hash.deep_dup BulkImports::Pipeline::ExtractedData.new( @@ -24,7 +25,7 @@ module BulkImports private - attr_reader :query + attr_reader :query_klass def graphql_client(context) @graphql_client ||= BulkImports::Clients::Graphql.new( diff --git a/lib/bulk_imports/groups/graphql/get_members_query.rb b/lib/bulk_imports/common/graphql/get_members_query.rb index e76c87cc521..00977f694d7 100644 --- a/lib/bulk_imports/groups/graphql/get_members_query.rb +++ b/lib/bulk_imports/common/graphql/get_members_query.rb @@ -1,15 +1,20 @@ # frozen_string_literal: true module BulkImports - module Groups + module Common module Graphql - module GetMembersQuery - extend self + class GetMembersQuery + attr_reader :context + + def initialize(context:) + @context = context + end + def to_s - <<-'GRAPHQL' + <<-GRAPHQL query($full_path: ID!, $cursor: String, $per_page: Int) { - group(fullPath: $full_path) { - group_members: groupMembers(relations: DIRECT, first: $per_page, after: $cursor) { + portable: #{context.entity.entity_type}(fullPath: $full_path) { + members: #{members_type}(relations: [DIRECT, INHERITED], first: $per_page, after: $cursor) { page_info: pageInfo { next_page: endCursor has_next_page: hasNextPage @@ -32,7 +37,7 @@ module BulkImports GRAPHQL end - def variables(context) + def variables { full_path: context.entity.source_full_path, cursor: context.tracker.next_page, @@ -40,10 +45,6 @@ module BulkImports } end - def base_path - %w[data group group_members] - end - def data_path base_path << 'nodes' end @@ -51,6 +52,20 @@ module BulkImports def page_info_path base_path << 'page_info' end + + private + + def base_path + %w[data portable members] + end + + def members_type + if context.entity.group? + 'groupMembers' + else + 'projectMembers' + end + end end end end diff --git a/lib/bulk_imports/common/pipelines/lfs_objects_pipeline.rb b/lib/bulk_imports/common/pipelines/lfs_objects_pipeline.rb new file mode 100644 index 00000000000..2e6a29f4738 --- /dev/null +++ b/lib/bulk_imports/common/pipelines/lfs_objects_pipeline.rb @@ -0,0 +1,134 @@ +# frozen_string_literal: true + +module BulkImports + module Common + module Pipelines + class LfsObjectsPipeline + include Pipeline + + def extract(_context) + download_service.execute + decompression_service.execute + extraction_service.execute + + file_paths = Dir.glob(File.join(tmpdir, '*')) + + BulkImports::Pipeline::ExtractedData.new(data: file_paths) + end + + # rubocop: disable CodeReuse/ActiveRecord + def load(_context, file_path) + Gitlab::Utils.check_path_traversal!(file_path) + Gitlab::Utils.check_allowed_absolute_path!(file_path, [Dir.tmpdir]) + + return if tar_filepath?(file_path) + return if lfs_json_filepath?(file_path) + return if File.directory?(file_path) + return if File.lstat(file_path).symlink? + + size = File.size(file_path) + oid = LfsObject.calculate_oid(file_path) + + lfs_object = LfsObject.find_or_initialize_by(oid: oid, size: size) + lfs_object.file = File.open(file_path) unless lfs_object.file&.exists? + lfs_object.save! if lfs_object.changed? + + repository_types(oid)&.each do |type| + create_lfs_objects_project(lfs_object, type) + end + end + # rubocop: enable CodeReuse/ActiveRecord + + def after_run(_) + FileUtils.remove_entry(tmpdir) if Dir.exist?(tmpdir) + end + + private + + def download_service + BulkImports::FileDownloadService.new( + configuration: context.configuration, + relative_url: context.entity.relation_download_url_path(relation), + tmpdir: tmpdir, + filename: targz_filename + ) + end + + def decompression_service + BulkImports::FileDecompressionService.new(tmpdir: tmpdir, filename: targz_filename) + end + + def extraction_service + BulkImports::ArchiveExtractionService.new(tmpdir: tmpdir, filename: tar_filename) + end + + def lfs_json + @lfs_json ||= Gitlab::Json.parse(File.read(lfs_json_filepath)) + rescue StandardError + raise BulkImports::Error, 'LFS Objects JSON read failed' + end + + def tmpdir + @tmpdir ||= Dir.mktmpdir('bulk_imports') + end + + def relation + BulkImports::FileTransfer::ProjectConfig::LFS_OBJECTS_RELATION + end + + def tar_filename + "#{relation}.tar" + end + + def targz_filename + "#{tar_filename}.gz" + end + + def lfs_json_filepath?(file_path) + file_path == lfs_json_filepath + end + + def tar_filepath?(file_path) + File.join(tmpdir, tar_filename) == file_path + end + + def lfs_json_filepath + File.join(tmpdir, "#{relation}.json") + end + + def create_lfs_objects_project(lfs_object, repository_type) + return unless allowed_repository_types.include?(repository_type) + + lfs_objects_project = LfsObjectsProject.create( + project: portable, + lfs_object: lfs_object, + repository_type: repository_type + ) + + return if lfs_objects_project.persisted? + + logger.warn( + project_id: portable.id, + message: 'Failed to save lfs objects project', + errors: lfs_objects_project.errors.full_messages.to_sentence, + **Gitlab::ApplicationContext.current + ) + end + + def repository_types(oid) + types = lfs_json[oid] + + return [] unless types + return [] unless types.is_a?(Array) + + # only return allowed repository types + types.uniq & allowed_repository_types + end + + def allowed_repository_types + @allowed_repository_types ||= LfsObjectsProject.repository_types.values.push(nil) + end + end + end + end +end diff --git a/lib/bulk_imports/common/pipelines/members_pipeline.rb b/lib/bulk_imports/common/pipelines/members_pipeline.rb new file mode 100644 index 00000000000..f35eb5ccf5e --- /dev/null +++ b/lib/bulk_imports/common/pipelines/members_pipeline.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module BulkImports + module Common + module Pipelines + class MembersPipeline + include Pipeline + + transformer Common::Transformers::ProhibitedAttributesTransformer + transformer BulkImports::Groups::Transformers::MemberAttributesTransformer + + def extract(context) + graphql_extractor.extract(context) + end + + def load(_context, data) + return unless data + + user_id = data[:user_id] + + # Current user is already a member + return if user_id == current_user.id + + user_membership = existing_user_membership(user_id) + + # User is already a member with higher existing (inherited) membership + return if user_membership && user_membership[:access_level] >= data[:access_level] + + # Create new membership for any other access level + portable.members.create!(data) + end + + private + + def graphql_extractor + @graphql_extractor ||= BulkImports::Common::Extractors::GraphqlExtractor + .new(query: BulkImports::Common::Graphql::GetMembersQuery) + end + + def existing_user_membership(user_id) + members_finder.execute.find_by_user_id(user_id) + end + + def members_finder + @members_finder ||= if context.entity.group? + ::GroupMembersFinder.new(portable, current_user) + else + ::MembersFinder.new(portable, current_user) + end + end + end + end + end +end diff --git a/lib/bulk_imports/groups/graphql/get_group_query.rb b/lib/bulk_imports/groups/graphql/get_group_query.rb index 6852e25c87f..911b2b67d8c 100644 --- a/lib/bulk_imports/groups/graphql/get_group_query.rb +++ b/lib/bulk_imports/groups/graphql/get_group_query.rb @@ -3,8 +3,12 @@ module BulkImports module Groups module Graphql - module GetGroupQuery - extend self + class GetGroupQuery + attr_reader :context + + def initialize(context:) + @context = context + end def to_s <<-'GRAPHQL' @@ -29,7 +33,7 @@ module BulkImports GRAPHQL end - def variables(context) + def variables { full_path: context.entity.source_full_path } end diff --git a/lib/bulk_imports/groups/graphql/get_projects_query.rb b/lib/bulk_imports/groups/graphql/get_projects_query.rb index 4cec1ad1462..3f74bbb8cce 100644 --- a/lib/bulk_imports/groups/graphql/get_projects_query.rb +++ b/lib/bulk_imports/groups/graphql/get_projects_query.rb @@ -3,8 +3,12 @@ module BulkImports module Groups module Graphql - module GetProjectsQuery - extend self + class GetProjectsQuery + attr_reader :context + + def initialize(context:) + @context = context + end def to_s <<-'GRAPHQL' @@ -25,7 +29,7 @@ module BulkImports GRAPHQL end - def variables(context) + def variables { full_path: context.entity.source_full_path, cursor: context.tracker.next_page, diff --git a/lib/bulk_imports/groups/pipelines/members_pipeline.rb b/lib/bulk_imports/groups/pipelines/members_pipeline.rb deleted file mode 100644 index 265abd5e3a7..00000000000 --- a/lib/bulk_imports/groups/pipelines/members_pipeline.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module BulkImports - module Groups - module Pipelines - class MembersPipeline - include Pipeline - - extractor BulkImports::Common::Extractors::GraphqlExtractor, - query: BulkImports::Groups::Graphql::GetMembersQuery - - transformer Common::Transformers::ProhibitedAttributesTransformer - transformer BulkImports::Groups::Transformers::MemberAttributesTransformer - - def load(context, data) - return unless data - - # Current user is already a member - return if data['user_id'].to_i == context.current_user.id - - context.group.members.create!(data) - end - end - end - end -end diff --git a/lib/bulk_imports/groups/stage.rb b/lib/bulk_imports/groups/stage.rb index 1a3babe1679..bc27220391d 100644 --- a/lib/bulk_imports/groups/stage.rb +++ b/lib/bulk_imports/groups/stage.rb @@ -16,7 +16,7 @@ module BulkImports stage: 1 }, members: { - pipeline: BulkImports::Groups::Pipelines::MembersPipeline, + pipeline: BulkImports::Common::Pipelines::MembersPipeline, stage: 1 }, labels: { diff --git a/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb b/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb index b9de375d0e9..da50a19ee62 100644 --- a/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb +++ b/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb @@ -5,50 +5,35 @@ module BulkImports module Transformers class MemberAttributesTransformer def transform(context, data) - data - .then { |data| add_user(data, context) } - .then { |data| add_access_level(data) } - .then { |data| add_author(data, context) } - end - - private - - def add_user(data, context) user = find_user(data&.dig('user', 'public_email')) + access_level = data&.dig('access_level', 'integer_value') + return unless data return unless user + return unless valid_access_level?(access_level) cache_source_user_id(data, user, context) - data - .except('user') - .merge('user_id' => user.id) + { + user_id: user.id, + access_level: access_level, + created_at: data['created_at'], + updated_at: data['updated_at'], + expires_at: data['expires_at'], + created_by_id: context.current_user.id + } end + private + def find_user(email) return unless email User.find_by_any_email(email, confirmed: true) end - def add_access_level(data) - access_level = data&.dig('access_level', 'integer_value') - - return unless valid_access_level?(access_level) - - data.merge('access_level' => access_level) - end - def valid_access_level?(access_level) - Gitlab::Access - .options_with_owner - .value?(access_level) - end - - def add_author(data, context) - return unless data - - data.merge('created_by_id' => context.current_user.id) + Gitlab::Access.options_with_owner.value?(access_level) end def cache_source_user_id(data, user, context) diff --git a/lib/bulk_imports/projects/graphql/get_project_query.rb b/lib/bulk_imports/projects/graphql/get_project_query.rb index 04ac0916bbc..b3d7f3f4683 100644 --- a/lib/bulk_imports/projects/graphql/get_project_query.rb +++ b/lib/bulk_imports/projects/graphql/get_project_query.rb @@ -3,9 +3,8 @@ module BulkImports module Projects module Graphql - module GetProjectQuery - extend Queryable - extend self + class GetProjectQuery + include Queryable def to_s <<-'GRAPHQL' diff --git a/lib/bulk_imports/projects/graphql/get_repository_query.rb b/lib/bulk_imports/projects/graphql/get_repository_query.rb index 24efce9e276..ca777c1a7e0 100644 --- a/lib/bulk_imports/projects/graphql/get_repository_query.rb +++ b/lib/bulk_imports/projects/graphql/get_repository_query.rb @@ -3,9 +3,8 @@ module BulkImports module Projects module Graphql - module GetRepositoryQuery - extend Queryable - extend self + class GetRepositoryQuery + include Queryable def to_s <<-'GRAPHQL' diff --git a/lib/bulk_imports/projects/graphql/get_snippet_repository_query.rb b/lib/bulk_imports/projects/graphql/get_snippet_repository_query.rb index 1ba57789612..c105b04c731 100644 --- a/lib/bulk_imports/projects/graphql/get_snippet_repository_query.rb +++ b/lib/bulk_imports/projects/graphql/get_snippet_repository_query.rb @@ -3,9 +3,8 @@ module BulkImports module Projects module Graphql - module GetSnippetRepositoryQuery - extend Queryable - extend self + class GetSnippetRepositoryQuery + include Queryable def to_s <<-'GRAPHQL' @@ -27,7 +26,7 @@ module BulkImports GRAPHQL end - def variables(context) + def variables { full_path: context.entity.source_full_path, cursor: context.tracker.next_page, diff --git a/lib/bulk_imports/projects/graphql/queryable.rb b/lib/bulk_imports/projects/graphql/queryable.rb index a897632dff3..bd3116cb838 100644 --- a/lib/bulk_imports/projects/graphql/queryable.rb +++ b/lib/bulk_imports/projects/graphql/queryable.rb @@ -4,7 +4,13 @@ module BulkImports module Projects module Graphql module Queryable - def variables(context) + attr_reader :context + + def initialize(context:) + @context = context + end + + def variables { full_path: context.entity.source_full_path } end diff --git a/lib/bulk_imports/projects/stage.rb b/lib/bulk_imports/projects/stage.rb index 0556395ca66..b920c1bf355 100644 --- a/lib/bulk_imports/projects/stage.rb +++ b/lib/bulk_imports/projects/stage.rb @@ -19,6 +19,10 @@ module BulkImports pipeline: BulkImports::Projects::Pipelines::ProjectAttributesPipeline, stage: 1 }, + members: { + pipeline: BulkImports::Common::Pipelines::MembersPipeline, + stage: 1 + }, labels: { pipeline: BulkImports::Common::Pipelines::LabelsPipeline, stage: 2 @@ -83,6 +87,10 @@ module BulkImports pipeline: BulkImports::Common::Pipelines::UploadsPipeline, stage: 5 }, + lfs_objects: { + pipeline: BulkImports::Common::Pipelines::LfsObjectsPipeline, + stage: 5 + }, auto_devops: { pipeline: BulkImports::Projects::Pipelines::AutoDevopsPipeline, stage: 5 |