From aee0a117a889461ce8ced6fcf73207fe017f1d99 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Mon, 20 Dec 2021 13:37:47 +0000 Subject: Add latest changes from gitlab-org/gitlab@14-6-stable-ee --- lib/bulk_imports/clients/http.rb | 2 +- .../common/pipelines/badges_pipeline.rb | 44 +++++++++++ .../common/pipelines/uploads_pipeline.rb | 29 +++++++- lib/bulk_imports/common/pipelines/wiki_pipeline.rb | 21 +++++- lib/bulk_imports/common/rest/get_badges_query.rb | 23 ++++++ .../groups/pipelines/badges_pipeline.rb | 32 -------- .../groups/pipelines/group_avatar_pipeline.rb | 49 ------------- lib/bulk_imports/groups/rest/get_badges_query.rb | 22 ------ lib/bulk_imports/groups/stage.rb | 10 +-- lib/bulk_imports/ndjson_pipeline.rb | 9 ++- .../projects/graphql/get_project_query.rb | 17 +---- .../projects/graphql/get_repository_query.rb | 17 +---- .../graphql/get_snippet_repository_query.rb | 48 ++++++++++++ lib/bulk_imports/projects/graphql/queryable.rb | 25 +++++++ .../projects/pipelines/auto_devops_pipeline.rb | 15 ++++ .../projects/pipelines/ci_pipelines_pipeline.rb | 15 ++++ .../container_expiration_policy_pipeline.rb | 15 ++++ .../pipelines/pipeline_schedules_pipeline.rb | 15 ++++ .../pipelines/project_attributes_pipeline.rb | 85 ++++++++++++++++++++++ .../projects/pipelines/project_feature_pipeline.rb | 15 ++++ .../projects/pipelines/repository_pipeline.rb | 2 + .../pipelines/service_desk_setting_pipeline.rb | 15 ++++ .../projects/pipelines/snippets_pipeline.rb | 15 ++++ .../pipelines/snippets_repository_pipeline.rb | 69 ++++++++++++++++++ lib/bulk_imports/projects/stage.rb | 40 ++++++++++ 25 files changed, 499 insertions(+), 150 deletions(-) create mode 100644 lib/bulk_imports/common/pipelines/badges_pipeline.rb create mode 100644 lib/bulk_imports/common/rest/get_badges_query.rb delete mode 100644 lib/bulk_imports/groups/pipelines/badges_pipeline.rb delete mode 100644 lib/bulk_imports/groups/pipelines/group_avatar_pipeline.rb delete mode 100644 lib/bulk_imports/groups/rest/get_badges_query.rb create mode 100644 lib/bulk_imports/projects/graphql/get_snippet_repository_query.rb create mode 100644 lib/bulk_imports/projects/graphql/queryable.rb create mode 100644 lib/bulk_imports/projects/pipelines/auto_devops_pipeline.rb create mode 100644 lib/bulk_imports/projects/pipelines/ci_pipelines_pipeline.rb create mode 100644 lib/bulk_imports/projects/pipelines/container_expiration_policy_pipeline.rb create mode 100644 lib/bulk_imports/projects/pipelines/pipeline_schedules_pipeline.rb create mode 100644 lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb create mode 100644 lib/bulk_imports/projects/pipelines/project_feature_pipeline.rb create mode 100644 lib/bulk_imports/projects/pipelines/service_desk_setting_pipeline.rb create mode 100644 lib/bulk_imports/projects/pipelines/snippets_pipeline.rb create mode 100644 lib/bulk_imports/projects/pipelines/snippets_repository_pipeline.rb (limited to 'lib/bulk_imports') diff --git a/lib/bulk_imports/clients/http.rb b/lib/bulk_imports/clients/http.rb index 90414a875c6..eb3d551d1d7 100644 --- a/lib/bulk_imports/clients/http.rb +++ b/lib/bulk_imports/clients/http.rb @@ -123,7 +123,7 @@ module BulkImports def with_error_handling response = yield - raise ::BulkImports::NetworkError.new(response: response) unless response.success? + raise ::BulkImports::NetworkError.new("Unsuccessful response #{response.code} from #{response.request.path.path}", response: response) unless response.success? response rescue *Gitlab::HTTP::HTTP_ERRORS => e diff --git a/lib/bulk_imports/common/pipelines/badges_pipeline.rb b/lib/bulk_imports/common/pipelines/badges_pipeline.rb new file mode 100644 index 00000000000..33a24e61a3f --- /dev/null +++ b/lib/bulk_imports/common/pipelines/badges_pipeline.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module BulkImports + module Common + module Pipelines + class BadgesPipeline + include Pipeline + + extractor BulkImports::Common::Extractors::RestExtractor, + query: BulkImports::Common::Rest::GetBadgesQuery + + transformer Common::Transformers::ProhibitedAttributesTransformer + + def transform(context, data) + return if data.blank? + # Project badges API returns badges of both group and project kind. To avoid creation of duplicates for the group we skip group badges when it's a project. + return if context.entity.project? && group_badge?(data) + + { + name: data['name'], + link_url: data['link_url'], + image_url: data['image_url'] + } + end + + def load(context, data) + return if data.blank? + + if context.entity.project? + context.portable.project_badges.create!(data) + else + context.portable.badges.create!(data) + end + end + + private + + def group_badge?(data) + data['kind'] == 'group' + end + end + end + end +end diff --git a/lib/bulk_imports/common/pipelines/uploads_pipeline.rb b/lib/bulk_imports/common/pipelines/uploads_pipeline.rb index 15e126e1646..49c16209661 100644 --- a/lib/bulk_imports/common/pipelines/uploads_pipeline.rb +++ b/lib/bulk_imports/common/pipelines/uploads_pipeline.rb @@ -8,6 +8,9 @@ module BulkImports include Gitlab::ImportExport::CommandLineUtil FILENAME = 'uploads.tar.gz' + AVATAR_PATTERN = %r{.*\/#{BulkImports::UploadsExportService::AVATAR_PATH}\/(?.*)}.freeze + + AvatarLoadingError = Class.new(StandardError) def extract(context) download_service(tmp_dir, context).execute @@ -18,14 +21,18 @@ module BulkImports end def load(context, file_path) - dynamic_path = FileUploader.extract_dynamic_path(file_path) + avatar_path = AVATAR_PATTERN.match(file_path) + + return save_avatar(file_path) if avatar_path + + dynamic_path = file_uploader.extract_dynamic_path(file_path) return unless dynamic_path return if File.directory?(file_path) named_captures = dynamic_path.named_captures.symbolize_keys - UploadService.new(context.portable, File.open(file_path, 'r'), FileUploader, **named_captures).execute + UploadService.new(context.portable, File.open(file_path, 'r'), file_uploader, **named_captures).execute end def after_run(_) @@ -46,6 +53,24 @@ module BulkImports def tmp_dir @tmp_dir ||= Dir.mktmpdir('bulk_imports') end + + def file_uploader + @file_uploader ||= if context.entity.group? + NamespaceFileUploader + else + FileUploader + end + end + + def save_avatar(file_path) + File.open(file_path) do |avatar| + service = context.entity.update_service.new(portable, current_user, avatar: avatar) + + unless service.execute + raise AvatarLoadingError, portable.errors.full_messages.to_sentence + end + end + end end end end diff --git a/lib/bulk_imports/common/pipelines/wiki_pipeline.rb b/lib/bulk_imports/common/pipelines/wiki_pipeline.rb index ccab0b979b2..6900835b14d 100644 --- a/lib/bulk_imports/common/pipelines/wiki_pipeline.rb +++ b/lib/bulk_imports/common/pipelines/wiki_pipeline.rb @@ -7,7 +7,9 @@ module BulkImports include Pipeline def extract(*) - BulkImports::Pipeline::ExtractedData.new(data: { url: url_from_parent_path(context.entity.source_full_path) }) + url = url_from_parent_path(context.entity.source_full_path) if source_wiki_exists? + + BulkImports::Pipeline::ExtractedData.new(data: { url: url }) end def transform(_, data) @@ -15,14 +17,15 @@ module BulkImports end def load(context, data) - return unless context.portable.wiki + return unless data&.dig(:url) + wiki = context.portable.wiki url = data[:url].sub("://", "://oauth2:#{context.configuration.access_token}@") Gitlab::UrlBlocker.validate!(url, allow_local_network: allow_local_requests?, allow_localhost: allow_local_requests?) - context.portable.wiki.ensure_repository - context.portable.wiki.repository.fetch_as_mirror(url) + wiki.ensure_repository + wiki.repository.fetch_as_mirror(url) end private @@ -36,6 +39,16 @@ module BulkImports def allow_local_requests? Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? end + + def source_wiki_exists? + wikis = client.get(context.entity.wikis_url_path).parsed_response + + wikis.any? + end + + def client + BulkImports::Clients::HTTP.new(url: context.configuration.url, token: context.configuration.access_token) + end end end end diff --git a/lib/bulk_imports/common/rest/get_badges_query.rb b/lib/bulk_imports/common/rest/get_badges_query.rb new file mode 100644 index 00000000000..60b2ebcc552 --- /dev/null +++ b/lib/bulk_imports/common/rest/get_badges_query.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module BulkImports + module Common + module Rest + module GetBadgesQuery + extend self + + def to_h(context) + resource = context.entity.pluralized_name + encoded_full_path = ERB::Util.url_encode(context.entity.source_full_path) + + { + resource: [resource, encoded_full_path, 'badges'].join('/'), + query: { + page: context.tracker.next_page + } + } + end + end + end + end +end diff --git a/lib/bulk_imports/groups/pipelines/badges_pipeline.rb b/lib/bulk_imports/groups/pipelines/badges_pipeline.rb deleted file mode 100644 index 8569ff3f77a..00000000000 --- a/lib/bulk_imports/groups/pipelines/badges_pipeline.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -module BulkImports - module Groups - module Pipelines - class BadgesPipeline - include Pipeline - - extractor BulkImports::Common::Extractors::RestExtractor, - query: BulkImports::Groups::Rest::GetBadgesQuery - - transformer Common::Transformers::ProhibitedAttributesTransformer - - def transform(_, data) - return if data.blank? - - { - name: data['name'], - link_url: data['link_url'], - image_url: data['image_url'] - } - end - - def load(context, data) - return if data.blank? - - context.group.badges.create!(data) - end - end - end - end -end diff --git a/lib/bulk_imports/groups/pipelines/group_avatar_pipeline.rb b/lib/bulk_imports/groups/pipelines/group_avatar_pipeline.rb deleted file mode 100644 index 6de8bbbc910..00000000000 --- a/lib/bulk_imports/groups/pipelines/group_avatar_pipeline.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -module BulkImports - module Groups - module Pipelines - class GroupAvatarPipeline - include Pipeline - - ALLOWED_AVATAR_DOWNLOAD_TYPES = (AvatarUploader::MIME_WHITELIST + %w(application/octet-stream)).freeze - - GroupAvatarLoadingError = Class.new(StandardError) - - def extract(context) - context.extra[:tmpdir] = Dir.mktmpdir - - filepath = BulkImports::FileDownloadService.new( - configuration: context.configuration, - relative_url: "/groups/#{context.entity.encoded_source_full_path}/avatar", - dir: context.extra[:tmpdir], - file_size_limit: Avatarable::MAXIMUM_FILE_SIZE, - allowed_content_types: ALLOWED_AVATAR_DOWNLOAD_TYPES - ).execute - - BulkImports::Pipeline::ExtractedData.new(data: { filepath: filepath }) - end - - def load(context, data) - return if data.blank? - - File.open(data[:filepath]) do |avatar| - service = ::Groups::UpdateService.new( - portable, - current_user, - avatar: avatar - ) - - unless service.execute - raise GroupAvatarLoadingError, portable.errors.full_messages.first - end - end - end - - def after_run(_) - FileUtils.remove_entry(context.extra[:tmpdir]) if context.extra[:tmpdir].present? - end - end - end - end -end diff --git a/lib/bulk_imports/groups/rest/get_badges_query.rb b/lib/bulk_imports/groups/rest/get_badges_query.rb deleted file mode 100644 index 79ffdd9a1f6..00000000000 --- a/lib/bulk_imports/groups/rest/get_badges_query.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -module BulkImports - module Groups - module Rest - module GetBadgesQuery - extend self - - def to_h(context) - encoded_full_path = ERB::Util.url_encode(context.entity.source_full_path) - - { - resource: ['groups', encoded_full_path, 'badges'].join('/'), - query: { - page: context.tracker.next_page - } - } - end - end - end - end -end diff --git a/lib/bulk_imports/groups/stage.rb b/lib/bulk_imports/groups/stage.rb index 241dd428dd5..1a3babe1679 100644 --- a/lib/bulk_imports/groups/stage.rb +++ b/lib/bulk_imports/groups/stage.rb @@ -11,10 +11,6 @@ module BulkImports pipeline: BulkImports::Groups::Pipelines::GroupPipeline, stage: 0 }, - avatar: { - pipeline: BulkImports::Groups::Pipelines::GroupAvatarPipeline, - stage: 1 - }, subgroups: { pipeline: BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline, stage: 1 @@ -32,13 +28,17 @@ module BulkImports stage: 1 }, badges: { - pipeline: BulkImports::Groups::Pipelines::BadgesPipeline, + pipeline: BulkImports::Common::Pipelines::BadgesPipeline, stage: 1 }, boards: { pipeline: BulkImports::Common::Pipelines::BoardsPipeline, stage: 2 }, + uploads: { + pipeline: BulkImports::Common::Pipelines::UploadsPipeline, + stage: 2 + }, finisher: { pipeline: BulkImports::Common::Pipelines::EntityFinisher, stage: 3 diff --git a/lib/bulk_imports/ndjson_pipeline.rb b/lib/bulk_imports/ndjson_pipeline.rb index 6cc29d63919..d5475a8b324 100644 --- a/lib/bulk_imports/ndjson_pipeline.rb +++ b/lib/bulk_imports/ndjson_pipeline.rb @@ -10,7 +10,12 @@ module BulkImports ndjson_pipeline! def transform(context, data) + return unless data + relation_hash, relation_index = data + + return unless relation_hash + relation_definition = import_export_config.top_relation_tree(relation) relation_object = deep_transform_relation!(relation_hash, relation, relation_definition) do |key, hash| @@ -31,9 +36,7 @@ module BulkImports end def load(_, object) - return unless object - - object.save! unless object.persisted? + object&.save! end def deep_transform_relation!(relation_hash, relation_key, relation_definition, &block) diff --git a/lib/bulk_imports/projects/graphql/get_project_query.rb b/lib/bulk_imports/projects/graphql/get_project_query.rb index 2aec496880f..04ac0916bbc 100644 --- a/lib/bulk_imports/projects/graphql/get_project_query.rb +++ b/lib/bulk_imports/projects/graphql/get_project_query.rb @@ -4,6 +4,7 @@ module BulkImports module Projects module Graphql module GetProjectQuery + extend Queryable extend self def to_s @@ -28,22 +29,6 @@ module BulkImports } GRAPHQL end - - def variables(context) - { full_path: context.entity.source_full_path } - end - - def base_path - %w[data project] - end - - def data_path - base_path - end - - def page_info_path - base_path << 'page_info' - end end end end diff --git a/lib/bulk_imports/projects/graphql/get_repository_query.rb b/lib/bulk_imports/projects/graphql/get_repository_query.rb index d3e377c1175..24efce9e276 100644 --- a/lib/bulk_imports/projects/graphql/get_repository_query.rb +++ b/lib/bulk_imports/projects/graphql/get_repository_query.rb @@ -4,6 +4,7 @@ module BulkImports module Projects module Graphql module GetRepositoryQuery + extend Queryable extend self def to_s @@ -15,22 +16,6 @@ module BulkImports } GRAPHQL end - - def variables(context) - { full_path: context.entity.source_full_path } - end - - def base_path - %w[data project] - end - - def data_path - base_path - end - - def page_info_path - base_path << 'page_info' - end end end end diff --git a/lib/bulk_imports/projects/graphql/get_snippet_repository_query.rb b/lib/bulk_imports/projects/graphql/get_snippet_repository_query.rb new file mode 100644 index 00000000000..1ba57789612 --- /dev/null +++ b/lib/bulk_imports/projects/graphql/get_snippet_repository_query.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Graphql + module GetSnippetRepositoryQuery + extend Queryable + extend self + + def to_s + <<-'GRAPHQL' + query($full_path: ID!) { + project(fullPath: $full_path) { + snippets { + page_info: pageInfo { + next_page: endCursor + has_next_page: hasNextPage + } + nodes { + title + createdAt + httpUrlToRepo + } + } + } + } + GRAPHQL + end + + def variables(context) + { + full_path: context.entity.source_full_path, + cursor: context.tracker.next_page, + per_page: ::BulkImports::Tracker::DEFAULT_PAGE_SIZE + } + end + + def base_path + %w[data project snippets] + end + + def data_path + base_path << 'nodes' + end + end + end + end +end diff --git a/lib/bulk_imports/projects/graphql/queryable.rb b/lib/bulk_imports/projects/graphql/queryable.rb new file mode 100644 index 00000000000..a897632dff3 --- /dev/null +++ b/lib/bulk_imports/projects/graphql/queryable.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Graphql + module Queryable + def variables(context) + { full_path: context.entity.source_full_path } + end + + def base_path + %w[data project] + end + + def data_path + base_path + end + + def page_info_path + base_path << 'page_info' + end + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/auto_devops_pipeline.rb b/lib/bulk_imports/projects/pipelines/auto_devops_pipeline.rb new file mode 100644 index 00000000000..1e54ca7017d --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/auto_devops_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class AutoDevopsPipeline + include NdjsonPipeline + + relation_name 'auto_devops' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/ci_pipelines_pipeline.rb b/lib/bulk_imports/projects/pipelines/ci_pipelines_pipeline.rb new file mode 100644 index 00000000000..4487835b88e --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/ci_pipelines_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class CiPipelinesPipeline + include NdjsonPipeline + + relation_name 'ci_pipelines' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/container_expiration_policy_pipeline.rb b/lib/bulk_imports/projects/pipelines/container_expiration_policy_pipeline.rb new file mode 100644 index 00000000000..796e2bd5293 --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/container_expiration_policy_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class ContainerExpirationPolicyPipeline + include NdjsonPipeline + + relation_name 'container_expiration_policy' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/pipeline_schedules_pipeline.rb b/lib/bulk_imports/projects/pipelines/pipeline_schedules_pipeline.rb new file mode 100644 index 00000000000..67053f4e0d4 --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/pipeline_schedules_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class PipelineSchedulesPipeline + include NdjsonPipeline + + relation_name 'pipeline_schedules' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb b/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb new file mode 100644 index 00000000000..4d742225ff7 --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class ProjectAttributesPipeline + include Pipeline + + transformer ::BulkImports::Common::Transformers::ProhibitedAttributesTransformer + + def extract(context) + download_service(tmp_dir, context).execute + decompression_service(tmp_dir).execute + project_attributes = json_decode(json_attributes) + + BulkImports::Pipeline::ExtractedData.new(data: project_attributes) + end + + def transform(_, data) + subrelations = config.portable_relations_tree.keys.map(&:to_s) + + Gitlab::ImportExport::AttributeCleaner.clean( + relation_hash: data, + relation_class: Project, + excluded_keys: config.relation_excluded_keys(:project) + ).except(*subrelations) + end + + def load(_, data) + portable.assign_attributes(data) + portable.reconcile_shared_runners_setting! + portable.drop_visibility_level! + portable.save! + end + + def after_run(_) + FileUtils.remove_entry(tmp_dir) + end + + def json_attributes + @json_attributes ||= File.read(File.join(tmp_dir, filename)) + end + + private + + def tmp_dir + @tmp_dir ||= Dir.mktmpdir + end + + def config + @config ||= BulkImports::FileTransfer.config_for(portable) + end + + def download_service(tmp_dir, context) + @download_service ||= BulkImports::FileDownloadService.new( + configuration: context.configuration, + relative_url: context.entity.relation_download_url_path(BulkImports::FileTransfer::BaseConfig::SELF_RELATION), + dir: tmp_dir, + filename: compressed_filename + ) + end + + def decompression_service(tmp_dir) + @decompression_service ||= BulkImports::FileDecompressionService.new(dir: tmp_dir, filename: compressed_filename) + end + + def compressed_filename + "#{filename}.gz" + end + + def filename + "#{BulkImports::FileTransfer::BaseConfig::SELF_RELATION}.json" + end + + def json_decode(string) + Gitlab::Json.parse(string) + rescue JSON::ParserError => e + Gitlab::ErrorTracking.log_exception(e) + + raise BulkImports::Error, 'Incorrect JSON format' + end + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/project_feature_pipeline.rb b/lib/bulk_imports/projects/pipelines/project_feature_pipeline.rb new file mode 100644 index 00000000000..ff5437efeef --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/project_feature_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class ProjectFeaturePipeline + include NdjsonPipeline + + relation_name 'project_feature' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/repository_pipeline.rb b/lib/bulk_imports/projects/pipelines/repository_pipeline.rb index 6bbd4d0688b..f5ccc1dd922 100644 --- a/lib/bulk_imports/projects/pipelines/repository_pipeline.rb +++ b/lib/bulk_imports/projects/pipelines/repository_pipeline.rb @@ -16,6 +16,8 @@ module BulkImports def load(context, data) url = data['httpUrlToRepo'] + return unless url.present? + url = url.sub("://", "://oauth2:#{context.configuration.access_token}@") project = context.portable diff --git a/lib/bulk_imports/projects/pipelines/service_desk_setting_pipeline.rb b/lib/bulk_imports/projects/pipelines/service_desk_setting_pipeline.rb new file mode 100644 index 00000000000..a50b5423366 --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/service_desk_setting_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class ServiceDeskSettingPipeline + include NdjsonPipeline + + relation_name 'service_desk_setting' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/snippets_pipeline.rb b/lib/bulk_imports/projects/pipelines/snippets_pipeline.rb new file mode 100644 index 00000000000..d543bcec383 --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/snippets_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class SnippetsPipeline + include NdjsonPipeline + + relation_name 'snippets' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/snippets_repository_pipeline.rb b/lib/bulk_imports/projects/pipelines/snippets_repository_pipeline.rb new file mode 100644 index 00000000000..6d423717a51 --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/snippets_repository_pipeline.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class SnippetsRepositoryPipeline + include Pipeline + + extractor Common::Extractors::GraphqlExtractor, query: Graphql::GetSnippetRepositoryQuery + + def transform(_context, data) + data.tap do |d| + d['createdAt'] = DateTime.parse(data['createdAt']) + end + end + + def load(context, data) + return unless data['httpUrlToRepo'].present? + + oauth2_url = oauth2(data['httpUrlToRepo']) + validate_url(oauth2_url) + + matched_snippet = find_matched_snippet(data) + # Skip snippets that we couldn't find a match. Probably because more snippets were + # added after the migration had already started, namely after the SnippetsPipeline + # has already run. + return unless matched_snippet + + matched_snippet.create_repository + matched_snippet.repository.fetch_as_mirror(oauth2_url) + response = Snippets::RepositoryValidationService.new(nil, matched_snippet).execute + + # skips matched_snippet repository creation if repository is invalid + return cleanup_snippet_repository(matched_snippet) if response.error? + + Snippets::UpdateStatisticsService.new(matched_snippet).execute + end + + private + + def find_matched_snippet(data) + Snippet.find_by_project_title_trunc_created_at( + context.portable, data['title'], data['createdAt']) + end + + def allow_local_requests? + Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? + end + + def oauth2(url) + url.sub("://", "://oauth2:#{context.configuration.access_token}@") + end + + def validate_url(url) + Gitlab::UrlBlocker.validate!( + url, + allow_local_network: allow_local_requests?, + allow_localhost: allow_local_requests?) + end + + def cleanup_snippet_repository(snippet) + snippet.repository.remove + snippet.snippet_repository.delete + snippet.repository.expire_exists_cache + end + end + end + end +end diff --git a/lib/bulk_imports/projects/stage.rb b/lib/bulk_imports/projects/stage.rb index 9ccc9efff1d..0556395ca66 100644 --- a/lib/bulk_imports/projects/stage.rb +++ b/lib/bulk_imports/projects/stage.rb @@ -15,6 +15,10 @@ module BulkImports pipeline: BulkImports::Projects::Pipelines::RepositoryPipeline, stage: 1 }, + project_attributes: { + pipeline: BulkImports::Projects::Pipelines::ProjectAttributesPipeline, + stage: 1 + }, labels: { pipeline: BulkImports::Common::Pipelines::LabelsPipeline, stage: 2 @@ -23,10 +27,22 @@ module BulkImports pipeline: BulkImports::Common::Pipelines::MilestonesPipeline, stage: 2 }, + badges: { + pipeline: BulkImports::Common::Pipelines::BadgesPipeline, + stage: 2 + }, issues: { pipeline: BulkImports::Projects::Pipelines::IssuesPipeline, stage: 3 }, + snippets: { + pipeline: BulkImports::Projects::Pipelines::SnippetsPipeline, + stage: 3 + }, + snippets_repository: { + pipeline: BulkImports::Projects::Pipelines::SnippetsRepositoryPipeline, + stage: 4 + }, boards: { pipeline: BulkImports::Common::Pipelines::BoardsPipeline, stage: 4 @@ -43,6 +59,22 @@ module BulkImports pipeline: BulkImports::Projects::Pipelines::ProtectedBranchesPipeline, stage: 4 }, + ci_pipelines: { + pipeline: BulkImports::Projects::Pipelines::CiPipelinesPipeline, + stage: 4 + }, + project_feature: { + pipeline: BulkImports::Projects::Pipelines::ProjectFeaturePipeline, + stage: 4 + }, + container_expiration_policy: { + pipeline: BulkImports::Projects::Pipelines::ContainerExpirationPolicyPipeline, + stage: 4 + }, + service_desk_setting: { + pipeline: BulkImports::Projects::Pipelines::ServiceDeskSettingPipeline, + stage: 4 + }, wiki: { pipeline: BulkImports::Common::Pipelines::WikiPipeline, stage: 5 @@ -51,6 +83,14 @@ module BulkImports pipeline: BulkImports::Common::Pipelines::UploadsPipeline, stage: 5 }, + auto_devops: { + pipeline: BulkImports::Projects::Pipelines::AutoDevopsPipeline, + stage: 5 + }, + pipeline_schedules: { + pipeline: BulkImports::Projects::Pipelines::PipelineSchedulesPipeline, + stage: 5 + }, finisher: { pipeline: BulkImports::Common::Pipelines::EntityFinisher, stage: 6 -- cgit v1.2.1