diff options
Diffstat (limited to 'lib/gitlab/import_export/project')
6 files changed, 496 insertions, 51 deletions
diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 618ef9a4f43..d815dd284ba 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -178,17 +178,7 @@ included_attributes: - :project_id - :key - :value - label: - - :title - - :color - - :project_id - - :group_id - - :created_at - - :updated_at - - :template - - :description - - :priority - labels: + label: &label_definition - :title - :color - :project_id @@ -198,23 +188,13 @@ included_attributes: - :template - :description - :priority + labels: *label_definition priorities: - :project_id - :priority - :created_at - :updated_at - milestone: - - :iid - - :title - - :project_id - - :group_id - - :description - - :due_date - - :created_at - - :updated_at - - :start_date - - :state - milestones: + milestone: &milestone_definition - :iid - :title - :project_id @@ -225,6 +205,7 @@ included_attributes: - :updated_at - :start_date - :state + milestones: *milestone_definition protected_branches: - :project_id - :name @@ -272,6 +253,385 @@ included_attributes: - :updated_at - :filepath - :link_type + container_expiration_policy: + - :created_at + - :updated_at + - :next_run_at + - :project_id + - :name_regex + - :cadence + - :older_than + - :keep_n + - :enabled + - :name_regex_keep + project_feature: + - :project_id + - :merge_requests_access_level + - :issues_access_level + - :wiki_access_level + - :snippets_access_level + - :builds_access_level + - :created_at + - :updated_at + - :repository_access_level + - :pages_access_level + - :forking_access_level + - :metrics_dashboard_access_level + - :operations_access_level + - :analytics_access_level + - :security_and_compliance_access_level + - :container_registry_access_level + prometheus_metrics: + - :created_at + - :updated_at + - :project_id + - :y_label + - :unit + - :legend + - :title + - :query + - :group + - :dashboard_path + service_desk_setting: + - :project_id + - :issue_template_key + - :project_key + snippets: + - :title + - :content + - :author_id + - :project_id + - :created_at + - :updated_at + - :file_name + - :visibility_level + - :description + project_members: + - :access_level + - :source_type + - :user_id + - :notification_level + - :created_at + - :updated_at + - :created_by_id + - :invite_email + - :invite_accepted_at + - :requested_at + - :expires_at + - :ldap + - :override + merge_request: &merge_request_definition + - :target_branch + - :source_branch + - :source_project_id + - :author_id + - :assignee_id + - :title + - :created_at + - :updated_at + - :state + - :merge_status + - :target_project_id + - :iid + - :description + - :updated_by_id + - :merge_error + - :merge_params + - :merge_when_pipeline_succeeds + - :merge_user_id + - :merge_commit_sha + - :squash_commit_sha + - :in_progress_merge_commit_sha + - :lock_version + - :approvals_before_merge + - :rebase_commit_sha + - :time_estimate + - :squash + - :last_edited_at + - :last_edited_by_id + - :discussion_locked + - :allow_maintainer_to_push + - :merge_ref_sha + - :draft + - :diff_head_sha + - :source_branch_sha + - :target_branch_sha + merge_requests: *merge_request_definition + award_emoji: + - :user_id + - :name + - :awardable_type + - :created_at + - :updated_at + commit_author: + - :name + - :email + committer: + - :name + - :email + events: + - :target_type + - :action + - :author_id + - :fingerprint + - :created_at + - :updated_at + label_links: + - :target_type + - :created_at + - :updated_at + merge_request_diff: + - :state + - :created_at + - :updated_at + - :base_commit_sha + - :real_size + - :head_commit_sha + - :start_commit_sha + - :commits_count + - :files_count + - :sorted + - :diff_type + merge_request_diff_commits: + - :author_name + - :author_email + - :committer_name + - :committer_email + - :relative_order + - :sha + - :authored_date + - :committed_date + - :message + - :trailers + merge_request_diff_files: + - :relative_order + - :new_file + - :renamed_file + - :deleted_file + - :new_path + - :old_path + - :a_mode + - :b_mode + - :too_large + - :binary + - :diff + metrics: + - :created_at + - :updated_at + - :latest_closed_by_id + - :latest_closed_at + - :merged_by_id + - :merged_at + - :latest_build_started_at + - :latest_build_finished_at + - :first_deployed_to_production_at + - :first_comment_at + - :first_commit_at + - :last_commit_at + - :diff_size + - :modified_paths_size + - :commits_count + - :first_approved_at + - :first_reassigned_at + - :added_lines + - :target_project_id + - :removed_lines + notes: + - :note + - :noteable_type + - :author_id + - :created_at + - :updated_at + - :project_id + - :attachment + - :line_code + - :commit_id + - :system + - :st_diff + - :updated_by_id + - :type + - :position + - :original_position + - :change_position + - :resolved_at + - :resolved_by_id + - :resolved_by_push + - :discussion_id + - :confidential + - :last_edited_at + push_event_payload: + - :commit_count + - :action + - :ref_type + - :commit_from + - :commit_to + - :ref + - :commit_title + - :ref_count + resource_label_events: + - :action + - :user_id + - :created_at + suggestions: + - :relative_order + - :applied + - :commit_id + - :from_content + - :to_content + - :outdated + - :lines_above + - :lines_below + system_note_metadata: + - :commit_count + - :action + - :created_at + - :updated_at + timelogs: + - :time_spent + - :user_id + - :project_id + - :spent_at + - :created_at + - :updated_at + - :summary + external_pull_request: &external_pull_request_definition + - :created_at + - :updated_at + - :project_id + - :pull_request_iid + - :status + - :source_branch + - :target_branch + - :source_repository + - :target_repository + - :source_sha + - :target_sha + external_pull_requests: *external_pull_request_definition + statuses: + - :project_id + - :status + - :finished_at + - :created_at + - :updated_at + - :started_at + - :coverage + - :commit_id + - :name + - :options + - :allow_failure + - :stage + - :stage_idx + - :tag + - :ref + - :user_id + - :type + - :target_url + - :description + - :erased_at + - :artifacts_expire_at + - :environment + - :yaml_variables + - :queued_at + - :lock_version + - :coverage_regex + - :retried + - :protected + - :failure_reason + - :scheduled_at + - :scheduling_type + ci_pipelines: + - :ref + - :sha + - :before_sha + - :created_at + - :updated_at + - :tag + - :yaml_errors + - :committed_at + - :project_id + - :status + - :started_at + - :finished_at + - :duration + - :user_id + - :lock_version + - :source + - :protected + - :config_source + - :failure_reason + - :iid + - :source_sha + - :target_sha + stages: + - :name + - :status + - :position + - :lock_version + - :project_id + - :created_at + - :updated_at + actions: + - :event + - :image_v432x230 + design: &design_definition + - :iid + - :project_id + - :filename + - :relative_position + designs: *design_definition + design_versions: + - :created_at + - :sha + - :author_id + issue_assignees: + - :user_id + sentry_issue: + - :sentry_issue_identifier + zoom_meetings: + - :project_id + - :issue_status + - :url + - :created_at + - :updated_at + issues: + - :title + - :author_id + - :project_id + - :created_at + - :updated_at + - :description + - :state + - :iid + - :updated_by_id + - :confidential + - :closed_at + - :closed_by_id + - :due_date + - :lock_version + - :weight + - :time_estimate + - :relative_position + - :external_author + - :last_edited_at + - :last_edited_by_id + - :discussion_locked + - :health_status + - :external_key + - :issue_type + group_members: + - :access_level + - :source_type + - :user_id + - :notification_level + - :created_at + - :updated_at + - :created_by_id + - :invite_email + - :invite_accepted_at + - :requested_at + - :expires_at + - :ldap + - :override # Do not include the following attributes for the models specified. excluded_attributes: @@ -387,16 +747,7 @@ excluded_attributes: - :service_desk_reply_to - :upvotes_count - :work_item_type_id - merge_request: - - :milestone_id - - :sprint_id - - :ref_fetched - - :merge_jid - - :rebase_jid - - :latest_merge_request_diff_id - - :head_pipeline_id - - :state_id - merge_requests: + merge_request: &merge_request_excluded_definition - :milestone_id - :sprint_id - :ref_fetched @@ -405,6 +756,7 @@ excluded_attributes: - :latest_merge_request_diff_id - :head_pipeline_id - :state_id + merge_requests: *merge_request_excluded_definition award_emoji: - :awardable_id statuses: @@ -473,10 +825,9 @@ excluded_attributes: - :issue_id zoom_meetings: - :issue_id - design: - - :issue_id - designs: + design: &design_excluded_definition - :issue_id + designs: *design_excluded_definition design_versions: - :issue_id actions: @@ -660,4 +1011,13 @@ ee: - :name - :created_at - :updated_at - + project_feature: + - :requirements_access_level + security_setting: + - :project_id + - :created_at + - :updated_at + - :auto_fix_container_scanning + - :auto_fix_dast + - :auto_fix_dependency_scanning + - :auto_fix_sast diff --git a/lib/gitlab/import_export/project/object_builder.rb b/lib/gitlab/import_export/project/object_builder.rb index b03dceba303..f7598ba1337 100644 --- a/lib/gitlab/import_export/project/object_builder.rb +++ b/lib/gitlab/import_export/project/object_builder.rb @@ -29,6 +29,7 @@ module Gitlab def find return if epic? && group.nil? return find_diff_commit_user if diff_commit_user? + return find_diff_commit if diff_commit? super end @@ -83,9 +84,38 @@ module Gitlab end def find_diff_commit_user - find_with_cache do - MergeRequest::DiffCommitUser - .find_or_create(@attributes['name'], @attributes['email']) + find_or_create_diff_commit_user(@attributes['name'], @attributes['email']) + end + + def find_diff_commit + row = @attributes.dup + + # Diff commits come in two formats: + # + # 1. The old format where author/committer details are separate fields + # 2. The new format where author/committer details are nested objects, + # and pre-processed by `find_diff_commit_user`. + # + # The code here ensures we support both the old and new format. + aname = row.delete('author_name') + amail = row.delete('author_email') + cname = row.delete('committer_name') + cmail = row.delete('committer_email') + author = row.delete('commit_author') + committer = row.delete('committer') + + row['commit_author'] = author || + find_or_create_diff_commit_user(aname, amail) + + row['committer'] = committer || + find_or_create_diff_commit_user(cname, cmail) + + MergeRequestDiffCommit.new(row) + end + + def find_or_create_diff_commit_user(name, email) + find_with_cache([MergeRequest::DiffCommitUser, name, email]) do + MergeRequest::DiffCommitUser.find_or_create(name, email) end end @@ -113,6 +143,10 @@ module Gitlab klass == MergeRequest::DiffCommitUser end + def diff_commit? + klass == MergeRequestDiffCommit + end + # If an existing group milestone used the IID # claim the IID back and set the group milestone to use one available # This is necessary to fix situations like the following: diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index 888a5a10f2c..d84db92fe69 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -33,7 +33,8 @@ module Gitlab links: 'Releases::Link', metrics_setting: 'ProjectMetricsSetting', commit_author: 'MergeRequest::DiffCommitUser', - committer: 'MergeRequest::DiffCommitUser' }.freeze + committer: 'MergeRequest::DiffCommitUser', + merge_request_diff_commits: 'MergeRequestDiffCommit' }.freeze BUILD_MODELS = %i[Ci::Build commit_status].freeze @@ -59,6 +60,7 @@ module Gitlab external_pull_requests DesignManagement::Design MergeRequest::DiffCommitUser + MergeRequestDiffCommit ].freeze def create diff --git a/lib/gitlab/import_export/project/relation_tree_restorer.rb b/lib/gitlab/import_export/project/relation_tree_restorer.rb new file mode 100644 index 00000000000..6e9548f393a --- /dev/null +++ b/lib/gitlab/import_export/project/relation_tree_restorer.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + class RelationTreeRestorer < ImportExport::Group::RelationTreeRestorer + # Relations which cannot be saved at project level (and have a group assigned) + GROUP_MODELS = [GroupLabel, Milestone, Epic].freeze + + private + + def bulk_insert_enabled + true + end + + def modify_attributes + @importable.reconcile_shared_runners_setting! + @importable.drop_visibility_level! + end + + def relation_invalid_for_importable?(relation_object) + GROUP_MODELS.include?(relation_object.class) && relation_object.group_id + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb b/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb index 4db92b12968..034122a9f14 100644 --- a/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb @@ -4,7 +4,7 @@ module Gitlab module ImportExport module Project module Sample - class RelationTreeRestorer < ImportExport::RelationTreeRestorer + class RelationTreeRestorer < ImportExport::Project::RelationTreeRestorer def initialize(...) super(...) @@ -18,10 +18,10 @@ module Gitlab end def dates - return [] if relation_reader.legacy? + return [] if @relation_reader.legacy? RelationFactory::DATE_MODELS.flat_map do |tag| - relation_reader.consume_relation(@importable_path, tag, mark_as_consumed: false).map do |model| + @relation_reader.consume_relation(@importable_path, tag, mark_as_consumed: false).map do |model| model.first['due_date'] end end diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index 1f0fa249390..aafed850afa 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -6,20 +6,16 @@ module Gitlab class TreeSaver attr_reader :full_path - def initialize(project:, current_user:, shared:, params: {}) + def initialize(project:, current_user:, shared:, params: {}, logger: Gitlab::Import::Logger) @params = params @project = project @current_user = current_user @shared = shared + @logger = logger end def save - ImportExport::Json::StreamingSerializer.new( - exportable, - reader.project_tree, - json_writer, - exportable_path: "project" - ).execute + stream_export true rescue StandardError => e @@ -31,6 +27,32 @@ module Gitlab private + def stream_export + on_retry = proc do |exception, try, elapsed_time, next_interval| + @logger.info( + message: "Project export retry triggered from streaming", + 'error.class': exception.class, + 'error.message': exception.message, + try_count: try, + elapsed_time_s: elapsed_time, + wait_to_retry_s: next_interval, + project_name: @project.name, + project_id: @project.id + ) + end + + serializer = ImportExport::Json::StreamingSerializer.new( + exportable, + reader.project_tree, + json_writer, + exportable_path: "project" + ) + + Retriable.retriable(on: Net::OpenTimeout, on_retry: on_retry) do + serializer.execute + end + end + def reader @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) end |