diff options
Diffstat (limited to 'app/workers/gitlab')
11 files changed, 129 insertions, 18 deletions
diff --git a/app/workers/gitlab/github_import/attachments/import_issue_worker.rb b/app/workers/gitlab/github_import/attachments/import_issue_worker.rb new file mode 100644 index 00000000000..1a9fa15b850 --- /dev/null +++ b/app/workers/gitlab/github_import/attachments/import_issue_worker.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Attachments + class ImportIssueWorker # rubocop:disable Scalability/IdempotentWorker + include ObjectImporter + + def representation_class + Representation::NoteText + end + + def importer_class + Importer::NoteAttachmentsImporter + end + + def object_type + :issue_attachment + end + end + end + end +end diff --git a/app/workers/gitlab/github_import/attachments/import_merge_request_worker.rb b/app/workers/gitlab/github_import/attachments/import_merge_request_worker.rb new file mode 100644 index 00000000000..a86852b094f --- /dev/null +++ b/app/workers/gitlab/github_import/attachments/import_merge_request_worker.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Attachments + class ImportMergeRequestWorker # rubocop:disable Scalability/IdempotentWorker + include ObjectImporter + + def representation_class + Representation::NoteText + end + + def importer_class + Importer::NoteAttachmentsImporter + end + + def object_type + :merge_request_attachment + end + end + end + end +end diff --git a/app/workers/gitlab/github_import/attachments/import_note_worker.rb b/app/workers/gitlab/github_import/attachments/import_note_worker.rb new file mode 100644 index 00000000000..2f5bc50ee0b --- /dev/null +++ b/app/workers/gitlab/github_import/attachments/import_note_worker.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Attachments + class ImportNoteWorker # rubocop:disable Scalability/IdempotentWorker + include ObjectImporter + + def representation_class + Representation::NoteText + end + + def importer_class + Importer::NoteAttachmentsImporter + end + + def object_type + :note_attachment + end + end + end + end +end diff --git a/app/workers/gitlab/github_import/attachments/import_release_worker.rb b/app/workers/gitlab/github_import/attachments/import_release_worker.rb new file mode 100644 index 00000000000..5eea5702d3c --- /dev/null +++ b/app/workers/gitlab/github_import/attachments/import_release_worker.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Attachments + class ImportReleaseWorker # rubocop:disable Scalability/IdempotentWorker + include ObjectImporter + + def representation_class + Representation::NoteText + end + + def importer_class + Importer::NoteAttachmentsImporter + end + + def object_type + :release_attachment + end + end + end + end +end diff --git a/app/workers/gitlab/github_import/import_issue_worker.rb b/app/workers/gitlab/github_import/import_issue_worker.rb index 8fdc0219ffd..7d6a28f4a96 100644 --- a/app/workers/gitlab/github_import/import_issue_worker.rb +++ b/app/workers/gitlab/github_import/import_issue_worker.rb @@ -16,6 +16,10 @@ module Gitlab def object_type :issue end + + def increment_object_counter?(object) + !object.pull_request? + end end end end diff --git a/app/workers/gitlab/github_import/import_release_attachments_worker.rb b/app/workers/gitlab/github_import/import_release_attachments_worker.rb index c6f45ec1d7c..bf901f2f7b8 100644 --- a/app/workers/gitlab/github_import/import_release_attachments_worker.rb +++ b/app/workers/gitlab/github_import/import_release_attachments_worker.rb @@ -1,16 +1,18 @@ # frozen_string_literal: true +# TODO: remove in 16.X milestone +# https://gitlab.com/gitlab-org/gitlab/-/issues/377059 module Gitlab module GithubImport class ImportReleaseAttachmentsWorker # rubocop:disable Scalability/IdempotentWorker include ObjectImporter def representation_class - Representation::ReleaseAttachments + Representation::NoteText end def importer_class - Importer::ReleaseAttachmentsImporter + Importer::NoteAttachmentsImporter end def object_type diff --git a/app/workers/gitlab/github_import/stage/import_attachments_worker.rb b/app/workers/gitlab/github_import/stage/import_attachments_worker.rb index e9086dca503..e4a413b4081 100644 --- a/app/workers/gitlab/github_import/stage/import_attachments_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_attachments_worker.rb @@ -28,7 +28,12 @@ module Gitlab # For future issue/mr/milestone/etc attachments importers def importers - [::Gitlab::GithubImport::Importer::ReleasesAttachmentsImporter] + [ + ::Gitlab::GithubImport::Importer::Attachments::ReleasesImporter, + ::Gitlab::GithubImport::Importer::Attachments::NotesImporter, + ::Gitlab::GithubImport::Importer::Attachments::IssuesImporter, + ::Gitlab::GithubImport::Importer::Attachments::MergeRequestsImporter + ] end def start_importer(project, importer, client) @@ -50,7 +55,7 @@ module Gitlab end def feature_disabled?(project) - Feature.disabled?(:github_importer_attachments_import, project.group, type: :ops) + import_settings(project).disabled?(:attachments_import) end end end diff --git a/app/workers/gitlab/github_import/stage/import_issue_events_worker.rb b/app/workers/gitlab/github_import/stage/import_issue_events_worker.rb index 0ec0a1b58d2..54ed4c47e78 100644 --- a/app/workers/gitlab/github_import/stage/import_issue_events_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_issue_events_worker.rb @@ -15,9 +15,9 @@ module Gitlab # client - An instance of Gitlab::GithubImport::Client. # project - An instance of Project. def import(client, project) - importer = importer_class(project) - return skip_to_next_stage(project) if importer.nil? + return skip_to_next_stage(project) if import_settings(project).disabled?(:single_endpoint_issue_events_import) + importer = ::Gitlab::GithubImport::Importer::SingleEndpointIssueEventsImporter info(project.id, message: "starting importer", importer: importer.name) waiter = importer.new(project, client).execute move_to_next_stage(project, { waiter.key => waiter.jobs_remaining }) @@ -25,16 +25,6 @@ module Gitlab private - def importer_class(project) - if Feature.enabled?(:github_importer_single_endpoint_issue_events_import, project.group, type: :ops) - ::Gitlab::GithubImport::Importer::SingleEndpointIssueEventsImporter - elsif Feature.enabled?(:github_importer_issue_events_import, project.group, type: :ops) - ::Gitlab::GithubImport::Importer::IssueEventsImporter - else - nil - end - end - def skip_to_next_stage(project) info(project.id, message: "skipping importer", importer: "IssueEventsImporter") move_to_next_stage(project) diff --git a/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb b/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb index 7922c1113c4..3d1a8437da2 100644 --- a/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb @@ -36,7 +36,7 @@ module Gitlab private def diff_notes_importer(project) - if project.group.present? && Feature.enabled?(:github_importer_single_endpoint_notes_import, project.group, type: :ops) + if import_settings(project).enabled?(:single_endpoint_notes_import) Importer::SingleEndpointDiffNotesImporter else Importer::DiffNotesImporter diff --git a/app/workers/gitlab/github_import/stage/import_notes_worker.rb b/app/workers/gitlab/github_import/stage/import_notes_worker.rb index b53e31ce40e..40ca12b130f 100644 --- a/app/workers/gitlab/github_import/stage/import_notes_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_notes_worker.rb @@ -25,7 +25,7 @@ module Gitlab end def importers(project) - if project.group.present? && Feature.enabled?(:github_importer_single_endpoint_notes_import, project.group, type: :ops) + if import_settings(project).enabled?(:single_endpoint_notes_import) [ Importer::SingleEndpointMergeRequestNotesImporter, Importer::SingleEndpointIssueNotesImporter diff --git a/app/workers/gitlab/github_import/stage/import_repository_worker.rb b/app/workers/gitlab/github_import/stage/import_repository_worker.rb index 3e914cc7590..8c1a2cd2677 100644 --- a/app/workers/gitlab/github_import/stage/import_repository_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_repository_worker.rb @@ -26,6 +26,11 @@ module Gitlab RefreshImportJidWorker.perform_in_the_future(project.id, jid) info(project.id, message: "starting importer", importer: 'Importer::RepositoryImporter') + + # If a user creates an issue while the import is in progress, this can lead to an import failure. + # The workaround is to allocate IIDs before starting the importer. + allocate_issues_internal_id!(project, client) + importer = Importer::RepositoryImporter.new(project, client) importer.execute @@ -56,6 +61,19 @@ module Gitlab def abort_on_failure true end + + private + + def allocate_issues_internal_id!(project, client) + return if InternalId.exists?(project: project, usage: :issues) # rubocop: disable CodeReuse/ActiveRecord + + options = { state: 'all', sort: 'number', direction: 'desc', per_page: '1' } + last_github_issue = client.each_object(:issues, project.import_source, options).first + + return unless last_github_issue + + Issue.track_project_iid!(project, last_github_issue[:number]) + end end end end |