From 619d0b6922a6cf95d291fbbf5fa3d09e772a1ea8 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 26 Feb 2020 18:09:24 +0000 Subject: Add latest changes from gitlab-org/gitlab@master --- app/controllers/projects/commits_controller.rb | 3 + .../repositories/git_http_controller.rb | 2 +- app/models/ci/pipeline.rb | 2 +- app/models/repository.rb | 1 + .../groups/import_export/export_service.rb | 16 +- .../groups/import_export/import_service.rb | 2 +- app/services/lfs/lock_file_service.rb | 4 +- app/services/lfs/unlock_file_service.rb | 4 +- .../projects/import_export/export_service.rb | 22 +- .../do-not-parse-undefined-severity-confidence.yml | 5 + changelogs/unreleased/find-commits-by-author.yml | 5 + ...ename-unauthorized-error-to-forbidden-error.yml | 5 + .../georgekoltsov-196188-cleanup-temp-exports.yml | 5 + danger/gemfile/Dangerfile | 36 - doc/administration/compliance.md | 1 + doc/administration/packages/index.md | 5 +- doc/development/import_project.md | 8 +- doc/user/clusters/applications.md | 21 +- doc/user/project/settings/import_export.md | 2 +- lib/api/internal/base.rb | 4 +- lib/gitlab/checks/branch_check.rb | 18 +- lib/gitlab/checks/diff_check.rb | 4 +- lib/gitlab/checks/lfs_check.rb | 2 +- lib/gitlab/checks/push_check.rb | 2 +- lib/gitlab/checks/tag_check.rb | 8 +- lib/gitlab/git/repository.rb | 1 + lib/gitlab/git_access.rb | 30 +- lib/gitlab/git_access_wiki.rb | 4 +- lib/gitlab/gitaly_client/commit_service.rb | 3 +- lib/gitlab/import_export.rb | 4 +- lib/gitlab/import_export/attribute_cleaner.rb | 4 +- lib/gitlab/import_export/base/object_builder.rb | 105 +++ lib/gitlab/import_export/base/relation_factory.rb | 308 ++++++++ lib/gitlab/import_export/base_object_builder.rb | 103 --- lib/gitlab/import_export/base_relation_factory.rb | 306 -------- lib/gitlab/import_export/group/import_export.yml | 78 ++ lib/gitlab/import_export/group/object_builder.rb | 57 ++ lib/gitlab/import_export/group/relation_factory.rb | 42 + lib/gitlab/import_export/group/tree_restorer.rb | 118 +++ lib/gitlab/import_export/group/tree_saver.rb | 57 ++ lib/gitlab/import_export/group_import_export.yml | 78 -- lib/gitlab/import_export/group_object_builder.rb | 55 -- .../import_export/group_project_object_builder.rb | 117 --- lib/gitlab/import_export/group_relation_factory.rb | 40 - lib/gitlab/import_export/group_tree_restorer.rb | 116 --- lib/gitlab/import_export/group_tree_saver.rb | 55 -- lib/gitlab/import_export/import_export.yml | 381 ---------- lib/gitlab/import_export/importer.rb | 4 +- lib/gitlab/import_export/members_mapper.rb | 4 +- lib/gitlab/import_export/project/import_export.yml | 381 ++++++++++ lib/gitlab/import_export/project/object_builder.rb | 119 +++ .../import_export/project/relation_factory.rb | 162 ++++ lib/gitlab/import_export/project/tree_loader.rb | 74 ++ lib/gitlab/import_export/project/tree_restorer.rb | 94 +++ lib/gitlab/import_export/project/tree_saver.rb | 70 ++ .../import_export/project_relation_factory.rb | 160 ---- lib/gitlab/import_export/project_tree_loader.rb | 72 -- lib/gitlab/import_export/project_tree_restorer.rb | 92 --- lib/gitlab/import_export/project_tree_saver.rb | 68 -- lib/gitlab/import_export/relation_tree_restorer.rb | 4 +- lib/gitlab_danger.rb | 1 - scripts/gemfile_lock_changed.sh | 26 + scripts/static-analysis | 3 +- spec/features/admin/admin_health_check_spec.rb | 2 +- spec/features/boards/multiple_boards_spec.rb | 10 +- spec/features/boards/new_issue_spec.rb | 8 +- spec/features/commits_spec.rb | 41 +- spec/features/dashboard/root_explore_spec.rb | 10 +- .../explore/user_explores_projects_spec.rb | 10 +- .../labels/user_sees_links_to_issuables_spec.rb | 2 +- spec/features/issues/user_views_issues_spec.rb | 2 +- .../merge_request/user_posts_notes_spec.rb | 3 +- .../user_sorts_merge_requests_spec.rb | 8 +- .../user_views_open_merge_requests_spec.rb | 6 +- .../milestones/user_creates_milestone_spec.rb | 4 +- .../milestones/user_edits_milestone_spec.rb | 6 +- .../milestones/user_promotes_milestone_spec.rb | 8 +- .../milestones/user_views_milestone_spec.rb | 8 +- .../milestones/user_views_milestones_spec.rb | 26 +- .../artifacts/user_downloads_artifacts_spec.rb | 6 +- .../projects/badges/pipeline_badge_spec.rb | 2 +- .../projects/branches/user_deletes_branch_spec.rb | 2 +- .../projects/branches/user_views_branches_spec.rb | 6 +- .../user_views_user_status_on_commit_spec.rb | 4 +- .../projects/labels/user_creates_labels_spec.rb | 6 +- .../projects/labels/user_edits_labels_spec.rb | 6 +- .../projects/labels/user_promotes_label_spec.rb | 8 +- .../labels/user_sees_links_to_issuables_spec.rb | 4 +- .../projects/labels/user_views_labels_spec.rb | 5 +- .../projects/settings/project_settings_spec.rb | 2 +- .../show/user_sees_git_instructions_spec.rb | 12 +- .../show/user_sees_last_commit_ci_status_spec.rb | 2 +- .../projects/show/user_sees_readme_spec.rb | 5 +- .../projects/user_sees_user_popover_spec.rb | 3 +- .../projects/wiki/markdown_preview_spec.rb | 2 +- .../projects/wiki/user_creates_wiki_page_spec.rb | 12 +- .../projects/wiki/user_views_wiki_page_spec.rb | 16 +- spec/features/read_only_spec.rb | 2 +- .../security/project/internal_access_spec.rb | 2 +- .../security/project/private_access_spec.rb | 2 +- .../security/project/public_access_spec.rb | 2 +- spec/features/user_sorts_things_spec.rb | 8 +- spec/fixtures/trace/sample_trace | 2 +- spec/lib/gitlab/checks/branch_check_spec.rb | 20 +- spec/lib/gitlab/checks/diff_check_spec.rb | 2 +- spec/lib/gitlab/checks/lfs_check_spec.rb | 2 +- spec/lib/gitlab/checks/push_check_spec.rb | 2 +- spec/lib/gitlab/checks/tag_check_spec.rb | 8 +- spec/lib/gitlab/git_access_spec.rb | 72 +- spec/lib/gitlab/git_access_wiki_spec.rb | 4 +- .../gitlab/gitaly_client/commit_service_spec.rb | 14 + .../import_export/base/object_builder_spec.rb | 53 ++ .../import_export/base/relation_factory_spec.rb | 143 ++++ .../import_export/base_object_builder_spec.rb | 53 -- .../import_export/base_relation_factory_spec.rb | 143 ---- spec/lib/gitlab/import_export/fork_spec.rb | 4 +- .../import_export/group/object_builder_spec.rb | 66 ++ .../import_export/group/relation_factory_spec.rb | 120 +++ .../import_export/group/tree_restorer_spec.rb | 153 ++++ .../gitlab/import_export/group/tree_saver_spec.rb | 202 +++++ .../import_export/group_object_builder_spec.rb | 66 -- .../group_project_object_builder_spec.rb | 153 ---- .../import_export/group_relation_factory_spec.rb | 120 --- .../import_export/group_tree_restorer_spec.rb | 153 ---- .../gitlab/import_export/group_tree_saver_spec.rb | 202 ----- spec/lib/gitlab/import_export/importer_spec.rb | 2 +- .../import_export/project/object_builder_spec.rb | 153 ++++ .../import_export/project/relation_factory_spec.rb | 326 ++++++++ .../import_export/project/tree_loader_spec.rb | 49 ++ .../import_export/project/tree_restorer_spec.rb | 843 ++++++++++++++++++++ .../import_export/project/tree_saver_spec.rb | 397 ++++++++++ .../import_export/project_relation_factory_spec.rb | 326 -------- .../import_export/project_tree_loader_spec.rb | 49 -- .../import_export/project_tree_restorer_spec.rb | 844 --------------------- .../import_export/project_tree_saver_spec.rb | 397 ---------- .../import_export/relation_rename_service_spec.rb | 4 +- .../import_export/relation_tree_restorer_spec.rb | 4 +- spec/lib/gitlab_danger_spec.rb | 2 +- spec/models/repository_spec.rb | 15 + spec/requests/api/internal/base_spec.rb | 18 +- .../groups/import_export/export_service_spec.rb | 51 +- .../projects/import_export/export_service_spec.rb | 25 +- spec/support/helpers/wiki_helpers.rb | 5 + spec/support/import_export/common_util.rb | 4 +- spec/support/import_export/configuration_helper.rb | 4 +- .../wiki_file_attachments_shared_examples.rb | 2 +- .../project_tree_restorer_shared_examples.rb | 2 +- 147 files changed, 4641 insertions(+), 4494 deletions(-) create mode 100644 changelogs/unreleased/do-not-parse-undefined-severity-confidence.yml create mode 100644 changelogs/unreleased/find-commits-by-author.yml create mode 100644 changelogs/unreleased/fj-rename-unauthorized-error-to-forbidden-error.yml create mode 100644 changelogs/unreleased/georgekoltsov-196188-cleanup-temp-exports.yml delete mode 100644 danger/gemfile/Dangerfile create mode 100644 lib/gitlab/import_export/base/object_builder.rb create mode 100644 lib/gitlab/import_export/base/relation_factory.rb delete mode 100644 lib/gitlab/import_export/base_object_builder.rb delete mode 100644 lib/gitlab/import_export/base_relation_factory.rb create mode 100644 lib/gitlab/import_export/group/import_export.yml create mode 100644 lib/gitlab/import_export/group/object_builder.rb create mode 100644 lib/gitlab/import_export/group/relation_factory.rb create mode 100644 lib/gitlab/import_export/group/tree_restorer.rb create mode 100644 lib/gitlab/import_export/group/tree_saver.rb delete mode 100644 lib/gitlab/import_export/group_import_export.yml delete mode 100644 lib/gitlab/import_export/group_object_builder.rb delete mode 100644 lib/gitlab/import_export/group_project_object_builder.rb delete mode 100644 lib/gitlab/import_export/group_relation_factory.rb delete mode 100644 lib/gitlab/import_export/group_tree_restorer.rb delete mode 100644 lib/gitlab/import_export/group_tree_saver.rb delete mode 100644 lib/gitlab/import_export/import_export.yml create mode 100644 lib/gitlab/import_export/project/import_export.yml create mode 100644 lib/gitlab/import_export/project/object_builder.rb create mode 100644 lib/gitlab/import_export/project/relation_factory.rb create mode 100644 lib/gitlab/import_export/project/tree_loader.rb create mode 100644 lib/gitlab/import_export/project/tree_restorer.rb create mode 100644 lib/gitlab/import_export/project/tree_saver.rb delete mode 100644 lib/gitlab/import_export/project_relation_factory.rb delete mode 100644 lib/gitlab/import_export/project_tree_loader.rb delete mode 100644 lib/gitlab/import_export/project_tree_restorer.rb delete mode 100644 lib/gitlab/import_export/project_tree_saver.rb create mode 100755 scripts/gemfile_lock_changed.sh create mode 100644 spec/lib/gitlab/import_export/base/object_builder_spec.rb create mode 100644 spec/lib/gitlab/import_export/base/relation_factory_spec.rb delete mode 100644 spec/lib/gitlab/import_export/base_object_builder_spec.rb delete mode 100644 spec/lib/gitlab/import_export/base_relation_factory_spec.rb create mode 100644 spec/lib/gitlab/import_export/group/object_builder_spec.rb create mode 100644 spec/lib/gitlab/import_export/group/relation_factory_spec.rb create mode 100644 spec/lib/gitlab/import_export/group/tree_restorer_spec.rb create mode 100644 spec/lib/gitlab/import_export/group/tree_saver_spec.rb delete mode 100644 spec/lib/gitlab/import_export/group_object_builder_spec.rb delete mode 100644 spec/lib/gitlab/import_export/group_project_object_builder_spec.rb delete mode 100644 spec/lib/gitlab/import_export/group_relation_factory_spec.rb delete mode 100644 spec/lib/gitlab/import_export/group_tree_restorer_spec.rb delete mode 100644 spec/lib/gitlab/import_export/group_tree_saver_spec.rb create mode 100644 spec/lib/gitlab/import_export/project/object_builder_spec.rb create mode 100644 spec/lib/gitlab/import_export/project/relation_factory_spec.rb create mode 100644 spec/lib/gitlab/import_export/project/tree_loader_spec.rb create mode 100644 spec/lib/gitlab/import_export/project/tree_restorer_spec.rb create mode 100644 spec/lib/gitlab/import_export/project/tree_saver_spec.rb delete mode 100644 spec/lib/gitlab/import_export/project_relation_factory_spec.rb delete mode 100644 spec/lib/gitlab/import_export/project_tree_loader_spec.rb delete mode 100644 spec/lib/gitlab/import_export/project_tree_restorer_spec.rb delete mode 100644 spec/lib/gitlab/import_export/project_tree_saver_spec.rb diff --git a/app/controllers/projects/commits_controller.rb b/app/controllers/projects/commits_controller.rb index 15bb35dd0be..b161e44660e 100644 --- a/app/controllers/projects/commits_controller.rb +++ b/app/controllers/projects/commits_controller.rb @@ -64,10 +64,13 @@ class Projects::CommitsController < Projects::ApplicationController render_404 unless @path.empty? || request.format == :atom || @repository.blob_at(@commit.id, @path) || @repository.tree(@commit.id, @path).entries.present? @limit, @offset = (params[:limit] || 40).to_i, (params[:offset] || 0).to_i search = params[:search] + author = params[:author] @commits = if search.present? @repository.find_commits_by_message(search, @ref, @path, @limit, @offset) + elsif author.present? + @repository.commits(@ref, author: author, path: @path, limit: @limit, offset: @offset) else @repository.commits(@ref, path: @path, limit: @limit, offset: @offset) end diff --git a/app/controllers/repositories/git_http_controller.rb b/app/controllers/repositories/git_http_controller.rb index 75c79881264..5c2b6089bff 100644 --- a/app/controllers/repositories/git_http_controller.rb +++ b/app/controllers/repositories/git_http_controller.rb @@ -7,7 +7,7 @@ module Repositories before_action :access_check prepend_before_action :deny_head_requests, only: [:info_refs] - rescue_from Gitlab::GitAccess::UnauthorizedError, with: :render_403_with_exception + rescue_from Gitlab::GitAccess::ForbiddenError, with: :render_403_with_exception rescue_from Gitlab::GitAccess::NotFoundError, with: :render_404_with_exception rescue_from Gitlab::GitAccess::ProjectCreationError, with: :render_422_with_exception rescue_from Gitlab::GitAccess::TimeoutError, with: :render_503_with_exception diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 4ae64b6c8f1..869a2e8da20 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -599,7 +599,7 @@ module Ci # Manually set the notes for a Ci::Pipeline # There is no ActiveRecord relation between Ci::Pipeline and notes # as they are related to a commit sha. This method helps importing - # them using the +Gitlab::ImportExport::ProjectRelationFactory+ class. + # them using the +Gitlab::ImportExport::Project::RelationFactory+ class. def notes=(notes) notes.each do |note| note[:id] = nil diff --git a/app/models/repository.rb b/app/models/repository.rb index cddffa9bb1d..a53850bb068 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -139,6 +139,7 @@ class Repository repo: raw_repository, ref: ref, path: opts[:path], + author: opts[:author], follow: Array(opts[:path]).length == 1, limit: opts[:limit], offset: opts[:offset], diff --git a/app/services/groups/import_export/export_service.rb b/app/services/groups/import_export/export_service.rb index 2c3975961a8..aa484e7203c 100644 --- a/app/services/groups/import_export/export_service.rb +++ b/app/services/groups/import_export/export_service.rb @@ -18,6 +18,8 @@ module Groups end save! + ensure + cleanup end private @@ -28,7 +30,7 @@ module Groups if savers.all?(&:save) notify_success else - cleanup_and_notify_error! + notify_error! end end @@ -37,21 +39,19 @@ module Groups end def tree_exporter - Gitlab::ImportExport::GroupTreeSaver.new(group: @group, current_user: @current_user, shared: @shared, params: @params) + Gitlab::ImportExport::Group::TreeSaver.new(group: @group, current_user: @current_user, shared: @shared, params: @params) end def file_saver Gitlab::ImportExport::Saver.new(exportable: @group, shared: @shared) end - def cleanup_and_notify_error - FileUtils.rm_rf(shared.export_path) - - notify_error + def cleanup + FileUtils.rm_rf(shared.archive_path) if shared&.archive_path end - def cleanup_and_notify_error! - cleanup_and_notify_error + def notify_error! + notify_error raise Gitlab::ImportExport::Error.new(shared.errors.to_sentence) end diff --git a/app/services/groups/import_export/import_service.rb b/app/services/groups/import_export/import_service.rb index 628c8f5bac0..57d2d9855d1 100644 --- a/app/services/groups/import_export/import_service.rb +++ b/app/services/groups/import_export/import_service.rb @@ -34,7 +34,7 @@ module Groups end def restorer - @restorer ||= Gitlab::ImportExport::GroupTreeRestorer.new(user: @current_user, + @restorer ||= Gitlab::ImportExport::Group::TreeRestorer.new(user: @current_user, shared: @shared, group: @group, group_hash: nil) diff --git a/app/services/lfs/lock_file_service.rb b/app/services/lfs/lock_file_service.rb index 383a0d6b4e3..1b283018c16 100644 --- a/app/services/lfs/lock_file_service.rb +++ b/app/services/lfs/lock_file_service.rb @@ -4,13 +4,13 @@ module Lfs class LockFileService < BaseService def execute unless can?(current_user, :push_code, project) - raise Gitlab::GitAccess::UnauthorizedError, 'You have no permissions' + raise Gitlab::GitAccess::ForbiddenError, 'You have no permissions' end create_lock! rescue ActiveRecord::RecordNotUnique error('already locked', 409, current_lock) - rescue Gitlab::GitAccess::UnauthorizedError => ex + rescue Gitlab::GitAccess::ForbiddenError => ex error(ex.message, 403) rescue => ex error(ex.message, 500) diff --git a/app/services/lfs/unlock_file_service.rb b/app/services/lfs/unlock_file_service.rb index ea5a67b727f..a13e89904a0 100644 --- a/app/services/lfs/unlock_file_service.rb +++ b/app/services/lfs/unlock_file_service.rb @@ -4,11 +4,11 @@ module Lfs class UnlockFileService < BaseService def execute unless can?(current_user, :push_code, project) - raise Gitlab::GitAccess::UnauthorizedError, _('You have no permissions') + raise Gitlab::GitAccess::ForbiddenError, _('You have no permissions') end unlock_file - rescue Gitlab::GitAccess::UnauthorizedError => ex + rescue Gitlab::GitAccess::ForbiddenError => ex error(ex.message, 403) rescue ActiveRecord::RecordNotFound error(_('Lock not found'), 404) diff --git a/app/services/projects/import_export/export_service.rb b/app/services/projects/import_export/export_service.rb index 38859c1efa4..77fddc44085 100644 --- a/app/services/projects/import_export/export_service.rb +++ b/app/services/projects/import_export/export_service.rb @@ -14,6 +14,8 @@ module Projects save_all! execute_after_export_action(after_export_strategy) + ensure + cleanup end private @@ -24,7 +26,7 @@ module Projects return unless after_export_strategy unless after_export_strategy.execute(current_user, project) - cleanup_and_notify_error + notify_error end end @@ -33,7 +35,7 @@ module Projects Gitlab::ImportExport::Saver.save(exportable: project, shared: shared) notify_success else - cleanup_and_notify_error! + notify_error! end end @@ -54,7 +56,7 @@ module Projects end def project_tree_saver - Gitlab::ImportExport::ProjectTreeSaver.new(project: project, current_user: current_user, shared: shared, params: params) + Gitlab::ImportExport::Project::TreeSaver.new(project: project, current_user: current_user, shared: shared, params: params) end def uploads_saver @@ -73,16 +75,12 @@ module Projects Gitlab::ImportExport::LfsSaver.new(project: project, shared: shared) end - def cleanup_and_notify_error - Rails.logger.error("Import/Export - Project #{project.name} with ID: #{project.id} export error - #{shared.errors.join(', ')}") # rubocop:disable Gitlab/RailsLogger - - FileUtils.rm_rf(shared.export_path) - - notify_error + def cleanup + FileUtils.rm_rf(shared.archive_path) if shared&.archive_path end - def cleanup_and_notify_error! - cleanup_and_notify_error + def notify_error! + notify_error raise Gitlab::ImportExport::Error.new(shared.errors.to_sentence) end @@ -92,6 +90,8 @@ module Projects end def notify_error + Rails.logger.error("Import/Export - Project #{project.name} with ID: #{project.id} export error - #{shared.errors.join(', ')}") # rubocop:disable Gitlab/RailsLogger + notification_service.project_not_exported(project, current_user, shared.errors) end end diff --git a/changelogs/unreleased/do-not-parse-undefined-severity-confidence.yml b/changelogs/unreleased/do-not-parse-undefined-severity-confidence.yml new file mode 100644 index 00000000000..32efeccf971 --- /dev/null +++ b/changelogs/unreleased/do-not-parse-undefined-severity-confidence.yml @@ -0,0 +1,5 @@ +--- +title: Do not parse undefined severity and confidence from reports +merge_request: 25884 +author: +type: other diff --git a/changelogs/unreleased/find-commits-by-author.yml b/changelogs/unreleased/find-commits-by-author.yml new file mode 100644 index 00000000000..a0ef9c1f3af --- /dev/null +++ b/changelogs/unreleased/find-commits-by-author.yml @@ -0,0 +1,5 @@ +--- +title: Filter commits by author +merge_request: 25597 +author: +type: added diff --git a/changelogs/unreleased/fj-rename-unauthorized-error-to-forbidden-error.yml b/changelogs/unreleased/fj-rename-unauthorized-error-to-forbidden-error.yml new file mode 100644 index 00000000000..ca2c28c13ae --- /dev/null +++ b/changelogs/unreleased/fj-rename-unauthorized-error-to-forbidden-error.yml @@ -0,0 +1,5 @@ +--- +title: Align git returned error codes +merge_request: 25936 +author: +type: changed diff --git a/changelogs/unreleased/georgekoltsov-196188-cleanup-temp-exports.yml b/changelogs/unreleased/georgekoltsov-196188-cleanup-temp-exports.yml new file mode 100644 index 00000000000..d74a628dfe5 --- /dev/null +++ b/changelogs/unreleased/georgekoltsov-196188-cleanup-temp-exports.yml @@ -0,0 +1,5 @@ +--- +title: Ensure temp export data is removed if Group/Project export failed +merge_request: 25828 +author: +type: fixed diff --git a/danger/gemfile/Dangerfile b/danger/gemfile/Dangerfile deleted file mode 100644 index 07c4c07cfe8..00000000000 --- a/danger/gemfile/Dangerfile +++ /dev/null @@ -1,36 +0,0 @@ -GEMFILE_LOCK_NOT_UPDATED_MESSAGE_SHORT = <<~MSG.freeze -%s was updated but %s wasn't updated. -MSG - -GEMFILE_LOCK_NOT_UPDATED_MESSAGE_FULL = <<~MSG.freeze -**#{GEMFILE_LOCK_NOT_UPDATED_MESSAGE_SHORT}** - -Usually, when %s is updated, you should run -``` -bundle install -``` - -or - -``` -bundle update -``` - -and commit the %s changes. -MSG - -gemfile_modified = git.modified_files.include?("Gemfile") -gemfile_lock_modified = git.modified_files.include?("Gemfile.lock") - -if gemfile_modified && !gemfile_lock_modified - gitlab_danger = GitlabDanger.new(helper.gitlab_helper) - - format_str = gitlab_danger.ci? ? GEMFILE_LOCK_NOT_UPDATED_MESSAGE_FULL : GEMFILE_LOCK_NOT_UPDATED_MESSAGE_SHORT - - message = format(format_str, - gemfile: gitlab_danger.html_link("Gemfile"), - gemfile_lock: gitlab_danger.html_link("Gemfile.lock") - ) - - warn(message) -end diff --git a/doc/administration/compliance.md b/doc/administration/compliance.md index 447b69e14b4..52b6867a310 100644 --- a/doc/administration/compliance.md +++ b/doc/administration/compliance.md @@ -17,3 +17,4 @@ GitLab’s [security features](../security/README.md) may also help you meet rel |**[Audit logs](audit_events.md)**
To maintain the integrity of your code, GitLab Enterprise Edition Premium gives admins the ability to view any modifications made within the GitLab server in an advanced audit log system, so you can control, analyze, and track every change.|Premium+|| |**[Auditor users](auditor_users.md)**
Auditor users are users who are given read-only access to all projects, groups, and other resources on the GitLab instance.|Premium+|| |**[Credentials inventory](../user/admin_area/credentials_inventory.md)**
With a credentials inventory, GitLab administrators can keep track of the credentials used by all of the users in their GitLab instance. |Ultimate|| +|**Separation of Duties using [Protected branches](../user/project/protected_branches.md#protected-branches-approval-by-code-owners-premium) and [custom CI Configuration Paths](../user/project/pipelines/settings.md#custom-ci-configuration-path)**
GitLab Silver and Premium users can leverage GitLab's cross-project YAML configuration's to define deployers of code and developers of code. View the [Separation of Duties Deploy Project](https://gitlab.com/guided-explorations/separation-of-duties-deploy/blob/master/README.md) and [Separation of Duties Project](https://gitlab.com/guided-explorations/separation-of-duties/blob/master/README.md) to see how to use this set up to define these roles.|Premium+|| diff --git a/doc/administration/packages/index.md b/doc/administration/packages/index.md index 421b70709b5..a12aec3c7b3 100644 --- a/doc/administration/packages/index.md +++ b/doc/administration/packages/index.md @@ -118,7 +118,10 @@ upload packages: #'path_style' => false # If true, use 'host/bucket_name/object' instead of 'bucket_name.host/object'. } ``` - + + NOTE: **Note:** + Some build tools, like Gradle, must make `HEAD` requests to Amazon S3 to pull a dependency’s metadata. The `gitlab_rails['packages_object_store_proxy_download']` property must be set to `true`. Without this setting, GitLab won't act as a proxy to the Amazon S3 service, and will instead return the signed URL. This will cause a `HTTP 403 Forbidden` response, since Amazon S3 expects a signed URL. + 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. diff --git a/doc/development/import_project.md b/doc/development/import_project.md index b969cb5f1c4..e92d18b7ace 100644 --- a/doc/development/import_project.md +++ b/doc/development/import_project.md @@ -81,7 +81,7 @@ The last option is to import a project using a Rails console: sudo -u git -H bundle exec rails console RAILS_ENV=production ``` -1. Create a project and run `ProjectTreeRestorer`: +1. Create a project and run `Project::TreeRestorer`: ```ruby shared_class = Struct.new(:export_path) do @@ -98,7 +98,7 @@ The last option is to import a project using a Rails console: begin #Enable Request store RequestStore.begin! - Gitlab::ImportExport::ProjectTreeRestorer.new(user: user, shared: shared, project: project).restore + Gitlab::ImportExport::Project::TreeRestorer.new(user: user, shared: shared, project: project).restore ensure RequestStore.end! RequestStore.clear! @@ -128,11 +128,11 @@ The last option is to import a project using a Rails console: For Performance testing, we should: - Import a quite large project, [`gitlabhq`](https://gitlab.com/gitlab-org/quality/performance-data#gitlab-performance-test-framework-data) should be a good example. -- Measure the execution time of `ProjectTreeRestorer`. +- Measure the execution time of `Project::TreeRestorer`. - Count the number of executed SQL queries during the restore. - Observe the number of GC cycles happening. -You can use this [snippet](https://gitlab.com/gitlab-org/gitlab/snippets/1924954), which will restore the project, and measure the execution time of `ProjectTreeRestorer`, number of SQL queries and number of GC cycles happening. +You can use this [snippet](https://gitlab.com/gitlab-org/gitlab/snippets/1924954), which will restore the project, and measure the execution time of `Project::TreeRestorer`, number of SQL queries and number of GC cycles happening. You can execute the script from the `gdk/gitlab` directory like this: diff --git a/doc/user/clusters/applications.md b/doc/user/clusters/applications.md index 7ea7feac9ba..33639c13b9d 100644 --- a/doc/user/clusters/applications.md +++ b/doc/user/clusters/applications.md @@ -257,7 +257,7 @@ use an A record. If your external endpoint is a hostname, use a CNAME record. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/21966) in GitLab 12.7. -A Web Application Firewall (WAF) is able to examine traffic being sent/received +A Web Application Firewall (WAF) examines traffic being sent or received, and can block malicious traffic before it reaches your application. The benefits of a WAF are: @@ -266,7 +266,7 @@ of a WAF are: - Access control for your application - Highly configurable logging and blocking rules -Out of the box, GitLab provides you with a WAF known as [`ModSecurity`](https://www.modsecurity.org/) +Out of the box, GitLab provides you with a WAF known as [`ModSecurity`](https://www.modsecurity.org/). ModSecurity is a toolkit for real-time web application monitoring, logging, and access control. With GitLab's offering, the [OWASP's Core Rule Set](https://www.modsecurity.org/CRS/Documentation/), @@ -288,9 +288,6 @@ when installing your [Ingress application](#ingress). If this is your first time using GitLab's WAF, we recommend you follow the [quick start guide](../../topics/web_application_firewall/quick_start_guide.md). -There is a small performance overhead by enabling ModSecurity. However, -if this is considered significant for your application, you can disable it. - There is a small performance overhead by enabling ModSecurity. If this is considered significant for your application, you can disable ModSecurity's rule engine for your deployed application by setting @@ -693,7 +690,7 @@ cilium: ``` The `clusterType` variable enables the recommended Helm variables for -a corresponding cluster type, the default value is blank. You can +a corresponding cluster type. The default value is blank. You can check the recommended variables for each cluster type in the official documentation: @@ -720,13 +717,13 @@ information. By default, Cilium will drop all non-whitelisted packets upon policy deployment. The audit mode is scheduled for release in [Cilium 1.8](https://github.com/cilium/cilium/pull/9970). In the audit -mode non-whitelisted packets will not be dropped, instead audit -notifications will be generated. GitLab provides alternative Docker +mode, non-whitelisted packets will not be dropped, and audit +notifications will be generated instead. GitLab provides alternative Docker images for Cilium with the audit patch included. You can switch to the custom build and enable the audit mode by adding the following to `.gitlab/managed-apps/cilium/values.yaml`: -```yml +```yaml global: registry: registry.gitlab.com/gitlab-org/defend/cilium policyAuditMode: true @@ -737,15 +734,15 @@ agent: ``` The Cilium monitor log for traffic is logged out by the -`cilium-monitor` sidecar container. You can check these logs via: +`cilium-monitor` sidecar container. You can check these logs with the following command: ```shell kubectl -n gitlab-managed-apps logs cilium-XXXX cilium-monitor ``` -You can disable the monitor log via `.gitlab/managed-apps/cilium/values.yaml`: +You can disable the monitor log in `.gitlab/managed-apps/cilium/values.yaml`: -```yml +```yaml agent: monitor: enabled: false diff --git a/doc/user/project/settings/import_export.md b/doc/user/project/settings/import_export.md index cdf6a789ec2..716078ed1d1 100644 --- a/doc/user/project/settings/import_export.md +++ b/doc/user/project/settings/import_export.md @@ -89,7 +89,7 @@ The following items will NOT be exported: NOTE: **Note:** For more details on the specific data persisted in a project export, see the -[`import_export.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/import_export/import_export.yml) file. +[`import_export.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/import_export/project/import_export.yml) file. ## Exporting a project and its data diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index 382bbeb66de..577a6e890d7 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -49,8 +49,8 @@ module API result = access_checker.check(params[:action], params[:changes]) @project ||= access_checker.project result - rescue Gitlab::GitAccess::UnauthorizedError => e - return response_with_status(code: 401, success: false, message: e.message) + rescue Gitlab::GitAccess::ForbiddenError => e + return response_with_status(code: 403, success: false, message: e.message) rescue Gitlab::GitAccess::TimeoutError => e return response_with_status(code: 503, success: false, message: e.message) rescue Gitlab::GitAccess::NotFoundError => e diff --git a/lib/gitlab/checks/branch_check.rb b/lib/gitlab/checks/branch_check.rb index 4ddc1c718c7..7be0ef05a49 100644 --- a/lib/gitlab/checks/branch_check.rb +++ b/lib/gitlab/checks/branch_check.rb @@ -28,7 +28,7 @@ module Gitlab logger.log_timed(LOG_MESSAGES[:delete_default_branch_check]) do if deletion? && branch_name == project.default_branch - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_default_branch] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:delete_default_branch] end end @@ -42,7 +42,7 @@ module Gitlab return unless ProtectedBranch.protected?(project, branch_name) # rubocop:disable Cop/AvoidReturnFromBlocks if forced_push? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:force_push_protected_branch] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:force_push_protected_branch] end end @@ -62,15 +62,15 @@ module Gitlab break if user_access.can_push_to_branch?(branch_name) unless user_access.can_merge_to_branch?(branch_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_branch] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:create_protected_branch] end unless safe_commit_for_new_protected_branch? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:invalid_commit_create_protected_branch] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:invalid_commit_create_protected_branch] end unless updated_from_web? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_create_protected_branch] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:non_web_create_protected_branch] end end end @@ -78,11 +78,11 @@ module Gitlab def protected_branch_deletion_checks logger.log_timed(LOG_MESSAGES[:protected_branch_deletion_checks]) do unless user_access.can_delete_branch?(branch_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_master_delete_protected_branch] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:non_master_delete_protected_branch] end unless updated_from_web? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_delete_protected_branch] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:non_web_delete_protected_branch] end end end @@ -91,11 +91,11 @@ module Gitlab logger.log_timed(LOG_MESSAGES[:protected_branch_push_checks]) do if matching_merge_request? unless user_access.can_merge_to_branch?(branch_name) || user_access.can_push_to_branch?(branch_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:merge_protected_branch] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:merge_protected_branch] end else unless user_access.can_push_to_branch?(branch_name) - raise GitAccess::UnauthorizedError, push_to_protected_branch_rejected_message + raise GitAccess::ForbiddenError, push_to_protected_branch_rejected_message end end end diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb index 5de71addd5f..0eb2b4c79ef 100644 --- a/lib/gitlab/checks/diff_check.rb +++ b/lib/gitlab/checks/diff_check.rb @@ -46,7 +46,7 @@ module Gitlab def validate_diff(diff) validations_for_diff.each do |validation| if error = validation.call(diff) - raise ::Gitlab::GitAccess::UnauthorizedError, error + raise ::Gitlab::GitAccess::ForbiddenError, error end end end @@ -77,7 +77,7 @@ module Gitlab logger.log_timed(LOG_MESSAGES[__method__]) do path_validations.each do |validation| if error = validation.call(file_paths) - raise ::Gitlab::GitAccess::UnauthorizedError, error + raise ::Gitlab::GitAccess::ForbiddenError, error end end end diff --git a/lib/gitlab/checks/lfs_check.rb b/lib/gitlab/checks/lfs_check.rb index 7b013567a03..f81c215d847 100644 --- a/lib/gitlab/checks/lfs_check.rb +++ b/lib/gitlab/checks/lfs_check.rb @@ -15,7 +15,7 @@ module Gitlab lfs_check = Checks::LfsIntegrity.new(project, newrev, logger.time_left) if lfs_check.objects_missing? - raise GitAccess::UnauthorizedError, ERROR_MESSAGE + raise GitAccess::ForbiddenError, ERROR_MESSAGE end end end diff --git a/lib/gitlab/checks/push_check.rb b/lib/gitlab/checks/push_check.rb index 91f8d0bdbc8..7cc5bc56cbb 100644 --- a/lib/gitlab/checks/push_check.rb +++ b/lib/gitlab/checks/push_check.rb @@ -6,7 +6,7 @@ module Gitlab def validate! logger.log_timed("Checking if you are allowed to push...") do unless can_push? - raise GitAccess::UnauthorizedError, GitAccess::ERROR_MESSAGES[:push_code] + raise GitAccess::ForbiddenError, GitAccess::ERROR_MESSAGES[:push_code] end end end diff --git a/lib/gitlab/checks/tag_check.rb b/lib/gitlab/checks/tag_check.rb index ced0612a7a3..a47e55cb160 100644 --- a/lib/gitlab/checks/tag_check.rb +++ b/lib/gitlab/checks/tag_check.rb @@ -20,7 +20,7 @@ module Gitlab logger.log_timed(LOG_MESSAGES[:tag_checks]) do if tag_exists? && user_access.cannot_do_action?(:admin_tag) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:change_existing_tags] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:change_existing_tags] end end @@ -33,11 +33,11 @@ module Gitlab logger.log_timed(LOG_MESSAGES[__method__]) do return unless ProtectedTag.protected?(project, tag_name) # rubocop:disable Cop/AvoidReturnFromBlocks - raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:update_protected_tag]) if update? - raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_protected_tag]) if deletion? + raise(GitAccess::ForbiddenError, ERROR_MESSAGES[:update_protected_tag]) if update? + raise(GitAccess::ForbiddenError, ERROR_MESSAGES[:delete_protected_tag]) if deletion? unless user_access.can_create_tag?(tag_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_tag] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:create_protected_tag] end end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 6bfe744a5cd..29324381cb5 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -322,6 +322,7 @@ module Gitlab limit: 10, offset: 0, path: nil, + author: nil, follow: false, skip_merges: false, after: nil, diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 906350e57c5..d6c87b858a8 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -6,7 +6,7 @@ module Gitlab class GitAccess include Gitlab::Utils::StrongMemoize - UnauthorizedError = Class.new(StandardError) + ForbiddenError = Class.new(StandardError) NotFoundError = Class.new(StandardError) ProjectCreationError = Class.new(StandardError) TimeoutError = Class.new(StandardError) @@ -125,7 +125,7 @@ module Gitlab return unless actor.is_a?(Key) unless actor.valid? - raise UnauthorizedError, "Your SSH key #{actor.errors[:key].first}." + raise ForbiddenError, "Your SSH key #{actor.errors[:key].first}." end end @@ -133,7 +133,7 @@ module Gitlab return if request_from_ci_build? unless protocol_allowed? - raise UnauthorizedError, "Git access over #{protocol.upcase} is not allowed" + raise ForbiddenError, "Git access over #{protocol.upcase} is not allowed" end end @@ -148,7 +148,7 @@ module Gitlab unless user_access.allowed? message = Gitlab::Auth::UserAccessDeniedReason.new(user).rejection_message - raise UnauthorizedError, message + raise ForbiddenError, message end end @@ -156,11 +156,11 @@ module Gitlab case cmd when *DOWNLOAD_COMMANDS unless authentication_abilities.include?(:download_code) || authentication_abilities.include?(:build_download_code) - raise UnauthorizedError, ERROR_MESSAGES[:auth_download] + raise ForbiddenError, ERROR_MESSAGES[:auth_download] end when *PUSH_COMMANDS unless authentication_abilities.include?(:push_code) - raise UnauthorizedError, ERROR_MESSAGES[:auth_upload] + raise ForbiddenError, ERROR_MESSAGES[:auth_upload] end end end @@ -189,19 +189,19 @@ module Gitlab def check_upload_pack_disabled! if http? && upload_pack_disabled_over_http? - raise UnauthorizedError, ERROR_MESSAGES[:upload_pack_disabled_over_http] + raise ForbiddenError, ERROR_MESSAGES[:upload_pack_disabled_over_http] end end def check_receive_pack_disabled! if http? && receive_pack_disabled_over_http? - raise UnauthorizedError, ERROR_MESSAGES[:receive_pack_disabled_over_http] + raise ForbiddenError, ERROR_MESSAGES[:receive_pack_disabled_over_http] end end def check_command_existence!(cmd) unless ALL_COMMANDS.include?(cmd) - raise UnauthorizedError, ERROR_MESSAGES[:command_not_allowed] + raise ForbiddenError, ERROR_MESSAGES[:command_not_allowed] end end @@ -209,7 +209,7 @@ module Gitlab return unless receive_pack?(cmd) if Gitlab::Database.read_only? - raise UnauthorizedError, push_to_read_only_message + raise ForbiddenError, push_to_read_only_message end end @@ -253,23 +253,23 @@ module Gitlab guest_can_download_code? unless passed - raise UnauthorizedError, ERROR_MESSAGES[:download] + raise ForbiddenError, ERROR_MESSAGES[:download] end end def check_push_access! if project.repository_read_only? - raise UnauthorizedError, ERROR_MESSAGES[:read_only] + raise ForbiddenError, ERROR_MESSAGES[:read_only] end if deploy_key? unless deploy_key.can_push_to?(project) - raise UnauthorizedError, ERROR_MESSAGES[:deploy_key_upload] + raise ForbiddenError, ERROR_MESSAGES[:deploy_key_upload] end elsif user # User access is verified in check_change_access! else - raise UnauthorizedError, ERROR_MESSAGES[:upload] + raise ForbiddenError, ERROR_MESSAGES[:upload] end check_change_access! @@ -284,7 +284,7 @@ module Gitlab project.any_branch_allows_collaboration?(user_access.user) unless can_push - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_code] + raise ForbiddenError, ERROR_MESSAGES[:push_code] end else # If there are worktrees with a HEAD pointing to a non-existent object, diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 3d0db753f6e..aad46937c32 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -19,11 +19,11 @@ module Gitlab def check_change_access! unless user_access.can_do_action?(:create_wiki) - raise UnauthorizedError, ERROR_MESSAGES[:write_to_wiki] + raise ForbiddenError, ERROR_MESSAGES[:write_to_wiki] end if Gitlab::Database.read_only? - raise UnauthorizedError, push_to_read_only_message + raise ForbiddenError, push_to_read_only_message end true diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index ac22f5bf419..1f914dc95d1 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -324,7 +324,8 @@ module Gitlab request.after = GitalyClient.timestamp(options[:after]) if options[:after] request.before = GitalyClient.timestamp(options[:before]) if options[:before] request.revision = encode_binary(options[:ref]) if options[:ref] - request.order = options[:order].upcase.sub('DEFAULT', 'NONE') if options[:order].present? + request.author = encode_binary(options[:author]) if options[:author] + request.order = options[:order].upcase.sub('DEFAULT', 'NONE') if options[:order].present? request.paths = encode_repeated(Array(options[:path])) if options[:path].present? diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 8ce6549c0c7..1033e6c4e05 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -43,7 +43,7 @@ module Gitlab end def config_file - Rails.root.join('lib/gitlab/import_export/import_export.yml') + Rails.root.join('lib/gitlab/import_export/project/import_export.yml') end def version_filename @@ -77,7 +77,7 @@ module Gitlab end def group_config_file - Rails.root.join('lib/gitlab/import_export/group_import_export.yml') + Rails.root.join('lib/gitlab/import_export/group/import_export.yml') end end end diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb index d1c20dff799..3bfc059dcd3 100644 --- a/lib/gitlab/import_export/attribute_cleaner.rb +++ b/lib/gitlab/import_export/attribute_cleaner.rb @@ -4,8 +4,8 @@ module Gitlab module ImportExport class AttributeCleaner ALLOWED_REFERENCES = [ - *ProjectRelationFactory::PROJECT_REFERENCES, - *ProjectRelationFactory::USER_REFERENCES, + *Gitlab::ImportExport::Project::RelationFactory::PROJECT_REFERENCES, + *Gitlab::ImportExport::Project::RelationFactory::USER_REFERENCES, 'group_id', 'commit_id', 'discussion_id', diff --git a/lib/gitlab/import_export/base/object_builder.rb b/lib/gitlab/import_export/base/object_builder.rb new file mode 100644 index 00000000000..109d2e233a5 --- /dev/null +++ b/lib/gitlab/import_export/base/object_builder.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Base + # Base class for Group & Project Object Builders. + # This class is not intended to be used on its own but + # rather inherited from. + # + # Cache keeps 1000 entries at most, 1000 is chosen based on: + # - one cache entry uses around 0.5K memory, 1000 items uses around 500K. + # (leave some buffer it should be less than 1M). It is afforable cost for project import. + # - for projects in Gitlab.com, it seems 1000 entries for labels/milestones is enough. + # For example, gitlab has ~970 labels and 26 milestones. + LRU_CACHE_SIZE = 1000 + + class ObjectBuilder + def self.build(*args) + new(*args).find + end + + def initialize(klass, attributes) + @klass = klass.ancestors.include?(Label) ? Label : klass + @attributes = attributes + + if Gitlab::SafeRequestStore.active? + @lru_cache = cache_from_request_store + @cache_key = [klass, attributes] + end + end + + def find + find_with_cache do + find_object || klass.create(prepare_attributes) + end + end + + protected + + def where_clauses + raise NotImplementedError + end + + # attributes wrapped in a method to be + # adjusted in sub-class if needed + def prepare_attributes + attributes + end + + private + + attr_reader :klass, :attributes, :lru_cache, :cache_key + + def find_with_cache + return yield unless lru_cache && cache_key + + lru_cache[cache_key] ||= yield + end + + def cache_from_request_store + Gitlab::SafeRequestStore[:lru_cache] ||= LruRedux::Cache.new(LRU_CACHE_SIZE) + end + + def find_object + klass.where(where_clause).first + end + + def where_clause + where_clauses.reduce(:and) + end + + def table + @table ||= klass.arel_table + end + + # Returns Arel clause: + # `"{table_name}"."{attrs.keys[0]}" = '{attrs.values[0]} AND {table_name}"."{attrs.keys[1]}" = '{attrs.values[1]}"` + # from the given Hash of attributes. + def attrs_to_arel(attrs) + attrs.map do |key, value| + table[key].eq(value) + end.reduce(:and) + end + + # Returns Arel clause `"{table_name}"."title" = '{attributes['title']}'` + # if attributes has 'title key, otherwise `nil`. + def where_clause_for_title + attrs_to_arel(attributes.slice('title')) + end + + # Returns Arel clause `"{table_name}"."description" = '{attributes['description']}'` + # if attributes has 'description key, otherwise `nil`. + def where_clause_for_description + attrs_to_arel(attributes.slice('description')) + end + + # Returns Arel clause `"{table_name}"."created_at" = '{attributes['created_at']}'` + # if attributes has 'created_at key, otherwise `nil`. + def where_clause_for_created_at + attrs_to_arel(attributes.slice('created_at')) + end + end + end + end +end diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb new file mode 100644 index 00000000000..688627d1f2f --- /dev/null +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -0,0 +1,308 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Base + class RelationFactory + include Gitlab::Utils::StrongMemoize + + IMPORTED_OBJECT_MAX_RETRIES = 5.freeze + + OVERRIDES = {}.freeze + EXISTING_OBJECT_RELATIONS = %i[].freeze + + # This represents all relations that have unique key on `project_id` or `group_id` + UNIQUE_RELATIONS = %i[].freeze + + USER_REFERENCES = %w[ + author_id + assignee_id + updated_by_id + merged_by_id + latest_closed_by_id + user_id + created_by_id + last_edited_by_id + merge_user_id + resolved_by_id + closed_by_id + owner_id + ].freeze + + TOKEN_RESET_MODELS = %i[Project Namespace Group Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze + + def self.create(*args) + new(*args).create + end + + def self.relation_class(relation_name) + # There are scenarios where the model is pluralized (e.g. + # MergeRequest::Metrics), and we don't want to force it to singular + # with #classify. + relation_name.to_s.classify.constantize + rescue NameError + relation_name.to_s.constantize + end + + def initialize(relation_sym:, relation_hash:, members_mapper:, object_builder:, user:, importable:, excluded_keys: []) + @relation_name = self.class.overrides[relation_sym]&.to_sym || relation_sym + @relation_hash = relation_hash.except('noteable_id') + @members_mapper = members_mapper + @object_builder = object_builder + @user = user + @importable = importable + @imported_object_retries = 0 + @relation_hash[importable_column_name] = @importable.id + + # Remove excluded keys from relation_hash + # We don't do this in the parsed_relation_hash because of the 'transformed attributes' + # For example, MergeRequestDiffFiles exports its diff attribute as utf8_diff. Then, + # in the create method that attribute is renamed to diff. And because diff is an excluded key, + # if we clean the excluded keys in the parsed_relation_hash, it will be removed + # from the object attributes and the export will fail. + @relation_hash.except!(*excluded_keys) + end + + # Creates an object from an actual model with name "relation_sym" with params from + # the relation_hash, updating references with new object IDs, mapping users using + # the "members_mapper" object, also updating notes if required. + def create + return if invalid_relation? + + setup_base_models + setup_models + + generate_imported_object + end + + def self.overrides + self::OVERRIDES + end + + def self.existing_object_relations + self::EXISTING_OBJECT_RELATIONS + end + + private + + def invalid_relation? + false + end + + def setup_models + raise NotImplementedError + end + + def unique_relations + # define in sub-class if any + self.class::UNIQUE_RELATIONS + end + + def setup_base_models + update_user_references + remove_duplicate_assignees + reset_tokens! + remove_encrypted_attributes! + end + + def update_user_references + self.class::USER_REFERENCES.each do |reference| + if @relation_hash[reference] + @relation_hash[reference] = @members_mapper.map[@relation_hash[reference]] + end + end + end + + def remove_duplicate_assignees + return unless @relation_hash['issue_assignees'] + + # When an assignee did not exist in the members mapper, the importer is + # assigned. We only need to assign each user once. + @relation_hash['issue_assignees'].uniq!(&:user_id) + end + + def generate_imported_object + imported_object + end + + def reset_tokens! + return unless Gitlab::ImportExport.reset_tokens? && self.class::TOKEN_RESET_MODELS.include?(@relation_name) + + # If we import/export to the same instance, tokens will have to be reset. + # We also have to reset them to avoid issues when the gitlab secrets file cannot be copied across. + relation_class.attribute_names.select { |name| name.include?('token') }.each do |token| + @relation_hash[token] = nil + end + end + + def remove_encrypted_attributes! + return unless relation_class.respond_to?(:encrypted_attributes) && relation_class.encrypted_attributes.any? + + relation_class.encrypted_attributes.each_key do |key| + @relation_hash[key.to_s] = nil + end + end + + def relation_class + @relation_class ||= self.class.relation_class(@relation_name) + end + + def importable_column_name + importable_class_name.concat('_id') + end + + def importable_class_name + @importable.class.to_s.downcase + end + + def imported_object + if existing_or_new_object.respond_to?(:importing) + existing_or_new_object.importing = true + end + + existing_or_new_object + rescue ActiveRecord::RecordNotUnique + # as the operation is not atomic, retry in the unlikely scenario an INSERT is + # performed on the same object between the SELECT and the INSERT + @imported_object_retries += 1 + retry if @imported_object_retries < IMPORTED_OBJECT_MAX_RETRIES + end + + def parsed_relation_hash + @parsed_relation_hash ||= Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: @relation_hash, + relation_class: relation_class) + end + + def existing_or_new_object + # Only find existing records to avoid mapping tables such as milestones + # Otherwise always create the record, skipping the extra SELECT clause. + @existing_or_new_object ||= begin + if existing_object? + attribute_hash = attribute_hash_for(['events']) + + existing_object.assign_attributes(attribute_hash) if attribute_hash.any? + + existing_object + else + # Because of single-type inheritance, we need to be careful to use the `type` field + # See https://gitlab.com/gitlab-org/gitlab/issues/34860#note_235321497 + inheritance_column = relation_class.try(:inheritance_column) + inheritance_attributes = parsed_relation_hash.slice(inheritance_column) + object = relation_class.new(inheritance_attributes) + object.assign_attributes(parsed_relation_hash) + object + end + end + end + + def attribute_hash_for(attributes) + attributes.each_with_object({}) do |hash, value| + hash[value] = parsed_relation_hash.delete(value) if parsed_relation_hash[value] + hash + end + end + + def existing_object + @existing_object ||= find_or_create_object! + end + + def unique_relation_object + unique_relation_object = relation_class.find_or_create_by(importable_column_name => @importable.id) + unique_relation_object.assign_attributes(parsed_relation_hash) + unique_relation_object + end + + def find_or_create_object! + return unique_relation_object if unique_relation? + + # Can't use IDs as validation exists calling `group` or `project` attributes + finder_hash = parsed_relation_hash.tap do |hash| + if relation_class.attribute_method?('group_id') && @importable.is_a?(::Project) + hash['group'] = @importable.group + end + + hash[importable_class_name] = @importable if relation_class.reflect_on_association(importable_class_name.to_sym) + hash.delete(importable_column_name) + end + + @object_builder.build(relation_class, finder_hash) + end + + def setup_note + set_note_author + # attachment is deprecated and note uploads are handled by Markdown uploader + @relation_hash['attachment'] = nil + end + + # Sets the author for a note. If the user importing the project + # has admin access, an actual mapping with new project members + # will be used. Otherwise, a note stating the original author name + # is left. + def set_note_author + old_author_id = @relation_hash['author_id'] + author = @relation_hash.delete('author') + + update_note_for_missing_author(author['name']) unless has_author?(old_author_id) + end + + def has_author?(old_author_id) + admin_user? && @members_mapper.include?(old_author_id) + end + + def missing_author_note(updated_at, author_name) + timestamp = updated_at.split('.').first + "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*" + end + + def update_note_for_missing_author(author_name) + @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank? + @relation_hash['note'] = "#{@relation_hash['note']}#{missing_author_note(@relation_hash['updated_at'], author_name)}" + end + + def admin_user? + @user.admin? + end + + def existing_object? + strong_memoize(:_existing_object) do + self.class.existing_object_relations.include?(@relation_name) || unique_relation? + end + end + + def unique_relation? + strong_memoize(:unique_relation) do + importable_foreign_key.present? && + (has_unique_index_on_importable_fk? || uses_importable_fk_as_primary_key?) + end + end + + def has_unique_index_on_importable_fk? + cache = cached_has_unique_index_on_importable_fk + table_name = relation_class.table_name + return cache[table_name] if cache.has_key?(table_name) + + index_exists = + ActiveRecord::Base.connection.index_exists?( + relation_class.table_name, + importable_foreign_key, + unique: true) + + cache[table_name] = index_exists + end + + # Avoid unnecessary DB requests + def cached_has_unique_index_on_importable_fk + Thread.current[:cached_has_unique_index_on_importable_fk] ||= {} + end + + def uses_importable_fk_as_primary_key? + relation_class.primary_key == importable_foreign_key + end + + def importable_foreign_key + relation_class.reflect_on_association(importable_class_name.to_sym)&.foreign_key + end + end + end + end +end diff --git a/lib/gitlab/import_export/base_object_builder.rb b/lib/gitlab/import_export/base_object_builder.rb deleted file mode 100644 index ec66b7a7a4f..00000000000 --- a/lib/gitlab/import_export/base_object_builder.rb +++ /dev/null @@ -1,103 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - # Base class for Group & Project Object Builders. - # This class is not intended to be used on its own but - # rather inherited from. - # - # Cache keeps 1000 entries at most, 1000 is chosen based on: - # - one cache entry uses around 0.5K memory, 1000 items uses around 500K. - # (leave some buffer it should be less than 1M). It is afforable cost for project import. - # - for projects in Gitlab.com, it seems 1000 entries for labels/milestones is enough. - # For example, gitlab has ~970 labels and 26 milestones. - LRU_CACHE_SIZE = 1000 - - class BaseObjectBuilder - def self.build(*args) - new(*args).find - end - - def initialize(klass, attributes) - @klass = klass.ancestors.include?(Label) ? Label : klass - @attributes = attributes - - if Gitlab::SafeRequestStore.active? - @lru_cache = cache_from_request_store - @cache_key = [klass, attributes] - end - end - - def find - find_with_cache do - find_object || klass.create(prepare_attributes) - end - end - - protected - - def where_clauses - raise NotImplementedError - end - - # attributes wrapped in a method to be - # adjusted in sub-class if needed - def prepare_attributes - attributes - end - - private - - attr_reader :klass, :attributes, :lru_cache, :cache_key - - def find_with_cache - return yield unless lru_cache && cache_key - - lru_cache[cache_key] ||= yield - end - - def cache_from_request_store - Gitlab::SafeRequestStore[:lru_cache] ||= LruRedux::Cache.new(LRU_CACHE_SIZE) - end - - def find_object - klass.where(where_clause).first - end - - def where_clause - where_clauses.reduce(:and) - end - - def table - @table ||= klass.arel_table - end - - # Returns Arel clause: - # `"{table_name}"."{attrs.keys[0]}" = '{attrs.values[0]} AND {table_name}"."{attrs.keys[1]}" = '{attrs.values[1]}"` - # from the given Hash of attributes. - def attrs_to_arel(attrs) - attrs.map do |key, value| - table[key].eq(value) - end.reduce(:and) - end - - # Returns Arel clause `"{table_name}"."title" = '{attributes['title']}'` - # if attributes has 'title key, otherwise `nil`. - def where_clause_for_title - attrs_to_arel(attributes.slice('title')) - end - - # Returns Arel clause `"{table_name}"."description" = '{attributes['description']}'` - # if attributes has 'description key, otherwise `nil`. - def where_clause_for_description - attrs_to_arel(attributes.slice('description')) - end - - # Returns Arel clause `"{table_name}"."created_at" = '{attributes['created_at']}'` - # if attributes has 'created_at key, otherwise `nil`. - def where_clause_for_created_at - attrs_to_arel(attributes.slice('created_at')) - end - end - end -end diff --git a/lib/gitlab/import_export/base_relation_factory.rb b/lib/gitlab/import_export/base_relation_factory.rb deleted file mode 100644 index fcb516fb3a1..00000000000 --- a/lib/gitlab/import_export/base_relation_factory.rb +++ /dev/null @@ -1,306 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class BaseRelationFactory - include Gitlab::Utils::StrongMemoize - - IMPORTED_OBJECT_MAX_RETRIES = 5.freeze - - OVERRIDES = {}.freeze - EXISTING_OBJECT_RELATIONS = %i[].freeze - - # This represents all relations that have unique key on `project_id` or `group_id` - UNIQUE_RELATIONS = %i[].freeze - - USER_REFERENCES = %w[ - author_id - assignee_id - updated_by_id - merged_by_id - latest_closed_by_id - user_id - created_by_id - last_edited_by_id - merge_user_id - resolved_by_id - closed_by_id - owner_id - ].freeze - - TOKEN_RESET_MODELS = %i[Project Namespace Group Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze - - def self.create(*args) - new(*args).create - end - - def self.relation_class(relation_name) - # There are scenarios where the model is pluralized (e.g. - # MergeRequest::Metrics), and we don't want to force it to singular - # with #classify. - relation_name.to_s.classify.constantize - rescue NameError - relation_name.to_s.constantize - end - - def initialize(relation_sym:, relation_hash:, members_mapper:, object_builder:, user:, importable:, excluded_keys: []) - @relation_name = self.class.overrides[relation_sym]&.to_sym || relation_sym - @relation_hash = relation_hash.except('noteable_id') - @members_mapper = members_mapper - @object_builder = object_builder - @user = user - @importable = importable - @imported_object_retries = 0 - @relation_hash[importable_column_name] = @importable.id - - # Remove excluded keys from relation_hash - # We don't do this in the parsed_relation_hash because of the 'transformed attributes' - # For example, MergeRequestDiffFiles exports its diff attribute as utf8_diff. Then, - # in the create method that attribute is renamed to diff. And because diff is an excluded key, - # if we clean the excluded keys in the parsed_relation_hash, it will be removed - # from the object attributes and the export will fail. - @relation_hash.except!(*excluded_keys) - end - - # Creates an object from an actual model with name "relation_sym" with params from - # the relation_hash, updating references with new object IDs, mapping users using - # the "members_mapper" object, also updating notes if required. - def create - return if invalid_relation? - - setup_base_models - setup_models - - generate_imported_object - end - - def self.overrides - self::OVERRIDES - end - - def self.existing_object_relations - self::EXISTING_OBJECT_RELATIONS - end - - private - - def invalid_relation? - false - end - - def setup_models - raise NotImplementedError - end - - def unique_relations - # define in sub-class if any - self.class::UNIQUE_RELATIONS - end - - def setup_base_models - update_user_references - remove_duplicate_assignees - reset_tokens! - remove_encrypted_attributes! - end - - def update_user_references - self.class::USER_REFERENCES.each do |reference| - if @relation_hash[reference] - @relation_hash[reference] = @members_mapper.map[@relation_hash[reference]] - end - end - end - - def remove_duplicate_assignees - return unless @relation_hash['issue_assignees'] - - # When an assignee did not exist in the members mapper, the importer is - # assigned. We only need to assign each user once. - @relation_hash['issue_assignees'].uniq!(&:user_id) - end - - def generate_imported_object - imported_object - end - - def reset_tokens! - return unless Gitlab::ImportExport.reset_tokens? && self.class::TOKEN_RESET_MODELS.include?(@relation_name) - - # If we import/export to the same instance, tokens will have to be reset. - # We also have to reset them to avoid issues when the gitlab secrets file cannot be copied across. - relation_class.attribute_names.select { |name| name.include?('token') }.each do |token| - @relation_hash[token] = nil - end - end - - def remove_encrypted_attributes! - return unless relation_class.respond_to?(:encrypted_attributes) && relation_class.encrypted_attributes.any? - - relation_class.encrypted_attributes.each_key do |key| - @relation_hash[key.to_s] = nil - end - end - - def relation_class - @relation_class ||= self.class.relation_class(@relation_name) - end - - def importable_column_name - importable_class_name.concat('_id') - end - - def importable_class_name - @importable.class.to_s.downcase - end - - def imported_object - if existing_or_new_object.respond_to?(:importing) - existing_or_new_object.importing = true - end - - existing_or_new_object - rescue ActiveRecord::RecordNotUnique - # as the operation is not atomic, retry in the unlikely scenario an INSERT is - # performed on the same object between the SELECT and the INSERT - @imported_object_retries += 1 - retry if @imported_object_retries < IMPORTED_OBJECT_MAX_RETRIES - end - - def parsed_relation_hash - @parsed_relation_hash ||= Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: @relation_hash, - relation_class: relation_class) - end - - def existing_or_new_object - # Only find existing records to avoid mapping tables such as milestones - # Otherwise always create the record, skipping the extra SELECT clause. - @existing_or_new_object ||= begin - if existing_object? - attribute_hash = attribute_hash_for(['events']) - - existing_object.assign_attributes(attribute_hash) if attribute_hash.any? - - existing_object - else - # Because of single-type inheritance, we need to be careful to use the `type` field - # See https://gitlab.com/gitlab-org/gitlab/issues/34860#note_235321497 - inheritance_column = relation_class.try(:inheritance_column) - inheritance_attributes = parsed_relation_hash.slice(inheritance_column) - object = relation_class.new(inheritance_attributes) - object.assign_attributes(parsed_relation_hash) - object - end - end - end - - def attribute_hash_for(attributes) - attributes.each_with_object({}) do |hash, value| - hash[value] = parsed_relation_hash.delete(value) if parsed_relation_hash[value] - hash - end - end - - def existing_object - @existing_object ||= find_or_create_object! - end - - def unique_relation_object - unique_relation_object = relation_class.find_or_create_by(importable_column_name => @importable.id) - unique_relation_object.assign_attributes(parsed_relation_hash) - unique_relation_object - end - - def find_or_create_object! - return unique_relation_object if unique_relation? - - # Can't use IDs as validation exists calling `group` or `project` attributes - finder_hash = parsed_relation_hash.tap do |hash| - if relation_class.attribute_method?('group_id') && @importable.is_a?(Project) - hash['group'] = @importable.group - end - - hash[importable_class_name] = @importable if relation_class.reflect_on_association(importable_class_name.to_sym) - hash.delete(importable_column_name) - end - - @object_builder.build(relation_class, finder_hash) - end - - def setup_note - set_note_author - # attachment is deprecated and note uploads are handled by Markdown uploader - @relation_hash['attachment'] = nil - end - - # Sets the author for a note. If the user importing the project - # has admin access, an actual mapping with new project members - # will be used. Otherwise, a note stating the original author name - # is left. - def set_note_author - old_author_id = @relation_hash['author_id'] - author = @relation_hash.delete('author') - - update_note_for_missing_author(author['name']) unless has_author?(old_author_id) - end - - def has_author?(old_author_id) - admin_user? && @members_mapper.include?(old_author_id) - end - - def missing_author_note(updated_at, author_name) - timestamp = updated_at.split('.').first - "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*" - end - - def update_note_for_missing_author(author_name) - @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank? - @relation_hash['note'] = "#{@relation_hash['note']}#{missing_author_note(@relation_hash['updated_at'], author_name)}" - end - - def admin_user? - @user.admin? - end - - def existing_object? - strong_memoize(:_existing_object) do - self.class.existing_object_relations.include?(@relation_name) || unique_relation? - end - end - - def unique_relation? - strong_memoize(:unique_relation) do - importable_foreign_key.present? && - (has_unique_index_on_importable_fk? || uses_importable_fk_as_primary_key?) - end - end - - def has_unique_index_on_importable_fk? - cache = cached_has_unique_index_on_importable_fk - table_name = relation_class.table_name - return cache[table_name] if cache.has_key?(table_name) - - index_exists = - ActiveRecord::Base.connection.index_exists?( - relation_class.table_name, - importable_foreign_key, - unique: true) - - cache[table_name] = index_exists - end - - # Avoid unnecessary DB requests - def cached_has_unique_index_on_importable_fk - Thread.current[:cached_has_unique_index_on_importable_fk] ||= {} - end - - def uses_importable_fk_as_primary_key? - relation_class.primary_key == importable_foreign_key - end - - def importable_foreign_key - relation_class.reflect_on_association(importable_class_name.to_sym)&.foreign_key - end - end - end -end diff --git a/lib/gitlab/import_export/group/import_export.yml b/lib/gitlab/import_export/group/import_export.yml new file mode 100644 index 00000000000..d4e0ff12373 --- /dev/null +++ b/lib/gitlab/import_export/group/import_export.yml @@ -0,0 +1,78 @@ +# Model relationships to be included in the group import/export +# +# This list _must_ only contain relationships that are available to both FOSS and +# Enterprise editions. EE specific relationships must be defined in the `ee` section further +# down below. +tree: + group: + - :milestones + - :badges + - labels: + - :priorities + - boards: + - lists: + - label: + - :priorities + - :board + - members: + - :user + +included_attributes: + user: + - :id + - :email + - :username + author: + - :name + +excluded_attributes: + group: + - :id + - :owner_id + - :parent_id + - :created_at + - :updated_at + - :runners_token + - :runners_token_encrypted + - :saml_discovery_token + - :visibility_level + +methods: + labels: + - :type + label: + - :type + badges: + - :type + notes: + - :type + events: + - :action + lists: + - :list_type + +preloads: + +# EE specific relationships and settings to include. All of this will be merged +# into the previous structures if EE is used. +ee: + tree: + group: + - epics: + - :parent + - :award_emoji + - events: + - :push_event_payload + - notes: + - :author + - :award_emoji + - events: + - :push_event_payload + - boards: + - :board_assignee + - labels: + - :priorities + - lists: + - milestone: + - events: + - :push_event_payload diff --git a/lib/gitlab/import_export/group/object_builder.rb b/lib/gitlab/import_export/group/object_builder.rb new file mode 100644 index 00000000000..e171a31348e --- /dev/null +++ b/lib/gitlab/import_export/group/object_builder.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Group + # Given a class, it finds or creates a new object at group level. + # + # Example: + # `Group::ObjectBuilder.build(Label, label_attributes)` + # finds or initializes a label with the given attributes. + class ObjectBuilder < Base::ObjectBuilder + def self.build(*args) + ::Group.transaction do + super + end + end + + def initialize(klass, attributes) + super + + @group = @attributes['group'] + + update_description + end + + private + + attr_reader :group + + # Convert description empty string to nil + # due to existing object being saved with description: nil + # Which makes object lookup to fail since nil != '' + def update_description + attributes['description'] = nil if attributes['description'] == '' + end + + def where_clauses + [ + where_clause_base, + where_clause_for_title, + where_clause_for_description, + where_clause_for_created_at + ].compact + end + + # Returns Arel clause `"{table_name}"."group_id" = {group.id}` + def where_clause_base + table[:group_id].in(group_and_ancestor_ids) + end + + def group_and_ancestor_ids + group.ancestors.map(&:id) << group.id + end + end + end + end +end diff --git a/lib/gitlab/import_export/group/relation_factory.rb b/lib/gitlab/import_export/group/relation_factory.rb new file mode 100644 index 00000000000..91637161377 --- /dev/null +++ b/lib/gitlab/import_export/group/relation_factory.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Group + class RelationFactory < Base::RelationFactory + OVERRIDES = { + labels: :group_labels, + priorities: :label_priorities, + label: :group_label, + parent: :epic + }.freeze + + EXISTING_OBJECT_RELATIONS = %i[ + epic + epics + milestone + milestones + label + labels + group_label + group_labels + ].freeze + + private + + def setup_models + setup_note if @relation_name == :notes + + update_group_references + end + + def update_group_references + return unless self.class.existing_object_relations.include?(@relation_name) + return unless @relation_hash['group_id'] + + @relation_hash['group_id'] = @importable.id + end + end + end + end +end diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb new file mode 100644 index 00000000000..e6f49dcac7a --- /dev/null +++ b/lib/gitlab/import_export/group/tree_restorer.rb @@ -0,0 +1,118 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Group + class TreeRestorer + attr_reader :user + attr_reader :shared + attr_reader :group + + def initialize(user:, shared:, group:, group_hash:) + @path = File.join(shared.export_path, 'group.json') + @user = user + @shared = shared + @group = group + @group_hash = group_hash + end + + def restore + @tree_hash = @group_hash || read_tree_hash + @group_members = @tree_hash.delete('members') + @children = @tree_hash.delete('children') + + if members_mapper.map && restorer.restore + @children&.each do |group_hash| + group = create_group(group_hash: group_hash, parent_group: @group) + shared = Gitlab::ImportExport::Shared.new(group) + + self.class.new( + user: @user, + shared: shared, + group: group, + group_hash: group_hash + ).restore + end + end + + return false if @shared.errors.any? + + true + rescue => e + @shared.error(e) + false + end + + private + + def read_tree_hash + json = IO.read(@path) + ActiveSupport::JSON.decode(json) + rescue => e + @shared.logger.error( + group_id: @group.id, + group_name: @group.name, + message: "Import/Export error: #{e.message}" + ) + + raise Gitlab::ImportExport::Error.new('Incorrect JSON format') + end + + def restorer + @relation_tree_restorer ||= RelationTreeRestorer.new( + user: @user, + shared: @shared, + importable: @group, + tree_hash: @tree_hash.except('name', 'path'), + members_mapper: members_mapper, + object_builder: object_builder, + relation_factory: relation_factory, + reader: reader + ) + end + + def create_group(group_hash:, parent_group:) + group_params = { + name: group_hash['name'], + path: group_hash['path'], + parent_id: parent_group&.id, + visibility_level: sub_group_visibility_level(group_hash, parent_group) + } + + ::Groups::CreateService.new(@user, group_params).execute + end + + def sub_group_visibility_level(group_hash, parent_group) + original_visibility_level = group_hash['visibility_level'] || Gitlab::VisibilityLevel::PRIVATE + + if parent_group && parent_group.visibility_level < original_visibility_level + Gitlab::VisibilityLevel.closest_allowed_level(parent_group.visibility_level) + else + original_visibility_level + end + end + + def members_mapper + @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @group_members, user: @user, importable: @group) + end + + def relation_factory + Gitlab::ImportExport::Group::RelationFactory + end + + def object_builder + Gitlab::ImportExport::Group::ObjectBuilder + end + + def reader + @reader ||= Gitlab::ImportExport::Reader.new( + shared: @shared, + config: Gitlab::ImportExport::Config.new( + config: Gitlab::ImportExport.group_config_file + ).to_h + ) + end + end + end + end +end diff --git a/lib/gitlab/import_export/group/tree_saver.rb b/lib/gitlab/import_export/group/tree_saver.rb new file mode 100644 index 00000000000..48f6925884b --- /dev/null +++ b/lib/gitlab/import_export/group/tree_saver.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Group + class TreeSaver + attr_reader :full_path, :shared + + def initialize(group:, current_user:, shared:, params: {}) + @params = params + @current_user = current_user + @shared = shared + @group = group + @full_path = File.join(@shared.export_path, ImportExport.group_filename) + end + + def save + group_tree = serialize(@group, reader.group_tree) + tree_saver.save(group_tree, @shared.export_path, ImportExport.group_filename) + + true + rescue => e + @shared.error(e) + false + end + + private + + def serialize(group, relations_tree) + group_tree = tree_saver.serialize(group, relations_tree) + + group.children.each do |child| + group_tree['children'] ||= [] + group_tree['children'] << serialize(child, relations_tree) + end + + group_tree + rescue => e + @shared.error(e) + end + + def reader + @reader ||= Gitlab::ImportExport::Reader.new( + shared: @shared, + config: Gitlab::ImportExport::Config.new( + config: Gitlab::ImportExport.group_config_file + ).to_h + ) + end + + def tree_saver + @tree_saver ||= RelationTreeSaver.new + end + end + end + end +end diff --git a/lib/gitlab/import_export/group_import_export.yml b/lib/gitlab/import_export/group_import_export.yml deleted file mode 100644 index d4e0ff12373..00000000000 --- a/lib/gitlab/import_export/group_import_export.yml +++ /dev/null @@ -1,78 +0,0 @@ -# Model relationships to be included in the group import/export -# -# This list _must_ only contain relationships that are available to both FOSS and -# Enterprise editions. EE specific relationships must be defined in the `ee` section further -# down below. -tree: - group: - - :milestones - - :badges - - labels: - - :priorities - - boards: - - lists: - - label: - - :priorities - - :board - - members: - - :user - -included_attributes: - user: - - :id - - :email - - :username - author: - - :name - -excluded_attributes: - group: - - :id - - :owner_id - - :parent_id - - :created_at - - :updated_at - - :runners_token - - :runners_token_encrypted - - :saml_discovery_token - - :visibility_level - -methods: - labels: - - :type - label: - - :type - badges: - - :type - notes: - - :type - events: - - :action - lists: - - :list_type - -preloads: - -# EE specific relationships and settings to include. All of this will be merged -# into the previous structures if EE is used. -ee: - tree: - group: - - epics: - - :parent - - :award_emoji - - events: - - :push_event_payload - - notes: - - :author - - :award_emoji - - events: - - :push_event_payload - - boards: - - :board_assignee - - labels: - - :priorities - - lists: - - milestone: - - events: - - :push_event_payload diff --git a/lib/gitlab/import_export/group_object_builder.rb b/lib/gitlab/import_export/group_object_builder.rb deleted file mode 100644 index 9796bfa07d4..00000000000 --- a/lib/gitlab/import_export/group_object_builder.rb +++ /dev/null @@ -1,55 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - # Given a class, it finds or creates a new object at group level. - # - # Example: - # `GroupObjectBuilder.build(Label, label_attributes)` - # finds or initializes a label with the given attributes. - class GroupObjectBuilder < BaseObjectBuilder - def self.build(*args) - Group.transaction do - super - end - end - - def initialize(klass, attributes) - super - - @group = @attributes['group'] - - update_description - end - - private - - attr_reader :group - - # Convert description empty string to nil - # due to existing object being saved with description: nil - # Which makes object lookup to fail since nil != '' - def update_description - attributes['description'] = nil if attributes['description'] == '' - end - - def where_clauses - [ - where_clause_base, - where_clause_for_title, - where_clause_for_description, - where_clause_for_created_at - ].compact - end - - # Returns Arel clause `"{table_name}"."group_id" = {group.id}` - def where_clause_base - table[:group_id].in(group_and_ancestor_ids) - end - - def group_and_ancestor_ids - group.ancestors.map(&:id) << group.id - end - end - end -end diff --git a/lib/gitlab/import_export/group_project_object_builder.rb b/lib/gitlab/import_export/group_project_object_builder.rb deleted file mode 100644 index 9e8f9d11393..00000000000 --- a/lib/gitlab/import_export/group_project_object_builder.rb +++ /dev/null @@ -1,117 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - # Given a class, it finds or creates a new object - # (initializes in the case of Label) at group or project level. - # If it does not exist in the group, it creates it at project level. - # - # Example: - # `GroupProjectObjectBuilder.build(Label, label_attributes)` - # finds or initializes a label with the given attributes. - # - # It also adds some logic around Group Labels/Milestones for edge cases. - class GroupProjectObjectBuilder < BaseObjectBuilder - def self.build(*args) - Project.transaction do - super - end - end - - def initialize(klass, attributes) - super - - @group = @attributes['group'] - @project = @attributes['project'] - end - - def find - return if epic? && group.nil? - - super - end - - private - - attr_reader :group, :project - - def where_clauses - [ - where_clause_base, - where_clause_for_title, - where_clause_for_klass - ].compact - end - - # Returns Arel clause `"{table_name}"."project_id" = {project.id}` if project is present - # For example: merge_request has :target_project_id, and we are searching by :iid - # or, if group is present: - # `"{table_name}"."project_id" = {project.id} OR "{table_name}"."group_id" = {group.id}` - def where_clause_base - [].tap do |clauses| - clauses << table[:project_id].eq(project.id) if project - clauses << table[:group_id].in(group.self_and_ancestors_ids) if group - end.reduce(:or) - end - - # Returns Arel clause for a particular model or `nil`. - def where_clause_for_klass - attrs_to_arel(attributes.slice('iid')) if merge_request? - end - - def prepare_attributes - attributes.dup.tap do |atts| - atts.delete('group') unless epic? - - if label? - atts['type'] = 'ProjectLabel' # Always create project labels - elsif milestone? - if atts['group_id'] # Transform new group milestones into project ones - atts['iid'] = nil - atts.delete('group_id') - else - claim_iid - end - end - - atts['importing'] = true if klass.ancestors.include?(Importable) - end - end - - def label? - klass == Label - end - - def milestone? - klass == Milestone - end - - def merge_request? - klass == MergeRequest - end - - def epic? - klass == Epic - end - - # If an existing group milestone used the IID - # claim the IID back and set the group milestone to use one available - # This is necessary to fix situations like the following: - # - Importing into a user namespace project with exported group milestones - # where the IID of the Group milestone could conflict with a project one. - def claim_iid - # The milestone has to be a group milestone, as it's the only case where - # we set the IID as the maximum. The rest of them are fixed. - milestone = project.milestones.find_by(iid: attributes['iid']) - - return unless milestone - - milestone.iid = nil - milestone.ensure_project_iid! - milestone.save! - end - end - end -end - -Gitlab::ImportExport::GroupProjectObjectBuilder.prepend_if_ee('EE::Gitlab::ImportExport::GroupProjectObjectBuilder') diff --git a/lib/gitlab/import_export/group_relation_factory.rb b/lib/gitlab/import_export/group_relation_factory.rb deleted file mode 100644 index e3597af44d2..00000000000 --- a/lib/gitlab/import_export/group_relation_factory.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class GroupRelationFactory < BaseRelationFactory - OVERRIDES = { - labels: :group_labels, - priorities: :label_priorities, - label: :group_label, - parent: :epic - }.freeze - - EXISTING_OBJECT_RELATIONS = %i[ - epic - epics - milestone - milestones - label - labels - group_label - group_labels - ].freeze - - private - - def setup_models - setup_note if @relation_name == :notes - - update_group_references - end - - def update_group_references - return unless self.class.existing_object_relations.include?(@relation_name) - return unless @relation_hash['group_id'] - - @relation_hash['group_id'] = @importable.id - end - end - end -end diff --git a/lib/gitlab/import_export/group_tree_restorer.rb b/lib/gitlab/import_export/group_tree_restorer.rb deleted file mode 100644 index 2f42843ed6c..00000000000 --- a/lib/gitlab/import_export/group_tree_restorer.rb +++ /dev/null @@ -1,116 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class GroupTreeRestorer - attr_reader :user - attr_reader :shared - attr_reader :group - - def initialize(user:, shared:, group:, group_hash:) - @path = File.join(shared.export_path, 'group.json') - @user = user - @shared = shared - @group = group - @group_hash = group_hash - end - - def restore - @tree_hash = @group_hash || read_tree_hash - @group_members = @tree_hash.delete('members') - @children = @tree_hash.delete('children') - - if members_mapper.map && restorer.restore - @children&.each do |group_hash| - group = create_group(group_hash: group_hash, parent_group: @group) - shared = Gitlab::ImportExport::Shared.new(group) - - self.class.new( - user: @user, - shared: shared, - group: group, - group_hash: group_hash - ).restore - end - end - - return false if @shared.errors.any? - - true - rescue => e - @shared.error(e) - false - end - - private - - def read_tree_hash - json = IO.read(@path) - ActiveSupport::JSON.decode(json) - rescue => e - @shared.logger.error( - group_id: @group.id, - group_name: @group.name, - message: "Import/Export error: #{e.message}" - ) - - raise Gitlab::ImportExport::Error.new('Incorrect JSON format') - end - - def restorer - @relation_tree_restorer ||= RelationTreeRestorer.new( - user: @user, - shared: @shared, - importable: @group, - tree_hash: @tree_hash.except('name', 'path'), - members_mapper: members_mapper, - object_builder: object_builder, - relation_factory: relation_factory, - reader: reader - ) - end - - def create_group(group_hash:, parent_group:) - group_params = { - name: group_hash['name'], - path: group_hash['path'], - parent_id: parent_group&.id, - visibility_level: sub_group_visibility_level(group_hash, parent_group) - } - - ::Groups::CreateService.new(@user, group_params).execute - end - - def sub_group_visibility_level(group_hash, parent_group) - original_visibility_level = group_hash['visibility_level'] || Gitlab::VisibilityLevel::PRIVATE - - if parent_group && parent_group.visibility_level < original_visibility_level - Gitlab::VisibilityLevel.closest_allowed_level(parent_group.visibility_level) - else - original_visibility_level - end - end - - def members_mapper - @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @group_members, user: @user, importable: @group) - end - - def relation_factory - Gitlab::ImportExport::GroupRelationFactory - end - - def object_builder - Gitlab::ImportExport::GroupObjectBuilder - end - - def reader - @reader ||= Gitlab::ImportExport::Reader.new( - shared: @shared, - config: Gitlab::ImportExport::Config.new( - config: Gitlab::ImportExport.group_config_file - ).to_h - ) - end - end - end -end diff --git a/lib/gitlab/import_export/group_tree_saver.rb b/lib/gitlab/import_export/group_tree_saver.rb deleted file mode 100644 index 2effcd01e30..00000000000 --- a/lib/gitlab/import_export/group_tree_saver.rb +++ /dev/null @@ -1,55 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class GroupTreeSaver - attr_reader :full_path, :shared - - def initialize(group:, current_user:, shared:, params: {}) - @params = params - @current_user = current_user - @shared = shared - @group = group - @full_path = File.join(@shared.export_path, ImportExport.group_filename) - end - - def save - group_tree = serialize(@group, reader.group_tree) - tree_saver.save(group_tree, @shared.export_path, ImportExport.group_filename) - - true - rescue => e - @shared.error(e) - false - end - - private - - def serialize(group, relations_tree) - group_tree = tree_saver.serialize(group, relations_tree) - - group.children.each do |child| - group_tree['children'] ||= [] - group_tree['children'] << serialize(child, relations_tree) - end - - group_tree - rescue => e - @shared.error(e) - end - - def reader - @reader ||= Gitlab::ImportExport::Reader.new( - shared: @shared, - config: Gitlab::ImportExport::Config.new( - config: Gitlab::ImportExport.group_config_file - ).to_h - ) - end - - def tree_saver - @tree_saver ||= RelationTreeSaver.new - end - end - end -end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml deleted file mode 100644 index 4fa909ac94b..00000000000 --- a/lib/gitlab/import_export/import_export.yml +++ /dev/null @@ -1,381 +0,0 @@ -# Model relationships to be included in the project import/export -# -# This list _must_ only contain relationships that are available to both CE and -# EE. EE specific relationships must be defined in the `ee` section further -# down below. -tree: - project: - - labels: - - :priorities - - milestones: - - events: - - :push_event_payload - - issues: - - events: - - :push_event_payload - - :timelogs - - notes: - - :author - - :award_emoji - - events: - - :push_event_payload - - label_links: - - label: - - :priorities - - milestone: - - events: - - :push_event_payload - - resource_label_events: - - label: - - :priorities - - :issue_assignees - - :zoom_meetings - - :sentry_issue - - :award_emoji - - snippets: - - :award_emoji - - notes: - - :author - - :award_emoji - - releases: - - :links - - project_members: - - :user - - merge_requests: - - :metrics - - :award_emoji - - notes: - - :author - - :award_emoji - - events: - - :push_event_payload - - :suggestions - - merge_request_diff: - - :merge_request_diff_commits - - :merge_request_diff_files - - events: - - :push_event_payload - - :timelogs - - label_links: - - label: - - :priorities - - milestone: - - events: - - :push_event_payload - - resource_label_events: - - label: - - :priorities - - ci_pipelines: - - notes: - - :author - - events: - - :push_event_payload - - stages: - - :statuses - - :external_pull_request - - :merge_request - - :external_pull_requests - - :auto_devops - - :triggers - - :pipeline_schedules - - :container_expiration_policy - - :services - - protected_branches: - - :merge_access_levels - - :push_access_levels - - protected_tags: - - :create_access_levels - - :project_feature - - :custom_attributes - - :prometheus_metrics - - :project_badges - - :ci_cd_settings - - :error_tracking_setting - - :metrics_setting - - boards: - - lists: - - label: - - :priorities - group_members: - - :user - -# Only include the following attributes for the models specified. -included_attributes: - user: - - :id - - :email - - :username - author: - - :name - ci_cd_settings: - - :group_runners_enabled - -# Do not include the following attributes for the models specified. -excluded_attributes: - project: - - :name - - :path - - :namespace_id - - :creator_id - - :pool_repository_id - - :import_url - - :import_status - - :avatar - - :import_type - - :import_source - - :mirror - - :runners_token - - :runners_token_encrypted - - :repository_storage - - :repository_read_only - - :lfs_enabled - - :created_at - - :updated_at - - :id - - :star_count - - :last_activity_at - - :last_repository_updated_at - - :last_repository_check_at - - :storage_version - - :remote_mirror_available_overridden - - :description_html - - :repository_languages - - :bfg_object_map - - :detected_repository_languages - - :tag_list - - :mirror_user_id - - :mirror_trigger_builds - - :only_mirror_protected_branches - - :pull_mirror_available_overridden - - :pull_mirror_branch_prefix - - :mirror_overwrites_diverged_branches - - :packages_enabled - - :mirror_last_update_at - - :mirror_last_successful_update_at - - :emails_disabled - - :max_pages_size - - :max_artifacts_size - - :marked_for_deletion_at - - :marked_for_deletion_by_user_id - namespaces: - - :runners_token - - :runners_token_encrypted - project_import_state: - - :last_error - - :jid - - :last_update_at - - :last_successful_update_at - prometheus_metrics: - - :common - - :identifier - snippets: - - :expired_at - - :secret - - :encrypted_secret_token - - :encrypted_secret_token_iv - merge_request_diff: - - :external_diff - - :stored_externally - - :external_diff_store - - :merge_request_id - merge_request_diff_commits: - - :merge_request_diff_id - merge_request_diff_files: - - :diff - - :external_diff_offset - - :external_diff_size - - :merge_request_diff_id - issues: - - :milestone_id - - :moved_to_id - - :state_id - - :duplicated_to_id - - :promoted_to_epic_id - merge_request: - - :milestone_id - - :ref_fetched - - :merge_jid - - :rebase_jid - - :latest_merge_request_diff_id - - :head_pipeline_id - - :state_id - merge_requests: - - :milestone_id - - :ref_fetched - - :merge_jid - - :rebase_jid - - :latest_merge_request_diff_id - - :head_pipeline_id - - :state_id - award_emoji: - - :awardable_id - statuses: - - :trace - - :token - - :token_encrypted - - :when - - :artifacts_file - - :artifacts_metadata - - :artifacts_file_store - - :artifacts_metadata_store - - :artifacts_size - - :commands - - :runner_id - - :trigger_request_id - - :erased_by_id - - :auto_canceled_by_id - - :stage_id - - :upstream_pipeline_id - - :resource_group_id - - :waiting_for_resource_at - - :processed - sentry_issue: - - :issue_id - push_event_payload: - - :event_id - project_badges: - - :group_id - resource_label_events: - - :reference - - :reference_html - - :epic_id - - :issue_id - - :merge_request_id - - :label_id - runners: - - :token - - :token_encrypted - services: - - :template - error_tracking_setting: - - :encrypted_token - - :encrypted_token_iv - - :enabled - service_desk_setting: - - :outgoing_name - priorities: - - :label_id - events: - - :target_id - timelogs: - - :issue_id - - :merge_request_id - notes: - - :noteable_id - - :review_id - label_links: - - :label_id - - :target_id - issue_assignees: - - :issue_id - zoom_meetings: - - :issue_id - design: - - :issue_id - designs: - - :issue_id - design_versions: - - :issue_id - actions: - - :design_id - - :version_id - links: - - :release_id - project_members: - - :source_id - metrics: - - :merge_request_id - - :pipeline_id - suggestions: - - :note_id - ci_pipelines: - - :auto_canceled_by_id - - :pipeline_schedule_id - - :merge_request_id - - :external_pull_request_id - stages: - - :pipeline_id - merge_access_levels: - - :protected_branch_id - push_access_levels: - - :protected_branch_id - unprotect_access_levels: - - :protected_branch_id - create_access_levels: - - :protected_tag_id - deploy_access_levels: - - :protected_environment_id - boards: - - :milestone_id - lists: - - :board_id - - :label_id - - :milestone_id - epic: - - :start_date_sourcing_milestone_id - - :due_date_sourcing_milestone_id - - :parent_id - - :state_id - - :start_date_sourcing_epic_id - - :due_date_sourcing_epic_id -methods: - notes: - - :type - labels: - - :type - label: - - :type - statuses: - - :type - services: - - :type - merge_request_diff_files: - - :utf8_diff - merge_requests: - - :diff_head_sha - - :source_branch_sha - - :target_branch_sha - events: - - :action - push_event_payload: - - :action - project_badges: - - :type - lists: - - :list_type - ci_pipelines: - - :notes - -preloads: - statuses: - # TODO: We cannot preload tags, as they are not part of `GenericCommitStatus` - # tags: # needed by tag_list - project: # deprecated: needed by coverage_regex of Ci::Build - merge_requests: - source_project: # needed by source_branch_sha and diff_head_sha - target_project: # needed by target_branch_sha - assignees: # needed by assigne_id that is implemented by DeprecatedAssignee - -# EE specific relationships and settings to include. All of this will be merged -# into the previous structures if EE is used. -ee: - tree: - project: - - issues: - - designs: - - notes: - - :author - - events: - - :push_event_payload - - design_versions: - - actions: - - :design # Duplicate export of issues.designs in order to link the record to both Issue and Action - - :epic - - protected_branches: - - :unprotect_access_levels - - protected_environments: - - :deploy_access_levels - - :service_desk_setting - excluded_attributes: - actions: - - image_v432x230 diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index a6463ed678c..4eeecc14067 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -49,7 +49,7 @@ module Gitlab end def project_tree - @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: current_user, + @project_tree ||= Gitlab::ImportExport::Project::TreeRestorer.new(user: current_user, shared: shared, project: project) end @@ -125,7 +125,7 @@ module Gitlab def project_to_overwrite strong_memoize(:project_to_overwrite) do - Project.find_by_full_path("#{project.namespace.full_path}/#{original_path}") + ::Project.find_by_full_path("#{project.namespace.full_path}/#{original_path}") end end end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index e7eae0a8c31..fd76252eb36 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -91,9 +91,9 @@ module Gitlab def relation_class case @importable - when Project + when ::Project ProjectMember - when Group + when ::Group GroupMember end end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml new file mode 100644 index 00000000000..4fa909ac94b --- /dev/null +++ b/lib/gitlab/import_export/project/import_export.yml @@ -0,0 +1,381 @@ +# Model relationships to be included in the project import/export +# +# This list _must_ only contain relationships that are available to both CE and +# EE. EE specific relationships must be defined in the `ee` section further +# down below. +tree: + project: + - labels: + - :priorities + - milestones: + - events: + - :push_event_payload + - issues: + - events: + - :push_event_payload + - :timelogs + - notes: + - :author + - :award_emoji + - events: + - :push_event_payload + - label_links: + - label: + - :priorities + - milestone: + - events: + - :push_event_payload + - resource_label_events: + - label: + - :priorities + - :issue_assignees + - :zoom_meetings + - :sentry_issue + - :award_emoji + - snippets: + - :award_emoji + - notes: + - :author + - :award_emoji + - releases: + - :links + - project_members: + - :user + - merge_requests: + - :metrics + - :award_emoji + - notes: + - :author + - :award_emoji + - events: + - :push_event_payload + - :suggestions + - merge_request_diff: + - :merge_request_diff_commits + - :merge_request_diff_files + - events: + - :push_event_payload + - :timelogs + - label_links: + - label: + - :priorities + - milestone: + - events: + - :push_event_payload + - resource_label_events: + - label: + - :priorities + - ci_pipelines: + - notes: + - :author + - events: + - :push_event_payload + - stages: + - :statuses + - :external_pull_request + - :merge_request + - :external_pull_requests + - :auto_devops + - :triggers + - :pipeline_schedules + - :container_expiration_policy + - :services + - protected_branches: + - :merge_access_levels + - :push_access_levels + - protected_tags: + - :create_access_levels + - :project_feature + - :custom_attributes + - :prometheus_metrics + - :project_badges + - :ci_cd_settings + - :error_tracking_setting + - :metrics_setting + - boards: + - lists: + - label: + - :priorities + group_members: + - :user + +# Only include the following attributes for the models specified. +included_attributes: + user: + - :id + - :email + - :username + author: + - :name + ci_cd_settings: + - :group_runners_enabled + +# Do not include the following attributes for the models specified. +excluded_attributes: + project: + - :name + - :path + - :namespace_id + - :creator_id + - :pool_repository_id + - :import_url + - :import_status + - :avatar + - :import_type + - :import_source + - :mirror + - :runners_token + - :runners_token_encrypted + - :repository_storage + - :repository_read_only + - :lfs_enabled + - :created_at + - :updated_at + - :id + - :star_count + - :last_activity_at + - :last_repository_updated_at + - :last_repository_check_at + - :storage_version + - :remote_mirror_available_overridden + - :description_html + - :repository_languages + - :bfg_object_map + - :detected_repository_languages + - :tag_list + - :mirror_user_id + - :mirror_trigger_builds + - :only_mirror_protected_branches + - :pull_mirror_available_overridden + - :pull_mirror_branch_prefix + - :mirror_overwrites_diverged_branches + - :packages_enabled + - :mirror_last_update_at + - :mirror_last_successful_update_at + - :emails_disabled + - :max_pages_size + - :max_artifacts_size + - :marked_for_deletion_at + - :marked_for_deletion_by_user_id + namespaces: + - :runners_token + - :runners_token_encrypted + project_import_state: + - :last_error + - :jid + - :last_update_at + - :last_successful_update_at + prometheus_metrics: + - :common + - :identifier + snippets: + - :expired_at + - :secret + - :encrypted_secret_token + - :encrypted_secret_token_iv + merge_request_diff: + - :external_diff + - :stored_externally + - :external_diff_store + - :merge_request_id + merge_request_diff_commits: + - :merge_request_diff_id + merge_request_diff_files: + - :diff + - :external_diff_offset + - :external_diff_size + - :merge_request_diff_id + issues: + - :milestone_id + - :moved_to_id + - :state_id + - :duplicated_to_id + - :promoted_to_epic_id + merge_request: + - :milestone_id + - :ref_fetched + - :merge_jid + - :rebase_jid + - :latest_merge_request_diff_id + - :head_pipeline_id + - :state_id + merge_requests: + - :milestone_id + - :ref_fetched + - :merge_jid + - :rebase_jid + - :latest_merge_request_diff_id + - :head_pipeline_id + - :state_id + award_emoji: + - :awardable_id + statuses: + - :trace + - :token + - :token_encrypted + - :when + - :artifacts_file + - :artifacts_metadata + - :artifacts_file_store + - :artifacts_metadata_store + - :artifacts_size + - :commands + - :runner_id + - :trigger_request_id + - :erased_by_id + - :auto_canceled_by_id + - :stage_id + - :upstream_pipeline_id + - :resource_group_id + - :waiting_for_resource_at + - :processed + sentry_issue: + - :issue_id + push_event_payload: + - :event_id + project_badges: + - :group_id + resource_label_events: + - :reference + - :reference_html + - :epic_id + - :issue_id + - :merge_request_id + - :label_id + runners: + - :token + - :token_encrypted + services: + - :template + error_tracking_setting: + - :encrypted_token + - :encrypted_token_iv + - :enabled + service_desk_setting: + - :outgoing_name + priorities: + - :label_id + events: + - :target_id + timelogs: + - :issue_id + - :merge_request_id + notes: + - :noteable_id + - :review_id + label_links: + - :label_id + - :target_id + issue_assignees: + - :issue_id + zoom_meetings: + - :issue_id + design: + - :issue_id + designs: + - :issue_id + design_versions: + - :issue_id + actions: + - :design_id + - :version_id + links: + - :release_id + project_members: + - :source_id + metrics: + - :merge_request_id + - :pipeline_id + suggestions: + - :note_id + ci_pipelines: + - :auto_canceled_by_id + - :pipeline_schedule_id + - :merge_request_id + - :external_pull_request_id + stages: + - :pipeline_id + merge_access_levels: + - :protected_branch_id + push_access_levels: + - :protected_branch_id + unprotect_access_levels: + - :protected_branch_id + create_access_levels: + - :protected_tag_id + deploy_access_levels: + - :protected_environment_id + boards: + - :milestone_id + lists: + - :board_id + - :label_id + - :milestone_id + epic: + - :start_date_sourcing_milestone_id + - :due_date_sourcing_milestone_id + - :parent_id + - :state_id + - :start_date_sourcing_epic_id + - :due_date_sourcing_epic_id +methods: + notes: + - :type + labels: + - :type + label: + - :type + statuses: + - :type + services: + - :type + merge_request_diff_files: + - :utf8_diff + merge_requests: + - :diff_head_sha + - :source_branch_sha + - :target_branch_sha + events: + - :action + push_event_payload: + - :action + project_badges: + - :type + lists: + - :list_type + ci_pipelines: + - :notes + +preloads: + statuses: + # TODO: We cannot preload tags, as they are not part of `GenericCommitStatus` + # tags: # needed by tag_list + project: # deprecated: needed by coverage_regex of Ci::Build + merge_requests: + source_project: # needed by source_branch_sha and diff_head_sha + target_project: # needed by target_branch_sha + assignees: # needed by assigne_id that is implemented by DeprecatedAssignee + +# EE specific relationships and settings to include. All of this will be merged +# into the previous structures if EE is used. +ee: + tree: + project: + - issues: + - designs: + - notes: + - :author + - events: + - :push_event_payload + - design_versions: + - actions: + - :design # Duplicate export of issues.designs in order to link the record to both Issue and Action + - :epic + - protected_branches: + - :unprotect_access_levels + - protected_environments: + - :deploy_access_levels + - :service_desk_setting + excluded_attributes: + actions: + - image_v432x230 diff --git a/lib/gitlab/import_export/project/object_builder.rb b/lib/gitlab/import_export/project/object_builder.rb new file mode 100644 index 00000000000..c3637b1c115 --- /dev/null +++ b/lib/gitlab/import_export/project/object_builder.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + # Given a class, it finds or creates a new object + # (initializes in the case of Label) at group or project level. + # If it does not exist in the group, it creates it at project level. + # + # Example: + # `ObjectBuilder.build(Label, label_attributes)` + # finds or initializes a label with the given attributes. + # + # It also adds some logic around Group Labels/Milestones for edge cases. + class ObjectBuilder < Base::ObjectBuilder + def self.build(*args) + ::Project.transaction do + super + end + end + + def initialize(klass, attributes) + super + + @group = @attributes['group'] + @project = @attributes['project'] + end + + def find + return if epic? && group.nil? + + super + end + + private + + attr_reader :group, :project + + def where_clauses + [ + where_clause_base, + where_clause_for_title, + where_clause_for_klass + ].compact + end + + # Returns Arel clause `"{table_name}"."project_id" = {project.id}` if project is present + # For example: merge_request has :target_project_id, and we are searching by :iid + # or, if group is present: + # `"{table_name}"."project_id" = {project.id} OR "{table_name}"."group_id" = {group.id}` + def where_clause_base + [].tap do |clauses| + clauses << table[:project_id].eq(project.id) if project + clauses << table[:group_id].in(group.self_and_ancestors_ids) if group + end.reduce(:or) + end + + # Returns Arel clause for a particular model or `nil`. + def where_clause_for_klass + attrs_to_arel(attributes.slice('iid')) if merge_request? + end + + def prepare_attributes + attributes.dup.tap do |atts| + atts.delete('group') unless epic? + + if label? + atts['type'] = 'ProjectLabel' # Always create project labels + elsif milestone? + if atts['group_id'] # Transform new group milestones into project ones + atts['iid'] = nil + atts.delete('group_id') + else + claim_iid + end + end + + atts['importing'] = true if klass.ancestors.include?(Importable) + end + end + + def label? + klass == Label + end + + def milestone? + klass == Milestone + end + + def merge_request? + klass == MergeRequest + end + + def epic? + klass == Epic + end + + # If an existing group milestone used the IID + # claim the IID back and set the group milestone to use one available + # This is necessary to fix situations like the following: + # - Importing into a user namespace project with exported group milestones + # where the IID of the Group milestone could conflict with a project one. + def claim_iid + # The milestone has to be a group milestone, as it's the only case where + # we set the IID as the maximum. The rest of them are fixed. + milestone = project.milestones.find_by(iid: attributes['iid']) + + return unless milestone + + milestone.iid = nil + milestone.ensure_project_iid! + milestone.save! + end + end + end + end +end + +Gitlab::ImportExport::Project::ObjectBuilder.prepend_if_ee('EE::Gitlab::ImportExport::Project::ObjectBuilder') diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb new file mode 100644 index 00000000000..951482a933a --- /dev/null +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -0,0 +1,162 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + class RelationFactory < Base::RelationFactory + prepend_if_ee('::EE::Gitlab::ImportExport::Project::RelationFactory') # rubocop: disable Cop/InjectEnterpriseEditionModule + + OVERRIDES = { snippets: :project_snippets, + ci_pipelines: 'Ci::Pipeline', + pipelines: 'Ci::Pipeline', + stages: 'Ci::Stage', + statuses: 'commit_status', + triggers: 'Ci::Trigger', + pipeline_schedules: 'Ci::PipelineSchedule', + builds: 'Ci::Build', + runners: 'Ci::Runner', + hooks: 'ProjectHook', + merge_access_levels: 'ProtectedBranch::MergeAccessLevel', + push_access_levels: 'ProtectedBranch::PushAccessLevel', + create_access_levels: 'ProtectedTag::CreateAccessLevel', + labels: :project_labels, + priorities: :label_priorities, + auto_devops: :project_auto_devops, + label: :project_label, + custom_attributes: 'ProjectCustomAttribute', + project_badges: 'Badge', + metrics: 'MergeRequest::Metrics', + ci_cd_settings: 'ProjectCiCdSetting', + error_tracking_setting: 'ErrorTracking::ProjectErrorTrackingSetting', + links: 'Releases::Link', + metrics_setting: 'ProjectMetricsSetting' }.freeze + + BUILD_MODELS = %i[Ci::Build commit_status].freeze + + GROUP_REFERENCES = %w[group_id].freeze + + PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze + + EXISTING_OBJECT_RELATIONS = %i[ + milestone + milestones + label + labels + project_label + project_labels + group_label + group_labels + project_feature + merge_request + epic + ProjectCiCdSetting + container_expiration_policy + ].freeze + + def create + @object = super + + # We preload the project, user, and group to re-use objects + @object = preload_keys(@object, PROJECT_REFERENCES, @importable) + @object = preload_keys(@object, GROUP_REFERENCES, @importable.group) + @object = preload_keys(@object, USER_REFERENCES, @user) + end + + private + + def invalid_relation? + # Do not create relation if it is: + # - An unknown service + # - A legacy trigger + unknown_service? || + (!Feature.enabled?(:use_legacy_pipeline_triggers, @importable) && legacy_trigger?) + end + + def setup_models + case @relation_name + when :merge_request_diff_files then setup_diff + when :notes then setup_note + when :'Ci::Pipeline' then setup_pipeline + when *BUILD_MODELS then setup_build + end + + update_project_references + update_group_references + end + + def generate_imported_object + if @relation_name == :merge_requests + MergeRequestParser.new(@importable, @relation_hash.delete('diff_head_sha'), super, @relation_hash).parse! + else + super + end + end + + def update_project_references + # If source and target are the same, populate them with the new project ID. + if @relation_hash['source_project_id'] + @relation_hash['source_project_id'] = same_source_and_target? ? @relation_hash['project_id'] : MergeRequestParser::FORKED_PROJECT_ID + end + + @relation_hash['target_project_id'] = @relation_hash['project_id'] if @relation_hash['target_project_id'] + end + + def same_source_and_target? + @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id'] + end + + def update_group_references + return unless existing_object? + return unless @relation_hash['group_id'] + + @relation_hash['group_id'] = @importable.namespace_id + end + + def setup_build + @relation_hash.delete('trace') # old export files have trace + @relation_hash.delete('token') + @relation_hash.delete('commands') + @relation_hash.delete('artifacts_file_store') + @relation_hash.delete('artifacts_metadata_store') + @relation_hash.delete('artifacts_size') + end + + def setup_diff + @relation_hash['diff'] = @relation_hash.delete('utf8_diff') + end + + def setup_pipeline + @relation_hash.fetch('stages', []).each do |stage| + stage.statuses.each do |status| + status.pipeline = imported_object + end + end + end + + def unknown_service? + @relation_name == :services && parsed_relation_hash['type'] && + !Object.const_defined?(parsed_relation_hash['type']) + end + + def legacy_trigger? + @relation_name == :'Ci::Trigger' && @relation_hash['owner_id'].nil? + end + + def preload_keys(object, references, value) + return object unless value + + references.each do |key| + attribute = "#{key.delete_suffix('_id')}=".to_sym + next unless object.respond_to?(key) && object.respond_to?(attribute) + + if object.read_attribute(key) == value&.id + object.public_send(attribute, value) # rubocop:disable GitlabSecurity/PublicSend + end + end + + object + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/tree_loader.rb b/lib/gitlab/import_export/project/tree_loader.rb new file mode 100644 index 00000000000..6d4737a2d00 --- /dev/null +++ b/lib/gitlab/import_export/project/tree_loader.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + class TreeLoader + def load(path, dedup_entries: false) + tree_hash = ActiveSupport::JSON.decode(IO.read(path)) + + if dedup_entries + dedup_tree(tree_hash) + else + tree_hash + end + end + + private + + # This function removes duplicate entries from the given tree recursively + # by caching nodes it encounters repeatedly. We only consider nodes for + # which there can actually be multiple equivalent instances (e.g. strings, + # hashes and arrays, but not `nil`s, numbers or booleans.) + # + # The algorithm uses a recursive depth-first descent with 3 cases, starting + # with a root node (the tree/hash itself): + # - a node has already been cached; in this case we return it from the cache + # - a node has not been cached yet but should be; descend into its children + # - a node is neither cached nor qualifies for caching; this is a no-op + def dedup_tree(node, nodes_seen = {}) + if nodes_seen.key?(node) && distinguishable?(node) + yield nodes_seen[node] + elsif should_dedup?(node) + nodes_seen[node] = node + + case node + when Array + node.each_index do |idx| + dedup_tree(node[idx], nodes_seen) do |cached_node| + node[idx] = cached_node + end + end + when Hash + node.each do |k, v| + dedup_tree(v, nodes_seen) do |cached_node| + node[k] = cached_node + end + end + end + else + node + end + end + + # We do not need to consider nodes for which there cannot be multiple instances + def should_dedup?(node) + node && !(node.is_a?(Numeric) || node.is_a?(TrueClass) || node.is_a?(FalseClass)) + end + + # We can only safely de-dup values that are distinguishable. True value objects + # are always distinguishable by nature. Hashes however can represent entities, + # which are identified by ID, not value. We therefore disallow de-duping hashes + # that do not have an `id` field, since we might risk dropping entities that + # have equal attributes yet different identities. + def distinguishable?(node) + if node.is_a?(Hash) + node.key?('id') + else + true + end + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb new file mode 100644 index 00000000000..a5123f16dbc --- /dev/null +++ b/lib/gitlab/import_export/project/tree_restorer.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + class TreeRestorer + LARGE_PROJECT_FILE_SIZE_BYTES = 500.megabyte + + attr_reader :user + attr_reader :shared + attr_reader :project + + def initialize(user:, shared:, project:) + @user = user + @shared = shared + @project = project + @tree_loader = TreeLoader.new + end + + def restore + @tree_hash = read_tree_hash + @project_members = @tree_hash.delete('project_members') + + RelationRenameService.rename(@tree_hash) + + if relation_tree_restorer.restore + import_failure_service.with_retry(action: 'set_latest_merge_request_diff_ids!') do + @project.merge_requests.set_latest_merge_request_diff_ids! + end + + true + else + false + end + rescue => e + @shared.error(e) + false + end + + private + + def large_project?(path) + File.size(path) >= LARGE_PROJECT_FILE_SIZE_BYTES + end + + def read_tree_hash + path = File.join(@shared.export_path, 'project.json') + dedup_entries = large_project?(path) && + Feature.enabled?(:dedup_project_import_metadata, project.group) + + @tree_loader.load(path, dedup_entries: dedup_entries) + rescue => e + Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger + raise Gitlab::ImportExport::Error.new('Incorrect JSON format') + end + + def relation_tree_restorer + @relation_tree_restorer ||= RelationTreeRestorer.new( + user: @user, + shared: @shared, + importable: @project, + tree_hash: @tree_hash, + object_builder: object_builder, + members_mapper: members_mapper, + relation_factory: relation_factory, + reader: reader + ) + end + + def members_mapper + @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members, + user: @user, + importable: @project) + end + + def object_builder + Project::ObjectBuilder + end + + def relation_factory + Project::RelationFactory + end + + def reader + @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) + end + + def import_failure_service + @import_failure_service ||= ImportFailureService.new(@project) + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb new file mode 100644 index 00000000000..58f33a04851 --- /dev/null +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + class TreeSaver + attr_reader :full_path + + def initialize(project:, current_user:, shared:, params: {}) + @params = params + @project = project + @current_user = current_user + @shared = shared + @full_path = File.join(@shared.export_path, ImportExport.project_filename) + end + + def save + project_tree = tree_saver.serialize(@project, reader.project_tree) + fix_project_tree(project_tree) + tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename) + + true + rescue => e + @shared.error(e) + false + end + + private + + # Aware that the resulting hash needs to be pure-hash and + # does not include any AR objects anymore, only objects that run `.to_json` + def fix_project_tree(project_tree) + if @params[:description].present? + project_tree['description'] = @params[:description] + end + + project_tree['project_members'] += group_members_array + + RelationRenameService.add_new_associations(project_tree) + end + + def reader + @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) + end + + def group_members_array + group_members.as_json(reader.group_members_tree).each do |group_member| + group_member['source_type'] = 'Project' # Make group members project members of the future import + end + end + + def group_members + return [] unless @current_user.can?(:admin_group, @project.group) + + # We need `.where.not(user_id: nil)` here otherwise when a group has an + # invitee, it would make the following query return 0 rows since a NULL + # user_id would be present in the subquery + # See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values + non_null_user_ids = @project.project_members.where.not(user_id: nil).select(:user_id) + + GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids) + end + + def tree_saver + @tree_saver ||= RelationTreeSaver.new + end + end + end + end +end diff --git a/lib/gitlab/import_export/project_relation_factory.rb b/lib/gitlab/import_export/project_relation_factory.rb deleted file mode 100644 index 0e08a66b89c..00000000000 --- a/lib/gitlab/import_export/project_relation_factory.rb +++ /dev/null @@ -1,160 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class ProjectRelationFactory < BaseRelationFactory - prepend_if_ee('::EE::Gitlab::ImportExport::ProjectRelationFactory') # rubocop: disable Cop/InjectEnterpriseEditionModule - - OVERRIDES = { snippets: :project_snippets, - ci_pipelines: 'Ci::Pipeline', - pipelines: 'Ci::Pipeline', - stages: 'Ci::Stage', - statuses: 'commit_status', - triggers: 'Ci::Trigger', - pipeline_schedules: 'Ci::PipelineSchedule', - builds: 'Ci::Build', - runners: 'Ci::Runner', - hooks: 'ProjectHook', - merge_access_levels: 'ProtectedBranch::MergeAccessLevel', - push_access_levels: 'ProtectedBranch::PushAccessLevel', - create_access_levels: 'ProtectedTag::CreateAccessLevel', - labels: :project_labels, - priorities: :label_priorities, - auto_devops: :project_auto_devops, - label: :project_label, - custom_attributes: 'ProjectCustomAttribute', - project_badges: 'Badge', - metrics: 'MergeRequest::Metrics', - ci_cd_settings: 'ProjectCiCdSetting', - error_tracking_setting: 'ErrorTracking::ProjectErrorTrackingSetting', - links: 'Releases::Link', - metrics_setting: 'ProjectMetricsSetting' }.freeze - - BUILD_MODELS = %i[Ci::Build commit_status].freeze - - GROUP_REFERENCES = %w[group_id].freeze - - PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze - - EXISTING_OBJECT_RELATIONS = %i[ - milestone - milestones - label - labels - project_label - project_labels - group_label - group_labels - project_feature - merge_request - epic - ProjectCiCdSetting - container_expiration_policy - ].freeze - - def create - @object = super - - # We preload the project, user, and group to re-use objects - @object = preload_keys(@object, PROJECT_REFERENCES, @importable) - @object = preload_keys(@object, GROUP_REFERENCES, @importable.group) - @object = preload_keys(@object, USER_REFERENCES, @user) - end - - private - - def invalid_relation? - # Do not create relation if it is: - # - An unknown service - # - A legacy trigger - unknown_service? || - (!Feature.enabled?(:use_legacy_pipeline_triggers, @importable) && legacy_trigger?) - end - - def setup_models - case @relation_name - when :merge_request_diff_files then setup_diff - when :notes then setup_note - when :'Ci::Pipeline' then setup_pipeline - when *BUILD_MODELS then setup_build - end - - update_project_references - update_group_references - end - - def generate_imported_object - if @relation_name == :merge_requests - MergeRequestParser.new(@importable, @relation_hash.delete('diff_head_sha'), super, @relation_hash).parse! - else - super - end - end - - def update_project_references - # If source and target are the same, populate them with the new project ID. - if @relation_hash['source_project_id'] - @relation_hash['source_project_id'] = same_source_and_target? ? @relation_hash['project_id'] : MergeRequestParser::FORKED_PROJECT_ID - end - - @relation_hash['target_project_id'] = @relation_hash['project_id'] if @relation_hash['target_project_id'] - end - - def same_source_and_target? - @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id'] - end - - def update_group_references - return unless existing_object? - return unless @relation_hash['group_id'] - - @relation_hash['group_id'] = @importable.namespace_id - end - - def setup_build - @relation_hash.delete('trace') # old export files have trace - @relation_hash.delete('token') - @relation_hash.delete('commands') - @relation_hash.delete('artifacts_file_store') - @relation_hash.delete('artifacts_metadata_store') - @relation_hash.delete('artifacts_size') - end - - def setup_diff - @relation_hash['diff'] = @relation_hash.delete('utf8_diff') - end - - def setup_pipeline - @relation_hash.fetch('stages', []).each do |stage| - stage.statuses.each do |status| - status.pipeline = imported_object - end - end - end - - def unknown_service? - @relation_name == :services && parsed_relation_hash['type'] && - !Object.const_defined?(parsed_relation_hash['type']) - end - - def legacy_trigger? - @relation_name == :'Ci::Trigger' && @relation_hash['owner_id'].nil? - end - - def preload_keys(object, references, value) - return object unless value - - references.each do |key| - attribute = "#{key.delete_suffix('_id')}=".to_sym - next unless object.respond_to?(key) && object.respond_to?(attribute) - - if object.read_attribute(key) == value&.id - object.public_send(attribute, value) # rubocop:disable GitlabSecurity/PublicSend - end - end - - object - end - end - end -end diff --git a/lib/gitlab/import_export/project_tree_loader.rb b/lib/gitlab/import_export/project_tree_loader.rb deleted file mode 100644 index fc21858043d..00000000000 --- a/lib/gitlab/import_export/project_tree_loader.rb +++ /dev/null @@ -1,72 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class ProjectTreeLoader - def load(path, dedup_entries: false) - tree_hash = ActiveSupport::JSON.decode(IO.read(path)) - - if dedup_entries - dedup_tree(tree_hash) - else - tree_hash - end - end - - private - - # This function removes duplicate entries from the given tree recursively - # by caching nodes it encounters repeatedly. We only consider nodes for - # which there can actually be multiple equivalent instances (e.g. strings, - # hashes and arrays, but not `nil`s, numbers or booleans.) - # - # The algorithm uses a recursive depth-first descent with 3 cases, starting - # with a root node (the tree/hash itself): - # - a node has already been cached; in this case we return it from the cache - # - a node has not been cached yet but should be; descend into its children - # - a node is neither cached nor qualifies for caching; this is a no-op - def dedup_tree(node, nodes_seen = {}) - if nodes_seen.key?(node) && distinguishable?(node) - yield nodes_seen[node] - elsif should_dedup?(node) - nodes_seen[node] = node - - case node - when Array - node.each_index do |idx| - dedup_tree(node[idx], nodes_seen) do |cached_node| - node[idx] = cached_node - end - end - when Hash - node.each do |k, v| - dedup_tree(v, nodes_seen) do |cached_node| - node[k] = cached_node - end - end - end - else - node - end - end - - # We do not need to consider nodes for which there cannot be multiple instances - def should_dedup?(node) - node && !(node.is_a?(Numeric) || node.is_a?(TrueClass) || node.is_a?(FalseClass)) - end - - # We can only safely de-dup values that are distinguishable. True value objects - # are always distinguishable by nature. Hashes however can represent entities, - # which are identified by ID, not value. We therefore disallow de-duping hashes - # that do not have an `id` field, since we might risk dropping entities that - # have equal attributes yet different identities. - def distinguishable?(node) - if node.is_a?(Hash) - node.key?('id') - else - true - end - end - end - end -end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb deleted file mode 100644 index aae07657ea0..00000000000 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ /dev/null @@ -1,92 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class ProjectTreeRestorer - LARGE_PROJECT_FILE_SIZE_BYTES = 500.megabyte - - attr_reader :user - attr_reader :shared - attr_reader :project - - def initialize(user:, shared:, project:) - @user = user - @shared = shared - @project = project - @tree_loader = ProjectTreeLoader.new - end - - def restore - @tree_hash = read_tree_hash - @project_members = @tree_hash.delete('project_members') - - RelationRenameService.rename(@tree_hash) - - if relation_tree_restorer.restore - import_failure_service.with_retry(action: 'set_latest_merge_request_diff_ids!') do - @project.merge_requests.set_latest_merge_request_diff_ids! - end - - true - else - false - end - rescue => e - @shared.error(e) - false - end - - private - - def large_project?(path) - File.size(path) >= LARGE_PROJECT_FILE_SIZE_BYTES - end - - def read_tree_hash - path = File.join(@shared.export_path, 'project.json') - dedup_entries = large_project?(path) && - Feature.enabled?(:dedup_project_import_metadata, project.group) - - @tree_loader.load(path, dedup_entries: dedup_entries) - rescue => e - Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger - raise Gitlab::ImportExport::Error.new('Incorrect JSON format') - end - - def relation_tree_restorer - @relation_tree_restorer ||= RelationTreeRestorer.new( - user: @user, - shared: @shared, - importable: @project, - tree_hash: @tree_hash, - object_builder: object_builder, - members_mapper: members_mapper, - relation_factory: relation_factory, - reader: reader - ) - end - - def members_mapper - @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members, - user: @user, - importable: @project) - end - - def object_builder - Gitlab::ImportExport::GroupProjectObjectBuilder - end - - def relation_factory - Gitlab::ImportExport::ProjectRelationFactory - end - - def reader - @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) - end - - def import_failure_service - @import_failure_service ||= ImportFailureService.new(@project) - end - end - end -end diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb deleted file mode 100644 index 386a4cfdfc6..00000000000 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ /dev/null @@ -1,68 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class ProjectTreeSaver - attr_reader :full_path - - def initialize(project:, current_user:, shared:, params: {}) - @params = params - @project = project - @current_user = current_user - @shared = shared - @full_path = File.join(@shared.export_path, ImportExport.project_filename) - end - - def save - project_tree = tree_saver.serialize(@project, reader.project_tree) - fix_project_tree(project_tree) - tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename) - - true - rescue => e - @shared.error(e) - false - end - - private - - # Aware that the resulting hash needs to be pure-hash and - # does not include any AR objects anymore, only objects that run `.to_json` - def fix_project_tree(project_tree) - if @params[:description].present? - project_tree['description'] = @params[:description] - end - - project_tree['project_members'] += group_members_array - - RelationRenameService.add_new_associations(project_tree) - end - - def reader - @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) - end - - def group_members_array - group_members.as_json(reader.group_members_tree).each do |group_member| - group_member['source_type'] = 'Project' # Make group members project members of the future import - end - end - - def group_members - return [] unless @current_user.can?(:admin_group, @project.group) - - # We need `.where.not(user_id: nil)` here otherwise when a group has an - # invitee, it would make the following query return 0 rows since a NULL - # user_id would be present in the subquery - # See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values - non_null_user_ids = @project.project_members.where.not(user_id: nil).select(:user_id) - - GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids) - end - - def tree_saver - @tree_saver ||= RelationTreeSaver.new - end - end - end -end diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb index 9b84ade1525..1797bbad51a 100644 --- a/lib/gitlab/import_export/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/relation_tree_restorer.rb @@ -69,7 +69,7 @@ module Gitlab def process_relation_item!(relation_key, relation_definition, relation_index, data_hash) relation_object = build_relation(relation_key, relation_definition, data_hash) return unless relation_object - return if importable_class == Project && group_model?(relation_object) + return if importable_class == ::Project && group_model?(relation_object) relation_object.assign_attributes(importable_class_sym => @importable) @@ -110,7 +110,7 @@ module Gitlab excluded_keys: excluded_keys_for_relation(importable_class_sym)) @importable.assign_attributes(params) - @importable.drop_visibility_level! if importable_class == Project + @importable.drop_visibility_level! if importable_class == ::Project Gitlab::Timeless.timeless(@importable) do @importable.save! diff --git a/lib/gitlab_danger.rb b/lib/gitlab_danger.rb index e776e2b7ea3..cf57b20790b 100644 --- a/lib/gitlab_danger.rb +++ b/lib/gitlab_danger.rb @@ -3,7 +3,6 @@ class GitlabDanger LOCAL_RULES ||= %w[ changes_size - gemfile documentation frozen_string duplicate_yarn_dependencies diff --git a/scripts/gemfile_lock_changed.sh b/scripts/gemfile_lock_changed.sh new file mode 100755 index 00000000000..24e2c685f11 --- /dev/null +++ b/scripts/gemfile_lock_changed.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +gemfile_lock_changed() { + if [ -n "$(git diff --name-only -- Gemfile.lock)" ]; then + cat << EOF + Gemfile was updated but Gemfile.lock was not updated. + + Usually, when Gemfile is updated, you should run + \`\`\` + bundle install + \`\`\` + + or + + \`\`\` + bundle update + \`\`\` + + and commit the Gemfile.lock changes. +EOF + + exit 1 + fi +} + +gemfile_lock_changed diff --git a/scripts/static-analysis b/scripts/static-analysis index 1f55c035ed1..251462fad33 100755 --- a/scripts/static-analysis +++ b/scripts/static-analysis @@ -49,7 +49,8 @@ def jobs_to_run(node_index, node_total) %w[scripts/lint-conflicts.sh], %w[scripts/lint-rugged], %w[scripts/frontend/check_no_partial_karma_jest.sh], - %w[scripts/lint-changelog-filenames] + %w[scripts/lint-changelog-filenames], + %w[scripts/gemfile_lock_changed.sh] ] case node_total diff --git a/spec/features/admin/admin_health_check_spec.rb b/spec/features/admin/admin_health_check_spec.rb index 97c34d55d73..9ce96fe8020 100644 --- a/spec/features/admin/admin_health_check_spec.rb +++ b/spec/features/admin/admin_health_check_spec.rb @@ -4,7 +4,7 @@ require 'spec_helper' describe "Admin Health Check", :feature do include StubENV - set(:admin) { create(:admin) } + let_it_be(:admin) { create(:admin) } before do stub_env('IN_MEMORY_APPLICATION_SETTINGS', 'false') diff --git a/spec/features/boards/multiple_boards_spec.rb b/spec/features/boards/multiple_boards_spec.rb index 2389707be9c..8e56be6bdd0 100644 --- a/spec/features/boards/multiple_boards_spec.rb +++ b/spec/features/boards/multiple_boards_spec.rb @@ -3,11 +3,11 @@ require 'spec_helper' describe 'Multiple Issue Boards', :js do - set(:user) { create(:user) } - set(:project) { create(:project, :public) } - set(:planning) { create(:label, project: project, name: 'Planning') } - set(:board) { create(:board, name: 'board1', project: project) } - set(:board2) { create(:board, name: 'board2', project: project) } + let_it_be(:user) { create(:user) } + let_it_be(:project) { create(:project, :public) } + let_it_be(:planning) { create(:label, project: project, name: 'Planning') } + let_it_be(:board) { create(:board, name: 'board1', project: project) } + let_it_be(:board2) { create(:board, name: 'board2', project: project) } let(:parent) { project } let(:boards_path) { project_boards_path(project) } diff --git a/spec/features/boards/new_issue_spec.rb b/spec/features/boards/new_issue_spec.rb index 730887370dd..2d41b5d612d 100644 --- a/spec/features/boards/new_issue_spec.rb +++ b/spec/features/boards/new_issue_spec.rb @@ -129,10 +129,10 @@ describe 'Issue Boards new issue', :js do end context 'group boards' do - set(:group) { create(:group, :public) } - set(:project) { create(:project, namespace: group) } - set(:group_board) { create(:board, group: group) } - set(:list) { create(:list, board: group_board, position: 0) } + let_it_be(:group) { create(:group, :public) } + let_it_be(:project) { create(:project, namespace: group) } + let_it_be(:group_board) { create(:board, group: group) } + let_it_be(:list) { create(:list, board: group_board, position: 0) } context 'for unauthorized users' do before do diff --git a/spec/features/commits_spec.rb b/spec/features/commits_spec.rb index f538df89fd3..d8b886b239f 100644 --- a/spec/features/commits_spec.rb +++ b/spec/features/commits_spec.rb @@ -3,8 +3,8 @@ require 'spec_helper' describe 'Commits' do - let(:project) { create(:project, :repository) } - let(:user) { create(:user) } + let_it_be(:project) { create(:project, :repository) } + let_it_be(:user) { create(:user) } describe 'CI' do before do @@ -183,4 +183,41 @@ describe 'Commits' do expect(find('.js-project-refs-dropdown')).to have_content branch_name end end + + context 'viewing commits for an author' do + let(:author_commit) { project.repository.commits(nil, limit: 1).first } + let(:commits) { project.repository.commits(nil, author: author, limit: 40) } + + before do + project.add_maintainer(user) + sign_in(user) + visit project_commits_path(project, nil, author: author) + end + + shared_examples 'show commits by author' do + it "includes the author's commits" do + commits.each do |commit| + expect(page).to have_content("#{author_commit.author_name} authored #{commit.authored_date.strftime("%b %d, %Y")}") + end + end + end + + context 'author is complete' do + let(:author) { "#{author_commit.author_name} <#{author_commit.author_email}>" } + + it_behaves_like 'show commits by author' + end + + context 'author is just a name' do + let(:author) { "#{author_commit.author_name}" } + + it_behaves_like 'show commits by author' + end + + context 'author is just an email' do + let(:author) { "#{author_commit.author_email}" } + + it_behaves_like 'show commits by author' + end + end end diff --git a/spec/features/dashboard/root_explore_spec.rb b/spec/features/dashboard/root_explore_spec.rb index 5b686d8b6f1..0e065dbed67 100644 --- a/spec/features/dashboard/root_explore_spec.rb +++ b/spec/features/dashboard/root_explore_spec.rb @@ -3,17 +3,17 @@ require 'spec_helper' describe 'Root explore' do - set(:public_project) { create(:project, :public) } - set(:archived_project) { create(:project, :archived) } - set(:internal_project) { create(:project, :internal) } - set(:private_project) { create(:project, :private) } + let_it_be(:public_project) { create(:project, :public) } + let_it_be(:archived_project) { create(:project, :archived) } + let_it_be(:internal_project) { create(:project, :internal) } + let_it_be(:private_project) { create(:project, :private) } before do allow(Gitlab).to receive(:com?).and_return(true) end context 'when logged in' do - set(:user) { create(:user) } + let_it_be(:user) { create(:user) } before do sign_in(user) diff --git a/spec/features/explore/user_explores_projects_spec.rb b/spec/features/explore/user_explores_projects_spec.rb index 9c3686dba2d..c64709c0b55 100644 --- a/spec/features/explore/user_explores_projects_spec.rb +++ b/spec/features/explore/user_explores_projects_spec.rb @@ -3,10 +3,10 @@ require 'spec_helper' describe 'User explores projects' do - set(:archived_project) { create(:project, :archived) } - set(:internal_project) { create(:project, :internal) } - set(:private_project) { create(:project, :private) } - set(:public_project) { create(:project, :public) } + let_it_be(:archived_project) { create(:project, :archived) } + let_it_be(:internal_project) { create(:project, :internal) } + let_it_be(:private_project) { create(:project, :private) } + let_it_be(:public_project) { create(:project, :public) } context 'when not signed in' do context 'when viewing public projects' do @@ -19,7 +19,7 @@ describe 'User explores projects' do end context 'when signed in' do - set(:user) { create(:user) } + let_it_be(:user) { create(:user) } before do sign_in(user) diff --git a/spec/features/groups/labels/user_sees_links_to_issuables_spec.rb b/spec/features/groups/labels/user_sees_links_to_issuables_spec.rb index 6199b566ebc..38561c71323 100644 --- a/spec/features/groups/labels/user_sees_links_to_issuables_spec.rb +++ b/spec/features/groups/labels/user_sees_links_to_issuables_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' describe 'Groups > Labels > User sees links to issuables' do - set(:group) { create(:group, :public) } + let_it_be(:group) { create(:group, :public) } before do create(:group_label, group: group, title: 'bug') diff --git a/spec/features/issues/user_views_issues_spec.rb b/spec/features/issues/user_views_issues_spec.rb index 8f174472f49..796e618c7c8 100644 --- a/spec/features/issues/user_views_issues_spec.rb +++ b/spec/features/issues/user_views_issues_spec.rb @@ -7,7 +7,7 @@ describe "User views issues" do let!(:open_issue1) { create(:issue, project: project) } let!(:open_issue2) { create(:issue, project: project) } - set(:user) { create(:user) } + let_it_be(:user) { create(:user) } shared_examples "opens issue from list" do it "opens issue" do diff --git a/spec/features/merge_request/user_posts_notes_spec.rb b/spec/features/merge_request/user_posts_notes_spec.rb index f24e7090605..b22f5a6c211 100644 --- a/spec/features/merge_request/user_posts_notes_spec.rb +++ b/spec/features/merge_request/user_posts_notes_spec.rb @@ -5,8 +5,7 @@ require 'spec_helper' describe 'Merge request > User posts notes', :js do include NoteInteractionHelpers - set(:project) { create(:project, :repository) } - + let_it_be(:project) { create(:project, :repository) } let(:user) { project.creator } let(:merge_request) do create(:merge_request, source_project: project, target_project: project) diff --git a/spec/features/merge_requests/user_sorts_merge_requests_spec.rb b/spec/features/merge_requests/user_sorts_merge_requests_spec.rb index 3c217786d43..5a84bcb0c44 100644 --- a/spec/features/merge_requests/user_sorts_merge_requests_spec.rb +++ b/spec/features/merge_requests/user_sorts_merge_requests_spec.rb @@ -10,10 +10,10 @@ describe 'User sorts merge requests' do create(:merge_request_with_diffs, source_project: project, target_project: project, source_branch: 'merge-test') end - set(:user) { create(:user) } - set(:group) { create(:group) } - set(:group_member) { create(:group_member, :maintainer, user: user, group: group) } - set(:project) { create(:project, :public, group: group) } + let_it_be(:user) { create(:user) } + let_it_be(:group) { create(:group) } + let_it_be(:group_member) { create(:group_member, :maintainer, user: user, group: group) } + let_it_be(:project) { create(:project, :public, group: group) } before do sign_in(user) diff --git a/spec/features/merge_requests/user_views_open_merge_requests_spec.rb b/spec/features/merge_requests/user_views_open_merge_requests_spec.rb index 932090bdbce..4aaa20f0455 100644 --- a/spec/features/merge_requests/user_views_open_merge_requests_spec.rb +++ b/spec/features/merge_requests/user_views_open_merge_requests_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' describe 'User views open merge requests' do - set(:user) { create(:user) } + let_it_be(:user) { create(:user) } shared_examples_for 'shows merge requests' do it 'shows merge requests' do @@ -12,7 +12,7 @@ describe 'User views open merge requests' do end context 'when project is public' do - set(:project) { create(:project, :public, :repository) } + let_it_be(:project) { create(:project, :public, :repository) } context 'when not signed in' do context "when the target branch is the project's default branch" do @@ -114,7 +114,7 @@ describe 'User views open merge requests' do context 'when project is internal' do let!(:merge_request) { create(:merge_request, source_project: project, target_project: project) } - set(:project) { create(:project, :internal, :repository) } + let_it_be(:project) { create(:project, :internal, :repository) } context 'when signed in' do before do diff --git a/spec/features/milestones/user_creates_milestone_spec.rb b/spec/features/milestones/user_creates_milestone_spec.rb index 5c93ddcf6f8..368a2ddecdf 100644 --- a/spec/features/milestones/user_creates_milestone_spec.rb +++ b/spec/features/milestones/user_creates_milestone_spec.rb @@ -3,8 +3,8 @@ require 'spec_helper' describe "User creates milestone", :js do - set(:user) { create(:user) } - set(:project) { create(:project) } + let_it_be(:user) { create(:user) } + let_it_be(:project) { create(:project) } before do project.add_developer(user) diff --git a/spec/features/milestones/user_edits_milestone_spec.rb b/spec/features/milestones/user_edits_milestone_spec.rb index b41b8f3282f..be05685aff7 100644 --- a/spec/features/milestones/user_edits_milestone_spec.rb +++ b/spec/features/milestones/user_edits_milestone_spec.rb @@ -3,9 +3,9 @@ require 'spec_helper' describe "User edits milestone", :js do - set(:user) { create(:user) } - set(:project) { create(:project) } - set(:milestone) { create(:milestone, project: project, start_date: Date.today, due_date: 5.days.from_now) } + let_it_be(:user) { create(:user) } + let_it_be(:project) { create(:project) } + let_it_be(:milestone) { create(:milestone, project: project, start_date: Date.today, due_date: 5.days.from_now) } before do project.add_developer(user) diff --git a/spec/features/milestones/user_promotes_milestone_spec.rb b/spec/features/milestones/user_promotes_milestone_spec.rb index 7678b6cbfa5..d14097e1ef4 100644 --- a/spec/features/milestones/user_promotes_milestone_spec.rb +++ b/spec/features/milestones/user_promotes_milestone_spec.rb @@ -3,10 +3,10 @@ require 'spec_helper' describe 'User promotes milestone' do - set(:group) { create(:group) } - set(:user) { create(:user) } - set(:project) { create(:project, namespace: group) } - set(:milestone) { create(:milestone, project: project) } + let_it_be(:group) { create(:group) } + let_it_be(:user) { create(:user) } + let_it_be(:project) { create(:project, namespace: group) } + let_it_be(:milestone) { create(:milestone, project: project) } context 'when user can admin group milestones' do before do diff --git a/spec/features/milestones/user_views_milestone_spec.rb b/spec/features/milestones/user_views_milestone_spec.rb index 71abb195ad1..cbc21dd02e5 100644 --- a/spec/features/milestones/user_views_milestone_spec.rb +++ b/spec/features/milestones/user_views_milestone_spec.rb @@ -3,10 +3,10 @@ require 'spec_helper' describe "User views milestone" do - set(:user) { create(:user) } - set(:project) { create(:project) } - set(:milestone) { create(:milestone, project: project) } - set(:labels) { create_list(:label, 2, project: project) } + let_it_be(:user) { create(:user) } + let_it_be(:project) { create(:project) } + let_it_be(:milestone) { create(:milestone, project: project) } + let_it_be(:labels) { create_list(:label, 2, project: project) } before do project.add_developer(user) diff --git a/spec/features/milestones/user_views_milestones_spec.rb b/spec/features/milestones/user_views_milestones_spec.rb index c91fe95aa77..e17797a8165 100644 --- a/spec/features/milestones/user_views_milestones_spec.rb +++ b/spec/features/milestones/user_views_milestones_spec.rb @@ -3,9 +3,9 @@ require 'spec_helper' describe "User views milestones" do - set(:user) { create(:user) } - set(:project) { create(:project) } - set(:milestone) { create(:milestone, project: project) } + let_it_be(:user) { create(:user) } + let_it_be(:project) { create(:project) } + let_it_be(:milestone) { create(:milestone, project: project) } before do project.add_developer(user) @@ -22,8 +22,8 @@ describe "User views milestones" do end context "with issues" do - set(:issue) { create(:issue, project: project, milestone: milestone) } - set(:closed_issue) { create(:closed_issue, project: project, milestone: milestone) } + let_it_be(:issue) { create(:issue, project: project, milestone: milestone) } + let_it_be(:closed_issue) { create(:closed_issue, project: project, milestone: milestone) } it "opens milestone" do click_link(milestone.title) @@ -38,7 +38,7 @@ describe "User views milestones" do end context "with associated releases" do - set(:first_release) { create(:release, project: project, name: "The first release", milestones: [milestone], released_at: Time.zone.parse('2019-10-07')) } + let_it_be(:first_release) { create(:release, project: project, name: "The first release", milestones: [milestone], released_at: Time.zone.parse('2019-10-07')) } context "with a single associated release" do it "shows the associated release" do @@ -48,10 +48,10 @@ describe "User views milestones" do end context "with lots of associated releases" do - set(:second_release) { create(:release, project: project, name: "The second release", milestones: [milestone], released_at: first_release.released_at + 1.day) } - set(:third_release) { create(:release, project: project, name: "The third release", milestones: [milestone], released_at: second_release.released_at + 1.day) } - set(:fourth_release) { create(:release, project: project, name: "The fourth release", milestones: [milestone], released_at: third_release.released_at + 1.day) } - set(:fifth_release) { create(:release, project: project, name: "The fifth release", milestones: [milestone], released_at: fourth_release.released_at + 1.day) } + let_it_be(:second_release) { create(:release, project: project, name: "The second release", milestones: [milestone], released_at: first_release.released_at + 1.day) } + let_it_be(:third_release) { create(:release, project: project, name: "The third release", milestones: [milestone], released_at: second_release.released_at + 1.day) } + let_it_be(:fourth_release) { create(:release, project: project, name: "The fourth release", milestones: [milestone], released_at: third_release.released_at + 1.day) } + let_it_be(:fifth_release) { create(:release, project: project, name: "The fifth release", milestones: [milestone], released_at: fourth_release.released_at + 1.day) } it "shows the associated releases and the truncation text" do expect(page).to have_content("Releases #{fifth_release.name} • #{fourth_release.name} • #{third_release.name} • 2 more releases") @@ -66,9 +66,9 @@ describe "User views milestones" do end describe "User views milestones with no MR" do - set(:user) { create(:user) } - set(:project) { create(:project, :merge_requests_disabled) } - set(:milestone) { create(:milestone, project: project) } + let_it_be(:user) { create(:user) } + let_it_be(:project) { create(:project, :merge_requests_disabled) } + let_it_be(:milestone) { create(:milestone, project: project) } before do project.add_developer(user) diff --git a/spec/features/projects/artifacts/user_downloads_artifacts_spec.rb b/spec/features/projects/artifacts/user_downloads_artifacts_spec.rb index fb70076fcf1..3cbf276c02d 100644 --- a/spec/features/projects/artifacts/user_downloads_artifacts_spec.rb +++ b/spec/features/projects/artifacts/user_downloads_artifacts_spec.rb @@ -3,9 +3,9 @@ require "spec_helper" describe "User downloads artifacts" do - set(:project) { create(:project, :repository, :public) } - set(:pipeline) { create(:ci_empty_pipeline, status: :success, sha: project.commit.id, project: project) } - set(:job) { create(:ci_build, :artifacts, :success, pipeline: pipeline) } + let_it_be(:project) { create(:project, :repository, :public) } + let_it_be(:pipeline) { create(:ci_empty_pipeline, status: :success, sha: project.commit.id, project: project) } + let_it_be(:job) { create(:ci_build, :artifacts, :success, pipeline: pipeline) } shared_examples "downloading" do it "downloads the zip" do diff --git a/spec/features/projects/badges/pipeline_badge_spec.rb b/spec/features/projects/badges/pipeline_badge_spec.rb index 5ddaf1e1591..b2f09a9d0b7 100644 --- a/spec/features/projects/badges/pipeline_badge_spec.rb +++ b/spec/features/projects/badges/pipeline_badge_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' describe 'Pipeline Badge' do - set(:project) { create(:project, :repository, :public) } + let_it_be(:project) { create(:project, :repository, :public) } let(:ref) { project.default_branch } context 'when the project has a pipeline' do diff --git a/spec/features/projects/branches/user_deletes_branch_spec.rb b/spec/features/projects/branches/user_deletes_branch_spec.rb index ad63a75a149..184954c1c78 100644 --- a/spec/features/projects/branches/user_deletes_branch_spec.rb +++ b/spec/features/projects/branches/user_deletes_branch_spec.rb @@ -3,7 +3,7 @@ require "spec_helper" describe "User deletes branch", :js do - set(:user) { create(:user) } + let_it_be(:user) { create(:user) } let(:project) { create(:project, :repository) } before do diff --git a/spec/features/projects/branches/user_views_branches_spec.rb b/spec/features/projects/branches/user_views_branches_spec.rb index f3810611094..e127e784b94 100644 --- a/spec/features/projects/branches/user_views_branches_spec.rb +++ b/spec/features/projects/branches/user_views_branches_spec.rb @@ -3,8 +3,8 @@ require "spec_helper" describe "User views branches" do - set(:project) { create(:project, :repository) } - set(:user) { project.owner } + let_it_be(:project) { create(:project, :repository) } + let_it_be(:user) { project.owner } before do sign_in(user) @@ -23,7 +23,7 @@ describe "User views branches" do end context "protected branches" do - set(:protected_branch) { create(:protected_branch, project: project) } + let_it_be(:protected_branch) { create(:protected_branch, project: project) } before do visit(project_protected_branches_path(project)) diff --git a/spec/features/projects/commit/user_views_user_status_on_commit_spec.rb b/spec/features/projects/commit/user_views_user_status_on_commit_spec.rb index e78b7f7ae08..c07f6081d2c 100644 --- a/spec/features/projects/commit/user_views_user_status_on_commit_spec.rb +++ b/spec/features/projects/commit/user_views_user_status_on_commit_spec.rb @@ -5,8 +5,8 @@ require 'spec_helper' describe 'Project > Commit > View user status' do include RepoHelpers - set(:project) { create(:project, :repository) } - set(:user) { create(:user) } + let_it_be(:project) { create(:project, :repository) } + let_it_be(:user) { create(:user) } let(:commit_author) { create(:user, email: sample_commit.author_email) } before do diff --git a/spec/features/projects/labels/user_creates_labels_spec.rb b/spec/features/projects/labels/user_creates_labels_spec.rb index 257e064ae3d..180cd8eff14 100644 --- a/spec/features/projects/labels/user_creates_labels_spec.rb +++ b/spec/features/projects/labels/user_creates_labels_spec.rb @@ -3,8 +3,8 @@ require "spec_helper" describe "User creates labels" do - set(:project) { create(:project_empty_repo, :public) } - set(:user) { create(:user) } + let_it_be(:project) { create(:project_empty_repo, :public) } + let_it_be(:user) { create(:user) } shared_examples_for "label creation" do it "creates new label" do @@ -66,7 +66,7 @@ describe "User creates labels" do end context "in another project" do - set(:another_project) { create(:project_empty_repo, :public) } + let_it_be(:another_project) { create(:project_empty_repo, :public) } before do create(:label, project: project, title: "bug") # Create label for `project` (not `another_project`) project. diff --git a/spec/features/projects/labels/user_edits_labels_spec.rb b/spec/features/projects/labels/user_edits_labels_spec.rb index da33ae3af3a..add959ccda6 100644 --- a/spec/features/projects/labels/user_edits_labels_spec.rb +++ b/spec/features/projects/labels/user_edits_labels_spec.rb @@ -3,9 +3,9 @@ require "spec_helper" describe "User edits labels" do - set(:project) { create(:project_empty_repo, :public) } - set(:label) { create(:label, project: project) } - set(:user) { create(:user) } + let_it_be(:project) { create(:project_empty_repo, :public) } + let_it_be(:label) { create(:label, project: project) } + let_it_be(:user) { create(:user) } before do project.add_maintainer(user) diff --git a/spec/features/projects/labels/user_promotes_label_spec.rb b/spec/features/projects/labels/user_promotes_label_spec.rb index fdecafd4c50..cf7320d3cf9 100644 --- a/spec/features/projects/labels/user_promotes_label_spec.rb +++ b/spec/features/projects/labels/user_promotes_label_spec.rb @@ -3,10 +3,10 @@ require 'spec_helper' describe 'User promotes label' do - set(:group) { create(:group) } - set(:user) { create(:user) } - set(:project) { create(:project, namespace: group) } - set(:label) { create(:label, project: project) } + let_it_be(:group) { create(:group) } + let_it_be(:user) { create(:user) } + let_it_be(:project) { create(:project, namespace: group) } + let_it_be(:label) { create(:label, project: project) } context 'when user can admin group labels' do before do diff --git a/spec/features/projects/labels/user_sees_links_to_issuables_spec.rb b/spec/features/projects/labels/user_sees_links_to_issuables_spec.rb index 7a9b9e6eac2..f60e7e9703f 100644 --- a/spec/features/projects/labels/user_sees_links_to_issuables_spec.rb +++ b/spec/features/projects/labels/user_sees_links_to_issuables_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' describe 'Projects > Labels > User sees links to issuables' do - set(:user) { create(:user) } + let_it_be(:user) { create(:user) } before do label # creates the label @@ -50,7 +50,7 @@ describe 'Projects > Labels > User sees links to issuables' do end context 'with a group label' do - set(:group) { create(:group) } + let_it_be(:group) { create(:group) } let(:label) { create(:group_label, group: group, title: 'bug') } context 'when merge requests and issues are enabled for the project' do diff --git a/spec/features/projects/labels/user_views_labels_spec.rb b/spec/features/projects/labels/user_views_labels_spec.rb index a6f7968c535..7f70ac903d6 100644 --- a/spec/features/projects/labels/user_views_labels_spec.rb +++ b/spec/features/projects/labels/user_views_labels_spec.rb @@ -3,9 +3,8 @@ require "spec_helper" describe "User views labels" do - set(:project) { create(:project_empty_repo, :public) } - set(:user) { create(:user) } - + let_it_be(:project) { create(:project_empty_repo, :public) } + let_it_be(:user) { create(:user) } let(:label_titles) { %w[bug enhancement feature] } let!(:prioritized_label) { create(:label, project: project, title: 'prioritized-label-name', priority: 1) } diff --git a/spec/features/projects/settings/project_settings_spec.rb b/spec/features/projects/settings/project_settings_spec.rb index b601866c96b..9fc91550667 100644 --- a/spec/features/projects/settings/project_settings_spec.rb +++ b/spec/features/projects/settings/project_settings_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' describe 'Projects settings' do - set(:project) { create(:project) } + let_it_be(:project) { create(:project) } let(:user) { project.owner } let(:panel) { find('.general-settings', match: :first) } let(:button) { panel.find('.btn.js-settings-toggle') } diff --git a/spec/features/projects/show/user_sees_git_instructions_spec.rb b/spec/features/projects/show/user_sees_git_instructions_spec.rb index dde9490a5e1..0c486056329 100644 --- a/spec/features/projects/show/user_sees_git_instructions_spec.rb +++ b/spec/features/projects/show/user_sees_git_instructions_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' describe 'Projects > Show > User sees Git instructions' do - set(:user) { create(:user) } + let_it_be(:user) { create(:user) } shared_examples_for 'redirects to the sign in page' do it 'redirects to the sign in page' do @@ -49,7 +49,7 @@ describe 'Projects > Show > User sees Git instructions' do context 'when project is public' do context 'when project has no repo' do - set(:project) { create(:project, :public) } + let_it_be(:project) { create(:project, :public) } before do sign_in(project.owner) @@ -60,7 +60,7 @@ describe 'Projects > Show > User sees Git instructions' do end context 'when project is empty' do - set(:project) { create(:project_empty_repo, :public) } + let_it_be(:project) { create(:project_empty_repo, :public) } context 'when not signed in' do before do @@ -98,7 +98,7 @@ describe 'Projects > Show > User sees Git instructions' do end context 'when project is not empty' do - set(:project) { create(:project, :public, :repository) } + let_it_be(:project) { create(:project, :public, :repository) } before do visit(project_path(project)) @@ -141,7 +141,7 @@ describe 'Projects > Show > User sees Git instructions' do end context 'when project is internal' do - set(:project) { create(:project, :internal, :repository) } + let_it_be(:project) { create(:project, :internal, :repository) } context 'when not signed in' do before do @@ -163,7 +163,7 @@ describe 'Projects > Show > User sees Git instructions' do end context 'when project is private' do - set(:project) { create(:project, :private) } + let_it_be(:project) { create(:project, :private) } before do visit(project_path(project)) diff --git a/spec/features/projects/show/user_sees_last_commit_ci_status_spec.rb b/spec/features/projects/show/user_sees_last_commit_ci_status_spec.rb index cf1a679102c..5aba16597b8 100644 --- a/spec/features/projects/show/user_sees_last_commit_ci_status_spec.rb +++ b/spec/features/projects/show/user_sees_last_commit_ci_status_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' describe 'Projects > Show > User sees last commit CI status' do - set(:project) { create(:project, :repository, :public) } + let_it_be(:project) { create(:project, :repository, :public) } it 'shows the project README', :js do project.enable_ci diff --git a/spec/features/projects/show/user_sees_readme_spec.rb b/spec/features/projects/show/user_sees_readme_spec.rb index 98906de4620..52745b06cd3 100644 --- a/spec/features/projects/show/user_sees_readme_spec.rb +++ b/spec/features/projects/show/user_sees_readme_spec.rb @@ -3,9 +3,8 @@ require 'spec_helper' describe 'Projects > Show > User sees README' do - set(:user) { create(:user) } - - set(:project) { create(:project, :repository, :public) } + let_it_be(:user) { create(:user) } + let_it_be(:project) { create(:project, :repository, :public) } it 'shows the project README', :js do visit project_path(project) diff --git a/spec/features/projects/user_sees_user_popover_spec.rb b/spec/features/projects/user_sees_user_popover_spec.rb index adbf9073d59..fafb3773866 100644 --- a/spec/features/projects/user_sees_user_popover_spec.rb +++ b/spec/features/projects/user_sees_user_popover_spec.rb @@ -3,8 +3,7 @@ require 'spec_helper' describe 'User sees user popover', :js do - set(:project) { create(:project, :repository) } - + let_it_be(:project) { create(:project, :repository) } let(:user) { project.creator } let(:merge_request) do create(:merge_request, source_project: project, target_project: project) diff --git a/spec/features/projects/wiki/markdown_preview_spec.rb b/spec/features/projects/wiki/markdown_preview_spec.rb index 331ba58d067..7d18c0f7a14 100644 --- a/spec/features/projects/wiki/markdown_preview_spec.rb +++ b/spec/features/projects/wiki/markdown_preview_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' describe 'Projects > Wiki > User previews markdown changes', :js do - set(:user) { create(:user) } + let_it_be(:user) { create(:user) } let(:project) { create(:project, :wiki_repo, namespace: user.namespace) } let(:wiki_page) { create(:wiki_page, wiki: project.wiki, attrs: { title: 'home', content: '[some link](other-page)' }) } let(:wiki_content) do diff --git a/spec/features/projects/wiki/user_creates_wiki_page_spec.rb b/spec/features/projects/wiki/user_creates_wiki_page_spec.rb index 7503c8aa52e..e67982bbd31 100644 --- a/spec/features/projects/wiki/user_creates_wiki_page_spec.rb +++ b/spec/features/projects/wiki/user_creates_wiki_page_spec.rb @@ -3,6 +3,8 @@ require "spec_helper" describe "User creates wiki page" do + include WikiHelpers + let(:user) { create(:user) } let(:wiki) { ProjectWiki.new(project, user) } let(:project) { create(:project) } @@ -14,9 +16,11 @@ describe "User creates wiki page" do end context "when wiki is empty" do - before do + before do |example| visit(project_wikis_path(project)) + wait_for_svg_to_be_loaded(example) + click_link "Create your first page" end @@ -45,7 +49,7 @@ describe "User creates wiki page" do expect(page).to have_content("Create New Page") end - it "shows non-escaped link in the pages list", :quarantine do + it "shows non-escaped link in the pages list" do fill_in(:wiki_title, with: "one/two/three-test") page.within(".wiki-form") do @@ -163,7 +167,7 @@ describe "User creates wiki page" do expect(page).to have_link('Link to Home', href: "/#{project.full_path}/-/wikis/home") end - it_behaves_like 'wiki file attachments', :quarantine + it_behaves_like 'wiki file attachments' end context "in a group namespace", :js do @@ -175,7 +179,7 @@ describe "User creates wiki page" do expect(page).to have_field("wiki[message]", with: "Create home") end - it "creates a page from the home page", :quarantine do + it "creates a page from the home page" do page.within(".wiki-form") do fill_in(:wiki_content, with: "My awesome wiki!") diff --git a/spec/features/projects/wiki/user_views_wiki_page_spec.rb b/spec/features/projects/wiki/user_views_wiki_page_spec.rb index c7856342fb2..1a9cde4571e 100644 --- a/spec/features/projects/wiki/user_views_wiki_page_spec.rb +++ b/spec/features/projects/wiki/user_views_wiki_page_spec.rb @@ -19,9 +19,12 @@ describe 'User views a wiki page' do sign_in(user) end - context 'when wiki is empty' do + context 'when wiki is empty', :js do before do - visit(project_wikis_path(project)) + visit project_wikis_path(project) + + wait_for_svg_to_be_loaded + click_link "Create your first page" fill_in(:wiki_title, with: 'one/two/three-test') @@ -32,7 +35,7 @@ describe 'User views a wiki page' do end end - it 'shows the history of a page that has a path', :js do + it 'shows the history of a page that has a path' do expect(current_path).to include('one/two/three-test') first(:link, text: 'three').click @@ -45,7 +48,7 @@ describe 'User views a wiki page' do end end - it 'shows an old version of a page', :js do + it 'shows an old version of a page' do expect(current_path).to include('one/two/three-test') expect(find('.wiki-pages')).to have_content('three') @@ -162,9 +165,12 @@ describe 'User views a wiki page' do end it 'opens a default wiki page', :js do - visit(project_path(project)) + visit project_path(project) find('.shortcuts-wiki').click + + wait_for_svg_to_be_loaded + click_link "Create your first page" expect(page).to have_content('Create New Page') diff --git a/spec/features/read_only_spec.rb b/spec/features/read_only_spec.rb index 619d34ebed4..a33535a7b0b 100644 --- a/spec/features/read_only_spec.rb +++ b/spec/features/read_only_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' describe 'read-only message' do - set(:user) { create(:user) } + let_it_be(:user) { create(:user) } before do sign_in(user) diff --git a/spec/features/security/project/internal_access_spec.rb b/spec/features/security/project/internal_access_spec.rb index ed1dbe15d65..45b57b5cb1b 100644 --- a/spec/features/security/project/internal_access_spec.rb +++ b/spec/features/security/project/internal_access_spec.rb @@ -5,7 +5,7 @@ require 'spec_helper' describe "Internal Project Access" do include AccessMatchers - set(:project) { create(:project, :internal, :repository) } + let_it_be(:project, reload: true) { create(:project, :internal, :repository) } describe "Project should be internal" do describe '#internal?' do diff --git a/spec/features/security/project/private_access_spec.rb b/spec/features/security/project/private_access_spec.rb index 97e6b3bd4ff..9aeb3ffbd43 100644 --- a/spec/features/security/project/private_access_spec.rb +++ b/spec/features/security/project/private_access_spec.rb @@ -5,7 +5,7 @@ require 'spec_helper' describe "Private Project Access" do include AccessMatchers - set(:project) { create(:project, :private, :repository, public_builds: false) } + let_it_be(:project, reload: true) { create(:project, :private, :repository, public_builds: false) } describe "Project should be private" do describe '#private?' do diff --git a/spec/features/security/project/public_access_spec.rb b/spec/features/security/project/public_access_spec.rb index 24bbb8d9b9e..4d8c2c7822c 100644 --- a/spec/features/security/project/public_access_spec.rb +++ b/spec/features/security/project/public_access_spec.rb @@ -5,7 +5,7 @@ require 'spec_helper' describe "Public Project Access" do include AccessMatchers - set(:project) { create(:project, :public, :repository) } + let_it_be(:project, reload: true) { create(:project, :public, :repository) } describe "Project should be public" do describe '#public?' do diff --git a/spec/features/user_sorts_things_spec.rb b/spec/features/user_sorts_things_spec.rb index 41f8f3761e8..8397854df27 100644 --- a/spec/features/user_sorts_things_spec.rb +++ b/spec/features/user_sorts_things_spec.rb @@ -10,10 +10,10 @@ describe "User sorts things" do include Spec::Support::Helpers::Features::SortingHelpers include DashboardHelper - set(:project) { create(:project_empty_repo, :public) } - set(:current_user) { create(:user) } # Using `current_user` instead of just `user` because of the hardoced call in `assigned_mrs_dashboard_path` which is used below. - set(:issue) { create(:issue, project: project, author: current_user) } - set(:merge_request) { create(:merge_request, target_project: project, source_project: project, author: current_user) } + let_it_be(:project) { create(:project_empty_repo, :public) } + let_it_be(:current_user) { create(:user) } # Using `current_user` instead of just `user` because of the hardoced call in `assigned_mrs_dashboard_path` which is used below. + let_it_be(:issue) { create(:issue, project: project, author: current_user) } + let_it_be(:merge_request) { create(:merge_request, target_project: project, source_project: project, author: current_user) } before do project.add_developer(current_user) diff --git a/spec/fixtures/trace/sample_trace b/spec/fixtures/trace/sample_trace index d774d154496..f768c038f9e 100644 --- a/spec/fixtures/trace/sample_trace +++ b/spec/fixtures/trace/sample_trace @@ -1442,7 +1442,7 @@ TodoService marks a single todo id as done caches the number of todos of a user -Gitlab::ImportExport::ProjectTreeSaver +Gitlab::ImportExport::Project::TreeSaver saves the project tree into a json object saves project successfully JSON diff --git a/spec/lib/gitlab/checks/branch_check_spec.rb b/spec/lib/gitlab/checks/branch_check_spec.rb index 7cc1722dfd4..fd7eaa1603f 100644 --- a/spec/lib/gitlab/checks/branch_check_spec.rb +++ b/spec/lib/gitlab/checks/branch_check_spec.rb @@ -15,7 +15,7 @@ describe Gitlab::Checks::BranchCheck do let(:ref) { 'refs/heads/master' } it 'raises an error' do - expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'The default branch of a project cannot be deleted.') + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::ForbiddenError, 'The default branch of a project cannot be deleted.') end end @@ -28,7 +28,7 @@ describe Gitlab::Checks::BranchCheck do it 'raises an error if the user is not allowed to do forced pushes to protected branches' do expect(Gitlab::Checks::ForcePush).to receive(:force_push?).and_return(true) - expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'You are not allowed to force push code to a protected branch on this project.') + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::ForbiddenError, 'You are not allowed to force push code to a protected branch on this project.') end it 'raises an error if the user is not allowed to merge to protected branches' do @@ -38,13 +38,13 @@ describe Gitlab::Checks::BranchCheck do expect(user_access).to receive(:can_merge_to_branch?).and_return(false) expect(user_access).to receive(:can_push_to_branch?).and_return(false) - expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'You are not allowed to merge code into protected branches on this project.') + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::ForbiddenError, 'You are not allowed to merge code into protected branches on this project.') end it 'raises an error if the user is not allowed to push to protected branches' do expect(user_access).to receive(:can_push_to_branch?).and_return(false) - expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'You are not allowed to push code to protected branches on this project.') + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::ForbiddenError, 'You are not allowed to push code to protected branches on this project.') end context 'when project repository is empty' do @@ -58,7 +58,7 @@ describe Gitlab::Checks::BranchCheck do end it 'raises an error' do - expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, /Ask a project Owner or Maintainer to create a default branch/) + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::ForbiddenError, /Ask a project Owner or Maintainer to create a default branch/) end end @@ -109,7 +109,7 @@ describe Gitlab::Checks::BranchCheck do end it 'raises an error' do - expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'You are not allowed to create protected branches on this project.') + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::ForbiddenError, 'You are not allowed to create protected branches on this project.') end end @@ -135,7 +135,7 @@ describe Gitlab::Checks::BranchCheck do end it 'raises an error' do - expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'You can only use an existing protected branch ref as the basis of a new protected branch.') + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::ForbiddenError, 'You can only use an existing protected branch ref as the basis of a new protected branch.') end end @@ -157,7 +157,7 @@ describe Gitlab::Checks::BranchCheck do context 'via SSH' do it 'raises an error' do - expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'You can only create protected branches using the web interface and API.') + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::ForbiddenError, 'You can only create protected branches using the web interface and API.') end end end @@ -171,7 +171,7 @@ describe Gitlab::Checks::BranchCheck do context 'if the user is not allowed to delete protected branches' do it 'raises an error' do - expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'You are not allowed to delete protected branches from this project. Only a project maintainer or owner can delete a protected branch.') + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::ForbiddenError, 'You are not allowed to delete protected branches from this project. Only a project maintainer or owner can delete a protected branch.') end end @@ -190,7 +190,7 @@ describe Gitlab::Checks::BranchCheck do context 'over SSH or HTTP' do it 'raises an error' do - expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'You can only delete protected branches using the web interface.') + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::ForbiddenError, 'You can only delete protected branches using the web interface.') end end end diff --git a/spec/lib/gitlab/checks/diff_check_spec.rb b/spec/lib/gitlab/checks/diff_check_spec.rb index b9134b8d6ab..467b4ed3a21 100644 --- a/spec/lib/gitlab/checks/diff_check_spec.rb +++ b/spec/lib/gitlab/checks/diff_check_spec.rb @@ -34,7 +34,7 @@ describe Gitlab::Checks::DiffCheck do context 'when change is sent by a different user' do it 'raises an error if the user is not allowed to update the file' do - expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, "The path 'README' is locked in Git LFS by #{lock.user.name}") + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::ForbiddenError, "The path 'README' is locked in Git LFS by #{lock.user.name}") end end diff --git a/spec/lib/gitlab/checks/lfs_check_spec.rb b/spec/lib/gitlab/checks/lfs_check_spec.rb index dad14e100a7..c86481d1abe 100644 --- a/spec/lib/gitlab/checks/lfs_check_spec.rb +++ b/spec/lib/gitlab/checks/lfs_check_spec.rb @@ -50,7 +50,7 @@ describe Gitlab::Checks::LfsCheck do end it 'fails if any LFS blobs are missing' do - expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, /LFS objects are missing/) + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::ForbiddenError, /LFS objects are missing/) end it 'succeeds if LFS objects have already been uploaded' do diff --git a/spec/lib/gitlab/checks/push_check_spec.rb b/spec/lib/gitlab/checks/push_check_spec.rb index e1bd52d6c0b..857d71732fe 100644 --- a/spec/lib/gitlab/checks/push_check_spec.rb +++ b/spec/lib/gitlab/checks/push_check_spec.rb @@ -15,7 +15,7 @@ describe Gitlab::Checks::PushCheck do expect(user_access).to receive(:can_do_action?).with(:push_code).and_return(false) expect(project).to receive(:branch_allows_collaboration?).with(user_access.user, 'master').and_return(false) - expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'You are not allowed to push code to this project.') + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::ForbiddenError, 'You are not allowed to push code to this project.') end end end diff --git a/spec/lib/gitlab/checks/tag_check_spec.rb b/spec/lib/gitlab/checks/tag_check_spec.rb index 80e9eb504ad..0c94171646e 100644 --- a/spec/lib/gitlab/checks/tag_check_spec.rb +++ b/spec/lib/gitlab/checks/tag_check_spec.rb @@ -11,7 +11,7 @@ describe Gitlab::Checks::TagCheck do it 'raises an error when user does not have access' do allow(user_access).to receive(:can_do_action?).with(:admin_tag).and_return(false) - expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'You are not allowed to change existing tags on this project.') + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::ForbiddenError, 'You are not allowed to change existing tags on this project.') end context 'with protected tag' do @@ -27,7 +27,7 @@ describe Gitlab::Checks::TagCheck do let(:newrev) { '0000000000000000000000000000000000000000' } it 'is prevented' do - expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, /cannot be deleted/) + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::ForbiddenError, /cannot be deleted/) end end @@ -36,7 +36,7 @@ describe Gitlab::Checks::TagCheck do let(:newrev) { '54fcc214b94e78d7a41a9a8fe6d87a5e59500e51' } it 'is prevented' do - expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, /cannot be updated/) + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::ForbiddenError, /cannot be updated/) end end end @@ -47,7 +47,7 @@ describe Gitlab::Checks::TagCheck do let(:ref) { 'refs/tags/v9.1.0' } it 'prevents creation below access level' do - expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, /allowed to create this tag as it is protected/) + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::ForbiddenError, /allowed to create this tag as it is protected/) end context 'when user has access' do diff --git a/spec/lib/gitlab/git_access_spec.rb b/spec/lib/gitlab/git_access_spec.rb index 0831021b22b..f95349a2125 100644 --- a/spec/lib/gitlab/git_access_spec.rb +++ b/spec/lib/gitlab/git_access_spec.rb @@ -32,8 +32,8 @@ describe Gitlab::GitAccess do it 'blocks ssh git push and pull' do aggregate_failures do - expect { push_access_check }.to raise_unauthorized('Git access over SSH is not allowed') - expect { pull_access_check }.to raise_unauthorized('Git access over SSH is not allowed') + expect { push_access_check }.to raise_forbidden('Git access over SSH is not allowed') + expect { pull_access_check }.to raise_forbidden('Git access over SSH is not allowed') end end end @@ -48,8 +48,8 @@ describe Gitlab::GitAccess do it 'blocks http push and pull' do aggregate_failures do - expect { push_access_check }.to raise_unauthorized('Git access over HTTP is not allowed') - expect { pull_access_check }.to raise_unauthorized('Git access over HTTP is not allowed') + expect { push_access_check }.to raise_forbidden('Git access over HTTP is not allowed') + expect { pull_access_check }.to raise_forbidden('Git access over HTTP is not allowed') end end @@ -58,7 +58,7 @@ describe Gitlab::GitAccess do it "doesn't block http pull" do aggregate_failures do - expect { pull_access_check }.not_to raise_unauthorized('Git access over HTTP is not allowed') + expect { pull_access_check }.not_to raise_forbidden('Git access over HTTP is not allowed') end end @@ -67,7 +67,7 @@ describe Gitlab::GitAccess do it "doesn't block http pull" do aggregate_failures do - expect { pull_access_check }.not_to raise_unauthorized('Git access over HTTP is not allowed') + expect { pull_access_check }.not_to raise_forbidden('Git access over HTTP is not allowed') end end end @@ -165,7 +165,7 @@ describe Gitlab::GitAccess do end it 'does not block pushes with "not found"' do - expect { push_access_check }.to raise_unauthorized(described_class::ERROR_MESSAGES[:auth_upload]) + expect { push_access_check }.to raise_forbidden(described_class::ERROR_MESSAGES[:auth_upload]) end end @@ -178,7 +178,7 @@ describe Gitlab::GitAccess do end it 'blocks the push' do - expect { push_access_check }.to raise_unauthorized(described_class::ERROR_MESSAGES[:upload]) + expect { push_access_check }.to raise_forbidden(described_class::ERROR_MESSAGES[:upload]) end end @@ -208,7 +208,7 @@ describe Gitlab::GitAccess do end it 'does not block pushes with "not found"' do - expect { push_access_check }.to raise_unauthorized(described_class::ERROR_MESSAGES[:upload]) + expect { push_access_check }.to raise_forbidden(described_class::ERROR_MESSAGES[:upload]) end end @@ -285,8 +285,8 @@ describe Gitlab::GitAccess do it 'does not allow keys which are too small', :aggregate_failures do expect(actor).not_to be_valid - expect { pull_access_check }.to raise_unauthorized('Your SSH key must be at least 4096 bits.') - expect { push_access_check }.to raise_unauthorized('Your SSH key must be at least 4096 bits.') + expect { pull_access_check }.to raise_forbidden('Your SSH key must be at least 4096 bits.') + expect { push_access_check }.to raise_forbidden('Your SSH key must be at least 4096 bits.') end end @@ -297,8 +297,8 @@ describe Gitlab::GitAccess do it 'does not allow keys which are too small', :aggregate_failures do expect(actor).not_to be_valid - expect { pull_access_check }.to raise_unauthorized(/Your SSH key type is forbidden/) - expect { push_access_check }.to raise_unauthorized(/Your SSH key type is forbidden/) + expect { pull_access_check }.to raise_forbidden(/Your SSH key type is forbidden/) + expect { push_access_check }.to raise_forbidden(/Your SSH key type is forbidden/) end end end @@ -363,7 +363,7 @@ describe Gitlab::GitAccess do let(:authentication_abilities) { [] } it 'raises unauthorized with download error' do - expect { pull_access_check }.to raise_unauthorized(described_class::ERROR_MESSAGES[:auth_download]) + expect { pull_access_check }.to raise_forbidden(described_class::ERROR_MESSAGES[:auth_download]) end context 'when authentication abilities include download code' do @@ -387,7 +387,7 @@ describe Gitlab::GitAccess do let(:authentication_abilities) { [] } it 'raises unauthorized with push error' do - expect { push_access_check }.to raise_unauthorized(described_class::ERROR_MESSAGES[:auth_upload]) + expect { push_access_check }.to raise_forbidden(described_class::ERROR_MESSAGES[:auth_upload]) end context 'when authentication abilities include push code' do @@ -414,7 +414,7 @@ describe Gitlab::GitAccess do end context 'when calling git-upload-pack' do - it { expect { pull_access_check }.to raise_unauthorized('Pulling over HTTP is not allowed.') } + it { expect { pull_access_check }.to raise_forbidden('Pulling over HTTP is not allowed.') } end context 'when calling git-receive-pack' do @@ -428,7 +428,7 @@ describe Gitlab::GitAccess do end context 'when calling git-receive-pack' do - it { expect { push_access_check }.to raise_unauthorized('Pushing over HTTP is not allowed.') } + it { expect { push_access_check }.to raise_forbidden('Pushing over HTTP is not allowed.') } end context 'when calling git-upload-pack' do @@ -445,7 +445,7 @@ describe Gitlab::GitAccess do allow(Gitlab::Database).to receive(:read_only?) { true } end - it { expect { push_access_check }.to raise_unauthorized(described_class::ERROR_MESSAGES[:cannot_push_to_read_only]) } + it { expect { push_access_check }.to raise_forbidden(described_class::ERROR_MESSAGES[:cannot_push_to_read_only]) } end end @@ -559,21 +559,21 @@ describe Gitlab::GitAccess do it 'disallows guests to pull' do project.add_guest(user) - expect { pull_access_check }.to raise_unauthorized(described_class::ERROR_MESSAGES[:download]) + expect { pull_access_check }.to raise_forbidden(described_class::ERROR_MESSAGES[:download]) end it 'disallows blocked users to pull' do project.add_maintainer(user) user.block - expect { pull_access_check }.to raise_unauthorized('Your account has been blocked.') + expect { pull_access_check }.to raise_forbidden('Your account has been blocked.') end it 'disallows deactivated users to pull' do project.add_maintainer(user) user.deactivate! - expect { pull_access_check }.to raise_unauthorized("Your account has been deactivated by your administrator. Please log back in from a web browser to reactivate your account at #{Gitlab.config.gitlab.url}") + expect { pull_access_check }.to raise_forbidden("Your account has been deactivated by your administrator. Please log back in from a web browser to reactivate your account at #{Gitlab.config.gitlab.url}") end context 'when the project repository does not exist' do @@ -610,7 +610,7 @@ describe Gitlab::GitAccess do it 'does not give access to download code' do public_project.project_feature.update_attribute(:repository_access_level, ProjectFeature::DISABLED) - expect { pull_access_check }.to raise_unauthorized(described_class::ERROR_MESSAGES[:download]) + expect { pull_access_check }.to raise_forbidden(described_class::ERROR_MESSAGES[:download]) end end end @@ -722,7 +722,7 @@ describe Gitlab::GitAccess do context 'when is not member of the project' do context 'pull code' do - it { expect { pull_access_check }.to raise_unauthorized(described_class::ERROR_MESSAGES[:download]) } + it { expect { pull_access_check }.to raise_forbidden(described_class::ERROR_MESSAGES[:download]) } end end end @@ -828,7 +828,7 @@ describe Gitlab::GitAccess do expect(&check).not_to raise_error, -> { "expected #{action} to be allowed" } else - expect(&check).to raise_error(Gitlab::GitAccess::UnauthorizedError), + expect(&check).to raise_error(Gitlab::GitAccess::ForbiddenError), -> { "expected #{action} to be disallowed" } end end @@ -965,7 +965,7 @@ describe Gitlab::GitAccess do it 'does not allow deactivated users to push' do user.deactivate! - expect { push_access_check }.to raise_unauthorized("Your account has been deactivated by your administrator. Please log back in from a web browser to reactivate your account at #{Gitlab.config.gitlab.url}") + expect { push_access_check }.to raise_forbidden("Your account has been deactivated by your administrator. Please log back in from a web browser to reactivate your account at #{Gitlab.config.gitlab.url}") end it 'cleans up the files' do @@ -1009,26 +1009,26 @@ describe Gitlab::GitAccess do project.add_reporter(user) end - it { expect { push_access_check }.to raise_unauthorized(described_class::ERROR_MESSAGES[:auth_upload]) } + it { expect { push_access_check }.to raise_forbidden(described_class::ERROR_MESSAGES[:auth_upload]) } end context 'when unauthorized' do context 'to public project' do let(:project) { create(:project, :public, :repository) } - it { expect { push_access_check }.to raise_unauthorized(described_class::ERROR_MESSAGES[:auth_upload]) } + it { expect { push_access_check }.to raise_forbidden(described_class::ERROR_MESSAGES[:auth_upload]) } end context 'to internal project' do let(:project) { create(:project, :internal, :repository) } - it { expect { push_access_check }.to raise_unauthorized(described_class::ERROR_MESSAGES[:auth_upload]) } + it { expect { push_access_check }.to raise_forbidden(described_class::ERROR_MESSAGES[:auth_upload]) } end context 'to private project' do let(:project) { create(:project, :private, :repository) } - it { expect { push_access_check }.to raise_unauthorized(described_class::ERROR_MESSAGES[:auth_upload]) } + it { expect { push_access_check }.to raise_forbidden(described_class::ERROR_MESSAGES[:auth_upload]) } end end end @@ -1039,7 +1039,7 @@ describe Gitlab::GitAccess do it 'denies push access' do project.add_maintainer(user) - expect { push_access_check }.to raise_unauthorized('The repository is temporarily read-only. Please try again later.') + expect { push_access_check }.to raise_forbidden('The repository is temporarily read-only. Please try again later.') end end @@ -1060,7 +1060,7 @@ describe Gitlab::GitAccess do context 'to public project' do let(:project) { create(:project, :public, :repository) } - it { expect { push_access_check }.to raise_unauthorized(described_class::ERROR_MESSAGES[:deploy_key_upload]) } + it { expect { push_access_check }.to raise_forbidden(described_class::ERROR_MESSAGES[:deploy_key_upload]) } end context 'to internal project' do @@ -1083,14 +1083,14 @@ describe Gitlab::GitAccess do key.deploy_keys_projects.create(project: project, can_push: false) end - it { expect { push_access_check }.to raise_unauthorized(described_class::ERROR_MESSAGES[:deploy_key_upload]) } + it { expect { push_access_check }.to raise_forbidden(described_class::ERROR_MESSAGES[:deploy_key_upload]) } end context 'when unauthorized' do context 'to public project' do let(:project) { create(:project, :public, :repository) } - it { expect { push_access_check }.to raise_unauthorized(described_class::ERROR_MESSAGES[:deploy_key_upload]) } + it { expect { push_access_check }.to raise_forbidden(described_class::ERROR_MESSAGES[:deploy_key_upload]) } end context 'to internal project' do @@ -1121,7 +1121,7 @@ describe Gitlab::GitAccess do it 'blocks access when the user did not accept terms', :aggregate_failures do actions.each do |action| - expect { action.call }.to raise_unauthorized(/must accept the Terms of Service in order to perform this action/) + expect { action.call }.to raise_forbidden(/must accept the Terms of Service in order to perform this action/) end end @@ -1211,8 +1211,8 @@ describe Gitlab::GitAccess do access.check('git-receive-pack', changes) end - def raise_unauthorized(message) - raise_error(Gitlab::GitAccess::UnauthorizedError, message) + def raise_forbidden(message) + raise_error(Gitlab::GitAccess::ForbiddenError, message) end def raise_not_found diff --git a/spec/lib/gitlab/git_access_wiki_spec.rb b/spec/lib/gitlab/git_access_wiki_spec.rb index 99c9369a2b9..b5e673c9e79 100644 --- a/spec/lib/gitlab/git_access_wiki_spec.rb +++ b/spec/lib/gitlab/git_access_wiki_spec.rb @@ -33,7 +33,7 @@ describe Gitlab::GitAccessWiki do end it 'does not give access to upload wiki code' do - expect { subject }.to raise_error(Gitlab::GitAccess::UnauthorizedError, "You can't push code to a read-only GitLab instance.") + expect { subject }.to raise_error(Gitlab::GitAccess::ForbiddenError, "You can't push code to a read-only GitLab instance.") end end end @@ -70,7 +70,7 @@ describe Gitlab::GitAccessWiki do it 'does not give access to download wiki code' do project.project_feature.update_attribute(:wiki_access_level, ProjectFeature::DISABLED) - expect { subject }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'You are not allowed to download code from this project.') + expect { subject }.to raise_error(Gitlab::GitAccess::ForbiddenError, 'You are not allowed to download code from this project.') end end end diff --git a/spec/lib/gitlab/gitaly_client/commit_service_spec.rb b/spec/lib/gitlab/gitaly_client/commit_service_spec.rb index 5c36d6d35af..00182983418 100644 --- a/spec/lib/gitlab/gitaly_client/commit_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/commit_service_spec.rb @@ -306,5 +306,19 @@ describe Gitlab::GitalyClient::CommitService do client.find_commits(order: 'topo') end + + it 'sends an RPC request with an author' do + request = Gitaly::FindCommitsRequest.new( + repository: repository_message, + disable_walk: true, + order: 'NONE', + author: "Billy Baggins " + ) + + expect_any_instance_of(Gitaly::CommitService::Stub).to receive(:find_commits) + .with(request, kind_of(Hash)).and_return([]) + + client.find_commits(order: 'default', author: "Billy Baggins ") + end end end diff --git a/spec/lib/gitlab/import_export/base/object_builder_spec.rb b/spec/lib/gitlab/import_export/base/object_builder_spec.rb new file mode 100644 index 00000000000..e5242ae0bfc --- /dev/null +++ b/spec/lib/gitlab/import_export/base/object_builder_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::ImportExport::Base::ObjectBuilder do + let(:project) do + create(:project, :repository, + :builds_disabled, + :issues_disabled, + name: 'project', + path: 'project') + end + let(:klass) { Milestone } + let(:attributes) { { 'title' => 'Test Base::ObjectBuilder Milestone', 'project' => project } } + + subject { described_class.build(klass, attributes) } + + describe '#build' do + context 'when object exists' do + context 'when where_clauses are implemented' do + before do + allow_next_instance_of(described_class) do |object_builder| + allow(object_builder).to receive(:where_clauses).and_return([klass.arel_table['title'].eq(attributes['title'])]) + end + end + + let!(:milestone) { create(:milestone, title: attributes['title'], project: project) } + + it 'finds existing object instead of creating one' do + expect(subject).to eq(milestone) + end + end + + context 'when where_clauses are not implemented' do + it 'raises NotImplementedError' do + expect { subject }.to raise_error(NotImplementedError) + end + end + end + + context 'when object does not exist' do + before do + allow_next_instance_of(described_class) do |object_builder| + allow(object_builder).to receive(:find_object).and_return(nil) + end + end + + it 'creates new object' do + expect { subject }.to change { Milestone.count }.from(0).to(1) + end + end + end +end diff --git a/spec/lib/gitlab/import_export/base/relation_factory_spec.rb b/spec/lib/gitlab/import_export/base/relation_factory_spec.rb new file mode 100644 index 00000000000..1011de83c95 --- /dev/null +++ b/spec/lib/gitlab/import_export/base/relation_factory_spec.rb @@ -0,0 +1,143 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::ImportExport::Base::RelationFactory do + let(:user) { create(:admin) } + let(:project) { create(:project) } + let(:members_mapper) { double('members_mapper').as_null_object } + let(:relation_sym) { :project_snippets } + let(:relation_hash) { {} } + let(:excluded_keys) { [] } + + subject do + described_class.create(relation_sym: relation_sym, + relation_hash: relation_hash, + object_builder: Gitlab::ImportExport::Project::ObjectBuilder, + members_mapper: members_mapper, + user: user, + importable: project, + excluded_keys: excluded_keys) + end + + describe '#create' do + context 'when relation is invalid' do + before do + expect_next_instance_of(described_class) do |relation_factory| + expect(relation_factory).to receive(:invalid_relation?).and_return(true) + end + end + + it 'returns without creating new relations' do + expect(subject).to be_nil + end + end + + context 'when #setup_models is not implemented' do + it 'raises NotImplementedError' do + expect { subject }.to raise_error(NotImplementedError) + end + end + + context 'when #setup_models is implemented' do + let(:relation_sym) { :notes } + let(:relation_hash) do + { + "id" => 4947, + "note" => "merged", + "noteable_type" => "MergeRequest", + "author_id" => 999, + "created_at" => "2016-11-18T09:29:42.634Z", + "updated_at" => "2016-11-18T09:29:42.634Z", + "project_id" => 1, + "attachment" => { + "url" => nil + }, + "noteable_id" => 377, + "system" => true, + "events" => [] + } + end + + before do + expect_next_instance_of(described_class) do |relation_factory| + expect(relation_factory).to receive(:setup_models).and_return(true) + end + end + + it 'creates imported object' do + expect(subject).to be_instance_of(Note) + end + + context 'when relation contains user references' do + let(:new_user) { create(:user) } + let(:exported_member) do + { + "id" => 111, + "access_level" => 30, + "source_id" => 1, + "source_type" => "Project", + "user_id" => 3, + "notification_level" => 3, + "created_at" => "2016-11-18T09:29:42.634Z", + "updated_at" => "2016-11-18T09:29:42.634Z", + "user" => { + "id" => 999, + "email" => new_user.email, + "username" => new_user.username + } + } + end + + let(:members_mapper) do + Gitlab::ImportExport::MembersMapper.new( + exported_members: [exported_member], + user: user, + importable: project) + end + + it 'maps the right author to the imported note' do + expect(subject.author).to eq(new_user) + end + end + + context 'when relation contains token attributes' do + let(:relation_sym) { 'ProjectHook' } + let(:relation_hash) { { token: 'secret' } } + + it 'removes token attributes' do + expect(subject.token).to be_nil + end + end + + context 'when relation contains encrypted attributes' do + let(:relation_sym) { 'Ci::Variable' } + let(:relation_hash) do + create(:ci_variable).as_json + end + + it 'removes encrypted attributes' do + expect(subject.value).to be_nil + end + end + end + end + + describe '.relation_class' do + context 'when relation name is pluralized' do + let(:relation_name) { 'MergeRequest::Metrics' } + + it 'returns constantized class' do + expect(described_class.relation_class(relation_name)).to eq(MergeRequest::Metrics) + end + end + + context 'when relation name is singularized' do + let(:relation_name) { 'Badge' } + + it 'returns constantized class' do + expect(described_class.relation_class(relation_name)).to eq(Badge) + end + end + end +end diff --git a/spec/lib/gitlab/import_export/base_object_builder_spec.rb b/spec/lib/gitlab/import_export/base_object_builder_spec.rb deleted file mode 100644 index fbb3b08cf56..00000000000 --- a/spec/lib/gitlab/import_export/base_object_builder_spec.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -describe Gitlab::ImportExport::BaseObjectBuilder do - let(:project) do - create(:project, :repository, - :builds_disabled, - :issues_disabled, - name: 'project', - path: 'project') - end - let(:klass) { Milestone } - let(:attributes) { { 'title' => 'Test BaseObjectBuilder Milestone', 'project' => project } } - - subject { described_class.build(klass, attributes) } - - describe '#build' do - context 'when object exists' do - context 'when where_clauses are implemented' do - before do - allow_next_instance_of(described_class) do |object_builder| - allow(object_builder).to receive(:where_clauses).and_return([klass.arel_table['title'].eq(attributes['title'])]) - end - end - - let!(:milestone) { create(:milestone, title: attributes['title'], project: project) } - - it 'finds existing object instead of creating one' do - expect(subject).to eq(milestone) - end - end - - context 'when where_clauses are not implemented' do - it 'raises NotImplementedError' do - expect { subject }.to raise_error(NotImplementedError) - end - end - end - - context 'when object does not exist' do - before do - allow_next_instance_of(described_class) do |object_builder| - allow(object_builder).to receive(:find_object).and_return(nil) - end - end - - it 'creates new object' do - expect { subject }.to change { Milestone.count }.from(0).to(1) - end - end - end -end diff --git a/spec/lib/gitlab/import_export/base_relation_factory_spec.rb b/spec/lib/gitlab/import_export/base_relation_factory_spec.rb deleted file mode 100644 index e02d8f3d2e6..00000000000 --- a/spec/lib/gitlab/import_export/base_relation_factory_spec.rb +++ /dev/null @@ -1,143 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -describe Gitlab::ImportExport::BaseRelationFactory do - let(:user) { create(:admin) } - let(:project) { create(:project) } - let(:members_mapper) { double('members_mapper').as_null_object } - let(:relation_sym) { :project_snippets } - let(:relation_hash) { {} } - let(:excluded_keys) { [] } - - subject do - described_class.create(relation_sym: relation_sym, - relation_hash: relation_hash, - object_builder: Gitlab::ImportExport::GroupProjectObjectBuilder, - members_mapper: members_mapper, - user: user, - importable: project, - excluded_keys: excluded_keys) - end - - describe '#create' do - context 'when relation is invalid' do - before do - expect_next_instance_of(described_class) do |relation_factory| - expect(relation_factory).to receive(:invalid_relation?).and_return(true) - end - end - - it 'returns without creating new relations' do - expect(subject).to be_nil - end - end - - context 'when #setup_models is not implemented' do - it 'raises NotImplementedError' do - expect { subject }.to raise_error(NotImplementedError) - end - end - - context 'when #setup_models is implemented' do - let(:relation_sym) { :notes } - let(:relation_hash) do - { - "id" => 4947, - "note" => "merged", - "noteable_type" => "MergeRequest", - "author_id" => 999, - "created_at" => "2016-11-18T09:29:42.634Z", - "updated_at" => "2016-11-18T09:29:42.634Z", - "project_id" => 1, - "attachment" => { - "url" => nil - }, - "noteable_id" => 377, - "system" => true, - "events" => [] - } - end - - before do - expect_next_instance_of(described_class) do |relation_factory| - expect(relation_factory).to receive(:setup_models).and_return(true) - end - end - - it 'creates imported object' do - expect(subject).to be_instance_of(Note) - end - - context 'when relation contains user references' do - let(:new_user) { create(:user) } - let(:exported_member) do - { - "id" => 111, - "access_level" => 30, - "source_id" => 1, - "source_type" => "Project", - "user_id" => 3, - "notification_level" => 3, - "created_at" => "2016-11-18T09:29:42.634Z", - "updated_at" => "2016-11-18T09:29:42.634Z", - "user" => { - "id" => 999, - "email" => new_user.email, - "username" => new_user.username - } - } - end - - let(:members_mapper) do - Gitlab::ImportExport::MembersMapper.new( - exported_members: [exported_member], - user: user, - importable: project) - end - - it 'maps the right author to the imported note' do - expect(subject.author).to eq(new_user) - end - end - - context 'when relation contains token attributes' do - let(:relation_sym) { 'ProjectHook' } - let(:relation_hash) { { token: 'secret' } } - - it 'removes token attributes' do - expect(subject.token).to be_nil - end - end - - context 'when relation contains encrypted attributes' do - let(:relation_sym) { 'Ci::Variable' } - let(:relation_hash) do - create(:ci_variable).as_json - end - - it 'removes encrypted attributes' do - expect(subject.value).to be_nil - end - end - end - end - - describe '.relation_class' do - context 'when relation name is pluralized' do - let(:relation_name) { 'MergeRequest::Metrics' } - - it 'returns constantized class' do - expect(described_class.relation_class(relation_name)).to eq(MergeRequest::Metrics) - end - end - - context 'when relation name is singularized' do - let(:relation_name) { 'Badge' } - - it 'returns constantized class' do - expect(described_class.relation_class(relation_name)).to eq(Badge) - end - end - end -end diff --git a/spec/lib/gitlab/import_export/fork_spec.rb b/spec/lib/gitlab/import_export/fork_spec.rb index 09e4f62c686..8aa28353c04 100644 --- a/spec/lib/gitlab/import_export/fork_spec.rb +++ b/spec/lib/gitlab/import_export/fork_spec.rb @@ -24,11 +24,11 @@ describe 'forked project import' do end let(:saver) do - Gitlab::ImportExport::ProjectTreeSaver.new(project: project_with_repo, current_user: user, shared: shared) + Gitlab::ImportExport::Project::TreeSaver.new(project: project_with_repo, current_user: user, shared: shared) end let(:restorer) do - Gitlab::ImportExport::ProjectTreeRestorer.new(user: user, shared: shared, project: project) + Gitlab::ImportExport::Project::TreeRestorer.new(user: user, shared: shared, project: project) end before do diff --git a/spec/lib/gitlab/import_export/group/object_builder_spec.rb b/spec/lib/gitlab/import_export/group/object_builder_spec.rb new file mode 100644 index 00000000000..781670b0aa5 --- /dev/null +++ b/spec/lib/gitlab/import_export/group/object_builder_spec.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::ImportExport::Group::ObjectBuilder do + let(:group) { create(:group) } + let(:base_attributes) do + { + 'title' => 'title', + 'description' => 'description', + 'group' => group + } + end + + context 'labels' do + let(:label_attributes) { base_attributes.merge('type' => 'GroupLabel') } + + it 'finds the existing group label' do + group_label = create(:group_label, base_attributes) + + expect(described_class.build(Label, label_attributes)).to eq(group_label) + end + + it 'creates a new label' do + label = described_class.build(Label, label_attributes) + + expect(label.persisted?).to be true + end + + context 'when description is an empty string' do + let(:label_attributes) { base_attributes.merge('type' => 'GroupLabel', 'description' => '') } + + it 'finds the existing group label' do + group_label = create(:group_label, label_attributes) + + expect(described_class.build(Label, label_attributes)).to eq(group_label) + end + end + end + + context 'milestones' do + it 'finds the existing group milestone' do + milestone = create(:milestone, base_attributes) + + expect(described_class.build(Milestone, base_attributes)).to eq(milestone) + end + + it 'creates a new milestone' do + milestone = described_class.build(Milestone, base_attributes) + + expect(milestone.persisted?).to be true + end + end + + describe '#initialize' do + context 'when attributes contain description as empty string' do + let(:attributes) { base_attributes.merge('description' => '') } + + it 'converts empty string to nil' do + builder = described_class.new(Label, attributes) + + expect(builder.send(:attributes)).to include({ 'description' => nil }) + end + end + end +end diff --git a/spec/lib/gitlab/import_export/group/relation_factory_spec.rb b/spec/lib/gitlab/import_export/group/relation_factory_spec.rb new file mode 100644 index 00000000000..332648d5c89 --- /dev/null +++ b/spec/lib/gitlab/import_export/group/relation_factory_spec.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::ImportExport::Group::RelationFactory do + let(:group) { create(:group) } + let(:members_mapper) { double('members_mapper').as_null_object } + let(:user) { create(:admin) } + let(:excluded_keys) { [] } + let(:created_object) do + described_class.create(relation_sym: relation_sym, + relation_hash: relation_hash, + members_mapper: members_mapper, + object_builder: Gitlab::ImportExport::Group::ObjectBuilder, + user: user, + importable: group, + excluded_keys: excluded_keys) + end + + context 'label object' do + let(:relation_sym) { :group_label } + let(:id) { random_id } + let(:original_group_id) { random_id } + + let(:relation_hash) do + { + 'id' => 123456, + 'title' => 'Bruffefunc', + 'color' => '#1d2da4', + 'project_id' => nil, + 'created_at' => '2019-11-20T17:02:20.546Z', + 'updated_at' => '2019-11-20T17:02:20.546Z', + 'template' => false, + 'description' => 'Description', + 'group_id' => original_group_id, + 'type' => 'GroupLabel', + 'priorities' => [], + 'textColor' => '#FFFFFF' + } + end + + it 'does not have the original ID' do + expect(created_object.id).not_to eq(id) + end + + it 'does not have the original group_id' do + expect(created_object.group_id).not_to eq(original_group_id) + end + + it 'has the new group_id' do + expect(created_object.group_id).to eq(group.id) + end + + context 'excluded attributes' do + let(:excluded_keys) { %w[description] } + + it 'are removed from the imported object' do + expect(created_object.description).to be_nil + end + end + end + + context 'Notes user references' do + let(:relation_sym) { :notes } + let(:new_user) { create(:user) } + let(:exported_member) do + { + 'id' => 111, + 'access_level' => 30, + 'source_id' => 1, + 'source_type' => 'Namespace', + 'user_id' => 3, + 'notification_level' => 3, + 'created_at' => '2016-11-18T09:29:42.634Z', + 'updated_at' => '2016-11-18T09:29:42.634Z', + 'user' => { + 'id' => 999, + 'email' => new_user.email, + 'username' => new_user.username + } + } + end + + let(:relation_hash) do + { + 'id' => 4947, + 'note' => 'note', + 'noteable_type' => 'Epic', + 'author_id' => 999, + 'created_at' => '2016-11-18T09:29:42.634Z', + 'updated_at' => '2016-11-18T09:29:42.634Z', + 'project_id' => 1, + 'attachment' => { + 'url' => nil + }, + 'noteable_id' => 377, + 'system' => true, + 'author' => { + 'name' => 'Administrator' + }, + 'events' => [] + } + end + + let(:members_mapper) do + Gitlab::ImportExport::MembersMapper.new( + exported_members: [exported_member], + user: user, + importable: group) + end + + it 'maps the right author to the imported note' do + expect(created_object.author).to eq(new_user) + end + end + + def random_id + rand(1000..10000) + end +end diff --git a/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb b/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb new file mode 100644 index 00000000000..5584f1503f7 --- /dev/null +++ b/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb @@ -0,0 +1,153 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::ImportExport::Group::TreeRestorer do + include ImportExport::CommonUtil + + let(:shared) { Gitlab::ImportExport::Shared.new(group) } + + describe 'restore group tree' do + before(:context) do + # Using an admin for import, so we can check assignment of existing members + user = create(:admin, email: 'root@gitlabexample.com') + create(:user, email: 'adriene.mcclure@gitlabexample.com') + create(:user, email: 'gwendolyn_robel@gitlabexample.com') + + RSpec::Mocks.with_temporary_scope do + @group = create(:group, name: 'group', path: 'group') + @shared = Gitlab::ImportExport::Shared.new(@group) + + setup_import_export_config('group_exports/complex') + + group_tree_restorer = described_class.new(user: user, shared: @shared, group: @group, group_hash: nil) + + @restored_group_json = group_tree_restorer.restore + end + end + + context 'JSON' do + it 'restores models based on JSON' do + expect(@restored_group_json).to be_truthy + end + + it 'has the group description' do + expect(Group.find_by_path('group').description).to eq('Group Description') + end + + it 'has group labels' do + expect(@group.labels.count).to eq(10) + end + + context 'issue boards' do + it 'has issue boards' do + expect(@group.boards.count).to eq(1) + end + + it 'has board label lists' do + lists = @group.boards.find_by(name: 'first board').lists + + expect(lists.count).to eq(3) + expect(lists.first.label.title).to eq('TSL') + expect(lists.second.label.title).to eq('Sosync') + end + end + + it 'has badges' do + expect(@group.badges.count).to eq(1) + end + + it 'has milestones' do + expect(@group.milestones.count).to eq(5) + end + + it 'has group children' do + expect(@group.children.count).to eq(2) + end + + it 'has group members' do + expect(@group.members.map(&:user).map(&:email)).to contain_exactly('root@gitlabexample.com', 'adriene.mcclure@gitlabexample.com', 'gwendolyn_robel@gitlabexample.com') + end + end + end + + context 'excluded attributes' do + let!(:source_user) { create(:user, id: 123) } + let!(:importer_user) { create(:user) } + let(:group) { create(:group) } + let(:shared) { Gitlab::ImportExport::Shared.new(group) } + let(:group_tree_restorer) { described_class.new(user: importer_user, shared: shared, group: group, group_hash: nil) } + let(:group_json) { ActiveSupport::JSON.decode(IO.read(File.join(shared.export_path, 'group.json'))) } + + shared_examples 'excluded attributes' do + excluded_attributes = %w[ + id + owner_id + parent_id + created_at + updated_at + runners_token + runners_token_encrypted + saml_discovery_token + ] + + before do + group.add_owner(importer_user) + + setup_import_export_config('group_exports/complex') + end + + excluded_attributes.each do |excluded_attribute| + it 'does not allow override of excluded attributes' do + expect(group_json[excluded_attribute]).not_to eq(group.public_send(excluded_attribute)) + end + end + end + + include_examples 'excluded attributes' + end + + context 'group.json file access check' do + let(:user) { create(:user) } + let!(:group) { create(:group, name: 'group2', path: 'group2') } + let(:group_tree_restorer) { described_class.new(user: user, shared: shared, group: group, group_hash: nil) } + let(:restored_group_json) { group_tree_restorer.restore } + + it 'does not read a symlink' do + Dir.mktmpdir do |tmpdir| + setup_symlink(tmpdir, 'group.json') + allow(shared).to receive(:export_path).and_call_original + + expect(group_tree_restorer.restore).to eq(false) + expect(shared.errors).to include('Incorrect JSON format') + end + end + end + + context 'group visibility levels' do + let(:user) { create(:user) } + let(:shared) { Gitlab::ImportExport::Shared.new(group) } + let(:group_tree_restorer) { described_class.new(user: user, shared: shared, group: group, group_hash: nil) } + + before do + setup_import_export_config(filepath) + + group_tree_restorer.restore + end + + shared_examples 'with visibility level' do |visibility_level, expected_visibilities| + context "when visibility level is #{visibility_level}" do + let(:group) { create(:group, visibility_level) } + let(:filepath) { "group_exports/visibility_levels/#{visibility_level}" } + + it "imports all subgroups as #{visibility_level}" do + expect(group.children.map(&:visibility_level)).to eq(expected_visibilities) + end + end + end + + include_examples 'with visibility level', :public, [20, 10, 0] + include_examples 'with visibility level', :private, [0, 0, 0] + include_examples 'with visibility level', :internal, [10, 10, 0] + end +end diff --git a/spec/lib/gitlab/import_export/group/tree_saver_spec.rb b/spec/lib/gitlab/import_export/group/tree_saver_spec.rb new file mode 100644 index 00000000000..845eb8e308b --- /dev/null +++ b/spec/lib/gitlab/import_export/group/tree_saver_spec.rb @@ -0,0 +1,202 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::ImportExport::Group::TreeSaver do + describe 'saves the group tree into a json object' do + let(:shared) { Gitlab::ImportExport::Shared.new(group) } + let(:group_tree_saver) { described_class.new(group: group, current_user: user, shared: shared) } + let(:export_path) { "#{Dir.tmpdir}/group_tree_saver_spec" } + let(:user) { create(:user) } + let!(:group) { setup_group } + + before do + group.add_maintainer(user) + allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) + end + + after do + FileUtils.rm_rf(export_path) + end + + it 'saves group successfully' do + expect(group_tree_saver.save).to be true + end + + context ':export_fast_serialize feature flag checks' do + before do + expect(Gitlab::ImportExport::Reader).to receive(:new).with(shared: shared, config: group_config).and_return(reader) + expect(reader).to receive(:group_tree).and_return(group_tree) + end + + let(:reader) { instance_double('Gitlab::ImportExport::Reader') } + let(:group_config) { Gitlab::ImportExport::Config.new(config: Gitlab::ImportExport.group_config_file).to_h } + let(:group_tree) do + { + include: [{ milestones: { include: [] } }], + preload: { milestones: nil } + } + end + + context 'when :export_fast_serialize feature is enabled' do + let(:serializer) { instance_double(Gitlab::ImportExport::FastHashSerializer) } + + before do + stub_feature_flags(export_fast_serialize: true) + + expect(Gitlab::ImportExport::FastHashSerializer).to receive(:new).with(group, group_tree).and_return(serializer) + end + + it 'uses FastHashSerializer' do + expect(serializer).to receive(:execute) + + group_tree_saver.save + end + end + + context 'when :export_fast_serialize feature is disabled' do + before do + stub_feature_flags(export_fast_serialize: false) + end + + it 'is serialized via built-in `as_json`' do + expect(group).to receive(:as_json).with(group_tree).and_call_original + + group_tree_saver.save + end + end + end + + # It is mostly duplicated in + # `spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb` + # except: + # context 'with description override' do + # context 'group members' do + # ^ These are specific for the Group::TreeSaver + context 'JSON' do + let(:saved_group_json) do + group_tree_saver.save + group_json(group_tree_saver.full_path) + end + + it 'saves the correct json' do + expect(saved_group_json).to include({ 'description' => 'description' }) + end + + it 'has milestones' do + expect(saved_group_json['milestones']).not_to be_empty + end + + it 'has labels' do + expect(saved_group_json['labels']).not_to be_empty + end + + it 'has boards' do + expect(saved_group_json['boards']).not_to be_empty + end + + it 'has board label list' do + expect(saved_group_json['boards'].first['lists']).not_to be_empty + end + + it 'has group members' do + expect(saved_group_json['members']).not_to be_empty + end + + it 'has priorities associated to labels' do + expect(saved_group_json['labels'].first['priorities']).not_to be_empty + end + + it 'has badges' do + expect(saved_group_json['badges']).not_to be_empty + end + + context 'group children' do + let(:children) { group.children } + + it 'exports group children' do + expect(saved_group_json['children'].length).to eq(children.count) + end + + it 'exports group children of children' do + expect(saved_group_json['children'].first['children'].length).to eq(children.first.children.count) + end + end + + context 'group members' do + let(:user2) { create(:user, email: 'group@member.com') } + let(:member_emails) do + saved_group_json['members'].map do |pm| + pm['user']['email'] + end + end + + before do + group.add_developer(user2) + end + + it 'exports group members as group owner' do + group.add_owner(user) + + expect(member_emails).to include('group@member.com') + end + + context 'as admin' do + let(:user) { create(:admin) } + + it 'exports group members as admin' do + expect(member_emails).to include('group@member.com') + end + + it 'exports group members' do + member_types = saved_group_json['members'].map { |pm| pm['source_type'] } + + expect(member_types).to all(eq('Namespace')) + end + end + end + + context 'group attributes' do + shared_examples 'excluded attributes' do + excluded_attributes = %w[ + id + owner_id + parent_id + created_at + updated_at + runners_token + runners_token_encrypted + saml_discovery_token + ] + + excluded_attributes.each do |excluded_attribute| + it 'does not contain excluded attribute' do + expect(saved_group_json).not_to include(excluded_attribute => group.public_send(excluded_attribute)) + end + end + end + + include_examples 'excluded attributes' + end + end + end + + def setup_group + group = create(:group, description: 'description') + sub_group = create(:group, description: 'description', parent: group) + create(:group, description: 'description', parent: sub_group) + create(:milestone, group: group) + create(:group_badge, group: group) + group_label = create(:group_label, group: group) + create(:label_priority, label: group_label, priority: 1) + board = create(:board, group: group) + create(:list, board: board, label: group_label) + create(:group_badge, group: group) + + group + end + + def group_json(filename) + JSON.parse(IO.read(filename)) + end +end diff --git a/spec/lib/gitlab/import_export/group_object_builder_spec.rb b/spec/lib/gitlab/import_export/group_object_builder_spec.rb deleted file mode 100644 index 08b2dae1147..00000000000 --- a/spec/lib/gitlab/import_export/group_object_builder_spec.rb +++ /dev/null @@ -1,66 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -describe Gitlab::ImportExport::GroupObjectBuilder do - let(:group) { create(:group) } - let(:base_attributes) do - { - 'title' => 'title', - 'description' => 'description', - 'group' => group - } - end - - context 'labels' do - let(:label_attributes) { base_attributes.merge('type' => 'GroupLabel') } - - it 'finds the existing group label' do - group_label = create(:group_label, base_attributes) - - expect(described_class.build(Label, label_attributes)).to eq(group_label) - end - - it 'creates a new label' do - label = described_class.build(Label, label_attributes) - - expect(label.persisted?).to be true - end - - context 'when description is an empty string' do - let(:label_attributes) { base_attributes.merge('type' => 'GroupLabel', 'description' => '') } - - it 'finds the existing group label' do - group_label = create(:group_label, label_attributes) - - expect(described_class.build(Label, label_attributes)).to eq(group_label) - end - end - end - - context 'milestones' do - it 'finds the existing group milestone' do - milestone = create(:milestone, base_attributes) - - expect(described_class.build(Milestone, base_attributes)).to eq(milestone) - end - - it 'creates a new milestone' do - milestone = described_class.build(Milestone, base_attributes) - - expect(milestone.persisted?).to be true - end - end - - describe '#initialize' do - context 'when attributes contain description as empty string' do - let(:attributes) { base_attributes.merge('description' => '') } - - it 'converts empty string to nil' do - builder = described_class.new(Label, attributes) - - expect(builder.send(:attributes)).to include({ 'description' => nil }) - end - end - end -end diff --git a/spec/lib/gitlab/import_export/group_project_object_builder_spec.rb b/spec/lib/gitlab/import_export/group_project_object_builder_spec.rb deleted file mode 100644 index 34049cbf570..00000000000 --- a/spec/lib/gitlab/import_export/group_project_object_builder_spec.rb +++ /dev/null @@ -1,153 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -describe Gitlab::ImportExport::GroupProjectObjectBuilder do - let!(:group) { create(:group, :private) } - let!(:subgroup) { create(:group, :private, parent: group) } - let!(:project) do - create(:project, :repository, - :builds_disabled, - :issues_disabled, - name: 'project', - path: 'project', - group: subgroup) - end - - let(:lru_cache) { subject.send(:lru_cache) } - let(:cache_key) { subject.send(:cache_key) } - - context 'request store is not active' do - subject do - described_class.new(Label, - 'title' => 'group label', - 'project' => project, - 'group' => project.group) - end - - it 'ignore cache initialize' do - expect(lru_cache).to be_nil - expect(cache_key).to be_nil - end - end - - context 'request store is active', :request_store do - subject do - described_class.new(Label, - 'title' => 'group label', - 'project' => project, - 'group' => project.group) - end - - it 'initialize cache in memory' do - expect(lru_cache).not_to be_nil - expect(cache_key).not_to be_nil - end - - it 'cache object when first time find the object' do - group_label = create(:group_label, name: 'group label', group: project.group) - - expect(subject).to receive(:find_object).and_call_original - expect { subject.find } - .to change { lru_cache[cache_key] } - .from(nil).to(group_label) - - expect(subject.find).to eq(group_label) - end - - it 'read from cache when object has been cached' do - group_label = create(:group_label, name: 'group label', group: project.group) - - subject.find - - expect(subject).not_to receive(:find_object) - expect { subject.find }.not_to change { lru_cache[cache_key] } - - expect(subject.find).to eq(group_label) - end - end - - context 'labels' do - it 'finds the existing group label' do - group_label = create(:group_label, name: 'group label', group: project.group) - - expect(described_class.build(Label, - 'title' => 'group label', - 'project' => project, - 'group' => project.group)).to eq(group_label) - end - - it 'finds the existing group label in root ancestor' do - group_label = create(:group_label, name: 'group label', group: group) - - expect(described_class.build(Label, - 'title' => 'group label', - 'project' => project, - 'group' => group)).to eq(group_label) - end - - it 'creates a new label' do - label = described_class.build(Label, - 'title' => 'group label', - 'project' => project, - 'group' => project.group) - - expect(label.persisted?).to be true - end - end - - context 'milestones' do - it 'finds the existing group milestone' do - milestone = create(:milestone, name: 'group milestone', group: project.group) - - expect(described_class.build(Milestone, - 'title' => 'group milestone', - 'project' => project, - 'group' => project.group)).to eq(milestone) - end - - it 'finds the existing group milestone in root ancestor' do - milestone = create(:milestone, name: 'group milestone', group: group) - - expect(described_class.build(Milestone, - 'title' => 'group milestone', - 'project' => project, - 'group' => group)).to eq(milestone) - end - - it 'creates a new milestone' do - milestone = described_class.build(Milestone, - 'title' => 'group milestone', - 'project' => project, - 'group' => project.group) - - expect(milestone.persisted?).to be true - end - end - - context 'merge_request' do - it 'finds the existing merge_request' do - merge_request = create(:merge_request, title: 'MergeRequest', iid: 7, target_project: project, source_project: project) - expect(described_class.build(MergeRequest, - 'title' => 'MergeRequest', - 'source_project_id' => project.id, - 'target_project_id' => project.id, - 'source_branch' => 'SourceBranch', - 'iid' => 7, - 'target_branch' => 'TargetBranch', - 'author_id' => project.creator.id)).to eq(merge_request) - end - - it 'creates a new merge_request' do - merge_request = described_class.build(MergeRequest, - 'title' => 'MergeRequest', - 'iid' => 8, - 'source_project_id' => project.id, - 'target_project_id' => project.id, - 'source_branch' => 'SourceBranch', - 'target_branch' => 'TargetBranch', - 'author_id' => project.creator.id) - expect(merge_request.persisted?).to be true - end - end -end diff --git a/spec/lib/gitlab/import_export/group_relation_factory_spec.rb b/spec/lib/gitlab/import_export/group_relation_factory_spec.rb deleted file mode 100644 index 9208b2ad203..00000000000 --- a/spec/lib/gitlab/import_export/group_relation_factory_spec.rb +++ /dev/null @@ -1,120 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -describe Gitlab::ImportExport::GroupRelationFactory do - let(:group) { create(:group) } - let(:members_mapper) { double('members_mapper').as_null_object } - let(:user) { create(:admin) } - let(:excluded_keys) { [] } - let(:created_object) do - described_class.create(relation_sym: relation_sym, - relation_hash: relation_hash, - members_mapper: members_mapper, - object_builder: Gitlab::ImportExport::GroupObjectBuilder, - user: user, - importable: group, - excluded_keys: excluded_keys) - end - - context 'label object' do - let(:relation_sym) { :group_label } - let(:id) { random_id } - let(:original_group_id) { random_id } - - let(:relation_hash) do - { - 'id' => 123456, - 'title' => 'Bruffefunc', - 'color' => '#1d2da4', - 'project_id' => nil, - 'created_at' => '2019-11-20T17:02:20.546Z', - 'updated_at' => '2019-11-20T17:02:20.546Z', - 'template' => false, - 'description' => 'Description', - 'group_id' => original_group_id, - 'type' => 'GroupLabel', - 'priorities' => [], - 'textColor' => '#FFFFFF' - } - end - - it 'does not have the original ID' do - expect(created_object.id).not_to eq(id) - end - - it 'does not have the original group_id' do - expect(created_object.group_id).not_to eq(original_group_id) - end - - it 'has the new group_id' do - expect(created_object.group_id).to eq(group.id) - end - - context 'excluded attributes' do - let(:excluded_keys) { %w[description] } - - it 'are removed from the imported object' do - expect(created_object.description).to be_nil - end - end - end - - context 'Notes user references' do - let(:relation_sym) { :notes } - let(:new_user) { create(:user) } - let(:exported_member) do - { - 'id' => 111, - 'access_level' => 30, - 'source_id' => 1, - 'source_type' => 'Namespace', - 'user_id' => 3, - 'notification_level' => 3, - 'created_at' => '2016-11-18T09:29:42.634Z', - 'updated_at' => '2016-11-18T09:29:42.634Z', - 'user' => { - 'id' => 999, - 'email' => new_user.email, - 'username' => new_user.username - } - } - end - - let(:relation_hash) do - { - 'id' => 4947, - 'note' => 'note', - 'noteable_type' => 'Epic', - 'author_id' => 999, - 'created_at' => '2016-11-18T09:29:42.634Z', - 'updated_at' => '2016-11-18T09:29:42.634Z', - 'project_id' => 1, - 'attachment' => { - 'url' => nil - }, - 'noteable_id' => 377, - 'system' => true, - 'author' => { - 'name' => 'Administrator' - }, - 'events' => [] - } - end - - let(:members_mapper) do - Gitlab::ImportExport::MembersMapper.new( - exported_members: [exported_member], - user: user, - importable: group) - end - - it 'maps the right author to the imported note' do - expect(created_object.author).to eq(new_user) - end - end - - def random_id - rand(1000..10000) - end -end diff --git a/spec/lib/gitlab/import_export/group_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/group_tree_restorer_spec.rb deleted file mode 100644 index b2c8398d358..00000000000 --- a/spec/lib/gitlab/import_export/group_tree_restorer_spec.rb +++ /dev/null @@ -1,153 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -describe Gitlab::ImportExport::GroupTreeRestorer do - include ImportExport::CommonUtil - - let(:shared) { Gitlab::ImportExport::Shared.new(group) } - - describe 'restore group tree' do - before(:context) do - # Using an admin for import, so we can check assignment of existing members - user = create(:admin, email: 'root@gitlabexample.com') - create(:user, email: 'adriene.mcclure@gitlabexample.com') - create(:user, email: 'gwendolyn_robel@gitlabexample.com') - - RSpec::Mocks.with_temporary_scope do - @group = create(:group, name: 'group', path: 'group') - @shared = Gitlab::ImportExport::Shared.new(@group) - - setup_import_export_config('group_exports/complex') - - group_tree_restorer = described_class.new(user: user, shared: @shared, group: @group, group_hash: nil) - - @restored_group_json = group_tree_restorer.restore - end - end - - context 'JSON' do - it 'restores models based on JSON' do - expect(@restored_group_json).to be_truthy - end - - it 'has the group description' do - expect(Group.find_by_path('group').description).to eq('Group Description') - end - - it 'has group labels' do - expect(@group.labels.count).to eq(10) - end - - context 'issue boards' do - it 'has issue boards' do - expect(@group.boards.count).to eq(1) - end - - it 'has board label lists' do - lists = @group.boards.find_by(name: 'first board').lists - - expect(lists.count).to eq(3) - expect(lists.first.label.title).to eq('TSL') - expect(lists.second.label.title).to eq('Sosync') - end - end - - it 'has badges' do - expect(@group.badges.count).to eq(1) - end - - it 'has milestones' do - expect(@group.milestones.count).to eq(5) - end - - it 'has group children' do - expect(@group.children.count).to eq(2) - end - - it 'has group members' do - expect(@group.members.map(&:user).map(&:email)).to contain_exactly('root@gitlabexample.com', 'adriene.mcclure@gitlabexample.com', 'gwendolyn_robel@gitlabexample.com') - end - end - end - - context 'excluded attributes' do - let!(:source_user) { create(:user, id: 123) } - let!(:importer_user) { create(:user) } - let(:group) { create(:group) } - let(:shared) { Gitlab::ImportExport::Shared.new(group) } - let(:group_tree_restorer) { described_class.new(user: importer_user, shared: shared, group: group, group_hash: nil) } - let(:group_json) { ActiveSupport::JSON.decode(IO.read(File.join(shared.export_path, 'group.json'))) } - - shared_examples 'excluded attributes' do - excluded_attributes = %w[ - id - owner_id - parent_id - created_at - updated_at - runners_token - runners_token_encrypted - saml_discovery_token - ] - - before do - group.add_owner(importer_user) - - setup_import_export_config('group_exports/complex') - end - - excluded_attributes.each do |excluded_attribute| - it 'does not allow override of excluded attributes' do - expect(group_json[excluded_attribute]).not_to eq(group.public_send(excluded_attribute)) - end - end - end - - include_examples 'excluded attributes' - end - - context 'group.json file access check' do - let(:user) { create(:user) } - let!(:group) { create(:group, name: 'group2', path: 'group2') } - let(:group_tree_restorer) { described_class.new(user: user, shared: shared, group: group, group_hash: nil) } - let(:restored_group_json) { group_tree_restorer.restore } - - it 'does not read a symlink' do - Dir.mktmpdir do |tmpdir| - setup_symlink(tmpdir, 'group.json') - allow(shared).to receive(:export_path).and_call_original - - expect(group_tree_restorer.restore).to eq(false) - expect(shared.errors).to include('Incorrect JSON format') - end - end - end - - context 'group visibility levels' do - let(:user) { create(:user) } - let(:shared) { Gitlab::ImportExport::Shared.new(group) } - let(:group_tree_restorer) { described_class.new(user: user, shared: shared, group: group, group_hash: nil) } - - before do - setup_import_export_config(filepath) - - group_tree_restorer.restore - end - - shared_examples 'with visibility level' do |visibility_level, expected_visibilities| - context "when visibility level is #{visibility_level}" do - let(:group) { create(:group, visibility_level) } - let(:filepath) { "group_exports/visibility_levels/#{visibility_level}" } - - it "imports all subgroups as #{visibility_level}" do - expect(group.children.map(&:visibility_level)).to eq(expected_visibilities) - end - end - end - - include_examples 'with visibility level', :public, [20, 10, 0] - include_examples 'with visibility level', :private, [0, 0, 0] - include_examples 'with visibility level', :internal, [10, 10, 0] - end -end diff --git a/spec/lib/gitlab/import_export/group_tree_saver_spec.rb b/spec/lib/gitlab/import_export/group_tree_saver_spec.rb deleted file mode 100644 index 7f49c7af8fa..00000000000 --- a/spec/lib/gitlab/import_export/group_tree_saver_spec.rb +++ /dev/null @@ -1,202 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -describe Gitlab::ImportExport::GroupTreeSaver do - describe 'saves the group tree into a json object' do - let(:shared) { Gitlab::ImportExport::Shared.new(group) } - let(:group_tree_saver) { described_class.new(group: group, current_user: user, shared: shared) } - let(:export_path) { "#{Dir.tmpdir}/group_tree_saver_spec" } - let(:user) { create(:user) } - let!(:group) { setup_group } - - before do - group.add_maintainer(user) - allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) - end - - after do - FileUtils.rm_rf(export_path) - end - - it 'saves group successfully' do - expect(group_tree_saver.save).to be true - end - - context ':export_fast_serialize feature flag checks' do - before do - expect(Gitlab::ImportExport::Reader).to receive(:new).with(shared: shared, config: group_config).and_return(reader) - expect(reader).to receive(:group_tree).and_return(group_tree) - end - - let(:reader) { instance_double('Gitlab::ImportExport::Reader') } - let(:group_config) { Gitlab::ImportExport::Config.new(config: Gitlab::ImportExport.group_config_file).to_h } - let(:group_tree) do - { - include: [{ milestones: { include: [] } }], - preload: { milestones: nil } - } - end - - context 'when :export_fast_serialize feature is enabled' do - let(:serializer) { instance_double(Gitlab::ImportExport::FastHashSerializer) } - - before do - stub_feature_flags(export_fast_serialize: true) - - expect(Gitlab::ImportExport::FastHashSerializer).to receive(:new).with(group, group_tree).and_return(serializer) - end - - it 'uses FastHashSerializer' do - expect(serializer).to receive(:execute) - - group_tree_saver.save - end - end - - context 'when :export_fast_serialize feature is disabled' do - before do - stub_feature_flags(export_fast_serialize: false) - end - - it 'is serialized via built-in `as_json`' do - expect(group).to receive(:as_json).with(group_tree).and_call_original - - group_tree_saver.save - end - end - end - - # It is mostly duplicated in - # `spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb` - # except: - # context 'with description override' do - # context 'group members' do - # ^ These are specific for the groupTreeSaver - context 'JSON' do - let(:saved_group_json) do - group_tree_saver.save - group_json(group_tree_saver.full_path) - end - - it 'saves the correct json' do - expect(saved_group_json).to include({ 'description' => 'description' }) - end - - it 'has milestones' do - expect(saved_group_json['milestones']).not_to be_empty - end - - it 'has labels' do - expect(saved_group_json['labels']).not_to be_empty - end - - it 'has boards' do - expect(saved_group_json['boards']).not_to be_empty - end - - it 'has board label list' do - expect(saved_group_json['boards'].first['lists']).not_to be_empty - end - - it 'has group members' do - expect(saved_group_json['members']).not_to be_empty - end - - it 'has priorities associated to labels' do - expect(saved_group_json['labels'].first['priorities']).not_to be_empty - end - - it 'has badges' do - expect(saved_group_json['badges']).not_to be_empty - end - - context 'group children' do - let(:children) { group.children } - - it 'exports group children' do - expect(saved_group_json['children'].length).to eq(children.count) - end - - it 'exports group children of children' do - expect(saved_group_json['children'].first['children'].length).to eq(children.first.children.count) - end - end - - context 'group members' do - let(:user2) { create(:user, email: 'group@member.com') } - let(:member_emails) do - saved_group_json['members'].map do |pm| - pm['user']['email'] - end - end - - before do - group.add_developer(user2) - end - - it 'exports group members as group owner' do - group.add_owner(user) - - expect(member_emails).to include('group@member.com') - end - - context 'as admin' do - let(:user) { create(:admin) } - - it 'exports group members as admin' do - expect(member_emails).to include('group@member.com') - end - - it 'exports group members' do - member_types = saved_group_json['members'].map { |pm| pm['source_type'] } - - expect(member_types).to all(eq('Namespace')) - end - end - end - - context 'group attributes' do - shared_examples 'excluded attributes' do - excluded_attributes = %w[ - id - owner_id - parent_id - created_at - updated_at - runners_token - runners_token_encrypted - saml_discovery_token - ] - - excluded_attributes.each do |excluded_attribute| - it 'does not contain excluded attribute' do - expect(saved_group_json).not_to include(excluded_attribute => group.public_send(excluded_attribute)) - end - end - end - - include_examples 'excluded attributes' - end - end - end - - def setup_group - group = create(:group, description: 'description') - sub_group = create(:group, description: 'description', parent: group) - create(:group, description: 'description', parent: sub_group) - create(:milestone, group: group) - create(:group_badge, group: group) - group_label = create(:group_label, group: group) - create(:label_priority, label: group_label, priority: 1) - board = create(:board, group: group) - create(:list, board: board, label: group_label) - create(:group_badge, group: group) - - group - end - - def group_json(filename) - JSON.parse(IO.read(filename)) - end -end diff --git a/spec/lib/gitlab/import_export/importer_spec.rb b/spec/lib/gitlab/import_export/importer_spec.rb index 942af4084e5..07857269004 100644 --- a/spec/lib/gitlab/import_export/importer_spec.rb +++ b/spec/lib/gitlab/import_export/importer_spec.rb @@ -63,7 +63,7 @@ describe Gitlab::ImportExport::Importer do end it 'restores the ProjectTree' do - expect(Gitlab::ImportExport::ProjectTreeRestorer).to receive(:new).and_call_original + expect(Gitlab::ImportExport::Project::TreeRestorer).to receive(:new).and_call_original importer.execute end diff --git a/spec/lib/gitlab/import_export/project/object_builder_spec.rb b/spec/lib/gitlab/import_export/project/object_builder_spec.rb new file mode 100644 index 00000000000..c9d1410400a --- /dev/null +++ b/spec/lib/gitlab/import_export/project/object_builder_spec.rb @@ -0,0 +1,153 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::ImportExport::Project::ObjectBuilder do + let!(:group) { create(:group, :private) } + let!(:subgroup) { create(:group, :private, parent: group) } + let!(:project) do + create(:project, :repository, + :builds_disabled, + :issues_disabled, + name: 'project', + path: 'project', + group: subgroup) + end + + let(:lru_cache) { subject.send(:lru_cache) } + let(:cache_key) { subject.send(:cache_key) } + + context 'request store is not active' do + subject do + described_class.new(Label, + 'title' => 'group label', + 'project' => project, + 'group' => project.group) + end + + it 'ignore cache initialize' do + expect(lru_cache).to be_nil + expect(cache_key).to be_nil + end + end + + context 'request store is active', :request_store do + subject do + described_class.new(Label, + 'title' => 'group label', + 'project' => project, + 'group' => project.group) + end + + it 'initialize cache in memory' do + expect(lru_cache).not_to be_nil + expect(cache_key).not_to be_nil + end + + it 'cache object when first time find the object' do + group_label = create(:group_label, name: 'group label', group: project.group) + + expect(subject).to receive(:find_object).and_call_original + expect { subject.find } + .to change { lru_cache[cache_key] } + .from(nil).to(group_label) + + expect(subject.find).to eq(group_label) + end + + it 'read from cache when object has been cached' do + group_label = create(:group_label, name: 'group label', group: project.group) + + subject.find + + expect(subject).not_to receive(:find_object) + expect { subject.find }.not_to change { lru_cache[cache_key] } + + expect(subject.find).to eq(group_label) + end + end + + context 'labels' do + it 'finds the existing group label' do + group_label = create(:group_label, name: 'group label', group: project.group) + + expect(described_class.build(Label, + 'title' => 'group label', + 'project' => project, + 'group' => project.group)).to eq(group_label) + end + + it 'finds the existing group label in root ancestor' do + group_label = create(:group_label, name: 'group label', group: group) + + expect(described_class.build(Label, + 'title' => 'group label', + 'project' => project, + 'group' => group)).to eq(group_label) + end + + it 'creates a new label' do + label = described_class.build(Label, + 'title' => 'group label', + 'project' => project, + 'group' => project.group) + + expect(label.persisted?).to be true + end + end + + context 'milestones' do + it 'finds the existing group milestone' do + milestone = create(:milestone, name: 'group milestone', group: project.group) + + expect(described_class.build(Milestone, + 'title' => 'group milestone', + 'project' => project, + 'group' => project.group)).to eq(milestone) + end + + it 'finds the existing group milestone in root ancestor' do + milestone = create(:milestone, name: 'group milestone', group: group) + + expect(described_class.build(Milestone, + 'title' => 'group milestone', + 'project' => project, + 'group' => group)).to eq(milestone) + end + + it 'creates a new milestone' do + milestone = described_class.build(Milestone, + 'title' => 'group milestone', + 'project' => project, + 'group' => project.group) + + expect(milestone.persisted?).to be true + end + end + + context 'merge_request' do + it 'finds the existing merge_request' do + merge_request = create(:merge_request, title: 'MergeRequest', iid: 7, target_project: project, source_project: project) + expect(described_class.build(MergeRequest, + 'title' => 'MergeRequest', + 'source_project_id' => project.id, + 'target_project_id' => project.id, + 'source_branch' => 'SourceBranch', + 'iid' => 7, + 'target_branch' => 'TargetBranch', + 'author_id' => project.creator.id)).to eq(merge_request) + end + + it 'creates a new merge_request' do + merge_request = described_class.build(MergeRequest, + 'title' => 'MergeRequest', + 'iid' => 8, + 'source_project_id' => project.id, + 'target_project_id' => project.id, + 'source_branch' => 'SourceBranch', + 'target_branch' => 'TargetBranch', + 'author_id' => project.creator.id) + expect(merge_request.persisted?).to be true + end + end +end diff --git a/spec/lib/gitlab/import_export/project/relation_factory_spec.rb b/spec/lib/gitlab/import_export/project/relation_factory_spec.rb new file mode 100644 index 00000000000..73ae6810706 --- /dev/null +++ b/spec/lib/gitlab/import_export/project/relation_factory_spec.rb @@ -0,0 +1,326 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::ImportExport::Project::RelationFactory do + let(:group) { create(:group) } + let(:project) { create(:project, :repository, group: group) } + let(:members_mapper) { double('members_mapper').as_null_object } + let(:user) { create(:admin) } + let(:excluded_keys) { [] } + let(:created_object) do + described_class.create(relation_sym: relation_sym, + relation_hash: relation_hash, + object_builder: Gitlab::ImportExport::Project::ObjectBuilder, + members_mapper: members_mapper, + user: user, + importable: project, + excluded_keys: excluded_keys) + end + + context 'hook object' do + let(:relation_sym) { :hooks } + let(:id) { 999 } + let(:service_id) { 99 } + let(:original_project_id) { 8 } + let(:token) { 'secret' } + + let(:relation_hash) do + { + 'id' => id, + 'url' => 'https://example.json', + 'project_id' => original_project_id, + 'created_at' => '2016-08-12T09:41:03.462Z', + 'updated_at' => '2016-08-12T09:41:03.462Z', + 'service_id' => service_id, + 'push_events' => true, + 'issues_events' => false, + 'confidential_issues_events' => false, + 'merge_requests_events' => true, + 'tag_push_events' => false, + 'note_events' => true, + 'enable_ssl_verification' => true, + 'job_events' => false, + 'wiki_page_events' => true, + 'token' => token + } + end + + it 'does not have the original ID' do + expect(created_object.id).not_to eq(id) + end + + it 'does not have the original service_id' do + expect(created_object.service_id).not_to eq(service_id) + end + + it 'does not have the original project_id' do + expect(created_object.project_id).not_to eq(original_project_id) + end + + it 'has the new project_id' do + expect(created_object.project_id).to eql(project.id) + end + + it 'has a nil token' do + expect(created_object.token).to eq(nil) + end + + context 'original service exists' do + let(:service_id) { create(:service, project: project).id } + + it 'does not have the original service_id' do + expect(created_object.service_id).not_to eq(service_id) + end + end + + context 'excluded attributes' do + let(:excluded_keys) { %w[url] } + + it 'are removed from the imported object' do + expect(created_object.url).to be_nil + end + end + end + + # Mocks an ActiveRecordish object with the dodgy columns + class FooModel + include ActiveModel::Model + + def initialize(params = {}) + params.each { |key, value| send("#{key}=", value) } + end + + def values + instance_variables.map { |ivar| instance_variable_get(ivar) } + end + end + + context 'merge_request object' do + let(:relation_sym) { :merge_requests } + + let(:exported_member) do + { + "id" => 111, + "access_level" => 30, + "source_id" => 1, + "source_type" => "Project", + "user_id" => 3, + "notification_level" => 3, + "created_at" => "2016-11-18T09:29:42.634Z", + "updated_at" => "2016-11-18T09:29:42.634Z", + "user" => { + "id" => user.id, + "email" => user.email, + "username" => user.username + } + } + end + + let(:members_mapper) do + Gitlab::ImportExport::MembersMapper.new( + exported_members: [exported_member], + user: user, + importable: project) + end + + let(:relation_hash) do + { + 'id' => 27, + 'target_branch' => "feature", + 'source_branch' => "feature_conflict", + 'source_project_id' => project.id, + 'target_project_id' => project.id, + 'author_id' => user.id, + 'assignee_id' => user.id, + 'updated_by_id' => user.id, + 'title' => "MR1", + 'created_at' => "2016-06-14T15:02:36.568Z", + 'updated_at' => "2016-06-14T15:02:56.815Z", + 'state' => "opened", + 'merge_status' => "unchecked", + 'description' => "Description", + 'position' => 0, + 'source_branch_sha' => "ABCD", + 'target_branch_sha' => "DCBA", + 'merge_when_pipeline_succeeds' => true + } + end + + it 'has preloaded author' do + expect(created_object.author).to equal(user) + end + + it 'has preloaded updated_by' do + expect(created_object.updated_by).to equal(user) + end + + it 'has preloaded source project' do + expect(created_object.source_project).to equal(project) + end + + it 'has preloaded target project' do + expect(created_object.source_project).to equal(project) + end + end + + context 'label object' do + let(:relation_sym) { :labels } + let(:relation_hash) do + { + "id": 3, + "title": "test3", + "color": "#428bca", + "group_id": project.group.id, + "created_at": "2016-07-22T08:55:44.161Z", + "updated_at": "2016-07-22T08:55:44.161Z", + "template": false, + "description": "", + "project_id": project.id, + "type": "GroupLabel" + } + end + + it 'has preloaded project' do + expect(created_object.project).to equal(project) + end + + it 'has preloaded group' do + expect(created_object.group).to equal(project.group) + end + end + + # `project_id`, `described_class.USER_REFERENCES`, noteable_id, target_id, and some project IDs are already + # re-assigned by described_class. + context 'Potentially hazardous foreign keys' do + let(:relation_sym) { :hazardous_foo_model } + let(:relation_hash) do + { + 'service_id' => 99, + 'moved_to_id' => 99, + 'namespace_id' => 99, + 'ci_id' => 99, + 'random_project_id' => 99, + 'random_id' => 99, + 'milestone_id' => 99, + 'project_id' => 99, + 'user_id' => 99 + } + end + + class HazardousFooModel < FooModel + attr_accessor :service_id, :moved_to_id, :namespace_id, :ci_id, :random_project_id, :random_id, :milestone_id, :project_id + end + + before do + allow(HazardousFooModel).to receive(:reflect_on_association).and_return(nil) + end + + it 'does not preserve any foreign key IDs' do + expect(created_object.values).not_to include(99) + end + end + + context 'overrided model with pluralized name' do + let(:relation_sym) { :metrics } + + let(:relation_hash) do + { + 'id' => 99, + 'merge_request_id' => 99, + 'merged_at' => Time.now, + 'merged_by_id' => 99, + 'latest_closed_at' => nil, + 'latest_closed_by_id' => nil + } + end + + it 'does not raise errors' do + expect { created_object }.not_to raise_error + end + end + + context 'Project references' do + let(:relation_sym) { :project_foo_model } + let(:relation_hash) do + Gitlab::ImportExport::Project::RelationFactory::PROJECT_REFERENCES.map { |ref| { ref => 99 } }.inject(:merge) + end + + class ProjectFooModel < FooModel + attr_accessor(*Gitlab::ImportExport::Project::RelationFactory::PROJECT_REFERENCES) + end + + before do + allow(ProjectFooModel).to receive(:reflect_on_association).and_return(nil) + end + + it 'does not preserve any project foreign key IDs' do + expect(created_object.values).not_to include(99) + end + end + + context 'Notes user references' do + let(:relation_sym) { :notes } + let(:new_user) { create(:user) } + let(:exported_member) do + { + "id" => 111, + "access_level" => 30, + "source_id" => 1, + "source_type" => "Project", + "user_id" => 3, + "notification_level" => 3, + "created_at" => "2016-11-18T09:29:42.634Z", + "updated_at" => "2016-11-18T09:29:42.634Z", + "user" => { + "id" => 999, + "email" => new_user.email, + "username" => new_user.username + } + } + end + + let(:relation_hash) do + { + "id" => 4947, + "note" => "merged", + "noteable_type" => "MergeRequest", + "author_id" => 999, + "created_at" => "2016-11-18T09:29:42.634Z", + "updated_at" => "2016-11-18T09:29:42.634Z", + "project_id" => 1, + "attachment" => { + "url" => nil + }, + "noteable_id" => 377, + "system" => true, + "author" => { + "name" => "Administrator" + }, + "events" => [] + } + end + + let(:members_mapper) do + Gitlab::ImportExport::MembersMapper.new( + exported_members: [exported_member], + user: user, + importable: project) + end + + it 'maps the right author to the imported note' do + expect(created_object.author).to eq(new_user) + end + end + + context 'encrypted attributes' do + let(:relation_sym) { 'Ci::Variable' } + let(:relation_hash) do + create(:ci_variable).as_json + end + + it 'has no value for the encrypted attribute' do + expect(created_object.value).to be_nil + end + end +end diff --git a/spec/lib/gitlab/import_export/project/tree_loader_spec.rb b/spec/lib/gitlab/import_export/project/tree_loader_spec.rb new file mode 100644 index 00000000000..e683eefa7c0 --- /dev/null +++ b/spec/lib/gitlab/import_export/project/tree_loader_spec.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::ImportExport::Project::TreeLoader do + let(:fixture) { 'spec/fixtures/lib/gitlab/import_export/with_duplicates.json' } + let(:project_tree) { JSON.parse(File.read(fixture)) } + + context 'without de-duplicating entries' do + let(:parsed_tree) do + subject.load(fixture) + end + + it 'parses the JSON into the expected tree' do + expect(parsed_tree).to eq(project_tree) + end + + it 'does not de-duplicate entries' do + expect(parsed_tree['duped_hash_with_id']).not_to be(parsed_tree['array'][0]['duped_hash_with_id']) + end + end + + context 'with de-duplicating entries' do + let(:parsed_tree) do + subject.load(fixture, dedup_entries: true) + end + + it 'parses the JSON into the expected tree' do + expect(parsed_tree).to eq(project_tree) + end + + it 'de-duplicates equal values' do + expect(parsed_tree['duped_hash_with_id']).to be(parsed_tree['array'][0]['duped_hash_with_id']) + expect(parsed_tree['duped_hash_with_id']).to be(parsed_tree['nested']['duped_hash_with_id']) + expect(parsed_tree['duped_array']).to be(parsed_tree['array'][1]['duped_array']) + expect(parsed_tree['duped_array']).to be(parsed_tree['nested']['duped_array']) + end + + it 'does not de-duplicate hashes without IDs' do + expect(parsed_tree['duped_hash_no_id']).to eq(parsed_tree['array'][2]['duped_hash_no_id']) + expect(parsed_tree['duped_hash_no_id']).not_to be(parsed_tree['array'][2]['duped_hash_no_id']) + end + + it 'keeps single entries intact' do + expect(parsed_tree['simple']).to eq(42) + expect(parsed_tree['nested']['array']).to eq(["don't touch"]) + end + end +end diff --git a/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb new file mode 100644 index 00000000000..312bbb58a28 --- /dev/null +++ b/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb @@ -0,0 +1,843 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::ImportExport::Project::TreeRestorer do + include ImportExport::CommonUtil + + let(:shared) { project.import_export_shared } + + describe 'restore project tree' do + before(:context) do + # Using an admin for import, so we can check assignment of existing members + @user = create(:admin) + @existing_members = [ + create(:user, email: 'bernard_willms@gitlabexample.com'), + create(:user, email: 'saul_will@gitlabexample.com') + ] + + RSpec::Mocks.with_temporary_scope do + @project = create(:project, :builds_enabled, :issues_disabled, name: 'project', path: 'project') + @shared = @project.import_export_shared + + setup_import_export_config('complex') + + allow_any_instance_of(Repository).to receive(:fetch_source_branch!).and_return(true) + allow_any_instance_of(Gitlab::Git::Repository).to receive(:branch_exists?).and_return(false) + + expect_any_instance_of(Gitlab::Git::Repository).to receive(:create_branch).with('feature', 'DCBA') + allow_any_instance_of(Gitlab::Git::Repository).to receive(:create_branch) + + project_tree_restorer = described_class.new(user: @user, shared: @shared, project: @project) + + @restored_project_json = project_tree_restorer.restore + end + end + + context 'JSON' do + it 'restores models based on JSON' do + expect(@restored_project_json).to be_truthy + end + + it 'restore correct project features' do + project = Project.find_by_path('project') + + expect(project.project_feature.issues_access_level).to eq(ProjectFeature::PRIVATE) + expect(project.project_feature.builds_access_level).to eq(ProjectFeature::PRIVATE) + expect(project.project_feature.snippets_access_level).to eq(ProjectFeature::PRIVATE) + expect(project.project_feature.wiki_access_level).to eq(ProjectFeature::PRIVATE) + expect(project.project_feature.merge_requests_access_level).to eq(ProjectFeature::PRIVATE) + end + + it 'has the project description' do + expect(Project.find_by_path('project').description).to eq('Nisi et repellendus ut enim quo accusamus vel magnam.') + end + + it 'has the same label associated to two issues' do + expect(ProjectLabel.find_by_title('test2').issues.count).to eq(2) + end + + it 'has milestones associated to two separate issues' do + expect(Milestone.find_by_description('test milestone').issues.count).to eq(2) + end + + context 'when importing a project with cached_markdown_version and note_html' do + context 'for an Issue' do + it 'does not import note_html' do + note_content = 'Quo reprehenderit aliquam qui dicta impedit cupiditate eligendi' + issue_note = Issue.find_by(description: 'Aliquam enim illo et possimus.').notes.select { |n| n.note.match(/#{note_content}/)}.first + + expect(issue_note.note_html).to match(/#{note_content}/) + end + end + + context 'for a Merge Request' do + it 'does not import note_html' do + note_content = 'Sit voluptatibus eveniet architecto quidem' + merge_request_note = MergeRequest.find_by(title: 'MR1').notes.select { |n| n.note.match(/#{note_content}/)}.first + + expect(merge_request_note.note_html).to match(/#{note_content}/) + end + end + end + + it 'creates a valid pipeline note' do + expect(Ci::Pipeline.find_by_sha('sha-notes').notes).not_to be_empty + end + + it 'pipeline has the correct user ID' do + expect(Ci::Pipeline.find_by_sha('sha-notes').user_id).to eq(@user.id) + end + + it 'restores pipelines with missing ref' do + expect(Ci::Pipeline.where(ref: nil)).not_to be_empty + end + + it 'restores pipeline for merge request' do + pipeline = Ci::Pipeline.find_by_sha('048721d90c449b244b7b4c53a9186b04330174ec') + + expect(pipeline).to be_valid + expect(pipeline.tag).to be_falsey + expect(pipeline.source).to eq('merge_request_event') + expect(pipeline.merge_request.id).to be > 0 + expect(pipeline.merge_request.target_branch).to eq('feature') + expect(pipeline.merge_request.source_branch).to eq('feature_conflict') + end + + it 'preserves updated_at on issues' do + issue = Issue.where(description: 'Aliquam enim illo et possimus.').first + + expect(issue.reload.updated_at.to_s).to eq('2016-06-14 15:02:47 UTC') + end + + it 'has multiple issue assignees' do + expect(Issue.find_by(title: 'Voluptatem').assignees).to contain_exactly(@user, *@existing_members) + expect(Issue.find_by(title: 'Issue without assignees').assignees).to be_empty + end + + it 'restores timelogs for issues' do + timelog = Issue.find_by(title: 'issue_with_timelogs').timelogs.last + + aggregate_failures do + expect(timelog.time_spent).to eq(72000) + expect(timelog.spent_at).to eq("2019-12-27T00:00:00.000Z") + end + end + + it 'contains the merge access levels on a protected branch' do + expect(ProtectedBranch.first.merge_access_levels).not_to be_empty + end + + it 'contains the push access levels on a protected branch' do + expect(ProtectedBranch.first.push_access_levels).not_to be_empty + end + + it 'contains the create access levels on a protected tag' do + expect(ProtectedTag.first.create_access_levels).not_to be_empty + end + + it 'restores issue resource label events' do + expect(Issue.find_by(title: 'Voluptatem').resource_label_events).not_to be_empty + end + + it 'restores merge requests resource label events' do + expect(MergeRequest.find_by(title: 'MR1').resource_label_events).not_to be_empty + end + + it 'restores suggestion' do + note = Note.find_by("note LIKE 'Saepe asperiores exercitationem non dignissimos laborum reiciendis et ipsum%'") + + expect(note.suggestions.count).to eq(1) + expect(note.suggestions.first.from_content).to eq("Original line\n") + end + + context 'event at forth level of the tree' do + let(:event) { Event.where(action: 6).first } + + it 'restores the event' do + expect(event).not_to be_nil + end + + it 'has the action' do + expect(event.action).not_to be_nil + end + + it 'event belongs to note, belongs to merge request, belongs to a project' do + expect(event.note.noteable.project).not_to be_nil + end + end + + it 'has the correct data for merge request diff files' do + expect(MergeRequestDiffFile.where.not(diff: nil).count).to eq(55) + end + + it 'has the correct data for merge request diff commits' do + expect(MergeRequestDiffCommit.count).to eq(77) + end + + it 'has the correct data for merge request latest_merge_request_diff' do + MergeRequest.find_each do |merge_request| + expect(merge_request.latest_merge_request_diff_id).to eq(merge_request.merge_request_diffs.maximum(:id)) + end + end + + it 'has labels associated to label links, associated to issues' do + expect(Label.first.label_links.first.target).not_to be_nil + end + + it 'has project labels' do + expect(ProjectLabel.count).to eq(3) + end + + it 'has no group labels' do + expect(GroupLabel.count).to eq(0) + end + + it 'has issue boards' do + expect(Project.find_by_path('project').boards.count).to eq(1) + end + + it 'has lists associated with the issue board' do + expect(Project.find_by_path('project').boards.find_by_name('TestBoardABC').lists.count).to eq(3) + end + + it 'has a project feature' do + expect(@project.project_feature).not_to be_nil + end + + it 'has custom attributes' do + expect(@project.custom_attributes.count).to eq(2) + end + + it 'has badges' do + expect(@project.project_badges.count).to eq(2) + end + + it 'has snippets' do + expect(@project.snippets.count).to eq(1) + end + + it 'has award emoji for a snippet' do + award_emoji = @project.snippets.first.award_emoji + + expect(award_emoji.map(&:name)).to contain_exactly('thumbsup', 'coffee') + end + + it 'snippet has notes' do + expect(@project.snippets.first.notes.count).to eq(1) + end + + it 'snippet has award emojis on notes' do + award_emoji = @project.snippets.first.notes.first.award_emoji.first + + expect(award_emoji.name).to eq('thumbsup') + end + + it 'restores `ci_cd_settings` : `group_runners_enabled` setting' do + expect(@project.ci_cd_settings.group_runners_enabled?).to eq(false) + end + + it 'restores `auto_devops`' do + expect(@project.auto_devops_enabled?).to eq(true) + expect(@project.auto_devops.deploy_strategy).to eq('continuous') + end + + it 'restores the correct service' do + expect(CustomIssueTrackerService.first).not_to be_nil + end + + it 'restores zoom meetings' do + meetings = @project.issues.first.zoom_meetings + + expect(meetings.count).to eq(1) + expect(meetings.first.url).to eq('https://zoom.us/j/123456789') + end + + it 'restores sentry issues' do + sentry_issue = @project.issues.first.sentry_issue + + expect(sentry_issue.sentry_issue_identifier).to eq(1234567891) + end + + it 'has award emoji for an issue' do + award_emoji = @project.issues.first.award_emoji.first + + expect(award_emoji.name).to eq('musical_keyboard') + end + + it 'has award emoji for a note in an issue' do + award_emoji = @project.issues.first.notes.first.award_emoji.first + + expect(award_emoji.name).to eq('clapper') + end + + it 'restores container_expiration_policy' do + policy = Project.find_by_path('project').container_expiration_policy + + aggregate_failures do + expect(policy).to be_an_instance_of(ContainerExpirationPolicy) + expect(policy).to be_persisted + expect(policy.cadence).to eq('3month') + end + end + + it 'restores error_tracking_setting' do + setting = @project.error_tracking_setting + + aggregate_failures do + expect(setting.api_url).to eq("https://gitlab.example.com/api/0/projects/sentry-org/sentry-project") + expect(setting.project_name).to eq("Sentry Project") + expect(setting.organization_name).to eq("Sentry Org") + end + end + + it 'restores external pull requests' do + external_pr = @project.external_pull_requests.last + + aggregate_failures do + expect(external_pr.pull_request_iid).to eq(4) + expect(external_pr.source_branch).to eq("feature") + expect(external_pr.target_branch).to eq("master") + expect(external_pr.status).to eq("open") + end + end + + it 'restores pipeline schedules' do + pipeline_schedule = @project.pipeline_schedules.last + + aggregate_failures do + expect(pipeline_schedule.description).to eq('Schedule Description') + expect(pipeline_schedule.ref).to eq('master') + expect(pipeline_schedule.cron).to eq('0 4 * * 0') + expect(pipeline_schedule.cron_timezone).to eq('UTC') + expect(pipeline_schedule.active).to eq(true) + end + end + + it 'restores releases with links' do + release = @project.releases.last + link = release.links.last + + aggregate_failures do + expect(release.tag).to eq('release-1.1') + expect(release.description).to eq('Some release notes') + expect(release.name).to eq('release-1.1') + expect(release.sha).to eq('901de3a8bd5573f4a049b1457d28bc1592ba6bf9') + expect(release.released_at).to eq('2019-12-26T10:17:14.615Z') + + expect(link.url).to eq('http://localhost/namespace6/project6/-/jobs/140463678/artifacts/download') + expect(link.name).to eq('release-1.1.dmg') + end + end + + context 'Merge requests' do + it 'always has the new project as a target' do + expect(MergeRequest.find_by_title('MR1').target_project).to eq(@project) + end + + it 'has the same source project as originally if source/target are the same' do + expect(MergeRequest.find_by_title('MR1').source_project).to eq(@project) + end + + it 'has the new project as target if source/target differ' do + expect(MergeRequest.find_by_title('MR2').target_project).to eq(@project) + end + + it 'has no source if source/target differ' do + expect(MergeRequest.find_by_title('MR2').source_project_id).to be_nil + end + + it 'has award emoji' do + award_emoji = MergeRequest.find_by_title('MR1').award_emoji + + expect(award_emoji.map(&:name)).to contain_exactly('thumbsup', 'drum') + end + + context 'notes' do + it 'has award emoji' do + award_emoji = MergeRequest.find_by_title('MR1').notes.first.award_emoji.first + + expect(award_emoji.name).to eq('tada') + end + end + end + + context 'tokens are regenerated' do + it 'has new CI trigger tokens' do + expect(Ci::Trigger.where(token: %w[cdbfasdf44a5958c83654733449e585 33a66349b5ad01fc00174af87804e40])) + .to be_empty + end + + it 'has a new CI build token' do + expect(Ci::Build.where(token: 'abcd')).to be_empty + end + end + + context 'has restored the correct number of records' do + it 'has the correct number of merge requests' do + expect(@project.merge_requests.size).to eq(9) + end + + it 'only restores valid triggers' do + expect(@project.triggers.size).to eq(1) + end + + it 'has the correct number of pipelines and statuses' do + expect(@project.ci_pipelines.size).to eq(7) + + @project.ci_pipelines.order(:id).zip([2, 2, 2, 2, 2, 0, 0]) + .each do |(pipeline, expected_status_size)| + expect(pipeline.statuses.size).to eq(expected_status_size) + end + end + end + + context 'when restoring hierarchy of pipeline, stages and jobs' do + it 'restores pipelines' do + expect(Ci::Pipeline.all.count).to be 7 + end + + it 'restores pipeline stages' do + expect(Ci::Stage.all.count).to be 6 + end + + it 'correctly restores association between stage and a pipeline' do + expect(Ci::Stage.all).to all(have_attributes(pipeline_id: a_value > 0)) + end + + it 'restores statuses' do + expect(CommitStatus.all.count).to be 10 + end + + it 'correctly restores association between a stage and a job' do + expect(CommitStatus.all).to all(have_attributes(stage_id: a_value > 0)) + end + + it 'correctly restores association between a pipeline and a job' do + expect(CommitStatus.all).to all(have_attributes(pipeline_id: a_value > 0)) + end + + it 'restores a Hash for CommitStatus options' do + expect(CommitStatus.all.map(&:options).compact).to all(be_a(Hash)) + end + + it 'restores external pull request for the restored pipeline' do + pipeline_with_external_pr = @project.ci_pipelines.order(:id).last + + expect(pipeline_with_external_pr.external_pull_request).to be_persisted + end + end + end + end + + shared_examples 'restores group correctly' do |**results| + it 'has group label' do + expect(project.group.labels.size).to eq(results.fetch(:labels, 0)) + expect(project.group.labels.where(type: "GroupLabel").where.not(project_id: nil).count).to eq(0) + end + + it 'has group milestone' do + expect(project.group.milestones.size).to eq(results.fetch(:milestones, 0)) + end + + it 'has the correct visibility level' do + # INTERNAL in the `project.json`, group's is PRIVATE + expect(project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) + end + end + + context 'project.json file access check' do + let(:user) { create(:user) } + let!(:project) { create(:project, :builds_disabled, :issues_disabled, name: 'project', path: 'project') } + let(:project_tree_restorer) do + described_class.new(user: user, shared: shared, project: project) + end + let(:restored_project_json) { project_tree_restorer.restore } + + it 'does not read a symlink' do + Dir.mktmpdir do |tmpdir| + setup_symlink(tmpdir, 'project.json') + allow(shared).to receive(:export_path).and_call_original + + expect(project_tree_restorer.restore).to eq(false) + expect(shared.errors).to include('Incorrect JSON format') + end + end + end + + context 'Light JSON' do + let(:user) { create(:user) } + let!(:project) { create(:project, :builds_disabled, :issues_disabled, name: 'project', path: 'project') } + let(:project_tree_restorer) { described_class.new(user: user, shared: shared, project: project) } + let(:restored_project_json) { project_tree_restorer.restore } + + context 'with a simple project' do + before do + setup_import_export_config('light') + expect(restored_project_json).to eq(true) + end + + it_behaves_like 'restores project successfully', + issues: 1, + labels: 2, + label_with_priorities: 'A project label', + milestones: 1, + first_issue_labels: 1, + services: 1 + + context 'when there is an existing build with build token' do + before do + create(:ci_build, token: 'abcd') + end + + it_behaves_like 'restores project successfully', + issues: 1, + labels: 2, + label_with_priorities: 'A project label', + milestones: 1, + first_issue_labels: 1 + end + end + + context 'when post import action throw non-retriable exception' do + let(:exception) { StandardError.new('post_import_error') } + + before do + setup_import_export_config('light') + expect(project) + .to receive(:merge_requests) + .and_raise(exception) + end + + it 'report post import error' do + expect(restored_project_json).to eq(false) + expect(shared.errors).to include('post_import_error') + end + end + + context 'when post import action throw retriable exception one time' do + let(:exception) { GRPC::DeadlineExceeded.new } + + before do + setup_import_export_config('light') + expect(project) + .to receive(:merge_requests) + .and_raise(exception) + expect(project) + .to receive(:merge_requests) + .and_call_original + expect(restored_project_json).to eq(true) + end + + it_behaves_like 'restores project successfully', + issues: 1, + labels: 2, + label_with_priorities: 'A project label', + milestones: 1, + first_issue_labels: 1, + services: 1, + import_failures: 1 + + it 'records the failures in the database' do + import_failure = ImportFailure.last + + expect(import_failure.project_id).to eq(project.id) + expect(import_failure.relation_key).to be_nil + expect(import_failure.relation_index).to be_nil + expect(import_failure.exception_class).to eq('GRPC::DeadlineExceeded') + expect(import_failure.exception_message).to be_present + expect(import_failure.correlation_id_value).not_to be_empty + expect(import_failure.created_at).to be_present + end + end + + context 'when the project has overridden params in import data' do + before do + setup_import_export_config('light') + end + + it 'handles string versions of visibility_level' do + # Project needs to be in a group for visibility level comparison + # to happen + group = create(:group) + project.group = group + + project.create_import_data(data: { override_params: { visibility_level: Gitlab::VisibilityLevel::INTERNAL.to_s } }) + + expect(restored_project_json).to eq(true) + expect(project.visibility_level).to eq(Gitlab::VisibilityLevel::INTERNAL) + end + + it 'overwrites the params stored in the JSON' do + project.create_import_data(data: { override_params: { description: "Overridden" } }) + + expect(restored_project_json).to eq(true) + expect(project.description).to eq("Overridden") + end + + it 'does not allow setting params that are excluded from import_export settings' do + project.create_import_data(data: { override_params: { lfs_enabled: true } }) + + expect(restored_project_json).to eq(true) + expect(project.lfs_enabled).to be_falsey + end + + it 'overrides project feature access levels' do + access_level_keys = project.project_feature.attributes.keys.select { |a| a =~ /_access_level/ } + + # `pages_access_level` is not included, since it is not available in the public API + # and has a dependency on project's visibility level + # see ProjectFeature model + access_level_keys.delete('pages_access_level') + + disabled_access_levels = Hash[access_level_keys.collect { |item| [item, 'disabled'] }] + + project.create_import_data(data: { override_params: disabled_access_levels }) + + expect(restored_project_json).to eq(true) + + aggregate_failures do + access_level_keys.each do |key| + expect(project.public_send(key)).to eq(ProjectFeature::DISABLED) + end + end + end + end + + context 'with a project that has a group' do + let!(:project) do + create(:project, + :builds_disabled, + :issues_disabled, + name: 'project', + path: 'project', + group: create(:group, visibility_level: Gitlab::VisibilityLevel::PRIVATE)) + end + + before do + setup_import_export_config('group') + expect(restored_project_json).to eq(true) + end + + it_behaves_like 'restores project successfully', + issues: 3, + labels: 2, + label_with_priorities: 'A project label', + milestones: 2, + first_issue_labels: 1 + + it_behaves_like 'restores group correctly', + labels: 0, + milestones: 0, + first_issue_labels: 1 + + it 'restores issue states' do + expect(project.issues.with_state(:closed).count).to eq(1) + expect(project.issues.with_state(:opened).count).to eq(2) + end + end + + context 'with existing group models' do + let!(:project) do + create(:project, + :builds_disabled, + :issues_disabled, + name: 'project', + path: 'project', + group: create(:group)) + end + + before do + setup_import_export_config('light') + end + + it 'does not import any templated services' do + expect(restored_project_json).to eq(true) + + expect(project.services.where(template: true).count).to eq(0) + end + + it 'imports labels' do + create(:group_label, name: 'Another label', group: project.group) + + expect_any_instance_of(Gitlab::ImportExport::Shared).not_to receive(:error) + + expect(restored_project_json).to eq(true) + expect(project.labels.count).to eq(1) + end + + it 'imports milestones' do + create(:milestone, name: 'A milestone', group: project.group) + + expect_any_instance_of(Gitlab::ImportExport::Shared).not_to receive(:error) + + expect(restored_project_json).to eq(true) + expect(project.group.milestones.count).to eq(1) + expect(project.milestones.count).to eq(0) + end + end + + context 'with clashing milestones on IID' do + let!(:project) do + create(:project, + :builds_disabled, + :issues_disabled, + name: 'project', + path: 'project', + group: create(:group)) + end + + before do + setup_import_export_config('milestone-iid') + end + + it 'preserves the project milestone IID' do + expect_any_instance_of(Gitlab::ImportExport::Shared).not_to receive(:error) + + expect(restored_project_json).to eq(true) + expect(project.milestones.count).to eq(2) + expect(Milestone.find_by_title('Another milestone').iid).to eq(1) + expect(Milestone.find_by_title('Group-level milestone').iid).to eq(2) + end + end + + context 'with external authorization classification labels' do + before do + setup_import_export_config('light') + end + + it 'converts empty external classification authorization labels to nil' do + project.create_import_data(data: { override_params: { external_authorization_classification_label: "" } }) + + expect(restored_project_json).to eq(true) + expect(project.external_authorization_classification_label).to be_nil + end + + it 'preserves valid external classification authorization labels' do + project.create_import_data(data: { override_params: { external_authorization_classification_label: "foobar" } }) + + expect(restored_project_json).to eq(true) + expect(project.external_authorization_classification_label).to eq("foobar") + end + end + end + + context 'Minimal JSON' do + let(:project) { create(:project) } + let(:user) { create(:user) } + let(:tree_hash) { { 'visibility_level' => visibility } } + let(:restorer) do + described_class.new(user: user, shared: shared, project: project) + end + + before do + expect(restorer).to receive(:read_tree_hash) { tree_hash } + end + + context 'no group visibility' do + let(:visibility) { Gitlab::VisibilityLevel::PRIVATE } + + it 'uses the project visibility' do + expect(restorer.restore).to eq(true) + expect(restorer.project.visibility_level).to eq(visibility) + end + end + + context 'with restricted internal visibility' do + describe 'internal project' do + let(:visibility) { Gitlab::VisibilityLevel::INTERNAL } + + it 'uses private visibility' do + stub_application_setting(restricted_visibility_levels: [Gitlab::VisibilityLevel::INTERNAL]) + + expect(restorer.restore).to eq(true) + expect(restorer.project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) + end + end + end + + context 'with group visibility' do + before do + group = create(:group, visibility_level: group_visibility) + + project.update(group: group) + end + + context 'private group visibility' do + let(:group_visibility) { Gitlab::VisibilityLevel::PRIVATE } + let(:visibility) { Gitlab::VisibilityLevel::PUBLIC } + + it 'uses the group visibility' do + expect(restorer.restore).to eq(true) + expect(restorer.project.visibility_level).to eq(group_visibility) + end + end + + context 'public group visibility' do + let(:group_visibility) { Gitlab::VisibilityLevel::PUBLIC } + let(:visibility) { Gitlab::VisibilityLevel::PRIVATE } + + it 'uses the project visibility' do + expect(restorer.restore).to eq(true) + expect(restorer.project.visibility_level).to eq(visibility) + end + end + + context 'internal group visibility' do + let(:group_visibility) { Gitlab::VisibilityLevel::INTERNAL } + let(:visibility) { Gitlab::VisibilityLevel::PUBLIC } + + it 'uses the group visibility' do + expect(restorer.restore).to eq(true) + expect(restorer.project.visibility_level).to eq(group_visibility) + end + + context 'with restricted internal visibility' do + it 'sets private visibility' do + stub_application_setting(restricted_visibility_levels: [Gitlab::VisibilityLevel::INTERNAL]) + + expect(restorer.restore).to eq(true) + expect(restorer.project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) + end + end + end + end + end + + context 'JSON with invalid records' do + subject(:restored_project_json) { project_tree_restorer.restore } + + let(:user) { create(:user) } + let!(:project) { create(:project, :builds_disabled, :issues_disabled, name: 'project', path: 'project') } + let(:project_tree_restorer) { described_class.new(user: user, shared: shared, project: project) } + + before do + setup_import_export_config('with_invalid_records') + + subject + end + + context 'when failures occur because a relation fails to be processed' do + it_behaves_like 'restores project successfully', + issues: 0, + labels: 0, + label_with_priorities: nil, + milestones: 1, + first_issue_labels: 0, + services: 0, + import_failures: 1 + + it 'records the failures in the database' do + import_failure = ImportFailure.last + + expect(import_failure.project_id).to eq(project.id) + expect(import_failure.relation_key).to eq('milestones') + expect(import_failure.relation_index).to be_present + expect(import_failure.exception_class).to eq('ActiveRecord::RecordInvalid') + expect(import_failure.exception_message).to be_present + expect(import_failure.correlation_id_value).not_to be_empty + expect(import_failure.created_at).to be_present + end + end + end +end diff --git a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb new file mode 100644 index 00000000000..151fdf8810f --- /dev/null +++ b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb @@ -0,0 +1,397 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::ImportExport::Project::TreeSaver do + describe 'saves the project tree into a json object' do + let(:shared) { project.import_export_shared } + let(:project_tree_saver) { described_class.new(project: project, current_user: user, shared: shared) } + let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } + let(:user) { create(:user) } + let!(:project) { setup_project } + + before do + project.add_maintainer(user) + allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) + allow_any_instance_of(MergeRequest).to receive(:source_branch_sha).and_return('ABCD') + allow_any_instance_of(MergeRequest).to receive(:target_branch_sha).and_return('DCBA') + end + + after do + FileUtils.rm_rf(export_path) + end + + it 'saves project successfully' do + expect(project_tree_saver.save).to be true + end + + context ':export_fast_serialize feature flag checks' do + before do + expect(Gitlab::ImportExport::Reader).to receive(:new).with(shared: shared).and_return(reader) + expect(reader).to receive(:project_tree).and_return(project_tree) + end + + let(:serializer) { instance_double('Gitlab::ImportExport::FastHashSerializer') } + let(:reader) { instance_double('Gitlab::ImportExport::Reader') } + let(:project_tree) do + { + include: [{ issues: { include: [] } }], + preload: { issues: nil } + } + end + + context 'when :export_fast_serialize feature is enabled' do + before do + stub_feature_flags(export_fast_serialize: true) + end + + it 'uses FastHashSerializer' do + expect(Gitlab::ImportExport::FastHashSerializer) + .to receive(:new) + .with(project, project_tree) + .and_return(serializer) + + expect(serializer).to receive(:execute) + + project_tree_saver.save + end + end + + context 'when :export_fast_serialize feature is disabled' do + before do + stub_feature_flags(export_fast_serialize: false) + end + + it 'is serialized via built-in `as_json`' do + expect(project).to receive(:as_json).with(project_tree) + + project_tree_saver.save + end + end + end + + # It is mostly duplicated in + # `spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb` + # except: + # context 'with description override' do + # context 'group members' do + # ^ These are specific for the Project::TreeSaver + context 'JSON' do + let(:saved_project_json) do + project_tree_saver.save + project_json(project_tree_saver.full_path) + end + + # It is not duplicated in + # `spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb` + context 'with description override' do + let(:params) { { description: 'Foo Bar' } } + let(:project_tree_saver) { described_class.new(project: project, current_user: user, shared: shared, params: params) } + + it 'overrides the project description' do + expect(saved_project_json).to include({ 'description' => params[:description] }) + end + end + + it 'saves the correct json' do + expect(saved_project_json).to include({ 'description' => 'description', 'visibility_level' => 20 }) + end + + it 'has approvals_before_merge set' do + expect(saved_project_json['approvals_before_merge']).to eq(1) + end + + it 'has milestones' do + expect(saved_project_json['milestones']).not_to be_empty + end + + it 'has merge requests' do + expect(saved_project_json['merge_requests']).not_to be_empty + end + + it 'has merge request\'s milestones' do + expect(saved_project_json['merge_requests'].first['milestone']).not_to be_empty + end + + it 'has merge request\'s source branch SHA' do + expect(saved_project_json['merge_requests'].first['source_branch_sha']).to eq('ABCD') + end + + it 'has merge request\'s target branch SHA' do + expect(saved_project_json['merge_requests'].first['target_branch_sha']).to eq('DCBA') + end + + it 'has events' do + expect(saved_project_json['merge_requests'].first['milestone']['events']).not_to be_empty + end + + it 'has snippets' do + expect(saved_project_json['snippets']).not_to be_empty + end + + it 'has snippet notes' do + expect(saved_project_json['snippets'].first['notes']).not_to be_empty + end + + it 'has releases' do + expect(saved_project_json['releases']).not_to be_empty + end + + it 'has no author on releases' do + expect(saved_project_json['releases'].first['author']).to be_nil + end + + it 'has the author ID on releases' do + expect(saved_project_json['releases'].first['author_id']).not_to be_nil + end + + it 'has issues' do + expect(saved_project_json['issues']).not_to be_empty + end + + it 'has issue comments' do + notes = saved_project_json['issues'].first['notes'] + + expect(notes).not_to be_empty + expect(notes.first['type']).to eq('DiscussionNote') + end + + it 'has issue assignees' do + expect(saved_project_json['issues'].first['issue_assignees']).not_to be_empty + end + + it 'has author on issue comments' do + expect(saved_project_json['issues'].first['notes'].first['author']).not_to be_empty + end + + it 'has project members' do + expect(saved_project_json['project_members']).not_to be_empty + end + + it 'has merge requests diffs' do + expect(saved_project_json['merge_requests'].first['merge_request_diff']).not_to be_empty + end + + it 'has merge request diff files' do + expect(saved_project_json['merge_requests'].first['merge_request_diff']['merge_request_diff_files']).not_to be_empty + end + + it 'has merge request diff commits' do + expect(saved_project_json['merge_requests'].first['merge_request_diff']['merge_request_diff_commits']).not_to be_empty + end + + it 'has merge requests comments' do + expect(saved_project_json['merge_requests'].first['notes']).not_to be_empty + end + + it 'has author on merge requests comments' do + expect(saved_project_json['merge_requests'].first['notes'].first['author']).not_to be_empty + end + + it 'has pipeline stages' do + expect(saved_project_json.dig('ci_pipelines', 0, 'stages')).not_to be_empty + end + + it 'has pipeline statuses' do + expect(saved_project_json.dig('ci_pipelines', 0, 'stages', 0, 'statuses')).not_to be_empty + end + + it 'has pipeline builds' do + builds_count = saved_project_json + .dig('ci_pipelines', 0, 'stages', 0, 'statuses') + .count { |hash| hash['type'] == 'Ci::Build' } + + expect(builds_count).to eq(1) + end + + it 'has no when YML attributes but only the DB column' do + expect_any_instance_of(Gitlab::Ci::YamlProcessor).not_to receive(:build_attributes) + + saved_project_json + end + + it 'has pipeline commits' do + expect(saved_project_json['ci_pipelines']).not_to be_empty + end + + it 'has ci pipeline notes' do + expect(saved_project_json['ci_pipelines'].first['notes']).not_to be_empty + end + + it 'has labels with no associations' do + expect(saved_project_json['labels']).not_to be_empty + end + + it 'has labels associated to records' do + expect(saved_project_json['issues'].first['label_links'].first['label']).not_to be_empty + end + + it 'has project and group labels' do + label_types = saved_project_json['issues'].first['label_links'].map { |link| link['label']['type'] } + + expect(label_types).to match_array(%w(ProjectLabel GroupLabel)) + end + + it 'has priorities associated to labels' do + priorities = saved_project_json['issues'].first['label_links'].flat_map { |link| link['label']['priorities'] } + + expect(priorities).not_to be_empty + end + + it 'has issue resource label events' do + expect(saved_project_json['issues'].first['resource_label_events']).not_to be_empty + end + + it 'has merge request resource label events' do + expect(saved_project_json['merge_requests'].first['resource_label_events']).not_to be_empty + end + + it 'saves the correct service type' do + expect(saved_project_json['services'].first['type']).to eq('CustomIssueTrackerService') + end + + it 'saves the properties for a service' do + expect(saved_project_json['services'].first['properties']).to eq('one' => 'value') + end + + it 'has project feature' do + project_feature = saved_project_json['project_feature'] + expect(project_feature).not_to be_empty + expect(project_feature["issues_access_level"]).to eq(ProjectFeature::DISABLED) + expect(project_feature["wiki_access_level"]).to eq(ProjectFeature::ENABLED) + expect(project_feature["builds_access_level"]).to eq(ProjectFeature::PRIVATE) + end + + it 'has custom attributes' do + expect(saved_project_json['custom_attributes'].count).to eq(2) + end + + it 'has badges' do + expect(saved_project_json['project_badges'].count).to eq(2) + end + + it 'does not complain about non UTF-8 characters in MR diff files' do + ActiveRecord::Base.connection.execute("UPDATE merge_request_diff_files SET diff = '---\n- :diff: !binary |-\n LS0tIC9kZXYvbnVsbAorKysgYi9pbWFnZXMvbnVjb3IucGRmCkBAIC0wLDAg\n KzEsMTY3OSBAQAorJVBERi0xLjUNJeLjz9MNCisxIDAgb2JqDTw8L01ldGFk\n YXR'") + + expect(project_tree_saver.save).to be true + end + + context 'group members' do + let(:user2) { create(:user, email: 'group@member.com') } + let(:member_emails) do + saved_project_json['project_members'].map do |pm| + pm['user']['email'] + end + end + + before do + Group.first.add_developer(user2) + end + + it 'does not export group members if it has no permission' do + Group.first.add_developer(user) + + expect(member_emails).not_to include('group@member.com') + end + + it 'does not export group members as maintainer' do + Group.first.add_maintainer(user) + + expect(member_emails).not_to include('group@member.com') + end + + it 'exports group members as group owner' do + Group.first.add_owner(user) + + expect(member_emails).to include('group@member.com') + end + + context 'as admin' do + let(:user) { create(:admin) } + + it 'exports group members as admin' do + expect(member_emails).to include('group@member.com') + end + + it 'exports group members as project members' do + member_types = saved_project_json['project_members'].map { |pm| pm['source_type'] } + + expect(member_types).to all(eq('Project')) + end + end + end + + context 'project attributes' do + it 'does not contain the runners token' do + expect(saved_project_json).not_to include("runners_token" => 'token') + end + end + + it 'has a board and a list' do + expect(saved_project_json['boards'].first['lists']).not_to be_empty + end + end + end + + def setup_project + release = create(:release) + group = create(:group) + + project = create(:project, + :public, + :repository, + :issues_disabled, + :wiki_enabled, + :builds_private, + description: 'description', + releases: [release], + group: group, + approvals_before_merge: 1 + ) + allow(project).to receive(:commit).and_return(Commit.new(RepoHelpers.sample_commit, project)) + + issue = create(:issue, assignees: [user], project: project) + snippet = create(:project_snippet, project: project) + project_label = create(:label, project: project) + group_label = create(:group_label, group: group) + create(:label_link, label: project_label, target: issue) + create(:label_link, label: group_label, target: issue) + create(:label_priority, label: group_label, priority: 1) + milestone = create(:milestone, project: project) + merge_request = create(:merge_request, source_project: project, milestone: milestone) + + ci_build = create(:ci_build, project: project, when: nil) + ci_build.pipeline.update(project: project) + create(:commit_status, project: project, pipeline: ci_build.pipeline) + + create(:milestone, project: project) + create(:discussion_note, noteable: issue, project: project) + create(:note, noteable: merge_request, project: project) + create(:note, noteable: snippet, project: project) + create(:note_on_commit, + author: user, + project: project, + commit_id: ci_build.pipeline.sha) + + create(:resource_label_event, label: project_label, issue: issue) + create(:resource_label_event, label: group_label, merge_request: merge_request) + + create(:event, :created, target: milestone, project: project, author: user) + create(:service, project: project, type: 'CustomIssueTrackerService', category: 'issue_tracker', properties: { one: 'value' }) + + create(:project_custom_attribute, project: project) + create(:project_custom_attribute, project: project) + + create(:project_badge, project: project) + create(:project_badge, project: project) + + board = create(:board, project: project, name: 'TestBoard') + create(:list, board: board, position: 0, label: project_label) + + project + end + + def project_json(filename) + JSON.parse(IO.read(filename)) + end +end diff --git a/spec/lib/gitlab/import_export/project_relation_factory_spec.rb b/spec/lib/gitlab/import_export/project_relation_factory_spec.rb deleted file mode 100644 index d0e89b2e57b..00000000000 --- a/spec/lib/gitlab/import_export/project_relation_factory_spec.rb +++ /dev/null @@ -1,326 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -describe Gitlab::ImportExport::ProjectRelationFactory do - let(:group) { create(:group) } - let(:project) { create(:project, :repository, group: group) } - let(:members_mapper) { double('members_mapper').as_null_object } - let(:user) { create(:admin) } - let(:excluded_keys) { [] } - let(:created_object) do - described_class.create(relation_sym: relation_sym, - relation_hash: relation_hash, - object_builder: Gitlab::ImportExport::GroupProjectObjectBuilder, - members_mapper: members_mapper, - user: user, - importable: project, - excluded_keys: excluded_keys) - end - - context 'hook object' do - let(:relation_sym) { :hooks } - let(:id) { 999 } - let(:service_id) { 99 } - let(:original_project_id) { 8 } - let(:token) { 'secret' } - - let(:relation_hash) do - { - 'id' => id, - 'url' => 'https://example.json', - 'project_id' => original_project_id, - 'created_at' => '2016-08-12T09:41:03.462Z', - 'updated_at' => '2016-08-12T09:41:03.462Z', - 'service_id' => service_id, - 'push_events' => true, - 'issues_events' => false, - 'confidential_issues_events' => false, - 'merge_requests_events' => true, - 'tag_push_events' => false, - 'note_events' => true, - 'enable_ssl_verification' => true, - 'job_events' => false, - 'wiki_page_events' => true, - 'token' => token - } - end - - it 'does not have the original ID' do - expect(created_object.id).not_to eq(id) - end - - it 'does not have the original service_id' do - expect(created_object.service_id).not_to eq(service_id) - end - - it 'does not have the original project_id' do - expect(created_object.project_id).not_to eq(original_project_id) - end - - it 'has the new project_id' do - expect(created_object.project_id).to eql(project.id) - end - - it 'has a nil token' do - expect(created_object.token).to eq(nil) - end - - context 'original service exists' do - let(:service_id) { create(:service, project: project).id } - - it 'does not have the original service_id' do - expect(created_object.service_id).not_to eq(service_id) - end - end - - context 'excluded attributes' do - let(:excluded_keys) { %w[url] } - - it 'are removed from the imported object' do - expect(created_object.url).to be_nil - end - end - end - - # Mocks an ActiveRecordish object with the dodgy columns - class FooModel - include ActiveModel::Model - - def initialize(params = {}) - params.each { |key, value| send("#{key}=", value) } - end - - def values - instance_variables.map { |ivar| instance_variable_get(ivar) } - end - end - - context 'merge_request object' do - let(:relation_sym) { :merge_requests } - - let(:exported_member) do - { - "id" => 111, - "access_level" => 30, - "source_id" => 1, - "source_type" => "Project", - "user_id" => 3, - "notification_level" => 3, - "created_at" => "2016-11-18T09:29:42.634Z", - "updated_at" => "2016-11-18T09:29:42.634Z", - "user" => { - "id" => user.id, - "email" => user.email, - "username" => user.username - } - } - end - - let(:members_mapper) do - Gitlab::ImportExport::MembersMapper.new( - exported_members: [exported_member], - user: user, - importable: project) - end - - let(:relation_hash) do - { - 'id' => 27, - 'target_branch' => "feature", - 'source_branch' => "feature_conflict", - 'source_project_id' => project.id, - 'target_project_id' => project.id, - 'author_id' => user.id, - 'assignee_id' => user.id, - 'updated_by_id' => user.id, - 'title' => "MR1", - 'created_at' => "2016-06-14T15:02:36.568Z", - 'updated_at' => "2016-06-14T15:02:56.815Z", - 'state' => "opened", - 'merge_status' => "unchecked", - 'description' => "Description", - 'position' => 0, - 'source_branch_sha' => "ABCD", - 'target_branch_sha' => "DCBA", - 'merge_when_pipeline_succeeds' => true - } - end - - it 'has preloaded author' do - expect(created_object.author).to equal(user) - end - - it 'has preloaded updated_by' do - expect(created_object.updated_by).to equal(user) - end - - it 'has preloaded source project' do - expect(created_object.source_project).to equal(project) - end - - it 'has preloaded target project' do - expect(created_object.source_project).to equal(project) - end - end - - context 'label object' do - let(:relation_sym) { :labels } - let(:relation_hash) do - { - "id": 3, - "title": "test3", - "color": "#428bca", - "group_id": project.group.id, - "created_at": "2016-07-22T08:55:44.161Z", - "updated_at": "2016-07-22T08:55:44.161Z", - "template": false, - "description": "", - "project_id": project.id, - "type": "GroupLabel" - } - end - - it 'has preloaded project' do - expect(created_object.project).to equal(project) - end - - it 'has preloaded group' do - expect(created_object.group).to equal(project.group) - end - end - - # `project_id`, `described_class.USER_REFERENCES`, noteable_id, target_id, and some project IDs are already - # re-assigned by described_class. - context 'Potentially hazardous foreign keys' do - let(:relation_sym) { :hazardous_foo_model } - let(:relation_hash) do - { - 'service_id' => 99, - 'moved_to_id' => 99, - 'namespace_id' => 99, - 'ci_id' => 99, - 'random_project_id' => 99, - 'random_id' => 99, - 'milestone_id' => 99, - 'project_id' => 99, - 'user_id' => 99 - } - end - - class HazardousFooModel < FooModel - attr_accessor :service_id, :moved_to_id, :namespace_id, :ci_id, :random_project_id, :random_id, :milestone_id, :project_id - end - - before do - allow(HazardousFooModel).to receive(:reflect_on_association).and_return(nil) - end - - it 'does not preserve any foreign key IDs' do - expect(created_object.values).not_to include(99) - end - end - - context 'overrided model with pluralized name' do - let(:relation_sym) { :metrics } - - let(:relation_hash) do - { - 'id' => 99, - 'merge_request_id' => 99, - 'merged_at' => Time.now, - 'merged_by_id' => 99, - 'latest_closed_at' => nil, - 'latest_closed_by_id' => nil - } - end - - it 'does not raise errors' do - expect { created_object }.not_to raise_error - end - end - - context 'Project references' do - let(:relation_sym) { :project_foo_model } - let(:relation_hash) do - Gitlab::ImportExport::ProjectRelationFactory::PROJECT_REFERENCES.map { |ref| { ref => 99 } }.inject(:merge) - end - - class ProjectFooModel < FooModel - attr_accessor(*Gitlab::ImportExport::ProjectRelationFactory::PROJECT_REFERENCES) - end - - before do - allow(ProjectFooModel).to receive(:reflect_on_association).and_return(nil) - end - - it 'does not preserve any project foreign key IDs' do - expect(created_object.values).not_to include(99) - end - end - - context 'Notes user references' do - let(:relation_sym) { :notes } - let(:new_user) { create(:user) } - let(:exported_member) do - { - "id" => 111, - "access_level" => 30, - "source_id" => 1, - "source_type" => "Project", - "user_id" => 3, - "notification_level" => 3, - "created_at" => "2016-11-18T09:29:42.634Z", - "updated_at" => "2016-11-18T09:29:42.634Z", - "user" => { - "id" => 999, - "email" => new_user.email, - "username" => new_user.username - } - } - end - - let(:relation_hash) do - { - "id" => 4947, - "note" => "merged", - "noteable_type" => "MergeRequest", - "author_id" => 999, - "created_at" => "2016-11-18T09:29:42.634Z", - "updated_at" => "2016-11-18T09:29:42.634Z", - "project_id" => 1, - "attachment" => { - "url" => nil - }, - "noteable_id" => 377, - "system" => true, - "author" => { - "name" => "Administrator" - }, - "events" => [] - } - end - - let(:members_mapper) do - Gitlab::ImportExport::MembersMapper.new( - exported_members: [exported_member], - user: user, - importable: project) - end - - it 'maps the right author to the imported note' do - expect(created_object.author).to eq(new_user) - end - end - - context 'encrypted attributes' do - let(:relation_sym) { 'Ci::Variable' } - let(:relation_hash) do - create(:ci_variable).as_json - end - - it 'has no value for the encrypted attribute' do - expect(created_object.value).to be_nil - end - end -end diff --git a/spec/lib/gitlab/import_export/project_tree_loader_spec.rb b/spec/lib/gitlab/import_export/project_tree_loader_spec.rb deleted file mode 100644 index b22de5a3f7b..00000000000 --- a/spec/lib/gitlab/import_export/project_tree_loader_spec.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -describe Gitlab::ImportExport::ProjectTreeLoader do - let(:fixture) { 'spec/fixtures/lib/gitlab/import_export/with_duplicates.json' } - let(:project_tree) { JSON.parse(File.read(fixture)) } - - context 'without de-duplicating entries' do - let(:parsed_tree) do - subject.load(fixture) - end - - it 'parses the JSON into the expected tree' do - expect(parsed_tree).to eq(project_tree) - end - - it 'does not de-duplicate entries' do - expect(parsed_tree['duped_hash_with_id']).not_to be(parsed_tree['array'][0]['duped_hash_with_id']) - end - end - - context 'with de-duplicating entries' do - let(:parsed_tree) do - subject.load(fixture, dedup_entries: true) - end - - it 'parses the JSON into the expected tree' do - expect(parsed_tree).to eq(project_tree) - end - - it 'de-duplicates equal values' do - expect(parsed_tree['duped_hash_with_id']).to be(parsed_tree['array'][0]['duped_hash_with_id']) - expect(parsed_tree['duped_hash_with_id']).to be(parsed_tree['nested']['duped_hash_with_id']) - expect(parsed_tree['duped_array']).to be(parsed_tree['array'][1]['duped_array']) - expect(parsed_tree['duped_array']).to be(parsed_tree['nested']['duped_array']) - end - - it 'does not de-duplicate hashes without IDs' do - expect(parsed_tree['duped_hash_no_id']).to eq(parsed_tree['array'][2]['duped_hash_no_id']) - expect(parsed_tree['duped_hash_no_id']).not_to be(parsed_tree['array'][2]['duped_hash_no_id']) - end - - it 'keeps single entries intact' do - expect(parsed_tree['simple']).to eq(42) - expect(parsed_tree['nested']['array']).to eq(["don't touch"]) - end - end -end diff --git a/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb deleted file mode 100644 index c899217d164..00000000000 --- a/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb +++ /dev/null @@ -1,844 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -include ImportExport::CommonUtil - -describe Gitlab::ImportExport::ProjectTreeRestorer do - include ImportExport::CommonUtil - - let(:shared) { project.import_export_shared } - - describe 'restore project tree' do - before(:context) do - # Using an admin for import, so we can check assignment of existing members - @user = create(:admin) - @existing_members = [ - create(:user, email: 'bernard_willms@gitlabexample.com'), - create(:user, email: 'saul_will@gitlabexample.com') - ] - - RSpec::Mocks.with_temporary_scope do - @project = create(:project, :builds_enabled, :issues_disabled, name: 'project', path: 'project') - @shared = @project.import_export_shared - - setup_import_export_config('complex') - - allow_any_instance_of(Repository).to receive(:fetch_source_branch!).and_return(true) - allow_any_instance_of(Gitlab::Git::Repository).to receive(:branch_exists?).and_return(false) - - expect_any_instance_of(Gitlab::Git::Repository).to receive(:create_branch).with('feature', 'DCBA') - allow_any_instance_of(Gitlab::Git::Repository).to receive(:create_branch) - - project_tree_restorer = described_class.new(user: @user, shared: @shared, project: @project) - - @restored_project_json = project_tree_restorer.restore - end - end - - context 'JSON' do - it 'restores models based on JSON' do - expect(@restored_project_json).to be_truthy - end - - it 'restore correct project features' do - project = Project.find_by_path('project') - - expect(project.project_feature.issues_access_level).to eq(ProjectFeature::PRIVATE) - expect(project.project_feature.builds_access_level).to eq(ProjectFeature::PRIVATE) - expect(project.project_feature.snippets_access_level).to eq(ProjectFeature::PRIVATE) - expect(project.project_feature.wiki_access_level).to eq(ProjectFeature::PRIVATE) - expect(project.project_feature.merge_requests_access_level).to eq(ProjectFeature::PRIVATE) - end - - it 'has the project description' do - expect(Project.find_by_path('project').description).to eq('Nisi et repellendus ut enim quo accusamus vel magnam.') - end - - it 'has the same label associated to two issues' do - expect(ProjectLabel.find_by_title('test2').issues.count).to eq(2) - end - - it 'has milestones associated to two separate issues' do - expect(Milestone.find_by_description('test milestone').issues.count).to eq(2) - end - - context 'when importing a project with cached_markdown_version and note_html' do - context 'for an Issue' do - it 'does not import note_html' do - note_content = 'Quo reprehenderit aliquam qui dicta impedit cupiditate eligendi' - issue_note = Issue.find_by(description: 'Aliquam enim illo et possimus.').notes.select { |n| n.note.match(/#{note_content}/)}.first - - expect(issue_note.note_html).to match(/#{note_content}/) - end - end - - context 'for a Merge Request' do - it 'does not import note_html' do - note_content = 'Sit voluptatibus eveniet architecto quidem' - merge_request_note = MergeRequest.find_by(title: 'MR1').notes.select { |n| n.note.match(/#{note_content}/)}.first - - expect(merge_request_note.note_html).to match(/#{note_content}/) - end - end - end - - it 'creates a valid pipeline note' do - expect(Ci::Pipeline.find_by_sha('sha-notes').notes).not_to be_empty - end - - it 'pipeline has the correct user ID' do - expect(Ci::Pipeline.find_by_sha('sha-notes').user_id).to eq(@user.id) - end - - it 'restores pipelines with missing ref' do - expect(Ci::Pipeline.where(ref: nil)).not_to be_empty - end - - it 'restores pipeline for merge request' do - pipeline = Ci::Pipeline.find_by_sha('048721d90c449b244b7b4c53a9186b04330174ec') - - expect(pipeline).to be_valid - expect(pipeline.tag).to be_falsey - expect(pipeline.source).to eq('merge_request_event') - expect(pipeline.merge_request.id).to be > 0 - expect(pipeline.merge_request.target_branch).to eq('feature') - expect(pipeline.merge_request.source_branch).to eq('feature_conflict') - end - - it 'preserves updated_at on issues' do - issue = Issue.where(description: 'Aliquam enim illo et possimus.').first - - expect(issue.reload.updated_at.to_s).to eq('2016-06-14 15:02:47 UTC') - end - - it 'has multiple issue assignees' do - expect(Issue.find_by(title: 'Voluptatem').assignees).to contain_exactly(@user, *@existing_members) - expect(Issue.find_by(title: 'Issue without assignees').assignees).to be_empty - end - - it 'restores timelogs for issues' do - timelog = Issue.find_by(title: 'issue_with_timelogs').timelogs.last - - aggregate_failures do - expect(timelog.time_spent).to eq(72000) - expect(timelog.spent_at).to eq("2019-12-27T00:00:00.000Z") - end - end - - it 'contains the merge access levels on a protected branch' do - expect(ProtectedBranch.first.merge_access_levels).not_to be_empty - end - - it 'contains the push access levels on a protected branch' do - expect(ProtectedBranch.first.push_access_levels).not_to be_empty - end - - it 'contains the create access levels on a protected tag' do - expect(ProtectedTag.first.create_access_levels).not_to be_empty - end - - it 'restores issue resource label events' do - expect(Issue.find_by(title: 'Voluptatem').resource_label_events).not_to be_empty - end - - it 'restores merge requests resource label events' do - expect(MergeRequest.find_by(title: 'MR1').resource_label_events).not_to be_empty - end - - it 'restores suggestion' do - note = Note.find_by("note LIKE 'Saepe asperiores exercitationem non dignissimos laborum reiciendis et ipsum%'") - - expect(note.suggestions.count).to eq(1) - expect(note.suggestions.first.from_content).to eq("Original line\n") - end - - context 'event at forth level of the tree' do - let(:event) { Event.where(action: 6).first } - - it 'restores the event' do - expect(event).not_to be_nil - end - - it 'has the action' do - expect(event.action).not_to be_nil - end - - it 'event belongs to note, belongs to merge request, belongs to a project' do - expect(event.note.noteable.project).not_to be_nil - end - end - - it 'has the correct data for merge request diff files' do - expect(MergeRequestDiffFile.where.not(diff: nil).count).to eq(55) - end - - it 'has the correct data for merge request diff commits' do - expect(MergeRequestDiffCommit.count).to eq(77) - end - - it 'has the correct data for merge request latest_merge_request_diff' do - MergeRequest.find_each do |merge_request| - expect(merge_request.latest_merge_request_diff_id).to eq(merge_request.merge_request_diffs.maximum(:id)) - end - end - - it 'has labels associated to label links, associated to issues' do - expect(Label.first.label_links.first.target).not_to be_nil - end - - it 'has project labels' do - expect(ProjectLabel.count).to eq(3) - end - - it 'has no group labels' do - expect(GroupLabel.count).to eq(0) - end - - it 'has issue boards' do - expect(Project.find_by_path('project').boards.count).to eq(1) - end - - it 'has lists associated with the issue board' do - expect(Project.find_by_path('project').boards.find_by_name('TestBoardABC').lists.count).to eq(3) - end - - it 'has a project feature' do - expect(@project.project_feature).not_to be_nil - end - - it 'has custom attributes' do - expect(@project.custom_attributes.count).to eq(2) - end - - it 'has badges' do - expect(@project.project_badges.count).to eq(2) - end - - it 'has snippets' do - expect(@project.snippets.count).to eq(1) - end - - it 'has award emoji for a snippet' do - award_emoji = @project.snippets.first.award_emoji - - expect(award_emoji.map(&:name)).to contain_exactly('thumbsup', 'coffee') - end - - it 'snippet has notes' do - expect(@project.snippets.first.notes.count).to eq(1) - end - - it 'snippet has award emojis on notes' do - award_emoji = @project.snippets.first.notes.first.award_emoji.first - - expect(award_emoji.name).to eq('thumbsup') - end - - it 'restores `ci_cd_settings` : `group_runners_enabled` setting' do - expect(@project.ci_cd_settings.group_runners_enabled?).to eq(false) - end - - it 'restores `auto_devops`' do - expect(@project.auto_devops_enabled?).to eq(true) - expect(@project.auto_devops.deploy_strategy).to eq('continuous') - end - - it 'restores the correct service' do - expect(CustomIssueTrackerService.first).not_to be_nil - end - - it 'restores zoom meetings' do - meetings = @project.issues.first.zoom_meetings - - expect(meetings.count).to eq(1) - expect(meetings.first.url).to eq('https://zoom.us/j/123456789') - end - - it 'restores sentry issues' do - sentry_issue = @project.issues.first.sentry_issue - - expect(sentry_issue.sentry_issue_identifier).to eq(1234567891) - end - - it 'has award emoji for an issue' do - award_emoji = @project.issues.first.award_emoji.first - - expect(award_emoji.name).to eq('musical_keyboard') - end - - it 'has award emoji for a note in an issue' do - award_emoji = @project.issues.first.notes.first.award_emoji.first - - expect(award_emoji.name).to eq('clapper') - end - - it 'restores container_expiration_policy' do - policy = Project.find_by_path('project').container_expiration_policy - - aggregate_failures do - expect(policy).to be_an_instance_of(ContainerExpirationPolicy) - expect(policy).to be_persisted - expect(policy.cadence).to eq('3month') - end - end - - it 'restores error_tracking_setting' do - setting = @project.error_tracking_setting - - aggregate_failures do - expect(setting.api_url).to eq("https://gitlab.example.com/api/0/projects/sentry-org/sentry-project") - expect(setting.project_name).to eq("Sentry Project") - expect(setting.organization_name).to eq("Sentry Org") - end - end - - it 'restores external pull requests' do - external_pr = @project.external_pull_requests.last - - aggregate_failures do - expect(external_pr.pull_request_iid).to eq(4) - expect(external_pr.source_branch).to eq("feature") - expect(external_pr.target_branch).to eq("master") - expect(external_pr.status).to eq("open") - end - end - - it 'restores pipeline schedules' do - pipeline_schedule = @project.pipeline_schedules.last - - aggregate_failures do - expect(pipeline_schedule.description).to eq('Schedule Description') - expect(pipeline_schedule.ref).to eq('master') - expect(pipeline_schedule.cron).to eq('0 4 * * 0') - expect(pipeline_schedule.cron_timezone).to eq('UTC') - expect(pipeline_schedule.active).to eq(true) - end - end - - it 'restores releases with links' do - release = @project.releases.last - link = release.links.last - - aggregate_failures do - expect(release.tag).to eq('release-1.1') - expect(release.description).to eq('Some release notes') - expect(release.name).to eq('release-1.1') - expect(release.sha).to eq('901de3a8bd5573f4a049b1457d28bc1592ba6bf9') - expect(release.released_at).to eq('2019-12-26T10:17:14.615Z') - - expect(link.url).to eq('http://localhost/namespace6/project6/-/jobs/140463678/artifacts/download') - expect(link.name).to eq('release-1.1.dmg') - end - end - - context 'Merge requests' do - it 'always has the new project as a target' do - expect(MergeRequest.find_by_title('MR1').target_project).to eq(@project) - end - - it 'has the same source project as originally if source/target are the same' do - expect(MergeRequest.find_by_title('MR1').source_project).to eq(@project) - end - - it 'has the new project as target if source/target differ' do - expect(MergeRequest.find_by_title('MR2').target_project).to eq(@project) - end - - it 'has no source if source/target differ' do - expect(MergeRequest.find_by_title('MR2').source_project_id).to be_nil - end - - it 'has award emoji' do - award_emoji = MergeRequest.find_by_title('MR1').award_emoji - - expect(award_emoji.map(&:name)).to contain_exactly('thumbsup', 'drum') - end - - context 'notes' do - it 'has award emoji' do - award_emoji = MergeRequest.find_by_title('MR1').notes.first.award_emoji.first - - expect(award_emoji.name).to eq('tada') - end - end - end - - context 'tokens are regenerated' do - it 'has new CI trigger tokens' do - expect(Ci::Trigger.where(token: %w[cdbfasdf44a5958c83654733449e585 33a66349b5ad01fc00174af87804e40])) - .to be_empty - end - - it 'has a new CI build token' do - expect(Ci::Build.where(token: 'abcd')).to be_empty - end - end - - context 'has restored the correct number of records' do - it 'has the correct number of merge requests' do - expect(@project.merge_requests.size).to eq(9) - end - - it 'only restores valid triggers' do - expect(@project.triggers.size).to eq(1) - end - - it 'has the correct number of pipelines and statuses' do - expect(@project.ci_pipelines.size).to eq(7) - - @project.ci_pipelines.order(:id).zip([2, 2, 2, 2, 2, 0, 0]) - .each do |(pipeline, expected_status_size)| - expect(pipeline.statuses.size).to eq(expected_status_size) - end - end - end - - context 'when restoring hierarchy of pipeline, stages and jobs' do - it 'restores pipelines' do - expect(Ci::Pipeline.all.count).to be 7 - end - - it 'restores pipeline stages' do - expect(Ci::Stage.all.count).to be 6 - end - - it 'correctly restores association between stage and a pipeline' do - expect(Ci::Stage.all).to all(have_attributes(pipeline_id: a_value > 0)) - end - - it 'restores statuses' do - expect(CommitStatus.all.count).to be 10 - end - - it 'correctly restores association between a stage and a job' do - expect(CommitStatus.all).to all(have_attributes(stage_id: a_value > 0)) - end - - it 'correctly restores association between a pipeline and a job' do - expect(CommitStatus.all).to all(have_attributes(pipeline_id: a_value > 0)) - end - - it 'restores a Hash for CommitStatus options' do - expect(CommitStatus.all.map(&:options).compact).to all(be_a(Hash)) - end - - it 'restores external pull request for the restored pipeline' do - pipeline_with_external_pr = @project.ci_pipelines.order(:id).last - - expect(pipeline_with_external_pr.external_pull_request).to be_persisted - end - end - end - end - - shared_examples 'restores group correctly' do |**results| - it 'has group label' do - expect(project.group.labels.size).to eq(results.fetch(:labels, 0)) - expect(project.group.labels.where(type: "GroupLabel").where.not(project_id: nil).count).to eq(0) - end - - it 'has group milestone' do - expect(project.group.milestones.size).to eq(results.fetch(:milestones, 0)) - end - - it 'has the correct visibility level' do - # INTERNAL in the `project.json`, group's is PRIVATE - expect(project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) - end - end - - context 'project.json file access check' do - let(:user) { create(:user) } - let!(:project) { create(:project, :builds_disabled, :issues_disabled, name: 'project', path: 'project') } - let(:project_tree_restorer) do - described_class.new(user: user, shared: shared, project: project) - end - let(:restored_project_json) { project_tree_restorer.restore } - - it 'does not read a symlink' do - Dir.mktmpdir do |tmpdir| - setup_symlink(tmpdir, 'project.json') - allow(shared).to receive(:export_path).and_call_original - - expect(project_tree_restorer.restore).to eq(false) - expect(shared.errors).to include('Incorrect JSON format') - end - end - end - - context 'Light JSON' do - let(:user) { create(:user) } - let!(:project) { create(:project, :builds_disabled, :issues_disabled, name: 'project', path: 'project') } - let(:project_tree_restorer) { described_class.new(user: user, shared: shared, project: project) } - let(:restored_project_json) { project_tree_restorer.restore } - - context 'with a simple project' do - before do - setup_import_export_config('light') - expect(restored_project_json).to eq(true) - end - - it_behaves_like 'restores project successfully', - issues: 1, - labels: 2, - label_with_priorities: 'A project label', - milestones: 1, - first_issue_labels: 1, - services: 1 - - context 'when there is an existing build with build token' do - before do - create(:ci_build, token: 'abcd') - end - - it_behaves_like 'restores project successfully', - issues: 1, - labels: 2, - label_with_priorities: 'A project label', - milestones: 1, - first_issue_labels: 1 - end - end - - context 'when post import action throw non-retriable exception' do - let(:exception) { StandardError.new('post_import_error') } - - before do - setup_import_export_config('light') - expect(project) - .to receive(:merge_requests) - .and_raise(exception) - end - - it 'report post import error' do - expect(restored_project_json).to eq(false) - expect(shared.errors).to include('post_import_error') - end - end - - context 'when post import action throw retriable exception one time' do - let(:exception) { GRPC::DeadlineExceeded.new } - - before do - setup_import_export_config('light') - expect(project) - .to receive(:merge_requests) - .and_raise(exception) - expect(project) - .to receive(:merge_requests) - .and_call_original - expect(restored_project_json).to eq(true) - end - - it_behaves_like 'restores project successfully', - issues: 1, - labels: 2, - label_with_priorities: 'A project label', - milestones: 1, - first_issue_labels: 1, - services: 1, - import_failures: 1 - - it 'records the failures in the database' do - import_failure = ImportFailure.last - - expect(import_failure.project_id).to eq(project.id) - expect(import_failure.relation_key).to be_nil - expect(import_failure.relation_index).to be_nil - expect(import_failure.exception_class).to eq('GRPC::DeadlineExceeded') - expect(import_failure.exception_message).to be_present - expect(import_failure.correlation_id_value).not_to be_empty - expect(import_failure.created_at).to be_present - end - end - - context 'when the project has overridden params in import data' do - before do - setup_import_export_config('light') - end - - it 'handles string versions of visibility_level' do - # Project needs to be in a group for visibility level comparison - # to happen - group = create(:group) - project.group = group - - project.create_import_data(data: { override_params: { visibility_level: Gitlab::VisibilityLevel::INTERNAL.to_s } }) - - expect(restored_project_json).to eq(true) - expect(project.visibility_level).to eq(Gitlab::VisibilityLevel::INTERNAL) - end - - it 'overwrites the params stored in the JSON' do - project.create_import_data(data: { override_params: { description: "Overridden" } }) - - expect(restored_project_json).to eq(true) - expect(project.description).to eq("Overridden") - end - - it 'does not allow setting params that are excluded from import_export settings' do - project.create_import_data(data: { override_params: { lfs_enabled: true } }) - - expect(restored_project_json).to eq(true) - expect(project.lfs_enabled).to be_falsey - end - - it 'overrides project feature access levels' do - access_level_keys = project.project_feature.attributes.keys.select { |a| a =~ /_access_level/ } - - # `pages_access_level` is not included, since it is not available in the public API - # and has a dependency on project's visibility level - # see ProjectFeature model - access_level_keys.delete('pages_access_level') - - disabled_access_levels = Hash[access_level_keys.collect { |item| [item, 'disabled'] }] - - project.create_import_data(data: { override_params: disabled_access_levels }) - - expect(restored_project_json).to eq(true) - - aggregate_failures do - access_level_keys.each do |key| - expect(project.public_send(key)).to eq(ProjectFeature::DISABLED) - end - end - end - end - - context 'with a project that has a group' do - let!(:project) do - create(:project, - :builds_disabled, - :issues_disabled, - name: 'project', - path: 'project', - group: create(:group, visibility_level: Gitlab::VisibilityLevel::PRIVATE)) - end - - before do - setup_import_export_config('group') - expect(restored_project_json).to eq(true) - end - - it_behaves_like 'restores project successfully', - issues: 3, - labels: 2, - label_with_priorities: 'A project label', - milestones: 2, - first_issue_labels: 1 - - it_behaves_like 'restores group correctly', - labels: 0, - milestones: 0, - first_issue_labels: 1 - - it 'restores issue states' do - expect(project.issues.with_state(:closed).count).to eq(1) - expect(project.issues.with_state(:opened).count).to eq(2) - end - end - - context 'with existing group models' do - let!(:project) do - create(:project, - :builds_disabled, - :issues_disabled, - name: 'project', - path: 'project', - group: create(:group)) - end - - before do - setup_import_export_config('light') - end - - it 'does not import any templated services' do - expect(restored_project_json).to eq(true) - - expect(project.services.where(template: true).count).to eq(0) - end - - it 'imports labels' do - create(:group_label, name: 'Another label', group: project.group) - - expect_any_instance_of(Gitlab::ImportExport::Shared).not_to receive(:error) - - expect(restored_project_json).to eq(true) - expect(project.labels.count).to eq(1) - end - - it 'imports milestones' do - create(:milestone, name: 'A milestone', group: project.group) - - expect_any_instance_of(Gitlab::ImportExport::Shared).not_to receive(:error) - - expect(restored_project_json).to eq(true) - expect(project.group.milestones.count).to eq(1) - expect(project.milestones.count).to eq(0) - end - end - - context 'with clashing milestones on IID' do - let!(:project) do - create(:project, - :builds_disabled, - :issues_disabled, - name: 'project', - path: 'project', - group: create(:group)) - end - - before do - setup_import_export_config('milestone-iid') - end - - it 'preserves the project milestone IID' do - expect_any_instance_of(Gitlab::ImportExport::Shared).not_to receive(:error) - - expect(restored_project_json).to eq(true) - expect(project.milestones.count).to eq(2) - expect(Milestone.find_by_title('Another milestone').iid).to eq(1) - expect(Milestone.find_by_title('Group-level milestone').iid).to eq(2) - end - end - - context 'with external authorization classification labels' do - before do - setup_import_export_config('light') - end - - it 'converts empty external classification authorization labels to nil' do - project.create_import_data(data: { override_params: { external_authorization_classification_label: "" } }) - - expect(restored_project_json).to eq(true) - expect(project.external_authorization_classification_label).to be_nil - end - - it 'preserves valid external classification authorization labels' do - project.create_import_data(data: { override_params: { external_authorization_classification_label: "foobar" } }) - - expect(restored_project_json).to eq(true) - expect(project.external_authorization_classification_label).to eq("foobar") - end - end - end - - context 'Minimal JSON' do - let(:project) { create(:project) } - let(:user) { create(:user) } - let(:tree_hash) { { 'visibility_level' => visibility } } - let(:restorer) do - described_class.new(user: user, shared: shared, project: project) - end - - before do - expect(restorer).to receive(:read_tree_hash) { tree_hash } - end - - context 'no group visibility' do - let(:visibility) { Gitlab::VisibilityLevel::PRIVATE } - - it 'uses the project visibility' do - expect(restorer.restore).to eq(true) - expect(restorer.project.visibility_level).to eq(visibility) - end - end - - context 'with restricted internal visibility' do - describe 'internal project' do - let(:visibility) { Gitlab::VisibilityLevel::INTERNAL } - - it 'uses private visibility' do - stub_application_setting(restricted_visibility_levels: [Gitlab::VisibilityLevel::INTERNAL]) - - expect(restorer.restore).to eq(true) - expect(restorer.project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) - end - end - end - - context 'with group visibility' do - before do - group = create(:group, visibility_level: group_visibility) - - project.update(group: group) - end - - context 'private group visibility' do - let(:group_visibility) { Gitlab::VisibilityLevel::PRIVATE } - let(:visibility) { Gitlab::VisibilityLevel::PUBLIC } - - it 'uses the group visibility' do - expect(restorer.restore).to eq(true) - expect(restorer.project.visibility_level).to eq(group_visibility) - end - end - - context 'public group visibility' do - let(:group_visibility) { Gitlab::VisibilityLevel::PUBLIC } - let(:visibility) { Gitlab::VisibilityLevel::PRIVATE } - - it 'uses the project visibility' do - expect(restorer.restore).to eq(true) - expect(restorer.project.visibility_level).to eq(visibility) - end - end - - context 'internal group visibility' do - let(:group_visibility) { Gitlab::VisibilityLevel::INTERNAL } - let(:visibility) { Gitlab::VisibilityLevel::PUBLIC } - - it 'uses the group visibility' do - expect(restorer.restore).to eq(true) - expect(restorer.project.visibility_level).to eq(group_visibility) - end - - context 'with restricted internal visibility' do - it 'sets private visibility' do - stub_application_setting(restricted_visibility_levels: [Gitlab::VisibilityLevel::INTERNAL]) - - expect(restorer.restore).to eq(true) - expect(restorer.project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) - end - end - end - end - end - - context 'JSON with invalid records' do - subject(:restored_project_json) { project_tree_restorer.restore } - - let(:user) { create(:user) } - let!(:project) { create(:project, :builds_disabled, :issues_disabled, name: 'project', path: 'project') } - let(:project_tree_restorer) { described_class.new(user: user, shared: shared, project: project) } - - before do - setup_import_export_config('with_invalid_records') - - subject - end - - context 'when failures occur because a relation fails to be processed' do - it_behaves_like 'restores project successfully', - issues: 0, - labels: 0, - label_with_priorities: nil, - milestones: 1, - first_issue_labels: 0, - services: 0, - import_failures: 1 - - it 'records the failures in the database' do - import_failure = ImportFailure.last - - expect(import_failure.project_id).to eq(project.id) - expect(import_failure.relation_key).to eq('milestones') - expect(import_failure.relation_index).to be_present - expect(import_failure.exception_class).to eq('ActiveRecord::RecordInvalid') - expect(import_failure.exception_message).to be_present - expect(import_failure.correlation_id_value).not_to be_empty - expect(import_failure.created_at).to be_present - end - end - end -end diff --git a/spec/lib/gitlab/import_export/project_tree_saver_spec.rb b/spec/lib/gitlab/import_export/project_tree_saver_spec.rb deleted file mode 100644 index 126ac289a56..00000000000 --- a/spec/lib/gitlab/import_export/project_tree_saver_spec.rb +++ /dev/null @@ -1,397 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -describe Gitlab::ImportExport::ProjectTreeSaver do - describe 'saves the project tree into a json object' do - let(:shared) { project.import_export_shared } - let(:project_tree_saver) { described_class.new(project: project, current_user: user, shared: shared) } - let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } - let(:user) { create(:user) } - let!(:project) { setup_project } - - before do - project.add_maintainer(user) - allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) - allow_any_instance_of(MergeRequest).to receive(:source_branch_sha).and_return('ABCD') - allow_any_instance_of(MergeRequest).to receive(:target_branch_sha).and_return('DCBA') - end - - after do - FileUtils.rm_rf(export_path) - end - - it 'saves project successfully' do - expect(project_tree_saver.save).to be true - end - - context ':export_fast_serialize feature flag checks' do - before do - expect(Gitlab::ImportExport::Reader).to receive(:new).with(shared: shared).and_return(reader) - expect(reader).to receive(:project_tree).and_return(project_tree) - end - - let(:serializer) { instance_double('Gitlab::ImportExport::FastHashSerializer') } - let(:reader) { instance_double('Gitlab::ImportExport::Reader') } - let(:project_tree) do - { - include: [{ issues: { include: [] } }], - preload: { issues: nil } - } - end - - context 'when :export_fast_serialize feature is enabled' do - before do - stub_feature_flags(export_fast_serialize: true) - end - - it 'uses FastHashSerializer' do - expect(Gitlab::ImportExport::FastHashSerializer) - .to receive(:new) - .with(project, project_tree) - .and_return(serializer) - - expect(serializer).to receive(:execute) - - project_tree_saver.save - end - end - - context 'when :export_fast_serialize feature is disabled' do - before do - stub_feature_flags(export_fast_serialize: false) - end - - it 'is serialized via built-in `as_json`' do - expect(project).to receive(:as_json).with(project_tree) - - project_tree_saver.save - end - end - end - - # It is mostly duplicated in - # `spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb` - # except: - # context 'with description override' do - # context 'group members' do - # ^ These are specific for the ProjectTreeSaver - context 'JSON' do - let(:saved_project_json) do - project_tree_saver.save - project_json(project_tree_saver.full_path) - end - - # It is not duplicated in - # `spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb` - context 'with description override' do - let(:params) { { description: 'Foo Bar' } } - let(:project_tree_saver) { described_class.new(project: project, current_user: user, shared: shared, params: params) } - - it 'overrides the project description' do - expect(saved_project_json).to include({ 'description' => params[:description] }) - end - end - - it 'saves the correct json' do - expect(saved_project_json).to include({ 'description' => 'description', 'visibility_level' => 20 }) - end - - it 'has approvals_before_merge set' do - expect(saved_project_json['approvals_before_merge']).to eq(1) - end - - it 'has milestones' do - expect(saved_project_json['milestones']).not_to be_empty - end - - it 'has merge requests' do - expect(saved_project_json['merge_requests']).not_to be_empty - end - - it 'has merge request\'s milestones' do - expect(saved_project_json['merge_requests'].first['milestone']).not_to be_empty - end - - it 'has merge request\'s source branch SHA' do - expect(saved_project_json['merge_requests'].first['source_branch_sha']).to eq('ABCD') - end - - it 'has merge request\'s target branch SHA' do - expect(saved_project_json['merge_requests'].first['target_branch_sha']).to eq('DCBA') - end - - it 'has events' do - expect(saved_project_json['merge_requests'].first['milestone']['events']).not_to be_empty - end - - it 'has snippets' do - expect(saved_project_json['snippets']).not_to be_empty - end - - it 'has snippet notes' do - expect(saved_project_json['snippets'].first['notes']).not_to be_empty - end - - it 'has releases' do - expect(saved_project_json['releases']).not_to be_empty - end - - it 'has no author on releases' do - expect(saved_project_json['releases'].first['author']).to be_nil - end - - it 'has the author ID on releases' do - expect(saved_project_json['releases'].first['author_id']).not_to be_nil - end - - it 'has issues' do - expect(saved_project_json['issues']).not_to be_empty - end - - it 'has issue comments' do - notes = saved_project_json['issues'].first['notes'] - - expect(notes).not_to be_empty - expect(notes.first['type']).to eq('DiscussionNote') - end - - it 'has issue assignees' do - expect(saved_project_json['issues'].first['issue_assignees']).not_to be_empty - end - - it 'has author on issue comments' do - expect(saved_project_json['issues'].first['notes'].first['author']).not_to be_empty - end - - it 'has project members' do - expect(saved_project_json['project_members']).not_to be_empty - end - - it 'has merge requests diffs' do - expect(saved_project_json['merge_requests'].first['merge_request_diff']).not_to be_empty - end - - it 'has merge request diff files' do - expect(saved_project_json['merge_requests'].first['merge_request_diff']['merge_request_diff_files']).not_to be_empty - end - - it 'has merge request diff commits' do - expect(saved_project_json['merge_requests'].first['merge_request_diff']['merge_request_diff_commits']).not_to be_empty - end - - it 'has merge requests comments' do - expect(saved_project_json['merge_requests'].first['notes']).not_to be_empty - end - - it 'has author on merge requests comments' do - expect(saved_project_json['merge_requests'].first['notes'].first['author']).not_to be_empty - end - - it 'has pipeline stages' do - expect(saved_project_json.dig('ci_pipelines', 0, 'stages')).not_to be_empty - end - - it 'has pipeline statuses' do - expect(saved_project_json.dig('ci_pipelines', 0, 'stages', 0, 'statuses')).not_to be_empty - end - - it 'has pipeline builds' do - builds_count = saved_project_json - .dig('ci_pipelines', 0, 'stages', 0, 'statuses') - .count { |hash| hash['type'] == 'Ci::Build' } - - expect(builds_count).to eq(1) - end - - it 'has no when YML attributes but only the DB column' do - expect_any_instance_of(Gitlab::Ci::YamlProcessor).not_to receive(:build_attributes) - - saved_project_json - end - - it 'has pipeline commits' do - expect(saved_project_json['ci_pipelines']).not_to be_empty - end - - it 'has ci pipeline notes' do - expect(saved_project_json['ci_pipelines'].first['notes']).not_to be_empty - end - - it 'has labels with no associations' do - expect(saved_project_json['labels']).not_to be_empty - end - - it 'has labels associated to records' do - expect(saved_project_json['issues'].first['label_links'].first['label']).not_to be_empty - end - - it 'has project and group labels' do - label_types = saved_project_json['issues'].first['label_links'].map { |link| link['label']['type'] } - - expect(label_types).to match_array(%w(ProjectLabel GroupLabel)) - end - - it 'has priorities associated to labels' do - priorities = saved_project_json['issues'].first['label_links'].flat_map { |link| link['label']['priorities'] } - - expect(priorities).not_to be_empty - end - - it 'has issue resource label events' do - expect(saved_project_json['issues'].first['resource_label_events']).not_to be_empty - end - - it 'has merge request resource label events' do - expect(saved_project_json['merge_requests'].first['resource_label_events']).not_to be_empty - end - - it 'saves the correct service type' do - expect(saved_project_json['services'].first['type']).to eq('CustomIssueTrackerService') - end - - it 'saves the properties for a service' do - expect(saved_project_json['services'].first['properties']).to eq('one' => 'value') - end - - it 'has project feature' do - project_feature = saved_project_json['project_feature'] - expect(project_feature).not_to be_empty - expect(project_feature["issues_access_level"]).to eq(ProjectFeature::DISABLED) - expect(project_feature["wiki_access_level"]).to eq(ProjectFeature::ENABLED) - expect(project_feature["builds_access_level"]).to eq(ProjectFeature::PRIVATE) - end - - it 'has custom attributes' do - expect(saved_project_json['custom_attributes'].count).to eq(2) - end - - it 'has badges' do - expect(saved_project_json['project_badges'].count).to eq(2) - end - - it 'does not complain about non UTF-8 characters in MR diff files' do - ActiveRecord::Base.connection.execute("UPDATE merge_request_diff_files SET diff = '---\n- :diff: !binary |-\n LS0tIC9kZXYvbnVsbAorKysgYi9pbWFnZXMvbnVjb3IucGRmCkBAIC0wLDAg\n KzEsMTY3OSBAQAorJVBERi0xLjUNJeLjz9MNCisxIDAgb2JqDTw8L01ldGFk\n YXR'") - - expect(project_tree_saver.save).to be true - end - - context 'group members' do - let(:user2) { create(:user, email: 'group@member.com') } - let(:member_emails) do - saved_project_json['project_members'].map do |pm| - pm['user']['email'] - end - end - - before do - Group.first.add_developer(user2) - end - - it 'does not export group members if it has no permission' do - Group.first.add_developer(user) - - expect(member_emails).not_to include('group@member.com') - end - - it 'does not export group members as maintainer' do - Group.first.add_maintainer(user) - - expect(member_emails).not_to include('group@member.com') - end - - it 'exports group members as group owner' do - Group.first.add_owner(user) - - expect(member_emails).to include('group@member.com') - end - - context 'as admin' do - let(:user) { create(:admin) } - - it 'exports group members as admin' do - expect(member_emails).to include('group@member.com') - end - - it 'exports group members as project members' do - member_types = saved_project_json['project_members'].map { |pm| pm['source_type'] } - - expect(member_types).to all(eq('Project')) - end - end - end - - context 'project attributes' do - it 'does not contain the runners token' do - expect(saved_project_json).not_to include("runners_token" => 'token') - end - end - - it 'has a board and a list' do - expect(saved_project_json['boards'].first['lists']).not_to be_empty - end - end - end - - def setup_project - release = create(:release) - group = create(:group) - - project = create(:project, - :public, - :repository, - :issues_disabled, - :wiki_enabled, - :builds_private, - description: 'description', - releases: [release], - group: group, - approvals_before_merge: 1 - ) - allow(project).to receive(:commit).and_return(Commit.new(RepoHelpers.sample_commit, project)) - - issue = create(:issue, assignees: [user], project: project) - snippet = create(:project_snippet, project: project) - project_label = create(:label, project: project) - group_label = create(:group_label, group: group) - create(:label_link, label: project_label, target: issue) - create(:label_link, label: group_label, target: issue) - create(:label_priority, label: group_label, priority: 1) - milestone = create(:milestone, project: project) - merge_request = create(:merge_request, source_project: project, milestone: milestone) - - ci_build = create(:ci_build, project: project, when: nil) - ci_build.pipeline.update(project: project) - create(:commit_status, project: project, pipeline: ci_build.pipeline) - - create(:milestone, project: project) - create(:discussion_note, noteable: issue, project: project) - create(:note, noteable: merge_request, project: project) - create(:note, noteable: snippet, project: project) - create(:note_on_commit, - author: user, - project: project, - commit_id: ci_build.pipeline.sha) - - create(:resource_label_event, label: project_label, issue: issue) - create(:resource_label_event, label: group_label, merge_request: merge_request) - - create(:event, :created, target: milestone, project: project, author: user) - create(:service, project: project, type: 'CustomIssueTrackerService', category: 'issue_tracker', properties: { one: 'value' }) - - create(:project_custom_attribute, project: project) - create(:project_custom_attribute, project: project) - - create(:project_badge, project: project) - create(:project_badge, project: project) - - board = create(:board, project: project, name: 'TestBoard') - create(:list, board: board, position: 0, label: project_label) - - project - end - - def project_json(filename) - JSON.parse(IO.read(filename)) - end -end diff --git a/spec/lib/gitlab/import_export/relation_rename_service_spec.rb b/spec/lib/gitlab/import_export/relation_rename_service_spec.rb index d62f5725f9e..2e251154e9f 100644 --- a/spec/lib/gitlab/import_export/relation_rename_service_spec.rb +++ b/spec/lib/gitlab/import_export/relation_rename_service_spec.rb @@ -22,7 +22,7 @@ describe Gitlab::ImportExport::RelationRenameService do end context 'when importing' do - let(:project_tree_restorer) { Gitlab::ImportExport::ProjectTreeRestorer.new(user: user, shared: shared, project: project) } + let(:project_tree_restorer) { Gitlab::ImportExport::Project::TreeRestorer.new(user: user, shared: shared, project: project) } let(:file_content) { IO.read(File.join(shared.export_path, 'project.json')) } let(:json_file) { ActiveSupport::JSON.decode(file_content) } @@ -99,7 +99,7 @@ describe Gitlab::ImportExport::RelationRenameService do let(:relation_tree_saver) { Gitlab::ImportExport::RelationTreeSaver.new } let(:project_tree_saver) do - Gitlab::ImportExport::ProjectTreeSaver.new( + Gitlab::ImportExport::Project::TreeSaver.new( project: project, current_user: user, shared: shared) end diff --git a/spec/lib/gitlab/import_export/relation_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/relation_tree_restorer_spec.rb index edb2c0a131a..80901feb893 100644 --- a/spec/lib/gitlab/import_export/relation_tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/relation_tree_restorer_spec.rb @@ -39,8 +39,8 @@ describe Gitlab::ImportExport::RelationTreeRestorer do context 'when restoring a project' do let(:path) { 'spec/fixtures/lib/gitlab/import_export/complex/project.json' } let(:importable) { create(:project, :builds_enabled, :issues_disabled, name: 'project', path: 'project') } - let(:object_builder) { Gitlab::ImportExport::GroupProjectObjectBuilder } - let(:relation_factory) { Gitlab::ImportExport::ProjectRelationFactory } + let(:object_builder) { Gitlab::ImportExport::Project::ObjectBuilder } + let(:relation_factory) { Gitlab::ImportExport::Project::RelationFactory } let(:reader) { Gitlab::ImportExport::Reader.new(shared: shared) } let(:tree_hash) { importable_hash } diff --git a/spec/lib/gitlab_danger_spec.rb b/spec/lib/gitlab_danger_spec.rb index 26bf5d76756..18321541221 100644 --- a/spec/lib/gitlab_danger_spec.rb +++ b/spec/lib/gitlab_danger_spec.rb @@ -9,7 +9,7 @@ describe GitlabDanger do describe '.local_warning_message' do it 'returns an informational message with rules that can run' do - expect(described_class.local_warning_message).to eq('==> Only the following Danger rules can be run locally: changes_size, gemfile, documentation, frozen_string, duplicate_yarn_dependencies, prettier, eslint, database, commit_messages') + expect(described_class.local_warning_message).to eq('==> Only the following Danger rules can be run locally: changes_size, documentation, frozen_string, duplicate_yarn_dependencies, prettier, eslint, database, commit_messages') end end diff --git a/spec/models/repository_spec.rb b/spec/models/repository_spec.rb index 00ffc3cae54..076897e6312 100644 --- a/spec/models/repository_spec.rb +++ b/spec/models/repository_spec.rb @@ -320,6 +320,21 @@ describe Repository do end end + context "when 'author' is set" do + it "returns commits from that author" do + commit = repository.commits(nil, limit: 1).first + known_author = "#{commit.author_name} <#{commit.author_email}>" + + expect(repository.commits(nil, author: known_author, limit: 1)).not_to be_empty + end + + it "doesn't returns commits from an unknown author" do + unknown_author = "The Man With No Name " + + expect(repository.commits(nil, author: unknown_author, limit: 1)).to be_empty + end + end + context "when 'all' flag is set" do it 'returns every commit from the repository' do expect(repository.commits(nil, all: true, limit: 60).size).to eq(60) diff --git a/spec/requests/api/internal/base_spec.rb b/spec/requests/api/internal/base_spec.rb index c3b5f9ded21..4249ce105c9 100644 --- a/spec/requests/api/internal/base_spec.rb +++ b/spec/requests/api/internal/base_spec.rb @@ -409,7 +409,7 @@ describe API::Internal::Base do it do pull(key, project) - expect(response).to have_gitlab_http_status(:unauthorized) + expect(response).to have_gitlab_http_status(:forbidden) expect(json_response["status"]).to be_falsey expect(user.reload.last_activity_on).to be_nil end @@ -419,7 +419,7 @@ describe API::Internal::Base do it do push(key, project) - expect(response).to have_gitlab_http_status(:unauthorized) + expect(response).to have_gitlab_http_status(:forbidden) expect(json_response["status"]).to be_falsey expect(user.reload.last_activity_on).to be_nil end @@ -518,7 +518,7 @@ describe API::Internal::Base do it do pull(key, personal_project) - expect(response).to have_gitlab_http_status(:unauthorized) + expect(response).to have_gitlab_http_status(:forbidden) expect(json_response["status"]).to be_falsey expect(user.reload.last_activity_on).to be_nil end @@ -528,7 +528,7 @@ describe API::Internal::Base do it do push(key, personal_project) - expect(response).to have_gitlab_http_status(:unauthorized) + expect(response).to have_gitlab_http_status(:forbidden) expect(json_response["status"]).to be_falsey expect(user.reload.last_activity_on).to be_nil end @@ -572,7 +572,7 @@ describe API::Internal::Base do it do push(key, project) - expect(response).to have_gitlab_http_status(:unauthorized) + expect(response).to have_gitlab_http_status(:forbidden) expect(json_response["status"]).to be_falsey end end @@ -654,7 +654,7 @@ describe API::Internal::Base do it 'rejects the SSH push' do push(key, project) - expect(response.status).to eq(401) + expect(response).to have_gitlab_http_status(:forbidden) expect(json_response['status']).to be_falsey expect(json_response['message']).to eq 'Git access over SSH is not allowed' end @@ -662,7 +662,7 @@ describe API::Internal::Base do it 'rejects the SSH pull' do pull(key, project) - expect(response.status).to eq(401) + expect(response).to have_gitlab_http_status(:forbidden) expect(json_response['status']).to be_falsey expect(json_response['message']).to eq 'Git access over SSH is not allowed' end @@ -676,7 +676,7 @@ describe API::Internal::Base do it 'rejects the HTTP push' do push(key, project, 'http') - expect(response.status).to eq(401) + expect(response).to have_gitlab_http_status(:forbidden) expect(json_response['status']).to be_falsey expect(json_response['message']).to eq 'Git access over HTTP is not allowed' end @@ -684,7 +684,7 @@ describe API::Internal::Base do it 'rejects the HTTP pull' do pull(key, project, 'http') - expect(response.status).to eq(401) + expect(response).to have_gitlab_http_status(:forbidden) expect(json_response['status']).to be_falsey expect(json_response['message']).to eq 'Git access over HTTP is not allowed' end diff --git a/spec/services/groups/import_export/export_service_spec.rb b/spec/services/groups/import_export/export_service_spec.rb index b1f76964722..5eebf08892a 100644 --- a/spec/services/groups/import_export/export_service_spec.rb +++ b/spec/services/groups/import_export/export_service_spec.rb @@ -7,7 +7,7 @@ describe Groups::ImportExport::ExportService do let!(:user) { create(:user) } let(:group) { create(:group) } let(:shared) { Gitlab::ImportExport::Shared.new(group) } - let(:export_path) { shared.export_path } + let(:archive_path) { shared.archive_path } let(:service) { described_class.new(group: group, user: user, params: { shared: shared }) } before do @@ -15,11 +15,11 @@ describe Groups::ImportExport::ExportService do end after do - FileUtils.rm_rf(export_path) + FileUtils.rm_rf(archive_path) end it 'saves the models' do - expect(Gitlab::ImportExport::GroupTreeSaver).to receive(:new).and_call_original + expect(Gitlab::ImportExport::Group::TreeSaver).to receive(:new).and_call_original service.execute end @@ -29,7 +29,7 @@ describe Groups::ImportExport::ExportService do service.execute expect(group.import_export_upload.export_file.file).not_to be_nil - expect(File.directory?(export_path)).to eq(false) + expect(File.directory?(archive_path)).to eq(false) expect(File.exist?(shared.archive_path)).to eq(false) end end @@ -46,25 +46,42 @@ describe Groups::ImportExport::ExportService do end end - context 'when saving services fail' do - before do - allow(service).to receive_message_chain(:tree_exporter, :save).and_return(false) + context 'when export fails' do + context 'when file saver fails' do + it 'removes the remaining exported data' do + allow_next_instance_of(Gitlab::ImportExport::Saver) do |saver| + allow(saver).to receive(:save).and_return(false) + end + + expect { service.execute }.to raise_error(Gitlab::ImportExport::Error) + + expect(group.import_export_upload).to be_nil + expect(File.exist?(shared.archive_path)).to eq(false) + end end - it 'removes the remaining exported data' do - allow_any_instance_of(Gitlab::ImportExport::Saver).to receive(:compress_and_save).and_return(false) + context 'when file compression fails' do + before do + allow(service).to receive_message_chain(:tree_exporter, :save).and_return(false) + end - expect { service.execute }.to raise_error(Gitlab::ImportExport::Error) + it 'removes the remaining exported data' do + allow_next_instance_of(Gitlab::ImportExport::Saver) do |saver| + allow(saver).to receive(:compress_and_save).and_return(false) + end - expect(group.import_export_upload).to be_nil - expect(File.directory?(export_path)).to eq(false) - expect(File.exist?(shared.archive_path)).to eq(false) - end + expect { service.execute }.to raise_error(Gitlab::ImportExport::Error) + + expect(group.import_export_upload).to be_nil + expect(File.exist?(shared.archive_path)).to eq(false) + end - it 'notifies logger' do - expect_any_instance_of(Gitlab::Import::Logger).to receive(:error) + it 'notifies logger' do + allow(service).to receive_message_chain(:tree_exporter, :save).and_return(false) + expect(shared.logger).to receive(:error) - expect { service.execute }.to raise_error(Gitlab::ImportExport::Error) + expect { service.execute }.to raise_error(Gitlab::ImportExport::Error) + end end end end diff --git a/spec/services/projects/import_export/export_service_spec.rb b/spec/services/projects/import_export/export_service_spec.rb index 906fef6edf5..ec1771e64c2 100644 --- a/spec/services/projects/import_export/export_service_spec.rb +++ b/spec/services/projects/import_export/export_service_spec.rb @@ -27,7 +27,7 @@ describe Projects::ImportExport::ExportService do end it 'saves the models' do - expect(Gitlab::ImportExport::ProjectTreeSaver).to receive(:new).and_call_original + expect(Gitlab::ImportExport::Project::TreeSaver).to receive(:new).and_call_original service.execute end @@ -91,10 +91,10 @@ describe Projects::ImportExport::ExportService do end it 'removes the remaining exported data' do - allow(shared).to receive(:export_path).and_return('whatever') + allow(shared).to receive(:archive_path).and_return('whatever') allow(FileUtils).to receive(:rm_rf) - expect(FileUtils).to receive(:rm_rf).with(shared.export_path) + expect(FileUtils).to receive(:rm_rf).with(shared.archive_path) end it 'notifies the user' do @@ -121,10 +121,10 @@ describe Projects::ImportExport::ExportService do end it 'removes the remaining exported data' do - allow(shared).to receive(:export_path).and_return('whatever') + allow(shared).to receive(:archive_path).and_return('whatever') allow(FileUtils).to receive(:rm_rf) - expect(FileUtils).to receive(:rm_rf).with(shared.export_path) + expect(FileUtils).to receive(:rm_rf).with(shared.archive_path) end it 'notifies the user' do @@ -142,6 +142,21 @@ describe Projects::ImportExport::ExportService do end end + context 'when one of the savers fail unexpectedly' do + let(:archive_path) { shared.archive_path } + + before do + allow(service).to receive_message_chain(:uploads_saver, :save).and_return(false) + end + + it 'removes the remaining exported data' do + expect { service.execute }.to raise_error(Gitlab::ImportExport::Error) + + expect(project.import_export_upload).to be_nil + expect(File.exist?(shared.archive_path)).to eq(false) + end + end + context 'when user does not have admin_project permission' do let!(:another_user) { create(:user) } diff --git a/spec/support/helpers/wiki_helpers.rb b/spec/support/helpers/wiki_helpers.rb index 06cea728b42..86eb1793707 100644 --- a/spec/support/helpers/wiki_helpers.rb +++ b/spec/support/helpers/wiki_helpers.rb @@ -3,6 +3,11 @@ module WikiHelpers extend self + def wait_for_svg_to_be_loaded(example = nil) + # Ensure the SVG is loaded first before clicking the button + find('.svg-content .js-lazy-loaded') if example.nil? || example.metadata.key?(:js) + end + def upload_file_to_wiki(project, user, file_name) opts = { file_name: file_name, diff --git a/spec/support/import_export/common_util.rb b/spec/support/import_export/common_util.rb index 912a8e0a2ab..9281937e4ba 100644 --- a/spec/support/import_export/common_util.rb +++ b/spec/support/import_export/common_util.rb @@ -34,13 +34,13 @@ module ImportExport end def get_project_restorer(project, import_path) - Gitlab::ImportExport::ProjectTreeRestorer.new( + Gitlab::ImportExport::Project::TreeRestorer.new( user: project.creator, shared: get_shared_env(path: import_path), project: project ) end def get_project_saver(project, export_path) - Gitlab::ImportExport::ProjectTreeSaver.new( + Gitlab::ImportExport::Project::TreeSaver.new( project: project, current_user: project.creator, shared: get_shared_env(path: export_path) ) end diff --git a/spec/support/import_export/configuration_helper.rb b/spec/support/import_export/configuration_helper.rb index 27819b5201a..4fe619225bb 100644 --- a/spec/support/import_export/configuration_helper.rb +++ b/spec/support/import_export/configuration_helper.rb @@ -36,8 +36,8 @@ module ConfigurationHelper end def relation_class_for_name(relation_name) - relation_name = Gitlab::ImportExport::ProjectRelationFactory.overrides[relation_name.to_sym] || relation_name - Gitlab::ImportExport::ProjectRelationFactory.relation_class(relation_name) + relation_name = Gitlab::ImportExport::Project::RelationFactory.overrides[relation_name.to_sym] || relation_name + Gitlab::ImportExport::Project::RelationFactory.relation_class(relation_name) end def parsed_attributes(relation_name, attributes, config: Gitlab::ImportExport.config_file) diff --git a/spec/support/shared_examples/features/wiki_file_attachments_shared_examples.rb b/spec/support/shared_examples/features/wiki_file_attachments_shared_examples.rb index 36d91d323b5..867290fb2d6 100644 --- a/spec/support/shared_examples/features/wiki_file_attachments_shared_examples.rb +++ b/spec/support/shared_examples/features/wiki_file_attachments_shared_examples.rb @@ -42,7 +42,7 @@ RSpec.shared_examples 'wiki file attachments' do end end - context 'uploading is complete', :quarantine do + context 'uploading is complete' do it 'shows "Attach a file" button on uploading complete' do attach_with_dropzone wait_for_requests diff --git a/spec/support/shared_examples/lib/gitlab/import_export/project_tree_restorer_shared_examples.rb b/spec/support/shared_examples/lib/gitlab/import_export/project_tree_restorer_shared_examples.rb index 968423176f1..14af98285fc 100644 --- a/spec/support/shared_examples/lib/gitlab/import_export/project_tree_restorer_shared_examples.rb +++ b/spec/support/shared_examples/lib/gitlab/import_export/project_tree_restorer_shared_examples.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -# Shared examples for ProjectTreeRestorer (shared to allow the testing +# Shared examples for Project::TreeRestorer (shared to allow the testing # of EE-specific features) RSpec.shared_examples 'restores project successfully' do |**results| it 'restores the project' do -- cgit v1.2.1