diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-10-21 07:08:36 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-10-21 07:08:36 +0000 |
commit | 48aff82709769b098321c738f3444b9bdaa694c6 (patch) | |
tree | e00c7c43e2d9b603a5a6af576b1685e400410dee /app/services/design_management | |
parent | 879f5329ee916a948223f8f43d77fba4da6cd028 (diff) | |
download | gitlab-ce-48aff82709769b098321c738f3444b9bdaa694c6.tar.gz |
Add latest changes from gitlab-org/gitlab@13-5-stable-eev13.5.0-rc42
Diffstat (limited to 'app/services/design_management')
8 files changed, 387 insertions, 7 deletions
diff --git a/app/services/design_management/copy_design_collection.rb b/app/services/design_management/copy_design_collection.rb new file mode 100644 index 00000000000..66cf6112062 --- /dev/null +++ b/app/services/design_management/copy_design_collection.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +module DesignManagement + module CopyDesignCollection + end +end diff --git a/app/services/design_management/copy_design_collection/copy_service.rb b/app/services/design_management/copy_design_collection/copy_service.rb new file mode 100644 index 00000000000..5099c2c5704 --- /dev/null +++ b/app/services/design_management/copy_design_collection/copy_service.rb @@ -0,0 +1,309 @@ +# frozen_string_literal: true + +# Service to copy a DesignCollection from one Issue to another. +# Copies the DesignCollection's Designs, Versions, and Notes on Designs. +module DesignManagement + module CopyDesignCollection + class CopyService < DesignService + # rubocop: disable CodeReuse/ActiveRecord + def initialize(project, user, params = {}) + super + + @target_issue = params.fetch(:target_issue) + @target_project = @target_issue.project + @target_repository = @target_project.design_repository + @target_design_collection = @target_issue.design_collection + @temporary_branch = "CopyDesignCollectionService_#{SecureRandom.hex}" + # The user who triggered the copy may not have permissions to push + # to the design repository. + @git_user = @target_project.default_owner + + @designs = DesignManagement::Design.unscoped.where(issue: issue).order(:id).load + @versions = DesignManagement::Version.unscoped.where(issue: issue).order(:id).includes(:designs).load + + @sha_attribute = Gitlab::Database::ShaAttribute.new + @shas = [] + @event_enum_map = DesignManagement::DesignAction::EVENT_FOR_GITALY_ACTION.invert + end + # rubocop: enable CodeReuse/ActiveRecord + + def execute + return error('User cannot copy design collection to issue') unless user_can_copy? + return error('Target design collection must first be queued') unless target_design_collection.copy_in_progress? + return error('Design collection has no designs') if designs.empty? + return error('Target design collection already has designs') unless target_design_collection.empty? + + with_temporary_branch do + copy_commits! + + ActiveRecord::Base.transaction do + design_ids = copy_designs! + version_ids = copy_versions! + copy_actions!(design_ids, version_ids) + link_lfs_files! + copy_notes!(design_ids) + finalize! + end + end + + ServiceResponse.success + rescue => error + log_exception(error) + + target_design_collection.error_copy! + + error('Designs were unable to be copied successfully') + end + + private + + attr_reader :designs, :event_enum_map, :git_user, :sha_attribute, :shas, + :temporary_branch, :target_design_collection, :target_issue, + :target_repository, :target_project, :versions + + alias_method :merge_branch, :target_branch + + def log_exception(exception) + payload = { + issue_id: issue.id, + project_id: project.id, + target_issue_id: target_issue.id, + target_project: target_project.id + } + + Gitlab::ErrorTracking.track_exception(exception, payload) + end + + def error(message) + ServiceResponse.error(message: message) + end + + def user_can_copy? + current_user.can?(:read_design, design_collection) && + current_user.can?(:admin_issue, target_issue) + end + + def with_temporary_branch(&block) + target_repository.create_if_not_exists + + create_master_branch! if target_repository.empty? + create_temporary_branch! + + yield + ensure + remove_temporary_branch! + end + + # A project that does not have any designs will have a blank design + # repository. To create a temporary branch from `master` we need + # create `master` first by adding a file to it. + def create_master_branch! + target_repository.create_file( + git_user, + ".CopyDesignCollectionService_#{Time.now.to_i}", + '.gitlab', + message: "Commit to create #{merge_branch} branch in CopyDesignCollectionService", + branch_name: merge_branch + ) + end + + def create_temporary_branch! + target_repository.add_branch( + git_user, + temporary_branch, + target_repository.root_ref + ) + end + + def remove_temporary_branch! + return unless target_repository.branch_exists?(temporary_branch) + + target_repository.rm_branch(git_user, temporary_branch) + end + + # Merge the temporary branch containing the commits to `master` + # and update the state of the target_design_collection. + def finalize! + source_sha = shas.last + + target_repository.raw.merge( + git_user, + source_sha, + merge_branch, + 'CopyDesignCollectionService finalize merge' + ) { nil } + + target_design_collection.end_copy! + end + + # rubocop: disable CodeReuse/ActiveRecord + def copy_commits! + # Execute another query to include actions and their designs + DesignManagement::Version.unscoped.where(id: versions).order(:id).includes(actions: :design).find_each(batch_size: 100) do |version| + gitaly_actions = version.actions.map do |action| + design = action.design + # Map the raw Action#event enum value to a Gitaly "action" for the + # `Repository#multi_action` call. + gitaly_action_name = @event_enum_map[action.event_before_type_cast] + # `content` will be the LfsPointer file and not the design file, + # and can be nil for deletions. + content = blobs.dig(version.sha, design.filename)&.data + file_path = DesignManagement::Design.build_full_path(target_issue, design) + + { + action: gitaly_action_name, + file_path: file_path, + content: content + }.compact + end + + sha = target_repository.multi_action( + git_user, + branch_name: temporary_branch, + message: commit_message(version), + actions: gitaly_actions + ) + + shas << sha + end + end + # rubocop: enable CodeReuse/ActiveRecord + + def copy_designs! + design_attributes = attributes_config[:design_attributes] + + new_rows = designs.map do |design| + design.attributes.slice(*design_attributes).merge( + issue_id: target_issue.id, + project_id: target_project.id + ) + end + + # TODO Replace `Gitlab::Database.bulk_insert` with `BulkInsertSafe` + # once https://gitlab.com/gitlab-org/gitlab/-/issues/247718 is fixed. + ::Gitlab::Database.bulk_insert( # rubocop:disable Gitlab/BulkInsert + DesignManagement::Design.table_name, + new_rows, + return_ids: true + ) + end + + def copy_versions! + version_attributes = attributes_config[:version_attributes] + # `shas` are the list of Git commits made during the Git copy phase, + # and will be ordered 1:1 with old versions + shas_enum = shas.to_enum + + new_rows = versions.map do |version| + version.attributes.slice(*version_attributes).merge( + issue_id: target_issue.id, + sha: sha_attribute.serialize(shas_enum.next) + ) + end + + # TODO Replace `Gitlab::Database.bulk_insert` with `BulkInsertSafe` + # once https://gitlab.com/gitlab-org/gitlab/-/issues/247718 is fixed. + ::Gitlab::Database.bulk_insert( # rubocop:disable Gitlab/BulkInsert + DesignManagement::Version.table_name, + new_rows, + return_ids: true + ) + end + + # rubocop: disable CodeReuse/ActiveRecord + def copy_actions!(new_design_ids, new_version_ids) + # Create a map of <Old design id> => <New design id> + design_id_map = new_design_ids.each_with_index.to_h do |design_id, i| + [designs[i].id, design_id] + end + + # Create a map of <Old version id> => <New version id> + version_id_map = new_version_ids.each_with_index.to_h do |version_id, i| + [versions[i].id, version_id] + end + + actions = DesignManagement::Action.unscoped.select(:design_id, :version_id, :event).where(design: designs, version: versions) + + new_rows = actions.map do |action| + { + design_id: design_id_map[action.design_id], + version_id: version_id_map[action.version_id], + event: action.event_before_type_cast + } + end + + # We cannot use `BulkInsertSafe` because of the uploader mounted in `Action`. + ::Gitlab::Database.bulk_insert( # rubocop:disable Gitlab/BulkInsert + DesignManagement::Action.table_name, + new_rows + ) + end + # rubocop: enable CodeReuse/ActiveRecord + + def commit_message(version) + "Copy commit #{version.sha} from issue #{issue.to_reference(full: true)}" + end + + # rubocop: disable CodeReuse/ActiveRecord + def copy_notes!(design_ids) + new_designs = DesignManagement::Design.unscoped.find(design_ids) + + # Execute another query to filter only designs with notes + DesignManagement::Design.unscoped.where(id: designs).joins(:notes).distinct.find_each(batch_size: 100) do |old_design| + new_design = new_designs.find { |d| d.filename == old_design.filename } + + Notes::CopyService.new(current_user, old_design, new_design).execute + end + end + # rubocop: enable CodeReuse/ActiveRecord + + # rubocop: disable CodeReuse/ActiveRecord + def link_lfs_files! + oids = blobs.values.flat_map(&:values).map(&:lfs_oid) + repository_type = LfsObjectsProject.repository_types[:design] + + new_rows = LfsObject.where(oid: oids).find_each(batch_size: 1000).map do |lfs_object| + { + project_id: target_project.id, + lfs_object_id: lfs_object.id, + repository_type: repository_type + } + end + + # We cannot use `BulkInsertSafe` due to the LfsObjectsProject#update_project_statistics + # callback that fires after_commit. + ::Gitlab::Database.bulk_insert( # rubocop:disable Gitlab/BulkInsert + LfsObjectsProject.table_name, + new_rows, + on_conflict: :do_nothing # Upsert + ) + end + # rubocop: enable CodeReuse/ActiveRecord + + # Blob data is used to find the oids for LfsObjects and to copy to Git. + # Blobs are reasonably small in memory, as their data are LFS Pointer files. + # + # Returns all blobs for the designs as a Hash of `{ Blob#commit_id => { Design#filename => Blob } }` + def blobs + @blobs ||= begin + items = versions.flat_map { |v| v.designs.map { |d| [v.sha, DesignManagement::Design.build_full_path(issue, d)] } } + + repository.blobs_at(items).each_with_object({}) do |blob, h| + design = designs.find { |d| DesignManagement::Design.build_full_path(issue, d) == blob.path } + + h[blob.commit_id] ||= {} + h[blob.commit_id][design.filename] = blob + end + end + end + + def attributes_config + @attributes_config ||= YAML.load_file(attributes_config_file).symbolize_keys + end + + def attributes_config_file + Rails.root.join('lib/gitlab/design_management/copy_design_collection_model_attributes.yml') + end + end + end +end diff --git a/app/services/design_management/copy_design_collection/queue_service.rb b/app/services/design_management/copy_design_collection/queue_service.rb new file mode 100644 index 00000000000..f76917dbe47 --- /dev/null +++ b/app/services/design_management/copy_design_collection/queue_service.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +# Service for setting the initial copy_state on the target DesignCollection +# and queuing a CopyDesignCollectionWorker. +module DesignManagement + module CopyDesignCollection + class QueueService + def initialize(current_user, issue, target_issue) + @current_user = current_user + @issue = issue + @target_issue = target_issue + @target_design_collection = target_issue.design_collection + end + + def execute + return error('User cannot copy designs to issue') unless user_can_copy? + return error('Target design collection copy state must be `ready`') unless target_design_collection.can_start_copy? + + target_design_collection.start_copy! + + DesignManagement::CopyDesignCollectionWorker.perform_async(current_user.id, issue.id, target_issue.id) + + ServiceResponse.success + end + + private + + delegate :design_collection, to: :issue + + attr_reader :current_user, :issue, :target_design_collection, :target_issue + + def error(message) + ServiceResponse.error(message: message) + end + + def user_can_copy? + current_user.can?(:read_design, issue) && + current_user.can?(:admin_issue, target_issue) + end + end + end +end diff --git a/app/services/design_management/delete_designs_service.rb b/app/services/design_management/delete_designs_service.rb index 5d875c630a0..a90c34d4e34 100644 --- a/app/services/design_management/delete_designs_service.rb +++ b/app/services/design_management/delete_designs_service.rb @@ -16,6 +16,7 @@ module DesignManagement version = delete_designs! EventCreateService.new.destroy_designs(designs, current_user) + Gitlab::UsageDataCounters::IssueActivityUniqueCounter.track_issue_designs_removed_action(author: current_user) success(version: version) end diff --git a/app/services/design_management/design_service.rb b/app/services/design_management/design_service.rb index 54e53609646..5aa2a2f73bc 100644 --- a/app/services/design_management/design_service.rb +++ b/app/services/design_management/design_service.rb @@ -19,6 +19,7 @@ module DesignManagement def collection issue.design_collection end + alias_method :design_collection, :collection def repository collection.repository diff --git a/app/services/design_management/generate_image_versions_service.rb b/app/services/design_management/generate_image_versions_service.rb index 213aac164ff..e56d163c461 100644 --- a/app/services/design_management/generate_image_versions_service.rb +++ b/app/services/design_management/generate_image_versions_service.rb @@ -48,6 +48,9 @@ module DesignManagement # Store and process the file action.image_v432x230.store!(raw_file) action.save! + rescue CarrierWave::IntegrityError => e + Gitlab::ErrorTracking.log_exception(e, project_id: project.id, design_id: action.design_id, version_id: action.version_id) + log_error(e.message) rescue CarrierWave::UploadError => e Gitlab::ErrorTracking.track_exception(e, project_id: project.id, design_id: action.design_id, version_id: action.version_id) log_error(e.message) diff --git a/app/services/design_management/runs_design_actions.rb b/app/services/design_management/runs_design_actions.rb index 4bd6bb45658..ee6aa9286d3 100644 --- a/app/services/design_management/runs_design_actions.rb +++ b/app/services/design_management/runs_design_actions.rb @@ -4,14 +4,15 @@ module DesignManagement module RunsDesignActions NoActions = Class.new(StandardError) - # this concern requires the following methods to be implemented: + # This concern requires the following methods to be implemented: # current_user, target_branch, repository, commit_message # # Before calling `run_actions`, you should ensure the repository exists, by # calling `repository.create_if_not_exists`. # # @raise [NoActions] if actions are empty - def run_actions(actions) + # @return [DesignManagement::Version] + def run_actions(actions, skip_system_notes: false) raise NoActions if actions.empty? sha = repository.multi_action(current_user, @@ -21,14 +22,14 @@ module DesignManagement ::DesignManagement::Version .create_for_designs(actions, sha, current_user) - .tap { |version| post_process(version) } + .tap { |version| post_process(version, skip_system_notes) } end private - def post_process(version) + def post_process(version, skip_system_notes) version.run_after_commit_or_now do - ::DesignManagement::NewVersionWorker.perform_async(id) + ::DesignManagement::NewVersionWorker.perform_async(id, skip_system_notes) end end end diff --git a/app/services/design_management/save_designs_service.rb b/app/services/design_management/save_designs_service.rb index 0446d2f1ee8..c26d2e7ab47 100644 --- a/app/services/design_management/save_designs_service.rb +++ b/app/services/design_management/save_designs_service.rb @@ -16,11 +16,15 @@ module DesignManagement def execute return error("Not allowed!") unless can_create_designs? return error("Only #{MAX_FILES} files are allowed simultaneously") if files.size > MAX_FILES + return error("Duplicate filenames are not allowed!") if files.map(&:original_filename).uniq.length != files.length + return error("Design copy is in progress") if design_collection.copy_in_progress? uploaded_designs, version = upload_designs! skipped_designs = designs - uploaded_designs create_events + design_collection.reset_copy! + success({ designs: uploaded_designs, version: version, skipped_designs: skipped_designs }) rescue ::ActiveRecord::RecordInvalid => e error(e.message) @@ -34,7 +38,10 @@ module DesignManagement ::DesignManagement::Version.with_lock(project.id, repository) do actions = build_actions - [actions.map(&:design), actions.presence && run_actions(actions)] + [ + actions.map(&:design), + actions.presence && run_actions(actions) + ] end end @@ -59,7 +66,7 @@ module DesignManagement action = new_file?(design) ? :create : :update on_success do - ::Gitlab::UsageDataCounters::DesignsCounter.count(action) + track_usage_metrics(action) end DesignManagement::DesignAction.new(design, action, content) @@ -121,6 +128,16 @@ module DesignManagement end end end + + def track_usage_metrics(action) + if action == :update + ::Gitlab::UsageDataCounters::IssueActivityUniqueCounter.track_issue_designs_modified_action(author: current_user) + else + ::Gitlab::UsageDataCounters::IssueActivityUniqueCounter.track_issue_designs_added_action(author: current_user) + end + + ::Gitlab::UsageDataCounters::DesignsCounter.count(action) + end end end |