diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/api/api.rb | 9 | ||||
| -rw-r--r-- | lib/api/entities.rb | 5 | ||||
| -rw-r--r-- | lib/api/group_milestones.rb | 14 | ||||
| -rw-r--r-- | lib/api/helpers/notes_helpers.rb | 5 | ||||
| -rw-r--r-- | lib/api/issues.rb | 11 | ||||
| -rw-r--r-- | lib/api/project_milestones.rb | 3 | ||||
| -rw-r--r-- | lib/api/protected_tags.rb | 79 | ||||
| -rw-r--r-- | lib/feature.rb | 3 | ||||
| -rw-r--r-- | lib/gitlab/bitbucket_server_import/importer.rb | 18 | ||||
| -rw-r--r-- | lib/gitlab/github_import.rb | 18 | ||||
| -rw-r--r-- | lib/gitlab/github_import/importer/issue_importer.rb | 4 | ||||
| -rw-r--r-- | lib/gitlab/github_import/importer/pull_request_importer.rb | 92 | ||||
| -rw-r--r-- | lib/gitlab/i18n.rb | 1 | ||||
| -rw-r--r-- | lib/gitlab/import/database_helpers.rb | 25 | ||||
| -rw-r--r-- | lib/gitlab/import/merge_request_creator.rb | 40 | ||||
| -rw-r--r-- | lib/gitlab/import/merge_request_helpers.rb | 70 |
16 files changed, 279 insertions, 118 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index e2ad3c5f4e3..c000666d992 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -99,12 +99,13 @@ module API mount ::API::Features mount ::API::Files mount ::API::GroupBoards - mount ::API::Groups mount ::API::GroupMilestones + mount ::API::Groups + mount ::API::GroupVariables mount ::API::Internal mount ::API::Issues - mount ::API::Jobs mount ::API::JobArtifacts + mount ::API::Jobs mount ::API::Keys mount ::API::Labels mount ::API::Lint @@ -122,11 +123,12 @@ module API mount ::API::ProjectExport mount ::API::ProjectImport mount ::API::ProjectHooks - mount ::API::Projects mount ::API::ProjectMilestones + mount ::API::Projects mount ::API::ProjectSnapshots mount ::API::ProjectSnippets mount ::API::ProtectedBranches + mount ::API::ProtectedTags mount ::API::Repositories mount ::API::Runner mount ::API::Runners @@ -143,7 +145,6 @@ module API mount ::API::Triggers mount ::API::Users mount ::API::Variables - mount ::API::GroupVariables mount ::API::Version mount ::API::Wikis diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 06262f0f991..95b25d7351a 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -429,6 +429,11 @@ module API expose :merge_access_levels, using: Entities::ProtectedRefAccess end + class ProtectedTag < Grape::Entity + expose :name + expose :create_access_levels, using: Entities::ProtectedRefAccess + end + class Milestone < Grape::Entity expose :id, :iid expose :project_id, if: -> (entity, options) { entity&.project_id } diff --git a/lib/api/group_milestones.rb b/lib/api/group_milestones.rb index 93fa0b95857..4b4352c2b27 100644 --- a/lib/api/group_milestones.rb +++ b/lib/api/group_milestones.rb @@ -41,7 +41,7 @@ module API use :optional_params end post ":id/milestones" do - authorize! :admin_milestones, user_group + authorize! :admin_milestone, user_group create_milestone_for(user_group) end @@ -53,11 +53,21 @@ module API use :update_params end put ":id/milestones/:milestone_id" do - authorize! :admin_milestones, user_group + authorize! :admin_milestone, user_group update_milestone_for(user_group) end + desc 'Remove a project milestone' + delete ":id/milestones/:milestone_id" do + authorize! :admin_milestone, user_group + + milestone = user_group.milestones.find(params[:milestone_id]) + Milestones::DestroyService.new(user_group, current_user).execute(milestone) + + status(204) + end + desc 'Get all issues for a single group milestone' do success Entities::IssueBasic end diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb index e2984b08eca..7b1f5c2584b 100644 --- a/lib/api/helpers/notes_helpers.rb +++ b/lib/api/helpers/notes_helpers.rb @@ -92,10 +92,7 @@ module API parent = noteable_parent(noteable) - if opts[:created_at] - opts.delete(:created_at) unless - current_user.admin? || parent.owned_by?(current_user) - end + opts.delete(:created_at) unless current_user.can?(:set_note_created_at, policy_object) opts[:updated_at] = opts[:created_at] if opts[:created_at] diff --git a/lib/api/issues.rb b/lib/api/issues.rb index bda05d1795b..cedfd2fbaa0 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -172,11 +172,8 @@ module API authorize! :create_issue, user_project - # Setting created_at time or iid only allowed for admins and project owners - unless current_user.admin? || user_project.owner == current_user - params.delete(:created_at) - params.delete(:iid) - end + params.delete(:created_at) unless current_user.can?(:set_issue_created_at, user_project) + params.delete(:iid) unless current_user.can?(:set_issue_iid, user_project) issue_params = declared_params(include_missing: false) @@ -216,8 +213,8 @@ module API issue = user_project.issues.find_by!(iid: params.delete(:issue_iid)) authorize! :update_issue, issue - # Setting created_at time only allowed for admins and project owners - unless current_user.admin? || user_project.owner == current_user + # Setting created_at time only allowed for admins and project/group owners + unless current_user.admin? || user_project.owner == current_user || current_user.owned_groups.include?(user_project.owner) params.delete(:updated_at) end diff --git a/lib/api/project_milestones.rb b/lib/api/project_milestones.rb index 306dc0e63d7..72cf32d7717 100644 --- a/lib/api/project_milestones.rb +++ b/lib/api/project_milestones.rb @@ -64,7 +64,8 @@ module API delete ":id/milestones/:milestone_id" do authorize! :admin_milestone, user_project - user_project.milestones.find(params[:milestone_id]).destroy + milestone = user_project.milestones.find(params[:milestone_id]) + Milestones::DestroyService.new(user_project, current_user).execute(milestone) status(204) end diff --git a/lib/api/protected_tags.rb b/lib/api/protected_tags.rb new file mode 100644 index 00000000000..bf0a7184e1c --- /dev/null +++ b/lib/api/protected_tags.rb @@ -0,0 +1,79 @@ +module API + class ProtectedTags < Grape::API + include PaginationParams + + TAG_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(name: API::NO_SLASH_URL_PART_REGEX) + + before { authorize_admin_project } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + desc "Get a project's protected tags" do + detail 'This feature was introduced in GitLab 11.3.' + success Entities::ProtectedTag + end + params do + use :pagination + end + get ':id/protected_tags' do + protected_tags = user_project.protected_tags.preload(:create_access_levels) + + present paginate(protected_tags), with: Entities::ProtectedTag, project: user_project + end + + desc 'Get a single protected tag' do + detail 'This feature was introduced in GitLab 11.3.' + success Entities::ProtectedTag + end + params do + requires :name, type: String, desc: 'The name of the tag or wildcard' + end + get ':id/protected_tags/:name', requirements: TAG_ENDPOINT_REQUIREMENTS do + protected_tag = user_project.protected_tags.find_by!(name: params[:name]) + + present protected_tag, with: Entities::ProtectedTag, project: user_project + end + + desc 'Protect a single tag or wildcard' do + detail 'This feature was introduced in GitLab 11.3.' + success Entities::ProtectedTag + end + params do + requires :name, type: String, desc: 'The name of the protected tag' + optional :create_access_level, type: Integer, default: Gitlab::Access::MAINTAINER, + values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS, + desc: 'Access levels allowed to create (defaults: `40`, maintainer access level)' + end + post ':id/protected_tags' do + protected_tags_params = { + name: params[:name], + create_access_levels_attributes: [{ access_level: params[:create_access_level] }] + } + + protected_tag = ::ProtectedTags::CreateService.new(user_project, + current_user, + protected_tags_params).execute + + if protected_tag.persisted? + present protected_tag, with: Entities::ProtectedTag, project: user_project + else + render_api_error!(protected_tag.errors.full_messages, 422) + end + end + + desc 'Unprotect a single tag' do + detail 'This feature was introduced in GitLab 11.3.' + end + params do + requires :name, type: String, desc: 'The name of the protected tag' + end + delete ':id/protected_tags/:name', requirements: TAG_ENDPOINT_REQUIREMENTS do + protected_tag = user_project.protected_tags.find_by!(name: params[:name]) + + destroy_conditionally!(protected_tag) + end + end + end +end diff --git a/lib/feature.rb b/lib/feature.rb index 09c5ef3ad94..24dbcb32fc0 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -47,7 +47,8 @@ class Feature end def disabled?(key, thing = nil) - !enabled?(key, thing) + # we need to make different method calls to make it easy to mock / define expectations in test mode + thing.nil? ? !enabled?(key) : !enabled?(key, thing) end def enable(key, thing = true) diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb index 268d21a77d1..b591d94668f 100644 --- a/lib/gitlab/bitbucket_server_import/importer.rb +++ b/lib/gitlab/bitbucket_server_import/importer.rb @@ -1,7 +1,10 @@ +# frozen_string_literal: true + module Gitlab module BitbucketServerImport class Importer include Gitlab::ShellAdapter + attr_reader :recover_missing_commits attr_reader :project, :project_key, :repository_slug, :client, :errors, :users @@ -175,21 +178,18 @@ module Gitlab description = '' description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author_email) description += pull_request.description if pull_request.description - - source_branch_sha = pull_request.source_branch_sha - target_branch_sha = pull_request.target_branch_sha author_id = gitlab_user_id(pull_request.author_email) attributes = { iid: pull_request.iid, title: pull_request.title, description: description, - source_project: project, + source_project_id: project.id, source_branch: Gitlab::Git.ref_name(pull_request.source_branch_name), - source_branch_sha: source_branch_sha, - target_project: project, + source_branch_sha: pull_request.source_branch_sha, + target_project_id: project.id, target_branch: Gitlab::Git.ref_name(pull_request.target_branch_name), - target_branch_sha: target_branch_sha, + target_branch_sha: pull_request.target_branch_sha, state: pull_request.state, author_id: author_id, assignee_id: nil, @@ -197,7 +197,9 @@ module Gitlab updated_at: pull_request.updated_at } - merge_request = project.merge_requests.create!(attributes) + creator = Gitlab::Import::MergeRequestCreator.new(project) + merge_request = creator.execute(attributes) + import_pull_request_comments(pull_request, merge_request) if merge_request.persisted? end diff --git a/lib/gitlab/github_import.rb b/lib/gitlab/github_import.rb index 65b5e30c70f..d40b06f969f 100644 --- a/lib/gitlab/github_import.rb +++ b/lib/gitlab/github_import.rb @@ -10,24 +10,6 @@ module Gitlab Client.new(token_to_use, parallel: parallel) end - # Inserts a raw row and returns the ID of the inserted row. - # - # attributes - The attributes/columns to set. - # relation - An ActiveRecord::Relation to use for finding the ID of the row - # when using MySQL. - def self.insert_and_return_id(attributes, relation) - # We use bulk_insert here so we can bypass any queries executed by - # callbacks or validation rules, as doing this wouldn't scale when - # importing very large projects. - result = Gitlab::Database - .bulk_insert(relation.table_name, [attributes], return_ids: true) - - # MySQL doesn't support returning the IDs of a bulk insert in a way that - # is not a pain, so in this case we'll issue an extra query instead. - result.first || - relation.where(iid: attributes[:iid]).limit(1).pluck(:id).first - end - # Returns the ID of the ghost user. def self.ghost_user_id key = 'github-import/ghost-user-id' diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb index cb4d7a6a0b6..4226eee85cc 100644 --- a/lib/gitlab/github_import/importer/issue_importer.rb +++ b/lib/gitlab/github_import/importer/issue_importer.rb @@ -4,6 +4,8 @@ module Gitlab module GithubImport module Importer class IssueImporter + include Gitlab::Import::DatabaseHelpers + attr_reader :project, :issue, :client, :user_finder, :milestone_finder, :issuable_finder @@ -55,7 +57,7 @@ module Gitlab updated_at: issue.updated_at } - GithubImport.insert_and_return_id(attributes, project.issues).tap do |id| + insert_and_return_id(attributes, project.issues).tap do |id| # We use .insert_and_return_id which effectively disables all callbacks. # Trigger iid logic here to make sure we track internal id values consistently. project.issues.find(id).ensure_project_iid! diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb index ed17aa54373..ae7c4cf1b38 100644 --- a/lib/gitlab/github_import/importer/pull_request_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_importer.rb @@ -4,6 +4,8 @@ module Gitlab module GithubImport module Importer class PullRequestImporter + include Gitlab::Import::MergeRequestHelpers + attr_reader :pull_request, :project, :client, :user_finder, :milestone_finder, :issuable_finder @@ -44,81 +46,27 @@ module Gitlab description = MarkdownText .format(pull_request.description, pull_request.author, author_found) - # This work must be wrapped in a transaction as otherwise we can leave - # behind incomplete data in the event of an error. This can then lead - # to duplicate key errors when jobs are retried. - MergeRequest.transaction do - attributes = { - iid: pull_request.iid, - title: pull_request.truncated_title, - description: description, - source_project_id: project.id, - target_project_id: project.id, - source_branch: pull_request.formatted_source_branch, - target_branch: pull_request.target_branch, - state: pull_request.state, - milestone_id: milestone_finder.id_for(pull_request), - author_id: author_id, - assignee_id: user_finder.assignee_id_for(pull_request), - created_at: pull_request.created_at, - updated_at: pull_request.updated_at - } - - # When creating merge requests there are a lot of hooks that may - # run, for many different reasons. Many of these hooks (e.g. the - # ones used for rendering Markdown) are completely unnecessary and - # may even lead to transaction timeouts. - # - # To ensure importing pull requests has a minimal impact and can - # complete in a reasonable time we bypass all the hooks by inserting - # the row and then retrieving it. We then only perform the - # additional work that is strictly necessary. - merge_request_id = GithubImport - .insert_and_return_id(attributes, project.merge_requests) - - merge_request = project.merge_requests.find(merge_request_id) - - # We use .insert_and_return_id which effectively disables all callbacks. - # Trigger iid logic here to make sure we track internal id values consistently. - merge_request.ensure_target_project_iid! + attributes = { + iid: pull_request.iid, + title: pull_request.truncated_title, + description: description, + source_project_id: project.id, + target_project_id: project.id, + source_branch: pull_request.formatted_source_branch, + target_branch: pull_request.target_branch, + state: pull_request.state, + milestone_id: milestone_finder.id_for(pull_request), + author_id: author_id, + assignee_id: user_finder.assignee_id_for(pull_request), + created_at: pull_request.created_at, + updated_at: pull_request.updated_at + } - [merge_request, false] - end - rescue ActiveRecord::InvalidForeignKey - # It's possible the project has been deleted since scheduling this - # job. In this case we'll just skip creating the merge request. - [] - rescue ActiveRecord::RecordNotUnique - # It's possible we previously created the MR, but failed when updating - # the Git data. In this case we'll just continue working on the - # existing row. - [project.merge_requests.find_by(iid: pull_request.iid), true] + create_merge_request_without_hooks(project, attributes, pull_request.iid) end - def insert_git_data(merge_request, already_exists = false) - # These fields are set so we can create the correct merge request - # diffs. - merge_request.source_branch_sha = pull_request.source_branch_sha - merge_request.target_branch_sha = pull_request.target_branch_sha - - merge_request.keep_around_commit - - # MR diffs normally use an "after_save" hook to pull data from Git. - # All of this happens in the transaction started by calling - # create/save/etc. This in turn can lead to these transactions being - # held open for much longer than necessary. To work around this we - # first save the diff, then populate it. - diff = - if already_exists - merge_request.merge_request_diffs.take || - merge_request.merge_request_diffs.build - else - merge_request.merge_request_diffs.build - end - - diff.importing = true - diff.save - diff.save_git_content + def insert_git_data(merge_request, already_exists) + insert_or_replace_git_data(merge_request, pull_request.source_branch_sha, pull_request.target_branch_sha, already_exists) end end end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index b8213929c6a..7346eab9e76 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -5,6 +5,7 @@ module Gitlab AVAILABLE_LANGUAGES = { 'en' => 'English', 'es' => 'Español', + 'gl_ES' => 'Galego', 'de' => 'Deutsch', 'fr' => 'Français', 'pt_BR' => 'Português (Brasil)', diff --git a/lib/gitlab/import/database_helpers.rb b/lib/gitlab/import/database_helpers.rb new file mode 100644 index 00000000000..80857061933 --- /dev/null +++ b/lib/gitlab/import/database_helpers.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Import + module DatabaseHelpers + # Inserts a raw row and returns the ID of the inserted row. + # + # attributes - The attributes/columns to set. + # relation - An ActiveRecord::Relation to use for finding the ID of the row + # when using MySQL. + def insert_and_return_id(attributes, relation) + # We use bulk_insert here so we can bypass any queries executed by + # callbacks or validation rules, as doing this wouldn't scale when + # importing very large projects. + result = Gitlab::Database + .bulk_insert(relation.table_name, [attributes], return_ids: true) + + # MySQL doesn't support returning the IDs of a bulk insert in a way that + # is not a pain, so in this case we'll issue an extra query instead. + result.first || + relation.where(iid: attributes[:iid]).limit(1).pluck(:id).first + end + end + end +end diff --git a/lib/gitlab/import/merge_request_creator.rb b/lib/gitlab/import/merge_request_creator.rb new file mode 100644 index 00000000000..a01951b0762 --- /dev/null +++ b/lib/gitlab/import/merge_request_creator.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +# This module is designed for importers that need to create many merge +# requests quickly. When creating merge requests there are a lot of hooks +# that may run, for many different reasons. Many of these hooks (e.g. the ones +# used for rendering Markdown) are completely unnecessary and may even lead to +# transaction timeouts. +# +# To ensure importing merge requests requests has a minimal impact and can +# complete in a reasonable time we bypass all the hooks by inserting the row +# and then retrieving it. We then only perform the additional work that is +# strictly necessary. +module Gitlab + module Import + class MergeRequestCreator + include ::Gitlab::Import::DatabaseHelpers + include ::Gitlab::Import::MergeRequestHelpers + + attr_accessor :project + + def initialize(project) + @project = project + end + + def execute(attributes) + source_branch_sha = attributes.delete(:source_branch_sha) + target_branch_sha = attributes.delete(:target_branch_sha) + iid = attributes[:iid] + + merge_request, already_exists = create_merge_request_without_hooks(project, attributes, iid) + + if merge_request + insert_or_replace_git_data(merge_request, source_branch_sha, target_branch_sha, already_exists) + end + + merge_request + end + end + end +end diff --git a/lib/gitlab/import/merge_request_helpers.rb b/lib/gitlab/import/merge_request_helpers.rb new file mode 100644 index 00000000000..8ba70700dc1 --- /dev/null +++ b/lib/gitlab/import/merge_request_helpers.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module Import + module MergeRequestHelpers + include DatabaseHelpers + + def create_merge_request_without_hooks(project, attributes, iid) + # This work must be wrapped in a transaction as otherwise we can leave + # behind incomplete data in the event of an error. This can then lead + # to duplicate key errors when jobs are retried. + MergeRequest.transaction do + # When creating merge requests there are a lot of hooks that may + # run, for many different reasons. Many of these hooks (e.g. the + # ones used for rendering Markdown) are completely unnecessary and + # may even lead to transaction timeouts. + # + # To ensure importing pull requests has a minimal impact and can + # complete in a reasonable time we bypass all the hooks by inserting + # the row and then retrieving it. We then only perform the + # additional work that is strictly necessary. + merge_request_id = insert_and_return_id(attributes, project.merge_requests) + + merge_request = project.merge_requests.find(merge_request_id) + + # We use .insert_and_return_id which effectively disables all callbacks. + # Trigger iid logic here to make sure we track internal id values consistently. + merge_request.ensure_target_project_iid! + + [merge_request, false] + end + rescue ActiveRecord::InvalidForeignKey + # It's possible the project has been deleted since scheduling this + # job. In this case we'll just skip creating the merge request. + [] + rescue ActiveRecord::RecordNotUnique + # It's possible we previously created the MR, but failed when updating + # the Git data. In this case we'll just continue working on the + # existing row. + [project.merge_requests.find_by(iid: iid), true] + end + + def insert_or_replace_git_data(merge_request, source_branch_sha, target_branch_sha, already_exists = false) + # These fields are set so we can create the correct merge request + # diffs. + merge_request.source_branch_sha = source_branch_sha + merge_request.target_branch_sha = target_branch_sha + + merge_request.keep_around_commit + + # MR diffs normally use an "after_save" hook to pull data from Git. + # All of this happens in the transaction started by calling + # create/save/etc. This in turn can lead to these transactions being + # held open for much longer than necessary. To work around this we + # first save the diff, then populate it. + diff = + if already_exists + merge_request.merge_request_diffs.take || + merge_request.merge_request_diffs.build + else + merge_request.merge_request_diffs.build + end + + diff.importing = true + diff.save + diff.save_git_content + end + end + end +end |