diff options
| author | Simon Knox <psimyn@gmail.com> | 2017-09-06 14:35:58 +1000 |
|---|---|---|
| committer | Simon Knox <psimyn@gmail.com> | 2017-09-06 14:35:58 +1000 |
| commit | b9aa55e1ea2ba226bd9bf4c6fb08fdec30e046c5 (patch) | |
| tree | b7770180f178086c78ef2ca25d6bb2267f739110 /lib | |
| parent | 74740604211dab6632771f1bfd7dd67902fea7ef (diff) | |
| parent | d68ff7f50a93ebbff537b5e795cf6bf80bd66a6e (diff) | |
| download | gitlab-ce-b9aa55e1ea2ba226bd9bf4c6fb08fdec30e046c5.tar.gz | |
Merge branch 'master' of gitlab.com:gitlab-org/gitlab-ce into ee_issue_928_backport
Diffstat (limited to 'lib')
117 files changed, 1997 insertions, 410 deletions
diff --git a/lib/after_commit_queue.rb b/lib/after_commit_queue.rb index b67575a3ac2..4750a2c373a 100644 --- a/lib/after_commit_queue.rb +++ b/lib/after_commit_queue.rb @@ -7,7 +7,7 @@ module AfterCommitQueue end def run_after_commit(method = nil, &block) - _after_commit_queue << proc { self.send(method) } if method + _after_commit_queue << proc { self.send(method) } if method # rubocop:disable GitlabSecurity/PublicSend _after_commit_queue << block if block true end diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index cdacf9839e5..374b611f55e 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -10,7 +10,7 @@ module API params do requires :id, type: String, desc: "The #{source_type} ID" end - resource source_type.pluralize, requirements: { id: %r{[^/]+} } do + resource source_type.pluralize, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc "Gets a list of access requests for a #{source_type}." do detail 'This feature was introduced in GitLab 8.11.' success Entities::AccessRequester @@ -67,10 +67,12 @@ module API end delete ":id/access_requests/:user_id" do source = find_source(source_type, params[:id]) + member = source.requesters.find_by!(user_id: params[:user_id]) - status 204 - ::Members::DestroyService.new(source, current_user, params) - .execute(:requesters) + destroy_conditionally!(member) do + ::Members::DestroyService.new(source, current_user, params) + .execute(:requesters) + end end end end diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index 5a028fc9d0b..c3d93996816 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -12,7 +12,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do AWARDABLES.each do |awardable_params| awardable_string = awardable_params[:type].pluralize awardable_id_string = "#{awardable_params[:type]}_#{awardable_params[:find_by]}" @@ -88,8 +88,7 @@ module API unauthorized! unless award.user == current_user || current_user.admin? - status 204 - award.destroy + destroy_conditionally!(award) end end end diff --git a/lib/api/boards.rb b/lib/api/boards.rb index 5a2d7a681e3..366b0dc9a6f 100644 --- a/lib/api/boards.rb +++ b/lib/api/boards.rb @@ -7,7 +7,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get all project boards' do detail 'This feature was introduced in 8.13' success Entities::Board @@ -122,13 +122,13 @@ module API end delete "/lists/:list_id" do authorize!(:admin_list, user_project) - list = board_lists.find(params[:list_id]) - service = ::Boards::Lists::DestroyService.new(user_project, current_user) - - unless service.execute(list) - render_api_error!({ error: 'List could not be deleted!' }, 400) + destroy_conditionally!(list) do |list| + service = ::Boards::Lists::DestroyService.new(user_project, current_user) + unless service.execute(list) + render_api_error!({ error: 'List could not be deleted!' }, 400) + end end end end diff --git a/lib/api/branches.rb b/lib/api/branches.rb index d3dbf941298..642c1140fcc 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -24,17 +24,22 @@ module API present paginate(branches), with: Entities::RepoBranch, project: user_project end - desc 'Get a single branch' do - success Entities::RepoBranch - end - params do - requires :branch, type: String, desc: 'The name of the branch' - end - get ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do - branch = user_project.repository.find_branch(params[:branch]) - not_found!("Branch") unless branch + resource ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do + desc 'Get a single branch' do + success Entities::RepoBranch + end + params do + requires :branch, type: String, desc: 'The name of the branch' + end + head do + user_project.repository.branch_exists?(params[:branch]) ? status(204) : status(404) + end + get do + branch = user_project.repository.find_branch(params[:branch]) + not_found!('Branch') unless branch - present branch, with: Entities::RepoBranch, project: user_project + present branch, with: Entities::RepoBranch, project: user_project + end end # Note: This API will be deprecated in favor of the protected branches API. @@ -125,11 +130,18 @@ module API delete ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do authorize_push_project - result = DeleteBranchService.new(user_project, current_user) - .execute(params[:branch]) + branch = user_project.repository.find_branch(params[:branch]) + not_found!('Branch') unless branch + + commit = user_project.repository.commit(branch.dereferenced_target) + + destroy_conditionally!(commit, last_updated: commit.authored_date) do + result = DeleteBranchService.new(user_project, current_user) + .execute(params[:branch]) - if result[:status] != :success - render_api_error!(result[:message], result[:return_code]) + if result[:status] != :success + render_api_error!(result[:message], result[:return_code]) + end end end diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb index 9980aec4752..0b45621ce7b 100644 --- a/lib/api/broadcast_messages.rb +++ b/lib/api/broadcast_messages.rb @@ -91,8 +91,7 @@ module API delete ':id' do message = find_message - status 204 - message.destroy + destroy_conditionally!(message) end end end diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 485b680cd5f..829eef18795 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -5,7 +5,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do include PaginationParams before { authenticate! } @@ -74,7 +74,8 @@ module API source: :external, sha: commit.sha, ref: ref, - user: current_user) + user: current_user, + protected: @project.protected_for?(ref)) end status = GenericCommitStatus.running_or_pending.find_or_initialize_by( @@ -82,7 +83,8 @@ module API pipeline: pipeline, name: name, ref: ref, - user: current_user + user: current_user, + protected: @project.protected_for?(ref) ) optional_attributes = @@ -101,7 +103,7 @@ module API when 'success' status.success! when 'failed' - status.drop! + status.drop!(:api_failure) when 'canceled' status.cancel! else diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index 42e7c1486b0..281269b1190 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -17,7 +17,7 @@ module API params do requires :id, type: String, desc: 'The ID of the project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do before { authorize_admin_project } desc "Get a specific project's deploy keys" do @@ -125,8 +125,7 @@ module API key = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id]) not_found!('Deploy Key') unless key - status 204 - key.destroy + destroy_conditionally!(key) end end end diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index 46b936897f6..1efee9a1324 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -8,7 +8,7 @@ module API params do requires :id, type: String, desc: 'The project ID' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get all deployments of the project' do detail 'This feature was introduced in GitLab 8.11.' success Entities::Deployment diff --git a/lib/api/entities.rb b/lib/api/entities.rb index e8dd61e493f..031dd02c6eb 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -1,11 +1,11 @@ module API module Entities class UserSafe < Grape::Entity - expose :name, :username + expose :id, :name, :username end class UserBasic < UserSafe - expose :id, :state + expose :state expose :avatar_url do |user, options| user.avatar_url(only_path: false) end @@ -320,7 +320,10 @@ module API end class IssueBasic < ProjectEntity - expose :label_names, as: :labels + expose :labels do |issue, options| + # Avoids an N+1 query since labels are preloaded + issue.labels.map(&:title).sort + end expose :milestone, using: Entities::Milestone expose :assignees, :author, using: Entities::UserBasic @@ -329,13 +332,32 @@ module API end expose :user_notes_count - expose :upvotes, :downvotes + expose :upvotes do |issue, options| + if options[:issuable_metadata] + # Avoids an N+1 query when metadata is included + options[:issuable_metadata][issue.id].upvotes + else + issue.upvotes + end + end + expose :downvotes do |issue, options| + if options[:issuable_metadata] + # Avoids an N+1 query when metadata is included + options[:issuable_metadata][issue.id].downvotes + else + issue.downvotes + end + end expose :due_date expose :confidential expose :web_url do |issue, options| Gitlab::UrlBuilder.build(issue) end + + expose :time_stats, using: 'API::Entities::IssuableTimeStats' do |issue| + issue + end end class Issue < IssueBasic @@ -365,10 +387,22 @@ module API end class IssuableTimeStats < Grape::Entity + format_with(:time_tracking_formatter) do |time_spent| + Gitlab::TimeTrackingFormatter.output(time_spent) + end + expose :time_estimate expose :total_time_spent expose :human_time_estimate - expose :human_total_time_spent + + with_options(format_with: :time_tracking_formatter) do + expose :total_time_spent, as: :human_total_time_spent + end + + def total_time_spent + # Avoids an N+1 query since timelogs are preloaded + object.timelogs.map(&:time_spent).sum + end end class ExternalIssue < Grape::Entity @@ -418,6 +452,10 @@ module API expose :web_url do |merge_request, options| Gitlab::UrlBuilder.build(merge_request) end + + expose :time_stats, using: 'API::Entities::IssuableTimeStats' do |merge_request| + merge_request + end end class MergeRequest < MergeRequestBasic @@ -453,6 +491,10 @@ module API expose :user, using: Entities::UserPublic end + class GPGKey < Grape::Entity + expose :id, :key, :created_at + end + class Note < Grape::Entity # Only Issue and MergeRequest have iid NOTEABLE_TYPES_WITH_IID = %w(Issue MergeRequest).freeze @@ -737,6 +779,7 @@ module API expose :tag_list expose :run_untagged expose :locked + expose :access_level expose :version, :revision, :platform, :architecture expose :contacted_at expose :token, if: lambda { |runner, options| options[:current_user].admin? || !runner.is_shared? } @@ -780,7 +823,7 @@ module API class Variable < Grape::Entity expose :key, :value - expose :protected?, as: :protected + expose :protected?, as: :protected, if: -> (entity, _) { entity.respond_to?(:protected?) } end class Pipeline < PipelineBasic @@ -801,6 +844,7 @@ module API class PipelineScheduleDetails < PipelineSchedule expose :last_pipeline, using: Entities::PipelineBasic + expose :variables, using: Entities::Variable end class EnvironmentBasic < Grape::Entity diff --git a/lib/api/environments.rb b/lib/api/environments.rb index c774a5c6685..5c63ec028d9 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -9,7 +9,7 @@ module API params do requires :id, type: String, desc: 'The project ID' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get all environments of the project' do detail 'This feature was introduced in GitLab 8.11.' success Entities::Environment @@ -79,8 +79,7 @@ module API environment = user_project.environments.find(params[:environment_id]) - status 204 - environment.destroy + destroy_conditionally!(environment) end desc 'Stops an existing environment' do diff --git a/lib/api/events.rb b/lib/api/events.rb index dabdf579119..b0713ff1d54 100644 --- a/lib/api/events.rb +++ b/lib/api/events.rb @@ -67,7 +67,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc "List a Project's visible events" do success Entities::Event end diff --git a/lib/api/files.rb b/lib/api/files.rb index e2ac7142bc4..1598d3c00b8 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -1,5 +1,7 @@ module API class Files < Grape::API + FILE_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(file_path: API::NO_SLASH_URL_PART_REGEX) + # Prevents returning plain/text responses for files with .txt extension after_validation { content_type "application/json" } @@ -58,13 +60,13 @@ module API params do requires :id, type: String, desc: 'The project ID' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: FILE_ENDPOINT_REQUIREMENTS do desc 'Get raw file contents from the repository' params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' requires :ref, type: String, desc: 'The name of branch, tag commit' end - get ":id/repository/files/:file_path/raw" do + get ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! send_git_blob @repo, @blob @@ -75,7 +77,7 @@ module API requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' requires :ref, type: String, desc: 'The name of branch, tag or commit' end - get ":id/repository/files/:file_path", requirements: { file_path: /.+/ } do + get ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! { @@ -95,7 +97,7 @@ module API params do use :extended_file_params end - post ":id/repository/files/:file_path", requirements: { file_path: /.+/ } do + post ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do authorize! :push_code, user_project file_params = declared_params(include_missing: false) @@ -113,7 +115,7 @@ module API params do use :extended_file_params end - put ":id/repository/files/:file_path", requirements: { file_path: /.+/ } do + put ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do authorize! :push_code, user_project file_params = declared_params(include_missing: false) @@ -137,7 +139,7 @@ module API params do use :simple_file_params end - delete ":id/repository/files/:file_path", requirements: { file_path: /.+/ } do + delete ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do authorize! :push_code, user_project file_params = declared_params(include_missing: false) diff --git a/lib/api/group_milestones.rb b/lib/api/group_milestones.rb index b85eb59dc0a..93fa0b95857 100644 --- a/lib/api/group_milestones.rb +++ b/lib/api/group_milestones.rb @@ -10,7 +10,7 @@ module API params do requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: { id: %r{[^/]+} } do + resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a list of group milestones' do success Entities::Milestone end diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb index f64da4ab77b..92800ce6450 100644 --- a/lib/api/group_variables.rb +++ b/lib/api/group_variables.rb @@ -9,7 +9,7 @@ module API requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: { id: %r{[^/]+} } do + resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get group-level variables' do success Entities::Variable end @@ -88,8 +88,7 @@ module API variable = user_group.variables.find_by(key: params[:key]) not_found!('GroupVariable') unless variable - status 204 - variable.destroy + destroy_conditionally!(variable) end end end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index e56427304a6..31a918eda60 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -7,7 +7,11 @@ module API helpers do params :optional_params_ce do optional :description, type: String, desc: 'The description of the group' - optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the group' + optional :visibility, type: String, + values: Gitlab::VisibilityLevel.string_values, + default: Gitlab::VisibilityLevel.string_level( + Gitlab::CurrentSettings.current_application_settings.default_group_visibility), + desc: 'The visibility of the group' optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group' optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' optional :share_with_group_lock, type: Boolean, desc: 'Prevent sharing a project with another group within this group' @@ -85,7 +89,7 @@ module API params do requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: { id: %r{[^/]+} } do + resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Update a group. Available only for users who can administrate groups.' do success Entities::Group end @@ -118,8 +122,9 @@ module API group = find_group!(params[:id]) authorize! :admin_group, group - status 204 - ::Groups::DestroyService.new(group, current_user).execute + destroy_conditionally!(group) do |group| + ::Groups::DestroyService.new(group, current_user).execute + end end desc 'Get a list of projects in this group.' do diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index b56fd2388b3..3d377fdb9eb 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -11,6 +11,27 @@ module API declared(params, options).to_h.symbolize_keys end + def check_unmodified_since!(last_modified) + if_unmodified_since = Time.parse(headers['If-Unmodified-Since']) rescue nil + + if if_unmodified_since && last_modified && last_modified > if_unmodified_since + render_api_error!('412 Precondition Failed', 412) + end + end + + def destroy_conditionally!(resource, last_updated: nil) + last_updated ||= resource.updated_at + + check_unmodified_since!(last_updated) + + status 204 + if block_given? + yield resource + else + resource.destroy + end + end + def current_user return @current_user if defined?(@current_user) diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index ecb79317093..f57ff0f2632 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -42,6 +42,10 @@ module API ::Users::ActivityService.new(actor, 'Git SSH').execute if commands.include?(params[:action]) end + def merge_request_urls + ::MergeRequests::GetUrlsService.new(project).execute(params[:changes]) + end + private def set_project diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index f8645e364ce..282af32ca94 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -1,6 +1,8 @@ module API module Helpers module Runner + include Gitlab::CurrentSettings + JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'.freeze JOB_TOKEN_PARAM = :token UPDATE_RUNNER_EVERY = 10 * 60 diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 8b007869dc3..622bd9650e4 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -68,7 +68,7 @@ module API end get "/merge_request_urls" do - ::MergeRequests::GetUrlsService.new(project).execute(params[:changes]) + merge_request_urls end # @@ -155,6 +155,21 @@ module API # render_api_error!(e, 500) # end end + + post '/post_receive' do + status 200 + + PostReceive.perform_async(params[:gl_repository], params[:identifier], + params[:changes]) + broadcast_message = BroadcastMessage.current&.last&.message + reference_counter_decreased = Gitlab::ReferenceCounter.new(params[:gl_repository]).decrease + + { + merge_request_urls: merge_request_urls, + broadcast_message: broadcast_message, + reference_counter_decreased: reference_counter_decreased + } + end end end end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 4cec1145f3a..1729df2aad0 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -4,6 +4,8 @@ module API before { authenticate! } + helpers ::Gitlab::IssuableMetadata + helpers do def find_issues(args = {}) args = params.merge(args) @@ -13,6 +15,7 @@ module API args[:label_name] = args.delete(:labels) issues = IssuesFinder.new(current_user, args).execute + .preload(:assignees, :labels, :notes, :timelogs) issues.reorder(args[:order_by] => args[:sort]) end @@ -33,6 +36,7 @@ module API optional :assignee_id, type: Integer, desc: 'Return issues which are assigned to the user with the given ID' optional :scope, type: String, values: %w[created-by-me assigned-to-me all], desc: 'Return issues for the given scope: `created-by-me`, `assigned-to-me` or `all`' + optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' use :pagination end @@ -65,14 +69,20 @@ module API get do issues = find_issues - present paginate(issues), with: Entities::IssueBasic, current_user: current_user + options = { + with: Entities::IssueBasic, + current_user: current_user, + issuable_metadata: issuable_meta_data(issues, 'Issue') + } + + present paginate(issues), options end end params do requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: { id: %r{[^/]+} } do + resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a list of group issues' do success Entities::IssueBasic end @@ -86,14 +96,20 @@ module API issues = find_issues(group_id: group.id) - present paginate(issues), with: Entities::IssueBasic, current_user: current_user + options = { + with: Entities::IssueBasic, + current_user: current_user, + issuable_metadata: issuable_meta_data(issues, 'Issue') + } + + present paginate(issues), options end end params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do include TimeTrackingEndpoints desc 'Get a list of project issues' do @@ -109,7 +125,14 @@ module API issues = find_issues(project_id: project.id) - present paginate(issues), with: Entities::IssueBasic, current_user: current_user, project: user_project + options = { + with: Entities::IssueBasic, + current_user: current_user, + project: user_project, + issuable_metadata: issuable_meta_data(issues, 'Issue') + } + + present paginate(issues), options end desc 'Get a single project issue' do @@ -230,8 +253,8 @@ module API not_found!('Issue') unless issue authorize!(:destroy_issue, issue) - status 204 - issue.destroy + + destroy_conditionally!(issue) end desc 'List merge requests closing issue' do diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index a40018b214e..5bab96398fd 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -7,7 +7,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do helpers do params :optional_scope do optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show', diff --git a/lib/api/labels.rb b/lib/api/labels.rb index 4520c98d951..e41a1720ac1 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -7,7 +7,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get all labels of the project' do success Entities::Label end @@ -56,8 +56,7 @@ module API label = user_project.labels.find_by(title: params[:name]) not_found!('Label') unless label - status 204 - label.destroy + destroy_conditionally!(label) end desc 'Update an existing label. At least one optional parameter is required.' do diff --git a/lib/api/members.rb b/lib/api/members.rb index bb970b7cd54..22e4bdead41 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -10,7 +10,7 @@ module API params do requires :id, type: String, desc: "The #{source_type} ID" end - resource source_type.pluralize, requirements: { id: %r{[^/]+} } do + resource source_type.pluralize, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Gets a list of group or project members viewable by the authenticated user.' do success Entities::Member end @@ -93,11 +93,11 @@ module API end delete ":id/members/:user_id" do source = find_source(source_type, params[:id]) - # Ensure that memeber exists - source.members.find_by!(user_id: params[:user_id]) + member = source.members.find_by!(user_id: params[:user_id]) - status 204 - ::Members::DestroyService.new(source, current_user, declared_params).execute + destroy_conditionally!(member) do + ::Members::DestroyService.new(source, current_user, declared_params).execute + end end end end diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb index 4b79eac2b8b..c3affcc6c6b 100644 --- a/lib/api/merge_request_diffs.rb +++ b/lib/api/merge_request_diffs.rb @@ -8,7 +8,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a list of merge request diff versions' do detail 'This feature was introduced in GitLab 8.12.' success Entities::MergeRequestDiff diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 8810d4e441d..56d72d511da 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -21,7 +21,7 @@ module API return merge_requests if args[:view] == 'simple' merge_requests - .preload(:notes, :author, :assignee, :milestone, :merge_request_diff, :labels) + .preload(:notes, :author, :assignee, :milestone, :merge_request_diff, :labels, :timelogs) end params :merge_requests_params do @@ -40,6 +40,7 @@ module API optional :assignee_id, type: Integer, desc: 'Return merge requests which are assigned to the user with the given ID' optional :scope, type: String, values: %w[created-by-me assigned-to-me all], desc: 'Return merge requests for the given scope: `created-by-me`, `assigned-to-me` or `all`' + optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' use :pagination end end @@ -72,7 +73,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do include TimeTrackingEndpoints helpers do @@ -164,8 +165,8 @@ module API merge_request = find_project_merge_request(params[:merge_request_iid]) authorize!(:destroy_merge_request, merge_request) - status 204 - merge_request.destroy + + destroy_conditionally!(merge_request) end params do diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 4e4e473994b..d6e7203adaf 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -9,7 +9,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do NOTEABLE_TYPES.each do |noteable_type| noteables_str = noteable_type.to_s.underscore.pluralize @@ -129,10 +129,12 @@ module API end delete ":id/#{noteables_str}/:noteable_id/notes/:note_id" do note = user_project.notes.find(params[:note_id]) + authorize! :admin_note, note - status 204 - ::Notes::DestroyService.new(user_project, current_user).execute(note) + destroy_conditionally!(note) do |note| + ::Notes::DestroyService.new(user_project, current_user).execute(note) + end end end end diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb index 5d113c94b22..bcc0833aa5c 100644 --- a/lib/api/notification_settings.rb +++ b/lib/api/notification_settings.rb @@ -54,7 +54,7 @@ module API params do requires :id, type: String, desc: "The #{source_type} ID" end - resource source_type.pluralize, requirements: { id: %r{[^/]+} } do + resource source_type.pluralize, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc "Get #{source_type} level notification level settings, defaults to Global" do detail 'This feature was introduced in GitLab 8.12' success Entities::NotificationSetting diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index dbeaf9e17ef..37f32411296 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -7,7 +7,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get all pipeline schedules' do success Entities::PipelineSchedule end @@ -31,10 +31,6 @@ module API requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' end get ':id/pipeline_schedules/:pipeline_schedule_id' do - authorize! :read_pipeline_schedule, user_project - - not_found!('PipelineSchedule') unless pipeline_schedule - present pipeline_schedule, with: Entities::PipelineScheduleDetails end @@ -74,9 +70,6 @@ module API optional :active, type: Boolean, desc: 'The activation of pipeline schedule' end put ':id/pipeline_schedules/:pipeline_schedule_id' do - authorize! :read_pipeline_schedule, user_project - - not_found!('PipelineSchedule') unless pipeline_schedule authorize! :update_pipeline_schedule, pipeline_schedule if pipeline_schedule.update(declared_params(include_missing: false)) @@ -93,9 +86,6 @@ module API requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' end post ':id/pipeline_schedules/:pipeline_schedule_id/take_ownership' do - authorize! :read_pipeline_schedule, user_project - - not_found!('PipelineSchedule') unless pipeline_schedule authorize! :update_pipeline_schedule, pipeline_schedule if pipeline_schedule.own!(current_user) @@ -112,22 +102,84 @@ module API requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' end delete ':id/pipeline_schedules/:pipeline_schedule_id' do - authorize! :read_pipeline_schedule, user_project + authorize! :admin_pipeline_schedule, pipeline_schedule + + destroy_conditionally!(pipeline_schedule) + end + + desc 'Create a new pipeline schedule variable' do + success Entities::Variable + end + params do + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + requires :key, type: String, desc: 'The key of the variable' + requires :value, type: String, desc: 'The value of the variable' + end + post ':id/pipeline_schedules/:pipeline_schedule_id/variables' do + authorize! :update_pipeline_schedule, pipeline_schedule - not_found!('PipelineSchedule') unless pipeline_schedule + variable_params = declared_params(include_missing: false) + variable = pipeline_schedule.variables.create(variable_params) + if variable.persisted? + present variable, with: Entities::Variable + else + render_validation_error!(variable) + end + end + + desc 'Edit a pipeline schedule variable' do + success Entities::Variable + end + params do + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + requires :key, type: String, desc: 'The key of the variable' + optional :value, type: String, desc: 'The value of the variable' + end + put ':id/pipeline_schedules/:pipeline_schedule_id/variables/:key' do + authorize! :update_pipeline_schedule, pipeline_schedule + + if pipeline_schedule_variable.update(declared_params(include_missing: false)) + present pipeline_schedule_variable, with: Entities::Variable + else + render_validation_error!(pipeline_schedule_variable) + end + end + + desc 'Delete a pipeline schedule variable' do + success Entities::Variable + end + params do + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + requires :key, type: String, desc: 'The key of the variable' + end + delete ':id/pipeline_schedules/:pipeline_schedule_id/variables/:key' do authorize! :admin_pipeline_schedule, pipeline_schedule status :accepted - present pipeline_schedule.destroy, with: Entities::PipelineScheduleDetails + present pipeline_schedule_variable.destroy, with: Entities::Variable end end helpers do def pipeline_schedule @pipeline_schedule ||= - user_project.pipeline_schedules - .preload(:owner, :last_pipeline) - .find_by(id: params.delete(:pipeline_schedule_id)) + user_project + .pipeline_schedules + .preload(:owner, :last_pipeline) + .find_by(id: params.delete(:pipeline_schedule_id)).tap do |pipeline_schedule| + unless can?(current_user, :read_pipeline_schedule, pipeline_schedule) + not_found!('Pipeline Schedule') + end + end + end + + def pipeline_schedule_variable + @pipeline_schedule_variable ||= + pipeline_schedule.variables.find_by(key: params[:key]).tap do |pipeline_schedule_variable| + unless pipeline_schedule_variable + not_found!('Pipeline Schedule Variable') + end + end end end end diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index e505cae3992..74b3376a1f3 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -7,7 +7,7 @@ module API params do requires :id, type: String, desc: 'The project ID' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get all Pipelines of the project' do detail 'This feature was introduced in GitLab 8.11.' success Entities::PipelineBasic diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 649dd891f56..86066e2b58f 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -24,7 +24,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get project hooks' do success Entities::ProjectHook end @@ -96,8 +96,7 @@ module API delete ":id/hooks/:hook_id" do hook = user_project.hooks.find(params.delete(:hook_id)) - status 204 - hook.destroy + destroy_conditionally!(hook) end end end diff --git a/lib/api/project_milestones.rb b/lib/api/project_milestones.rb index 451998c726a..0cb209a02d0 100644 --- a/lib/api/project_milestones.rb +++ b/lib/api/project_milestones.rb @@ -10,7 +10,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a list of project milestones' do success Entities::Milestone end diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index f3d905b0068..2ccda1c1aa1 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -7,7 +7,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do helpers do def handle_project_member_errors(errors) if errors[:project_access].any? @@ -116,8 +116,8 @@ module API not_found!('Snippet') unless snippet authorize! :admin_project_snippet, snippet - status 204 - snippet.destroy + + destroy_conditionally!(snippet) end desc 'Get a raw project snippet' diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 15c3832b032..4845242a173 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -1,7 +1,6 @@ require_dependency 'declarative_policy' module API - # Projects API class Projects < Grape::API include PaginationParams @@ -96,7 +95,7 @@ module API end end - resource :users, requirements: { user_id: %r{[^/]+} } do + resource :users, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a user projects' do success Entities::BasicProjectDetails end @@ -184,7 +183,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a single project' do success Entities::ProjectWithAccess end @@ -334,7 +333,10 @@ module API desc 'Remove a project' delete ":id" do authorize! :remove_project, user_project - ::Projects::DestroyService.new(user_project, current_user, {}).async_execute + + destroy_conditionally!(user_project) do + ::Projects::DestroyService.new(user_project, current_user, {}).async_execute + end accepted! end @@ -363,8 +365,7 @@ module API authorize! :remove_fork_project, user_project if user_project.forked? - status 204 - user_project.forked_project_link.destroy + destroy_conditionally!(user_project.forked_project_link) else not_modified! end @@ -408,8 +409,7 @@ module API link = user_project.project_group_links.find_by(group_id: params[:group_id]) not_found!('Group Link') unless link - status 204 - link.destroy + destroy_conditionally!(link) end desc 'Upload a file' diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index dccf4fa27a7..15fcb9e8e27 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -76,9 +76,7 @@ module API delete ':id/protected_branches/:name', requirements: BRANCH_ENDPOINT_REQUIREMENTS do protected_branch = user_project.protected_branches.find_by!(name: params[:name]) - protected_branch.destroy - - status 204 + destroy_conditionally!(protected_branch) end end end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 14d2bff9cb5..2255fb1b70d 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -9,7 +9,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do helpers do def handle_project_member_errors(errors) if errors[:project_access].any? diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 88fc62d33df..a3987c560dd 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -45,8 +45,10 @@ module API end delete '/' do authenticate_runner! - status 204 - Ci::Runner.find_by_token(params[:token]).destroy + + runner = Ci::Runner.find_by_token(params[:token]) + + destroy_conditionally!(runner) end desc 'Validates authentication credentials' do @@ -78,7 +80,7 @@ module API no_content! unless current_runner.active? update_runner_info - if current_runner.is_runner_queue_value_latest?(params[:last_update]) + if current_runner.runner_queue_value_latest?(params[:last_update]) header 'X-GitLab-Last-Update', params[:last_update] Gitlab::Metrics.add_event(:build_not_found_cached) return no_content! @@ -112,6 +114,8 @@ module API requires :id, type: Integer, desc: %q(Job's ID) optional :trace, type: String, desc: %q(Job's full trace) optional :state, type: String, desc: %q(Job's status: success, failed) + optional :failure_reason, type: String, values: CommitStatus.failure_reasons.keys, + desc: %q(Job's failure_reason) end put '/:id' do job = authenticate_job! @@ -125,7 +129,7 @@ module API when 'success' job.success when 'failed' - job.drop + job.drop(params[:failure_reason] || :unknown_failure) end end diff --git a/lib/api/runners.rb b/lib/api/runners.rb index 31f940fe96b..d3559ef71be 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -55,7 +55,9 @@ module API optional :tag_list, type: Array[String], desc: 'The list of tags for a runner' optional :run_untagged, type: Boolean, desc: 'Flag indicating the runner can execute untagged jobs' optional :locked, type: Boolean, desc: 'Flag indicating the runner is locked' - at_least_one_of :description, :active, :tag_list, :run_untagged, :locked + optional :access_level, type: String, values: Ci::Runner.access_levels.keys, + desc: 'The access_level of the runner' + at_least_one_of :description, :active, :tag_list, :run_untagged, :locked, :access_level end put ':id' do runner = get_runner(params.delete(:id)) @@ -77,17 +79,17 @@ module API end delete ':id' do runner = get_runner(params[:id]) + authenticate_delete_runner!(runner) - status 204 - runner.destroy! + destroy_conditionally!(runner) end end params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do before { authorize_admin_project } desc 'Get runners available for project' do @@ -135,8 +137,7 @@ module API runner = runner_project.runner forbidden!("Only one project associated with the runner. Please remove the runner instead") if runner.projects.count == 1 - status 204 - runner_project.destroy + destroy_conditionally!(runner_project) end end diff --git a/lib/api/services.rb b/lib/api/services.rb index 843c05ae32e..2cbd0517dc3 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -601,7 +601,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do before { authenticate! } before { authorize_admin_project } @@ -656,12 +656,14 @@ module API delete ":id/services/:service_slug" do service = user_project.find_or_initialize_service(params[:service_slug].underscore) - attrs = service_attributes(service).inject({}) do |hash, key| - hash.merge!(key => nil) - end + destroy_conditionally!(service) do + attrs = service_attributes(service).inject({}) do |hash, key| + hash.merge!(key => nil) + end - unless service.update_attributes(attrs.merge(active: false)) - render_api_error!('400 Bad Request', 400) + unless service.update_attributes(attrs.merge(active: false)) + render_api_error!('400 Bad Request', 400) + end end end @@ -689,7 +691,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc "Trigger a slash command for #{service_slug}" do detail 'Added in GitLab 8.13' end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 667ba468ce6..851b226e9e5 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -122,6 +122,13 @@ module API optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.' optional :polling_interval_multiplier, type: BigDecimal, desc: 'Interval multiplier used by endpoints that perform polling. Set to 0 to disable polling.' + ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type| + optional :"#{type}_key_restriction", + type: Integer, + values: KeyRestrictionValidator.supported_key_restrictions(type), + desc: "Restrictions on the complexity of uploaded #{type.upcase} keys. A value of #{ApplicationSetting::FORBIDDEN_KEY_VALUE} disables all #{type.upcase} keys." + end + optional(*::ApplicationSettingsHelper.visible_attributes) at_least_one_of(*::ApplicationSettingsHelper.visible_attributes) end diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index 35ece56c65c..00eb7c60f16 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -123,8 +123,7 @@ module API authorize! :destroy_personal_snippet, snippet - status 204 - snippet.destroy + destroy_conditionally!(snippet) end desc 'Get a raw snippet' do diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb index 91567909998..b3e1e23031a 100644 --- a/lib/api/subscriptions.rb +++ b/lib/api/subscriptions.rb @@ -12,7 +12,7 @@ module API requires :id, type: String, desc: 'The ID of a project' requires :subscribable_id, type: String, desc: 'The ID of a resource' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do subscribable_types.each do |type, finder| type_singularized = type.singularize entity_class = Entities.const_get(type_singularized.camelcase) diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb index c0179037440..6b6a03e3300 100644 --- a/lib/api/system_hooks.rb +++ b/lib/api/system_hooks.rb @@ -66,8 +66,7 @@ module API hook = SystemHook.find_by(id: params[:id]) not_found!('System hook') unless hook - status 204 - hook.destroy + destroy_conditionally!(hook) end end end diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 1333747cced..912415e3a7f 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -65,11 +65,18 @@ module API delete ':id/repository/tags/:tag_name', requirements: TAG_ENDPOINT_REQUIREMENTS do authorize_push_project - result = ::Tags::DestroyService.new(user_project, current_user) - .execute(params[:tag_name]) + tag = user_project.repository.find_tag(params[:tag_name]) + not_found!('Tag') unless tag + + commit = user_project.repository.commit(tag.dereferenced_target) + + destroy_conditionally!(commit, last_updated: commit.authored_date) do + result = ::Tags::DestroyService.new(user_project, current_user) + .execute(params[:tag_name]) - if result[:status] != :success - render_api_error!(result[:message], result[:return_code]) + if result[:status] != :success + render_api_error!(result[:message], result[:return_code]) + end end end diff --git a/lib/api/todos.rb b/lib/api/todos.rb index 55191169dd4..ffccfebe752 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -12,7 +12,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do ISSUABLE_TYPES.each do |type, finder| type_id_str = "#{type.singularize}_iid".to_sym diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index edfdb63d183..dd6801664b1 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -5,7 +5,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Trigger a GitLab project pipeline' do success Entities::Pipeline end @@ -140,8 +140,7 @@ module API trigger = user_project.triggers.find(params.delete(:trigger_id)) return not_found!('Trigger') unless trigger - status 204 - trigger.destroy + destroy_conditionally!(trigger) end end end diff --git a/lib/api/users.rb b/lib/api/users.rb index e2019d6d512..1825c90a23b 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -230,10 +230,89 @@ module API key = user.keys.find_by(id: params[:key_id]) not_found!('Key') unless key + destroy_conditionally!(key) + end + + desc 'Add a GPG key to a specified user. Available only for admins.' do + detail 'This feature was added in GitLab 10.0' + success Entities::GPGKey + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + requires :key, type: String, desc: 'The new GPG key' + end + post ':id/gpg_keys' do + authenticated_as_admin! + + user = User.find_by(id: params.delete(:id)) + not_found!('User') unless user + + key = user.gpg_keys.new(declared_params(include_missing: false)) + + if key.save + present key, with: Entities::GPGKey + else + render_validation_error!(key) + end + end + + desc 'Get the GPG keys of a specified user. Available only for admins.' do + detail 'This feature was added in GitLab 10.0' + success Entities::GPGKey + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + use :pagination + end + get ':id/gpg_keys' do + authenticated_as_admin! + + user = User.find_by(id: params[:id]) + not_found!('User') unless user + + present paginate(user.gpg_keys), with: Entities::GPGKey + end + + desc 'Delete an existing GPG key from a specified user. Available only for admins.' do + detail 'This feature was added in GitLab 10.0' + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + requires :key_id, type: Integer, desc: 'The ID of the GPG key' + end + delete ':id/gpg_keys/:key_id' do + authenticated_as_admin! + + user = User.find_by(id: params[:id]) + not_found!('User') unless user + + key = user.gpg_keys.find_by(id: params[:key_id]) + not_found!('GPG Key') unless key + status 204 key.destroy end + desc 'Revokes an existing GPG key from a specified user. Available only for admins.' do + detail 'This feature was added in GitLab 10.0' + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + requires :key_id, type: Integer, desc: 'The ID of the GPG key' + end + post ':id/gpg_keys/:key_id/revoke' do + authenticated_as_admin! + + user = User.find_by(id: params[:id]) + not_found!('User') unless user + + key = user.gpg_keys.find_by(id: params[:key_id]) + not_found!('GPG Key') unless key + + key.revoke + status :accepted + end + desc 'Add an email address to a specified user. Available only for admins.' do success Entities::Email end @@ -287,7 +366,11 @@ module API email = user.emails.find_by(id: params[:email_id]) not_found!('Email') unless email - Emails::DestroyService.new(user, email: email.email).execute + destroy_conditionally!(email) do |email| + Emails::DestroyService.new(current_user, email: email.email).execute + end + + user.update_secondary_emails! end desc 'Delete a user. Available only for admins.' do @@ -299,11 +382,13 @@ module API end delete ":id" do authenticated_as_admin! + user = User.find_by(id: params[:id]) not_found!('User') unless user - status 204 - user.delete_async(deleted_by: current_user, params: params) + destroy_conditionally!(user) do + user.delete_async(deleted_by: current_user, params: params) + end end desc 'Block a user. Available only for admins.' @@ -403,8 +488,11 @@ module API requires :impersonation_token_id, type: Integer, desc: 'The ID of the impersonation token' end delete ':impersonation_token_id' do - status 204 - find_impersonation_token.revoke! + token = find_impersonation_token + + destroy_conditionally!(token) do + token.revoke! + end end end end @@ -481,6 +569,75 @@ module API key = current_user.keys.find_by(id: params[:key_id]) not_found!('Key') unless key + destroy_conditionally!(key) + end + + desc "Get the currently authenticated user's GPG keys" do + detail 'This feature was added in GitLab 10.0' + success Entities::GPGKey + end + params do + use :pagination + end + get 'gpg_keys' do + present paginate(current_user.gpg_keys), with: Entities::GPGKey + end + + desc 'Get a single GPG key owned by currently authenticated user' do + detail 'This feature was added in GitLab 10.0' + success Entities::GPGKey + end + params do + requires :key_id, type: Integer, desc: 'The ID of the GPG key' + end + get 'gpg_keys/:key_id' do + key = current_user.gpg_keys.find_by(id: params[:key_id]) + not_found!('GPG Key') unless key + + present key, with: Entities::GPGKey + end + + desc 'Add a new GPG key to the currently authenticated user' do + detail 'This feature was added in GitLab 10.0' + success Entities::GPGKey + end + params do + requires :key, type: String, desc: 'The new GPG key' + end + post 'gpg_keys' do + key = current_user.gpg_keys.new(declared_params) + + if key.save + present key, with: Entities::GPGKey + else + render_validation_error!(key) + end + end + + desc 'Revoke a GPG key owned by currently authenticated user' do + detail 'This feature was added in GitLab 10.0' + end + params do + requires :key_id, type: Integer, desc: 'The ID of the GPG key' + end + post 'gpg_keys/:key_id/revoke' do + key = current_user.gpg_keys.find_by(id: params[:key_id]) + not_found!('GPG Key') unless key + + key.revoke + status :accepted + end + + desc 'Delete a GPG key from the currently authenticated user' do + detail 'This feature was added in GitLab 10.0' + end + params do + requires :key_id, type: Integer, desc: 'The ID of the SSH key' + end + delete 'gpg_keys/:key_id' do + key = current_user.gpg_keys.find_by(id: params[:key_id]) + not_found!('GPG Key') unless key + status 204 key.destroy end @@ -533,8 +690,11 @@ module API email = current_user.emails.find_by(id: params[:email_id]) not_found!('Email') unless email - status 204 - Emails::DestroyService.new(current_user, email: email.email).execute + destroy_conditionally!(email) do |email| + Emails::DestroyService.new(current_user, email: email.email).execute + end + + current_user.update_secondary_emails! end desc 'Get a list of user activities' diff --git a/lib/api/v3/triggers.rb b/lib/api/v3/triggers.rb index e9d4c35307b..534911fde5c 100644 --- a/lib/api/v3/triggers.rb +++ b/lib/api/v3/triggers.rb @@ -16,25 +16,31 @@ module API optional :variables, type: Hash, desc: 'The list of variables to be injected into build' end post ":id/(ref/:ref/)trigger/builds", requirements: { ref: /.+/ } do - project = find_project(params[:id]) - trigger = Ci::Trigger.find_by_token(params[:token].to_s) - not_found! unless project && trigger - unauthorized! unless trigger.project == project - # validate variables - variables = params[:variables].to_h - unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) } + params[:variables] = params[:variables].to_h + unless params[:variables].all? { |key, value| key.is_a?(String) && value.is_a?(String) } render_api_error!('variables needs to be a map of key-valued strings', 400) end - # create request and trigger builds - result = Ci::CreateTriggerRequestService.execute(project, trigger, params[:ref].to_s, variables) - pipeline = result.pipeline + project = find_project(params[:id]) + not_found! unless project + + result = Ci::PipelineTriggerService.new(project, nil, params).execute + not_found! unless result - if pipeline.persisted? - present result.trigger_request, with: ::API::V3::Entities::TriggerRequest + if result[:http_status] + render_api_error!(result[:message], result[:http_status]) else - render_validation_error!(pipeline) + pipeline = result[:pipeline] + + # We switched to Ci::PipelineVariable from Ci::TriggerRequest.variables. + # Ci::TriggerRequest doesn't save variables anymore. + # Here is copying Ci::PipelineVariable to Ci::TriggerRequest.variables for presenting the variables. + # The same endpoint in v4 API pressents Pipeline instead of TriggerRequest, so it doesn't need such a process. + trigger_request = pipeline.trigger_requests.last + trigger_request.variables = params[:variables] + + present trigger_request, with: ::API::V3::Entities::TriggerRequest end end diff --git a/lib/api/variables.rb b/lib/api/variables.rb index 7c0fdd3d1be..d08876ae1b9 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -9,7 +9,7 @@ module API requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get project variables' do success Entities::Variable end @@ -88,6 +88,7 @@ module API variable = user_project.variables.find_by(key: params[:key]) not_found!('Variable') unless variable + # Variables don't have any timestamp. Therfore, destroy unconditionally. status 204 variable.destroy end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 3a4911b23b0..62b44389b15 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -20,24 +20,6 @@ module Ci raise ValidationError, e.message end - def jobs_for_ref(ref, tag = false, source = nil) - @jobs.select do |_, job| - process?(job[:only], job[:except], ref, tag, source) - end - end - - def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil) - jobs_for_ref(ref, tag, source).select do |_, job| - job[:stage] == stage - end - end - - def builds_for_ref(ref, tag = false, source = nil) - jobs_for_ref(ref, tag, source).map do |name, _| - build_attributes(name) - end - end - def builds_for_stage_and_ref(stage, ref, tag = false, source = nil) jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _| build_attributes(name) @@ -52,8 +34,7 @@ module Ci def stage_seeds(pipeline) seeds = @stages.uniq.map do |stage| - builds = builds_for_stage_and_ref( - stage, pipeline.ref, pipeline.tag?, pipeline.source) + builds = pipeline_stage_builds(stage, pipeline) Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any? end @@ -101,6 +82,34 @@ module Ci private + def pipeline_stage_builds(stage, pipeline) + builds = builds_for_stage_and_ref( + stage, pipeline.ref, pipeline.tag?, pipeline.source) + + builds.select do |build| + job = @jobs[build.fetch(:name).to_sym] + has_kubernetes = pipeline.has_kubernetes_active? + only_kubernetes = job.dig(:only, :kubernetes) + except_kubernetes = job.dig(:except, :kubernetes) + + [!only_kubernetes && !except_kubernetes, + only_kubernetes && has_kubernetes, + except_kubernetes && !has_kubernetes].any? + end + end + + def jobs_for_ref(ref, tag = false, source = nil) + @jobs.select do |_, job| + process?(job.dig(:only, :refs), job.dig(:except, :refs), ref, tag, source) + end + end + + def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil) + jobs_for_ref(ref, tag, source).select do |_, job| + job[:stage] == stage + end + end + def initial_parsing ## # Global config diff --git a/lib/email_template_interceptor.rb b/lib/email_template_interceptor.rb index 63f9f8d7a5a..f2bf3d0fb2b 100644 --- a/lib/email_template_interceptor.rb +++ b/lib/email_template_interceptor.rb @@ -1,6 +1,6 @@ # Read about interceptors in http://guides.rubyonrails.org/action_mailer_basics.html#intercepting-emails class EmailTemplateInterceptor - include Gitlab::CurrentSettings + extend Gitlab::CurrentSettings def self.delivering_email(message) # Remove HTML part if HTML emails are disabled. diff --git a/lib/github/import.rb b/lib/github/import.rb index 7b848081e85..9354e142d3d 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -226,49 +226,51 @@ module Github while url response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) - response.body.each do |raw| - representation = Github::Representation::Issue.new(raw, options) + response.body.each { |raw| populate_issue(raw) } - begin - # Every pull request is an issue, but not every issue - # is a pull request. For this reason, "shared" actions - # for both features, like manipulating assignees, labels - # and milestones, are provided within the Issues API. - if representation.pull_request? - next unless representation.has_labels? - - merge_request = MergeRequest.find_by!(target_project_id: project.id, iid: representation.iid) - merge_request.update_attribute(:label_ids, label_ids(representation.labels)) - else - next if Issue.where(iid: representation.iid, project_id: project.id).exists? - - author_id = user_id(representation.author, project.creator_id) - issue = Issue.new - issue.iid = representation.iid - issue.project_id = project.id - issue.title = representation.title - issue.description = format_description(representation.description, representation.author) - issue.state = representation.state - issue.label_ids = label_ids(representation.labels) - issue.milestone_id = milestone_id(representation.milestone) - issue.author_id = author_id - issue.assignee_ids = [user_id(representation.assignee)] - issue.created_at = representation.created_at - issue.updated_at = representation.updated_at - issue.save!(validate: false) - - # Fetch comments - if representation.has_comments? - comments_url = "/repos/#{repo}/issues/#{issue.iid}/comments" - fetch_comments(issue, :comment, comments_url) - end - end - rescue => e - error(:issue, representation.url, e.message) + url = response.rels[:next] + end + end + + def populate_issue(raw) + representation = Github::Representation::Issue.new(raw, options) + + begin + # Every pull request is an issue, but not every issue + # is a pull request. For this reason, "shared" actions + # for both features, like manipulating assignees, labels + # and milestones, are provided within the Issues API. + if representation.pull_request? + return unless representation.has_labels? + + merge_request = MergeRequest.find_by!(target_project_id: project.id, iid: representation.iid) + merge_request.update_attribute(:label_ids, label_ids(representation.labels)) + else + return if Issue.where(iid: representation.iid, project_id: project.id).exists? + + author_id = user_id(representation.author, project.creator_id) + issue = Issue.new + issue.iid = representation.iid + issue.project_id = project.id + issue.title = representation.title + issue.description = format_description(representation.description, representation.author) + issue.state = representation.state + issue.label_ids = label_ids(representation.labels) + issue.milestone_id = milestone_id(representation.milestone) + issue.author_id = author_id + issue.assignee_ids = [user_id(representation.assignee)] + issue.created_at = representation.created_at + issue.updated_at = representation.updated_at + issue.save!(validate: false) + + # Fetch comments + if representation.has_comments? + comments_url = "/repos/#{repo}/issues/#{issue.iid}/comments" + fetch_comments(issue, :comment, comments_url) end end - - url = response.rels[:next] + rescue => e + error(:issue, representation.url, e.message) end end diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index 3d41ac76406..cead1c7eacd 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -6,6 +6,8 @@ module Gitlab # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters # the resulting HTML through HTML pipeline filters. module Asciidoc + extend Gitlab::CurrentSettings + DEFAULT_ADOC_ATTRS = [ 'showtitle', 'idprefix=user-content-', 'idseparator=-', 'env=gitlab', 'env-gitlab', 'source-highlighter=html-pipeline', 'icons=font' diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 8cb4060cd97..3fd81759d25 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -19,6 +19,8 @@ module Gitlab OPTIONAL_SCOPES = (AVAILABLE_SCOPES + OPENID_SCOPES - DEFAULT_SCOPES).freeze class << self + include Gitlab::CurrentSettings + def find_for_git_client(login, password, project:, ip:) raise "Must provide an IP for rate limiting" if ip.nil? @@ -48,10 +50,6 @@ module Gitlab # Avoid resource intensive login checks if password is not provided return unless password.present? - # Nothing to do here if internal auth is disabled and LDAP is - # not configured - return unless current_application_settings.password_authentication_enabled? || Gitlab::LDAP::Config.enabled? - Gitlab::Auth::UniqueIpsLimiter.limit_user! do user = User.by_login(login) diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb index 714464fd5e7..dc5d285ea65 100644 --- a/lib/gitlab/checks/force_push.rb +++ b/lib/gitlab/checks/force_push.rb @@ -12,7 +12,7 @@ module Gitlab !project .repository .gitaly_commit_client - .is_ancestor(oldrev, newrev) + .ancestor?(oldrev, newrev) else Gitlab::Git::RevList.new( path_to_repo: project.repository.path_to_repo, diff --git a/lib/gitlab/ci/config/entry/attributable.rb b/lib/gitlab/ci/config/entry/attributable.rb index 1c8b55ee4c4..3e87a09704e 100644 --- a/lib/gitlab/ci/config/entry/attributable.rb +++ b/lib/gitlab/ci/config/entry/attributable.rb @@ -8,6 +8,10 @@ module Gitlab class_methods do def attributes(*attributes) attributes.flatten.each do |attribute| + if method_defined?(attribute) + raise ArgumentError, 'Method already defined!' + end + define_method(attribute) do return unless config.is_a?(Hash) diff --git a/lib/gitlab/ci/config/entry/configurable.rb b/lib/gitlab/ci/config/entry/configurable.rb index e05aca9881b..68b6742385a 100644 --- a/lib/gitlab/ci/config/entry/configurable.rb +++ b/lib/gitlab/ci/config/entry/configurable.rb @@ -15,9 +15,10 @@ module Gitlab # module Configurable extend ActiveSupport::Concern - include Validatable included do + include Validatable + validations do validates :config, type: Hash end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 32f5c6ab142..91aac6df4b1 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -59,10 +59,10 @@ module Gitlab entry :services, Entry::Services, description: 'Services that will be used to execute this job.' - entry :only, Entry::Trigger, + entry :only, Entry::Policy, description: 'Refs policy this job will be executed for.' - entry :except, Entry::Trigger, + entry :except, Entry::Policy, description: 'Refs policy this job will be executed for.' entry :variables, Entry::Variables, diff --git a/lib/gitlab/ci/config/entry/node.rb b/lib/gitlab/ci/config/entry/node.rb index a6a914d79c1..c868943c42e 100644 --- a/lib/gitlab/ci/config/entry/node.rb +++ b/lib/gitlab/ci/config/entry/node.rb @@ -16,8 +16,9 @@ module Gitlab @metadata = metadata @entries = {} - @validator = self.class.validator.new(self) - @validator.validate(:new) + self.class.aspects.to_a.each do |aspect| + instance_exec(&aspect) + end end def [](key) @@ -47,7 +48,7 @@ module Gitlab end def errors - @validator.messages + descendants.flat_map(&:errors) + [] end def value @@ -70,6 +71,13 @@ module Gitlab true end + def location + name = @key.presence || self.class.name.to_s.demodulize + .underscore.humanize.downcase + + ancestors.map(&:key).append(name).compact.join(':') + end + def inspect val = leaf? ? config : descendants unspecified = specified? ? '' : '(unspecified) ' @@ -79,8 +87,8 @@ module Gitlab def self.default end - def self.validator - Validator + def self.aspects + @aspects ||= [] end end end diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb new file mode 100644 index 00000000000..0027e9ec8c5 --- /dev/null +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -0,0 +1,53 @@ +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents an only/except trigger policy for the job. + # + class Policy < Simplifiable + strategy :RefsPolicy, if: -> (config) { config.is_a?(Array) } + strategy :ComplexPolicy, if: -> (config) { config.is_a?(Hash) } + + class RefsPolicy < Entry::Node + include Entry::Validatable + + validations do + validates :config, array_of_strings_or_regexps: true + end + + def value + { refs: @config } + end + end + + class ComplexPolicy < Entry::Node + include Entry::Validatable + include Entry::Attributable + + attributes :refs, :kubernetes + + validations do + validates :config, presence: true + validates :config, allowed_keys: %i[refs kubernetes] + + with_options allow_nil: true do + validates :refs, array_of_strings_or_regexps: true + validates :kubernetes, allowed_values: %w[active] + end + end + end + + class UnknownStrategy < Entry::Node + def errors + ["#{location} has to be either an array of conditions or a hash"] + end + end + + def self.default + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/simplifiable.rb b/lib/gitlab/ci/config/entry/simplifiable.rb new file mode 100644 index 00000000000..12764629686 --- /dev/null +++ b/lib/gitlab/ci/config/entry/simplifiable.rb @@ -0,0 +1,43 @@ +module Gitlab + module Ci + class Config + module Entry + class Simplifiable < SimpleDelegator + EntryStrategy = Struct.new(:name, :condition) + + def initialize(config, **metadata) + unless self.class.const_defined?(:UnknownStrategy) + raise ArgumentError, 'UndefinedStrategy not available!' + end + + strategy = self.class.strategies.find do |variant| + variant.condition.call(config) + end + + entry = self.class.entry_class(strategy) + + super(entry.new(config, metadata)) + end + + def self.strategy(name, **opts) + EntryStrategy.new(name, opts.fetch(:if)).tap do |strategy| + strategies.append(strategy) + end + end + + def self.strategies + @strategies ||= [] + end + + def self.entry_class(strategy) + if strategy.present? + self.const_get(strategy.name) + else + self::UnknownStrategy + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/trigger.rb b/lib/gitlab/ci/config/entry/trigger.rb deleted file mode 100644 index 16b234e6c59..00000000000 --- a/lib/gitlab/ci/config/entry/trigger.rb +++ /dev/null @@ -1,18 +0,0 @@ -module Gitlab - module Ci - class Config - module Entry - ## - # Entry that represents a trigger policy for the job. - # - class Trigger < Node - include Validatable - - validations do - validates :config, array_of_strings_or_regexps: true - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/validatable.rb b/lib/gitlab/ci/config/entry/validatable.rb index f7f1b111571..5ced778d311 100644 --- a/lib/gitlab/ci/config/entry/validatable.rb +++ b/lib/gitlab/ci/config/entry/validatable.rb @@ -5,6 +5,17 @@ module Gitlab module Validatable extend ActiveSupport::Concern + def self.included(node) + node.aspects.append -> do + @validator = self.class.validator.new(self) + @validator.validate(:new) + end + end + + def errors + @validator.messages + descendants.flat_map(&:errors) + end + class_methods do def validator @validator ||= Class.new(Entry::Validator).tap do |validator| diff --git a/lib/gitlab/ci/config/entry/validator.rb b/lib/gitlab/ci/config/entry/validator.rb index 55343005fe3..2df23a3edcd 100644 --- a/lib/gitlab/ci/config/entry/validator.rb +++ b/lib/gitlab/ci/config/entry/validator.rb @@ -8,7 +8,6 @@ module Gitlab def initialize(entry) super(entry) - @entry = entry end def messages @@ -20,21 +19,6 @@ module Gitlab def self.name 'Validator' end - - private - - def location - predecessors = ancestors.map(&:key).compact - predecessors.append(key_name).join(':') - end - - def key_name - if key.blank? - @entry.class.name.demodulize.underscore.humanize - else - key - end - end end end end diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb index b2ca3c881e4..0159179f0a9 100644 --- a/lib/gitlab/ci/config/entry/validators.rb +++ b/lib/gitlab/ci/config/entry/validators.rb @@ -14,6 +14,14 @@ module Gitlab end end + class AllowedValuesValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unless options[:in].include?(value.to_s) + record.errors.add(attribute, "unknown value: #{value}") + end + end + end + class ArrayOfStringsValidator < ActiveModel::EachValidator include LegacyValidationHelpers diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb index f81f9347b4d..e19aae35a81 100644 --- a/lib/gitlab/ci/stage/seed.rb +++ b/lib/gitlab/ci/stage/seed.rb @@ -28,7 +28,8 @@ module Gitlab attributes.merge(project: project, ref: pipeline.ref, tag: pipeline.tag, - trigger_request: trigger) + trigger_request: trigger, + protected: protected_ref?) end end @@ -43,6 +44,12 @@ module Gitlab end end end + + private + + def protected_ref? + @protected_ref ||= project.protected_for?(pipeline.ref) + end end end end diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index d671867e7c7..90f83e0f810 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -18,7 +18,7 @@ module Gitlab new(merge_request, project).tap do |file_collection| project .repository - .with_repo_branch_commit(merge_request.target_project.repository, merge_request.target_branch) do + .with_repo_branch_commit(merge_request.target_project.repository.raw_repository, merge_request.target_branch) do yield file_collection end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 7fa02f3d7b3..642f0944354 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -1,5 +1,7 @@ module Gitlab module CurrentSettings + extend self + def current_application_settings if RequestStore.active? RequestStore.fetch(:current_application_settings) { ensure_application_settings! } diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index e001d25e7b7..a6ec75da385 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -9,6 +9,14 @@ module Gitlab ActiveRecord::Base.configurations[Rails.env] end + def self.username + config['username'] || ENV['USER'] + end + + def self.database_name + config['database'] + end + def self.adapter_name config['adapter'] end diff --git a/lib/gitlab/database/grant.rb b/lib/gitlab/database/grant.rb new file mode 100644 index 00000000000..aee3981e79a --- /dev/null +++ b/lib/gitlab/database/grant.rb @@ -0,0 +1,34 @@ +module Gitlab + module Database + # Model that can be used for querying permissions of a SQL user. + class Grant < ActiveRecord::Base + self.table_name = + if Database.postgresql? + 'information_schema.role_table_grants' + else + 'mysql.user' + end + + def self.scope_to_current_user + if Database.postgresql? + where('grantee = user') + else + where("CONCAT(User, '@', Host) = current_user()") + end + end + + # Returns true if the current user can create and execute triggers on the + # given table. + def self.create_and_execute_trigger?(table) + priv = + if Database.postgresql? + where(privilege_type: 'TRIGGER', table_name: table) + else + where(Trigger_priv: 'Y') + end + + priv.scope_to_current_user.any? + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 5e2c6cc5cad..fb14798efe6 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -358,6 +358,8 @@ module Gitlab raise 'rename_column_concurrently can not be run inside a transaction' end + check_trigger_permissions!(table) + old_col = column_for(table, old) new_type = type || old_col.type @@ -430,6 +432,8 @@ module Gitlab def cleanup_concurrent_column_rename(table, old, new) trigger_name = rename_trigger_name(table, old, new) + check_trigger_permissions!(table) + if Database.postgresql? remove_rename_triggers_for_postgresql(table, trigger_name) else @@ -485,14 +489,14 @@ module Gitlab # Removes the triggers used for renaming a PostgreSQL column concurrently. def remove_rename_triggers_for_postgresql(table, trigger) - execute("DROP TRIGGER #{trigger} ON #{table}") - execute("DROP FUNCTION #{trigger}()") + execute("DROP TRIGGER IF EXISTS #{trigger} ON #{table}") + execute("DROP FUNCTION IF EXISTS #{trigger}()") end # Removes the triggers used for renaming a MySQL column concurrently. def remove_rename_triggers_for_mysql(trigger) - execute("DROP TRIGGER #{trigger}_insert") - execute("DROP TRIGGER #{trigger}_update") + execute("DROP TRIGGER IF EXISTS #{trigger}_insert") + execute("DROP TRIGGER IF EXISTS #{trigger}_update") end # Returns the (base) name to use for triggers when renaming columns. @@ -625,6 +629,30 @@ module Gitlab conn.llen("queue:#{queue_name}") end end + + def check_trigger_permissions!(table) + unless Grant.create_and_execute_trigger?(table) + dbname = Database.database_name + user = Database.username + + raise <<-EOF +Your database user is not allowed to create, drop, or execute triggers on the +table #{table}. + +If you are using PostgreSQL you can solve this by logging in to the GitLab +database (#{dbname}) using a super user and running: + + ALTER #{user} WITH SUPERUSER + +For MySQL you instead need to run: + + GRANT ALL PRIVILEGES ON *.* TO #{user}@'%' + +Both queries will grant the user super user permissions, ensuring you don't run +into similar problems in the future (e.g. when new tables are created). + EOF + end + end end end end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index b6449f27034..8c9acbc9fbe 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -5,6 +5,7 @@ module Gitlab BRANCH_REF_PREFIX = "refs/heads/".freeze CommandError = Class.new(StandardError) + CommitError = Class.new(StandardError) class << self include Gitlab::EncodingHelper diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb new file mode 100644 index 00000000000..9e6fca8c80c --- /dev/null +++ b/lib/gitlab/git/operation_service.rb @@ -0,0 +1,168 @@ +module Gitlab + module Git + class OperationService + attr_reader :committer, :repository + + def initialize(committer, new_repository) + committer = Gitlab::Git::Committer.from_user(committer) if committer.is_a?(User) + @committer = committer + + # Refactoring aid + unless new_repository.is_a?(Gitlab::Git::Repository) + raise "expected a Gitlab::Git::Repository, got #{new_repository}" + end + + @repository = new_repository + end + + def add_branch(branch_name, newrev) + ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name + oldrev = Gitlab::Git::BLANK_SHA + + update_ref_in_hooks(ref, newrev, oldrev) + end + + def rm_branch(branch) + ref = Gitlab::Git::BRANCH_REF_PREFIX + branch.name + oldrev = branch.target + newrev = Gitlab::Git::BLANK_SHA + + update_ref_in_hooks(ref, newrev, oldrev) + end + + def add_tag(tag_name, newrev, options = {}) + ref = Gitlab::Git::TAG_REF_PREFIX + tag_name + oldrev = Gitlab::Git::BLANK_SHA + + with_hooks(ref, newrev, oldrev) do |service| + # We want to pass the OID of the tag object to the hooks. For an + # annotated tag we don't know that OID until after the tag object + # (raw_tag) is created in the repository. That is why we have to + # update the value after creating the tag object. Only the + # "post-receive" hook will receive the correct value in this case. + raw_tag = repository.rugged.tags.create(tag_name, newrev, options) + service.newrev = raw_tag.target_id + end + end + + def rm_tag(tag) + ref = Gitlab::Git::TAG_REF_PREFIX + tag.name + oldrev = tag.target + newrev = Gitlab::Git::BLANK_SHA + + update_ref_in_hooks(ref, newrev, oldrev) do + repository.rugged.tags.delete(tag_name) + end + end + + # Whenever `start_branch_name` is passed, if `branch_name` doesn't exist, + # it would be created from `start_branch_name`. + # If `start_project` is passed, and the branch doesn't exist, + # it would try to find the commits from it instead of current repository. + def with_branch( + branch_name, + start_branch_name: nil, + start_repository: repository, + &block) + + # Refactoring aid + unless start_repository.is_a?(Gitlab::Git::Repository) + raise "expected a Gitlab::Git::Repository, got #{start_repository}" + end + + start_branch_name = nil if start_repository.empty_repo? + + if start_branch_name && !start_repository.branch_exists?(start_branch_name) + raise ArgumentError, "Cannot find branch #{start_branch_name} in #{start_repository.full_path}" + end + + update_branch_with_hooks(branch_name) do + repository.with_repo_branch_commit( + start_repository, + start_branch_name || branch_name, + &block) + end + end + + private + + # Returns [newrev, should_run_after_create, should_run_after_create_branch] + def update_branch_with_hooks(branch_name) + update_autocrlf_option + + was_empty = repository.empty? + + # Make commit + newrev = yield + + unless newrev + raise Gitlab::Git::CommitError.new('Failed to create commit') + end + + branch = repository.find_branch(branch_name) + oldrev = find_oldrev_from_branch(newrev, branch) + + ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name + update_ref_in_hooks(ref, newrev, oldrev) + + [newrev, was_empty, was_empty || Gitlab::Git.blank_ref?(oldrev)] + end + + def find_oldrev_from_branch(newrev, branch) + return Gitlab::Git::BLANK_SHA unless branch + + oldrev = branch.target + + if oldrev == repository.rugged.merge_base(newrev, branch.target) + oldrev + else + raise Gitlab::Git::CommitError.new('Branch diverged') + end + end + + def update_ref_in_hooks(ref, newrev, oldrev) + with_hooks(ref, newrev, oldrev) do + update_ref(ref, newrev, oldrev) + end + end + + def with_hooks(ref, newrev, oldrev) + Gitlab::Git::HooksService.new.execute( + committer, + repository, + oldrev, + newrev, + ref) do |service| + + yield(service) + end + end + + # Gitaly note: JV: wait with migrating #update_ref until we know how to migrate its call sites. + def update_ref(ref, newrev, oldrev) + # We use 'git update-ref' because libgit2/rugged currently does not + # offer 'compare and swap' ref updates. Without compare-and-swap we can + # (and have!) accidentally reset the ref to an earlier state, clobbering + # commits. See also https://github.com/libgit2/libgit2/issues/1534. + command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z] + _, status = Gitlab::Popen.popen( + command, + repository.path) do |stdin| + stdin.write("update #{ref}\x00#{newrev}\x00#{oldrev}\x00") + end + + unless status.zero? + raise Gitlab::Git::CommitError.new( + "Could not update branch #{Gitlab::Git.branch_name(ref)}." \ + " Please refresh and try again.") + end + end + + def update_autocrlf_option + if repository.autocrlf != :input + repository.autocrlf = :input + end + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index b835dec24eb..75d4efc0bc5 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -17,6 +17,7 @@ module Gitlab NoRepository = Class.new(StandardError) InvalidBlobName = Class.new(StandardError) InvalidRef = Class.new(StandardError) + GitError = Class.new(StandardError) class << self # Unlike `new`, `create` takes the storage path, not the storage name @@ -46,6 +47,9 @@ module Gitlab # Directory name of repo attr_reader :name + # Relative path of repo + attr_reader :relative_path + # Rugged repo object attr_reader :rugged @@ -69,6 +73,10 @@ module Gitlab delegate :exists?, to: :gitaly_repository_client + def ==(other) + path == other.path + end + # Default branch in the repository def root_ref @root_ref ||= gitaly_migrate(:root_ref) do |is_enabled| @@ -126,15 +134,19 @@ module Gitlab # This is to work around a bug in libgit2 that causes in-memory refs to # be stale/invalid when packed-refs is changed. # See https://gitlab.com/gitlab-org/gitlab-ce/issues/15392#note_14538333 - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/474 def find_branch(name, force_reload = false) - reload_rugged if force_reload + gitaly_migrate(:find_branch) do |is_enabled| + if is_enabled + gitaly_ref_client.find_branch(name) + else + reload_rugged if force_reload - rugged_ref = rugged.branches[name] - if rugged_ref - target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) + rugged_ref = rugged.branches[name] + if rugged_ref + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) + end + end end end @@ -246,6 +258,19 @@ module Gitlab branch_names + tag_names end + def delete_all_refs_except(prefixes) + delete_refs(*all_ref_names_except(prefixes)) + end + + # Returns an Array of all ref names, except when it's matching pattern + # + # regexp - The pattern for ref names we don't want + def all_ref_names_except(prefixes) + rugged.references.reject do |ref| + prefixes.any? { |p| ref.name.start_with?(p) } + end.map(&:name) + end + # Discovers the default branch based on the repository's available branches # # - If no branches are present, returns nil @@ -439,8 +464,8 @@ module Gitlab end # Returns true is +from+ is direct ancestor to +to+, otherwise false - def is_ancestor?(from, to) - gitaly_commit_client.is_ancestor(from, to) + def ancestor?(from, to) + gitaly_commit_client.ancestor?(from, to) end # Return an array of Diff objects that represent the diff @@ -584,6 +609,49 @@ module Gitlab # TODO: implement this method end + def add_branch(branch_name, committer:, target:) + target_object = Ref.dereference_object(lookup(target)) + raise InvalidRef.new("target not found: #{target}") unless target_object + + OperationService.new(committer, self).add_branch(branch_name, target_object.oid) + find_branch(branch_name) + rescue Rugged::ReferenceError => ex + raise InvalidRef, ex + end + + def add_tag(tag_name, committer:, target:, message: nil) + target_object = Ref.dereference_object(lookup(target)) + raise InvalidRef.new("target not found: #{target}") unless target_object + + committer = Committer.from_user(committer) if committer.is_a?(User) + + options = nil # Use nil, not the empty hash. Rugged cares about this. + if message + options = { + message: message, + tagger: Gitlab::Git.committer_hash(email: committer.email, name: committer.name) + } + end + + OperationService.new(committer, self).add_tag(tag_name, target_object.oid, options) + + find_tag(tag_name) + rescue Rugged::ReferenceError => ex + raise InvalidRef, ex + end + + def rm_branch(branch_name, committer:) + OperationService.new(committer, self).rm_branch(find_branch(branch_name)) + end + + def rm_tag(tag_name, committer:) + OperationService.new(committer, self).rm_tag(find_tag(tag_name)) + end + + def find_tag(name) + tags.find { |tag| tag.name == name } + end + # Delete the specified branch from the repository # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/476 @@ -591,6 +659,23 @@ module Gitlab rugged.branches.delete(branch_name) end + def delete_refs(*ref_names) + instructions = ref_names.map do |ref| + "delete #{ref}\x00\x00" + end + + command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z] + message, status = Gitlab::Popen.popen( + command, + path) do |stdin| + stdin.write(instructions.join) + end + + unless status.zero? + raise GitError.new("Could not delete refs #{ref_names}: #{message}") + end + end + # Create a new branch named **ref+ based on **stat_point+, HEAD by default # # Examples: @@ -706,6 +791,106 @@ module Gitlab end end + def with_repo_branch_commit(start_repository, start_branch_name) + raise "expected Gitlab::Git::Repository, got #{start_repository}" unless start_repository.is_a?(Gitlab::Git::Repository) + + return yield nil if start_repository.empty_repo? + + if start_repository == self + yield commit(start_branch_name) + else + sha = start_repository.commit(start_branch_name).sha + + if branch_commit = commit(sha) + yield branch_commit + else + with_repo_tmp_commit( + start_repository, start_branch_name, sha) do |tmp_commit| + yield tmp_commit + end + end + end + end + + def with_repo_tmp_commit(start_repository, start_branch_name, sha) + tmp_ref = fetch_ref( + start_repository.path, + "#{Gitlab::Git::BRANCH_REF_PREFIX}#{start_branch_name}", + "refs/tmp/#{SecureRandom.hex}/head" + ) + + yield commit(sha) + ensure + delete_refs(tmp_ref) if tmp_ref + end + + def fetch_source_branch(source_repository, source_branch, local_ref) + with_repo_branch_commit(source_repository, source_branch) do |commit| + if commit + write_ref(local_ref, commit.sha) + else + raise Rugged::ReferenceError, 'source repository is empty' + end + end + end + + def compare_source_branch(target_branch_name, source_repository, source_branch_name, straight:) + with_repo_branch_commit(source_repository, source_branch_name) do |commit| + break unless commit + + Gitlab::Git::Compare.new( + self, + target_branch_name, + commit.sha, + straight: straight + ) + end + end + + def write_ref(ref_path, sha) + rugged.references.create(ref_path, sha, force: true) + end + + def fetch_ref(source_path, source_ref, target_ref) + args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) + message, status = run_git(args) + + # Make sure ref was created, and raise Rugged::ReferenceError when not + raise Rugged::ReferenceError, message if status != 0 + + target_ref + end + + # Refactoring aid; allows us to copy code from app/models/repository.rb + def run_git(args) + circuit_breaker.perform do + popen([Gitlab.config.git.bin_path, *args], path) + end + end + + # Refactoring aid; allows us to copy code from app/models/repository.rb + def commit(ref = 'HEAD') + Gitlab::Git::Commit.find(self, ref) + end + + # Refactoring aid; allows us to copy code from app/models/repository.rb + def empty_repo? + !exists? || !has_visible_content? + end + + # + # Git repository can contains some hidden refs like: + # /refs/notes/* + # /refs/git-as-svn/* + # /refs/pulls/* + # This refs by default not visible in project page and not cloned to client side. + # + # This method return true if repository contains some content visible in project page. + # + def has_visible_content? + branch_count > 0 + end + def gitaly_repository Gitlab::GitalyClient::Util.repository(@storage, @relative_path) end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 3e8b83c0f90..62d1ecae676 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -35,6 +35,7 @@ module Gitlab def check(cmd, changes) check_protocol! + check_valid_actor! check_active_user! check_project_accessibility! check_project_moved! @@ -70,6 +71,14 @@ module Gitlab private + def check_valid_actor! + return unless actor.is_a?(Key) + + unless actor.valid? + raise UnauthorizedError, "Your SSH key #{actor.errors[:key].first}." + end + end + def check_protocol! unless protocol_allowed? raise UnauthorizedError, "Git access over #{protocol.upcase} is not allowed" diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 57f42bd35ee..21a32a7e0db 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -22,7 +22,7 @@ module Gitlab end end - def is_ancestor(ancestor_id, child_id) + def ancestor?(ancestor_id, child_id) request = Gitaly::CommitIsAncestorRequest.new( repository: @gitaly_repo, ancestor_id: ancestor_id, diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 8c0008c6971..a1a25cf2079 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -78,6 +78,20 @@ module Gitlab raise ArgumentError, e.message end + def find_branch(branch_name) + request = Gitaly::DeleteBranchRequest.new( + repository: @gitaly_repo, + name: GitalyClient.encode(branch_name) + ) + + response = GitalyClient.call(@repository.storage, :ref_service, :find_branch, request) + branch = response.branch + return unless branch + + target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) + Gitlab::Git::Branch.new(@repository, encode!(branch.name.dup), branch.target_commit.id, target_commit) + end + private def consume_refs_response(response) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index a74a6dc6e78..177a1284f38 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -37,6 +37,22 @@ module Gitlab request = Gitaly::ApplyGitattributesRequest.new(repository: @gitaly_repo, revision: revision) GitalyClient.call(@storage, :repository_service, :apply_gitattributes, request) end + + def fetch_remote(remote, ssh_auth: nil, forced: false, no_tags: false) + request = Gitaly::FetchRemoteRequest.new(repository: @gitaly_repo, remote: remote, force: forced, no_tags: no_tags) + + if ssh_auth&.ssh_import? + if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present? + request.ssh_key = ssh_auth.ssh_private_key + end + + if ssh_auth.ssh_known_hosts.present? + request.known_hosts = ssh_auth.ssh_known_hosts + end + end + + GitalyClient.call(@storage, :repository_service, :fetch_remote, request) + end end end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 2d1ae6a5925..9bcc579278f 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -3,6 +3,7 @@ module Gitlab module GonHelper include WebpackHelper + include Gitlab::CurrentSettings def add_gon_variables gon.api_version = 'v4' diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb index 45e9f9d65ae..025f826e65f 100644 --- a/lib/gitlab/gpg.rb +++ b/lib/gitlab/gpg.rb @@ -39,7 +39,7 @@ module Gitlab fingerprints = CurrentKeyChain.fingerprints_from_key(key) GPGME::Key.find(:public, fingerprints).flat_map do |raw_key| - raw_key.uids.map { |uid| { name: uid.name, email: uid.email } } + raw_key.uids.map { |uid| { name: uid.name, email: uid.email.downcase } } end end end diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 606c7576f70..86bd9f5b125 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -1,17 +1,12 @@ module Gitlab module Gpg class Commit - def self.for_commit(commit) - new(commit.project, commit.sha) - end - - def initialize(project, sha) - @project = project - @sha = sha + def initialize(commit) + @commit = commit @signature_text, @signed_text = begin - Rugged::Commit.extract_signature(project.repository.rugged, sha) + Rugged::Commit.extract_signature(@commit.project.repository.rugged, @commit.sha) rescue Rugged::OdbError nil end @@ -26,7 +21,7 @@ module Gitlab return @signature if @signature - cached_signature = GpgSignature.find_by(commit_sha: @sha) + cached_signature = GpgSignature.find_by(commit_sha: @commit.sha) return @signature = cached_signature if cached_signature.present? @signature = create_cached_signature! @@ -73,20 +68,31 @@ module Gitlab def attributes(gpg_key) user_infos = user_infos(gpg_key) + verification_status = verification_status(gpg_key) { - commit_sha: @sha, - project: @project, + commit_sha: @commit.sha, + project: @commit.project, gpg_key: gpg_key, gpg_key_primary_keyid: gpg_key&.primary_keyid || verified_signature.fingerprint, gpg_key_user_name: user_infos[:name], gpg_key_user_email: user_infos[:email], - valid_signature: gpg_signature_valid_signature_value(gpg_key) + verification_status: verification_status } end - def gpg_signature_valid_signature_value(gpg_key) - !!(gpg_key && gpg_key.verified? && verified_signature.valid?) + def verification_status(gpg_key) + return :unknown_key unless gpg_key + return :unverified_key unless gpg_key.verified? + return :unverified unless verified_signature.valid? + + if gpg_key.verified_and_belongs_to_email?(@commit.committer_email) + :verified + elsif gpg_key.user.all_emails.include?(@commit.committer_email) + :same_user_different_email + else + :other_user + end end def user_infos(gpg_key) diff --git a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb index a525ee7a9ee..e085eab26c9 100644 --- a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb +++ b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb @@ -8,7 +8,7 @@ module Gitlab def run GpgSignature .select(:id, :commit_sha, :project_id) - .where('gpg_key_id IS NULL OR valid_signature = ?', false) + .where('gpg_key_id IS NULL OR verification_status <> ?', GpgSignature.verification_statuses[:verified]) .where(gpg_key_primary_keyid: @gpg_key.primary_keyid) .find_each { |sig| sig.gpg_commit.update_signature!(sig) } end diff --git a/lib/gitlab/health_checks/db_check.rb b/lib/gitlab/health_checks/db_check.rb index fd94984f8a2..e27e16ddaf6 100644 --- a/lib/gitlab/health_checks/db_check.rb +++ b/lib/gitlab/health_checks/db_check.rb @@ -10,7 +10,7 @@ module Gitlab 'db_ping' end - def is_successful?(result) + def successful?(result) result == '1' end diff --git a/lib/gitlab/health_checks/redis/cache_check.rb b/lib/gitlab/health_checks/redis/cache_check.rb index a28658d42d4..0eb9b77634a 100644 --- a/lib/gitlab/health_checks/redis/cache_check.rb +++ b/lib/gitlab/health_checks/redis/cache_check.rb @@ -15,7 +15,7 @@ module Gitlab 'redis_cache_ping' end - def is_successful?(result) + def successful?(result) result == 'PONG' end diff --git a/lib/gitlab/health_checks/redis/queues_check.rb b/lib/gitlab/health_checks/redis/queues_check.rb index f97d50d3947..f322fe831b8 100644 --- a/lib/gitlab/health_checks/redis/queues_check.rb +++ b/lib/gitlab/health_checks/redis/queues_check.rb @@ -15,7 +15,7 @@ module Gitlab 'redis_queues_ping' end - def is_successful?(result) + def successful?(result) result == 'PONG' end diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb index fe4e3c4a3ab..8ceb0a0aa46 100644 --- a/lib/gitlab/health_checks/redis/redis_check.rb +++ b/lib/gitlab/health_checks/redis/redis_check.rb @@ -11,7 +11,7 @@ module Gitlab 'redis_ping' end - def is_successful?(result) + def successful?(result) result == 'PONG' end diff --git a/lib/gitlab/health_checks/redis/shared_state_check.rb b/lib/gitlab/health_checks/redis/shared_state_check.rb index e3244392902..07e6f707998 100644 --- a/lib/gitlab/health_checks/redis/shared_state_check.rb +++ b/lib/gitlab/health_checks/redis/shared_state_check.rb @@ -15,7 +15,7 @@ module Gitlab 'redis_shared_state_ping' end - def is_successful?(result) + def successful?(result) result == 'PONG' end diff --git a/lib/gitlab/health_checks/simple_abstract_check.rb b/lib/gitlab/health_checks/simple_abstract_check.rb index f5026171ba4..96945ce5b20 100644 --- a/lib/gitlab/health_checks/simple_abstract_check.rb +++ b/lib/gitlab/health_checks/simple_abstract_check.rb @@ -5,7 +5,7 @@ module Gitlab def readiness check_result = check - if is_successful?(check_result) + if successful?(check_result) HealthChecks::Result.new(true) elsif check_result.is_a?(Timeout::Error) HealthChecks::Result.new(false, "#{human_name} check timed out") @@ -16,10 +16,10 @@ module Gitlab def metrics result, elapsed = with_timing(&method(:check)) - Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless is_successful?(result) + Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless successful?(result) [ metric("#{metric_prefix}_timeout", result.is_a?(Timeout::Error) ? 1 : 0), - metric("#{metric_prefix}_success", is_successful?(result) ? 1 : 0), + metric("#{metric_prefix}_success", successful?(result) ? 1 : 0), metric("#{metric_prefix}_latency_seconds", elapsed) ] end @@ -30,7 +30,7 @@ module Gitlab raise NotImplementedError end - def is_successful?(result) + def successful?(result) raise NotImplementedError end diff --git a/lib/gitlab/i18n/metadata_entry.rb b/lib/gitlab/i18n/metadata_entry.rb new file mode 100644 index 00000000000..35d57459a3d --- /dev/null +++ b/lib/gitlab/i18n/metadata_entry.rb @@ -0,0 +1,27 @@ +module Gitlab + module I18n + class MetadataEntry + attr_reader :entry_data + + def initialize(entry_data) + @entry_data = entry_data + end + + def expected_plurals + return nil unless plural_information + + plural_information['nplurals'].to_i + end + + private + + def plural_information + return @plural_information if defined?(@plural_information) + + if plural_line = entry_data[:msgstr].detect { |metadata_line| metadata_line.starts_with?('Plural-Forms: ') } + @plural_information = Hash[plural_line.scan(/(\w+)=([^;\n]+)/)] + end + end + end + end +end diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb new file mode 100644 index 00000000000..7d3ff8c7f58 --- /dev/null +++ b/lib/gitlab/i18n/po_linter.rb @@ -0,0 +1,214 @@ +module Gitlab + module I18n + class PoLinter + attr_reader :po_path, :translation_entries, :metadata_entry, :locale + + VARIABLE_REGEX = /%{\w*}|%[a-z]/.freeze + + def initialize(po_path, locale = I18n.locale.to_s) + @po_path = po_path + @locale = locale + end + + def errors + @errors ||= validate_po + end + + def validate_po + if parse_error = parse_po + return 'PO-syntax errors' => [parse_error] + end + + validate_entries + end + + def parse_po + entries = SimplePoParser.parse(po_path) + + # The first entry is the metadata entry if there is one. + # This is an entry when empty `msgid` + if entries.first[:msgid].empty? + @metadata_entry = Gitlab::I18n::MetadataEntry.new(entries.shift) + else + return 'Missing metadata entry.' + end + + @translation_entries = entries.map do |entry_data| + Gitlab::I18n::TranslationEntry.new(entry_data, metadata_entry.expected_plurals) + end + + nil + rescue SimplePoParser::ParserError => e + @translation_entries = [] + e.message + end + + def validate_entries + errors = {} + + translation_entries.each do |entry| + errors_for_entry = validate_entry(entry) + errors[join_message(entry.msgid)] = errors_for_entry if errors_for_entry.any? + end + + errors + end + + def validate_entry(entry) + errors = [] + + validate_flags(errors, entry) + validate_variables(errors, entry) + validate_newlines(errors, entry) + validate_number_of_plurals(errors, entry) + validate_unescaped_chars(errors, entry) + + errors + end + + def validate_unescaped_chars(errors, entry) + if entry.msgid_contains_unescaped_chars? + errors << 'contains unescaped `%`, escape it using `%%`' + end + + if entry.plural_id_contains_unescaped_chars? + errors << 'plural id contains unescaped `%`, escape it using `%%`' + end + + if entry.translations_contain_unescaped_chars? + errors << 'translation contains unescaped `%`, escape it using `%%`' + end + end + + def validate_number_of_plurals(errors, entry) + return unless metadata_entry&.expected_plurals + return unless entry.translated? + + if entry.has_plural? && entry.all_translations.size != metadata_entry.expected_plurals + errors << "should have #{metadata_entry.expected_plurals} "\ + "#{'translations'.pluralize(metadata_entry.expected_plurals)}" + end + end + + def validate_newlines(errors, entry) + if entry.msgid_contains_newlines? + errors << 'is defined over multiple lines, this breaks some tooling.' + end + + if entry.plural_id_contains_newlines? + errors << 'plural is defined over multiple lines, this breaks some tooling.' + end + + if entry.translations_contain_newlines? + errors << 'has translations defined over multiple lines, this breaks some tooling.' + end + end + + def validate_variables(errors, entry) + if entry.has_singular_translation? + validate_variables_in_message(errors, entry.msgid, entry.singular_translation) + end + + if entry.has_plural? + entry.plural_translations.each do |translation| + validate_variables_in_message(errors, entry.plural_id, translation) + end + end + end + + def validate_variables_in_message(errors, message_id, message_translation) + message_id = join_message(message_id) + required_variables = message_id.scan(VARIABLE_REGEX) + + validate_unnamed_variables(errors, required_variables) + validate_translation(errors, message_id, required_variables) + validate_variable_usage(errors, message_translation, required_variables) + end + + def validate_translation(errors, message_id, used_variables) + variables = fill_in_variables(used_variables) + + begin + Gitlab::I18n.with_locale(locale) do + translated = if message_id.include?('|') + FastGettext::Translation.s_(message_id) + else + FastGettext::Translation._(message_id) + end + + translated % variables + end + + # `sprintf` could raise an `ArgumentError` when invalid passing something + # other than a Hash when using named variables + # + # `sprintf` could raise `TypeError` when passing a wrong type when using + # unnamed variables + # + # FastGettext::Translation could raise `RuntimeError` (raised as a string), + # or as subclassess `NoTextDomainConfigured` & `InvalidFormat` + # + # `FastGettext::Translation` could raise `ArgumentError` as subclassess + # `InvalidEncoding`, `IllegalSequence` & `InvalidCharacter` + rescue ArgumentError, TypeError, RuntimeError => e + errors << "Failure translating to #{locale} with #{variables}: #{e.message}" + end + end + + def fill_in_variables(variables) + if variables.empty? + [] + elsif variables.any? { |variable| unnamed_variable?(variable) } + variables.map do |variable| + variable == '%d' ? Random.rand(1000) : Gitlab::Utils.random_string + end + else + variables.inject({}) do |hash, variable| + variable_name = variable[/\w+/] + hash[variable_name] = Gitlab::Utils.random_string + hash + end + end + end + + def validate_unnamed_variables(errors, variables) + if variables.size > 1 && variables.any? { |variable_name| unnamed_variable?(variable_name) } + errors << 'is combining multiple unnamed variables' + end + end + + def validate_variable_usage(errors, translation, required_variables) + translation = join_message(translation) + + # We don't need to validate when the message is empty. + # In this case we fall back to the default, which has all the the + # required variables. + return if translation.empty? + + found_variables = translation.scan(VARIABLE_REGEX) + + missing_variables = required_variables - found_variables + if missing_variables.any? + errors << "<#{translation}> is missing: [#{missing_variables.to_sentence}]" + end + + unknown_variables = found_variables - required_variables + if unknown_variables.any? + errors << "<#{translation}> is using unknown variables: [#{unknown_variables.to_sentence}]" + end + end + + def unnamed_variable?(variable_name) + !variable_name.start_with?('%{') + end + + def validate_flags(errors, entry) + errors << "is marked #{entry.flag}" if entry.flag + end + + def join_message(message) + Array(message).join + end + end + end +end diff --git a/lib/gitlab/i18n/translation_entry.rb b/lib/gitlab/i18n/translation_entry.rb new file mode 100644 index 00000000000..e6c95afca7e --- /dev/null +++ b/lib/gitlab/i18n/translation_entry.rb @@ -0,0 +1,92 @@ +module Gitlab + module I18n + class TranslationEntry + PERCENT_REGEX = /(?:^|[^%])%(?!{\w*}|[a-z%])/.freeze + + attr_reader :nplurals, :entry_data + + def initialize(entry_data, nplurals) + @entry_data = entry_data + @nplurals = nplurals + end + + def msgid + entry_data[:msgid] + end + + def plural_id + entry_data[:msgid_plural] + end + + def has_plural? + plural_id.present? + end + + def singular_translation + all_translations.first if has_singular_translation? + end + + def all_translations + @all_translations ||= entry_data.fetch_values(*translation_keys) + .reject(&:empty?) + end + + def translated? + all_translations.any? + end + + def plural_translations + return [] unless has_plural? + return [] unless translated? + + @plural_translations ||= if has_singular_translation? + all_translations.drop(1) + else + all_translations + end + end + + def flag + entry_data[:flag] + end + + def has_singular_translation? + nplurals > 1 || !has_plural? + end + + def msgid_contains_newlines? + msgid.is_a?(Array) + end + + def plural_id_contains_newlines? + plural_id.is_a?(Array) + end + + def translations_contain_newlines? + all_translations.any? { |translation| translation.is_a?(Array) } + end + + def msgid_contains_unescaped_chars? + contains_unescaped_chars?(msgid) + end + + def plural_id_contains_unescaped_chars? + contains_unescaped_chars?(plural_id) + end + + def translations_contain_unescaped_chars? + all_translations.any? { |translation| contains_unescaped_chars?(translation) } + end + + def contains_unescaped_chars?(string) + string =~ PERCENT_REGEX + end + + private + + def translation_keys + @translation_keys ||= entry_data.keys.select { |key| key.to_s =~ /\Amsgstr(\[\d+\])?\z/ } + end + end + end +end diff --git a/lib/gitlab/issuables_count_for_state.rb b/lib/gitlab/issuables_count_for_state.rb new file mode 100644 index 00000000000..505810964bc --- /dev/null +++ b/lib/gitlab/issuables_count_for_state.rb @@ -0,0 +1,50 @@ +module Gitlab + # Class for counting and caching the number of issuables per state. + class IssuablesCountForState + # The name of the RequestStore cache key. + CACHE_KEY = :issuables_count_for_state + + # The state values that can be safely casted to a Symbol. + STATES = %w[opened closed merged all].freeze + + # finder - The finder class to use for retrieving the issuables. + def initialize(finder) + @finder = finder + @cache = + if RequestStore.active? + RequestStore[CACHE_KEY] ||= initialize_cache + else + initialize_cache + end + end + + def for_state_or_opened(state = nil) + self[state || :opened] + end + + # Returns the count for the given state. + # + # state - The name of the state as either a String or a Symbol. + # + # Returns an Integer. + def [](state) + state = state.to_sym if cast_state_to_symbol?(state) + + cache_for_finder[state] || 0 + end + + private + + def cache_for_finder + @cache[@finder] + end + + def cast_state_to_symbol?(state) + state.is_a?(String) && STATES.include?(state) + end + + def initialize_cache + Hash.new { |hash, finder| hash[finder] = finder.count_by_state } + end + end +end diff --git a/lib/gitlab/key_fingerprint.rb b/lib/gitlab/key_fingerprint.rb deleted file mode 100644 index d9a79f7c291..00000000000 --- a/lib/gitlab/key_fingerprint.rb +++ /dev/null @@ -1,48 +0,0 @@ -module Gitlab - class KeyFingerprint - attr_reader :key, :ssh_key - - # Unqualified MD5 fingerprint for compatibility - delegate :fingerprint, to: :ssh_key, allow_nil: true - - def initialize(key) - @key = key - - @ssh_key = - begin - Net::SSH::KeyFactory.load_data_public_key(key) - rescue Net::SSH::Exception, NotImplementedError - end - end - - def valid? - ssh_key.present? - end - - def type - return unless valid? - - parts = ssh_key.ssh_type.split('-') - parts.shift if parts[0] == 'ssh' - - parts[0].upcase - end - - def bits - return unless valid? - - case type - when 'RSA' - ssh_key.n.num_bits - when 'DSS', 'DSA' - ssh_key.p.num_bits - when 'ECDSA' - ssh_key.group.order.num_bits - when 'ED25519' - 256 - else - raise "Unsupported key type: #{type}" - end - end - end -end diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb index d7c56463aac..7b06bb953aa 100644 --- a/lib/gitlab/metrics/influx_db.rb +++ b/lib/gitlab/metrics/influx_db.rb @@ -1,7 +1,7 @@ module Gitlab module Metrics module InfluxDb - extend Gitlab::CurrentSettings + include Gitlab::CurrentSettings extend self MUTEX = Mutex.new diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index 56112ec2301..e73245b82c1 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -1,6 +1,6 @@ module Gitlab module PerformanceBar - include Gitlab::CurrentSettings + extend Gitlab::CurrentSettings ALLOWED_USER_IDS_KEY = 'performance_bar_allowed_user_ids:v2'.freeze EXPIRY_TIME = 5.minutes diff --git a/lib/gitlab/polling_interval.rb b/lib/gitlab/polling_interval.rb index f0c50584f07..4780675a492 100644 --- a/lib/gitlab/polling_interval.rb +++ b/lib/gitlab/polling_interval.rb @@ -1,6 +1,6 @@ module Gitlab class PollingInterval - include Gitlab::CurrentSettings + extend Gitlab::CurrentSettings HEADER_NAME = 'Poll-Interval'.freeze diff --git a/lib/gitlab/prometheus/queries/matched_metrics_query.rb b/lib/gitlab/prometheus/queries/matched_metrics_query.rb index d4894c87f8d..4c3edccc71a 100644 --- a/lib/gitlab/prometheus/queries/matched_metrics_query.rb +++ b/lib/gitlab/prometheus/queries/matched_metrics_query.rb @@ -42,13 +42,13 @@ module Gitlab lookup = series.each_slice(MAX_QUERY_ITEMS).flat_map do |batched_series| client_series(*batched_series, start: timeframe_start, stop: timeframe_end) - .select(&method(:has_matching_label)) + .select(&method(:has_matching_label?)) .map { |series_info| [series_info['__name__'], true] } end lookup.to_h end - def has_matching_label(series_info) + def has_matching_label?(series_info) series_info.key?('environment') end diff --git a/lib/gitlab/protocol_access.rb b/lib/gitlab/protocol_access.rb index 21aefc884be..09fa14764e6 100644 --- a/lib/gitlab/protocol_access.rb +++ b/lib/gitlab/protocol_access.rb @@ -1,5 +1,7 @@ module Gitlab module ProtocolAccess + extend Gitlab::CurrentSettings + def self.allowed?(protocol) if protocol == 'web' true diff --git a/lib/gitlab/recaptcha.rb b/lib/gitlab/recaptcha.rb index 4bc76ea033f..c463dd487a0 100644 --- a/lib/gitlab/recaptcha.rb +++ b/lib/gitlab/recaptcha.rb @@ -1,5 +1,7 @@ module Gitlab module Recaptcha + extend Gitlab::CurrentSettings + def self.load_configurations! if current_application_settings.recaptcha_enabled ::Recaptcha.configure do |config| diff --git a/lib/gitlab/reference_counter.rb b/lib/gitlab/reference_counter.rb new file mode 100644 index 00000000000..bb26f1b610a --- /dev/null +++ b/lib/gitlab/reference_counter.rb @@ -0,0 +1,44 @@ +module Gitlab + class ReferenceCounter + REFERENCE_EXPIRE_TIME = 600 + + attr_reader :gl_repository, :key + + def initialize(gl_repository) + @gl_repository = gl_repository + @key = "git-receive-pack-reference-counter:#{gl_repository}" + end + + def value + Gitlab::Redis::SharedState.with { |redis| (redis.get(key) || 0).to_i } + end + + def increase + redis_cmd do |redis| + redis.incr(key) + redis.expire(key, REFERENCE_EXPIRE_TIME) + end + end + + def decrease + redis_cmd do |redis| + current_value = redis.decr(key) + if current_value < 0 + Rails.logger.warn("Reference counter for #{gl_repository} decreased" \ + " when its value was less than 1. Reseting the counter.") + redis.del(key) + end + end + end + + private + + def redis_cmd + Gitlab::Redis::SharedState.with { |redis| yield(redis) } + true + rescue => e + Rails.logger.warn("GitLab: An unexpected error occurred in writing to Redis: #{e}") + false + end + end +end diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb index 2442c2ded3b..159d0e7952e 100644 --- a/lib/gitlab/sentry.rb +++ b/lib/gitlab/sentry.rb @@ -1,5 +1,7 @@ module Gitlab module Sentry + extend Gitlab::CurrentSettings + def self.enabled? Rails.env.production? && current_application_settings.sentry_enabled? end @@ -7,6 +9,8 @@ module Gitlab def self.context(current_user = nil) return unless self.enabled? + Raven.tags_context(locale: I18n.locale) + if current_user Raven.user_context( id: current_user.id, diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 280a9abf03e..81ecdf43ef9 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -98,33 +98,24 @@ module Gitlab # Fetch remote for repository # - # name - project path with namespace + # repository - an instance of Git::Repository # remote - remote name # forced - should we use --force flag? # no_tags - should we use --no-tags flag? # # Ex. - # fetch_remote("gitlab/gitlab-ci", "upstream") + # fetch_remote(my_repo, "upstream") # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 - def fetch_remote(storage, name, remote, ssh_auth: nil, forced: false, no_tags: false) - args = [gitlab_shell_projects_path, 'fetch-remote', storage, "#{name}.git", remote, "#{Gitlab.config.gitlab_shell.git_timeout}"] - args << '--force' if forced - args << '--no-tags' if no_tags - - vars = {} - - if ssh_auth&.ssh_import? - if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present? - vars['GITLAB_SHELL_SSH_KEY'] = ssh_auth.ssh_private_key - end - - if ssh_auth.ssh_known_hosts.present? - vars['GITLAB_SHELL_KNOWN_HOSTS'] = ssh_auth.ssh_known_hosts + def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false) + gitaly_migrate(:fetch_remote) do |is_enabled| + if is_enabled + repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags) + else + storage_path = Gitlab.config.repositories.storages[repository.storage]["path"] + local_fetch_remote(storage_path, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags) end end - - gitlab_shell_fast_execute_raise_error(args, vars) end # Move repository @@ -302,6 +293,26 @@ module Gitlab private + def local_fetch_remote(storage, name, remote, ssh_auth: nil, forced: false, no_tags: false) + args = [gitlab_shell_projects_path, 'fetch-remote', storage, name, remote, "#{Gitlab.config.gitlab_shell.git_timeout}"] + args << '--force' if forced + args << '--no-tags' if no_tags + + vars = {} + + if ssh_auth&.ssh_import? + if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present? + vars['GITLAB_SHELL_SSH_KEY'] = ssh_auth.ssh_private_key + end + + if ssh_auth.ssh_known_hosts.present? + vars['GITLAB_SHELL_KNOWN_HOSTS'] = ssh_auth.ssh_known_hosts + end + end + + gitlab_shell_fast_execute_raise_error(args, vars) + end + def gitlab_shell_fast_execute(cmd) output, status = gitlab_shell_fast_execute_helper(cmd) @@ -325,5 +336,13 @@ module Gitlab # from wasting I/O by searching through GEM_PATH Bundler.with_original_env { Popen.popen(cmd, nil, vars) } end + + def gitaly_migrate(method, &block) + Gitlab::GitalyClient.migrate(method, &block) + rescue GRPC::NotFound, GRPC::BadStatus => e + # Old Popen code returns [Error, output] to the caller, so we + # need to do the same here... + raise Error, e + end end end diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb new file mode 100644 index 00000000000..7c2d1d8f887 --- /dev/null +++ b/lib/gitlab/sql/pattern.rb @@ -0,0 +1,46 @@ +module Gitlab + module SQL + module Pattern + extend ActiveSupport::Concern + + MIN_CHARS_FOR_PARTIAL_MATCHING = 3 + REGEX_QUOTED_WORD = /(?<=^| )"[^"]+"(?= |$)/ + + class_methods do + def to_pattern(query) + if partial_matching?(query) + "%#{sanitize_sql_like(query)}%" + else + sanitize_sql_like(query) + end + end + + def partial_matching?(query) + query.length >= MIN_CHARS_FOR_PARTIAL_MATCHING + end + + def to_fuzzy_arel(column, query) + words = select_fuzzy_words(query) + + matches = words.map { |word| arel_table[column].matches(to_pattern(word)) } + + matches.reduce { |result, match| result.and(match) } + end + + def select_fuzzy_words(query) + quoted_words = query.scan(REGEX_QUOTED_WORD) + + query = quoted_words.reduce(query) { |q, quoted_word| q.sub(quoted_word, '') } + + words = query.split(/\s+/) + + quoted_words.map! { |quoted_word| quoted_word[1..-2] } + + words.concat(quoted_words) + + words.select { |word| partial_matching?(word) } + end + end + end + end +end diff --git a/lib/gitlab/ssh_public_key.rb b/lib/gitlab/ssh_public_key.rb new file mode 100644 index 00000000000..89ca1298120 --- /dev/null +++ b/lib/gitlab/ssh_public_key.rb @@ -0,0 +1,71 @@ +module Gitlab + class SSHPublicKey + Technology = Struct.new(:name, :key_class, :supported_sizes) + + Technologies = [ + Technology.new(:rsa, OpenSSL::PKey::RSA, [1024, 2048, 3072, 4096]), + Technology.new(:dsa, OpenSSL::PKey::DSA, [1024, 2048, 3072]), + Technology.new(:ecdsa, OpenSSL::PKey::EC, [256, 384, 521]), + Technology.new(:ed25519, Net::SSH::Authentication::ED25519::PubKey, [256]) + ].freeze + + def self.technology(name) + Technologies.find { |tech| tech.name.to_s == name.to_s } + end + + def self.technology_for_key(key) + Technologies.find { |tech| key.is_a?(tech.key_class) } + end + + def self.supported_sizes(name) + technology(name)&.supported_sizes + end + + attr_reader :key_text, :key + + # Unqualified MD5 fingerprint for compatibility + delegate :fingerprint, to: :key, allow_nil: true + + def initialize(key_text) + @key_text = key_text + + @key = + begin + Net::SSH::KeyFactory.load_data_public_key(key_text) + rescue StandardError, NotImplementedError + end + end + + def valid? + key.present? + end + + def type + technology.name if valid? + end + + def bits + return unless valid? + + case type + when :rsa + key.n.num_bits + when :dsa + key.p.num_bits + when :ecdsa + key.group.order.num_bits + when :ed25519 + 256 + else + raise "Unsupported key type: #{type}" + end + end + + private + + def technology + @technology ||= + self.class.technology_for_key(key) || raise("Unsupported key type: #{key.class}") + end + end +end diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb index 7ebec8e2cff..7393574ac13 100644 --- a/lib/gitlab/template/base_template.rb +++ b/lib/gitlab/template/base_template.rb @@ -18,6 +18,10 @@ module Gitlab { name: name, content: content } end + def <=>(other) + name <=> other.name + end + class << self def all(project = nil) if categories.any? @@ -58,7 +62,7 @@ module Gitlab directory = category_directory(category) files = finder(project).list_files_for(directory) - files.map { |f| new(f, project) } + files.map { |f| new(f, project) }.sort end def category_directory(category) diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 748e0a29184..3cf26625108 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -1,8 +1,8 @@ module Gitlab class UsageData - include Gitlab::CurrentSettings - class << self + include Gitlab::CurrentSettings + def data(force_refresh: false) Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data } end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 9670c93759e..abb3d3a02c3 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -42,5 +42,9 @@ module Gitlab 'No' end end + + def random_string + Random.rand(Float::MAX.to_i).to_s(36) + end end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index a362a3a0bc6..e5ad9b5a40c 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -35,10 +35,7 @@ module Gitlab when 'git_receive_pack' Gitlab::GitalyClient.feature_enabled?(:post_receive_pack) when 'git_upload_pack' - Gitlab::GitalyClient.feature_enabled?( - :post_upload_pack, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT - ) + true when 'info_refs' true else diff --git a/lib/system_check/app/git_config_check.rb b/lib/system_check/app/git_config_check.rb index 198867f7ac6..d08a81639e3 100644 --- a/lib/system_check/app/git_config_check.rb +++ b/lib/system_check/app/git_config_check.rb @@ -20,7 +20,7 @@ module SystemCheck # Returns true if all subcommands were successful (according to their exit code) # Returns false if any or all subcommands failed. def repair! - return false unless is_gitlab_user? + return false unless gitlab_user? command_success = OPTIONS.map do |name, value| system(*%W(#{Gitlab.config.git.bin_path} config --global #{name} #{value})) diff --git a/lib/system_check/app/git_user_default_ssh_config_check.rb b/lib/system_check/app/git_user_default_ssh_config_check.rb new file mode 100644 index 00000000000..7b486d78cf0 --- /dev/null +++ b/lib/system_check/app/git_user_default_ssh_config_check.rb @@ -0,0 +1,69 @@ +module SystemCheck + module App + class GitUserDefaultSSHConfigCheck < SystemCheck::BaseCheck + # These files are allowed in the .ssh directory. The `config` file is not + # whitelisted as it may change the SSH client's behaviour dramatically. + WHITELIST = %w[ + authorized_keys + authorized_keys2 + known_hosts + ].freeze + + set_name 'Git user has default SSH configuration?' + set_skip_reason 'skipped (git user is not present or configured)' + + def skip? + !home_dir || !File.directory?(home_dir) + end + + def check? + forbidden_files.empty? + end + + def show_error + backup_dir = "~/gitlab-check-backup-#{Time.now.to_i}" + + instructions = forbidden_files.map do |filename| + "sudo mv #{Shellwords.escape(filename)} #{backup_dir}" + end + + try_fixing_it("mkdir #{backup_dir}", *instructions) + for_more_information('doc/ssh/README.md in section "SSH on the GitLab server"') + fix_and_rerun + end + + private + + def git_user + Gitlab.config.gitlab.user + end + + def home_dir + return @home_dir if defined?(@home_dir) + + @home_dir = + begin + File.expand_path("~#{git_user}") + rescue ArgumentError + nil + end + end + + def ssh_dir + return nil unless home_dir + + File.join(home_dir, '.ssh') + end + + def forbidden_files + @forbidden_files ||= + begin + present = Dir[File.join(ssh_dir, '*')] + whitelisted = WHITELIST.map { |basename| File.join(ssh_dir, basename) } + + present - whitelisted + end + end + end + end +end diff --git a/lib/system_check/base_check.rb b/lib/system_check/base_check.rb index 5dcb3f0886b..7f9e2ffffc2 100644 --- a/lib/system_check/base_check.rb +++ b/lib/system_check/base_check.rb @@ -73,7 +73,7 @@ module SystemCheck self.class.instance_methods(false).include?(:skip?) end - def is_multi_check? + def multi_check? self.class.instance_methods(false).include?(:multi_check) end diff --git a/lib/system_check/simple_executor.rb b/lib/system_check/simple_executor.rb index e5986612908..6604b1078cf 100644 --- a/lib/system_check/simple_executor.rb +++ b/lib/system_check/simple_executor.rb @@ -53,7 +53,7 @@ module SystemCheck end # When implements a multi check, we don't control the output - if check.is_multi_check? + if check.multi_check? check.multi_check return end diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index b48e4dce445..35ba729c156 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -19,4 +19,46 @@ namespace :gettext do Rake::Task['gettext:pack'].invoke Rake::Task['gettext:po_to_json'].invoke end + + desc 'Lint all po files in `locale/' + task lint: :environment do + require 'simple_po_parser' + + FastGettext.silence_errors + files = Dir.glob(Rails.root.join('locale/*/gitlab.po')) + + linters = files.map do |file| + locale = File.basename(File.dirname(file)) + + Gitlab::I18n::PoLinter.new(file, locale) + end + + pot_file = Rails.root.join('locale/gitlab.pot') + linters.unshift(Gitlab::I18n::PoLinter.new(pot_file)) + + failed_linters = linters.select { |linter| linter.errors.any? } + + if failed_linters.empty? + puts 'All PO files are valid.' + else + failed_linters.each do |linter| + report_errors_for_file(linter.po_path, linter.errors) + end + + raise "Not all PO-files are valid: #{failed_linters.map(&:po_path).to_sentence}" + end + end + + def report_errors_for_file(file, errors_for_file) + puts "Errors in `#{file}`:" + + errors_for_file.each do |message_id, errors| + puts " #{message_id}" + errors.each do |error| + spaces = ' ' * 4 + error = error.lines.join("#{spaces}") + puts "#{spaces}#{error}" + end + end + end end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 1bd36bbe20a..92a3f503fcb 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -33,6 +33,7 @@ namespace :gitlab do SystemCheck::App::RedisVersionCheck, SystemCheck::App::RubyVersionCheck, SystemCheck::App::GitVersionCheck, + SystemCheck::App::GitUserDefaultSSHConfigCheck, SystemCheck::App::ActiveUsersCheck ] diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index f76bef5f4bf..8ae1b6a626a 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -111,7 +111,7 @@ namespace :gitlab do next unless id > max_iid project.deployments.find(id).create_ref - rugged.references.delete(ref) + project.repository.delete_refs(ref) end end end diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb index d85b810ac66..8a63f486fa3 100644 --- a/lib/tasks/gitlab/task_helpers.rb +++ b/lib/tasks/gitlab/task_helpers.rb @@ -104,7 +104,7 @@ module Gitlab Gitlab.config.gitlab.user end - def is_gitlab_user? + def gitlab_user? return @is_gitlab_user unless @is_gitlab_user.nil? current_user = run_command(%w(whoami)).chomp @@ -114,7 +114,7 @@ module Gitlab def warn_user_is_not_gitlab return if @warned_user_not_gitlab - unless is_gitlab_user? + unless gitlab_user? current_user = run_command(%w(whoami)).chomp puts " Warning ".color(:black).background(:yellow) diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 1206302cb76..4d485108cf6 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -80,7 +80,7 @@ class GithubImport end def visibility_level - @repo['private'] ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility + @repo['private'] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::CurrentSettings.current_application_settings.default_project_visibility end end |